diff --git a/RELEASES.md b/RELEASES.md index ea15648f..5c968145 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -20,6 +20,8 @@ Enhancements * `Scope` is now `Clone + Hash`. * `Engine::register_static_module` now supports sub-module paths (e.g. `foo::bar::baz`). +* `Engine::register_custom_operator` now accepts reserved symbols. +* `Engine::register_custom_operator` now returns an error if given a precedence of zero. Version 0.19.8 diff --git a/src/engine_settings.rs b/src/engine_settings.rs index e7757636..1a291ccc 100644 --- a/src/engine_settings.rs +++ b/src/engine_settings.rs @@ -1,7 +1,7 @@ //! Configuration settings for [`Engine`]. use crate::stdlib::{format, num::NonZeroU8, string::String}; -use crate::token::{is_valid_identifier, Token}; +use crate::token::Token; use crate::Engine; #[cfg(not(feature = "no_module"))] @@ -252,17 +252,29 @@ impl Engine { if precedence.is_none() { return Err("precedence cannot be zero".into()); } - if !is_valid_identifier(keyword.chars()) { - return Err(format!("not a valid identifier: '{}'", keyword).into()); - } match Token::lookup_from_syntax(keyword) { // Standard identifiers, reserved keywords and custom keywords are OK None | Some(Token::Reserved(_)) | Some(Token::Custom(_)) => (), - // Disabled keywords are also OK - Some(token) if !self.disabled_symbols.contains(token.syntax().as_ref()) => (), // Active standard keywords cannot be made custom - Some(_) => return Err(format!("'{}' is a reserved keyword", keyword).into()), + // Disabled keywords are OK + Some(token) if token.is_keyword() => { + if !self.disabled_symbols.contains(token.syntax().as_ref()) { + return Err(format!("'{}' is a reserved keyword", keyword).into()); + } + } + // Active standard operators cannot be made custom + Some(token) if token.is_operator() => { + if !self.disabled_symbols.contains(token.syntax().as_ref()) { + return Err(format!("'{}' is a reserved operator", keyword).into()); + } + } + // Active standard symbols cannot be made custom + Some(token) if !self.disabled_symbols.contains(token.syntax().as_ref()) => { + return Err(format!("'{}' is a reserved symbol", keyword).into()) + } + // Disabled symbols are OK + Some(_) => (), } // Add to custom keywords diff --git a/src/parser.rs b/src/parser.rs index e3e521eb..9c3721f5 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1665,15 +1665,24 @@ fn parse_binary_op( loop { let (current_op, current_pos) = input.peek().unwrap(); - let precedence = if let Token::Custom(c) = current_op { - // Custom operators - if let Some(Some(p)) = state.engine.custom_keywords.get(c) { - *p - } else { - return Err(PERR::Reserved(c.clone()).into_err(*current_pos)); + let precedence = match current_op { + Token::Custom(c) => { + if state + .engine + .custom_keywords + .get(c) + .map(Option::is_some) + .unwrap_or(false) + { + state.engine.custom_keywords.get(c).unwrap().unwrap().get() + } else { + return Err(PERR::Reserved(c.clone()).into_err(*current_pos)); + } } - } else { - current_op.precedence() + Token::Reserved(c) if !is_valid_identifier(c.chars()) => { + return Err(PERR::UnknownOperator(c.into()).into_err(*current_pos)) + } + _ => current_op.precedence(), }; let bind_right = current_op.is_bind_right(); @@ -1698,15 +1707,24 @@ fn parse_binary_op( let rhs = parse_unary(input, state, lib, settings)?; let (next_op, next_pos) = input.peek().unwrap(); - let next_precedence = if let Token::Custom(c) = next_op { - // Custom operators - if let Some(Some(p)) = state.engine.custom_keywords.get(c) { - *p - } else { - return Err(PERR::Reserved(c.clone()).into_err(*next_pos)); + let next_precedence = match next_op { + Token::Custom(c) => { + if state + .engine + .custom_keywords + .get(c) + .map(Option::is_some) + .unwrap_or(false) + { + state.engine.custom_keywords.get(c).unwrap().unwrap().get() + } else { + return Err(PERR::Reserved(c.clone()).into_err(*next_pos)); + } } - } else { - next_op.precedence() + Token::Reserved(c) if !is_valid_identifier(c.chars()) => { + return Err(PERR::UnknownOperator(c.into()).into_err(*next_pos)) + } + _ => next_op.precedence(), }; // Bind to right if the next operator has higher precedence @@ -1809,11 +1827,24 @@ fn parse_binary_op( make_dot_expr(state, current_lhs, rhs, pos)? } - Token::Custom(s) if state.engine.custom_keywords.contains_key(&s) => { - // Accept non-native functions for custom operators + Token::Custom(s) + if state + .engine + .custom_keywords + .get(&s) + .map(Option::is_some) + .unwrap_or(false) => + { + let hash_script = if is_valid_identifier(s.chars()) { + // Accept non-native functions for custom operators + calc_script_fn_hash(empty(), &s, 2) + } else { + None + }; + Expr::FnCall( Box::new(FnCallExpr { - hash_script: calc_script_fn_hash(empty(), &s, 2), + hash_script, args, ..op_base }), diff --git a/src/syntax.rs b/src/syntax.rs index 55db8cce..8ce578fa 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -121,20 +121,26 @@ impl Engine { continue; } + let token = Token::lookup_from_syntax(s); + let seg = match s { // Markers not in first position MARKER_IDENT | MARKER_EXPR | MARKER_BLOCK if !segments.is_empty() => s.into(), // Standard or reserved keyword/symbol not in first position - s if !segments.is_empty() && Token::lookup_from_syntax(s).is_some() => { - // Make it a custom keyword/symbol - if !self.custom_keywords.contains_key(s) { + s if !segments.is_empty() && token.is_some() => { + // Make it a custom keyword/symbol if it is disabled or reserved + if (self.disabled_symbols.contains(s) + || matches!(token, Some(Token::Reserved(_)))) + && !self.custom_keywords.contains_key(s) + { self.custom_keywords.insert(s.into(), None); } s.into() } // Standard keyword in first position s if segments.is_empty() - && Token::lookup_from_syntax(s) + && token + .as_ref() .map(|v| v.is_keyword() || v.is_reserved()) .unwrap_or(false) => { @@ -151,7 +157,11 @@ impl Engine { } // Identifier in first position s if segments.is_empty() && is_valid_identifier(s.chars()) => { - if !self.custom_keywords.contains_key(s) { + // Make it a custom keyword/symbol if it is disabled or reserved + if (self.disabled_symbols.contains(s) + || matches!(token, Some(Token::Reserved(_)))) + && !self.custom_keywords.contains_key(s) + { self.custom_keywords.insert(s.into(), None); } s.into() diff --git a/src/token.rs b/src/token.rs index ee3febb0..1d3fbead 100644 --- a/src/token.rs +++ b/src/token.rs @@ -532,10 +532,10 @@ impl Token { #[cfg(feature = "no_module")] "import" | "export" | "as" => Reserved(syntax.into()), - "===" | "!==" | "->" | "<-" | ":=" | "::<" | "(*" | "*)" | "#" | "public" | "new" - | "use" | "module" | "package" | "var" | "static" | "begin" | "end" | "shared" - | "with" | "each" | "then" | "goto" | "unless" | "exit" | "match" | "case" - | "default" | "void" | "null" | "nil" | "spawn" | "thread" | "go" | "sync" + "===" | "!==" | "->" | "<-" | ":=" | "**" | "::<" | "(*" | "*)" | "#" | "public" + | "new" | "use" | "module" | "package" | "var" | "static" | "begin" | "end" + | "shared" | "with" | "each" | "then" | "goto" | "unless" | "exit" | "match" + | "case" | "default" | "void" | "null" | "nil" | "spawn" | "thread" | "go" | "sync" | "async" | "await" | "yield" => Reserved(syntax.into()), KEYWORD_PRINT | KEYWORD_DEBUG | KEYWORD_TYPE_OF | KEYWORD_EVAL | KEYWORD_FN_PTR @@ -1742,27 +1742,21 @@ impl<'a> Iterator for TokenIterator<'a, '_> { Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.contains_key(&s) => { Some((Token::Custom(s), pos)) } - // Custom standard keyword - must be disabled - Some((token, pos)) if token.is_keyword() && self.engine.custom_keywords.contains_key(token.syntax().as_ref()) => { + // Custom standard keyword/symbol - must be disabled + Some((token, pos)) if self.engine.custom_keywords.contains_key(token.syntax().as_ref()) => { if self.engine.disabled_symbols.contains(token.syntax().as_ref()) { - // Disabled standard keyword + // Disabled standard keyword/symbol Some((Token::Custom(token.syntax().into()), pos)) } else { // Active standard keyword - should never be a custom keyword! - unreachable!() + unreachable!("{:?}", token) } } - // Disabled operator - Some((token, pos)) if token.is_operator() && self.engine.disabled_symbols.contains(token.syntax().as_ref()) => { - Some(( - Token::LexError(LexError::UnexpectedInput(token.syntax().into())), - pos, - )) - } - // Disabled standard keyword - Some((token, pos)) if token.is_keyword() && self.engine.disabled_symbols.contains(token.syntax().as_ref()) => { + // Disabled symbol + Some((token, pos)) if self.engine.disabled_symbols.contains(token.syntax().as_ref()) => { Some((Token::Reserved(token.syntax().into()), pos)) } + // Normal symbol r => r, }; diff --git a/tests/tokens.rs b/tests/tokens.rs index 0070c23d..71158313 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -21,12 +21,17 @@ fn test_tokens_disabled() { .compile("let x = 40 + 2; x += 1;") .expect_err("should error") .0, - ParseErrorType::BadInput(LexError::UnexpectedInput("+=".to_string())) + ParseErrorType::UnknownOperator("+=".to_string()) ); + + assert!(matches!( + *engine.compile("let x = += 0;").expect_err("should error").0, + ParseErrorType::BadInput(LexError::UnexpectedInput(err)) if err == "+=" + )); } #[test] -fn test_tokens_custom_operator() -> Result<(), Box> { +fn test_tokens_custom_operator_identifiers() -> Result<(), Box> { let mut engine = Engine::new(); // Register a custom operator called `foo` and give it @@ -55,6 +60,29 @@ fn test_tokens_custom_operator() -> Result<(), Box> { Ok(()) } +#[test] +fn test_tokens_custom_operator_symbol() -> Result<(), Box> { + let mut engine = Engine::new(); + + // Register a custom operator `#` and give it + // a precedence of 160 (i.e. between +|- and *|/). + engine.register_custom_operator("#", 160).unwrap(); + + // Register a binary function named `#` + engine.register_fn("#", |x: INT, y: INT| (x * y) - (x + y)); + + assert_eq!(engine.eval_expression::("1 + 2 * 3 # 4 - 5 / 6")?, 15); + + // Register a custom operator named `=>` + assert!(engine.register_custom_operator("=>", 160).is_err()); + engine.disable_symbol("=>"); + engine.register_custom_operator("=>", 160).unwrap(); + engine.register_fn("=>", |x: INT, y: INT| (x * y) - (x + y)); + assert_eq!(engine.eval_expression::("1 + 2 * 3 => 4 - 5 / 6")?, 15); + + Ok(()) +} + #[test] fn test_tokens_unicode_xid_ident() -> Result<(), Box> { let engine = Engine::new();