diff --git a/CHANGELOG.md b/CHANGELOG.md index 2965b74e..a982c131 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Enhancements * `Engine::consume_XXX` methods are renamed to `Engine::run_XXX` to make meanings clearer. The `consume_XXX` API is deprecated. * `Engine::register_type_XXX` are now available even under `no_object`. +* Added `Engine::on_parse_token` to allow remapping certain tokens during parsing. ### Custom Syntax diff --git a/src/engine.rs b/src/engine.rs index 4f4ce8bb..51391745 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -5,7 +5,8 @@ use crate::custom_syntax::CustomSyntax; use crate::dynamic::{map_std_type_name, AccessMode, Union, Variant}; use crate::fn_hash::get_hasher; use crate::fn_native::{ - CallableFunction, IteratorFn, OnDebugCallback, OnPrintCallback, OnVarCallback, + CallableFunction, IteratorFn, OnDebugCallback, OnParseTokenCallback, OnPrintCallback, + OnVarCallback, }; use crate::module::NamespaceRef; use crate::optimize::OptimizationLevel; @@ -921,6 +922,8 @@ pub struct Engine { pub(crate) custom_syntax: BTreeMap>, /// Callback closure for resolving variable access. pub(crate) resolve_var: Option, + /// Callback closure to remap tokens during parsing. + pub(crate) token_mapper: Option>, /// Callback closure for implementing the `print` command. pub(crate) print: Option, @@ -1045,6 +1048,7 @@ impl Engine { custom_syntax: Default::default(), resolve_var: None, + token_mapper: None, print: None, debug: None, diff --git a/src/engine_api.rs b/src/engine_api.rs index c33f0356..ece66787 100644 --- a/src/engine_api.rs +++ b/src/engine_api.rs @@ -1181,7 +1181,8 @@ impl Engine { scripts: &[&str], optimization_level: OptimizationLevel, ) -> Result { - let (stream, tokenizer_control) = self.lex_raw(scripts, None); + let (stream, tokenizer_control) = + self.lex_raw(scripts, self.token_mapper.as_ref().map(Box::as_ref)); let mut state = ParseState::new(self, tokenizer_control); self.parse( &mut stream.peekable(), @@ -1368,13 +1369,13 @@ impl Engine { let (stream, tokenizer_control) = engine.lex_raw( &scripts, Some(if has_null { - |token| match token { + &|token| match token { // If `null` is present, make sure `null` is treated as a variable Token::Reserved(s) if s == "null" => Token::Identifier(s), _ => token, } } else { - |t| t + &|t| t }), ); let mut state = ParseState::new(engine, tokenizer_control); @@ -1462,7 +1463,8 @@ impl Engine { script: &str, ) -> Result { let scripts = [script]; - let (stream, tokenizer_control) = self.lex_raw(&scripts, None); + let (stream, tokenizer_control) = + self.lex_raw(&scripts, self.token_mapper.as_ref().map(Box::as_ref)); let mut peekable = stream.peekable(); let mut state = ParseState::new(self, tokenizer_control); @@ -1624,7 +1626,8 @@ impl Engine { script: &str, ) -> Result> { let scripts = [script]; - let (stream, tokenizer_control) = self.lex_raw(&scripts, None); + let (stream, tokenizer_control) = + self.lex_raw(&scripts, self.token_mapper.as_ref().map(Box::as_ref)); let mut state = ParseState::new(self, tokenizer_control); // No need to optimize a lone expression @@ -1769,7 +1772,8 @@ impl Engine { script: &str, ) -> Result<(), Box> { let scripts = [script]; - let (stream, tokenizer_control) = self.lex_raw(&scripts, None); + let (stream, tokenizer_control) = + self.lex_raw(&scripts, self.token_mapper.as_ref().map(Box::as_ref)); let mut state = ParseState::new(self, tokenizer_control); let ast = self.parse( @@ -2114,6 +2118,46 @@ impl Engine { self.resolve_var = Some(Box::new(callback)); self } + /// _(internals)_ Provide a callback that will be invoked during parsing to remap certain tokens. + /// Exported under the `internals` feature only. + /// + /// # Example + /// + /// ``` + /// # fn main() -> Result<(), Box> { + /// use rhai::{Engine, Token}; + /// + /// let mut engine = Engine::new(); + /// + /// // Register a token mapper. + /// engine.on_parse_token(|token| { + /// match token { + /// // Convert all integer literals to strings + /// Token::IntegerConstant(n) => Token::StringConstant(n.to_string()), + /// // Convert 'begin' .. 'end' to '{' .. '}' + /// Token::Identifier(s) if &s == "begin" => Token::LeftBrace, + /// Token::Identifier(s) if &s == "end" => Token::RightBrace, + /// // Pass through all other tokens unchanged + /// _ => token + /// } + /// }); + /// + /// assert_eq!(engine.eval::("42")?, "42"); + /// assert_eq!(engine.eval::("true")?, true); + /// assert_eq!(engine.eval::("let x = 42; begin let x = 0; end; x")?, "42"); + /// + /// # Ok(()) + /// # } + /// ``` + #[cfg(feature = "internals")] + #[inline(always)] + pub fn on_parse_token( + &mut self, + callback: impl Fn(crate::token::Token) -> crate::token::Token + SendSync + 'static, + ) -> &mut Self { + self.token_mapper = Some(Box::new(callback)); + self + } /// Register a callback for script evaluation progress. /// /// Not available under `unchecked`. diff --git a/src/fn_native.rs b/src/fn_native.rs index a5cc3bc5..60f46274 100644 --- a/src/fn_native.rs +++ b/src/fn_native.rs @@ -4,6 +4,7 @@ use crate::ast::{FnAccess, FnCallHashes}; use crate::engine::Imports; use crate::fn_call::FnCallArgs; use crate::plugin::PluginFunction; +use crate::token::Token; use crate::{ calc_fn_hash, Dynamic, Engine, EvalAltResult, EvalContext, Module, Position, RhaiResult, }; @@ -307,6 +308,13 @@ pub type OnDebugCallback = Box, Position) + 'static>; #[cfg(feature = "sync")] pub type OnDebugCallback = Box, Position) + Send + Sync + 'static>; +/// A standard callback function for mapping tokens during parsing. +#[cfg(not(feature = "sync"))] +pub type OnParseTokenCallback = dyn Fn(Token) -> Token; +/// A standard callback function for mapping tokens during parsing. +#[cfg(feature = "sync")] +pub type OnParseTokenCallback = dyn Fn(Token) -> Token + Send + Sync + 'static; + /// A standard callback function for variable access. #[cfg(not(feature = "sync"))] pub type OnVarCallback = diff --git a/src/token.rs b/src/token.rs index 00725d77..9baaf22a 100644 --- a/src/token.rs +++ b/src/token.rs @@ -4,6 +4,7 @@ use crate::engine::{ Precedence, KEYWORD_DEBUG, KEYWORD_EVAL, KEYWORD_FN_PTR, KEYWORD_FN_PTR_CALL, KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_THIS, KEYWORD_TYPE_OF, }; +use crate::fn_native::OnParseTokenCallback; use crate::{Engine, LexError, StaticVec, INT}; #[cfg(feature = "no_std")] use std::prelude::v1::*; @@ -2123,12 +2124,12 @@ pub struct TokenIterator<'a> { pub state: TokenizeState, /// Current position. pub pos: Position, - /// External buffer containing the next character to read, if any. + /// Shared object to allow controlling the tokenizer externally. pub tokenizer_control: TokenizerControl, /// Input character stream. pub stream: MultiInputsStream<'a>, /// A processor function that maps a token to another. - pub map: Option Token>, + pub token_mapper: Option<&'a OnParseTokenCallback>, } impl<'a> Iterator for TokenIterator<'a> { @@ -2222,7 +2223,7 @@ impl<'a> Iterator for TokenIterator<'a> { }; // Run the mapper, if any - let token = if let Some(map_func) = self.map { + let token = if let Some(map_func) = self.token_mapper { map_func(token) } else { token @@ -2254,9 +2255,9 @@ impl Engine { pub fn lex_with_map<'a>( &'a self, input: impl IntoIterator, - map: fn(Token) -> Token, + token_mapper: &'a OnParseTokenCallback, ) -> (TokenIterator<'a>, TokenizerControl) { - self.lex_raw(input, Some(map)) + self.lex_raw(input, Some(token_mapper)) } /// Tokenize an input text stream with an optional mapping function. #[inline] @@ -2264,7 +2265,7 @@ impl Engine { pub(crate) fn lex_raw<'a>( &'a self, input: impl IntoIterator, - map: Option Token>, + token_mapper: Option<&'a OnParseTokenCallback>, ) -> (TokenIterator<'a>, TokenizerControl) { let buffer: TokenizerControl = Default::default(); let buffer2 = buffer.clone(); @@ -2289,7 +2290,7 @@ impl Engine { streams: input.into_iter().map(|s| s.chars().peekable()).collect(), index: 0, }, - map, + token_mapper, }, buffer2, )