kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Add raw API for custom syntax.

Browse Source
This commit is contained in:
Stephen Chung
2020-10-25 21:57:18 +08:00
parent f670d55871
commit b607a3a9ba
11 changed files with 298 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/api.rs
View File

@@ -164,14 +164,8 @@ impl Engine {
#[cfg(not(feature = "no_object"))]
#[inline(always)]
pub fn register_type_with_name<T: Variant + Clone>(&mut self, name: &str) -> &mut Self {
if self.type_names.is_none() {
self.type_names = Some(Default::default());
}
// Add the pretty-print type name into the map
self.type_names
.as_mut()
.unwrap()
.insert(type_name::<T>().into(), name.into());
self.type_names.insert(type_name::<T>().into(), name.into());
self
}

28
src/engine.rs
View File

@@ -512,14 +512,14 @@ pub struct Engine {
pub(crate) module_resolver: Option<Box<dyn ModuleResolver>>,
/// A hashmap mapping type names to pretty-print names.
pub(crate) type_names: Option<HashMap<String, String>>,
pub(crate) type_names: HashMap<String, String>,
/// A hashset containing symbols to disable.
pub(crate) disabled_symbols: Option<HashSet<String>>,
pub(crate) disabled_symbols: HashSet<String>,
/// A hashset containing custom keywords and precedence to recognize.
pub(crate) custom_keywords: Option<HashMap<String, u8>>,
pub(crate) custom_keywords: HashMap<String, Option<u8>>,
/// Custom syntax.
pub(crate) custom_syntax: Option<HashMap<String, CustomSyntax>>,
pub(crate) custom_syntax: HashMap<ImmutableString, CustomSyntax>,
/// Callback closure for resolving variable access.
pub(crate) resolve_var: Option<OnVarCallback>,
@@ -658,10 +658,10 @@ impl Engine {
#[cfg(any(feature = "no_std", target_arch = "wasm32",))]
module_resolver: None,
type_names: None,
disabled_symbols: None,
custom_keywords: None,
custom_syntax: None,
type_names: Default::default(),
disabled_symbols: Default::default(),
custom_keywords: Default::default(),
custom_syntax: Default::default(),
// variable resolver
resolve_var: None,
@@ -715,10 +715,10 @@ impl Engine {
#[cfg(not(feature = "no_module"))]
module_resolver: None,
type_names: None,
disabled_symbols: None,
custom_keywords: None,
custom_syntax: None,
type_names: Default::default(),
disabled_symbols: Default::default(),
custom_keywords: Default::default(),
custom_syntax: Default::default(),
resolve_var: None,
@@ -2274,8 +2274,8 @@ impl Engine {
#[inline(always)]
pub(crate) fn map_type_name<'a>(&'a self, name: &'a str) -> &'a str {
self.type_names
.as_ref()
.and_then(|t| t.get(name).map(String::as_str))
.get(name)
.map(String::as_str)
.unwrap_or_else(|| map_std_type_name(name))
}

2
src/fn_call.rs
View File

@@ -1055,7 +1055,7 @@ impl Engine {
} else {
// If the first argument is a variable, and there is no curried arguments, convert to method-call style
// in order to leverage potential &mut first argument and avoid cloning the value
if args_expr[0].get_variable_access(false).is_some() && curry.is_empty() {
if curry.is_empty() && args_expr[0].get_variable_access(false).is_some() {
// func(x, ...) -> x.func(...)
arg_values = args_expr
.iter()

87
src/parser.rs
View File

@@ -2549,9 +2549,17 @@ fn parse_binary_op(
let mut root = lhs;
loop {
let (current_op, _) = input.peek().unwrap();
let custom = state.engine.custom_keywords.as_ref();
let precedence = current_op.precedence(custom);
let (current_op, current_pos) = input.peek().unwrap();
let precedence = if let Token::Custom(c) = current_op {
// Custom operators
if let Some(Some(p)) = state.engine.custom_keywords.get(c) {
*p
} else {
return Err(PERR::Reserved(c.clone()).into_err(*current_pos));
}
} else {
current_op.precedence()
};
let bind_right = current_op.is_bind_right();
// Bind left to the parent lhs expression if precedence is higher
@@ -2574,7 +2582,17 @@ fn parse_binary_op(
let rhs = parse_unary(input, state, lib, settings)?;
let next_precedence = input.peek().unwrap().0.precedence(custom);
let (next_op, next_pos) = input.peek().unwrap();
let next_precedence = if let Token::Custom(c) = next_op {
// Custom operators
if let Some(Some(p)) = state.engine.custom_keywords.get(c) {
*p
} else {
return Err(PERR::Reserved(c.clone()).into_err(*next_pos));
}
} else {
next_op.precedence()
};
// Bind to right if the next operator has higher precedence
// If same precedence, then check if the operator binds right
@@ -2646,14 +2664,7 @@ fn parse_binary_op(
make_dot_expr(current_lhs, rhs, pos)?
}
Token::Custom(s)
if state
.engine
.custom_keywords
.as_ref()
.map(|c| c.contains_key(&s))
.unwrap_or(false) =>
{
Token::Custom(s) if state.engine.custom_keywords.contains_key(&s) => {
// Accept non-native functions for custom operators
let op = (op.0, false, op.2, op.3);
Expr::FnCall(Box::new((op, None, hash, args, None)))
@@ -2665,12 +2676,12 @@ fn parse_binary_op(
}
/// Parse a custom syntax.
fn parse_custom(
fn parse_custom_syntax(
input: &mut TokenStream,
state: &mut ParseState,
lib: &mut FunctionsLib,
mut settings: ParseSettings,
key: &str,
key: String,
syntax: &CustomSyntax,
pos: Position,
) -> Result<Expr, ParseError> {
@@ -2691,13 +2702,26 @@ fn parse_custom(
_ => (),
}
for segment in syntax.segments.iter() {
let mut segments: StaticVec<_> = Default::default();
segments.push(key);
loop {
settings.pos = input.peek().unwrap().1;
let token = if let Some(seg) = (syntax.parse)(&segments.iter().collect::<StaticVec<_>>())
.map_err(|err| err.0.into_err(settings.pos))?
{
seg
} else {
break;
};
let settings = settings.level_up();
match segment.as_str() {
match token.as_str() {
MARKER_IDENT => match input.next().unwrap() {
(Token::Identifier(s), pos) => {
segments.push(s.to_string());
exprs.push(Expr::Variable(Box::new(((s, pos), None, 0, None))));
}
(Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => {
@@ -2705,20 +2729,25 @@ fn parse_custom(
}
(_, pos) => return Err(PERR::VariableExpected.into_err(pos)),
},
MARKER_EXPR => exprs.push(parse_expr(input, state, lib, settings)?),
MARKER_EXPR => {
exprs.push(parse_expr(input, state, lib, settings)?);
segments.push(MARKER_EXPR.to_string());
}
MARKER_BLOCK => {
let stmt = parse_block(input, state, lib, settings)?;
let pos = stmt.position();
exprs.push(Expr::Stmt(Box::new((stmt, pos))))
exprs.push(Expr::Stmt(Box::new((stmt, pos))));
segments.push(MARKER_BLOCK.to_string());
}
s => match input.peek().unwrap() {
(Token::LexError(err), pos) => return Err(err.into_err(*pos)),
(t, _) if t.syntax().as_ref() == s => {
input.next().unwrap();
segments.push(input.next().unwrap().0.syntax().into_owned());
}
(_, pos) => {
return Err(PERR::MissingToken(
s.to_string(),
format!("for '{}' expression", key),
format!("for '{}' expression", segments[0]),
)
.into_err(*pos))
}
@@ -2745,16 +2774,22 @@ fn parse_expr(
settings.ensure_level_within_max_limit(state.max_expr_depth)?;
// Check if it is a custom syntax.
if let Some(ref custom) = state.engine.custom_syntax {
if !state.engine.custom_syntax.is_empty() {
let (token, pos) = input.peek().unwrap();
let token_pos = *pos;
match token {
Token::Custom(key) if custom.contains_key(key) => {
let custom = custom.get_key_value(key).unwrap();
let (key, syntax) = custom;
input.next().unwrap();
return parse_custom(input, state, lib, settings, key, syntax, token_pos);
Token::Custom(key) | Token::Reserved(key) | Token::Identifier(key) => {
match state.engine.custom_syntax.get_key_value(key) {
Some((key, syntax)) => {
let key = key.to_string();
input.next().unwrap();
return parse_custom_syntax(
input, state, lib, settings, key, syntax, token_pos,
);
}
_ => (),
}
}
_ => (),
}

28
src/settings.rs
View File

@@ -240,15 +240,7 @@ impl Engine {
/// ```
#[inline(always)]
pub fn disable_symbol(&mut self, symbol: &str) -> &mut Self {
if self.disabled_symbols.is_none() {
self.disabled_symbols = Some(Default::default());
}
self.disabled_symbols
.as_mut()
.unwrap()
.insert(symbol.into());
self.disabled_symbols.insert(symbol.into());
self
}
@@ -291,28 +283,14 @@ impl Engine {
// Standard identifiers, reserved keywords and custom keywords are OK
None | Some(Token::Reserved(_)) | Some(Token::Custom(_)) => (),
// Disabled keywords are also OK
Some(token)
if !self
.disabled_symbols
.as_ref()
.map(|d| d.contains(token.syntax().as_ref()))
.unwrap_or(false) =>
{
()
}
Some(token) if !self.disabled_symbols.contains(token.syntax().as_ref()) => (),
// Active standard keywords cannot be made custom
Some(_) => return Err(format!("'{}' is a reserved keyword", keyword).into()),
}
// Add to custom keywords
if self.custom_keywords.is_none() {
self.custom_keywords = Some(Default::default());
}
self.custom_keywords
.as_mut()
.unwrap()
.insert(keyword.into(), precedence);
.insert(keyword.into(), Some(precedence));
Ok(self)
}

140
src/syntax.rs
View File

@@ -7,13 +7,10 @@ use crate::fn_native::{SendSync, Shared};
use crate::parser::Expr;
use crate::result::EvalAltResult;
use crate::token::{is_valid_identifier, Position, Token};
use crate::utils::ImmutableString;
use crate::StaticVec;
use crate::stdlib::{
boxed::Box,
fmt, format,
string::{String, ToString},
};
use crate::stdlib::{boxed::Box, format, string::ToString};
/// A general expression evaluation trait object.
#[cfg(not(feature = "sync"))]
@@ -24,6 +21,14 @@ pub type FnCustomSyntaxEval =
pub type FnCustomSyntaxEval =
dyn Fn(&mut EvalContext, &[Expression]) -> Result<Dynamic, Box<EvalAltResult>> + Send + Sync;
/// A general expression parsing trait object.
#[cfg(not(feature = "sync"))]
pub type FnCustomSyntaxParse = dyn Fn(&[&String]) -> Result<Option<ImmutableString>, ParseError>;
/// A general expression parsing trait object.
#[cfg(feature = "sync")]
pub type FnCustomSyntaxParse =
dyn Fn(&[&String]) -> Result<Option<ImmutableString>, ParseError> + Send + Sync;
/// An expression sub-tree in an AST.
#[derive(Debug, Clone, Hash)]
pub struct Expression<'a>(&'a Expr);
@@ -76,20 +81,12 @@ impl EvalContext<'_, '_, '_, '_, '_, '_, '_, '_, '_> {
}
}
#[derive(Clone)]
pub struct CustomSyntax {
pub segments: StaticVec<String>,
pub parse: Box<FnCustomSyntaxParse>,
pub func: Shared<FnCustomSyntaxEval>,
pub scope_delta: isize,
}
impl fmt::Debug for CustomSyntax {
#[inline(always)]
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&self.segments, f)
}
}
impl Engine {
/// Register a custom syntax with the `Engine`.
///
@@ -106,7 +103,7 @@ impl Engine {
) -> Result<&mut Self, ParseError> {
let keywords = keywords.as_ref();
let mut segments: StaticVec<_> = Default::default();
let mut segments: StaticVec<ImmutableString> = Default::default();
for s in keywords {
let s = s.as_ref().trim();
@@ -119,47 +116,40 @@ impl Engine {
let seg = match s {
// Markers not in first position
MARKER_EXPR | MARKER_BLOCK | MARKER_IDENT if !segments.is_empty() => s.to_string(),
// Standard symbols not in first position
// Standard or reserved keyword/symbol not in first position
s if !segments.is_empty() && Token::lookup_from_syntax(s).is_some() => {
// Make it a custom keyword/operator if it is a disabled standard keyword/operator
// or a reserved keyword/operator.
if self
.disabled_symbols
.as_ref()
.map(|d| d.contains(s))
.unwrap_or(false)
|| Token::lookup_from_syntax(s)
.map(|token| token.is_reserved())
.unwrap_or(false)
{
// If symbol is disabled, make it a custom keyword
if self.custom_keywords.is_none() {
self.custom_keywords = Some(Default::default());
}
if !self.custom_keywords.as_ref().unwrap().contains_key(s) {
self.custom_keywords.as_mut().unwrap().insert(s.into(), 0);
}
// Make it a custom keyword/symbol
if !self.custom_keywords.contains_key(s) {
self.custom_keywords.insert(s.into(), None);
}
s.into()
}
// Identifier
s if is_valid_identifier(s.chars()) => {
if self.custom_keywords.is_none() {
self.custom_keywords = Some(Default::default());
// Standard keyword in first position
s if segments.is_empty()
&& Token::lookup_from_syntax(s)
.map(|v| v.is_keyword() || v.is_reserved())
.unwrap_or(false) =>
{
return Err(LexError::ImproperSymbol(format!(
"Improper symbol for custom syntax at position #{}: '{}'",
segments.len() + 1,
s
))
.into_err(Position::none())
.into());
}
// Identifier in first position
s if segments.is_empty() && is_valid_identifier(s.chars()) => {
if !self.custom_keywords.contains_key(s) {
self.custom_keywords.insert(s.into(), None);
}
if !self.custom_keywords.as_ref().unwrap().contains_key(s) {
self.custom_keywords.as_mut().unwrap().insert(s.into(), 0);
}
s.into()
}
// Anything else is an error
_ => {
return Err(LexError::ImproperSymbol(format!(
"Improper symbol for custom syntax: '{}'",
"Improper symbol for custom syntax at position #{}: '{}'",
segments.len() + 1,
s
))
.into_err(Position::none())
@@ -167,7 +157,7 @@ impl Engine {
}
};
segments.push(seg);
segments.push(seg.into());
}
// If the syntax has no keywords, just ignore the registration
@@ -175,24 +165,54 @@ impl Engine {
return Ok(self);
}
// Remove the first keyword as the discriminator
let key = segments.remove(0);
// The first keyword is the discriminator
let key = segments[0].clone();
self.register_custom_syntax_raw(
key,
// Construct the parsing function
move |stream| {
if stream.len() >= segments.len() {
Ok(None)
} else {
Ok(Some(segments[stream.len()].clone()))
}
},
new_vars,
func,
);
Ok(self)
}
/// Register a custom syntax with the `Engine`.
///
/// ## WARNING - Low Level API
///
/// This function is very low level.
///
/// * `new_vars` is the number of new variables declared by this custom syntax, or the number of variables removed (if negative).
/// * `parse` is the parsing function.
/// * `func` is the implementation function.
///
/// All custom keywords must be manually registered via `Engine::register_custom_operator`.
/// Otherwise, custom keywords won't be recognized.
pub fn register_custom_syntax_raw(
&mut self,
key: impl Into<ImmutableString>,
parse: impl Fn(&[&String]) -> Result<Option<ImmutableString>, ParseError> + SendSync + 'static,
new_vars: isize,
func: impl Fn(&mut EvalContext, &[Expression]) -> Result<Dynamic, Box<EvalAltResult>>
+ SendSync
+ 'static,
) -> &mut Self {
let syntax = CustomSyntax {
segments,
parse: Box::new(parse),
func: (Box::new(func) as Box<FnCustomSyntaxEval>).into(),
scope_delta: new_vars,
};
if self.custom_syntax.is_none() {
self.custom_syntax = Some(Default::default());
}
self.custom_syntax
.as_mut()
.unwrap()
.insert(key, syntax.into());
Ok(self)
self.custom_syntax.insert(key.into(), syntax);
self
}
}

43
src/token.rs
View File

@@ -18,9 +18,7 @@ use crate::parser::FLOAT;
use crate::stdlib::{
borrow::Cow,
boxed::Box,
char,
collections::HashMap,
fmt, format,
char, fmt, format,
iter::Peekable,
str::{Chars, FromStr},
string::{String, ToString},
@@ -610,7 +608,7 @@ impl Token {
}
/// Get the precedence number of the token.
pub fn precedence(&self, custom: Option<&HashMap<String, u8>>) -> u8 {
pub fn precedence(&self) -> u8 {
use Token::*;
match self {
@@ -639,9 +637,6 @@ impl Token {
Period => 240,
// Custom operators
Custom(s) => custom.map_or(0, |c| *c.get(s).unwrap()),
_ => 0,
}
}
@@ -692,7 +687,7 @@ impl Token {
Import | Export | As => true,
True | False | Let | Const | If | Else | While | Loop | For | In | Continue | Break
| Return | Throw => true,
| Return | Throw | Try | Catch => true,
_ => false,
}
@@ -1647,16 +1642,12 @@ impl<'a> Iterator for TokenIterator<'a, '_> {
type Item = (Token, Position);
fn next(&mut self) -> Option<Self::Item> {
let token = match (
get_next_token(&mut self.stream, &mut self.state, &mut self.pos),
self.engine.disabled_symbols.as_ref(),
self.engine.custom_keywords.as_ref(),
) {
let token = match get_next_token(&mut self.stream, &mut self.state, &mut self.pos) {
// {EOF}
(None, _, _) => None,
None => None,
// Reserved keyword/symbol
(Some((Token::Reserved(s), pos)), disabled, custom) => Some((match
(s.as_str(), custom.map(|c| c.contains_key(&s)).unwrap_or(false))
Some((Token::Reserved(s), pos)) => Some((match
(s.as_str(), self.engine.custom_keywords.contains_key(&s))
{
("===", false) => Token::LexError(Box::new(LERR::ImproperSymbol(
"'===' is not a valid operator. This is not JavaScript! Should it be '=='?".to_string(),
@@ -1691,21 +1682,19 @@ impl<'a> Iterator for TokenIterator<'a, '_> {
format!("'{}' is a reserved symbol", token)
))),
// Reserved keyword that is not custom and disabled.
(token, false) if disabled.map(|d| d.contains(token)).unwrap_or(false) => Token::LexError(Box::new(LERR::ImproperSymbol(
(token, false) if self.engine.disabled_symbols.contains(token) => Token::LexError(Box::new(LERR::ImproperSymbol(
format!("reserved symbol '{}' is disabled", token)
))),
// Reserved keyword/operator that is not custom.
(_, false) => Token::Reserved(s),
}, pos)),
// Custom keyword
(Some((Token::Identifier(s), pos)), _, Some(custom)) if custom.contains_key(&s) => {
Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.contains_key(&s) => {
Some((Token::Custom(s), pos))
}
// Custom standard keyword - must be disabled
(Some((token, pos)), Some(disabled), Some(custom))
if token.is_keyword() && custom.contains_key(token.syntax().as_ref()) =>
{
if disabled.contains(token.syntax().as_ref()) {
Some((token, pos)) if token.is_keyword() && self.engine.custom_keywords.contains_key(token.syntax().as_ref()) => {
if self.engine.disabled_symbols.contains(token.syntax().as_ref()) {
// Disabled standard keyword
Some((Token::Custom(token.syntax().into()), pos))
} else {
@@ -1714,21 +1703,17 @@ impl<'a> Iterator for TokenIterator<'a, '_> {
}
}
// Disabled operator
(Some((token, pos)), Some(disabled), _)
if token.is_operator() && disabled.contains(token.syntax().as_ref()) =>
{
Some((token, pos)) if token.is_operator() && self.engine.disabled_symbols.contains(token.syntax().as_ref()) => {
Some((
Token::LexError(Box::new(LexError::UnexpectedInput(token.syntax().into()))),
pos,
))
}
// Disabled standard keyword
(Some((token, pos)), Some(disabled), _)
if token.is_keyword() && disabled.contains(token.syntax().as_ref()) =>
{
Some((token, pos)) if token.is_keyword() && self.engine.disabled_symbols.contains(token.syntax().as_ref()) => {
Some((Token::Reserved(token.syntax().into()), pos))
}
(r, _, _) => r,
r => r,
};
match token {