kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Merge branch 'master' into plugins

Browse Source
This commit is contained in:
Stephen Chung
2020-06-14 00:14:52 +08:00
parent 96e5a1c56d 22d30c95c9
commit 27ad7bbebe
13 changed files with 626 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/api.rs
View File

@@ -547,7 +547,7 @@ impl Engine {
scripts: &[&str],
optimization_level: OptimizationLevel,
) -> Result<AST, ParseError> {
let stream = lex(scripts);
let stream = lex(scripts, self.max_string_size);
self.parse(&mut stream.peekable(), scope, optimization_level)
}
@@ -669,7 +669,7 @@ impl Engine {
// Trims the JSON string and add a '#' in front
let scripts = ["#", json.trim()];
let stream = lex(&scripts);
let stream = lex(&scripts, self.max_string_size);
let ast =
self.parse_global_expr(&mut stream.peekable(), &scope, OptimizationLevel::None)?;
@@ -750,7 +750,7 @@ impl Engine {
script: &str,
) -> Result<AST, ParseError> {
let scripts = [script];
let stream = lex(&scripts);
let stream = lex(&scripts, self.max_string_size);
{
let mut peekable = stream.peekable();
@@ -904,7 +904,7 @@ impl Engine {
script: &str,
) -> Result<T, Box<EvalAltResult>> {
let scripts = [script];
let stream = lex(&scripts);
let stream = lex(&scripts, self.max_string_size);
let ast = self.parse_global_expr(
&mut stream.peekable(),
@@ -1034,7 +1034,7 @@ impl Engine {
script: &str,
) -> Result<(), Box<EvalAltResult>> {
let scripts = [script];
let stream = lex(&scripts);
let stream = lex(&scripts, self.max_string_size);
let ast = self.parse(&mut stream.peekable(), scope, self.optimization_level)?;
self.consume_ast_with_scope(scope, &ast)
@@ -1114,7 +1114,7 @@ impl Engine {
args: A,
) -> Result<T, Box<EvalAltResult>> {
let mut arg_values = args.into_vec();
let result = self.call_fn_dynamic(scope, ast, name, arg_values.as_mut())?;
let result = self.call_fn_dynamic_raw(scope, ast, name, arg_values.as_mut())?;
let return_type = self.map_type_name(result.type_name());
@@ -1128,13 +1128,6 @@ impl Engine {
/// Call a script function defined in an `AST` with multiple `Dynamic` arguments.
///
/// ## WARNING
///
/// All the arguments are _consumed_, meaning that they're replaced by `()`.
/// This is to avoid unnecessarily cloning the arguments.
/// Do you use the arguments after this call. If you need them afterwards,
/// clone them _before_ calling this function.
///
/// # Example
///
/// ```
@@ -1155,13 +1148,13 @@ impl Engine {
/// scope.push("foo", 42_i64);
///
/// // Call the script-defined function
/// let result = engine.call_fn_dynamic(&mut scope, &ast, "add", &mut [ String::from("abc").into(), 123_i64.into() ])?;
/// let result = engine.call_fn_dynamic(&mut scope, &ast, "add", vec![ String::from("abc").into(), 123_i64.into() ])?;
/// assert_eq!(result.cast::<i64>(), 168);
///
/// let result = engine.call_fn_dynamic(&mut scope, &ast, "add1", &mut [ String::from("abc").into() ])?;
/// let result = engine.call_fn_dynamic(&mut scope, &ast, "add1", vec![ String::from("abc").into() ])?;
/// assert_eq!(result.cast::<i64>(), 46);
///
/// let result= engine.call_fn_dynamic(&mut scope, &ast, "bar", &mut [])?;
/// let result= engine.call_fn_dynamic(&mut scope, &ast, "bar", vec![])?;
/// assert_eq!(result.cast::<i64>(), 21);
/// # }
/// # Ok(())
@@ -1169,6 +1162,25 @@ impl Engine {
/// ```
#[cfg(not(feature = "no_function"))]
pub fn call_fn_dynamic(
&self,
scope: &mut Scope,
ast: &AST,
name: &str,
arg_values: impl IntoIterator<Item = Dynamic>,
) -> Result<Dynamic, Box<EvalAltResult>> {
let mut arg_values: StaticVec<_> = arg_values.into_iter().collect();
self.call_fn_dynamic_raw(scope, ast, name, arg_values.as_mut())
}
/// Call a script function defined in an `AST` with multiple `Dynamic` arguments.
///
/// ## WARNING
///
/// All the arguments are _consumed_, meaning that they're replaced by `()`.
/// This is to avoid unnecessarily cloning the arguments.
/// Do not use the arguments after this call. If they are needed afterwards,
/// clone them _before_ calling this function.
pub(crate) fn call_fn_dynamic_raw(
&self,
scope: &mut Scope,
ast: &AST,

97
src/engine.rs
View File

@@ -67,6 +67,13 @@ pub const MAX_EXPR_DEPTH: usize = usize::MAX;
#[cfg(feature = "unchecked")]
pub const MAX_FUNCTION_EXPR_DEPTH: usize = usize::MAX;
#[cfg(feature = "unchecked")]
pub const MAX_STRING_SIZE: usize = usize::MAX;
#[cfg(feature = "unchecked")]
pub const MAX_ARRAY_SIZE: usize = usize::MAX;
#[cfg(feature = "unchecked")]
pub const MAX_MAP_SIZE: usize = usize::MAX;
pub const KEYWORD_PRINT: &str = "print";
pub const KEYWORD_DEBUG: &str = "debug";
pub const KEYWORD_TYPE_OF: &str = "type_of";
@@ -262,6 +269,12 @@ pub struct Engine {
pub(crate) max_operations: u64,
/// Maximum number of modules allowed to load.
pub(crate) max_modules: u64,
/// Maximum length of a string.
pub(crate) max_string_size: usize,
/// Maximum length of an array.
pub(crate) max_array_size: usize,
/// Maximum number of properties in a map.
pub(crate) max_map_size: usize,
}
impl Default for Engine {
@@ -298,6 +311,9 @@ impl Default for Engine {
max_function_expr_depth: MAX_FUNCTION_EXPR_DEPTH,
max_operations: u64::MAX,
max_modules: u64::MAX,
max_string_size: usize::MAX,
max_array_size: usize::MAX,
max_map_size: usize::MAX,
};
engine.load_package(StandardPackage::new().get());
@@ -442,6 +458,9 @@ impl Engine {
max_function_expr_depth: MAX_FUNCTION_EXPR_DEPTH,
max_operations: u64::MAX,
max_modules: u64::MAX,
max_string_size: usize::MAX,
max_array_size: usize::MAX,
max_map_size: usize::MAX,
}
}
@@ -495,13 +514,33 @@ impl Engine {
self.max_modules = if modules == 0 { u64::MAX } else { modules };
}
/// Set the depth limits for expressions/statements.
/// Set the depth limits for expressions/statements (0 for unlimited).
#[cfg(not(feature = "unchecked"))]
pub fn set_max_expr_depths(&mut self, max_expr_depth: usize, max_function_expr_depth: usize) {
self.max_expr_depth = max_expr_depth;
self.max_function_expr_depth = max_function_expr_depth;
}
/// Set the maximum length of strings (0 for unlimited).
#[cfg(not(feature = "unchecked"))]
pub fn set_max_string_size(&mut self, max_size: usize) {
self.max_string_size = max_size;
}
/// Set the maximum length of arrays (0 for unlimited).
#[cfg(not(feature = "unchecked"))]
#[cfg(not(feature = "no_index"))]
pub fn set_max_array_size(&mut self, max_size: usize) {
self.max_array_size = max_size;
}
/// Set the maximum length of object maps (0 for unlimited).
#[cfg(not(feature = "unchecked"))]
#[cfg(not(feature = "no_object"))]
pub fn set_max_map_size(&mut self, max_size: usize) {
self.max_map_size = max_size;
}
/// Set the module resolution service used by the `Engine`.
///
/// Not available under the `no_module` feature.
@@ -1395,7 +1434,7 @@ impl Engine {
self.inc_operations(state)
.map_err(|err| EvalAltResult::new_position(err, expr.position()))?;
match expr {
let result = match expr {
Expr::Expr(x) => self.eval_expr(scope, state, lib, x.as_ref(), level),
Expr::IntegerConstant(x) => Ok(x.0.into()),
@@ -1732,7 +1771,13 @@ impl Engine {
Expr::Unit(_) => Ok(().into()),
_ => unreachable!(),
};
if let Ok(val) = &result {
self.check_data_size(val)?;
}
result
}
/// Evaluate a statement
@@ -1747,7 +1792,7 @@ impl Engine {
self.inc_operations(state)
.map_err(|err| EvalAltResult::new_position(err, stmt.position()))?;
match stmt {
let result = match stmt {
// No-op
Stmt::Noop(_) => Ok(Default::default()),
@@ -1999,6 +2044,52 @@ impl Engine {
}
Ok(Default::default())
}
};
if let Ok(val) = &result {
self.check_data_size(val)?;
}
result
}
/// Check a `Dynamic` value to ensure that its size is within allowable limit.
fn check_data_size(&self, value: &Dynamic) -> Result<(), Box<EvalAltResult>> {
#[cfg(feature = "unchecked")]
return Ok(());
match value {
Dynamic(Union::Str(s))
if self.max_string_size > 0 && s.len() > self.max_string_size =>
{
Err(Box::new(EvalAltResult::ErrorDataTooLarge(
"Length of string".to_string(),
self.max_string_size,
s.len(),
Position::none(),
)))
}
#[cfg(not(feature = "no_index"))]
Dynamic(Union::Array(arr))
if self.max_array_size > 0 && arr.len() > self.max_array_size =>
{
Err(Box::new(EvalAltResult::ErrorDataTooLarge(
"Length of array".to_string(),
self.max_array_size,
arr.len(),
Position::none(),
)))
}
#[cfg(not(feature = "no_object"))]
Dynamic(Union::Map(map)) if self.max_map_size > 0 && map.len() > self.max_map_size => {
Err(Box::new(EvalAltResult::ErrorDataTooLarge(
"Number of properties in object map".to_string(),
self.max_map_size,
map.len(),
Position::none(),
)))
}
_ => Ok(()),
}
}

19
src/error.rs
View File

@@ -12,6 +12,8 @@ pub enum LexError {
UnexpectedChar(char),
/// A string literal is not terminated before a new-line or EOF.
UnterminatedString,
/// An identifier is in an invalid format.
StringTooLong(usize),
/// An string/character/numeric escape sequence is in an invalid format.
MalformedEscapeSequence(String),
/// An numeric literal is in an invalid format.
@@ -35,6 +37,11 @@ impl fmt::Display for LexError {
Self::MalformedChar(s) => write!(f, "Invalid character: '{}'", s),
Self::MalformedIdentifier(s) => write!(f, "Variable name is not proper: '{}'", s),
Self::UnterminatedString => write!(f, "Open string is not terminated"),
Self::StringTooLong(max) => write!(
f,
"Length of string literal exceeds the maximum limit ({})",
max
),
Self::ImproperKeyword(s) => write!(f, "{}", s),
}
}
@@ -109,12 +116,16 @@ pub enum ParseErrorType {
WrongExport,
/// Assignment to a copy of a value.
AssignmentToCopy,
/// Assignment to an a constant variable.
/// Assignment to an a constant variable. Wrapped value is the constant variable name.
AssignmentToConstant(String),
/// Expression exceeding the maximum levels of complexity.
///
/// Never appears under the `unchecked` feature.
ExprTooDeep,
/// Literal exceeding the maximum size. Wrapped values are the data type name and the maximum size.
///
/// Never appears under the `unchecked` feature.
LiteralTooLarge(String, usize),
/// Break statement not inside a loop.
LoopBreak,
}
@@ -149,13 +160,14 @@ impl ParseErrorType {
Self::AssignmentToCopy => "Only a copy of the value is change with this assignment",
Self::AssignmentToConstant(_) => "Cannot assign to a constant value",
Self::ExprTooDeep => "Expression exceeds maximum complexity",
Self::LiteralTooLarge(_, _) => "Literal exceeds maximum limit",
Self::LoopBreak => "Break statement should only be used inside a loop"
}
}
}
impl fmt::Display for ParseErrorType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::BadInput(s) | ParseErrorType::MalformedCallExpr(s) => {
write!(f, "{}", if s.is_empty() { self.desc() } else { s })
@@ -197,6 +209,9 @@ impl fmt::Display for ParseErrorType {
Self::AssignmentToConstant(s) if s.is_empty() => write!(f, "{}", self.desc()),
Self::AssignmentToConstant(s) => write!(f, "Cannot assign to constant '{}'", s),
Self::LiteralTooLarge(typ, max) => {
write!(f, "{} exceeds the maximum limit ({})", typ, max)
}
_ => write!(f, "{}", self.desc()),
}
}

244
src/parser.rs
View File

@@ -201,13 +201,27 @@ struct ParseState {
stack: Vec<(String, ScopeEntryType)>,
/// Maximum levels of expression nesting.
max_expr_depth: usize,
/// Maximum length of a string.
pub max_string_size: usize,
/// Maximum length of an array.
pub max_array_size: usize,
/// Maximum number of properties in a map.
pub max_map_size: usize,
}
impl ParseState {
/// Create a new `ParseState`.
pub fn new(max_expr_depth: usize) -> Self {
pub fn new(
max_expr_depth: usize,
max_string_size: usize,
max_array_size: usize,
max_map_size: usize,
) -> Self {
Self {
max_expr_depth,
max_string_size,
max_array_size,
max_map_size,
..Default::default()
}
}
@@ -1070,6 +1084,14 @@ fn parse_array_literal(
if !match_token(input, Token::RightBracket)? {
while !input.peek().unwrap().0.is_eof() {
if state.max_array_size > 0 && arr.len() >= state.max_array_size {
return Err(PERR::LiteralTooLarge(
"Size of array literal".to_string(),
state.max_array_size,
)
.into_err(input.peek().unwrap().1));
}
let expr = parse_expr(input, state, settings.level_up())?;
arr.push(expr);
@@ -1155,8 +1177,15 @@ fn parse_map_literal(
}
};
let expr = parse_expr(input, state, settings.level_up())?;
if state.max_map_size > 0 && map.len() >= state.max_map_size {
return Err(PERR::LiteralTooLarge(
"Number of properties in object map literal".to_string(),
state.max_map_size,
)
.into_err(input.peek().unwrap().1));
}
let expr = parse_expr(input, state, settings.level_up())?;
map.push(((name, pos), expr));
match input.peek().unwrap() {
@@ -2408,102 +2437,6 @@ fn parse_fn(
})
}
/// Parse the global level statements.
fn parse_global_level(
input: &mut TokenStream,
max_expr_depth: usize,
max_function_expr_depth: usize,
) -> Result<(Vec<Stmt>, Vec<ScriptFnDef>), ParseError> {
let mut statements = Vec::<Stmt>::new();
let mut functions = HashMap::<u64, ScriptFnDef, _>::with_hasher(StraightHasherBuilder);
let mut state = ParseState::new(max_expr_depth);
while !input.peek().unwrap().0.is_eof() {
// Collect all the function definitions
#[cfg(not(feature = "no_function"))]
{
let (access, must_be_fn) = if match_token(input, Token::Private)? {
(FnAccess::Private, true)
} else {
(FnAccess::Public, false)
};
match input.peek().unwrap() {
#[cfg(not(feature = "no_function"))]
(Token::Fn, pos) => {
let mut state = ParseState::new(max_function_expr_depth);
let settings = ParseSettings {
allow_if_expr: true,
allow_stmt_expr: true,
is_global: false,
is_breakable: false,
level: 0,
pos: *pos,
};
let func = parse_fn(input, &mut state, access, settings)?;
// Qualifiers (none) + function name + number of arguments.
let hash = calc_fn_hash(empty(), &func.name, func.params.len(), empty());
functions.insert(hash, func);
continue;
}
(_, pos) if must_be_fn => {
return Err(PERR::MissingToken(
Token::Fn.into(),
format!("following '{}'", Token::Private.syntax()),
)
.into_err(*pos))
}
_ => (),
}
}
// Actual statement
let settings = ParseSettings {
allow_if_expr: true,
allow_stmt_expr: true,
is_global: true,
is_breakable: false,
level: 0,
pos: Position::none(),
};
let stmt = parse_stmt(input, &mut state, settings)?;
let need_semicolon = !stmt.is_self_terminated();
statements.push(stmt);
match input.peek().unwrap() {
// EOF
(Token::EOF, _) => break,
// stmt ;
(Token::SemiColon, _) if need_semicolon => {
eat_token(input, Token::SemiColon);
}
// stmt ;
(Token::SemiColon, _) if !need_semicolon => (),
// { stmt } ???
(_, _) if !need_semicolon => (),
// stmt <error>
(Token::LexError(err), pos) => {
return Err(PERR::BadInput(err.to_string()).into_err(*pos))
}
// stmt ???
(_, pos) => {
// Semicolons are not optional between statements
return Err(PERR::MissingToken(
Token::SemiColon.into(),
"to terminate this statement".into(),
)
.into_err(*pos));
}
}
}
Ok((statements, functions.into_iter().map(|(_, v)| v).collect()))
}
impl Engine {
pub(crate) fn parse_global_expr(
&self,
@@ -2511,7 +2444,12 @@ impl Engine {
scope: &Scope,
optimization_level: OptimizationLevel,
) -> Result<AST, ParseError> {
let mut state = ParseState::new(self.max_expr_depth);
let mut state = ParseState::new(
self.max_expr_depth,
self.max_string_size,
self.max_array_size,
self.max_map_size,
);
let settings = ParseSettings {
allow_if_expr: false,
allow_stmt_expr: false,
@@ -2540,6 +2478,111 @@ impl Engine {
)
}
/// Parse the global level statements.
fn parse_global_level(
&self,
input: &mut TokenStream,
) -> Result<(Vec<Stmt>, Vec<ScriptFnDef>), ParseError> {
let mut statements = Vec::<Stmt>::new();
let mut functions = HashMap::<u64, ScriptFnDef, _>::with_hasher(StraightHasherBuilder);
let mut state = ParseState::new(
self.max_expr_depth,
self.max_string_size,
self.max_array_size,
self.max_map_size,
);
while !input.peek().unwrap().0.is_eof() {
// Collect all the function definitions
#[cfg(not(feature = "no_function"))]
{
let (access, must_be_fn) = if match_token(input, Token::Private)? {
(FnAccess::Private, true)
} else {
(FnAccess::Public, false)
};
match input.peek().unwrap() {
#[cfg(not(feature = "no_function"))]
(Token::Fn, pos) => {
let mut state = ParseState::new(
self.max_function_expr_depth,
self.max_string_size,
self.max_array_size,
self.max_map_size,
);
let settings = ParseSettings {
allow_if_expr: true,
allow_stmt_expr: true,
is_global: false,
is_breakable: false,
level: 0,
pos: *pos,
};
let func = parse_fn(input, &mut state, access, settings)?;
// Qualifiers (none) + function name + number of arguments.
let hash = calc_fn_hash(empty(), &func.name, func.params.len(), empty());
functions.insert(hash, func);
continue;
}
(_, pos) if must_be_fn => {
return Err(PERR::MissingToken(
Token::Fn.into(),
format!("following '{}'", Token::Private.syntax()),
)
.into_err(*pos))
}
_ => (),
}
}
// Actual statement
let settings = ParseSettings {
allow_if_expr: true,
allow_stmt_expr: true,
is_global: true,
is_breakable: false,
level: 0,
pos: Position::none(),
};
let stmt = parse_stmt(input, &mut state, settings)?;
let need_semicolon = !stmt.is_self_terminated();
statements.push(stmt);
match input.peek().unwrap() {
// EOF
(Token::EOF, _) => break,
// stmt ;
(Token::SemiColon, _) if need_semicolon => {
eat_token(input, Token::SemiColon);
}
// stmt ;
(Token::SemiColon, _) if !need_semicolon => (),
// { stmt } ???
(_, _) if !need_semicolon => (),
// stmt <error>
(Token::LexError(err), pos) => {
return Err(PERR::BadInput(err.to_string()).into_err(*pos))
}
// stmt ???
(_, pos) => {
// Semicolons are not optional between statements
return Err(PERR::MissingToken(
Token::SemiColon.into(),
"to terminate this statement".into(),
)
.into_err(*pos));
}
}
}
Ok((statements, functions.into_iter().map(|(_, v)| v).collect()))
}
/// Run the parser on an input stream, returning an AST.
pub(crate) fn parse(
&self,
@@ -2547,8 +2590,7 @@ impl Engine {
scope: &Scope,
optimization_level: OptimizationLevel,
) -> Result<AST, ParseError> {
let (statements, lib) =
parse_global_level(input, self.max_expr_depth, self.max_function_expr_depth)?;
let (statements, lib) = self.parse_global_level(input)?;
Ok(
// Optimize AST

8
src/result.rs
View File

@@ -81,6 +81,8 @@ pub enum EvalAltResult {
ErrorTooManyModules(Position),
/// Call stack over maximum limit.
ErrorStackOverflow(Position),
/// Data value over maximum size limit. Wrapped values are the data type, maximum size and current size.
ErrorDataTooLarge(String, usize, usize, Position),
/// The script is prematurely terminated.
ErrorTerminated(Position),
/// Run-time error encountered. Wrapped value is the error message.
@@ -139,6 +141,7 @@ impl EvalAltResult {
Self::ErrorTooManyOperations(_) => "Too many operations",
Self::ErrorTooManyModules(_) => "Too many modules imported",
Self::ErrorStackOverflow(_) => "Stack overflow",
Self::ErrorDataTooLarge(_, _, _, _) => "Data size exceeds maximum limit",
Self::ErrorTerminated(_) => "Script terminated.",
Self::ErrorRuntime(_, _) => "Runtime error",
Self::ErrorLoopBreak(true, _) => "Break statement not inside a loop",
@@ -228,6 +231,9 @@ impl fmt::Display for EvalAltResult {
"String index {} is out of bounds: only {} characters in the string",
index, max
)?,
Self::ErrorDataTooLarge(typ, max, size, _) => {
write!(f, "{} ({}) exceeds the maximum limit ({})", typ, size, max)?
}
}
// Do not write any position if None
@@ -279,6 +285,7 @@ impl EvalAltResult {
| Self::ErrorTooManyOperations(pos)
| Self::ErrorTooManyModules(pos)
| Self::ErrorStackOverflow(pos)
| Self::ErrorDataTooLarge(_, _, _, pos)
| Self::ErrorTerminated(pos)
| Self::ErrorRuntime(_, pos)
| Self::ErrorLoopBreak(_, pos)
@@ -316,6 +323,7 @@ impl EvalAltResult {
| Self::ErrorTooManyOperations(pos)
| Self::ErrorTooManyModules(pos)
| Self::ErrorStackOverflow(pos)
| Self::ErrorDataTooLarge(_, _, _, pos)
| Self::ErrorTerminated(pos)
| Self::ErrorRuntime(_, pos)
| Self::ErrorLoopBreak(_, pos)

58
src/token.rs
View File

@@ -429,6 +429,8 @@ impl From<Token> for String {
/// An iterator on a `Token` stream.
pub struct TokenIterator<'a> {
/// Maximum length of a string (0 = unlimited).
max_string_size: usize,
/// Can the next token be a unary operator?
can_be_unary: bool,
/// Current position.
@@ -494,6 +496,7 @@ impl<'a> TokenIterator<'a> {
pub fn parse_string_literal(
&mut self,
enclosing_char: char,
max_length: usize,
) -> Result<String, (LexError, Position)> {
let mut result = Vec::new();
let mut escape = String::with_capacity(12);
@@ -505,6 +508,10 @@ impl<'a> TokenIterator<'a> {
self.advance();
if max_length > 0 && result.len() > max_length {
return Err((LexError::StringTooLong(max_length), self.pos));
}
match next_char {
// \...
'\\' if escape.is_empty() => {
@@ -592,7 +599,13 @@ impl<'a> TokenIterator<'a> {
}
}
Ok(result.iter().collect())
let s = result.iter().collect::<String>();
if max_length > 0 && s.len() > max_length {
return Err((LexError::StringTooLong(max_length), self.pos));
}
Ok(s)
}
/// Get the next token.
@@ -779,10 +792,12 @@ impl<'a> TokenIterator<'a> {
// " - string literal
('"', _) => {
return self.parse_string_literal('"').map_or_else(
|err| Some((Token::LexError(Box::new(err.0)), err.1)),
|out| Some((Token::StringConst(out), pos)),
);
return self
.parse_string_literal('"', self.max_string_size)
.map_or_else(
|err| Some((Token::LexError(Box::new(err.0)), err.1)),
|out| Some((Token::StringConst(out), pos)),
);
}
// ' - character literal
@@ -793,19 +808,25 @@ impl<'a> TokenIterator<'a> {
));
}
('\'', _) => {
return Some(self.parse_string_literal('\'').map_or_else(
|err| (Token::LexError(Box::new(err.0)), err.1),
|result| {
let mut chars = result.chars();
let first = chars.next();
return Some(
self.parse_string_literal('\'', self.max_string_size)
.map_or_else(
|err| (Token::LexError(Box::new(err.0)), err.1),
|result| {
let mut chars = result.chars();
let first = chars.next();
if chars.next().is_some() {
(Token::LexError(Box::new(LERR::MalformedChar(result))), pos)
} else {
(Token::CharConstant(first.expect("should be Some")), pos)
}
},
));
if chars.next().is_some() {
(
Token::LexError(Box::new(LERR::MalformedChar(result))),
pos,
)
} else {
(Token::CharConstant(first.expect("should be Some")), pos)
}
},
),
);
}
// Braces
@@ -1047,8 +1068,9 @@ impl<'a> Iterator for TokenIterator<'a> {
}
/// Tokenize an input text stream.
pub fn lex<'a>(input: &'a [&'a str]) -> TokenIterator<'a> {
pub fn lex<'a>(input: &'a [&'a str], max_string_size: usize) -> TokenIterator<'a> {
TokenIterator {
max_string_size,
can_be_unary: true,
pos: Position::new(1, 0),
streams: input.iter().map(|s| s.chars().peekable()).collect(),

9
src/utils.rs
View File

@@ -216,6 +216,15 @@ impl<T> FromIterator<T> for StaticVec<T> {
}
}
impl<T: 'static> IntoIterator for StaticVec<T> {
type Item = T;
type IntoIter = Box<dyn Iterator<Item = T>>;
fn into_iter(self) -> Self::IntoIter {
self.into_iter()
}
}
impl<T> StaticVec<T> {
/// Create a new `StaticVec`.
#[inline(always)]