From 9ef522b6997026146dedc6b8cb858bf0fb00022d Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sat, 26 Feb 2022 17:28:58 +0800 Subject: [PATCH] Use SmartString in parsing. --- src/ast/flags.rs | 9 --------- src/ast/mod.rs | 4 ++-- src/ast/script_fn.rs | 11 ++++++++++- src/lib.rs | 2 +- src/parser.rs | 32 ++++++++++++++++++-------------- src/tokenizer.rs | 41 ++++++++++++++++++++--------------------- 6 files changed, 51 insertions(+), 48 deletions(-) diff --git a/src/ast/flags.rs b/src/ast/flags.rs index 264a6374..1e0ad969 100644 --- a/src/ast/flags.rs +++ b/src/ast/flags.rs @@ -4,15 +4,6 @@ use bitflags::bitflags; #[cfg(feature = "no_std")] use std::prelude::v1::*; -/// A type representing the access mode of a function. -#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum FnAccess { - /// Private function. - Private, - /// Public function. - Public, -} - bitflags! { /// _(internals)_ A type that holds a configuration option with bit-flags. /// Exported under the `internals` feature only. diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 16be3bb9..9b24ddb3 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -9,13 +9,13 @@ pub mod stmt; pub use ast::{ASTNode, AST}; pub use expr::{BinaryExpr, CustomExpr, Expr, FnCallExpr, FnCallHashes}; -pub use flags::{ASTFlags, FnAccess}; +pub use flags::ASTFlags; pub use ident::Ident; #[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_function"))] pub use script_fn::EncapsulatedEnviron; #[cfg(not(feature = "no_function"))] -pub use script_fn::{ScriptFnDef, ScriptFnMetadata}; +pub use script_fn::{FnAccess, ScriptFnDef, ScriptFnMetadata}; pub use stmt::{ ConditionalStmtBlock, OpAssignment, Stmt, StmtBlock, StmtBlockContainer, SwitchCases, TryCatchBlock, diff --git a/src/ast/script_fn.rs b/src/ast/script_fn.rs index 4c103cf7..29f8e0c8 100644 --- a/src/ast/script_fn.rs +++ b/src/ast/script_fn.rs @@ -1,12 +1,21 @@ //! Module defining script-defined functions. #![cfg(not(feature = "no_function"))] -use super::{FnAccess, StmtBlock}; +use super::StmtBlock; use crate::{Identifier, StaticVec}; #[cfg(feature = "no_std")] use std::prelude::v1::*; use std::{fmt, hash::Hash}; +/// A type representing the access mode of a function. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum FnAccess { + /// Private function. + Private, + /// Public function. + Public, +} + /// _(internals)_ Encapsulated AST environment. /// Exported under the `internals` feature only. /// diff --git a/src/lib.rs b/src/lib.rs index ad8d640a..0c089e60 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -365,7 +365,7 @@ pub type StaticVec = smallvec::SmallVec<[T; 3]>; /// /// Under `no_closure`, this type aliases to [`StaticVec`][crate::StaticVec] instead. #[cfg(not(feature = "no_closure"))] -type FnArgsVec = smallvec::SmallVec<[T; 8]>; +type FnArgsVec = smallvec::SmallVec<[T; 5]>; /// Inline arguments storage for function calls. /// This type aliases to [`StaticVec`][crate::StaticVec]. diff --git a/src/parser.rs b/src/parser.rs index bb751e4e..0dedab87 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -19,7 +19,7 @@ use crate::types::StringsInterner; use crate::{ calc_fn_hash, Dynamic, Engine, EvalAltResult, EvalContext, ExclusiveRange, Identifier, ImmutableString, InclusiveRange, LexError, OptimizationLevel, ParseError, Position, Scope, - Shared, StaticVec, AST, INT, PERR, + Shared, SmartString, StaticVec, AST, INT, PERR, }; #[cfg(feature = "no_std")] use std::prelude::v1::*; @@ -387,7 +387,7 @@ fn match_token(input: &mut TokenStream, token: Token) -> (bool, Position) { } /// Parse a variable name. -fn parse_var_name(input: &mut TokenStream) -> ParseResult<(Box, Position)> { +fn parse_var_name(input: &mut TokenStream) -> ParseResult<(SmartString, Position)> { match input.next().expect(NEVER_ENDS) { // Variable name (Token::Identifier(s), pos) => Ok((s, pos)), @@ -403,7 +403,7 @@ fn parse_var_name(input: &mut TokenStream) -> ParseResult<(Box, Position)> } /// Parse a symbol. -fn parse_symbol(input: &mut TokenStream) -> ParseResult<(Box, Position)> { +fn parse_symbol(input: &mut TokenStream) -> ParseResult<(SmartString, Position)> { match input.next().expect(NEVER_ENDS) { // Symbol (token, pos) if token.is_standard_symbol() => Ok((token.literal_syntax().into(), pos)), @@ -2059,7 +2059,7 @@ fn parse_binary_op( Token::Custom(c) => state .engine .custom_keywords - .get(c.as_ref()) + .get(c) .cloned() .ok_or_else(|| PERR::Reserved(c.to_string()).into_err(*current_pos))?, Token::Reserved(c) if !is_valid_identifier(c.chars()) => { @@ -2084,7 +2084,7 @@ fn parse_binary_op( Token::Custom(c) => state .engine .custom_keywords - .get(c.as_ref()) + .get(c) .cloned() .ok_or_else(|| PERR::Reserved(c.to_string()).into_err(*next_pos))?, Token::Reserved(c) if !is_valid_identifier(c.chars()) => { @@ -2182,7 +2182,7 @@ fn parse_binary_op( if state .engine .custom_keywords - .get(s.as_ref()) + .get(s.as_str()) .map_or(false, Option::is_some) => { let hash = calc_fn_hash(&s, 2); @@ -2630,14 +2630,12 @@ fn parse_let( // let name ... let (name, pos) = parse_var_name(input)?; - if !settings.default_options.allow_shadowing - && state.stack.iter().any(|(v, ..)| v == name.as_ref()) - { + if !settings.default_options.allow_shadowing && state.stack.iter().any(|(v, ..)| v == &name) { return Err(PERR::VariableExists(name.to_string()).into_err(pos)); } if let Some(ref filter) = state.engine.def_var_filter { - let will_shadow = state.stack.iter().any(|(v, ..)| v == name.as_ref()); + let will_shadow = state.stack.iter().any(|(v, ..)| v == &name); let level = settings.level; let is_const = access == AccessMode::ReadOnly; let info = VarDefInfo { @@ -2820,7 +2818,7 @@ fn parse_block( } }; - let mut statements = Vec::with_capacity(8); + let mut statements = StaticVec::new(); let prev_entry_stack_len = state.block_stack_len; state.block_stack_len = state.stack.len(); @@ -2924,7 +2922,7 @@ fn parse_stmt( #[cfg(not(feature = "no_function"))] #[cfg(feature = "metadata")] let comments = { - let mut comments = Vec::>::new(); + let mut comments = StaticVec::::new(); let mut comments_pos = Position::NONE; // Handle doc-comments. @@ -3201,7 +3199,7 @@ fn parse_fn( settings: ParseSettings, #[cfg(not(feature = "no_function"))] #[cfg(feature = "metadata")] - comments: Vec>, + comments: StaticVec, ) -> ParseResult { #[cfg(not(feature = "unchecked"))] settings.ensure_level_within_max_limit(state.max_expr_depth)?; @@ -3285,7 +3283,13 @@ fn parse_fn( comments: if comments.is_empty() { None } else { - Some(comments.into()) + Some( + comments + .into_iter() + .map(|s| s.to_string().into_boxed_str()) + .collect::>() + .into_boxed_slice(), + ) }, }) } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index bf8326ac..a579042c 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -5,7 +5,7 @@ use crate::engine::{ KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_THIS, KEYWORD_TYPE_OF, }; use crate::func::native::OnParseTokenCallback; -use crate::{Engine, LexError, StaticVec, INT, UNSIGNED_INT}; +use crate::{Engine, LexError, SmartString, StaticVec, INT, UNSIGNED_INT}; #[cfg(feature = "no_std")] use std::prelude::v1::*; use std::{ @@ -363,13 +363,13 @@ pub enum Token { #[cfg(feature = "decimal")] DecimalConstant(rust_decimal::Decimal), /// An identifier. - Identifier(Box), + Identifier(SmartString), /// A character constant. CharConstant(char), /// A string constant. - StringConstant(Box), + StringConstant(SmartString), /// An interpolated string. - InterpolatedString(Box), + InterpolatedString(SmartString), /// `{` LeftBrace, /// `}` @@ -536,11 +536,11 @@ pub enum Token { /// A lexer error. LexError(LexError), /// A comment block. - Comment(Box), + Comment(SmartString), /// A reserved symbol. - Reserved(Box), + Reserved(SmartString), /// A custom keyword. - Custom(Box), + Custom(SmartString), /// End of the input stream. EOF, } @@ -1022,9 +1022,9 @@ impl Token { /// Convert a token into a function name, if possible. #[cfg(not(feature = "no_function"))] #[inline] - pub(crate) fn into_function_name_for_override(self) -> Result, Self> { + pub(crate) fn into_function_name_for_override(self) -> Result { match self { - Self::Custom(s) | Self::Identifier(s) if is_valid_function_name(&*s) => Ok(s), + Self::Custom(s) | Self::Identifier(s) if is_valid_function_name(&s) => Ok(s), _ => Err(self), } } @@ -1112,9 +1112,9 @@ pub fn parse_string_literal( verbatim: bool, allow_line_continuation: bool, allow_interpolation: bool, -) -> Result<(Box, bool, Position), (LexError, Position)> { - let mut result = String::with_capacity(12); - let mut escape = String::with_capacity(12); +) -> Result<(SmartString, bool, Position), (LexError, Position)> { + let mut result = SmartString::new(); + let mut escape = SmartString::new(); let start = *pos; let mut first_char = Position::NONE; @@ -1146,7 +1146,6 @@ pub fn parse_string_literal( break; } None => { - result += &escape; pos.advance(); state.is_within_text_terminated_by = None; return Err((LERR::UnterminatedString, start)); @@ -1242,7 +1241,7 @@ pub fn parse_string_literal( result.push( char::from_u32(out_val) - .ok_or_else(|| (LERR::MalformedEscapeSequence(seq), *pos))?, + .ok_or_else(|| (LERR::MalformedEscapeSequence(seq.to_string()), *pos))?, ); } @@ -1283,7 +1282,7 @@ pub fn parse_string_literal( _ if !escape.is_empty() => { escape.push(next_char); - return Err((LERR::MalformedEscapeSequence(escape), *pos)); + return Err((LERR::MalformedEscapeSequence(escape.to_string()), *pos)); } // Whitespace to skip @@ -1309,7 +1308,7 @@ pub fn parse_string_literal( } } - Ok((result.into(), interpolated, first_char)) + Ok((result, interpolated, first_char)) } /// Consume the next character. @@ -2291,18 +2290,18 @@ impl<'a> Iterator for TokenIterator<'a> { (Token::Custom(s), pos) } // Custom keyword/symbol - must be disabled - Some((token, pos)) if self.engine.custom_keywords.contains_key(&*token.syntax()) => { - if self.engine.disabled_symbols.contains(&*token.syntax()) { + Some((token, pos)) if self.engine.custom_keywords.contains_key(token.literal_syntax()) => { + if self.engine.disabled_symbols.contains(token.literal_syntax()) { // Disabled standard keyword/symbol - (Token::Custom(token.syntax().into()), pos) + (Token::Custom(token.literal_syntax().into()), pos) } else { // Active standard keyword - should never be a custom keyword! unreachable!("{:?} is an active keyword", token) } } // Disabled symbol - Some((token, pos)) if self.engine.disabled_symbols.contains(&*token.syntax()) => { - (Token::Reserved(token.syntax().into()), pos) + Some((token, pos)) if self.engine.disabled_symbols.contains(token.literal_syntax()) => { + (Token::Reserved(token.literal_syntax().into()), pos) } // Normal symbol Some(r) => r,