kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

More code refactor.

Browse Source
This commit is contained in:
Stephen Chung
2022-11-25 20:42:16 +08:00
parent fbe30b8d0e
commit d645d8271c
30 changed files with 422 additions and 434 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/tokenizer.rs
View File

@@ -584,7 +584,10 @@ pub enum Token {
#[cfg(not(feature = "no_custom_syntax"))]
Custom(Box<SmartString>),
/// End of the input stream.
/// Used as a placeholder for the end of input.
EOF,
/// Placeholder to indicate the lack of a token.
NonToken,
}
impl fmt::Display for Token {
@@ -610,6 +613,7 @@ impl fmt::Display for Token {
Comment(s) => f.write_str(s),
EOF => f.write_str("{EOF}"),
NonToken => f.write_str("{NONE}"),
token => f.write_str(token.literal_syntax()),
}
@@ -638,7 +642,7 @@ impl Token {
Custom(..) => false,
LexError(..) | Comment(..) => false,
EOF => false,
EOF | NonToken => false,
_ => true,
}
@@ -956,13 +960,6 @@ impl Token {
}
}
/// Is this token [`EOF`][Token::EOF]?
#[inline(always)]
#[must_use]
pub const fn is_eof(&self) -> bool {
matches!(self, Self::EOF)
}
/// If another operator is after these, it's probably a unary operator
/// (not sure about `fn` name).
#[must_use]
@@ -2435,7 +2432,7 @@ impl<'a> Iterator for TokenIterator<'a> {
Some((Token::Reserved(s), pos)) => (match
(s.as_str(),
#[cfg(not(feature = "no_custom_syntax"))]
self.engine.custom_keywords.as_ref().map_or(false, |m| m.contains_key(&*s)),
self.engine.custom_keywords.as_deref().map_or(false, |m| m.contains_key(&*s)),
#[cfg(feature = "no_custom_syntax")]
false
)
@@ -2472,7 +2469,7 @@ impl<'a> Iterator for TokenIterator<'a> {
#[cfg(feature = "no_custom_syntax")]
(.., true) => unreachable!("no custom operators"),
// Reserved keyword that is not custom and disabled.
(token, false) if self.engine.disabled_symbols.as_ref().map_or(false,|m| m.contains(token)) => {
(token, false) if self.engine.disabled_symbols.as_deref().map_or(false,|m| m.contains(token)) => {
let msg = format!("reserved {} '{token}' is disabled", if is_valid_identifier(token) { "keyword"} else {"symbol"});
Token::LexError(LERR::ImproperSymbol(s.to_string(), msg).into())
},
@@ -2481,13 +2478,13 @@ impl<'a> Iterator for TokenIterator<'a> {
}, pos),
// Custom keyword
#[cfg(not(feature = "no_custom_syntax"))]
Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.as_ref().map_or(false,|m| m.contains_key(&*s)) => {
Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.as_deref().map_or(false,|m| m.contains_key(&*s)) => {
(Token::Custom(s), pos)
}
// Custom keyword/symbol - must be disabled
#[cfg(not(feature = "no_custom_syntax"))]
Some((token, pos)) if token.is_literal() && self.engine.custom_keywords.as_ref().map_or(false,|m| m.contains_key(token.literal_syntax())) => {
if self.engine.disabled_symbols.as_ref().map_or(false,|m| m.contains(token.literal_syntax())) {
Some((token, pos)) if token.is_literal() && self.engine.custom_keywords.as_deref().map_or(false,|m| m.contains_key(token.literal_syntax())) => {
if self.engine.disabled_symbols.as_deref().map_or(false,|m| m.contains(token.literal_syntax())) {
// Disabled standard keyword/symbol
(Token::Custom(Box::new(token.literal_syntax().into())), pos)
} else {
@@ -2496,7 +2493,7 @@ impl<'a> Iterator for TokenIterator<'a> {
}
}
// Disabled symbol
Some((token, pos)) if token.is_literal() && self.engine.disabled_symbols.as_ref().map_or(false,|m| m.contains(token.literal_syntax())) => {
Some((token, pos)) if token.is_literal() && self.engine.disabled_symbols.as_deref().map_or(false,|m| m.contains(token.literal_syntax())) => {
(Token::Reserved(Box::new(token.literal_syntax().into())), pos)
}
// Normal symbol