diff --git a/Cargo.toml b/Cargo.toml index 2f2ab525..f0fdaa33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ categories = [ "no-std", "embedded", "parser-implementations" ] [dependencies] num-traits = { version = "0.2.11", default-features = false } +unicode-xid = "0.2.1" [features] #default = ["unchecked", "sync", "no_optimize", "no_float", "only_i32", "no_index", "no_object", "no_function", "no_module"] diff --git a/src/token.rs b/src/token.rs index d79c5f72..6cb1a4d2 100644 --- a/src/token.rs +++ b/src/token.rs @@ -1338,6 +1338,9 @@ fn get_next_token_inner( ('\0', _) => unreachable!(), (ch, _) if ch.is_whitespace() => (), + (ch, _) if unicode_xid::UnicodeXID::is_xid_start(ch) => { + return get_identifier(stream, pos, start_pos, c); + } (ch, _) => { return Some(( Token::LexError(Box::new(LERR::UnexpectedInput(ch.to_string()))), @@ -1409,6 +1412,15 @@ pub fn is_valid_identifier(name: impl Iterator) -> bool { first_alphabetic } +fn is_id_first_alphabetic(x: char) -> bool { + unicode_xid::UnicodeXID::is_xid_start(x) +} + +fn is_id_continue(x: char) -> bool { + unicode_xid::UnicodeXID::is_xid_continue(x) +} + +/* fn is_id_first_alphabetic(x: char) -> bool { x.is_ascii_alphabetic() } @@ -1416,6 +1428,7 @@ fn is_id_first_alphabetic(x: char) -> bool { fn is_id_continue(x: char) -> bool { x.is_ascii_alphanumeric() || x == '_' } +*/ /// A type that implements the `InputStream` trait. /// Multiple character streams are jointed together to form one single stream.