diff --git a/Cargo.toml b/Cargo.toml
index 66bdba28..f049777e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,7 +14,7 @@ include = [
"Cargo.toml"
]
keywords = [ "scripting" ]
-categories = [ "no-std", "embedded", "parser-implementations" ]
+categories = [ "no-std", "embedded", "wasm", "parser-implementations" ]
[dependencies]
num-traits = { version = "0.2.11", default-features = false }
@@ -34,7 +34,7 @@ no_object = [] # no custom objects
no_function = [] # no script-defined functions
no_module = [] # no modules
internals = [] # expose internal data structures
-unicode-xid-ident = ["unicode-xid"] # allow unicode-xid for identifiers.
+unicode-xid-ident = ["unicode-xid"] # allow Unicode Standard Annex #31 for identifiers.
# compiling for no-std
no_std = [ "num-traits/libm", "hashbrown", "core-error", "libm", "ahash" ]
diff --git a/RELEASES.md b/RELEASES.md
index 87d12d44..7f59cddf 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -20,7 +20,7 @@ New features
* Custom syntax now works even without the `internals` feature.
* Currying of function pointers is supported via the new `curry` keyword.
* `Module::set_indexer_get_set_fn` is added as a shorthand of both `Module::set_indexer_get_fn` and `Module::set_indexer_set_fn`.
-* New `unicode-xid-ident` feature to allow unicode-xid for identifiers.
+* New `unicode-xid-ident` feature to allow [Unicode Standard Annex #31](http://www.unicode.org/reports/tr31/) for identifiers.
Breaking changes
----------------
diff --git a/doc/src/language/variables.md b/doc/src/language/variables.md
index be4abbce..455c075b 100644
--- a/doc/src/language/variables.md
+++ b/doc/src/language/variables.md
@@ -21,6 +21,11 @@ Variable names are case _sensitive_.
Variable names also cannot be the same as a [keyword].
+### Unicode Standard Annex #31 Identifiers
+
+The [`unicode-xid-ident`] feature expands the allowed characters for variable names to the set defined by
+[Unicode Standard Annex #31](http://www.unicode.org/reports/tr31/).
+
Declare a Variable
------------------
diff --git a/doc/src/start/features.md b/doc/src/start/features.md
index 83311cf1..0f6b0f53 100644
--- a/doc/src/start/features.md
+++ b/doc/src/start/features.md
@@ -11,22 +11,22 @@ Notice that this deviates from Rust norm where features are _additive_.
Excluding unneeded functionalities can result in smaller, faster builds as well as
more control over what a script can (or cannot) do.
-| Feature | Description |
-| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `unchecked` | Disable arithmetic checking (such as over-flows and division by zero), call stack depth limit, operations count limit and modules loading limit.
Beware that a bad script may panic the entire system! |
-| `sync` | Restrict all values types to those that are `Send + Sync`. Under this feature, all Rhai types, including [`Engine`], [`Scope`] and [`AST`], are all `Send + Sync`. |
-| `no_optimize` | Disable [script optimization]. |
-| `no_float` | Disable floating-point numbers and math. |
-| `only_i32` | Set the system integer type to `i32` and disable all other integer types. `INT` is set to `i32`. |
-| `only_i64` | Set the system integer type to `i64` and disable all other integer types. `INT` is set to `i64`. |
-| `no_index` | Disable [arrays] and indexing features. |
-| `no_object` | Disable support for [custom types] and [object maps]. |
-| `no_function` | Disable script-defined [functions]. |
-| `no_module` | Disable loading external [modules]. |
-| `no_std` | Build for `no-std`. Notice that additional dependencies will be pulled in to replace `std` features. |
-| `serde` | Enable serialization/deserialization via `serde`. Notice that the [`serde`](https://crates.io/crates/serde) crate will be pulled in together with its dependencies. |
-| `internals` | Expose internal data structures (e.g. [`AST`] nodes). Beware that Rhai internals are volatile and may change from version to version. |
-| `unicode-xid-ident` | Allow unicode-xid for identifiers. |
+| Feature | Description |
+| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `unchecked` | Disable arithmetic checking (such as over-flows and division by zero), call stack depth limit, operations count limit and modules loading limit.
Beware that a bad script may panic the entire system! |
+| `sync` | Restrict all values types to those that are `Send + Sync`. Under this feature, all Rhai types, including [`Engine`], [`Scope`] and [`AST`], are all `Send + Sync`. |
+| `no_optimize` | Disable [script optimization]. |
+| `no_float` | Disable floating-point numbers and math. |
+| `only_i32` | Set the system integer type to `i32` and disable all other integer types. `INT` is set to `i32`. |
+| `only_i64` | Set the system integer type to `i64` and disable all other integer types. `INT` is set to `i64`. |
+| `no_index` | Disable [arrays] and indexing features. |
+| `no_object` | Disable support for [custom types] and [object maps]. |
+| `no_function` | Disable script-defined [functions]. |
+| `no_module` | Disable loading external [modules]. |
+| `no_std` | Build for `no-std`. Notice that additional dependencies will be pulled in to replace `std` features. |
+| `serde` | Enable serialization/deserialization via `serde`. Notice that the [`serde`](https://crates.io/crates/serde) crate will be pulled in together with its dependencies. |
+| `internals` | Expose internal data structures (e.g. [`AST`] nodes). Beware that Rhai internals are volatile and may change from version to version. |
+| `unicode-xid-ident` | Allow [Unicode Standard Annex #31](http://www.unicode.org/reports/tr31/) as identifiers. |
Example
diff --git a/src/token.rs b/src/token.rs
index 9f5456d9..77187304 100644
--- a/src/token.rs
+++ b/src/token.rs
@@ -1104,7 +1104,7 @@ fn get_next_token_inner(
|err| (Token::LexError(Box::new(err.0)), err.1),
|result| {
let mut chars = result.chars();
- let first = chars.next();
+ let first = chars.next().unwrap();
if chars.next().is_some() {
(
@@ -1112,10 +1112,7 @@ fn get_next_token_inner(
start_pos,
)
} else {
- (
- Token::CharConstant(first.expect("should be Some")),
- start_pos,
- )
+ (Token::CharConstant(first), start_pos)
}
},
))
@@ -1419,6 +1416,7 @@ fn get_identifier(
}
/// Is this keyword allowed as a function?
+#[inline(always)]
pub fn is_keyword_function(name: &str) -> bool {
name == KEYWORD_PRINT
|| name == KEYWORD_DEBUG
@@ -1446,22 +1444,25 @@ pub fn is_valid_identifier(name: impl Iterator- ) -> bool {
}
#[cfg(feature = "unicode-xid-ident")]
+#[inline(always)]
fn is_id_first_alphabetic(x: char) -> bool {
unicode_xid::UnicodeXID::is_xid_start(x)
}
#[cfg(feature = "unicode-xid-ident")]
+#[inline(always)]
fn is_id_continue(x: char) -> bool {
unicode_xid::UnicodeXID::is_xid_continue(x)
}
#[cfg(not(feature = "unicode-xid-ident"))]
-
+#[inline(always)]
fn is_id_first_alphabetic(x: char) -> bool {
x.is_ascii_alphabetic()
}
#[cfg(not(feature = "unicode-xid-ident"))]
+#[inline(always)]
fn is_id_continue(x: char) -> bool {
x.is_ascii_alphanumeric() || x == '_'
}
diff --git a/tests/tokens.rs b/tests/tokens.rs
index 843fc719..e393663d 100644
--- a/tests/tokens.rs
+++ b/tests/tokens.rs
@@ -57,9 +57,9 @@ fn test_tokens_unicode_xid_ident() -> Result<(), Box> {
let engine = Engine::new();
let result = engine.eval::(
r"
- fn すべての答え() { 42 }
- すべての答え()
- ",
+ fn すべての答え() { 42 }
+ すべての答え()
+ ",
);
#[cfg(feature = "unicode-xid-ident")]
assert_eq!(result?, 42);
@@ -69,9 +69,9 @@ fn test_tokens_unicode_xid_ident() -> Result<(), Box> {
let result = engine.eval::(
r"
- fn _1() { 1 }
- _1()
- ",
+ fn _1() { 1 }
+ _1()
+ ",
);
assert!(result.is_err());