Add unicode support for zenscripts

The unicode characters can be recognized in strings and comments.
The encoding of scripts should be utf-8.
This commit is contained in:
Skydark Chen committed 2015-11-15 17:22:47 +08:00
1 parent 89b2548d02
commit 33c2bd0c94
3 files changed
+15 -4

No files matched your search

@@ -21,6 +21,10 @@ public class NFA {
public static final int NOFINAL = Integer.MIN_VALUE;
public static final int EPSILON = Integer.MIN_VALUE + 1;
/* UNICODE_ESCAPE should be between 128 and 256, so unicode chars can be
matched by regexp like '.' and '[^a]' and not by '[a-zA-Z]' */
public static final int UNICODE_ESCAPE = 256;
private final NFAState initial;
private HashMap<NodeSet, DFA.DFAState> converted;
@@ -318,6 +322,8 @@ public class NFA {
/* Processes a single character */
private int processChar(CharStream stream) {
if (stream.optional('\\')) {
if (stream.optional('u'))
return UNICODE_ESCAPE;
if (stream.optional('e'))
return -1;
if (stream.optional('r'))