Add unicode support for zenscripts

The unicode characters can be recognized in strings and comments.
The encoding of scripts should be utf-8.
This commit is contained in:
Skydark Chen committed 2015-11-15 17:22:47 +08:00
1 parent 89b2548d02
commit 33c2bd0c94
3 files changed
+15 -4

No files matched your search

@@ -21,6 +21,10 @@ public class NFA {
public static final int NOFINAL = Integer.MIN_VALUE;
public static final int EPSILON = Integer.MIN_VALUE + 1;
/* UNICODE_ESCAPE should be between 128 and 256, so unicode chars can be
matched by regexp like '.' and '[^a]' and not by '[a-zA-Z]' */
public static final int UNICODE_ESCAPE = 256;
private final NFAState initial;
private HashMap<NodeSet, DFA.DFAState> converted;
@@ -318,6 +322,8 @@ public class NFA {
/* Processes a single character */
private int processChar(CharStream stream) {
if (stream.optional('\\')) {
if (stream.optional('u'))
return UNICODE_ESCAPE;
if (stream.optional('e'))
return -1;
if (stream.optional('r'))
@@ -212,12 +212,17 @@ public class TokenStream implements Iterator<Token> {
StringBuilder value = new StringBuilder();
int tLine = line;
int tLineOffset = lineOffset;
while (dfa.transitions[state].containsKey(nextChar)) {
value.append((char) nextChar);
state = dfa.transitions[state].get(nextChar);
int _fakeNextChar = nextChar > 127 ? NFA.UNICODE_ESCAPE : nextChar;
while (dfa.transitions[state].containsKey(_fakeNextChar)) {
if (nextChar < 0)
value.append((char) nextChar);
else
value.appendCodePoint(nextChar);
state = dfa.transitions[state].get(_fakeNextChar);
line = reader.line;
lineOffset = reader.lineOffset;
nextChar = reader.read();
_fakeNextChar = nextChar > 127 ? NFA.UNICODE_ESCAPE : nextChar;
}
if (line < 0)