mirror of
https://github.com/GTNewHorizons/CraftTweaker.git
synced 2026-11-17 19:05:19 +00:00
Add unicode support for zenscripts
The unicode characters can be recognized in strings and comments. The encoding of scripts should be utf-8.
This commit is contained in:
1 parent
89b2548d02
commit
33c2bd0c94
3 files changed
+15
-4
No files matched your search
@@ -21,6 +21,10 @@ public class NFA {
|
||||
public static final int NOFINAL = Integer.MIN_VALUE;
|
||||
public static final int EPSILON = Integer.MIN_VALUE + 1;
|
||||
|
||||
/* UNICODE_ESCAPE should be between 128 and 256, so unicode chars can be
|
||||
matched by regexp like '.' and '[^a]' and not by '[a-zA-Z]' */
|
||||
public static final int UNICODE_ESCAPE = 256;
|
||||
|
||||
private final NFAState initial;
|
||||
|
||||
private HashMap<NodeSet, DFA.DFAState> converted;
|
||||
@@ -318,6 +322,8 @@ public class NFA {
|
||||
/* Processes a single character */
|
||||
private int processChar(CharStream stream) {
|
||||
if (stream.optional('\\')) {
|
||||
if (stream.optional('u'))
|
||||
return UNICODE_ESCAPE;
|
||||
if (stream.optional('e'))
|
||||
return -1;
|
||||
if (stream.optional('r'))
|
||||
|
||||
@@ -212,12 +212,17 @@ public class TokenStream implements Iterator<Token> {
|
||||
StringBuilder value = new StringBuilder();
|
||||
int tLine = line;
|
||||
int tLineOffset = lineOffset;
|
||||
while (dfa.transitions[state].containsKey(nextChar)) {
|
||||
value.append((char) nextChar);
|
||||
state = dfa.transitions[state].get(nextChar);
|
||||
int _fakeNextChar = nextChar > 127 ? NFA.UNICODE_ESCAPE : nextChar;
|
||||
while (dfa.transitions[state].containsKey(_fakeNextChar)) {
|
||||
if (nextChar < 0)
|
||||
value.append((char) nextChar);
|
||||
else
|
||||
value.appendCodePoint(nextChar);
|
||||
state = dfa.transitions[state].get(_fakeNextChar);
|
||||
line = reader.line;
|
||||
lineOffset = reader.lineOffset;
|
||||
nextChar = reader.read();
|
||||
_fakeNextChar = nextChar > 127 ? NFA.UNICODE_ESCAPE : nextChar;
|
||||
}
|
||||
|
||||
if (line < 0)
|
||||
|
||||
Reference in new issue
Block a user