Files
CraftTweaker/ZenScript/src/main/java/stanhebben/zenscript/ZenTokener.java
T
Skydark Chen c35e3b1f78 Squashed commit of the following:
commit 0a3f5772b9450672fb033a6306fbf7fb0d972d5c
Author: Skydark Chen <[email protected]>
Date:   Fri Nov 27 10:54:46 2015 +0800

    Fix for-each iterator
    It should not pop from stack at the end while compiling.

commit 07c057e7502eb51ab3e6088eb2aee9ceb7d07576
Author: Skydark Chen <[email protected]>
Date:   Tue Nov 24 11:45:07 2015 +0800

    Fix non-int type arithmetic compare
    It was an error if we compare two double/float/long in scripts
    since the bytecode compiled in ExpressionArithmeticCompare.java were
    ICMPEQ/ICMPNE/... which assumed that two expressions had an int type.

commit 941b32c8be
Author: Skydark Chen <[email protected]>
Date:   Sun Nov 15 21:24:25 2015 +0800

    Fix ZenGetter and ZenSetter annotations
    Before this commit, ZenGetter and ZenSetter in ZenExpansion could not
    be defined. Since ZenExpansion requires static methods to work, but
    the getters and setters will be invoked as virtual.

commit bd55380b5a
Author: Skydark Chen <[email protected]>
Date:   Sun Nov 15 19:58:21 2015 +0800

    Fix block comments regexp

    Before this commit, comments like this could not be recognized:

        /* blah blah * blah  / <- this slash failed */

commit 2924666ad4
Author: Skydark Chen <[email protected]>
Date:   Sun Nov 15 19:44:02 2015 +0800

    Add support to return void in a function

    * statement `return;` is valid now.
    * functions return void by default, an explicit return is not necessary now

commit 33c2bd0c94
Author: Skydark Chen <[email protected]>
Date:   Sat Nov 14 13:48:53 2015 +0800

    Add unicode support for zenscripts

    The unicode characters can be recognized in strings and comments.
    The encoding of scripts should be utf-8.

commit 89b2548d02
Author: Skydark Chen <[email protected]>
Date:   Thu Nov 12 23:39:25 2015 +0800

    Fix nested foreach loop cause out of index
2015-11-30 19:15:25 +08:00

283 lines
6.8 KiB
Java

package stanhebben.zenscript;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.HashMap;
import stanhebben.zenscript.parser.CompiledDFA;
import stanhebben.zenscript.parser.NFA;
import stanhebben.zenscript.parser.Token;
import stanhebben.zenscript.parser.TokenStream;
/**
* A tokener is capable of splitting a single file into tokens. It's intended
* for use by LL(*) parsers.
*
* @author Stan Hebben
*/
public class ZenTokener extends TokenStream {
private static final HashMap<String, Integer> KEYWORDS;
public static final int T_ID = 1;
public static final int T_INTVALUE = 2;
public static final int T_FLOATVALUE = 3;
public static final int T_STRINGVALUE = 4;
public static final int T_AOPEN = 5;
public static final int T_ACLOSE = 6;
public static final int T_SQBROPEN = 7;
public static final int T_SQBRCLOSE = 8;
public static final int T_DOT2 = 9;
public static final int T_DOT = 10;
public static final int T_COMMA = 11;
public static final int T_PLUSASSIGN = 12;
public static final int T_PLUS = 13;
public static final int T_MINUSASSIGN = 14;
public static final int T_MINUS = 15;
public static final int T_MULASSIGN = 16;
public static final int T_MUL = 17;
public static final int T_DIVASSIGN = 18;
public static final int T_DIV = 19;
public static final int T_MODASSIGN = 20;
public static final int T_MOD = 21;
public static final int T_ORASSIGN = 22;
public static final int T_OR = 23;
public static final int T_OR2 = 40;
public static final int T_ANDASSIGN = 24;
public static final int T_AND2 = 41;
public static final int T_AND = 25;
public static final int T_XORASSIGN = 26;
public static final int T_XOR = 27;
public static final int T_QUEST = 28;
public static final int T_COLON = 29;
public static final int T_BROPEN = 30;
public static final int T_BRCLOSE = 31;
public static final int T_TILDEASSIGN = 45;
public static final int T_TILDE = 32;
public static final int T_SEMICOLON = 33;
public static final int T_LTEQ = 34;
public static final int T_LT = 35;
public static final int T_GTEQ = 36;
public static final int T_GT = 37;
public static final int T_EQ = 38;
public static final int T_ASSIGN = 39;
public static final int T_NOTEQ = 42;
public static final int T_NOT = 43;
public static final int T_DOLLAR = 44;
public static final int T_ANY = 99;
public static final int T_BOOL = 100;
public static final int T_BYTE = 101;
public static final int T_SHORT = 102;
public static final int T_INT = 103;
public static final int T_LONG = 104;
public static final int T_FLOAT = 105;
public static final int T_DOUBLE = 106;
public static final int T_STRING = 107;
public static final int T_FUNCTION = 108;
public static final int T_IN = 109;
public static final int T_VOID = 110;
public static final int T_AS = 120;
public static final int T_VERSION = 121;
public static final int T_IF = 122;
public static final int T_ELSE = 123;
public static final int T_FOR = 124;
public static final int T_RETURN = 125;
public static final int T_VAR = 126;
public static final int T_VAL = 127;
public static final int T_NULL = 140;
public static final int T_TRUE = 141;
public static final int T_FALSE = 142;
public static final int T_IMPORT = 160;
private static final String[] REGEXPS = {
"#[^\n]*[\n\\e]",
"//[^\n]*[\n\\e]",
"/\\*[^\\*]*(\\*|\\*[^/\\*][^\\*]*)*\\*/",
"[ \t\r\n]*",
"[a-zA-Z_][a-zA-Z_0-9]*",
"\\-?(0|[1-9][0-9]*)\\.[0-9]+([eE][\\+\\-]?[0-9]+)?",
"\\-?(0|[1-9][0-9]*)",
"\"([^\"\\\\]|\\\\([\'\"\\\\/bfnrt]|u[0-9a-fA-F]{4}))*\"",
"\'([^\'\\\\]|\\\\([\'\"\\\\/bfnrt]|u[0-9a-fA-F]{4}))*\'",
"\\{",
"\\}",
"\\[",
"\\]",
"\\.\\.",
"\\.",
",",
"\\+=",
"\\+",
"\\-=",
"\\-",
"\\*=",
"\\*",
"/=",
"/",
"%=",
"%",
"\\|=",
"\\|\\|",
"\\|",
"&=",
"&&",
"&",
"\\^=",
"\\^",
"\\?",
":",
"\\(",
"\\)",
"~=",
"~",
";",
"<=",
"<",
">=",
">",
"==",
"=",
"!=",
"!",
"$"
};
private static final int[] FINALS = {
-1,
-1,
-1,
-1,
T_ID,
T_FLOATVALUE,
T_INTVALUE,
T_STRINGVALUE,
T_STRINGVALUE,
T_AOPEN,
T_ACLOSE,
T_SQBROPEN,
T_SQBRCLOSE,
T_DOT2,
T_DOT,
T_COMMA,
T_PLUSASSIGN,
T_PLUS,
T_MINUSASSIGN,
T_MINUS,
T_MULASSIGN,
T_MUL,
T_DIVASSIGN,
T_DIV,
T_MODASSIGN,
T_MOD,
T_ORASSIGN,
T_OR2,
T_OR,
T_ANDASSIGN,
T_AND2,
T_AND,
T_XORASSIGN,
T_XOR,
T_QUEST,
T_COLON,
T_BROPEN,
T_BRCLOSE,
T_TILDEASSIGN,
T_TILDE,
T_SEMICOLON,
T_LTEQ,
T_LT,
T_GTEQ,
T_GT,
T_EQ,
T_ASSIGN,
T_NOTEQ,
T_NOT,
T_DOLLAR
};
private static final CompiledDFA DFA = new NFA(REGEXPS, FINALS).toDFA().optimize().compile();
static {
KEYWORDS = new HashMap<String, Integer>();
KEYWORDS.put("any", T_ANY);
KEYWORDS.put("bool", T_BOOL);
KEYWORDS.put("byte", T_BYTE);
KEYWORDS.put("short", T_SHORT);
KEYWORDS.put("int", T_INT);
KEYWORDS.put("long", T_LONG);
KEYWORDS.put("float", T_FLOAT);
KEYWORDS.put("double", T_DOUBLE);
KEYWORDS.put("string", T_STRING);
KEYWORDS.put("function", T_FUNCTION);
KEYWORDS.put("in", T_IN);
KEYWORDS.put("void", T_VOID);
KEYWORDS.put("as", T_AS);
KEYWORDS.put("version", T_VERSION);
KEYWORDS.put("if", T_IF);
KEYWORDS.put("else", T_ELSE);
KEYWORDS.put("for", T_FOR);
KEYWORDS.put("return", T_RETURN);
KEYWORDS.put("var", T_VAR);
KEYWORDS.put("val", T_VAL);
KEYWORDS.put("null", T_NULL);
KEYWORDS.put("true", T_TRUE);
KEYWORDS.put("false", T_FALSE);
KEYWORDS.put("import", T_IMPORT);
}
private final IZenCompileEnvironment environment;
/**
* Constructs a tokener from the given reader.
*
* @param contents file reader
* @param environment compile environment
* @throws IOException if the file could not be read properly
*/
public ZenTokener(Reader contents, IZenCompileEnvironment environment) throws IOException {
super(contents, DFA);
this.environment = environment;
}
/**
* Constructs a tokener from the given string.
*
* @param contents content string
* @param environment compile environment
* @throws IOException shouldn't happen
*/
public ZenTokener(String contents, IZenCompileEnvironment environment) throws IOException {
super(new StringReader(contents), DFA);
this.environment = environment;
}
/**
* Retrieves the compile environment of this tokener.
*
* @return compile environment
*/
public IZenCompileEnvironment getEnvironment() {
return environment;
}
// ##################################
// ### TokenStream implementation ###
// ##################################
@Override
public Token process(Token token) {
if (token.getType() == T_ID) {
if (KEYWORDS.containsKey(token.getValue())) {
return new Token(token.getValue(), KEYWORDS.get(token.getValue()), token.getPosition());
}
}
return token;
}
}