From 816d33de28d10983881b9322e6c6cf4a448ae3b6 Mon Sep 17 00:00:00 2001 From: LexManos Date: Wed, 7 Feb 2018 00:43:32 -0800 Subject: [PATCH] Initial implementation of JSON based annotation scanning. Disabled by default for now, until proven to be fully function. Enable using -Dfml.enableJsonAnnotations=true --- .../fml/common/discovery/JarDiscoverer.java | 128 +++++++++---- .../fml/common/discovery/ModDiscoverer.java | 1 + .../common/discovery/asm/ModAnnotation.java | 2 +- .../fml/common/discovery/json/ASMInfo.java | 165 +++++++++++++++++ .../discovery/json/JsonAnnotationLoader.java | 77 ++++++++ .../test/TestAnnotationParser.java | 172 ++++++++++++++++++ 6 files changed, 505 insertions(+), 40 deletions(-) create mode 100644 src/main/java/net/minecraftforge/fml/common/discovery/json/ASMInfo.java create mode 100644 src/main/java/net/minecraftforge/fml/common/discovery/json/JsonAnnotationLoader.java create mode 100644 src/test/java/net/minecraftforge/test/TestAnnotationParser.java diff --git a/src/main/java/net/minecraftforge/fml/common/discovery/JarDiscoverer.java b/src/main/java/net/minecraftforge/fml/common/discovery/JarDiscoverer.java index 4fd756ebb2..b825e960d9 100644 --- a/src/main/java/net/minecraftforge/fml/common/discovery/JarDiscoverer.java +++ b/src/main/java/net/minecraftforge/fml/common/discovery/JarDiscoverer.java @@ -19,9 +19,12 @@ package net.minecraftforge.fml.common.discovery; +import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Constructor; import java.util.Collections; import java.util.List; +import java.util.Map.Entry; import java.util.jar.JarFile; import net.minecraftforge.fml.common.FMLLog; @@ -29,18 +32,22 @@ import net.minecraftforge.fml.common.LoaderException; import net.minecraftforge.fml.common.MetadataCollection; import net.minecraftforge.fml.common.ModContainer; import net.minecraftforge.fml.common.ModContainerFactory; +import net.minecraftforge.fml.common.discovery.ASMDataTable.ASMData; import net.minecraftforge.fml.common.discovery.asm.ASMModParser; - -import org.apache.commons.io.IOUtils; -import org.apache.logging.log4j.Level; +import net.minecraftforge.fml.common.discovery.json.JsonAnnotationLoader; import java.util.regex.Matcher; import java.util.zip.ZipEntry; +import org.objectweb.asm.Type; + import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; public class JarDiscoverer implements ITypeDiscoverer { + private static final boolean ENABLE_JSON_TEST = "true".equals(System.getProperty("fml.enableJsonAnnotations", "false")); + @Override public List discover(ModCandidate candidate, ASMDataTable table) { @@ -63,42 +70,11 @@ public class JarDiscoverer implements ITypeDiscoverer FMLLog.log.debug("The mod container {} appears to be missing an mcmod.info file", candidate.getModContainer().getName()); mc = MetadataCollection.from(null, ""); } - for (ZipEntry ze : Collections.list(jar.entries())) - { - if (ze.getName()!=null && ze.getName().startsWith("__MACOSX")) - { - continue; - } - Matcher match = classFile.matcher(ze.getName()); - if (match.matches()) - { - ASMModParser modParser; - try - { - try (InputStream inputStream = jar.getInputStream(ze)) - { - modParser = new ASMModParser(inputStream); - } - candidate.addClassEntry(ze.getName()); - } - catch (LoaderException e) - { - FMLLog.log.error("There was a problem reading the entry {} in the jar {} - probably a corrupt zip", candidate.getModContainer().getPath(), e); - jar.close(); - throw e; - } - modParser.validate(); - modParser.sendToTable(table, candidate); - ModContainer container = ModContainerFactory.instance().build(modParser, candidate.getModContainer(), candidate); - if (container!=null) - { - table.addContainer(container); - foundMods.add(container); - container.bindMetadata(mc); - container.setClassVersion(modParser.getClassVersion()); - } - } - } + + if (ENABLE_JSON_TEST && jar.getEntry(JsonAnnotationLoader.ANNOTATION_JSON) != null) + findClassesJSON(candidate, table, jar, foundMods, mc); + else + findClassesASM(candidate, table, jar, foundMods, mc); } catch (Exception e) { @@ -107,4 +83,78 @@ public class JarDiscoverer implements ITypeDiscoverer return foundMods; } + private void findClassesASM(ModCandidate candidate, ASMDataTable table, JarFile jar, List foundMods, MetadataCollection mc) throws IOException + { + for (ZipEntry ze : Collections.list(jar.entries())) + { + if (ze.getName()!=null && ze.getName().startsWith("__MACOSX")) + { + continue; + } + Matcher match = classFile.matcher(ze.getName()); + if (match.matches()) + { + ASMModParser modParser; + try + { + try (InputStream inputStream = jar.getInputStream(ze)) + { + modParser = new ASMModParser(inputStream); + } + candidate.addClassEntry(ze.getName()); + } + catch (LoaderException e) + { + FMLLog.log.error("There was a problem reading the entry {} in the jar {} - probably a corrupt zip", candidate.getModContainer().getPath(), e); + jar.close(); + throw e; + } + modParser.validate(); + modParser.sendToTable(table, candidate); + ModContainer container = ModContainerFactory.instance().build(modParser, candidate.getModContainer(), candidate); + if (container!=null) + { + table.addContainer(container); + foundMods.add(container); + container.bindMetadata(mc); + container.setClassVersion(modParser.getClassVersion()); + } + } + } + } + + private void findClassesJSON(ModCandidate candidate, ASMDataTable table, JarFile jar, List foundMods, MetadataCollection mc) throws IOException + { + FMLLog.log.info("Loading jar {} annotation data from json", candidate.getModContainer().getPath()); + ZipEntry json = jar.getEntry(JsonAnnotationLoader.ANNOTATION_JSON); + Multimap annos = JsonAnnotationLoader.loadJson(jar.getInputStream(json), candidate, table); + + for (Entry> entry : ModContainerFactory.modTypes.entrySet()) + { + Type type = entry.getKey(); + Constructor ctr = entry.getValue(); + + for (ASMData data : annos.get(type.getClassName())) + { + FMLLog.log.debug("Identified a mod of type {} ({}) - loading", type.getClassName(), data.getClassName()); + try + { + ModContainer ret = ctr.newInstance(data.getClassName(), candidate, data.getAnnotationInfo()); + if (!ret.shouldLoadInEnvironment()) + FMLLog.log.debug("Skipping mod {}, container opted to not load.", data.getClassName()); + else + { + table.addContainer(ret); + foundMods.add(ret); + ret.bindMetadata(mc); + //ret.setClassVersion(classVersion); // Not really needed anymore as we're forcing J8. Maybe think of reinstating for J9 support? After LaunchWraper re-do. + } + } + catch (Exception e) + { + FMLLog.log.error("Unable to construct {} container", data.getClassName(), e); + } + } + } + } } diff --git a/src/main/java/net/minecraftforge/fml/common/discovery/ModDiscoverer.java b/src/main/java/net/minecraftforge/fml/common/discovery/ModDiscoverer.java index 4c18d8bfa6..0720f3b188 100644 --- a/src/main/java/net/minecraftforge/fml/common/discovery/ModDiscoverer.java +++ b/src/main/java/net/minecraftforge/fml/common/discovery/ModDiscoverer.java @@ -107,6 +107,7 @@ public class ModDiscoverer } else if (modFile.isDirectory()) { + //TODO Remove in 1.13+ Mods should never be directory based anymore. FMLLog.log.debug("Found a candidate mod directory {}", modFile.getName()); addCandidate(new ModCandidate(modFile, modFile, ContainerType.DIR)); } diff --git a/src/main/java/net/minecraftforge/fml/common/discovery/asm/ModAnnotation.java b/src/main/java/net/minecraftforge/fml/common/discovery/asm/ModAnnotation.java index 1279368de9..5bbafc89ed 100644 --- a/src/main/java/net/minecraftforge/fml/common/discovery/asm/ModAnnotation.java +++ b/src/main/java/net/minecraftforge/fml/common/discovery/asm/ModAnnotation.java @@ -32,7 +32,7 @@ import com.google.common.collect.Maps; public class ModAnnotation { - public class EnumHolder + public static class EnumHolder { private final String desc; private final String value; diff --git a/src/main/java/net/minecraftforge/fml/common/discovery/json/ASMInfo.java b/src/main/java/net/minecraftforge/fml/common/discovery/json/ASMInfo.java new file mode 100644 index 0000000000..bd8019a6be --- /dev/null +++ b/src/main/java/net/minecraftforge/fml/common/discovery/json/ASMInfo.java @@ -0,0 +1,165 @@ +/* + * Minecraft Forge + * Copyright (c) 2018. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation version 2.1 + * of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package net.minecraftforge.fml.common.discovery.json; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import org.apache.commons.lang3.Validate; +import org.objectweb.asm.Type; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import net.minecraftforge.fml.common.FMLLog; +import net.minecraftforge.fml.common.discovery.asm.ModAnnotation.EnumHolder; + +//Package private, modders shouldn't access this. Do it through ASMDataTable. +class ASMInfo +{ + String name; + String[] interfaces; + List annotations; + private Map byID; + + public Annotation getSubAnnotation(int id) + { + if (byID == null) + { + byID = Maps.newHashMap(); + annotations.forEach(a -> { if (a.id != null) byID.put(a.id, a); }); + } + return byID.get(id); + } + + public enum TargetType { CLASS, FIELD, METHOD, SUBTYPE }; + public enum ValueType + { + BOOL(Boolean::valueOf, v -> {boolean[] ret = new boolean[v.length]; for (int x = 0; x < v.length; x++) ret[x] = Boolean.parseBoolean(v[x]); return ret; }), + BYTE(Byte::valueOf, v -> {byte[] ret = new byte[v.length]; for (int x = 0; x < v.length; x++) ret[x] = Byte.parseByte(v[x]); return ret; }), + CHAR(x -> x.charAt(0), v -> {char[] ret = new char[v.length]; for (int x = 0; x < v.length; x++) ret[x] = v[x].charAt(0); return ret; }), + SHORT(Short::valueOf, v -> {short[] ret = new short[v.length]; for (int x = 0; x < v.length; x++) ret[x] = Short.parseShort(v[x]); return ret; }), + INT(Integer::valueOf, v -> {int[] ret = new int[v.length]; for (int x = 0; x < v.length; x++) ret[x] = Integer.parseInt(v[x]); return ret; }), + LONG(Long::valueOf, v -> {long[] ret = new long[v.length]; for (int x = 0; x < v.length; x++) ret[x] = Long.parseLong(v[x]); return ret; }), + FLOAT(Float::valueOf, v -> {float[] ret = new float[v.length]; for (int x = 0; x < v.length; x++) ret[x] = Float.parseFloat(v[x]); return ret; }), + DOUBLE(Double::valueOf, v -> {double[] ret = new double[v.length]; for (int x = 0; x < v.length; x++) ret[x] = Double.parseDouble(v[x]); return ret; }), + STRING(x -> x, x -> x), + CLASS(Type::getType, v -> {Type[] ret = new Type[v.length]; for (int x = 0; x < v.length; x++) ret[x] = Type.getType(v[x]); return ret; }), + ENUM(ValueType::getEnumHolder, v -> {EnumHolder[] ret = new EnumHolder[v.length]; for (int x = 0; x < v.length; x++) ret[x] = ValueType.getEnumHolder(v[x]); return ret; }), + ANNOTATION(null, null), + NULL(x -> null, x -> null); + + public final Function single; + public final Function array; + + private ValueType(Function single, Function array) + { + this.single = single; + this.array = array; + } + + private static EnumHolder getEnumHolder(String value) + { + int idx = value.lastIndexOf('/'); + if (idx <= 1) + throw new IllegalArgumentException("Can not create a EnumHolder for value: " + value); + return new EnumHolder(value.substring(0, idx - 1), value.substring(idx)); + } + }; + + static class Annotation + { + TargetType type; + String name; + String target; + Integer id; + ValueHolder value; + Map values; + private Map _values; + + public Map getValues(ASMInfo pool) + { + if (_values == null) + { + _values = Maps.newHashMap(); + if (values != null) + values.forEach((k, v) -> _values.put(k, v.get(pool))); + else + _values.put("value", value); + } + + return _values; + } + } + + static class ValueHolder + { + ValueType type; + String value; + String[] values; + + private Object _value; + + private ValueType getType() + { + return type == null ? ValueType.STRING : type; + } + + public Object get(ASMInfo pool) + { + if (_value == null) + { + if (values != null) + { + if (type == ValueType.ANNOTATION) + { + List> list = Lists.newArrayList(); + _value = list; + + for (String s : values) + { + Annotation sub = pool.getSubAnnotation(Integer.parseInt(s)); + if (sub == null) + FMLLog.log.error("Invalid Sub-Annotation in Annotation JSON: " + s); + else + list.add(sub.getValues(pool)); + } + } + else + _value = getType().array.apply(values); + } + else + { + if (type == ValueType.ANNOTATION) + { + Annotation sub = pool.getSubAnnotation(Integer.parseInt(value)); + if (sub == null) + FMLLog.log.error("Invalid Sub-Annotation in Annotation JSON: " + value); + else + _value = sub.getValues(pool); + } + else + _value = getType().single.apply(value); + } + } + return _value; + } + } +} diff --git a/src/main/java/net/minecraftforge/fml/common/discovery/json/JsonAnnotationLoader.java b/src/main/java/net/minecraftforge/fml/common/discovery/json/JsonAnnotationLoader.java new file mode 100644 index 0000000000..8af098ee49 --- /dev/null +++ b/src/main/java/net/minecraftforge/fml/common/discovery/json/JsonAnnotationLoader.java @@ -0,0 +1,77 @@ +/* + * Minecraft Forge + * Copyright (c) 2018. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation version 2.1 + * of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package net.minecraftforge.fml.common.discovery.json; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.util.Map; +import java.util.Map.Entry; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import net.minecraftforge.fml.common.discovery.ASMDataTable; +import net.minecraftforge.fml.common.discovery.ASMDataTable.ASMData; +import net.minecraftforge.fml.common.discovery.ModCandidate; +import net.minecraftforge.fml.common.discovery.json.ASMInfo.Annotation; + +public class JsonAnnotationLoader +{ + public static final String ANNOTATION_JSON = "META-INF/fml_cache_annotation.json"; + private static final Gson GSON = new GsonBuilder().create(); + private static final Type INFO_TABLE = new TypeToken>(){}.getType(); + + public static Multimap loadJson(InputStream data, ModCandidate candidate, ASMDataTable table) + { + Map map = GSON.fromJson(new InputStreamReader(data), INFO_TABLE); + Multimap ret = HashMultimap.create(); + + for (Entry entry : map.entrySet()) + { + //TODO: Java9 Multi-Release Jars, picking the correct class for the current platform. For now we just ignore them. + if (entry.getKey().startsWith("META-INF/")) + continue; + + ASMInfo asm_info = entry.getValue(); + if (asm_info.interfaces != null) + { + for (String type : asm_info.interfaces) + { + table.addASMData(candidate, type, asm_info.name, null, null); + ret.put(type, new ASMData(candidate, type, asm_info.name, null, null)); + } + } + + if (asm_info.annotations != null) + { + for (Annotation anno : asm_info.annotations) + { + table.addASMData(candidate, anno.name, asm_info.name, anno.target, anno.getValues(asm_info)); + ret.put(anno.name, new ASMData(candidate, anno.name, asm_info.name, anno.target, anno.getValues(asm_info))); + } + } + } + + return ret; + } +} diff --git a/src/test/java/net/minecraftforge/test/TestAnnotationParser.java b/src/test/java/net/minecraftforge/test/TestAnnotationParser.java new file mode 100644 index 0000000000..109364ad68 --- /dev/null +++ b/src/test/java/net/minecraftforge/test/TestAnnotationParser.java @@ -0,0 +1,172 @@ +/* + * Minecraft Forge + * Copyright (c) 2018. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation version 2.1 + * of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package net.minecraftforge.test; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Collections; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import org.junit.Test; + +import net.minecraftforge.fml.common.discovery.ASMDataTable; +import net.minecraftforge.fml.common.discovery.ContainerType; +import net.minecraftforge.fml.common.discovery.ModCandidate; +import net.minecraftforge.fml.common.discovery.asm.ASMModParser; +import net.minecraftforge.fml.common.discovery.json.JsonAnnotationLoader; + +public class TestAnnotationParser +{ + private static String TEST_JAR = "forestry_1.12.2-5.7.0.0.jar"; + public static Pattern classFile = Pattern.compile("[^\\s\\$]+(\\$[^\\s]+)?\\.class$"); + private static final int RUN_COUNT = 100; + + private File getFile() + { + ClassLoader cl = getClass().getClassLoader(); + URL url = cl.getResource(TEST_JAR); + return url == null ? null : new File(url.getFile()); + } + + @Test + public void testAnnotationLoaderASM() throws IOException + { + File jar = getFile(); + if (jar == null) + return; //Skip this test if the test jar doesn't exist. + + Timer timer = new Timer(); + for( int x = 0; x < RUN_COUNT; x++) + { + timer.start(); + loadAnnotationsASM(jar); + timer.end(null); + } + System.out.println("LoaderASM: " + timer.finish()); + } + + + @Test + public void testAnnotationLoaderJSON() throws IOException + { + File jar = getFile(); + if (jar == null) + return; //Skip this test if the test jar doesn't exist. + + Timer timer = new Timer(); + for( int x = 0; x < RUN_COUNT; x++) + { + timer.start(); + loadAnnotationsJSON(jar); + timer.end(null); + } + System.out.println("LoaderJSON: " + timer.finish()); + } + + private void loadAnnotationsASM(File jar) throws IOException + { + ASMDataTable dataTable = new ASMDataTable(); + ModCandidate candidate = new ModCandidate(jar, jar, ContainerType.JAR); + try (ZipFile in = new ZipFile(jar)) + { + for (ZipEntry e : Collections.list(in.entries())) + { + if (e.getName() != null && e.getName().startsWith("__MACOSX")) + continue; + + Matcher match = classFile.matcher(e.getName()); + if (match.matches()) + { + ASMModParser modParser; + try (InputStream inputStream = in.getInputStream(e)) + { + modParser = new ASMModParser(inputStream); + } + //candidate.addClassEntry(e.getName()); + if (modParser != null) + modParser.sendToTable(dataTable, candidate); + } + } + } + } + + private void loadAnnotationsJSON(File jar) throws IOException + { + ASMDataTable dataTable = new ASMDataTable(); + ModCandidate candidate = new ModCandidate(jar, jar, ContainerType.JAR); + try (ZipFile in = new ZipFile(jar)) + { + // We need to loop everything to gather a list of class files, but as we're not reading every entry we should be faster? + for (ZipEntry e : Collections.list(in.entries())) + { + if (e.getName() != null && e.getName().startsWith("__MACOSX")) + continue; + + Matcher match = classFile.matcher(e.getName()); + if (match.matches()) + { + //We check for classes, make this fancier and support multi-release jars? + //candidate.addClassEntry(e.getName()); + } + } + + InputStream json_input = in.getInputStream(in.getEntry(JsonAnnotationLoader.ANNOTATION_JSON)); + JsonAnnotationLoader.loadJson(json_input, candidate, dataTable); + } + } + + private static class Timer + { + private long start; + private long min = Long.MAX_VALUE; + private long max = Long.MIN_VALUE; + private long total = 0; + private int count = 0; + + public void start() + { + this.start = System.currentTimeMillis(); + } + public void end(String message) + { + long now = System.currentTimeMillis(); + long time = now - start; + if (message != null) + System.out.println(String.format(message, time)); + min = Long.min(min, time); + max = Long.max(max, time); + total += time; + count++; + } + + public String finish() + { + return "Runs: " + count + + " Min: " + min + + " Max: " + max + + " Total: " + total + + " Average: " + (total/count); + } + } +}