From aae628cd18729dafd63698c0e41cdf9ce976a90b Mon Sep 17 00:00:00 2001 From: Marc Hermans Date: Wed, 12 Oct 2022 21:00:25 +0200 Subject: [PATCH] Fix performance issue and logging when resource caching is enabled (#9029) --- .../minecraftforge/fml/ModLoadingContext.java | 19 ++ fmlloader/src/main/resources/log4j2.xml | 1 + .../packs/VanillaPackResources.java.patch | 2 +- .../FallbackResourceManager.java.patch | 12 - src/fmllauncher/resources/log4j2.xml | 1 + .../minecraftforge/common/ForgeConfig.java | 11 +- .../resource/PathPackResources.java | 3 +- .../resource/ResourceCacheManager.java | 257 +++++++++++++++--- 8 files changed, 242 insertions(+), 64 deletions(-) delete mode 100644 patches/minecraft/net/minecraft/server/packs/resources/FallbackResourceManager.java.patch diff --git a/fmlcore/src/main/java/net/minecraftforge/fml/ModLoadingContext.java b/fmlcore/src/main/java/net/minecraftforge/fml/ModLoadingContext.java index 0d36c1b308..e549437fda 100644 --- a/fmlcore/src/main/java/net/minecraftforge/fml/ModLoadingContext.java +++ b/fmlcore/src/main/java/net/minecraftforge/fml/ModLoadingContext.java @@ -5,13 +5,18 @@ package net.minecraftforge.fml; +import com.mojang.logging.LogUtils; import net.minecraftforge.fml.config.IConfigSpec; import net.minecraftforge.fml.config.ModConfig; +import net.minecraftforge.fml.loading.FMLLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.function.Supplier; public class ModLoadingContext { + private static final Logger LOGGER = LogUtils.getLogger(); private static final ThreadLocal context = ThreadLocal.withInitial(ModLoadingContext::new); private Object languageExtension; private ModLoadingStage stage; @@ -46,10 +51,24 @@ public class ModLoadingContext } public void registerConfig(ModConfig.Type type, IConfigSpec spec) { + if (spec.isEmpty()) + { + // This handles the case where a mod tries to register a config, without any options configured inside it. + LOGGER.debug("Attempted to register an empty config for type {} on mod {}", type, getActiveContainer().getModId()); + return; + } + getActiveContainer().addConfig(new ModConfig(type, spec, getActiveContainer())); } public void registerConfig(ModConfig.Type type, IConfigSpec spec, String fileName) { + if (spec.isEmpty()) + { + // This handles the case where a mod tries to register a config, without any options configured inside it. + LOGGER.debug("Attempted to register an empty config for type {} on mod {} using file name {}", type, getActiveContainer().getModId(), fileName); + return; + } + getActiveContainer().addConfig(new ModConfig(type, spec, getActiveContainer(), fileName)); } diff --git a/fmlloader/src/main/resources/log4j2.xml b/fmlloader/src/main/resources/log4j2.xml index 7e3ae72954..6276295182 100644 --- a/fmlloader/src/main/resources/log4j2.xml +++ b/fmlloader/src/main/resources/log4j2.xml @@ -13,6 +13,7 @@ + diff --git a/patches/minecraft/net/minecraft/server/packs/VanillaPackResources.java.patch b/patches/minecraft/net/minecraft/server/packs/VanillaPackResources.java.patch index 6c3a11bc93..cd64436086 100644 --- a/patches/minecraft/net/minecraft/server/packs/VanillaPackResources.java.patch +++ b/patches/minecraft/net/minecraft/server/packs/VanillaPackResources.java.patch @@ -54,7 +54,7 @@ }; + } + -+ private final net.minecraftforge.resource.ResourceCacheManager cacheManager = new net.minecraftforge.resource.ResourceCacheManager(false, net.minecraftforge.common.ForgeConfig.COMMON.indexVanillaPackCachesOnThread, (packType, namespace) -> f_182296_.get(packType).resolve(namespace)); ++ private final net.minecraftforge.resource.ResourceCacheManager cacheManager = new net.minecraftforge.resource.ResourceCacheManager(false, "indexVanillaPackCachesOnThread", (packType, namespace) -> f_182296_.get(packType).resolve(namespace)); + + @Override + public void initForNamespace(final String nameSpace) { diff --git a/patches/minecraft/net/minecraft/server/packs/resources/FallbackResourceManager.java.patch b/patches/minecraft/net/minecraft/server/packs/resources/FallbackResourceManager.java.patch deleted file mode 100644 index 71f7578d6a..0000000000 --- a/patches/minecraft/net/minecraft/server/packs/resources/FallbackResourceManager.java.patch +++ /dev/null @@ -1,12 +0,0 @@ ---- a/net/minecraft/server/packs/resources/FallbackResourceManager.java -+++ b/net/minecraft/server/packs/resources/FallbackResourceManager.java -@@ -53,6 +_,9 @@ - - private void m_215395_(String p_215396_, @Nullable PackResources p_215397_, @Nullable Predicate p_215398_) { - this.f_10599_.add(new FallbackResourceManager.PackEntry(p_215396_, p_215397_, p_215398_)); -+ if (p_215397_ != null) { -+ p_215397_.initForNamespace(this.f_10602_); -+ } - } - - public Set m_7187_() { diff --git a/src/fmllauncher/resources/log4j2.xml b/src/fmllauncher/resources/log4j2.xml index 0e434457e9..d7ef415f85 100644 --- a/src/fmllauncher/resources/log4j2.xml +++ b/src/fmllauncher/resources/log4j2.xml @@ -13,6 +13,7 @@ + diff --git a/src/main/java/net/minecraftforge/common/ForgeConfig.java b/src/main/java/net/minecraftforge/common/ForgeConfig.java index f350c27756..2f4440010e 100644 --- a/src/main/java/net/minecraftforge/common/ForgeConfig.java +++ b/src/main/java/net/minecraftforge/common/ForgeConfig.java @@ -83,28 +83,31 @@ public class ForgeConfig { * General configuration that doesn't need to be synchronized but needs to be available before server startup */ public static class Common { + @Deprecated(since = "1.19.2", forRemoval = true) public final BooleanValue cachePackAccess; + @Deprecated(since = "1.19.2", forRemoval = true) public final BooleanValue indexVanillaPackCachesOnThread; + @Deprecated(since = "1.19.2", forRemoval = true) public final BooleanValue indexModPackCachesOnThread; Common(ForgeConfigSpec.Builder builder) { - builder.comment("General configuration settings") + builder.comment("[DEPRECATED / NO EFFECT]: General configuration settings") .push("general"); cachePackAccess = builder - .comment("Set this to true to cache resource listings in resource and data packs") + .comment("[DEPRECATED / NO EFFECT] [NOW IN RESOURCE-CACHING CONFIG]: Set this to true to cache resource listings in resource and data packs") .translation("forge.configgui.cachePackAccess") .worldRestart() .define("cachePackAccess", true); indexVanillaPackCachesOnThread = builder - .comment("Set this to true to index vanilla resource and data packs on thread") + .comment("[DEPRECATED / NO EFFECT] [NOW IN RESOURCE-CACHING CONFIG]: Set this to true to index vanilla resource and data packs on thread") .translation("forge.configgui.indexVanillaPackCachesOnThread") .worldRestart() .define("indexVanillaPackCachesOnThread", false); indexModPackCachesOnThread = builder - .comment("Set this to true to index mod resource and data packs on thread") + .comment("[DEPRECATED / NO EFFECT] [NOW IN RESOURCE-CACHING CONFIG]: Set this to true to index mod resource and data packs on thread") .translation("forge.configgui.indexModPackCachesOnThread") .worldRestart() .define("indexModPackCachesOnThread", false); diff --git a/src/main/java/net/minecraftforge/resource/PathPackResources.java b/src/main/java/net/minecraftforge/resource/PathPackResources.java index 7e9980d50f..2817ef2efb 100644 --- a/src/main/java/net/minecraftforge/resource/PathPackResources.java +++ b/src/main/java/net/minecraftforge/resource/PathPackResources.java @@ -9,7 +9,6 @@ import com.google.common.base.Joiner; import net.minecraft.resources.ResourceLocation; import net.minecraft.server.packs.AbstractPackResources; import net.minecraft.server.packs.PackType; -import net.minecraftforge.common.ForgeConfig; import org.jetbrains.annotations.NotNull; import java.io.File; @@ -38,7 +37,7 @@ public class PathPackResources extends AbstractPackResources private final Path source; private final String packName; - private final ResourceCacheManager cacheManager = new ResourceCacheManager(true, ForgeConfig.COMMON.indexModPackCachesOnThread, (packType, namespace) -> resolve(packType.getDirectory(), namespace).toAbsolutePath()); + private final ResourceCacheManager cacheManager = new ResourceCacheManager(true, "indexModPackCachesOnThread", (packType, namespace) -> resolve(packType.getDirectory(), namespace).toAbsolutePath()); /** * Constructs a java.nio.Path-based resource pack. diff --git a/src/main/java/net/minecraftforge/resource/ResourceCacheManager.java b/src/main/java/net/minecraftforge/resource/ResourceCacheManager.java index 0a8d4cfe95..5d4d70f38e 100644 --- a/src/main/java/net/minecraftforge/resource/ResourceCacheManager.java +++ b/src/main/java/net/minecraftforge/resource/ResourceCacheManager.java @@ -5,21 +5,32 @@ package net.minecraftforge.resource; +import com.electronwill.nightconfig.core.ConfigFormat; +import com.electronwill.nightconfig.core.ConfigSpec; +import com.electronwill.nightconfig.core.file.CommentedFileConfig; +import com.electronwill.nightconfig.core.io.ParsingException; +import com.electronwill.nightconfig.core.io.WritingMode; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterators; import com.google.common.collect.Maps; import com.mojang.logging.LogUtils; import net.minecraft.Util; import net.minecraft.resources.ResourceLocation; import net.minecraft.server.packs.PackType; -import net.minecraftforge.common.ForgeConfig; import net.minecraftforge.common.ForgeConfigSpec; +import net.minecraftforge.fml.loading.FMLPaths; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; import java.io.IOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -31,7 +42,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -40,23 +50,30 @@ import java.util.stream.Stream; *

* This class handles caching the resource listing calls on a pack, pack type and namespace level. */ +@ApiStatus.Internal public class ResourceCacheManager { + /** + * Logging marker to handle the hiding of the entries from the debug file. + */ + private static final Marker RESOURCE_CACHE = MarkerFactory.getMarker("RESOURCE-CACHE"); /** * Indicates if the underlying namespaced managers need to support reloading. * Disabled for the (in-jar / downloaded) default vanilla pack. */ private final boolean supportsReloading; + /** - * Indicates if the indexing of the file-tree should happen off-thread or on-thread and block the process accordingly. + * The config key which can be used to check if the indexing of the file-tree should happen off-thread or on-thread and block the process accordingly. */ - private final ForgeConfigSpec.BooleanValue indexOffThreadConfigOption; + private final String indexOnThreadConfigurationKey; /** * The path builder (different users have different requirements for how they want to handle this) */ private final BiFunction pathBuilder; + /** * The individual sub managers by pack type and namespace */ @@ -69,35 +86,50 @@ public class ResourceCacheManager * @param indexOffThreadConfig True to index the file-tree off-thread. * @param pathBuilder The path builder to use. */ + @Deprecated(since = "1.19.2", forRemoval = true) public ResourceCacheManager(final boolean supportsReloading, final ForgeConfigSpec.BooleanValue indexOffThreadConfig, final BiFunction pathBuilder) { this.supportsReloading = supportsReloading; - this.indexOffThreadConfigOption = indexOffThreadConfig; + this.indexOnThreadConfigurationKey = Iterators.getLast(indexOffThreadConfig.getPath().iterator()); this.pathBuilder = pathBuilder; } - public boolean shouldIndexOffThread() + public ResourceCacheManager(final boolean supportsReloading, final String indexOnThreadConfigurationKey, final BiFunction pathBuilder) { - return !getConfigValue(this.indexOffThreadConfigOption); + this.supportsReloading = supportsReloading; + this.indexOnThreadConfigurationKey = indexOnThreadConfigurationKey; + this.pathBuilder = pathBuilder; } + /** + * Indicates if the caching system is enabled or not. + * + * @return {@code true} if the caching system is enabled, {@code false} otherwise. + */ public static boolean shouldUseCache() { - return getConfigValue(ForgeConfig.COMMON.cachePackAccess); + return ResourceManagerBootCacheConfigurationHandler.getInstance().getConfigValue("cachePackAccess", true); } - private static boolean getConfigValue(Supplier configValue) + /** + * Indicates if this cache manager requires the indexing of the file-tree to happen on the main thread. + * + * @return {@code true} if the indexing of the file-tree should happen on the main thread, {@code false} otherwise. + */ + public boolean shouldIndexOnThread() { - // Yes we catch the early loading error on purpose. - // Both the indexing off-thread and if caching is enabled are configurable, so we need to catch the error. - // So when they are loaded early we preserve the default behaviour which is to not cache or index on thread. - try - { - return configValue.get(); - } catch (IllegalStateException e) - { - return false; - } + return ResourceManagerBootCacheConfigurationHandler.getInstance().getConfigValue(this.indexOnThreadConfigurationKey, false); + } + + /** + * Indicates if this cache manager requires the indexing of the file-tree to happen off the main thread. + * + * @return {@code true} if the indexing of the file-tree should happen off the main thread, {@code true} otherwise. + */ + @Deprecated(forRemoval = true, since = "1.19.2") + public boolean shouldIndexOffThread() + { + return !this.shouldIndexOnThread(); } /** @@ -110,13 +142,13 @@ public class ResourceCacheManager for (final PackType packType : PackType.values()) { final PackTypeAndNamespace key = new PackTypeAndNamespace(packType, namespace); // Make a key. - if (managersByNamespace.containsKey(key) && !supportsReloading) - { // If we do not support reloading we just skip this. + if (managersByNamespace.containsKey(key) && !supportsReloading) // If we do not support reloading we just skip this. + { return; } // Create a new manager, overriding the previous one if it exists. - final NamespacedResourceCacheManager newManager = new NamespacedResourceCacheManager(packType, namespace, this.shouldIndexOffThread(), pathBuilder, this::createWalkingStream); + final NamespacedResourceCacheManager newManager = new NamespacedResourceCacheManager(packType, namespace, this.shouldIndexOnThread(), pathBuilder, this::createWalkingStream); this.managersByNamespace.put(key, newManager); // Index the inner manager (which will for sure exist, and will know the pack type and namespace already) @@ -233,9 +265,9 @@ public class ResourceCacheManager */ private final String namespace; /** - * Indicates if the indexing should run off-thread. + * Indicates if the indexing should run on-thread. */ - private final boolean indexOffThread; + private final boolean indexOnThread; /** * The path builder to use. */ @@ -257,17 +289,17 @@ public class ResourceCacheManager /** * Creates a new namespaced resource cache manager. * - * @param packType The pack type that this manager handles. - * @param namespace The namespace that this manager handles. - * @param indexOffThread True to enable indexing off-thread. - * @param pathBuilder The path builder to use. - * @param pathFinder The path walker stream factory to use. + * @param packType The pack type that this manager handles. + * @param namespace The namespace that this manager handles. + * @param indexOnThread True to enable indexing on-thread. + * @param pathBuilder The path builder to use. + * @param pathFinder The path walker stream factory to use. */ - private NamespacedResourceCacheManager(final PackType packType, final String namespace, final boolean indexOffThread, BiFunction pathBuilder, final PathWalkerFactory pathFinder) + private NamespacedResourceCacheManager(final PackType packType, final String namespace, final boolean indexOnThread, BiFunction pathBuilder, final PathWalkerFactory pathFinder) { this.packType = packType; this.namespace = namespace; - this.indexOffThread = indexOffThread; + this.indexOnThread = indexOnThread; this.pathBuilder = pathBuilder; this.pathFinder = pathFinder; } @@ -277,17 +309,18 @@ public class ResourceCacheManager */ public void index() { - if (indexOffThread) + if (indexOnThread) + { + // Run on-thread + doIndex(); + } + else { // Run off-thread. CompletableFuture.runAsync( this::doIndex, Util.backgroundExecutor() ); - } else - { - // Run on-thread - doIndex(); } } @@ -297,6 +330,8 @@ public class ResourceCacheManager */ private void doIndex() { + LOGGER.debug(RESOURCE_CACHE, "Indexing resources for pack type {} and namespace {}, on thread: {}", packType, namespace, Thread.currentThread().getName()); + // Get the path to the root of the namespace in the current pack. final Path rootPath = pathBuilder.apply(packType, namespace); @@ -314,16 +349,20 @@ public class ResourceCacheManager .filter(path -> ResourceLocation.isValidPath(path.locationPath())) // Only process valid paths .map(path -> new ResourceCacheEntry(packType, namespace, path.path(), new ResourceLocation(namespace, path.locationPath()))) // Create a cache entry. .forEach(this::injectIntoCache); // Inject the entry into the cache. - } catch (NoSuchFileException noSuchFileException) + } + catch (NoSuchFileException noSuchFileException) { - LOGGER.debug("Failed to cache resources, the directory does not exist!", noSuchFileException); - } catch (IOException ioException) + LOGGER.debug(RESOURCE_CACHE, "Failed to cache resources, the directory does not exist!", noSuchFileException); + } + catch (IOException ioException) { - LOGGER.error("Failed to cache resources, some stuff might be missing!", ioException); - } catch (Exception exception) + LOGGER.error(RESOURCE_CACHE, "Failed to cache resources, some stuff might be missing!", ioException); + } + catch (Exception exception) { - LOGGER.error("Failed to cache resources, some stuff might be missing! Unknown exception!", exception); - } finally + LOGGER.error(RESOURCE_CACHE, "Failed to cache resources, some stuff might be missing! Unknown exception!", exception); + } + finally { cacheLoaded.set(true); } @@ -353,7 +392,8 @@ public class ResourceCacheManager { // We have null, generally this only happens on the root directory. pathEntry = ""; - } else + } + else { // Use the parent paths, we can not depend on a toString call here, since we need the / as separator. pathEntry = Joiner.on("/").join(parentPath); @@ -379,7 +419,8 @@ public class ResourceCacheManager public Collection getResources(final Path inputPath, final Predicate filter) { // We only have a cache once the atomic flag has been set. - if (!cacheLoaded()) { + if (!cacheLoaded()) + { // We need to return a mutable object here because the callers actually mutate the collection. return new ArrayList<>(); } @@ -413,6 +454,132 @@ public class ResourceCacheManager @FunctionalInterface private interface PathWalkerFactory { + /** + * Create a new walkable stream of paths. + * The stream will be closed by the caller. + * + * @param path The path to create the stream for. + * @return A new stream of paths that are children (potentially several generations deep) of the given path. + * @throws IOException If the stream can not be created. + */ Stream createWalkingStream(Path path) throws IOException; } + + /** + * Class to handle the reading, initial creation, and watching of the boot configuration file, for the resource cache manager. + */ + private static final class ResourceManagerBootCacheConfigurationHandler + { + /** + * The logger for the {@link ResourceManagerBootCacheConfigurationHandler} + */ + private static final Logger LOGGER = LogUtils.getLogger(); + + /** + * The path to the boot configuration file. + */ + private static final Path CONFIG_PATH = FMLPaths.CONFIGDIR.get().resolve("forge-resource-caching.toml"); + + /** + * The configuration specification for the resource cache configuration. + */ + private static final ConfigSpec configSpec = new ConfigSpec(); + static { + configSpec.define("cacheResources", Boolean.TRUE); + configSpec.define("indexVanillaPackCachesOnThread", Boolean.FALSE); + configSpec.define("indexModPackCachesOnThread", Boolean.FALSE); + } + + /** + * The current instance of the handler. + * @implNote This needs to be the last static field (or at least below the config spec and the config path, else some of them create NPEs because they are not initialized yet) + */ + private static final ResourceManagerBootCacheConfigurationHandler INSTANCE = new ResourceManagerBootCacheConfigurationHandler(); + + /** + * The configuration handle for the resource cache configuration. + */ + private final CommentedFileConfig configurationHandle; + + /** + * Creates a new instance of the handler. + * Registers the watchdog thread. + */ + private ResourceManagerBootCacheConfigurationHandler() + { + this.configurationHandle = createConfiguration(); + } + + /** + * Creates the configuration handle for the resource cache configuration. + */ + private static CommentedFileConfig createConfiguration() + { + final CommentedFileConfig configData = CommentedFileConfig.builder(CONFIG_PATH).sync() + .onFileNotFound(ResourceManagerBootCacheConfigurationHandler::onConfigFileNotFound) + .autosave() + .autoreload() + .concurrent() + .writingMode(WritingMode.REPLACE) + .build(); + try + { + configData.load(); + } + catch (ParsingException e) + { + throw new RuntimeException("Failed to load Force Resource Cache Configuration from %s".formatted(CONFIG_PATH), e); + } + if (!configSpec.isCorrect(configData)) { + LOGGER.warn("Configuration file {} is not correct. Correcting", CONFIG_PATH); + configSpec.correct(configData, (action, path, incorrectValue, correctedValue) -> + LOGGER.warn("Incorrect key {} was corrected from {} to {}", path, incorrectValue, correctedValue)); + } + + configData.save(); + + return configData; + } + + private static boolean onConfigFileNotFound(Path file, ConfigFormat configFormat) throws IOException { + Files.write(file, ImmutableList.of( + "# This TOML configuration file controls the resource caching system which is used before the mod loading environment starts.", + "# This file is read by the Forge boot loader, and is not used by the game itself.", + "#", + "# Set this to false to disable the resource cache. This will cause the game to scan the resource packs everytime it needs a list of resources.", + "cacheResources=true", + "", + "# Set this to true to force the caching of vanilla resources to happen on the main thread.", + "indexVanillaPackCachesOnThread=false", + "", + "# Set this to true to force the caching of mod resources to happen on the main thread.", + "indexModPackCachesOnThread=false" + ), + StandardOpenOption.CREATE_NEW); + + return true; + } + + /** + * Gives access to the current singleton instance of the handler. + * + * @return The instance of this handler. + */ + public static ResourceManagerBootCacheConfigurationHandler getInstance() + { + return INSTANCE; + } + + /** + * Reads a config value from the current configuration file. + * + * @param configKey The key of the config value to read. + * @param defaultValue The default value to return if the config value is not present. + * @return The value of the config value, or the default value if it is not present. + */ + private boolean getConfigValue(final String configKey, final boolean defaultValue) + { + return this.configurationHandle.getOptional(configKey).orElse(defaultValue); + } + } }