From b70d99489bddc5a53b1752f7b7a65d221370757b Mon Sep 17 00:00:00 2001 From: Marc Hermans Date: Sat, 9 Jul 2022 16:58:11 +0200 Subject: [PATCH] Cache resource listing calls in resource packs (#8829) * Make the resource lookups cached. * Include configurability and handle patch cleanup. * Document and comment the cache manager. * Make thread selection configurable. * Implement a configurable loading mechanic that falls back to default behaviour when the config is not bound yet. * Use boolean supplier and fix wildcard import. * Clean up the VPR since this is more elegant. * Clean up the VPR since this is more elegant. * Address review comments. * Address more review comments. * Fix formatting on `getSource` * Address comments by ichtt * Adapt to pups requests. * Stupid idea. * Attempt this again with a copy on write list. * Fix a concurrency and loading issue. * Fix #8813 Checks if the paths are valid resource paths. * Move the new methods on vanilla Patch. --- .../packs/VanillaPackResources.java.patch | 41 +- .../FallbackResourceManager.java.patch | 12 + .../MultiPackResourceManager.java.patch | 10 + .../minecraftforge/common/ForgeConfig.java | 23 + .../extensions/IForgePackResources.java | 6 + .../resource/DelegatingResourcePack.java | 20 +- .../resource/PathResourcePack.java | 84 +++- .../resource/ResourceCacheManager.java | 418 ++++++++++++++++++ .../resources/META-INF/accesstransformer.cfg | 3 + .../resources/assets/forge/lang/en_us.json | 7 + 10 files changed, 594 insertions(+), 30 deletions(-) create mode 100644 patches/minecraft/net/minecraft/server/packs/resources/FallbackResourceManager.java.patch create mode 100644 patches/minecraft/net/minecraft/server/packs/resources/MultiPackResourceManager.java.patch create mode 100644 src/main/java/net/minecraftforge/resource/ResourceCacheManager.java diff --git a/patches/minecraft/net/minecraft/server/packs/VanillaPackResources.java.patch b/patches/minecraft/net/minecraft/server/packs/VanillaPackResources.java.patch index df8e08b103..02eb76dae5 100644 --- a/patches/minecraft/net/minecraft/server/packs/VanillaPackResources.java.patch +++ b/patches/minecraft/net/minecraft/server/packs/VanillaPackResources.java.patch @@ -1,5 +1,15 @@ --- a/net/minecraft/server/packs/VanillaPackResources.java +++ b/net/minecraft/server/packs/VanillaPackResources.java +@@ -147,6 +_,9 @@ + try { + Path path = f_182296_.get(p_215346_); + if (path != null) { ++ if (net.minecraftforge.resource.ResourceCacheManager.shouldUseCache() && this.cacheManager.hasCached(p_215346_, p_215347_)) { ++ set.addAll(this.cacheManager.getResources(p_215346_, p_215347_, path.getFileSystem().getPath(p_215348_), p_215349_)); ++ } else + m_215357_(set, p_215347_, path, p_215348_, p_215349_); + } else { + f_10315_.error("Can't access assets root for type: {}", (Object)p_215346_); @@ -209,7 +_,7 @@ try { @@ -18,12 +28,10 @@ } public boolean m_7211_(PackType p_10355_, ResourceLocation p_10356_) { -@@ -298,6 +_,20 @@ +@@ -300,11 +_,39 @@ + public void close() { } - public void close() { -+ } -+ + //Vanilla used to just grab from the classpath, this breaks dev environments, and Forge runtime + //as forge ships vanilla assets in an 'extra' jar with no classes. + //So find that extra jar using the .mcassetsroot marker. @@ -36,6 +44,27 @@ + } catch (IOException e) { + return VanillaPackResources.class.getResourceAsStream(resource); + } - } - ++ } ++ public ResourceProvider m_215363_() { + return (p_215344_) -> { + return Optional.of(new Resource(this.m_8017_(), () -> { + return this.m_8031_(PackType.CLIENT_RESOURCES, p_215344_); + })); + }; ++ } ++ ++ private final net.minecraftforge.resource.ResourceCacheManager cacheManager = new net.minecraftforge.resource.ResourceCacheManager(false, net.minecraftforge.common.ForgeConfig.COMMON.indexVanillaPackCachesOnThread, (packType, namespace) -> f_182296_.get(packType).resolve(namespace)); ++ ++ @Override ++ public void initForNamespace(final String nameSpace) { ++ if (net.minecraftforge.resource.ResourceCacheManager.shouldUseCache()) ++ this.cacheManager.index(nameSpace); ++ } ++ ++ @Override ++ public void init(final PackType packType) { ++ initForNamespace("minecraft"); //Default namespaces. ++ initForNamespace("realms"); + } + } diff --git a/patches/minecraft/net/minecraft/server/packs/resources/FallbackResourceManager.java.patch b/patches/minecraft/net/minecraft/server/packs/resources/FallbackResourceManager.java.patch new file mode 100644 index 0000000000..71f7578d6a --- /dev/null +++ b/patches/minecraft/net/minecraft/server/packs/resources/FallbackResourceManager.java.patch @@ -0,0 +1,12 @@ +--- a/net/minecraft/server/packs/resources/FallbackResourceManager.java ++++ b/net/minecraft/server/packs/resources/FallbackResourceManager.java +@@ -53,6 +_,9 @@ + + private void m_215395_(String p_215396_, @Nullable PackResources p_215397_, @Nullable Predicate p_215398_) { + this.f_10599_.add(new FallbackResourceManager.PackEntry(p_215396_, p_215397_, p_215398_)); ++ if (p_215397_ != null) { ++ p_215397_.initForNamespace(this.f_10602_); ++ } + } + + public Set m_7187_() { diff --git a/patches/minecraft/net/minecraft/server/packs/resources/MultiPackResourceManager.java.patch b/patches/minecraft/net/minecraft/server/packs/resources/MultiPackResourceManager.java.patch new file mode 100644 index 0000000000..9d9f724485 --- /dev/null +++ b/patches/minecraft/net/minecraft/server/packs/resources/MultiPackResourceManager.java.patch @@ -0,0 +1,10 @@ +--- a/net/minecraft/server/packs/resources/MultiPackResourceManager.java ++++ b/net/minecraft/server/packs/resources/MultiPackResourceManager.java +@@ -24,6 +_,7 @@ + public MultiPackResourceManager(PackType p_203797_, List p_203798_) { + this.f_203795_ = List.copyOf(p_203798_); + Map map = new HashMap<>(); ++ p_203798_.forEach(resourcePack -> resourcePack.init(p_203797_)); + List list = p_203798_.stream().flatMap((p_215471_) -> { + return p_215471_.m_5698_(p_203797_).stream(); + }).toList(); diff --git a/src/main/java/net/minecraftforge/common/ForgeConfig.java b/src/main/java/net/minecraftforge/common/ForgeConfig.java index 80af297e7f..f350c27756 100644 --- a/src/main/java/net/minecraftforge/common/ForgeConfig.java +++ b/src/main/java/net/minecraftforge/common/ForgeConfig.java @@ -83,10 +83,33 @@ public class ForgeConfig { * General configuration that doesn't need to be synchronized but needs to be available before server startup */ public static class Common { + public final BooleanValue cachePackAccess; + public final BooleanValue indexVanillaPackCachesOnThread; + public final BooleanValue indexModPackCachesOnThread; + Common(ForgeConfigSpec.Builder builder) { builder.comment("General configuration settings") .push("general"); + cachePackAccess = builder + .comment("Set this to true to cache resource listings in resource and data packs") + .translation("forge.configgui.cachePackAccess") + .worldRestart() + .define("cachePackAccess", true); + + indexVanillaPackCachesOnThread = builder + .comment("Set this to true to index vanilla resource and data packs on thread") + .translation("forge.configgui.indexVanillaPackCachesOnThread") + .worldRestart() + .define("indexVanillaPackCachesOnThread", false); + + indexModPackCachesOnThread = builder + .comment("Set this to true to index mod resource and data packs on thread") + .translation("forge.configgui.indexModPackCachesOnThread") + .worldRestart() + .define("indexModPackCachesOnThread", false); + + builder.pop(); } } diff --git a/src/main/java/net/minecraftforge/common/extensions/IForgePackResources.java b/src/main/java/net/minecraftforge/common/extensions/IForgePackResources.java index c794ec8be4..025626636c 100644 --- a/src/main/java/net/minecraftforge/common/extensions/IForgePackResources.java +++ b/src/main/java/net/minecraftforge/common/extensions/IForgePackResources.java @@ -5,10 +5,16 @@ package net.minecraftforge.common.extensions; +import net.minecraft.server.packs.PackType; + public interface IForgePackResources { default boolean isHidden() { return false; } + + default void initForNamespace(final String nameSpace) {} + + default void init(final PackType packType) {} } diff --git a/src/main/java/net/minecraftforge/resource/DelegatingResourcePack.java b/src/main/java/net/minecraftforge/resource/DelegatingResourcePack.java index 6a314b3245..9778f29e98 100644 --- a/src/main/java/net/minecraftforge/resource/DelegatingResourcePack.java +++ b/src/main/java/net/minecraftforge/resource/DelegatingResourcePack.java @@ -24,10 +24,9 @@ import net.minecraft.resources.ResourceLocation; public class DelegatingResourcePack extends AbstractPackResources { - private final List delegates; - private final Map> namespacesAssets; - private final Map> namespacesData; + private Map> namespacesAssets; + private Map> namespacesData; private final String name; private final PackMetadataSection packInfo; @@ -42,6 +41,21 @@ public class DelegatingResourcePack extends AbstractPackResources this.namespacesData = this.buildNamespaceMap(PackType.SERVER_DATA, delegates); } + @Override + public void initForNamespace(final String nameSpace) + { + this.delegates.forEach(delegate -> delegate.initForNamespace(nameSpace)); + } + + @Override + public void init(final PackType packType) + { + this.delegates.forEach(packResources -> packResources.init(packType)); + + this.namespacesAssets = buildNamespaceMap(PackType.CLIENT_RESOURCES, delegates); + this.namespacesData = buildNamespaceMap(PackType.SERVER_DATA, delegates); + } + private Map> buildNamespaceMap(PackType type, List packList) { Map> map = new HashMap<>(); diff --git a/src/main/java/net/minecraftforge/resource/PathResourcePack.java b/src/main/java/net/minecraftforge/resource/PathResourcePack.java index 4e2fa350b7..e89802de8b 100644 --- a/src/main/java/net/minecraftforge/resource/PathResourcePack.java +++ b/src/main/java/net/minecraftforge/resource/PathResourcePack.java @@ -9,6 +9,8 @@ import com.google.common.base.Joiner; import net.minecraft.resources.ResourceLocation; import net.minecraft.server.packs.AbstractPackResources; import net.minecraft.server.packs.PackType; +import net.minecraftforge.common.ForgeConfig; +import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.FileNotFoundException; @@ -26,7 +28,7 @@ import java.util.stream.Collectors; /** * Defines a resource pack from an arbitrary Path. - * + *

* This is primarily intended to support including optional resource packs inside a mod, * such as to have alternative textures to use along with Programmer Art, or optional * alternative recipes for compatibility ot to replace vanilla recipes. @@ -36,12 +38,14 @@ public class PathResourcePack extends AbstractPackResources private final Path source; private final String packName; + private final ResourceCacheManager cacheManager = new ResourceCacheManager(true, ForgeConfig.COMMON.indexModPackCachesOnThread, (packType, namespace) -> resolve(packType.getDirectory(), namespace).toAbsolutePath()); + /** * Constructs a java.nio.Path-based resource pack. * * @param packName the identifying name of the pack. * This name should be unique within the pack finder, preferably the name of the file or folder containing the resources. - * @param source the root path of the pack. This needs to point to the folder that contains "assets" and/or "data", not the asset folder itself! + * @param source the root path of the pack. This needs to point to the folder that contains "assets" and/or "data", not the asset folder itself! */ public PathResourcePack(String packName, final Path source) { @@ -50,13 +54,29 @@ public class PathResourcePack extends AbstractPackResources this.packName = packName; } + @Override + public void initForNamespace(final String namespace) + { + if (ResourceCacheManager.shouldUseCache()) + { + this.cacheManager.index(namespace); + } + } + + @Override + public void init(final PackType packType) + { + getNamespacesFromDisk(packType).forEach(this::initForNamespace); + } + /** * Returns the source path containing the resource pack. * This is used for error display. * * @return the root path of the resources. */ - public Path getSource() { + public Path getSource() + { return this.source; } @@ -73,13 +93,14 @@ public class PathResourcePack extends AbstractPackResources /** * Implement to return a file or folder path for the given set of path components. + * * @param paths One or more path strings to resolve. Can include slash-separated paths. * @return the resulting path, which may not exist. */ protected Path resolve(String... paths) { Path path = getSource(); - for(String name : paths) + for (String name : paths) path = path.resolve(name); return path; } @@ -88,7 +109,7 @@ public class PathResourcePack extends AbstractPackResources protected InputStream getResource(String name) throws IOException { final Path path = resolve(name); - if(!Files.exists(path)) + if (!Files.exists(path)) throw new FileNotFoundException("Can't find resource " + name + " at " + getSource()); return Files.newInputStream(path, StandardOpenOption.READ); } @@ -108,16 +129,21 @@ public class PathResourcePack extends AbstractPackResources Path root = resolve(type.getDirectory(), resourceNamespace).toAbsolutePath(); Path inputPath = root.getFileSystem().getPath(pathIn); + if (ResourceCacheManager.shouldUseCache() && this.cacheManager.hasCached(type, resourceNamespace)) + { + return this.cacheManager.getResources(type, resourceNamespace, inputPath, filter); + } + return Files.walk(root) .map(root::relativize) .filter(path -> !path.toString().endsWith(".mcmeta") && path.startsWith(inputPath)) // It is VERY IMPORTANT that we do not rely on Path.toString as this is inconsistent between operating systems - // Join the path names ourselves to force forward slashes + // Join the path names ourselves to force forward slashes #8813 + .filter(path -> ResourceLocation.isValidPath(Joiner.on('/').join(path))) // Only process valid paths Fixes the case where people put invalid resources in their jar. .map(path -> new ResourceLocation(resourceNamespace, Joiner.on('/').join(path))) .filter(filter) .collect(Collectors.toList()); - } - catch (IOException e) + } catch (IOException e) { return Collections.emptyList(); } @@ -126,22 +152,32 @@ public class PathResourcePack extends AbstractPackResources @Override public Set getNamespaces(PackType type) { - try { + if (ResourceCacheManager.shouldUseCache()) + { + return this.cacheManager.getNamespaces(type); + } + + return getNamespacesFromDisk(type); + } + + @NotNull + private Set getNamespacesFromDisk(final PackType type) + { + try + { Path root = resolve(type.getDirectory()); - return Files.walk(root,1) + return Files.walk(root, 1) .map(path -> root.relativize(path)) .filter(path -> path.getNameCount() > 0) // skip the root entry - .map(p->p.toString().replaceAll("/$","")) // remove the trailing slash, if present + .map(p -> p.toString().replaceAll("/$", "")) // remove the trailing slash, if present .filter(s -> !s.isEmpty()) //filter empty strings, otherwise empty strings default to minecraft in ResourceLocations .collect(Collectors.toSet()); - } - catch (IOException e) + } catch (IOException e) { if (type == PackType.SERVER_DATA) //We still have to add the resource namespace if client resources exist, as we load langs (which are in assets) on server { return this.getNamespaces(PackType.CLIENT_RESOURCES); - } - else + } else { return Collections.emptySet(); } @@ -149,19 +185,25 @@ public class PathResourcePack extends AbstractPackResources } @Override - public InputStream getResource(PackType type, ResourceLocation location) throws IOException { - if (location.getPath().startsWith("lang/")) { + public InputStream getResource(PackType type, ResourceLocation location) throws IOException + { + if (location.getPath().startsWith("lang/")) + { return super.getResource(PackType.CLIENT_RESOURCES, location); - } else { + } else + { return super.getResource(type, location); } } @Override - public boolean hasResource(PackType type, ResourceLocation location) { - if (location.getPath().startsWith("lang/")) { + public boolean hasResource(PackType type, ResourceLocation location) + { + if (location.getPath().startsWith("lang/")) + { return super.hasResource(PackType.CLIENT_RESOURCES, location); - } else { + } else + { return super.hasResource(type, location); } } diff --git a/src/main/java/net/minecraftforge/resource/ResourceCacheManager.java b/src/main/java/net/minecraftforge/resource/ResourceCacheManager.java new file mode 100644 index 0000000000..0a8d4cfe95 --- /dev/null +++ b/src/main/java/net/minecraftforge/resource/ResourceCacheManager.java @@ -0,0 +1,418 @@ +/* + * Copyright (c) Forge Development LLC and contributors + * SPDX-License-Identifier: LGPL-2.1-only + */ + +package net.minecraftforge.resource; + +import com.google.common.base.Joiner; +import com.google.common.collect.Maps; +import com.mojang.logging.LogUtils; +import net.minecraft.Util; +import net.minecraft.resources.ResourceLocation; +import net.minecraft.server.packs.PackType; +import net.minecraftforge.common.ForgeConfig; +import net.minecraftforge.common.ForgeConfigSpec; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiFunction; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Cache manager for resources. + *

+ * This class handles caching the resource listing calls on a pack, pack type and namespace level. + */ +public class ResourceCacheManager +{ + + /** + * Indicates if the underlying namespaced managers need to support reloading. + * Disabled for the (in-jar / downloaded) default vanilla pack. + */ + private final boolean supportsReloading; + /** + * Indicates if the indexing of the file-tree should happen off-thread or on-thread and block the process accordingly. + */ + private final ForgeConfigSpec.BooleanValue indexOffThreadConfigOption; + + /** + * The path builder (different users have different requirements for how they want to handle this) + */ + private final BiFunction pathBuilder; + /** + * The individual sub managers by pack type and namespace + */ + private final Map managersByNamespace = Maps.newConcurrentMap(); + + /** + * Creates a new instance of a resource cache manager. + * + * @param supportsReloading True to make the namespace specific managers support reloading the cache. + * @param indexOffThreadConfig True to index the file-tree off-thread. + * @param pathBuilder The path builder to use. + */ + public ResourceCacheManager(final boolean supportsReloading, final ForgeConfigSpec.BooleanValue indexOffThreadConfig, final BiFunction pathBuilder) + { + this.supportsReloading = supportsReloading; + this.indexOffThreadConfigOption = indexOffThreadConfig; + this.pathBuilder = pathBuilder; + } + + public boolean shouldIndexOffThread() + { + return !getConfigValue(this.indexOffThreadConfigOption); + } + + public static boolean shouldUseCache() + { + return getConfigValue(ForgeConfig.COMMON.cachePackAccess); + } + + private static boolean getConfigValue(Supplier configValue) + { + // Yes we catch the early loading error on purpose. + // Both the indexing off-thread and if caching is enabled are configurable, so we need to catch the error. + // So when they are loaded early we preserve the default behaviour which is to not cache or index on thread. + try + { + return configValue.get(); + } catch (IllegalStateException e) + { + return false; + } + } + + /** + * Invoked to trigger indexing of a given namespaces resources. + * + * @param namespace The namespace to index. + */ + public void index(final String namespace) + { + for (final PackType packType : PackType.values()) + { + final PackTypeAndNamespace key = new PackTypeAndNamespace(packType, namespace); // Make a key. + if (managersByNamespace.containsKey(key) && !supportsReloading) + { // If we do not support reloading we just skip this. + return; + } + + // Create a new manager, overriding the previous one if it exists. + final NamespacedResourceCacheManager newManager = new NamespacedResourceCacheManager(packType, namespace, this.shouldIndexOffThread(), pathBuilder, this::createWalkingStream); + this.managersByNamespace.put(key, newManager); + + // Index the inner manager (which will for sure exist, and will know the pack type and namespace already) + newManager.index(); + } + } + + /** + * Handling method to create a new stream of paths in the target directory, which can then be walked. + * The caller needs to close the stream manually. + * + * @param path The path to walk. + * @return A stream of paths which reside in the given path. + * @throws IOException Thrown when the path does not exist or reading of path information failed. + */ + @SuppressWarnings("resource") + // We close this automatically later on when the result is actually used. This is just a factory. + private Stream createWalkingStream(final Path path) throws IOException + { + return Files.walk(path) + .filter(resourcePath -> !resourcePath.toString().endsWith(".mcmeta")); + } + + /** + * Returns the cached resources for the given pack type, namespace, path prefix and filter. + * + * @param type The type of pack to look up in (data or resource) + * @param resourceNamespace The namespace of the resources to look up. + * @param inputPath The input path prefix to check for. + * @param filter The ternary filter to apply. + * @return A collection of resource locations which match the given filter, have the input path as prefix, are in the right pack type and have the given namespace as namespace. + */ + public Collection getResources(final PackType type, final String resourceNamespace, final Path inputPath, final Predicate filter) + { + final PackTypeAndNamespace key = new PackTypeAndNamespace(type, resourceNamespace); + final NamespacedResourceCacheManager manager = managersByNamespace.get(key); + + // Check if we even have a cached manager with the given namespace, else return an empty collection. + if (manager == null) + { + return Collections.emptyList(); + } + + // Request the data from the manager. + return manager.getResources(inputPath, filter); + } + + /** + * Looks up all already indexed namespaces. + * Note even though a namespace might be returned from this method, it does not mean that the indexing has completed + * on that namespace if it is running off-thread. + * + * @param type The type of the pack to get the namespaces for. + * @return The namespaces which are known for the given pack type. + */ + public Set getNamespaces(final PackType type) + { + return managersByNamespace.keySet().stream() + .filter(key -> key.packType() == type) + .map(PackTypeAndNamespace::namespace) + .collect(Collectors.toSet()); + } + + /** + * Indicates if the given namespace is completely cached for the given pack type. + * + * @param packType The pack type to check. + * @param namespace The namespace to check. + * @return True if the namespace is completely cached for the given pack type. + */ + public boolean hasCached(final PackType packType, final String namespace) + { + final PackTypeAndNamespace key = new PackTypeAndNamespace(packType, namespace); + final NamespacedResourceCacheManager manager = managersByNamespace.get(key); + return manager != null && manager.cacheLoaded(); + } + + /** + * Record for the delegated namespaced manager map. + * + * @param packType The pack type. + * @param namespace The namespace. + */ + private record PackTypeAndNamespace(PackType packType, String namespace) + { + } + + /** + * Record for the individual cache entries. + * + * @param packType The pack type that the resource resides in. + * @param namespace The namespace that the resource resides in. + * @param path The path in the pack that the resource resides in. + * @param resourceLocation The resource location for the resource. + */ + private record ResourceCacheEntry(PackType packType, String namespace, Path path, + ResourceLocation resourceLocation) + { + } + + /** + * A namespaced and pack type specific cache which the {@link ResourceCacheManager} can delegate the handling to. + */ + private static class NamespacedResourceCacheManager + { + private static final Logger LOGGER = LogUtils.getLogger(); + + /** + * The pack type this manager is responsible for. + */ + private final PackType packType; + /** + * The namespace this manager is responsible for. + */ + private final String namespace; + /** + * Indicates if the indexing should run off-thread. + */ + private final boolean indexOffThread; + /** + * The path builder to use. + */ + private final BiFunction pathBuilder; + /** + * The path walker stream factory to use. + */ + private final PathWalkerFactory pathFinder; + + /** + * The cache entries for this manager. + */ + private final Map> entriesByPathPrefix = Maps.newConcurrentMap(); + /** + * Indicates if the cache has been loaded successfully. + */ + private final AtomicBoolean cacheLoaded = new AtomicBoolean(false); + + /** + * Creates a new namespaced resource cache manager. + * + * @param packType The pack type that this manager handles. + * @param namespace The namespace that this manager handles. + * @param indexOffThread True to enable indexing off-thread. + * @param pathBuilder The path builder to use. + * @param pathFinder The path walker stream factory to use. + */ + private NamespacedResourceCacheManager(final PackType packType, final String namespace, final boolean indexOffThread, BiFunction pathBuilder, final PathWalkerFactory pathFinder) + { + this.packType = packType; + this.namespace = namespace; + this.indexOffThread = indexOffThread; + this.pathBuilder = pathBuilder; + this.pathFinder = pathFinder; + } + + /** + * Triggers indexing of this manager. + */ + public void index() + { + if (indexOffThread) + { + // Run off-thread. + CompletableFuture.runAsync( + this::doIndex, + Util.backgroundExecutor() + ); + } else + { + // Run on-thread + doIndex(); + } + } + + /** + * Performs the actual indexing of this manager. + * Can be invoked on- or off-thread. + */ + private void doIndex() + { + // Get the path to the root of the namespace in the current pack. + final Path rootPath = pathBuilder.apply(packType, namespace); + + // Stream element resource that combines a normalized path and a joined path using the "/" as separator. + record PathWithLocationPath(Path path, String locationPath) + { + } + + // Build a walkable stream, process it + try (final Stream paths = pathFinder.createWalkingStream(rootPath)) + { + paths.parallel() // Run the stream in parallel + .map(rootPath::relativize) // Relative to the given root. + .map(path -> new PathWithLocationPath(path, Joiner.on('/').join(path))) // Create a common hierarchy. + .filter(path -> ResourceLocation.isValidPath(path.locationPath())) // Only process valid paths + .map(path -> new ResourceCacheEntry(packType, namespace, path.path(), new ResourceLocation(namespace, path.locationPath()))) // Create a cache entry. + .forEach(this::injectIntoCache); // Inject the entry into the cache. + } catch (NoSuchFileException noSuchFileException) + { + LOGGER.debug("Failed to cache resources, the directory does not exist!", noSuchFileException); + } catch (IOException ioException) + { + LOGGER.error("Failed to cache resources, some stuff might be missing!", ioException); + } catch (Exception exception) + { + LOGGER.error("Failed to cache resources, some stuff might be missing! Unknown exception!", exception); + } finally + { + cacheLoaded.set(true); + } + } + + /** + * Injects the given cache entry into the cache. + * By first injecting it into the parent of the entry (so the directory it resides in) and then recursively walking up. + * + * @param entry The entry to inject into the cache, must not be null. + */ + private void injectIntoCache(final ResourceCacheEntry entry) + { + injectIntoCache(entry.path().getParent(), entry); + } + + /** + * Injects the given cache entry into the cache, recursively into all of its parent paths. + * + * @param parentPath The parent path to inject into the cache, can be null. Null will be treated as the root path. + * @param entry The entry to inject into the cache, must not be null. + */ + private void injectIntoCache(@Nullable final Path parentPath, final ResourceCacheEntry entry) + { + String pathEntry; + if (parentPath == null) + { + // We have null, generally this only happens on the root directory. + pathEntry = ""; + } else + { + // Use the parent paths, we can not depend on a toString call here, since we need the / as separator. + pathEntry = Joiner.on("/").join(parentPath); + } + + // Inject into the cache, we can use a normal array list here since we have guarded against cache access later on. + this.entriesByPathPrefix.computeIfAbsent(pathEntry, e -> new CopyOnWriteArrayList<>()).add(entry); + + // Recursively walk to the top, while preventing duplicate entries. + if (parentPath != null && !pathEntry.isEmpty()) + { + this.injectIntoCache(parentPath.getParent(), entry); + } + } + + /** + * Looks up resources in the given input path prefix as well as those that match the filter. + * + * @param inputPath The input path prefix to look in. + * @param filter The filter which the resources must match. + * @return A collection of resource location which match the given path prefix and who match the filter. + */ + public Collection getResources(final Path inputPath, final Predicate filter) + { + // We only have a cache once the atomic flag has been set. + if (!cacheLoaded()) { + // We need to return a mutable object here because the callers actually mutate the collection. + return new ArrayList<>(); + } + + // Use the input path prefix, we can not depend on a toString call here, since we need the / as separator. + final String pathEntry = Joiner.on('/').join(inputPath); + + // Since we inject into the cache recursively we can now just grab the map entry that is the prefix and loop over its values. + // We use getOrDefault with a stream combination since the returned list needs to be mutable. + return entriesByPathPrefix.getOrDefault(pathEntry, Collections.emptyList()).stream() + .map(ResourceCacheEntry::resourceLocation) + .filter(filter) + .collect(Collectors.toList()); + } + + /** + * Indicates if the cache of this manager has been loaded. + * + * @return True if the cache has been loaded, false otherwise. + */ + public boolean cacheLoaded() + { + return cacheLoaded.get(); + } + } + + /** + * Functional callback interface to get a walkable stream of paths. + * Supports throwing {@link IOException} if the stream can not be created. + */ + @FunctionalInterface + private interface PathWalkerFactory + { + Stream createWalkingStream(Path path) throws IOException; + } +} diff --git a/src/main/resources/META-INF/accesstransformer.cfg b/src/main/resources/META-INF/accesstransformer.cfg index e08320e5e9..4c8184d32d 100644 --- a/src/main/resources/META-INF/accesstransformer.cfg +++ b/src/main/resources/META-INF/accesstransformer.cfg @@ -269,6 +269,9 @@ public-f net.minecraft.data.tags.TagsProvider m_6055_()Ljava/lang/String; # getN public net.minecraft.data.tags.TagsProvider$TagAppender f_126569_ # registry public net.minecraft.gametest.framework.GameTestServer (Ljava/lang/Thread;Lnet/minecraft/world/level/storage/LevelStorageSource$LevelStorageAccess;Lnet/minecraft/server/packs/repository/PackRepository;Lnet/minecraft/server/WorldStem;Ljava/util/Collection;Lnet/minecraft/core/BlockPos;)V # constructor public net.minecraft.network.protocol.status.ClientboundStatusResponsePacket f_134885_ # GSON +public net.minecraft.resources.ResourceLocation m_135835_(C)Z # validNamespaceChar +public net.minecraft.resources.ResourceLocation m_135841_(Ljava/lang/String;)Z # isValidPath +public net.minecraft.resources.ResourceLocation m_135843_(Ljava/lang/String;)Z # isValidNamespace protected net.minecraft.server.MinecraftServer f_129726_ # nextTickTime public net.minecraft.server.MinecraftServer$ReloadableResources public net.minecraft.server.dedicated.DedicatedServer f_139600_ # consoleInput diff --git a/src/main/resources/assets/forge/lang/en_us.json b/src/main/resources/assets/forge/lang/en_us.json index 8437ad38c1..602dcdd6b0 100644 --- a/src/main/resources/assets/forge/lang/en_us.json +++ b/src/main/resources/assets/forge/lang/en_us.json @@ -189,6 +189,13 @@ "forge.configgui.disableVersionCheck.tooltip": "Set to true to disable Forge version check mechanics. Forge queries a small json file on our server for version information. For more details see the ForgeVersion class in our github.", "forge.configgui.disableVersionCheck": "Disable Forge Version Check", + "forge.configgui.cachePackAccess.tooltip": "Set this to true to cache resource listings in resource and data packs", + "forge.configgui.cachePackAccess": "Cache Pack Access", + "forge.configgui.indexVanillaPackCachesOnThread.tooltip": "Set this to true to index vanilla resource and data packs on thread", + "forge.configgui.indexVanillaPackCachesOnThread": "Index vanilla resource packs on thread", + "forge.configgui.indexModPackCachesOnThread.tooltip": "Set this to true to index mod resource and data packs on thread", + "forge.configgui.indexModPackCachesOnThread": "Index mod resource packs on thread", + "forge.controlsgui.shift": "SHIFT + %s", "forge.controlsgui.control": "CTRL + %s", "forge.controlsgui.control.mac": "CMD + %s",