[1.18.x] Backport: Cache resource listing calls in resource packs (#8868)

This commit is contained in:
embeddedt authored and GitHub committed 2022-08-08 15:05:43 +02:00
1 parent 49c927a505
commit 4df024ef43
10 files changed
+561 -8

No files matched your search

@@ -1,5 +1,15 @@
--- a/net/minecraft/server/packs/VanillaPackResources.java
+++ b/net/minecraft/server/packs/VanillaPackResources.java
@@ -148,6 +_,9 @@
try {
Path path = f_182296_.get(p_10324_);
if (path != null) {
+ if (net.minecraftforge.resource.ResourceCacheManager.shouldUseCache() && this.cacheManager.hasCached(p_10324_, p_10325_)) {
+ set.addAll(this.cacheManager.getResources(p_10324_, p_10325_, path.getFileSystem().getPath(p_10326_), p_10328_));
+ } else
m_10342_(set, p_10327_, p_10325_, path, p_10326_, p_10328_);
} else {
f_10315_.error("Can't access assets root for type: {}", (Object)p_10324_);
@@ -203,7 +_,7 @@
try {
@@ -18,12 +28,10 @@
}
public boolean m_7211_(PackType p_10355_, ResourceLocation p_10356_) {
@@ -292,6 +_,20 @@
@@ -294,6 +_,20 @@
public void close() {
}
public void close() {
+ }
+
+ //Vanilla used to just grab from the classpath, this breaks dev environments, and Forge runtime
+ //as forge ships vanilla assets in an 'extra' jar with no classes.
+ //So find that extra jar using the .mcassetsroot marker.
@@ -36,6 +44,28 @@
+ } catch (IOException e) {
+ return VanillaPackResources.class.getResourceAsStream(resource);
+ }
}
+ }
+
public Resource m_142591_(final ResourceLocation p_143764_) throws IOException {
return new Resource() {
@Nullable
@@ -333,5 +_,19 @@
return p_143764_.toString();
}
};
+ }
+
+ private final net.minecraftforge.resource.ResourceCacheManager cacheManager = new net.minecraftforge.resource.ResourceCacheManager(false, net.minecraftforge.common.ForgeConfig.COMMON.indexVanillaPackCachesOnThread, (packType, namespace) -> f_182296_.get(packType).resolve(namespace));
+
+ @Override
+ public void initForNamespace(final String nameSpace) {
+ if (net.minecraftforge.resource.ResourceCacheManager.shouldUseCache())
+ this.cacheManager.index(nameSpace);
+ }
+
+ @Override
+ public void init(final PackType packType) {
+ initForNamespace("minecraft"); //Default namespaces.
+ initForNamespace("realms");
}
}
@@ -0,0 +1,10 @@
--- a/net/minecraft/server/packs/resources/FallbackResourceManager.java
+++ b/net/minecraft/server/packs/resources/FallbackResourceManager.java
@@ -33,6 +_,7 @@
public void m_10608_(PackResources p_10609_) {
this.f_10599_.add(p_10609_);
+ p_10609_.initForNamespace(this.f_10602_);
}
public Set<String> m_7187_() {
@@ -0,0 +1,10 @@
--- a/net/minecraft/server/packs/resources/MultiPackResourceManager.java
+++ b/net/minecraft/server/packs/resources/MultiPackResourceManager.java
@@ -24,6 +_,7 @@
this.f_203795_ = List.copyOf(p_203798_);
Map<String, FallbackResourceManager> map = new HashMap<>();
+ p_203798_.forEach(resourcePack -> resourcePack.init(p_203797_));
for(PackResources packresources : p_203798_) {
for(String s : packresources.m_5698_(p_203797_)) {
map.computeIfAbsent(s, (p_203802_) -> {
@@ -98,6 +98,9 @@ public class ForgeConfig {
*/
public static class Common {
public final ForgeConfigSpec.ConfigValue<? extends String> defaultWorldType;
public final BooleanValue cachePackAccess;
public final BooleanValue indexVanillaPackCachesOnThread;
public final BooleanValue indexModPackCachesOnThread;
Common(ForgeConfigSpec.Builder builder) {
builder.comment("General configuration settings")
@@ -109,6 +112,24 @@ public class ForgeConfig {
.translation("forge.configgui.defaultWorldType")
.define("defaultWorldType", "default");
cachePackAccess = builder
.comment("Set this to true to cache resource listings in resource and data packs")
.translation("forge.configgui.cachePackAccess")
.worldRestart()
.define("cachePackAccess", true);
indexVanillaPackCachesOnThread = builder
.comment("Set this to true to index vanilla resource and data packs on thread")
.translation("forge.configgui.indexVanillaPackCachesOnThread")
.worldRestart()
.define("indexVanillaPackCachesOnThread", false);
indexModPackCachesOnThread = builder
.comment("Set this to true to index mod resource and data packs on thread")
.translation("forge.configgui.indexModPackCachesOnThread")
.worldRestart()
.define("indexModPackCachesOnThread", false);
builder.pop();
}
@@ -5,10 +5,16 @@
package net.minecraftforge.common.extensions;
import net.minecraft.server.packs.PackType;
public interface IForgePackResources
{
default boolean isHidden()
{
return false;
}
default void initForNamespace(final String nameSpace) {}
default void init(final PackType packType) {}
}
@@ -26,8 +26,8 @@ public class DelegatingResourcePack extends AbstractPackResources
{
private final List<PackResources> delegates;
private final Map<String, List<PackResources>> namespacesAssets;
private final Map<String, List<PackResources>> namespacesData;
private Map<String, List<PackResources>> namespacesAssets;
private Map<String, List<PackResources>> namespacesData;
private final String name;
private final PackMetadataSection packInfo;
@@ -42,6 +42,21 @@ public class DelegatingResourcePack extends AbstractPackResources
this.namespacesData = this.buildNamespaceMap(PackType.SERVER_DATA, delegates);
}
@Override
public void initForNamespace(final String nameSpace)
{
this.delegates.forEach(delegate -> delegate.initForNamespace(nameSpace));
}
@Override
public void init(final PackType packType)
{
this.delegates.forEach(packResources -> packResources.init(packType));
this.namespacesAssets = buildNamespaceMap(PackType.CLIENT_RESOURCES, delegates);
this.namespacesData = buildNamespaceMap(PackType.SERVER_DATA, delegates);
}
private Map<String, List<PackResources>> buildNamespaceMap(PackType type, List<PackResources> packList)
{
Map<String, List<PackResources>> map = new HashMap<>();
@@ -9,6 +9,7 @@ import com.google.common.base.Joiner;
import net.minecraft.resources.ResourceLocation;
import net.minecraft.server.packs.AbstractPackResources;
import net.minecraft.server.packs.PackType;
import net.minecraftforge.common.ForgeConfig;
import java.io.File;
import java.io.FileNotFoundException;
@@ -36,6 +37,8 @@ public class PathResourcePack extends AbstractPackResources
private final Path source;
private final String packName;
private final ResourceCacheManager cacheManager = new ResourceCacheManager(true, ForgeConfig.COMMON.indexModPackCachesOnThread, (packType, namespace) -> resolve(packType.getDirectory(), namespace).toAbsolutePath());
/**
* Constructs a java.nio.Path-based resource pack.
*
@@ -50,6 +53,21 @@ public class PathResourcePack extends AbstractPackResources
this.packName = packName;
}
@Override
public void initForNamespace(final String namespace)
{
if (ResourceCacheManager.shouldUseCache())
{
this.cacheManager.index(namespace);
}
}
@Override
public void init(final PackType packType)
{
getNamespacesFromDisk(packType).forEach(this::initForNamespace);
}
/**
* Returns the source path containing the resource pack.
* This is used for error display.
@@ -108,6 +126,11 @@ public class PathResourcePack extends AbstractPackResources
Path root = resolve(type.getDirectory(), resourceNamespace).toAbsolutePath();
Path inputPath = root.getFileSystem().getPath(pathIn);
if (ResourceCacheManager.shouldUseCache() && this.cacheManager.hasCached(type, resourceNamespace))
{
return this.cacheManager.getResources(type, resourceNamespace, inputPath, filter);
}
return Files.walk(root)
.map(root::relativize)
.filter(path -> path.getNameCount() <= maxDepth && !path.toString().endsWith(".mcmeta") && path.startsWith(inputPath))
@@ -125,6 +148,16 @@ public class PathResourcePack extends AbstractPackResources
@Override
public Set<String> getNamespaces(PackType type)
{
if (ResourceCacheManager.shouldUseCache())
{
return this.cacheManager.getNamespaces(type);
}
return getNamespacesFromDisk(type);
}
public Set<String> getNamespacesFromDisk(PackType type)
{
try {
Path root = resolve(type.getDirectory());
@@ -0,0 +1,418 @@
/*
* Copyright (c) Forge Development LLC and contributors
* SPDX-License-Identifier: LGPL-2.1-only
*/
package net.minecraftforge.resource;
import com.google.common.base.Joiner;
import com.google.common.collect.Maps;
import com.mojang.logging.LogUtils;
import net.minecraft.Util;
import net.minecraft.resources.ResourceLocation;
import net.minecraft.server.packs.PackType;
import net.minecraftforge.common.ForgeConfig;
import net.minecraftforge.common.ForgeConfigSpec;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Cache manager for resources.
* <p>
* This class handles caching the resource listing calls on a pack, pack type and namespace level.
*/
public class ResourceCacheManager
{
/**
* Indicates if the underlying namespaced managers need to support reloading.
* Disabled for the (in-jar / downloaded) default vanilla pack.
*/
private final boolean supportsReloading;
/**
* Indicates if the indexing of the file-tree should happen off-thread or on-thread and block the process accordingly.
*/
private final ForgeConfigSpec.BooleanValue indexOffThreadConfigOption;
/**
* The path builder (different users have different requirements for how they want to handle this)
*/
private final BiFunction<PackType, String, Path> pathBuilder;
/**
* The individual sub managers by pack type and namespace
*/
private final Map<PackTypeAndNamespace, NamespacedResourceCacheManager> managersByNamespace = Maps.newConcurrentMap();
/**
* Creates a new instance of a resource cache manager.
*
* @param supportsReloading True to make the namespace specific managers support reloading the cache.
* @param indexOffThreadConfig True to index the file-tree off-thread.
* @param pathBuilder The path builder to use.
*/
public ResourceCacheManager(final boolean supportsReloading, final ForgeConfigSpec.BooleanValue indexOffThreadConfig, final BiFunction<PackType, String, Path> pathBuilder)
{
this.supportsReloading = supportsReloading;
this.indexOffThreadConfigOption = indexOffThreadConfig;
this.pathBuilder = pathBuilder;
}
public boolean shouldIndexOffThread()
{
return !getConfigValue(this.indexOffThreadConfigOption);
}
public static boolean shouldUseCache()
{
return getConfigValue(ForgeConfig.COMMON.cachePackAccess);
}
private static boolean getConfigValue(Supplier<Boolean> configValue)
{
// Yes we catch the early loading error on purpose.
// Both the indexing off-thread and if caching is enabled are configurable, so we need to catch the error.
// So when they are loaded early we preserve the default behaviour which is to not cache or index on thread.
try
{
return configValue.get();
} catch (IllegalStateException e)
{
return false;
}
}
/**
* Invoked to trigger indexing of a given namespaces resources.
*
* @param namespace The namespace to index.
*/
public void index(final String namespace)
{
for (final PackType packType : PackType.values())
{
final PackTypeAndNamespace key = new PackTypeAndNamespace(packType, namespace); // Make a key.
if (managersByNamespace.containsKey(key) && !supportsReloading)
{ // If we do not support reloading we just skip this.
return;
}
// Create a new manager, overriding the previous one if it exists.
final NamespacedResourceCacheManager newManager = new NamespacedResourceCacheManager(packType, namespace, this.shouldIndexOffThread(), pathBuilder, this::createWalkingStream);
this.managersByNamespace.put(key, newManager);
// Index the inner manager (which will for sure exist, and will know the pack type and namespace already)
newManager.index();
}
}
/**
* Handling method to create a new stream of paths in the target directory, which can then be walked.
* The caller needs to close the stream manually.
*
* @param path The path to walk.
* @return A stream of paths which reside in the given path.
* @throws IOException Thrown when the path does not exist or reading of path information failed.
*/
@SuppressWarnings("resource")
// We close this automatically later on when the result is actually used. This is just a factory.
private Stream<Path> createWalkingStream(final Path path) throws IOException
{
return Files.walk(path)
.filter(resourcePath -> !resourcePath.toString().endsWith(".mcmeta"));
}
/**
* Returns the cached resources for the given pack type, namespace, path prefix and filter.
*
* @param type The type of pack to look up in (data or resource)
* @param resourceNamespace The namespace of the resources to look up.
* @param inputPath The input path prefix to check for.
* @param filter The ternary filter to apply.
* @return A collection of resource locations which match the given filter, have the input path as prefix, are in the right pack type and have the given namespace as namespace.
*/
public Collection<ResourceLocation> getResources(final PackType type, final String resourceNamespace, final Path inputPath, final Predicate<String> filter)
{
final PackTypeAndNamespace key = new PackTypeAndNamespace(type, resourceNamespace);
final NamespacedResourceCacheManager manager = managersByNamespace.get(key);
// Check if we even have a cached manager with the given namespace, else return an empty collection.
if (manager == null)
{
return Collections.emptyList();
}
// Request the data from the manager.
return manager.getResources(inputPath, filter);
}
/**
* Looks up all already indexed namespaces.
* Note even though a namespace might be returned from this method, it does not mean that the indexing has completed
* on that namespace if it is running off-thread.
*
* @param type The type of the pack to get the namespaces for.
* @return The namespaces which are known for the given pack type.
*/
public Set<String> getNamespaces(final PackType type)
{
return managersByNamespace.keySet().stream()
.filter(key -> key.packType() == type)
.map(PackTypeAndNamespace::namespace)
.collect(Collectors.toSet());
}
/**
* Indicates if the given namespace is completely cached for the given pack type.
*
* @param packType The pack type to check.
* @param namespace The namespace to check.
* @return True if the namespace is completely cached for the given pack type.
*/
public boolean hasCached(final PackType packType, final String namespace)
{
final PackTypeAndNamespace key = new PackTypeAndNamespace(packType, namespace);
final NamespacedResourceCacheManager manager = managersByNamespace.get(key);
return manager != null && manager.cacheLoaded();
}
/**
* Record for the delegated namespaced manager map.
*
* @param packType The pack type.
* @param namespace The namespace.
*/
private record PackTypeAndNamespace(PackType packType, String namespace)
{
}
/**
* Record for the individual cache entries.
*
* @param packType The pack type that the resource resides in.
* @param namespace The namespace that the resource resides in.
* @param path The path in the pack that the resource resides in.
* @param resourceLocation The resource location for the resource.
*/
private record ResourceCacheEntry(PackType packType, String namespace, Path path,
ResourceLocation resourceLocation)
{
}
/**
* A namespaced and pack type specific cache which the {@link ResourceCacheManager} can delegate the handling to.
*/
private static class NamespacedResourceCacheManager
{
private static final Logger LOGGER = LogUtils.getLogger();
/**
* The pack type this manager is responsible for.
*/
private final PackType packType;
/**
* The namespace this manager is responsible for.
*/
private final String namespace;
/**
* Indicates if the indexing should run off-thread.
*/
private final boolean indexOffThread;
/**
* The path builder to use.
*/
private final BiFunction<PackType, String, Path> pathBuilder;
/**
* The path walker stream factory to use.
*/
private final PathWalkerFactory pathFinder;
/**
* The cache entries for this manager.
*/
private final Map<String, List<ResourceCacheEntry>> entriesByPathPrefix = Maps.newConcurrentMap();
/**
* Indicates if the cache has been loaded successfully.
*/
private final AtomicBoolean cacheLoaded = new AtomicBoolean(false);
/**
* Creates a new namespaced resource cache manager.
*
* @param packType The pack type that this manager handles.
* @param namespace The namespace that this manager handles.
* @param indexOffThread True to enable indexing off-thread.
* @param pathBuilder The path builder to use.
* @param pathFinder The path walker stream factory to use.
*/
private NamespacedResourceCacheManager(final PackType packType, final String namespace, final boolean indexOffThread, BiFunction<PackType, String, Path> pathBuilder, final PathWalkerFactory pathFinder)
{
this.packType = packType;
this.namespace = namespace;
this.indexOffThread = indexOffThread;
this.pathBuilder = pathBuilder;
this.pathFinder = pathFinder;
}
/**
* Triggers indexing of this manager.
*/
public void index()
{
if (indexOffThread)
{
// Run off-thread.
CompletableFuture.runAsync(
this::doIndex,
Util.backgroundExecutor()
);
} else
{
// Run on-thread
doIndex();
}
}
/**
* Performs the actual indexing of this manager.
* Can be invoked on- or off-thread.
*/
private void doIndex()
{
// Get the path to the root of the namespace in the current pack.
final Path rootPath = pathBuilder.apply(packType, namespace);
// Stream element resource that combines a normalized path and a joined path using the "/" as separator.
record PathWithLocationPath(Path path, String locationPath)
{
}
// Build a walkable stream, process it
try (final Stream<Path> paths = pathFinder.createWalkingStream(rootPath))
{
paths.parallel() // Run the stream in parallel
.map(rootPath::relativize) // Relative to the given root.
.map(path -> new PathWithLocationPath(path, Joiner.on('/').join(path))) // Create a common hierarchy.
.filter(path -> ResourceLocation.isValidPath(path.locationPath())) // Only process valid paths
.map(path -> new ResourceCacheEntry(packType, namespace, path.path(), new ResourceLocation(namespace, path.locationPath()))) // Create a cache entry.
.forEach(this::injectIntoCache); // Inject the entry into the cache.
} catch (NoSuchFileException noSuchFileException)
{
LOGGER.debug("Failed to cache resources, the directory does not exist!", noSuchFileException);
} catch (IOException ioException)
{
LOGGER.error("Failed to cache resources, some stuff might be missing!", ioException);
} catch (Exception exception)
{
LOGGER.error("Failed to cache resources, some stuff might be missing! Unknown exception!", exception);
} finally
{
cacheLoaded.set(true);
}
}
/**
* Injects the given cache entry into the cache.
* By first injecting it into the parent of the entry (so the directory it resides in) and then recursively walking up.
*
* @param entry The entry to inject into the cache, must not be null.
*/
private void injectIntoCache(final ResourceCacheEntry entry)
{
injectIntoCache(entry.path().getParent(), entry);
}
/**
* Injects the given cache entry into the cache, recursively into all of its parent paths.
*
* @param parentPath The parent path to inject into the cache, can be null. Null will be treated as the root path.
* @param entry The entry to inject into the cache, must not be null.
*/
private void injectIntoCache(@Nullable final Path parentPath, final ResourceCacheEntry entry)
{
String pathEntry;
if (parentPath == null)
{
// We have null, generally this only happens on the root directory.
pathEntry = "";
} else
{
// Use the parent paths, we can not depend on a toString call here, since we need the / as separator.
pathEntry = Joiner.on("/").join(parentPath);
}
// Inject into the cache, we can use a normal array list here since we have guarded against cache access later on.
this.entriesByPathPrefix.computeIfAbsent(pathEntry, e -> new CopyOnWriteArrayList<>()).add(entry);
// Recursively walk to the top, while preventing duplicate entries.
if (parentPath != null && !pathEntry.isEmpty())
{
this.injectIntoCache(parentPath.getParent(), entry);
}
}
/**
* Looks up resources in the given input path prefix as well as those that match the filter.
*
* @param inputPath The input path prefix to look in.
* @param filter The filter which the resources must match.
* @return A collection of resource location which match the given path prefix and who match the filter.
*/
public Collection<ResourceLocation> getResources(final Path inputPath, final Predicate<String> filter)
{
// We only have a cache once the atomic flag has been set.
if (!cacheLoaded()) {
// We need to return a mutable object here because the callers actually mutate the collection.
return new ArrayList<>();
}
// Use the input path prefix, we can not depend on a toString call here, since we need the / as separator.
final String pathEntry = Joiner.on('/').join(inputPath);
// Since we inject into the cache recursively we can now just grab the map entry that is the prefix and loop over its values.
// We use getOrDefault with a stream combination since the returned list needs to be mutable.
return entriesByPathPrefix.getOrDefault(pathEntry, Collections.emptyList()).stream()
.filter(entry -> filter.test(entry.path.getFileName().toString()))
.map(ResourceCacheEntry::resourceLocation)
.collect(Collectors.toList());
}
/**
* Indicates if the cache of this manager has been loaded.
*
* @return True if the cache has been loaded, false otherwise.
*/
public boolean cacheLoaded()
{
return cacheLoaded.get();
}
}
/**
* Functional callback interface to get a walkable stream of paths.
* Supports throwing {@link IOException} if the stream can not be created.
*/
@FunctionalInterface
private interface PathWalkerFactory
{
Stream<Path> createWalkingStream(Path path) throws IOException;
}
}
@@ -263,6 +263,9 @@ protected net.minecraft.data.tags.TagsProvider m_6648_(Lnet/minecraft/resources/
public net.minecraft.data.tags.TagsProvider$TagAppender f_126569_ # registry
public net.minecraft.gametest.framework.GameTestServer <init>(Ljava/lang/Thread;Lnet/minecraft/world/level/storage/LevelStorageSource$LevelStorageAccess;Lnet/minecraft/server/packs/repository/PackRepository;Lnet/minecraft/server/WorldStem;Ljava/util/Collection;Lnet/minecraft/core/BlockPos;)V # constructor
public net.minecraft.network.protocol.status.ClientboundStatusResponsePacket f_134885_ # GSON
public net.minecraft.resources.ResourceLocation m_135835_(C)Z # validNamespaceChar
public net.minecraft.resources.ResourceLocation m_135841_(Ljava/lang/String;)Z # isValidPath
public net.minecraft.resources.ResourceLocation m_135843_(Ljava/lang/String;)Z # isValidNamespace
protected net.minecraft.server.MinecraftServer f_129726_ # nextTickTime
public net.minecraft.server.MinecraftServer$ReloadableResources
public net.minecraft.server.dedicated.DedicatedServer f_139600_ # consoleInput
@@ -188,6 +188,13 @@
"forge.configgui.disableVersionCheck.tooltip": "Set to true to disable Forge version check mechanics. Forge queries a small json file on our server for version information. For more details see the ForgeVersion class in our github.",
"forge.configgui.disableVersionCheck": "Disable Forge Version Check",
"forge.configgui.cachePackAccess.tooltip": "Set this to true to cache resource listings in resource and data packs",
"forge.configgui.cachePackAccess": "Cache Pack Access",
"forge.configgui.indexVanillaPackCachesOnThread.tooltip": "Set this to true to index vanilla resource and data packs on thread",
"forge.configgui.indexVanillaPackCachesOnThread": "Index vanilla resource packs on thread",
"forge.configgui.indexModPackCachesOnThread.tooltip": "Set this to true to index mod resource and data packs on thread",
"forge.configgui.indexModPackCachesOnThread": "Index mod resource packs on thread",
"forge.controlsgui.shift": "SHIFT + %s",
"forge.controlsgui.control": "CTRL + %s",
"forge.controlsgui.control.mac": "CMD + %s",