Initial commit for shader-loading-optimizations (#2213)

This commit is contained in:
Eclipse-Sol authored and GitHub committed 2026-10-04 15:36:50 -07:00
1 parent 10922edca3
commit 38af64f10c
35 files changed
+1239 -194

No files matched your search

+178 -16
View File
@@ -17,6 +17,7 @@ import com.gtnewhorizons.angelica.proxy.ClientProxy;
import com.gtnewhorizons.angelica.rendering.StateAwareTessellator;
import com.gtnewhorizons.angelica.rendering.celeritas.api.IrisShaderProviderHolder;
import com.gtnewhorizons.angelica.sdlgpu.SDLGPUGate;
import com.gtnewhorizons.angelica.utils.AngelicaJar;
import cpw.mods.fml.client.registry.ClientRegistry;
import cpw.mods.fml.common.eventhandler.SubscribeEvent;
import cpw.mods.fml.common.gameevent.InputEvent;
@@ -29,6 +30,7 @@ import net.coderbot.iris.celeritas.IrisCeleritasShaderProvider;
import net.coderbot.iris.compat.dh.DHCompat;
import net.coderbot.iris.config.IrisConfig;
import net.coderbot.iris.gbuffer_overrides.matching.InputAvailability;
import net.coderbot.iris.gl.program.RetainedPrograms;
import net.coderbot.iris.gl.shader.StandardMacros;
import net.coderbot.iris.gui.screen.ShaderPackScreen;
import net.coderbot.iris.pipeline.DeferredWorldRenderingPipeline;
@@ -38,6 +40,7 @@ import net.coderbot.iris.pipeline.WorldRenderingPipeline;
import net.coderbot.iris.pipeline.transform.ShaderTransformer;
import net.coderbot.iris.pipeline.transform.TransformPatcher;
import net.coderbot.iris.shaderpack.OptionalBoolean;
import net.coderbot.iris.shaderpack.IdMap;
import net.coderbot.iris.shaderpack.ProgramSet;
import net.coderbot.iris.shaderpack.ShaderPack;
import net.coderbot.iris.shaderpack.StringPair;
@@ -67,19 +70,30 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.lang.ref.SoftReference;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystem;
import java.nio.file.FileSystemLoopException;
import java.nio.file.FileSystemNotFoundException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.LinkOption;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.TreeMap;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;
import java.util.stream.Stream;
import java.util.zip.ZipException;
@@ -107,6 +121,7 @@ public class Iris {
@Getter
private static String currentPackName;
private static String currentPackSettings;
private static String currentPackState;
@Getter
private static boolean initialized;
@Getter
@@ -288,6 +303,10 @@ public class Iris {
IrisShaderProviderHolder.setProvider(new IrisCeleritasShaderProvider());
// The pack loads below, before mod preInit would otherwise set up the cache
final Path angelicaJar = AngelicaJar.location();
AngelicaJar.configureShaderDiskCache(angelicaJar != null ? angelicaJar.toFile() : null);
// Warm up the threadpool so shader transformations are faster when we need them
ShaderTransformExecutor.warmup();
@@ -485,9 +504,7 @@ public class Iris {
}
}
@SuppressWarnings("unchecked")
final Map<String, String> changedConfigs = tryReadConfigProperties(shaderPackConfigTxt).map(properties -> (Map<String, String>) (Map<?, ?>) properties)
.orElse(new HashMap<>());
final Map<String, String> changedConfigs = readChangedConfigs(shaderPackConfigTxt);
changedConfigs.putAll(shaderPackOptionQueue);
clearShaderPackOptionQueue();
@@ -499,7 +516,12 @@ public class Iris {
try {
final Iterable<StringPair> environmentDefines = StandardMacros.createStandardEnvironmentDefines();
currentPack = new ShaderPack(shaderPackPath, changedConfigs, environmentDefines);
final String packState = describeSettings(name, shaderPackRoot, toProperties(changedConfigs), environmentDefines, false);
final ShaderPack prepared = takePreparedPack(name, packState);
currentPack = prepared != null ? prepared : new ShaderPack(shaderPackPath, changedConfigs, environmentDefines);
currentPackState = packState;
RetainedPrograms.useConfiguration(packState);
currentPack.activate();
final MutableOptionValues changedConfigsValues = currentPack.getShaderPackOptions().getOptionValues().mutableCopy();
@@ -509,7 +531,7 @@ public class Iris {
changedConfigsValues.getStringValues().forEach(configsToSave::setProperty);
tryUpdateConfigPropertiesFile(shaderPackConfigTxt, configsToSave);
currentPackSettings = describeSettings(name, shaderPackRoot, configsToSave, environmentDefines);
currentPackSettings = irisConfig.shouldSaveCompiledShaders() ? describeSettings(name, shaderPackRoot, configsToSave, environmentDefines, true) : null;
} catch (Exception e) {
logger.error("Failed to load the shaderpack \"{}\"!", name);
logger.error("", e);
@@ -526,10 +548,83 @@ public class Iris {
return true;
}
private record PreparedPack(String settings, ShaderPack pack) {}
private static volatile SoftReference<PreparedPack> preparedPack;
@Nullable
public static Runnable prepareSelectedShaderPack() {
if (irisConfig == null || irisConfig.areShadersEnabled() || currentPack != null) return null;
final Optional<String> name = irisConfig.getShaderPackName();
if (name.isEmpty()) return null;
final List<StringPair> environmentDefines = new ArrayList<>();
StandardMacros.createStandardEnvironmentDefines().forEach(environmentDefines::add);
return () -> {
try {
final PreparedPack prepared = readPreparedPack(name.get(), environmentDefines);
if (prepared != null) preparedPack = new SoftReference<>(prepared);
} catch (Exception e) {
logger.debug("Could not read shaderpack \"{}\" ahead of time", name.get(), e);
}
};
}
@Nullable
private static PreparedPack readPreparedPack(String name, List<StringPair> environmentDefines) throws IOException {
final Path shaderPackRoot = getShaderpacksDirectory().resolve(name);
final Map<String, String> changedConfigs = readChangedConfigs(getShaderpacksDirectory().resolve(name + ".txt"));
final String settings = describeSettings(name, shaderPackRoot, toProperties(changedConfigs), environmentDefines, false);
if (settings == null) return null;
if (Files.isDirectory(shaderPackRoot)) {
return new PreparedPack(settings, parsePack(shaderPackRoot.resolve("shaders"), changedConfigs, environmentDefines));
}
try (FileSystem zipSystem = FileSystems.newFileSystem(shaderPackRoot, Iris.class.getClassLoader())) {
final Optional<Path> shadersDirectory = findShadersDirectory(zipSystem);
if (shadersDirectory.isEmpty()) return null;
return new PreparedPack(settings, parsePack(shadersDirectory.get(), changedConfigs, environmentDefines));
}
}
private static ShaderPack parsePack(Path shadersDirectory, Map<String, String> changedConfigs, List<StringPair> environmentDefines) throws IOException {
final ShaderPack pack = new ShaderPack(shadersDirectory, changedConfigs, environmentDefines);
// Most worlds open in the overworld; other dimensions build their program sets on demand as usual.
// Considered having tracking of what dim a user is in but with it being so fast now, not sure if
// it's worth saving the 50-80ms "time waste" on first load, especially when we're already pre-loading at game init.
pack.getProgramSet("Overworld");
return pack;
}
@Nullable
private static ShaderPack takePreparedPack(String name, @Nullable String packState) {
final SoftReference<PreparedPack> reference = preparedPack;
preparedPack = null;
final PreparedPack prepared = reference != null ? reference.get() : null;
if (prepared == null || !prepared.settings().equals(packState)) return null;
logger.info("Reusing shaderpack \"{}\" read earlier, nothing it was read from has changed", name);
return prepared.pack();
}
private static Map<String, String> readChangedConfigs(Path shaderPackConfigTxt) {
@SuppressWarnings("unchecked")
final Map<String, String> changedConfigs = tryReadConfigProperties(shaderPackConfigTxt).map(properties -> (Map<String, String>) (Map<?, ?>) properties)
.orElse(new HashMap<>());
return new HashMap<>(changedConfigs);
}
private static Properties toProperties(Map<String, String> values) {
final Properties properties = new Properties();
properties.putAll(values);
return properties;
}
private static Optional<Path> loadExternalZipShaderpack(Path shaderpackPath) throws IOException {
final FileSystem zipSystem = FileSystems.newFileSystem(shaderpackPath, Iris.class.getClassLoader());
zipFileSystem = zipSystem;
return findShadersDirectory(zipSystem);
}
private static Optional<Path> findShadersDirectory(FileSystem zipSystem) throws IOException {
// Should only be one root directory for a zip shaderpack
final Path root = zipSystem.getRootDirectories().iterator().next();
@@ -552,6 +647,7 @@ public class Iris {
private static void setShadersDisabled() {
currentPack = null;
currentPackState = null;
currentPackSettings = null;
ProgramBinaryCache.usePack(null);
fallback = false;
@@ -710,7 +806,11 @@ public class Iris {
*/
private static void destroyEverything() {
awaitProgramSetWarmup();
if (currentPack != null && currentPackState != null) {
preparedPack = new SoftReference<>(new PreparedPack(currentPackState, currentPack));
}
currentPack = null;
currentPackState = null;
currentPackSettings = null;
ProgramBinaryCache.usePack(null);
@@ -809,6 +909,8 @@ public class Iris {
} catch (Exception e) {
everyFolderCompiled = false;
logger.error("Failed to compile shader folder '{}' for saving", folder, e);
} finally {
RetainedPrograms.afterPipelineBuilt();
}
}
loopFinished = true;
@@ -818,8 +920,65 @@ public class Iris {
}
}
private record HashedFile(long size, long modified, String hash) {}
private static final Map<Path, HashedFile> contentHashes = new ConcurrentHashMap<>();
private static String contentHash(Path file, BasicFileAttributes attributes) throws IOException {
final long size = attributes.size();
final long modified = attributes.lastModifiedTime().toMillis();
final HashedFile known = contentHashes.get(file);
if (known != null && known.size() == size && known.modified() == modified) return known.hash();
final String hash;
try {
hash = Base64.getEncoder().encodeToString(MessageDigest.getInstance("SHA-256").digest(Files.readAllBytes(file)));
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException(e);
}
contentHashes.put(file, new HashedFile(size, modified, hash));
return hash;
}
private record VisitedFolder(Path path, @Nullable Object key) {}
private static Map<String, String> describeFolder(Path root, VisitedFolder folder, List<VisitedFolder> ancestors, boolean byContent) {
final Map<String, String> files = new HashMap<>();
final List<Supplier<Map<String, String>>> subfolders = new ArrayList<>();
final List<VisitedFolder> lineage = new ArrayList<>(ancestors);
lineage.add(folder);
try (DirectoryStream<Path> entries = Files.newDirectoryStream(folder.path())) {
for (Path entry : entries) {
BasicFileAttributes attributes;
try {
attributes = Files.readAttributes(entry, BasicFileAttributes.class);
} catch (IOException e) {
attributes = Files.readAttributes(entry, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
}
if (attributes.isDirectory()) {
final VisitedFolder subfolder = new VisitedFolder(entry, attributes.fileKey());
for (VisitedFolder ancestor : lineage) {
final boolean same = subfolder.key() != null && ancestor.key() != null
? subfolder.key().equals(ancestor.key())
: Files.isSameFile(ancestor.path(), entry);
if (same) throw new FileSystemLoopException(entry.toString());
}
subfolders.add(() -> describeFolder(root, subfolder, lineage, byContent));
} else if (attributes.isRegularFile()) {
files.put(root.relativize(entry).toString(),
byContent ? contentHash(entry, attributes) : attributes.size() + "|" + attributes.lastModifiedTime().toMillis());
}
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
for (Map<String, String> nested : AngelicaWorkers.invokeAll(subfolders)) {
files.putAll(nested);
}
return files;
}
@Nullable
private static String describeSettings(String packName, Path packRoot, Properties options, Iterable<StringPair> environmentDefines) {
private static String describeSettings(String packName, Path packRoot, Properties options, Iterable<StringPair> environmentDefines, boolean byContent) {
final StringBuilder settings = new StringBuilder(packName).append('\n');
if (Files.isRegularFile(packRoot)) {
try {
@@ -828,18 +987,15 @@ public class Iris {
return null;
}
} else if (Files.isDirectory(packRoot)) {
try (Stream<Path> files = Files.walk(packRoot)) {
files.filter(Files::isRegularFile).sorted().forEach(file -> {
try {
settings.append(packRoot.relativize(file)).append('|').append(Files.size(file)).append('|')
.append(Files.getLastModifiedTime(file).toMillis()).append('\n');
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
final TreeMap<String, String> files = new TreeMap<>();
try {
// One stat per file; this runs on the render thread when shaders turn on, so folders are read in parallel
final Object rootKey = Files.readAttributes(packRoot, BasicFileAttributes.class).fileKey();
files.putAll(describeFolder(packRoot, new VisitedFolder(packRoot, rootKey), List.of(), byContent));
} catch (IOException | UncheckedIOException e) {
return null;
}
files.forEach((file, state) -> settings.append(file).append('|').append(state).append('\n'));
} else {
return null;
}
@@ -848,6 +1004,7 @@ public class Iris {
for (StringPair define : environmentDefines) {
settings.append('#').append(define.getKey()).append(' ').append(define.getValue()).append('\n');
}
settings.append("modernFallbackMcVersion=").append(IdMap.modernFallbackMcVersion()).append('\n');
return settings.toString();
}
@@ -870,7 +1027,12 @@ public class Iris {
ShaderTransformExecutor.prepare();
PerFrameUniformBlockHarvester.harvest(programs);
long startTime = System.nanoTime();
WorldRenderingPipeline pipeline = new DeferredWorldRenderingPipeline(programs);
final WorldRenderingPipeline pipeline;
try {
pipeline = new DeferredWorldRenderingPipeline(programs);
} finally {
RetainedPrograms.afterPipelineBuilt();
}
long endTime = System.nanoTime();
logger.info("[Load #{}] Total shaderpack load time for '{}' in dimension '{}': {} ms", shaderPackLoadId, currentPackName, dimensionName, String.format("%.1f", (endTime - startTime) / 1_000_000.0));
return pipeline;
@@ -10,6 +10,7 @@ public abstract class GlResource {
}
public final void destroy() {
if (!isValid) return;
destroyInternal();
isValid = false;
}
@@ -6,6 +6,7 @@ import net.coderbot.iris.gl.GlResource;
import net.coderbot.iris.gl.blending.DepthColorStorage;
import net.coderbot.iris.pipeline.WorldRenderingPipeline;
import net.coderbot.iris.shaderpack.FilledIndirectPointer;
import org.jetbrains.annotations.Nullable;
import org.joml.Vector2f;
import org.joml.Vector3i;
import org.lwjgl.BufferUtils;
@@ -27,9 +28,14 @@ public final class ComputeProgram extends GlResource {
private float cachedHeight;
private Vector3i cachedWorkGroups;
ComputeProgram(int program, ProgramUniforms uniforms, ProgramSamplers samplers, ProgramImages images) {
@Nullable
private final RetainedPrograms.Sources sources;
ComputeProgram(int program, @Nullable RetainedPrograms.Sources sources, ProgramUniforms uniforms, ProgramSamplers samplers, ProgramImages images) {
super(program);
this.sources = sources;
localSizeBuffer = BufferUtils.createIntBuffer(3);
RenderSystem.getProgramiv(program, GL43.GL_COMPUTE_WORK_GROUP_SIZE, localSizeBuffer);
this.uniforms = uniforms;
@@ -93,7 +99,11 @@ public final class ComputeProgram extends GlResource {
@Override
public void destroyInternal() {
DepthColorStorage.unregisterOwnedProgram(getGlId());
GLStateManager.glDeleteProgram(getGlId());
if (sources != null) {
RetainedPrograms.retain(sources, getGlId());
} else {
GLStateManager.glDeleteProgram(getGlId());
}
}
/**
@@ -5,6 +5,7 @@ import net.coderbot.iris.gl.GlResource;
import net.coderbot.iris.gl.blending.DepthColorStorage;
import com.gtnewhorizons.angelica.glsm.GLStateManager;
import com.gtnewhorizons.angelica.glsm.RenderSystem;
import org.jetbrains.annotations.Nullable;
import org.lwjgl.opengl.GL42;
import org.lwjgl.opengl.GL43;
@@ -25,9 +26,14 @@ public final class Program extends GlResource {
@Getter
private final ProgramImages images;
Program(int program, ProgramUniforms uniforms, ProgramSamplers samplers, ProgramImages images) {
@Nullable
private final RetainedPrograms.Sources sources;
Program(int program, @Nullable RetainedPrograms.Sources sources, ProgramUniforms uniforms, ProgramSamplers samplers, ProgramImages images) {
super(program);
this.sources = sources;
this.uniforms = uniforms;
this.samplers = samplers;
this.images = images;
@@ -63,7 +69,11 @@ public final class Program extends GlResource {
@Override
public void destroyInternal() {
DepthColorStorage.unregisterOwnedProgram(getGlId());
GLStateManager.glDeleteProgram(getGlId());
if (sources != null) {
RetainedPrograms.retain(sources, getGlId());
} else {
GLStateManager.glDeleteProgram(getGlId());
}
}
public int getProgramId() {
@@ -21,6 +21,7 @@ import java.util.function.IntSupplier;
public class ProgramBuilder extends ProgramUniforms.Builder implements SamplerHolder, ImageHolder {
private final int program;
private RetainedPrograms.Sources sources;
private final ProgramSamplers.Builder samplers;
private final ProgramImages.Builder images;
@@ -44,17 +45,29 @@ public class ProgramBuilder extends ProgramUniforms.Builder implements SamplerHo
public static ProgramBuilder begin(String name, @Nullable String vertexSource, @Nullable String geometrySource,
@Nullable String tessControlSource, @Nullable String tessEvalSource,
@Nullable String fragmentSource, ImmutableSet<Integer> reservedTextureUnits) {
final ProgramBinaryCache.Key cacheKey = ProgramCreator.cacheKey();
final RetainedPrograms.Sources sources = new RetainedPrograms.Sources(vertexSource, geometrySource, tessControlSource, tessEvalSource, fragmentSource, null);
final int retained = RetainedPrograms.take(sources);
final ProgramBinaryCache.Key cacheKey = retained != 0 && !ProgramBinaryCache.isFullCompileRunning() ? null : ProgramCreator.cacheKey();
if (cacheKey != null) {
cacheKey.stage(ShaderType.VERTEX.id, vertexSource);
if (geometrySource != null) cacheKey.stage(ShaderType.GEOMETRY.id, geometrySource);
if (tessControlSource != null) cacheKey.stage(ShaderType.TESSELATION_CONTROL.id, tessControlSource);
if (tessEvalSource != null) cacheKey.stage(ShaderType.TESSELATION_EVAL.id, tessEvalSource);
cacheKey.stage(ShaderType.FRAGMENT.id, fragmentSource);
}
if (retained != 0) {
if (cacheKey == null || ProgramBinaryCache.saveRetained(cacheKey, retained)) {
initializeFallbackUniforms(retained);
return new ProgramBuilder(name, retained, reservedTextureUnits).retainAs(sources);
}
GLStateManager.glDeleteProgram(retained);
}
if (cacheKey != null) {
final int saved = ProgramCreator.load(name, cacheKey);
if (saved != 0) {
initializeFallbackUniforms(saved);
return new ProgramBuilder(name, saved, reservedTextureUnits);
return new ProgramBuilder(name, saved, reservedTextureUnits).retainAs(sources);
}
}
@@ -78,7 +91,7 @@ public class ProgramBuilder extends ProgramUniforms.Builder implements SamplerHo
shader.destroy();
}
return new ProgramBuilder(name, programId, reservedTextureUnits);
return new ProgramBuilder(name, programId, reservedTextureUnits).retainAs(sources);
}
/** Initialize alpha and color uniforms to neutral values until the first pass update. */
@@ -122,12 +135,23 @@ public class ProgramBuilder extends ProgramUniforms.Builder implements SamplerHo
throw new IllegalStateException("This PC does not support compute shaders, but it's attempting to be used???");
}
final ProgramBinaryCache.Key cacheKey = ProgramCreator.cacheKey();
final RetainedPrograms.Sources sources = new RetainedPrograms.Sources(null, null, null, null, null, source);
final int retained = RetainedPrograms.take(sources);
final ProgramBinaryCache.Key cacheKey = retained != 0 && !ProgramBinaryCache.isFullCompileRunning() ? null : ProgramCreator.cacheKey();
if (cacheKey != null) {
cacheKey.stage(ShaderType.COMPUTE.id, source);
}
if (retained != 0) {
if (cacheKey == null || ProgramBinaryCache.saveRetained(cacheKey, retained)) {
return new ProgramBuilder(name, retained, reservedTextureUnits).retainAs(sources);
}
GLStateManager.glDeleteProgram(retained);
}
if (cacheKey != null) {
final int saved = ProgramCreator.load(name, cacheKey);
if (saved != 0) {
return new ProgramBuilder(name, saved, reservedTextureUnits);
return new ProgramBuilder(name, saved, reservedTextureUnits).retainAs(sources);
}
}
@@ -137,15 +161,20 @@ public class ProgramBuilder extends ProgramUniforms.Builder implements SamplerHo
compute.destroy();
return new ProgramBuilder(name, programId, reservedTextureUnits);
return new ProgramBuilder(name, programId, reservedTextureUnits).retainAs(sources);
}
private ProgramBuilder retainAs(RetainedPrograms.Sources sources) {
this.sources = sources;
return this;
}
public Program build() {
return new Program(program, super.buildUniforms(), this.samplers.build(), this.images.build());
return new Program(program, sources, super.buildUniforms(), this.samplers.build(), this.images.build());
}
public ComputeProgram buildCompute() {
return new ComputeProgram(program, super.buildUniforms(), this.samplers.build(), this.images.build());
return new ComputeProgram(program, sources, super.buildUniforms(), this.samplers.build(), this.images.build());
}
private static GlShader buildShader(ShaderType shaderType, String name, @Nullable String source) {
@@ -0,0 +1,60 @@
package net.coderbot.iris.gl.program;
import com.gtnewhorizons.angelica.glsm.GLStateManager;
import org.jetbrains.annotations.Nullable;
import java.util.ArrayDeque;
import java.util.HashMap;
import java.util.Map;
/**
* Linked programs kept from destroyed pipelines. Turning shaders off and on, reloading an unchanged pack, or
* travelling between two dimensions rebuilds programs from identical sources; taking them from here skips compile and
* link.
*/
public final class RetainedPrograms {
record Sources(@Nullable String vertex, @Nullable String geometry, @Nullable String tessControl,
@Nullable String tessEval, @Nullable String fragment, @Nullable String compute) {}
private static Map<Sources, ArrayDeque<Integer>> recent = new HashMap<>();
private static Map<Sources, ArrayDeque<Integer>> older = new HashMap<>();
private static String configuration;
private RetainedPrograms() {}
public static void useConfiguration(@Nullable String settings) {
if (settings == null || !settings.equals(configuration)) {
afterPipelineBuilt();
afterPipelineBuilt();
}
configuration = settings;
}
static void retain(Sources sources, int program) {
recent.computeIfAbsent(sources, key -> new ArrayDeque<>()).add(program);
}
static int take(Sources sources) {
final int program = take(recent, sources);
return program != 0 ? program : take(older, sources);
}
private static int take(Map<Sources, ArrayDeque<Integer>> pool, Sources sources) {
if (pool.isEmpty()) return 0;
final ArrayDeque<Integer> programs = pool.get(sources);
if (programs == null) return 0;
final int program = programs.poll();
if (programs.isEmpty()) pool.remove(sources);
return program;
}
public static void afterPipelineBuilt() {
for (ArrayDeque<Integer> programs : older.values()) {
for (int program : programs) {
GLStateManager.glDeleteProgram(program);
}
}
older = recent;
recent = new HashMap<>();
}
}
@@ -543,26 +543,12 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R
this.shadowComputes = createShadowComputes(programs.getShadowCompute());
final Map<SharedProgramKey, BuiltProgram> builtPrograms = new HashMap<>();
this.table = new ProgramTable<>((condition, availability) -> {
final int idx;
final ProgramId finalId = gbufferProgramId(ids, condition, availability);
if (availability.texture && availability.lightmap) {
idx = 2;
} else if (availability.texture) {
idx = 1;
} else {
idx = 0;
}
ProgramId id = ids[condition.ordinal() * 3 + idx];
if (id == null) {
id = ids[idx];
}
final ProgramId finalId = id;
return cachedPasses.computeIfAbsent(Pair.of(id, availability), p -> {
return cachedPasses.computeIfAbsent(Pair.of(finalId, availability), p -> {
final ProgramSource source = resolver.resolveNullable(p.getLeft());
if (condition == RenderCondition.SHADOW || condition == RenderCondition.SHADOW_TRANSLUCENT) {
@@ -583,7 +569,7 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R
try {
return createPass(source, availability,
condition == RenderCondition.SHADOW || condition == RenderCondition.SHADOW_TRANSLUCENT, finalId);
condition == RenderCondition.SHADOW || condition == RenderCondition.SHADOW_TRANSLUCENT, finalId, builtPrograms);
} catch (Exception e) {
throw new RuntimeException("Failed to create pass for " + source.getName() + " for rendering condition "
+ condition + " specialized to input availability " + availability, e);
@@ -1131,6 +1117,25 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R
null, Collections.emptyList(), false);
}
private static ProgramId gbufferProgramId(ProgramId[] ids, RenderCondition condition, InputAvailability availability) {
final int idx;
if (availability.texture && availability.lightmap) {
idx = 2;
} else if (availability.texture) {
idx = 1;
} else {
idx = 0;
}
final ProgramId id = ids[condition.ordinal() * 3 + idx];
return id != null ? id : ids[idx];
}
private record SharedProgramKey(String sourceName, InputAvailability availability, boolean shadow) {}
private record BuiltProgram(Program program, Pass firstPass) {}
private static ProgramBuilder beginBuilder(String name, Map<PatchShaderType, String> transformed) {
final String vertex = transformed.get(PatchShaderType.VERTEX);
final String geometry = transformed.get(PatchShaderType.GEOMETRY);
@@ -1143,11 +1148,22 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R
return ProgramBuilder.begin(name, vertex, geometry, tessControl, tessEval, fragment, IrisSamplers.WORLD_RESERVED_TEXTURE_UNITS);
}
private Pass createPass(ProgramSource source, InputAvailability availability, boolean shadow, ProgramId id) {
private Pass createPass(ProgramSource source, InputAvailability availability, boolean shadow, ProgramId id,
Map<SharedProgramKey, BuiltProgram> builtPrograms) {
// Use pre-computed transform if available, otherwise transform synchronously
Pair<String, InputAvailability> key = Pair.of(source.getName(), availability);
Map<PatchShaderType, String> transformed = attributeTransforms.get(key);
// The synchronous fallback below depends on the program id, so only precomputed programs are shared
final SharedProgramKey shareKey = transformed != null ? new SharedProgramKey(source.getName(), availability, shadow) : null;
final BuiltProgram shared = shareKey != null ? builtPrograms.get(shareKey) : null;
if (shared != null) {
final Pass pass = createPassInner(shared.program(), source.getDirectives(), shadow, id);
this.customUniforms.mapPassLike(shared.firstPass(), pass);
pass.setInstancingKey(source.getName(), availability, shadow);
return pass;
}
if (transformed == null) {
// Fallback to synchronous transform if not pre-computed
String vertex = source.getVertexSource().orElseThrow(NullPointerException::new);
@@ -1163,9 +1179,15 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R
}
ProgramBuilder builder = beginBuilder(source.getName(), transformed);
wireGbufferProgram(builder, availability, shadow);
final Program program = builder.build();
final Pass pass = createPassInner(builder, source.getDirectives(), availability, shadow, id);
final Pass pass = createPassInner(program, source.getDirectives(), shadow, id);
this.customUniforms.mapholderToPass(builder, pass);
pass.setInstancingKey(source.getName(), availability, shadow);
if (shareKey != null) {
builtPrograms.put(shareKey, new BuiltProgram(program, pass));
}
return pass;
}
@@ -1212,10 +1234,7 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R
}
}
private Pass createPassInner(ProgramBuilder builder, ProgramDirectives programDirectives, InputAvailability availability, boolean shadow, ProgramId id) {
wireGbufferProgram(builder, availability, shadow);
private Pass createPassInner(Program program, ProgramDirectives programDirectives, boolean shadow, ProgramId id) {
GlFramebuffer framebufferBeforeTranslucents;
GlFramebuffer framebufferAfterTranslucents;
@@ -1240,12 +1259,8 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R
}
});
Pass pass = new Pass(builder.build(), framebufferBeforeTranslucents, framebufferAfterTranslucents, alphaTestOverride,
return new Pass(program, framebufferBeforeTranslucents, framebufferAfterTranslucents, alphaTestOverride,
programDirectives.getBlendModeOverride().orElse(id.getBlendModeOverride()), bufferOverrides, shadow);
this.customUniforms.mapholderToPass(builder, pass);
return pass;
}
public void addGbufferOrShadowSamplers(SamplerHolder samplers, ImageHolder images, Supplier<ImmutableSet<Integer>> flipped,
@@ -1468,7 +1483,7 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R
if (!instancedVariantAttempted[i]) {
instancedVariantAttempted[i] = true;
if (instancingSourceName != null && supportsInstancing[i]) {
instancedVariants[i] = buildInstancedVariant(instancingSourceName, instancingAvailability, instancingShadow, kind);
instancedVariants[i] = sharedInstancedVariant(instancingSourceName, instancingAvailability, instancingShadow, kind);
}
}
return instancedVariants[i];
@@ -1487,6 +1502,20 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R
}
}
private record InstancedVariantKey(String sourceName, InputAvailability availability, boolean shadow, Instancing kind) {}
// Passes sharing a program also share its instanced variants
private final Map<InstancedVariantKey, Program> instancedVariantPrograms = new HashMap<>();
@Nullable
private Program sharedInstancedVariant(String sourceName, InputAvailability availability, boolean shadow, Instancing kind) {
final InstancedVariantKey key = new InstancedVariantKey(sourceName, availability, shadow, kind);
if (instancedVariantPrograms.containsKey(key)) return instancedVariantPrograms.get(key);
final Program variant = buildInstancedVariant(sourceName, availability, shadow, kind);
instancedVariantPrograms.put(key, variant);
return variant;
}
@Nullable
private Program buildInstancedVariant(String sourceName, InputAvailability availability, boolean shadow, Instancing kind) {
final Map<Pair<String, InputAvailability>, Map<PatchShaderType, String>> transforms = instancedAttributeTransforms.get(kind);
@@ -2440,6 +2469,16 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R
if (source == null) {
return false;
}
boolean mentioned = false;
for (String builtin : MV_BUILTINS) {
if (source.contains(builtin)) {
mentioned = true;
break;
}
}
if (!mentioned) {
return false;
}
try {
final Transformer transformer = new Transformer(ShaderParser.parseShader(source).full());
for (String builtin : MV_BUILTINS) {
@@ -576,16 +576,16 @@ public class ShaderTransformer {
if (vshResult != null && fshResult != null) {
throw new RuntimeException("Program " + patchType.name() + " writes images from both VSH and FSH; only one stage may write per program under SDL_GPU");
}
if (vshResult != null) {
result.put(PatchShaderType.VERTEX, vshResult.strippedSource());
result.put(PatchShaderType.COMPUTE, vshResult.computeSource());
if (artifactsOut != null) artifactsOut.remove(PatchShaderType.VERTEX);
Iris.logger.info("[RwImageStoreExtractor] Extracted compute pre-pass for {} (mode={}, written={})", patchType.name(), vshResult.mode(), vshResult.writtenImages());
} else if (fshResult != null) {
result.put(PatchShaderType.FRAGMENT, fshResult.strippedSource());
result.put(PatchShaderType.COMPUTE, fshResult.computeSource());
if (artifactsOut != null) artifactsOut.remove(PatchShaderType.FRAGMENT);
Iris.logger.info("[RwImageStoreExtractor] Extracted compute pre-pass for {} (mode={}, written={})", patchType.name(), fshResult.mode(), fshResult.writtenImages());
final RwImageStoreExtractor.Result extracted = vshResult != null ? vshResult : fshResult;
if (extracted == null) return;
final PatchShaderType stage = vshResult != null ? PatchShaderType.VERTEX : PatchShaderType.FRAGMENT;
result.put(stage, extracted.strippedSource());
if (artifactsOut != null) artifactsOut.remove(stage);
if (patchType == Patch.COMPOSITE || patchType == Patch.CELERITAS_TERRAIN) {
result.put(PatchShaderType.COMPUTE, extracted.computeSource());
Iris.logger.info("[RwImageStoreExtractor] Extracted compute pre-pass for {} (mode={}, written={})", patchType.name(), extracted.mode(), extracted.writtenImages());
} else {
Iris.logger.info("[RwImageStoreExtractor] Dropped image writes from {} (written={})", patchType.name(), extracted.writtenImages());
}
}
@@ -8,15 +8,25 @@ public class ConstDirectiveParser {
public static List<ConstDirective> findDirectives(String source) {
List<ConstDirective> directives = new ArrayList<>();
// Match any valid newline sequence
// https://stackoverflow.com/a/31060125
for (String line : source.split("\\R")) {
findDirectiveInLine(line).ifPresent(directives::add);
int from = 0;
while (true) {
final int match = source.indexOf("const", from);
if (match < 0) break;
int start = match;
while (start > 0 && !isLineBreak(source.charAt(start - 1))) start--;
int end = match + "const".length();
while (end < source.length() && !isLineBreak(source.charAt(end))) end++;
findDirectiveInLine(source.substring(start, end)).ifPresent(directives::add);
from = end;
}
return directives;
}
private static boolean isLineBreak(char c) {
return c == '\n' || c == '\r' || c == '\u000B' || c == '\f' || c == '\u0085' || c == '\u2028' || c == '\u2029';
}
public static Optional<ConstDirective> findDirectiveInLine(String line) {
// Valid const directives contain the following elements:
// * Zero or more whitespace characters
@@ -81,8 +81,18 @@ public class IdMap {
* Detects whether a pack's block.properties has a dedicated 1.7.10 section.
*/
public static boolean detectLegacySection(Path shaderPath) {
String rawBlockProperties = readProperties(shaderPath, "block.properties");
return rawBlockProperties != null && LEGACY_DIRECTIVE_PATTERN.matcher(rawBlockProperties).find();
final String rawBlockProperties = readProperties(shaderPath, "block.properties");
if (rawBlockProperties == null) return false;
int from = 0;
while (true) {
final int match = rawBlockProperties.indexOf("10710", from);
if (match < 0) return false;
final int start = rawBlockProperties.lastIndexOf('\n', match) + 1;
int end = rawBlockProperties.indexOf('\n', match);
if (end < 0) end = rawBlockProperties.length();
if (LEGACY_DIRECTIVE_PATTERN.matcher(rawBlockProperties.substring(start, end)).find()) return true;
from = end;
}
}
public static String modernFallbackMcVersion() {
@@ -90,9 +100,8 @@ public class IdMap {
? String.valueOf(AngelicaConfig.modernFallbackMcVersion) : "260101";
}
IdMap(Path shaderPath, ShaderPackOptions shaderPackOptions, Iterable<StringPair> environmentDefines) {
// Check if block.properties has a dedicated 1.7.10 section
this.hasLegacySection = detectLegacySection(shaderPath);
IdMap(Path shaderPath, boolean hasLegacySection, ShaderPackOptions shaderPackOptions, Iterable<StringPair> environmentDefines) {
this.hasLegacySection = hasLegacySection;
Iterable<StringPair> resolvedDefines;
if (this.hasLegacySection) {
@@ -5,6 +5,7 @@ import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.stream.JsonReader;
import com.gtnewhorizons.angelica.glsm.threading.AngelicaWorkers;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
@@ -36,6 +37,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
@@ -53,7 +55,10 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
public class ShaderPack {
@@ -79,6 +84,7 @@ public class ShaderPack {
private final boolean mappedByDimensionProperties;
private final String legacyDefaultFolder;
private final Function<AbsolutePackPath, String> sourceProvider;
private final ImmutableList<AbsolutePackPath> sourceFiles;
private final ShaderProperties shaderProperties;
private boolean hasLoggedCacheLimitReached = false;
@@ -90,7 +96,8 @@ public class ShaderPack {
@Getter private final Object2ObjectMap<String, ImageInformation> customImages;
@Getter private final Int2ObjectMap<ShaderStorageInfo> bufferObjects;
@Getter private final ShaderPackOptions shaderPackOptions;
@Getter private final OptionMenuContainer menuContainer;
private final ProfileSet profiles;
private OptionMenuContainer menuContainer;
private final ProfileSet.ProfileResult profile;
private final String profileInfo;
@@ -200,8 +207,18 @@ public class ShaderPack {
}
this.legacyDefaultFolder = legacyDefault;
// Only the options screen reads translations, so they load alongside the rest of the pack
final CompletableFuture<LanguageMap> languageMapFuture = AngelicaWorkers.submit(() -> {
try {
return new LanguageMap(root.resolve("lang"));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
// Read all files and included files recursively
IncludeGraph graph = new IncludeGraph(root, starts.build());
this.sourceFiles = starts.build();
IncludeGraph graph = new IncludeGraph(root, sourceFiles);
if (!graph.getFailures().isEmpty()) {
graph.getFailures().forEach((path, error) -> {
@@ -211,13 +228,14 @@ public class ShaderPack {
throw new IOException("Failed to resolve some #include directives, see previous messages for details");
}
this.languageMap = new LanguageMap(root.resolve("lang"));
// Discover, merge, and apply shader pack options
this.shaderPackOptions = new ShaderPackOptions(graph, changedConfigs);
graph = this.shaderPackOptions.getIncludes();
final String mcVersionOverride = IdMap.detectLegacySection(root) ? null : IdMap.modernFallbackMcVersion();
final boolean hasLegacyIdSection = IdMap.detectLegacySection(root);
final String mcVersionOverride = hasLegacyIdSection ? null : IdMap.modernFallbackMcVersion();
final ShaderPackOptions options = this.shaderPackOptions;
final CompletableFuture<IdMap> idMapFuture = AngelicaWorkers.submit(() -> new IdMap(root, hasLegacyIdSection, options, environmentDefines));
List<StringPair> finalEnvironmentDefines = new ArrayList<>();
for (StringPair define : environmentDefines) {
@@ -275,7 +293,7 @@ public class ShaderPack {
}
});
this.menuContainer = new OptionMenuContainer(shaderProperties, this.shaderPackOptions, profiles);
this.profiles = profiles;
{
String profileName = getCurrentProfileName();
@@ -305,38 +323,39 @@ public class ShaderPack {
return null;
}
ImmutableList<String> lines = includeProcessor.getIncludedFile(path);
final byte[] contentHash = includeProcessor.getIncludedFileHash(path);
if (lines == null) {
if (contentHash == null) {
return null;
}
StringBuilder builder = new StringBuilder();
for (String line : lines) {
builder.append(line);
builder.append('\n');
}
// Apply GLSL preprocessor to source, while making environment defines available.
//
// This uses similar techniques to the *.properties preprocessor to avoid actually putting
// #define statements in the actual source - instead, we tell the preprocessor about them
// directly. This removes one obstacle to accurate reporting of line numbers for errors,
// though there exist many more (such as relocating all #extension directives and similar things)
String source = builder.toString();
return JcppProcessor.glslPreprocessSource(contentHash, () -> {
final ImmutableList<String> lines = includeProcessor.getIncludedFile(path);
int length = 0;
for (String line : lines) {
length += line.length() + 1;
}
// Apply shader pack workarounds for version compatibility (before preprocessing)
source = ShaderPackWorkarounds.apply(source);
final StringBuilder builder = new StringBuilder(length);
source = JcppProcessor.glslPreprocessSource(source, finalEnvironmentDefines1);
for (String line : lines) {
builder.append(line);
builder.append('\n');
}
return source;
// Apply shader pack workarounds for version compatibility (before preprocessing)
return ShaderPackWorkarounds.apply(builder.toString());
}, finalEnvironmentDefines1);
};
this.base = new ProgramSet(AbsolutePackPath.fromAbsolutePath("/"), sourceProvider, shaderProperties, this);
this.idMap = new IdMap(root, shaderPackOptions, environmentDefines);
final AbsolutePackPath rootDirectory = AbsolutePackPath.fromAbsolutePath("/");
this.base = new ProgramSet(rootDirectory, preprocessInParallel(rootDirectory), shaderProperties, this);
customNoiseTexture = shaderProperties.getNoiseTexturePath().map(path -> {
try {
@@ -375,10 +394,35 @@ public class ShaderPack {
this.customImages = shaderProperties.getCustomImages();
this.bufferObjects = shaderProperties.getBufferObjects();
this.idMap = await(idMapFuture);
this.languageMap = await(languageMapFuture);
}
// Only the options screen reads the menu, don't build unless it's needed
public OptionMenuContainer getMenuContainer() {
if (menuContainer == null) {
menuContainer = new OptionMenuContainer(shaderProperties, shaderPackOptions, profiles);
}
return menuContainer;
}
/** Publishes this pack's custom images to the shader transformers; call when the pack becomes the current one. */
public void activate() {
SamplerToStorageImageRewriter.setActiveCandidates(SamplerToStorageImageRewriter.buildCandidates(this.customImages.values()));
RwImageStoreExtractor.setActiveCustomImages(this.customImages);
}
private static <T> T await(CompletableFuture<T> future) throws IOException {
try {
return future.join();
} catch (CompletionException e) {
if (e.getCause() instanceof UncheckedIOException io) throw io.getCause();
if (e.getCause() instanceof RuntimeException runtime) throw runtime;
if (e.getCause() instanceof Error error) throw error;
throw e;
}
}
private String getCurrentProfileName() {
return profile.current.map(p -> p.name).orElse("Custom");
}
@@ -667,6 +711,26 @@ public class ShaderPack {
}
}
private Function<AbsolutePackPath, String> preprocessInParallel(AbsolutePackPath directory) {
final String directoryPath = directory.getPathString();
final String prefix = directoryPath.endsWith("/") ? directoryPath : directoryPath + "/";
final List<AbsolutePackPath> files = new ArrayList<>();
final List<Supplier<String>> tasks = new ArrayList<>();
for (AbsolutePackPath file : sourceFiles) {
final String path = file.getPathString();
if (path.startsWith(prefix) && path.indexOf('/', prefix.length()) < 0) {
files.add(file);
tasks.add(() -> sourceProvider.apply(file));
}
}
final List<String> sources = AngelicaWorkers.invokeAll(tasks);
final Map<AbsolutePackPath, String> preprocessed = new HashMap<>();
for (int i = 0; i < files.size(); i++) {
preprocessed.put(files.get(i), sources.get(i));
}
return path -> preprocessed.containsKey(path) ? preprocessed.get(path) : sourceProvider.apply(path);
}
/**
* Gets or creates the appropriate ProgramSet for the given dimension.
*
@@ -721,12 +785,8 @@ public class ShaderPack {
if (programSet == null) {
// Create ProgramSet on-demand for dimension folder
try {
programSet = new ProgramSet(
AbsolutePackPath.fromAbsolutePath("/" + folderName),
sourceProvider,
shaderProperties,
this
);
final AbsolutePackPath directory = AbsolutePackPath.fromAbsolutePath("/" + folderName);
programSet = new ProgramSet(directory, preprocessInParallel(directory), shaderProperties, this);
dimensionProgramSets.put(folderName, programSet);
// Check cache size limit and evict LRU dimensions if needed
@@ -2,6 +2,7 @@ package net.coderbot.iris.shaderpack.include;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.gtnewhorizons.angelica.glsm.threading.AngelicaWorkers;
import net.coderbot.iris.Iris;
import net.coderbot.iris.shaderpack.error.RusticError;
import net.coderbot.iris.shaderpack.transform.line.LineTransform;
@@ -18,6 +19,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
/**
@@ -75,14 +77,18 @@ public class IncludeGraph {
List<AbsolutePackPath> queue = new ArrayList<>(startingPaths);
Set<AbsolutePackPath> seen = new HashSet<>(startingPaths);
Map<AbsolutePackPath, CompletableFuture<ReadResult>> reads = new HashMap<>();
for (AbsolutePackPath start : startingPaths) {
reads.put(start, readLater(root, start));
}
while (!queue.isEmpty()) {
AbsolutePackPath next = queue.remove(queue.size() - 1);
String source;
final ReadResult read = reads.remove(next).join();
final IOException e = read.failure();
try {
source = readFile(next.resolved(root));
} catch (IOException e) {
if (e != null) {
AbsolutePackPath src = cameFrom.get(next);
if (src == null) {
@@ -110,7 +116,7 @@ public class IncludeGraph {
continue;
}
ImmutableList<String> lines = ImmutableList.copyOf(source.split("\\R"));
ImmutableList<String> lines = read.lines();
FileNode node = new FileNode(next, lines);
boolean selfInclude = false;
@@ -128,6 +134,7 @@ public class IncludeGraph {
} else if (!seen.contains(included)) {
queue.add(included);
seen.add(included);
reads.put(included, readLater(root, included));
cameFrom.put(included, next);
lineNumberInclude.put(included, line);
}
@@ -254,6 +261,18 @@ public class IncludeGraph {
return failures;
}
private record ReadResult(ImmutableList<String> lines, IOException failure) {}
private static CompletableFuture<ReadResult> readLater(Path root, AbsolutePackPath path) {
return AngelicaWorkers.submit(() -> {
try {
return new ReadResult(ImmutableList.copyOf(readFile(path.resolved(root)).split("\\R")), null);
} catch (IOException e) {
return new ReadResult(null, e);
}
});
}
private static String readFile(Path path) throws IOException {
return new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
}
@@ -3,18 +3,69 @@ package net.coderbot.iris.shaderpack.include;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import java.util.HashMap;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
// TODO: Write tests for this code
public class IncludeProcessor {
private final IncludeGraph graph;
private final Map<AbsolutePackPath, ImmutableList<String>> cache;
private final Map<AbsolutePackPath, byte[]> hashes;
public IncludeProcessor(IncludeGraph graph) {
this.graph = graph;
this.cache = new HashMap<>();
this.cache = new ConcurrentHashMap<>();
this.hashes = new ConcurrentHashMap<>();
}
public byte[] getIncludedFileHash(AbsolutePackPath path) {
byte[] hash = hashes.get(path);
if (hash == null) {
hash = hash(path);
if (hash != null) hashes.put(path, hash);
}
return hash;
}
private byte[] hash(AbsolutePackPath path) {
final FileNode fileNode = graph.getNodes().get(path);
if (fileNode == null) {
return null;
}
final MessageDigest digest;
try {
digest = MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException(e);
}
final ImmutableList<String> lines = fileNode.getLines();
final ImmutableMap<Integer, AbsolutePackPath> includes = fileNode.getIncludes();
// A line can't contain '\n' and an include is a fixed 32 bytes, so the encoding is unambiguous
for (int i = 0; i < lines.size(); i++) {
final AbsolutePackPath include = includes.get(i);
if (include != null) {
digest.update((byte) 1);
digest.update(Objects.requireNonNull(getIncludedFileHash(include)));
} else {
digest.update((byte) 0);
digest.update(lines.get(i).getBytes(StandardCharsets.UTF_8));
digest.update((byte) '\n');
}
}
return digest.digest();
}
// TODO: Actual error handling
@@ -24,7 +75,7 @@ public class IncludeProcessor {
if (lines == null) {
lines = process(path);
cache.put(path, lines);
if (lines != null) cache.put(path, lines);
}
return lines;
@@ -1,5 +1,6 @@
package net.coderbot.iris.shaderpack.preprocessor;
import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache;
import net.coderbot.iris.shaderpack.StringPair;
import org.anarres.cpp.Feature;
import org.anarres.cpp.LexerException;
@@ -7,7 +8,18 @@ import org.anarres.cpp.Preprocessor;
import org.anarres.cpp.StringLexerSource;
import org.anarres.cpp.Token;
import java.util.function.Supplier;
public class JcppProcessor {
public static String glslPreprocessSource(byte[] contentHash, Supplier<String> source, Iterable<StringPair> environmentDefines) {
final ShaderDiskCache.Key cacheKey = PreprocessedSourceCache.key("glsl", contentHash, environmentDefines);
final String cached = PreprocessedSourceCache.get(cacheKey);
if (cached != null) return cached;
final String preprocessed = glslPreprocessSource(source.get(), environmentDefines);
PreprocessedSourceCache.put(cacheKey, preprocessed);
return preprocessed;
}
// Derived from GlShader from Canvas, licenced under LGPL
public static String glslPreprocessSource(String source, Iterable<StringPair> environmentDefines) {
if (source.contains(GlslCollectingListener.VERSION_MARKER)
@@ -0,0 +1,36 @@
package net.coderbot.iris.shaderpack.preprocessor;
import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache;
import net.coderbot.iris.shaderpack.StringPair;
import org.jetbrains.annotations.Nullable;
/** Disk cache of preprocessor output, keyed on the input and every macro handed to the preprocessor. */
final class PreprocessedSourceCache {
private PreprocessedSourceCache() {}
@Nullable
static ShaderDiskCache.Key key(String kind, String source, Iterable<StringPair> defines) {
return ShaderDiskCache.isEnabled() ? withDefines(ShaderDiskCache.key("iris-preprocess").str(kind).str(source), defines) : null;
}
@Nullable
static ShaderDiskCache.Key key(String kind, byte[] contentHash, Iterable<StringPair> defines) {
return ShaderDiskCache.isEnabled() ? withDefines(ShaderDiskCache.key("iris-preprocess").str(kind).bytes(contentHash), defines) : null;
}
private static ShaderDiskCache.Key withDefines(ShaderDiskCache.Key key, Iterable<StringPair> defines) {
for (StringPair define : defines) {
key.str(define.getKey()).str(define.getValue());
}
return key;
}
@Nullable
static String get(@Nullable ShaderDiskCache.Key key) {
return key == null ? null : ShaderDiskCache.getString(key);
}
static void put(@Nullable ShaderDiskCache.Key key, String preprocessed) {
if (key != null) ShaderDiskCache.putString(key, preprocessed);
}
}
@@ -1,5 +1,6 @@
package net.coderbot.iris.shaderpack.preprocessor;
import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache;
import net.coderbot.iris.Iris;
import net.coderbot.iris.shaderpack.StringPair;
import net.coderbot.iris.shaderpack.option.ShaderPackOptions;
@@ -17,7 +18,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;
public class PropertiesPreprocessor {
// Derived from ShaderProcessor.glslPreprocessSource, which is derived from GlShader from Canvas, licenced under LGPL
@@ -29,6 +29,14 @@ public class PropertiesPreprocessor {
final List<String> booleanValues = getBooleanValues(shaderPackOptions);
final Map<String, String> stringValues = getStringValues(shaderPackOptions);
final List<StringPair> macros = new ArrayList<>();
for (String value : booleanValues) macros.add(new StringPair(value, ""));
for (StringPair envDefine : environmentDefines) macros.add(envDefine);
stringValues.forEach((name, value) -> macros.add(new StringPair(name, value)));
final ShaderDiskCache.Key cacheKey = PreprocessedSourceCache.key("properties", source, macros);
final String cached = PreprocessedSourceCache.get(cacheKey);
if (cached != null) return cached;
try (Preprocessor pp = new Preprocessor()) {
for (String value : booleanValues) {
pp.addMacro(value);
@@ -46,7 +54,9 @@ public class PropertiesPreprocessor {
}
});
return process(pp, source);
final String preprocessed = process(pp, source);
PreprocessedSourceCache.put(cacheKey, preprocessed);
return preprocessed;
} catch (IOException e) {
throw new RuntimeException("Unexpected IOException while processing macros", e);
} catch (LexerException e) {
@@ -77,24 +87,10 @@ public class PropertiesPreprocessor {
PropertyCollectingListener listener = new PropertyCollectingListener();
preprocessor.setListener(listener);
// Not super efficient, but this removes trailing whitespace on lines, fixing an issue with whitespace after
// This removes trailing whitespace on lines, fixing an issue with whitespace after
// line continuations (see PreprocessorTest#testWeirdPropertiesLineContinuation)
// Required for Voyager Shader
source = Arrays.stream(source.split("\\r\\n|\\n|\\r")).map(String::trim).filter(s -> !s.isEmpty())
.map(line -> {
if (line.startsWith("#")) {
for (PreprocessorCommand command : PreprocessorCommand.values()) {
if (line.startsWith("#" + (command.name().replace("PP_", "").toLowerCase(Locale.ROOT)))) {
return line;
}
}
return "";
}
// In PropertyCollectingListener we suppress "unknown preprocessor directive errors" and
// assume the line to be a comment, since in .properties files `#` also functions as a comment
// marker.
return line.replace("#", "");
}).collect(Collectors.joining("\n")) + "\n";
source = cleanLines(source);
// TODO: This is a horrible fix to trick the preprocessor into not seeing the backslashes during processing. We need a better way to do this.
source = source.replace("\\", "IRIS_PASSTHROUGHBACKSLASH");
@@ -119,6 +115,46 @@ public class PropertiesPreprocessor {
return (listener.collectLines() + source).replace("IRIS_PASSTHROUGHBACKSLASH", "\\");
}
private static final String[] DIRECTIVE_PREFIXES = Arrays.stream(PreprocessorCommand.values())
.map(command -> "#" + command.name().replace("PP_", "").toLowerCase(Locale.ROOT))
.toArray(String[]::new);
// Same result as splitting on \r\n|\n|\r, trimming, dropping empty lines and joining with \n
static String cleanLines(String source) {
final StringBuilder out = new StringBuilder(source.length());
boolean first = true;
int start = 0;
final int length = source.length();
while (start <= length) {
int end = start;
while (end < length && source.charAt(end) != '\n' && source.charAt(end) != '\r') end++;
final String line = source.substring(start, end).trim();
if (!line.isEmpty()) {
if (!first) out.append('\n');
first = false;
out.append(cleanLine(line));
}
if (end < length && source.charAt(end) == '\r' && end + 1 < length && source.charAt(end + 1) == '\n') end++;
start = end + 1;
}
return out.append('\n').toString();
}
private static String cleanLine(String line) {
if (line.startsWith("#")) {
for (String prefix : DIRECTIVE_PREFIXES) {
if (line.startsWith(prefix)) {
return line;
}
}
return "";
}
// In PropertyCollectingListener we suppress "unknown preprocessor directive errors" and
// assume the line to be a comment, since in .properties files `#` also functions as a comment
// marker.
return line.indexOf('#') < 0 ? line : line.replace("#", "");
}
private static List<String> getBooleanValues(ShaderPackOptions shaderPackOptions) {
List<String> booleanValues = new ArrayList<>();
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.List;
@@ -46,6 +47,7 @@ public final class PBRResourceIndex {
private static IResourceManager owner;
private static CompletableFuture<Map<IResourcePack, PackIndex>> pending;
private static Map<IResourcePack, PackIndex> built;
private static final Map<ResourceLocation, Boolean> answers = new HashMap<>();
private static boolean disabled;
private static boolean warned;
@@ -75,6 +77,7 @@ public final class PBRResourceIndex {
owner = null;
pending = null;
built = null;
answers.clear();
}
public static synchronized boolean mayExist(IResourceManager resourceManager, ResourceLocation location) {
@@ -99,6 +102,16 @@ public final class PBRResourceIndex {
return true;
}
}
final Boolean known = answers.get(location);
if (known != null) {
return known;
}
final boolean present = lookUp(manager, location, path);
answers.put(location, present);
return present;
}
private static boolean lookUp(SimpleReloadableResourceManager manager, ResourceLocation location, String path) {
final Object domain = manager.domainResourceManagers.get(location.getResourceDomain());
if (domain == null) {
return false;
@@ -234,6 +234,11 @@ public class CustomUniforms implements FunctionContext {
locationMap.put(pass, locationMap.remove(holder));
}
public void mapPassLike(Object existingPass, Object pass) {
final PassUniforms uniforms = locationMap.get(existingPass);
if (uniforms != null) locationMap.put(pass, uniforms);
}
public void update() {
if (Tracy.ENABLED) {