diff --git a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/GlslTransformUtils.java b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/GlslTransformUtils.java index 3789cbb53..59b1cda49 100644 --- a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/GlslTransformUtils.java +++ b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/GlslTransformUtils.java @@ -72,6 +72,10 @@ public class GlslTransformUtils { } public static String renameReservedWords(String source, int targetVersion) { + if (!containsWholeWord(source, "sample") && !containsWholeWord(source, "new") + && (targetVersion < 400 || !containsWholeWord(source, "sampler"))) { + return source; + } for (var entry : VERSIONED_RESERVED_WORDS.entrySet()) { // Rename identifiers that become reserved words at or above this GLSL version if (targetVersion >= entry.getKey()) { @@ -83,6 +87,23 @@ public class GlslTransformUtils { return source; } + static boolean containsWholeWord(String source, String word) { + int from = 0; + while (true) { + final int match = source.indexOf(word, from); + if (match < 0) return false; + final int end = match + word.length(); + if ((match == 0 || !isAsciiWordChar(source.charAt(match - 1))) && (end == source.length() || !isAsciiWordChar(source.charAt(end)))) { + return true; + } + from = match + 1; + } + } + + private static boolean isAsciiWordChar(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; + } + public static String restoreReservedWords(String source) { source = source.replace(RENAMED_PREFIX + "texture", "texture"); source = source.replace(RENAMED_PREFIX + "sampler", "sampler"); diff --git a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ProgramBinaryCache.java b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ProgramBinaryCache.java index 0f7858dbf..cfdc8a0c4 100644 --- a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ProgramBinaryCache.java +++ b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ProgramBinaryCache.java @@ -106,14 +106,20 @@ public final class ProgramBinaryCache { } public static void save(Key key, int program) { + if (!saveBinary(key, program) && fullCompileWrites != null) { + fullCompileWrites.add(CompletableFuture.completedFuture(false)); + } + } + + private static boolean saveBinary(Key key, int program) { final int length = GLStateManager.glGetProgrami(program, GL41.GL_PROGRAM_BINARY_LENGTH); - if (length <= 0) return; + if (length <= 0) return false; final ByteBuffer data = MemoryUtilities.memAlloc(length); final IntBuffer written = MemoryUtilities.memAllocInt(1); final IntBuffer format = MemoryUtilities.memAllocInt(1); try { GLStateManager.glGetProgramBinary(program, written, format, data); - if (written.get(0) <= 0) return; + if (written.get(0) <= 0) return false; data.limit(written.get(0)); final byte[] binary = ShaderCacheIO.toHeap(data); final String formatTag = Integer.toString(format.get(0)); @@ -130,6 +136,7 @@ public final class ProgramBinaryCache { }); final List> writes = fullCompileWrites; if (writes != null) writes.add(write); + return true; } finally { MemoryUtilities.memFree(data); MemoryUtilities.memFree(written); @@ -137,6 +144,18 @@ public final class ProgramBinaryCache { } } + public static boolean isFullCompileRunning() { + return keysInUse != null; + } + + public static boolean saveRetained(Key key, int program) { + if (ShaderDiskCache.getBlob(key.diskKey) != null) { + keep(key.diskKey); + return true; + } + return saveBinary(key, program); + } + public static void beginFullCompile() { fullCompileLayer = packLayer; keysInUse = new HashSet<>(); diff --git a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCache.java b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCache.java index 572da34b8..d47e8492b 100644 --- a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCache.java +++ b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCache.java @@ -53,7 +53,10 @@ public final class ShaderDiskCache { private ShaderDiskCache() {} - public static void configure(Path dir, String salt) { + private static String configuredSalt; + + public static synchronized void configure(Path dir, String salt) { + if (dir.equals(root) && salt.equals(configuredSalt)) return; try { Files.createDirectories(dir); final Path saltFile = dir.resolve("salt"); @@ -63,6 +66,7 @@ public final class ShaderDiskCache { Files.write(saltFile, salt.getBytes(StandardCharsets.UTF_8)); } root = dir; + configuredSalt = salt; AngelicaWorkers.run(() -> enforceCap(dir, MAX_BYTES)); LOGGER.info("Shader disk cache at {}", dir); } catch (IOException | RuntimeException e) { diff --git a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/threading/AngelicaWorkers.java b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/threading/AngelicaWorkers.java index 19cf61567..7bb68a757 100644 --- a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/threading/AngelicaWorkers.java +++ b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/threading/AngelicaWorkers.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; @@ -16,6 +17,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; public final class AngelicaWorkers { @@ -153,6 +155,51 @@ public final class AngelicaWorkers { } } + /** + * Runs every task and returns the results in task order. + */ + public static List invokeAll(List> tasks) { + final int count = tasks.size(); + final Object[] results = new Object[count]; + if (count == 0) return new ArrayList<>(); + final AtomicInteger next = new AtomicInteger(); + final CountDownLatch finished = new CountDownLatch(count); + final AtomicReference failure = new AtomicReference<>(); + final Runnable drain = () -> { + int i; + while ((i = next.getAndIncrement()) < count) { + try { + results[i] = tasks.get(i).get(); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } finally { + finished.countDown(); + } + } + }; + final int helpers = Math.min(threads(), count - 1); + for (int h = 0; h < helpers; h++) { + run(drain); + } + drain.run(); + try { + finished.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted waiting for worker tasks", e); + } + final Throwable t = failure.get(); + if (t instanceof RuntimeException re) throw re; + if (t instanceof Error err) throw err; + if (t != null) throw new IllegalStateException(t); + final List out = new ArrayList<>(count); + for (Object result : results) { + @SuppressWarnings("unchecked") final T typed = (T) result; + out.add(typed); + } + return out; + } + private static void checkIdleShutdown() { synchronized (lock) { final ExecutorService current = executor; diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/SDLGPURenderBackend.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/SDLGPURenderBackend.java index 5c68afa54..253c8ec50 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/SDLGPURenderBackend.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/SDLGPURenderBackend.java @@ -4109,6 +4109,15 @@ public class SDLGPURenderBackend extends RenderBackend { } private static final int CLEAR_CHUNK_BYTES = 64 * 1024; + private static final int FILL_CHUNK_BYTES = 4 * 1024 * 1024; + + private static boolean isAllZero(ByteBuffer data, int elemBytes) { + final int n = Math.min(elemBytes, data.remaining()); + for (int i = 0; i < n; i++) { + if (data.get(data.position() + i) != 0) return false; + } + return true; + } @Override public void clearBufferSubData(int target, int internalFormat, long offset, long size, int format, int type, ByteBuffer data) { final int glId = getBoundBuffer(target); if (glId == 0 || size <= 0) return; @@ -4116,29 +4125,32 @@ public class SDLGPURenderBackend extends RenderBackend { if (handle == 0 || frameManager.getCommandBuffer() == 0) return; final int elemBytes = Math.max(1, PixelOps.glPixelSize(format, type)); - final int chunkBytes = (int) Math.min(size, CLEAR_CHUNK_BYTES); + final boolean defer = s().deferUploads && target != GL43.GL_SHADER_STORAGE_BUFFER; + // Undeferred clears stage one large pattern and upload it repeatedly + final int chunkBytes = (int) Math.min(size, defer ? CLEAR_CHUNK_BYTES : FILL_CHUNK_BYTES); final int alignedChunk = (chunkBytes / elemBytes) * elemBytes; if (alignedChunk == 0) return; resourceManager.markBufferContentsDefined(glId); - final ByteBuffer chunk = MemoryUtil.memAlloc(alignedChunk); + final boolean zero = data == null || isAllZero(data, elemBytes); + final ByteBuffer chunk = zero ? MemoryUtil.memCalloc(alignedChunk) : MemoryUtil.memAlloc(alignedChunk); try { - for (int i = 0; i < alignedChunk; i++) { - final byte v = (data != null && (i % elemBytes) < data.remaining()) - ? data.get(data.position() + (i % elemBytes)) : 0; - chunk.put(i, v); + if (!zero) { + for (int i = 0; i < alignedChunk; i++) { + final byte v = (i % elemBytes) < data.remaining() ? data.get(data.position() + (i % elemBytes)) : 0; + chunk.put(i, v); + } } chunk.position(0).limit(alignedChunk); + if (!defer) { + resourceManager.fillBuffer(frameManager.ensureCopyPass(), chunk, handle, offset, size); + return; + } long remaining = size; long writeOffset = offset; - final boolean defer = s().deferUploads && target != GL43.GL_SHADER_STORAGE_BUFFER; while (remaining > 0) { final int n = (int) Math.min(remaining, alignedChunk); chunk.position(0).limit(n); - if (defer) { - enqueuePreCopied(chunk, handle, writeOffset, false); - } else { - resourceManager.uploadToBuffer(frameManager.ensureCopyPass(), chunk, handle, writeOffset, false); - } + enqueuePreCopied(chunk, handle, writeOffset, false); remaining -= n; writeOffset += n; } @@ -4163,11 +4175,14 @@ public class SDLGPURenderBackend extends RenderBackend { return; } - // Volume-sized staging, so it is only reached for 3D formats the compute clear cannot address. - final long totalBytes = (long) mipW * mipH * mipD * bpp; - if (totalBytes <= 0 || totalBytes > Integer.MAX_VALUE) return; final long cp = frameManager.ensureCopyPass(); if (cp == 0) return; + // A volume-sized staging buffer would allocate and upload each byte (1 GB for Euphoria Patches' floodfill!!) + if (mipD > 1 && resourceManager.zeroTexture3D(cp, texHandle, mipW, mipH, mipD, level, bpp)) { + return; + } + final long totalBytes = (long) mipW * mipH * mipD * bpp; + if (totalBytes <= 0 || totalBytes > Integer.MAX_VALUE) return; final ByteBuffer zeros = MemoryUtil.memCalloc((int) totalBytes); try { if (mipD > 1) { diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/resource/ResourceManager.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/resource/ResourceManager.java index 1466de118..d5f049690 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/resource/ResourceManager.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/resource/ResourceManager.java @@ -1631,6 +1631,76 @@ public final class ResourceManager { returnTransferBuffer(xfer, size); } + /** + * Fills {@code size} bytes of a buffer by staging {@code pattern} once and uploading it repeatedly. Staging every + * byte instead maps a transfer buffer per chunk, which stalls for seconds on the hundreds of megabytes some shader + * packs allocate as storage buffers. + */ + public void fillBuffer(long copyPass, ByteBuffer pattern, long gpuBuffer, long offset, long size) { + final int chunk = pattern.remaining(); + if (chunk <= 0 || size <= 0) return; + final long xfer = acquireTransferBuffer(chunk); + if (xfer == 0) return; + try { + final ByteBuffer mapped = SDL_MapGPUTransferBuffer(device.getDevice(), xfer, true, getTransferBufferMapSize(chunk)); + if (mapped == null || mapped.capacity() < chunk) { + SDL_UnmapGPUTransferBuffer(device.getDevice(), xfer); + throw new IllegalStateException("fillBuffer: bad mapping size=" + chunk + " mapped=" + (mapped == null ? "null" : "cap=" + mapped.capacity())); + } + copyMappedFromData(mapped, pattern, chunk, COPY_CALLSITE_UPLOAD_BUFFER); + SDL_UnmapGPUTransferBuffer(device.getDevice(), xfer); + + int uploads = 0; + try (var stack = stackPush()) { + final SDL_GPUTransferBufferLocation src = SDL_GPUTransferBufferLocation.calloc(stack).transfer_buffer(xfer).offset(0); + final SDL_GPUBufferRegion dst = SDL_GPUBufferRegion.calloc(stack).buffer(gpuBuffer); + for (long done = 0; done < size; done += chunk) { + dst.offset((int) (offset + done)).size((int) Math.min(chunk, size - done)); + SDL_UploadToGPUBuffer(copyPass, src, dst, false); + uploads++; + } + } + frameManager.recordUploadCommands(size, uploads); + } finally { + returnTransferBuffer(xfer, chunk); + } + } + + public boolean zeroTexture3D(long copyPass, long gpuTexture, int w, int h, int d, int level, int bytesPerTexel) { + final long sliceBytes = (long) w * h * bytesPerTexel; + if (sliceBytes <= 0 || sliceBytes > Integer.MAX_VALUE) return false; + final long xfer = acquireTransferBuffer(sliceBytes); + if (xfer == 0) return false; + try { + final ByteBuffer mapped = SDL_MapGPUTransferBuffer(device.getDevice(), xfer, true, getTransferBufferMapSize(sliceBytes)); + if (mapped == null || mapped.capacity() < sliceBytes) { + SDL_UnmapGPUTransferBuffer(device.getDevice(), xfer); + return false; + } + MemoryUtil.memSet(MemoryUtil.memAddress(mapped), 0, sliceBytes); + SDL_UnmapGPUTransferBuffer(device.getDevice(), xfer); + + flushBatchedUploads(copyPass); + try (var stack = stackPush()) { + final SDL_GPUTextureTransferInfo src = SDL_GPUTextureTransferInfo.calloc(stack).transfer_buffer(xfer).offset(0); + final SDL_GPUTextureRegion dst = SDL_GPUTextureRegion.calloc(stack) + .texture(gpuTexture) + .mip_level(level) + .x(0).y(0) + .w(w).h(h).d(1); + for (int z = 0; z < d; z++) { + dst.z(z); + SDL_UploadToGPUTexture(copyPass, src, dst, false); + } + } + markTextureContentDefined(gpuTexture); + frameManager.recordUploadCommands(sliceBytes * d, d); + return true; + } finally { + returnTransferBuffer(xfer, sliceBytes); + } + } + public static final int BATCH_SEGMENT_CAPACITY = 4 * 1024 * 1024; public void uploadToTexture(long copyPass, ByteBuffer data, long gpuTexture, int x, int y, int w, int h, int level) { diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderManager.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderManager.java index 99d4d257a..a206da3f3 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderManager.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderManager.java @@ -53,6 +53,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static org.lwjgl.sdl.SDLGPU.*; import static org.lwjgl.system.MemoryStack.*; @@ -64,6 +65,7 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntArrays; import it.unimi.dsi.fastutil.ints.IntIterator; +import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; @@ -234,7 +236,35 @@ public final class ShaderManager { final int shaderKind = shaderKindFor(obj); final int glType = obj.type; final String finalSrc = obj.source; - obj.spirvFuture = AngelicaWorkers.submit(() -> compileStageAsync(finalSrc, shaderKind, glType, shader)); + final QueuedCompile compile = new QueuedCompile(() -> compileStageAsync(finalSrc, shaderKind, glType, shader)); + AngelicaWorkers.run(compile); + obj.queuedCompile = compile; + obj.spirvFuture = compile.result; + } + + private static final class QueuedCompile implements Runnable { + private final AtomicBoolean claimed = new AtomicBoolean(); + private final Supplier work; + final CompletableFuture result = new CompletableFuture<>(); + + QueuedCompile(Supplier work) { + this.work = work; + } + + @Override + public void run() { + if (!claimed.compareAndSet(false, true)) return; + try { + result.complete(work.get()); + } catch (Throwable t) { + result.completeExceptionally(t); + } + } + + /** Stops the compile if no thread has started it; false when it is running or done. */ + boolean cancel() { + return claimed.compareAndSet(false, true); + } } private static AsyncCompile compileStageAsync(String src, int shaderKind, int glType, int shader) { @@ -377,6 +407,10 @@ public final class ShaderManager { final SpirvCompiler.Result r; final boolean alreadyRemapped; if (obj.spirvFuture != null) { + if (obj.queuedCompile != null) { + obj.queuedCompile.run(); + obj.queuedCompile = null; + } final AsyncCompile compiled = obj.spirvFuture.join(); obj.spirvFuture = null; r = compiled.result(); @@ -431,6 +465,9 @@ public final class ShaderManager { final CompletableFuture future = obj.spirvFuture; if (future == null) return; obj.spirvFuture = null; + final QueuedCompile queued = obj.queuedCompile; + obj.queuedCompile = null; + if (queued != null && queued.cancel()) return; future.thenAccept(ShaderManager::freeSpirvResult); } @@ -567,7 +604,7 @@ public final class ShaderManager { registerExtraNames(prog, vs.reflection); registerExtraNames(prog, fs.reflection); - for (final String name : prog.allSamplerNames) ensureLocation(prog, name); + registerSamplerLocations(prog, vs.reflection, fs.reflection); for (final String name : prog.allImageNames) ensureLocation(prog, name); prog.buildUniformSlotArrays(); @@ -611,7 +648,7 @@ public final class ShaderManager { applyUboMembers(prog, refl, false); registerExtraNames(prog, refl); - for (final String name : prog.allSamplerNames) ensureLocation(prog, name); + registerSamplerLocations(prog, refl); for (final String name : prog.allImageNames) ensureLocation(prog, name); prog.buildUniformSlotArrays(); @@ -820,8 +857,52 @@ public final class ShaderManager { return built; } + private static ShaderDiskCache.Key variantKey(String vertexSource, List attribs) { + final ShaderDiskCache.Key key = ShaderDiskCache.key("sdl-vertex-variant").str(SpirvCompiler.toolchainId()).str(CrossCompileUtil.spvcId()) + .str(vertexSource).i(attribs.size()); + for (UscaledRetype.Attrib a : attribs) { + key.str(a.name()).i(a.location()).i(a.declVecSize()).i(a.boundVecSize()).i(a.signed() ? 1 : 0); + } + return key; + } + private VertexVariant buildVertexVariant(int program, ProgramObject prog, List attribs) { - final String preprocessed = SpirvCompiler.preprocess(prog.vertexSource, Shaderc.shaderc_vertex_shader, "variant" + program, SpirvCompiler.Options.vulkanForced460Core()); + final ShaderDiskCache.Key diskKey = ShaderDiskCache.isEnabled() ? variantKey(prog.vertexSource, attribs) : null; + final ShaderStageSerializer.Stage cached = diskKey != null ? loadStage(diskKey) : null; + final ShaderStageSerializer.Stage stage = cached != null ? cached : compileVertexVariant(program, prog.vertexSource, attribs, diskKey); + if (stage == null) return null; + + final StageReflection refl = stage.reflection(); + if (!sameVaryingLayout(prog.vertexReflection.vsOutputs(), refl.vsOutputs())) { + LOG.error("Vertex attribute conversion: program {} variant changed VS output locations; the fragment stage is patched against the base layout. Falling back to normalized coercion.", program); + return null; + } + final VertexVariant v = new VertexVariant(); + v.spirv = ShaderCacheIO.toNative(stage.spirv()); + + final Object2IntOpenHashMap bindings = new Object2IntOpenHashMap<>(prog.attribLocationBindings); + bindings.defaultReturnValue(-1); + for (UscaledRetype.Attrib a : attribs) bindings.put(a.name() + UscaledRetype.SUFFIX, a.location()); + v.inputMask = patchAttribLocations(v.spirv, refl, bindings, v.inputVecSize, v.inputBaseType, null, null); + if (v.inputMask != prog.vertexInputMask) { + memFree(v.spirv); + LOG.error("Vertex attribute conversion: program {} variant resolved inputs to 0x{} but the program binds 0x{}; a renamed input landed on the wrong location. Falling back to normalized coercion.", + program, Integer.toHexString(v.inputMask), Integer.toHexString(prog.vertexInputMask)); + return null; + } + v.sdlShader = createSDLShader(v.spirv, SDL_GPU_SHADERSTAGE_VERTEX, prog.vertexResources.numSamplers(), prog.vertexResources.numUBOs(), prog.vertexResources.numStorageBuffers(), prog.vertexResources.numStorageTextures()); + if (v.sdlShader == 0) { + memFree(v.spirv); + LOG.error("Vertex attribute conversion: program {} variant shader creation failed: {}", program, SDLError.SDL_GetError()); + return null; + } + LOG.info("Vertex attribute conversion: program {} converted {}", program, attribs); + if (SystemProperties.dumpShaders()) dumpVariantSource(program, stage.source()); + return v; + } + + private static ShaderStageSerializer.Stage compileVertexVariant(int program, String vertexSource, List attribs, ShaderDiskCache.Key diskKey) { + final String preprocessed = SpirvCompiler.preprocess(vertexSource, Shaderc.shaderc_vertex_shader, "variant" + program, SpirvCompiler.Options.vulkanForced460Core()); if (preprocessed == null) { LOG.warn("Vertex attribute conversion: program {} preprocessing failed; falling back to normalized coercion, values will be scaled", program); return null; @@ -839,32 +920,11 @@ public final class ShaderManager { try { remapSpirvForSDLGPU(r.spirv(), GL20.GL_VERTEX_SHADER); final StageReflection refl = reflectStage(r.spirv(), true); - if (!sameVaryingLayout(prog.vertexReflection.vsOutputs(), refl.vsOutputs())) { - LOG.error("Vertex attribute conversion: program {} variant changed VS output locations; the fragment stage is patched against the base layout. Falling back to normalized coercion.", program); - return null; + final byte[] heap = ShaderCacheIO.toHeap(r.spirv()); + if (diskKey != null && refl != StageReflection.EMPTY) { + ShaderDiskCache.put(diskKey, ShaderStageSerializer.encode(heap, refl, GraphicsBindingMap.EMPTY, null, src)); } - final VertexVariant v = new VertexVariant(); - v.spirv = copyBuffer(r.spirv()); - - final Object2IntOpenHashMap bindings = new Object2IntOpenHashMap<>(prog.attribLocationBindings); - bindings.defaultReturnValue(-1); - for (UscaledRetype.Attrib a : attribs) bindings.put(a.name() + UscaledRetype.SUFFIX, a.location()); - v.inputMask = patchAttribLocations(v.spirv, refl, bindings, v.inputVecSize, v.inputBaseType, null, null); - if (v.inputMask != prog.vertexInputMask) { - memFree(v.spirv); - LOG.error("Vertex attribute conversion: program {} variant resolved inputs to 0x{} but the program binds 0x{}; a renamed input landed on the wrong location. Falling back to normalized coercion.", - program, Integer.toHexString(v.inputMask), Integer.toHexString(prog.vertexInputMask)); - return null; - } - v.sdlShader = createSDLShader(v.spirv, SDL_GPU_SHADERSTAGE_VERTEX, prog.vertexResources.numSamplers(), prog.vertexResources.numUBOs(), prog.vertexResources.numStorageBuffers(), prog.vertexResources.numStorageTextures()); - if (v.sdlShader == 0) { - memFree(v.spirv); - LOG.error("Vertex attribute conversion: program {} variant shader creation failed: {}", program, SDLError.SDL_GetError()); - return null; - } - LOG.info("Vertex attribute conversion: program {} converted {}", program, attribs); - if (SystemProperties.dumpShaders()) dumpVariantSource(program, src); - return v; + return new ShaderStageSerializer.Stage(heap, refl, GraphicsBindingMap.EMPTY, Set.of(), src); } finally { memFree(r.spirv()); } @@ -1476,6 +1536,7 @@ public final class ShaderManager { public boolean deletePending; public int attachCount; public CompletableFuture spirvFuture; + QueuedCompile queuedCompile; public StageReflection reflection = StageReflection.EMPTY; public GraphicsBindingMap graphicsBindingMap = GraphicsBindingMap.EMPTY; public Set boolUniforms = Set.of(); @@ -1520,6 +1581,7 @@ public final class ShaderManager { public record StageReflection( ResourceCounts counts, List samplerNames, // SAMPLED_IMAGE, sorted by binding (post-remap) + List unusedSamplerNames, // declared samplers the entry point never reads List extraUniformNames, // SEPARATE_SAMPLERS - registered in nameToLocation only List storageImageNames, // SEPARATE_IMAGE + STORAGE_IMAGE (RO + RW), sorted by binding (post-remap) int uboSize, // binding-0 UBO total size in bytes; 0 if absent @@ -1535,7 +1597,7 @@ public final class ShaderManager { BlockReflection[] blocks ) { public static final StageReflection EMPTY = new StageReflection( - ResourceCounts.EMPTY, List.of(), List.of(), List.of(), 0, List.of(), List.of(), List.of(), List.of(), + ResourceCounts.EMPTY, List.of(), List.of(), List.of(), List.of(), 0, List.of(), List.of(), List.of(), List.of(), -1, 0, 0, 0, 0, BlockReflection.emptyBlocks()); } @@ -1739,6 +1801,7 @@ public final class ShaderManager { return new StageReflection( new ResourceCounts(samplerNames.size(), numUBOs, numStorageBuffers, numStorageTextures), samplerNames, + unusedSamplerNames(compiler, resources, stack), extraNames, storageImageNames, uboSize, @@ -1822,6 +1885,36 @@ public final class ShaderManager { }); } + private static List unusedSamplerNames(long compiler, long resources, MemoryStack stack) { + final PointerBuffer pActive = stack.mallocPointer(1); + final PointerBuffer pActiveResources = stack.mallocPointer(1); + if (Spvc.spvc_compiler_get_active_interface_variables(compiler, pActive) != Spvc.SPVC_SUCCESS + || Spvc.spvc_compiler_create_shader_resources_for_active_variables(compiler, pActiveResources, pActive.get(0)) != Spvc.SPVC_SUCCESS) { + return List.of(); + } + final IntOpenHashSet active = new IntOpenHashSet(); + forEachResource(pActiveResources.get(0), Spvc.SPVC_RESOURCE_TYPE_SAMPLED_IMAGE, stack, (i, r) -> active.add(r.id())); + final List unused = new ArrayList<>(); + forEachResource(resources, Spvc.SPVC_RESOURCE_TYPE_SAMPLED_IMAGE, stack, (i, r) -> { + if (active.contains(r.id())) return; + final String name = Spvc.spvc_compiler_get_name(compiler, r.id()); + if (name != null && !name.isEmpty()) unused.add(name); + }); + return unused; + } + + private static void registerSamplerLocations(ProgramObject prog, StageReflection... stages) { + final Set read = new HashSet<>(); + for (StageReflection stage : stages) { + for (String name : stage.samplerNames()) { + if (!stage.unusedSamplerNames().contains(name)) read.add(name); + } + } + for (String name : prog.allSamplerNames) { + if (read.contains(name)) ensureLocation(prog, name); + } + } + private static void collectResourceNames(long resources, long compiler, int resourceType, List out, MemoryStack stack) { forEachResource(resources, resourceType, stack, (i, r) -> { final String name = r.nameString(); diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializer.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializer.java index 0b8019bae..a3ca2aa1a 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializer.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializer.java @@ -102,6 +102,7 @@ final class ShaderStageSerializer { out.writeInt(r.counts().numStorageBuffers()); out.writeInt(r.counts().numStorageTextures()); writeStrList(out, r.samplerNames()); + writeStrList(out, r.unusedSamplerNames()); writeStrList(out, r.extraUniformNames()); writeStrList(out, r.storageImageNames()); out.writeInt(r.uboSize()); @@ -139,6 +140,7 @@ final class ShaderStageSerializer { private static StageReflection readReflection(ByteBuffer in) { final ResourceCounts counts = new ResourceCounts(in.getInt(), in.getInt(), in.getInt(), in.getInt()); final List samplerNames = readStrList(in); + final List unusedSamplerNames = readStrList(in); final List extraNames = readStrList(in); final List storageImageNames = readStrList(in); final int uboSize = in.getInt(); @@ -156,7 +158,7 @@ final class ShaderStageSerializer { for (int b = 0; b < blockCount; b++) { blocks[b] = new BlockReflection(in.getInt(), in.getInt(), readList(in, ShaderStageSerializer::readMember), in.get() != 0); } - return new StageReflection(counts, samplerNames, extraNames, storageImageNames, uboSize, uboMembers, vsInputs, vsOutputs, fsInputs, + return new StageReflection(counts, samplerNames, unusedSamplerNames, extraNames, storageImageNames, uboSize, uboMembers, vsInputs, vsOutputs, fsInputs, maxOutputLocation, roBuf, rwBuf, roTex, rwTex, blocks); } diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/AttribLocationPatchTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/AttribLocationPatchTest.java index 799318cfd..9adcb8193 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/AttribLocationPatchTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/AttribLocationPatchTest.java @@ -36,7 +36,7 @@ class AttribLocationPatchTest { inputs[i] = new ShaderManager.VsInput(names[i], i, i, i < VEC_SIZES.length ? VEC_SIZES[i] : 4, i < BASE_TYPES.length ? BASE_TYPES[i] : Spvc.SPVC_BASETYPE_FP32); } final ShaderManager.StageReflection e = ShaderManager.StageReflection.EMPTY; - return new ShaderManager.StageReflection(e.counts(), e.samplerNames(), e.extraUniformNames(), + return new ShaderManager.StageReflection(e.counts(), e.samplerNames(), e.unusedSamplerNames(), e.extraUniformNames(), e.storageImageNames(), e.uboSize(), e.uboMembers(), List.of(inputs), e.vsOutputs(), e.fsInputs(), e.maxOutputLocation(), e.numReadonlyStorageBuffers(), e.numReadwriteStorageBuffers(), e.numReadonlyStorageTextures(), e.numReadwriteStorageTextures(), e.blocks()); diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/AttribLocationReflectionTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/AttribLocationReflectionTest.java index 53dd68ef0..30f2df949 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/AttribLocationReflectionTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/AttribLocationReflectionTest.java @@ -19,7 +19,7 @@ class AttribLocationReflectionTest { private static StageReflection reflectionWith(VsInput... inputs) { return new StageReflection( - ShaderManager.ResourceCounts.EMPTY, List.of(), List.of(), List.of(), 0, List.of(), + ShaderManager.ResourceCounts.EMPTY, List.of(), List.of(), List.of(), List.of(), 0, List.of(), List.of(inputs), List.of(), List.of(), -1, 0, 0, 0, 0, ShaderManager.BlockReflection.emptyBlocks()); } diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/StageDiskCacheTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/StageDiskCacheTest.java index 3fc6b9eb5..b41a14111 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/StageDiskCacheTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/StageDiskCacheTest.java @@ -73,7 +73,7 @@ class StageDiskCacheTest { private static byte[] poison(String marker) { final ShaderManager.StageReflection reflection = new ShaderManager.StageReflection( - ShaderManager.ResourceCounts.EMPTY, List.of(marker), List.of(), List.of(), 0, List.of(), List.of(), List.of(), List.of(), + ShaderManager.ResourceCounts.EMPTY, List.of(marker), List.of(), List.of(), List.of(), 0, List.of(), List.of(), List.of(), List.of(), -1, 0, 0, 0, 0, ShaderManager.BlockReflection.emptyBlocks()); return ShaderStageSerializer.encode(POISON_SPIRV, reflection, ShaderManager.GraphicsBindingMap.EMPTY, Set.of("poison_bool"), "poison_source"); } diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/UnusedSamplerReflectionTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/UnusedSamplerReflectionTest.java new file mode 100644 index 000000000..cb011ea74 --- /dev/null +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/UnusedSamplerReflectionTest.java @@ -0,0 +1,45 @@ +package com.gtnewhorizons.angelica.sdlgpu.shader; + +import com.gtnewhorizons.angelica.glsm.shader.SpirvCompiler; +import org.junit.jupiter.api.Test; +import org.lwjgl.system.MemoryUtil; +import org.lwjgl.util.shaderc.Shaderc; + +import java.nio.ByteBuffer; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +class UnusedSamplerReflectionTest { + + private static final String FRAGMENT = """ + #version 330 core + uniform sampler2D tex; + uniform sampler2D normals; + uniform sampler2D specular; + in vec2 uv; + out vec4 fragColor; + float highlight(float d) { + float specular = d * d; + return specular; + } + void main() { + fragColor = texture(tex, uv) * highlight(uv.x); + } + """; + + @Test + void samplersOnlyShadowedOrDeclaredAreUnused() { + final SpirvCompiler.Result r = SpirvCompiler.compile(FRAGMENT, Shaderc.shaderc_fragment_shader, "test", SpirvCompiler.Options.vulkanRelaxed()); + if (r.spirv() == null) fail("Compilation failed: " + r.error()); + final ByteBuffer spirv = r.spirv(); + try { + final ShaderManager.StageReflection refl = ShaderManager.reflectStage(spirv, false); + assertEquals(List.of("normals", "specular"), refl.unusedSamplerNames().stream().sorted().toList(), + "declared samplers: " + refl.samplerNames()); + } finally { + MemoryUtil.memFree(spirv); + } + } +} diff --git a/src/main/java/com/gtnewhorizons/angelica/iris/ShaderWarmup.java b/src/main/java/com/gtnewhorizons/angelica/iris/ShaderWarmup.java new file mode 100644 index 000000000..b7d4ffe0b --- /dev/null +++ b/src/main/java/com/gtnewhorizons/angelica/iris/ShaderWarmup.java @@ -0,0 +1,96 @@ +package com.gtnewhorizons.angelica.iris; + +import com.gtnewhorizons.angelica.utils.AngelicaJar; +import net.coderbot.iris.Iris; +import net.coderbot.iris.parsing.IrisFunctions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Prepares for the first shader pack load on a background thread while the game sits at the main menu. + */ +public final class ShaderWarmup { + private static final Logger LOGGER = LogManager.getLogger("Angelica/ShaderWarmup"); + private static final String[] PACKAGES = { "net/coderbot/iris/", "kroppeb/stareval/" }; + private static final String SKIPPED_PACKAGE = "net/coderbot/iris/compat/dh/"; + + private static boolean started; + + private ShaderWarmup() {} + + /** Main thread only */ + public static void start() { + if (started) return; + started = true; + final Runnable readSelectedPack = Iris.prepareSelectedShaderPack(); + final Thread thread = new Thread(() -> { + preloadClasses(); + if (readSelectedPack != null) readSelectedPack.run(); + }, "Angelica-Shader-Warmup"); + thread.setDaemon(true); + thread.setPriority(Thread.MIN_PRIORITY); + thread.start(); + } + + private static void preloadClasses() { + final List names; + try { + names = listClasses(); + } catch (Exception e) { + LOGGER.debug("Could not list shader classes to preload", e); + return; + } + final ClassLoader loader = Iris.class.getClassLoader(); + for (String name : names) { + try { + Class.forName(name, false, loader); + } catch (Throwable ignored) { + // A class that cannot load here fails the same way when the pipeline reaches it + } + } + try { + Class.forName(IrisFunctions.class.getName(), true, loader); + } catch (Throwable ignored) { + } + } + + private static List listClasses() throws Exception { + final Path source = AngelicaJar.location(); + final List names = new ArrayList<>(); + if (source == null) return names; + if (Files.isDirectory(source)) { + try (Stream files = Files.walk(source)) { + files.forEach(file -> addIfShaderClass(names, source.relativize(file).toString().replace('\\', '/'))); + } + } else { + try (ZipFile jar = new ZipFile(source.toFile())) { + final Enumeration entries = jar.entries(); + while (entries.hasMoreElements()) { + addIfShaderClass(names, entries.nextElement().getName()); + } + } + } + Collections.sort(names); + return names; + } + + private static void addIfShaderClass(List names, String entry) { + if (!entry.endsWith(".class") || entry.startsWith(SKIPPED_PACKAGE)) return; + for (String packagePath : PACKAGES) { + if (entry.startsWith(packagePath)) { + names.add(entry.substring(0, entry.length() - ".class".length()).replace('/', '.')); + return; + } + } + } +} diff --git a/src/main/java/com/gtnewhorizons/angelica/proxy/ClientProxy.java b/src/main/java/com/gtnewhorizons/angelica/proxy/ClientProxy.java index cb20d0020..594254a87 100644 --- a/src/main/java/com/gtnewhorizons/angelica/proxy/ClientProxy.java +++ b/src/main/java/com/gtnewhorizons/angelica/proxy/ClientProxy.java @@ -4,7 +4,6 @@ import com.gtnewhorizons.angelica.debug.DebugKeyHandler; import com.gtnewhorizons.angelica.rendering.culling.GpuCulling; import static com.gtnewhorizons.angelica.AngelicaMod.MOD_ID; -import java.io.File; import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.Locale; @@ -41,7 +40,6 @@ import com.gtnewhorizon.gtnhlib.client.renderer.vao.VAOManager; import com.gtnewhorizons.angelica.client.font.DarkModeUtils; import com.gtnewhorizons.angelica.commands.AngelicaCommand; import com.gtnewhorizons.angelica.AngelicaMod; -import com.gtnewhorizons.angelica.Tags; import com.gtnewhorizons.angelica.common.BlockError; import com.gtnewhorizons.angelica.common.BlockIsbrhTest; import com.gtnewhorizons.angelica.compat.ModStatus; @@ -65,9 +63,9 @@ import com.gtnewhorizons.angelica.glsm.GLStateManager; import com.gtnewhorizons.angelica.glsm.backend.BackendStartGuard; import com.gtnewhorizons.angelica.glsm.backend.VSyncMode; import com.gtnewhorizons.angelica.glsm.profiling.Tracy; -import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import com.gtnewhorizons.angelica.hudcaching.HUDCaching; import com.gtnewhorizons.angelica.iris.IrisGLSMBridge; +import com.gtnewhorizons.angelica.iris.ShaderWarmup; import com.gtnewhorizons.angelica.mixins.interfaces.IGameSettingsExt; import com.gtnewhorizons.angelica.render.CloudRenderer; import com.gtnewhorizons.angelica.render.EmissiveTextureAutoloader; @@ -84,6 +82,7 @@ import com.gtnewhorizons.angelica.rendering.celeritas.threading.ThreadedChunkTas import com.gtnewhorizons.angelica.rendering.items.BlockRenderListManager; import com.gtnewhorizons.angelica.rendering.items.ItemRenderListManager; import com.gtnewhorizons.angelica.rendering.tesr.TesrLifecycle; +import com.gtnewhorizons.angelica.utils.AngelicaJar; import com.gtnewhorizons.angelica.utils.AnimationMode; import com.gtnewhorizons.angelica.utils.ManagedEnum; import com.gtnewhorizons.angelica.zoom.Zoom; @@ -149,11 +148,7 @@ public final class ClientProxy extends CommonProxy { BlockIsbrhTest.renderId = RenderingRegistry.getNextAvailableRenderId(); RenderingRegistry.registerBlockHandler(BlockIsbrhTest.renderId, new IsbrhTestRenderer()); } - final File source = event.getSourceFile(); - if (source != null && source.isFile()) { - ShaderDiskCache.configure(Minecraft.getMinecraft().mcDataDir.toPath().resolve("angelica").resolve("shadercache"), - Tags.VERSION + "|" + source.length() + "|" + source.lastModified()); - } + AngelicaJar.configureShaderDiskCache(event.getSourceFile()); if (AngelicaConfig.enableIris) { IrisGLSMBridge.installImmediateExtendedHandler(); Iris.warmupShaderTransforms(); @@ -391,6 +386,7 @@ public final class ClientProxy extends CommonProxy { if (event.gui instanceof GuiMainMenu && gameStartTime == -1) { gameStartTime = ManagementFactory.getRuntimeMXBean().getUptime() / 1000f; LOGGER.info("The game loaded in {} seconds.", gameStartTime); + if (AngelicaConfig.enableIris) ShaderWarmup.start(); } // force reset zoom when a GUI is opened diff --git a/src/main/java/com/gtnewhorizons/angelica/utils/AngelicaJar.java b/src/main/java/com/gtnewhorizons/angelica/utils/AngelicaJar.java new file mode 100644 index 000000000..3e43e0dd7 --- /dev/null +++ b/src/main/java/com/gtnewhorizons/angelica/utils/AngelicaJar.java @@ -0,0 +1,36 @@ +package com.gtnewhorizons.angelica.utils; + +import com.gtnewhorizons.angelica.Tags; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; +import net.minecraft.client.Minecraft; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; + +public final class AngelicaJar { + private AngelicaJar() {} + + /** The jar or classes directory Angelica was loaded from, or null if something really went wrong. */ + @Nullable + public static Path location() { + try { + String location = AngelicaJar.class.getProtectionDomain().getCodeSource().getLocation().toString(); + if (location.startsWith("jar:")) { + final int separator = location.indexOf("!/"); + location = location.substring("jar:".length(), separator < 0 ? location.length() : separator); + } + return Paths.get(new URI(location)); + } catch (Exception e) { + return null; + } + } + + public static void configureShaderDiskCache(@Nullable File jar) { + if (jar == null || !jar.isFile()) return; + ShaderDiskCache.configure(Minecraft.getMinecraft().mcDataDir.toPath().resolve("angelica").resolve("shadercache"), + Tags.VERSION + "|" + jar.length() + "|" + jar.lastModified()); + } +} diff --git a/src/main/java/net/coderbot/iris/Iris.java b/src/main/java/net/coderbot/iris/Iris.java index 7817a486a..c23195713 100644 --- a/src/main/java/net/coderbot/iris/Iris.java +++ b/src/main/java/net/coderbot/iris/Iris.java @@ -17,6 +17,7 @@ import com.gtnewhorizons.angelica.proxy.ClientProxy; import com.gtnewhorizons.angelica.rendering.StateAwareTessellator; import com.gtnewhorizons.angelica.rendering.celeritas.api.IrisShaderProviderHolder; import com.gtnewhorizons.angelica.sdlgpu.SDLGPUGate; +import com.gtnewhorizons.angelica.utils.AngelicaJar; import cpw.mods.fml.client.registry.ClientRegistry; import cpw.mods.fml.common.eventhandler.SubscribeEvent; import cpw.mods.fml.common.gameevent.InputEvent; @@ -29,6 +30,7 @@ import net.coderbot.iris.celeritas.IrisCeleritasShaderProvider; import net.coderbot.iris.compat.dh.DHCompat; import net.coderbot.iris.config.IrisConfig; import net.coderbot.iris.gbuffer_overrides.matching.InputAvailability; +import net.coderbot.iris.gl.program.RetainedPrograms; import net.coderbot.iris.gl.shader.StandardMacros; import net.coderbot.iris.gui.screen.ShaderPackScreen; import net.coderbot.iris.pipeline.DeferredWorldRenderingPipeline; @@ -38,6 +40,7 @@ import net.coderbot.iris.pipeline.WorldRenderingPipeline; import net.coderbot.iris.pipeline.transform.ShaderTransformer; import net.coderbot.iris.pipeline.transform.TransformPatcher; import net.coderbot.iris.shaderpack.OptionalBoolean; +import net.coderbot.iris.shaderpack.IdMap; import net.coderbot.iris.shaderpack.ProgramSet; import net.coderbot.iris.shaderpack.ShaderPack; import net.coderbot.iris.shaderpack.StringPair; @@ -67,19 +70,30 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; +import java.lang.ref.SoftReference; +import java.nio.file.DirectoryStream; import java.nio.file.FileSystem; +import java.nio.file.FileSystemLoopException; import java.nio.file.FileSystemNotFoundException; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.InvalidPathException; +import java.nio.file.LinkOption; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Base64; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; import java.util.stream.Stream; import java.util.zip.ZipException; @@ -107,6 +121,7 @@ public class Iris { @Getter private static String currentPackName; private static String currentPackSettings; + private static String currentPackState; @Getter private static boolean initialized; @Getter @@ -288,6 +303,10 @@ public class Iris { IrisShaderProviderHolder.setProvider(new IrisCeleritasShaderProvider()); + // The pack loads below, before mod preInit would otherwise set up the cache + final Path angelicaJar = AngelicaJar.location(); + AngelicaJar.configureShaderDiskCache(angelicaJar != null ? angelicaJar.toFile() : null); + // Warm up the threadpool so shader transformations are faster when we need them ShaderTransformExecutor.warmup(); @@ -485,9 +504,7 @@ public class Iris { } } - @SuppressWarnings("unchecked") - final Map changedConfigs = tryReadConfigProperties(shaderPackConfigTxt).map(properties -> (Map) (Map) properties) - .orElse(new HashMap<>()); + final Map changedConfigs = readChangedConfigs(shaderPackConfigTxt); changedConfigs.putAll(shaderPackOptionQueue); clearShaderPackOptionQueue(); @@ -499,7 +516,12 @@ public class Iris { try { final Iterable environmentDefines = StandardMacros.createStandardEnvironmentDefines(); - currentPack = new ShaderPack(shaderPackPath, changedConfigs, environmentDefines); + final String packState = describeSettings(name, shaderPackRoot, toProperties(changedConfigs), environmentDefines, false); + final ShaderPack prepared = takePreparedPack(name, packState); + currentPack = prepared != null ? prepared : new ShaderPack(shaderPackPath, changedConfigs, environmentDefines); + currentPackState = packState; + RetainedPrograms.useConfiguration(packState); + currentPack.activate(); final MutableOptionValues changedConfigsValues = currentPack.getShaderPackOptions().getOptionValues().mutableCopy(); @@ -509,7 +531,7 @@ public class Iris { changedConfigsValues.getStringValues().forEach(configsToSave::setProperty); tryUpdateConfigPropertiesFile(shaderPackConfigTxt, configsToSave); - currentPackSettings = describeSettings(name, shaderPackRoot, configsToSave, environmentDefines); + currentPackSettings = irisConfig.shouldSaveCompiledShaders() ? describeSettings(name, shaderPackRoot, configsToSave, environmentDefines, true) : null; } catch (Exception e) { logger.error("Failed to load the shaderpack \"{}\"!", name); logger.error("", e); @@ -526,10 +548,83 @@ public class Iris { return true; } + private record PreparedPack(String settings, ShaderPack pack) {} + + private static volatile SoftReference preparedPack; + + @Nullable + public static Runnable prepareSelectedShaderPack() { + if (irisConfig == null || irisConfig.areShadersEnabled() || currentPack != null) return null; + final Optional name = irisConfig.getShaderPackName(); + if (name.isEmpty()) return null; + final List environmentDefines = new ArrayList<>(); + StandardMacros.createStandardEnvironmentDefines().forEach(environmentDefines::add); + return () -> { + try { + final PreparedPack prepared = readPreparedPack(name.get(), environmentDefines); + if (prepared != null) preparedPack = new SoftReference<>(prepared); + } catch (Exception e) { + logger.debug("Could not read shaderpack \"{}\" ahead of time", name.get(), e); + } + }; + } + + @Nullable + private static PreparedPack readPreparedPack(String name, List environmentDefines) throws IOException { + final Path shaderPackRoot = getShaderpacksDirectory().resolve(name); + final Map changedConfigs = readChangedConfigs(getShaderpacksDirectory().resolve(name + ".txt")); + final String settings = describeSettings(name, shaderPackRoot, toProperties(changedConfigs), environmentDefines, false); + if (settings == null) return null; + + if (Files.isDirectory(shaderPackRoot)) { + return new PreparedPack(settings, parsePack(shaderPackRoot.resolve("shaders"), changedConfigs, environmentDefines)); + } + try (FileSystem zipSystem = FileSystems.newFileSystem(shaderPackRoot, Iris.class.getClassLoader())) { + final Optional shadersDirectory = findShadersDirectory(zipSystem); + if (shadersDirectory.isEmpty()) return null; + return new PreparedPack(settings, parsePack(shadersDirectory.get(), changedConfigs, environmentDefines)); + } + } + + private static ShaderPack parsePack(Path shadersDirectory, Map changedConfigs, List environmentDefines) throws IOException { + final ShaderPack pack = new ShaderPack(shadersDirectory, changedConfigs, environmentDefines); + // Most worlds open in the overworld; other dimensions build their program sets on demand as usual. + // Considered having tracking of what dim a user is in but with it being so fast now, not sure if + // it's worth saving the 50-80ms "time waste" on first load, especially when we're already pre-loading at game init. + pack.getProgramSet("Overworld"); + return pack; + } + + @Nullable + private static ShaderPack takePreparedPack(String name, @Nullable String packState) { + final SoftReference reference = preparedPack; + preparedPack = null; + final PreparedPack prepared = reference != null ? reference.get() : null; + if (prepared == null || !prepared.settings().equals(packState)) return null; + logger.info("Reusing shaderpack \"{}\" read earlier, nothing it was read from has changed", name); + return prepared.pack(); + } + + private static Map readChangedConfigs(Path shaderPackConfigTxt) { + @SuppressWarnings("unchecked") + final Map changedConfigs = tryReadConfigProperties(shaderPackConfigTxt).map(properties -> (Map) (Map) properties) + .orElse(new HashMap<>()); + return new HashMap<>(changedConfigs); + } + + private static Properties toProperties(Map values) { + final Properties properties = new Properties(); + properties.putAll(values); + return properties; + } + private static Optional loadExternalZipShaderpack(Path shaderpackPath) throws IOException { final FileSystem zipSystem = FileSystems.newFileSystem(shaderpackPath, Iris.class.getClassLoader()); zipFileSystem = zipSystem; + return findShadersDirectory(zipSystem); + } + private static Optional findShadersDirectory(FileSystem zipSystem) throws IOException { // Should only be one root directory for a zip shaderpack final Path root = zipSystem.getRootDirectories().iterator().next(); @@ -552,6 +647,7 @@ public class Iris { private static void setShadersDisabled() { currentPack = null; + currentPackState = null; currentPackSettings = null; ProgramBinaryCache.usePack(null); fallback = false; @@ -710,7 +806,11 @@ public class Iris { */ private static void destroyEverything() { awaitProgramSetWarmup(); + if (currentPack != null && currentPackState != null) { + preparedPack = new SoftReference<>(new PreparedPack(currentPackState, currentPack)); + } currentPack = null; + currentPackState = null; currentPackSettings = null; ProgramBinaryCache.usePack(null); @@ -809,6 +909,8 @@ public class Iris { } catch (Exception e) { everyFolderCompiled = false; logger.error("Failed to compile shader folder '{}' for saving", folder, e); + } finally { + RetainedPrograms.afterPipelineBuilt(); } } loopFinished = true; @@ -818,8 +920,65 @@ public class Iris { } } + private record HashedFile(long size, long modified, String hash) {} + + private static final Map contentHashes = new ConcurrentHashMap<>(); + + private static String contentHash(Path file, BasicFileAttributes attributes) throws IOException { + final long size = attributes.size(); + final long modified = attributes.lastModifiedTime().toMillis(); + final HashedFile known = contentHashes.get(file); + if (known != null && known.size() == size && known.modified() == modified) return known.hash(); + final String hash; + try { + hash = Base64.getEncoder().encodeToString(MessageDigest.getInstance("SHA-256").digest(Files.readAllBytes(file))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + contentHashes.put(file, new HashedFile(size, modified, hash)); + return hash; + } + + private record VisitedFolder(Path path, @Nullable Object key) {} + + private static Map describeFolder(Path root, VisitedFolder folder, List ancestors, boolean byContent) { + final Map files = new HashMap<>(); + final List>> subfolders = new ArrayList<>(); + final List lineage = new ArrayList<>(ancestors); + lineage.add(folder); + try (DirectoryStream entries = Files.newDirectoryStream(folder.path())) { + for (Path entry : entries) { + BasicFileAttributes attributes; + try { + attributes = Files.readAttributes(entry, BasicFileAttributes.class); + } catch (IOException e) { + attributes = Files.readAttributes(entry, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + } + if (attributes.isDirectory()) { + final VisitedFolder subfolder = new VisitedFolder(entry, attributes.fileKey()); + for (VisitedFolder ancestor : lineage) { + final boolean same = subfolder.key() != null && ancestor.key() != null + ? subfolder.key().equals(ancestor.key()) + : Files.isSameFile(ancestor.path(), entry); + if (same) throw new FileSystemLoopException(entry.toString()); + } + subfolders.add(() -> describeFolder(root, subfolder, lineage, byContent)); + } else if (attributes.isRegularFile()) { + files.put(root.relativize(entry).toString(), + byContent ? contentHash(entry, attributes) : attributes.size() + "|" + attributes.lastModifiedTime().toMillis()); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + for (Map nested : AngelicaWorkers.invokeAll(subfolders)) { + files.putAll(nested); + } + return files; + } + @Nullable - private static String describeSettings(String packName, Path packRoot, Properties options, Iterable environmentDefines) { + private static String describeSettings(String packName, Path packRoot, Properties options, Iterable environmentDefines, boolean byContent) { final StringBuilder settings = new StringBuilder(packName).append('\n'); if (Files.isRegularFile(packRoot)) { try { @@ -828,18 +987,15 @@ public class Iris { return null; } } else if (Files.isDirectory(packRoot)) { - try (Stream files = Files.walk(packRoot)) { - files.filter(Files::isRegularFile).sorted().forEach(file -> { - try { - settings.append(packRoot.relativize(file)).append('|').append(Files.size(file)).append('|') - .append(Files.getLastModifiedTime(file).toMillis()).append('\n'); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); + final TreeMap files = new TreeMap<>(); + try { + // One stat per file; this runs on the render thread when shaders turn on, so folders are read in parallel + final Object rootKey = Files.readAttributes(packRoot, BasicFileAttributes.class).fileKey(); + files.putAll(describeFolder(packRoot, new VisitedFolder(packRoot, rootKey), List.of(), byContent)); } catch (IOException | UncheckedIOException e) { return null; } + files.forEach((file, state) -> settings.append(file).append('|').append(state).append('\n')); } else { return null; } @@ -848,6 +1004,7 @@ public class Iris { for (StringPair define : environmentDefines) { settings.append('#').append(define.getKey()).append(' ').append(define.getValue()).append('\n'); } + settings.append("modernFallbackMcVersion=").append(IdMap.modernFallbackMcVersion()).append('\n'); return settings.toString(); } @@ -870,7 +1027,12 @@ public class Iris { ShaderTransformExecutor.prepare(); PerFrameUniformBlockHarvester.harvest(programs); long startTime = System.nanoTime(); - WorldRenderingPipeline pipeline = new DeferredWorldRenderingPipeline(programs); + final WorldRenderingPipeline pipeline; + try { + pipeline = new DeferredWorldRenderingPipeline(programs); + } finally { + RetainedPrograms.afterPipelineBuilt(); + } long endTime = System.nanoTime(); logger.info("[Load #{}] Total shaderpack load time for '{}' in dimension '{}': {} ms", shaderPackLoadId, currentPackName, dimensionName, String.format("%.1f", (endTime - startTime) / 1_000_000.0)); return pipeline; diff --git a/src/main/java/net/coderbot/iris/gl/GlResource.java b/src/main/java/net/coderbot/iris/gl/GlResource.java index 7cf0a4fff..36c5945e5 100644 --- a/src/main/java/net/coderbot/iris/gl/GlResource.java +++ b/src/main/java/net/coderbot/iris/gl/GlResource.java @@ -10,6 +10,7 @@ public abstract class GlResource { } public final void destroy() { + if (!isValid) return; destroyInternal(); isValid = false; } diff --git a/src/main/java/net/coderbot/iris/gl/program/ComputeProgram.java b/src/main/java/net/coderbot/iris/gl/program/ComputeProgram.java index 1c25921ff..b46fb2e9c 100644 --- a/src/main/java/net/coderbot/iris/gl/program/ComputeProgram.java +++ b/src/main/java/net/coderbot/iris/gl/program/ComputeProgram.java @@ -6,6 +6,7 @@ import net.coderbot.iris.gl.GlResource; import net.coderbot.iris.gl.blending.DepthColorStorage; import net.coderbot.iris.pipeline.WorldRenderingPipeline; import net.coderbot.iris.shaderpack.FilledIndirectPointer; +import org.jetbrains.annotations.Nullable; import org.joml.Vector2f; import org.joml.Vector3i; import org.lwjgl.BufferUtils; @@ -27,9 +28,14 @@ public final class ComputeProgram extends GlResource { private float cachedHeight; private Vector3i cachedWorkGroups; - ComputeProgram(int program, ProgramUniforms uniforms, ProgramSamplers samplers, ProgramImages images) { + @Nullable + private final RetainedPrograms.Sources sources; + + ComputeProgram(int program, @Nullable RetainedPrograms.Sources sources, ProgramUniforms uniforms, ProgramSamplers samplers, ProgramImages images) { super(program); + this.sources = sources; + localSizeBuffer = BufferUtils.createIntBuffer(3); RenderSystem.getProgramiv(program, GL43.GL_COMPUTE_WORK_GROUP_SIZE, localSizeBuffer); this.uniforms = uniforms; @@ -93,7 +99,11 @@ public final class ComputeProgram extends GlResource { @Override public void destroyInternal() { DepthColorStorage.unregisterOwnedProgram(getGlId()); - GLStateManager.glDeleteProgram(getGlId()); + if (sources != null) { + RetainedPrograms.retain(sources, getGlId()); + } else { + GLStateManager.glDeleteProgram(getGlId()); + } } /** diff --git a/src/main/java/net/coderbot/iris/gl/program/Program.java b/src/main/java/net/coderbot/iris/gl/program/Program.java index cd2b62f1d..7e4f3d807 100644 --- a/src/main/java/net/coderbot/iris/gl/program/Program.java +++ b/src/main/java/net/coderbot/iris/gl/program/Program.java @@ -5,6 +5,7 @@ import net.coderbot.iris.gl.GlResource; import net.coderbot.iris.gl.blending.DepthColorStorage; import com.gtnewhorizons.angelica.glsm.GLStateManager; import com.gtnewhorizons.angelica.glsm.RenderSystem; +import org.jetbrains.annotations.Nullable; import org.lwjgl.opengl.GL42; import org.lwjgl.opengl.GL43; @@ -25,9 +26,14 @@ public final class Program extends GlResource { @Getter private final ProgramImages images; - Program(int program, ProgramUniforms uniforms, ProgramSamplers samplers, ProgramImages images) { + @Nullable + private final RetainedPrograms.Sources sources; + + Program(int program, @Nullable RetainedPrograms.Sources sources, ProgramUniforms uniforms, ProgramSamplers samplers, ProgramImages images) { super(program); + this.sources = sources; + this.uniforms = uniforms; this.samplers = samplers; this.images = images; @@ -63,7 +69,11 @@ public final class Program extends GlResource { @Override public void destroyInternal() { DepthColorStorage.unregisterOwnedProgram(getGlId()); - GLStateManager.glDeleteProgram(getGlId()); + if (sources != null) { + RetainedPrograms.retain(sources, getGlId()); + } else { + GLStateManager.glDeleteProgram(getGlId()); + } } public int getProgramId() { diff --git a/src/main/java/net/coderbot/iris/gl/program/ProgramBuilder.java b/src/main/java/net/coderbot/iris/gl/program/ProgramBuilder.java index 2d0b7ac9f..f08c1b6f4 100644 --- a/src/main/java/net/coderbot/iris/gl/program/ProgramBuilder.java +++ b/src/main/java/net/coderbot/iris/gl/program/ProgramBuilder.java @@ -21,6 +21,7 @@ import java.util.function.IntSupplier; public class ProgramBuilder extends ProgramUniforms.Builder implements SamplerHolder, ImageHolder { private final int program; + private RetainedPrograms.Sources sources; private final ProgramSamplers.Builder samplers; private final ProgramImages.Builder images; @@ -44,17 +45,29 @@ public class ProgramBuilder extends ProgramUniforms.Builder implements SamplerHo public static ProgramBuilder begin(String name, @Nullable String vertexSource, @Nullable String geometrySource, @Nullable String tessControlSource, @Nullable String tessEvalSource, @Nullable String fragmentSource, ImmutableSet reservedTextureUnits) { - final ProgramBinaryCache.Key cacheKey = ProgramCreator.cacheKey(); + final RetainedPrograms.Sources sources = new RetainedPrograms.Sources(vertexSource, geometrySource, tessControlSource, tessEvalSource, fragmentSource, null); + final int retained = RetainedPrograms.take(sources); + final ProgramBinaryCache.Key cacheKey = retained != 0 && !ProgramBinaryCache.isFullCompileRunning() ? null : ProgramCreator.cacheKey(); if (cacheKey != null) { cacheKey.stage(ShaderType.VERTEX.id, vertexSource); if (geometrySource != null) cacheKey.stage(ShaderType.GEOMETRY.id, geometrySource); if (tessControlSource != null) cacheKey.stage(ShaderType.TESSELATION_CONTROL.id, tessControlSource); if (tessEvalSource != null) cacheKey.stage(ShaderType.TESSELATION_EVAL.id, tessEvalSource); cacheKey.stage(ShaderType.FRAGMENT.id, fragmentSource); + } + if (retained != 0) { + if (cacheKey == null || ProgramBinaryCache.saveRetained(cacheKey, retained)) { + initializeFallbackUniforms(retained); + return new ProgramBuilder(name, retained, reservedTextureUnits).retainAs(sources); + } + GLStateManager.glDeleteProgram(retained); + } + + if (cacheKey != null) { final int saved = ProgramCreator.load(name, cacheKey); if (saved != 0) { initializeFallbackUniforms(saved); - return new ProgramBuilder(name, saved, reservedTextureUnits); + return new ProgramBuilder(name, saved, reservedTextureUnits).retainAs(sources); } } @@ -78,7 +91,7 @@ public class ProgramBuilder extends ProgramUniforms.Builder implements SamplerHo shader.destroy(); } - return new ProgramBuilder(name, programId, reservedTextureUnits); + return new ProgramBuilder(name, programId, reservedTextureUnits).retainAs(sources); } /** Initialize alpha and color uniforms to neutral values until the first pass update. */ @@ -122,12 +135,23 @@ public class ProgramBuilder extends ProgramUniforms.Builder implements SamplerHo throw new IllegalStateException("This PC does not support compute shaders, but it's attempting to be used???"); } - final ProgramBinaryCache.Key cacheKey = ProgramCreator.cacheKey(); + final RetainedPrograms.Sources sources = new RetainedPrograms.Sources(null, null, null, null, null, source); + final int retained = RetainedPrograms.take(sources); + final ProgramBinaryCache.Key cacheKey = retained != 0 && !ProgramBinaryCache.isFullCompileRunning() ? null : ProgramCreator.cacheKey(); if (cacheKey != null) { cacheKey.stage(ShaderType.COMPUTE.id, source); + } + if (retained != 0) { + if (cacheKey == null || ProgramBinaryCache.saveRetained(cacheKey, retained)) { + return new ProgramBuilder(name, retained, reservedTextureUnits).retainAs(sources); + } + GLStateManager.glDeleteProgram(retained); + } + + if (cacheKey != null) { final int saved = ProgramCreator.load(name, cacheKey); if (saved != 0) { - return new ProgramBuilder(name, saved, reservedTextureUnits); + return new ProgramBuilder(name, saved, reservedTextureUnits).retainAs(sources); } } @@ -137,15 +161,20 @@ public class ProgramBuilder extends ProgramUniforms.Builder implements SamplerHo compute.destroy(); - return new ProgramBuilder(name, programId, reservedTextureUnits); + return new ProgramBuilder(name, programId, reservedTextureUnits).retainAs(sources); + } + + private ProgramBuilder retainAs(RetainedPrograms.Sources sources) { + this.sources = sources; + return this; } public Program build() { - return new Program(program, super.buildUniforms(), this.samplers.build(), this.images.build()); + return new Program(program, sources, super.buildUniforms(), this.samplers.build(), this.images.build()); } public ComputeProgram buildCompute() { - return new ComputeProgram(program, super.buildUniforms(), this.samplers.build(), this.images.build()); + return new ComputeProgram(program, sources, super.buildUniforms(), this.samplers.build(), this.images.build()); } private static GlShader buildShader(ShaderType shaderType, String name, @Nullable String source) { diff --git a/src/main/java/net/coderbot/iris/gl/program/RetainedPrograms.java b/src/main/java/net/coderbot/iris/gl/program/RetainedPrograms.java new file mode 100644 index 000000000..3c8aff160 --- /dev/null +++ b/src/main/java/net/coderbot/iris/gl/program/RetainedPrograms.java @@ -0,0 +1,60 @@ +package net.coderbot.iris.gl.program; + +import com.gtnewhorizons.angelica.glsm.GLStateManager; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.Map; + +/** + * Linked programs kept from destroyed pipelines. Turning shaders off and on, reloading an unchanged pack, or + * travelling between two dimensions rebuilds programs from identical sources; taking them from here skips compile and + * link. + */ +public final class RetainedPrograms { + record Sources(@Nullable String vertex, @Nullable String geometry, @Nullable String tessControl, + @Nullable String tessEval, @Nullable String fragment, @Nullable String compute) {} + + private static Map> recent = new HashMap<>(); + private static Map> older = new HashMap<>(); + private static String configuration; + + private RetainedPrograms() {} + + public static void useConfiguration(@Nullable String settings) { + if (settings == null || !settings.equals(configuration)) { + afterPipelineBuilt(); + afterPipelineBuilt(); + } + configuration = settings; + } + + static void retain(Sources sources, int program) { + recent.computeIfAbsent(sources, key -> new ArrayDeque<>()).add(program); + } + + static int take(Sources sources) { + final int program = take(recent, sources); + return program != 0 ? program : take(older, sources); + } + + private static int take(Map> pool, Sources sources) { + if (pool.isEmpty()) return 0; + final ArrayDeque programs = pool.get(sources); + if (programs == null) return 0; + final int program = programs.poll(); + if (programs.isEmpty()) pool.remove(sources); + return program; + } + + public static void afterPipelineBuilt() { + for (ArrayDeque programs : older.values()) { + for (int program : programs) { + GLStateManager.glDeleteProgram(program); + } + } + older = recent; + recent = new HashMap<>(); + } +} diff --git a/src/main/java/net/coderbot/iris/pipeline/DeferredWorldRenderingPipeline.java b/src/main/java/net/coderbot/iris/pipeline/DeferredWorldRenderingPipeline.java index 2a8f4ae8d..e0d2d68af 100644 --- a/src/main/java/net/coderbot/iris/pipeline/DeferredWorldRenderingPipeline.java +++ b/src/main/java/net/coderbot/iris/pipeline/DeferredWorldRenderingPipeline.java @@ -543,26 +543,12 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R this.shadowComputes = createShadowComputes(programs.getShadowCompute()); + final Map builtPrograms = new HashMap<>(); + this.table = new ProgramTable<>((condition, availability) -> { - final int idx; + final ProgramId finalId = gbufferProgramId(ids, condition, availability); - if (availability.texture && availability.lightmap) { - idx = 2; - } else if (availability.texture) { - idx = 1; - } else { - idx = 0; - } - - ProgramId id = ids[condition.ordinal() * 3 + idx]; - - if (id == null) { - id = ids[idx]; - } - - final ProgramId finalId = id; - - return cachedPasses.computeIfAbsent(Pair.of(id, availability), p -> { + return cachedPasses.computeIfAbsent(Pair.of(finalId, availability), p -> { final ProgramSource source = resolver.resolveNullable(p.getLeft()); if (condition == RenderCondition.SHADOW || condition == RenderCondition.SHADOW_TRANSLUCENT) { @@ -583,7 +569,7 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R try { return createPass(source, availability, - condition == RenderCondition.SHADOW || condition == RenderCondition.SHADOW_TRANSLUCENT, finalId); + condition == RenderCondition.SHADOW || condition == RenderCondition.SHADOW_TRANSLUCENT, finalId, builtPrograms); } catch (Exception e) { throw new RuntimeException("Failed to create pass for " + source.getName() + " for rendering condition " + condition + " specialized to input availability " + availability, e); @@ -1131,6 +1117,25 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R null, Collections.emptyList(), false); } + private static ProgramId gbufferProgramId(ProgramId[] ids, RenderCondition condition, InputAvailability availability) { + final int idx; + + if (availability.texture && availability.lightmap) { + idx = 2; + } else if (availability.texture) { + idx = 1; + } else { + idx = 0; + } + + final ProgramId id = ids[condition.ordinal() * 3 + idx]; + return id != null ? id : ids[idx]; + } + + private record SharedProgramKey(String sourceName, InputAvailability availability, boolean shadow) {} + + private record BuiltProgram(Program program, Pass firstPass) {} + private static ProgramBuilder beginBuilder(String name, Map transformed) { final String vertex = transformed.get(PatchShaderType.VERTEX); final String geometry = transformed.get(PatchShaderType.GEOMETRY); @@ -1143,11 +1148,22 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R return ProgramBuilder.begin(name, vertex, geometry, tessControl, tessEval, fragment, IrisSamplers.WORLD_RESERVED_TEXTURE_UNITS); } - private Pass createPass(ProgramSource source, InputAvailability availability, boolean shadow, ProgramId id) { + private Pass createPass(ProgramSource source, InputAvailability availability, boolean shadow, ProgramId id, + Map builtPrograms) { // Use pre-computed transform if available, otherwise transform synchronously Pair key = Pair.of(source.getName(), availability); Map transformed = attributeTransforms.get(key); + // The synchronous fallback below depends on the program id, so only precomputed programs are shared + final SharedProgramKey shareKey = transformed != null ? new SharedProgramKey(source.getName(), availability, shadow) : null; + final BuiltProgram shared = shareKey != null ? builtPrograms.get(shareKey) : null; + if (shared != null) { + final Pass pass = createPassInner(shared.program(), source.getDirectives(), shadow, id); + this.customUniforms.mapPassLike(shared.firstPass(), pass); + pass.setInstancingKey(source.getName(), availability, shadow); + return pass; + } + if (transformed == null) { // Fallback to synchronous transform if not pre-computed String vertex = source.getVertexSource().orElseThrow(NullPointerException::new); @@ -1163,9 +1179,15 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R } ProgramBuilder builder = beginBuilder(source.getName(), transformed); + wireGbufferProgram(builder, availability, shadow); + final Program program = builder.build(); - final Pass pass = createPassInner(builder, source.getDirectives(), availability, shadow, id); + final Pass pass = createPassInner(program, source.getDirectives(), shadow, id); + this.customUniforms.mapholderToPass(builder, pass); pass.setInstancingKey(source.getName(), availability, shadow); + if (shareKey != null) { + builtPrograms.put(shareKey, new BuiltProgram(program, pass)); + } return pass; } @@ -1212,10 +1234,7 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R } } - private Pass createPassInner(ProgramBuilder builder, ProgramDirectives programDirectives, InputAvailability availability, boolean shadow, ProgramId id) { - - wireGbufferProgram(builder, availability, shadow); - + private Pass createPassInner(Program program, ProgramDirectives programDirectives, boolean shadow, ProgramId id) { GlFramebuffer framebufferBeforeTranslucents; GlFramebuffer framebufferAfterTranslucents; @@ -1240,12 +1259,8 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R } }); - Pass pass = new Pass(builder.build(), framebufferBeforeTranslucents, framebufferAfterTranslucents, alphaTestOverride, + return new Pass(program, framebufferBeforeTranslucents, framebufferAfterTranslucents, alphaTestOverride, programDirectives.getBlendModeOverride().orElse(id.getBlendModeOverride()), bufferOverrides, shadow); - - this.customUniforms.mapholderToPass(builder, pass); - - return pass; } public void addGbufferOrShadowSamplers(SamplerHolder samplers, ImageHolder images, Supplier> flipped, @@ -1468,7 +1483,7 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R if (!instancedVariantAttempted[i]) { instancedVariantAttempted[i] = true; if (instancingSourceName != null && supportsInstancing[i]) { - instancedVariants[i] = buildInstancedVariant(instancingSourceName, instancingAvailability, instancingShadow, kind); + instancedVariants[i] = sharedInstancedVariant(instancingSourceName, instancingAvailability, instancingShadow, kind); } } return instancedVariants[i]; @@ -1487,6 +1502,20 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R } } + private record InstancedVariantKey(String sourceName, InputAvailability availability, boolean shadow, Instancing kind) {} + + // Passes sharing a program also share its instanced variants + private final Map instancedVariantPrograms = new HashMap<>(); + + @Nullable + private Program sharedInstancedVariant(String sourceName, InputAvailability availability, boolean shadow, Instancing kind) { + final InstancedVariantKey key = new InstancedVariantKey(sourceName, availability, shadow, kind); + if (instancedVariantPrograms.containsKey(key)) return instancedVariantPrograms.get(key); + final Program variant = buildInstancedVariant(sourceName, availability, shadow, kind); + instancedVariantPrograms.put(key, variant); + return variant; + } + @Nullable private Program buildInstancedVariant(String sourceName, InputAvailability availability, boolean shadow, Instancing kind) { final Map, Map> transforms = instancedAttributeTransforms.get(kind); @@ -2440,6 +2469,16 @@ public class DeferredWorldRenderingPipeline implements WorldRenderingPipeline, R if (source == null) { return false; } + boolean mentioned = false; + for (String builtin : MV_BUILTINS) { + if (source.contains(builtin)) { + mentioned = true; + break; + } + } + if (!mentioned) { + return false; + } try { final Transformer transformer = new Transformer(ShaderParser.parseShader(source).full()); for (String builtin : MV_BUILTINS) { diff --git a/src/main/java/net/coderbot/iris/pipeline/transform/ShaderTransformer.java b/src/main/java/net/coderbot/iris/pipeline/transform/ShaderTransformer.java index 48b433b30..dc8d95986 100644 --- a/src/main/java/net/coderbot/iris/pipeline/transform/ShaderTransformer.java +++ b/src/main/java/net/coderbot/iris/pipeline/transform/ShaderTransformer.java @@ -576,16 +576,16 @@ public class ShaderTransformer { if (vshResult != null && fshResult != null) { throw new RuntimeException("Program " + patchType.name() + " writes images from both VSH and FSH; only one stage may write per program under SDL_GPU"); } - if (vshResult != null) { - result.put(PatchShaderType.VERTEX, vshResult.strippedSource()); - result.put(PatchShaderType.COMPUTE, vshResult.computeSource()); - if (artifactsOut != null) artifactsOut.remove(PatchShaderType.VERTEX); - Iris.logger.info("[RwImageStoreExtractor] Extracted compute pre-pass for {} (mode={}, written={})", patchType.name(), vshResult.mode(), vshResult.writtenImages()); - } else if (fshResult != null) { - result.put(PatchShaderType.FRAGMENT, fshResult.strippedSource()); - result.put(PatchShaderType.COMPUTE, fshResult.computeSource()); - if (artifactsOut != null) artifactsOut.remove(PatchShaderType.FRAGMENT); - Iris.logger.info("[RwImageStoreExtractor] Extracted compute pre-pass for {} (mode={}, written={})", patchType.name(), fshResult.mode(), fshResult.writtenImages()); + final RwImageStoreExtractor.Result extracted = vshResult != null ? vshResult : fshResult; + if (extracted == null) return; + final PatchShaderType stage = vshResult != null ? PatchShaderType.VERTEX : PatchShaderType.FRAGMENT; + result.put(stage, extracted.strippedSource()); + if (artifactsOut != null) artifactsOut.remove(stage); + if (patchType == Patch.COMPOSITE || patchType == Patch.CELERITAS_TERRAIN) { + result.put(PatchShaderType.COMPUTE, extracted.computeSource()); + Iris.logger.info("[RwImageStoreExtractor] Extracted compute pre-pass for {} (mode={}, written={})", patchType.name(), extracted.mode(), extracted.writtenImages()); + } else { + Iris.logger.info("[RwImageStoreExtractor] Dropped image writes from {} (written={})", patchType.name(), extracted.writtenImages()); } } diff --git a/src/main/java/net/coderbot/iris/shaderpack/ConstDirectiveParser.java b/src/main/java/net/coderbot/iris/shaderpack/ConstDirectiveParser.java index 6e5515a24..65975875f 100644 --- a/src/main/java/net/coderbot/iris/shaderpack/ConstDirectiveParser.java +++ b/src/main/java/net/coderbot/iris/shaderpack/ConstDirectiveParser.java @@ -8,15 +8,25 @@ public class ConstDirectiveParser { public static List findDirectives(String source) { List directives = new ArrayList<>(); - // Match any valid newline sequence - // https://stackoverflow.com/a/31060125 - for (String line : source.split("\\R")) { - findDirectiveInLine(line).ifPresent(directives::add); + int from = 0; + while (true) { + final int match = source.indexOf("const", from); + if (match < 0) break; + int start = match; + while (start > 0 && !isLineBreak(source.charAt(start - 1))) start--; + int end = match + "const".length(); + while (end < source.length() && !isLineBreak(source.charAt(end))) end++; + findDirectiveInLine(source.substring(start, end)).ifPresent(directives::add); + from = end; } return directives; } + private static boolean isLineBreak(char c) { + return c == '\n' || c == '\r' || c == '\u000B' || c == '\f' || c == '\u0085' || c == '\u2028' || c == '\u2029'; + } + public static Optional findDirectiveInLine(String line) { // Valid const directives contain the following elements: // * Zero or more whitespace characters diff --git a/src/main/java/net/coderbot/iris/shaderpack/IdMap.java b/src/main/java/net/coderbot/iris/shaderpack/IdMap.java index 8b049d28e..898059fa3 100644 --- a/src/main/java/net/coderbot/iris/shaderpack/IdMap.java +++ b/src/main/java/net/coderbot/iris/shaderpack/IdMap.java @@ -81,8 +81,18 @@ public class IdMap { * Detects whether a pack's block.properties has a dedicated 1.7.10 section. */ public static boolean detectLegacySection(Path shaderPath) { - String rawBlockProperties = readProperties(shaderPath, "block.properties"); - return rawBlockProperties != null && LEGACY_DIRECTIVE_PATTERN.matcher(rawBlockProperties).find(); + final String rawBlockProperties = readProperties(shaderPath, "block.properties"); + if (rawBlockProperties == null) return false; + int from = 0; + while (true) { + final int match = rawBlockProperties.indexOf("10710", from); + if (match < 0) return false; + final int start = rawBlockProperties.lastIndexOf('\n', match) + 1; + int end = rawBlockProperties.indexOf('\n', match); + if (end < 0) end = rawBlockProperties.length(); + if (LEGACY_DIRECTIVE_PATTERN.matcher(rawBlockProperties.substring(start, end)).find()) return true; + from = end; + } } public static String modernFallbackMcVersion() { @@ -90,9 +100,8 @@ public class IdMap { ? String.valueOf(AngelicaConfig.modernFallbackMcVersion) : "260101"; } - IdMap(Path shaderPath, ShaderPackOptions shaderPackOptions, Iterable environmentDefines) { - // Check if block.properties has a dedicated 1.7.10 section - this.hasLegacySection = detectLegacySection(shaderPath); + IdMap(Path shaderPath, boolean hasLegacySection, ShaderPackOptions shaderPackOptions, Iterable environmentDefines) { + this.hasLegacySection = hasLegacySection; Iterable resolvedDefines; if (this.hasLegacySection) { diff --git a/src/main/java/net/coderbot/iris/shaderpack/ShaderPack.java b/src/main/java/net/coderbot/iris/shaderpack/ShaderPack.java index 966116f08..ecf4501e5 100644 --- a/src/main/java/net/coderbot/iris/shaderpack/ShaderPack.java +++ b/src/main/java/net/coderbot/iris/shaderpack/ShaderPack.java @@ -5,6 +5,7 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; import com.google.gson.stream.JsonReader; +import com.gtnewhorizons.angelica.glsm.threading.AngelicaWorkers; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.objects.Object2ObjectMap; import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; @@ -36,6 +37,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.StringReader; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -53,7 +55,10 @@ import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; public class ShaderPack { @@ -79,6 +84,7 @@ public class ShaderPack { private final boolean mappedByDimensionProperties; private final String legacyDefaultFolder; private final Function sourceProvider; + private final ImmutableList sourceFiles; private final ShaderProperties shaderProperties; private boolean hasLoggedCacheLimitReached = false; @@ -90,7 +96,8 @@ public class ShaderPack { @Getter private final Object2ObjectMap customImages; @Getter private final Int2ObjectMap bufferObjects; @Getter private final ShaderPackOptions shaderPackOptions; - @Getter private final OptionMenuContainer menuContainer; + private final ProfileSet profiles; + private OptionMenuContainer menuContainer; private final ProfileSet.ProfileResult profile; private final String profileInfo; @@ -200,8 +207,18 @@ public class ShaderPack { } this.legacyDefaultFolder = legacyDefault; + // Only the options screen reads translations, so they load alongside the rest of the pack + final CompletableFuture languageMapFuture = AngelicaWorkers.submit(() -> { + try { + return new LanguageMap(root.resolve("lang")); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + // Read all files and included files recursively - IncludeGraph graph = new IncludeGraph(root, starts.build()); + this.sourceFiles = starts.build(); + IncludeGraph graph = new IncludeGraph(root, sourceFiles); if (!graph.getFailures().isEmpty()) { graph.getFailures().forEach((path, error) -> { @@ -211,13 +228,14 @@ public class ShaderPack { throw new IOException("Failed to resolve some #include directives, see previous messages for details"); } - this.languageMap = new LanguageMap(root.resolve("lang")); - // Discover, merge, and apply shader pack options this.shaderPackOptions = new ShaderPackOptions(graph, changedConfigs); graph = this.shaderPackOptions.getIncludes(); - final String mcVersionOverride = IdMap.detectLegacySection(root) ? null : IdMap.modernFallbackMcVersion(); + final boolean hasLegacyIdSection = IdMap.detectLegacySection(root); + final String mcVersionOverride = hasLegacyIdSection ? null : IdMap.modernFallbackMcVersion(); + final ShaderPackOptions options = this.shaderPackOptions; + final CompletableFuture idMapFuture = AngelicaWorkers.submit(() -> new IdMap(root, hasLegacyIdSection, options, environmentDefines)); List finalEnvironmentDefines = new ArrayList<>(); for (StringPair define : environmentDefines) { @@ -275,7 +293,7 @@ public class ShaderPack { } }); - this.menuContainer = new OptionMenuContainer(shaderProperties, this.shaderPackOptions, profiles); + this.profiles = profiles; { String profileName = getCurrentProfileName(); @@ -305,38 +323,39 @@ public class ShaderPack { return null; } - ImmutableList lines = includeProcessor.getIncludedFile(path); + final byte[] contentHash = includeProcessor.getIncludedFileHash(path); - if (lines == null) { + if (contentHash == null) { return null; } - StringBuilder builder = new StringBuilder(); - - for (String line : lines) { - builder.append(line); - builder.append('\n'); - } - // Apply GLSL preprocessor to source, while making environment defines available. // // This uses similar techniques to the *.properties preprocessor to avoid actually putting // #define statements in the actual source - instead, we tell the preprocessor about them // directly. This removes one obstacle to accurate reporting of line numbers for errors, // though there exist many more (such as relocating all #extension directives and similar things) - String source = builder.toString(); + return JcppProcessor.glslPreprocessSource(contentHash, () -> { + final ImmutableList lines = includeProcessor.getIncludedFile(path); + int length = 0; + for (String line : lines) { + length += line.length() + 1; + } - // Apply shader pack workarounds for version compatibility (before preprocessing) - source = ShaderPackWorkarounds.apply(source); + final StringBuilder builder = new StringBuilder(length); - source = JcppProcessor.glslPreprocessSource(source, finalEnvironmentDefines1); + for (String line : lines) { + builder.append(line); + builder.append('\n'); + } - return source; + // Apply shader pack workarounds for version compatibility (before preprocessing) + return ShaderPackWorkarounds.apply(builder.toString()); + }, finalEnvironmentDefines1); }; - this.base = new ProgramSet(AbsolutePackPath.fromAbsolutePath("/"), sourceProvider, shaderProperties, this); - - this.idMap = new IdMap(root, shaderPackOptions, environmentDefines); + final AbsolutePackPath rootDirectory = AbsolutePackPath.fromAbsolutePath("/"); + this.base = new ProgramSet(rootDirectory, preprocessInParallel(rootDirectory), shaderProperties, this); customNoiseTexture = shaderProperties.getNoiseTexturePath().map(path -> { try { @@ -375,10 +394,35 @@ public class ShaderPack { this.customImages = shaderProperties.getCustomImages(); this.bufferObjects = shaderProperties.getBufferObjects(); + this.idMap = await(idMapFuture); + this.languageMap = await(languageMapFuture); + } + + // Only the options screen reads the menu, don't build unless it's needed + public OptionMenuContainer getMenuContainer() { + if (menuContainer == null) { + menuContainer = new OptionMenuContainer(shaderProperties, shaderPackOptions, profiles); + } + return menuContainer; + } + + /** Publishes this pack's custom images to the shader transformers; call when the pack becomes the current one. */ + public void activate() { SamplerToStorageImageRewriter.setActiveCandidates(SamplerToStorageImageRewriter.buildCandidates(this.customImages.values())); RwImageStoreExtractor.setActiveCustomImages(this.customImages); } + private static T await(CompletableFuture future) throws IOException { + try { + return future.join(); + } catch (CompletionException e) { + if (e.getCause() instanceof UncheckedIOException io) throw io.getCause(); + if (e.getCause() instanceof RuntimeException runtime) throw runtime; + if (e.getCause() instanceof Error error) throw error; + throw e; + } + } + private String getCurrentProfileName() { return profile.current.map(p -> p.name).orElse("Custom"); } @@ -667,6 +711,26 @@ public class ShaderPack { } } + private Function preprocessInParallel(AbsolutePackPath directory) { + final String directoryPath = directory.getPathString(); + final String prefix = directoryPath.endsWith("/") ? directoryPath : directoryPath + "/"; + final List files = new ArrayList<>(); + final List> tasks = new ArrayList<>(); + for (AbsolutePackPath file : sourceFiles) { + final String path = file.getPathString(); + if (path.startsWith(prefix) && path.indexOf('/', prefix.length()) < 0) { + files.add(file); + tasks.add(() -> sourceProvider.apply(file)); + } + } + final List sources = AngelicaWorkers.invokeAll(tasks); + final Map preprocessed = new HashMap<>(); + for (int i = 0; i < files.size(); i++) { + preprocessed.put(files.get(i), sources.get(i)); + } + return path -> preprocessed.containsKey(path) ? preprocessed.get(path) : sourceProvider.apply(path); + } + /** * Gets or creates the appropriate ProgramSet for the given dimension. * @@ -721,12 +785,8 @@ public class ShaderPack { if (programSet == null) { // Create ProgramSet on-demand for dimension folder try { - programSet = new ProgramSet( - AbsolutePackPath.fromAbsolutePath("/" + folderName), - sourceProvider, - shaderProperties, - this - ); + final AbsolutePackPath directory = AbsolutePackPath.fromAbsolutePath("/" + folderName); + programSet = new ProgramSet(directory, preprocessInParallel(directory), shaderProperties, this); dimensionProgramSets.put(folderName, programSet); // Check cache size limit and evict LRU dimensions if needed diff --git a/src/main/java/net/coderbot/iris/shaderpack/include/IncludeGraph.java b/src/main/java/net/coderbot/iris/shaderpack/include/IncludeGraph.java index e1188115b..c15406056 100644 --- a/src/main/java/net/coderbot/iris/shaderpack/include/IncludeGraph.java +++ b/src/main/java/net/coderbot/iris/shaderpack/include/IncludeGraph.java @@ -2,6 +2,7 @@ package net.coderbot.iris.shaderpack.include; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.gtnewhorizons.angelica.glsm.threading.AngelicaWorkers; import net.coderbot.iris.Iris; import net.coderbot.iris.shaderpack.error.RusticError; import net.coderbot.iris.shaderpack.transform.line.LineTransform; @@ -18,6 +19,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.function.Function; /** @@ -75,14 +77,18 @@ public class IncludeGraph { List queue = new ArrayList<>(startingPaths); Set seen = new HashSet<>(startingPaths); + Map> reads = new HashMap<>(); + for (AbsolutePackPath start : startingPaths) { + reads.put(start, readLater(root, start)); + } + while (!queue.isEmpty()) { AbsolutePackPath next = queue.remove(queue.size() - 1); - String source; + final ReadResult read = reads.remove(next).join(); + final IOException e = read.failure(); - try { - source = readFile(next.resolved(root)); - } catch (IOException e) { + if (e != null) { AbsolutePackPath src = cameFrom.get(next); if (src == null) { @@ -110,7 +116,7 @@ public class IncludeGraph { continue; } - ImmutableList lines = ImmutableList.copyOf(source.split("\\R")); + ImmutableList lines = read.lines(); FileNode node = new FileNode(next, lines); boolean selfInclude = false; @@ -128,6 +134,7 @@ public class IncludeGraph { } else if (!seen.contains(included)) { queue.add(included); seen.add(included); + reads.put(included, readLater(root, included)); cameFrom.put(included, next); lineNumberInclude.put(included, line); } @@ -254,6 +261,18 @@ public class IncludeGraph { return failures; } + private record ReadResult(ImmutableList lines, IOException failure) {} + + private static CompletableFuture readLater(Path root, AbsolutePackPath path) { + return AngelicaWorkers.submit(() -> { + try { + return new ReadResult(ImmutableList.copyOf(readFile(path.resolved(root)).split("\\R")), null); + } catch (IOException e) { + return new ReadResult(null, e); + } + }); + } + private static String readFile(Path path) throws IOException { return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); } diff --git a/src/main/java/net/coderbot/iris/shaderpack/include/IncludeProcessor.java b/src/main/java/net/coderbot/iris/shaderpack/include/IncludeProcessor.java index b06db48cb..0df2f3287 100644 --- a/src/main/java/net/coderbot/iris/shaderpack/include/IncludeProcessor.java +++ b/src/main/java/net/coderbot/iris/shaderpack/include/IncludeProcessor.java @@ -3,18 +3,69 @@ package net.coderbot.iris.shaderpack.include; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.util.HashMap; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; // TODO: Write tests for this code public class IncludeProcessor { private final IncludeGraph graph; private final Map> cache; + private final Map hashes; + public IncludeProcessor(IncludeGraph graph) { this.graph = graph; - this.cache = new HashMap<>(); + this.cache = new ConcurrentHashMap<>(); + this.hashes = new ConcurrentHashMap<>(); + } + + public byte[] getIncludedFileHash(AbsolutePackPath path) { + byte[] hash = hashes.get(path); + + if (hash == null) { + hash = hash(path); + if (hash != null) hashes.put(path, hash); + } + + return hash; + } + + private byte[] hash(AbsolutePackPath path) { + final FileNode fileNode = graph.getNodes().get(path); + + if (fileNode == null) { + return null; + } + + final MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + + final ImmutableList lines = fileNode.getLines(); + final ImmutableMap includes = fileNode.getIncludes(); + + // A line can't contain '\n' and an include is a fixed 32 bytes, so the encoding is unambiguous + for (int i = 0; i < lines.size(); i++) { + final AbsolutePackPath include = includes.get(i); + + if (include != null) { + digest.update((byte) 1); + digest.update(Objects.requireNonNull(getIncludedFileHash(include))); + } else { + digest.update((byte) 0); + digest.update(lines.get(i).getBytes(StandardCharsets.UTF_8)); + digest.update((byte) '\n'); + } + } + + return digest.digest(); } // TODO: Actual error handling @@ -24,7 +75,7 @@ public class IncludeProcessor { if (lines == null) { lines = process(path); - cache.put(path, lines); + if (lines != null) cache.put(path, lines); } return lines; diff --git a/src/main/java/net/coderbot/iris/shaderpack/preprocessor/JcppProcessor.java b/src/main/java/net/coderbot/iris/shaderpack/preprocessor/JcppProcessor.java index bbc217e90..a6acefd49 100644 --- a/src/main/java/net/coderbot/iris/shaderpack/preprocessor/JcppProcessor.java +++ b/src/main/java/net/coderbot/iris/shaderpack/preprocessor/JcppProcessor.java @@ -1,5 +1,6 @@ package net.coderbot.iris.shaderpack.preprocessor; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import net.coderbot.iris.shaderpack.StringPair; import org.anarres.cpp.Feature; import org.anarres.cpp.LexerException; @@ -7,7 +8,18 @@ import org.anarres.cpp.Preprocessor; import org.anarres.cpp.StringLexerSource; import org.anarres.cpp.Token; +import java.util.function.Supplier; + public class JcppProcessor { + public static String glslPreprocessSource(byte[] contentHash, Supplier source, Iterable environmentDefines) { + final ShaderDiskCache.Key cacheKey = PreprocessedSourceCache.key("glsl", contentHash, environmentDefines); + final String cached = PreprocessedSourceCache.get(cacheKey); + if (cached != null) return cached; + final String preprocessed = glslPreprocessSource(source.get(), environmentDefines); + PreprocessedSourceCache.put(cacheKey, preprocessed); + return preprocessed; + } + // Derived from GlShader from Canvas, licenced under LGPL public static String glslPreprocessSource(String source, Iterable environmentDefines) { if (source.contains(GlslCollectingListener.VERSION_MARKER) diff --git a/src/main/java/net/coderbot/iris/shaderpack/preprocessor/PreprocessedSourceCache.java b/src/main/java/net/coderbot/iris/shaderpack/preprocessor/PreprocessedSourceCache.java new file mode 100644 index 000000000..30856c492 --- /dev/null +++ b/src/main/java/net/coderbot/iris/shaderpack/preprocessor/PreprocessedSourceCache.java @@ -0,0 +1,36 @@ +package net.coderbot.iris.shaderpack.preprocessor; + +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; +import net.coderbot.iris.shaderpack.StringPair; +import org.jetbrains.annotations.Nullable; + +/** Disk cache of preprocessor output, keyed on the input and every macro handed to the preprocessor. */ +final class PreprocessedSourceCache { + private PreprocessedSourceCache() {} + + @Nullable + static ShaderDiskCache.Key key(String kind, String source, Iterable defines) { + return ShaderDiskCache.isEnabled() ? withDefines(ShaderDiskCache.key("iris-preprocess").str(kind).str(source), defines) : null; + } + + @Nullable + static ShaderDiskCache.Key key(String kind, byte[] contentHash, Iterable defines) { + return ShaderDiskCache.isEnabled() ? withDefines(ShaderDiskCache.key("iris-preprocess").str(kind).bytes(contentHash), defines) : null; + } + + private static ShaderDiskCache.Key withDefines(ShaderDiskCache.Key key, Iterable defines) { + for (StringPair define : defines) { + key.str(define.getKey()).str(define.getValue()); + } + return key; + } + + @Nullable + static String get(@Nullable ShaderDiskCache.Key key) { + return key == null ? null : ShaderDiskCache.getString(key); + } + + static void put(@Nullable ShaderDiskCache.Key key, String preprocessed) { + if (key != null) ShaderDiskCache.putString(key, preprocessed); + } +} diff --git a/src/main/java/net/coderbot/iris/shaderpack/preprocessor/PropertiesPreprocessor.java b/src/main/java/net/coderbot/iris/shaderpack/preprocessor/PropertiesPreprocessor.java index f2fb0e53d..9a07a90af 100644 --- a/src/main/java/net/coderbot/iris/shaderpack/preprocessor/PropertiesPreprocessor.java +++ b/src/main/java/net/coderbot/iris/shaderpack/preprocessor/PropertiesPreprocessor.java @@ -1,5 +1,6 @@ package net.coderbot.iris.shaderpack.preprocessor; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import net.coderbot.iris.Iris; import net.coderbot.iris.shaderpack.StringPair; import net.coderbot.iris.shaderpack.option.ShaderPackOptions; @@ -17,7 +18,6 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.stream.Collectors; public class PropertiesPreprocessor { // Derived from ShaderProcessor.glslPreprocessSource, which is derived from GlShader from Canvas, licenced under LGPL @@ -29,6 +29,14 @@ public class PropertiesPreprocessor { final List booleanValues = getBooleanValues(shaderPackOptions); final Map stringValues = getStringValues(shaderPackOptions); + final List macros = new ArrayList<>(); + for (String value : booleanValues) macros.add(new StringPair(value, "")); + for (StringPair envDefine : environmentDefines) macros.add(envDefine); + stringValues.forEach((name, value) -> macros.add(new StringPair(name, value))); + final ShaderDiskCache.Key cacheKey = PreprocessedSourceCache.key("properties", source, macros); + final String cached = PreprocessedSourceCache.get(cacheKey); + if (cached != null) return cached; + try (Preprocessor pp = new Preprocessor()) { for (String value : booleanValues) { pp.addMacro(value); @@ -46,7 +54,9 @@ public class PropertiesPreprocessor { } }); - return process(pp, source); + final String preprocessed = process(pp, source); + PreprocessedSourceCache.put(cacheKey, preprocessed); + return preprocessed; } catch (IOException e) { throw new RuntimeException("Unexpected IOException while processing macros", e); } catch (LexerException e) { @@ -77,24 +87,10 @@ public class PropertiesPreprocessor { PropertyCollectingListener listener = new PropertyCollectingListener(); preprocessor.setListener(listener); - // Not super efficient, but this removes trailing whitespace on lines, fixing an issue with whitespace after + // This removes trailing whitespace on lines, fixing an issue with whitespace after // line continuations (see PreprocessorTest#testWeirdPropertiesLineContinuation) // Required for Voyager Shader - source = Arrays.stream(source.split("\\r\\n|\\n|\\r")).map(String::trim).filter(s -> !s.isEmpty()) - .map(line -> { - if (line.startsWith("#")) { - for (PreprocessorCommand command : PreprocessorCommand.values()) { - if (line.startsWith("#" + (command.name().replace("PP_", "").toLowerCase(Locale.ROOT)))) { - return line; - } - } - return ""; - } - // In PropertyCollectingListener we suppress "unknown preprocessor directive errors" and - // assume the line to be a comment, since in .properties files `#` also functions as a comment - // marker. - return line.replace("#", ""); - }).collect(Collectors.joining("\n")) + "\n"; + source = cleanLines(source); // TODO: This is a horrible fix to trick the preprocessor into not seeing the backslashes during processing. We need a better way to do this. source = source.replace("\\", "IRIS_PASSTHROUGHBACKSLASH"); @@ -119,6 +115,46 @@ public class PropertiesPreprocessor { return (listener.collectLines() + source).replace("IRIS_PASSTHROUGHBACKSLASH", "\\"); } + private static final String[] DIRECTIVE_PREFIXES = Arrays.stream(PreprocessorCommand.values()) + .map(command -> "#" + command.name().replace("PP_", "").toLowerCase(Locale.ROOT)) + .toArray(String[]::new); + + // Same result as splitting on \r\n|\n|\r, trimming, dropping empty lines and joining with \n + static String cleanLines(String source) { + final StringBuilder out = new StringBuilder(source.length()); + boolean first = true; + int start = 0; + final int length = source.length(); + while (start <= length) { + int end = start; + while (end < length && source.charAt(end) != '\n' && source.charAt(end) != '\r') end++; + final String line = source.substring(start, end).trim(); + if (!line.isEmpty()) { + if (!first) out.append('\n'); + first = false; + out.append(cleanLine(line)); + } + if (end < length && source.charAt(end) == '\r' && end + 1 < length && source.charAt(end + 1) == '\n') end++; + start = end + 1; + } + return out.append('\n').toString(); + } + + private static String cleanLine(String line) { + if (line.startsWith("#")) { + for (String prefix : DIRECTIVE_PREFIXES) { + if (line.startsWith(prefix)) { + return line; + } + } + return ""; + } + // In PropertyCollectingListener we suppress "unknown preprocessor directive errors" and + // assume the line to be a comment, since in .properties files `#` also functions as a comment + // marker. + return line.indexOf('#') < 0 ? line : line.replace("#", ""); + } + private static List getBooleanValues(ShaderPackOptions shaderPackOptions) { List booleanValues = new ArrayList<>(); diff --git a/src/main/java/net/coderbot/iris/texture/pbr/PBRResourceIndex.java b/src/main/java/net/coderbot/iris/texture/pbr/PBRResourceIndex.java index 8ce3e7f1a..bccd625bc 100644 --- a/src/main/java/net/coderbot/iris/texture/pbr/PBRResourceIndex.java +++ b/src/main/java/net/coderbot/iris/texture/pbr/PBRResourceIndex.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; import java.util.Enumeration; +import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; import java.util.List; @@ -46,6 +47,7 @@ public final class PBRResourceIndex { private static IResourceManager owner; private static CompletableFuture> pending; private static Map built; + private static final Map answers = new HashMap<>(); private static boolean disabled; private static boolean warned; @@ -75,6 +77,7 @@ public final class PBRResourceIndex { owner = null; pending = null; built = null; + answers.clear(); } public static synchronized boolean mayExist(IResourceManager resourceManager, ResourceLocation location) { @@ -99,6 +102,16 @@ public final class PBRResourceIndex { return true; } } + final Boolean known = answers.get(location); + if (known != null) { + return known; + } + final boolean present = lookUp(manager, location, path); + answers.put(location, present); + return present; + } + + private static boolean lookUp(SimpleReloadableResourceManager manager, ResourceLocation location, String path) { final Object domain = manager.domainResourceManagers.get(location.getResourceDomain()); if (domain == null) { return false; diff --git a/src/main/java/net/coderbot/iris/uniforms/custom/CustomUniforms.java b/src/main/java/net/coderbot/iris/uniforms/custom/CustomUniforms.java index 99d0c42ff..ca649bd33 100644 --- a/src/main/java/net/coderbot/iris/uniforms/custom/CustomUniforms.java +++ b/src/main/java/net/coderbot/iris/uniforms/custom/CustomUniforms.java @@ -234,6 +234,11 @@ public class CustomUniforms implements FunctionContext { locationMap.put(pass, locationMap.remove(holder)); } + public void mapPassLike(Object existingPass, Object pass) { + final PassUniforms uniforms = locationMap.get(existingPass); + if (uniforms != null) locationMap.put(pass, uniforms); + } + public void update() { if (Tracy.ENABLED) { diff --git a/src/test/java/net/coderbot/iris/PackSettingsFingerprintTest.java b/src/test/java/net/coderbot/iris/PackSettingsFingerprintTest.java new file mode 100644 index 000000000..d76e6a926 --- /dev/null +++ b/src/test/java/net/coderbot/iris/PackSettingsFingerprintTest.java @@ -0,0 +1,39 @@ +package net.coderbot.iris; + +import com.gtnewhorizons.angelica.config.AngelicaConfig; +import net.coderbot.iris.shaderpack.StringPair; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.*; + +class PackSettingsFingerprintTest { + @TempDir Path dir; + + @Test + void effectiveFallbackVersionInvalidatesPreparedAndCompiledPackSettings() throws Exception { + final Path pack = Files.write(dir.resolve("pack.zip"), new byte[] {1}); + final Method describe = Iris.class.getDeclaredMethod("describeSettings", String.class, Path.class, + Properties.class, Iterable.class, boolean.class); + describe.setAccessible(true); + final int previous = AngelicaConfig.modernFallbackMcVersion; + try { + for (boolean byContent : new boolean[] {false, true}) { + AngelicaConfig.modernFallbackMcVersion = 0; + final Object defaults = describe.invoke(null, "pack", pack, new Properties(), List.of(new StringPair("MC_VERSION", "10710")), byContent); + AngelicaConfig.modernFallbackMcVersion = 260101; + assertEquals(defaults, describe.invoke(null, "pack", pack, new Properties(), List.of(new StringPair("MC_VERSION", "10710")), byContent)); + AngelicaConfig.modernFallbackMcVersion = 12001; + assertNotEquals(defaults, describe.invoke(null, "pack", pack, new Properties(), List.of(new StringPair("MC_VERSION", "10710")), byContent)); + } + } finally { + AngelicaConfig.modernFallbackMcVersion = previous; + } + } +} diff --git a/src/test/java/net/coderbot/iris/gl/program/ProgramBindingRegressionTest.java b/src/test/java/net/coderbot/iris/gl/program/ProgramBindingRegressionTest.java index a651ed39e..13db39eaf 100644 --- a/src/test/java/net/coderbot/iris/gl/program/ProgramBindingRegressionTest.java +++ b/src/test/java/net/coderbot/iris/gl/program/ProgramBindingRegressionTest.java @@ -109,7 +109,7 @@ class ProgramBindingRegressionTest { int id = linkedProgram(); int other = linkedProgram(); ProgramUniforms uniforms = mock(ProgramUniforms.class); - Program program = new Program(id, uniforms, mock(ProgramSamplers.class), mock(ProgramImages.class)); + Program program = new Program(id, null, uniforms, mock(ProgramSamplers.class), mock(ProgramImages.class)); DeferredWorldRenderingPipeline pipeline = mock(DeferredWorldRenderingPipeline.class); DeferredWorldRenderingPipeline.Pass pass = Reflect.allocate(DeferredWorldRenderingPipeline.Pass.class); Reflect.set(pass, "program", program);