From d4f6df5a81f3c0f72e4a49c646ff4f4dc4b6c316 Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Sun, 27 Feb 2022 06:56:20 -0300 Subject: [PATCH 01/14] chore: add `zstd` and modify encoding on compile time to `UTF-8` --- build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9afd07516..7cd22a284 100644 --- a/build.gradle +++ b/build.gradle @@ -42,7 +42,7 @@ dependencies { // for more info... // http://www.gradle.org/docs/current/userguide/artifact_dependencies_tutorial.html // http://www.gradle.org/docs/current/userguide/dependency_management.html - + compile 'com.github.luben:zstd-jni:1.5.2-2' } processResources @@ -65,4 +65,8 @@ processResources } } +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + sourceSets {main { output.resourcesDir = output.classesDir }} From c3bba543f0672f5e0ddd25d4fe153600a3565921 Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Sun, 27 Feb 2022 06:56:57 -0300 Subject: [PATCH 02/14] feat: add `ZstdCompressor` to write/read almost compressed data --- .../luizotavio/compressor/ZstdCompressor.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 src/main/java/me/luizotavio/compressor/ZstdCompressor.java diff --git a/src/main/java/me/luizotavio/compressor/ZstdCompressor.java b/src/main/java/me/luizotavio/compressor/ZstdCompressor.java new file mode 100644 index 000000000..694229539 --- /dev/null +++ b/src/main/java/me/luizotavio/compressor/ZstdCompressor.java @@ -0,0 +1,61 @@ +package me.luizotavio.compressor; + +import com.github.luben.zstd.Zstd; +import net.minecraft.nbt.CompressedStreamTools; +import net.minecraft.nbt.NBTSizeTracker; +import net.minecraft.nbt.NBTTagCompound; + +import java.io.*; + +public class ZstdCompressor { + + public static byte[] readCompressed(DataInput inputStream) throws IOException { + int compressedLength = inputStream.readInt(); + int uncompressedLength = inputStream.readInt(); + + if (compressedLength < 0 || uncompressedLength < 0) { + throw new IllegalArgumentException("Invalid length"); + } + + byte[] compressed = new byte[compressedLength]; + + inputStream.readFully(compressed); + + return Zstd.decompress(compressed, uncompressedLength); + } + + public static NBTTagCompound readCompressCompound(DataInput inputStream) throws IOException { + NBTTagCompound compound; + + byte[] compressed = readCompressed(inputStream); + + try (DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(compressed))) { + compound = CompressedStreamTools.func_152456_a(dataInputStream, NBTSizeTracker.field_152451_a); + } + + return compound; + } + + public static byte[] writeCompound(NBTTagCompound compound) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(), + copy = new ByteArrayOutputStream(); + + try (DataOutputStream dataOutputStream = new DataOutputStream(outputStream)) { + CompressedStreamTools.write(compound, dataOutputStream); + } + + byte[] uncompressed = outputStream.toByteArray(); + + byte[] compress = Zstd.compress(uncompressed); + + try (DataOutputStream dataOutputStream = new DataOutputStream(copy)) { + dataOutputStream.writeInt(compress.length); + dataOutputStream.writeInt(uncompressed.length); + + dataOutputStream.write(compress); + + return copy.toByteArray(); + } + } + +} From 4bdbe0efee1bc349d320d9009bd51dcbe42360f6 Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Sun, 27 Feb 2022 06:57:37 -0300 Subject: [PATCH 03/14] refactor: ensure almost compression types to load/write each `NBTTagCompound` --- .../controllers/PlayerDataController.java | 106 ++++++++++++++++-- 1 file changed, 94 insertions(+), 12 deletions(-) diff --git a/src/main/java/noppes/npcs/controllers/PlayerDataController.java b/src/main/java/noppes/npcs/controllers/PlayerDataController.java index 4c72212d7..c50a9ff44 100644 --- a/src/main/java/noppes/npcs/controllers/PlayerDataController.java +++ b/src/main/java/noppes/npcs/controllers/PlayerDataController.java @@ -1,21 +1,24 @@ package noppes.npcs.controllers; -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; import java.util.HashMap; import java.util.Map; +import me.luizotavio.compressor.ZstdCompressor; import net.minecraft.entity.player.EntityPlayer; import net.minecraft.nbt.CompressedStreamTools; +import net.minecraft.nbt.NBTSizeTracker; import net.minecraft.nbt.NBTTagCompound; import net.minecraft.server.MinecraftServer; import noppes.npcs.CustomNpcs; import noppes.npcs.LogWriter; +import noppes.npcs.util.JsonException; import noppes.npcs.util.NBTJsonUtil; +import static me.luizotavio.compressor.executor.IOExecutor.IO_EXECUTOR; + public class PlayerDataController { public static PlayerDataController instance; @@ -74,10 +77,9 @@ public class PlayerDataController { filename = "noplayername"; filename += ".json"; try { - File file = new File(saveDir, filename); - if(file.exists()){ - return NBTJsonUtil.LoadFile(file); - } + return ensureCompressor( + new File(saveDir, filename) + ); } catch (Exception e) { LogWriter.error("Error loading: " + filename, e); } @@ -87,12 +89,23 @@ public class PlayerDataController { public void savePlayerData(PlayerData data){ NBTTagCompound compound = data.getNBT(); - String filename = data.uuid + ".json"; + String filename; + + if (CustomNpcs.CompressorType.equalsIgnoreCase("lz4")) { + filename = data.uuid + ".lz4"; + } else if (CustomNpcs.CompressorType.equalsIgnoreCase("json")) { + filename = data.uuid + ".json"; + } else { + filename = data.uuid + ".dat"; + } + try { File saveDir = getSaveDir(); File file = new File(saveDir, filename+"_new"); File file1 = new File(saveDir, filename); - NBTJsonUtil.SaveFile(file, compound); + + writeCompressor(file, compound); + if(file1.exists()){ file1.delete(); } @@ -164,7 +177,8 @@ public class PlayerDataController { if(file.isDirectory() || !file.getName().endsWith(".json")) continue; try { - NBTTagCompound compound = NBTJsonUtil.LoadFile(file); + NBTTagCompound compound = ensureCompressor(file); + if(compound.hasKey("PlayerName")){ map.put(compound.getString("PlayerName"), compound); } @@ -178,4 +192,72 @@ public class PlayerDataController { public boolean hasMail(EntityPlayer player) { return getPlayerData(player).mailData.hasMail(); } + + private void writeCompressor(File file, NBTTagCompound compound) { + if (compound == null) { + throw new IllegalArgumentException("Compound cannot be null!"); + } + + if (CustomNpcs.CompressorType.equalsIgnoreCase("json")) { + IO_EXECUTOR.submit(() -> { + try { + NBTJsonUtil.SaveFile(file, compound); + } catch (IOException | JsonException e) { + e.printStackTrace(); + } + }); + } else if (CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) { + IO_EXECUTOR.submit(() -> { + try { + byte[] compressed = ZstdCompressor.writeCompound(compound); + + Files.write( + file.toPath(), + compressed, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE + ); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + } + + private NBTTagCompound ensureCompressor(File file) throws JsonException, IOException { + if (!file.exists()) { + return new NBTTagCompound(); + } + + if (CustomNpcs.CompressorType.equalsIgnoreCase("json")) { + if (file.getName().endsWith(".zstd")) { + byte[] readCompressed = ZstdCompressor.readCompressed( + new RandomAccessFile(file, "r") + ); + + try (DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(readCompressed))) { + return CompressedStreamTools.func_152456_a(dataInputStream, NBTSizeTracker.field_152451_a); + } + } + + return NBTJsonUtil.LoadFile(file); + } + + if (CustomNpcs.CompressorType.equalsIgnoreCase("lz4")) { + if (file.getName().endsWith(".json")) { + return NBTJsonUtil.LoadFile(file); + } + + byte[] readCompressed = ZstdCompressor.readCompressed( + new RandomAccessFile(file, "r") + ); + + try (DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(readCompressed))) { + return CompressedStreamTools.func_152456_a(dataInputStream, NBTSizeTracker.field_152451_a); + } + } + + throw new IOException("Unknown compressor type: " + CustomNpcs.CompressorType); + } } From 34b6eb05e58041de01ac1683f434242f221dcea3 Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Sun, 27 Feb 2022 06:58:53 -0300 Subject: [PATCH 04/14] feat: integrate the compressor type to config property and add number of IO-Workers to write compressed files --- .../compressor/executor/IOExecutor.java | 16 +++++ src/main/java/noppes/npcs/CustomNpcs.java | 64 +++++++------------ 2 files changed, 40 insertions(+), 40 deletions(-) create mode 100644 src/main/java/me/luizotavio/compressor/executor/IOExecutor.java diff --git a/src/main/java/me/luizotavio/compressor/executor/IOExecutor.java b/src/main/java/me/luizotavio/compressor/executor/IOExecutor.java new file mode 100644 index 000000000..f0f5da14e --- /dev/null +++ b/src/main/java/me/luizotavio/compressor/executor/IOExecutor.java @@ -0,0 +1,16 @@ +package me.luizotavio.compressor.executor; + +import java.util.concurrent.ExecutorService; + +public class IOExecutor { + + /** + * Please, do not change this value unless you know what you are doing. + * This value is used to define the number of threads that will be used to + * compress the files. + * + * If you want to change this value, please, do it with care. + */ + public static ExecutorService IO_EXECUTOR; + +} diff --git a/src/main/java/noppes/npcs/CustomNpcs.java b/src/main/java/noppes/npcs/CustomNpcs.java index a5aaa037e..75c76c0cf 100644 --- a/src/main/java/noppes/npcs/CustomNpcs.java +++ b/src/main/java/noppes/npcs/CustomNpcs.java @@ -1,8 +1,6 @@ package noppes.npcs; -import java.io.File; -import java.util.Set; - +import com.google.common.util.concurrent.ThreadFactoryBuilder; import cpw.mods.fml.common.FMLCommonHandler; import cpw.mods.fml.common.Mod; import cpw.mods.fml.common.Mod.EventHandler; @@ -12,6 +10,7 @@ import cpw.mods.fml.common.network.FMLEventChannel; import cpw.mods.fml.common.network.NetworkRegistry; import cpw.mods.fml.common.registry.EntityRegistry; import foxz.command.CommandNoppes; +import me.luizotavio.compressor.executor.IOExecutor; import net.minecraft.block.Block; import net.minecraft.block.BlockIce; import net.minecraft.block.BlockLeavesBase; @@ -25,50 +24,30 @@ import net.minecraftforge.common.MinecraftForge; import nikedemos.markovnames.generators.*; import noppes.npcs.config.ConfigLoader; import noppes.npcs.config.ConfigProp; -import noppes.npcs.controllers.BankController; -import noppes.npcs.controllers.ChunkController; -import noppes.npcs.controllers.DialogController; -import noppes.npcs.controllers.FactionController; -import noppes.npcs.controllers.GlobalDataController; -import noppes.npcs.controllers.LinkedNpcController; -import noppes.npcs.controllers.PixelmonHelper; -import noppes.npcs.controllers.PlayerDataController; -import noppes.npcs.controllers.QuestController; -import noppes.npcs.controllers.RecipeController; -import noppes.npcs.controllers.ScriptController; -import noppes.npcs.controllers.ServerCloneController; -import noppes.npcs.controllers.SpawnController; -import noppes.npcs.controllers.TransportController; +import noppes.npcs.controllers.*; import noppes.npcs.enchants.EnchantInterface; import noppes.npcs.entity.*; -import noppes.npcs.entity.old.EntityNPCDwarfFemale; -import noppes.npcs.entity.old.EntityNPCDwarfMale; -import noppes.npcs.entity.old.EntityNPCElfFemale; -import noppes.npcs.entity.old.EntityNPCElfMale; -import noppes.npcs.entity.old.EntityNPCEnderman; -import noppes.npcs.entity.old.EntityNPCFurryFemale; -import noppes.npcs.entity.old.EntityNPCFurryMale; -import noppes.npcs.entity.old.EntityNPCHumanFemale; -import noppes.npcs.entity.old.EntityNPCHumanMale; -import noppes.npcs.entity.old.EntityNPCOrcFemale; -import noppes.npcs.entity.old.EntityNPCOrcMale; -import noppes.npcs.entity.old.EntityNPCVillager; -import noppes.npcs.entity.old.EntityNpcEnderchibi; -import noppes.npcs.entity.old.EntityNpcMonsterFemale; -import noppes.npcs.entity.old.EntityNpcMonsterMale; -import noppes.npcs.entity.old.EntityNpcNagaFemale; -import noppes.npcs.entity.old.EntityNpcNagaMale; -import noppes.npcs.entity.old.EntityNpcSkeleton; +import noppes.npcs.entity.old.*; import noppes.npcs.scripted.wrapper.WrapperNpcAPI; +import java.io.File; +import java.util.Set; +import java.util.concurrent.Executors; + @Mod(modid = "customnpcs", name = "CustomNpcs", version = "1.5.1-beta") public class CustomNpcs { - @ConfigProp(info = "Disable Chat Bubbles") + @ConfigProp(info = "Disable Chat Bubbles") public static boolean EnableChatBubbles = true; - + private static int NewEntityStartId = 0; - + + @ConfigProp(info = "The type of compressor which is used to compress and decompress player data. If you are experiencing issues with the server lagging/crashing, try changing this to 'zstd'") + public static String CompressorType = "json"; + + @ConfigProp(info = "The size of IO executors used for compression and decompression. This is the number of threads used for compression and decompression. If you are experiencing issues with the server lagging/crashing, try changing this to a higher number.") + public static int CompressorThreads = 2; + @ConfigProp(info = "Navigation search range for NPCs. Not recommended to increase if you have a slow pc or on a server. Minimum of 16, maximum of 96.") public static int NpcNavRange = 32; @@ -227,14 +206,19 @@ public class CustomNpcs { registerNpc(EntityNPCGolem.class, "npcGolem"); registerNpc(EntityCustomNpc.class, "CustomNpc"); - + IOExecutor.IO_EXECUTOR = Executors.newFixedThreadPool( + CompressorThreads, + new ThreadFactoryBuilder() + .setNameFormat("CustomNpcs-IO-%d") + .build() + ); registerNewEntity(EntityChairMount.class, "CustomNpcChairMount", 64, 10, false); registerNewEntity(EntityProjectile.class, "throwableitem", 64, 3, true); registerNewEntity(EntityMagicProjectile.class, "magicprojectile", 64, 3, true); new RecipeController(); - + ForgeChunkManager.setForcedChunkLoadingCallback(this, new ChunkController()); new CustomNpcsPermissions(); From 3ac531a49da882dc0c2930a47314a9c9586fbcbc Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Sun, 27 Feb 2022 06:59:24 -0300 Subject: [PATCH 05/14] feat: add `compressor` command to convert data files to `zstd` or `json` --- src/main/java/foxz/command/CmdCompressor.java | 185 ++++++++++++++++++ src/main/java/foxz/command/CmdNoppes.java | 2 +- 2 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 src/main/java/foxz/command/CmdCompressor.java diff --git a/src/main/java/foxz/command/CmdCompressor.java b/src/main/java/foxz/command/CmdCompressor.java new file mode 100644 index 000000000..630d332d5 --- /dev/null +++ b/src/main/java/foxz/command/CmdCompressor.java @@ -0,0 +1,185 @@ +package foxz.command; + +import foxz.commandhelper.ChMcLogger; +import foxz.commandhelper.annotations.Command; +import foxz.commandhelper.annotations.SubCommand; +import foxz.commandhelper.permissions.OpOnly; +import me.luizotavio.compressor.ZstdCompressor; +import net.minecraft.nbt.NBTTagCompound; +import noppes.npcs.controllers.PlayerDataController; +import noppes.npcs.util.NBTJsonUtil; + +import java.io.File; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static me.luizotavio.compressor.executor.IOExecutor.IO_EXECUTOR; + +@Command( + name = "compressor", + desc = "Convert a file to a compressed file and vice versa", + usage = "update [file|all] [json|zstd]" +) +public class CmdCompressor extends ChMcLogger { + + public CmdCompressor(Object sender) { + super(sender); + } + + @SubCommand( + name = "update", + desc = "Convert all files in the current directory to compressed files", + permissions = {OpOnly.class} + ) + public boolean onConvert(String[] args) throws ExecutionException, InterruptedException, TimeoutException { + if (args.length == 1) { + sendmessage("Please specify the type of compression to use"); + return true; + } + + String type; + + if (args[1].equalsIgnoreCase("json")) { + type = "json"; + } else if (args[1].equalsIgnoreCase("zstd")) { + type = "zstd"; + } else { + sendmessage("Please specify the type of compression to use"); + return true; + } + + File directory = PlayerDataController.instance.getSaveDir(); + + if (args[0].equalsIgnoreCase("all")) { + for (File file : directory.listFiles()) { + if (file.isDirectory()) { + continue; + } + + if (!file.getName().endsWith(".json") && type.equalsIgnoreCase("json")) { + submit(() -> { + try { + NBTTagCompound compound = NBTJsonUtil.LoadFile(file); + + File compressed = new File(file.getParentFile(), file.getName().replace(".json", ".zstd")); + + byte[] bytes = ZstdCompressor.writeCompound(compound); + + Files.write( + compressed.toPath(), + bytes, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE + ); + } catch (Exception exception) { + exception.printStackTrace(); + } + }); + } else if (!file.getName().endsWith(".zstd") && type.equalsIgnoreCase("zstd")) { + submit(() -> { + try { + NBTTagCompound compound = ZstdCompressor.readCompressCompound( + new RandomAccessFile(file, "r") + ); + + if (compound == null) { + return; + } + + File compressed = new File(file.getParentFile(), file.getName().replace(".zstd", ".json")); + + NBTJsonUtil.SaveFile(compressed, compound); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + } + + sendmessage("Done! Almost file was converted."); + } else { + File file = new File(directory, args[0] + ".json"); + + if (!file.exists()) { + file = new File(directory, args[0] + ".zstd"); + } + + if (!file.exists()) { + sendmessage("File not found"); + return true; + } + + File finalFile = file; + + if (file.getName().endsWith(".json")) { + if (type.equalsIgnoreCase("json")) { + sendmessage("File is already in json format"); + return true; + } + + submit(() -> { + try { + NBTTagCompound compound = NBTJsonUtil.LoadFile(finalFile); + + File compressed = new File(finalFile.getParentFile(), finalFile.getName().replace(".json", ".zstd")); + + byte[] bytes = ZstdCompressor.writeCompound(compound); + + Files.write( + compressed.toPath(), + bytes, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE + ); + + sendmessage("File compressed to zstd!"); + } catch (Exception exception) { + exception.printStackTrace(); + } + }); + } + + if (file.getName().endsWith(".zstd")) { + if (type.equalsIgnoreCase("zstd")) { + sendmessage("File is already in zstd format"); + return true; + } + + submit(() -> { + try { + NBTTagCompound compound = ZstdCompressor.readCompressCompound( + new RandomAccessFile(finalFile, "r") + ); + + if (compound == null) { + return; + } + + File compressed = new File(finalFile.getParentFile(), finalFile.getName().replace(".zstd", ".json")); + + NBTJsonUtil.SaveFile(compressed, compound); + + sendmessage("File converted to json!"); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + } + + return true; + } + + private void submit(Runnable runnable) throws ExecutionException, InterruptedException, TimeoutException { + CompletableFuture.runAsync(runnable, IO_EXECUTOR).get(5, TimeUnit.MINUTES); + } + + +} diff --git a/src/main/java/foxz/command/CmdNoppes.java b/src/main/java/foxz/command/CmdNoppes.java index 9e95fe665..7ba24b61e 100644 --- a/src/main/java/foxz/command/CmdNoppes.java +++ b/src/main/java/foxz/command/CmdNoppes.java @@ -37,7 +37,7 @@ import foxz.commandhelper.permissions.PlayerOnly; @Command( name = "noppes", desc = "noppes root command", - sub = {CmdClone.class, CmdScript.class, CmdQuest.class, CmdDialog.class, CmdConfig.class} + sub = {CmdClone.class, CmdScript.class, CmdQuest.class, CmdDialog.class, CmdConfig.class, CmdCompressor.class} ) public class CmdNoppes extends ChMcLogger { From 451a60f2973d5c889cad280f461d4898b2a09c61 Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Sun, 27 Feb 2022 07:00:42 -0300 Subject: [PATCH 06/14] test: make `CompressorTest` to retrieve the measure time of compression to see how much each one will take --- .../luizotavio/compressor/CompressorTest.java | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/test/java/me/luizotavio/compressor/CompressorTest.java diff --git a/src/test/java/me/luizotavio/compressor/CompressorTest.java b/src/test/java/me/luizotavio/compressor/CompressorTest.java new file mode 100644 index 000000000..1dfc0977e --- /dev/null +++ b/src/test/java/me/luizotavio/compressor/CompressorTest.java @@ -0,0 +1,77 @@ +package me.luizotavio.compressor; + +import net.minecraft.nbt.NBTTagCompound; +import noppes.npcs.util.JsonException; +import noppes.npcs.util.NBTJsonUtil; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.util.logging.Logger; + +public class CompressorTest { + + public static final int MAX_ENTRIES = 1000; + + public static void main(String[] args) throws JsonException, IOException { + Logger logger = Logger.getLogger("CompressorTest"); + + logger.info("Starting CompressorTest with " + MAX_ENTRIES + " entries"); + + NBTTagCompound compound = new NBTTagCompound(); + File file = new File("test.json"); + + fill(compound); + + long start = System.currentTimeMillis(); + + logger.info("Writing the data into JSON..."); + + NBTJsonUtil.SaveFile(file, compound); + + long end = System.currentTimeMillis(); + + logger.info("Done! Took " + (end - start) + "ms"); + logger.info("Reading the data from JSON..."); + + start = System.currentTimeMillis(); + + NBTJsonUtil.LoadFile(new File("test.json")); + + end = System.currentTimeMillis(); + + logger.info("Done! Took " + (end - start) + "ms"); + logger.info("Now, we will compress the data into zstd..."); + + start = System.currentTimeMillis(); + + byte[] bytes = ZstdCompressor.writeCompound(compound); + + end = System.currentTimeMillis(); + + logger.info("Done! Took " + (end - start) + "ms"); + logger.info("Now, we will decompress the data from zstd..."); + + start = System.currentTimeMillis(); + + ZstdCompressor.readCompressed( + new DataInputStream( + new ByteArrayInputStream(bytes) + ) + ); + + end = System.currentTimeMillis(); + + logger.info("Done! Took " + (end - start) + "ms"); + + file.deleteOnExit(); + } + + private static void fill(NBTTagCompound compound) { + for (int i = 0; i < MAX_ENTRIES; i++) { + compound.setInteger("" + i, i); + } + } + +} From 54fa6d106f2310e29ed3003cf40b5eb265ffc8b6 Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Sun, 27 Feb 2022 08:10:53 -0300 Subject: [PATCH 07/14] chore: add shade configuration --- build.gradle | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 7cd22a284..508e5cc5d 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,11 @@ minecraft { sourceCompatibility = 1.8 targetCompatibility = 1.8 +configurations { + shade + compile.extendsFrom shade +} + dependencies { // you may put jars on which you depend on in ./libs // or you may define them like so.. @@ -42,7 +47,7 @@ dependencies { // for more info... // http://www.gradle.org/docs/current/userguide/artifact_dependencies_tutorial.html // http://www.gradle.org/docs/current/userguide/dependency_management.html - compile 'com.github.luben:zstd-jni:1.5.2-2' + shade 'com.github.luben:zstd-jni:1.5.2-2' } processResources @@ -69,4 +74,10 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } -sourceSets {main { output.resourcesDir = output.classesDir }} +jar { + configurations.shade.each { dep -> + from(project.zipTree(dep)) { + exclude 'META-INF', 'META-INF/**' + } + } +} \ No newline at end of file From 7b0a6b7497dbbb8a78e07d253204a4e9e5fa805c Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Sun, 27 Feb 2022 08:11:48 -0300 Subject: [PATCH 08/14] feat: add some utilities messages on each case of compressor command --- src/main/java/foxz/command/CmdCompressor.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/foxz/command/CmdCompressor.java b/src/main/java/foxz/command/CmdCompressor.java index 630d332d5..d022d9b84 100644 --- a/src/main/java/foxz/command/CmdCompressor.java +++ b/src/main/java/foxz/command/CmdCompressor.java @@ -37,6 +37,10 @@ public class CmdCompressor extends ChMcLogger { permissions = {OpOnly.class} ) public boolean onConvert(String[] args) throws ExecutionException, InterruptedException, TimeoutException { + if (args.length == 0) { + sendmessage("Usage: /noppes compressor update [|all] [json|zstd]"); + return true; + } if (args.length == 1) { sendmessage("Please specify the type of compression to use"); return true; @@ -56,12 +60,14 @@ public class CmdCompressor extends ChMcLogger { File directory = PlayerDataController.instance.getSaveDir(); if (args[0].equalsIgnoreCase("all")) { + sendmessage("Converting all files in the current directory to compressed files"); + for (File file : directory.listFiles()) { if (file.isDirectory()) { continue; } - if (!file.getName().endsWith(".json") && type.equalsIgnoreCase("json")) { + if (file.getName().endsWith(".json") && type.equalsIgnoreCase("zstd")) { submit(() -> { try { NBTTagCompound compound = NBTJsonUtil.LoadFile(file); @@ -74,24 +80,21 @@ public class CmdCompressor extends ChMcLogger { compressed.toPath(), bytes, StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE ); } catch (Exception exception) { exception.printStackTrace(); } }); - } else if (!file.getName().endsWith(".zstd") && type.equalsIgnoreCase("zstd")) { + } + + if (file.getName().endsWith(".zstd") && type.equalsIgnoreCase("json")) { submit(() -> { try { NBTTagCompound compound = ZstdCompressor.readCompressCompound( new RandomAccessFile(file, "r") ); - if (compound == null) { - return; - } - File compressed = new File(file.getParentFile(), file.getName().replace(".zstd", ".json")); NBTJsonUtil.SaveFile(compressed, compound); @@ -135,7 +138,6 @@ public class CmdCompressor extends ChMcLogger { compressed.toPath(), bytes, StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE ); From 90c657412568d6b94a13143d6d721bf30dbc2a10 Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Sun, 27 Feb 2022 08:12:24 -0300 Subject: [PATCH 09/14] fix: ensure `playerdata` type and remove useless checking about transforming --- .../controllers/PlayerDataController.java | 49 ++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/src/main/java/noppes/npcs/controllers/PlayerDataController.java b/src/main/java/noppes/npcs/controllers/PlayerDataController.java index c50a9ff44..27bdef592 100644 --- a/src/main/java/noppes/npcs/controllers/PlayerDataController.java +++ b/src/main/java/noppes/npcs/controllers/PlayerDataController.java @@ -73,9 +73,18 @@ public class PlayerDataController { public NBTTagCompound loadPlayerData(String player){ File saveDir = getSaveDir(); String filename = player; - if(filename.isEmpty()) - filename = "noplayername"; - filename += ".json"; + if(filename.isEmpty()) { + return new NBTTagCompound(); + } + + if(CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) { + filename += ".zstd"; + } else if(CustomNpcs.CompressorType.equalsIgnoreCase("json")) { + filename += ".json"; + } else { + filename += ".dat"; + } + try { return ensureCompressor( new File(saveDir, filename) @@ -91,8 +100,8 @@ public class PlayerDataController { NBTTagCompound compound = data.getNBT(); String filename; - if (CustomNpcs.CompressorType.equalsIgnoreCase("lz4")) { - filename = data.uuid + ".lz4"; + if (CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) { + filename = data.uuid + ".zstd"; } else if (CustomNpcs.CompressorType.equalsIgnoreCase("json")) { filename = data.uuid + ".json"; } else { @@ -101,15 +110,14 @@ public class PlayerDataController { try { File saveDir = getSaveDir(); - File file = new File(saveDir, filename+"_new"); - File file1 = new File(saveDir, filename); + + File file = new File(saveDir, filename); + + if (file.exists()) { + file.delete(); + } writeCompressor(file, compound); - - if(file1.exists()){ - file1.delete(); - } - file.renameTo(file1); } catch (Exception e) { LogWriter.except(e); } @@ -215,7 +223,6 @@ public class PlayerDataController { file.toPath(), compressed, StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE ); } catch (Exception e) { @@ -231,24 +238,10 @@ public class PlayerDataController { } if (CustomNpcs.CompressorType.equalsIgnoreCase("json")) { - if (file.getName().endsWith(".zstd")) { - byte[] readCompressed = ZstdCompressor.readCompressed( - new RandomAccessFile(file, "r") - ); - - try (DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(readCompressed))) { - return CompressedStreamTools.func_152456_a(dataInputStream, NBTSizeTracker.field_152451_a); - } - } - return NBTJsonUtil.LoadFile(file); } - if (CustomNpcs.CompressorType.equalsIgnoreCase("lz4")) { - if (file.getName().endsWith(".json")) { - return NBTJsonUtil.LoadFile(file); - } - + if (CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) { byte[] readCompressed = ZstdCompressor.readCompressed( new RandomAccessFile(file, "r") ); From e23620d724c46dbe1d97170e92f80847b3eac98a Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Thu, 3 Mar 2022 17:43:18 -0300 Subject: [PATCH 10/14] refactor: use `JSON`as default codec for `PlayerData` instead of `.dat` files --- .../controllers/PlayerDataController.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/main/java/noppes/npcs/controllers/PlayerDataController.java b/src/main/java/noppes/npcs/controllers/PlayerDataController.java index 27bdef592..345ba4130 100644 --- a/src/main/java/noppes/npcs/controllers/PlayerDataController.java +++ b/src/main/java/noppes/npcs/controllers/PlayerDataController.java @@ -1,11 +1,5 @@ package noppes.npcs.controllers; -import java.io.*; -import java.nio.file.Files; -import java.nio.file.StandardOpenOption; -import java.util.HashMap; -import java.util.Map; - import me.luizotavio.compressor.ZstdCompressor; import net.minecraft.entity.player.EntityPlayer; import net.minecraft.nbt.CompressedStreamTools; @@ -17,6 +11,12 @@ import noppes.npcs.LogWriter; import noppes.npcs.util.JsonException; import noppes.npcs.util.NBTJsonUtil; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Map; + import static me.luizotavio.compressor.executor.IOExecutor.IO_EXECUTOR; public class PlayerDataController { @@ -79,10 +79,8 @@ public class PlayerDataController { if(CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) { filename += ".zstd"; - } else if(CustomNpcs.CompressorType.equalsIgnoreCase("json")) { - filename += ".json"; } else { - filename += ".dat"; + filename += ".json"; } try { @@ -102,10 +100,8 @@ public class PlayerDataController { if (CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) { filename = data.uuid + ".zstd"; - } else if (CustomNpcs.CompressorType.equalsIgnoreCase("json")) { + } else { filename = data.uuid + ".json"; - } else { - filename = data.uuid + ".dat"; } try { From 5578f2055e695c5a9fef79582210b9a69f409855 Mon Sep 17 00:00:00 2001 From: luiz-otavio Date: Thu, 3 Mar 2022 17:50:28 -0300 Subject: [PATCH 11/14] fix(gradle): assign `resourcesDir` to `classesDir` from `sourcesSets` --- build.gradle | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 508e5cc5d..4e2f429c2 100644 --- a/build.gradle +++ b/build.gradle @@ -80,4 +80,6 @@ jar { exclude 'META-INF', 'META-INF/**' } } -} \ No newline at end of file +} + +sourceSets {main { output.resourcesDir = output.classesDir }} From 1a7267349f6301d39e1e0bfd952730b7815c2443 Mon Sep 17 00:00:00 2001 From: Kamron Vaswani Date: Thu, 3 Mar 2022 15:58:21 -0500 Subject: [PATCH 12/14] Update Config Description with Warning --- src/main/java/noppes/npcs/CustomNpcs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/noppes/npcs/CustomNpcs.java b/src/main/java/noppes/npcs/CustomNpcs.java index 75c76c0cf..250e6fb14 100644 --- a/src/main/java/noppes/npcs/CustomNpcs.java +++ b/src/main/java/noppes/npcs/CustomNpcs.java @@ -42,7 +42,7 @@ public class CustomNpcs { private static int NewEntityStartId = 0; - @ConfigProp(info = "The type of compressor which is used to compress and decompress player data. If you are experiencing issues with the server lagging/crashing, try changing this to 'zstd'") + @ConfigProp(info = "*BACKUP BEFORE CHANGING* File Type used with PlayerData. Converting files may reduce lag. To convert existing files: /noppes compressor update [|all] [json|zstd]") public static String CompressorType = "json"; @ConfigProp(info = "The size of IO executors used for compression and decompression. This is the number of threads used for compression and decompression. If you are experiencing issues with the server lagging/crashing, try changing this to a higher number.") From 3339488a6fb797397aaa198b8d213a11c41cb95d Mon Sep 17 00:00:00 2001 From: Kamron Vaswani Date: Thu, 3 Mar 2022 15:59:17 -0500 Subject: [PATCH 13/14] Version Correction --- src/main/java/noppes/npcs/CustomNpcs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/noppes/npcs/CustomNpcs.java b/src/main/java/noppes/npcs/CustomNpcs.java index 250e6fb14..087c9b525 100644 --- a/src/main/java/noppes/npcs/CustomNpcs.java +++ b/src/main/java/noppes/npcs/CustomNpcs.java @@ -34,7 +34,7 @@ import java.io.File; import java.util.Set; import java.util.concurrent.Executors; -@Mod(modid = "customnpcs", name = "CustomNpcs", version = "1.5.1-beta") +@Mod(modid = "customnpcs", name = "CustomNpcs", version = "1.5.1") public class CustomNpcs { @ConfigProp(info = "Disable Chat Bubbles") From 89b7019e29969f04ead1d4192d7e739dbf26d0b8 Mon Sep 17 00:00:00 2001 From: Kamron Vaswani Date: Thu, 3 Mar 2022 16:02:02 -0500 Subject: [PATCH 14/14] Delete CompressorTest.java Not needed in final version --- .../luizotavio/compressor/CompressorTest.java | 77 ------------------- 1 file changed, 77 deletions(-) delete mode 100644 src/test/java/me/luizotavio/compressor/CompressorTest.java diff --git a/src/test/java/me/luizotavio/compressor/CompressorTest.java b/src/test/java/me/luizotavio/compressor/CompressorTest.java deleted file mode 100644 index 1dfc0977e..000000000 --- a/src/test/java/me/luizotavio/compressor/CompressorTest.java +++ /dev/null @@ -1,77 +0,0 @@ -package me.luizotavio.compressor; - -import net.minecraft.nbt.NBTTagCompound; -import noppes.npcs.util.JsonException; -import noppes.npcs.util.NBTJsonUtil; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.File; -import java.io.IOException; -import java.util.logging.Logger; - -public class CompressorTest { - - public static final int MAX_ENTRIES = 1000; - - public static void main(String[] args) throws JsonException, IOException { - Logger logger = Logger.getLogger("CompressorTest"); - - logger.info("Starting CompressorTest with " + MAX_ENTRIES + " entries"); - - NBTTagCompound compound = new NBTTagCompound(); - File file = new File("test.json"); - - fill(compound); - - long start = System.currentTimeMillis(); - - logger.info("Writing the data into JSON..."); - - NBTJsonUtil.SaveFile(file, compound); - - long end = System.currentTimeMillis(); - - logger.info("Done! Took " + (end - start) + "ms"); - logger.info("Reading the data from JSON..."); - - start = System.currentTimeMillis(); - - NBTJsonUtil.LoadFile(new File("test.json")); - - end = System.currentTimeMillis(); - - logger.info("Done! Took " + (end - start) + "ms"); - logger.info("Now, we will compress the data into zstd..."); - - start = System.currentTimeMillis(); - - byte[] bytes = ZstdCompressor.writeCompound(compound); - - end = System.currentTimeMillis(); - - logger.info("Done! Took " + (end - start) + "ms"); - logger.info("Now, we will decompress the data from zstd..."); - - start = System.currentTimeMillis(); - - ZstdCompressor.readCompressed( - new DataInputStream( - new ByteArrayInputStream(bytes) - ) - ); - - end = System.currentTimeMillis(); - - logger.info("Done! Took " + (end - start) + "ms"); - - file.deleteOnExit(); - } - - private static void fill(NBTTagCompound compound) { - for (int i = 0; i < MAX_ENTRIES; i++) { - compound.setInteger("" + i, i); - } - } - -}