diff --git a/build.gradle b/build.gradle index 379148646..0fa989b42 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,11 @@ minecraft { sourceCompatibility = 1.8 targetCompatibility = 1.8 +configurations { + shade + compile.extendsFrom shade +} + dependencies { // you may put jars on which you depend on in ./libs // or you may define them like so.. @@ -42,7 +47,7 @@ dependencies { // for more info... // http://www.gradle.org/docs/current/userguide/artifact_dependencies_tutorial.html // http://www.gradle.org/docs/current/userguide/dependency_management.html - + shade 'com.github.luben:zstd-jni:1.5.2-2' } processResources @@ -65,4 +70,16 @@ processResources } } +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +jar { + configurations.shade.each { dep -> + from(project.zipTree(dep)) { + exclude 'META-INF', 'META-INF/**' + } + } +} + sourceSets {main { output.resourcesDir = output.classesDir }} diff --git a/src/main/java/foxz/command/CmdCompressor.java b/src/main/java/foxz/command/CmdCompressor.java new file mode 100644 index 000000000..d022d9b84 --- /dev/null +++ b/src/main/java/foxz/command/CmdCompressor.java @@ -0,0 +1,187 @@ +package foxz.command; + +import foxz.commandhelper.ChMcLogger; +import foxz.commandhelper.annotations.Command; +import foxz.commandhelper.annotations.SubCommand; +import foxz.commandhelper.permissions.OpOnly; +import me.luizotavio.compressor.ZstdCompressor; +import net.minecraft.nbt.NBTTagCompound; +import noppes.npcs.controllers.PlayerDataController; +import noppes.npcs.util.NBTJsonUtil; + +import java.io.File; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static me.luizotavio.compressor.executor.IOExecutor.IO_EXECUTOR; + +@Command( + name = "compressor", + desc = "Convert a file to a compressed file and vice versa", + usage = "update [file|all] [json|zstd]" +) +public class CmdCompressor extends ChMcLogger { + + public CmdCompressor(Object sender) { + super(sender); + } + + @SubCommand( + name = "update", + desc = "Convert all files in the current directory to compressed files", + permissions = {OpOnly.class} + ) + public boolean onConvert(String[] args) throws ExecutionException, InterruptedException, TimeoutException { + if (args.length == 0) { + sendmessage("Usage: /noppes compressor update [|all] [json|zstd]"); + return true; + } + if (args.length == 1) { + sendmessage("Please specify the type of compression to use"); + return true; + } + + String type; + + if (args[1].equalsIgnoreCase("json")) { + type = "json"; + } else if (args[1].equalsIgnoreCase("zstd")) { + type = "zstd"; + } else { + sendmessage("Please specify the type of compression to use"); + return true; + } + + File directory = PlayerDataController.instance.getSaveDir(); + + if (args[0].equalsIgnoreCase("all")) { + sendmessage("Converting all files in the current directory to compressed files"); + + for (File file : directory.listFiles()) { + if (file.isDirectory()) { + continue; + } + + if (file.getName().endsWith(".json") && type.equalsIgnoreCase("zstd")) { + submit(() -> { + try { + NBTTagCompound compound = NBTJsonUtil.LoadFile(file); + + File compressed = new File(file.getParentFile(), file.getName().replace(".json", ".zstd")); + + byte[] bytes = ZstdCompressor.writeCompound(compound); + + Files.write( + compressed.toPath(), + bytes, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE + ); + } catch (Exception exception) { + exception.printStackTrace(); + } + }); + } + + if (file.getName().endsWith(".zstd") && type.equalsIgnoreCase("json")) { + submit(() -> { + try { + NBTTagCompound compound = ZstdCompressor.readCompressCompound( + new RandomAccessFile(file, "r") + ); + + File compressed = new File(file.getParentFile(), file.getName().replace(".zstd", ".json")); + + NBTJsonUtil.SaveFile(compressed, compound); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + } + + sendmessage("Done! Almost file was converted."); + } else { + File file = new File(directory, args[0] + ".json"); + + if (!file.exists()) { + file = new File(directory, args[0] + ".zstd"); + } + + if (!file.exists()) { + sendmessage("File not found"); + return true; + } + + File finalFile = file; + + if (file.getName().endsWith(".json")) { + if (type.equalsIgnoreCase("json")) { + sendmessage("File is already in json format"); + return true; + } + + submit(() -> { + try { + NBTTagCompound compound = NBTJsonUtil.LoadFile(finalFile); + + File compressed = new File(finalFile.getParentFile(), finalFile.getName().replace(".json", ".zstd")); + + byte[] bytes = ZstdCompressor.writeCompound(compound); + + Files.write( + compressed.toPath(), + bytes, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE + ); + + sendmessage("File compressed to zstd!"); + } catch (Exception exception) { + exception.printStackTrace(); + } + }); + } + + if (file.getName().endsWith(".zstd")) { + if (type.equalsIgnoreCase("zstd")) { + sendmessage("File is already in zstd format"); + return true; + } + + submit(() -> { + try { + NBTTagCompound compound = ZstdCompressor.readCompressCompound( + new RandomAccessFile(finalFile, "r") + ); + + if (compound == null) { + return; + } + + File compressed = new File(finalFile.getParentFile(), finalFile.getName().replace(".zstd", ".json")); + + NBTJsonUtil.SaveFile(compressed, compound); + + sendmessage("File converted to json!"); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + } + + return true; + } + + private void submit(Runnable runnable) throws ExecutionException, InterruptedException, TimeoutException { + CompletableFuture.runAsync(runnable, IO_EXECUTOR).get(5, TimeUnit.MINUTES); + } + + +} diff --git a/src/main/java/foxz/command/CmdNoppes.java b/src/main/java/foxz/command/CmdNoppes.java index 9e95fe665..7ba24b61e 100644 --- a/src/main/java/foxz/command/CmdNoppes.java +++ b/src/main/java/foxz/command/CmdNoppes.java @@ -37,7 +37,7 @@ import foxz.commandhelper.permissions.PlayerOnly; @Command( name = "noppes", desc = "noppes root command", - sub = {CmdClone.class, CmdScript.class, CmdQuest.class, CmdDialog.class, CmdConfig.class} + sub = {CmdClone.class, CmdScript.class, CmdQuest.class, CmdDialog.class, CmdConfig.class, CmdCompressor.class} ) public class CmdNoppes extends ChMcLogger { diff --git a/src/main/java/me/luizotavio/compressor/ZstdCompressor.java b/src/main/java/me/luizotavio/compressor/ZstdCompressor.java new file mode 100644 index 000000000..694229539 --- /dev/null +++ b/src/main/java/me/luizotavio/compressor/ZstdCompressor.java @@ -0,0 +1,61 @@ +package me.luizotavio.compressor; + +import com.github.luben.zstd.Zstd; +import net.minecraft.nbt.CompressedStreamTools; +import net.minecraft.nbt.NBTSizeTracker; +import net.minecraft.nbt.NBTTagCompound; + +import java.io.*; + +public class ZstdCompressor { + + public static byte[] readCompressed(DataInput inputStream) throws IOException { + int compressedLength = inputStream.readInt(); + int uncompressedLength = inputStream.readInt(); + + if (compressedLength < 0 || uncompressedLength < 0) { + throw new IllegalArgumentException("Invalid length"); + } + + byte[] compressed = new byte[compressedLength]; + + inputStream.readFully(compressed); + + return Zstd.decompress(compressed, uncompressedLength); + } + + public static NBTTagCompound readCompressCompound(DataInput inputStream) throws IOException { + NBTTagCompound compound; + + byte[] compressed = readCompressed(inputStream); + + try (DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(compressed))) { + compound = CompressedStreamTools.func_152456_a(dataInputStream, NBTSizeTracker.field_152451_a); + } + + return compound; + } + + public static byte[] writeCompound(NBTTagCompound compound) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(), + copy = new ByteArrayOutputStream(); + + try (DataOutputStream dataOutputStream = new DataOutputStream(outputStream)) { + CompressedStreamTools.write(compound, dataOutputStream); + } + + byte[] uncompressed = outputStream.toByteArray(); + + byte[] compress = Zstd.compress(uncompressed); + + try (DataOutputStream dataOutputStream = new DataOutputStream(copy)) { + dataOutputStream.writeInt(compress.length); + dataOutputStream.writeInt(uncompressed.length); + + dataOutputStream.write(compress); + + return copy.toByteArray(); + } + } + +} diff --git a/src/main/java/me/luizotavio/compressor/executor/IOExecutor.java b/src/main/java/me/luizotavio/compressor/executor/IOExecutor.java new file mode 100644 index 000000000..f0f5da14e --- /dev/null +++ b/src/main/java/me/luizotavio/compressor/executor/IOExecutor.java @@ -0,0 +1,16 @@ +package me.luizotavio.compressor.executor; + +import java.util.concurrent.ExecutorService; + +public class IOExecutor { + + /** + * Please, do not change this value unless you know what you are doing. + * This value is used to define the number of threads that will be used to + * compress the files. + * + * If you want to change this value, please, do it with care. + */ + public static ExecutorService IO_EXECUTOR; + +} diff --git a/src/main/java/noppes/npcs/CustomNpcs.java b/src/main/java/noppes/npcs/CustomNpcs.java index 82ae020d9..a1c50e79b 100644 --- a/src/main/java/noppes/npcs/CustomNpcs.java +++ b/src/main/java/noppes/npcs/CustomNpcs.java @@ -1,8 +1,6 @@ package noppes.npcs; -import java.io.File; -import java.util.Set; - +import com.google.common.util.concurrent.ThreadFactoryBuilder; import cpw.mods.fml.common.FMLCommonHandler; import cpw.mods.fml.common.Mod; import cpw.mods.fml.common.Mod.EventHandler; @@ -12,6 +10,7 @@ import cpw.mods.fml.common.network.FMLEventChannel; import cpw.mods.fml.common.network.NetworkRegistry; import cpw.mods.fml.common.registry.EntityRegistry; import foxz.command.CommandNoppes; +import me.luizotavio.compressor.executor.IOExecutor; import net.minecraft.block.Block; import net.minecraft.block.BlockIce; import net.minecraft.block.BlockLeavesBase; @@ -25,50 +24,29 @@ import net.minecraftforge.common.MinecraftForge; import nikedemos.markovnames.generators.*; import noppes.npcs.config.ConfigLoader; import noppes.npcs.config.ConfigProp; -import noppes.npcs.controllers.BankController; -import noppes.npcs.controllers.ChunkController; -import noppes.npcs.controllers.DialogController; -import noppes.npcs.controllers.FactionController; -import noppes.npcs.controllers.GlobalDataController; -import noppes.npcs.controllers.LinkedNpcController; -import noppes.npcs.controllers.PixelmonHelper; -import noppes.npcs.controllers.PlayerDataController; -import noppes.npcs.controllers.QuestController; -import noppes.npcs.controllers.RecipeController; -import noppes.npcs.controllers.ScriptController; -import noppes.npcs.controllers.ServerCloneController; -import noppes.npcs.controllers.SpawnController; -import noppes.npcs.controllers.TransportController; +import noppes.npcs.controllers.*; import noppes.npcs.enchants.EnchantInterface; import noppes.npcs.entity.*; -import noppes.npcs.entity.old.EntityNPCDwarfFemale; -import noppes.npcs.entity.old.EntityNPCDwarfMale; -import noppes.npcs.entity.old.EntityNPCElfFemale; -import noppes.npcs.entity.old.EntityNPCElfMale; -import noppes.npcs.entity.old.EntityNPCEnderman; -import noppes.npcs.entity.old.EntityNPCFurryFemale; -import noppes.npcs.entity.old.EntityNPCFurryMale; -import noppes.npcs.entity.old.EntityNPCHumanFemale; -import noppes.npcs.entity.old.EntityNPCHumanMale; -import noppes.npcs.entity.old.EntityNPCOrcFemale; -import noppes.npcs.entity.old.EntityNPCOrcMale; -import noppes.npcs.entity.old.EntityNPCVillager; -import noppes.npcs.entity.old.EntityNpcEnderchibi; -import noppes.npcs.entity.old.EntityNpcMonsterFemale; -import noppes.npcs.entity.old.EntityNpcMonsterMale; -import noppes.npcs.entity.old.EntityNpcNagaFemale; -import noppes.npcs.entity.old.EntityNpcNagaMale; -import noppes.npcs.entity.old.EntityNpcSkeleton; +import noppes.npcs.entity.old.*; import noppes.npcs.scripted.wrapper.WrapperNpcAPI; +import java.io.File; +import java.util.Set; +import java.util.concurrent.Executors; @Mod(modid = "customnpcs", name = "CustomNpcs", version = "1.5.1") public class CustomNpcs { - @ConfigProp(info = "Disable Chat Bubbles") + @ConfigProp(info = "Disable Chat Bubbles") public static boolean EnableChatBubbles = true; - + private static int NewEntityStartId = 0; - + + @ConfigProp(info = "*BACKUP BEFORE CHANGING* File Type used with PlayerData. Converting files may reduce lag. To convert existing files: /noppes compressor update [|all] [json|zstd]") + public static String CompressorType = "json"; + + @ConfigProp(info = "The size of IO executors used for compression and decompression. This is the number of threads used for compression and decompression. If you are experiencing issues with the server lagging/crashing, try changing this to a higher number.") + public static int CompressorThreads = 2; + @ConfigProp(info = "Navigation search range for NPCs. Not recommended to increase if you have a slow pc or on a server. Minimum of 16, maximum of 96.") public static int NpcNavRange = 32; @@ -227,14 +205,19 @@ public class CustomNpcs { registerNpc(EntityNPCGolem.class, "npcGolem"); registerNpc(EntityCustomNpc.class, "CustomNpc"); - + IOExecutor.IO_EXECUTOR = Executors.newFixedThreadPool( + CompressorThreads, + new ThreadFactoryBuilder() + .setNameFormat("CustomNpcs-IO-%d") + .build() + ); registerNewEntity(EntityChairMount.class, "CustomNpcChairMount", 64, 10, false); registerNewEntity(EntityProjectile.class, "throwableitem", 64, 3, true); registerNewEntity(EntityMagicProjectile.class, "magicprojectile", 64, 3, true); new RecipeController(); - + ForgeChunkManager.setForcedChunkLoadingCallback(this, new ChunkController()); new CustomNpcsPermissions(); diff --git a/src/main/java/noppes/npcs/controllers/PlayerDataController.java b/src/main/java/noppes/npcs/controllers/PlayerDataController.java index 4c72212d7..345ba4130 100644 --- a/src/main/java/noppes/npcs/controllers/PlayerDataController.java +++ b/src/main/java/noppes/npcs/controllers/PlayerDataController.java @@ -1,21 +1,24 @@ package noppes.npcs.controllers; -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - +import me.luizotavio.compressor.ZstdCompressor; import net.minecraft.entity.player.EntityPlayer; import net.minecraft.nbt.CompressedStreamTools; +import net.minecraft.nbt.NBTSizeTracker; import net.minecraft.nbt.NBTTagCompound; import net.minecraft.server.MinecraftServer; import noppes.npcs.CustomNpcs; import noppes.npcs.LogWriter; +import noppes.npcs.util.JsonException; import noppes.npcs.util.NBTJsonUtil; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Map; + +import static me.luizotavio.compressor.executor.IOExecutor.IO_EXECUTOR; + public class PlayerDataController { public static PlayerDataController instance; @@ -70,14 +73,20 @@ public class PlayerDataController { public NBTTagCompound loadPlayerData(String player){ File saveDir = getSaveDir(); String filename = player; - if(filename.isEmpty()) - filename = "noplayername"; - filename += ".json"; + if(filename.isEmpty()) { + return new NBTTagCompound(); + } + + if(CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) { + filename += ".zstd"; + } else { + filename += ".json"; + } + try { - File file = new File(saveDir, filename); - if(file.exists()){ - return NBTJsonUtil.LoadFile(file); - } + return ensureCompressor( + new File(saveDir, filename) + ); } catch (Exception e) { LogWriter.error("Error loading: " + filename, e); } @@ -87,16 +96,24 @@ public class PlayerDataController { public void savePlayerData(PlayerData data){ NBTTagCompound compound = data.getNBT(); - String filename = data.uuid + ".json"; + String filename; + + if (CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) { + filename = data.uuid + ".zstd"; + } else { + filename = data.uuid + ".json"; + } + try { File saveDir = getSaveDir(); - File file = new File(saveDir, filename+"_new"); - File file1 = new File(saveDir, filename); - NBTJsonUtil.SaveFile(file, compound); - if(file1.exists()){ - file1.delete(); - } - file.renameTo(file1); + + File file = new File(saveDir, filename); + + if (file.exists()) { + file.delete(); + } + + writeCompressor(file, compound); } catch (Exception e) { LogWriter.except(e); } @@ -164,7 +181,8 @@ public class PlayerDataController { if(file.isDirectory() || !file.getName().endsWith(".json")) continue; try { - NBTTagCompound compound = NBTJsonUtil.LoadFile(file); + NBTTagCompound compound = ensureCompressor(file); + if(compound.hasKey("PlayerName")){ map.put(compound.getString("PlayerName"), compound); } @@ -178,4 +196,57 @@ public class PlayerDataController { public boolean hasMail(EntityPlayer player) { return getPlayerData(player).mailData.hasMail(); } + + private void writeCompressor(File file, NBTTagCompound compound) { + if (compound == null) { + throw new IllegalArgumentException("Compound cannot be null!"); + } + + if (CustomNpcs.CompressorType.equalsIgnoreCase("json")) { + IO_EXECUTOR.submit(() -> { + try { + NBTJsonUtil.SaveFile(file, compound); + } catch (IOException | JsonException e) { + e.printStackTrace(); + } + }); + } else if (CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) { + IO_EXECUTOR.submit(() -> { + try { + byte[] compressed = ZstdCompressor.writeCompound(compound); + + Files.write( + file.toPath(), + compressed, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE + ); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + } + + private NBTTagCompound ensureCompressor(File file) throws JsonException, IOException { + if (!file.exists()) { + return new NBTTagCompound(); + } + + if (CustomNpcs.CompressorType.equalsIgnoreCase("json")) { + return NBTJsonUtil.LoadFile(file); + } + + if (CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) { + byte[] readCompressed = ZstdCompressor.readCompressed( + new RandomAccessFile(file, "r") + ); + + try (DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(readCompressed))) { + return CompressedStreamTools.func_152456_a(dataInputStream, NBTSizeTracker.field_152451_a); + } + } + + throw new IOException("Unknown compressor type: " + CustomNpcs.CompressorType); + } }