Merge pull request #36 from luiz-otavio/feat/zstd-compressor

feat: add config property to use `Zstd` compression instead of `JSON`
This commit is contained in:
Kamron Vaswani authored and GitHub committed 2022-03-14 00:08:54 -04:00
commit a10d79da9b
7 files changed
+401 -66

No files matched your search

+18 -1
View File
@@ -29,6 +29,11 @@ minecraft {
sourceCompatibility = 1.8
targetCompatibility = 1.8
configurations {
shade
compile.extendsFrom shade
}
dependencies {
// you may put jars on which you depend on in ./libs
// or you may define them like so..
@@ -42,7 +47,7 @@ dependencies {
// for more info...
// http://www.gradle.org/docs/current/userguide/artifact_dependencies_tutorial.html
// http://www.gradle.org/docs/current/userguide/dependency_management.html
shade 'com.github.luben:zstd-jni:1.5.2-2'
}
processResources
@@ -65,4 +70,16 @@ processResources
}
}
tasks.withType(JavaCompile) {
options.encoding = 'UTF-8'
}
jar {
configurations.shade.each { dep ->
from(project.zipTree(dep)) {
exclude 'META-INF', 'META-INF/**'
}
}
}
sourceSets {main { output.resourcesDir = output.classesDir }}
@@ -0,0 +1,187 @@
package foxz.command;
import foxz.commandhelper.ChMcLogger;
import foxz.commandhelper.annotations.Command;
import foxz.commandhelper.annotations.SubCommand;
import foxz.commandhelper.permissions.OpOnly;
import me.luizotavio.compressor.ZstdCompressor;
import net.minecraft.nbt.NBTTagCompound;
import noppes.npcs.controllers.PlayerDataController;
import noppes.npcs.util.NBTJsonUtil;
import java.io.File;
import java.io.RandomAccessFile;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static me.luizotavio.compressor.executor.IOExecutor.IO_EXECUTOR;
@Command(
name = "compressor",
desc = "Convert a file to a compressed file and vice versa",
usage = "update [file|all] [json|zstd]"
)
public class CmdCompressor extends ChMcLogger {
public CmdCompressor(Object sender) {
super(sender);
}
@SubCommand(
name = "update",
desc = "Convert all files in the current directory to compressed files",
permissions = {OpOnly.class}
)
public boolean onConvert(String[] args) throws ExecutionException, InterruptedException, TimeoutException {
if (args.length == 0) {
sendmessage("Usage: /noppes compressor update [<filename>|all] [json|zstd]");
return true;
}
if (args.length == 1) {
sendmessage("Please specify the type of compression to use");
return true;
}
String type;
if (args[1].equalsIgnoreCase("json")) {
type = "json";
} else if (args[1].equalsIgnoreCase("zstd")) {
type = "zstd";
} else {
sendmessage("Please specify the type of compression to use");
return true;
}
File directory = PlayerDataController.instance.getSaveDir();
if (args[0].equalsIgnoreCase("all")) {
sendmessage("Converting all files in the current directory to compressed files");
for (File file : directory.listFiles()) {
if (file.isDirectory()) {
continue;
}
if (file.getName().endsWith(".json") && type.equalsIgnoreCase("zstd")) {
submit(() -> {
try {
NBTTagCompound compound = NBTJsonUtil.LoadFile(file);
File compressed = new File(file.getParentFile(), file.getName().replace(".json", ".zstd"));
byte[] bytes = ZstdCompressor.writeCompound(compound);
Files.write(
compressed.toPath(),
bytes,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE
);
} catch (Exception exception) {
exception.printStackTrace();
}
});
}
if (file.getName().endsWith(".zstd") && type.equalsIgnoreCase("json")) {
submit(() -> {
try {
NBTTagCompound compound = ZstdCompressor.readCompressCompound(
new RandomAccessFile(file, "r")
);
File compressed = new File(file.getParentFile(), file.getName().replace(".zstd", ".json"));
NBTJsonUtil.SaveFile(compressed, compound);
} catch (Exception e) {
e.printStackTrace();
}
});
}
}
sendmessage("Done! Almost file was converted.");
} else {
File file = new File(directory, args[0] + ".json");
if (!file.exists()) {
file = new File(directory, args[0] + ".zstd");
}
if (!file.exists()) {
sendmessage("File not found");
return true;
}
File finalFile = file;
if (file.getName().endsWith(".json")) {
if (type.equalsIgnoreCase("json")) {
sendmessage("File is already in json format");
return true;
}
submit(() -> {
try {
NBTTagCompound compound = NBTJsonUtil.LoadFile(finalFile);
File compressed = new File(finalFile.getParentFile(), finalFile.getName().replace(".json", ".zstd"));
byte[] bytes = ZstdCompressor.writeCompound(compound);
Files.write(
compressed.toPath(),
bytes,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE
);
sendmessage("File compressed to zstd!");
} catch (Exception exception) {
exception.printStackTrace();
}
});
}
if (file.getName().endsWith(".zstd")) {
if (type.equalsIgnoreCase("zstd")) {
sendmessage("File is already in zstd format");
return true;
}
submit(() -> {
try {
NBTTagCompound compound = ZstdCompressor.readCompressCompound(
new RandomAccessFile(finalFile, "r")
);
if (compound == null) {
return;
}
File compressed = new File(finalFile.getParentFile(), finalFile.getName().replace(".zstd", ".json"));
NBTJsonUtil.SaveFile(compressed, compound);
sendmessage("File converted to json!");
} catch (Exception e) {
e.printStackTrace();
}
});
}
}
return true;
}
private void submit(Runnable runnable) throws ExecutionException, InterruptedException, TimeoutException {
CompletableFuture.runAsync(runnable, IO_EXECUTOR).get(5, TimeUnit.MINUTES);
}
}
+1 -1
View File
@@ -37,7 +37,7 @@ import foxz.commandhelper.permissions.PlayerOnly;
@Command(
name = "noppes",
desc = "noppes root command",
sub = {CmdClone.class, CmdScript.class, CmdQuest.class, CmdDialog.class, CmdConfig.class}
sub = {CmdClone.class, CmdScript.class, CmdQuest.class, CmdDialog.class, CmdConfig.class, CmdCompressor.class}
)
public class CmdNoppes extends ChMcLogger {
@@ -0,0 +1,61 @@
package me.luizotavio.compressor;
import com.github.luben.zstd.Zstd;
import net.minecraft.nbt.CompressedStreamTools;
import net.minecraft.nbt.NBTSizeTracker;
import net.minecraft.nbt.NBTTagCompound;
import java.io.*;
public class ZstdCompressor {
public static byte[] readCompressed(DataInput inputStream) throws IOException {
int compressedLength = inputStream.readInt();
int uncompressedLength = inputStream.readInt();
if (compressedLength < 0 || uncompressedLength < 0) {
throw new IllegalArgumentException("Invalid length");
}
byte[] compressed = new byte[compressedLength];
inputStream.readFully(compressed);
return Zstd.decompress(compressed, uncompressedLength);
}
public static NBTTagCompound readCompressCompound(DataInput inputStream) throws IOException {
NBTTagCompound compound;
byte[] compressed = readCompressed(inputStream);
try (DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(compressed))) {
compound = CompressedStreamTools.func_152456_a(dataInputStream, NBTSizeTracker.field_152451_a);
}
return compound;
}
public static byte[] writeCompound(NBTTagCompound compound) throws IOException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream(),
copy = new ByteArrayOutputStream();
try (DataOutputStream dataOutputStream = new DataOutputStream(outputStream)) {
CompressedStreamTools.write(compound, dataOutputStream);
}
byte[] uncompressed = outputStream.toByteArray();
byte[] compress = Zstd.compress(uncompressed);
try (DataOutputStream dataOutputStream = new DataOutputStream(copy)) {
dataOutputStream.writeInt(compress.length);
dataOutputStream.writeInt(uncompressed.length);
dataOutputStream.write(compress);
return copy.toByteArray();
}
}
}
@@ -0,0 +1,16 @@
package me.luizotavio.compressor.executor;
import java.util.concurrent.ExecutorService;
public class IOExecutor {
/**
* Please, do not change this value unless you know what you are doing.
* This value is used to define the number of threads that will be used to
* compress the files.
*
* If you want to change this value, please, do it with care.
*/
public static ExecutorService IO_EXECUTOR;
}
+23 -40
View File
@@ -1,8 +1,6 @@
package noppes.npcs;
import java.io.File;
import java.util.Set;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import cpw.mods.fml.common.FMLCommonHandler;
import cpw.mods.fml.common.Mod;
import cpw.mods.fml.common.Mod.EventHandler;
@@ -12,6 +10,7 @@ import cpw.mods.fml.common.network.FMLEventChannel;
import cpw.mods.fml.common.network.NetworkRegistry;
import cpw.mods.fml.common.registry.EntityRegistry;
import foxz.command.CommandNoppes;
import me.luizotavio.compressor.executor.IOExecutor;
import net.minecraft.block.Block;
import net.minecraft.block.BlockIce;
import net.minecraft.block.BlockLeavesBase;
@@ -25,50 +24,29 @@ import net.minecraftforge.common.MinecraftForge;
import nikedemos.markovnames.generators.*;
import noppes.npcs.config.ConfigLoader;
import noppes.npcs.config.ConfigProp;
import noppes.npcs.controllers.BankController;
import noppes.npcs.controllers.ChunkController;
import noppes.npcs.controllers.DialogController;
import noppes.npcs.controllers.FactionController;
import noppes.npcs.controllers.GlobalDataController;
import noppes.npcs.controllers.LinkedNpcController;
import noppes.npcs.controllers.PixelmonHelper;
import noppes.npcs.controllers.PlayerDataController;
import noppes.npcs.controllers.QuestController;
import noppes.npcs.controllers.RecipeController;
import noppes.npcs.controllers.ScriptController;
import noppes.npcs.controllers.ServerCloneController;
import noppes.npcs.controllers.SpawnController;
import noppes.npcs.controllers.TransportController;
import noppes.npcs.controllers.*;
import noppes.npcs.enchants.EnchantInterface;
import noppes.npcs.entity.*;
import noppes.npcs.entity.old.EntityNPCDwarfFemale;
import noppes.npcs.entity.old.EntityNPCDwarfMale;
import noppes.npcs.entity.old.EntityNPCElfFemale;
import noppes.npcs.entity.old.EntityNPCElfMale;
import noppes.npcs.entity.old.EntityNPCEnderman;
import noppes.npcs.entity.old.EntityNPCFurryFemale;
import noppes.npcs.entity.old.EntityNPCFurryMale;
import noppes.npcs.entity.old.EntityNPCHumanFemale;
import noppes.npcs.entity.old.EntityNPCHumanMale;
import noppes.npcs.entity.old.EntityNPCOrcFemale;
import noppes.npcs.entity.old.EntityNPCOrcMale;
import noppes.npcs.entity.old.EntityNPCVillager;
import noppes.npcs.entity.old.EntityNpcEnderchibi;
import noppes.npcs.entity.old.EntityNpcMonsterFemale;
import noppes.npcs.entity.old.EntityNpcMonsterMale;
import noppes.npcs.entity.old.EntityNpcNagaFemale;
import noppes.npcs.entity.old.EntityNpcNagaMale;
import noppes.npcs.entity.old.EntityNpcSkeleton;
import noppes.npcs.entity.old.*;
import noppes.npcs.scripted.wrapper.WrapperNpcAPI;
import java.io.File;
import java.util.Set;
import java.util.concurrent.Executors;
@Mod(modid = "customnpcs", name = "CustomNpcs", version = "1.5.1")
public class CustomNpcs {
@ConfigProp(info = "Disable Chat Bubbles")
@ConfigProp(info = "Disable Chat Bubbles")
public static boolean EnableChatBubbles = true;
private static int NewEntityStartId = 0;
@ConfigProp(info = "*BACKUP BEFORE CHANGING* File Type used with PlayerData. Converting files may reduce lag. To convert existing files: /noppes compressor update [<filename>|all] [json|zstd]")
public static String CompressorType = "json";
@ConfigProp(info = "The size of IO executors used for compression and decompression. This is the number of threads used for compression and decompression. If you are experiencing issues with the server lagging/crashing, try changing this to a higher number.")
public static int CompressorThreads = 2;
@ConfigProp(info = "Navigation search range for NPCs. Not recommended to increase if you have a slow pc or on a server. Minimum of 16, maximum of 96.")
public static int NpcNavRange = 32;
@@ -227,14 +205,19 @@ public class CustomNpcs {
registerNpc(EntityNPCGolem.class, "npcGolem");
registerNpc(EntityCustomNpc.class, "CustomNpc");
IOExecutor.IO_EXECUTOR = Executors.newFixedThreadPool(
CompressorThreads,
new ThreadFactoryBuilder()
.setNameFormat("CustomNpcs-IO-%d")
.build()
);
registerNewEntity(EntityChairMount.class, "CustomNpcChairMount", 64, 10, false);
registerNewEntity(EntityProjectile.class, "throwableitem", 64, 3, true);
registerNewEntity(EntityMagicProjectile.class, "magicprojectile", 64, 3, true);
new RecipeController();
ForgeChunkManager.setForcedChunkLoadingCallback(this, new ChunkController());
new CustomNpcsPermissions();
@@ -1,21 +1,24 @@
package noppes.npcs.controllers;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import me.luizotavio.compressor.ZstdCompressor;
import net.minecraft.entity.player.EntityPlayer;
import net.minecraft.nbt.CompressedStreamTools;
import net.minecraft.nbt.NBTSizeTracker;
import net.minecraft.nbt.NBTTagCompound;
import net.minecraft.server.MinecraftServer;
import noppes.npcs.CustomNpcs;
import noppes.npcs.LogWriter;
import noppes.npcs.util.JsonException;
import noppes.npcs.util.NBTJsonUtil;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.HashMap;
import java.util.Map;
import static me.luizotavio.compressor.executor.IOExecutor.IO_EXECUTOR;
public class PlayerDataController {
public static PlayerDataController instance;
@@ -70,14 +73,20 @@ public class PlayerDataController {
public NBTTagCompound loadPlayerData(String player){
File saveDir = getSaveDir();
String filename = player;
if(filename.isEmpty())
filename = "noplayername";
filename += ".json";
if(filename.isEmpty()) {
return new NBTTagCompound();
}
if(CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) {
filename += ".zstd";
} else {
filename += ".json";
}
try {
File file = new File(saveDir, filename);
if(file.exists()){
return NBTJsonUtil.LoadFile(file);
}
return ensureCompressor(
new File(saveDir, filename)
);
} catch (Exception e) {
LogWriter.error("Error loading: " + filename, e);
}
@@ -87,16 +96,24 @@ public class PlayerDataController {
public void savePlayerData(PlayerData data){
NBTTagCompound compound = data.getNBT();
String filename = data.uuid + ".json";
String filename;
if (CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) {
filename = data.uuid + ".zstd";
} else {
filename = data.uuid + ".json";
}
try {
File saveDir = getSaveDir();
File file = new File(saveDir, filename+"_new");
File file1 = new File(saveDir, filename);
NBTJsonUtil.SaveFile(file, compound);
if(file1.exists()){
file1.delete();
}
file.renameTo(file1);
File file = new File(saveDir, filename);
if (file.exists()) {
file.delete();
}
writeCompressor(file, compound);
} catch (Exception e) {
LogWriter.except(e);
}
@@ -164,7 +181,8 @@ public class PlayerDataController {
if(file.isDirectory() || !file.getName().endsWith(".json"))
continue;
try {
NBTTagCompound compound = NBTJsonUtil.LoadFile(file);
NBTTagCompound compound = ensureCompressor(file);
if(compound.hasKey("PlayerName")){
map.put(compound.getString("PlayerName"), compound);
}
@@ -178,4 +196,57 @@ public class PlayerDataController {
public boolean hasMail(EntityPlayer player) {
return getPlayerData(player).mailData.hasMail();
}
private void writeCompressor(File file, NBTTagCompound compound) {
if (compound == null) {
throw new IllegalArgumentException("Compound cannot be null!");
}
if (CustomNpcs.CompressorType.equalsIgnoreCase("json")) {
IO_EXECUTOR.submit(() -> {
try {
NBTJsonUtil.SaveFile(file, compound);
} catch (IOException | JsonException e) {
e.printStackTrace();
}
});
} else if (CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) {
IO_EXECUTOR.submit(() -> {
try {
byte[] compressed = ZstdCompressor.writeCompound(compound);
Files.write(
file.toPath(),
compressed,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE
);
} catch (Exception e) {
e.printStackTrace();
}
});
}
}
private NBTTagCompound ensureCompressor(File file) throws JsonException, IOException {
if (!file.exists()) {
return new NBTTagCompound();
}
if (CustomNpcs.CompressorType.equalsIgnoreCase("json")) {
return NBTJsonUtil.LoadFile(file);
}
if (CustomNpcs.CompressorType.equalsIgnoreCase("zstd")) {
byte[] readCompressed = ZstdCompressor.readCompressed(
new RandomAccessFile(file, "r")
);
try (DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(readCompressed))) {
return CompressedStreamTools.func_152456_a(dataInputStream, NBTSizeTracker.field_152451_a);
}
}
throw new IOException("Unknown compressor type: " + CustomNpcs.CompressorType);
}
}