使用 kala-encoding-detector 作为编码推测库 (#6477)

This commit is contained in:
Glavo authored and GitHub committed 2026-07-26 00:47:43 +08:00
1 parent c2e807055a
commit 1277bf8b5d
6 files changed
+7 -55

No files matched your search

@@ -17,8 +17,8 @@
*/
package org.jackhuang.hmcl.game;
import kala.encdet.EncodingDetector;
import org.jackhuang.hmcl.util.StringUtils;
import org.jackhuang.hmcl.util.io.IOUtils;
import org.jackhuang.hmcl.util.io.Zipper;
import org.jackhuang.hmcl.util.logging.Logger;
import org.jackhuang.hmcl.util.platform.OperatingSystem;
@@ -91,7 +91,7 @@ public final class LogExporter {
for (Path file : stream) {
if (Files.isRegularFile(file)) {
if (logMatcher == null || logMatcher.matches(file)) {
try (BufferedReader reader = IOUtils.newBufferedReaderMaybeNativeEncoding(file)) {
try (BufferedReader reader = EncodingDetector.MODERN_WEB.newBufferedReader(file)) {
zipper.putLines(reader.lines().map(Logger::filterForbiddenToken), file.getFileName().toString());
} catch (IOException e) {
LOG.warning("Failed to read log file: " + file, e);
@@ -33,6 +33,7 @@ import javafx.scene.layout.VBox;
import javafx.scene.text.Text;
import javafx.scene.text.TextFlow;
import javafx.stage.Stage;
import kala.encdet.EncodingDetector;
import org.jackhuang.hmcl.Metadata;
import org.jackhuang.hmcl.download.LibraryAnalyzer;
import org.jackhuang.hmcl.game.*;
@@ -48,7 +49,6 @@ import org.jackhuang.hmcl.util.Lang;
import org.jackhuang.hmcl.util.Log4jLevel;
import org.jackhuang.hmcl.util.Pair;
import org.jackhuang.hmcl.util.StringUtils;
import org.jackhuang.hmcl.util.io.FileUtils;
import org.jackhuang.hmcl.util.logging.Logger;
import org.jackhuang.hmcl.util.platform.*;
@@ -149,7 +149,7 @@ public class GameCrashWindow extends Stage {
String log;
try {
log = FileUtils.readTextMaybeNativeEncoding(latestLog);
log = EncodingDetector.MODERN_WEB.readString(latestLog);
} catch (IOException e) {
LOG.warning("Failed to read logs/latest.log", e);
return pair(new HashSet<CrashReportAnalyzer.Result>(), new HashSet<String>());
+1 -1
View File
@@ -14,6 +14,7 @@ tasks.compileJava {
dependencies {
api(libs.kala.compress.zip)
api(libs.kala.compress.tar)
api(libs.kala.encoding.detctor)
api(libs.simple.png.javafx)
api(libs.gson)
api(libs.tomlj)
@@ -23,7 +24,6 @@ dependencies {
api(libs.constant.pool.scanner)
api(libs.nanohttpd)
api(libs.jsoup)
api(libs.chardet)
api(libs.jna)
api(libs.pci.ids)
api(libs.hello.nbt)
@@ -18,8 +18,6 @@
package org.jackhuang.hmcl.util.io;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import org.glavo.chardet.DetectedCharset;
import org.glavo.chardet.UniversalDetector;
import org.jackhuang.hmcl.util.StringUtils;
import org.jackhuang.hmcl.util.function.ExceptionalConsumer;
import org.jackhuang.hmcl.util.platform.OperatingSystem;
@@ -38,7 +36,6 @@ import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Stream;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.jackhuang.hmcl.util.logging.Logger.LOG;
/**
@@ -236,24 +233,6 @@ public final class FileUtils {
}
}
public static String readTextMaybeNativeEncoding(Path file) throws IOException {
byte[] bytes = Files.readAllBytes(file);
if (OperatingSystem.NATIVE_CHARSET == UTF_8)
return new String(bytes, UTF_8);
UniversalDetector detector = new UniversalDetector();
detector.handleData(bytes);
detector.dataEnd();
DetectedCharset detectedCharset = detector.getDetectedCharset();
if (detectedCharset != null && detectedCharset.isSupported()
&& (detectedCharset == DetectedCharset.UTF_8 || detectedCharset == DetectedCharset.US_ASCII))
return new String(bytes, UTF_8);
else
return new String(bytes, OperatingSystem.NATIVE_CHARSET);
}
public static void deleteDirectory(Path directory) throws IOException {
if (!Files.exists(directory))
return;
@@ -17,19 +17,11 @@
*/
package org.jackhuang.hmcl.util.io;
import org.glavo.chardet.DetectedCharset;
import org.glavo.chardet.UniversalDetector;
import java.io.*;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.zip.GZIPInputStream;
import static java.nio.charset.StandardCharsets.*;
import static org.jackhuang.hmcl.util.platform.OperatingSystem.NATIVE_CHARSET;
/**
* This utility class consists of some util methods operating on InputStream/OutputStream.
@@ -43,25 +35,6 @@ public final class IOUtils {
public static final int DEFAULT_BUFFER_SIZE = 32 * 1024;
public static BufferedReader newBufferedReaderMaybeNativeEncoding(Path file) throws IOException {
if (NATIVE_CHARSET == UTF_8)
return Files.newBufferedReader(file);
FileChannel channel = FileChannel.open(file);
try {
long oldPosition = channel.position();
DetectedCharset detectedCharset = UniversalDetector.detectCharset(channel);
Charset charset = detectedCharset != null && detectedCharset.isSupported()
&& (detectedCharset.getCharset() == UTF_8 || detectedCharset.getCharset() == US_ASCII)
? UTF_8 : NATIVE_CHARSET;
channel.position(oldPosition);
return new BufferedReader(new InputStreamReader(Channels.newInputStream(channel), charset));
} catch (Throwable e) {
closeQuietly(channel, e);
throw e;
}
}
public static byte[] readFully(InputStream stream) throws IOException {
try (stream) {
return stream.readAllBytes();
+2 -2
View File
@@ -2,6 +2,7 @@
hmclauncher = "3.7.0.1"
jetbrains-annotations = "26.1.0"
kala-compress = "1.27.1-3"
kala-encoding-detctor = "0.1.0"
simple-png-javafx = "0.3.0"
gson = "2.13.2"
tomlj = "1.1.1"
@@ -12,7 +13,6 @@ constant-pool-scanner = "1.2"
hello-nbt = "0.3.0"
nanohttpd = "2.3.1"
jsoup = "1.21.2"
chardet = "2.5.0"
fxsvgimage = "1.3"
jna = "5.18.1"
pci-ids = "0.4.0"
@@ -39,6 +39,7 @@ jetbrains-annotations = { module = "org.jetbrains:annotations", version.ref = "j
kala-compress-zip = { module = "org.glavo.kala:kala-compress-archivers-zip", version.ref = "kala-compress" }
kala-compress-tar = { module = "org.glavo.kala:kala-compress-archivers-tar", version.ref = "kala-compress" }
kala-compress-ar = { module = "org.glavo.kala:kala-compress-archivers-ar", version.ref = "kala-compress" }
kala-encoding-detctor = { module = "org.glavo:kala-encoding-detector", version.ref = "kala-encoding-detctor" }
simple-png-javafx = { module = "org.glavo:simple-png-javafx", version.ref = "simple-png-javafx" }
gson = { module = "com.google.code.gson:gson", version.ref = "gson" }
tomlj = { module = "org.tomlj:tomlj", version.ref = "tomlj" }
@@ -49,7 +50,6 @@ constant-pool-scanner = { module = "org.jenkins-ci:constant-pool-scanner", versi
hello-nbt = { module = "org.glavo:HelloNBT", version.ref = "hello-nbt" }
nanohttpd = { module = "org.nanohttpd:nanohttpd", version.ref = "nanohttpd" }
jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" }
chardet = { module = "org.glavo:chardet", version.ref = "chardet" }
fxsvgimage = { module = "com.github.hervegirod:fxsvgimage", version.ref = "fxsvgimage" }
jna = { module = "net.java.dev.jna:jna", version.ref = "jna" }
jna-platform = { module = "net.java.dev.jna:jna-platform", version.ref = "jna" }