使用 kala-encoding-detector 作为编码推测库 (#6477)
This commit is contained in:
1 parent
c2e807055a
commit
1277bf8b5d
6 files changed
+7
-55
No files matched your search
@@ -17,8 +17,8 @@
|
||||
*/
|
||||
package org.jackhuang.hmcl.game;
|
||||
|
||||
import kala.encdet.EncodingDetector;
|
||||
import org.jackhuang.hmcl.util.StringUtils;
|
||||
import org.jackhuang.hmcl.util.io.IOUtils;
|
||||
import org.jackhuang.hmcl.util.io.Zipper;
|
||||
import org.jackhuang.hmcl.util.logging.Logger;
|
||||
import org.jackhuang.hmcl.util.platform.OperatingSystem;
|
||||
@@ -91,7 +91,7 @@ public final class LogExporter {
|
||||
for (Path file : stream) {
|
||||
if (Files.isRegularFile(file)) {
|
||||
if (logMatcher == null || logMatcher.matches(file)) {
|
||||
try (BufferedReader reader = IOUtils.newBufferedReaderMaybeNativeEncoding(file)) {
|
||||
try (BufferedReader reader = EncodingDetector.MODERN_WEB.newBufferedReader(file)) {
|
||||
zipper.putLines(reader.lines().map(Logger::filterForbiddenToken), file.getFileName().toString());
|
||||
} catch (IOException e) {
|
||||
LOG.warning("Failed to read log file: " + file, e);
|
||||
|
||||
@@ -33,6 +33,7 @@ import javafx.scene.layout.VBox;
|
||||
import javafx.scene.text.Text;
|
||||
import javafx.scene.text.TextFlow;
|
||||
import javafx.stage.Stage;
|
||||
import kala.encdet.EncodingDetector;
|
||||
import org.jackhuang.hmcl.Metadata;
|
||||
import org.jackhuang.hmcl.download.LibraryAnalyzer;
|
||||
import org.jackhuang.hmcl.game.*;
|
||||
@@ -48,7 +49,6 @@ import org.jackhuang.hmcl.util.Lang;
|
||||
import org.jackhuang.hmcl.util.Log4jLevel;
|
||||
import org.jackhuang.hmcl.util.Pair;
|
||||
import org.jackhuang.hmcl.util.StringUtils;
|
||||
import org.jackhuang.hmcl.util.io.FileUtils;
|
||||
import org.jackhuang.hmcl.util.logging.Logger;
|
||||
import org.jackhuang.hmcl.util.platform.*;
|
||||
|
||||
@@ -149,7 +149,7 @@ public class GameCrashWindow extends Stage {
|
||||
|
||||
String log;
|
||||
try {
|
||||
log = FileUtils.readTextMaybeNativeEncoding(latestLog);
|
||||
log = EncodingDetector.MODERN_WEB.readString(latestLog);
|
||||
} catch (IOException e) {
|
||||
LOG.warning("Failed to read logs/latest.log", e);
|
||||
return pair(new HashSet<CrashReportAnalyzer.Result>(), new HashSet<String>());
|
||||
|
||||
@@ -14,6 +14,7 @@ tasks.compileJava {
|
||||
dependencies {
|
||||
api(libs.kala.compress.zip)
|
||||
api(libs.kala.compress.tar)
|
||||
api(libs.kala.encoding.detctor)
|
||||
api(libs.simple.png.javafx)
|
||||
api(libs.gson)
|
||||
api(libs.tomlj)
|
||||
@@ -23,7 +24,6 @@ dependencies {
|
||||
api(libs.constant.pool.scanner)
|
||||
api(libs.nanohttpd)
|
||||
api(libs.jsoup)
|
||||
api(libs.chardet)
|
||||
api(libs.jna)
|
||||
api(libs.pci.ids)
|
||||
api(libs.hello.nbt)
|
||||
|
||||
@@ -18,8 +18,6 @@
|
||||
package org.jackhuang.hmcl.util.io;
|
||||
|
||||
import com.google.errorprone.annotations.CanIgnoreReturnValue;
|
||||
import org.glavo.chardet.DetectedCharset;
|
||||
import org.glavo.chardet.UniversalDetector;
|
||||
import org.jackhuang.hmcl.util.StringUtils;
|
||||
import org.jackhuang.hmcl.util.function.ExceptionalConsumer;
|
||||
import org.jackhuang.hmcl.util.platform.OperatingSystem;
|
||||
@@ -38,7 +36,6 @@ import java.util.*;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.jackhuang.hmcl.util.logging.Logger.LOG;
|
||||
|
||||
/**
|
||||
@@ -236,24 +233,6 @@ public final class FileUtils {
|
||||
}
|
||||
}
|
||||
|
||||
public static String readTextMaybeNativeEncoding(Path file) throws IOException {
|
||||
byte[] bytes = Files.readAllBytes(file);
|
||||
|
||||
if (OperatingSystem.NATIVE_CHARSET == UTF_8)
|
||||
return new String(bytes, UTF_8);
|
||||
|
||||
UniversalDetector detector = new UniversalDetector();
|
||||
detector.handleData(bytes);
|
||||
detector.dataEnd();
|
||||
|
||||
DetectedCharset detectedCharset = detector.getDetectedCharset();
|
||||
if (detectedCharset != null && detectedCharset.isSupported()
|
||||
&& (detectedCharset == DetectedCharset.UTF_8 || detectedCharset == DetectedCharset.US_ASCII))
|
||||
return new String(bytes, UTF_8);
|
||||
else
|
||||
return new String(bytes, OperatingSystem.NATIVE_CHARSET);
|
||||
}
|
||||
|
||||
public static void deleteDirectory(Path directory) throws IOException {
|
||||
if (!Files.exists(directory))
|
||||
return;
|
||||
|
||||
@@ -17,19 +17,11 @@
|
||||
*/
|
||||
package org.jackhuang.hmcl.util.io;
|
||||
|
||||
import org.glavo.chardet.DetectedCharset;
|
||||
import org.glavo.chardet.UniversalDetector;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.*;
|
||||
import static org.jackhuang.hmcl.util.platform.OperatingSystem.NATIVE_CHARSET;
|
||||
|
||||
/**
|
||||
* This utility class consists of some util methods operating on InputStream/OutputStream.
|
||||
@@ -43,25 +35,6 @@ public final class IOUtils {
|
||||
|
||||
public static final int DEFAULT_BUFFER_SIZE = 32 * 1024;
|
||||
|
||||
public static BufferedReader newBufferedReaderMaybeNativeEncoding(Path file) throws IOException {
|
||||
if (NATIVE_CHARSET == UTF_8)
|
||||
return Files.newBufferedReader(file);
|
||||
|
||||
FileChannel channel = FileChannel.open(file);
|
||||
try {
|
||||
long oldPosition = channel.position();
|
||||
DetectedCharset detectedCharset = UniversalDetector.detectCharset(channel);
|
||||
Charset charset = detectedCharset != null && detectedCharset.isSupported()
|
||||
&& (detectedCharset.getCharset() == UTF_8 || detectedCharset.getCharset() == US_ASCII)
|
||||
? UTF_8 : NATIVE_CHARSET;
|
||||
channel.position(oldPosition);
|
||||
return new BufferedReader(new InputStreamReader(Channels.newInputStream(channel), charset));
|
||||
} catch (Throwable e) {
|
||||
closeQuietly(channel, e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
public static byte[] readFully(InputStream stream) throws IOException {
|
||||
try (stream) {
|
||||
return stream.readAllBytes();
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
hmclauncher = "3.7.0.1"
|
||||
jetbrains-annotations = "26.1.0"
|
||||
kala-compress = "1.27.1-3"
|
||||
kala-encoding-detctor = "0.1.0"
|
||||
simple-png-javafx = "0.3.0"
|
||||
gson = "2.13.2"
|
||||
tomlj = "1.1.1"
|
||||
@@ -12,7 +13,6 @@ constant-pool-scanner = "1.2"
|
||||
hello-nbt = "0.3.0"
|
||||
nanohttpd = "2.3.1"
|
||||
jsoup = "1.21.2"
|
||||
chardet = "2.5.0"
|
||||
fxsvgimage = "1.3"
|
||||
jna = "5.18.1"
|
||||
pci-ids = "0.4.0"
|
||||
@@ -39,6 +39,7 @@ jetbrains-annotations = { module = "org.jetbrains:annotations", version.ref = "j
|
||||
kala-compress-zip = { module = "org.glavo.kala:kala-compress-archivers-zip", version.ref = "kala-compress" }
|
||||
kala-compress-tar = { module = "org.glavo.kala:kala-compress-archivers-tar", version.ref = "kala-compress" }
|
||||
kala-compress-ar = { module = "org.glavo.kala:kala-compress-archivers-ar", version.ref = "kala-compress" }
|
||||
kala-encoding-detctor = { module = "org.glavo:kala-encoding-detector", version.ref = "kala-encoding-detctor" }
|
||||
simple-png-javafx = { module = "org.glavo:simple-png-javafx", version.ref = "simple-png-javafx" }
|
||||
gson = { module = "com.google.code.gson:gson", version.ref = "gson" }
|
||||
tomlj = { module = "org.tomlj:tomlj", version.ref = "tomlj" }
|
||||
@@ -49,7 +50,6 @@ constant-pool-scanner = { module = "org.jenkins-ci:constant-pool-scanner", versi
|
||||
hello-nbt = { module = "org.glavo:HelloNBT", version.ref = "hello-nbt" }
|
||||
nanohttpd = { module = "org.nanohttpd:nanohttpd", version.ref = "nanohttpd" }
|
||||
jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" }
|
||||
chardet = { module = "org.glavo:chardet", version.ref = "chardet" }
|
||||
fxsvgimage = { module = "com.github.hervegirod:fxsvgimage", version.ref = "fxsvgimage" }
|
||||
jna = { module = "net.java.dev.jna:jna", version.ref = "jna" }
|
||||
jna-platform = { module = "net.java.dev.jna:jna-platform", version.ref = "jna" }
|
||||
|
||||
Reference in new issue
Block a user