Skip to content

GitLab

Commit 09944172 authored by AnimeGitB's avatar AnimeGitB Committed by Luke H-W
Browse files

Cache used strings from TextMaps

parent 4790158a
Show whitespace changes
Inline Side-by-side
.gitignore
View file @ 09944172
......@@ -52,6 +52,7 @@ tmp/
.vscode
# Grasscutter
/cache
/resources
/logs
/plugins
......
src/main/java/emu/grasscutter/data/ResourceLoader.java
View file @ 09944172
......@@ -53,7 +53,9 @@ public class ResourceLoader {
return classList;
}
private static boolean loadedAll = false;
public static void loadAll() {
if (loadedAll) return;
Grasscutter.getLogger().info(translate("messages.status.resources.loading"));
// Load ability lists
......@@ -75,6 +77,7 @@ public class ResourceLoader {
loadNpcBornData();
Grasscutter.getLogger().info(translate("messages.status.resources.finish"));
loadedAll = true;
}
public static void loadResources() {
......
src/main/java/emu/grasscutter/tools/Tools.java
View file @ 09944172
package emu.grasscutter.tools;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
......@@ -12,10 +10,7 @@ import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import com.google.gson.reflect.TypeToken;
......@@ -33,119 +28,25 @@ import emu.grasscutter.data.excels.QuestData;
import emu.grasscutter.data.excels.SceneData;
import emu.grasscutter.utils.Language;
import emu.grasscutter.utils.Utils;
import emu.grasscutter.utils.Language.TextStrings;
import it.unimi.dsi.fastutil.ints.Int2IntMap;
import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import lombok.EqualsAndHashCode;
import static emu.grasscutter.config.Configuration.*;
import static emu.grasscutter.utils.Language.translate;
public final class Tools {
@EqualsAndHashCode public static class TextStrings {
public static final String[] ARR_LANGUAGES = {"EN", "CHS", "CHT", "JP", "KR", "DE", "ES", "FR", "ID", "PT", "RU", "TH", "VI"};
public static final String[] ARR_GC_LANGUAGES = {"en-US", "zh-CN", "zh-TW", "JP", "KR", "DE", "es-ES", "fr-FR", "ID", "PT", "ru-RU", "TH", "VI"};
public static final int NUM_LANGUAGES = ARR_LANGUAGES.length;
public static final List<String> LIST_LANGUAGES = Arrays.asList(ARR_LANGUAGES);
public static final Object2IntMap<String> MAP_LANGUAGES = // Map "EN": 0, "CHS": 1, ..., "VI": 12
new Object2IntOpenHashMap<>(
IntStream.range(0, ARR_LANGUAGES.length)
.boxed()
.collect(Collectors.toMap(i -> ARR_LANGUAGES[i], i -> i)));
public String[] strings = new String[ARR_LANGUAGES.length];
public TextStrings() {};
public TextStrings(String init) {
for (int i = 0; i < NUM_LANGUAGES; i++)
this.strings[i] = init;
};
public TextStrings(Collection<String> strings) {
this.strings = strings.toArray(new String[0]);
}
public String get(String languageCode) {
return strings[MAP_LANGUAGES.getOrDefault(languageCode, 0)];
}
public boolean set(String languageCode, String string) {
int index = MAP_LANGUAGES.getOrDefault(languageCode, -1);
if (index < 0) return false;
strings[index] = string;
return true;
}
}
private static final Pattern textMapKeyValueRegex = Pattern.compile("\"(\\d+)\": \"(.+)\"");
private static Int2ObjectMap<String> loadTextMap(String language, IntSet nameHashes) {
Int2ObjectMap<String> output = new Int2ObjectOpenHashMap<>();
try (BufferedReader file = new BufferedReader(new FileReader(Utils.toFilePath(RESOURCE("TextMap/TextMap"+language+".json")), StandardCharsets.UTF_8))) {
Matcher matcher = textMapKeyValueRegex.matcher("");
return new Int2ObjectOpenHashMap<>(
file.lines()
.sequential()
.map(matcher::reset) // Side effects, but it's faster than making a new one
.filter(Matcher::find)
.filter(m -> nameHashes.contains((int) Long.parseLong(m.group(1)))) // TODO: Cache this parse somehow
.collect(Collectors.toMap(
m -> (int) Long.parseLong(m.group(1)),
m -> m.group(2))));
} catch (Exception e) {
Grasscutter.getLogger().error("Error loading textmap: " + language);
Grasscutter.getLogger().error(e.toString());
}
return output;
}
public static Int2ObjectMap<TextStrings> loadTextMaps(IntSet nameHashes) {
Map<Integer, Int2ObjectMap<String>> mapLanguageMaps = // Separate step to process the textmaps in parallel
TextStrings.LIST_LANGUAGES.parallelStream().collect(
Collectors.toConcurrentMap(s -> TextStrings.MAP_LANGUAGES.getInt(s), s -> loadTextMap(s, nameHashes)));
List<Int2ObjectMap<String>> languageMaps =
IntStream.range(0, TextStrings.NUM_LANGUAGES)
.mapToObj(i -> mapLanguageMaps.get(i))
.collect(Collectors.toList());
Map<TextStrings, TextStrings> canonicalTextStrings = new HashMap<>();
return new Int2ObjectOpenHashMap<TextStrings>(
nameHashes
.intStream()
.boxed()
.collect(Collectors.toMap(key -> key, key -> {
TextStrings t = new TextStrings(
IntStream.range(0, TextStrings.NUM_LANGUAGES)
.mapToObj(i -> languageMaps.get(i).getOrDefault((int) key, "[N/A] - hash key %d".formatted(key)))
.collect(Collectors.toList()));
return canonicalTextStrings.computeIfAbsent(t, x -> t);
}))
);
}
public static void createGmHandbooks() throws Exception {
ResourceLoader.loadAll();
Int2IntMap avatarNames = new Int2IntOpenHashMap(GameData.getAvatarDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash())));
Int2IntMap itemNames = new Int2IntOpenHashMap(GameData.getItemDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash())));
Int2IntMap monsterNames = new Int2IntOpenHashMap(GameData.getMonsterDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getNameTextMapHash())));
Int2IntMap mainQuestTitles = new Int2IntOpenHashMap(GameData.getMainQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getTitleTextMapHash())));
Int2IntMap questDescs = new Int2IntOpenHashMap(GameData.getQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getDescTextMapHash())));
IntSet usedHashes = new IntOpenHashSet();
usedHashes.addAll(avatarNames.values());
usedHashes.addAll(itemNames.values());
usedHashes.addAll(monsterNames.values());
usedHashes.addAll(mainQuestTitles.values());
usedHashes.addAll(questDescs.values());
// Int2IntMap questDescs = new Int2IntOpenHashMap(GameData.getQuestDataMap().int2ObjectEntrySet().stream().collect(Collectors.toMap(e -> (int) e.getIntKey(), e -> (int) e.getValue().getDescTextMapHash())));
Int2ObjectMap<TextStrings> textMaps = loadTextMaps(usedHashes);
Int2ObjectMap<TextStrings> textMaps = Language.getTextMapStrings();
Language savedLanguage = Grasscutter.getLanguage();
......@@ -233,10 +134,8 @@ public final class Tools {
var questDataMap = GameData.getQuestDataMap();
questDataMap.keySet().intStream().sorted().forEach(id -> {
QuestData data = questDataMap.get(id);
int titleKey = (int) mainQuestTitles.get(data.getMainId());
int descKey = (int) data.getDescTextMapHash();
TextStrings title = textMaps.get(titleKey);
TextStrings desc = textMaps.get(descKey);
TextStrings title = textMaps.get((int) mainQuestTitles.get(data.getMainId()));
TextStrings desc = textMaps.get((int) data.getDescTextMapHash());
for (int i = 0; i < TextStrings.NUM_LANGUAGES; i++) {
handbookBuilders[i]
.append(id)
......
src/main/java/emu/grasscutter/utils/Language.java
View file @ 09944172
......@@ -3,14 +3,43 @@ package emu.grasscutter.utils;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.data.GameData;
import emu.grasscutter.data.ResourceLoader;
import emu.grasscutter.game.player.Player;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import lombok.EqualsAndHashCode;
import javax.annotation.Nullable;
import static emu.grasscutter.config.Configuration.*;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public final class Language {
......@@ -207,4 +236,187 @@ public final class Language {
return languageFile;
}
}
private static final int TEXTMAP_CACHE_VERSION = 0x9CCACE02;
@EqualsAndHashCode public static class TextStrings implements Serializable {
public static final String[] ARR_LANGUAGES = {"EN", "CHS", "CHT", "JP", "KR", "DE", "ES", "FR", "ID", "PT", "RU", "TH", "VI"};
public static final String[] ARR_GC_LANGUAGES = {"en-US", "zh-CN", "zh-TW", "JP", "KR", "DE", "es-ES", "fr-FR", "ID", "PT", "ru-RU", "TH", "VI"};
public static final int NUM_LANGUAGES = ARR_LANGUAGES.length;
public static final List<String> LIST_LANGUAGES = Arrays.asList(ARR_LANGUAGES);
public static final Object2IntMap<String> MAP_LANGUAGES = // Map "EN": 0, "CHS": 1, ..., "VI": 12
new Object2IntOpenHashMap<>(
IntStream.range(0, ARR_LANGUAGES.length)
.boxed()
.collect(Collectors.toMap(i -> ARR_LANGUAGES[i], i -> i)));
public String[] strings = new String[ARR_LANGUAGES.length];
public TextStrings() {};
public TextStrings(String init) {
for (int i = 0; i < NUM_LANGUAGES; i++)
this.strings[i] = init;
};
public TextStrings(List<String> strings, int key) {
// Some hashes don't have strings for some languages :(
String nullReplacement = "[N/A] %d".formatted((long) key & 0xFFFFFFFFL);
for (int i = 0; i < NUM_LANGUAGES; i++) { // Find first non-null if there is any
String s = strings.get(i);
if (s != null) {
nullReplacement = "[%s] - %s".formatted(ARR_LANGUAGES[i], s);
break;
}
}
for (int i = 0; i < NUM_LANGUAGES; i++) {
String s = strings.get(i);
if (s != null)
this.strings[i] = s;
else
this.strings[i] = nullReplacement;
}
}
public String get(String languageCode) {
return strings[MAP_LANGUAGES.getOrDefault(languageCode, 0)];
}
public boolean set(String languageCode, String string) {
int index = MAP_LANGUAGES.getOrDefault(languageCode, -1);
if (index < 0) return false;
strings[index] = string;
return true;
}
}
private static final Pattern textMapKeyValueRegex = Pattern.compile("\"(\\d+)\": \"(.+)\"");
private static Int2ObjectMap<String> loadTextMapFile(String language, IntSet nameHashes) {
Int2ObjectMap<String> output = new Int2ObjectOpenHashMap<>();
try (BufferedReader file = new BufferedReader(new FileReader(Utils.toFilePath(RESOURCE("TextMap/TextMap"+language+".json")), StandardCharsets.UTF_8))) {
Matcher matcher = textMapKeyValueRegex.matcher("");
return new Int2ObjectOpenHashMap<>(
file.lines()
.sequential()
.map(matcher::reset) // Side effects, but it's faster than making a new one
.filter(Matcher::find)
.filter(m -> nameHashes.contains((int) Long.parseLong(m.group(1)))) // TODO: Cache this parse somehow
.collect(Collectors.toMap(
m -> (int) Long.parseLong(m.group(1)),
m -> m.group(2).replace("\\\"", "\""))));
} catch (Exception e) {
Grasscutter.getLogger().error("Error loading textmap: " + language);
Grasscutter.getLogger().error(e.toString());
}
return output;
}
private static Int2ObjectMap<TextStrings> loadTextMapFiles(IntSet nameHashes) {
Map<Integer, Int2ObjectMap<String>> mapLanguageMaps = // Separate step to process the textmaps in parallel
TextStrings.LIST_LANGUAGES.parallelStream().collect(
Collectors.toConcurrentMap(s -> TextStrings.MAP_LANGUAGES.getInt(s), s -> loadTextMapFile(s, nameHashes)));
List<Int2ObjectMap<String>> languageMaps =
IntStream.range(0, TextStrings.NUM_LANGUAGES)
.mapToObj(i -> mapLanguageMaps.get(i))
.collect(Collectors.toList());
Map<TextStrings, TextStrings> canonicalTextStrings = new HashMap<>();
return new Int2ObjectOpenHashMap<TextStrings>(
nameHashes
.intStream()
.boxed()
.collect(Collectors.toMap(key -> key, key -> {
TextStrings t = new TextStrings(
IntStream.range(0, TextStrings.NUM_LANGUAGES)
.mapToObj(i -> languageMaps.get(i).get((int) key))
.collect(Collectors.toList()), (int) key);
return canonicalTextStrings.computeIfAbsent(t, x -> t);
}))
);
}
private static Int2ObjectMap<TextStrings> loadTextMapsCache() throws Exception {
try (ObjectInputStream file = new ObjectInputStream(new BufferedInputStream(Files.newInputStream(TEXTMAP_CACHE_PATH), 0x100000))) {
final int fileVersion = file.readInt();
if (fileVersion != TEXTMAP_CACHE_VERSION)
throw new Exception("Invalid cache version");
return (Int2ObjectMap<TextStrings>) file.readObject();
}
}
private static void saveTextMapsCache(Int2ObjectMap<TextStrings> input) throws IOException {
try {
Files.createDirectory(Path.of("cache"));
} catch (FileAlreadyExistsException ignored) {};
try (ObjectOutputStream file = new ObjectOutputStream(new BufferedOutputStream(Files.newOutputStream(TEXTMAP_CACHE_PATH, StandardOpenOption.CREATE), 0x100000))) {
file.writeInt(TEXTMAP_CACHE_VERSION);
file.writeObject(input);
}
}
private static Int2ObjectMap<TextStrings> textMapStrings;
private static final Path TEXTMAP_CACHE_PATH = Path.of(Utils.toFilePath("cache/TextMapCache.bin"));
public static Int2ObjectMap<TextStrings> getTextMapStrings() {
if (textMapStrings == null)
loadTextMaps();
return textMapStrings;
}
public static TextStrings getTextMapKey(long hash) {
return textMapStrings.get((int) hash);
}
public static void loadTextMaps() {
// Check system timestamps on cache and resources
try {
long cacheModified = Files.getLastModifiedTime(TEXTMAP_CACHE_PATH).toMillis();
long textmapsModified = Files.list(Path.of(RESOURCE("TextMap")))
.filter(path -> path.toString().endsWith(".json"))
.map(path -> {
try {
return Files.getLastModifiedTime(path).toMillis();
} catch (Exception ignored) {
Grasscutter.getLogger().debug("Exception while checking modified time: ", path);
return Long.MAX_VALUE; // Don't use cache, something has gone wrong
}
})
.max(Long::compare)
.get();
Grasscutter.getLogger().debug("Cache modified %d, textmap modified %d".formatted(cacheModified, textmapsModified));
if (textmapsModified < cacheModified) {
// Try loading from cache
Grasscutter.getLogger().info("Loading cached TextMaps");
textMapStrings = loadTextMapsCache();
return;
}
} catch (Exception e) {
Grasscutter.getLogger().debug("Exception while checking cache: ", e);
};
// Regenerate cache
Grasscutter.getLogger().info("Generating TextMaps cache");
ResourceLoader.loadAll();
IntSet usedHashes = new IntOpenHashSet();
GameData.getAvatarDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
GameData.getItemDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
GameData.getMonsterDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
GameData.getMainQuestDataMap().forEach((k, v) -> usedHashes.add((int) v.getTitleTextMapHash()));
GameData.getQuestDataMap().forEach((k, v) -> usedHashes.add((int) v.getDescTextMapHash()));
// Incidental strings
usedHashes.add((int) 4233146695L); // Character
usedHashes.add((int) 4231343903L); // Weapon
usedHashes.add((int) 332935371L); // Standard Wish
usedHashes.add((int) 2272170627L); // Character Event Wish
usedHashes.add((int) 3352513147L); // Character Event Wish-2
usedHashes.add((int) 2864268523L); // Weapon Event Wish
textMapStrings = loadTextMapFiles(usedHashes);
try {
saveTextMapsCache(textMapStrings);
} catch (IOException e) {
Grasscutter.getLogger().error("Failed to save TextMap cache: ", e);
};
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment