Unverified Commit 0b532951 authored by Luke H-W's avatar Luke H-W Committed by GitHub
Browse files

TSJ and TSV parsing (#1962)

* Deserialization support for tsv files

* Benchmarking

* Apparently moving the setter out of the lambda fixed the setAccessible issue

* Thread it

* Use AllArgsConstructor instead of field reflection

* Clean up AllArgsConstructor TSV deserialization

* Refactor TsvUtils

* Remove AllArgsConstructors from Excels

* Set field accessible

* [WIP] TSJ improvements

* [WIP] More TSV stuff

* [WIP] More TSV stuff

* Working TSV parser (slow)

* Load Excels in TSJ > JSON > TSV priority
parent 46b0c7cf
......@@ -11,7 +11,10 @@ import emu.grasscutter.game.world.SpawnDataEntry;
import emu.grasscutter.game.world.SpawnDataEntry.GridBlockId;
import emu.grasscutter.game.world.SpawnDataEntry.SpawnGroupEntry;
import emu.grasscutter.scripts.SceneIndexManager;
import emu.grasscutter.utils.FileUtils;
import emu.grasscutter.utils.JsonUtils;
import emu.grasscutter.utils.TsvUtils;
import it.unimi.dsi.fastutil.Pair;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntArraySet;
......@@ -23,6 +26,8 @@ import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.regex.Pattern;
import java.util.stream.Stream;
......@@ -32,8 +37,9 @@ import static emu.grasscutter.utils.Language.translate;
public class ResourceLoader {
private static final List<String> loadedResources = new ArrayList<>();
private static final Set<String> loadedResources = new CopyOnWriteArraySet<>();
// Get a list of all resource classes, sorted by loadPriority
public static List<Class<?>> getResourceDefClasses() {
Reflections reflections = new Reflections(ResourceLoader.class.getPackage().getName());
Set<?> classes = reflections.getSubTypesOf(GameResource.class);
......@@ -51,6 +57,25 @@ public class ResourceLoader {
return classList;
}
// Get a list containing sets of all resource classes, sorted by loadPriority
protected static List<Set<Class<?>>> getResourceDefClassesPrioritySets() {
val reflections = new Reflections(ResourceLoader.class.getPackage().getName());
val classes = reflections.getSubTypesOf(GameResource.class);
val priorities = ResourceType.LoadPriority.getInOrder();
Grasscutter.getLogger().debug("Priorities are "+priorities);
val map = new LinkedHashMap<ResourceType.LoadPriority, Set<Class<?>>>(priorities.size());
priorities.forEach(p -> map.put(p, new HashSet<>()));
classes.forEach(c -> {
// val c = (Class<?>) o;
val annotation = c.getAnnotation(ResourceType.class);
if (annotation != null) {
map.get(annotation.loadPriority()).add(c);
}
});
return List.copyOf(map.values());
}
private static boolean loadedAll = false;
public static void loadAll() {
if (loadedAll) return;
......@@ -86,48 +111,66 @@ public class ResourceLoader {
}
public static void loadResources(boolean doReload) {
for (Class<?> resourceDefinition : getResourceDefClasses()) {
ResourceType type = resourceDefinition.getAnnotation(ResourceType.class);
long startTime = System.nanoTime();
val errors = new ConcurrentLinkedQueue<Pair<String, Exception>>(); // Logger in a parallel stream will deadlock
if (type == null) {
continue;
}
@SuppressWarnings("rawtypes")
Int2ObjectMap map = GameData.getMapByResourceDef(resourceDefinition);
getResourceDefClassesPrioritySets().forEach(classes -> {
classes.stream()
.parallel().unordered()
.forEach(c -> {
val type = c.getAnnotation(ResourceType.class);
if (type == null) return;
if (map == null) {
continue;
}
val map = GameData.getMapByResourceDef(c);
if (map == null) return;
try {
loadFromResource(resourceDefinition, type, map, doReload);
} catch (Exception e) {
Grasscutter.getLogger().error("Error loading resource file: " + Arrays.toString(type.name()), e.getLocalizedMessage());
}
}
try {
loadFromResource(c, type, map, doReload);
} catch (Exception e) {
errors.add(Pair.of(Arrays.toString(type.name()), e));
}
});
});
errors.forEach(pair -> Grasscutter.getLogger().error("Error loading resource file: " + pair.left(), pair.right()));
long endTime = System.nanoTime();
long ns = (endTime - startTime); //divide by 1000000 to get milliseconds.
Grasscutter.getLogger().debug("Loading resources took "+ns+"ns == "+ns/1000000+"ms");
}
@SuppressWarnings("rawtypes")
protected static void loadFromResource(Class<?> c, ResourceType type, Int2ObjectMap map, boolean doReload) throws Exception {
if (!loadedResources.contains(c.getSimpleName()) || doReload) {
val simpleName = c.getSimpleName();
if (doReload || !loadedResources.contains(simpleName)) {
for (String name : type.name()) {
loadFromResource(c, name, map);
loadFromResource(c, FileUtils.getExcelPath(name), map);
}
loadedResources.add(c.getSimpleName());
Grasscutter.getLogger().debug("Loaded " + map.size() + " " + c.getSimpleName() + "s.");
loadedResources.add(simpleName);
}
}
@SuppressWarnings({"rawtypes", "unchecked"})
protected static <T> void loadFromResource(Class<T> c, String fileName, Int2ObjectMap map) throws Exception {
List<T> list = JsonUtils.loadToList(getResourcePath("ExcelBinOutput/" + fileName), c);
protected static <T> void loadFromResource(Class<T> c, Path filename, Int2ObjectMap map) throws Exception {
val results = switch (FileUtils.getFileExtension(filename)) {
case "json" -> JsonUtils.loadToList(filename, c);
case "tsj" -> TsvUtils.loadTsjToListSetField(c, filename);
case "tsv" -> TsvUtils.loadTsvToListSetField(c, filename);
default -> null;
};
if (results == null) return;
results.forEach(o -> {
GameResource res = (GameResource) o;
res.onLoad();
map.put(res.getId(), res);
});
}
for (T o : list) {
@SuppressWarnings({"rawtypes", "unchecked"})
protected static <T> void loadFromResource(Class<T> c, String fileName, Int2ObjectMap map) throws Exception {
JsonUtils.loadToList(getResourcePath("ExcelBinOutput/" + fileName), c).forEach(o -> {
GameResource res = (GameResource) o;
res.onLoad();
map.put(res.getId(), res);
}
});
}
public class ScenePointConfig { // Sadly this doesn't work as a local class in loadScenePoints()
......
......@@ -2,6 +2,8 @@ package emu.grasscutter.data;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.List;
import java.util.stream.Stream;
@Retention(RetentionPolicy.RUNTIME)
public @interface ResourceType {
......@@ -28,5 +30,9 @@ public @interface ResourceType {
public int value() {
return value;
}
public static List<LoadPriority> getInOrder() {
return Stream.of(LoadPriority.values()).sorted((x, y) -> y.value() - x.value()).toList();
}
}
}
......@@ -21,7 +21,7 @@ public class ActivityWatcherData extends GameResource {
@Override
public void onLoad() {
triggerConfig.paramList = triggerConfig.paramList.stream().filter(x -> !x.isBlank()).toList();
triggerConfig.paramList = triggerConfig.paramList.stream().filter(x -> (x != null) && !x.isBlank()).toList();
triggerConfig.watcherTriggerType = WatcherTriggerType.getTypeByName(triggerConfig.triggerType);
}
......
......@@ -40,8 +40,11 @@ public class BattlePassMissionData extends GameResource {
@Override
public void onLoad() {
if (this.getTriggerConfig() != null && getTriggerConfig().getParamList()[0].length() > 0) {
this.mainParams = Arrays.stream(getTriggerConfig().getParamList()[0].split("[:;,]")).map(Integer::parseInt).collect(Collectors.toSet());
if (this.getTriggerConfig() != null) {
var params = getTriggerConfig().getParamList()[0];
if ((params != null) && !params.isEmpty()) {
this.mainParams = Arrays.stream(params.split("[:;,]")).map(Integer::parseInt).collect(Collectors.toSet());
}
}
}
......
package emu.grasscutter.game.props.ItemUseAction;
import emu.grasscutter.game.props.ItemUseOp;
import lombok.Getter;
public class ItemUseAddExp extends ItemUseAction {
@Getter private int exp = 0;
public class ItemUseAddExp extends ItemUseInt {
@Override
public ItemUseOp getItemUseOp() {
return ItemUseOp.ITEM_USE_ADD_EXP;
}
public ItemUseAddExp(String[] useParam) {
try {
this.exp = Integer.parseInt(useParam[0]);
} catch (NumberFormatException ignored) {}
super(useParam);
}
public int getExp() {
return this.i;
}
}
......@@ -14,7 +14,7 @@ public class ItemUseAddItem extends ItemUseInt {
super(useParam);
try {
this.count = Integer.parseInt(useParam[1]);
} catch (NumberFormatException ignored) {}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
}
@Override
......
package emu.grasscutter.game.props.ItemUseAction;
import emu.grasscutter.game.props.ItemUseOp;
import lombok.Getter;
public class ItemUseAddReliquaryExp extends ItemUseAction {
@Getter private int exp = 0;
public class ItemUseAddReliquaryExp extends ItemUseAddExp {
@Override
public ItemUseOp getItemUseOp() {
return ItemUseOp.ITEM_USE_ADD_RELIQUARY_EXP;
}
public ItemUseAddReliquaryExp(String[] useParam) {
try {
this.exp = Integer.parseInt(useParam[0]);
} catch (NumberFormatException ignored) {}
super(useParam);
}
}
......@@ -14,7 +14,7 @@ public class ItemUseAddServerBuff extends ItemUseInt {
super(useParam);
try {
this.duration = Integer.parseInt(useParam[1]);
} catch (NumberFormatException ignored) {}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
}
@Override
......
package emu.grasscutter.game.props.ItemUseAction;
import emu.grasscutter.game.props.ItemUseOp;
import lombok.Getter;
public class ItemUseAddWeaponExp extends ItemUseAction {
@Getter private int exp = 0;
public class ItemUseAddWeaponExp extends ItemUseAddExp {
@Override
public ItemUseOp getItemUseOp() {
return ItemUseOp.ITEM_USE_ADD_WEAPON_EXP;
}
public ItemUseAddWeaponExp(String[] useParam) {
try {
this.exp = Integer.parseInt(useParam[0]);
} catch (NumberFormatException ignored) {}
super(useParam);
}
}
......@@ -15,10 +15,10 @@ public class ItemUseCombineItem extends ItemUseInt {
super(useParam);
try {
this.resultId = Integer.parseInt(useParam[1]);
} catch (NumberFormatException ignored) {}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
try {
this.resultCount = Integer.parseInt(useParam[2]);
} catch (NumberFormatException ignored) {}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
}
@Override
......
......@@ -19,10 +19,10 @@ public class ItemUseGainAvatar extends ItemUseInt {
super(useParam);
try {
this.level = Integer.parseInt(useParam[1]);
} catch (NumberFormatException ignored) {}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
try {
this.constellation = Integer.parseInt(useParam[2]);
} catch (NumberFormatException ignored) {}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
}
@Override
......
......@@ -8,6 +8,6 @@ public abstract class ItemUseInt extends ItemUseAction {
public ItemUseInt(String[] useParam) {
try {
this.i = Integer.parseInt(useParam[0]);
} catch (NumberFormatException ignored) {}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
}
}
package emu.grasscutter.utils;
import emu.grasscutter.Grasscutter;
import lombok.val;
import java.io.File;
import java.io.IOException;
......@@ -111,6 +112,24 @@ public final class FileUtils {
return RESOURCES_PATH.resolve(path);
}
public static Path getExcelPath(String filename) {
return getTsjJsonTsv(RESOURCES_PATH.resolve("ExcelBinOutput"), filename);
}
// Gets path of a resource.
// If multiple formats of it exist, priority is TSJ > JSON > TSV
// If none exist, return the TSJ path, in case it wants to create a file
public static Path getTsjJsonTsv(Path root, String filename) {
val name = getFilenameWithoutExtension(filename);
val tsj = root.resolve(name + ".tsj");
if (Files.exists(tsj)) return tsj;
val json = root.resolve(name + ".json");
if (Files.exists(json)) return json;
val tsv = root.resolve(name + ".tsv");
if (Files.exists(tsv)) return tsv;
return tsj;
}
public static Path getScriptPath(String path) {
return SCRIPTS_PATH.resolve(path);
}
......@@ -167,14 +186,19 @@ public final class FileUtils {
}
}
@Deprecated // No current uses of this anyway
public static String getFilenameWithoutPath(String fileName) {
int i = fileName.lastIndexOf(".");
if (i > 0) {
return fileName.substring(0, i);
} else {
return fileName;
}
@Deprecated // Misnamed legacy function
public static String getFilenameWithoutPath(String filename) {
return getFilenameWithoutExtension(filename);
}
public static String getFilenameWithoutExtension(String filename) {
int i = filename.lastIndexOf(".");
return (i < 0) ? filename : filename.substring(0, i);
}
public static String getFileExtension(Path path) {
val filename = path.toString();
int i = filename.lastIndexOf(".");
return (i < 0) ? "" : filename.substring(i+1);
}
public static List<Path> getPathsFromResource(String folder) throws URISyntaxException {
......
......@@ -4,6 +4,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
......@@ -18,6 +19,7 @@ import com.google.gson.reflect.TypeToken;
import emu.grasscutter.data.common.DynamicFloat;
import emu.grasscutter.utils.JsonAdapters.*;
import it.unimi.dsi.fastutil.ints.IntList;
public final class JsonUtils {
......@@ -102,4 +104,12 @@ public final class JsonUtils {
return null;
}
}
public static <T> T decode(String jsonData, Type type) {
try {
return gson.fromJson(jsonData, type);
} catch (Exception ignored) {
return null;
}
}
}
This diff is collapsed.
......@@ -395,4 +395,22 @@ public final class Utils {
public static <T> T drawRandomListElement(List<T> list) {
return drawRandomListElement(list, null);
}
/***
* Splits a string by a character, into a list
* @param input The string to split
* @param separator The character to use as the split points
* @return A list of all the substrings
*/
public static List<String> nonRegexSplit(String input, int separator) {
var output = new ArrayList<String>();
int start = 0;
for (int next = input.indexOf(separator); next > 0; next = input.indexOf(separator, start)) {
output.add(input.substring(start, next));
start = next + 1;
}
if (start < input.length())
output.add(input.substring(start));
return output;
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment