TSJ and TSV parsing (#1962)

* Deserialization support for tsv files

* Benchmarking

* Apparently moving the setter out of the lambda fixed the setAccessible issue

* Thread it

* Use AllArgsConstructor instead of field reflection

* Clean up AllArgsConstructor TSV deserialization

* Refactor TsvUtils

* Remove AllArgsConstructors from Excels

* Set field accessible

* [WIP] TSJ improvements

* [WIP] More TSV stuff

* [WIP] More TSV stuff

* Working TSV parser (slow)

* Load Excels in TSJ > JSON > TSV priority

src/main/java/emu/grasscutter/data/ResourceLoader.java

@@ -11,7 +11,10 @@ import emu.grasscutter.game.world.SpawnDataEntry;
 import emu.grasscutter.game.world.SpawnDataEntry.GridBlockId;
 import emu.grasscutter.game.world.SpawnDataEntry.SpawnGroupEntry;
 import emu.grasscutter.scripts.SceneIndexManager;
+import emu.grasscutter.utils.FileUtils;
 import emu.grasscutter.utils.JsonUtils;
+import emu.grasscutter.utils.TsvUtils;
+import it.unimi.dsi.fastutil.Pair;
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
 import it.unimi.dsi.fastutil.ints.IntArrayList;
 import it.unimi.dsi.fastutil.ints.IntArraySet;
@@ -23,6 +26,8 @@ import java.io.*;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
@@ -32,8 +37,9 @@ import static emu.grasscutter.utils.Language.translate;
 
 public class ResourceLoader {
 
-    private static final List<String> loadedResources = new ArrayList<>();
+    private static final Set<String> loadedResources = new CopyOnWriteArraySet<>();
 
+    // Get a list of all resource classes, sorted by loadPriority
     public static List<Class<?>> getResourceDefClasses() {
         Reflections reflections = new Reflections(ResourceLoader.class.getPackage().getName());
         Set<?> classes = reflections.getSubTypesOf(GameResource.class);
@@ -51,6 +57,25 @@ public class ResourceLoader {
         return classList;
     }
 
+    // Get a list containing sets of all resource classes, sorted by loadPriority
+    protected static List<Set<Class<?>>> getResourceDefClassesPrioritySets() {
+        val reflections = new Reflections(ResourceLoader.class.getPackage().getName());
+        val classes = reflections.getSubTypesOf(GameResource.class);
+        val priorities = ResourceType.LoadPriority.getInOrder();
+        Grasscutter.getLogger().debug("Priorities are "+priorities);
+        val map = new LinkedHashMap<ResourceType.LoadPriority, Set<Class<?>>>(priorities.size());
+        priorities.forEach(p -> map.put(p, new HashSet<>()));
+
+        classes.forEach(c -> {
+            // val c = (Class<?>) o;
+            val annotation = c.getAnnotation(ResourceType.class);
+            if (annotation != null) {
+                map.get(annotation.loadPriority()).add(c);
+            }
+        });
+        return List.copyOf(map.values());
+    }
+
     private static boolean loadedAll = false;
     public static void loadAll() {
         if (loadedAll) return;
@@ -86,48 +111,66 @@ public class ResourceLoader {
     }
 
     public static void loadResources(boolean doReload) {
-        for (Class<?> resourceDefinition : getResourceDefClasses()) {
-            ResourceType type = resourceDefinition.getAnnotation(ResourceType.class);
+        long startTime = System.nanoTime();
+        val errors = new ConcurrentLinkedQueue<Pair<String, Exception>>();  // Logger in a parallel stream will deadlock
 
-            if (type == null) {
-                continue;
-            }
-
-            @SuppressWarnings("rawtypes")
-            Int2ObjectMap map = GameData.getMapByResourceDef(resourceDefinition);
+        getResourceDefClassesPrioritySets().forEach(classes -> {
+            classes.stream()
+                .parallel().unordered()
+                .forEach(c -> {
+                    val type = c.getAnnotation(ResourceType.class);
+                    if (type == null) return;
 
-            if (map == null) {
-                continue;
-            }
+                    val map = GameData.getMapByResourceDef(c);
+                    if (map == null) return;
 
-            try {
-                loadFromResource(resourceDefinition, type, map, doReload);
-            } catch (Exception e) {
-                Grasscutter.getLogger().error("Error loading resource file: " + Arrays.toString(type.name()), e.getLocalizedMessage());
-            }
-        }
+                    try {
+                        loadFromResource(c, type, map, doReload);
+                    } catch (Exception e) {
+                        errors.add(Pair.of(Arrays.toString(type.name()), e));
+                    }
+                });
+        });
+        errors.forEach(pair -> Grasscutter.getLogger().error("Error loading resource file: " + pair.left(), pair.right()));
+        long endTime = System.nanoTime();
+        long ns = (endTime - startTime);  //divide by 1000000 to get milliseconds.
+        Grasscutter.getLogger().debug("Loading resources took "+ns+"ns == "+ns/1000000+"ms");
     }
 
     @SuppressWarnings("rawtypes")
     protected static void loadFromResource(Class<?> c, ResourceType type, Int2ObjectMap map, boolean doReload) throws Exception {
-        if (!loadedResources.contains(c.getSimpleName()) || doReload) {
+        val simpleName = c.getSimpleName();
+        if (doReload || !loadedResources.contains(simpleName)) {
             for (String name : type.name()) {
-                loadFromResource(c, name, map);
+                loadFromResource(c, FileUtils.getExcelPath(name), map);
             }
-            loadedResources.add(c.getSimpleName());
-            Grasscutter.getLogger().debug("Loaded " + map.size() + " " + c.getSimpleName() + "s.");
+            loadedResources.add(simpleName);
         }
     }
 
     @SuppressWarnings({"rawtypes", "unchecked"})
-    protected static <T> void loadFromResource(Class<T> c, String fileName, Int2ObjectMap map) throws Exception {
-        List<T> list = JsonUtils.loadToList(getResourcePath("ExcelBinOutput/" + fileName), c);
+    protected static <T> void loadFromResource(Class<T> c, Path filename, Int2ObjectMap map) throws Exception {
+        val results = switch (FileUtils.getFileExtension(filename)) {
+            case "json" -> JsonUtils.loadToList(filename, c);
+            case "tsj" -> TsvUtils.loadTsjToListSetField(c, filename);
+            case "tsv" -> TsvUtils.loadTsvToListSetField(c, filename);
+            default -> null;
+        };
+        if (results == null) return;
+        results.forEach(o -> {
+            GameResource res = (GameResource) o;
+            res.onLoad();
+            map.put(res.getId(), res);
+        });
+    }
 
-        for (T o : list) {
+    @SuppressWarnings({"rawtypes", "unchecked"})
+    protected static <T> void loadFromResource(Class<T> c, String fileName, Int2ObjectMap map) throws Exception {
+        JsonUtils.loadToList(getResourcePath("ExcelBinOutput/" + fileName), c).forEach(o -> {
             GameResource res = (GameResource) o;
             res.onLoad();
             map.put(res.getId(), res);
-        }
+        });
     }
 
     public class ScenePointConfig {  // Sadly this doesn't work as a local class in loadScenePoints()

src/main/java/emu/grasscutter/data/ResourceType.java

@@ -2,6 +2,8 @@ package emu.grasscutter.data;
 
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
+import java.util.List;
+import java.util.stream.Stream;
 
 @Retention(RetentionPolicy.RUNTIME) 
 public @interface ResourceType {
@@ -28,5 +30,9 @@ public @interface ResourceType {
 		public int value() {
 			return value;
 		}
+
+        public static List<LoadPriority> getInOrder() {
+            return Stream.of(LoadPriority.values()).sorted((x, y) -> y.value() - x.value()).toList();
+        }
 	}
 }

src/main/java/emu/grasscutter/data/excels/ActivityWatcherData.java

@@ -21,7 +21,7 @@ public class ActivityWatcherData extends GameResource {
 
     @Override
     public void onLoad() {
-        triggerConfig.paramList = triggerConfig.paramList.stream().filter(x -> !x.isBlank()).toList();
+        triggerConfig.paramList = triggerConfig.paramList.stream().filter(x -> (x != null) && !x.isBlank()).toList();
         triggerConfig.watcherTriggerType = WatcherTriggerType.getTypeByName(triggerConfig.triggerType);
     }
 

src/main/java/emu/grasscutter/data/excels/BattlePassMissionData.java

@@ -40,8 +40,11 @@ public class BattlePassMissionData extends GameResource {
 
     @Override
     public void onLoad() {
-        if (this.getTriggerConfig() != null && getTriggerConfig().getParamList()[0].length() > 0) {
-            this.mainParams = Arrays.stream(getTriggerConfig().getParamList()[0].split("[:;,]")).map(Integer::parseInt).collect(Collectors.toSet());
+        if (this.getTriggerConfig() != null) {
+            var params = getTriggerConfig().getParamList()[0];
+            if ((params != null) && !params.isEmpty()) {
+                this.mainParams = Arrays.stream(params.split("[:;,]")).map(Integer::parseInt).collect(Collectors.toSet());
+            }
         }
     }
 

src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddExp.java

 package emu.grasscutter.game.props.ItemUseAction;
 
 import emu.grasscutter.game.props.ItemUseOp;
-import lombok.Getter;
-
-public class ItemUseAddExp extends ItemUseAction {
-    @Getter private int exp = 0;
 
+public class ItemUseAddExp extends ItemUseInt {
     @Override
     public ItemUseOp getItemUseOp() {
         return ItemUseOp.ITEM_USE_ADD_EXP;
     }
 
     public ItemUseAddExp(String[] useParam) {
-        try {
-            this.exp = Integer.parseInt(useParam[0]);
-        } catch (NumberFormatException ignored) {}
+        super(useParam);
+    }
+
+    public int getExp() {
+        return this.i;
     }
 }

src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddItem.java

@@ -14,7 +14,7 @@ public class ItemUseAddItem extends ItemUseInt {
         super(useParam);
         try {
             this.count = Integer.parseInt(useParam[1]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
     }
 
     @Override

src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddReliquaryExp.java

 package emu.grasscutter.game.props.ItemUseAction;
 
 import emu.grasscutter.game.props.ItemUseOp;
-import lombok.Getter;
-
-public class ItemUseAddReliquaryExp extends ItemUseAction {
-    @Getter private int exp = 0;
 
+public class ItemUseAddReliquaryExp extends ItemUseAddExp {
     @Override
     public ItemUseOp getItemUseOp() {
         return ItemUseOp.ITEM_USE_ADD_RELIQUARY_EXP;
     }
 
     public ItemUseAddReliquaryExp(String[] useParam) {
-        try {
-            this.exp = Integer.parseInt(useParam[0]);
-        } catch (NumberFormatException ignored) {}
+        super(useParam);
     }
 }

src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddServerBuff.java

@@ -14,7 +14,7 @@ public class ItemUseAddServerBuff extends ItemUseInt {
         super(useParam);
         try {
             this.duration = Integer.parseInt(useParam[1]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
     }
 
     @Override

src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddWeaponExp.java

 package emu.grasscutter.game.props.ItemUseAction;
 
 import emu.grasscutter.game.props.ItemUseOp;
-import lombok.Getter;
-
-public class ItemUseAddWeaponExp extends ItemUseAction {
-    @Getter private int exp = 0;
 
+public class ItemUseAddWeaponExp extends ItemUseAddExp {
     @Override
     public ItemUseOp getItemUseOp() {
         return ItemUseOp.ITEM_USE_ADD_WEAPON_EXP;
     }
 
     public ItemUseAddWeaponExp(String[] useParam) {
-        try {
-            this.exp = Integer.parseInt(useParam[0]);
-        } catch (NumberFormatException ignored) {}
+        super(useParam);
     }
 }

src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseCombineItem.java

@@ -15,10 +15,10 @@ public class ItemUseCombineItem extends ItemUseInt {
         super(useParam);
         try {
             this.resultId = Integer.parseInt(useParam[1]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
         try {
             this.resultCount = Integer.parseInt(useParam[2]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
     }
 
     @Override

src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseGainAvatar.java

@@ -19,10 +19,10 @@ public class ItemUseGainAvatar extends ItemUseInt {
         super(useParam);
         try {
             this.level = Integer.parseInt(useParam[1]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
         try {
             this.constellation = Integer.parseInt(useParam[2]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
     }
 
     @Override

src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseInt.java

@@ -8,6 +8,6 @@ public abstract class ItemUseInt extends ItemUseAction {
     public ItemUseInt(String[] useParam) {
         try {
             this.i = Integer.parseInt(useParam[0]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
     }
 }

src/main/java/emu/grasscutter/utils/FileUtils.java

 package emu.grasscutter.utils;
 
 import emu.grasscutter.Grasscutter;
+import lombok.val;
 
 import java.io.File;
 import java.io.IOException;
@@ -111,6 +112,24 @@ public final class FileUtils {
         return RESOURCES_PATH.resolve(path);
     }
 
+    public static Path getExcelPath(String filename) {
+        return getTsjJsonTsv(RESOURCES_PATH.resolve("ExcelBinOutput"), filename);
+    }
+
+    // Gets path of a resource.
+    // If multiple formats of it exist, priority is TSJ > JSON > TSV
+    // If none exist, return the TSJ path, in case it wants to create a file
+    public static Path getTsjJsonTsv(Path root, String filename) {
+        val name = getFilenameWithoutExtension(filename);
+        val tsj = root.resolve(name + ".tsj");
+        if (Files.exists(tsj)) return tsj;
+        val json = root.resolve(name + ".json");
+        if (Files.exists(json)) return json;
+        val tsv = root.resolve(name + ".tsv");
+        if (Files.exists(tsv)) return tsv;
+        return tsj;
+    }
+
     public static Path getScriptPath(String path) {
         return SCRIPTS_PATH.resolve(path);
     }
@@ -167,14 +186,19 @@ public final class FileUtils {
         }
     }
 
-    @Deprecated  // No current uses of this anyway
-    public static String getFilenameWithoutPath(String fileName) {
-        int i = fileName.lastIndexOf(".");
-        if (i > 0) {
-           return fileName.substring(0, i);
-        } else {
-           return fileName;
-        }
+    @Deprecated  // Misnamed legacy function
+    public static String getFilenameWithoutPath(String filename) {
+        return getFilenameWithoutExtension(filename);
+    }
+    public static String getFilenameWithoutExtension(String filename) {
+        int i = filename.lastIndexOf(".");
+        return (i < 0) ? filename : filename.substring(0, i);
+    }
+
+    public static String getFileExtension(Path path) {
+        val filename = path.toString();
+        int i = filename.lastIndexOf(".");
+        return (i < 0) ? "" : filename.substring(i+1);
     }
 
     public static List<Path> getPathsFromResource(String folder) throws URISyntaxException {

src/main/java/emu/grasscutter/utils/JsonUtils.java

@@ -4,6 +4,7 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.lang.reflect.Type;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -18,6 +19,7 @@ import com.google.gson.reflect.TypeToken;
 
 import emu.grasscutter.data.common.DynamicFloat;
 import emu.grasscutter.utils.JsonAdapters.*;
+
 import it.unimi.dsi.fastutil.ints.IntList;
 
 public final class JsonUtils {
@@ -102,4 +104,12 @@ public final class JsonUtils {
             return null;
         }
     }
+
+    public static <T> T decode(String jsonData, Type type) {
+        try {
+            return gson.fromJson(jsonData, type);
+        } catch (Exception ignored) {
+            return null;
+        }
+    }
 }

src/main/java/emu/grasscutter/utils/Utils.java

@@ -395,4 +395,22 @@ public final class Utils {
     public static <T> T drawRandomListElement(List<T> list) {
         return drawRandomListElement(list, null);
     }
+
+    /***
+     * Splits a string by a character, into a list
+     * @param input The string to split
+     * @param separator The character to use as the split points
+     * @return A list of all the substrings
+     */
+    public static List<String> nonRegexSplit(String input, int separator) {
+        var output = new ArrayList<String>();
+        int start = 0;
+        for (int next = input.indexOf(separator); next > 0; next = input.indexOf(separator, start)) {
+            output.add(input.substring(start, next));
+            start = next + 1;
+        }
+        if (start < input.length())
+            output.add(input.substring(start));
+        return output;
+    }
 }