"Autogenerate" data files with data fallbacks and moved keys folder into jar resources (#927)

* Autogenerate keys and data files

* Update gacha html files

Accidentally pushed with old html files

* Keys no longer copied. No more manually retrieving listing files. Recursive directory creation

Removed unused code from old GC as well.

* Moved somethings and better errors

* Fixed resources from loading twice

* Data files fallback

.gitignore

@@ -52,21 +52,23 @@ tmp/
 .vscode
 
 # Grasscutter
-resources/
-logs/
-plugins/
-data/AbilityEmbryos.json
-data/OpenConfig.json
+/resources
+/logs
+/plugins
+/data
+/keys
+/language
+/languages
+/src/generated
+
+/*.jar
+/*.sh
+
 GM Handbook.txt
 config.json
 mitmdump.exe
-*.jar
-!lib/*.jar
 mongod.exe
-/src/generated/
-/*.sh
-language/
-languages/
+
 gacha-mapping.js
 mappings.js
 BuildConfig.java

src/main/java/emu/grasscutter/Configuration.java

@@ -28,7 +28,6 @@ public final class Configuration extends ConfigContainer {
     public static final Locale FALLBACK_LANGUAGE = config.language.fallback;
     private static final String DATA_FOLDER = config.folderStructure.data;
     private static final String RESOURCES_FOLDER = config.folderStructure.resources;
-    private static final String KEYS_FOLDER = config.folderStructure.keys;
     private static final String PLUGINS_FOLDER = config.folderStructure.plugins;
     private static final String SCRIPTS_FOLDER = config.folderStructure.scripts;
     private static final String PACKETS_FOLDER = config.folderStructure.packets;
@@ -62,10 +61,6 @@ public final class Configuration extends ConfigContainer {
     public static String RESOURCE(String path) {
         return Paths.get(RESOURCES_FOLDER, path).toString();
     }
-
-    public static String KEY(String path) {
-        return Paths.get(KEYS_FOLDER, path).toString();
-    }
     
     public static String PLUGIN() {
         return PLUGINS_FOLDER;

src/main/java/emu/grasscutter/data/DataLoader.java

+package emu.grasscutter.data;
+
+import emu.grasscutter.Grasscutter;
+import emu.grasscutter.server.http.handlers.GachaHandler;
+import emu.grasscutter.tools.Tools;
+import emu.grasscutter.utils.FileUtils;
+import emu.grasscutter.utils.Utils;
+
+import java.io.*;
+import java.nio.file.Path;
+import java.util.List;
+
+import static emu.grasscutter.Configuration.DATA;
+
+public class DataLoader {
+
+    /**
+     * Load a data file by its name. If the file isn't found within the /data directory then it will fallback to the default within the jar resources
+     * @see #load(String, boolean)
+     * @param resourcePath The path to the data file to be loaded.
+     * @return InputStream of the data file.
+     * @throws FileNotFoundException
+     */
+    public static InputStream load(String resourcePath) throws FileNotFoundException {
+        return load(resourcePath, true);
+    }
+
+    /**
+     * Load a data file by its name.
+     * @param resourcePath The path to the data file to be loaded.
+     * @param useFallback If the file does not exist in the /data directory, should it use the default file in the jar?
+     * @return InputStream of the data file.
+     * @throws FileNotFoundException
+     */
+    public static InputStream load(String resourcePath, boolean useFallback) throws FileNotFoundException {
+        if(Utils.fileExists(DATA(resourcePath))) {
+            // Data is in the resource directory
+            return new FileInputStream(DATA(resourcePath));
+        } else {
+            if(useFallback) {
+                return FileUtils.readResourceAsStream("/defaults/data/" + resourcePath);
+            }
+        }
+
+        return null;
+    }
+
+    public static void CheckAllFiles() {
+
+        try {
+            List<Path> filenames = FileUtils.getPathsFromResource("/defaults/data/");
+
+            for (Path file : filenames) {
+                String relativePath = String.valueOf(file).split("/defaults/data/")[1];
+
+                CheckAndCopyData(relativePath);
+            }
+        } catch (Exception e) {
+            Grasscutter.getLogger().error("An error occurred while trying to check the data folder. \n" + e);
+        }
+
+        GenerateGachaMappings();
+    }
+
+    private static void CheckAndCopyData(String name) {
+        String filePath = Utils.toFilePath(DATA(name));
+
+        if (!Utils.fileExists(filePath)) {
+            // Check if file is in subdirectory
+            if (name.indexOf("/") != -1) {
+                String[] path = name.split("/");
+
+                String folder = "";
+                for(int i = 0; i < (path.length - 1); i++) {
+                    folder += path[i] + "/";
+
+                    // Make sure the current folder exists
+                    String folderToCreate = Utils.toFilePath(DATA(folder));
+                    if(!Utils.fileExists(folderToCreate)) {
+                        Grasscutter.getLogger().info("Creating data folder '" + folder + "'");
+                        Utils.createFolder(folderToCreate);
+                    }
+                }
+            }
+
+            Grasscutter.getLogger().info("Creating default '" + name + "' data");
+            FileUtils.copyResource("/defaults/data/" + name, filePath);
+        }
+    }
+
+    private static void GenerateGachaMappings() {
+        if (!Utils.fileExists(GachaHandler.gachaMappings)) {
+            try {
+                Grasscutter.getLogger().info("Creating default '" + GachaHandler.gachaMappings + "' data");
+                Tools.createGachaMapping(GachaHandler.gachaMappings);
+            } catch (Exception exception) {
+                Grasscutter.getLogger().warn("Failed to create gacha mappings. \n" + exception);
+            }
+        }
+    }
+}

src/main/java/emu/grasscutter/data/ResourceLoader.java

 package emu.grasscutter.data;
 
-import java.io.File;
-import java.io.FileReader;
+import java.io.*;
 import java.util.*;
 import java.util.Map.Entry;
 import java.util.regex.Matcher;
@@ -33,6 +32,8 @@ import static emu.grasscutter.Configuration.*;
 
 public class ResourceLoader {
 
+	private static List<String> loadedResources = new ArrayList<String>();
+
 	public static List<Class<?>> getResourceDefClasses() {
 		Reflections reflections = new Reflections(ResourceLoader.class.getPackage().getName());
 		Set<?> classes = reflections.getSubTypesOf(GameResource.class);
@@ -98,6 +99,10 @@ public class ResourceLoader {
 	}
 
 	public static void loadResources() {
+		loadResources(false);
+	}
+
+	public static void loadResources(boolean doReload) {
 		for (Class<?> resourceDefinition : getResourceDefClasses()) {
 			ResourceType type = resourceDefinition.getAnnotation(ResourceType.class);
 
@@ -113,7 +118,7 @@ public class ResourceLoader {
 			}
 
 			try {
-				loadFromResource(resourceDefinition, type, map);
+				loadFromResource(resourceDefinition, type, map, doReload);
 			} catch (Exception e) {
 				Grasscutter.getLogger().error("Error loading resource file: " + Arrays.toString(type.name()), e);
 			}
@@ -121,13 +126,16 @@ public class ResourceLoader {
 	}
 	
 	@SuppressWarnings("rawtypes")
-	protected static void loadFromResource(Class<?> c, ResourceType type, Int2ObjectMap map) throws Exception {
-		for (String name : type.name()) {
-			loadFromResource(c, name, map);
+	protected static void loadFromResource(Class<?> c, ResourceType type, Int2ObjectMap map, boolean doReload) throws Exception {
+		if(!loadedResources.contains(c.getSimpleName()) || doReload) {
+			for (String name : type.name()) {
+				loadFromResource(c, name, map);
+			}
+			Grasscutter.getLogger().info("Loaded " + map.size() + " " + c.getSimpleName() + "s.");
+			loadedResources.add(c.getSimpleName());
 		}
-		Grasscutter.getLogger().info("Loaded " + map.size() + " " + c.getSimpleName() + "s.");
 	}
-	
+
 	@SuppressWarnings({"rawtypes", "unchecked"})
 	protected static void loadFromResource(Class<?> c, String fileName, Int2ObjectMap map) throws Exception {
 		FileReader fileReader = new FileReader(RESOURCE("ExcelBinOutput/" + fileName));
@@ -138,6 +146,9 @@ public class ResourceLoader {
 			Map<String, Object> tempMap = Utils.switchPropertiesUpperLowerCase((Map<String, Object>) o, c);
 			GameResource res = gson.fromJson(gson.toJson(tempMap), TypeToken.get(c).getType());
 			res.onLoad();
+			if(map.containsKey(res.getId())) {
+				map.remove(res.getId());
+			}
 			map.put(res.getId(), res);
 		}
 	}
@@ -191,18 +202,14 @@ public class ResourceLoader {
 	}
 
 	private static void loadAbilityEmbryos() {
-		// Read from cached file if exists
-		File embryoCache = new File(DATA("AbilityEmbryos.json"));
 		List<AbilityEmbryoEntry> embryoList = null;
-		
-		if (embryoCache.exists()) {
-			// Load from cache
-			try (FileReader fileReader = new FileReader(embryoCache)) {
-				embryoList = Grasscutter.getGsonFactory().fromJson(fileReader, TypeToken.getParameterized(Collection.class, AbilityEmbryoEntry.class).getType());
-			} catch (Exception e) {
-				e.printStackTrace();
-			}
-		} else {
+
+		// Read from cached file if exists
+		try(InputStream embryoCache = DataLoader.load("AbilityEmbryos.json", false)) {
+			embryoList = Grasscutter.getGsonFactory().fromJson(new InputStreamReader(embryoCache), TypeToken.getParameterized(Collection.class, AbilityEmbryoEntry.class).getType());
+		} catch(Exception ignored) {}
+
+		if(embryoList == null) {
 			// Load from BinOutput
 			Pattern pattern = Pattern.compile("(?<=ConfigAvatar_)(.*?)(?=.json)");
 
@@ -316,18 +323,12 @@ public class ResourceLoader {
 	}
 	
 	private static void loadSpawnData() {
-		// Read from cached file if exists
-		File spawnDataEntries = new File(DATA("Spawns.json"));
 		List<SpawnGroupEntry> spawnEntryList = null;
-		
-		if (spawnDataEntries.exists()) {
-			// Load from cache
-			try (FileReader fileReader = new FileReader(spawnDataEntries)) {
-				spawnEntryList = Grasscutter.getGsonFactory().fromJson(fileReader, TypeToken.getParameterized(Collection.class, SpawnGroupEntry.class).getType());
-			} catch (Exception e) {
-				e.printStackTrace();
-			}
-		}
+
+		// Read from cached file if exists
+		try(InputStream spawnDataEntries = DataLoader.load("Spawns.json")) {
+			spawnEntryList = Grasscutter.getGsonFactory().fromJson(new InputStreamReader(spawnDataEntries), TypeToken.getParameterized(Collection.class, SpawnGroupEntry.class).getType());
+		} catch (Exception ignored) {}
 		
 		if (spawnEntryList == null || spawnEntryList.isEmpty()) {
 			Grasscutter.getLogger().error("No spawn data loaded!");
@@ -342,16 +343,13 @@ public class ResourceLoader {
 	
 	private static void loadOpenConfig() {
 		// Read from cached file if exists
-		File openConfigCache = new File(DATA("OpenConfig.json"));
 		List<OpenConfigEntry> list = null;
-		
-		if (openConfigCache.exists()) {
-			try (FileReader fileReader = new FileReader(openConfigCache)) {
-				list = Grasscutter.getGsonFactory().fromJson(fileReader, TypeToken.getParameterized(Collection.class, OpenConfigEntry.class).getType());
-			} catch (Exception e) {
-				e.printStackTrace();
-			}
-		} else {
+
+		try(InputStream openConfigCache = DataLoader.load("OpenConfig.json", false)) {
+			list = Grasscutter.getGsonFactory().fromJson(new InputStreamReader(openConfigCache), TypeToken.getParameterized(Collection.class, SpawnGroupEntry.class).getType());
+		} catch (Exception ignored) {}
+
+		if (list == null) {
 			Map<String, OpenConfigEntry> map = new TreeMap<>();
 			java.lang.reflect.Type type = new TypeToken<Map<String, OpenConfigData[]>>() {}.getType();
 			String[] folderNames = {"BinOutput/Talent/EquipTalents/", "BinOutput/Talent/AvatarTalents/"};

src/main/java/emu/grasscutter/game/drop/DropManager.java

@@ -2,6 +2,7 @@ package emu.grasscutter.game.drop;
 
 import com.google.gson.reflect.TypeToken;
 import emu.grasscutter.Grasscutter;
+import emu.grasscutter.data.DataLoader;
 import emu.grasscutter.data.GameData;
 import emu.grasscutter.data.def.ItemData;
 import emu.grasscutter.game.entity.EntityItem;
@@ -17,12 +18,11 @@ import emu.grasscutter.utils.Utils;
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 
-import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.util.Collection;
 import java.util.List;
 
-import static emu.grasscutter.Configuration.*;
-
 public class DropManager {
     public GameServer getGameServer() {
         return gameServer;
@@ -43,7 +43,7 @@ public class DropManager {
     }
 
     public synchronized void load() {
-        try (FileReader fileReader = new FileReader(DATA("Drop.json"))) {
+        try (Reader fileReader = new InputStreamReader(DataLoader.load("Drop.json"))) {
             getDropData().clear();
             List<DropInfo> banners = Grasscutter.getGsonFactory().fromJson(fileReader, TypeToken.getParameterized(Collection.class, DropInfo.class).getType());
             if(banners.size() > 0) {

src/main/java/emu/grasscutter/game/expedition/ExpeditionManager.java

@@ -2,11 +2,14 @@ package emu.grasscutter.game.expedition;
 
 import com.google.gson.reflect.TypeToken;
 import emu.grasscutter.Grasscutter;
+import emu.grasscutter.data.DataLoader;
 import emu.grasscutter.server.game.GameServer;
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 
 import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.util.Collection;
 import java.util.List;
 
@@ -30,7 +33,7 @@ public class ExpeditionManager {
     }
 
     public synchronized void load() {
-        try (FileReader fileReader = new FileReader(DATA("ExpeditionReward.json"))) {
+        try (Reader fileReader = new InputStreamReader(DataLoader.load("ExpeditionReward.json"))) {
             getExpeditionRewardDataList().clear();
             List<ExpeditionRewardInfo> banners = Grasscutter.getGsonFactory().fromJson(fileReader, TypeToken.getParameterized(Collection.class, ExpeditionRewardInfo.class).getType());
             if(banners.size() > 0) {

src/main/java/emu/grasscutter/game/gacha/GachaManager.java

@@ -2,6 +2,8 @@ package emu.grasscutter.game.gacha;
 
 import java.io.File;
 import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.nio.file.*;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -13,6 +15,7 @@ import com.google.gson.reflect.TypeToken;
 
 import com.sun.nio.file.SensitivityWatchEventModifier;
 import emu.grasscutter.Grasscutter;
+import emu.grasscutter.data.DataLoader;
 import emu.grasscutter.data.GameData;
 import emu.grasscutter.data.common.ItemParamData;
 import emu.grasscutter.data.def.ItemData;
@@ -74,7 +77,7 @@ public class GachaManager {
 	}
 	
 	public synchronized void load() {
-		try (FileReader fileReader = new FileReader(DATA("Banners.json"))) {
+		try (Reader fileReader = new InputStreamReader(DataLoader.load("Banners.json"))) {
 			getGachaBanners().clear();
 			List<GachaBanner> banners = Grasscutter.getGsonFactory().fromJson(fileReader, TypeToken.getParameterized(Collection.class, GachaBanner.class).getType());
 			if(banners.size() > 0) {

src/main/java/emu/grasscutter/game/shop/ShopManager.java

@@ -2,6 +2,7 @@ package emu.grasscutter.game.shop;
 
 import com.google.gson.reflect.TypeToken;
 import emu.grasscutter.Grasscutter;
+import emu.grasscutter.data.DataLoader;
 import emu.grasscutter.data.GameData;
 import emu.grasscutter.data.common.ItemParamData;
 import emu.grasscutter.data.def.ShopGoodsData;
@@ -11,6 +12,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 
 import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
@@ -58,7 +61,7 @@ public class ShopManager {
 	}
 
 	private void loadShop() {
-		try (FileReader fileReader = new FileReader(DATA("Shop.json"))) {
+		try (Reader fileReader = new InputStreamReader(DataLoader.load("Shop.json"))) {
 			getShopData().clear();
 			List<ShopTable> banners = Grasscutter.getGsonFactory().fromJson(fileReader, TypeToken.getParameterized(Collection.class, ShopTable.class).getType());
 			if(banners.size() > 0) {
@@ -102,7 +105,7 @@ public class ShopManager {
 	}
 
 	private void loadShopChest() {
-		try (FileReader fileReader = new FileReader(DATA("ShopChest.json"))) {
+		try (Reader fileReader = new InputStreamReader(DataLoader.load("ShopChest.json"))) {
 			getShopChestData().clear();
 			List<ShopChestTable> shopChestTableList = Grasscutter.getGsonFactory().fromJson(fileReader, TypeToken.getParameterized(Collection.class, ShopChestTable.class).getType());
 			if (shopChestTableList.size() > 0) {
@@ -117,7 +120,7 @@ public class ShopManager {
 	}
 
 	private void loadShopChestBatchUse() {
-		try (FileReader fileReader = new FileReader(DATA("ShopChestBatchUse.json"))) {
+		try (Reader fileReader = new InputStreamReader(DataLoader.load("ShopChestBatchUse.json"))) {
 			getShopChestBatchUseData().clear();
 			List<ShopChestBatchUseTable> shopChestBatchUseTableList = Grasscutter.getGsonFactory().fromJson(fileReader, TypeToken.getParameterized(Collection.class, ShopChestBatchUseTable.class).getType());
 			if (shopChestBatchUseTableList.size() > 0) {

src/main/java/emu/grasscutter/game/tower/TowerScheduleManager.java

 package emu.grasscutter.game.tower;
 
 import emu.grasscutter.Grasscutter;
+import emu.grasscutter.data.DataLoader;
 import emu.grasscutter.data.GameData;
 import emu.grasscutter.data.def.TowerScheduleData;
 import emu.grasscutter.server.game.GameServer;
 
 import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.util.List;
 
 import static emu.grasscutter.Configuration.*;
@@ -25,7 +28,7 @@ public class TowerScheduleManager {
     private TowerScheduleConfig towerScheduleConfig;
 
     public synchronized void load(){
-        try (FileReader fileReader = new FileReader(DATA("TowerSchedule.json"))) {
+        try (Reader fileReader = new InputStreamReader(DataLoader.load("TowerSchedule.json"))) {
             towerScheduleConfig = Grasscutter.getGsonFactory().fromJson(fileReader, TowerScheduleConfig.class);
         } catch (Exception e) {
             Grasscutter.getLogger().error("Unable to load tower schedule config.", e);

src/main/java/emu/grasscutter/server/http/handlers/AnnouncementsHandler.java

 package emu.grasscutter.server.http.handlers;
 
 import emu.grasscutter.Grasscutter;
+import emu.grasscutter.data.DataLoader;
 import emu.grasscutter.server.http.objects.HttpJsonResponse;
 import emu.grasscutter.server.http.Router;
 import emu.grasscutter.utils.FileUtils;
@@ -14,6 +15,7 @@ import io.javalin.Javalin;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 
@@ -41,9 +43,21 @@ public final class AnnouncementsHandler implements Router {
     private static void getAnnouncement(Request request, Response response) {
         String data = "";
         if (Objects.equals(request.baseUrl(), "/common/hk4e_global/announcement/api/getAnnContent")) {
-            data = readToString(new File(Utils.toFilePath(DATA("GameAnnouncement.json"))));
+            try {
+                data = FileUtils.readToString(DataLoader.load("GameAnnouncement.json"));
+            } catch (Exception e) {
+                if(e.getClass() == IOException.class) {
+                    Grasscutter.getLogger().info("Unable to read file 'GameAnnouncementList.json'. \n" + e);
+                }
+            }
         } else if (Objects.equals(request.baseUrl(), "/common/hk4e_global/announcement/api/getAnnList")) {
-            data = readToString(new File(Utils.toFilePath(DATA("GameAnnouncementList.json"))));
+            try {
+                data = FileUtils.readToString(DataLoader.load("GameAnnouncementList.json"));
+            } catch (Exception e) {
+                if(e.getClass() == IOException.class) {
+                    Grasscutter.getLogger().info("Unable to read file 'GameAnnouncementList.json'. \n" + e);
+                }
+            }
         } else {
             response.send("{\"retcode\":404,\"message\":\"Unknown request path\"}");
         }
@@ -64,29 +78,15 @@ public final class AnnouncementsHandler implements Router {
     }
     
     private static void getPageResources(Request request, Response response) {
-        String filename = Utils.toFilePath(DATA(request.path()));
-        File file = new File(filename);
-        if (file.exists() && file.isFile()) {
-            MediaType fromExtension = MediaType.getByExtension(filename.substring(filename.lastIndexOf(".") + 1));
+        try(InputStream filestream = DataLoader.load(request.path())) {
+            String possibleFilename = Utils.toFilePath(DATA(request.path()));
+
+            MediaType fromExtension = MediaType.getByExtension(possibleFilename.substring(possibleFilename.lastIndexOf(".") + 1));
             response.type((fromExtension != null) ? fromExtension.getMIME() : "application/octet-stream");
-            response.send(FileUtils.read(file));
-        } else {
-            Grasscutter.getLogger().warn("File does not exist: " + file);
+            response.send(filestream.readAllBytes());
+        } catch (Exception e) {
+            Grasscutter.getLogger().warn("File does not exist: " + request.path());
             response.status(404);
         }
     }
-
-    @SuppressWarnings("ResultOfMethodCallIgnored")
-    private static String readToString(File file) {
-        byte[] content = new byte[(int) file.length()];
-        
-        try {
-            FileInputStream in = new FileInputStream(file);
-            in.read(content); in.close();
-        } catch (IOException ignored) {
-            Grasscutter.getLogger().warn("File does not exist: " + file);
-        }
-
-        return new String(content, StandardCharsets.UTF_8);
-    }
 }

src/main/java/emu/grasscutter/server/http/handlers/GachaHandler.java

@@ -29,18 +29,7 @@ import static emu.grasscutter.utils.Language.translate;
  * Handles all gacha-related HTTP requests.
  */
 public final class GachaHandler implements Router {
-    private final String gachaMappings;
-    
-    public GachaHandler() {
-        this.gachaMappings = Utils.toFilePath(DATA("gacha/mappings.js"));
-        if(!(new File(this.gachaMappings).exists())) {
-            try {
-                Tools.createGachaMapping(this.gachaMappings);
-            } catch (Exception exception) {
-                Grasscutter.getLogger().warn("Failed to create gacha mappings.", exception);
-            }
-        }
-    }
+    public static final String gachaMappings = DATA(Utils.toFilePath("gacha/mappings.js"));
     
     @Override public void applyRoutes(Express express, Javalin handle) {
         express.get("/gacha", GachaHandler::gachaRecords);

src/main/java/emu/grasscutter/utils/ConfigContainer.java

@@ -84,7 +84,6 @@ public class ConfigContainer {
         public String resources = "./resources/";
         public String data = "./data/";
         public String packets = "./packets/";
-        public String keys = "./keys/";
         public String scripts = "./resources/scripts/";
         public String plugins = "./plugins/";
 

src/main/java/emu/grasscutter/utils/Crypto.java

@@ -20,11 +20,11 @@ public final class Crypto {
 	public static byte[] ENCRYPT_SEED_BUFFER = new byte[0];
 	
 	public static void loadKeys() {
-		DISPATCH_KEY = FileUtils.read(KEY("dispatchKey.bin"));
-		DISPATCH_SEED = FileUtils.read(KEY("dispatchSeed.bin"));
+		DISPATCH_KEY = FileUtils.readResource("/keys/dispatchKey.bin");
+		DISPATCH_SEED = FileUtils.readResource("/keys/dispatchSeed.bin");
 
-		ENCRYPT_KEY = FileUtils.read(KEY("secretKey.bin"));
-		ENCRYPT_SEED_BUFFER = FileUtils.read(KEY("secretKeyBuffer.bin"));
+		ENCRYPT_KEY = FileUtils.readResource("/keys/secretKey.bin");
+		ENCRYPT_SEED_BUFFER = FileUtils.readResource("/keys/secretKeyBuffer.bin");
 	}
 	
 	public static void xor(byte[] packet, byte[] key) {
@@ -37,25 +37,6 @@ public final class Crypto {
 		}
 	}
 	
-	public static void extractSecretKeyBuffer(byte[] data) {
-		try {
-			GetPlayerTokenRsp p = GetPlayerTokenRsp.parseFrom(data);
-			FileUtils.write(KEY("/secretKeyBuffer.bin"), p.getSecretKeyBytes().toByteArray());
-			Grasscutter.getLogger().info("Secret Key: " + p.getSecretKey());
-		} catch (Exception e) {
-			Grasscutter.getLogger().error("Crypto error.", e);
-		}
-	}
-	
-	public static void extractDispatchSeed(String data) {
-		try {
-			QueryCurrRegionHttpRsp p = QueryCurrRegionHttpRsp.parseFrom(Base64.getDecoder().decode(data));
-			FileUtils.write(KEY("/dispatchSeed.bin"), p.getRegionInfo().getSecretKey().toByteArray());
-		} catch (Exception e) {
-			Grasscutter.getLogger().error("Crypto error.", e);
-		}
-	}
-	
 	public static byte[] createSessionKey(int length) {
 		byte[] bytes = new byte[length];
 		secureRandom.nextBytes(bytes);

src/main/java/emu/grasscutter/utils/FileUtils.java

@@ -4,9 +4,14 @@ import emu.grasscutter.Grasscutter;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.*;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
 
 public final class FileUtils {
 	public static void write(String dest, byte[] bytes) {
@@ -32,10 +37,34 @@ public final class FileUtils {
 		
 		return new byte[0];
 	}
+
+	public static InputStream readResourceAsStream(String resourcePath) {
+		return Grasscutter.class.getResourceAsStream(resourcePath);
+	}
+
+	public static byte[] readResource(String resourcePath) {
+		try (InputStream is = Grasscutter.class.getResourceAsStream(resourcePath)) {
+			return is.readAllBytes();
+		} catch (Exception exception) {
+			Grasscutter.getLogger().warn("Failed to read resource: " + resourcePath);
+			exception.printStackTrace();
+		}
+
+		return new byte[0];
+	}
 	
 	public static byte[] read(File file) {
 		return read(file.getPath());
 	}
+
+	public static void copyResource(String resourcePath, String destination) {
+		try {
+			byte[] resource = FileUtils.readResource(resourcePath);
+			FileUtils.write(destination, resource);
+		} catch (Exception exception) {
+			Grasscutter.getLogger().warn("Failed to copy resource: " + resourcePath + "\n" + exception);
+		}
+	}
 	
 	public static String getFilenameWithoutPath(String fileName) {
 		if (fileName.indexOf(".") > 0) {
@@ -44,4 +73,33 @@ public final class FileUtils {
 		   return fileName;
 		}
 	}
+
+	// From https://mkyong.com/java/java-read-a-file-from-resources-folder/
+	public static List<Path> getPathsFromResource(String folder) throws URISyntaxException, IOException {
+		List<Path> result;
+
+		// get path of the current running JAR
+		String jarPath = Grasscutter.class.getProtectionDomain()
+				.getCodeSource()
+				.getLocation()
+				.toURI()
+				.getPath();
+
+		// file walks JAR
+		URI uri = URI.create("jar:file:" + jarPath);
+		try (FileSystem fs = FileSystems.newFileSystem(uri, Collections.emptyMap())) {
+			result = Files.walk(fs.getPath(folder))
+					.filter(Files::isRegularFile)
+					.collect(Collectors.toList());
+		}
+
+		return result;
+	}
+
+	@SuppressWarnings("ResultOfMethodCallIgnored")
+	public static String readToString(InputStream file) throws IOException {
+		byte[] content = file.readAllBytes();
+
+		return new String(content, StandardCharsets.UTF_8);
+	}
 }

src/main/java/emu/grasscutter/utils/Utils.java

@@ -9,6 +9,7 @@ import java.time.temporal.TemporalAdjusters;
 import java.util.*;
 
 import emu.grasscutter.Grasscutter;
+import emu.grasscutter.data.DataLoader;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.ByteBufUtil;
 import io.netty.buffer.Unpooled;
@@ -198,6 +199,9 @@ public final class Utils {
 		if(!fileExists(dataFolder))
 			createFolder(dataFolder);
 
+		// Make sure the data folder is populated, if there are any missing files copy them from resources
+		DataLoader.CheckAllFiles();
+
 		if(exit) System.exit(1);
 	}