Merge pull request #1704 from Grasscutters/development

Merge `development` into `stable`

build.gradle

@@ -43,8 +43,7 @@ sourceCompatibility = JavaVersion.VERSION_17
 targetCompatibility = JavaVersion.VERSION_17
 
 group = 'xyz.grasscutters'
-version = '1.2.0'
-
+version = '1.3.1-dev'
 
 sourceCompatibility = 17
 targetCompatibility = 17
@@ -61,35 +60,48 @@ repositories {
 dependencies {
     implementation fileTree(dir: 'lib', include: ['*.jar'])
 
-    implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.32'
-    implementation group: 'ch.qos.logback', name: 'logback-core', version: '1.2.9'
-    implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.9'
+    implementation group: 'it.unimi.dsi', name: 'fastutil', version: '8.5.8'
+
+    implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.36'
+    implementation group: 'ch.qos.logback', name: 'logback-core', version: '1.2.11'
+    implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.11'
 
     implementation group: 'org.jline', name: 'jline', version: '3.21.0'
     implementation group: 'org.jline', name: 'jline-terminal-jna', version: '3.21.0'
     implementation group: 'net.java.dev.jna', name: 'jna', version: '5.10.0'
 
-    implementation group: 'io.netty', name: 'netty-all', version: '4.1.71.Final'
+    implementation group: 'io.netty', name: 'netty-common', version: '4.1.79.Final'
+    implementation group: 'io.netty', name: 'netty-handler', version: '4.1.79.Final'
+    implementation group: 'io.netty', name: 'netty-transport-native-epoll', version: '4.1.79.Final'
+    implementation group: 'io.netty', name: 'netty-transport-native-kqueue', version: '4.1.79.Final'
 
     implementation group: 'com.google.code.gson', name: 'gson', version: '2.9.0'
     implementation group: 'com.google.protobuf', name: 'protobuf-java', version: '3.18.2'
 
     implementation group: 'org.reflections', name: 'reflections', version: '0.10.2'
 
-    implementation group: 'dev.morphia.morphia', name: 'morphia-core', version: '2.2.6'
+    implementation group: 'dev.morphia.morphia', name: 'morphia-core', version: '2.2.7'
 
     implementation group: 'org.greenrobot', name: 'eventbus-java', version: '3.3.1'
-    implementation group: 'org.danilopianini', name: 'java-quadtree', version: '0.1.9'
+    //implementation group: 'org.danilopianini', name: 'java-quadtree', version: '0.1.9'
 
     implementation group: 'org.quartz-scheduler', name: 'quartz', version: '2.3.2'
     implementation group: 'org.quartz-scheduler', name: 'quartz-jobs', version: '2.3.2'
 
     implementation group: 'org.luaj', name: 'luaj-jse', version: '3.0.1'
 
+    implementation group:  'com.esotericsoftware', name : 'reflectasm', version: '1.11.9'
+    implementation group:  'com.github.davidmoten', name : 'rtree-multi', version: '0.1'
+
+    implementation group: 'io.javalin', name: 'javalin', version: '4.6.4'
+    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.3'
+
     protobuf files('proto/')
 
     compileOnly 'org.projectlombok:lombok:1.18.24'
     annotationProcessor 'org.projectlombok:lombok:1.18.24'
+    testCompileOnly 'org.projectlombok:lombok:1.18.24'
+    testAnnotationProcessor 'org.projectlombok:lombok:1.18.24'
 }
 
 configurations.all {
@@ -98,29 +110,33 @@ configurations.all {
 
 application {
     // Define the main class for the application
-    mainClassName = 'emu.grasscutter.Grasscutter'
+    getMainClass().set('emu.grasscutter.Grasscutter')
 }
 
 
 jar {
+    exclude '*.proto'
+
     manifest {
         attributes 'Main-Class': 'emu.grasscutter.Grasscutter'
     }
 
-    jar.baseName = 'grasscutter'
-    jar.archiveName = project.hasProperty('jarFilename') ? "${jarFilename}.${extension}" : archiveName
+    archiveBaseName = 'grasscutter'
+    if (project.hasProperty('jarFilename')) {
+        archiveFileName = "${jarFilename}.${extension}"
+    }
 
     from {
         configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
     }
-    
+
     duplicatesStrategy = DuplicatesStrategy.INCLUDE
 
     from('src/main/java') {
         include '*.xml'
     }
 
-    destinationDir = file(".")
+    destinationDirectory = file(".")
 }
 
 publishing {
@@ -150,12 +166,7 @@ publishing {
                     developer {
                         id = 'meledy'
                         name = 'Meledy'
-                        email = 'meledy@xigam.tech' // not a real email kek
-                    }
-                    developer {
-                        id = 'magix'
-                        name = 'Magix'
-                        email = 'magix@xigam.tech'
+                        email = 'meledy@grasscutter.io' // not a real email kek
                     }
                 }
                 scm {
@@ -168,13 +179,23 @@ publishing {
     }
     repositories {
         maven {
-            // change URLs to point to your repos, e.g. http://my.org/repo
-            def releasesRepoUrl = 'https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/'
-            def snapshotsRepoUrl = 'https://s01.oss.sonatype.org/content/repositories/snapshots/'
-            url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
-
-            name = 'sonatype'
-            credentials(PasswordCredentials)
+            if(version.endsWith('-dev')) {
+                println ("Publishing to 4benj-maven")
+                url 'https://repo.4benj.com/releases'
+                name '4benj-maven'
+                credentials {
+                    username System.getenv('benj_maven_username')
+                    password System.getenv('benj_maven_token')
+                }
+            } else {
+                println ("Publishing to sonatype")
+                def releasesRepoUrl = 'https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/'
+                def snapshotsRepoUrl = 'https://s01.oss.sonatype.org/content/repositories/snapshots/'
+                url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
+
+                name = 'sonatype'
+                credentials(PasswordCredentials)
+            }
         }
     }
 }
@@ -222,7 +243,9 @@ eclipse {
 }
 
 signing {
-    sign publishing.publications.mavenJava
+    if(!version.endsWith('-dev')) {
+        sign publishing.publications.mavenJava
+    }
 }
 
 javadoc {
@@ -236,17 +259,19 @@ task injectGitHash {
     def gitCommitHash = {
         try {
             return 'git rev-parse --verify --short HEAD'.execute().text.trim()
-        } catch (e) {
+        } catch (ignored) {
             return "GIT_NOT_FOUND"
         }
     }
-    new File(projectDir, "src/main/java/emu/grasscutter/BuildConfig.java").text = """
-        package emu.grasscutter;
-        public class BuildConfig {
-                public static final String VERSION = \"${version}\";
-                public static final String GIT_HASH = \"${gitCommitHash()}\";
-        }
-    """
+
+    new File(projectDir, "src/main/java/emu/grasscutter/BuildConfig.java").text =
+        """package emu.grasscutter;
+
+public final class BuildConfig {
+    public static final String VERSION = \"${version}\";
+    public static final String GIT_HASH = \"${gitCommitHash()}\";
+}"""
+
 }
 
 processResources {

format_whitespace.py

+import re
+import subprocess
+
+
+UPSTREAM = 'https://github.com/Grasscutters/Grasscutter.git'
+RATCHET = 'LintRatchet'
+RATCHET_FALLBACK = 'c517b8a2c95473811eb07e12e73c4a69e59fbbdc'
+
+
+re_leading_whitespace = re.compile(r'^[ \t]+', re.MULTILINE)  # Replace with \1.replace('\t', '    ')
+re_trailing_whitespace = re.compile(r'[ \t]+$', re.MULTILINE)  # Replace with ''
+# Replace 'for (foo){bar' with 'for (foo) {bar'
+re_bracket_space = re.compile(r'\) *\{(?!\})')  # Replace with ') {'
+# Replace 'for(foo)' with 'foo (bar)'
+re_keyword_space = re.compile(r'(?<=\b)(if|for|while|switch|try|else|catch|finally|synchronized) *(?=[\(\{])')  # Replace with '\1 '
+
+
+def get_changed_filelist():
+    # subprocess.run(['git', 'fetch', UPSTREAM, f'{RATCHET}:{RATCHET}'])  # Ensure LintRatchet ref is matched to upstream
+    # result = subprocess.run(['git', 'diff', RATCHET, '--name-only'], capture_output=True, text=True)
+    # if result.returncode != 0:
+        # print(f'{RATCHET} not found, trying fallback {RATCHET_FALLBACK}')
+    print(f'Attempting to diff against {RATCHET_FALLBACK}')
+    result = subprocess.run(['git', 'diff', RATCHET_FALLBACK, '--name-only'], capture_output=True, text=True)
+    if result.returncode != 0:
+        # print('Fallback is also missing, aborting.')
+        print(f'Could not find {RATCHET_FALLBACK}, aborting.')
+        exit(1)
+    return result.stdout.strip().split('\n')
+
+
+def format_string(data: str):
+    data = re_leading_whitespace.sub(lambda m: m.group(0).replace('\t', '    '), data)
+    data = re_trailing_whitespace.sub('', data)
+    data = re_bracket_space.sub(') {', data)
+    data = re_keyword_space.sub(r'\1 ', data)
+    if not data.endswith('\n'):  # Enforce trailing \n
+        data = data + '\n'
+    return data
+
+
+def format_file(filename: str) -> bool:
+    try:
+        with open(filename, 'r') as file:
+            data = file.read()
+        data = format_string(data)
+        with open(filename, 'w') as file:
+            file.write(data)
+        return True
+    except FileNotFoundError:
+        print(f'File not found, probably deleted: {filename}')
+        return False
+
+
+def main():
+    filelist = [f for f in get_changed_filelist() if f.endswith('.java') and not f.startswith('src/generated')]
+    replaced = 0
+    not_found = 0
+    if not filelist:
+        print('No changed files due for formatting!')
+        return
+    print('Changed files due for formatting: ', filelist)
+    for file in filelist:
+        if format_file(file):
+            replaced += 1
+        else:
+            not_found += 1
+    print(f'Format complete! {replaced} formatted, {not_found} missing.')
+
+
+if __name__ == '__main__':
+    main()

install_without_dependencies.sh

+#!/usr/bin/env bash
+
+# Grasscutter install script for linux - Simpler version
+# This installer doesn't ask you to install dependencies, you have to install them manually
+# Made by TurtleIdiot and modified by syktyvkar (and then again modified by Blue)
+
+# Stops the installer if any command has a non-zero exit status
+set -e
+
+# Checks for root
+if [ $EUID != 0 ]; then
+    echo "Please run the installer as root (sudo)!"
+    exit
+fi
+
+is_command() {
+    # Checks if given command is available
+    local check_command="$1"
+    command -v "${check_command}" > /dev/null 2>&1
+}
+
+# IP validation
+valid_ip() {
+    local ip=$1
+    local stat=1
+
+    if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
+        OIFS=$IFS
+        IFS="."
+        ip=($ip)
+        IFS=$OIFS
+        [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
+            && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
+        stat=$?
+    fi
+    return $stat
+}
+echo "#################################"
+echo ""
+echo "This script will take for granted that you have all dependencies installed (mongodb, openjdk-17-jre/jre17-openjdk, wget, openssl, unzip, git, curl, base-devel), in fact, this script is recommended to update your current server installation, and it should run from the same folder as grasscutter.jar"
+echo "#################################"
+echo ""
+echo "If you are using version > 2.8 of the client, make sure to use the patched metadata if you don't use Cultivation."
+echo "Search for METADATA here: https://discord.gg/grasscutter."
+echo ""
+echo "#################################"
+echo "You can find plugins here: https://discord.com/channels/965284035985305680/970830969919664218"
+echo ""
+echo "Grasscutter will be installed to script's running directory"
+echo "Do you wish to proceed and install Grasscutter?"
+select yn in "Yes" "No" ; do
+    case $yn in
+        Yes ) break;;
+        No )
+			echo "Aborting..."
+			exit;;
+    esac
+done
+
+if [ -d "./resources" ]
+then
+    echo "It's recommended to remove resources folder"
+    echo "Remove resources folder?"
+    select yn in "Yes" "No" ; do
+        case $yn in
+            Yes ) 
+                rm -rf resources
+                break;;
+            No )
+                echo "Aborting..."
+                exit;;
+        esac
+    done
+echo "You may need to remove data folder and config.json to apply some updates"
+echo "#################################"
+fi
+
+
+
+# Allows choice between stable and dev branch
+echo "Please select the branch you wish to install"
+echo -e "!!NOTE!!: stable is the recommended branch.\nDo *NOT* use development unless you have a reason to and know what you're doing"
+select branch in "stable" "development" ; do
+    case $branch in
+        stable )
+            break;;
+        development )
+            break;;
+    esac
+done
+
+echo -e "Using $branch branch for installing server \n"
+
+# Prompt IP address for config.json and for generating new keystore.p12 file
+echo "Please enter the IP address that will be used to connect to the server"
+echo "This can be a local or a public IP address"
+echo "This IP address will be used to generate SSL certificates, so it is important it is correct!"
+
+while : ; do
+    read -p "Enter server IP: " SERVER_IP
+    if valid_ip $SERVER_IP; then
+        break;
+    else
+        echo "Invalid IP address. Try again."
+    fi
+done
+
+echo "Beginning Grasscutter installation..."
+
+
+# Download resources
+echo "Downloading Grasscutter BinOutputs..."
+git clone --single-branch https://github.com/Koko-boya/Grasscutter_Resources.git Grasscutter-bins
+mv ./Grasscutter-bins/Resources ./resources
+rm -rf Grasscutter-bins # takes ~350M of drive space after moving BinOutputs... :sob:
+
+# Download and build jar
+echo "Downloading Grasscutter source code..."
+git clone --single-branch -b $branch https://github.com/Grasscutters/Grasscutter.git Grasscutter-src #change this to download a fork
+
+echo "Building grasscutter.jar..."
+cd Grasscutter-src
+chmod +x ./gradlew #just in case
+./gradlew --no-daemon jar
+mv $(find -name "grasscutter*.jar" -type f) ../grasscutter.jar
+echo "Building grasscutter.jar done!"
+cd ..
+
+# Generate handbook/config
+echo "Grasscutter will be started to generate data files"
+java -jar grasscutter.jar -version
+
+# Replaces "127.0.0.1" with given IP
+echo "Replacing IP address in server config..."
+sed -i "s/127.0.0.1/$SERVER_IP/g" config.json
+# Generates new keystore.p12 with the server's IP address
+# This is done to prevent a "Connection Timed Out" error from appearing
+#	   after clicking to enter the door in the main menu/title screen
+# This issue only exists when connecting to a server *other* than localhost
+#	   since the default keystore.p12 has only been made for localhost
+
+mkdir certs
+cd certs
+echo "Generating CA key and certificate pair..."
+openssl req -x509 -nodes -days 25202 -newkey rsa:2048 -subj "/C=GB/ST=Essex/L=London/O=Grasscutters/OU=Grasscutters/CN=$SERVER_IP" -keyout CAkey.key -out CAcert.crt
+
+echo "Generating SSL key and certificate pair..."
+openssl genpkey -out ssl.key -algorithm rsa
+
+# Creates a conf file in order to generate a csr
+cat > csr.conf <<EOF
+[ req ]
+default_bits = 2048
+prompt = no
+default_md = sha256
+req_extensions = req_ext
+distinguished_name = dn
+
+[ dn ]
+C = GB
+ST = Essex
+L = London
+O = Grasscutters
+OU = Grasscutters
+CN = $SERVER_IP
+
+[ req_ext ]
+subjectAltName = @alt_names
+
+[ alt_names ]
+IP.1 = $SERVER_IP
+EOF
+
+# Creates csr using key and conf
+openssl req -new -key ssl.key -out ssl.csr -config csr.conf
+
+# Creates conf to finalise creation of certificate
+cat > cert.conf <<EOF
+
+authorityKeyIdentifier=keyid,issuer
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, keyAgreement, dataEncipherment
+subjectAltName = @alt_names
+
+[alt_names]
+IP.1 = $SERVER_IP
+
+EOF
+
+# Creates SSL cert
+openssl x509 -req -in ssl.csr -CA CAcert.crt -CAkey CAkey.key -CAcreateserial -out ssl.crt -days 25202 -sha256 -extfile cert.conf
+
+echo "Generating keystore.p12 from key and certificate..."
+openssl pkcs12 -export -out keystore.p12 -inkey ssl.key -in ssl.crt -certfile CAcert.crt -passout pass:123456
+
+cd ../
+mv ./certs/keystore.p12 ./keystore.p12
+echo "Done!"
+
+# Running scripts as sudo makes all Grasscutter files to be owned by root
+# which may cause problems editing .jsons...
+if [ $SUDO_USER ]; then
+	echo "Changing Grasscutter files owner to current user..."
+	chown -R $SUDO_USER:$SUDO_USER ./*
+fi
+
+echo "Removing unnecessary files..."
+rm -rf ./certs ./Grasscutter-src
+
+echo "All done!"
+echo "-=-=-=-=-=--- !! IMPORTANT !! ---=-=-=-=-=-"
+echo "Please make sure that ports 80, 443, 8888 and 22102 are OPEN (both tcp and udp)"
+echo "In order to run the server, run the following command:"
+echo "    sudo java -jar grasscutter.jar"
+echo "The GM Handbook of all supported languages will be generated automatically when you start the server for the first time."
+echo "You must run it using sudo as port 443 is a privileged port"
+echo "To play, use the IP you provided earlier ($SERVER_IP) via GrassClipper or Fiddler"
+echo ""
+
+exit

manage_languages.py

+# Written for Python 3.6+
+# Older versions don't retain insertion order of regular dicts
+import argparse
+import cmd
+import json
+import os
+import re
+from pprint import pprint
+
+INDENT = 2
+PRIMARY_LANGUAGE = 'en-US.json'
+PRIMARY_FALLBACK_PREFIX = '🇺🇸'  # This is invisible in-game, terminal emulators might render it
+LANGUAGE_FOLDER = 'src/main/resources/languages/'
+LANGUAGE_FILENAMES = sorted(os.listdir(LANGUAGE_FOLDER), key=lambda x: 'AAA' if x == PRIMARY_LANGUAGE else x)
+SOURCE_FOLDER = 'src/'
+SOURCE_EXTENSIONS = ('java')
+
+
+def ppprint(data):
+    pprint(data, width=130, sort_dicts=False, compact=True)
+
+
+class JsonHelpers:
+    @staticmethod
+    def load(filename: str) -> dict:
+        with open(filename, 'r', encoding='utf-8') as file:
+            return json.load(file)
+
+    @staticmethod
+    def save(filename: str, data: dict) -> None:
+        with open(filename, 'w', encoding='utf-8', newline='\n') as file:
+            json.dump(data, file, ensure_ascii=False, indent=INDENT)
+            file.write('\n')  # json.dump doesn't terminate last line
+
+    @staticmethod
+    def flatten(data: dict, prefix='') -> dict:
+        output = {}
+        for key, value in data.items():
+            if isinstance(value, dict):
+                for k,v in JsonHelpers.flatten(value, f'{prefix}{key}.').items():
+                    output[k] = v
+            else:
+                output[f'{prefix}{key}'] = value
+        return output
+
+    @staticmethod
+    def unflatten(data: dict) -> dict:
+        output = {}
+        def add_key(k: list, value, d: dict):
+            if len(k) == 1:
+                d[k[0]] = value
+            else:
+                d[k[0]] = d.get(k[0], {})
+                add_key(k[1:], value, d[k[0]])
+        for key, value in data.items():
+            add_key(key.split('.'), value, output)
+        return output
+
+    @staticmethod
+    def pprint_keys(keys, indent=4) -> str:
+        # Only strip down to one level
+        padding = ' ' * indent
+        roots = {}
+        for key in keys:
+            root, _, k = key.rpartition('.')
+            roots[root] = roots.get(root, [])
+            roots[root].append(k)
+        lines = []
+        for root, ks in roots.items():
+            if len(ks) > 1:
+                lines.append(f'{padding}{root}.[{", ".join(ks)}]')
+            else:
+                lines.append(f'{padding}{root}.{ks[0]}')
+        return ',\n'.join(lines)
+
+    @staticmethod
+    def deep_clone_and_fill(d1: dict, d2: dict, fallback_prefix=PRIMARY_FALLBACK_PREFIX) -> dict:
+        out = {}
+        for key, value in d1.items():
+            if isinstance(value, dict):
+                out[key] = JsonHelpers.deep_clone_and_fill(value, d2.get(key, {}), fallback_prefix)
+            else:
+                v2 = d2.get(key, value)
+                if type(value) == str and v2 == value:
+                    out[key] = fallback_prefix + value
+                else:
+                    out[key] = v2
+        return out
+
+
+class LanguageManager:
+    TRANSLATION_KEY = re.compile(r'[Tt]ranslate.*"(\w+\.[\w\.]+)"')
+    POTENTIAL_KEY = re.compile(r'"(\w+\.[\w\.]+)"')
+    COMMAND_LABEL = re.compile(r'@Command\s*\([\W\w]*?label\s*=\s*"(\w+)"', re.MULTILINE)  # [\W\w] is a cheeky way to match everything including \n
+
+    def __init__(self):
+        self.load_jsons()
+
+    def load_jsons(self):
+        self.language_jsons = [JsonHelpers.load(LANGUAGE_FOLDER + filename) for filename in LANGUAGE_FILENAMES]
+        self.flattened_jsons = [JsonHelpers.flatten(j) for j in self.language_jsons]
+        self.update_keys()
+
+    def update_keys(self):
+        self.key_sets = [set(j.keys()) for j in self.flattened_jsons]
+        self.common_keys = set.intersection(*self.key_sets)
+        self.all_keys = set.union(*self.key_sets)
+        self.used_keys = self.find_all_used_keys(self.all_keys)
+        self.missing_keys = self.used_keys - self.common_keys
+        self.unused_keys = self.all_keys - self.used_keys
+
+    def find_all_used_keys(self, expected_keys=[]) -> set:
+        # Note that this will only find string literals passed to the translate() or sendTranslatedMessage() methods!
+        # String variables passed to them can be checked against expected_keys
+        used = set()
+        potential = set()
+        for root, dirs, files in os.walk(SOURCE_FOLDER):
+            for file in files:
+                if file.rpartition('.')[-1] in SOURCE_EXTENSIONS:
+                    filename = os.path.join(root, file)
+                    with open(filename, 'r', encoding='utf-8') as f:
+                        data = f.read()  # Loads in entire file at once
+                        for k in self.TRANSLATION_KEY.findall(data):
+                            used.add(k)
+                        for k in self.POTENTIAL_KEY.findall(data):
+                            potential.add(k)
+                        for label in self.COMMAND_LABEL.findall(data):
+                            used.add(f'commands.{label}.description')
+        return used | (potential & expected_keys)
+
+    def _lint_report_language(self, lang: str, keys: set, flattened: dict, primary_language_flattened: dict) -> None:
+        missing = self.used_keys - keys
+        unused = keys - self.used_keys
+        identical_keys = set() if (lang == PRIMARY_LANGUAGE) else {key for key in keys if primary_language_flattened.get(key, None) == flattened.get(key)}
+        placeholder_keys = {key for key in keys if flattened.get(key).startswith(PRIMARY_FALLBACK_PREFIX)}
+        p1 = f'Language {lang} has {len(missing)} missing keys and {len(unused)} unused keys.'
+        p2 = 'This is the primary language.' if (lang == PRIMARY_LANGUAGE) else f'{len(identical_keys)} match {PRIMARY_LANGUAGE}, {len(placeholder_keys)} have the placeholder mark.'
+        print(f'{p1} {p2}')
+
+        lint_categories = {
+            'Missing': missing,
+            'Unused': unused,
+            f'Matches {PRIMARY_LANGUAGE}': identical_keys,
+            'Placeholder': placeholder_keys,
+        }
+        for name, category in lint_categories.items():
+            if len(category) > 0:
+                print(name + ':')
+                print(JsonHelpers.pprint_keys(sorted(category)))
+
+    def lint_report(self) -> None:
+        print(f'There are {len(self.missing_keys)} translation keys in use that are missing from one or more language files.')
+        print(f'There are {len(self.unused_keys)} translation keys in language files that are not used.')
+        primary_language_flattened = self.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
+        for lang, keys, flattened in zip(LANGUAGE_FILENAMES, self.key_sets, self.flattened_jsons):
+            print('')
+            self._lint_report_language(lang, keys, flattened, primary_language_flattened)
+
+    def rename_keys(self, key_remappings: dict) -> None:
+        # Unfortunately we can't rename keys in-place preserving insertion order, so we have to make new dicts
+        for i in range(len(self.flattened_jsons)):
+            self.flattened_jsons[i] = {key_remappings.get(k,k):v for k,v in self.flattened_jsons[i].items()}
+
+    def update_secondary_languages(self):
+        # Push en_US fallback
+        primary_language_json = self.language_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
+        for filename, lang in zip(LANGUAGE_FILENAMES, self.language_jsons):
+            if filename != PRIMARY_LANGUAGE:
+                js = JsonHelpers.deep_clone_and_fill(primary_language_json, lang)
+                JsonHelpers.save(LANGUAGE_FOLDER + filename, js)
+
+    def update_all_languages_from_flattened(self):
+        for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons):
+            JsonHelpers.save(LANGUAGE_FOLDER + filename, JsonHelpers.unflatten(flat))
+
+    def save_flattened_languages(self, prefix='flat_'):
+        for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons):
+            JsonHelpers.save(prefix + filename, flat)
+
+
+class InteractiveRename(cmd.Cmd):
+    intro = 'Welcome to the interactive rename shell.   Type help or ? to list commands.\n'
+    prompt = '(rename) '
+    file = None
+
+    def __init__(self, language_manager: LanguageManager) -> None:
+        super().__init__()
+        self.language_manager = language_manager
+        self.flat_keys = [key for key in language_manager.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)].keys()]
+        self.mappings = {}
+
+    def do_add(self, arg):
+        '''
+        Prepare to rename an existing translation key. Will not actually rename anything until you confirm all your pending changes with 'rename'.
+        e.g. a single string:  add commands.execution.argument_error commands.generic.invalid.argument
+        e.g. a group:          add commands.enter_dungeon commands.new_enter_dungeon
+        '''
+        args = arg.split()
+        if len(args) < 2:
+            self.do_help('add')
+            return
+        old, new = args[:2]
+        if old in self.flat_keys:
+            self.mappings[old] = new
+        else:
+            # Check if we are renaming a higher level
+            if not old.endswith('.'):
+                old = old + '.'
+            results = [key for key in self.flat_keys if key.startswith(old)]
+            if len(results) > 0:
+                if not new.endswith('.'):
+                    new = new + '.'
+                new_mappings = {key: key.replace(old, new) for key in results}
+                # Ask for confirmation
+                print('Will add the following mappings:')
+                ppprint(new_mappings)
+                print('Add these mappings? [y/N]')
+                if self.prompt_yn():
+                    for k,v in new_mappings.items():
+                        self.mappings[k] = v
+            else:
+                print('No translation keys matched!')
+    
+    def complete_add(self, text: str, line: str, begidx: int, endidx: int) -> list:
+        if text == '':
+            return [k for k in {key.partition('.')[0] for key in self.flat_keys}]
+        results = [key for key in self.flat_keys if key.startswith(text)]
+        if len(results) > 40:
+            # Collapse categories
+            if text[-1] != '.':
+                text = text + '.'
+            level = text.count('.') + 1
+            new_results = {'.'.join(key.split('.')[:level]) for key in results}
+            return list(new_results)
+        return results
+
+    def do_remove(self, arg):
+        '''
+        Remove a pending rename mapping. Takes the old name of the key, not the new one.
+        e.g. a single key:  remove commands.execution.argument_error
+        e.g. a group:       remove commands.enter_dungeon
+        '''
+        old = arg.split()[0]
+        if old in self.mappings:
+            self.mappings.pop(old)
+        else:
+            # Check if we are renaming a higher level
+            if not old.endswith('.'):
+                old = old + '.'
+            results = [key for key in self.mappings if key.startswith(old)]
+            if len(results) > 0:
+                # Ask for confirmation
+                print('Will remove the following pending mappings:')
+                print(JsonHelpers.pprint_keys(results))
+                print('Delete these mappings? [y/N]')
+                if self.prompt_yn():
+                    for key in results:
+                        self.mappings.pop(key)
+            else:
+                print('No pending rename mappings matched!')
+    
+    def complete_remove(self, text: str, line: str, begidx: int, endidx: int) -> list:
+        return [key for key in self.mappings if key.startswith(text)]
+
+    def do_rename(self, _arg):
+        'Applies pending renames and overwrites language jsons.'
+        # Ask for confirmation
+        print('Will perform the following mappings:')
+        ppprint(self.mappings)
+        print('Perform and save these rename mappings? [y/N]')
+        if self.prompt_yn():
+            self.language_manager.rename_keys(self.mappings)
+            self.language_manager.update_all_languages_from_flattened()
+            print('Renamed keys, closing')
+            return True
+        else:
+            print('Do you instead wish to quit without saving? [yes/N]')
+            if self.prompt_yn(True):
+                print('Left rename shell without renaming')
+                return True
+
+    def prompt_yn(self, strict_yes=False):
+        if strict_yes:
+            return input('(yes/N) ').lower() == 'yes'
+        return input('(y/N) ').lower()[0] == 'y'
+
+
+def main(args: argparse.Namespace):
+    # print(args)
+    language_manager = LanguageManager()
+    errors = None
+    if args.lint_report:
+        language_manager.lint_report()
+        missing = language_manager.used_keys - language_manager.key_sets[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
+        if len(missing) > 0:
+            errors = f'[ERROR] {len(missing)} keys missing from primary language json!\n{JsonHelpers.pprint_keys(missing)}'
+    if prefix := args.save_flattened:
+        language_manager.save_flattened_languages(prefix)
+    if args.update:
+        print('Updating secondary languages')
+        language_manager.update_secondary_languages()
+    if args.interactive_rename:
+        language_manager.load_jsons()  # Previous actions may have changed them on-disk
+        try:
+            InteractiveRename(language_manager).cmdloop()
+        except KeyboardInterrupt:
+            print('Left rename shell without renaming')
+    if errors:
+        print(errors)
+        exit(1)
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Manage Grasscutter's language json files.")
+    parser.add_argument('-u', '--update', action='store_true',
+        help=f'Update secondary language files to conform to the layout of the primary language file ({PRIMARY_LANGUAGE}) and contain any new keys from it.')
+    parser.add_argument('-l', '--lint-report', action='store_true',
+        help='Prints a lint report, listing unused, missing, and untranslated keys among all language jsons.')
+    parser.add_argument('-f', '--save-flattened', const='./flat_', metavar='prefix', nargs='?',
+        help='Save copies of all the language jsons in a flattened key form.')
+    parser.add_argument('-i', '--interactive-rename', action='store_true',
+        help='Enter interactive rename mode, in which you can specify keys in flattened form to be renamed.')
+    args = parser.parse_args()
+    main(args)
\ No newline at end of file

plugin-schema.json

 {
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "JSON schema for a Grasscutter Plugin",
-  "type": "object",
-  "additionalProperties": true,
-  "definitions": {
-    "plugin-name": {
-      "type": "string",
-      "pattern": "^[A-Za-z\\d_.-]+$"
-    }
-  },
-  "required": [ "name", "description", "mainClass" ],
-  "properties": {
-    "name": {
-      "description": "The unique name of plugin.",
-      "$ref": "#/definitions/plugin-name"
-    },
-    "mainClass": {
-      "description": "The plugin's initial class file.",
-      "type": "string",
-      "pattern": "^(?!org\\.bukkit\\.)([a-zA-Z_$][a-zA-Z\\d_$]*\\.)*[a-zA-Z_$][a-zA-Z\\d_$]*$"
-    },
-    "version": {
-      "description": "A plugin revision identifier.",
-      "type": [ "string", "number" ]
-    },
-    "description": {
-      "description": "Human readable plugin summary.",
-      "type": "string"
-    },
-    "author": {
-      "description": "The plugin author.",
-      "type": "string"
-    },
-    "authors": {
-      "description": "The plugin contributors.",
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "JSON schema for a Grasscutter Plugin",
+    "type": "object",
+    "additionalProperties": true,
+    "definitions": {
+        "plugin-name": {
+            "type": "string",
+            "pattern": "^[A-Za-z\\d_.-]+$"
+        }
     },
-    "website": {
-      "title": "Website",
-      "description": "The URL to the plugin's site",
-      "type": "string",
-      "format": "uri"
+    "required": [
+        "name",
+        "description",
+        "mainClass"
+    ],
+    "properties": {
+        "name": {
+            "description": "The unique name of plugin.",
+            "$ref": "#/definitions/plugin-name"
+        },
+        "mainClass": {
+            "description": "The plugin's initial class file.",
+            "type": "string",
+            "pattern": "^(?!org\\.bukkit\\.)([a-zA-Z_$][a-zA-Z\\d_$]*\\.)*[a-zA-Z_$][a-zA-Z\\d_$]*$"
+        },
+        "version": {
+            "description": "A plugin revision identifier.",
+            "type": [
+                "string",
+                "number"
+            ]
+        },
+        "description": {
+            "description": "Human readable plugin summary.",
+            "type": "string"
+        },
+        "author": {
+            "description": "The plugin author.",
+            "type": "string"
+        },
+        "authors": {
+            "description": "The plugin contributors.",
+            "type": "array",
+            "items": {
+                "type": "string"
+            }
+        },
+        "website": {
+            "title": "Website",
+            "description": "The URL to the plugin's site",
+            "type": "string",
+            "format": "uri"
+        },
+        "loadAfter": {
+            "description": "Plugins to load before this plugin.",
+            "type": "array",
+            "items": {
+                "type": "string"
+            }
+        }
     }
-  }
 }
\ No newline at end of file

proxy.py

@@ -6,7 +6,7 @@
 
 ##
 #
-#   Genshin Impact script for mitmproxy
+#   Animation Company script for mitmproxy
 #
 #   https://github.com/MlgmXyysd/
 #
@@ -20,12 +20,17 @@
 #
 ##
 
-from mitmproxy import http
+import collections
+import random
+from mitmproxy import http, connection, ctx, tls
+from abc import ABC, abstractmethod
+from enum import Enum
+from mitmproxy.utils import human
 from proxy_config import USE_SSL
 from proxy_config import REMOTE_HOST
 from proxy_config import REMOTE_PORT
 
-class MlgmXyysd_Genshin_Impact_Proxy:
+class MlgmXyysd_Animation_Company_Proxy:
 
     LIST_DOMAINS = [
         "api-os-takumi.mihoyo.com",
@@ -71,6 +76,80 @@ class MlgmXyysd_Genshin_Impact_Proxy:
             flow.request.host = REMOTE_HOST
             flow.request.port = REMOTE_PORT
 
+class InterceptionResult(Enum):
+    SUCCESS = 1
+    FAILURE = 2
+    SKIPPED = 3
+
+
+class TlsStrategy(ABC):
+    def __init__(self):
+        self.history = collections.defaultdict(lambda: collections.deque(maxlen=200))
+
+    @abstractmethod
+    def should_intercept(self, server_address: connection.Address) -> bool:
+        raise NotImplementedError()
+
+    def record_success(self, server_address):
+        self.history[server_address].append(InterceptionResult.SUCCESS)
+
+    def record_failure(self, server_address):
+        self.history[server_address].append(InterceptionResult.FAILURE)
+
+    def record_skipped(self, server_address):
+        self.history[server_address].append(InterceptionResult.SKIPPED)
+
+
+class ConservativeStrategy(TlsStrategy):
+    def should_intercept(self, server_address: connection.Address) -> bool:
+        return InterceptionResult.FAILURE not in self.history[server_address]
+
+
+class ProbabilisticStrategy(TlsStrategy):
+    def __init__(self, p: float):
+        self.p = p
+        super().__init__()
+
+    def should_intercept(self, server_address: connection.Address) -> bool:
+        return random.uniform(0, 1) < self.p
+
+
+class MaybeTls:
+    strategy: TlsStrategy
+
+    def load(self, l):
+        l.add_option(
+            "tls_strategy", int, 0,
+            "TLS passthrough strategy. If set to 0, connections will be passed through after the first unsuccessful "
+            "handshake. If set to 0 < p <= 100, connections with be passed through with probability p.",
+        )
+
+    def configure(self, updated):
+        if "tls_strategy" not in updated:
+            return
+        if ctx.options.tls_strategy > 0:
+            self.strategy = ProbabilisticStrategy(ctx.options.tls_strategy / 100)
+        else:
+            self.strategy = ConservativeStrategy()
+
+    def tls_clienthello(self, data: tls.ClientHelloData):
+        server_address = data.context.server.peername
+        if not self.strategy.should_intercept(server_address):
+            ctx.log(f"TLS passthrough: {human.format_address(server_address)}.")
+            data.ignore_connection = True
+            self.strategy.record_skipped(server_address)
+
+    def tls_established_client(self, data: tls.TlsData):
+        server_address = data.context.server.peername
+        ctx.log(f"TLS handshake successful: {human.format_address(server_address)}")
+        self.strategy.record_success(server_address)
+
+    def tls_failed_client(self, data: tls.TlsData):
+        server_address = data.context.server.peername
+        ctx.log(f"TLS handshake failed: {human.format_address(server_address)}")
+        self.strategy.record_failure(server_address)
+        
 addons = [
-	MlgmXyysd_Genshin_Impact_Proxy()
+	MlgmXyysd_Animation_Company_Proxy(),
+    MaybeTls()
 ]