Language linting (#1382)

.github/workflows/language_lint.yml

+name: "Language Lint"
+on:
+  workflow_dispatch: ~
+  push:
+    paths:
+      - "**.java"
+      - "**.json"
+    branches:
+      - "stable"
+      - "development"
+  pull_request:
+    paths:
+      - "**.java"
+      - "**.json"
+    types:
+      - opened
+      - synchronize
+      - reopened
+jobs:
+  Lint-Language-Keys:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10' # Version range or exact version of a Python version to use, using SemVer's version range syntax
+      - run: python3 manage_languages.py -l

manage_languages.py

+# Written for Python 3.6+
+# Older versions don't retain insertion order of regular dicts
+import argparse
+import cmd
+import json
+import os
+import re
+from pprint import pprint
+
+INDENT = 2
+PRIMARY_LANGUAGE = 'en-US.json'
+PRIMARY_FALLBACK_PREFIX = '🇺🇸'  # This is invisible in-game, terminal emulators might render it
+LANGUAGE_FOLDER = 'src/main/resources/languages/'
+LANGUAGE_FILENAMES = sorted(os.listdir(LANGUAGE_FOLDER), key=lambda x: 'AAA' if x == PRIMARY_LANGUAGE else x)
+SOURCE_FOLDER = 'src/'
+SOURCE_EXTENSIONS = ('java')
+
+
+def ppprint(data):
+    pprint(data, width=130, sort_dicts=False, compact=True)
+
+
+class JsonHelpers:
+    @staticmethod
+    def load(filename: str) -> dict:
+        with open(filename, 'r') as file:
+            return json.load(file)
+
+    @staticmethod
+    def save(filename: str, data: dict) -> None:
+        with open(filename, 'w', encoding='utf-8', newline='\n') as file:
+            json.dump(data, file, ensure_ascii=False, indent=INDENT)
+
+    @staticmethod
+    def flatten(data: dict, prefix='') -> dict:
+        output = {}
+        for key, value in data.items():
+            if isinstance(value, dict):
+                for k,v in JsonHelpers.flatten(value, f'{prefix}{key}.').items():
+                    output[k] = v
+            else:
+                output[f'{prefix}{key}'] = value
+        return output
+
+    @staticmethod
+    def unflatten(data: dict) -> dict:
+        output = {}
+        def add_key(k: list, value, d: dict):
+            if len(k) == 1:
+                d[k[0]] = value
+            else:
+                d[k[0]] = d.get(k[0], {})
+                add_key(k[1:], value, d[k[0]])
+        for key, value in data.items():
+            add_key(key.split('.'), value, output)
+        return output
+
+    @staticmethod
+    def pprint_keys(keys, indent=4) -> str:
+        # Only strip down to one level
+        padding = ' ' * indent
+        roots = {}
+        for key in keys:
+            root, _, k = key.rpartition('.')
+            roots[root] = roots.get(root, [])
+            roots[root].append(k)
+        lines = []
+        for root, ks in roots.items():
+            if len(ks) > 1:
+                lines.append(f'{padding}{root}.[{", ".join(ks)}]')
+            else:
+                lines.append(f'{padding}{root}.{ks[0]}')
+        return ',\n'.join(lines)
+
+    @staticmethod
+    def deep_clone_and_fill(d1: dict, d2: dict, fallback_prefix=PRIMARY_FALLBACK_PREFIX) -> dict:
+        out = {}
+        for key, value in d1.items():
+            if isinstance(value, dict):
+                out[key] = JsonHelpers.deep_clone_and_fill(value, d2.get(key, {}), fallback_prefix)
+            else:
+                v2 = d2.get(key, value)
+                if type(value) == str and v2 == value:
+                    out[key] = fallback_prefix + value
+                else:
+                    out[key] = v2
+        return out
+
+
+class LanguageManager:
+    TRANSLATION_KEY = re.compile(r'[Tt]ranslate.*"(\w+\.[\w\.]+)"')
+    POTENTIAL_KEY = re.compile(r'"(\w+\.[\w\.]+)"')
+
+    def __init__(self):
+        self.load_jsons()
+
+    def load_jsons(self):
+        self.language_jsons = [JsonHelpers.load(LANGUAGE_FOLDER + filename) for filename in LANGUAGE_FILENAMES]
+        self.flattened_jsons = [JsonHelpers.flatten(j) for j in self.language_jsons]
+        self.update_keys()
+
+    def update_keys(self):
+        self.key_sets = [set(j.keys()) for j in self.flattened_jsons]
+        self.common_keys = set.intersection(*self.key_sets)
+        self.all_keys = set.union(*self.key_sets)
+        self.used_keys = self.find_all_used_keys(self.all_keys)
+        self.missing_keys = self.used_keys - self.common_keys
+        self.unused_keys = self.all_keys - self.used_keys
+
+    def find_all_used_keys(self, expected_keys=[]) -> set:
+        # Note that this will only find string literals passed to the translate() or sendTranslatedMessage() methods!
+        # String variables passed to them can be checked against expected_keys
+        used = set()
+        potential = set()
+        for root, dirs, files in os.walk(SOURCE_FOLDER):
+            for file in files:
+                if file.rpartition('.')[-1] in SOURCE_EXTENSIONS:
+                    filename = os.path.join(root, file)
+                    with open(filename, 'r') as f:
+                        data = f.read()  # Loads in entire file at once
+                        for k in self.TRANSLATION_KEY.findall(data):
+                            used.add(k)
+                        for k in self.POTENTIAL_KEY.findall(data):
+                            potential.add(k)
+        return used | (potential & expected_keys)
+
+    def _lint_report_language(self, lang: str, keys: set, flattened: dict, primary_language_flattened: dict) -> None:
+        missing = self.used_keys - keys
+        unused = keys - self.used_keys
+        identical_keys = set() if (lang == PRIMARY_LANGUAGE) else {key for key in keys if primary_language_flattened.get(key, None) == flattened.get(key)}
+        placeholder_keys = {key for key in keys if flattened.get(key).startswith(PRIMARY_FALLBACK_PREFIX)}
+        p1 = f'Language {lang} has {len(missing)} missing keys and {len(unused)} unused keys.'
+        p2 = 'This is the primary language.' if (lang == PRIMARY_LANGUAGE) else f'{len(identical_keys)} match {PRIMARY_LANGUAGE}, {len(placeholder_keys)} have the placeholder mark.'
+        print(f'{p1} {p2}')
+
+        lint_categories = {
+            'Missing': missing,
+            'Unused': unused,
+            f'Matches {PRIMARY_LANGUAGE}': identical_keys,
+            'Placeholder': placeholder_keys,
+        }
+        for name, category in lint_categories.items():
+            if len(category) > 0:
+                print(name + ':')
+                print(JsonHelpers.pprint_keys(sorted(category)))
+
+    def lint_report(self) -> None:
+        print(f'There are {len(self.missing_keys)} translation keys in use that are missing from one or more language files.')
+        print(f'There are {len(self.unused_keys)} translation keys in language files that are not used.')
+        primary_language_flattened = self.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
+        for lang, keys, flattened in zip(LANGUAGE_FILENAMES, self.key_sets, self.flattened_jsons):
+            print('')
+            self._lint_report_language(lang, keys, flattened, primary_language_flattened)
+
+    def rename_keys(self, key_remappings: dict) -> None:
+        # Unfortunately we can't rename keys in-place preserving insertion order, so we have to make new dicts
+        for i in range(len(self.flattened_jsons)):
+            self.flattened_jsons[i] = {key_remappings.get(k,k):v for k,v in self.flattened_jsons[i].items()}
+
+    def update_secondary_languages(self):
+        # Push en_US fallback
+        primary_language_json = self.language_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
+        for filename, lang in zip(LANGUAGE_FILENAMES, self.language_jsons):
+            if filename != PRIMARY_LANGUAGE:
+                js = JsonHelpers.deep_clone_and_fill(primary_language_json, lang)
+                JsonHelpers.save(LANGUAGE_FOLDER + filename, js)
+
+    def update_all_languages_from_flattened(self):
+        for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons):
+            JsonHelpers.save(LANGUAGE_FOLDER + filename, JsonHelpers.unflatten(flat))
+
+    def save_flattened_languages(self, prefix='flat_'):
+        for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons):
+            JsonHelpers.save(prefix + filename, flat)
+
+
+class InteractiveRename(cmd.Cmd):
+    intro = 'Welcome to the interactive rename shell.   Type help or ? to list commands.\n'
+    prompt = '(rename) '
+    file = None
+
+    def __init__(self, language_manager: LanguageManager) -> None:
+        super().__init__()
+        self.language_manager = language_manager
+        self.flat_keys = [key for key in language_manager.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)].keys()]
+        self.mappings = {}
+
+    def do_add(self, arg):
+        '''
+        Prepare to rename an existing translation key. Will not actually rename anything until you confirm all your pending changes with 'rename'.
+        e.g. a single string:  add commands.execution.argument_error commands.generic.invalid.argument
+        e.g. a group:          add commands.enter_dungeon commands.new_enter_dungeon
+        '''
+        args = arg.split()
+        if len(args) < 2:
+            self.do_help('add')
+            return
+        old, new = args[:2]
+        if old in self.flat_keys:
+            self.mappings[old] = new
+        else:
+            # Check if we are renaming a higher level
+            if not old.endswith('.'):
+                old = old + '.'
+            results = [key for key in self.flat_keys if key.startswith(old)]
+            if len(results) > 0:
+                if not new.endswith('.'):
+                    new = new + '.'
+                new_mappings = {key: key.replace(old, new) for key in results}
+                # Ask for confirmation
+                print('Will add the following mappings:')
+                ppprint(new_mappings)
+                print('Add these mappings? [y/N]')
+                if self.prompt_yn():
+                    for k,v in new_mappings.items():
+                        self.mappings[k] = v
+            else:
+                print('No translation keys matched!')
+    
+    def complete_add(self, text: str, line: str, begidx: int, endidx: int) -> list:
+        if text == '':
+            return [k for k in {key.partition('.')[0] for key in self.flat_keys}]
+        results = [key for key in self.flat_keys if key.startswith(text)]
+        if len(results) > 40:
+            # Collapse categories
+            if text[-1] != '.':
+                text = text + '.'
+            level = text.count('.') + 1
+            new_results = {'.'.join(key.split('.')[:level]) for key in results}
+            return list(new_results)
+        return results
+
+    def do_remove(self, arg):
+        '''
+        Remove a pending rename mapping. Takes the old name of the key, not the new one.
+        e.g. a single key:  remove commands.execution.argument_error
+        e.g. a group:       remove commands.enter_dungeon
+        '''
+        old = arg.split()[0]
+        if old in self.mappings:
+            self.mappings.pop(old)
+        else:
+            # Check if we are renaming a higher level
+            if not old.endswith('.'):
+                old = old + '.'
+            results = [key for key in self.mappings if key.startswith(old)]
+            if len(results) > 0:
+                # Ask for confirmation
+                print('Will remove the following pending mappings:')
+                print(JsonHelpers.pprint_keys(results))
+                print('Delete these mappings? [y/N]')
+                if self.prompt_yn():
+                    for key in results:
+                        self.mappings.pop(key)
+            else:
+                print('No pending rename mappings matched!')
+    
+    def complete_remove(self, text: str, line: str, begidx: int, endidx: int) -> list:
+        return [key for key in self.mappings if key.startswith(text)]
+
+    def do_rename(self, _arg):
+        'Applies pending renames and overwrites language jsons.'
+        # Ask for confirmation
+        print('Will perform the following mappings:')
+        ppprint(self.mappings)
+        print('Perform and save these rename mappings? [y/N]')
+        if self.prompt_yn():
+            self.language_manager.rename_keys(self.mappings)
+            self.language_manager.update_all_languages_from_flattened()
+            print('Renamed keys, closing')
+            return True
+        else:
+            print('Do you instead wish to quit without saving? [yes/N]')
+            if self.prompt_yn(True):
+                print('Left rename shell without renaming')
+                return True
+
+    def prompt_yn(self, strict_yes=False):
+        if strict_yes:
+            return input('(yes/N) ').lower() == 'yes'
+        return input('(y/N) ').lower()[0] == 'y'
+
+
+def main(args: argparse.Namespace):
+    # print(args)
+    language_manager = LanguageManager()
+    errors = None
+    if args.lint_report:
+        language_manager.lint_report()
+        missing = language_manager.used_keys - language_manager.key_sets[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
+        if len(missing) > 0:
+            errors = f'[ERROR] {len(missing)} keys missing from primary language json!\n{JsonHelpers.pprint_keys(missing)}'
+    if prefix := args.save_flattened:
+        language_manager.save_flattened_languages(prefix)
+    if args.update:
+        print('Updating secondary languages')
+        language_manager.update_secondary_languages()
+    if args.interactive_rename:
+        language_manager.load_jsons()  # Previous actions may have changed them on-disk
+        try:
+            InteractiveRename(language_manager).cmdloop()
+        except KeyboardInterrupt:
+            print('Left rename shell without renaming')
+    if errors:
+        print(errors)
+        exit(1)
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Manage Grasscutter's language json files.")
+    parser.add_argument('-u', '--update', action='store_true',
+        help=f'Update secondary language files to conform to the layout of the primary language file ({PRIMARY_LANGUAGE}) and contain any new keys from it.')
+    parser.add_argument('-l', '--lint-report', action='store_true',
+        help='Prints a lint report, listing unused, missing, and untranslated keys among all language jsons.')
+    parser.add_argument('-f', '--save-flattened', const='./flat_', metavar='prefix', nargs='?',
+        help='Save copies of all the language jsons in a flattened key form.')
+    parser.add_argument('-i', '--interactive-rename', action='store_true',
+        help='Enter interactive rename mode, in which you can specify keys in flattened form to be renamed.')
+    args = parser.parse_args()
+    main(args)
\ No newline at end of file

src/main/java/emu/grasscutter/auth/DefaultAuthentication.java

@@ -31,7 +31,7 @@ public final class DefaultAuthentication implements AuthenticationSystem {
 
     @Override
     public Account verifyUser(String details) {
-        Grasscutter.getLogger().info(translate("dispatch.authentication.default_unable_to_verify"));
+        Grasscutter.getLogger().info(translate("messages.dispatch.authentication.default_unable_to_verify"));
         return null;
     }
 

src/main/java/emu/grasscutter/command/CommandMap.java

@@ -106,7 +106,12 @@ public final class CommandMap {
      * @return The command handler.
      */
     public CommandHandler getHandler(String label) {
-        return this.commands.get(label);
+        CommandHandler handler = this.commands.get(label);
+        if (handler == null) {
+            // Try getting by alias
+            handler = this.aliases.get(label);
+        }
+        return handler;
     }
 
     private Player getTargetPlayer(String playerId, Player player, Player targetPlayer, List<String> args) {
@@ -129,7 +134,7 @@ public final class CommandMap {
                     }
                     return targetPlayer;
                 } catch (NumberFormatException e) {
-                    CommandHandler.sendTranslatedMessage(player, "commands.execution.uid_error");
+                    CommandHandler.sendTranslatedMessage(player, "commands.generic.invalid.uid");
                     throw new IllegalArgumentException();
                 }
             }
@@ -177,7 +182,7 @@ public final class CommandMap {
             CommandHandler.sendTranslatedMessage(player, targetPlayer.isOnline()? "commands.execution.set_target_online" : "commands.execution.set_target_offline", targetUid);
             return true;
         } catch (NumberFormatException e) {
-            CommandHandler.sendTranslatedMessage(player, "commands.execution.uid_error");
+            CommandHandler.sendTranslatedMessage(player, "commands.generic.invalid.uid");
             return false;
         }
     }
@@ -220,12 +225,9 @@ public final class CommandMap {
         }
 
         // Get command handler.
-        CommandHandler handler = this.commands.get(label);
-        if(handler == null)
-            // Try to get the handler by alias.
-            handler = this.aliases.get(label);
+        CommandHandler handler = this.getHandler(label);
 
-        // Check if the handler is still null.
+        // Check if the handler is null.
         if (handler == null) {
             CommandHandler.sendTranslatedMessage(player, "commands.generic.unknown_command", label);
             return;

src/main/java/emu/grasscutter/command/commands/GiveCommand.java

@@ -221,7 +221,6 @@ public final class GiveCommand implements CommandHandler {
                 case ITEM_RELIQUARY:
                     targetPlayer.getInventory().addItems(makeArtifacts(param), ActionReason.SubfieldDrop);
                     CommandHandler.sendTranslatedMessage(sender, "commands.give.given_level", Integer.toString(param.id), Integer.toString(param.lvl), Integer.toString(param.amount), Integer.toString(targetPlayer.getUid()));
-                    //CommandHandler.sendTranslatedMessage(sender, "commands.giveArtifact.success", Integer.toString(param.id), Integer.toString(targetPlayer.getUid()));
                     return;
                 default:
                     targetPlayer.getInventory().addItem(new GameItem(param.data, param.amount), ActionReason.SubfieldDrop);

src/main/java/emu/grasscutter/command/commands/HelpCommand.java

 package emu.grasscutter.command.commands;
 
-import emu.grasscutter.Grasscutter;
 import emu.grasscutter.command.Command;
 import emu.grasscutter.command.CommandHandler;
 import emu.grasscutter.command.CommandMap;
@@ -13,6 +12,28 @@ import static emu.grasscutter.utils.Language.translate;
 @Command(label = "help", usage = "help [command]", description = "commands.help.description", targetRequirement = Command.TargetRequirement.NONE)
 public final class HelpCommand implements CommandHandler {
 
+    private void createCommand(StringBuilder builder, Player player, Command annotation) {
+        builder.append("\n").append(annotation.label()).append(" - ").append(translate(player, annotation.description()));
+        builder.append("\n\t").append(translate(player, "commands.help.usage"));
+        if (annotation.aliases().length >= 1) {
+            builder.append("\n\t").append(translate(player, "commands.help.aliases"));
+            for (String alias : annotation.aliases()) {
+                builder.append(alias).append(" ");
+            }
+        }
+        builder.append("\n\t").append(translate(player, "commands.help.tip_need_permission"));
+        if(annotation.permission().isEmpty() || annotation.permission().isBlank()) {
+            builder.append(translate(player, "commands.help.tip_need_no_permission"));
+        } else {
+            builder.append(annotation.permission());
+        }
+
+        if(!annotation.permissionTargeted().isEmpty() && !annotation.permissionTargeted().isBlank()) {
+            String permissionTargeted = annotation.permissionTargeted();
+            builder.append(" ").append(translate(player, "commands.help.tip_permission_targeted", permissionTargeted));
+        }
+    }
+
     @Override
     public void execute(Player player, Player targetPlayer, List<String> args) {
         if (args.size() < 1) {
@@ -32,38 +53,16 @@ public final class HelpCommand implements CommandHandler {
         } else {
             String command = args.get(0);
             CommandHandler handler = CommandMap.getInstance().getHandler(command);
-            StringBuilder builder = new StringBuilder(player == null ? "\n" + translate(player, "commands.status.help") + " - " : translate(player, "commands.status.help") + " - ").append(command).append(": \n");
+            StringBuilder builder = new StringBuilder("");
             if (handler == null) {
                 builder.append(translate(player, "commands.generic.command_exist_error"));
             } else {
                 Command annotation = handler.getClass().getAnnotation(Command.class);
 
-                builder.append("   ").append(translate(player, annotation.description())).append("\n");
-                builder.append(translate(player, "commands.help.usage")).append(annotation.usage());
-                if (annotation.aliases().length >= 1) {
-                    builder.append("\n").append(translate(player, "commands.help.aliases"));
-                    for (String alias : annotation.aliases()) {
-                        builder.append(alias).append(" ");
-                    }
-                }
-
-                builder.append("\n").append(translate(player, "commands.help.tip_need_permission"));
-                if(annotation.permission().isEmpty() || annotation.permission().isBlank()) {
-                    builder.append(translate(player, "commands.help.tip_need_no_permission"));
-                }
-                else {
-                    builder.append(annotation.permission());
-                }
-                builder.append(" ");
-
-                if(!annotation.permissionTargeted().isEmpty() && !annotation.permissionTargeted().isBlank()) {
-                    String permissionTargeted = annotation.permissionTargeted();
-                    builder.append(translate(player, "commands.help.tip_permission_targeted", permissionTargeted));
-                }
+                this.createCommand(builder, player, annotation);
 
                 if (player != null && !Objects.equals(annotation.permission(), "") && !player.getAccount().hasPermission(annotation.permission())) {
-                    builder.append("\n ");
-                    builder.append(translate(player, "commands.help.warn_player_has_no_permission"));
+                    builder.append("\n\t").append(translate(player, "commands.help.warn_player_has_no_permission"));
                 }
             }
 
@@ -72,67 +71,12 @@ public final class HelpCommand implements CommandHandler {
     }
 
     void SendAllHelpMessage(Player player, List<Command> annotations) {
-        if (player == null) {
-            StringBuilder builder = new StringBuilder("\n" + translate(player, "commands.help.available_commands") + "\n");
+        StringBuilder builder = new StringBuilder(translate(player, "commands.help.available_commands"));
         annotations.forEach(annotation -> {
-                builder.append(annotation.label()).append("\n");
-                builder.append("   ").append(translate(player, annotation.description())).append("\n");
-                builder.append(translate(player, "commands.help.usage")).append(annotation.usage());
-                if (annotation.aliases().length >= 1) {
-                    builder.append("\n").append(translate(player, "commands.help.aliases"));
-                    for (String alias : annotation.aliases()) {
-                        builder.append(alias).append(" ");
-                    }
-                }
-                builder.append("\n").append(translate(player, "commands.help.tip_need_permission"));
-                if(annotation.permission().isEmpty() || annotation.permission().isBlank()) {
-                    builder.append(translate(player, "commands.help.tip_need_no_permission"));
-                }
-                else {
-                    builder.append(annotation.permission());
-                }
-
-                builder.append(" ");
-
-                if(!annotation.permissionTargeted().isEmpty() && !annotation.permissionTargeted().isBlank()) {
-                    String permissionTargeted = annotation.permissionTargeted();
-                    builder.append(translate(player, "commands.help.tip_permission_targeted", permissionTargeted));
-                }
-
+            this.createCommand(builder, player, annotation);
             builder.append("\n");
         });
 
-            CommandHandler.sendMessage(null, builder.toString());
-        } else {
-            CommandHandler.sendMessage(player, translate(player, "commands.help.available_commands"));
-            annotations.forEach(annotation -> {
-                StringBuilder builder = new StringBuilder(annotation.label()).append("\n");
-                builder.append("   ").append(translate(player, annotation.description())).append("\n");
-                builder.append(translate(player, "commands.help.usage")).append(annotation.usage());
-                if (annotation.aliases().length >= 1) {
-                    builder.append("\n").append(translate(player, "commands.help.aliases"));
-                    for (String alias : annotation.aliases()) {
-                        builder.append(alias).append(" ");
-                    }
-                }
-                builder.append("\n").append(translate(player, "commands.help.tip_need_permission"));
-                if(annotation.permission().isEmpty() || annotation.permission().isBlank()) {
-                    builder.append(translate(player, "commands.help.tip_need_no_permission"));
-                }
-                else {
-                    builder.append(annotation.permission());
-                }
-
-                builder.append(" ");
-
-                if(!annotation.permissionTargeted().isEmpty() && !annotation.permissionTargeted().isBlank()) {
-                    String permissionTargeted = annotation.permissionTargeted();
-                    builder.append(translate(player, "commands.help.tip_permission_targeted", permissionTargeted));
-                }
-
-
         CommandHandler.sendMessage(player, builder.toString());
-            });
-        }
     }
 }

src/main/java/emu/grasscutter/command/commands/SendMailCommand.java

@@ -65,7 +65,7 @@ public final class SendMailCommand implements CommandHandler {
                 switch (args.get(0).toLowerCase()) {
                     case "stop" -> {
                         mailBeingConstructed.remove(senderId);
-                        CommandHandler.sendMessage(sender, translate(sender, "commands.sendMail.sendCancel"));
+                        CommandHandler.sendMessage(sender, translate(sender, "commands.sendMail.send_cancel"));
                         return;
                     }
                     case "finish" -> {