Commit 8843276c authored by Luke H-W's avatar Luke H-W Committed by GitHub
Browse files

Language linting (#1382)

parent bb84432b
name: "Language Lint"
on:
workflow_dispatch: ~
push:
paths:
- "**.java"
- "**.json"
branches:
- "stable"
- "development"
pull_request:
paths:
- "**.java"
- "**.json"
types:
- opened
- synchronize
- reopened
jobs:
Lint-Language-Keys:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.10' # Version range or exact version of a Python version to use, using SemVer's version range syntax
- run: python3 manage_languages.py -l
# Written for Python 3.6+
# Older versions don't retain insertion order of regular dicts
import argparse
import cmd
import json
import os
import re
from pprint import pprint
INDENT = 2
PRIMARY_LANGUAGE = 'en-US.json'
PRIMARY_FALLBACK_PREFIX = '🇺🇸' # This is invisible in-game, terminal emulators might render it
LANGUAGE_FOLDER = 'src/main/resources/languages/'
LANGUAGE_FILENAMES = sorted(os.listdir(LANGUAGE_FOLDER), key=lambda x: 'AAA' if x == PRIMARY_LANGUAGE else x)
SOURCE_FOLDER = 'src/'
SOURCE_EXTENSIONS = ('java')
def ppprint(data):
pprint(data, width=130, sort_dicts=False, compact=True)
class JsonHelpers:
@staticmethod
def load(filename: str) -> dict:
with open(filename, 'r') as file:
return json.load(file)
@staticmethod
def save(filename: str, data: dict) -> None:
with open(filename, 'w', encoding='utf-8', newline='\n') as file:
json.dump(data, file, ensure_ascii=False, indent=INDENT)
@staticmethod
def flatten(data: dict, prefix='') -> dict:
output = {}
for key, value in data.items():
if isinstance(value, dict):
for k,v in JsonHelpers.flatten(value, f'{prefix}{key}.').items():
output[k] = v
else:
output[f'{prefix}{key}'] = value
return output
@staticmethod
def unflatten(data: dict) -> dict:
output = {}
def add_key(k: list, value, d: dict):
if len(k) == 1:
d[k[0]] = value
else:
d[k[0]] = d.get(k[0], {})
add_key(k[1:], value, d[k[0]])
for key, value in data.items():
add_key(key.split('.'), value, output)
return output
@staticmethod
def pprint_keys(keys, indent=4) -> str:
# Only strip down to one level
padding = ' ' * indent
roots = {}
for key in keys:
root, _, k = key.rpartition('.')
roots[root] = roots.get(root, [])
roots[root].append(k)
lines = []
for root, ks in roots.items():
if len(ks) > 1:
lines.append(f'{padding}{root}.[{", ".join(ks)}]')
else:
lines.append(f'{padding}{root}.{ks[0]}')
return ',\n'.join(lines)
@staticmethod
def deep_clone_and_fill(d1: dict, d2: dict, fallback_prefix=PRIMARY_FALLBACK_PREFIX) -> dict:
out = {}
for key, value in d1.items():
if isinstance(value, dict):
out[key] = JsonHelpers.deep_clone_and_fill(value, d2.get(key, {}), fallback_prefix)
else:
v2 = d2.get(key, value)
if type(value) == str and v2 == value:
out[key] = fallback_prefix + value
else:
out[key] = v2
return out
class LanguageManager:
TRANSLATION_KEY = re.compile(r'[Tt]ranslate.*"(\w+\.[\w\.]+)"')
POTENTIAL_KEY = re.compile(r'"(\w+\.[\w\.]+)"')
def __init__(self):
self.load_jsons()
def load_jsons(self):
self.language_jsons = [JsonHelpers.load(LANGUAGE_FOLDER + filename) for filename in LANGUAGE_FILENAMES]
self.flattened_jsons = [JsonHelpers.flatten(j) for j in self.language_jsons]
self.update_keys()
def update_keys(self):
self.key_sets = [set(j.keys()) for j in self.flattened_jsons]
self.common_keys = set.intersection(*self.key_sets)
self.all_keys = set.union(*self.key_sets)
self.used_keys = self.find_all_used_keys(self.all_keys)
self.missing_keys = self.used_keys - self.common_keys
self.unused_keys = self.all_keys - self.used_keys
def find_all_used_keys(self, expected_keys=[]) -> set:
# Note that this will only find string literals passed to the translate() or sendTranslatedMessage() methods!
# String variables passed to them can be checked against expected_keys
used = set()
potential = set()
for root, dirs, files in os.walk(SOURCE_FOLDER):
for file in files:
if file.rpartition('.')[-1] in SOURCE_EXTENSIONS:
filename = os.path.join(root, file)
with open(filename, 'r') as f:
data = f.read() # Loads in entire file at once
for k in self.TRANSLATION_KEY.findall(data):
used.add(k)
for k in self.POTENTIAL_KEY.findall(data):
potential.add(k)
return used | (potential & expected_keys)
def _lint_report_language(self, lang: str, keys: set, flattened: dict, primary_language_flattened: dict) -> None:
missing = self.used_keys - keys
unused = keys - self.used_keys
identical_keys = set() if (lang == PRIMARY_LANGUAGE) else {key for key in keys if primary_language_flattened.get(key, None) == flattened.get(key)}
placeholder_keys = {key for key in keys if flattened.get(key).startswith(PRIMARY_FALLBACK_PREFIX)}
p1 = f'Language {lang} has {len(missing)} missing keys and {len(unused)} unused keys.'
p2 = 'This is the primary language.' if (lang == PRIMARY_LANGUAGE) else f'{len(identical_keys)} match {PRIMARY_LANGUAGE}, {len(placeholder_keys)} have the placeholder mark.'
print(f'{p1} {p2}')
lint_categories = {
'Missing': missing,
'Unused': unused,
f'Matches {PRIMARY_LANGUAGE}': identical_keys,
'Placeholder': placeholder_keys,
}
for name, category in lint_categories.items():
if len(category) > 0:
print(name + ':')
print(JsonHelpers.pprint_keys(sorted(category)))
def lint_report(self) -> None:
print(f'There are {len(self.missing_keys)} translation keys in use that are missing from one or more language files.')
print(f'There are {len(self.unused_keys)} translation keys in language files that are not used.')
primary_language_flattened = self.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
for lang, keys, flattened in zip(LANGUAGE_FILENAMES, self.key_sets, self.flattened_jsons):
print('')
self._lint_report_language(lang, keys, flattened, primary_language_flattened)
def rename_keys(self, key_remappings: dict) -> None:
# Unfortunately we can't rename keys in-place preserving insertion order, so we have to make new dicts
for i in range(len(self.flattened_jsons)):
self.flattened_jsons[i] = {key_remappings.get(k,k):v for k,v in self.flattened_jsons[i].items()}
def update_secondary_languages(self):
# Push en_US fallback
primary_language_json = self.language_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
for filename, lang in zip(LANGUAGE_FILENAMES, self.language_jsons):
if filename != PRIMARY_LANGUAGE:
js = JsonHelpers.deep_clone_and_fill(primary_language_json, lang)
JsonHelpers.save(LANGUAGE_FOLDER + filename, js)
def update_all_languages_from_flattened(self):
for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons):
JsonHelpers.save(LANGUAGE_FOLDER + filename, JsonHelpers.unflatten(flat))
def save_flattened_languages(self, prefix='flat_'):
for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons):
JsonHelpers.save(prefix + filename, flat)
class InteractiveRename(cmd.Cmd):
intro = 'Welcome to the interactive rename shell. Type help or ? to list commands.\n'
prompt = '(rename) '
file = None
def __init__(self, language_manager: LanguageManager) -> None:
super().__init__()
self.language_manager = language_manager
self.flat_keys = [key for key in language_manager.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)].keys()]
self.mappings = {}
def do_add(self, arg):
'''
Prepare to rename an existing translation key. Will not actually rename anything until you confirm all your pending changes with 'rename'.
e.g. a single string: add commands.execution.argument_error commands.generic.invalid.argument
e.g. a group: add commands.enter_dungeon commands.new_enter_dungeon
'''
args = arg.split()
if len(args) < 2:
self.do_help('add')
return
old, new = args[:2]
if old in self.flat_keys:
self.mappings[old] = new
else:
# Check if we are renaming a higher level
if not old.endswith('.'):
old = old + '.'
results = [key for key in self.flat_keys if key.startswith(old)]
if len(results) > 0:
if not new.endswith('.'):
new = new + '.'
new_mappings = {key: key.replace(old, new) for key in results}
# Ask for confirmation
print('Will add the following mappings:')
ppprint(new_mappings)
print('Add these mappings? [y/N]')
if self.prompt_yn():
for k,v in new_mappings.items():
self.mappings[k] = v
else:
print('No translation keys matched!')
def complete_add(self, text: str, line: str, begidx: int, endidx: int) -> list:
if text == '':
return [k for k in {key.partition('.')[0] for key in self.flat_keys}]
results = [key for key in self.flat_keys if key.startswith(text)]
if len(results) > 40:
# Collapse categories
if text[-1] != '.':
text = text + '.'
level = text.count('.') + 1
new_results = {'.'.join(key.split('.')[:level]) for key in results}
return list(new_results)
return results
def do_remove(self, arg):
'''
Remove a pending rename mapping. Takes the old name of the key, not the new one.
e.g. a single key: remove commands.execution.argument_error
e.g. a group: remove commands.enter_dungeon
'''
old = arg.split()[0]
if old in self.mappings:
self.mappings.pop(old)
else:
# Check if we are renaming a higher level
if not old.endswith('.'):
old = old + '.'
results = [key for key in self.mappings if key.startswith(old)]
if len(results) > 0:
# Ask for confirmation
print('Will remove the following pending mappings:')
print(JsonHelpers.pprint_keys(results))
print('Delete these mappings? [y/N]')
if self.prompt_yn():
for key in results:
self.mappings.pop(key)
else:
print('No pending rename mappings matched!')
def complete_remove(self, text: str, line: str, begidx: int, endidx: int) -> list:
return [key for key in self.mappings if key.startswith(text)]
def do_rename(self, _arg):
'Applies pending renames and overwrites language jsons.'
# Ask for confirmation
print('Will perform the following mappings:')
ppprint(self.mappings)
print('Perform and save these rename mappings? [y/N]')
if self.prompt_yn():
self.language_manager.rename_keys(self.mappings)
self.language_manager.update_all_languages_from_flattened()
print('Renamed keys, closing')
return True
else:
print('Do you instead wish to quit without saving? [yes/N]')
if self.prompt_yn(True):
print('Left rename shell without renaming')
return True
def prompt_yn(self, strict_yes=False):
if strict_yes:
return input('(yes/N) ').lower() == 'yes'
return input('(y/N) ').lower()[0] == 'y'
def main(args: argparse.Namespace):
# print(args)
language_manager = LanguageManager()
errors = None
if args.lint_report:
language_manager.lint_report()
missing = language_manager.used_keys - language_manager.key_sets[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
if len(missing) > 0:
errors = f'[ERROR] {len(missing)} keys missing from primary language json!\n{JsonHelpers.pprint_keys(missing)}'
if prefix := args.save_flattened:
language_manager.save_flattened_languages(prefix)
if args.update:
print('Updating secondary languages')
language_manager.update_secondary_languages()
if args.interactive_rename:
language_manager.load_jsons() # Previous actions may have changed them on-disk
try:
InteractiveRename(language_manager).cmdloop()
except KeyboardInterrupt:
print('Left rename shell without renaming')
if errors:
print(errors)
exit(1)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Manage Grasscutter's language json files.")
parser.add_argument('-u', '--update', action='store_true',
help=f'Update secondary language files to conform to the layout of the primary language file ({PRIMARY_LANGUAGE}) and contain any new keys from it.')
parser.add_argument('-l', '--lint-report', action='store_true',
help='Prints a lint report, listing unused, missing, and untranslated keys among all language jsons.')
parser.add_argument('-f', '--save-flattened', const='./flat_', metavar='prefix', nargs='?',
help='Save copies of all the language jsons in a flattened key form.')
parser.add_argument('-i', '--interactive-rename', action='store_true',
help='Enter interactive rename mode, in which you can specify keys in flattened form to be renamed.')
args = parser.parse_args()
main(args)
\ No newline at end of file
......@@ -31,7 +31,7 @@ public final class DefaultAuthentication implements AuthenticationSystem {
@Override
public Account verifyUser(String details) {
Grasscutter.getLogger().info(translate("dispatch.authentication.default_unable_to_verify"));
Grasscutter.getLogger().info(translate("messages.dispatch.authentication.default_unable_to_verify"));
return null;
}
......
......@@ -106,7 +106,12 @@ public final class CommandMap {
* @return The command handler.
*/
public CommandHandler getHandler(String label) {
return this.commands.get(label);
CommandHandler handler = this.commands.get(label);
if (handler == null) {
// Try getting by alias
handler = this.aliases.get(label);
}
return handler;
}
private Player getTargetPlayer(String playerId, Player player, Player targetPlayer, List<String> args) {
......@@ -129,7 +134,7 @@ public final class CommandMap {
}
return targetPlayer;
} catch (NumberFormatException e) {
CommandHandler.sendTranslatedMessage(player, "commands.execution.uid_error");
CommandHandler.sendTranslatedMessage(player, "commands.generic.invalid.uid");
throw new IllegalArgumentException();
}
}
......@@ -177,7 +182,7 @@ public final class CommandMap {
CommandHandler.sendTranslatedMessage(player, targetPlayer.isOnline()? "commands.execution.set_target_online" : "commands.execution.set_target_offline", targetUid);
return true;
} catch (NumberFormatException e) {
CommandHandler.sendTranslatedMessage(player, "commands.execution.uid_error");
CommandHandler.sendTranslatedMessage(player, "commands.generic.invalid.uid");
return false;
}
}
......@@ -220,12 +225,9 @@ public final class CommandMap {
}
// Get command handler.
CommandHandler handler = this.commands.get(label);
if(handler == null)
// Try to get the handler by alias.
handler = this.aliases.get(label);
CommandHandler handler = this.getHandler(label);
// Check if the handler is still null.
// Check if the handler is null.
if (handler == null) {
CommandHandler.sendTranslatedMessage(player, "commands.generic.unknown_command", label);
return;
......
......@@ -221,7 +221,6 @@ public final class GiveCommand implements CommandHandler {
case ITEM_RELIQUARY:
targetPlayer.getInventory().addItems(makeArtifacts(param), ActionReason.SubfieldDrop);
CommandHandler.sendTranslatedMessage(sender, "commands.give.given_level", Integer.toString(param.id), Integer.toString(param.lvl), Integer.toString(param.amount), Integer.toString(targetPlayer.getUid()));
//CommandHandler.sendTranslatedMessage(sender, "commands.giveArtifact.success", Integer.toString(param.id), Integer.toString(targetPlayer.getUid()));
return;
default:
targetPlayer.getInventory().addItem(new GameItem(param.data, param.amount), ActionReason.SubfieldDrop);
......
package emu.grasscutter.command.commands;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.command.Command;
import emu.grasscutter.command.CommandHandler;
import emu.grasscutter.command.CommandMap;
......@@ -13,6 +12,28 @@ import static emu.grasscutter.utils.Language.translate;
@Command(label = "help", usage = "help [command]", description = "commands.help.description", targetRequirement = Command.TargetRequirement.NONE)
public final class HelpCommand implements CommandHandler {
private void createCommand(StringBuilder builder, Player player, Command annotation) {
builder.append("\n").append(annotation.label()).append(" - ").append(translate(player, annotation.description()));
builder.append("\n\t").append(translate(player, "commands.help.usage"));
if (annotation.aliases().length >= 1) {
builder.append("\n\t").append(translate(player, "commands.help.aliases"));
for (String alias : annotation.aliases()) {
builder.append(alias).append(" ");
}
}
builder.append("\n\t").append(translate(player, "commands.help.tip_need_permission"));
if(annotation.permission().isEmpty() || annotation.permission().isBlank()) {
builder.append(translate(player, "commands.help.tip_need_no_permission"));
} else {
builder.append(annotation.permission());
}
if(!annotation.permissionTargeted().isEmpty() && !annotation.permissionTargeted().isBlank()) {
String permissionTargeted = annotation.permissionTargeted();
builder.append(" ").append(translate(player, "commands.help.tip_permission_targeted", permissionTargeted));
}
}
@Override
public void execute(Player player, Player targetPlayer, List<String> args) {
if (args.size() < 1) {
......@@ -32,38 +53,16 @@ public final class HelpCommand implements CommandHandler {
} else {
String command = args.get(0);
CommandHandler handler = CommandMap.getInstance().getHandler(command);
StringBuilder builder = new StringBuilder(player == null ? "\n" + translate(player, "commands.status.help") + " - " : translate(player, "commands.status.help") + " - ").append(command).append(": \n");
StringBuilder builder = new StringBuilder("");
if (handler == null) {
builder.append(translate(player, "commands.generic.command_exist_error"));
} else {
Command annotation = handler.getClass().getAnnotation(Command.class);
builder.append(" ").append(translate(player, annotation.description())).append("\n");
builder.append(translate(player, "commands.help.usage")).append(annotation.usage());
if (annotation.aliases().length >= 1) {
builder.append("\n").append(translate(player, "commands.help.aliases"));
for (String alias : annotation.aliases()) {
builder.append(alias).append(" ");
}
}
builder.append("\n").append(translate(player, "commands.help.tip_need_permission"));
if(annotation.permission().isEmpty() || annotation.permission().isBlank()) {
builder.append(translate(player, "commands.help.tip_need_no_permission"));
}
else {
builder.append(annotation.permission());
}
builder.append(" ");
if(!annotation.permissionTargeted().isEmpty() && !annotation.permissionTargeted().isBlank()) {
String permissionTargeted = annotation.permissionTargeted();
builder.append(translate(player, "commands.help.tip_permission_targeted", permissionTargeted));
}
this.createCommand(builder, player, annotation);
if (player != null && !Objects.equals(annotation.permission(), "") && !player.getAccount().hasPermission(annotation.permission())) {
builder.append("\n ");
builder.append(translate(player, "commands.help.warn_player_has_no_permission"));
builder.append("\n\t").append(translate(player, "commands.help.warn_player_has_no_permission"));
}
}
......@@ -72,67 +71,12 @@ public final class HelpCommand implements CommandHandler {
}
void SendAllHelpMessage(Player player, List<Command> annotations) {
if (player == null) {
StringBuilder builder = new StringBuilder("\n" + translate(player, "commands.help.available_commands") + "\n");
annotations.forEach(annotation -> {
builder.append(annotation.label()).append("\n");
builder.append(" ").append(translate(player, annotation.description())).append("\n");
builder.append(translate(player, "commands.help.usage")).append(annotation.usage());
if (annotation.aliases().length >= 1) {
builder.append("\n").append(translate(player, "commands.help.aliases"));
for (String alias : annotation.aliases()) {
builder.append(alias).append(" ");
}
}
builder.append("\n").append(translate(player, "commands.help.tip_need_permission"));
if(annotation.permission().isEmpty() || annotation.permission().isBlank()) {
builder.append(translate(player, "commands.help.tip_need_no_permission"));
}
else {
builder.append(annotation.permission());
}
builder.append(" ");
if(!annotation.permissionTargeted().isEmpty() && !annotation.permissionTargeted().isBlank()) {
String permissionTargeted = annotation.permissionTargeted();
builder.append(translate(player, "commands.help.tip_permission_targeted", permissionTargeted));
}
StringBuilder builder = new StringBuilder(translate(player, "commands.help.available_commands"));
annotations.forEach(annotation -> {
this.createCommand(builder, player, annotation);
builder.append("\n");
});
builder.append("\n");
});
CommandHandler.sendMessage(null, builder.toString());
} else {
CommandHandler.sendMessage(player, translate(player, "commands.help.available_commands"));
annotations.forEach(annotation -> {
StringBuilder builder = new StringBuilder(annotation.label()).append("\n");
builder.append(" ").append(translate(player, annotation.description())).append("\n");
builder.append(translate(player, "commands.help.usage")).append(annotation.usage());
if (annotation.aliases().length >= 1) {
builder.append("\n").append(translate(player, "commands.help.aliases"));
for (String alias : annotation.aliases()) {
builder.append(alias).append(" ");
}
}
builder.append("\n").append(translate(player, "commands.help.tip_need_permission"));
if(annotation.permission().isEmpty() || annotation.permission().isBlank()) {
builder.append(translate(player, "commands.help.tip_need_no_permission"));
}
else {
builder.append(annotation.permission());
}
builder.append(" ");
if(!annotation.permissionTargeted().isEmpty() && !annotation.permissionTargeted().isBlank()) {
String permissionTargeted = annotation.permissionTargeted();
builder.append(translate(player, "commands.help.tip_permission_targeted", permissionTargeted));
}
CommandHandler.sendMessage(player, builder.toString());
});
}
CommandHandler.sendMessage(player, builder.toString());
}
}
......@@ -65,7 +65,7 @@ public final class SendMailCommand implements CommandHandler {
switch (args.get(0).toLowerCase()) {
case "stop" -> {
mailBeingConstructed.remove(senderId);
CommandHandler.sendMessage(sender, translate(sender, "commands.sendMail.sendCancel"));
CommandHandler.sendMessage(sender, translate(sender, "commands.sendMail.send_cancel"));
return;
}
case "finish" -> {
......
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment