diff --git a/client/strings/.gitignore b/client/strings/.gitignore new file mode 100644 index 000000000..e14da3b62 --- /dev/null +++ b/client/strings/.gitignore @@ -0,0 +1 @@ +string_syncer_paths.txt diff --git a/client/strings/string_syncer.py b/client/strings/string_syncer.py new file mode 100644 index 000000000..79f41739c --- /dev/null +++ b/client/strings/string_syncer.py @@ -0,0 +1,328 @@ +# Python 3.7 + +############## +# Summary +############## +# This script copies existing translations between the Audiobookshelf server and app repos. +# Both repositiories must be cloned locally. + +# The relative path to both repositories is stored in `string_syncer_paths.txt` in this same +# directory and is not tracked by Git (so there is not a hard coded path when you clone the +# file). + +############## +# Rules +############## +# Any (non-English) files are copied between the two repositories if they do not exist. This +# is to help add new language translations. After copying these files, any keys which were +# copied where the English string for the key does not match between the two repositories +# are removed from the destination file (in case of name collisions or the English string +# not being updated yet). + +# If an English string exists in BOTH repositories (both the key and value are identical) and +# a translation of the string does not exist in ONE repository, the translation is copied. + +# All keys in translation files which do not exist in the corresponding English file are +# deleted to handle server/app specific strings. + +# If the English key exists in both repositories but the value is not identical, the translations +# are not copied to help ensure there is not a name conflict. + + + +############## +# Imports +############## + +import sys +import os +import json +import shutil + +############## +# Stat handling +############## +stats = {} +dupes_found = 0 + +def print_stats(data): + # Get a set of all column keys from nested dictionaries + column_keys = sorted(set().union(*(inner_dict.keys() for inner_dict in data.values()))) + + # Create a list of headers including an empty space for the row labels + headers = [""] + list(column_keys) + + # Create a list to hold rows of the table + table = [headers] + table.append(["-" * len(cell) for cell in headers]) + + # Iterate through the top-level dictionary and create rows for each top-level key + for top_level_key, nested_dict in data.items(): + row = [top_level_key] + [nested_dict.get(column, "") for column in headers[1:]] + table.append(row) + + # Determine the maximum column width for each column + col_widths = [max(len(str(row[i])) for row in table) for i in range(len(headers))] + + # Create the table with proper alignment + formatted_table = [] + for row in table: + formatted_row = [" | ".join(f"{cell:{col_widths[i]}}" for i, cell in enumerate(row))] + formatted_table.append(formatted_row) + + # Join the rows to create the table and print it + for row in formatted_table: + print(" ".join(row)) + +############## +# Helper functions +############## + +# Write JSON object to file +def write_to_file(dictionary, path): + with open(path, "w") as file: + json.dump(dictionary, file, indent=2, sort_keys=True) + +# Check that directory exists +def is_dir(path): + return os.path.exists(path) and os.path.isdir(path) + +# Get list of all translation languages +def get_languages(path): + langs = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))] + langs.remove("en-us") + return langs + +############## +# Creation and moving +############## + +# Create missing language directories +def create_directories_from_list(directory_list, base_directory): + for directory_name in directory_list: + # Create the full path by joining the base directory and the directory name + full_path = os.path.join(base_directory, directory_name) + # Check if the directory already exists; if not, create it + if not os.path.exists(full_path): + os.makedirs(full_path) + +# Copy all missing translation files between the two repositories +def copy_missing_files(source_dir, target_dir, remove_keys): + had_missing_files = False + if not os.path.exists(source_dir) or not os.path.exists(target_dir): + raise ValueError("Source and target directories must exist.") + + for root, _, files in os.walk(source_dir): + for filename in files: + source_path = os.path.join(root, filename) + target_path = os.path.join(target_dir, filename) + + if not os.path.exists(target_path) and target_path.endswith('.json'): + shutil.copy(source_path, target_path) + print(f"Copied {source_path} -> {target_path}") + + remove_after_copy(remove_keys, target_path) + had_missing_files = True + return had_missing_files + +############## +# Key management +############## + +# Return a dictionary of all key/values in a directory +# (supports multiple files in case the JSON is broken into multiple files) +# Throws an error if a key exists more than once. +def load_json_files(directory_path): + data = {} + for filename in os.listdir(directory_path): + if filename.endswith('.json'): + file_path = os.path.join(directory_path, filename) + with open(file_path, 'r') as file: + file_data = json.load(file) + for key in file_data: + if key in data: + raise ValueError(f'Duplicate key found: {key} in {filename}') + data.update(file_data) + return data + +# Return matching key/value pairs from two dicts +def dict_same(dict1, dict2): + result = {} + diffs = {} + + print(f"=======================") + print(f"The English values differ between the two repositories for the following keys.") + print(f"Consider updating the keys to remove the name collision.") + for k,v in dict1.items(): + if k in dict2: + if dict2[k] == v: + result[k] = v + else: + diffs[k] = v + print(f" {k}") + print() + print(f"=======================") + + dupes_found = len(result.keys()) + + return result, diffs + +# This function is called after copying a file to ensure +# that the destination file does not have the wrong translation in case +# of a name collision or an out-of-date English string. +def remove_after_copy(remove_keys, path): + with open(path, 'r') as file: + file_data = json.load(file) + + # Delete all keys which had different English values + for key in remove_keys: + file_data.pop(key, None) + + write_to_file(file_data, path) + +# Get list of translations which need to be added between repositories +def translations_to_add(english_dict, server_path, app_path, lang=None): + # Load the data + with open(server_path, 'r') as file: + server_dict = json.load(file) + with open( app_path, 'r') as file: + app_dict = json.load(file) + + app_adds = 0 + server_adds = 0 + + # Copy from server to app + for key, value in server_dict.items(): + if (key in english_dict) and (key not in app_dict): + app_adds += 1 + app_dict[key] = value + + # Copy from app to server + for key, value in app_dict.items(): + if (key in english_dict) and (key not in server_dict): + server_adds += 1 + server_dict[key] = value + + # Store the data + write_to_file(server_dict, server_path) + write_to_file( app_dict, app_path) + + file_path = '/'.join(server_path.split("/")[-2:]) + if lang: + stats[lang]['Server add'] = server_adds + stats[lang]['App add'] = app_adds + +# Given the English lookup and a translation file, remove any keys that do +# not exist in the English lookup +# +# Return how many keys were removed +def remove_nonexistant(english_dict, translation_path, repo=None, lang=None): + # Load data + with open(translation_path, 'r') as file: + file_data = json.load(file) + + # Remove keys which don't exist in English + keys_to_remove = [] + for key in file_data: + if key not in english_dict: + keys_to_remove.append(key) + for key in keys_to_remove: + file_data.pop(key) + + # Write back to file + write_to_file(file_data, translation_path) + + if lang: + stats[lang][f"{repo} del"] += len(keys_to_remove) + + +############## +# Main +############## + +# Given two server directories, Do The Thing! +def main(): + stored_path = "string_syncer_paths.txt" + + # Check to see if the repository paths are set and exist + try: + with open(stored_path, 'r') as file: + repos = json.load(file) + print(f"Loaded repo paths from {stored_path}. To use different repo paths, please delete this file.") + print() + except: + # File does not exist, need command line arguments + if len(sys.argv) < 3: + print("Stored repositories not found and no command line arguments given.") + print("Command usage:") + print("") + print(" python string_syncer.py [audiobookshelf string dir] [audiobookshelf-app string dir]") + print("") + print("") + print("For example, if you are running this script from the server strings directory and the") + print("app repo is located at ../../../audiobookshelf-app, the command would be") + print("") + print(' python string_syncer.py "." "../../../audiobookshelf-app/strings"') + return 1 + + repos = {'server': sys.argv[1], 'app': sys.argv[2]} + # Store the repo paths + write_to_file(repos, stored_path) + + # Load English resource files + server_english = load_json_files( repos['server'] + "/en-us" ) + app_english = load_json_files( repos['app'] + "/en-us" ) + + # Get identical English key/value pairs + english_common, remove_keys = dict_same(server_english, app_english) + + # Get list of languages + server_langs = get_languages( repos['server'] ) + app_langs = get_languages( repos['app'] ) + + all_langs = sorted(list(set(server_langs) | set(app_langs))) + + # add all languages to the stats + for lang in all_langs: + stats[lang] = {'Server add' : 0, + 'Server del' : 0, + 'App add' : 0, + 'App del' : 0 + } + + + # Create all language directories in both repositories + create_directories_from_list( all_langs, repos['server'] ) + create_directories_from_list( all_langs, repos['app'] ) + + # Copy all missing translation files + for lang in all_langs: + server_path = repos['server'] + "/" + lang + app_path = repos['app'] + "/" + lang + + if copy_missing_files( server_path, app_path, remove_keys ): + stats[lang]['Server del'] += len(remove_keys) + if copy_missing_files( app_path, server_path, remove_keys ): + stats[lang]['App del'] += len(remove_keys) + + # After copying the files, update all files for language + for filename in os.listdir(server_path): + if filename.endswith(".json"): + server_file = server_path + "/" + filename + app_file = app_path + "/" + filename + + # Remove extra keys + remove_nonexistant(server_english, server_file, repo="Server", lang=lang) + remove_nonexistant( app_english, app_file, repo="App" , lang=lang) + + # Add identical key/values + translations_to_add(english_common, server_file, app_file, lang=lang) + + print("") + print("") + print("") + + print_stats(stats) + +if __name__ == "__main__": + main()