Added string_syncer script

client/strings/.gitignore

@@ -0,0 +1 @@
+string_syncer_paths.txt

client/strings/string_syncer.py

@@ -0,0 +1,328 @@
+# Python 3.7
+
+##############
+# Summary
+##############
+# This script copies existing translations between the Audiobookshelf server and app repos.
+# Both repositiories must be cloned locally.
+
+# The relative path to both repositories is stored in `string_syncer_paths.txt` in this same
+# directory and is not tracked by Git (so there is not a hard coded path when you clone the
+# file).
+
+##############
+# Rules
+##############
+# Any (non-English) files are copied between the two repositories if they do not exist. This
+# is to help add new language translations. After copying these files, any keys which were
+# copied where the English string for the key does not match between the two repositories
+# are removed from the destination file (in case of name collisions or the English string
+# not being updated yet).
+
+# If an English string exists in BOTH repositories (both the key and value are identical) and
+# a translation of the string does not exist in ONE repository, the translation is copied.
+
+# All keys in translation files which do not exist in the corresponding English file are
+# deleted to handle server/app specific strings.
+
+# If the English key exists in both repositories but the value is not identical, the translations
+# are not copied to help ensure there is not a name conflict.
+
+
+
+##############
+# Imports
+##############
+
+import sys
+import os
+import json
+import shutil
+
+##############
+# Stat handling
+##############
+stats = {}
+dupes_found = 0
+
+def print_stats(data):
+    # Get a set of all column keys from nested dictionaries
+    column_keys = sorted(set().union(*(inner_dict.keys() for inner_dict in data.values())))
+
+    # Create a list of headers including an empty space for the row labels
+    headers = [""] + list(column_keys)
+
+    # Create a list to hold rows of the table
+    table = [headers]
+    table.append(["-" * len(cell) for cell in headers])
+
+    # Iterate through the top-level dictionary and create rows for each top-level key
+    for top_level_key, nested_dict in data.items():
+        row = [top_level_key] + [nested_dict.get(column, "") for column in headers[1:]]
+        table.append(row)
+
+    # Determine the maximum column width for each column
+    col_widths = [max(len(str(row[i])) for row in table) for i in range(len(headers))]
+
+    # Create the table with proper alignment
+    formatted_table = []
+    for row in table:
+        formatted_row = [" | ".join(f"{cell:{col_widths[i]}}" for i, cell in enumerate(row))]
+        formatted_table.append(formatted_row)
+
+    # Join the rows to create the table and print it
+    for row in formatted_table:
+        print(" ".join(row))
+
+##############
+# Helper functions
+##############
+
+# Write JSON object to file
+def write_to_file(dictionary, path):
+    with open(path, "w") as file:
+        json.dump(dictionary, file, indent=2, sort_keys=True)
+
+# Check that directory exists
+def is_dir(path):
+    return os.path.exists(path) and os.path.isdir(path)
+
+# Get list of all translation languages
+def get_languages(path):
+    langs = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))]
+    langs.remove("en-us")
+    return langs
+
+##############
+# Creation and moving
+##############
+
+# Create missing language directories
+def create_directories_from_list(directory_list, base_directory):
+    for directory_name in directory_list:
+        # Create the full path by joining the base directory and the directory name
+        full_path = os.path.join(base_directory, directory_name)
+        # Check if the directory already exists; if not, create it
+        if not os.path.exists(full_path):
+            os.makedirs(full_path)
+
+# Copy all missing translation files between the two repositories
+def copy_missing_files(source_dir, target_dir, remove_keys):
+    had_missing_files = False
+    if not os.path.exists(source_dir) or not os.path.exists(target_dir):
+        raise ValueError("Source and target directories must exist.")
+
+    for root, _, files in os.walk(source_dir):
+        for filename in files:
+            source_path = os.path.join(root, filename)
+            target_path = os.path.join(target_dir, filename)
+
+            if not os.path.exists(target_path) and target_path.endswith('.json'):
+                shutil.copy(source_path, target_path)
+                print(f"Copied {source_path}    ->    {target_path}")
+
+                remove_after_copy(remove_keys, target_path)
+                had_missing_files = True
+    return had_missing_files
+
+##############
+# Key management
+##############
+
+# Return a dictionary of all key/values in a directory
+# (supports multiple files in case the JSON is broken into multiple files)
+# Throws an error if a key exists more than once.
+def load_json_files(directory_path):
+    data = {}
+    for filename in os.listdir(directory_path):
+        if filename.endswith('.json'):
+            file_path = os.path.join(directory_path, filename)
+            with open(file_path, 'r') as file:
+                file_data = json.load(file)
+                for key in file_data:
+                    if key in data:
+                        raise ValueError(f'Duplicate key found: {key} in {filename}')
+                data.update(file_data)
+    return data
+
+# Return matching key/value pairs from two dicts
+def dict_same(dict1, dict2):
+    result = {}
+    diffs  = {}
+
+    print(f"=======================")
+    print(f"The English values differ between the two repositories for the following keys.")
+    print(f"Consider updating the keys to remove the name collision.")
+    for k,v in dict1.items():
+        if k in dict2:
+            if dict2[k] == v:
+                result[k] = v
+            else:
+                diffs[k] = v
+                print(f"  {k}")
+    print()
+    print(f"=======================")
+    
+    dupes_found = len(result.keys())
+
+    return result, diffs
+
+# This function is called after copying a file to ensure
+# that the destination file does not have the wrong translation in case
+# of a name collision or an out-of-date English string.
+def remove_after_copy(remove_keys, path):
+    with open(path, 'r') as file:
+        file_data = json.load(file)
+
+    # Delete all keys which had different English values
+    for key in remove_keys:
+        file_data.pop(key, None)
+
+    write_to_file(file_data, path)
+
+# Get list of translations which need to be added between repositories
+def translations_to_add(english_dict, server_path, app_path, lang=None):
+    # Load the data
+    with open(server_path, 'r') as file:
+        server_dict = json.load(file)
+    with open(   app_path, 'r') as file:
+        app_dict    = json.load(file)
+
+    app_adds    = 0
+    server_adds = 0
+
+    # Copy from server to app
+    for key, value in server_dict.items():
+        if (key in english_dict) and (key not in app_dict):
+            app_adds      += 1
+            app_dict[key] = value
+
+    # Copy from app to server
+    for key, value in app_dict.items():
+        if (key in english_dict) and (key not in server_dict):
+            server_adds      += 1
+            server_dict[key] = value
+
+    # Store the data
+    write_to_file(server_dict, server_path)
+    write_to_file(   app_dict,    app_path)
+
+    file_path = '/'.join(server_path.split("/")[-2:])
+    if lang:
+        stats[lang]['Server add'] = server_adds
+        stats[lang]['App add']    = app_adds
+
+# Given the English lookup and a translation file, remove any keys that do
+# not exist in the English lookup
+# 
+# Return how many keys were removed
+def remove_nonexistant(english_dict, translation_path, repo=None, lang=None):
+    # Load data
+    with open(translation_path, 'r') as file:
+        file_data = json.load(file)
+
+    # Remove keys which don't exist in English
+    keys_to_remove = []
+    for key in file_data:
+        if key not in english_dict:
+            keys_to_remove.append(key)
+    for key in keys_to_remove:
+        file_data.pop(key)
+
+    # Write back to file
+    write_to_file(file_data, translation_path)
+
+    if lang:
+        stats[lang][f"{repo} del"] += len(keys_to_remove)
+
+
+##############
+# Main
+##############
+
+# Given two server directories, Do The Thing!
+def main():
+    stored_path = "string_syncer_paths.txt"
+
+    # Check to see if the repository paths are set and exist
+    try:
+        with open(stored_path, 'r') as file:
+            repos = json.load(file)
+        print(f"Loaded repo paths from {stored_path}. To use different repo paths, please delete this file.")
+        print()
+    except:
+        # File does not exist, need command line arguments
+        if len(sys.argv) < 3:
+            print("Stored repositories not found and no command line arguments given.")
+            print("Command usage:")
+            print("")
+            print("  python string_syncer.py [audiobookshelf string dir] [audiobookshelf-app string dir]")
+            print("")
+            print("")
+            print("For example, if you are running this script from the server strings directory and the")
+            print("app repo is located at ../../../audiobookshelf-app, the command would be")
+            print("")
+            print('  python string_syncer.py "." "../../../audiobookshelf-app/strings"')
+            return 1
+
+        repos = {'server': sys.argv[1], 'app': sys.argv[2]}
+        # Store the repo paths
+        write_to_file(repos, stored_path)
+
+    # Load English resource files
+    server_english = load_json_files( repos['server'] + "/en-us" )
+    app_english    = load_json_files( repos['app']    + "/en-us" )
+
+    # Get identical English key/value pairs
+    english_common, remove_keys = dict_same(server_english, app_english)
+    
+    # Get list of languages
+    server_langs   = get_languages( repos['server'] )
+    app_langs      = get_languages( repos['app']    )
+
+    all_langs      = sorted(list(set(server_langs) | set(app_langs)))
+
+    # add all languages to the stats
+    for lang in all_langs:
+        stats[lang] = {'Server add' : 0,
+                       'Server del' : 0,
+                       'App add'    : 0,
+                       'App del'    : 0
+                      }
+
+
+    # Create all language directories in both repositories
+    create_directories_from_list( all_langs, repos['server'] )
+    create_directories_from_list( all_langs, repos['app']    )
+
+    # Copy all missing translation files
+    for lang in all_langs:
+        server_path = repos['server'] + "/" + lang
+        app_path    = repos['app']    + "/" + lang
+
+        if copy_missing_files( server_path,    app_path, remove_keys ):
+            stats[lang]['Server del'] += len(remove_keys)
+        if copy_missing_files(    app_path, server_path, remove_keys ):
+            stats[lang]['App del'] += len(remove_keys)
+
+        # After copying the files, update all files for language
+        for filename in os.listdir(server_path):
+            if filename.endswith(".json"):
+                server_file = server_path + "/" + filename
+                app_file    = app_path    + "/" + filename
+
+                # Remove extra keys
+                remove_nonexistant(server_english, server_file, repo="Server", lang=lang)
+                remove_nonexistant(   app_english,    app_file, repo="App"   , lang=lang)
+
+                # Add identical key/values
+                translations_to_add(english_common, server_file, app_file, lang=lang)
+
+    print("")
+    print("")
+    print("")
+
+    print_stats(stats)
+
+if __name__ == "__main__":
+    main()