everness/i18n.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Script to generate the template file and update the translation files.
# Copy the script into the mod or modpack root folder and run it there.
#
# Copyright (C) 2019 Joachim Stolberg, 2020 FaceDeer, 2020 Louis Royer
# LGPLv2.1+
#
# See https://github.com/minetest-tools/update_translations for
# potential future updates to this script.

from __future__ import print_function
import os, fnmatch, re, shutil, errno
from sys import argv as _argv
from sys import stderr as _stderr

# Running params
params = {"recursive": False,
    "help": False,
    "mods": False,
    "verbose": False,
    "folders": [],
    "no-old-file": False,
    "break-long-lines": False,
    "sort": False,
    "print-source": False,
    "truncate-unused": False,
}
# Available CLI options
options = {"recursive": ['--recursive', '-r'],
    "help": ['--help', '-h'],
    "mods": ['--installed-mods', '-m'],
    "verbose": ['--verbose', '-v'],
    "no-old-file": ['--no-old-file', '-O'],
    "break-long-lines": ['--break-long-lines', '-b'],
    "sort": ['--sort', '-s'],
    "print-source": ['--print-source', '-p'],
    "truncate-unused": ['--truncate-unused', '-t'],
}

# Strings longer than this will have extra space added between
# them in the translation files to make it easier to distinguish their
# beginnings and endings at a glance
doublespace_threshold = 80

def set_params_folders(tab: list):
    '''Initialize params["folders"] from CLI arguments.'''
    # Discarding argument 0 (tool name)
    for param in tab[1:]:
        stop_param = False
        for option in options:
            if param in options[option]:
                stop_param = True
                break
        if not stop_param:
            params["folders"].append(os.path.abspath(param))

def set_params(tab: list):
    '''Initialize params from CLI arguments.'''
    for option in options:
        for option_name in options[option]:
            if option_name in tab:
                params[option] = True
                break

def print_help(name):
    '''Prints some help message.'''
    print(f'''SYNOPSIS
    {name} [OPTIONS] [PATHS...]
DESCRIPTION
    {', '.join(options["help"])}
        prints this help message
    {', '.join(options["recursive"])}
        run on all subfolders of paths given
    {', '.join(options["mods"])}
        run on locally installed modules
    {', '.join(options["no-old-file"])}
        do not create *.old files
    {', '.join(options["sort"])}
        sort output strings alphabetically
    {', '.join(options["break-long-lines"])}
        add extra line breaks before and after long strings
    {', '.join(options["print-source"])}
	add comments denoting the source file
    {', '.join(options["verbose"])}
        add output information
    {', '.join(options["truncate-unused"])}
        delete unused strings from files
''')


def main():
    '''Main function'''
    set_params(_argv)
    set_params_folders(_argv)
    if params["help"]:
        print_help(_argv[0])
    elif params["recursive"] and params["mods"]:
        print("Option --installed-mods is incompatible with --recursive")
    else:
        # Add recursivity message
        print("Running ", end='')
        if params["recursive"]:
            print("recursively ", end='')
        # Running
        if params["mods"]:
            print(f"on all locally installed modules in {os.path.expanduser('~/.minetest/mods/')}")
            run_all_subfolders(os.path.expanduser("~/.minetest/mods"))
        elif len(params["folders"]) >= 2:
            print("on folder list:", params["folders"])
            for f in params["folders"]:
                if params["recursive"]:
                    run_all_subfolders(f)
                else:
                    update_folder(f)
        elif len(params["folders"]) == 1:
            print("on folder", params["folders"][0])
            if params["recursive"]:
                run_all_subfolders(params["folders"][0])
            else:
                update_folder(params["folders"][0])
        else:
            print("on folder", os.path.abspath("./"))
            if params["recursive"]:
                run_all_subfolders(os.path.abspath("./"))
            else:
                update_folder(os.path.abspath("./"))

#group 2 will be the string, groups 1 and 3 will be the delimiters (" or ')
#See https://stackoverflow.com/questions/46967465/regex-match-text-in-either-single-or-double-quote
pattern_lua_s = re.compile(r'[\.=^\t,{\(\s]N?S\(\s*(["\'])((?:\\\1|(?:(?!\1)).)*)(\1)[\s,\)]', re.DOTALL)
pattern_lua_fs = re.compile(r'[\.=^\t,{\(\s]N?FS\(\s*(["\'])((?:\\\1|(?:(?!\1)).)*)(\1)[\s,\)]', re.DOTALL)
pattern_lua_bracketed_s = re.compile(r'[\.=^\t,{\(\s]N?S\(\s*\[\[(.*?)\]\][\s,\)]', re.DOTALL)
pattern_lua_bracketed_fs = re.compile(r'[\.=^\t,{\(\s]N?FS\(\s*\[\[(.*?)\]\][\s,\)]', re.DOTALL)

# Handles "concatenation" .. " of strings"
pattern_concat = re.compile(r'["\'][\s]*\.\.[\s]*["\']', re.DOTALL)

pattern_tr = re.compile(r'(.*?[^@])=(.*)')
pattern_name = re.compile(r'^name[ ]*=[ ]*([^ \n]*)')
pattern_tr_filename = re.compile(r'\.tr$')
pattern_po_language_code = re.compile(r'(.*)\.po$')

#attempt to read the mod's name from the mod.conf file or folder name. Returns None on failure
def get_modname(folder):
    try:
        with open(os.path.join(folder, "mod.conf"), "r", encoding='utf-8') as mod_conf:
            for line in mod_conf:
                match = pattern_name.match(line)
                if match:
                    return match.group(1)
    except FileNotFoundError:
        if not os.path.isfile(os.path.join(folder, "modpack.txt")):
            folder_name = os.path.basename(folder)
            # Special case when run in Minetest's builtin directory
            if folder_name == "builtin":
                return "__builtin"
            else:
                return folder_name
        else:
            return None
    return None

#If there are already .tr files in /locale, returns a list of their names
def get_existing_tr_files(folder):
    out = []
    for root, dirs, files in os.walk(os.path.join(folder, 'locale/')):
        for name in files:
            if pattern_tr_filename.search(name):
                out.append(name)
    return out

# A series of search and replaces that massage a .po file's contents into
# a .tr file's equivalent
def process_po_file(text):
    # The first three items are for unused matches
    text = re.sub(r'#~ msgid "', "", text)
    text = re.sub(r'"\n#~ msgstr ""\n"', "=", text)
    text = re.sub(r'"\n#~ msgstr "', "=", text)
    # comment lines
    text = re.sub(r'#.*\n', "", text)
    # converting msg pairs into "=" pairs
    text = re.sub(r'msgid "', "", text)
    text = re.sub(r'"\nmsgstr ""\n"', "=", text)
    text = re.sub(r'"\nmsgstr "', "=", text)
    # various line breaks and escape codes
    text = re.sub(r'"\n"', "", text)
    text = re.sub(r'"\n', "\n", text)
    text = re.sub(r'\\"', '"', text)
    text = re.sub(r'\\n', '@n', text)
    # remove header text
    text = re.sub(r'=Project-Id-Version:.*\n', "", text)
    # remove double-spaced lines
    text = re.sub(r'\n\n', '\n', text)
    return text

# Go through existing .po files and, if a .tr file for that language
# *doesn't* exist, convert it and create it.
# The .tr file that results will subsequently be reprocessed so
# any "no longer used" strings will be preserved.
# Note that "fuzzy" tags will be lost in this process.
def process_po_files(folder, modname):
    for root, dirs, files in os.walk(os.path.join(folder, 'locale/')):
        for name in files:
            code_match = pattern_po_language_code.match(name)
            if code_match == None:
                continue
            language_code = code_match.group(1)
            tr_name = f'{modname}.{language_code}.tr'
            tr_file = os.path.join(root, tr_name)
            if os.path.exists(tr_file):
                if params["verbose"]:
                    print(f"{tr_name} already exists, ignoring {name}")
                continue
            fname = os.path.join(root, name)
            with open(fname, "r", encoding='utf-8') as po_file:
                if params["verbose"]:
                    print(f"Importing translations from {name}")
                text = process_po_file(po_file.read())
                with open(tr_file, "wt", encoding='utf-8') as tr_out:
                    tr_out.write(text)

# from https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python/600612#600612
# Creates a directory if it doesn't exist, silently does
# nothing if it already exists
def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc: # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else: raise

# Converts the template dictionary to a text to be written as a file
# dKeyStrings is a dictionary of localized string to source file sets
# dOld is a dictionary of existing translations and comments from
# the previous version of this text
def strings_to_text(dkeyStrings, dOld, mod_name, header_comments):
    lOut = [f"# textdomain: {mod_name}"]
    if header_comments is not None:
        lOut.append(header_comments)
    
    dGroupedBySource = {}

    for key in dkeyStrings:
        sourceList = list(dkeyStrings[key])
        if params["sort"]:
            sourceList.sort()
        sourceString = "\n".join(sourceList)
        listForSource = dGroupedBySource.get(sourceString, [])
        listForSource.append(key)
        dGroupedBySource[sourceString] = listForSource
    
    lSourceKeys = list(dGroupedBySource.keys())
    lSourceKeys.sort()
    for source in lSourceKeys:
        localizedStrings = dGroupedBySource[source]
        if params["sort"]:
            localizedStrings.sort()
        if params["print-source"]:
            if lOut[-1] != "":
                lOut.append("")
            lOut.append(source)
        for localizedString in localizedStrings:
            val = dOld.get(localizedString, {})
            translation = val.get("translation", "")
            comment = val.get("comment")
            if params["break-long-lines"] and len(localizedString) > doublespace_threshold and not lOut[-1] == "":
                lOut.append("")
            if comment != None and comment != "" and not comment.startswith("# textdomain:"):
                lOut.append(comment)
            lOut.append(f"{localizedString}={translation}")
            if params["break-long-lines"] and len(localizedString) > doublespace_threshold:
                lOut.append("")


    unusedExist = False
    if not params["truncate-unused"]:
        for key in dOld:
            if key not in dkeyStrings:
                val = dOld[key]
                translation = val.get("translation")
                comment = val.get("comment")
                # only keep an unused translation if there was translated
                # text or a comment associated with it
                if translation != None and (translation != "" or comment):
                    if not unusedExist:
                        unusedExist = True
                        lOut.append("\n\n##### not used anymore #####\n")
                    if params["break-long-lines"] and len(key) > doublespace_threshold and not lOut[-1] == "":
                        lOut.append("")
                    if comment != None:
                        lOut.append(comment)
                    lOut.append(f"{key}={translation}")
                    if params["break-long-lines"] and len(key) > doublespace_threshold:
                        lOut.append("")
    return "\n".join(lOut) + '\n'

# Writes a template.txt file
# dkeyStrings is the dictionary returned by generate_template
def write_template(templ_file, dkeyStrings, mod_name):
    # read existing template file to preserve comments
    existing_template = import_tr_file(templ_file)
    
    text = strings_to_text(dkeyStrings, existing_template[0], mod_name, existing_template[2])
    mkdir_p(os.path.dirname(templ_file))
    with open(templ_file, "wt", encoding='utf-8') as template_file:
        template_file.write(text)


# Gets all translatable strings from a lua file
def read_lua_file_strings(lua_file):
    lOut = []
    with open(lua_file, encoding='utf-8') as text_file:
        text = text_file.read()
        #TODO remove comments here

        text = re.sub(pattern_concat, "", text)

        strings = []
        for s in pattern_lua_s.findall(text):
            strings.append(s[1])
        for s in pattern_lua_bracketed_s.findall(text):
            strings.append(s)
        for s in pattern_lua_fs.findall(text):
            strings.append(s[1])
        for s in pattern_lua_bracketed_fs.findall(text):
            strings.append(s)
                
        for s in strings:
            s = re.sub(r'"\.\.\s+"', "", s)
            s = re.sub("@[^@=0-9]", "@@", s)
            s = s.replace('\\"', '"')
            s = s.replace("\\'", "'")
            s = s.replace("\n", "@n")
            s = s.replace("\\n", "@n")
            s = s.replace("=", "@=")
            lOut.append(s)
    return lOut

# Gets strings from an existing translation file
# returns both a dictionary of translations
# and the full original source text so that the new text
# can be compared to it for changes.
# Returns also header comments in the third return value.
def import_tr_file(tr_file):
    dOut = {}
    text = None
    header_comment = None
    if os.path.exists(tr_file):
        with open(tr_file, "r", encoding='utf-8') as existing_file :
            # save the full text to allow for comparison
            # of the old version with the new output
            text = existing_file.read()
            existing_file.seek(0)
            # a running record of the current comment block
            # we're inside, to allow preceeding multi-line comments
            # to be retained for a translation line
            latest_comment_block = None
            for line in existing_file.readlines():
                line = line.rstrip('\n')
                if line.startswith("###"):
                    if header_comment is None and not latest_comment_block is None:
                        # Save header comments
                        header_comment = latest_comment_block
                        # Strip textdomain line
                        tmp_h_c = ""
                        for l in header_comment.split('\n'):
                            if not l.startswith("# textdomain:"):
                                tmp_h_c += l + '\n'
                        header_comment = tmp_h_c

                    # Reset comment block if we hit a header
                    latest_comment_block = None
                    continue
                elif line.startswith("#"):
                    # Save the comment we're inside
                    if not latest_comment_block:
                        latest_comment_block = line
                    else:
                        latest_comment_block = latest_comment_block + "\n" + line
                    continue
                match = pattern_tr.match(line)
                if match:
                    # this line is a translated line
                    outval = {}
                    outval["translation"] = match.group(2)
                    if latest_comment_block:
                        # if there was a comment, record that.
                        outval["comment"] = latest_comment_block
                    latest_comment_block = None
                    dOut[match.group(1)] = outval
    return (dOut, text, header_comment)

# Walks all lua files in the mod folder, collects translatable strings,
# and writes it to a template.txt file
# Returns a dictionary of localized strings to source file sets
# that can be used with the strings_to_text function.
def generate_template(folder, mod_name):
    dOut = {}
    for root, dirs, files in os.walk(folder):
        for name in files:
            if fnmatch.fnmatch(name, "*.lua"):
                fname = os.path.join(root, name)
                found = read_lua_file_strings(fname)
                if params["verbose"]:
                    print(f"{fname}: {str(len(found))} translatable strings")

                for s in found:
                    sources = dOut.get(s, set())
                    sources.add(f"### {os.path.basename(fname)} ###")
                    dOut[s] = sources
                    
    if len(dOut) == 0:
        return None
    templ_file = os.path.join(folder, "locale/template.txt")
    write_template(templ_file, dOut, mod_name)
    return dOut

# Updates an existing .tr file, copying the old one to a ".old" file
# if any changes have happened
# dNew is the data used to generate the template, it has all the
# currently-existing localized strings
def update_tr_file(dNew, mod_name, tr_file):
    if params["verbose"]:
        print(f"updating {tr_file}")

    tr_import = import_tr_file(tr_file)
    dOld = tr_import[0]
    textOld = tr_import[1]

    textNew = strings_to_text(dNew, dOld, mod_name, tr_import[2])

    if textOld and textOld != textNew:
        print(f"{tr_file} has changed.")
        if not params["no-old-file"]:
            shutil.copyfile(tr_file, f"{tr_file}.old")

    with open(tr_file, "w", encoding='utf-8') as new_tr_file:
        new_tr_file.write(textNew)

# Updates translation files for the mod in the given folder
def update_mod(folder):
    modname = get_modname(folder)
    if modname is not None:
        process_po_files(folder, modname)
        print(f"Updating translations for {modname}")
        data = generate_template(folder, modname)
        if data == None:
            print(f"No translatable strings found in {modname}")
        else:
            for tr_file in get_existing_tr_files(folder):
                update_tr_file(data, modname, os.path.join(folder, "locale/", tr_file))
    else:
        print(f"\033[31mUnable to find modname in folder {folder}.\033[0m", file=_stderr)
        exit(1)

# Determines if the folder being pointed to is a mod or a mod pack
# and then runs update_mod accordingly
def update_folder(folder):
    is_modpack = os.path.exists(os.path.join(folder, "modpack.txt")) or os.path.exists(os.path.join(folder, "modpack.conf"))
    if is_modpack:
        subfolders = [f.path for f in os.scandir(folder) if f.is_dir() and not f.name.startswith('.')]
        for subfolder in subfolders:
            update_mod(subfolder)
    else:
        update_mod(folder)
    print("Done.")

def run_all_subfolders(folder):
    for modfolder in [f.path for f in os.scandir(folder) if f.is_dir() and not f.name.startswith('.')]:
        update_folder(modfolder)


main()