lint_po

#!/usr/bin/env python3

"""Checks and Unifies PO headers and rewraps PO files to 79 chars.

Usage:
./lint_po --help

Default is check mode where the error are listed but not fixed.
With --fix the files get changed and unified.

Run for all po files in the working directory (including subdirs):
./lint_po

Run with a list of files:
./lint_po file1.de.po file2.fr.po

Run for all po files that are staged for git commit:
./lint_po --cached

Run for all po files of one language in the current directory (recursively):
./lint_po --lang de

When modifying lint_po (this script), you should check if the current type
annotations match, using `mypy` (`apt install mypy`):

mypy lint_po
"""

import argparse
import contextlib
import copy
import functools
import glob
import itertools
import logging
import multiprocessing
import os.path
import re
import shutil
import subprocess
import sys
import tempfile

try:
    import polib  # type: ignore
except ImportError:
    sys.exit("You need to install python3-polib to use this program.")

from typing import Dict, List, Tuple

# i18nspector issues, that we accept
I18NSPECTOR_ACCEPT = [
        "boilerplate-in-date",
        "boilerplate-in-initial-comments",
        "boilerplate-in-language-team",
        "boilerplate-in-last-translator",
        "boilerplate-in-project-id-version",
        "codomain-error-in-plural-forms",
        "codomain-error-in-unused-plural-forms",
        "conflict-marker-in-header-entry",
        "fuzzy-header-entry",
        "incorrect-plural-forms",
        "invalid-content-transfer-encoding",
        "invalid-date",
        "invalid-language",
        "invalid-last-translator",
        "language-team-equal-to-last-translator",
        "no-language-header-field",
        "no-package-name-in-project-id-version",
        "no-plural-forms-header-field",
        "no-report-msgid-bugs-to-header-field",
        "no-version-in-project-id-version",
        "stray-previous-msgid",
        "unable-to-determine-language",
        "unknown-message-flag",
        "unknown-poedit-language",
        "unusual-plural-forms",
        "unusual-unused-plural-forms",
        ]


class NoLanguageError(Exception):
    def __init__(self, fname):
        self.fname = fname

    def __str__(self):
        return(
            "Can't detect expect file suffix .XX.po for '{fname}'."
            .format(fname=self.fname))
    pass


class PoFile:
    def __init__(self, fname: str) -> None:
        self.fname = fname
        self.wrapwidth = 79

    def fixedHeaders(self) -> Dict[str, str]:
        """@returns: a dict of key,value parts that should be fixed
        within the po file"""
        return {"Language": self.lang(),
                "Content-Type": "text/plain; charset=UTF-8",
                "Project-Id-Version": "",
                "Language-Team": "Tails translators <tails-l10n@boum.org>",
                "Last-Translator": "Tails translators",
                }

    def lang(self) -> str:
        """@returns: language of filename, possibly ending with an underscore
        followed by the territory or script"""
        name = os.path.basename(self.fname)
        m = re.match(r"^(?:[^.].*\.)?(?P<lang>[A-Za-z0-9_@]+)\.po$", name)
        if not m:
            raise NoLanguageError(self.fname)
        return m.group("lang")

    def lang_without_script(self) -> str:
        """@returns: language of filename, with any script suffix stripped"""
        lang = self.lang()
        # a script suffix (e.g. "_Latn") starts with an underscore,
        # followed by the script name in title case, which we
        # approximate as an upper case letter followed by a lower case one
        m = re.match(r"^(?P<lang_without_script>.*?)(_[A-Z][a-z][A-Za-z]*)?(@[A-Za-z0-9]+)?$", lang)
        if m:
            return m.group("lang_without_script")
        else:
            return lang

    def check(self, key: str, value: str) -> bool:
        """check if there is "key: value\\n" in PO header"""
        try:
            return (self.pf.metadata[key] == value)
        except KeyError:
            return False

    def unifyKey(self, key: str, value: str) -> None:
        """ set value of PO header key to "key: value\\n" """
        if not self.check(key, value):
            self.pf.metadata[key] = value
            self.__changed = True

    def open(self) -> None:
        """read po file content"""
        if not os.path.exists(self.fname):
            raise FileNotFoundError(self.fname)
        self.pf = polib.pofile(self.fname)
        self.pf.wrapwidth = self.wrapwidth
        self.__changed = False

    def write(self) -> None:
        """write file, if content was changed"""
        if self.__changed:
            _prefix = os.path.basename(self.fname)
            _dir = os.path.dirname(self.fname)
            with tempfile.NamedTemporaryFile(
                    prefix=_prefix, dir=_dir, delete=False) as fd:
                try:
                    self.pf.save(fd.name)
                    fd.flush()
                    os.fdatasync(fd.fileno())
                except Exception:
                    os.unlink(fd.name)
                    raise
                else:
                    os.rename(fd.name, self.fname)

    def needs_rewrap(self) -> bool:
        """checks if lines are wrapped propperly.
        @returns: returns True if content is fine.
        """
        _pf = polib.pofile(self.fname)
        _pf.wrapwidth = self.wrapwidth
        with open(self.fname, 'r', encoding='utf-8') as f:
            content = f.read()
            if str(_pf) != content:
                self.__changed = True
                return True
            else:
                return False

    def i18nspector(self, i18nspector_env: Dict[str, str]) -> List[str]:
        """i18nspector_env sets the environment variables for i18nspector
           @returns a list of issues raised by i18nspector removes
                    allowed issues from @I18NINSPECTOR_ACCEPT.
        """
        cmd = ["i18nspector", "--language",
               self.lang_without_script(), self.fname]
        process = subprocess.run(
                cmd,
                env = i18nspector_env,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                universal_newlines=True,
                check=True)
        issues = []
        for line in process.stdout.strip().split("\n"):
            severity, fname, issue, *content = line.split(" ")
            if issue not in I18NSPECTOR_ACCEPT:
                issues.append(" ".join([severity, issue, *content]))

        return issues

@contextlib.contextmanager
def pofile_readonly(fname: str):
    pf = PoFile(fname)
    pf.open()
    yield pf


@contextlib.contextmanager
def pofile_writable(fname: str):
    pf = PoFile(fname)
    pf.open()
    yield pf
    pf.write()


def check_po_file(fname: str, extended: bool, i18nspector_env: Dict[str, str]) -> Tuple[str, List[str]]:
    """check PO file for issues.
    @returns: nothing or a list of errors
    @extended: is used to check the header fields in more detail.
    """
    errors = list()
    with pofile_readonly(fname) as poFile:
        try:
            # Make sure to have a specific subdirectory for each call to
            # avoid race conditions regarding the rply cache (#17359):
            with tempfile.TemporaryDirectory() as tmpdir:
                my_env = copy.copy(i18nspector_env)
                my_env["XDG_CACHE_HOME"] = tmpdir

                issues = poFile.i18nspector(my_env)
                if issues:
                    errors.append(
                        "i18nspector is not happy:\n\t"+"\n\t".join(issues))
        except subprocess.CalledProcessError as e:
            errors.append("i18nspector exited with {e.returncode} - stderr:\n"
                          "{e.stderr}".format(e=e))

        if extended:
            for key, value in poFile.fixedHeaders().items():
                if not poFile.check(key, value):
                    errors.append("{key} is not '{value}'."
                                  .format(key=key, value=value))

    return (fname, errors)


def unify_po_file(fname: str) -> None:
    """unify PO header and rewrapps file named `fname`"""
    with pofile_writable(fname) as poFile:
        for key, value in poFile.fixedHeaders().items():
            poFile.unifyKey(key, value)
        # As a side-effect this updates the store flag,
        # if the file is not properly wrapped:
        poFile.needs_rewrap()


def main(logger) -> None:
    parser = argparse.ArgumentParser(description='Unify PO files')
    parser.add_argument('--fix', dest='fix', action='store_true',
                        help='Fixes issues found in PO headers.')
    parser.add_argument('--check-extended', dest='extended',
                        action='store_true',
                        help='Do extended checks of PO headers.')
    parser.add_argument('--lang', dest='lang',
                        help='Check all PO files of the specified language.')
    parser.add_argument('--cached', dest='cached', action='store_true',
                        help='Only check PO files staged with Git.')
    parser.add_argument('files', metavar='file', type=str, nargs='*',
                        help='List of files to process.')
    args = parser.parse_args()

    if args.lang:
        args.files += glob.glob("**/*.{lang}.po".format(lang=args.lang),
                                recursive=True)
        args.files += glob.glob("**/{lang}.po".format(lang=args.lang),
                                recursive=True)

    if args.cached:
        # get top level directory of the current git repository
        # git diff returns always relative paths to the top level directory
        toplevel = subprocess.check_output(
            ["git", "rev-parse", "--show-toplevel"],
            universal_newlines=True).rstrip()

        # get a list of changes and added files in stage for the next commit
        output = subprocess.check_output(
            ["git", "diff", "--name-only", "--cached",
             "--ignore-submodules", "--diff-filter=d"],
            universal_newlines=True)

        # add all po files to list to unify
        args.files += [os.path.join(toplevel, f) for f in output.splitlines()
                       if f.endswith(".po")]

    if not args.files and not args.cached and not args.lang:
        args.files += glob.glob("**/*.po", recursive=True)

    # Skip files in ./tmp/
    files = list(itertools.filterfalse(
        lambda f:
        f.startswith('tmp/')
        or f.startswith('submodules/torbrowser-launcher/'),
        args.files))

    if not files:
        if not args.cached:
            logger.warning("No file to process.\n"
                           "You may want to add files to operate on."
                           " See --help for further information.")
        sys.exit(0)

    for prog in ("i18nspector",):
        if shutil.which(prog) is None:
            sys.exit("{prog}: command not found\n"
                     "You need to install {prog} first.\n"
                     "See /contribute/l10n_tricks."
                     .format(prog=prog))

    pool = multiprocessing.Pool()
    if args.fix:
        # unify PO headers for a list of files
        list(pool.map(unify_po_file, files))
    else:
        fine = True
        # check only the headers
        pool = multiprocessing.Pool()

        # The environment is no longer set here, but let's keep the
        # signatures unchanged, in case we need to tweak some more
        # things later on:
        i18nspector_env = {}
        _check_po_file = functools.partial(check_po_file,
                                           extended=args.extended,
                                           i18nspector_env=i18nspector_env)
        for fname, issues in pool.imap_unordered(_check_po_file,
                                                 files, 10):
            if issues:
                fine = False
                # indent sub-issues:
                issues = [i.replace("\n", "\n\t") for i in issues]
                logger.error("{fname}:\n\t{issues}"
                             .format(fname=fname, issues="\n\t".join(issues)))
            else:
                logger.debug("{fname} - No issue found.".format(fname=fname))

        if not fine:
            sys.exit("checked files are not clean.")


if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO,
                        format='%(levelname)s: %(message)s')
    main(logging.getLogger())