Commit 53c414fa authored by intrigeri's avatar intrigeri
Browse files

Import lint_po from tails.git at commit 52afa659e1a7815c2b37e2103bb627f9b9735457.

parent 8472ad28
#!/bin/sh
# -*- mode: sh; sh-basic-offset: 4; indent-tabs-mode: nil; -*-
# vim: set filetype=sh sw=4 sts=4 expandtab autoindent:
# Usage: check_po.sh [LANGUAGE]
set -u
if ! [ -x "`which i18nspector`" ] ; then
echo "i18nspector: command not found"
echo "You need to install i18nspector first. See /contribute/l10n_tricks."
exit 2
fi
if [ $# -ge 1 ] ; then
FILE_GLOB="*.${1}.po"
else
FILE_GLOB='*.po'
fi
PATTERNS_FILE="$(mktemp -t XXXXXX.patterns)"
echo "
boilerplate-in-date
boilerplate-in-initial-comments
boilerplate-in-language-team
boilerplate-in-last-translator
boilerplate-in-project-id-version
codomain-error-in-plural-forms
codomain-error-in-unused-plural-forms
conflict-marker-in-header-entry
fuzzy-header-entry
incorrect-plural-forms
invalid-content-transfer-encoding
invalid-date
invalid-language
invalid-last-translator
language-team-equal-to-last-translator
no-language-header-field
no-package-name-in-project-id-version
no-plural-forms-header-field
no-report-msgid-bugs-to-header-field
no-version-in-project-id-version
stray-previous-msgid
unable-to-determine-language
unknown-poedit-language
unusual-plural-forms
unusual-unused-plural-forms
" | grep -v '^$' > "$PATTERNS_FILE"
CPUS=$(egrep '^processor[[:space:]]+:' /proc/cpuinfo | wc -l)
OUTPUT=$(find -wholename ./tmp -prune -o \( -iname "$FILE_GLOB" -print0 \) \
| xargs -0 --max-procs="$CPUS" --max-args=64 i18nspector \
| grep -v --line-regexp '' \
| grep -v -f "$PATTERNS_FILE")
### Output and exit code
# Our automated testing jobs depend on it, beware!
if [ -z "$OUTPUT" ]; then
exit 0
else
# Output the filtered i18nspector's output
echo "$OUTPUT"
# Exit code: 0 iff. the filtered i18nspector's output was empty
exit 1
fi
lint_po
\ No newline at end of file
#!/usr/bin/env python3
"""Checks and Unifies PO headers and rewraps PO files to 79 chars.
Usage:
./lint_po --help
Default is check mode where the error are listed but not fixed.
With --fix the files get changed and unified.
Run for all po files in the working directory (including subdirs):
./lint_po
Run with a list of files:
./lint_po file1.de.po file2.fr.po
Run for all po files that are staged for git commit:
./lint_po --cached
Run for all po files of one language in the current directory (including subdirs):
./lint_po --lang de
When modifying lint_po (this script), you should check if the current type
annotations match, using `mypy` (`apt install mypy`):
mypy lint_po
"""
import argparse
import contextlib
import functools
import glob
import logging
import multiprocessing
import os.path
import re
import shutil
import subprocess
import sys
import tempfile
try:
import polib
except ImportError:
sys.exit("You need to install python3-polib to use this program.")
from typing import Dict, List, Tuple
# i18nspector issues, that we accept
I18NSPECTOR_ACCEPT = [
"boilerplate-in-date",
"boilerplate-in-initial-comments",
"boilerplate-in-language-team",
"boilerplate-in-last-translator",
"boilerplate-in-project-id-version",
"codomain-error-in-plural-forms",
"codomain-error-in-unused-plural-forms",
"conflict-marker-in-header-entry",
"fuzzy-header-entry",
"incorrect-plural-forms",
"invalid-content-transfer-encoding",
"invalid-date",
"invalid-language",
"invalid-last-translator",
"language-team-equal-to-last-translator",
"no-language-header-field",
"no-package-name-in-project-id-version",
"no-plural-forms-header-field",
"no-report-msgid-bugs-to-header-field",
"no-version-in-project-id-version",
"stray-previous-msgid",
"unable-to-determine-language",
"unknown-poedit-language",
"unusual-plural-forms",
"unusual-unused-plural-forms",
]
class NoLanguageError(Exception):
def __init__(self, fname):
self.fname = fname
def __str__(self):
return f"Can't detect expect file suffix .XX.po for '{self.fname}'."
pass
class PoFile:
def __init__(self, fname: str) -> None:
self.fname = fname
self.wrapwidth = 79
def fixedHeaders(self) -> Dict[str, str]:
"""@returns: a dict of key,value parts that should be fixed within the po file"""
return {"Language": self.lang(),
"Content-Type": "text/plain; charset=UTF-8",
"Project-Id-Version": "",
"Language-Team": "Tails translators <tails-l10n@boum.org>",
"Last-Translator": "Tails translators",
}
def lang(self) -> str:
"""@returns: language of filename"""
name = os.path.basename(self.fname)
m = re.match(r"^[^.].*\.(?P<lang>[A-Za-z_]+)\.po$", name)
if not m:
raise NoLanguageError(self.fname)
return m.group("lang")
def check(self, key: str, value: str) -> bool:
"""check if there is "key: value\\n" in PO header"""
try:
return (self.pf.metadata[key] == value)
except KeyError:
return False
def unifyKey(self, key: str, value: str) -> None:
""" set value of PO header key to "key: value\\n" """
if not self.check(key, value):
self.pf.metadata[key] = value
self.__changed = True
def open(self) -> None:
"""read po file content"""
if not os.path.exists(self.fname):
raise FileNotFoundError(self.fname)
self.pf = polib.pofile(self.fname)
self.pf.wrapwidth = self.wrapwidth
self.__changed = False
def write(self) -> None:
"""write file, if content was changed"""
if self.__changed:
_prefix = os.path.basename(self.fname)
_dir = os.path.dirname(self.fname)
with tempfile.NamedTemporaryFile(prefix=_prefix, dir=_dir, delete=False) as fd:
try:
self.pf.save(fd.name)
fd.flush()
os.fdatasync(fd.fileno())
except Exception:
os.unlink(fd.name)
raise
else:
os.rename(fd.name, self.fname)
def needs_rewrap(self) -> bool:
"""checks if lines are wrapped propperly.
@returns: returns True if content is fine.
"""
_pf = polib.pofile(self.fname)
_pf.wrapwidth = self.wrapwidth
with open(self.fname, 'r', encoding='utf-8') as f:
content = f.read()
if str(_pf) != content:
self.__changed = True
return True
else:
return False
def i18nspector(self) -> List[str]:
"""@returns a list of issues raised by i18nspector removes
allowed issues from @I18NINSPECTOR_ACCEPT.
"""
cmd = ["i18nspector", "-l", self.lang(), self.fname]
process = subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
check=True)
issues = []
for line in process.stdout.strip().split("\n"):
severity, fname, issue, *content = line.split(" ")
if issue not in I18NSPECTOR_ACCEPT:
issues.append(" ".join([severity, issue, *content]))
return issues
@contextlib.contextmanager
def pofile_readonly(fname: str):
pf = PoFile(fname)
pf.open()
yield pf
@contextlib.contextmanager
def pofile_writable(fname: str):
pf = PoFile(fname)
pf.open()
yield pf
pf.write()
def check_po_file(fname: str, extended: bool) -> Tuple[str, List[str]]:
"""check PO file for issues.
@returns: nothing or a list of errors
@extended: is used to check the header fields in more detail.
"""
errors = list()
with pofile_readonly(fname) as poFile:
try:
issues = poFile.i18nspector()
if issues:
errors.append("i18nspector is not happy:\n\t"+"\n\t".join(issues))
except subprocess.CalledProcessError as e:
errors.append("i18nspector exited with {e.returncode} - stderr:\n"
"{e.stderr}".format(e=e))
if extended:
for key, value in poFile.fixedHeaders().items():
if not poFile.check(key, value):
errors.append("{key} is not '{value}'.".format(key=key, value=value))
return (fname, errors)
def unify_po_file(fname: str) -> None:
"""unify PO header and rewrapps file named `fname`"""
with pofile_writable(fname) as poFile:
for key, value in poFile.fixedHeaders().items():
poFile.unifyKey(key, value)
poFile.needs_rewrap() # as sideeffect it updates the store flag,
# if file is not wrapped properly
def main(logger) -> None:
parser = argparse.ArgumentParser(description='Unify PO files')
parser.add_argument('--fix', dest='fix', action='store_true',
help='Fixes the issues of the PO headers, otherwise only check is done.')
parser.add_argument('--check-extended', dest='extended', action='store_true',
help='Do extended checks of PO headers.')
parser.add_argument('--lang', dest='lang', help='all files of a specific language.')
parser.add_argument('--cached', dest='cached', action='store_true',
help='all git staged PO files.')
parser.add_argument('files', metavar='file', type=str, nargs='*',
help='list of files to process.')
args = parser.parse_args()
if args.lang:
args.files += glob.glob("**/*.{lang}.po".format(lang=args.lang), recursive=True)
if args.cached:
# get top level directory of the current git repository
# git diff returns always relative paths to the top level directory
toplevel = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], universal_newlines=True).rstrip()
# get a list of changes and added files in stage for the next commit
output = subprocess.check_output(
["git", "diff", "--name-only", "--cached", "--ignore-submodules", "--diff-filter=d"],
universal_newlines=True)
# add all po files to list to unify
args.files += [os.path.join(toplevel, f) for f in output.splitlines() if f.endswith(".po")]
if not args.files and not args.cached and not args.lang:
args.files += glob.glob("**/*.po", recursive=True)
if not args.files:
logger.warning("no file to process :("
" You may want to add files to operate on. See --help for further information.")
for prog in ("i18nspector",):
if shutil.which(prog) is None:
sys.exit("{prog}: command not found\n"
"You need to install {prog} first. See /contribute/l10n_tricks."
.format(prog=prog))
pool = multiprocessing.Pool()
if args.fix:
# unify PO headers for a list of files
list(pool.map(unify_po_file, args.files))
else:
fine = True
# check only the headers
pool = multiprocessing.Pool()
_check_po_file = functools.partial(check_po_file, extended=args.extended)
for fname, issues in pool.imap_unordered(_check_po_file, args.files, 10):
if issues:
fine = False
issues = [i.replace("\n", "\n\t") for i in issues] # indent subissues
logger.error("{fname}:\n\t{issues}".format(fname=fname, issues="\n\t".join(issues)))
else:
logger.debug("{fname} - No issue found.".format(fname=fname))
if not fine:
sys.exit("checked files are not clean.")
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
main(logging.getLogger())
wrongLang:
- |-
i18nspector is not happy:
W: language-disparity en (command-line) != de_DE (Language header field)
clean: []
invalidHeaderfilds: []
length: []
# SOME DESCRIPTIVE TITLE
# Copyright (C) YEAR Free Software Foundation, Inc.
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: \n"
"POT-Creation-Date: 2018-06-17 12:49+0200\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: Tails translators\n"
"Language-Team: Tails translators <tails-l10n@boum.org>\n"
"Language: en\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
msgid "test"
msgstr "test"
# SOME DESCRIPTIVE TITLE
# Copyright (C) YEAR Free Software Foundation, Inc.
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: tails-l10n@boum.org\n"
"POT-Creation-Date: 2018-08-20 15:43+0200\n"
"PO-Revision-Date: 2018-05-09 17:53+0000\n"
"Last-Translator: Tails translators <tails-l10n@boum.org>\n"
"Language-Team: Spanish <http://translate.tails.boum.org/projects/tails/"
"upgrade-tails-overview/es/>\n"
"Language: en\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=n != 1;\n"
"X-Generator: Weblate 2.10.1\n"
msgid "test"
msgstr "test"
# SOME DESCRIPTIVE TITLE
# Copyright (C) YEAR Free Software Foundation, Inc.
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
msgid ""
msgstr ""
"Project-Id-Version: \n"
"POT-Creation-Date: 2017-09-16 01:19+0200\n"
"PO-Revision-Date: 2017-08-20 19:43+0200\n"
"Last-Translator: Tails translators\n"
"Language-Team: Tails translators <tails-l10n@boum.org>\n"
"Language: en\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"X-Generator: Poedit 1.8.11\n"
msgid ""
"test asdf asdfaera asdr vsdafaser sasdfasdra sdfasdras asdfaser scvasdr ascvsadta sdfa sdfa ad"
"rsdasdasersfasdras asdfaser sadfdrase asdfasra sdf asdfaser ssa"
msgstr ""
"test adsfads asdf asdf asdf asdf asdf adsf adsf asdf dsf sdf sasdf asdf adsf asdf asdfasd adsfads adf asdf asdf asdfaser faser asd"
# SOME DESCRIPTIVE TITLE
# Copyright (C) YEAR Free Software Foundation, Inc.
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
msgid ""
msgstr ""
"Project-Id-Version: Tails website\n"
"POT-Creation-Date: 2017-09-16 01:19+0200\n"
"PO-Revision-Date: 2017-08-20 19:43+0200\n"
"Last-Translator: Tails translators\n"
"Language-Team: \n"
"Language: de_DE\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"X-Generator: Poedit 1.8.11\n"
msgid "test"
msgstr "test"
wrongLang:
- |-
i18nspector is not happy:
W: language-disparity en (command-line) != de_DE (Language header field)
- Language is not 'en'.
- Project-Id-Version is not ''.
- Language-Team is not 'Tails translators <tails-l10n@boum.org>'.
clean: []
invalidHeaderfilds:
- Project-Id-Version is not ''.
- Language-Team is not 'Tails translators <tails-l10n@boum.org>'.
- Last-Translator is not 'Tails translators'.
length: []
#!/usb/bin/env python3
import glob
import importlib.machinery
import logging
import os
import unittest
import tempfile
import shutil
import yaml
lint_po = importlib.machinery.SourceFileLoader('lint_po', 'lint_po').load_module()
DIRNAME = os.path.dirname(__file__)
class TestCheckPo(unittest.TestCase):
def test_checkPo(self):
with open(os.path.join(DIRNAME, "checkPo.yml")) as f:
expected = yaml.load(f)
with tempfile.TemporaryDirectory() as tmpdir:
for fpath in glob.glob(os.path.join(DIRNAME, "checkPo/*")):
name = os.path.basename(fpath)
newPath = os.path.join(tmpdir, name + ".en.po")
os.symlink(os.path.abspath(fpath), newPath)
path, issues = lint_po.check_po_file(newPath, extended=False)
self.assertEqual(path, newPath)
self.assertEqual(issues, expected[name], msg=name)
def test_checkPoExtended(self):
with open(os.path.join(DIRNAME, "checkPoExtended.yml")) as f:
expected = yaml.load(f)
with tempfile.TemporaryDirectory() as tmpdir:
for fpath in glob.glob(os.path.join(DIRNAME, "checkPo/*")):
name = os.path.basename(fpath)
newPath = os.path.join(tmpdir, name + ".en.po")
os.symlink(os.path.abspath(fpath), newPath)
path, issues = lint_po.check_po_file(newPath, extended=True)
self.assertEqual(path, newPath)
self.assertEqual(issues, expected[name], msg=name)
def test_nonexistingPo(self):
with tempfile.TemporaryDirectory() as tmpdir:
newPath = os.path.join(tmpdir, "nonexisting.en.po")
with self.assertRaises(FileNotFoundError, msg=newPath):
path, issues = lint_po.check_po_file(newPath, extended=False)
with self.assertRaises(FileNotFoundError, msg=newPath):
path, issues = lint_po.check_po_file(newPath, extended=True)
def test_defaultOption(self):
with open(os.path.join(DIRNAME, "checkPo.yml")) as f:
expected = yaml.load(f)
expectedOutput = []
with tempfile.TemporaryDirectory() as tmpdir:
for fpath in glob.glob(os.path.join(DIRNAME, "checkPo/*")):
name = os.path.basename(fpath)
newPath = os.path.join(tmpdir, name+ "/" + name + ".en.po")
os.mkdir(os.path.join(tmpdir, name))
os.symlink(os.path.abspath(fpath), newPath)
if not expected[name]:
expectedOutput.append("DEBUG:root:{} - No issue found.".format(name+ "/" + name + ".en.po"))
else:
expectedOutput.append("ERROR:root:{}:\n\t{}".format(name+ "/" + name + ".en.po", expected[name][0].replace("\n","\n\t")))
cwd = os.getcwd()
try:
os.chdir(tmpdir)
with self.assertRaises(SystemExit) as e:
with self.assertLogs(level='DEBUG') as cm:
lint_po.main(logging.getLogger())
finally:
os.chdir(cwd)
self.assertEqual(e.exception.args, ("checked files are not clean.",))
self.assertEqual(sorted(cm.output), sorted(expectedOutput))
def test_lint_po(self):
self.maxDiff = None
with tempfile.TemporaryDirectory() as tmpdir:
for fpath in glob.glob(os.path.join(DIRNAME, "lint_po/*")):
with open(fpath) as f:
expectedContent = f.read()
name = os.path.basename(fpath)
newPath = os.path.join(tmpdir, name + ".en.po")
shutil.copy(os.path.join(DIRNAME, "checkPo", name), newPath)
lint_po.unify_po_file(newPath)
with open(newPath) as f:
self.assertEqual(f.read(), expectedContent, msg=name)
_, issues = lint_po.check_po_file(newPath, extended=True)
self.assertEqual(issues, [], msg=name)
def test_lang(self):
self.assertEqual(lint_po.PoFile("index.de.po").lang(), "de")
self.assertEqual(lint_po.PoFile("x/a/a.fb.xx.po").lang(), "xx")
_p = lint_po.PoFile(".de.po")
with self.assertRaises(lint_po.NoLanguageError, msg=_p.fname) as e:
_p.lang()
self.assertEqual(str(e.exception), "Can't detect expect file suffix .XX.po for '.de.po'.")
_p = lint_po.PoFile(".a.d.de.po")
with self.assertRaises(lint_po.NoLanguageError, msg=_p.fname):
_p.lang()
_p = lint_po.PoFile("a.po")
with self.assertRaises(lint_po.NoLanguageError, msg=_p.fname):
_p.lang()
_p = lint_po.PoFile("/a/d/d..po")
with self.assertRaises(lint_po.NoLanguageError, msg=_p.fname):
_p.lang()
def test_needs_rewrap(self):
with lint_po.pofile_readonly(os.path.join(DIRNAME, "checkPo/length")) as poFile:
self.assertEqual(poFile.needs_rewrap(), True)
with lint_po.pofile_readonly(os.path.join(DIRNAME, "unifyPo/length")) as poFile:
self.assertEqual(poFile.needs_rewrap(), False)
if __name__ == '__main__':
unittest.main()
# SOME DESCRIPTIVE TITLE
# Copyright (C) YEAR Free Software Foundation, Inc.
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
#, fuzzy