save-suggestions.py 5.32 KB
Newer Older
1
#!/usr/bin/env python3
2 3

# This script generates a temporary copy of all translations including suggestions.
4 5
# We have created a local git clone before running this script.
# It takes the default branch to work on.
6 7 8
#
# Usage:
#
9
#   /var/lib/weblate/scripts/save-suggestions.py repopath
10

11
import argparse
12 13
import logging
import logging.config
14
import os
15
import pathlib
16 17
import subprocess
import sys
18
import tempfile
19

20 21 22
sys.path.insert(0, "/usr/local/share/weblate")
os.environ["DJANGO_SETTINGS_MODULE"] = "weblate.settings"
os.environ["DJANGO_IS_MANAGEMENT_COMMAND"] = "1"
23 24 25 26

from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()

27
from weblate.trans import models
28

29
NAME = "saveSuggestions"
30

31
logging.config.fileConfig('/var/lib/weblate/config/{NAME}.conf'.format(NAME=NAME))
32

33
logger = logging.getLogger(NAME)
34

intrigeri's avatar
Lint.  
intrigeri committed
35

36 37
def log_subprocess_output(cmd, **kwargs):
    """Add subprocess output to logger."""
intrigeri's avatar
Lint.  
intrigeri committed
38
    logger.info("Running command '%s':", " ".join(cmd))
39
    popen = subprocess.Popen(cmd, universal_newlines=True,
intrigeri's avatar
Lint.  
intrigeri committed
40 41 42
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
                             **kwargs)
groente's avatar
groente committed
43 44 45
    for stdout_line in iter(popen.stdout.readline, ""):
        logger.debug(stdout_line.strip())
    popen.stdout.close()
groente's avatar
groente committed
46 47 48
    returncode = popen.wait()
    if returncode:
        raise subprocess.CalledProcessError(returncode, cmd)
groente's avatar
groente committed
49

intrigeri's avatar
Lint.  
intrigeri committed
50

51
def update_unit(translation, unit, target):
52
    # copied from weblate/trans/models/translation.py: update_units
53 54 55 56 57 58 59 60
    src = unit.get_source_plurals()[0]
    add = False

    pounit, add = translation.store.find_unit(unit.context, src)

    # Bail out if we have not found anything
    if pounit is None or pounit.is_obsolete():
        logger.warning('message %s disappeared!', unit)
61
        return
62 63

    pounit.set_target(target)     # update po file with first suggestion
64
    pounit.mark_fuzzy(False)           # mark change as non fuzzy
65 66


67
def commandline():
68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-v", "--verbose",
        action='store_true',
        help="verbose logging.")
    parser.add_argument(
        "repopath",
        type=pathlib.Path,
        help="path to the repository.")
    args = parser.parse_args()
    prog = parser.prog

    if args.verbose:
        for handler in logging.getLogger().handlers:
            if isinstance(handler, logging.StreamHandler):
                handler.level = logging.DEBUG

    repopath = args.repopath

87
    logger.info("-- Start to run %s for %s.", NAME, repopath)
88
    try:
89 90 91 92
        log_subprocess_output(["git", "clean", "-fd"], cwd=str(repopath))
        log_subprocess_output(["git", "fetch"], cwd=str(repopath))
        log_subprocess_output(["git", "reset", 'FETCH_HEAD', "--hard"],
                                 cwd=str(repopath))
93
    except:
94
        logger.exception("-- Something unexpected happened. Giving up. --")
95 96
        raise

97 98
    logger.info("Start search for suggestions.")

99
    subprojects = models.Component.objects.all()
100

101
    for i, subproject in enumerate(subprojects):
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
        try:
            for translation in subproject.translation_set.all():
                changed = False
                for unit in translation.unit_set.all():
                    if unit.suggestions:
                        # Get newest most voted suggestion
                        date_sorted = sorted(unit.suggestions, key=lambda i: i.timestamp, reverse=True)
                        s = sorted(date_sorted, key=lambda i: i.get_num_votes(), reverse=True)[0]
                        logger.debug("found suggestion for %s", unit)
                        update_unit(translation, unit, s.target)
                        changed = True
                    elif unit.pending:
                        # Save uncommitted changes
                        logger.debug("uncommitted changes found for %s", unit)
                        update_unit(translation, unit, unit.target)
                        changed = True

                # save with suggestions
                if changed:
121 122
                    filepath = repopath/translation.filename
                    with tempfile.NamedTemporaryFile() as f:
123
                        translation.store.store.serialize(f)
124 125 126 127 128 129 130 131 132 133 134 135 136 137
                        f.flush()
                        try:
                            subprocess.check_output(['diff', "-q", str(filepath), f.name], stderr=subprocess.STDOUT)
                            logger.debug("don't update {}, because no diff was found".format(translation.filename))
                            # The file is still the same, so no need to touch the original file
                        except subprocess.CalledProcessError:
                            stat = filepath.stat()
                            os.fchmod(f.fileno(), stat.st_mode)
                            os.fchown(f.fileno(), stat.st_uid, stat.st_gid)
                            f.delete = False
                            # There is a bug in stdlib, that the delete flag is not forward to the real object
                            f._closer.delete = False
                            f.close()
                            os.rename(f.name, str(filepath))
138
        except:
139
            logger.exception("-- Got an exception for %s(%i) --", subproject.name, i)
140
            raise
141
    logger.info("-- Successfully run %s for %s.", NAME, repopath)
142 143 144

if __name__ == "__main__":
    commandline()