Commit 677867e4 authored by Zen Fu's avatar Zen Fu
Browse files

Merge remote-tracking branch 'hefee/hefee/cleanup_save_suggestion'

parents d0dd0de6 3221c66f
#!/usr/bin/env python3
# This script generates a temporary copy of all translations including suggestions.
# TMPDIR - the directory where the temporary copy is located.
# We have created TMPDIR manually, by git cloning locally.
# We have created a local git clone before running this script.
# It takes the default branch to work on.
#
# Usage:
#
# export DJANGO_SETTINGS_MODULE=weblate.settings
# export DJANGO_IS_MANAGEMENT_COMMAND=1
# /var/lib/weblate/scripts/save-suggestions.py
# /var/lib/weblate/scripts/save-suggestions.py repopath
import argparse
import logging
import logging.config
import os
import pathlib
import subprocess
import sys
import tempfile
sys.path.append("/usr/local/share/weblate")
sys.path.insert(0, "/usr/local/share/weblate")
os.environ["DJANGO_SETTINGS_MODULE"] = "weblate.settings"
os.environ["DJANGO_IS_MANAGEMENT_COMMAND"] = "1"
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
from weblate.trans.models import *
TMPDIR = "/var/lib/weblate/repositories/vcs/staging/"
from weblate.trans import models
logging.config.fileConfig('/var/lib/weblate/config/saveSuggestions.conf')
NAME = "saveSuggestions"
logger = logging.getLogger('saveSuggestions')
logging.config.fileConfig('/var/lib/weblate/config/{NAME}.conf'.format(NAME=NAME))
logger.info("Start a new run.")
logger = logging.getLogger(NAME)
def subprocessOutputToLogger(cmd, **kwargs):
def log_subprocess_output(cmd, **kwargs):
"""Add subprocess output to logger."""
logger.info("Running command '%s':", " ".join(cmd))
popen = subprocess.Popen(cmd, universal_newlines=True,
stdout=subprocess.PIPE,
......@@ -47,7 +49,7 @@ def subprocessOutputToLogger(cmd, **kwargs):
def update_unit(translation, unit, target):
# weblate/trans/models/translation.py: update_units
# copied from weblate/trans/models/translation.py: update_units
src = unit.get_source_plurals()[0]
add = False
......@@ -56,50 +58,87 @@ def update_unit(translation, unit, target):
# Bail out if we have not found anything
if pounit is None or pounit.is_obsolete():
logger.warning('message %s disappeared!', unit)
return
pounit.set_target(target) # update po file with first suggestion
pounit.mark_fuzzy(False) # mark change as non fuzzy
try:
subprocessOutputToLogger(["git", "clean", "-fd"], cwd=TMPDIR)
subprocessOutputToLogger(["git", "fetch"], cwd=TMPDIR)
subprocessOutputToLogger(["git", "reset", 'FETCH_HEAD', "--hard"],
cwd=TMPDIR)
except:
logger.exception("Got an exception")
raise
def commandline():
parser = argparse.ArgumentParser()
parser.add_argument(
"-v", "--verbose",
action='store_true',
help="verbose logging.")
parser.add_argument(
"repopath",
type=pathlib.Path,
help="path to the repository.")
args = parser.parse_args()
prog = parser.prog
logger.info("Start search for suggestions.")
len_subprojects = len(Component.objects.all())
subprojects = list(Component.objects.all())
if args.verbose:
for handler in logging.getLogger().handlers:
if isinstance(handler, logging.StreamHandler):
handler.level = logging.DEBUG
repopath = args.repopath
for i in range(len_subprojects):
subproject = subprojects[i]
logger.info("-- Start to run %s for %s.", NAME, repopath)
try:
for translation in subproject.translation_set.all():
changed = False
for unit in translation.unit_set.all():
if unit.suggestions:
# Get newest most voted suggestion
date_sorted = sorted(unit.suggestions, key=lambda i: i.timestamp, reverse=True)
s = sorted(date_sorted, key=lambda i: i.get_num_votes(), reverse=True)[0]
logger.debug("found suggestion for %s", unit)
update_unit(translation, unit, s.target)
changed = True
elif unit.pending:
# Save uncommitted changes
logger.debug("uncommitted changes found for %s", unit)
update_unit(translation, unit, unit.target)
changed = True
# save with suggestions
if changed:
with open(TMPDIR+"/"+translation.filename, "wb") as f:
translation.store.store.serialize(f)
log_subprocess_output(["git", "clean", "-fd"], cwd=str(repopath))
log_subprocess_output(["git", "fetch"], cwd=str(repopath))
log_subprocess_output(["git", "reset", 'FETCH_HEAD', "--hard"],
cwd=str(repopath))
except:
logger.exception("Got an exception for %s(%i)", subproject.name, i)
logger.exception("-- Something unexpected happened. Giving up. --")
raise
logger.info("Successfully updated %s.", TMPDIR)
logger.info("Start search for suggestions.")
subprojects = models.Component.objects.all()
for i, subproject in enumerate(subprojects):
try:
for translation in subproject.translation_set.all():
changed = False
for unit in translation.unit_set.all():
if unit.suggestions:
# Get newest most voted suggestion
date_sorted = sorted(unit.suggestions, key=lambda i: i.timestamp, reverse=True)
s = sorted(date_sorted, key=lambda i: i.get_num_votes(), reverse=True)[0]
logger.debug("found suggestion for %s", unit)
update_unit(translation, unit, s.target)
changed = True
elif unit.pending:
# Save uncommitted changes
logger.debug("uncommitted changes found for %s", unit)
update_unit(translation, unit, unit.target)
changed = True
# save with suggestions
if changed:
filepath = repopath/translation.filename
with tempfile.NamedTemporaryFile() as f:
translation.store.store.serialize(f)
f.flush()
try:
subprocess.check_output(['diff', "-q", str(filepath), f.name], stderr=subprocess.STDOUT)
logger.debug("don't update {}, because no diff was found".format(translation.filename))
# The file is still the same, so no need to touch the original file
except subprocess.CalledProcessError:
stat = filepath.stat()
os.fchmod(f.fileno(), stat.st_mode)
os.fchown(f.fileno(), stat.st_uid, stat.st_gid)
f.delete = False
# There is a bug in stdlib, that the delete flag is not forward to the real object
f._closer.delete = False
f.close()
os.rename(f.name, str(filepath))
except:
logger.exception("-- Got an exception for %s(%i) --", subproject.name, i)
raise
logger.info("-- Successfully run %s for %s.", NAME, repopath)
if __name__ == "__main__":
commandline()
......@@ -7,16 +7,14 @@ set -e
set -u
STAGING_WEB_DIR=/var/www/staging
STAGING_REPO_DIR=/var/lib/weblate/repositories/vcs/staging
SANITY_CHECK_LOGFILE_BASENAME=last-sanity-errors.txt
SANITY_CHECK_LOGFILE="${STAGING_WEB_DIR}/${SANITY_CHECK_LOGFILE_BASENAME}"
IKIWIKI_LOGFILE=/var/log/weblate/update-staging-website_ikiwiki.log
cd /usr/local/share/weblate
export DJANGO_SETTINGS_MODULE=weblate.settings
export DJANGO_IS_MANAGEMENT_COMMAND=1
/var/lib/weblate/scripts/save-suggestions.py
/var/lib/weblate/scripts/save-suggestions.py "$STAGING_REPO_DIR"
cd /var/lib/weblate/repositories/vcs/staging
cd "STAGING_REPO_DIR"
# Note: cron ignores the exit code so on errors, we print a message to
# STDERR, which cron will then send over email
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment