Merge remote-tracking branch 'hefee/hefee/cleanup_save_suggestion'

files/weblate/scripts/save-suggestions.py

 #!/usr/bin/env python3
 
 # This script generates a temporary copy of all translations including suggestions.
-# TMPDIR - the directory where the temporary copy is located.
-# We have created TMPDIR manually, by git cloning locally.
+# We have created a local git clone before running this script.
+# It takes the default branch to work on.
 #
 # Usage:
 #
-#   export DJANGO_SETTINGS_MODULE=weblate.settings
-#   export DJANGO_IS_MANAGEMENT_COMMAND=1
-#   /var/lib/weblate/scripts/save-suggestions.py
+#   /var/lib/weblate/scripts/save-suggestions.py repopath
 
+import argparse
 import logging
 import logging.config
-
+import os
+import pathlib
 import subprocess
 import sys
+import tempfile
 
-sys.path.append("/usr/local/share/weblate")
+sys.path.insert(0, "/usr/local/share/weblate")
+os.environ["DJANGO_SETTINGS_MODULE"] = "weblate.settings"
+os.environ["DJANGO_IS_MANAGEMENT_COMMAND"] = "1"
 
 from django.core.wsgi import get_wsgi_application
 application = get_wsgi_application()
 
-from weblate.trans.models import *
-
-TMPDIR = "/var/lib/weblate/repositories/vcs/staging/"
+from weblate.trans import models
 
-logging.config.fileConfig('/var/lib/weblate/config/saveSuggestions.conf')
+NAME = "saveSuggestions"
 
-logger = logging.getLogger('saveSuggestions')
+logging.config.fileConfig('/var/lib/weblate/config/{NAME}.conf'.format(NAME=NAME))
 
-logger.info("Start a new run.")
+logger = logging.getLogger(NAME)
 
 
-def subprocessOutputToLogger(cmd, **kwargs):
+def log_subprocess_output(cmd, **kwargs):
+    """Add subprocess output to logger."""
     logger.info("Running command '%s':", " ".join(cmd))
     popen = subprocess.Popen(cmd, universal_newlines=True,
                              stdout=subprocess.PIPE,
@@ -47,7 +49,7 @@ def subprocessOutputToLogger(cmd, **kwargs):
 
 
 def update_unit(translation, unit, target):
-    # weblate/trans/models/translation.py: update_units
+    # copied from weblate/trans/models/translation.py: update_units
     src = unit.get_source_plurals()[0]
     add = False
 
@@ -56,50 +58,87 @@ def update_unit(translation, unit, target):
     # Bail out if we have not found anything
     if pounit is None or pounit.is_obsolete():
         logger.warning('message %s disappeared!', unit)
+        return
 
     pounit.set_target(target)     # update po file with first suggestion
     pounit.mark_fuzzy(False)           # mark change as non fuzzy
 
 
-try:
-    subprocessOutputToLogger(["git", "clean", "-fd"], cwd=TMPDIR)
-    subprocessOutputToLogger(["git", "fetch"], cwd=TMPDIR)
-    subprocessOutputToLogger(["git", "reset", 'FETCH_HEAD', "--hard"],
-                             cwd=TMPDIR)
-except:
-    logger.exception("Got an exception")
-    raise
+def commandline():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v", "--verbose",
+        action='store_true',
+        help="verbose logging.")
+    parser.add_argument(
+        "repopath",
+        type=pathlib.Path,
+        help="path to the repository.")
+    args = parser.parse_args()
+    prog = parser.prog
 
-logger.info("Start search for suggestions.")
-len_subprojects = len(Component.objects.all())
-subprojects = list(Component.objects.all())
+    if args.verbose:
+        for handler in logging.getLogger().handlers:
+            if isinstance(handler, logging.StreamHandler):
+                handler.level = logging.DEBUG
 
+    repopath = args.repopath
 
-for i in range(len_subprojects):
-    subproject = subprojects[i]
+    logger.info("-- Start to run %s for %s.", NAME, repopath)
     try:
-        for translation in subproject.translation_set.all():
-            changed = False
-            for unit in translation.unit_set.all():
-                if unit.suggestions:
-                    # Get newest most voted suggestion
-                    date_sorted = sorted(unit.suggestions, key=lambda i: i.timestamp, reverse=True)
-                    s = sorted(date_sorted, key=lambda i: i.get_num_votes(), reverse=True)[0]
-                    logger.debug("found suggestion for %s", unit)
-                    update_unit(translation, unit, s.target)
-                    changed = True
-                elif unit.pending:
-                    # Save uncommitted changes
-                    logger.debug("uncommitted changes found for %s", unit)
-                    update_unit(translation, unit, unit.target)
-                    changed = True
-
-            # save with suggestions
-            if changed:
-                with open(TMPDIR+"/"+translation.filename, "wb") as f:
-                    translation.store.store.serialize(f)
+        log_subprocess_output(["git", "clean", "-fd"], cwd=str(repopath))
+        log_subprocess_output(["git", "fetch"], cwd=str(repopath))
+        log_subprocess_output(["git", "reset", 'FETCH_HEAD', "--hard"],
+                                 cwd=str(repopath))
     except:
-        logger.exception("Got an exception for %s(%i)", subproject.name, i)
+        logger.exception("-- Something unexpected happened. Giving up. --")
         raise
 
-logger.info("Successfully updated %s.", TMPDIR)
+    logger.info("Start search for suggestions.")
+
+    subprojects = models.Component.objects.all()
+
+    for i, subproject in enumerate(subprojects):
+        try:
+            for translation in subproject.translation_set.all():
+                changed = False
+                for unit in translation.unit_set.all():
+                    if unit.suggestions:
+                        # Get newest most voted suggestion
+                        date_sorted = sorted(unit.suggestions, key=lambda i: i.timestamp, reverse=True)
+                        s = sorted(date_sorted, key=lambda i: i.get_num_votes(), reverse=True)[0]
+                        logger.debug("found suggestion for %s", unit)
+                        update_unit(translation, unit, s.target)
+                        changed = True
+                    elif unit.pending:
+                        # Save uncommitted changes
+                        logger.debug("uncommitted changes found for %s", unit)
+                        update_unit(translation, unit, unit.target)
+                        changed = True
+
+                # save with suggestions
+                if changed:
+                    filepath = repopath/translation.filename
+                    with tempfile.NamedTemporaryFile() as f:
+                        translation.store.store.serialize(f)
+                        f.flush()
+                        try:
+                            subprocess.check_output(['diff', "-q", str(filepath), f.name], stderr=subprocess.STDOUT)
+                            logger.debug("don't update {}, because no diff was found".format(translation.filename))
+                            # The file is still the same, so no need to touch the original file
+                        except subprocess.CalledProcessError:
+                            stat = filepath.stat()
+                            os.fchmod(f.fileno(), stat.st_mode)
+                            os.fchown(f.fileno(), stat.st_uid, stat.st_gid)
+                            f.delete = False
+                            # There is a bug in stdlib, that the delete flag is not forward to the real object
+                            f._closer.delete = False
+                            f.close()
+                            os.rename(f.name, str(filepath))
+        except:
+            logger.exception("-- Got an exception for %s(%i) --", subproject.name, i)
+            raise
+    logger.info("-- Successfully run %s for %s.", NAME, repopath)
+
+if __name__ == "__main__":
+    commandline()

files/weblate/scripts/update-staging-website.sh

@@ -7,16 +7,14 @@ set -e
 set -u
 
 STAGING_WEB_DIR=/var/www/staging
+STAGING_REPO_DIR=/var/lib/weblate/repositories/vcs/staging
 SANITY_CHECK_LOGFILE_BASENAME=last-sanity-errors.txt
 SANITY_CHECK_LOGFILE="${STAGING_WEB_DIR}/${SANITY_CHECK_LOGFILE_BASENAME}"
 IKIWIKI_LOGFILE=/var/log/weblate/update-staging-website_ikiwiki.log
 
-cd /usr/local/share/weblate
-export DJANGO_SETTINGS_MODULE=weblate.settings
-export DJANGO_IS_MANAGEMENT_COMMAND=1
-/var/lib/weblate/scripts/save-suggestions.py
+/var/lib/weblate/scripts/save-suggestions.py "$STAGING_REPO_DIR"
 
-cd /var/lib/weblate/repositories/vcs/staging
+cd "STAGING_REPO_DIR"
 
 # Note: cron ignores the exit code so on errors, we print a message to
 # STDERR, which cron will then send over email