merge_weblate_changes.py 11 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
#!/usr/bin/env python3

"""Script to merge weblate changes from git(remote/master) into local git.
Tails have different rules, what Weblate is allowed to do and how merge conficts
should be handled:
 * po files only modified by Weblate(remote) - remote version is used.
 * po files modifed by remote and local are merged with @mergePo:
   - mergePo understands the po file formate and does a merge based on the po msgids.
   - if there is a merge conflict on one msgid (remote and local modified the
     content for one msgid), local is preferred. This makes sure, that Tails
     developers can fix issues in main repository and updates translations
     over Weblate.
 * otherwise local is preferred.
"""

import git

import logging
intrigeri's avatar
intrigeri committed
19
import logging.config
20 21 22 23 24
import os
import subprocess
import tempfile
import pathlib

25
import merge_canonical_changes as mcc
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49

logger = logging.getLogger(__name__)
parser = None
prog = 'merge-weblate-changes'
temporaryDirectory = lambda x: tempfile.TemporaryDirectory()


def mergePo(origPath, lang, output):
    """merges po files.
    it expects, that @origPath is a directory with following files (pot, local, base and remote)
    returns the merge content to the file @output."""
    pot = origPath/'pot'
    local = origPath/'local'
    base = origPath/'base'
    remote = origPath/'remote'

    with temporaryDirectory(origPath/"merge") as tempdir:
        path = pathlib.Path(tempdir)
        header = path/'header'
        localChanges = path/'localChanges'
        remoteChanges = path/'remoteChanges'
        unchanged = path/'unchanged'

        # get only header of pot file
50
        header.write_bytes(mcc.poHeader(pot.read_bytes()))
51 52

        # messages changed on local/remote
53 54
        localChanges.write_bytes(mcc.extractChanges(local, base))
        remoteChanges.write_bytes(mcc.extractChanges(remote, base))
55 56

        # unchanged messages
57
        unchanged.write_bytes(mcc.unchangedChanges(base, local, remote))
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95

        # messages changed on local, not on remote; and vice-versa
        unique = subprocess.run(['msgcat', '-o', '-', '--unique', str(localChanges), str(remoteChanges)],
                                stdout=subprocess.PIPE,
                                check=True)

        # the big merge (unchanged, unique and localChanges)
        # Keep in mind, that we DO NOT add remoteChanges as localChanges are preferred
        # over local changes for tails this is fine as everything form main git is
        # preferred over weblate and we would create conflicts in po files

        msgcat = subprocess.run(['msgcat', '-o', '-', str(unchanged), '-', str(localChanges)],
                                input=unique.stdout,
                                stdout=subprocess.PIPE,
                                check=True)

        # filter messages actually needed (those on local and remote)
        msgmerge = subprocess.run(['msgmerge', '--force-po', '--quiet', '--no-fuzzy-matching', '-o', '-', '-', str(pot)],
                                  input=msgcat.stdout,
                                  stdout=subprocess.PIPE,
                                  check=True)

        # final merge, adds saved header
        subprocess.run(['msgcat', "--lang", lang, '-w', '79', '-o', str(output), '--use-first', str(header), '-'],
                       input=msgmerge.stdout,
                       check=True)


def exists_in_tree(tree, path):
    length = path.rfind("/")
    if length != -1:
        try:
            return path in tree/path[:length]
        except KeyError:
            return False
    else:
        return path in tree

96
def main(repopath, local, localCommit, remote):
97 98 99 100 101 102
    r = git.Repo(str(repopath))
    msg = "merge Weblate changes using {}.\n\n".format(prog)

    try:
        local = r.refs[local]
    except IndexError:
103
        mcc.logInputError('{} is not a valid git reference.'.format(local))
104 105 106 107 108

    if localCommit:
        try:
            local.commit = r.commit(localCommit)
        except git.BadName:
109
            mcc.logInputError('{} is not a valid git reference.'.format(local))
110 111 112

    local.checkout()

113 114
    oldcommit = local.commit

115 116 117 118 119 120 121 122 123 124 125
    logger.info("local: %s", local.commit)

    remoteBranch = None
    try:
        remote = r.refs[remote]
        remoteBranch = remote
        remote = remote.commit
    except IndexError:
        try:
            remote = r.commit(remote)
        except git.BadName:
126
            mcc.logInputError('{} is not a valid git reference or a commit.'.format(remote))
127 128 129

    logger.info("remote: %s", remote)

130
    if local.commit == remote:
Sandro Knauß's avatar
Sandro Knauß committed
131
        logger.info('remote and local are on the same commit, nothing to do.')
132 133
        return

134 135 136 137 138
    # get common ancestor
    base = r.merge_base(local, remote)
    logger.info("base: %s", base[0])

    # create index to prepare the merge
139
    index = mcc.Index(git.IndexFile.from_tree(r, base, local, remote))
140 141

    # is a fast-forward possible
142 143
    ff_possible_local = (base[0] == local.commit)    # remote branch has commits on top of local
    local_uptodate = (base[0] == remote)             # local has commits on top of remote branch
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181

    try:
        # A set of all local modified files
        local_modified_files = index.local_modified_files

        # Process all remote modified files
        basediff = base[0].diff(remote)

        for i in base[0].diff(local.commit):
            path = i.b_path
            if i.change_type == "D":
                path = i.a_path
            local_modified_files.add(path)

        for i in basediff:
            path = i.b_path
            if i.change_type == "D":
                path = i.a_path
                if exists_in_tree(local.commit.tree, path):
                    # Reset file, Weblate is not allowed to delete files.
                    m = "{path}: reset (Weblate is not allowed to delete files).\n".format(path=path)
                    logger.debug(m.strip())
                    msg += m
                    blob = local.commit.tree/path
                    index.updateFile(path, blob)
                else:
                    index.removeFile(path)
                continue

            if not exists_in_tree(local.commit.tree, path):
                # File does not exists on master, that's why delete file from index
                # as Weblate is not allowed to create files
                m = "{path}: reset (Weblate is not allowed to add files).\n".format(path=path)
                logger.debug(m.strip())
                msg += m
                index.removeFile(path)
                continue

182
            if not mcc.isWikiPo(path):
183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
                # Reset file, that is not allowed to be touched by Weblate.
                m = "{path}: reset (Weblate is only to touch po files).\n".format(path=path)
                logger.debug(m.strip())
                msg += m
                blob = local.commit.tree/path
                index.updateFile(path, blob)
                continue

            if i.a_path not in local_modified_files:
                logger.debug("%s: only remote changes - apply them.", i.a_path)
                # Only remote changes - apply them
                blob = i.b_blob
                index.updateFile(path, blob)
                continue

            # remote and local have modified the file
            m = "{path}: merging.\n".format(path=path)
            logger.debug(m.strip())
            with temporaryDirectory(path) as tempdir:
                temppath = pathlib.Path(tempdir)
                potpath = path
204 205 206 207
                (temppath/'pot').write_bytes(mcc.pot(local.commit.tree/potpath))
                mcc.msguniq(remote.tree/path, temppath/'remote')
                mcc.msguniq(local.commit.tree/i.a_path, temppath/'local')
                mcc.msguniq(base[0].tree/i.a_path, temppath/'base')
208

209
                mergePo(temppath, mcc.getLang(path), repopath/path)
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234

            index.removeFile(path)
            index.add([path])
            msg += m

            try:
                index.removeFromUnmergedBlobMap(path)
            except KeyError:
                pass

        # We need to write the index to file, before we can commit
        index.index.write()

        # Set the working tree to the new status of "local" and empty the index
        local.checkout(force=True)

        # We need to commit before we can test, if we simply can do a fast-forward
        # It seems that the diff against an index is very error-prone.
        index.index.commit(msg.strip(), parent_commits=(local.commit, remote))
    finally:
        # Get rid of temporary index file, that we don't need anymore
        if os.path.exists(index.index.path):
            logger.debug("Deleting temporary index file {index_path}.".format(index_path=index.index.path))
            os.unlink(index.index.path)

235
    if ff_possible_local and not local.commit.diff(remote):
236 237 238
        # We can fast-forward so let's do it
        local.commit = remote
        logger.info("Fast-forwarded local branch to {remote}.".format(remote=remote))
239 240 241
    elif local_uptodate and not local.commit.diff(oldcommit):
        # We don't have to do anything.
        local.commit = oldcommit
intrigeri's avatar
intrigeri committed
242
        logger.info("Nothing to commit - local branch can be fast-forwarded from {remote}.".format(remote=remote))
243 244 245 246

    # Set the working tree to the new status of "local" and empty the index
    local.checkout(force=True)

Sandro Knauß's avatar
Sandro Knauß committed
247

248 249 250 251
def commandline():
    import argparse
    import shutil

252 253 254
    global logger
    global prog

255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
    debugPath = pathlib.Path("__debug")

    logging.config.fileConfig('/var/lib/weblate/config/mergeWeblateChanges.conf')
    logger = logging.getLogger('')
    logger.level = logging.INFO

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-v", "--verbose",
        action='store_true',
        help="verbose logging.")
    parser.add_argument(
        "-d", "--debug",
        action='store_true',
        help="keep intermediate step files in {}/.".format(debugPath))
    parser.add_argument(
        "--local",
        default="master",
        help="name of local branch.")
    parser.add_argument(
        "--localCommit",
        help="reset the branch to commit first before merging. Useful for testing.")
    parser.add_argument(
        "repopath",
        type=pathlib.Path,
        help="path to the repository.")
    parser.add_argument(
        "remote",
        help="name of remote branch or commit id.")
    args = parser.parse_args()
    prog = parser.prog

    if args.verbose:
        logger.level = logging.DEBUG

    if args.debug:
        logger.info('Store all temporary files in %s/.', debugPath)
        if debugPath.exists():
            shutil.rmtree(debugPath)

        def temporaryDirectory(path):
            temppath = pathlib.Path(path)
            if debugPath not in temppath.parents:
                temppath = debugPath/temppath
            temppath.mkdir(parents=True)
            return temppath
    else:
        def temporaryDirectory(path):
            return tempfile.TemporaryDirectory()
304
    try:
305
        main(args.repopath, args.local, args.localCommit, args.remote)
306 307 308 309 310
    except:
        logger.exception("-- Something unexpected happened. Giving up. --")
        raise
    else:
        logger.info("-- Ended successfully.")
311 312 313 314


if __name__ == "__main__":
    commandline()