Merge branch 'bugfix/drop-parallel-collect-iuks' into 'stable'

Release process: drop dependency on parallel_collect_IUKs

See merge request tails/tails!267

bin/copy-iuks-to-rsync-server-and-verify

@@ -2,13 +2,19 @@
 
 import argparse
 import logging
+import os
+import re
 import subprocess
 import sys
 
 from typing import List
 from pathlib import Path
+from urllib.parse import urlparse
+from urllib.request import Request, urlopen
 
-JENKINS_IUKS_BASE_URL = "https://nightly.tails.boum.org/parallel_collect_IUKs/builds"
+from bs4 import BeautifulSoup  # type: ignore
+
+JENKINS_IUKS_BASE_URL = "https://nightly.tails.boum.org/build_IUKs"
 RSYNC_SERVER_HOSTNAME = "rsync.lizard"
 LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
 log = logging.getLogger()
@@ -17,17 +23,26 @@ log = logging.getLogger()
 def main():
     parser = argparse.ArgumentParser(
         description="Copy IUKs from Jenkins to our rsync server \
-        and verify that they match those built locally"
-    )
-    parser.add_argument("--hashes-file", type=str, action="store", required=True)
-    parser.add_argument("--jenkins-build-id", type=int, action="store", required=True)
+        and verify that they match those built locally")
+    parser.add_argument("--hashes-file",
+                        type=str,
+                        action="store",
+                        required=True)
+    parser.add_argument("--jenkins-build-id",
+                        type=int,
+                        action="store",
+                        required=True)
     parser.add_argument("--work-dir", type=str, action="store", default=".")
-    parser.add_argument("-q", "--quiet", action="store_true",
+    parser.add_argument("-q",
+                        "--quiet",
+                        action="store_true",
                         help="quiet output")
     parser.add_argument("--debug", action="store_true", help="debug output")
-    parser.add_argument("--skip-sending-hashes-file", action="store_true",
+    parser.add_argument("--skip-sending-hashes-file",
+                        action="store_true",
                         help="Assume the hashes file was uploaded already")
-    parser.add_argument("--skip-downloading-iuks", action="store_true",
+    parser.add_argument("--skip-downloading-iuks",
+                        action="store_true",
                         help="Assume the IUKs were already downloaded")
     args = parser.parse_args()
 
@@ -39,7 +54,7 @@ def main():
         logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
 
     if not Path(args.hashes_file).exists():
-        log.error("%s does not exist" % (args.hashes_file))
+        log.error("%s does not exist", args.hashes_file)
         sys.exit(1)
 
     if not args.skip_sending_hashes_file:
@@ -65,61 +80,112 @@ def main():
     )
 
 
-def send_hashes_file(
-        hashes_file: str,
-        desthost: str,
-        destdir: str) -> None:
+def send_hashes_file(hashes_file: str, desthost: str, destdir: str) -> None:
     log.info("Sending %(f)s to %(d)s on %(h)s…" % {
         "f": hashes_file,
         "d": destdir,
         "h": desthost,
     })
     subprocess.run(
-        ["scp", hashes_file, "%s:%s" % (desthost, destdir)],
-        check=True
-    )
+        ["scp", hashes_file, "%s:%s" % (desthost, destdir)], check=True)
 
 
 def iuks_listed_in(hashes_file: str) -> List[str]:
     with Path(hashes_file).open() as f:
         lines = f.readlines()
-    return [l.split('  ')[-1].rstrip() for l in lines]
-
-
-def download_iuks_from_jenkins(
-        hashes_file: str,
-        desthost: str,
-        destdir: str,
-        jenkins_iuks_base_url: str,
-        jenkins_build_id: int) -> None:
-    log.info("Downloading IUKs from Jenkins to %s…" % (desthost))
-    iuks = iuks_listed_in(hashes_file)
-    log.debug("IUKS: %s" % ', '.join(iuks))
-    for iuk in iuks:
-        log.debug("Downloading %s to %s" % (iuk, destdir))
-        url = "%s/%s/archive/%s" % (
-            jenkins_iuks_base_url,
-            jenkins_build_id,
-            iuk
-        )
-        subprocess.run(
-            ["ssh", desthost, "wget", "--quiet", "--no-clobber",
-             "-O", "%s/%s" % (destdir, iuk), url],
-            check=True
-        )
+    return [line.split('  ')[-1].rstrip() for line in lines]
+
+
+def get_jenkins_iuks_urls(jenkins_iuks_base_url: str,
+                          jenkins_build_id: int) -> List[str]:
+    urls: List[str] = []
+    source_version_index_url = jenkins_iuks_base_url + \
+        "/configurations/axis-SOURCE_VERSION"
+    for source_version_url in [
+            source_version_index_url + '/' + link.get('href')
+            for link in BeautifulSoup(
+                urlopen(Request(source_version_index_url)),
+                'html.parser').find_all(href=re.compile('^[1-9]'))
+    ]:
+        axis_label_index_url = source_version_url + "axis-label_exp/"
+        log.debug("Looking at %s", axis_label_index_url)
+        label_urls = [
+            axis_label_index_url + link.get('href')
+            for link in BeautifulSoup(urlopen(Request(axis_label_index_url)),
+                                      'html.parser').find_all(
+                                          href=re.compile('^[a-z]'))
+        ]
+        if len(label_urls) == 0:
+            log.debug("Found no label URL in %s, ignoring this source version",
+                      axis_label_index_url)
+            continue
+        if len(label_urls) > 1:
+            log.error("Found too many label URLs in %s: %s",
+                      axis_label_index_url, label_urls)
+            sys.exit(1)
+        label_url = label_urls[0]
+
+        artifacts_index_url = label_url + '/builds/' + str(
+            jenkins_build_id) + '/archive/'
+        log.debug("Looking at %s", artifacts_index_url)
+        iuk_urls = [
+            artifacts_index_url + link.get('href') for link in BeautifulSoup(
+                urlopen(Request(artifacts_index_url)), 'html.parser').find_all(
+                    href=re.compile('[.]iuk$'))
+        ]
+        if len(iuk_urls) == 0:
+            log.debug("Found no IUK URL in %s, ignoring this source version",
+                      artifacts_index_url)
+            continue
+        if len(iuk_urls) > 1:
+            log.error("Found too many IUK URLs in %s: %s", artifacts_index_url,
+                      iuk_urls)
+            sys.exit(1)
+        else:
+            iuk_url = iuk_urls[0]
+        urls.append(iuk_url)
+    log.debug("Found IUK URLs: %s", urls)
+    return urls
+
+
+def download_iuks_from_jenkins(hashes_file: str, desthost: str, destdir: str,
+                               jenkins_iuks_base_url: str,
+                               jenkins_build_id: int) -> None:
+    log.info("Downloading IUKs from Jenkins to %s…", desthost)
+    expected_iuks = iuks_listed_in(hashes_file)
+    log.debug("IUKS: %s", ', '.join(expected_iuks)
+    jenkins_iuks_urls = get_jenkins_iuks_urls(jenkins_iuks_base_url,
+                                              jenkins_build_id)
+    jenkins_iuks = [
+        os.path.basename(urlparse(url).path) for url in jenkins_iuks_urls
+    ]
+    if set(expected_iuks) != set(jenkins_iuks):
+        log.error(
+            "Jenkins' set of IUKs differs from local one:\n"
+            " - locally: %s\n"
+            " - Jenkins: %s\n",
+            expected_iuks, jenkins_iuks)
+        sys.exit(1)
+    for iuk_url in jenkins_iuks_urls:
+        log.debug("Downloading %s to %s", iuk_url, destdir)
+        subprocess.run([
+            "ssh", desthost, "wget", "--quiet", "--no-clobber",
+            "--directory-prefix=%s" % destdir, iuk_url
+        ],
+                       check=True)
 
 
 def verify_iuks(desthost: str, iuks_dir: str, hashes_file: str) -> None:
     log.info("Verifying that IUKs built on Jenkins match those you've built…")
     try:
-        subprocess.run(
-            ["ssh", desthost,
-             "cd '%(d)s' && sha256sum --check --strict '%(f)s'" % {
-                 "d": iuks_dir,
-                 "f": Path(hashes_file).name,
-             }],
-            check=True
-        )
+        subprocess.run([
+            "ssh", desthost,
+            "cd '%(d)s' && sha256sum --check --strict '%(f)s'" % {
+                "d": iuks_dir,
+                "f": Path(hashes_file).name,
+            }
+        ],
+                       check=True)
     except subprocess.CalledProcessError:
         print("\nERROR: IUKs built on Jenkins don't match yours\n",
               file=sys.stderr)

config/release_management/templates/built-iuks.yml

-# The ID of the Jenkins `parallel_collect_IUKs` build.
+# The ID of the Jenkins `build_IUKs` build.
 candidate_jenkins_iuks_build_id: FIXME

wiki/src/contribute/release_process.mdwn

@@ -17,7 +17,7 @@ Packages
 
 To release Tails you'll need some packages installed:
 
-* `docker.io gitlab-cli jq tidy mktorrent python3-debian python3-gitlab python3-jinja2 python3-voluptuous transmission-cli`
+* `docker.io gitlab-cli jq tidy mktorrent python3-bs4 python3-debian python3-gitlab python3-jinja2 python3-voluptuous transmission-cli`
 * [[!debpts squashfs-tools]] 1:4.4-1+0.tails1
   from our custom `iukbuilder-stretch` APT suite.
 * `perl5lib` [[dependencies|contribute/release_process/perl5lib#build-deps]]
@@ -826,10 +826,7 @@ Build the Incremental Upgrade Kits on Jenkins
 3. After a few seconds, a new build appears on top of the _Build
    History_ sidebar. Click on the progress bar of this new build.
 
-3. Once this build is completed, Jenkins will trigger a build of the downstream
-   `parallel_collect_IUKs` job.
-
-4. Record the ID of the candidate Jenkins `parallel_collect_IUKs` build:
+4. Record the ID of the candidate Jenkins `build_IUKs` build:
 
    1. To update the `~/.config/tails/release_management/current.yml` template
       with newly required variables, run:
@@ -838,7 +835,7 @@ Build the Incremental Upgrade Kits on Jenkins
 
    2. Edit `~/.config/tails/release_management/current.yml`
       and set the `candidate_jenkins_iuks_build_id` value
-      to the ID of that `parallel_collect_IUKs` build (an integer):
+      to the ID of that `build_IUKs` build (an integer):
 
           "${EDITOR:?}" ~/.config/tails/release_management/current.yml
 
@@ -847,12 +844,11 @@ Build the Incremental Upgrade Kits on Jenkins
 
           . $(./bin/rm-config generate-environment --stage built-iuks)
 
-5. Wait until both `build_IUKs` and `parallel_collect_IUKs` jobs complete successfully.
+5. Wait until the `build_IUKs` build completes successfully.
    It should take about 10-15 minutes for each member of
    the `$IUK_SOURCE_VERSIONS` list, distributed across `isobuilderN` workers.
 
-6. Add the time it took for `build_IUKs` and
-   `parallel_collect_IUKs` to the table of [[!tails_ticket 17750]].
+6. Add the time it took for `build_IUKs` to the table of [[!tails_ticket 17750]].
 
 <a id="reproducibility-sanity-check-iuk"></a>
 
@@ -862,7 +858,6 @@ Verify that Jenkins reproduced your IUKs
     "${RELEASE_CHECKOUT:?}"/bin/copy-iuks-to-rsync-server-and-verify \
        --hashes-file "${IUKS_HASHES:?}" \
        --work-dir /srv/tmp \
-       --debug \
        --jenkins-build-id "${CANDIDATE_JENKINS_IUKS_BUILD_ID:?}"
 
 If this verification succeeds, move on to the next section.