Unverified Commit b5c897a4 authored by intrigeri's avatar intrigeri
Browse files

Release process: drop dependency on parallel_collect_IUKs

The only reason why we needed parallel_collect_IUKs so far was so that
bin/copy-iuks-to-rsync-server-and-verify, which is run by the RM during the
release process, can fetch every IUK built by Jenkins from a single place.

But parallel_collect_IUKs is painfully slow (sysadmin#17744).

So let's instead teach bin/copy-iuks-to-rsync-server-and-verify how to locate
the IUKs it needs under
https://nightly.tails.boum.org/build_IUKs/configurations/ and download them from
there on rsync.lizard.
parent f8f7fbf3
...@@ -2,13 +2,19 @@ ...@@ -2,13 +2,19 @@
import argparse import argparse
import logging import logging
import os
import re
import subprocess import subprocess
import sys import sys
from typing import List from typing import List
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse
from urllib.request import Request, urlopen
JENKINS_IUKS_BASE_URL = "https://nightly.tails.boum.org/parallel_collect_IUKs/builds" from bs4 import BeautifulSoup # type: ignore
JENKINS_IUKS_BASE_URL = "https://nightly.tails.boum.org/build_IUKs"
RSYNC_SERVER_HOSTNAME = "rsync.lizard" RSYNC_SERVER_HOSTNAME = "rsync.lizard"
LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s" LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
log = logging.getLogger() log = logging.getLogger()
...@@ -90,24 +96,83 @@ def iuks_listed_in(hashes_file: str) -> List[str]: ...@@ -90,24 +96,83 @@ def iuks_listed_in(hashes_file: str) -> List[str]:
return [l.split(' ')[-1].rstrip() for l in lines] return [l.split(' ')[-1].rstrip() for l in lines]
def get_jenkins_iuks_urls(jenkins_iuks_base_url: str,
jenkins_build_id: int) -> List[str]:
urls: List[str] = []
source_version_index_url = jenkins_iuks_base_url + \
"/configurations/axis-SOURCE_VERSION"
for source_version_url in [
source_version_index_url + '/' + link.get('href')
for link in BeautifulSoup(
urlopen(Request(source_version_index_url)),
'html.parser').find_all(href=re.compile('^[1-9]'))
]:
axis_label_index_url = source_version_url + "axis-label_exp/"
log.debug("Looking at %s", axis_label_index_url)
label_urls = [
axis_label_index_url + link.get('href')
for link in BeautifulSoup(urlopen(Request(axis_label_index_url)),
'html.parser').find_all(
href=re.compile('^[a-z]'))
]
if len(label_urls) == 0:
log.debug("Found no label URL in %s, ignoring this source version",
axis_label_index_url)
continue
if len(label_urls) > 1:
log.error("Found too many label URLs in %s: %s",
axis_label_index_url, label_urls)
sys.exit(1)
label_url = label_urls[0]
artifacts_index_url = label_url + '/builds/' + str(
jenkins_build_id) + '/archive/'
log.debug("Looking at %s", artifacts_index_url)
iuk_urls = [
artifacts_index_url + link.get('href') for link in BeautifulSoup(
urlopen(Request(artifacts_index_url)), 'html.parser').find_all(
href=re.compile('[.]iuk$'))
]
if len(iuk_urls) == 0:
log.debug("Found no IUK URL in %s, ignoring this source version",
artifacts_index_url)
continue
if len(iuk_urls) > 1:
log.error("Found too many IUK URLs in %s: %s", artifacts_index_url,
iuk_urls)
sys.exit(1)
else:
iuk_url = iuk_urls[0]
urls.append(iuk_url)
log.debug("Found IUK URLs: %s", urls)
return urls
def download_iuks_from_jenkins(hashes_file: str, desthost: str, destdir: str, def download_iuks_from_jenkins(hashes_file: str, desthost: str, destdir: str,
jenkins_iuks_base_url: str, jenkins_iuks_base_url: str,
jenkins_build_id: int) -> None: jenkins_build_id: int) -> None:
log.info("Downloading IUKs from Jenkins to %s…" % (desthost)) log.info("Downloading IUKs from Jenkins to %s…" % (desthost))
iuks = iuks_listed_in(hashes_file) expected_iuks = iuks_listed_in(hashes_file)
log.debug("IUKS: %s" % ', '.join(iuks)) log.debug("IUKS: %s" % ', '.join(expected_iuks))
for iuk in iuks: jenkins_iuks_urls = get_jenkins_iuks_urls(jenkins_iuks_base_url,
log.debug("Downloading %s to %s" % (iuk, destdir)) jenkins_build_id)
url = "%s/%s/archive/%s" % ( jenkins_iuks = [
jenkins_iuks_base_url, os.path.basename(urlparse(url).path) for url in jenkins_iuks_urls
jenkins_build_id, ]
iuk if set(expected_iuks) != set(jenkins_iuks):
) log.error(
subprocess.run( "Jenkins' set of IUKs differs from local one:\n"
["ssh", desthost, "wget", "--quiet", "--no-clobber", " - locally: %s\n"
"-O", "%s/%s" % (destdir, iuk), url], " - Jenkins: %s\n",
check=True expected_iuks, jenkins_iuks)
) sys.exit(1)
for iuk_url in jenkins_iuks_urls:
log.debug("Downloading %s to %s", iuk_url, destdir)
subprocess.run([
"ssh", desthost, "wget", "--quiet", "--no-clobber",
"--directory-prefix=%s" % destdir, iuk_url
],
check=True)
def verify_iuks(desthost: str, iuks_dir: str, hashes_file: str) -> None: def verify_iuks(desthost: str, iuks_dir: str, hashes_file: str) -> None:
......
# The ID of the Jenkins `parallel_collect_IUKs` build. # The ID of the Jenkins `build_IUKs` build.
candidate_jenkins_iuks_build_id: FIXME candidate_jenkins_iuks_build_id: FIXME
...@@ -17,7 +17,7 @@ Packages ...@@ -17,7 +17,7 @@ Packages
To release Tails you'll need some packages installed: To release Tails you'll need some packages installed:
* `docker.io gitlab-cli jq tidy mktorrent python3-debian python3-gitlab python3-jinja2 python3-voluptuous transmission-cli` * `docker.io gitlab-cli jq tidy mktorrent python3-bs4 python3-debian python3-gitlab python3-jinja2 python3-voluptuous transmission-cli`
* [[!debpts squashfs-tools]] 1:4.4-1+0.tails1 * [[!debpts squashfs-tools]] 1:4.4-1+0.tails1
from our custom `iukbuilder-stretch` APT suite. from our custom `iukbuilder-stretch` APT suite.
* `perl5lib` [[dependencies|contribute/release_process/perl5lib#build-deps]] * `perl5lib` [[dependencies|contribute/release_process/perl5lib#build-deps]]
...@@ -826,10 +826,7 @@ Build the Incremental Upgrade Kits on Jenkins ...@@ -826,10 +826,7 @@ Build the Incremental Upgrade Kits on Jenkins
3. After a few seconds, a new build appears on top of the _Build 3. After a few seconds, a new build appears on top of the _Build
History_ sidebar. Click on the progress bar of this new build. History_ sidebar. Click on the progress bar of this new build.
3. Once this build is completed, Jenkins will trigger a build of the downstream 4. Record the ID of the candidate Jenkins `build_IUKs` build:
`parallel_collect_IUKs` job.
4. Record the ID of the candidate Jenkins `parallel_collect_IUKs` build:
1. To update the `~/.config/tails/release_management/current.yml` template 1. To update the `~/.config/tails/release_management/current.yml` template
with newly required variables, run: with newly required variables, run:
...@@ -838,7 +835,7 @@ Build the Incremental Upgrade Kits on Jenkins ...@@ -838,7 +835,7 @@ Build the Incremental Upgrade Kits on Jenkins
2. Edit `~/.config/tails/release_management/current.yml` 2. Edit `~/.config/tails/release_management/current.yml`
and set the `candidate_jenkins_iuks_build_id` value and set the `candidate_jenkins_iuks_build_id` value
to the ID of that `parallel_collect_IUKs` build (an integer): to the ID of that `build_IUKs` build (an integer):
"${EDITOR:?}" ~/.config/tails/release_management/current.yml "${EDITOR:?}" ~/.config/tails/release_management/current.yml
...@@ -847,12 +844,11 @@ Build the Incremental Upgrade Kits on Jenkins ...@@ -847,12 +844,11 @@ Build the Incremental Upgrade Kits on Jenkins
. $(./bin/rm-config generate-environment --stage built-iuks) . $(./bin/rm-config generate-environment --stage built-iuks)
5. Wait until both `build_IUKs` and `parallel_collect_IUKs` jobs complete successfully. 5. Wait until the `build_IUKs` build completes successfully.
It should take about 10-15 minutes for each member of It should take about 10-15 minutes for each member of
the `$IUK_SOURCE_VERSIONS` list, distributed across `isobuilderN` workers. the `$IUK_SOURCE_VERSIONS` list, distributed across `isobuilderN` workers.
6. Add the time it took for `build_IUKs` and 6. Add the time it took for `build_IUKs` to the table of [[!tails_ticket 17750]].
`parallel_collect_IUKs` to the table of [[!tails_ticket 17750]].
<a id="reproducibility-sanity-check-iuk"></a> <a id="reproducibility-sanity-check-iuk"></a>
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment