Commit b02eef89 authored by anonym's avatar anonym
Browse files

Merge branch 'bugfix/drop-parallel-collect-iuks' into 'stable'

Release process: drop dependency on parallel_collect_IUKs

See merge request !267
parents bbc87dd5 a1176534
...@@ -2,13 +2,19 @@ ...@@ -2,13 +2,19 @@
import argparse import argparse
import logging import logging
import os
import re
import subprocess import subprocess
import sys import sys
from typing import List from typing import List
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse
from urllib.request import Request, urlopen
JENKINS_IUKS_BASE_URL = "https://nightly.tails.boum.org/parallel_collect_IUKs/builds" from bs4 import BeautifulSoup # type: ignore
JENKINS_IUKS_BASE_URL = "https://nightly.tails.boum.org/build_IUKs"
RSYNC_SERVER_HOSTNAME = "rsync.lizard" RSYNC_SERVER_HOSTNAME = "rsync.lizard"
LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s" LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
log = logging.getLogger() log = logging.getLogger()
...@@ -17,17 +23,26 @@ log = logging.getLogger() ...@@ -17,17 +23,26 @@ log = logging.getLogger()
def main(): def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Copy IUKs from Jenkins to our rsync server \ description="Copy IUKs from Jenkins to our rsync server \
and verify that they match those built locally" and verify that they match those built locally")
) parser.add_argument("--hashes-file",
parser.add_argument("--hashes-file", type=str, action="store", required=True) type=str,
parser.add_argument("--jenkins-build-id", type=int, action="store", required=True) action="store",
required=True)
parser.add_argument("--jenkins-build-id",
type=int,
action="store",
required=True)
parser.add_argument("--work-dir", type=str, action="store", default=".") parser.add_argument("--work-dir", type=str, action="store", default=".")
parser.add_argument("-q", "--quiet", action="store_true", parser.add_argument("-q",
"--quiet",
action="store_true",
help="quiet output") help="quiet output")
parser.add_argument("--debug", action="store_true", help="debug output") parser.add_argument("--debug", action="store_true", help="debug output")
parser.add_argument("--skip-sending-hashes-file", action="store_true", parser.add_argument("--skip-sending-hashes-file",
action="store_true",
help="Assume the hashes file was uploaded already") help="Assume the hashes file was uploaded already")
parser.add_argument("--skip-downloading-iuks", action="store_true", parser.add_argument("--skip-downloading-iuks",
action="store_true",
help="Assume the IUKs were already downloaded") help="Assume the IUKs were already downloaded")
args = parser.parse_args() args = parser.parse_args()
...@@ -39,7 +54,7 @@ def main(): ...@@ -39,7 +54,7 @@ def main():
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
if not Path(args.hashes_file).exists(): if not Path(args.hashes_file).exists():
log.error("%s does not exist" % (args.hashes_file)) log.error("%s does not exist", args.hashes_file)
sys.exit(1) sys.exit(1)
if not args.skip_sending_hashes_file: if not args.skip_sending_hashes_file:
...@@ -65,61 +80,112 @@ def main(): ...@@ -65,61 +80,112 @@ def main():
) )
def send_hashes_file( def send_hashes_file(hashes_file: str, desthost: str, destdir: str) -> None:
hashes_file: str,
desthost: str,
destdir: str) -> None:
log.info("Sending %(f)s to %(d)s on %(h)s…" % { log.info("Sending %(f)s to %(d)s on %(h)s…" % {
"f": hashes_file, "f": hashes_file,
"d": destdir, "d": destdir,
"h": desthost, "h": desthost,
}) })
subprocess.run( subprocess.run(
["scp", hashes_file, "%s:%s" % (desthost, destdir)], ["scp", hashes_file, "%s:%s" % (desthost, destdir)], check=True)
check=True
)
def iuks_listed_in(hashes_file: str) -> List[str]: def iuks_listed_in(hashes_file: str) -> List[str]:
with Path(hashes_file).open() as f: with Path(hashes_file).open() as f:
lines = f.readlines() lines = f.readlines()
return [l.split(' ')[-1].rstrip() for l in lines] return [line.split(' ')[-1].rstrip() for line in lines]
def download_iuks_from_jenkins( def get_jenkins_iuks_urls(jenkins_iuks_base_url: str,
hashes_file: str, jenkins_build_id: int) -> List[str]:
desthost: str, urls: List[str] = []
destdir: str, source_version_index_url = jenkins_iuks_base_url + \
jenkins_iuks_base_url: str, "/configurations/axis-SOURCE_VERSION"
jenkins_build_id: int) -> None: for source_version_url in [
log.info("Downloading IUKs from Jenkins to %s…" % (desthost)) source_version_index_url + '/' + link.get('href')
iuks = iuks_listed_in(hashes_file) for link in BeautifulSoup(
log.debug("IUKS: %s" % ', '.join(iuks)) urlopen(Request(source_version_index_url)),
for iuk in iuks: 'html.parser').find_all(href=re.compile('^[1-9]'))
log.debug("Downloading %s to %s" % (iuk, destdir)) ]:
url = "%s/%s/archive/%s" % ( axis_label_index_url = source_version_url + "axis-label_exp/"
jenkins_iuks_base_url, log.debug("Looking at %s", axis_label_index_url)
jenkins_build_id, label_urls = [
iuk axis_label_index_url + link.get('href')
) for link in BeautifulSoup(urlopen(Request(axis_label_index_url)),
subprocess.run( 'html.parser').find_all(
["ssh", desthost, "wget", "--quiet", "--no-clobber", href=re.compile('^[a-z]'))
"-O", "%s/%s" % (destdir, iuk), url], ]
check=True if len(label_urls) == 0:
) log.debug("Found no label URL in %s, ignoring this source version",
axis_label_index_url)
continue
if len(label_urls) > 1:
log.error("Found too many label URLs in %s: %s",
axis_label_index_url, label_urls)
sys.exit(1)
label_url = label_urls[0]
artifacts_index_url = label_url + '/builds/' + str(
jenkins_build_id) + '/archive/'
log.debug("Looking at %s", artifacts_index_url)
iuk_urls = [
artifacts_index_url + link.get('href') for link in BeautifulSoup(
urlopen(Request(artifacts_index_url)), 'html.parser').find_all(
href=re.compile('[.]iuk$'))
]
if len(iuk_urls) == 0:
log.debug("Found no IUK URL in %s, ignoring this source version",
artifacts_index_url)
continue
if len(iuk_urls) > 1:
log.error("Found too many IUK URLs in %s: %s", artifacts_index_url,
iuk_urls)
sys.exit(1)
else:
iuk_url = iuk_urls[0]
urls.append(iuk_url)
log.debug("Found IUK URLs: %s", urls)
return urls
def download_iuks_from_jenkins(hashes_file: str, desthost: str, destdir: str,
jenkins_iuks_base_url: str,
jenkins_build_id: int) -> None:
log.info("Downloading IUKs from Jenkins to %s…", desthost)
expected_iuks = iuks_listed_in(hashes_file)
log.debug("IUKS: %s", ', '.join(expected_iuks)
jenkins_iuks_urls = get_jenkins_iuks_urls(jenkins_iuks_base_url,
jenkins_build_id)
jenkins_iuks = [
os.path.basename(urlparse(url).path) for url in jenkins_iuks_urls
]
if set(expected_iuks) != set(jenkins_iuks):
log.error(
"Jenkins' set of IUKs differs from local one:\n"
" - locally: %s\n"
" - Jenkins: %s\n",
expected_iuks, jenkins_iuks)
sys.exit(1)
for iuk_url in jenkins_iuks_urls:
log.debug("Downloading %s to %s", iuk_url, destdir)
subprocess.run([
"ssh", desthost, "wget", "--quiet", "--no-clobber",
"--directory-prefix=%s" % destdir, iuk_url
],
check=True)
def verify_iuks(desthost: str, iuks_dir: str, hashes_file: str) -> None: def verify_iuks(desthost: str, iuks_dir: str, hashes_file: str) -> None:
log.info("Verifying that IUKs built on Jenkins match those you've built…") log.info("Verifying that IUKs built on Jenkins match those you've built…")
try: try:
subprocess.run( subprocess.run([
["ssh", desthost, "ssh", desthost,
"cd '%(d)s' && sha256sum --check --strict '%(f)s'" % { "cd '%(d)s' && sha256sum --check --strict '%(f)s'" % {
"d": iuks_dir, "d": iuks_dir,
"f": Path(hashes_file).name, "f": Path(hashes_file).name,
}], }
check=True ],
) check=True)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
print("\nERROR: IUKs built on Jenkins don't match yours\n", print("\nERROR: IUKs built on Jenkins don't match yours\n",
file=sys.stderr) file=sys.stderr)
......
# The ID of the Jenkins `parallel_collect_IUKs` build. # The ID of the Jenkins `build_IUKs` build.
candidate_jenkins_iuks_build_id: FIXME candidate_jenkins_iuks_build_id: FIXME
...@@ -17,7 +17,7 @@ Packages ...@@ -17,7 +17,7 @@ Packages
To release Tails you'll need some packages installed: To release Tails you'll need some packages installed:
* `docker.io gitlab-cli jq tidy mktorrent python3-debian python3-gitlab python3-jinja2 python3-voluptuous transmission-cli` * `docker.io gitlab-cli jq tidy mktorrent python3-bs4 python3-debian python3-gitlab python3-jinja2 python3-voluptuous transmission-cli`
* [[!debpts squashfs-tools]] 1:4.4-1+0.tails1 * [[!debpts squashfs-tools]] 1:4.4-1+0.tails1
from our custom `iukbuilder-stretch` APT suite. from our custom `iukbuilder-stretch` APT suite.
* `perl5lib` [[dependencies|contribute/release_process/perl5lib#build-deps]] * `perl5lib` [[dependencies|contribute/release_process/perl5lib#build-deps]]
...@@ -826,10 +826,7 @@ Build the Incremental Upgrade Kits on Jenkins ...@@ -826,10 +826,7 @@ Build the Incremental Upgrade Kits on Jenkins
3. After a few seconds, a new build appears on top of the _Build 3. After a few seconds, a new build appears on top of the _Build
History_ sidebar. Click on the progress bar of this new build. History_ sidebar. Click on the progress bar of this new build.
3. Once this build is completed, Jenkins will trigger a build of the downstream 4. Record the ID of the candidate Jenkins `build_IUKs` build:
`parallel_collect_IUKs` job.
4. Record the ID of the candidate Jenkins `parallel_collect_IUKs` build:
1. To update the `~/.config/tails/release_management/current.yml` template 1. To update the `~/.config/tails/release_management/current.yml` template
with newly required variables, run: with newly required variables, run:
...@@ -838,7 +835,7 @@ Build the Incremental Upgrade Kits on Jenkins ...@@ -838,7 +835,7 @@ Build the Incremental Upgrade Kits on Jenkins
2. Edit `~/.config/tails/release_management/current.yml` 2. Edit `~/.config/tails/release_management/current.yml`
and set the `candidate_jenkins_iuks_build_id` value and set the `candidate_jenkins_iuks_build_id` value
to the ID of that `parallel_collect_IUKs` build (an integer): to the ID of that `build_IUKs` build (an integer):
"${EDITOR:?}" ~/.config/tails/release_management/current.yml "${EDITOR:?}" ~/.config/tails/release_management/current.yml
...@@ -847,12 +844,11 @@ Build the Incremental Upgrade Kits on Jenkins ...@@ -847,12 +844,11 @@ Build the Incremental Upgrade Kits on Jenkins
. $(./bin/rm-config generate-environment --stage built-iuks) . $(./bin/rm-config generate-environment --stage built-iuks)
5. Wait until both `build_IUKs` and `parallel_collect_IUKs` jobs complete successfully. 5. Wait until the `build_IUKs` build completes successfully.
It should take about 10-15 minutes for each member of It should take about 10-15 minutes for each member of
the `$IUK_SOURCE_VERSIONS` list, distributed across `isobuilderN` workers. the `$IUK_SOURCE_VERSIONS` list, distributed across `isobuilderN` workers.
6. Add the time it took for `build_IUKs` and 6. Add the time it took for `build_IUKs` to the table of [[!tails_ticket 17750]].
`parallel_collect_IUKs` to the table of [[!tails_ticket 17750]].
<a id="reproducibility-sanity-check-iuk"></a> <a id="reproducibility-sanity-check-iuk"></a>
...@@ -862,7 +858,6 @@ Verify that Jenkins reproduced your IUKs ...@@ -862,7 +858,6 @@ Verify that Jenkins reproduced your IUKs
"${RELEASE_CHECKOUT:?}"/bin/copy-iuks-to-rsync-server-and-verify \ "${RELEASE_CHECKOUT:?}"/bin/copy-iuks-to-rsync-server-and-verify \
--hashes-file "${IUKS_HASHES:?}" \ --hashes-file "${IUKS_HASHES:?}" \
--work-dir /srv/tmp \ --work-dir /srv/tmp \
--debug \
--jenkins-build-id "${CANDIDATE_JENKINS_IUKS_BUILD_ID:?}" --jenkins-build-id "${CANDIDATE_JENKINS_IUKS_BUILD_ID:?}"
If this verification succeeds, move on to the next section. If this verification succeeds, move on to the next section.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment