Commit b02eef89 authored by anonym's avatar anonym
Browse files

Merge branch 'bugfix/drop-parallel-collect-iuks' into 'stable'

Release process: drop dependency on parallel_collect_IUKs

See merge request tails/tails!267
parents bbc87dd5 a1176534
......@@ -2,13 +2,19 @@
import argparse
import logging
import os
import re
import subprocess
import sys
from typing import List
from pathlib import Path
from urllib.parse import urlparse
from urllib.request import Request, urlopen
JENKINS_IUKS_BASE_URL = "https://nightly.tails.boum.org/parallel_collect_IUKs/builds"
from bs4 import BeautifulSoup # type: ignore
JENKINS_IUKS_BASE_URL = "https://nightly.tails.boum.org/build_IUKs"
RSYNC_SERVER_HOSTNAME = "rsync.lizard"
LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
log = logging.getLogger()
......@@ -17,17 +23,26 @@ log = logging.getLogger()
def main():
parser = argparse.ArgumentParser(
description="Copy IUKs from Jenkins to our rsync server \
and verify that they match those built locally"
)
parser.add_argument("--hashes-file", type=str, action="store", required=True)
parser.add_argument("--jenkins-build-id", type=int, action="store", required=True)
and verify that they match those built locally")
parser.add_argument("--hashes-file",
type=str,
action="store",
required=True)
parser.add_argument("--jenkins-build-id",
type=int,
action="store",
required=True)
parser.add_argument("--work-dir", type=str, action="store", default=".")
parser.add_argument("-q", "--quiet", action="store_true",
parser.add_argument("-q",
"--quiet",
action="store_true",
help="quiet output")
parser.add_argument("--debug", action="store_true", help="debug output")
parser.add_argument("--skip-sending-hashes-file", action="store_true",
parser.add_argument("--skip-sending-hashes-file",
action="store_true",
help="Assume the hashes file was uploaded already")
parser.add_argument("--skip-downloading-iuks", action="store_true",
parser.add_argument("--skip-downloading-iuks",
action="store_true",
help="Assume the IUKs were already downloaded")
args = parser.parse_args()
......@@ -39,7 +54,7 @@ def main():
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
if not Path(args.hashes_file).exists():
log.error("%s does not exist" % (args.hashes_file))
log.error("%s does not exist", args.hashes_file)
sys.exit(1)
if not args.skip_sending_hashes_file:
......@@ -65,61 +80,112 @@ def main():
)
def send_hashes_file(
hashes_file: str,
desthost: str,
destdir: str) -> None:
def send_hashes_file(hashes_file: str, desthost: str, destdir: str) -> None:
log.info("Sending %(f)s to %(d)s on %(h)s…" % {
"f": hashes_file,
"d": destdir,
"h": desthost,
})
subprocess.run(
["scp", hashes_file, "%s:%s" % (desthost, destdir)],
check=True
)
["scp", hashes_file, "%s:%s" % (desthost, destdir)], check=True)
def iuks_listed_in(hashes_file: str) -> List[str]:
with Path(hashes_file).open() as f:
lines = f.readlines()
return [l.split(' ')[-1].rstrip() for l in lines]
def download_iuks_from_jenkins(
hashes_file: str,
desthost: str,
destdir: str,
jenkins_iuks_base_url: str,
jenkins_build_id: int) -> None:
log.info("Downloading IUKs from Jenkins to %s…" % (desthost))
iuks = iuks_listed_in(hashes_file)
log.debug("IUKS: %s" % ', '.join(iuks))
for iuk in iuks:
log.debug("Downloading %s to %s" % (iuk, destdir))
url = "%s/%s/archive/%s" % (
jenkins_iuks_base_url,
jenkins_build_id,
iuk
)
subprocess.run(
["ssh", desthost, "wget", "--quiet", "--no-clobber",
"-O", "%s/%s" % (destdir, iuk), url],
check=True
)
return [line.split(' ')[-1].rstrip() for line in lines]
def get_jenkins_iuks_urls(jenkins_iuks_base_url: str,
jenkins_build_id: int) -> List[str]:
urls: List[str] = []
source_version_index_url = jenkins_iuks_base_url + \
"/configurations/axis-SOURCE_VERSION"
for source_version_url in [
source_version_index_url + '/' + link.get('href')
for link in BeautifulSoup(
urlopen(Request(source_version_index_url)),
'html.parser').find_all(href=re.compile('^[1-9]'))
]:
axis_label_index_url = source_version_url + "axis-label_exp/"
log.debug("Looking at %s", axis_label_index_url)
label_urls = [
axis_label_index_url + link.get('href')
for link in BeautifulSoup(urlopen(Request(axis_label_index_url)),
'html.parser').find_all(
href=re.compile('^[a-z]'))
]
if len(label_urls) == 0:
log.debug("Found no label URL in %s, ignoring this source version",
axis_label_index_url)
continue
if len(label_urls) > 1:
log.error("Found too many label URLs in %s: %s",
axis_label_index_url, label_urls)
sys.exit(1)
label_url = label_urls[0]
artifacts_index_url = label_url + '/builds/' + str(
jenkins_build_id) + '/archive/'
log.debug("Looking at %s", artifacts_index_url)
iuk_urls = [
artifacts_index_url + link.get('href') for link in BeautifulSoup(
urlopen(Request(artifacts_index_url)), 'html.parser').find_all(
href=re.compile('[.]iuk$'))
]
if len(iuk_urls) == 0:
log.debug("Found no IUK URL in %s, ignoring this source version",
artifacts_index_url)
continue
if len(iuk_urls) > 1:
log.error("Found too many IUK URLs in %s: %s", artifacts_index_url,
iuk_urls)
sys.exit(1)
else:
iuk_url = iuk_urls[0]
urls.append(iuk_url)
log.debug("Found IUK URLs: %s", urls)
return urls
def download_iuks_from_jenkins(hashes_file: str, desthost: str, destdir: str,
jenkins_iuks_base_url: str,
jenkins_build_id: int) -> None:
log.info("Downloading IUKs from Jenkins to %s…", desthost)
expected_iuks = iuks_listed_in(hashes_file)
log.debug("IUKS: %s", ', '.join(expected_iuks)
jenkins_iuks_urls = get_jenkins_iuks_urls(jenkins_iuks_base_url,
jenkins_build_id)
jenkins_iuks = [
os.path.basename(urlparse(url).path) for url in jenkins_iuks_urls
]
if set(expected_iuks) != set(jenkins_iuks):
log.error(
"Jenkins' set of IUKs differs from local one:\n"
" - locally: %s\n"
" - Jenkins: %s\n",
expected_iuks, jenkins_iuks)
sys.exit(1)
for iuk_url in jenkins_iuks_urls:
log.debug("Downloading %s to %s", iuk_url, destdir)
subprocess.run([
"ssh", desthost, "wget", "--quiet", "--no-clobber",
"--directory-prefix=%s" % destdir, iuk_url
],
check=True)
def verify_iuks(desthost: str, iuks_dir: str, hashes_file: str) -> None:
log.info("Verifying that IUKs built on Jenkins match those you've built…")
try:
subprocess.run(
["ssh", desthost,
"cd '%(d)s' && sha256sum --check --strict '%(f)s'" % {
"d": iuks_dir,
"f": Path(hashes_file).name,
}],
check=True
)
subprocess.run([
"ssh", desthost,
"cd '%(d)s' && sha256sum --check --strict '%(f)s'" % {
"d": iuks_dir,
"f": Path(hashes_file).name,
}
],
check=True)
except subprocess.CalledProcessError:
print("\nERROR: IUKs built on Jenkins don't match yours\n",
file=sys.stderr)
......
# The ID of the Jenkins `parallel_collect_IUKs` build.
# The ID of the Jenkins `build_IUKs` build.
candidate_jenkins_iuks_build_id: FIXME
......@@ -17,7 +17,7 @@ Packages
To release Tails you'll need some packages installed:
* `docker.io gitlab-cli jq tidy mktorrent python3-debian python3-gitlab python3-jinja2 python3-voluptuous transmission-cli`
* `docker.io gitlab-cli jq tidy mktorrent python3-bs4 python3-debian python3-gitlab python3-jinja2 python3-voluptuous transmission-cli`
* [[!debpts squashfs-tools]] 1:4.4-1+0.tails1
from our custom `iukbuilder-stretch` APT suite.
* `perl5lib` [[dependencies|contribute/release_process/perl5lib#build-deps]]
......@@ -826,10 +826,7 @@ Build the Incremental Upgrade Kits on Jenkins
3. After a few seconds, a new build appears on top of the _Build
History_ sidebar. Click on the progress bar of this new build.
3. Once this build is completed, Jenkins will trigger a build of the downstream
`parallel_collect_IUKs` job.
4. Record the ID of the candidate Jenkins `parallel_collect_IUKs` build:
4. Record the ID of the candidate Jenkins `build_IUKs` build:
1. To update the `~/.config/tails/release_management/current.yml` template
with newly required variables, run:
......@@ -838,7 +835,7 @@ Build the Incremental Upgrade Kits on Jenkins
2. Edit `~/.config/tails/release_management/current.yml`
and set the `candidate_jenkins_iuks_build_id` value
to the ID of that `parallel_collect_IUKs` build (an integer):
to the ID of that `build_IUKs` build (an integer):
"${EDITOR:?}" ~/.config/tails/release_management/current.yml
......@@ -847,12 +844,11 @@ Build the Incremental Upgrade Kits on Jenkins
. $(./bin/rm-config generate-environment --stage built-iuks)
5. Wait until both `build_IUKs` and `parallel_collect_IUKs` jobs complete successfully.
5. Wait until the `build_IUKs` build completes successfully.
It should take about 10-15 minutes for each member of
the `$IUK_SOURCE_VERSIONS` list, distributed across `isobuilderN` workers.
6. Add the time it took for `build_IUKs` and
`parallel_collect_IUKs` to the table of [[!tails_ticket 17750]].
6. Add the time it took for `build_IUKs` to the table of [[!tails_ticket 17750]].
<a id="reproducibility-sanity-check-iuk"></a>
......@@ -862,7 +858,6 @@ Verify that Jenkins reproduced your IUKs
"${RELEASE_CHECKOUT:?}"/bin/copy-iuks-to-rsync-server-and-verify \
--hashes-file "${IUKS_HASHES:?}" \
--work-dir /srv/tmp \
--debug \
--jenkins-build-id "${CANDIDATE_JENKINS_IUKS_BUILD_ID:?}"
If this verification succeeds, move on to the next section.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment