Commit 034b2df9 authored by intrigeri's avatar intrigeri
Browse files

Merge remote-tracking branch 'origin/master' into update-jenkins-core-and-plugins

parents 8f58ff06 bb7742eb
......@@ -8,7 +8,7 @@ Many classes depend on the puppetlabs-stdlib module.
Many classes depend on the saz/sudo module.
tails::reprepro depends on the reprepro module:
http://git-tails.immerda.ch/puppet-reprepro/
https://gitlab.tails.boum.org/tails/puppet-reprepro
The tails and tails::reprepro classes depend on the shared-common module:
https://gitlab.com/shared-puppet-modules-group/common
......@@ -22,10 +22,10 @@ The tails::check_mirrors class depends on the vcsrepo module:
https://github.com/puppetlabs/puppetlabs-vcsrepo
The tails::gitolite class depends on the gitolite module:
https://git-tails.immerda.ch/puppet-gitolite
https://gitlab.tails.boum.org/tails/puppet-gitolite
The tails::jenkins class depends on the jenkins module:
https://git-tails.immerda.ch/puppet-jenkins
https://gitlab.tails.boum.org/tails/puppet-jenkins
Classes
=======
......
#!/bin/sh
set -eu
/usr/bin/git push --mirror labs
......@@ -5,9 +5,6 @@ set -eu
# it will error out if any of the hooks fails, this may be good for hooks that
# could depend on each other running, but could be annoying if we don't have that
/usr/bin/find -L "${GIT_DIR}/hooks/post-update.d" \
-type f -executable -print0 \
| while IFS= read -r -d '' file
do
"$file" "$@"
for hook in "${GIT_DIR}"/hooks/post-update.d/* ; do
"$hook" "$@"
done
......@@ -4,7 +4,6 @@ set -e
gitolite_root=/var/lib/gitolite3
repository_root="${gitolite_root}/repositories"
tails_common_hookdir="${gitolite_root}/.gitolite/hooks/tails_common"
jenkins_jobs_hookdir="${gitolite_root}/.gitolite/hooks/jenkins_jobs"
install_post_update_dot_d_hook () {
repository="$1"
......@@ -61,13 +60,11 @@ install_website_underlays_post_update_hook () {
case $(basename "$repository") in
etcher-binary.git|mirror-pool-dispatcher.git|mirror-pool.git|promotion-material.git|uui-binary.git)
install_post_update_dot_d_hook "$repository"
deinstall_post_update_hook "$repository" mirror boum
install_mirror_post_update_hook "$repository" immerda
install_website_underlays_post_update_hook "$repository" www
;;
jenkins-jobs.git)
install_post_update_dot_d_hook "$repository"
install_mirror_post_update_hook "$repository" immerda
install_mirror_post_update_hook "$repository" gitlab
;;
puppet-*)
if ! grep -qs -x "$(basename "$repository")" \
......@@ -75,15 +72,15 @@ install_website_underlays_post_update_hook () {
continue
fi
install_post_update_dot_d_hook "$repository"
install_mirror_post_update_hook "$repository" immerda
install_mirror_post_update_hook "$repository" gitlab
;;
tails.git)
install_post_update_dot_d_hook "$repository"
deinstall_post_update_hook "$repository" website_ping boum
install_website_ping_post_update_hook "$repository" www
install_mirror_post_update_hook "$repository" gitlab
# Needed because many pieces of infra deployed by this Puppet
# module use the https://git-tails.immerda.ch/tails Git remote,
# so for now we maintain an up-to-date mirror of tails.git there
install_mirror_post_update_hook "$repository" immerda
install_mirror_post_update_hook "$repository" labs
install_mirror_post_update_hook "$repository" salsa
;;
*)
......
#!/bin/bash
set -e
gitolite_root=/var/lib/gitolite3
......@@ -18,19 +19,34 @@ repository_root="${gitolite_root}/repositories"
fi
(
cd "$repository_dir"
git remote | grep -qs -x gitlab || \
git remote add gitlab "git@gitlab.com:Tails/$repository"
git remote | grep -qs -x immerda || \
git remote add immerda "tails@git.tails.boum.org:$repository"
if git remote | grep -qs -x labs; then
git remote set-url labs "tails_git@redmine.tails.boum.org:/srv/repositories/$repository"
else
git remote add labs "tails_git@redmine.tails.boum.org:/srv/repositories/$repository"
fi
git remote | grep -qs -x salsa || \
git remote add salsa "git@salsa.debian.org:tails-team/$repository"
case "$repository" in
# The puppet-* and jenkins-jobs repos should now also
# mirror to Tails GitLab instance.
puppet-*|jenkins-jobs.git)
if git remote | grep -qs -x gitlab; then
git remote set-url gitlab "git@gitlab-ssh.tails.boum.org:tails/$repository"
else
git remote add gitlab "git@gitlab-ssh.tails.boum.org:tails/$repository"
fi
;;
# All the others are now mirrors of their counterpart repos
# in Tails GitLab instance.
*)
;;
esac
)
;;
weblate-gatekeeper.git)
cd "$repository_dir"
git remote | grep -qs -x gitlab || \
git remote add gitlab "git@gitlab.tails.boum.org:tails/tails.git"
;;
*)
;;
esac
......
#!/bin/sh
# This script is called by the cleanup_build_jobs_leftovers jenkins-job-builder
# builder, defined in jenkins-jobs.git.
set -e
set -u
set -x
......@@ -20,7 +23,7 @@ if [ -b "$vmdebootstrap_device" ]; then
while ! sudo umount -f --verbose ${vmdebootstrap_device} && [ $tries -lt 12 ]; do
sudo fuser --mount ${vmdebootstrap_device} --kill || true
sleep 5
tries=$(expr $tries + 1)
tries=$(("$tries" + 1))
done
fi
......
#!/bin/sh
# This script is called by the collect_build_environment jenkins-job-builder
# builder, defined in jenkins-jobs.git.
set -e
set -u
set -x
OUT_FILE_NAME="$1"
[ $OUT_FILE_NAME ] || exit 2
[ $GIT_COMMIT ] || exit 3
[ $GIT_BRANCH ] || exit 4
[ $BUILD_NUMBER ] || exit 5
[ $WORKSPACE ] || exit 6
[ "$OUT_FILE_NAME" ] || exit 2
[ "$GIT_COMMIT" ] || exit 3
[ "$GIT_BRANCH" ] || exit 4
[ "$BUILD_NUMBER" ] || exit 5
[ "$WORKSPACE" ] || exit 6
BASE_BRANCH=$(head -n1 config/base_branch)
BASE_BRANCH_HEAD=$(git rev-parse "origin/${BASE_BRANCH}")
[ $BASE_BRANCH_HEAD ] || exit 7
[ "$BASE_BRANCH_HEAD" ] || exit 7
OUT_FILE="build-artifacts/${OUT_FILE_NAME}"
. ${WORKSPACE}/tmp/recipient
echo "UPSTREAMJOB_GIT_COMMIT=${GIT_COMMIT}" >> ${OUT_FILE}
echo "UPSTREAMJOB_GIT_BRANCH=${GIT_BRANCH}" >> ${OUT_FILE}
echo "UPSTREAMJOB_GIT_BASE_BRANCH=${BASE_BRANCH}" >> ${OUT_FILE}
echo "UPSTREAMJOB_GIT_BASE_BRANCH_HEAD=${BASE_BRANCH_HEAD}" >> ${OUT_FILE}
echo "UPSTREAMJOB_BUILD_NUMBER=${BUILD_NUMBER}" >> ${OUT_FILE}
echo "UPSTREAMJOB_NAME=${JOB_NAME}" >> ${OUT_FILE}
{
echo "UPSTREAMJOB_GIT_COMMIT=${GIT_COMMIT}"
echo "UPSTREAMJOB_GIT_BRANCH=${GIT_BRANCH}"
echo "UPSTREAMJOB_GIT_BASE_BRANCH=${BASE_BRANCH}"
echo "UPSTREAMJOB_GIT_BASE_BRANCH_HEAD=${BASE_BRANCH_HEAD}"
echo "UPSTREAMJOB_BUILD_NUMBER=${BUILD_NUMBER}"
echo "UPSTREAMJOB_NAME=${JOB_NAME}"
} >> "${OUT_FILE}"
case "${GIT_BRANCH}" in
origin/devel|origin/feature/stretch|origin/master|origin/stable|origin/testing)
break
true
;;
*)
# Collect necessary notification variable for the test and reproducible child job.
[ $NOTIFY_TEST_TO ] || NOTIFY_TEST_TO=""
echo "NOTIFY_TEST_TO=${NOTIFY_TEST_TO}" >> ${OUT_FILE}
[ $NOTIFY_BUILD_TO ] || NOTIFY_BUILD_TO=""
echo "NOTIFY_BUILD_TO=${NOTIFY_BUILD_TO}" >> ${OUT_FILE}
[ "$NOTIFY_TEST_TO" ] || NOTIFY_TEST_TO=""
echo "NOTIFY_TEST_TO=${NOTIFY_TEST_TO}" >> "${OUT_FILE}"
[ "$NOTIFY_BUILD_TO" ] || NOTIFY_BUILD_TO=""
echo "NOTIFY_BUILD_TO=${NOTIFY_BUILD_TO}" >> "${OUT_FILE}"
;;
esac
#!/bin/sh
# This script is called by the compare_artifacts jenkins-job-builder
# builder, defined in jenkins-jobs.git.
set -e
set -u
set -x
......@@ -34,7 +37,7 @@ dothediffo()
exit_code=$((exit_code + $?))
sudo chown -R jenkins:jenkins "${ARTIFACTS_DIR}"
if [ -r "${ARTIFACTS_DIR}/diffoscope.${EXT}.txt" ] \
&& [ $((3 * 1024)) -ge $(stat --format='%s' "${ARTIFACTS_DIR}/diffoscope.${EXT}.txt") ]; then
&& [ $((3 * 1024)) -ge "$(stat --format='%s' "${ARTIFACTS_DIR}/diffoscope.${EXT}.txt")" ]; then
cat "${ARTIFACTS_DIR}/diffoscope.${EXT}.txt"
fi
fi
......
#!/usr/bin/python3
# This script is called by the decide_if_reproduce jenkins-job-builder
# builder, defined in jenkins-jobs.git.
import os
from redmine import Redmine
import requests
API_URL = 'https://gitlab.tails.boum.org/api/v4/projects/tails%2Ftails/issues/{}'
def ticket_status(apikeyfile, ticket):
def needs_validation(apikeyfile, ticket):
with open(apikeyfile) as apikey_file:
apikey = apikey_file.read()
redmine = Redmine('https://redmine.tails.boum.org/code', key=apikey)
return str(redmine.issue.get(ticket).status)
apikey = apikey_file.read().rstrip()
headers = {'Private-Token': apikey}
json = requests.get(API_URL.format(ticket), headers=headers).json()
return 'Needs Validation' in json['labels']
BASE_BRANCHES_JOBS = ['build_Tails_ISO_stable',
......@@ -24,7 +31,7 @@ if __name__ == "__main__":
"--apikey_file",
default=None,
required=True,
help="Path to a file containing the Redmine API key.")
help="Path to a file containing the GitLab API key.")
parser.add_argument(
"--reproduce_file",
default=None,
......@@ -35,7 +42,7 @@ if __name__ == "__main__":
if os.environ['JOB_NAME'] in ALWAYS_REPRODUCE_JOBS or \
(int(os.environ['TAILS_TICKET']) != 0 and
ticket_status(
needs_validation(
apikeyfile=args.apikey_file,
ticket=os.environ['TAILS_TICKET']) == 'Needs Validation'):
ticket=os.environ['TAILS_TICKET'])):
open(args.reproduce_file, 'a').close()
#!/usr/bin/python3
# This script is called by the configure_notification_recipient
# jenkins-job-builder builder, defined in jenkins-jobs.git.
import os
import requests
import sys
from sh import git
from redmine import Redmine
API_URL = 'https://gitlab.tails.boum.org/api/v4/projects/tails%2Ftails/issues/{}'
def branch_last_author(git_repo_root, git_commit):
......@@ -27,11 +33,13 @@ def calculate_notification_vars(disabled_for_builds, disabled_for_tests, recipie
return build_recipient_var + "\n" + test_recipient_var
def ticket_status(apikeyfile, ticket):
def needs_validation(apikeyfile, ticket):
with open(apikeyfile) as apikey_file:
apikey = apikey_file.read()
redmine = Redmine('https://redmine.tails.boum.org/code', key=apikey)
return str(redmine.issue.get(ticket).status)
apikey = apikey_file.read().rstrip()
headers = {'Private-Token': apikey}
json = requests.get(API_URL.format(ticket), headers=headers).json()
return 'Needs Validation' in json['labels']
if __name__ == "__main__":
......@@ -40,7 +48,7 @@ if __name__ == "__main__":
parser.add_argument(
"--apikey_file",
default=None,
help="Path to a file containing the Redmine API key.")
help="Path to a file containing the GitLab API key.")
parser.add_argument(
"--no_notify_builds",
action='store_true',
......@@ -67,9 +75,9 @@ if __name__ == "__main__":
print("Err: you must specify apikey_file.")
sys.exit(2)
if int(os.environ['TAILS_TICKET']) != 0 and \
ticket_status(
needs_validation(
apikeyfile=args.apikey_file,
ticket=os.environ['TAILS_TICKET']) == 'Needs Validation':
ticket=os.environ['TAILS_TICKET']):
print(calculate_notification_vars(
disabled_for_builds=args.no_notify_builds,
disabled_for_tests=args.no_notify_tests,
......
#!/bin/sh
# This script is called by the sign_artifacts jenkins-job-builder builder,
# defined in jenkins-jobs.git.
set -e
set -u
set -x
......
......@@ -8,9 +8,11 @@
import argparse
import logging
import os
import shutil
import subprocess
import sys
from multiprocessing import Pool
from pathlib import Path
ISO_HISTORY_URL = 'https://iso-history.tails.boum.org'
......@@ -53,6 +55,9 @@ def main():
parser.add_argument("--create-iuk-extra-args", type=str,
action="store",
default="")
parser.add_argument("--jobs", type=int,
action="store",
default=1)
parser.add_argument("-v", "--verbose", action="store_true",
help="verbose output")
parser.add_argument("--debug", action="store_true", help="debug output")
......@@ -68,7 +73,8 @@ def main():
logging.basicConfig(level=logging.WARN, stream=sys.stderr,
format=LOG_FORMAT)
set_up_git_clone(remote=args.tails_git_remote, commit=args.tails_git_commit,
set_up_git_clone(remote=args.tails_git_remote,
commit=args.tails_git_commit,
use_existing_clone=args.use_existing_clone)
Path(args.output_dir).resolve().mkdir(parents=True, exist_ok=True)
get_iso(
......@@ -81,15 +87,24 @@ def main():
iso_history_url=args.iso_history_url,
local_isos_dir=args.local_isos_dir,
version=source_version)
create_iuk(
source_date_epoch=args.source_date_epoch,
source_version=source_version,
new_version=args.new_version,
output_dir=args.output_dir,
tmp_dir=args.tmp_dir,
local_isos_dir=args.local_isos_dir,
extra_args=args.create_iuk_extra_args
)
with Pool(processes=args.jobs) as pool:
log.info("Running up to %d jobs at once" % args.jobs)
lock_file = os.path.join(args.tmp_dir, 'mksquashfs-%d.lock' % os.getpid())
args.create_iuk_extra_args += ' --mksquashfs-lock-file=%s' % lock_file
for source_version in args.source_versions.split(" "):
pool.apply_async(create_iuk, (), {
'source_date_epoch': args.source_date_epoch,
'source_version': source_version,
'new_version': args.new_version,
'output_dir': args.output_dir,
'tmp_dir': args.tmp_dir,
'local_isos_dir': args.local_isos_dir,
'extra_args': args.create_iuk_extra_args,
'parallel': args.jobs > 1,
})
pool.close()
pool.join()
subprocess.run(["sudo", "rm", '-f', lock_file], check=True)
def set_up_git_clone(remote, commit, use_existing_clone):
......@@ -134,11 +149,26 @@ def get_iso(iso_history_url, local_isos_dir, version):
def create_iuk(source_date_epoch, source_version, new_version, local_isos_dir,
output_dir, tmp_dir, extra_args):
output_dir, tmp_dir, extra_args, parallel):
output_file = str(
Path(output_dir).resolve().joinpath(
"Tails_amd64_" + source_version + "_to_" + new_version + ".iuk"
))
new_iso_orig = find_iso(new_version, local_isos_dir)
if parallel:
new_iso_copy = str(
Path(tmp_dir).resolve().joinpath(
"Tails_amd64_" + new_version + "_from_" + source_version + ".iso"
))
log.info("Duplicating %s image for %s processing" %
(new_version, source_version))
shutil.copy(new_iso_orig, new_iso_copy)
else:
log.info("Using original %s image for %s processing" %
(new_version, source_version))
new_iso_copy = new_iso_orig
log.info("Creating %s" % (output_file))
cmd = "sudo" + \
" SOURCE_DATE_EPOCH='%s'" % source_date_epoch + \
......@@ -148,7 +178,7 @@ def create_iuk(source_date_epoch, source_version, new_version, local_isos_dir,
" ./tails/config/chroot_local-includes/usr/src/iuk/bin/tails-create-iuk" + \
" --squashfs-diff-name '" + new_version + ".squashfs'" + \
" --old-iso '" + find_iso(source_version, local_isos_dir) + "'" + \
" --new-iso '" + find_iso(new_version, local_isos_dir) + "'" \
" --new-iso '" + new_iso_copy + "'" \
" --outfile '" + output_file + "'"
if extra_args:
cmd = cmd + " " + extra_args
......@@ -160,6 +190,10 @@ def create_iuk(source_date_epoch, source_version, new_version, local_isos_dir,
output_file
], check=True)
if parallel:
log.info("Removing copy of %s image" % new_version)
os.unlink(new_iso_copy)
if __name__ == "__main__":
try:
......
#!/bin/sh
# This script is called by the post_test_cleanup jenkins-job-builder builder
# defined in jenkins-jobs.git.
set -e
set -u
set -x
......@@ -7,31 +10,24 @@ set -x
WORKSPACE="$1"
TEMP_DIR="$2"
[ ! -z "$WORKSPACE" ] || exit 2
[ ! -z "$TEMP_DIR" ] || exit 3
[ -n "$WORKSPACE" ] || exit 2
[ -n "$TEMP_DIR" ] || exit 3
[ -d "$WORKSPACE" ] || exit 4
[ -d "$TEMP_DIR" ] || exit 5
sudo -n chown -R jenkins "${TEMP_DIR}"/*
sudo -n chown -R jenkins "${TEMP_DIR}"
mkdir -p "${WORKSPACE}/build-artifacts"
# Move sikuli_candidates first and separately to avoid the pictures
# in there to be caught by "-name '*.png'" in the next command
# and thus copied to the root of build-artifacts/
find "${TEMP_DIR}" \
\( -type d -a -name sikuli_candidates \) -print0 | \
xargs --null --no-run-if-empty \
mv --target-directory="${WORKSPACE}/build-artifacts/"
find "${TEMP_DIR}" \
\( -name '*.png' -o -name '*.mkv' -o -name '*.pcap' \
-o -name debug.log -o -name cucumber.json \
-o -name '*_chutney-data' -o -name '*.htpdate' \
-o -name '*.journal' -o -name '*.tor' \
\( -name '*.png' -o -name '*.mkv' -o -name '*.pcap' \
-o -name debug.log -o -name cucumber.json \
-o -name '*_chutney-data' -o -name '*.htpdate' \
-o -name '*.journal' -o -name '*.tor' \
-o -name '*.cmd_output_*' -o -name '*.file_content_*' \
\) \
-print0 | \
xargs --null --no-run-if-empty \
cp -r --target-directory="${WORKSPACE}/build-artifacts/"
mv --target-directory="${WORKSPACE}/build-artifacts/"
sudo -n rm -rf "${TEMP_DIR}"/*
#!/bin/sh
# This script is called from by test_Tails_ISO_* jobs,
# defined in jenkins-jobs.git.
set -e
set -u
set -x
......@@ -10,7 +13,7 @@ ISO_HISTORY_URL="$1"
shift
TMP_DIR="$1"
shift
: ${USE_LAST_RELEASE_AS_OLD_ISO:=yes}
: "${USE_LAST_RELEASE_AS_OLD_ISO:=yes}"
### Constants
......@@ -19,20 +22,21 @@ FIRST_BUSTER_SPECIFIC_COMMIT=2f106487f6a118e76cf0d7d47ba260e087a64ab2
### Sanity checks
[ $WORKSPACE ] || exit 2
[ $ISO_HISTORY_URL ] || exit 3
[ $TMP_DIR ] || exit 4
[ $UPSTREAMJOB_GIT_COMMIT ] || exit 5
[ $UPSTREAMJOB_GIT_BASE_BRANCH_HEAD ] || exit 6
[ $UPSTREAMJOB_BUILD_NUMBER ] || exit 7
[ $JENKINS_URL ] || exit 8
[ $JOB_NAME ] || exit 9
[ $BUILD_NUMBER ] || exit 10
[ "$WORKSPACE" ] || exit 2
[ "$ISO_HISTORY_URL" ] || exit 3
[ "$TMP_DIR" ] || exit 4
[ "$UPSTREAMJOB_GIT_COMMIT" ] || exit 5
# shellcheck disable=SC2153
[ "$UPSTREAMJOB_GIT_BASE_BRANCH_HEAD" ] || exit 6
[ "$UPSTREAMJOB_BUILD_NUMBER" ] || exit 7
[ "$JENKINS_URL" ] || exit 8
[ "$JOB_NAME" ] || exit 9
[ "$BUILD_NUMBER" ] || exit 10
[ "$USE_LAST_RELEASE_AS_OLD_ISO" = yes ] \
|| [ "$USE_LAST_RELEASE_AS_OLD_ISO" = no ] \
|| exit 13
[ -d $TMP_DIR ] || exit 11
[ -d "$TMP_DIR" ] || exit 11
if [ -f "$ALREADY_RUN_FLAG_FILE" ]; then
echo "A test suite job has already been run on this ISO tester." >&2
......@@ -56,7 +60,7 @@ only_changes_doc() {
local upstreamjob_git_base_branch_head="$1"
local base_branch_non_doc_diff="$(git diff \
$upstreamjob_git_base_branch_head... \
"${upstreamjob_git_base_branch_head}..." \
-- \
'*' \
':!/wiki' \
......@@ -134,4 +138,4 @@ as_root_do ./run_test_suite \
--capture \
-- \
$TAGS_ARGS \
${@}
"${@}"
......@@ -37,10 +37,3 @@ object ServiceGroup "memory" {
assign where match("mem", service.check_command)
}
object ServiceGroup "redmine" {
display_name = "Redmine Checks"
assign where match("buse.tails.boum.org", return_servicegroup_host(service))
assign where match("redmine.tails.boum.org", service.vars.http_vhost)
}
#!/bin/sh
TMPDIR=$(mktemp -d)
set -e
set -u
git clone /srv/repositories/mirrors.git $TMPDIR
cd $TMPDIR
TMPDIR=$(mktemp -d)
if [ -f ./subdomains.txt ]; then
echo "updating subdomains"
/usr/local/bin/update-subdomains.py
fi
git clone /srv/repositories/mirrors.git "$TMPDIR"
cd "$TMPDIR"
<