Commit 9080b9e8 authored by intrigeri's avatar intrigeri
Browse files

Merge remote-tracking branch 'origin/devel' into feature/stretch

parents b887c45f b5171a7a
......@@ -3,6 +3,7 @@
*.po~
*.pot~
*.swp
/*.apt-sources
/*.build-manifest
/*.buildlog
/*.img
......
......@@ -77,7 +77,15 @@ chmod -R go+rX config/chroot_sources
# build the image
# we need /debootstrap/deburis to build a manifest of used packages:
export DEBOOTSTRAP_OPTIONS='--keep-debootstrap-dir'
DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keep-debootstrap-dir"
# use our own APT repository's key:
DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
gpg --homedir "$DEBOOTSTRAP_GNUPG_HOMEDIR" \
--import config/chroot_sources/tails.chroot.gpg
DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keyring=$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.gpg"
export DEBOOTSTRAP_OPTIONS
: ${MKSQUASHFS_OPTIONS:='-comp xz -Xbcj x86 -b 1024K -Xdict-size 1024K'}
MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS} -wildcards -ef chroot/usr/share/amnesia/build/mksquashfs-excludes"
......@@ -165,7 +173,8 @@ case "$LB_BINARY_IMAGES" in
;;
esac
BUILD_DEST_FILENAME="${BUILD_BASENAME}.${BUILD_FILENAME_EXT}"
BUILD_MANIFEST="${BUILD_DEST_FILENAME}.list"
BUILD_MANIFEST="${BUILD_DEST_FILENAME}.build-manifest"
BUILD_APT_SOURCES="${BUILD_DEST_FILENAME}.apt-sources"
BUILD_PACKAGES="${BUILD_DEST_FILENAME}.packages"
BUILD_LOG="${BUILD_DEST_FILENAME}.buildlog"
BUILD_START_FILENAME="${BUILD_DEST_FILENAME}.start.timestamp"
......@@ -177,6 +186,15 @@ trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
exec 2> >(tee -a "$BUILD_LOG" >&2)
trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
(
echo "Mirrors:"
apt-mirror debian
apt-mirror debian-security
apt-mirror torproject
echo "Additional sources:"
cat config/chroot_sources/*.chroot
) > "$BUILD_APT_SOURCES"
echo "Building $LB_BINARY_IMAGES image ${BUILD_BASENAME}..."
set -o pipefail
[ -z "$JENKINS_URL" ] || date --utc '+%s' > "$BUILD_START_FILENAME"
......@@ -199,7 +217,7 @@ if [ -e "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" ]; then
echo "Renaming generated files..."
mv -i "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" "${BUILD_DEST_FILENAME}"
mv -i binary.packages "${BUILD_PACKAGES}"
generate-build-manifest chroot/debootstrap "${BUILD_DEST_FILENAME}.build-manifest"
generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
else
fatal "lb build failed ($?)."
fi
......@@ -16,12 +16,39 @@ fi
export LB_BOOTSTRAP_INCLUDE='eatmydata'
# sanity checks
if grep -qs -E '^Pin:\s+release\s+.*a=' config/chroot_apt/preferences ; then
echo "Found unsupported a= syntax in config/chroot_apt/preferences,"
echo "use n= instead. Exiting."
exit 1
fi
if grep -qs -E '^Pin:\s+release\s+.*o=Debian Backports' \
config/chroot_apt/preferences ; then
echo "Found unsupported 'o=Debian Backports' syntax,"
echo "in config/chroot_apt/preferences. Use o=Debian instead. Exiting."
exit 1
fi
# init variables
RUN_LB_CONFIG="lb config noauto"
# init config/ with defaults for the target distribution
$RUN_LB_CONFIG --distribution stretch ${@}
# set up everything for time-based snapshots:
apt-snapshots-serials prepare-build
DEBIAN_MIRROR="$(apt-mirror debian)"
DEBIAN_SECURITY_MIRROR="$(apt-mirror debian-security)"
TORPROJECT_MIRROR="$(apt-mirror torproject)"
perl -pi \
-E \
"s|^(deb(?:-src)?\s+)https?://ftp[.]us[.]debian[.]org/debian/?(\s+)|\$1$DEBIAN_MIRROR\$2| ; \
s|^(deb(?:-src)?\s+)https?://deb[.]torproject[.]org/torproject[.]org/?(\s+)|\$1$TORPROJECT_MIRROR\$2|" \
config/chroot_sources/*.chroot \
|| exit 1
# set Amnesia's general options
$RUN_LB_CONFIG \
--verbose \
......@@ -39,9 +66,11 @@ $RUN_LB_CONFIG \
--iso-publisher="https://tails.boum.org/" \
--iso-volume="TAILS ${AMNESIA_FULL_VERSION}" \
--memtest none \
--mirror-binary "http://ftp.us.debian.org/debian/" \
--mirror-bootstrap "http://ftp.us.debian.org/debian/" \
--mirror-chroot "http://ftp.us.debian.org/debian/" \
--mirror-binary "$DEBIAN_MIRROR" \
--mirror-bootstrap "$DEBIAN_MIRROR" \
--mirror-chroot "$DEBIAN_MIRROR" \
--mirror-binary-security "$DEBIAN_SECURITY_MIRROR" \
--mirror-chroot-security "$DEBIAN_SECURITY_MIRROR" \
--packages-lists="standard" \
--tasks="standard" \
--linux-packages="linux-image-3.16.0-4" \
......
#!/bin/bash
set -e
set -u
. "$(dirname $0)/utils.sh"
ARCHIVE="$1"
output_tagged_snapshot() {
local archive="$1"
local tag="$2"
echo "http://tagged.snapshots.deb.tails.boum.org/$tag/$archive"
}
output_time_based_snapshot() {
local archive="$1"
local serial="$2"
echo "http://time-based.snapshots.deb.tails.boum.org/$archive/$serial"
}
### Sanity checks
[ -n "$ARCHIVE" ] || exit 1
### Main
SERIAL=$(cat "config/APT_snapshots.d/$ARCHIVE/serial")
RESOLVED_SERIAL=$(cat "tmp/APT_snapshots.d/$ARCHIVE/serial")
if [ "$(base_branch)" = stable ]; then
if version_was_released "$(version_in_changelog)"; then
[ "$(current_branch)" = stable ] \
|| fatal "Not building from stable, but last version in changelog" \
"was released"
output_tagged_snapshot "$ARCHIVE" "$(version_in_changelog)"
else
version_was_released "$(previous_version_in_changelog)" \
|| fatal "None of the two last version in changelog were released"
case "$ARCHIVE" in
debian-security)
[ "$SERIAL" = latest ] \
|| fatal "APT snapshots are frozen for debian-security, which" \
"should not happen on a branch based on stable"
output_time_based_snapshot "$ARCHIVE" "$RESOLVED_SERIAL"
;;
*)
if [ "$SERIAL" = latest ]; then
# In this case, "latest" means "do what I mean", that is stick
# to previous release's tagged snapshot
output_tagged_snapshot "$ARCHIVE" "$(previous_version_in_changelog)"
else
output_time_based_snapshot "$ARCHIVE" "$SERIAL"
fi
esac
fi
elif [ "$(base_branch)" = testing ]; then
if version_was_released "$(version_in_changelog)"; then
[ "$(current_branch)" = testing ] \
|| fatal "Not building from testing, but last version in changelog" \
"was released"
[ "$ARCHIVE" = debian-security ] || [ "$SERIAL" != latest ] \
|| fatal "APT snapshots for $ARCHIVE are not frozen, which should" \
"not happen on a tagged testing branch"
output_tagged_snapshot "$ARCHIVE" "$(version_in_changelog)"
else
output_time_based_snapshot "$ARCHIVE" "$RESOLVED_SERIAL"
fi
else
if [ "$(base_branch)" = devel ] && [ "$SERIAL" != latest ]; then
fatal "APT snapshots are frozen, which should not happen on a branch" \
"based on the devel one"
fi
output_time_based_snapshot "$ARCHIVE" "$RESOLVED_SERIAL"
fi
#!/bin/bash
set -e
set -u
set -o pipefail
BASE_URL=http://time-based.snapshots.deb.tails.boum.org/
CONFIG=config/APT_snapshots.d
ORIGINS=$(cd $CONFIG; ls -d *)
get_latest_serial() {
origin=$1
wget -q $BASE_URL/$origin/project/trace/$origin -O - \
| awk -F': ' '/^Archive serial: / {print $2}'
}
action="$1"
case "$action" in
get-latest)
for origin in $ORIGINS; do
online=$(get_latest_serial $origin)
echo "$origin: $online"
done
;;
freeze)
for origin in $ORIGINS; do
serial_file="$CONFIG/$origin/serial"
git=$(cat $serial_file)
case "$origin" in
debian-security)
new=latest
;;
*)
new=$(get_latest_serial $origin)
esac
printf "Origin $origin:\n old: $git\n new: $new\n"
echo $new > $serial_file
done
printf "\nAll files ($CONFIG/*/serial) have been updated with new serials\n" >&2
;;
thaw)
for origin in $ORIGINS; do
serial_file="$CONFIG/$origin/serial"
git=$(cat $serial_file)
printf "Origin $origin:\n old: $git\n new: latest\n"
echo 'latest' > $serial_file
done
;;
prepare-build)
rm -rf tmp/APT_snapshots.d
mkdir -p tmp
cp -r config/APT_snapshots.d tmp/
$0 get-latest > tmp/cached_APT_snapshots_serials
for origin_dir in tmp/APT_snapshots.d/*; do
origin=$(basename $origin_dir)
if grep -qs '^latest$' $origin_dir/serial; then
awk -F': ' "/^$origin: / {print \$2}" \
tmp/cached_APT_snapshots_serials \
> $origin_dir/serial
fi
done
;;
*)
printf "unknown action ($action), use either 'get-latest', 'prepare-build', 'freeze' or 'thaw'\n" >&2
exit 1
;;
esac
......@@ -33,6 +33,12 @@ if (! -d $debootstrap) {
usage;
}
if (-f "$debootstrap/unknown") {
print "E: actions unsupported by the apt-get wrapper were logged ",
"in $debootstrap/unknown. Aborting.";
exit 1;
}
my $extra_packages_file = 'config/build-manifest-extra-packages.yml';
my $extra_packages;
......@@ -82,11 +88,12 @@ foreach my $type (keys %package_type) {
}
}
### Extract list of (origin, reference) from the build configuration:
### Extract list of (origin, reference) from the build configuration
### (the resolved serials, stored under tmp by "apt-snapshots-serials prepare-build"):
my %origin_reference;
while (my $origin_dir = glob('config/APT_snapshots.d/*')) {
while (my $origin_dir = glob('tmp/APT_snapshots.d/*')) {
my $origin_name = $origin_dir;
$origin_name =~ s{\A config/APT_snapshots[.]d/}{}xms;
$origin_name =~ s{\A tmp/APT_snapshots[.]d/}{}xms;
$origin_reference{$origin_name} = read_file("$origin_dir/serial");
chomp $origin_reference{$origin_name};
$data->{origin_references}->{ $origin_name }->{reference} = $origin_reference{ $origin_name } || 'unknown';
......
#!/bin/bash
set -e
set -u
. "$(dirname $0)/utils.sh"
APT_MIRROR_URL="http://deb.tails.boum.org/"
DEFAULT_COMPONENTS="main"
BASE_BRANCHES="stable testing devel feature/jessie"
fatal() {
echo "$*" >&2
exit 1
}
git_tag_exists() {
local tag="$1"
test -n "$(git tag -l "$tag")"
}
version_was_released() {
local version="$1"
version="$(echo "$version" | tr '~' '-')"
git_tag_exists "$version"
}
version_in_changelog() {
dpkg-parsechangelog | awk '/^Version: / { print $2 }'
}
output_apt_binary_source() {
local suite="$1"
local components="${2:-$DEFAULT_COMPONENTS}"
......@@ -41,10 +22,6 @@ output_overlay_apt_binary_sources() {
done
}
current_branch() {
git branch | awk '/^\* / { print $2 }'
}
on_base_branch() {
local current_branch=$(current_branch)
......@@ -57,10 +34,6 @@ on_base_branch() {
return 1
}
base_branch() {
cat config/base_branch | head -n1
}
branch_name_to_suite() {
local branch="$1"
......
#!/bin/bash
set -e
set -u
set -o pipefail
list_origins () {
(
cd config/APT_snapshots.d/
ls --color=never -1 | grep -v --line-regexp '\.placeholder'
)
}
print_tagged_snapshots_pool_url () {
origin="$1"
version="$2"
printf \
'http://tagged.snapshots.deb.tails.boum.org/%s/%s/pool/\n' \
"$version" "$origin"
}
conf=/etc/apt-cacher-ng/tails-snapshots.conf
for origin in $(list_origins) ; do
[ "$origin" != .placeholder ] || continue
origin_without_dashes=$(echo "$origin" | sed -e 's,-,,g')
echo "Remap-tailssnapshots${origin_without_dashes}pool: file:tails-time-based-snapshots-$origin-pool.list file:tails-tagged-snapshots-$origin-pool.list"
done > "$conf"
chmod 644 "$conf"
# Generate .list files for time-based snapshots
for origin in $(list_origins) ; do
list="/etc/apt-cacher-ng/tails-time-based-snapshots-$origin-pool.list"
current_year=$(date '+%Y')
for year in $(seq $(($current_year - 1)) $(($current_year + 1))) ; do
for month in $(seq 1 12); do
# We need the config file to contain _at least_ everything
# that can possibly exists, and we don't care if it has some extra
# lines, so to simplify we do as if each month had 31 days.
for day in $(seq 1 31) ; do
for n in $(seq 1 4) ; do
printf 'http://time-based.snapshots.deb.tails.boum.org/%s/%04u%02u%02u%02u/pool/\n' \
"$origin" "$year" "$month" "$day" "$n"
done
done
done
done \
> "$list"
chmod 644 "$list"
done
# Generate .list files for tagged snapshots
for origin in $(list_origins) ; do
list="/etc/apt-cacher-ng/tails-tagged-snapshots-$origin-pool.list"
# We need the config file to contain _at least_ everything
# that can possibly exists, and we don't care if it has some extra
# lines, so here we try to build the smallest possible superset of
# all realistic Tails version numbers; it could certainly be a tiny
# bit smaller, at the cost of more assumptions (=> more risk of not
# including some version number we'll end up using) or of more
# code complexity (=> higher maintenance cost).
#
# XXX: Stretch: bump the end of the range of major versions
for major in $(seq 2 3) ; do
for minor in $(seq 0 32); do
for suffix in "" alpha beta rc ; do
for suffix_n in "" $(seq 1 8); do
if [ -z "$suffix" ]; then
version="${major}.${minor}"
elif [ -z "$suffix_n" ]; then
version="${major}.${minor}-${suffix}"
else
version="${major}.${minor}-${suffix}${suffix_n}"
fi
print_tagged_snapshots_pool_url "$origin" "$version"
done
done
for emergency in $(seq 1 4) ; do
version="${major}.${minor}.${emergency}"
print_tagged_snapshots_pool_url "$origin" "$version"
done
done
done > "$list"
chmod 644 "$list"
done
# This library is meant to be used in bash, with "set -e" and "set -u".
current_branch() {
git branch | awk '/^\* / { print $2 }'
}
base_branch() {
cat config/base_branch | head -n1
}
fatal() {
echo "$*" >&2
exit 1
}
git_tag_exists() {
local tag="$1"
test -n "$(git tag -l "$tag")"
}
version_was_released() {
local version="$1"
version="$(echo "$version" | tr '~' '-')"
git_tag_exists "$version"
}
version_in_changelog() {
dpkg-parsechangelog | awk '/^Version: / { print $2 }'
}
previous_version_in_changelog() {
dpkg-parsechangelog --offset 1 --count 1 | awk '/^Version: / { print $2 }'
}
# for each upstream APT repository:
# tell time-based snapshots infra to keep last snapshot
# -> returns us the corresponding serial
# write serial of the last snapshot > config/APT_snapshots.d/$origin/serial
#!/bin/sh
set -e
set -u
TIME_BASED_SNAPSHOTS_HOST='apt.lizard'
TIME_BASED_SNAPSHOTS_USER='reprepro-time-based-snapshots'
TIME_BASED_SNAPSHOTS_USER_AT_HOST="${TIME_BASED_SNAPSHOTS_USER}@${TIME_BASED_SNAPSHOTS_HOST}"
fail_with_usage() {
echo "$(basename $0) BUILD_MANIFEST TAG" >&2
}
[ $# -eq 2 ] || fail_with_usage
BUILD_MANIFEST="$1"
TAG="$2"
[ -r "$BUILD_MANIFEST" ] || fail_with_usage
[ -n "$TAG" ] || fail_with_usage
echo "I: Preparing a workspace on ${TIME_BASED_SNAPSHOTS_HOST}"
ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" install -d '$HOME'/tmp
REMOTE_BUILD_MANIFEST=$(ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" \
mktemp --tmpdir='$HOME'/tmp)
REMOTE_DEST_DIR=$(ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" \
mktemp -d --tmpdir='$HOME'/tmp)
echo "I: Sending build manifest to ${TIME_BASED_SNAPSHOTS_HOST}"
scp "$BUILD_MANIFEST" \
"${TIME_BASED_SNAPSHOTS_USER_AT_HOST}:${REMOTE_BUILD_MANIFEST}"
echo "I: Preparing reprepro configuration"
ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" \
tails-prepare-tagged-apt-snapshot-import \
"$REMOTE_BUILD_MANIFEST" \
"$REMOTE_DEST_DIR"
echo "I: Pulling packages from the time-based snapshots into the tagged ones"
for archive in $(ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" \
ls "$REMOTE_DEST_DIR") ; do
echo "I: - $archive"
ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" \
reprepro --basedir "${REMOTE_DEST_DIR}/${archive}" update
done
echo "I: Publishing the tagged APT snapshot"
ssh "${TIME_BASED_SNAPSHOTS_USER}@${TIME_BASED_SNAPSHOTS_HOST}" \
sudo -n /usr/local/sbin/tails-publish-tagged-apt-snapshot \
"$REMOTE_DEST_DIR" "$TAG"
echo "I: Cleaning up"
ssh "${TIME_BASED_SNAPSHOTS_USER}@${TIME_BASED_SNAPSHOTS_HOST}" \
rm "$REMOTE_BUILD_MANIFEST"
......@@ -43,6 +43,10 @@ cp "$CHROOT_SYSLINUX_MBR" "$BINARY_MBR_DIR/mbr.bin"
cat chroot/etc/apt/sources.list chroot/etc/apt/sources.list.d/*.list \
| grep --extended-regexp --line-regexp --invert-match \
'deb\s+file:/root/local-packages\s+\./' \
| grep --extended-regexp --invert-match \
'^deb\s+http://tagged\.snapshots\.deb\.tails\.boum.org/[^/]+/torproject/' \
| grep --extended-regexp --invert-match \
'^deb\s+http://time-based\.snapshots\.deb\.tails\.boum.org/torproject/' \
| sed --regexp-extended -e 's,^deb(\s+),deb-src\1,' \
> "$CHROOT_TEMP_APT_SOURCES"
Chroot chroot apt-get --yes update
......
Package: b43-fwcutter
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-amd-graphics
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-atheros
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-b43-installer
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-b43legacy-installer
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-brcm80211
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-ipw2x00
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-iwlwifi
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-libertas
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-linux
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-linux-free
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-linux-nonfree
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-misc-nonfree
Pin: release o=Debian,a=unstable
Pin: release o=Debian,n=sid
Pin-Priority: 999