Merge branch 'feature/15281-single-squashfs-diff' into stable (Closes: #15281,...

Merge branch 'feature/15281-single-squashfs-diff' into stable (Closes: #15281, #15279, #15283, #15286)

auto/build

@@ -118,7 +118,7 @@ export MKSQUASHFS_OPTIONS
 ./refresh-translations || fatal "refresh-translations failed ($?)."
 
 # generate list of supported languages
-./generate-languages-list || fatal "generate-languages-list failed ($?)."
+generate-languages-list || fatal "generate-languages-list failed ($?)."
 
 BUILD_ISO_FILENAME="${BUILD_BASENAME}.iso"
 BUILD_MANIFEST="${BUILD_BASENAME}.build-manifest"

bin/copy-iuks-to-rsync-server-and-verify

+#!/usr/bin/python3
+
+import argparse
+import logging
+import subprocess
+import sys
+
+from typing import List
+from pathlib import Path
+
+JENKINS_IUKS_BASE_URL = "https://nightly.tails.boum.org/build_IUKs/builds"
+RSYNC_SERVER_HOSTNAME = "rsync.lizard"
+LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
+log = logging.getLogger()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Copy IUKs from Jenkins to our rsync server \
+        and verify that they match those built locally"
+    )
+    parser.add_argument("--hashes-file", type=str, action="store")
+    parser.add_argument("--jenkins-build-id", type=int, action="store")
+    parser.add_argument("-q", "--quiet", action="store_true",
+                        help="quiet output")
+    parser.add_argument("--debug", action="store_true", help="debug output")
+    parser.add_argument("--skip-sending-hashes-file", action="store_true",
+                        help="Assume the hashes file was uploaded already")
+    parser.add_argument("--skip-downloading-iuks", action="store_true",
+                        help="Assume the IUKs were already downloaded")
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG, stream=sys.stderr,
+                            format=LOG_FORMAT)
+    elif args.quiet:
+        logging.basicConfig(level=logging.WARN, stream=sys.stderr,
+                            format=LOG_FORMAT)
+    else:
+        logging.basicConfig(level=logging.INFO, stream=sys.stderr,
+                            format=LOG_FORMAT)
+
+    if args.hashes_file is None:
+        log.error("Please pass --hashes-file")
+        sys.exit(1)
+
+    if args.jenkins_build_id is None:
+        log.error("Please pass --jenkins-build-id")
+        sys.exit(1)
+
+    if not Path(args.hashes_file).exists():
+        log.error("%s does not exist" % (args.hashes_file))
+        sys.exit(1)
+
+    if not args.skip_sending_hashes_file:
+        send_hashes_file(
+            hashes_file=args.hashes_file,
+            desthost=RSYNC_SERVER_HOSTNAME,
+        )
+
+    if not args.skip_downloading_iuks:
+        download_iuks_from_jenkins(
+            hashes_file=args.hashes_file,
+            desthost=RSYNC_SERVER_HOSTNAME,
+            jenkins_iuks_base_url=JENKINS_IUKS_BASE_URL,
+            jenkins_build_id=args.jenkins_build_id,
+        )
+
+    verify_iuks(
+        desthost=RSYNC_SERVER_HOSTNAME,
+        hashes_file=Path(args.hashes_file).name,
+    )
+
+
+def send_hashes_file(
+        hashes_file: str,
+        desthost: str) -> None:
+    log.info("Sending %(f)s to %(h)s…" % {
+        "f": hashes_file,
+        "h": desthost,
+    })
+    subprocess.run(
+        ["scp", hashes_file, "%s:" % (desthost)],
+        check=True
+    )
+
+
+def iuks_listed_in(hashes_file: str) -> List[str]:
+    with Path(hashes_file).open() as f:
+        lines = f.readlines()
+    return [l.split('  ')[-1].rstrip() for l in lines]
+
+
+def download_iuks_from_jenkins(
+        hashes_file: str,
+        desthost: str,
+        jenkins_iuks_base_url: str,
+        jenkins_build_id: int) -> None:
+    log.info("Downloading IUKs from Jenkins to %s…" % (desthost))
+    iuks = iuks_listed_in(hashes_file)
+    log.debug("IUKS: %s" % ', '.join(iuks))
+    for iuk in iuks:
+        log.debug("Downloading %s" % (iuk))
+        url = "%s/%s/archive/%s" % (
+            jenkins_iuks_base_url,
+            jenkins_build_id,
+            iuk
+        )
+        subprocess.run(
+            ["ssh", desthost, "wget", "--quiet", "--no-clobber",
+             "-O", iuk, url],
+            check=True
+        )
+
+
+def verify_iuks(desthost: str, hashes_file: str) -> None:
+    log.info("Verifying that IUKs built on Jenkins match those you've built…")
+    try:
+        subprocess.run(
+            ["ssh", desthost, "sha256sum", "--check", "--strict",
+             Path(hashes_file).name],
+            check=True
+        )
+    except subprocess.CalledProcessError:
+        print("\nERROR: IUKs built on Jenkins don't match yours\n",
+              file=sys.stderr)
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except Exception as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)

bin/create-test-iuks

+#!/bin/sh
+
+set -e
+set -u
+set -x
+
+VERSIONS="2.0~test 2.2~test 2.3~test"
+export SOURCE_DATE_EPOCH=$(date --utc '+%s')
+
+[ -d "$TAILS_CHECKOUT" ] || exit 2
+
+WORKDIR=$(mktemp -d)
+
+for version in $VERSIONS; do
+   ISO_SRC="$WORKDIR/$version"/iso_src
+   for dir in EFI/BOOT isolinux live utils/linux utils/mbr; do
+      mkdir -p "$ISO_SRC/$dir"
+   done
+   SQUASHFS_SRC="$WORKDIR/$version"/squashfs_src
+   mkdir -p "$SQUASHFS_SRC"
+   mkdir -p "$SQUASHFS_SRC"/etc/amnesia "$SQUASHFS_SRC"/usr/share
+
+   cp -a /usr/share/common-licenses "$SQUASHFS_SRC"/usr/share/
+   if [ "$version" != '2.0~test' ]; then
+      echo "Some content" > "$SQUASHFS_SRC"/some_new_file
+      rm "$SQUASHFS_SRC"/usr/share/common-licenses/BSD
+   fi
+   if [ "$version" = '2.3~test' ]; then
+      echo "Some content 2.3" > "$SQUASHFS_SRC"/some_new_file_2.3
+      rm "$SQUASHFS_SRC"/usr/share/common-licenses/MPL-1.1
+   fi
+   cat > "$SQUASHFS_SRC"/etc/amnesia/version <<EOF
+$version - 20380119
+ffffffffffffffffffffffffffffffffffffffff
+live-build: 3.0.5+really+is+2.0.12-0.tails2
+live-boot: 4.0.2-1
+live-config: 4.0.4-1
+EOF
+   cat > "$SQUASHFS_SRC"/etc/os-release <<EOF
+TAILS_PRODUCT_NAME="Tails"
+TAILS_VERSION_ID="$version"
+EOF
+   mksquashfs \
+      "$SQUASHFS_SRC" \
+      "$ISO_SRC"/live/filesystem.squashfs \
+      -no-progress -noappend -comp xz -Xbcj x86 -b 1024K -Xdict-size 1024K
+
+   echo vmlinuz > "$ISO_SRC"/live/vmlinuz
+   echo initrd > "$ISO_SRC"/live/initrd.img
+   echo isolinux > "$ISO_SRC"/isolinux/isolinux.cfg
+   echo 'filesystem.squashfs' > "$ISO_SRC"/live/Tails.module
+   cp /usr/lib/syslinux/mbr/gptmbr.bin "$ISO_SRC"/utils/mbr/mbr.bin
+   if [ "$version" = '2.0~test' ]; then
+      cp /usr/bin/syslinux "$ISO_SRC"/utils/linux
+   fi
+   if [ "$version" = '2.3~test' ]; then
+      rm "$ISO_SRC"/utils/mbr/mbr.bin
+   fi
+   xorriso \
+      -as mkisofs -R -r -J -joliet-long -l -cache-inodes -iso-level 3 \
+      --modification-date=2019112316114600 \
+      -o "$WORKDIR/$version.iso" "$ISO_SRC"
+done
+
+for dest_version in 2.2~test 2.3~test; do
+   echo "Generating IUK file from 2.0~test to $dest_version"
+   sudo su -c \
+         "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \
+          LC_ALL=C \
+          TMPDIR=\"${TMPDIR:-/tmp}\" \
+          PERL5LIB=\"${TAILS_CHECKOUT:?}/config/chroot_local-includes/usr/src/perl5lib/lib\" \
+          ${TAILS_CHECKOUT:?}/config/chroot_local-includes/usr/src/iuk/bin/tails-create-iuk \
+               --squashfs_diff_name \"${dest_version}.squashfs\"           \
+               --old_iso \"$WORKDIR/2.0~test.iso\" \
+               --new_iso \"$WORKDIR/${dest_version}.iso\" \
+               --outfile \"$WORKDIR/Tails_amd64_2.0~test_to_${dest_version}.iuk\""
+done
+
+echo "Generated test IUKS:"
+ls -lh "$WORKDIR"/*.iuk

bin/iuk-source-versions

+#!/bin/sh
+
+set -e
+set -u
+
+major_version () {
+   local version="$1"
+   echo "$version" | perl -p -E 's,[.].*,,'
+}
+
+RELEASING_VERSION="$1"
+RELEASING_MAJOR_VERSION=$(major_version "$RELEASING_VERSION")
+
+git tag --color=never | while read tag ; do
+   version=$(echo "$tag" | perl -p -E 's,-,~,')
+   major_version=$(major_version "$version")
+   if [ "$major_version" = "$RELEASING_MAJOR_VERSION" ] && \
+         dpkg --compare-versions "$version" lt "$RELEASING_VERSION" ; then
+      echo "$version"
+   fi
+done
+

config/chroot_local-hooks/06-adduser_tails-upgrade-frontend

@@ -11,4 +11,6 @@ set -e
 echo "Creating the tails-upgrade-frontend user"
 
 addgroup --system --quiet --gid 126 tails-upgrade-frontend
-adduser --system --quiet --uid 118 --gid 126 --no-create-home tails-upgrade-frontend
+adduser --system --quiet --uid 118 --gid 126 \
+        --home /var/lib/tails-upgrade-frontend \
+        tails-upgrade-frontend

config/chroot_local-hooks/08-install-Perl-programs

+#!/bin/sh
+
+set -e
+set -u
+
+echo "Installing Perl programs"
+
+# Import ensure_hook_dependency_is_installed()
+. /usr/local/lib/tails-shell-library/build.sh
+
+ensure_hook_dependency_is_installed \
+   cpanminus \
+   libdist-zilla-perl \
+   libdist-zilla-plugin-test-notabs-perl \
+   libdist-zilla-plugin-test-perl-critic-perl
+
+for dist in perl5lib iuk; do
+   dist_dir="/usr/src/${dist}"
+   cd "$dist_dir"
+   PERL5LIB=/usr/src/perl5lib/lib PERL_CPANM_OPT=--notest dzil install
+   cd
+   rm -r "$dist_dir"
+done
+rm -r /root/.cpanm
+
+# Satisfy the dependency of the tails-persistence-setup package
+# on tails-perl5lib
+install_fake_package tails-perl5lib 4.0
+
+apt-get install --yes tails-persistence-setup
+
+for patch in /usr/share/tails/build/run_t-p-s_as_its_dedicated_user.diff ; do
+   (cd / && patch --forward --batch -p1 < "$patch")
+   rm "$patch"
+done

config/chroot_local-hooks/99-set_mtimes

@@ -9,3 +9,6 @@ touch --no-create -t 197001010000 \
    /usr/share/ppd/hplip/HP/*.ppd  \
    /var/lib/anthy/anthy.dic       \
    /var/lib/anthy/mkworddic/anthy.wdic
+
+find /usr/share/doc/tails/website -depth -exec \
+   touch --no-create -t 197001010000 '{}' \;

config/chroot_local-includes/etc/sudoers.d/zzz_upgrade

-Cmnd_Alias INSTALL_IUK = /bin/chmod, /bin/dd, /bin/mkdir, /bin/mktemp, /bin/mount, /bin/rm, /bin/tar, /lib/live/mount/medium/utils/linux/syslinux, /usr/bin/nocache /bin/cp *
-Cmnd_Alias IUK_GET_TARGET_FILE = /usr/bin/tails-iuk-get-target-file
-Cmnd_Alias UPGRADE_FRONTEND = /usr/bin/tails-upgrade-frontend ""
+Cmnd_Alias INSTALL_IUK = /bin/dd, /bin/mount, /bin/umount, /bin/rm, /lib/live/mount/medium/utils/linux/syslinux, /usr/bin/rsync, /usr/bin/nocache /bin/cp *
+Cmnd_Alias IUK_GET_TARGET_FILE = /usr/local/bin/tails-iuk-get-target-file
+Cmnd_Alias UPGRADE_FRONTEND = /usr/local/bin/tails-upgrade-frontend ""
 
 Defaults!IUK_GET_TARGET_FILE env_keep+="HARNESS_ACTIVE DISABLE_PROXY"
 Defaults!UPGRADE_FRONTEND env_keep+="DISABLE_PROXY SSL_NO_VERIFY"
 
 amnesia                ALL = (tails-upgrade-frontend)    NOPASSWD: UPGRADE_FRONTEND
-tails-upgrade-frontend ALL =                             NOPASSWD: /usr/bin/tails-shutdown-network ""
-tails-upgrade-frontend ALL = (tails-install-iuk)         NOPASSWD: /usr/bin/tails-install-iuk
+tails-upgrade-frontend ALL =                             NOPASSWD: /usr/local/bin/tails-shutdown-network ""
+tails-upgrade-frontend ALL = (tails-install-iuk)         NOPASSWD: /usr/local/bin/tails-install-iuk
 tails-upgrade-frontend ALL = (tails-iuk-get-target-file) NOPASSWD: IUK_GET_TARGET_FILE
-tails-upgrade-frontend ALL = (tails-iuk-get-target-file) NOPASSWD: /usr/bin/tails-iuk-mktemp-get-target-file ""
+tails-upgrade-frontend ALL = (tails-iuk-get-target-file) NOPASSWD: /usr/local/bin/tails-iuk-mktemp-get-target-file ""
 tails-upgrade-frontend ALL =                             NOPASSWD: /sbin/reboot ""
 
 tails-install-iuk     ALL =                             NOPASSWD: INSTALL_IUK

config/chroot_local-includes/lib/live/config/2000-import-gnupg-key

@@ -6,9 +6,9 @@ Import_GnuPG_key ()
 	sudo -H -u "${LIVE_USERNAME}" gpg --batch --import /usr/share/doc/tails/website/*.key
 
 	echo "- importing Tails' GnuPG signing key into tails-iuk's trusted keyring"
-	gpg --batch --homedir /usr/share/tails-iuk/trusted_gnupg_homedir \
-	    --import /usr/share/doc/tails/website/tails-signing.key
-	chmod -R go+rX /usr/share/tails-iuk
+	sudo -H -u tails-upgrade-frontend \
+             gpg --batch --import \
+                 /usr/share/doc/tails/website/tails-signing-minimal.key
 
 	echo "- importing Tails help desk's GnuPG key into WhisperBack's keyring"
 	gpg --batch --no-default-keyring \

config/chroot_local-includes/usr/lib/tmpfiles.d/tails-upgrader.conf

 # Type	Path						Mode	UID		GID			Age	Argument
 d	/run/tails-upgrader				00775	root		tails-upgrade-frontend	-	-
-d	/usr/share/tails-iuk/trusted_gnupg_homedir	00700	root		root			-	-

config/chroot_local-includes/usr/local/bin/tails-upgrade-frontend-wrapper

@@ -29,7 +29,10 @@ CMD = os.path.basename(sys.argv[0])
 TORDATE_DIR = '/run/tordate'
 TORDATE_DONE_FILE = '{}/done'.format(TORDATE_DIR)
 INOTIFY_TIMEOUT = 60
-MIN_AVAILABLE_MEMORY = (300 * 1024 * 1024)  # In Bytes
+# While running iuk.git:features/frontend:
+#  - tails-upgrade-frontend never uses more than 100 * 10^6 bytes
+#  - tails-iuk-get-upgrade-description never uses more than 95 * 10^6 bytes
+MIN_AVAILABLE_MEMORY = (200 * 1024 * 1024)  # In Bytes
 RUN_AS_USER = 'tails-upgrade-frontend'
 
 ERROR_MESSAGE = gettext('''\"<b>Not enough memory available to check for upgrades.</b>
@@ -61,7 +64,7 @@ def main(*args):
         (
             "/bin/sh", "-c",
             "xhost +SI:localuser:{user};"
-            "sudo -u {user} /usr/bin/tails-upgrade-frontend {args};"
+            "sudo -u {user} /usr/local/bin/tails-upgrade-frontend {args};"
             "xhost -SI:localuser:{user}".format(user=RUN_AS_USER, args=" ".join(args))
         )
     )

config/chroot_local-includes/usr/share/tails/build/passwd

@@ -34,6 +34,6 @@ tails-persistence-setup:x:115:122::/home/tails-persistence-setup:/usr/sbin/nolog
 clearnet:x:114:123::/home/clearnet:/usr/sbin/nologin
 htp:x:116:124::/home/htp:/usr/sbin/nologin
 tails-iuk-get-target-file:x:117:125::/home/tails-iuk-get-target-file:/usr/sbin/nologin
-tails-upgrade-frontend:x:118:126::/home/tails-upgrade-frontend:/usr/sbin/nologin
+tails-upgrade-frontend:x:118:126::/var/lib/tails-upgrade-frontend:/usr/sbin/nologin
 tor-launcher:x:119:127::/home/tor-launcher:/usr/sbin/nologin
 tails-install-iuk:x:120:128::/home/tails-install-iuk:/usr/sbin/nologin

config/chroot_local-includes/usr/src/iuk/.gitignore

+blib*
+/.build
+/Makefile
+Makefile.old
+Build
+Build.bat
+_build*
+pm_to_blib*
+*.tar.gz
+.lwpcookies
+cover_db
+pod2htm*.tmp
+Tails-IUK-*
+*.bak
+debian/
+*.mo

config/chroot_local-includes/usr/src/iuk/.plsense

+lib-path=lib

config/chroot_local-includes/usr/src/iuk/MANIFEST.SKIP

+^cover_db
+~$
+^debian
+\.bak$
+\.old$

config/chroot_local-includes/usr/src/iuk/README

+Test suite
+==========
+
+Some parts of the test suite need to run some commands using
+passwordless sudo. Most of it contents itself with cached sudo
+credentials, but some parts change the TTY, so if the tty_tickets sudo
+option is enabled, then one needs to explicitly allow running, as
+root, using passwordless sudo, the following commands: /bin/chmod, /bin/mount, /bin/rm.
+
+Licence
+=======
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.

config/chroot_local-includes/usr/src/iuk/bin/tails-create-iuk

+#!/usr/bin/perl
+
+=head1 NAME
+
+tails-create-iuk - create an Incremental upgrade Kit
+
+=head1 VERSION
+
+Version 
+
+=cut
+
+use strictures 2;
+use 5.10.1;
+
+our $VERSION = '4.0.3';
+
+use FindBin;
+use lib "$FindBin::Bin/../lib";
+
+use Tails::IUK;
+
+umask 022;
+Tails::IUK->new_with_options()->run;

config/chroot_local-includes/usr/src/iuk/bin/tails-install-iuk

+#!/usr/bin/perl
+
+=head1 NAME
+
+tails-install-iuk - install an Incremental Upgrade Kit
+
+=cut
+
+use strictures 2;
+use 5.10.1;
+
+use FindBin;
+use lib "$FindBin::Bin/../lib";
+
+use Tails::IUK::Install;
+
+my $iuk = pop;
+Tails::IUK::Install->new_with_options(from_file => $iuk)->run;