Merge branch 'web/16170-other-ways-above-fold' of d53ykjpeekuikgoq.onion:tails...

Merge branch 'web/16170-other-ways-above-fold' of d53ykjpeekuikgoq.onion:tails into web/16170-other-ways-above-fold

.gitignore

@@ -53,7 +53,7 @@
 /config/chroot_local-includes/usr/share/desktop-directories/Tails.directory
 /config/chroot_local-includes/usr/share/polkit-1/actions/org.boum.tails.root-terminal.policy
 /config/chroot_local-includes/usr/share/polkit-1/actions/org.boum.tails.additional-software.policy
-/config/chroot_local-includes/usr/share/unlock-veracrypt-volumes/ui/*.ui
+/config/chroot_local-includes/usr/share/tails/unlock-veracrypt-volumes/*.ui
 /tmp/
 
 # The test suite's local configuration files

Rakefile

@@ -370,7 +370,7 @@ task :setup_environment => ['validate_git_state'] do
     end
   end
 
-  ENV['BASE_BRANCH_GIT_COMMIT'] = git_helper('git_base_branch_head')
+  ENV['BASE_BRANCH_GIT_COMMIT'] ||= git_helper('git_base_branch_head')
   ['GIT_COMMIT', 'GIT_REF', 'BASE_BRANCH_GIT_COMMIT'].each do |var|
     if ENV[var].empty?
       raise "Variable '#{var}' is empty, which should not be possible: " +

auto/build

@@ -11,21 +11,21 @@ set -x
 
 umask 022
 
-### functions
+### Clone all output, from this point on, to the log file
 
-syslinux_utils_upstream_version () {
-   dpkg-query -W -f='${Version}\n' syslinux-utils | \
-       # drop epoch
-       sed -e 's,.*:,,' | \
-       # drop +dfsg and everything that follows
-       sed -e 's,\+dfsg.*,,'
-}
+BUILD_LOG="${BUILD_BASENAME}.buildlog"
+exec >  >(tee -a "$BUILD_LOG")
+trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
+exec 2> >(tee -a "$BUILD_LOG" >&2)
+trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
+
+### functions
 
 print_iso_size () {
    local isofile="$1"
    [ -f "$isofile" ] || return 23
    size=$(stat --printf='%s' "$isofile")
-   echo "The ISO is ${size} bytes large."
+   echo "I: The ISO is ${size} bytes large."
 }
 
 ### Main
@@ -100,13 +100,9 @@ DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
 gpg --homedir "$DEBOOTSTRAP_GNUPG_HOMEDIR" \
     --no-tty \
     --import config/chroot_sources/tails.chroot.gpg
-if [ -e "$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.gpg" ]; then
-    DEBOOTSTRAP_GNUPG_KEYRING="$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.gpg"
-elif [ -e "$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.kbx" ]; then
-    DEBOOTSTRAP_GNUPG_KEYRING="$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.kbx"
-else
-   fatal "No debootstrap GnuPG keyring was created."
-fi
+DEBOOTSTRAP_GNUPG_KEYRING="$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.kbx"
+[ -e "$DEBOOTSTRAP_GNUPG_KEYRING" ] \
+   || fatal "No debootstrap GnuPG keyring was created."
 DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keyring=$DEBOOTSTRAP_GNUPG_KEYRING"
 
 export DEBOOTSTRAP_OPTIONS
@@ -121,36 +117,12 @@ export MKSQUASHFS_OPTIONS
 # refresh translations of our programs
 ./refresh-translations || fatal "refresh-translations failed ($?)."
 
-case "$LB_BINARY_IMAGES" in
-   iso)
-      which isohybrid >/dev/null || fatal 'Cannot find isohybrid in $PATH'
-      installed_syslinux_utils_upstream_version="$(syslinux_utils_upstream_version)"
-      if dpkg --compare-versions \
-	   "$installed_syslinux_utils_upstream_version" \
-	   'lt' \
-	   "$REQUIRED_SYSLINUX_UTILS_UPSTREAM_VERSION" ; then
-	  fatal \
-	      "syslinux-utils '${installed_syslinux_utils_upstream_version}' is installed, " \
-	      "while we need at least '${REQUIRED_SYSLINUX_UTILS_UPSTREAM_VERSION}'."
-      fi
-      ;;
-   *)
-      fatal "Image type ${LB_BINARY_IMAGES} is not supported."
-      ;;
-esac
 BUILD_ISO_FILENAME="${BUILD_BASENAME}.iso"
 BUILD_MANIFEST="${BUILD_BASENAME}.build-manifest"
 BUILD_APT_SOURCES="${BUILD_BASENAME}.apt-sources"
 BUILD_PACKAGES="${BUILD_BASENAME}.packages"
-BUILD_LOG="${BUILD_BASENAME}.buildlog"
 BUILD_USB_IMAGE_FILENAME="${BUILD_BASENAME}.img"
 
-# Clone all output, from this point on, to the log file
-exec >  >(tee -a "$BUILD_LOG")
-trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
-exec 2> >(tee -a "$BUILD_LOG" >&2)
-trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
-
 (
    echo "Mirrors:"
    apt-mirror debian
@@ -160,25 +132,25 @@ trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
    cat config/chroot_sources/*.chroot
 ) > "$BUILD_APT_SOURCES"
 
-echo "Building ISO image ${BUILD_ISO_FILENAME}..."
+echo "I: Building ISO image ${BUILD_ISO_FILENAME}..."
 time lb build noauto ${@}
 [ -e binary.iso ] || fatal "lb build failed ($?)."
 
-echo "ISO image was successfully created"
+echo "I: ISO image was successfully created"
 print_iso_size binary.iso
 
-echo "Hybriding it..."
+echo "I: Hybriding it..."
 isohybrid $AMNESIA_ISOHYBRID_OPTS binary.iso || fatal "isohybrid failed"
 print_iso_size binary.iso
 truncate -s %2048 binary.iso
 print_iso_size binary.iso
 
-echo "Renaming generated files..."
+echo "I: Renaming generated files..."
 mv -i binary.iso "${BUILD_ISO_FILENAME}"
 mv -i binary.packages "${BUILD_PACKAGES}"
 
-echo "Generating build manifest..."
+echo "I: Generating build manifest..."
 generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
 
-echo "Creating USB image ${BUILD_USB_IMAGE_FILENAME}..."
+echo "I: Creating USB image ${BUILD_USB_IMAGE_FILENAME}..."
 create-usb-image-from-iso "${BUILD_ISO_FILENAME}"

auto/config

@@ -7,12 +7,6 @@ set -x
 
 . "$(dirname $0)/scripts/utils.sh"
 
-# we require building from git
-if ! git rev-parse --is-inside-work-tree; then
-    echo "${PWD} is not a Git tree. Exiting."
-    exit 1
-fi
-
 . config/amnesia
 if [ -e config/amnesia.local ] ; then
    . config/amnesia.local
@@ -21,12 +15,10 @@ fi
 if [ -n "${SOURCE_DATE_EPOCH}" ]; then
     CURRENT_EPOCH="$(date --utc +%s)"
     if [ "${SOURCE_DATE_EPOCH}" -gt "${CURRENT_EPOCH}" ]; then
-        echo "SOURCE_DATE_EPOCH is set before the current time. Exiting."
-        exit 1
+        fatal "SOURCE_DATE_EPOCH is set before the current time. Exiting."
     fi
 else
-    echo "SOURCE_DATE_EPOCH is not set. Exiting."
-    exit 1
+    fatal "SOURCE_DATE_EPOCH is not set. Exiting."
 fi
 
 # get git branch or tag so we can set the basename appropriately, i.e.:
@@ -53,20 +45,19 @@ GIT_BASE_BRANCH=$(base_branch) \
 
 if [ "${TAILS_MERGE_BASE_BRANCH:-}" = 1 ] && \
        ! git_on_a_tag && [ "$GIT_BRANCH" != "$GIT_BASE_BRANCH" ] ; then
-    GIT_BASE_BRANCH_COMMIT=$(git_base_branch_head)
-    [ -n "${GIT_BASE_BRANCH_COMMIT}" ] \
-        || fatal "Base branch's top commit could not be guessed."
+    [ -n "${BASE_BRANCH_GIT_COMMIT}" ] \
+        || fatal "Base branch's top commit is not set."
 
-    echo "Merging base branch origin/${GIT_BASE_BRANCH}"
-    echo "(at commit ${GIT_BASE_BRANCH_COMMIT})..."
+    echo "I: Merging base branch ${GIT_BASE_BRANCH}" \
+         "(at commit ${BASE_BRANCH_GIT_COMMIT})..."
     faketime -f "${SOURCE_DATE_FAKETIME}" \
-        git merge --no-edit "origin/${GIT_BASE_BRANCH}" \
+        git merge --no-edit "${BASE_BRANCH_GIT_COMMIT}" \
 	    || fatal "Failed to merge base branch."
     git submodule update --init
 
     # Adjust BUILD_BASENAME to embed the base branch name and its top commit
     CLEAN_GIT_BASE_BRANCH=$(echo "$GIT_BASE_BRANCH" | sed 's,/,_,g')
-    GIT_BASE_BRANCH_SHORT_ID=$(git_base_branch_head --short)
+    GIT_BASE_BRANCH_SHORT_ID=$(git rev-parse --verify --short "${BASE_BRANCH_GIT_COMMIT}")
     [ -n "${GIT_BASE_BRANCH_SHORT_ID}" ] \
         || fatal "Base branch's top commit short ID could not be guessed."
     BUILD_BASENAME="${BUILD_BASENAME}+${CLEAN_GIT_BASE_BRANCH}"
@@ -79,19 +70,16 @@ echo "BUILD_BASENAME='${BUILD_BASENAME}'" > tmp/build_environment
 
 # sanity checks
 if grep -qs -E '^Pin:\s+release\s+.*a=' config/chroot_apt/preferences ; then
-    echo "Found unsupported a= syntax in config/chroot_apt/preferences,"
-    echo "use n= instead. Exiting."
-    exit 1
+    fatal "Found unsupported a= syntax in config/chroot_apt/preferences," \
+          "use n= instead. Exiting."
 fi
 if grep -qs -E '^Pin:\s+release\s+.*o=Debian Backports' \
 	config/chroot_apt/preferences ; then
-    echo "Found unsupported 'o=Debian Backports' syntax,"
-    echo "in config/chroot_apt/preferences. Use o=Debian instead. Exiting."
-    exit 1
+    fatal "Found unsupported 'o=Debian Backports' syntax," \
+          "in config/chroot_apt/preferences. Use o=Debian instead. Exiting."
 fi
 if [ $(dpkg --print-architecture) != amd64 ] ; then
-    echo "Only amd64 build systems are supported"
-    exit 1
+    fatal "Only amd64 build systems are supported"
 fi
 
 # init variables
@@ -102,7 +90,7 @@ $RUN_LB_CONFIG --distribution stretch ${@}
 
 # set up everything for time-based snapshots:
 if [ -n "${APT_SNAPSHOTS_SERIALS:-}" ]; then
-    echo "Fixing 'latest' APT snapshots serials to: '${APT_SNAPSHOTS_SERIALS}'."
+    echo "I: Fixing 'latest' APT snapshots serials to: '${APT_SNAPSHOTS_SERIALS}'."
     apt-snapshots-serials prepare-build "${APT_SNAPSHOTS_SERIALS}"
 else
     apt-snapshots-serials prepare-build
@@ -119,16 +107,16 @@ DEBIAN_MIRROR="$(apt-mirror debian)"
 DEBIAN_SECURITY_MIRROR="$(apt-mirror debian-security)"
 TORPROJECT_MIRROR="$(apt-mirror torproject)"
 
-[ -n "$DEBIAN_MIRROR" ]          || exit 1
-[ -n "$DEBIAN_SECURITY_MIRROR" ] || exit 1
-[ -n "$TORPROJECT_MIRROR" ]      || exit 1
+[ -n "$DEBIAN_MIRROR" ]          || fatal "\$DEBIAN_MIRROR is empty"
+[ -n "$DEBIAN_SECURITY_MIRROR" ] || fatal "\$DEBIAN_SECURITY_MIRROR is empty"
+[ -n "$TORPROJECT_MIRROR" ]      || fatal "\$TORPROJECT_MIRROR is empty"
 
 perl -pi \
      -E \
        "s|^(deb(?:-src)?\s+)https?://ftp[.]us[.]debian[.]org/debian/?(\s+)|\$1$DEBIAN_MIRROR\$2| ; \
         s|^(deb(?:-src)?\s+)https?://deb[.]torproject[.]org/torproject[.]org/?(\s+)|\$1$TORPROJECT_MIRROR\$2|" \
     config/chroot_sources/*.chroot \
-    || exit 1
+    || fatal "APT mirror substitution failed with exit code $?"
 
 # set Amnesia's general options
 $RUN_LB_CONFIG \
@@ -171,7 +159,8 @@ install -d config/chroot_local-includes/etc/amnesia/
 
 # environment
 TAILS_WIKI_SUPPORTED_LANGUAGES="$(ikiwiki-supported-languages ikiwiki.setup)"
-[ -n "$TAILS_WIKI_SUPPORTED_LANGUAGES" ] || exit 16
+[ -n "$TAILS_WIKI_SUPPORTED_LANGUAGES" ] \
+   || fatal "\$TAILS_WIKI_SUPPORTED_LANGUAGES is empty"
 echo "TAILS_WIKI_SUPPORTED_LANGUAGES='${TAILS_WIKI_SUPPORTED_LANGUAGES}'" \
    >> config/chroot_local-includes/etc/amnesia/environment
 
@@ -186,6 +175,7 @@ echo "live-build: `dpkg-query -W -f='${Version}\n' live-build`" \
 cat >> config/chroot_local-includes/etc/os-release <<EOF
 TAILS_PRODUCT_NAME="Tails"
 TAILS_VERSION_ID="$AMNESIA_VERSION"
+TAILS_DISTRIBUTION="$TAILS_DISTRIBUTION"
 EOF
 if echo "$AMNESIA_VERSION" | grep -qs -E '~(alpha|beta|rc)[0-9]*$' ; then
     echo 'TAILS_CHANNEL="alpha"' >> config/chroot_local-includes/etc/os-release

auto/scripts/create-usb-image-from-iso

@@ -39,8 +39,12 @@ GET_UDISKS_OBJECT_TIMEOUT = 2
 # the partition table, reserved sectors, and filesystem metadata.
 SYSTEM_PARTITION_ADDITIONAL_SIZE = 10
 
-SYSLINUX_COM32MODULES_DIR = '/usr/lib/syslinux/modules/bios'
-
+# We use the syslinux from the chroot here, because it's the same one
+# that will be available to Tails Installer in the running Tails. Using
+# the same syslinux version here and in Tails Installer is important to
+# prevent issues when upgrading a Tails device via Tails Installer.
+CHROOT_SYSLINUX_COM32MODULES_DIR = 'chroot/usr/lib/syslinux/modules/bios'
+CHROOT_SYSLINUX_BIN='chroot/usr/bin/syslinux'
 
 class ImageCreationError(Exception):
     pass
@@ -252,7 +256,7 @@ class ImageCreator(object):
         com32modules = [f for f in os.listdir(syslinux_dir) if f.endswith('.c32')]
 
         for module in sorted(com32modules):
-            src_path = os.path.join(SYSLINUX_COM32MODULES_DIR, module)
+            src_path = os.path.join(CHROOT_SYSLINUX_COM32MODULES_DIR, module)
             if not os.path.isfile(src_path):
                 raise ImageCreationError("Could not find the '%s' COM32 module" % module)
 
@@ -269,7 +273,7 @@ class ImageCreator(object):
         # device would cause this issue:
         # https://bugs.chromium.org/p/chromium/issues/detail?id=508713#c8
         execute([
-            'syslinux',
+            CHROOT_SYSLINUX_BIN,
             '--offset', str(self.partition.props.partition.props.offset),
             '--directory', '/syslinux/',
             '--install', self.image

auto/scripts/utils.sh

@@ -97,7 +97,7 @@ branch_name_to_suite() {
 }
 
 fatal() {
-	echo "$*" >&2
+	echo "E: $*" >&2
 	exit 1
 }
 

config/APT_snapshots.d/debian/serial

-2019042801
+2019061901

config/APT_snapshots.d/torproject/serial

-2019031301
+2019060802

config/amnesia

@@ -17,16 +17,13 @@ export SOURCE_DATE_FAKETIME="$(date --utc --date="$(dpkg-parsechangelog --show-f
 # Base for the string that will be passed to "lb config --bootappend-live"
 # FIXME: see [[bugs/sdmem_on_eject_broken_for_CD]] for explanation why we
 # need to set block.events_dfl_poll_msecs
-AMNESIA_APPEND="live-media=removable nopersistence noprompt timezone=Etc/UTC block.events_dfl_poll_msecs=1000 splash noautologin module=Tails slab_nomerge slub_debug=FZP mce=0 vsyscall=none page_poison=1 union=aufs"
+AMNESIA_APPEND="live-media=removable nopersistence noprompt timezone=Etc/UTC block.events_dfl_poll_msecs=1000 splash noautologin module=Tails slab_nomerge slub_debug=FZP mce=0 vsyscall=none page_poison=1 mds=full,nosmt union=aufs"
 
 # Options passed to isohybrid
 AMNESIA_ISOHYBRID_OPTS="-h 255 -s 63 --id 42 --verbose"
 
-# Minimal upstream version of syslinux-utils we need
-REQUIRED_SYSLINUX_UTILS_UPSTREAM_VERSION="6.03~pre20"
-
 # Kernel version
-KERNEL_VERSION='4.19.0-4'
+KERNEL_VERSION='4.19.0-5'
 KERNEL_SOURCE_VERSION=$(
    echo "$KERNEL_VERSION" \
        | perl -p -E 's{\A (\d+ [.] \d+) [.] .*}{$1}xms'
@@ -49,3 +46,6 @@ AMNESIA_FULL_VERSION="${AMNESIA_VERSION} - ${SOURCE_DATE_YYYYMMDD}"
 AMNESIA_DEV_FULLNAME='Tails developers'
 AMNESIA_DEV_EMAIL="tails@boum.org"
 AMNESIA_DEV_KEYID="A490 D0F4 D311 A415 3E2B  B7CA DBB8 02B2 58AC D84F"
+
+# Used to set a custom home page if the distribution is UNRELEASED
+TAILS_DISTRIBUTION="`dpkg-parsechangelog -SDistribution`"

config/chroot_apt/preferences

@@ -10,9 +10,9 @@ Package: b43-fwcutter
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
 
-Explanation: unavailable in stretch and stretch-backports
+Explanation: unavailable in stretch and stretch-backports, version in sid is intentionally broken (Debian#928518)
 Package: electrum python3-electrum
-Pin: release o=Debian,n=sid
+Pin: origin deb.tails.boum.org
 Pin-Priority: 999
 
 Explanation: Electrum dependencies
@@ -37,10 +37,15 @@ Pin: release o=Debian,n=sid
 Pin-Priority: 999
 
 Explanation: src:firmware-nonfree
-Package: firmware-linux firmware-linux-nonfree firmware-amd-graphics firmware-atheros firmware-brcm80211 firmware-intel-sound firmware-ipw2x00 firmware-iwlwifi firmware-libertas firmware-misc-nonfree firmware-realtek firmware-ti-connectivity
+Package: firmware-linux firmware-linux-nonfree firmware-atheros firmware-brcm80211 firmware-intel-sound firmware-ipw2x00 firmware-iwlwifi firmware-libertas firmware-misc-nonfree firmware-realtek firmware-ti-connectivity
 Pin: release o=Debian,n=sid
 Pin-Priority: 990
 
+Explanation: Exception to src:firmware-nonfree pinning due to Debian#928631
+Package: firmware-amd-graphics
+Pin: release o=Debian,n=stretch-backports
+Pin-Priority: 990
+
 Package: firmware-zd1211
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
@@ -125,6 +130,10 @@ Package: tails-installer
 Pin: origin deb.tails.boum.org
 Pin-Priority: 999
 
+Package: tor tor-geoipdb
+Pin: release o=TorProject,n=tor-experimental-0.4.0.x-stretch
+Pin-Priority: 999
+
 Package: virtualbox*
 Pin: release o=Debian,n=stretch-backports
 Pin-Priority: 999
@@ -158,10 +167,6 @@ Package: xul-ext-torbirdy
 Pin: release o=Debian,n=stretch-backports
 Pin-Priority: 999
 
-Package: pdf-redact-tools
-Pin: release o=Debian,n=sid
-Pin-Priority: 999
-
 Explanation: weirdness in chroot_apt install-binary
 Package: *
 Pin: release o=chroot_local-packages

config/chroot_local-hooks/04-change-gids-and-uids

@@ -95,13 +95,20 @@ Change_gid () {
 	fi
 }
 
-# Temporarily give these groups a GID that's out of the way, to avoid collisions
+# Temporarily give these users and groups a UID/GID that's out of the way,
+# to avoid collisions
+Change_uid debian-tor   1070
+Change_uid speech-dispatcher 1080
+Change_uid colord       1090
+Change_uid saned        1100
+Change_uid pulse        1110
+Change_uid hplip        1120
+Change_uid Debian-gdm   1130
 Change_gid messagebus   1050
 Change_gid ssh          1090
 Change_gid memlockd     1100
 Change_gid ssl-cert     1110
 Change_gid vboxsf       1120
-Change_gid monkeysphere 1130
 Change_gid debian-tor   1140
 Change_gid lpadmin      1150
 Change_gid scanner      1160
@@ -112,14 +119,21 @@ Change_gid pulse-access 1200
 Change_gid Debian-gdm   1210
 Change_gid kvm          1500
 Change_gid render       1510
+Change_gid Debian-exim  1520
 
-# Finally, give these groups the desired GID
+# Finally, give these users and groups the desired UID/GID
+Change_uid debian-tor   107
+Change_uid speech-dispatcher 108
+Change_uid colord       109
+Change_uid saned        110
+Change_uid pulse        111
+Change_uid hplip        112
+Change_uid Debian-gdm   113
 Change_gid messagebus   105
 Change_gid ssh          109
 Change_gid memlockd     110
 Change_gid ssl-cert     111
 Change_gid vboxsf       112
-Change_gid monkeysphere 113
 Change_gid debian-tor   114
 Change_gid lpadmin      115
 Change_gid scanner      116
@@ -130,3 +144,4 @@ Change_gid pulse-access 120
 Change_gid Debian-gdm   121
 Change_gid kvm          150
 Change_gid render       151
+Change_gid Debian-exim  152

config/chroot_local-hooks/06-adduser_clearnet

@@ -9,4 +9,5 @@ set -e
 
 echo "Creating the clearnet user"
 
-adduser --system --quiet --group clearnet
+addgroup --system --quiet --gid 123 clearnet
+adduser --system --quiet --uid 114 --gid 123 clearnet

config/chroot_local-hooks/06-adduser_htp

@@ -9,4 +9,5 @@ set -e
 
 echo "Creating the htp user"