Merge branch 'master' into stable

.gitmodules

@@ -11,3 +11,6 @@
 [submodule "submodules/mirror-pool-dispatcher"]
 	path = submodules/mirror-pool-dispatcher
 	url = https://git-tails.immerda.ch/mirror-pool-dispatcher
+[submodule "submodules/aufs4-standalone"]
+	path = submodules/aufs4-standalone
+	url = https://github.com/sfjro/aufs4-standalone.git

auto/config

@@ -192,7 +192,8 @@ if [ -e config/binary_rootfs/squashfs.sort ]; then
 fi
 
 # custom APT sources
-tails-custom-apt-sources > config/chroot_sources/tails.chroot
+tails-custom-apt-sources > config/chroot_sources/tails.chroot \
+   || fatal "tails-custom-apt-sources failed with exit code $?"
 
 # tails-transform-mirror-url and its dependencies
 install -m 0755 \
@@ -203,6 +204,10 @@ install -m 0755 \
    submodules/mirror-pool-dispatcher/lib/js/mirror-dispatcher.js \
    config/chroot_local-includes/usr/local/lib/nodejs/
 
+# aufs4-standalone
+rm -rf config/chroot_local-includes/usr/src/aufs4-standalone
+cp -a submodules/aufs4-standalone config/chroot_local-includes/usr/src/
+
 # custom debootstrap script, setting some APT magic to log downloads:
 patch \
     --follow-symlinks \
@@ -210,3 +215,7 @@ patch \
     /usr/share/debootstrap/scripts/jessie \
     data/debootstrap/scripts/jessie.patch
 sed -i "s,%%topdir%%,$(pwd)," /usr/share/debootstrap/scripts/tails-build-jessie
+
+# Make the python library available in Tails
+install -d -m 2777 config/chroot_local-includes/tmp/
+cp -r submodules/pythonlib config/chroot_local-includes/tmp/

config/APT_snapshots.d/debian/serial

-2018011503
+2018031004

config/APT_snapshots.d/torproject/serial

-2017120803
+2018030601

config/amnesia

@@ -26,7 +26,7 @@ AMNESIA_ISOHYBRID_OPTS="-h 255 -s 63 --id 42 --verbose"
 REQUIRED_SYSLINUX_UTILS_UPSTREAM_VERSION="6.03~pre20"
 
 # Kernel version
-KERNEL_VERSION='4.14.0-3'
+KERNEL_VERSION='4.15.0-1'
 KERNEL_SOURCE_VERSION=$(
    echo "$KERNEL_VERSION" \
        | perl -p -E 's{\A (\d+ [.] \d+) [.] .*}{$1}xms'

config/binary_rootfs/squashfs.sort

@@ -3597,14 +3597,14 @@ usr/share/gnome-shell/extensions/places-menu@gnome-shell-extensions.gcampax.gith
 usr/share/gnome-shell/extensions/places-menu@gnome-shell-extensions.gcampax.github.com/placeDisplay.js 27832
 usr/share/gnome-shell/extensions/places-menu@gnome-shell-extensions.gcampax.github.com/stylesheet.css 27831
 usr/share/gnome-shell/extensions/screenshot-window-sizer@gnome-shell-extensions.gcampax.github.com/metadata.json 27830
-usr/share/gnome-shell/extensions/shutdown-helper@tails.boum.org/metadata.json 27829
+usr/share/gnome-shell/extensions/status-menu-helper@tails.boum.org/metadata.json 27829
 usr/sbin/cupsd                                                       27824
 usr/lib/x86_64-linux-gnu/libcupsmime.so.1                            27823
 usr/lib/x86_64-linux-gnu/libpaper.so.1.1.2                           27822
 etc/cups/cups-files.conf                                             27821
 etc/cups/cupsd.conf                                                  27820
-usr/share/gnome-shell/extensions/shutdown-helper@tails.boum.org/extension.js 27817
-usr/share/gnome-shell/extensions/shutdown-helper@tails.boum.org/lib.js 27816
+usr/share/gnome-shell/extensions/status-menu-helper@tails.boum.org/extension.js 27817
+usr/share/gnome-shell/extensions/status-menu-helper@tails.boum.org/lib.js 27816
 usr/share/gnome-shell/extensions/torstatus@tails.boum.org/metadata.json 27815
 usr/share/gnome-shell/extensions/torstatus@tails.boum.org/extension.js 27814
 usr/share/gnome-shell/extensions/user-theme@gnome-shell-extensions.gcampax.github.com/metadata.json 27813

config/chroot_apt/preferences

@@ -36,6 +36,10 @@ Package: gir1.2-gdkpixbuf-2.0 libgdk-pixbuf2.0-*
 Pin: version 2.36.5-2.0tails*
 Pin-Priority: -1
 
+Package: intel-microcode
+Pin: release o=Debian,n=stretch-backports
+Pin-Priority: 999
+
 Package: linux-compiler-* linux-headers-* linux-image-* linux-kbuild-* linux-source-*
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
@@ -49,10 +53,20 @@ Package: obfs4proxy
 Pin: release o=TorProject,n=obfs4proxy
 Pin-Priority: 990
 
+Explanation: src:systemd
+Explanation: systemd >= v233 required for meek_lite and enable the unsafe browser and Tor launcher applications to do clearnet DNS resolution. (#8243)
+Package: systemd systemd-sysv systemd-container systemd-journal-remote systemd-coredump systemd-tests libpam-systemd libnss-myhostname libnss-mymachines libnss-resolve libnss-systemd libsystemd0 libsystemd-dev udev libudev1 libudev-dev udev-udeb libudev1-udeb
+Pin: release o=Debian,n=stretch-backports
+Pin-Priority: 999
+
 Package: onionshare
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
 
+Package: openpgp-applet
+Pin: release o=Debian,n=sid
+Pin-Priority: 999
+
 Package: tails-installer
 Pin: origin deb.tails.boum.org
 Pin-Priority: 999
@@ -61,10 +75,19 @@ Package: virtualbox*
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
 
+Explanation: src:xorg-server
+Package: xserver-xorg-core xserver-xorg-dev xdmx xdmx-tools xnest xvfb xserver-xephyr xserver-common xorg-server-source xwayland xserver-xorg-legacy
+Pin: release o=Debian,n=stretch
+Pin-Priority: 999
+
 Package: xul-ext-ublock-origin
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
 
+Package: pdf-redact-tools
+Pin: release o=Debian,n=sid
+Pin-Priority: 999
+
 Explanation: weirdness in chroot_apt install-binary
 Package: *
 Pin: release o=chroot_local-packages
@@ -101,3 +124,7 @@ Pin-Priority: -10
 Package: *
 Pin: release o=TorProject
 Pin-Priority: -10
+
+Package: electrum python3-electrum python3-jsonrpclib-pelix python3-pyaes
+Pin: release o=Debian,n=stretch-backports
+Pin-Priority: 999

config/chroot_local-hooks/00-install-tailslib

+#!/bin/sh
+
+set -e
+set -u
+
+echo "Installing the tailslib python library"
+
+# Import ensure_hook_dependency_is_installed()
+. /usr/local/lib/tails-shell-library/build.sh
+
+ensure_hook_dependency_is_installed python3-setuptools
+
+(
+    cd /tmp/pythonlib
+    python3 setup.py clean
+    python3 setup.py install
+)
+rm -rf /tmp/pythonlib

config/chroot_local-hooks/01-check-for-dot-orig-files

@@ -4,19 +4,21 @@ set -e
 
 echo "Checking for .orig files"
 
-DOT_ORIG_WHITELIST=$(cat <<EOF
+DOT_ORIG_WHITELIST_DELETE=$(cat <<EOF
 /bin/hostname.orig
 /etc/resolv.conf.orig
 /lib/systemd/system/alsa-utils.service.orig
-/sbin/start-stop-daemon.orig
 EOF
 )
 
-rm -f ${DOT_ORIG_WHITELIST}
+# live-build creates this backup copy and restores it later in the build process
+DOT_ORIG_WHITELIST_KEEP="/sbin/start-stop-daemon.orig"
+
+rm -f ${DOT_ORIG_WHITELIST_DELETE}
 
 DOT_ORIG_FILES=$(find / -type f -name *.orig || :)
 
-if [ -n "$DOT_ORIG_FILES" ]; then
+if [ "$DOT_ORIG_FILES" != "$DOT_ORIG_WHITELIST_KEEP" ]; then
     echo "Some patches are fuzzy and leave .orig files around:" >&2
     echo "$DOT_ORIG_FILES" >&2
     exit 1

config/chroot_local-hooks/01-check-for-outdated-AppArmor-feature-set

-#! /bin/sh
-
-set -e
-set -u
-set -x
-
-echo "Checking if we should stop shipping our own AppArmor feature set"
-
-if [ -f /usr/share/apparmor-features/features ]; then
-    if cmp --quiet /usr/share/apparmor-features/features.Tails \
-              /usr/share/apparmor-features/features; then
-        echo "Debian ships the same AppArmor feature set as ours. " \
-             "Likely we can now remove our own one." >&2
-    else
-        echo "Debian ships a different AppArmor feature set from ours. " \
-             "Likely our own one is outdated and can be removed:" >&2
-        diff -Naur \
-            /usr/share/apparmor-features/features.Tails \
-            /usr/share/apparmor-features/features \
-            >&2
-    fi
-    # In any case, we probably have to do something about it.
-    exit 1
-fi

config/chroot_local-hooks/10-tbb

@@ -275,8 +275,9 @@ create_default_profile() {
     rsync -a --exclude bookmarks.html --exclude extensions \
           "${tbb_profile}"/ "${destination}"/
 
-    # Remove TBB's default bridges
-    sed -i '/extensions\.torlauncher\.default_bridge\./d' "${destination}"/preferences/extension-overrides.js
+    # Remove TBB's Tor Launcher settings since we don't enable it in
+    # our Tor Browser.
+    sed -i '/extensions\.torlauncher\./d' "${destination}"/preferences/extension-overrides.js
 
     mkdir -p "${destination}"/extensions
     for ext in "${tbb_extensions_dir}"/*; do

config/chroot_local-hooks/11-localize_browser

@@ -11,12 +11,14 @@ echo "Localize each supported browser locale"
 # Import language_code_from_locale()
 . /usr/local/lib/tails-shell-library/localization.sh
 
-# Import strip_nondeterminism_wrapper()
+# Import strip_nondeterminism_wrapper() and ensure_hook_dependency_is_installed()
 . /usr/local/lib/tails-shell-library/build.sh
 
 # Import TAILS_WIKI_SUPPORTED_LANGUAGES
 . /etc/amnesia/environment
 
+ensure_hook_dependency_is_installed p7zip imagemagick
+
 TBB_LOCALIZED_SEARCHPLUGINS_DIR="${TBB_INSTALL}/distribution/searchplugins/locale/"
 BROWSER_LOCALIZATION_DIR="/usr/share/tails/browser-localization"
 DESCRIPTIONS_FILE="${BROWSER_LOCALIZATION_DIR}/descriptions"

config/chroot_local-hooks/12-generate-ublock-origin-filter

@@ -4,7 +4,10 @@ set -e
 
 echo "Converting uBlock database dump into sqlite blob"
 
-apt-get install --yes sqlite3
+# Import ensure_hook_dependency_is_installed()
+. /usr/local/lib/tails-shell-library/build.sh
+
+ensure_hook_dependency_is_installed sqlite3
 
 DUMP="/usr/share/tails/ublock-origin/ublock0.dump"
 DATABASE="/etc/tor-browser/profile/extension-data/ublock0.sqlite"
@@ -18,5 +21,3 @@ mkdir -p "$(dirname "${DATABASE}")"
 sed ':a;N;$!ba;s_\r\n__g' "${DUMP}" | sqlite3 "${DATABASE}"
 
 echo "Created uBlock sqlite blob successfully"
-
-apt-get purge --yes sqlite3

config/chroot_local-hooks/12-kernel-modules-build-environment

+#!/bin/sh
+
+set -e
+set -u
+set -x
+
+echo "Setting up a build environment for kernel modules"
+
+. /usr/share/amnesia/build/variables
+
+# Import ensure_hook_dependency_is_installed() and
+# install_fake_package()
+. /usr/local/lib/tails-shell-library/build.sh
+
+# Install gcc-6 and fake linux-compiler-gcc-7-x86
+# (linux-headers-4.14+ depends on it, but Stretch hasn't GCC 7)
+# XXX:Buster: remove this hack.
+ensure_hook_dependency_is_installed gcc-6
+NEWEST_INSTALLED_KERNEL_VERSION="$(
+    dpkg-query --showformat '${Version}\n' --show 'linux-image-*-amd64' \
+    | sort --version-sort | tail -n1
+)"
+install_fake_package \
+    linux-compiler-gcc-7-x86 \
+    "${NEWEST_INSTALLED_KERNEL_VERSION}~0tails1"
+ln -s /usr/bin/gcc-6 /usr/bin/gcc-7
+
+ensure_hook_dependency_is_installed \
+    build-essential \
+    libelf-dev \
+    "linux-headers-${KERNEL_VERSION}-amd64"

config/chroot_local-hooks/13-aufs

+#! /bin/sh
+
+set -e
+set -u
+
+echo "Building the aufs module"
+
+. /usr/share/amnesia/build/variables
+
+# Import ensure_hook_dependency_is_installed()
+. /usr/local/lib/tails-shell-library/build.sh
+
+ensure_hook_dependency_is_installed \
+    "linux-source-${KERNEL_SOURCE_VERSION}"
+
+# aufs build needs fs/mount.h, which is in linux-source-* but not
+# in linux-headers-*, so we'll symlink it.
+tar --directory=/usr/src \
+    -xf "/usr/src/linux-source-${KERNEL_SOURCE_VERSION}.tar."*
+
+arch=amd64
+ln -s \
+   "/usr/src/linux-source-${KERNEL_SOURCE_VERSION}/fs" \
+   "/usr/src/linux-headers-${KERNEL_VERSION}-${arch}/fs"
+(
+   cd /usr/src/aufs4-standalone
+   perl -pi -E \
+        's{\A CONFIG_AUFS_DEBUG \s* = \s* y $}{CONFIG_AUFS_DEBUG =}xms' \
+        config.mk
+   KDIR="/usr/src/linux-headers-${KERNEL_VERSION}-${arch}"
+   make clean   KDIR="$KDIR"
+   make install KDIR="$KDIR"
+)
+
+for modules_dir in /lib/modules/*/extra ; do
+   if [ ! -f "${modules_dir}/aufs.ko" ]; then
+       echo "Can not find aufs.ko module in '${modules_dir}" >&2
+       exit 1
+   fi
+done
+
+depmod "${KERNEL_VERSION}-${arch}"
+rm -r /usr/src/aufs4-standalone
+rm -r "/usr/src/linux-source-${KERNEL_SOURCE_VERSION}"

config/chroot_local-hooks/15-tor-browser-bookmarks

@@ -4,6 +4,11 @@ set -e
 
 echo "Set up Tor Browser bookmarks"
 
+# Import ensure_hook_dependency_is_installed()
+. /usr/local/lib/tails-shell-library/build.sh
+
+ensure_hook_dependency_is_installed sqlite3
+
 # Create a symlink to places.sqlite in browser profile from a
 # dedicated "bookmarks" directory, so that it can be easily made
 # persistent
@@ -11,8 +16,6 @@ ln -s /home/amnesia/.mozilla/firefox/bookmarks/places.sqlite \
    /etc/skel/.tor-browser/profile.default/places.sqlite
 
 # Create the bookmarks database
-apt install --yes sqlite3
 sqlite3 /etc/skel/.mozilla/firefox/bookmarks/places.sqlite \
         < /etc/skel/.mozilla/firefox/bookmarks/places.sqlite.in
-apt purge --yes sqlite3
 rm /etc/skel/.mozilla/firefox/bookmarks/places.sqlite.in

config/chroot_local-hooks/19-install-tor-browser-AppArmor-profile

@@ -2,10 +2,14 @@
 
 set -e
 
-echo "Installing AppArmor profile for Tor Browser"
+echo "Installing AppArmor profiles for Tor Browser"
+
+# Import ensure_hook_dependency_is_installed()
+. /usr/local/lib/tails-shell-library/build.sh
+
+ensure_hook_dependency_is_installed patch
 
 PATCH='/usr/share/tails/torbrowser-AppArmor-profile.patch'
-PROFILE='/etc/apparmor.d/torbrowser'
 
 ### Functions
 
@@ -33,14 +37,17 @@ toggle_src_APT_sources() {
    apt-get --yes update
 }
 
-install_torbrowser_AppArmor_profile() {
+install_torbrowser_AppArmor_profiles() {
    tmpdir="$(mktemp -d)"
    (
       cd "$tmpdir"
       apt-get source torbrowser-launcher/sid
       install -m 0644 \
-              torbrowser-launcher-*/apparmor/torbrowser.Browser.firefox \
-              "$PROFILE"
+              torbrowser-launcher-*/apparmor/torbrowser.Browser.* \
+              /etc/apparmor.d/
+      install -m 0644 \
+              torbrowser-launcher-*/apparmor/tunables/* \
+              /etc/apparmor.d/tunables/
    )
    rm -r "$tmpdir"
 }
@@ -48,7 +55,7 @@ install_torbrowser_AppArmor_profile() {
 ### Main
 
 toggle_src_APT_sources on
-install_torbrowser_AppArmor_profile
+install_torbrowser_AppArmor_profiles
 toggle_src_APT_sources off
-patch --forward --batch "$PROFILE" < "$PATCH"
+(cd / && patch --forward --batch -p1 < "$PATCH")
 rm "$PATCH"

config/chroot_local-hooks/20-dconf_update

@@ -7,5 +7,10 @@ set -e
 
 echo "Updating the system DConf databases"
 
+# Import ensure_hook_dependency_is_installed()
+. /usr/local/lib/tails-shell-library/build.sh
+
+ensure_hook_dependency_is_installed dconf-cli
+
 dconf update
 chmod 0644 /etc/dconf/db/local

config/chroot_local-hooks/32-logind-NAutoVTs

+#!/bin/sh
+set -e
+set -u
+
+# Make room for tails-gdm-failed-to-start.service
+echo "Lower logind's NAutoVTs"
+
+sed --in-place --regexp-extended \
+    's/^#NAutoVTs=.*$/NAutoVTs=4/' \
+    /etc/systemd/logind.conf

config/chroot_local-hooks/42-wrap-gdm-x-session

+#!/bin/sh
+
+set -eu
+
+echo "Wrapping gdm-x-session to limit the number of allowed failures"
+
+dpkg-divert --add --rename --divert \
+            /usr/lib/gdm3/gdm-x-session.real \
+	    /usr/lib/gdm3/gdm-x-session
+
+ln -s /usr/lib/gdm3/gdm-x-session.tails /usr/lib/gdm3/gdm-x-session