Merge branch 'feature/stretch' into bugfix/12565-reorder-features

HACKING.mdwn

+wiki/src/contribute/how/code/HACKING.mdwn
\ No newline at end of file

Rakefile

@@ -34,6 +34,7 @@ STABLE_BRANCH_NAMES = ['stable', 'testing']
 
 EXPORTED_VARIABLES = [
   'MKSQUASHFS_OPTIONS',
+  'TAILS_DATE_OFFSET',
   'TAILS_MERGE_BASE_BRANCH',
   'TAILS_OFFLINE_MODE',
   'TAILS_PROXY',
@@ -244,12 +245,18 @@ task :parse_build_options do
       end
     when 'defaultcomp'
       ENV['MKSQUASHFS_OPTIONS'] = nil
-    # Virtual CPUs settings
+    # Virtual hardware settings
+    when /machinetype=([a-zA-Z0-9_.-]+)/
+      ENV['TAILS_BUILD_MACHINE_TYPE'] = $1
     when /cpus=(\d+)/
       ENV['TAILS_BUILD_CPUS'] = $1
+    when /cpumodel=([a-zA-Z0-9_-]+)/
+      ENV['TAILS_BUILD_CPU_MODEL'] = $1
     # Git settings
     when 'ignorechanges'
       ENV['TAILS_BUILD_IGNORE_CHANGES'] = '1'
+    when /dateoffset=([-+]\d+)/
+      ENV['TAILS_DATE_OFFSET'] = $1
     # Developer convenience features
     when 'keeprunning'
       $keep_running = true

auto/build

@@ -76,6 +76,10 @@ chmod -R go+rX config/chroot_local-includes/var
 chmod -R go+rX config/chroot_apt
 chmod -R go+rX config/chroot_sources
 
+# normalize file timestamps
+find config/binary_local-includes config/chroot_local-includes \
+     -exec touch --date="@$SOURCE_DATE_EPOCH" '{}' \;
+
 # build the image
 
 # we need /debootstrap/deburis to build a manifest of used packages:
@@ -99,7 +103,7 @@ DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keyring=$DEBOOTSTRAP_GNUPG_KEYRING"
 
 export DEBOOTSTRAP_OPTIONS
 
-: ${MKSQUASHFS_OPTIONS:='-comp xz -Xbcj x86 -b 1024K -Xdict-size 1024K'}
+: ${MKSQUASHFS_OPTIONS:='-comp xz -Xbcj x86 -b 1024K -Xdict-size 1024K -no-exports'}
 MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS} -wildcards -ef chroot/usr/share/amnesia/build/mksquashfs-excludes"
 export MKSQUASHFS_OPTIONS
 
@@ -215,7 +219,7 @@ if [ -e "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" ]; then
        ISO_FILE="${BUILD_FILENAME}.${BUILD_FILENAME_EXT}"
        print_iso_size "$ISO_FILE"
        echo "Hybriding it..."
-       isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE"
+       isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE" || fatal "isohybrid failed"
        print_iso_size "$ISO_FILE"
        truncate -s %2048 "$ISO_FILE"
        print_iso_size "$ISO_FILE"

config/amnesia

@@ -10,13 +10,16 @@
 # These configuration files are actually shell scripts, and are
 # sourced by various other scripts.
 
+export SOURCE_DATE_EPOCH="$(date --utc --date="$(dpkg-parsechangelog --show-field=Date)" +%s)"
+export SOURCE_DATE_YYYYMMDD="$(date --utc --date="$(dpkg-parsechangelog --show-field=Date)" +%Y%m%d)"
+
 # Base for the string that will be passed to "lb config --bootappend-live"
 # FIXME: see [[bugs/sdmem_on_eject_broken_for_CD]] for explanation why we
 # need to set block.events_dfl_poll_msecs
 AMNESIA_APPEND="live-media=removable apparmor=1 security=apparmor nopersistence noprompt timezone=Etc/UTC block.events_dfl_poll_msecs=1000 splash noautologin module=Tails kaslr slab_nomerge slub_debug=FZP mce=0 vsyscall=none page_poison=1 union=aufs"
 
 # Options passed to isohybrid
-AMNESIA_ISOHYBRID_OPTS="-h 255 -s 63"
+AMNESIA_ISOHYBRID_OPTS="-h 255 -s 63 --id 42 --verbose"
 
 # Minimal upstream version of syslinux-utils we need
 REQUIRED_SYSLINUX_UTILS_UPSTREAM_VERSION="6.03~pre20"
@@ -38,9 +41,8 @@ fi
 
 # Compute the current Amnesia's version once for all
 AMNESIA_NOW="`date --utc '+%Y%m%dT%H%MZ'`"
-AMNESIA_TODAY="`date '+%Y%m%d'`"
 AMNESIA_VERSION="`dpkg-parsechangelog -SVersion`"
-AMNESIA_FULL_VERSION="${AMNESIA_VERSION} - ${AMNESIA_TODAY}"
+AMNESIA_FULL_VERSION="${AMNESIA_VERSION} - ${SOURCE_DATE_YYYYMMDD}"
 
 # Developpers' data used by git-dch, debcommit and friends in the release script
 AMNESIA_DEV_FULLNAME='Tails developers'

config/chroot_local-hooks/10-tbb

@@ -51,8 +51,16 @@ install_tor_browser() {
     destination="${2}"
 
     tmp="$(mktemp -d)"
-    tar -xf "${bundle}" -C "${tmp}" tor-browser_en-US
-    prep="${tmp}"/tor-browser_en-US/Browser
+    tar -xf "${bundle}" -C "${tmp}"
+    if [ -d "${tmp}"/tor-browser_en-US ]; then
+        prep="${tmp}"/tor-browser_en-US/Browser
+    elif [ -d "${tmp}"/tor-browser ]; then
+        # TBB nightly builds
+        prep="${tmp}"/tor-browser/Browser
+    else
+        echo "The main bundle's top level directory is wrong" >&2
+        exit 1
+    fi
 
     # Enable our myspell/hunspell dictionaries. TBB only provides the
     # one for en-US, but Debian's seems more comprehensive, so we'll
@@ -78,12 +86,13 @@ install_tor_browser() {
     torlauncher_version="$(sed -n \
         's,^        <em:version>\([0-9\.]\+\)</em:version>,\1,p' \
         "${TOR_LAUNCHER_INSTALL}/install.rdf")"
+    SOURCE_DATE_YYYYMMDD=$(date --utc --date="@$SOURCE_DATE_EPOCH" '+%Y%m%d')
     cat > "${TOR_LAUNCHER_INSTALL}/application.ini" << EOF
 [App]
 Vendor=TorProject
 Name=TorLauncher
 Version=${torlauncher_version}
-BuildID=$(date +%Y%m%d)
+BuildID=${SOURCE_DATE_YYYYMMDD}
 ID=tor-launcher@torproject.org
 
 [Gecko]
@@ -106,6 +115,82 @@ EOF
     rm -r "${tmp}"
 }
 
+# TBB works around the lack of code signing for its extensions by
+# hacking in exceptions. We do the same!
+apply_extension_code_signing_hacks () {
+    local destination tmp
+    destination="${1}"
+
+    tmp="$(mktemp -d)"
+    (
+        cd "${tmp}"
+        7z x -tzip "${TBB_INSTALL}/omni.ja" \
+           modules/addons/XPIProvider.jsm \
+           chrome/toolkit/content/mozapps/extensions/extensions.js
+        patch -p1 <<EOF
+diff -Naur a/chrome/toolkit/content/mozapps/extensions/extensions.js b/chrome/toolkit/content/mozapps/extensions/extensions.js
+--- a/chrome/toolkit/content/mozapps/extensions/extensions.js 2000-01-01 00:00:00.000000000 +0000
++++ b/chrome/toolkit/content/mozapps/extensions/extensions.js 2000-01-01 00:00:00.000000000 +0000
+@@ -282,7 +282,9 @@
+   // they aren't the correct type for signing.
+   if (aAddon.id == "torbutton@torproject.org" ||
+       aAddon.id == "tor-launcher@torproject.org" ||
+-      aAddon.id == "https-everywhere-eff@eff.org") {
++      aAddon.id == "https-everywhere-eff@eff.org" ||
++      aAddon.id == "branding@amnesia.boum.org" ||
++      aAddon.id == "uBlock0@raymondhill.net") {
+     return true;
+   }
+   return aAddon.isCorrectlySigned !== false;
+diff -Naur a/modules/addons/XPIProvider.jsm b/modules/addons/XPIProvider.jsm
+--- a/modules/addons/XPIProvider.jsm 2000-01-01 00:00:00.000000000 +0000
++++ b/modules/addons/XPIProvider.jsm 2000-01-01 00:00:00.000000000 +0000
+@@ -749,7 +749,9 @@
+   if (aAddon.id == "torbutton@torproject.org" ||
+       aAddon.id == "tor-launcher@torproject.org" ||
+       aAddon.id == "https-everywhere-eff@eff.org" ||
+-      aAddon.id == "meek-http-helper@bamsoftware.com") {
++      aAddon.id == "meek-http-helper@bamsoftware.com" ||
++      aAddon.id == "branding@amnesia.boum.org" ||
++      aAddon.id == "uBlock0@raymondhill.net") {
+     return true;
+   }
+EOF
+        7z u -tzip "${TBB_INSTALL}/omni.ja" \
+           modules/addons/XPIProvider.jsm \
+           chrome/toolkit/content/mozapps/extensions/extensions.js
+
+        7z x -tzip "${TBB_INSTALL}/browser/omni.ja" \
+           components/nsBrowserGlue.js
+        patch -p1 <<EOF
+diff -Naur x/components/nsBrowserGlue.js y/components/nsBrowserGlue.js
+--- a/components/nsBrowserGlue.js 2000-01-01 00:00:00.000000000 +0000
++++ b/components/nsBrowserGlue.js 2000-01-01 00:00:00.000000000 +0000
+@@ -1122,7 +1122,9 @@
+           if ((addon.signedState <= AddonManager.SIGNEDSTATE_MISSING) &&
+               !(addon.id == "torbutton@torproject.org" ||
+                 addon.id == "tor-launcher@torproject.org" ||
+-                addon.id == "https-everywhere-eff@eff.org")) {
++                addon.id == "https-everywhere-eff@eff.org" ||
++                addon.id == "branding@amnesia.boum.org" ||
++                addon.id == "uBlock0@raymondhill.net")) {
+             this._notifyUnsignedAddonsDisabled();
+             break;
+           }
+EOF
+        7z u -tzip "${TBB_INSTALL}/browser/omni.ja" \
+           components/nsBrowserGlue.js
+        # These binaries are generated from the above modified files
+        # so we have to remove them. This will have a performance
+        # impact that probably is unnoticeable for humans, but TBB 7.5
+        # won't ship any of these binaries any way, so we'll converge.
+        7z d -tzip "${TBB_INSTALL}/omni.ja" \
+           jsloader/resource/gre/modules/addons/XPIProvider.jsm
+        7z d -tzip "${TBB_INSTALL}/browser/omni.ja" \
+           jsloader/resource/app/components/nsBrowserGlue.js
+    )
+}
+
 install_langpacks_from_bundles() {
     local bundles_dir destination
     bundles_dir="${1}"
@@ -164,7 +249,14 @@ TBB_TARBALLS="$(grep "\<tor-browser-linux64-.*\.tar.xz$" "${TBB_SHA256SUMS_FILE}
 
 # We'll use the en-US bundle as our basis; only langpacks will be
 # installed from the other bundles.
-MAIN_TARBALL="$(echo "${TBB_TARBALLS}" | grep -o "tor-browser-linux64-.*_en-US.tar.xz")"
+MAIN_TARBALL="$(echo "${TBB_TARBALLS}" | grep -o "tor-browser-linux64-.*_en-US.tar.xz" || :)"
+NIGHTLY_BUILD=
+if [ -z "${MAIN_TARBALL}" ] && [ "$(echo $TBB_TARBALLS | awk '{ print $2 }')" = 'tor-browser-linux64-tbb-nightly_ALL.tar.xz' ]; then
+    # Except for TBB nightly builds; then there is only one bundle
+    # containing all langpacks
+    MAIN_TARBALL='tor-browser-linux64-tbb-nightly_ALL.tar.xz'
+    NIGHTLY_BUILD=yes
+fi
 TBB_DIST_URL_FILE=/usr/share/tails/tbb-dist-url.txt
 TBB_TARBALLS_BASE_URL="$(cat "${TBB_DIST_URL_FILE}")"
 
@@ -176,9 +268,12 @@ TMP="$(mktemp -d)"
 download_and_verify_files "${TBB_TARBALLS_BASE_URL}" "${TBB_TARBALLS}" "${TMP}"
 
 install_tor_browser "${TMP}/${MAIN_TARBALL}" "${TBB_INSTALL}"
+apply_extension_code_signing_hacks "${TBB_INSTALL}"
 
 mkdir -p "${TBB_EXT}"
-install_langpacks_from_bundles "${TMP}" "${TBB_EXT}"
+if [ "${NIGHTLY_BUILD}" != yes ]; then
+    install_langpacks_from_bundles "${TMP}" "${TBB_EXT}"
+fi
 
 rm -r "${TMP}"
 

config/chroot_local-hooks/11-localize_browser

@@ -14,6 +14,9 @@ echo "Localize each supported browser locale"
 # Import language_code_from_locale()
 . /usr/local/lib/tails-shell-library/localization.sh
 
+# Import strip_nondeterminism_wrapper()
+. /usr/local/lib/tails-shell-library/build.sh
+
 # Import TAILS_WIKI_SUPPORTED_LANGUAGES
 . /etc/amnesia/environment
 
@@ -162,17 +165,24 @@ rm -r "${BROWSER_LOCALIZATION_DIR}"
 # present, otherwise they won't work. It's not a problem to list
 # nonexisting ones, so as long as we delete plugins we do not have to
 # alter it.
-7z d -tzip "${TBB_INSTALL}/browser/omni.ja" \
+7z d -mtc=off -tzip "${TBB_INSTALL}/browser/omni.ja" \
    'chrome/en-US/locale/browser/searchplugins/ddg*.xml' \
    'chrome/en-US/locale/browser/searchplugins/startpage*.xml' \
    'chrome/en-US/locale/browser/searchplugins/wikipedia*.xml' \
    'chrome/en-US/locale/browser/searchplugins/yahoo*.xml'
+# For consistency, fixup the internal timestamps of these archives with
+# the same ones used by the Tor Browser instead of SOURCE_DATE_EPOCH.
+tbb_timestamp="$(date --date='2000-01-01 00:00:00' +%s)"
+strip_nondeterminism_wrapper --type zip --timestamp "${tbb_timestamp}" \
+                             "${TBB_INSTALL}/browser/omni.ja" 2>/dev/null
 for pack in "${TBB_EXT}"/langpack-*.xpi; do
-    7z d -tzip "${pack}" \
+    7z d -mtc=off -tzip "${pack}" \
        'browser/chrome/*/locale/browser/searchplugins/ddg*.xml' \
        'browser/chrome/*/locale/browser/searchplugins/startpage*.xml' \
        'browser/chrome/*/locale/browser/searchplugins/wikipedia*.xml' \
        'browser/chrome/*/locale/browser/searchplugins/yahoo*.xml'
+    strip_nondeterminism_wrapper --type zip --timestamp "${tbb_timestamp}" \
+                                 "${pack}" 2>/dev/null
 done
 
 # We want our localized English Wikipedia plugin to be available in

config/chroot_local-hooks/52-update-rc.d

@@ -16,6 +16,7 @@ systemctl enable tails-shutdown-on-media-removal.service
 systemctl enable tails-tor-has-bootstrapped.target
 systemctl enable tails-wait-until-tor-has-bootstrapped.service
 systemctl enable tails-tor-has-bootstrapped-flag-file.service
+systemctl enable update-ca-certificates.service
 systemctl enable var-tmp.mount
 
 # Enable our own systemd user unit files

config/chroot_local-hooks/55-create-tails-keyring

-#!/bin/sh
-
-set -e
-
-echo "Creating system-wide Tails GnuPG keyring"
-
-gpg --batch --no-default-keyring --keyring /usr/share/keyrings/tails-keyring.gpg --import /usr/share/doc/tails/website/*.key
-chmod a+r /usr/share/keyrings/tails-keyring.gpg

config/chroot_local-hooks/80-block-network

@@ -4,7 +4,43 @@ set -e
 
 echo "Generating blocklist for all network devices"
 
-find /lib/modules/*/kernel/drivers/net \
-     -name "*.ko" -printf "install %f /bin/true\n" | \
-  sed 's/\.ko / /' | \
-  sort -u > /etc/modprobe.d/all-net-blacklist.conf
+is_net_module() {
+    # Here we assume that if any of the patterns below are matched, it
+    # is a network driver. This is not comprehensive, but should be
+    # enough for the staging directory (worst case we blacklist some
+    # shitty non-network driver by mistake).
+    /sbin/modinfo "${1}" | \
+        grep -q --extended-regexp \
+             -e "^depends:\s*(cfg|lib|mac)80211" \
+             -e "^parm:\s*ifname:"
+}
+
+net_module_filter() {
+    local path
+    while read path; do
+        if is_net_module "${path}"; then
+            echo "${path}"
+        fi
+    done
+}
+
+generate_blocking_line() {
+    local name
+    local path
+    while read path; do
+        name="$(basename "${path}" .ko)"
+        printf "install ${name} /bin/true\n"
+    done
+}
+
+BLACKLIST=/etc/modprobe.d/all-net-blacklist.conf
+
+(
+    find /lib/modules/*/kernel/drivers/net -name "*.ko" | \
+        generate_blocking_line && \
+
+    # Let's try to find the network drivers in the staging directory as well
+    find /lib/modules/*/kernel/drivers/staging/ -name "*.ko" | \
+        net_module_filter | \
+        generate_blocking_line
+) | sort -u > "${BLACKLIST}"

config/chroot_local-hooks/99-zzzzzz_reproducible-builds-post-processing

+#! /bin/sh
+
+set -e
+
+echo "Post processing filesystem to make it reproducible"
+
+if [ -z "${SOURCE_DATE_EPOCH}" ]; then
+    echo "SOURCE_DATE_EPOCH was not set!" >&2
+    exit 1
+fi
+
+# These files are pretty useless for us and mainly occupy space on the
+# image. They are, for instance, not useful for checking the
+# authenticity of the filesystem (an external verification tool and
+# source of these checksums would be required), and checking for
+# corruption is less relevant in Tails' context, where the system
+# partition is read-only (the point being: if they do differ, chances
+# are problems would manifest in much more obvious ways).
+rm /var/lib/dpkg/info/*.md5sums
+
+# Clear caches and remove precompiled code. These will be generated
+# on-the-fly when needed instead of being shipped on the image, so
+# we'll require a bit more RAM and startup times, while the image will
+# be smaller (and more reproducible!).
+rm /etc/console-setup/cached_setup_keyboard.sh
+rm /var/cache/ldconfig/aux-cache
+rm /var/lib/systemd/catalog/database
+
+# Delete non-deterministically generated files, that should not be shared among
+# all Tails systems anyway. We don't ship SSHd, so we don't bother generating
+# them at boot.
+rm -r /var/lib/monkeysphere/authentication/
+
+# Empty non-deterministically generated file. If it exists and is empty, systemd
+# will automatically set up a new unique ID. But if does not exist, systemd
+# will populate /etc with preset unit settings, which will for example re-enable
+# units we have disabled (#11970).
+: > /etc/machine-id
+
+# Remove logs.
+rm -r /var/lib/dkms/*/*/*/*/log
+
+# Set various timestamps according to SOURCE_DATE_EPOCH.
+find / -name '%gconf-tree.xml' -print0 | \
+    xargs -0r \
+        sed -i -e 's@\bmtime="[0-9][0-9]*"@mtime="'${SOURCE_DATE_EPOCH}'"@g'
+
+# Post-process /etc/shadow by setting the sp_lstchg field to the number of days
+# since SOURCE_DATE_EPOCH instead of 1st Jan 1970. (#12339)
+# XXX:Buster: drop this if https://bugs.debian.org/857803 is fixed.
+cut -d: -f1 /etc/shadow | \
+    xargs -L1 \
+        chage --lastday \
+            "$(($(date --utc --date "@${SOURCE_DATE_EPOCH}" "+%s") / 86400))"

config/chroot_local-includes/etc/NetworkManager/dispatcher.d/10-tor.sh

@@ -43,9 +43,7 @@ systemctl --no-block restart tails-tor-has-bootstrapped.target
 # a HTTP proxy or allowed firewall ports won't get the sandboxing, but
 # much better than nothing.
 if [ "$(tails_netconf)" = "direct" ]; then
-    # https://trac.torproject.org/projects/tor/ticket/21943
-    # tor_set_in_torrc Sandbox 1
-    true
+    tor_set_in_torrc Sandbox 1
 fi
 
 # We would like Tor to be started during init time, even before the

config/chroot_local-includes/etc/NetworkManager/dispatcher.d/20-time.sh

@@ -28,7 +28,6 @@ TOR_UNVERIFIED_CONSENSUS=${TOR_DIR}/unverified-microdesc-consensus
 TOR_UNVERIFIED_CONSENSUS_HARDLINK=${TOR_UNVERIFIED_CONSENSUS}.bak
 INOTIFY_TIMEOUT=60
 DATE_RE='[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]'
-VERSION_FILE=/etc/amnesia/version
 
 ### Exit conditions
 

config/chroot_local-includes/etc/default/htpdate.pools

-HTP_POOL_PAL="boum.org,espiv.net,db.debian.org,epic.org,mail.riseup.net,www.sarava.org,squat.net,tachanka.org,www.1984.is,www.eff.org,www.immerda.ch,www.privacyinternational.org,www.torproject.org"
+HTP_POOL_PAL="boum.org,espiv.net,db.debian.org,epic.org,mail.riseup.net,leap.se,squat.net,tachanka.org,www.1984.is,www.eff.org,www.immerda.ch,www.privacyinternational.org,www.torproject.org"
 HTP_POOL_NEUTRAL="cve.mitre.org,en.wikipedia.org,lkml.org,thepiratebay.org,www.apache.org,www.centos.org,www.democracynow.org,www.duckduckgo.com,www.gnu.org,www.kernel.org,www.mozilla.org,www.stackexchange.com,www.startpage.com,www.xkcd.com"
 HTP_POOL_FOE="encrypted.google.com,github.com,login.live.com,login.yahoo.com,secure.flickr.com,tumblr.com,twitter.com,www.adobe.com,www.gandi.net,www.myspace.com,www.paypal.com,www.rackspace.com,www.sony.com"

config/chroot_local-includes/etc/tor-browser/profile/preferences/0000tails.js

@@ -91,7 +91,6 @@ pref("browser.download.panel.shown", true);
 // and instead only propose them to save downloaded files.
 pref("browser.download.forbid_open_with", true);
 
-// uBlock Origin and the amnesia branding extensions are not signed and
-// therefore disabled by default for FF 45+ *unless* we set this
-// option. This is only a temporary stop gap. See #11419.
-pref("xpinstall.signatures.required", false);
+// Web pages does not render when e10s is enabled, so we have to
+// disable it. Note that the "user_"-prefix is required.
+user_pref("browser.tabs.remote.autostart.2", false);

config/chroot_local-includes/etc/tor/torrc

@@ -174,3 +174,8 @@ WarnUnsafeSocks 0
 ## Disable default warnings on StartTLS for email. Let's not train our