Merge remote-tracking branch 'origin/master' into web/16128-verification-on-page

.gitignore

@@ -57,12 +57,16 @@
 /config/chroot_local-includes/usr/share/applications/tails-shutdown.desktop
 /config/chroot_local-includes/usr/share/applications/tor-browser.desktop
 /config/chroot_local-includes/usr/share/applications/tails-about.desktop
+/config/chroot_local-includes/usr/share/applications/tails-installer.desktop
 /config/chroot_local-includes/usr/share/applications/unlock-veracrypt-volumes.desktop
+/config/chroot_local-includes/usr/share/applications/whisperback.desktop
 /config/chroot_local-includes/usr/share/desktop-directories/Tails.directory
 /config/chroot_local-includes/usr/share/polkit-1/actions/org.boum.tails.root-terminal.policy
 /config/chroot_local-includes/usr/share/polkit-1/actions/org.boum.tails.additional-software.policy
 /config/chroot_local-includes/usr/share/tails/greeter/*.ui
+/config/chroot_local-includes/usr/share/tails-installer/*.ui
 /config/chroot_local-includes/usr/share/tails/unlock-veracrypt-volumes/*.ui
+/config/chroot_local-includes/usr/share/whisperback/*.ui
 /tmp/
 
 # The test suite's local configuration files

.gitmodules

@@ -11,3 +11,7 @@
 [submodule "submodules/tails-workarounds"]
 	path = submodules/tails-workarounds
 	url = https://gitlab.tails.boum.org/tails/workarounds.git
+[submodule "submodules/sof"]
+	path = submodules/sof
+	url = https://github.com/thesofproject/sof-bin.git
+	branch = stable-v1.5.1

auto/build

@@ -3,6 +3,7 @@
 set -e
 set -u
 set -x
+set -o pipefail
 
 . "$(dirname $0)/scripts/utils.sh"
 
@@ -59,6 +60,12 @@ echo "POTFILES_DOT_IN='$(
        | sed -e 's,^config/chroot_local-includes,,' | tr "\n" ' '
    )'" \
    >> config/chroot_local-includes/usr/share/tails/build/variables
+echo "SOF_VERSION='$(
+         git -C submodules/sof branch --all --contains HEAD \
+             --format '%(refname:short)' 'origin/stable-v*' \
+       | cut -d"-" -f 2
+   )'" \
+   >> config/chroot_local-includes/usr/share/tails/build/variables
 
 # fix permissions on some source files that will be copied as is to the chroot.
 # they may be wrong, e.g. if the Git repository was cloned with a strict umask.
@@ -146,13 +153,11 @@ BUILD_USB_IMAGE_FILENAME="${BUILD_BASENAME}.img"
    cat config/chroot_sources/*.chroot
 ) > "$BUILD_APT_SOURCES"
 
-# make workarounds available in the chroot, if any:
-WORKAROUNDS_SRC="submodules/tails-workarounds"
-WORKAROUNDS_DST="config/chroot_local-includes/tmp/"
-if [ -d "$WORKAROUNDS_SRC" ]; then
-  mkdir -p "$WORKAROUNDS_DST"
-  cp -a "$WORKAROUNDS_SRC" "$WORKAROUNDS_DST"
-fi
+# make submodules available in the chroot:
+SUBMODULES_SRC="submodules/sof submodules/tails-workarounds"
+SUBMODULES_DST="config/chroot_local-includes/tmp/submodules"
+mkdir -p "$SUBMODULES_DST"
+cp -a $SUBMODULES_SRC "$SUBMODULES_DST"/
 
 echo "I: Building ISO image ${BUILD_ISO_FILENAME}..."
 time lb build noauto "${@}"

bin/add-release-to-iso-history

+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+NAME=$(basename "${0}")
+LONGOPTS="version:,isos:,release-branch:,matching-jenkins-images-build-id:"
+OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
+eval set -- "$OPTS"
+while [ $# -gt 0 ]; do
+    case $1 in
+        --version)
+	    shift
+            VERSION="$1"
+            ;;
+	--isos)
+	    shift
+	    ISOS="$1"
+	    ;;
+	--release-branch)
+	    shift
+	    RELEASE_BRANCH="$1"
+	    ;;
+	--matching-jenkins-images-build-id)
+	    shift
+	    MATCHING_JENKINS_IMAGES_BUILD_ID="$1"
+	    ;;
+    esac
+    shift
+done
+
+ssh misc.lizard mkdir "tails-amd64-${VERSION:?}"
+
+scp "${ISOS:?}/tails-amd64-${VERSION:?}/tails-amd64-${VERSION:?}."{apt-sources,build-manifest,buildlog,packages,iso.sig,img.sig} \
+    "misc.lizard:tails-amd64-${VERSION:?}"
+
+ssh misc.lizard gpg --import < "wiki/src/tails-signing.key"
+
+ssh misc.lizard << EOF
+   cd tails-amd64-${VERSION:?} && \
+   wget --quiet \
+      "https://nightly.tails.boum.org/build_Tails_ISO_${RELEASE_BRANCH:?}/builds/${MATCHING_JENKINS_IMAGES_BUILD_ID:?}/archive/build-artifacts/tails-amd64-${VERSION:?}.iso" \
+      "https://nightly.tails.boum.org/build_Tails_ISO_${RELEASE_BRANCH:?}/builds/${MATCHING_JENKINS_IMAGES_BUILD_ID:?}/archive/build-artifacts/tails-amd64-${VERSION:?}.img" && \
+   gpg --verify tails-amd64-${VERSION:?}.iso{.sig,} && \
+   gpg --verify tails-amd64-${VERSION:?}.img{.sig,}
+EOF
+
+ssh misc.lizard << EOF
+   ( [ -d isos ] || git clone gitolite@puppet-git.lizard:isos.git ) && \
+   cd isos && \
+   git annex init && \
+   git annex sync && \
+   git annex import ../tails-amd64-${VERSION:?} && \
+   rmdir ../tails-amd64-${VERSION:?} && \
+   git commit -m "Add Tails ${VERSION:?}" && \
+   git annex sync && \
+   git annex copy tails-amd64-${VERSION:?} --to origin && \
+   git annex drop tails-amd64-${VERSION:?} && \
+   git annex sync
+EOF

bin/apt-snapshots-expiry

+#!/bin/sh
+
+set -eu
+
+for dir in config/APT_snapshots.d vagrant/definitions/tails-builder/config/APT_snapshots.d; do
+(
+    set -eu
+    echo "${dir:?}:"
+    cd "${dir:?}"
+    for ARCHIVE in * ; do
+        SERIAL="$(cat ${ARCHIVE:?}/serial)"
+        if [ "${SERIAL:?}" = 'latest' ]; then
+            EXPIRY='never'
+            if [ "${ARCHIVE:?}" != 'debian-security' ]; then
+                echo "Warning: origin '${ARCHIVE:?}' is using the 'latest' snapshot, which is unexpected" >&2
+            fi
+        else
+            case "${ARCHIVE:?}" in
+                'debian-security')
+                    DIST='buster/updates'
+                    ;;
+                'torproject')
+                    DIST='buster'
+                    ;;
+                *)
+                    DIST='stable'
+                    ;;
+            esac
+            EXPIRY="$(curl --silent "https://time-based.snapshots.deb.tails.boum.org/${ARCHIVE:?}/dists/${DIST:?}/snapshots/${SERIAL:?}/Release" | sed -n 's/^Valid-Until:\s\+\(.*\)$/\1/p')"
+        fi
+        echo "* Archive '${ARCHIVE:?}' uses snapshot '${SERIAL:?}' which expires on: ${EXPIRY:?}"
+    done
+    echo ---
+)
+done

bin/copy-images-to-rsync-server-and-verify

+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+NAME=$(basename "${0}")
+LONGOPTS="version:,dist:,release-branch:,matching-jenkins-images-build-id:"
+OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
+eval set -- "$OPTS"
+while [ $# -gt 0 ]; do
+    case $1 in
+        --version)
+	    shift
+            VERSION="$1"
+            ;;
+	--dist)
+	    shift
+	    DIST="$1"
+	    ;;
+	--release-branch)
+	    shift
+	    RELEASE_BRANCH="$1"
+	    ;;
+	--matching-jenkins-images-build-id)
+	    shift
+	    MATCHING_JENKINS_IMAGES_BUILD_ID="$1"
+	    ;;
+    esac
+    shift
+done
+
+ssh rsync.lizard gpg --import < wiki/src/tails-signing.key
+
+ssh rsync.lizard << EOF
+   wget --quiet \
+      "https://nightly.tails.boum.org/build_Tails_ISO_${RELEASE_BRANCH:?}/builds/${MATCHING_JENKINS_IMAGES_BUILD_ID:?}/archive/build-artifacts/tails-amd64-${VERSION:?}.iso" \
+      "https://nightly.tails.boum.org/build_Tails_ISO_${RELEASE_BRANCH:?}/builds/${MATCHING_JENKINS_IMAGES_BUILD_ID:?}/archive/build-artifacts/tails-amd64-${VERSION:?}.img" && \
+   gpg --verify tails-amd64-${VERSION:?}.iso{.sig,} && \
+   gpg --verify tails-amd64-${VERSION:?}.img{.sig,}
+EOF
+
+ssh rsync.lizard << EOF
+  sudo install -o root -g rsync_tails -m 0755 -d \
+     /srv/rsync/tails/tails/${DIST:?}/tails-amd64-${VERSION:?} && \
+  sudo chown root:rsync_tails tails-amd64-${VERSION:?}.{iso,img}* && \
+  sudo chmod u=rwX,go=rX tails-amd64-${VERSION:?}.{iso,img}* && \
+  sudo mv tails-amd64-${VERSION:?}.{iso,img}* \
+          /srv/rsync/tails/tails/${DIST:?}/tails-amd64-${VERSION:?}
+EOF

bin/copy-iuks-to-rsync-server-and-verify

@@ -92,41 +92,6 @@ def download_iuks_from_jenkins(
         destdir: str,
         jenkins_iuks_base_url: str,
         jenkins_build_id: int) -> None:
-
-    # This assumes same basename for hashes, locally and in Jenkins:
-    log.info("Downloading IUK hashes (if available) from Jenkins to %s…" % (desthost))
-    try:
-        url = "%s/%s/archive/%s" % (
-            jenkins_iuks_base_url,
-            jenkins_build_id,
-            Path(hashes_file).name
-        )
-        jenkins_hashes = '%(d)s/%(f)s' % {
-            "d": destdir,
-            "f": '%s.jenkins' % Path(hashes_file).name
-        }
-        our_hashes = '%(d)s/%(f)s' % {
-            "d": destdir,
-            "f": Path(hashes_file).name,
-        }
-        subprocess.run(
-            ["ssh", desthost, "wget", "--quiet", "--no-clobber",
-             "-O", jenkins_hashes, url],
-            check=True
-        )
-        subprocess.run(
-            ["ssh", desthost,
-             "sh -c \"if ! cmp -s '%(j_h)s' '%(o_h)s'; then "
-             "echo 'WARNING: IUK hashes seem different'; else "
-             "echo 'OK: IUK hashes seem similar'; fi\"" % {
-                 "j_h": jenkins_hashes,
-                 "o_h": our_hashes,
-             }],
-            check=True
-        )
-    except subprocess.CalledProcessError:
-        log.error("Unable to download/validate IUK hashes from Jenkins")
-
     log.info("Downloading IUKs from Jenkins to %s…" % (desthost))
     iuks = iuks_listed_in(hashes_file)
     log.debug("IUKS: %s" % ', '.join(iuks))

bin/generate-call-for-testing

+#! /usr/bin/python3
+
+import email.utils
+import subprocess
+
+from datetime import datetime, timedelta
+import jinja2
+
+
+def feedback_deadline(final_date: datetime) -> datetime:
+    return final_date - timedelta(days=2)
+
+
+def call_for_testing_contents(args) -> str:
+    jinja2_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader('config/release_management/templates'))
+
+    return (jinja2_env.get_template('call_for_testing.mdwn').render(
+        date=email.utils.format_datetime(datetime.fromisoformat(args.date)),
+        version=args.version,
+        tag=args.tag,
+        final_date=datetime.fromisoformat(args.final_date).strftime("%B %d"),
+        final_version=args.final_version,
+        deadline=feedback_deadline(datetime.fromisoformat(
+            args.final_date)).strftime("%B %d")))
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--version', required=True)
+    parser.add_argument('--tag', required=True)
+    parser.add_argument('--date', required=True)
+    parser.add_argument('--final-version', required=True)
+    parser.add_argument('--final-date', required=True)
+    args = parser.parse_args()
+
+    print(call_for_testing_contents(args))

bin/generate-changelog

@@ -24,9 +24,7 @@ GROUP_NAME = 'tails'
 PROJECTS = [
     GROUP_NAME + '/' + project for project in [
         'chutney',
-        'installer',
         'tails',
-        'whisperback',
         'workarounds',
     ]
 ]

bin/generate-report

+#! /usr/bin/python3
+
+import functools
+import gitlab
+import logging
+import os
+
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+from pathlib import Path
+
+PYTHON_GITLAB_CONFIG_FILE = os.getenv('PYTHON_GITLAB_CONFIG_FILE',
+                                      default=Path.home() /
+                                      '.python-gitlab.cfg')
+
+PYTHON_GITLAB_NAME = os.getenv('GITLAB_NAME', default='Tails')
+
+GROUP_NAME = 'tails'
+
+# By default, only changes in these projects are considered
+PROJECTS = [
+    GROUP_NAME + '/' + project for project in [
+        'chutney',
+        'installer',
+        'tails',
+        'whisperback',
+        'workarounds',
+    ]
+]
+
+LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
+log = logging.getLogger()
+
+
+class GitLabWrapper(gitlab.Gitlab):
+    @functools.lru_cache
+    def project(self, project_id):
+        return self.projects.get(project_id)
+
+    @functools.lru_cache
+    def project_from_name(self, project_name):
+        project = [
+            p for p in self.projects.list(all=True)
+            # Disambiguate between projects whose names share a common prefix
+            if p.path_with_namespace == project_name
+        ][0]
+        assert isinstance(project, gitlab.v4.objects.Project)
+        return project
+
+
+class ReportGenerator(object):
+    def __init__(self, gl, group, projects: list, label: str, year: int,
+                 month: int):
+        self.gl = gl
+        self.group = group
+        self.projects = projects
+        self.label = label
+        self.after = end_of_previous_month(year, month)
+        self.before = beginning_of_next_month(year, month)
+
+    def closed_issues_in_project(self, project_name) -> list:
+        closed_issues = []
+        project = self.gl.project_from_name(project_name)
+        closed_issues_events = project.events.list(as_list=False,
+                                                   target_type='issue',
+                                                   action='closed',
+                                                   after=self.after,
+                                                   before=self.before)
+
+        gl_closed_issues = [{
+            "project_id": event.project_id,
+            "iid": event.target_iid
+        } for event in closed_issues_events]
+
+        for issue in gl_closed_issues:
+            project = self.gl.project(issue["project_id"])
+            issue = project.issues.get(issue["iid"])
+            if self.label is not None and self.label not in issue.labels:
+                continue
+            closed_issues.append({
+                "title": issue.title,
+                "web_url": issue.web_url,
+            })
+
+        return closed_issues
+
+    def closed_issues(self) -> list:
+        closed_issues = []
+        for project in self.projects:
+            closed_issues = closed_issues + self.closed_issues_in_project(
+                project)
+        return closed_issues
+
+
+def beginning_of_next_month(year, month):
+    return (datetime(year, month, 1) + relativedelta(months=1)).replace(day=1)
+
+
+def end_of_previous_month(year, month):
+    return datetime(year, month, 1) + relativedelta(seconds=-1)
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--year', type=int, required=True)
+    parser.add_argument('--month', type=int, required=True)
+    parser.add_argument('--label', default=None)
+    parser.add_argument('--project')
+    parser.add_argument("--debug", action="store_true", help="debug output")
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
+    else:
+        logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+
+    gl = GitLabWrapper.from_config(PYTHON_GITLAB_NAME,
+                                   config_files=[PYTHON_GITLAB_CONFIG_FILE])
+    gl.auth()
+
+    group = gl.groups.list(search=GROUP_NAME)[0]
+    assert isinstance(group, gitlab.v4.objects.Group)
+
+    if args.project:
+        projects = [args.project]
+    else:
+        projects = PROJECTS
+
+    report_generator = ReportGenerator(gl, group, projects, args.label,
+                                       args.year, args.month)
+
+    print("Closed issues")
+    print("=============")
+    print()
+    for closed_issue in report_generator.closed_issues():
+        print(f'- {closed_issue["title"]}')
+        print(f'  {closed_issue["web_url"]}')
+        print()

bin/wait-for-new-mfsa

+#!/bin/sh
+
+set -u
+
+current_mfsa() {
+    local current
+    current="$(
+        torsocks --isolate curl --silent https://www.mozilla.org/en-US/security/advisories/ | \
+           sed --regexp-extended -n 's@.*<a href="/en-US/security/advisories/(mfsa[0-9]+-[0-9]+)/".*>@\1@p' | \
+           sort -n | \
+           tail -n 1
+    )"
+    echo "$(date --rfc-3339=s): got ${current}" >&2
+    echo "${current}"
+}
+
+initial="$(current_mfsa)"
+while true; do
+    new="$(current_mfsa)"
+    [ -n "${new}" ] || continue
+    if [ "${new}" != "${initial}" ]; then
+	echo "${new}"
+	exit 0
+    fi
+    sleep 60
+done

config/APT_snapshots.d/debian/serial

-2020081601
+2020101002

config/APT_snapshots.d/torproject/serial

-2020073001
+2020091901

config/amnesia

@@ -17,13 +17,13 @@ export SOURCE_DATE_FAKETIME="$(date --utc --date="$(dpkg-parsechangelog --show-f
 # Base for the string that will be passed to "lb config --bootappend-live"
 # FIXME: see [[bugs/sdmem_on_eject_broken_for_CD]] for explanation why we
 # need to set block.events_dfl_poll_msecs
-AMNESIA_APPEND="live-media=removable nopersistence noprompt timezone=Etc/UTC block.events_dfl_poll_msecs=1000 splash noautologin module=Tails slab_nomerge slub_debug=FZP mce=0 vsyscall=none page_poison=1 init_on_alloc=1 init_on_free=1 mds=full,nosmt"
+AMNESIA_APPEND="live-media=removable nopersistence noprompt timezone=Etc/UTC block.events_dfl_poll_msecs=1000 splash noautologin module=Tails slab_nomerge slub_debug=FZP mce=0 vsyscall=none page_poison=1 init_on_free=1 mds=full,nosmt"
 
 # Options passed to isohybrid
 AMNESIA_ISOHYBRID_OPTS="-h 255 -s 63 --id 42 --verbose"
 
 # Kernel version
-KERNEL_VERSION='5.7.0-2'
+KERNEL_VERSION='5.8.0-2'
 KERNEL_SOURCE_VERSION=$(
    echo "$KERNEL_VERSION" \
        | perl -p -E 's{\A (\d+ [.] \d+) [.] .*}{$1}xms'

config/chroot_apt/preferences

@@ -18,12 +18,12 @@ Pin-Priority: 990
 
 Explanation: Electrum 4.0.2 and recent TREZOR firmware need 0.12
 Package: python3-trezor trezor
-Pin: release o=Debian,n=bullseye
+Pin: release o=Debian,n=buster-backports
 Pin-Priority: 999
 
 Explanation: python3-trezor needs a version newer than the one in Buster
-Package: python3-usb1
-Pin: release o=Debian,n=bullseye
+Package: python3-construct
+Pin: release o=Debian,n=buster-backports
 Pin-Priority: 999
 
 Package: firmware-b43-installer
@@ -52,6 +52,12 @@ Package: grub*
 Pin: release o=Debian,n=bullseye
 Pin-Priority: 999
 
+Explanation: We want to set default database directory to ~/Persistent, which
+Explanation: is only possible since 2.4.0, which is unavailable in Buster.
+Package: keepassxc
+Pin: release o=Debian,n=buster-backports
+Pin-Priority: 999
+
 Package: linux-compiler-* linux-headers-* linux-image-* linux-kbuild-* linux-source-*
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
@@ -73,7 +79,8 @@ Package: squashfs-tools
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
 
-Package: tails-installer
+Explanation: install Thunderbird 68 until we're ready for 78 (#17962)
+Package: calendar-google-provider lightning* thunderbird*
 Pin: origin deb.tails.boum.org
 Pin-Priority: 999
 
@@ -81,7 +88,7 @@ Package: virtualbox*
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
 
-Package: webext-ublock-origin
+Package: webext-ublock-origin-firefox
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
 

config/chroot_local-hooks/10-tbb

@@ -74,9 +74,7 @@ install_tor_browser() {
     # instead of the system one, whenever ours is too old.
     # For details see projects/firefox/abicheck.cc in
     # https://git.torproject.org/builders/tor-browser-build.git
-    # Tor Browser 9.0a7 requires GLIBCXX_3.4.25, which Buster has,
-    # so disable this for now.
-    # cp "${prep}"/TorBrowser/Tor/libstdc++/libstdc++.so.6 "${prep}"
+    cp "${prep}"/TorBrowser/Tor/libstdc++/libstdc++.so.6 "${prep}"
 
     # We don't need the Tor binary, the shared libraries Tor needs
     # (but Firefox doesn't) and documentation shipped in the TBB.
@@ -141,11 +139,8 @@ EOF
     rm -r "${tmp}"
 }
 
-# TBB works around the lack of code signing for its extensions by
-# hacking in exceptions. We do the same!
-# Improving this is tracked on #12571.
-apply_extension_code_signing_hacks () {
-    local tbb_install tbb_timestamp
+embed_extensions_in_omni_ja () {
+    local tbb_install tbb_timestamp tmp
     tbb_install="${1}"
     tbb_timestamp="${2}"
 
@@ -153,62 +148,8 @@ apply_extension_code_signing_hacks () {
     (
         cd "${tmp}"
         7z x -tzip "${tbb_install}/omni.ja"
-        patch -p1 <<EOF
-diff -Naur a/chrome/toolkit/content/mozapps/extensions/aboutaddonsCommon.js b/chrome/toolkit/content/mozapps/extensions/aboutaddonsCommon.js
---- a/chrome/toolkit/content/mozapps/extensions/aboutaddonsCommon.js	2019-09-02 15:24:00.000000000 +0200
-+++ b/chrome/toolkit/content/mozapps/extensions/aboutaddonsCommon.js	2019-09-08 20:42:24.198382292 +0200
-@@ -195,6 +195,10 @@
-   if (addon.id == "https-everywhere-eff@eff.org") {
-     return true;
-   }
-+  // Allow uBlock installed from Debian (Tails#12571)
-+  if (addon.id == "uBlock0@raymondhill.net") {
-+    return true;
-+  }
-   return addon.isCorrectlySigned !== false;
- }
- 
-diff -Naur a/modules/addons/XPIDatabase.jsm b/modules/addons/XPIDatabase.jsm
---- a/modules/addons/XPIDatabase.jsm	2019-09-02 15:24:00.000000000 +0200