Merge remote-tracking branch 'origin/web/release-4.12' into master

.gitignore

@@ -57,12 +57,16 @@
 /config/chroot_local-includes/usr/share/applications/tails-shutdown.desktop
 /config/chroot_local-includes/usr/share/applications/tor-browser.desktop
 /config/chroot_local-includes/usr/share/applications/tails-about.desktop
+/config/chroot_local-includes/usr/share/applications/tails-installer.desktop
 /config/chroot_local-includes/usr/share/applications/unlock-veracrypt-volumes.desktop
+/config/chroot_local-includes/usr/share/applications/whisperback.desktop
 /config/chroot_local-includes/usr/share/desktop-directories/Tails.directory
 /config/chroot_local-includes/usr/share/polkit-1/actions/org.boum.tails.root-terminal.policy
 /config/chroot_local-includes/usr/share/polkit-1/actions/org.boum.tails.additional-software.policy
 /config/chroot_local-includes/usr/share/tails/greeter/*.ui
+/config/chroot_local-includes/usr/share/tails-installer/*.ui
 /config/chroot_local-includes/usr/share/tails/unlock-veracrypt-volumes/*.ui
+/config/chroot_local-includes/usr/share/whisperback/*.ui
 /tmp/
 
 # The test suite's local configuration files

bin/copy-iuks-to-rsync-server-and-verify

@@ -92,41 +92,6 @@ def download_iuks_from_jenkins(
         destdir: str,
         jenkins_iuks_base_url: str,
         jenkins_build_id: int) -> None:
-
-    # This assumes same basename for hashes, locally and in Jenkins:
-    log.info("Downloading IUK hashes (if available) from Jenkins to %s…" % (desthost))
-    try:
-        url = "%s/%s/archive/%s" % (
-            jenkins_iuks_base_url,
-            jenkins_build_id,
-            Path(hashes_file).name
-        )
-        jenkins_hashes = '%(d)s/%(f)s' % {
-            "d": destdir,
-            "f": '%s.jenkins' % Path(hashes_file).name
-        }
-        our_hashes = '%(d)s/%(f)s' % {
-            "d": destdir,
-            "f": Path(hashes_file).name,
-        }
-        subprocess.run(
-            ["ssh", desthost, "wget", "--quiet", "--no-clobber",
-             "-O", jenkins_hashes, url],
-            check=True
-        )
-        subprocess.run(
-            ["ssh", desthost,
-             "sh -c \"if ! cmp -s '%(j_h)s' '%(o_h)s'; then "
-             "echo 'WARNING: IUK hashes seem different'; else "
-             "echo 'OK: IUK hashes seem similar'; fi\"" % {
-                 "j_h": jenkins_hashes,
-                 "o_h": our_hashes,
-             }],
-            check=True
-        )
-    except subprocess.CalledProcessError:
-        log.error("Unable to download/validate IUK hashes from Jenkins")
-
     log.info("Downloading IUKs from Jenkins to %s…" % (desthost))
     iuks = iuks_listed_in(hashes_file)
     log.debug("IUKS: %s" % ', '.join(iuks))

bin/generate-call-for-testing

+#! /usr/bin/python3
+
+import email.utils
+import subprocess
+
+from datetime import datetime, timedelta
+import jinja2
+
+
+def feedback_deadline(final_date: datetime) -> datetime:
+    return final_date - timedelta(days=2)
+
+
+def call_for_testing_contents(args) -> str:
+    jinja2_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader('config/release_management/templates'))
+
+    return (jinja2_env.get_template('call_for_testing.mdwn').render(
+        date=email.utils.format_datetime(datetime.fromisoformat(args.date)),
+        version=args.version,
+        tag=args.tag,
+        final_date=datetime.fromisoformat(args.final_date).strftime("%B %d"),
+        final_version=args.final_version,
+        deadline=feedback_deadline(datetime.fromisoformat(
+            args.final_date)).strftime("%B %d")))
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--version', required=True)
+    parser.add_argument('--tag', required=True)
+    parser.add_argument('--date', required=True)
+    parser.add_argument('--final-version', required=True)
+    parser.add_argument('--final-date', required=True)
+    args = parser.parse_args()
+
+    print(call_for_testing_contents(args))

bin/generate-changelog

@@ -24,9 +24,7 @@ GROUP_NAME = 'tails'
 PROJECTS = [
     GROUP_NAME + '/' + project for project in [
         'chutney',
-        'installer',
         'tails',
-        'whisperback',
         'workarounds',
     ]
 ]

config/APT_snapshots.d/debian/serial

-2020090702
+2020101002

config/APT_snapshots.d/torproject/serial

-2020090701
+2020091901

config/amnesia

@@ -17,13 +17,13 @@ export SOURCE_DATE_FAKETIME="$(date --utc --date="$(dpkg-parsechangelog --show-f
 # Base for the string that will be passed to "lb config --bootappend-live"
 # FIXME: see [[bugs/sdmem_on_eject_broken_for_CD]] for explanation why we
 # need to set block.events_dfl_poll_msecs
-AMNESIA_APPEND="live-media=removable nopersistence noprompt timezone=Etc/UTC block.events_dfl_poll_msecs=1000 splash noautologin module=Tails slab_nomerge slub_debug=FZP mce=0 vsyscall=none page_poison=1 init_on_alloc=1 init_on_free=1 mds=full,nosmt"
+AMNESIA_APPEND="live-media=removable nopersistence noprompt timezone=Etc/UTC block.events_dfl_poll_msecs=1000 splash noautologin module=Tails slab_nomerge slub_debug=FZP mce=0 vsyscall=none page_poison=1 init_on_free=1 mds=full,nosmt"
 
 # Options passed to isohybrid
 AMNESIA_ISOHYBRID_OPTS="-h 255 -s 63 --id 42 --verbose"
 
 # Kernel version
-KERNEL_VERSION='5.7.0-3'
+KERNEL_VERSION='5.8.0-2'
 KERNEL_SOURCE_VERSION=$(
    echo "$KERNEL_VERSION" \
        | perl -p -E 's{\A (\d+ [.] \d+) [.] .*}{$1}xms'

config/chroot_apt/preferences

@@ -79,7 +79,8 @@ Package: squashfs-tools
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
 
-Package: tails-installer
+Explanation: install Thunderbird 68 until we're ready for 78 (#17962)
+Package: calendar-google-provider lightning* thunderbird*
 Pin: origin deb.tails.boum.org
 Pin-Priority: 999
 
@@ -87,7 +88,7 @@ Package: virtualbox*
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
 
-Package: webext-ublock-origin
+Package: webext-ublock-origin-firefox
 Pin: release o=Debian,n=sid
 Pin-Priority: 999
 

config/chroot_local-hooks/10-tbb

@@ -88,44 +88,6 @@ install_tor_browser() {
     # Otherwise the "General" section in the preferences is not displayed.
     install -d -m 0755 "${prep}"/TorBrowser/UpdateInfo
 
-    # Apply 10.0-build2 → 10.0-build3 changes:
-    (
-        local tmp
-        tmp="$(mktemp -d)"
-        cd "${tmp}"
-        7z x -tzip "${prep}/browser/omni.ja"
-        # Any $ in the below in-line patch must be escaped!
-        patch -p1 <<EOF
-commit fb9428098b5b85eed400daa6e0010ac63faf8848 (tag: tor-browser-78.3.0esr-10.0-2-build2, origin/tor-browser-78.3.0esr-10.0-2)
-Author: Matthew Finkel <sysrqb@torproject.org>
-Date:   Sat Sep 19 17:03:53 2020 +0000
-
-    Revert "fixup! TB4: Tor Browser's Firefox preference overrides."
-    
-    This reverts commit c386fb3312237fd6c0d123ba9aaad662f8740e56.
-    
-    We continue using the old webextensions storage backend due to #40137.
-
-diff --git a/browser/app/profile/000-tor-browser.js b/browser/app/profile/000-tor-browser.js
-index bac98ce06540..7e29c788b720 100644
---- a/defaults/preferences/000-tor-browser.js
-+++ b/defaults/preferences/000-tor-browser.js
-@@ -286,6 +286,8 @@ pref("extensions.htmlaboutaddons.recommendations.enabled", false);
- pref("extensions.legacy.exceptions", "{972ce4c6-7e08-4474-a285-3208198ce6fd},torbutton@torproject.org");
- // Bug 26114: Allow NoScript to access addons.mozilla.org etc.
- pref("extensions.webextensions.restrictedDomains", "");
-+// Bug 31396: Disable indexedDB WebExtension storage backend.
-+pref("extensions.webextensions.ExtensionStorageIDB.enabled", false);
- // Bug 28896: Make sure our bundled WebExtensions are running in Private Browsing Mode
- pref("extensions.allowPrivateBrowsingByDefault", true);
- 
-EOF
-        touch --date="@${TBB_TIMESTAMP:?}" defaults/preferences/000-tor-browser.js
-        rm "${prep}/browser/omni.ja"
-        7z a -mtc=off -tzip "${prep}/browser/omni.ja" *
-        rm -r "${tmp}"
-    )
-    
     mv "${prep}" "${destination}"
     rm -r "${tmp}"
 }
@@ -325,7 +287,7 @@ install_debian_extensions() {
     fake_firefox_version=${firefox_version}+fake1
     install_fake_package firefox "${fake_firefox_version}" web
 
-    apt-get install --yes webext-ublock-origin
+    apt-get install --yes webext-ublock-origin-firefox
     patch -p1 < /usr/share/tails/uBlock-disable-autoUpdate.diff
 
     # Apply the same hack for our extension as the Tor Browser does
@@ -335,7 +297,7 @@ install_debian_extensions() {
     embed_extensions_in_omni_ja "${destination}" "${timestamp}"
     # ... and then remove the packages we just installed, since we
     # don't need them outside of omni.ja.
-    apt purge --yes firefox webext-ublock-origin
+    apt purge --yes firefox webext-ublock-origin-firefox
 }
 
 create_default_profile() {

config/chroot_local-hooks/12-kernel-modules-build-environment

@@ -11,17 +11,17 @@ echo "Setting up a build environment for kernel modules"
 # Import ensure_hook_dependency_is_installed()
 . /usr/local/lib/tails-shell-library/build.sh
 
-# Install gcc-8 and fake linux-compiler-gcc-9-x86
-# (linux-headers-5.3.0+ depends on it, but Buster hasn't GCC 9)
+# Install gcc-8 and fake linux-compiler-gcc-10-x86
+# (linux-headers-5.8.0+ depends on it, but Buster hasn't GCC 10)
 ensure_hook_dependency_is_installed gcc-8
 NEWEST_INSTALLED_KERNEL_VERSION="$(
     dpkg-query --showformat '${Version}\n' --show 'linux-image-*-amd64' \
     | sort --version-sort | tail -n1
 )"
 install_fake_package \
-    linux-compiler-gcc-9-x86 \
+    linux-compiler-gcc-10-x86 \
     "${NEWEST_INSTALLED_KERNEL_VERSION}~0tails1"
-ln -s /usr/bin/gcc-8 /usr/bin/gcc-9
+ln -s /usr/bin/gcc-8 /usr/bin/gcc-10
 
 ensure_hook_dependency_is_installed \
     build-essential \

config/chroot_local-includes/etc/sudoers.d/zzz_upgrade

@@ -11,5 +11,6 @@ tails-upgrade-frontend ALL = (tails-install-iuk)         NOPASSWD: /usr/local/bi
 tails-upgrade-frontend ALL = (tails-iuk-get-target-file) NOPASSWD: IUK_GET_TARGET_FILE
 tails-upgrade-frontend ALL = (tails-iuk-get-target-file) NOPASSWD: /usr/local/bin/tails-iuk-mktemp-get-target-file ""
 tails-upgrade-frontend ALL =                             NOPASSWD: /sbin/reboot ""
+tails-upgrade-frontend ALL =                             NOPASSWD: /usr/local/bin/tails-iuk-cancel-download ""
 
 tails-install-iuk     ALL =                             NOPASSWD: INSTALL_IUK

config/chroot_local-includes/lib/live/config/1500-reconfigure-APT

@@ -85,7 +85,7 @@ s{
    tor[+]https?://deb[.]tails[.]boum[.]org
    /?
    (\s+)
-}{$1tor+http://jenw7xbd6tf7vfhp.onion/$2}xms;
+}{$1tor+http://umjqavufhoix3smyq6az2sx4istmuvsgmz4bq5u5x56rnayejoo6l2qd.onion/$2}xms;
 
 ' | perl -pi - /etc/apt/sources.list /etc/apt/sources.list.d/*.list
 
@@ -99,7 +99,7 @@ s{
    (Pin:\s+origin\s+)
    deb[.]tails[.]boum[.]org
    $
-}{$1jenw7xbd6tf7vfhp.onion}xms;
+}{$1umjqavufhoix3smyq6az2sx4istmuvsgmz4bq5u5x56rnayejoo6l2qd.onion}xms;
 
 ### Fix origin for backports
 

config/chroot_local-includes/usr/lib/python2.7/dist-packages/tails_installer/__init__.py

+# -*- coding: utf-8 -*-
+#
+# Copyright © 2008  Red Hat, Inc. All rights reserved.
+#
+# This copyrighted material is made available to anyone wishing to use, modify,
+# copy, or redistribute it subject to the terms and conditions of the GNU
+# General Public License v.2.  This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
+# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.  You should have
+# received a copy of the GNU General Public License along with this program; if
+# not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth
+# Floor, Boston, MA 02110-1301, USA. Any Red Hat trademarks that are
+# incorporated in the source code or documentation are not subject to the GNU
+# General Public License and may only be used or replicated with the express
+# permission of Red Hat, Inc.
+#
+# Author(s): Luke Macken <lmacken@redhat.com>
+
+import os
+import sys
+import gettext
+import locale
+
+# Add sbin to PATH to support unprivileged mode
+if os.path.exists('/usr/sbin') or os.path.exists('/usr/local/sbin'):
+    try:
+        os.environ['PATH'] = '/usr/local/sbin:/usr/sbin:' + os.environ['PATH']
+    except KeyError, e:
+        os.environ['PATH'] = '/usr/local/sbin:/usr/sbin'
+
+def utf8_gettext(*args, **kwargs):
+    " Translate string, converting it to a UTF-8 encoded bytestring "
+    return gettext.translation(
+               'tails', '/usr/share/locale', fallback=True
+           ).gettext(*args, **kwargs)
+
+_ = utf8_gettext
+
+from tails_installer.creator import TailsInstallerError
+
+from tails_installer.creator import LinuxTailsInstallerCreator as TailsInstallerCreator
+
+from tails_installer.config import config
+branding = {
+    'distribution': config['branding']['distribution'],
+    'header':       config['branding']['header']
+}
+
+__all__ = ("TailsInstallerCreator", "TailsInstallerError", "TailsInstallerDialog", "_", "utf8_gettext", "branding")

config/chroot_local-includes/usr/lib/python2.7/dist-packages/tails_installer/config.py

+# -*- coding: utf-8 -*-
+
+config = {
+    # Minimum device size we accept as valid target for initial
+    # installation, in MiB as in 1 MiB = 1024**2 bytes. I've seen USB
+    # sticks labeled "8 GB" that were 7759462400 bytes = 7400 MiB
+    # large, and one can probably fine even smaller ones, so let's be
+    # nice with users who believed what was written on the box and
+    # accept slightly smaller devices than what the theory
+    # would dictate.
+    'min_installation_device_size': 7200,
+    # Minimum device size we tell the user they should get, in MB
+    # as in 1000 MB = 1 GB, i.e. let's use a unit close to what they will
+    # see displayed in shops.
+    'official_min_installation_device_size': 8000,
+    'main_liveos_dir': 'live',
+    'running_liveos_mountpoint': '/lib/live/mount/medium',
+    'liveos_toplevel_files': [ 'autorun.bat', 'autorun.inf', 'boot', '.disk',
+                               'doc', 'EFI', 'live', 'isolinux', 'syslinux',
+                               'tmp', 'utils' ],
+    'persistence': { 'enabled': False,
+                },
+    'branding': { 'distribution': 'Tails',
+                  'header': 'tails-liveusb-header.png',
+                  'color': '#56347c',
+                  'partition_label': 'Tails',
+                },
+}

config/chroot_local-includes/usr/lib/python2.7/dist-packages/tails_installer/source.py

+# -*- coding: utf-8 -*-
+
+import os
+import shutil
+import sys
+import subprocess
+from stat import ST_SIZE
+from tails_installer import _
+from tails_installer.config import config
+from tails_installer.utils import (_to_unicode, _dir_size, iso_is_live_system,
+                           unicode_to_utf8, unicode_to_filesystemencoding,
+                           _set_liberal_perms_recursive,
+                           underlying_physical_device, TailsError)
+
+class SourceError(TailsError):
+    """ A generic error message that is thrown by the Source classes """
+    pass
+
+class Source(object):
+    def clone(self, destination):
+        raise NotImplementedError
+
+class LocalIsoSource(Source):
+    def __init__(self, path):
+        self.path = os.path.abspath(_to_unicode(path))
+        self.size = os.stat(self.path)[ST_SIZE]
+        if not iso_is_live_system(self.path):
+            raise SourceError(_("Unable to find LiveOS on ISO"))
+        self.dev  = None
+        # This can fail for devices not supported by UDisks such as aufs mounts
+        try:
+            self.dev = underlying_physical_device(self.path)
+        except Exception, e:
+            print >> sys.stderr, _("Could not guess underlying block device: %s") % e.args[0]
+            pass
+
+    def clone(self, destination):
+        cmd = ['7z', 'x', self.path,
+               '-x![BOOT]', '-y', '-o%s' % (destination)]
+        cmd_decoded = u' '.join(cmd)
+        cmd_bytes = [ unicode_to_filesystemencoding(el) for el in cmd ]
+        proc = subprocess.Popen(cmd_bytes, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        out, err = proc.communicate()
+        out = out.decode('utf-8')
+        err = err.decode('utf-8')
+        if proc.returncode:
+            raise SourceError(_("There was a problem executing `%(cmd)s`.\n"
+                                "%(out)s\n%(err)s") % {
+                                    'cmd': cmd_decoded,
+                                    'out': out,
+                                    'err': err
+                                })
+        _set_liberal_perms_recursive(destination)
+
+
+class RunningLiveSystemSource(Source):
+    def __init__(self, path):
+        if not os.path.exists(path):
+            raise SourceError(_("'%s' does not exist") % path)
+        if not os.path.isdir(path):
+            raise SourceError(_("'%s' is not a directory") % path)
+        self.path = path
+        self.size = _dir_size(self.path)
+        self.dev  = underlying_physical_device(self.path)
+    def clone(self, destination):
+        for f in config['liveos_toplevel_files']:
+            src = os.path.join(self.path, f)
+            dst = os.path.join(destination, f)
+            if os.path.isfile(src):
+                if src.lower().endswith('.iso'):
+                    print >> sys.stderr, _("Skipping '%(filename)s'") % {
+                        'filename': src
+                    }
+                else:
+                    shutil.copy(src, dst)
+            elif os.path.islink(src):
+                linkto = os.readlink(src)
+                os.symlink(linkto, dst)
+            elif os.path.isdir(src):
+                shutil.copytree(src, dst)
+        _set_liberal_perms_recursive(destination)

config/chroot_local-includes/usr/lib/python2.7/dist-packages/tails_installer/utils.py

+# -*- coding: utf-8 -*-
+
+import os
+import re
+import subprocess
+import shutil
+import stat
+import sys
+from tails_installer import _
+from tails_installer.config import config
+
+from gi.repository import GLib
+
+class TailsError(Exception):
+    """ A generic Exception the allows us to manage error
+        messages encoded in unicode """
+    def __init__(self, message):
+        encoded_message=unicode_to_utf8(message)
+        super(TailsError, self).__init__(encoded_message)
+        self.message = encoded_message
+
+    def __unicode__(self):
+        return self.message
+
+def _to_unicode(obj, encoding='utf-8'):
+    if hasattr(obj, 'toUtf8'): # PyQt4.QtCore.QString
+        obj = str(obj.toUtf8())
+    if isinstance(obj, basestring):
+        if not isinstance(obj, unicode):
+            obj = unicode(obj, encoding)
+    return obj
+
+def unicode_to_utf8(string):
+    if isinstance(string, unicode):
+        return string.encode('utf-8')
+    return string
+
+def unicode_to_filesystemencoding(string):
+    if isinstance(string, unicode):
+        return string.encode(sys.getfilesystemencoding(), 'replace')
+    return string
+
+def extract_file_content_from_iso(iso_path, path):
+    """ Return the content of that file read from inside self.iso """
+
+    cmd = ['isoinfo', '-R', '-i', unicode_to_utf8(iso_path),
+           '-x', unicode_to_utf8(path)]
+
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = proc.communicate()
+    out = unicode_to_utf8(out)
+    err = unicode_to_utf8(err)
+    if proc.returncode:
+        raise Exception(_("There was a problem executing `%(cmd)s`."
+                          "%(out)s\n%(err)s") % {
+                              'cmd': cmd,
+                              'out': out,
+                              'err': err
+                          })
+    return out
+
+def iso_is_live_system(iso_path):
+    """ Return true iff a Live system is detected inside the iso_path file """
+    version = extract_file_content_from_iso(iso_path, '/.disk/info')
+    return version.startswith('Debian GNU/Linux')
+
+def _dir_size(source):
+    total_size = os.path.getsize(source)
+    for item in os.listdir(source):
+        itempath = os.path.join(source, item)
+        if os.path.isfile(itempath):
+            total_size += os.path.getsize(itempath)
+        elif os.path.isdir(itempath):
+            total_size += _dir_size(itempath)
+    return total_size
+
+def _move_if_exists(src, dest):
+    if os.path.exists(src):
+        shutil.move(src, dest)
+
+def _unlink_if_exists(path):
+    if os.path.exists(path):
+        os.unlink(path)
+
+def _set_liberal_perms_recursive(destination):
+    def _set_liberal_perms(arg, dirname, fnames):
+        if dirname == 'lost+found':
+            return
+        os.chmod(dirname, 0755)
+        for f in fnames:
+            if f == 'lost+found':
+                continue
+            file = os.path.join(dirname, f)
+            if os.path.isdir(file):
+                os.chmod(file, 0755)
+            elif os.path.isfile(file):
+                os.chmod(file, 0644)
+    os.path.walk(destination, _set_liberal_perms, None)
+
+def underlying_physical_device(path):
+    """ Returns the physical block device UDI on which the specified file is
+    stored (e.g. /org/freedesktop/UDisks2/block_devices/sdb).
+    """
+    rawdev = os.stat(path)[stat.ST_DEV]
+    from gi.repository import UDisks
+    udisksclient = UDisks.Client.new_sync()
+    block = udisksclient.get_block_for_dev(rawdev)
+    drive = udisksclient.get_drive_for_block(block)
+    parentblock = udisksclient.get_block_for_drive(drive, get_physical=False)
+    return parentblock.get_object_path()
+
+def _format_bytes_in_gb(value):
+    return '%0.1f GB' % (value / 10.0**9)
+
+
+def MiB_to_bytes(size_in_MiB):
+    return size_in_MiB * 1024**2
+
+
+def _get_datadir():
+    script_path = os.path.abspath(sys.argv[0])
+    if not script_path.startswith('/usr/'):
+        if os.path.exists('data/tails-installer.ui'):
+            return('data')
+    else: