Merge remote-tracking branch 'origin/devel' into feature/tor-nightly-master

Rakefile

@@ -38,7 +38,6 @@ EXPORTED_VARIABLES = [
   'TAILS_ACNG_PROXY',
   'TAILS_BUILD_FAILURE_RESCUE',
   'TAILS_DATE_OFFSET',
-  'TAILS_MERGE_BASE_BRANCH',
   'TAILS_OFFLINE_MODE',
   'TAILS_PROXY',
   'TAILS_PROXY_TYPE',
@@ -298,9 +297,16 @@ task :parse_build_options do
       $keep_running = true
       ENV['TAILS_BUILD_FAILURE_RESCUE'] = '1'
     # Jenkins
-    when 'mergebasebranch'
-      ENV['TAILS_MERGE_BASE_BRANCH'] = '1'
+    when 'nomergebasebranch'
+      $skip_mergebasebranch = true
     else
+      # Handle jenkins passing the now obsolete mergebasebranch.
+      # XXX: the commit adding this comment should be reverted
+      # once jenkins stops passing the mergebasebranch option.
+      if opt == 'mergebasebranch' && on_jenkins?
+        warn "The 'mergebasebranch' option is obsolete!"
+        next
+      end
       raise "Unknown Tails build option '#{opt}'"
     end
   end
@@ -409,6 +415,38 @@ task setup_environment: ['validate_git_state'] do
   end
 end
 
+task merge_base_branch: ['parse_build_options', 'setup_environment'] do
+  next if $skip_mergebasebranch
+  branch = git_helper('git_current_branch')
+  base_branch = git_helper('base_branch')
+  source_date_faketime = `date --utc --date="$(dpkg-parsechangelog --show-field=Date)" '+%Y-%m-%d %H:%M:%S'`.chomp
+  next if releasing? || branch == base_branch
+  commit_before_merge = git_helper('git_current_commit')
+  warn "Merging base branch '#{base_branch}' (at commit " \
+       "#{ENV['BASE_BRANCH_GIT_COMMIT']}) ..."
+  begin
+    run_command('faketime', '-f', source_date_faketime, \
+                'git', 'merge', '--no-edit', ENV['BASE_BRANCH_GIT_COMMIT'])
+  rescue CommandError
+    run_command('git', 'merge', '--abort')
+    raise <<-END_OF_MESSAGE.gsub(/^        /, '')
+
+          There were conflicts when merging the base branch; either
+          merge it yourself and resolve conflicts, or skip this merge
+          by rebuilding with the 'nomergebasebranch' option.
+
+    END_OF_MESSAGE
+  end
+  run_command('git', 'submodule', 'update', '--init')
+
+  # If we actually merged anything we'll re-run rake in the new Git
+  # state in order to avoid subtle build errors due to mixed state.
+  next if commit_before_merge == git_helper('git_current_commit')
+  ENV['TAILS_BUILD_OPTIONS'] = (ENV['TAILS_BUILD_OPTIONS'] || '') + \
+                               ' nomergebasebranch'
+  Kernel.exec('rake', *ARGV)
+end
+
 task :maybe_clean_up_builder_vms do
   clean_up_builder_vms if $force_cleanup
 end
@@ -447,6 +485,7 @@ task build: [
   'maybe_clean_up_builder_vms',
   'validate_git_state',
   'setup_environment',
+  'merge_base_branch',
   'validate_http_proxy',
   'ensure_correct_permissions',
   'vm:up',

auto/config

@@ -21,14 +21,18 @@ else
     fatal "SOURCE_DATE_EPOCH is not set. Exiting."
 fi
 
-# get git branch or tag so we can set the basename appropriately, i.e.:
-# * if we build from a tag: tails-$ARCH-$TAG.iso
-# * otherwise:              tails-$ARCH-$BRANCH-$VERSION-$TIME-$COMMIT.iso
+# get git branch or tag so we can set the basename appropriately.
 GIT_BRANCH="$(git_current_branch)"
 if [ -n "${GIT_BRANCH}" ]; then
     CLEAN_GIT_BRANCH=$(echo "$GIT_BRANCH" | sed 's,/,_,g')
     GIT_SHORT_ID="$(git_current_commit --short)"
-    BUILD_BASENAME="tails-amd64-${CLEAN_GIT_BRANCH}-${AMNESIA_VERSION}-${AMNESIA_NOW}-${GIT_SHORT_ID}"
+    BASE_BRANCH_PART=''
+    if [ "${GIT_BRANCH}" != "$(base_branch)" ]; then
+        CLEAN_GIT_BASE_BRANCH=$(base_branch | sed 's,/,_,g')
+        GIT_BASE_BRANCH_SHORT_ID="$(git rev-parse --verify --short $(git_base_branch_head))"
+        BASE_BRANCH_PART="+${CLEAN_GIT_BASE_BRANCH}@${GIT_BASE_BRANCH_SHORT_ID}"
+    fi
+    BUILD_BASENAME="tails-amd64-${CLEAN_GIT_BRANCH}@${GIT_SHORT_ID}${BASE_BRANCH_PART}-${AMNESIA_NOW}"
 else
     if git_on_a_tag; then
         CLEAN_GIT_TAG=$(git_current_tag | tr '/-' '_~')
@@ -40,30 +44,6 @@ else
     fi
 fi
 
-GIT_BASE_BRANCH=$(base_branch) \
-    || fatal "GIT_BASE_BRANCH could not be guessed."
-
-if [ "${TAILS_MERGE_BASE_BRANCH:-}" = 1 ] && \
-       ! git_on_a_tag && [ "$GIT_BRANCH" != "$GIT_BASE_BRANCH" ] ; then
-    [ -n "${BASE_BRANCH_GIT_COMMIT}" ] \
-        || fatal "Base branch's top commit is not set."
-
-    echo "I: Merging base branch ${GIT_BASE_BRANCH}" \
-         "(at commit ${BASE_BRANCH_GIT_COMMIT})..."
-    faketime -f "${SOURCE_DATE_FAKETIME}" \
-        git merge --no-edit "${BASE_BRANCH_GIT_COMMIT}" \
-	    || fatal "Failed to merge base branch."
-    git submodule update --init
-
-    # Adjust BUILD_BASENAME to embed the base branch name and its top commit
-    CLEAN_GIT_BASE_BRANCH=$(echo "$GIT_BASE_BRANCH" | sed 's,/,_,g')
-    GIT_BASE_BRANCH_SHORT_ID=$(git rev-parse --verify --short "${BASE_BRANCH_GIT_COMMIT}")
-    [ -n "${GIT_BASE_BRANCH_SHORT_ID}" ] \
-        || fatal "Base branch's top commit short ID could not be guessed."
-    BUILD_BASENAME="${BUILD_BASENAME}+${CLEAN_GIT_BASE_BRANCH}"
-    BUILD_BASENAME="${BUILD_BASENAME}@${GIT_BASE_BRANCH_SHORT_ID}"
-fi
-
 # save variables that lb build needs
 mkdir -p tmp
 echo "BUILD_BASENAME='${BUILD_BASENAME}'" > tmp/build_environment

bin/generate-report

@@ -33,6 +33,7 @@ PROJECTS = [
     GROUP_NAME + '/' + project for project in [
         'chutney',
         'installer',
+        'sysadmin',
         'tails',
         'whisperback',
         'workarounds',

bin/po4a-version

+#!/bin/sh
+
+set -eu
+
+po4a --version | head -n1 | perl -pE 's{\Apo4a version ([0-9.]+)[.]$}{$1}'

config/amnesia

@@ -12,7 +12,6 @@
 
 export SOURCE_DATE_EPOCH="$(date --utc --date="$(dpkg-parsechangelog --show-field=Date)" +%s)"
 export SOURCE_DATE_YYYYMMDD="$(date --utc --date="$(dpkg-parsechangelog --show-field=Date)" +%Y%m%d)"
-export SOURCE_DATE_FAKETIME="$(date --utc --date="$(dpkg-parsechangelog --show-field=Date)" '+%Y-%m-%d %H:%M:%S')"
 
 # Base for the string that will be passed to "lb config --bootappend-live"
 AMNESIA_APPEND="live-media=removable nopersistence noprompt timezone=Etc/UTC splash noautologin module=Tails slab_nomerge slub_debug=FZP mce=0 vsyscall=none page_poison=1 init_on_free=1 mds=full,nosmt"

config/chroot_local-hooks/52-update-rc.d

@@ -15,6 +15,7 @@ systemctl enable tails-allow-external-TailsData-access.service
 systemctl enable tails-synchronize-data-to-new-persistent-volume-on-shutdown.service
 systemctl enable tails-autotest-broken-Xorg.service
 systemctl enable tails-autotest-remote-shell.service
+systemctl enable tails-create-netns.service
 systemctl enable tails-remove-overlayfs-dirs.service
 systemctl enable tails-set-wireless-devices-state.service
 systemctl enable tails-shutdown-on-media-removal.service
@@ -34,6 +35,7 @@ systemctl --global enable tails-security-check.service
 systemctl --global enable tails-upgrade-frontend.service
 systemctl --global enable tails-virt-notify-user.service
 systemctl --global enable tails-wait-until-tor-has-bootstrapped.service
+systemctl --global enable tails-a11y-proxy-netns@onioncircs.service
 
 # Use socket activation only, to delay the startup of cupsd.
 # In practice, this means that cupsd is started during

config/chroot_local-hooks/80-block-network

@@ -4,6 +4,15 @@ set -e
 
 echo "Generating blocklist for all network devices"
 
+is_allowed() {
+    mod="$(basename "$1" .ko)"
+    shift
+    # the heredoc is the allowlist
+    grep -qwF "$mod" <<END
+veth
+END
+}
+
 is_net_module() {
     # Here we assume that if any of the patterns below are matched, it
     # is a network driver. This is not comprehensive, but should be
@@ -14,11 +23,18 @@ is_net_module() {
              -e "^depends:\s*(cfg|lib|mac)80211" \
              -e "^parm:\s*ifname:"
 }
-
 net_module_filter() {
     local path
     while read path; do
-        if is_net_module "${path}"; then
+        if ! is_allowed "${path}" && is_net_module "${path}"; then
+            echo "${path}"
+        fi
+    done
+}
+remove_allowlist_filter() {
+    local path
+    while read path; do
+        if ! is_allowed "${path}"; then
             echo "${path}"
         fi
     done
@@ -37,6 +53,7 @@ BLACKLIST=/etc/modprobe.d/all-net-blacklist.conf
 
 (
     find /lib/modules/*/kernel/drivers/net -name "*.ko" | \
+        remove_allowlist_filter | \
         generate_blocking_line && \
 
     # Let's try to find the network drivers in the staging directory as well

config/chroot_local-includes/etc/ferm/ferm.conf

@@ -18,6 +18,12 @@ domain ip {
 
             # Traffic on the loopback interface is accepted.
             interface lo ACCEPT;
+
+            # netns configuration; see config/chroot_local-includes/usr/local/lib/tails-create-netns
+            interface veth-tbb saddr 10.200.1.2 daddr 10.200.1.1 proto tcp mod multiport destination-ports (9050 9051) ACCEPT;
+            interface veth-onioncircs saddr 10.200.1.6 daddr 10.200.1.5 proto tcp mod multiport destination-ports (9051) ACCEPT;
+            interface veth-torlaunch saddr 10.200.1.10 daddr 10.200.1.9 proto tcp mod multiport destination-ports (9051) ACCEPT;
+            interface veth-onionshare saddr 10.200.1.14 daddr 10.200.1.13 proto tcp mod multiport destination-ports (9050 9051) ACCEPT;
         }
 
         chain OUTPUT {

config/chroot_local-includes/etc/onion-grater.d/onioncircuits.yml

 ---
-- apparmor-profiles:
-    - '/usr/bin/onioncircuits'
-  users:
-    - 'amnesia'
+- hosts:
+    - '10.200.1.6'
   commands:
     GETINFO:
       - 'version'

config/chroot_local-includes/etc/sudoers.d/zzz_onioncircuits

+amnesia   ALL = NOPASSWD: /usr/local/bin/onioncircuits ""
+

config/chroot_local-includes/etc/sudoers.d/zzz_upgrade

@@ -2,8 +2,9 @@ Cmnd_Alias INSTALL_IUK = /bin/dd, /bin/mount, /bin/umount, /bin/rm, /lib/live/mo
 Cmnd_Alias IUK_GET_TARGET_FILE = /usr/local/bin/tails-iuk-get-target-file
 Cmnd_Alias UPGRADE_FRONTEND = /usr/local/bin/tails-upgrade-frontend ""
 
-Defaults!IUK_GET_TARGET_FILE env_keep+="HARNESS_ACTIVE DISABLE_PROXY"
-Defaults!UPGRADE_FRONTEND env_keep+="DISABLE_PROXY SSL_NO_VERIFY"
+## Settings that might be useful for developers
+# Defaults!IUK_GET_TARGET_FILE env_keep+="HARNESS_ACTIVE DISABLE_PROXY"
+# Defaults!UPGRADE_FRONTEND env_keep+="DISABLE_PROXY"
 
 amnesia                ALL = (tails-upgrade-frontend)    NOPASSWD: UPGRADE_FRONTEND
 tails-upgrade-frontend ALL =                             NOPASSWD: /usr/local/bin/tails-shutdown-network ""

config/chroot_local-includes/lib/live/config/1500-reconfigure-APT

@@ -23,7 +23,7 @@ s{
    /[0-9]{10} # serial
    /?
    (\s+)
-}{$1tor+http://vwakviie2ienjx6t.onion/debian/$2}xms;
+}{$1tor+https://cdn-fastly.deb.debian.org/debian/$2}xms;
 
 s{
    ^
@@ -33,7 +33,7 @@ s{
    /[0-9]{10} # serial
    /?
    (\s+)
-}{$1tor+http://sgvtcaew4bxjd7ln.onion/$2}xms;
+}{$1tor+https://cdn-fastly.deb.debian.org/debian-security/$2}xms;
 
 s{
    ^
@@ -55,7 +55,7 @@ s{
    /debian
    /?
    (\s+)
-}{$1tor+http://vwakviie2ienjx6t.onion/debian/$2}xms;
+}{$1tor+https://cdn-fastly.deb.debian.org/debian/$2}xms;
 
 s{
    ^
@@ -65,7 +65,7 @@ s{
    /debian-security
    /?
    (\s+)
-}{$1tor+http://sgvtcaew4bxjd7ln.onion/$2}xms;
+}{$1tor+https://cdn-fastly.deb.debian.org/debian-security/$2}xms;
 
 s{
    ^

config/chroot_local-includes/lib/systemd/system/onion-grater.service

@@ -4,7 +4,7 @@ Documentation=https://tails.boum.org/contribute/design/
 
 [Service]
 Type=simple
-ExecStart=/usr/local/lib/onion-grater
+ExecStart=/usr/local/lib/onion-grater --listen-address 0.0.0.0
 CapabilityBoundingSet=CAP_DAC_OVERRIDE CAP_SYS_PTRACE
 PrivateDevices=yes
 PrivateTmp=yes

config/chroot_local-includes/lib/systemd/system/tails-create-netns.service

+[Unit]
+Description=Prepare network namespaces
+Documentation=https://gitlab.tails.boum.org/tails/tails/-/issues/18123
+Wants=network.target
+Before=network.target
+Before=NetworkManager.service
+Before=onion-grater.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/lib/tails-create-netns start
+ExecStop=/usr/local/lib/tails-create-netns stop
+
+[Install]
+WantedBy=sysinit.target

config/chroot_local-includes/usr/lib/python3/dist-packages/tailslib/gnome.py

 #!/usr/bin/python3
 
-import os
 import shlex
 import subprocess
 
-def _gnome_sh_wrapper(cmd):
+def _gnome_sh_wrapper(cmd) -> str:
     command = shlex.split(
         "env -i sh -c '. {lib} && {cmd}'".format(lib=GNOME_SH_PATH, cmd=cmd)
     )
@@ -13,7 +12,8 @@ def _gnome_sh_wrapper(cmd):
 GNOME_SH_PATH = "/usr/local/lib/tails-shell-library/gnome.sh"
 GNOME_ENV_VARS = _gnome_sh_wrapper("echo ${GNOME_ENV_VARS}").strip().split()
 
-def gnome_env_vars():
+
+def gnome_env_vars() -> list:
     ret = []
     for line in _gnome_sh_wrapper("export_gnome_env && env").split("\n"):
         (key, _, value) = line.rstrip().partition("=")

config/chroot_local-includes/usr/lib/systemd/user/tails-a11y-proxy-netns@.service

+[Unit]
+Description=Proxy AT-SPI bus inside a netns
+After=at-spi-dbus-bus.service
+Requires=at-spi-dbus-bus.service
+
+[Service]
+Type=notify
+NotifyAccess=all
+ExecStart=/usr/local/bin/a11y-proxy-netns --log-level DEBUG %i
+ExecStop=/bin/kill -INT $MAINPID
+
+
+[Install]
+WantedBy=desktop.target

config/chroot_local-includes/usr/local/bin/a11y-proxy-netns

+#!/usr/bin/python3
+
+import os.path
+import time
+import subprocess
+from logging import getLogger, basicConfig
+from argparse import ArgumentParser
+
+import dbus
+
+log = getLogger(os.path.basename(__file__))
+
+
+def get_parser():
+    p = ArgumentParser()
+    p.add_argument(
+        "--log-level", choices=["DEBUG", "INFO", "WARNING", "ERROR"], default="DEBUG"
+    )
+    p.add_argument("netns")
+    return p
+
+
+def get_bus() -> str:
+    bus = dbus.SessionBus()
+    obj = bus.get_object("org.a11y.Bus", "/org/a11y/bus")
+    iface = dbus.Interface(obj, dbus_interface="org.a11y.Bus")
+    response = iface.GetAddress()
+    return str(response)
+
+
+def netns_exists(name: str) -> bool:
+    return os.path.exists(os.path.join("/var/run/netns", name))
+
+
+def wait_netns(name: str, sleep_time=1):
+    notified = False
+    while not netns_exists(name):
+        if not notified:
+            log.info("Waiting for netns %s to be ready", name)
+        time.sleep(1)
+
+
+def systemd_ready():
+    try:
+        # XXX: discard stdout/stderr
+        subprocess.Popen(["systemd-notify", "--ready"])
+    except FileNotFoundError:
+        # systemd not installed
+        pass
+    else:
+        log.info("systemd was notified")
+
+
+def main():
+    args = get_parser().parse_args()
+    basicConfig(level=args.log_level)
+    wait_netns(args.netns)
+    log.debug("get address")
+    at_bus_address = get_bus()
+    log.debug("address got! %s", at_bus_address)
+
+    dirname = os.path.join("/tmp/netns-specific/", args.netns)
+    os.makedirs(dirname, exist_ok=True)
+    dest_bus_path = os.path.join(dirname, "at.sock")
+    log.debug("Binding at %s", dest_bus_path)
+    if os.path.exists(dest_bus_path):
+        os.unlink(dest_bus_path)
+
+    args = ["xdg-dbus-proxy", at_bus_address, dest_bus_path]
+    log.debug("Running %r", args)
+    # we fork-exec to handle systemd notifications. though not strictly needed, they are nice!
+    p = subprocess.Popen(args)
+    log.debug("Started!")
+    # XXX: we could wait for dest_bus_path to appear, before signaling us ready.
+    systemd_ready()
+    try:
+        p.communicate()
+    except KeyboardInterrupt:
+        # this except clause will handle SIGINT, but not other signals
+        # we should probably explicitly do that!
+        p.kill()
+        log.debug("Killed %s", args[0])
+    return
+
+
+if __name__ == "__main__":
+    main()

config/chroot_local-includes/usr/local/bin/onioncircuits

+#!/usr/bin/env python3
+import os
+import logging
+
+from tailslib.gnome import gnome_env_vars
+
+
+def run_in_netns(*args, netns, user="amnesia"):
+    # base bwrap sharing most of the system
+    bwrap = ["bwrap", "--bind", "/", "/", "--proc", "/proc", "--dev", "/dev"]
+    # passes data to us
+    bwrap += [
+        "--bind",
+        os.path.join("/tmp/netns-specific/", netns),
+        "/tmp/shared-with-me/",
+    ]
+    # hide data not for us
+    bwrap += ["--tmpfs", "/tmp/netns-specific/"]
+    cmd = [
+        "/bin/ip",
+        "netns",
+        "exec",
+        netns,
+        "/sbin/runuser",
+        "-u",
+        user,
+        "--",
+        *bwrap,
+        "/usr/bin/env",
+        *gnome_env_vars(),
+        "AT_SPI_BUS_ADDRESS=unix:path=/tmp/shared-with-me/at.sock",
+        *args,
+    ]
+    logging.info("Running %s", cmd)
+    os.execvp(cmd[0], cmd)
+
+
+def drop_and_run():
+    run_in_netns("/usr/bin/onioncircuits", netns="onioncircs")
+
+
+def main():
+    if os.getuid() == 0:
+        drop_and_run()
+    else:
+        os.execlp("sudo", "sudo", "--non-interactive", "/usr/local/bin/onioncircuits")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    main()

config/chroot_local-includes/usr/local/lib/initramfs-pre-shutdown-hook

@@ -43,6 +43,7 @@ mkdir -p /mnt/live/run
 
 # Finally, really unmount relevant filesystems
 /bin/umount /oldroot
+/bin/umount /mnt/live/medium
 
 # Debugging
 /bin/mount