Merge remote-tracking branch 'origin/devel' into feature/shellcheck+force-all-tests

bin/po4a-version

+#!/bin/sh
+
+set -eu
+
+po4a --version | head -n1 | perl -pE 's{\Apo4a version ([0-9.]+)[.]$}{$1}'

config/chroot_local-hooks/52-update-rc.d

@@ -15,6 +15,7 @@ systemctl enable tails-allow-external-TailsData-access.service
 systemctl enable tails-synchronize-data-to-new-persistent-volume-on-shutdown.service
 systemctl enable tails-autotest-broken-Xorg.service
 systemctl enable tails-autotest-remote-shell.service
+systemctl enable tails-create-netns.service
 systemctl enable tails-remove-overlayfs-dirs.service
 systemctl enable tails-set-wireless-devices-state.service
 systemctl enable tails-shutdown-on-media-removal.service
@@ -34,6 +35,7 @@ systemctl --global enable tails-security-check.service
 systemctl --global enable tails-upgrade-frontend.service
 systemctl --global enable tails-virt-notify-user.service
 systemctl --global enable tails-wait-until-tor-has-bootstrapped.service
+systemctl --global enable tails-a11y-proxy-netns@onioncircs.service
 
 # Use socket activation only, to delay the startup of cupsd.
 # In practice, this means that cupsd is started during

config/chroot_local-hooks/80-block-network

@@ -4,6 +4,15 @@ set -e
 
 echo "Generating blocklist for all network devices"
 
+is_allowed() {
+    mod="$(basename "$1" .ko)"
+    shift
+    # the heredoc is the allowlist
+    grep -qwF "$mod" <<END
+veth
+END
+}
+
 is_net_module() {
     # Here we assume that if any of the patterns below are matched, it
     # is a network driver. This is not comprehensive, but should be
@@ -14,11 +23,18 @@ is_net_module() {
              -e "^depends:\s*(cfg|lib|mac)80211" \
              -e "^parm:\s*ifname:"
 }
-
 net_module_filter() {
     local path
     while read -r path; do
-        if is_net_module "${path}"; then
+        if ! is_allowed "${path}" && is_net_module "${path}"; then
+            echo "${path}"
+        fi
+    done
+}
+remove_allowlist_filter() {
+    local path
+    while read -r path; do
+        if ! is_allowed "${path}"; then
             echo "${path}"
         fi
     done
@@ -37,6 +53,7 @@ BLACKLIST=/etc/modprobe.d/all-net-blacklist.conf
 
 (
     find /lib/modules/*/kernel/drivers/net -name "*.ko" | \
+        remove_allowlist_filter | \
         generate_blocking_line && \
 
     # Let's try to find the network drivers in the staging directory as well

config/chroot_local-includes/etc/ferm/ferm.conf

@@ -18,6 +18,12 @@ domain ip {
 
             # Traffic on the loopback interface is accepted.
             interface lo ACCEPT;
+
+            # netns configuration; see config/chroot_local-includes/usr/local/lib/tails-create-netns
+            interface veth-tbb saddr 10.200.1.2 daddr 10.200.1.1 proto tcp mod multiport destination-ports (9050 9051) ACCEPT;
+            interface veth-onioncircs saddr 10.200.1.6 daddr 10.200.1.5 proto tcp mod multiport destination-ports (9051) ACCEPT;
+            interface veth-torlaunch saddr 10.200.1.10 daddr 10.200.1.9 proto tcp mod multiport destination-ports (9051) ACCEPT;
+            interface veth-onionshare saddr 10.200.1.14 daddr 10.200.1.13 proto tcp mod multiport destination-ports (9050 9051) ACCEPT;
         }
 
         chain OUTPUT {

config/chroot_local-includes/etc/onion-grater.d/onioncircuits.yml

 ---
-- apparmor-profiles:
-    - '/usr/bin/onioncircuits'
-  users:
-    - 'amnesia'
+- hosts:
+    - '10.200.1.6'
   commands:
     GETINFO:
       - 'version'

config/chroot_local-includes/etc/sudoers.d/zzz_onioncircuits

+amnesia   ALL = NOPASSWD: /usr/local/bin/onioncircuits ""
+

config/chroot_local-includes/etc/sudoers.d/zzz_upgrade

@@ -2,8 +2,9 @@ Cmnd_Alias INSTALL_IUK = /bin/dd, /bin/mount, /bin/umount, /bin/rm, /lib/live/mo
 Cmnd_Alias IUK_GET_TARGET_FILE = /usr/local/bin/tails-iuk-get-target-file
 Cmnd_Alias UPGRADE_FRONTEND = /usr/local/bin/tails-upgrade-frontend ""
 
-Defaults!IUK_GET_TARGET_FILE env_keep+="HARNESS_ACTIVE DISABLE_PROXY"
-Defaults!UPGRADE_FRONTEND env_keep+="DISABLE_PROXY SSL_NO_VERIFY"
+## Settings that might be useful for developers
+# Defaults!IUK_GET_TARGET_FILE env_keep+="HARNESS_ACTIVE DISABLE_PROXY"
+# Defaults!UPGRADE_FRONTEND env_keep+="DISABLE_PROXY"
 
 amnesia                ALL = (tails-upgrade-frontend)    NOPASSWD: UPGRADE_FRONTEND
 tails-upgrade-frontend ALL =                             NOPASSWD: /usr/local/bin/tails-shutdown-network ""

config/chroot_local-includes/lib/systemd/system/onion-grater.service

@@ -4,7 +4,7 @@ Documentation=https://tails.boum.org/contribute/design/
 
 [Service]
 Type=simple
-ExecStart=/usr/local/lib/onion-grater
+ExecStart=/usr/local/lib/onion-grater --listen-address 0.0.0.0
 CapabilityBoundingSet=CAP_DAC_OVERRIDE CAP_SYS_PTRACE
 PrivateDevices=yes
 PrivateTmp=yes

config/chroot_local-includes/lib/systemd/system/tails-create-netns.service

+[Unit]
+Description=Prepare network namespaces
+Documentation=https://gitlab.tails.boum.org/tails/tails/-/issues/18123
+Wants=network.target
+Before=network.target
+Before=NetworkManager.service
+Before=onion-grater.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/lib/tails-create-netns start
+ExecStop=/usr/local/lib/tails-create-netns stop
+
+[Install]
+WantedBy=sysinit.target

config/chroot_local-includes/usr/lib/python3/dist-packages/tailslib/gnome.py

 #!/usr/bin/python3
 
-import os
 import shlex
 import subprocess
 
-def _gnome_sh_wrapper(cmd):
+def _gnome_sh_wrapper(cmd) -> str:
     command = shlex.split(
         "env -i sh -c '. {lib} && {cmd}'".format(lib=GNOME_SH_PATH, cmd=cmd)
     )
@@ -13,7 +12,8 @@ def _gnome_sh_wrapper(cmd):
 GNOME_SH_PATH = "/usr/local/lib/tails-shell-library/gnome.sh"
 GNOME_ENV_VARS = _gnome_sh_wrapper("echo ${GNOME_ENV_VARS}").strip().split()
 
-def gnome_env_vars():
+
+def gnome_env_vars() -> list:
     ret = []
     for line in _gnome_sh_wrapper("export_gnome_env && env").split("\n"):
         (key, _, value) = line.rstrip().partition("=")

config/chroot_local-includes/usr/lib/systemd/user/tails-a11y-proxy-netns@.service

+[Unit]
+Description=Proxy AT-SPI bus inside a netns
+After=at-spi-dbus-bus.service
+Requires=at-spi-dbus-bus.service
+
+[Service]
+Type=notify
+NotifyAccess=all
+ExecStart=/usr/local/bin/a11y-proxy-netns --log-level DEBUG %i
+ExecStop=/bin/kill -INT $MAINPID
+
+
+[Install]
+WantedBy=desktop.target

config/chroot_local-includes/usr/local/bin/a11y-proxy-netns

+#!/usr/bin/python3
+
+import os.path
+import time
+import subprocess
+from logging import getLogger, basicConfig
+from argparse import ArgumentParser
+
+import dbus
+
+log = getLogger(os.path.basename(__file__))
+
+
+def get_parser():
+    p = ArgumentParser()
+    p.add_argument(
+        "--log-level", choices=["DEBUG", "INFO", "WARNING", "ERROR"], default="DEBUG"
+    )
+    p.add_argument("netns")
+    return p
+
+
+def get_bus() -> str:
+    bus = dbus.SessionBus()
+    obj = bus.get_object("org.a11y.Bus", "/org/a11y/bus")
+    iface = dbus.Interface(obj, dbus_interface="org.a11y.Bus")
+    response = iface.GetAddress()
+    return str(response)
+
+
+def netns_exists(name: str) -> bool:
+    return os.path.exists(os.path.join("/var/run/netns", name))
+
+
+def wait_netns(name: str, sleep_time=1):
+    notified = False
+    while not netns_exists(name):
+        if not notified:
+            log.info("Waiting for netns %s to be ready", name)
+        time.sleep(1)
+
+
+def systemd_ready():
+    try:
+        # XXX: discard stdout/stderr
+        subprocess.Popen(["systemd-notify", "--ready"])
+    except FileNotFoundError:
+        # systemd not installed
+        pass
+    else:
+        log.info("systemd was notified")
+
+
+def main():
+    args = get_parser().parse_args()
+    basicConfig(level=args.log_level)
+    wait_netns(args.netns)
+    log.debug("get address")
+    at_bus_address = get_bus()
+    log.debug("address got! %s", at_bus_address)
+
+    dirname = os.path.join("/tmp/netns-specific/", args.netns)
+    os.makedirs(dirname, exist_ok=True)
+    dest_bus_path = os.path.join(dirname, "at.sock")
+    log.debug("Binding at %s", dest_bus_path)
+    if os.path.exists(dest_bus_path):
+        os.unlink(dest_bus_path)
+
+    args = ["xdg-dbus-proxy", at_bus_address, dest_bus_path]
+    log.debug("Running %r", args)
+    # we fork-exec to handle systemd notifications. though not strictly needed, they are nice!
+    p = subprocess.Popen(args)
+    log.debug("Started!")
+    # XXX: we could wait for dest_bus_path to appear, before signaling us ready.
+    systemd_ready()
+    try:
+        p.communicate()
+    except KeyboardInterrupt:
+        # this except clause will handle SIGINT, but not other signals
+        # we should probably explicitly do that!
+        p.kill()
+        log.debug("Killed %s", args[0])
+    return
+
+
+if __name__ == "__main__":
+    main()

config/chroot_local-includes/usr/local/bin/onioncircuits

+#!/usr/bin/env python3
+import os
+import logging
+
+from tailslib.gnome import gnome_env_vars
+
+
+def run_in_netns(*args, netns, user="amnesia"):
+    # base bwrap sharing most of the system
+    bwrap = ["bwrap", "--bind", "/", "/", "--proc", "/proc", "--dev", "/dev"]
+    # passes data to us
+    bwrap += [
+        "--bind",
+        os.path.join("/tmp/netns-specific/", netns),
+        "/tmp/shared-with-me/",
+    ]
+    # hide data not for us
+    bwrap += ["--tmpfs", "/tmp/netns-specific/"]
+    cmd = [
+        "/bin/ip",
+        "netns",
+        "exec",
+        netns,
+        "/sbin/runuser",
+        "-u",
+        user,
+        "--",
+        *bwrap,
+        "/usr/bin/env",
+        *gnome_env_vars(),
+        "AT_SPI_BUS_ADDRESS=unix:path=/tmp/shared-with-me/at.sock",
+        *args,
+    ]
+    logging.info("Running %s", cmd)
+    os.execvp(cmd[0], cmd)
+
+
+def drop_and_run():
+    run_in_netns("/usr/bin/onioncircuits", netns="onioncircs")
+
+
+def main():
+    if os.getuid() == 0:
+        drop_and_run()
+    else:
+        os.execlp("sudo", "sudo", "--non-interactive", "/usr/local/bin/onioncircuits")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    main()

config/chroot_local-includes/usr/local/lib/tails-create-netns

+#!/bin/sh
+
+
+#ns=tbbNs
+guestVeth="veth0"
+
+set -ue
+
+increment_ip_address() {
+    echo "$1" | \
+        python3  -c 'base, host = input().rsplit(".", 1); print("%s.%s" % (base, int(host)+1))'
+}
+
+decrement_ip_address() {
+    echo "$1" | \
+        python3  -c 'base, host = input().rsplit(".", 1); print("%s.%s" % (base, int(host)-1))'
+}
+
+get_netns_guest_address() {
+    ns="$1"
+    ip netns exec "$ns" ip -4 a show dev "$guestVeth" |  grep -Po 'inet \K[\d.]+'
+}
+
+get_netns_host_address() {
+    ns="$1"
+    decrement_ip_address "$(get_netns_guest_address "$ns")"
+}
+
+expose() {
+    if [ $# -ne 3 ]
+    then
+        echo 'Wrong expose usage' >&2
+        exit 2
+    fi
+    ns="$1"
+    guestPort="$2"
+    hostPort="$3"
+    hostAddress="$(get_netns_host_address "$ns")"
+    guestAddress="$(get_netns_guest_address "$ns")"
+    hostVeth="veth-${ns}"
+    # $1 is netNs name
+    # $2 is netNs port
+    # $3 is host port
+    ip netns exec "$ns" iptables -t nat \
+        -A OUTPUT -o lo -d 127.0.0.1 -p tcp --dport "$guestPort" \
+        -j DNAT  --to-destination "$hostAddress:$hostPort"
+
+}
+
+delete_netns() {
+    # $1 = netns basename
+    basename="$1"
+    nsName="${basename}"
+    hostVeth="veth-${basename}"
+    ip link del "$hostVeth" || true
+    ip netns del "$nsName" || true
+}
+
+create_netns() {
+    # $1 = netns basename
+    # $2 = first address; implies /30
+    if ! [ $# -eq 2 ]; then
+        echo "Wrong usage for create_netns" >&2
+        exit 2
+    fi
+    basename="$1"
+    hostAddress="$2"
+    netmask=30
+    nsName="${basename}"
+    hostVeth="veth-${basename}"
+    if [ "${#hostVeth}" -ge 16 ]
+    then
+        echo "netns name too long '${hostVeth}'; it would have a veth name >= 16"
+        exit 2
+    fi
+    guestAddress="$(increment_ip_address "$hostAddress")"
+    ip netns add "$nsName"
+
+    # create veth
+    ip netns exec "$nsName" ip link set dev lo up
+    ip link add "$hostVeth" type veth peer name "$guestVeth"
+
+    # setup veth
+    ip link set veth0 netns "$nsName"
+    ip addr add "${hostAddress}/$netmask" dev "$hostVeth"
+    ip link set dev "$hostVeth" up
+    ip netns exec "$nsName" ip addr add "${guestAddress}/$netmask" dev "$guestVeth"
+    ip netns exec "$nsName" ip link set dev "$guestVeth" up
+
+    # setup iptables
+    ## forbid IP spoofing
+    ip netns exec "$nsName" iptables -A OUTPUT -o veth0 ! --src "$guestAddress" -j REJECT
+    ip netns exec "$nsName" sysctl net.ipv4.ip_forward=0
+    ip netns exec "$nsName" sysctl net.ipv4.conf.all.forwarding=0
+    ip netns exec "$nsName" sysctl net.ipv4.conf.lo.forwarding=0
+    ip netns exec "$nsName" sysctl net.ipv4.conf.all.route_localnet=1
+
+    ip netns exec "$nsName" iptables -t nat -A POSTROUTING -j MASQUERADE
+
+    sysctl net.ipv4.ip_forward=0
+    sysctl net.ipv4.conf.all.forwarding=0
+    sysctl "net.ipv4.conf.${hostVeth}.forwarding=0"
+}
+
+if [ "$#" -ne 1 ]
+then
+    echo "Wrong usage: $0 start|stop" >&2
+    exit 2
+fi
+
+if [ "$1" = stop ]
+then
+    delete_netns tbb
+    delete_netns onioncircs
+    delete_netns torlaunch
+    delete_netns onionshare
+    exit
+fi
+
+if [ "$1" = start ]
+then
+    modprobe veth
+    modprobe xt_MASQUERADE
+    modprobe xt_nat
+    netBase='10.200.1'
+    create_netns tbb "${netBase}.1"
+    create_netns onioncircs "${netBase}.5"
+    create_netns torlaunch "${netBase}.9"
+    create_netns onionshare "${netBase}.13"
+
+    # Exposing specific services to applications confined in netns
+    expose tbb 9050 9050
+    expose tbb 9051 9051
+    expose onioncircs 9051 9051
+    expose torlaunch 9051 9051
+    expose onionshare 9050 9050
+    expose onionshare 9051 9051
+fi
+

config/chroot_local-includes/usr/local/lib/tails-shell-library/hardware.sh

@@ -7,7 +7,30 @@ get_all_ethernet_nics() {
         if [ "$(cat "${i}"/type)" = 1 ]; then
             basename "${i}"
         fi
-    done
+    done | grep -Fv --line-regexp -f /dev/fd/3  3<<EOT
+$(get_all_veth_nics)
+EOT
+   # the above command just "removes" from the output of the for loop
+   # everything that is outputted by get_all_veth_nics
+   # On bash, you might have used process substitution: for-loop | grep -Fxv -f <(get_all_veth_nics)
+   # but we're in posix-compatible shell scripting!
+   ret=$?
+   if [ "$ret" -lt 2 ]; then
+       return 0
+   else
+       return $ret
+   fi
+}
+
+get_all_veth_nics() {
+    ip -brief link show type veth|
+        awk '{ print $1 }' |
+        cut -d '@' -f 1
+}
+
+is_veth_nic() {
+    # checks if argument is a veth nic
+    get_all_veth_nics | grep -F --line-regexp -q "${1}"
 }
 
 nic_exists() {

config/chroot_local-includes/usr/local/lib/tails-spoof-mac

@@ -107,6 +107,12 @@ set_log_tag spoof-mac
 
 NIC="${1}"
 
+if is_veth_nic "${NIC}"
+then
+    log -t debug "NIC ${NIC} is actually a veth; no spoofing"
+    exit 0
+fi
+
 if ! mac_spoof_is_enabled; then
     exit 0
 fi

config/chroot_local-includes/usr/src/iuk/dist.ini

@@ -62,6 +62,7 @@ Sys::Filesystem = 1.28
 Test::BDD::Cucumber = 0.16
 Test::EOL = 0.9
 Test::Fatal = 0.010
+Test::LWP::UserAgent = 0.031
 Test::Most = 0.22
 Test::Perl::Critic = 1.02
 Test::Spec = 0.40

config/chroot_local-includes/usr/src/iuk/features/download_target_file/Download_Target_File.feature

@@ -29,6 +29,42 @@ Feature: download and verify a target file
       | whatever1.file | /        | abc     |    3 | ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad |
       | whatever2.file | /sub/dir | 123     |    3 | a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3 |
 
+  @retry
+  Scenario: Successfully resuming an interrupted download, from the same mirror
+    Given a HTTP server that supports Range requests and serves "<file>" in "<webdir>" with content "<content>" and hash "<sha256>"
+    When I download "<file>" (of expected size <size>) from "<webdir>", failing 1 time, and check its hash is "<sha256>"
+    Then it should succeed
+    And I should be told "Sending HTTP request: attempt no. 1"
+    And I should be told "Resuming download after 1 bytes"
+    And I should be told "Sending HTTP request: attempt no. 2"
+    And I should not be told "Falling back to DNS mirror pool"
+    And I should see the downloaded file in the temporary directory
+    And the SHA-256 of the downloaded file should be "<sha256>"
+    And the downloaded file should be world-readable
+    Examples:
+      | file           | webdir   | content | size | sha256                                                           |
+      | whatever1.file | /        | abc     |    3 | ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad |
+      | whatever2.file | /sub/dir | 123     |    3 | a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3 |
+
+  @retry
+  # Keep in sync with Tails::IUK::TargetFile::Download: here we must
+  # simulate (max_attempts - 1) failures
+  Scenario: Successfully resuming an interrupted download, using the fallback mirror pool
+    Given a HTTP server that supports Range requests and serves "<file>" in "<webdir>" with content "<content>" and hash "<sha256>"
+    When I download "<file>" (of expected size <size>) from "<webdir>", failing 5 times, and check its hash is "<sha256>"
+    Then it should succeed
+    And I should be told "Sending HTTP request: attempt no. 1"
+    And I should be told "Resuming download after 1 bytes"
+    And I should be told "Sending HTTP request: attempt no. 2"
+    And I should be told "Falling back to DNS mirror pool"
+    And I should see the downloaded file in the temporary directory
+    And the SHA-256 of the downloaded file should be "<sha256>"
+    And the downloaded file should be world-readable
+    Examples:
+      | file           | webdir   | content | size | sha256                                                           |
+      | whatever1.file | /        | abc     |    3 | ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad |
+      | whatever2.file | /sub/dir | 123     |    3 | a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3 |
+
   Scenario: Successful download and verification with redirect to another hostname over HTTPS
     Given a HTTP server that redirects to 127.0.0.2 over HTTPS
     And a HTTPS server on 127.0.0.2 that serves "<file>" in "<webdir>" with content "<content>" and hash "<sha256>"

config/chroot_local-includes/usr/src/iuk/features/download_target_file/step_definitions/Download_Target_File_steps.pl

@@ -12,6 +12,8 @@ use Data::Dumper;
 use English qw{-no_match_vars};
 use Fcntl ':mode';
 use Function::Parameters;
+use IPC::System::Simple qw{systemx};
+use LWP::UserAgent;
 use Test::More;
 use Test::BDD::Cucumber::StepFile;
 
@@ -22,6 +24,7 @@ use Test::WebServer::RedirectToHTTPS;
 use Test::WebServer::Static;
 use Test::WebServer::Static::SSL;
 use Types::Path::Tiny qw{AbsPath};
+use Types::Standard qw{Int};
 
 my $bindir = path(__FILE__)->parent->parent->parent->parent->child('bin')->absolute;
 
@@ -77,21 +80,74 @@ fun prepare_webroot (AbsPath $webroot, $filename, $present, $webdir, $spec_type,
     }
 }
 
-Given qr{^a HTTP server that(| does not) serve[s]? "([^"]+)" in "([^"]+)"(?: with (content|size) "?([^"]*)"?)?}, fun ($c) {
-    my $present    = $c->matches->[0] ? 0 : 1;
-    my $filename   = $c->matches->[1];
-    my $webdir     = $c->matches->[2];
-    my $spec_type  = $c->matches->[3];
-    my $spec_value = $c->matches->[4];
+fun generate_nginx_conf (
+    AbsPath :$pid_file,
+    AbsPath :$conf_file,
+    AbsPath :$webroot,
+    Int :$port
+  ) {