Commit 00379539 authored by intrigeri's avatar intrigeri
Browse files

Merge branch 'devel' into hefee/bugfix/16186-disable-autocrypt+force-all-tests

parents a6786409 d50830f2
......@@ -98,6 +98,7 @@ DEBOOTSTRAP_OPTIONS="${DEBOOTSTRAP_OPTIONS:-} --no-merged-usr"
# use our own APT repository's key:
DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
gpg --homedir "$DEBOOTSTRAP_GNUPG_HOMEDIR" \
--no-tty \
--import config/chroot_sources/tails.chroot.gpg
if [ -e "$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.gpg" ]; then
DEBOOTSTRAP_GNUPG_KEYRING="$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.gpg"
......
......@@ -4,7 +4,7 @@ import argparse
import os
import logging
from contextlib import contextmanager
import re
import tempfile
import time
import subprocess
......@@ -30,7 +30,6 @@ PARTITION_LABEL = 'Tails'
FILESYSTEM_LABEL = 'Tails'
GET_UDISKS_OBJECT_TIMEOUT = 2
WAIT_FOR_PARTITION_TIMEOUT = 2
# The size of the system partition (in MiB) will be:
#
......@@ -56,7 +55,7 @@ class ImageCreator(object):
self._loop_device = None # type: str
self._partition = None # type: str
self._system_partition_size = None # type: int
self.mountpoint = None # type: str
self.iso_mountpoint = None # type: str
@property
def loop_device(self) -> UDisks.ObjectProxy:
......@@ -108,14 +107,12 @@ class ImageCreator(object):
self.set_partition_flags()
# XXX: Rescan?
self.format_partition()
with self.mount_partition():
with self.mount_iso():
self.extract_iso()
self.set_permissions()
self.update_configs()
self.adjust_syslinux_configuration()
self.install_mbr()
self.copy_syslinux_modules()
# We have to install syslinux after the partition was unmounted.
# This sleep is a workaround for a race condition which causes the
# syslinux installation to return without errors, even though the
# bootloader isn't actually installed
......@@ -126,12 +123,20 @@ class ImageCreator(object):
self.set_guids()
self.set_fsuuid()
with self.mount_partition():
self.reset_timestamps()
def extract_iso(self):
logger.info("Extracting ISO contents to the partition")
execute(['7z', 'x', self.iso, '-x![BOOT]', '-y', '-o%s' % self.mountpoint])
for root, dirs, files in os.walk(self.iso_mountpoint):
for d in sorted(dirs):
with set_env("MTOOLS_SKIP_CHECK", "1"):
execute(['mmd',
"-i", self.partition.props.block.props.device,
"::%s" % os.path.join(os.path.relpath(root, start=self.iso_mountpoint), d)])
for f in sorted(files):
with set_env("MTOOLS_SKIP_CHECK", "1"):
execute(['mcopy',
"-i", self.partition.props.block.props.device,
os.path.join(root, f),
"::%s" % os.path.join(os.path.relpath(root, start=self.iso_mountpoint), f)])
def create_empty_image(self):
logger.info("Creating empty image %r", self.image)
......@@ -165,6 +170,18 @@ class ImageCreator(object):
arg_options=GLib.Variant('a{sv}', None),
)
@contextmanager
def mount_iso(self):
with tempfile.TemporaryDirectory() as mountpoint:
logger.info("Mounting ISO on %s" % mountpoint)
execute(['mount', '-o', 'loop,ro', self.iso, mountpoint])
self.iso_mountpoint = mountpoint
try:
yield mountpoint
finally:
logger.info("Unmounting ISO")
execute(['umount', mountpoint])
def create_gpt(self):
logger.info("Creating GPT")
self.loop_device.props.block.call_format_sync(
......@@ -198,102 +215,53 @@ class ImageCreator(object):
def format_partition(self):
logger.info("Formatting partition")
options = GLib.Variant('a{sv}', {
'label': GLib.Variant('s', FILESYSTEM_LABEL),
'update-partition-type': GLib.Variant('b', False)
})
self.partition.props.block.call_format_sync(
arg_type='vfat',
arg_options=options
)
@contextmanager
def mount_partition(self):
logger.info("Mounting partition")
try:
self.mountpoint = self.partition.props.filesystem.call_mount_sync(
arg_options=GLib.Variant('a{sv}', None)
)
except GLib.Error as e:
if "org.freedesktop.UDisks2.Error.AlreadyMounted" in e.message and \
self.partition.props.filesystem.props.mount_points:
self.mountpoint = self.partition.props.filesystem.props.mount_points[0]
logger.info("Partition is already mounted at {}".format(self.mountpoint))
else:
raise
try:
yield
finally:
logger.info("Unmounting partition")
self.partition.props.filesystem.call_unmount_sync(
arg_options=GLib.Variant('a{sv}', {'force': GLib.Variant('b', True)}),
)
execute([
'mkfs.msdos',
'-v',
# Use constants for normally randomly generated or time-based data
# such as volume ID and creation time
'--invariant',
# Fill all 11 chars of the volume label to avoid any uninitialized
# memory from sneaking in
'-n', 'TAILS' + 6 * ' ',
self.partition.props.block.props.device,
])
def set_permissions(self):
logger.info("Setting file access permissions")
for root, dirs, files in os.walk(self.mountpoint):
for d in dirs:
os.chmod(os.path.join(root, d), 0o755)
for f in files:
os.chmod(os.path.join(root, f), 0o644)
def update_configs(self):
logger.info("Updating config files")
grubconf = os.path.join(self.mountpoint, "EFI", "BOOT", "grub.conf")
bootconf = os.path.join(self.mountpoint, "EFI", "BOOT", "boot.conf")
isolinux_dir = os.path.join(self.mountpoint, "isolinux")
syslinux_dir = os.path.join(self.mountpoint, "syslinux")
isolinux_cfg = os.path.join(syslinux_dir, "isolinux.cfg")
files_to_update = [
(os.path.join(self.mountpoint, "isolinux", "isolinux.cfg"),
os.path.join(self.mountpoint, "isolinux", "syslinux.cfg")),
(os.path.join(self.mountpoint, "isolinux", "stdmenu.cfg"),
os.path.join(self.mountpoint, "isolinux", "stdmenu.cfg")),
(os.path.join(self.mountpoint, "isolinux", "exithelp.cfg"),
os.path.join(self.mountpoint, "isolinux", "exithelp.cfg")),
(os.path.join(self.mountpoint, "EFI", "BOOT", "isolinux.cfg"),
os.path.join(self.mountpoint, "EFI", "BOOT", "syslinux.cfg")),
(grubconf, bootconf)
]
for (infile, outfile) in files_to_update:
if os.path.exists(infile):
self.update_config(infile, outfile)
if os.path.exists(isolinux_dir):
execute(["mv", isolinux_dir, syslinux_dir])
if os.path.exists(isolinux_cfg):
os.remove(isolinux_cfg)
def update_config(self, infile, outfile):
with open(infile) as f_in:
lines = [re.sub('/isolinux/', '/syslinux/', line) for line in f_in]
with open(outfile, "w") as f_out:
f_out.writelines(lines)
def adjust_syslinux_configuration(self):
logger.info("Adjusting syslinux configuration")
with set_env("MTOOLS_SKIP_CHECK", "1"):
execute(['mren', "-i", self.partition.props.block.props.device,
"::isolinux", "::syslinux"])
execute(['mren', "-i", self.partition.props.block.props.device,
"::syslinux/isolinux.cfg", "::syslinux/syslinux.cfg"])
def install_mbr(self):
logger.info("Installing MBR")
mbr_path = os.path.join(self.mountpoint, "utils/mbr/mbr.bin")
execute(["dd", "bs=440", "count=1", "conv=notrunc", "if=%s" % mbr_path, "of=%s" % self.image])
# Only required if using the running system's syslinux instead of the one on the ISO
mbr_path = os.path.join(self.iso_mountpoint, "utils/mbr/mbr.bin")
execute(["dd", "bs=440", "count=1", "conv=notrunc",
"if=%s" % mbr_path, "of=%s" % self.image])
# Ensure the syslinux modules come from the same package as the
# binary we'll use in install_syslinux, i.e. the system's one
# and not the one that's in the ISO; otherwise their ABI may
# be incompatible
def copy_syslinux_modules(self):
logger.info("Copying syslinux modules to device")
syslinux_dir = os.path.join(self.mountpoint, 'syslinux')
syslinux_dir = os.path.join(self.iso_mountpoint, 'isolinux')
com32modules = [f for f in os.listdir(syslinux_dir) if f.endswith('.c32')]
for module in com32modules:
for module in sorted(com32modules):
src_path = os.path.join(SYSLINUX_COM32MODULES_DIR, module)
if not os.path.isfile(src_path):
raise ImageCreationError("Could not find the '%s' COM32 module" % module)
logger.debug('Copying %s to the device' % src_path)
execute(["cp", "-a", src_path, os.path.join(syslinux_dir, module)])
with set_env("MTOOLS_SKIP_CHECK", "1"):
execute(['mcopy', '-D', 'o',
'-i', self.partition.props.block.props.device,
src_path,
"::%s" % os.path.join('syslinux', module)])
def install_syslinux(self):
logger.info("Installing bootloader")
......@@ -307,13 +275,6 @@ class ImageCreator(object):
'--install', self.image
])
def reset_timestamps(self):
logger.info("Resetting timestamps")
for root, dirs, files in os.walk(self.mountpoint):
os.utime(root, (0, 0), follow_symlinks=False)
for file in files:
os.utime(os.path.join(root, file), (0, 0), follow_symlinks=False)
def set_guids(self):
logger.info("Setting disk and partition GUID")
execute(["/sbin/sgdisk", "--disk-guid", "17B81DA0-8B1E-4269-9C39-FE5C7B9B58A3",
......@@ -335,7 +296,7 @@ def execute(cmd: list):
@contextmanager
def set_env(name: str, value:str):
def set_env(name: str, value: str):
old_value = os.getenv(name)
os.putenv(name, value)
try:
......
......@@ -27,32 +27,30 @@ def target_file_url(channel, filename):
}
def idf_content(build_target, channel, product_name, version, img, iso):
installation_paths = [
{
'type': 'iso',
'target-files': [{
'url': target_file_url(channel, iso),
'sha256': sha256_file(iso),
'size': Path(iso).stat().st_size,
}],
},
]
if img is not None:
installation_paths += {
'type': 'img',
'target-files': [{
'url': target_file_url(channel, img),
'sha256': sha256_file(img),
'size': Path(img).stat().st_size,
}],
}
return to_json({
'build_target': build_target,
'channel': channel,
'product-name': product_name,
'installations': [{
'version': version,
'installation-paths': installation_paths,
'installation-paths': [
{
'type': 'img',
'target-files': [{
'url': target_file_url(channel, img),
'sha256': sha256_file(img),
'size': Path(img).stat().st_size,
}],
},
{
'type': 'iso',
'target-files': [{
'url': target_file_url(channel, iso),
'sha256': sha256_file(iso),
'size': Path(iso).stat().st_size,
}],
},
],
}],
})
......@@ -64,7 +62,7 @@ if __name__ == '__main__':
parser.add_argument('--product-name', dest='product_name', default='Tails')
parser.add_argument('--version', default=None, required=True,
help='Version of Tails .')
parser.add_argument('--img', default=None,
parser.add_argument('--img', default=None, required=True,
help='Path to the USB image.')
parser.add_argument('--iso', default=None, required=True,
help='Path to the ISO file.')
......
......@@ -41,6 +41,7 @@ sed -i -e "s/Boot menu//" "${CFG_FILE}"
sed -i -e "s/menu label Live/menu label Tails/" "${SYSLINUX_PATH}"/live*.cfg
sed -i -r -e 's/(menu label .* )\(failsafe\)/\1(Troubleshooting Mode)/' \
"${SYSLINUX_PATH}"/live*.cfg
sed -i -r -e 's/(append .*)/\1\n\tsysappend 0x40000/g' "${SYSLINUX_PATH}"/live*.cfg
cat > "${SYSLINUX_PATH}/tails.cfg" << EOF
menu color sel * #ffffffff #55555555 *
......
......@@ -7,7 +7,7 @@ echo "Configuring compression of the initramfs"
# Import ensure_hook_dependency_is_installed()
. /usr/local/lib/tails-shell-library/build.sh
ensure_hook_dependency_is_installed initramfs-tools xz-utils
ensure_hook_dependency_is_installed initramfs-tools xz-utils fatresize
# Compress the initramfs using a more size-wise efficient algorithm.
......
#!/bin/sh -e
PREREQS=""
prereqs() { echo "$PREREQS"; }
case "$1" in
prereqs)
prereqs
exit 0
;;
esac
. /usr/share/initramfs-tools/hook-functions
copy_exec /sbin/sgdisk /sbin
copy_exec /sbin/partprobe /sbin
copy_exec /usr/sbin/fatresize /sbin
copy_exec /usr/bin/mlabel /sbin
# Needed by mlabel that uses iconv_open
copy_exec /usr/lib/x86_64-linux-gnu/gconv/IBM850.so
copy_file "regular file" /usr/lib/x86_64-linux-gnu/gconv/gconv-modules
#!/bin/sh
PREREQS="plymouth"
prereqs() { echo "$PREREQS"; }
case "$1" in
prereqs)
prereqs
exit 0
;;
esac
set -eu
# Print executed commands for debugging
if [ -n "$debug" ]; then
set -x
fi
. /scripts/functions
if [ -z "${FSUUID:-}" ]; then
echo "\$FSUUID is unset, probably because the boot medium is an ISO, and not a USB image. Skipping partitioning."
exit 0
fi
# Wait for system partition
log_begin_msg "Waiting for system partition to become available"
success=
i=0
while [ "$i" -lt 300 ]; do
if [ -b "/dev/disk/by-uuid/${FSUUID}" ]; then
success=1
break
fi
sleep 0.2
i="$(($i + 1))"
done
if [ -n "$success" ]; then
log_end_msg
else
log_failure_msg "Reached timeout waiting for system partition. Skipping partitioning."
# The exit code is irrelevant in the current implementation of live-config
exit 1
fi
# Get parent device
# XXX: We assume here that the parent device is also ready if the
# system partition is.
SYSTEM_PARTITION=$(readlink -f "/dev/disk/by-uuid/${FSUUID}")
PARENT_PATH=$(udevadm info --query=property --name="${SYSTEM_PARTITION}" \
| sed -n '/^ID_PATH=/ s/^ID_PATH=// p')
PARENT_DEVICE=$(readlink -f "/dev/disk/by-path/${PARENT_PATH}")
DEVICE_NAME=$(basename "${PARENT_DEVICE}")
# Check if this is first boot
GUID=$(sgdisk --print "${PARENT_DEVICE}" \
| sed -n '/^Disk identifier (GUID)/ s/^Disk identifier (GUID): // p')
if [ "${GUID}" != "17B81DA0-8B1E-4269-9C39-FE5C7B9B58A3" ]; then
_log_msg "Skipping partitioning because this is not the first boot"
exit 0
fi
DEVICE_TOO_SMALL_ERROR_MESSAGE="Sorry, this device is too small to run Tails. Please use a device with at least 8 GiB. Press ENTER to shut down."
# Get new system partition size
DEVICE_SIZE=$(cat "/sys/block/${DEVICE_NAME}/size")
DEVICE_SIZE_IN_MiB=$((${DEVICE_SIZE} / 2 / 1024))
if [ "${DEVICE_SIZE_IN_MiB}" -lt 7200 ]; then
plymouth message --text="${DEVICE_TOO_SMALL_ERROR_MESSAGE}"
plymouth watch-keystroke > /dev/null
poweroff -f
elif [ "${DEVICE_SIZE_IN_MiB}" -lt 14500 ]; then
SYSTEM_PARTITION_SIZE=4096M
else
SYSTEM_PARTITION_SIZE=8192M
fi
# Update partition table. This includes the following operations:
# * Move GPT backup header to the end of the device
# * Set a random disk GUID and partition GUID
# * Delete the old system partition
# * Create a new system partition of size ${SYSTEM_PARTITION_SIZE}
# * Set the partition label to "Tails"
ESP_GUID="C12A7328-F81F-11D2-BA4B-00A0C93EC93B"
log_begin_msg "Updating partition table"
sgdisk \
--move-second-header \
--randomize-guids \
--delete=1 \
--new=1:0:+"${SYSTEM_PARTITION_SIZE}" \
--typecode="1:${ESP_GUID}" \
--change-name=1:Tails \
"${PARENT_DEVICE}"
log_end_msg
# Tell the kernel to reload the partition table
partprobe
# fatresize overwrites the VBR, so we have to back it up to be able to
# restore the boot code later
dd if="${SYSTEM_PARTITION}" of=/tmp/vbr bs=512 count=1
# Grow the filesystem
# Note that fatresize resets partition attributes
# fatresize uses "Mi" for MiB, so we have to append an "i"
FS_SIZE="${SYSTEM_PARTITION_SIZE}"i
fatresize --size="${FS_SIZE}" "${SYSTEM_PARTITION}"
# Restore boot code overwritten by fatresize
dd if=/tmp/vbr of="${SYSTEM_PARTITION}" bs=1 skip=90 seek=90 count=414
# Restore JMP instruction which jumps to the bootcode
dd if=/tmp/vbr of="${SYSTEM_PARTITION}" bs=3 count=1
# Set a random filesystem UUID (aka. FAT "Volume ID" / "serial number")
MTOOLS_SKIP_CHECK=1 mlabel -i "${SYSTEM_PARTITION}" -n ::Tails
# Set the following attributes on the system partition (we have to
# set them after running fatresize, because fatresize resets them):
# 0: system partition
# 2: legacy BIOS bootable
# 60: read-only
# 62: hidden
# 63: do not automount
sgdisk \
--attributes=1:set:0 \
--attributes=1:set:2 \
--attributes=1:set:60 \
--attributes=1:set:62 \
--attributes=1:set:63 \
"${PARENT_DEVICE}"
# Tell the kernel to reload the partition table
partprobe
......@@ -424,3 +424,6 @@ python3-sh
# For Unlock VeraCrypt Volumes
gir1.2-gudev-1.0
# For partitioning script in initramfs
fatresize
......@@ -64,14 +64,11 @@ def checkpoints
'usb-install-tails-greeter' => {
:description => "I have started Tails without network from a USB drive without a persistent partition and stopped at Tails Greeter's login screen",
:parent_checkpoint => 'no-network-logged-in',
:parent_checkpoint => nil,
:steps => [
'I create a 7200 MiB disk named "__internal"',
'I plug USB drive "__internal"',
'I install Tails to USB drive "__internal" by cloning',
'the running Tails is installed on USB drive "__internal"',
'there is no persistence partition on USB drive "__internal"',
'I shutdown Tails and wait for the computer to power off',
'I write the Tails USB image to disk "__internal"',
'I start Tails from USB drive "__internal" with network unplugged',
'the boot device has safe access rights',
'Tails is running from USB drive "__internal"',
......
......@@ -29,9 +29,10 @@ Given /^I create an?( (\d+) ([[:alpha:]]+))? ([[:alnum:]]+) partition( labeled "
$vm.storage.disk_mkpartfs(name, parttype, fstype, opts)
end
Given /^I write the Tails ISO image to disk "([^"]+)"$/ do |name|
Given /^I write (|an old version of )the Tails (ISO|USB) image to disk "([^"]+)"$/ do |old, type, name|
src_disk = {
:path => TAILS_ISO,
:path => (old == '' ? (type == 'ISO' ? TAILS_ISO : TAILS_IMG)
: (type == 'ISO' ? OLD_TAILS_ISO : OLD_TAILS_IMG)),
:opts => {
:format => "raw",
:readonly => true
......
......@@ -274,7 +274,7 @@ def check_disk_integrity(name, dev, scheme)
"Unexpected partition scheme on USB drive '#{name}', '#{dev}'")
end
def check_part_integrity(name, dev, usage, fs_type, part_label, part_type = nil)
def check_part_integrity(name, dev, usage, fs_type, part_label = nil, part_type = nil)
info = $vm.execute("udisksctl info --block-device '#{dev}'").stdout
info_split = info.split("\n org\.freedesktop\.UDisks2\.Partition:\n")
dev_info = info_split[0]
......@@ -283,8 +283,10 @@ def check_part_integrity(name, dev, usage, fs_type, part_label, part_type = nil)
"Unexpected device field 'usage' on USB drive '#{name}', '#{dev}'")
assert(dev_info.match("^ IdType: +#{fs_type}$"),
"Unexpected device field 'IdType' on USB drive '#{name}', '#{dev}'")
assert(part_info.match("^ Name: +#{part_label}$"),
"Unexpected partition label on USB drive '#{name}', '#{dev}'")
if part_label
assert(part_info.match("^ Name: +#{part_label}$"),
"Unexpected partition label on USB drive '#{name}', '#{dev}'")
end
if part_type
assert(part_info.match("^ Type: +#{part_type}$"),
"Unexpected partition type on USB drive '#{name}', '#{dev}'")
......@@ -453,6 +455,32 @@ def boot_device_type
device_info(boot_device)['ID_BUS']
end
# Turn udisksctl info output into something more manipulable:
def parse_udisksctl_info(input)
tree = {}
section = nil
key = nil
input.chomp.split("\n").each { |line|
case line
when /^\/org\/freedesktop\/UDisks2\/block_devices\//
# no-op, ignore first line = device
when /^ (org\.freedesktop\.UDisks2\..+):$/
section = $1
tree[section] = {}
when /^\s+(.+?):\s+(.+)$/
key = $1
value = $2
tree[section][key] = value
else
# XXX: Best effort = consider this a continuation from previous
# line (e.g. Symlinks), and add the whole line, without
# stripping anything (e.g. leading whitespaces)
tree[section][key] += line
end
}
return tree
end
Then /^Tails is running from (.*) drive "([^"]+)"$/ do |bus, name|
bus = bus.downcase