Skip to content
GitLab
Commit 8b1452be authored by intrigeri's avatar intrigeri
Browse files

Merge branch 'feature/15292-usb-image' into doc/15999-integrate-usb-image-in-the-release-process

parents 5c80b335 64d58e43
Expand all Hide whitespace changes
Inline Side-by-side
auto/build
View file @ 8b1452be
......@@ -98,6 +98,7 @@ DEBOOTSTRAP_OPTIONS="${DEBOOTSTRAP_OPTIONS:-} --no-merged-usr"
# use our own APT repository's key:
DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
gpg --homedir "$DEBOOTSTRAP_GNUPG_HOMEDIR" \
--no-tty \
--import config/chroot_sources/tails.chroot.gpg
if [ -e "$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.gpg" ]; then
DEBOOTSTRAP_GNUPG_KEYRING="$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.gpg"
......
auto/scripts/create-usb-image-from-iso
View file @ 8b1452be
......@@ -4,7 +4,7 @@ import argparse
import os
import logging
from contextlib import contextmanager
import re
import tempfile
import time
import subprocess
......@@ -30,7 +30,6 @@ PARTITION_LABEL = 'Tails'
FILESYSTEM_LABEL = 'Tails'
GET_UDISKS_OBJECT_TIMEOUT = 2
WAIT_FOR_PARTITION_TIMEOUT = 2
# The size of the system partition (in MiB) will be:
#
......@@ -56,7 +55,7 @@ class ImageCreator(object):
self._loop_device = None # type: str
self._partition = None # type: str
self._system_partition_size = None # type: int
self.mountpoint = None # type: str
self.iso_mountpoint = None # type: str
@property
def loop_device(self) -> UDisks.ObjectProxy:
......@@ -101,17 +100,19 @@ class ImageCreator(object):
with self.setup_loop_device():
self.create_gpt()
self.create_partition()
# udisks' create_partition function seems to ignore arg_type
# in Stretch, so we set it via sgdisk.
# XXX:Buster: Remove set_partition_type
self.set_partition_type()
self.set_partition_flags()
# XXX: Rescan?
self.format_partition()
with self.mount_partition():
with self.mount_iso():
self.extract_iso()
self.set_permissions()
self.update_configs()
self.adjust_syslinux_configuration()
self.install_mbr()
self.copy_syslinux_modules()
# We have to install syslinux after the partition was unmounted.
# This sleep is a workaround for a race condition which causes the
# syslinux installation to return without errors, even though the
# bootloader isn't actually installed
......@@ -122,12 +123,20 @@ class ImageCreator(object):
self.set_guids()
self.set_fsuuid()
with self.mount_partition():
self.reset_timestamps()
def extract_iso(self):
logger.info("Extracting ISO contents to the partition")
execute(['7z', 'x', self.iso, '-x![BOOT]', '-y', '-o%s' % self.mountpoint])
for root, dirs, files in os.walk(self.iso_mountpoint):
for d in sorted(dirs):
with set_env("MTOOLS_SKIP_CHECK", "1"):
execute(['mmd',
"-i", self.partition.props.block.props.device,
"::%s" % os.path.join(os.path.relpath(root, start=self.iso_mountpoint), d)])
for f in sorted(files):
with set_env("MTOOLS_SKIP_CHECK", "1"):
execute(['mcopy',
"-i", self.partition.props.block.props.device,
os.path.join(root, f),
"::%s" % os.path.join(os.path.relpath(root, start=self.iso_mountpoint), f)])
def create_empty_image(self):
logger.info("Creating empty image %r", self.image)
......@@ -145,7 +154,6 @@ class ImageCreator(object):
arg_fd=GLib.Variant('h', 0),
arg_options=GLib.Variant('a{sv}', None),
fd_list=Gio.UnixFDList.new_from_array([image_fd]),
cancellable=None,
)
if not resulting_device:
......@@ -160,15 +168,25 @@ class ImageCreator(object):
logger.info("Tearing down loop device")
self.loop_device.props.loop.call_delete_sync(
arg_options=GLib.Variant('a{sv}', None),
cancellable=None,
)
@contextmanager
def mount_iso(self):
with tempfile.TemporaryDirectory() as mountpoint:
logger.info("Mounting ISO on %s" % mountpoint)
execute(['mount', '-o', 'loop,ro', self.iso, mountpoint])
self.iso_mountpoint = mountpoint
try:
yield mountpoint
finally:
logger.info("Unmounting ISO")
execute(['umount', mountpoint])
def create_gpt(self):
logger.info("Creating GPT")
self.loop_device.props.block.call_format_sync(
arg_type='gpt',
arg_options=GLib.Variant('a{sv}', None),
cancellable=None
arg_options=GLib.Variant('a{sv}', None)
)
def create_partition(self):
......@@ -178,8 +196,7 @@ class ImageCreator(object):
arg_size=self.system_partition_size * 2**20,
arg_type=ESP_GUID,
arg_name=PARTITION_LABEL,
arg_options=GLib.Variant('a{sv}', None),
cancellable=None
arg_options=GLib.Variant('a{sv}', None)
)
# XXX: Tails Installer ignores GLib errors here
......@@ -187,124 +204,64 @@ class ImageCreator(object):
self._partition = partition
def set_partition_flags(self):
logger.info("Setting partition flags")
# We use sgdisk directly instead of udisks' set_flags method, because the
# latter sometimes resets the partition type / partition table type
# before Buster's udisks2 + libblockdev 2.15-1
# (https://github.com/storaged-project/udisks/issues/418)
execute(["/sbin/sgdisk", "--attributes=1:=:%x" % SYSTEM_PARTITION_FLAGS, self.image])
start_time = time.perf_counter()
while time.perf_counter() - start_time < WAIT_FOR_PARTITION_TIMEOUT:
try:
self.partition.props.partition.call_set_flags_sync(
arg_flags=SYSTEM_PARTITION_FLAGS,
arg_options=GLib.Variant('a{sv}', None),
cancellable=None
)
except GLib.Error as e:
if "GDBus.Error:org.freedesktop.DBus.Error.UnknownMethod: No such interface" in e.message:
time.sleep(0.1)
continue
raise
return
def set_partition_type(self):
execute(["/sbin/sgdisk", "--typecode=1:%s" % ESP_GUID, self.image])
def format_partition(self):
logger.info("Formatting partition")
options = GLib.Variant('a{sv}', {
'label': GLib.Variant('s', FILESYSTEM_LABEL),
'update-partition-type': GLib.Variant('b', False)
})
self.partition.props.block.call_format_sync(
arg_type='vfat',
arg_options=options,
cancellable=None
)
@contextmanager
def mount_partition(self):
logger.info("Mounting partition")
try:
self.mountpoint = self.partition.props.filesystem.call_mount_sync(
arg_options=GLib.Variant('a{sv}', None),
cancellable=None
)
except GLib.Error as e:
if "org.freedesktop.UDisks2.Error.AlreadyMounted" in e.message and \
self.partition.props.filesystem.props.mount_points:
self.mountpoint = self.partition.props.filesystem.props.mount_points[0]
logger.info("Partition is already mounted at {}".format(self.mountpoint))
else:
raise
try:
yield
finally:
logger.info("Unmounting partition")
self.partition.props.filesystem.call_unmount_sync(
arg_options=GLib.Variant('a{sv}', {'force': GLib.Variant('b', True)}),
cancellable=None,
)
def set_permissions(self):
logger.info("Setting file access permissions")
for root, dirs, files in os.walk(self.mountpoint):
for d in dirs:
os.chmod(os.path.join(root, d), 0o755)
for f in files:
os.chmod(os.path.join(root, f), 0o644)
def update_configs(self):
logger.info("Updating config files")
grubconf = os.path.join(self.mountpoint, "EFI", "BOOT", "grub.conf")
bootconf = os.path.join(self.mountpoint, "EFI", "BOOT", "boot.conf")
isolinux_dir = os.path.join(self.mountpoint, "isolinux")
syslinux_dir = os.path.join(self.mountpoint, "syslinux")
isolinux_cfg = os.path.join(syslinux_dir, "isolinux.cfg")
files_to_update = [
(os.path.join(self.mountpoint, "isolinux", "isolinux.cfg"),
os.path.join(self.mountpoint, "isolinux", "syslinux.cfg")),
(os.path.join(self.mountpoint, "isolinux", "stdmenu.cfg"),
os.path.join(self.mountpoint, "isolinux", "stdmenu.cfg")),
(os.path.join(self.mountpoint, "isolinux", "exithelp.cfg"),
os.path.join(self.mountpoint, "isolinux", "exithelp.cfg")),
(os.path.join(self.mountpoint, "EFI", "BOOT", "isolinux.cfg"),
os.path.join(self.mountpoint, "EFI", "BOOT", "syslinux.cfg")),
(grubconf, bootconf)
]
for (infile, outfile) in files_to_update:
if os.path.exists(infile):
self.update_config(infile, outfile)
if os.path.exists(isolinux_dir):
execute(["mv", isolinux_dir, syslinux_dir])
if os.path.exists(isolinux_cfg):
os.remove(isolinux_cfg)
def update_config(self, infile, outfile):
with open(infile) as f_in:
lines = [re.sub('/isolinux/', '/syslinux/', line) for line in f_in]
with open(outfile, "w") as f_out:
f_out.writelines(lines)
execute([
'mkfs.msdos',
'-v',
# Use constants for normally randomly generated or time-based data
# such as volume ID and creation time
'--invariant',
# Fill all 11 chars of the volume label to avoid any uninitialized
# memory from sneaking in
'-n', 'TAILS' + 6 * ' ',
self.partition.props.block.props.device,
])
def adjust_syslinux_configuration(self):
logger.info("Adjusting syslinux configuration")
with set_env("MTOOLS_SKIP_CHECK", "1"):
execute(['mren', "-i", self.partition.props.block.props.device,
"::isolinux", "::syslinux"])
execute(['mren', "-i", self.partition.props.block.props.device,
"::syslinux/isolinux.cfg", "::syslinux/syslinux.cfg"])
def install_mbr(self):
logger.info("Installing MBR")
mbr_path = os.path.join(self.mountpoint, "utils/mbr/mbr.bin")
execute(["dd", "bs=440", "count=1", "conv=notrunc", "if=%s" % mbr_path, "of=%s" % self.image])
# Only required if using the running system's syslinux instead of the one on the ISO
mbr_path = os.path.join(self.iso_mountpoint, "utils/mbr/mbr.bin")
execute(["dd", "bs=440", "count=1", "conv=notrunc",
"if=%s" % mbr_path, "of=%s" % self.image])
# Ensure the syslinux modules come from the same package as the
# binary we'll use in install_syslinux, i.e. the system's one
# and not the one that's in the ISO; otherwise their ABI may
# be incompatible
def copy_syslinux_modules(self):
logger.info("Copying syslinux modules to device")
syslinux_dir = os.path.join(self.mountpoint, 'syslinux')
syslinux_dir = os.path.join(self.iso_mountpoint, 'isolinux')
com32modules = [f for f in os.listdir(syslinux_dir) if f.endswith('.c32')]
for module in com32modules:
for module in sorted(com32modules):
src_path = os.path.join(SYSLINUX_COM32MODULES_DIR, module)
if not os.path.isfile(src_path):
raise ImageCreationError("Could not find the '%s' COM32 module" % module)
logger.debug('Copying %s to the device' % src_path)
execute(["cp", "-a", src_path, os.path.join(syslinux_dir, module)])
with set_env("MTOOLS_SKIP_CHECK", "1"):
execute(['mcopy', '-D', 'o',
'-i', self.partition.props.block.props.device,
src_path,
"::%s" % os.path.join('syslinux', module)])
def install_syslinux(self):
logger.info("Installing bootloader")
......@@ -316,16 +273,7 @@ class ImageCreator(object):
'--offset', str(self.partition.props.partition.props.offset),
'--directory', '/syslinux/',
'--install', self.image
],
as_root=True # XXX: Why does this only work as root?
)
def reset_timestamps(self):
logger.info("Resetting timestamps")
for root, dirs, files in os.walk(self.mountpoint):
os.utime(root, (0, 0), follow_symlinks=False)
for file in files:
os.utime(os.path.join(root, file), (0, 0), follow_symlinks=False)
])
def set_guids(self):
logger.info("Setting disk and partition GUID")
......@@ -336,18 +284,19 @@ class ImageCreator(object):
"""Set a fixed filesystem UUID aka. FAT Volume ID / serial number"""
logger.info("Setting filesystem UUID")
with set_env("MTOOLS_SKIP_CHECK", "1"):
execute(["mlabel", "-i", self.partition.props.block.props.device, "-N", "a69020d2"])
execute(["mlabel", "-i", self.partition.props.block.props.device,
"-N", "a69020d2",
# Otherwise mlabel -N will remove the pre-existing label
"::%s" % FILESYSTEM_LABEL])
def execute(cmd: list, as_root=False):
if as_root and os.geteuid() != 0:
cmd = ['pkexec'] + cmd
def execute(cmd: list):
logger.info("Executing '%s'" % ' '.join(cmd))
subprocess.check_call(cmd)
@contextmanager
def set_env(name: str, value:str):
def set_env(name: str, value: str):
old_value = os.getenv(name)
os.putenv(name, value)
try:
......@@ -374,6 +323,9 @@ def main():
if not args.ISO.endswith(".iso"):
parser.error("Input file is not an ISO (no .iso extension)")
if os.geteuid() != 0:
raise PermissionError("This script must be run as root")
logging.basicConfig(level=logging.INFO)
logging.getLogger('sh').setLevel(logging.WARNING)
......
bin/idf-content 0 → 100755
View file @ 8b1452be
#! /usr/bin/python3
import hashlib
import io
import json
import sys
from pathlib import Path, PurePath
def sha256_file(filename):
sha256 = hashlib.sha256()
with io.open(filename, mode="rb") as fd:
content = fd.read()
sha256.update(content)
return sha256.hexdigest()
def to_json(data):
return json.dumps(data, indent=4, sort_keys=True)
def target_file_url(channel, filename):
basename = PurePath(filename).name
return '%(baseurl)s/%(channel)s/%(subdir)s/%(basename)s' % {
'baseurl': 'http://dl.amnesia.boum.org/tails',
'channel': channel,
'subdir': PurePath(basename).stem,
'basename': basename,
}
def idf_content(build_target, channel, product_name, version, img, iso):
installation_paths = [
{
'type': 'iso',
'target-files': [{
'url': target_file_url(channel, iso),
'sha256': sha256_file(iso),
'size': Path(iso).stat().st_size,
}],
},
]
if img is not None:
installation_paths += {
'type': 'img',
'target-files': [{
'url': target_file_url(channel, img),
'sha256': sha256_file(img),
'size': Path(img).stat().st_size,
}],
}
return to_json({
'build_target': build_target,
'channel': channel,
'product-name': product_name,
'installations': [{
'version': version,
'installation-paths': installation_paths,
}],
})
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--build-target', dest='build_target', default='amd64')
parser.add_argument('--channel', default='stable')
parser.add_argument('--product-name', dest='product_name', default='Tails')
parser.add_argument('--version', default=None, required=True,
help='Version of Tails .')
parser.add_argument('--img', default=None,
help='Path to the USB image.')
parser.add_argument('--iso', default=None, required=True,
help='Path to the ISO file.')
args = parser.parse_args()
print(idf_content(build_target=args.build_target,
channel=args.channel,
product_name=args.product_name,
version=args.version,
img=args.img,
iso=args.iso))
config/amnesia
View file @ 8b1452be
......@@ -26,7 +26,7 @@ AMNESIA_ISOHYBRID_OPTS="-h 255 -s 63 --id 42 --verbose"
REQUIRED_SYSLINUX_UTILS_UPSTREAM_VERSION="6.03~pre20"
# Kernel version
KERNEL_VERSION='4.18.0-2'
KERNEL_VERSION='4.18.0-3'
KERNEL_SOURCE_VERSION=$(
echo "$KERNEL_VERSION" \
| perl -p -E 's{\A (\d+ [.] \d+) [.] .*}{$1}xms'
......
config/binary_local-hooks/10-syslinux_customize
View file @ 8b1452be
......@@ -41,6 +41,7 @@ sed -i -e "s/Boot menu//" "${CFG_FILE}"
sed -i -e "s/menu label Live/menu label Tails/" "${SYSLINUX_PATH}"/live*.cfg
sed -i -r -e 's/(menu label .* )\(failsafe\)/\1(Troubleshooting Mode)/' \
"${SYSLINUX_PATH}"/live*.cfg
sed -i -r -e 's/(append .*)/\1\n\tsysappend 0x40000/g' "${SYSLINUX_PATH}"/live*.cfg
cat > "${SYSLINUX_PATH}/tails.cfg" << EOF
menu color sel * #ffffffff #55555555 *
......
config/binary_rootfs/squashfs.sort
View file @ 8b1452be
This diff is collapsed.
config/chroot_local-hooks/04-change-gids-and-uids
View file @ 8b1452be
......@@ -2,9 +2,17 @@
set -e
# Free the fixed GIDs and UIDs we're using.
echo "Set fixed GIDs and UIDs"
echo "Change GIDs and UIDs"
Debug_gids_and_uids () {
for file in passwd group; do
diff -Naur "/usr/share/tails/build/${file}" "/etc/${file}" >&2 || :
echo >&2
echo "Content of '/etc/${file}':" >&2
cat "/etc/${file}" >&2
echo >&2
done
}
Change_uid () {
NAME="$1"
......@@ -13,7 +21,10 @@ Change_uid () {
if [ -n "$OLD" ]; then
echo "Changing UID for $NAME ($OLD -> $NEW)"
usermod --uid "$NEW" "$NAME"
if ! usermod --uid "$NEW" "$NAME"; then
Debug_gids_and_uids
exit 1
fi
find / -wholename /proc -prune -o \( \! -type l -a -uid "$OLD" -exec chown "$NEW" '{}' \; \)
fi
}
......@@ -25,27 +36,48 @@ Change_gid () {
if [ -n "$OLD" ]; then
echo "Changing GID for $NAME ($OLD -> $NEW)"
groupmod --gid "$NEW" "$NAME"
if ! groupmod --gid "$NEW" "$NAME"; then
Debug_gids_and_uids
exit 1
fi
find / -wholename /proc -prune -o \( \! -type l -a -gid "$OLD" -exec chgrp "$NEW" '{}' \; \)
fi
}
Change_uid tails-persistent-setup 150
Change_gid tails-persistent-setup 150
### Ensure GIDs are stable accross releases
# ... otherwise, things such as tor@service are broken
# after applying an automatic upgrade (#15695, #15424, #13426, #15407)
# Temporarily give these groups a GID that's out of the way, to avoid collisions
Change_gid messagebus 1050
Change_gid ssh 1090
Change_gid memlockd 1100
Change_gid ssl-cert 1110
Change_gid vboxsf 1120
Change_gid monkeysphere 1130
Change_gid debian-tor 1140
Change_gid lpadmin 1150
Change_gid scanner 1160
Change_gid colord 1170
Change_gid saned 1180
Change_gid pulse 1190
Change_gid pulse-access 1200
Change_gid Debian-gdm 1210
Change_gid kvm 1500
# Finally, give these groups the desired GID
Change_gid messagebus 105
Change_gid ssh 109
Change_gid memlockd 110
Change_gid ssl-cert 111
Change_gid vboxsf 112
Change_gid monkeysphere 113
Change_gid debian-tor 114
Change_gid lpadmin 115
Change_gid scanner 116
Change_gid colord 117
Change_gid saned 118
Change_gid pulse 119
Change_gid pulse-access 120
Change_gid Debian-gdm 121
Change_gid kvm 150
config/chroot_local-hooks/05-adduser_tails-persistence-setup
View file @ 8b1452be
......@@ -11,5 +11,8 @@ set -e
echo "Creating the tails-persistence-setup user"
# If this fails because UID 115 or GID 122 are already in use,
# move the "stealer" user/group out of the way in 04-change-gids-and-uids.
addgroup --system --gid 122 tails-persistence-setup
adduser --system --uid 115 --gid 122 --no-create-home tails-persistence-setup
config/chroot_local-hooks/09-torsocks-apps
View file @ 8b1452be
......@@ -4,9 +4,9 @@ set -e
echo "Wrapping some applications with torsocks"
APPS="gobby-0.5 net.sourceforge.liferea openpgp-applet seahorse"
APPS="gobby-0.5 openpgp-applet seahorse"
DBUS_SERVICES="org.gnome.seahorse.Application org.fedoraproject.Config.Printing"
WRAPPED_DBUS_SERVICES="net.sourceforge.liferea"
WRAPPED_DBUS_SERVICES=""
for app in $APPS; do
sed -i'' --regexp-extended 's,^Exec=(.*),Exec=torsocks \1,' \
......
config/chroot_local-hooks/99-initramfs-compress
View file @ 8b1452be
......@@ -7,7 +7,7 @@ echo "Configuring compression of the initramfs"
# Import ensure_hook_dependency_is_installed()
. /usr/local/lib/tails-shell-library/build.sh
ensure_hook_dependency_is_installed initramfs-tools xz-utils
ensure_hook_dependency_is_installed initramfs-tools xz-utils fatresize
# Compress the initramfs using a more size-wise efficient algorithm.
......
config/chroot_local-includes/usr/local/bin/liferea deleted 100755 → 0
View file @ 5c80b335
#!/usr/bin/env python3
"""
Tails Liferea wrapper
"""
import os
import sh
import sys
from gettext import gettext
LIFEREA = '/usr/bin/liferea'
os.environ['TEXTDOMAIN'] = 'tails'
def main():
disabled_text = gettext('Liferea is deprecated')
question_text = gettext('Do you wish to start Liferea anyway?')
warning_text = gettext(
"Due to security concerns the Liferea feed reader will be removed "
"from Tails by the end of 2018. Please migrate your feeds to "
"Thunderbird."
)