Commit c478dfe3 authored by intrigeri's avatar intrigeri
Browse files

Merge branch 'feature/15292-generate-usb-image'

parents bb025b01 e2bade70
......@@ -314,7 +314,7 @@ end
def list_artifacts
user = vagrant_ssh_config('User')
stdout = capture_vagrant_ssh("find '/home/#{user}/amnesia/' -maxdepth 1 " +
"-name 'tails-*.iso*'").first
"-name 'tails-amd64-*'").first
stdout.split("\n")
rescue VagrantCommandError
return Array.new
......
......@@ -122,8 +122,6 @@ export MKSQUASHFS_OPTIONS
case "$LB_BINARY_IMAGES" in
iso)
BUILD_FILENAME_EXT=iso
BUILD_FILENAME=binary
which isohybrid >/dev/null || fatal 'Cannot find isohybrid in $PATH'
installed_syslinux_utils_upstream_version="$(syslinux_utils_upstream_version)"
if dpkg --compare-versions \
......@@ -135,27 +133,16 @@ case "$LB_BINARY_IMAGES" in
"while we need at least '${REQUIRED_SYSLINUX_UTILS_UPSTREAM_VERSION}'."
fi
;;
iso-hybrid)
BUILD_FILENAME_EXT=iso
BUILD_FILENAME=binary-hybrid
;;
tar)
BUILD_FILENAME_EXT=tar.gz
BUILD_FILENAME=binary-tar
;;
usb-hdd)
BUILD_FILENAME_EXT=img
BUILD_FILENAME=binary
;;
*)
fatal "Image type ${LB_BINARY_IMAGES} is not supported."
;;
esac
BUILD_DEST_FILENAME="${BUILD_BASENAME}.${BUILD_FILENAME_EXT}"
BUILD_MANIFEST="${BUILD_DEST_FILENAME}.build-manifest"
BUILD_APT_SOURCES="${BUILD_DEST_FILENAME}.apt-sources"
BUILD_PACKAGES="${BUILD_DEST_FILENAME}.packages"
BUILD_LOG="${BUILD_DEST_FILENAME}.buildlog"
BUILD_ISO_FILENAME="${BUILD_BASENAME}.iso"
BUILD_MANIFEST="${BUILD_BASENAME}.build-manifest"
BUILD_APT_SOURCES="${BUILD_BASENAME}.apt-sources"
BUILD_PACKAGES="${BUILD_BASENAME}.packages"
BUILD_LOG="${BUILD_BASENAME}.buildlog"
BUILD_USB_IMAGE_FILENAME="${BUILD_BASENAME}.img"
# Clone all output, from this point on, to the log file
exec > >(tee -a "$BUILD_LOG")
......@@ -172,27 +159,25 @@ trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
cat config/chroot_sources/*.chroot
) > "$BUILD_APT_SOURCES"
echo "Building $LB_BINARY_IMAGES image ${BUILD_BASENAME}..."
set -o pipefail
echo "Building ISO image ${BUILD_ISO_FILENAME}..."
time lb build noauto ${@}
RET=$?
if [ -e "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" ]; then
echo "Image was successfully created"
[ "$RET" -eq 0 ] || \
echo "Warning: lb build exited with code $RET"
if [ "$LB_BINARY_IMAGES" = iso ]; then
ISO_FILE="${BUILD_FILENAME}.${BUILD_FILENAME_EXT}"
print_iso_size "$ISO_FILE"
echo "Hybriding it..."
isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE" || fatal "isohybrid failed"
print_iso_size "$ISO_FILE"
truncate -s %2048 "$ISO_FILE"
print_iso_size "$ISO_FILE"
fi
echo "Renaming generated files..."
mv -i "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" "${BUILD_DEST_FILENAME}"
mv -i binary.packages "${BUILD_PACKAGES}"
generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
else
fatal "lb build failed ($?)."
fi
[ -e binary.iso ] || fatal "lb build failed ($?)."
echo "ISO image was successfully created"
print_iso_size binary.iso
echo "Hybriding it..."
isohybrid $AMNESIA_ISOHYBRID_OPTS binary.iso || fatal "isohybrid failed"
print_iso_size binary.iso
truncate -s %2048 binary.iso
print_iso_size binary.iso
echo "Renaming generated files..."
mv -i binary.iso "${BUILD_ISO_FILENAME}"
mv -i binary.packages "${BUILD_PACKAGES}"
echo "Generating build manifest..."
generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
echo "Creating USB image ${BUILD_USB_IMAGE_FILENAME}..."
create-usb-image-from-iso "${BUILD_ISO_FILENAME}"
#!/usr/bin/env python3
import argparse
import os
import logging
from contextlib import contextmanager
import re
import time
import subprocess
import gi
gi.require_version('UDisks', '2.0')
from gi.repository import UDisks, GLib, Gio
logger = logging.getLogger(__name__)
SYSTEM_PARTITION_FLAGS = (
1 << 0 | # system partition
1 << 2 | # legacy BIOS bootable
1 << 60 | # read-only
1 << 62 | # hidden
1 << 63 # do not automount
)
# EFI System Partition
ESP_GUID = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B'
PARTITION_LABEL = 'Tails'
FILESYSTEM_LABEL = 'Tails'
GET_UDISKS_OBJECT_TIMEOUT = 2
WAIT_FOR_PARTITION_TIMEOUT = 2
# The size of the system partition (in MiB) will be:
#
# SYSTEM_PARTITION_ADDITIONAL_SIZE + size of the ISO
#
# SYSTEM_PARTITION_ADDITIONAL_SIZE must be large enough to fit
# the partition table, reserved sectors, and filesystem metadata.
SYSTEM_PARTITION_ADDITIONAL_SIZE = 10
SYSLINUX_COM32MODULES_DIR = '/usr/lib/syslinux/modules/bios'
class ImageCreationError(Exception):
pass
class ImageCreator(object):
def __init__(self, iso: str, image: str, free_space: int):
self.iso = iso
self.image = image
self.free_space = free_space
self._loop_device = None # type: str
self._partition = None # type: str
self._system_partition_size = None # type: int
self.mountpoint = None # type: str
@property
def loop_device(self) -> UDisks.ObjectProxy:
if not self._loop_device:
raise ImageCreationError("Loop device not set up")
return self.try_getting_udisks_object(self._loop_device)
@property
def partition(self) -> UDisks.ObjectProxy:
if not self._partition:
raise ImageCreationError("Partition not created")
return self.try_getting_udisks_object(self._partition)
@property
def system_partition_size(self) -> int:
if self._system_partition_size is None:
self._system_partition_size = get_file_size(self.iso) + SYSTEM_PARTITION_ADDITIONAL_SIZE
return self._system_partition_size
def try_getting_udisks_object(self, object_path: str) -> UDisks.Object:
start_time = time.perf_counter()
while time.perf_counter() - start_time < GET_UDISKS_OBJECT_TIMEOUT:
with self.get_udisks_client() as udisks_client:
udisks_object = udisks_client.get_object(object_path)
if udisks_object:
return udisks_object
time.sleep(0.1)
raise ImageCreationError("Couldn't get UDisksObject for path '%s' (timeout: %s)" %
(object_path, GET_UDISKS_OBJECT_TIMEOUT))
@contextmanager
def get_udisks_client(self):
client = UDisks.Client().new_sync()
yield client
client.settle()
def create_image(self):
self.create_empty_image()
with self.setup_loop_device():
self.create_gpt()
self.create_partition()
self.set_partition_flags()
# XXX: Rescan?
self.format_partition()
with self.mount_partition():
self.extract_iso()
self.set_permissions()
self.update_configs()
self.install_mbr()
self.copy_syslinux_modules()
# We have to install syslinux after the partition was unmounted.
# This sleep is a workaround for a race condition which causes the
# syslinux installation to return without errors, even though the
# bootloader isn't actually installed
# XXX: Investigate and report this race condition
# Might it be https://bugs.chromium.org/p/chromium/issues/detail?id=508713 ?
time.sleep(1)
self.install_syslinux()
self.set_guids()
self.set_fsuuid()
with self.mount_partition():
self.reset_timestamps()
def extract_iso(self):
logger.info("Extracting ISO contents to the partition")
execute(['7z', 'x', self.iso, '-x![BOOT]', '-y', '-o%s' % self.mountpoint])
def create_empty_image(self):
logger.info("Creating empty image %r", self.image)
image_size = self.system_partition_size + self.free_space
execute(["dd", "if=/dev/zero", "of=%s" % self.image, "bs=1M", "count=%s" % image_size])
@contextmanager
def setup_loop_device(self):
logger.info("Setting up loop device")
with self.get_udisks_client() as udisks_client:
manager = udisks_client.get_manager()
image_fd = os.open(self.image, os.O_RDWR)
resulting_device, fd_list = manager.call_loop_setup_sync(
arg_fd=GLib.Variant('h', 0),
arg_options=GLib.Variant('a{sv}', None),
fd_list=Gio.UnixFDList.new_from_array([image_fd]),
cancellable=None,
)
if not resulting_device:
raise ImageCreationError("Failed to set up loop device")
logger.info("Loop device: %r", resulting_device)
self._loop_device = resulting_device
try:
yield
finally:
logger.info("Tearing down loop device")
self.loop_device.props.loop.call_delete_sync(
arg_options=GLib.Variant('a{sv}', None),
cancellable=None,
)
def create_gpt(self):
logger.info("Creating GPT")
self.loop_device.props.block.call_format_sync(
arg_type='gpt',
arg_options=GLib.Variant('a{sv}', None),
cancellable=None
)
def create_partition(self):
logger.info("Creating partition")
partition = self.loop_device.props.partition_table.call_create_partition_sync(
arg_offset=0,
arg_size=self.system_partition_size * 2**20,
arg_type=ESP_GUID,
arg_name=PARTITION_LABEL,
arg_options=GLib.Variant('a{sv}', None),
cancellable=None
)
# XXX: Tails Installer ignores GLib errors here
logger.info("Partition: %r", partition)
self._partition = partition
def set_partition_flags(self):
logger.info("Setting partition flags")
start_time = time.perf_counter()
while time.perf_counter() - start_time < WAIT_FOR_PARTITION_TIMEOUT:
try:
self.partition.props.partition.call_set_flags_sync(
arg_flags=SYSTEM_PARTITION_FLAGS,
arg_options=GLib.Variant('a{sv}', None),
cancellable=None
)
except GLib.Error as e:
if "GDBus.Error:org.freedesktop.DBus.Error.UnknownMethod: No such interface" in e.message:
time.sleep(0.1)
continue
raise
return
def format_partition(self):
logger.info("Formatting partition")
options = GLib.Variant('a{sv}', {
'label': GLib.Variant('s', FILESYSTEM_LABEL),
'update-partition-type': GLib.Variant('b', False)
})
self.partition.props.block.call_format_sync(
arg_type='vfat',
arg_options=options,
cancellable=None
)
@contextmanager
def mount_partition(self):
logger.info("Mounting partition")
try:
self.mountpoint = self.partition.props.filesystem.call_mount_sync(
arg_options=GLib.Variant('a{sv}', None),
cancellable=None
)
except GLib.Error as e:
if "org.freedesktop.UDisks2.Error.AlreadyMounted" in e.message and \
self.partition.props.filesystem.props.mount_points:
self.mountpoint = self.partition.props.filesystem.props.mount_points[0]
logger.info("Partition is already mounted at {}".format(self.mountpoint))
else:
raise
try:
yield
finally:
logger.info("Unmounting partition")
self.partition.props.filesystem.call_unmount_sync(
arg_options=GLib.Variant('a{sv}', {'force': GLib.Variant('b', True)}),
cancellable=None,
)
def set_permissions(self):
logger.info("Setting file access permissions")
for root, dirs, files in os.walk(self.mountpoint):
for d in dirs:
os.chmod(os.path.join(root, d), 0o755)
for f in files:
os.chmod(os.path.join(root, f), 0o644)
def update_configs(self):
logger.info("Updating config files")
grubconf = os.path.join(self.mountpoint, "EFI", "BOOT", "grub.conf")
bootconf = os.path.join(self.mountpoint, "EFI", "BOOT", "boot.conf")
isolinux_dir = os.path.join(self.mountpoint, "isolinux")
syslinux_dir = os.path.join(self.mountpoint, "syslinux")
isolinux_cfg = os.path.join(syslinux_dir, "isolinux.cfg")
files_to_update = [
(os.path.join(self.mountpoint, "isolinux", "isolinux.cfg"),
os.path.join(self.mountpoint, "isolinux", "syslinux.cfg")),
(os.path.join(self.mountpoint, "isolinux", "stdmenu.cfg"),
os.path.join(self.mountpoint, "isolinux", "stdmenu.cfg")),
(os.path.join(self.mountpoint, "isolinux", "exithelp.cfg"),
os.path.join(self.mountpoint, "isolinux", "exithelp.cfg")),
(os.path.join(self.mountpoint, "EFI", "BOOT", "isolinux.cfg"),
os.path.join(self.mountpoint, "EFI", "BOOT", "syslinux.cfg")),
(grubconf, bootconf)
]
for (infile, outfile) in files_to_update:
if os.path.exists(infile):
self.update_config(infile, outfile)
if os.path.exists(isolinux_dir):
execute(["mv", isolinux_dir, syslinux_dir])
if os.path.exists(isolinux_cfg):
os.remove(isolinux_cfg)
def update_config(self, infile, outfile):
with open(infile) as f_in:
lines = [re.sub('/isolinux/', '/syslinux/', line) for line in f_in]
with open(outfile, "w") as f_out:
f_out.writelines(lines)
def install_mbr(self):
logger.info("Installing MBR")
mbr_path = os.path.join(self.mountpoint, "utils/mbr/mbr.bin")
execute(["dd", "bs=440", "count=1", "conv=notrunc", "if=%s" % mbr_path, "of=%s" % self.image])
# Only required if using the running system's syslinux instead of the one on the ISO
def copy_syslinux_modules(self):
logger.info("Copying syslinux modules to device")
syslinux_dir = os.path.join(self.mountpoint, 'syslinux')
com32modules = [f for f in os.listdir(syslinux_dir) if f.endswith('.c32')]
for module in com32modules:
src_path = os.path.join(SYSLINUX_COM32MODULES_DIR, module)
if not os.path.isfile(src_path):
raise ImageCreationError("Could not find the '%s' COM32 module" % module)
logger.debug('Copying %s to the device' % src_path)
execute(["cp", "-a", src_path, os.path.join(syslinux_dir, module)])
def install_syslinux(self):
logger.info("Installing bootloader")
# We install syslinux directly on the image. Installing it on the loop
# device would cause this issue:
# https://bugs.chromium.org/p/chromium/issues/detail?id=508713#c8
execute([
'syslinux',
'--offset', str(self.partition.props.partition.props.offset),
'--directory', '/syslinux/',
'--install', self.image
],
as_root=True # XXX: Why does this only work as root?
)
def reset_timestamps(self):
logger.info("Resetting timestamps")
for root, dirs, files in os.walk(self.mountpoint):
os.utime(root, (0, 0), follow_symlinks=False)
for file in files:
os.utime(os.path.join(root, file), (0, 0), follow_symlinks=False)
def set_guids(self):
logger.info("Setting disk and partition GUID")
execute(["/sbin/sgdisk", "--disk-guid", "17B81DA0-8B1E-4269-9C39-FE5C7B9B58A3",
"--partition-guid", "1:34BF027A-8001-4B93-8243-1F9D3DCE7DE7", self.image])
def set_fsuuid(self):
"""Set a fixed filesystem UUID aka. FAT Volume ID / serial number"""
logger.info("Setting filesystem UUID")
with set_env("MTOOLS_SKIP_CHECK", "1"):
execute(["mlabel", "-i", self.partition.props.block.props.device, "-N", "a69020d2"])
def execute(cmd: list, as_root=False):
if as_root and os.geteuid() != 0:
cmd = ['pkexec'] + cmd
logger.info("Executing '%s'" % ' '.join(cmd))
subprocess.check_call(cmd)
@contextmanager
def set_env(name: str, value:str):
old_value = os.getenv(name)
os.putenv(name, value)
try:
yield
finally:
if old_value is not None:
os.putenv(name, value)
else:
os.unsetenv(name)
def get_file_size(path: str) -> int:
"""Returns the size of a file in MiB"""
size_in_bytes = os.path.getsize(path)
return round(size_in_bytes // 1024 ** 2)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("ISO", help="Path to the ISO")
parser.add_argument("-d", "--directory", default=".", help="Output directory for the resulting image (the current directory by default)")
parser.add_argument("--free-space", type=int, default=0, help="Additional free space (for a persistent volume) in MiB")
args = parser.parse_args()
if not args.ISO.endswith(".iso"):
parser.error("Input file is not an ISO (no .iso extension)")
logging.basicConfig(level=logging.INFO)
logging.getLogger('sh').setLevel(logging.WARNING)
iso = args.ISO
image = os.path.realpath(os.path.join(args.directory, os.path.basename(iso).replace(".iso", ".img")))
image_creator = ImageCreator(iso, image, args.free_space)
image_creator.create_image()
if __name__ == "__main__":
main()
#!/bin/sh
set -e
set -u
set -x
# Based on ypcs' scripts found at:
# https://github.com/ypcs/vmdebootstrap-vagrant/
......
......@@ -72,10 +72,13 @@ sed -i 's,^GRUB_TIMEOUT=5,GRUB_TIMEOUT=1,g' /etc/default/grub
echo "I: Installing Tails build dependencies."
apt-get -y install \
debootstrap \
dosfstools \
dpkg-dev \
eatmydata \
faketime \
gdisk \
gettext \
gir1.2-udisks-2.0 \
git \
ikiwiki \
intltool \
......@@ -93,12 +96,18 @@ apt-get -y install \
libyaml-syck-perl \
live-build \
lsof \
mtools \
p7zip-full \
perlmagick \
psmisc \
python3-gi \
rsync \
ruby \
syslinux \
syslinux-common \
syslinux-utils \
time \
udisks2 \
whois
# Ensure we can use timedatectl
......
......@@ -34,7 +34,7 @@ remove_build_dirs() {
tries=0
sudo lsof | grep --fixed-strings "${mountpoint}" || true
while ! sudo umount -f --verbose "${mountpoint}" && [ $tries -lt 12 ]; do
sudo fuser --ismountpoint --mount "${mountpoint}" --kill
sudo fuser --ismountpoint --mount "${mountpoint}" --kill || true
sleep 5
tries=$(expr $tries + 1)
done
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment