Commit 437a053a authored by intrigeri's avatar intrigeri

Merge remote-tracking branch 'origin/devel' into feature/11123-new-mirror-pool

parents ea9e9aad 9e2df212
......@@ -3,6 +3,7 @@
*.po~
*.pot~
*.swp
/*.build-manifest
/*.buildlog
/*.img
/*.iso
......
This diff is collapsed.
......@@ -76,6 +76,9 @@ chmod -R go+rX config/chroot_sources
# build the image
# we need /debootstrap/deburis to build a manifest of used packages:
export DEBOOTSTRAP_OPTIONS='--keep-debootstrap-dir'
: ${MKSQUASHFS_OPTIONS:='-comp xz -Xbcj x86 -b 1024K -Xdict-size 1024K'}
MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS} -wildcards -ef chroot/usr/share/amnesia/build/mksquashfs-excludes"
export MKSQUASHFS_OPTIONS
......@@ -180,24 +183,23 @@ set -o pipefail
time eatmydata lb build noauto ${@}
RET=$?
if [ -e "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" ]; then
if [ "$RET" -eq 0 ]; then
[ -z "$JENKINS_URL" ] || date --utc '+%s' > "$BUILD_END_FILENAME"
echo "Image was successfully created"
if [ "$LB_BINARY_IMAGES" = iso ]; then
ISO_FILE="${BUILD_FILENAME}.${BUILD_FILENAME_EXT}"
print_iso_size "$ISO_FILE"
echo "Hybriding it..."
isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE"
print_iso_size "$ISO_FILE"
truncate -s %2048 "$ISO_FILE"
print_iso_size "$ISO_FILE"
fi
else
echo "Warning: image created, but lb build exited with code $RET"
echo "Image was successfully created"
[ "$RET" -eq 0 ] || \
echo "Warning: lb build exited with code $RET"
[ -z "$JENKINS_URL" ] || date --utc '+%s' > "$BUILD_END_FILENAME"
if [ "$LB_BINARY_IMAGES" = iso ]; then
ISO_FILE="${BUILD_FILENAME}.${BUILD_FILENAME_EXT}"
print_iso_size "$ISO_FILE"
echo "Hybriding it..."
isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE"
print_iso_size "$ISO_FILE"
truncate -s %2048 "$ISO_FILE"
print_iso_size "$ISO_FILE"
fi
echo "Renaming generated files..."
mv -i "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" "${BUILD_DEST_FILENAME}"
mv -i binary.packages "${BUILD_PACKAGES}"
generate-build-manifest chroot/debootstrap "${BUILD_DEST_FILENAME}.build-manifest"
else
fatal "lb build failed ($?)."
fi
......@@ -24,13 +24,15 @@ $RUN_LB_CONFIG --distribution jessie ${@}
# set Amnesia's general options
$RUN_LB_CONFIG \
--verbose \
--apt-recommends false \
--backports false \
--binary-images iso \
--binary-indices false \
--checksums none \
--bootappend-live "${AMNESIA_APPEND}" \
--bootstrap "cdebootstrap" \
--bootstrap debootstrap \
--bootstrap-config tails-build-jessie \
--archive-areas "main contrib non-free" \
--includes none \
--iso-application="The Amnesic Incognito Live System" \
......@@ -106,3 +108,11 @@ install -m 0755 -d config/chroot_local-includes/usr/local/lib/nodejs
install -m 0755 \
submodules/mirror-pool-dispatcher/lib/js/mirror-dispatcher.js \
config/chroot_local-includes/usr/local/lib/nodejs/
# custom debootstrap script, setting some APT magic to log downloads:
patch \
--follow-symlinks \
--output=/usr/share/debootstrap/scripts/tails-build-jessie \
/usr/share/debootstrap/scripts/jessie \
data/debootstrap/scripts/jessie.patch
sed -i "s,%%topdir%%,$(pwd)," /usr/share/debootstrap/scripts/tails-build-jessie
#!/usr/bin/perl
# © 2015 Cyril Brulebois <cyril@debamax.com>, for the Tails project.
# © 2016 Tails developers <tails@boum.org>
use strict;
use warnings;
use File::Slurp;
use List::MoreUtils qw(uniq);
use YAML::XS;
# NOTE: For reference, the first one is generated by debootstrap
# (>= 1.0.73), while the other two are generated by the apt-get
# wrapper installed in the chroot during the build.
my %package_type = qw(
deburis binary
binuris binary
srcuris source
);
### Various usability checks:
sub usage {
die "Usage: $0 debootstrap-dir manifest-file";
}
my $debootstrap = shift @ARGV
or usage;
my $manifest = shift @ARGV
or usage;
if (! -d $debootstrap) {
print "E: $debootstrap isn't a directory\n";
usage;
}
my $extra_packages_file = 'config/build-manifest-extra-packages.yml';
my $extra_packages;
if (-e $extra_packages_file) {
my $yaml = read_file($extra_packages_file);
my $extra_packages_data = Load $yaml
or die "E: failed to load $extra_packages_file: $!";
$extra_packages = $extra_packages_data->{packages};
}
### Read (package, version, uri) tuples and generate a single (package, version) list:
my $data;
foreach my $type (keys %package_type) {
my $path = "$debootstrap/$type";
if (! -f $path ) {
print "E: $path is missing, wrong debootstrap-dir parameter? (got: $debootstrap)\n";
usage;
}
print "I: processing $path\n";
foreach my $line (read_file($path)) {
chomp $line;
my ($package, $version, $uri) = split / /, $line;
# Store package_version_arch to ease sort+uniq for deduplication:
my $arch = 'source';
if ($package_type{$type} eq 'binary') {
if ($uri =~ /_([^_]+)\.deb$/) {
$arch = $1;
}
else {
die "unable to determine architecture for uri=$uri";
}
}
push @{ $data->{ packages_tmp }->{ $package_type{$type} } }, "${package}_${version}_${arch}";
}
# Add extra packages
if ($extra_packages->{$package_type{$type}}) {
foreach my $pkginfo (@{ $extra_packages->{$package_type{$type}} }) {
my $package = $pkginfo->{package};
my $version = $pkginfo->{version};
my $arch = $package_type{$type} eq 'binary'
? $pkginfo->{arch}
: 'source';
push @{ $data->{ packages_tmp }->{ $package_type{$type} } },
"${package}_${version}_${arch}";
}
}
}
### Extract list of (origin, reference) from the build configuration:
my %origin_reference;
while (my $origin_dir = glob('config/APT_snapshots.d/*')) {
my $origin_name = $origin_dir;
$origin_name =~ s{\A config/APT_snapshots[.]d/}{}xms;
$origin_reference{$origin_name} = read_file("$origin_dir/serial");
chomp $origin_reference{$origin_name};
$data->{origin_references}->{ $origin_name }->{reference} = $origin_reference{ $origin_name } || 'unknown';
}
### Deduplicate:
foreach my $type (uniq values %package_type) {
foreach my $entry (uniq sort @{ $data->{ packages_tmp }->{ $type } }) {
if ($entry =~ m{^(.+)_(.+)_(.+)$}) {
my ($package, $version, $arch) = ($1, $2, $3);
my $item = { package => $package, version => $version, arch => $arch, };
# Reduce clutter:
delete $item->{arch}
if $type eq 'source';
push @{ $data->{ packages }->{ $type } }, $item;
}
}
}
delete $data->{ packages_tmp };
my $yaml = Dump $data;
write_file($manifest, $yaml);
#!/usr/bin/perl
use strict;
use warnings FATAL => 'all';
use 5.10.1;
use autodie;
use Carp::Assert;
use Carp::Assert::More;
use IO::All;
use List::MoreUtils qw{uniq};
my $usage = "Usage: $0 ACNG_LOG IP EPOCH_START EPOCH_END [OUTPUT_BIN_PKGS OUTPUT_SRC_PKGS]";
my $logline_re = qr{
\A
(\d+) [|]
([^|]+) [|]
(\d+) [|]
([^|]+) [|]
([^\n]+)
\z
}xms;
### Subs
sub logline_timestamp_is_between {
my $logline = shift;
my $epoch_start = shift;
my $epoch_end = shift;
if (my ($timestamp) = ($logline =~ m{\A (\d+) [|]}xms)) {
return $timestamp >= $epoch_start && $timestamp <= $epoch_end;
}
return;
}
sub logline_is_from_ip {
my $logline = shift;
my $client_ip = shift;
if (my ($time, $whatever, $size, $ip, $url) = ($logline =~ m{$logline_re}xms)) {
return $ip eq $client_ip;
}
return;
}
sub interesting_loglines {
my $logfile = shift;
my $client_ip = shift;
my $epoch_start = shift;
my $epoch_end = shift;
my @content = grep {
logline_timestamp_is_between($_, $epoch_start, $epoch_end)
and
logline_is_from_ip($_, $client_ip)
} io->file($logfile)->chomp->slurp;
}
sub url_in_logline {
my $logline = shift;
if (my ($time, $whatever, $size, $ip, $url) = ($logline =~ m{$logline_re}xms)) {
return $url;
}
return;
}
### Extract and validate arguments
@ARGV == 4 or @ARGV == 6 or die $usage;
my ($logfile, $client_ip, $epoch_start, $epoch_end, $output_binpkgs,
$output_srcpkgs) = @ARGV;
assert(-e $logfile);
assert(-f $logfile);
assert(-r $logfile);
for my $epoch ($epoch_start, $epoch_end) {
assert_integer($epoch);
assert_positive($epoch);
}
### Extract urls from loglines within the build time range
my @urls = map {
url_in_logline($_)
} interesting_loglines(
$logfile, $client_ip, $epoch_start, $epoch_end
);
my @deb_urls = uniq grep { /[.]deb\z/ } @urls;
my @dsc_urls = uniq grep { /[.]dsc\z/ } @urls;
my @bin_pkgs = map { s{\A .*/ ([^/_]+) [_] .* [.]deb \z}{$1}xms; $_; } @deb_urls;
my @src_pkgs = map { s{\A .*/ ([^/_]+) [_] .* [.]dsc \z}{$1}xms; $_; } @dsc_urls;
if (defined $output_binpkgs && defined $output_srcpkgs) {
open(my $bin_fh, '>', $output_binpkgs);
print $bin_fh join("\n", sort @bin_pkgs), "\n";
open(my $src_fh, '>', $output_srcpkgs);
print $src_fh join("\n", sort @src_pkgs), "\n";
}
else {
use Data::Dumper;
say Dumper \@bin_pkgs;
say Dumper \@src_pkgs;
}
......@@ -13,7 +13,7 @@
# Base for the string that will be passed to "lb config --bootappend-live"
# FIXME: see [[bugs/sdmem_on_eject_broken_for_CD]] for explanation why we
# need to set block.events_dfl_poll_msecs
AMNESIA_APPEND="live-media=removable apparmor=1 security=apparmor nopersistence noprompt timezone=Etc/UTC block.events_dfl_poll_msecs=1000 splash noautologin module=Tails"
AMNESIA_APPEND="live-media=removable apparmor=1 security=apparmor nopersistence noprompt timezone=Etc/UTC block.events_dfl_poll_msecs=1000 splash noautologin module=Tails slab_nomerge slub_debug=FZ mce=0 vsyscall=none"
# Options passed to isohybrid
AMNESIA_ISOHYBRID_OPTS="-h 255 -s 63"
......
# Extra packages that shall be added to the build manifest.
#
# Add here any package needed during the build, that is not identified by our
# debootstrap + apt-get wrapper tricks, when there is no better solution.
#
packages:
binary:
- package: squashfs-tools
arch: i386
version: 1:4.2+20130409-2
explanation: pulled by lb_binary_rootfs, outside of the reach of our apt-get wrapper
......@@ -29,7 +29,7 @@ apt-get --yes purge \
### since they have Priority: standard.
apt-get --yes purge \
apt-listchanges at bsd-mailx dc debian-faq doc-debian dselect \
'^exim4*' ftp m4 mlocate mutt ncurses-term nfs-common portmap procmail python-apt \
'^exim4*' ftp m4 mlocate mutt ncurses-term nfs-common portmap procmail \
python-reportbug reportbug telnet texinfo time w3m wamerican
### Deinstall some other unwanted packages.
......
......@@ -15,7 +15,7 @@ domain ip {
policy DROP;
# Established incoming connections are accepted.
mod state state (RELATED ESTABLISHED) ACCEPT;
mod state state (ESTABLISHED) ACCEPT;
# Traffic on the loopback interface is accepted.
interface lo ACCEPT;
......@@ -25,10 +25,13 @@ domain ip {
policy DROP;
# Established outgoing connections are accepted.
mod state state (RELATED ESTABLISHED) ACCEPT;
mod state state (ESTABLISHED) ACCEPT;
# White-list access to local resources
outerface lo {
# Related outgoing ICMP packets are accepted.
mod state state (RELATED) proto icmp ACCEPT;
# White-list access to Tor's SOCKSPort's
daddr 127.0.0.1 proto tcp syn dport 9050 {
mod owner uid-owner root ACCEPT;
......@@ -141,7 +144,9 @@ domain ip {
}
# Tor is allowed to do anything it wants to.
mod owner uid-owner debian-tor ACCEPT;
mod owner uid-owner debian-tor {
proto tcp syn mod state state (NEW) ACCEPT;
}
# i2p is allowed to do anything it wants to on the internet.
outerface ! lo mod owner uid-owner i2psvc {
......@@ -188,7 +193,7 @@ domain ip6 {
# White-list access to the accessibility daemon
interface lo saddr ::1 daddr ::1 proto tcp {
dport 4101 ACCEPT;
sport 4101 mod state state (RELATED ESTABLISHED) ACCEPT;
sport 4101 mod state state (ESTABLISHED) ACCEPT;
}
}
......@@ -203,7 +208,7 @@ domain ip6 {
# White-list access to the accessibility daemon
outerface lo saddr ::1 daddr ::1 proto tcp {
dport 4101 mod owner uid-owner amnesia ACCEPT;
sport 4101 mod state state (RELATED ESTABLISHED) ACCEPT;
sport 4101 mod state state (ESTABLISHED) ACCEPT;
}
# Everything else is logged and dropped.
......
......@@ -4,9 +4,6 @@
<blist>
<group name='Discussions'>
<setting name='collapsed' type='bool'>0</setting>
<chat proto='prpl-irc' account='XXX_NICK_XXX@irc.oftc.net'>
<component name='channel'>#tails</component>
</chat>
<chat proto='prpl-irc' account='XXX_NICK_XXX@127.0.0.1'>
<component name='channel'>#i2p</component>
</chat>
......
#!/bin/sh
echo "- undiverting APT"
if [ -f /usr/bin/apt-get.real ]; then
rm -f usr/bin/apt-get
dpkg-divert --rename --remove /usr/bin/apt-get
fi
boot/initrd.img-*
boot/vmlinux-*
boot/vmlinuz-*
debootstrap/*
tmp/*
usr/share/amnesia/packages/*
usr/share/doc/tails/website/blueprint/*
......
......@@ -110,7 +110,6 @@ gnome-system-monitor
gnome-terminal
gnome-themes
gnome-themes-standard
gnome-tweak-tool
gnome-user-guide
gnupg-agent
gnupg-curl
......@@ -134,7 +133,6 @@ hardlink
haveged
# needed by laptop-mode-tools to spin-down hard drives
hdparm
hledger
hopenpgp-tools
icedove
icedove-l10n-all
......@@ -392,6 +390,7 @@ crda
wireless-regdb
### Automated test suite
python-dogtail
python3-serial
python3-systemd
xdotool
......
--- /usr/share/debootstrap/scripts/sid 2016-05-11 15:43:45.396062439 +0000
+++ data/debootstrap/tails-wheezy 2016-05-11 15:38:08.949103098 +0000
@@ -201,4 +201,8 @@
progress $bases $bases CONFBASE "Configuring base system"
info BASESUCCESS "Base system installed successfully."
+
+ # Tails-specific part:
+ chroot $TARGET /usr/sbin/dpkg-divert --divert /usr/bin/apt-get.real --rename /usr/bin/apt-get
+ cp -f %%topdir%%/data/wrappers/apt-get $TARGET/usr/bin/apt-get
}
#!/bin/sh
set -e
set -u
mode=unknown
for param in "$@"; do
case "$param" in
install | download | purge | remove | upgrade | dist-upgrade)
mode=binuris
break
;;
source)
mode=srcuris
break
;;
check | update | autoclean | autoremove)
mode=noop
break
;;
esac
done
# let's fail as early as possible:
if [ "$mode" = unknown ]; then
echo "E: unsupported apt-get operation, mode is still unknown" >&2
echo "E: requested operation follows:" "$@" >&2
exit 1
fi
if [ "$mode" = binuris ]; then
apt-get.real "$@" --print-uris|perl -ne 'if (/^'\''(.+)'\'' ([^_]+)_([^_]+)_/) { my ($url, $package, $version)=($1,$2,$3); $version =~ s/%3a/:/g; print "$package $version $url\n"; }' >> /debootstrap/$mode
apt-get.real "$@"
elif [ "$mode" = srcuris ]; then
# all uris: perl -ne 'if (/^'\''(.+)'\'' (\S+)/) { my ($url, $filename) = ($1, $2); print "$filename $url\n"; }'
# only dsc: perl -ne 'if (/^'\''(.+)'\'' (\S+\.dsc)/) { my ($url, $filename) = ($1, $2); print "$filename $url\n"; }'
apt-get.real "$@" --print-uris|perl -ne 'if (/^'\''(.+)'\'' (\S+\.dsc)/) { my ($url, $filename) = ($1, $2); print "$filename $url\n"; }' >> /debootstrap/$mode.tmp
apt-get.real "$@"
while read filename uri; do
# extract source and version w/o taking the GnuPG version in the signature section, and add uri after that
s_v=$(awk '/^(Source|Version):/ {print $2}' "$filename" | head -2 | xargs)
echo "$s_v $uri" >> /debootstrap/$mode
done < /debootstrap/$mode.tmp
rm /debootstrap/$mode.tmp
else
# handle both noop and unknown here, each into its own file; unknown should be empty:
echo "command:" "$@" >> /debootstrap/$mode
apt-get.real "$@"
fi
......@@ -103,5 +103,4 @@ Feature: Various checks
And I enable more Tails Greeter options
And I disable all networking in the Tails Greeter
And I log in to a new session
And the Tails desktop is ready
Then no network interfaces are enabled
......@@ -29,6 +29,9 @@ Feature: Encryption and verification using GnuPG
And I both encrypt and sign the message using my OpenPGP key
Then I can decrypt and verify the encrypted message
#11394
#11398
@fragile
Scenario: Symmetric encryption and decryption using OpenPGP Applet
When I type a message into gedit
And I symmetrically encrypt the message with password "asdf"
......
......@@ -21,6 +21,8 @@ Feature: Using Evince
Then I see "CupsTestPage.png" after at most 20 seconds
And I can print the current document to "/home/amnesia/output.pdf"
#11398
@fragile
Scenario: I cannot view a PDF file stored in non-persistent /home/amnesia/.gnupg
Given I have started Tails from DVD without network and logged in
And I copy "/usr/share/cups/data/default-testpage.pdf" to "/home/amnesia/.gnupg" as user "amnesia"
......
......@@ -68,7 +68,7 @@ Feature: I2P
When I activate the "I2P" Pidgin account
And I close Pidgin's account manager window
Then Pidgin successfully connects to the "I2P" account
And I can join the "#i2p" channel on "I2P"
And I can join the pre-configured "#i2p" channel on "I2P"
@fragile
Scenario: I2P displays a notice when bootstrapping fails
......