Commit a8358a00 authored by sajolida's avatar sajolida
Browse files

Merge remote-tracking branch 'origin/master' into web/10628-consistency-router-overview

parents 65a20764 d7a21489

Too many changes to show.

To preserve performance only 1000 of 1000+ files are displayed.
......@@ -3,6 +3,8 @@
*.po~
*.pot~
*.swp
/*.apt-sources
/*.build-manifest
/*.buildlog
/*.img
/*.iso
......
......@@ -4,3 +4,10 @@
[submodule "submodules/jenkins-tools"]
path = submodules/jenkins-tools
url = https://git-tails.immerda.ch/jenkins-tools
[submodule "submodules/chutney"]
path = submodules/chutney
url = https://git-tails.immerda.ch/chutney
branch = feature/tails_test_suite
[submodule "submodules/mirror-pool-dispatcher"]
path = submodules/mirror-pool-dispatcher
url = https://git-tails.immerda.ch/mirror-pool-dispatcher
......@@ -18,14 +18,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
require 'open3'
require 'rbconfig'
require 'rubygems'
require 'vagrant'
require 'uri'
$:.unshift File.expand_path('../vagrant/lib', __FILE__)
require 'tails_build_settings'
require 'vagrant_version'
require_relative 'vagrant/lib/tails_build_settings'
# Path to the directory which holds our Vagrantfile
VAGRANT_PATH = File.expand_path('../vagrant', __FILE__)
......@@ -40,58 +37,61 @@ EXPORTED_VARIABLES = ['http_proxy', 'MKSQUASHFS_OPTIONS', 'TAILS_RAM_BUILD', 'TA
EXTERNAL_HTTP_PROXY = ENV['http_proxy']
# In-VM proxy URL
INTERNEL_HTTP_PROXY = "http://#{VIRTUAL_MACHINE_HOSTNAME}:3142"
INTERNAL_HTTP_PROXY = "http://#{VIRTUAL_MACHINE_HOSTNAME}:3142"
def primary_vm
env = Vagrant::Environment.new(:cwd => VAGRANT_PATH, :ui_class => Vagrant::UI::Basic)
if vagrant_old
return env.primary_vm
else
name = env.primary_machine_name
return env.machine(name, env.default_provider)
end
end
def primary_vm_state
if vagrant_old
return primary_vm.state
else
return primary_vm.state.id
end
class VagrantCommandError < StandardError
end
def primary_vm_chan
if vagrant_old
return primary_vm.channel
else
return primary_vm.communicate
# Runs the vagrant command, letting stdout/stderr through. Throws an
# exception unless the vagrant command succeeds.
def run_vagrant(*args)
Process.wait Kernel.spawn('vagrant', *args, :chdir => './vagrant')
if $?.exitstatus != 0
raise(VagrantCommandError, "'vagrant #{args}' command failed: " +
"#{$?.exitstatus}")
end
end
def vm_id
if vagrant_old
primary_vm.uuid
else
primary_vm.id
# Runs the vagrant command, not letting stdout/stderr through, and
# returns [stdout, stderr, Preocess:Status].
def capture_vagrant(*args)
stdout, stderr, proc_status =
Open3.capture3('vagrant', *args, :chdir => './vagrant')
if proc_status.exitstatus != 0
raise(VagrantCommandError, "'vagrant #{args}' command failed: " +
"#{proc_status.exitstatus}")
end
return stdout, stderr
end
def vm_driver
if vagrant_old
primary_vm.driver
else
primary_vm.provider.driver
def vagrant_ssh_config(key)
# Cache results
if $vagrant_ssh_config.nil?
$vagrant_ssh_config = capture_vagrant('ssh-config').first.split("\n") \
.map { |line| line.strip.split(/\s+/, 2) } .to_h
# The path in the ssh-config output is quoted, which is not what
# is expected outside of a shell, so let's get rid of the quotes.
$vagrant_ssh_config['IdentityFile'].gsub!(/^"|"$/, '')
end
$vagrant_ssh_config[key]
end
def current_vm_cpus
info = vm_driver.execute 'showvminfo', vm_id, '--machinereadable'
$1.to_i if info =~ /^cpus=(\d+)/
capture_vagrant('ssh', '-c', 'grep -c "^processor\s*:" /proc/cpuinfo').first.chomp.to_i
end
def vm_running?
primary_vm_state == :running
def vm_state
out, _ = capture_vagrant('status')
status_line = out.split("\n")[2]
if status_line['not created']
return :not_created
elsif status_line['shutoff']
return :poweroff
elsif status_line['running']
return :running
else
raise "could not determine VM state"
end
end
def enough_free_host_memory_for_ram_build?
......@@ -106,9 +106,7 @@ def enough_free_host_memory_for_ram_build?
end
def free_vm_memory
primary_vm_chan.execute("free", :error_check => false) do |fd, data|
return data.split[16].to_i
end
capture_vagrant('ssh', '-c', 'free').first.chomp.split[16].to_i
end
def enough_free_vm_memory_for_ram_build?
......@@ -116,7 +114,7 @@ def enough_free_vm_memory_for_ram_build?
end
def enough_free_memory_for_ram_build?
if vm_running?
if vm_state == :running
enough_free_vm_memory_for_ram_build?
else
enough_free_host_memory_for_ram_build?
......@@ -145,8 +143,8 @@ task :parse_build_options do
# Default to in-memory builds if there is enough RAM available
options += 'ram ' if enough_free_memory_for_ram_build?
# Use in-VM proxy unless an external proxy is set
options += 'vmproxy ' unless EXTERNAL_HTTP_PROXY
# Default to build using the in-VM proxy
options += 'vmproxy '
# Default to fast compression on development branches
options += 'gzipcomp ' unless is_release?
......@@ -172,7 +170,7 @@ task :parse_build_options do
abort "No HTTP proxy set, but one is required by TAILS_BUILD_OPTIONS. Aborting." unless EXTERNAL_HTTP_PROXY
ENV['http_proxy'] = EXTERNAL_HTTP_PROXY
when 'vmproxy'
ENV['http_proxy'] = INTERNEL_HTTP_PROXY
ENV['http_proxy'] = INTERNAL_HTTP_PROXY
when 'noproxy'
ENV['http_proxy'] = nil
# SquashFS compression settings
......@@ -189,6 +187,8 @@ task :parse_build_options do
# Git settings
when 'ignorechanges'
ENV['TAILS_BUILD_IGNORE_CHANGES'] = '1'
when 'noprovision'
ENV['TAILS_NO_AUTO_PROVISION'] = '1'
end
end
end
......@@ -221,6 +221,26 @@ task :ensure_clean_repository do
end
end
def list_artifacts
user = vagrant_ssh_config('User')
stdout = capture_vagrant('ssh', '-c', "find '/home/#{user}/' -maxdepth 1 " +
"-name 'tails-*.iso*'").first
stdout.split("\n")
rescue VagrantCommandError
return Array.new
end
def remove_artifacts
list_artifacts.each do |artifact|
run_vagrant('ssh', '-c', "sudo rm -f '#{artifact}'")
end
end
desc "Make sure the vagrant user's home directory has no undesired artifacts"
task :ensure_clean_home_directory => ['vm:up'] do
remove_artifacts
end
task :validate_http_proxy do
if ENV['http_proxy']
proxy_host = URI.parse(ENV['http_proxy']).host
......@@ -242,7 +262,7 @@ task :validate_http_proxy do
end
desc 'Build Tails'
task :build => ['parse_build_options', 'ensure_clean_repository', 'validate_http_proxy', 'vm:up'] do
task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_home_directory', 'validate_http_proxy', 'vm:up'] do
if ENV['TAILS_RAM_BUILD'] && not(enough_free_memory_for_ram_build?)
$stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
......@@ -268,27 +288,47 @@ task :build => ['parse_build_options', 'ensure_clean_repository', 'validate_http
abort 'The virtual machine needs to be reloaded to change the number of CPUs. Aborting.'
end
# Let's make sure that, unless you know what you are doing and
# explicitly disable this, we always provision in order to ensure
# a valid, up-to-date build system.
run_vagrant('provision') unless ENV['TAILS_NO_AUTO_PROVISION']
exported_env = EXPORTED_VARIABLES.select { |k| ENV[k] }.
collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
status = primary_vm_chan.execute("#{exported_env} build-tails",
:error_check => false) do |fd, data|
(fd == :stdout ? $stdout : $stderr).write data
collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
run_vagrant('ssh', '-c', "#{exported_env} build-tails")
artifacts = list_artifacts
raise 'No build artifacts was found!' if artifacts.empty?
user = vagrant_ssh_config('User')
hostname = vagrant_ssh_config('HostName')
key_file = vagrant_ssh_config('IdentityFile')
$stderr.puts "Retrieving artifacts from Vagrant build box."
artifacts.each do |artifact|
run_vagrant('ssh', '-c', "sudo chown #{user} '#{artifact}'")
Process.wait(
Kernel.spawn(
'scp',
'-i', key_file,
# We need this since the user will not necessarily have a
# known_hosts entry. It is safe since an attacker must
# compromise libvirt's network config or the user running the
# command to modify the #{hostname} below.
'-o', 'StrictHostKeyChecking=no',
"#{user}@#{hostname}:#{artifact}", '.'
)
)
raise "Failed to fetch artifact '#{artifact}'" unless $?.success?
end
# Move build products to the current directory
FileUtils.mv Dir.glob("#{VAGRANT_PATH}/tails-*"),
File.expand_path('..', __FILE__), :force => true
exit status
remove_artifacts
end
namespace :vm do
desc 'Start the build virtual machine'
task :up => ['parse_build_options', 'validate_http_proxy'] do
case primary_vm_state
case vm_state
when :not_created
# Do not use non-existant in-VM proxy to download the basebox
if ENV['http_proxy'] == INTERNEL_HTTP_PROXY
if ENV['http_proxy'] == INTERNAL_HTTP_PROXY
ENV['http_proxy'] = nil
restore_internal_proxy = true
end
......@@ -315,31 +355,56 @@ namespace :vm do
END_OF_MESSAGE
end
env = Vagrant::Environment.new(:cwd => VAGRANT_PATH, :ui_class => Vagrant::UI::Basic)
result = env.cli('up')
abort "'vagrant up' failed" unless result
run_vagrant('up')
ENV['http_proxy'] = INTERNAL_HTTP_PROXY if restore_internal_proxy
end
ENV['http_proxy'] = INTERNEL_HTTP_PROXY if restore_internal_proxy
desc 'SSH into the builder VM'
task :ssh do
run_vagrant('ssh')
end
desc 'Stop the build virtual machine'
task :halt do
env = Vagrant::Environment.new(:cwd => VAGRANT_PATH, :ui_class => Vagrant::UI::Basic)
result = env.cli('halt')
abort "'vagrant halt' failed" unless result
run_vagrant('halt')
end
desc 'Re-run virtual machine setup'
task :provision => ['parse_build_options', 'validate_http_proxy'] do
env = Vagrant::Environment.new(:cwd => VAGRANT_PATH, :ui_class => Vagrant::UI::Basic)
result = env.cli('provision')
abort "'vagrant provision' failed" unless result
run_vagrant('provision')
end
desc 'Destroy build virtual machine (clean up all files)'
task :destroy do
env = Vagrant::Environment.new(:cwd => VAGRANT_PATH, :ui_class => Vagrant::UI::Basic)
result = env.cli('destroy', '--force')
abort "'vagrant destroy' failed" unless result
run_vagrant('destroy', '--force')
end
end
namespace :basebox do
desc 'Generate a new base box'
task :create do
box_dir = VAGRANT_PATH + '/definitions/tails-builder'
Dir.chdir(box_dir) do
`./generate-tails-builder-box.sh`
raise 'Base box generation failed!' unless $?.success?
end
box = Dir.glob("#{box_dir}/*.box").sort_by {|f| File.mtime(f) } .last
$stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
You have successfully generated a new Vagrant base box:
#{box}
To install the new base box, please run:
$ vagrant box add #{box}
To actually make Tails build using this base box, the `config.vm.box` key
in `vagrant/Vagrantfile` has to be updated. Please check the documentation
for details.
END_OF_MESSAGE
end
end
......@@ -76,6 +76,17 @@ chmod -R go+rX config/chroot_sources
# build the image
# we need /debootstrap/deburis to build a manifest of used packages:
DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keep-debootstrap-dir"
# use our own APT repository's key:
DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
gpg --homedir "$DEBOOTSTRAP_GNUPG_HOMEDIR" \
--import config/chroot_sources/tails.chroot.gpg
DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keyring=$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.gpg"
export DEBOOTSTRAP_OPTIONS
: ${MKSQUASHFS_OPTIONS:='-comp xz -Xbcj x86 -b 1024K -Xdict-size 1024K'}
MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS} -wildcards -ef chroot/usr/share/amnesia/build/mksquashfs-excludes"
export MKSQUASHFS_OPTIONS
......@@ -162,7 +173,8 @@ case "$LB_BINARY_IMAGES" in
;;
esac
BUILD_DEST_FILENAME="${BUILD_BASENAME}.${BUILD_FILENAME_EXT}"
BUILD_MANIFEST="${BUILD_DEST_FILENAME}.list"
BUILD_MANIFEST="${BUILD_DEST_FILENAME}.build-manifest"
BUILD_APT_SOURCES="${BUILD_DEST_FILENAME}.apt-sources"
BUILD_PACKAGES="${BUILD_DEST_FILENAME}.packages"
BUILD_LOG="${BUILD_DEST_FILENAME}.buildlog"
BUILD_START_FILENAME="${BUILD_DEST_FILENAME}.start.timestamp"
......@@ -174,30 +186,38 @@ trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
exec 2> >(tee -a "$BUILD_LOG" >&2)
trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
(
echo "Mirrors:"
apt-mirror debian
apt-mirror debian-security
apt-mirror torproject
echo "Additional sources:"
cat config/chroot_sources/*.chroot
) > "$BUILD_APT_SOURCES"
echo "Building $LB_BINARY_IMAGES image ${BUILD_BASENAME}..."
set -o pipefail
[ -z "$JENKINS_URL" ] || date --utc '+%s' > "$BUILD_START_FILENAME"
time eatmydata lb build noauto ${@}
RET=$?
if [ -e "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" ]; then
if [ "$RET" -eq 0 ]; then
[ -z "$JENKINS_URL" ] || date --utc '+%s' > "$BUILD_END_FILENAME"
echo "Image was successfully created"
if [ "$LB_BINARY_IMAGES" = iso ]; then
ISO_FILE="${BUILD_FILENAME}.${BUILD_FILENAME_EXT}"
print_iso_size "$ISO_FILE"
echo "Hybriding it..."
isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE"
print_iso_size "$ISO_FILE"
truncate -s %2048 "$ISO_FILE"
print_iso_size "$ISO_FILE"
fi
else
echo "Warning: image created, but lb build exited with code $RET"
echo "Image was successfully created"
[ "$RET" -eq 0 ] || \
echo "Warning: lb build exited with code $RET"
[ -z "$JENKINS_URL" ] || date --utc '+%s' > "$BUILD_END_FILENAME"
if [ "$LB_BINARY_IMAGES" = iso ]; then
ISO_FILE="${BUILD_FILENAME}.${BUILD_FILENAME_EXT}"
print_iso_size "$ISO_FILE"
echo "Hybriding it..."
isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE"
print_iso_size "$ISO_FILE"
truncate -s %2048 "$ISO_FILE"
print_iso_size "$ISO_FILE"
fi
echo "Renaming generated files..."
mv -i "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" "${BUILD_DEST_FILENAME}"
mv -i binary.packages "${BUILD_PACKAGES}"
generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
else
fatal "lb build failed ($?)."
fi
......@@ -16,30 +16,65 @@ fi
export LB_BOOTSTRAP_INCLUDE='eatmydata'
# sanity checks
if grep -qs -E '^Pin:\s+release\s+.*a=' config/chroot_apt/preferences ; then
echo "Found unsupported a= syntax in config/chroot_apt/preferences,"
echo "use n= instead. Exiting."
exit 1
fi
if grep -qs -E '^Pin:\s+release\s+.*o=Debian Backports' \
config/chroot_apt/preferences ; then
echo "Found unsupported 'o=Debian Backports' syntax,"
echo "in config/chroot_apt/preferences. Use o=Debian instead. Exiting."
exit 1
fi
# init variables
RUN_LB_CONFIG="lb config noauto"
# init config/ with defaults for the target distribution
$RUN_LB_CONFIG --distribution jessie ${@}
# set up everything for time-based snapshots:
apt-snapshots-serials prepare-build
DEBIAN_MIRROR="$(apt-mirror debian)"
DEBIAN_SECURITY_MIRROR="$(apt-mirror debian-security)"
TORPROJECT_MIRROR="$(apt-mirror torproject)"
[ -n "$DEBIAN_MIRROR" ] || exit 1
[ -n "$DEBIAN_SECURITY_MIRROR" ] || exit 1
[ -n "$TORPROJECT_MIRROR" ] || exit 1
perl -pi \
-E \
"s|^(deb(?:-src)?\s+)https?://ftp[.]us[.]debian[.]org/debian/?(\s+)|\$1$DEBIAN_MIRROR\$2| ; \
s|^(deb(?:-src)?\s+)https?://deb[.]torproject[.]org/torproject[.]org/?(\s+)|\$1$TORPROJECT_MIRROR\$2|" \
config/chroot_sources/*.chroot \
|| exit 1
# set Amnesia's general options
$RUN_LB_CONFIG \
--verbose \
--apt-recommends false \
--backports false \
--binary-images iso \
--binary-indices false \
--checksums none \
--bootappend-live "${AMNESIA_APPEND}" \
--bootstrap "cdebootstrap" \
--bootstrap debootstrap \
--bootstrap-config tails-build-jessie \
--archive-areas "main contrib non-free" \
--includes none \
--iso-application="The Amnesic Incognito Live System" \
--iso-publisher="https://tails.boum.org/" \
--iso-volume="TAILS ${AMNESIA_FULL_VERSION}" \
--memtest none \
--mirror-binary "http://ftp.us.debian.org/debian/" \
--mirror-bootstrap "http://ftp.us.debian.org/debian/" \
--mirror-chroot "http://ftp.us.debian.org/debian/" \
--mirror-binary "$DEBIAN_MIRROR" \
--mirror-bootstrap "$DEBIAN_MIRROR" \
--mirror-chroot "$DEBIAN_MIRROR" \
--mirror-binary-security "$DEBIAN_SECURITY_MIRROR" \
--mirror-chroot-security "$DEBIAN_SECURITY_MIRROR" \
--packages-lists="standard" \
--tasks="standard" \
--linux-packages="linux-image-3.16.0-4" \
......@@ -97,3 +132,20 @@ fi
# custom APT sources
tails-custom-apt-sources > config/chroot_sources/tails.chroot
# tails-transform-mirror-url and its dependencies
install -m 0755 \
submodules/mirror-pool-dispatcher/bin/tails-transform-mirror-url \
config/chroot_local-includes/usr/local/bin/
install -m 0755 -d config/chroot_local-includes/usr/local/lib/nodejs
install -m 0755 \
submodules/mirror-pool-dispatcher/lib/js/mirror-dispatcher.js \
config/chroot_local-includes/usr/local/lib/nodejs/
# custom debootstrap script, setting some APT magic to log downloads:
patch \
--follow-symlinks \
--output=/usr/share/debootstrap/scripts/tails-build-jessie \
/usr/share/debootstrap/scripts/jessie \
data/debootstrap/scripts/jessie.patch
sed -i "s,%%topdir%%,$(pwd)," /usr/share/debootstrap/scripts/tails-build-jessie
#!/bin/bash
set -e
set -u
. "$(dirname $0)/utils.sh"
ARCHIVE="$1"
output_tagged_snapshot() {
local archive="$1"
local tag="$2"
local snapshot=$(branch_name_to_suite "$tag")
echo "http://tagged.snapshots.deb.tails.boum.org/$snapshot/$archive"
}
output_time_based_snapshot() {
local archive="$1"
local serial="$2"
echo "http://time-based.snapshots.deb.tails.boum.org/$archive/$serial"
}
### Sanity checks
[ -n "$ARCHIVE" ] || exit 1
### Main
SERIAL=$(cat "config/APT_snapshots.d/$ARCHIVE/serial")
RESOLVED_SERIAL=$(cat "tmp/APT_snapshots.d/$ARCHIVE/serial")
if [ "$(base_branch)" = stable ]; then
if version_was_released "$(version_in_changelog)"; then
on_a_tag \
|| fatal "Not building from stable, but last version in changelog" \
"was released"
output_tagged_snapshot "$ARCHIVE" "$(version_in_changelog)"
else
version_was_released "$(previous_version_in_changelog)" \
|| fatal "None of the two last version in changelog were released"
case "$ARCHIVE" in
debian-security)
[ "$SERIAL" = latest ] \
|| fatal "APT snapshots are frozen for debian-security, which" \
"should not happen on a branch based on stable"
output_time_based_snapshot "$ARCHIVE" "$RESOLVED_SERIAL"
;;
*)
if [ "$SERIAL" = latest ]; then
# In this case, "latest" means "do what I mean", that is stick
# to previous release's tagged snapshot
output_tagged_snapshot "$ARCHIVE" "$(previous_version_in_changelog)"
else
output_time_based_snapshot "$ARCHIVE" "$SERIAL"
fi
esac
fi
elif [ "$(base_branch)" = testing ]; then
if version_was_released "$(version_in_changelog)"; then
on_a_tag \
|| fatal "Not building from a tag, but last version in changelog" \
"was released"
[ "$ARCHIVE" = debian-security ] || [ "$SERIAL" != latest ] \
|| fatal "APT snapshots for $ARCHIVE are not frozen, which should" \
"not happen on a tagged testing branch"
output_tagged_snapshot "$ARCHIVE" "$(version_in_changelog)"
else
output_time_based_snapshot "$ARCHIVE" "$RESOLVED_SERIAL"
fi
else
if [ "$(base_branch)" = devel ] && [ "$SERIAL" != latest ]; then
fatal "APT snapshots are frozen, which should not happen on a branch" \
"based on the devel one"
fi
output_time_based_snapshot "$ARCHIVE" "$RESOLVED_SERIAL"
fi
#!/bin/bash
set -e
set -u
set -o pipefail
BASE_URL=http://time-based.snapshots.deb.tails.boum.org/
CONFIG=config/APT_snapshots.d