Commit 1267d7c7 authored by tchou's avatar tchou
Browse files

Merge branch 'master' into web/10623-css-improvements

Conflicts:
	wiki/src/install/expert/usb.de.po
	wiki/src/install/expert/usb.fa.po
	wiki/src/install/expert/usb.fr.po
	wiki/src/install/expert/usb.mdwn
	wiki/src/install/expert/usb.pt.po
	wiki/src/install/inc/overview.de.po
	wiki/src/install/inc/overview.fa.po
	wiki/src/install/inc/overview.fr.po
	wiki/src/install/inc/overview.html
	wiki/src/install/inc/overview.it.po
	wiki/src/install/inc/overview.pt.po
	wiki/src/install/inc/steps/create_persistence.inline.mdwn
	wiki/src/install/inc/steps/install_intermediary_outro.inline.mdwn
	wiki/src/install/inc/steps/you_are_done.inline.mdwn
	wiki/src/install/inc/stylesheets/assistant.css
	wiki/src/install/inc/stylesheets/expert.css
	wiki/src/install/mac/dvd.mdwn
	wiki/src/upgrade.de.po
	wiki/src/upgrade.fa.po
	wiki/src/upgrade.fr.po
	wiki/src/upgrade.it.po
	wiki/src/upgrade.pt.po
parents 27d7724b dbeac4be

Too many changes to show.

To preserve performance only 1000 of 1000+ files are displayed.
......@@ -3,11 +3,14 @@
*.po~
*.pot~
*.swp
/*.apt-sources
/*.build-manifest
/*.buildlog
/*.img
/*.iso
/*.list
/*.packages
/*.rake_tasks~
/*.vdi
/binary
/cache
......
......@@ -4,3 +4,10 @@
[submodule "submodules/jenkins-tools"]
path = submodules/jenkins-tools
url = https://git-tails.immerda.ch/jenkins-tools
[submodule "submodules/chutney"]
path = submodules/chutney
url = https://git-tails.immerda.ch/chutney
branch = feature/tails_test_suite
[submodule "submodules/mirror-pool-dispatcher"]
path = submodules/mirror-pool-dispatcher
url = https://git-tails.immerda.ch/mirror-pool-dispatcher
......@@ -18,14 +18,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
require 'open3'
require 'rbconfig'
require 'rubygems'
require 'vagrant'
require 'uri'
$:.unshift File.expand_path('../vagrant/lib', __FILE__)
require 'tails_build_settings'
require 'vagrant_version'
require_relative 'vagrant/lib/tails_build_settings'
# Path to the directory which holds our Vagrantfile
VAGRANT_PATH = File.expand_path('../vagrant', __FILE__)
......@@ -40,58 +37,61 @@ EXPORTED_VARIABLES = ['http_proxy', 'MKSQUASHFS_OPTIONS', 'TAILS_RAM_BUILD', 'TA
EXTERNAL_HTTP_PROXY = ENV['http_proxy']
# In-VM proxy URL
INTERNEL_HTTP_PROXY = "http://#{VIRTUAL_MACHINE_HOSTNAME}:3142"
INTERNAL_HTTP_PROXY = "http://#{VIRTUAL_MACHINE_HOSTNAME}:3142"
def primary_vm
env = Vagrant::Environment.new(:cwd => VAGRANT_PATH, :ui_class => Vagrant::UI::Basic)
if vagrant_old
return env.primary_vm
else
name = env.primary_machine_name
return env.machine(name, env.default_provider)
end
end
def primary_vm_state
if vagrant_old
return primary_vm.state
else
return primary_vm.state.id
end
class VagrantCommandError < StandardError
end
def primary_vm_chan
if vagrant_old
return primary_vm.channel
else
return primary_vm.communicate
# Runs the vagrant command, letting stdout/stderr through. Throws an
# exception unless the vagrant command succeeds.
def run_vagrant(*args)
Process.wait Kernel.spawn('vagrant', *args, :chdir => './vagrant')
if $?.exitstatus != 0
raise(VagrantCommandError, "'vagrant #{args}' command failed: " +
"#{$?.exitstatus}")
end
end
def vm_id
if vagrant_old
primary_vm.uuid
else
primary_vm.id
# Runs the vagrant command, not letting stdout/stderr through, and
# returns [stdout, stderr, Preocess:Status].
def capture_vagrant(*args)
stdout, stderr, proc_status =
Open3.capture3('vagrant', *args, :chdir => './vagrant')
if proc_status.exitstatus != 0
raise(VagrantCommandError, "'vagrant #{args}' command failed: " +
"#{proc_status.exitstatus}")
end
return stdout, stderr
end
def vm_driver
if vagrant_old
primary_vm.driver
else
primary_vm.provider.driver
def vagrant_ssh_config(key)
# Cache results
if $vagrant_ssh_config.nil?
$vagrant_ssh_config = capture_vagrant('ssh-config').first.split("\n") \
.map { |line| line.strip.split(/\s+/, 2) } .to_h
# The path in the ssh-config output is quoted, which is not what
# is expected outside of a shell, so let's get rid of the quotes.
$vagrant_ssh_config['IdentityFile'].gsub!(/^"|"$/, '')
end
$vagrant_ssh_config[key]
end
def current_vm_cpus
info = vm_driver.execute 'showvminfo', vm_id, '--machinereadable'
$1.to_i if info =~ /^cpus=(\d+)/
capture_vagrant('ssh', '-c', 'grep -c "^processor\s*:" /proc/cpuinfo').first.chomp.to_i
end
def vm_running?
primary_vm_state == :running
def vm_state
out, _ = capture_vagrant('status')
status_line = out.split("\n")[2]
if status_line['not created']
return :not_created
elsif status_line['shutoff']
return :poweroff
elsif status_line['running']
return :running
else
raise "could not determine VM state"
end
end
def enough_free_host_memory_for_ram_build?
......@@ -106,9 +106,7 @@ def enough_free_host_memory_for_ram_build?
end
def free_vm_memory
primary_vm_chan.execute("free", :error_check => false) do |fd, data|
return data.split[16].to_i
end
capture_vagrant('ssh', '-c', 'free').first.chomp.split[16].to_i
end
def enough_free_vm_memory_for_ram_build?
......@@ -116,7 +114,7 @@ def enough_free_vm_memory_for_ram_build?
end
def enough_free_memory_for_ram_build?
if vm_running?
if vm_state == :running
enough_free_vm_memory_for_ram_build?
else
enough_free_host_memory_for_ram_build?
......@@ -145,8 +143,8 @@ task :parse_build_options do
# Default to in-memory builds if there is enough RAM available
options += 'ram ' if enough_free_memory_for_ram_build?
# Use in-VM proxy unless an external proxy is set
options += 'vmproxy ' unless EXTERNAL_HTTP_PROXY
# Default to build using the in-VM proxy
options += 'vmproxy '
# Default to fast compression on development branches
options += 'gzipcomp ' unless is_release?
......@@ -172,7 +170,7 @@ task :parse_build_options do
abort "No HTTP proxy set, but one is required by TAILS_BUILD_OPTIONS. Aborting." unless EXTERNAL_HTTP_PROXY
ENV['http_proxy'] = EXTERNAL_HTTP_PROXY
when 'vmproxy'
ENV['http_proxy'] = INTERNEL_HTTP_PROXY
ENV['http_proxy'] = INTERNAL_HTTP_PROXY
when 'noproxy'
ENV['http_proxy'] = nil
# SquashFS compression settings
......@@ -189,35 +187,60 @@ task :parse_build_options do
# Git settings
when 'ignorechanges'
ENV['TAILS_BUILD_IGNORE_CHANGES'] = '1'
when 'noprovision'
ENV['TAILS_NO_AUTO_PROVISION'] = '1'
end
end
end
task :ensure_clean_repository do
unless `git status --porcelain`.empty?
git_status = `git status --porcelain`
unless git_status.empty?
if ENV['TAILS_BUILD_IGNORE_CHANGES']
$stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
You have uncommited changes in the Git repository. They will
be ignored for the upcoming build.
You have uncommitted changes in the Git repository. They will
be ignored for the upcoming build:
#{git_status}
END_OF_MESSAGE
else
$stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
You have uncommited changes in the Git repository. Due to limitations
of the build system, you need to commit them before building Tails.
You have uncommitted changes in the Git repository. Due to limitations
of the build system, you need to commit them before building Tails:
#{git_status}
If you don't care about those changes and want to build Tails nonetheless,
please add `ignorechanges` to the TAILS_BUILD_OPTIONS environment
variable.
END_OF_MESSAGE
abort 'Uncommited changes. Aborting.'
abort 'Uncommitted changes. Aborting.'
end
end
end
def list_artifacts
user = vagrant_ssh_config('User')
stdout = capture_vagrant('ssh', '-c', "find '/home/#{user}/' -maxdepth 1 " +
"-name 'tails-*.iso*'").first
stdout.split("\n")
rescue VagrantCommandError
return Array.new
end
def remove_artifacts
list_artifacts.each do |artifact|
run_vagrant('ssh', '-c', "sudo rm -f '#{artifact}'")
end
end
desc "Make sure the vagrant user's home directory has no undesired artifacts"
task :ensure_clean_home_directory => ['vm:up'] do
remove_artifacts
end
task :validate_http_proxy do
if ENV['http_proxy']
proxy_host = URI.parse(ENV['http_proxy']).host
......@@ -239,7 +262,7 @@ task :validate_http_proxy do
end
desc 'Build Tails'
task :build => ['parse_build_options', 'ensure_clean_repository', 'validate_http_proxy', 'vm:up'] do
task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_home_directory', 'validate_http_proxy', 'vm:up'] do
if ENV['TAILS_RAM_BUILD'] && not(enough_free_memory_for_ram_build?)
$stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
......@@ -265,27 +288,47 @@ task :build => ['parse_build_options', 'ensure_clean_repository', 'validate_http
abort 'The virtual machine needs to be reloaded to change the number of CPUs. Aborting.'
end
# Let's make sure that, unless you know what you are doing and
# explicitly disable this, we always provision in order to ensure
# a valid, up-to-date build system.
run_vagrant('provision') unless ENV['TAILS_NO_AUTO_PROVISION']
exported_env = EXPORTED_VARIABLES.select { |k| ENV[k] }.
collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
status = primary_vm_chan.execute("#{exported_env} build-tails",
:error_check => false) do |fd, data|
(fd == :stdout ? $stdout : $stderr).write data
collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
run_vagrant('ssh', '-c', "#{exported_env} build-tails")
artifacts = list_artifacts
raise 'No build artifacts was found!' if artifacts.empty?
user = vagrant_ssh_config('User')
hostname = vagrant_ssh_config('HostName')
key_file = vagrant_ssh_config('IdentityFile')
$stderr.puts "Retrieving artifacts from Vagrant build box."
artifacts.each do |artifact|
run_vagrant('ssh', '-c', "sudo chown #{user} '#{artifact}'")
Process.wait(
Kernel.spawn(
'scp',
'-i', key_file,
# We need this since the user will not necessarily have a
# known_hosts entry. It is safe since an attacker must
# compromise libvirt's network config or the user running the
# command to modify the #{hostname} below.
'-o', 'StrictHostKeyChecking=no',
"#{user}@#{hostname}:#{artifact}", '.'
)
)
raise "Failed to fetch artifact '#{artifact}'" unless $?.success?
end
# Move build products to the current directory
FileUtils.mv Dir.glob("#{VAGRANT_PATH}/tails-*"),
File.expand_path('..', __FILE__), :force => true
exit status
remove_artifacts
end
namespace :vm do
desc 'Start the build virtual machine'
task :up => ['parse_build_options', 'validate_http_proxy'] do
case primary_vm_state
case vm_state
when :not_created
# Do not use non-existant in-VM proxy to download the basebox
if ENV['http_proxy'] == INTERNEL_HTTP_PROXY
if ENV['http_proxy'] == INTERNAL_HTTP_PROXY
ENV['http_proxy'] = nil
restore_internal_proxy = true
end
......@@ -312,31 +355,56 @@ namespace :vm do
END_OF_MESSAGE
end
env = Vagrant::Environment.new(:cwd => VAGRANT_PATH, :ui_class => Vagrant::UI::Basic)
result = env.cli('up')
abort "'vagrant up' failed" unless result
run_vagrant('up')
ENV['http_proxy'] = INTERNAL_HTTP_PROXY if restore_internal_proxy
end
ENV['http_proxy'] = INTERNEL_HTTP_PROXY if restore_internal_proxy
desc 'SSH into the builder VM'
task :ssh do
run_vagrant('ssh')
end
desc 'Stop the build virtual machine'
task :halt do
env = Vagrant::Environment.new(:cwd => VAGRANT_PATH, :ui_class => Vagrant::UI::Basic)
result = env.cli('halt')
abort "'vagrant halt' failed" unless result
run_vagrant('halt')
end
desc 'Re-run virtual machine setup'
task :provision => ['parse_build_options', 'validate_http_proxy'] do
env = Vagrant::Environment.new(:cwd => VAGRANT_PATH, :ui_class => Vagrant::UI::Basic)
result = env.cli('provision')
abort "'vagrant provision' failed" unless result
run_vagrant('provision')
end
desc 'Destroy build virtual machine (clean up all files)'
task :destroy do
env = Vagrant::Environment.new(:cwd => VAGRANT_PATH, :ui_class => Vagrant::UI::Basic)
result = env.cli('destroy', '--force')
abort "'vagrant destroy' failed" unless result
run_vagrant('destroy', '--force')
end
end
namespace :basebox do
desc 'Generate a new base box'
task :create do
box_dir = VAGRANT_PATH + '/definitions/tails-builder'
Dir.chdir(box_dir) do
`./generate-tails-builder-box.sh`
raise 'Base box generation failed!' unless $?.success?
end
box = Dir.glob("#{box_dir}/*.box").sort_by {|f| File.mtime(f) } .last
$stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
You have successfully generated a new Vagrant base box:
#{box}
To install the new base box, please run:
$ vagrant box add #{box}
To actually make Tails build using this base box, the `config.vm.box` key
in `vagrant/Vagrantfile` has to be updated. Please check the documentation
for details.
END_OF_MESSAGE
end
end
......@@ -63,6 +63,8 @@ chmod go+rX config/chroot_local-includes/home
chmod go+rX config/chroot_local-includes/lib
chmod go+rX config/chroot_local-includes/lib/live
chmod -R go+rx config/chroot_local-includes/lib/live/config
chmod go+rX config/chroot_local-includes/lib/live/mount
chmod -R go+rX config/chroot_local-includes/lib/systemd
chmod go+rX config/chroot_local-includes/live
chmod -R go+rX config/chroot_local-includes/usr
chmod -R go+rx config/chroot_local-includes/usr/local/bin
......@@ -74,6 +76,17 @@ chmod -R go+rX config/chroot_sources
# build the image
# we need /debootstrap/deburis to build a manifest of used packages:
DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keep-debootstrap-dir"
# use our own APT repository's key:
DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
gpg --homedir "$DEBOOTSTRAP_GNUPG_HOMEDIR" \
--import config/chroot_sources/tails.chroot.gpg
DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keyring=$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.gpg"
export DEBOOTSTRAP_OPTIONS
: ${MKSQUASHFS_OPTIONS:='-comp xz -Xbcj x86 -b 1024K -Xdict-size 1024K'}
MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS} -wildcards -ef chroot/usr/share/amnesia/build/mksquashfs-excludes"
export MKSQUASHFS_OPTIONS
......@@ -160,7 +173,8 @@ case "$LB_BINARY_IMAGES" in
;;
esac
BUILD_DEST_FILENAME="${BUILD_BASENAME}.${BUILD_FILENAME_EXT}"
BUILD_MANIFEST="${BUILD_DEST_FILENAME}.list"
BUILD_MANIFEST="${BUILD_DEST_FILENAME}.build-manifest"
BUILD_APT_SOURCES="${BUILD_DEST_FILENAME}.apt-sources"
BUILD_PACKAGES="${BUILD_DEST_FILENAME}.packages"
BUILD_LOG="${BUILD_DEST_FILENAME}.buildlog"
BUILD_START_FILENAME="${BUILD_DEST_FILENAME}.start.timestamp"
......@@ -172,30 +186,38 @@ trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
exec 2> >(tee -a "$BUILD_LOG" >&2)
trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
(
echo "Mirrors:"
apt-mirror debian
apt-mirror debian-security
apt-mirror torproject
echo "Additional sources:"
cat config/chroot_sources/*.chroot
) > "$BUILD_APT_SOURCES"
echo "Building $LB_BINARY_IMAGES image ${BUILD_BASENAME}..."
set -o pipefail
[ -z "$JENKINS_URL" ] || date --utc '+%s' > "$BUILD_START_FILENAME"
time eatmydata lb build noauto ${@}
RET=$?
if [ -e "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" ]; then
if [ "$RET" -eq 0 ]; then
[ -z "$JENKINS_URL" ] || date --utc '+%s' > "$BUILD_END_FILENAME"
echo "Image was successfully created"
if [ "$LB_BINARY_IMAGES" = iso ]; then
ISO_FILE="${BUILD_FILENAME}.${BUILD_FILENAME_EXT}"
print_iso_size "$ISO_FILE"
echo "Hybriding it..."
isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE"
print_iso_size "$ISO_FILE"
truncate -s %2048 "$ISO_FILE"
print_iso_size "$ISO_FILE"
fi
else
echo "Warning: image created, but lb build exited with code $RET"
echo "Image was successfully created"
[ "$RET" -eq 0 ] || \
echo "Warning: lb build exited with code $RET"
[ -z "$JENKINS_URL" ] || date --utc '+%s' > "$BUILD_END_FILENAME"
if [ "$LB_BINARY_IMAGES" = iso ]; then
ISO_FILE="${BUILD_FILENAME}.${BUILD_FILENAME_EXT}"
print_iso_size "$ISO_FILE"
echo "Hybriding it..."
isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE"
print_iso_size "$ISO_FILE"
truncate -s %2048 "$ISO_FILE"
print_iso_size "$ISO_FILE"
fi
echo "Renaming generated files..."
mv -i "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" "${BUILD_DEST_FILENAME}"
mv -i binary.packages "${BUILD_PACKAGES}"
generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
else
fatal "lb build failed ($?)."
fi
......@@ -16,30 +16,65 @@ fi
export LB_BOOTSTRAP_INCLUDE='eatmydata'
# sanity checks
if grep -qs -E '^Pin:\s+release\s+.*a=' config/chroot_apt/preferences ; then
echo "Found unsupported a= syntax in config/chroot_apt/preferences,"
echo "use n= instead. Exiting."
exit 1
fi
if grep -qs -E '^Pin:\s+release\s+.*o=Debian Backports' \
config/chroot_apt/preferences ; then
echo "Found unsupported 'o=Debian Backports' syntax,"
echo "in config/chroot_apt/preferences. Use o=Debian instead. Exiting."
exit 1
fi
# init variables
RUN_LB_CONFIG="lb config noauto"
# init config/ with defaults for the target distribution
$RUN_LB_CONFIG --distribution wheezy ${@}
$RUN_LB_CONFIG --distribution jessie ${@}
# set up everything for time-based snapshots:
apt-snapshots-serials prepare-build
DEBIAN_MIRROR="$(apt-mirror debian)"
DEBIAN_SECURITY_MIRROR="$(apt-mirror debian-security)"
TORPROJECT_MIRROR="$(apt-mirror torproject)"
[ -n "$DEBIAN_MIRROR" ] || exit 1
[ -n "$DEBIAN_SECURITY_MIRROR" ] || exit 1
[ -n "$TORPROJECT_MIRROR" ] || exit 1
perl -pi \
-E \
"s|^(deb(?:-src)?\s+)https?://ftp[.]us[.]debian[.]org/debian/?(\s+)|\$1$DEBIAN_MIRROR\$2| ; \
s|^(deb(?:-src)?\s+)https?://deb[.]torproject[.]org/torproject[.]org/?(\s+)|\$1$TORPROJECT_MIRROR\$2|" \
config/chroot_sources/*.chroot \
|| exit 1
# set Amnesia's general options
$RUN_LB_CONFIG \
--verbose \
--apt-recommends false \
--backports false \
--binary-images iso \
--binary-indices false \
--checksums none \
--bootappend-live "${AMNESIA_APPEND}" \
--bootstrap "cdebootstrap" \
--bootstrap debootstrap \
--bootstrap-config tails-build-jessie \
--archive-areas "main contrib non-free" \
--includes none \
--iso-application="The Amnesic Incognito Live System" \
--iso-publisher="https://tails.boum.org/" \
--iso-volume="TAILS ${AMNESIA_FULL_VERSION}" \
--memtest none \
--mirror-binary "http://ftp.us.debian.org/debian/" \
--mirror-bootstrap "http://ftp.us.debian.org/debian/" \
--mirror-chroot "http://ftp.us.debian.org/debian/" \
--mirror-binary "$DEBIAN_MIRROR" \
--mirror-bootstrap "$DEBIAN_MIRROR" \
--mirror-chroot "$DEBIAN_MIRROR" \
--mirror-binary-security "$DEBIAN_SECURITY_MIRROR" \
--mirror-chroot-security "$DEBIAN_SECURITY_MIRROR" \
--packages-lists="standard" \
--tasks="standard" \
--linux-packages="linux-image-3.16.0-4" \
......@@ -97,3 +132,20 @@ fi
# custom APT sources
tails-custom-apt-sources > config/chroot_sources/tails.chroot
# tails-transform-mirror-url and its dependencies
install -m 0755 \
submodules/mirror-pool-dispatcher/bin/tails-transform-mirror-url \
config/chroot_local-includes/usr/local/bin/
install -m 0755 -d config/chroot_local-includes/usr/local/lib/nodejs
install -m 0755 \
submodules/mirror-pool-dispatcher/lib/js/mirror-dispatcher.js \
config/chroot_local-includes/usr/local/lib/nodejs/
# custom debootstrap script, setting some APT magic to log downloads:
patch \
--follow-symlinks \
--output=/usr/share/debootstrap/scripts/tails-build-jessie \
/usr/share/debootstrap/scripts/jessie \
data/debootstrap/scripts/jessie.patch
sed -i "s,%%topdir%%,$(pwd)," /usr/share/debootstrap/scripts/tails-build-jessie
#!/bin/bash
set -e
set -u
. "$(dirname $0)/utils.sh"
ARCHIVE="$1"
output_tagged_snapshot() {
local archive="$1"
local tag="$2"
local snapshot=$(branch_name_to_suite "$tag")
echo "http://tagged.snapshots.deb.tails.boum.org/$snapshot/$archive"
}
output_time_based_snapshot() {
local archive="$1"
local serial="$2"
echo "http://time-based.snapshots.deb.tails.boum.org/$archive/$serial"
}
### Sanity checks
[ -n "$ARCHIVE" ] || exit 1
### Main
SERIAL=$(cat "config/APT_snapshots.d/$ARCHIVE/serial")
RESOLVED_SERIAL=$(cat "tmp/APT_snapshots.d/$ARCHIVE/serial")
if [ "$(base_branch)" = stable ]; then
if version_was_released "$(version_in_changelog)"; then
on_a_tag \
|| fatal "Not building from stable, but last version in changelog" \
"was released"
output_tagged_snapshot "$ARCHIVE" "$(version_in_changelog)"
else