Merge branch 'devel' into feature/stretch

58521f9b · bertagaz · e82b1b6e · 9e90b7c3 · 58521f9b · 58521f9b
Commit 58521f9b authored May 12, 2017 by bertagaz
--- a/Rakefile
+++ b/Rakefile
@@ -404,46 +404,39 @@ task :build => ['parse_build_options', 'ensure_clean_repository', 'maybe_clean_u
    run_vagrant_ssh("#{exported_env} build-tails")

    artifacts = list_artifacts
-    raise 'No build artifacts was found!' if artifacts.empty?
+    raise 'No build artifacts were found!' if artifacts.empty?
    user     = vagrant_ssh_config('User')
    hostname = vagrant_ssh_config('HostName')
    key_file = vagrant_ssh_config('IdentityFile')
    $stderr.puts "Retrieving artifacts from Vagrant build box."
-    artifacts.each do |artifact|
-      run_vagrant_ssh("sudo chown #{user} '#{artifact}'")
-      Process.wait(
-        Kernel.spawn(
-          'scp',
-          '-i', key_file,
-          # We need this since the user will not necessarily have a
-          # known_hosts entry. It is safe since an attacker must
-          # compromise libvirt's network config or the user running the
-          # command to modify the #{hostname} below.
-          '-o', 'StrictHostKeyChecking=no',
-          '-o', 'UserKnownHostsFile=/dev/null',
-          "#{user}@#{hostname}:#{artifact}", "#{ENV['ARTIFACTS']}"
-        )
-      )
-      raise "Failed to fetch artifact '#{artifact}'" unless $?.success?
-    end
+    run_vagrant_ssh(
+      "sudo chown #{user} " + artifacts.map { |a| "'#{a}'" } .join(' ')
+    )
+    fetch_command = [
+      'scp',
+      '-i', key_file,
+      # We need this since the user will not necessarily have a
+      # known_hosts entry. It is safe since an attacker must
+      # compromise libvirt's network config or the user running the
+      # command to modify the #{hostname} below.
+      '-o', 'StrictHostKeyChecking=no',
+      '-o', 'UserKnownHostsFile=/dev/null',
+    ]
+    fetch_command += artifacts.map { |a| "#{user}@#{hostname}:#{a}" }
+    fetch_command << ENV['ARTIFACTS']
+    run_command(*fetch_command)
    clean_up_builder_vms unless $keep_running
  ensure
    clean_up_builder_vms if $force_cleanup
  end
 end

-def box_name(vagrantfile_contents = open('vagrant/Vagrantfile') { |f| f.read })
-  /^\s*config.vm.box = '([^']+)'/.match(vagrantfile_contents)[1]
-end
-
-def has_box?(name = box_name)
-  not(capture_vagrant('box', 'list').grep(/^#{name}\s+\(libvirt,/).empty?)
+def has_box?
+  not(capture_vagrant('box', 'list').grep(/^#{box_name}\s+\(libvirt,/).empty?)
 end

-def domain_name(name = box_name)
-  # Vagrant drops some characters when creating the domain and volumes
-  # based on the box name.
-  "#{name.delete('+')}_default"
+def domain_name
+  "#{box_name}_default"
 end

 def clean_up_builder_vms
@@ -577,15 +570,11 @@ namespace :basebox do
      time needed for downloading around 250 MiB of Debian packages.

    END_OF_MESSAGE
-    # args will be either [serial] or [serial, comment]
-    args = /^tails-builder-(?:[^-]+)-(?:[^-]+)-(\d{10})(?:-(.+))?$/.match(box_name)[1,2].select { |m| not(m.nil?) }
    box_dir = VAGRANT_PATH + '/definitions/tails-builder'
-    Dir.chdir(box_dir) do
-      run_command('./generate-tails-builder-box.sh', *args)
-    end
+    run_command("#{box_dir}/generate-tails-builder-box.sh")
    # Let's use an absolute path since run_vagrant changes the working
    # directory but File.delete doesn't
-    box_path = "#{Dir.pwd}/vagrant/definitions/tails-builder/#{box_name}.box"
+    box_path = "#{box_dir}/#{box_name}.box"
    run_vagrant('box', 'add', '--name', box_name, box_path)
    File.delete(box_path)
    end

--- a/auto/scripts/apt-snapshots-serials
+++ b/auto/scripts/apt-snapshots-serials
@@ -6,6 +6,8 @@ set -o pipefail

 BASE_URL=http://time-based.snapshots.deb.tails.boum.org/
 CONFIG=config/APT_snapshots.d
+SERIAL_ONLY=
+FREEZE_EXCEPTIONS=debian-security

 get_latest_serial() {
 	origin=$1
@@ -19,6 +21,14 @@ if [ $# -eq 0 ]; then
 else
 	action="${1}"
 	shift
+	if [ "${1:-}" = --print-serials-only ]; then
+		SERIAL_ONLY=yes
+		shift
+	fi
+	if [ "${1:-}" = --freeze-debian-security ]; then
+		FREEZE_EXCEPTIONS=
+		shift
+	fi
 	if [ $# -eq 0 ]; then
 		ORIGINS="$(cd ${CONFIG}; ls -d *)"
 	else
@@ -29,14 +39,14 @@ fi
 case "$action" in
 	cat)
 		for origin in $ORIGINS; do
-			echo -n "$origin: "
+			[ -z "${SERIAL_ONLY}" ] && echo -n "$origin: "
 			cat "$CONFIG/$origin/serial"
 		done
 	;;
 	get-latest)
 		for origin in $ORIGINS; do
-			online=$(get_latest_serial $origin)
-			echo "$origin: $online"
+			[ -z "${SERIAL_ONLY}" ] && echo -n "$origin: "
+			get_latest_serial $origin
 		done
 	;;
 	freeze)
@@ -44,7 +54,7 @@ case "$action" in
 			serial_file="$CONFIG/$origin/serial"
 			git=$(cat $serial_file)
 			case "$origin" in
-				debian-security)
+				${FREEZE_EXCEPTIONS})
 					new=latest
 					;;
 				*)

--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -21,10 +21,9 @@ require_relative 'lib/tails_build_settings'
 ENV['VAGRANT_DEFAULT_PROVIDER'] = 'libvirt'

 Vagrant.configure("2") do |config|
-  config.vm.box = 'tails-builder-amd64-jessie-2017042704-feature-11980'
+  config.vm.box = box_name
  config.vm.box_url = "http://dl.amnesia.boum.org/tails/project/vagrant/#{config.vm.box}.box"
-  config.vm.box_download_checksum_type = "sha256"
-  config.vm.box_download_checksum = '9cd2df4b0b5f520b46fbfc126f46a56d0f7c2f20462d9b64bbdf75acd75edc49'
+  config.vm.box_url = "http://127.0.0.1/dev/null"
  ENV['EXPORTED_VARIABLES'] ||= ""
  env_vars = ENV.select { |k,v| ENV['EXPORTED_VARIABLES'].split.include?(k) }
                   .map { |k, v| "#{k}='#{v}'" } .join(' ')

--- a/vagrant/definitions/tails-builder/config/APT_snapshots.d/.placeholder
+++ b/vagrant/definitions/tails-builder/config/APT_snapshots.d/.placeholder
--- a/vagrant/definitions/tails-builder/config/APT_snapshots.d/debian-security/serial
+++ b/vagrant/definitions/tails-builder/config/APT_snapshots.d/debian-security/serial
+2017051002
--- a/vagrant/definitions/tails-builder/config/APT_snapshots.d/debian/serial
+++ b/vagrant/definitions/tails-builder/config/APT_snapshots.d/debian/serial
+2017051002
--- a/vagrant/definitions/tails-builder/config/APT_snapshots.d/tails/serial
+++ b/vagrant/definitions/tails-builder/config/APT_snapshots.d/tails/serial
+2017051002
--- a/vagrant/definitions/tails-builder/generate-tails-builder-box.sh
+++ b/vagrant/definitions/tails-builder/generate-tails-builder-box.sh
@@ -5,27 +5,33 @@ set -u
 # Based on ypcs' scripts found at:
 #     https://github.com/ypcs/vmdebootstrap-vagrant/

-SERIAL="${1}"
-COMMENT="${2:-}"
-ARCHITECTURE="amd64"
-DISTRIBUTION="jessie"
+GIT_DIR="$(git rev-parse --show-toplevel)"
+cd "${GIT_DIR}/vagrant/definitions/tails-builder"
+
+build_setting() {
+    ruby -I "${GIT_DIR}/vagrant/lib" \
+         -e "require 'tails_build_settings.rb'; print ${1}"
+}
+
+get_serial() {
+    "${GIT_DIR}/auto/scripts/apt-snapshots-serials" \
+        cat --print-serials-only "${1}"
+}
+
+TARGET_NAME="$(build_setting box_name)"
+TARGET_IMG="${TARGET_NAME}.qcow2"
+TARGET_BOX="${TARGET_NAME}.box"
+ARCHITECTURE="$(build_setting ARCHITECTURE)"
+DISTRIBUTION="$(build_setting DISTRIBUTION)"
+HOSTNAME="vagrant-${DISTRIBUTION}"
 USERNAME="vagrant"
 PASSWORD="vagrant"
 SIZE="20G"
-HOSTNAME="vagrant-${DISTRIBUTION}"
-TARGET_NAME="tails-builder-${ARCHITECTURE}-${DISTRIBUTION}-${SERIAL}"
-if [ -n "${COMMENT}" ]; then
-   TARGET_NAME="${TARGET_NAME}-${COMMENT}"
-fi
-TARGET_IMG="${TARGET_NAME}.qcow2"
-TARGET_BOX="${TARGET_NAME}.box"
 LC_ALL=C

-SECURITY_SERIAL="$(
-    cd ../../.. && \
-    auto/scripts/apt-snapshots-serials get-latest debian-security | \
-        sed s/^debian-security:[^0-9]*// \
-)"
+DEBIAN_SERIAL="$(get_serial debian)"
+DEBIAN_SECURITY_SERIAL="$(get_serial debian-security)"
+TAILS_SERIAL="$(get_serial tails)"

 DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
 gpg --homedir "${DEBOOTSTRAP_GNUPG_HOMEDIR}" \
@@ -43,8 +49,9 @@ sudo ${http_proxy:+http_proxy="$http_proxy"} \
     LC_ALL=${LC_ALL} \
     ARCHITECTURE=${ARCHITECTURE} \
     DISTRIBUTION=${DISTRIBUTION} \
-     SECURITY_SERIAL=${SECURITY_SERIAL} \
-     SERIAL=${SERIAL} \
+     DEBIAN_SERIAL=${DEBIAN_SERIAL} \
+     DEBIAN_SECURITY_SERIAL=${DEBIAN_SECURITY_SERIAL} \
+     TAILS_SERIAL=${TAILS_SERIAL} \
     vmdebootstrap \
     --arch "${ARCHITECTURE}" \
     --distribution "${DISTRIBUTION}" \
@@ -54,7 +61,7 @@ sudo ${http_proxy:+http_proxy="$http_proxy"} \
     --grub \
     --hostname "${HOSTNAME}" \
     --log-level "debug" \
-     --mirror "http://time-based.snapshots.deb.tails.boum.org/debian/${SERIAL}" \
+     --mirror "http://time-based.snapshots.deb.tails.boum.org/debian/${DEBIAN_SERIAL}" \
     --debootstrapopts "keyring=${DEBOOTSTRAP_GNUPG_PUBRING}" \
     --owner "${SUDO_USER:-${USER}}" \
     --kernel-package "linux-image-${ARCHITECTURE}" \

--- a/vagrant/definitions/tails-builder/postinstall.sh
+++ b/vagrant/definitions/tails-builder/postinstall.sh
@@ -43,11 +43,11 @@ cat "/etc/apt/sources.list" | \
 	sed -e 's/jessie/jessie-backports/' \
 	> "/etc/apt/sources.list.d/jessie-backports.list"

-echo "deb http://time-based.snapshots.deb.tails.boum.org/debian-security/${SECURITY_SERIAL}/ jessie/updates main" \
+echo "deb http://time-based.snapshots.deb.tails.boum.org/debian-security/${DEBIAN_SECURITY_SERIAL}/ jessie/updates main" \
 	> "/etc/apt/sources.list.d/jessie-security.list"

 echo "I: Adding our builder-jessie suite with live-build, pin it low."
-echo "deb http://time-based.snapshots.deb.tails.boum.org/tails/${SERIAL}/ builder-jessie main" > "/etc/apt/sources.list.d/tails.list"
+echo "deb http://time-based.snapshots.deb.tails.boum.org/tails/${TAILS_SERIAL}/ builder-jessie main" > "/etc/apt/sources.list.d/tails.list"
 sed -e 's/^[[:blank:]]*//' > /etc/apt/preferences.d/tails <<EOF
 	Package: *
 	Pin: release o=Tails,n=builder-jessie
@@ -159,7 +159,6 @@ rm -rf \
   /var/cache/apt/*.bin \
   /var/cache/apt/archives/*.deb \
   /var/log/installer \
-   /var/lib/dhcp/* \
-    || :
+   /var/lib/dhcp/*

 exit 0
--- a/vagrant/lib/tails_build_settings.rb
+++ b/vagrant/lib/tails_build_settings.rb
@@ -13,3 +13,15 @@ VM_MEMORY_FOR_DISK_BUILDS = 512

 # Virtual machine memory size for in-memory builds
 VM_MEMORY_FOR_RAM_BUILDS = VM_MEMORY_FOR_DISK_BUILDS + BUILD_SPACE_REQUIREMENT
+
+# The builder VM's platform
+ARCHITECTURE = "amd64"
+DISTRIBUTION = "jessie"
+
+# The name of the Vagrant box
+def box_name
+  git_root = `git rev-parse --show-toplevel`.chomp
+  shortid, date = `git log -1 --date="format:%Y%m%d" --pretty="%h %ad" -- \
+                   #{git_root}/vagrant/definitions/tails-builder/`.chomp.split
+  return "tails-builder-#{ARCHITECTURE}-#{DISTRIBUTION}-#{date}-#{shortid}"
+end
--- a/vagrant/provision/setup-tails-builder
+++ b/vagrant/provision/setup-tails-builder
@@ -17,11 +17,10 @@ fi

 latest_serial(){
 	local archive="${1}"
-        get_serial="$(
-            cd /amnesia.git
-            auto/scripts/apt-snapshots-serials get-latest ${archive}
-        )"
-	echo "${get_serial#${1}: }"
+	(
+		cd /amnesia.git
+		auto/scripts/apt-snapshots-serials get-latest --print-serials-only ${archive}
+        )
 }

 if [ "${TAILS_PROXY_TYPE}" = "vmproxy" ]; then

--- a/wiki/src/contribute/APT_repository/time-based_snapshots.mdwn
+++ b/wiki/src/contribute/APT_repository/time-based_snapshots.mdwn
@@ -77,9 +77,21 @@ that shall be used during the freeze:

        git checkout "$RELEASE_BRANCH" && \
        ./auto/scripts/apt-snapshots-serials freeze && \
+
+Normally when freezing the APT snapshots used when building Tails we
+also freeze the snapshots used to generate our builder VM:
+
+        ( \
+            cd vagrant/definitions/tails-builder && \
+            ../../../auto/scripts/apt-snapshots-serials freeze --freeze-debian-security \
+        )
+
+Then commit:
+
        git commit \
            -m "Freeze APT snapshots for ${VERSION}." \
-            config/APT_snapshots.d/*/serial
+            config/APT_snapshots.d/*/serial \
+            vagrant/definitions/tails-builder/config/APT_snapshots.d/*/serial

 <a id="thaw"></a>

@@ -127,15 +139,17 @@ days from now:

 	git checkout "$RELEASE_BRANCH" && \
 	(
-	   cd config/APT_snapshots.d && \
-	   for ARCHIVE in * ; do
-           if ! grep -qs '^latest$' "$ARCHIVE"/serial; then
-               ssh reprepro-time-based-snapshots@incoming.deb.tails.boum.org \
-                   tails-bump-apt-snapshot-valid-until \
-                       "$ARCHIVE" "$(cat "$ARCHIVE"/serial)" \
-                       "$DAYS_FROM_NOW"
-           fi
-	   done
+        for dir in config/APT_snapshots.d vagrant/definitions/tails-builder/config/APT_snapshots.d; do
+            cd "${dir}" && \
+	        for ARCHIVE in * ; do
+                if ! grep -qs '^latest$' "$ARCHIVE"/serial; then
+                    ssh reprepro-time-based-snapshots@incoming.deb.tails.boum.org \
+                        tails-bump-apt-snapshot-valid-until \
+                            "$ARCHIVE" "$(cat "$ARCHIVE"/serial)" \
+                            "$DAYS_FROM_NOW"
+                fi
+	        done
+        done
 	)

 Stop tracking a distribution