Merge remote-tracking branch 'origin/devel' into test/wip-improved-snapshots

Conflicts:
	features/step_definitions/common_steps.rb
	features/step_definitions/firewall_leaks.rb
	features/step_definitions/pidgin.rb
	features/step_definitions/time_syncing.rb
	features/step_definitions/torified_gnupg.rb
	features/step_definitions/torified_misc.rb
	features/time_syncing.feature
	features/torified_browsing.feature

Includes s/@vm/$vm/ fixes, removal of "next if @skip_..." lines and
adaptations of new scenarios to use the appropriate
snapshots.

config/binary_local-hooks/40-include_syslinux_in_ISO_filesystem

@@ -21,23 +21,44 @@ Set_defaults
 # Seems like we'll have work to do
 Echo_message 'including syslinux in the ISO filesystem'
 
-# Variables
+### Functions
+
+syslinux_deb_version_in_chroot () {
+   chroot chroot dpkg-query -W -f='${Version}\n' syslinux
+}
+
+### Variables
 LINUX_BINARY_UTILS_DIR='binary/utils/linux'
 WIN32_BINARY_UTILS_DIR='binary/utils/win32'
 BINARY_MBR_DIR='binary/utils/mbr'
 CHROOT_SYSLINUX_BIN='chroot/usr/bin/syslinux'
 CHROOT_SYSLINUX_MBR='chroot/usr/lib/SYSLINUX/gptmbr.bin'
 CHROOT_TEMP_APT_SOURCES='chroot/etc/apt/sources.list.d/tmp-deb-src.list'
+SYSLINUX_DEB_VERSION_IN_CHROOT=$(syslinux_deb_version_in_chroot)
 
-# Functions
-
-syslinux_deb_version_in_chroot () {
-   chroot chroot dpkg-query -W -f='${Version}\n' syslinux
-}
-
-# Main
+### Main
 mkdir -p "$LINUX_BINARY_UTILS_DIR" "$WIN32_BINARY_UTILS_DIR" "$BINARY_MBR_DIR"
+
+# Copy 32-bit syslinux binary
 cp "$CHROOT_SYSLINUX_BIN" "$LINUX_BINARY_UTILS_DIR/"
+
+# Copy 64-bit syslinux binary
+(
+   olddir=$(pwd)
+   workdir=$(mktemp -d)
+   cd "$workdir"
+   chroot="$olddir/chroot"
+   Chroot "$chroot" \
+	  apt-get --yes download \
+	  syslinux:amd64="$SYSLINUX_DEB_VERSION_IN_CHROOT"
+   dpkg-deb --extract "$chroot"/syslinux_*.deb .
+   rm "$chroot"/syslinux_*.deb
+   cp ./usr/bin/syslinux "$olddir/$LINUX_BINARY_UTILS_DIR/syslinux-amd64"
+   cd "$olddir"
+   rm -r "$workdir"
+)
+
+# Copy syslinux MBR
 cp "$CHROOT_SYSLINUX_MBR" "$BINARY_MBR_DIR/mbr.bin"
 
 cat chroot/etc/apt/sources.list chroot/etc/apt/sources.list.d/*.list \
@@ -47,7 +68,7 @@ cat chroot/etc/apt/sources.list chroot/etc/apt/sources.list.d/*.list \
    > "$CHROOT_TEMP_APT_SOURCES"
 Chroot chroot apt-get --yes update
 Chroot chroot apt-get --yes install dpkg-dev
-Chroot chroot apt-get source syslinux="$(syslinux_deb_version_in_chroot)"
+Chroot chroot apt-get source syslinux="$SYSLINUX_DEB_VERSION_IN_CHROOT"
 cp chroot/syslinux-*/bios/win32/syslinux.exe "$WIN32_BINARY_UTILS_DIR/"
 rm -r chroot/syslinux*
 rm "$CHROOT_TEMP_APT_SOURCES"

config/chroot_local-hooks/03-dpkg-architectures

+#! /bin/sh
+
+set -e
+
+echo "Configuring dpkg architectures"
+
+dpkg --add-architecture amd64

config/chroot_local-includes/etc/apt/apt.conf.d/13architectures

+APT::Architectures {"i386"; "amd64";};

config/chroot_local-includes/etc/tor-browser/profile/preferences/0000tails.js

@@ -78,4 +78,3 @@ pref("extensions.update.enabled", false);
 pref("layout.spellcheckDefault", 0);
 pref("network.dns.disableIPv6", true);
 pref("security.warn_submit_insecure", true);
-pref("network.proxy.no_proxies_on", "10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16");

config/chroot_local-includes/usr/share/tails/tbb-dist-url.txt

-http://torbrowser-archive.tails.boum.org/4.5.1/
+http://torbrowser-archive.tails.boum.org/4.5.3-build2/

config/chroot_local-includes/usr/share/tails/tbb-sha256sums.txt

-77c272f9ed7cb2703d559f6dde24b3caa86c8f43db1b1a36454e0992d3922256  tor-browser-linux32-4.5.1_ar.tar.xz
-e37fd8e2731206fa06f219237bf9a437b3b490c17f18cb532b887f2d4b218e13  tor-browser-linux32-4.5.1_de.tar.xz
-1ccf6a3bb6cdcef95639f090939e172ba860fc5c5582ebfc1cf49dd87d7b7f3a  tor-browser-linux32-4.5.1_en-US.tar.xz
-26f5810b7f93823a86d78d09476eb01f65e1f99d431b37ca4d3c29bd4370b44d  tor-browser-linux32-4.5.1_es-ES.tar.xz
-4f6ba52e2d740a1df8f121c3b598ac56abfaebf1714213f731cec818022629ca  tor-browser-linux32-4.5.1_fa.tar.xz
-db01d0f30994f65fe41bb820d86b50f166615d52648d6b233ba544b9e68ba3fe  tor-browser-linux32-4.5.1_fr.tar.xz
-531722b9189e54d3b4e492ba2e37bdce9e996fee5d76401d2317fb36c571435f  tor-browser-linux32-4.5.1_it.tar.xz
-1bab54c31dfc9c1e06d8c6f002576fb0bc0f962888c44d568bea74ff0da6c759  tor-browser-linux32-4.5.1_ko.tar.xz
-75d4868e7694ede63ca17c5e3d44ad88475595e1f16bdbe95e611c493efa742e  tor-browser-linux32-4.5.1_nl.tar.xz
-a6071a76a613a4955139ddd28d790f8d5d645ce17dde802d1a4ac9fb5465ac1c  tor-browser-linux32-4.5.1_pl.tar.xz
-5d450616d746635dd4a6886771c5e88dc92b2fb12c6a7af7c7bcb525b81bfc25  tor-browser-linux32-4.5.1_pt-PT.tar.xz
-10ddf846984e80ec65a1bf29f480c3c82dc3ba23439d779619800101221a5de2  tor-browser-linux32-4.5.1_ru.tar.xz
-bbde08f326cf2e3594d6a150d22b161c16f0c7b5e9d29591a43a85aba118be4f  tor-browser-linux32-4.5.1_tr.tar.xz
-2e9748478e974a81634ee2523e0f704de82796b71caf270de5ff459a3ebfc1dc  tor-browser-linux32-4.5.1_vi.tar.xz
-3a2b2c089ac03cebd1507eff7af2c707f441c07894a51481376581448fbcde7c  tor-browser-linux32-4.5.1_zh-CN.tar.xz
+9a6425afeeae40f1cd7ebd3dd0a8672b8ed13e4df1863282ee39607c5b3cea2d  tor-browser-linux32-4.5.3_ar.tar.xz
+d9af6bf2585ac7905534fecd8c9df016fa833ba90b95babdfbf7b2225760c774  tor-browser-linux32-4.5.3_de.tar.xz
+154d659583048e91870c40921561f0519babf6d3c9ac439f6fb74ed66824463f  tor-browser-linux32-4.5.3_en-US.tar.xz
+196cfd81e726d0e1f7ecf0fe0183eac6b7e2cf8e8c5bc89b9105ce4d82e0922a  tor-browser-linux32-4.5.3_es-ES.tar.xz
+61dfee81c930f1b6c12911917b880bd62f99a7e61fdcc3d8d5cb71657770adeb  tor-browser-linux32-4.5.3_fa.tar.xz
+dcb98570ac927298a086771524b283ee4f64268b72d0b5ec36268817073612aa  tor-browser-linux32-4.5.3_fr.tar.xz
+6d94c31bf8ed708a49e0e4ecdbfd7c20db1b382fe4d8039971aadd8c87240fc6  tor-browser-linux32-4.5.3_it.tar.xz
+59dad0fbcbc8fd02953af03cb587c1b89d534812c6580d9cd78e883c55f2b34c  tor-browser-linux32-4.5.3_ko.tar.xz
+0cf9401dd5383212871c12bc7c7db44953aa4796db2692528c949ebc2b0219e3  tor-browser-linux32-4.5.3_nl.tar.xz
+487ebbe2260666476a9a72e927025783567767dc3fc1c5403ad60e7f37c80734  tor-browser-linux32-4.5.3_pl.tar.xz
+78688df77ad688c0458e5ff959b5ff1bf5579435ee480093e98f9ad537400dbd  tor-browser-linux32-4.5.3_pt-PT.tar.xz
+0a1cbdae6e13dad17ab228970a3b7c0c1dd86c71c1ad4950d4c591947be1b04b  tor-browser-linux32-4.5.3_ru.tar.xz
+4ebce91c056b67e4b2aed183d816d299b005111f895f726d2bd49494231a76cb  tor-browser-linux32-4.5.3_tr.tar.xz
+71a80614bee6a0349b3e9e297c08925d0a6682768bf3f43352586d4c28e7dacd  tor-browser-linux32-4.5.3_vi.tar.xz
+6675936035691de5e20ac899c5c1a004462b41e38bda29040aba7351ca4fb3b2  tor-browser-linux32-4.5.3_zh-CN.tar.xz

debian/changelog

@@ -4,11 +4,88 @@ tails (1.5) UNRELEASED; urgency=medium
 
  -- Tails developers <tails@boum.org>  Mon, 04 May 2015 14:48:25 +0200
 
-tails (1.4.1) UNRELEASED; urgency=medium
+tails (1.4.1) unstable; urgency=medium
 
-  * Dummy entry.
+  * Security fixes
+    - Upgrade Tor Browser to 4.5.3, based on Firefox 31.8.0 ESR. (Closes: #9649)
+    - Upgrade Tor to 0.2.6.9-1~d70.wheezy+1+tails2, which includes a circuit
+      isolation bugfix. (Closes: #9560)
+    - AppArmor: deny Tor Browser access to the list of recently used files.
+      (Closes: #9126)
+    - Upgrade OpenSSL to 1.0.1e-2+deb7u17.
+    - Upgrade Linux to 3.16.7-ckt11-1.
+    - Upgrade CUPS to 1.5.3-5+deb7u6.
+    - Upgrade FUSE to 2.9.0-2+deb7u2.
+    - Upgrade libsqlite3-0 to 3.7.13-1+deb7u2.
+    - Upgrade ntfs-3g and ntfsprogs to 1:2012.1.15AR.5-2.1+deb7u2.
+    - Upgrade p7zip-full to 9.20.1~dfsg.1-4+deb7u1.
+
+  * Bugfixes
+    - Fix automatic upgrades in Windows Camouflage mode. (Closes: #9413)
+    - Don't ship the snakeoil SSL key pair generated by ssl-cert in the ISO.
+      (Closes: #9416)
+    - Partially fix the truncated notifications issue. (#7249)
+
+  * Minor improvements
+    - Disable the hwclock.sh initscript at reboot/shutdown time.
+      This is an additional safety measure to ensure that the hardware clock
+      is not modified. (Closes: #9364)
+    - Stop shipping /var/cache/man/*, to make ISOs and IUKs smaller.
+      (Closes: #9417)
+    - Update torbrowser-AppArmor-profile.patch to apply cleanly on top of the
+      profile shipped with torbrowser-launcher 0.2.0-1.
+    - Add the jessie/updates APT repo and set appropriate pinning.
+    - Upgrade Electrum to 1.9.8-4~bpo70+1.
+    - Upgrade kernel firmware packages to 0.44.
+
+  * Build system
+    - Install the Linux kernel from Debian Jessie. (Closes: #9341)
+    - Remove files that are not under version control when building in Jenkins.
+      (Closes: #9406)
+    - Don't modify files in the source tree before having possibly merged
+      the base branch into it. (Closes: #9406)
+    - Make it so eatmydata is actually used during a greater part of the build
+      process. This includes using eatmydata from wheezy-backports.
+      (Closes: #9419, #9523)
+    - release script: adjust to support current Debian sid.
 
- -- Tails developers <tails@boum.org>  Tue, 12 May 2015 17:19:13 +0200
+  * Test suite
+    - Test the system clock sanity check we do at boot. (Closes: #9377)
+    - Remove the impossible "Clock way in the past" scenarios.
+      Thanks to config/chroot_local-includes/lib/live/config/0001-sane-clock,
+      these scenarios cannot happen, and since we test that it works they
+      can be safely removed.
+    - Test that the hardware clock is not modified at shutdown. (Closes: #9557)
+    - Pidgin: retry looking for the roadmap URL in the topic.
+    - Avoid showing Pidgin's tooltips during test, potentially confusing Sikuli.
+      (Closes: #9317)
+    - Test all OpenPGP keys shipped with Tails. (Closes: #9402)
+    - Check that notification-daemon is running when looking for notifications
+      fails. (Closes: #9332)
+    - Allow using the cucumber formatters however we want. (Closes: #9424)
+    - Enable Spice in the guest, and blacklist the psmouse kernel module,
+      to help with lost mouse events. (Closes: #9425)
+    - Automate testing Torbutton's 'New Identity' feature. (Closes: #9286)
+    - Test that Seahorse is configured to use the correct keyserver.
+      (Closes: #9339)
+    - Always export TMPDIR back to the test suite's shell environment.
+      (Closes: #9479)
+    - Make OpenPGP tests more reliable:
+      · Retry accessing the OpenPGP applet menus on failure. (Closes: #9355)
+      · Retry accessing menus in Seahorse on failure. (Closes: #9344)
+    - Focus the Pidgin conversation window before any attempt to interact
+      with it. (Closes: #9317)
+    - Use convertkey from the (backported to Jessie) Debian package,
+      instead of our own copy of that script. (Closes: #9066)
+    - Make the memory erasure tests more robust (Closes: #9329):
+      · Bump /proc/sys/vm/min_free_kbytes when running fillram.
+      · Actually set oom_adj for the remote shell when running fillram.
+      · Try to be more sure that we OOM kill fillram.
+      · Run fillram as non-root.
+    - Only try to build the storage pool if TailsToasterStorage isn't found.
+      (Closes: #9568)
+
+ -- Tails developers <tails@boum.org>  Sun, 28 Jun 2015 19:46:25 +0200
 
 tails (1.4) unstable; urgency=medium
 

features/config/defaults.yml

@@ -5,6 +5,7 @@ LIBVIRT_NETWORK_UUID: f2305af3-2a64-4f16-afe6-b9dbf02a597e
 DEBUG: false
 PAUSE_ON_FAIL: false
 SIKULI_RETRY_FINDFAILED: false
+MAX_NEW_TOR_CIRCUIT_RETRIES: 5
 TMPDIR: "/tmp/TailsToaster"
 
 Unsafe_SSH_private_key: |

features/step_definitions/common_steps.rb

@@ -384,7 +384,8 @@ Then /^I see "([^"]*)" after at most (\d+) seconds$/ do |image, time|
 end
 
 Then /^all Internet traffic has only flowed through Tor$/ do
-  leaks = FirewallLeakCheck.new(@sniffer.pcap_file, get_all_tor_nodes)
+  leaks = FirewallLeakCheck.new(@sniffer.pcap_file,
+                                :accepted_hosts => get_all_tor_nodes)
   leaks.assert_no_leaks
 end
 
@@ -894,3 +895,96 @@ end
 When /^I accept to import the key with Seahorse$/ do
   @screen.wait_and_click("TorBrowserOkButton.png", 10)
 end
+
+Given /^a web server is running on the LAN$/ do
+  web_server_ip_addr = $vmnet.bridge_ip_addr
+  web_server_port = 8000
+  @web_server_url = "http://#{web_server_ip_addr}:#{web_server_port}"
+  web_server_hello_msg = "Welcome to the LAN web server!"
+
+  # I've tested ruby Thread:s, fork(), etc. but nothing works due to
+  # various strange limitations in the ruby interpreter. For instance,
+  # apparently concurrent IO has serious limits in the thread
+  # scheduler (e.g. sikuli's wait() would block WEBrick from reading
+  # from its socket), and fork():ing results in a lot of complex
+  # cucumber stuff (like our hooks!) ending up in the child process,
+  # breaking stuff in the parent process. After asking some supposed
+  # ruby pros, I've settled on the following.
+  code = <<-EOF
+  require "webrick"
+  STDOUT.reopen("/dev/null", "w")
+  STDERR.reopen("/dev/null", "w")
+  server = WEBrick::HTTPServer.new(:BindAddress => "#{web_server_ip_addr}",
+                                   :Port => #{web_server_port},
+                                   :DocumentRoot => "/dev/null")
+  server.mount_proc("/") do |req, res|
+    res.body = "#{web_server_hello_msg}"
+  end
+  server.start
+EOF
+  proc = IO.popen(['ruby', '-e', code])
+  try_for(10, :msg => "It seems the LAN web server failed to start") do
+    Process.kill(0, proc.pid) == 1
+  end
+
+  add_after_scenario_hook { Process.kill("TERM", proc.pid) }
+
+  # It seems necessary to actually check that the LAN server is
+  # serving, possibly because it isn't doing so reliably when setting
+  # up. If e.g. the Unsafe Browser (which *should* be able to access
+  # the web server) tries to access it too early, Firefox seems to
+  # take some random amount of time to retry fetching. Curl gives a
+  # more consistent result, so let's rely on that instead. Note that
+  # this forces us to capture traffic *after* this step in case
+  # accessing this server matters, like when testing the Tor Browser..
+  try_for(30, :msg => "Something is wrong with the LAN web server") do
+    msg = $vm.execute_successfully("curl #{@web_server_url}",
+                                   LIVE_USER).stdout.chomp
+    web_server_hello_msg == msg
+  end
+end
+
+When /^I open a page on the LAN web server in the (.*)$/ do |browser|
+  step "I open the address \"#{@web_server_url}\" in the #{browser}"
+end
+
+def force_new_tor_circuit(with_vidalia=nil)
+  assert(!@new_circuit_tries.nil? && @new_circuit_tries >= 0,
+         '@new_circuit_tries was not initialized before it was used')
+  @new_circuit_tries += 1
+  STDERR.puts "Forcing new Tor circuit... (attempt ##{@new_circuit_tries})" if $config["DEBUG"]
+  if with_vidalia
+    assert_equal('gnome', @theme, "Vidalia is not available in the #{@theme} theme.")
+    begin
+      step 'process "vidalia" is running'
+    rescue Test::Unit::AssertionFailedError
+      STDERR.puts "Vidalia was not running. Attempting to start Vidalia..." if $config["DEBUG"]
+      $vm.spawn('restart-vidalia')
+      step 'process "vidalia" is running within 15 seconds'
+    end
+    # Sometimes Sikuli gets confused and recognizes the yellow-colored vidalia systray
+    # icon as the green one. This has been seen when Vidalia needed to be
+    # restarted in the above 'begin' block.
+    #
+    # try_for is used here for that reason, otherwise this step may fail
+    # because sikuli presumaturely right-clicked the Vidalia icon and the 'New
+    # Identity' option isn't clickable yet..
+    try_for(3 * 60) do
+      # Let's be *sure* that vidalia is still running. I'd hate to spend up to
+      # three minutes waiting for an icon that isn't there because Vidalia, for
+      # whatever reason, is no longer running...
+      step 'process "vidalia" is running'
+      @screen.wait_and_right_click('VidaliaSystrayReady.png', 10)
+      @screen.wait_and_click('VidaliaMenuNewIdentity.png', 10)
+    end
+    @screen.wait('VidaliaNewIdentityNotification.png', 20)
+    @screen.waitVanish('VidaliaNewIdentityNotification.png', 60)
+  else
+    $vm.execute_successfully('. /usr/local/lib/tails-shell-library/tor.sh; tor_control_send "signal NEWNYM"')
+  end
+end
+
+Then /^I force Tor to use a new circuit( in Vidalia)?$/ do |with_vidalia|
+  @new_circuit_tries = 1 if @new_circuit_tries.nil?
+  force_new_tor_circuit(with_vidalia)
+end