Merge remote-tracking branch 'origin/stable' into bugfix/10182-improve-i2p-notification

config/chroot_local-hooks/16-i2p_config

@@ -5,11 +5,29 @@ set -e
 echo "Configuring I2P"
 
 I2P="/usr/share/i2p"
+I2PROUTER="/usr/bin/i2prouter"
 WRAPPER="/etc/i2p/wrapper.config"
 
 # This must be set in order for the i2p init script to work
 sed -i 's/^RUN_DAEMON=.*$/RUN_DAEMON="true"/' /etc/default/i2p
 
+# Remove the "i2prouter" script, its man page, and its apparmor profile
+# since these are not used by Tails:
+rm /etc/apparmor.d/usr.bin.i2prouter /usr/share/man/man1/i2prouter.1.gz
+
+# Install custom i2prouter stub scripts
+for script in ${I2PROUTER} ${I2PROUTER}-nowrapper; do
+    echo "Removing $script"
+    dpkg-divert --rename --add "${script}"
+    cat > "$script" << EOF
+#!/bin/sh
+echo "This script is not used by Tails."
+echo "See https://tails.boum.org/doc/anonymous_internet/i2p/ for more information."
+exit 0
+EOF
+    chmod 755 "$script"
+done
+
 # Remove the outproxy from the tunnel on port 4444
 # This will remove the following lines:
 #      tunnel.0.proxyList=false.i2p
@@ -48,3 +66,7 @@ EOF
 cat > "$I2P/susimail.config" << EOF
 susimail.pop3.leave.on.server=true
 EOF
+
+# enforce apparmor
+echo Setting the I2P apparmor profile to enforce mode
+sed  -i -re 's|flags=\(complain\)||' /etc/apparmor.d/system_i2p

config/chroot_local-includes/usr/local/bin/tor-browser

@@ -41,24 +41,6 @@ tor_has_bootstrapped() {
     sudo -n -u debian-tor /usr/local/sbin/tor-has-bootstrapped
 }
 
-# Workaround bug #8036 by copying any localized search plugins into
-# the profile.
-enable_localized_searchplugins() {
-    local locale plugin
-    locale=$(cat "${PROFILE}"/preferences/0000locale.js | \
-                 sed 's@^pref("general\.useragent\.locale", "\([^"]*\)");$@\1@')
-    if [ "${locale}" = en-US ] || [ -e "${PROFILE}"/searchplugins ]; then
-        return
-    fi
-    # Fallback to a similar locale if there is no exact match
-    plugin="$(ls -1 "${TBB_INSTALL}"/distribution/searchplugins/locale/ | grep -m1 "^${locale}\(-[A-Z]\+\)\?$" || true)"
-    if [ -n "${plugin}" ]; then
-        mkdir -p "${PROFILE}"/searchplugins
-        # The plugins do not load if they are symlinks
-        cp --dereference "${TBB_INSTALL}"/distribution/searchplugins/locale/"${plugin}"/*  "${PROFILE}"/searchplugins
-    fi
-}
-
 start_browser() {
     if [ ! -d "${PROFILE}" ]; then
         /usr/local/bin/generate-tor-browser-profile
@@ -68,11 +50,11 @@ start_browser() {
     mkdir --mode=0700 -p "$TMPDIR"
     export TMPDIR
 
+    # We need to set general.useragent.locale properly to get
+    # localized search plugins (and perhaps other things too). It is
+    # not enough to simply set intl.locale.matchOS to true.
     configure_best_tor_browser_locale "${PROFILE}"
 
-    # Workaround bug #8036
-    enable_localized_searchplugins
-
     if [ -z "$XAUTHORITY" ]; then
         XAUTHORITY=~/.Xauthority
         export XAUTHORITY

config/chroot_local-includes/usr/local/sbin/tails-restricted-network-detector

@@ -61,9 +61,15 @@ while(my $sl = $parser->next) {
         # The beginning of *all* (not only wireless) new
         # connections. We drop any previous state so it won't
         # interfere.
-        $state{$1} = undef;
-    } elsif ($text =~ /\(([^)]+)\): supplicant connection state:.*-> (.*)$/) {
-        # Wireless connection state transition.
+        $state{$1} = "";
+    } elsif ($text =~ /\(([^)]+)\): supplicant (?:connection|interface) state: \S+ -> (\S+)/ ||
+             $text =~ /\(([^)]+)\): device state change: \S+ -> (\S+)/) {
+        # NetworkManager logs state transitions with the above
+        # messages, but the really important part is that we
+        # accurately log the state changes *to* and *from*
+        # "associating" (for the next case). Hence the safest bet
+        # seems to be to deal with all observed types of transitions
+        # that NetworkManager logs.
         $state{$1} = $2;
     } elsif ($text =~ /Activation \(([^)]+)\/[^)]*\): association took too long/) {
         # Wireless connection failure. If it happens during

config/chroot_local-includes/usr/share/tails/i2p-browser/prefs.js

@@ -22,3 +22,6 @@ pref("browser.newtabpage.directory.source", "");
 pref("browser.newtabpage.directory.ping", "");
 // ... and disable the explanation shown the first time
 pref("browser.newtabpage.introShown", true);
+
+// Never  add 'www' or '.com' to hostnames in I2P Browser.
+pref("browser.fixup.alternate.enabled", false);

features/config/defaults.yml

+CAPTURE: false
+CAPTURE_ALL: false
 PAUSE_ON_FAIL: false
 SIKULI_RETRY_FINDFAILED: false
 MAX_NEW_TOR_CIRCUIT_RETRIES: 5

features/step_definitions/common_steps.rb

@@ -110,7 +110,7 @@ end
 
 Then /^drive "([^"]+)" is detected by Tails$/ do |name|
   next if @skip_steps_while_restoring_background
-  raise "Tails is not running" if @vm.is_running?
+  raise "Tails is not running" unless @vm.is_running?
   try_for(10, :msg => "Drive '#{name}' is not detected by Tails") do
     @vm.disk_detected?(name)
   end

features/step_definitions/pidgin.rb

@@ -385,6 +385,7 @@ def pidgin_add_certificate_from (cert_file)
   # Here, we need a certificate that is not already in the NSS database
   step "I copy \"/usr/share/ca-certificates/spi-inc.org/spi-cacert-2008.crt\" to \"#{cert_file}\" as user \"amnesia\""
 
+  @vm.focus_window('Buddy List')
   @screen.wait_and_click('PidginToolsMenu.png', 10)
   @screen.wait_and_click('PidginCertificatesMenuItem.png', 10)
   @screen.wait('PidginCertificateManagerDialog.png', 10)

features/support/helpers/sikuli_helper.rb

@@ -43,9 +43,11 @@ end
 
 # Bind Java's stdout to debug_log() via our magical pseudo fifo
 # logger.
-file_output_stream = Java::Io::FileOutputStream.new(DEBUG_LOG_PSEUDO_FIFO)
-print_stream = Java::Io::PrintStream.new(file_output_stream)
-Java::Lang::System.setOut(print_stream)
+def bind_java_to_pseudo_fifo_logger
+  file_output_stream = Java::Io::FileOutputStream.new(DEBUG_LOG_PSEUDO_FIFO)
+  print_stream = Java::Io::PrintStream.new(file_output_stream)
+  Java::Lang::System.setOut(print_stream)
+end
 
 def findfailed_hook(pic)
   STDERR.puts ""
@@ -72,6 +74,12 @@ end
 sikuli_script_proxy = Sikuli::Screen
 $_original_sikuli_screen_new ||= Sikuli::Screen.method :new
 
+# For waitAny()/findAny() we are forced to throw this exception since
+# Rjb::throw doesn't block until the Java exception has been received
+# by Ruby, so strange things can happen.
+class FindAnyFailed < StandardError
+end
+
 def sikuli_script_proxy.new(*args)
   s = $_original_sikuli_screen_new.call(*args)
 
@@ -143,6 +151,14 @@ def sikuli_script_proxy.new(*args)
     self.hover(self.wait(pic, time))
   end
 
+  def s.existsAny(images)
+    images.each do |image|
+      region = self.exists(image)
+      return [image, region] if region
+    end
+    return nil
+  end
+
   def s.findAny(images)
     images.each do |image|
       begin
@@ -153,23 +169,20 @@ def sikuli_script_proxy.new(*args)
       end
     end
     # If we've reached this point, none of the images could be found.
-    Rjb::throw('org.sikuli.script.FindFailed',
-               "can not find any of the images #{images} on the screen")
+    raise FindAnyFailed.new("can not find any of the images #{images} on the " +
+                            "screen")
   end
 
   def s.waitAny(images, time)
     Timeout::timeout(time) do
       loop do
-        begin
-          return self.findAny(images)
-        rescue FindFailed
-          # Ignore. We want to retry until we timeout.
-        end
+        result = self.existsAny(images)
+        return result if result
       end
     end
   rescue Timeout::Error
-    Rjb::throw('org.sikuli.script.FindFailed',
-               "can not find any of the images #{images} on the screen")
+    raise FindAnyFailed.new("can not find any of the images #{images} on the " +
+                            "screen")
   end
 
   def s.hover_point(x, y)

features/support/helpers/vm_helper.rb

@@ -175,7 +175,7 @@ class VM
 
   def set_cdrom_boot(image)
     if is_running?
-      raise "boot settings can only be set for inactice vms"
+      raise "boot settings can only be set for inactive vms"
     end
     set_boot_device('cdrom')
     set_cdrom_image(image)
@@ -269,7 +269,7 @@ class VM
   # XXX-9p in common_steps.rb for more information.
   def add_share(source, tag)
     if is_running?
-      raise "shares can only be added to inactice vms"
+      raise "shares can only be added to inactive vms"
     end
     # The complete source directory must be group readable by the user
     # running the virtual machine, and world readable so the user inside
@@ -294,7 +294,7 @@ class VM
   end
 
   def set_ram_size(size, unit = "KiB")
-    raise "System memory can only be added to inactice vms" if is_running?
+    raise "System memory can only be added to inactive vms" if is_running?
     domain_xml = REXML::Document.new(@domain.xml_desc)
     domain_xml.elements['domain/memory'].text = size
     domain_xml.elements['domain/memory'].attributes['unit'] = unit
@@ -311,21 +311,21 @@ class VM
   end
 
   def set_arch(arch)
-    raise "System architecture can only be set to inactice vms" if is_running?
+    raise "System architecture can only be set to inactive vms" if is_running?
     domain_xml = REXML::Document.new(@domain.xml_desc)
     domain_xml.elements['domain/os/type'].attributes['arch'] = arch
     update(domain_xml.to_s)
   end
 
   def add_hypervisor_feature(feature)
-    raise "Hypervisor features can only be added to inactice vms" if is_running?
+    raise "Hypervisor features can only be added to inactive vms" if is_running?
     domain_xml = REXML::Document.new(@domain.xml_desc)
     domain_xml.elements['domain/features'].add_element(feature)
     update(domain_xml.to_s)
   end
 
   def drop_hypervisor_feature(feature)
-    raise "Hypervisor features can only be fropped from inactice vms" if is_running?
+    raise "Hypervisor features can only be fropped from inactive vms" if is_running?
     domain_xml = REXML::Document.new(@domain.xml_desc)
     domain_xml.elements['domain/features'].delete_element(feature)
     update(domain_xml.to_s)
@@ -348,7 +348,7 @@ EOF
 
   def set_os_loader(type)
     if is_running?
-      raise "boot settings can only be set for inactice vms"
+      raise "boot settings can only be set for inactive vms"
     end
     if type == 'UEFI'
       domain_xml = REXML::Document.new(@domain.xml_desc)

features/support/hooks.rb

@@ -5,6 +5,23 @@ require 'tmpdir'
 
 # Run once, before any feature
 AfterConfiguration do |config|
+  if File.exist?($config["TMPDIR"])
+    if !File.directory?($config["TMPDIR"])
+      raise "Temporary directory '#{$config["TMPDIR"]}' exists but is not a " +
+            "directory"
+    end
+    if !File.owned?($config["TMPDIR"])
+      raise "Temporary directory '#{$config["TMPDIR"]}' must be owned by the " +
+            "current user"
+    end
+    FileUtils.chmod(0755, $config["TMPDIR"])
+  else
+    begin
+      FileUtils.mkdir_p($config["TMPDIR"])
+    rescue Errno::EACCES => e
+      raise "Cannot create temporary directory: #{e.to_s}"
+    end
+  end
   # Start a thread that monitors a pseudo fifo file and debug_log():s
   # anything written to it "immediately" (well, as fast as inotify
   # detects it). We're forced to a convoluted solution like this
@@ -25,6 +42,8 @@ AfterConfiguration do |config|
       watcher.run
     end
   end
+  # Fix Sikuli's debug_log():ing.
+  bind_java_to_pseudo_fifo_logger
 end
 
 # For @product tests
@@ -50,23 +69,6 @@ def add_after_scenario_hook(&block)
 end
 
 BeforeFeature('@product') do |feature|
-  if File.exist?($config["TMPDIR"])
-    if !File.directory?($config["TMPDIR"])
-      raise "Temporary directory '#{$config["TMPDIR"]}' exists but is not a " +
-            "directory"
-    end
-    if !File.owned?($config["TMPDIR"])
-      raise "Temporary directory '#{$config["TMPDIR"]}' must be owned by the " +
-            "current user"
-    end
-    FileUtils.chmod(0755, $config["TMPDIR"])
-  else
-    begin
-      Dir.mkdir($config["TMPDIR"])
-    rescue Errno::EACCES => e
-      raise "Cannot create temporary directory: #{e.to_s}"
-    end
-  end
   delete_all_snapshots if !KEEP_SNAPSHOTS
   if TAILS_ISO.nil?
     raise "No Tails ISO image specified, and none could be found in the " +
@@ -109,8 +111,27 @@ AfterFeature('@product') do
 end
 
 # BeforeScenario
-Before('@product') do
+Before('@product') do |scenario|
   @screen = Sikuli::Screen.new
+  if $config["CAPTURE"]
+    video_name = "capture-" + "#{scenario.name}-#{TIME_AT_START}.mkv"
+    # Sanitize the filename from unix-hostile filename characters
+    bad_filename_chars = Regexp.new("[^A-Za-z0-9_\\-.,+:]")
+    video_name.gsub!(bad_filename_chars, '_')
+    @video_path = "#{$config['TMPDIR']}/#{video_name}"
+    capture = IO.popen(['avconv',
+                        '-f', 'x11grab',
+                        '-s', '1024x768',
+                        '-r', '15',
+                        '-i', "#{$config['DISPLAY']}.0",
+                        '-an',
+                        '-c:v', 'libx264',
+                        '-y',
+                        @video_path,
+                        :err => ['/dev/null', 'w'],
+                       ])
+    @video_capture_pid = capture.pid
+  end
   if File.size?($background_snapshot)
     @skip_steps_while_restoring_background = true
   else
@@ -124,6 +145,14 @@ end
 
 # AfterScenario
 After('@product') do |scenario|
+  if @video_capture_pid
+    # We can be incredibly fast at detecting errors sometimes, so the
+    # screen barely "settles" when we end up here and kill the video
+    # capture. Let's wait a few seconds more to make it easier to see
+    # what the error was.
+    sleep 3 if scenario.failed?
+    Process.kill("INT", @video_capture_pid)
+  end
   if scenario.failed?
     time_of_fail = Time.now - TIME_AT_START
     secs = "%02d" % (time_of_fail % 60)
@@ -140,6 +169,10 @@ After('@product') do |scenario|
       STDERR.puts "Press ENTER to continue running the test suite"
       STDIN.gets
     end
+  else
+    if @video_path && File.exist?(@video_path) && not($config['CAPTURE_ALL'])
+      FileUtils.rm(@video_path)
+    end
   end
   @vm.destroy_and_undefine if @vm
 end

run_test_suite

@@ -51,8 +51,11 @@ Sets up an appropriate environment and invokes cucumber. Note that this script
 must be run from the Tails source directory root.
 
 Options for '@product' features:
-  --capture FILE     Captures the test session into FILE using VP8 encoding.
-                     Requires libvpx1.
+  --capture          Captures failed scenarios into videos stored in the
+                     temporary directory (see --tmpdir below) using x264
+                     encoding. Requires x264.
+  --capture-all      Keep videos for all scenarios, including those that
+                     succeed (implies --capture).
   --pause-on-fail    On failure, pause test suite until pressing Enter. This is
                      useful for investigating the state of the VM guest to see
                      exactly why a test failed.
@@ -79,12 +82,12 @@ Options for '@product' features:
 Note that '@source' features has no relevant options.
 
 CUCUMBER_ARGS can be used to specify which features to be run, but also any
-cucumber option, although then you must pass `--` first to let this wrapper
+cucumber option, although then you must pass \`--\` first to let this wrapper
 script know that we're done with *its* options. For debugging purposes, a
 'debug' formatter has been added so pretty debugging can be enabled with
-`--format debug`. You could even combine the default (pretty) formatter with
-pretty debugging printed to a file with `--format pretty --format debug
---out debug.log`.
+\`--format debug\`. You could even combine the default (pretty) formatter with
+pretty debugging printed to a file with \`--format pretty --format debug
+--out debug.log\`.
 "
 }
 
@@ -168,18 +171,18 @@ capture_session() {
 # Unset all environment variables used by this script to pass options
 # to cucumber, except TMPDIR since we explicitly want to support
 # setting it that way.
-CAPTURE_FILE=
+CAPTURE=
+CAPTURE_ALL=
 LOG_FILE=
 VNC_VIEWER=
 VNC_SERVER=
-DEBUG=
 PAUSE_ON_FAIL=
 KEEP_SNAPSHOTS=
 SIKULI_RETRY_FINDFAILED=
 TAILS_ISO=
 OLD_TAILS_ISO=
 
-LONGOPTS="view,vnc-server-only,capture:,help,tmpdir:,keep-snapshots,retry-find,iso:,old-iso:,pause-on-fail"
+LONGOPTS="view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-snapshots,retry-find,iso:,old-iso:,pause-on-fail"
 OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
 eval set -- "$OPTS"
 while [ $# -gt 0 ]; do
@@ -193,8 +196,13 @@ while [ $# -gt 0 ]; do
             VNC_SERVER=yes
             ;;
         --capture)
-            shift
-            CAPTURE_FILE="$1"
+            check_dependencies x264
+            export CAPTURE="yes"
+            ;;
+        --capture-all)
+            check_dependencies x264
+            export CAPTURE="yes"
+            export CAPTURE_ALL="yes"
             ;;
         --pause-on-fail)
             export PAUSE_ON_FAIL="yes"
@@ -237,9 +245,6 @@ TARGET_DISPLAY=$(next_free_display)
 
 start_xvfb
 
-if [ -n "${CAPTURE_FILE}" ]; then
-    capture_session
-fi
 if [ -n "${VNC_SERVER}" ]; then
     start_vnc_server
 fi

wiki/src/contribute/release_process/test/usage.mdwn

@@ -32,6 +32,17 @@ feature, we'd simply run something like:
 
     ./run_test_suite features/build.feature
 
+Actually, `run_test_suite` is just a wrapper around `cucumber`, so any
+`cucumber` option can be passed too, although after an `--` so they
+are not confused with the wrapper's options. For instance:
+
+    ./run_test_suite ... -- --format debug features/apt.feature
+
+will enable the `debug` formatter, which in Tails' Cucumber setup will
+enable debugging information to be printed (which is *very* useful
+when debugging or developing for the test suite) unlike in vanilla
+Cucumber, where it's used for debugging the formatting subsystem.
+
 For full instructions, see its `--help`.
 
 Note: since the test suite itself uses `virt-viewer` to interact with
@@ -72,8 +83,12 @@ can be use by concurrent test suite runs):
 Here's a list of all non-secret key-value pairs that can be supported
 by the local configuration file:
 
-* `DEBUG`: Boolean value. If set to `true`, various debugging info
-  will be printed. Defaults to `false`.
+* `CAPTURE`: Captures failed scenarios into videos stored in the
+  temporary directory (see `TMPDIR` below) using x264
+  encoding. Defaults to `false`.
+
+* `CAPTURE_ALL`: Keep videos for all scenarios, including those that
+  succeed (implies `CAPTURE`). Defaults to `false`.
 
 * `MAX_NEW_TOR_CIRCUIT_RETRIES`: Integer. Upon failure, some test steps may be
   run again after requesting that connections are made using new Tor circuits. This
@@ -90,7 +105,7 @@ by the local configuration file:
   after pressing ENTER. This is useful for updating outdated images,
   or when extracting new images. Defaults to `false`.
 
-* `TEMP_DIR`: String value. Directory where various temporary files
+* `TMPDIR`: String value. Directory where various temporary files
   are written during a test, e.g. VM snapshots and memory dumps,
   failure screenshots, pcap files and disk images. Defaults to
   `"/tmp/TailsToaster"`.