Merge branch 'stable' into test/10116-make-webm-video-test-more-robust

Conflicts:
	features/step_definitions/common_steps.rb

config/chroot_local-hooks/16-i2p_config

@@ -5,11 +5,29 @@ set -e
 echo "Configuring I2P"
 
 I2P="/usr/share/i2p"
+I2PROUTER="/usr/bin/i2prouter"
 WRAPPER="/etc/i2p/wrapper.config"
 
 # This must be set in order for the i2p init script to work
 sed -i 's/^RUN_DAEMON=.*$/RUN_DAEMON="true"/' /etc/default/i2p
 
+# Remove the "i2prouter" script, its man page, and its apparmor profile
+# since these are not used by Tails:
+rm /etc/apparmor.d/usr.bin.i2prouter /usr/share/man/man1/i2prouter.1.gz
+
+# Install custom i2prouter stub scripts
+for script in ${I2PROUTER} ${I2PROUTER}-nowrapper; do
+    echo "Removing $script"
+    dpkg-divert --rename --add "${script}"
+    cat > "$script" << EOF
+#!/bin/sh
+echo "This script is not used by Tails."
+echo "See https://tails.boum.org/doc/anonymous_internet/i2p/ for more information."
+exit 0
+EOF
+    chmod 755 "$script"
+done
+
 # Remove the outproxy from the tunnel on port 4444
 # This will remove the following lines:
 #      tunnel.0.proxyList=false.i2p
@@ -48,3 +66,7 @@ EOF
 cat > "$I2P/susimail.config" << EOF
 susimail.pop3.leave.on.server=true
 EOF
+
+# enforce apparmor
+echo Setting the I2P apparmor profile to enforce mode
+sed  -i -re 's|flags=\(complain\)||' /etc/apparmor.d/system_i2p

config/chroot_local-hooks/19-install-tor-browser-AppArmor-profile

@@ -33,7 +33,7 @@ install_torbrowser_AppArmor_profile() {
    tmpdir="$(mktemp -d)"
    (
       cd "$tmpdir"
-      apt-get source torbrowser-launcher/testing
+      apt-get source torbrowser-launcher/sid
       install -m 0644 \
               torbrowser-launcher-*/apparmor/torbrowser.Browser.firefox \
               "$PROFILE"

config/chroot_local-includes/usr/local/bin/tor-browser

@@ -41,24 +41,6 @@ tor_has_bootstrapped() {
     sudo -n -u debian-tor /usr/local/sbin/tor-has-bootstrapped
 }
 
-# Workaround bug #8036 by copying any localized search plugins into
-# the profile.
-enable_localized_searchplugins() {
-    local locale plugin
-    locale=$(cat "${PROFILE}"/preferences/0000locale.js | \
-                 sed 's@^pref("general\.useragent\.locale", "\([^"]*\)");$@\1@')
-    if [ "${locale}" = en-US ] || [ -e "${PROFILE}"/searchplugins ]; then
-        return
-    fi
-    # Fallback to a similar locale if there is no exact match
-    plugin="$(ls -1 "${TBB_INSTALL}"/distribution/searchplugins/locale/ | grep -m1 "^${locale}\(-[A-Z]\+\)\?$" || true)"
-    if [ -n "${plugin}" ]; then
-        mkdir -p "${PROFILE}"/searchplugins
-        # The plugins do not load if they are symlinks
-        cp --dereference "${TBB_INSTALL}"/distribution/searchplugins/locale/"${plugin}"/*  "${PROFILE}"/searchplugins
-    fi
-}
-
 start_browser() {
     if [ ! -d "${PROFILE}" ]; then
         /usr/local/bin/generate-tor-browser-profile
@@ -68,11 +50,11 @@ start_browser() {
     mkdir --mode=0700 -p "$TMPDIR"
     export TMPDIR
 
+    # We need to set general.useragent.locale properly to get
+    # localized search plugins (and perhaps other things too). It is
+    # not enough to simply set intl.locale.matchOS to true.
     configure_best_tor_browser_locale "${PROFILE}"
 
-    # Workaround bug #8036
-    enable_localized_searchplugins
-
     if [ -z "$XAUTHORITY" ]; then
         XAUTHORITY=~/.Xauthority
         export XAUTHORITY

config/chroot_local-includes/usr/local/sbin/tails-restricted-network-detector

@@ -61,9 +61,15 @@ while(my $sl = $parser->next) {
         # The beginning of *all* (not only wireless) new
         # connections. We drop any previous state so it won't
         # interfere.
-        $state{$1} = undef;
-    } elsif ($text =~ /\(([^)]+)\): supplicant connection state:.*-> (.*)$/) {
-        # Wireless connection state transition.
+        $state{$1} = "";
+    } elsif ($text =~ /\(([^)]+)\): supplicant (?:connection|interface) state: \S+ -> (\S+)/ ||
+             $text =~ /\(([^)]+)\): device state change: \S+ -> (\S+)/) {
+        # NetworkManager logs state transitions with the above
+        # messages, but the really important part is that we
+        # accurately log the state changes *to* and *from*
+        # "associating" (for the next case). Hence the safest bet
+        # seems to be to deal with all observed types of transitions
+        # that NetworkManager logs.
         $state{$1} = $2;
     } elsif ($text =~ /Activation \(([^)]+)\/[^)]*\): association took too long/) {
         # Wireless connection failure. If it happens during

config/chroot_local-includes/usr/local/sbin/tails-spoof-mac

@@ -14,6 +14,14 @@ set -e
 TEXTDOMAIN="tails"
 export TEXTDOMAIN
 
+disable_networking() {
+    service network-manager stop || :
+    for f in /etc/init.d/network-manager /usr/sbin/NetworkManager; do
+        [ -e "${f}" ] && mv "${f}" "${f}.disabled"
+    done
+    log "Networking disabled"
+}
+
 show_notification() {
     # We must wait until all the facilities necessary for showing the
     # notification to the Live user is available to prevent it from
@@ -63,8 +71,8 @@ mac_spoof_panic() {
     echo "blacklist ${module}" >> /etc/modprobe.d/"${module}"-blacklist.conf
     unload_module_and_rev_deps "${module}" || :
     if nic_exists "${nic}"; then
-        log "Failed to unload module ${module} of NIC ${nic}. Stopping NetworkManager."
-        service network-manager stop
+        log "Failed to unload module ${module} of NIC ${nic}"
+        disable_networking
         notify_panic_failure "${nic}" "${nic_name}" &
     else
         log "Successfully unloaded module ${module} of NIC ${nic}."
@@ -130,8 +138,8 @@ then
         # If mac_spoof_panic() fails we're quite screwed, so we kill
         # NetworkManager without notification to do our best to
         # prevent a MAC address leak.
-        log "Panic mode failed for NIC ${NIC}. Killing NetworkManager."
-        service network-manager stop
+        log "Panic mode failed for NIC ${NIC}"
+        disable_networking
     fi
     exit 1
 fi

config/chroot_local-includes/usr/local/sbin/tails-unblock-network

@@ -19,4 +19,10 @@ rm -f "${BLACKLIST}"
 # if NM wins, the udev trigger's run of tails-spoof-mac will fail.
 /sbin/udevadm settle
 
-service network-manager start
+# If tails-spoof-mac goes into panic mode but fails to disable the
+# problematic device, networking will be disabled by having these
+# removed.
+if [ -e "/etc/init.d/network-manager" ] && \
+   [ -e "/usr/sbin/NetworkManager" ]; then
+    service network-manager start
+fi

config/chroot_local-includes/usr/share/tails/i2p-browser/prefs.js

@@ -22,3 +22,6 @@ pref("browser.newtabpage.directory.source", "");
 pref("browser.newtabpage.directory.ping", "");
 // ... and disable the explanation shown the first time
 pref("browser.newtabpage.introShown", true);
+
+// Never  add 'www' or '.com' to hostnames in I2P Browser.
+pref("browser.fixup.alternate.enabled", false);

features/config/defaults.yml

-DEBUG: false
+CAPTURE: false
+CAPTURE_ALL: false
 PAUSE_ON_FAIL: false
 SIKULI_RETRY_FINDFAILED: false
 MAX_NEW_TOR_CIRCUIT_RETRIES: 5

features/scripts/otr-bot.py

@@ -3,6 +3,7 @@ import sys
 import jabberbot
 import xmpp
 import potr
+import logging
 from argparse import ArgumentParser
 
 class OtrContext(potr.context.Context):
@@ -51,9 +52,11 @@ class OtrBot(jabberbot.JabberBot):
 
     PING_FREQUENCY = 60
 
-    def __init__(self, account, password, otr_key_path, connect_server = None):
+    def __init__(self, account, password, otr_key_path,
+                 connect_server = None, log_file = None):
         self.__connect_server = connect_server
         self.__password = password
+        self.__log_file = log_file
         super(OtrBot, self).__init__(account, password)
         self.__otr_manager = OtrContextManager(account, otr_key_path)
         self.send_raw_message_fn = super(OtrBot, self).send_message
@@ -71,6 +74,8 @@ class OtrBot(jabberbot.JabberBot):
     # completely (copy-paste mostly) in order to add support for using
     # an XMPP "Connect Server".
     def connect(self):
+        logging.basicConfig(filename = self.__log_file,
+                            level = logging.DEBUG)
         if not self.conn:
             conn = xmpp.Client(self.jid.getDomain(), debug=[])
             if self.__connect_server:
@@ -185,10 +190,14 @@ if __name__ == '__main__':
                         "(port defaults to 5222)")
     parser.add_argument("-j", "--auto-join", nargs = '+', metavar = 'ROOMS',
                         help = "auto-join multi-user chatrooms on start")
+    parser.add_argument("-l", "--log-file", metavar = 'LOGFILE',
+                        help = "Log to file instead of stderr")
     args = parser.parse_args()
     otr_bot_opt_args = dict()
     if args.connect_server:
         otr_bot_opt_args["connect_server"] = args.connect_server
+    if args.log_file:
+        otr_bot_opt_args["log_file"] = args.log_file
     otr_bot = OtrBot(args.account, args.password, args.otr_key_path,
                      **otr_bot_opt_args)
     if args.auto_join:

features/scripts/vm-execute

@@ -7,7 +7,8 @@ rescue LoadError => e
   raise "This script must be run from within Tails' Git directory."
 end
 $config = Hash.new
-$config["DEBUG"] = false
+
+def debug_log(*args) ; end
 
 class FakeVM
   def get_remote_shell_port

features/step_definitions/common_steps.rb

@@ -110,13 +110,9 @@ end
 
 Then /^drive "([^"]+)" is detected by Tails$/ do |name|
   next if @skip_steps_while_restoring_background
-  if @vm.is_running?
-    try_for(10, :msg => "Drive '#{name}' is not detected by Tails") {
-      @vm.disk_detected?(name)
-    }
-  else
-    STDERR.puts "Cannot tell if drive '#{name}' is detected by Tails: " +
-                "Tails is not running"
+  raise "Tails is not running" unless @vm.is_running?
+  try_for(10, :msg => "Drive '#{name}' is not detected by Tails") do
+    @vm.disk_detected?(name)
   end
 end
 
@@ -1078,12 +1074,13 @@ When /^I open a page on the LAN web server in the (.*)$/ do |browser|
 end
 
 def force_new_tor_circuit(with_vidalia=nil)
+  debug_log("Forcing new Tor circuit...")
   if with_vidalia
     assert_equal('gnome', @theme, "Vidalia is not available in the #{@theme} theme.")
     begin
       step 'process "vidalia" is running'
     rescue Test::Unit::AssertionFailedError
-      STDERR.puts "Vidalia was not running. Attempting to start Vidalia..." if $config["DEBUG"]
+      debug_log("Vidalia was not running. Attempting to start Vidalia...")
       @vm.spawn('restart-vidalia')
       step 'process "vidalia" is running within 15 seconds'
     end

features/step_definitions/erase_memory.rb

@@ -140,18 +140,19 @@ Given /^I fill the guest's memory with a known pattern(| without verifying)$/ do
   @vm.pidof("fillram").each do |pid|
     @vm.execute_successfully("echo 15 > /proc/#{pid}/oom_adj")
   end
-  STDERR.print "Memory fill progress: "
-  ram_usage = ""
-  remove_chars = 0
+  prev_used_ram_ratio = -1
   # ... and that it finishes
   try_for(instances*2*60, { :msg => "fillram didn't complete, probably the VM crashed" }) do
-    used_ram = used_ram_in_MiB
-    remove_chars = ram_usage.size
-    ram_usage = "%3d%% " % ((used_ram.to_f/@detected_ram_m)*100)
-    STDERR.print "\b"*remove_chars + ram_usage
+    used_ram_ratio = (used_ram_in_MiB.to_f/@detected_ram_m)*100
+    # Round down to closest multiple of 10 to limit the logging a bit.
+    used_ram_ratio = (used_ram_ratio/10).round*10
+    if used_ram_ratio - prev_used_ram_ratio >= 10
+      debug_log("Memory fill progress: %3d%%" % used_ram_ratio)
+      prev_used_ram_ratio = used_ram_ratio
+    end
     ! @vm.has_process?("fillram")
   end
-  STDERR.print "\b"*remove_chars + "finished.\n"
+  debug_log("Memory fill progress: finished")
   if verify
     coverage = pattern_coverage_in_guest_ram()
     # Let's aim for having the pattern cover at least 80% of the free RAM.

features/step_definitions/pidgin.rb

@@ -378,6 +378,7 @@ def pidgin_add_certificate_from (cert_file)
   # Here, we need a certificate that is not already in the NSS database
   step "I copy \"/usr/share/ca-certificates/spi-inc.org/spi-cacert-2008.crt\" to \"#{cert_file}\" as user \"amnesia\""
 
+  @vm.focus_window('Buddy List')
   @screen.wait_and_click('PidginToolsMenu.png', 10)
   @screen.wait_and_click('PidginCertificatesMenuItem.png', 10)
   @screen.wait('PidginCertificateManagerDialog.png', 10)

features/support/config.rb

@@ -36,12 +36,13 @@ ENV['TMPDIR'] = $config['TMPDIR']
 # Dynamic constants initialized through the environment or similar,
 # e.g. options we do not want to be configurable through the YAML
 # configuration files.
+DEBUG_LOG_PSEUDO_FIFO = "#{$config["TMPDIR"]}/debug_log_pseudo_fifo"
 DISPLAY = ENV['DISPLAY']
 GIT_DIR = ENV['PWD']
 KEEP_SNAPSHOTS = !ENV['KEEP_SNAPSHOTS'].nil?
 LIVE_USER = cmd_helper(". config/chroot_local-includes/etc/live/config.d/username.conf; echo ${LIVE_USERNAME}").chomp
-OLD_TAILS_ISO = ENV['OLD_TAILS_ISO']
 TAILS_ISO = ENV['TAILS_ISO']
+OLD_TAILS_ISO = ENV['OLD_TAILS_ISO'] || TAILS_ISO
 TIME_AT_START = Time.now
 
 # Constants that are statically initialized.

features/support/extra_hooks.rb

@@ -46,20 +46,18 @@ def AfterFeature(*tag_expressions, &block)
   $after_feature_hooks << SimpleHook.new(tag_expressions, block)
 end
 
-AfterConfiguration do |config|
-  # Cucumber may read this file multiple times, and hence run this
-  # AfterConfiguration hook multiple times. Patching the formatter
-  # more than once will lead to problems, so let's ensure we only do
-  # it once.
-  next if $formatters_are_patched
-  # Multiple formatters can be registered, but we only patch one of
-  # them, since we only want our hooks to run once in total, not once
-  # for each formatter.
-  formatter_name, _ = config.formats.first
-  formatter = config.formatter_class(formatter_name)
-  formatter.class_exec do
-    if method_defined?(:before_feature)
-      alias old_before_feature before_feature
+def debug_log(message)
+  $debug_log_fns.each { |fn| fn.call(message) } if $debug_log_fns
+end
+
+require 'cucumber/formatter/pretty'
+module ExtraFormatters
+  # This is a null formatter in the sense that it doesn't ever output
+  # anything. We only use it do hook into the correct events so we can
+  # add our extra hooks.
+  class ExtraHooks
+    def initialize(*args)
+      # We do not care about any of the arguments.
     end
 
     def before_feature(feature)
@@ -68,19 +66,9 @@ AfterConfiguration do |config|
           hook.invoke(feature) if feature.accept_hook?(hook)
         end
       end
-      if self.class.method_defined?(:old_before_feature)
-        old_before_feature(feature)
-      end
-    end
-
-    if method_defined?(:after_feature)
-      alias old_after_feature after_feature
     end
 
     def after_feature(feature)
-      if self.class.method_defined?(:old_after_feature)
-        old_after_feature(feature)
-      end
       if $after_feature_hooks
         $after_feature_hooks.each do |hook|
           hook.invoke(feature) if feature.accept_hook?(hook)
@@ -88,5 +76,40 @@ AfterConfiguration do |config|
       end
     end
   end
-  $formatters_are_patched = true
+
+  # The pretty formatter with debug logging mixed into its output.
+  class PrettyDebug < Cucumber::Formatter::Pretty
+    def initialize(*args)
+      super(*args)
+      $debug_log_fns ||= []
+      $debug_log_fns << self.method(:debug_log)
+    end
+
+    def debug_log(message)
+      @io.puts(format_string(message, :blue))
+    end
+  end
+
+end
+
+module Cucumber
+  module Cli
+    class Options
+      BUILTIN_FORMATS['pretty_debug'] =
+        [
+          'ExtraFormatters::PrettyDebug',
+          'Prints the feature with debugging information - in colours.'
+        ]
+      BUILTIN_FORMATS['debug'] = BUILTIN_FORMATS['pretty_debug']
+    end
+  end
+end
+
+AfterConfiguration do |config|
+  # Cucumber may read this file multiple times, and hence run this
+  # AfterConfiguration hook multiple times. We only want our
+  # ExtraHooks formatter to be loaded once, otherwise the hooks would
+  # be run miltiple times.
+  extra_hooks = ['ExtraFormatters::ExtraHooks', '/dev/null']
+  config.formats << extra_hooks if not(config.formats.include?(extra_hooks))
 end

features/support/helpers/chatbot_helper.rb

@@ -27,6 +27,7 @@ class ChatBot
           ]
     cmd += ["--connect-server", @opts["connect_server"]] if @opts["connect_server"]
     cmd += ["--auto-join"] + @opts["auto_join"] if @opts["auto_join"]
+    cmd += ["--log-file", DEBUG_LOG_PSEUDO_FIFO]
 
     job = IO.popen(cmd)
     @pid = job.pid

features/support/helpers/ctcp_helper.rb

@@ -19,7 +19,7 @@ class CtcpChecker < Net::IRC::Client
       :user => nickname,
       :real => nickname,
     }
-    opts[:logger] = Logger.new("/dev/null") if !$config["DEBUG"]
+    opts[:logger] = Logger.new(DEBUG_LOG_PSEUDO_FIFO)
     super(host, port, opts)
   end
 

features/support/helpers/exec_helper.rb

@@ -32,14 +32,14 @@ class VMCommand
     options[:spawn] ||= false
     type = options[:spawn] ? "spawn" : "call"
     socket = TCPSocket.new("127.0.0.1", vm.get_remote_shell_port)
-    STDERR.puts "#{type}ing as #{options[:user]}: #{cmd}" if $config["DEBUG"]
+    debug_log("#{type}ing as #{options[:user]}: #{cmd}")
     begin
       socket.puts(JSON.dump([type, options[:user], cmd]))
       s = socket.readline(sep = "\0").chomp("\0")
     ensure
       socket.close
     end
-    STDERR.puts "#{type} returned: #{s}" if $config["DEBUG"]
+    debug_log("#{type} returned: #{s}") if not(options[:spawn])
     begin
       return JSON.load(s)
     rescue JSON::ParserError

features/support/helpers/sikuli_helper.rb

@@ -5,6 +5,9 @@ require 'sikuli-script.jar'
 Rjb::load
 
 package_members = [
+                   "java.io.FileOutputStream",
+                   "java.io.PrintStream",
+                   "java.lang.System",
                    "org.sikuli.script.Finder",
                    "org.sikuli.script.Key",
                    "org.sikuli.script.KeyModifier",
@@ -18,6 +21,8 @@ package_members = [
 
 translations = Hash[
                     "org.sikuli.script", "Sikuli",
+                    "java.lang", "Java::Lang",
+                    "java.io", "Java::Io",
                    ]
 
 for p in package_members
@@ -36,6 +41,14 @@ for p in package_members
   mod.const_set(class_name, imported_class)
 end
 
+# Bind Java's stdout to debug_log() via our magical pseudo fifo
+# logger.
+def bind_java_to_pseudo_fifo_logger
+  file_output_stream = Java::Io::FileOutputStream.new(DEBUG_LOG_PSEUDO_FIFO)
+  print_stream = Java::Io::PrintStream.new(file_output_stream)
+  Java::Lang::System.setOut(print_stream)
+end
+
 def findfailed_hook(pic)
   STDERR.puts ""
   STDERR.puts "FindFailed for: #{pic}"
@@ -61,6 +74,12 @@ end
 sikuli_script_proxy = Sikuli::Screen
 $_original_sikuli_screen_new ||= Sikuli::Screen.method :new
 
+# For waitAny()/findAny() we are forced to throw this exception since
+# Rjb::throw doesn't block until the Java exception has been received
+# by Ruby, so strange things can happen.
+class FindAnyFailed < StandardError
+end
+
 def sikuli_script_proxy.new(*args)
   s = $_original_sikuli_screen_new.call(*args)
 
@@ -132,6 +151,14 @@ def sikuli_script_proxy.new(*args)
     self.hover(self.wait(pic, time))
   end
 
+  def s.existsAny(images)
+    images.each do |image|
+      region = self.exists(image)
+      return [image, region] if region
+    end
+    return nil
+  end
+
   def s.findAny(images)
     images.each do |image|
       begin
@@ -142,23 +169,20 @@ def sikuli_script_proxy.new(*args)
       end
     end
     # If we've reached this point, none of the images could be found.
-    Rjb::throw('org.sikuli.script.FindFailed',
-               "can not find any of the images #{images} on the screen")
+    raise FindAnyFailed.new("can not find any of the images #{images} on the " +
+                            "screen")
   end
 
   def s.waitAny(images, time)
     Timeout::timeout(time) do
       loop do
-        begin
-          return self.findAny(images)
-        rescue FindFailed
-          # Ignore. We want to retry until we timeout.
-        end
+        result = self.existsAny(images)
+        return result if result
       end
     end
   rescue Timeout::Error
-    Rjb::throw('org.sikuli.script.FindFailed',
-               "can not find any of the images #{images} on the screen")
+    raise FindAnyFailed.new("can not find any of the images #{images} on the " +
+                            "screen")
   end
 
   def s.hover_point(x, y)
@@ -188,7 +212,7 @@ sikuli_settings.OcrDataPath = $config["TMPDIR"]
 # Also, Sikuli's default of 0.7 is simply too low (many false
 # positives).
 sikuli_settings.MinSimilarity = 0.9
-sikuli_settings.ActionLogs = $config["DEBUG"]
-sikuli_settings.DebugLogs = $config["DEBUG"]
-sikuli_settings.InfoLogs = $config["DEBUG"]
-sikuli_settings.ProfileLogs = $config["DEBUG"]
+sikuli_settings.ActionLogs = true
+sikuli_settings.DebugLogs = true
+sikuli_settings.InfoLogs = true
+sikuli_settings.ProfileLogs = true