Revert "Reset test suite to commit 7837bcbf."

This reverts commit 75449088.

config/chroot_local-includes/usr/local/lib/tails-autotest-remote-shell

@@ -6,6 +6,7 @@
 
 import base64
 import fcntl
+import io
 import json
 import os
 import pwd
@@ -14,22 +15,22 @@ import signal
 import subprocess
 import sys
 import systemd.daemon
+import textwrap
 import traceback
 
 REMOTE_SHELL_DEV = '/dev/ttyS0'
 
 
-def mk_switch_user_fn(uid, gid):
+def mk_switch_user_fn(user):
+    pwd_user = pwd.getpwnam(user)
     def switch_user():
-        os.setgid(gid)
-        os.setuid(uid)
+        os.initgroups(user, pwd_user.pw_gid)
+        os.setgid(pwd_user.pw_gid)
+        os.setuid(pwd_user.pw_uid)
     return switch_user
 
 
-def run_cmd_as_user(cmd, user):
-    pwd_user = pwd.getpwnam(user)
-    switch_user_fn = mk_switch_user_fn(pwd_user.pw_uid,
-                                       pwd_user.pw_gid)
+def get_user_env(user):
     # We try to create an environment identical to what's expected
     # inside Tails for the user by logging in (via `su`) as the user,
     # setting up the GNOME shell environment, and extracting the
@@ -41,7 +42,80 @@ def run_cmd_as_user(cmd, user):
     wrapped_env_cmd = "su -c '{}' {}".format(env_cmd, user)
     pipe = subprocess.Popen(wrapped_env_cmd, stdout=subprocess.PIPE, shell=True)
     env_data = pipe.communicate()[0].decode('utf-8')
-    env = dict((line.split('=', 1) for line in env_data.splitlines()))
+    return dict((line.split('=', 1) for line in env_data.splitlines()))
+
+
+# Dogtail does not seem to support the root user interacting with
+# other users' applications, and it does not support Python 3 (which
+# this script is written in) so let's wrap around an interactive
+# Python shell started as a subprocess.
+class PythonSession:
+    def __init__(self, user = None):
+        interactive_shell_code = '; '.join([
+            "import sys",
+            "import code",
+            "sys.ps1 = ''",
+            "sys.ps2 = ''",
+            "code.interact(banner='')",
+            ])
+        if not user:
+            user = pwd.getpwuid(os.getuid()).pw_name
+        env = get_user_env(user)
+        cwd = env['HOME']
+        self.process = subprocess.Popen(
+            ["python2", "-u", "-c", interactive_shell_code],
+            bufsize = 0,
+            shell=False,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+            cwd=cwd,
+            preexec_fn=mk_switch_user_fn(user)
+        )
+        init_code = """
+            import cStringIO
+            import json
+            import sys
+            orig_stdout = sys.stdout
+            orig_stderr = sys.stderr
+            """.replace('            ', '').lstrip()
+        self.process.stdin.write(init_code.encode())
+        self.process.stdin.flush()
+
+    def execute(self, code):
+        # This wrapping ensures that we can run almost any reasonable
+        # code and capture what it does.
+        wrapper = """
+            fake_stdout = cStringIO.StringIO()
+            fake_stderr = cStringIO.StringIO()
+            sys.stdout = fake_stdout
+            sys.stderr = fake_stderr
+            exc = None
+            try:
+            {code}
+            except Exception as e:
+                exc = '%s: %s' % (type(e).__name__, str(e))
+            # This newline is needed to end the `try` statement.
+
+            sys.stdout = orig_stdout
+            sys.stderr = orig_stderr
+            out_data = fake_stdout.getvalue()
+            err_data = fake_stderr.getvalue()
+            fake_stdout.close()
+            fake_stderr.close()
+            print(json.dumps([exc, out_data, err_data]))
+            """.replace('            ', '').lstrip()
+        indented_code = textwrap.indent(code, prefix=' '*4)
+        wrapped_code = wrapper.format(code=indented_code)
+        self.process.stdin.write(wrapped_code.encode())
+        self.process.stdin.flush()
+        return str(self.process.stdout.readline().strip(), 'utf-8')
+
+
+def run_cmd_as_user(cmd, user):
+    switch_user_fn = mk_switch_user_fn(user)
+    env = get_user_env(user)
     cwd = env['HOME']
     return subprocess.Popen(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
@@ -51,6 +125,7 @@ def run_cmd_as_user(cmd, user):
 
 def main():
     port = serial.Serial(port = REMOTE_SHELL_DEV, baudrate = 4000000)
+    python_sessions = dict()
 
     # Notify systemd that we're ready
     systemd.daemon.notify('READY=1')
@@ -61,10 +136,10 @@ def main():
         try:
             id, cmd_type, *rest = json.loads(line)
             ret = ""
-            if cmd_type in ['call', 'spawn']:
+            if cmd_type in ['sh_call', 'sh_spawn']:
                 user, cmd = rest
                 p = run_cmd_as_user(cmd, user)
-                if cmd_type == "spawn":
+                if cmd_type == "sh_spawn":
                     returncode, stdout, stderr = 0, "", ""
                 else:
                     stdout_b, stderr_b = p.communicate()
@@ -72,14 +147,22 @@ def main():
                     stderr = stderr_b.decode('utf-8')
                     returncode = p.returncode
                 ret = json.dumps([id, 'success', returncode, stdout, stderr])
-            elif cmd_type in ['read', 'write', 'append']:
+            elif cmd_type == 'python_execute':
+                user, code = rest
+                if user not in python_sessions:
+                    python_sessions[user] = PythonSession(user)
+                session = python_sessions[user]
+                result_str = session.execute(code)
+                result = json.loads(result_str)
+                ret = json.dumps([id, 'success'] + result)
+            elif cmd_type in ['file_read', 'file_write', 'file_append']:
                 path, *rest = rest
-                open_mode = cmd_type[0] + 'b'
+                open_mode = cmd_type[5] + 'b'
                 with open(path, open_mode) as f:
-                    if cmd_type == 'read':
+                    if cmd_type == 'file_read':
                         assert(rest == [])
                         ret = str(base64.b64encode(f.read()), 'utf-8')
-                    elif cmd_type in ['write', 'append']:
+                    elif cmd_type in ['file_write', 'file_append']:
                         assert(len(rest) == 1)
                         data = base64.b64decode(rest[0])
                         ret = f.write(data)

features/apt.feature

@@ -8,6 +8,7 @@ Feature: Installing packages through APT
   Scenario: APT sources are configured correctly
     Given I have started Tails from DVD without network and logged in
     Then the only hosts in APT sources are "vwakviie2ienjx6t.onion,sgvtcaew4bxjd7ln.onion,jenw7xbd6tf7vfhp.onion,sdscoq7snqtznauu.onion"
+    And no proposed-updates APT suite is enabled
 
   @check_tor_leaks
   Scenario: Install packages using apt

features/documentation.feature

@@ -3,8 +3,5 @@ Feature: Tails documentation
 
   Scenario: The "Report an Error" launcher will open the support documentation
     Given I have started Tails from DVD without network and logged in
-    And the network is plugged
-    And Tor is ready
-    And all notifications have disappeared
     When I double-click the Report an Error launcher on the desktop
     Then the support documentation page opens in Tor Browser

features/localization.feature

@@ -7,9 +7,7 @@ Feature: Localization
   @doc
   Scenario: The Report an Error launcher will open the support documentation in supported non-English locales
     Given I have started Tails from DVD without network and stopped at Tails Greeter's login screen
-    And the network is plugged
     And I log in to a new session in German
-    And Tor is ready
     When I double-click the Report an Error launcher on the desktop
     Then the support documentation page opens in Tor Browser
 

features/scripts/vm-execute

@@ -46,7 +46,7 @@ opt_parser = OptionParser.new do |opts|
 end
 opt_parser.parse!(ARGV)
 cmd = ARGV.join(" ")
-c = RemoteShell::Command.new(FakeVM.new, cmd, cmd_opts)
+c = RemoteShell::ShellCommand.new(FakeVM.new, cmd, cmd_opts)
 puts "Return status: #{c.returncode}"
 puts "STDOUT:\n#{c.stdout}"
 puts "STDERR:\n#{c.stderr}"

features/step_definitions/apt.rb

@@ -14,6 +14,13 @@ Given /^the only hosts in APT sources are "([^"]*)"$/ do |hosts_str|
   end
 end
 
+Given /^no proposed-updates APT suite is enabled$/ do
+  apt_sources = $vm.execute_successfully(
+    'cat /etc/apt/sources.list /etc/apt/sources.list.d/*'
+  ).stdout
+  assert_no_match(/\s\S+-proposed-updates\s/, apt_sources)
+end
+
 When /^I configure APT to use non-onion sources$/ do
   script = <<-EOF
   use strict;
@@ -64,8 +71,9 @@ When /^I start Synaptic$/ do
   @synaptic = Dogtail::Application.new('synaptic')
   # The seemingly spurious space is needed because that is how this
   # frame is named...
-  @synaptic.child('Synaptic Package Manager ', roleName: 'frame',
-                  recursive: false).wait
+  @synaptic.child(
+    'Synaptic Package Manager ', roleName: 'frame', recursive: false
+  )
 end
 
 When /^I update APT using Synaptic$/ do
@@ -96,7 +104,6 @@ Then /^I install "(.+)" using Synaptic$/ do |package_name|
   retry_tor(recovery_proc) do
     @synaptic.button('Search').click
     find_dialog = @synaptic.dialog('Find')
-    find_dialog.wait(10)
     find_dialog.child(roleName: 'text').typeText(package_name)
     find_dialog.button('Search').click
     package_list = @synaptic.child('Installed Version',
@@ -104,10 +111,12 @@ Then /^I install "(.+)" using Synaptic$/ do |package_name|
     package_entry = package_list.child(package_name, roleName: 'table cell')
     package_entry.doubleClick
     @synaptic.button('Apply').click
-    apply_prompt = @synaptic.dialog('Summary')
-    apply_prompt.wait(60)
+    apply_prompt = nil
+    try_for(60) { apply_prompt = @synaptic.dialog('Summary'); true }
     apply_prompt.button('Apply').click
-    @synaptic.child('Changes applied', roleName: 'frame',
-                    recursive: false).wait(4*60)
+    try_for(4*60) do
+      @synaptic.child('Changes applied', roleName: 'frame', recursive: false)
+      true
+    end
   end
 end

features/step_definitions/browser.rb

@@ -124,12 +124,12 @@ Then /^"([^"]+)" has loaded in the Tor Browser$/ do |title|
   end
   expected_title = "#{title} - #{browser_name}"
   app = Dogtail::Application.new('Firefox')
-  app.child(expected_title, roleName: 'frame').wait(60)
+  try_for(60) { app.child(expected_title, roleName: 'frame') }
   # The 'Reload current page' button (graphically shown as a looping
   # arrow) is only shown when a page has loaded, so once we see the
   # expected title *and* this button has appeared, then we can be sure
   # that the page has fully loaded.
-  app.child(reload_action, roleName: 'push button').wait(60)
+  try_for(60) { app.child(reload_action, roleName: 'push button') }
 end
 
 Then /^the (.*) has no plugins installed$/ do |browser|
@@ -228,9 +228,6 @@ end
 
 Then /^the Tor Browser shows the "([^"]+)" error$/ do |error|
   page = @torbrowser.child("Problem loading page", roleName: "document frame")
-  # Important to wait here since children() won't retry but return the
-  # immediate results
-  page.wait
   headers = page.children(roleName: "heading")
   found = headers.any? { |heading| heading.text == error }
   raise "Could not find the '#{error}' error in the Tor Browser" unless found

features/step_definitions/common_steps.rb

@@ -381,7 +381,6 @@ When /^I start the Tor Browser( in offline mode)?$/ do |offline|
   if offline
     offline_prompt = Dogtail::Application.new('zenity')
                      .dialog('Tor is not ready')
-    offline_prompt.wait(10)
     offline_prompt.button('Start Tor Browser').click
   end
   @torbrowser = Dogtail::Application.new('Firefox')

features/step_definitions/gnome.rb

@@ -20,5 +20,5 @@ When /^the "(.+)" notification is sent$/ do |title|
 end
 
 Then /^the "(.+)" notification is shown to the user$/ do |title|
-  Dogtail::Application.new('gnome-shell').child(title).wait
+  Dogtail::Application.new('gnome-shell').child(title)
 end

features/step_definitions/icedove.rb

@@ -32,7 +32,7 @@ When /^I start Icedove$/ do
     $vm.file_append('/etc/icedove/pref/icedove.js ', line)
   end
   step 'I start "Icedove" via the GNOME "Internet" applications menu'
-  icedove_main.wait(60)
+  try_for(60) { icedove_main }
 end
 
 When /^I have not configured an email account$/ do
@@ -44,7 +44,7 @@ When /^I have not configured an email account$/ do
 end
 
 Then /^I am prompted to setup an email account$/ do
-  icedove_wizard.wait(30)
+  icedove_wizard
 end
 
 Then /^I cancel setting up an email account$/ do
@@ -57,7 +57,6 @@ Then /^I open Icedove's Add-ons Manager$/ do
   @icedove_addons = icedove_app.child(
     'Add-ons Manager - Icedove Mail/News', roleName: 'frame'
   )
-  @icedove_addons.wait
 end
 
 Then /^I click the extensions tab$/ do
@@ -80,7 +79,7 @@ end
 
 Then /^I see that Torbirdy is configured to use Tor$/ do
   icedove_main.child(roleName: 'status bar')
-    .child('TorBirdy Enabled:    Tor', roleName: 'label').wait
+    .child('TorBirdy Enabled:    Tor', roleName: 'label')
 end
 
 When /^I enter my email credentials into the autoconfiguration wizard$/ do
@@ -90,7 +89,7 @@ When /^I enter my email credentials into the autoconfiguration wizard$/ do
     .typeText($config['Icedove']['password'])
   icedove_wizard.button('Continue').click
   # This button is shown if and only if a configuration has been found
-  icedove_wizard.button('Done').wait(120)
+  try_for(120) { icedove_wizard.button('Done') }
 end
 
 Then /^the autoconfiguration wizard's choice for the (incoming|outgoing) server is secure (.+)$/ do |type, protocol|
@@ -109,24 +108,31 @@ When /^I fetch my email$/ do
 
   icedove_main.child('Mail Toolbar', roleName: 'tool bar')
     .button('Get Messages').click
-  fetch_progress = icedove_main.child(roleName: 'status bar')
-                     .child(roleName: 'progress bar')
-  fetch_progress.wait_vanish(120)
+  try_for(120) do
+    begin
+      icedove_main.child(roleName: 'status bar', retry: false)
+        .child(roleName: 'progress bar', retry: false)
+      false
+    rescue
+      true
+    end
+  end
 end
 
 When /^I accept the (?:autoconfiguration wizard's|manual) configuration$/ do
   # The password check can fail due to bad Tor circuits.
   retry_tor do
     try_for(120) do
-      if icedove_wizard.exist?
+      begin
         # Spam the button, even if it is disabled (while it is still
         # testing the password).
         icedove_wizard.button('Done').click
         false
-      else
+      rescue
         true
       end
     end
+    true
   end
   # The account isn't fully created before we fetch our mail. For
   # instance, if we'd try to send an email before this, yet another
@@ -165,7 +171,6 @@ end
 When /^I send an email to myself$/ do
   icedove_main.child('Mail Toolbar', roleName: 'tool bar').button('Write').click
   compose_window = icedove_app.child('Write: (no subject)')
-  compose_window.wait(10)
   compose_window.child('To:', roleName: 'autocomplete').child(roleName: 'entry')
     .typeText($config['Icedove']['address'])
   # The randomness of the subject will make it easier for us to later
@@ -179,7 +184,9 @@ When /^I send an email to myself$/ do
     .typeText('test')
   compose_window.child('Composition Toolbar', roleName: 'tool bar')
     .button('Send').click
-  compose_window.wait_vanish(120)
+  try_for(120) do
+    not compose_window.exist?
+  end
 end
 
 Then /^I can find the email I sent to myself in my inbox$/ do
@@ -190,7 +197,6 @@ Then /^I can find the email I sent to myself in my inbox$/ do
                                 roleName: 'entry')
     filter.typeText(@subject)
     hit_counter = icedove_main.child('1 message')
-    hit_counter.wait
     inbox_view = hit_counter.parent
     message_list = inbox_view.child(roleName: 'table')
     the_message = message_list.child(@subject, roleName: 'table cell')

features/step_definitions/ssh.rb

@@ -127,7 +127,7 @@ Then /^I connect to an SFTP server on the Internet$/ do
   retry_tor(recovery_proc) do
     step 'I start "Nautilus" via the GNOME "Accessories" applications menu'
     nautilus = Dogtail::Application.new('nautilus')
-    nautilus.child(roleName: 'frame').wait
+    nautilus.child(roleName: 'frame')
     nautilus.child('Other Locations', roleName: 'label').click
     connect_bar = nautilus.child('Connect to Server', roleName: 'label').parent
     connect_bar

features/step_definitions/tor.rb

@@ -325,22 +325,21 @@ end
 
 When /^I connect Gobby to "([^"]+)"$/ do |host|
   gobby = Dogtail::Application.new('gobby-0.5')
-  gobby.child('Welcome to Gobby', roleName: 'label').wait(30)
+  gobby.child('Welcome to Gobby', roleName: 'label')
   gobby.button('Close').click
   # This indicates that Gobby has finished initializing itself
   # (generating DH parameters, etc.) -- before, the UI is not responsive
   # and our CTRL-t is lost.
-  gobby.child('Failed to share documents', roleName: 'label').wait(30)
+  gobby.child('Failed to share documents', roleName: 'label')
   gobby.menu('File').click
   gobby.menuItem('Connect to Server...').click
   @screen.type("t", Sikuli::KeyModifier.CTRL)
   connect_dialog = gobby.dialog('Connect to Server')
-  connect_dialog.wait(10)
   connect_dialog.child('', roleName: 'text').typeText(host)
   connect_dialog.button('Connect').click
   # This looks for the live user's presence entry in the chat, which
   # will only be shown if the connection succeeded.
-  gobby.child(LIVE_USER, roleName: 'table cell').wait(60)
+  try_for(60) { gobby.child(LIVE_USER, roleName: 'table cell'); true }
 end
 
 When /^the Tor Launcher autostarts$/ do

features/step_definitions/usb.rb

@@ -98,8 +98,12 @@ def usb_install_helper(name)
   begin
     @installer.button('Install Tails').click
     @installer.child('Question', roleName: 'alert').button('Yes').click
-    @installer.child('Information', roleName: 'alert')
-      .child('Installation complete!', roleName: 'label').wait(30*60)
+    try_for(30*60) do
+      @installer
+        .child('Information', roleName: 'alert')
+        .child('Installation complete!', roleName: 'label')
+      true
+    end
   rescue FindFailed => e
     path = $vm.execute_successfully('ls -1 /tmp/tails-installer-*').stdout.chomp
     debug_log("Tails Installer debug log:\n" + $vm.file_content(path))
@@ -110,10 +114,9 @@ end
 When /^I start Tails Installer in "([^"]+)" mode$/ do |mode|
   step 'I run "export DEBUG=1 ; tails-installer-launcher" in GNOME Terminal'
   installer_launcher = Dogtail::Application.new('tails-installer-launcher')
-  installer_launcher.wait(10)
   installer_launcher.button(mode).click
   @installer = Dogtail::Application.new('tails-installer')
-  @installer.child('Tails Installer', roleName: 'frame').wait
+  @installer.child('Tails Installer', roleName: 'frame')
 end
 
 Then /^Tails Installer detects that a device is too small$/ do
@@ -137,8 +140,9 @@ When /^I am suggested to do a "Install by cloning"$/ do
 end
 
 Then /^a suitable USB device is (?:still )?not found$/ do
-  @installer.child('No device suitable to install Tails could be found',
-                   roleName: 'label').wait(30)
+  @installer.child(
+    'No device suitable to install Tails could be found', roleName: 'label'
+  )
 end
 
 Then /^(no|the "([^"]+)") USB drive is selected$/ do |mode, name|
@@ -176,7 +180,6 @@ When /^I do a "Upgrade from ISO" on USB drive "([^"]+)"$/ do |name|
   @installer.child('Use existing Live system ISO:', roleName: 'label')
     .parent.button('(None)').click
   file_chooser = @installer.child('Select a File', roleName: 'file chooser')
-  file_chooser.wait(10)
   @screen.type("l", Sikuli::KeyModifier.CTRL)
   # The only visible text element will be the path entry
   file_chooser.child(roleName: 'text').text = @iso_path

features/support/helpers/dogtail.rb

@@ -46,59 +46,51 @@ module Dogtail
   #     menu.click()
   #
   # i.e. the object referenced by `menu` is never modified by method
-  # calls and can be used as expected. This explains why
-  # `proxy_call()` below returns a new instance instead of adding
-  # appending the new component the proxied method call would result
-  # in.
+  # calls and can be used as expected.
 
   class Application
+    @@node_counter ||= 0
 
     def initialize(app_name, opts = {})
+      @var = "node#{@@node_counter += 1}"
       @app_name = app_name
       @opts = opts
-      @init_lines = @opts[:init_lines] || [
-        "from dogtail import tree",
-        "from dogtail.config import config",
-        "config.logDebugToFile = False",
-        "config.logDebugToStdOut = False",
-        "config.blinkOnActions = True",
-        "config.searchShowingOnly = True",
-        "application = tree.root.application('#{@app_name}')",
+      @opts[:user] ||= LIVE_USER
+      @find_code = "dogtail.tree.root.application('#{@app_name}')"
+      script_lines = [
+        "import dogtail.config",
+        "import dogtail.tree",
+        "import dogtail.predicate",
+        "dogtail.config.logDebugToFile = False",
+        "dogtail.config.logDebugToStdOut = False",
+        "dogtail.config.blinkOnActions = True",
+        "dogtail.config.searchShowingOnly = True",
+        "#{@var} = #{@find_code}",
       ]
-      @components = @opts[:components] || ['application']
+      run(script_lines)
     end
 
-    def build_script(lines)
-      (
-        ["#!/usr/bin/python"] +
-        @init_lines +
-        lines
-      ).join("\n")
+    def to_s
+      @var
     end
 
-    def build_line
-      @components.join('.')
+    def run(code)
+      code = code.join("\n") if code.class == Array
+      c = RemoteShell::PythonCommand.new($vm, code, user: @opts[:user])
+      if c.failure?
+        raise RuntimeError.new("The Dogtail script raised: #{c.exception}")
+      end
+      return c
     end
 
-    def run(lines = nil)
-      @opts[:user] ||= LIVE_USER
-      lines ||= [build_line]
-      lines = [lines] if lines.class != Array
-      script = build_script(lines)
-      script_path = $vm.execute_successfully('mktemp', @opts).stdout.chomp
-      $vm.file_overwrite(script_path, script)
-      args = ["/usr/bin/python '#{script_path}'", @opts]
-      if @opts[:allow_failure]
-        return $vm.execute(*args)
-      else
-        begin
-          return $vm.execute_successfully(*args)
-        rescue Exception => e
-          debug_log("Failing Dogtail script (#{script_path}):")
-          script.split("\n").each { |line| debug_log(" "*4 + line) }
-          raise e
-        end
-      end
+    def exist?
+      run("dogtail.config.searchCutoffCount = 0")
+      run(@find_code)
+      return true
+    rescue
+      return false
+    ensure
+      run("dogtail.config.searchCutoffCount = 20")
     end
 
     def self.value_to_s(v)
@@ -133,28 +125,6 @@ module Dogtail
       ).join(', ')
     end
 
-    def wait(timeout = nil)
-      if timeout
-        try_for(timeout) { run }
-      else
-        run
-      end
-    end
-
-    def exist?
-      @opts[:allow_failure] = true
-      # We do not want any retries since this method should return the
-      # result for the immediate situation, not for the situation up
-      # to 20 retries in the future.
-      optimization = "config.searchCutoffCount = 1"
-      @init_lines << optimization unless @init_lines.include?(optimization)
-      run.