Commit c493a5f5 authored by intrigeri's avatar intrigeri
Browse files

Merge branch 'devel' into feature/stretch

parents 959b01cf 246559f3
[Unit]
Description=Remote shell (over serial link) used in Tails test suite
Description=Remote shell used in Tails test suite
Documentation=https://tails.boum.org/contribute/release_process/test/automated_tests/
ConditionKernelCommandLine=autotest_never_use_this_option
Before=gdm.service
[Service]
Type=notify
ExecStart=/usr/local/lib/tails-autotest-remote-shell /dev/ttyS0
ExecStart=/usr/local/lib/tails-autotest-remote-shell
OOMScoreAdjust=-1000
[Install]
......
......@@ -4,76 +4,105 @@
# adversary with access to you *physical* serial port, which means
# that you are screwed any way.
from subprocess import Popen, PIPE
from sys import argv
from json import dumps, loads
from pwd import getpwnam
from os import setgid, setuid, environ
import base64
import fcntl
import json
import os
import pwd
import serial
from systemd.daemon import notify as sd_notify
import signal
import subprocess
import sys
import systemd.daemon
import traceback
REMOTE_SHELL_DEV = '/dev/ttyS0'
def mk_switch_user_fn(uid, gid):
def switch_user():
setgid(gid)
setuid(uid)
os.setgid(gid)
os.setuid(uid)
return switch_user
def run_cmd_as_user(cmd, user):
pwd_user = getpwnam(user)
switch_user_fn = mk_switch_user_fn(pwd_user.pw_uid,
pwd_user.pw_gid)
# We try to create an environment identical to what's expected
# inside Tails for the user by logging in (via `su`) as the user,
# setting up the GNOME shell environment, and extracting the
# environment via `env`; not that we will run `env` unconditionally
# since the former command could fail, e.g. if GNOME is not running.
env_cmd = '. /usr/local/lib/tails-shell-library/gnome.sh && ' + \
'export_gnome_env ; ' + \
'env'
wrapped_env_cmd = "su -c '{}' {}".format(env_cmd, user)
pipe = Popen(wrapped_env_cmd, stdout=PIPE, shell=True)
env_data = pipe.communicate()[0].decode('utf-8')
env = dict((line.split('=', 1) for line in env_data.splitlines()))
cwd = env['HOME']
return Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, env=env, cwd=cwd,
preexec_fn=switch_user_fn)
pwd_user = pwd.getpwnam(user)
switch_user_fn = mk_switch_user_fn(pwd_user.pw_uid,
pwd_user.pw_gid)
# We try to create an environment identical to what's expected
# inside Tails for the user by logging in (via `su`) as the user,
# setting up the GNOME shell environment, and extracting the
# environment via `env`; not that we will run `env` unconditionally
# since the former command could fail, e.g. if GNOME is not running.
env_cmd = '. /usr/local/lib/tails-shell-library/gnome.sh && ' + \
'export_gnome_env ; ' + \
'env'
wrapped_env_cmd = "su -c '{}' {}".format(env_cmd, user)
pipe = subprocess.Popen(wrapped_env_cmd, stdout=subprocess.PIPE, shell=True)
env_data = pipe.communicate()[0].decode('utf-8')
env = dict((line.split('=', 1) for line in env_data.splitlines()))
cwd = env['HOME']
return subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
shell=True, env=env, cwd=cwd, preexec_fn=switch_user_fn
)
def main():
dev = argv[1]
port = serial.Serial(port = dev, baudrate = 4000000)
if not port.isOpen():
port.open()
port = serial.Serial(port = REMOTE_SHELL_DEV, baudrate = 4000000)
# Notify systemd that we're ready
sd_notify('READY=1')
sd_notify('STATUS=Processing requests...\n')
# Notify systemd that we're ready
systemd.daemon.notify('READY=1')
systemd.daemon.notify('STATUS=Processing requests...\n')
while True:
try:
line = port.readline().decode('utf-8')
except Exception as e:
# port must be opened wrong, so we restart everything and pray
# that it works.
print(str(e))
port.close()
return main()
try:
id, cmd_type, user, cmd = loads(line)
except Exception as e:
# We had a parse/pack error, so we just send a \0 as an ACK,
# releasing the client from blocking.
print(str(e))
port.write(b"\0")
continue
p = run_cmd_as_user(cmd, user)
if cmd_type == "spawn":
returncode, stdout, stderr = 0, "", ""
else:
stdout_b, stderr_b = p.communicate()
stdout = stdout_b.decode('utf-8')
stderr = stderr_b.decode('utf-8')
returncode = p.returncode
port.write(dumps([id, returncode, stdout, stderr]).encode('utf-8') + b"\0")
while True:
line = port.readline().decode('utf-8')
try:
id, cmd_type, *rest = json.loads(line)
ret = ""
if cmd_type in ['call', 'spawn']:
user, cmd = rest
p = run_cmd_as_user(cmd, user)
if cmd_type == "spawn":
returncode, stdout, stderr = 0, "", ""
else:
stdout_b, stderr_b = p.communicate()
stdout = stdout_b.decode('utf-8')
stderr = stderr_b.decode('utf-8')
returncode = p.returncode
ret = json.dumps([id, 'success', returncode, stdout, stderr])
elif cmd_type in ['read', 'write', 'append']:
path, *rest = rest
open_mode = cmd_type[0] + 'b'
with open(path, open_mode) as f:
if cmd_type == 'read':
assert(rest == [])
ret = str(base64.b64encode(f.read()), 'utf-8')
elif cmd_type in ['write', 'append']:
assert(len(rest) == 1)
data = base64.b64decode(rest[0])
ret = f.write(data)
if ret != len(data):
raise IOError("we only wrote {} bytes out of {}"
.format(ret, len(data)))
ret = json.dumps([id, 'success'] + [ret])
else:
raise ValueError("unknown command type")
response = (ret + "\n").encode('utf-8')
port.write(response)
port.flush()
except Exception as e:
print("Error caught while processing line:", file=sys.stderr)
print(" " + line, file=sys.stderr)
print("The error was:", file=sys.stderr)
traceback.print_exc(file=sys.stdout)
print("-----", file=sys.stderr)
sys.stderr.flush()
exc_str = '{}: {}'.format(type(e).__name__, str(e))
port.write(json.dumps([id, 'error', exc_str]).encode('utf-8') + b"\n")
port.flush()
continue
if __name__ == "__main__":
main()
main()
......@@ -2,7 +2,8 @@
require 'optparse'
begin
require "#{`git rev-parse --show-toplevel`.chomp}/features/support/helpers/exec_helper.rb"
require "#{`git rev-parse --show-toplevel`.chomp}/features/support/helpers/remote_shell.rb"
require "#{`git rev-parse --show-toplevel`.chomp}/features/support/helpers/misc_helpers.rb"
rescue LoadError => e
raise "This script must be run from within Tails' Git directory."
end
......@@ -45,7 +46,7 @@ opt_parser = OptionParser.new do |opts|
end
opt_parser.parse!(ARGV)
cmd = ARGV.join(" ")
c = VMCommand.new(FakeVM.new, cmd, cmd_opts)
c = RemoteShell::Command.new(FakeVM.new, cmd, cmd_opts)
puts "Return status: #{c.returncode}"
puts "STDOUT:\n#{c.stdout}"
puts "STDERR:\n#{c.stderr}"
......
......@@ -2,13 +2,16 @@ require 'uri'
Given /^the only hosts in APT sources are "([^"]*)"$/ do |hosts_str|
hosts = hosts_str.split(',')
$vm.file_content("/etc/apt/sources.list /etc/apt/sources.list.d/*").chomp.each_line { |line|
apt_sources = $vm.execute_successfully(
"cat /etc/apt/sources.list /etc/apt/sources.list.d/*"
).stdout.chomp
apt_sources.each_line do |line|
next if ! line.start_with? "deb"
source_host = URI(line.split[1]).host
if !hosts.include?(source_host)
raise "Bad APT source '#{line}'"
end
}
end
end
When /^I configure APT to use non-onion sources$/ do
......
......@@ -23,10 +23,10 @@ Given /^I generate an OpenPGP key named "([^"]+)" with password "([^"]+)"$/ do |
Passphrase: #{pwd}
%commit
EOF
gpg_key_recipie.split("\n").each do |line|
$vm.execute("echo '#{line}' >> /tmp/gpg_key_recipie", :user => LIVE_USER)
end
c = $vm.execute("gpg --batch --gen-key < /tmp/gpg_key_recipie",
recipe_path = '/tmp/gpg_key_recipe'
$vm.file_overwrite(recipe_path, gpg_key_recipie)
$vm.execute("chown #{LIVE_USER}:#{LIVE_USER} #{recipe_path}")
c = $vm.execute("gpg --batch --gen-key < #{recipe_path}",
:user => LIVE_USER)
assert(c.success?, "Failed to generate OpenPGP key:\n#{c.stderr}")
end
......
......@@ -204,8 +204,9 @@ end
def configured_pidgin_accounts
accounts = Hash.new
xml = REXML::Document.new($vm.file_content('$HOME/.purple/accounts.xml',
LIVE_USER))
xml = REXML::Document.new(
$vm.file_content("/home/#{LIVE_USER}/.purple/accounts.xml")
)
xml.elements.each("account/account") do |e|
account = e.elements["name"].text
account_name, network = account.split("@")
......@@ -264,7 +265,7 @@ def default_chan (account)
end
def pidgin_otr_keys
return $vm.file_content('$HOME/.purple/otr.private_key', LIVE_USER)
return $vm.file_content("/home/#{LIVE_USER}/.purple/otr.private_key")
end
Given /^Pidgin has the expected accounts configured with random nicknames$/ do
......
......@@ -167,12 +167,12 @@ def reach_checkpoint(name)
post_snapshot_restore_hook
end
debug_log(scenario_indent + "Checkpoint: #{checkpoint_description}",
:color => :white)
color: :white, timestamp: false)
step_action = "Given"
if parent_checkpoint
parent_description = checkpoints[parent_checkpoint][:description]
debug_log(step_indent + "#{step_action} #{parent_description}",
:color => :green)
color: :green, timestamp: false)
step_action = "And"
end
steps.each do |s|
......@@ -181,10 +181,11 @@ def reach_checkpoint(name)
rescue Exception => e
debug_log(scenario_indent +
"Step failed while creating checkpoint: #{s}",
:color => :red)
color: :red, timestamp: false)
raise e
end
debug_log(step_indent + "#{step_action} #{s}", :color => :green)
debug_log(step_indent + "#{step_action} #{s}",
color: :green, timestamp: false)
step_action = "And"
end
$vm.save_snapshot(name)
......
......@@ -101,7 +101,8 @@ def usb_install_helper(name)
@installer.child('Information', roleName: 'alert')
.child('Installation complete!', roleName: 'label').wait(30*60)
rescue FindFailed => e
debug_log("Tails Installer debug log:\n" + $vm.file_content('/tmp/tails-installer-*'))
path = $vm.execute_successfully('ls -1 /tmp/tails-installer-*').stdout.chomp
debug_log("Tails Installer debug log:\n" + $vm.file_content(path))
raise e
end
end
......
......@@ -77,7 +77,16 @@ def info_log(message = "", options = {})
end
def debug_log(message, options = {})
$debug_log_fns.each { |fn| fn.call(message, options) } if $debug_log_fns
options[:timestamp] = true unless options.has_key?(:timestamp)
if $debug_log_fns
if options[:timestamp]
# Force UTC so the local timezone difference vs UTC won't be
# added to the result.
elapsed = (Time.now - TIME_AT_START.to_f).utc.strftime("%H:%M:%S.%9N")
message = "#{elapsed}: #{message}"
end
$debug_log_fns.each { |fn| fn.call(message, options) }
end
end
require 'cucumber/formatter/pretty'
......
......@@ -86,15 +86,19 @@ module Dogtail
lines = [lines] if lines.class != Array
script = build_script(lines)
script_path = $vm.execute_successfully('mktemp', @opts).stdout.chomp
$vm.file_overwrite(script_path, script, @opts[:user])
$vm.file_overwrite(script_path, script)
args = ["/usr/bin/python '#{script_path}'", @opts]
if @opts[:allow_failure]
ret = $vm.execute(*args)
return $vm.execute(*args)
else
ret = $vm.execute_successfully(*args)
begin
return $vm.execute_successfully(*args)
rescue Exception => e
debug_log("Failing Dogtail script (#{script_path}):")
script.split("\n").each { |line| debug_log(" "*4 + line) }
raise e
end
end
$vm.execute("rm -f '#{script_path}'")
ret
end
def self.value_to_s(v)
......
require 'json'
require 'socket'
class VMCommand
@@request_id ||= 0
attr_reader :cmd, :returncode, :stdout, :stderr
def initialize(vm, cmd, options = {})
@cmd = cmd
@returncode, @stdout, @stderr = VMCommand.execute(vm, cmd, options)
end
def VMCommand.wait_until_remote_shell_is_up(vm, timeout = 90)
try_for(timeout, :msg => "Remote shell seems to be down") do
Timeout::timeout(3) do
VMCommand.execute(vm, "echo 'hello?'")
end
end
end
# If `:spawn` is false the server will block until it has finished
# executing `cmd`. If it's true the server won't block, and the
# response will always be [0, "", ""] (only used as an
# ACK). execute() will always block until a response is received,
# though. Spawning is useful when starting processes in the
# background (or running scripts that does the same) or any
# application we want to interact with.
def VMCommand.execute(vm, cmd, options = {})
options[:user] ||= "root"
options[:spawn] ||= false
type = options[:spawn] ? "spawn" : "call"
id = (@@request_id += 1)
socket = TCPSocket.new("127.0.0.1", vm.get_remote_shell_port)
debug_log("#{type}ing as #{options[:user]}: #{cmd}")
socket.puts(JSON.dump([id, type, options[:user], cmd]))
loop do
s = socket.readline(sep = "\0").chomp("\0")
response_id, *rest = JSON.load(s)
if response_id == id
debug_log("#{type} returned: #{s}") if not(options[:spawn])
return rest
else
debug_log("Dropped out-of-order remote shell response: " +
"got id #{response_id} but expected id #{id}")
end
end
ensure
socket.close if defined?(socket) && socket
end
def success?
return @returncode == 0
end
def failure?
return not(success?)
end
def to_s
"Return status: #{@returncode}\n" +
"STDOUT:\n" +
@stdout +
"STDERR:\n" +
@stderr
end
end
......@@ -3,7 +3,7 @@ require 'packetfu'
# Returns the unique edges (based on protocol, source/destination
# address/port) in the graph of all network flows.
def pcap_connections_helper(pcap_file, opts = {})
opts[:ignore_dhcp] ||= true
opts[:ignore_dhcp] = true unless opts.has_key?(:ignore_dhcp)
connections = Array.new
packets = PacketFu::PcapFile.new.file_to_array(:filename => pcap_file)
packets.each do |p|
......
......@@ -30,8 +30,12 @@ end
# Call block (ignoring any exceptions it may throw) repeatedly with
# one second breaks until it returns true, or until `timeout` seconds have
# passed when we throw a Timeout::Error exception.
# passed when we throw a Timeout::Error exception. If `timeout` is `nil`,
# then we just run the code block with no timeout.
def try_for(timeout, options = {})
if block_given? && timeout.nil?
return yield
end
options[:delay] ||= 1
last_exception = nil
# Create a unique exception used only for this particular try_for
......@@ -78,11 +82,12 @@ def try_for(timeout, options = {})
# ends up there immediately.
rescue unique_timeout_exception => e
msg = options[:msg] || 'try_for() timeout expired'
exc_class = options[:exception] || Timeout::Error
if last_exception
msg += "\nLast ignored exception was: " +
"#{last_exception.class}: #{last_exception}"
end
raise Timeout::Error.new(msg)
raise exc_class.new(msg)
end
class TorFailure < StandardError
......@@ -275,6 +280,9 @@ end
def pause(message = "Paused")
STDERR.puts
STDERR.puts message
# Ring the ASCII bell for a helpful notification in most terminal
# emulators.
STDOUT.write "\a"
STDERR.puts
loop do
STDERR.puts "Return: Continue; d: Debugging REPL"
......
require 'base64'
require 'json'
require 'socket'
require 'timeout'
module RemoteShell
class ServerFailure < StandardError
end
# Used to differentiate vs Timeout::Error, which is thrown by
# try_for() (by default) and often wraps around remote shell usage
# -- in that case we don't want to catch that "outer" exception in
# our handling of remote shell timeouts below.
class Timeout < ServerFailure
end
DEFAULT_TIMEOUT = 20*60
# Counter providing unique id:s for each communicate() call.
@@request_id ||= 0
def communicate(vm, *args, **opts)
opts[:timeout] ||= DEFAULT_TIMEOUT
socket = TCPSocket.new("127.0.0.1", vm.get_remote_shell_port)
id = (@@request_id += 1)
# Since we already have defined our own Timeout in the current
# scope, we have to be more careful when referring to the Timeout
# class from the 'timeout' module. However, note that we want it
# to throw our own Timeout exception.
Object::Timeout.timeout(opts[:timeout], Timeout) do
socket.puts(JSON.dump([id] + args))
socket.flush
loop do
line = socket.readline("\n").chomp("\n")
response_id, status, *rest = JSON.load(line)
if response_id == id
if status != "success"
if status == "error" and rest.class == Array and rest.size == 1
msg = rest.first
raise ServerFailure.new("#{msg}")
else
raise ServerFailure.new("Uncaught exception: #{status}: #{rest}")
end
end
return rest
else
debug_log("Dropped out-of-order remote shell response: " +
"got id #{response_id} but expected id #{id}")
end
end
end
ensure
socket.close if defined?(socket) && socket
end
module_function :communicate
private :communicate
class Command
# If `:spawn` is false the server will block until it has finished
# executing `cmd`. If it's true the server won't block, and the
# response will always be [0, "", ""] (only used as an
# ACK). execute() will always block until a response is received,
# though. Spawning is useful when starting processes in the
# background (or running scripts that does the same) or any
# application we want to interact with.
def self.execute(vm, cmd, **opts)
opts[:user] ||= "root"
opts[:spawn] = false unless opts.has_key?(:spawn)
type = opts[:spawn] ? "spawn" : "call"
debug_log("#{type}ing as #{opts[:user]}: #{cmd}")
ret = RemoteShell.communicate(vm, type, opts[:user], cmd, **opts)
debug_log("#{type} returned: #{ret}") if not(opts[:spawn])
return ret
end
attr_reader :cmd, :returncode, :stdout, :stderr
def initialize(vm, cmd, **opts)
@cmd = cmd
@returncode, @stdout, @stderr = self.class.execute(vm, cmd, **opts)
end
def success?
return @returncode == 0
end
def failure?
return not(success?)
end
def to_s
"Return status: #{@returncode}\n" +
"STDOUT:\n" +
@stdout +
"STDERR:\n" +
@stderr
end
end
# An IO-like object that is more or less equivalent to a File object
# opened in rw mode.
class File
def self.open(vm, mode, path, *args, **opts)
debug_log("opening file #{path} in '#{mode}' mode")
ret = RemoteShell.communicate(vm, mode, path, *args, **opts)
if ret.size != 1
raise ServerFailure.new("expected 1 value but got #{ret.size}")
end
debug_log("#{mode} complete")
return ret.first
end
attr_reader :vm, :path
def initialize(vm, path)
@vm, @path = vm, path
end
def read()
Base64.decode64(self.class.open(@vm, 'read', @path))
end