misc_helpers.rb 9.87 KB
Newer Older
1
require 'date'
2
3
require 'io/console'
require 'pry'
4
require 'timeout'
5
require 'test/unit'
6

7
8
9
10
11
12
13
14
15
# Test::Unit adds an at_exit hook which, among other things, consumes
# the command-line arguments that were intended for cucumber. If
# e.g. `--format` was passed it will throw an error since it's not a
# valid option for Test::Unit, and it throwing an error at this time
# (at_exit) will make Cucumber think it failed and consequently exit
# with an error. Fooling Test::Unit that this hook has already run
# works around this craziness.
Test::Unit.run = true

16
17
# Make all the assert_* methods easily accessible in any context.
include Test::Unit::Assertions
18

19
20
21
22
23
24
25
def assert_vmcommand_success(p, msg = nil)
  assert(p.success?, msg.nil? ? "Command failed: #{p.cmd}\n" + \
                                "error code: #{p.returncode}\n" \
                                "stderr: #{p.stderr}" : \
                                msg)
end

26
27
28
# It's forbidden to throw this exception (or subclasses) in anything
# but try_for() below. Just don't use it anywhere else!
class UniqueTryForTimeoutError < Exception
29
30
31
end

# Call block (ignoring any exceptions it may throw) repeatedly with
anonym's avatar
anonym committed
32
# one second breaks until it returns true, or until `timeout` seconds have
anonym's avatar
anonym committed
33
34
# passed when we throw a Timeout::Error exception. If `timeout` is `nil`,
# then we just run the code block with no timeout.
anonym's avatar
anonym committed
35
def try_for(timeout, options = {})
anonym's avatar
anonym committed
36
37
38
  if block_given? && timeout.nil?
    return yield
  end
39
  options[:delay] ||= 1
40
  last_exception = nil
41
42
43
44
45
46
  # Create a unique exception used only for this particular try_for
  # call's Timeout to allow nested try_for:s. If we used the same one,
  # the innermost try_for would catch all outer ones', creating a
  # really strange situation.
  unique_timeout_exception = Class.new(UniqueTryForTimeoutError)
  Timeout::timeout(timeout, unique_timeout_exception) do
anonym's avatar
anonym committed
47
48
    loop do
      begin
anonym's avatar
anonym committed
49
        return if yield
50
51
52
53
      rescue NameError, UniqueTryForTimeoutError => e
        # NameError most likely means typos, and hiding that is rarely
        # (never?) a good idea, so we rethrow them. See below why we
        # also rethrow *all* the unique exceptions.
anonym's avatar
anonym committed
54
        raise e
55
56
57
58
59
      rescue Exception => e
        # All other exceptions are ignored while trying the
        # block. Well we save the last exception so we can print it in
        # case of a timeout.
        last_exception = e
60
      end
anonym's avatar
anonym committed
61
      sleep options[:delay]
62
63
    end
  end
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
  # At this point the block above either succeeded and we'll return,
  # or we are throwing an exception. If the latter, we either have a
  # NameError that we'll not catch (and will any try_for below us in
  # the stack), or we have a unique exception. That can mean one of
  # two things:
  # 1. it's the one unique to this try_for, and in that case we'll
  #    catch it, rethrowing it as something that will be ignored by
  #    inside the blocks of all try_for:s below us in the stack.
  # 2. it's an exception unique to another try_for. Assuming that we
  #    do not throw the unique exceptions in any other place or way
  #    than we do it in this function, this means that there is a
  #    try_for below us in the stack to which this exception must be
  #    unique to.
  # Let 1 be the base step, and 2 the inductive step, and we sort of
  # an inductive proof for the correctness of try_for when it's
  # nested. It shows that for an infinite stack of try_for:s, any of
  # the unique exceptions will be caught only by the try_for instance
  # it is unique to, and all try_for:s in between will ignore it so it
  # ends up there immediately.
rescue unique_timeout_exception => e
84
  msg = options[:msg] || 'try_for() timeout expired'
85
  exc_class = options[:exception] || Timeout::Error
86
87
88
89
  if last_exception
    msg += "\nLast ignored exception was: " +
           "#{last_exception.class}: #{last_exception}"
  end
90
  raise exc_class.new(msg)
91
92
end

anonym's avatar
anonym committed
93
94
95
class TorFailure < StandardError
end

96
97
98
class MaxRetriesFailure < StandardError
end

anonym's avatar
anonym committed
99
100
def force_new_tor_circuit()
  debug_log("Forcing new Tor circuit...")
101
  # Tor rate limits NEWNYM to at most one per 10 second period.
102
  interval = 10
103
104
105
  if $__last_newnym
    elapsed = Time.now - $__last_newnym
    # We sleep an extra second to avoid tight timings.
106
    sleep interval - elapsed + 1 if 0 < elapsed && elapsed < interval
107
  end
anonym's avatar
anonym committed
108
  $vm.execute_successfully('tor_control_send "signal NEWNYM"', :libs => 'tor')
109
  $__last_newnym = Time.now
anonym's avatar
anonym committed
110
111
end

anonym's avatar
anonym committed
112
113
114
115
# This will retry the block up to MAX_NEW_TOR_CIRCUIT_RETRIES
# times. The block must raise an exception for a run to be considered
# as a failure. After a failure recovery_proc will be called (if
# given) and the intention with it is to bring us back to the state
intrigeri's avatar
intrigeri committed
116
# expected by the block, so it can be retried.
anonym's avatar
anonym committed
117
def retry_tor(recovery_proc = nil, &block)
kytv's avatar
kytv committed
118
119
120
121
122
123
124
125
  tor_recovery_proc = Proc.new do
    force_new_tor_circuit
    recovery_proc.call if recovery_proc
  end

  retry_action($config['MAX_NEW_TOR_CIRCUIT_RETRIES'],
               :recovery_proc => tor_recovery_proc,
               :operation_name => 'Tor operation', &block)
126
127
end

kytv's avatar
kytv committed
128
129
130
131
132
def retry_action(max_retries, options = {}, &block)
  assert(max_retries.is_a?(Integer), "max_retries must be an integer")
  options[:recovery_proc] ||= nil
  options[:operation_name] ||= 'Operation'

133
  retries = 1
anonym's avatar
anonym committed
134
135
136
137
  loop do
    begin
      block.call
      return
138
139
140
141
    rescue NameError => e
      # NameError most likely means typos, and hiding that is rarely
      # (never?) a good idea, so we rethrow them.
      raise e
anonym's avatar
anonym committed
142
143
    rescue Exception => e
      if retries <= max_retries
kytv's avatar
kytv committed
144
        debug_log("#{options[:operation_name]} failed (Try #{retries} of " +
145
146
                  "#{max_retries}) with:\n" +
                  "#{e.class}: #{e.message}")
kytv's avatar
kytv committed
147
        options[:recovery_proc].call if options[:recovery_proc]
anonym's avatar
anonym committed
148
149
        retries += 1
      else
kytv's avatar
kytv committed
150
151
152
        raise MaxRetriesFailure.new("#{options[:operation_name]} failed (despite retrying " +
                                    "#{max_retries} times) with\n" +
                                    "#{e.class}: #{e.message}")
anonym's avatar
anonym committed
153
154
155
156
157
      end
    end
  end
end

158
159
alias :retry_times :retry_action

160
def wait_until_tor_is_working
161
  try_for(270) { $vm.execute('/usr/local/sbin/tor-has-bootstrapped').success? }
162
rescue Timeout::Error => e
163
  c = $vm.execute("journalctl SYSLOG_IDENTIFIER=restart-tor")
164
  if c.success?
anonym's avatar
anonym committed
165
    debug_log("From the journal:\n" + c.stdout.sub(/^/, "  "))
166
  else
anonym's avatar
anonym committed
167
    debug_log("Nothing was in the journal about 'restart-tor'")
168
169
  end
  raise e
170
end
171

Tails developers's avatar
Tails developers committed
172
def convert_bytes_mod(unit)
173
  case unit
Tails developers's avatar
Tails developers committed
174
175
176
177
178
179
180
181
182
  when "bytes", "b" then mod = 1
  when "KB"         then mod = 10**3
  when "k", "KiB"   then mod = 2**10
  when "MB"         then mod = 10**6
  when "M", "MiB"   then mod = 2**20
  when "GB"         then mod = 10**9
  when "G", "GiB"   then mod = 2**30
  when "TB"         then mod = 10**12
  when "T", "TiB"   then mod = 2**40
183
  else
Tails developers's avatar
Tails developers committed
184
    raise "invalid memory unit '#{unit}'"
185
  end
Tails developers's avatar
Tails developers committed
186
187
188
189
190
191
192
  return mod
end

def convert_to_bytes(size, unit)
  return (size*convert_bytes_mod(unit)).to_i
end

193
194
195
196
def convert_to_MiB(size, unit)
  return (size*convert_bytes_mod(unit) / (2**20)).to_i
end

Tails developers's avatar
Tails developers committed
197
198
def convert_from_bytes(size, unit)
  return size.to_f/convert_bytes_mod(unit).to_f
199
end
200

201
def cmd_helper(cmd, env = {})
202
203
204
205
206
  if cmd.instance_of?(Array)
    cmd << {:err => [:child, :out]}
  elsif cmd.instance_of?(String)
    cmd += " 2>&1"
  end
207
208
  env = ENV.to_h.merge(env)
  IO.popen(env, cmd) do |p|
209
    out = p.readlines.join("\n")
210
211
    p.close
    ret = $?
212
    assert_equal(0, ret, "Command failed (returned #{ret}): #{cmd}:\n#{out}")
213
    return out
214
215
  end
end
216

217
def all_tor_hosts
218
219
220
221
222
223
224
225
226
227
228
229
  nodes = Array.new
  chutney_torrcs = Dir.glob(
    "#{$config['TMPDIR']}/chutney-data/nodes/*/torrc"
  )
  chutney_torrcs.each do |torrc|
    open(torrc) do |f|
      nodes += f.grep(/^(Or|Dir)Port\b/).map do |line|
        { address: $vmnet.bridge_ip_addr, port: line.split.last.to_i }
      end
    end
  end
  return nodes
230
231
end

232
233
234
235
def allowed_hosts_under_tor_enforcement
  all_tor_hosts + @lan_hosts
end

236
237
238
239
def get_free_space(machine, path)
  case machine
  when 'host'
    assert(File.exists?(path), "Path '#{path}' not found on #{machine}.")
240
    free = cmd_helper(["df", path])
241
  when 'guest'
242
243
    assert($vm.file_exist?(path), "Path '#{path}' not found on #{machine}.")
    free = $vm.execute_successfully("df '#{path}'")
244
245
246
247
248
249
  else
    raise 'Unsupported machine type #{machine} passed.'
  end
  output = free.split("\n").last
  return output.match(/[^\s]\s+[0-9]+\s+[0-9]+\s+([0-9]+)\s+.*/)[1].chomp.to_i
end
250

251
def random_string_from_set(set, min_len, max_len)
252
253
254
255
256
  len = (min_len..max_len).to_a.sample
  len ||= min_len
  (0..len-1).map { |n| set.sample }.join
end

257
def random_alpha_string(min_len, max_len = 0)
258
259
  alpha_set = ('A'..'Z').to_a + ('a'..'z').to_a
  random_string_from_set(alpha_set, min_len, max_len)
260
261
end

262
def random_alnum_string(min_len, max_len = 0)
263
264
  alnum_set = ('A'..'Z').to_a + ('a'..'z').to_a + (0..9).to_a.map { |n| n.to_s }
  random_string_from_set(alnum_set, min_len, max_len)
265
end
266
267

# Sanitize the filename from unix-hostile filename characters
268
269
def sanitize_filename(filename, options = {})
  options[:replacement] ||= '_'
270
  bad_unix_filename_chars = Regexp.new("[^A-Za-z0-9_\\-.,+:]")
271
  filename.gsub(bad_unix_filename_chars, options[:replacement])
272
end
273
274

def info_log_artifact_location(type, path)
anonym's avatar
anonym committed
275
  if $config['ARTIFACTS_BASE_URI']
276
    # Remove any trailing slashes, we'll add one ourselves
anonym's avatar
anonym committed
277
    base_url = $config['ARTIFACTS_BASE_URI'].gsub(/\/*$/, "")
278
279
280
281
    path = "#{base_url}/#{File.basename(path)}"
  end
  info_log("#{type.capitalize}: #{path}")
end
282

283
284
285
286
287
288
def notify_user(message)
  alarm_script = $config['NOTIFY_USER_COMMAND']
  return if alarm_script.nil? || alarm_script.empty?
  cmd_helper(alarm_script.gsub('%m', message))
end

289
def pause(message = "Paused")
290
  notify_user(message)
291
292
  STDERR.puts
  STDERR.puts message
293
294
295
  # Ring the ASCII bell for a helpful notification in most terminal
  # emulators.
  STDOUT.write "\a"
296
  STDERR.puts
297
298
299
300
301
302
303
304
305
306
  loop do
    STDERR.puts "Return: Continue; d: Debugging REPL"
    c = STDIN.getch
    case c
    when "\r"
      return
    when "d"
      binding.pry(quiet: true)
    end
  end
307
end