Rakefile 23.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# -*- mode: ruby -*-
# vi: set ft=ruby :
#
# Tails: The Amnesic Incognito Live System
# Copyright © 2012 Tails developers <tails@boum.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

20
require 'date'
21
require 'English'
22
require 'libvirt'
anonym's avatar
anonym committed
23
require 'open3'
24
require 'rbconfig'
25
require 'uri'
26

27
require_relative 'vagrant/lib/tails_build_settings'
28

29
# Path to the directory which holds our Vagrantfile
30
VAGRANT_PATH = File.expand_path('vagrant', __dir__)
31

32
# Branches that are considered 'stable' (used to select SquashFS compression)
33
STABLE_BRANCH_NAMES = ['stable', 'testing'].freeze
34

35
36
EXPORTED_VARIABLES = [
  'MKSQUASHFS_OPTIONS',
37
  'APT_SNAPSHOTS_SERIALS',
38
  'TAILS_ACNG_PROXY',
39
  'TAILS_BUILD_FAILURE_RESCUE',
40
  'TAILS_DATE_OFFSET',
anonym's avatar
anonym committed
41
  'TAILS_OFFLINE_MODE',
42
43
44
  'TAILS_PROXY',
  'TAILS_PROXY_TYPE',
  'TAILS_RAM_BUILD',
45
  'TAILS_WEBSITE_CACHE',
bertagaz's avatar
bertagaz committed
46
47
48
  'GIT_COMMIT',
  'GIT_REF',
  'BASE_BRANCH_GIT_COMMIT',
49
].freeze
50
ENV['EXPORTED_VARIABLES'] = EXPORTED_VARIABLES.join(' ')
Tails developers's avatar
Tails developers committed
51

52
53
EXTERNAL_HTTP_PROXY = ENV['http_proxy']

54
# In-VM proxy URL
55
INTERNAL_HTTP_PROXY = "http://#{VIRTUAL_MACHINE_HOSTNAME}:3142".freeze
56

57
58
ENV['ARTIFACTS'] ||= '.'

59
ENV['APT_SNAPSHOTS_SERIALS'] ||= ''
60

61
class CommandError < StandardError
anonym's avatar
anonym committed
62
63
  attr_reader :status, :stderr

64
  def initialize(message, **opts)
anonym's avatar
anonym committed
65
66
67
68
    opts[:status] ||= nil
    opts[:stderr] ||= nil
    @status = opts[:status]
    @stderr = opts[:stderr]
intrigeri's avatar
intrigeri committed
69
    super(format(message, status: @status, stderr: @stderr))
70
71
72
73
74
  end
end

def run_command(*args)
  Process.wait Kernel.spawn(*args)
75
76
  return if $CHILD_STATUS.exitstatus.zero?

77
78
  raise CommandError.new("command #{args} failed with exit status %<status>s",
                         status: $CHILD_STATUS.exitstatus)
79
80
81
end

def capture_command(*args)
anonym's avatar
anonym committed
82
  stdout, stderr, proc_status = Open3.capture3(*args)
83
  if proc_status.exitstatus != 0
84
    raise CommandError.new("command #{args} failed with exit status " \
85
                           '%<status>s: %<stderr>s',
anonym's avatar
anonym committed
86
                           stderr: stderr, status: proc_status.exitstatus)
87
  end
88
  [stdout, stderr]
89
90
end

anonym's avatar
anonym committed
91
def git_helper(*args)
92
93
  question = args.first.end_with?('?')
  args.first.sub!(/\?$/, '')
anonym's avatar
anonym committed
94
95
96
  status = 0
  stdout = ''
  begin
97
    stdout, = capture_command('auto/scripts/utils.sh', *args)
anonym's avatar
anonym committed
98
99
100
  rescue CommandError => e
    status = e.status
  end
intrigeri's avatar
intrigeri committed
101
  question ? status.zero? : stdout.chomp
anonym's avatar
anonym committed
102
103
end

104
class VagrantCommandError < CommandError
105
106
end

anonym's avatar
anonym committed
107
108
# Runs the vagrant command, letting stdout/stderr through. Throws an
# exception unless the vagrant command succeeds.
anonym's avatar
anonym committed
109
def run_vagrant(*args)
110
  run_command('vagrant', *args, chdir: './vagrant')
anonym's avatar
anonym committed
111
rescue CommandError => e
112
  raise(VagrantCommandError, "'vagrant #{args}' command failed with exit " \
anonym's avatar
anonym committed
113
                             "status #{e.status}")
114
115
end

anonym's avatar
anonym committed
116
# Runs the vagrant command, not letting stdout/stderr through, and
intrigeri's avatar
intrigeri committed
117
# returns [stdout, stderr, Process::Status].
anonym's avatar
anonym committed
118
def capture_vagrant(*args)
119
  capture_command('vagrant', *args, chdir: './vagrant')
120
rescue CommandError => e
121
  raise(VagrantCommandError, "'vagrant #{args}' command failed with exit " \
anonym's avatar
anonym committed
122
                             "status #{e.status}: #{e.stderr}")
123
124
end

anonym's avatar
anonym committed
125
126
[:run_vagrant, :capture_vagrant].each do |m|
  define_method "#{m}_ssh" do |*args|
anonym's avatar
anonym committed
127
    method(m).call('ssh', '-c', *args, '--', '-q')
128
  end
129
130
end

anonym's avatar
anonym committed
131
def vagrant_ssh_config(key)
anonym's avatar
anonym committed
132
  # Cache results
anonym's avatar
anonym committed
133
  if $vagrant_ssh_config.nil?
intrigeri's avatar
Linting    
intrigeri committed
134
135
136
    $vagrant_ssh_config = capture_vagrant('ssh-config')
                          .first.split("\n") \
                          .map { |line| line.strip.split(/\s+/, 2) } .to_h
anonym's avatar
anonym committed
137
138
139
140
141
142
143
    # The path in the ssh-config output is quoted, which is not what
    # is expected outside of a shell, so let's get rid of the quotes.
    $vagrant_ssh_config['IdentityFile'].gsub!(/^"|"$/, '')
  end
  $vagrant_ssh_config[key]
end

anonym's avatar
anonym committed
144
def current_vm_cpus
anonym's avatar
anonym committed
145
  capture_vagrant_ssh('grep -c "^processor\s*:" /proc/cpuinfo').first.chomp.to_i
146
147
end

anonym's avatar
anonym committed
148
def vm_state
149
  out, = capture_vagrant('status')
anonym's avatar
anonym committed
150
151
  status_line = out.split("\n")[2]
  if    status_line['not created']
152
    :not_created
anonym's avatar
anonym committed
153
  elsif status_line['shutoff']
154
    :poweroff
anonym's avatar
anonym committed
155
  elsif status_line['running']
156
    :running
157
  else
158
    raise 'could not determine VM state'
159
160
161
  end
end

162
def enough_free_host_memory_for_ram_build?
163
164
165
  return false unless RbConfig::CONFIG['host_os'] =~ /linux/i

  begin
166
    usable_free_mem = `free`.split[12].to_i
167
    usable_free_mem > VM_MEMORY_FOR_RAM_BUILDS * 1024
168
  rescue StandardError
169
170
171
172
    false
  end
end

173
def free_vm_memory
174
  capture_vagrant_ssh('free').first.chomp.split[12].to_i
175
176
177
178
179
180
181
end

def enough_free_vm_memory_for_ram_build?
  free_vm_memory > BUILD_SPACE_REQUIREMENT * 1024
end

def enough_free_memory_for_ram_build?
anonym's avatar
anonym committed
182
  if vm_state == :running
183
184
185
186
187
188
    enough_free_vm_memory_for_ram_build?
  else
    enough_free_host_memory_for_ram_build?
  end
end

189
def releasing?
190
  git_helper('git_on_a_tag?')
191
192
end

193
def system_cpus
194
  return unless RbConfig::CONFIG['host_os'] =~ /linux/i
195
196
197

  begin
    File.read('/proc/cpuinfo').scan(/^processor\s+:/).count
198
  rescue StandardError
199
200
201
202
    nil
  end
end

203
ENV['TAILS_WEBSITE_CACHE'] = releasing? ? '0' : '1'
204

205
task :parse_build_options do
206
  options = []
207

208
  # Default to in-memory builds if there is enough RAM available
209
  options << 'ram' if enough_free_memory_for_ram_build?
210
  # Default to build using the in-VM proxy
211
  options << 'vmproxy'
212
  # Default to fast compression on development branches
213
  options << 'fastcomp' unless releasing?
214
215
  # Default to the number of system CPUs when we can figure it out
  cpus = system_cpus
216
  options << "cpus=#{cpus}" if cpus
217

218
  options += ENV['TAILS_BUILD_OPTIONS'].split if ENV['TAILS_BUILD_OPTIONS']
219

220
  options.uniq.each do |opt| # rubocop:disable Metrics/BlockLength
221
    case opt
222
223
224
225
226
    # Memory build settings
    when 'ram'
      ENV['TAILS_RAM_BUILD'] = '1'
    when 'noram'
      ENV['TAILS_RAM_BUILD'] = nil
227
    # Bootstrap cache settings
228
229
    # HTTP proxy settings
    when 'extproxy'
230
      unless EXTERNAL_HTTP_PROXY
231
232
        abort 'No HTTP proxy set, but one is required by ' \
              'TAILS_BUILD_OPTIONS. Aborting.'
233
      end
234
      ENV['TAILS_PROXY'] = EXTERNAL_HTTP_PROXY
235
      ENV['TAILS_PROXY_TYPE'] = 'extproxy'
236
    when 'vmproxy', 'vmproxy+extproxy'
237
      ENV['TAILS_PROXY'] = INTERNAL_HTTP_PROXY
238
      ENV['TAILS_PROXY_TYPE'] = 'vmproxy'
239
      if opt == 'vmproxy+extproxy'
240
        unless EXTERNAL_HTTP_PROXY
241
242
          abort 'No HTTP proxy set, but one is required by ' \
                'TAILS_BUILD_OPTIONS. Aborting.'
243
        end
244
245
        ENV['TAILS_ACNG_PROXY'] = EXTERNAL_HTTP_PROXY
      end
246
    when 'noproxy'
247
      ENV['TAILS_PROXY'] = nil
248
      ENV['TAILS_PROXY_TYPE'] = 'noproxy'
249
250
    when 'offline'
      ENV['TAILS_OFFLINE_MODE'] = '1'
251
    when /cachewebsite(?:=([a-z]+))?/
252
      value = Regexp.last_match(1)
253
      if releasing?
254
        warn "Building a release ⇒ ignoring #{opt} build option"
255
256
        ENV['TAILS_WEBSITE_CACHE'] = '0'
      else
257
258
259
260
261
262
263
264
265
        value = 'yes' if value.nil?
        case value
        when 'yes'
          ENV['TAILS_WEBSITE_CACHE'] = '1'
        when 'no'
          ENV['TAILS_WEBSITE_CACHE'] = '0'
        else
          raise "Unsupported value for cachewebsite option: #{value}"
        end
266
      end
267
    # SquashFS compression settings
268
    when 'fastcomp', 'gzipcomp'
269
      if releasing?
270
        warn "Building a release ⇒ ignoring #{opt} build option"
271
272
273
        ENV['MKSQUASHFS_OPTIONS'] = nil
      else
        ENV['MKSQUASHFS_OPTIONS'] = '-comp xz -no-exports'
274
      end
275
276
    when 'defaultcomp'
      ENV['MKSQUASHFS_OPTIONS'] = nil
277
278
    # Virtual hardware settings
    when /machinetype=([a-zA-Z0-9_.-]+)/
279
      ENV['TAILS_BUILD_MACHINE_TYPE'] = Regexp.last_match(1)
280
    when /cpus=(\d+)/
281
      ENV['TAILS_BUILD_CPUS'] = Regexp.last_match(1)
282
    when /cpumodel=([a-zA-Z0-9_-]+)/
283
      ENV['TAILS_BUILD_CPU_MODEL'] = Regexp.last_match(1)
284
285
286
    # Git settings
    when 'ignorechanges'
      ENV['TAILS_BUILD_IGNORE_CHANGES'] = '1'
287
    when /dateoffset=([-+]\d+)/
288
      ENV['TAILS_DATE_OFFSET'] = Regexp.last_match(1)
289
290
291
    # Developer convenience features
    when 'keeprunning'
      $keep_running = true
292
293
294
295
      $force_cleanup = false
    when 'forcecleanup'
      $force_cleanup = true
      $keep_running = false
296
297
298
    when 'rescue'
      $keep_running = true
      ENV['TAILS_BUILD_FAILURE_RESCUE'] = '1'
299
    # Jenkins
300
301
    when 'nomergebasebranch'
      $skip_mergebasebranch = true
302
    else
303
304
305
306
307
308
309
      # Handle jenkins passing the now obsolete mergebasebranch.
      # XXX: the commit adding this comment should be reverted
      # once jenkins stops passing the mergebasebranch option.
      if opt == 'mergebasebranch' && on_jenkins?
        warn "The 'mergebasebranch' option is obsolete!"
        next
      end
310
      raise "Unknown Tails build option '#{opt}'"
311
312
    end
  end
313
314

  if ENV['TAILS_OFFLINE_MODE'] == '1'
315
    if ENV['TAILS_PROXY'].nil?
316
      abort 'You must use a caching proxy when building offline'
317
318
    end
  end
319
320
321
end

task :ensure_clean_repository do
322
323
  git_status = `git status --porcelain`
  unless git_status.empty?
324
    if ENV['TAILS_BUILD_IGNORE_CHANGES']
325
      warn <<-END_OF_MESSAGE.gsub(/^        /, '')
326

hybridwipe's avatar
hybridwipe committed
327
        You have uncommitted changes in the Git repository. They will
328
329
        be ignored for the upcoming build:
        #{git_status}
330
331
332

      END_OF_MESSAGE
    else
333
      warn <<-END_OF_MESSAGE.gsub(/^        /, '')
334

hybridwipe's avatar
hybridwipe committed
335
        You have uncommitted changes in the Git repository. Due to limitations
336
337
        of the build system, you need to commit them before building Tails:
        #{git_status}
338
339
340
341
342
343

        If you don't care about those changes and want to build Tails nonetheless,
        please add `ignorechanges` to the TAILS_BUILD_OPTIONS environment
        variable.

      END_OF_MESSAGE
hybridwipe's avatar
hybridwipe committed
344
      abort 'Uncommitted changes. Aborting.'
345
346
347
348
    end
  end
end

anonym's avatar
anonym committed
349
350
def list_artifacts
  user = vagrant_ssh_config('User')
351
352
  stdout = capture_vagrant_ssh("find '/home/#{user}/amnesia/' -maxdepth 1 " \
                                        "-name 'tails-amd64-*' " \
353
                                        '-o -name tails-build-env.list').first
anonym's avatar
anonym committed
354
  stdout.split("\n")
355
rescue VagrantCommandError
356
  []
anonym's avatar
anonym committed
357
358
359
end

def remove_artifacts
360
  list_artifacts.each do |artifact|
anonym's avatar
anonym committed
361
    run_vagrant_ssh("sudo rm -f '#{artifact}'")
362
  end
anonym's avatar
anonym committed
363
364
end

365
task ensure_clean_home_directory: ['vm:up'] do
anonym's avatar
anonym committed
366
  remove_artifacts
367
368
end

369
task :validate_http_proxy do
370
371
  if ENV['TAILS_PROXY']
    proxy_host = URI.parse(ENV['TAILS_PROXY']).host
372

373
    if proxy_host.nil?
374
      ENV['TAILS_PROXY'] = nil
375
      warn 'Ignoring invalid HTTP proxy.'
376
377
378
      return
    end

379
380
381
382
    if ['localhost', '[::1]'].include?(proxy_host) \
       || proxy_host.start_with?('127.0.0.')
      abort 'Using an HTTP proxy listening on the loopback is doomed ' \
            'to fail. Aborting.'
383
384
    end

385
    warn "Using HTTP proxy: #{ENV['TAILS_PROXY']}"
386
  else
387
    warn 'No HTTP proxy set.'
388
389
390
  end
end

391
task :validate_git_state do
392
  if git_helper('git_in_detached_head?') && !git_helper('git_on_a_tag?')
393
394
395
396
    raise 'We are in detached head but the current commit is not tagged'
  end
end

397
task setup_environment: ['validate_git_state'] do
398
  ENV['GIT_COMMIT'] ||= git_helper('git_current_commit')
399
400
  ENV['GIT_REF'] ||= git_helper('git_current_head_name')
  if on_jenkins?
401
    jenkins_branch = (ENV['GIT_BRANCH'] || '').sub(%r{^origin/}, '')
402
    if !releasing? && jenkins_branch != ENV['GIT_REF']
403
404
      raise "We expected to build the Git ref '#{ENV['GIT_REF']}', " \
            "but GIT_REF in the environment says '#{jenkins_branch}'. Aborting!"
405
406
    end
  end
407

408
  ENV['BASE_BRANCH_GIT_COMMIT'] ||= git_helper('git_base_branch_head')
bertagaz's avatar
bertagaz committed
409
  ['GIT_COMMIT', 'GIT_REF', 'BASE_BRANCH_GIT_COMMIT'].each do |var|
intrigeri's avatar
intrigeri committed
410
    next unless ENV[var].empty?
411

412
413
    raise "Variable '#{var}' is empty, which should not be possible: " \
          "either validate_git_state is buggy or the 'origin' remote " \
intrigeri's avatar
intrigeri committed
414
          'does not point to the official Tails Git repository.'
415
416
417
  end
end

418
419
420
421
422
423
task merge_base_branch: ['parse_build_options', 'setup_environment'] do
  next if $skip_mergebasebranch
  branch = git_helper('git_current_branch')
  base_branch = git_helper('base_branch')
  source_date_faketime = `date --utc --date="$(dpkg-parsechangelog --show-field=Date)" '+%Y-%m-%d %H:%M:%S'`.chomp
  next if releasing? || branch == base_branch
424
  commit_before_merge = git_helper('git_current_commit')
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
  warn "Merging base branch '#{base_branch}' (at commit " \
       "#{ENV['BASE_BRANCH_GIT_COMMIT']}) ..."
  begin
    run_command('faketime', '-f', source_date_faketime, \
                'git', 'merge', '--no-edit', ENV['BASE_BRANCH_GIT_COMMIT'])
  rescue CommandError
    run_command('git', 'merge', '--abort')
    raise <<-END_OF_MESSAGE.gsub(/^        /, '')

          There were conflicts when merging the base branch; either
          merge it yourself and resolve conflicts, or skip this merge
          by rebuilding with the 'nomergebasebranch' option.

    END_OF_MESSAGE
  end
  run_command('git', 'submodule', 'update', '--init')
441
442
443
444
445
446
447

  # If we actually merged anything we'll re-run rake in the new Git
  # state in order to avoid subtle build errors due to mixed state.
  next if commit_before_merge == git_helper('git_current_commit')
  ENV['TAILS_BUILD_OPTIONS'] = (ENV['TAILS_BUILD_OPTIONS'] || '') + \
                               ' nomergebasebranch'
  Kernel.exec('rake', *ARGV)
448
449
end

450
451
task :maybe_clean_up_builder_vms do
  clean_up_builder_vms if $force_cleanup
452
453
end

454
455
456
task :ensure_correct_permissions do
  FileUtils.chmod('go+x', '.')
  FileUtils.chmod_R('go+rX', ['.git', 'submodules', 'vagrant'])
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474

  # Changing permissions outside of the working copy, in particular on
  # parent directories such as $HOME, feels too blunt and can have
  # problematic security consequences, so we don't forcibly do that.
  # Instead, when the permissions are not OK, display a nicer error
  # message than "Virtio-9p Failed to initialize fs-driver […]"
  begin
    capture_command('sudo', '-u', 'libvirt-qemu', 'stat', '.git')
  rescue CommandError
    abort <<-END_OF_MESSAGE.gsub(/^      /, '')

      Incorrect permissions: the libvirt-qemu user needs to be allowed
      to traverse the filesystem up to #{ENV['PWD']}.

      To fix this, you can for example run the following command
      on every parent directory of #{ENV['PWD']} up to #{ENV['HOME']}
      (inclusive):

475
        chmod g+rx DIR && setfacl -m user:libvirt-qemu:rx DIR
476
477
478

    END_OF_MESSAGE
  end
479
480
end

Tails developers's avatar
Tails developers committed
481
desc 'Build Tails'
482
task build: [
483
484
485
486
487
  'parse_build_options',
  'ensure_clean_repository',
  'maybe_clean_up_builder_vms',
  'validate_git_state',
  'setup_environment',
488
  'merge_base_branch',
489
490
491
  'validate_http_proxy',
  'ensure_correct_permissions',
  'vm:up',
492
  'ensure_clean_home_directory',
493
] do
anonym's avatar
anonym committed
494
  begin
495
    if ENV['TAILS_RAM_BUILD'] && !enough_free_memory_for_ram_build?
496
      warn <<-END_OF_MESSAGE.gsub(/^        /, '')
497

anonym's avatar
anonym committed
498
499
500
501
        The virtual machine is not currently set with enough memory to
        perform an in-memory build. Either remove the `ram` option from
        the TAILS_BUILD_OPTIONS environment variable, or shut the
        virtual machine down using `rake vm:halt` before trying again.
502

anonym's avatar
anonym committed
503
      END_OF_MESSAGE
504
505
      abort 'Not enough memory for the virtual machine to run an in-memory ' \
            'build. Aborting.'
anonym's avatar
anonym committed
506
    end
507

508
509
    if ENV['TAILS_BUILD_CPUS'] \
       && current_vm_cpus != ENV['TAILS_BUILD_CPUS'].to_i
510
      warn <<-END_OF_MESSAGE.gsub(/^        /, '')
511

anonym's avatar
anonym committed
512
513
514
515
        The virtual machine is currently running with #{current_vm_cpus}
        virtual CPU(s). In order to change that number, you need to
        stop the VM first, using `rake vm:halt`. Otherwise, please
        adjust the `cpus` options accordingly.
516

anonym's avatar
anonym committed
517
      END_OF_MESSAGE
518
519
      abort 'The virtual machine needs to be reloaded to change the number ' \
            'of CPUs. Aborting.'
anonym's avatar
anonym committed
520
    end
521

intrigeri's avatar
Linting    
intrigeri committed
522
523
    exported_env = EXPORTED_VARIABLES
                   .select { |k| ENV[k] }
524
                   .map    { |k| "#{k}='#{ENV[k]}'" }.join(' ')
525
526
527
528
529

    begin
      retrieved_artifacts = false
      run_vagrant_ssh("#{exported_env} build-tails")
    rescue VagrantCommandError
530
      retrieve_artifacts(missing_ok: true)
531
532
      retrieved_artifacts = true
    ensure
533
      retrieve_artifacts(missing_ok: false) unless retrieved_artifacts
534
535
      clean_up_builder_vms unless $keep_running
    end
anonym's avatar
anonym committed
536
  ensure
537
    clean_up_builder_vms if $force_cleanup
538
  end
Tails developers's avatar
Tails developers committed
539
540
end

541
desc 'Retrieve build artifacts from the Vagrant box'
542
543
544
545
task :retrieve_artifacts do
  retrieve_artifacts
end

546
547
548
549
def retrieve_artifacts(missing_ok: false)
  artifacts = list_artifacts
  if artifacts.empty?
    msg = 'No build artifacts were found!'
550
551
552
553
    raise msg unless missing_ok

    warn msg
    return
554
  end
555
  user = vagrant_ssh_config('User')
556
557
  hostname = vagrant_ssh_config('HostName')
  key_file = vagrant_ssh_config('IdentityFile')
558
  warn 'Retrieving artifacts from Vagrant build box.'
559
560
561
562
563
564
  run_vagrant_ssh(
    "sudo chown #{user} " + artifacts.map { |a| "'#{a}'" } .join(' ')
  )
  fetch_command = [
    'scp',
    '-i', key_file,
565
566
    # We don't want to use any identity saved in ssh agent'
    '-o', 'IdentityAgent=none',
567
568
569
570
571
572
573
574
575
576
577
578
579
580
    # We need this since the user will not necessarily have a
    # known_hosts entry. It is safe since an attacker must
    # compromise libvirt's network config or the user running the
    # command to modify the #{hostname} below.
    '-o', 'StrictHostKeyChecking=no',
    '-o', 'UserKnownHostsFile=/dev/null',
    # Speed up the copy
    '-o', 'Compression=no',
  ]
  fetch_command += artifacts.map { |a| "#{user}@#{hostname}:#{a}" }
  fetch_command << ENV['ARTIFACTS']
  run_command(*fetch_command)
end

581
def box?
582
  !capture_vagrant('box', 'list').grep(/^#{box_name}\s+\(libvirt,/).empty?
583
end
584

anonym's avatar
anonym committed
585
586
def domain_name
  "#{box_name}_default"
587
end
588

intrigeri's avatar
intrigeri committed
589
590
# XXX: giving up on a few worst offenders for now
# rubocop:disable Metrics/AbcSize
591
# rubocop:disable Metrics/MethodLength
592
def clean_up_builder_vms
593
  libvirt = Libvirt.open('qemu:///system')
594

intrigeri's avatar
intrigeri committed
595
  clean_up_domain = proc do |domain|
596
    next if domain.nil?
597

598
    domain.destroy if domain.active?
599
    domain.undefine
600
    begin
601
      libvirt
602
603
604
605
606
607
        .lookup_storage_pool_by_name('default')
        .lookup_volume_by_name("#{domain.name}.img")
        .delete
    rescue Libvirt::RetrieveError
      # Expected if the pool or disk does not exist
    end
608
  end
609

610
  # Let's ensure that the VM we are about to create is cleaned up ...
611
  previous_domain = libvirt.list_all_domains.find { |d| d.name == domain_name }
612
  if previous_domain&.active?
613
    begin
614
      run_vagrant_ssh('mountpoint -q /var/cache/apt-cacher-ng')
615
616
617
    rescue VagrantCommandError
    # Nothing to unmount.
    else
618
619
620
      run_vagrant_ssh('sudo systemctl stop apt-cacher-ng.service')
      run_vagrant_ssh('sudo umount /var/cache/apt-cacher-ng')
      run_vagrant_ssh('sudo sync')
621
    end
622
    begin
623
      run_vagrant_ssh('mountpoint -q /var/cache/tails-website')
624
625
626
    rescue VagrantCommandError
    # Nothing to unmount.
    else
627
628
      run_vagrant_ssh('sudo umount /var/cache/tails-website')
      run_vagrant_ssh('sudo sync')
629
    end
630
  end
631
632
633
634
635
636
637
  clean_up_domain.call(previous_domain)

  # ... and the same for any residual VM based on another box (=>
  # another domain name) that Vagrant still keeps track of.
  old_domain =
    begin
      old_domain_uuid =
638
        open('vagrant/.vagrant/machines/default/libvirt/id', 'r', &:read)
639
        .strip
640
      libvirt.lookup_domain_by_uuid(old_domain_uuid)
641
642
643
644
645
646
647
648
649
650
    rescue Errno::ENOENT, Libvirt::RetrieveError
      # Expected if we don't have vagrant/.vagrant, or if the VM was
      # undefined for other reasons (e.g. manually).
      nil
    end
  clean_up_domain.call(old_domain)

  # We could use `vagrant destroy` here but due to vagrant-libvirt's
  # upstream issue #746 we then risk losing the apt-cacher-ng data.
  # Since we essentially implement `vagrant destroy` without this bug
intrigeri's avatar
intrigeri committed
651
  # above, but in a way so it works even if `vagrant/.vagrant` does
anonym's avatar
anonym committed
652
  # not exist, let's just do what is safest, i.e. avoiding `vagrant
653
654
655
  # destroy`. For details, see the upstream issue:
  #   https://github.com/vagrant-libvirt/vagrant-libvirt/issues/746
  FileUtils.rm_rf('vagrant/.vagrant')
656
ensure
657
  libvirt.close
Tails developers's avatar
Tails developers committed
658
end
intrigeri's avatar
intrigeri committed
659
# rubocop:enable Metrics/AbcSize
660
# rubocop:enable Metrics/MethodLength
661

662
desc 'Remove all libvirt volumes named tails-builder-* (run at your own risk!)'
663
task :clean_up_libvirt_volumes do
664
  libvirt = Libvirt.open('qemu:///system')
665
  begin
666
    pool = libvirt.lookup_storage_pool_by_name('default')
667
668
669
  rescue Libvirt::RetrieveError
    # Expected if the pool does not exist
  else
intrigeri's avatar
intrigeri committed
670
    pool.list_volumes.each do |disk|
intrigeri's avatar
intrigeri committed
671
      next unless /^tails-builder-/.match(disk)
672

intrigeri's avatar
intrigeri committed
673
674
675
676
      begin
        pool.lookup_volume_by_name(disk).delete
      rescue Libvirt::RetrieveError
        # Expected if the disk does not exist
677
678
679
      end
    end
  ensure
680
    libvirt.close
681
682
683
  end
end

684
def on_jenkins?
685
  !ENV['JENKINS_URL'].nil?
686
end
687

anonym's avatar
anonym committed
688
desc 'Clean up all build related files'
689
task clean_all: ['vm:destroy', 'basebox:clean_all']
690
691
692

namespace :vm do
  desc 'Start the build virtual machine'
693
  task up: [
694
695
696
697
698
    'parse_build_options',
    'validate_http_proxy',
    'setup_environment',
    'basebox:create',
  ] do
anonym's avatar
anonym committed
699
    case vm_state
700
    when :not_created
701
      clean_up_builder_vms
702
    end
703
    begin
704
      run_vagrant('up', '--provision')
705
706
707
    rescue VagrantCommandError => e
      clean_up_builder_vms if $force_cleanup
      raise e
708
709
710
    end
  end

711
712
713
714
715
  desc 'SSH into the builder VM'
  task :ssh do
    run_vagrant('ssh')
  end

716
717
  desc 'Stop the build virtual machine'
  task :halt do
718
    run_vagrant('halt')
719
720
721
  end

  desc 'Re-run virtual machine setup'
722
  task provision: [
723
724
725
726
    'parse_build_options',
    'validate_http_proxy',
    'setup_environment',
  ] do
727
    run_vagrant('provision')
728
729
  end

730
  desc 'Destroy build virtual machine (clean up all files except the ' \
731
       "vmproxy's apt-cacher-ng data and the website cache)"
732
  task :destroy do
733
    clean_up_builder_vms
734
735
  end
end
736
737

namespace :basebox do
738
  desc 'Create and import the base box unless already done'
739
  task :create do
740
    next if box?
741

742
    warn <<-END_OF_MESSAGE.gsub(/^      /, '')
743

744
745
746
747
      This is the first time we are using this Vagrant base box so we
      will have to bootstrap by building it from scratch. This will
      take around 20 minutes (depending on your hardware) plus the
      time needed for downloading around 250 MiB of Debian packages.
748
749

    END_OF_MESSAGE
750
    box_dir = VAGRANT_PATH + '/definitions/tails-builder'
751
    run_command("#{box_dir}/generate-tails-builder-box.sh")
752
753
    # Let's use an absolute path since run_vagrant changes the working
    # directory but File.delete doesn't
754
    box_path = "#{box_dir}/#{box_name}.box"
755
756
    run_vagrant('box', 'add', '--name', box_name, box_path)
    File.delete(box_path)
757
  end
758

759
760
761
  def basebox_date(box)
    Date.parse(/^tails-builder-[^-]+-[^-]+-(\d{8})/.match(box)[1])
  end
762

763
  def baseboxes
intrigeri's avatar
Linting    
intrigeri committed
764
765
766
767
    capture_vagrant('box', 'list')
      .first.lines
      .grep(/^tails-builder-.*/)
      .map { |x| x.chomp.sub(/\s.*$/, '') }
768
  end
769

770
771
772
  def clean_up_basebox(box)
    run_vagrant('box', 'remove', '--force', box)
    begin
773
774
      libvirt = Libvirt.open('qemu:///system')
      libvirt
775
776
777
778
779
780
        .lookup_storage_pool_by_name('default')
        .lookup_volume_by_name("#{box}_vagrant_box_image_0.img")
        .delete
    rescue Libvirt::RetrieveError
      # Expected if the pool or disk does not exist
    ensure
781
      libvirt.close
782
783
    end
  end
784

785
786
  desc 'Remove all base boxes'
  task :clean_all do
787
    baseboxes.each { |box| clean_up_basebox(box) }
788
789
  end

790
791
  desc 'Remove all base boxes older than six months'
  task :clean_old do
792
    boxes = baseboxes
793
    # We always want to keep the newest basebox
794
    boxes.sort! { |a, b| basebox_date(a) <=> basebox_date(b) }
795
796
    boxes.pop
    boxes.each do |box|
797
      clean_up_basebox(box) if basebox_date(box) < Date.today - 365.0 / 2.0
798
799
    end
  end
800
end