Rakefile 12.7 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# -*- mode: ruby -*-
# vi: set ft=ruby :
#
# Tails: The Amnesic Incognito Live System
# Copyright © 2012 Tails developers <tails@boum.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

anonym's avatar
anonym committed
21
require 'open3'
22
require 'rbconfig'
23
require 'uri'
24

25
require_relative 'vagrant/lib/tails_build_settings'
26

27
28
29
# Path to the directory which holds our Vagrantfile
VAGRANT_PATH = File.expand_path('../vagrant', __FILE__)

30
31
32
# Branches that are considered 'stable' (used to select SquashFS compression)
STABLE_BRANCH_NAMES = ['stable', 'testing']

Tails developers's avatar
Tails developers committed
33
# Environment variables that will be exported to the build script
34
EXPORTED_VARIABLES = ['http_proxy', 'MKSQUASHFS_OPTIONS', 'TAILS_RAM_BUILD', 'TAILS_CLEAN_BUILD', 'TAILS_OFFLINE_MODE']
Tails developers's avatar
Tails developers committed
35

36
37
38
# Let's save the http_proxy set before playing with it
EXTERNAL_HTTP_PROXY = ENV['http_proxy']

39
# In-VM proxy URL
anonym's avatar
anonym committed
40
INTERNAL_HTTP_PROXY = "http://#{VIRTUAL_MACHINE_HOSTNAME}:3142"
41

42
43
44
class VagrantCommandError < StandardError
end

anonym's avatar
anonym committed
45
46
# Runs the vagrant command, letting stdout/stderr through. Throws an
# exception unless the vagrant command succeeds.
anonym's avatar
anonym committed
47
48
def run_vagrant(*args)
  Process.wait Kernel.spawn('vagrant', *args, :chdir => './vagrant')
intrigeri's avatar
intrigeri committed
49
  if $?.exitstatus != 0
50
51
    raise(VagrantCommandError, "'vagrant #{args}' command failed: " +
                               "#{$?.exitstatus}")
intrigeri's avatar
intrigeri committed
52
  end
53
54
end

anonym's avatar
anonym committed
55
# Runs the vagrant command, not letting stdout/stderr through, and
anonym's avatar
anonym committed
56
# returns [stdout, stderr, Preocess:Status].
anonym's avatar
anonym committed
57
def capture_vagrant(*args)
58
59
60
  stdout, stderr, proc_status =
    Open3.capture3('vagrant', *args, :chdir => './vagrant')
  if proc_status.exitstatus != 0
61
62
    raise(VagrantCommandError, "'vagrant #{args}' command failed: " +
                               "#{proc_status.exitstatus}")
63
64
  end
  return stdout, stderr
65
66
end

anonym's avatar
anonym committed
67
def vagrant_ssh_config(key)
anonym's avatar
anonym committed
68
  # Cache results
anonym's avatar
anonym committed
69
70
71
72
73
74
75
76
77
78
  if $vagrant_ssh_config.nil?
    $vagrant_ssh_config = capture_vagrant('ssh-config').first.split("\n") \
                           .map { |line| line.strip.split(/\s+/, 2) } .to_h
    # The path in the ssh-config output is quoted, which is not what
    # is expected outside of a shell, so let's get rid of the quotes.
    $vagrant_ssh_config['IdentityFile'].gsub!(/^"|"$/, '')
  end
  $vagrant_ssh_config[key]
end

anonym's avatar
anonym committed
79
80
def current_vm_cpus
  capture_vagrant('ssh', '-c', 'grep -c "^processor\s*:" /proc/cpuinfo').first.chomp.to_i
81
82
end

anonym's avatar
anonym committed
83
def vm_state
84
  out, _ = capture_vagrant('status')
anonym's avatar
anonym committed
85
86
87
88
89
90
91
  status_line = out.split("\n")[2]
  if    status_line['not created']
    return :not_created
  elsif status_line['shutoff']
    return :poweroff
  elsif status_line['running']
    return :running
92
  else
anonym's avatar
anonym committed
93
    raise "could not determine VM state"
94
95
96
  end
end

97
def enough_free_host_memory_for_ram_build?
98
99
100
101
102
103
104
105
106
107
  return false unless RbConfig::CONFIG['host_os'] =~ /linux/i

  begin
    usable_free_mem = `free`.split[16].to_i
    usable_free_mem > VM_MEMORY_FOR_RAM_BUILDS * 1024
  rescue
    false
  end
end

108
def free_vm_memory
109
  capture_vagrant('ssh', '-c', 'free').first.chomp.split[16].to_i
110
111
112
113
114
115
116
end

def enough_free_vm_memory_for_ram_build?
  free_vm_memory > BUILD_SPACE_REQUIREMENT * 1024
end

def enough_free_memory_for_ram_build?
anonym's avatar
anonym committed
117
  if vm_state == :running
118
119
120
121
122
123
    enough_free_vm_memory_for_ram_build?
  else
    enough_free_host_memory_for_ram_build?
  end
end

124
def is_release?
anonym's avatar
anonym committed
125
126
127
128
  detached_head = `git symbolic-ref HEAD` == ""
  `git describe --tags --exact-match HEAD 2>/dev/null`
  is_tag = $?.success?
  detached_head && is_tag
129
130
end

131
132
133
134
135
136
137
138
139
140
def system_cpus
  return nil unless RbConfig::CONFIG['host_os'] =~ /linux/i

  begin
    File.read('/proc/cpuinfo').scan(/^processor\s+:/).count
  rescue
    nil
  end
end

141
task :parse_build_options do
142
  options = []
143

144
  # Default to in-memory builds if there is enough RAM available
145
  options << 'ram' if enough_free_memory_for_ram_build?
146

147
  # Default to build using the in-VM proxy
148
  options << 'vmproxy'
149

150
  # Default to fast compression on development branches
151
  options << 'gzipcomp' unless is_release?
152

153
154
  # Default to the number of system CPUs when we can figure it out
  cpus = system_cpus
155
  options << "cpus=#{cpus}" if cpus
156

157
  options += ENV['TAILS_BUILD_OPTIONS'].split if ENV['TAILS_BUILD_OPTIONS']
158
159

  # Make sure release builds are clean
160
  options << 'cleanall' if is_release?
161

162
  options.uniq.each do |opt|
163
    case opt
164
165
166
167
168
    # Memory build settings
    when 'ram'
      ENV['TAILS_RAM_BUILD'] = '1'
    when 'noram'
      ENV['TAILS_RAM_BUILD'] = nil
169
    # Bootstrap cache settings
170
171
172
173
174
    # HTTP proxy settings
    when 'extproxy'
      abort "No HTTP proxy set, but one is required by TAILS_BUILD_OPTIONS. Aborting." unless EXTERNAL_HTTP_PROXY
      ENV['http_proxy'] = EXTERNAL_HTTP_PROXY
    when 'vmproxy'
anonym's avatar
anonym committed
175
      ENV['http_proxy'] = INTERNAL_HTTP_PROXY
176
177
    when 'noproxy'
      ENV['http_proxy'] = nil
178
179
    when 'offline'
      ENV['TAILS_OFFLINE_MODE'] = '1'
180
181
    # SquashFS compression settings
    when 'gzipcomp'
182
      ENV['MKSQUASHFS_OPTIONS'] = '-comp gzip -Xcompression-level 1'
183
184
    when 'defaultcomp'
      ENV['MKSQUASHFS_OPTIONS'] = nil
185
186
187
    # Clean-up settings
    when 'cleanall'
      ENV['TAILS_CLEAN_BUILD'] = '1'
188
189
190
    # Virtual CPUs settings
    when /cpus=(\d+)/
      ENV['TAILS_BUILD_CPUS'] = $1
191
192
193
    # Git settings
    when 'ignorechanges'
      ENV['TAILS_BUILD_IGNORE_CHANGES'] = '1'
194
195
    when 'noprovision'
      ENV['TAILS_NO_AUTO_PROVISION'] = '1'
196
197
    else
      raise "Unknown Tails build option '#{opt}'"
198
199
    end
  end
200
201
202
203
204
205
206
207
208

  if ENV['TAILS_OFFLINE_MODE'] == '1'
    if ENV['http_proxy'].nil?
      abort "You must use a caching proxy when building offline"
    end
    if ENV['TAILS_NO_AUTO_PROVISION'] == '1'
      abort "Offline mode requires provisioning"
    end
  end
209
210
211
end

task :ensure_clean_repository do
212
213
  git_status = `git status --porcelain`
  unless git_status.empty?
214
215
216
    if ENV['TAILS_BUILD_IGNORE_CHANGES']
      $stderr.puts <<-END_OF_MESSAGE.gsub(/^        /, '')

hybridwipe's avatar
hybridwipe committed
217
        You have uncommitted changes in the Git repository. They will
218
219
        be ignored for the upcoming build:
        #{git_status}
220
221
222
223
224

      END_OF_MESSAGE
    else
      $stderr.puts <<-END_OF_MESSAGE.gsub(/^        /, '')

hybridwipe's avatar
hybridwipe committed
225
        You have uncommitted changes in the Git repository. Due to limitations
226
227
        of the build system, you need to commit them before building Tails:
        #{git_status}
228
229
230
231
232
233

        If you don't care about those changes and want to build Tails nonetheless,
        please add `ignorechanges` to the TAILS_BUILD_OPTIONS environment
        variable.

      END_OF_MESSAGE
hybridwipe's avatar
hybridwipe committed
234
      abort 'Uncommitted changes. Aborting.'
235
236
237
238
    end
  end
end

anonym's avatar
anonym committed
239
240
241
242
243
def list_artifacts
  user = vagrant_ssh_config('User')
  stdout = capture_vagrant('ssh', '-c', "find '/home/#{user}/' -maxdepth 1 " +
                                        "-name 'tails-*.iso*'").first
  stdout.split("\n")
244
245
rescue VagrantCommandError
  return Array.new
anonym's avatar
anonym committed
246
247
248
end

def remove_artifacts
249
250
251
  list_artifacts.each do |artifact|
    run_vagrant('ssh', '-c', "sudo rm -f '#{artifact}'")
  end
anonym's avatar
anonym committed
252
253
end

254
255
desc "Make sure the vagrant user's home directory has no undesired artifacts"
task :ensure_clean_home_directory => ['vm:up'] do
anonym's avatar
anonym committed
256
  remove_artifacts
257
258
end

259
260
261
262
task :validate_http_proxy do
  if ENV['http_proxy']
    proxy_host = URI.parse(ENV['http_proxy']).host

263
264
265
266
267
268
    if proxy_host.nil?
      ENV['http_proxy'] = nil
      $stderr.puts "Ignoring invalid HTTP proxy."
      return
    end

269
270
271
272
273
274
275
276
277
278
    if ['localhost', '[::1]'].include?(proxy_host) || proxy_host.start_with?('127.0.0.')
      abort 'Using an HTTP proxy listening on the loopback is doomed to fail. Aborting.'
    end

    $stderr.puts "Using HTTP proxy: #{ENV['http_proxy']}"
  else
    $stderr.puts "No HTTP proxy set."
  end
end

Tails developers's avatar
Tails developers committed
279
desc 'Build Tails'
280
task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_home_directory', 'validate_http_proxy', 'vm:up'] do
281

282
  if ENV['TAILS_RAM_BUILD'] && not(enough_free_memory_for_ram_build?)
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
    $stderr.puts <<-END_OF_MESSAGE.gsub(/^      /, '')

      The virtual machine is not currently set with enough memory to
      perform an in-memory build. Either remove the `ram` option from
      the TAILS_BUILD_OPTIONS environment variable, or shut the
      virtual machine down using `rake vm:halt` before trying again.

    END_OF_MESSAGE
    abort 'Not enough memory for the virtual machine to run an in-memory build. Aborting.'
  end

  if ENV['TAILS_BUILD_CPUS'] && current_vm_cpus != ENV['TAILS_BUILD_CPUS'].to_i
    $stderr.puts <<-END_OF_MESSAGE.gsub(/^      /, '')

      The virtual machine is currently running with #{current_vm_cpus}
      virtual CPU(s). In order to change that number, you need to
      stop the VM first, using `rake vm:halt`. Otherwise, please
      adjust the `cpus` options accordingly.

    END_OF_MESSAGE
    abort 'The virtual machine needs to be reloaded to change the number of CPUs. Aborting.'
  end

306
307
308
309
310
  # Let's make sure that, unless you know what you are doing and
  # explicitly disable this, we always provision in order to ensure
  # a valid, up-to-date build system.
  run_vagrant('provision') unless ENV['TAILS_NO_AUTO_PROVISION']

Tails developers's avatar
Tails developers committed
311
  exported_env = EXPORTED_VARIABLES.select { |k| ENV[k] }.
anonym's avatar
anonym committed
312
                 collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
313
  run_vagrant('ssh', '-c', "#{exported_env} build-tails")
314

anonym's avatar
anonym committed
315
  artifacts = list_artifacts
anonym's avatar
anonym committed
316
317
318
319
320
321
322
  raise 'No build artifacts was found!' if artifacts.empty?
  user     = vagrant_ssh_config('User')
  hostname = vagrant_ssh_config('HostName')
  key_file = vagrant_ssh_config('IdentityFile')
  $stderr.puts "Retrieving artifacts from Vagrant build box."
  artifacts.each do |artifact|
    run_vagrant('ssh', '-c', "sudo chown #{user} '#{artifact}'")
323
324
325
326
327
328
329
330
331
332
333
334
    Process.wait(
      Kernel.spawn(
        'scp',
        '-i', key_file,
        # We need this since the user will not necessarily have a
        # known_hosts entry. It is safe since an attacker must
        # compromise libvirt's network config or the user running the
        # command to modify the #{hostname} below.
        '-o', 'StrictHostKeyChecking=no',
        "#{user}@#{hostname}:#{artifact}", '.'
      )
    )
anonym's avatar
anonym committed
335
    raise "Failed to fetch artifact '#{artifact}'" unless $?.success?
336
  end
anonym's avatar
anonym committed
337
  remove_artifacts
Tails developers's avatar
Tails developers committed
338
339
end

340
341
namespace :vm do
  desc 'Start the build virtual machine'
342
  task :up => ['parse_build_options', 'validate_http_proxy'] do
anonym's avatar
anonym committed
343
    case vm_state
344
    when :not_created
345
      # Do not use non-existant in-VM proxy to download the basebox
anonym's avatar
anonym committed
346
      if ENV['http_proxy'] == INTERNAL_HTTP_PROXY
347
348
349
350
        ENV['http_proxy'] = nil
        restore_internal_proxy = true
      end

351
      $stderr.puts <<-END_OF_MESSAGE.gsub(/^        /, '')
352
353
354
355
356
357

        This is the first time that the Tails builder virtual machine is
        started. The virtual machine template is about 300 MB to download,
        so the process might take some time.

        Please remember to shut the virtual machine down once your work on
358
        Tails is done:
359
360
361
362
363

            $ rake vm:halt

      END_OF_MESSAGE
    when :poweroff
364
      $stderr.puts <<-END_OF_MESSAGE.gsub(/^        /, '')
365
366

        Starting Tails builder virtual machine. This might take a short while.
367
        Please remember to shut it down once your work on Tails is done:
368
369
370
371
372

            $ rake vm:halt

      END_OF_MESSAGE
    end
373
    run_vagrant('up')
anonym's avatar
anonym committed
374
    ENV['http_proxy'] = INTERNAL_HTTP_PROXY if restore_internal_proxy
375
376
  end

377
378
379
380
381
  desc 'SSH into the builder VM'
  task :ssh do
    run_vagrant('ssh')
  end

382
383
  desc 'Stop the build virtual machine'
  task :halt do
384
    run_vagrant('halt')
385
386
387
  end

  desc 'Re-run virtual machine setup'
388
  task :provision => ['parse_build_options', 'validate_http_proxy'] do
389
    run_vagrant('provision')
390
391
392
393
  end

  desc 'Destroy build virtual machine (clean up all files)'
  task :destroy do
394
    run_vagrant('destroy', '--force')
395
396
  end
end
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425

namespace :basebox do

  desc 'Generate a new base box'
  task :create do
    box_dir = VAGRANT_PATH + '/definitions/tails-builder'
    Dir.chdir(box_dir) do
      `./generate-tails-builder-box.sh`
      raise 'Base box generation failed!' unless $?.success?
    end
    box = Dir.glob("#{box_dir}/*.box").sort_by {|f| File.mtime(f) } .last
    $stderr.puts <<-END_OF_MESSAGE.gsub(/^      /, '')

      You have successfully generated a new Vagrant base box:

          #{box}

      To install the new base box, please run:

          $ vagrant box add #{box}

      To actually make Tails build using this base box, the `config.vm.box` key
      in `vagrant/Vagrantfile` has to be updated. Please check the documentation
      for details.

    END_OF_MESSAGE
  end

end