Skip to content

GitLab

Commit 7913474e authored by Ulrike Uhlig's avatar Ulrike Uhlig
Browse files

Merge remote-tracking branch 'tails/master'

parents 0d13f5b0 d5868241
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 5172 additions and 4930 deletions
.gitignore
View file @ 7913474e
......@@ -49,6 +49,7 @@
/config/chroot_local-includes/usr/share/applications/tor-browser.desktop
/config/chroot_local-includes/usr/share/applications/tails-about.desktop
/config/chroot_local-includes/usr/share/desktop-directories/Tails.directory
/config/chroot_local-includes/usr/share/polkit-1/actions/org.boum.tails.root-terminal.policy
/tmp/
# The test suite's local configuration files
......
.gitmodules
View file @ 7913474e
......@@ -11,6 +11,3 @@
[submodule "submodules/mirror-pool-dispatcher"]
path = submodules/mirror-pool-dispatcher
url = https://git-tails.immerda.ch/mirror-pool-dispatcher
[submodule "submodules/gnome-shell-extension-florence-indicator"]
path = submodules/gnome-shell-extension-florence-indicator
url = https://github.com/UshakovVasilii/gnome-shell-extension-florence-indicator.git
Rakefile
View file @ 7913474e
......@@ -34,6 +34,7 @@ STABLE_BRANCH_NAMES = ['stable', 'testing']
EXPORTED_VARIABLES = [
'MKSQUASHFS_OPTIONS',
'TAILS_BUILD_FAILURE_RESCUE',
'TAILS_DATE_OFFSET',
'TAILS_MERGE_BASE_BRANCH',
'TAILS_OFFLINE_MODE',
......@@ -161,7 +162,7 @@ def enough_free_host_memory_for_ram_build?
return false unless RbConfig::CONFIG['host_os'] =~ /linux/i
begin
usable_free_mem = `free`.split[16].to_i
usable_free_mem = `free`.split[12].to_i
usable_free_mem > VM_MEMORY_FOR_RAM_BUILDS * 1024
rescue
false
......@@ -169,7 +170,7 @@ def enough_free_host_memory_for_ram_build?
end
def free_vm_memory
capture_vagrant_ssh('free').first.chomp.split[16].to_i
capture_vagrant_ssh('free').first.chomp.split[12].to_i
end
def enough_free_vm_memory_for_ram_build?
......@@ -368,8 +369,9 @@ task :setup_environment => ['validate_git_state'] do
ENV['BASE_BRANCH_GIT_COMMIT'] = git_helper('git_base_branch_head')
['GIT_COMMIT', 'GIT_REF', 'BASE_BRANCH_GIT_COMMIT'].each do |var|
if ENV[var].empty?
raise "Variable '#{var}' is empty, which should not be possible" +
"(validate_git_state must be buggy)"
raise "Variable '#{var}' is empty, which should not be possible: " +
"either validate_git_state is buggy or the 'origin' remote " +
"does not point to the official Tails Git repository."
end
end
end
......
auto/build
View file @ 7913474e
#!/bin/bash
set -e
set -u
set -x
. "$(dirname $0)/scripts/utils.sh"
# get $BUILD_BASENAME
. tmp/build_environment
umask 022
### functions
......@@ -61,7 +66,6 @@ chown 0:0 config/chroot_local-includes/etc/resolv.conf
chmod -R go+rX config/binary_local-includes/
chmod -R go+rX config/chroot_local-includes/etc
chmod 0440 config/chroot_local-includes/etc/sudoers.d/*
chmod go+rX config/chroot_local-includes/home
chmod go+rX config/chroot_local-includes/lib
chmod go+rX config/chroot_local-includes/lib/live
chmod -R go+rx config/chroot_local-includes/lib/live/config
......@@ -71,22 +75,25 @@ chmod go+rX config/chroot_local-includes/live
chmod -R go+rX config/chroot_local-includes/usr
chmod -R go+rx config/chroot_local-includes/usr/local/bin
chmod -R go+rx config/chroot_local-includes/usr/local/sbin
chmod -R go+rX config/chroot_local-includes/usr/share/doc/tails
chmod -R go+rX config/chroot_local-includes/usr/share/doc
chmod -R go+rX config/chroot_local-includes/var
chmod -R go+rX config/chroot_apt
chmod -R go+rX config/chroot_sources
# normalize file timestamps
find config/binary_local-includes config/chroot_local-includes \
-exec touch --date="@$SOURCE_DATE_EPOCH" '{}' \;
find \
config/binary_local-includes \
config/chroot_local-includes \
wiki/src \
-exec touch --date="@$SOURCE_DATE_EPOCH" '{}' \;
# build the image
# we need /debootstrap/deburis to build a manifest of used packages:
DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keep-debootstrap-dir"
DEBOOTSTRAP_OPTIONS="${DEBOOTSTRAP_OPTIONS:-} --keep-debootstrap-dir"
# we're not ready for merged-/usr yet: Debian#843461, Tails#11903
DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --no-merged-usr"
DEBOOTSTRAP_OPTIONS="${DEBOOTSTRAP_OPTIONS:-} --no-merged-usr"
# use our own APT repository's key:
DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
......@@ -107,49 +114,6 @@ export DEBOOTSTRAP_OPTIONS
MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS} -wildcards -ef chroot/usr/share/amnesia/build/mksquashfs-excludes"
export MKSQUASHFS_OPTIONS
# get git branch or tag so we can set the basename appropriately, i.e.:
# * if we build from a tag: tails-$ARCH-$TAG.iso
# * otherwise: tails-$ARCH-$BRANCH-$VERSION-$TIME-$COMMIT.iso
GIT_BRANCH="$(git_current_branch)"
if [ -n "${GIT_BRANCH}" ]; then
CLEAN_GIT_BRANCH=$(echo "$GIT_BRANCH" | sed 's,/,_,g')
GIT_SHORT_ID="$(git_current_commit --short)"
BUILD_BASENAME="tails-${LB_ARCHITECTURE}-${CLEAN_GIT_BRANCH}-${AMNESIA_VERSION}-${AMNESIA_NOW}-${GIT_SHORT_ID}"
else
if git_on_a_tag; then
CLEAN_GIT_TAG=$(git_current_tag | tr '/-' '_~')
BUILD_BASENAME="tails-${LB_ARCHITECTURE}-${CLEAN_GIT_TAG}"
else
# this shouldn't reasonably happen (e.g. only if you checkout a
# tag, remove the tag and then build)
fatal "Neither a Git branch nor a tag, exiting."
fi
fi
GIT_BASE_BRANCH=$(base_branch) \
|| fatal "GIT_BASE_BRANCH could not be guessed."
if [ "${TAILS_MERGE_BASE_BRANCH}" = 1 ] && \
! git_on_a_tag && [ "$GIT_BRANCH" != "$GIT_BASE_BRANCH" ] ; then
GIT_BASE_BRANCH_COMMIT=$(git_base_branch_head)
[ -n "${GIT_BASE_BRANCH_COMMIT}" ] \
|| fatal "Base branch's top commit could not be guessed."
echo "Merging base branch origin/${GIT_BASE_BRANCH}"
echo "(at commit ${GIT_BASE_BRANCH_COMMIT})..."
git merge --no-edit "origin/${GIT_BASE_BRANCH}" \
|| fatal "Failed to merge base branch."
git submodule update --init
# Adjust BUILD_BASENAME to embed the base branch name and its top commit
CLEAN_GIT_BASE_BRANCH=$(echo "$GIT_BASE_BRANCH" | sed 's,/,_,g')
GIT_BASE_BRANCH_SHORT_ID=$(git_base_branch_head --short)
[ -n "${GIT_BASE_BRANCH_SHORT_ID}" ] \
|| fatal "Base branch's top commit short ID could not be guessed."
BUILD_BASENAME="${BUILD_BASENAME}+${CLEAN_GIT_BASE_BRANCH}"
BUILD_BASENAME="${BUILD_BASENAME}@${GIT_BASE_BRANCH_SHORT_ID}"
fi
# build the doc wiki
./build-website
......
auto/clean
View file @ 7913474e
#!/bin/sh
set -e
set -u
set -x
for dir in chroot/{dev/pts,proc,sys,var/lib/dpkg} ; do
......
auto/config
View file @ 7913474e
#! /bin/sh
# automatically run by "lb config"
set -e
set -u
set -x
. "$(dirname $0)/scripts/utils.sh"
# we require building from git
if ! git rev-parse --is-inside-work-tree; then
echo "${PWD} is not a Git tree. Exiting."
......@@ -14,6 +18,65 @@ if [ -e config/amnesia.local ] ; then
. config/amnesia.local
fi
if [ -n "${SOURCE_DATE_EPOCH}" ]; then
CURRENT_EPOCH="$(date --utc +%s)"
if [ "${SOURCE_DATE_EPOCH}" -gt "${CURRENT_EPOCH}" ]; then
echo "SOURCE_DATE_EPOCH is set before the current time. Exiting."
exit 1
fi
else
echo "SOURCE_DATE_EPOCH is not set. Exiting."
exit 1
fi
# get git branch or tag so we can set the basename appropriately, i.e.:
# * if we build from a tag: tails-$ARCH-$TAG.iso
# * otherwise: tails-$ARCH-$BRANCH-$VERSION-$TIME-$COMMIT.iso
GIT_BRANCH="$(git_current_branch)"
if [ -n "${GIT_BRANCH}" ]; then
CLEAN_GIT_BRANCH=$(echo "$GIT_BRANCH" | sed 's,/,_,g')
GIT_SHORT_ID="$(git_current_commit --short)"
BUILD_BASENAME="tails-amd64-${CLEAN_GIT_BRANCH}-${AMNESIA_VERSION}-${AMNESIA_NOW}-${GIT_SHORT_ID}"
else
if git_on_a_tag; then
CLEAN_GIT_TAG=$(git_current_tag | tr '/-' '_~')
BUILD_BASENAME="tails-amd64-${CLEAN_GIT_TAG}"
else
# this shouldn't reasonably happen (e.g. only if you checkout a
# tag, remove the tag and then build)
fatal "Neither a Git branch nor a tag, exiting."
fi
fi
GIT_BASE_BRANCH=$(base_branch) \
|| fatal "GIT_BASE_BRANCH could not be guessed."
if [ "${TAILS_MERGE_BASE_BRANCH:-}" = 1 ] && \
! git_on_a_tag && [ "$GIT_BRANCH" != "$GIT_BASE_BRANCH" ] ; then
GIT_BASE_BRANCH_COMMIT=$(git_base_branch_head)
[ -n "${GIT_BASE_BRANCH_COMMIT}" ] \
|| fatal "Base branch's top commit could not be guessed."
echo "Merging base branch origin/${GIT_BASE_BRANCH}"
echo "(at commit ${GIT_BASE_BRANCH_COMMIT})..."
faketime -f "${SOURCE_DATE_FAKETIME}" \
git merge --no-edit "origin/${GIT_BASE_BRANCH}" \
|| fatal "Failed to merge base branch."
git submodule update --init
# Adjust BUILD_BASENAME to embed the base branch name and its top commit
CLEAN_GIT_BASE_BRANCH=$(echo "$GIT_BASE_BRANCH" | sed 's,/,_,g')
GIT_BASE_BRANCH_SHORT_ID=$(git_base_branch_head --short)
[ -n "${GIT_BASE_BRANCH_SHORT_ID}" ] \
|| fatal "Base branch's top commit short ID could not be guessed."
BUILD_BASENAME="${BUILD_BASENAME}+${CLEAN_GIT_BASE_BRANCH}"
BUILD_BASENAME="${BUILD_BASENAME}@${GIT_BASE_BRANCH_SHORT_ID}"
fi
# save variables that lb build needs
mkdir -p tmp
echo "BUILD_BASENAME='${BUILD_BASENAME}'" > tmp/build_environment
# sanity checks
if grep -qs -E '^Pin:\s+release\s+.*a=' config/chroot_apt/preferences ; then
echo "Found unsupported a= syntax in config/chroot_apt/preferences,"
......@@ -86,7 +149,6 @@ $RUN_LB_CONFIG \
--packages-lists="standard" \
--tasks="standard" \
--linux-packages="linux-image-${KERNEL_VERSION}" \
--security false \
--syslinux-menu vesamenu \
--syslinux-splash data/splash.png \
--syslinux-timeout 4 \
......@@ -123,8 +185,10 @@ cp debian/changelog config/chroot_local-includes/usr/share/doc/amnesia/Changelog
# create readahead-list from squashfs.sort
if [ -e config/binary_rootfs/squashfs.sort ]; then
mkdir -p config/chroot_local-includes/usr/share/amnesia
sort -k2 -n -r config/binary_rootfs/squashfs.sort |
cut -d' ' -f1 > config/chroot_local-includes/usr/share/amnesia/readahead-list
sort -k2 -n -r config/binary_rootfs/squashfs.sort | \
cut -d' ' -f1 | \
grep --invert-match --extended-regexp "$READAHEAD_EXCLUDE_PATTERN" \
> config/chroot_local-includes/usr/share/amnesia/readahead-list
fi
# custom APT sources
......@@ -139,12 +203,6 @@ install -m 0755 \
submodules/mirror-pool-dispatcher/lib/js/mirror-dispatcher.js \
config/chroot_local-includes/usr/local/lib/nodejs/
# gnome-shell-extension-florence-indicator
rm -rf \
config/chroot_local-includes/usr/share/gnome-shell/extensions/florenceIndicator@UshakovVasilii_Github.yahoo.com
cp -a submodules/gnome-shell-extension-florence-indicator/florenceIndicator@UshakovVasilii_Github.yahoo.com/ \
config/chroot_local-includes/usr/share/gnome-shell/extensions/
# custom debootstrap script, setting some APT magic to log downloads:
patch \
--follow-symlinks \
......
auto/scripts/ikiwiki-supported-languages
View file @ 7913474e
......@@ -21,7 +21,7 @@ Tails developers <amnesia@boum.org>
=head1 LICENSE AND COPYRIGHT
Copyright (C) 2011 Tails developers <amnesia@boum.org>
Copyright (C) 2011 Tails developers <tails@boum.org>
Licensed under the GNU GPL version 3 or any later version.
......
bin/doc-impacted-by 0 → 100755
View file @ 7913474e
#!/usr/bin/env ruby
# coding: utf-8
require 'deep_merge'
require 'git'
require 'optparse'
require 'yaml'
require 'test/unit'
Test::Unit.run = true
include Test::Unit::Assertions
# The Ruby Git module we use needs the Git root directory, and this
# prevents it from being able to run the command below.
GIT_DIR = `git rev-parse --show-toplevel`.chomp
assert_equal(0, $?.exitstatus)
DEFAULT_RELATIONSHIP_FILE = "#{GIT_DIR}/doc-source-relationships.yml"
class Object
def arrayify
self.instance_of?(Array) ? self : [self]
end
end
class Array
def glob(glob)
self.select do |e|
e.instance_of?(String) && File.fnmatch(glob, e, File::FNM_EXTGLOB)
end
end
end
def parse_argv!
options = Hash.new
opt_parser = OptionParser.new do |opts|
opts.banner = "Usage: [OPTION]... COMMITISH1 COMMITISH2 MANIFEST1 MANIFEST2"
opts.separator ""
opts.separator "Produces a list of documentation pages that might need " \
"attention due to the changes from COMMITISH1 to " \
"COMMITISH2. The corresponding .build-manifest files must " \
"be passed as MANIFEST1 and MANIFEST2."
opts.separator ""
opts.separator "Example:"
opts.separator " bin/doc-impacted-by 3.0 3.2 " \
"tails-amd64-3.0.build-manifest " \
"tails-amd64-3.2.build-manifest"
opts.separator ""
opts.separator "Options:"
opts.on("-h", "--help", "Show this message") do
puts opts
exit
end
opts.on("-f PATH", "--relationship-file=PATH",
"Use a custom PATH for the doc-source relationship description " +
"file (default: #{File.basename(DEFAULT_RELATIONSHIP_FILE)} in " +
"the Git root)") do |path|
options['relationship-file'] = path
end
opts.on("-s", "--skip-packages", "Skip looking at packages, " +
"only look at Git") do
options['skip-packages'] = true
end
end
parameters = opt_parser.parse(ARGV)
req_nr_parameters = options['skip-packages'] ? 2 : 4
assert_equal(req_nr_parameters, parameters.size,
"You must pass exactly #{req_nr_parameters} parameters")
return [options, parameters]
end
# From a .build-manifest, from its list of packages, generate a
# Hash mapping `package` to a Hash containing the remaining package
# fields from the .build-manifest (e.g. `arch`, `version`).
def read_package_manifest_file_as_package_map(path)
package_manifest = YAML.load(File.read(path))
packages = package_manifest['packages']['binary'] +
package_manifest['packages']['source']
packages.map do |entry|
[
entry['package'],
entry.clone.delete_if { |k, _| k == 'package' }
]
end
.to_h
end
def canonicalize_relationship(orig_entry)
entry = orig_entry.clone
field_abbreviations = {
'file' => 'files',
'package' => 'packages',
'page' => 'pages',
'test' => 'tests',
}
fields = field_abbreviations.values
field_abbreviations.each do |short, long|
next unless entry.has_key?(short)
assert(not(entry.has_key?(long)),
"contains both '#{long}' and its abbreviation '#{short}'")
v = entry[short]
entry.delete(short)
entry[long] = v
end
assert(entry.has_key?('pages'),
"lacks the obligatory 'pages' field")
assert(entry.keys.size > 1,
"entries with only a 'pages' field are meaningless")
# Note: `(a - b).empty?` <==> "a is a subset of b?"
assert((entry.keys - fields).empty?,
"contains invalid fields: #{entry.keys - fields}")
fields.each do |field|
next unless entry.has_key?(field)
entry[field] = entry[field].arrayify
end
return entry
rescue Exception => e
STDERR.puts 'Problematic entry:'
STDERR.puts YAML.dump([orig_entry])
STDERR.puts
raise e
end
# Reads the `relationship_file` and returns a "documentation impact
# map", a Hash which maps all documentation pages to the sources it is
# impacted by.
def read_relationship_file_as_impact_map(relationship_file)
impact_map = Hash.new
relationships = YAML.load(File.read(relationship_file))
relationships.map { |e| canonicalize_relationship(e) } .each do |entry|
entry['pages'].each do |page|
source_files = entry.clone.delete_if { |k, _| k == 'pages' }
impact_map.deep_merge({page => source_files})
end
end
return impact_map
end
# Given the "documentation impact map" and the "old" and "new" state,
# look at the changes between "old" and "new" and find which
# documentation pages are impacted. The return value is a mapping
# from each affected documentation page to the list of "reasons",
# explanations how the sources impact the page.
def find_impacted_docs(impact_map,
old_commit, new_commit,
old_manifest, new_manifest)
git = Git.open(GIT_DIR)
git_diff = git.diff(old_commit, new_commit)
# Create the list of all wiki files, and use it as an approximation
# of all documentation pages. It's a super set, so it only impacts
# performance when we search in it later. Ideally we'd like to do
# something like `git.object(new_commit).path('wiki/src')` but the
# Git module we use seem to not support listing files at a certain
# commit.
git_cmd_wiki_files = "git ls-tree -r --full-tree " +
"--name-only #{new_commit} -- wiki/src"
doc_pages = `#{git_cmd_wiki_files}`.chomp.split("\n")
assert_equal(0, $?.exitstatus, "Error: `git ls-tree` failed")
old_packages = old_manifest.keys
new_packages = new_manifest.keys
removed_packages = old_packages - new_packages
introduced_packages = new_packages - old_packages
updated_packages = (new_packages & old_packages).select do |package|
old_manifest[package] != new_manifest[package]
end
impacted_docs = Hash.new
impact_map.each do |page, sources|
file_paths = []
package_globs = []
test_paths = []
sources.each do |type, source|
case type
when 'packages'
package_globs = source
when 'tests'
test_paths = source.map { |path| "features/#{path}" }
when 'files'
file_paths = source
else
raise "Unknown field '#{type}' in impact map; this should not " +
"happen, and probably means canonicalize_relationship() " +
"is buggy"
end
end
all_source_file_paths = file_paths + test_paths
doc_pages.glob("wiki/src/#{page}.{html,mdwn}").each do |page_path|
all_source_file_paths.each do |source_path|
# Git::Diff#path() alters the object so it cannot be used for a
# successive call for another path.
_git_diff = git_diff.clone
source_path_diff = _git_diff.path(source_path)
if source_path_diff.size > 0
changed_files = source_path_diff.map { |file| file.path }
reasons = changed_files.map do |path|
"Changes in source file: #{path}"
end
impacted_docs.deep_merge({page_path => reasons})
end
end
package_globs.each do |package_glob|
reasons = []
removed_impacted_packages = removed_packages.glob(package_glob)
introduced_impacted_packages = introduced_packages.glob(package_glob)
updated_impacted_packages = updated_packages.glob(package_glob)
reasons += removed_impacted_packages.map do |package|
"Removed package: #{package}"
end
reasons += introduced_impacted_packages.map do |package|
"Introduced package: #{package}"
end
reasons += updated_impacted_packages.map do |package|
old = old_manifest[package]
new = new_manifest[package]
assert_not_equal(
old, new,
"'#{package}' has identical data in both manifests so it is " +
"a bug that we ended up here"
)
package_changes = old_manifest[package].keys.sort.map do |key|
old_val = old[key]
new_val = new[key]
old_val != new_val ? "#{old_val} → #{new_val}" : nil
end
.compact.join(', ')
"Updated package: #{package} (#{package_changes})"
end
impacted_docs.deep_merge({page_path => reasons}) unless reasons.empty?
end
end
end
return impacted_docs
end
# Main
options, parameters = parse_argv!
relationship_file = options['relationship-file'] || DEFAULT_RELATIONSHIP_FILE
old_commit, new_commit, old_manifest_path, new_manifest_path = parameters
impact_map = read_relationship_file_as_impact_map(relationship_file)
if options['skip-packages']
old_manifest = Hash.new
new_manifest = Hash.new
else
old_manifest = read_package_manifest_file_as_package_map(old_manifest_path)
new_manifest = read_package_manifest_file_as_package_map(new_manifest_path)
end
impacted_docs = find_impacted_docs(
impact_map,
old_commit, new_commit,
old_manifest, new_manifest
)
if impacted_docs.size > 0
result =
impacted_docs.sort.map do |page, reasons|
"#{page}\n" +
reasons.sort.map do |reason|
"- #{reason}"
end
.join("\n")
end
.join("\n\n")
puts "The following documentation pages need investigation:"
puts
puts result
end
if options['skip-packages']
STDERR.puts "Warning! The --skip-packages option makes this " +
"report incomplete!"
end
bin/sanity-check-website 0 → 100755
View file @ 7913474e
#!/usr/bin/env ruby
require 'date'
require 'test/unit'
Test::Unit.run = true
include Test::Unit::Assertions
# Force UTF-8. Ruby will default to the system locale, and if it is
# non-UTF-8, String-methods will fail when operating on non-ASCII
# strings.
Encoding.default_external = Encoding::UTF_8
Encoding.default_internal = Encoding::UTF_8
GIT_DIR=`git rev-parse --show-toplevel`
assert_equal(
0, $?.exitstatus,
"Failed to find Tails' Git root; this command must be run " +
"inside Tails Git repo"
)
def rfc2822_date?(date)
!!DateTime.rfc2822(date)
rescue ArgumentError
false
end
def page_meta_date_is_ok?(path)
meta_date_post_re = /^\[\[!meta\s+date="(?<date>.*)"\]\]$/
success = true
content_lines = File.new(path).read.split("\n")
matches = content_lines.grep(meta_date_post_re)
if matches.size != 1
STDERR.puts "#{path}: has #{matches.size} well-formed 'meta date' " +
"directives (must be 1)"
success = false
else
meta_date_line = matches.first
m = meta_date_post_re.match(meta_date_line)
meta_date = m['date']
if not(rfc2822_date?(meta_date))
STDERR.puts "#{path}: 'meta date' directive contains non-rfc2822 " +
"date: #{meta_date}"
success = false
end
end
return success
end
def po_file_meta_date_is_ok?(path)
meta_date_po_re_str = '\[\[!meta\s+date=\\\"(?<date>.*)\\\"\]\]\\\n'
success = true
content_lines = File.new(path).read.split("\n")
matches = content_lines.grep(Regexp.new("^msgid \"#{meta_date_po_re_str}\"$"))
if matches.size != 1
STDERR.puts "#{path}: has #{matches.size} 'meta date' msgid:s (must be 1)"
success = false
else
msgid = matches.first