Commit 0439d624 authored by intrigeri's avatar intrigeri
Browse files

Merge branch 'feature/8654-encode-apt-suite-in-git' into feature/jessie

parents fce8ec29 347473a9
#!/bin/bash
# set -x
set -x
umask 022
......@@ -103,6 +103,34 @@ else
fi
fi
GIT_BASE_BRANCH=$(head -n1 config/base_branch) \
|| fatal "GIT_BASE_BRANCH could not be guessed."
# Merge base branch into the branch being built, iff. we're building
# in Jenkins, and not building from a tag, and not building the base
# branch itself
if [ -n "$JENKINS_URL" ] && [ -z "$GIT_TAG" ] \
&& [ "$GIT_BRANCH" != "$GIT_BASE_BRANCH" ] ; then
GIT_BASE_BRANCH_COMMIT=$(git rev-parse "origin/${GIT_BASE_BRANCH}") \
|| fatal "Base branch's top commit could not be guessed."
cat > .git/info/attributes <<-EOF
config/APT_overlays merge=union
EOF
echo "Merging base branch origin/${GIT_BASE_BRANCH}"
echo "(at commit ${GIT_BASE_BRANCH_COMMIT})..."
git merge --no-edit "origin/${GIT_BASE_BRANCH}" \
|| fatal "Failed to merge base branch."
# Adjust BUILD_BASENAME to embed the base branch name and its top commit
CLEAN_GIT_BASE_BRANCH=$(echo "$GIT_BASE_BRANCH" | sed 's,/,_,g')
GIT_BASE_BRANCH_SHORT_ID=$(git rev-parse --short "origin/${GIT_BASE_BRANCH}") \
|| fatal "Base branch's top commit short ID could not be guessed."
BUILD_BASENAME="${BUILD_BASENAME}+${CLEAN_GIT_BASE_BRANCH}"
BUILD_BASENAME="${BUILD_BASENAME}@${GIT_BASE_BRANCH_SHORT_ID}"
fi
case "$LB_BINARY_IMAGES" in
iso)
BUILD_FILENAME_EXT=iso
......
#!/bin/sh
set -x
for dir in chroot/{dev/pts,proc,sys,var/lib/dpkg} ; do
if mountpoint -q "$dir" ; then
umount "$dir"
......
#! /bin/sh
# automatically run by "lb config"
set -x
# we require building from git
if ! git rev-parse --is-inside-work-tree; then
echo "${PWD} is not a Git tree. Exiting."
......
#!/bin/sh
#!/bin/bash
set -e
APT_MIRROR_URL="http://deb.tails.boum.org/"
DEFAULT_COMPONENTS="main"
BASE_BRANCHES="stable testing devel feature/jessie"
fatal() {
echo "$*" >&2
exit 1
}
git_tag_exists() {
local tag="$1"
......@@ -29,12 +35,30 @@ output_apt_binary_source() {
echo "deb $APT_MIRROR_URL $suite $components"
}
output_overlay_apt_binary_sources() {
for suite in $(cat config/APT_overlays) ; do
output_apt_binary_source "$suite"
done
}
current_branch() {
git branch | awk '/^\* / { print $2 }'
}
on_topic_branch() {
current_branch | grep -qE '^(feature|bug|bugfix)/'
on_base_branch() {
local current_branch=$(current_branch)
for base_branch in $BASE_BRANCHES ; do
if [ "$current_branch" = "$base_branch" ] ; then
return 0
fi
done
return 1
}
base_branch() {
cat config/base_branch | head -n1
}
branch_name_to_suite() {
......@@ -43,19 +67,32 @@ branch_name_to_suite() {
echo "$branch" | sed -e 's,[^.a-z0-9-],-,ig' | tr '[A-Z]' '[a-z]'
}
### Sanity checks
[ -e config/APT_overlays ] || fatal 'config/APT_overlays does not exist'
[ -e config/base_branch ] || fatal 'config/base_branch does not exist'
[ "$(cat config/base_branch | wc -l)" -eq 1 ] \
|| fatal 'config/base_branch must contain exactly one line'
if on_base_branch && ! [ "$(base_branch)" = "$(current_branch)" ] ; then
echo "base_branch: $(base_branch)" >&2
echo "current_branch: $(current_branch)" >&2
fatal "In a base branch, config/base_branch must match the current branch."
fi
cmp --quiet config/APT_overlays <(sort -u config/APT_overlays) \
|| fatal "There are duplicate lines in config/APT_overlays."
### Main
if version_was_released "$(version_in_changelog)"; then
if [ -s config/APT_overlays ]; then
fatal 'config/APT_overlays must be empty while releasing'
fi
output_apt_binary_source "$(branch_name_to_suite "$(version_in_changelog)")"
elif [ "$(current_branch)" = "stable" ]; then
output_apt_binary_source stable
elif [ "$(current_branch)" = "testing" ]; then
output_apt_binary_source testing
elif [ "$(current_branch)" = "experimental" ]; then
output_apt_binary_source experimental
else
output_apt_binary_source devel
output_apt_binary_source "$(branch_name_to_suite "$(base_branch)")"
output_overlay_apt_binary_sources
fi
if on_topic_branch; then
output_apt_binary_source "$(branch_name_to_suite $(current_branch))"
fi
......@@ -4,72 +4,209 @@ Feature: custom APT sources to build branches
the proper APT sources were automatically picked depending
on which Git branch I am working on.
Scenario: build from an untagged stable branch
Given I am working on the stable branch
And last released version mentioned in debian/changelog is 1.0
Scenario: build from an untagged stable branch where config/APT_overlays is empty
Given I am working on the stable base branch
And the last version mentioned in debian/changelog is 1.0
And Tails 1.0 has not been released yet
When I run tails-custom-apt-sources
And config/APT_overlays is empty
When I successfully run tails-custom-apt-sources
Then I should see only the 'stable' suite
Scenario: build from an untagged stable branch where config/APT_overlays is not empty
Given I am working on the stable base branch
And the last version mentioned in debian/changelog is 1.0
And Tails 1.0 has not been released yet
And config/APT_overlays contains 'feature-foo'
And config/APT_overlays contains 'bugfix-bar'
When I successfully run tails-custom-apt-sources
Then I should see the 'stable' suite
Then I should not see the '1.0' suite
And I should see the 'feature-foo' suite
And I should see the 'bugfix-bar' suite
But I should not see the '1.0' suite
Scenario: build from a tagged stable branch
Scenario: build from a tagged stable branch where config/APT_overlays is empty
Given Tails 0.10 has been released
And last released version mentioned in debian/changelog is 0.10
And I am working on the stable branch
When I run tails-custom-apt-sources
Then I should see the '0.10' suite
And the last version mentioned in debian/changelog is 0.10
And I am working on the stable base branch
And config/APT_overlays is empty
When I successfully run tails-custom-apt-sources
Then I should see only the '0.10' suite
Scenario: build from a bugfix branch for a stable release
Scenario: build from a tagged stable branch where config/APT_overlays is not empty
Given Tails 0.10 has been released
And last released version mentioned in debian/changelog is 0.10
And I am working on the bugfix/disable_gdomap branch based on 0.10
And the last version mentioned in debian/changelog is 0.10
And I am working on the stable base branch
And config/APT_overlays contains 'feature-foo'
When I run tails-custom-apt-sources
Then I should see the '0.10' suite
Then it should fail
Scenario: build from a bugfix branch without overlays for a stable release
Given Tails 0.10 has been released
And the last version mentioned in debian/changelog is 0.10.1
And Tails 0.10.1 has not been released yet
And I am working on the bugfix/disable_gdomap branch based on stable
And config/APT_overlays is empty
When I successfully run tails-custom-apt-sources
Then I should see only the 'stable' suite
Scenario: build from a bugfix branch with overlays for a stable release
Given Tails 0.10 has been released
And the last version mentioned in debian/changelog is 0.10.1
And Tails 0.10.1 has not been released yet
And I am working on the bugfix/disable_gdomap branch based on stable
And config/APT_overlays contains 'bugfix-disable-gdomap'
And config/APT_overlays contains 'bugfix-bar'
When I successfully run tails-custom-apt-sources
Then I should see the 'stable' suite
And I should see the 'bugfix-disable-gdomap' suite
And I should see the 'bugfix-bar' suite
But I should not see the '0.10' suite
Scenario: build from an untagged testing branch
Given I am working on the testing branch
And last released version mentioned in debian/changelog is 0.11
Scenario: build from an untagged testing branch where config/APT_overlays is empty
Given I am working on the testing base branch
And the last version mentioned in debian/changelog is 0.11
And Tails 0.11 has not been released yet
When I run tails-custom-apt-sources
And config/APT_overlays is empty
When I successfully run tails-custom-apt-sources
Then I should see the 'testing' suite
And I should not see the '0.11' suite
And I should not see the 'feature-foo' suite
And I should not see the 'bugfix-bar' suite
Scenario: build from a tagged testing branch
Given I am working on the testing branch
And last released version mentioned in debian/changelog is 0.11
Scenario: build from an untagged testing branch where config/APT_overlays is not empty
Given I am working on the testing base branch
And the last version mentioned in debian/changelog is 0.11
And Tails 0.11 has not been released yet
And config/APT_overlays contains 'feature-foo'
And config/APT_overlays contains 'bugfix-bar'
When I successfully run tails-custom-apt-sources
Then I should see the 'testing' suite
And I should see the 'feature-foo' suite
And I should see the 'bugfix-bar' suite
But I should not see the '0.11' suite
Scenario: build from a tagged testing branch where config/APT_overlays is empty
Given I am working on the testing base branch
And the last version mentioned in debian/changelog is 0.11
And Tails 0.11 has been released
And config/APT_overlays is empty
When I successfully run tails-custom-apt-sources
Then I should see only the '0.11' suite
Scenario: build from a tagged testing branch where config/APT_overlays is not empty
Given I am working on the testing base branch
And the last version mentioned in debian/changelog is 0.11
And Tails 0.11 has been released
And config/APT_overlays contains 'feature-foo'
When I run tails-custom-apt-sources
Then I should see the '0.11' suite
And I should not see the 'testing' suite
Then it should fail
Scenario: build a release candidate from a tagged testing branch
Given I am working on the testing branch
Given I am working on the testing base branch
And Tails 0.11 has been released
And last released version mentioned in debian/changelog is 0.12~rc1
And the last version mentioned in debian/changelog is 0.12~rc1
And Tails 0.12-rc1 has been tagged
When I run tails-custom-apt-sources
Then I should see the '0.12-rc1' suite
And I should not see the 'testing' suite
And config/APT_overlays is empty
When I successfully run tails-custom-apt-sources
Then I should see only the '0.12-rc1' suite
Scenario: build from the devel branch
Given I am working on the devel branch
Scenario: build a release candidate from a tagged testing branch where config/APT_overlays is not empty
Given I am working on the testing base branch
And Tails 0.11 has been released
And the last version mentioned in debian/changelog is 0.12~rc1
And Tails 0.12-rc1 has been tagged
And config/APT_overlays contains 'bugfix-bar'
When I run tails-custom-apt-sources
Then it should fail
Scenario: build from the devel branch without overlays
Given I am working on the devel base branch
And config/APT_overlays is empty
When I successfully run tails-custom-apt-sources
Then I should see only the 'devel' suite
Scenario: build from the devel branch with overlays
Given I am working on the devel base branch
And config/APT_overlays contains 'feature-foo'
And config/APT_overlays contains 'bugfix-bar'
When I successfully run tails-custom-apt-sources
Then I should see the 'devel' suite
And I should see the 'feature-foo' suite
And I should see the 'bugfix-bar' suite
Scenario: build from the feature/jessie branch without overlays
Given I am working on the feature/jessie base branch
And config/APT_overlays is empty
When I successfully run tails-custom-apt-sources
Then I should see only the 'feature-jessie' suite
Scenario: build from the feature/jessie branch with overlays
Given I am working on the feature/jessie base branch
And config/APT_overlays contains 'feature-7756-reintroduce-whisperback'
When I successfully run tails-custom-apt-sources
Then I should see the 'feature-jessie' suite
And I should see the 'feature-7756-reintroduce-whisperback' suite
Scenario: build from the experimental branch
Given I am working on the experimental branch
When I run tails-custom-apt-sources
Then I should see the 'experimental' suite
Given I am working on the experimental branch based on devel
And config/APT_overlays contains 'feature-foo'
And config/APT_overlays contains 'bugfix-bar'
When I successfully run tails-custom-apt-sources
Then I should see the 'devel' suite
And I should see the 'feature-foo' suite
And I should see the 'bugfix-bar' suite
Scenario: build from a feature branch based on devel
Scenario: build from a feature branch with overlays based on devel
Given I am working on the feature/icedove branch based on devel
When I run tails-custom-apt-sources
And config/APT_overlays contains 'feature-icedove'
And config/APT_overlays contains 'bugfix-bar'
When I successfully run tails-custom-apt-sources
Then I should see the 'devel' suite
And I should see the 'feature-icedove' suite
And I should see the 'bugfix-bar' suite
Scenario: build from a feature branch without overlays based on devel
Given I am working on the feature/icedove branch based on devel
And config/APT_overlays is empty
When I successfully run tails-custom-apt-sources
Then I should see only the 'devel' suite
Scenario: build from a feature branch with overlays based on feature/jessie
Given I am working on the feature/7756-reintroduce-whisperback branch based on feature/jessie
And config/APT_overlays contains 'feature-7756-reintroduce-whisperback'
And config/APT_overlays contains 'bugfix-bar'
When I successfully run tails-custom-apt-sources
Then I should see the 'feature-jessie' suite
And I should see the 'feature-7756-reintroduce-whisperback' suite
And I should see the 'bugfix-bar' suite
Scenario: build from a feature branch without overlays based on feature/jessie
Given I am working on the feature/icedove branch based on feature/jessie
And config/APT_overlays is empty
When I successfully run tails-custom-apt-sources
Then I should see only the 'feature-jessie' suite
Scenario: build from a feature branch based on devel with dots in its name
Given I am working on the feature/live-boot-3.x branch based on devel
When I run tails-custom-apt-sources
And config/APT_overlays contains 'feature-live-boot-3.x'
When I successfully run tails-custom-apt-sources
Then I should see the 'devel' suite
And I should see the 'feature-live-boot-3.x' suite
Scenario: build from a branch that has no config/APT_overlays file
Given I am working on the stable base branch
And config/APT_overlays does not exist
When I run tails-custom-apt-sources
Then it should fail
Scenario: build from a branch that has no config/base_branch file
Given I am working on the stable base branch
And config/base_branch does not exist
When I run tails-custom-apt-sources
Then it should fail
Scenario: build from a branch where config/base_branch is empty
Given I am working on the stable base branch
And config/base_branch is empty
When I run tails-custom-apt-sources
Then it should fail
Given /^Tails ([[:alnum:].]+) has been released$/ do |version|
create_git unless git_exists?
old_branch = current_branch
fatal_system "git checkout --quiet stable"
old_entries = File.open('debian/changelog') { |f| f.read }
File.open('debian/changelog', 'w') do |changelog|
......@@ -16,6 +18,11 @@ END_OF_CHANGELOG
end
fatal_system "git commit --quiet debian/changelog -m 'Release #{version}'"
fatal_system "git tag '#{version}'"
if old_branch != 'stable'
fatal_system "git checkout --quiet '#{old_branch}'"
fatal_system "git merge --quiet 'stable'"
end
end
Given /^Tails ([[:alnum:].-]+) has been tagged$/ do |version|
......@@ -26,7 +33,7 @@ Given /^Tails ([[:alnum:].]+) has not been released yet$/ do |version|
!File.exists? ".git/refs/tags/#{version}"
end
Given /^last released version mentioned in debian\/changelog is ([[:alnum:]~.]+)$/ do |version|
Given /^the last version mentioned in debian\/changelog is ([[:alnum:]~.]+)$/ do |version|
last = `dpkg-parsechangelog | awk '/^Version: / { print $2 }'`.strip
raise StandardError.new('dpkg-parsechangelog failed.') if $? != 0
......@@ -35,37 +42,71 @@ Given /^last released version mentioned in debian\/changelog is ([[:alnum:]~.]+)
end
end
Given %r{I am working on the ([[:alnum:]./_-]+) branch$} do |branch|
Given %r{I am working on the ([[:alnum:]./_-]+) base branch$} do |branch|
create_git unless git_exists?
current_branch = `git branch | awk '/^\*/ { print $2 }'`.strip
raise StandardError.new('git-branch failed.') if $? != 0
if current_branch != branch
fatal_system "git checkout --quiet '#{branch}'"
end
File.open('config/base_branch', 'w+') do |base_branch_file|
base_branch_file.write("#{branch}\n")
end
end
Given %r{I am working on the ([[:alnum:]./_-]+) branch based on ([[:alnum:]./_-]+)$} do |branch, base|
create_git unless git_exists?
current_branch = `git branch | awk '/^\*/ { print $2 }'`.strip
raise StandardError.new('git-branch failed.') if $? != 0
if current_branch != branch
fatal_system "git checkout --quiet -b '#{branch}' '#{base}'"
end
File.open('config/base_branch', 'w+') do |base_branch_file|
base_branch_file.write("#{base}\n")
end
end
When /^I run ([[:alnum:]-]+)$/ do |command|
When /^I successfully run ([[:alnum:]-]+)$/ do |command|
@output = `#{File.expand_path("../../../auto/scripts/#{command}", __FILE__)}`
raise StandardError.new("#{command} failed. Exit code: #{$?}") if $? != 0
end
When /^I run ([[:alnum:]-]+)$/ do |command|
@output = `#{File.expand_path("../../../auto/scripts/#{command}", __FILE__)}`
@exit_code = $?
end
Then /^I should see the ['"]?([[:alnum:].-]+)['"]? suite$/ do |suite|
@output.should have_suite(suite)
end
Then /^I should see only the ['"]?([[:alnum:].-]+)['"]? suite$/ do |suite|
assert_equal(1, @output.lines.count)
@output.should have_suite(suite)
end
Then /^I should not see the ['"]?([[:alnum:].-]+)['"]? suite$/ do |suite|
@output.should_not have_suite(suite)
end
Given(/^config\/APT_overlays is empty$/) do
assert_equal(0, File.new('config/APT_overlays').size)
end
Given(/^config\/APT_overlays contains ['"]?([[:alnum:].-]+)['"]?$/) do |suite|
File.open('config/APT_overlays', 'a') do |overlays|
overlays.write("#{suite}\n")
end
end
Then(/^it should fail$/) do
assert_not_equal(0, @exit_code)
end
Given(/^(config\/(?:APT_overlays|base_branch)) does not exist$/) do |file|
File.delete(file)
end
Given(/^config\/base_branch is empty$/) do
File.open('config/base_branch', 'w+') { }
end
......@@ -14,6 +14,9 @@ def git_exists?
end
def create_git
Dir.mkdir 'config'
FileUtils.touch('config/base_branch')
FileUtils.touch('config/APT_overlays')
Dir.mkdir 'debian'
File.open('debian/changelog', 'w') do |changelog|
changelog.write(<<END_OF_CHANGELOG)
......@@ -33,7 +36,14 @@ END_OF_CHANGELOG
fatal_system "git branch -M stable"
fatal_system "git branch testing stable"
fatal_system "git branch devel stable"
fatal_system "git branch experimental devel"
fatal_system "git branch feature/jessie devel"
end
def current_branch
branch = `git branch | awk '/^\*/ { print $2 }'`.strip
raise StandardError.new('git-branch failed.') if $? != 0
return branch
end
RSpec::Matchers.define :have_suite do |suite|
......
......@@ -49,9 +49,12 @@ error() {
exit 1
}
package_installed() {
dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"
}
check_dependency() {
if ! which "${1}" >/dev/null && \
! dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
if ! which "${1}" >/dev/null && ! package_installed "${1}" ; then
error "'${1}' is missing, please install it and run again. Aborting..."
fi
}
......@@ -197,6 +200,11 @@ for dep in git libvirt-daemon-system libvirt-clients libvirt-dev libav-tools lib
check_dependency "${dep}"
done
package_installed libavcodec-extra-53 \
|| package_installed libav-tools \
|| error "Both libavcodec-extra-53 and libav-tools, are missing, " \
"please install one of those and run again. Aborting..."
TARGET_DISPLAY=$(next_free_display)
start_xvfb
......
......@@ -12,9 +12,10 @@ We use one single APT repository hosting multiple *suites*:
* We have a (read-only) suite for every past release: `0.9`,
`0.10.1`, etc.
* We have a suite for each *main* branch: `stable`, `testing`, `devel`
* We have a suite for each *topic* branch: `bugfix/*`, `feature/*`.
**Important note**: the APT suite corresponding to a given Git topic
* We have a suite for each *main* branch: `stable`, `testing`,
`devel`, `feature-jessie`
* We have an overlay suite for each *topic* branch: `bugfix/*`, `feature/*`.
**Note**: the APT suite corresponding to a given Git topic
branch contains *only* the packages this branch adds to the tag or
*main* branch it diverged from. Think of it as an overlay.
* We also have a less formal `unstable` suite, that should not be used
......@@ -30,8 +31,9 @@ We use one single APT repository hosting multiple *suites*:
packages needed on a Wheezy system to build Tails.
The suite(s) to use as sources for APT, during the build and inside
the resulting system, are determined at Tails build time
(`auto/config`). See details in the *Build system* section below.
the resulting system, are determined by the content of the
`config/base_branch` and `config/APT_overlays` files. See details in
the *Build system* section below.
We manage our APT repository with
[reprepro](http://mirrorer.alioth.debian.org/).
......@@ -49,18 +51,50 @@ few minutes, detects new branches, and accordingly:
Build system
============
The build system adds the relevant APT sources:
The Tails ISO build system dynamically adds APT sources that will be
used during the build, and inside the resulting ISO itself.
* if the version in `debian/changelog` was released already (i.e.
a matching tag exists), then add the suite corresponding to this
release (e.g. `0.10` or `0.10.1`);
* else, if building from the `testing` branch, add the `testing` suite
* else, if building from the `devel` branch, add the `devel` suite
* else, if building from the `experimental` branch, add the `experimental` suite
* else, add the `devel` suite
If the last version in `debian/changelog` was released already (i.e.
a matching tag exists), then the build system adds the suite
corresponding to this release (e.g. `1.5` or `3.0`), and that's all.
Also, if we're building from a bugfix or feature branch, add its
own suite.
Else, it adds:
* one APT source for the base branch of the one being built, as found
in `config/base_branch`;
* one APT source for each suite listed in `config/APT_overlays`.
In practice, `config/APT_overlays` contains:
* for a topic branch:
- if needed, the branch's own overlay APT suite;
- any APT suite that came from merging its base branch into this
topic branch, that is: