Commit 6087ca6f authored by segfault's avatar segfault
Browse files

Move pythonlib to main repo and remove the submodule (refs: #16935)

parent 1be30b68
[submodule "submodules/pythonlib"]
path = submodules/pythonlib
url = https://git-tails.immerda.ch/pythonlib
[submodule "submodules/jenkins-tools"]
path = submodules/jenkins-tools
url = https://git-tails.immerda.ch/jenkins-tools
......
......@@ -218,4 +218,3 @@ sed -i "s,%%topdir%%,$(pwd)," /usr/share/debootstrap/scripts/tails-build-jessie
# Make the python library available in Tails
install -d -m 2777 config/chroot_local-includes/tmp/
cp -r submodules/pythonlib config/chroot_local-includes/tmp/
#!/bin/sh
set -e
set -u
echo "Installing the tailslib python library"
# Import ensure_hook_dependency_is_installed() and
# strip_nondeterminism_wrapper()
. /usr/local/lib/tails-shell-library/build.sh
ensure_hook_dependency_is_installed python3-setuptools
(
cd /tmp/pythonlib
python3 setup.py clean
python3 setup.py install
package_glob_pattern="/usr/local/lib/python3.*/dist-packages/Tailslib*.egg"
strip_nondeterminism_wrapper --type zip ${package_glob_pattern}
)
rm -rf /tmp/pythonlib
#!/usr/bin/python3
#
LIVE_USERNAME = "amnesia"
PERSISTENCE_SETUP_USERNAME = "tails-persistence-setup"
"""Tails Additional Software configuration file management."""
import grp
import logging
import os
import os.path
import pwd
import re
import atomicwrites
from tailslib import PERSISTENCE_SETUP_USERNAME
from tailslib.persistence import get_persistence_path
PACKAGES_LIST_FILE = "live-additional-software.conf"
class ASPError(Exception):
"""Base class for exceptions raised by ASP."""
class ASPDataError(ASPError):
"""Raised when the data read does not have the expected format."""
pass
def _write_config(packages, search_new_persistence=False):
config_file_owner_uid = pwd.getpwnam(PERSISTENCE_SETUP_USERNAME).pw_uid
config_file_owner_gid = grp.getgrnam(PERSISTENCE_SETUP_USERNAME).gr_gid
packages_list_path = get_packages_list_path(search_new_persistence)
try:
os.setegid(config_file_owner_gid)
os.seteuid(config_file_owner_uid)
with atomicwrites.atomic_write(packages_list_path,
overwrite=True) as f:
for package in sorted(packages):
f.write(package + '\n')
os.chmod(packages_list_path, 0o0644)
finally:
os.seteuid(0)
os.setegid(0)
def filter_package_details(pkg):
"""Filter target release, version and architecture from pkg."""
return re.split("[/:=]", pkg)[0]
def get_packages_list_path(search_new_persistence=False,
return_nonexistent=False):
"""Return the package list file path in current or new persistence.
The search_new_persistence and return_nonexistent arguments are passed to
get_persistence_path.
"""
persistence_dir = get_persistence_path(search_new_persistence,
return_nonexistent)
return os.path.join(persistence_dir, PACKAGES_LIST_FILE)
def get_additional_packages(search_new_persistence=False):
"""Return the list of all additional packages configured.
The search_new_persistence argument is passed to get_persistence_path.
"""
packages = set()
try:
with open(get_packages_list_path(search_new_persistence)) as f:
for line in f:
line = line.strip()
if line:
packages.add(line)
except FileNotFoundError:
# Just return an empty set.
pass
return packages
def add_additional_packages(new_packages, search_new_persistence=False):
"""Add packages to additional packages configuration.
Add the packages to additional packages configuration.
The new_packages argument should be a list of packages names.
The search_new_persistence argument is passed to get_persistence_path.
"""
logging.info("Adding to additional packages list: %s" % new_packages)
packages = get_additional_packages(search_new_persistence)
# The list of packages was initially provided by apt after installing them,
# so we don't check the names.
packages |= new_packages
_write_config(packages, search_new_persistence)
def remove_additional_packages(old_packages, search_new_persistence=False):
"""Remove packages from additional packages configuration.
Removes the packages from additional packages configuration.
The old_packages argument should be a list of packages names.
The search_new_persistence argument is passed to get_persistence_path.
"""
logging.info("Removing from additional packages list: %s" % old_packages)
packages = get_additional_packages(search_new_persistence)
# The list of packages was initially provided by apt after removing them,
# so we don't check the names.
packages -= old_packages
_write_config(packages, search_new_persistence)
#!/usr/bin/env python3
# This file is part of Tails.
import subprocess
def is_password_set():
output = subprocess.check_output(["passwd", "--status"])
return output.split()[1] == b"P"
#!/usr/bin/python3
#
# Handles our errors.
class TailsLibException(Exception):
pass
class UnreadableGitRepo(TailsLibException):
pass
class CheckDirectoryExists(TailsLibException):
pass
class TorFailedToBoostrapError(Exception):
pass
#!/usr/bin/python3
#
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
import functools
import os
import re
from distutils.version import StrictVersion
from sh import git as _real_git
class Git:
"""
Provide facilities to inspect the content of a Tails Git repository.
Note that some methods are memoized, so an instance of this class
is not guaranteed to produce up-to-date results if the Git repository
is modified externally during the lifetime of the object.
"""
# These branches must always exist. They will never be automatically deleted.
STATIC_BRANCHES=[
'devel',
'master',
'stable',
'testing',
'feature/buster',
]
def __init__(self, git_repo):
self.git_repo = git_repo
assert os.access(self.git_repo, os.R_OK)
assert os.path.isdir(self.git_repo)
self.git_args = '--git-dir=%s' % self.git_repo
assert self.is_bare_repo(), 'Please use a git bare repository.'
def is_bare_repo(self):
if self.git('rev-parse', '--is-bare-repository').splitlines() == ['false']:
return False
return True
def all_branches(self):
return [self.clean_branch_name(branch) for branch in
self.git('branch', '--no-color').splitlines()]
def new_revs_in_branch(self, oldrev, branch):
return self.git('rev-list',
'%s..%s' % (oldrev, branch)).splitlines()
def revs_in_branch_after(self, since, branch):
return self.git('rev-list',
'--since=%s' % since,
'%s' % branch).splitlines()
def tags(self):
return self.git('tag').splitlines()
@functools.lru_cache()
def release_tags(self):
return [tag for tag in self.tags()
if not re.search(r'-(?:alpha|beta|rc)\d*$', tag)]
@functools.lru_cache()
def last_release_tag(self):
return '%s' % max([StrictVersion(tag) for tag in self.release_tags()])
@functools.lru_cache(maxsize=None)
def clean_branch_name(self, branch):
"""Clean the branch name."""
return re.fullmatch(r'[*]?\s*(.+)', branch).group(1)
def has_commits_since_last_release(self, branch):
last_release_tag = self.last_release_tag()
revs_on_top_of_last_release = self.new_revs_in_branch(
last_release_tag, branch)
date_of_last_release = self.committer_date(last_release_tag)
for commit in revs_on_top_of_last_release:
if self.committer_date(commit) > date_of_last_release:
return True
return False
def has_commits_since(self, since, branch):
return self.revs_in_branch_after(since, branch)
@functools.lru_cache(maxsize=None)
def committer_date(self, commit):
return self.git('--no-pager',
'show',
'-1',
'--no-patch',
'--pretty=format:"%ct"',
commit).__str__()
def branches_merged_into(self, commit):
return [self.clean_branch_name(branch) for branch in
self.git('--no-pager',
'branch',
'--no-color',
'--merged',
commit).splitlines()]
def branches_to_delete(self):
return [branch for branch in self.branches_merged_into('master')
if not branch in Git.STATIC_BRANCHES]
def push(self, remote, refs):
self.git('push', remote, *refs)
def git(self, *args):
return _real_git(self.git_args, args)
#!/usr/bin/python3
import os
import shlex
import subprocess
def _gnome_sh_wrapper(cmd):
command = shlex.split(
"env -i sh -c '. {lib} && {cmd}'".format(lib=GNOME_SH_PATH, cmd=cmd)
)
return subprocess.check_output(command).decode()
GNOME_SH_PATH = "/usr/local/lib/tails-shell-library/gnome.sh"
GNOME_ENV_VARS = _gnome_sh_wrapper("echo ${GNOME_ENV_VARS}").strip().split()
def gnome_env_vars():
ret = []
for line in _gnome_sh_wrapper("export_gnome_env && env").split("\n"):
(key, _, value) = line.rstrip().partition("=")
if key in GNOME_ENV_VARS:
ret.append(key + "=" + value)
return ret
#!/usr/bin/python3
#
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
import re
from datetime import datetime, timedelta
from operator import itemgetter
from tailslib.git import Git
class ActiveBranches (Git):
"""
Compute set of active Git branches that should be built on Jenkins,
along with some parameters that the Jenkins build jobs would need.
"""
# These branches are always considered as worthy to be built.
ALWAYS_ACTIVE_BRANCHES = [
'devel',
'feature/buster',
'feature/tor-nightly-master',
'stable',
]
# These branches are used for the "has this topic branch been
# merged already?" logic. The "base branch" concept here is
# different from its homonyms (e.g. what's in config/base_branch,
# or the base APT suite concept).
BASE_BRANCHES = [
'devel',
'master',
'stable',
'testing',
]
def __init__(self, git_repo, active_days):
# A branch that has not seen activity since this number of days
# is not active.
Git.__init__(self, git_repo)
self.active_days = active_days
assert isinstance(self.active_days, int)
def compute(self):
return sorted([
self.job_params(branch)
for branch in self.active_branches()
], key=itemgetter('branch_name'))
def job_params(self, branch):
return {'branch_name': branch,
'project_name': self.project_name(branch),
'recipients': self.recipients(branch),
'is_base_branch': self.is_base_branch(branch),
'ticket_number': self.ticket_number(branch)}
def active_branches(self):
since = int(
(datetime.now() - timedelta(days=self.active_days))
.timestamp()
)
branches = list(set(
ActiveBranches.ALWAYS_ACTIVE_BRANCHES +
[branch for branch in self.all_branches()
if self.has_commits_since(since, branch)
and self.not_merged_into_any_base_branch(branch)]
))
if self.new_revs_in_branch('stable', 'testing'):
branches.append('testing')
return branches
def is_base_branch(self, branch):
return branch in ActiveBranches.BASE_BRANCHES
def project_name(self, branch):
project_name_re_match = re.compile('[^a-zA-Z0-9-.]')
return project_name_re_match.sub('-', branch).lower()
def recipients(self, branch):
if self.is_base_branch(branch):
return 'release_managers'
else:
return 'whoever_broke_the_build'
def ticket_number(self, branch):
ticket_re_match = re.search(r'/(\d{4,6})+', branch)
if ticket_re_match:
return ticket_re_match.group(1)
else:
return 0
def not_merged_into_any_base_branch(self, branch):
# XXX: might be faster to iterate over the base branches
# and not on the containing branches? (use new_revs_in_branch)
for containing_branch in self.git('branch',
'--no-color',
'--contains',
branch).splitlines():
if self.is_base_branch(self.clean_branch_name(containing_branch)):
return False
return True
"""Tails persistence related tests."""
import os
import subprocess
from tailslib import PERSISTENCE_SETUP_USERNAME
from tailslib.utils import launch_x_application
PERSISTENCE_DIR = "/live/persistence/TailsData_unlocked"
NEWLY_CREATED_PERSISTENCE_DIR = "/media/tails-persistence-setup/TailsData"
PERSISTENCE_PARTITION = "/dev/disk/by-partlabel/TailsData"
def get_persistence_path(search_new_persistence=False,
return_nonexistent=False):
"""Return the path of the (newly created) persistence.
Return PERSISTENCE_DIR if it exists. If search_new_persistence is True,
also try NEWLY_CREATED_PERSISTENCE_DIR.
If return_nonexistent is true, return the path that the file would have
after new persistence creation.
If no persistence directory exists and return_nonexistent is true, raise
FileNotFoundError.
"""
if os.path.isdir(PERSISTENCE_DIR):
return PERSISTENCE_DIR
elif search_new_persistence:
if os.path.isdir(NEWLY_CREATED_PERSISTENCE_DIR) or return_nonexistent:
return NEWLY_CREATED_PERSISTENCE_DIR
else:
raise FileNotFoundError(
"No persistence directory found. Neither {dir} not {alt_dir} "
"exist.".format(dir=PERSISTENCE_DIR,
alt_dir=NEWLY_CREATED_PERSISTENCE_DIR))
else:
raise FileNotFoundError(
"No persistence directory found in {dir}".format(
dir=PERSISTENCE_DIR))
def has_persistence():
"""Return true iff PERSISTENCE_PARTITION exists."""
return os.path.exists(PERSISTENCE_PARTITION)
def has_unlocked_persistence(search_new_persistence=False):
"""Return true iff a persistence directory exists.
The search_new_persistence argument is passed to get_persistence_path.
"""
try:
get_persistence_path(search_new_persistence)
except FileNotFoundError:
return False
else:
return True
def is_tails_media_writable():
"""Return true iff tails is started from a writable media."""
return not subprocess.run(
"/usr/local/lib/tails-boot-device-can-have-persistence").returncode
def launch_persistence_setup(*args):
"""Launch tails-persistence-setup and wait for its completion."""
launch_x_application(PERSISTENCE_SETUP_USERNAME,
"/usr/bin/tails-persistence-setup",
*args)
import stem
from stem.control import Controller
CONTROL_SOCKET_PATH = '/var/run/tor/control'
def tor_has_bootstrapped():
try:
c = Controller.from_socket_file(CONTROL_SOCKET_PATH)
except stem.SocketError:
# Socket is not available
return False
c.authenticate()
info = c.get_info("status/bootstrap-phase")
return "PROGRESS=100" in info.split()
#!/usr/bin/python3
import contextlib
import os
import logging
import pwd
import subprocess
from tailslib import LIVE_USERNAME
X_ENV = {"DISPLAY": ":1",
"XAUTHORITY": "/run/user/1000/gdm/Xauthority"}
# Credits go to kurin from this Reddit thread:
# https://www.reddit.com/r/Python/comments/1sxil3/chdir_a_context_manager_for_switching_working/ce29rcm
@contextlib.contextmanager
def chdir(path):
curdir = os.getcwd()
os.chdir(path)
try:
yield
finally:
os.chdir(curdir)
def launch_x_application(username, command, *args):
"""Launch an X application and wait for its completion."""
live_user_uid = pwd.getpwnam(LIVE_USERNAME).pw_uid
xhost_cmd = ["xhost", "+SI:localuser:" + username]
if os.geteuid() != live_user_uid:
xhost_cmd = ["sudo", "-u", LIVE_USERNAME] + xhost_cmd
subprocess.run(
xhost_cmd,
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
env=X_ENV,
check=True)
cmdline = ["sudo", "-u", username, command]
cmdline.extend(args)
try:
subprocess.run(cmdline,
stderr=subprocess.PIPE,
env=X_ENV,
check=True,
universal_newlines=True)
except subprocess.CalledProcessError as e:
logging.error("{command} returned with {returncode}".format(
command=command, returncode=e.returncode))
for line in e.stderr.splitlines():
logging.error(line)
raise
finally:
xhost_cmd = ["xhost", "-SI:localuser:" + username]
if os.geteuid() != live_user_uid:
xhost_cmd = ["sudo", "-u", LIVE_USERNAME] + xhost_cmd
subprocess.run(
xhost_cmd,
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
env=X_ENV,
check=True)