Commit 2251c449 authored by Sandro Knauß's avatar Sandro Knauß Committed by Zen Fu
Browse files

Add update_puppet_dependecy_files.py with templates and config

parent 2f433712
# Manage Weblate dependencies that are in Debian
class tails::weblate::debian_packages () {
# some packages are commented out because the needed versions for Weblate
# $WEBLATE_VERSION are still not available in Debian stable. Fore more details, see:
# https://redmine.tails.boum.org/code/issues/10038
$$packages = [
'apache2',
'ccze',
'ipython3', # for more convenient debugging
'memcached',
'mercurial', # so we can use pip to install from hg repos
'libapache2-mod-security2',
'libapache2-mod-wsgi-py3',
'modsecurity-crs',
'$WEBLATE_DEPENDS',
'sqlite3',
]
ensure_packages($$packages)
}
---
skip:
- pillow # Pillow is packaged as python3-pil in Debian
- brotli # extra depdendecies by kombu, we don't need
- brotlipy # extra depdendecies by kombu, we don't need
- configparser # it is part of PYthon 3.5
- setuptools # we need this anyway to install packages via pip
require_pip:
- django
- django-appconf
- django-crispy-forms
- django-compressor
- jellyfish
- translate-toolkit
require_debian:
- python3-levenshtein # optional Weblate dependency to speed up DB migrations
- python3-memcache
- python3-mysqldb # Tails uses MariaDB as Weblate database
- python3-pil # Pillow is packaged as python3-pil in Debian
- python3-pyuca # optional Weblate depdendency for Unicode support
- python3-yaml # For our own scripts
# Manage Weblate dependencies that are in not in Debian
class tails::weblate::python_modules () {
$$pip_packages = [
'python3-pip',
'python3-setuptools',
]
ensure_packages($$pip_packages)
# The following deps are in Debian Stretch but with a version that is older
# than needed by Weblate $WEBLATE_VERSION. The versions noted as comments are the
# requirements for the current Weblate version we are running (check
# manifests/weblate.pp to know which version it is).
$PIP_DEPENDS
}
#!/usr/bin/env python3
from operator import attrgetter
from string import Template
import git
import io
import itertools
import pathlib
import re
import requests
import yaml
from debian.debian_support import Version
import apt_pkg
apt_cache = apt_pkg.Cache()
class Dependency:
def __init__(self, line, split=True):
self.line = line
self.python_version = None
self._pypi = None
self.comment = bool(re.match("^\s*#", line))
if self.comment:
return
if ";" in line:
m = re.search(r"; python_version ([<>=]+) '([0-9.]+)'", line)
if m:
self.python_version = Version(m.group(2))
self.python_constrain = m.group(1)
line = line[:line.find(";")].strip()
self.dep = line
first = True
self.constrains=[]
if split:
for part in line.split(","):
m = re.match(r"^(?P<name>[^!=<>]*)\s*((?P<constrain>[!=<>~]+)\s*(?P<version>[0-9][0-9\.]*))?\s*$", part)
if first:
self.name = m.group("name")
first = False
if m.group("constrain"):
self.constrains.append((m.group("constrain"), m.group("version")))
else:
m = re.match(r"^(?P<name>[^!=<>]*)\s*(?P<constrains>\((([!=<>~]+)?\s*([0-9][0-9ab\.]*|dev),?)+\))?\s*$", line)
self.name = m.group("name")
if m.group("constrains"):
for part in m.group("constrains")[1:-1].split(","):
n = re.match(r"^(?P<constrain>[!=<>~]+)\s*(?P<version>[0-9][0-9ab\.]*|dev)$", part.strip())
self.constrains.append((n.group("constrain"), n.group("version")))
self.name = self.name.lower().strip().replace("_","-")
def pypi(self):
if self._pypi:
return self._pypi
ret = requests.get(f"https://pypi.org/pypi/{self.name}/json")
ret.raise_for_status()
self._pypi = ret.json()
return self._pypi
def python_match(self, ver):
if self.comment:
return False
if not self.python_version:
return True
if self.python_constrain == ">":
return ver > self.python_version
elif self.python_constrain == "<":
return ver < self.python_version
elif self.python_constrain == ">=":
return ver >= self.python_version
else:
print(f"Unknown constrain: {self.python_constrain}, {self.line}")
def match(self, version):
if self.comment:
return False
for constrain in self.constrains:
c = constrain[0]
v = constrain[1]
if c == "==":
if not Version(version.upstream_version) == Version(v):
return False
elif c == ">=":
if not Version(version.upstream_version) >= Version(v):
return False
elif c == "<=":
if not Version(version.upstream_version) <= Version(v):
return False
elif c == "!=":
if not Version(version.upstream_version) != Version(v):
return False
return True
@property
def text_constrains(self):
strconstrains = []
for constrain in self.constrains:
strconstrains.append(constrain[0]+constrain[1])
return ",".join(strconstrains)
def __repr__(self):
if self.comment:
return f"<Dependency comment('{self.line}')>"
if self.python_version:
return f"<Dependency {self.name}({self.text_constrains}) (only for Python {self.python_constrain} {self.python_version})>"
else:
return f"<Dependency {self.name}({self.text_constrains})>"
@property
def url(self):
return self.pypi()['info']['home_page']
@property
def newest_pip_version(self):
return self.pypi()['info']['version']
@property
def tag(self):
return None
def require(self, via_pip):
if not self.pypi()['info']['requires_dist']:
return []
for i in self.pypi()['info']['requires_dist']:
if "; extra == 'redis'" in i:
i = i[:i.find("; extra")].strip()
if "; extra" in i:
print(i)
continue
dep = Dependency(i, split=False)
if via_pip and dep.name not in via_pip:
continue
yield dep
def puppet_code(self, via_pip):
head = f"tails::pip_package_from_repo {{ '{self.name}':"
tail = " }"
lines = []
if self.text_constrains:
lines.append(f"version => '{self.newest_pip_version}', # {self.text_constrains}")
else:
lines.append(f"version => '{self.newest_pip_version}',")
if self.tag:
lines.append("tag => '{self.tag}',")
lines.append(f"url => '{self.url}',")
for dep in self.require(via_pip):
lines.append(f"require => 'Exec[pip_install_{dep.name}]',")
inner = '\n '.join(lines)
return f"{head}\n {inner}\n{tail}\n"
class DebianSuite:
def stretch(v):
for f in v.file_list:
if f[0].site != 'deb.debian.org':
continue
archive = f[0].archive
if archive.startswith("oldstable"):
return archive
elif archive.startswith("stretch"):
return archive
else:
return False
def buster(v):
for f in v.file_list:
if f[0].site != 'deb.debian.org':
continue
archive = f[0].archive
if archive.startswith("stable"):
return archive
elif archive.startswith("buster"):
return archive
else:
return False
def bullseye(v):
for f in v.file_list:
if f[0].site != 'deb.debian.org':
continue
archive = f[0].archive
if archive.startswith("testing"):
return archive
elif archive.startswith("bullseye"):
return archive
else:
return False
def names(basename):
if basename.startswith("python"):
yield from names(basename[basename.find("-")+1:])
elif basename.startswith("py"):
yield from names(basename[2:])
yield "python3-"+basename
yield basename
def deb_versions(pkgname,suite):
err = None
once = False
for n in names(pkgname.lower()):
try:
a_pkg = apt_cache[n]
except KeyError as e:
if not err:
err = e
continue
for v in a_pkg.version_list:
archive = suite(v)
if not archive:
continue
yield Version(v.ver_str), archive, v
once = True
if once:
return
else:
raise err
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"--weblate-version",
help="What Weblate version.")
parser.add_argument(
"--debian-suite",
choices=['stretch', 'buster', 'bullseye'],
help="Debian suite.")
args = parser.parse_args()
suite_name = args.debian_suite
suite = getattr(DebianSuite, suite_name)
repo = git.Repo('/home/hefee/git/weblate')
output = io.BytesIO()
b = repo.commit("weblate-{}".format(args.weblate_version)).tree/"requirements.txt"
b.stream_data(output)
deps = [Dependency(i) for i in output.getvalue().decode().splitlines()]
config = yaml.safe_load(pathlib.Path(__file__).with_name("override.yaml").open())
SKIP_LIST = set(config.get('skip'))
PIP_ONLY = set(config.get('require_pip'))
pip_needs = set()
debian_needs = set(config.get('require_debian'))
a_pkg = apt_cache['python3']
for v in a_pkg.version_list:
archive = suite(v)
if not archive:
continue
running_python = Version(v.ver_str)
break
def in_debian(dep):
try:
debversions = list(deb_versions(dep.name, suite))
if not debversions:
return False
for version, archive, pkg in debversions:
if dep.match(version):
return pkg.parent_pkg.name
except KeyError:
return False
known = set()
for dep in deps:
if dep.comment:
continue
known.add(dep.name)
if not dep.python_match(running_python.upstream_version):
continue
if dep.name in SKIP_LIST:
continue
if dep.name in PIP_ONLY:
pip_needs.add(dep)
continue
debian_package = in_debian(dep)
if debian_package:
debian_needs.add(debian_package)
else:
pip_needs.add(dep)
add = True
while add:
add = False
pip_adds = set()
for dep in pip_needs:
try:
for require in dep.require([]):
if require.comment:
continue
if require.name in known:
continue
known.add(require.name)
if not require.python_match(running_python.upstream_version):
continue
if require.name in SKIP_LIST:
continue
if require.name in PIP_ONLY:
pip_adds.add(require)
continue
debian_package = in_debian(require)
if debian_package:
debian_needs.add(debian_package)
elif require not in pip_needs:
pip_adds.add(require)
add = True
except requests.HTTPError:
pass
pip_needs |= pip_adds
template_file = pathlib.Path(__file__).with_name("debian_packages.pp").read_text()
template = Template(template_file)
debian_depends = template.substitute(
WEBLATE_DEPENDS="',\n '".join(sorted(debian_needs)),
WEBLATE_VERSION= args.weblate_version,
)
pathlib.Path("manifests/weblate/debian_packages.pp").write_text(debian_depends)
template_file = pathlib.Path(__file__).with_name("python_modules.pp").read_text()
template = Template(template_file)
via_pip = set((i.name for i in pip_needs))
pip_depends = []
for dep in sorted(pip_needs, key=attrgetter('name')):
try:
pip_depends.append(dep.puppet_code(via_pip))
except requests.HTTPError:
print(f"Didn't found {dep.name} on pypi.")
python_modules = template.substitute(
PIP_DEPENDS="\n ".join(pip_depends),
WEBLATE_VERSION= args.weblate_version,
)
pathlib.Path("manifests/weblate/python_modules.pp").write_text(python_modules)
if __name__ == "__main__":
main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment