Commit e0c83c15 authored by Sandro Knauß's avatar Sandro Knauß Committed by Zen Fu
Browse files

Fix url, repo_type and tag of pip packages.

parent 60f36192
......@@ -22,3 +22,30 @@ require_debian:
- python3-pil # Pillow is packaged as python3-pil in Debian
- python3-pyuca # optional Weblate depdendency for Unicode support
- python3-yaml # For our own scripts
pip:
celery:
url: https://github.com/celery/celery
django:
additional_condition: <2.0
url: https://github.com/django/django
django-appconf:
url: https://github.com/django-compressor/django-appconf
django-compressor:
url: https://github.com/django-compressor/django-compressor
djangorestframework:
url: https://github.com/encode/django-rest-framework
importlib-metadata:
url: https://gitlab.com/python-devs/importlib_metadata
jellyfish:
url: https://github.com/jamesturk/jellyfish
kombu:
url: http://github.com/celery/kombu
openpyxl:
url: https://bitbucket.org/openpyxl/openpyxl/src/default
repo_type: hg
rjsmin:
url: https://github.com/ndparker/rjsmin
translate-toolkit:
url: https://github.com/translate/translate
translation-finder:
url: https://github.com/WeblateOrg/translation-finder.git
#!/usr/bin/env python3
from bs4 import BeautifulSoup
from operator import attrgetter
from string import Template
import git
......@@ -7,6 +8,7 @@ import itertools
import pathlib
import re
import requests
import urllib
import yaml
from debian.debian_support import Version
......@@ -15,8 +17,9 @@ import apt_pkg
apt_cache = apt_pkg.Cache()
class Dependency:
def __init__(self, line, split=True):
def __init__(self, line, config, split=True):
self.line = line
self.config = config
self.python_version = None
self._pypi = None
self.comment = bool(re.match("^\s*#", line))
......@@ -110,6 +113,10 @@ class Dependency:
@property
def url(self):
if self.name in self.config:
if "url" in self.config[self.name]:
return self.config[self.name]["url"]
return self.pypi()['info']['home_page']
@property
......@@ -118,6 +125,47 @@ class Dependency:
@property
def tag(self):
urls = [
f"{self.url}/tags", # Github tag page
f"{self.url}/-/tags", # Gitlab tag page
]
for url in urls:
r = requests.get(f"{self.url}/tags")
if r.status_code == 200:
break
else:
return None
if url.endswith("-/tags"):
#gitlab
url_path = urllib.parse.urljoin(urllib.parse.urlparse(self.url).path+"/", "-/tag/")
else:
#github
url_path = urllib.parse.urljoin(urllib.parse.urlparse(self.url).path+"/", "releases/tag/")
soup = BeautifulSoup(r.text, 'html.parser')
links = soup.find_all('a')
ll = [i.get("href") for i in links if i.get("href") is not None and i.get("href").startswith(url_path)]
versions = [self.newest_pip_version]
if self.newest_pip_version.endswith(".0"):
versions.append(self.newest_pip_version[:-2])
for v in versions:
try:
version_tag = next(filter(lambda i: i.endswith(v), ll)).split("/")[-1]
break
except StopIteration:
pass
else:
return None
if version_tag == self.newest_pip_version:
return None
else:
return version_tag
@property
def repo_type(self):
if self.name in self.config:
if "repo_type" in self.config[self.name]:
return self.config[self.name]["repo_type"]
return None
def require(self, via_pip):
......@@ -128,9 +176,8 @@ class Dependency:
if "; extra == 'redis'" in i:
i = i[:i.find("; extra")].strip()
if "; extra" in i:
print(i)
continue
dep = Dependency(i, split=False)
dep = Dependency(i, self.config, split=False)
if via_pip and dep.name not in via_pip:
continue
yield dep
......@@ -145,10 +192,13 @@ class Dependency:
lines.append(f"version => '{self.newest_pip_version}',")
if self.tag:
lines.append("tag => '{self.tag}',")
lines.append(f"tag => '{self.tag}',")
lines.append(f"url => '{self.url}',")
if self.repo_type:
lines.append(f"repo_type => '{self.repo_type}',")
for dep in self.require(via_pip):
lines.append(f"require => 'Exec[pip_install_{dep.name}]',")
......@@ -243,7 +293,6 @@ def main():
output = io.BytesIO()
b = repo.commit("weblate-{}".format(args.weblate_version)).tree/"requirements.txt"
b.stream_data(output)
deps = [Dependency(i) for i in output.getvalue().decode().splitlines()]
config = yaml.safe_load(pathlib.Path(__file__).with_name("override.yaml").open())
......@@ -253,6 +302,8 @@ def main():
pip_needs = set()
debian_needs = set(config.get('require_debian'))
deps = [Dependency(i, config['pip']) for i in output.getvalue().decode().splitlines()]
a_pkg = apt_cache['python3']
for v in a_pkg.version_list:
archive = suite(v)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment