Update recompile_requirements for newer pips
pip now depends on (instead of vendoring) its dependencies, so we needed some updates there to only install the affected packages once. Fixes #2255
This commit is contained in:
Florian Bruhin
parent
288c484cdf
commit
8afc215c3d
@ -60,6 +60,20 @@ def convert_line(line, comments):
|
||||
return line
|
||||
|
||||
|
||||
def get_requirements(requirements_file, exclude=()):
|
||||
"""Get the requirements after freezing with the given file."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
pip_bin = os.path.join(tmpdir, 'bin', 'pip')
|
||||
subprocess.check_call(['virtualenv', tmpdir])
|
||||
if requirements_file is not None:
|
||||
subprocess.check_call([pip_bin, 'install', '-r',
|
||||
requirements_file])
|
||||
out = subprocess.check_output([pip_bin, 'freeze', '--all'],
|
||||
universal_newlines=True)
|
||||
|
||||
return [line for line in out.splitlines() if line not in exclude]
|
||||
|
||||
|
||||
def read_comments(fobj):
|
||||
"""Find special comments in the config.
|
||||
|
||||
@ -98,36 +112,52 @@ def get_all_names():
|
||||
"""Get all requirement names based on filenames."""
|
||||
for filename in glob.glob(os.path.join(REQ_DIR, 'requirements-*.txt-raw')):
|
||||
basename = os.path.basename(filename)
|
||||
yield basename[len('requirements-'):-len('.txt-raw')]
|
||||
name = basename[len('requirements-'):-len('.txt-raw')]
|
||||
if name == 'cxfreeze' and sys.hexversion >= 0x030600:
|
||||
print("Warning: Skipping cxfreeze")
|
||||
else:
|
||||
yield name
|
||||
yield 'pip'
|
||||
|
||||
|
||||
def main():
|
||||
"""Re-compile the given (or all) requirement files."""
|
||||
names = sys.argv[1:] if len(sys.argv) > 1 else get_all_names()
|
||||
names = sys.argv[1:] if len(sys.argv) > 1 else sorted(get_all_names())
|
||||
|
||||
utils.print_title('pip')
|
||||
pip_requirements = get_requirements(None)
|
||||
|
||||
for name in names:
|
||||
utils.print_title(name)
|
||||
filename = os.path.join(REQ_DIR,
|
||||
'requirements-{}.txt-raw'.format(name))
|
||||
|
||||
if name == 'qutebrowser':
|
||||
outfile = os.path.join(REPO_DIR, 'requirements.txt')
|
||||
else:
|
||||
outfile = os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name))
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
pip_bin = os.path.join(tmpdir, 'bin', 'pip')
|
||||
subprocess.check_call(['virtualenv', tmpdir])
|
||||
subprocess.check_call([pip_bin, 'install', '-r', filename])
|
||||
reqs = subprocess.check_output([pip_bin, 'freeze']).decode('utf-8')
|
||||
if name == 'pip':
|
||||
requirements = [req for req in pip_requirements
|
||||
if not req.startswith('pip==')]
|
||||
comments = {
|
||||
'filter': {},
|
||||
'comment': {},
|
||||
'ignore': [],
|
||||
'replace': {},
|
||||
}
|
||||
else:
|
||||
filename = os.path.join(REQ_DIR,
|
||||
'requirements-{}.txt-raw'.format(name))
|
||||
requirements = get_requirements(filename, exclude=pip_requirements)
|
||||
|
||||
with open(filename, 'r', encoding='utf-8') as f:
|
||||
comments = read_comments(f)
|
||||
with open(filename, 'r', encoding='utf-8') as f:
|
||||
comments = read_comments(f)
|
||||
|
||||
with open(outfile, 'w', encoding='utf-8') as f:
|
||||
f.write("# This file is automatically generated by "
|
||||
"scripts/dev/recompile_requirements.py\n\n")
|
||||
for line in reqs.splitlines():
|
||||
f.write(convert_line(line, comments) + '\n')
|
||||
for line in requirements:
|
||||
converted = convert_line(line, comments)
|
||||
f.write(converted + '\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
10
tox.ini
10
tox.ini
@ -15,6 +15,7 @@ setenv =
|
||||
PYTEST_QT_API=pyqt5
|
||||
passenv = PYTHON DISPLAY XAUTHORITY HOME USERNAME USER CI TRAVIS XDG_* QUTE_* DOCKER
|
||||
deps =
|
||||
-r{toxinidir}/misc/requirements/requirements-pip.txt
|
||||
-r{toxinidir}/requirements.txt
|
||||
-r{toxinidir}/misc/requirements/requirements-tests.txt
|
||||
commands =
|
||||
@ -57,6 +58,7 @@ commands = {envpython} scripts/link_pyqt.py --tox {envdir}
|
||||
envdir = {toxinidir}/.venv
|
||||
usedevelop = true
|
||||
deps =
|
||||
-r{toxinidir}/misc/requirements/requirements-pip.txt
|
||||
-r{toxinidir}/requirements.txt
|
||||
|
||||
# This is used for Windows, since binary name is different
|
||||
@ -66,6 +68,7 @@ commands = {envpython} scripts/link_pyqt.py --tox {envdir}
|
||||
envdir = {toxinidir}/.venv
|
||||
usedevelop = true
|
||||
deps =
|
||||
-r{toxinidir}/misc/requirements/requirements-pip.txt
|
||||
-r{toxinidir}/requirements.txt
|
||||
|
||||
# This is undocumented, but it's a common typo, so let's make it work
|
||||
@ -102,6 +105,7 @@ commands =
|
||||
[testenv:vulture]
|
||||
basepython = python3
|
||||
deps =
|
||||
-r{toxinidir}/misc/requirements/requirements-pip.txt
|
||||
-r{toxinidir}/requirements.txt
|
||||
-r{toxinidir}/misc/requirements/requirements-vulture.txt
|
||||
setenv = PYTHONPATH={toxinidir}
|
||||
@ -136,6 +140,7 @@ commands =
|
||||
basepython = python3
|
||||
passenv =
|
||||
deps =
|
||||
-r{toxinidir}/misc/requirements/requirements-pip.txt
|
||||
-r{toxinidir}/requirements.txt
|
||||
-r{toxinidir}/misc/requirements/requirements-flake8.txt
|
||||
commands =
|
||||
@ -145,6 +150,7 @@ commands =
|
||||
basepython = python3
|
||||
passenv =
|
||||
deps =
|
||||
-r{toxinidir}/misc/requirements/requirements-pip.txt
|
||||
-r{toxinidir}/misc/requirements/requirements-pyroma.txt
|
||||
commands =
|
||||
{envdir}/bin/pyroma .
|
||||
@ -153,6 +159,7 @@ commands =
|
||||
basepython = python3
|
||||
passenv =
|
||||
deps =
|
||||
-r{toxinidir}/misc/requirements/requirements-pip.txt
|
||||
-r{toxinidir}/misc/requirements/requirements-check-manifest.txt
|
||||
commands =
|
||||
{envdir}/bin/check-manifest --ignore 'qutebrowser/git-commit-id,qutebrowser/html/doc,qutebrowser/html/doc/*,*/__pycache__'
|
||||
@ -162,6 +169,7 @@ basepython = python3
|
||||
whitelist_externals = git
|
||||
passenv = TRAVIS_PULL_REQUEST
|
||||
deps =
|
||||
-r{toxinidir}/misc/requirements/requirements-pip.txt
|
||||
-r{toxinidir}/requirements.txt
|
||||
commands =
|
||||
{envpython} scripts/link_pyqt.py --tox {envdir}
|
||||
@ -174,6 +182,7 @@ commands =
|
||||
# fail if we didn't have a fallback defined.
|
||||
basepython = {env:PYTHON:}/python.exe
|
||||
deps =
|
||||
-r{toxinidir}/misc/requirements/requirements-pip.txt
|
||||
-r{toxinidir}/requirements.txt
|
||||
-r{toxinidir}/misc/requirements/requirements-cxfreeze.txt
|
||||
commands =
|
||||
@ -184,6 +193,7 @@ commands =
|
||||
[testenv:pyinstaller]
|
||||
basepython = python3
|
||||
deps =
|
||||
-r{toxinidir}/misc/requirements/requirements-pip.txt
|
||||
-r{toxinidir}/requirements.txt
|
||||
-r{toxinidir}/misc/requirements/requirements-pyinstaller.txt
|
||||
commands =
|
||||
|
||||
Loading…
Reference in New Issue
Block a user