From 8afc215c3d299140e0b5f50b1c638987a36c59f9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 5 Feb 2017 01:29:21 +0100 Subject: [PATCH] Update recompile_requirements for newer pips pip now depends on (instead of vendoring) its dependencies, so we needed some updates there to only install the affected packages once. Fixes #2255 --- scripts/dev/recompile_requirements.py | 56 ++++++++++++++++++++------- tox.ini | 10 +++++ 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scripts/dev/recompile_requirements.py b/scripts/dev/recompile_requirements.py index 65c6ad5c8..1e3181344 100644 --- a/scripts/dev/recompile_requirements.py +++ b/scripts/dev/recompile_requirements.py @@ -60,6 +60,20 @@ def convert_line(line, comments): return line +def get_requirements(requirements_file, exclude=()): + """Get the requirements after freezing with the given file.""" + with tempfile.TemporaryDirectory() as tmpdir: + pip_bin = os.path.join(tmpdir, 'bin', 'pip') + subprocess.check_call(['virtualenv', tmpdir]) + if requirements_file is not None: + subprocess.check_call([pip_bin, 'install', '-r', + requirements_file]) + out = subprocess.check_output([pip_bin, 'freeze', '--all'], + universal_newlines=True) + + return [line for line in out.splitlines() if line not in exclude] + + def read_comments(fobj): """Find special comments in the config. @@ -98,36 +112,52 @@ def get_all_names(): """Get all requirement names based on filenames.""" for filename in glob.glob(os.path.join(REQ_DIR, 'requirements-*.txt-raw')): basename = os.path.basename(filename) - yield basename[len('requirements-'):-len('.txt-raw')] + name = basename[len('requirements-'):-len('.txt-raw')] + if name == 'cxfreeze' and sys.hexversion >= 0x030600: + print("Warning: Skipping cxfreeze") + else: + yield name + yield 'pip' def main(): """Re-compile the given (or all) requirement files.""" - names = sys.argv[1:] if len(sys.argv) > 1 else get_all_names() + names = sys.argv[1:] if len(sys.argv) > 1 else sorted(get_all_names()) + + utils.print_title('pip') + pip_requirements = get_requirements(None) for name in names: utils.print_title(name) - filename = os.path.join(REQ_DIR, - 'requirements-{}.txt-raw'.format(name)) + if name == 'qutebrowser': outfile = os.path.join(REPO_DIR, 'requirements.txt') else: outfile = os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name)) - with tempfile.TemporaryDirectory() as tmpdir: - pip_bin = os.path.join(tmpdir, 'bin', 'pip') - subprocess.check_call(['virtualenv', tmpdir]) - subprocess.check_call([pip_bin, 'install', '-r', filename]) - reqs = subprocess.check_output([pip_bin, 'freeze']).decode('utf-8') + if name == 'pip': + requirements = [req for req in pip_requirements + if not req.startswith('pip==')] + comments = { + 'filter': {}, + 'comment': {}, + 'ignore': [], + 'replace': {}, + } + else: + filename = os.path.join(REQ_DIR, + 'requirements-{}.txt-raw'.format(name)) + requirements = get_requirements(filename, exclude=pip_requirements) - with open(filename, 'r', encoding='utf-8') as f: - comments = read_comments(f) + with open(filename, 'r', encoding='utf-8') as f: + comments = read_comments(f) with open(outfile, 'w', encoding='utf-8') as f: f.write("# This file is automatically generated by " "scripts/dev/recompile_requirements.py\n\n") - for line in reqs.splitlines(): - f.write(convert_line(line, comments) + '\n') + for line in requirements: + converted = convert_line(line, comments) + f.write(converted + '\n') if __name__ == '__main__': diff --git a/tox.ini b/tox.ini index 0ff46d232..8d31dd961 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,7 @@ setenv = PYTEST_QT_API=pyqt5 passenv = PYTHON DISPLAY XAUTHORITY HOME USERNAME USER CI TRAVIS XDG_* QUTE_* DOCKER deps = + -r{toxinidir}/misc/requirements/requirements-pip.txt -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-tests.txt commands = @@ -57,6 +58,7 @@ commands = {envpython} scripts/link_pyqt.py --tox {envdir} envdir = {toxinidir}/.venv usedevelop = true deps = + -r{toxinidir}/misc/requirements/requirements-pip.txt -r{toxinidir}/requirements.txt # This is used for Windows, since binary name is different @@ -66,6 +68,7 @@ commands = {envpython} scripts/link_pyqt.py --tox {envdir} envdir = {toxinidir}/.venv usedevelop = true deps = + -r{toxinidir}/misc/requirements/requirements-pip.txt -r{toxinidir}/requirements.txt # This is undocumented, but it's a common typo, so let's make it work @@ -102,6 +105,7 @@ commands = [testenv:vulture] basepython = python3 deps = + -r{toxinidir}/misc/requirements/requirements-pip.txt -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-vulture.txt setenv = PYTHONPATH={toxinidir} @@ -136,6 +140,7 @@ commands = basepython = python3 passenv = deps = + -r{toxinidir}/misc/requirements/requirements-pip.txt -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-flake8.txt commands = @@ -145,6 +150,7 @@ commands = basepython = python3 passenv = deps = + -r{toxinidir}/misc/requirements/requirements-pip.txt -r{toxinidir}/misc/requirements/requirements-pyroma.txt commands = {envdir}/bin/pyroma . @@ -153,6 +159,7 @@ commands = basepython = python3 passenv = deps = + -r{toxinidir}/misc/requirements/requirements-pip.txt -r{toxinidir}/misc/requirements/requirements-check-manifest.txt commands = {envdir}/bin/check-manifest --ignore 'qutebrowser/git-commit-id,qutebrowser/html/doc,qutebrowser/html/doc/*,*/__pycache__' @@ -162,6 +169,7 @@ basepython = python3 whitelist_externals = git passenv = TRAVIS_PULL_REQUEST deps = + -r{toxinidir}/misc/requirements/requirements-pip.txt -r{toxinidir}/requirements.txt commands = {envpython} scripts/link_pyqt.py --tox {envdir} @@ -174,6 +182,7 @@ commands = # fail if we didn't have a fallback defined. basepython = {env:PYTHON:}/python.exe deps = + -r{toxinidir}/misc/requirements/requirements-pip.txt -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-cxfreeze.txt commands = @@ -184,6 +193,7 @@ commands = [testenv:pyinstaller] basepython = python3 deps = + -r{toxinidir}/misc/requirements/requirements-pip.txt -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-pyinstaller.txt commands =