Update recompile_requirements for newer pips

pip now depends on (instead of vendoring) its dependencies, so we needed some
updates there to only install the affected packages once.

Fixes #2255

scripts/dev/recompile_requirements.py

@@ -60,6 +60,20 @@ def convert_line(line, comments):
     return line
 
 
+def get_requirements(requirements_file, exclude=()):
+    """Get the requirements after freezing with the given file."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        pip_bin = os.path.join(tmpdir, 'bin', 'pip')
+        subprocess.check_call(['virtualenv', tmpdir])
+        if requirements_file is not None:
+            subprocess.check_call([pip_bin, 'install', '-r',
+                                   requirements_file])
+        out = subprocess.check_output([pip_bin, 'freeze', '--all'],
+                                      universal_newlines=True)
+
+    return [line for line in out.splitlines() if line not in exclude]
+
+
 def read_comments(fobj):
     """Find special comments in the config.
 
@@ -98,36 +112,52 @@ def get_all_names():
     """Get all requirement names based on filenames."""
     for filename in glob.glob(os.path.join(REQ_DIR, 'requirements-*.txt-raw')):
         basename = os.path.basename(filename)
-        yield basename[len('requirements-'):-len('.txt-raw')]
+        name = basename[len('requirements-'):-len('.txt-raw')]
+        if name == 'cxfreeze' and sys.hexversion >= 0x030600:
+            print("Warning: Skipping cxfreeze")
+        else:
+            yield name
+    yield 'pip'
 
 
 def main():
     """Re-compile the given (or all) requirement files."""
-    names = sys.argv[1:] if len(sys.argv) > 1 else get_all_names()
+    names = sys.argv[1:] if len(sys.argv) > 1 else sorted(get_all_names())
+
+    utils.print_title('pip')
+    pip_requirements = get_requirements(None)
 
     for name in names:
         utils.print_title(name)
-        filename = os.path.join(REQ_DIR,
+
-                                'requirements-{}.txt-raw'.format(name))
         if name == 'qutebrowser':
             outfile = os.path.join(REPO_DIR, 'requirements.txt')
         else:
             outfile = os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name))
 
-        with tempfile.TemporaryDirectory() as tmpdir:
+        if name == 'pip':
-            pip_bin = os.path.join(tmpdir, 'bin', 'pip')
+            requirements = [req for req in pip_requirements
-            subprocess.check_call(['virtualenv', tmpdir])
+                            if not req.startswith('pip==')]
-            subprocess.check_call([pip_bin, 'install', '-r', filename])
+            comments = {
-            reqs = subprocess.check_output([pip_bin, 'freeze']).decode('utf-8')
+                'filter': {},
+                'comment': {},
+                'ignore': [],
+                'replace': {},
+            }
+        else:
+            filename = os.path.join(REQ_DIR,
+                                    'requirements-{}.txt-raw'.format(name))
+            requirements = get_requirements(filename, exclude=pip_requirements)
 
-        with open(filename, 'r', encoding='utf-8') as f:
+            with open(filename, 'r', encoding='utf-8') as f:
-            comments = read_comments(f)
+                comments = read_comments(f)
 
         with open(outfile, 'w', encoding='utf-8') as f:
             f.write("# This file is automatically generated by "
                     "scripts/dev/recompile_requirements.py\n\n")
-            for line in reqs.splitlines():
+            for line in requirements:
-                f.write(convert_line(line, comments) + '\n')
+                converted = convert_line(line, comments)
+                f.write(converted + '\n')
 
 
 if __name__ == '__main__':

tox.ini

@@ -15,6 +15,7 @@ setenv =
     PYTEST_QT_API=pyqt5
 passenv = PYTHON DISPLAY XAUTHORITY HOME USERNAME USER CI TRAVIS XDG_* QUTE_* DOCKER
 deps =
+    -r{toxinidir}/misc/requirements/requirements-pip.txt
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/misc/requirements/requirements-tests.txt
 commands =
@@ -57,6 +58,7 @@ commands = {envpython} scripts/link_pyqt.py --tox {envdir}
 envdir = {toxinidir}/.venv
 usedevelop = true
 deps =
+    -r{toxinidir}/misc/requirements/requirements-pip.txt
     -r{toxinidir}/requirements.txt
 
 # This is used for Windows, since binary name is different
@@ -66,6 +68,7 @@ commands = {envpython} scripts/link_pyqt.py --tox {envdir}
 envdir = {toxinidir}/.venv
 usedevelop = true
 deps =
+    -r{toxinidir}/misc/requirements/requirements-pip.txt
     -r{toxinidir}/requirements.txt
 
 # This is undocumented, but it's a common typo, so let's make it work
@@ -102,6 +105,7 @@ commands =
 [testenv:vulture]
 basepython = python3
 deps =
+    -r{toxinidir}/misc/requirements/requirements-pip.txt
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/misc/requirements/requirements-vulture.txt
 setenv = PYTHONPATH={toxinidir}
@@ -136,6 +140,7 @@ commands =
 basepython = python3
 passenv =
 deps =
+    -r{toxinidir}/misc/requirements/requirements-pip.txt
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/misc/requirements/requirements-flake8.txt
 commands =
@@ -145,6 +150,7 @@ commands =
 basepython = python3
 passenv =
 deps =
+    -r{toxinidir}/misc/requirements/requirements-pip.txt
     -r{toxinidir}/misc/requirements/requirements-pyroma.txt
 commands =
     {envdir}/bin/pyroma .
@@ -153,6 +159,7 @@ commands =
 basepython = python3
 passenv =
 deps =
+    -r{toxinidir}/misc/requirements/requirements-pip.txt
      -r{toxinidir}/misc/requirements/requirements-check-manifest.txt
 commands =
     {envdir}/bin/check-manifest --ignore 'qutebrowser/git-commit-id,qutebrowser/html/doc,qutebrowser/html/doc/*,*/__pycache__'
@@ -162,6 +169,7 @@ basepython = python3
 whitelist_externals = git
 passenv = TRAVIS_PULL_REQUEST
 deps =
+    -r{toxinidir}/misc/requirements/requirements-pip.txt
     -r{toxinidir}/requirements.txt
 commands =
     {envpython} scripts/link_pyqt.py --tox {envdir}
@@ -174,6 +182,7 @@ commands =
 # fail if we didn't have a fallback defined.
 basepython = {env:PYTHON:}/python.exe
 deps =
+    -r{toxinidir}/misc/requirements/requirements-pip.txt
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/misc/requirements/requirements-cxfreeze.txt
 commands =
@@ -184,6 +193,7 @@ commands =
 [testenv:pyinstaller]
 basepython = python3
 deps =
+    -r{toxinidir}/misc/requirements/requirements-pip.txt
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/misc/requirements/requirements-pyinstaller.txt
 commands =