Add a script to compile requirement files

.gitignore

@@ -36,3 +36,4 @@ TODO
 /scripts/testbrowser_cpp/Makefile
 /scripts/testbrowser_cpp/main.o
 /scripts/testbrowser_cpp/testbrowser
+/scripts/dev/pylint_checkers/qute_pylint.egg-info

misc/requirements/requirements-check-manifest-raw.txt

@@ -0,0 +1 @@
+check-manifest

misc/requirements/requirements-check-manifest.txt

@@ -1 +1,3 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
 check-manifest==0.31

misc/requirements/requirements-codecov-raw.txt

@@ -0,0 +1 @@
+codecov

misc/requirements/requirements-codecov.txt

@@ -1,3 +1,5 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
 codecov==2.0.5
 coverage==4.1
 requests==2.10.0

misc/requirements/requirements-cxfreeze-raw.txt

@@ -0,0 +1 @@
+cx_Freeze

misc/requirements/requirements-cxfreeze.txt

@@ -1 +1,3 @@
-cx_Freeze==4.3.4
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+cx-Freeze==4.3.4

misc/requirements/requirements-flake8-raw.txt

@@ -0,0 +1,21 @@
+ebb-lint
+flake8
+flake8-copyright
+flake8-debugger
+flake8-deprecated
+flake8-docstrings
+flake8-future-import
+flake8-mock
+flake8-pep3101
+flake8-putty
+flake8-string-format
+flake8-tidy-imports
+flake8-tuple
+hacking
+pep8-naming
+pydocstyle
+pyflakes
+
+mccabe==0.5.0
+
+# pep257: still needed by flake8-docstrings but ignored

misc/requirements/requirements-flake8.txt

@@ -1,3 +1,5 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
 ebb-lint==0.4.4
 flake8==2.5.4
 flake8-copyright==0.1

misc/requirements/requirements-pyinstaller-raw.txt

@@ -0,0 +1 @@
+PyInstaller

misc/requirements/requirements-pyinstaller.txt

@@ -1 +1,3 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
 PyInstaller==3.2

misc/requirements/requirements-pylint-master-raw.txt

@@ -0,0 +1,6 @@
+-e git+https://github.com/PyCQA/astroid.git#egg=astroid
+-e git+https://github.com/PyCQA/pylint.git#egg=pylint
+./scripts/dev/pylint_checkers
+
+# https://github.com/PyCQA/pylint/issues/932
+mccabe==0.5.0

misc/requirements/requirements-pylint-master.in

@@ -0,0 +1,2 @@
+-e git+https://github.com/PyCQA/astroid.git@master#egg=astroid
+-e git+https://github.com/PyCQA/pylint.git@master#egg=pylint

misc/requirements/requirements-pylint-master.txt

@@ -1,9 +1,10 @@
-git+https://github.com/PyCQA/astroid.git
+# This file is automatically generated by scripts/dev/recompile_requirements.py
-git+https://github.com/PyCQA/pylint.git
+
+-e git+https://github.com/PyCQA/astroid.git#egg=astroid
 isort==4.2.5
 lazy-object-proxy==1.2.2
 mccabe==0.5.0
+-e git+https://github.com/PyCQA/pylint.git#egg=pylint
+./scripts/dev/pylint_checkers
 six==1.10.0
 wrapt==1.10.8
-requests==2.10.0
-./scripts/dev/pylint_checkers

misc/requirements/requirements-pylint-raw.txt

@@ -0,0 +1,2 @@
+pylint
+./scripts/dev/pylint_checkers

misc/requirements/requirements-pylint.txt

@@ -1,8 +1,9 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
 astroid==1.4.6
 colorama==0.3.7
 lazy-object-proxy==1.2.2
 pylint==1.5.6
-requests==2.10.0
+./scripts/dev/pylint_checkers
 six==1.10.0
 wrapt==1.10.8
-./scripts/dev/pylint_checkers

misc/requirements/requirements-pyroma-raw.txt

@@ -0,0 +1 @@
+pyroma

misc/requirements/requirements-pyroma.txt

@@ -1,2 +1,4 @@
-pyroma==2.0.2
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
 docutils==0.12
+pyroma==2.0.2

misc/requirements/requirements-tests-raw.txt

@@ -0,0 +1,22 @@
+beautifulsoup4
+CherryPy
+coverage
+Flask==0.10.1
+httpbin
+hypothesis
+pytest
+pytest-bdd
+pytest-catchlog
+pytest-cov
+pytest-faulthandler
+pytest-instafail
+pytest-mock
+pytest-qt
+pytest-repeat
+pytest-rerunfailures
+pytest-travis-fold
+pytest-xvfb
+vulture
+
+# Flask: rq.filter: != 0.11.0
+# ignore: Jinja2

misc/requirements/requirements-tests.txt

@@ -1,3 +1,5 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
 beautifulsoup4==4.4.1
 CherryPy==6.0.1
 coverage==4.1
@@ -8,6 +10,7 @@ httpbin==0.4.1
 hypothesis==3.4.0
 itsdangerous==0.24
 Mako==1.0.4
+MarkupSafe==0.23
 parse==1.6.6
 parse-type==0.3.4
 py==1.4.31

misc/requirements/requirements-tox-raw.txt

@@ -0,0 +1 @@
+tox

misc/requirements/requirements-tox.txt

@@ -1,3 +1,5 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
 pluggy==0.3.1
 py==1.4.31
 tox==2.3.1

misc/requirements/requirements-vulture-raw.txt

@@ -0,0 +1 @@
+vulture

misc/requirements/requirements-vulture.txt

@@ -1 +1,3 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
 vulture==0.8.1

scripts/dev/recompile_requirements.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Script to regenerate requirements files in misc/requirements."""
+
+import re
+import sys
+import os.path
+import glob
+import subprocess
+import tempfile
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
+                                os.pardir))
+
+from scripts import utils
+
+REQ_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                       '..', '..',  # /scripts/dev -> /scripts -> /
+                       'misc', 'requirements')
+
+
+def convert_line(line, comments):
+    replacements = {
+        (r'@.*#', '#'),  # remove @commit-id for scm installs
+        (r'qute-pylint==.*', './scripts/dev/pylint_checkers'),
+    }
+    for pattern, repl in replacements:
+        line = re.sub(pattern, repl, line)
+
+    pkgname = line.split('=')[0]
+    ignored = comments.get('ignore', '').split(',')
+
+    if pkgname in ignored:
+        return None
+
+    if pkgname in comments:
+        line += '  # ' + comments[pkgname]
+
+    return line
+
+
+def read_comments(fobj):
+    comments = {}
+    for line in fobj:
+        if line.startswith('#') and ':' in line:
+            pkg, comment = line.split(':', maxsplit=1)
+            pkg = pkg.lstrip('# ')
+            comment = comment.strip()
+            comments[pkg] = comment
+    return comments
+
+
+def get_all_names():
+    for filename in glob.glob(os.path.join(REQ_DIR, 'requirements-*-raw.txt')):
+        basename = os.path.basename(filename)
+        yield basename[len('requirements-'):-len('-raw.txt')]
+
+
+def main():
+    names = sys.argv[1:] if len(sys.argv) > 1 else get_all_names()
+
+    for name in names:
+        utils.print_title(name)
+        filename = os.path.join(REQ_DIR,
+                                'requirements-{}-raw.txt'.format(name))
+        outfile = os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name))
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            pip_bin = os.path.join(tmpdir, 'bin', 'pip')
+            subprocess.check_call(['virtualenv', tmpdir])
+            subprocess.check_call([pip_bin, 'install', '-r', filename])
+            reqs = subprocess.check_output([pip_bin, 'freeze']).decode('utf-8')
+
+        with open(filename, 'r', encoding='utf-8') as f:
+            comments = read_comments(f)
+
+        with open(outfile, 'w', encoding='utf-8') as f:
+            f.write("# This file is automatically generated by "
+                    "scripts/dev/recompile_requirements.py\n\n")
+            for line in reqs.splitlines():
+                converted = convert_line(line, comments)
+                if converted is not None:
+                    f.write(converted + '\n')
+
+
+if __name__ == '__main__':
+    main()