diff --git a/.run_checks b/.run_checks deleted file mode 100644 index 2c435ab40..000000000 --- a/.run_checks +++ /dev/null @@ -1,21 +0,0 @@ -# vim: ft=dosini - -[DEFAULT] -targets=qutebrowser,scripts - -[pep257] -# D102: Docstring missing, will be handled by others -# D209: Blank line before closing """ (removed from PEP257) -# D402: First line should not be function's signature (false-positives) -disable=D102,D209,D402 -exclude=resources.py,test_content_disposition.py - -[pylint] -args=--output-format=colorized,--reports=no,--rcfile=.pylintrc -plugins=config,crlf,modeline,settrace,openencoding -# excluding command.py is a WORKAROUND for https://bitbucket.org/logilab/pylint/issue/395/horrible-performance-related-to-inspect -exclude=resources.py - -[flake8] -args=--config=.flake8 -exclude=resources.py diff --git a/scripts/init_venv.py b/scripts/init_venv.py deleted file mode 100644 index 3df298b61..000000000 --- a/scripts/init_venv.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env python3 -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2014-2015 Florian Bruhin (The Compiler) - -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Initialize a venv suitable to be used for qutebrowser.""" - -import os -import re -import sys -import glob -import os.path -import shutil -import argparse -import subprocess -import distutils.sysconfig # pylint: disable=import-error -# see https://bitbucket.org/logilab/pylint/issue/73/ -import venv -import urllib.request -import tempfile - -from PyQt5.QtCore import QStandardPaths - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) -from scripts import utils - - -try: - import ensurepip # pylint: disable=import-error -except ImportError: - # Debian-like systems don't have ensurepip... - ensurepip = None - - -g_path = None -g_args = None - - -def parse_args(): - """Parse the commandline arguments.""" - parser = argparse.ArgumentParser() - parser.add_argument('--clear', help="Clear venv in case it already " - "exists.", action='store_true') - parser.add_argument('--upgrade', help="Upgrade venv to use this version " - "of Python, assuming Python has been upgraded " - "in-place.", action='store_true') - parser.add_argument('--force', help=argparse.SUPPRESS, - action='store_true') - parser.add_argument('--dev', help="Set up an environment suitable for " - "developing qutebrowser.", - action='store_true') - parser.add_argument('--cache', help="Cache the clean virtualenv and " - "copy it when a new one is requested.", - default=None, nargs='?', const='', metavar='NAME') - parser.add_argument('path', help="Path to the venv folder", - default='.venv', nargs='?') - return parser.parse_args() - - -def get_dev_packages(short=False): - """Get a list of packages to install. - - Args: - short: Remove the version specification. - """ - packages = ['colorlog', 'flake8', 'astroid', 'pylint', 'pep257', - 'colorama', 'beautifulsoup4', 'coverage', 'pyroma', - 'check-manifest'] - if short: - packages = [re.split(r'[<>=]', p)[0] for p in packages] - return packages - - -def install_dev_packages(): - """Install the packages needed for development.""" - for pkg in get_dev_packages(): - utils.print_subtitle("Installing {}".format(pkg)) - venv_python('-m', 'pip', 'install', '--upgrade', pkg) - - -def venv_python(*args, output=False): - """Call the venv's python with the given arguments.""" - subdir = 'Scripts' if os.name == 'nt' else 'bin' - executable = os.path.join(g_path, subdir, 'python') - env = dict(os.environ) - if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env: - # WORKAROUND for https://github.com/pypa/pip/issues/2031 - del env['__PYVENV_LAUNCHER__'] - if output: - return subprocess.check_output([executable] + list(args), - universal_newlines=True, env=env) - else: - subprocess.check_call([executable] + list(args), env=env) - - -def test_toolchain(): - """Test if imports work properly.""" - utils.print_title("Checking toolchain") - - packages = ['sip', 'PyQt5.QtCore', 'PyQt5.QtWebKit', 'qutebrowser.app'] - renames = {'beautifulsoup4': 'bs4', 'check-manifest': 'check_manifest'} - if g_args.dev: - packages += get_dev_packages(short=True) - for pkg in packages: - try: - pkg = renames[pkg] - except KeyError: - pass - print("Importing {}".format(pkg)) - venv_python('-c', 'import {}'.format(pkg)) - - -def verbose_copy(src, dst, *, follow_symlinks=True): - """Copy function for shutil.copytree which prints copied files.""" - print('{} -> {}'.format(src, dst)) - shutil.copy(src, dst, follow_symlinks=follow_symlinks) - - -def get_ignored_files(directory, files): - """Get the files which should be ignored for link_pyqt() on Windows.""" - needed_exts = ('.py', '.dll', '.pyd', '.so') - ignored_dirs = ('examples', 'qml', 'uic', 'doc') - filtered = [] - for f in files: - ext = os.path.splitext(f)[1] - full_path = os.path.join(directory, f) - if os.path.isdir(full_path) and f in ignored_dirs: - filtered.append(f) - elif (ext not in needed_exts) and os.path.isfile(full_path): - filtered.append(f) - return filtered - - -def link_pyqt(): - """Symlink the systemwide PyQt/sip into the venv.""" - action = "Copying" if os.name == 'nt' else "Softlinking" - utils.print_title("{} PyQt5".format(action)) - sys_path = distutils.sysconfig.get_python_lib() - venv_path = venv_python( - '-c', 'from distutils.sysconfig import get_python_lib\n' - 'print(get_python_lib())', output=True).rstrip() - - globbed_sip = (glob.glob(os.path.join(sys_path, 'sip*.so')) + - glob.glob(os.path.join(sys_path, 'sip*.pyd'))) - if not globbed_sip: - print("Did not find sip in {}!".format(sys_path), file=sys.stderr) - sys.exit(1) - - files = [ - 'PyQt5', - ] - files += [os.path.basename(e) for e in globbed_sip] - for fn in files: - source = os.path.join(sys_path, fn) - dest = os.path.join(venv_path, fn) - if not os.path.exists(source): - raise FileNotFoundError(source) - if os.path.exists(dest): - if os.path.isdir(dest) and not os.path.islink(dest): - shutil.rmtree(dest) - else: - os.unlink(dest) - if os.name == 'nt': - if os.path.isdir(source): - shutil.copytree(source, dest, ignore=get_ignored_files, - copy_function=verbose_copy) - else: - print('{} -> {}'.format(source, dest)) - shutil.copy(source, dest) - else: - print('{} -> {}'.format(source, dest)) - os.symlink(source, dest) - - -def install_pip(): - """Install pip on Debian-like systems which don't have ensurepip. - - WORKAROUND for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=772730 and - https://bugs.launchpad.net/ubuntu/+source/python3.4/+bug/1290847 - """ - utils.print_title("Installing pip/setuptools") - f = urllib.request.urlopen('https://bootstrap.pypa.io/get-pip.py') - with tempfile.NamedTemporaryFile() as tmp: - tmp.write(f.read()) - venv_python(tmp.name) - - -def create_venv(): - """Create a new venv.""" - utils.print_title("Creating venv") - if os.name == 'nt': - symlinks = False - else: - symlinks = True - clear = g_args.clear or g_args.force - upgrade = g_args.upgrade or g_args.cache is not None - builder = venv.EnvBuilder(system_site_packages=False, - clear=clear, upgrade=upgrade, - symlinks=symlinks, with_pip=ensurepip) - builder.create(g_path) - # If we don't have ensurepip, we have to do it by hand... - if not ensurepip: - install_pip() - - -def restore_cache(cache_path): - """Restore a cache if one is present and --cache is given.""" - if g_args.cache is not None: - utils.print_title("Restoring cache") - print("Restoring {} to {}...".format(cache_path, g_args.path)) - try: - shutil.rmtree(g_args.path) - except FileNotFoundError: - pass - try: - shutil.copytree(cache_path, g_args.path, symlinks=True) - except FileNotFoundError: - print("No cache present!") - else: - return True - return False - - -def save_cache(cache_path): - """Save the cache if --cache is given.""" - if g_args.cache is not None: - utils.print_title("Saving cache") - print("Saving {} to {}...".format(g_args.path, cache_path)) - try: - shutil.rmtree(cache_path) - except FileNotFoundError: - pass - shutil.copytree(g_args.path, cache_path, symlinks=True) - - -def main(): - """Main entry point.""" - global g_path, g_args - g_args = parse_args() - if not g_args.path: - print("Refusing to run with empty path!", file=sys.stderr) - sys.exit(1) - g_path = os.path.abspath(g_args.path) - - if os.path.exists(g_args.path) and not (g_args.force or g_args.clear or - g_args.upgrade): - print("{} does already exist! Use --clear or " - "--upgrade.".format(g_path), file=sys.stderr) - sys.exit(1) - - os_cache_dir = QStandardPaths.writableLocation( - QStandardPaths.CacheLocation) - file_name = 'qutebrowser-venv' - if g_args.cache: - file_name += '-' + g_args.cache - cache_path = os.path.join(os_cache_dir, file_name) - - restored = restore_cache(cache_path) - if not restored: - create_venv() - - utils.print_title("Calling: setup.py develop") - venv_python('setup.py', 'develop') - - if g_args.dev: - utils.print_title("Installing developer packages") - install_dev_packages() - link_pyqt() - test_toolchain() - save_cache(cache_path) - - -if __name__ == '__main__': - main() diff --git a/scripts/run_checks.py b/scripts/run_checks.py deleted file mode 100755 index 16c168672..000000000 --- a/scripts/run_checks.py +++ /dev/null @@ -1,392 +0,0 @@ -#!/usr/bin/env python3 -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2014-2015 Florian Bruhin (The Compiler) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Run different codecheckers over a codebase. - -Runs flake8, pylint, pep257, a CRLF/whitespace/conflict-checker and -pyroma/check-manifest by default. - -Module attributes: - option: A dictionary with options. -""" - -import sys -import subprocess -import os -import io -import os.path -import unittest -import tokenize -import configparser -import argparse -import collections -import functools -import contextlib -import traceback - -import pep257 -import coverage - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) - -from scripts import utils - - -config = configparser.ConfigParser() - - -@contextlib.contextmanager -def _adjusted_pythonpath(name): - """Adjust PYTHONPATH for pylint.""" - if name == 'pylint': - scriptdir = os.path.abspath(os.path.dirname(__file__)) - if 'PYTHONPATH' in os.environ: - old_pythonpath = os.environ['PYTHONPATH'] - os.environ['PYTHONPATH'] += os.pathsep + scriptdir - else: - old_pythonpath = None - os.environ['PYTHONPATH'] = scriptdir - yield - if name == 'pylint': - if old_pythonpath is not None: - os.environ['PYTHONPATH'] = old_pythonpath - else: - del os.environ['PYTHONPATH'] - - -def run(name, target=None, print_version=False): - """Run a checker via distutils with optional args. - - Arguments: - name: Name of the checker/binary - target: The package to check - print_version: Whether to print the checker version. - """ - args = _get_args(name) - if target is not None: - args.append(target) - with _adjusted_pythonpath(name): - if os.name == 'nt': - exename = name + '.exe' - else: - exename = name - # for virtualenvs - executable = os.path.join(os.path.dirname(sys.executable), exename) - if not os.path.exists(executable): - # in $PATH - executable = name - if print_version: - subprocess.call([executable, '--version']) - try: - status = subprocess.call([executable] + args) - except OSError: - traceback.print_exc() - status = None - print() - return status - - -def check_pep257(target, print_version=False): - """Run pep257 checker with args passed. - - We use this rather than run() because on some systems (e.g. Windows) no - pep257 binary is available. - """ - if print_version: - print(pep257.__version__) - args = _get_args('pep257') - sys.argv = ['pep257', target] - if args is not None: - sys.argv += args - try: - # pylint: disable=assignment-from-no-return,no-member - if hasattr(pep257, 'run_pep257'): - # newer pep257 versions - status = pep257.run_pep257() - else: - # older pep257 versions - status = pep257.main(*pep257.parse_options()) - print() - return status - except Exception: - traceback.print_exc() - return None - - -def check_init(target): - """Check if every subdir of target has an __init__.py file.""" - ok = True - for dirpath, _dirnames, filenames in os.walk(target): - if any(f.endswith('.py') for f in filenames): - if '__init__.py' not in filenames: - utils.print_col("Missing __init__.py in {}!".format(dirpath), - 'red') - ok = False - print() - return ok - - -def check_unittest(run_coverage, verbose): - """Run the unittest checker. - - Args: - run_coverage: Whether to also run coverage.py. - verbose: For verbose output. - """ - if run_coverage: - cov = coverage.coverage(branch=True, source=['qutebrowser']) - cov.erase() - cov.start() - suite = unittest.TestLoader().discover('.') - verbosity = 2 if verbose else 1 - result = unittest.TextTestRunner(verbosity=verbosity).run(suite) - if run_coverage: - cov.stop() - perc = cov.report(file=io.StringIO()) - print("COVERAGE: {}%".format(round(perc))) - cov.html_report() - print() - return result.wasSuccessful() - - -def check_git(): - """Check for uncommited git files..""" - if not os.path.isdir(".git"): - print("No .git dir, ignoring") - print() - return False - untracked = [] - changed = [] - gitst = subprocess.check_output(['git', 'status', '--porcelain']) - gitst = gitst.decode('UTF-8').strip() - for line in gitst.splitlines(): - s, name = line.split(maxsplit=1) - if s == '??' and name != '.venv/': - untracked.append(name) - elif s == 'M': - changed.append(name) - status = True - if untracked: - status = False - utils.print_col("Untracked files:", 'red') - print('\n'.join(untracked)) - if changed: - status = False - utils.print_col("Uncommited changes:", 'red') - print('\n'.join(changed)) - print() - return status - - -def check_vcs_conflict(target): - """Check VCS conflict markers.""" - try: - ok = True - for (dirpath, _dirnames, filenames) in os.walk(target): - for name in (e for e in filenames if e.endswith('.py')): - fn = os.path.join(dirpath, name) - with tokenize.open(fn) as f: - for line in f: - if any(line.startswith(c * 7) for c in '<>=|'): - print("Found conflict marker in {}".format(fn)) - ok = False - print() - return ok - except Exception: - traceback.print_exc() - return None - - -def _get_optional_args(checker): - """Get a list of arguments based on a comma-separated args config.""" - # pylint: disable=no-member - # https://bitbucket.org/logilab/pylint/issue/487/ - try: - return config.get(checker, 'args').split(',') - except configparser.NoOptionError: - return [] - - -def _get_flag(arg, checker, option): - """Get a list of arguments based on a config option.""" - try: - return ['--{}={}'.format(arg, config.get(checker, option))] - except configparser.NoOptionError: - return [] - - -def _get_args(checker): - """Construct the arguments for a given checker. - - Return: - A list of commandline arguments. - """ - # pylint: disable=no-member - # https://bitbucket.org/logilab/pylint/issue/487/ - args = [] - if checker == 'pylint': - args += _get_flag('disable', 'pylint', 'disable') - args += _get_flag('ignore', 'pylint', 'exclude') - args += _get_optional_args('pylint') - plugins = [] - for plugin in config.get('pylint', 'plugins').split(','): - plugins.append('pylint_checkers.{}'.format(plugin)) - args.append('--load-plugins={}'.format(','.join(plugins))) - elif checker == 'flake8': - args += _get_flag('ignore', 'flake8', 'disable') - args += _get_flag('exclude', 'flake8', 'exclude') - args += _get_optional_args('flake8') - elif checker == 'pep257': - args += _get_flag('ignore', 'pep257', 'disable') - try: - excluded = config.get('pep257', 'exclude').split(',') - except configparser.NoOptionError: - excluded = [] - if os.name == 'nt': - # FIXME find a better solution - # pep257 uses cp1252 by default on Windows, which can't handle the - # unicode chars in some files. - # https://github.com/The-Compiler/qutebrowser/issues/105 - excluded += ['configdata', 'misc'] - args.append(r'--match=(?!{})\.py'.format('|'.join(excluded))) - args += _get_optional_args('pep257') - elif checker == 'pyroma': - args = ['.'] - elif checker == 'check-manifest': - args = [] - return args - - -def _get_checkers(args): - """Get a dict of checkers we need to execute.""" - # pylint: disable=no-member - # https://bitbucket.org/logilab/pylint/issue/487/ - # "Static" checkers - checkers = collections.OrderedDict([ - ('global', collections.OrderedDict([ - ('unittest', functools.partial(check_unittest, args.coverage, - args.verbose)), - ('git', check_git), - ])), - ('setup', collections.OrderedDict([ - ('pyroma', functools.partial(run, 'pyroma', - print_version=args.version)), - ('check-manifest', functools.partial(run, 'check-manifest', - print_version=args.version)), - ])), - ]) - # "Dynamic" checkers which exist once for each target. - for target in config.get('DEFAULT', 'targets').split(','): - checkers[target] = collections.OrderedDict([ - ('init', functools.partial(check_init, target)), - ('pep257', functools.partial(check_pep257, target, args.version)), - ('flake8', functools.partial(run, 'flake8', target, args.version)), - ('vcs', functools.partial(check_vcs_conflict, target)), - ('pylint', functools.partial(run, 'pylint', target, args.version)), - ]) - return checkers - - -def _checker_enabled(args, group, name): - """Check if a named checker is enabled.""" - # pylint: disable=no-member - # https://bitbucket.org/logilab/pylint/issue/487/ - if args.checkers == 'all': - if not args.setup and group == 'setup': - return False - else: - return True - else: - return name in args.checkers.split(',') - - -def _parse_args(): - """Parse commandline args via argparse.""" - parser = argparse.ArgumentParser(description='Run various checkers.') - parser.add_argument('-c', '--coverage', help="Also run coverage.py and " - "generate a HTML report.", action='store_true') - parser.add_argument('-s', '--setup', help="Run additional setup checks", - action='store_true') - parser.add_argument('-q', '--quiet', - help="Don't print unnecessary headers.", - action='store_true') - parser.add_argument('-V', '--version', - help="Print checker versions.", action='store_true') - parser.add_argument('-v', '--verbose', help="Run some checkers verbose.", - action='store_true') - parser.add_argument('checkers', help="Checkers to run (or 'all')", - default='all', nargs='?') - return parser.parse_args() - - -def main(): - """Main entry point.""" - # pylint: disable=no-member - # https://bitbucket.org/logilab/pylint/issue/487/ - utils.change_cwd() - read_files = config.read('.run_checks') - if not read_files: - raise OSError("Could not read config!") - exit_status = collections.OrderedDict() - exit_status_bool = {} - - args = _parse_args() - checkers = _get_checkers(args) - - groups = ['global'] - groups += config.get('DEFAULT', 'targets').split(',') - groups.append('setup') - - for group in groups: - print() - utils.print_title(group) - for name, func in checkers[group].items(): - if _checker_enabled(args, group, name): - utils.print_subtitle(name) - status = func() - key = '{}_{}'.format(group, name) - exit_status[key] = status - if name == 'flake8': - # pyflakes uses True for errors and False for ok. - exit_status_bool[key] = not status - elif isinstance(status, bool): - exit_status_bool[key] = status - else: - # sys.exit(0) means no problems -> True, anything != 0 - # means problems. - exit_status_bool[key] = (status == 0) - elif not args.quiet: - utils.print_subtitle(name) - utils.print_col("Checker disabled.", 'blue') - print() - utils.print_col("Exit status values:", 'yellow') - for (k, v) in exit_status.items(): - ok = exit_status_bool[k] - color = 'green' if ok else 'red' - utils.print_col( - ' {} - {} ({})'.format(k, 'ok' if ok else 'FAIL', v), color) - if all(exit_status_bool.values()): - return 0 - else: - return 1 - - -if __name__ == '__main__': - sys.exit(main())