rnhmjoj/qutebrowser
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Remove run_checks/init_venv.

Browse Source 
This will be replaced by tox.
This commit is contained in:
Florian Bruhin 2015-03-26 07:59:01 +01:00
parent 513fbb1539
commit 07da31e2a0
3 changed files with 0 additions and 702 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
.run_checks
View File

@ -1,21 +0,0 @@
# vim: ft=dosini
[DEFAULT]
targets=qutebrowser,scripts
[pep257]
# D102: Docstring missing, will be handled by others
# D209: Blank line before closing """ (removed from PEP257)
# D402: First line should not be function's signature (false-positives)
disable=D102,D209,D402
exclude=resources.py,test_content_disposition.py
[pylint]
args=--output-format=colorized,--reports=no,--rcfile=.pylintrc
plugins=config,crlf,modeline,settrace,openencoding
# excluding command.py is a WORKAROUND for https://bitbucket.org/logilab/pylint/issue/395/horrible-performance-related-to-inspect
exclude=resources.py
[flake8]
args=--config=.flake8
exclude=resources.py

289
scripts/init_venv.py
View File

@ -1,289 +0,0 @@
#!/usr/bin/env python3
# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
# Copyright 2014-2015 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
"""Initialize a venv suitable to be used for qutebrowser."""
import os
import re
import sys
import glob
import os.path
import shutil
import argparse
import subprocess
import distutils.sysconfig # pylint: disable=import-error
# see https://bitbucket.org/logilab/pylint/issue/73/
import venv
import urllib.request
import tempfile
from PyQt5.QtCore import QStandardPaths
sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir))
from scripts import utils
try:
import ensurepip # pylint: disable=import-error
except ImportError:
# Debian-like systems don't have ensurepip...
ensurepip = None
g_path = None
g_args = None
def parse_args():
"""Parse the commandline arguments."""
parser = argparse.ArgumentParser()
parser.add_argument('--clear', help="Clear venv in case it already "
"exists.", action='store_true')
parser.add_argument('--upgrade', help="Upgrade venv to use this version "
"of Python, assuming Python has been upgraded "
"in-place.", action='store_true')
parser.add_argument('--force', help=argparse.SUPPRESS,
action='store_true')
parser.add_argument('--dev', help="Set up an environment suitable for "
"developing qutebrowser.",
action='store_true')
parser.add_argument('--cache', help="Cache the clean virtualenv and "
"copy it when a new one is requested.",
default=None, nargs='?', const='', metavar='NAME')
parser.add_argument('path', help="Path to the venv folder",
default='.venv', nargs='?')
return parser.parse_args()
def get_dev_packages(short=False):
"""Get a list of packages to install.
Args:
short: Remove the version specification.
"""
packages = ['colorlog', 'flake8', 'astroid', 'pylint', 'pep257',
'colorama', 'beautifulsoup4', 'coverage', 'pyroma',
'check-manifest']
if short:
packages = [re.split(r'[<>=]', p)[0] for p in packages]
return packages
def install_dev_packages():
"""Install the packages needed for development."""
for pkg in get_dev_packages():
utils.print_subtitle("Installing {}".format(pkg))
venv_python('-m', 'pip', 'install', '--upgrade', pkg)
def venv_python(*args, output=False):
"""Call the venv's python with the given arguments."""
subdir = 'Scripts' if os.name == 'nt' else 'bin'
executable = os.path.join(g_path, subdir, 'python')
env = dict(os.environ)
if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env:
# WORKAROUND for https://github.com/pypa/pip/issues/2031
del env['__PYVENV_LAUNCHER__']
if output:
return subprocess.check_output([executable] + list(args),
universal_newlines=True, env=env)
else:
subprocess.check_call([executable] + list(args), env=env)
def test_toolchain():
"""Test if imports work properly."""
utils.print_title("Checking toolchain")
packages = ['sip', 'PyQt5.QtCore', 'PyQt5.QtWebKit', 'qutebrowser.app']
renames = {'beautifulsoup4': 'bs4', 'check-manifest': 'check_manifest'}
if g_args.dev:
packages += get_dev_packages(short=True)
for pkg in packages:
try:
pkg = renames[pkg]
except KeyError:
pass
print("Importing {}".format(pkg))
venv_python('-c', 'import {}'.format(pkg))
def verbose_copy(src, dst, *, follow_symlinks=True):
"""Copy function for shutil.copytree which prints copied files."""
print('{} -> {}'.format(src, dst))
shutil.copy(src, dst, follow_symlinks=follow_symlinks)
def get_ignored_files(directory, files):
"""Get the files which should be ignored for link_pyqt() on Windows."""
needed_exts = ('.py', '.dll', '.pyd', '.so')
ignored_dirs = ('examples', 'qml', 'uic', 'doc')
filtered = []
for f in files:
ext = os.path.splitext(f)[1]
full_path = os.path.join(directory, f)
if os.path.isdir(full_path) and f in ignored_dirs:
filtered.append(f)
elif (ext not in needed_exts) and os.path.isfile(full_path):
filtered.append(f)
return filtered
def link_pyqt():
"""Symlink the systemwide PyQt/sip into the venv."""
action = "Copying" if os.name == 'nt' else "Softlinking"
utils.print_title("{} PyQt5".format(action))
sys_path = distutils.sysconfig.get_python_lib()
venv_path = venv_python(
'-c', 'from distutils.sysconfig import get_python_lib\n'
'print(get_python_lib())', output=True).rstrip()
globbed_sip = (glob.glob(os.path.join(sys_path, 'sip*.so')) +
glob.glob(os.path.join(sys_path, 'sip*.pyd')))
if not globbed_sip:
print("Did not find sip in {}!".format(sys_path), file=sys.stderr)
sys.exit(1)
files = [
'PyQt5',
]
files += [os.path.basename(e) for e in globbed_sip]
for fn in files:
source = os.path.join(sys_path, fn)
dest = os.path.join(venv_path, fn)
if not os.path.exists(source):
raise FileNotFoundError(source)
if os.path.exists(dest):
if os.path.isdir(dest) and not os.path.islink(dest):
shutil.rmtree(dest)
else:
os.unlink(dest)
if os.name == 'nt':
if os.path.isdir(source):
shutil.copytree(source, dest, ignore=get_ignored_files,
copy_function=verbose_copy)
else:
print('{} -> {}'.format(source, dest))
shutil.copy(source, dest)
else:
print('{} -> {}'.format(source, dest))
os.symlink(source, dest)
def install_pip():
"""Install pip on Debian-like systems which don't have ensurepip.
WORKAROUND for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=772730 and
https://bugs.launchpad.net/ubuntu/+source/python3.4/+bug/1290847
"""
utils.print_title("Installing pip/setuptools")
f = urllib.request.urlopen('https://bootstrap.pypa.io/get-pip.py')
with tempfile.NamedTemporaryFile() as tmp:
tmp.write(f.read())
venv_python(tmp.name)
def create_venv():
"""Create a new venv."""
utils.print_title("Creating venv")
if os.name == 'nt':
symlinks = False
else:
symlinks = True
clear = g_args.clear or g_args.force
upgrade = g_args.upgrade or g_args.cache is not None
builder = venv.EnvBuilder(system_site_packages=False,
clear=clear, upgrade=upgrade,
symlinks=symlinks, with_pip=ensurepip)
builder.create(g_path)
# If we don't have ensurepip, we have to do it by hand...
if not ensurepip:
install_pip()
def restore_cache(cache_path):
"""Restore a cache if one is present and --cache is given."""
if g_args.cache is not None:
utils.print_title("Restoring cache")
print("Restoring {} to {}...".format(cache_path, g_args.path))
try:
shutil.rmtree(g_args.path)
except FileNotFoundError:
pass
try:
shutil.copytree(cache_path, g_args.path, symlinks=True)
except FileNotFoundError:
print("No cache present!")
else:
return True
return False
def save_cache(cache_path):
"""Save the cache if --cache is given."""
if g_args.cache is not None:
utils.print_title("Saving cache")
print("Saving {} to {}...".format(g_args.path, cache_path))
try:
shutil.rmtree(cache_path)
except FileNotFoundError:
pass
shutil.copytree(g_args.path, cache_path, symlinks=True)
def main():
"""Main entry point."""
global g_path, g_args
g_args = parse_args()
if not g_args.path:
print("Refusing to run with empty path!", file=sys.stderr)
sys.exit(1)
g_path = os.path.abspath(g_args.path)
if os.path.exists(g_args.path) and not (g_args.force or g_args.clear or
g_args.upgrade):
print("{} does already exist! Use --clear or "
"--upgrade.".format(g_path), file=sys.stderr)
sys.exit(1)
os_cache_dir = QStandardPaths.writableLocation(
QStandardPaths.CacheLocation)
file_name = 'qutebrowser-venv'
if g_args.cache:
file_name += '-' + g_args.cache
cache_path = os.path.join(os_cache_dir, file_name)
restored = restore_cache(cache_path)
if not restored:
create_venv()
utils.print_title("Calling: setup.py develop")
venv_python('setup.py', 'develop')
if g_args.dev:
utils.print_title("Installing developer packages")
install_dev_packages()
link_pyqt()
test_toolchain()
save_cache(cache_path)
if __name__ == '__main__':
main()

392
scripts/run_checks.py
View File

@ -1,392 +0,0 @@
#!/usr/bin/env python3
# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
# Copyright 2014-2015 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
#
# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
"""Run different codecheckers over a codebase.
Runs flake8, pylint, pep257, a CRLF/whitespace/conflict-checker and
pyroma/check-manifest by default.
Module attributes:
option: A dictionary with options.
"""
import sys
import subprocess
import os
import io
import os.path
import unittest
import tokenize
import configparser
import argparse
import collections
import functools
import contextlib
import traceback
import pep257
import coverage
sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir))
from scripts import utils
config = configparser.ConfigParser()
@contextlib.contextmanager
def _adjusted_pythonpath(name):
"""Adjust PYTHONPATH for pylint."""
if name == 'pylint':
scriptdir = os.path.abspath(os.path.dirname(__file__))
if 'PYTHONPATH' in os.environ:
old_pythonpath = os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] += os.pathsep + scriptdir
else:
old_pythonpath = None
os.environ['PYTHONPATH'] = scriptdir
yield
if name == 'pylint':
if old_pythonpath is not None:
os.environ['PYTHONPATH'] = old_pythonpath
else:
del os.environ['PYTHONPATH']
def run(name, target=None, print_version=False):
"""Run a checker via distutils with optional args.
Arguments:
name: Name of the checker/binary
target: The package to check
print_version: Whether to print the checker version.
"""
args = _get_args(name)
if target is not None:
args.append(target)
with _adjusted_pythonpath(name):
if os.name == 'nt':
exename = name + '.exe'
else:
exename = name
# for virtualenvs
executable = os.path.join(os.path.dirname(sys.executable), exename)
if not os.path.exists(executable):
# in $PATH
executable = name
if print_version:
subprocess.call([executable, '--version'])
try:
status = subprocess.call([executable] + args)
except OSError:
traceback.print_exc()
status = None
print()
return status
def check_pep257(target, print_version=False):
"""Run pep257 checker with args passed.
We use this rather than run() because on some systems (e.g. Windows) no
pep257 binary is available.
"""
if print_version:
print(pep257.__version__)
args = _get_args('pep257')
sys.argv = ['pep257', target]
if args is not None:
sys.argv += args
try:
# pylint: disable=assignment-from-no-return,no-member
if hasattr(pep257, 'run_pep257'):
# newer pep257 versions
status = pep257.run_pep257()
else:
# older pep257 versions
status = pep257.main(*pep257.parse_options())
print()
return status
except Exception:
traceback.print_exc()
return None
def check_init(target):
"""Check if every subdir of target has an __init__.py file."""
ok = True
for dirpath, _dirnames, filenames in os.walk(target):
if any(f.endswith('.py') for f in filenames):
if '__init__.py' not in filenames:
utils.print_col("Missing __init__.py in {}!".format(dirpath),
'red')
ok = False
print()
return ok
def check_unittest(run_coverage, verbose):
"""Run the unittest checker.
Args:
run_coverage: Whether to also run coverage.py.
verbose: For verbose output.
"""
if run_coverage:
cov = coverage.coverage(branch=True, source=['qutebrowser'])
cov.erase()
cov.start()
suite = unittest.TestLoader().discover('.')
verbosity = 2 if verbose else 1
result = unittest.TextTestRunner(verbosity=verbosity).run(suite)
if run_coverage:
cov.stop()
perc = cov.report(file=io.StringIO())
print("COVERAGE: {}%".format(round(perc)))
cov.html_report()
print()
return result.wasSuccessful()
def check_git():
"""Check for uncommited git files.."""
if not os.path.isdir(".git"):
print("No .git dir, ignoring")
print()
return False
untracked = []
changed = []
gitst = subprocess.check_output(['git', 'status', '--porcelain'])
gitst = gitst.decode('UTF-8').strip()
for line in gitst.splitlines():
s, name = line.split(maxsplit=1)
if s == '??' and name != '.venv/':
untracked.append(name)
elif s == 'M':
changed.append(name)
status = True
if untracked:
status = False
utils.print_col("Untracked files:", 'red')
print('\n'.join(untracked))
if changed:
status = False
utils.print_col("Uncommited changes:", 'red')
print('\n'.join(changed))
print()
return status
def check_vcs_conflict(target):
"""Check VCS conflict markers."""
try:
ok = True
for (dirpath, _dirnames, filenames) in os.walk(target):
for name in (e for e in filenames if e.endswith('.py')):
fn = os.path.join(dirpath, name)
with tokenize.open(fn) as f:
for line in f:
if any(line.startswith(c * 7) for c in '<>=|'):
print("Found conflict marker in {}".format(fn))
ok = False
print()
return ok
except Exception:
traceback.print_exc()
return None
def _get_optional_args(checker):
"""Get a list of arguments based on a comma-separated args config."""
# pylint: disable=no-member
# https://bitbucket.org/logilab/pylint/issue/487/
try:
return config.get(checker, 'args').split(',')
except configparser.NoOptionError:
return []
def _get_flag(arg, checker, option):
"""Get a list of arguments based on a config option."""
try:
return ['--{}={}'.format(arg, config.get(checker, option))]
except configparser.NoOptionError:
return []
def _get_args(checker):
"""Construct the arguments for a given checker.
Return:
A list of commandline arguments.
"""
# pylint: disable=no-member
# https://bitbucket.org/logilab/pylint/issue/487/
args = []
if checker == 'pylint':
args += _get_flag('disable', 'pylint', 'disable')
args += _get_flag('ignore', 'pylint', 'exclude')
args += _get_optional_args('pylint')
plugins = []
for plugin in config.get('pylint', 'plugins').split(','):
plugins.append('pylint_checkers.{}'.format(plugin))
args.append('--load-plugins={}'.format(','.join(plugins)))
elif checker == 'flake8':
args += _get_flag('ignore', 'flake8', 'disable')
args += _get_flag('exclude', 'flake8', 'exclude')
args += _get_optional_args('flake8')
elif checker == 'pep257':
args += _get_flag('ignore', 'pep257', 'disable')
try:
excluded = config.get('pep257', 'exclude').split(',')
except configparser.NoOptionError:
excluded = []
if os.name == 'nt':
# FIXME find a better solution
# pep257 uses cp1252 by default on Windows, which can't handle the
# unicode chars in some files.
# https://github.com/The-Compiler/qutebrowser/issues/105
excluded += ['configdata', 'misc']
args.append(r'--match=(?!{})\.py'.format('|'.join(excluded)))
args += _get_optional_args('pep257')
elif checker == 'pyroma':
args = ['.']
elif checker == 'check-manifest':
args = []
return args
def _get_checkers(args):
"""Get a dict of checkers we need to execute."""
# pylint: disable=no-member
# https://bitbucket.org/logilab/pylint/issue/487/
# "Static" checkers
checkers = collections.OrderedDict([
('global', collections.OrderedDict([
('unittest', functools.partial(check_unittest, args.coverage,
args.verbose)),
('git', check_git),
])),
('setup', collections.OrderedDict([
('pyroma', functools.partial(run, 'pyroma',
print_version=args.version)),
('check-manifest', functools.partial(run, 'check-manifest',
print_version=args.version)),
])),
])
# "Dynamic" checkers which exist once for each target.
for target in config.get('DEFAULT', 'targets').split(','):
checkers[target] = collections.OrderedDict([
('init', functools.partial(check_init, target)),
('pep257', functools.partial(check_pep257, target, args.version)),
('flake8', functools.partial(run, 'flake8', target, args.version)),
('vcs', functools.partial(check_vcs_conflict, target)),
('pylint', functools.partial(run, 'pylint', target, args.version)),
])
return checkers
def _checker_enabled(args, group, name):
"""Check if a named checker is enabled."""
# pylint: disable=no-member
# https://bitbucket.org/logilab/pylint/issue/487/
if args.checkers == 'all':
if not args.setup and group == 'setup':
return False
else:
return True
else:
return name in args.checkers.split(',')
def _parse_args():
"""Parse commandline args via argparse."""
parser = argparse.ArgumentParser(description='Run various checkers.')
parser.add_argument('-c', '--coverage', help="Also run coverage.py and "
"generate a HTML report.", action='store_true')
parser.add_argument('-s', '--setup', help="Run additional setup checks",
action='store_true')
parser.add_argument('-q', '--quiet',
help="Don't print unnecessary headers.",
action='store_true')
parser.add_argument('-V', '--version',
help="Print checker versions.", action='store_true')
parser.add_argument('-v', '--verbose', help="Run some checkers verbose.",
action='store_true')
parser.add_argument('checkers', help="Checkers to run (or 'all')",
default='all', nargs='?')
return parser.parse_args()
def main():
"""Main entry point."""
# pylint: disable=no-member
# https://bitbucket.org/logilab/pylint/issue/487/
utils.change_cwd()
read_files = config.read('.run_checks')
if not read_files:
raise OSError("Could not read config!")
exit_status = collections.OrderedDict()
exit_status_bool = {}
args = _parse_args()
checkers = _get_checkers(args)
groups = ['global']
groups += config.get('DEFAULT', 'targets').split(',')
groups.append('setup')
for group in groups:
print()
utils.print_title(group)
for name, func in checkers[group].items():
if _checker_enabled(args, group, name):
utils.print_subtitle(name)
status = func()
key = '{}_{}'.format(group, name)
exit_status[key] = status
if name == 'flake8':
# pyflakes uses True for errors and False for ok.
exit_status_bool[key] = not status
elif isinstance(status, bool):
exit_status_bool[key] = status
else:
# sys.exit(0) means no problems -> True, anything != 0
# means problems.
exit_status_bool[key] = (status == 0)
elif not args.quiet:
utils.print_subtitle(name)
utils.print_col("Checker disabled.", 'blue')
print()
utils.print_col("Exit status values:", 'yellow')
for (k, v) in exit_status.items():
ok = exit_status_bool[k]
color = 'green' if ok else 'red'
utils.print_col(
' {} - {} ({})'.format(k, 'ok' if ok else 'FAIL', v), color)
if all(exit_status_bool.values()):
return 0
else:
return 1
if __name__ == '__main__':
sys.exit(main())