qutebrowser/scripts/run_checks.py

328 lines
10 KiB
Python
Raw Normal View History

2014-09-22 20:21:00 +02:00
#!/usr/bin/env python3
2014-06-19 09:04:37 +02:00
# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
2014-02-06 14:01:23 +01:00
# Copyright 2014 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
#
# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
2014-09-08 07:44:32 +02:00
# pylint: disable=broad-except
2014-05-15 08:24:10 +02:00
2014-04-17 17:44:27 +02:00
""" Run different codecheckers over a codebase.
Runs flake8, pylint, pep257, a CRLF/whitespace/conflict-checker and
pyroma/check-manifest by default.
2014-04-17 17:44:27 +02:00
Module attributes:
option: A dictionary with options.
"""
2014-01-28 15:25:40 +01:00
import sys
import subprocess
2014-01-28 22:44:39 +01:00
import os
import os.path
2014-05-06 14:37:47 +02:00
import unittest
2014-05-06 19:00:35 +02:00
import logging
import tokenize
2014-08-12 18:37:53 +02:00
import configparser
2014-08-14 15:32:42 +02:00
import argparse
2014-08-26 19:10:14 +02:00
import collections
import functools
import contextlib
import traceback
2014-01-28 15:25:40 +01:00
2014-08-12 18:47:11 +02:00
import pep257
2014-01-29 04:07:27 +01:00
2014-09-22 20:41:12 +02:00
sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir))
from scripts import utils
2014-05-06 19:00:35 +02:00
# We need to do this because pyroma is braindead enough to use logging instead
# of print...
logging.basicConfig(level=logging.INFO, format='%(msg)s')
2014-08-12 18:37:53 +02:00
2014-08-12 22:38:28 +02:00
config = configparser.ConfigParser()
2014-02-10 11:07:21 +01:00
2014-05-15 08:24:10 +02:00
2014-08-26 19:10:14 +02:00
@contextlib.contextmanager
2014-08-14 18:02:38 +02:00
def _adjusted_pythonpath(name):
"""Adjust PYTHONPATH for pylint."""
2014-08-06 14:42:05 +02:00
if name == 'pylint':
scriptdir = os.path.abspath(os.path.dirname(__file__))
if 'PYTHONPATH' in os.environ:
old_pythonpath = os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] += os.pathsep + scriptdir
else:
old_pythonpath = None
os.environ['PYTHONPATH'] = scriptdir
2014-08-14 18:02:38 +02:00
yield
if name == 'pylint':
if old_pythonpath is not None:
os.environ['PYTHONPATH'] = old_pythonpath
else:
del os.environ['PYTHONPATH']
def run(name, target=None):
"""Run a checker via distutils with optional args.
Arguments:
name: Name of the checker/binary
target: The package to check
"""
# pylint: disable=too-many-branches
args = _get_args(name)
if target is not None:
args.append(target)
with _adjusted_pythonpath(name):
try:
status = subprocess.call([name] + args)
except OSError:
traceback.print_exc()
status = None
2014-01-28 15:25:40 +01:00
print()
2014-08-14 18:02:38 +02:00
return status
2014-01-28 15:25:40 +01:00
2014-02-10 11:07:21 +01:00
def check_pep257(target):
2014-04-17 17:44:27 +02:00
"""Run pep257 checker with args passed."""
args = _get_args('pep257')
2014-05-15 07:29:12 +02:00
sys.argv = ['pep257', target]
2014-02-07 19:44:56 +01:00
if args is not None:
sys.argv += args
try:
2014-08-14 18:02:38 +02:00
status = pep257.main(*pep257.parse_options())
print()
return status
except Exception:
traceback.print_exc()
2014-08-14 18:02:38 +02:00
return None
2014-02-07 19:44:56 +01:00
2014-02-10 11:07:21 +01:00
2014-05-06 14:37:47 +02:00
def check_unittest():
2014-05-15 08:24:10 +02:00
"""Run the unittest checker."""
2014-05-06 14:37:47 +02:00
suite = unittest.TestLoader().discover('.')
result = unittest.TextTestRunner().run(suite)
print()
2014-08-14 18:02:38 +02:00
return result.wasSuccessful()
2014-05-06 14:37:47 +02:00
2014-05-15 08:24:10 +02:00
2014-05-22 15:19:35 +02:00
def check_git():
"""Check for uncommited git files.."""
if not os.path.isdir(".git"):
print("No .git dir, ignoring")
print()
2014-08-14 18:02:38 +02:00
return False
2014-05-22 15:19:35 +02:00
untracked = []
changed = []
2014-05-22 15:19:35 +02:00
gitst = subprocess.check_output(['git', 'status', '--porcelain'])
gitst = gitst.decode('UTF-8').strip()
for line in gitst.splitlines():
s, name = line.split(maxsplit=1)
2014-12-05 23:40:33 +01:00
if s == '??' and name != '.venv':
2014-05-22 15:19:35 +02:00
untracked.append(name)
elif s == 'M':
changed.append(name)
status = True
2014-05-22 15:19:35 +02:00
if untracked:
2014-08-14 18:02:38 +02:00
status = False
utils.print_col("Untracked files:", 'red')
2014-05-22 15:19:35 +02:00
print('\n'.join(untracked))
if changed:
status = False
utils.print_col("Uncommited changes:", 'red')
print('\n'.join(changed))
2014-05-22 15:19:35 +02:00
print()
2014-08-14 18:02:38 +02:00
return status
2014-05-22 15:19:35 +02:00
def check_vcs_conflict(target):
"""Check VCS conflict markers."""
2014-01-28 22:44:39 +01:00
try:
ok = True
2014-05-15 08:24:10 +02:00
for (dirpath, _dirnames, filenames) in os.walk(target):
for name in (e for e in filenames if e.endswith('.py')):
2014-01-28 22:44:39 +01:00
fn = os.path.join(dirpath, name)
with tokenize.open(fn) as f:
for line in f:
if any(line.startswith(c * 7) for c in '<>=|'):
print("Found conflict marker in {}".format(fn))
ok = False
2014-08-14 18:02:38 +02:00
print()
return ok
except Exception:
traceback.print_exc()
2014-08-14 18:02:38 +02:00
return None
2014-01-28 22:44:39 +01:00
2014-02-10 11:07:21 +01:00
2014-02-07 19:39:55 +01:00
def _get_args(checker):
2014-04-17 17:44:27 +02:00
"""Construct the arguments for a given checker.
Return:
A list of commandline arguments.
"""
2014-08-12 18:37:53 +02:00
def _get_optional_args(checker):
"""Get a list of arguments based on a comma-separated args config."""
2014-02-07 19:39:55 +01:00
try:
2014-08-12 22:38:28 +02:00
return config.get(checker, 'args').split(',')
2014-08-12 18:37:53 +02:00
except configparser.NoOptionError:
return []
def _get_flag(arg, checker, option):
"""Get a list of arguments based on a config option."""
2014-04-16 11:05:58 +02:00
try:
2014-08-12 22:38:28 +02:00
return ['--{}={}'.format(arg, config.get(checker, option))]
2014-08-12 18:37:53 +02:00
except configparser.NoOptionError:
return []
args = []
if checker == 'pylint':
args += _get_flag('disable', 'pylint', 'disable')
args += _get_flag('ignore', 'pylint', 'exclude')
args += _get_optional_args('pylint')
2014-08-12 22:38:36 +02:00
plugins = []
for plugin in config.get('pylint', 'plugins').split(','):
plugins.append('pylint_checkers.{}'.format(plugin))
args.append('--load-plugins={}'.format(','.join(plugins)))
2014-02-07 19:39:55 +01:00
elif checker == 'flake8':
2014-08-12 18:37:53 +02:00
args += _get_flag('ignore', 'flake8', 'disable')
args += _get_flag('exclude', 'flake8', 'exclude')
args += _get_optional_args('flake8')
2014-02-07 19:44:56 +01:00
elif checker == 'pep257':
2014-08-12 18:37:53 +02:00
args += _get_flag('ignore', 'pep257', 'disable')
2014-04-16 11:05:58 +02:00
try:
2014-08-12 22:38:28 +02:00
excluded = config.get('pep257', 'exclude').split(',')
2014-08-12 18:37:53 +02:00
except configparser.NoOptionError:
excluded = []
if os.name == 'nt':
# FIXME find a better solution
# pep257 uses cp1252 by default on Windows, which can't handle the
# unicode chars in some files.
2014-10-01 22:23:27 +02:00
# https://github.com/The-Compiler/qutebrowser/issues/105
2014-08-12 18:37:53 +02:00
excluded += ['configdata', 'misc']
args.append(r'--match=(?!{})\.py'.format('|'.join(excluded)))
args += _get_optional_args('pep257')
2014-04-29 08:34:49 +02:00
elif checker == 'pyroma':
args = ['.']
elif checker == 'check-manifest':
args = []
2014-02-07 19:39:55 +01:00
return args
2014-02-07 10:58:53 +01:00
2014-05-15 07:29:12 +02:00
2014-08-14 18:02:38 +02:00
def _get_checkers():
"""Get a dict of checkers we need to execute."""
2014-08-14 18:02:38 +02:00
# "Static" checkers
2014-08-26 19:10:14 +02:00
checkers = collections.OrderedDict([
('global', collections.OrderedDict([
('unittest', check_unittest),
('git', check_git),
])),
2014-08-26 19:10:14 +02:00
('setup', collections.OrderedDict([
('pyroma', functools.partial(run, 'pyroma')),
('check-manifest', functools.partial(run, 'check-manifest')),
])),
])
2014-08-14 18:02:38 +02:00
# "Dynamic" checkers which exist once for each target.
for target in config.get('DEFAULT', 'targets').split(','):
2014-08-26 19:10:14 +02:00
checkers[target] = collections.OrderedDict([
('pep257', functools.partial(check_pep257, target)),
('flake8', functools.partial(run, 'flake8', target)),
('vcs', functools.partial(check_vcs_conflict, target)),
('pylint', functools.partial(run, 'pylint', target)),
])
return checkers
def _checker_enabled(args, group, name):
"""Check if a named checker is enabled."""
if args.checkers == 'all':
if not args.setup and group == 'setup':
return False
else:
return True
else:
return name in args.checkers.split(',')
2014-10-26 22:09:06 +01:00
def _parse_args():
"""Parse commandline args via argparse."""
parser = argparse.ArgumentParser(description='Run various checkers.')
parser.add_argument('-s', '--setup', help="Run additional setup checks",
action='store_true')
2014-12-03 23:10:23 +01:00
parser.add_argument('-q', '--quiet',
help="Don't print unnecessary headers.",
action='store_true')
2014-10-26 22:09:06 +01:00
parser.add_argument('checkers', help="Checkers to run (or 'all')",
default='all', nargs='?')
return parser.parse_args()
2014-08-12 18:48:31 +02:00
def main():
2014-08-12 21:12:31 +02:00
"""Main entry point."""
utils.change_cwd()
read_files = config.read('.run_checks')
if not read_files:
raise IOError("Could not read config!")
2014-08-26 19:10:14 +02:00
exit_status = collections.OrderedDict()
2014-08-14 20:58:39 +02:00
exit_status_bool = {}
2014-08-14 15:32:42 +02:00
2014-10-26 22:09:06 +01:00
args = _parse_args()
2014-08-14 18:02:38 +02:00
checkers = _get_checkers()
2014-08-12 18:48:31 +02:00
groups = ['global']
groups += config.get('DEFAULT', 'targets').split(',')
groups.append('setup')
for group in groups:
print()
utils.print_title(group)
for name, func in checkers[group].items():
if _checker_enabled(args, group, name):
utils.print_subtitle(name)
2014-08-14 18:02:38 +02:00
status = func()
2014-08-14 20:58:39 +02:00
key = '{}_{}'.format(group, name)
exit_status[key] = status
if name == 'flake8':
# pyflakes uses True for errors and False for ok.
exit_status_bool[key] = not status
elif isinstance(status, bool):
exit_status_bool[key] = status
else:
# sys.exit(0) means no problems -> True, anything != 0
# means problems.
exit_status_bool[key] = (status == 0)
2014-12-03 23:10:23 +01:00
elif not args.quiet:
utils.print_subtitle(name)
utils.print_col("Checker disabled.", 'blue')
print()
utils.print_col("Exit status values:", 'yellow')
2014-08-14 18:02:38 +02:00
for (k, v) in exit_status.items():
2014-08-14 20:58:39 +02:00
ok = exit_status_bool[k]
color = 'green' if ok else 'red'
utils.print_col(
' {} - {} ({})'.format(k, 'ok' if ok else 'FAIL', v), color)
2014-12-03 23:11:37 +01:00
if all(exit_status_bool.values()):
2014-08-12 18:48:31 +02:00
return 0
else:
return 1
if __name__ == '__main__':
sys.exit(main())