qutebrowser/scripts/generate_doc.py

#!/usr/bin/python3
# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:

# Copyright 2014 Florian Bruhin (The Compiler) <mail@qutebrowser.org>

# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.

"""Generate asciidoc source for qutebrowser based on docstrings."""

import re
import os
import sys
import html
import shutil
import inspect
import subprocess
from collections import Counter, OrderedDict
from tempfile import mkstemp

sys.path.insert(0, os.getcwd())

import qutebrowser
# We import qutebrowser.app so all @cmdutils-register decorators are run.
import qutebrowser.app
import qutebrowser.commands.utils as cmdutils
import qutebrowser.config.configdata as configdata
import qutebrowser.qutebrowser as qutequtebrowser
from qutebrowser.utils.usertypes import enum


def _open_file(name, mode='w'):
    """Open a file with a preset newline/encoding mode."""
    return open(name, mode, newline='\n', encoding='utf-8')


def _parse_docstring(func):  # noqa
    """Generate documentation based on a docstring of a command handler.

    The docstring needs to follow the format described in HACKING.

    Args:
        func: The function to generate the docstring for.

    Return:
        A (short_desc, long_desc, arg_descs) tuple.
    """
    # pylint: disable=too-many-branches
    State = enum('State', 'short', 'desc',  # pylint: disable=invalid-name
                 'desc_hidden', 'arg_start', 'arg_inside', 'misc')
    doc = inspect.getdoc(func)
    lines = doc.splitlines()

    cur_state = State.short

    short_desc = []
    long_desc = []
    arg_descs = OrderedDict()
    cur_arg_name = None

    for line in lines:
        if cur_state == State.short:
            if not line:
                cur_state = State.desc
            else:
                short_desc.append(line.strip())
        elif cur_state == State.desc:
            if line.startswith('Args:'):
                cur_state = State.arg_start
            elif line.startswith('Emit:') or line.startswith('Raise:'):
                cur_state = State.misc
            elif line.strip() == '//':
                cur_state = State.desc_hidden
            elif line.strip():
                long_desc.append(line.strip())
        elif cur_state == State.misc:
            if line.startswith('Args:'):
                cur_state = State.arg_start
            else:
                pass
        elif cur_state == State.desc_hidden:
            if line.startswith('Args:'):
                cur_state = State.arg_start
        elif cur_state == State.arg_start:
            cur_arg_name, argdesc = line.split(':', maxsplit=1)
            cur_arg_name = cur_arg_name.strip().lstrip('*')
            arg_descs[cur_arg_name] = [argdesc.strip()]
            cur_state = State.arg_inside
        elif cur_state == State.arg_inside:
            if re.match('^[A-Z][a-z]+:$', line):
                if not arg_descs[cur_arg_name][-1].strip():
                    arg_descs[cur_arg_name] = arg_descs[cur_arg_name][:-1]
                    break
            elif not line.strip():
                arg_descs[cur_arg_name].append('\n\n')
            elif line[4:].startswith(' '):
                arg_descs[cur_arg_name].append(line.strip() + '\n')
            else:
                cur_arg_name, argdesc = line.split(':', maxsplit=1)
                cur_arg_name = cur_arg_name.strip().lstrip('*')
                arg_descs[cur_arg_name] = [argdesc.strip()]
    return (short_desc, long_desc, arg_descs)


def _get_cmd_syntax(name, cmd):
    """Get the command syntax for a command."""
    words = []
    argspec = inspect.getfullargspec(cmd.handler)
    if argspec.defaults is not None:
        defaults = dict(zip(reversed(argspec.args),
                        reversed(list(argspec.defaults))))
    else:
        defaults = {}
    words.append(name)
    minargs, maxargs = cmd.nargs
    i = 1
    for arg in argspec.args:
        if arg in ['self', 'count']:
            continue
        if minargs is not None and i <= minargs:
            words.append('<{}>'.format(arg))
        elif maxargs is None or i <= maxargs:
            words.append('[<{}>]'.format(arg))
        i += 1
    if argspec.varargs is not None:
        words.append('[<{name}> [...]]'.format(name=argspec.varargs))
    return (' '.join(words), defaults)


def _get_command_quickref(cmds):
    """Generate the command quick reference."""
    out = []
    out.append('[options="header",width="75%",cols="25%,75%"]')
    out.append('|==============')
    out.append('|Command|Description')
    for name, cmd in cmds:
        desc = inspect.getdoc(cmd.handler).splitlines()[0]
        out.append('|<<cmd-{},{}>>|{}'.format(name, name, desc))
    out.append('|==============')
    return '\n'.join(out)


def _get_setting_quickref():
    """Generate the settings quick reference."""
    out = []
    for sectname, sect in configdata.DATA.items():
        if not getattr(sect, 'descriptions'):
            continue
        out.append(".Quick reference for section ``{}''".format(sectname))
        out.append('[options="header",width="75%",cols="25%,75%"]')
        out.append('|==============')
        out.append('|Setting|Description')
        for optname, _option in sect.items():
            desc = sect.descriptions[optname].splitlines()[0]
            out.append('|<<setting-{}-{},{}>>|{}'.format(
                sectname, optname, optname, desc))
        out.append('|==============')
    return '\n'.join(out)


def _get_command_doc(name, cmd):
    """Generate the documentation for a command."""
    output = ['[[cmd-{}]]'.format(name)]
    output += ['==== {}'.format(name)]
    syntax, defaults = _get_cmd_syntax(name, cmd)
    if syntax != name:
        output.append('Syntax: +:{}+'.format(syntax))
    output.append("")
    short_desc, long_desc, arg_descs = _parse_docstring(cmd.handler)
    output.append(' '.join(short_desc))
    output.append("")
    output.append(' '.join(long_desc))
    if arg_descs:
        output.append("")
        for arg, desc in arg_descs.items():
            text = ' '.join(desc).splitlines()
            firstline = text[0].replace(', or None', '')
            item = "* +{}+: {}".format(arg, firstline)
            if arg in defaults:
                val = defaults[arg]
                if val is None:
                    item += " (optional)\n"
                else:
                    item += " (default: +{}+)\n".format(defaults[arg])
            item += '\n'.join(text[1:])
            output.append(item)
        output.append("")
    output.append("")
    return '\n'.join(output)


def _get_action_metavar(action):
    """Get the metavar to display for an argparse action."""
    if action.metavar is not None:
        return "'{}'".format(action.metavar)
    elif action.choices is not None:
        choices = ','.join(map(str, action.choices))
        return "'{{{}}}'".format(choices)
    else:
        return "'{}'".format(action.dest.upper())


def _format_action_args(action):
    """Get an argument string based on an argparse action."""
    if action.nargs is None:
        return _get_action_metavar(action)
    elif action.nargs == '?':
        return '[{}]'.format(_get_action_metavar(action))
    elif action.nargs == '*':
        return '[{mv} [{mv} ...]]'.format(mv=_get_action_metavar(action))
    elif action.nargs == '+':
        return '{mv} [{mv} ...]'.format(mv=_get_action_metavar(action))
    elif action.nargs == '...':
        return '...'
    else:
        return ' '.join([_get_action_metavar(action)] * action.nargs)


def _format_action(action):
    """Get an invocation string/help from an argparse action."""
    if not action.option_strings:
        invocation = '*{}*::'.format(_get_action_metavar(action))
    else:
        parts = []
        if action.nargs == 0:
            # Doesn't take a value, so the syntax is -s, --long
            parts += ['*{}*'.format(s) for s in action.option_strings]
        else:
            # Takes a value, so the syntax is -s ARGS or --long ARGS.
            args_string = _format_action_args(action)
            for opt in action.option_strings:
                parts.append('*{}* {}'.format(opt, args_string))
        invocation = ', '.join(parts) + '::'
    return '{}\n    {}\n\n'.format(invocation, action.help)


def generate_manpage_header(f):
    """Generate an asciidoc header for the manpage."""
    f.write("// DO NOT EDIT THIS FILE BY HAND!\n")
    f.write("// It is generated by `scripts/generate_doc.py`.\n")
    f.write("// Most likely you'll need to rerun that script, or edit that "
            "instead of this file.\n")
    f.write('= qutebrowser(1)\n')
    f.write(':doctype: manpage\n')
    f.write(':man source: qutebrowser\n')
    f.write(':man manual: qutebrowser manpage\n')
    f.write(':toc:\n')
    f.write(':homepage: http://www.qutebrowser.org/\n')
    f.write('\n')


def generate_manpage_name(f):
    """Generate the NAME-section of the manpage."""
    f.write('== NAME\n')
    f.write('qutebrowser - {}\n'.format(qutebrowser.__description__))
    f.write('\n')


def generate_manpage_synopsis(f):
    """Generate the SYNOPSIS-section of the manpage."""
    f.write('== SYNOPSIS\n')
    f.write("*qutebrowser* ['-OPTION' ['...']] [':COMMAND' ['...']] "
            "['URL' ['...']]\n")
    f.write('\n')


def generate_manpage_description(f):
    """Generate the DESCRIPTION-section of the manpage."""
    f.write('== DESCRIPTION\n')
    f.write("qutebrowser is a keyboard-focused browser with with a minimal "
            "GUI. It's based on Python, PyQt5 and QtWebKit and free software, "
            "licensed under the GPL.\n\n")
    f.write("It was inspired by other browsers/addons like dwb and "
            "Vimperator/Pentadactyl.\n\n")


def generate_manpage_options(f):
    """Generate the OPTIONS-section of the manpage from an argparse parser."""
    # pylint: disable=protected-access
    parser = qutequtebrowser.get_argparser()
    f.write('== OPTIONS\n')

    # positionals, optionals and user-defined groups
    for group in parser._action_groups:
        f.write('=== {}\n'.format(group.title))
        if group.description is not None:
            f.write(group.description + '\n')
        for action in group._group_actions:
            f.write(_format_action(action))
        f.write('\n')
    # epilog
    if parser.epilog is not None:
        f.write(parser.epilog)
    f.write('\n')


def generate_commands(f):
    """Generate the complete commands section."""
    f.write('\n')
    f.write("== COMMANDS\n")
    normal_cmds = []
    hidden_cmds = []
    debug_cmds = []
    for name, cmd in cmdutils.cmd_dict.items():
        if cmd.hide:
            hidden_cmds.append((name, cmd))
        elif cmd.debug:
            debug_cmds.append((name, cmd))
        else:
            normal_cmds.append((name, cmd))
    normal_cmds.sort()
    hidden_cmds.sort()
    debug_cmds.sort()
    f.write("\n")
    f.write("=== Normal commands\n")
    f.write(".Quick reference\n")
    f.write(_get_command_quickref(normal_cmds) + "\n")
    for name, cmd in normal_cmds:
        f.write(_get_command_doc(name, cmd) + "\n")
    f.write("\n")
    f.write("=== Hidden commands\n")
    f.write(".Quick reference\n")
    f.write(_get_command_quickref(hidden_cmds) + "\n")
    for name, cmd in hidden_cmds:
        f.write(_get_command_doc(name, cmd) + "\n")
    f.write("\n")
    f.write("=== Debugging commands\n")
    f.write("These commands are mainly intended for debugging. They are "
            "hidden if qutebrowser was started without the `--debug`-flag.\n")
    f.write("\n")
    f.write(".Quick reference\n")
    f.write(_get_command_quickref(debug_cmds) + "\n")
    for name, cmd in debug_cmds:
        f.write(_get_command_doc(name, cmd) + "\n")


def generate_settings(f):
    """Generate the complete settings section."""
    f.write("\n")
    f.write("== SETTINGS\n")
    f.write(_get_setting_quickref() + "\n")
    for sectname, sect in configdata.DATA.items():
        f.write("\n")
        f.write("=== {}".format(sectname) + "\n")
        f.write(configdata.SECTION_DESC[sectname] + "\n")
        if not getattr(sect, 'descriptions'):
            pass
        else:
            for optname, option in sect.items():
                f.write("\n")
                f.write('[[setting-{}-{}]]'.format(sectname, optname) + "\n")
                f.write("==== {}".format(optname) + "\n")
                f.write(sect.descriptions[optname] + "\n")
                f.write("\n")
                valid_values = option.typ.valid_values
                if valid_values is not None:
                    f.write("Valid values:\n")
                    f.write("\n")
                    for val in valid_values:
                        try:
                            desc = valid_values.descriptions[val]
                            f.write(" * +{}+: {}".format(val, desc) + "\n")
                        except KeyError:
                            f.write(" * +{}+".format(val) + "\n")
                    f.write("\n")
                if option.default:
                    f.write("Default: +pass:[{}]+\n".format(html.escape(
                        option.default)))
                else:
                    f.write("Default: empty\n")


def _get_authors():
    """Get a list of authors based on git commit logs."""
    commits = subprocess.check_output(['git', 'log', '--format=%aN'])
    cnt = Counter(commits.decode('utf-8').splitlines())
    return sorted(cnt, key=lambda k: cnt[k])


def generate_manpage_author(f):
    """Generate the manpage AUTHOR section."""
    f.write("== AUTHOR\n")
    f.write("Contributors, sorted by the number of commits in descending "
            "order:\n\n")
    for author in _get_authors():
        f.write('* {}\n'.format(author))
    f.write('\n')


def generate_manpage_bugs(f):
    """Generate the manpage BUGS section."""
    f.write('== BUGS\n')
    f.write("Bugs are tracked at two locations:\n\n")
    f.write("* The link:BUGS[doc/BUGS] and link:TODO[doc/TODO] files shipped "
            "with qutebrowser.\n")
    f.write("* The Github issue tracker at https://github.com/The-Compiler/"
            "qutebrowser/issues .\n\n")
    f.write("If you found a bug or have a suggestion, either open a ticket "
            "in the github issue tracker, or write a mail to the "
            "https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser["
            "mailinglist] at mailto:qutebrowser@lists.qutebrowser.org[].\n\n")


def generate_manpage_copyright(f):
    """Generate the COPYRIGHT section of the manpage."""
    f.write('== COPYRIGHT\n')
    f.write("This program is free software: you can redistribute it and/or "
            "modify it under the terms of the GNU General Public License as "
            "published by the Free Software Foundation, either version 3 of "
            "the License, or (at your option) any later version.\n\n")
    f.write("This program is distributed in the hope that it will be useful, "
            "but WITHOUT ANY WARRANTY; without even the implied warranty of "
            "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the "
            "GNU General Public License for more details.\n\n")
    f.write("You should have received a copy of the GNU General Public "
            "License along with this program.  If not, see "
            "<http://www.gnu.org/licenses/>.\n")


def generate_manpage_resources(f):
    """Generate the RESOURCES section of the manpage."""
    f.write('== RESOURCES\n\n')
    f.write("* Website: http://www.qutebrowser.org/\n")
    f.write("* Mailinglist: mailto:qutebrowser@lists.qutebrowser.org[] / "
            "https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser\n")
    f.write("* IRC: irc://irc.freenode.org/#qutebrowser[`#qutebrowser`] on "
            "http://freenode.net/[Freenode]\n")
    f.write("* Github: https://github.com/The-Compiler/qutebrowser\n\n")


def regenerate_authors(filename):
    """Re-generate the authors inside README based on the commits made."""
    oshandle, tmpname = mkstemp()
    with _open_file(filename, mode='r') as infile, \
            _open_file(oshandle, mode='w') as temp:
        ignore = False
        for line in infile:
            if line.strip() == '// QUTE_AUTHORS_START':
                ignore = True
                temp.write(line)
                for author in _get_authors():
                    temp.write('* {}\n'.format(author))
            elif line.strip() == '// QUTE_AUTHORS_END':
                temp.write(line)
                ignore = False
            elif not ignore:
                temp.write(line)
    os.remove(filename)
    shutil.move(tmpname, filename)


if __name__ == '__main__':
    with _open_file('doc/qutebrowser.1.asciidoc') as fobj:
        generate_manpage_header(fobj)
        generate_manpage_name(fobj)
        generate_manpage_synopsis(fobj)
        generate_manpage_description(fobj)
        generate_manpage_options(fobj)
        generate_settings(fobj)
        generate_commands(fobj)
        generate_manpage_bugs(fobj)
        generate_manpage_author(fobj)
        generate_manpage_resources(fobj)
        generate_manpage_copyright(fobj)
    regenerate_authors('README.asciidoc')