Clean up docstring parsing and move it into qutebrowser for commands.

qutebrowser/commands/cmdutils.py

@@ -137,6 +137,7 @@ class register:  # pylint: disable=invalid-name
         self.ignore_args = ignore_args
         self.parser = None
         self.func = None
+        self.docparser = None
         if modes is not None:
             for m in modes:
                 if not isinstance(m, usertypes.KeyMode):
@@ -166,10 +167,13 @@ class register:  # pylint: disable=invalid-name
         for name in names:
             if name in cmd_dict:
                 raise ValueError("{} is already registered!".format(name))
-        self.parser = argparser.ArgumentParser(names[0])
+        self.docparser = utils.DocstringParser(func)
+        self.parser = argparser.ArgumentParser(
+            names[0], description=self.docparser.short_desc,
+            epilog=self.docparser.long_desc)
         self.parser.add_argument('-h', '--help', action=argparser.HelpAction,
                                  default=argparser.SUPPRESS, nargs=0,
-                                help="Show this help message.")
+                                 help=argparser.SUPPRESS)
         has_count, desc, type_conv = self._inspect_func()
         cmd = command.Command(
             name=names[0], split=self.split, hide=self.hide, count=has_count,
@@ -267,7 +271,13 @@ class register:  # pylint: disable=invalid-name
             annotation_info: An AnnotationInfo tuple for the parameter.
         """
         kwargs = {}
+
+        try:
+            kwargs['help'] = self.docparser.arg_descs[param.name]
+        except KeyError:
+            pass
         typ = self._get_type(param, annotation_info)
+
         if isinstance(typ, tuple):
             pass
         elif utils.is_enum(typ):
@@ -282,6 +292,7 @@ class register:  # pylint: disable=invalid-name
         elif typ is not bool and param.default is not inspect.Parameter.empty:
             kwargs['default'] = param.default
             kwargs['nargs'] = '?'
+
         return kwargs
 
     def _parse_annotation(self, param):

qutebrowser/utils/utils.py

@@ -21,9 +21,11 @@
 
 import os
 import io
+import re
 import sys
 import enum
 import shlex
+import inspect
 import os.path
 import urllib.request
 import urllib.parse
@@ -36,7 +38,7 @@ from PyQt5.QtGui import QKeySequence, QColor
 import pkg_resources
 
 import qutebrowser
-from qutebrowser.utils import qtutils
+from qutebrowser.utils import usertypes, qtutils
 
 
 def elide(text, length):
@@ -503,3 +505,92 @@ def is_enum(obj):
         return issubclass(obj, enum.Enum)
     except TypeError:
         return False
+
+
+class DocstringParser:
+
+    """Generate documentation based on a docstring of a command handler.
+
+    The docstring needs to follow the format described in HACKING.
+    """
+
+    State = usertypes.enum('State', 'short', 'desc', 'desc_hidden',
+                           'arg_start', 'arg_inside', 'misc')
+
+    def __init__(self, func):
+        """Constructor.
+
+        Args:
+            func: The function to parse the docstring for.
+        """
+        self.state = self.State.short
+        self.short_desc = []
+        self.long_desc = []
+        self.arg_descs = collections.OrderedDict()
+        self.cur_arg_name = None
+        self.handlers = {
+            self.State.short: self._parse_short,
+            self.State.desc: self._parse_desc,
+            self.State.desc_hidden: self._skip,
+            self.State.arg_start: self._parse_arg_start,
+            self.State.arg_inside: self._parse_arg_inside,
+            self.State.misc: self._skip,
+        }
+        doc = inspect.getdoc(func)
+        for line in doc.splitlines():
+            handler = self.handlers[self.state]
+            stop = handler(line)
+            if stop:
+                break
+        for k, v in self.arg_descs.items():
+            self.arg_descs[k] = ' '.join(v)
+        self.long_desc = ' '.join(self.long_desc)
+        self.short_desc = ' '.join(self.short_desc)
+
+    def _process_arg(self, line):
+        """Helper method to process a line like 'fooarg: Blah blub'."""
+        self.cur_arg_name, argdesc = line.split(':', maxsplit=1)
+        self.cur_arg_name = self.cur_arg_name.strip().lstrip('*')
+        self.arg_descs[self.cur_arg_name] = [argdesc.strip()]
+
+    def _skip(self, line):
+        """Handler to ignore everything until we get 'Args:'."""
+        if line.startswith('Args:'):
+            self.state = self.State.arg_start
+
+    def _parse_short(self, line):
+        """Parse the short description (first block) in the docstring."""
+        if not line:
+            self.state = self.State.desc
+        else:
+            self.short_desc.append(line.strip())
+
+    def _parse_desc(self, line):
+        """Parse the long description in the docstring."""
+        if line.startswith('Args:'):
+            self.state = self.State.arg_start
+        elif line.startswith('Emit:') or line.startswith('Raise:'):
+            self.state = self.State.misc
+        elif line.strip() == '//':
+            self.state = self.State.desc_hidden
+        elif line.strip():
+            self.long_desc.append(line.strip())
+
+    def _parse_arg_start(self, line):
+        """Parse first argument line."""
+        self._process_arg(line)
+        self.state = self.State.arg_inside
+
+    def _parse_arg_inside(self, line):
+        """Parse subsequent argument lines."""
+        argname = self.cur_arg_name
+        if re.match(r'^[A-Z][a-z]+:$', line):
+            if not self.arg_descs[argname][-1].strip():
+                self.arg_descs[argname] = self.arg_descs[argname][:-1]
+                return True
+        elif not line.strip():
+            self.arg_descs[argname].append('\n\n')
+        elif line[4:].startswith(' '):
+            self.arg_descs[argname].append(line.strip() + '\n')
+        else:
+            self._process_arg(line)

scripts/generate_doc.py

@@ -20,7 +20,6 @@
 
 """Generate asciidoc source for qutebrowser based on docstrings."""
 
-import re
 import os
 import sys
 import html
@@ -38,7 +37,7 @@ import qutebrowser.app
 from qutebrowser import qutebrowser as qutequtebrowser
 from qutebrowser.commands import cmdutils
 from qutebrowser.config import configdata
-from qutebrowser.utils import usertypes
+from qutebrowser.utils import utils
 
 
 def _open_file(name, mode='w'):
@@ -46,75 +45,6 @@ def _open_file(name, mode='w'):
     return open(name, mode, newline='\n', encoding='utf-8')
 
 
-def _parse_docstring(func):  # noqa
-    """Generate documentation based on a docstring of a command handler.
-
-    The docstring needs to follow the format described in HACKING.
-
-    Args:
-        func: The function to generate the docstring for.
-
-    Return:
-        A (short_desc, long_desc, arg_descs) tuple.
-    """
-    # pylint: disable=too-many-branches
-    State = usertypes.enum('State', 'short',  # pylint: disable=invalid-name
-                           'desc', 'desc_hidden', 'arg_start', 'arg_inside',
-                           'misc')
-    doc = inspect.getdoc(func)
-    lines = doc.splitlines()
-
-    cur_state = State.short
-
-    short_desc = []
-    long_desc = []
-    arg_descs = collections.OrderedDict()
-    cur_arg_name = None
-
-    for line in lines:
-        if cur_state == State.short:
-            if not line:
-                cur_state = State.desc
-            else:
-                short_desc.append(line.strip())
-        elif cur_state == State.desc:
-            if line.startswith('Args:'):
-                cur_state = State.arg_start
-            elif line.startswith('Emit:') or line.startswith('Raise:'):
-                cur_state = State.misc
-            elif line.strip() == '//':
-                cur_state = State.desc_hidden
-            elif line.strip():
-                long_desc.append(line.strip())
-        elif cur_state == State.misc:
-            if line.startswith('Args:'):
-                cur_state = State.arg_start
-            else:
-                pass
-        elif cur_state == State.desc_hidden:
-            if line.startswith('Args:'):
-                cur_state = State.arg_start
-        elif cur_state == State.arg_start:
-            cur_arg_name, argdesc = line.split(':', maxsplit=1)
-            cur_arg_name = cur_arg_name.strip().lstrip('*')
-            arg_descs[cur_arg_name] = [argdesc.strip()]
-            cur_state = State.arg_inside
-        elif cur_state == State.arg_inside:
-            if re.match('^[A-Z][a-z]+:$', line):
-                if not arg_descs[cur_arg_name][-1].strip():
-                    arg_descs[cur_arg_name] = arg_descs[cur_arg_name][:-1]
-                    break
-            elif not line.strip():
-                arg_descs[cur_arg_name].append('\n\n')
-            elif line[4:].startswith(' '):
-                arg_descs[cur_arg_name].append(line.strip() + '\n')
-            else:
-                cur_arg_name, argdesc = line.split(':', maxsplit=1)
-                cur_arg_name = cur_arg_name.strip().lstrip('*')
-                arg_descs[cur_arg_name] = [argdesc.strip()]
-    return (short_desc, long_desc, arg_descs)
-
-
 def _get_cmd_syntax(name, cmd):
     """Get the command syntax for a command."""
     words = []
@@ -179,14 +109,14 @@ def _get_command_doc(name, cmd):
     if syntax != name:
         output.append('Syntax: +:{}+'.format(syntax))
     output.append("")
-    short_desc, long_desc, arg_descs = _parse_docstring(cmd.handler)
+    parser = utils.DocstringParser(cmd.handler)
-    output.append(' '.join(short_desc))
+    output.append(parser.short_desc)
     output.append("")
-    output.append(' '.join(long_desc))
+    output.append(parser.long_desc)
-    if arg_descs:
+    if parser.arg_descs:
         output.append("")
-        for arg, desc in arg_descs.items():
+        for arg, desc in parser.arg_descs.items():
-            text = ' '.join(desc).splitlines()
+            text = desc.splitlines()
             firstline = text[0].replace(', or None', '')
             item = "* +{}+: {}".format(arg, firstline)
             if arg in defaults: