From 46bde66147ead1bb234932a48e2c56fc4b698ec0 Mon Sep 17 00:00:00 2001
From: smolck <46855713+smolck@users.noreply.github.com>
Date: Sat, 26 Oct 2019 14:54:54 -0500
Subject: gen_vimdoc.py: dump API docs to msgpack #11296

Convenient for API clients who want to reuse the API docs in their own
docs. Could be used e.g. to eliminate nvim.net's own doxygen parser:
https://github.com/neovim/nvim.net/tree/3a736232a4e7b7a2a1eff4bded24d2bf27a918c2/src/NvimClient.APIGenerator/Docs

TODO: currently the result values are formatted as Vim help docs. We
should change the values to have structure, something like this:

    [{
      'nvim_win_get_var': [
        'line1,
        'line2',
        [ 'item1', 'item2', ... ]
      ],
      'nvim_win_set_var': [
        ...
      ],
      ...
    }]

close #11296
---
 scripts/gen_vimdoc.py | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 373a58d11e..aa50132d4c 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -36,11 +36,12 @@ import shutil
 import textwrap
 import subprocess
 import collections
+import msgpack
 
 from xml.dom import minidom
 
-if sys.version_info[0] < 3:
-    print("use Python 3")
+if sys.version_info[0] < 3 or sys.version_info[1] < 5:
+    print("requires Python 3.5+")
     sys.exit(1)
 
 DEBUG = ('DEBUG' in os.environ)
@@ -453,7 +454,7 @@ def parse_source_xml(filename, mode):
     """
     global xrefs
     xrefs = set()
-    functions = []
+    functions = {}  # Map of func_name:docstring.
     deprecated_functions = []
 
     dom = minidom.parse(filename)
@@ -577,11 +578,11 @@ def parse_source_xml(filename, mode):
         if 'Deprecated' in xrefs:
             deprecated_functions.append(func_doc)
         elif name.startswith(CONFIG[mode]['func_name_prefix']):
-            functions.append(func_doc)
+            functions[name] = func_doc
 
         xrefs.clear()
 
-    return '\n\n'.join(functions), '\n\n'.join(deprecated_functions)
+    return '\n\n'.join(list(functions.values())), '\n\n'.join(deprecated_functions), functions
 
 
 def delete_lines_below(filename, tokenstr):
@@ -604,6 +605,12 @@ def gen_docs(config):
     Doxygen is called and configured through stdin.
     """
     for mode in CONFIG:
+        functions = {}  # Map of func_name:docstring.
+        mpack_file = os.path.join(base_dir, 'runtime', 'doc',
+                CONFIG[mode]['filename'].replace('.txt', '.mpack'))
+        if os.path.exists(mpack_file):
+            os.remove(mpack_file)
+
         output_dir = out_dir.format(mode=mode)
         p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE)
         p.communicate(
@@ -645,14 +652,15 @@ def gen_docs(config):
 
             filename = get_text(find_first(compound, 'name'))
             if filename.endswith('.c') or filename.endswith('.lua'):
-                functions, deprecated = parse_source_xml(
+                functions_text, deprecated_text, fns = parse_source_xml(
                     os.path.join(base, '%s.xml' %
                                  compound.getAttribute('refid')), mode)
+                # Collect functions from all modules (for the current `mode`).
+                functions = {**functions, **fns}
 
-                if not functions and not deprecated:
+                if not functions_text and not deprecated_text:
                     continue
-
-                if functions or deprecated:
+                else:
                     name = os.path.splitext(os.path.basename(filename))[0]
                     if name == 'ui':
                         name = name.upper()
@@ -665,12 +673,12 @@ def gen_docs(config):
                     if intro:
                         doc += '\n\n' + intro
 
-                    if functions:
-                        doc += '\n\n' + functions
+                    if functions_text:
+                        doc += '\n\n' + functions_text
 
-                    if INCLUDE_DEPRECATED and deprecated:
+                    if INCLUDE_DEPRECATED and deprecated_text:
                         doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name
-                        doc += deprecated
+                        doc += deprecated_text
 
                     if doc:
                         filename = os.path.basename(filename)
@@ -713,6 +721,8 @@ def gen_docs(config):
         delete_lines_below(doc_file, CONFIG[mode]['section_start_token'])
         with open(doc_file, 'ab') as fp:
             fp.write(docs.encode('utf8'))
+        with open(mpack_file, 'wb') as fp:
+            fp.write(msgpack.packb(functions, use_bin_type=True))
 
         shutil.rmtree(output_dir)
 
-- 
cgit 


From 6d8fe9b3f40d0d3f4ddeed56f592dfd2be810f7f Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Sat, 26 Oct 2019 16:21:22 -0700
Subject: lint

---
 scripts/gen_vimdoc.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index aa50132d4c..8d0cf54828 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -582,7 +582,9 @@ def parse_source_xml(filename, mode):
 
         xrefs.clear()
 
-    return '\n\n'.join(list(functions.values())), '\n\n'.join(deprecated_functions), functions
+    return ('\n\n'.join(list(functions.values())),
+            '\n\n'.join(deprecated_functions),
+            functions)
 
 
 def delete_lines_below(filename, tokenstr):
@@ -606,7 +608,8 @@ def gen_docs(config):
     """
     for mode in CONFIG:
         functions = {}  # Map of func_name:docstring.
-        mpack_file = os.path.join(base_dir, 'runtime', 'doc',
+        mpack_file = os.path.join(
+                base_dir, 'runtime', 'doc',
                 CONFIG[mode]['filename'].replace('.txt', '.mpack'))
         if os.path.exists(mpack_file):
             os.remove(mpack_file)
@@ -653,8 +656,8 @@ def gen_docs(config):
             filename = get_text(find_first(compound, 'name'))
             if filename.endswith('.c') or filename.endswith('.lua'):
                 functions_text, deprecated_text, fns = parse_source_xml(
-                    os.path.join(base, '%s.xml' %
-                                 compound.getAttribute('refid')), mode)
+                    os.path.join(base, '{}.xml'.format(
+                                 compound.getAttribute('refid'))), mode)
                 # Collect functions from all modules (for the current `mode`).
                 functions = {**functions, **fns}
 
-- 
cgit 


From 4bd7881fce0c7183b97e7541b7a7ac872e08bbda Mon Sep 17 00:00:00 2001
From: smolck <46855713+smolck@users.noreply.github.com>
Date: Wed, 6 Nov 2019 16:22:20 -0600
Subject: [scripts/gen_vimdoc.py] Generate better-formatted mpack

Changes the generated msgpack result values in the runtime/doc/*.mpack
files to be formatted like this (instead of being formatted like Vim help text):
    [
        'nvim_win_get_var': {
                'signature': 'nvim_win_get_var({window}, {name}, {err})',
                'parameters': [('Window', 'window'), ('String', 'name'), ('Error *', 'err')],
                'parameters_doc': [{'window': 'Window handle, or 0 for current window', 'name': 'Variable name'}],
                'doc': ['Gets a window-scoped (w:) variable'],
                'return': ['Variable value'],
                'seealso': []
            }
        },
        ...
    ]
---
 scripts/gen_vimdoc.py | 259 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 254 insertions(+), 5 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 8d0cf54828..df06644118 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -262,6 +262,36 @@ def has_nonexcluded_params(nodes):
             return True
 
 
+def params_as_map(parent, width=62):
+    """Returns a Doxygen <parameterlist> as a map of
+    <parameter name>:<description>"""
+    items = []
+    for node in parent.childNodes:
+        if node.nodeType == node.TEXT_NODE:
+            continue
+
+        name_node = find_first(node, 'parametername')
+        if name_node.getAttribute('direction') == 'out':
+            continue
+
+        name = get_text(name_node)
+        if name in param_exclude:
+            continue
+
+        items.append((name.strip(), node))
+
+    out = {}  # Map of name:desc
+    for name, node in items:
+        desc = ''
+        desc_node = get_child(node, 'parameterdescription')
+        if desc_node:
+            desc = parse_parblock(desc_node, width=width,
+                                  indent=(' ' * len(name)))
+            out[name] = desc
+
+    return out
+
+
 def render_params(parent, width=62):
     """Renders Doxygen <parameterlist> tag as Vim help text."""
     name_length = 0
@@ -365,6 +395,86 @@ def render_node(n, text, prefix='', indent='', width=62):
     return text
 
 
+def para_as_map(parent, indent='', width=62):
+    """Parses XML <para> as a map
+
+    The map returned may or may not contain
+    all of the following keys:
+        'text': Text in the para tag
+        'params': Information from <parameterlist>'s
+        'return': TODO
+        'seealso': TODO
+        'xrefs': TODO
+    """
+    if is_inline(parent):
+        return {
+            'text': clean_lines(doc_wrap(render_node(parent, ''),
+                                indent=indent, width=width).strip())
+        }
+
+    # Ordered dict of ordered lists.
+    groups = collections.OrderedDict([
+        ('params', []),
+        ('return', []),
+        ('seealso', []),
+        ('xrefs', []),
+    ])
+
+    # Gather nodes into groups.  Mostly this is because we want "parameterlist"
+    # nodes to appear together.
+    text = ''
+    kind = ''
+    last = ''
+    for child in parent.childNodes:
+        if child.nodeName == 'parameterlist':
+            groups['params'].append(child)
+        elif child.nodeName == 'xrefsect':
+            groups['xrefs'].append(child)
+        elif child.nodeName == 'simplesect':
+            last = kind
+            kind = child.getAttribute('kind')
+            if kind == 'return' or (kind == 'note' and last == 'return'):
+                groups['return'].append(child)
+            elif kind == 'see':
+                groups['seealso'].append(child)
+            elif kind in ('note', 'warning'):
+                text += render_node(child, text, indent=indent, width=width)
+            else:
+                raise RuntimeError('unhandled simplesect: {}\n{}'.format(
+                    child.nodeName, child.toprettyxml(indent='  ', newl='\n')))
+        else:
+            text += render_node(child, text, indent=indent, width=width)
+
+    chunks = {
+        'text': text,
+        'params': [],
+        'return': [],
+        'seealso': [],
+        'xrefs': []
+    }
+
+    # Generate text from the gathered items.
+    if len(groups['params']) > 0 and has_nonexcluded_params(groups['params']):
+        for child in groups['params']:
+            chunks['params'].append(params_as_map(child, width=width))
+    if len(groups['return']) > 0:
+        for child in groups['return']:
+            chunks['return'].append(render_node(
+                child, '', indent=indent, width=width).lstrip())
+    if len(groups['seealso']) > 0:
+        for child in groups['seealso']:
+            chunks['seealso'].append(render_node(
+                child, '', indent=indent, width=width))
+    for child in groups['xrefs']:
+        title = get_text(get_child(child, 'xreftitle'))
+        xrefs.add(title)
+        xrefdesc = render_para(get_child(child, 'xrefdescription'), width=width)
+        chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title),
+                                        width=width) + '\n')
+
+    return chunks
+
+
 def render_para(parent, indent='', width=62):
     """Renders Doxygen <para> containing arbitrary nodes.
 
@@ -433,6 +543,16 @@ def render_para(parent, indent='', width=62):
     return clean_lines('\n'.join(chunks).strip())
 
 
+def parse_parblock_as_array(parent, prefix='', width=62, indent=''):
+    """Parses a nested block of <para> tags as an array of maps"""
+    lines = []
+    for child in parent.childNodes:
+        rendered = para_as_map(child, width=width, indent=indent)
+        lines.append(rendered)
+
+    return lines
+
+
 def parse_parblock(parent, prefix='', width=62, indent=''):
     """Renders a nested block of <para> tags as Vim help text."""
     paragraphs = []
@@ -443,13 +563,136 @@ def parse_parblock(parent, prefix='', width=62, indent=''):
 # }}}
 
 
+def parse_source_xml_for_mpack(filename, mode):
+    """Collects API functions to be packed as msgpack.
+
+    Returns two maps:
+      1. API functions
+      2. Deprecated API functions
+
+    Caller decides what to do with the deprecated documentation.
+    """
+    global xrefs
+    xrefs = set()
+    functions = {}  # Map of func_name:docstring.
+    deprecated_functions = {}  # Map of func_name:docstring.
+
+    dom = minidom.parse(filename)
+    for member in dom.getElementsByTagName('memberdef'):
+        if member.getAttribute('static') == 'yes' or \
+                member.getAttribute('kind') != 'function' or \
+                member.getAttribute('prot') == 'private' or \
+                get_text(get_child(member, 'name')).startswith('_'):
+            continue
+
+        loc = find_first(member, 'location')
+        if 'private' in loc.getAttribute('file'):
+            continue
+
+        return_type = get_text(get_child(member, 'type'))
+        if return_type == '':
+            continue
+
+        if return_type.startswith(('ArrayOf', 'DictionaryOf')):
+            parts = return_type.strip('_').split('_')
+            return_type = '{}({})'.format(parts[0], ', '.join(parts[1:]))
+
+        name = get_text(get_child(member, 'name'))
+
+        annotations = get_text(get_child(member, 'argsstring'))
+        if annotations and ')' in annotations:
+            annotations = annotations.rsplit(')', 1)[-1].strip()
+        # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of
+        # non-void functions.  Special-case void functions here.
+        if name == 'nvim_get_mode' and len(annotations) == 0:
+            annotations += 'FUNC_API_FAST'
+        annotations = filter(None, map(lambda x: annotation_map.get(x),
+                                       annotations.split()))
+
+        params = []
+        type_length = 0
+
+        for param in get_children(member, 'param'):
+            param_type = get_text(get_child(param, 'type')).strip()
+            param_name = ''
+            declname = get_child(param, 'declname')
+            if declname:
+                param_name = get_text(declname).strip()
+            elif mode == 'lua':
+                # that's how it comes out of lua2dox
+                param_name = param_type
+                param_type = ''
+
+            if param_name in param_exclude:
+                continue
+
+            type_length = max(type_length, len(param_type))
+            params.append((param_type, param_name))
+
+        c_args = []
+        for param_type, param_name in params:
+            c_args.append((
+                '%s %s' % (param_type.ljust(type_length), param_name)).strip())
+
+        c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args))
+
+        doc = ''
+        desc = find_first(member, 'detaileddescription')
+        if desc:
+            doc = parse_parblock_as_array(desc)
+            if DEBUG:
+                print(textwrap.indent(
+                    re.sub(r'\n\s*\n+', '\n',
+                           desc.toprettyxml(indent='  ', newl='\n')), ' ' * 16))
+
+        prefix = '%s(' % name
+        suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params
+                                   if a[0] not in ('void', 'Error'))
+
+        signature = prefix + suffix
+
+        parameters_doc = []
+        func_doc = []
+        func_return = ''
+        seealso = ''
+        for m in doc:
+            if 'text' in m:
+                if not m['text'] == '':
+                    func_doc.append(m['text'])
+            if 'params' in m:
+                parameters_doc += m['params']
+            if 'return' in m:
+                func_return = m['return']
+            if 'seealso' in m:
+                seealso = m['xrefs']
+
+        function_map = {
+            'signature': signature,
+            'parameters': params,
+            'parameters_doc': parameters_doc,
+            'doc': func_doc,
+            'return': func_return,
+            'seealso': seealso
+        }
+
+        if INCLUDE_C_DECL:
+            function_map['c_decl'] = c_decl
+
+        if 'Deprecated' in xrefs:
+            deprecated_functions[name] = function_map
+        elif name.startswith(CONFIG[mode]['func_name_prefix']):
+            functions[name] = function_map
+
+        xrefs.clear()
+
+    return (functions, deprecated_functions)
+
+
 def parse_source_xml(filename, mode):
     """Collects API functions.
-
     Returns two strings:
       1. API functions
       2. Deprecated API functions
-
     Caller decides what to do with the deprecated documentation.
     """
     global xrefs
@@ -609,8 +852,8 @@ def gen_docs(config):
     for mode in CONFIG:
         functions = {}  # Map of func_name:docstring.
         mpack_file = os.path.join(
-                base_dir, 'runtime', 'doc',
-                CONFIG[mode]['filename'].replace('.txt', '.mpack'))
+            base_dir, 'runtime', 'doc',
+            CONFIG[mode]['filename'].replace('.txt', '.mpack'))
         if os.path.exists(mpack_file):
             os.remove(mpack_file)
 
@@ -627,6 +870,7 @@ def gen_docs(config):
         if p.returncode:
             sys.exit(p.returncode)
 
+        mpacks = {}
         sections = {}
         intros = {}
         sep = '=' * text_width
@@ -655,6 +899,9 @@ def gen_docs(config):
 
             filename = get_text(find_first(compound, 'name'))
             if filename.endswith('.c') or filename.endswith('.lua'):
+                mpack = parse_source_xml_for_mpack(os.path.join(base, '{}.xml'.format(
+                    compound.getAttribute('refid'))), mode)
+
                 functions_text, deprecated_text, fns = parse_source_xml(
                     os.path.join(base, '{}.xml'.format(
                                  compound.getAttribute('refid'))), mode)
@@ -694,6 +941,7 @@ def gen_docs(config):
                             title = '{} Functions'.format(name)
                             helptag = '*api-{}*'.format(name.lower())
                         sections[filename] = (title, helptag, doc)
+                        mpacks[filename] = mpack
 
         if not sections:
             return
@@ -724,8 +972,9 @@ def gen_docs(config):
         delete_lines_below(doc_file, CONFIG[mode]['section_start_token'])
         with open(doc_file, 'ab') as fp:
             fp.write(docs.encode('utf8'))
+
         with open(mpack_file, 'wb') as fp:
-            fp.write(msgpack.packb(functions, use_bin_type=True))
+            fp.write(msgpack.packb(mpacks, use_bin_type=True))
 
         shutil.rmtree(output_dir)
 
-- 
cgit 


From af53a0c0123338575dd59934449d7fe836835d1c Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Sun, 17 Nov 2019 19:06:59 -0800
Subject: doc: Lua [ci skip] #11378

- Rework :help lua-commands
- Rename if_lua.txt => lua.txt
---
 scripts/gen_vimdoc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 8d0cf54828..4d71d5e15e 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -85,7 +85,7 @@ CONFIG = {
         'append_only': [],
     },
     'lua': {
-        'filename': 'if_lua.txt',
+        'filename': 'lua.txt',
         'section_start_token': '*lua-vim*',
         'section_order': [
             'vim.lua',
-- 
cgit 


From 01adb81c55c16491372842001f58d810ea2524c2 Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Sun, 17 Nov 2019 22:49:28 -0800
Subject: gen_vimdoc.py: DRY

---
 scripts/gen_vimdoc.py | 542 ++++++++++++++++++--------------------------------
 1 file changed, 198 insertions(+), 344 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index df06644118..cd44667ef3 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -9,25 +9,22 @@ This would be easier using lxml and XSLT, but:
   2. I wouldn't know how to deal with nested indentation in <para> tags using
      XSLT.
 
-Each function documentation is formatted with the following rules:
+Each function :help block is formatted as follows:
 
-  - Maximum width of 78 characters (`text_width`).
-  - Spaces for indentation.
-  - Function signature and helptag are on the same line.
-    - Helptag is right aligned.
+  - Max width of 78 columns (`text_width`).
+  - Indent with spaces (not tabs).
+  - Indent of 16 columns for body text.
+  - Function signature and helptag (right-aligned) on the same line.
     - Signature and helptag must have a minimum of 8 spaces between them.
-    - If the signature is too long, it is placed on the line after the
-      helptag.  The signature wraps at `text_width - 8` characters with
-      subsequent lines indented to the open parenthesis.
-  - Documentation body will be indented by 16 spaces.
+    - If the signature is too long, it is placed on the line after the helptag.
+      Signature wraps at `text_width - 8` characters with subsequent
+      lines indented to the open parenthesis.
     - Subsection bodies are indented an additional 4 spaces.
-  - Documentation body consists of the function description, parameter details,
-    return description, and C declaration.
+  - Body consists of function description, parameters, return description, and
+    C declaration (`INCLUDE_C_DECL`).
   - Parameters are omitted for the `void` and `Error *` types, or if the
     parameter is marked as [out].
   - Each function documentation is separated by a single line.
-
-The C declaration is added to the end to show actual argument types.
 """
 import os
 import re
@@ -123,14 +120,21 @@ annotation_map = {
 xrefs = set()
 
 
-def debug_this(s, n):
-    o = n if isinstance(n, str) else n.toprettyxml(indent='  ', newl='\n')
-    name = '' if isinstance(n, str) else n.nodeName
-    if s in o:
+# Raises an error with details about `o`, if `cond` is in object `o`,
+# or if `cond()` is callable and returns True.
+def debug_this(cond, o):
+    name = ''
+    if not isinstance(o, str):
+        try:
+            name = o.nodeName
+            o = o.toprettyxml(indent='  ', newl='\n')
+        except:
+            pass
+    if ((callable(cond) and cond())
+            or (not callable(cond) and cond in o)):
         raise RuntimeError('xxx: {}\n{}'.format(name, o))
 
 
-# XML Parsing Utilities {{{
 def find_first(parent, name):
     """Finds the first matching node within parent."""
     sub = parent.getElementsByTagName(name)
@@ -254,74 +258,42 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None):
     return result
 
 
-def has_nonexcluded_params(nodes):
-    """Returns true if any of the given <parameterlist> elements has at least
-    one non-excluded item."""
-    for n in nodes:
-        if render_params(n) != '':
-            return True
 
 
-def params_as_map(parent, width=62):
-    """Returns a Doxygen <parameterlist> as a map of
-    <parameter name>:<description>"""
+def update_params_map(parent, ret_map, width=62):
+    """Updates `ret_map` with name:desc key-value pairs extracted
+    from a Doxygen <parameterlist>.
+    """
     items = []
     for node in parent.childNodes:
         if node.nodeType == node.TEXT_NODE:
             continue
-
         name_node = find_first(node, 'parametername')
         if name_node.getAttribute('direction') == 'out':
             continue
-
         name = get_text(name_node)
         if name in param_exclude:
             continue
-
         items.append((name.strip(), node))
-
-    out = {}  # Map of name:desc
+    # `ret_map` is a name:desc map.
     for name, node in items:
         desc = ''
         desc_node = get_child(node, 'parameterdescription')
         if desc_node:
             desc = parse_parblock(desc_node, width=width,
                                   indent=(' ' * len(name)))
-            out[name] = desc
-
-    return out
+            ret_map[name] = desc
+    return ret_map
 
 
-def render_params(parent, width=62):
-    """Renders Doxygen <parameterlist> tag as Vim help text."""
+def fmt_params_map_as_vimhelp(m, width=62):
+    """Renders a params map as Vim :help text."""
     name_length = 0
-    items = []
-    for node in parent.childNodes:
-        if node.nodeType == node.TEXT_NODE:
-            continue
-
-        name_node = find_first(node, 'parametername')
-        if name_node.getAttribute('direction') == 'out':
-            continue
-
-        name = get_text(name_node)
-        if name in param_exclude:
-            continue
-
-        name = '{%s}' % name
-        name_length = max(name_length, len(name) + 2)
-        items.append((name.strip(), node))
-
+    for name, desc in m.items():
+        name_length = max(name_length, len(name) + 4)
     out = ''
-    for name, node in items:
-        name = '    {}'.format(name.ljust(name_length))
-
-        desc = ''
-        desc_node = get_child(node, 'parameterdescription')
-        if desc_node:
-            desc = parse_parblock(desc_node, width=width,
-                                  indent=(' ' * len(name)))
-
+    for name, desc in m.items():
+        name = '    {}'.format('{{{}}}'.format(name).ljust(name_length))
         out += '{}{}\n'.format(name, desc)
     return out.rstrip()
 
@@ -396,21 +368,26 @@ def render_node(n, text, prefix='', indent='', width=62):
 
 
 def para_as_map(parent, indent='', width=62):
-    """Parses XML <para> as a map
-
-    The map returned may or may not contain
-    all of the following keys:
-        'text': Text in the para tag
-        'params': Information from <parameterlist>'s
-        'return': TODO
-        'seealso': TODO
-        'xrefs': TODO
+    """Extracts a Doxygen XML <para> node to a map.
+
+    Keys:
+        'text': Text from this <para> element
+        'params': <parameterlist> map
+        'return': List of @return strings
+        'seealso': List of @see strings
+        'xrefs': ?
     """
+    chunks = {
+        'text': '',
+        'params': collections.OrderedDict(),
+        'return': [],
+        'seealso': [],
+        'xrefs': []
+    }
+
     if is_inline(parent):
-        return {
-            'text': clean_lines(doc_wrap(render_node(parent, ''),
-                                indent=indent, width=width).strip())
-        }
+        chunks['text'] = clean_lines(doc_wrap(render_node(parent, ''),
+            indent=indent, width=width).strip())
 
     # Ordered dict of ordered lists.
     groups = collections.OrderedDict([
@@ -445,26 +422,18 @@ def para_as_map(parent, indent='', width=62):
         else:
             text += render_node(child, text, indent=indent, width=width)
 
-    chunks = {
-        'text': text,
-        'params': [],
-        'return': [],
-        'seealso': [],
-        'xrefs': []
-    }
+    chunks['text'] = text
 
-    # Generate text from the gathered items.
-    if len(groups['params']) > 0 and has_nonexcluded_params(groups['params']):
+    # Generate map from the gathered items.
+    if len(groups['params']) > 0:
         for child in groups['params']:
-            chunks['params'].append(params_as_map(child, width=width))
-    if len(groups['return']) > 0:
-        for child in groups['return']:
-            chunks['return'].append(render_node(
-                child, '', indent=indent, width=width).lstrip())
-    if len(groups['seealso']) > 0:
-        for child in groups['seealso']:
-            chunks['seealso'].append(render_node(
-                child, '', indent=indent, width=width))
+            update_params_map(child, ret_map=chunks['params'], width=width)
+    for child in groups['return']:
+        chunks['return'].append(render_node(
+            child, '', indent=indent, width=width).lstrip())
+    for child in groups['seealso']:
+        chunks['seealso'].append(render_node(
+            child, '', indent=indent, width=width))
     for child in groups['xrefs']:
         title = get_text(get_child(child, 'xreftitle'))
         xrefs.add(title)
@@ -480,79 +449,33 @@ def render_para(parent, indent='', width=62):
 
     NB: Blank lines in a docstring manifest as <para> tags.
     """
-    if is_inline(parent):
-        return clean_lines(doc_wrap(render_node(parent, ''),
-                                    indent=indent, width=width).strip())
+    para = para_as_map(parent, indent, width)
 
-    # Ordered dict of ordered lists.
-    groups = collections.OrderedDict([
-        ('params', []),
-        ('return', []),
-        ('seealso', []),
-        ('xrefs', []),
-    ])
-
-    # Gather nodes into groups.  Mostly this is because we want "parameterlist"
-    # nodes to appear together.
-    text = ''
-    kind = ''
-    last = ''
-    for child in parent.childNodes:
-        if child.nodeName == 'parameterlist':
-            groups['params'].append(child)
-        elif child.nodeName == 'xrefsect':
-            groups['xrefs'].append(child)
-        elif child.nodeName == 'simplesect':
-            last = kind
-            kind = child.getAttribute('kind')
-            if kind == 'return' or (kind == 'note' and last == 'return'):
-                groups['return'].append(child)
-            elif kind == 'see':
-                groups['seealso'].append(child)
-            elif kind in ('note', 'warning'):
-                text += render_node(child, text, indent=indent, width=width)
-            else:
-                raise RuntimeError('unhandled simplesect: {}\n{}'.format(
-                    child.nodeName, child.toprettyxml(indent='  ', newl='\n')))
-        else:
-            text += render_node(child, text, indent=indent, width=width)
+    def has_nonexcluded_params(m):
+        """Returns true if any of the given params has at least
+        one non-excluded item."""
+        if fmt_params_map_as_vimhelp(m) != '':
+            return True
 
-    chunks = [text]
     # Generate text from the gathered items.
-    if len(groups['params']) > 0 and has_nonexcluded_params(groups['params']):
+    chunks = [para['text']]
+    if len(para['params']) > 0 and has_nonexcluded_params(para['params']):
         chunks.append('\nParameters: ~')
-        for child in groups['params']:
-            chunks.append(render_params(child, width=width))
-    if len(groups['return']) > 0:
+        chunks.append(fmt_params_map_as_vimhelp(para['params'], width=width))
+    if len(para['return']) > 0:
         chunks.append('\nReturn: ~')
-        for child in groups['return']:
-            chunks.append(render_node(
-                child, chunks[-1][-1], indent=indent, width=width))
-    if len(groups['seealso']) > 0:
+        for s in para['return']:
+            chunks.append(s)
+    if len(para['seealso']) > 0:
         chunks.append('\nSee also: ~')
-        for child in groups['seealso']:
-            chunks.append(render_node(
-                child, chunks[-1][-1], indent=indent, width=width))
-    for child in groups['xrefs']:
-        title = get_text(get_child(child, 'xreftitle'))
-        xrefs.add(title)
-        xrefdesc = render_para(get_child(child, 'xrefdescription'), width=width)
-        chunks.append(doc_wrap(xrefdesc, prefix='{}: '.format(title),
-                               width=width) + '\n')
+        for s in para['seealso']:
+            chunks.append(s)
+    for s in para['xrefs']:
+        chunks.append(s)
 
     return clean_lines('\n'.join(chunks).strip())
 
 
-def parse_parblock_as_array(parent, prefix='', width=62, indent=''):
-    """Parses a nested block of <para> tags as an array of maps"""
-    lines = []
-    for child in parent.childNodes:
-        rendered = para_as_map(child, width=width, indent=indent)
-        lines.append(rendered)
-
-    return lines
-
-
 def parse_parblock(parent, prefix='', width=62, indent=''):
     """Renders a nested block of <para> tags as Vim help text."""
     paragraphs = []
@@ -560,17 +483,17 @@ def parse_parblock(parent, prefix='', width=62, indent=''):
         paragraphs.append(render_para(child, width=width, indent=indent))
         paragraphs.append('')
     return clean_lines('\n'.join(paragraphs).strip())
-# }}}
 
 
-def parse_source_xml_for_mpack(filename, mode):
-    """Collects API functions to be packed as msgpack.
+def extract_from_xml(filename, mode, fmt_vimhelp):
+    """Extracts Doxygen info as maps without formatting the text.
 
     Returns two maps:
-      1. API functions
-      2. Deprecated API functions
+      1. Functions
+      2. Deprecated functions
 
-    Caller decides what to do with the deprecated documentation.
+    The `fmt_vimhelp` parameter controls some special cases for use by
+    fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :)
     """
     global xrefs
     xrefs = set()
@@ -578,6 +501,7 @@ def parse_source_xml_for_mpack(filename, mode):
     deprecated_functions = {}  # Map of func_name:docstring.
 
     dom = minidom.parse(filename)
+    compoundname = get_text(dom.getElementsByTagName('compoundname')[0])
     for member in dom.getElementsByTagName('memberdef'):
         if member.getAttribute('static') == 'yes' or \
                 member.getAttribute('kind') != 'function' or \
@@ -609,6 +533,15 @@ def parse_source_xml_for_mpack(filename, mode):
         annotations = filter(None, map(lambda x: annotation_map.get(x),
                                        annotations.split()))
 
+        if not fmt_vimhelp:
+            pass
+        elif mode == 'lua':
+            fstem = compoundname.split('.')[0]
+            fstem = CONFIG[mode]['module_override'].get(fstem, fstem)
+            vimtag = '*{}.{}()*'.format(fstem, name)
+        else:
+            vimtag = '*{}()*'.format(name)
+
         params = []
         type_length = 0
 
@@ -626,180 +559,103 @@ def parse_source_xml_for_mpack(filename, mode):
             if param_name in param_exclude:
                 continue
 
+            if fmt_vimhelp and param_type.endswith('*'):
+                param_type = param_type.strip('* ')
+                param_name = '*' + param_name
             type_length = max(type_length, len(param_type))
             params.append((param_type, param_name))
 
         c_args = []
         for param_type, param_name in params:
-            c_args.append((
+            c_args.append(('    ' if fmt_vimhelp else '') + (
                 '%s %s' % (param_type.ljust(type_length), param_name)).strip())
 
-        c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args))
+        prefix = '%s(' % name
+        suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params
+                                   if a[0] not in ('void', 'Error'))
+        if not fmt_vimhelp:
+            c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args))
+            signature = prefix + suffix
+        else:
+            c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name,
+                                                         ',\n'.join(c_args)),
+                                     '    ')
+
+            # Minimum 8 chars between signature and vimtag
+            lhs = (text_width - 8) - len(prefix)
+
+            if len(prefix) + len(suffix) > lhs:
+                signature = vimtag.rjust(text_width) + '\n'
+                signature += doc_wrap(suffix, width=text_width-8, prefix=prefix,
+                                      func=True)
+            else:
+                signature = prefix + suffix
+                signature += vimtag.rjust(text_width - len(signature))
 
-        doc = ''
+        paras = []
         desc = find_first(member, 'detaileddescription')
         if desc:
-            doc = parse_parblock_as_array(desc)
+            for child in desc.childNodes:
+                paras.append(para_as_map(child))  #, width=width, indent=indent))
             if DEBUG:
                 print(textwrap.indent(
                     re.sub(r'\n\s*\n+', '\n',
                            desc.toprettyxml(indent='  ', newl='\n')), ' ' * 16))
 
-        prefix = '%s(' % name
-        suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params
-                                   if a[0] not in ('void', 'Error'))
-
-        signature = prefix + suffix
+        fn = {
+            'annotations': list(annotations),
+            'signature': signature,
+            'parameters': params,
+            'parameters_doc': [],
+            'doc': [],
+            'return': '',
+            'seealso': '',
+        }
+        if fmt_vimhelp:
+            fn['desc_node'] = desc  # HACK :(
 
-        parameters_doc = []
-        func_doc = []
-        func_return = ''
-        seealso = ''
-        for m in doc:
+        for m in paras:
             if 'text' in m:
                 if not m['text'] == '':
-                    func_doc.append(m['text'])
+                    fn['doc'].append(m['text'])
             if 'params' in m:
-                parameters_doc += m['params']
+                fn['parameters_doc'] += m['params']
             if 'return' in m:
-                func_return = m['return']
+                fn['return'] = m['return']
             if 'seealso' in m:
-                seealso = m['xrefs']
-
-        function_map = {
-            'signature': signature,
-            'parameters': params,
-            'parameters_doc': parameters_doc,
-            'doc': func_doc,
-            'return': func_return,
-            'seealso': seealso
-        }
+                fn['seealso'] = m['xrefs']
 
         if INCLUDE_C_DECL:
-            function_map['c_decl'] = c_decl
+            fn['c_decl'] = c_decl
 
         if 'Deprecated' in xrefs:
-            deprecated_functions[name] = function_map
+            deprecated_functions[name] = fn
         elif name.startswith(CONFIG[mode]['func_name_prefix']):
-            functions[name] = function_map
+            functions[name] = fn
 
         xrefs.clear()
 
     return (functions, deprecated_functions)
 
 
-def parse_source_xml(filename, mode):
-    """Collects API functions.
+def fmt_doxygen_xml_as_vimhelp(filename, mode):
+    """Formats functions from doxygen XML into Vim :help format.
+
     Returns two strings:
-      1. API functions
-      2. Deprecated API functions
-    Caller decides what to do with the deprecated documentation.
+      1. Functions in Vim :help format
+      2. Deprecated functions (handled by caller, or ignored)
     """
-    global xrefs
-    xrefs = set()
     functions = {}  # Map of func_name:docstring.
-    deprecated_functions = []
-
-    dom = minidom.parse(filename)
-    compoundname = get_text(dom.getElementsByTagName('compoundname')[0])
-    for member in dom.getElementsByTagName('memberdef'):
-        if member.getAttribute('static') == 'yes' or \
-                member.getAttribute('kind') != 'function' or \
-                member.getAttribute('prot') == 'private' or \
-                get_text(get_child(member, 'name')).startswith('_'):
-            continue
-
-        loc = find_first(member, 'location')
-        if 'private' in loc.getAttribute('file'):
-            continue
-
-        return_type = get_text(get_child(member, 'type'))
-        if return_type == '':
-            continue
-
-        if return_type.startswith(('ArrayOf', 'DictionaryOf')):
-            parts = return_type.strip('_').split('_')
-            return_type = '{}({})'.format(parts[0], ', '.join(parts[1:]))
-
-        name = get_text(get_child(member, 'name'))
-
-        annotations = get_text(get_child(member, 'argsstring'))
-        if annotations and ')' in annotations:
-            annotations = annotations.rsplit(')', 1)[-1].strip()
-        # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of
-        # non-void functions.  Special-case void functions here.
-        if name == 'nvim_get_mode' and len(annotations) == 0:
-            annotations += 'FUNC_API_FAST'
-        annotations = filter(None, map(lambda x: annotation_map.get(x),
-                                       annotations.split()))
-
-        if mode == 'lua':
-            fstem = compoundname.split('.')[0]
-            fstem = CONFIG[mode]['module_override'].get(fstem, fstem)
-            vimtag = '*{}.{}()*'.format(fstem, name)
-        else:
-            vimtag = '*{}()*'.format(name)
-
-        params = []
-        type_length = 0
-
-        for param in get_children(member, 'param'):
-            param_type = get_text(get_child(param, 'type')).strip()
-            param_name = ''
-            declname = get_child(param, 'declname')
-            if declname:
-                param_name = get_text(declname).strip()
-            elif mode == 'lua':
-                # that's how it comes out of lua2dox
-                param_name = param_type
-                param_type = ''
-
-            if param_name in param_exclude:
-                continue
-
-            if param_type.endswith('*'):
-                param_type = param_type.strip('* ')
-                param_name = '*' + param_name
-            type_length = max(type_length, len(param_type))
-            params.append((param_type, param_name))
-
-        c_args = []
-        for param_type, param_name in params:
-            c_args.append('    ' + (
-                '%s %s' % (param_type.ljust(type_length), param_name)).strip())
-
-        c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name,
-                                                     ',\n'.join(c_args)),
-                                 '    ')
-
-        prefix = '%s(' % name
-        suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params
-                                   if a[0] not in ('void', 'Error'))
-
-        # Minimum 8 chars between signature and vimtag
-        lhs = (text_width - 8) - len(prefix)
-
-        if len(prefix) + len(suffix) > lhs:
-            signature = vimtag.rjust(text_width) + '\n'
-            signature += doc_wrap(suffix, width=text_width-8, prefix=prefix,
-                                  func=True)
-        else:
-            signature = prefix + suffix
-            signature += vimtag.rjust(text_width - len(signature))
-
-        doc = ''
-        desc = find_first(member, 'detaileddescription')
-        if desc:
-            doc = parse_parblock(desc)
-            if DEBUG:
-                print(textwrap.indent(
-                    re.sub(r'\n\s*\n+', '\n',
-                           desc.toprettyxml(indent='  ', newl='\n')), ' ' * 16))
+    deprecated_functions = {}  # Map of func_name:docstring.
+    fns, deprecated_fns = extract_from_xml(filename, mode, True)
 
+    for name, fn in fns.items():
+        if fn['desc_node']:
+            doc = parse_parblock(fn['desc_node'])
         if not doc:
             doc = 'TODO: Documentation'
 
-        annotations = '\n'.join(annotations)
+        annotations = '\n'.join(fn['annotations'])
         if annotations:
             annotations = ('\n\nAttributes: ~\n' +
                            textwrap.indent(annotations, '    '))
@@ -811,10 +667,10 @@ def parse_source_xml(filename, mode):
 
         if INCLUDE_C_DECL:
             doc += '\n\nC Declaration: ~\n>\n'
-            doc += c_decl
+            doc += fn['c_decl']
             doc += '\n<'
 
-        func_doc = signature + '\n'
+        func_doc = fn['signature'] + '\n'
         func_doc += textwrap.indent(clean_lines(doc), ' ' * 16)
         func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M)
 
@@ -826,7 +682,7 @@ def parse_source_xml(filename, mode):
         xrefs.clear()
 
     return ('\n\n'.join(list(functions.values())),
-            '\n\n'.join(deprecated_functions),
+            '\n\n'.join(deprecated_fns),
             functions)
 
 
@@ -870,7 +726,7 @@ def gen_docs(config):
         if p.returncode:
             sys.exit(p.returncode)
 
-        mpacks = {}
+        doc_maps = {}
         sections = {}
         intros = {}
         sep = '=' * text_width
@@ -899,10 +755,10 @@ def gen_docs(config):
 
             filename = get_text(find_first(compound, 'name'))
             if filename.endswith('.c') or filename.endswith('.lua'):
-                mpack = parse_source_xml_for_mpack(os.path.join(base, '{}.xml'.format(
-                    compound.getAttribute('refid'))), mode)
+                mpack = extract_from_xml(os.path.join(base, '{}.xml'.format(
+                    compound.getAttribute('refid'))), mode, False)
 
-                functions_text, deprecated_text, fns = parse_source_xml(
+                functions_text, deprecated_text, fns = fmt_doxygen_xml_as_vimhelp(
                     os.path.join(base, '{}.xml'.format(
                                  compound.getAttribute('refid'))), mode)
                 # Collect functions from all modules (for the current `mode`).
@@ -941,7 +797,7 @@ def gen_docs(config):
                             title = '{} Functions'.format(name)
                             helptag = '*api-{}*'.format(name.lower())
                         sections[filename] = (title, helptag, doc)
-                        mpacks[filename] = mpack
+                        doc_maps[filename] = mpack
 
         if not sections:
             return
@@ -974,7 +830,7 @@ def gen_docs(config):
             fp.write(docs.encode('utf8'))
 
         with open(mpack_file, 'wb') as fp:
-            fp.write(msgpack.packb(mpacks, use_bin_type=True))
+            fp.write(msgpack.packb(doc_maps, use_bin_type=True))
 
         shutil.rmtree(output_dir)
 
@@ -994,42 +850,40 @@ def filter_source(filename):
                          fp.read(), flags=re.M))
 
 
-# Doxygen Config {{{
-Doxyfile = '''
-OUTPUT_DIRECTORY       = {output}
-INPUT                  = {input}
-INPUT_ENCODING         = UTF-8
-FILE_PATTERNS          = {file_patterns}
-RECURSIVE              = YES
-INPUT_FILTER           = "{filter}"
-EXCLUDE                =
-EXCLUDE_SYMLINKS       = NO
-EXCLUDE_PATTERNS       = */private/*
-EXCLUDE_SYMBOLS        =
-EXTENSION_MAPPING      = lua=C
-EXTRACT_PRIVATE        = NO
-
-GENERATE_HTML          = NO
-GENERATE_DOCSET        = NO
-GENERATE_HTMLHELP      = NO
-GENERATE_QHP           = NO
-GENERATE_TREEVIEW      = NO
-GENERATE_LATEX         = NO
-GENERATE_RTF           = NO
-GENERATE_MAN           = NO
-GENERATE_DOCBOOK       = NO
-GENERATE_AUTOGEN_DEF   = NO
-
-GENERATE_XML           = YES
-XML_OUTPUT             = xml
-XML_PROGRAMLISTING     = NO
-
-ENABLE_PREPROCESSING   = YES
-MACRO_EXPANSION        = YES
-EXPAND_ONLY_PREDEF     = NO
-MARKDOWN_SUPPORT       = YES
-'''
-# }}}
+Doxyfile = textwrap.dedent('''
+    OUTPUT_DIRECTORY       = {output}
+    INPUT                  = {input}
+    INPUT_ENCODING         = UTF-8
+    FILE_PATTERNS          = {file_patterns}
+    RECURSIVE              = YES
+    INPUT_FILTER           = "{filter}"
+    EXCLUDE                =
+    EXCLUDE_SYMLINKS       = NO
+    EXCLUDE_PATTERNS       = */private/*
+    EXCLUDE_SYMBOLS        =
+    EXTENSION_MAPPING      = lua=C
+    EXTRACT_PRIVATE        = NO
+
+    GENERATE_HTML          = NO
+    GENERATE_DOCSET        = NO
+    GENERATE_HTMLHELP      = NO
+    GENERATE_QHP           = NO
+    GENERATE_TREEVIEW      = NO
+    GENERATE_LATEX         = NO
+    GENERATE_RTF           = NO
+    GENERATE_MAN           = NO
+    GENERATE_DOCBOOK       = NO
+    GENERATE_AUTOGEN_DEF   = NO
+
+    GENERATE_XML           = YES
+    XML_OUTPUT             = xml
+    XML_PROGRAMLISTING     = NO
+
+    ENABLE_PREPROCESSING   = YES
+    MACRO_EXPANSION        = YES
+    EXPAND_ONLY_PREDEF     = NO
+    MARKDOWN_SUPPORT       = YES
+''')
 
 if __name__ == "__main__":
     if len(sys.argv) > 1:
@@ -1037,4 +891,4 @@ if __name__ == "__main__":
     else:
         gen_docs(Doxyfile)
 
-# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker :
+# vim: set ft=python ts=4 sw=4 tw=79 et :
-- 
cgit 


From 481da1ce4060fd3045ac6553cef69f4fa4f9829d Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Thu, 19 Dec 2019 16:52:37 -0800
Subject: gen_vimdoc.py: fix mpack generator

- make parameters_doc a dict intead of a list
  BEFORE:
    "parameters_doc": [
        {
            "buffer": "Buffer handle, or 0 for current buffer"
        }
    ],
  AFTER:
    "parameters_doc": {
        "buffer": "Buffer handle, or 0 for current buffer"
    },
- make "return", "seealso", lists instead of strings
---
 scripts/gen_vimdoc.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index cd44667ef3..368eaf48f1 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -606,10 +606,10 @@ def extract_from_xml(filename, mode, fmt_vimhelp):
             'annotations': list(annotations),
             'signature': signature,
             'parameters': params,
-            'parameters_doc': [],
+            'parameters_doc': collections.OrderedDict(),
             'doc': [],
-            'return': '',
-            'seealso': '',
+            'return': [],
+            'seealso': [],
         }
         if fmt_vimhelp:
             fn['desc_node'] = desc  # HACK :(
@@ -619,11 +619,12 @@ def extract_from_xml(filename, mode, fmt_vimhelp):
                 if not m['text'] == '':
                     fn['doc'].append(m['text'])
             if 'params' in m:
-                fn['parameters_doc'] += m['params']
-            if 'return' in m:
-                fn['return'] = m['return']
-            if 'seealso' in m:
-                fn['seealso'] = m['xrefs']
+                # Merge OrderedDicts.
+                fn['parameters_doc'].update(m['params'])
+            if 'return' in m and len(m['return']) > 0:
+                fn['return'] += m['return']
+            if 'seealso' in m and len(m['xrefs']) > 0:
+                fn['seealso'].append(str(m['xrefs']))
 
         if INCLUDE_C_DECL:
             fn['c_decl'] = c_decl
-- 
cgit 


From 62e365f5776cfe8e460fdd29a1d7fa89a5c9c0bc Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Thu, 19 Dec 2019 17:10:34 -0800
Subject: gen_vimdoc.py: mpack: exclude deprecated functions

The `mpack` variable was a tuple, which manifests as an array in the
generated msgpack structure.
- Removes noise from the mpack data (deprecated functions are
  deprecated).
- Eliminates 1 level of nesting.

BEFORE:
    [
      {
        "buffer.c": [
          {
            "nvim__buf_stats": { ... },
            ...
          },
          {
            "buffer_del_line": { ... },
            ...
          },
        ],
        ...
      }
    ]

AFTER:
    [
      {
        "buffer.c": {
          "nvim__buf_stats": { ... },
          ...
        },
        ...
    ]
---
 scripts/gen_vimdoc.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 368eaf48f1..2ca121c299 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -727,7 +727,7 @@ def gen_docs(config):
         if p.returncode:
             sys.exit(p.returncode)
 
-        doc_maps = {}
+        fn_maps = {}
         sections = {}
         intros = {}
         sep = '=' * text_width
@@ -756,7 +756,7 @@ def gen_docs(config):
 
             filename = get_text(find_first(compound, 'name'))
             if filename.endswith('.c') or filename.endswith('.lua'):
-                mpack = extract_from_xml(os.path.join(base, '{}.xml'.format(
+                fn_map, _ = extract_from_xml(os.path.join(base, '{}.xml'.format(
                     compound.getAttribute('refid'))), mode, False)
 
                 functions_text, deprecated_text, fns = fmt_doxygen_xml_as_vimhelp(
@@ -798,7 +798,7 @@ def gen_docs(config):
                             title = '{} Functions'.format(name)
                             helptag = '*api-{}*'.format(name.lower())
                         sections[filename] = (title, helptag, doc)
-                        doc_maps[filename] = mpack
+                        fn_maps[filename] = fn_map
 
         if not sections:
             return
@@ -831,7 +831,7 @@ def gen_docs(config):
             fp.write(docs.encode('utf8'))
 
         with open(mpack_file, 'wb') as fp:
-            fp.write(msgpack.packb(doc_maps, use_bin_type=True))
+            fp.write(msgpack.packb(fn_maps, use_bin_type=True))
 
         shutil.rmtree(output_dir)
 
-- 
cgit 


From f968dad3bff5dd4e0d8d797b561fcb2c76553a1c Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Thu, 19 Dec 2019 17:48:40 -0800
Subject: gen_vimdoc.py: fix "seealso", "xrefs"

- Also fix xrefs ("Deprecated" section)
- Fix "Deprecated" rendering by a weird hack (see comment).
- Eliminate unnecessary use of render_para()
---
 scripts/gen_vimdoc.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 2ca121c299..676e6b5be9 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -435,9 +435,11 @@ def para_as_map(parent, indent='', width=62):
         chunks['seealso'].append(render_node(
             child, '', indent=indent, width=width))
     for child in groups['xrefs']:
-        title = get_text(get_child(child, 'xreftitle'))
+        # XXX: Add a space (or any char) to `title` here, otherwise xrefs
+        # ("Deprecated" section) acts very weird...
+        title = get_text(get_child(child, 'xreftitle')) + ' '
         xrefs.add(title)
-        xrefdesc = render_para(get_child(child, 'xrefdescription'), width=width)
+        xrefdesc = get_text(get_child(child, 'xrefdescription'))
         chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title),
                                         width=width) + '\n')
 
@@ -496,7 +498,7 @@ def extract_from_xml(filename, mode, fmt_vimhelp):
     fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :)
     """
     global xrefs
-    xrefs = set()
+    xrefs.clear()
     functions = {}  # Map of func_name:docstring.
     deprecated_functions = {}  # Map of func_name:docstring.
 
@@ -623,8 +625,8 @@ def extract_from_xml(filename, mode, fmt_vimhelp):
                 fn['parameters_doc'].update(m['params'])
             if 'return' in m and len(m['return']) > 0:
                 fn['return'] += m['return']
-            if 'seealso' in m and len(m['xrefs']) > 0:
-                fn['seealso'].append(str(m['xrefs']))
+            if 'seealso' in m and len(m['seealso']) > 0:
+                fn['seealso'] += m['seealso']
 
         if INCLUDE_C_DECL:
             fn['c_decl'] = c_decl
-- 
cgit 


From 4657819e80e3493564c0dd7cf16864aa9bf73c69 Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Thu, 19 Dec 2019 18:17:43 -0800
Subject: gen_vimdoc.py: mpack: collect functions in 1 dict

All Nvim API, core Vimscript, and core Lua functions are globally
unique, so there is no need for per-module nested dicts.

BEFORE (generated mpack structure):
    [
      {
        "buffer.c": {
          "nvim__buf_stats": { ... },
          ...
        },
        "window.c": {
          "nvim_win_close": { ... },
          ...
        },
        ...
      }
    ]

AFTER (generated mpack structure):
    [
      {
        "nvim__buf_stats": {
          ...
        },
        "nvim_buf_attach": {
          ...
        },
        "nvim_tabpage_set_var": {
          ...
        },
        "nvim_ui_attach": {
          ...
        },
        "nvim_win_close": {
          ...
        }
      }
    ]
---
 scripts/gen_vimdoc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 676e6b5be9..229a68a462 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -729,7 +729,7 @@ def gen_docs(config):
         if p.returncode:
             sys.exit(p.returncode)
 
-        fn_maps = {}
+        fn_map_full = {}  # Collects all functions as each module is processed.
         sections = {}
         intros = {}
         sep = '=' * text_width
@@ -800,7 +800,7 @@ def gen_docs(config):
                             title = '{} Functions'.format(name)
                             helptag = '*api-{}*'.format(name.lower())
                         sections[filename] = (title, helptag, doc)
-                        fn_maps[filename] = fn_map
+                        fn_map_full.update(fn_map)
 
         if not sections:
             return
@@ -833,7 +833,7 @@ def gen_docs(config):
             fp.write(docs.encode('utf8'))
 
         with open(mpack_file, 'wb') as fp:
-            fp.write(msgpack.packb(fn_maps, use_bin_type=True))
+            fp.write(msgpack.packb(fn_map_full, use_bin_type=True))
 
         shutil.rmtree(output_dir)
 
-- 
cgit 


From 9ea7aeb3086bf1226f673c0736a757f6e6256ec1 Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Thu, 19 Dec 2019 19:42:24 -0800
Subject: gen_vimdoc.py: rename for clarity

- render_para => fmt_node_as_vimhelp
- Inline parse_parblock() in fmt_node_as_vimhelp()
---
 scripts/gen_vimdoc.py | 102 +++++++++++++++++++++++++++-----------------------
 1 file changed, 56 insertions(+), 46 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 229a68a462..50accbdde8 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -1,5 +1,18 @@
 #!/usr/bin/env python3
-"""Generates Nvim help docs from C docstrings, by parsing Doxygen XML.
+"""Generates Nvim help docs from C/Lua docstrings, using Doxygen.
+
+Also generates *.mpack files. To inspect the *.mpack structure:
+
+    :new | put=json_encode(msgpackparse(readfile('runtime/doc/api.mpack')))
+
+Flow:
+    gen_docs
+      extract_from_xml
+        fmt_node_as_vimhelp
+          fmt_params_map_as_vimhelp
+            render_node
+          para_as_map
+            render_node
 
 This would be easier using lxml and XSLT, but:
 
@@ -262,9 +275,9 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None):
 
 def update_params_map(parent, ret_map, width=62):
     """Updates `ret_map` with name:desc key-value pairs extracted
-    from a Doxygen <parameterlist>.
+    from Doxygen XML node `parent`.
     """
-    items = []
+    params = []
     for node in parent.childNodes:
         if node.nodeType == node.TEXT_NODE:
             continue
@@ -274,13 +287,13 @@ def update_params_map(parent, ret_map, width=62):
         name = get_text(name_node)
         if name in param_exclude:
             continue
-        items.append((name.strip(), node))
+        params.append((name.strip(), node))
     # `ret_map` is a name:desc map.
-    for name, node in items:
+    for name, node in params:
         desc = ''
         desc_node = get_child(node, 'parameterdescription')
         if desc_node:
-            desc = parse_parblock(desc_node, width=width,
+            desc = fmt_node_as_vimhelp(desc_node, width=width,
                                   indent=(' ' * len(name)))
             ret_map[name] = desc
     return ret_map
@@ -288,12 +301,12 @@ def update_params_map(parent, ret_map, width=62):
 
 def fmt_params_map_as_vimhelp(m, width=62):
     """Renders a params map as Vim :help text."""
-    name_length = 0
+    max_name_len = 0
     for name, desc in m.items():
-        name_length = max(name_length, len(name) + 4)
+        max_name_len = max(max_name_len, len(name) + 4)
     out = ''
     for name, desc in m.items():
-        name = '    {}'.format('{{{}}}'.format(name).ljust(name_length))
+        name = '    {}'.format('{{{}}}'.format(name).ljust(max_name_len))
         out += '{}{}\n'.format(name, desc)
     return out.rstrip()
 
@@ -446,45 +459,40 @@ def para_as_map(parent, indent='', width=62):
     return chunks
 
 
-def render_para(parent, indent='', width=62):
-    """Renders Doxygen <para> containing arbitrary nodes.
+def fmt_node_as_vimhelp(parent, width=62, indent=''):
+    """Renders (nested) Doxygen <para> nodes as Vim :help text.
 
     NB: Blank lines in a docstring manifest as <para> tags.
     """
-    para = para_as_map(parent, indent, width)
-
-    def has_nonexcluded_params(m):
-        """Returns true if any of the given params has at least
-        one non-excluded item."""
-        if fmt_params_map_as_vimhelp(m) != '':
-            return True
-
-    # Generate text from the gathered items.
-    chunks = [para['text']]
-    if len(para['params']) > 0 and has_nonexcluded_params(para['params']):
-        chunks.append('\nParameters: ~')
-        chunks.append(fmt_params_map_as_vimhelp(para['params'], width=width))
-    if len(para['return']) > 0:
-        chunks.append('\nReturn: ~')
-        for s in para['return']:
-            chunks.append(s)
-    if len(para['seealso']) > 0:
-        chunks.append('\nSee also: ~')
-        for s in para['seealso']:
+    rendered_blocks = []
+    for child in parent.childNodes:
+        para = para_as_map(child, indent, width)
+
+        def has_nonexcluded_params(m):
+            """Returns true if any of the given params has at least
+            one non-excluded item."""
+            if fmt_params_map_as_vimhelp(m) != '':
+                return True
+
+        # Generate text from the gathered items.
+        chunks = [para['text']]
+        if len(para['params']) > 0 and has_nonexcluded_params(para['params']):
+            chunks.append('\nParameters: ~')
+            chunks.append(fmt_params_map_as_vimhelp(para['params'], width=width))
+        if len(para['return']) > 0:
+            chunks.append('\nReturn: ~')
+            for s in para['return']:
+                chunks.append(s)
+        if len(para['seealso']) > 0:
+            chunks.append('\nSee also: ~')
+            for s in para['seealso']:
+                chunks.append(s)
+        for s in para['xrefs']:
             chunks.append(s)
-    for s in para['xrefs']:
-        chunks.append(s)
-
-    return clean_lines('\n'.join(chunks).strip())
 
-
-def parse_parblock(parent, prefix='', width=62, indent=''):
-    """Renders a nested block of <para> tags as Vim help text."""
-    paragraphs = []
-    for child in parent.childNodes:
-        paragraphs.append(render_para(child, width=width, indent=indent))
-        paragraphs.append('')
-    return clean_lines('\n'.join(paragraphs).strip())
+        rendered_blocks.append(clean_lines('\n'.join(chunks).strip()))
+        rendered_blocks.append('')
+    return clean_lines('\n'.join(rendered_blocks).strip())
 
 
 def extract_from_xml(filename, mode, fmt_vimhelp):
@@ -653,8 +661,9 @@ def fmt_doxygen_xml_as_vimhelp(filename, mode):
     fns, deprecated_fns = extract_from_xml(filename, mode, True)
 
     for name, fn in fns.items():
+        # Generate Vim :help for parameters.
         if fn['desc_node']:
-            doc = parse_parblock(fn['desc_node'])
+            doc = fmt_node_as_vimhelp(fn['desc_node'])
         if not doc:
             doc = 'TODO: Documentation'
 
@@ -704,7 +713,8 @@ def delete_lines_below(filename, tokenstr):
 
 
 def gen_docs(config):
-    """Generate documentation.
+    """Generate formatted Vim :help docs and unformatted *.mpack files for use
+    by API clients.
 
     Doxygen is called and configured through stdin.
     """
@@ -748,7 +758,7 @@ def gen_docs(config):
 
             desc = find_first(minidom.parse(groupxml), 'detaileddescription')
             if desc:
-                doc = parse_parblock(desc)
+                doc = fmt_node_as_vimhelp(desc)
                 if doc:
                     intros[groupname] = doc
 
-- 
cgit 


From 2e280dac7df504b0681043647c8cc02abcbcc686 Mon Sep 17 00:00:00 2001
From: Daniel Hahler <git@thequod.de>
Date: Mon, 23 Dec 2019 07:08:49 +0100
Subject: gen_vimdoc.py: lint #11593

---
 scripts/gen_vimdoc.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 876d46c18f..3ceb5d99e2 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -141,7 +141,7 @@ def debug_this(cond, o):
         try:
             name = o.nodeName
             o = o.toprettyxml(indent='  ', newl='\n')
-        except:
+        except Exception:
             pass
     if ((callable(cond) and cond())
             or (not callable(cond) and cond in o)):
@@ -271,8 +271,6 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None):
     return result
 
 
-
-
 def update_params_map(parent, ret_map, width=62):
     """Updates `ret_map` with name:desc key-value pairs extracted
     from Doxygen XML node `parent`.
@@ -293,8 +291,7 @@ def update_params_map(parent, ret_map, width=62):
         desc = ''
         desc_node = get_child(node, 'parameterdescription')
         if desc_node:
-            desc = fmt_node_as_vimhelp(desc_node, width=width,
-                                  indent=(' ' * len(name)))
+            desc = fmt_node_as_vimhelp(desc_node, width=width, indent=(" " * len(name)))
             ret_map[name] = desc
     return ret_map
 
@@ -399,8 +396,9 @@ def para_as_map(parent, indent='', width=62):
     }
 
     if is_inline(parent):
-        chunks['text'] = clean_lines(doc_wrap(render_node(parent, ''),
-            indent=indent, width=width).strip())
+        chunks["text"] = clean_lines(
+            doc_wrap(render_node(parent, ""), indent=indent, width=width).strip()
+        )
 
     # Ordered dict of ordered lists.
     groups = collections.OrderedDict([
@@ -606,7 +604,7 @@ def extract_from_xml(filename, mode, fmt_vimhelp):
         desc = find_first(member, 'detaileddescription')
         if desc:
             for child in desc.childNodes:
-                paras.append(para_as_map(child))  #, width=width, indent=indent))
+                paras.append(para_as_map(child))
             if DEBUG:
                 print(textwrap.indent(
                     re.sub(r'\n\s*\n+', '\n',
-- 
cgit 


From 27b678f577eb422ccc67e803727c3b26756787f3 Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Sat, 28 Dec 2019 02:15:06 -0800
Subject: gen_vimdoc.py: fix deprecated check

---
 scripts/gen_vimdoc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 3ceb5d99e2..93851ed17e 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -637,7 +637,7 @@ def extract_from_xml(filename, mode, fmt_vimhelp):
         if INCLUDE_C_DECL:
             fn['c_decl'] = c_decl
 
-        if 'Deprecated' in xrefs:
+        if 'Deprecated' in str(xrefs):
             deprecated_functions[name] = fn
         elif name.startswith(CONFIG[mode]['func_name_prefix']):
             functions[name] = fn
-- 
cgit 


From b81547ce6d182e2a7518e5a2f80d1ee90c5cdb8f Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Tue, 24 Dec 2019 00:04:14 -0800
Subject: gen_vimdoc.py: better handling of inline (non-block) nodes

---
 scripts/gen_vimdoc.py | 258 +++++++++++++++++++++++++++-----------------------
 1 file changed, 141 insertions(+), 117 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 93851ed17e..891336f571 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -1,18 +1,18 @@
 #!/usr/bin/env python3
-"""Generates Nvim help docs from C/Lua docstrings, using Doxygen.
+"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen.
 
 Also generates *.mpack files. To inspect the *.mpack structure:
 
-    :new | put=json_encode(msgpackparse(readfile('runtime/doc/api.mpack')))
+    :new | put=v:lua.vim.inspect(msgpackparse(readfile('runtime/doc/api.mpack')))
+
 
 Flow:
-    gen_docs
+    main
       extract_from_xml
-        fmt_node_as_vimhelp
-          fmt_params_map_as_vimhelp
-            render_node
-          para_as_map
-            render_node
+        fmt_node_as_vimhelp   \
+          para_as_map          } recursive
+            update_params_map /
+              render_node
 
 This would be easier using lxml and XSLT, but:
 
@@ -58,6 +58,7 @@ DEBUG = ('DEBUG' in os.environ)
 INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ)
 INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ)
 
+fmt_vimhelp = False  # HACK
 text_width = 78
 script_path = os.path.abspath(__file__)
 base_dir = os.path.dirname(os.path.dirname(script_path))
@@ -144,6 +145,7 @@ def debug_this(cond, o):
         except Exception:
             pass
     if ((callable(cond) and cond())
+            or (not callable(cond) and cond == True)
             or (not callable(cond) and cond in o)):
         raise RuntimeError('xxx: {}\n{}'.format(name, o))
 
@@ -156,20 +158,27 @@ def find_first(parent, name):
     return sub[0]
 
 
-def get_children(parent, name):
-    """Yield matching child nodes within parent."""
+def iter_children(parent, name):
+    """Yields matching child nodes within parent."""
     for child in parent.childNodes:
         if child.nodeType == child.ELEMENT_NODE and child.nodeName == name:
             yield child
 
 
 def get_child(parent, name):
-    """Get the first matching child node."""
-    for child in get_children(parent, name):
+    """Gets the first matching child node."""
+    for child in iter_children(parent, name):
         return child
     return None
 
 
+def self_or_child(n):
+    """Gets the first child node, or self."""
+    if len(n.childNodes) == 0:
+        return n
+    return n.childNodes[0]
+
+
 def clean_text(text):
     """Cleans text.
 
@@ -190,18 +199,21 @@ def is_blank(text):
     return '' == clean_lines(text)
 
 
-def get_text(parent, preformatted=False):
-    """Combine all text in a node."""
-    if parent.nodeType == parent.TEXT_NODE:
-        return parent.data
-
-    out = ''
-    for node in parent.childNodes:
+def get_text(n, preformatted=False):
+    """Recursively concatenates all text in a node tree."""
+    text = ''
+    if n.nodeType == n.TEXT_NODE:
+        return n.data
+    if n.nodeName == 'computeroutput':
+        for node in n.childNodes:
+            text += get_text(node)
+        return '`{}` '.format(text)
+    for node in n.childNodes:
         if node.nodeType == node.TEXT_NODE:
-            out += node.data if preformatted else clean_text(node.data)
+            text += node.data if preformatted else clean_text(node.data)
         elif node.nodeType == node.ELEMENT_NODE:
-            out += ' ' + get_text(node, preformatted)
-    return out
+            text += ' ' + get_text(node, preformatted)
+    return text
 
 
 # Gets the length of the last line in `text`, excluding newline ("\n") char.
@@ -221,6 +233,8 @@ def len_lastline_withoutindent(text, indent):
 
 # Returns True if node `n` contains only inline (not block-level) elements.
 def is_inline(n):
+    # if len(n.childNodes) == 0:
+    #     return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput'
     for c in n.childNodes:
         if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput':
             return False
@@ -271,11 +285,17 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None):
     return result
 
 
+def max_name(names):
+    if len(names) == 0:
+        return 0
+    return max(len(name) for name in names)
+
+
 def update_params_map(parent, ret_map, width=62):
     """Updates `ret_map` with name:desc key-value pairs extracted
     from Doxygen XML node `parent`.
     """
-    params = []
+    params = collections.OrderedDict()
     for node in parent.childNodes:
         if node.nodeType == node.TEXT_NODE:
             continue
@@ -285,48 +305,34 @@ def update_params_map(parent, ret_map, width=62):
         name = get_text(name_node)
         if name in param_exclude:
             continue
-        params.append((name.strip(), node))
+        params[name.strip()] = node
+    max_name_len = max_name(params.keys()) + 8
     # `ret_map` is a name:desc map.
-    for name, node in params:
+    for name, node in params.items():
         desc = ''
         desc_node = get_child(node, 'parameterdescription')
         if desc_node:
-            desc = fmt_node_as_vimhelp(desc_node, width=width, indent=(" " * len(name)))
+            desc = fmt_node_as_vimhelp(desc_node, width=width,
+                                  indent=(' ' * max_name_len))
             ret_map[name] = desc
     return ret_map
 
 
-def fmt_params_map_as_vimhelp(m, width=62):
-    """Renders a params map as Vim :help text."""
-    max_name_len = 0
-    for name, desc in m.items():
-        max_name_len = max(max_name_len, len(name) + 4)
-    out = ''
-    for name, desc in m.items():
-        name = '    {}'.format('{{{}}}'.format(name).ljust(max_name_len))
-        out += '{}{}\n'.format(name, desc)
-    return out.rstrip()
-
-
 def render_node(n, text, prefix='', indent='', width=62):
     """Renders a node as Vim help text, recursively traversing all descendants."""
+    global fmt_vimhelp
+    def ind(s):
+        return s if fmt_vimhelp else ''
     text = ''
     # space_preceding = (len(text) > 0 and ' ' == text[-1][-1])
     # text += (int(not space_preceding) * ' ')
 
-    if n.nodeType == n.TEXT_NODE:
-        # `prefix` is NOT sent to doc_wrap, it was already handled by now.
-        text += doc_wrap(n.data, indent=indent, width=width)
-    elif n.nodeName == 'computeroutput':
-        text += ' `{}` '.format(get_text(n))
-    elif n.nodeName == 'preformatted':
+    if n.nodeName == 'preformatted':
         o = get_text(n, preformatted=True)
         ensure_nl = '' if o[-1] == '\n' else '\n'
-        text += ' >{}{}\n<'.format(ensure_nl, o)
+        text += '>{}{}\n<'.format(ensure_nl, o)
     elif is_inline(n):
-        for c in n.childNodes:
-            text += render_node(c, text)
-        text = doc_wrap(text, indent=indent, width=width)
+        text = doc_wrap(get_text(n), indent=indent, width=width)
     elif n.nodeName == 'verbatim':
         # TODO: currently we don't use this. The "[verbatim]" hint is there as
         # a reminder that we must decide how to format this if we do use it.
@@ -341,8 +347,6 @@ def render_node(n, text, prefix='', indent='', width=62):
     elif n.nodeName in ('para', 'heading'):
         for c in n.childNodes:
             text += render_node(c, text, indent=indent, width=width)
-        if is_inline(n):
-            text = doc_wrap(text, indent=indent, width=width)
     elif n.nodeName == 'itemizedlist':
         for c in n.childNodes:
             text += '{}\n'.format(render_node(c, text, prefix='• ',
@@ -368,7 +372,7 @@ def render_node(n, text, prefix='', indent='', width=62):
         text += '\n'
     elif (n.nodeName == 'simplesect'
             and n.getAttribute('kind') in ('return', 'see')):
-        text += '    '
+        text += ind('    ')
         for c in n.childNodes:
             text += render_node(c, text, indent='    ', width=width)
     else:
@@ -395,11 +399,6 @@ def para_as_map(parent, indent='', width=62):
         'xrefs': []
     }
 
-    if is_inline(parent):
-        chunks["text"] = clean_lines(
-            doc_wrap(render_node(parent, ""), indent=indent, width=width).strip()
-        )
-
     # Ordered dict of ordered lists.
     groups = collections.OrderedDict([
         ('params', []),
@@ -413,27 +412,39 @@ def para_as_map(parent, indent='', width=62):
     text = ''
     kind = ''
     last = ''
-    for child in parent.childNodes:
-        if child.nodeName == 'parameterlist':
-            groups['params'].append(child)
-        elif child.nodeName == 'xrefsect':
-            groups['xrefs'].append(child)
-        elif child.nodeName == 'simplesect':
-            last = kind
-            kind = child.getAttribute('kind')
-            if kind == 'return' or (kind == 'note' and last == 'return'):
-                groups['return'].append(child)
-            elif kind == 'see':
-                groups['seealso'].append(child)
-            elif kind in ('note', 'warning'):
-                text += render_node(child, text, indent=indent, width=width)
+    if is_inline(parent):
+        # Flatten inline text from a tree of non-block nodes.
+        text = doc_wrap(render_node(parent, ""), indent=indent, width=width)
+    else:
+        prev = None  # Previous node
+        for child in parent.childNodes:
+            if child.nodeName == 'parameterlist':
+                groups['params'].append(child)
+            elif child.nodeName == 'xrefsect':
+                groups['xrefs'].append(child)
+            elif child.nodeName == 'simplesect':
+                last = kind
+                kind = child.getAttribute('kind')
+                if kind == 'return' or (kind == 'note' and last == 'return'):
+                    groups['return'].append(child)
+                elif kind == 'see':
+                    groups['seealso'].append(child)
+                elif kind in ('note', 'warning'):
+                    text += render_node(child, text, indent=indent, width=width)
+                else:
+                    raise RuntimeError('unhandled simplesect: {}\n{}'.format(
+                        child.nodeName, child.toprettyxml(indent='  ', newl='\n')))
             else:
-                raise RuntimeError('unhandled simplesect: {}\n{}'.format(
-                    child.nodeName, child.toprettyxml(indent='  ', newl='\n')))
-        else:
-            text += render_node(child, text, indent=indent, width=width)
+                if (prev is not None
+                        and is_inline(self_or_child(prev))
+                        and is_inline(self_or_child(child))
+                        and '' != get_text(self_or_child(child)).strip()
+                        and ' ' != text[-1]):
+                    text += ' '
+                text += render_node(child, text, indent=indent, width=width)
+                prev = child
 
-    chunks['text'] = text
+    chunks['text'] += text
 
     # Generate map from the gathered items.
     if len(groups['params']) > 0:
@@ -441,7 +452,7 @@ def para_as_map(parent, indent='', width=62):
             update_params_map(child, ret_map=chunks['params'], width=width)
     for child in groups['return']:
         chunks['return'].append(render_node(
-            child, '', indent=indent, width=width).lstrip())
+            child, '', indent=indent, width=width))
     for child in groups['seealso']:
         chunks['seealso'].append(render_node(
             child, '', indent=indent, width=width))
@@ -463,20 +474,30 @@ def fmt_node_as_vimhelp(parent, width=62, indent=''):
     NB: Blank lines in a docstring manifest as <para> tags.
     """
     rendered_blocks = []
+
+    def fmt_param_doc(m):
+        """Renders a params map as Vim :help text."""
+        max_name_len = max_name(m.keys()) + 4
+        out = ''
+        for name, desc in m.items():
+            name = '    {}'.format('{{{}}}'.format(name).ljust(max_name_len))
+            out += '{}{}\n'.format(name, desc)
+        return out.rstrip()
+
+    def has_nonexcluded_params(m):
+        """Returns true if any of the given params has at least
+        one non-excluded item."""
+        if fmt_param_doc(m) != '':
+            return True
+
     for child in parent.childNodes:
         para = para_as_map(child, indent, width)
 
-        def has_nonexcluded_params(m):
-            """Returns true if any of the given params has at least
-            one non-excluded item."""
-            if fmt_params_map_as_vimhelp(m) != '':
-                return True
-
         # Generate text from the gathered items.
         chunks = [para['text']]
         if len(para['params']) > 0 and has_nonexcluded_params(para['params']):
             chunks.append('\nParameters: ~')
-            chunks.append(fmt_params_map_as_vimhelp(para['params'], width=width))
+            chunks.append(fmt_param_doc(para['params']))
         if len(para['return']) > 0:
             chunks.append('\nReturn: ~')
             for s in para['return']:
@@ -493,20 +514,21 @@ def fmt_node_as_vimhelp(parent, width=62, indent=''):
     return clean_lines('\n'.join(rendered_blocks).strip())
 
 
-def extract_from_xml(filename, mode, fmt_vimhelp):
+def extract_from_xml(filename, mode, width):
     """Extracts Doxygen info as maps without formatting the text.
 
     Returns two maps:
       1. Functions
       2. Deprecated functions
 
-    The `fmt_vimhelp` parameter controls some special cases for use by
+    The `fmt_vimhelp` global controls some special cases for use by
     fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :)
     """
     global xrefs
+    global fmt_vimhelp
     xrefs.clear()
-    functions = {}  # Map of func_name:docstring.
-    deprecated_functions = {}  # Map of func_name:docstring.
+    fns = {}  # Map of func_name:docstring.
+    deprecated_fns = {}  # Map of func_name:docstring.
 
     dom = minidom.parse(filename)
     compoundname = get_text(dom.getElementsByTagName('compoundname')[0])
@@ -553,7 +575,7 @@ def extract_from_xml(filename, mode, fmt_vimhelp):
         params = []
         type_length = 0
 
-        for param in get_children(member, 'param'):
+        for param in iter_children(member, 'param'):
             param_type = get_text(get_child(param, 'type')).strip()
             param_name = ''
             declname = get_child(param, 'declname')
@@ -590,15 +612,15 @@ def extract_from_xml(filename, mode, fmt_vimhelp):
                                      '    ')
 
             # Minimum 8 chars between signature and vimtag
-            lhs = (text_width - 8) - len(prefix)
+            lhs = (width - 8) - len(prefix)
 
             if len(prefix) + len(suffix) > lhs:
-                signature = vimtag.rjust(text_width) + '\n'
-                signature += doc_wrap(suffix, width=text_width-8, prefix=prefix,
+                signature = vimtag.rjust(width) + '\n'
+                signature += doc_wrap(suffix, width=width-8, prefix=prefix,
                                       func=True)
             else:
                 signature = prefix + suffix
-                signature += vimtag.rjust(text_width - len(signature))
+                signature += vimtag.rjust(width - len(signature))
 
         paras = []
         desc = find_first(member, 'detaileddescription')
@@ -638,25 +660,27 @@ def extract_from_xml(filename, mode, fmt_vimhelp):
             fn['c_decl'] = c_decl
 
         if 'Deprecated' in str(xrefs):
-            deprecated_functions[name] = fn
+            deprecated_fns[name] = fn
         elif name.startswith(CONFIG[mode]['func_name_prefix']):
-            functions[name] = fn
+            fns[name] = fn
 
         xrefs.clear()
 
-    return (functions, deprecated_functions)
+    return (fns, deprecated_fns)
 
 
 def fmt_doxygen_xml_as_vimhelp(filename, mode):
-    """Formats functions from doxygen XML into Vim :help format.
+    """Entrypoint for generating Vim :help from from Doxygen XML.
 
-    Returns two strings:
-      1. Functions in Vim :help format
-      2. Deprecated functions (handled by caller, or ignored)
+    Returns 3 items:
+      1. Vim help text for functions found in `filename`.
+      2. Vim help text for deprecated functions.
     """
-    functions = {}  # Map of func_name:docstring.
-    deprecated_functions = {}  # Map of func_name:docstring.
-    fns, deprecated_fns = extract_from_xml(filename, mode, True)
+    global fmt_vimhelp
+    fmt_vimhelp = True
+    fns_txt = {}  # Map of func_name:vim-help-text.
+    deprecated_fns_txt = {}  # Map of func_name:vim-help-text.
+    fns, _ = extract_from_xml(filename, mode, width=text_width)
 
     for name, fn in fns.items():
         # Generate Vim :help for parameters.
@@ -685,15 +709,15 @@ def fmt_doxygen_xml_as_vimhelp(filename, mode):
         func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M)
 
         if 'Deprecated' in xrefs:
-            deprecated_functions.append(func_doc)
+            deprecated_fns_txt[name] = func_doc
         elif name.startswith(CONFIG[mode]['func_name_prefix']):
-            functions[name] = func_doc
+            fns_txt[name] = func_doc
 
         xrefs.clear()
 
-    return ('\n\n'.join(list(functions.values())),
-            '\n\n'.join(deprecated_fns),
-            functions)
+    fmt_vimhelp = False
+    return ('\n\n'.join(list(fns_txt.values())),
+            '\n\n'.join(list(deprecated_fns_txt.values())))
 
 
 def delete_lines_below(filename, tokenstr):
@@ -710,14 +734,15 @@ def delete_lines_below(filename, tokenstr):
         fp.writelines(lines[0:i])
 
 
-def gen_docs(config):
-    """Generate formatted Vim :help docs and unformatted *.mpack files for use
-    by API clients.
+def main(config):
+    """Generates:
+
+    1. Vim :help docs
+    2. *.mpack files for use by API clients
 
     Doxygen is called and configured through stdin.
     """
     for mode in CONFIG:
-        functions = {}  # Map of func_name:docstring.
         mpack_file = os.path.join(
             base_dir, 'runtime', 'doc',
             CONFIG[mode]['filename'].replace('.txt', '.mpack'))
@@ -766,14 +791,13 @@ def gen_docs(config):
 
             filename = get_text(find_first(compound, 'name'))
             if filename.endswith('.c') or filename.endswith('.lua'):
+                # Extract unformatted (*.mpack).
                 fn_map, _ = extract_from_xml(os.path.join(base, '{}.xml'.format(
-                    compound.getAttribute('refid'))), mode, False)
-
-                functions_text, deprecated_text, fns = fmt_doxygen_xml_as_vimhelp(
+                    compound.getAttribute('refid'))), mode, width=9999)
+                # Extract formatted (:help).
+                functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp(
                     os.path.join(base, '{}.xml'.format(
                                  compound.getAttribute('refid'))), mode)
-                # Collect functions from all modules (for the current `mode`).
-                functions = {**functions, **fns}
 
                 if not functions_text and not deprecated_text:
                     continue
@@ -900,6 +924,6 @@ if __name__ == "__main__":
     if len(sys.argv) > 1:
         filter_source(sys.argv[1])
     else:
-        gen_docs(Doxyfile)
+        main(Doxyfile)
 
 # vim: set ft=python ts=4 sw=4 tw=79 et :
-- 
cgit 


From c24f8f46b45f097dbe0abf7e6ae686bc9bec5568 Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Sat, 28 Dec 2019 03:27:25 -0800
Subject: gen_vimdoc.py: sort by name

---
 scripts/gen_vimdoc.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 891336f571..e30ed92e8f 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -145,7 +145,7 @@ def debug_this(cond, o):
         except Exception:
             pass
     if ((callable(cond) and cond())
-            or (not callable(cond) and cond == True)
+            or (not callable(cond) and cond)
             or (not callable(cond) and cond in o)):
         raise RuntimeError('xxx: {}\n{}'.format(name, o))
 
@@ -312,8 +312,8 @@ def update_params_map(parent, ret_map, width=62):
         desc = ''
         desc_node = get_child(node, 'parameterdescription')
         if desc_node:
-            desc = fmt_node_as_vimhelp(desc_node, width=width,
-                                  indent=(' ' * max_name_len))
+            desc = fmt_node_as_vimhelp(
+                    desc_node, width=width, indent=(' ' * max_name_len))
             ret_map[name] = desc
     return ret_map
 
@@ -321,8 +321,10 @@ def update_params_map(parent, ret_map, width=62):
 def render_node(n, text, prefix='', indent='', width=62):
     """Renders a node as Vim help text, recursively traversing all descendants."""
     global fmt_vimhelp
+
     def ind(s):
         return s if fmt_vimhelp else ''
+
     text = ''
     # space_preceding = (len(text) > 0 and ' ' == text[-1][-1])
     # text += (int(not space_preceding) * ' ')
@@ -666,6 +668,8 @@ def extract_from_xml(filename, mode, width):
 
         xrefs.clear()
 
+    fns = collections.OrderedDict(sorted(fns.items()))
+    deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items()))
     return (fns, deprecated_fns)
 
 
@@ -864,6 +868,7 @@ def main(config):
         with open(doc_file, 'ab') as fp:
             fp.write(docs.encode('utf8'))
 
+        fn_map_full = collections.OrderedDict(sorted(fn_map_full.items()))
         with open(mpack_file, 'wb') as fp:
             fp.write(msgpack.packb(fn_map_full, use_bin_type=True))
 
-- 
cgit 


From b112fe828fd2457692f556626d7657615e53cb0b Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Tue, 31 Dec 2019 06:52:14 -0800
Subject: gen_vimdoc.py: generate LSP docs

---
 scripts/gen_vimdoc.py | 126 +++++++++++++++++++++++++++++++++++---------------
 1 file changed, 89 insertions(+), 37 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index e30ed92e8f..c1b05e16ff 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -55,6 +55,7 @@ if sys.version_info[0] < 3 or sys.version_info[1] < 5:
     sys.exit(1)
 
 DEBUG = ('DEBUG' in os.environ)
+TARGET = os.environ.get('TARGET', None)
 INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ)
 INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ)
 
@@ -69,6 +70,7 @@ lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter')
 
 CONFIG = {
     'api': {
+        'mode': 'c',
         'filename': 'api.txt',
         # String used to find the start of the generated part of the doc.
         'section_start_token': '*api-global*',
@@ -85,17 +87,24 @@ CONFIG = {
         # file patterns used by doxygen
         'file_patterns': '*.h *.c',
         # Only function with this prefix are considered
-        'func_name_prefix': 'nvim_',
+        'fn_name_prefix': 'nvim_',
         # Section name overrides.
         'section_name': {
             'vim.c': 'Global',
         },
+        # For generated section names.
+        'section_fmt': lambda name: f'{name} Functions',
+        # Section helptag.
+        'helptag_fmt': lambda name: f'*api-{name.lower()}*',
+        # Per-function helptag.
+        'fn_helptag_fmt': lambda fstem, name: f'*{name}()*',
         # Module name overrides (for Lua).
         'module_override': {},
         # Append the docs for these modules, do not start a new section.
         'append_only': [],
     },
     'lua': {
+        'mode': 'lua',
         'filename': 'lua.txt',
         'section_start_token': '*lua-vim*',
         'section_order': [
@@ -107,8 +116,13 @@ CONFIG = {
             os.path.join(base_dir, 'runtime/lua/vim/shared.lua'),
         ]),
         'file_patterns': '*.lua',
-        'func_name_prefix': '',
-        'section_name': {},
+        'fn_name_prefix': '',
+        'section_name': {
+            'lsp.lua': 'core',
+        },
+        'section_fmt': lambda name: f'Lua module: {name.lower()}',
+        'helptag_fmt': lambda name: f'*lua-{name.lower()}*',
+        'fn_helptag_fmt': lambda fstem, name: f'*{fstem}.{name}()*',
         'module_override': {
             # `shared` functions are exposed on the `vim` module.
             'shared': 'vim',
@@ -117,6 +131,44 @@ CONFIG = {
             'shared.lua',
         ],
     },
+    'lsp': {
+        'mode': 'lua',
+        'filename': 'lsp.txt',
+        'section_start_token': '*lsp-core*',
+        'section_order': [
+            'lsp.lua',
+            'protocol.lua',
+            'buf.lua',
+            'callbacks.lua',
+            'log.lua',
+            'rpc.lua',
+            'util.lua'
+        ],
+        'files': ' '.join([
+            os.path.join(base_dir, 'runtime/lua/vim/lsp'),
+            os.path.join(base_dir, 'runtime/lua/vim/lsp.lua'),
+        ]),
+        'file_patterns': '*.lua',
+        'fn_name_prefix': '',
+        'section_name': {},
+        'section_fmt': lambda name: ('Lua module: vim.lsp'
+            if name.lower() == 'lsp'
+            else f'Lua module: vim.lsp.{name.lower()}'),
+        'helptag_fmt': lambda name: ('*lsp-core*'
+            if name.lower() == 'lsp'
+            else f'*lsp-{name.lower()}*'),
+        'fn_helptag_fmt': lambda fstem, name: (f'*vim.lsp.{name}()*'
+            if fstem == 'lsp' and name != 'client'
+            else ('*vim.lsp.client*'
+                # HACK. TODO(justinmk): class/structure support in lua2dox
+                if 'lsp.client' == f'{fstem}.{name}'
+                else f'*vim.lsp.{fstem}.{name}()*')),
+        'module_override': {
+            # Combine  are exposed on the `vim` module.
+            'shared': 'vim',
+        },
+        'append_only': [],
+    },
 }
 
 param_exclude = (
@@ -567,12 +619,12 @@ def extract_from_xml(filename, mode, width):
 
         if not fmt_vimhelp:
             pass
-        elif mode == 'lua':
-            fstem = compoundname.split('.')[0]
-            fstem = CONFIG[mode]['module_override'].get(fstem, fstem)
-            vimtag = '*{}.{}()*'.format(fstem, name)
         else:
-            vimtag = '*{}()*'.format(name)
+            fstem = '?'
+            if '.' in compoundname:
+                fstem = compoundname.split('.')[0]
+                fstem = CONFIG[mode]['module_override'].get(fstem, fstem)
+            vimtag = CONFIG[mode]['fn_helptag_fmt'](fstem, name)
 
         params = []
         type_length = 0
@@ -583,8 +635,8 @@ def extract_from_xml(filename, mode, width):
             declname = get_child(param, 'declname')
             if declname:
                 param_name = get_text(declname).strip()
-            elif mode == 'lua':
-                # that's how it comes out of lua2dox
+            elif CONFIG[mode]['mode'] == 'lua':
+                # XXX: this is what lua2dox gives us...
                 param_name = param_type
                 param_type = ''
 
@@ -614,7 +666,7 @@ def extract_from_xml(filename, mode, width):
                                      '    ')
 
             # Minimum 8 chars between signature and vimtag
-            lhs = (width - 8) - len(prefix)
+            lhs = (width - 8) - len(vimtag)
 
             if len(prefix) + len(suffix) > lhs:
                 signature = vimtag.rjust(width) + '\n'
@@ -663,7 +715,7 @@ def extract_from_xml(filename, mode, width):
 
         if 'Deprecated' in str(xrefs):
             deprecated_fns[name] = fn
-        elif name.startswith(CONFIG[mode]['func_name_prefix']):
+        elif name.startswith(CONFIG[mode]['fn_name_prefix']):
             fns[name] = fn
 
         xrefs.clear()
@@ -714,7 +766,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, mode):
 
         if 'Deprecated' in xrefs:
             deprecated_fns_txt[name] = func_doc
-        elif name.startswith(CONFIG[mode]['func_name_prefix']):
+        elif name.startswith(CONFIG[mode]['fn_name_prefix']):
             fns_txt[name] = func_doc
 
         xrefs.clear()
@@ -730,9 +782,13 @@ def delete_lines_below(filename, tokenstr):
     """
     lines = open(filename).readlines()
     i = 0
+    found = False
     for i, line in enumerate(lines, 1):
         if tokenstr in line:
+            found = True
             break
+    if not found:
+        raise RuntimeError(f'not found: "{tokenstr}"')
     i = max(0, i - 2)
     with open(filename, 'wt') as fp:
         fp.writelines(lines[0:i])
@@ -747,6 +803,8 @@ def main(config):
     Doxygen is called and configured through stdin.
     """
     for mode in CONFIG:
+        if TARGET is not None and mode != TARGET:
+            continue
         mpack_file = os.path.join(
             base_dir, 'runtime', 'doc',
             CONFIG[mode]['filename'].replace('.txt', '.mpack'))
@@ -754,7 +812,10 @@ def main(config):
             os.remove(mpack_file)
 
         output_dir = out_dir.format(mode=mode)
-        p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE)
+        p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE,
+                # silence warnings
+                # runtime/lua/vim/lsp.lua:209: warning: argument 'trace' of command @param is not found in the argument list
+                stderr=subprocess.DEVNULL)
         p.communicate(
             config.format(
                 input=CONFIG[mode]['files'],
@@ -806,15 +867,11 @@ def main(config):
                 if not functions_text and not deprecated_text:
                     continue
                 else:
-                    name = os.path.splitext(os.path.basename(filename))[0]
-                    if name == 'ui':
-                        name = name.upper()
-                    else:
-                        name = name.title()
-
+                    name = os.path.splitext(
+                            os.path.basename(filename))[0].lower()
+                    sectname = name.upper() if name == 'ui' else name.title()
                     doc = ''
-
-                    intro = intros.get('api-%s' % name.lower())
+                    intro = intros.get(f'api-{name}')
                     if intro:
                         doc += '\n\n' + intro
 
@@ -822,33 +879,28 @@ def main(config):
                         doc += '\n\n' + functions_text
 
                     if INCLUDE_DEPRECATED and deprecated_text:
-                        doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name
+                        doc += f'\n\n\nDeprecated {sectname} Functions: ~\n\n'
                         doc += deprecated_text
 
                     if doc:
                         filename = os.path.basename(filename)
-                        name = CONFIG[mode]['section_name'].get(filename, name)
-
-                        if mode == 'lua':
-                            title = 'Lua module: {}'.format(name.lower())
-                            helptag = '*lua-{}*'.format(name.lower())
-                        else:
-                            title = '{} Functions'.format(name)
-                            helptag = '*api-{}*'.format(name.lower())
+                        sectname = CONFIG[mode]['section_name'].get(
+                                filename, sectname)
+                        title = CONFIG[mode]['section_fmt'](sectname)
+                        helptag = CONFIG[mode]['helptag_fmt'](sectname)
                         sections[filename] = (title, helptag, doc)
                         fn_map_full.update(fn_map)
 
-        if not sections:
-            return
+        assert sections
+        if len(sections) > len(CONFIG[mode]['section_order']):
+            raise RuntimeError(
+                'found new modules "{}"; update the "section_order" map'.format(
+                    set(sections).difference(CONFIG[mode]['section_order'])))
 
         docs = ''
 
         i = 0
         for filename in CONFIG[mode]['section_order']:
-            if filename not in sections:
-                raise RuntimeError(
-                    'found new module "{}"; update the "section_order" map'.format(
-                        filename))
             title, helptag, section_doc = sections.pop(filename)
             i += 1
             if filename not in CONFIG[mode]['append_only']:
-- 
cgit 


From 0dcf4bd3ebfec77c5d3816d2053521190080f7e0 Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Tue, 31 Dec 2019 07:12:10 -0800
Subject: gen_vimdoc.py: rename `mode` to `target`

---
 scripts/gen_vimdoc.py | 70 ++++++++++++++++++++++++++++-----------------------
 1 file changed, 38 insertions(+), 32 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index c1b05e16ff..bdfe52addb 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -63,7 +63,7 @@ fmt_vimhelp = False  # HACK
 text_width = 78
 script_path = os.path.abspath(__file__)
 base_dir = os.path.dirname(os.path.dirname(script_path))
-out_dir = os.path.join(base_dir, 'tmp-{mode}-doc')
+out_dir = os.path.join(base_dir, 'tmp-{target}-doc')
 filter_cmd = '%s %s' % (sys.executable, script_path)
 seen_funcs = set()
 lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter')
@@ -151,15 +151,19 @@ CONFIG = {
         'file_patterns': '*.lua',
         'fn_name_prefix': '',
         'section_name': {},
-        'section_fmt': lambda name: ('Lua module: vim.lsp'
+        'section_fmt': lambda name: (
+            'Lua module: vim.lsp'
             if name.lower() == 'lsp'
             else f'Lua module: vim.lsp.{name.lower()}'),
-        'helptag_fmt': lambda name: ('*lsp-core*'
+        'helptag_fmt': lambda name: (
+            '*lsp-core*'
             if name.lower() == 'lsp'
             else f'*lsp-{name.lower()}*'),
-        'fn_helptag_fmt': lambda fstem, name: (f'*vim.lsp.{name}()*'
+        'fn_helptag_fmt': lambda fstem, name: (
+            f'*vim.lsp.{name}()*'
             if fstem == 'lsp' and name != 'client'
-            else ('*vim.lsp.client*'
+            else (
+                '*vim.lsp.client*'
                 # HACK. TODO(justinmk): class/structure support in lua2dox
                 if 'lsp.client' == f'{fstem}.{name}'
                 else f'*vim.lsp.{fstem}.{name}()*')),
@@ -568,7 +572,7 @@ def fmt_node_as_vimhelp(parent, width=62, indent=''):
     return clean_lines('\n'.join(rendered_blocks).strip())
 
 
-def extract_from_xml(filename, mode, width):
+def extract_from_xml(filename, target, width):
     """Extracts Doxygen info as maps without formatting the text.
 
     Returns two maps:
@@ -623,8 +627,8 @@ def extract_from_xml(filename, mode, width):
             fstem = '?'
             if '.' in compoundname:
                 fstem = compoundname.split('.')[0]
-                fstem = CONFIG[mode]['module_override'].get(fstem, fstem)
-            vimtag = CONFIG[mode]['fn_helptag_fmt'](fstem, name)
+                fstem = CONFIG[target]['module_override'].get(fstem, fstem)
+            vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name)
 
         params = []
         type_length = 0
@@ -635,7 +639,7 @@ def extract_from_xml(filename, mode, width):
             declname = get_child(param, 'declname')
             if declname:
                 param_name = get_text(declname).strip()
-            elif CONFIG[mode]['mode'] == 'lua':
+            elif CONFIG[target]['mode'] == 'lua':
                 # XXX: this is what lua2dox gives us...
                 param_name = param_type
                 param_type = ''
@@ -715,7 +719,7 @@ def extract_from_xml(filename, mode, width):
 
         if 'Deprecated' in str(xrefs):
             deprecated_fns[name] = fn
-        elif name.startswith(CONFIG[mode]['fn_name_prefix']):
+        elif name.startswith(CONFIG[target]['fn_name_prefix']):
             fns[name] = fn
 
         xrefs.clear()
@@ -725,7 +729,7 @@ def extract_from_xml(filename, mode, width):
     return (fns, deprecated_fns)
 
 
-def fmt_doxygen_xml_as_vimhelp(filename, mode):
+def fmt_doxygen_xml_as_vimhelp(filename, target):
     """Entrypoint for generating Vim :help from from Doxygen XML.
 
     Returns 3 items:
@@ -736,7 +740,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, mode):
     fmt_vimhelp = True
     fns_txt = {}  # Map of func_name:vim-help-text.
     deprecated_fns_txt = {}  # Map of func_name:vim-help-text.
-    fns, _ = extract_from_xml(filename, mode, width=text_width)
+    fns, _ = extract_from_xml(filename, target, width=text_width)
 
     for name, fn in fns.items():
         # Generate Vim :help for parameters.
@@ -766,7 +770,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, mode):
 
         if 'Deprecated' in xrefs:
             deprecated_fns_txt[name] = func_doc
-        elif name.startswith(CONFIG[mode]['fn_name_prefix']):
+        elif name.startswith(CONFIG[target]['fn_name_prefix']):
             fns_txt[name] = func_doc
 
         xrefs.clear()
@@ -802,26 +806,28 @@ def main(config):
 
     Doxygen is called and configured through stdin.
     """
-    for mode in CONFIG:
-        if TARGET is not None and mode != TARGET:
+    for target in CONFIG:
+        if TARGET is not None and target != TARGET:
             continue
         mpack_file = os.path.join(
             base_dir, 'runtime', 'doc',
-            CONFIG[mode]['filename'].replace('.txt', '.mpack'))
+            CONFIG[target]['filename'].replace('.txt', '.mpack'))
         if os.path.exists(mpack_file):
             os.remove(mpack_file)
 
-        output_dir = out_dir.format(mode=mode)
-        p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE,
+        output_dir = out_dir.format(target=target)
+        p = subprocess.Popen(
+                ['doxygen', '-'],
+                stdin=subprocess.PIPE,
                 # silence warnings
-                # runtime/lua/vim/lsp.lua:209: warning: argument 'trace' of command @param is not found in the argument list
+                # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found
                 stderr=subprocess.DEVNULL)
         p.communicate(
             config.format(
-                input=CONFIG[mode]['files'],
+                input=CONFIG[target]['files'],
                 output=output_dir,
                 filter=filter_cmd,
-                file_patterns=CONFIG[mode]['file_patterns'])
+                file_patterns=CONFIG[target]['file_patterns'])
             .encode('utf8')
         )
         if p.returncode:
@@ -858,11 +864,11 @@ def main(config):
             if filename.endswith('.c') or filename.endswith('.lua'):
                 # Extract unformatted (*.mpack).
                 fn_map, _ = extract_from_xml(os.path.join(base, '{}.xml'.format(
-                    compound.getAttribute('refid'))), mode, width=9999)
+                    compound.getAttribute('refid'))), target, width=9999)
                 # Extract formatted (:help).
                 functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp(
                     os.path.join(base, '{}.xml'.format(
-                                 compound.getAttribute('refid'))), mode)
+                                 compound.getAttribute('refid'))), target)
 
                 if not functions_text and not deprecated_text:
                     continue
@@ -884,26 +890,26 @@ def main(config):
 
                     if doc:
                         filename = os.path.basename(filename)
-                        sectname = CONFIG[mode]['section_name'].get(
+                        sectname = CONFIG[target]['section_name'].get(
                                 filename, sectname)
-                        title = CONFIG[mode]['section_fmt'](sectname)
-                        helptag = CONFIG[mode]['helptag_fmt'](sectname)
+                        title = CONFIG[target]['section_fmt'](sectname)
+                        helptag = CONFIG[target]['helptag_fmt'](sectname)
                         sections[filename] = (title, helptag, doc)
                         fn_map_full.update(fn_map)
 
         assert sections
-        if len(sections) > len(CONFIG[mode]['section_order']):
+        if len(sections) > len(CONFIG[target]['section_order']):
             raise RuntimeError(
                 'found new modules "{}"; update the "section_order" map'.format(
-                    set(sections).difference(CONFIG[mode]['section_order'])))
+                    set(sections).difference(CONFIG[target]['section_order'])))
 
         docs = ''
 
         i = 0
-        for filename in CONFIG[mode]['section_order']:
+        for filename in CONFIG[target]['section_order']:
             title, helptag, section_doc = sections.pop(filename)
             i += 1
-            if filename not in CONFIG[mode]['append_only']:
+            if filename not in CONFIG[target]['append_only']:
                 docs += sep
                 docs += '\n%s%s' % (title,
                                     helptag.rjust(text_width - len(title)))
@@ -914,9 +920,9 @@ def main(config):
         docs += ' vim:tw=78:ts=8:ft=help:norl:\n'
 
         doc_file = os.path.join(base_dir, 'runtime', 'doc',
-                                CONFIG[mode]['filename'])
+                                CONFIG[target]['filename'])
 
-        delete_lines_below(doc_file, CONFIG[mode]['section_start_token'])
+        delete_lines_below(doc_file, CONFIG[target]['section_start_token'])
         with open(doc_file, 'ab') as fp:
             fp.write(docs.encode('utf8'))
 
-- 
cgit 


From 92316849863bb2661ee5b4bb284f56163fed27ad Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Sun, 12 Jan 2020 23:41:55 -0800
Subject: doc [ci skip] #11656

---
 scripts/gen_vimdoc.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index bdfe52addb..3c51b2aa81 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -167,10 +167,7 @@ CONFIG = {
                 # HACK. TODO(justinmk): class/structure support in lua2dox
                 if 'lsp.client' == f'{fstem}.{name}'
                 else f'*vim.lsp.{fstem}.{name}()*')),
-        'module_override': {
-            # Combine  are exposed on the `vim` module.
-            'shared': 'vim',
-        },
+        'module_override': {},
         'append_only': [],
     },
 }
@@ -821,7 +818,7 @@ def main(config):
                 stdin=subprocess.PIPE,
                 # silence warnings
                 # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found
-                stderr=subprocess.DEVNULL)
+                stderr=(subprocess.STDOUT if DEBUG else subprocess.DEVNULL))
         p.communicate(
             config.format(
                 input=CONFIG[target]['files'],
-- 
cgit 


From 7b529e7912517af078e005dd7b06b3d042be9cb7 Mon Sep 17 00:00:00 2001
From: TJ DeVries <devries.timothyj@gmail.com>
Date: Thu, 2 Jul 2020 07:09:17 -0400
Subject: doc: fix scripts and regenerate (#12506)

* Fix some small doc issues

* doc: fixup

* doc: fixup

* Fix lint and rebase

* Remove bad advice

* Ugh, stupid mpack files...

* Don't let people include these for now until they specifically want to

* Prevent duplicate tag
---
 scripts/gen_vimdoc.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 60 insertions(+), 6 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 3c51b2aa81..328a903b46 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -374,6 +374,7 @@ def update_params_map(parent, ret_map, width=62):
 def render_node(n, text, prefix='', indent='', width=62):
     """Renders a node as Vim help text, recursively traversing all descendants."""
     global fmt_vimhelp
+    global has_seen_preformatted
 
     def ind(s):
         return s if fmt_vimhelp else ''
@@ -386,6 +387,7 @@ def render_node(n, text, prefix='', indent='', width=62):
         o = get_text(n, preformatted=True)
         ensure_nl = '' if o[-1] == '\n' else '\n'
         text += '>{}{}\n<'.format(ensure_nl, o)
+
     elif is_inline(n):
         text = doc_wrap(get_text(n), indent=indent, width=width)
     elif n.nodeName == 'verbatim':
@@ -394,11 +396,17 @@ def render_node(n, text, prefix='', indent='', width=62):
         text += ' [verbatim] {}'.format(get_text(n))
     elif n.nodeName == 'listitem':
         for c in n.childNodes:
-            text += (
-                indent
-                + prefix
-                + render_node(c, text, indent=indent + (' ' * len(prefix)), width=width)
+            result = render_node(
+                c,
+                text,
+                indent=indent + (' ' * len(prefix)),
+                width=width
             )
+
+            if is_blank(result):
+                continue
+
+            text += indent + prefix + result
     elif n.nodeName in ('para', 'heading'):
         for c in n.childNodes:
             text += render_node(c, text, indent=indent, width=width)
@@ -433,6 +441,7 @@ def render_node(n, text, prefix='', indent='', width=62):
     else:
         raise RuntimeError('unhandled node type: {}\n{}'.format(
             n.nodeName, n.toprettyxml(indent='  ', newl='\n')))
+
     return text
 
 
@@ -496,6 +505,7 @@ def para_as_map(parent, indent='', width=62):
                         and '' != get_text(self_or_child(child)).strip()
                         and ' ' != text[-1]):
                     text += ' '
+
                 text += render_node(child, text, indent=indent, width=width)
                 prev = child
 
@@ -566,6 +576,7 @@ def fmt_node_as_vimhelp(parent, width=62, indent=''):
 
         rendered_blocks.append(clean_lines('\n'.join(chunks).strip()))
         rendered_blocks.append('')
+
     return clean_lines('\n'.join(rendered_blocks).strip())
 
 
@@ -678,6 +689,11 @@ def extract_from_xml(filename, target, width):
                 signature += vimtag.rjust(width - len(signature))
 
         paras = []
+        brief_desc = find_first(member, 'briefdescription')
+        if brief_desc:
+            for child in brief_desc.childNodes:
+                paras.append(para_as_map(child))
+
         desc = find_first(member, 'detaileddescription')
         if desc:
             for child in desc.childNodes:
@@ -763,8 +779,36 @@ def fmt_doxygen_xml_as_vimhelp(filename, target):
 
         func_doc = fn['signature'] + '\n'
         func_doc += textwrap.indent(clean_lines(doc), ' ' * 16)
+
+        # Verbatim handling.
         func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M)
 
+        split_lines = func_doc.split('\n')
+        start = 0
+        while True:
+            try:
+                start = split_lines.index('>', start)
+            except ValueError:
+                break
+
+            try:
+                end = split_lines.index('<', start)
+            except ValueError:
+                break
+
+            split_lines[start + 1:end] = [
+                ('    ' + x).rstrip()
+                for x in textwrap.dedent(
+                    "\n".join(
+                        split_lines[start+1:end]
+                    )
+                ).split("\n")
+            ]
+
+            start = end
+
+        func_doc = "\n".join(split_lines)
+
         if 'Deprecated' in xrefs:
             deprecated_fns_txt[name] = func_doc
         elif name.startswith(CONFIG[target]['fn_name_prefix']):
@@ -847,11 +891,21 @@ def main(config):
             groupxml = os.path.join(base, '%s.xml' %
                                     compound.getAttribute('refid'))
 
-            desc = find_first(minidom.parse(groupxml), 'detaileddescription')
+            group_parsed = minidom.parse(groupxml)
+            doc_list = []
+            brief_desc = find_first(group_parsed, 'briefdescription')
+            if brief_desc:
+                for child in brief_desc.childNodes:
+                    doc_list.append(fmt_node_as_vimhelp(child))
+
+            desc = find_first(group_parsed, 'detaileddescription')
             if desc:
                 doc = fmt_node_as_vimhelp(desc)
+
                 if doc:
-                    intros[groupname] = doc
+                    doc_list.append(doc)
+
+            intros[groupname] = "\n".join(doc_list)
 
         for compound in dom.getElementsByTagName('compound'):
             if compound.getAttribute('kind') != 'file':
-- 
cgit 


From 44cbf288ec4407c6fddf79b6b39041d5bbee940b Mon Sep 17 00:00:00 2001
From: jnozsc <jnozsc@gmail.com>
Date: Fri, 24 Jul 2020 13:42:25 -0700
Subject: script: simplify python version check (#12672)

---
 scripts/gen_vimdoc.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 328a903b46..a61690e99f 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -50,8 +50,10 @@ import msgpack
 
 from xml.dom import minidom
 
-if sys.version_info[0] < 3 or sys.version_info[1] < 5:
-    print("requires Python 3.5+")
+MIN_PYTHON_VERSION = (3, 5)
+
+if sys.version_info < MIN_PYTHON_VERSION:
+    print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION))
     sys.exit(1)
 
 DEBUG = ('DEBUG' in os.environ)
-- 
cgit