From 5dc0bdfe98b59bb03226167ed541d17cc5af30b1 Mon Sep 17 00:00:00 2001
From: Mathias Fußenegger <mfussenegger@users.noreply.github.com>
Date: Tue, 2 Jan 2024 14:32:43 +0100
Subject: docs(glob): add glob module (#26853)

---
 scripts/gen_vimdoc.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 8ed88cb8f5..b51bd5fbf5 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -164,6 +164,7 @@ CONFIG = {
             'filetype.lua',
             'keymap.lua',
             'fs.lua',
+            'glob.lua',
             'secure.lua',
             'version.lua',
             'iter.lua',
@@ -187,6 +188,7 @@ CONFIG = {
             'runtime/lua/vim/_inspector.lua',
             'runtime/lua/vim/snippet.lua',
             'runtime/lua/vim/text.lua',
+            'runtime/lua/vim/glob.lua',
             'runtime/lua/vim/_meta/builtin.lua',
             'runtime/lua/vim/_meta/diff.lua',
             'runtime/lua/vim/_meta/mpack.lua',
@@ -251,6 +253,7 @@ CONFIG = {
             'spell': 'vim.spell',
             'snippet': 'vim.snippet',
             'text': 'vim.text',
+            'glob': 'vim.glob',
         },
         'append_only': [
             'shared.lua',
-- 
cgit 


From 5e2d4b3c4dbd56342cabb6993d354f690e0a1575 Mon Sep 17 00:00:00 2001
From: Jongwook Choi <wookayin@gmail.com>
Date: Thu, 28 Dec 2023 13:04:34 -0500
Subject: refactor(gen_vimdoc): use stronger typing for CONFIG, avoid dict

---
 scripts/gen_vimdoc.py | 279 +++++++++++++++++++++++++++++---------------------
 1 file changed, 165 insertions(+), 114 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index b51bd5fbf5..f22d11ed6b 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen.
+
+r"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen.
 
 Also generates *.mpack files. To inspect the *.mpack structure:
     :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B')))
@@ -32,24 +33,29 @@ The generated :help text for each function is formatted as follows:
     parameter is marked as [out].
   - Each function documentation is separated by a single line.
 """
+
+from __future__ import annotations
+
 import argparse
+import collections
+import dataclasses
+import logging
 import os
 import re
-import sys
 import shutil
-import textwrap
 import subprocess
-import collections
-import msgpack
-import logging
-from typing import Tuple
+import sys
+import textwrap
 from pathlib import Path
-
+from typing import Any, Callable, Dict, List, Literal, Tuple
 from xml.dom import minidom
+
+import msgpack
+
 Element = minidom.Element
 Document = minidom.Document
 
-MIN_PYTHON_VERSION = (3, 6)
+MIN_PYTHON_VERSION = (3, 7)
 MIN_DOXYGEN_VERSION = (1, 9, 0)
 
 if sys.version_info < MIN_PYTHON_VERSION:
@@ -68,7 +74,7 @@ if doxygen_version < MIN_DOXYGEN_VERSION:
 # Need a `nvim` that supports `-l`, try the local build
 nvim_path = Path(__file__).parent / "../build/bin/nvim"
 if nvim_path.exists():
-    nvim = str(nvim_path)
+    nvim = nvim_path.resolve()
 else:
     # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works.
     nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True)
@@ -103,12 +109,59 @@ filter_cmd = '%s %s' % (sys.executable, script_path)
 msgs = []  # Messages to show on exit.
 lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua')
 
-CONFIG = {
-    'api': {
-        'mode': 'c',
-        'filename': 'api.txt',
+
+SectionName = str
+
+@dataclasses.dataclass
+class Config:
+    """Config for documentation."""
+
+    mode: Literal['c', 'lua']
+
+    filename: str
+    """Generated documentation target, e.g. api.txt"""
+
+    section_order: List[str]
+    """Section ordering."""
+
+    files: List[str]
+    """List of files/directories for doxygen to read, relative to `base_dir`."""
+
+    file_patterns: str
+    """file patterns used by doxygen."""
+
+    section_name: Dict[str, SectionName]
+    """Section name overrides. Key: filename (e.g., vim.c)"""
+
+    section_fmt: Callable[[SectionName], str]
+    """For generated section names."""
+
+    helptag_fmt: Callable[[SectionName], str]
+    """Section helptag."""
+
+    fn_helptag_fmt: Callable[[str, str, bool], str]
+    """Per-function helptag."""
+
+    module_override: Dict[str, str]
+    """Module name overrides (for Lua)."""
+
+    append_only: List[str]
+    """Append the docs for these modules, do not start a new section."""
+
+    fn_name_prefix: str
+    """Only function with this prefix are considered"""
+
+    fn_name_fmt: Callable[[str, str], str] | None = None
+
+    include_tables: bool = True
+
+
+CONFIG: Dict[str, Config] = {
+    'api': Config(
+        mode = 'c',
+        filename = 'api.txt',
         # Section ordering.
-        'section_order': [
+        section_order=[
             'vim.c',
             'vimscript.c',
             'command.c',
@@ -121,31 +174,22 @@ CONFIG = {
             'autocmd.c',
             'ui.c',
         ],
-        # List of files/directories for doxygen to read, relative to `base_dir`
-        'files': ['src/nvim/api'],
-        # file patterns used by doxygen
-        'file_patterns': '*.h *.c',
-        # Only function with this prefix are considered
-        'fn_name_prefix': 'nvim_',
-        # Section name overrides.
-        'section_name': {
+        files=['src/nvim/api'],
+        file_patterns = '*.h *.c',
+        fn_name_prefix = 'nvim_',
+        section_name={
             'vim.c': 'Global',
         },
-        # For generated section names.
-        'section_fmt': lambda name: f'{name} Functions',
-        # Section helptag.
-        'helptag_fmt': lambda name: f'*api-{name.lower()}*',
-        # Per-function helptag.
-        'fn_helptag_fmt': lambda fstem, name, istbl: f'*{name}()*',
-        # Module name overrides (for Lua).
-        'module_override': {},
-        # Append the docs for these modules, do not start a new section.
-        'append_only': [],
-    },
-    'lua': {
-        'mode': 'lua',
-        'filename': 'lua.txt',
-        'section_order': [
+        section_fmt=lambda name: f'{name} Functions',
+        helptag_fmt=lambda name: f'*api-{name.lower()}*',
+        fn_helptag_fmt=lambda fstem, name, istbl: f'*{name}()*',
+        module_override={},
+        append_only=[],
+    ),
+    'lua': Config(
+        mode='lua',
+        filename='lua.txt',
+        section_order=[
             'highlight.lua',
             'regex.lua',
             'diff.lua',
@@ -171,7 +215,7 @@ CONFIG = {
             'snippet.lua',
             'text.lua',
         ],
-        'files': [
+        files=[
             'runtime/lua/vim/iter.lua',
             'runtime/lua/vim/_editor.lua',
             'runtime/lua/vim/_options.lua',
@@ -197,30 +241,30 @@ CONFIG = {
             'runtime/lua/vim/_meta/regex.lua',
             'runtime/lua/vim/_meta/spell.lua',
         ],
-        'file_patterns': '*.lua',
-        'fn_name_prefix': '',
-        'fn_name_fmt': lambda fstem, name: (
+        file_patterns='*.lua',
+        fn_name_prefix='',
+        fn_name_fmt=lambda fstem, name: (
             name if fstem in [ 'vim.iter' ] else
             f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else
             f'vim.{name}' if fstem == '_options' and not name[0].isupper() else
             f'{fstem}.{name}' if fstem.startswith('vim') else
             name
         ),
-        'section_name': {
+        section_name={
             'lsp.lua': 'core',
             '_inspector.lua': 'inspector',
         },
-        'section_fmt': lambda name: (
+        section_fmt=lambda name: (
             'Lua module: vim' if name.lower() == '_editor' else
             'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else
             f'VIM.{name.upper()}' if name.lower() in [ 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', 'regex' ] else
             'VIM' if name.lower() == 'builtin' else
             f'Lua module: vim.{name.lower()}'),
-        'helptag_fmt': lambda name: (
+        helptag_fmt=lambda name: (
             '*lua-vim*' if name.lower() == '_editor' else
             '*lua-vimscript*' if name.lower() == '_options' else
             f'*vim.{name.lower()}*'),
-        'fn_helptag_fmt': lambda fstem, name, istbl: (
+        fn_helptag_fmt=lambda fstem, name, istbl: (
             f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else
             # Exclude fstem for methods
             f'*{name}()*' if ':' in name else
@@ -230,7 +274,7 @@ CONFIG = {
             f'*{fstem}()*' if fstem.endswith('.' + name) else
             f'*{fstem}.{name}{"" if istbl else "()"}*'
             ),
-        'module_override': {
+        module_override={
             # `shared` functions are exposed on the `vim` module.
             'shared': 'vim',
             '_inspector': 'vim',
@@ -255,14 +299,14 @@ CONFIG = {
             'text': 'vim.text',
             'glob': 'vim.glob',
         },
-        'append_only': [
+        append_only=[
             'shared.lua',
         ],
-    },
-    'lsp': {
-        'mode': 'lua',
-        'filename': 'lsp.txt',
-        'section_order': [
+    ),
+    'lsp': Config(
+        mode='lua',
+        filename='lsp.txt',
+        section_order=[
             'lsp.lua',
             'buf.lua',
             'diagnostic.lua',
@@ -276,50 +320,50 @@ CONFIG = {
             'rpc.lua',
             'protocol.lua',
         ],
-        'files': [
+        files=[
             'runtime/lua/vim/lsp',
             'runtime/lua/vim/lsp.lua',
         ],
-        'file_patterns': '*.lua',
-        'fn_name_prefix': '',
-        'section_name': {'lsp.lua': 'lsp'},
-        'section_fmt': lambda name: (
+        file_patterns='*.lua',
+        fn_name_prefix='',
+        section_name={'lsp.lua': 'lsp'},
+        section_fmt=lambda name: (
             'Lua module: vim.lsp'
             if name.lower() == 'lsp'
             else f'Lua module: vim.lsp.{name.lower()}'),
-        'helptag_fmt': lambda name: (
+        helptag_fmt=lambda name: (
             '*lsp-core*'
             if name.lower() == 'lsp'
             else f'*lsp-{name.lower()}*'),
-        'fn_helptag_fmt': lambda fstem, name, istbl: (
+        fn_helptag_fmt=lambda fstem, name, istbl: (
             f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else
             # HACK. TODO(justinmk): class/structure support in lua2dox
             '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else
             f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'),
-        'module_override': {},
-        'append_only': [],
-    },
-    'diagnostic': {
-        'mode': 'lua',
-        'filename': 'diagnostic.txt',
-        'section_order': [
+        module_override={},
+        append_only=[],
+    ),
+    'diagnostic': Config(
+        mode='lua',
+        filename='diagnostic.txt',
+        section_order=[
             'diagnostic.lua',
         ],
-        'files': ['runtime/lua/vim/diagnostic.lua'],
-        'file_patterns': '*.lua',
-        'fn_name_prefix': '',
-        'include_tables': False,
-        'section_name': {'diagnostic.lua': 'diagnostic'},
-        'section_fmt': lambda _: 'Lua module: vim.diagnostic',
-        'helptag_fmt': lambda _: '*diagnostic-api*',
-        'fn_helptag_fmt': lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*',
-        'module_override': {},
-        'append_only': [],
-    },
-    'treesitter': {
-        'mode': 'lua',
-        'filename': 'treesitter.txt',
-        'section_order': [
+        files=['runtime/lua/vim/diagnostic.lua'],
+        file_patterns='*.lua',
+        fn_name_prefix='',
+        include_tables=False,
+        section_name={'diagnostic.lua': 'diagnostic'},
+        section_fmt=lambda _: 'Lua module: vim.diagnostic',
+        helptag_fmt=lambda _: '*diagnostic-api*',
+        fn_helptag_fmt=lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*',
+        module_override={},
+        append_only=[],
+    ),
+    'treesitter': Config(
+        mode='lua',
+        filename='treesitter.txt',
+        section_order=[
             'treesitter.lua',
             'language.lua',
             'query.lua',
@@ -327,30 +371,30 @@ CONFIG = {
             'languagetree.lua',
             'dev.lua',
         ],
-        'files': [
+        files=[
             'runtime/lua/vim/treesitter.lua',
             'runtime/lua/vim/treesitter/',
         ],
-        'file_patterns': '*.lua',
-        'fn_name_prefix': '',
-        'section_name': {},
-        'section_fmt': lambda name: (
+        file_patterns='*.lua',
+        fn_name_prefix='',
+        section_name={},
+        section_fmt=lambda name: (
             'Lua module: vim.treesitter'
             if name.lower() == 'treesitter'
             else f'Lua module: vim.treesitter.{name.lower()}'),
-        'helptag_fmt': lambda name: (
+        helptag_fmt=lambda name: (
             '*lua-treesitter-core*'
             if name.lower() == 'treesitter'
             else f'*lua-treesitter-{name.lower()}*'),
-        'fn_helptag_fmt': lambda fstem, name, istbl: (
+        fn_helptag_fmt=lambda fstem, name, istbl: (
             f'*vim.{fstem}.{name}()*'
             if fstem == 'treesitter'
             else f'*{name}()*'
             if name[0].isupper()
             else f'*vim.treesitter.{fstem}.{name}()*'),
-        'module_override': {},
-        'append_only': [],
-    }
+        module_override={},
+        append_only=[],
+    ),
 }
 
 param_exclude = (
@@ -814,6 +858,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
 
     return chunks, xrefs
 
+
 def is_program_listing(para):
     """
     Return True if `para` contains a "programlisting" (i.e. a Markdown code
@@ -835,6 +880,7 @@ def is_program_listing(para):
 
     return len(children) == 1 and children[0].nodeName == 'programlisting'
 
+
 def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='',
                         fmt_vimhelp=False):
     """Renders (nested) Doxygen <para> nodes as Vim :help text.
@@ -910,6 +956,8 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
     The `fmt_vimhelp` variable controls some special cases for use by
     fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :)
     """
+    config: Config = CONFIG[target]
+
     fns = {}  # Map of func_name:docstring.
     deprecated_fns = {}  # Map of func_name:docstring.
 
@@ -934,7 +982,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
             continue
 
         istbl = return_type.startswith('table')  # Special from lua2dox.lua.
-        if istbl and not CONFIG[target].get('include_tables', True):
+        if istbl and not config.include_tables:
             continue
 
         if return_type.startswith(('ArrayOf', 'DictionaryOf')):
@@ -962,7 +1010,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
             declname = get_child(param, 'declname')
             if declname:
                 param_name = get_text(declname).strip()
-            elif CONFIG[target]['mode'] == 'lua':
+            elif config.mode == 'lua':
                 # XXX: this is what lua2dox gives us...
                 param_name = param_type
                 param_type = ''
@@ -998,11 +1046,11 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
             fstem = '?'
             if '.' in compoundname:
                 fstem = compoundname.split('.')[0]
-                fstem = CONFIG[target]['module_override'].get(fstem, fstem)
-            vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name, istbl)
+                fstem = config.module_override.get(fstem, fstem)
+            vimtag = config.fn_helptag_fmt(fstem, name, istbl)
 
-            if 'fn_name_fmt' in CONFIG[target]:
-                name = CONFIG[target]['fn_name_fmt'](fstem, name)
+            if config.fn_name_fmt:
+                name = config.fn_name_fmt(fstem, name)
 
         if istbl:
             aopen, aclose = '', ''
@@ -1085,7 +1133,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
 
         if 'Deprecated' in str(xrefs_all):
             deprecated_fns[name] = fn
-        elif name.startswith(CONFIG[target]['fn_name_prefix']):
+        elif name.startswith(config.fn_name_prefix):
             fns[name] = fn
 
     fns = collections.OrderedDict(sorted(
@@ -1102,6 +1150,8 @@ def fmt_doxygen_xml_as_vimhelp(filename, target):
       1. Vim help text for functions found in `filename`.
       2. Vim help text for deprecated functions.
     """
+    config: Config = CONFIG[target]
+
     fns_txt = {}  # Map of func_name:vim-help-text.
     deprecated_fns_txt = {}  # Map of func_name:vim-help-text.
     fns, _ = extract_from_xml(filename, target, text_width, True)
@@ -1164,7 +1214,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, target):
 
         func_doc = "\n".join(map(align_tags, split_lines))
 
-        if (name.startswith(CONFIG[target]['fn_name_prefix'])
+        if (name.startswith(config.fn_name_prefix)
            and name != "nvim_error_event"):
             fns_txt[name] = func_doc
 
@@ -1237,9 +1287,12 @@ def main(doxygen_config, args):
     for target in CONFIG:
         if args.target is not None and target != args.target:
             continue
+
+        config: Config = CONFIG[target]
+
         mpack_file = os.path.join(
             base_dir, 'runtime', 'doc',
-            CONFIG[target]['filename'].replace('.txt', '.mpack'))
+            config.filename.replace('.txt', '.mpack'))
         if os.path.exists(mpack_file):
             os.remove(mpack_file)
 
@@ -1255,11 +1308,10 @@ def main(doxygen_config, args):
                 stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL))
         p.communicate(
             doxygen_config.format(
-                input=' '.join(
-                    [f'"{file}"' for file in CONFIG[target]['files']]),
+                input=' '.join([f'"{file}"' for file in config.files]),
                 output=output_dir,
                 filter=filter_cmd,
-                file_patterns=CONFIG[target]['file_patterns'])
+                file_patterns=config.file_patterns)
             .encode('utf8')
         )
         if p.returncode:
@@ -1294,9 +1346,9 @@ def main(doxygen_config, args):
                     filename = os.path.basename(filename)
                     name = os.path.splitext(filename)[0].lower()
                     sectname = name.upper() if name == 'ui' else name.title()
-                    sectname = CONFIG[target]['section_name'].get(filename, sectname)
-                    title = CONFIG[target]['section_fmt'](sectname)
-                    section_tag = CONFIG[target]['helptag_fmt'](sectname)
+                    sectname = config.section_name.get(filename, sectname)
+                    title = config.section_fmt(sectname)
+                    section_tag = config.helptag_fmt(sectname)
                     # Module/Section id matched against @defgroup.
                     #   "*api-buffer*" => "api-buffer"
                     section_id = section_tag.strip('*')
@@ -1319,22 +1371,22 @@ def main(doxygen_config, args):
 
         if len(sections) == 0:
             fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)')
-        if len(sections) > len(CONFIG[target]['section_order']):
+        if len(sections) > len(config.section_order):
             raise RuntimeError(
                 'found new modules "{}"; update the "section_order" map'.format(
-                    set(sections).difference(CONFIG[target]['section_order'])))
-        first_section_tag = sections[CONFIG[target]['section_order'][0]][1]
+                    set(sections).difference(config.section_order)))
+        first_section_tag = sections[config.section_order[0]][1]
 
         docs = ''
 
-        for filename in CONFIG[target]['section_order']:
+        for filename in config.section_order:
             try:
                 title, section_tag, section_doc = sections.pop(filename)
             except KeyError:
                 msg(f'warning: empty docs, skipping (target={target}): {filename}')
                 msg(f'    existing docs: {sections.keys()}')
                 continue
-            if filename not in CONFIG[target]['append_only']:
+            if filename not in config.append_only:
                 docs += sep
                 docs += '\n{}{}'.format(title, section_tag.rjust(text_width - len(title)))
             docs += section_doc
@@ -1343,8 +1395,7 @@ def main(doxygen_config, args):
         docs = docs.rstrip() + '\n\n'
         docs += f' vim:tw=78:ts=8:sw={indentation}:sts={indentation}:et:ft=help:norl:\n'
 
-        doc_file = os.path.join(base_dir, 'runtime', 'doc',
-                                CONFIG[target]['filename'])
+        doc_file = os.path.join(base_dir, 'runtime', 'doc', config.filename)
 
         if os.path.exists(doc_file):
             delete_lines_below(doc_file, first_section_tag)
-- 
cgit 


From 1a31d4cf2b88e9f677d7a2f23b10e4dacf803a9d Mon Sep 17 00:00:00 2001
From: Jongwook Choi <wookayin@gmail.com>
Date: Thu, 28 Dec 2023 14:11:13 -0500
Subject: refactor(gen_vimdoc): use typing for function API vimdoc generation

---
 scripts/gen_vimdoc.py | 143 +++++++++++++++++++++++++++++++++++---------------
 1 file changed, 102 insertions(+), 41 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index f22d11ed6b..40ece01ba3 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -112,6 +112,11 @@ lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua')
 
 SectionName = str
 
+Docstring = str  # Represents (formatted) vimdoc string
+
+FunctionName = str
+
+
 @dataclasses.dataclass
 class Config:
     """Config for documentation."""
@@ -881,6 +886,44 @@ def is_program_listing(para):
     return len(children) == 1 and children[0].nodeName == 'programlisting'
 
 
+FunctionParam = Tuple[
+    str,  # type
+    str,  # parameter name
+]
+
+@dataclasses.dataclass
+class FunctionDoc:
+    """Data structure for function documentation. Also exported as msgpack."""
+
+    annotations: List[str]
+    """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map"""
+
+    signature: str
+    """Function signature with *tags*."""
+
+    parameters: List[FunctionParam]
+    """Parameters: (type, name)"""
+
+    parameters_doc: Dict[str, Docstring]
+    """Parameters documentation. Key is parameter name, value is doc."""
+
+    doc: List[Docstring]
+    """Main description for the function. Separated by paragraph."""
+
+    return_: List[Docstring]
+    """Return:, or Return (multiple): (@return strings)"""
+
+    seealso: List[Docstring]
+    """See also: (@see strings)"""
+
+    # for fmt_node_as_vimhelp
+    desc_node: Element | None = None
+    brief_desc_node: Element | None = None
+
+    # for INCLUDE_C_DECL
+    c_decl: str | None = None
+
+
 def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='',
                         fmt_vimhelp=False):
     """Renders (nested) Doxygen <para> nodes as Vim :help text.
@@ -946,7 +989,10 @@ def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=
     return clean_lines('\n'.join(rendered_blocks).strip())
 
 
-def extract_from_xml(filename, target, width, fmt_vimhelp):
+def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[
+    Dict[FunctionName, FunctionDoc],
+    Dict[FunctionName, FunctionDoc],
+]:
     """Extracts Doxygen info as maps without formatting the text.
 
     Returns two maps:
@@ -958,8 +1004,8 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
     """
     config: Config = CONFIG[target]
 
-    fns = {}  # Map of func_name:docstring.
-    deprecated_fns = {}  # Map of func_name:docstring.
+    fns: Dict[FunctionName, FunctionDoc] = {}
+    deprecated_fns: Dict[FunctionName, FunctionDoc] = {}
 
     dom = minidom.parse(filename)
     compoundname = get_text(dom.getElementsByTagName('compoundname')[0])
@@ -1084,7 +1130,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
         # Tracks `xrefsect` titles.  As of this writing, used only for separating
         # deprecated functions.
         xrefs_all = set()
-        paras = []
+        paras: List[Dict[str, Any]] = []
         brief_desc = find_first(member, 'briefdescription')
         if brief_desc:
             for child in brief_desc.childNodes:
@@ -1103,47 +1149,48 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
                            desc.toprettyxml(indent='  ', newl='\n')),
                     ' ' * indentation))
 
-        fn = {
-            'annotations': list(annotations),
-            'signature': signature,
-            'parameters': params,
-            'parameters_doc': collections.OrderedDict(),
-            'doc': [],
-            'return': [],
-            'seealso': [],
-        }
+        fn = FunctionDoc(
+            annotations=list(annotations),
+            signature=signature,
+            parameters=params,
+            parameters_doc=collections.OrderedDict(),
+            doc=[],
+            return_=[],
+            seealso=[],
+        )
         if fmt_vimhelp:
-            fn['desc_node'] = desc
-            fn['brief_desc_node'] = brief_desc
+            fn.desc_node = desc
+            fn.brief_desc_node = brief_desc
 
         for m in paras:
-            if 'text' in m:
-                if not m['text'] == '':
-                    fn['doc'].append(m['text'])
+            if m.get('text', ''):
+                fn.doc.append(m['text'])
             if 'params' in m:
                 # Merge OrderedDicts.
-                fn['parameters_doc'].update(m['params'])
+                fn.parameters_doc.update(m['params'])
             if 'return' in m and len(m['return']) > 0:
-                fn['return'] += m['return']
+                fn.return_ += m['return']
             if 'seealso' in m and len(m['seealso']) > 0:
-                fn['seealso'] += m['seealso']
+                fn.seealso += m['seealso']
 
         if INCLUDE_C_DECL:
-            fn['c_decl'] = c_decl
+            fn.c_decl = c_decl
 
         if 'Deprecated' in str(xrefs_all):
             deprecated_fns[name] = fn
         elif name.startswith(config.fn_name_prefix):
             fns[name] = fn
 
+    # sort functions by name (lexicographically)
     fns = collections.OrderedDict(sorted(
         fns.items(),
-        key=lambda key_item_tuple: key_item_tuple[0].lower()))
+        key=lambda key_item_tuple: key_item_tuple[0].lower(),
+    ))
     deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items()))
     return fns, deprecated_fns
 
 
-def fmt_doxygen_xml_as_vimhelp(filename, target):
+def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]:
     """Entrypoint for generating Vim :help from from Doxygen XML.
 
     Returns 2 items:
@@ -1154,20 +1201,26 @@ def fmt_doxygen_xml_as_vimhelp(filename, target):
 
     fns_txt = {}  # Map of func_name:vim-help-text.
     deprecated_fns_txt = {}  # Map of func_name:vim-help-text.
+
+    fns: Dict[FunctionName, FunctionDoc]
     fns, _ = extract_from_xml(filename, target, text_width, True)
 
-    for name, fn in fns.items():
+    for fn_name, fn in fns.items():
         # Generate Vim :help for parameters.
-        if fn['desc_node']:
-            doc = fmt_node_as_vimhelp(fn['desc_node'], fmt_vimhelp=True)
-        if not doc and fn['brief_desc_node']:
-            doc = fmt_node_as_vimhelp(fn['brief_desc_node'])
-        if not doc and name.startswith("nvim__"):
+
+        # Generate body.
+        doc = ''
+        if fn.desc_node:
+            doc = fmt_node_as_vimhelp(fn.desc_node, fmt_vimhelp=True)
+        if not doc and fn.brief_desc_node:
+            doc = fmt_node_as_vimhelp(fn.brief_desc_node)
+        if not doc and fn_name.startswith("nvim__"):
             continue
         if not doc:
             doc = 'TODO: Documentation'
 
-        annotations = '\n'.join(fn['annotations'])
+        # Annotations: put before Parameters
+        annotations: str = '\n'.join(fn.annotations)
         if annotations:
             annotations = ('\n\nAttributes: ~\n' +
                            textwrap.indent(annotations, '    '))
@@ -1177,18 +1230,22 @@ def fmt_doxygen_xml_as_vimhelp(filename, target):
             else:
                 doc = doc[:i] + annotations + '\n\n' + doc[i:]
 
+        # C Declaration: (debug only)
         if INCLUDE_C_DECL:
             doc += '\n\nC Declaration: ~\n>\n'
-            doc += fn['c_decl']
+            assert fn.c_decl is not None
+            doc += fn.c_decl
             doc += '\n<'
 
-        func_doc = fn['signature'] + '\n'
+        # Start of function documentations. e.g.,
+        # nvim_cmd({*cmd}, {*opts})                                         *nvim_cmd()*
+        func_doc = fn.signature + '\n'
         func_doc += textwrap.indent(clean_lines(doc), ' ' * indentation)
 
         # Verbatim handling.
         func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M)
 
-        split_lines = func_doc.split('\n')
+        split_lines: List[str] = func_doc.split('\n')
         start = 0
         while True:
             try:
@@ -1214,12 +1271,14 @@ def fmt_doxygen_xml_as_vimhelp(filename, target):
 
         func_doc = "\n".join(map(align_tags, split_lines))
 
-        if (name.startswith(config.fn_name_prefix)
-           and name != "nvim_error_event"):
-            fns_txt[name] = func_doc
+        if (fn_name.startswith(config.fn_name_prefix)
+            and fn_name != "nvim_error_event"):
+            fns_txt[fn_name] = func_doc
 
-    return ('\n\n'.join(list(fns_txt.values())),
-            '\n\n'.join(list(deprecated_fns_txt.values())))
+    return (
+        '\n\n'.join(list(fns_txt.values())),
+        '\n\n'.join(list(deprecated_fns_txt.values())),
+    )
 
 
 def delete_lines_below(filename, tokenstr):
@@ -1402,9 +1461,11 @@ def main(doxygen_config, args):
         with open(doc_file, 'ab') as fp:
             fp.write(docs.encode('utf8'))
 
-        fn_map_full = collections.OrderedDict(sorted(fn_map_full.items()))
+        fn_map_full = collections.OrderedDict(sorted(
+            (name, fn_doc.__dict__) for (name, fn_doc) in fn_map_full.items()
+        ))
         with open(mpack_file, 'wb') as fp:
-            fp.write(msgpack.packb(fn_map_full, use_bin_type=True))
+            fp.write(msgpack.packb(fn_map_full, use_bin_type=True))  # type: ignore
 
         if not args.keep_tmpfiles:
             shutil.rmtree(output_dir)
-- 
cgit 


From 4e9298ecdf945b4d16c2c6e6e4ed82b97880917c Mon Sep 17 00:00:00 2001
From: Jongwook Choi <wookayin@gmail.com>
Date: Thu, 28 Dec 2023 17:50:05 -0500
Subject: refactor(gen_vimdoc): generate function doc from metadata, not from
 xml

Problem:

For function definitions to be included in the vimdoc (formatted) and
to be exported as mpack data (unformatted), we had two internal
representations of the same function/API metadata in duplicate;
one is FunctionDoc (which was previously a dict), and the other is
doxygen XML DOM from which vimdoc (functions sections) was generated.

Solution:

We should have a single path and unified data representation
(i.e. FunctionDoc) that contains all the metadata and information about
function APIs, from which both of mpack export and vimdoc are generated.
I.e., vimdocs are no longer generated directly from doxygen XML nodes,
but generated via:

  (XML DOM Nodes) ------------> FunctionDoc ------> mpack (unformatted)
                   Recursive     Internal     |
                   Formatting    Metadata     +---> vimdoc (formatted)

This refactoring eliminates the hacky and ugly use of `fmt_vimhelp` in
`fmt_node_as_vimhelp()` and all other helper functions! This way,
`fmt_node_as_vimhelp()` can simplified as it no longer needs to handle
generating of function docs, which needs to be done only in the topmost
level of recursion.
---
 scripts/gen_vimdoc.py | 220 ++++++++++++++++++++++++++++++++++----------------
 1 file changed, 151 insertions(+), 69 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 40ece01ba3..eede5b76a2 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -751,7 +751,10 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation,
     return text
 
 
-def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=False):
+def para_as_map(parent: Element,
+                indent: str = '',
+                width: int = (text_width - indentation),
+                ):
     """Extracts a Doxygen XML <para> node to a map.
 
     Keys:
@@ -787,7 +790,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
     kind = ''
     if is_inline(parent):
         # Flatten inline text from a tree of non-block nodes.
-        text = doc_wrap(render_node(parent, "", fmt_vimhelp=fmt_vimhelp),
+        text = doc_wrap(render_node(parent, ""),
                         indent=indent, width=width)
     else:
         prev = None  # Previous node
@@ -805,8 +808,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
                 elif kind == 'see':
                     groups['seealso'].append(child)
                 elif kind == 'warning':
-                    text += render_node(child, text, indent=indent,
-                                        width=width, fmt_vimhelp=fmt_vimhelp)
+                    text += render_node(child, text, indent=indent, width=width)
                 elif kind == 'since':
                     since_match = re.match(r'^(\d+)', get_text(child))
                     since = int(since_match.group(1)) if since_match else 0
@@ -827,8 +829,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
                         and ' ' != text[-1]):
                     text += ' '
 
-                text += render_node(child, text, indent=indent, width=width,
-                                    fmt_vimhelp=fmt_vimhelp)
+                text += render_node(child, text, indent=indent, width=width)
                 prev = child
 
     chunks['text'] += text
@@ -839,17 +840,17 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
             update_params_map(child, ret_map=chunks['params'], width=width)
     for child in groups['note']:
         chunks['note'].append(render_node(
-            child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp).rstrip())
+            child, '', indent=indent, width=width).rstrip())
     for child in groups['return']:
         chunks['return'].append(render_node(
-            child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp))
+            child, '', indent=indent, width=width))
     for child in groups['seealso']:
         # Example:
         #   <simplesect kind="see">
         #     <para>|autocommand|</para>
         #   </simplesect>
         chunks['seealso'].append(render_node(
-            child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp))
+            child, '', indent=indent, width=width))
 
     xrefs = set()
     for child in groups['xrefs']:
@@ -898,6 +899,9 @@ class FunctionDoc:
     annotations: List[str]
     """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map"""
 
+    notes: List[Docstring]
+    """Notes: (@note strings)"""
+
     signature: str
     """Function signature with *tags*."""
 
@@ -916,41 +920,122 @@ class FunctionDoc:
     seealso: List[Docstring]
     """See also: (@see strings)"""
 
-    # for fmt_node_as_vimhelp
-    desc_node: Element | None = None
-    brief_desc_node: Element | None = None
+    xrefs: List[Docstring]
+    """XRefs. Currently only used to track Deprecated functions."""
 
     # for INCLUDE_C_DECL
     c_decl: str | None = None
 
+    prerelease: bool = False
+
+    def export_mpack(self) -> Dict[str, Any]:
+        """Convert a dict to be exported as mpack data."""
+        exported = self.__dict__.copy()
+        del exported['notes']
+        del exported['c_decl']
+        del exported['prerelease']
+        del exported['xrefs']
+        exported['return'] = exported.pop('return_')
+        return exported
+
+    def doc_concatenated(self) -> Docstring:
+        """Concatenate all the paragraphs in `doc` into a single string, but
+        remove blank lines before 'programlisting' blocks. #25127
+
+        BEFORE (without programlisting processing):
+            ```vimdoc
+            Example:
+
+            >vim
+                :echo nvim_get_color_by_name("Pink")
+            <
+            ```
+
+        AFTER:
+            ```vimdoc
+            Example: >vim
+                :echo nvim_get_color_by_name("Pink")
+            <
+            ```
+        """
+        def is_program_listing(paragraph: str) -> bool:
+            lines = paragraph.strip().split('\n')
+            return lines[0].startswith('>') and lines[-1] == '<'
+
+        rendered = []
+        for paragraph in self.doc:
+            if is_program_listing(paragraph):
+                rendered.append(' ')  # Example: >vim
+            elif rendered:
+                rendered.append('\n\n')
+            rendered.append(paragraph)
+        return ''.join(rendered)
+
+    def render(self) -> Docstring:
+        """Renders function documentation as Vim :help text."""
+        rendered_blocks: List[Docstring] = []
+
+        def fmt_param_doc(m):
+            """Renders a params map as Vim :help text."""
+            max_name_len = max_name(m.keys()) + 4
+            out = ''
+            for name, desc in m.items():
+                if name == 'self':
+                    continue
+                name = '  • {}'.format('{{{}}}'.format(name).ljust(max_name_len))
+                out += '{}{}\n'.format(name, desc)
+            return out.rstrip()
+
+        # Generate text from the gathered items.
+        chunks: List[Docstring] = [self.doc_concatenated()]
+
+        notes = []
+        if self.prerelease:
+            notes = ["  This API is pre-release (unstable)."]
+        notes += self.notes
+        if len(notes) > 0:
+            chunks.append('\nNote: ~')
+            for s in notes:
+                chunks.append('  ' + s)
+
+        if self.parameters_doc:
+            chunks.append('\nParameters: ~')
+            chunks.append(fmt_param_doc(self.parameters_doc))
+
+        if self.return_:
+            chunks.append('\nReturn (multiple): ~' if len(self.return_) > 1
+                          else '\nReturn: ~')
+            for s in self.return_:
+                chunks.append('    ' + s)
+
+        if self.seealso:
+            chunks.append('\nSee also: ~')
+            for s in self.seealso:
+                chunks.append('  ' + s)
+
+        # Note: xrefs are currently only used to remark "Deprecated: "
+        # for deprecated functions; visible when INCLUDE_DEPRECATED is set
+        for s in self.xrefs:
+            chunks.append('\n' + s)
+
+        rendered_blocks.append(clean_lines('\n'.join(chunks).strip()))
+        rendered_blocks.append('')
 
-def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='',
-                        fmt_vimhelp=False):
+        return clean_lines('\n'.join(rendered_blocks).strip())
+
+
+def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=''):
     """Renders (nested) Doxygen <para> nodes as Vim :help text.
 
+    Only handles "text" nodes. Used for individual elements (see render_node())
+    and in extract_defgroups().
+
     NB: Blank lines in a docstring manifest as <para> tags.
     """
     rendered_blocks = []
 
-    def fmt_param_doc(m):
-        """Renders a params map as Vim :help text."""
-        max_name_len = max_name(m.keys()) + 4
-        out = ''
-        for name, desc in m.items():
-            if name == 'self':
-                continue
-            name = '  • {}'.format('{{{}}}'.format(name).ljust(max_name_len))
-            out += '{}{}\n'.format(name, desc)
-        return out.rstrip()
-
-    def has_nonexcluded_params(m):
-        """Returns true if any of the given params has at least
-        one non-excluded item."""
-        if fmt_param_doc(m) != '':
-            return True
-
     for child in parent.childNodes:
-        para, _ = para_as_map(child, indent, width, fmt_vimhelp)
+        para, _ = para_as_map(child, indent, width)
 
         # 'programlisting' blocks are Markdown code blocks. Do not include
         # these as a separate paragraph, but append to the last non-empty line
@@ -963,25 +1048,6 @@ def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=
 
         # Generate text from the gathered items.
         chunks = [para['text']]
-        notes = ["    This API is pre-release (unstable)."] if para['prerelease'] else []
-        notes += para['note']
-        if len(notes) > 0:
-            chunks.append('\nNote: ~')
-            for s in notes:
-                chunks.append(s)
-        if len(para['params']) > 0 and has_nonexcluded_params(para['params']):
-            chunks.append('\nParameters: ~')
-            chunks.append(fmt_param_doc(para['params']))
-        if len(para['return']) > 0:
-            chunks.append('\nReturn (multiple): ~' if len(para['return']) > 1 else '\nReturn: ~')
-            for s in para['return']:
-                chunks.append(s)
-        if len(para['seealso']) > 0:
-            chunks.append('\nSee also: ~')
-            for s in para['seealso']:
-                chunks.append(s)
-        for s in para['xrefs']:
-            chunks.append(s)
 
         rendered_blocks.append(clean_lines('\n'.join(chunks).strip()))
         rendered_blocks.append('')
@@ -989,7 +1055,8 @@ def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=
     return clean_lines('\n'.join(rendered_blocks).strip())
 
 
-def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[
+def extract_from_xml(filename, target, *,
+                     width: int, fmt_vimhelp: bool) -> Tuple[
     Dict[FunctionName, FunctionDoc],
     Dict[FunctionName, FunctionDoc],
 ]:
@@ -1130,18 +1197,20 @@ def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[
         # Tracks `xrefsect` titles.  As of this writing, used only for separating
         # deprecated functions.
         xrefs_all = set()
-        paras: List[Dict[str, Any]] = []
+        paras: List[Dict[str, Any]] = []  # paras means paragraphs!
         brief_desc = find_first(member, 'briefdescription')
         if brief_desc:
             for child in brief_desc.childNodes:
                 para, xrefs = para_as_map(child)
+                paras.append(para)
                 xrefs_all.update(xrefs)
 
         desc = find_first(member, 'detaileddescription')
         if desc:
+            paras_detail = []  # override briefdescription
             for child in desc.childNodes:
                 para, xrefs = para_as_map(child)
-                paras.append(para)
+                paras_detail.append(para)
                 xrefs_all.update(xrefs)
             log.debug(
                 textwrap.indent(
@@ -1149,18 +1218,25 @@ def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[
                            desc.toprettyxml(indent='  ', newl='\n')),
                     ' ' * indentation))
 
+            # override briefdescription, if detaileddescription is not empty
+            # (note: briefdescription can contain some erroneous luadoc
+            #  comments from preceding comments, this is a bug of lua2dox)
+            if any((para['text'] or para['note'] or para['params'] or
+                    para['return'] or para['seealso']
+                    ) for para in paras_detail):
+                paras = paras_detail
+
         fn = FunctionDoc(
             annotations=list(annotations),
+            notes=[],
             signature=signature,
             parameters=params,
             parameters_doc=collections.OrderedDict(),
             doc=[],
             return_=[],
             seealso=[],
+            xrefs=[],
         )
-        if fmt_vimhelp:
-            fn.desc_node = desc
-            fn.brief_desc_node = brief_desc
 
         for m in paras:
             if m.get('text', ''):
@@ -1172,6 +1248,12 @@ def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[
                 fn.return_ += m['return']
             if 'seealso' in m and len(m['seealso']) > 0:
                 fn.seealso += m['seealso']
+            if m.get('prerelease', False):
+                fn.prerelease = True
+            if 'note' in m:
+                fn.notes += m['note']
+            if 'xrefs' in m:
+                fn.xrefs += m['xrefs']
 
         if INCLUDE_C_DECL:
             fn.c_decl = c_decl
@@ -1203,17 +1285,14 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]:
     deprecated_fns_txt = {}  # Map of func_name:vim-help-text.
 
     fns: Dict[FunctionName, FunctionDoc]
-    fns, _ = extract_from_xml(filename, target, text_width, True)
+    fns, _ = extract_from_xml(filename, target,
+                              width=text_width, fmt_vimhelp=True)
 
     for fn_name, fn in fns.items():
         # Generate Vim :help for parameters.
 
-        # Generate body.
-        doc = ''
-        if fn.desc_node:
-            doc = fmt_node_as_vimhelp(fn.desc_node, fmt_vimhelp=True)
-        if not doc and fn.brief_desc_node:
-            doc = fmt_node_as_vimhelp(fn.brief_desc_node)
+        # Generate body from FunctionDoc, not XML nodes
+        doc = fn.render()
         if not doc and fn_name.startswith("nvim__"):
             continue
         if not doc:
@@ -1393,11 +1472,14 @@ def main(doxygen_config, args):
             filename = get_text(find_first(compound, 'name'))
             if filename.endswith('.c') or filename.endswith('.lua'):
                 xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid')))
+
                 # Extract unformatted (*.mpack).
-                fn_map, _ = extract_from_xml(xmlfile, target, 9999, False)
+                fn_map, _ = extract_from_xml(
+                    xmlfile, target, width=9999, fmt_vimhelp=False)
+
                 # Extract formatted (:help).
                 functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp(
-                    os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))), target)
+                    xmlfile, target)
 
                 if not functions_text and not deprecated_text:
                     continue
@@ -1461,11 +1543,11 @@ def main(doxygen_config, args):
         with open(doc_file, 'ab') as fp:
             fp.write(docs.encode('utf8'))
 
-        fn_map_full = collections.OrderedDict(sorted(
-            (name, fn_doc.__dict__) for (name, fn_doc) in fn_map_full.items()
+        fn_map_full_exported = collections.OrderedDict(sorted(
+            (name, fn_doc.export_mpack()) for (name, fn_doc) in fn_map_full.items()
         ))
         with open(mpack_file, 'wb') as fp:
-            fp.write(msgpack.packb(fn_map_full, use_bin_type=True))  # type: ignore
+            fp.write(msgpack.packb(fn_map_full_exported, use_bin_type=True))  # type: ignore
 
         if not args.keep_tmpfiles:
             shutil.rmtree(output_dir)
-- 
cgit 


From f74f52a1a5ff668187bf12c397319972764a9704 Mon Sep 17 00:00:00 2001
From: Jongwook Choi <wookayin@gmail.com>
Date: Fri, 29 Dec 2023 05:52:31 -0500
Subject: refactor(gen_vimdoc): refactor section and defgroup doc generation

Problem: main() has too much logic implemented there, too difficult to
read.

Solution: Do more OOP, introduce `Section` dataclass that stores
information about a "section", with documentation and concrete examples
about what each field and variable would mean. Extract all the lines for
rendering a section into `section.render()` pulled out of `main()`.
---
 scripts/gen_vimdoc.py | 185 +++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 138 insertions(+), 47 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index eede5b76a2..e5f2e61dfd 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -102,6 +102,8 @@ LOG_LEVELS = {
 
 text_width = 78
 indentation = 4
+SECTION_SEP = '=' * text_width
+
 script_path = os.path.abspath(__file__)
 base_dir = os.path.dirname(os.path.dirname(script_path))
 out_dir = os.path.join(base_dir, 'tmp-{target}-doc')
@@ -1378,7 +1380,7 @@ def delete_lines_below(filename, tokenstr):
         fp.writelines(lines[0:i])
 
 
-def extract_defgroups(base: str, dom: Document):
+def extract_defgroups(base: str, dom: Document) -> Dict[SectionName, Docstring]:
     '''Generate module-level (section) docs (@defgroup).'''
     section_docs = {}
 
@@ -1414,6 +1416,101 @@ def extract_defgroups(base: str, dom: Document):
     return section_docs
 
 
+@dataclasses.dataclass
+class Section:
+    """Represents a section. Includes section heading (defgroup)
+    and all the FunctionDoc that belongs to this section."""
+
+    name: str
+    '''Name of the section. Usually derived from basename of lua/c src file.
+    Example: "Autocmd".'''
+
+    title: str
+    '''Formatted section config. see config.section_fmt().
+    Example: "Autocmd Functions". '''
+
+    helptag: str
+    '''see config.helptag_fmt(). Example: *api-autocmd*'''
+
+    @property
+    def id(self) -> str:
+        '''section id: Module/Section id matched against @defgroup.
+           e.g., "*api-autocmd*" => "api-autocmd"
+        '''
+        return self.helptag.strip('*')
+
+    doc: str = ""
+    '''Section heading docs extracted from @defgroup.'''
+
+    # TODO: Do not carry rendered text, but handle FunctionDoc for better OOP
+    functions_text: Docstring | None = None
+    '''(Rendered) doc of all the functions that belong to this section.'''
+
+    deprecated_functions_text: Docstring | None = None
+    '''(Rendered) doc of all the deprecated functions that belong to this
+    section.'''
+
+    def __repr__(self):
+        return f"Section(title='{self.title}', helptag='{self.helptag}')"
+
+    @classmethod
+    def make_from(cls, filename: str, config: Config,
+                  section_docs: Dict[SectionName, str],
+                  *,
+                  functions_text: Docstring,
+                  deprecated_functions_text: Docstring,
+                  ):
+        # filename: e.g., 'autocmd.c'
+        # name: e.g. 'autocmd'
+        name = os.path.splitext(filename)[0].lower()
+
+        # section name: e.g. "Autocmd"
+        sectname: SectionName
+        sectname = name.upper() if name == 'ui' else name.title()
+        sectname = config.section_name.get(filename, sectname)
+
+        # Formatted (this is what's going to be written in the vimdoc)
+        # e.g., "Autocmd Functions"
+        title: str = config.section_fmt(sectname)
+
+        # section tag: e.g., "*api-autocmd*"
+        section_tag: str = config.helptag_fmt(sectname)
+
+        section = cls(name=sectname, title=title, helptag=section_tag,
+                      functions_text=functions_text,
+                      deprecated_functions_text=deprecated_functions_text,
+                      )
+        section.doc = section_docs.get(section.id) or ''
+        return section
+
+    def render(self, add_header=True) -> str:
+        """Render as vimdoc."""
+        doc = ''
+
+        if add_header:
+            doc += SECTION_SEP
+            doc += '\n{}{}'.format(
+                self.title,
+                self.helptag.rjust(text_width - len(self.title))
+            )
+
+        if self.doc:
+            doc += '\n\n' + self.doc
+
+        if self.functions_text:
+            doc += '\n\n' + self.functions_text
+
+        if INCLUDE_DEPRECATED and self.deprecated_functions_text:
+            doc += f'\n\n\nDeprecated {self.name} Functions: ~\n\n'
+            doc += self.deprecated_functions_text
+
+        return doc
+
+    def __bool__(self) -> bool:
+        """Whether this section has contents. Used for skipping empty ones."""
+        return bool(self.doc or self.functions_text)
+
+
 def main(doxygen_config, args):
     """Generates:
 
@@ -1455,14 +1552,16 @@ def main(doxygen_config, args):
         if p.returncode:
             sys.exit(p.returncode)
 
-        fn_map_full = {}  # Collects all functions as each module is processed.
-        sections = {}
-        sep = '=' * text_width
+        # Collects all functions as each module is processed.
+        fn_map_full: Dict[FunctionName, FunctionDoc] = {}
+        # key: filename (e.g. autocmd.c)
+        sections: Dict[str, Section] = {}
 
         base = os.path.join(output_dir, 'xml')
         dom = minidom.parse(os.path.join(base, 'index.xml'))
 
-        section_docs = extract_defgroups(base, dom)
+        # Collect all @defgroups (section headings after the '===...' separator
+        section_docs: Dict[SectionName, Docstring] = extract_defgroups(base, dom)
 
         # Generate docs for all functions in the current module.
         for compound in dom.getElementsByTagName('compound'):
@@ -1470,45 +1569,38 @@ def main(doxygen_config, args):
                 continue
 
             filename = get_text(find_first(compound, 'name'))
-            if filename.endswith('.c') or filename.endswith('.lua'):
-                xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid')))
+            if not (
+                filename.endswith('.c') or
+                filename.endswith('.lua')
+            ):
+                continue
 
-                # Extract unformatted (*.mpack).
-                fn_map, _ = extract_from_xml(
-                    xmlfile, target, width=9999, fmt_vimhelp=False)
+            xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid')))
 
-                # Extract formatted (:help).
-                functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp(
-                    xmlfile, target)
+            # Extract unformatted (*.mpack).
+            fn_map, _ = extract_from_xml(
+                xmlfile, target, width=9999, fmt_vimhelp=False)
 
-                if not functions_text and not deprecated_text:
-                    continue
-                else:
-                    filename = os.path.basename(filename)
-                    name = os.path.splitext(filename)[0].lower()
-                    sectname = name.upper() if name == 'ui' else name.title()
-                    sectname = config.section_name.get(filename, sectname)
-                    title = config.section_fmt(sectname)
-                    section_tag = config.helptag_fmt(sectname)
-                    # Module/Section id matched against @defgroup.
-                    #   "*api-buffer*" => "api-buffer"
-                    section_id = section_tag.strip('*')
-
-                    doc = ''
-                    section_doc = section_docs.get(section_id)
-                    if section_doc:
-                        doc += '\n\n' + section_doc
-
-                    if functions_text:
-                        doc += '\n\n' + functions_text
-
-                    if INCLUDE_DEPRECATED and deprecated_text:
-                        doc += f'\n\n\nDeprecated {sectname} Functions: ~\n\n'
-                        doc += deprecated_text
-
-                    if doc:
-                        sections[filename] = (title, section_tag, doc)
-                        fn_map_full.update(fn_map)
+            # Extract formatted (:help).
+            functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp(
+                xmlfile, target)
+
+            if not functions_text and not deprecated_text:
+                continue
+
+            filename = os.path.basename(filename)
+
+            section: Section = Section.make_from(
+                filename, config, section_docs,
+                functions_text=functions_text,
+                deprecated_functions_text=deprecated_text,
+            )
+
+            if section:  # if not empty
+                sections[filename] = section
+                fn_map_full.update(fn_map)
+            else:
+                log.debug("Skipping empty section: %s", section)
 
         if len(sections) == 0:
             fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)')
@@ -1516,21 +1608,20 @@ def main(doxygen_config, args):
             raise RuntimeError(
                 'found new modules "{}"; update the "section_order" map'.format(
                     set(sections).difference(config.section_order)))
-        first_section_tag = sections[config.section_order[0]][1]
+        first_section_tag = sections[config.section_order[0]].helptag
 
         docs = ''
 
         for filename in config.section_order:
             try:
-                title, section_tag, section_doc = sections.pop(filename)
+                section: Section = sections.pop(filename)
             except KeyError:
                 msg(f'warning: empty docs, skipping (target={target}): {filename}')
                 msg(f'    existing docs: {sections.keys()}')
                 continue
-            if filename not in config.append_only:
-                docs += sep
-                docs += '\n{}{}'.format(title, section_tag.rjust(text_width - len(title)))
-            docs += section_doc
+
+            add_sep_and_header = filename not in config.append_only
+            docs += section.render(add_header=add_sep_and_header)
             docs += '\n\n\n'
 
         docs = docs.rstrip() + '\n\n'
-- 
cgit 


From 765729a145d3d8204ff68b1da0b5bb45c70262e2 Mon Sep 17 00:00:00 2001
From: Jongwook Choi <wookayin@gmail.com>
Date: Fri, 29 Dec 2023 06:32:37 -0500
Subject: fix(gen_vimdoc): INCLUDE_DEPRECATED not generating docs for
 deprecateds

Since some point INCLUDE_DEPRECATED stopped working as it is usually
turned off when generating an actual vimdoc. This commit fixes this
hidden feature back again (used for devel purposes only).
---
 scripts/gen_vimdoc.py | 37 +++++++++++++++++++++++++------------
 1 file changed, 25 insertions(+), 12 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index e5f2e61dfd..804f0fd198 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -89,8 +89,8 @@ else:
 
 
 # DEBUG = ('DEBUG' in os.environ)
-INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ)
-INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ)
+INCLUDE_C_DECL = os.environ.get('INCLUDE_C_DECL', '0') != '0'
+INCLUDE_DEPRECATED = os.environ.get('INCLUDE_DEPRECATED', '0') != '0'
 
 log = logging.getLogger(__name__)
 
@@ -168,7 +168,7 @@ CONFIG: Dict[str, Config] = {
         mode = 'c',
         filename = 'api.txt',
         # Section ordering.
-        section_order=[
+        section_order=[x for x in [
             'vim.c',
             'vimscript.c',
             'command.c',
@@ -180,7 +180,8 @@ CONFIG: Dict[str, Config] = {
             'tabpage.c',
             'autocmd.c',
             'ui.c',
-        ],
+            'deprecated.c' if INCLUDE_DEPRECATED else ''
+        ] if x],
         files=['src/nvim/api'],
         file_patterns = '*.h *.c',
         fn_name_prefix = 'nvim_',
@@ -1287,18 +1288,21 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]:
     deprecated_fns_txt = {}  # Map of func_name:vim-help-text.
 
     fns: Dict[FunctionName, FunctionDoc]
-    fns, _ = extract_from_xml(filename, target,
-                              width=text_width, fmt_vimhelp=True)
+    deprecated_fns: Dict[FunctionName, FunctionDoc]
+    fns, deprecated_fns = extract_from_xml(
+        filename, target, width=text_width, fmt_vimhelp=True)
 
-    for fn_name, fn in fns.items():
+    def _handle_fn(fn_name: FunctionName, fn: FunctionDoc,
+                   fns_txt: Dict[FunctionName, Docstring], deprecated=False):
         # Generate Vim :help for parameters.
 
         # Generate body from FunctionDoc, not XML nodes
         doc = fn.render()
         if not doc and fn_name.startswith("nvim__"):
-            continue
+            return
         if not doc:
-            doc = 'TODO: Documentation'
+            doc = ('TODO: Documentation' if not deprecated
+                   else 'Deprecated.')
 
         # Annotations: put before Parameters
         annotations: str = '\n'.join(fn.annotations)
@@ -1356,6 +1360,11 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]:
             and fn_name != "nvim_error_event"):
             fns_txt[fn_name] = func_doc
 
+    for fn_name, fn in fns.items():
+        _handle_fn(fn_name, fn, fns_txt)
+    for fn_name, fn in deprecated_fns.items():
+        _handle_fn(fn_name, fn, deprecated_fns_txt, deprecated=True)
+
     return (
         '\n\n'.join(list(fns_txt.values())),
         '\n\n'.join(list(deprecated_fns_txt.values())),
@@ -1508,7 +1517,8 @@ class Section:
 
     def __bool__(self) -> bool:
         """Whether this section has contents. Used for skipping empty ones."""
-        return bool(self.doc or self.functions_text)
+        return bool(self.doc or self.functions_text or
+                    (INCLUDE_DEPRECATED and self.deprecated_functions_text))
 
 
 def main(doxygen_config, args):
@@ -1606,8 +1616,11 @@ def main(doxygen_config, args):
             fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)')
         if len(sections) > len(config.section_order):
             raise RuntimeError(
-                'found new modules "{}"; update the "section_order" map'.format(
-                    set(sections).difference(config.section_order)))
+                '{}: found new modules {}; '
+                'update the "section_order" map'.format(
+                    target,
+                    set(sections).difference(config.section_order))
+            )
         first_section_tag = sections[config.section_order[0]].helptag
 
         docs = ''
-- 
cgit 


From f40df63bdca33d343cada6ceaafbc8b765ed7cc6 Mon Sep 17 00:00:00 2001
From: Jongwook Choi <wookayin@gmail.com>
Date: Thu, 4 Jan 2024 11:09:13 -0500
Subject: fix(docs): make lines not overflow in vim docs

Problem: Some lines in the generated vim doc are overflowing, not
correctly wrapped at 78 characters. This happens when docs body contains
several consecutive 'inline' elements generated by doxygen.

Solution: Take into account the current column offset of the last line,
and prepend some padding before doc_wrap().
---
 scripts/gen_vimdoc.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 804f0fd198..698336cf3e 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -34,7 +34,7 @@ The generated :help text for each function is formatted as follows:
   - Each function documentation is separated by a single line.
 """
 
-from __future__ import annotations
+from __future__ import annotations  # PEP-563, python 3.7+
 
 import argparse
 import collections
@@ -47,9 +47,12 @@ import subprocess
 import sys
 import textwrap
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Literal, Tuple
+from typing import Any, Callable, Dict, List, Tuple
 from xml.dom import minidom
 
+if sys.version_info >= (3, 8):
+    from typing import Literal
+
 import msgpack
 
 Element = minidom.Element
@@ -165,7 +168,7 @@ class Config:
 
 CONFIG: Dict[str, Config] = {
     'api': Config(
-        mode = 'c',
+        mode='c',
         filename = 'api.txt',
         # Section ordering.
         section_order=[x for x in [
@@ -576,7 +579,7 @@ def is_inline(n):
     return True
 
 
-def doc_wrap(text, prefix='', width=70, func=False, indent=None):
+def doc_wrap(text, prefix='', width=70, func=False, indent=None) -> str:
     """Wraps text to `width`.
 
     First line is prefixed with `prefix`, subsequent lines are aligned.
@@ -651,13 +654,19 @@ def update_params_map(parent, ret_map, width=text_width - indentation):
     return ret_map
 
 
-def render_node(n, text, prefix='', indent='', width=text_width - indentation,
-                fmt_vimhelp=False):
+def render_node(n: Element, text: str, prefix='', *,
+                indent: str = '',
+                width: int = (text_width - indentation),
+                fmt_vimhelp: bool = False):
     """Renders a node as Vim help text, recursively traversing all descendants."""
 
     def ind(s):
         return s if fmt_vimhelp else ''
 
+    # Get the current column offset from the last line of `text`
+    # (needed to appropriately wrap multiple and contiguous inline elements)
+    col_offset: int = len_lastline(text)
+
     text = ''
     # space_preceding = (len(text) > 0 and ' ' == text[-1][-1])
     # text += (int(not space_preceding) * ' ')
@@ -682,7 +691,14 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation,
 
         text += '\n{}\n<'.format(textwrap.indent(o, ' ' * 4))
     elif is_inline(n):
-        text = doc_wrap(get_text(n), prefix=prefix, indent=indent, width=width)
+        o = get_text(n).strip()
+        if o:
+            DEL = chr(127)  # a dummy character to pad for proper line wrap
+            assert len(DEL) == 1
+            dummy_padding = DEL * max(0, col_offset - len(prefix))
+            text += doc_wrap(dummy_padding + o,
+                             prefix=prefix, indent=indent, width=width
+                            ).replace(DEL, "")
     elif n.nodeName == 'verbatim':
         # TODO: currently we don't use this. The "[verbatim]" hint is there as
         # a reminder that we must decide how to format this if we do use it.
-- 
cgit 


From 2f9ee9b6cfc61a0504fc0bc22bdf481828e2ea91 Mon Sep 17 00:00:00 2001
From: Lewis Russell <lewis6991@gmail.com>
Date: Tue, 9 Jan 2024 17:36:46 +0000
Subject: fix(doc): improve doc generation of types using lpeg

Added a lpeg grammar for LuaCATS and use it in lua2dox.lua
---
 scripts/gen_vimdoc.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 698336cf3e..01532cc3d3 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -717,12 +717,14 @@ def render_node(n: Element, text: str, prefix='', *,
     elif n.nodeName in ('para', 'heading'):
         did_prefix = False
         for c in n.childNodes:
+            c_text = render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width)
             if (is_inline(c)
-                    and '' != get_text(c).strip()
+                    and '' != c_text.strip()
                     and text
-                    and ' ' != text[-1]):
+                    and text[-1] not in (' ', '(', '|')
+                    and not c_text.startswith(')')):
                 text += ' '
-            text += render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width)
+            text += c_text
             did_prefix = True
     elif n.nodeName == 'itemizedlist':
         for c in n.childNodes:
@@ -840,15 +842,17 @@ def para_as_map(parent: Element,
                     raise RuntimeError('unhandled simplesect: {}\n{}'.format(
                         child.nodeName, child.toprettyxml(indent='  ', newl='\n')))
             else:
+                child_text = render_node(child, text, indent=indent, width=width)
                 if (prev is not None
                         and is_inline(self_or_child(prev))
                         and is_inline(self_or_child(child))
                         and '' != get_text(self_or_child(child)).strip()
                         and text
-                        and ' ' != text[-1]):
+                        and text[-1] not in (' ', '(', '|')
+                        and not child_text.startswith(')')):
                     text += ' '
 
-                text += render_node(child, text, indent=indent, width=width)
+                text += child_text
                 prev = child
 
     chunks['text'] += text
-- 
cgit 


From 2cdea852e8934beb89012f2127f333e4dd8aada8 Mon Sep 17 00:00:00 2001
From: Jongwook Choi <wookayin@gmail.com>
Date: Thu, 11 Jan 2024 12:24:44 -0500
Subject: docs: auto-generate docs for `vim.lpeg` and `vim.re`

- Add section `VIM.LPEG` and `VIM.RE` to docs/lua.txt.

- Add `_meta/re.lua` which adds luadoc and type annotations, for the
  vendored `vim.re` package.

- Fix minor style issues on `_meta/lpeg.lua` luadoc for better vimdocs
  generation.

- Fix a bug on `gen_vimdoc` where non-helptags in verbatim code blocks
  were parsed as helptags, affecting code examples on `vim.lpeg.Cf`,
  etc.

- Also move the `vim.regex` section below so that it can be located
  closer to `vim.lpeg` and `vim.re`.
---
 scripts/gen_vimdoc.py | 52 +++++++++++++++++++++++++--------------------------
 1 file changed, 25 insertions(+), 27 deletions(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 01532cc3d3..4cb90a4588 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -202,7 +202,6 @@ CONFIG: Dict[str, Config] = {
         filename='lua.txt',
         section_order=[
             'highlight.lua',
-            'regex.lua',
             'diff.lua',
             'mpack.lua',
             'json.lua',
@@ -220,6 +219,9 @@ CONFIG: Dict[str, Config] = {
             'keymap.lua',
             'fs.lua',
             'glob.lua',
+            'lpeg.lua',
+            're.lua',
+            'regex.lua',
             'secure.lua',
             'version.lua',
             'iter.lua',
@@ -250,6 +252,8 @@ CONFIG: Dict[str, Config] = {
             'runtime/lua/vim/_meta/json.lua',
             'runtime/lua/vim/_meta/base64.lua',
             'runtime/lua/vim/_meta/regex.lua',
+            'runtime/lua/vim/_meta/lpeg.lua',
+            'runtime/lua/vim/_meta/re.lua',
             'runtime/lua/vim/_meta/spell.lua',
         ],
         file_patterns='*.lua',
@@ -268,7 +272,10 @@ CONFIG: Dict[str, Config] = {
         section_fmt=lambda name: (
             'Lua module: vim' if name.lower() == '_editor' else
             'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else
-            f'VIM.{name.upper()}' if name.lower() in [ 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', 'regex' ] else
+            f'VIM.{name.upper()}' if name.lower() in [
+                'highlight', 'mpack', 'json', 'base64', 'diff', 'spell',
+                'regex', 'lpeg', 're',
+            ] else
             'VIM' if name.lower() == 'builtin' else
             f'Lua module: vim.{name.lower()}'),
         helptag_fmt=lambda name: (
@@ -305,6 +312,8 @@ CONFIG: Dict[str, Config] = {
             'json': 'vim.json',
             'base64': 'vim.base64',
             'regex': 'vim.regex',
+            'lpeg': 'vim.lpeg',
+            're': 'vim.re',
             'spell': 'vim.spell',
             'snippet': 'vim.snippet',
             'text': 'vim.text',
@@ -1350,31 +1359,20 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]:
         # Verbatim handling.
         func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M)
 
-        split_lines: List[str] = func_doc.split('\n')
-        start = 0
-        while True:
-            try:
-                start = split_lines.index('>', start)
-            except ValueError:
-                break
-
-            try:
-                end = split_lines.index('<', start)
-            except ValueError:
-                break
-
-            split_lines[start + 1:end] = [
-                ('    ' + x).rstrip()
-                for x in textwrap.dedent(
-                    "\n".join(
-                        split_lines[start+1:end]
-                    )
-                ).split("\n")
-            ]
-
-            start = end
-
-        func_doc = "\n".join(map(align_tags, split_lines))
+        def process_helptags(func_doc: str) -> str:
+            lines: List[str] = func_doc.split('\n')
+            # skip ">lang ... <" regions
+            is_verbatim: bool = False
+            for i in range(len(lines)):
+                if re.search(' >([a-z])*$', lines[i]):
+                    is_verbatim = True
+                elif is_verbatim and lines[i].strip() == '<':
+                    is_verbatim = False
+                if not is_verbatim:
+                    lines[i] = align_tags(lines[i])
+            return "\n".join(lines)
+
+        func_doc = process_helptags(func_doc)
 
         if (fn_name.startswith(config.fn_name_prefix)
             and fn_name != "nvim_error_event"):
-- 
cgit 


From f9d81c43d2296d212c9cebcbdce401cd76cf0f1f Mon Sep 17 00:00:00 2001
From: bfredl <bjorn.linse@gmail.com>
Date: Wed, 31 Jan 2024 22:02:06 +0100
Subject: refactor(api): use keydict and arena for more api return values

Implement api_keydict_to_dict as the complement to api_dict_to_keydict

Fix a conversion error when nvim_get_win_config gets called from lua,
where Float values "x" and "y" didn't get converted to lua numbers.
---
 scripts/gen_vimdoc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 4cb90a4588..c1a2183f24 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -1688,7 +1688,7 @@ def filter_source(filename, keep_tmpfiles):
     else:
         """Filters the source to fix macros that confuse Doxygen."""
         with open(filename, 'rt') as fp:
-            print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))',
+            print(re.sub(r'^(ArrayOf|DictionaryOf|Dict)(\(.*?\))',
                          lambda m: m.group(1)+'_'.join(
                              re.split(r'[^\w]+', m.group(2))),
                          fp.read(), flags=re.M))
-- 
cgit 


From 9beb40a4db5613601fc1a4b828a44e5977eca046 Mon Sep 17 00:00:00 2001
From: Lewis Russell <lewis6991@gmail.com>
Date: Thu, 15 Feb 2024 17:16:04 +0000
Subject: feat(docs): replace lua2dox.lua

Problem:

The documentation flow (`gen_vimdoc.py`) has several issues:
- it's not very versatile
- depends on doxygen
- doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C.
- The intermediate XML files and filters makes it too much like a rube goldberg machine.

Solution:

Re-implement the flow using Lua, LPEG and treesitter.

- `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic.
- `lua2dox.lua` is gone!
- No more XML files.
- Doxygen is now longer used and instead we now use:
  - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`).
  - LPEG for C parsing (see `scripts/cdoc_parser.lua`)
  - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`).
  - Treesitter for Markdown parsing (see `scripts/text_utils.lua`).
- The generated `runtime/doc/*.mpack` files have been removed.
   - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly.
- Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change).
---
 scripts/gen_vimdoc.py | 1766 -------------------------------------------------
 1 file changed, 1766 deletions(-)
 delete mode 100755 scripts/gen_vimdoc.py

(limited to 'scripts/gen_vimdoc.py')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
deleted file mode 100755
index c1a2183f24..0000000000
--- a/scripts/gen_vimdoc.py
+++ /dev/null
@@ -1,1766 +0,0 @@
-#!/usr/bin/env python3
-
-r"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen.
-
-Also generates *.mpack files. To inspect the *.mpack structure:
-    :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B')))
-
-Flow:
-    main
-      extract_from_xml
-        fmt_node_as_vimhelp   \
-          para_as_map          } recursive
-            update_params_map /
-              render_node
-
-TODO: eliminate this script and use Lua+treesitter (requires parsers for C and
-Lua markdown-style docstrings).
-
-The generated :help text for each function is formatted as follows:
-
-  - Max width of 78 columns (`text_width`).
-  - Indent with spaces (not tabs).
-  - Indent of 4 columns for body text (`indentation`).
-  - Function signature and helptag (right-aligned) on the same line.
-    - Signature and helptag must have a minimum of 8 spaces between them.
-    - If the signature is too long, it is placed on the line after the helptag.
-      Signature wraps at `text_width - 8` characters with subsequent
-      lines indented to the open parenthesis.
-    - Subsection bodies are indented an additional 4 spaces.
-  - Body consists of function description, parameters, return description, and
-    C declaration (`INCLUDE_C_DECL`).
-  - Parameters are omitted for the `void` and `Error *` types, or if the
-    parameter is marked as [out].
-  - Each function documentation is separated by a single line.
-"""
-
-from __future__ import annotations  # PEP-563, python 3.7+
-
-import argparse
-import collections
-import dataclasses
-import logging
-import os
-import re
-import shutil
-import subprocess
-import sys
-import textwrap
-from pathlib import Path
-from typing import Any, Callable, Dict, List, Tuple
-from xml.dom import minidom
-
-if sys.version_info >= (3, 8):
-    from typing import Literal
-
-import msgpack
-
-Element = minidom.Element
-Document = minidom.Document
-
-MIN_PYTHON_VERSION = (3, 7)
-MIN_DOXYGEN_VERSION = (1, 9, 0)
-
-if sys.version_info < MIN_PYTHON_VERSION:
-    print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION))
-    sys.exit(1)
-
-doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"],
-                        universal_newlines=True).split()[0].split('.')))
-
-if doxygen_version < MIN_DOXYGEN_VERSION:
-    print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION))
-    print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version))
-    sys.exit(1)
-
-
-# Need a `nvim` that supports `-l`, try the local build
-nvim_path = Path(__file__).parent / "../build/bin/nvim"
-if nvim_path.exists():
-    nvim = nvim_path.resolve()
-else:
-    # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works.
-    nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True)
-    nvim_version = [line for line in nvim_out.split('\n')
-                    if '-l ' in line]
-    if len(nvim_version) == 0:
-        print((
-            "\nYou need to have a local Neovim build or a `nvim` version 0.9 for `-l` "
-            "support to build the documentation."))
-        sys.exit(1)
-    nvim = 'nvim'
-
-
-# DEBUG = ('DEBUG' in os.environ)
-INCLUDE_C_DECL = os.environ.get('INCLUDE_C_DECL', '0') != '0'
-INCLUDE_DEPRECATED = os.environ.get('INCLUDE_DEPRECATED', '0') != '0'
-
-log = logging.getLogger(__name__)
-
-LOG_LEVELS = {
-    logging.getLevelName(level): level for level in [
-        logging.DEBUG, logging.INFO, logging.ERROR
-    ]
-}
-
-text_width = 78
-indentation = 4
-SECTION_SEP = '=' * text_width
-
-script_path = os.path.abspath(__file__)
-base_dir = os.path.dirname(os.path.dirname(script_path))
-out_dir = os.path.join(base_dir, 'tmp-{target}-doc')
-filter_cmd = '%s %s' % (sys.executable, script_path)
-msgs = []  # Messages to show on exit.
-lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua')
-
-
-SectionName = str
-
-Docstring = str  # Represents (formatted) vimdoc string
-
-FunctionName = str
-
-
-@dataclasses.dataclass
-class Config:
-    """Config for documentation."""
-
-    mode: Literal['c', 'lua']
-
-    filename: str
-    """Generated documentation target, e.g. api.txt"""
-
-    section_order: List[str]
-    """Section ordering."""
-
-    files: List[str]
-    """List of files/directories for doxygen to read, relative to `base_dir`."""
-
-    file_patterns: str
-    """file patterns used by doxygen."""
-
-    section_name: Dict[str, SectionName]
-    """Section name overrides. Key: filename (e.g., vim.c)"""
-
-    section_fmt: Callable[[SectionName], str]
-    """For generated section names."""
-
-    helptag_fmt: Callable[[SectionName], str]
-    """Section helptag."""
-
-    fn_helptag_fmt: Callable[[str, str, bool], str]
-    """Per-function helptag."""
-
-    module_override: Dict[str, str]
-    """Module name overrides (for Lua)."""
-
-    append_only: List[str]
-    """Append the docs for these modules, do not start a new section."""
-
-    fn_name_prefix: str
-    """Only function with this prefix are considered"""
-
-    fn_name_fmt: Callable[[str, str], str] | None = None
-
-    include_tables: bool = True
-
-
-CONFIG: Dict[str, Config] = {
-    'api': Config(
-        mode='c',
-        filename = 'api.txt',
-        # Section ordering.
-        section_order=[x for x in [
-            'vim.c',
-            'vimscript.c',
-            'command.c',
-            'options.c',
-            'buffer.c',
-            'extmark.c',
-            'window.c',
-            'win_config.c',
-            'tabpage.c',
-            'autocmd.c',
-            'ui.c',
-            'deprecated.c' if INCLUDE_DEPRECATED else ''
-        ] if x],
-        files=['src/nvim/api'],
-        file_patterns = '*.h *.c',
-        fn_name_prefix = 'nvim_',
-        section_name={
-            'vim.c': 'Global',
-        },
-        section_fmt=lambda name: f'{name} Functions',
-        helptag_fmt=lambda name: f'*api-{name.lower()}*',
-        fn_helptag_fmt=lambda fstem, name, istbl: f'*{name}()*',
-        module_override={},
-        append_only=[],
-    ),
-    'lua': Config(
-        mode='lua',
-        filename='lua.txt',
-        section_order=[
-            'highlight.lua',
-            'diff.lua',
-            'mpack.lua',
-            'json.lua',
-            'base64.lua',
-            'spell.lua',
-            'builtin.lua',
-            '_options.lua',
-            '_editor.lua',
-            '_inspector.lua',
-            'shared.lua',
-            'loader.lua',
-            'uri.lua',
-            'ui.lua',
-            'filetype.lua',
-            'keymap.lua',
-            'fs.lua',
-            'glob.lua',
-            'lpeg.lua',
-            're.lua',
-            'regex.lua',
-            'secure.lua',
-            'version.lua',
-            'iter.lua',
-            'snippet.lua',
-            'text.lua',
-        ],
-        files=[
-            'runtime/lua/vim/iter.lua',
-            'runtime/lua/vim/_editor.lua',
-            'runtime/lua/vim/_options.lua',
-            'runtime/lua/vim/shared.lua',
-            'runtime/lua/vim/loader.lua',
-            'runtime/lua/vim/uri.lua',
-            'runtime/lua/vim/ui.lua',
-            'runtime/lua/vim/filetype.lua',
-            'runtime/lua/vim/keymap.lua',
-            'runtime/lua/vim/fs.lua',
-            'runtime/lua/vim/highlight.lua',
-            'runtime/lua/vim/secure.lua',
-            'runtime/lua/vim/version.lua',
-            'runtime/lua/vim/_inspector.lua',
-            'runtime/lua/vim/snippet.lua',
-            'runtime/lua/vim/text.lua',
-            'runtime/lua/vim/glob.lua',
-            'runtime/lua/vim/_meta/builtin.lua',
-            'runtime/lua/vim/_meta/diff.lua',
-            'runtime/lua/vim/_meta/mpack.lua',
-            'runtime/lua/vim/_meta/json.lua',
-            'runtime/lua/vim/_meta/base64.lua',
-            'runtime/lua/vim/_meta/regex.lua',
-            'runtime/lua/vim/_meta/lpeg.lua',
-            'runtime/lua/vim/_meta/re.lua',
-            'runtime/lua/vim/_meta/spell.lua',
-        ],
-        file_patterns='*.lua',
-        fn_name_prefix='',
-        fn_name_fmt=lambda fstem, name: (
-            name if fstem in [ 'vim.iter' ] else
-            f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else
-            f'vim.{name}' if fstem == '_options' and not name[0].isupper() else
-            f'{fstem}.{name}' if fstem.startswith('vim') else
-            name
-        ),
-        section_name={
-            'lsp.lua': 'core',
-            '_inspector.lua': 'inspector',
-        },
-        section_fmt=lambda name: (
-            'Lua module: vim' if name.lower() == '_editor' else
-            'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else
-            f'VIM.{name.upper()}' if name.lower() in [
-                'highlight', 'mpack', 'json', 'base64', 'diff', 'spell',
-                'regex', 'lpeg', 're',
-            ] else
-            'VIM' if name.lower() == 'builtin' else
-            f'Lua module: vim.{name.lower()}'),
-        helptag_fmt=lambda name: (
-            '*lua-vim*' if name.lower() == '_editor' else
-            '*lua-vimscript*' if name.lower() == '_options' else
-            f'*vim.{name.lower()}*'),
-        fn_helptag_fmt=lambda fstem, name, istbl: (
-            f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else
-            # Exclude fstem for methods
-            f'*{name}()*' if ':' in name else
-            f'*vim.{name}()*' if fstem.lower() == '_editor' else
-            f'*vim.{name}*' if fstem.lower() == '_options' and istbl else
-            # Prevents vim.regex.regex
-            f'*{fstem}()*' if fstem.endswith('.' + name) else
-            f'*{fstem}.{name}{"" if istbl else "()"}*'
-            ),
-        module_override={
-            # `shared` functions are exposed on the `vim` module.
-            'shared': 'vim',
-            '_inspector': 'vim',
-            'uri': 'vim',
-            'ui': 'vim.ui',
-            'loader': 'vim.loader',
-            'filetype': 'vim.filetype',
-            'keymap': 'vim.keymap',
-            'fs': 'vim.fs',
-            'highlight': 'vim.highlight',
-            'secure': 'vim.secure',
-            'version': 'vim.version',
-            'iter': 'vim.iter',
-            'diff': 'vim',
-            'builtin': 'vim',
-            'mpack': 'vim.mpack',
-            'json': 'vim.json',
-            'base64': 'vim.base64',
-            'regex': 'vim.regex',
-            'lpeg': 'vim.lpeg',
-            're': 'vim.re',
-            'spell': 'vim.spell',
-            'snippet': 'vim.snippet',
-            'text': 'vim.text',
-            'glob': 'vim.glob',
-        },
-        append_only=[
-            'shared.lua',
-        ],
-    ),
-    'lsp': Config(
-        mode='lua',
-        filename='lsp.txt',
-        section_order=[
-            'lsp.lua',
-            'buf.lua',
-            'diagnostic.lua',
-            'codelens.lua',
-            'inlay_hint.lua',
-            'tagfunc.lua',
-            'semantic_tokens.lua',
-            'handlers.lua',
-            'util.lua',
-            'log.lua',
-            'rpc.lua',
-            'protocol.lua',
-        ],
-        files=[
-            'runtime/lua/vim/lsp',
-            'runtime/lua/vim/lsp.lua',
-        ],
-        file_patterns='*.lua',
-        fn_name_prefix='',
-        section_name={'lsp.lua': 'lsp'},
-        section_fmt=lambda name: (
-            'Lua module: vim.lsp'
-            if name.lower() == 'lsp'
-            else f'Lua module: vim.lsp.{name.lower()}'),
-        helptag_fmt=lambda name: (
-            '*lsp-core*'
-            if name.lower() == 'lsp'
-            else f'*lsp-{name.lower()}*'),
-        fn_helptag_fmt=lambda fstem, name, istbl: (
-            f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else
-            # HACK. TODO(justinmk): class/structure support in lua2dox
-            '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else
-            f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'),
-        module_override={},
-        append_only=[],
-    ),
-    'diagnostic': Config(
-        mode='lua',
-        filename='diagnostic.txt',
-        section_order=[
-            'diagnostic.lua',
-        ],
-        files=['runtime/lua/vim/diagnostic.lua'],
-        file_patterns='*.lua',
-        fn_name_prefix='',
-        include_tables=False,
-        section_name={'diagnostic.lua': 'diagnostic'},
-        section_fmt=lambda _: 'Lua module: vim.diagnostic',
-        helptag_fmt=lambda _: '*diagnostic-api*',
-        fn_helptag_fmt=lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*',
-        module_override={},
-        append_only=[],
-    ),
-    'treesitter': Config(
-        mode='lua',
-        filename='treesitter.txt',
-        section_order=[
-            'treesitter.lua',
-            'language.lua',
-            'query.lua',
-            'highlighter.lua',
-            'languagetree.lua',
-            'dev.lua',
-        ],
-        files=[
-            'runtime/lua/vim/treesitter.lua',
-            'runtime/lua/vim/treesitter/',
-        ],
-        file_patterns='*.lua',
-        fn_name_prefix='',
-        section_name={},
-        section_fmt=lambda name: (
-            'Lua module: vim.treesitter'
-            if name.lower() == 'treesitter'
-            else f'Lua module: vim.treesitter.{name.lower()}'),
-        helptag_fmt=lambda name: (
-            '*lua-treesitter-core*'
-            if name.lower() == 'treesitter'
-            else f'*lua-treesitter-{name.lower()}*'),
-        fn_helptag_fmt=lambda fstem, name, istbl: (
-            f'*vim.{fstem}.{name}()*'
-            if fstem == 'treesitter'
-            else f'*{name}()*'
-            if name[0].isupper()
-            else f'*vim.treesitter.{fstem}.{name}()*'),
-        module_override={},
-        append_only=[],
-    ),
-}
-
-param_exclude = (
-    'channel_id',
-)
-
-# Annotations are displayed as line items after API function descriptions.
-annotation_map = {
-    'FUNC_API_FAST': '|api-fast|',
-    'FUNC_API_TEXTLOCK': 'not allowed when |textlock| is active or in the |cmdwin|',
-    'FUNC_API_TEXTLOCK_ALLOW_CMDWIN': 'not allowed when |textlock| is active',
-    'FUNC_API_REMOTE_ONLY': '|RPC| only',
-    'FUNC_API_LUA_ONLY': 'Lua |vim.api| only',
-}
-
-
-def nvim_api_info() -> Tuple[int, bool]:
-    """Returns NVIM_API_LEVEL, NVIM_API_PRERELEASE from CMakeLists.txt"""
-    if not hasattr(nvim_api_info, 'LEVEL'):
-        script_dir = os.path.dirname(os.path.abspath(__file__))
-        cmake_file_path = os.path.join(script_dir, '..', 'CMakeLists.txt')
-        with open(cmake_file_path, 'r') as cmake_file:
-            cmake_content = cmake_file.read()
-
-        api_level_match = re.search(r'set\(NVIM_API_LEVEL (\d+)\)', cmake_content)
-        api_prerelease_match = re.search(
-            r'set\(NVIM_API_PRERELEASE (\w+)\)', cmake_content
-        )
-
-        if not api_level_match or not api_prerelease_match:
-            raise RuntimeError(
-                'Could not find NVIM_API_LEVEL or NVIM_API_PRERELEASE in CMakeLists.txt'
-            )
-
-        nvim_api_info.LEVEL = int(api_level_match.group(1))
-        nvim_api_info.PRERELEASE = api_prerelease_match.group(1).lower() == 'true'
-
-    return nvim_api_info.LEVEL, nvim_api_info.PRERELEASE
-
-
-# Raises an error with details about `o`, if `cond` is in object `o`,
-# or if `cond()` is callable and returns True.
-def debug_this(o, cond=True):
-    name = ''
-    if cond is False:
-        return
-    if not isinstance(o, str):
-        try:
-            name = o.nodeName
-            o = o.toprettyxml(indent='  ', newl='\n')
-        except Exception:
-            pass
-    if (cond is True
-            or (callable(cond) and cond())
-            or (not callable(cond) and cond in o)):
-        raise RuntimeError('xxx: {}\n{}'.format(name, o))
-
-
-# Appends a message to a list which will be printed on exit.
-def msg(s):
-    msgs.append(s)
-
-
-# Print all collected messages.
-def msg_report():
-    for m in msgs:
-        print(f'    {m}')
-
-
-# Print collected messages, then throw an exception.
-def fail(s):
-    msg_report()
-    raise RuntimeError(s)
-
-
-def find_first(parent, name):
-    """Finds the first matching node within parent."""
-    sub = parent.getElementsByTagName(name)
-    if not sub:
-        return None
-    return sub[0]
-
-
-def iter_children(parent, name):
-    """Yields matching child nodes within parent."""
-    for child in parent.childNodes:
-        if child.nodeType == child.ELEMENT_NODE and child.nodeName == name:
-            yield child
-
-
-def get_child(parent, name):
-    """Gets the first matching child node."""
-    for child in iter_children(parent, name):
-        return child
-    return None
-
-
-def self_or_child(n):
-    """Gets the first child node, or self."""
-    if len(n.childNodes) == 0:
-        return n
-    return n.childNodes[0]
-
-
-def align_tags(line):
-    tag_regex = r"\s(\*.+?\*)(?:\s|$)"
-    tags = re.findall(tag_regex, line)
-
-    if len(tags) > 0:
-        line = re.sub(tag_regex, "", line)
-        tags = " " + " ".join(tags)
-        line = line + (" " * (78 - len(line) - len(tags))) + tags
-    return line
-
-
-def clean_lines(text):
-    """Removes superfluous lines.
-
-    The beginning and end of the string is trimmed.  Empty lines are collapsed.
-    """
-    return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text))
-
-
-def is_blank(text):
-    return '' == clean_lines(text)
-
-
-def get_text(n):
-    """Recursively concatenates all text in a node tree."""
-    text = ''
-    if n.nodeType == n.TEXT_NODE:
-        return n.data
-    if n.nodeName == 'computeroutput':
-        for node in n.childNodes:
-            text += get_text(node)
-        return '`{}`'.format(text)
-    if n.nodeName == 'sp': # space, used in "programlisting" nodes
-        return ' '
-    for node in n.childNodes:
-        if node.nodeType == node.TEXT_NODE:
-            text += node.data
-        elif node.nodeType == node.ELEMENT_NODE:
-            text += get_text(node)
-    return text
-
-
-# Gets the length of the last line in `text`, excluding newline ("\n") char.
-def len_lastline(text):
-    lastnl = text.rfind('\n')
-    if -1 == lastnl:
-        return len(text)
-    if '\n' == text[-1]:
-        return lastnl - (1 + text.rfind('\n', 0, lastnl))
-    return len(text) - (1 + lastnl)
-
-
-def len_lastline_withoutindent(text, indent):
-    n = len_lastline(text)
-    return (n - len(indent)) if n > len(indent) else 0
-
-
-# Returns True if node `n` contains only inline (not block-level) elements.
-def is_inline(n):
-    # if len(n.childNodes) == 0:
-    #     return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput'
-    for c in n.childNodes:
-        if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput':
-            return False
-        if not is_inline(c):
-            return False
-    return True
-
-
-def doc_wrap(text, prefix='', width=70, func=False, indent=None) -> str:
-    """Wraps text to `width`.
-
-    First line is prefixed with `prefix`, subsequent lines are aligned.
-    If `func` is True, only wrap at commas.
-    """
-    if not width:
-        # return prefix + text
-        return text
-
-    # Whitespace used to indent all lines except the first line.
-    indent = ' ' * len(prefix) if indent is None else indent
-    indent_only = (prefix == '' and indent is not None)
-
-    if func:
-        lines = [prefix]
-        for part in text.split(', '):
-            if part[-1] not in ');':
-                part += ', '
-            if len(lines[-1]) + len(part) > width:
-                lines.append(indent)
-            lines[-1] += part
-        return '\n'.join(x.rstrip() for x in lines).rstrip()
-
-    # XXX: Dummy prefix to force TextWrapper() to wrap the first line.
-    if indent_only:
-        prefix = indent
-
-    tw = textwrap.TextWrapper(break_long_words=False,
-                              break_on_hyphens=False,
-                              width=width,
-                              initial_indent=prefix,
-                              subsequent_indent=indent)
-    result = '\n'.join(tw.wrap(text.strip()))
-
-    # XXX: Remove the dummy prefix.
-    if indent_only:
-        result = result[len(indent):]
-
-    return result
-
-
-def max_name(names):
-    if len(names) == 0:
-        return 0
-    return max(len(name) for name in names)
-
-
-def update_params_map(parent, ret_map, width=text_width - indentation):
-    """Updates `ret_map` with name:desc key-value pairs extracted
-    from Doxygen XML node `parent`.
-    """
-    params = collections.OrderedDict()
-    for node in parent.childNodes:
-        if node.nodeType == node.TEXT_NODE:
-            continue
-        name_node = find_first(node, 'parametername')
-        if name_node.getAttribute('direction') == 'out':
-            continue
-        name = get_text(name_node)
-        if name in param_exclude:
-            continue
-        params[name.strip()] = node
-    max_name_len = max_name(params.keys()) + 8
-    # `ret_map` is a name:desc map.
-    for name, node in params.items():
-        desc = ''
-        desc_node = get_child(node, 'parameterdescription')
-        if desc_node:
-            desc = fmt_node_as_vimhelp(
-                    desc_node, width=width, indent=(' ' * max_name_len))
-            ret_map[name] = desc
-    return ret_map
-
-
-def render_node(n: Element, text: str, prefix='', *,
-                indent: str = '',
-                width: int = (text_width - indentation),
-                fmt_vimhelp: bool = False):
-    """Renders a node as Vim help text, recursively traversing all descendants."""
-
-    def ind(s):
-        return s if fmt_vimhelp else ''
-
-    # Get the current column offset from the last line of `text`
-    # (needed to appropriately wrap multiple and contiguous inline elements)
-    col_offset: int = len_lastline(text)
-
-    text = ''
-    # space_preceding = (len(text) > 0 and ' ' == text[-1][-1])
-    # text += (int(not space_preceding) * ' ')
-
-    if n.nodeName == 'preformatted':
-        o = get_text(n)
-        ensure_nl = '' if o[-1] == '\n' else '\n'
-        if o[0:4] == 'lua\n':
-            text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1])
-        elif o[0:4] == 'vim\n':
-            text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1])
-        elif o[0:5] == 'help\n':
-            text += o[4:-1]
-        else:
-            text += '>{}{}\n<'.format(ensure_nl, o)
-    elif n.nodeName == 'programlisting': # codeblock (```)
-        o = get_text(n)
-        text += '>'
-        if 'filename' in n.attributes:
-            filename = n.attributes['filename'].value
-            text += filename.lstrip('.')
-
-        text += '\n{}\n<'.format(textwrap.indent(o, ' ' * 4))
-    elif is_inline(n):
-        o = get_text(n).strip()
-        if o:
-            DEL = chr(127)  # a dummy character to pad for proper line wrap
-            assert len(DEL) == 1
-            dummy_padding = DEL * max(0, col_offset - len(prefix))
-            text += doc_wrap(dummy_padding + o,
-                             prefix=prefix, indent=indent, width=width
-                            ).replace(DEL, "")
-    elif n.nodeName == 'verbatim':
-        # TODO: currently we don't use this. The "[verbatim]" hint is there as
-        # a reminder that we must decide how to format this if we do use it.
-        text += ' [verbatim] {}'.format(get_text(n))
-    elif n.nodeName == 'listitem':
-        for c in n.childNodes:
-            result = render_node(
-                c,
-                text,
-                indent=indent + (' ' * len(prefix)),
-                width=width
-            )
-            if is_blank(result):
-                continue
-            text += indent + prefix + result
-    elif n.nodeName in ('para', 'heading'):
-        did_prefix = False
-        for c in n.childNodes:
-            c_text = render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width)
-            if (is_inline(c)
-                    and '' != c_text.strip()
-                    and text
-                    and text[-1] not in (' ', '(', '|')
-                    and not c_text.startswith(')')):
-                text += ' '
-            text += c_text
-            did_prefix = True
-    elif n.nodeName == 'itemizedlist':
-        for c in n.childNodes:
-            text += '{}\n'.format(render_node(c, text, prefix='• ',
-                                              indent=indent, width=width))
-    elif n.nodeName == 'orderedlist':
-        i = 1
-        for c in n.childNodes:
-            if is_blank(get_text(c)):
-                text += '\n'
-                continue
-            text += '{}\n'.format(render_node(c, text, prefix='{}. '.format(i),
-                                              indent=indent, width=width))
-            i = i + 1
-    elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'):
-        text += ind('  ')
-        for c in n.childNodes:
-            if is_blank(render_node(c, text, prefix='• ', indent='    ', width=width)):
-                continue
-            text += render_node(c, text, prefix='• ', indent='    ', width=width)
-        # text += '\n'
-    elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'):
-        text += 'Warning:\n    '
-        for c in n.childNodes:
-            text += render_node(c, text, indent='    ', width=width)
-        text += '\n'
-    elif n.nodeName == 'simplesect' and 'see' == n.getAttribute('kind'):
-        text += ind('  ')
-        # Example:
-        #   <simplesect kind="see">
-        #     <para>|autocommand|</para>
-        #   </simplesect>
-        for c in n.childNodes:
-            text += render_node(c, text, prefix='• ', indent='    ', width=width)
-    elif n.nodeName == 'simplesect' and 'return' == n.getAttribute('kind'):
-        text += ind('    ')
-        for c in n.childNodes:
-            text += render_node(c, text, indent='    ', width=width)
-    elif n.nodeName == 'computeroutput':
-        return get_text(n)
-    else:
-        raise RuntimeError('unhandled node type: {}\n{}'.format(
-            n.nodeName, n.toprettyxml(indent='  ', newl='\n')))
-
-    return text
-
-
-def para_as_map(parent: Element,
-                indent: str = '',
-                width: int = (text_width - indentation),
-                ):
-    """Extracts a Doxygen XML <para> node to a map.
-
-    Keys:
-        'text': Text from this <para> element
-        'note': List of @note strings
-        'params': <parameterlist> map
-        'return': List of @return strings
-        'seealso': List of @see strings
-        'xrefs': ?
-    """
-    chunks = {
-        'text': '',
-        'note': [],
-        'params': collections.OrderedDict(),
-        'return': [],
-        'seealso': [],
-        'prerelease': False,
-        'xrefs': []
-    }
-
-    # Ordered dict of ordered lists.
-    groups = collections.OrderedDict([
-        ('note', []),
-        ('params', []),
-        ('return', []),
-        ('seealso', []),
-        ('xrefs', []),
-    ])
-
-    # Gather nodes into groups.  Mostly this is because we want "parameterlist"
-    # nodes to appear together.
-    text = ''
-    kind = ''
-    if is_inline(parent):
-        # Flatten inline text from a tree of non-block nodes.
-        text = doc_wrap(render_node(parent, ""),
-                        indent=indent, width=width)
-    else:
-        prev = None  # Previous node
-        for child in parent.childNodes:
-            if child.nodeName == 'parameterlist':
-                groups['params'].append(child)
-            elif child.nodeName == 'xrefsect':
-                groups['xrefs'].append(child)
-            elif child.nodeName == 'simplesect':
-                kind = child.getAttribute('kind')
-                if kind == 'note':
-                    groups['note'].append(child)
-                elif kind == 'return':
-                    groups['return'].append(child)
-                elif kind == 'see':
-                    groups['seealso'].append(child)
-                elif kind == 'warning':
-                    text += render_node(child, text, indent=indent, width=width)
-                elif kind == 'since':
-                    since_match = re.match(r'^(\d+)', get_text(child))
-                    since = int(since_match.group(1)) if since_match else 0
-                    NVIM_API_LEVEL, NVIM_API_PRERELEASE = nvim_api_info()
-                    if since > NVIM_API_LEVEL or (
-                        since == NVIM_API_LEVEL and NVIM_API_PRERELEASE
-                    ):
-                        chunks['prerelease'] = True
-                else:
-                    raise RuntimeError('unhandled simplesect: {}\n{}'.format(
-                        child.nodeName, child.toprettyxml(indent='  ', newl='\n')))
-            else:
-                child_text = render_node(child, text, indent=indent, width=width)
-                if (prev is not None
-                        and is_inline(self_or_child(prev))
-                        and is_inline(self_or_child(child))
-                        and '' != get_text(self_or_child(child)).strip()
-                        and text
-                        and text[-1] not in (' ', '(', '|')
-                        and not child_text.startswith(')')):
-                    text += ' '
-
-                text += child_text
-                prev = child
-
-    chunks['text'] += text
-
-    # Generate map from the gathered items.
-    if len(groups['params']) > 0:
-        for child in groups['params']:
-            update_params_map(child, ret_map=chunks['params'], width=width)
-    for child in groups['note']:
-        chunks['note'].append(render_node(
-            child, '', indent=indent, width=width).rstrip())
-    for child in groups['return']:
-        chunks['return'].append(render_node(
-            child, '', indent=indent, width=width))
-    for child in groups['seealso']:
-        # Example:
-        #   <simplesect kind="see">
-        #     <para>|autocommand|</para>
-        #   </simplesect>
-        chunks['seealso'].append(render_node(
-            child, '', indent=indent, width=width))
-
-    xrefs = set()
-    for child in groups['xrefs']:
-        # XXX: Add a space (or any char) to `title` here, otherwise xrefs
-        # ("Deprecated" section) acts very weird...
-        title = get_text(get_child(child, 'xreftitle')) + ' '
-        xrefs.add(title)
-        xrefdesc = get_text(get_child(child, 'xrefdescription'))
-        chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title),
-                                        width=width) + '\n')
-
-    return chunks, xrefs
-
-
-def is_program_listing(para):
-    """
-    Return True if `para` contains a "programlisting" (i.e. a Markdown code
-    block ```).
-
-    Sometimes a <para> element will have only a single "programlisting" child
-    node, but othertimes it will have extra whitespace around the
-    "programlisting" node.
-
-    @param para XML <para> node
-    @return True if <para> is a programlisting
-    """
-
-    # Remove any child text nodes that are only whitespace
-    children = [
-        n for n in para.childNodes
-        if n.nodeType != n.TEXT_NODE or n.data.strip() != ''
-    ]
-
-    return len(children) == 1 and children[0].nodeName == 'programlisting'
-
-
-FunctionParam = Tuple[
-    str,  # type
-    str,  # parameter name
-]
-
-@dataclasses.dataclass
-class FunctionDoc:
-    """Data structure for function documentation. Also exported as msgpack."""
-
-    annotations: List[str]
-    """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map"""
-
-    notes: List[Docstring]
-    """Notes: (@note strings)"""
-
-    signature: str
-    """Function signature with *tags*."""
-
-    parameters: List[FunctionParam]
-    """Parameters: (type, name)"""
-
-    parameters_doc: Dict[str, Docstring]
-    """Parameters documentation. Key is parameter name, value is doc."""
-
-    doc: List[Docstring]
-    """Main description for the function. Separated by paragraph."""
-
-    return_: List[Docstring]
-    """Return:, or Return (multiple): (@return strings)"""
-
-    seealso: List[Docstring]
-    """See also: (@see strings)"""
-
-    xrefs: List[Docstring]
-    """XRefs. Currently only used to track Deprecated functions."""
-
-    # for INCLUDE_C_DECL
-    c_decl: str | None = None
-
-    prerelease: bool = False
-
-    def export_mpack(self) -> Dict[str, Any]:
-        """Convert a dict to be exported as mpack data."""
-        exported = self.__dict__.copy()
-        del exported['notes']
-        del exported['c_decl']
-        del exported['prerelease']
-        del exported['xrefs']
-        exported['return'] = exported.pop('return_')
-        return exported
-
-    def doc_concatenated(self) -> Docstring:
-        """Concatenate all the paragraphs in `doc` into a single string, but
-        remove blank lines before 'programlisting' blocks. #25127
-
-        BEFORE (without programlisting processing):
-            ```vimdoc
-            Example:
-
-            >vim
-                :echo nvim_get_color_by_name("Pink")
-            <
-            ```
-
-        AFTER:
-            ```vimdoc
-            Example: >vim
-                :echo nvim_get_color_by_name("Pink")
-            <
-            ```
-        """
-        def is_program_listing(paragraph: str) -> bool:
-            lines = paragraph.strip().split('\n')
-            return lines[0].startswith('>') and lines[-1] == '<'
-
-        rendered = []
-        for paragraph in self.doc:
-            if is_program_listing(paragraph):
-                rendered.append(' ')  # Example: >vim
-            elif rendered:
-                rendered.append('\n\n')
-            rendered.append(paragraph)
-        return ''.join(rendered)
-
-    def render(self) -> Docstring:
-        """Renders function documentation as Vim :help text."""
-        rendered_blocks: List[Docstring] = []
-
-        def fmt_param_doc(m):
-            """Renders a params map as Vim :help text."""
-            max_name_len = max_name(m.keys()) + 4
-            out = ''
-            for name, desc in m.items():
-                if name == 'self':
-                    continue
-                name = '  • {}'.format('{{{}}}'.format(name).ljust(max_name_len))
-                out += '{}{}\n'.format(name, desc)
-            return out.rstrip()
-
-        # Generate text from the gathered items.
-        chunks: List[Docstring] = [self.doc_concatenated()]
-
-        notes = []
-        if self.prerelease:
-            notes = ["  This API is pre-release (unstable)."]
-        notes += self.notes
-        if len(notes) > 0:
-            chunks.append('\nNote: ~')
-            for s in notes:
-                chunks.append('  ' + s)
-
-        if self.parameters_doc:
-            chunks.append('\nParameters: ~')
-            chunks.append(fmt_param_doc(self.parameters_doc))
-
-        if self.return_:
-            chunks.append('\nReturn (multiple): ~' if len(self.return_) > 1
-                          else '\nReturn: ~')
-            for s in self.return_:
-                chunks.append('    ' + s)
-
-        if self.seealso:
-            chunks.append('\nSee also: ~')
-            for s in self.seealso:
-                chunks.append('  ' + s)
-
-        # Note: xrefs are currently only used to remark "Deprecated: "
-        # for deprecated functions; visible when INCLUDE_DEPRECATED is set
-        for s in self.xrefs:
-            chunks.append('\n' + s)
-
-        rendered_blocks.append(clean_lines('\n'.join(chunks).strip()))
-        rendered_blocks.append('')
-
-        return clean_lines('\n'.join(rendered_blocks).strip())
-
-
-def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=''):
-    """Renders (nested) Doxygen <para> nodes as Vim :help text.
-
-    Only handles "text" nodes. Used for individual elements (see render_node())
-    and in extract_defgroups().
-
-    NB: Blank lines in a docstring manifest as <para> tags.
-    """
-    rendered_blocks = []
-
-    for child in parent.childNodes:
-        para, _ = para_as_map(child, indent, width)
-
-        # 'programlisting' blocks are Markdown code blocks. Do not include
-        # these as a separate paragraph, but append to the last non-empty line
-        # in the text
-        if is_program_listing(child):
-            while rendered_blocks and rendered_blocks[-1] == '':
-                rendered_blocks.pop()
-            rendered_blocks[-1] += ' ' + para['text']
-            continue
-
-        # Generate text from the gathered items.
-        chunks = [para['text']]
-
-        rendered_blocks.append(clean_lines('\n'.join(chunks).strip()))
-        rendered_blocks.append('')
-
-    return clean_lines('\n'.join(rendered_blocks).strip())
-
-
-def extract_from_xml(filename, target, *,
-                     width: int, fmt_vimhelp: bool) -> Tuple[
-    Dict[FunctionName, FunctionDoc],
-    Dict[FunctionName, FunctionDoc],
-]:
-    """Extracts Doxygen info as maps without formatting the text.
-
-    Returns two maps:
-      1. Functions
-      2. Deprecated functions
-
-    The `fmt_vimhelp` variable controls some special cases for use by
-    fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :)
-    """
-    config: Config = CONFIG[target]
-
-    fns: Dict[FunctionName, FunctionDoc] = {}
-    deprecated_fns: Dict[FunctionName, FunctionDoc] = {}
-
-    dom = minidom.parse(filename)
-    compoundname = get_text(dom.getElementsByTagName('compoundname')[0])
-    for member in dom.getElementsByTagName('memberdef'):
-        if member.getAttribute('static') == 'yes' or \
-                member.getAttribute('kind') != 'function' or \
-                member.getAttribute('prot') == 'private' or \
-                get_text(get_child(member, 'name')).startswith('_'):
-            continue
-
-        loc = find_first(member, 'location')
-        if 'private' in loc.getAttribute('file'):
-            continue
-
-        return_type = get_text(get_child(member, 'type'))
-        if return_type == '':
-            continue
-
-        if 'local_function' in return_type:  # Special from lua2dox.lua.
-            continue
-
-        istbl = return_type.startswith('table')  # Special from lua2dox.lua.
-        if istbl and not config.include_tables:
-            continue
-
-        if return_type.startswith(('ArrayOf', 'DictionaryOf')):
-            parts = return_type.strip('_').split('_')
-            return_type = '{}({})'.format(parts[0], ', '.join(parts[1:]))
-
-        name = get_text(get_child(member, 'name'))
-
-        annotations = get_text(get_child(member, 'argsstring'))
-        if annotations and ')' in annotations:
-            annotations = annotations.rsplit(')', 1)[-1].strip()
-        # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of
-        # non-void functions.  Special-case void functions here.
-        if name == 'nvim_get_mode' and len(annotations) == 0:
-            annotations += 'FUNC_API_FAST'
-        annotations = filter(None, map(lambda x: annotation_map.get(x),
-                                       annotations.split()))
-
-        params = []
-        type_length = 0
-
-        for param in iter_children(member, 'param'):
-            param_type = get_text(get_child(param, 'type')).strip()
-            param_name = ''
-            declname = get_child(param, 'declname')
-            if declname:
-                param_name = get_text(declname).strip()
-            elif config.mode == 'lua':
-                # XXX: this is what lua2dox gives us...
-                param_name = param_type
-                param_type = ''
-
-            if param_name in param_exclude:
-                continue
-
-            if fmt_vimhelp and param_type.endswith('*'):
-                param_type = param_type.strip('* ')
-                param_name = '*' + param_name
-
-            type_length = max(type_length, len(param_type))
-            params.append((param_type, param_name))
-
-        # Handle Object Oriented style functions here.
-        #   We make sure they have "self" in the parameters,
-        #   and a parent function
-        if return_type.startswith('function') \
-                and len(return_type.split(' ')) >= 2 \
-                and any(x[1] == 'self' for x in params):
-            split_return = return_type.split(' ')
-            name = f'{split_return[1]}:{name}'
-            params = [x for x in params if x[1] != 'self']
-
-        c_args = []
-        for param_type, param_name in params:
-            c_args.append(('    ' if fmt_vimhelp else '') + (
-                '%s %s' % (param_type.ljust(type_length), param_name)).strip())
-
-        if not fmt_vimhelp:
-            pass
-        else:
-            fstem = '?'
-            if '.' in compoundname:
-                fstem = compoundname.split('.')[0]
-                fstem = config.module_override.get(fstem, fstem)
-            vimtag = config.fn_helptag_fmt(fstem, name, istbl)
-
-            if config.fn_name_fmt:
-                name = config.fn_name_fmt(fstem, name)
-
-        if istbl:
-            aopen, aclose = '', ''
-        else:
-            aopen, aclose = '(', ')'
-
-        prefix = name + aopen
-        suffix = ', '.join('{%s}' % a[1] for a in params
-                           if a[0] not in ('void', 'Error', 'Arena',
-                                           'lua_State')) + aclose
-
-        if not fmt_vimhelp:
-            c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args))
-            signature = prefix + suffix
-        else:
-            c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name,
-                                                         ',\n'.join(c_args)),
-                                     '    ')
-
-            # Minimum 8 chars between signature and vimtag
-            lhs = (width - 8) - len(vimtag)
-
-            if len(prefix) + len(suffix) > lhs:
-                signature = vimtag.rjust(width) + '\n'
-                signature += doc_wrap(suffix, width=width, prefix=prefix,
-                                      func=True)
-            else:
-                signature = prefix + suffix
-                signature += vimtag.rjust(width - len(signature))
-
-        # Tracks `xrefsect` titles.  As of this writing, used only for separating
-        # deprecated functions.
-        xrefs_all = set()
-        paras: List[Dict[str, Any]] = []  # paras means paragraphs!
-        brief_desc = find_first(member, 'briefdescription')
-        if brief_desc:
-            for child in brief_desc.childNodes:
-                para, xrefs = para_as_map(child)
-                paras.append(para)
-                xrefs_all.update(xrefs)
-
-        desc = find_first(member, 'detaileddescription')
-        if desc:
-            paras_detail = []  # override briefdescription
-            for child in desc.childNodes:
-                para, xrefs = para_as_map(child)
-                paras_detail.append(para)
-                xrefs_all.update(xrefs)
-            log.debug(
-                textwrap.indent(
-                    re.sub(r'\n\s*\n+', '\n',
-                           desc.toprettyxml(indent='  ', newl='\n')),
-                    ' ' * indentation))
-
-            # override briefdescription, if detaileddescription is not empty
-            # (note: briefdescription can contain some erroneous luadoc
-            #  comments from preceding comments, this is a bug of lua2dox)
-            if any((para['text'] or para['note'] or para['params'] or
-                    para['return'] or para['seealso']
-                    ) for para in paras_detail):
-                paras = paras_detail
-
-        fn = FunctionDoc(
-            annotations=list(annotations),
-            notes=[],
-            signature=signature,
-            parameters=params,
-            parameters_doc=collections.OrderedDict(),
-            doc=[],
-            return_=[],
-            seealso=[],
-            xrefs=[],
-        )
-
-        for m in paras:
-            if m.get('text', ''):
-                fn.doc.append(m['text'])
-            if 'params' in m:
-                # Merge OrderedDicts.
-                fn.parameters_doc.update(m['params'])
-            if 'return' in m and len(m['return']) > 0:
-                fn.return_ += m['return']
-            if 'seealso' in m and len(m['seealso']) > 0:
-                fn.seealso += m['seealso']
-            if m.get('prerelease', False):
-                fn.prerelease = True
-            if 'note' in m:
-                fn.notes += m['note']
-            if 'xrefs' in m:
-                fn.xrefs += m['xrefs']
-
-        if INCLUDE_C_DECL:
-            fn.c_decl = c_decl
-
-        if 'Deprecated' in str(xrefs_all):
-            deprecated_fns[name] = fn
-        elif name.startswith(config.fn_name_prefix):
-            fns[name] = fn
-
-    # sort functions by name (lexicographically)
-    fns = collections.OrderedDict(sorted(
-        fns.items(),
-        key=lambda key_item_tuple: key_item_tuple[0].lower(),
-    ))
-    deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items()))
-    return fns, deprecated_fns
-
-
-def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]:
-    """Entrypoint for generating Vim :help from from Doxygen XML.
-
-    Returns 2 items:
-      1. Vim help text for functions found in `filename`.
-      2. Vim help text for deprecated functions.
-    """
-    config: Config = CONFIG[target]
-
-    fns_txt = {}  # Map of func_name:vim-help-text.
-    deprecated_fns_txt = {}  # Map of func_name:vim-help-text.
-
-    fns: Dict[FunctionName, FunctionDoc]
-    deprecated_fns: Dict[FunctionName, FunctionDoc]
-    fns, deprecated_fns = extract_from_xml(
-        filename, target, width=text_width, fmt_vimhelp=True)
-
-    def _handle_fn(fn_name: FunctionName, fn: FunctionDoc,
-                   fns_txt: Dict[FunctionName, Docstring], deprecated=False):
-        # Generate Vim :help for parameters.
-
-        # Generate body from FunctionDoc, not XML nodes
-        doc = fn.render()
-        if not doc and fn_name.startswith("nvim__"):
-            return
-        if not doc:
-            doc = ('TODO: Documentation' if not deprecated
-                   else 'Deprecated.')
-
-        # Annotations: put before Parameters
-        annotations: str = '\n'.join(fn.annotations)
-        if annotations:
-            annotations = ('\n\nAttributes: ~\n' +
-                           textwrap.indent(annotations, '    '))
-            i = doc.rfind('Parameters: ~')
-            if i == -1:
-                doc += annotations
-            else:
-                doc = doc[:i] + annotations + '\n\n' + doc[i:]
-
-        # C Declaration: (debug only)
-        if INCLUDE_C_DECL:
-            doc += '\n\nC Declaration: ~\n>\n'
-            assert fn.c_decl is not None
-            doc += fn.c_decl
-            doc += '\n<'
-
-        # Start of function documentations. e.g.,
-        # nvim_cmd({*cmd}, {*opts})                                         *nvim_cmd()*
-        func_doc = fn.signature + '\n'
-        func_doc += textwrap.indent(clean_lines(doc), ' ' * indentation)
-
-        # Verbatim handling.
-        func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M)
-
-        def process_helptags(func_doc: str) -> str:
-            lines: List[str] = func_doc.split('\n')
-            # skip ">lang ... <" regions
-            is_verbatim: bool = False
-            for i in range(len(lines)):
-                if re.search(' >([a-z])*$', lines[i]):
-                    is_verbatim = True
-                elif is_verbatim and lines[i].strip() == '<':
-                    is_verbatim = False
-                if not is_verbatim:
-                    lines[i] = align_tags(lines[i])
-            return "\n".join(lines)
-
-        func_doc = process_helptags(func_doc)
-
-        if (fn_name.startswith(config.fn_name_prefix)
-            and fn_name != "nvim_error_event"):
-            fns_txt[fn_name] = func_doc
-
-    for fn_name, fn in fns.items():
-        _handle_fn(fn_name, fn, fns_txt)
-    for fn_name, fn in deprecated_fns.items():
-        _handle_fn(fn_name, fn, deprecated_fns_txt, deprecated=True)
-
-    return (
-        '\n\n'.join(list(fns_txt.values())),
-        '\n\n'.join(list(deprecated_fns_txt.values())),
-    )
-
-
-def delete_lines_below(filename, tokenstr):
-    """Deletes all lines below the line containing `tokenstr`, the line itself,
-    and one line above it.
-    """
-    lines = open(filename).readlines()
-    i = 0
-    found = False
-    for i, line in enumerate(lines, 1):
-        if tokenstr in line:
-            found = True
-            break
-    if not found:
-        raise RuntimeError(f'not found: "{tokenstr}"')
-    i = max(0, i - 2)
-    with open(filename, 'wt') as fp:
-        fp.writelines(lines[0:i])
-
-
-def extract_defgroups(base: str, dom: Document) -> Dict[SectionName, Docstring]:
-    '''Generate module-level (section) docs (@defgroup).'''
-    section_docs = {}
-
-    for compound in dom.getElementsByTagName('compound'):
-        if compound.getAttribute('kind') != 'group':
-            continue
-
-        # Doxygen "@defgroup" directive.
-        groupname = get_text(find_first(compound, 'name'))
-        groupxml = os.path.join(base, '%s.xml' %
-                                compound.getAttribute('refid'))
-
-        group_parsed = minidom.parse(groupxml)
-        doc_list = []
-        brief_desc = find_first(group_parsed, 'briefdescription')
-        if brief_desc:
-            for child in brief_desc.childNodes:
-                doc_list.append(fmt_node_as_vimhelp(child))
-
-        desc = find_first(group_parsed, 'detaileddescription')
-        if desc:
-            doc = fmt_node_as_vimhelp(desc)
-
-            if doc:
-                doc_list.append(doc)
-
-        # Can't use '.' in @defgroup, so convert to '--'
-        # "vim.json" => "vim-dot-json"
-        groupname = groupname.replace('-dot-', '.')
-
-        section_docs[groupname] = "\n".join(doc_list)
-
-    return section_docs
-
-
-@dataclasses.dataclass
-class Section:
-    """Represents a section. Includes section heading (defgroup)
-    and all the FunctionDoc that belongs to this section."""
-
-    name: str
-    '''Name of the section. Usually derived from basename of lua/c src file.
-    Example: "Autocmd".'''
-
-    title: str
-    '''Formatted section config. see config.section_fmt().
-    Example: "Autocmd Functions". '''
-
-    helptag: str
-    '''see config.helptag_fmt(). Example: *api-autocmd*'''
-
-    @property
-    def id(self) -> str:
-        '''section id: Module/Section id matched against @defgroup.
-           e.g., "*api-autocmd*" => "api-autocmd"
-        '''
-        return self.helptag.strip('*')
-
-    doc: str = ""
-    '''Section heading docs extracted from @defgroup.'''
-
-    # TODO: Do not carry rendered text, but handle FunctionDoc for better OOP
-    functions_text: Docstring | None = None
-    '''(Rendered) doc of all the functions that belong to this section.'''
-
-    deprecated_functions_text: Docstring | None = None
-    '''(Rendered) doc of all the deprecated functions that belong to this
-    section.'''
-
-    def __repr__(self):
-        return f"Section(title='{self.title}', helptag='{self.helptag}')"
-
-    @classmethod
-    def make_from(cls, filename: str, config: Config,
-                  section_docs: Dict[SectionName, str],
-                  *,
-                  functions_text: Docstring,
-                  deprecated_functions_text: Docstring,
-                  ):
-        # filename: e.g., 'autocmd.c'
-        # name: e.g. 'autocmd'
-        name = os.path.splitext(filename)[0].lower()
-
-        # section name: e.g. "Autocmd"
-        sectname: SectionName
-        sectname = name.upper() if name == 'ui' else name.title()
-        sectname = config.section_name.get(filename, sectname)
-
-        # Formatted (this is what's going to be written in the vimdoc)
-        # e.g., "Autocmd Functions"
-        title: str = config.section_fmt(sectname)
-
-        # section tag: e.g., "*api-autocmd*"
-        section_tag: str = config.helptag_fmt(sectname)
-
-        section = cls(name=sectname, title=title, helptag=section_tag,
-                      functions_text=functions_text,
-                      deprecated_functions_text=deprecated_functions_text,
-                      )
-        section.doc = section_docs.get(section.id) or ''
-        return section
-
-    def render(self, add_header=True) -> str:
-        """Render as vimdoc."""
-        doc = ''
-
-        if add_header:
-            doc += SECTION_SEP
-            doc += '\n{}{}'.format(
-                self.title,
-                self.helptag.rjust(text_width - len(self.title))
-            )
-
-        if self.doc:
-            doc += '\n\n' + self.doc
-
-        if self.functions_text:
-            doc += '\n\n' + self.functions_text
-
-        if INCLUDE_DEPRECATED and self.deprecated_functions_text:
-            doc += f'\n\n\nDeprecated {self.name} Functions: ~\n\n'
-            doc += self.deprecated_functions_text
-
-        return doc
-
-    def __bool__(self) -> bool:
-        """Whether this section has contents. Used for skipping empty ones."""
-        return bool(self.doc or self.functions_text or
-                    (INCLUDE_DEPRECATED and self.deprecated_functions_text))
-
-
-def main(doxygen_config, args):
-    """Generates:
-
-    1. Vim :help docs
-    2. *.mpack files for use by API clients
-
-    Doxygen is called and configured through stdin.
-    """
-    for target in CONFIG:
-        if args.target is not None and target != args.target:
-            continue
-
-        config: Config = CONFIG[target]
-
-        mpack_file = os.path.join(
-            base_dir, 'runtime', 'doc',
-            config.filename.replace('.txt', '.mpack'))
-        if os.path.exists(mpack_file):
-            os.remove(mpack_file)
-
-        output_dir = out_dir.format(target=target)
-        log.info("Generating documentation for %s in folder %s",
-                 target, output_dir)
-        debug = args.log_level >= logging.DEBUG
-        p = subprocess.Popen(
-                ['doxygen', '-'],
-                stdin=subprocess.PIPE,
-                # silence warnings
-                # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found
-                stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL))
-        p.communicate(
-            doxygen_config.format(
-                input=' '.join([f'"{file}"' for file in config.files]),
-                output=output_dir,
-                filter=filter_cmd,
-                file_patterns=config.file_patterns)
-            .encode('utf8')
-        )
-        if p.returncode:
-            sys.exit(p.returncode)
-
-        # Collects all functions as each module is processed.
-        fn_map_full: Dict[FunctionName, FunctionDoc] = {}
-        # key: filename (e.g. autocmd.c)
-        sections: Dict[str, Section] = {}
-
-        base = os.path.join(output_dir, 'xml')
-        dom = minidom.parse(os.path.join(base, 'index.xml'))
-
-        # Collect all @defgroups (section headings after the '===...' separator
-        section_docs: Dict[SectionName, Docstring] = extract_defgroups(base, dom)
-
-        # Generate docs for all functions in the current module.
-        for compound in dom.getElementsByTagName('compound'):
-            if compound.getAttribute('kind') != 'file':
-                continue
-
-            filename = get_text(find_first(compound, 'name'))
-            if not (
-                filename.endswith('.c') or
-                filename.endswith('.lua')
-            ):
-                continue
-
-            xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid')))
-
-            # Extract unformatted (*.mpack).
-            fn_map, _ = extract_from_xml(
-                xmlfile, target, width=9999, fmt_vimhelp=False)
-
-            # Extract formatted (:help).
-            functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp(
-                xmlfile, target)
-
-            if not functions_text and not deprecated_text:
-                continue
-
-            filename = os.path.basename(filename)
-
-            section: Section = Section.make_from(
-                filename, config, section_docs,
-                functions_text=functions_text,
-                deprecated_functions_text=deprecated_text,
-            )
-
-            if section:  # if not empty
-                sections[filename] = section
-                fn_map_full.update(fn_map)
-            else:
-                log.debug("Skipping empty section: %s", section)
-
-        if len(sections) == 0:
-            fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)')
-        if len(sections) > len(config.section_order):
-            raise RuntimeError(
-                '{}: found new modules {}; '
-                'update the "section_order" map'.format(
-                    target,
-                    set(sections).difference(config.section_order))
-            )
-        first_section_tag = sections[config.section_order[0]].helptag
-
-        docs = ''
-
-        for filename in config.section_order:
-            try:
-                section: Section = sections.pop(filename)
-            except KeyError:
-                msg(f'warning: empty docs, skipping (target={target}): {filename}')
-                msg(f'    existing docs: {sections.keys()}')
-                continue
-
-            add_sep_and_header = filename not in config.append_only
-            docs += section.render(add_header=add_sep_and_header)
-            docs += '\n\n\n'
-
-        docs = docs.rstrip() + '\n\n'
-        docs += f' vim:tw=78:ts=8:sw={indentation}:sts={indentation}:et:ft=help:norl:\n'
-
-        doc_file = os.path.join(base_dir, 'runtime', 'doc', config.filename)
-
-        if os.path.exists(doc_file):
-            delete_lines_below(doc_file, first_section_tag)
-        with open(doc_file, 'ab') as fp:
-            fp.write(docs.encode('utf8'))
-
-        fn_map_full_exported = collections.OrderedDict(sorted(
-            (name, fn_doc.export_mpack()) for (name, fn_doc) in fn_map_full.items()
-        ))
-        with open(mpack_file, 'wb') as fp:
-            fp.write(msgpack.packb(fn_map_full_exported, use_bin_type=True))  # type: ignore
-
-        if not args.keep_tmpfiles:
-            shutil.rmtree(output_dir)
-
-    msg_report()
-
-
-def filter_source(filename, keep_tmpfiles):
-    output_dir = out_dir.format(target='lua2dox')
-    name, extension = os.path.splitext(filename)
-    if extension == '.lua':
-        args = [str(nvim), '-l', lua2dox, filename] + (['--outdir', output_dir] if keep_tmpfiles else [])
-        p = subprocess.run(args, stdout=subprocess.PIPE)
-        op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8'))
-        print(op)
-    else:
-        """Filters the source to fix macros that confuse Doxygen."""
-        with open(filename, 'rt') as fp:
-            print(re.sub(r'^(ArrayOf|DictionaryOf|Dict)(\(.*?\))',
-                         lambda m: m.group(1)+'_'.join(
-                             re.split(r'[^\w]+', m.group(2))),
-                         fp.read(), flags=re.M))
-
-
-def parse_args():
-    targets = ', '.join(CONFIG.keys())
-    ap = argparse.ArgumentParser(
-        description="Generate helpdoc from source code")
-    ap.add_argument(
-        "--log-level", "-l", choices=LOG_LEVELS.keys(),
-        default=logging.getLevelName(logging.ERROR), help="Set log verbosity"
-    )
-    ap.add_argument('source_filter', nargs='*',
-                    help="Filter source file(s)")
-    ap.add_argument('-k', '--keep-tmpfiles', action='store_true',
-                    help="Keep temporary files (tmp-xx-doc/ directories, including tmp-lua2dox-doc/ for lua2dox.lua quasi-C output)")
-    ap.add_argument('-t', '--target',
-                    help=f'One of ({targets}), defaults to "all"')
-    return ap.parse_args()
-
-
-Doxyfile = textwrap.dedent('''
-    OUTPUT_DIRECTORY       = {output}
-    INPUT                  = {input}
-    INPUT_ENCODING         = UTF-8
-    FILE_PATTERNS          = {file_patterns}
-    RECURSIVE              = YES
-    INPUT_FILTER           = "{filter}"
-    EXCLUDE                =
-    EXCLUDE_SYMLINKS       = NO
-    EXCLUDE_PATTERNS       = */private/* */health.lua */_*.lua
-    EXCLUDE_SYMBOLS        =
-    EXTENSION_MAPPING      = lua=C
-    EXTRACT_PRIVATE        = NO
-
-    GENERATE_HTML          = NO
-    GENERATE_DOCSET        = NO
-    GENERATE_HTMLHELP      = NO
-    GENERATE_QHP           = NO
-    GENERATE_TREEVIEW      = NO
-    GENERATE_LATEX         = NO
-    GENERATE_RTF           = NO
-    GENERATE_MAN           = NO
-    GENERATE_DOCBOOK       = NO
-    GENERATE_AUTOGEN_DEF   = NO
-
-    GENERATE_XML           = YES
-    XML_OUTPUT             = xml
-    XML_PROGRAMLISTING     = NO
-
-    ENABLE_PREPROCESSING   = YES
-    MACRO_EXPANSION        = YES
-    EXPAND_ONLY_PREDEF     = NO
-    MARKDOWN_SUPPORT       = YES
-''')
-
-if __name__ == "__main__":
-    args = parse_args()
-    print("Setting log level to %s" % args.log_level)
-    args.log_level = LOG_LEVELS[args.log_level]
-    log.setLevel(args.log_level)
-    log.addHandler(logging.StreamHandler())
-
-    # When invoked as a filter, args won't be passed, so use an env var.
-    if args.keep_tmpfiles:
-        os.environ['NVIM_KEEP_TMPFILES'] = '1'
-    keep_tmpfiles = ('NVIM_KEEP_TMPFILES' in os.environ)
-
-    if len(args.source_filter) > 0:
-        filter_source(args.source_filter[0], keep_tmpfiles)
-    else:
-        main(Doxyfile, args)
-
-# vim: set ft=python ts=4 sw=4 tw=79 et :
-- 
cgit