feat(docs): replace lua2dox.lua

Problem:

The documentation flow (`gen_vimdoc.py`) has several issues:
- it's not very versatile
- depends on doxygen
- doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C.
- The intermediate XML files and filters makes it too much like a rube goldberg machine.

Solution:

Re-implement the flow using Lua, LPEG and treesitter.

- `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic.
- `lua2dox.lua` is gone!
- No more XML files.
- Doxygen is now longer used and instead we now use:
  - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`).
  - LPEG for C parsing (see `scripts/cdoc_parser.lua`)
  - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`).
  - Treesitter for Markdown parsing (see `scripts/text_utils.lua`).
- The generated `runtime/doc/*.mpack` files have been removed.
   - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly.
- Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change).

1 files changed, 0 insertions, 1766 deletions

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
deleted file mode 100755
index c1a2183f24..0000000000
--- a/scripts/gen_vimdoc.py
+++ /dev/null
@@ -1,1766 +0,0 @@
-#!/usr/bin/env python3
-
-r"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen.
-
-Also generates *.mpack files. To inspect the *.mpack structure:
-    :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B')))
-
-Flow:
-    main
-      extract_from_xml
-        fmt_node_as_vimhelp   \
-          para_as_map          } recursive
-            update_params_map /
-              render_node
-
-TODO: eliminate this script and use Lua+treesitter (requires parsers for C and
-Lua markdown-style docstrings).
-
-The generated :help text for each function is formatted as follows:
-
-  - Max width of 78 columns (`text_width`).
-  - Indent with spaces (not tabs).
-  - Indent of 4 columns for body text (`indentation`).
-  - Function signature and helptag (right-aligned) on the same line.
-    - Signature and helptag must have a minimum of 8 spaces between them.
-    - If the signature is too long, it is placed on the line after the helptag.
-      Signature wraps at `text_width - 8` characters with subsequent
-      lines indented to the open parenthesis.
-    - Subsection bodies are indented an additional 4 spaces.
-  - Body consists of function description, parameters, return description, and
-    C declaration (`INCLUDE_C_DECL`).
-  - Parameters are omitted for the `void` and `Error *` types, or if the
-    parameter is marked as [out].
-  - Each function documentation is separated by a single line.
-"""
-
-from __future__ import annotations  # PEP-563, python 3.7+
-
-import argparse
-import collections
-import dataclasses
-import logging
-import os
-import re
-import shutil
-import subprocess
-import sys
-import textwrap
-from pathlib import Path
-from typing import Any, Callable, Dict, List, Tuple
-from xml.dom import minidom
-
-if sys.version_info >= (3, 8):
-    from typing import Literal
-
-import msgpack
-
-Element = minidom.Element
-Document = minidom.Document
-
-MIN_PYTHON_VERSION = (3, 7)
-MIN_DOXYGEN_VERSION = (1, 9, 0)
-
-if sys.version_info < MIN_PYTHON_VERSION:
-    print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION))
-    sys.exit(1)
-
-doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"],
-                        universal_newlines=True).split()[0].split('.')))
-
-if doxygen_version < MIN_DOXYGEN_VERSION:
-    print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION))
-    print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version))
-    sys.exit(1)
-
-
-# Need a `nvim` that supports `-l`, try the local build
-nvim_path = Path(__file__).parent / "../build/bin/nvim"
-if nvim_path.exists():
-    nvim = nvim_path.resolve()
-else:
-    # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works.
-    nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True)
-    nvim_version = [line for line in nvim_out.split('\n')
-                    if '-l ' in line]
-    if len(nvim_version) == 0:
-        print((
-            "\nYou need to have a local Neovim build or a `nvim` version 0.9 for `-l` "
-            "support to build the documentation."))
-        sys.exit(1)
-    nvim = 'nvim'
-
-
-# DEBUG = ('DEBUG' in os.environ)
-INCLUDE_C_DECL = os.environ.get('INCLUDE_C_DECL', '0') != '0'
-INCLUDE_DEPRECATED = os.environ.get('INCLUDE_DEPRECATED', '0') != '0'
-
-log = logging.getLogger(__name__)
-
-LOG_LEVELS = {
-    logging.getLevelName(level): level for level in [
-        logging.DEBUG, logging.INFO, logging.ERROR
-    ]
-}
-
-text_width = 78
-indentation = 4
-SECTION_SEP = '=' * text_width
-
-script_path = os.path.abspath(__file__)
-base_dir = os.path.dirname(os.path.dirname(script_path))
-out_dir = os.path.join(base_dir, 'tmp-{target}-doc')
-filter_cmd = '%s %s' % (sys.executable, script_path)
-msgs = []  # Messages to show on exit.
-lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua')
-
-
-SectionName = str
-
-Docstring = str  # Represents (formatted) vimdoc string
-
-FunctionName = str
-
-
-@dataclasses.dataclass
-class Config:
-    """Config for documentation."""
-
-    mode: Literal['c', 'lua']
-
-    filename: str
-    """Generated documentation target, e.g. api.txt"""
-
-    section_order: List[str]
-    """Section ordering."""
-
-    files: List[str]
-    """List of files/directories for doxygen to read, relative to `base_dir`."""
-
-    file_patterns: str
-    """file patterns used by doxygen."""
-
-    section_name: Dict[str, SectionName]
-    """Section name overrides. Key: filename (e.g., vim.c)"""
-
-    section_fmt: Callable[[SectionName], str]
-    """For generated section names."""
-
-    helptag_fmt: Callable[[SectionName], str]
-    """Section helptag."""
-
-    fn_helptag_fmt: Callable[[str, str, bool], str]
-    """Per-function helptag."""
-
-    module_override: Dict[str, str]
-    """Module name overrides (for Lua)."""
-
-    append_only: List[str]
-    """Append the docs for these modules, do not start a new section."""
-
-    fn_name_prefix: str
-    """Only function with this prefix are considered"""
-
-    fn_name_fmt: Callable[[str, str], str] | None = None
-
-    include_tables: bool = True
-
-
-CONFIG: Dict[str, Config] = {
-    'api': Config(
-        mode='c',
-        filename = 'api.txt',
-        # Section ordering.
-        section_order=[x for x in [
-            'vim.c',
-            'vimscript.c',
-            'command.c',
-            'options.c',
-            'buffer.c',
-            'extmark.c',
-            'window.c',
-            'win_config.c',
-            'tabpage.c',
-            'autocmd.c',
-            'ui.c',
-            'deprecated.c' if INCLUDE_DEPRECATED else ''
-        ] if x],
-        files=['src/nvim/api'],
-        file_patterns = '*.h *.c',
-        fn_name_prefix = 'nvim_',
-        section_name={
-            'vim.c': 'Global',
-        },
-        section_fmt=lambda name: f'{name} Functions',
-        helptag_fmt=lambda name: f'*api-{name.lower()}*',
-        fn_helptag_fmt=lambda fstem, name, istbl: f'*{name}()*',
-        module_override={},
-        append_only=[],
-    ),
-    'lua': Config(
-        mode='lua',
-        filename='lua.txt',
-        section_order=[
-            'highlight.lua',
-            'diff.lua',
-            'mpack.lua',
-            'json.lua',
-            'base64.lua',
-            'spell.lua',
-            'builtin.lua',
-            '_options.lua',
-            '_editor.lua',
-            '_inspector.lua',
-            'shared.lua',
-            'loader.lua',
-            'uri.lua',
-            'ui.lua',
-            'filetype.lua',
-            'keymap.lua',
-            'fs.lua',
-            'glob.lua',
-            'lpeg.lua',
-            're.lua',
-            'regex.lua',
-            'secure.lua',
-            'version.lua',
-            'iter.lua',
-            'snippet.lua',
-            'text.lua',
-        ],
-        files=[
-            'runtime/lua/vim/iter.lua',
-            'runtime/lua/vim/_editor.lua',
-            'runtime/lua/vim/_options.lua',
-            'runtime/lua/vim/shared.lua',
-            'runtime/lua/vim/loader.lua',
-            'runtime/lua/vim/uri.lua',
-            'runtime/lua/vim/ui.lua',
-            'runtime/lua/vim/filetype.lua',
-            'runtime/lua/vim/keymap.lua',
-            'runtime/lua/vim/fs.lua',
-            'runtime/lua/vim/highlight.lua',
-            'runtime/lua/vim/secure.lua',
-            'runtime/lua/vim/version.lua',
-            'runtime/lua/vim/_inspector.lua',
-            'runtime/lua/vim/snippet.lua',
-            'runtime/lua/vim/text.lua',
-            'runtime/lua/vim/glob.lua',
-            'runtime/lua/vim/_meta/builtin.lua',
-            'runtime/lua/vim/_meta/diff.lua',
-            'runtime/lua/vim/_meta/mpack.lua',
-            'runtime/lua/vim/_meta/json.lua',
-            'runtime/lua/vim/_meta/base64.lua',
-            'runtime/lua/vim/_meta/regex.lua',
-            'runtime/lua/vim/_meta/lpeg.lua',
-            'runtime/lua/vim/_meta/re.lua',
-            'runtime/lua/vim/_meta/spell.lua',
-        ],
-        file_patterns='*.lua',
-        fn_name_prefix='',
-        fn_name_fmt=lambda fstem, name: (
-            name if fstem in [ 'vim.iter' ] else
-            f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else
-            f'vim.{name}' if fstem == '_options' and not name[0].isupper() else
-            f'{fstem}.{name}' if fstem.startswith('vim') else
-            name
-        ),
-        section_name={
-            'lsp.lua': 'core',
-            '_inspector.lua': 'inspector',
-        },
-        section_fmt=lambda name: (
-            'Lua module: vim' if name.lower() == '_editor' else
-            'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else
-            f'VIM.{name.upper()}' if name.lower() in [
-                'highlight', 'mpack', 'json', 'base64', 'diff', 'spell',
-                'regex', 'lpeg', 're',
-            ] else
-            'VIM' if name.lower() == 'builtin' else
-            f'Lua module: vim.{name.lower()}'),
-        helptag_fmt=lambda name: (
-            '*lua-vim*' if name.lower() == '_editor' else
-            '*lua-vimscript*' if name.lower() == '_options' else
-            f'*vim.{name.lower()}*'),
-        fn_helptag_fmt=lambda fstem, name, istbl: (
-            f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else
-            # Exclude fstem for methods
-            f'*{name}()*' if ':' in name else
-            f'*vim.{name}()*' if fstem.lower() == '_editor' else
-            f'*vim.{name}*' if fstem.lower() == '_options' and istbl else
-            # Prevents vim.regex.regex
-            f'*{fstem}()*' if fstem.endswith('.' + name) else
-            f'*{fstem}.{name}{"" if istbl else "()"}*'
-            ),
-        module_override={
-            # `shared` functions are exposed on the `vim` module.
-            'shared': 'vim',
-            '_inspector': 'vim',
-            'uri': 'vim',
-            'ui': 'vim.ui',
-            'loader': 'vim.loader',
-            'filetype': 'vim.filetype',
-            'keymap': 'vim.keymap',
-            'fs': 'vim.fs',
-            'highlight': 'vim.highlight',
-            'secure': 'vim.secure',
-            'version': 'vim.version',
-            'iter': 'vim.iter',
-            'diff': 'vim',
-            'builtin': 'vim',
-            'mpack': 'vim.mpack',
-            'json': 'vim.json',
-            'base64': 'vim.base64',
-            'regex': 'vim.regex',
-            'lpeg': 'vim.lpeg',
-            're': 'vim.re',
-            'spell': 'vim.spell',
-            'snippet': 'vim.snippet',
-            'text': 'vim.text',
-            'glob': 'vim.glob',
-        },
-        append_only=[
-            'shared.lua',
-        ],
-    ),
-    'lsp': Config(
-        mode='lua',
-        filename='lsp.txt',
-        section_order=[
-            'lsp.lua',
-            'buf.lua',
-            'diagnostic.lua',
-            'codelens.lua',
-            'inlay_hint.lua',
-            'tagfunc.lua',
-            'semantic_tokens.lua',
-            'handlers.lua',
-            'util.lua',
-            'log.lua',
-            'rpc.lua',
-            'protocol.lua',
-        ],
-        files=[
-            'runtime/lua/vim/lsp',
-            'runtime/lua/vim/lsp.lua',
-        ],
-        file_patterns='*.lua',
-        fn_name_prefix='',
-        section_name={'lsp.lua': 'lsp'},
-        section_fmt=lambda name: (
-            'Lua module: vim.lsp'
-            if name.lower() == 'lsp'
-            else f'Lua module: vim.lsp.{name.lower()}'),
-        helptag_fmt=lambda name: (
-            '*lsp-core*'
-            if name.lower() == 'lsp'
-            else f'*lsp-{name.lower()}*'),
-        fn_helptag_fmt=lambda fstem, name, istbl: (
-            f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else
-            # HACK. TODO(justinmk): class/structure support in lua2dox
-            '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else
-            f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'),
-        module_override={},
-        append_only=[],
-    ),
-    'diagnostic': Config(
-        mode='lua',
-        filename='diagnostic.txt',
-        section_order=[
-            'diagnostic.lua',
-        ],
-        files=['runtime/lua/vim/diagnostic.lua'],
-        file_patterns='*.lua',
-        fn_name_prefix='',
-        include_tables=False,
-        section_name={'diagnostic.lua': 'diagnostic'},
-        section_fmt=lambda _: 'Lua module: vim.diagnostic',
-        helptag_fmt=lambda _: '*diagnostic-api*',
-        fn_helptag_fmt=lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*',
-        module_override={},
-        append_only=[],
-    ),
-    'treesitter': Config(
-        mode='lua',
-        filename='treesitter.txt',
-        section_order=[
-            'treesitter.lua',
-            'language.lua',
-            'query.lua',
-            'highlighter.lua',
-            'languagetree.lua',
-            'dev.lua',
-        ],
-        files=[
-            'runtime/lua/vim/treesitter.lua',
-            'runtime/lua/vim/treesitter/',
-        ],
-        file_patterns='*.lua',
-        fn_name_prefix='',
-        section_name={},
-        section_fmt=lambda name: (
-            'Lua module: vim.treesitter'
-            if name.lower() == 'treesitter'
-            else f'Lua module: vim.treesitter.{name.lower()}'),
-        helptag_fmt=lambda name: (
-            '*lua-treesitter-core*'
-            if name.lower() == 'treesitter'
-            else f'*lua-treesitter-{name.lower()}*'),
-        fn_helptag_fmt=lambda fstem, name, istbl: (
-            f'*vim.{fstem}.{name}()*'
-            if fstem == 'treesitter'
-            else f'*{name}()*'
-            if name[0].isupper()
-            else f'*vim.treesitter.{fstem}.{name}()*'),
-        module_override={},
-        append_only=[],
-    ),
-}
-
-param_exclude = (
-    'channel_id',
-)
-
-# Annotations are displayed as line items after API function descriptions.
-annotation_map = {
-    'FUNC_API_FAST': '|api-fast|',
-    'FUNC_API_TEXTLOCK': 'not allowed when |textlock| is active or in the |cmdwin|',
-    'FUNC_API_TEXTLOCK_ALLOW_CMDWIN': 'not allowed when |textlock| is active',
-    'FUNC_API_REMOTE_ONLY': '|RPC| only',
-    'FUNC_API_LUA_ONLY': 'Lua |vim.api| only',
-}
-
-
-def nvim_api_info() -> Tuple[int, bool]:
-    """Returns NVIM_API_LEVEL, NVIM_API_PRERELEASE from CMakeLists.txt"""
-    if not hasattr(nvim_api_info, 'LEVEL'):
-        script_dir = os.path.dirname(os.path.abspath(__file__))
-        cmake_file_path = os.path.join(script_dir, '..', 'CMakeLists.txt')
-        with open(cmake_file_path, 'r') as cmake_file:
-            cmake_content = cmake_file.read()
-
-        api_level_match = re.search(r'set\(NVIM_API_LEVEL (\d+)\)', cmake_content)
-        api_prerelease_match = re.search(
-            r'set\(NVIM_API_PRERELEASE (\w+)\)', cmake_content
-        )
-
-        if not api_level_match or not api_prerelease_match:
-            raise RuntimeError(
-                'Could not find NVIM_API_LEVEL or NVIM_API_PRERELEASE in CMakeLists.txt'
-            )
-
-        nvim_api_info.LEVEL = int(api_level_match.group(1))
-        nvim_api_info.PRERELEASE = api_prerelease_match.group(1).lower() == 'true'
-
-    return nvim_api_info.LEVEL, nvim_api_info.PRERELEASE
-
-
-# Raises an error with details about `o`, if `cond` is in object `o`,
-# or if `cond()` is callable and returns True.
-def debug_this(o, cond=True):
-    name = ''
-    if cond is False:
-        return
-    if not isinstance(o, str):
-        try:
-            name = o.nodeName
-            o = o.toprettyxml(indent='  ', newl='\n')
-        except Exception:
-            pass
-    if (cond is True
-            or (callable(cond) and cond())
-            or (not callable(cond) and cond in o)):
-        raise RuntimeError('xxx: {}\n{}'.format(name, o))
-
-
-# Appends a message to a list which will be printed on exit.
-def msg(s):
-    msgs.append(s)
-
-
-# Print all collected messages.
-def msg_report():
-    for m in msgs:
-        print(f'    {m}')
-
-
-# Print collected messages, then throw an exception.
-def fail(s):
-    msg_report()
-    raise RuntimeError(s)
-
-
-def find_first(parent, name):
-    """Finds the first matching node within parent."""
-    sub = parent.getElementsByTagName(name)
-    if not sub:
-        return None
-    return sub[0]
-
-
-def iter_children(parent, name):
-    """Yields matching child nodes within parent."""
-    for child in parent.childNodes:
-        if child.nodeType == child.ELEMENT_NODE and child.nodeName == name:
-            yield child
-
-
-def get_child(parent, name):
-    """Gets the first matching child node."""
-    for child in iter_children(parent, name):
-        return child
-    return None
-
-
-def self_or_child(n):
-    """Gets the first child node, or self."""
-    if len(n.childNodes) == 0:
-        return n
-    return n.childNodes[0]
-
-
-def align_tags(line):
-    tag_regex = r"\s(\*.+?\*)(?:\s|$)"
-    tags = re.findall(tag_regex, line)
-
-    if len(tags) > 0:
-        line = re.sub(tag_regex, "", line)
-        tags = " " + " ".join(tags)
-        line = line + (" " * (78 - len(line) - len(tags))) + tags
-    return line
-
-
-def clean_lines(text):
-    """Removes superfluous lines.
-
-    The beginning and end of the string is trimmed.  Empty lines are collapsed.
-    """
-    return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text))
-
-
-def is_blank(text):
-    return '' == clean_lines(text)
-
-
-def get_text(n):
-    """Recursively concatenates all text in a node tree."""
-    text = ''
-    if n.nodeType == n.TEXT_NODE:
-        return n.data
-    if n.nodeName == 'computeroutput':
-        for node in n.childNodes:
-            text += get_text(node)
-        return '`{}`'.format(text)
-    if n.nodeName == 'sp': # space, used in "programlisting" nodes
-        return ' '
-    for node in n.childNodes:
-        if node.nodeType == node.TEXT_NODE:
-            text += node.data
-        elif node.nodeType == node.ELEMENT_NODE:
-            text += get_text(node)
-    return text
-
-
-# Gets the length of the last line in `text`, excluding newline ("\n") char.
-def len_lastline(text):
-    lastnl = text.rfind('\n')
-    if -1 == lastnl:
-        return len(text)
-    if '\n' == text[-1]:
-        return lastnl - (1 + text.rfind('\n', 0, lastnl))
-    return len(text) - (1 + lastnl)
-
-
-def len_lastline_withoutindent(text, indent):
-    n = len_lastline(text)
-    return (n - len(indent)) if n > len(indent) else 0
-
-
-# Returns True if node `n` contains only inline (not block-level) elements.
-def is_inline(n):
-    # if len(n.childNodes) == 0:
-    #     return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput'
-    for c in n.childNodes:
-        if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput':
-            return False
-        if not is_inline(c):
-            return False
-    return True
-
-
-def doc_wrap(text, prefix='', width=70, func=False, indent=None) -> str:
-    """Wraps text to `width`.
-
-    First line is prefixed with `prefix`, subsequent lines are aligned.
-    If `func` is True, only wrap at commas.
-    """
-    if not width:
-        # return prefix + text
-        return text
-
-    # Whitespace used to indent all lines except the first line.
-    indent = ' ' * len(prefix) if indent is None else indent
-    indent_only = (prefix == '' and indent is not None)
-
-    if func:
-        lines = [prefix]
-        for part in text.split(', '):
-            if part[-1] not in ');':
-                part += ', '
-            if len(lines[-1]) + len(part) > width:
-                lines.append(indent)
-            lines[-1] += part
-        return '\n'.join(x.rstrip() for x in lines).rstrip()
-
-    # XXX: Dummy prefix to force TextWrapper() to wrap the first line.
-    if indent_only:
-        prefix = indent
-
-    tw = textwrap.TextWrapper(break_long_words=False,
-                              break_on_hyphens=False,
-                              width=width,
-                              initial_indent=prefix,
-                              subsequent_indent=indent)
-    result = '\n'.join(tw.wrap(text.strip()))
-
-    # XXX: Remove the dummy prefix.
-    if indent_only:
-        result = result[len(indent):]
-
-    return result
-
-
-def max_name(names):
-    if len(names) == 0:
-        return 0
-    return max(len(name) for name in names)
-
-
-def update_params_map(parent, ret_map, width=text_width - indentation):
-    """Updates `ret_map` with name:desc key-value pairs extracted
-    from Doxygen XML node `parent`.
-    """
-    params = collections.OrderedDict()
-    for node in parent.childNodes:
-        if node.nodeType == node.TEXT_NODE:
-            continue
-        name_node = find_first(node, 'parametername')
-        if name_node.getAttribute('direction') == 'out':
-            continue
-        name = get_text(name_node)
-        if name in param_exclude:
-            continue
-        params[name.strip()] = node
-    max_name_len = max_name(params.keys()) + 8
-    # `ret_map` is a name:desc map.
-    for name, node in params.items():
-        desc = ''
-        desc_node = get_child(node, 'parameterdescription')
-        if desc_node:
-            desc = fmt_node_as_vimhelp(
-                    desc_node, width=width, indent=(' ' * max_name_len))
-            ret_map[name] = desc
-    return ret_map
-
-
-def render_node(n: Element, text: str, prefix='', *,
-                indent: str = '',
-                width: int = (text_width - indentation),
-                fmt_vimhelp: bool = False):
-    """Renders a node as Vim help text, recursively traversing all descendants."""
-
-    def ind(s):
-        return s if fmt_vimhelp else ''
-
-    # Get the current column offset from the last line of `text`
-    # (needed to appropriately wrap multiple and contiguous inline elements)
-    col_offset: int = len_lastline(text)
-
-    text = ''
-    # space_preceding = (len(text) > 0 and ' ' == text[-1][-1])
-    # text += (int(not space_preceding) * ' ')
-
-    if n.nodeName == 'preformatted':
-        o = get_text(n)
-        ensure_nl = '' if o[-1] == '\n' else '\n'
-        if o[0:4] == 'lua\n':
-            text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1])
-        elif o[0:4] == 'vim\n':
-            text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1])
-        elif o[0:5] == 'help\n':
-            text += o[4:-1]
-        else:
-            text += '>{}{}\n<'.format(ensure_nl, o)
-    elif n.nodeName == 'programlisting': # codeblock (```)
-        o = get_text(n)
-        text += '>'
-        if 'filename' in n.attributes:
-            filename = n.attributes['filename'].value
-            text += filename.lstrip('.')
-
-        text += '\n{}\n<'.format(textwrap.indent(o, ' ' * 4))
-    elif is_inline(n):
-        o = get_text(n).strip()
-        if o:
-            DEL = chr(127)  # a dummy character to pad for proper line wrap
-            assert len(DEL) == 1
-            dummy_padding = DEL * max(0, col_offset - len(prefix))
-            text += doc_wrap(dummy_padding + o,
-                             prefix=prefix, indent=indent, width=width
-                            ).replace(DEL, "")
-    elif n.nodeName == 'verbatim':
-        # TODO: currently we don't use this. The "[verbatim]" hint is there as
-        # a reminder that we must decide how to format this if we do use it.
-        text += ' [verbatim] {}'.format(get_text(n))
-    elif n.nodeName == 'listitem':
-        for c in n.childNodes:
-            result = render_node(
-                c,
-                text,
-                indent=indent + (' ' * len(prefix)),
-                width=width
-            )
-            if is_blank(result):
-                continue
-            text += indent + prefix + result
-    elif n.nodeName in ('para', 'heading'):
-        did_prefix = False
-        for c in n.childNodes:
-            c_text = render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width)
-            if (is_inline(c)
-                    and '' != c_text.strip()
-                    and text
-                    and text[-1] not in (' ', '(', '|')
-                    and not c_text.startswith(')')):
-                text += ' '
-            text += c_text
-            did_prefix = True
-    elif n.nodeName == 'itemizedlist':
-        for c in n.childNodes:
-            text += '{}\n'.format(render_node(c, text, prefix='• ',
-                                              indent=indent, width=width))
-    elif n.nodeName == 'orderedlist':
-        i = 1
-        for c in n.childNodes:
-            if is_blank(get_text(c)):
-                text += '\n'
-                continue
-            text += '{}\n'.format(render_node(c, text, prefix='{}. '.format(i),
-                                              indent=indent, width=width))
-            i = i + 1
-    elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'):
-        text += ind('  ')
-        for c in n.childNodes:
-            if is_blank(render_node(c, text, prefix='• ', indent='    ', width=width)):
-                continue
-            text += render_node(c, text, prefix='• ', indent='    ', width=width)
-        # text += '\n'
-    elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'):
-        text += 'Warning:\n    '
-        for c in n.childNodes:
-            text += render_node(c, text, indent='    ', width=width)
-        text += '\n'
-    elif n.nodeName == 'simplesect' and 'see' == n.getAttribute('kind'):
-        text += ind('  ')
-        # Example:
-        #   <simplesect kind="see">
-        #     <para>|autocommand|</para>
-        #   </simplesect>
-        for c in n.childNodes:
-            text += render_node(c, text, prefix='• ', indent='    ', width=width)
-    elif n.nodeName == 'simplesect' and 'return' == n.getAttribute('kind'):
-        text += ind('    ')
-        for c in n.childNodes:
-            text += render_node(c, text, indent='    ', width=width)
-    elif n.nodeName == 'computeroutput':
-        return get_text(n)
-    else:
-        raise RuntimeError('unhandled node type: {}\n{}'.format(
-            n.nodeName, n.toprettyxml(indent='  ', newl='\n')))
-
-    return text
-
-
-def para_as_map(parent: Element,
-                indent: str = '',
-                width: int = (text_width - indentation),
-                ):
-    """Extracts a Doxygen XML <para> node to a map.
-
-    Keys:
-        'text': Text from this <para> element
-        'note': List of @note strings
-        'params': <parameterlist> map
-        'return': List of @return strings
-        'seealso': List of @see strings
-        'xrefs': ?
-    """
-    chunks = {
-        'text': '',
-        'note': [],
-        'params': collections.OrderedDict(),
-        'return': [],
-        'seealso': [],
-        'prerelease': False,
-        'xrefs': []
-    }
-
-    # Ordered dict of ordered lists.
-    groups = collections.OrderedDict([
-        ('note', []),
-        ('params', []),
-        ('return', []),
-        ('seealso', []),
-        ('xrefs', []),
-    ])
-
-    # Gather nodes into groups.  Mostly this is because we want "parameterlist"
-    # nodes to appear together.
-    text = ''
-    kind = ''
-    if is_inline(parent):
-        # Flatten inline text from a tree of non-block nodes.
-        text = doc_wrap(render_node(parent, ""),
-                        indent=indent, width=width)
-    else:
-        prev = None  # Previous node
-        for child in parent.childNodes:
-            if child.nodeName == 'parameterlist':
-                groups['params'].append(child)
-            elif child.nodeName == 'xrefsect':
-                groups['xrefs'].append(child)
-            elif child.nodeName == 'simplesect':
-                kind = child.getAttribute('kind')
-                if kind == 'note':
-                    groups['note'].append(child)
-                elif kind == 'return':
-                    groups['return'].append(child)
-                elif kind == 'see':
-                    groups['seealso'].append(child)
-                elif kind == 'warning':
-                    text += render_node(child, text, indent=indent, width=width)
-                elif kind == 'since':
-                    since_match = re.match(r'^(\d+)', get_text(child))
-                    since = int(since_match.group(1)) if since_match else 0
-                    NVIM_API_LEVEL, NVIM_API_PRERELEASE = nvim_api_info()
-                    if since > NVIM_API_LEVEL or (
-                        since == NVIM_API_LEVEL and NVIM_API_PRERELEASE
-                    ):
-                        chunks['prerelease'] = True
-                else:
-                    raise RuntimeError('unhandled simplesect: {}\n{}'.format(
-                        child.nodeName, child.toprettyxml(indent='  ', newl='\n')))
-            else:
-                child_text = render_node(child, text, indent=indent, width=width)
-                if (prev is not None
-                        and is_inline(self_or_child(prev))
-                        and is_inline(self_or_child(child))
-                        and '' != get_text(self_or_child(child)).strip()
-                        and text
-                        and text[-1] not in (' ', '(', '|')
-                        and not child_text.startswith(')')):
-                    text += ' '
-
-                text += child_text
-                prev = child
-
-    chunks['text'] += text
-
-    # Generate map from the gathered items.
-    if len(groups['params']) > 0:
-        for child in groups['params']:
-            update_params_map(child, ret_map=chunks['params'], width=width)
-    for child in groups['note']:
-        chunks['note'].append(render_node(
-            child, '', indent=indent, width=width).rstrip())
-    for child in groups['return']:
-        chunks['return'].append(render_node(
-            child, '', indent=indent, width=width))
-    for child in groups['seealso']:
-        # Example:
-        #   <simplesect kind="see">
-        #     <para>|autocommand|</para>
-        #   </simplesect>
-        chunks['seealso'].append(render_node(
-            child, '', indent=indent, width=width))
-
-    xrefs = set()
-    for child in groups['xrefs']:
-        # XXX: Add a space (or any char) to `title` here, otherwise xrefs
-        # ("Deprecated" section) acts very weird...
-        title = get_text(get_child(child, 'xreftitle')) + ' '
-        xrefs.add(title)
-        xrefdesc = get_text(get_child(child, 'xrefdescription'))
-        chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title),
-                                        width=width) + '\n')
-
-    return chunks, xrefs
-
-
-def is_program_listing(para):
-    """
-    Return True if `para` contains a "programlisting" (i.e. a Markdown code
-    block ```).
-
-    Sometimes a <para> element will have only a single "programlisting" child
-    node, but othertimes it will have extra whitespace around the
-    "programlisting" node.
-
-    @param para XML <para> node
-    @return True if <para> is a programlisting
-    """
-
-    # Remove any child text nodes that are only whitespace
-    children = [
-        n for n in para.childNodes
-        if n.nodeType != n.TEXT_NODE or n.data.strip() != ''
-    ]
-
-    return len(children) == 1 and children[0].nodeName == 'programlisting'
-
-
-FunctionParam = Tuple[
-    str,  # type
-    str,  # parameter name
-]
-
-@dataclasses.dataclass
-class FunctionDoc:
-    """Data structure for function documentation. Also exported as msgpack."""
-
-    annotations: List[str]
-    """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map"""
-
-    notes: List[Docstring]
-    """Notes: (@note strings)"""
-
-    signature: str
-    """Function signature with *tags*."""
-
-    parameters: List[FunctionParam]
-    """Parameters: (type, name)"""
-
-    parameters_doc: Dict[str, Docstring]
-    """Parameters documentation. Key is parameter name, value is doc."""
-
-    doc: List[Docstring]
-    """Main description for the function. Separated by paragraph."""
-
-    return_: List[Docstring]
-    """Return:, or Return (multiple): (@return strings)"""
-
-    seealso: List[Docstring]
-    """See also: (@see strings)"""
-
-    xrefs: List[Docstring]
-    """XRefs. Currently only used to track Deprecated functions."""
-
-    # for INCLUDE_C_DECL
-    c_decl: str | None = None
-
-    prerelease: bool = False
-
-    def export_mpack(self) -> Dict[str, Any]:
-        """Convert a dict to be exported as mpack data."""
-        exported = self.__dict__.copy()
-        del exported['notes']
-        del exported['c_decl']
-        del exported['prerelease']
-        del exported['xrefs']
-        exported['return'] = exported.pop('return_')
-        return exported
-
-    def doc_concatenated(self) -> Docstring:
-        """Concatenate all the paragraphs in `doc` into a single string, but
-        remove blank lines before 'programlisting' blocks. #25127
-
-        BEFORE (without programlisting processing):
-            ```vimdoc
-            Example:
-
-            >vim
-                :echo nvim_get_color_by_name("Pink")
-            <
-            ```
-
-        AFTER:
-            ```vimdoc
-            Example: >vim
-                :echo nvim_get_color_by_name("Pink")
-            <
-            ```
-        """
-        def is_program_listing(paragraph: str) -> bool:
-            lines = paragraph.strip().split('\n')
-            return lines[0].startswith('>') and lines[-1] == '<'
-
-        rendered = []
-        for paragraph in self.doc:
-            if is_program_listing(paragraph):
-                rendered.append(' ')  # Example: >vim