feat(docs): replace lua2dox.lua

Problem:

The documentation flow (`gen_vimdoc.py`) has several issues:
- it's not very versatile
- depends on doxygen
- doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C.
- The intermediate XML files and filters makes it too much like a rube goldberg machine.

Solution:

Re-implement the flow using Lua, LPEG and treesitter.

- `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic.
- `lua2dox.lua` is gone!
- No more XML files.
- Doxygen is now longer used and instead we now use:
  - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`).
  - LPEG for C parsing (see `scripts/cdoc_parser.lua`)
  - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`).
  - Treesitter for Markdown parsing (see `scripts/text_utils.lua`).
- The generated `runtime/doc/*.mpack` files have been removed.
   - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly.
- Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change).

16 files changed, 141 insertions, 236 deletions

diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
index 54ad43e8f3..5ec0e4d9ec 100644
--- a/src/nvim/CMakeLists.txt
+++ b/src/nvim/CMakeLists.txt
@@ -902,14 +902,6 @@ set(VIMDOC_FILES
   ${NVIM_RUNTIME_DIR}/doc/treesitter.txt
 )
 
-set(MPACK_FILES
-  ${NVIM_RUNTIME_DIR}/doc/api.mpack
-  ${NVIM_RUNTIME_DIR}/doc/diagnostic.mpack
-  ${NVIM_RUNTIME_DIR}/doc/lsp.mpack
-  ${NVIM_RUNTIME_DIR}/doc/lua.mpack
-  ${NVIM_RUNTIME_DIR}/doc/treesitter.mpack
-)
-
 file(GLOB API_SOURCES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/src/nvim/api/*.c)
 
 file(GLOB LUA_SOURCES CONFIGURE_DEPENDS
@@ -921,25 +913,25 @@ file(GLOB LUA_SOURCES CONFIGURE_DEPENDS
 )
 
 add_custom_command(
-  OUTPUT ${MPACK_FILES}
-  COMMAND ${PROJECT_SOURCE_DIR}/scripts/gen_vimdoc.py
+  OUTPUT ${VIMDOC_FILES}
+  COMMAND ${CMAKE_COMMAND} -E env "VIMRUNTIME=${NVIM_RUNTIME_DIR}"
+    $<TARGET_FILE:nvim_bin> -l scripts/gen_vimdoc.lua
   DEPENDS
-    nvim_bin
+    nvim
     ${API_SOURCES}
     ${LUA_SOURCES}
-    ${VIMDOC_FILES}
-    ${PROJECT_SOURCE_DIR}/scripts/gen_vimdoc.py
-    ${PROJECT_SOURCE_DIR}/scripts/lua2dox.lua
+    ${PROJECT_SOURCE_DIR}/scripts/gen_vimdoc.lua
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
 )
 
 add_custom_command(
   OUTPUT ${GEN_EVAL_TOUCH}
   COMMAND ${CMAKE_COMMAND} -E touch ${GEN_EVAL_TOUCH}
-  COMMAND $<TARGET_FILE:nvim_bin> -l ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua
+  COMMAND ${CMAKE_COMMAND} -E env "VIMRUNTIME=${NVIM_RUNTIME_DIR}"
+    $<TARGET_FILE:nvim_bin> -l ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua
   DEPENDS
+    nvim
     ${API_METADATA}
-    ${NVIM_RUNTIME_DIR}/doc/api.mpack
     ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua
     ${PROJECT_SOURCE_DIR}/src/nvim/eval.lua
     ${PROJECT_SOURCE_DIR}/src/nvim/options.lua
diff --git a/src/nvim/api/autocmd.c b/src/nvim/api/autocmd.c
index 7e33a77d24..06f21919f9 100644
--- a/src/nvim/api/autocmd.c
+++ b/src/nvim/api/autocmd.c
@@ -75,7 +75,7 @@ static int64_t next_autocmd_id = 1;
 ///             - buffer: Buffer number or list of buffer numbers for buffer local autocommands
 ///             |autocmd-buflocal|. Cannot be used with {pattern}
 /// @return Array of autocommands matching the criteria, with each item
-///         containing the following fields:
+///             containing the following fields:
 ///             - id (number): the autocommand id (only when defined with the API).
 ///             - group (integer): the autocommand group id.
 ///             - group_name (string): the autocommand group name.
@@ -83,10 +83,10 @@ static int64_t next_autocmd_id = 1;
 ///             - event (string): the autocommand event.
 ///             - command (string): the autocommand command. Note: this will be empty if a callback is set.
 ///             - callback (function|string|nil): Lua function or name of a Vim script function
-///             which is executed when this autocommand is triggered.
+///               which is executed when this autocommand is triggered.
 ///             - once (boolean): whether the autocommand is only run once.
 ///             - pattern (string): the autocommand pattern.
-///             If the autocommand is buffer local |autocmd-buffer-local|:
+///               If the autocommand is buffer local |autocmd-buffer-local|:
 ///             - buflocal (boolean): true if the autocommand is buffer local.
 ///             - buffer (number): the buffer number.
 Array nvim_get_autocmds(Dict(get_autocmds) *opts, Arena *arena, Error *err)
@@ -536,9 +536,9 @@ void nvim_del_autocmd(Integer id, Error *err)
 /// @param opts Parameters
 ///         - event: (string|table)
 ///              Examples:
-///                 - event: "pat1"
-///                 - event: { "pat1" }
-///                 - event: { "pat1", "pat2", "pat3" }
+///              - event: "pat1"
+///              - event: { "pat1" }
+///              - event: { "pat1", "pat2", "pat3" }
 ///         - pattern: (string|table)
 ///             - pattern or patterns to match exactly.
 ///                 - For example, if you have `*.py` as that pattern for the autocmd,
diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c
index 41a3743895..7f195de959 100644
--- a/src/nvim/api/buffer.c
+++ b/src/nvim/api/buffer.c
@@ -50,9 +50,8 @@
 # include "api/buffer.c.generated.h"
 #endif
 
-/// \defgroup api-buffer
-///
-/// \brief For more information on buffers, see |buffers|
+/// @brief <pre>help
+/// For more information on buffers, see |buffers|.
 ///
 /// Unloaded Buffers: ~
 ///
@@ -64,6 +63,7 @@
 ///
 /// You can use |nvim_buf_is_loaded()| or |nvim_buf_line_count()| to check
 /// whether a buffer is loaded.
+/// </pre>
 
 /// Returns the number of lines in the given buffer.
 ///
@@ -229,6 +229,7 @@ Boolean nvim_buf_detach(uint64_t channel_id, Buffer buffer, Error *err)
   return true;
 }
 
+/// @nodoc
 void nvim__buf_redraw_range(Buffer buffer, Integer first, Integer last, Error *err)
 {
   buf_T *buf = find_buffer_by_handle(buffer, err);
@@ -875,8 +876,8 @@ Integer nvim_buf_get_changedtick(Buffer buffer, Error *err)
 
 /// Gets a list of buffer-local |mapping| definitions.
 ///
-/// @param  mode       Mode short-name ("n", "i", "v", ...)
 /// @param  buffer     Buffer handle, or 0 for current buffer
+/// @param  mode       Mode short-name ("n", "i", "v", ...)
 /// @param[out]  err   Error details, if any
 /// @returns Array of |maparg()|-like dictionaries describing mappings.
 ///          The "buffer" key holds the associated buffer handle.
@@ -1223,6 +1224,7 @@ Object nvim_buf_call(Buffer buffer, LuaRef fun, Error *err)
   return res;
 }
 
+/// @nodoc
 Dictionary nvim__buf_stats(Buffer buffer, Arena *arena, Error *err)
 {
   buf_T *buf = find_buffer_by_handle(buffer, err);
diff --git a/src/nvim/api/command.c b/src/nvim/api/command.c
index f2d5342a5f..e08035b0eb 100644
--- a/src/nvim/api/command.c
+++ b/src/nvim/api/command.c
@@ -67,7 +67,7 @@
 ///             - file: (boolean) The command expands filenames. Which means characters such as "%",
 ///                               "#" and wildcards are expanded.
 ///             - bar: (boolean) The "|" character is treated as a command separator and the double
-///                              quote character (\") is treated as the start of a comment.
+///                              quote character (") is treated as the start of a comment.
 ///         - mods: (dictionary) |:command-modifiers|.
 ///             - filter: (dictionary) |:filter|.
 ///                 - pattern: (string) Filter pattern. Empty string if there is no filter.
diff --git a/src/nvim/api/deprecated.c b/src/nvim/api/deprecated.c
index 166b04adf0..6254e9fbd8 100644
--- a/src/nvim/api/deprecated.c
+++ b/src/nvim/api/deprecated.c
@@ -61,8 +61,7 @@ Object nvim_execute_lua(String code, Array args, Arena *arena, Error *err)
 
 /// Gets the buffer number
 ///
-/// @deprecated The buffer number now is equal to the object id,
-///             so there is no need to use this function.
+/// @deprecated The buffer number now is equal to the object id
 ///
 /// @param buffer     Buffer handle, or 0 for current buffer
 /// @param[out] err   Error details, if any
@@ -100,8 +99,7 @@ void nvim_buf_clear_highlight(Buffer buffer, Integer ns_id, Integer line_start,
 
 /// Set the virtual text (annotation) for a buffer line.
 ///
-/// @deprecated use nvim_buf_set_extmark to use full virtual text
-///             functionality.
+/// @deprecated use nvim_buf_set_extmark to use full virtual text functionality.
 ///
 /// The text will be placed after the buffer text. Virtual text will never
 /// cause reflow, rather virtual text will be truncated at the end of the screen
@@ -119,7 +117,7 @@ void nvim_buf_clear_highlight(Buffer buffer, Integer ns_id, Integer line_start,
 /// virtual text, the allocated id is then returned.
 ///
 /// @param buffer     Buffer handle, or 0 for current buffer
-/// @param ns_id      Namespace to use or 0 to create a namespace,
+/// @param src_id     Namespace to use or 0 to create a namespace,
 ///                   or -1 for a ungrouped annotation
 /// @param line       Line to annotate with virtual text (zero-indexed)
 /// @param chunks     A list of [text, hl_group] arrays, each representing a
diff --git a/src/nvim/api/extmark.c b/src/nvim/api/extmark.c
index 653140a7ae..e35840915f 100644
--- a/src/nvim/api/extmark.c
+++ b/src/nvim/api/extmark.c
@@ -1205,6 +1205,7 @@ free_exit:
   return virt_text;
 }
 
+/// @nodoc
 String nvim__buf_debug_extmarks(Buffer buffer, Boolean keys, Boolean dot, Error *err)
   FUNC_API_SINCE(7) FUNC_API_RET_ALLOC
 {
diff --git a/src/nvim/api/options.c b/src/nvim/api/options.c
index 7deadb8eb5..d9bc0ccc92 100644
--- a/src/nvim/api/options.c
+++ b/src/nvim/api/options.c
@@ -272,21 +272,21 @@ Dictionary nvim_get_all_options_info(Arena *arena, Error *err)
 /// Gets the option information for one option from arbitrary buffer or window
 ///
 /// Resulting dictionary has keys:
-///     - name: Name of the option (like 'filetype')
-///     - shortname: Shortened name of the option (like 'ft')
-///     - type: type of option ("string", "number" or "boolean")
-///     - default: The default value for the option
-///     - was_set: Whether the option was set.
+/// - name: Name of the option (like 'filetype')
+/// - shortname: Shortened name of the option (like 'ft')
+/// - type: type of option ("string", "number" or "boolean")
+/// - default: The default value for the option
+/// - was_set: Whether the option was set.
 ///
-///     - last_set_sid: Last set script id (if any)
-///     - last_set_linenr: line number where option was set
-///     - last_set_chan: Channel where option was set (0 for local)
+/// - last_set_sid: Last set script id (if any)
+/// - last_set_linenr: line number where option was set
+/// - last_set_chan: Channel where option was set (0 for local)
 ///
-///     - scope: one of "global", "win", or "buf"
-///     - global_local: whether win or buf option has a global value
+/// - scope: one of "global", "win", or "buf"
+/// - global_local: whether win or buf option has a global value
 ///
-///     - commalist: List of comma separated values
-///     - flaglist: List of single char flags
+/// - commalist: List of comma separated values
+/// - flaglist: List of single char flags
 ///
 /// When {scope} is not provided, the last set information applies to the local
 /// value in the current buffer or window if it is available, otherwise the
diff --git a/src/nvim/api/ui.c b/src/nvim/api/ui.c
index d5843caa96..0f016d2f29 100644
--- a/src/nvim/api/ui.c
+++ b/src/nvim/api/ui.c
@@ -165,7 +165,7 @@ void remote_ui_wait_for_attach(bool only_stdio)
 
 /// Activates UI events on the channel.
 ///
-/// Entry point of all UI clients.  Allows |\-\-embed| to continue startup.
+/// Entry point of all UI clients.  Allows |--embed| to continue startup.
 /// Implies that the client is ready to show the UI.  Adds the client to the
 /// list of UIs. |nvim_list_uis()|
 ///
@@ -541,7 +541,7 @@ void nvim_ui_pum_set_bounds(uint64_t channel_id, Float width, Float height, Floa
 ///
 /// @param channel_id
 /// @param event Event name
-/// @param payload Event payload
+/// @param value Event payload
 /// @param[out] err Error details, if any.
 void nvim_ui_term_event(uint64_t channel_id, String event, Object value, Error *err)
   FUNC_API_SINCE(12) FUNC_API_REMOTE_ONLY
diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index b3d38cde69..54bf290e6b 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -123,7 +123,7 @@ Dictionary nvim_get_hl(Integer ns_id, Dict(get_highlight) *opts, Arena *arena, E
 ///
 /// @note Unlike the `:highlight` command which can update a highlight group,
 ///       this function completely replaces the definition. For example:
-///       ``nvim_set_hl(0, 'Visual', {})`` will clear the highlight group
+///       `nvim_set_hl(0, 'Visual', {})` will clear the highlight group
 ///       'Visual'.
 ///
 /// @note The fg and bg keys also accept the string values `"fg"` or `"bg"`
@@ -549,6 +549,7 @@ ArrayOf(String) nvim_list_runtime_paths(Arena *arena, Error *err)
   return nvim_get_runtime_file(NULL_STRING, true, arena, err);
 }
 
+/// @nodoc
 Array nvim__runtime_inspect(Arena *arena)
 {
   return runtime_inspect(arena);
@@ -600,6 +601,7 @@ static bool find_runtime_cb(int num_fnames, char **fnames, bool all, void *c)
   return num_fnames > 0;
 }
 
+/// @nodoc
 String nvim__get_lib_dir(void)
   FUNC_API_RET_ALLOC
 {
@@ -1547,14 +1549,14 @@ Array nvim_get_api_info(uint64_t channel_id, Arena *arena)
 /// @param channel_id
 /// @param name Short name for the connected client
 /// @param version  Dictionary describing the version, with these
-///                 (optional) keys:
+///     (optional) keys:
 ///     - "major" major version (defaults to 0 if not set, for no release yet)
 ///     - "minor" minor version
 ///     - "patch" patch number
 ///     - "prerelease" string describing a prerelease, like "dev" or "beta1"
 ///     - "commit" hash or similar identifier of commit
 /// @param type Must be one of the following values. Client libraries should
-///             default to "remote" unless overridden by the user.
+///     default to "remote" unless overridden by the user.
 ///     - "remote" remote client connected "Nvim flavored" MessagePack-RPC (responses
 ///                must be in reverse order of requests). |msgpack-rpc|
 ///     - "msgpack-rpc" remote client connected to Nvim via fully MessagePack-RPC
@@ -1565,12 +1567,12 @@ Array nvim_get_api_info(uint64_t channel_id, Arena *arena)
 ///     - "host" plugin host, typically started by nvim
 ///     - "plugin" single plugin, started by nvim
 /// @param methods Builtin methods in the client. For a host, this does not
-///                include plugin methods which will be discovered later.
-///                The key should be the method name, the values are dicts with
-///                these (optional) keys (more keys may be added in future
-///                versions of Nvim, thus unknown keys are ignored. Clients
-///                must only use keys defined in this or later versions of
-///                Nvim):
+///     include plugin methods which will be discovered later.
+///     The key should be the method name, the values are dicts with
+///     these (optional) keys (more keys may be added in future
+///     versions of Nvim, thus unknown keys are ignored. Clients
+///     must only use keys defined in this or later versions of
+///     Nvim):
 ///     - "async"  if true, send as a notification. If false or unspecified,
 ///                use a blocking request
 ///     - "nargs" Number of arguments. Could be a single integer or an array
@@ -1979,7 +1981,7 @@ void nvim_select_popupmenu_item(Integer item, Boolean insert, Boolean finish, Di
   pum_ext_select_item((int)item, insert, finish);
 }
 
-/// NB: if your UI doesn't use hlstate, this will not return hlstate first time
+/// NB: if your UI doesn't use hlstate, this will not return hlstate first time.
 Array nvim__inspect_cell(Integer grid, Integer row, Integer col, Arena *arena, Error *err)
 {
   Array ret = ARRAY_DICT_INIT;
@@ -2015,6 +2017,7 @@ Array nvim__inspect_cell(Integer grid, Integer row, Integer col, Arena *arena, E
   return ret;
 }
 
+/// @nodoc
 void nvim__screenshot(String path)
   FUNC_API_FAST
 {
@@ -2029,6 +2032,7 @@ void nvim__invalidate_glyph_cache(void)
   must_redraw = UPD_CLEAR;
 }
 
+/// @nodoc
 Object nvim__unpack(String str, Arena *arena, Error *err)
   FUNC_API_FAST
 {
@@ -2319,6 +2323,7 @@ Dictionary nvim_eval_statusline(String str, Dict(eval_statusline) *opts, Arena *
   return result;
 }
 
+/// @nodoc
 void nvim_error_event(uint64_t channel_id, Integer lvl, String data)
   FUNC_API_REMOTE_ONLY
 {
diff --git a/src/nvim/api/vimscript.c b/src/nvim/api/vimscript.c
index 7239971c21..8f57e61c76 100644
--- a/src/nvim/api/vimscript.c
+++ b/src/nvim/api/vimscript.c
@@ -351,9 +351,7 @@ typedef struct {
   Object *ret_node_p;
 } ExprASTConvStackItem;
 
-/// @cond DOXYGEN_NOT_A_FUNCTION
 typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack;
-/// @endcond
 
 /// Parse a Vimscript expression.
 ///
@@ -387,8 +385,8 @@ typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack;
 ///          - "arg": String, error message argument.
 ///        - "len": Amount of bytes successfully parsed. With flags equal to ""
 ///                 that should be equal to the length of expr string.
-///                 (“Successfully parsed” here means “participated in AST
-///                  creation”, not “till the first error”.)
+///                 ("Successfully parsed" here means "participated in AST
+///                  creation", not "till the first error".)
 ///        - "ast": AST, either nil or a dictionary with these keys:
 ///          - "type": node type, one of the value names from ExprASTNodeType
 ///                    stringified without "kExprNode" prefix.
diff --git a/src/nvim/api/win_config.c b/src/nvim/api/win_config.c
index e76db82c61..3cc520dc78 100644
--- a/src/nvim/api/win_config.c
+++ b/src/nvim/api/win_config.c
@@ -116,12 +116,12 @@
 ///   - width: Window width (in character cells). Minimum of 1.
 ///   - height: Window height (in character cells). Minimum of 1.
 ///   - bufpos: Places float relative to buffer text (only when
-///               relative="win"). Takes a tuple of zero-indexed [line, column].
-///               `row` and `col` if given are applied relative to this
-///               position, else they default to:
-///               - `row=1` and `col=0` if `anchor` is "NW" or "NE"
-///               - `row=0` and `col=0` if `anchor` is "SW" or "SE"
-///               (thus like a tooltip near the buffer text).
+///       relative="win"). Takes a tuple of zero-indexed [line, column].
+///       `row` and `col` if given are applied relative to this
+///       position, else they default to:
+///       - `row=1` and `col=0` if `anchor` is "NW" or "NE"
+///       - `row=0` and `col=0` if `anchor` is "SW" or "SE"
+///         (thus like a tooltip near the buffer text).
 ///   - row: Row position in units of "screen cell height", may be fractional.
 ///   - col: Column position in units of "screen cell width", may be
 ///            fractional.
@@ -153,7 +153,7 @@
 ///                    'fillchars' to a space char, and clearing the
 ///                    |hl-EndOfBuffer| region in 'winhighlight'.
 ///   - border: Style of (optional) window border. This can either be a string
-///      or an array. The string values are
+///     or an array. The string values are
 ///     - "none": No border (default).
 ///     - "single": A single line box.
 ///     - "double": A double line box.
@@ -161,21 +161,31 @@
 ///     - "solid": Adds padding by a single whitespace cell.
 ///     - "shadow": A drop shadow effect by blending with the background.
 ///     - If it is an array, it should have a length of eight or any divisor of
-///     eight. The array will specify the eight chars building up the border
-///     in a clockwise fashion starting with the top-left corner. As an
-///     example, the double box style could be specified as
+///       eight. The array will specify the eight chars building up the border
+///       in a clockwise fashion starting with the top-left corner. As an
+///       example, the double box style could be specified as:
+///       ```
 ///       [ "╔", "═" ,"╗", "║", "╝", "═", "╚", "║" ].
-///     If the number of chars are less than eight, they will be repeated. Thus
-///     an ASCII border could be specified as
+///       ```
+///       If the number of chars are less than eight, they will be repeated. Thus
+///       an ASCII border could be specified as
+///       ```
 ///       [ "/", "-", \"\\\\\", "|" ],
-///     or all chars the same as
+///       ```
+///       or all chars the same as
+///       ```
 ///       [ "x" ].
+///       ```
 ///     An empty string can be used to turn off a specific border, for instance,
+///     ```
 ///       [ "", "", "", ">", "", "", "", "<" ]
+///     ```
 ///     will only make vertical borders but not horizontal ones.
 ///     By default, `FloatBorder` highlight is used, which links to `WinSeparator`
 ///     when not defined.  It could also be specified by character:
+///     ```
 ///       [ ["+", "MyCorner"], ["x", "MyBorder"] ].
+///     ```
 ///   - title: Title (optional) in window border, string or list.
 ///     List should consist of `[text, highlight]` tuples.
 ///     If string, the default highlight group is `FloatTitle`.
diff --git a/src/nvim/api/window.c b/src/nvim/api/window.c
index c41c5d4b07..ed51eedf1b 100644
--- a/src/nvim/api/window.c
+++ b/src/nvim/api/window.c
@@ -452,6 +452,7 @@ Object nvim_win_call(Window window, LuaRef fun, Error *err)
 ///
 /// This takes precedence over the 'winhighlight' option.
 ///
+/// @param window
 /// @param ns_id the namespace to use
 /// @param[out] err Error details, if any
 void nvim_win_set_hl_ns(Window window, Integer ns_id, Error *err)
diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua
index 8a3e70990a..9ce9f3d7a6 100644
--- a/src/nvim/generators/c_grammar.lua
+++ b/src/nvim/generators/c_grammar.lua
@@ -6,46 +6,74 @@ local lpeg = vim.lpeg
 local P, R, S = lpeg.P, lpeg.R, lpeg.S
 local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
 
+--- @param pat vim.lpeg.Pattern
+local function rep(pat)
+  return pat ^ 0
+end
+
+--- @param pat vim.lpeg.Pattern
+local function rep1(pat)
+  return pat ^ 1
+end
+
+--- @param pat vim.lpeg.Pattern
+local function opt(pat)
+  return pat ^ -1
+end
+
 local any = P(1) -- (consume one character)
 local letter = R('az', 'AZ') + S('_$')
 local num = R('09')
 local alpha = letter + num
 local nl = P('\r\n') + P('\n')
 local not_nl = any - nl
-local ws = S(' \t') + nl
-local fill = ws ^ 0
-local c_comment = P('//') * (not_nl ^ 0)
-local c_preproc = P('#') * (not_nl ^ 0)
-local dllexport = P('DLLEXPORT') * (ws ^ 1)
-local typed_container = (P('ArrayOf(') + P('DictionaryOf(') + P('Dict('))
-  * ((any - P(')')) ^ 1)
+local space = S(' \t')
+local ws = space + nl
+local fill = rep(ws)
+local c_comment = P('//') * rep(not_nl)
+local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(not_nl), 'comment')))
+local c_preproc = P('#') * rep(not_nl)
+local dllexport = P('DLLEXPORT') * rep1(ws)
+
+local typed_container = (
+  (P('ArrayOf(') + P('DictionaryOf(') + P('Dict('))
+  * rep1(any - P(')'))
   * P(')')
-local c_id = (typed_container + (letter * (alpha ^ 0)))
+)
+
+local c_id = (typed_container + (letter * rep(alpha)))
 local c_void = P('void')
+
 local c_param_type = (
   ((P('Error') * fill * P('*') * fill) * Cc('error'))
   + ((P('Arena') * fill * P('*') * fill) * Cc('arena'))
   + ((P('lua_State') * fill * P('*') * fill) * Cc('lstate'))
-  + C((P('const ') ^ -1) * c_id * (ws ^ 1) * P('*'))
-  + (C(c_id) * (ws ^ 1))
+  + C(opt(P('const ')) * c_id * rep1(ws) * rep1(P('*')))
+  + (C(c_id) * rep1(ws))
 )
+
 local c_type = (C(c_void) * (ws ^ 1)) + c_param_type
 local c_param = Ct(c_param_type * C(c_id))
 local c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0)
 local c_params = Ct(c_void + c_param_list)
+
+local impl_line = (any - P('}')) * opt(rep(not_nl)) * nl
+
+local ignore_line = rep1(not_nl) * nl
+
+local empty_line = Ct(Cc('empty') * nl * nl)
+
 local c_proto = Ct(
-  (dllexport ^ -1)
+  Cc('proto')
+    * opt(dllexport)
+    * opt(Cg(P('static') * fill * Cc(true), 'static'))
     * Cg(c_type, 'return_type')
     * Cg(c_id, 'name')
     * fill
-    * P('(')
-    * fill
-    * Cg(c_params, 'parameters')
-    * fill
-    * P(')')
+    * (P('(') * fill * Cg(c_params, 'parameters') * fill * P(')'))
     * Cg(Cc(false), 'fast')
-    * (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1)
-    * (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(num ^ 1)) * P(')'), 'deprecated_since') ^ -1)
+    * (fill * Cg((P('FUNC_API_SINCE(') * C(rep1(num))) * P(')'), 'since') ^ -1)
+    * (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(rep1(num))) * P(')'), 'deprecated_since') ^ -1)
     * (fill * Cg((P('FUNC_API_FAST') * Cc(true)), 'fast') ^ -1)
     * (fill * Cg((P('FUNC_API_RET_ALLOC') * Cc(true)), 'ret_alloc') ^ -1)
     * (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1)
@@ -60,7 +88,7 @@ local c_proto = Ct(
     * (fill * Cg((P('FUNC_API_CLIENT_IMPL') * Cc(true)), 'client_impl') ^ -1)
     * (fill * Cg((P('FUNC_API_CLIENT_IGNORE') * Cc(true)), 'client_ignore') ^ -1)
     * fill
-    * P(';')
+    * (P(';') + (P('{') * nl + (impl_line ^ 0) * P('}')))
 )
 
 local c_field = Ct(Cg(c_id, 'type') * ws * Cg(c_id, 'name') * fill * P(';') * fill)
@@ -83,5 +111,7 @@ local c_keyset = Ct(
     * P(';')
 )
 
-local grammar = Ct((c_proto + c_comment + c_preproc + ws + c_keyset) ^ 1)
+local grammar = Ct(
+  rep1(empty_line + c_proto + cdoc_comment + c_comment + c_preproc + ws + c_keyset + ignore_line)
+)
 return { grammar = grammar, typed_container = typed_container }
diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua
index c3fc5aa0a3..56331e4162 100644
--- a/src/nvim/generators/gen_api_dispatch.lua
+++ b/src/nvim/generators/gen_api_dispatch.lua
@@ -23,9 +23,7 @@ local function_names = {}
 
 local c_grammar = require('generators.c_grammar')
 
-local function startswith(String, Start)
-  return string.sub(String, 1, string.len(Start)) == Start
-end
+local startswith = vim.startswith
 
 local function add_function(fn)
   local public = startswith(fn.name, 'nvim_') or fn.deprecated_since
@@ -112,10 +110,12 @@ for i = 6, #arg do
   local tmp = c_grammar.grammar:match(input:read('*all'))
   for j = 1, #tmp do
     local val = tmp[j]
-    if val.keyset_name then
-      add_keyset(val)
-    else
-      add_function(val)
+    if val[1] ~= 'empty' then
+      if val.keyset_name then
+        add_keyset(val)
+      else
+        add_function(val)
+      end
     end
   end
   input:close()
diff --git a/src/nvim/generators/gen_api_ui_events.lua b/src/nvim/generators/gen_api_ui_events.lua
index 89f8c654f4..697626b793 100644
--- a/src/nvim/generators/gen_api_ui_events.lua
+++ b/src/nvim/generators/gen_api_ui_events.lua
@@ -93,6 +93,10 @@ local function call_ui_event_method(output, ev)
   output:write('}\n\n')
 end
 
+events = vim.tbl_filter(function(ev)
+  return ev[1] ~= 'empty'
+end, events)
+
 for i = 1, #events do
   local ev = events[i]
   assert(ev.return_type == 'void')
diff --git a/src/nvim/generators/luacats_grammar.lua b/src/nvim/generators/luacats_grammar.lua
deleted file mode 100644
index dcccd028ce..0000000000
--- a/src/nvim/generators/luacats_grammar.lua
+++ /dev/null
@@ -1,136 +0,0 @@
---[[!
-LPEG grammar for LuaCATS
-
-Currently only partially supports:
-- @param
-- @return
-]]
-
-local lpeg = vim.lpeg
-local P, R, S = lpeg.P, lpeg.R, lpeg.S
-local Ct, Cg = lpeg.Ct, lpeg.Cg
-
---- @param x vim.lpeg.Pattern
-local function rep(x)
-  return x ^ 0
-end
-
---- @param x vim.lpeg.Pattern
-local function rep1(x)
-  return x ^ 1
-end
-
---- @param x vim.lpeg.Pattern
-local function opt(x)
-  return x ^ -1
-end
-
-local nl = P('\r\n') + P('\n')
-local ws = rep1(S(' \t') + nl)
-local fill = opt(ws)
-
-local any = P(1) -- (consume one character)
-local letter = R('az', 'AZ') + S('_$')
-local num = R('09')
-local ident = letter * rep(letter + num + S '-.')
-local string_single = P "'" * rep(any - P "'") * P "'"
-local string_double = P '"' * rep(any - P '"') * P '"'
-
-local literal = (string_single + string_double + (opt(P '-') * num) + P 'false' + P 'true')
-
-local lname = (ident + P '...') * opt(P '?')
-
---- @param x string
-local function Pf(x)
-  return fill * P(x) * fill
-end
-
---- @param x string
-local function Sf(x)
-  return fill * S(x) * fill
-end
-
---- @param x vim.lpeg.Pattern
-local function comma(x)
-  return x * rep(Pf ',' * x)
-end
-
---- @param x vim.lpeg.Pattern
-local function parenOpt(x)
-  return (Pf('(') * x ^ -1 * fill * P(')')) + x ^ -1
-end
-
---- @type table<string,vim.lpeg.Pattern>
-local v = setmetatable({}, {
-  __index = function(_, k)
-    return lpeg.V(k)
-  end,
-})
-
-local desc_delim = Sf '#:' + ws
-
---- @class luacats.Param
---- @field kind 'param'
---- @field name string
---- @field type string
---- @field desc? string
-
---- @class luacats.Return
---- @field kind 'return'
---- @field [integer] { type: string, name?: string}
---- @field desc? string
-
---- @class luacats.Generic
---- @field kind 'generic'
---- @field name string
---- @field type? string
-
---- @alias luacats.grammar.result
---- | luacats.Param
---- | luacats.Return
---- | luacats.Generic
-
---- @class luacats.grammar
---- @field match fun(self, input: string): luacats.grammar.result?
-
-local grammar = P {
-  rep1(P('@') * v.ats),
-
-  ats = (v.at_param + v.at_return + v.at_generic),
-
-  at_param = Ct(
-    Cg(P('param'), 'kind')
-      * ws
-      * Cg(lname, 'name')
-      * ws
-      * Cg(v.ltype, 'type')
-      * opt(desc_delim * Cg(rep(any), 'desc'))
-  ),
-
-  at_return = Ct(
-    Cg(P('return'), 'kind')
-      * ws
-      * parenOpt(comma(Ct(Cg(v.ltype, 'type') * opt(ws * Cg(ident, 'name')))))
-      * opt(desc_delim * Cg(rep(any), 'desc'))
-  ),
-
-  at_generic = Ct(
-    Cg(P('generic'), 'kind') * ws * Cg(ident, 'name') * opt(Pf ':' * Cg(v.ltype, 'type'))
-  ),
-
-  ltype = v.ty_union + Pf '(' * v.ty_union * fill * P ')',
-
-  ty_union = v.ty_opt * rep(Pf '|' * v.ty_opt),
-  ty = v.ty_fun + ident + v.ty_table + literal,
-  ty_param = Pf '<' * comma(v.ltype) * fill * P '>',
-  ty_opt = v.ty * opt(v.ty_param) * opt(P '[]') * opt(P '?'),
-
-  table_key = (Pf '[' * literal * Pf ']') + lname,
-  table_elem = v.table_key * Pf ':' * v.ltype,
-  ty_table = Pf '{' * comma(v.table_elem) * Pf '}',
-
-  fun_param = lname * opt(Pf ':' * v.ltype),
-  ty_fun = Pf 'fun(' * rep(comma(v.fun_param)) * fill * P ')' * opt(Pf ':' * v.ltype),
-}
-
-return grammar --[[@as luacats.grammar]]