feat(docs): replace lua2dox.lua

Problem:

The documentation flow (`gen_vimdoc.py`) has several issues:
- it's not very versatile
- depends on doxygen
- doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C.
- The intermediate XML files and filters makes it too much like a rube goldberg machine.

Solution:

Re-implement the flow using Lua, LPEG and treesitter.

- `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic.
- `lua2dox.lua` is gone!
- No more XML files.
- Doxygen is now longer used and instead we now use:
  - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`).
  - LPEG for C parsing (see `scripts/cdoc_parser.lua`)
  - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`).
  - Treesitter for Markdown parsing (see `scripts/text_utils.lua`).
- The generated `runtime/doc/*.mpack` files have been removed.
   - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly.
- Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change).

8 files changed, 36 insertions, 26 deletions

diff --git a/runtime/lua/vim/lsp/buf.lua b/runtime/lua/vim/lsp/buf.lua
index 7fc5286a78..d2e92de083 100644
--- a/runtime/lua/vim/lsp/buf.lua
+++ b/runtime/lua/vim/lsp/buf.lua
@@ -12,7 +12,7 @@ local M = {}
 ---@param method (string) LSP method name
 ---@param params (table|nil) Parameters to send to the server
 ---@param handler (function|nil) See |lsp-handler|. Follows |lsp-handler-resolution|
---
+---
 ---@return table<integer, integer> client_request_ids Map of client-id:request-id pairs
 ---for all successful requests.
 ---@return function _cancel_all_requests Function which can be used to
@@ -172,12 +172,13 @@ end
 ---
 ---     - filter (function|nil):
 ---         Predicate used to filter clients. Receives a client as argument and must return a
----         boolean. Clients matching the predicate are included. Example: <pre>lua
----                     -- Never request typescript-language-server for formatting
----                     vim.lsp.buf.format {
----                       filter = function(client) return client.name ~= "tsserver" end
----                     }
----         </pre>
+---         boolean. Clients matching the predicate are included. Example:
+---         ```lua
+---            -- Never request typescript-language-server for formatting
+---              vim.lsp.buf.format {
+---                filter = function(client) return client.name ~= "tsserver" end
+---              }
+---         ```
 ---
 ---     - async boolean|nil
 ---         If true the method won't block. Defaults to false.
@@ -472,6 +473,7 @@ end
 
 --- Add the folder at path to the workspace folders. If {path} is
 --- not provided, the user will be prompted for a path using |input()|.
+--- @param workspace_folder? string
 function M.add_workspace_folder(workspace_folder)
   workspace_folder = workspace_folder
     or npcall(vim.fn.input, 'Workspace Folder: ', vim.fn.expand('%:p:h'), 'dir')
@@ -511,6 +513,7 @@ end
 --- Remove the folder at path from the workspace folders. If
 --- {path} is not provided, the user will be prompted for
 --- a path using |input()|.
+--- @param workspace_folder? string
 function M.remove_workspace_folder(workspace_folder)
   workspace_folder = workspace_folder
     or npcall(vim.fn.input, 'Workspace Folder: ', vim.fn.expand('%:p:h'))
@@ -725,15 +728,15 @@ end
 ---
 ---@param options table|nil Optional table which holds the following optional fields:
 ---  - context: (table|nil)
----      Corresponds to `CodeActionContext` of the LSP specification:
----        - diagnostics (table|nil):
----                      LSP `Diagnostic[]`. Inferred from the current
----                      position if not provided.
----        - only (table|nil):
----               List of LSP `CodeActionKind`s used to filter the code actions.
----               Most language servers support values like `refactor`
----               or `quickfix`.
----        - triggerKind (number|nil): The reason why code actions were requested.
+---    Corresponds to `CodeActionContext` of the LSP specification:
+---      - diagnostics (table|nil):
+---                    LSP `Diagnostic[]`. Inferred from the current
+---                    position if not provided.
+---      - only (table|nil):
+---             List of LSP `CodeActionKind`s used to filter the code actions.
+---             Most language servers support values like `refactor`
+---             or `quickfix`.
+---      - triggerKind (number|nil): The reason why code actions were requested.
 ---  - filter: (function|nil)
 ---           Predicate taking an `CodeAction` and returning a boolean.
 ---  - apply: (boolean|nil)
diff --git a/runtime/lua/vim/lsp/codelens.lua b/runtime/lua/vim/lsp/codelens.lua
index 966c7f4d03..7aed6f99e3 100644
--- a/runtime/lua/vim/lsp/codelens.lua
+++ b/runtime/lua/vim/lsp/codelens.lua
@@ -258,6 +258,8 @@ end
 
 --- |lsp-handler| for the method `textDocument/codeLens`
 ---
+---@param err lsp.ResponseError?
+---@param result lsp.CodeLens[]
 ---@param ctx lsp.HandlerContext
 function M.on_codelens(err, result, ctx, _)
   if err then
diff --git a/runtime/lua/vim/lsp/diagnostic.lua b/runtime/lua/vim/lsp/diagnostic.lua
index 33051ab61c..1fa67fc473 100644
--- a/runtime/lua/vim/lsp/diagnostic.lua
+++ b/runtime/lua/vim/lsp/diagnostic.lua
@@ -1,5 +1,3 @@
----@brief lsp-diagnostic
-
 local protocol = require('vim.lsp.protocol')
 local ms = protocol.Methods
 
@@ -287,6 +285,7 @@ end
 --- )
 --- ```
 ---
+---@param _ lsp.ResponseError?
 ---@param result lsp.PublishDiagnosticsParams
 ---@param ctx lsp.HandlerContext
 ---@param config? vim.diagnostic.Opts Configuration table (see |vim.diagnostic.config()|).
@@ -319,6 +318,7 @@ end
 --- )
 --- ```
 ---
+---@param _ lsp.ResponseError?
 ---@param result lsp.DocumentDiagnosticReport
 ---@param ctx lsp.HandlerContext
 ---@param config table Configuration table (see |vim.diagnostic.config()|).
diff --git a/runtime/lua/vim/lsp/handlers.lua b/runtime/lua/vim/lsp/handlers.lua
index a9da812231..781d720486 100644
--- a/runtime/lua/vim/lsp/handlers.lua
+++ b/runtime/lua/vim/lsp/handlers.lua
@@ -368,6 +368,8 @@ end
 --- )
 --- ```
 ---
+---@param _ lsp.ResponseError?
+---@param result lsp.Hover
 ---@param ctx lsp.HandlerContext
 ---@param config table Configuration table.
 ---     - border:     (default=nil)
@@ -464,7 +466,8 @@ M[ms.textDocument_implementation] = location_handler
 --- )
 --- ```
 ---
----@param result table Response from the language server
+---@param _ lsp.ResponseError?
+---@param result lsp.SignatureHelp  Response from the language server
 ---@param ctx lsp.HandlerContext Client context
 ---@param config table Configuration table.
 ---     - border:     (default=nil)
diff --git a/runtime/lua/vim/lsp/log.lua b/runtime/lua/vim/lsp/log.lua
index a9d49bc8f4..018003bb81 100644
--- a/runtime/lua/vim/lsp/log.lua
+++ b/runtime/lua/vim/lsp/log.lua
@@ -165,7 +165,7 @@ end
 
 --- Checks whether the level is sufficient for logging.
 ---@param level integer log level
----@returns (bool) true if would log, false if not
+---@return bool : true if would log, false if not
 function log.should_log(level)
   return level >= current_log_level
 end
diff --git a/runtime/lua/vim/lsp/rpc.lua b/runtime/lua/vim/lsp/rpc.lua
index e849bb4f2a..1455ab51fa 100644
--- a/runtime/lua/vim/lsp/rpc.lua
+++ b/runtime/lua/vim/lsp/rpc.lua
@@ -273,8 +273,6 @@ end
 ---@field notify_reply_callbacks table<integer, function> dict of message_id to callback
 ---@field transport vim.lsp.rpc.Transport
 ---@field dispatchers vim.lsp.rpc.Dispatchers
-
----@class vim.lsp.rpc.Client
 local Client = {}
 
 ---@private
diff --git a/runtime/lua/vim/lsp/sync.lua b/runtime/lua/vim/lsp/sync.lua
index 62fa0b33f4..936579e003 100644
--- a/runtime/lua/vim/lsp/sync.lua
+++ b/runtime/lua/vim/lsp/sync.lua
@@ -53,7 +53,7 @@ local str_utf_end = vim.str_utf_end
 ---@param line string the line to index into
 ---@param byte integer the byte idx
 ---@param offset_encoding string utf-8|utf-16|utf-32|nil (default: utf-8)
---@returns integer the utf idx for the given encoding
+---@return integer utf_idx for the given encoding
 local function byte_to_utf(line, byte, offset_encoding)
   -- convert to 0 based indexing for str_utfindex
   byte = byte - 1
@@ -204,7 +204,7 @@ end
 --- Normalized to the next codepoint.
 --- prev_end_range is the text range sent to the server representing the changed region.
 --- curr_end_range is the text that should be collected and sent to the server.
---
+---
 ---@param prev_lines string[] list of lines
 ---@param curr_lines string[] list of lines
 ---@param start_range vim.lsp.sync.Range
diff --git a/runtime/lua/vim/lsp/util.lua b/runtime/lua/vim/lsp/util.lua
index b60135f851..e371cb0e15 100644
--- a/runtime/lua/vim/lsp/util.lua
+++ b/runtime/lua/vim/lsp/util.lua
@@ -574,6 +574,7 @@ end
 ---
 ---@param text_document_edit table: a `TextDocumentEdit` object
 ---@param index integer: Optional index of the edit, if from a list of edits (or nil, if not from a list)
+---@param offset_encoding? string
 ---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocumentEdit
 function M.apply_text_document_edit(text_document_edit, index, offset_encoding)
   local text_document = text_document_edit.textDocument
@@ -770,7 +771,7 @@ end
 ---
 ---@param workspace_edit table `WorkspaceEdit`
 ---@param offset_encoding string utf-8|utf-16|utf-32 (required)
---see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#workspace_applyEdit
+---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#workspace_applyEdit
 function M.apply_workspace_edit(workspace_edit, offset_encoding)
   if offset_encoding == nil then
     vim.notify_once(
@@ -1130,6 +1131,7 @@ end
 ---   - for LocationLink, targetRange is shown (e.g., body of function definition)
 ---
 ---@param location table a single `Location` or `LocationLink`
+---@param opts table
 ---@return integer|nil buffer id of float window
 ---@return integer|nil window id of float window
 function M.preview_location(location, opts)
@@ -1243,6 +1245,7 @@ end
 ---
 --- If you want to open a popup with fancy markdown, use `open_floating_preview` instead
 ---
+---@param bufnr integer
 ---@param contents table of lines to show in window
 ---@param opts table with optional fields
 ---  - height    of floating window
@@ -1603,7 +1606,7 @@ end
 ---@param contents table of lines to show in window
 ---@param syntax string of syntax to set for opened buffer
 ---@param opts table with optional fields (additional keys are filtered with |vim.lsp.util.make_floating_popup_options()|
----                  before they are passed on to |nvim_open_win()|)
+---             before they are passed on to |nvim_open_win()|)
 ---             - height: (integer) height of floating window
 ---             - width: (integer) width of floating window
 ---             - wrap: (boolean, default true) wrap long lines
@@ -1868,6 +1871,7 @@ end
 --- Converts symbols to quickfix list items.
 ---
 ---@param symbols table DocumentSymbol[] or SymbolInformation[]
+---@param bufnr integer
 function M.symbols_to_items(symbols, bufnr)
   local function _symbols_to_items(_symbols, _items, _bufnr)
     for _, symbol in ipairs(_symbols) do