feat(docs): replace lua2dox.lua

Problem:

The documentation flow (`gen_vimdoc.py`) has several issues:
- it's not very versatile
- depends on doxygen
- doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C.
- The intermediate XML files and filters makes it too much like a rube goldberg machine.

Solution:

Re-implement the flow using Lua, LPEG and treesitter.

- `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic.
- `lua2dox.lua` is gone!
- No more XML files.
- Doxygen is now longer used and instead we now use:
  - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`).
  - LPEG for C parsing (see `scripts/cdoc_parser.lua`)
  - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`).
  - Treesitter for Markdown parsing (see `scripts/text_utils.lua`).
- The generated `runtime/doc/*.mpack` files have been removed.
   - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly.
- Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change).

4 files changed, 61 insertions, 163 deletions

diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua
index 8a3e70990a..9ce9f3d7a6 100644
--- a/src/nvim/generators/c_grammar.lua
+++ b/src/nvim/generators/c_grammar.lua
@@ -6,46 +6,74 @@ local lpeg = vim.lpeg
 local P, R, S = lpeg.P, lpeg.R, lpeg.S
 local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
 
+--- @param pat vim.lpeg.Pattern
+local function rep(pat)
+  return pat ^ 0
+end
+
+--- @param pat vim.lpeg.Pattern
+local function rep1(pat)
+  return pat ^ 1
+end
+
+--- @param pat vim.lpeg.Pattern
+local function opt(pat)
+  return pat ^ -1
+end
+
 local any = P(1) -- (consume one character)
 local letter = R('az', 'AZ') + S('_$')
 local num = R('09')
 local alpha = letter + num
 local nl = P('\r\n') + P('\n')
 local not_nl = any - nl
-local ws = S(' \t') + nl
-local fill = ws ^ 0
-local c_comment = P('//') * (not_nl ^ 0)
-local c_preproc = P('#') * (not_nl ^ 0)
-local dllexport = P('DLLEXPORT') * (ws ^ 1)
-local typed_container = (P('ArrayOf(') + P('DictionaryOf(') + P('Dict('))
-  * ((any - P(')')) ^ 1)
+local space = S(' \t')
+local ws = space + nl
+local fill = rep(ws)
+local c_comment = P('//') * rep(not_nl)
+local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(not_nl), 'comment')))
+local c_preproc = P('#') * rep(not_nl)
+local dllexport = P('DLLEXPORT') * rep1(ws)
+
+local typed_container = (
+  (P('ArrayOf(') + P('DictionaryOf(') + P('Dict('))
+  * rep1(any - P(')'))
   * P(')')
-local c_id = (typed_container + (letter * (alpha ^ 0)))
+)
+
+local c_id = (typed_container + (letter * rep(alpha)))
 local c_void = P('void')
+
 local c_param_type = (
   ((P('Error') * fill * P('*') * fill) * Cc('error'))
   + ((P('Arena') * fill * P('*') * fill) * Cc('arena'))
   + ((P('lua_State') * fill * P('*') * fill) * Cc('lstate'))
-  + C((P('const ') ^ -1) * c_id * (ws ^ 1) * P('*'))
-  + (C(c_id) * (ws ^ 1))
+  + C(opt(P('const ')) * c_id * rep1(ws) * rep1(P('*')))
+  + (C(c_id) * rep1(ws))
 )
+
 local c_type = (C(c_void) * (ws ^ 1)) + c_param_type
 local c_param = Ct(c_param_type * C(c_id))
 local c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0)
 local c_params = Ct(c_void + c_param_list)
+
+local impl_line = (any - P('}')) * opt(rep(not_nl)) * nl
+
+local ignore_line = rep1(not_nl) * nl
+
+local empty_line = Ct(Cc('empty') * nl * nl)
+
 local c_proto = Ct(
-  (dllexport ^ -1)
+  Cc('proto')
+    * opt(dllexport)
+    * opt(Cg(P('static') * fill * Cc(true), 'static'))
     * Cg(c_type, 'return_type')
     * Cg(c_id, 'name')
     * fill
-    * P('(')
-    * fill
-    * Cg(c_params, 'parameters')
-    * fill
-    * P(')')
+    * (P('(') * fill * Cg(c_params, 'parameters') * fill * P(')'))
     * Cg(Cc(false), 'fast')
-    * (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1)
-    * (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(num ^ 1)) * P(')'), 'deprecated_since') ^ -1)
+    * (fill * Cg((P('FUNC_API_SINCE(') * C(rep1(num))) * P(')'), 'since') ^ -1)
+    * (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(rep1(num))) * P(')'), 'deprecated_since') ^ -1)
     * (fill * Cg((P('FUNC_API_FAST') * Cc(true)), 'fast') ^ -1)
     * (fill * Cg((P('FUNC_API_RET_ALLOC') * Cc(true)), 'ret_alloc') ^ -1)
     * (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1)
@@ -60,7 +88,7 @@ local c_proto = Ct(
     * (fill * Cg((P('FUNC_API_CLIENT_IMPL') * Cc(true)), 'client_impl') ^ -1)
     * (fill * Cg((P('FUNC_API_CLIENT_IGNORE') * Cc(true)), 'client_ignore') ^ -1)
     * fill
-    * P(';')
+    * (P(';') + (P('{') * nl + (impl_line ^ 0) * P('}')))
 )
 
 local c_field = Ct(Cg(c_id, 'type') * ws * Cg(c_id, 'name') * fill * P(';') * fill)
@@ -83,5 +111,7 @@ local c_keyset = Ct(
     * P(';')
 )
 
-local grammar = Ct((c_proto + c_comment + c_preproc + ws + c_keyset) ^ 1)
+local grammar = Ct(
+  rep1(empty_line + c_proto + cdoc_comment + c_comment + c_preproc + ws + c_keyset + ignore_line)
+)
 return { grammar = grammar, typed_container = typed_container }
diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua
index c3fc5aa0a3..56331e4162 100644
--- a/src/nvim/generators/gen_api_dispatch.lua
+++ b/src/nvim/generators/gen_api_dispatch.lua
@@ -23,9 +23,7 @@ local function_names = {}
 
 local c_grammar = require('generators.c_grammar')
 
-local function startswith(String, Start)
-  return string.sub(String, 1, string.len(Start)) == Start
-end
+local startswith = vim.startswith
 
 local function add_function(fn)
   local public = startswith(fn.name, 'nvim_') or fn.deprecated_since
@@ -112,10 +110,12 @@ for i = 6, #arg do
   local tmp = c_grammar.grammar:match(input:read('*all'))
   for j = 1, #tmp do
     local val = tmp[j]
-    if val.keyset_name then
-      add_keyset(val)
-    else
-      add_function(val)
+    if val[1] ~= 'empty' then
+      if val.keyset_name then
+        add_keyset(val)
+      else
+        add_function(val)
+      end
     end
   end
   input:close()
diff --git a/src/nvim/generators/gen_api_ui_events.lua b/src/nvim/generators/gen_api_ui_events.lua
index 89f8c654f4..697626b793 100644
--- a/src/nvim/generators/gen_api_ui_events.lua
+++ b/src/nvim/generators/gen_api_ui_events.lua
@@ -93,6 +93,10 @@ local function call_ui_event_method(output, ev)
   output:write('}\n\n')
 end
 
+events = vim.tbl_filter(function(ev)
+  return ev[1] ~= 'empty'
+end, events)
+
 for i = 1, #events do
   local ev = events[i]
   assert(ev.return_type == 'void')
diff --git a/src/nvim/generators/luacats_grammar.lua b/src/nvim/generators/luacats_grammar.lua
deleted file mode 100644
index dcccd028ce..0000000000
--- a/src/nvim/generators/luacats_grammar.lua
+++ /dev/null
@@ -1,136 +0,0 @@
---[[!
-LPEG grammar for LuaCATS
-
-Currently only partially supports:
-- @param
-- @return
-]]
-
-local lpeg = vim.lpeg
-local P, R, S = lpeg.P, lpeg.R, lpeg.S
-local Ct, Cg = lpeg.Ct, lpeg.Cg
-
---- @param x vim.lpeg.Pattern
-local function rep(x)
-  return x ^ 0
-end
-
---- @param x vim.lpeg.Pattern
-local function rep1(x)
-  return x ^ 1
-end
-
---- @param x vim.lpeg.Pattern
-local function opt(x)
-  return x ^ -1
-end
-
-local nl = P('\r\n') + P('\n')
-local ws = rep1(S(' \t') + nl)
-local fill = opt(ws)
-
-local any = P(1) -- (consume one character)
-local letter = R('az', 'AZ') + S('_$')
-local num = R('09')
-local ident = letter * rep(letter + num + S '-.')
-local string_single = P "'" * rep(any - P "'") * P "'"
-local string_double = P '"' * rep(any - P '"') * P '"'
-
-local literal = (string_single + string_double + (opt(P '-') * num) + P 'false' + P 'true')
-
-local lname = (ident + P '...') * opt(P '?')
-
---- @param x string
-local function Pf(x)
-  return fill * P(x) * fill
-end
-
---- @param x string
-local function Sf(x)
-  return fill * S(x) * fill
-end
-
---- @param x vim.lpeg.Pattern
-local function comma(x)
-  return x * rep(Pf ',' * x)
-end
-
---- @param x vim.lpeg.Pattern
-local function parenOpt(x)
-  return (Pf('(') * x ^ -1 * fill * P(')')) + x ^ -1
-end
-
---- @type table<string,vim.lpeg.Pattern>
-local v = setmetatable({}, {
-  __index = function(_, k)
-    return lpeg.V(k)
-  end,
-})
-
-local desc_delim = Sf '#:' + ws
-
---- @class luacats.Param
---- @field kind 'param'
---- @field name string
---- @field type string
---- @field desc? string
-
---- @class luacats.Return
---- @field kind 'return'
---- @field [integer] { type: string, name?: string}
---- @field desc? string
-
---- @class luacats.Generic
---- @field kind 'generic'
---- @field name string
---- @field type? string
-
---- @alias luacats.grammar.result
---- | luacats.Param
---- | luacats.Return
---- | luacats.Generic
-
---- @class luacats.grammar
---- @field match fun(self, input: string): luacats.grammar.result?
-
-local grammar = P {
-  rep1(P('@') * v.ats),
-
-  ats = (v.at_param + v.at_return + v.at_generic),
-
-  at_param = Ct(
-    Cg(P('param'), 'kind')
-      * ws
-      * Cg(lname, 'name')
-      * ws
-      * Cg(v.ltype, 'type')
-      * opt(desc_delim * Cg(rep(any), 'desc'))
-  ),
-
-  at_return = Ct(
-    Cg(P('return'), 'kind')
-      * ws
-      * parenOpt(comma(Ct(Cg(v.ltype, 'type') * opt(ws * Cg(ident, 'name')))))
-      * opt(desc_delim * Cg(rep(any), 'desc'))
-  ),
-
-  at_generic = Ct(
-    Cg(P('generic'), 'kind') * ws * Cg(ident, 'name') * opt(Pf ':' * Cg(v.ltype, 'type'))
-  ),
-
-  ltype = v.ty_union + Pf '(' * v.ty_union * fill * P ')',
-
-  ty_union = v.ty_opt * rep(Pf '|' * v.ty_opt),
-  ty = v.ty_fun + ident + v.ty_table + literal,
-  ty_param = Pf '<' * comma(v.ltype) * fill * P '>',
-  ty_opt = v.ty * opt(v.ty_param) * opt(P '[]') * opt(P '?'),
-
-  table_key = (Pf '[' * literal * Pf ']') + lname,
-  table_elem = v.table_key * Pf ':' * v.ltype,
-  ty_table = Pf '{' * comma(v.table_elem) * Pf '}',
-
-  fun_param = lname * opt(Pf ':' * v.ltype),
-  ty_fun = Pf 'fun(' * rep(comma(v.fun_param)) * fill * P ')' * opt(Pf ':' * v.ltype),
-}
-
-return grammar --[[@as luacats.grammar]]