feat(docs): replace lua2dox.lua

Problem:

The documentation flow (`gen_vimdoc.py`) has several issues:
- it's not very versatile
- depends on doxygen
- doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C.
- The intermediate XML files and filters makes it too much like a rube goldberg machine.

Solution:

Re-implement the flow using Lua, LPEG and treesitter.

- `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic.
- `lua2dox.lua` is gone!
- No more XML files.
- Doxygen is now longer used and instead we now use:
  - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`).
  - LPEG for C parsing (see `scripts/cdoc_parser.lua`)
  - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`).
  - Treesitter for Markdown parsing (see `scripts/text_utils.lua`).
- The generated `runtime/doc/*.mpack` files have been removed.
   - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly.
- Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change).

1 files changed, 133 insertions, 0 deletions

diff --git a/test/functional/script/luacats_grammar_spec.lua b/test/functional/script/luacats_grammar_spec.lua
new file mode 100644
index 0000000000..931fe42dd0
--- /dev/null
+++ b/test/functional/script/luacats_grammar_spec.lua
@@ -0,0 +1,133 @@
+local helpers = require('test.functional.helpers')(after_each)
+local eq = helpers.eq
+
+local grammar = require('scripts/luacats_grammar')
+
+describe('luacats grammar', function()
+  --- @param text string
+  --- @param exp table<string,string>
+  local function test(text, exp)
+    it(string.format('can parse %q', text), function()
+      eq(exp, grammar:match(text))
+    end)
+  end
+
+  test('@param hello vim.type', {
+    kind = 'param',
+    name = 'hello',
+    type = 'vim.type',
+  })
+
+  test('@param hello vim.type this is a description', {
+    kind = 'param',
+    name = 'hello',
+    type = 'vim.type',
+    desc = 'this is a description',
+  })
+
+  test('@param hello vim.type|string this is a description', {
+    kind = 'param',
+    name = 'hello',
+    type = 'vim.type|string',
+    desc = 'this is a description',
+  })
+
+  test('@param hello vim.type?|string? this is a description', {
+    kind = 'param',
+    name = 'hello',
+    type = 'vim.type?|string?',
+    desc = 'this is a description',
+  })
+
+  test('@return string hello this is a description', {
+    kind = 'return',
+    {
+      name = 'hello',
+      type = 'string',
+    },
+    desc = 'this is a description',
+  })
+
+  test('@return fun() hello this is a description', {
+    kind = 'return',
+    {
+      name = 'hello',
+      type = 'fun()',
+    },
+    desc = 'this is a description',
+  })
+
+  test('@return fun(a: string[]): string hello this is a description', {
+    kind = 'return',
+    {
+      name = 'hello',
+      type = 'fun(a: string[]): string',
+    },
+    desc = 'this is a description',
+  })
+
+  test('@return fun(a: table<string,any>): string hello this is a description', {
+    kind = 'return',
+    {
+      name = 'hello',
+      type = 'fun(a: table<string,any>): string',
+    },
+    desc = 'this is a description',
+  })
+
+  test('@param ... string desc', {
+    kind = 'param',
+    name = '...',
+    type = 'string',
+    desc = 'desc',
+  })
+
+  test('@param level (integer|string) desc', {
+    kind = 'param',
+    name = 'level',
+    type = 'integer|string',
+    desc = 'desc',
+  })
+
+  test('@return (string command) the command and arguments', {
+    kind = 'return',
+    {
+      name = 'command',
+      type = 'string',
+    },
+    desc = 'the command and arguments',
+  })
+
+  test('@return (string command, string[] args) the command and arguments', {
+    kind = 'return',
+    {
+      name = 'command',
+      type = 'string',
+    },
+    {
+      name = 'args',
+      type = 'string[]',
+    },
+    desc = 'the command and arguments',
+  })
+
+  test('@param rfc "rfc2396" | "rfc2732" | "rfc3986" | nil', {
+    kind = 'param',
+    name = 'rfc',
+    type = '"rfc2396" | "rfc2732" | "rfc3986" | nil',
+  })
+
+  test('@param offset_encoding "utf-8" | "utf-16" | "utf-32" | nil', {
+    kind = 'param',
+    name = 'offset_encoding',
+    type = '"utf-8" | "utf-16" | "utf-32" | nil',
+  })
+
+  -- handle a : after the param type
+  test('@param a b: desc', {
+    kind = 'param',
+    name = 'a',
+    type = 'b',
+    desc = 'desc',
+  })
+end)