Merge #22493 lua2dox.lua, vim.treesitter tags

8 files changed, 114 insertions, 116 deletions

diff --git a/runtime/doc/deprecated.txt b/runtime/doc/deprecated.txt
index 42dfb53e77..3d861b07b3 100644
--- a/runtime/doc/deprecated.txt
+++ b/runtime/doc/deprecated.txt
@@ -120,7 +120,7 @@ LSP FUNCTIONS
 					or |vim.lsp.buf.format()| instead.
 
 TREESITTER FUNCTIONS
-- *vim.treesitter.language.require_language()*	Use |vim.treesitter.language.add()|
+- *vim.treesitter.language.require_language()*	Use |vim.treesitter.add()|
 						instead.
 - *vim.treesitter.get_node_at_pos()*		Use |vim.treesitter.get_node()|
 						instead.
diff --git a/runtime/doc/news.txt b/runtime/doc/news.txt
index 0ffd335520..cda4792c9e 100644
--- a/runtime/doc/news.txt
+++ b/runtime/doc/news.txt
@@ -162,11 +162,10 @@ The following new APIs or features were added.
 • Treesitter captures can now be transformed by directives. This will allow
   more complicated dynamic language injections.
 
-• |vim.treesitter.query.get_node_text()| now accepts a `metadata` option for
-  writing custom directives using |vim.treesitter.query.add_directive()|.
+• |vim.treesitter.get_node_text()| now accepts a `metadata` option for
+  writing custom directives using |vim.treesitter.add_directive()|.
 
-• |vim.treesitter.language.add()| as a new replacement for
-  `vim.treesitter.language.require_language`.
+• |vim.treesitter.add()| replaces `vim.treesitter.language.require_language`.
 
 • `require'bit'` is now always available |lua-bit|
 
@@ -225,8 +224,7 @@ DEPRECATIONS                                                *news-deprecations*
 The following functions are now deprecated and will be removed in the next
 release.
 
-• `vim.treesitter.language.require_language()` has been deprecated in favour
-  of |vim.treesitter.language.add()|.
+• |vim.treesitter.add()| replaces `vim.treesitter.language.require_language()`
 
 • |vim.treesitter.get_node_at_pos()| and |vim.treesitter.get_node_at_cursor()|
   are both deprecated in favor of |vim.treesitter.get_node()|.
diff --git a/runtime/doc/treesitter.txt b/runtime/doc/treesitter.txt
index 5ac24de70a..94a93bdbbb 100644
--- a/runtime/doc/treesitter.txt
+++ b/runtime/doc/treesitter.txt
@@ -272,9 +272,8 @@ The following predicates are built in:
 Each predicate has a `not-` prefixed predicate that is just the negation of
 the predicate.
 
-Further predicates can be added via |vim.treesitter.query.add_predicate()|.
-Use |vim.treesitter.query.list_predicates()| to list all available
-predicates.
+Further predicates can be added via |vim.treesitter.add_predicate()|.
+Use |vim.treesitter.list_predicates()| to list all available predicates.
 
 
 TREESITTER QUERY DIRECTIVES                            *treesitter-directives*
@@ -316,9 +315,8 @@ The following directives are built in:
             ((identifier) @constant (#offset! @constant 0 1 0 -1))
 <
 
-Further directives can be added via |vim.treesitter.query.add_directive()|.
-Use |vim.treesitter.query.list_directives()| to list all available
-directives.
+Further directives can be added via |vim.treesitter.add_directive()|.
+Use |vim.treesitter.list_directives()| to list all available directives.
 
 
 TREESITTER QUERY MODELINES                          *treesitter-query-modeline*
@@ -690,7 +688,7 @@ stop({bufnr})                                          *vim.treesitter.stop()*
 ==============================================================================
 Lua module: vim.treesitter.language                  *lua-treesitter-language*
 
-add({lang}, {opts})                            *vim.treesitter.language.add()*
+add({lang}, {opts})                                     *vim.treesitter.add()*
     Asserts that a parser for the language {lang} is installed.
 
     Parsers are searched in the `parser` runtime directory, or the provided
@@ -708,14 +706,14 @@ add({lang}, {opts})                            *vim.treesitter.language.add()*
                 • symbol_name (string|nil) Internal symbol name for the
                   language to load
 
-get_lang({filetype})                      *vim.treesitter.language.get_lang()*
+get_lang({filetype})                               *vim.treesitter.get_lang()*
     Parameters: ~
       • {filetype}  (string)
 
     Return: ~
         (string|nil)
 
-inspect_language({lang})          *vim.treesitter.language.inspect_language()*
+inspect_language({lang})                   *vim.treesitter.inspect_language()*
     Inspects the provided language.
 
     Inspecting provides some useful information on the language like node
@@ -727,7 +725,7 @@ inspect_language({lang})          *vim.treesitter.language.inspect_language()*
     Return: ~
         (table)
 
-register({lang}, {filetype})              *vim.treesitter.language.register()*
+register({lang}, {filetype})                       *vim.treesitter.register()*
     Register a lang to be used for a filetype (or list of filetypes).
 
     Parameters: ~
@@ -738,7 +736,7 @@ register({lang}, {filetype})              *vim.treesitter.language.register()*
 ==============================================================================
 Lua module: vim.treesitter.query                        *lua-treesitter-query*
 
-                                        *vim.treesitter.query.add_directive()*
+                                              *vim.treesitter.add_directive()*
 add_directive({name}, {handler}, {force})
     Adds a new directive to be used in queries
 
@@ -760,7 +758,7 @@ add_directive({name}, {handler}, {force})
                      the predicate `{ "#set!", "conceal", "-" }`
       • {force}    (boolean|nil)
 
-                                        *vim.treesitter.query.add_predicate()*
+                                              *vim.treesitter.add_predicate()*
 add_predicate({name}, {handler}, {force})
     Adds a new predicate to be used in queries
 
@@ -768,11 +766,11 @@ add_predicate({name}, {handler}, {force})
       • {name}     (string) Name of the predicate, without leading #
       • {handler}  function(match:table<string,|TSNode|>, pattern:string,
                    bufnr:number, predicate:string[])
-                   • see |vim.treesitter.query.add_directive()| for argument
+                   • see |vim.treesitter.add_directive()| for argument
                      meanings
       • {force}    (boolean|nil)
 
-                                        *vim.treesitter.query.get_node_text()*
+                                              *vim.treesitter.get_node_text()*
 get_node_text({node}, {source}, {opts})
     Gets the text corresponding to a given node
 
@@ -785,12 +783,12 @@ get_node_text({node}, {source}, {opts})
                     true)
                   • metadata (table) Metadata of a specific capture. This
                     would be set to `metadata[capture_id]` when using
-                    |vim.treesitter.query.add_directive()|.
+                    |vim.treesitter.add_directive()|.
 
     Return: ~
         (string[]|string|nil)
 
-get_query({lang}, {query_name})             *vim.treesitter.query.get_query()*
+get_query({lang}, {query_name})                   *vim.treesitter.get_query()*
     Returns the runtime query {query_name} for {lang}.
 
     Parameters: ~
@@ -800,7 +798,7 @@ get_query({lang}, {query_name})             *vim.treesitter.query.get_query()*
     Return: ~
         Query|nil Parsed query
 
-                                      *vim.treesitter.query.get_query_files()*
+                                            *vim.treesitter.get_query_files()*
 get_query_files({lang}, {query_name}, {is_included})
     Gets the list of files used to make up a query
 
@@ -814,19 +812,19 @@ get_query_files({lang}, {query_name}, {is_included})
         string[] query_files List of files to load for given query and
         language
 
-list_directives()                     *vim.treesitter.query.list_directives()*
+list_directives()                           *vim.treesitter.list_directives()*
     Lists the currently available directives to use in queries.
 
     Return: ~
         string[] List of supported directives.
 
-list_predicates()                     *vim.treesitter.query.list_predicates()*
+list_predicates()                           *vim.treesitter.list_predicates()*
     Lists the currently available predicates to use in queries.
 
     Return: ~
         string[] List of supported predicates.
 
-parse_query({lang}, {query})              *vim.treesitter.query.parse_query()*
+parse_query({lang}, {query})                    *vim.treesitter.parse_query()*
     Parse {query} as a string. (If the query is in a file, the caller should
     read the contents into a string before calling).
 
@@ -915,8 +913,7 @@ Query:iter_matches({self}, {node}, {source}, {start}, {stop})
         (fun(): integer, table<integer,TSNode>, table): pattern id, match,
         metadata
 
-                                            *vim.treesitter.query.set_query()*
-set_query({lang}, {query_name}, {text})
+set_query({lang}, {query_name}, {text})           *vim.treesitter.set_query()*
     Sets the runtime query named {query_name} for {lang}
 
     This allows users to override any runtime files and/or configuration set
@@ -931,17 +928,6 @@ set_query({lang}, {query_name}, {text})
 ==============================================================================
 Lua module: vim.treesitter.highlighter            *lua-treesitter-highlighter*
 
-new({tree}, {opts})                         *vim.treesitter.highlighter.new()*
-    Creates a new highlighter using
-
-    Parameters: ~
-      • {tree}  |LanguageTree| parser object to use for highlighting
-      • {opts}  (table|nil) Configuration of the highlighter:
-                • queries table overwrite queries used by the highlighter
-
-    Return: ~
-        TSHighlighter Created highlighter object
-
 TSHighlighter:destroy({self})                        *TSHighlighter:destroy()*
     Removes all internal references to the highlighter
 
@@ -1103,21 +1089,4 @@ LanguageTree:trees({self})                              *LanguageTree:trees()*
     Parameters: ~
       • {self}
 
-new({source}, {lang}, {opts})              *vim.treesitter.languagetree.new()*
-    A |LanguageTree| holds the treesitter parser for a given language {lang}
-    used to parse a buffer. As the buffer may contain injected languages, the LanguageTree needs to store parsers for these child languages as well (which in turn
-    may contain child languages themselves, hence the name).
-
-    Parameters: ~
-      • {source}  (integer|string) Buffer or a string of text to parse
-      • {lang}    (string) Root language this tree represents
-      • {opts}    (table|nil) Optional keyword arguments:
-                  • injections table Mapping language to injection query
-                    strings. This is useful for overriding the built-in
-                    runtime file searching for the injection language query
-                    per language.
-
-    Return: ~
-        |LanguageTree| parser object
-
  vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:
diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index 8adaa4ef2f..e3deaf6ba6 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -58,7 +58,9 @@ function TSHighlighterQuery:query()
   return self._query
 end
 
---- Creates a new highlighter using @param tree
+---@private
+---
+--- Creates a highlighter for `tree`.
 ---
 ---@param tree LanguageTree parser object to use for highlighting
 ---@param opts (table|nil) Configuration of the highlighter:
diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
index 43fb866896..1bc7971eba 100644
--- a/runtime/lua/vim/treesitter/languagetree.lua
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -38,18 +38,16 @@ local LanguageTree = {}
 
 LanguageTree.__index = LanguageTree
 
---- A |LanguageTree| holds the treesitter parser for a given language {lang} used
---- to parse a buffer. As the buffer may contain injected languages, the LanguageTree
---- needs to store parsers for these child languages as well (which in turn may contain
---- child languages themselves, hence the name).
+--- @private
 ---
----@param source (integer|string) Buffer or a string of text to parse
----@param lang string Root language this tree represents
----@param opts (table|nil) Optional keyword arguments:
----             - injections table Mapping language to injection query strings.
----                                This is useful for overriding the built-in
----                                runtime file searching for the injection language
----                                query per language.
+--- |LanguageTree| contains a tree of parsers: the root treesitter parser for {lang} and any
+--- "injected" language parsers, which themselves may inject other languages, recursively.
+---
+---@param source (integer|string) Buffer or text string to parse
+---@param lang string Root language of this tree
+---@param opts (table|nil) Optional arguments:
+---             - injections table Map of language to injection query strings. Overrides the
+---                                built-in runtime file searching for language injections.
 ---@return LanguageTree parser object
 function LanguageTree.new(source, lang, opts)
   language.add(lang)
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 13d98a0625..4e9871b59d 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -273,8 +273,7 @@ end
 ---@param opts (table|nil) Optional parameters.
 ---          - concat: (boolean) Concatenate result in a string (default true)
 ---          - metadata (table) Metadata of a specific capture. This would be
----            set to `metadata[capture_id]` when using
----            |vim.treesitter.query.add_directive()|.
+---            set to `metadata[capture_id]` when using |vim.treesitter.add_directive()|.
 ---@return (string[]|string|nil)
 function M.get_node_text(node, source, opts)
   opts = opts or {}
@@ -486,7 +485,7 @@ local directive_handlers = {
 ---
 ---@param name string Name of the predicate, without leading #
 ---@param handler function(match:table<string,TSNode>, pattern:string, bufnr:number, predicate:string[])
----   - see |vim.treesitter.query.add_directive()| for argument meanings
+---   - see |vim.treesitter.add_directive()| for argument meanings
 ---@param force boolean|nil
 function M.add_predicate(name, handler, force)
   if predicate_handlers[name] and not force then
diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index dd593475e2..9e9e966627 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -265,6 +265,7 @@ CONFIG = {
             'query.lua',
             'highlighter.lua',
             'languagetree.lua',
+            'playground.lua',
         ],
         'files': [
             'runtime/lua/vim/treesitter.lua',
@@ -286,7 +287,7 @@ CONFIG = {
             if fstem == 'treesitter'
             else f'*{name}()*'
             if name[0].isupper()
-            else f'*vim.treesitter.{fstem}.{name}()*'),
+            else f'*vim.treesitter.{name}()*'),
         'module_override': {},
         'append_only': [],
     }
@@ -1171,10 +1172,12 @@ def main(doxygen_config, args):
     msg_report()
 
 
-def filter_source(filename):
+def filter_source(filename, keep_tmpfiles):
+    output_dir = out_dir.format(target='lua2dox')
     name, extension = os.path.splitext(filename)
     if extension == '.lua':
-        p = subprocess.run([str(nvim), '-l', lua2dox, filename], stdout=subprocess.PIPE)
+        args = [str(nvim), '-l', lua2dox, filename] + (['--outdir', output_dir] if keep_tmpfiles else [])
+        p = subprocess.run(args, stdout=subprocess.PIPE)
         op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8'))
         print(op)
     else:
@@ -1197,7 +1200,7 @@ def parse_args():
     ap.add_argument('source_filter', nargs='*',
                     help="Filter source file(s)")
     ap.add_argument('-k', '--keep-tmpfiles', action='store_true',
-                    help="Keep temporary files")
+                    help="Keep temporary files (tmp-xx-doc/ directories, including tmp-lua2dox-doc/ for lua2dox.lua quasi-C output)")
     ap.add_argument('-t', '--target',
                     help=f'One of ({targets}), defaults to "all"')
     return ap.parse_args()
@@ -1245,8 +1248,13 @@ if __name__ == "__main__":
     log.setLevel(args.log_level)
     log.addHandler(logging.StreamHandler())
 
+    # When invoked as a filter, args won't be passed, so use an env var.
+    if args.keep_tmpfiles:
+        os.environ['NVIM_KEEP_TMPFILES'] = '1'
+    keep_tmpfiles = ('NVIM_KEEP_TMPFILES' in os.environ)
+
     if len(args.source_filter) > 0:
-        filter_source(args.source_filter[0])
+        filter_source(args.source_filter[0], keep_tmpfiles)
     else:
         main(Doxyfile, args)
 
diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua
index 17de0ea9b4..b99cd955f4 100644
--- a/scripts/lua2dox.lua
+++ b/scripts/lua2dox.lua
@@ -24,11 +24,18 @@ Lua-to-Doxygen converter
 Partially from lua2dox
 http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm
 
-Running
+RUNNING
 -------
 
 This script "lua2dox.lua" gets called by "gen_vimdoc.py".
 
+DEBUGGING/DEVELOPING
+---------------------
+
+1. To debug, run gen_vimdoc.py with --keep-tmpfiles:
+   python3 scripts/gen_vimdoc.py -t treesitter --keep-tmpfiles
+2. The filtered result will be written to ./tmp-lua2dox-doc/….lua.c
+
 Doxygen must be on your system. You can experiment like so:
 
 - Run "doxygen -g" to create a default Doxyfile.
@@ -49,6 +56,9 @@ However I have put in a hack that will insert the "missing" close paren.
 The effect is that you will get the function documented, but not with the parameter list you might expect.
 ]]
 
+local _debug_outfile = nil
+local _debug_output = {}
+
 local function class()
   local newClass = {} -- a new class newClass
   -- the class will be the metatable for all its newInstanceects,
@@ -71,27 +81,28 @@ local function class()
   return newClass
 end
 
---! \brief write to stdout
+-- write to stdout
 local function TCore_IO_write(Str)
   if Str then
     io.write(Str)
+    if _debug_outfile then
+      table.insert(_debug_output, Str)
+    end
   end
 end
 
---! \brief write to stdout
+-- write to stdout
 local function TCore_IO_writeln(Str)
-  if Str then
-    io.write(Str)
-  end
-  io.write('\n')
+  TCore_IO_write(Str)
+  TCore_IO_write('\n')
 end
 
---! \brief trims a string
+-- trims a string
 local function string_trim(Str)
   return Str:match('^%s*(.-)%s*$')
 end
 
---! \brief split a string
+-- split a string
 --!
 --! \param Str
 --! \param Pattern
@@ -117,12 +128,12 @@ local function string_split(Str, Pattern)
 end
 
 -------------------------------
---! \brief file buffer
+-- file buffer
 --!
 --! an input file buffer
 local TStream_Read = class()
 
---! \brief get contents of file
+-- get contents of file
 --!
 --! \param Filename name of file to read (or nil == stdin)
 function TStream_Read.getContents(this, Filename)
@@ -143,12 +154,12 @@ function TStream_Read.getContents(this, Filename)
   return filecontents
 end
 
---! \brief get lineno
+-- get lineno
 function TStream_Read.getLineNo(this)
   return this.currentLineNo
 end
 
---! \brief get a line
+-- get a line
 function TStream_Read.getLine(this)
   local line
   if this.currentLine then
@@ -166,12 +177,12 @@ function TStream_Read.getLine(this)
   return line
 end
 
---! \brief save line fragment
+-- save line fragment
 function TStream_Read.ungetLine(this, LineFrag)
   this.currentLine = LineFrag
 end
 
---! \brief is it eof?
+-- is it eof?
 function TStream_Read.eof(this)
   if this.currentLine or this.currentLineNo <= this.contentsLen then
     return false
@@ -179,31 +190,31 @@ function TStream_Read.eof(this)
   return true
 end
 
---! \brief output stream
+-- output stream
 local TStream_Write = class()
 
---! \brief constructor
+-- constructor
 function TStream_Write.init(this)
   this.tailLine = {}
 end
 
---! \brief write immediately
+-- write immediately
 function TStream_Write.write(_, Str)
   TCore_IO_write(Str)
 end
 
---! \brief write immediately
+-- write immediately
 function TStream_Write.writeln(_, Str)
   TCore_IO_writeln(Str)
 end
 
---! \brief write immediately
+-- write immediately
 function TStream_Write.writelnComment(_, Str)
   TCore_IO_write('// ZZ: ')
   TCore_IO_writeln(Str)
 end
 
---! \brief write to tail
+-- write to tail
 function TStream_Write.writelnTail(this, Line)
   if not Line then
     Line = ''
@@ -211,7 +222,7 @@ function TStream_Write.writelnTail(this, Line)
   table.insert(this.tailLine, Line)
 end
 
---! \brief output tail lines
+-- output tail lines
 function TStream_Write.write_tailLines(this)
   for _, line in ipairs(this.tailLine) do
     TCore_IO_writeln(line)
@@ -219,17 +230,17 @@ function TStream_Write.write_tailLines(this)
   TCore_IO_write('// Lua2DoX new eof')
 end
 
---! \brief input filter
+-- input filter
 local TLua2DoX_filter = class()
 
---! \brief allow us to do errormessages
+-- allow us to do errormessages
 function TLua2DoX_filter.warning(this, Line, LineNo, Legend)
   this.outStream:writelnTail(
     '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"'
   )
 end
 
---! \brief trim comment off end of string
+-- trim comment off end of string
 --!
 --! If the string has a comment on the end, this trims it off.
 --!
@@ -243,7 +254,7 @@ local function TString_removeCommentFromLine(Line)
   return Line, tailComment
 end
 
---! \brief get directive from magic
+-- get directive from magic
 local function getMagicDirective(Line)
   local macro, tail
   local macroStr = '[\\@]'
@@ -264,7 +275,7 @@ local function getMagicDirective(Line)
   return macro, tail
 end
 
---! \brief check comment for fn
+-- check comment for fn
 local function checkComment4fn(Fn_magic, MagicLines)
   local fn_magic = Fn_magic
   --    TCore_IO_writeln('// checkComment4fn "' .. MagicLines .. '"')
@@ -293,8 +304,8 @@ local tagged_types = { 'TSNode', 'LanguageTree' }
 -- Document these as 'table'
 local alias_types = { 'Range4', 'Range6' }
 
---! \brief run the filter
-function TLua2DoX_filter.readfile(this, AppStamp, Filename)
+-- Processes the file and writes filtered output to stdout.
+function TLua2DoX_filter.filter(this, AppStamp, Filename)
   local inStream = TStream_Read()
   local outStream = TStream_Write()
   this.outStream = outStream -- save to this obj
@@ -524,10 +535,10 @@ function TLua2DoX_filter.readfile(this, AppStamp, Filename)
   end
 end
 
---! \brief this application
+-- this application
 local TApp = class()
 
---! \brief constructor
+-- constructor
 function TApp.init(this)
   this.timestamp = os.date('%c %Z', os.time())
   this.name = 'Lua2DoX'
@@ -551,8 +562,7 @@ local This_app = TApp()
 
 --main
 
-local argv1 = arg[1]
-if argv1 == '--help' then
+if arg[1] == '--help' then
   TCore_IO_writeln(This_app:getVersion())
   TCore_IO_writeln(This_app:getCopyright())
   TCore_IO_writeln([[
@@ -563,16 +573,30 @@ if argv1 == '--help' then
   <filename> : interprets filename
   --version  : show version/copyright info
   --help     : this help text]])
-elseif argv1 == '--version' then
+elseif arg[1] == '--version' then
   TCore_IO_writeln(This_app:getVersion())
   TCore_IO_writeln(This_app:getCopyright())
-else
-  -- it's a filter
-  local appStamp = This_app:getRunStamp()
-  local filename = argv1
+else  -- It's a filter.
+  local filename = arg[1]
+
+  if arg[2] == '--outdir' then
+    local outdir = arg[3]
+    if type(outdir) ~= 'string' or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir)) then
+      error(('invalid --outdir: "%s"'):format(tostring(outdir)))
+    end
+    vim.fn.mkdir(outdir, 'p')
+    _debug_outfile = string.format('%s/%s.c', outdir, vim.fs.basename(filename))
+  end
 
+  local appStamp = This_app:getRunStamp()
   local filter = TLua2DoX_filter()
-  filter:readfile(appStamp, filename)
+  filter:filter(appStamp, filename)
+
+  if _debug_outfile then
+    local f = assert(io.open(_debug_outfile, 'w'))
+    f:write(table.concat(_debug_output))
+    f:close()
+  end
 end
 
 --eof