feat(docs): generate builtin.txt (#24493)

- eval.lua is now the source of truth.

- Formatting is much more consistent.

- Fixed Lua type generation for polymorphic functions (get(), etc).

- Removed "Overview" section from builtin.txt
  - Can generate this if we really want it.

- Moved functions from sign.txt and testing.txt into builtin.txt.

- Removed the *timer* *timers* tags since libuv timers via vim.uv should be preferred.

- Removed the temp-file-name tag from tempname()

- Moved lueval() from lua.txt to builtin.txt.

* Fix indent

* fixup!

* fixup! fixup!

* fixup! better tag formatting

* fixup: revert changes no longer needed

* fixup! CI

---------

Co-authored-by: zeertzjq <zeertzjq@outlook.com>

3 files changed, 277 insertions, 249 deletions

diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua
new file mode 100755
index 0000000000..7193346758
--- /dev/null
+++ b/scripts/gen_eval_files.lua
@@ -0,0 +1,274 @@
+#!/usr/bin/env -S nvim -l
+-- Generator for src/nvim/eval.lua
+
+local funcs = require('src/nvim/eval').funcs
+
+local LUA_KEYWORDS = {
+  ['and'] = true,
+  ['end'] = true,
+  ['function'] = true,
+  ['or'] = true,
+  ['if'] = true,
+  ['while'] = true,
+  ['repeat'] = true,
+}
+
+--- @param f string
+--- @param params {[1]:string,[2]:string}[]|true
+local function render_fun_sig(f, params)
+  local param_str --- @type string
+  if params == true then
+    param_str = '...'
+  else
+    param_str = table.concat(
+      vim.tbl_map(
+        --- @param v {[1]:string,[2]:string}
+        function(v)
+          return v[1]
+        end,
+        params
+      ),
+      ', '
+    )
+  end
+
+  if LUA_KEYWORDS[f] then
+    return string.format("vim.fn['%s'] = function(%s) end", f, param_str)
+  else
+    return string.format('function vim.fn.%s(%s) end', f, param_str)
+  end
+end
+
+--- Uniquify names
+--- Fix any names that are lua keywords
+--- @param params {[1]:string,[2]:string}[]
+--- @return {[1]:string,[2]:string}[]
+local function process_params(params)
+  local seen = {} --- @type table<string,true>
+  local sfx = 1
+
+  for _, p in ipairs(params) do
+    if LUA_KEYWORDS[p[1]] then
+      p[1] = p[1] .. '_'
+    end
+    if seen[p[1]] then
+      p[1] = p[1] .. sfx
+      sfx = sfx + 1
+    else
+      seen[p[1]] = true
+    end
+  end
+
+  return params
+end
+
+--- @param f string
+--- @param fun vim.EvalFn
+--- @param write fun(line: string)
+local function render_vimfn(f, fun, write)
+  if fun.lua == false then
+    return
+  end
+
+  local funname = fun.name or f
+
+  local params = process_params(fun.params)
+
+  if fun.signature then
+    write('')
+    if fun.deprecated then
+      write('--- @deprecated')
+    end
+
+    local desc = fun.desc
+
+    if desc then
+      desc = desc:gsub('\n%s*\n%s*$', '\n')
+      for _, l in ipairs(vim.split(desc, '\n', { plain = true })) do
+        l = l:gsub('^      ', ''):gsub('\t', '  '):gsub('@', '\\@')
+        write('--- ' .. l)
+      end
+    end
+
+    local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0
+
+    for i, param in ipairs(params) do
+      local pname, ptype = param[1], param[2]
+      local optional = (pname ~= '...' and i > req_args) and '?' or ''
+      write(string.format('--- @param %s%s %s', pname, optional, ptype))
+    end
+
+    if fun.returns ~= false then
+      write('--- @return ' .. (fun.returns or 'any'))
+    end
+
+    write(render_fun_sig(funname, params))
+
+    return
+  end
+
+  print('no doc for', funname)
+end
+
+--- @type table<string,true>
+local rendered_tags = {}
+
+--- @param f string
+--- @param fun vim.EvalFn
+--- @param write fun(line: string)
+local function render_eval_doc(f, fun, write)
+  if fun.deprecated then
+    return
+  end
+
+  if not fun.signature then
+    return
+  end
+
+  local desc = fun.desc
+
+  if not desc then
+    write(fun.signature)
+    return
+  end
+
+  local name = fun.name or f
+  local tags = { '*' .. name .. '()*' }
+  if fun.tags then
+    for _, t in ipairs(fun.tags) do
+      tags[#tags + 1] = '*' .. t .. '*'
+    end
+  end
+  local tag = table.concat(tags, ' ')
+
+  local siglen = #fun.signature
+  if rendered_tags[name] then
+    write(fun.signature)
+  else
+    if siglen + #tag > 80 then
+      write(string.rep('\t', 6) .. tag)
+      write(fun.signature)
+    else
+      local tt = math.max(1, (76 - siglen - #tag) / 8)
+      write(string.format('%s%s%s', fun.signature, string.rep('\t', tt), tag))
+    end
+  end
+  rendered_tags[name] = true
+
+  desc = vim.trim(desc)
+  local desc_l = vim.split(desc, '\n', { plain = true })
+  for _, l in ipairs(desc_l) do
+    l = l:gsub('^      ', '')
+    if vim.startswith(l, '<') and not l:match('^<[A-Z][A-Z]') then
+      write('<\t\t' .. l:sub(2))
+    else
+      write('\t\t' .. l)
+    end
+  end
+
+  if #desc_l > 0 and not desc_l[#desc_l]:match('^<?$') then
+    write('')
+  end
+end
+
+--- @class nvim.gen_eval_files.elem
+--- @field path string
+--- @field render fun(f:string,fun:vim.EvalFn,write:fun(line:string))
+--- @field header? string[]
+--- @field footer? string[]
+
+--- @type nvim.gen_eval_files.elem[]
+local CONFIG = {
+  {
+    path = 'runtime/lua/vim/_meta/vimfn.lua',
+    render = render_vimfn,
+    header = {
+      '--- @meta',
+      '-- THIS FILE IS GENERATED',
+      '-- DO NOT EDIT',
+    },
+  },
+  {
+    path = 'runtime/doc/builtin.txt',
+    render = render_eval_doc,
+    header = {
+      '*builtin.txt*	Nvim',
+      '',
+      '',
+      '\t\t  VIM REFERENCE MANUAL\t  by Bram Moolenaar',
+      '',
+      '',
+      'Builtin functions\t\t*vimscript-functions* *builtin-functions*',
+      '',
+      'For functions grouped by what they are used for see |function-list|.',
+      '',
+      '\t\t\t\t      Type |gO| to see the table of contents.',
+      '==============================================================================',
+      '1. Details					*builtin-function-details*',
+      '',
+    },
+    footer = {
+      '==============================================================================',
+      '2. Matching a pattern in a String			*string-match*',
+      '',
+      'This is common between several functions. A regexp pattern as explained at',
+      '|pattern| is normally used to find a match in the buffer lines.  When a',
+      'pattern is used to find a match in a String, almost everything works in the',
+      'same way.  The difference is that a String is handled like it is one line.',
+      'When it contains a "\\n" character, this is not seen as a line break for the',
+      'pattern.  It can be matched with a "\\n" in the pattern, or with ".".  Example:',
+      '>',
+      '\t:let a = "aaaa\\nxxxx"',
+      '\t:echo matchstr(a, "..\\n..")',
+      '\taa',
+      '\txx',
+      '\t:echo matchstr(a, "a.x")',
+      '\ta',
+      '\tx',
+      '',
+      'Don\'t forget that "^" will only match at the first character of the String and',
+      '"$" at the last character of the string.  They don\'t match after or before a',
+      '"\\n".',
+      '',
+      ' vim:tw=78:ts=8:noet:ft=help:norl:',
+    },
+  },
+}
+
+--- @param elem nvim.gen_eval_files.elem
+local function render(elem)
+  local o = assert(io.open(elem.path, 'w'))
+
+  --- @param l string
+  local function write(l)
+    local l1 = l:gsub('%s+$', '')
+    o:write(l1)
+    o:write('\n')
+  end
+
+  for _, l in ipairs(elem.header or {}) do
+    write(l)
+  end
+
+  --- @type string[]
+  local fnames = vim.tbl_keys(funcs)
+  table.sort(fnames)
+
+  for _, f in ipairs(fnames) do
+    elem.render(f, funcs[f], write)
+  end
+
+  for _, l in ipairs(elem.footer or {}) do
+    write(l)
+  end
+
+  o:close()
+end
+
+local function main()
+  for _, c in ipairs(CONFIG) do
+    render(c)
+  end
+end
+
+main()
diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index dfad1f000c..61f18ad794 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -1359,4 +1359,7 @@ if __name__ == "__main__":
     else:
         main(Doxyfile, args)
 
+    print('Running ./scripts/gen_eval_files.lua')
+    subprocess.call(['./scripts/gen_eval_files.lua'])
+
 # vim: set ft=python ts=4 sw=4 tw=79 et :
diff --git a/scripts/gen_vimfn_types.lua b/scripts/gen_vimfn_types.lua
deleted file mode 100755
index 32de1d3c95..0000000000
--- a/scripts/gen_vimfn_types.lua
+++ /dev/null
@@ -1,249 +0,0 @@
-#!/usr/bin/env -S nvim -l
-
---- @class vim.EvalFn2 : vim.EvalFn
---- @field signature string
---- @field desc string[]
---- @field params {[1]: string, [2]: string}[]
-
---- @param filename string
---- @return string
-local function safe_read(filename)
-  local file, err = io.open(filename, 'r')
-  if not file then
-    error(err)
-  end
-  local content = file:read('*a')
-  io.close(file)
-  return content
-end
-
-local nvim_eval = require'src/nvim/eval'
-
-local funcs = nvim_eval.funcs --[[@as table<string,vim.EvalFn2>]]
-
-local LUA_KEYWORDS = {
-  ['and'] = true,
-  ['end'] = true,
-  ['function'] = true,
-  ['or'] = true,
-  ['if'] = true,
-  ['while'] = true,
-  ['repeat'] = true
-}
-
-local SOURCES = {
-  {
-    path = 'runtime/doc/builtin.txt',
-    from = '^2. Details',
-    to = '==========',
-  },
-  {
-    path = 'runtime/doc/sign.txt',
-    from = '^3. Functions',
-    to = 'vim:'
-  },
-  {
-    path = 'runtime/doc/testing.txt',
-    from = '^3. Assert functions',
-    to = 'vim:'
-  }
-}
-
-local ARG_NAME_TYPES = {
-  col = 'integer',
-  nosuf = 'boolean',
-  dir = 'string',
-  mode = 'string',
-  width = 'integer',
-  height = 'integer',
-  timeout = 'integer',
-  libname = 'string',
-  funcname = 'string',
-  end_ = 'integer',
-  file = 'string',
-  flags = 'string',
-  fname = 'integer',
-  idx = 'integer',
-  lnum = 'integer',
-  mods = 'string',
-  name = 'string',
-  nr = 'integer',
-  options = 'table',
-  opts = 'table',
-  path = 'string',
-  regname = 'string',
-  silent = 'boolean',
-  string = 'string',
-  tabnr = 'integer',
-  varname = 'string',
-  winid = 'integer',
-  winnr = 'integer',
-}
-
-local function process_source(source)
-  local src_txt = safe_read(source.path)
-
-  --- @type string[]
-  local src_lines = vim.split(src_txt, '\n', { plain = true })
-
-  local s = 0
-  for i, l in ipairs(src_lines) do
-    if l:match(source.from) then
-      s = i+1
-    end
-  end
-
-  local lines = {} --- @type string[]
-  local last_f --- @type string?
-  local last_l --- @type string?
-
-  for i = s, #src_lines do
-    local l = src_lines[i]
-    if not l or l:match(source.to) then
-      break
-    end
-    local f = l:match('^([a-z][a-zA-Z0-9_]*)%(')
-    if f then
-      if last_f then
-        if last_l and last_l:find('*' .. f .. '()*', 1, true) then
-          lines[#lines] = nil
-        end
-        funcs[last_f].desc = lines
-      end
-      last_f = f
-      local sig = l:match('[^)]+%)')
-      local params = {} --- @type table[]
-      if sig then
-        for param in string.gmatch(sig, '{([a-z][a-zA-Z0-9_]*)}') do
-          local t = ARG_NAME_TYPES[param] or 'any'
-          params[#params+1] = {param, t}
-        end
-      else
-        print('error parsing', l)
-      end
-
-      funcs[last_f].signature = sig
-      funcs[last_f].params = params
-
-      lines = {}
-    else
-      lines[#lines+1] = l:gsub('^(<?)\t\t', '%1'):gsub('\t', '  ')
-    end
-    last_l = l
-  end
-
-  if last_f then
-    funcs[last_f].desc = lines
-  end
-end
-
-local function render_fun_sig(f, params)
-  local param_str --- @type string
-  if params == true then
-    param_str = '...'
-  else
-    param_str = table.concat(vim.tbl_map(function(v)
-      return v[1]
-    end, params), ', ')
-  end
-
-  if LUA_KEYWORDS[f] then
-    return string.format('vim.fn[\'%s\'] = function(%s) end', f, param_str)
-  else
-    return string.format('function vim.fn.%s(%s) end', f, param_str)
-  end
-end
-
---- Uniquify names
---- Fix any names that are lua keywords
---- @param fun vim.EvalFn2
-local function process_params(fun)
-  if not fun.params then
-    return
-  end
-
-  local seen = {} --- @type table<string,true>
-  local sfx = 1
-
-  for _, p in ipairs(fun.params) do
-    if LUA_KEYWORDS[p[1]] then
-      p[1] = p[1]..'_'
-    end
-    if seen[p[1]] then
-      p[1] = p[1]..sfx
-      sfx = sfx + 1
-    else
-      seen[p[1]] = true
-    end
-  end
-end
-
---- @param funname string
---- @param fun vim.EvalFn2
---- @param write fun(line: string)
-local function render_fun(funname, fun, write)
-  if fun.deprecated then
-    write('')
-    write('--- @deprecated')
-    for _, l in ipairs(fun.deprecated) do
-      write('--- '.. l)
-    end
-    write(render_fun_sig(funname, true))
-    return
-  end
-
-  if fun.desc and fun.signature then
-    write('')
-    for _, l in ipairs(fun.desc) do
-      write('--- '.. l:gsub('@', '\\@'))
-    end
-
-    local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0
-
-    for i, param in ipairs(fun.params) do
-      if i <= req_args then
-        write('--- @param '..param[1]..' '..param[2])
-      else
-        write('--- @param '..param[1]..'? '..param[2])
-      end
-    end
-    if fun.returns ~= false then
-      write('--- @return '..(fun.returns or 'any'))
-    end
-    write(render_fun_sig(funname, fun.params))
-    return
-  end
-
-  print('no doc for', funname)
-end
-
-local function main(outfile)
-  local o = assert(io.open(outfile, 'w'))
-
-  local function write(l)
-    local l1 = l:gsub('%s+$', '')
-    o:write(l1)
-    o:write('\n')
-  end
-
-  for _, source in ipairs(SOURCES) do
-    process_source(source)
-  end
-
-  --- @type string[]
-  local fnames = vim.tbl_keys(funcs)
-  table.sort(fnames)
-
-  write('--- @meta')
-  write('-- THIS FILE IS GENERATED')
-  write('-- DO NOT EDIT')
-
-  for _, f in ipairs(fnames) do
-    local fun = funcs[f]
-    process_params(fun)
-    render_fun(f, fun, write)
-  end
-end
-
-main('runtime/lua/vim/_meta/vimfn.lua')
-