fix(lua): pesc, tbl_islist result types #20751

Problem:
- pesc() returns multiple results, it should return a single result.
- tbl_islist() returns non-boolean in some branches.
- Docstring: @generic must be declared first

Solution:
Constrain docstring annotations.
Fix return types.

Co-authored-by: Justin M. Keyes <justinkz@gmail.com>

3 files changed, 15 insertions, 16 deletions

diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt
index ce16c208cd..70a28a7519 100644
--- a/runtime/doc/lua.txt
+++ b/runtime/doc/lua.txt
@@ -1573,7 +1573,7 @@ gsplit({s}, {sep}, {plain})                                     *vim.gsplit()*
     Parameters: ~
       • {s}      (string) String to split
       • {sep}    (string) Separator or pattern
-      • {plain}  (boolean) If `true` use `sep` literally (passed to
+      • {plain}  (boolean|nil) If `true` use `sep` literally (passed to
                  string.find)
 
     Return: ~
@@ -1645,15 +1645,11 @@ split({s}, {sep}, {kwargs})                                      *vim.split()*
       split("|x|y|z|", "|", {trimempty=true}) => {'x', 'y', 'z'}
 <
 
-    @alias split_kwargs {plain: boolean, trimempty: boolean} | boolean | nil
-
-    See also: ~
-        |vim.gsplit()|
-
     Parameters: ~
       • {s}       (string) String to split
       • {sep}     (string) Separator or pattern
-      • {kwargs}  (table|nil) Keyword arguments:
+      • {kwargs}  ({plain: boolean, trimempty: boolean}|nil) Keyword
+                  arguments:
                   • plain: (boolean) If `true` use `sep` literally (passed to
                     string.find)
                   • trimempty: (boolean) If `true` remove empty items from the
@@ -1662,6 +1658,9 @@ split({s}, {sep}, {kwargs})                                      *vim.split()*
     Return: ~
         string[] List of split components
 
+    See also: ~
+        |vim.gsplit()|
+
 startswith({s}, {prefix})                                   *vim.startswith()*
     Tests if `s` starts with `prefix`.
 
diff --git a/runtime/lua/vim/shared.lua b/runtime/lua/vim/shared.lua
index f03d608e56..f980547ae4 100644
--- a/runtime/lua/vim/shared.lua
+++ b/runtime/lua/vim/shared.lua
@@ -62,7 +62,7 @@ end)()
 ---
 ---@param s string String to split
 ---@param sep string Separator or pattern
----@param plain boolean If `true` use `sep` literally (passed to string.find)
+---@param plain (boolean|nil) If `true` use `sep` literally (passed to string.find)
 ---@return fun():string (function) Iterator over the split components
 function vim.gsplit(s, sep, plain)
   vim.validate({ s = { s, 's' }, sep = { sep, 's' }, plain = { plain, 'b', true } })
@@ -108,11 +108,9 @@ end
 ---
 ---@see |vim.gsplit()|
 ---
----@alias split_kwargs {plain: boolean, trimempty: boolean} | boolean | nil
----
 ---@param s string String to split
 ---@param sep string Separator or pattern
----@param kwargs? {plain: boolean, trimempty: boolean} (table|nil) Keyword arguments:
+---@param kwargs ({plain: boolean, trimempty: boolean}|nil) Keyword arguments:
 ---       - plain: (boolean) If `true` use `sep` literally (passed to string.find)
 ---       - trimempty: (boolean) If `true` remove empty items from the front
 ---         and back of the list
@@ -159,8 +157,8 @@ end
 ---
 ---@see From https://github.com/premake/premake-core/blob/master/src/base/table.lua
 ---
----@param t table<T, any> (table) Table
 ---@generic T: table
+---@param t table<T, any> (table) Table
 ---@return T[] (list) List of keys
 function vim.tbl_keys(t)
   assert(type(t) == 'table', string.format('Expected table, got %s', type(t)))
@@ -417,8 +415,8 @@ end
 ---@generic T: table
 ---@param dst T List which will be modified and appended to
 ---@param src table List from which values will be inserted
----@param start? number Start index on src. Defaults to 1
----@param finish? number Final index on src. Defaults to `#src`
+---@param start (number|nil) Start index on src. Defaults to 1
+---@param finish (number|nil) Final index on src. Defaults to `#src`
 ---@return T dst
 function vim.list_extend(dst, src, start, finish)
   vim.validate({
@@ -486,7 +484,7 @@ function vim.tbl_islist(t)
     -- TODO(bfredl): in the future, we will always be inside nvim
     -- then this check can be deleted.
     if vim._empty_dict_mt == nil then
-      return nil
+      return false
     end
     return getmetatable(t) ~= vim._empty_dict_mt
   end
@@ -544,7 +542,7 @@ end
 ---@return string %-escaped pattern string
 function vim.pesc(s)
   vim.validate({ s = { s, 's' } })
-  return s:gsub('[%(%)%.%%%+%-%*%?%[%]%^%$]', '%%%1')
+  return (s:gsub('[%(%)%.%%%+%-%*%?%[%]%^%$]', '%%%1'))
 end
 
 --- Tests if `s` starts with `prefix`.
diff --git a/test/functional/lua/vim_spec.lua b/test/functional/lua/vim_spec.lua
index ecfb7275cd..38f772c5cf 100644
--- a/test/functional/lua/vim_spec.lua
+++ b/test/functional/lua/vim_spec.lua
@@ -430,6 +430,8 @@ describe('lua stdlib', function()
   it('vim.pesc', function()
     eq('foo%-bar', exec_lua([[return vim.pesc('foo-bar')]]))
     eq('foo%%%-bar', exec_lua([[return vim.pesc(vim.pesc('foo-bar'))]]))
+    -- pesc() returns one result. #20751
+    eq({'x'}, exec_lua([[return {vim.pesc('x')}]]))
 
     -- Validates args.
     matches('s: expected string, got number',