Merge remote-tracking branch 'upstream/master' into mix_20240309HEADrahm

1 files changed, 259 insertions, 74 deletions

diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua
index ed6e30ea10..890c260843 100644
--- a/src/nvim/generators/c_grammar.lua
+++ b/src/nvim/generators/c_grammar.lua
@@ -1,10 +1,37 @@
-local lpeg = vim.lpeg
-
 -- lpeg grammar for building api metadata from a set of header files. It
 -- ignores comments and preprocessor commands and parses a very small subset
 -- of C prototypes with a limited set of types
-local P, R, S = lpeg.P, lpeg.R, lpeg.S
-local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
+
+--- @class nvim.c_grammar.Proto
+--- @field [1] 'proto'
+--- @field pos integer
+--- @field endpos integer
+--- @field fast boolean
+--- @field name string
+--- @field return_type string
+--- @field parameters [string, string][]
+--- @field static true?
+--- @field inline true?
+
+--- @class nvim.c_grammar.Preproc
+--- @field [1] 'preproc'
+--- @field content string
+
+--- @class nvim.c_grammar.Empty
+--- @field [1] 'empty'
+
+--- @alias nvim.c_grammar.result
+--- | nvim.c_grammar.Proto
+--- | nvim.c_grammar.Preproc
+--- | nvim.c_grammar.Empty
+
+--- @class nvim.c_grammar
+--- @field match fun(self, input: string): nvim.c_grammar.result[]
+
+local lpeg = vim.lpeg
+
+local P, R, S, V = lpeg.P, lpeg.R, lpeg.S, lpeg.V
+local C, Ct, Cc, Cg, Cp = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg, lpeg.Cp
 
 --- @param pat vim.lpeg.Pattern
 local function rep(pat)
@@ -21,95 +48,253 @@ local function opt(pat)
   return pat ^ -1
 end
 
-local any = P(1) -- (consume one character)
+local any = P(1)
 local letter = R('az', 'AZ') + S('_$')
 local num = R('09')
 local alpha = letter + num
 local nl = P('\r\n') + P('\n')
-local not_nl = any - nl
 local space = S(' \t')
+local str = P('"') * rep((P('\\') * any) + (1 - P('"'))) * P('"')
+local char = P("'") * (any - P("'")) * P("'")
 local ws = space + nl
-local fill = rep(ws)
-local c_comment = P('//') * rep(not_nl)
-local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(not_nl), 'comment')))
-local c_preproc = P('#') * rep(not_nl)
-local dllexport = P('DLLEXPORT') * rep1(ws)
-
-local typed_container = ((P('ArrayOf(') + P('DictOf(') + P('Dict(')) * rep1(any - P(')')) * P(')'))
-
-local c_id = (typed_container + (letter * rep(alpha)))
-local c_void = P('void')
-
-local c_param_type = (
-  ((P('Error') * fill * P('*') * fill) * Cc('error'))
-  + ((P('Arena') * fill * P('*') * fill) * Cc('arena'))
-  + ((P('lua_State') * fill * P('*') * fill) * Cc('lstate'))
-  + C(opt(P('const ')) * c_id * rep1(ws) * rep1(P('*')))
-  + (C(c_id) * rep1(ws))
+local wb = #-alpha -- word boundary
+local id = letter * rep(alpha)
+
+local comment_inline = P('/*') * rep(1 - P('*/')) * P('*/')
+local comment = P('//') * rep(1 - nl) * nl
+local preproc = Ct(Cc('preproc') * P('#') * Cg(rep(1 - nl) * nl, 'content'))
+
+local fill = rep(ws + comment_inline + comment + preproc)
+
+--- @param s string
+--- @return vim.lpeg.Pattern
+local function word(s)
+  return fill * P(s) * wb * fill
+end
+
+--- @param x vim.lpeg.Pattern
+local function comma1(x)
+  return x * rep(fill * P(',') * fill * x)
+end
+
+--- @param v string
+local function Pf(v)
+  return fill * P(v) * fill
+end
+
+--- @param x vim.lpeg.Pattern
+local function paren(x)
+  return P('(') * fill * x * fill * P(')')
+end
+
+local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(1 - nl), 'comment')))
+
+local braces = P({
+  'S',
+  A = comment_inline + comment + preproc + str + char + (any - S('{}')),
+  S = P('{') * rep(V('A')) * rep(V('S') + V('A')) * P('}'),
+})
+
+-- stylua: ignore start
+local typed_container = P({
+  'S',
+  S = (
+    (P('Union') * paren(comma1(V('ID'))))
+    + (P('ArrayOf') * paren(id * opt(P(',') * fill * rep1(num))))
+    + (P('DictOf') * paren(id))
+    + (P('LuaRefOf') * paren(
+      paren(comma1((V('ID') + str) * rep1(ws) * opt(P('*')) * id))
+      * opt(P(',') * fill * opt(P('*')) * V('ID'))
+    ))
+    + (P('Dict') * paren(id))),
+  ID = V('S') + id,
+})
+-- stylua: ignore end
+
+local ptr_mod = word('restrict') + word('__restrict') + word('const')
+local opt_ptr = rep(Pf('*') * opt(ptr_mod))
+
+--- @param name string
+--- @param var string
+--- @return vim.lpeg.Pattern
+local function attr(name, var)
+  return Cg((P(name) * Cc(true)), var)
+end
+
+--- @param name string
+--- @param var string
+--- @return vim.lpeg.Pattern
+local function attr_num(name, var)
+  return Cg((P(name) * paren(C(rep1(num)))), var)
+end
+
+local fattr = (
+  attr_num('FUNC_API_SINCE', 'since')
+  + attr_num('FUNC_API_DEPRECATED_SINCE', 'deprecated_since')
+  + attr('FUNC_API_FAST', 'fast')
+  + attr('FUNC_API_RET_ALLOC', 'ret_alloc')
+  + attr('FUNC_API_NOEXPORT', 'noexport')
+  + attr('FUNC_API_REMOTE_ONLY', 'remote_only')
+  + attr('FUNC_API_LUA_ONLY', 'lua_only')
+  + attr('FUNC_API_TEXTLOCK_ALLOW_CMDWIN', 'textlock_allow_cmdwin')
+  + attr('FUNC_API_TEXTLOCK', 'textlock')
+  + attr('FUNC_API_REMOTE_IMPL', 'remote_impl')
+  + attr('FUNC_API_COMPOSITOR_IMPL', 'compositor_impl')
+  + attr('FUNC_API_CLIENT_IMPL', 'client_impl')
+  + attr('FUNC_API_CLIENT_IGNORE', 'client_ignore')
+  + (P('FUNC_') * rep(alpha) * opt(fill * paren(rep(1 - P(')') * any))))
 )
 
-local c_type = (C(c_void) * (ws ^ 1)) + c_param_type
-local c_param = Ct(c_param_type * C(c_id))
-local c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0)
-local c_params = Ct(c_void + c_param_list)
+local void = P('void') * wb
 
-local impl_line = (any - P('}')) * opt(rep(not_nl)) * nl
+local api_param_type = (
+  (word('Error') * opt_ptr * Cc('error'))
+  + (word('Arena') * opt_ptr * Cc('arena'))
+  + (word('lua_State') * opt_ptr * Cc('lstate'))
+)
 
-local ignore_line = rep1(not_nl) * nl
+local ctype = C(
+  opt(word('const'))
+    * (
+      typed_container
+      -- 'unsigned' is a type modifier, and a type itself
+      + (word('unsigned char') + word('unsigned'))
+      + (word('struct') * fill * id)
+      + id
+    )
+    * opt(word('const'))
+    * opt_ptr
+)
+
+local return_type = (C(void) * fill) + ctype
 
+-- stylua: ignore start
+local params = Ct(
+  (void * #P(')'))
+  + comma1(Ct(
+      (api_param_type + ctype)
+      * fill
+      * C(id)
+      * rep(Pf('[') * rep(alpha) * Pf(']'))
+      * rep(fill * fattr)
+    ))
+    * opt(Pf(',') * P('...'))
+)
+-- stylua: ignore end
+
+local ignore_line = rep1(1 - nl) * nl
 local empty_line = Ct(Cc('empty') * nl * nl)
 
-local c_proto = Ct(
-  Cc('proto')
-    * opt(dllexport)
-    * opt(Cg(P('static') * fill * Cc(true), 'static'))
-    * Cg(c_type, 'return_type')
-    * Cg(c_id, 'name')
-    * fill
-    * (P('(') * fill * Cg(c_params, 'parameters') * fill * P(')'))
-    * Cg(Cc(false), 'fast')
-    * (fill * Cg((P('FUNC_API_SINCE(') * C(rep1(num))) * P(')'), 'since') ^ -1)
-    * (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(rep1(num))) * P(')'), 'deprecated_since') ^ -1)
-    * (fill * Cg((P('FUNC_API_FAST') * Cc(true)), 'fast') ^ -1)
-    * (fill * Cg((P('FUNC_API_RET_ALLOC') * Cc(true)), 'ret_alloc') ^ -1)
-    * (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1)
-    * (fill * Cg((P('FUNC_API_REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1)
-    * (fill * Cg((P('FUNC_API_LUA_ONLY') * Cc(true)), 'lua_only') ^ -1)
-    * (fill * (Cg(P('FUNC_API_TEXTLOCK_ALLOW_CMDWIN') * Cc(true), 'textlock_allow_cmdwin') + Cg(
-      P('FUNC_API_TEXTLOCK') * Cc(true),
-      'textlock'
-    )) ^ -1)
-    * (fill * Cg((P('FUNC_API_REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1)
-    * (fill * Cg((P('FUNC_API_COMPOSITOR_IMPL') * Cc(true)), 'compositor_impl') ^ -1)
-    * (fill * Cg((P('FUNC_API_CLIENT_IMPL') * Cc(true)), 'client_impl') ^ -1)
-    * (fill * Cg((P('FUNC_API_CLIENT_IGNORE') * Cc(true)), 'client_ignore') ^ -1)
-    * fill
-    * (P(';') + (P('{') * nl + (impl_line ^ 0) * P('}')))
+local proto_name = opt_ptr * fill * id
+
+-- __inline is used in MSVC
+local decl_mod = (
+  Cg(word('static') * Cc(true), 'static')
+  + Cg((word('inline') + word('__inline')) * Cc(true), 'inline')
 )
 
-local dict_key = P('DictKey(') * Cg(rep1(any - P(')')), 'dict_key') * P(')')
-local keyset_field =
-  Ct(Cg(c_id, 'type') * ws * Cg(c_id, 'name') * fill * (dict_key ^ -1) * fill * P(';') * fill)
-local c_keyset = Ct(
-  P('typedef')
-    * ws
-    * P('struct')
+local proto = Ct(
+  Cg(Cp(), 'pos')
+    * Cc('proto')
+    * -#P('typedef')
+    * #alpha
+    * opt(P('DLLEXPORT') * rep1(ws))
+    * rep(decl_mod)
+    * Cg(return_type, 'return_type')
     * fill
-    * P('{')
+    * Cg(proto_name, 'name')
     * fill
-    * Cg(Ct(keyset_field ^ 1), 'fields')
-    * P('}')
-    * fill
-    * P('Dict')
+    * paren(Cg(params, 'parameters'))
+    * Cg(Cc(false), 'fast')
+    * rep(fill * fattr)
+    * Cg(Cp(), 'endpos')
+    * (fill * (S(';') + braces))
+)
+
+local keyset_field = Ct(
+  Cg(ctype, 'type')
     * fill
-    * P('(')
-    * Cg(c_id, 'keyset_name')
+    * Cg(id, 'name')
     * fill
-    * P(')')
-    * P(';')
+    * opt(P('DictKey') * paren(Cg(rep1(1 - P(')')), 'dict_key')))
+    * Pf(';')
 )
 
-local grammar = Ct(
-  rep1(empty_line + c_proto + cdoc_comment + c_comment + c_preproc + ws + c_keyset + ignore_line)
+local keyset = Ct(
+  P('typedef')
+    * word('struct')
+    * Pf('{')
+    * Cg(Ct(rep1(keyset_field)), 'fields')
+    * Pf('}')
+    * P('Dict')
+    * paren(Cg(id, 'keyset_name'))
+    * Pf(';')
 )
-return { grammar = grammar, typed_container = typed_container }
+
+local grammar =
+  Ct(rep1(empty_line + proto + cdoc_comment + comment + preproc + ws + keyset + ignore_line))
+
+if arg[1] == '--test' then
+  for i, t in ipairs({
+    'void multiqueue_put_event(MultiQueue *self, Event event) {} ',
+    'void *xmalloc(size_t size) {} ',
+    {
+      'struct tm *os_localtime_r(const time_t *restrict clock,',
+      '                          struct tm *restrict result) FUNC_ATTR_NONNULL_ALL {}',
+    },
+    {
+      '_Bool',
+      '# 163 "src/nvim/event/multiqueue.c"',
+      '    multiqueue_empty(MultiQueue *self)',
+      '{}',
+    },
+    'const char *find_option_end(const char *arg, OptIndex *opt_idxp) {}',
+    'bool semsg(const char *const fmt, ...) {}',
+    'int32_t utf_ptr2CharInfo_impl(uint8_t const *p, uintptr_t const len) {}',
+    'void ex_argdedupe(exarg_T *eap FUNC_ATTR_UNUSED) {}',
+    'static TermKeySym register_c0(TermKey *tk, TermKeySym sym, unsigned char ctrl, const char *name) {}',
+    'unsigned get_bkc_flags(buf_T *buf) {}',
+    'char *xstpcpy(char *restrict dst, const char *restrict src) {}',
+    'bool try_leave(const TryState *const tstate, Error *const err) {}',
+    'void api_set_error(ErrorType errType) {}',
+    {
+      'void nvim_subscribe(uint64_t channel_id, String event)',
+      'FUNC_API_SINCE(1) FUNC_API_DEPRECATED_SINCE(13) FUNC_API_REMOTE_ONLY',
+      '{}',
+    },
+
+    -- Do not consume leading preproc statements
+    {
+      '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"',
+      'static __inline int mark_global_index(const char name)',
+      '  FUNC_ATTR_CONST',
+      '{}',
+    },
+    {
+      '',
+      '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"',
+      'static __inline int mark_global_index(const char name)',
+      '{}',
+    },
+    {
+      'size_t xstrlcpy(char *__restrict dst, const char *__restrict src, size_t dsize)',
+      ' FUNC_ATTR_NONNULL_ALL',
+      ' {}',
+    },
+  }) do
+    if type(t) == 'table' then
+      t = table.concat(t, '\n') .. '\n'
+    end
+    t = t:gsub(' +', ' ')
+    local r = grammar:match(t)
+    if not r then
+      print('Test ' .. i .. ' failed')
+      print('    |' .. table.concat(vim.split(t, '\n'), '\n    |'))
+    end
+  end
+end
+
+return {
+  grammar = grammar --[[@as nvim.c_grammar]],
+  typed_container = typed_container,
+}