Merge remote-tracking branch 'upstream/master' into mix_20240309HEADrahm

8 files changed, 797 insertions, 632 deletions

diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua
index ed6e30ea10..890c260843 100644
--- a/src/nvim/generators/c_grammar.lua
+++ b/src/nvim/generators/c_grammar.lua
@@ -1,10 +1,37 @@
-local lpeg = vim.lpeg
-
 -- lpeg grammar for building api metadata from a set of header files. It
 -- ignores comments and preprocessor commands and parses a very small subset
 -- of C prototypes with a limited set of types
-local P, R, S = lpeg.P, lpeg.R, lpeg.S
-local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
+
+--- @class nvim.c_grammar.Proto
+--- @field [1] 'proto'
+--- @field pos integer
+--- @field endpos integer
+--- @field fast boolean
+--- @field name string
+--- @field return_type string
+--- @field parameters [string, string][]
+--- @field static true?
+--- @field inline true?
+
+--- @class nvim.c_grammar.Preproc
+--- @field [1] 'preproc'
+--- @field content string
+
+--- @class nvim.c_grammar.Empty
+--- @field [1] 'empty'
+
+--- @alias nvim.c_grammar.result
+--- | nvim.c_grammar.Proto
+--- | nvim.c_grammar.Preproc
+--- | nvim.c_grammar.Empty
+
+--- @class nvim.c_grammar
+--- @field match fun(self, input: string): nvim.c_grammar.result[]
+
+local lpeg = vim.lpeg
+
+local P, R, S, V = lpeg.P, lpeg.R, lpeg.S, lpeg.V
+local C, Ct, Cc, Cg, Cp = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg, lpeg.Cp
 
 --- @param pat vim.lpeg.Pattern
 local function rep(pat)
@@ -21,95 +48,253 @@ local function opt(pat)
   return pat ^ -1
 end
 
-local any = P(1) -- (consume one character)
+local any = P(1)
 local letter = R('az', 'AZ') + S('_$')
 local num = R('09')
 local alpha = letter + num
 local nl = P('\r\n') + P('\n')
-local not_nl = any - nl
 local space = S(' \t')
+local str = P('"') * rep((P('\\') * any) + (1 - P('"'))) * P('"')
+local char = P("'") * (any - P("'")) * P("'")
 local ws = space + nl
-local fill = rep(ws)
-local c_comment = P('//') * rep(not_nl)
-local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(not_nl), 'comment')))
-local c_preproc = P('#') * rep(not_nl)
-local dllexport = P('DLLEXPORT') * rep1(ws)
-
-local typed_container = ((P('ArrayOf(') + P('DictOf(') + P('Dict(')) * rep1(any - P(')')) * P(')'))
-
-local c_id = (typed_container + (letter * rep(alpha)))
-local c_void = P('void')
-
-local c_param_type = (
-  ((P('Error') * fill * P('*') * fill) * Cc('error'))
-  + ((P('Arena') * fill * P('*') * fill) * Cc('arena'))
-  + ((P('lua_State') * fill * P('*') * fill) * Cc('lstate'))
-  + C(opt(P('const ')) * c_id * rep1(ws) * rep1(P('*')))
-  + (C(c_id) * rep1(ws))
+local wb = #-alpha -- word boundary
+local id = letter * rep(alpha)
+
+local comment_inline = P('/*') * rep(1 - P('*/')) * P('*/')
+local comment = P('//') * rep(1 - nl) * nl
+local preproc = Ct(Cc('preproc') * P('#') * Cg(rep(1 - nl) * nl, 'content'))
+
+local fill = rep(ws + comment_inline + comment + preproc)
+
+--- @param s string
+--- @return vim.lpeg.Pattern
+local function word(s)
+  return fill * P(s) * wb * fill
+end
+
+--- @param x vim.lpeg.Pattern
+local function comma1(x)
+  return x * rep(fill * P(',') * fill * x)
+end
+
+--- @param v string
+local function Pf(v)
+  return fill * P(v) * fill
+end
+
+--- @param x vim.lpeg.Pattern
+local function paren(x)
+  return P('(') * fill * x * fill * P(')')
+end
+
+local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(1 - nl), 'comment')))
+
+local braces = P({
+  'S',
+  A = comment_inline + comment + preproc + str + char + (any - S('{}')),
+  S = P('{') * rep(V('A')) * rep(V('S') + V('A')) * P('}'),
+})
+
+-- stylua: ignore start
+local typed_container = P({
+  'S',
+  S = (
+    (P('Union') * paren(comma1(V('ID'))))
+    + (P('ArrayOf') * paren(id * opt(P(',') * fill * rep1(num))))
+    + (P('DictOf') * paren(id))
+    + (P('LuaRefOf') * paren(
+      paren(comma1((V('ID') + str) * rep1(ws) * opt(P('*')) * id))
+      * opt(P(',') * fill * opt(P('*')) * V('ID'))
+    ))
+    + (P('Dict') * paren(id))),
+  ID = V('S') + id,
+})
+-- stylua: ignore end
+
+local ptr_mod = word('restrict') + word('__restrict') + word('const')
+local opt_ptr = rep(Pf('*') * opt(ptr_mod))
+
+--- @param name string
+--- @param var string
+--- @return vim.lpeg.Pattern
+local function attr(name, var)
+  return Cg((P(name) * Cc(true)), var)
+end
+
+--- @param name string
+--- @param var string
+--- @return vim.lpeg.Pattern
+local function attr_num(name, var)
+  return Cg((P(name) * paren(C(rep1(num)))), var)
+end
+
+local fattr = (
+  attr_num('FUNC_API_SINCE', 'since')
+  + attr_num('FUNC_API_DEPRECATED_SINCE', 'deprecated_since')
+  + attr('FUNC_API_FAST', 'fast')
+  + attr('FUNC_API_RET_ALLOC', 'ret_alloc')
+  + attr('FUNC_API_NOEXPORT', 'noexport')
+  + attr('FUNC_API_REMOTE_ONLY', 'remote_only')
+  + attr('FUNC_API_LUA_ONLY', 'lua_only')
+  + attr('FUNC_API_TEXTLOCK_ALLOW_CMDWIN', 'textlock_allow_cmdwin')
+  + attr('FUNC_API_TEXTLOCK', 'textlock')
+  + attr('FUNC_API_REMOTE_IMPL', 'remote_impl')
+  + attr('FUNC_API_COMPOSITOR_IMPL', 'compositor_impl')
+  + attr('FUNC_API_CLIENT_IMPL', 'client_impl')
+  + attr('FUNC_API_CLIENT_IGNORE', 'client_ignore')
+  + (P('FUNC_') * rep(alpha) * opt(fill * paren(rep(1 - P(')') * any))))
 )
 
-local c_type = (C(c_void) * (ws ^ 1)) + c_param_type
-local c_param = Ct(c_param_type * C(c_id))
-local c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0)
-local c_params = Ct(c_void + c_param_list)
+local void = P('void') * wb
 
-local impl_line = (any - P('}')) * opt(rep(not_nl)) * nl
+local api_param_type = (
+  (word('Error') * opt_ptr * Cc('error'))
+  + (word('Arena') * opt_ptr * Cc('arena'))
+  + (word('lua_State') * opt_ptr * Cc('lstate'))
+)
 
-local ignore_line = rep1(not_nl) * nl
+local ctype = C(
+  opt(word('const'))
+    * (
+      typed_container
+      -- 'unsigned' is a type modifier, and a type itself
+      + (word('unsigned char') + word('unsigned'))
+      + (word('struct') * fill * id)
+      + id
+    )
+    * opt(word('const'))
+    * opt_ptr
+)
+
+local return_type = (C(void) * fill) + ctype
 
+-- stylua: ignore start
+local params = Ct(
+  (void * #P(')'))
+  + comma1(Ct(
+      (api_param_type + ctype)
+      * fill
+      * C(id)
+      * rep(Pf('[') * rep(alpha) * Pf(']'))
+      * rep(fill * fattr)
+    ))
+    * opt(Pf(',') * P('...'))
+)
+-- stylua: ignore end
+
+local ignore_line = rep1(1 - nl) * nl
 local empty_line = Ct(Cc('empty') * nl * nl)
 
-local c_proto = Ct(
-  Cc('proto')
-    * opt(dllexport)
-    * opt(Cg(P('static') * fill * Cc(true), 'static'))
-    * Cg(c_type, 'return_type')
-    * Cg(c_id, 'name')
-    * fill
-    * (P('(') * fill * Cg(c_params, 'parameters') * fill * P(')'))
-    * Cg(Cc(false), 'fast')
-    * (fill * Cg((P('FUNC_API_SINCE(') * C(rep1(num))) * P(')'), 'since') ^ -1)
-    * (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(rep1(num))) * P(')'), 'deprecated_since') ^ -1)
-    * (fill * Cg((P('FUNC_API_FAST') * Cc(true)), 'fast') ^ -1)
-    * (fill * Cg((P('FUNC_API_RET_ALLOC') * Cc(true)), 'ret_alloc') ^ -1)
-    * (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1)
-    * (fill * Cg((P('FUNC_API_REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1)
-    * (fill * Cg((P('FUNC_API_LUA_ONLY') * Cc(true)), 'lua_only') ^ -1)
-    * (fill * (Cg(P('FUNC_API_TEXTLOCK_ALLOW_CMDWIN') * Cc(true), 'textlock_allow_cmdwin') + Cg(
-      P('FUNC_API_TEXTLOCK') * Cc(true),
-      'textlock'
-    )) ^ -1)
-    * (fill * Cg((P('FUNC_API_REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1)
-    * (fill * Cg((P('FUNC_API_COMPOSITOR_IMPL') * Cc(true)), 'compositor_impl') ^ -1)
-    * (fill * Cg((P('FUNC_API_CLIENT_IMPL') * Cc(true)), 'client_impl') ^ -1)
-    * (fill * Cg((P('FUNC_API_CLIENT_IGNORE') * Cc(true)), 'client_ignore') ^ -1)
-    * fill
-    * (P(';') + (P('{') * nl + (impl_line ^ 0) * P('}')))
+local proto_name = opt_ptr * fill * id
+
+-- __inline is used in MSVC
+local decl_mod = (
+  Cg(word('static') * Cc(true), 'static')
+  + Cg((word('inline') + word('__inline')) * Cc(true), 'inline')
 )
 
-local dict_key = P('DictKey(') * Cg(rep1(any - P(')')), 'dict_key') * P(')')
-local keyset_field =
-  Ct(Cg(c_id, 'type') * ws * Cg(c_id, 'name') * fill * (dict_key ^ -1) * fill * P(';') * fill)
-local c_keyset = Ct(
-  P('typedef')
-    * ws
-    * P('struct')
+local proto = Ct(
+  Cg(Cp(), 'pos')
+    * Cc('proto')
+    * -#P('typedef')
+    * #alpha
+    * opt(P('DLLEXPORT') * rep1(ws))
+    * rep(decl_mod)
+    * Cg(return_type, 'return_type')
     * fill
-    * P('{')
+    * Cg(proto_name, 'name')
     * fill
-    * Cg(Ct(keyset_field ^ 1), 'fields')
-    * P('}')
-    * fill
-    * P('Dict')
+    * paren(Cg(params, 'parameters'))
+    * Cg(Cc(false), 'fast')
+    * rep(fill * fattr)
+    * Cg(Cp(), 'endpos')
+    * (fill * (S(';') + braces))
+)
+
+local keyset_field = Ct(
+  Cg(ctype, 'type')
     * fill
-    * P('(')
-    * Cg(c_id, 'keyset_name')
+    * Cg(id, 'name')
     * fill
-    * P(')')
-    * P(';')
+    * opt(P('DictKey') * paren(Cg(rep1(1 - P(')')), 'dict_key')))
+    * Pf(';')
 )
 
-local grammar = Ct(
-  rep1(empty_line + c_proto + cdoc_comment + c_comment + c_preproc + ws + c_keyset + ignore_line)
+local keyset = Ct(
+  P('typedef')
+    * word('struct')
+    * Pf('{')
+    * Cg(Ct(rep1(keyset_field)), 'fields')
+    * Pf('}')
+    * P('Dict')
+    * paren(Cg(id, 'keyset_name'))
+    * Pf(';')
 )
-return { grammar = grammar, typed_container = typed_container }
+
+local grammar =
+  Ct(rep1(empty_line + proto + cdoc_comment + comment + preproc + ws + keyset + ignore_line))
+
+if arg[1] == '--test' then
+  for i, t in ipairs({
+    'void multiqueue_put_event(MultiQueue *self, Event event) {} ',
+    'void *xmalloc(size_t size) {} ',
+    {
+      'struct tm *os_localtime_r(const time_t *restrict clock,',
+      '                          struct tm *restrict result) FUNC_ATTR_NONNULL_ALL {}',
+    },
+    {
+      '_Bool',
+      '# 163 "src/nvim/event/multiqueue.c"',
+      '    multiqueue_empty(MultiQueue *self)',
+      '{}',
+    },
+    'const char *find_option_end(const char *arg, OptIndex *opt_idxp) {}',
+    'bool semsg(const char *const fmt, ...) {}',
+    'int32_t utf_ptr2CharInfo_impl(uint8_t const *p, uintptr_t const len) {}',
+    'void ex_argdedupe(exarg_T *eap FUNC_ATTR_UNUSED) {}',
+    'static TermKeySym register_c0(TermKey *tk, TermKeySym sym, unsigned char ctrl, const char *name) {}',
+    'unsigned get_bkc_flags(buf_T *buf) {}',
+    'char *xstpcpy(char *restrict dst, const char *restrict src) {}',
+    'bool try_leave(const TryState *const tstate, Error *const err) {}',
+    'void api_set_error(ErrorType errType) {}',
+    {
+      'void nvim_subscribe(uint64_t channel_id, String event)',
+      'FUNC_API_SINCE(1) FUNC_API_DEPRECATED_SINCE(13) FUNC_API_REMOTE_ONLY',
+      '{}',
+    },
+
+    -- Do not consume leading preproc statements
+    {
+      '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"',
+      'static __inline int mark_global_index(const char name)',
+      '  FUNC_ATTR_CONST',
+      '{}',
+    },
+    {
+      '',
+      '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"',
+      'static __inline int mark_global_index(const char name)',
+      '{}',
+    },
+    {
+      'size_t xstrlcpy(char *__restrict dst, const char *__restrict src, size_t dsize)',
+      ' FUNC_ATTR_NONNULL_ALL',
+      ' {}',
+    },
+  }) do
+    if type(t) == 'table' then
+      t = table.concat(t, '\n') .. '\n'
+    end
+    t = t:gsub(' +', ' ')
+    local r = grammar:match(t)
+    if not r then
+      print('Test ' .. i .. ' failed')
+      print('    |' .. table.concat(vim.split(t, '\n'), '\n    |'))
+    end
+  end
+end
+
+return {
+  grammar = grammar --[[@as nvim.c_grammar]],
+  typed_container = typed_container,
+}
diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua
index a78f746fee..c987037324 100644
--- a/src/nvim/generators/gen_api_dispatch.lua
+++ b/src/nvim/generators/gen_api_dispatch.lua
@@ -1,3 +1,10 @@
+-- Example (manual) invocation:
+--
+--    make
+--    cp build/nvim_version.lua src/nvim/
+--    cd src/nvim
+--    nvim -l generators/gen_api_dispatch.lua "../../build/src/nvim/auto/api/private/dispatch_wrappers.generated.h" "../../build/src/nvim/auto/api/private/api_metadata.generated.h" "../../build/funcs_metadata.mpack" "../../build/src/nvim/auto/lua_api_c_bindings.generated.h" "../../build/src/nvim/auto/keysets_defs.generated.h" "../../build/ui_metadata.mpack" "../../build/cmake.config/auto/versiondef_git.h" "./api/autocmd.h" "./api/buffer.h" "./api/command.h" "./api/deprecated.h" "./api/extmark.h" "./api/keysets_defs.h" "./api/options.h" "./api/tabpage.h" "./api/ui.h" "./api/vim.h" "./api/vimscript.h" "./api/win_config.h" "./api/window.h" "../../build/include/api/autocmd.h.generated.h" "../../build/include/api/buffer.h.generated.h" "../../build/include/api/command.h.generated.h" "../../build/include/api/deprecated.h.generated.h" "../../build/include/api/extmark.h.generated.h" "../../build/include/api/options.h.generated.h" "../../build/include/api/tabpage.h.generated.h" "../../build/include/api/ui.h.generated.h" "../../build/include/api/vim.h.generated.h" "../../build/include/api/vimscript.h.generated.h" "../../build/include/api/win_config.h.generated.h" "../../build/include/api/window.h.generated.h"
+
 local mpack = vim.mpack
 
 local hashy = require 'generators.hashy'
@@ -8,7 +15,7 @@ assert(#arg >= pre_args)
 local dispatch_outputf = arg[1]
 -- output h file with packed metadata (api_metadata.generated.h)
 local api_metadata_outputf = arg[2]
--- output metadata mpack file, for use by other build scripts (api_metadata.mpack)
+-- output metadata mpack file, for use by other build scripts (funcs_metadata.mpack)
 local mpack_outputf = arg[3]
 local lua_c_bindings_outputf = arg[4] -- lua_api_c_bindings.generated.c
 local keysets_outputf = arg[5] -- keysets_defs.generated.h
@@ -235,7 +242,7 @@ for x in string.gmatch(ui_options_text, '"([a-z][a-z_]+)"') do
   table.insert(ui_options, x)
 end
 
-local version = require 'nvim_version'
+local version = require 'nvim_version' -- `build/nvim_version.lua` file.
 local git_version = io.open(git_version_inputf):read '*a'
 local version_build = string.match(git_version, '#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL
 
@@ -266,10 +273,7 @@ fixdict(1 + #version)
 for _, item in ipairs(version) do
   -- NB: all items are mandatory. But any error will be less confusing
   -- with placeholder vim.NIL (than invalid mpack data)
-  local val = item[2]
-  if val == nil then
-    val = vim.NIL
-  end
+  local val = item[2] == nil and vim.NIL or item[2]
   put(item[1], val)
 end
 put('build', version_build)
@@ -347,12 +351,16 @@ for _, k in ipairs(keysets) do
   local function typename(type)
     if type == 'HLGroupID' then
       return 'kObjectTypeInteger'
+    elseif not type or vim.startswith(type, 'Union') then
+      return 'kObjectTypeNil'
+    elseif vim.startswith(type, 'LuaRefOf') then
+      return 'kObjectTypeLuaRef'
     elseif type == 'StringArray' then
       return 'kUnpackTypeStringArray'
-    elseif type ~= nil then
-      return 'kObjectType' .. type
+    elseif vim.startswith(type, 'ArrayOf') then
+      return 'kObjectTypeArray'
     else
-      return 'kObjectTypeNil'
+      return 'kObjectType' .. type
     end
   end
 
diff --git a/src/nvim/generators/gen_api_ui_events.lua b/src/nvim/generators/gen_api_ui_events.lua
index 30a83330eb..a3bb76cb91 100644
--- a/src/nvim/generators/gen_api_ui_events.lua
+++ b/src/nvim/generators/gen_api_ui_events.lua
@@ -98,7 +98,7 @@ local function call_ui_event_method(output, ev)
 end
 
 events = vim.tbl_filter(function(ev)
-  return ev[1] ~= 'empty'
+  return ev[1] ~= 'empty' and ev[1] ~= 'preproc'
 end, events)
 
 for i = 1, #events do
diff --git a/src/nvim/generators/gen_declarations.lua b/src/nvim/generators/gen_declarations.lua
index 2ec0e9ab68..6e1ea92572 100644
--- a/src/nvim/generators/gen_declarations.lua
+++ b/src/nvim/generators/gen_declarations.lua
@@ -1,136 +1,105 @@
-local fname = arg[1]
-local static_fname = arg[2]
-local non_static_fname = arg[3]
-local preproc_fname = arg[4]
-local static_basename = arg[5]
-
-local lpeg = vim.lpeg
-
-local fold = function(func, ...)
-  local result = nil
-  for _, v in ipairs({ ... }) do
-    if result == nil then
-      result = v
-    else
-      result = func(result, v)
+local grammar = require('generators.c_grammar').grammar
+
+--- @param fname string
+--- @return string?
+local function read_file(fname)
+  local f = io.open(fname, 'r')
+  if not f then
+    return
+  end
+  local contents = f:read('*a')
+  f:close()
+  return contents
+end
+
+--- @param fname string
+--- @param contents string[]
+local function write_file(fname, contents)
+  local contents_s = table.concat(contents, '\n') .. '\n'
+  local fcontents = read_file(fname)
+  if fcontents == contents_s then
+    return
+  end
+  local f = assert(io.open(fname, 'w'))
+  f:write(contents_s)
+  f:close()
+end
+
+--- @param fname string
+--- @param non_static_fname string
+--- @return string? non_static
+local function add_iwyu_non_static(fname, non_static_fname)
+  if fname:find('.*/src/nvim/.*%.c$') then
+    -- Add an IWYU pragma comment if the corresponding .h file exists.
+    local header_fname = fname:sub(1, -3) .. '.h'
+    local header_f = io.open(header_fname, 'r')
+    if header_f then
+      header_f:close()
+      return (header_fname:gsub('.*/src/nvim/', 'nvim/'))
     end
+  elseif non_static_fname:find('/include/api/private/dispatch_wrappers%.h%.generated%.h$') then
+    return 'nvim/api/private/dispatch.h'
+  elseif non_static_fname:find('/include/ui_events_call%.h%.generated%.h$') then
+    return 'nvim/ui.h'
+  elseif non_static_fname:find('/include/ui_events_client%.h%.generated%.h$') then
+    return 'nvim/ui_client.h'
+  elseif non_static_fname:find('/include/ui_events_remote%.h%.generated%.h$') then
+    return 'nvim/api/ui.h'
   end
-  return result
 end
 
-local folder = function(func)
-  return function(...)
-    return fold(func, ...)
+--- @param d string
+local function process_decl(d)
+  -- Comments are really handled by preprocessor, so the following is not
+  -- needed
+  d = d:gsub('/%*.-%*/', '')
+  d = d:gsub('//.-\n', '\n')
+  d = d:gsub('# .-\n', '')
+  d = d:gsub('\n', ' ')
+  d = d:gsub('%s+', ' ')
+  d = d:gsub(' ?%( ?', '(')
+  d = d:gsub(' ?, ?', ', ')
+  d = d:gsub(' ?(%*+) ?', ' %1')
+  d = d:gsub(' ?(FUNC_ATTR_)', ' %1')
+  d = d:gsub(' $', '')
+  d = d:gsub('^ ', '')
+  return d .. ';'
+end
+
+--- @param fname string
+--- @param text string
+--- @return string[] static
+--- @return string[] non_static
+--- @return boolean any_static
+local function gen_declarations(fname, text)
+  local non_static = {} --- @type string[]
+  local static = {} --- @type string[]
+
+  local neededfile = fname:match('[^/]+$')
+  local curfile = nil
+  local any_static = false
+  for _, node in ipairs(grammar:match(text)) do
+    if node[1] == 'preproc' then
+      curfile = node.content:match('^%a* %d+ "[^"]-/?([^"/]+)"') or curfile
+    elseif node[1] == 'proto' and curfile == neededfile then
+      local node_text = text:sub(node.pos, node.endpos - 1)
+      local declaration = process_decl(node_text)
+
+      if node.static then
+        if not any_static and declaration:find('FUNC_ATTR_') then
+          any_static = true
+        end
+        static[#static + 1] = declaration
+      else
+        non_static[#non_static + 1] = 'DLLEXPORT ' .. declaration
+      end
+    end
   end
-end
 
-local lit = lpeg.P
-local set = function(...)
-  return lpeg.S(fold(function(a, b)
-    return a .. b
-  end, ...))
-end
-local any_character = lpeg.P(1)
-local rng = function(s, e)
-  return lpeg.R(s .. e)
-end
-local concat = folder(function(a, b)
-  return a * b
-end)
-local branch = folder(function(a, b)
-  return a + b
-end)
-local one_or_more = function(v)
-  return v ^ 1
-end
-local two_or_more = function(v)
-  return v ^ 2
-end
-local any_amount = function(v)
-  return v ^ 0
-end
-local one_or_no = function(v)
-  return v ^ -1
-end
-local look_behind = lpeg.B
-local look_ahead = function(v)
-  return #v
-end
-local neg_look_ahead = function(v)
-  return -v
-end
-local neg_look_behind = function(v)
-  return -look_behind(v)
+  return static, non_static, any_static
 end
 
-local w = branch(rng('a', 'z'), rng('A', 'Z'), lit('_'))
-local aw = branch(w, rng('0', '9'))
-local s = set(' ', '\n', '\t')
-local raw_word = concat(w, any_amount(aw))
-local right_word = concat(raw_word, neg_look_ahead(aw))
-local word = branch(
-  concat(
-    branch(lit('ArrayOf('), lit('DictOf('), lit('Dict(')), -- typed container macro
-    one_or_more(any_character - lit(')')),
-    lit(')')
-  ),
-  concat(neg_look_behind(aw), right_word)
-)
-local inline_comment =
-  concat(lit('/*'), any_amount(concat(neg_look_ahead(lit('*/')), any_character)), lit('*/'))
-local spaces = any_amount(branch(
-  s,
-  -- Comments are really handled by preprocessor, so the following is not needed
-  inline_comment,
-  concat(lit('//'), any_amount(concat(neg_look_ahead(lit('\n')), any_character)), lit('\n')),
-  -- Linemarker inserted by preprocessor
-  concat(lit('# '), any_amount(concat(neg_look_ahead(lit('\n')), any_character)), lit('\n'))
-))
-local typ_part = concat(word, any_amount(concat(spaces, lit('*'))), spaces)
-
-local typ_id = two_or_more(typ_part)
-local arg = typ_id -- argument name is swallowed by typ
-local pattern = concat(
-  any_amount(branch(set(' ', '\t'), inline_comment)),
-  typ_id, -- return type with function name
-  spaces,
-  lit('('),
-  spaces,
-  one_or_no(branch( -- function arguments
-    concat(
-      arg, -- first argument, does not require comma
-      any_amount(concat( -- following arguments, start with a comma
-        spaces,
-        lit(','),
-        spaces,
-        arg,
-        any_amount(concat(lit('['), spaces, any_amount(aw), spaces, lit(']')))
-      )),
-      one_or_no(concat(spaces, lit(','), spaces, lit('...')))
-    ),
-    lit('void') -- also accepts just void
-  )),
-  spaces,
-  lit(')'),
-  any_amount(concat( -- optional attributes
-    spaces,
-    lit('FUNC_'),
-    any_amount(aw),
-    one_or_no(concat( -- attribute argument
-      spaces,
-      lit('('),
-      any_amount(concat(neg_look_ahead(lit(')')), any_character)),
-      lit(')')
-    ))
-  )),
-  look_ahead(concat( -- definition must be followed by "{"
-    spaces,
-    lit('{')
-  ))
-)
-
-if fname == '--help' then
-  print([[
+local usage = [[
 Usage:
 
     gen_declarations.lua definitions.c static.h non-static.h definitions.i
@@ -141,204 +110,77 @@ non-static.h. File `definitions.i' should contain an already preprocessed
 version of definitions.c and it is the only one which is actually parsed,
 definitions.c is needed only to determine functions from which file out of all
 functions found in definitions.i are needed and to generate an IWYU comment.
-
-Additionally uses the following environment variables:
-
-    NVIM_GEN_DECLARATIONS_LINE_NUMBERS:
-        If set to 1 then all generated declarations receive a comment with file
-        name and line number after the declaration. This may be useful for
-        debugging gen_declarations script, but not much beyond that with
-        configured development environment (i.e. with with clang/etc).
-
-        WARNING: setting this to 1 will cause extensive rebuilds: declarations
-                 generator script will not regenerate non-static.h file if its
-                 contents did not change, but including line numbers will make
-                 contents actually change.
-
-                 With contents changed timestamp of the file is regenerated even
-                 when no real changes were made (e.g. a few lines were added to
-                 a function which is not at the bottom of the file).
-
-                 With changed timestamp build system will assume that header
-                 changed, triggering rebuilds of all C files which depend on the
-                 "changed" header.
-]])
-  os.exit()
-end
-
-local preproc_f = io.open(preproc_fname)
-local text = preproc_f:read('*all')
-preproc_f:close()
-
-local non_static = [[
-#define DEFINE_FUNC_ATTRIBUTES
-#include "nvim/func_attr.h"
-#undef DEFINE_FUNC_ATTRIBUTES
-#ifndef DLLEXPORT
-#  ifdef MSWIN
-#    define DLLEXPORT __declspec(dllexport)
-#  else
-#    define DLLEXPORT
-#  endif
-#endif
-]]
-
-local static = [[
-#define DEFINE_FUNC_ATTRIBUTES
-#include "nvim/func_attr.h"
-#undef DEFINE_FUNC_ATTRIBUTES
 ]]
 
-local non_static_footer = [[
-#include "nvim/func_attr.h"
-]]
+local function main()
+  local fname = arg[1]
+  local static_fname = arg[2]
+  local non_static_fname = arg[3]
+  local preproc_fname = arg[4]
+  local static_basename = arg[5]
 
-local static_footer = [[
-#define DEFINE_EMPTY_ATTRIBUTES
-#include "nvim/func_attr.h"  // IWYU pragma: export
-]]
-
-if fname:find('.*/src/nvim/.*%.c$') then
-  -- Add an IWYU pragma comment if the corresponding .h file exists.
-  local header_fname = fname:sub(1, -3) .. '.h'
-  local header_f = io.open(header_fname, 'r')
-  if header_f ~= nil then
-    header_f:close()
-    non_static = ([[
-// IWYU pragma: private, include "%s"
-]]):format(header_fname:gsub('.*/src/nvim/', 'nvim/')) .. non_static
+  if fname == '--help' or #arg < 5 then
+    print(usage)
+    os.exit()
   end
-elseif fname:find('.*/src/nvim/.*%.h$') then
-  static = ([[
-// IWYU pragma: private, include "%s"
-]]):format(fname:gsub('.*/src/nvim/', 'nvim/')) .. static
-elseif non_static_fname:find('/include/api/private/dispatch_wrappers%.h%.generated%.h$') then
-  non_static = [[
-// IWYU pragma: private, include "nvim/api/private/dispatch.h"
-]] .. non_static
-elseif non_static_fname:find('/include/ui_events_call%.h%.generated%.h$') then
-  non_static = [[
-// IWYU pragma: private, include "nvim/ui.h"
-]] .. non_static
-elseif non_static_fname:find('/include/ui_events_client%.h%.generated%.h$') then
-  non_static = [[
-// IWYU pragma: private, include "nvim/ui_client.h"
-]] .. non_static
-elseif non_static_fname:find('/include/ui_events_remote%.h%.generated%.h$') then
-  non_static = [[
-// IWYU pragma: private, include "nvim/api/ui.h"
-]] .. non_static
-end
 
-local filepattern = '^#%a* (%d+) "([^"]-)/?([^"/]+)"'
+  local text = assert(read_file(preproc_fname))
 
-local init = 1
-local curfile = nil
-local neededfile = fname:match('[^/]+$')
-local declline = 0
-local declendpos = 0
-local curdir = nil
-local is_needed_file = false
-local init_is_nl = true
-local any_static = false
-while init ~= nil do
-  if init_is_nl and text:sub(init, init) == '#' then
-    local line, dir, file = text:match(filepattern, init)
-    if file ~= nil then
-      curfile = file
-      is_needed_file = (curfile == neededfile)
-      declline = tonumber(line) - 1
-      curdir = dir:gsub('.*/src/nvim/', '')
-    else
-      declline = declline - 1
-    end
-  elseif init < declendpos then -- luacheck: ignore 542
-    -- Skipping over declaration
-  elseif is_needed_file then
-    s = init
-    local e = pattern:match(text, init)
-    if e ~= nil then
-      local declaration = text:sub(s, e - 1)
-      -- Comments are really handled by preprocessor, so the following is not
-      -- needed
-      declaration = declaration:gsub('/%*.-%*/', '')
-      declaration = declaration:gsub('//.-\n', '\n')
-
-      declaration = declaration:gsub('# .-\n', '')
+  local static_decls, non_static_decls, any_static = gen_declarations(fname, text)
 
-      declaration = declaration:gsub('\n', ' ')
-      declaration = declaration:gsub('%s+', ' ')
-      declaration = declaration:gsub(' ?%( ?', '(')
-      -- declaration = declaration:gsub(' ?%) ?', ')')
-      declaration = declaration:gsub(' ?, ?', ', ')
-      declaration = declaration:gsub(' ?(%*+) ?', ' %1')
-      declaration = declaration:gsub(' ?(FUNC_ATTR_)', ' %1')
-      declaration = declaration:gsub(' $', '')
-      declaration = declaration:gsub('^ ', '')
-      declaration = declaration .. ';'
-
-      if os.getenv('NVIM_GEN_DECLARATIONS_LINE_NUMBERS') == '1' then
-        declaration = declaration .. ('  // %s/%s:%u'):format(curdir, curfile, declline)
-      end
-      declaration = declaration .. '\n'
-      if declaration:sub(1, 6) == 'static' then
-        if declaration:find('FUNC_ATTR_') then
-          any_static = true
-        end
-        static = static .. declaration
-      else
-        declaration = 'DLLEXPORT ' .. declaration
-        non_static = non_static .. declaration
-      end
-      declendpos = e
-    end
-  end
-  init = text:find('[\n;}]', init)
-  if init == nil then
-    break
-  end
-  init_is_nl = text:sub(init, init) == '\n'
-  init = init + 1
-  if init_is_nl and is_needed_file then
-    declline = declline + 1
+  local static = {} --- @type string[]
+  if fname:find('.*/src/nvim/.*%.h$') then
+    static[#static + 1] = ('// IWYU pragma: private, include "%s"'):format(
+      fname:gsub('.*/src/nvim/', 'nvim/')
+    )
   end
-end
-
-non_static = non_static .. non_static_footer
-static = static .. static_footer
-
-local F
-F = io.open(static_fname, 'w')
-F:write(static)
-F:close()
-
-if any_static then
-  F = io.open(fname, 'r')
-  local orig_text = F:read('*a')
-  local pat = '\n#%s?include%s+"' .. static_basename .. '"\n'
-  local pat_comment = '\n#%s?include%s+"' .. static_basename .. '"%s*//'
-  if not string.find(orig_text, pat) and not string.find(orig_text, pat_comment) then
-    error('fail: missing include for ' .. static_basename .. ' in ' .. fname)
+  vim.list_extend(static, {
+    '#define DEFINE_FUNC_ATTRIBUTES',
+    '#include "nvim/func_attr.h"',
+    '#undef DEFINE_FUNC_ATTRIBUTES',
+  })
+  vim.list_extend(static, static_decls)
+  vim.list_extend(static, {
+    '#define DEFINE_EMPTY_ATTRIBUTES',
+    '#include "nvim/func_attr.h"  // IWYU pragma: export',
+    '',
+  })
+
+  write_file(static_fname, static)
+
+  if any_static then
+    local orig_text = assert(read_file(fname))
+    local pat = '\n#%s?include%s+"' .. static_basename .. '"\n'
+    local pat_comment = '\n#%s?include%s+"' .. static_basename .. '"%s*//'
+    if not orig_text:find(pat) and not orig_text:find(pat_comment) then
+      error(('fail: missing include for %s in %s'):format(static_basename, fname))
+    end
   end
-  F:close()
-end
 
-if non_static_fname == 'SKIP' then
-  return -- only want static declarations
-end
-
--- Before generating the non-static headers, check if the current file (if
--- exists) is different from the new one. If they are the same, we won't touch
--- the current version to avoid triggering an unnecessary rebuilds of modules
--- that depend on this one
-F = io.open(non_static_fname, 'r')
-if F ~= nil then
-  if F:read('*a') == non_static then
-    os.exit(0)
+  if non_static_fname ~= 'SKIP' then
+    local non_static = {} --- @type string[]
+    local iwyu_non_static = add_iwyu_non_static(fname, non_static_fname)
+    if iwyu_non_static then
+      non_static[#non_static + 1] = ('// IWYU pragma: private, include "%s"'):format(
+        iwyu_non_static
+      )
+    end
+    vim.list_extend(non_static, {
+      '#define DEFINE_FUNC_ATTRIBUTES',
+      '#include "nvim/func_attr.h"',
+      '#undef DEFINE_FUNC_ATTRIBUTES',
+      '#ifndef DLLEXPORT',
+      '#  ifdef MSWIN',
+      '#    define DLLEXPORT __declspec(dllexport)',
+      '#  else',
+      '#    define DLLEXPORT',
+      '#  endif',
+      '#endif',
+    })
+    vim.list_extend(non_static, non_static_decls)
+    non_static[#non_static + 1] = '#include "nvim/func_attr.h"'
+    write_file(non_static_fname, non_static)
   end
-  F:close()
 end
 
-F = io.open(non_static_fname, 'w')
-F:write(non_static)
-F:close()
+return main()
diff --git a/src/nvim/generators/gen_eval.lua b/src/nvim/generators/gen_eval.lua
index 443c68e008..0b6ee6cb24 100644
--- a/src/nvim/generators/gen_eval.lua
+++ b/src/nvim/generators/gen_eval.lua
@@ -19,6 +19,7 @@ hashpipe:write([[
 #include "nvim/digraph.h"
 #include "nvim/eval.h"
 #include "nvim/eval/buffer.h"
+#include "nvim/eval/deprecated.h"
 #include "nvim/eval/fs.h"
 #include "nvim/eval/funcs.h"
 #include "nvim/eval/typval.h"
diff --git a/src/nvim/generators/gen_options.lua b/src/nvim/generators/gen_options.lua
index 02f3ac3257..e5dba90925 100644
--- a/src/nvim/generators/gen_options.lua
+++ b/src/nvim/generators/gen_options.lua
@@ -1,143 +1,30 @@
-local options_file = arg[1]
-local options_enum_file = arg[2]
-local options_map_file = arg[3]
-
-local opt_fd = assert(io.open(options_file, 'w'))
-local opt_enum_fd = assert(io.open(options_enum_file, 'w'))
-local opt_map_fd = assert(io.open(options_map_file, 'w'))
-
-local w = function(s)
-  if s:match('^    %.') then
-    opt_fd:write(s .. ',\n')
-  else
-    opt_fd:write(s .. '\n')
-  end
-end
-
---- @param s string
-local function enum_w(s)
-  opt_enum_fd:write(s .. '\n')
-end
-
---- @param s string
-local function map_w(s)
-  opt_map_fd:write(s .. '\n')
-end
-
 --- @module 'nvim.options'
 local options = require('options')
 local options_meta = options.options
-
 local cstr = options.cstr
 local valid_scopes = options.valid_scopes
 
---- Options for each scope.
---- @type table<string, vim.option_meta[]>
-local scope_options = {}
-for _, scope in ipairs(valid_scopes) do
-  scope_options[scope] = {}
+--- @param o vim.option_meta
+--- @return string
+local function get_values_var(o)
+  return ('opt_%s_values'):format(o.abbreviation or o.full_name)
 end
 
 --- @param s string
 --- @return string
-local lowercase_to_titlecase = function(s)
-  return s:sub(1, 1):upper() .. s:sub(2)
+local function lowercase_to_titlecase(s)
+  return table.concat(vim.tbl_map(function(word) --- @param word string
+    return word:sub(1, 1):upper() .. word:sub(2)
+  end, vim.split(s, '[-_]')))
 end
 
--- Generate options enum file
-enum_w('// IWYU pragma: private, include "nvim/option_defs.h"')
-enum_w('')
-
---- Map of option name to option index
---- @type table<string, string>
-local option_index = {}
-
--- Generate option index enum and populate the `option_index` and `scope_option` dicts.
-enum_w('typedef enum {')
-enum_w('  kOptInvalid = -1,')
-
-for i, o in ipairs(options_meta) do
-  local enum_val_name = 'kOpt' .. lowercase_to_titlecase(o.full_name)
-  enum_w(('  %s = %u,'):format(enum_val_name, i - 1))
-
-  option_index[o.full_name] = enum_val_name
-
-  if o.abbreviation then
-    option_index[o.abbreviation] = enum_val_name
-  end
-
-  if o.alias then
-    o.alias = type(o.alias) == 'string' and { o.alias } or o.alias
-
-    for _, v in ipairs(o.alias) do
-      option_index[v] = enum_val_name
-    end
-  end
-
-  for _, scope in ipairs(o.scope) do
-    table.insert(scope_options[scope], o)
-  end
-end
-
-enum_w('  // Option count')
-enum_w('#define kOptCount ' .. tostring(#options_meta))
-enum_w('} OptIndex;')
-
 --- @param scope string
 --- @param option_name string
 --- @return string
-local get_scope_option = function(scope, option_name)
+local function get_scope_option(scope, option_name)
   return ('k%sOpt%s'):format(lowercase_to_titlecase(scope), lowercase_to_titlecase(option_name))
 end
 
--- Generate option index enum for each scope
-for _, scope in ipairs(valid_scopes) do
-  enum_w('')
-
-  local scope_name = lowercase_to_titlecase(scope)
-  enum_w('typedef enum {')
-  enum_w(('  %s = -1,'):format(get_scope_option(scope, 'Invalid')))
-
-  for idx, option in ipairs(scope_options[scope]) do
-    enum_w(('  %s = %u,'):format(get_scope_option(scope, option.full_name), idx - 1))
-  end
-
-  enum_w(('  // %s option count'):format(scope_name))
-  enum_w(('#define %s %d'):format(get_scope_option(scope, 'Count'), #scope_options[scope]))
-  enum_w(('} %sOptIndex;'):format(scope_name))
-end
-
--- Generate reverse lookup from option scope index to option index for each scope.
-for _, scope in ipairs(valid_scopes) do
-  enum_w('')
-  enum_w(('EXTERN const OptIndex %s_opt_idx[] INIT( = {'):format(scope))
-  for _, option in ipairs(scope_options[scope]) do
-    local idx = option_index[option.full_name]
-    enum_w(('  [%s] = %s,'):format(get_scope_option(scope, option.full_name), idx))
-  end
-  enum_w('});')
-end
-
-opt_enum_fd:close()
-
--- Generate option index map.
-local hashy = require('generators.hashy')
-local neworder, hashfun = hashy.hashy_hash('find_option', vim.tbl_keys(option_index), function(idx)
-  return ('option_hash_elems[%s].name'):format(idx)
-end)
-
-map_w('static const struct { const char *name; OptIndex opt_idx; } option_hash_elems[] = {')
-
-for _, name in ipairs(neworder) do
-  assert(option_index[name] ~= nil)
-  map_w(('  { .name = "%s", .opt_idx = %s },'):format(name, option_index[name]))
-end
-
-map_w('};\n')
-map_w('static ' .. hashfun)
-
-opt_map_fd:close()
-
 local redraw_flags = {
   ui_option = 'kOptFlagUIOption',
   tabline = 'kOptFlagRedrTabl',
@@ -161,28 +48,29 @@ local list_flags = {
 --- @param o vim.option_meta
 --- @return string
 local function get_flags(o)
-  --- @type string
-  local flags = '0'
+  --- @type string[]
+  local flags = { '0' }
 
   --- @param f string
-  local add_flag = function(f)
-    flags = flags .. '|' .. f
+  local function add_flag(f)
+    table.insert(flags, f)
   end
 
   if o.list then
     add_flag(list_flags[o.list])
   end
-  if o.redraw then
-    for _, r_flag in ipairs(o.redraw) do
-      add_flag(redraw_flags[r_flag])
-    end
+
+  for _, r_flag in ipairs(o.redraw or {}) do
+    add_flag(redraw_flags[r_flag])
   end
+
   if o.expand then
     add_flag('kOptFlagExpand')
     if o.expand == 'nodefault' then
       add_flag('kOptFlagNoDefExp')
     end
   end
+
   for _, flag_desc in ipairs({
     { 'nodefault', 'NoDefault' },
     { 'no_mkrc', 'NoMkrc' },
@@ -197,13 +85,14 @@ local function get_flags(o)
     { 'modelineexpr', 'MLE' },
     { 'func' },
   }) do
-    local key_name = flag_desc[1]
-    local def_name = 'kOptFlag' .. (flag_desc[2] or lowercase_to_titlecase(key_name))
+    local key_name, flag_suffix = flag_desc[1], flag_desc[2]
     if o[key_name] then
+      local def_name = 'kOptFlag' .. (flag_suffix or lowercase_to_titlecase(key_name))
       add_flag(def_name)
     end
   end
-  return flags
+
+  return table.concat(flags, '|')
 end
 
 --- @param opt_type vim.option_type
@@ -220,21 +109,6 @@ end
 
 --- @param o vim.option_meta
 --- @return string
-local function get_type_flags(o)
-  local opt_types = (type(o.type) == 'table') and o.type or { o.type }
-  local type_flags = '0'
-  assert(type(opt_types) == 'table')
-
-  for _, opt_type in ipairs(opt_types) do
-    assert(type(opt_type) == 'string')
-    type_flags = ('%s | (1 << %s)'):format(type_flags, opt_type_enum(opt_type))
-  end
-
-  return type_flags
-end
-
---- @param o vim.option_meta
---- @return string
 local function get_scope_flags(o)
   local scope_flags = '0'
 
@@ -262,35 +136,15 @@ local function get_scope_idx(o)
   return ('{\n%s\n    }'):format(table.concat(strs, ',\n'))
 end
 
---- @param c string|string[]
---- @param base_string? string
---- @return string
-local function get_cond(c, base_string)
-  local cond_string = base_string or '#if '
-  if type(c) == 'table' then
-    cond_string = cond_string .. get_cond(c[1], '')
-    for i, subc in ipairs(c) do
-      if i > 1 then
-        cond_string = cond_string .. ' && ' .. get_cond(subc, '')
-      end
-    end
-  elseif c:sub(1, 1) == '!' then
-    cond_string = cond_string .. '!defined(' .. c:sub(2) .. ')'
-  else
-    cond_string = cond_string .. 'defined(' .. c .. ')'
-  end
-  return cond_string
-end
-
 --- @param s string
 --- @return string
-local static_cstr_as_string = function(s)
+local function static_cstr_as_string(s)
   return ('{ .data = %s, .size = sizeof(%s) - 1 }'):format(s, s)
 end
 
 --- @param v vim.option_value|function
 --- @return string
-local get_opt_val = function(v)
+local function get_opt_val(v)
   --- @type vim.option_type
   local v_type
 
@@ -308,6 +162,7 @@ local get_opt_val = function(v)
     elseif v_type == 'number' then
       v = ('%iL'):format(v)
     elseif v_type == 'string' then
+      --- @cast v string
       v = static_cstr_as_string(cstr(v))
     end
   end
@@ -318,7 +173,7 @@ end
 --- @param d vim.option_value|function
 --- @param n string
 --- @return string
-local get_defaults = function(d, n)
+local function get_defaults(d, n)
   if d == nil then
     error("option '" .. n .. "' should have a default value")
   end
@@ -327,84 +182,354 @@ end
 
 --- @param i integer
 --- @param o vim.option_meta
-local function dump_option(i, o)
-  w('  [' .. ('%u'):format(i - 1) .. ']={')
-  w('    .fullname=' .. cstr(o.full_name))
+--- @param write fun(...: string)
+local function dump_option(i, o, write)
+  write('  [', ('%u'):format(i - 1) .. ']={')
+  write('    .fullname=', cstr(o.full_name))
   if o.abbreviation then
-    w('    .shortname=' .. cstr(o.abbreviation))
+    write('    .shortname=', cstr(o.abbreviation))
   end
-  w('    .flags=' .. get_flags(o))
-  w('    .type_flags=' .. get_type_flags(o))
-  w('    .scope_flags=' .. get_scope_flags(o))
-  w('    .scope_idx=' .. get_scope_idx(o))
+  write('    .type=', opt_type_enum(o.type))
+  write('    .flags=', get_flags(o))
+  write('    .scope_flags=', get_scope_flags(o))
+  write('    .scope_idx=', get_scope_idx(o))
+  write('    .values=', (o.values and get_values_var(o) or 'NULL'))
+  write('    .values_len=', (o.values and #o.values or '0'))
+  write('    .flags_var=', (o.flags_varname and ('&%s'):format(o.flags_varname) or 'NULL'))
   if o.enable_if then
-    w(get_cond(o.enable_if))
+    write(('#if defined(%s)'):format(o.enable_if))
   end
 
   local is_window_local = #o.scope == 1 and o.scope[1] == 'win'
 
-  if not is_window_local then
-    if o.varname then
-      w('    .var=&' .. o.varname)
-    elseif o.immutable then
-      -- Immutable options can directly point to the default value.
-      w(('    .var=&options[%u].def_val.data'):format(i - 1))
-    else
-      -- Option must be immutable or have a variable.
-      assert(false)
-    end
+  if is_window_local then
+    write('    .var=NULL')
+  elseif o.varname then
+    write('    .var=&', o.varname)
+  elseif o.immutable then
+    -- Immutable options can directly point to the default value.
+    write(('    .var=&options[%u].def_val.data'):format(i - 1))
   else
-    w('    .var=NULL')
-  end
-  w('    .immutable=' .. (o.immutable and 'true' or 'false'))
-  if o.cb then
-    w('    .opt_did_set_cb=' .. o.cb)
-  end
-  if o.expand_cb then
-    w('    .opt_expand_cb=' .. o.expand_cb)
+    error('Option must be immutable or have a variable.')
   end
+
+  write('    .immutable=', (o.immutable and 'true' or 'false'))
+  write('    .opt_did_set_cb=', o.cb or 'NULL')
+  write('    .opt_expand_cb=', o.expand_cb or 'NULL')
+
   if o.enable_if then
-    w('#else')
+    write('#else')
     -- Hidden option directly points to default value.
-    w(('    .var=&options[%u].def_val.data'):format(i - 1))
+    write(('    .var=&options[%u].def_val.data'):format(i - 1))
     -- Option is always immutable on the false branch of `enable_if`.
-    w('    .immutable=true')
-    w('#endif')
+    write('    .immutable=true')
+    write('#endif')
   end
-  if o.defaults then
-    if o.defaults.condition then
-      w(get_cond(o.defaults.condition))
+
+  if not o.defaults then
+    write('    .def_val=NIL_OPTVAL')
+  elseif o.defaults.condition then
+    write(('#if defined(%s)'):format(o.defaults.condition))
+    write('    .def_val=', get_defaults(o.defaults.if_true, o.full_name))
+    if o.defaults.if_false then
+      write('#else')
+      write('    .def_val=', get_defaults(o.defaults.if_false, o.full_name))
     end
-    w('    .def_val=' .. get_defaults(o.defaults.if_true, o.full_name))
-    if o.defaults.condition then
-      if o.defaults.if_false then
-        w('#else')
-        w('    .def_val=' .. get_defaults(o.defaults.if_false, o.full_name))
-      end
-      w('#endif')
+    write('#endif')
+  else
+    write('    .def_val=', get_defaults(o.defaults.if_true, o.full_name))
+  end
+
+  write('  },')
+end
+
+--- @param prefix string
+--- @param values vim.option_valid_values
+local function preorder_traversal(prefix, values)
+  local out = {} --- @type string[]
+
+  local function add(s)
+    table.insert(out, s)
+  end
+
+  add('')
+  add(('EXTERN const char *(%s_values[%s]) INIT( = {'):format(prefix, #vim.tbl_keys(values) + 1))
+
+  --- @type [string,vim.option_valid_values][]
+  local children = {}
+
+  for _, value in ipairs(values) do
+    if type(value) == 'string' then
+      add(('  "%s",'):format(value))
+    else
+      assert(type(value) == 'table' and type(value[1]) == 'string' and type(value[2]) == 'table')
+      add(('  "%s",'):format(value[1]))
+      table.insert(children, value)
     end
+  end
+
+  add('  NULL')
+  add('});')
+
+  for _, value in pairs(children) do
+    -- Remove trailing colon from the added prefix to prevent syntax errors.
+    add(preorder_traversal(prefix .. '_' .. value[1]:gsub(':$', ''), value[2]))
+  end
+
+  return table.concat(out, '\n')
+end
+
+--- @param o vim.option_meta
+--- @return string
+local function gen_opt_enum(o)
+  local out = {} --- @type string[]
+
+  local function add(s)
+    table.insert(out, s)
+  end
+
+  add('')
+  add('typedef enum {')
+
+  local opt_name = lowercase_to_titlecase(o.abbreviation or o.full_name)
+  --- @type table<string,integer>
+  local enum_values
+
+  if type(o.flags) == 'table' then
+    enum_values = o.flags --[[ @as table<string,integer> ]]
   else
-    w('    .def_val=NIL_OPTVAL')
+    enum_values = {}
+    for i, flag_name in ipairs(o.values) do
+      assert(type(flag_name) == 'string')
+      enum_values[flag_name] = math.pow(2, i - 1)
+    end
+  end
+
+  -- Sort the keys by the flag value so that the enum can be generated in order.
+  --- @type string[]
+  local flag_names = vim.tbl_keys(enum_values)
+  table.sort(flag_names, function(a, b)
+    return enum_values[a] < enum_values[b]
+  end)
+
+  for _, flag_name in pairs(flag_names) do
+    add(
+      ('  kOpt%sFlag%s = 0x%02x,'):format(
+        opt_name,
+        lowercase_to_titlecase(flag_name:gsub(':$', '')),
+        enum_values[flag_name]
+      )
+    )
+  end
+
+  add(('} Opt%sFlags;'):format(opt_name))
+
+  return table.concat(out, '\n')
+end
+
+--- @param output_file string
+--- @return table<string,string> options_index Map of option name to option index
+local function gen_enums(output_file)
+  --- Options for each scope.
+  --- @type table<string, vim.option_meta[]>
+  local scope_options = {}
+  for _, scope in ipairs(valid_scopes) do
+    scope_options[scope] = {}
+  end
+
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param s string
+  local function write(s)
+    fd:write(s)
+    fd:write('\n')
+  end
+
+  -- Generate options enum file
+  write('// IWYU pragma: private, include "nvim/option_defs.h"')
+  write('')
+
+  --- Map of option name to option index
+  --- @type table<string, string>
+  local option_index = {}
+
+  -- Generate option index enum and populate the `option_index` and `scope_option` dicts.
+  write('typedef enum {')
+  write('  kOptInvalid = -1,')
+
+  for i, o in ipairs(options_meta) do
+    local enum_val_name = 'kOpt' .. lowercase_to_titlecase(o.full_name)
+    write(('  %s = %u,'):format(enum_val_name, i - 1))
+
+    option_index[o.full_name] = enum_val_name
+
+    if o.abbreviation then
+      option_index[o.abbreviation] = enum_val_name
+    end
+
+    local alias = o.alias or {} --[[@as string[] ]]
+    for _, v in ipairs(alias) do
+      option_index[v] = enum_val_name
+    end
+
+    for _, scope in ipairs(o.scope) do
+      table.insert(scope_options[scope], o)
+    end
+  end
+
+  write('  // Option count')
+  write('#define kOptCount ' .. tostring(#options_meta))
+  write('} OptIndex;')
+
+  -- Generate option index enum for each scope
+  for _, scope in ipairs(valid_scopes) do
+    write('')
+
+    local scope_name = lowercase_to_titlecase(scope)
+    write('typedef enum {')
+    write(('  %s = -1,'):format(get_scope_option(scope, 'Invalid')))
+
+    for idx, option in ipairs(scope_options[scope]) do
+      write(('  %s = %u,'):format(get_scope_option(scope, option.full_name), idx - 1))
+    end
+
+    write(('  // %s option count'):format(scope_name))
+    write(('#define %s %d'):format(get_scope_option(scope, 'Count'), #scope_options[scope]))
+    write(('} %sOptIndex;'):format(scope_name))
   end
-  w('  },')
+
+  -- Generate reverse lookup from option scope index to option index for each scope.
+  for _, scope in ipairs(valid_scopes) do
+    write('')
+    write(('EXTERN const OptIndex %s_opt_idx[] INIT( = {'):format(scope))
+    for _, option in ipairs(scope_options[scope]) do
+      local idx = option_index[option.full_name]
+      write(('  [%s] = %s,'):format(get_scope_option(scope, option.full_name), idx))
+    end
+    write('});')
+  end
+
+  fd:close()
+
+  return option_index
 end
 
--- Generate options[] array.
-w([[
-#include "nvim/ex_docmd.h"
-#include "nvim/ex_getln.h"
-#include "nvim/insexpand.h"
-#include "nvim/mapping.h"
-#include "nvim/ops.h"
-#include "nvim/option.h"
-#include "nvim/optionstr.h"
-#include "nvim/quickfix.h"
-#include "nvim/runtime.h"
-#include "nvim/tag.h"
-#include "nvim/window.h"
-
-static vimoption_T options[] = {]])
-for i, o in ipairs(options.options) do
-  dump_option(i, o)
+--- @param output_file string
+--- @param option_index table<string,string>
+local function gen_map(output_file, option_index)
+  -- Generate option index map.
+  local hashy = require('generators.hashy')
+
+  local neworder, hashfun = hashy.hashy_hash(
+    'find_option',
+    vim.tbl_keys(option_index),
+    function(idx)
+      return ('option_hash_elems[%s].name'):format(idx)
+    end
+  )
+
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param s string
+  local function write(s)
+    fd:write(s)
+    fd:write('\n')
+  end
+
+  write('static const struct { const char *name; OptIndex opt_idx; } option_hash_elems[] = {')
+
+  for _, name in ipairs(neworder) do
+    assert(option_index[name] ~= nil)
+    write(('  { .name = "%s", .opt_idx = %s },'):format(name, option_index[name]))
+  end
+
+  write('};')
+  write('')
+  write('static ' .. hashfun)
+
+  fd:close()
 end
-w('};')
+
+--- @param output_file string
+local function gen_vars(output_file)
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param s string
+  local function write(s)
+    fd:write(s)
+    fd:write('\n')
+  end
+
+  write('// IWYU pragma: private, include "nvim/option_vars.h"')
+
+  -- Generate enums for option flags.
+  for _, o in ipairs(options_meta) do
+    if o.flags and (type(o.flags) == 'table' or o.values) then
+      write(gen_opt_enum(o))
+    end
+  end
+
+  -- Generate valid values for each option.
+  for _, option in ipairs(options_meta) do
+    -- Since option values can be nested, we need to do preorder traversal to generate the values.
+    if option.values then
+      local values_var = ('opt_%s'):format(option.abbreviation or option.full_name)
+      write(preorder_traversal(values_var, option.values))
+    end
+  end
+
+  fd:close()
+end
+
+--- @param output_file string
+local function gen_options(output_file)
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param ... string
+  local function write(...)
+    local s = table.concat({ ... }, '')
+    fd:write(s)
+    if s:match('^    %.') then
+      fd:write(',')
+    end
+    fd:write('\n')
+  end
+
+  -- Generate options[] array.
+  write([[
+  #include "nvim/ex_docmd.h"
+  #include "nvim/ex_getln.h"
+  #include "nvim/insexpand.h"
+  #include "nvim/mapping.h"
+  #include "nvim/ops.h"
+  #include "nvim/option.h"
+  #include "nvim/optionstr.h"
+  #include "nvim/quickfix.h"
+  #include "nvim/runtime.h"
+  #include "nvim/tag.h"
+  #include "nvim/window.h"
+
+  static vimoption_T options[] = {]])
+
+  for i, o in ipairs(options_meta) do
+    dump_option(i, o, write)
+  end
+
+  write('};')
+
+  fd:close()
+end
+
+local function main()
+  local options_file = arg[1]
+  local options_enum_file = arg[2]
+  local options_map_file = arg[3]
+  local option_vars_file = arg[4]
+
+  local option_index = gen_enums(options_enum_file)
+  gen_map(options_map_file, option_index)
+  gen_vars(option_vars_file)
+  gen_options(options_file)
+end
+
+main()
diff --git a/src/nvim/generators/gen_vimvim.lua b/src/nvim/generators/gen_vimvim.lua
index 0675f04b73..d8053822bf 100644
--- a/src/nvim/generators/gen_vimvim.lua
+++ b/src/nvim/generators/gen_vimvim.lua
@@ -52,11 +52,13 @@ local function is_special_cased_cmd(cmd)
 end
 
 local vimcmd_start = 'syn keyword vimCommand contained '
+local vimcmd_end = ' nextgroup=vimBang'
 w(vimcmd_start)
+
 local prev_cmd = nil
 for _, cmd_desc in ipairs(ex_cmds.cmds) do
   if lld.line_length > 850 then
-    w('\n' .. vimcmd_start)
+    w(vimcmd_end .. '\n' .. vimcmd_start)
   end
   local cmd = cmd_desc.command
   if cmd:match('%w') and cmd ~= 'z' and not is_special_cased_cmd(cmd) then
@@ -79,9 +81,11 @@ for _, cmd_desc in ipairs(ex_cmds.cmds) do
   prev_cmd = cmd
 end
 
+w(vimcmd_end .. '\n')
+
 local vimopt_start = 'syn keyword vimOption contained '
 local vimopt_end = ' skipwhite nextgroup=vimSetEqual,vimSetMod'
-w('\n\n' .. vimopt_start)
+w('\n' .. vimopt_start)
 
 for _, opt_desc in ipairs(options.options) do
   if not opt_desc.immutable then
diff --git a/src/nvim/generators/hashy.lua b/src/nvim/generators/hashy.lua
index ea35064962..74b7655324 100644
--- a/src/nvim/generators/hashy.lua
+++ b/src/nvim/generators/hashy.lua
@@ -55,7 +55,7 @@ function M.build_pos_hash(strings)
 end
 
 function M.switcher(put, tab, maxlen, worst_buck_size)
-  local neworder = {}
+  local neworder = {} --- @type string[]
   put '  switch (len) {\n'
   local bucky = worst_buck_size > 1
   for len = 1, maxlen do