feat(keymap): return nil from an expr keymap

For Lua callback expr keymaps, returning `nil` or `false` is equivalent
to an empty string

2 files changed, 15 insertions, 3 deletions

diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt
index bd821c4f9e..e5bc60a8cd 100644
--- a/runtime/doc/lua.txt
+++ b/runtime/doc/lua.txt
@@ -2007,7 +2007,9 @@ set({mode}, {lhs}, {rhs}, {opts})                           *vim.keymap.set()*
                             create mapping on multiple modes.
                     {lhs}   string Left-hand side |{lhs}| of the mapping.
                     {rhs}   string|function Right-hand side |{rhs}| of the
-                            mapping. Can also be a Lua function.
+                            mapping. Can also be a Lua function. If a Lua
+                            function and `opts.expr == true`, returning `nil`
+                            or `false` is equivalent to an empty string.
                     {opts}  table A table of |:map-arguments| such as
                             "silent". In addition to the options listed in
                             |nvim_set_keymap()|, this table also accepts the
diff --git a/runtime/lua/vim/keymap.lua b/runtime/lua/vim/keymap.lua
index 1be40b0081..c193a13a7b 100644
--- a/runtime/lua/vim/keymap.lua
+++ b/runtime/lua/vim/keymap.lua
@@ -35,6 +35,8 @@ local keymap = {}
 ---                            Can also be list of modes to create mapping on multiple modes.
 ---@param lhs string          Left-hand side |{lhs}| of the mapping.
 ---@param rhs string|function  Right-hand side |{rhs}| of the mapping. Can also be a Lua function.
+---                            If a Lua function and `opts.expr == true`, returning `nil` or `false`
+---                            is equivalent to an empty string.
 --
 ---@param opts table A table of |:map-arguments| such as "silent". In addition to the options
 ---                  listed in |nvim_set_keymap()|, this table also accepts the following keys:
@@ -56,10 +58,18 @@ function keymap.set(mode, lhs, rhs, opts)
   local is_rhs_luaref = type(rhs) == "function"
   mode = type(mode) == 'string' and {mode} or mode
 
-  if is_rhs_luaref and opts.expr and opts.replace_keycodes ~= false then
+  if is_rhs_luaref and opts.expr then
     local user_rhs = rhs
     rhs = function ()
-      return vim.api.nvim_replace_termcodes(user_rhs(), true, true, true)
+      local res = user_rhs()
+      if not res then
+        -- TODO(lewis6991): Handle this in C?
+        return ''
+      elseif opts.replace_keycodes ~= false then
+        return vim.api.nvim_replace_termcodes(res, true, true, true)
+      else
+        return res
+      end
     end
   end
   -- clear replace_keycodes from opts table