fix(getchar): do not simplify keycodes in terminal mode

The code represents a useful pattern in normal mode where remapping
`<tab>` will implicitly also remap `<c-i>` unless you remap that
explicitly. This relies on the _unmapped_ behavior being identical which
is not true in terminal mode, as vterm can distinguish these keys.

Vim seems to entangle this with kitty keyboard mode detection which
is irrelevant for us. Conditional fallbacks depending on
keyboard mode could be done completely inside `vterm/` without getchar.c
getting involved, I would think.

3 files changed, 83 insertions, 9 deletions

diff --git a/src/nvim/getchar.c b/src/nvim/getchar.c
index 60aa1055c3..6cf4556a9f 100644
--- a/src/nvim/getchar.c
+++ b/src/nvim/getchar.c
@@ -1518,9 +1518,11 @@ int merge_modifiers(int c_arg, int *modifiers)
 
   if (*modifiers & MOD_MASK_CTRL) {
     if ((c >= '`' && c <= 0x7f) || (c >= '@' && c <= '_')) {
-      c &= 0x1f;
-      if (c == NUL) {
-        c = K_ZERO;
+      if (!(State & MODE_TERMINAL) || !(c == 'I' || c == 'J' || c == 'M' || c == '[')) {
+        c &= 0x1f;
+        if (c == NUL) {
+          c = K_ZERO;
+        }
       }
     } else if (c == '6') {
       // CTRL-6 is equivalent to CTRL-^
diff --git a/src/nvim/terminal.c b/src/nvim/terminal.c
index d7ed709906..ad343bad67 100644
--- a/src/nvim/terminal.c
+++ b/src/nvim/terminal.c
@@ -1011,7 +1011,7 @@ static void terminal_send_key(Terminal *term, int c)
     c = Ctrl_AT;
   }
 
-  VTermKey key = convert_key(c, &mod);
+  VTermKey key = convert_key(&c, &mod);
 
   if (key) {
     vterm_keyboard_key(term->vt, key, mod);
@@ -1415,19 +1415,23 @@ static int term_selection_set(VTermSelectionMask mask, VTermStringFragment frag,
 // }}}
 // input handling {{{
 
-static void convert_modifiers(int key, VTermModifier *statep)
+static void convert_modifiers(int *key, VTermModifier *statep)
 {
   if (mod_mask & MOD_MASK_SHIFT) {
     *statep |= VTERM_MOD_SHIFT;
   }
   if (mod_mask & MOD_MASK_CTRL) {
     *statep |= VTERM_MOD_CTRL;
+    if (!(mod_mask & MOD_MASK_SHIFT) && *key >= 'A' && *key <= 'Z') {
+      // vterm interprets CTRL+A as SHIFT+CTRL, change to CTRL+a
+      *key += ('a' - 'A');
+    }
   }
   if (mod_mask & MOD_MASK_ALT) {
     *statep |= VTERM_MOD_ALT;
   }
 
-  switch (key) {
+  switch (*key) {
   case K_S_TAB:
   case K_S_UP:
   case K_S_DOWN:
@@ -1459,11 +1463,11 @@ static void convert_modifiers(int key, VTermModifier *statep)
   }
 }
 
-static VTermKey convert_key(int key, VTermModifier *statep)
+static VTermKey convert_key(int *key, VTermModifier *statep)
 {
   convert_modifiers(key, statep);
 
-  switch (key) {
+  switch (*key) {
   case K_BS:
     return VTERM_KEY_BACKSPACE;
   case K_S_TAB:
@@ -1791,7 +1795,7 @@ static bool send_mouse_event(Terminal *term, int c)
     }
 
     VTermModifier mod = VTERM_MOD_NONE;
-    convert_modifiers(c, &mod);
+    convert_modifiers(&c, &mod);
     mouse_action(term, button, row, col - offset, pressed, mod);
     return false;
   }
diff --git a/test/functional/terminal/buffer_spec.lua b/test/functional/terminal/buffer_spec.lua
index cc807ba555..50e23d9e23 100644
--- a/test/functional/terminal/buffer_spec.lua
+++ b/test/functional/terminal/buffer_spec.lua
@@ -625,6 +625,74 @@ describe('terminal input', function()
       ]]):format(key))
     end
   end)
+
+  -- TODO(bfredl): getcharstr() erases the distinction between <C-I> and <Tab>.
+  -- If it was enhanced or replaced this could get folded into the test above.
+  it('can send TAB/C-I and ESC/C-[ separately', function()
+    clear()
+    local screen = tt.setup_child_nvim({
+      '-u',
+      'NONE',
+      '-i',
+      'NONE',
+      '--cmd',
+      'colorscheme vim',
+      '--cmd',
+      'set notermguicolors',
+      '--cmd',
+      'noremap <Tab> <cmd>echo "Tab!"<cr>',
+      '--cmd',
+      'noremap <C-i> <cmd>echo "Ctrl-I!"<cr>',
+      '--cmd',
+      'noremap <Esc> <cmd>echo "Esc!"<cr>',
+      '--cmd',
+      'noremap <C-[> <cmd>echo "Ctrl-[!"<cr>',
+    })
+
+    screen:expect([[
+      ^                                                  |
+      {4:~                                                 }|*3
+      {5:[No Name]                       0,0-1          All}|
+                                                        |
+      {3:-- TERMINAL --}                                    |
+    ]])
+
+    feed('<tab>')
+    screen:expect([[
+      ^                                                  |
+      {4:~                                                 }|*3
+      {5:[No Name]                       0,0-1          All}|
+      Tab!                                              |
+      {3:-- TERMINAL --}                                    |
+    ]])
+
+    feed('<c-i>')
+    screen:expect([[
+      ^                                                  |
+      {4:~                                                 }|*3
+      {5:[No Name]                       0,0-1          All}|
+      Ctrl-I!                                           |
+      {3:-- TERMINAL --}                                    |
+    ]])
+
+    feed('<Esc>')
+    screen:expect([[
+      ^                                                  |
+      {4:~                                                 }|*3
+      {5:[No Name]                       0,0-1          All}|
+      Esc!                                              |
+      {3:-- TERMINAL --}                                    |
+    ]])
+
+    feed('<c-[>')
+    screen:expect([[
+      ^                                                  |
+      {4:~                                                 }|*3
+      {5:[No Name]                       0,0-1          All}|
+      Ctrl-[!                                           |
+      {3:-- TERMINAL --}                                    |
+    ]])
+  end)
 end)
 
 if is_os('win') then