feat(terminal)!: cursor shape and blink (#31562)

When a terminal application running inside the terminal emulator sets
the cursor shape or blink status of the cursor, update the cursor in the
parent terminal to match.

This removes the "virtual cursor" that has been in use by the terminal
emulator since the beginning. The original rationale for using the
virtual cursor was to avoid having to support additional UI methods to
change the cursor color for other (non-TUI) UIs, instead relying on the
TermCursor and TermCursorNC highlight groups.

The TermCursor highlight group is now used in the default 'guicursor'
value, which has a new entry for Terminal mode. However, the
TermCursorNC highlight group is no longer supported: since terminal
windows now use the real cursor, when the window is not focused there is
no cursor displayed in the window at all, so there is nothing to
highlight. Users can still use the StatusLineTermNC highlight group to
differentiate non-focused terminal windows.

BREAKING CHANGE: The TermCursorNC highlight group is no longer supported.

5 files changed, 89 insertions, 73 deletions

diff --git a/test/functional/ui/cursor_spec.lua b/test/functional/ui/cursor_spec.lua
index edf826a1d9..825a90fbc8 100644
--- a/test/functional/ui/cursor_spec.lua
+++ b/test/functional/ui/cursor_spec.lua
@@ -190,6 +190,19 @@ describe('ui/cursor', function()
         attr_lm = {},
         short_name = 'sm',
       },
+      [18] = {
+        blinkoff = 500,
+        blinkon = 500,
+        blinkwait = 0,
+        cell_percentage = 0,
+        cursor_shape = 'block',
+        name = 'terminal',
+        hl_id = 3,
+        id_lm = 3,
+        attr = { reverse = true },
+        attr_lm = { reverse = true },
+        short_name = 't',
+      },
     }
 
     screen:expect(function()
@@ -245,17 +258,20 @@ describe('ui/cursor', function()
         end
       end
       if m.hl_id then
-        m.hl_id = 66
+        m.hl_id = 65
         m.attr = { background = Screen.colors.DarkGray }
       end
       if m.id_lm then
-        m.id_lm = 73
+        m.id_lm = 72
+        m.attr_lm = {}
       end
     end
 
     -- Assert the new expectation.
     screen:expect(function()
-      eq(expected_mode_info, screen._mode_info)
+      for i, v in ipairs(expected_mode_info) do
+        eq(v, screen._mode_info[i])
+      end
       eq(true, screen._cursor_style_enabled)
       eq('normal', screen.mode)
     end)
diff --git a/test/functional/ui/hlstate_spec.lua b/test/functional/ui/hlstate_spec.lua
index f8f5ee9488..745ad70efe 100644
--- a/test/functional/ui/hlstate_spec.lua
+++ b/test/functional/ui/hlstate_spec.lua
@@ -227,7 +227,7 @@ describe('ext_hlstate detailed highlights', function()
     command(("enew | call termopen(['%s'])"):format(testprg('tty-test')))
     screen:expect([[
       ^tty ready                               |
-      {1: }                                       |
+                                              |
                                               |*5
       {7:                                        }|
     ]])
@@ -242,7 +242,7 @@ describe('ext_hlstate detailed highlights', function()
       screen:expect([[
         ^tty ready                               |
         x {5:y z}                                   |
-        {1: }                                       |
+                                                |
                                                 |*4
         {7:                                        }|
       ]])
@@ -250,7 +250,7 @@ describe('ext_hlstate detailed highlights', function()
       screen:expect([[
         ^tty ready                               |
         x {2:y }{3:z}                                   |
-        {1: }                                       |
+                                                |
                                                 |*4
         {7:                                        }|
       ]])
@@ -268,7 +268,7 @@ describe('ext_hlstate detailed highlights', function()
     else
       screen:expect([[
         ^tty ready                               |
-        x {4:y}{2: }{3:z}                                   |
+        x {2:y }{3:z}                                   |
                                                 |*5
         {7:                                        }|
       ]])
diff --git a/test/functional/ui/messages_spec.lua b/test/functional/ui/messages_spec.lua
index 2c1297b768..287db81a12 100644
--- a/test/functional/ui/messages_spec.lua
+++ b/test/functional/ui/messages_spec.lua
@@ -49,7 +49,7 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { '\ntest\n[O]k: ', 6, 11 } },
+          content = { { '\ntest\n[O]k: ', 6, 10 } },
           kind = 'confirm',
         },
       },
@@ -77,7 +77,7 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { '\ntest\n[O]k: ', 6, 11 } },
+          content = { { '\ntest\n[O]k: ', 6, 10 } },
           kind = 'confirm',
         },
       },
@@ -91,7 +91,7 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { '\ntest\n[O]k: ', 6, 11 } },
+          content = { { '\ntest\n[O]k: ', 6, 10 } },
           kind = 'confirm',
         },
         {
@@ -99,7 +99,7 @@ describe('ui/ext_messages', function()
           kind = 'echo',
         },
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
@@ -116,7 +116,7 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { 'replace with X (y/n/a/q/l/^E/^Y)?', 6, 19 } },
+          content = { { 'replace with X (y/n/a/q/l/^E/^Y)?', 6, 18 } },
           kind = 'confirm_sub',
         },
       },
@@ -135,7 +135,7 @@ describe('ui/ext_messages', function()
       ]],
       messages = {
         {
-          content = { { 'W10: Warning: Changing a readonly file', 19, 27 } },
+          content = { { 'W10: Warning: Changing a readonly file', 19, 26 } },
           kind = 'wmsg',
         },
       },
@@ -151,7 +151,7 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { 'search hit BOTTOM, continuing at TOP', 19, 27 } },
+          content = { { 'search hit BOTTOM, continuing at TOP', 19, 26 } },
           kind = 'wmsg',
         },
       },
@@ -167,15 +167,15 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { 'Error detected while processing :', 9, 7 } },
+          content = { { 'Error detected while processing :', 9, 6 } },
           kind = 'emsg',
         },
         {
-          content = { { 'E605: Exception not caught: foo', 9, 7 } },
+          content = { { 'E605: Exception not caught: foo', 9, 6 } },
           kind = 'emsg',
         },
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
@@ -225,15 +225,15 @@ describe('ui/ext_messages', function()
         {
           content = {
             { '\nErrorMsg      ' },
-            { 'xxx', 9, 7 },
+            { 'xxx', 9, 6 },
             { ' ' },
-            { 'ctermfg=', 18, 6 },
+            { 'ctermfg=', 18, 5 },
             { '15 ' },
-            { 'ctermbg=', 18, 6 },
+            { 'ctermbg=', 18, 5 },
             { '1 ' },
-            { 'guifg=', 18, 6 },
+            { 'guifg=', 18, 5 },
             { 'White ' },
-            { 'guibg=', 18, 6 },
+            { 'guibg=', 18, 5 },
             { 'Red' },
           },
           kind = 'list_cmd',
@@ -280,7 +280,7 @@ describe('ui/ext_messages', function()
       {1:~                        }|*4
     ]],
       messages = { {
-        content = { { 'raa', 9, 7 } },
+        content = { { 'raa', 9, 6 } },
         kind = 'echoerr',
       } },
     }
@@ -307,15 +307,15 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { 'bork', 9, 7 } },
+          content = { { 'bork', 9, 6 } },
           kind = 'echoerr',
         },
         {
-          content = { { 'fail', 9, 7 } },
+          content = { { 'fail', 9, 6 } },
           kind = 'echoerr',
         },
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
@@ -329,19 +329,19 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { 'bork', 9, 7 } },
+          content = { { 'bork', 9, 6 } },
           kind = 'echoerr',
         },
         {
-          content = { { 'fail', 9, 7 } },
+          content = { { 'fail', 9, 6 } },
           kind = 'echoerr',
         },
         {
-          content = { { 'extrafail', 9, 7 } },
+          content = { { 'extrafail', 9, 6 } },
           kind = 'echoerr',
         },
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
@@ -363,7 +363,7 @@ describe('ui/ext_messages', function()
       {1:~                        }|*4
     ]],
       messages = { {
-        content = { { 'problem', 9, 7 } },
+        content = { { 'problem', 9, 6 } },
         kind = 'echoerr',
       } },
       cmdline = {
@@ -391,15 +391,15 @@ describe('ui/ext_messages', function()
       {1:~                        }|*4
     ]],
       msg_history = {
-        { kind = 'echoerr', content = { { 'raa', 9, 7 } } },
-        { kind = 'echoerr', content = { { 'bork', 9, 7 } } },
-        { kind = 'echoerr', content = { { 'fail', 9, 7 } } },
-        { kind = 'echoerr', content = { { 'extrafail', 9, 7 } } },
-        { kind = 'echoerr', content = { { 'problem', 9, 7 } } },
+        { kind = 'echoerr', content = { { 'raa', 9, 6 } } },
+        { kind = 'echoerr', content = { { 'bork', 9, 6 } } },
+        { kind = 'echoerr', content = { { 'fail', 9, 6 } } },
+        { kind = 'echoerr', content = { { 'extrafail', 9, 6 } } },
+        { kind = 'echoerr', content = { { 'problem', 9, 6 } } },
       },
       messages = {
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
@@ -424,7 +424,7 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { 'bork\nfail', 9, 7 } },
+          content = { { 'bork\nfail', 9, 6 } },
           kind = 'echoerr',
         },
       },
@@ -438,13 +438,13 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
       msg_history = {
         {
-          content = { { 'bork\nfail', 9, 7 } },
+          content = { { 'bork\nfail', 9, 6 } },
           kind = 'echoerr',
         },
       },
@@ -492,7 +492,7 @@ describe('ui/ext_messages', function()
         { content = { { 'x                     #1' } }, kind = 'list_cmd' },
         { content = { { 'y                     #2' } }, kind = 'list_cmd' },
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
@@ -507,7 +507,7 @@ describe('ui/ext_messages', function()
       ^                         |
       {1:~                        }|*4
     ]],
-      showmode = { { '-- INSERT --', 5, 12 } },
+      showmode = { { '-- INSERT --', 5, 11 } },
     }
 
     feed('alphpabet<cr>alphanum<cr>')
@@ -518,7 +518,7 @@ describe('ui/ext_messages', function()
       ^                         |
       {1:~                        }|*2
     ]],
-      showmode = { { '-- INSERT --', 5, 12 } },
+      showmode = { { '-- INSERT --', 5, 11 } },
     }
 
     feed('<c-x>')
@@ -529,7 +529,7 @@ describe('ui/ext_messages', function()
       ^                         |
       {1:~                        }|*2
     ]],
-      showmode = { { '-- ^X mode (^]^D^E^F^I^K^L^N^O^Ps^U^V^Y)', 5, 12 } },
+      showmode = { { '-- ^X mode (^]^D^E^F^I^K^L^N^O^Ps^U^V^Y)', 5, 11 } },
     }
 
     feed('<c-p>')
@@ -545,7 +545,7 @@ describe('ui/ext_messages', function()
         items = { { 'alphpabet', '', '', '' }, { 'alphanum', '', '', '' } },
         pos = 1,
       },
-      showmode = { { '-- Keyword Local completion (^N^P) ', 5, 12 }, { 'match 1 of 2', 6, 19 } },
+      showmode = { { '-- Keyword Local completion (^N^P) ', 5, 11 }, { 'match 1 of 2', 6, 18 } },
     }
 
     -- echomsg and showmode don't overwrite each other, this is the same
@@ -567,7 +567,7 @@ describe('ui/ext_messages', function()
         content = { { 'stuff' } },
         kind = 'echomsg',
       } },
-      showmode = { { '-- Keyword Local completion (^N^P) ', 5, 12 }, { 'match 1 of 2', 6, 19 } },
+      showmode = { { '-- Keyword Local completion (^N^P) ', 5, 11 }, { 'match 1 of 2', 6, 18 } },
     }
 
     feed('<c-p>')
@@ -587,7 +587,7 @@ describe('ui/ext_messages', function()
         content = { { 'stuff' } },
         kind = 'echomsg',
       } },
-      showmode = { { '-- Keyword Local completion (^N^P) ', 5, 12 }, { 'match 2 of 2', 6, 19 } },
+      showmode = { { '-- Keyword Local completion (^N^P) ', 5, 11 }, { 'match 2 of 2', 6, 18 } },
     }
 
     feed('<esc>:messages<cr>')
@@ -604,7 +604,7 @@ describe('ui/ext_messages', function()
       } },
       messages = {
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
@@ -618,7 +618,7 @@ describe('ui/ext_messages', function()
       ^                         |
       {1:~                        }|*4
     ]],
-      showmode = { { 'recording @q', 5, 12 } },
+      showmode = { { 'recording @q', 5, 11 } },
     }
 
     feed('i')
@@ -627,7 +627,7 @@ describe('ui/ext_messages', function()
       ^                         |
       {1:~                        }|*4
     ]],
-      showmode = { { '-- INSERT --recording @q', 5, 12 } },
+      showmode = { { '-- INSERT --recording @q', 5, 11 } },
     }
 
     feed('<esc>')
@@ -636,7 +636,7 @@ describe('ui/ext_messages', function()
       ^                         |
       {1:~                        }|*4
     ]],
-      showmode = { { 'recording @q', 5, 12 } },
+      showmode = { { 'recording @q', 5, 11 } },
     }
 
     feed('q')
@@ -655,7 +655,7 @@ describe('ui/ext_messages', function()
       ^                         |
       {1:~                        }|*4
     ]],
-      showmode = { { 'recording @q', 5, 12 } },
+      showmode = { { 'recording @q', 5, 11 } },
       mode = 'normal',
     }
 
@@ -665,7 +665,7 @@ describe('ui/ext_messages', function()
       ^                         |
       {1:~                        }|*4
     ]],
-      showmode = { { 'recording @q', 5, 12 } },
+      showmode = { { 'recording @q', 5, 11 } },
       mode = 'insert',
     }
 
@@ -675,7 +675,7 @@ describe('ui/ext_messages', function()
       ^                         |
       {1:~                        }|*4
     ]],
-      showmode = { { 'recording @q', 5, 12 } },
+      showmode = { { 'recording @q', 5, 11 } },
       mode = 'normal',
     }
 
@@ -697,7 +697,7 @@ describe('ui/ext_messages', function()
         ^                         |
         {1:~                        }|*4
       ]],
-      ruler = { { '0,0-1   All', 9, 62 } },
+      ruler = { { '0,0-1   All', 9, 61 } },
     })
     command('hi clear MsgArea')
     feed('i')
@@ -706,7 +706,7 @@ describe('ui/ext_messages', function()
       ^                         |
       {1:~                        }|*4
     ]],
-      showmode = { { '-- INSERT --', 5, 12 } },
+      showmode = { { '-- INSERT --', 5, 11 } },
       ruler = { { '0,1     All' } },
     }
     feed('abcde<cr>12345<esc>')
@@ -744,7 +744,7 @@ describe('ui/ext_messages', function()
         {17:123}45                    |
         {1:~                        }|*3
       ]],
-      showmode = { { '-- VISUAL BLOCK --', 5, 12 } },
+      showmode = { { '-- VISUAL BLOCK --', 5, 11 } },
       showcmd = { { '2x3' } },
       ruler = { { '1,3     All' } },
     })
@@ -825,7 +825,7 @@ describe('ui/ext_messages', function()
       {1:~                        }|*4
     ]],
       messages = { {
-        content = { { 'bork', 9, 7 } },
+        content = { { 'bork', 9, 6 } },
         kind = 'echoerr',
       } },
     }
@@ -850,7 +850,7 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { 'E117: Unknown function: nosuchfunction', 9, 7 } },
+          content = { { 'E117: Unknown function: nosuchfunction', 9, 6 } },
           kind = 'emsg',
         },
       },
@@ -865,12 +865,12 @@ describe('ui/ext_messages', function()
       msg_history = {
         { kind = 'echomsg', content = { { 'howdy' } } },
         { kind = '', content = { { 'Type  :qa  and press <Enter> to exit Nvim' } } },
-        { kind = 'echoerr', content = { { 'bork', 9, 7 } } },
-        { kind = 'emsg', content = { { 'E117: Unknown function: nosuchfunction', 9, 7 } } },
+        { kind = 'echoerr', content = { { 'bork', 9, 6 } } },
+        { kind = 'emsg', content = { { 'E117: Unknown function: nosuchfunction', 9, 6 } } },
       },
       messages = {
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
@@ -943,7 +943,7 @@ stack traceback:
 	[C]: in function 'error'
 	[string ":lua"]:1: in main chunk]],
               9,
-              7,
+              6,
             },
           },
           kind = 'lua_error',
@@ -963,7 +963,7 @@ stack traceback:
       messages = {
         {
           content = {
-            { "Error invoking 'test_method' on channel 1:\ncomplete\nerror\n\nmessage", 9, 7 },
+            { "Error invoking 'test_method' on channel 1:\ncomplete\nerror\n\nmessage", 9, 6 },
           },
           kind = 'rpc_error',
         },
@@ -1092,7 +1092,7 @@ stack traceback:
     ]],
       messages = {
         {
-          content = { { 'wow, ', 10, 9 }, { 'such\n\nvery ', 9, 7 }, { 'color', 8, 13 } },
+          content = { { 'wow, ', 10, 8 }, { 'such\n\nvery ', 9, 6 }, { 'color', 8, 12 } },
           kind = 'echomsg',
         },
       },
@@ -1117,13 +1117,13 @@ stack traceback:
     ]],
       messages = {
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
       msg_history = {
         {
-          content = { { 'wow, ', 10, 9 }, { 'such\n\nvery ', 9, 7 }, { 'color', 8, 13 } },
+          content = { { 'wow, ', 10, 8 }, { 'such\n\nvery ', 9, 6 }, { 'color', 8, 12 } },
           kind = 'echomsg',
         },
       },
@@ -1783,7 +1783,7 @@ describe('ui/ext_messages', function()
       {1:~                }type  :help iccf{18:<Enter>}       for information {1:                 }|
       {1:~                                                                               }|*5
     ]]
-    local showmode = { { '-- INSERT --', 5, 12 } }
+    local showmode = { { '-- INSERT --', 5, 11 } }
     screen:expect(introscreen)
 
     -- <c-l> (same as :mode) does _not_ clear intro message
@@ -1858,7 +1858,7 @@ describe('ui/ext_messages', function()
     ]],
       messages = {
         {
-          content = { { 'Press ENTER or type command to continue', 6, 19 } },
+          content = { { 'Press ENTER or type command to continue', 6, 18 } },
           kind = 'return_prompt',
         },
       },
diff --git a/test/functional/ui/output_spec.lua b/test/functional/ui/output_spec.lua
index b5a09d814c..37e0e1344b 100644
--- a/test/functional/ui/output_spec.lua
+++ b/test/functional/ui/output_spec.lua
@@ -34,7 +34,7 @@ describe('shell command :!', function()
       n.nvim_set .. ' notermguicolors',
     })
     screen:expect([[
-      {1: }                                                 |
+      ^                                                  |
       {4:~                                                 }|*4
                                                         |
       {3:-- TERMINAL --}                                    |
@@ -78,7 +78,7 @@ describe('shell command :!', function()
       29999: foo                                        |
       30000: foo                                        |
                                                         |
-      {10:Press ENTER or type command to continue}{1: }          |
+      {10:Press ENTER or type command to continue}^           |
       {3:-- TERMINAL --}                                    |
     ]],
       {
diff --git a/test/functional/ui/screen.lua b/test/functional/ui/screen.lua
index f5cb914299..42734d07ca 100644
--- a/test/functional/ui/screen.lua
+++ b/test/functional/ui/screen.lua
@@ -967,11 +967,11 @@ function Screen:_handle_mode_info_set(cursor_style_enabled, mode_info)
   self._cursor_style_enabled = cursor_style_enabled
   for _, item in pairs(mode_info) do
     -- attr IDs are not stable, but their value should be
-    if item.attr_id ~= nil then
+    if item.attr_id ~= nil and self._attr_table[item.attr_id] ~= nil then
       item.attr = self._attr_table[item.attr_id][1]
       item.attr_id = nil
     end
-    if item.attr_id_lm ~= nil then
+    if item.attr_id_lm ~= nil and self._attr_table[item.attr_id_lm] ~= nil then
       item.attr_lm = self._attr_table[item.attr_id_lm][1]
       item.attr_id_lm = nil
     end