ui: make 'mouse' handling in external UI more consistent

before the behaviour of 'mouse' was inconsistent in external UI,
as some remapping logic would check has_mouse() and others don't
(no difference in TUI or vim classic). With this change, the behaviour
is consistently up to the UI decide (see ui.txt edit)

Behaviour of tui.c is unaffected by this change.

10 files changed, 106 insertions, 93 deletions

diff --git a/runtime/doc/ui.txt b/runtime/doc/ui.txt
index e5c6b9b1b7..0a8584927e 100644
--- a/runtime/doc/ui.txt
+++ b/runtime/doc/ui.txt
@@ -213,11 +213,9 @@ the editor.
 
 ["mouse_on"]
 ["mouse_off"]
-	|'mouse'| was enabled/disabled in the current editor mode. Useful for
-	a terminal UI, or other situations where Nvim mouse would conflict
-	with other usages of the mouse. UIs may ignore this and always send
-	mouse input, because 'mouse' decides the behavior of |nvim_input()|
-	implicitly.
+	'mouse' was enabled/disabled in the current editor mode. Useful for
+	a terminal UI, or embedding into an application where Nvim mouse would
+	conflict with other usages of the mouse. Other UI:s may ignore this event.
 
 ["busy_start"]
 ["busy_stop"]
diff --git a/src/nvim/edit.c b/src/nvim/edit.c
index 45bd84261d..2d9dba839d 100644
--- a/src/nvim/edit.c
+++ b/src/nvim/edit.c
@@ -8328,9 +8328,6 @@ static void ins_mouse(int c)
   pos_T tpos;
   win_T       *old_curwin = curwin;
 
-  if (!mouse_has(MOUSE_INSERT))
-    return;
-
   undisplay_dollar();
   tpos = curwin->w_cursor;
   if (do_mouse(NULL, c, BACKWARD, 1, 0)) {
diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c
index 129cb323ea..53dd2776c1 100644
--- a/src/nvim/ex_getln.c
+++ b/src/nvim/ex_getln.c
@@ -1880,9 +1880,6 @@ static int command_line_handle_key(CommandLineState *s)
     return command_line_not_changed(s);                 // Ignore mouse
 
   case K_MIDDLEMOUSE:
-    if (!mouse_has(MOUSE_COMMAND)) {
-      return command_line_not_changed(s);                   // Ignore mouse
-    }
     cmdline_paste(eval_has_provider("clipboard") ? '*' : 0, true, true);
     redrawcmd();
     return command_line_changed(s);
@@ -1906,10 +1903,6 @@ static int command_line_handle_key(CommandLineState *s)
       s->ignore_drag_release = false;
     }
 
-    if (!mouse_has(MOUSE_COMMAND)) {
-      return command_line_not_changed(s);                   // Ignore mouse
-    }
-
     ccline.cmdspos = cmd_startcol();
     for (ccline.cmdpos = 0; ccline.cmdpos < ccline.cmdlen;
          ccline.cmdpos++) {
diff --git a/src/nvim/message.c b/src/nvim/message.c
index 02a7732f5c..f94529c687 100644
--- a/src/nvim/message.c
+++ b/src/nvim/message.c
@@ -1157,15 +1157,7 @@ void wait_return(int redraw)
              || c == K_MIDDLEDRAG || c == K_MIDDLERELEASE
              || c == K_RIGHTDRAG  || c == K_RIGHTRELEASE
              || c == K_MOUSELEFT  || c == K_MOUSERIGHT
-             || c == K_MOUSEDOWN  || c == K_MOUSEUP
-             || (!mouse_has(MOUSE_RETURN)
-                 && mouse_row < msg_row
-                 && (c == K_LEFTMOUSE
-                     || c == K_MIDDLEMOUSE
-                     || c == K_RIGHTMOUSE
-                     || c == K_X1MOUSE
-                     || c == K_X2MOUSE))
-             );
+             || c == K_MOUSEDOWN  || c == K_MOUSEUP);
     os_breakcheck();
     /*
      * Avoid that the mouse-up event causes visual mode to start.
diff --git a/src/nvim/mouse.c b/src/nvim/mouse.c
index f05dade73f..ff471ea978 100644
--- a/src/nvim/mouse.c
+++ b/src/nvim/mouse.c
@@ -526,53 +526,9 @@ static win_T *mouse_find_grid_win(int *gridp, int *rowp, int *colp)
 void setmouse(void)
 {
   ui_cursor_shape();
-
-  // Be quick when mouse is off.
-  if (*p_mouse == NUL) {
-    return;
-  }
-
-  int checkfor = MOUSE_NORMAL;  // assume normal mode
-  if (VIsual_active) {
-    checkfor = MOUSE_VISUAL;
-  } else if (State == HITRETURN || State == ASKMORE || State == SETWSIZE) {
-    checkfor = MOUSE_RETURN;
-  } else if (State & INSERT) {
-    checkfor = MOUSE_INSERT;
-  } else if (State & CMDLINE) {
-    checkfor = MOUSE_COMMAND;
-  } else if (State == CONFIRM || State == EXTERNCMD) {
-    checkfor = ' ';  // don't use mouse for ":confirm" or ":!cmd"
-  }
-
-  if (mouse_has(checkfor)) {
-    ui_call_mouse_on();
-  } else {
-    ui_call_mouse_off();
-  }
+  ui_check_mouse();
 }
 
-/*
- * Return true if
- * - "c" is in 'mouse', or
- * - 'a' is in 'mouse' and "c" is in MOUSE_A, or
- * - the current buffer is a help file and 'h' is in 'mouse' and we are in a
- *   normal editing mode (not at hit-return message).
- */
-int mouse_has(int c)
-{
-  for (char_u *p = p_mouse; *p; ++p)
-    switch (*p) {
-    case 'a': if (vim_strchr((char_u *)MOUSE_A, c) != NULL)
-        return true;
-      break;
-    case MOUSE_HELP: if (c != MOUSE_RETURN && curbuf->b_help)
-        return true;
-      break;
-    default: if (c == *p) return true; break;
-    }
-  return false;
-}
 
 // Set orig_topline.  Used when jumping to another window, so that a double
 // click still works.
diff --git a/src/nvim/normal.c b/src/nvim/normal.c
index 4b81a22f55..3ecac615b3 100644
--- a/src/nvim/normal.c
+++ b/src/nvim/normal.c
@@ -2375,10 +2375,10 @@ do_mouse (
      * Also paste at the cursor if the current mode isn't in 'mouse' (only
      * happens for the GUI).
      */
-    if ((State & INSERT) || !mouse_has(MOUSE_NORMAL)) {
-      if (regname == '.')
+    if ((State & INSERT)) {
+      if (regname == '.') {
         insert_reg(regname, true);
-      else {
+      } else {
         if (regname == 0 && eval_has_provider("clipboard")) {
           regname = '*';
         }
@@ -2558,8 +2558,9 @@ do_mouse (
          * on a status line */
         if (VIsual_active)
           jump_flags |= MOUSE_MAY_STOP_VIS;
-      } else if (mouse_has(MOUSE_VISUAL))
+      } else {
         jump_flags |= MOUSE_MAY_VIS;
+      }
     } else if (which_button == MOUSE_RIGHT) {
       if (is_click && VIsual_active) {
         /*
@@ -2575,8 +2576,7 @@ do_mouse (
         }
       }
       jump_flags |= MOUSE_FOCUS;
-      if (mouse_has(MOUSE_VISUAL))
-        jump_flags |= MOUSE_MAY_VIS;
+      jump_flags |= MOUSE_MAY_VIS;
     }
   }
 
@@ -2790,8 +2790,7 @@ do_mouse (
   /* Handle double clicks, unless on status line */
   else if (in_status_line) {
   } else if (in_sep_line) {
-  } else if ((mod_mask & MOD_MASK_MULTI_CLICK) && (State & (NORMAL | INSERT))
-             && mouse_has(MOUSE_VISUAL)) {
+  } else if ((mod_mask & MOD_MASK_MULTI_CLICK) && (State & (NORMAL | INSERT))) {
     if (is_click || !VIsual_active) {
       if (VIsual_active) {
         orig_cursor = VIsual;
diff --git a/src/nvim/option.c b/src/nvim/option.c
index bac289a959..7786feddf3 100644
--- a/src/nvim/option.c
+++ b/src/nvim/option.c
@@ -3210,11 +3210,7 @@ ambw_end:
   }
 
   if (varp == &p_mouse) {
-    if (*p_mouse == NUL) {
-      ui_call_mouse_off();
-    } else {
-      setmouse();  // in case 'mouse' changed
-    }
+    setmouse();  // in case 'mouse' changed
   }
 
   if (curwin->w_curswant != MAXCOL
@@ -4983,11 +4979,7 @@ void ui_refresh_options(void)
     ui_call_option_set(name, value);
   }
   if (p_mouse != NULL) {
-    if (*p_mouse == NUL) {
-      ui_call_mouse_off();
-    } else {
-      setmouse();
-    }
+    setmouse();
   }
 }
 
diff --git a/src/nvim/ui.c b/src/nvim/ui.c
index 685da77b39..c6c09c80d7 100644
--- a/src/nvim/ui.c
+++ b/src/nvim/ui.c
@@ -61,6 +61,9 @@ static bool pending_mode_info_update = false;
 static bool pending_mode_update = false;
 static handle_T cursor_grid_handle = DEFAULT_GRID_HANDLE;
 
+static bool has_mouse = false;
+static int pending_has_mouse = -1;
+
 #if MIN_LOG_LEVEL > DEBUG_LOG_LEVEL
 # define UI_LOG(funname)
 #else
@@ -220,6 +223,7 @@ void ui_refresh(void)
   ui_mode_info_set();
   pending_mode_update = true;
   ui_cursor_shape();
+  pending_has_mouse = -1;
 }
 
 int ui_pum_get_height(void)
@@ -459,10 +463,69 @@ void ui_flush(void)
     ui_call_mode_change(cstr_as_string(full_name), ui_mode_idx);
     pending_mode_update = false;
   }
+  if (pending_has_mouse != has_mouse) {
+    (has_mouse ? ui_call_mouse_on : ui_call_mouse_off)();
+    pending_has_mouse = has_mouse;
+  }
   ui_call_flush();
 }
 
+
+/// Check if 'mouse' is active for the current mode
+///
+/// TODO(bfredl): precompute the State -> active mapping when 'mouse' changes,
+/// then this can be checked directly in ui_flush()
+void ui_check_mouse(void)
+{
+  has_mouse = false;
+  // Be quick when mouse is off.
+  if (*p_mouse == NUL) {
+    return;
+  }
+
+  int checkfor = MOUSE_NORMAL;  // assume normal mode
+  if (VIsual_active) {
+    checkfor = MOUSE_VISUAL;
+  } else if (State == HITRETURN || State == ASKMORE || State == SETWSIZE) {
+    checkfor = MOUSE_RETURN;
+  } else if (State & INSERT) {
+    checkfor = MOUSE_INSERT;
+  } else if (State & CMDLINE) {
+    checkfor = MOUSE_COMMAND;
+  } else if (State == CONFIRM || State == EXTERNCMD) {
+    checkfor = ' ';  // don't use mouse for ":confirm" or ":!cmd"
+  }
+
+  // mouse should be active if at least one of the following is true:
+  // - "c" is in 'mouse', or
+  // - 'a' is in 'mouse' and "c" is in MOUSE_A, or
+  // - the current buffer is a help file and 'h' is in 'mouse' and we are in a
+  //   normal editing mode (not at hit-return message).
+  for (char_u *p = p_mouse; *p; p++) {
+    switch (*p) {
+      case 'a':
+        if (vim_strchr((char_u *)MOUSE_A, checkfor) != NULL) {
+          has_mouse = true;
+          return;
+        }
+        break;
+      case MOUSE_HELP:
+        if (checkfor != MOUSE_RETURN && curbuf->b_help) {
+          has_mouse = true;
+          return;
+        }
+        break;
+      default:
+        if (checkfor == *p) {
+          has_mouse = true;
+          return;
+        }
+    }
+  }
+}
+
 /// Check if current mode has changed.
+///
 /// May update the shape of the cursor.
 void ui_cursor_shape(void)
 {
diff --git a/test/functional/ui/mouse_spec.lua b/test/functional/ui/mouse_spec.lua
index a741136111..7bca741ae3 100644
--- a/test/functional/ui/mouse_spec.lua
+++ b/test/functional/ui/mouse_spec.lua
@@ -45,13 +45,33 @@ describe('ui/mouse/input', function()
 
   it('single left click moves cursor', function()
     feed('<LeftMouse><2,1>')
-    screen:expect([[
+    screen:expect{grid=[[
       testing                  |
       mo^use                    |
       support and selection    |
       {0:~                        }|
                                |
+    ]], mouse_enabled=true}
+    feed('<LeftMouse><0,0>')
+    screen:expect([[
+      ^testing                  |
+      mouse                    |
+      support and selection    |
+      {0:~                        }|
+                               |
     ]])
+  end)
+
+  it("in external ui works with unset 'mouse'", function()
+    meths.set_option('mouse', '')
+    feed('<LeftMouse><2,1>')
+    screen:expect{grid=[[
+      testing                  |
+      mo^use                    |
+      support and selection    |
+      {0:~                        }|
+                               |
+    ]], mouse_enabled=false}
     feed('<LeftMouse><0,0>')
     screen:expect([[
       ^testing                  |
diff --git a/test/functional/ui/screen.lua b/test/functional/ui/screen.lua
index 8fa9fcc42f..fcf6926433 100644
--- a/test/functional/ui/screen.lua
+++ b/test/functional/ui/screen.lua
@@ -170,7 +170,7 @@ function Screen.new(width, height)
     ruler = {},
     hl_groups = {},
     _default_attr_ids = nil,
-    _mouse_enabled = true,
+    mouse_enabled = true,
     _attrs = {},
     _hl_info = {[0]={}},
     _attr_table = {[0]={{},{}}},
@@ -318,7 +318,7 @@ function Screen:expect(expected, attr_ids, ...)
   assert(next({...}) == nil, "invalid args to expect()")
   if type(expected) == "table" then
     assert(not (attr_ids ~= nil))
-    local is_key = {grid=true, attr_ids=true, condition=true,
+    local is_key = {grid=true, attr_ids=true, condition=true, mouse_enabled=true,
                     any=true, mode=true, unchanged=true, intermediate=true,
                     reset=true, timeout=true, request_cb=true, hl_groups=true}
     for _, v in ipairs(ext_keys) do
@@ -422,12 +422,15 @@ screen:redraw_debug() to show all intermediate screen states.  ]])
     if expected.mode ~= nil then
       extstate.mode = self.mode
     end
+    if expected.mouse_enabled ~= nil then
+      extstate.mouse_enabled = self.mouse_enabled
+    end
     if expected.win_viewport == nil then
       extstate.win_viewport = nil
     end
 
     -- Convert assertion errors into invalid screen state descriptions.
-    for _, k in ipairs(concat_tables(ext_keys, {'mode'})) do
+    for _, k in ipairs(concat_tables(ext_keys, {'mode', 'mouse_enabled'})) do
       -- Empty states are considered the default and need not be mentioned.
       if (not (expected[k] == nil and isempty(extstate[k]))) then
         local status, res = pcall(eq, expected[k], extstate[k], k)
@@ -799,11 +802,11 @@ function Screen:_handle_busy_stop()
 end
 
 function Screen:_handle_mouse_on()
-  self._mouse_enabled = true
+  self.mouse_enabled = true
 end
 
 function Screen:_handle_mouse_off()
-  self._mouse_enabled = false
+  self.mouse_enabled = false
 end
 
 function Screen:_handle_mode_change(mode, idx)