ui: make 'mouse' handling in external UI more consistent

before the behaviour of 'mouse' was inconsistent in external UI,
as some remapping logic would check has_mouse() and others don't
(no difference in TUI or vim classic). With this change, the behaviour
is consistently up to the UI decide (see ui.txt edit)

Behaviour of tui.c is unaffected by this change.

7 files changed, 74 insertions, 82 deletions

diff --git a/src/nvim/edit.c b/src/nvim/edit.c
index 45bd84261d..2d9dba839d 100644
--- a/src/nvim/edit.c
+++ b/src/nvim/edit.c
@@ -8328,9 +8328,6 @@ static void ins_mouse(int c)
   pos_T tpos;
   win_T       *old_curwin = curwin;
 
-  if (!mouse_has(MOUSE_INSERT))
-    return;
-
   undisplay_dollar();
   tpos = curwin->w_cursor;
   if (do_mouse(NULL, c, BACKWARD, 1, 0)) {
diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c
index 129cb323ea..53dd2776c1 100644
--- a/src/nvim/ex_getln.c
+++ b/src/nvim/ex_getln.c
@@ -1880,9 +1880,6 @@ static int command_line_handle_key(CommandLineState *s)
     return command_line_not_changed(s);                 // Ignore mouse
 
   case K_MIDDLEMOUSE:
-    if (!mouse_has(MOUSE_COMMAND)) {
-      return command_line_not_changed(s);                   // Ignore mouse
-    }
     cmdline_paste(eval_has_provider("clipboard") ? '*' : 0, true, true);
     redrawcmd();
     return command_line_changed(s);
@@ -1906,10 +1903,6 @@ static int command_line_handle_key(CommandLineState *s)
       s->ignore_drag_release = false;
     }
 
-    if (!mouse_has(MOUSE_COMMAND)) {
-      return command_line_not_changed(s);                   // Ignore mouse
-    }
-
     ccline.cmdspos = cmd_startcol();
     for (ccline.cmdpos = 0; ccline.cmdpos < ccline.cmdlen;
          ccline.cmdpos++) {
diff --git a/src/nvim/message.c b/src/nvim/message.c
index 02a7732f5c..f94529c687 100644
--- a/src/nvim/message.c
+++ b/src/nvim/message.c
@@ -1157,15 +1157,7 @@ void wait_return(int redraw)
              || c == K_MIDDLEDRAG || c == K_MIDDLERELEASE
              || c == K_RIGHTDRAG  || c == K_RIGHTRELEASE
              || c == K_MOUSELEFT  || c == K_MOUSERIGHT
-             || c == K_MOUSEDOWN  || c == K_MOUSEUP
-             || (!mouse_has(MOUSE_RETURN)
-                 && mouse_row < msg_row
-                 && (c == K_LEFTMOUSE
-                     || c == K_MIDDLEMOUSE
-                     || c == K_RIGHTMOUSE
-                     || c == K_X1MOUSE
-                     || c == K_X2MOUSE))
-             );
+             || c == K_MOUSEDOWN  || c == K_MOUSEUP);
     os_breakcheck();
     /*
      * Avoid that the mouse-up event causes visual mode to start.
diff --git a/src/nvim/mouse.c b/src/nvim/mouse.c
index f05dade73f..ff471ea978 100644
--- a/src/nvim/mouse.c
+++ b/src/nvim/mouse.c
@@ -526,53 +526,9 @@ static win_T *mouse_find_grid_win(int *gridp, int *rowp, int *colp)
 void setmouse(void)
 {
   ui_cursor_shape();
-
-  // Be quick when mouse is off.
-  if (*p_mouse == NUL) {
-    return;
-  }
-
-  int checkfor = MOUSE_NORMAL;  // assume normal mode
-  if (VIsual_active) {
-    checkfor = MOUSE_VISUAL;
-  } else if (State == HITRETURN || State == ASKMORE || State == SETWSIZE) {
-    checkfor = MOUSE_RETURN;
-  } else if (State & INSERT) {
-    checkfor = MOUSE_INSERT;
-  } else if (State & CMDLINE) {
-    checkfor = MOUSE_COMMAND;
-  } else if (State == CONFIRM || State == EXTERNCMD) {
-    checkfor = ' ';  // don't use mouse for ":confirm" or ":!cmd"
-  }
-
-  if (mouse_has(checkfor)) {
-    ui_call_mouse_on();
-  } else {
-    ui_call_mouse_off();
-  }
+  ui_check_mouse();
 }
 
-/*
- * Return true if
- * - "c" is in 'mouse', or
- * - 'a' is in 'mouse' and "c" is in MOUSE_A, or
- * - the current buffer is a help file and 'h' is in 'mouse' and we are in a
- *   normal editing mode (not at hit-return message).
- */
-int mouse_has(int c)
-{
-  for (char_u *p = p_mouse; *p; ++p)
-    switch (*p) {
-    case 'a': if (vim_strchr((char_u *)MOUSE_A, c) != NULL)
-        return true;
-      break;
-    case MOUSE_HELP: if (c != MOUSE_RETURN && curbuf->b_help)
-        return true;
-      break;
-    default: if (c == *p) return true; break;
-    }
-  return false;
-}
 
 // Set orig_topline.  Used when jumping to another window, so that a double
 // click still works.
diff --git a/src/nvim/normal.c b/src/nvim/normal.c
index 4b81a22f55..3ecac615b3 100644
--- a/src/nvim/normal.c
+++ b/src/nvim/normal.c
@@ -2375,10 +2375,10 @@ do_mouse (
      * Also paste at the cursor if the current mode isn't in 'mouse' (only
      * happens for the GUI).
      */
-    if ((State & INSERT) || !mouse_has(MOUSE_NORMAL)) {
-      if (regname == '.')
+    if ((State & INSERT)) {
+      if (regname == '.') {
         insert_reg(regname, true);
-      else {
+      } else {
         if (regname == 0 && eval_has_provider("clipboard")) {
           regname = '*';
         }
@@ -2558,8 +2558,9 @@ do_mouse (
          * on a status line */
         if (VIsual_active)
           jump_flags |= MOUSE_MAY_STOP_VIS;
-      } else if (mouse_has(MOUSE_VISUAL))
+      } else {
         jump_flags |= MOUSE_MAY_VIS;
+      }
     } else if (which_button == MOUSE_RIGHT) {
       if (is_click && VIsual_active) {
         /*
@@ -2575,8 +2576,7 @@ do_mouse (
         }
       }
       jump_flags |= MOUSE_FOCUS;
-      if (mouse_has(MOUSE_VISUAL))
-        jump_flags |= MOUSE_MAY_VIS;
+      jump_flags |= MOUSE_MAY_VIS;
     }
   }
 
@@ -2790,8 +2790,7 @@ do_mouse (
   /* Handle double clicks, unless on status line */
   else if (in_status_line) {
   } else if (in_sep_line) {
-  } else if ((mod_mask & MOD_MASK_MULTI_CLICK) && (State & (NORMAL | INSERT))
-             && mouse_has(MOUSE_VISUAL)) {
+  } else if ((mod_mask & MOD_MASK_MULTI_CLICK) && (State & (NORMAL | INSERT))) {
     if (is_click || !VIsual_active) {
       if (VIsual_active) {
         orig_cursor = VIsual;
diff --git a/src/nvim/option.c b/src/nvim/option.c
index bac289a959..7786feddf3 100644
--- a/src/nvim/option.c
+++ b/src/nvim/option.c
@@ -3210,11 +3210,7 @@ ambw_end:
   }
 
   if (varp == &p_mouse) {
-    if (*p_mouse == NUL) {
-      ui_call_mouse_off();
-    } else {
-      setmouse();  // in case 'mouse' changed
-    }
+    setmouse();  // in case 'mouse' changed
   }
 
   if (curwin->w_curswant != MAXCOL
@@ -4983,11 +4979,7 @@ void ui_refresh_options(void)
     ui_call_option_set(name, value);
   }
   if (p_mouse != NULL) {
-    if (*p_mouse == NUL) {
-      ui_call_mouse_off();
-    } else {
-      setmouse();
-    }
+    setmouse();
   }
 }
 
diff --git a/src/nvim/ui.c b/src/nvim/ui.c
index 685da77b39..c6c09c80d7 100644
--- a/src/nvim/ui.c
+++ b/src/nvim/ui.c
@@ -61,6 +61,9 @@ static bool pending_mode_info_update = false;
 static bool pending_mode_update = false;
 static handle_T cursor_grid_handle = DEFAULT_GRID_HANDLE;
 
+static bool has_mouse = false;
+static int pending_has_mouse = -1;
+
 #if MIN_LOG_LEVEL > DEBUG_LOG_LEVEL
 # define UI_LOG(funname)
 #else
@@ -220,6 +223,7 @@ void ui_refresh(void)
   ui_mode_info_set();
   pending_mode_update = true;
   ui_cursor_shape();
+  pending_has_mouse = -1;
 }
 
 int ui_pum_get_height(void)
@@ -459,10 +463,69 @@ void ui_flush(void)
     ui_call_mode_change(cstr_as_string(full_name), ui_mode_idx);
     pending_mode_update = false;
   }
+  if (pending_has_mouse != has_mouse) {
+    (has_mouse ? ui_call_mouse_on : ui_call_mouse_off)();
+    pending_has_mouse = has_mouse;
+  }
   ui_call_flush();
 }
 
+
+/// Check if 'mouse' is active for the current mode
+///
+/// TODO(bfredl): precompute the State -> active mapping when 'mouse' changes,
+/// then this can be checked directly in ui_flush()
+void ui_check_mouse(void)
+{
+  has_mouse = false;
+  // Be quick when mouse is off.
+  if (*p_mouse == NUL) {
+    return;
+  }
+
+  int checkfor = MOUSE_NORMAL;  // assume normal mode
+  if (VIsual_active) {
+    checkfor = MOUSE_VISUAL;
+  } else if (State == HITRETURN || State == ASKMORE || State == SETWSIZE) {
+    checkfor = MOUSE_RETURN;
+  } else if (State & INSERT) {
+    checkfor = MOUSE_INSERT;
+  } else if (State & CMDLINE) {
+    checkfor = MOUSE_COMMAND;
+  } else if (State == CONFIRM || State == EXTERNCMD) {
+    checkfor = ' ';  // don't use mouse for ":confirm" or ":!cmd"
+  }
+
+  // mouse should be active if at least one of the following is true:
+  // - "c" is in 'mouse', or
+  // - 'a' is in 'mouse' and "c" is in MOUSE_A, or
+  // - the current buffer is a help file and 'h' is in 'mouse' and we are in a
+  //   normal editing mode (not at hit-return message).
+  for (char_u *p = p_mouse; *p; p++) {
+    switch (*p) {
+      case 'a':
+        if (vim_strchr((char_u *)MOUSE_A, checkfor) != NULL) {
+          has_mouse = true;
+          return;
+        }
+        break;
+      case MOUSE_HELP:
+        if (checkfor != MOUSE_RETURN && curbuf->b_help) {
+          has_mouse = true;
+          return;
+        }
+        break;
+      default:
+        if (checkfor == *p) {
+          has_mouse = true;
+          return;
+        }
+    }
+  }
+}
+
 /// Check if current mode has changed.
+///
 /// May update the shape of the cursor.
 void ui_cursor_shape(void)
 {