fix(vim.ui.open): return (don't show) error message

Problem:
Showing an error via vim.notify() makes it awkward for callers such as
lsp/handlers.lua to avoid showing redundant errors.

Solution:
Return the message instead of showing it. Let the caller decide whether
and when to show the message.

4 files changed, 31 insertions, 19 deletions

diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt
index b09e308e80..77a89a123d 100644
--- a/runtime/doc/lua.txt
+++ b/runtime/doc/lua.txt
@@ -2345,7 +2345,8 @@ input({opts}, {on_confirm})                                   *vim.ui.input()*
 
 open({path})                                                   *vim.ui.open()*
     Opens `path` with the system default handler (macOS `open`, Windows
-    `explorer.exe`, Linux `xdg-open`, …), or shows a message on failure.
+    `explorer.exe`, Linux `xdg-open`, …), or returns (but does not show) an
+    error message on failure.
 
     Expands "~/" and environment variables in filesystem paths.
 
@@ -2360,7 +2361,8 @@ open({path})                                                   *vim.ui.open()*
       • {path}  (string) Path or URL to open
 
     Return: ~
-        SystemCompleted|nil result Command result, or nil if not found.
+        SystemCompleted|nil # Command result, or nil if not found.
+        (string|nil) # Error message on failure
 
     See also: ~
       • |vim.system()|
diff --git a/runtime/lua/vim/lsp/handlers.lua b/runtime/lua/vim/lsp/handlers.lua
index 79d3f7aab0..70781cb7a6 100644
--- a/runtime/lua/vim/lsp/handlers.lua
+++ b/runtime/lua/vim/lsp/handlers.lua
@@ -573,15 +573,14 @@ M['window/showDocument'] = function(_, result, ctx, _)
 
   if result.external then
     -- TODO(lvimuser): ask the user for confirmation
-
-    local ret = vim.ui.open(uri)
+    local ret, err = vim.ui.open(uri)
 
     if ret == nil or ret.code ~= 0 then
       return {
         success = false,
         error = {
           code = protocol.ErrorCodes.UnknownErrorCode,
-          message = ret and ret.stderr or 'No handler found',
+          message = ret and ret.stderr or err,
         },
       }
     end
diff --git a/runtime/lua/vim/ui.lua b/runtime/lua/vim/ui.lua
index 3ffa329f74..fd06611da2 100644
--- a/runtime/lua/vim/ui.lua
+++ b/runtime/lua/vim/ui.lua
@@ -105,7 +105,7 @@ function M.input(opts, on_confirm)
 end
 
 --- Opens `path` with the system default handler (macOS `open`, Windows `explorer.exe`, Linux
---- `xdg-open`, …), or shows a message on failure.
+--- `xdg-open`, …), or returns (but does not show) an error message on failure.
 ---
 --- Expands "~/" and environment variables in filesystem paths.
 ---
@@ -118,13 +118,14 @@ end
 ---
 ---@param path string Path or URL to open
 ---
----@return SystemCompleted|nil result Command result, or nil if not found.
+---@return SystemCompleted|nil # Command result, or nil if not found.
+---@return string|nil # Error message on failure
 ---
 ---@see |vim.system()|
 function M.open(path)
-  vim.validate{
-    path={path, 'string'}
-  }
+  vim.validate({
+    path = { path, 'string' },
+  })
   local is_uri = path:match('%w+:')
   if not is_uri then
     path = vim.fn.expand(path)
@@ -141,17 +142,16 @@ function M.open(path)
   elseif vim.fn.executable('xdg-open') == 1 then
     cmd = { 'xdg-open', path }
   else
-    vim.notify('vim.ui.open: no handler found (tried: wslview, xdg-open)', vim.log.levels.ERROR)
-    return nil
+    return nil, 'vim.ui.open: no handler found (tried: wslview, xdg-open)'
   end
 
-  local rv = vim.system(cmd, { text = true, detach = true, }):wait()
+  local rv = vim.system(cmd, { text = true, detach = true }):wait()
   if rv.code ~= 0 then
     local msg = ('vim.ui.open: command failed (%d): %s'):format(rv.code, vim.inspect(cmd))
-    vim.notify(msg, vim.log.levels.ERROR)
+    return rv, msg
   end
 
-  return rv
+  return rv, nil
 end
 
 return M
diff --git a/runtime/plugin/nvim.lua b/runtime/plugin/nvim.lua
index 9fff6246e3..33d399e577 100644
--- a/runtime/plugin/nvim.lua
+++ b/runtime/plugin/nvim.lua
@@ -22,12 +22,23 @@ end, { desc = 'Inspect treesitter language tree for buffer', count = true })
 -- TODO: use vim.region() when it lands... #13896 #16843
 local function get_visual_selection()
   local save_a = vim.fn.getreginfo('a')
-  vim.cmd[[norm! "ay]]
+  vim.cmd([[norm! "ay]])
   local selection = vim.fn.getreg('a', 1)
   vim.fn.setreg('a', save_a)
   return selection
 end
 
-local gx_desc = 'Opens filepath or URI under cursor with the system handler (file explorer, web browser, …)'
-vim.keymap.set({ 'n' }, 'gx', function() vim.ui.open(vim.fn.expand('<cfile>')) end, { desc = gx_desc })
-vim.keymap.set({ 'x' }, 'gx', function() vim.ui.open(get_visual_selection()) end, { desc = gx_desc })
+local gx_desc =
+  'Opens filepath or URI under cursor with the system handler (file explorer, web browser, …)'
+local function do_open(uri)
+  local _, err = vim.ui.open(uri)
+  if err then
+    vim.notify(err, vim.log.levels.ERROR)
+  end
+end
+vim.keymap.set({ 'n' }, 'gx', function()
+  do_open(vim.fn.expand('<cfile>'))
+end, { desc = gx_desc })
+vim.keymap.set({ 'x' }, 'gx', function()
+  do_open(get_visual_selection())
+end, { desc = gx_desc })