fix(vim.ui)!: change open() to return `result|nil, errmsg|nil` #28612

reverts e0d92b9cc20b58179599f53dfa74ca821935a539 #28502

Problem:
`vim.ui.open()` has a `pcall()` like signature, under the assumption
that this is the Lua idiom for returning result-or-error. However, the
`result|nil, errmsg|nil` pattern:
- has precedent in:
  - `io.open`
  - `vim.uv` (`:help luv-error-handling`)
- has these advantages:
  - Can be used with `assert()`:
    ```
    local result, err = assert(foobar())
    ```
  - Allows LuaLS to infer the type of `result`:
    ```
    local result, err = foobar()
    if err then
      ...
    elseif result then
      ...
    end
    ```

Solution:
- Revert to the `result|nil, errmsg|nil` pattern.
- Document the pattern in our guidelines.

1 files changed, 7 insertions, 8 deletions

diff --git a/runtime/lua/vim/_defaults.lua b/runtime/lua/vim/_defaults.lua
index 29f8e71264..6d31a3ea93 100644
--- a/runtime/lua/vim/_defaults.lua
+++ b/runtime/lua/vim/_defaults.lua
@@ -98,19 +98,18 @@ do
   --- Map |gx| to call |vim.ui.open| on the <cfile> at cursor.
   do
     local function do_open(uri)
-      local ok, cmd_or_err = vim.ui.open(uri)
-      local rv = ok and (cmd_or_err --[[@as vim.SystemObj]]):wait(1000) or nil
-      if rv and rv.code ~= 0 then
-        ok = false
-        cmd_or_err = ('vim.ui.open: command %s (%d): %s'):format(
+      local cmd, err = vim.ui.open(uri)
+      local rv = cmd and cmd:wait(1000) or nil
+      if cmd and rv and rv.code ~= 0 then
+        err = ('vim.ui.open: command %s (%d): %s'):format(
           (rv.code == 124 and 'timeout' or 'failed'),
           rv.code,
-          vim.inspect(cmd_or_err.cmd)
+          vim.inspect(cmd.cmd)
         )
       end
 
-      if not ok then
-        vim.notify(cmd_or_err --[[@as string]], vim.log.levels.ERROR)
+      if err then
+        vim.notify(err, vim.log.levels.ERROR)
       end
     end