fix(vim.json)!: remove global options, "null", "array_mt" #24070

Problem:
- `vim.json` exposes various global options which:
  - affect all Nvim Lua plugins (especially the LSP client)
  - are undocumented and untested
  - can cause confusing problems such as: https://github.com/codota/tabnine-nvim/commit/cc76ae3abe2f129d44b5a8edee2529e0ee0dcf69
- `vim.json` exposes redundant mechanisms:
  - `vim.json.null` is redundant with `vim.NIL`.
  - `array_mt` is redundant because Nvim uses a metatable
    (`vim.empty_dict()`) for empty dict instead, which `vim.json` is
    configured to use by default (see `as_empty_dict`).
    Example:
    ```
    :lua vim.print(vim.json.decode('{"bar":[],"foo":{}}'))
    --> { bar = {},  foo = vim.empty_dict() }
    ```
    Thus we don't need to also decorate empty arrays with `array_mt`.

Solution:
Remove the functions from the public vim.json interface.
Comment-out the implementation code to minimize drift from upstream.

TODO:
- Expose the options as arguments to `vim.json.new()`

4 files changed, 142 insertions, 48 deletions

diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt
index 1110fb7fc3..80ef2e358c 100644
--- a/runtime/doc/lua.txt
+++ b/runtime/doc/lua.txt
@@ -747,9 +747,22 @@ vim.json.encode({obj})                                       *vim.json.encode*
 vim.json.decode({str}[, {opts}])                             *vim.json.decode*
     Decodes (or "unpacks") the JSON-encoded {str} to a Lua object.
 
-    {opts} is a table with the key `luanil = { object: bool, array: bool }`
-    that controls whether `null` in JSON objects or arrays should be converted
-    to Lua `nil` instead of `vim.NIL`.
+    - Decodes JSON "null" as |vim.NIL| (controllable by {opts}, see below).
+    - Decodes empty object as |vim.empty_dict()|.
+    - Decodes empty array as `{}` (empty Lua table).
+
+    Example: >lua
+    :lua vim.print(vim.json.decode('{"bar":[],"foo":{},"zub":null}'))
+    --> { bar = {}, foo = vim.empty_dict(), zub = vim.NIL }
+<
+    Parameters: ~
+      • {str}    Stringified JSON data.
+      • {opts}   Options map keys:
+                 • luanil: { object: bool, array: bool }
+                   • `luanil.object=true` converts `null` in JSON objects to
+                     Lua `nil` instead of `vim.NIL`.
+                   • `luanil.array=true` converts `null` in JSON arrays to Lua
+                     `nil` instead of `vim.NIL`.
 
 ------------------------------------------------------------------------------
 VIM.SPELL                                                          *lua-spell*
diff --git a/runtime/doc/news.txt b/runtime/doc/news.txt
index 2a25edc4eb..550b69010d 100644
--- a/runtime/doc/news.txt
+++ b/runtime/doc/news.txt
@@ -43,6 +43,11 @@ The following changes may require adaptations in user config or plugins.
 
 • Renamed `vim.treesitter.playground` to `vim.treesitter.dev`.
 
+• Removed functions from the |vim.json| module:
+  • Unnecessary, undocumented functions which caused global side-effects.
+  • `vim.json.null` is redundant with `vim.NIL`.
+  • `vim.json.array_mt` (and related) is redundant with `vim.empty_dict()`.
+
 ==============================================================================
 NEW FEATURES                                                    *news-features*
 
diff --git a/src/cjson/lua_cjson.c b/src/cjson/lua_cjson.c
index c243f93c05..6af43d8eb7 100644
--- a/src/cjson/lua_cjson.c
+++ b/src/cjson/lua_cjson.c
@@ -1,3 +1,5 @@
+// Upstream: https://github.com/openresty/lua-cjson/blob/master/lua_cjson.c
+
 /* Lua CJSON - JSON support for Lua
  *
  * Copyright (c) 2010-2012  Mark Pulford <mark@kyne.com.au>
@@ -252,6 +254,7 @@ static json_config_t *json_fetch_config(lua_State *l)
 /* Ensure the correct number of arguments have been provided.
  * Pad with nil to allow other functions to simply check arg[i]
  * to find whether an argument was provided */
+/*
 static json_config_t *json_arg_init(lua_State *l, int args)
 {
     luaL_argcheck(l, lua_gettop(l) <= args, args + 1,
@@ -262,8 +265,10 @@ static json_config_t *json_arg_init(lua_State *l, int args)
 
     return json_fetch_config(l);
 }
+*/
 
 /* Process integer options for configuration functions */
+/*
 static int json_integer_option(lua_State *l, int optindex, int *setting,
                                int min, int max)
 {
@@ -281,8 +286,10 @@ static int json_integer_option(lua_State *l, int optindex, int *setting,
 
     return 1;
 }
+*/
 
 /* Process enumerated arguments for a configuration function */
+/*
 static int json_enum_option(lua_State *l, int optindex, int *setting,
                             const char **options, int bool_true)
 {
@@ -307,11 +314,13 @@ static int json_enum_option(lua_State *l, int optindex, int *setting,
 
     return 1;
 }
+*/
 
 /* Configures handling of extremely sparse arrays:
  * convert: Convert extremely sparse arrays into objects? Otherwise error.
  * ratio: 0: always allow sparse; 1: never allow sparse; >1: use ratio
  * safe: Always use an array when the max index <= safe */
+/*
 static int json_cfg_encode_sparse_array(lua_State *l)
 {
     json_config_t *cfg = json_arg_init(l, 3);
@@ -322,42 +331,52 @@ static int json_cfg_encode_sparse_array(lua_State *l)
 
     return 3;
 }
+*/
 
 /* Configures the maximum number of nested arrays/objects allowed when
  * encoding */
+/*
 static int json_cfg_encode_max_depth(lua_State *l)
 {
     json_config_t *cfg = json_arg_init(l, 1);
 
     return json_integer_option(l, 1, &cfg->encode_max_depth, 1, INT_MAX);
 }
+*/
 
 /* Configures the maximum number of nested arrays/objects allowed when
  * encoding */
+/*
 static int json_cfg_decode_max_depth(lua_State *l)
 {
     json_config_t *cfg = json_arg_init(l, 1);
 
     return json_integer_option(l, 1, &cfg->decode_max_depth, 1, INT_MAX);
 }
+*/
 
 /* Configures number precision when converting doubles to text */
+/*
 static int json_cfg_encode_number_precision(lua_State *l)
 {
     json_config_t *cfg = json_arg_init(l, 1);
 
     return json_integer_option(l, 1, &cfg->encode_number_precision, 1, 16);
 }
+*/
 
 /* Configures how to treat empty table when encode lua table */
+/*
 static int json_cfg_encode_empty_table_as_object(lua_State *l)
 {
     json_config_t *cfg = json_arg_init(l, 1);
 
     return json_enum_option(l, 1, &cfg->encode_empty_table_as_object, NULL, 1);
 }
+*/
 
 /* Configures how to decode arrays */
+/*
 static int json_cfg_decode_array_with_array_mt(lua_State *l)
 {
     json_config_t *cfg = json_arg_init(l, 1);
@@ -366,8 +385,10 @@ static int json_cfg_decode_array_with_array_mt(lua_State *l)
 
     return 1;
 }
+*/
 
 /* Configures JSON encoding buffer persistence */
+/*
 static int json_cfg_encode_keep_buffer(lua_State *l)
 {
     json_config_t *cfg = json_arg_init(l, 1);
@@ -377,7 +398,7 @@ static int json_cfg_encode_keep_buffer(lua_State *l)
 
     json_enum_option(l, 1, &cfg->encode_keep_buffer, NULL, 1);
 
-    /* Init / free the buffer if the setting has changed */
+    // Init / free the buffer if the setting has changed
     if (old_value ^ cfg->encode_keep_buffer) {
         if (cfg->encode_keep_buffer)
             strbuf_init(&cfg->encode_buf, 0);
@@ -387,6 +408,7 @@ static int json_cfg_encode_keep_buffer(lua_State *l)
 
     return 1;
 }
+*/
 
 #if defined(DISABLE_INVALID_NUMBERS) && !defined(USE_INTERNAL_FPCONV)
 void json_verify_invalid_number_setting(lua_State *l, int *setting)
@@ -400,6 +422,7 @@ void json_verify_invalid_number_setting(lua_State *l, int *setting)
 #define json_verify_invalid_number_setting(l, s)    do { } while(0)
 #endif
 
+/*
 static int json_cfg_encode_invalid_numbers(lua_State *l)
 {
     static const char *options[] = { "off", "on", "null", NULL };
@@ -411,7 +434,9 @@ static int json_cfg_encode_invalid_numbers(lua_State *l)
 
     return 1;
 }
+*/
 
+/*
 static int json_cfg_decode_invalid_numbers(lua_State *l)
 {
     json_config_t *cfg = json_arg_init(l, 1);
@@ -422,7 +447,9 @@ static int json_cfg_decode_invalid_numbers(lua_State *l)
 
     return 1;
 }
+*/
 
+/*
 static int json_cfg_encode_escape_forward_slash(lua_State *l)
 {
     int            ret;
@@ -436,6 +463,7 @@ static int json_cfg_encode_escape_forward_slash(lua_State *l)
     }
     return ret;
 }
+*/
 
 static int json_destroy_config(lua_State *l)
 {
@@ -1417,9 +1445,9 @@ static int json_decode(lua_State *l)
         lua_getfield(l, 2, "luanil");
 
         /* We only handle the luanil option for now */
-        if (lua_isnil(l, -1)) { 
-            lua_pop(l, 1); 
-            break; 
+        if (lua_isnil(l, -1)) {
+            lua_pop(l, 1);
+            break;
         }
 
         luaL_checktype(l, -1, LUA_TTABLE);
@@ -1534,6 +1562,8 @@ int lua_cjson_new(lua_State *l)
     luaL_Reg reg[] = {
         { "encode", json_encode },
         { "decode", json_decode },
+        // Nvim: don't expose options which cause global side-effects.
+        /*
         { "encode_empty_table_as_object", json_cfg_encode_empty_table_as_object },
         { "decode_array_with_array_mt", json_cfg_decode_array_with_array_mt },
         { "encode_sparse_array", json_cfg_encode_sparse_array },
@@ -1544,6 +1574,7 @@ int lua_cjson_new(lua_State *l)
         { "encode_invalid_numbers", json_cfg_encode_invalid_numbers },
         { "decode_invalid_numbers", json_cfg_decode_invalid_numbers },
         { "encode_escape_forward_slash", json_cfg_encode_escape_forward_slash },
+        */
         { "new", lua_cjson_new },
         { NULL, NULL }
     };
@@ -1589,23 +1620,31 @@ int lua_cjson_new(lua_State *l)
     json_create_config(l);
     compat_luaL_setfuncs(l, reg, 1);
 
-    /* Set cjson.null */
+    // Nvim: don't expose "null", it is identical to vim.NIL.
+    /*
     nlua_pushref(l, nlua_get_nil_ref(l));
     lua_setfield(l, -2, "null");
+    */
 
-    /* Set cjson.empty_array_mt */
+    // Nvim: don't expose empty_array_mt.
+    /*
     lua_pushlightuserdata(l, json_lightudata_mask(&json_empty_array));
     lua_rawget(l, LUA_REGISTRYINDEX);
     lua_setfield(l, -2, "empty_array_mt");
+    */
 
-    /* Set cjson.array_mt */
+    // Nvim: don't expose array_mt.
+    /*
     lua_pushlightuserdata(l, json_lightudata_mask(&json_array));
     lua_rawget(l, LUA_REGISTRYINDEX);
     lua_setfield(l, -2, "array_mt");
+    */
 
-    /* Set cjson.empty_array */
+    // Nvim: don't expose empty_array.
+    /*
     lua_pushlightuserdata(l, json_lightudata_mask(&json_array));
     lua_setfield(l, -2, "empty_array");
+    */
 
     /* Set module name / version fields */
     lua_pushliteral(l, CJSON_MODNAME);
diff --git a/test/functional/lua/json_spec.lua b/test/functional/lua/json_spec.lua
index fbb21bfd57..25fdb48eea 100644
--- a/test/functional/lua/json_spec.lua
+++ b/test/functional/lua/json_spec.lua
@@ -1,20 +1,57 @@
 local helpers = require('test.functional.helpers')(after_each)
 local clear = helpers.clear
-local NIL = helpers.NIL
 local exec_lua = helpers.exec_lua
 local eq = helpers.eq
+local pcall_err = helpers.pcall_err
 
-describe('vim.json.decode function', function()
+describe('vim.json.decode()', function()
   before_each(function()
     clear()
   end)
 
   it('parses null, true, false', function()
-    eq(NIL, exec_lua([[return vim.json.decode('null')]]))
+    eq(vim.NIL, exec_lua([[return vim.json.decode('null')]]))
     eq(true, exec_lua([[return vim.json.decode('true')]]))
     eq(false, exec_lua([[return vim.json.decode('false')]]))
   end)
 
+  it('validation', function()
+    eq('Expected object key string but found invalid token at character 2',
+      pcall_err(exec_lua, [[return vim.json.decode('{a:"b"}')]]))
+  end)
+
+  it('options', function()
+    local jsonstr = '{"arr":[1,2,null],"bar":[3,7],"foo":{"a":"b"},"baz":null}'
+    eq({
+        arr = { 1, 2, vim.NIL },
+        bar = { 3, 7 },
+        baz = vim.NIL,
+        foo = { a = 'b' },
+      },
+      exec_lua([[return vim.json.decode(..., {})]], jsonstr))
+    eq({
+        arr = { 1, 2, vim.NIL },
+        bar = { 3, 7 },
+        -- baz = nil,
+        foo = { a = 'b' },
+      },
+      exec_lua([[return vim.json.decode(..., { luanil = { object = true } })]], jsonstr))
+    eq({
+        arr = { 1, 2 },
+        bar = { 3, 7 },
+        baz = vim.NIL,
+        foo = { a = 'b' },
+      },
+      exec_lua([[return vim.json.decode(..., { luanil = { array = true } })]], jsonstr))
+    eq({
+        arr = { 1, 2 },
+        bar = { 3, 7 },
+        -- baz = nil,
+        foo = { a = 'b' },
+      },
+      exec_lua([[return vim.json.decode(..., { luanil = { array = true, object = true } })]], jsonstr))
+  end)
+
   it('parses integer numbers', function()
     eq(100000, exec_lua([[return vim.json.decode('100000')]]))
     eq(-100000, exec_lua([[return vim.json.decode('-100000')]]))
@@ -60,7 +97,7 @@ describe('vim.json.decode function', function()
 
   it('parses containers', function()
     eq({1}, exec_lua([[return vim.json.decode('[1]')]]))
-    eq({NIL, 1}, exec_lua([[return vim.json.decode('[null, 1]')]]))
+    eq({vim.NIL, 1}, exec_lua([[return vim.json.decode('[null, 1]')]]))
     eq({['1']=2}, exec_lua([[return vim.json.decode('{"1": 2}')]]))
     eq({['1']=2, ['3']={{['4']={['5']={{}, 1}}}}},
        exec_lua([[return vim.json.decode('{"1": 2, "3": [{"4": {"5": [ [], 1]}}]}')]]))
@@ -88,43 +125,43 @@ describe('vim.json.decode function', function()
 
 end)
 
-describe('vim.json.encode function', function()
+describe('vim.json.encode()', function()
   before_each(function()
     clear()
   end)
 
-   it('dumps strings', function()
-     eq('"Test"', exec_lua([[return vim.json.encode('Test')]]))
-     eq('""', exec_lua([[return vim.json.encode('')]]))
-     eq('"\\t"', exec_lua([[return vim.json.encode('\t')]]))
-     eq('"\\n"', exec_lua([[return vim.json.encode('\n')]]))
-     -- vim.fn.json_encode return \\u001B
-     eq('"\\u001b"', exec_lua([[return vim.json.encode('\27')]]))
-     eq('"þÿþ"', exec_lua([[return vim.json.encode('þÿþ')]]))
-   end)
-
-   it('dumps numbers', function()
-     eq('0', exec_lua([[return vim.json.encode(0)]]))
-     eq('10', exec_lua([[return vim.json.encode(10)]]))
-     eq('-10', exec_lua([[return vim.json.encode(-10)]]))
-   end)
-
-   it('dumps floats', function()
-     eq('10.5', exec_lua([[return vim.json.encode(10.5)]]))
-     eq('-10.5', exec_lua([[return vim.json.encode(-10.5)]]))
-     eq('-1e-05', exec_lua([[return vim.json.encode(-1e-5)]]))
-   end)
-
-   it('dumps lists', function()
-     eq('[]', exec_lua([[return vim.json.encode({})]]))
-     eq('[[]]', exec_lua([[return vim.json.encode({{}})]]))
-     eq('[[],[]]', exec_lua([[return vim.json.encode({{}, {}})]]))
-   end)
-
-   it('dumps dictionaries', function()
-     eq('{}', exec_lua([[return vim.json.encode(vim.empty_dict())]]))
-     eq('{"d":[]}', exec_lua([[return vim.json.encode({d={}})]]))
-   end)
+  it('dumps strings', function()
+    eq('"Test"', exec_lua([[return vim.json.encode('Test')]]))
+    eq('""', exec_lua([[return vim.json.encode('')]]))
+    eq('"\\t"', exec_lua([[return vim.json.encode('\t')]]))
+    eq('"\\n"', exec_lua([[return vim.json.encode('\n')]]))
+    -- vim.fn.json_encode return \\u001B
+    eq('"\\u001b"', exec_lua([[return vim.json.encode('\27')]]))
+    eq('"þÿþ"', exec_lua([[return vim.json.encode('þÿþ')]]))
+  end)
+
+  it('dumps numbers', function()
+    eq('0', exec_lua([[return vim.json.encode(0)]]))
+    eq('10', exec_lua([[return vim.json.encode(10)]]))
+    eq('-10', exec_lua([[return vim.json.encode(-10)]]))
+  end)
+
+  it('dumps floats', function()
+    eq('10.5', exec_lua([[return vim.json.encode(10.5)]]))
+    eq('-10.5', exec_lua([[return vim.json.encode(-10.5)]]))
+    eq('-1e-05', exec_lua([[return vim.json.encode(-1e-5)]]))
+  end)
+
+  it('dumps lists', function()
+    eq('[]', exec_lua([[return vim.json.encode({})]]))
+    eq('[[]]', exec_lua([[return vim.json.encode({{}})]]))
+    eq('[[],[]]', exec_lua([[return vim.json.encode({{}, {}})]]))
+  end)
+
+  it('dumps dictionaries', function()
+    eq('{}', exec_lua([[return vim.json.encode(vim.empty_dict())]]))
+    eq('{"d":[]}', exec_lua([[return vim.json.encode({d={}})]]))
+  end)
 
   it('dumps vim.NIL', function()
     eq('null', exec_lua([[return vim.json.encode(vim.NIL)]]))