api/helpers: Use typval_encode.h for vim_to_object

This ought to prevent stack overflow, but I do not see this actually working:
*lua* code crashes with stack overflow when trying to deserialize msgpack from
Neovim, Neovim is fine even if nesting level is increased 100x (though test
becomes very slow); not sure how recursive function may survive this. So it
looks like there are currently only two positive effects:

1. NULL lists are returned as empty (#4596).
2. Functional tests are slightly more fast. Very slightly. Checked for Release
   build for test/functional/eval tests because benchmarking of debug mode is
   not very useful.

1 files changed, 13 insertions, 0 deletions

diff --git a/test/functional/api/server_notifications_spec.lua b/test/functional/api/server_notifications_spec.lua
index d68ce411c0..110bc6cbf1 100644
--- a/test/functional/api/server_notifications_spec.lua
+++ b/test/functional/api/server_notifications_spec.lua
@@ -3,6 +3,7 @@ local helpers = require('test.functional.helpers')(after_each)
 local eq, clear, eval, execute, nvim, next_message =
   helpers.eq, helpers.clear, helpers.eval, helpers.execute, helpers.nvim,
   helpers.next_message
+local meths = helpers.meths
 
 describe('notify', function()
   local channel
@@ -36,5 +37,17 @@ describe('notify', function()
       eval('rpcnotify(0, "event1", 13, 14, 15)')
       eq({'notification', 'event1', {13, 14, 15}}, next_message())
     end)
+
+    it('does not crash for deeply nested variable', function()
+      meths.set_var('l', {})
+      local nest_level = 100000
+      meths.command(('call map(range(%u), "extend(g:, {\'l\': [g:l]})")'):format(nest_level))
+      local ret = {}
+      for i = 1, nest_level do
+        ret = {ret}
+      end
+      eval('rpcnotify('..channel..', "event", g:l)')
+      -- eq({'notification', 'event', ret}, next_message())
+    end)
   end)
 end)