api/helpers: Use typval_encode.h for vim_to_object

This ought to prevent stack overflow, but I do not see this actually working:
*lua* code crashes with stack overflow when trying to deserialize msgpack from
Neovim, Neovim is fine even if nesting level is increased 100x (though test
becomes very slow); not sure how recursive function may survive this. So it
looks like there are currently only two positive effects:

1. NULL lists are returned as empty (#4596).
2. Functional tests are slightly more fast. Very slightly. Checked for Release
   build for test/functional/eval tests because benchmarking of debug mode is
   not very useful.

2 files changed, 4 insertions, 5 deletions

diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c
index 54daf7557e..771655fee5 100644
--- a/src/nvim/eval/encode.c
+++ b/src/nvim/eval/encode.c
@@ -379,7 +379,6 @@ int encode_read_from_list(ListReaderState *const state, char *const buf,
       } \
       vim_snprintf(ebuf, ARRAY_SIZE(ebuf), "{E724@%zu}", backref); \
       ga_concat(gap, &ebuf[0]); \
-      return OK; \
     } while (0)
 
 #define TYPVAL_ENCODE_ALLOW_SPECIALS false
@@ -426,7 +425,6 @@ TYPVAL_ENCODE_DEFINE_CONV_FUNCTIONS(, echo, garray_T *const, gap)
         EMSG(_("E724: unable to correctly dump variable " \
                "with self-referencing container")); \
       } \
-      return OK; \
     } while (0)
 
 #undef TYPVAL_ENCODE_ALLOW_SPECIALS
@@ -662,9 +660,8 @@ static inline int convert_to_json_string(garray_T *const gap,
 /// Check whether given key can be used in json_encode()
 ///
 /// @param[in]  tv  Key to check.
-static inline bool check_json_key(const typval_T *const tv)
+bool encode_check_json_key(const typval_T *const tv)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_PURE
-  FUNC_ATTR_ALWAYS_INLINE
 {
   if (tv->v_type == VAR_STRING) {
     return true;
@@ -701,7 +698,7 @@ static inline bool check_json_key(const typval_T *const tv)
 #undef TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK
 #define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(label, kv_pair) \
     do { \
-      if (!check_json_key(&kv_pair->lv_first->li_tv)) { \
+      if (!encode_check_json_key(&kv_pair->lv_first->li_tv)) { \
         EMSG(_("E474: Invalid key in special dictionary")); \
         goto label; \
       } \
diff --git a/src/nvim/eval/typval_encode.h b/src/nvim/eval/typval_encode.h
index f70a6c9e94..c2e8403475 100644
--- a/src/nvim/eval/typval_encode.h
+++ b/src/nvim/eval/typval_encode.h
@@ -184,10 +184,12 @@ typedef kvec_t(MPConvStackVal) MPConvStack;
 /// @param  copyID_attr  Name of the container attribute that holds copyID.
 ///                      After checking whether value of this attribute is
 ///                      copyID (variable) it is set to copyID.
+/// @param  conv_type  Type of the conversion, @see MPConvStackValType.
 #define _TYPVAL_ENCODE_CHECK_SELF_REFERENCE(val, copyID_attr, conv_type) \
     do { \
       if ((val)->copyID_attr == copyID) { \
         TYPVAL_ENCODE_CONV_RECURSE((val), conv_type); \
+        return OK; \
       } \
       (val)->copyID_attr = copyID; \
     } while (0)