diff options
-rw-r--r-- | runtime/doc/eval.txt | 93 | ||||
-rw-r--r-- | src/nvim/eval.c | 951 | ||||
-rw-r--r-- | src/nvim/eval.h | 1 | ||||
-rw-r--r-- | src/nvim/eval_defs.h | 5 | ||||
-rw-r--r-- | src/nvim/ex_cmds.c | 2 | ||||
-rw-r--r-- | src/nvim/ex_cmds2.c | 2 | ||||
-rw-r--r-- | test/functional/eval/msgpack_functions_spec.lua | 606 |
7 files changed, 1646 insertions, 14 deletions
diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 9182eb76b5..d12871f4ab 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -1517,6 +1517,12 @@ v:mouse_col Column number for a mouse click obtained with |getchar()|. This is the screen column number, like with |virtcol()|. The value is zero when there was no mouse button click. + *v:msgpack_types* *msgpack_types-variable* +v:msgpack_types Dictionary containing msgpack types used by |msgpackparse()| + and |msgpackdump()|. All types inside dictionary are fixed + (not editable) empty lists. To check whether some list is one + of msgpack types, use |is| operator. + *v:oldfiles* *oldfiles-variable* v:oldfiles List of file names that is loaded from the |viminfo| file on startup. These are the files that Vim remembers marks for. @@ -1904,6 +1910,8 @@ min( {list}) Number minimum value of items in {list} mkdir( {name} [, {path} [, {prot}]]) Number create directory {name} mode( [expr]) String current editing mode +msgpackdump( {list}) List dump a list of objects to msgpack +msgpackparse( {list}) List parse msgpack to a list of objects nextnonblank( {lnum}) Number line nr of non-blank line >= {lnum} nr2char( {expr}[, {utf8}]) String single char with ASCII/UTF8 value {expr} or( {expr}, {expr}) Number bitwise OR @@ -4625,6 +4633,91 @@ mode([expr]) Return a string that indicates the current mode. "c" or "n". Also see |visualmode()|. +msgpackdump({list}) {Nvim} *msgpackdump()* + Convert a list of VimL objects to msgpack. Returned value is + |readfile()|-style list. Example: > + call writefile(msgpackdump([{}]), 'fname.mpack', 'b') +< This will write the single 0x80 byte to `fname.mpack` file + (dictionary with zero items is represented by 0x80 byte in + messagepack). + + Limitations: + 1. |Funcref|s cannot be dumped. + 2. Containers that reference themselves cannot be dumped. + 3. Dictionary keys are always dumped as STR strings. + 4. Other strings are always dumped as BIN strings. + 5. Points 3. and 4. do not apply to |msgpack-special-dict|s. + +msgpackparse({list}) {Nvim} *msgpackparse()* + Convert a |readfile()|-style list to a list of VimL objects. + Example: > + let fname = expand('~/.nvim/shada/main.shada') + let mpack = readfile(fname, 'b') + let shada_objects = msgpackparse(mpack) +< This will read |shada-file| to `shada_objects` list. + + Limitations: + 1. Mapping ordering is not preserved unless messagepack + mapping is dumped using generic mapping + (|msgpack-special-map|). + 2. Since the parser aims to preserve all data untouched + (except for 1.) some strings are parsed to + |msgpack-special-dict| format which is not convenient to + use. + *msgpack-special-dict* + Some messagepack strings may be parsed to special + dictionaries. Special dictionaries are dictionaries which + + 1. Contain exactly two keys: `_TYPE` and `_VALUE`. + 2. `_TYPE` key is one of the types found in |v:msgpack_types| + variable. + 3. Value for `_VALUE` has the following format (Key column + contains name of the key from |v:msgpack_types|): + + Key Value ~ + nil Zero, ignored when dumping. + boolean One or zero. When dumping it is only checked that + value is a |Number|. + integer |List| with four numbers: sign (-1 or 1), highest two + bits, number with bits from 62nd to 31st, lowest 31 + bits. I.e. to get actual number one will need to use + code like > + _VALUE[0] * ((_VALUE[1] << 62) + & (_VALUE[2] << 31) + & _VALUE[3]) +< Special dictionary with this type will appear in + |msgpackparse()| output under one of the following + circumstances: + 1. |Number| is 32-bit and value is either above + INT32_MAX or below INT32_MIN. + 2. |Number| is 64-bit and value is above INT64_MAX. It + cannot possibly be below INT64_MIN because msgpack + C parser does not support such values. + float |Float|. This value cannot possibly appear in + |msgpackparse()| output. + string |readfile()|-style list of strings. This value will + appear in |msgpackparse()| output if string contains + zero byte or if string is a mapping key and mapping is + being represented as special dictionary for other + reasons. + binary |readfile()|-style list of strings. This value will + appear in |msgpackparse()| output if binary string + contains zero byte. + array |List|. This value cannot appear in |msgpackparse()| + output. + *msgpack-special-map* + map |List| of |List|s with two items (key and value) each. + This value will appear in |msgpackparse()| output if + parsed mapping contains one of the following keys: + 1. Any key that is not a string (including keys which + are binary strings). + 2. String with NUL byte inside. + 3. Duplicate key. + 4. Empty key. + ext |List| with two values: first is a signed integer + representing extension type. Second is + |readfile()|-style list of strings. + nextnonblank({lnum}) *nextnonblank()* Return the line number of the first line at or below {lnum} that is not blank. Example: > diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 18188696b9..48f9faa777 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -20,6 +20,7 @@ #include <stdbool.h> #include <math.h> #include <limits.h> +#include <msgpack.h> #include "nvim/assert.h" #include "nvim/vim.h" @@ -90,11 +91,13 @@ #include "nvim/os/time.h" #include "nvim/msgpack_rpc/channel.h" #include "nvim/msgpack_rpc/server.h" +#include "nvim/msgpack_rpc/helpers.h" #include "nvim/api/private/helpers.h" #include "nvim/api/vim.h" #include "nvim/os/dl.h" #include "nvim/os/input.h" #include "nvim/event/loop.h" +#include "nvim/lib/kvec.h" #define DICT_MAXNEST 100 /* maximum nesting of lists and dicts */ @@ -160,6 +163,13 @@ typedef struct lval_S { char_u *ll_newkey; /* New key for Dict in alloc. mem or NULL. */ } lval_T; +/// Structure defining state for read_from_list() +typedef struct { + const listitem_T *li; ///< Item currently read. + size_t offset; ///< Byte offset inside the read item. + size_t li_length; ///< Length of the string inside the read item. +} ListReaderState; + static char *e_letunexp = N_("E18: Unexpected characters in :let"); static char *e_listidx = N_("E684: list index out of range: %" PRId64); @@ -429,6 +439,7 @@ static struct vimvar { {VV_NAME("progpath", VAR_STRING), VV_RO}, {VV_NAME("command_output", VAR_STRING), 0}, {VV_NAME("completed_item", VAR_DICT), VV_RO}, + {VV_NAME("msgpack_types", VAR_DICT), VV_RO}, }; /* shorthand */ @@ -459,6 +470,29 @@ typedef struct { uint64_t id; } TerminalJobData; +/// Structure representing current VimL to messagepack conversion state +typedef struct { + enum { + kMPConvDict, ///< Convert dict_T *dictionary. + kMPConvList, ///< Convert list_T *list. + kMPConvPairs, ///< Convert mapping represented as a list_T* of pairs. + } type; + union { + struct { + const dict_T *dict; ///< Currently converted dictionary. + const hashitem_T *hi; ///< Currently converted dictionary item. + size_t todo; ///< Amount of items left to process. + } d; ///< State of dictionary conversion. + struct { + const list_T *list; ///< Currently converted list. + const listitem_T *li; ///< Currently converted list item. + } l; ///< State of list or generic mapping conversion. + } data; ///< Data to convert. +} MPConvStackVal; + +/// Stack used to convert VimL values to messagepack. +typedef kvec_t(MPConvStackVal) MPConvStack; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "eval.c.generated.h" @@ -479,6 +513,40 @@ static int disable_job_defer = 0; static uint64_t current_job_id = 1; static PMap(uint64_t) *jobs = NULL; +typedef enum { + kMPNil, + kMPBoolean, + kMPInteger, + kMPFloat, + kMPString, + kMPBinary, + kMPArray, + kMPMap, + kMPExt, +} MessagePackType; +static const char *const msgpack_type_names[] = { + [kMPNil] = "nil", + [kMPBoolean] = "boolean", + [kMPInteger] = "integer", + [kMPFloat] = "float", + [kMPString] = "string", + [kMPBinary] = "binary", + [kMPArray] = "array", + [kMPMap] = "map", + [kMPExt] = "ext", +}; +static const list_T *msgpack_type_lists[] = { + [kMPNil] = NULL, + [kMPBoolean] = NULL, + [kMPInteger] = NULL, + [kMPFloat] = NULL, + [kMPString] = NULL, + [kMPBinary] = NULL, + [kMPArray] = NULL, + [kMPMap] = NULL, + [kMPExt] = NULL, +}; + /* * Initialize the global and v: variables. */ @@ -511,6 +579,26 @@ void eval_init(void) /* add to compat scope dict */ hash_add(&compat_hashtab, p->vv_di.di_key); } + + dict_T *const msgpack_types_dict = dict_alloc(); + for (size_t i = 0; i < ARRAY_SIZE(msgpack_type_names); i++) { + list_T *const type_list = list_alloc(); + type_list->lv_lock = VAR_FIXED; + dictitem_T *const di = dictitem_alloc((char_u *) msgpack_type_names[i]); + di->di_flags = DI_FLAGS_RO | DI_FLAGS_FIX; + di->di_tv = (typval_T) { + .v_type = VAR_LIST, + .vval = { .v_list = type_list, }, + }; + msgpack_type_lists[i] = type_list; + if (dict_add(msgpack_types_dict, di) == FAIL) { + // There must not be duplicate items in this dictionary by definition. + assert(false); + } + } + msgpack_types_dict->dv_lock = VAR_FIXED; + + set_vim_var_dict(VV_MSGPACK_TYPES, msgpack_types_dict); set_vim_var_dict(VV_COMPLETED_ITEM, dict_alloc()); set_vim_var_nr(VV_SEARCHFORWARD, 1L); set_vim_var_nr(VV_HLSEARCH, 1L); @@ -5156,24 +5244,40 @@ void list_append_dict(list_T *list, dict_T *dict) ++dict->dv_refcount; } -/* - * Make a copy of "str" and append it as an item to list "l". - * When "len" >= 0 use "str[len]". - */ -void list_append_string(list_T *l, char_u *str, int len) +/// Make a copy of "str" and append it as an item to list "l" +/// +/// @param[out] l List to append to. +/// @param[in] str String to append. +/// @param[in] len Length of the appended string. May be negative, in this +/// case string is considered to be usual zero-terminated +/// string. +void list_append_string(list_T *l, const char_u *str, int len) + FUNC_ATTR_NONNULL_ARG(1) +{ + if (str == NULL) { + list_append_allocated_string(l, NULL); + } else { + list_append_allocated_string(l, (len >= 0 + ? xmemdupz((char *) str, len) + : xstrdup((char *) str))); + } +} + +/// Append given string to the list +/// +/// Unlike list_append_string this function does not copy the string. +/// +/// @param[out] l List to append to. +/// @param[in] str String to append. +void list_append_allocated_string(list_T *l, char *const str) + FUNC_ATTR_NONNULL_ARG(1) { listitem_T *li = listitem_alloc(); list_append(l, li); li->li_tv.v_type = VAR_STRING; li->li_tv.v_lock = 0; - - if (str == NULL) { - li->li_tv.vval.v_string = NULL; - } else { - li->li_tv.vval.v_string = (len >= 0) ? vim_strnsave(str, len) - : vim_strsave(str); - } + li->li_tv.vval.v_string = (char_u *) str; } /* @@ -6582,6 +6686,8 @@ static struct fst { {"min", 1, 1, f_min}, {"mkdir", 1, 3, f_mkdir}, {"mode", 0, 1, f_mode}, + {"msgpackdump", 1, 1, f_msgpackdump}, + {"msgpackparse", 1, 1, f_msgpackparse}, {"nextnonblank", 1, 1, f_nextnonblank}, {"nr2char", 1, 2, f_nr2char}, {"or", 2, 2, f_or}, @@ -11782,6 +11888,827 @@ static void f_mode(typval_T *argvars, typval_T *rettv) rettv->v_type = VAR_STRING; } +/// Msgpack callback for writing to readfile()-style list +static int msgpack_list_write(void *data, const char *buf, size_t len) +{ + if (len == 0) { + return 0; + } + list_T *const list = (list_T *) data; + const char *const end = buf + len; + const char *line_end = buf; + if (list->lv_last == NULL) { + list_append_string(list, NULL, 0); + } + listitem_T *li = list->lv_last; + do { + const char *line_start = line_end; + line_end = xmemscan(line_start, NL, end - line_start); + if (line_end == line_start) { + list_append_allocated_string(list, NULL); + } else { + const size_t line_length = line_end - line_start; + char *str; + if (li == NULL) { + str = xmemdupz(line_start, line_length); + } else { + const size_t li_len = (li->li_tv.vval.v_string == NULL + ? 0 + : STRLEN(li->li_tv.vval.v_string)); + li->li_tv.vval.v_string = xrealloc(li->li_tv.vval.v_string, + li_len + line_length + 1); + str = (char *) li->li_tv.vval.v_string + li_len; + memmove(str, line_start, line_length); + str[line_length] = 0; + } + for (size_t i = 0; i < line_length; i++) { + if (str[i] == NUL) { + str[i] = NL; + } + } + if (li == NULL) { + list_append_allocated_string(list, str); + } else { + li = NULL; + } + if (line_end == end - 1) { + list_append_allocated_string(list, NULL); + } + } + line_end++; + } while (line_end < end); + return 0; +} + +/// Convert readfile()-style list to a char * buffer with length +/// +/// @param[in] list Converted list. +/// @param[out] ret_len Resulting buffer length. +/// @param[out] ret_buf Allocated buffer with the result or NULL if ret_len is +/// zero. +/// +/// @return true in case of success, false in case of failure. +static inline bool vim_list_to_buf(const list_T *const list, + size_t *const ret_len, char **const ret_buf) + FUNC_ATTR_NONNULL_ARG(2,3) FUNC_ATTR_WARN_UNUSED_RESULT +{ + size_t len = 0; + if (list != NULL) { + for (const listitem_T *li = list->lv_first; + li != NULL; + li = li->li_next) { + if (li->li_tv.v_type != VAR_STRING) { + return false; + } + len++; + if (li->li_tv.vval.v_string != 0) { + len += STRLEN(li->li_tv.vval.v_string); + } + } + if (len) { + len--; + } + } + *ret_len = len; + if (len == 0) { + *ret_buf = NULL; + return true; + } + ListReaderState lrstate = init_lrstate(list); + char *const buf = xmalloc(len); + size_t read_bytes; + if (read_from_list(&lrstate, buf, len, &read_bytes) != OK) { + assert(false); + } + assert(len == read_bytes); + *ret_buf = buf; + return true; +} + +/// Convert one VimL value to msgpack +/// +/// @param packer Messagepack packer. +/// @param[out] mpstack Stack with values to convert. Only used for pushing +/// values to it. +/// @param[in] tv Converted value. +/// +/// @return OK in case of success, FAIL otherwise. +static int convert_one_value(msgpack_packer *const packer, + MPConvStack *const mpstack, + const typval_T *const tv) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT +{ + switch (tv->v_type) { +#define CHECK_SELF_REFERENCE(conv_type, vval_name, ptr) \ + do { \ + for (size_t i = 0; i < kv_size(*mpstack); i++) { \ + if (kv_A(*mpstack, i).type == conv_type \ + && kv_A(*mpstack, i).data.vval_name == ptr) { \ + EMSG2(_(e_invarg2), "container references itself"); \ + return FAIL; \ + } \ + } \ + } while (0) + case VAR_STRING: { + if (tv->vval.v_string == NULL) { + msgpack_pack_bin(packer, 0); + } else { + const size_t len = STRLEN(tv->vval.v_string); + msgpack_pack_bin(packer, len); + msgpack_pack_bin_body(packer, tv->vval.v_string, len); + } + break; + } + case VAR_NUMBER: { + msgpack_pack_int64(packer, (int64_t) tv->vval.v_number); + break; + } + case VAR_FLOAT: { + msgpack_pack_double(packer, (double) tv->vval.v_float); + break; + } + case VAR_FUNC: { + EMSG2(_(e_invarg2), "attempt to dump function reference"); + return FAIL; + } + case VAR_LIST: { + if (tv->vval.v_list == NULL) { + msgpack_pack_array(packer, 0); + break; + } + CHECK_SELF_REFERENCE(kMPConvList, l.list, tv->vval.v_list); + msgpack_pack_array(packer, tv->vval.v_list->lv_len); + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { + .type = kMPConvList, + .data = { + .l = { + .list = tv->vval.v_list, + .li = tv->vval.v_list->lv_first, + }, + }, + })); + break; + } + case VAR_DICT: { + if (tv->vval.v_dict == NULL) { + msgpack_pack_map(packer, 0); + break; + } + const dictitem_T *type_di; + const dictitem_T *val_di; + if (tv->vval.v_dict->dv_hashtab.ht_used == 2 + && (type_di = dict_find((dict_T *) tv->vval.v_dict, + (char_u *) "_TYPE", -1)) != NULL + && type_di->di_tv.v_type == VAR_LIST + && (val_di = dict_find((dict_T *) tv->vval.v_dict, + (char_u *) "_VAL", -1)) != NULL) { + size_t i; + for (i = 0; i < ARRAY_SIZE(msgpack_type_lists); i++) { + if (type_di->di_tv.vval.v_list == msgpack_type_lists[i]) { + break; + } + } + if (i == ARRAY_SIZE(msgpack_type_lists)) { + goto vim_to_msgpack_regural_dict; + } + switch ((MessagePackType) i) { + case kMPNil: { + msgpack_pack_nil(packer); + break; + } + case kMPBoolean: { + if (val_di->di_tv.v_type != VAR_NUMBER) { + goto vim_to_msgpack_regural_dict; + } + if (val_di->di_tv.vval.v_number) { + msgpack_pack_true(packer); + } else { + msgpack_pack_false(packer); + } + break; + } + case kMPInteger: { + const list_T *val_list; + varnumber_T sign; + varnumber_T highest_bits; + varnumber_T high_bits; + varnumber_T low_bits; + // List of 4 integers; first is signed (should be 1 or -1, but this + // is not checked), second is unsigned and have at most one (sign is + // -1) or two (sign is 1) non-zero bits (number of bits is not + // checked), other unsigned and have at most 31 non-zero bits + // (number of bits is not checked). + if (val_di->di_tv.v_type != VAR_LIST + || (val_list = val_di->di_tv.vval.v_list) == NULL + || val_list->lv_len != 4 + || val_list->lv_first->li_tv.v_type != VAR_NUMBER + || (sign = val_list->lv_first->li_tv.vval.v_number) == 0 + || val_list->lv_first->li_next->li_tv.v_type != VAR_NUMBER + || (highest_bits = + val_list->lv_first->li_next->li_tv.vval.v_number) < 0 + || val_list->lv_last->li_prev->li_tv.v_type != VAR_NUMBER + || (high_bits = + val_list->lv_last->li_prev->li_tv.vval.v_number) < 0 + || val_list->lv_last->li_tv.v_type != VAR_NUMBER + || (low_bits = val_list->lv_last->li_tv.vval.v_number) < 0) { + goto vim_to_msgpack_regural_dict; + } + uint64_t number = ((uint64_t) (((uint64_t) highest_bits) << 62) + | (uint64_t) (((uint64_t) high_bits) << 31) + | (uint64_t) low_bits); + if (sign > 0) { + msgpack_pack_uint64(packer, number); + } else { + msgpack_pack_int64(packer, (int64_t) (-number)); + } + break; + } + case kMPFloat: { + if (val_di->di_tv.v_type != VAR_FLOAT) { + goto vim_to_msgpack_regural_dict; + } + msgpack_pack_double(packer, val_di->di_tv.vval.v_float); + break; + } + case kMPString: + case kMPBinary: { + const bool is_string = ((MessagePackType) i == kMPString); + if (val_di->di_tv.v_type != VAR_LIST) { + goto vim_to_msgpack_regural_dict; + } + size_t len; + char *buf; + if (!vim_list_to_buf(val_di->di_tv.vval.v_list, &len, &buf)) { + goto vim_to_msgpack_regural_dict; + } + if (is_string) { + msgpack_pack_str(packer, len); + } else { + msgpack_pack_bin(packer, len); + } + if (len == 0) { + break; + } + if (is_string) { + msgpack_pack_str_body(packer, buf, len); + } else { + msgpack_pack_bin_body(packer, buf, len); + } + xfree(buf); + break; + } + case kMPArray: { + if (val_di->di_tv.v_type != VAR_LIST) { + goto vim_to_msgpack_regural_dict; + } + CHECK_SELF_REFERENCE(kMPConvList, l.list, + val_di->di_tv.vval.v_list); + msgpack_pack_array(packer, val_di->di_tv.vval.v_list->lv_len); + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { + .type = kMPConvList, + .data = { + .l = { + .list = val_di->di_tv.vval.v_list, + .li = val_di->di_tv.vval.v_list->lv_first, + }, + }, + })); + break; + } + case kMPMap: { + if (val_di->di_tv.v_type != VAR_LIST) { + goto vim_to_msgpack_regural_dict; + } + if (val_di->di_tv.vval.v_list == NULL) { + msgpack_pack_map(packer, 0); + break; + } + const list_T *const val_list = val_di->di_tv.vval.v_list; + for (const listitem_T *li = val_list->lv_first; li != NULL; + li = li->li_next) { + if (li->li_tv.v_type != VAR_LIST + || li->li_tv.vval.v_list->lv_len != 2) { + goto vim_to_msgpack_regural_dict; + } + } + CHECK_SELF_REFERENCE(kMPConvPairs, l.list, val_list); + msgpack_pack_map(packer, val_list->lv_len); + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { + .type = kMPConvPairs, + .data = { + .l = { + .list = val_list, + .li = val_list->lv_first, + }, + }, + })); + break; + } + case kMPExt: { + const list_T *val_list; + varnumber_T type; + if (val_di->di_tv.v_type != VAR_LIST + || (val_list = val_di->di_tv.vval.v_list) == NULL + || val_list->lv_len != 2 + || (val_list->lv_first->li_tv.v_type != VAR_NUMBER) + || (type = val_list->lv_first->li_tv.vval.v_number) > INT8_MAX + || type < INT8_MIN + || (val_list->lv_last->li_tv.v_type != VAR_LIST)) { + goto vim_to_msgpack_regural_dict; + } + size_t len; + char *buf; + if (!vim_list_to_buf(val_list->lv_last->li_tv.vval.v_list, + &len, &buf)) { + goto vim_to_msgpack_regural_dict; + } + msgpack_pack_ext(packer, len, (int8_t) type); + msgpack_pack_ext_body(packer, buf, len); + xfree(buf); + break; + } + } + break; + } +vim_to_msgpack_regural_dict: + CHECK_SELF_REFERENCE(kMPConvDict, d.dict, tv->vval.v_dict); + msgpack_pack_map(packer, tv->vval.v_dict->dv_hashtab.ht_used); + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { + .type = kMPConvDict, + .data = { + .d = { + .dict = tv->vval.v_dict, + .hi = tv->vval.v_dict->dv_hashtab.ht_array, + .todo = tv->vval.v_dict->dv_hashtab.ht_used, + }, + }, + })); + break; + } + } +#undef CHECK_SELF_REFERENCE + return OK; +} + +/// Convert typval_T to messagepack +static int vim_to_msgpack(msgpack_packer *const packer, + const typval_T *const tv) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + MPConvStack mpstack; + kv_init(mpstack); + if (convert_one_value(packer, &mpstack, tv) == FAIL) { + goto vim_to_msgpack_error_ret; + } + while (kv_size(mpstack)) { + MPConvStackVal *cur_mpsv = &kv_A(mpstack, kv_size(mpstack) - 1); + const typval_T *cur_tv = NULL; + switch (cur_mpsv->type) { + case kMPConvDict: { + if (!cur_mpsv->data.d.todo) { + (void) kv_pop(mpstack); + continue; + } + while (HASHITEM_EMPTY(cur_mpsv->data.d.hi)) { + cur_mpsv->data.d.hi++; + } + const dictitem_T *const di = HI2DI(cur_mpsv->data.d.hi); + cur_mpsv->data.d.todo--; + cur_mpsv->data.d.hi++; + const size_t key_len = STRLEN(&di->di_key[0]); + msgpack_pack_str(packer, key_len); + msgpack_pack_str_body(packer, &di->di_key[0], key_len); + cur_tv = &di->di_tv; + break; + } + case kMPConvList: { + if (cur_mpsv->data.l.li == NULL) { + (void) kv_pop(mpstack); + continue; + } + cur_tv = &cur_mpsv->data.l.li->li_tv; + cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next; + break; + } + case kMPConvPairs: { + if (cur_mpsv->data.l.li == NULL) { + (void) kv_pop(mpstack); + continue; + } + const list_T *const kv_pair = cur_mpsv->data.l.li->li_tv.vval.v_list; + if (convert_one_value(packer, &mpstack, &kv_pair->lv_first->li_tv) + == FAIL) { + goto vim_to_msgpack_error_ret; + } + cur_tv = &kv_pair->lv_last->li_tv; + cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next; + break; + } + } + if (convert_one_value(packer, &mpstack, cur_tv) == FAIL) { + goto vim_to_msgpack_error_ret; + } + } + kv_destroy(mpstack); + return OK; +vim_to_msgpack_error_ret: + kv_destroy(mpstack); + return FAIL; +} + +/// "msgpackdump()" function +static void f_msgpackdump(typval_T *argvars, typval_T *rettv) + FUNC_ATTR_NONNULL_ALL +{ + if (argvars[0].v_type != VAR_LIST) { + EMSG2(_(e_listarg), "msgpackdump()"); + return; + } + list_T *ret_list = rettv_list_alloc(rettv); + const list_T *list = argvars[0].vval.v_list; + if (list == NULL) { + return; + } + msgpack_packer *lpacker = msgpack_packer_new(ret_list, &msgpack_list_write); + for (const listitem_T *li = list->lv_first; li != NULL; li = li->li_next) { + if (vim_to_msgpack(lpacker, &li->li_tv) == FAIL) { + break; + } + } + msgpack_packer_free(lpacker); +} + +/// Read bytes from list +/// +/// @param[in,out] state Structure describing position in list from which +/// reading should start. Is updated to reflect position +/// at which reading ended. +/// @param[out] buf Buffer to write to. +/// @param[in] nbuf Buffer length. +/// @param[out] read_bytes Is set to amount of bytes read. +/// +/// @return OK when reading was finished, FAIL in case of error (i.e. list item +/// was not a string), NOTDONE if reading was successfull, but there are +/// more bytes to read. +static int read_from_list(ListReaderState *const state, char *const buf, + const size_t nbuf, size_t *const read_bytes) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT +{ + char *const buf_end = buf + nbuf; + char *p = buf; + while (p < buf_end) { + for (size_t i = state->offset; i < state->li_length && p < buf_end; i++) { + const char ch = state->li->li_tv.vval.v_string[state->offset++]; + *p++ = (ch == NL ? NUL : ch); + } + if (p < buf_end) { + state->li = state->li->li_next; + if (state->li == NULL) { + *read_bytes = (size_t) (p - buf); + return OK; + } + *p++ = NL; + if (state->li->li_tv.v_type != VAR_STRING) { + *read_bytes = (size_t) (p - buf); + return FAIL; + } + state->offset = 0; + state->li_length = (state->li->li_tv.vval.v_string == NULL + ? 0 + : STRLEN(state->li->li_tv.vval.v_string)); + } + } + *read_bytes = nbuf; + return (state->offset < state->li_length || state->li->li_next != NULL + ? NOTDONE + : OK); +} + +/// Initialize ListReaderState structure +static inline ListReaderState init_lrstate(const list_T *const list) + FUNC_ATTR_NONNULL_ALL +{ + return (ListReaderState) { + .li = list->lv_first, + .offset = 0, + .li_length = (list->lv_first->li_tv.vval.v_string == NULL + ? 0 + : STRLEN(list->lv_first->li_tv.vval.v_string)), + }; +} + +/// Convert msgpack object to a VimL one +static int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT +{ +#define INIT_SPECIAL_DICT(tv, type, val) \ + do { \ + dict_T *const dict = dict_alloc(); \ + dictitem_T *const type_di = dictitem_alloc((char_u *) "_TYPE"); \ + type_di->di_tv.v_type = VAR_LIST; \ + type_di->di_tv.v_lock = 0; \ + type_di->di_tv.vval.v_list = (list_T *) msgpack_type_lists[type]; \ + type_di->di_tv.vval.v_list->lv_refcount++; \ + dict_add(dict, type_di); \ + dictitem_T *const val_di = dictitem_alloc((char_u *) "_VAL"); \ + val_di->di_tv = val; \ + dict_add(dict, val_di); \ + tv->v_type = VAR_DICT; \ + dict->dv_refcount++; \ + tv->vval.v_dict = dict; \ + } while (0) + switch (mobj.type) { + case MSGPACK_OBJECT_NIL: { + INIT_SPECIAL_DICT(rettv, kMPNil, ((typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { .v_number = 0 }, + })); + break; + } + case MSGPACK_OBJECT_BOOLEAN: { + INIT_SPECIAL_DICT(rettv, kMPBoolean, + ((typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { + .v_number = (varnumber_T) mobj.via.boolean, + }, + })); + break; + } + case MSGPACK_OBJECT_POSITIVE_INTEGER: { + if (mobj.via.u64 <= VARNUMBER_MAX) { + *rettv = (typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { .v_number = (varnumber_T) mobj.via.u64 }, + }; + } else { + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPInteger, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + uint64_t n = mobj.via.u64; + list_append_number(list, 1); + list_append_number(list, (varnumber_T) ((n >> 62) & 0x3)); + list_append_number(list, (varnumber_T) ((n >> 31) & 0x7FFFFFFF)); + list_append_number(list, (varnumber_T) (n & 0x7FFFFFFF)); + } + break; + } + case MSGPACK_OBJECT_NEGATIVE_INTEGER: { + if (mobj.via.i64 >= VARNUMBER_MIN) { + *rettv = (typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { .v_number = (varnumber_T) mobj.via.i64 }, + }; + } else { + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPInteger, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + uint64_t n = -((uint64_t) mobj.via.i64); + list_append_number(list, -1); + list_append_number(list, (varnumber_T) ((n >> 62) & 0x3)); + list_append_number(list, (varnumber_T) ((n >> 31) & 0x7FFFFFFF)); + list_append_number(list, (varnumber_T) (n & 0x7FFFFFFF)); + } + break; + } + case MSGPACK_OBJECT_FLOAT: { + *rettv = (typval_T) { + .v_type = VAR_FLOAT, + .v_lock = 0, + .vval = { .v_float = mobj.via.f64 }, + }; + break; + } + case MSGPACK_OBJECT_STR: { + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPString, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + if (msgpack_list_write((void *) list, mobj.via.str.ptr, mobj.via.str.size) + == -1) { + return FAIL; + } + break; + } + case MSGPACK_OBJECT_BIN: { + if (memchr(mobj.via.bin.ptr, NUL, mobj.via.bin.size) == NULL) { + *rettv = (typval_T) { + .v_type = VAR_STRING, + .v_lock = 0, + .vval = { .v_string = xmemdupz(mobj.via.bin.ptr, mobj.via.bin.size) }, + }; + break; + } + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPBinary, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + if (msgpack_list_write((void *) list, mobj.via.bin.ptr, mobj.via.bin.size) + == -1) { + return FAIL; + } + break; + } + case MSGPACK_OBJECT_ARRAY: { + list_T *const list = list_alloc(); + list->lv_refcount++; + *rettv = (typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + }; + for (size_t i = 0; i < mobj.via.array.size; i++) { + listitem_T *const li = listitem_alloc(); + li->li_tv.v_type = VAR_UNKNOWN; + list_append(list, li); + if (msgpack_to_vim(mobj.via.array.ptr[i], &li->li_tv) == FAIL) { + return FAIL; + } + } + break; + } + case MSGPACK_OBJECT_MAP: { + for (size_t i = 0; i < mobj.via.map.size; i++) { + if (mobj.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR + || mobj.via.map.ptr[i].key.via.str.size == 0 + || memchr(mobj.via.map.ptr[i].key.via.str.ptr, NUL, + mobj.via.map.ptr[i].key.via.str.size) != NULL) { + goto msgpack_to_vim_generic_map; + } + } + dict_T *const dict = dict_alloc(); + dict->dv_refcount++; + *rettv = (typval_T) { + .v_type = VAR_DICT, + .v_lock = 0, + .vval = { .v_dict = dict }, + }; + for (size_t i = 0; i < mobj.via.map.size; i++) { + dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key) + + mobj.via.map.ptr[i].key.via.str.size); + memcpy(&di->di_key[0], mobj.via.map.ptr[i].key.via.str.ptr, + mobj.via.map.ptr[i].key.via.str.size); + di->di_tv.v_type = VAR_UNKNOWN; + if (dict_add(dict, di) == FAIL) { + // Duplicate key: fallback to generic map + clear_tv(rettv); + xfree(di); + goto msgpack_to_vim_generic_map; + } + if (msgpack_to_vim(mobj.via.map.ptr[i].val, &di->di_tv) == FAIL) { + return FAIL; + } + } + break; +msgpack_to_vim_generic_map: {} + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPMap, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + for (size_t i = 0; i < mobj.via.map.size; i++) { + list_T *const kv_pair = list_alloc(); + list_append_list(list, kv_pair); + listitem_T *const key_li = listitem_alloc(); + key_li->li_tv.v_type = VAR_UNKNOWN; + list_append(kv_pair, key_li); + listitem_T *const val_li = listitem_alloc(); + val_li->li_tv.v_type = VAR_UNKNOWN; + list_append(kv_pair, val_li); + if (msgpack_to_vim(mobj.via.map.ptr[i].key, &key_li->li_tv) == FAIL) { + return FAIL; + } + if (msgpack_to_vim(mobj.via.map.ptr[i].val, &val_li->li_tv) == FAIL) { + return FAIL; + } + } + break; + } + case MSGPACK_OBJECT_EXT: { + list_T *const list = list_alloc(); + list->lv_refcount++; + list_append_number(list, mobj.via.ext.type); + list_T *const ext_val_list = list_alloc(); + list_append_list(list, ext_val_list); + INIT_SPECIAL_DICT(rettv, kMPExt, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + if (msgpack_list_write((void *) ext_val_list, mobj.via.ext.ptr, + mobj.via.ext.size) == -1) { + return FAIL; + } + break; + } + } +#undef INIT_SPECIAL_DICT + return OK; +} + +/// "msgpackparse" function +static void f_msgpackparse(typval_T *argvars, typval_T *rettv) + FUNC_ATTR_NONNULL_ALL +{ + if (argvars[0].v_type != VAR_LIST) { + EMSG2(_(e_listarg), "msgpackparse()"); + } + list_T *ret_list = rettv_list_alloc(rettv); + const list_T *list = argvars[0].vval.v_list; + if (list == NULL || list->lv_first == NULL) { + return; + } + if (list->lv_first->li_tv.v_type != VAR_STRING) { + EMSG2(_(e_invarg2), "List item is not a string"); + return; + } + ListReaderState lrstate = init_lrstate(list); + msgpack_unpacker *const unpacker = msgpack_unpacker_new(IOSIZE); + if (unpacker == NULL) { + EMSG(_(e_outofmem)); + return; + } + msgpack_unpacked unpacked; + msgpack_unpacked_init(&unpacked); + do { + if (!msgpack_unpacker_reserve_buffer(unpacker, IOSIZE)) { + EMSG(_(e_outofmem)); + goto f_msgpackparse_exit; + } + size_t read_bytes; + const int rlret = read_from_list( + &lrstate, msgpack_unpacker_buffer(unpacker), IOSIZE, &read_bytes); + if (rlret == FAIL) { + EMSG2(_(e_invarg2), "List item is not a string"); + goto f_msgpackparse_exit; + } + msgpack_unpacker_buffer_consumed(unpacker, read_bytes); + if (read_bytes == 0) { + break; + } + while (unpacker->off < unpacker->used) { + const msgpack_unpack_return result = msgpack_unpacker_next(unpacker, + &unpacked); + if (result == MSGPACK_UNPACK_PARSE_ERROR) { + EMSG2(_(e_invarg2), "Failed to parse msgpack string"); + goto f_msgpackparse_exit; + } + if (result == MSGPACK_UNPACK_NOMEM_ERROR) { + EMSG(_(e_outofmem)); + goto f_msgpackparse_exit; + } + if (result == MSGPACK_UNPACK_SUCCESS) { + listitem_T *li = listitem_alloc(); + li->li_tv.v_type = VAR_UNKNOWN; + list_append(ret_list, li); + if (msgpack_to_vim(unpacked.data, &li->li_tv) == FAIL) { + EMSG2(_(e_invarg2), "Failed to convert msgpack string"); + goto f_msgpackparse_exit; + } + } + if (result == MSGPACK_UNPACK_CONTINUE) { + if (rlret == OK) { + EMSG2(_(e_invarg2), "Incomplete msgpack string"); + } + break; + } + } + if (rlret == OK) { + break; + } + } while (true); + +f_msgpackparse_exit: + msgpack_unpacked_destroy(&unpacked); + msgpack_unpacker_free(unpacker); + return; +} /* * "nextnonblank()" function diff --git a/src/nvim/eval.h b/src/nvim/eval.h index bf48bc74b8..8f065eda33 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -65,6 +65,7 @@ enum { VV_PROGPATH, VV_COMMAND_OUTPUT, VV_COMPLETED_ITEM, + VV_MSGPACK_TYPES, VV_LEN, /* number of v: vars */ }; diff --git a/src/nvim/eval_defs.h b/src/nvim/eval_defs.h index 34a36004d6..0faf860588 100644 --- a/src/nvim/eval_defs.h +++ b/src/nvim/eval_defs.h @@ -1,11 +1,16 @@ #ifndef NVIM_EVAL_DEFS_H #define NVIM_EVAL_DEFS_H +#include <limits.h> + #include "nvim/hashtab.h" typedef int varnumber_T; typedef double float_T; +#define VARNUMBER_MAX INT_MAX +#define VARNUMBER_MIN INT_MIN + typedef struct listvar_S list_T; typedef struct dictvar_S dict_T; diff --git a/src/nvim/ex_cmds.c b/src/nvim/ex_cmds.c index 2c92113404..cacef01b19 100644 --- a/src/nvim/ex_cmds.c +++ b/src/nvim/ex_cmds.c @@ -6125,7 +6125,7 @@ char_u * sign_typenr2name(int typenr) /* * Undefine/free all signs. */ -void free_signs() +void free_signs(void) { while (first_sign != NULL) sign_undefine(first_sign, NULL); diff --git a/src/nvim/ex_cmds2.c b/src/nvim/ex_cmds2.c index d610aa6d19..efa18aa681 100644 --- a/src/nvim/ex_cmds2.c +++ b/src/nvim/ex_cmds2.c @@ -2624,7 +2624,7 @@ char_u *get_scriptname(scid_T id) } # if defined(EXITFREE) -void free_scriptnames() +void free_scriptnames(void) { # define FREE_SCRIPTNAME(item) xfree((item)->sn_name) GA_DEEP_CLEAR(&script_items, scriptitem_T, FREE_SCRIPTNAME); diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua new file mode 100644 index 0000000000..3b349f1f72 --- /dev/null +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -0,0 +1,606 @@ +local helpers = require('test.functional.helpers') +local clear, feed, execute = helpers.clear, helpers.feed, helpers.execute +local eval, eq, neq = helpers.eval, helpers.eq, helpers.neq +local execute, source = helpers.execute, helpers.source +local nvim = helpers.nvim +describe('msgpack*() functions', function() + before_each(function() + clear() + end) + local obj_test = function(msg, obj) + it(msg, function() + nvim('set_var', 'obj', obj) + eq(obj, eval('msgpackparse(msgpackdump(g:obj))')) + end) + end + -- Regression test: msgpack_list_write was failing to write buffer with zero + -- length. + obj_test('are able to dump and restore {"file": ""}', {{file=''}}) + -- Regression test: msgpack_list_write was failing to write buffer with NL at + -- the end. + obj_test('are able to dump and restore {0, "echo mpack"}', {{0, 'echo mpack'}}) + obj_test('are able to dump and restore "Test\\n"', {'Test\n'}) + -- Regression test: msgpack_list_write was failing to write buffer with NL + -- inside. + obj_test('are able to dump and restore "Test\\nTest 2"', {'Test\nTest 2'}) + -- Test that big objects (requirement: dump to something that is bigger then + -- IOSIZE) are also fine. This particular object is obtained by concatenating + -- 5 identical shada files. + local big_obj = { + 1, 1436711454, 78, { + encoding="utf-8", + max_kbyte=10, + pid=19269, + version="NVIM 0.0.0-alpha+201507121634" + }, + 8, 1436711451, 40, { file="/home/zyx/.nvim/shada/main.shada" }, + 8, 1436711391, 8, { file="" }, + 4, 1436700940, 30, { 0, "call mkdir('/tmp/tty/tty')" }, + 4, 1436701355, 35, { 0, "call mkdir('/tmp/tty/tty', 'p')" }, + 4, 1436701368, 24, { 0, "call mkdir('/', 'p')" }, + 4, 1436701375, 26, { 0, "call mkdir('/tty/tty')" }, + 4, 1436701383, 30, { 0, "call mkdir('/tty/tty/tty')" }, + 4, 1436701407, 35, { 0, "call mkdir('/usr/tty/tty', 'p')" }, + 4, 1436701666, 35, { 0, "call mkdir('/tty/tty/tty', 'p')" }, + 4, 1436708101, 25, { 0, "echo msgpackdump([1])" }, + 4, 1436708966, 6, { 0, "cq" }, + 4, 1436709606, 25, { 0, "echo msgpackdump([5])" }, + 4, 1436709610, 26, { 0, "echo msgpackdump([10])" }, + 4, 1436709615, 31, { 0, "echo msgpackdump([5, 5, 5])" }, + 4, 1436709618, 35, { 0, "echo msgpackdump([5, 5, 5, 10])" }, + 4, 1436709634, 57, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1}]])" + }, + 4, 1436709651, 67, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1, \"def\": 0}]])" + }, + 4, 1436709660, 70, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1, \"def\": 0}], 0])" + }, + 4, 1436710095, 29, { 0, "echo msgpackparse([\"\\n\"])" }, + 4, 1436710100, 28, { 0, "echo msgpackparse([\"j\"])" }, + 4, 1436710109, 31, { 0, "echo msgpackparse([\"\", \"\"])" }, + 4, 1436710424, 33, { 0, "echo msgpackparse([\"\", \"\\n\"])" }, + 4, 1436710428, 32, { 0, "echo msgpackparse([\"\", \"j\"])" }, + 4, 1436711142, 14, { 0, "echo mpack" }, + 4, 1436711196, 45, { 0, "let lengths = map(mpack[:], 'len(v:val)')" }, + 4, 1436711206, 16, { 0, "echo lengths" }, + 4, 1436711244, 92, { + 0, + ("let sum = len(lengths) - 1 | call map(copy(lengths), " + .. "'extend(g:, {\"sum\": sum + v:val})')") + }, + 4, 1436711245, 12, { 0, "echo sum" }, + 4, 1436711398, 10, { 0, "echo s" }, + 4, 1436711404, 41, { 0, "let mpack = readfile('/tmp/foo', 'b')" }, + 4, 1436711408, 41, { 0, "let shada_objects=msgpackparse(mpack)" }, + 4, 1436711415, 22, { 0, "echo shada_objects" }, + 4, 1436711451, 30, { 0, "e ~/.nvim/shada/main.shada" }, + 4, 1436711454, 6, { 0, "qa" }, + 4, 1436711442, 9, { 1, "test", 47 }, + 4, 1436711443, 15, { 1, "aontsuesan", 47 }, + 2, 1436711443, 38, { hlsearch=1, pat="aontsuesan", smartcase=1 }, + 2, 0, 31, { islast=0, pat="", smartcase=1, sub=1 }, + 3, 0, 3, { "" }, + 10, 1436711451, 40, { file="/home/zyx/.nvim/shada/main.shada" }, + 1, 1436711454, 78, { + encoding="utf-8", + max_kbyte=10, + pid=19269, + version="NVIM 0.0.0-alpha+201507121634" + }, + 8, 1436711451, 40, { file="/home/zyx/.nvim/shada/main.shada" }, + 8, 1436711391, 8, { file="" }, + 4, 1436700940, 30, { 0, "call mkdir('/tmp/tty/tty')" }, + 4, 1436701355, 35, { 0, "call mkdir('/tmp/tty/tty', 'p')" }, + 4, 1436701368, 24, { 0, "call mkdir('/', 'p')" }, + 4, 1436701375, 26, { 0, "call mkdir('/tty/tty')" }, + 4, 1436701383, 30, { 0, "call mkdir('/tty/tty/tty')" }, + 4, 1436701407, 35, { 0, "call mkdir('/usr/tty/tty', 'p')" }, + 4, 1436701666, 35, { 0, "call mkdir('/tty/tty/tty', 'p')" }, + 4, 1436708101, 25, { 0, "echo msgpackdump([1])" }, + 4, 1436708966, 6, { 0, "cq" }, + 4, 1436709606, 25, { 0, "echo msgpackdump([5])" }, + 4, 1436709610, 26, { 0, "echo msgpackdump([10])" }, + 4, 1436709615, 31, { 0, "echo msgpackdump([5, 5, 5])" }, + 4, 1436709618, 35, { 0, "echo msgpackdump([5, 5, 5, 10])" }, + 4, 1436709634, 57, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1}]])" + }, + 4, 1436709651, 67, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1, \"def\": 0}]])" + }, + 4, 1436709660, 70, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1, \"def\": 0}], 0])" + }, + 4, 1436710095, 29, { 0, "echo msgpackparse([\"\\n\"])" }, + 4, 1436710100, 28, { 0, "echo msgpackparse([\"j\"])" }, + 4, 1436710109, 31, { 0, "echo msgpackparse([\"\", \"\"])" }, + 4, 1436710424, 33, { 0, "echo msgpackparse([\"\", \"\\n\"])" }, + 4, 1436710428, 32, { 0, "echo msgpackparse([\"\", \"j\"])" }, + 4, 1436711142, 14, { 0, "echo mpack" }, + 4, 1436711196, 45, { 0, "let lengths = map(mpack[:], 'len(v:val)')" }, + 4, 1436711206, 16, { 0, "echo lengths" }, + 4, 1436711244, 92, { + 0, + ("let sum = len(lengths) - 1 | call map(copy(lengths), " + .. "'extend(g:, {\"sum\": sum + v:val})')") + }, + 4, 1436711245, 12, { 0, "echo sum" }, + 4, 1436711398, 10, { 0, "echo s" }, + 4, 1436711404, 41, { 0, "let mpack = readfile('/tmp/foo', 'b')" }, + 4, 1436711408, 41, { 0, "let shada_objects=msgpackparse(mpack)" }, + 4, 1436711415, 22, { 0, "echo shada_objects" }, + 4, 1436711451, 30, { 0, "e ~/.nvim/shada/main.shada" }, + 4, 1436711454, 6, { 0, "qa" }, + 4, 1436711442, 9, { 1, "test", 47 }, + 4, 1436711443, 15, { 1, "aontsuesan", 47 }, + 2, 1436711443, 38, { hlsearch=1, pat="aontsuesan", smartcase=1 }, + 2, 0, 31, { islast=0, pat="", smartcase=1, sub=1 }, + 3, 0, 3, { "" }, + 10, 1436711451, 40, { file="/home/zyx/.nvim/shada/main.shada" }, + 1, 1436711454, 78, { + encoding="utf-8", + max_kbyte=10, + pid=19269, + version="NVIM 0.0.0-alpha+201507121634" + }, + 8, 1436711451, 40, { file="/home/zyx/.nvim/shada/main.shada" }, + 8, 1436711391, 8, { file="" }, + 4, 1436700940, 30, { 0, "call mkdir('/tmp/tty/tty')" }, + 4, 1436701355, 35, { 0, "call mkdir('/tmp/tty/tty', 'p')" }, + 4, 1436701368, 24, { 0, "call mkdir('/', 'p')" }, + 4, 1436701375, 26, { 0, "call mkdir('/tty/tty')" }, + 4, 1436701383, 30, { 0, "call mkdir('/tty/tty/tty')" }, + 4, 1436701407, 35, { 0, "call mkdir('/usr/tty/tty', 'p')" }, + 4, 1436701666, 35, { 0, "call mkdir('/tty/tty/tty', 'p')" }, + 4, 1436708101, 25, { 0, "echo msgpackdump([1])" }, + 4, 1436708966, 6, { 0, "cq" }, + 4, 1436709606, 25, { 0, "echo msgpackdump([5])" }, + 4, 1436709610, 26, { 0, "echo msgpackdump([10])" }, + 4, 1436709615, 31, { 0, "echo msgpackdump([5, 5, 5])" }, + 4, 1436709618, 35, { 0, "echo msgpackdump([5, 5, 5, 10])" }, + 4, 1436709634, 57, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1}]])" + }, + 4, 1436709651, 67, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1, \"def\": 0}]])" + }, + 4, 1436709660, 70, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1, \"def\": 0}], 0])" + }, + 4, 1436710095, 29, { 0, "echo msgpackparse([\"\\n\"])" }, + 4, 1436710100, 28, { 0, "echo msgpackparse([\"j\"])" }, + 4, 1436710109, 31, { 0, "echo msgpackparse([\"\", \"\"])" }, + 4, 1436710424, 33, { 0, "echo msgpackparse([\"\", \"\\n\"])" }, + 4, 1436710428, 32, { 0, "echo msgpackparse([\"\", \"j\"])" }, + 4, 1436711142, 14, { 0, "echo mpack" }, + 4, 1436711196, 45, { 0, "let lengths = map(mpack[:], 'len(v:val)')" }, + 4, 1436711206, 16, { 0, "echo lengths" }, + 4, 1436711244, 92, { + 0, + ("let sum = len(lengths) - 1 | call map(copy(lengths), " + .. "'extend(g:, {\"sum\": sum + v:val})')") + }, + 4, 1436711245, 12, { 0, "echo sum" }, + 4, 1436711398, 10, { 0, "echo s" }, + 4, 1436711404, 41, { 0, "let mpack = readfile('/tmp/foo', 'b')" }, + 4, 1436711408, 41, { 0, "let shada_objects=msgpackparse(mpack)" }, + 4, 1436711415, 22, { 0, "echo shada_objects" }, + 4, 1436711451, 30, { 0, "e ~/.nvim/shada/main.shada" }, + 4, 1436711454, 6, { 0, "qa" }, + 4, 1436711442, 9, { 1, "test", 47 }, + 4, 1436711443, 15, { 1, "aontsuesan", 47 }, + 2, 1436711443, 38, { hlsearch=1, pat="aontsuesan", smartcase=1 }, + 2, 0, 31, { islast=0, pat="", smartcase=1, sub=1 }, + 3, 0, 3, { "" }, + 10, 1436711451, 40, { file="/home/zyx/.nvim/shada/main.shada" }, + 1, 1436711454, 78, { + encoding="utf-8", + max_kbyte=10, + pid=19269, + version="NVIM 0.0.0-alpha+201507121634" + }, + 8, 1436711451, 40, { file="/home/zyx/.nvim/shada/main.shada" }, + 8, 1436711391, 8, { file="" }, + 4, 1436700940, 30, { 0, "call mkdir('/tmp/tty/tty')" }, + 4, 1436701355, 35, { 0, "call mkdir('/tmp/tty/tty', 'p')" }, + 4, 1436701368, 24, { 0, "call mkdir('/', 'p')" }, + 4, 1436701375, 26, { 0, "call mkdir('/tty/tty')" }, + 4, 1436701383, 30, { 0, "call mkdir('/tty/tty/tty')" }, + 4, 1436701407, 35, { 0, "call mkdir('/usr/tty/tty', 'p')" }, + 4, 1436701666, 35, { 0, "call mkdir('/tty/tty/tty', 'p')" }, + 4, 1436708101, 25, { 0, "echo msgpackdump([1])" }, + 4, 1436708966, 6, { 0, "cq" }, + 4, 1436709606, 25, { 0, "echo msgpackdump([5])" }, + 4, 1436709610, 26, { 0, "echo msgpackdump([10])" }, + 4, 1436709615, 31, { 0, "echo msgpackdump([5, 5, 5])" }, + 4, 1436709618, 35, { 0, "echo msgpackdump([5, 5, 5, 10])" }, + 4, 1436709634, 57, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1}]])" + }, + 4, 1436709651, 67, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1, \"def\": 0}]])" + }, + 4, 1436709660, 70, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1, \"def\": 0}], 0])" + }, + 4, 1436710095, 29, { 0, "echo msgpackparse([\"\\n\"])" }, + 4, 1436710100, 28, { 0, "echo msgpackparse([\"j\"])" }, + 4, 1436710109, 31, { 0, "echo msgpackparse([\"\", \"\"])" }, + 4, 1436710424, 33, { 0, "echo msgpackparse([\"\", \"\\n\"])" }, + 4, 1436710428, 32, { 0, "echo msgpackparse([\"\", \"j\"])" }, + 4, 1436711142, 14, { 0, "echo mpack" }, + 4, 1436711196, 45, { 0, "let lengths = map(mpack[:], 'len(v:val)')" }, + 4, 1436711206, 16, { 0, "echo lengths" }, + 4, 1436711244, 92, { + 0, + ("let sum = len(lengths) - 1 | call map(copy(lengths), " + .. "'extend(g:, {\"sum\": sum + v:val})')") + }, + 4, 1436711245, 12, { 0, "echo sum" }, + 4, 1436711398, 10, { 0, "echo s" }, + 4, 1436711404, 41, { 0, "let mpack = readfile('/tmp/foo', 'b')" }, + 4, 1436711408, 41, { 0, "let shada_objects=msgpackparse(mpack)" }, + 4, 1436711415, 22, { 0, "echo shada_objects" }, + 4, 1436711451, 30, { 0, "e ~/.nvim/shada/main.shada" }, + 4, 1436711454, 6, { 0, "qa" }, + 4, 1436711442, 9, { 1, "test", 47 }, + 4, 1436711443, 15, { 1, "aontsuesan", 47 }, + 2, 1436711443, 38, { hlsearch=1, pat="aontsuesan", smartcase=1 }, + 2, 0, 31, { islast=0, pat="", smartcase=1, sub=1 }, + 3, 0, 3, { "" }, + 10, 1436711451, 40, { file="/home/zyx/.nvim/shada/main.shada" }, + 1, 1436711454, 78, { + encoding="utf-8", + max_kbyte=10, + pid=19269, + version="NVIM 0.0.0-alpha+201507121634" + }, + 8, 1436711451, 40, { file="/home/zyx/.nvim/shada/main.shada" }, + 8, 1436711391, 8, { file="" }, + 4, 1436700940, 30, { 0, "call mkdir('/tmp/tty/tty')" }, + 4, 1436701355, 35, { 0, "call mkdir('/tmp/tty/tty', 'p')" }, + 4, 1436701368, 24, { 0, "call mkdir('/', 'p')" }, + 4, 1436701375, 26, { 0, "call mkdir('/tty/tty')" }, + 4, 1436701383, 30, { 0, "call mkdir('/tty/tty/tty')" }, + 4, 1436701407, 35, { 0, "call mkdir('/usr/tty/tty', 'p')" }, + 4, 1436701666, 35, { 0, "call mkdir('/tty/tty/tty', 'p')" }, + 4, 1436708101, 25, { 0, "echo msgpackdump([1])" }, + 4, 1436708966, 6, { 0, "cq" }, + 4, 1436709606, 25, { 0, "echo msgpackdump([5])" }, + 4, 1436709610, 26, { 0, "echo msgpackdump([10])" }, + 4, 1436709615, 31, { 0, "echo msgpackdump([5, 5, 5])" }, + 4, 1436709618, 35, { 0, "echo msgpackdump([5, 5, 5, 10])" }, + 4, 1436709634, 57, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1}]])" + }, + 4, 1436709651, 67, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1, \"def\": 0}]])" + }, + 4, 1436709660, 70, { + 0, + "echo msgpackdump([5, 5, 5, 10, [10, 20, {\"abc\": 1, \"def\": 0}], 0])" + }, + 4, 1436710095, 29, { 0, "echo msgpackparse([\"\\n\"])" }, + 4, 1436710100, 28, { 0, "echo msgpackparse([\"j\"])" }, + 4, 1436710109, 31, { 0, "echo msgpackparse([\"\", \"\"])" }, + 4, 1436710424, 33, { 0, "echo msgpackparse([\"\", \"\\n\"])" }, + 4, 1436710428, 32, { 0, "echo msgpackparse([\"\", \"j\"])" }, + 4, 1436711142, 14, { 0, "echo mpack" }, + 4, 1436711196, 45, { 0, "let lengths = map(mpack[:], 'len(v:val)')" }, + 4, 1436711206, 16, { 0, "echo lengths" }, + 4, 1436711244, 92, { + 0, + ("let sum = len(lengths) - 1 | call map(copy(lengths), " + .. "'extend(g:, {\"sum\": sum + v:val})')") + }, + 4, 1436711245, 12, { 0, "echo sum" }, + 4, 1436711398, 10, { 0, "echo s" }, + 4, 1436711404, 41, { 0, "let mpack = readfile('/tmp/foo', 'b')" }, + 4, 1436711408, 41, { 0, "let shada_objects=msgpackparse(mpack)" }, + 4, 1436711415, 22, { 0, "echo shada_objects" }, + 4, 1436711451, 30, { 0, "e ~/.nvim/shada/main.shada" }, + 4, 1436711454, 6, { 0, "qa" }, + 4, 1436711442, 9, { 1, "test", 47 }, + 4, 1436711443, 15, { 1, "aontsuesan", 47 }, + 2, 1436711443, 38, { hlsearch=1, pat="aontsuesan", smartcase=1 }, + 2, 0, 31, { islast=0, pat="", smartcase=1, sub=1 }, + 3, 0, 3, { "" }, + 10, 1436711451, 40, { file="/home/zyx/.nvim/shada/main.shada" } + } + obj_test('are able to dump and restore rather big object', big_obj) + + obj_test('are able to dump and restore floating-point value', {0.125}) + + it('restore nil as special dict', function() + execute('let dumped = ["\\xC0"]') + execute('let parsed = msgpackparse(dumped)') + eq({{_TYPE={}, _VAL=0}}, eval('parsed')) + eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.nil')) + end) + + it('restore boolean false as zero', function() + execute('let dumped = ["\\xC2"]') + execute('let parsed = msgpackparse(dumped)') + eq({{_TYPE={}, _VAL=0}}, eval('parsed')) + eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.boolean')) + end) + + it('restore boolean true as one', function() + execute('let dumped = ["\\xC3"]') + execute('let parsed = msgpackparse(dumped)') + eq({{_TYPE={}, _VAL=1}}, eval('parsed')) + eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.boolean')) + end) + + it('dump string as BIN 8', function() + nvim('set_var', 'obj', {'Test'}) + eq({"\196\004Test"}, eval('msgpackdump(obj)')) + end) + + it('restore FIXSTR as special dict', function() + execute('let dumped = ["\\xa2ab"]') + execute('let parsed = msgpackparse(dumped)') + eq({{_TYPE={}, _VAL={'ab'}}}, eval('parsed')) + eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.string')) + end) + + it('restore BIN 8 as string', function() + execute('let dumped = ["\\xC4\\x02ab"]') + eq({'ab'}, eval('msgpackparse(dumped)')) + end) + + it('restore FIXEXT1 as special dictionary', function() + execute('let dumped = ["\\xD4\\x10", ""]') + execute('let parsed = msgpackparse(dumped)') + eq({{_TYPE={}, _VAL={0x10, {"", ""}}}}, eval('parsed')) + eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.ext')) + end) + + it('restore MAP with BIN key as special dictionary', function() + execute('let dumped = ["\\x81\\xC4\\x01a\\xC4\\n"]') + execute('let parsed = msgpackparse(dumped)') + eq({{_TYPE={}, _VAL={{'a', ''}}}}, eval('parsed')) + eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map')) + end) + + it('restore MAP with duplicate STR keys as special dictionary', function() + execute('let dumped = ["\\x82\\xA1a\\xC4\\n\\xA1a\\xC4\\n"]') + execute('let parsed = msgpackparse(dumped)') + eq({{_TYPE={}, _VAL={{{_TYPE={}, _VAL={'a'}}, ''}, + {{_TYPE={}, _VAL={'a'}}, ''}}}}, eval('parsed')) + eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map')) + eq(1, eval('g:parsed[0]._VAL[0][0]._TYPE is v:msgpack_types.string')) + eq(1, eval('g:parsed[0]._VAL[1][0]._TYPE is v:msgpack_types.string')) + end) + + it('restore MAP with MAP key as special dictionary', function() + execute('let dumped = ["\\x81\\x80\\xC4\\n"]') + execute('let parsed = msgpackparse(dumped)') + eq({{_TYPE={}, _VAL={{{}, ''}}}}, eval('parsed')) + eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map')) + end) + + it('can restore and dump UINT64_MAX', function() + execute('let dumped = ["\\xCF" . repeat("\\xFF", 8)]') + execute('let parsed = msgpackparse(dumped)') + execute('let dumped2 = msgpackdump(parsed)') + eq(1, eval('type(parsed[0]) == type(0) ' .. + '|| parsed[0]._TYPE is v:msgpack_types.integer')) + if eval('type(parsed[0]) == type(0)') == 1 then + eq(1, eval('0xFFFFFFFFFFFFFFFF == parsed[0]')) + else + eq({_TYPE={}, _VAL={1, 3, 0x7FFFFFFF, 0x7FFFFFFF}}, eval('parsed[0]')) + end + eq(1, eval('dumped ==# dumped2')) + end) + + it('can restore and dump INT64_MIN', function() + execute('let dumped = ["\\xD3\\x80" . repeat("\\n", 7)]') + execute('let parsed = msgpackparse(dumped)') + execute('let dumped2 = msgpackdump(parsed)') + eq(1, eval('type(parsed[0]) == type(0) ' .. + '|| parsed[0]._TYPE is v:msgpack_types.integer')) + if eval('type(parsed[0]) == type(0)') == 1 then + eq(1, eval('-0x8000000000000000 == parsed[0]')) + else + eq({_TYPE={}, _VAL={-1, 2, 0, 0}}, eval('parsed[0]')) + end + eq(1, eval('dumped ==# dumped2')) + end) + + it('can restore and dump BIN string with zero byte', function() + execute('let dumped = ["\\xC4\\x01\\n"]') + execute('let parsed = msgpackparse(dumped)') + execute('let dumped2 = msgpackdump(parsed)') + eq({{_TYPE={}, _VAL={'\n'}}}, eval('parsed')) + eq(1, eval('parsed[0]._TYPE is v:msgpack_types.binary')) + eq(1, eval('dumped ==# dumped2')) + end) + + it('can restore and dump STR string with zero byte', function() + execute('let dumped = ["\\xA1\\n"]') + execute('let parsed = msgpackparse(dumped)') + execute('let dumped2 = msgpackdump(parsed)') + eq({{_TYPE={}, _VAL={'\n'}}}, eval('parsed')) + eq(1, eval('parsed[0]._TYPE is v:msgpack_types.string')) + eq(1, eval('dumped ==# dumped2')) + end) + + it('can restore and dump BIN string with NL', function() + execute('let dumped = ["\\xC4\\x01", ""]') + execute('let parsed = msgpackparse(dumped)') + execute('let dumped2 = msgpackdump(parsed)') + eq({"\n"}, eval('parsed')) + eq(1, eval('dumped ==# dumped2')) + end) + + it('can dump generic mapping with generic mapping keys and values', function() + execute('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": []}') + execute('let todumpv1 = {"_TYPE": v:msgpack_types.map, "_VAL": []}') + execute('let todumpv2 = {"_TYPE": v:msgpack_types.map, "_VAL": []}') + execute('call add(todump._VAL, [todumpv1, todumpv2])') + eq({'\129\128\128'}, eval('msgpackdump([todump])')) + end) + + it('can dump generic mapping with ext', function() + execute('let todump = {"_TYPE": v:msgpack_types.ext, "_VAL": [5, ["",""]]}') + eq({'\212\005', ''}, eval('msgpackdump([todump])')) + end) + + it('can dump generic mapping with array', function() + execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": [5, [""]]}') + eq({'\146\005\145\196\n'}, eval('msgpackdump([todump])')) + end) + + it('can dump generic mapping with UINT64_MAX', function() + execute('let todump = {"_TYPE": v:msgpack_types.integer}') + execute('let todump._VAL = [1, 3, 0x7FFFFFFF, 0x7FFFFFFF]') + eq({'\207\255\255\255\255\255\255\255\255'}, eval('msgpackdump([todump])')) + end) + + it('can dump generic mapping with INT64_MIN', function() + execute('let todump = {"_TYPE": v:msgpack_types.integer}') + execute('let todump._VAL = [-1, 2, 0, 0]') + eq({'\211\128\n\n\n\n\n\n\n'}, eval('msgpackdump([todump])')) + end) + + it('dump and restore generic mapping with floating-point value', function() + execute('let todump = {"_TYPE": v:msgpack_types.float, "_VAL": 0.125}') + eq({0.125}, eval('msgpackparse(msgpackdump([todump]))')) + end) + + it('fails to dump a function reference', function() + execute('let Todump = function("tr")') + execute([[ + try + let dumped = msgpackdump([Todump]) + let exception = 0 + catch + let exception = v:exception + endtry + ]]) + eq('Vim(let):E475: Invalid argument: attempt to dump function reference', + eval('exception')) + end) + + it('fails to dump a function reference in a list', function() + execute('let todump = [function("tr")]') + execute([[ + try + let dumped = msgpackdump([todump]) + let exception = 0 + catch + let exception = v:exception + endtry + ]]) + eq('Vim(let):E475: Invalid argument: attempt to dump function reference', + eval('exception')) + end) + + it('fails to dump a recursive list', function() + execute('let todump = [[[]]]') + execute('call add(todump[0][0], todump)') + execute([[ + try + let dumped = msgpackdump([todump]) + let exception = 0 + catch + let exception = v:exception + endtry + ]]) + eq('Vim(let):E475: Invalid argument: container references itself', + eval('exception')) + end) + + it('fails to dump a recursive dict', function() + execute('let todump = {"d": {"d": {}}}') + execute('call extend(todump.d.d, {"d": todump})') + execute([[ + try + let dumped = msgpackdump([todump]) + let exception = 0 + catch + let exception = v:exception + endtry + ]]) + eq('Vim(let):E475: Invalid argument: container references itself', + eval('exception')) + end) + + it('fails to dump a recursive list in a special dict', function() + execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}') + execute('call add(todump._VAL, todump)') + execute([[ + try + let dumped = msgpackdump([todump]) + let exception = 0 + catch + let exception = v:exception + endtry + ]]) + eq('Vim(let):E475: Invalid argument: container references itself', + eval('exception')) + end) + + it('fails to dump a recursive (key) map in a special dict', function() + execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}') + execute('call add(todump._VAL, [todump, 0])') + execute([[ + try + let dumped = msgpackdump([todump]) + let exception = 0 + catch + let exception = v:exception + endtry + ]]) + eq('Vim(let):E475: Invalid argument: container references itself', + eval('exception')) + end) + + it('fails to dump a recursive (val) map in a special dict', function() + execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}') + execute('call add(todump._VAL, [0, todump])') + execute([[ + try + let dumped = msgpackdump([todump]) + let exception = 0 + catch + let exception = v:exception + endtry + ]]) + eq('Vim(let):E475: Invalid argument: container references itself', + eval('exception')) + end) + + it('fails to dump a recursive (val) special list in a special dict', + function() + execute('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": []}') + execute('call add(todump._VAL, [0, todump._VAL])') + execute([[ + try + let dumped = msgpackdump([todump]) + let exception = 0 + catch + let exception = v:exception + endtry + ]]) + eq('Vim(let):E475: Invalid argument: container references itself', + eval('exception')) + end) +end) |