diff options
Diffstat (limited to 'src/nvim/eval.c')
| -rw-r--r-- | src/nvim/eval.c | 1148 |
1 files changed, 1045 insertions, 103 deletions
diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 18188696b9..bf9a219e28 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -20,6 +20,7 @@ #include <stdbool.h> #include <math.h> #include <limits.h> +#include <msgpack.h> #include "nvim/assert.h" #include "nvim/vim.h" @@ -90,11 +91,13 @@ #include "nvim/os/time.h" #include "nvim/msgpack_rpc/channel.h" #include "nvim/msgpack_rpc/server.h" +#include "nvim/msgpack_rpc/helpers.h" #include "nvim/api/private/helpers.h" #include "nvim/api/vim.h" #include "nvim/os/dl.h" #include "nvim/os/input.h" #include "nvim/event/loop.h" +#include "nvim/lib/kvec.h" #define DICT_MAXNEST 100 /* maximum nesting of lists and dicts */ @@ -160,6 +163,13 @@ typedef struct lval_S { char_u *ll_newkey; /* New key for Dict in alloc. mem or NULL. */ } lval_T; +/// Structure defining state for read_from_list() +typedef struct { + const listitem_T *li; ///< Item currently read. + size_t offset; ///< Byte offset inside the read item. + size_t li_length; ///< Length of the string inside the read item. +} ListReaderState; + static char *e_letunexp = N_("E18: Unexpected characters in :let"); static char *e_listidx = N_("E684: list index out of range: %" PRId64); @@ -429,6 +439,7 @@ static struct vimvar { {VV_NAME("progpath", VAR_STRING), VV_RO}, {VV_NAME("command_output", VAR_STRING), 0}, {VV_NAME("completed_item", VAR_DICT), VV_RO}, + {VV_NAME("msgpack_types", VAR_DICT), VV_RO}, }; /* shorthand */ @@ -457,17 +468,32 @@ typedef struct { dict_T *self; int *status_ptr; uint64_t id; + Queue *events; } TerminalJobData; +/// Structure representing current VimL to messagepack conversion state +typedef struct { + enum { + kMPConvDict, ///< Convert dict_T *dictionary. + kMPConvList, ///< Convert list_T *list. + kMPConvPairs, ///< Convert mapping represented as a list_T* of pairs. + } type; + union { + struct { + const dict_T *dict; ///< Currently converted dictionary. + const hashitem_T *hi; ///< Currently converted dictionary item. + size_t todo; ///< Amount of items left to process. + } d; ///< State of dictionary conversion. + struct { + const list_T *list; ///< Currently converted list. + const listitem_T *li; ///< Currently converted list item. + } l; ///< State of list or generic mapping conversion. + } data; ///< Data to convert. +} MPConvStackVal; + +/// Stack used to convert VimL values to messagepack. +typedef kvec_t(MPConvStackVal) MPConvStack; -#ifdef INCLUDE_GENERATED_DECLARATIONS -# include "eval.c.generated.h" -#endif - -#define FNE_INCL_BR 1 /* find_name_end(): include [] in name */ -#define FNE_CHECK_START 2 /* find_name_end(): check name starts with - valid character */ -// Memory pool for reusing JobEvent structures typedef struct { TerminalJobData *data; ufunc_T *callback; @@ -475,10 +501,51 @@ typedef struct { list_T *received; int status; } JobEvent; -static int disable_job_defer = 0; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "eval.c.generated.h" +#endif + +#define FNE_INCL_BR 1 /* find_name_end(): include [] in name */ +#define FNE_CHECK_START 2 /* find_name_end(): check name starts with + valid character */ static uint64_t current_job_id = 1; static PMap(uint64_t) *jobs = NULL; +typedef enum { + kMPNil, + kMPBoolean, + kMPInteger, + kMPFloat, + kMPString, + kMPBinary, + kMPArray, + kMPMap, + kMPExt, +} MessagePackType; +static const char *const msgpack_type_names[] = { + [kMPNil] = "nil", + [kMPBoolean] = "boolean", + [kMPInteger] = "integer", + [kMPFloat] = "float", + [kMPString] = "string", + [kMPBinary] = "binary", + [kMPArray] = "array", + [kMPMap] = "map", + [kMPExt] = "ext", +}; +static const list_T *msgpack_type_lists[] = { + [kMPNil] = NULL, + [kMPBoolean] = NULL, + [kMPInteger] = NULL, + [kMPFloat] = NULL, + [kMPString] = NULL, + [kMPBinary] = NULL, + [kMPArray] = NULL, + [kMPMap] = NULL, + [kMPExt] = NULL, +}; + /* * Initialize the global and v: variables. */ @@ -511,6 +578,27 @@ void eval_init(void) /* add to compat scope dict */ hash_add(&compat_hashtab, p->vv_di.di_key); } + + dict_T *const msgpack_types_dict = dict_alloc(); + for (size_t i = 0; i < ARRAY_SIZE(msgpack_type_names); i++) { + list_T *const type_list = list_alloc(); + type_list->lv_lock = VAR_FIXED; + type_list->lv_refcount = 1; + dictitem_T *const di = dictitem_alloc((char_u *) msgpack_type_names[i]); + di->di_flags = DI_FLAGS_RO | DI_FLAGS_FIX; + di->di_tv = (typval_T) { + .v_type = VAR_LIST, + .vval = { .v_list = type_list, }, + }; + msgpack_type_lists[i] = type_list; + if (dict_add(msgpack_types_dict, di) == FAIL) { + // There must not be duplicate items in this dictionary by definition. + assert(false); + } + } + msgpack_types_dict->dv_lock = VAR_FIXED; + + set_vim_var_dict(VV_MSGPACK_TYPES, msgpack_types_dict); set_vim_var_dict(VV_COMPLETED_ITEM, dict_alloc()); set_vim_var_nr(VV_SEARCHFORWARD, 1L); set_vim_var_nr(VV_HLSEARCH, 1L); @@ -5156,24 +5244,40 @@ void list_append_dict(list_T *list, dict_T *dict) ++dict->dv_refcount; } -/* - * Make a copy of "str" and append it as an item to list "l". - * When "len" >= 0 use "str[len]". - */ -void list_append_string(list_T *l, char_u *str, int len) +/// Make a copy of "str" and append it as an item to list "l" +/// +/// @param[out] l List to append to. +/// @param[in] str String to append. +/// @param[in] len Length of the appended string. May be negative, in this +/// case string is considered to be usual zero-terminated +/// string. +void list_append_string(list_T *l, const char_u *str, int len) + FUNC_ATTR_NONNULL_ARG(1) +{ + if (str == NULL) { + list_append_allocated_string(l, NULL); + } else { + list_append_allocated_string(l, (len >= 0 + ? xmemdupz((char *) str, len) + : xstrdup((char *) str))); + } +} + +/// Append given string to the list +/// +/// Unlike list_append_string this function does not copy the string. +/// +/// @param[out] l List to append to. +/// @param[in] str String to append. +void list_append_allocated_string(list_T *l, char *const str) + FUNC_ATTR_NONNULL_ARG(1) { listitem_T *li = listitem_alloc(); list_append(l, li); li->li_tv.v_type = VAR_STRING; li->li_tv.v_lock = 0; - - if (str == NULL) { - li->li_tv.vval.v_string = NULL; - } else { - li->li_tv.vval.v_string = (len >= 0) ? vim_strnsave(str, len) - : vim_strsave(str); - } + li->li_tv.vval.v_string = (char_u *) str; } /* @@ -6582,6 +6686,8 @@ static struct fst { {"min", 1, 1, f_min}, {"mkdir", 1, 3, f_mkdir}, {"mode", 0, 1, f_mode}, + {"msgpackdump", 1, 1, f_msgpackdump}, + {"msgpackparse", 1, 1, f_msgpackparse}, {"nextnonblank", 1, 1, f_nextnonblank}, {"nr2char", 1, 2, f_nr2char}, {"or", 2, 2, f_or}, @@ -8456,27 +8562,12 @@ static void f_feedkeys(typval_T *argvars, typval_T *rettv) } } -/* - * "filereadable()" function - */ +/// "filereadable()" function static void f_filereadable(typval_T *argvars, typval_T *rettv) { - int fd; - char_u *p; - int n; - -#ifndef O_NONBLOCK -# define O_NONBLOCK 0 -#endif - p = get_tv_string(&argvars[0]); - if (*p && !os_isdir(p) && (fd = os_open((char *)p, - O_RDONLY | O_NONBLOCK, 0)) >= 0) { - n = TRUE; - close(fd); - } else - n = FALSE; - - rettv->vval.v_number = n; + char_u *p = get_tv_string(&argvars[0]); + rettv->vval.v_number = + (*p && !os_isdir(p) && os_file_is_readable((char*)p)); } /* @@ -10671,7 +10762,7 @@ static void f_jobclose(typval_T *argvars, typval_T *rettv) return; } - TerminalJobData *data = pmap_get(uint64_t)(jobs, argvars[0].vval.v_number); + TerminalJobData *data = find_job(argvars[0].vval.v_number); if (!data) { EMSG(_(e_invjob)); return; @@ -10712,7 +10803,7 @@ static void f_jobsend(typval_T *argvars, typval_T *rettv) return; } - TerminalJobData *data = pmap_get(uint64_t)(jobs, argvars[0].vval.v_number); + TerminalJobData *data = find_job(argvars[0].vval.v_number); if (!data) { EMSG(_(e_invjob)); return; @@ -10753,7 +10844,7 @@ static void f_jobresize(typval_T *argvars, typval_T *rettv) } - TerminalJobData *data = pmap_get(uint64_t)(jobs, argvars[0].vval.v_number); + TerminalJobData *data = find_job(argvars[0].vval.v_number); if (!data) { EMSG(_(e_invjob)); return; @@ -10900,8 +10991,8 @@ static void f_jobstop(typval_T *argvars, typval_T *rettv) } - TerminalJobData *data = pmap_get(uint64_t)(jobs, argvars[0].vval.v_number); - if (!data || data->stopped) { + TerminalJobData *data = find_job(argvars[0].vval.v_number); + if (!data) { EMSG(_(e_invjob)); return; } @@ -10931,28 +11022,24 @@ static void f_jobwait(typval_T *argvars, typval_T *rettv) list_T *rv = list_alloc(); ui_busy_start(); - // disable breakchecks, which could result in job callbacks being executed - // at unexpected places - disable_breakcheck++; - // disable job event deferring so the callbacks are processed while waiting. - if (!disable_job_defer++) { - // process any pending job events in the deferred queue, but only do this if - // deferred is not disabled(at the top-level `jobwait()` call) - loop_process_event(&loop); - } + Queue *waiting_jobs = queue_new_parent(loop_on_put, &loop); // For each item in the input list append an integer to the output list. -3 // is used to represent an invalid job id, -2 is for a interrupted job and // -1 for jobs that were skipped or timed out. for (listitem_T *arg = args->lv_first; arg != NULL; arg = arg->li_next) { TerminalJobData *data = NULL; if (arg->li_tv.v_type != VAR_NUMBER - || !(data = pmap_get(uint64_t)(jobs, arg->li_tv.vval.v_number))) { + || !(data = find_job(arg->li_tv.vval.v_number))) { list_append_number(rv, -3); } else { // append the list item and set the status pointer so we'll collect the // status code when the job exits list_append_number(rv, -1); data->status_ptr = &rv->lv_last->li_tv.vval.v_number; + // Process any pending events for the job because we'll temporarily + // replace the parent queue + queue_process_events(data->events); + queue_replace_parent(data->events, waiting_jobs); } } @@ -10970,10 +11057,10 @@ static void f_jobwait(typval_T *argvars, typval_T *rettv) break; } if (arg->li_tv.v_type != VAR_NUMBER - || !(data = pmap_get(uint64_t)(jobs, arg->li_tv.vval.v_number))) { + || !(data = find_job(arg->li_tv.vval.v_number))) { continue; } - int status = process_wait((Process *)&data->proc, remaining); + int status = process_wait((Process *)&data->proc, remaining, waiting_jobs); if (status < 0) { // interrupted or timed out, skip remaining jobs. if (status == -2) { @@ -10993,23 +11080,31 @@ static void f_jobwait(typval_T *argvars, typval_T *rettv) } } - // poll to ensure any pending callbacks from the last job are invoked - loop_poll_events(&loop, 0); - for (listitem_T *arg = args->lv_first; arg != NULL; arg = arg->li_next) { TerminalJobData *data = NULL; if (arg->li_tv.v_type != VAR_NUMBER - || !(data = pmap_get(uint64_t)(jobs, arg->li_tv.vval.v_number))) { + || !(data = find_job(arg->li_tv.vval.v_number))) { continue; } // remove the status pointer because the list may be freed before the // job exits data->status_ptr = NULL; } - disable_job_defer--; - disable_breakcheck--; - ui_busy_stop(); + // restore the parent queue for any jobs still alive + for (listitem_T *arg = args->lv_first; arg != NULL; arg = arg->li_next) { + TerminalJobData *data = NULL; + if (arg->li_tv.v_type != VAR_NUMBER + || !(data = pmap_get(uint64_t)(jobs, arg->li_tv.vval.v_number))) { + continue; + } + // restore the parent queue for the job + queue_process_events(data->events); + queue_replace_parent(data->events, loop.events); + } + + queue_free(waiting_jobs); + ui_busy_stop(); rv->lv_refcount++; rettv->v_type = VAR_LIST; rettv->vval.v_list = rv; @@ -11782,6 +11877,827 @@ static void f_mode(typval_T *argvars, typval_T *rettv) rettv->v_type = VAR_STRING; } +/// Msgpack callback for writing to readfile()-style list +static int msgpack_list_write(void *data, const char *buf, size_t len) +{ + if (len == 0) { + return 0; + } + list_T *const list = (list_T *) data; + const char *const end = buf + len; + const char *line_end = buf; + if (list->lv_last == NULL) { + list_append_string(list, NULL, 0); + } + listitem_T *li = list->lv_last; + do { + const char *line_start = line_end; + line_end = xmemscan(line_start, NL, end - line_start); + if (line_end == line_start) { + list_append_allocated_string(list, NULL); + } else { + const size_t line_length = line_end - line_start; + char *str; + if (li == NULL) { + str = xmemdupz(line_start, line_length); + } else { + const size_t li_len = (li->li_tv.vval.v_string == NULL + ? 0 + : STRLEN(li->li_tv.vval.v_string)); + li->li_tv.vval.v_string = xrealloc(li->li_tv.vval.v_string, + li_len + line_length + 1); + str = (char *) li->li_tv.vval.v_string + li_len; + memmove(str, line_start, line_length); + str[line_length] = 0; + } + for (size_t i = 0; i < line_length; i++) { + if (str[i] == NUL) { + str[i] = NL; + } + } + if (li == NULL) { + list_append_allocated_string(list, str); + } else { + li = NULL; + } + if (line_end == end - 1) { + list_append_allocated_string(list, NULL); + } + } + line_end++; + } while (line_end < end); + return 0; +} + +/// Convert readfile()-style list to a char * buffer with length +/// +/// @param[in] list Converted list. +/// @param[out] ret_len Resulting buffer length. +/// @param[out] ret_buf Allocated buffer with the result or NULL if ret_len is +/// zero. +/// +/// @return true in case of success, false in case of failure. +static inline bool vim_list_to_buf(const list_T *const list, + size_t *const ret_len, char **const ret_buf) + FUNC_ATTR_NONNULL_ARG(2,3) FUNC_ATTR_WARN_UNUSED_RESULT +{ + size_t len = 0; + if (list != NULL) { + for (const listitem_T *li = list->lv_first; + li != NULL; + li = li->li_next) { + if (li->li_tv.v_type != VAR_STRING) { + return false; + } + len++; + if (li->li_tv.vval.v_string != 0) { + len += STRLEN(li->li_tv.vval.v_string); + } + } + if (len) { + len--; + } + } + *ret_len = len; + if (len == 0) { + *ret_buf = NULL; + return true; + } + ListReaderState lrstate = init_lrstate(list); + char *const buf = xmalloc(len); + size_t read_bytes; + if (read_from_list(&lrstate, buf, len, &read_bytes) != OK) { + assert(false); + } + assert(len == read_bytes); + *ret_buf = buf; + return true; +} + +/// Convert one VimL value to msgpack +/// +/// @param packer Messagepack packer. +/// @param[out] mpstack Stack with values to convert. Only used for pushing +/// values to it. +/// @param[in] tv Converted value. +/// +/// @return OK in case of success, FAIL otherwise. +static int convert_one_value(msgpack_packer *const packer, + MPConvStack *const mpstack, + const typval_T *const tv) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT +{ + switch (tv->v_type) { +#define CHECK_SELF_REFERENCE(conv_type, vval_name, ptr) \ + do { \ + for (size_t i = 0; i < kv_size(*mpstack); i++) { \ + if (kv_A(*mpstack, i).type == conv_type \ + && kv_A(*mpstack, i).data.vval_name == ptr) { \ + EMSG2(_(e_invarg2), "container references itself"); \ + return FAIL; \ + } \ + } \ + } while (0) + case VAR_STRING: { + if (tv->vval.v_string == NULL) { + msgpack_pack_bin(packer, 0); + } else { + const size_t len = STRLEN(tv->vval.v_string); + msgpack_pack_bin(packer, len); + msgpack_pack_bin_body(packer, tv->vval.v_string, len); + } + break; + } + case VAR_NUMBER: { + msgpack_pack_int64(packer, (int64_t) tv->vval.v_number); + break; + } + case VAR_FLOAT: { + msgpack_pack_double(packer, (double) tv->vval.v_float); + break; + } + case VAR_FUNC: { + EMSG2(_(e_invarg2), "attempt to dump function reference"); + return FAIL; + } + case VAR_LIST: { + if (tv->vval.v_list == NULL) { + msgpack_pack_array(packer, 0); + break; + } + CHECK_SELF_REFERENCE(kMPConvList, l.list, tv->vval.v_list); + msgpack_pack_array(packer, tv->vval.v_list->lv_len); + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { + .type = kMPConvList, + .data = { + .l = { + .list = tv->vval.v_list, + .li = tv->vval.v_list->lv_first, + }, + }, + })); + break; + } + case VAR_DICT: { + if (tv->vval.v_dict == NULL) { + msgpack_pack_map(packer, 0); + break; + } + const dictitem_T *type_di; + const dictitem_T *val_di; + if (tv->vval.v_dict->dv_hashtab.ht_used == 2 + && (type_di = dict_find((dict_T *) tv->vval.v_dict, + (char_u *) "_TYPE", -1)) != NULL + && type_di->di_tv.v_type == VAR_LIST + && (val_di = dict_find((dict_T *) tv->vval.v_dict, + (char_u *) "_VAL", -1)) != NULL) { + size_t i; + for (i = 0; i < ARRAY_SIZE(msgpack_type_lists); i++) { + if (type_di->di_tv.vval.v_list == msgpack_type_lists[i]) { + break; + } + } + if (i == ARRAY_SIZE(msgpack_type_lists)) { + goto vim_to_msgpack_regural_dict; + } + switch ((MessagePackType) i) { + case kMPNil: { + msgpack_pack_nil(packer); + break; + } + case kMPBoolean: { + if (val_di->di_tv.v_type != VAR_NUMBER) { + goto vim_to_msgpack_regural_dict; + } + if (val_di->di_tv.vval.v_number) { + msgpack_pack_true(packer); + } else { + msgpack_pack_false(packer); + } + break; + } + case kMPInteger: { + const list_T *val_list; + varnumber_T sign; + varnumber_T highest_bits; + varnumber_T high_bits; + varnumber_T low_bits; + // List of 4 integers; first is signed (should be 1 or -1, but this + // is not checked), second is unsigned and have at most one (sign is + // -1) or two (sign is 1) non-zero bits (number of bits is not + // checked), other unsigned and have at most 31 non-zero bits + // (number of bits is not checked). + if (val_di->di_tv.v_type != VAR_LIST + || (val_list = val_di->di_tv.vval.v_list) == NULL + || val_list->lv_len != 4 + || val_list->lv_first->li_tv.v_type != VAR_NUMBER + || (sign = val_list->lv_first->li_tv.vval.v_number) == 0 + || val_list->lv_first->li_next->li_tv.v_type != VAR_NUMBER + || (highest_bits = + val_list->lv_first->li_next->li_tv.vval.v_number) < 0 + || val_list->lv_last->li_prev->li_tv.v_type != VAR_NUMBER + || (high_bits = + val_list->lv_last->li_prev->li_tv.vval.v_number) < 0 + || val_list->lv_last->li_tv.v_type != VAR_NUMBER + || (low_bits = val_list->lv_last->li_tv.vval.v_number) < 0) { + goto vim_to_msgpack_regural_dict; + } + uint64_t number = ((uint64_t) (((uint64_t) highest_bits) << 62) + | (uint64_t) (((uint64_t) high_bits) << 31) + | (uint64_t) low_bits); + if (sign > 0) { + msgpack_pack_uint64(packer, number); + } else { + msgpack_pack_int64(packer, (int64_t) (-number)); + } + break; + } + case kMPFloat: { + if (val_di->di_tv.v_type != VAR_FLOAT) { + goto vim_to_msgpack_regural_dict; + } + msgpack_pack_double(packer, val_di->di_tv.vval.v_float); + break; + } + case kMPString: + case kMPBinary: { + const bool is_string = ((MessagePackType) i == kMPString); + if (val_di->di_tv.v_type != VAR_LIST) { + goto vim_to_msgpack_regural_dict; + } + size_t len; + char *buf; + if (!vim_list_to_buf(val_di->di_tv.vval.v_list, &len, &buf)) { + goto vim_to_msgpack_regural_dict; + } + if (is_string) { + msgpack_pack_str(packer, len); + } else { + msgpack_pack_bin(packer, len); + } + if (len == 0) { + break; + } + if (is_string) { + msgpack_pack_str_body(packer, buf, len); + } else { + msgpack_pack_bin_body(packer, buf, len); + } + xfree(buf); + break; + } + case kMPArray: { + if (val_di->di_tv.v_type != VAR_LIST) { + goto vim_to_msgpack_regural_dict; + } + CHECK_SELF_REFERENCE(kMPConvList, l.list, + val_di->di_tv.vval.v_list); + msgpack_pack_array(packer, val_di->di_tv.vval.v_list->lv_len); + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { + .type = kMPConvList, + .data = { + .l = { + .list = val_di->di_tv.vval.v_list, + .li = val_di->di_tv.vval.v_list->lv_first, + }, + }, + })); + break; + } + case kMPMap: { + if (val_di->di_tv.v_type != VAR_LIST) { + goto vim_to_msgpack_regural_dict; + } + if (val_di->di_tv.vval.v_list == NULL) { + msgpack_pack_map(packer, 0); + break; + } + const list_T *const val_list = val_di->di_tv.vval.v_list; + for (const listitem_T *li = val_list->lv_first; li != NULL; + li = li->li_next) { + if (li->li_tv.v_type != VAR_LIST + || li->li_tv.vval.v_list->lv_len != 2) { + goto vim_to_msgpack_regural_dict; + } + } + CHECK_SELF_REFERENCE(kMPConvPairs, l.list, val_list); + msgpack_pack_map(packer, val_list->lv_len); + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { + .type = kMPConvPairs, + .data = { + .l = { + .list = val_list, + .li = val_list->lv_first, + }, + }, + })); + break; + } + case kMPExt: { + const list_T *val_list; + varnumber_T type; + if (val_di->di_tv.v_type != VAR_LIST + || (val_list = val_di->di_tv.vval.v_list) == NULL + || val_list->lv_len != 2 + || (val_list->lv_first->li_tv.v_type != VAR_NUMBER) + || (type = val_list->lv_first->li_tv.vval.v_number) > INT8_MAX + || type < INT8_MIN + || (val_list->lv_last->li_tv.v_type != VAR_LIST)) { + goto vim_to_msgpack_regural_dict; + } + size_t len; + char *buf; + if (!vim_list_to_buf(val_list->lv_last->li_tv.vval.v_list, + &len, &buf)) { + goto vim_to_msgpack_regural_dict; + } + msgpack_pack_ext(packer, len, (int8_t) type); + msgpack_pack_ext_body(packer, buf, len); + xfree(buf); + break; + } + } + break; + } +vim_to_msgpack_regural_dict: + CHECK_SELF_REFERENCE(kMPConvDict, d.dict, tv->vval.v_dict); + msgpack_pack_map(packer, tv->vval.v_dict->dv_hashtab.ht_used); + kv_push(MPConvStackVal, *mpstack, ((MPConvStackVal) { + .type = kMPConvDict, + .data = { + .d = { + .dict = tv->vval.v_dict, + .hi = tv->vval.v_dict->dv_hashtab.ht_array, + .todo = tv->vval.v_dict->dv_hashtab.ht_used, + }, + }, + })); + break; + } + } +#undef CHECK_SELF_REFERENCE + return OK; +} + +/// Convert typval_T to messagepack +static int vim_to_msgpack(msgpack_packer *const packer, + const typval_T *const tv) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + MPConvStack mpstack; + kv_init(mpstack); + if (convert_one_value(packer, &mpstack, tv) == FAIL) { + goto vim_to_msgpack_error_ret; + } + while (kv_size(mpstack)) { + MPConvStackVal *cur_mpsv = &kv_A(mpstack, kv_size(mpstack) - 1); + const typval_T *cur_tv = NULL; + switch (cur_mpsv->type) { + case kMPConvDict: { + if (!cur_mpsv->data.d.todo) { + (void) kv_pop(mpstack); + continue; + } + while (HASHITEM_EMPTY(cur_mpsv->data.d.hi)) { + cur_mpsv->data.d.hi++; + } + const dictitem_T *const di = HI2DI(cur_mpsv->data.d.hi); + cur_mpsv->data.d.todo--; + cur_mpsv->data.d.hi++; + const size_t key_len = STRLEN(&di->di_key[0]); + msgpack_pack_str(packer, key_len); + msgpack_pack_str_body(packer, &di->di_key[0], key_len); + cur_tv = &di->di_tv; + break; + } + case kMPConvList: { + if (cur_mpsv->data.l.li == NULL) { + (void) kv_pop(mpstack); + continue; + } + cur_tv = &cur_mpsv->data.l.li->li_tv; + cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next; + break; + } + case kMPConvPairs: { + if (cur_mpsv->data.l.li == NULL) { + (void) kv_pop(mpstack); + continue; + } + const list_T *const kv_pair = cur_mpsv->data.l.li->li_tv.vval.v_list; + if (convert_one_value(packer, &mpstack, &kv_pair->lv_first->li_tv) + == FAIL) { + goto vim_to_msgpack_error_ret; + } + cur_tv = &kv_pair->lv_last->li_tv; + cur_mpsv->data.l.li = cur_mpsv->data.l.li->li_next; + break; + } + } + if (convert_one_value(packer, &mpstack, cur_tv) == FAIL) { + goto vim_to_msgpack_error_ret; + } + } + kv_destroy(mpstack); + return OK; +vim_to_msgpack_error_ret: + kv_destroy(mpstack); + return FAIL; +} + +/// "msgpackdump()" function +static void f_msgpackdump(typval_T *argvars, typval_T *rettv) + FUNC_ATTR_NONNULL_ALL +{ + if (argvars[0].v_type != VAR_LIST) { + EMSG2(_(e_listarg), "msgpackdump()"); + return; + } + list_T *ret_list = rettv_list_alloc(rettv); + const list_T *list = argvars[0].vval.v_list; + if (list == NULL) { + return; + } + msgpack_packer *lpacker = msgpack_packer_new(ret_list, &msgpack_list_write); + for (const listitem_T *li = list->lv_first; li != NULL; li = li->li_next) { + if (vim_to_msgpack(lpacker, &li->li_tv) == FAIL) { + break; + } + } + msgpack_packer_free(lpacker); +} + +/// Read bytes from list +/// +/// @param[in,out] state Structure describing position in list from which +/// reading should start. Is updated to reflect position +/// at which reading ended. +/// @param[out] buf Buffer to write to. +/// @param[in] nbuf Buffer length. +/// @param[out] read_bytes Is set to amount of bytes read. +/// +/// @return OK when reading was finished, FAIL in case of error (i.e. list item +/// was not a string), NOTDONE if reading was successfull, but there are +/// more bytes to read. +static int read_from_list(ListReaderState *const state, char *const buf, + const size_t nbuf, size_t *const read_bytes) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT +{ + char *const buf_end = buf + nbuf; + char *p = buf; + while (p < buf_end) { + for (size_t i = state->offset; i < state->li_length && p < buf_end; i++) { + const char ch = state->li->li_tv.vval.v_string[state->offset++]; + *p++ = (ch == NL ? NUL : ch); + } + if (p < buf_end) { + state->li = state->li->li_next; + if (state->li == NULL) { + *read_bytes = (size_t) (p - buf); + return OK; + } + *p++ = NL; + if (state->li->li_tv.v_type != VAR_STRING) { + *read_bytes = (size_t) (p - buf); + return FAIL; + } + state->offset = 0; + state->li_length = (state->li->li_tv.vval.v_string == NULL + ? 0 + : STRLEN(state->li->li_tv.vval.v_string)); + } + } + *read_bytes = nbuf; + return (state->offset < state->li_length || state->li->li_next != NULL + ? NOTDONE + : OK); +} + +/// Initialize ListReaderState structure +static inline ListReaderState init_lrstate(const list_T *const list) + FUNC_ATTR_NONNULL_ALL +{ + return (ListReaderState) { + .li = list->lv_first, + .offset = 0, + .li_length = (list->lv_first->li_tv.vval.v_string == NULL + ? 0 + : STRLEN(list->lv_first->li_tv.vval.v_string)), + }; +} + +/// Convert msgpack object to a VimL one +static int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT +{ +#define INIT_SPECIAL_DICT(tv, type, val) \ + do { \ + dict_T *const dict = dict_alloc(); \ + dictitem_T *const type_di = dictitem_alloc((char_u *) "_TYPE"); \ + type_di->di_tv.v_type = VAR_LIST; \ + type_di->di_tv.v_lock = 0; \ + type_di->di_tv.vval.v_list = (list_T *) msgpack_type_lists[type]; \ + type_di->di_tv.vval.v_list->lv_refcount++; \ + dict_add(dict, type_di); \ + dictitem_T *const val_di = dictitem_alloc((char_u *) "_VAL"); \ + val_di->di_tv = val; \ + dict_add(dict, val_di); \ + tv->v_type = VAR_DICT; \ + dict->dv_refcount++; \ + tv->vval.v_dict = dict; \ + } while (0) + switch (mobj.type) { + case MSGPACK_OBJECT_NIL: { + INIT_SPECIAL_DICT(rettv, kMPNil, ((typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { .v_number = 0 }, + })); + break; + } + case MSGPACK_OBJECT_BOOLEAN: { + INIT_SPECIAL_DICT(rettv, kMPBoolean, + ((typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { + .v_number = (varnumber_T) mobj.via.boolean, + }, + })); + break; + } + case MSGPACK_OBJECT_POSITIVE_INTEGER: { + if (mobj.via.u64 <= VARNUMBER_MAX) { + *rettv = (typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { .v_number = (varnumber_T) mobj.via.u64 }, + }; + } else { + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPInteger, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + uint64_t n = mobj.via.u64; + list_append_number(list, 1); + list_append_number(list, (varnumber_T) ((n >> 62) & 0x3)); + list_append_number(list, (varnumber_T) ((n >> 31) & 0x7FFFFFFF)); + list_append_number(list, (varnumber_T) (n & 0x7FFFFFFF)); + } + break; + } + case MSGPACK_OBJECT_NEGATIVE_INTEGER: { + if (mobj.via.i64 >= VARNUMBER_MIN) { + *rettv = (typval_T) { + .v_type = VAR_NUMBER, + .v_lock = 0, + .vval = { .v_number = (varnumber_T) mobj.via.i64 }, + }; + } else { + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPInteger, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + uint64_t n = -((uint64_t) mobj.via.i64); + list_append_number(list, -1); + list_append_number(list, (varnumber_T) ((n >> 62) & 0x3)); + list_append_number(list, (varnumber_T) ((n >> 31) & 0x7FFFFFFF)); + list_append_number(list, (varnumber_T) (n & 0x7FFFFFFF)); + } + break; + } + case MSGPACK_OBJECT_FLOAT: { + *rettv = (typval_T) { + .v_type = VAR_FLOAT, + .v_lock = 0, + .vval = { .v_float = mobj.via.f64 }, + }; + break; + } + case MSGPACK_OBJECT_STR: { + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPString, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + if (msgpack_list_write((void *) list, mobj.via.str.ptr, mobj.via.str.size) + == -1) { + return FAIL; + } + break; + } + case MSGPACK_OBJECT_BIN: { + if (memchr(mobj.via.bin.ptr, NUL, mobj.via.bin.size) == NULL) { + *rettv = (typval_T) { + .v_type = VAR_STRING, + .v_lock = 0, + .vval = { .v_string = xmemdupz(mobj.via.bin.ptr, mobj.via.bin.size) }, + }; + break; + } + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPBinary, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + if (msgpack_list_write((void *) list, mobj.via.bin.ptr, mobj.via.bin.size) + == -1) { + return FAIL; + } + break; + } + case MSGPACK_OBJECT_ARRAY: { + list_T *const list = list_alloc(); + list->lv_refcount++; + *rettv = (typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + }; + for (size_t i = 0; i < mobj.via.array.size; i++) { + listitem_T *const li = listitem_alloc(); + li->li_tv.v_type = VAR_UNKNOWN; + list_append(list, li); + if (msgpack_to_vim(mobj.via.array.ptr[i], &li->li_tv) == FAIL) { + return FAIL; + } + } + break; + } + case MSGPACK_OBJECT_MAP: { + for (size_t i = 0; i < mobj.via.map.size; i++) { + if (mobj.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR + || mobj.via.map.ptr[i].key.via.str.size == 0 + || memchr(mobj.via.map.ptr[i].key.via.str.ptr, NUL, + mobj.via.map.ptr[i].key.via.str.size) != NULL) { + goto msgpack_to_vim_generic_map; + } + } + dict_T *const dict = dict_alloc(); + dict->dv_refcount++; + *rettv = (typval_T) { + .v_type = VAR_DICT, + .v_lock = 0, + .vval = { .v_dict = dict }, + }; + for (size_t i = 0; i < mobj.via.map.size; i++) { + dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key) + + mobj.via.map.ptr[i].key.via.str.size); + memcpy(&di->di_key[0], mobj.via.map.ptr[i].key.via.str.ptr, + mobj.via.map.ptr[i].key.via.str.size); + di->di_tv.v_type = VAR_UNKNOWN; + if (dict_add(dict, di) == FAIL) { + // Duplicate key: fallback to generic map + clear_tv(rettv); + xfree(di); + goto msgpack_to_vim_generic_map; + } + if (msgpack_to_vim(mobj.via.map.ptr[i].val, &di->di_tv) == FAIL) { + return FAIL; + } + } + break; +msgpack_to_vim_generic_map: {} + list_T *const list = list_alloc(); + list->lv_refcount++; + INIT_SPECIAL_DICT(rettv, kMPMap, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + for (size_t i = 0; i < mobj.via.map.size; i++) { + list_T *const kv_pair = list_alloc(); + list_append_list(list, kv_pair); + listitem_T *const key_li = listitem_alloc(); + key_li->li_tv.v_type = VAR_UNKNOWN; + list_append(kv_pair, key_li); + listitem_T *const val_li = listitem_alloc(); + val_li->li_tv.v_type = VAR_UNKNOWN; + list_append(kv_pair, val_li); + if (msgpack_to_vim(mobj.via.map.ptr[i].key, &key_li->li_tv) == FAIL) { + return FAIL; + } + if (msgpack_to_vim(mobj.via.map.ptr[i].val, &val_li->li_tv) == FAIL) { + return FAIL; + } + } + break; + } + case MSGPACK_OBJECT_EXT: { + list_T *const list = list_alloc(); + list->lv_refcount++; + list_append_number(list, mobj.via.ext.type); + list_T *const ext_val_list = list_alloc(); + list_append_list(list, ext_val_list); + INIT_SPECIAL_DICT(rettv, kMPExt, + ((typval_T) { + .v_type = VAR_LIST, + .v_lock = 0, + .vval = { .v_list = list }, + })); + if (msgpack_list_write((void *) ext_val_list, mobj.via.ext.ptr, + mobj.via.ext.size) == -1) { + return FAIL; + } + break; + } + } +#undef INIT_SPECIAL_DICT + return OK; +} + +/// "msgpackparse" function +static void f_msgpackparse(typval_T *argvars, typval_T *rettv) + FUNC_ATTR_NONNULL_ALL +{ + if (argvars[0].v_type != VAR_LIST) { + EMSG2(_(e_listarg), "msgpackparse()"); + } + list_T *ret_list = rettv_list_alloc(rettv); + const list_T *list = argvars[0].vval.v_list; + if (list == NULL || list->lv_first == NULL) { + return; + } + if (list->lv_first->li_tv.v_type != VAR_STRING) { + EMSG2(_(e_invarg2), "List item is not a string"); + return; + } + ListReaderState lrstate = init_lrstate(list); + msgpack_unpacker *const unpacker = msgpack_unpacker_new(IOSIZE); + if (unpacker == NULL) { + EMSG(_(e_outofmem)); + return; + } + msgpack_unpacked unpacked; + msgpack_unpacked_init(&unpacked); + do { + if (!msgpack_unpacker_reserve_buffer(unpacker, IOSIZE)) { + EMSG(_(e_outofmem)); + goto f_msgpackparse_exit; + } + size_t read_bytes; + const int rlret = read_from_list( + &lrstate, msgpack_unpacker_buffer(unpacker), IOSIZE, &read_bytes); + if (rlret == FAIL) { + EMSG2(_(e_invarg2), "List item is not a string"); + goto f_msgpackparse_exit; + } + msgpack_unpacker_buffer_consumed(unpacker, read_bytes); + if (read_bytes == 0) { + break; + } + while (unpacker->off < unpacker->used) { + const msgpack_unpack_return result = msgpack_unpacker_next(unpacker, + &unpacked); + if (result == MSGPACK_UNPACK_PARSE_ERROR) { + EMSG2(_(e_invarg2), "Failed to parse msgpack string"); + goto f_msgpackparse_exit; + } + if (result == MSGPACK_UNPACK_NOMEM_ERROR) { + EMSG(_(e_outofmem)); + goto f_msgpackparse_exit; + } + if (result == MSGPACK_UNPACK_SUCCESS) { + listitem_T *li = listitem_alloc(); + li->li_tv.v_type = VAR_UNKNOWN; + list_append(ret_list, li); + if (msgpack_to_vim(unpacked.data, &li->li_tv) == FAIL) { + EMSG2(_(e_invarg2), "Failed to convert msgpack string"); + goto f_msgpackparse_exit; + } + } + if (result == MSGPACK_UNPACK_CONTINUE) { + if (rlret == OK) { + EMSG2(_(e_invarg2), "Incomplete msgpack string"); + } + break; + } + } + if (rlret == OK) { + break; + } + } while (true); + +f_msgpackparse_exit: + msgpack_unpacked_destroy(&unpacked); + msgpack_unpacker_free(unpacker); + return; +} /* * "nextnonblank()" function @@ -20125,17 +21041,21 @@ static inline TerminalJobData *common_job_init(char **argv, ufunc_T *on_stdout, data->on_stderr = on_stderr; data->on_exit = on_exit; data->self = self; + data->events = queue_new_child(loop.events); if (pty) { - data->proc.pty = pty_process_init(data); + data->proc.pty = pty_process_init(&loop, data); } else { - data->proc.uv = uv_process_init(data); + data->proc.uv = uv_process_init(&loop, data); } Process *proc = (Process *)&data->proc; proc->argv = argv; proc->in = &data->in; proc->out = &data->out; - proc->err = &data->err; + if (!pty) { + proc->err = &data->err; + } proc->cb = on_process_exit; + proc->events = data->events; return data; } @@ -20166,8 +21086,12 @@ static inline bool common_job_start(TerminalJobData *data, typval_T *rettv) { data->refcount++; Process *proc = (Process *)&data->proc; - if (!process_spawn(&loop, proc)) { + if (!process_spawn(proc)) { EMSG(_(e_jobexe)); + if (proc->type == kProcessTypePty) { + xfree(data->proc.pty.term_name); + free_term_job_data(data); + } return false; } @@ -20186,7 +21110,9 @@ static inline bool common_job_start(TerminalJobData *data, typval_T *rettv) return true; } -static inline void free_term_job_data(TerminalJobData *data) { +static inline void free_term_job_data_event(void **argv) +{ + TerminalJobData *data = argv[0]; if (data->on_stdout) { user_func_unref(data->on_stdout); } @@ -20201,17 +21127,25 @@ static inline void free_term_job_data(TerminalJobData *data) { data->self->internal_refcount--; dict_unref(data->self); } + queue_free(data->events); xfree(data); } +static inline void free_term_job_data(TerminalJobData *data) +{ + // data->queue may still be used after this function returns(process_wait), so + // only free in the next event loop iteration + queue_put(loop.fast_events, free_term_job_data_event, 1, data); +} + // vimscript job callbacks must be executed on Nvim main loop -static inline void push_job_event(TerminalJobData *data, ufunc_T *callback, +static inline void process_job_event(TerminalJobData *data, ufunc_T *callback, const char *type, char *buf, size_t count, int status) { - JobEvent *event_data = xmalloc(sizeof(JobEvent)); - event_data->received = NULL; + JobEvent event_data; + event_data.received = NULL; if (buf) { - event_data->received = list_alloc(); + event_data.received = list_alloc(); char *ptr = buf; size_t remaining = count; size_t off = 0; @@ -20219,7 +21153,7 @@ static inline void push_job_event(TerminalJobData *data, ufunc_T *callback, while (off < remaining) { // append the line if (ptr[off] == NL) { - list_append_string(event_data->received, (uint8_t *)ptr, off); + list_append_string(event_data.received, (uint8_t *)ptr, off); size_t skip = off + 1; ptr += skip; remaining -= skip; @@ -20232,51 +21166,53 @@ static inline void push_job_event(TerminalJobData *data, ufunc_T *callback, } off++; } - list_append_string(event_data->received, (uint8_t *)ptr, off); + list_append_string(event_data.received, (uint8_t *)ptr, off); } else { - event_data->status = status; + event_data.status = status; } - event_data->data = data; - event_data->callback = callback; - event_data->type = type; - loop_push_event(&loop, (Event) { - .handler = on_job_event, - .data = event_data - }, !disable_job_defer); + event_data.data = data; + event_data.callback = callback; + event_data.type = type; + on_job_event(&event_data); } -static void on_job_stdout(Stream *stream, RBuffer *buf, void *job, bool eof) +static void on_job_stdout(Stream *stream, RBuffer *buf, size_t count, + void *job, bool eof) { TerminalJobData *data = job; - on_job_output(stream, job, buf, eof, data->on_stdout, "stdout"); + on_job_output(stream, job, buf, count, eof, data->on_stdout, "stdout"); } -static void on_job_stderr(Stream *stream, RBuffer *buf, void *job, bool eof) +static void on_job_stderr(Stream *stream, RBuffer *buf, size_t count, + void *job, bool eof) { TerminalJobData *data = job; - on_job_output(stream, job, buf, eof, data->on_stderr, "stderr"); + on_job_output(stream, job, buf, count, eof, data->on_stderr, "stderr"); } static void on_job_output(Stream *stream, TerminalJobData *data, RBuffer *buf, - bool eof, ufunc_T *callback, const char *type) + size_t count, bool eof, ufunc_T *callback, const char *type) { if (eof) { return; } - RBUFFER_UNTIL_EMPTY(buf, ptr, len) { - // The order here matters, the terminal must receive the data first because - // push_job_event will modify the read buffer(convert NULs into NLs) - if (data->term) { - terminal_receive(data->term, ptr, len); - } + // stub variable, to keep reading consistent with the order of events, only + // consider the count parameter. + size_t r; + char *ptr = rbuffer_read_ptr(buf, &r); - if (callback) { - push_job_event(data, callback, type, ptr, len, 0); - } + // The order here matters, the terminal must receive the data first because + // process_job_event will modify the read buffer(convert NULs into NLs) + if (data->term) { + terminal_receive(data->term, ptr, count); + } - rbuffer_consumed(buf, len); + if (callback) { + process_job_event(data, callback, type, ptr, count, 0); } + + rbuffer_consumed(buf, count); } static void on_process_exit(Process *proc, int status, void *d) @@ -20292,7 +21228,7 @@ static void on_process_exit(Process *proc, int status, void *d) *data->status_ptr = status; } - push_job_event(data, data->on_exit, "exit", NULL, 0, status); + process_job_event(data, data->on_exit, "exit", NULL, 0, status); } static void term_write(char *buf, size_t size, void *d) @@ -20326,10 +21262,8 @@ static void term_job_data_decref(TerminalJobData *data) } } -static void on_job_event(Event event) +static void on_job_event(JobEvent *ev) { - JobEvent *ev = event.data; - if (!ev->callback) { goto end; } @@ -20374,7 +21308,15 @@ end: pmap_del(uint64_t)(jobs, ev->data->id); term_job_data_decref(ev->data); } - xfree(ev); +} + +static TerminalJobData *find_job(uint64_t id) +{ + TerminalJobData *data = pmap_get(uint64_t)(jobs, id); + if (!data || data->stopped) { + return NULL; + } + return data; } static void script_host_eval(char *name, typval_T *argvars, typval_T *rettv) |