diff options
-rw-r--r-- | src/nvim/eval/decode.c | 39 | ||||
-rw-r--r-- | src/nvim/eval/encode.c | 56 | ||||
-rw-r--r-- | src/nvim/shada.c | 234 |
3 files changed, 45 insertions, 284 deletions
diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index e8c06b2927..c7ca3a8ce5 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -222,8 +222,6 @@ static inline int json_decoder_pop(ValuesStackItem obj, /// Parse JSON double-quoted string /// -/// @param[in] conv Defines conversion necessary to convert UTF-8 string to -/// &encoding. /// @param[in] buf Buffer being converted. /// @param[in] buf_len Length of the buffer. /// @param[in,out] pp Pointer to the start of the string. Must point to '"'. @@ -240,8 +238,7 @@ static inline int json_decoder_pop(ValuesStackItem obj, /// value when decoder is restarted, otherwise unused. /// /// @return OK in case of success, FAIL in case of error. -static inline int parse_json_string(vimconv_T *const conv, - const char *const buf, const size_t buf_len, +static inline int parse_json_string(const char *const buf, const size_t buf_len, const char **const pp, ValuesStack *const stack, ContainerStack *const container_stack, @@ -416,20 +413,6 @@ static inline int parse_json_string(vimconv_T *const conv, } PUT_FST_IN_PAIR(fst_in_pair, str_end); #undef PUT_FST_IN_PAIR - if (conv->vc_type != CONV_NONE) { - size_t str_len = (size_t) (str_end - str); - char *const new_str = (char *) string_convert(conv, (char_u *) str, - &str_len); - if (new_str == NULL) { - emsgf(_("E474: Failed to convert string \"%.*s\" from UTF-8"), - (int) str_len, str); - xfree(str); - goto parse_json_string_fail; - } - xfree(str); - str = new_str; - str_end = new_str + str_len; - } if (hasnul) { typval_T obj; list_T *const list = tv_list_alloc(); @@ -606,17 +589,6 @@ parse_json_number_ret: } \ } while (0) -/// Last used p_enc value -/// -/// Generic pointer: it is not used as a string, only pointer comparisons are -/// performed. Must not be freed. -static const void *last_p_enc = NULL; - -/// Conversion setup for converting from UTF-8 to last_p_enc -static vimconv_T p_enc_conv = { - .vc_type = CONV_NONE, -}; - /// Convert JSON string into VimL object /// /// @param[in] buf String to convert. UTF-8 encoding is assumed. @@ -637,12 +609,7 @@ int json_decode_string(const char *const buf, const size_t buf_len, EMSG(_("E474: Attempt to decode a blank string")); return FAIL; } - if (last_p_enc != (const void *)p_enc) { - p_enc_conv.vc_type = CONV_NONE; - convert_setup(&p_enc_conv, (char_u *)"utf-8", p_enc); - p_enc_conv.vc_fail = true; - last_p_enc = p_enc; - } + assert(STRCMP(p_enc, "utf-8") == 0); int ret = OK; ValuesStack stack = KV_INITIAL_VALUE; ContainerStack container_stack = KV_INITIAL_VALUE; @@ -789,7 +756,7 @@ json_decode_string_cycle_start: } case '"': { if (parse_json_string( - &p_enc_conv, buf, buf_len, &p, &stack, &container_stack, + buf, buf_len, &p, &stack, &container_stack, &next_map_special, &didcomma, &didcolon) == FAIL) { // Error message was already given goto json_decode_string_fail; diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 26f9aaa27d..b64217f969 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -11,7 +11,7 @@ #include <math.h> #include "nvim/eval/encode.h" -#include "nvim/buffer_defs.h" // vimconv_T +#include "nvim/buffer_defs.h" #include "nvim/eval.h" #include "nvim/eval/typval.h" #include "nvim/garray.h" @@ -29,10 +29,6 @@ #define utf_ptr2char(b) utf_ptr2char((char_u *)b) #define utf_ptr2len(b) ((size_t)utf_ptr2len((char_u *)b)) #define utf_char2len(b) ((size_t)utf_char2len(b)) -#define string_convert(a, b, c) \ - ((char *)string_convert((vimconv_T *)a, (char_u *)b, c)) -#define convert_setup(vcp, from, to) \ - (convert_setup(vcp, (char_u *)from, (char_u *)to)) const char *const encode_special_var_names[] = { [kSpecialVarNull] = "null", @@ -537,17 +533,6 @@ int encode_read_from_list(ListReaderState *const state, char *const buf, } \ } while (0) -/// Last used p_enc value -/// -/// Generic pointer: it is not used as a string, only pointer comparisons are -/// performed. Must not be freed. -static const void *last_p_enc = NULL; - -/// Conversion setup for converting from last_p_enc to UTF-8 -static vimconv_T p_enc_conv = { - .vc_type = CONV_NONE, -}; - /// Escape sequences used in JSON static const char escapes[][3] = { [BS] = "\\b", @@ -579,33 +564,16 @@ static inline int convert_to_json_string(garray_T *const gap, } else { size_t utf_len = len; char *tofree = NULL; - if (last_p_enc != (const void *) p_enc) { - p_enc_conv.vc_type = CONV_NONE; - convert_setup(&p_enc_conv, p_enc, "utf-8"); - p_enc_conv.vc_fail = true; - last_p_enc = p_enc; - } - if (p_enc_conv.vc_type != CONV_NONE) { - tofree = string_convert(&p_enc_conv, buf, &utf_len); - if (tofree == NULL) { - emsgf(_("E474: Failed to convert string \"%.*s\" to UTF-8"), - utf_len, utf_buf); - return FAIL; - } - utf_buf = tofree; - } + assert(STRCMP(p_enc, "utf-8") == 0); size_t str_len = 0; - // Encode character as \u0000 if - // 1. It is an ASCII control character (0x0 .. 0x1F, 0x7F). - // 2. &encoding is not UTF-8 and code point is above 0x7F. - // 3. &encoding is UTF-8 and code point is not printable according to - // utf_printable(). - // This is done to make it possible to :echo values when &encoding is not - // UTF-8. -#define ENCODE_RAW(p_enc_conv, ch) \ - (ch >= 0x20 && (p_enc_conv.vc_type == CONV_NONE \ - ? utf_printable(ch) \ - : ch < 0x7F)) + // Encode character as \uNNNN if + // 1. It is an ASCII control character (0x0 .. 0x1F; 0x7F not + // utf_printable and thus not checked specially). + // 2. Code point is not printable according to utf_printable(). + // This is done to make resulting values displayable on screen also not from + // Neovim. +#define ENCODE_RAW(ch) \ + (ch >= 0x20 && utf_printable(ch)) for (size_t i = 0; i < utf_len;) { const int ch = utf_ptr2char(utf_buf + i); const size_t shift = (ch == 0? 1: utf_ptr2len(utf_buf + i)); @@ -636,7 +604,7 @@ static inline int convert_to_json_string(garray_T *const gap, utf_len - (i - shift), utf_buf + i - shift); xfree(tofree); return FAIL; - } else if (ENCODE_RAW(p_enc_conv, ch)) { + } else if (ENCODE_RAW(ch)) { str_len += shift; } else { str_len += ((sizeof("\\u1234") - 1) @@ -666,7 +634,7 @@ static inline int convert_to_json_string(garray_T *const gap, break; } default: { - if (ENCODE_RAW(p_enc_conv, ch)) { + if (ENCODE_RAW(ch)) { ga_concat_len(gap, utf_buf + i, shift); } else if (ch < SURROGATE_FIRST_CHAR) { ga_concat_len(gap, ((const char[]) { diff --git a/src/nvim/shada.c b/src/nvim/shada.c index 3c4fd8b992..cfe0bd8774 100644 --- a/src/nvim/shada.c +++ b/src/nvim/shada.c @@ -73,15 +73,10 @@ KHASH_SET_INIT_STR(strset) (vim_rename((char_u *)a, (char_u *)b)) #define mb_strnicmp(a, b, c) \ (mb_strnicmp((char_u *)a, (char_u *)b, c)) -#define has_non_ascii(a) (has_non_ascii((char_u *)a)) -#define string_convert(a, b, c) \ - ((char *)string_convert((vimconv_T *)a, (char_u *)b, c)) #define path_shorten_fname_if_possible(b) \ ((char *)path_shorten_fname_if_possible((char_u *)b)) #define buflist_new(ffname, sfname, ...) \ (buflist_new((char_u *)ffname, (char_u *)sfname, __VA_ARGS__)) -#define convert_setup(vcp, from, to) \ - (convert_setup(vcp, (char_u *)from, (char_u *)to)) #define os_isdir(f) (os_isdir((char_u *) f)) #define regtilde(s, m) ((char *) regtilde((char_u *) s, m)) #define path_tail_with_sep(f) ((char *) path_tail_with_sep((char_u *)f)) @@ -413,8 +408,6 @@ typedef struct sd_read_def { const char *error; ///< Error message in case of error. uintmax_t fpos; ///< Current position (amount of bytes read since ///< reader structure initialization). May overflow. - vimconv_T sd_conv; ///< Structure used for converting encodings of some - ///< items. } ShaDaReadDef; struct sd_write_def; @@ -435,8 +428,6 @@ typedef struct sd_write_def { ShaDaWriteCloser close; ///< Close function. void *cookie; ///< Data describing object written to. const char *error; ///< Error message in case of error. - vimconv_T sd_conv; ///< Structure used for converting encodings of some - ///< items. } ShaDaWriteDef; #ifdef INCLUDE_GENERATED_DECLARATIONS @@ -709,7 +700,6 @@ static ptrdiff_t write_file(ShaDaWriteDef *const sd_writer, static void close_sd_reader(ShaDaReadDef *const sd_reader) FUNC_ATTR_NONNULL_ALL { - convert_setup(&sd_reader->sd_conv, NULL, NULL); close_file(sd_reader->cookie); } @@ -717,7 +707,6 @@ static void close_sd_reader(ShaDaReadDef *const sd_reader) static void close_sd_writer(ShaDaWriteDef *const sd_writer) FUNC_ATTR_NONNULL_ALL { - convert_setup(&sd_writer->sd_conv, NULL, NULL); close_file(sd_writer->cookie); } @@ -800,13 +789,12 @@ static int open_shada_file_for_reading(const char *const fname, .eof = false, .fpos = 0, .cookie = file_open_new(&error, fname, kFileReadOnly, 0), - .sd_conv.vc_type = CONV_NONE, }; if (sd_reader->cookie == NULL) { return error; } - convert_setup(&sd_reader->sd_conv, "utf-8", p_enc); + assert(STRCMP(p_enc, "utf-8") == 0); return 0; } @@ -1902,127 +1890,24 @@ shada_pack_entry_error: } #undef PACK_STRING -/// Write single ShaDa entry, converting it if needed +/// Write single ShaDa entry and free it afterwards /// -/// @warning Frees entry after packing. +/// Will not free if entry could not be freed. /// /// @param[in] packer Packer used to write entry. -/// @param[in] sd_conv Conversion definitions. -/// @param[in] entry Entry written. If entry.can_free_entry is false then -/// it assumes that entry was not converted, otherwise it -/// is assumed that entry was already converted. +/// @param[in] entry Entry written. /// @param[in] max_kbyte Maximum size of an item in KiB. Zero means no /// restrictions. -static ShaDaWriteResult shada_pack_encoded_entry(msgpack_packer *const packer, - const vimconv_T *const sd_conv, - PossiblyFreedShadaEntry entry, - const size_t max_kbyte) - FUNC_ATTR_NONNULL_ALL +static inline ShaDaWriteResult shada_pack_pfreed_entry( + msgpack_packer *const packer, PossiblyFreedShadaEntry entry, + const size_t max_kbyte) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE { ShaDaWriteResult ret = kSDWriteSuccessfull; + ret = shada_pack_entry(packer, entry.data, max_kbyte); if (entry.can_free_entry) { - ret = shada_pack_entry(packer, entry.data, max_kbyte); shada_free_shada_entry(&entry.data); - return ret; } -#define RUN_WITH_CONVERTED_STRING(cstr, code) \ - do { \ - bool did_convert = false; \ - if (sd_conv->vc_type != CONV_NONE && has_non_ascii((cstr))) { \ - char *const converted_string = string_convert(sd_conv, (cstr), NULL); \ - if (converted_string != NULL) { \ - (cstr) = converted_string; \ - did_convert = true; \ - } \ - } \ - code \ - if (did_convert) { \ - xfree((cstr)); \ - } \ - } while (0) - switch (entry.data.type) { - case kSDItemUnknown: - case kSDItemMissing: { - assert(false); - } - case kSDItemSearchPattern: { - RUN_WITH_CONVERTED_STRING(entry.data.data.search_pattern.pat, { - ret = shada_pack_entry(packer, entry.data, max_kbyte); - }); - break; - } - case kSDItemHistoryEntry: { - RUN_WITH_CONVERTED_STRING(entry.data.data.history_item.string, { - ret = shada_pack_entry(packer, entry.data, max_kbyte); - }); - break; - } - case kSDItemSubString: { - RUN_WITH_CONVERTED_STRING(entry.data.data.sub_string.sub, { - ret = shada_pack_entry(packer, entry.data, max_kbyte); - }); - break; - } - case kSDItemVariable: { - if (sd_conv->vc_type != CONV_NONE) { - typval_T tgttv; - var_item_copy(sd_conv, &entry.data.data.global_var.value, &tgttv, - true, 0); - tv_clear(&entry.data.data.global_var.value); - entry.data.data.global_var.value = tgttv; - } - ret = shada_pack_entry(packer, entry.data, max_kbyte); - break; - } - case kSDItemRegister: { - bool did_convert = false; - if (sd_conv->vc_type != CONV_NONE) { - size_t first_non_ascii = 0; - for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) { - if (has_non_ascii(entry.data.data.reg.contents[i])) { - first_non_ascii = i; - did_convert = true; - break; - } - } - if (did_convert) { - entry.data.data.reg.contents = - xmemdup(entry.data.data.reg.contents, - (entry.data.data.reg.contents_size - * sizeof(entry.data.data.reg.contents[0]))); - for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) { - if (i >= first_non_ascii) { - entry.data.data.reg.contents[i] = get_converted_string( - sd_conv, - entry.data.data.reg.contents[i], - strlen(entry.data.data.reg.contents[i])); - } else { - entry.data.data.reg.contents[i] = - xstrdup(entry.data.data.reg.contents[i]); - } - } - } - } - ret = shada_pack_entry(packer, entry.data, max_kbyte); - if (did_convert) { - for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) { - xfree(entry.data.data.reg.contents[i]); - } - xfree(entry.data.data.reg.contents); - } - break; - } - case kSDItemHeader: - case kSDItemGlobalMark: - case kSDItemJump: - case kSDItemBufferList: - case kSDItemLocalMark: - case kSDItemChange: { - ret = shada_pack_entry(packer, entry.data, max_kbyte); - break; - } - } -#undef RUN_WITH_CONVERTED_STRING return ret; } @@ -2559,11 +2444,7 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, break; } typval_T tgttv; - if (sd_writer->sd_conv.vc_type != CONV_NONE) { - var_item_copy(&sd_writer->sd_conv, &vartv, &tgttv, true, 0); - } else { - tv_copy(&vartv, &tgttv); - } + tv_copy(&vartv, &tgttv); ShaDaWriteResult spe_ret; if ((spe_ret = shada_pack_entry(packer, (ShadaEntry) { .type = kSDItemVariable, @@ -2814,9 +2695,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, do { \ for (size_t i_ = 0; i_ < ARRAY_SIZE(wms_array); i_++) { \ if (wms_array[i_].data.type != kSDItemMissing) { \ - if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, \ - wms_array[i_], \ - max_kbyte) == kSDWriteFailed) { \ + if (shada_pack_pfreed_entry(packer, wms_array[i_], max_kbyte) \ + == kSDWriteFailed) { \ ret = kSDWriteFailed; \ goto shada_write_exit; \ } \ @@ -2826,8 +2706,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, PACK_WMS_ARRAY(wms->global_marks); PACK_WMS_ARRAY(wms->registers); for (size_t i = 0; i < wms->jumps_size; i++) { - if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, wms->jumps[i], - max_kbyte) == kSDWriteFailed) { + if (shada_pack_pfreed_entry(packer, wms->jumps[i], max_kbyte) + == kSDWriteFailed) { ret = kSDWriteFailed; goto shada_write_exit; } @@ -2835,8 +2715,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, #define PACK_WMS_ENTRY(wms_entry) \ do { \ if (wms_entry.data.type != kSDItemMissing) { \ - if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, wms_entry, \ - max_kbyte) == kSDWriteFailed) { \ + if (shada_pack_pfreed_entry(packer, wms_entry, max_kbyte) \ + == kSDWriteFailed) { \ ret = kSDWriteFailed; \ goto shada_write_exit; \ } \ @@ -2863,9 +2743,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, for (size_t i = 0; i < file_markss_to_dump; i++) { PACK_WMS_ARRAY(all_file_markss[i]->marks); for (size_t j = 0; j < all_file_markss[i]->changes_size; j++) { - if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, - all_file_markss[i]->changes[j], - max_kbyte) == kSDWriteFailed) { + if (shada_pack_pfreed_entry(packer, all_file_markss[i]->changes[j], + max_kbyte) == kSDWriteFailed) { ret = kSDWriteFailed; goto shada_write_exit; } @@ -2889,8 +2768,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, if (dump_one_history[i]) { hms_insert_whole_neovim_history(&wms->hms[i]); HMS_ITER(&wms->hms[i], cur_entry, { - if (shada_pack_encoded_entry( - packer, &sd_writer->sd_conv, (PossiblyFreedShadaEntry) { + if (shada_pack_pfreed_entry( + packer, (PossiblyFreedShadaEntry) { .data = cur_entry->data, .can_free_entry = cur_entry->can_free_entry, }, max_kbyte) == kSDWriteFailed) { @@ -2940,7 +2819,6 @@ int shada_write_file(const char *const file, bool nomerge) .write = &write_file, .close = &close_sd_writer, .error = NULL, - .sd_conv.vc_type = CONV_NONE, }; ShaDaReadDef sd_reader = { .close = NULL }; @@ -3042,7 +2920,7 @@ shada_write_file_nomerge: {} verbose_leave(); } - convert_setup(&sd_writer.sd_conv, p_enc, "utf-8"); + assert(STRCMP(p_enc, "utf-8") == 0); const ShaDaWriteResult sw_ret = shada_write(&sd_writer, (nomerge ? NULL @@ -3331,29 +3209,6 @@ static ShaDaReadResult msgpack_read_uint64(ShaDaReadDef *const sd_reader, return kSDReadStatusSuccess; } -/// Convert or copy and return a string -/// -/// @param[in] sd_conv Conversion definition. -/// @param[in] str String to convert. -/// @param[in] len String length. -/// -/// @return [allocated] converted string or copy of the original string. -static inline char *get_converted_string(const vimconv_T *const sd_conv, - const char *const str, - const size_t len) - FUNC_ATTR_NONNULL_ALL FUNC_ATTR_MALLOC FUNC_ATTR_WARN_UNUSED_RESULT -{ - if (!has_non_ascii_len(str, len)) { - return xmemdupz(str, len); - } - size_t new_len = len; - char *const new_str = string_convert(sd_conv, str, &new_len); - if (new_str == NULL) { - return xmemdupz(str, len); - } - return new_str; -} - #define READERR(entry_name, error_desc) \ RERR "Error while reading ShaDa file: " \ entry_name " entry at position %" PRIu64 " " \ @@ -3431,10 +3286,7 @@ static inline char *get_converted_string(const vimconv_T *const sd_conv, sizeof(*unpacked.data.via.map.ptr)); \ ad_ga.ga_len++; \ } -#define CONVERTED(str, len) ( \ - sd_reader->sd_conv.vc_type != CONV_NONE \ - ? get_converted_string(&sd_reader->sd_conv, (str), (len)) \ - : xmemdupz((str), (len))) +#define CONVERTED(str, len) (xmemdupz((str), (len))) #define BIN_CONVERTED(b) CONVERTED(b.ptr, b.size) #define SET_ADDITIONAL_DATA(tgt, name) \ do { \ @@ -3807,30 +3659,14 @@ shada_read_next_item_start: (char) unpacked.data.via.array.ptr[2].via.u64; } size_t strsize; - if (sd_reader->sd_conv.vc_type == CONV_NONE - || !has_non_ascii_len(unpacked.data.via.array.ptr[1].via.bin.ptr, - unpacked.data.via.array.ptr[1].via.bin.size)) { -shada_read_next_item_hist_no_conv: - strsize = ( - unpacked.data.via.array.ptr[1].via.bin.size - + 1 // Zero byte - + 1); // Separator character - entry->data.history_item.string = xmalloc(strsize); - memcpy(entry->data.history_item.string, - unpacked.data.via.array.ptr[1].via.bin.ptr, - unpacked.data.via.array.ptr[1].via.bin.size); - } else { - size_t len = unpacked.data.via.array.ptr[1].via.bin.size; - char *const converted = string_convert( - &sd_reader->sd_conv, unpacked.data.via.array.ptr[1].via.bin.ptr, - &len); - if (converted != NULL) { - strsize = len + 2; - entry->data.history_item.string = xrealloc(converted, strsize); - } else { - goto shada_read_next_item_hist_no_conv; - } - } + strsize = ( + unpacked.data.via.array.ptr[1].via.bin.size + + 1 // Zero byte + + 1); // Separator character + entry->data.history_item.string = xmalloc(strsize); + memcpy(entry->data.history_item.string, + unpacked.data.via.array.ptr[1].via.bin.ptr, + unpacked.data.via.array.ptr[1].via.bin.size); entry->data.history_item.string[strsize - 2] = 0; entry->data.history_item.string[strsize - 1] = entry->data.history_item.sep; @@ -3863,16 +3699,6 @@ shada_read_next_item_hist_no_conv: "be converted to the VimL value")), initial_fpos); goto shada_read_next_item_error; } - if (sd_reader->sd_conv.vc_type != CONV_NONE) { - typval_T tgttv; - var_item_copy(&sd_reader->sd_conv, - &entry->data.global_var.value, - &tgttv, - true, - 0); - tv_clear(&entry->data.global_var.value); - entry->data.global_var.value = tgttv; - } SET_ADDITIONAL_ELEMENTS(unpacked.data.via.array, 2, entry->data.global_var.additional_elements, "variable"); |