From bb6190bec5f18c1f9e2c1d29ef1f7cf7912ea625 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sat, 1 Jun 2024 08:19:41 -0700 Subject: refactor: move shared messages to errors.h #26214 --- src/nvim/lua/api_wrappers.c | 1 + src/nvim/lua/executor.c | 1 + src/nvim/lua/secure.c | 1 + src/nvim/lua/spell.c | 1 + 4 files changed, 4 insertions(+) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/api_wrappers.c b/src/nvim/lua/api_wrappers.c index 2b7b0c6471..36847d1fc9 100644 --- a/src/nvim/lua/api_wrappers.c +++ b/src/nvim/lua/api_wrappers.c @@ -5,6 +5,7 @@ #include "nvim/api/private/defs.h" #include "nvim/api/private/dispatch.h" #include "nvim/api/private/helpers.h" +#include "nvim/errors.h" #include "nvim/ex_docmd.h" #include "nvim/ex_getln.h" #include "nvim/func_attr.h" diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c index a76b8213e5..6246452b92 100644 --- a/src/nvim/lua/executor.c +++ b/src/nvim/lua/executor.c @@ -22,6 +22,7 @@ #include "nvim/cmdexpand_defs.h" #include "nvim/cursor.h" #include "nvim/drawscreen.h" +#include "nvim/errors.h" #include "nvim/eval.h" #include "nvim/eval/funcs.h" #include "nvim/eval/typval.h" diff --git a/src/nvim/lua/secure.c b/src/nvim/lua/secure.c index f62e0ace04..a84d6c4d65 100644 --- a/src/nvim/lua/secure.c +++ b/src/nvim/lua/secure.c @@ -3,6 +3,7 @@ #include #include "nvim/charset.h" +#include "nvim/errors.h" #include "nvim/ex_cmds_defs.h" #include "nvim/gettext_defs.h" #include "nvim/globals.h" diff --git a/src/nvim/lua/spell.c b/src/nvim/lua/spell.c index ba83239dc5..f4dacd7a55 100644 --- a/src/nvim/lua/spell.c +++ b/src/nvim/lua/spell.c @@ -7,6 +7,7 @@ #include "nvim/ascii_defs.h" #include "nvim/buffer_defs.h" +#include "nvim/errors.h" #include "nvim/gettext_defs.h" #include "nvim/globals.h" #include "nvim/highlight_defs.h" -- cgit From 200e7ad1578619e78c664bd0c6be024168433412 Mon Sep 17 00:00:00 2001 From: James Tirta Halim Date: Mon, 3 Jun 2024 11:10:30 +0700 Subject: fixup: apply the change on more files --- src/nvim/lua/secure.c | 4 ++-- src/nvim/lua/treesitter.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/secure.c b/src/nvim/lua/secure.c index a84d6c4d65..61277949c4 100644 --- a/src/nvim/lua/secure.c +++ b/src/nvim/lua/secure.c @@ -104,12 +104,12 @@ void ex_trust(exarg_T *eap) action = "deny"; } else if (strcmp(arg1, "++remove") == 0) { action = "remove"; - } else if (*arg1 != '\0') { + } else if (*arg1 != NUL) { semsg(e_invarg2, arg1); goto theend; } - if (path[0] == '\0') { + if (path[0] == NUL) { path = NULL; } diff --git a/src/nvim/lua/treesitter.c b/src/nvim/lua/treesitter.c index e87cf756a8..b13a162074 100644 --- a/src/nvim/lua/treesitter.c +++ b/src/nvim/lua/treesitter.c @@ -279,7 +279,7 @@ static const char *input_cb(void *payload, uint32_t byte_index, TSPoint position memcpy(buf, line + position.column, tocopy); // Translate embedded \n to NUL - memchrsub(buf, '\n', '\0', tocopy); + memchrsub(buf, '\n', NUL, tocopy); *bytes_read = (uint32_t)tocopy; if (tocopy < BUFSIZE) { // now add the final \n. If it didn't fit, input_cb will be called again -- cgit From 9afa1fd35510c5fe485f4a1dfdabf94e5f051a1c Mon Sep 17 00:00:00 2001 From: dundargoc Date: Mon, 3 Jun 2024 19:04:28 +0200 Subject: feat(lua): add `vim._with` It's a function to perform operations in their own sealed context, similar to pythons `with`. This helps ease operations where you need to perform an operation in a specific context, and then restore the context. Marked as private for now as it's not ready for public use. The current plan is to start using this internally so we can discover and fix any problems. Once this is ready to be exposed it will be renamed to `vim.with`. Usage: ```lua local ret = vim._with({context = val}, function() return "hello" end) ``` , where `context` is any combination of: - `buf` - `emsg_silent` - `hide` - `horizontal` - `keepalt` - `keepjumps` - `keepmarks` - `keeppatterns` - `lockmarks` - `noautocmd` - `options` - `sandbox` - `silent` - `unsilent` - `win` (except for `win` and `buf` which can't be used at the same time). This list will most likely be expanded in the future. Work on https://github.com/neovim/neovim/issues/19832. Co-authored-by: Lewis Russell --- src/nvim/lua/stdlib.c | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/stdlib.c b/src/nvim/lua/stdlib.c index 22ee0a1c98..ee0eabbebb 100644 --- a/src/nvim/lua/stdlib.c +++ b/src/nvim/lua/stdlib.c @@ -17,10 +17,13 @@ #include "nvim/api/private/defs.h" #include "nvim/api/private/helpers.h" #include "nvim/ascii_defs.h" +#include "nvim/autocmd.h" #include "nvim/buffer_defs.h" #include "nvim/eval/typval.h" #include "nvim/eval/typval_defs.h" #include "nvim/eval/vars.h" +#include "nvim/eval/window.h" +#include "nvim/ex_docmd.h" #include "nvim/ex_eval.h" #include "nvim/fold.h" #include "nvim/globals.h" @@ -40,6 +43,7 @@ #include "nvim/runtime.h" #include "nvim/strings.h" #include "nvim/types_defs.h" +#include "nvim/window.h" #ifdef INCLUDE_GENERATED_DECLARATIONS # include "lua/stdlib.c.generated.h" @@ -568,6 +572,99 @@ static int nlua_foldupdate(lua_State *lstate) return 0; } +static int nlua_with(lua_State *L) +{ + int flags = 0; + buf_T *buf = NULL; + win_T *win = NULL; + +#define APPLY_FLAG(key, flag) \ + if (strequal((key), k) && (v)) { \ + flags |= (flag); \ + } + + luaL_argcheck(L, lua_istable(L, 1), 1, "table expected"); + lua_pushnil(L); // [dict, ..., nil] + while (lua_next(L, 1)) { + // [dict, ..., key, value] + if (lua_type(L, -2) == LUA_TSTRING) { + const char *k = lua_tostring(L, -2); + bool v = lua_toboolean(L, -1); + if (strequal("buf", k)) { \ + buf = handle_get_buffer((int)luaL_checkinteger(L, -1)); + } else if (strequal("win", k)) { \ + win = handle_get_window((int)luaL_checkinteger(L, -1)); + } else { + APPLY_FLAG("sandbox", CMOD_SANDBOX); + APPLY_FLAG("silent", CMOD_SILENT); + APPLY_FLAG("emsg_silent", CMOD_ERRSILENT); + APPLY_FLAG("unsilent", CMOD_UNSILENT); + APPLY_FLAG("noautocmd", CMOD_NOAUTOCMD); + APPLY_FLAG("hide", CMOD_HIDE); + APPLY_FLAG("keepalt", CMOD_KEEPALT); + APPLY_FLAG("keepmarks", CMOD_KEEPMARKS); + APPLY_FLAG("keepjumps", CMOD_KEEPJUMPS); + APPLY_FLAG("lockmarks", CMOD_LOCKMARKS); + APPLY_FLAG("keeppatterns", CMOD_KEEPPATTERNS); + } + } + // pop the value; lua_next will pop the key. + lua_pop(L, 1); // [dict, ..., key] + } + int status = 0; + int rets = 0; + + cmdmod_T save_cmdmod = cmdmod; + cmdmod.cmod_flags = flags; + apply_cmdmod(&cmdmod); + + if (buf || win) { + try_start(); + } + + aco_save_T aco; + win_execute_T win_execute_args; + Error err = ERROR_INIT; + + if (win) { + tabpage_T *tabpage = win_find_tabpage(win); + if (!win_execute_before(&win_execute_args, win, tabpage)) { + goto end; + } + } else if (buf) { + aucmd_prepbuf(&aco, buf); + } + + int s = lua_gettop(L); + lua_pushvalue(L, 2); + status = lua_pcall(L, 0, LUA_MULTRET, 0); + rets = lua_gettop(L) - s; + + if (win) { + win_execute_after(&win_execute_args); + } else if (buf) { + aucmd_restbuf(&aco); + } + +end: + if (buf || win) { + try_end(&err); + } + + undo_cmdmod(&cmdmod); + cmdmod = save_cmdmod; + + if (status) { + return lua_error(L); + } else if (ERROR_SET(&err)) { + nlua_push_errstr(L, "%s", err.msg); + api_clear_error(&err); + return lua_error(L); + } + + return rets; +} + // Access to internal functions. For use in runtime/ static void nlua_state_add_internal(lua_State *const lstate) { @@ -582,6 +679,9 @@ static void nlua_state_add_internal(lua_State *const lstate) // _updatefolds lua_pushcfunction(lstate, &nlua_foldupdate); lua_setfield(lstate, -2, "_foldupdate"); + + lua_pushcfunction(lstate, &nlua_with); + lua_setfield(lstate, -2, "_with_c"); } void nlua_state_add_stdlib(lua_State *const lstate, bool is_thread) -- cgit From 948f2beed4ea55a9c2cce3cff894359b94fba748 Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Tue, 18 Jun 2024 09:47:10 +0800 Subject: fix(lua): find length of completion prefix earlier (#29384) Do the expansion right after setting the expand context, so that the length of the completion prefix can be set, but don't do that directly in set_one_cmd_context(), as that's also called by getcmdcompltype(). --- src/nvim/lua/executor.c | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c index 6246452b92..af218443b9 100644 --- a/src/nvim/lua/executor.c +++ b/src/nvim/lua/executor.c @@ -1924,10 +1924,14 @@ static void nlua_add_treesitter(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL lua_setfield(lstate, -2, "_ts_get_minimum_language_version"); } -int nlua_expand_pat(expand_T *xp, char *pat, int *num_results, char ***results) +static garray_T expand_result_array = GA_EMPTY_INIT_VALUE; + +/// Finds matches for Lua cmdline completion and advances xp->xp_pattern after prefix. +/// This should be called before xp->xp_pattern is first used. +void nlua_expand_pat(expand_T *xp, const char *pat) { lua_State *const lstate = global_lstate; - int ret = OK; + int status = OK; // [ vim ] lua_getglobal(lstate, "vim"); @@ -1942,54 +1946,54 @@ int nlua_expand_pat(expand_T *xp, char *pat, int *num_results, char ***results) if (nlua_pcall(lstate, 1, 2) != 0) { nlua_error(lstate, _("Error executing vim._expand_pat: %.*s")); - return FAIL; + return; } Error err = ERROR_INIT; - *num_results = 0; - *results = NULL; - Arena arena = ARENA_EMPTY; int prefix_len = (int)nlua_pop_Integer(lstate, &arena, &err); if (ERROR_SET(&err)) { - ret = FAIL; + status = FAIL; goto cleanup; } Array completions = nlua_pop_Array(lstate, &arena, &err); if (ERROR_SET(&err)) { - ret = FAIL; + status = FAIL; goto cleanup_array; } - garray_T result_array; - ga_init(&result_array, (int)sizeof(char *), 80); + ga_clear(&expand_result_array); + ga_init(&expand_result_array, (int)sizeof(char *), 80); for (size_t i = 0; i < completions.size; i++) { Object v = completions.items[i]; if (v.type != kObjectTypeString) { - ret = FAIL; + status = FAIL; goto cleanup_array; } - GA_APPEND(char *, &result_array, string_to_cstr(v.data.string)); + GA_APPEND(char *, &expand_result_array, string_to_cstr(v.data.string)); } xp->xp_pattern += prefix_len; - *results = result_array.ga_data; - *num_results = result_array.ga_len; cleanup_array: arena_mem_free(arena_finish(&arena)); cleanup: - - if (ret == FAIL) { - ga_clear(&result_array); + if (status == FAIL) { + ga_clear(&expand_result_array); } +} - return ret; +int nlua_expand_get_matches(int *num_results, char ***results) +{ + *results = expand_result_array.ga_data; + *num_results = expand_result_array.ga_len; + expand_result_array = (garray_T)GA_EMPTY_INIT_VALUE; + return *num_results > 0; } static int nlua_is_thread(lua_State *lstate) -- cgit From b0c336eaf8e7dd0e52e08195f46fd309fc138ea1 Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Tue, 18 Jun 2024 12:02:31 +0800 Subject: refactor(lua): remove unnecessary strlen() in nlua_expand_pat() (#29388) Also change the initial value of `status` to `FAIL`, as that'll avoid unnecessary assignments. --- src/nvim/lua/executor.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c index af218443b9..bac6f4ca9a 100644 --- a/src/nvim/lua/executor.c +++ b/src/nvim/lua/executor.c @@ -1931,7 +1931,7 @@ static garray_T expand_result_array = GA_EMPTY_INIT_VALUE; void nlua_expand_pat(expand_T *xp, const char *pat) { lua_State *const lstate = global_lstate; - int status = OK; + int status = FAIL; // [ vim ] lua_getglobal(lstate, "vim"); @@ -1940,12 +1940,11 @@ void nlua_expand_pat(expand_T *xp, const char *pat) lua_getfield(lstate, -1, "_expand_pat"); luaL_checktype(lstate, -1, LUA_TFUNCTION); - // [ vim, vim._expand_pat, buf ] - lua_pushlstring(lstate, pat, strlen(pat)); + // [ vim, vim._expand_pat, pat ] + lua_pushstring(lstate, pat); if (nlua_pcall(lstate, 1, 2) != 0) { - nlua_error(lstate, - _("Error executing vim._expand_pat: %.*s")); + nlua_error(lstate, _("Error executing vim._expand_pat: %.*s")); return; } @@ -1954,30 +1953,27 @@ void nlua_expand_pat(expand_T *xp, const char *pat) Arena arena = ARENA_EMPTY; int prefix_len = (int)nlua_pop_Integer(lstate, &arena, &err); if (ERROR_SET(&err)) { - status = FAIL; goto cleanup; } Array completions = nlua_pop_Array(lstate, &arena, &err); if (ERROR_SET(&err)) { - status = FAIL; goto cleanup_array; } ga_clear(&expand_result_array); ga_init(&expand_result_array, (int)sizeof(char *), 80); + for (size_t i = 0; i < completions.size; i++) { Object v = completions.items[i]; - if (v.type != kObjectTypeString) { - status = FAIL; goto cleanup_array; } - GA_APPEND(char *, &expand_result_array, string_to_cstr(v.data.string)); } xp->xp_pattern += prefix_len; + status = OK; cleanup_array: arena_mem_free(arena_finish(&arena)); -- cgit From 2bb1a18631c4035e4a582b7d995968acbac874bf Mon Sep 17 00:00:00 2001 From: bfredl Date: Sat, 15 Jun 2024 19:50:44 +0200 Subject: refactor(typval): don't use msgpack_packer for msgpackdump() Step towords completely eliminating msgpack_packer. --- src/nvim/lua/converter.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c index 38ccb03cfc..1b1794e244 100644 --- a/src/nvim/lua/converter.c +++ b/src/nvim/lua/converter.c @@ -425,6 +425,8 @@ static bool typval_conv_special = false; #define TYPVAL_ENCODE_ALLOW_SPECIALS true +#define TYPVAL_ENCODE_CHECK_BEFORE + #define TYPVAL_ENCODE_CONV_NIL(tv) \ do { \ if (typval_conv_special) { \ @@ -574,6 +576,7 @@ static bool typval_conv_special = false; #undef TYPVAL_ENCODE_CONV_LIST_START #undef TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START #undef TYPVAL_ENCODE_CONV_EMPTY_DICT +#undef TYPVAL_ENCODE_CHECK_BEFORE #undef TYPVAL_ENCODE_CONV_NIL #undef TYPVAL_ENCODE_CONV_BOOL #undef TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER -- cgit From bda63d5b97dfb333de6f4bd757dbb978906062a2 Mon Sep 17 00:00:00 2001 From: bfredl Date: Tue, 25 Jun 2024 15:33:47 +0200 Subject: refactor(typval)!: remove distinction of binary and nonbinary strings This is a breaking change which will make refactor of typval and shada code a lot easier. In particular, code that would use or check for v:msgpack_types.binary in the wild would be broken. This appears to be rarely used in existing plugins. Also some cases where v:msgpack_type.string would be used to represent a binary string of "string" type, we use a BLOB instead, which is vimscripts native type for binary blobs, and already was used for BIN formats when necessary. msgpackdump(msgpackparse(data)) no longer preserves the distinction of BIN and STR strings. This is very common behavior for language-specific msgpack bindings. Nvim uses msgpack as a tool to serialize its data. Nvim is not a tool to bit-perfectly manipulate arbitrary msgpack data out in the wild. The changed tests should indicate how behavior changes in various edge cases. --- src/nvim/lua/converter.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c index 1b1794e244..c8ad6606bf 100644 --- a/src/nvim/lua/converter.c +++ b/src/nvim/lua/converter.c @@ -219,12 +219,7 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv) if (cur.special) { list_T *const kv_pair = tv_list_alloc(2); - typval_T s_tv = decode_string(s, len, kTrue, false, false); - if (s_tv.v_type == VAR_UNKNOWN) { - ret = false; - tv_list_unref(kv_pair); - continue; - } + typval_T s_tv = decode_string(s, len, true, false); tv_list_append_owned_tv(kv_pair, s_tv); // Value: not populated yet, need to create list item to push. @@ -280,10 +275,7 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv) case LUA_TSTRING: { size_t len; const char *s = lua_tolstring(lstate, -1, &len); - *cur.tv = decode_string(s, len, kNone, true, false); - if (cur.tv->v_type == VAR_UNKNOWN) { - ret = false; - } + *cur.tv = decode_string(s, len, false, false); break; } case LUA_TNUMBER: { -- cgit From bdc6e38781321895331057cbcfb099f8ad31e6db Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Sat, 6 Jul 2024 22:25:35 +0800 Subject: fix(lua): don't include text after cursor in completion pattern (#29587) --- src/nvim/lua/executor.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c index bac6f4ca9a..c29e670c33 100644 --- a/src/nvim/lua/executor.c +++ b/src/nvim/lua/executor.c @@ -1928,7 +1928,7 @@ static garray_T expand_result_array = GA_EMPTY_INIT_VALUE; /// Finds matches for Lua cmdline completion and advances xp->xp_pattern after prefix. /// This should be called before xp->xp_pattern is first used. -void nlua_expand_pat(expand_T *xp, const char *pat) +void nlua_expand_pat(expand_T *xp) { lua_State *const lstate = global_lstate; int status = FAIL; @@ -1941,7 +1941,10 @@ void nlua_expand_pat(expand_T *xp, const char *pat) luaL_checktype(lstate, -1, LUA_TFUNCTION); // [ vim, vim._expand_pat, pat ] - lua_pushstring(lstate, pat); + const char *pat = xp->xp_pattern; + assert(xp->xp_line + xp->xp_col >= pat); + ptrdiff_t patlen = xp->xp_line + xp->xp_col - pat; + lua_pushlstring(lstate, pat, (size_t)patlen); if (nlua_pcall(lstate, 1, 2) != 0) { nlua_error(lstate, _("Error executing vim._expand_pat: %.*s")); @@ -1951,8 +1954,8 @@ void nlua_expand_pat(expand_T *xp, const char *pat) Error err = ERROR_INIT; Arena arena = ARENA_EMPTY; - int prefix_len = (int)nlua_pop_Integer(lstate, &arena, &err); - if (ERROR_SET(&err)) { + ptrdiff_t prefix_len = nlua_pop_Integer(lstate, &arena, &err); + if (ERROR_SET(&err) || prefix_len > patlen) { goto cleanup; } -- cgit From 16f63b964fc4a7842fd9a2fd5e0ba8c997d549c9 Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Sat, 13 Jul 2024 04:14:29 +0800 Subject: fix(input): handle vim.on_key() properly with ALT and K_SPECIAL (#29677) --- src/nvim/lua/executor.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c index c29e670c33..4d5b2c3698 100644 --- a/src/nvim/lua/executor.c +++ b/src/nvim/lua/executor.c @@ -2057,10 +2057,11 @@ char *nlua_register_table_as_callable(const typval_T *const arg) return name; } -void nlua_execute_on_key(int c, char *typed_buf, size_t typed_len) +void nlua_execute_on_key(int c, char *typed_buf) { char buf[MB_MAXBYTES * 3 + 4]; size_t buf_len = special_to_buf(c, mod_mask, false, buf); + vim_unescape_ks(typed_buf); lua_State *const lstate = global_lstate; @@ -2079,7 +2080,7 @@ void nlua_execute_on_key(int c, char *typed_buf, size_t typed_len) lua_pushlstring(lstate, buf, buf_len); // [ vim, vim._on_key, buf, typed_buf ] - lua_pushlstring(lstate, typed_buf, typed_len); + lua_pushstring(lstate, typed_buf); int save_got_int = got_int; got_int = false; // avoid interrupts when the key typed is Ctrl-C -- cgit From 970a27927eb31bdc735f0d7d5e3d07784486c6de Mon Sep 17 00:00:00 2001 From: Amit Singh Date: Thu, 11 Jul 2024 15:56:52 +0530 Subject: fix(lua)!: do not use typed table for empty dict Problem: Empty dictionaries are converted into typed tables of the form `{ [true] = 6}` instead of an empty dictionary representation `{}`. This leads to incorrect table representation, along with failure in JSON encoding of such tables as currently tables with only string and number type keys can be encoded. Solution: The typed table logic has been removed from `nlua_push_Dictionary`. The typed table logic is required only for float value conversions which is already handled in `nlua_push_Float`. So, it is(was) no longer required here. Fixes neovim/neovim#29218 --- src/nvim/lua/converter.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c index c8ad6606bf..c879d731bc 100644 --- a/src/nvim/lua/converter.c +++ b/src/nvim/lua/converter.c @@ -703,14 +703,10 @@ void nlua_push_Boolean(lua_State *lstate, const Boolean b, int flags) void nlua_push_Dictionary(lua_State *lstate, const Dictionary dict, int flags) FUNC_ATTR_NONNULL_ALL { - if (dict.size == 0 && (flags & kNluaPushSpecial)) { - nlua_create_typed_table(lstate, 0, 0, kObjectTypeDictionary); - } else { - lua_createtable(lstate, 0, (int)dict.size); - if (dict.size == 0 && !(flags & kNluaPushSpecial)) { - nlua_pushref(lstate, nlua_global_refs->empty_dict_ref); - lua_setmetatable(lstate, -2); - } + lua_createtable(lstate, 0, (int)dict.size); + if (dict.size == 0) { + nlua_pushref(lstate, nlua_global_refs->empty_dict_ref); + lua_setmetatable(lstate, -2); } for (size_t i = 0; i < dict.size; i++) { nlua_push_String(lstate, dict.items[i].key, flags); -- cgit From 688b961d13bd54a14836f08c3ded3121d3fb15a0 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Fri, 19 Apr 2024 16:04:57 +0100 Subject: feat(treesitter): add support for wasm parsers Problem: Installing treesitter parser is hard (harder than climbing to heaven). Solution: Add optional support for wasm parsers with `wasmtime`. Notes: * Needs to be enabled by setting `ENABLE_WASMTIME` for tree-sitter and Neovim. Build with `make CMAKE_EXTRA_FLAGS=-DENABLE_WASMTIME=ON DEPS_CMAKE_FLAGS=-DENABLE_WASMTIME=ON` * Adds optional Rust (obviously) and C11 dependencies. * Wasmtime comes with a lot of features that can negatively affect Neovim performance due to library and symbol table size. Make sure to build with minimal features and full LTO. * To reduce re-compilation times, install `sccache` and build with `RUSTC_WRAPPER= make ...` --- src/nvim/lua/executor.c | 10 +++- src/nvim/lua/treesitter.c | 146 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 145 insertions(+), 11 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c index 4d5b2c3698..d4940f3add 100644 --- a/src/nvim/lua/executor.c +++ b/src/nvim/lua/executor.c @@ -924,6 +924,7 @@ void nlua_free_all_mem(void) lua_State *lstate = global_lstate; nlua_unref_global(lstate, require_ref); nlua_common_free_all_mem(lstate); + tslua_free(); } static void nlua_common_free_all_mem(lua_State *lstate) @@ -1902,8 +1903,13 @@ static void nlua_add_treesitter(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL lua_pushcfunction(lstate, tslua_push_querycursor); lua_setfield(lstate, -2, "_create_ts_querycursor"); - lua_pushcfunction(lstate, tslua_add_language); - lua_setfield(lstate, -2, "_ts_add_language"); + lua_pushcfunction(lstate, tslua_add_language_from_object); + lua_setfield(lstate, -2, "_ts_add_language_from_object"); + +#ifdef HAVE_WASMTIME + lua_pushcfunction(lstate, tslua_add_language_from_wasm); + lua_setfield(lstate, -2, "_ts_add_language_from_wasm"); +#endif lua_pushcfunction(lstate, tslua_has_language); lua_setfield(lstate, -2, "_ts_has_language"); diff --git a/src/nvim/lua/treesitter.c b/src/nvim/lua/treesitter.c index b13a162074..eb92ae8d0e 100644 --- a/src/nvim/lua/treesitter.c +++ b/src/nvim/lua/treesitter.c @@ -15,6 +15,10 @@ #include #include +#ifdef HAVE_WASMTIME +# include +#endif + #include "klib/kvec.h" #include "nvim/api/private/helpers.h" #include "nvim/buffer_defs.h" @@ -24,6 +28,7 @@ #include "nvim/map_defs.h" #include "nvim/memline.h" #include "nvim/memory.h" +#include "nvim/os/fs.h" #include "nvim/pos_defs.h" #include "nvim/strings.h" #include "nvim/types_defs.h" @@ -53,6 +58,11 @@ typedef struct { static PMap(cstr_t) langs = MAP_INIT; +#ifdef HAVE_WASMTIME +static wasm_engine_t *wasmengine; +static TSWasmStore *ts_wasmstore; +#endif + // TSLanguage int tslua_has_language(lua_State *L) @@ -62,8 +72,59 @@ int tslua_has_language(lua_State *L) return 1; } -static TSLanguage *load_language(lua_State *L, const char *path, const char *lang_name, - const char *symbol) +#ifdef HAVE_WASMTIME +static char *read_file(const char *path, size_t *len) + FUNC_ATTR_MALLOC +{ + FILE *file = os_fopen(path, "r"); + if (file == NULL) { + return NULL; + } + fseek(file, 0L, SEEK_END); + *len = (size_t)ftell(file); + fseek(file, 0L, SEEK_SET); + char *data = xmalloc(*len); + if (fread(data, *len, 1, file) != 1) { + xfree(data); + fclose(file); + return NULL; + } + fclose(file); + return data; +} + +static const char *wasmerr_to_str(TSWasmErrorKind werr) +{ + switch (werr) { + case TSWasmErrorKindParse: + return "PARSE"; + case TSWasmErrorKindCompile: + return "COMPILE"; + case TSWasmErrorKindInstantiate: + return "INSTANTIATE"; + case TSWasmErrorKindAllocate: + return "ALLOCATE"; + default: + return "UNKNOWN"; + } +} +#endif + +int tslua_add_language_from_wasm(lua_State *L) +{ + return add_language(L, true); +} + +// Creates the language into the internal language map. +// +// Returns true if the language is correctly loaded in the language map +int tslua_add_language_from_object(lua_State *L) +{ + return add_language(L, false); +} + +static const TSLanguage *load_language_from_object(lua_State *L, const char *path, + const char *lang_name, const char *symbol) { uv_lib_t lib; if (uv_dlopen(path, &lib)) { @@ -91,16 +152,59 @@ static TSLanguage *load_language(lua_State *L, const char *path, const char *lan return lang; } -// Creates the language into the internal language map. -// -// Returns true if the language is correctly loaded in the language map -int tslua_add_language(lua_State *L) +static const TSLanguage *load_language_from_wasm(lua_State *L, const char *path, + const char *lang_name) +{ +#ifndef HAVE_WASMTIME + luaL_error(L, "Not supported"); + return NULL; +#else + if (wasmengine == NULL) { + wasmengine = wasm_engine_new(); + } + assert(wasmengine != NULL); + + TSWasmError werr = { 0 }; + if (ts_wasmstore == NULL) { + ts_wasmstore = ts_wasm_store_new(wasmengine, &werr); + } + + if (werr.kind > 0) { + luaL_error(L, "Error creating wasm store: (%s) %s", wasmerr_to_str(werr.kind), werr.message); + } + + size_t file_size = 0; + char *data = read_file(path, &file_size); + + if (data == NULL) { + luaL_error(L, "Unable to read file", path); + } + + const TSLanguage *lang = ts_wasm_store_load_language(ts_wasmstore, lang_name, data, + (uint32_t)file_size, &werr); + + xfree(data); + + if (werr.kind > 0) { + luaL_error(L, "Failed to load WASM parser %s: (%s) %s", path, wasmerr_to_str(werr.kind), + werr.message); + } + + if (lang == NULL) { + luaL_error(L, "Failed to load parser %s: internal error", path); + } + + return lang; +#endif +} + +static int add_language(lua_State *L, bool is_wasm) { const char *path = luaL_checkstring(L, 1); const char *lang_name = luaL_checkstring(L, 2); const char *symbol_name = lang_name; - if (lua_gettop(L) >= 3 && !lua_isnil(L, 3)) { + if (!is_wasm && lua_gettop(L) >= 3 && !lua_isnil(L, 3)) { symbol_name = luaL_checkstring(L, 3); } @@ -109,7 +213,9 @@ int tslua_add_language(lua_State *L) return 1; } - TSLanguage *lang = load_language(L, path, lang_name, symbol_name); + const TSLanguage *lang = is_wasm + ? load_language_from_wasm(L, path, lang_name) + : load_language_from_object(L, path, lang_name, symbol_name); uint32_t lang_version = ts_language_version(lang); if (lang_version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION @@ -121,7 +227,7 @@ int tslua_add_language(lua_State *L) TREE_SITTER_LANGUAGE_VERSION, lang_version); } - pmap_put(cstr_t)(&langs, xstrdup(lang_name), lang); + pmap_put(cstr_t)(&langs, xstrdup(lang_name), (TSLanguage *)lang); lua_pushboolean(L, true); return 1; @@ -186,6 +292,9 @@ int tslua_inspect_lang(lua_State *L) lua_setfield(L, -2, "fields"); // [retval] + lua_pushboolean(L, ts_language_is_wasm(lang)); + lua_setfield(L, -2, "_wasm"); + lua_pushinteger(L, ts_language_version(lang)); // [retval, version] lua_setfield(L, -2, "_abi_version"); @@ -215,6 +324,13 @@ int tslua_push_parser(lua_State *L) TSParser **parser = lua_newuserdata(L, sizeof(TSParser *)); *parser = ts_parser_new(); +#ifdef HAVE_WASMTIME + if (ts_language_is_wasm(lang)) { + assert(wasmengine != NULL); + ts_parser_set_wasm_store(*parser, ts_wasmstore); + } +#endif + if (!ts_parser_set_language(*parser, lang)) { ts_parser_delete(*parser); const char *lang_name = luaL_checkstring(L, 1); @@ -1561,3 +1677,15 @@ void tslua_init(lua_State *L) ts_set_allocator(xmalloc, xcalloc, xrealloc, xfree); } + +void tslua_free(void) +{ +#ifdef HAVE_WASMTIME + if (wasmengine != NULL) { + wasm_engine_delete(wasmengine); + } + if (ts_wasmstore != NULL) { + ts_wasm_store_delete(ts_wasmstore); + } +#endif +} -- cgit From 176bfea1356bd151a8aaef61e02ac5a175969a59 Mon Sep 17 00:00:00 2001 From: bfredl Date: Mon, 8 Jan 2024 14:34:49 +0100 Subject: fix(build): issues with s390x CI Does not fix everything, but at least let's test run to finish before timeout --- src/nvim/lua/xdiff.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/xdiff.c b/src/nvim/lua/xdiff.c index 035c171a14..8d791a7e74 100644 --- a/src/nvim/lua/xdiff.c +++ b/src/nvim/lua/xdiff.c @@ -185,7 +185,12 @@ static mmfile_t get_string_arg(lua_State *lstate, int idx) luaL_argerror(lstate, idx, "expected string"); } mmfile_t mf; - mf.ptr = (char *)lua_tolstring(lstate, idx, (size_t *)&mf.size); + size_t size; + mf.ptr = (char *)lua_tolstring(lstate, idx, &size); + if (size > INT_MAX) { + luaL_argerror(lstate, idx, "string too long"); + } + mf.size = (int)size; return mf; } -- cgit From 737f58e23230ea14f1648ac1fc7f442ea0f8563c Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Fri, 20 Sep 2024 07:34:50 +0200 Subject: refactor(api)!: rename Dictionary => Dict In the api_info() output: :new|put =map(filter(api_info().functions, '!has_key(v:val,''deprecated_since'')'), 'v:val') ... {'return_type': 'ArrayOf(Integer, 2)', 'name': 'nvim_win_get_position', 'method': v:true, 'parameters': [['Window', 'window']], 'since': 1} The `ArrayOf(Integer, 2)` return type didn't break clients when we added it, which is evidence that clients don't use the `return_type` field, thus renaming Dictionary => Dict in api_info() is not (in practice) a breaking change. --- src/nvim/lua/converter.c | 149 +++++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 76 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c index c879d731bc..027ae61007 100644 --- a/src/nvim/lua/converter.c +++ b/src/nvim/lua/converter.c @@ -26,13 +26,13 @@ #include "nvim/types_defs.h" #include "nvim/vim_defs.h" -/// Determine, which keys lua table contains +/// Determine, which keys Lua table contains typedef struct { size_t maxidx; ///< Maximum positive integral value found. size_t string_keys_num; ///< Number of string keys. bool has_string_with_nul; ///< True if there is string key with NUL byte. ObjectType type; ///< If has_type_key is true then attached value. Otherwise - ///< either kObjectTypeNil, kObjectTypeDictionary or + ///< either kObjectTypeNil, kObjectTypeDict or ///< kObjectTypeArray, depending on other properties. lua_Number val; ///< If has_val_key and val_type == LUA_TNUMBER: value. bool has_type_key; ///< True if type key is present. @@ -52,7 +52,7 @@ static LuaTableProps nlua_traverse_table(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT { size_t tsize = 0; // Total number of keys. - int val_type = 0; // If has_val_key: lua type of the value. + int val_type = 0; // If has_val_key: Lua type of the value. bool has_val_key = false; // True if val key was found, // @see nlua_push_val_idx(). size_t other_keys_num = 0; // Number of keys that are not string, integral @@ -96,7 +96,7 @@ static LuaTableProps nlua_traverse_table(lua_State *const lstate) lua_Number n = lua_tonumber(lstate, -1); if (n == (lua_Number)kObjectTypeFloat || n == (lua_Number)kObjectTypeArray - || n == (lua_Number)kObjectTypeDictionary) { + || n == (lua_Number)kObjectTypeDict) { ret.has_type_key = true; ret.type = (ObjectType)n; } else { @@ -156,12 +156,12 @@ static LuaTableProps nlua_traverse_table(lua_State *const lstate) if (tsize == 0 && lua_getmetatable(lstate, -1)) { nlua_pushref(lstate, nlua_global_refs->empty_dict_ref); if (lua_rawequal(lstate, -2, -1)) { - ret.type = kObjectTypeDictionary; + ret.type = kObjectTypeDict; } lua_pop(lstate, 2); } } else if (ret.string_keys_num == tsize) { - ret.type = kObjectTypeDictionary; + ret.type = kObjectTypeDict; } else { ret.type = kObjectTypeNil; } @@ -174,14 +174,14 @@ typedef struct { typval_T *tv; ///< Location where conversion result is saved. size_t list_len; ///< Maximum length when tv is a list. bool container; ///< True if tv is a container. - bool special; ///< If true then tv is a _VAL part of special dictionary + bool special; ///< If true then tv is a _VAL part of special dict. ///< that represents mapping. int idx; ///< Container index (used to detect self-referencing structures). } TVPopStackItem; -/// Convert lua object to Vimscript typval_T +/// Convert Lua object to Vimscript typval_T /// -/// Should pop exactly one value from lua stack. +/// Should pop exactly one value from Lua stack. /// /// @param lstate Lua state. /// @param[out] ret_tv Where to put the result. @@ -322,7 +322,7 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv) kvi_push(stack, cur); } break; - case kObjectTypeDictionary: + case kObjectTypeDict: if (table_props.string_keys_num == 0) { cur.tv->v_type = VAR_DICT; cur.tv->vval.v_dict = tv_dict_alloc(); @@ -357,7 +357,7 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv) cur.tv->vval.v_float = (float_T)table_props.val; break; case kObjectTypeNil: - emsg(_("E5100: Cannot convert given lua table: table should " + emsg(_("E5100: Cannot convert given Lua table: table should " "contain either only integer keys or only string keys")); ret = false; break; @@ -385,13 +385,13 @@ nlua_pop_typval_table_processing_end: cur.tv->v_type = VAR_SPECIAL; cur.tv->vval.v_special = kSpecialVarNull; } else { - emsg(_("E5101: Cannot convert given lua type")); + emsg(_("E5101: Cannot convert given Lua type")); ret = false; } break; } default: - emsg(_("E5101: Cannot convert given lua type")); + emsg(_("E5101: Cannot convert given Lua type")); ret = false; break; } @@ -474,7 +474,7 @@ static bool typval_conv_special = false; #define TYPVAL_ENCODE_CONV_EMPTY_DICT(tv, dict) \ do { \ if (typval_conv_special) { \ - nlua_create_typed_table(lstate, 0, 0, kObjectTypeDictionary); \ + nlua_create_typed_table(lstate, 0, 0, kObjectTypeDict); \ } else { \ lua_createtable(lstate, 0, 0); \ nlua_pushref(lstate, nlua_global_refs->empty_dict_ref); \ @@ -583,10 +583,9 @@ static bool typval_conv_special = false; #undef TYPVAL_ENCODE_CONV_RECURSE #undef TYPVAL_ENCODE_ALLOW_SPECIALS -/// Convert Vimscript typval_T to lua value +/// Convert Vimscript typval_T to Lua value /// -/// Should leave single value in lua stack. May only fail if lua failed to grow -/// stack. +/// Should leave single value in Lua stack. May only fail if Lua failed to grow stack. /// /// @param lstate Lua interpreter state. /// @param[in] tv typval_T to convert. @@ -654,7 +653,7 @@ static inline void nlua_create_typed_table(lua_State *lstate, const size_t narr, lua_rawset(lstate, -3); } -/// Convert given String to lua string +/// Convert given String to Lua string /// /// Leaves converted string on top of the stack. void nlua_push_String(lua_State *lstate, const String s, int flags) @@ -663,7 +662,7 @@ void nlua_push_String(lua_State *lstate, const String s, int flags) lua_pushlstring(lstate, s.data, s.size); } -/// Convert given Integer to lua number +/// Convert given Integer to Lua number /// /// Leaves converted number on top of the stack. void nlua_push_Integer(lua_State *lstate, const Integer n, int flags) @@ -672,7 +671,7 @@ void nlua_push_Integer(lua_State *lstate, const Integer n, int flags) lua_pushnumber(lstate, (lua_Number)n); } -/// Convert given Float to lua table +/// Convert given Float to Lua table /// /// Leaves converted table on top of the stack. void nlua_push_Float(lua_State *lstate, const Float f, int flags) @@ -688,7 +687,7 @@ void nlua_push_Float(lua_State *lstate, const Float f, int flags) } } -/// Convert given Float to lua boolean +/// Convert given Float to Lua boolean /// /// Leaves converted value on top of the stack. void nlua_push_Boolean(lua_State *lstate, const Boolean b, int flags) @@ -697,10 +696,10 @@ void nlua_push_Boolean(lua_State *lstate, const Boolean b, int flags) lua_pushboolean(lstate, b); } -/// Convert given Dictionary to lua table +/// Convert given Dict to Lua table /// /// Leaves converted table on top of the stack. -void nlua_push_Dictionary(lua_State *lstate, const Dictionary dict, int flags) +void nlua_push_Dict(lua_State *lstate, const Dict dict, int flags) FUNC_ATTR_NONNULL_ALL { lua_createtable(lstate, 0, (int)dict.size); @@ -715,7 +714,7 @@ void nlua_push_Dictionary(lua_State *lstate, const Dictionary dict, int flags) } } -/// Convert given Array to lua table +/// Convert given Array to Lua table /// /// Leaves converted table on top of the stack. void nlua_push_Array(lua_State *lstate, const Array array, int flags) @@ -741,7 +740,7 @@ GENERATE_INDEX_FUNCTION(Tabpage) #undef GENERATE_INDEX_FUNCTION -/// Convert given Object to lua value +/// Convert given Object to Lua value /// /// Leaves converted value on top of the stack. void nlua_push_Object(lua_State *lstate, Object *obj, int flags) @@ -768,12 +767,12 @@ void nlua_push_Object(lua_State *lstate, Object *obj, int flags) nlua_push_##type(lstate, obj->data.data_key, flags); \ break; \ } - ADD_TYPE(Boolean, boolean) - ADD_TYPE(Integer, integer) - ADD_TYPE(Float, floating) - ADD_TYPE(String, string) - ADD_TYPE(Array, array) - ADD_TYPE(Dictionary, dictionary) + ADD_TYPE(Boolean, boolean) + ADD_TYPE(Integer, integer) + ADD_TYPE(Float, floating) + ADD_TYPE(String, string) + ADD_TYPE(Array, array) + ADD_TYPE(Dict, dict) #undef ADD_TYPE #define ADD_REMOTE_TYPE(type) \ case kObjectType##type: { \ @@ -787,7 +786,7 @@ void nlua_push_Object(lua_State *lstate, Object *obj, int flags) } } -/// Convert lua value to string +/// Convert Lua value to string /// /// Always pops one value from the stack. String nlua_pop_String(lua_State *lstate, Arena *arena, Error *err) @@ -802,16 +801,16 @@ String nlua_pop_String(lua_State *lstate, Arena *arena, Error *err) ret.data = (char *)lua_tolstring(lstate, -1, &(ret.size)); assert(ret.data != NULL); - // TODO(bfredl): it would be "nice" to just use the memory of the lua string + // TODO(bfredl): it would be "nice" to just use the memory of the Lua string // directly, although ensuring the lifetime of such strings is a bit tricky - // (an API call could invoke nested lua, which triggers GC, and kaboom?) + // (an API call could invoke nested Lua, which triggers GC, and kaboom?) ret.data = arena_memdupz(arena, ret.data, ret.size); lua_pop(lstate, 1); return ret; } -/// Convert lua value to integer +/// Convert Lua value to integer /// /// Always pops one value from the stack. Integer nlua_pop_Integer(lua_State *lstate, Arena *arena, Error *err) @@ -832,10 +831,10 @@ Integer nlua_pop_Integer(lua_State *lstate, Arena *arena, Error *err) return (Integer)n; } -/// Convert lua value to boolean +/// Convert Lua value to boolean /// -/// Despite the name of the function, this uses lua semantics for booleans. -/// thus `err` is never set as any lua value can be co-erced into a lua bool +/// Despite the name of the function, this uses Lua semantics for booleans. +/// thus `err` is never set as any Lua value can be co-erced into a Lua bool /// /// Always pops one value from the stack. Boolean nlua_pop_Boolean(lua_State *lstate, Arena *arena, Error *err) @@ -846,7 +845,7 @@ Boolean nlua_pop_Boolean(lua_State *lstate, Arena *arena, Error *err) return ret; } -/// Convert lua value to boolean +/// Convert Lua value to boolean /// /// This follows API conventions for a Boolean value, compare api_object_to_bool /// @@ -896,9 +895,9 @@ static inline LuaTableProps nlua_check_type(lua_State *const lstate, Error *cons } LuaTableProps table_props = nlua_traverse_table(lstate); - if (type == kObjectTypeDictionary && table_props.type == kObjectTypeArray + if (type == kObjectTypeDict && table_props.type == kObjectTypeArray && table_props.maxidx == 0 && !table_props.has_type_key) { - table_props.type = kObjectTypeDictionary; + table_props.type = kObjectTypeDict; } if (table_props.type != type) { @@ -910,7 +909,7 @@ static inline LuaTableProps nlua_check_type(lua_State *const lstate, Error *cons return table_props; } -/// Convert lua table to float +/// Convert Lua table to float /// /// Always pops one value from the stack. Float nlua_pop_Float(lua_State *lstate, Arena *arena, Error *err) @@ -931,7 +930,7 @@ Float nlua_pop_Float(lua_State *lstate, Arena *arena, Error *err) return (Float)table_props.val; } -/// Convert lua table to array without determining whether it is array +/// Convert Lua table to array without determining whether it is array /// /// @param lstate Lua state. /// @param[in] table_props nlua_traverse_table() output. @@ -966,7 +965,7 @@ static Array nlua_pop_Array_unchecked(lua_State *const lstate, const LuaTablePro return ret; } -/// Convert lua table to array +/// Convert Lua table to array /// /// Always pops one value from the stack. Array nlua_pop_Array(lua_State *lstate, Arena *arena, Error *err) @@ -979,7 +978,7 @@ Array nlua_pop_Array(lua_State *lstate, Arena *arena, Error *err) return nlua_pop_Array_unchecked(lstate, table_props, arena, err); } -/// Convert lua table to dictionary +/// Convert Lua table to dictionary /// /// Always pops one value from the stack. Does not check whether whether topmost /// value on the stack is a table. @@ -987,11 +986,11 @@ Array nlua_pop_Array(lua_State *lstate, Arena *arena, Error *err) /// @param lstate Lua interpreter state. /// @param[in] table_props nlua_traverse_table() output. /// @param[out] err Location where error will be saved. -static Dictionary nlua_pop_Dictionary_unchecked(lua_State *lstate, const LuaTableProps table_props, - bool ref, Arena *arena, Error *err) +static Dict nlua_pop_Dict_unchecked(lua_State *lstate, const LuaTableProps table_props, bool ref, + Arena *arena, Error *err) FUNC_ATTR_NONNULL_ARG(1, 5) FUNC_ATTR_WARN_UNUSED_RESULT { - Dictionary ret = arena_dict(arena, table_props.string_keys_num); + Dict ret = arena_dict(arena, table_props.string_keys_num); if (table_props.string_keys_num == 0) { lua_pop(lstate, 1); @@ -1020,11 +1019,11 @@ static Dictionary nlua_pop_Dictionary_unchecked(lua_State *lstate, const LuaTabl if (ERROR_SET(err)) { if (!arena) { - api_free_dictionary(ret); + api_free_dict(ret); } lua_pop(lstate, 2); // stack: - return (Dictionary) { .size = 0, .items = NULL }; + return (Dict) { .size = 0, .items = NULL }; } i++; } else { @@ -1037,20 +1036,20 @@ static Dictionary nlua_pop_Dictionary_unchecked(lua_State *lstate, const LuaTabl return ret; } -/// Convert lua table to dictionary +/// Convert Lua table to dictionary /// /// Always pops one value from the stack. -Dictionary nlua_pop_Dictionary(lua_State *lstate, bool ref, Arena *arena, Error *err) +Dict nlua_pop_Dict(lua_State *lstate, bool ref, Arena *arena, Error *err) FUNC_ATTR_NONNULL_ARG(1, 4) FUNC_ATTR_WARN_UNUSED_RESULT { const LuaTableProps table_props = nlua_check_type(lstate, err, - kObjectTypeDictionary); - if (table_props.type != kObjectTypeDictionary) { + kObjectTypeDict); + if (table_props.type != kObjectTypeDict) { lua_pop(lstate, 1); - return (Dictionary) { .size = 0, .items = NULL }; + return (Dict) { .size = 0, .items = NULL }; } - return nlua_pop_Dictionary_unchecked(lstate, table_props, ref, arena, err); + return nlua_pop_Dict_unchecked(lstate, table_props, ref, arena, err); } /// Helper structure for nlua_pop_Object @@ -1059,7 +1058,7 @@ typedef struct { bool container; ///< True if tv is a container. } ObjPopStackItem; -/// Convert lua table to object +/// Convert Lua table to object /// /// Always pops one value from the stack. Object nlua_pop_Object(lua_State *const lstate, bool ref, Arena *arena, Error *const err) @@ -1077,9 +1076,9 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Arena *arena, Error *c api_set_error(err, kErrorTypeException, "Lua failed to grow stack"); break; } - if (cur.obj->type == kObjectTypeDictionary) { + if (cur.obj->type == kObjectTypeDict) { // stack: …, dict, key - if (cur.obj->data.dictionary.size == cur.obj->data.dictionary.capacity) { + if (cur.obj->data.dict.size == cur.obj->data.dict.capacity) { lua_pop(lstate, 2); continue; } @@ -1097,10 +1096,10 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Arena *arena, Error *c // stack: …, dict, new key, val size_t len; const char *s = lua_tolstring(lstate, -2, &len); - const size_t idx = cur.obj->data.dictionary.size++; - cur.obj->data.dictionary.items[idx].key = CBUF_TO_ARENA_STR(arena, s, len); + const size_t idx = cur.obj->data.dict.size++; + cur.obj->data.dict.items[idx].key = CBUF_TO_ARENA_STR(arena, s, len); kvi_push(stack, cur); - cur = (ObjPopStackItem){ .obj = &cur.obj->data.dictionary.items[idx].value }; + cur = (ObjPopStackItem){ .obj = &cur.obj->data.dict.items[idx].value }; } else { // stack: …, dict lua_pop(lstate, 1); @@ -1154,10 +1153,10 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Arena *arena, Error *c kvi_push(stack, cur); } break; - case kObjectTypeDictionary: - *cur.obj = DICTIONARY_OBJ(((Dictionary)ARRAY_DICT_INIT)); + case kObjectTypeDict: + *cur.obj = DICT_OBJ(((Dict)ARRAY_DICT_INIT)); if (table_props.string_keys_num != 0) { - cur.obj->data.dictionary = arena_dict(arena, table_props.string_keys_num); + cur.obj->data.dict = arena_dict(arena, table_props.string_keys_num); cur.container = true; kvi_push(stack, cur); lua_pushnil(lstate); @@ -1191,16 +1190,14 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Arena *arena, Error *c if (is_nil) { *cur.obj = NIL; } else { - api_set_error(err, kErrorTypeValidation, - "Cannot convert userdata"); + api_set_error(err, kErrorTypeValidation, "Cannot convert userdata"); } break; } default: type_error: - api_set_error(err, kErrorTypeValidation, - "Cannot convert given lua type"); + api_set_error(err, kErrorTypeValidation, "Cannot convert given Lua type"); break; } if (!cur.container) { @@ -1278,16 +1275,16 @@ void nlua_init_types(lua_State *const lstate) lua_rawset(lstate, -3); LUA_PUSH_STATIC_STRING(lstate, "dictionary"); - lua_pushnumber(lstate, (lua_Number)kObjectTypeDictionary); + lua_pushnumber(lstate, (lua_Number)kObjectTypeDict); lua_rawset(lstate, -3); - lua_pushnumber(lstate, (lua_Number)kObjectTypeDictionary); + lua_pushnumber(lstate, (lua_Number)kObjectTypeDict); LUA_PUSH_STATIC_STRING(lstate, "dictionary"); lua_rawset(lstate, -3); lua_rawset(lstate, -3); } -// lua specific variant of api_dict_to_keydict +// Lua specific variant of api_dict_to_keydict void nlua_pop_keydict(lua_State *L, void *retval, FieldHashfn hashy, char **err_opt, Arena *arena, Error *err) { @@ -1337,8 +1334,8 @@ void nlua_pop_keydict(lua_State *L, void *retval, FieldHashfn hashy, char **err_ *(handle_T *)mem = nlua_pop_handle(L, arena, err); } else if (field->type == kObjectTypeArray) { *(Array *)mem = nlua_pop_Array(L, arena, err); - } else if (field->type == kObjectTypeDictionary) { - *(Dictionary *)mem = nlua_pop_Dictionary(L, false, arena, err); + } else if (field->type == kObjectTypeDict) { + *(Dict *)mem = nlua_pop_Dict(L, false, arena, err); } else if (field->type == kObjectTypeLuaRef) { *(LuaRef *)mem = nlua_pop_LuaRef(L, arena, err); } else { @@ -1387,8 +1384,8 @@ void nlua_push_keydict(lua_State *L, void *value, KeySetLink *table) nlua_push_String(L, *(String *)mem, 0); } else if (field->type == kObjectTypeArray) { nlua_push_Array(L, *(Array *)mem, 0); - } else if (field->type == kObjectTypeDictionary) { - nlua_push_Dictionary(L, *(Dictionary *)mem, 0); + } else if (field->type == kObjectTypeDict) { + nlua_push_Dict(L, *(Dict *)mem, 0); } else if (field->type == kObjectTypeLuaRef) { nlua_pushref(L, *(LuaRef *)mem); } else { -- cgit From 069451bb214bd9d97273ac92b37a25054df0f1a8 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Wed, 25 Sep 2024 03:25:49 -0700 Subject: fix(coverity): size_t overflow #30497 CID 497370: Overflowed constant (INTEGER_OVERFLOW) Expression `tsize - ret.has_type_key`, where tsize=0 and ret.has_type_key=1, underflows the type that receives it, an unsigned integer 64 bits wide. CID 509910: Overflowed constant (INTEGER_OVERFLOW) Expression stack.size++, which is equal to 0, where stack.size is known to be equal to 18446744073709551615, overflows the type that receives it, an unsigned integer 64 bits wide --- src/nvim/lua/converter.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c index 027ae61007..45ead154bc 100644 --- a/src/nvim/lua/converter.c +++ b/src/nvim/lua/converter.c @@ -122,6 +122,7 @@ static LuaTableProps nlua_traverse_table(lua_State *const lstate) lua_pop(lstate, 1); } if (ret.has_type_key) { + assert(tsize > 0); if (ret.type == kObjectTypeFloat && (!has_val_key || val_type != LUA_TNUMBER)) { ret.type = kObjectTypeNil; @@ -1150,6 +1151,7 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Arena *arena, Error *c if (table_props.maxidx != 0) { cur.obj->data.array = arena_array(arena, table_props.maxidx); cur.container = true; + assert(kv_size(stack) < SIZE_MAX); kvi_push(stack, cur); } break; @@ -1158,6 +1160,7 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Arena *arena, Error *c if (table_props.string_keys_num != 0) { cur.obj->data.dict = arena_dict(arena, table_props.string_keys_num); cur.container = true; + assert(kv_size(stack) < SIZE_MAX); kvi_push(stack, cur); lua_pushnil(lstate); } -- cgit From c65646c2474d22948c604168a68f6626a645d1d2 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Thu, 26 Sep 2024 16:10:11 +0100 Subject: fix(diff): use mmfile_t in linematch Problem: Linematch used to use strchr to navigate a string, however strchr does not supoprt embedded NULs. Solution: Use `mmfile_t` instead of `char *` in linematch and introduce `strnchr()`. Also remove heap allocations from `matching_char_iwhite()` Fixes: #30505 --- src/nvim/lua/xdiff.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/xdiff.c b/src/nvim/lua/xdiff.c index 8d791a7e74..b9f96abf73 100644 --- a/src/nvim/lua/xdiff.c +++ b/src/nvim/lua/xdiff.c @@ -67,11 +67,11 @@ static void get_linematch_results(lua_State *lstate, mmfile_t *ma, mmfile_t *mb, int count_a, int start_b, int count_b, bool iwhite) { // get the pointer to char of the start of the diff to pass it to linematch algorithm - const char *diff_begin[2] = { ma->ptr, mb->ptr }; - int diff_length[2] = { count_a, count_b }; + mmfile_t ma0 = fastforward_buf_to_lnum(*ma, (linenr_T)start_a + 1); + mmfile_t mb0 = fastforward_buf_to_lnum(*mb, (linenr_T)start_b + 1); - fastforward_buf_to_lnum(&diff_begin[0], (linenr_T)start_a + 1); - fastforward_buf_to_lnum(&diff_begin[1], (linenr_T)start_b + 1); + const mmfile_t *diff_begin[2] = { &ma0, &mb0 }; + int diff_length[2] = { count_a, count_b }; int *decisions = NULL; size_t decisions_length = linematch_nbuffers(diff_begin, diff_length, 2, &decisions, iwhite); -- cgit From c6abc97006eee7fc89baefad2e1bddb248187f2e Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Wed, 2 Oct 2024 11:48:05 +0100 Subject: perf(treesitter): do not use tree cursors with a small lifetime Problem: Tree cursors can only be efficient when they are re-used. Short-lived cursors are very slow. Solution: Reimplement functions that use short-lived cursors. --- src/nvim/lua/treesitter.c | 86 +++++++++++++---------------------------------- 1 file changed, 24 insertions(+), 62 deletions(-) (limited to 'src/nvim/lua') diff --git a/src/nvim/lua/treesitter.c b/src/nvim/lua/treesitter.c index eb92ae8d0e..ab97704dfe 100644 --- a/src/nvim/lua/treesitter.c +++ b/src/nvim/lua/treesitter.c @@ -39,7 +39,6 @@ #define TS_META_QUERY "treesitter_query" #define TS_META_QUERYCURSOR "treesitter_querycursor" #define TS_META_QUERYMATCH "treesitter_querymatch" -#define TS_META_TREECURSOR "treesitter_treecursor" typedef struct { LuaRef cb; @@ -802,20 +801,6 @@ static int tree_root(lua_State *L) return 1; } -// TSTreeCursor - -static struct luaL_Reg treecursor_meta[] = { - { "__gc", treecursor_gc }, - { NULL, NULL } -}; - -static int treecursor_gc(lua_State *L) -{ - TSTreeCursor *cursor = luaL_checkudata(L, 1, TS_META_TREECURSOR); - ts_tree_cursor_delete(cursor); - return 0; -} - // TSNode static struct luaL_Reg node_meta[] = { { "__tostring", node_tostring }, @@ -1006,23 +991,14 @@ static int node_field(lua_State *L) size_t name_len; const char *field_name = luaL_checklstring(L, 2, &name_len); - TSTreeCursor cursor = ts_tree_cursor_new(node); - lua_newtable(L); // [table] - size_t curr_index = 0; - if (ts_tree_cursor_goto_first_child(&cursor)) { - do { - const char *current_field = ts_tree_cursor_current_field_name(&cursor); - - if (current_field != NULL && !strcmp(field_name, current_field)) { - push_node(L, ts_tree_cursor_current_node(&cursor), 1); // [table, node] - lua_rawseti(L, -2, (int)++curr_index); - } - } while (ts_tree_cursor_goto_next_sibling(&cursor)); + TSNode field = ts_node_child_by_field_name(node, field_name, (uint32_t)name_len); + if (!ts_node_is_null(field)) { + push_node(L, field, 1); // [table, node] + lua_rawseti(L, -2, 1); } - ts_tree_cursor_delete(&cursor); return 1; } @@ -1118,45 +1094,35 @@ static int node_named_descendant_for_range(lua_State *L) static int node_next_child(lua_State *L) { - TSTreeCursor *cursor = luaL_checkudata(L, lua_upvalueindex(1), TS_META_TREECURSOR); + uint32_t *child_index = lua_touserdata(L, lua_upvalueindex(1)); TSNode source = node_check(L, lua_upvalueindex(2)); - // First call should return first child - if (ts_node_eq(source, ts_tree_cursor_current_node(cursor))) { - if (ts_tree_cursor_goto_first_child(cursor)) { - goto push; - } else { - return 0; - } - } - - if (!ts_tree_cursor_goto_next_sibling(cursor)) { + if (*child_index >= ts_node_child_count(source)) { return 0; } -push: - push_node(L, ts_tree_cursor_current_node(cursor), lua_upvalueindex(2)); // [node] - - const char *field = ts_tree_cursor_current_field_name(cursor); + TSNode child = ts_node_child(source, *child_index); + push_node(L, child, lua_upvalueindex(2)); + const char *field = ts_node_field_name_for_child(source, *child_index); if (field != NULL) { - lua_pushstring(L, ts_tree_cursor_current_field_name(cursor)); + lua_pushstring(L, field); } else { lua_pushnil(L); } // [node, field_name_or_nil] + + (*child_index)++; + return 2; } static int node_iter_children(lua_State *L) { - TSNode node = node_check(L, 1); - - TSTreeCursor *ud = lua_newuserdata(L, sizeof(TSTreeCursor)); // [udata] - *ud = ts_tree_cursor_new(node); + node_check(L, 1); + uint32_t *child_index = lua_newuserdata(L, sizeof(uint32_t)); // [source_node,..., udata] + *child_index = 0; - lua_getfield(L, LUA_REGISTRYINDEX, TS_META_TREECURSOR); // [udata, mt] - lua_setmetatable(L, -2); // [udata] - lua_pushvalue(L, 1); // [udata, source_node] + lua_pushvalue(L, 1); // [source_node, ..., udata, source_node] lua_pushcclosure(L, node_next_child, 2); return 1; @@ -1248,22 +1214,19 @@ static int node_prev_named_sibling(lua_State *L) static int node_named_children(lua_State *L) { TSNode source = node_check(L, 1); - TSTreeCursor cursor = ts_tree_cursor_new(source); lua_newtable(L); int curr_index = 0; - if (ts_tree_cursor_goto_first_child(&cursor)) { - do { - TSNode node = ts_tree_cursor_current_node(&cursor); - if (ts_node_is_named(node)) { - push_node(L, node, 1); - lua_rawseti(L, -2, ++curr_index); - } - } while (ts_tree_cursor_goto_next_sibling(&cursor)); + uint32_t n = ts_node_child_count(source); + for (uint32_t i = 0; i < n; i++) { + TSNode child = ts_node_child(source, i); + if (ts_node_is_named(child)) { + push_node(L, child, 1); + lua_rawseti(L, -2, ++curr_index); + } } - ts_tree_cursor_delete(&cursor); return 1; } @@ -1673,7 +1636,6 @@ void tslua_init(lua_State *L) build_meta(L, TS_META_QUERY, query_meta); build_meta(L, TS_META_QUERYCURSOR, querycursor_meta); build_meta(L, TS_META_QUERYMATCH, querymatch_meta); - build_meta(L, TS_META_TREECURSOR, treecursor_meta); ts_set_allocator(xmalloc, xcalloc, xrealloc, xfree); } -- cgit