diff options
37 files changed, 13855 insertions, 567 deletions
diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index a166ee6c02..d3e07326bf 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -86,6 +86,8 @@ foreach(subdir event eval lua + viml + viml/parser ) if(${subdir} MATCHES "tui" AND NOT FEAT_TUI) continue() diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 98f4410347..9daa2fb398 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -886,6 +886,72 @@ theend: return rv; } +/// Parse a VimL expression +/// +/// @param[in] expr Expression to parse. Is always treated as a single line. +/// @param[in] flags Flags: "m" if multiple expressions in a row are allowed +/// (only the first one will be parsed), "E" if EOC tokens +/// are not allowed (determines whether they will stop +/// parsing process or be recognized as an operator/space, +/// though also yielding an error). +/// +/// Use only "m" to parse like for "<C-r>=", only "E" to +/// parse like for ":echo", empty string for ":let". +/// +/// @return AST: top-level dectionary holds keys +/// +/// "error": Dictionary with error, present only if parser saw some +/// error. Contains the following keys: +/// +/// "message": String, error message in printf format, translated. +/// Must contain exactly one "%.*s". +/// "arg": String, error message argument. +/// +/// "ast": actual AST, a dictionary with the following keys: +/// +/// "type": node type, one of the value names from ExprASTNodeType +/// stringified without "kExprNode" prefix. +/// "start": a pair [line, column] describing where node is “started” +/// where "line" is always 0 (will not be 0 if you will be +/// using nvim_parse_viml() on e.g. ":let", but that is not +/// present yet). Both elements are Integers. +/// "len": “length” of the node. This and "start" are there for +/// debugging purposes primary (debugging parser and providing +/// debug information). +/// "children": a list of nodes described in top/"ast". There always +/// is zero, one or two children, key will contain an +/// empty array if node can have children, but has no and +/// will not be present at all if node can’t have any +/// children. Maximum number of children may be found in +/// node_maxchildren array. +/// +/// Local values (present only for certain nodes): +/// +/// "scope": a single Integer, specifies scope for "Option" and +/// "PlainIdentifier" nodes. For "Option" it is one of +/// ExprOptScope values, for "PlainIdentifier" it is one of +/// ExprVarScope values. +/// "ident": identifier (without scope, if any), present for "Option", +/// "PlainIdentifier", "PlainKey" and "Environment" nodes. +/// "name": Integer, register name (one character) or -1. Only present +/// for "Register" nodes. +/// "cmp_type": String, comparison type, one of the value names from +/// ExprComparisonType, stringified without "kExprCmp" +/// prefix. Only present for "Comparison" nodes. +/// "ccs_strategy": String, case comparison strategy, one of the +/// value names from ExprCaseCompareStrategy, +/// stringified without "kCCStrategy" prefix. Only +/// present for "Comparison" nodes. +/// "ivalue": Integer, integer value for "Integer" nodes. +/// "fvalue": Float, floating-point value for "Float" nodes. +/// "svalue": String, value for "SingleQuotedString" and +/// "DoubleQuotedString" nodes. +Dictionary nvim_parse_expression(String expr, String flags, Error *err) + FUNC_API_SINCE(4) +{ + return (Dictionary)ARRAY_DICT_INIT; +} + /// Writes a message to vim output or error buffer. The string is split /// and flushed after each newline. Incomplete lines are kept for writing diff --git a/src/nvim/charset.c b/src/nvim/charset.c index 577fc13a31..c895d65eb7 100644 --- a/src/nvim/charset.c +++ b/src/nvim/charset.c @@ -1621,11 +1621,13 @@ bool vim_isblankline(char_u *lbuf) void vim_str2nr(const char_u *const start, int *const prep, int *const len, const int what, varnumber_T *const nptr, uvarnumber_T *const unptr, const int maxlen) + FUNC_ATTR_NONNULL_ARG(1) { - const char_u *ptr = start; + const char *ptr = (const char *)start; +#define STRING_ENDED(ptr) \ + (!(maxlen == 0 || (int)((ptr) - (const char *)start) < maxlen)) int pre = 0; // default is decimal bool negative = false; - uvarnumber_T un = 0; if (ptr[0] == '-') { negative = true; @@ -1633,119 +1635,80 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } // Recognize hex, octal and bin. - if ((ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9') - && (maxlen == 0 || maxlen > 1)) { + if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) + && !STRING_ENDED(ptr + 1) + && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { pre = ptr[1]; if ((what & STR2NR_HEX) - && ((pre == 'X') || (pre == 'x')) - && ascii_isxdigit(ptr[2]) - && (maxlen == 0 || maxlen > 2)) { + && !STRING_ENDED(ptr + 2) + && (pre == 'X' || pre == 'x') + && ascii_isxdigit(ptr[2])) { // hexadecimal ptr += 2; } else if ((what & STR2NR_BIN) - && ((pre == 'B') || (pre == 'b')) - && ascii_isbdigit(ptr[2]) - && (maxlen == 0 || maxlen > 2)) { + && !STRING_ENDED(ptr + 2) + && (pre == 'B' || pre == 'b') + && ascii_isbdigit(ptr[2])) { // binary ptr += 2; } else { // decimal or octal, default is decimal pre = 0; - if (what & STR2NR_OCT) { - // Don't interpret "0", "08" or "0129" as octal. - for (int n = 1; ascii_isdigit(ptr[n]); ++n) { - if (ptr[n] > '7') { - // can't be octal + if (what & STR2NR_OCT + && !STRING_ENDED(ptr + 1) + && ('0' <= ptr[1] && ptr[1] <= '7')) { + // Assume octal now: what we already know is that string starts with + // zero and some octal digit. + pre = '0'; + // Don’t interpret "0", "008" or "0129" as octal. + for (int i = 2; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { + if (ptr[i] > '7') { + // Can’t be octal. pre = 0; break; } - if (ptr[n] >= '0') { - // assume octal - pre = '0'; - } - if (n == maxlen) { - break; - } } } } } // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. - int n = 1; - if ((pre == 'B') || (pre == 'b') || what == STR2NR_BIN + STR2NR_FORCE) { - // bin - if (pre != 0) { - n += 2; // skip over "0b" - } - while ('0' <= *ptr && *ptr <= '1') { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 2) { - un = 2 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - if (n++ == maxlen) { - break; - } - } - } else if ((pre == '0') || what == STR2NR_OCT + STR2NR_FORCE) { - // octal - while ('0' <= *ptr && *ptr <= '7') { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 8) { - un = 8 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - if (n++ == maxlen) { - break; - } - } - } else if ((pre == 'X') || (pre == 'x') - || what == STR2NR_HEX + STR2NR_FORCE) { - // hex - if (pre != 0) { - n += 2; // skip over "0x" - } - while (ascii_isxdigit(*ptr)) { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 16) { - un = 16 * un + (uvarnumber_T)hex2nr(*ptr); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - if (n++ == maxlen) { - break; - } - } + uvarnumber_T un = 0; +#define PARSE_NUMBER(base, cond, conv) \ + do { \ + while (!STRING_ENDED(ptr) && (cond)) { \ + /* avoid ubsan error for overflow */ \ + if (un < UVARNUMBER_MAX / base) { \ + un = base * un + (uvarnumber_T)(conv); \ + } else { \ + un = UVARNUMBER_MAX; \ + } \ + ptr++; \ + } \ + } while (0) + if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { + // Binary number. + PARSE_NUMBER(2, (*ptr == '0' || *ptr == '1'), (*ptr - '0')); + } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { + // Octal number. + PARSE_NUMBER(8, ('0' <= *ptr && *ptr <= '7'), (*ptr - '0')); + } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { + // Hexadecimal number. + PARSE_NUMBER(16, (ascii_isxdigit(*ptr)), (hex2nr(*ptr))); } else { - // decimal - while (ascii_isdigit(*ptr)) { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 10) { - un = 10 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - if (n++ == maxlen) { - break; - } - } + // Decimal number. + PARSE_NUMBER(10, (ascii_isdigit(*ptr)), (*ptr - '0')); } +#undef PARSE_NUMBER if (prep != NULL) { *prep = pre; } if (len != NULL) { - *len = (int)(ptr - start); + *len = (int)(ptr - (const char *)start); } if (nptr != NULL) { @@ -1767,6 +1730,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, if (unptr != NULL) { *unptr = un; } +#undef STRING_ENDED } /// Return the value of a single hex character. diff --git a/src/nvim/edit.c b/src/nvim/edit.c index 2bafb77fef..28722a4d10 100644 --- a/src/nvim/edit.c +++ b/src/nvim/edit.c @@ -6066,27 +6066,30 @@ void free_last_insert(void) #endif -/* - * Add character "c" to buffer "s". Escape the special meaning of K_SPECIAL - * and CSI. Handle multi-byte characters. - * Returns a pointer to after the added bytes. - */ +/// Add character "c" to buffer "s" +/// +/// Escapes the special meaning of K_SPECIAL and CSI, handles multi-byte +/// characters. +/// +/// @param[in] c Character to add. +/// @param[out] s Buffer to add to. Must have at least MB_MAXBYTES + 1 bytes. +/// +/// @return Pointer to after the added bytes. char_u *add_char2buf(int c, char_u *s) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT { char_u temp[MB_MAXBYTES + 1]; - int i; - int len; - - len = (*mb_char2bytes)(c, temp); - for (i = 0; i < len; ++i) { + const int len = utf_char2bytes(c, temp); + for (int i = 0; i < len; ++i) { c = temp[i]; - /* Need to escape K_SPECIAL and CSI like in the typeahead buffer. */ + // Need to escape K_SPECIAL and CSI like in the typeahead buffer. if (c == K_SPECIAL) { *s++ = K_SPECIAL; *s++ = KS_SPECIAL; *s++ = KE_FILLER; - } else + } else { *s++ = c; + } } return s; } diff --git a/src/nvim/ex_docmd.c b/src/nvim/ex_docmd.c index 096187b162..3bcea4b666 100644 --- a/src/nvim/ex_docmd.c +++ b/src/nvim/ex_docmd.c @@ -5912,9 +5912,10 @@ static void ex_colorscheme(exarg_T *eap) static void ex_highlight(exarg_T *eap) { - if (*eap->arg == NUL && eap->cmd[2] == '!') + if (*eap->arg == NUL && eap->cmd[2] == '!') { MSG(_("Greetings, Vim user!")); - do_highlight(eap->arg, eap->forceit, FALSE); + } + do_highlight((const char *)eap->arg, eap->forceit, false); } diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c index e79476ab53..785038f5d6 100644 --- a/src/nvim/ex_getln.c +++ b/src/nvim/ex_getln.c @@ -66,6 +66,8 @@ #include "nvim/lib/kvec.h" #include "nvim/api/private/helpers.h" #include "nvim/highlight_defs.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/viml/parser/expressions.h" /// Command-line colors: one chunk /// @@ -2360,6 +2362,62 @@ void free_cmdline_buf(void) enum { MAX_CB_ERRORS = 1 }; +/// Color expression cmdline using built-in expressions parser +/// +/// @param[in] colored_ccline Command-line to color. +/// @param[out] ret_ccline_colors What should be colored. +/// +/// Always colors the whole cmdline. +static void color_expr_cmdline(const CmdlineInfo *const colored_ccline, + ColoredCmdline *const ret_ccline_colors) + FUNC_ATTR_NONNULL_ALL +{ + ParserLine plines[] = { + { + .data = (const char *)colored_ccline->cmdbuff, + .size = STRLEN(colored_ccline->cmdbuff), + .allocated = false, + }, + { NULL, 0, false }, + }; + ParserLine *plines_p = plines; + ParserHighlight colors; + kvi_init(colors); + ParserState pstate; + viml_parser_init( + &pstate, parser_simple_get_line, &plines_p, &colors); + ExprAST east = viml_pexpr_parse(&pstate, kExprFlagsDisallowEOC); + viml_pexpr_free_ast(east); + viml_parser_destroy(&pstate); + kv_resize(ret_ccline_colors->colors, kv_size(colors)); + size_t prev_end = 0; + for (size_t i = 0 ; i < kv_size(colors) ; i++) { + const ParserHighlightChunk chunk = kv_A(colors, i); + if (chunk.start.col != prev_end) { + kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) { + .start = prev_end, + .end = chunk.start.col, + .attr = 0, + })); + } + const int id = syn_name2id((const char_u *)chunk.group); + const int attr = (id == 0 ? 0 : syn_id2attr(id)); + kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) { + .start = chunk.start.col, + .end = chunk.end_col, + .attr = attr, + })); + prev_end = chunk.end_col; + } + if (prev_end < (size_t)colored_ccline->cmdlen) { + kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) { + .start = prev_end, + .end = (size_t)colored_ccline->cmdlen, + .attr = 0, + })); + } +} + /// Color command-line /// /// Should use built-in command parser or user-specified one. Currently only the @@ -2442,13 +2500,8 @@ static bool color_cmdline(CmdlineInfo *colored_ccline) tl_ret = try_leave(&tstate, &err); can_free_cb = true; } else if (colored_ccline->cmdfirstc == '=') { - try_enter(&tstate); - err_errmsg = N_( - "E5409: Unable to get g:Nvim_color_expr callback: %s"); - dgc_ret = tv_dict_get_callback(&globvardict, S_LEN("Nvim_color_expr"), - &color_cb); - tl_ret = try_leave(&tstate, &err); - can_free_cb = true; + color_expr_cmdline(colored_ccline, ret_ccline_colors); + can_free_cb = false; } if (!tl_ret || !dgc_ret) { goto color_cmdline_error; diff --git a/src/nvim/generators/gen_declarations.lua b/src/nvim/generators/gen_declarations.lua index e999e53e4a..0c73376ba0 100755 --- a/src/nvim/generators/gen_declarations.lua +++ b/src/nvim/generators/gen_declarations.lua @@ -164,9 +164,40 @@ local pattern = concat( ) if fname == '--help' then - print'Usage:' - print() - print' gendeclarations.lua definitions.c static.h non-static.h preprocessor.i' + print([[ +Usage: + + gendeclarations.lua definitions.c static.h non-static.h definitions.i + +Generates declarations for a C file defitions.c, putting declarations for +static functions into static.h and declarations for non-static functions into +non-static.h. File `definitions.i' should contain an already preprocessed +version of defintions.c and it is the only one which is actually parsed, +definitions.c is needed only to determine functions from which file out of all +functions found in definitions.i are needed. + +Additionally uses the following environment variables: + + NVIM_GEN_DECLARATIONS_LINE_NUMBERS: + If set to 1 then all generated declarations receive a comment with file + name and line number after the declaration. This may be useful for + debugging gen_declarations script, but not much beyound that with + configured development environment (i.e. with ctags/cscope/finding + definitions with clang/etc). + + WARNING: setting this to 1 will cause extensive rebuilds: declarations + generator script will not regenerate non-static.h file if its + contents did not change, but including line numbers will make + contents actually change. + + With contents changed timestamp of the file is regenerated even + when no real changes were made (e.g. a few lines were added to + a function which is not at the bottom of the file). + + With changed timestamp build system will assume that header + changed, triggering rebuilds of all C files which depend on the + "changed" header. +]]) os.exit() end @@ -249,8 +280,10 @@ while init ~= nil do declaration = declaration:gsub(' $', '') declaration = declaration:gsub('^ ', '') declaration = declaration .. ';' - declaration = declaration .. (' // %s/%s:%u'):format( - curdir, curfile, declline) + if os.getenv('NVIM_GEN_DECLARATIONS_LINE_NUMBERS') == '1' then + declaration = declaration .. (' // %s/%s:%u'):format( + curdir, curfile, declline) + end declaration = declaration .. '\n' if declaration:sub(1, 6) == 'static' then static = static .. declaration diff --git a/src/nvim/globals.h b/src/nvim/globals.h index 0f3e132bc0..2a041177d4 100644 --- a/src/nvim/globals.h +++ b/src/nvim/globals.h @@ -728,29 +728,6 @@ EXTERN int vr_lines_changed INIT(= 0); /* #Lines changed by "gR" so far */ /// Encoding used when 'fencs' is set to "default" EXTERN char_u *fenc_default INIT(= NULL); -// To speed up BYTELEN(); keep a lookup table to quickly get the length in -// bytes of a UTF-8 character from the first byte of a UTF-8 string. Bytes -// which are illegal when used as the first byte have a 1. The NUL byte has -// length 1. -EXTERN char utf8len_tab[256] INIT(= { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, -}); - # if defined(USE_ICONV) && defined(DYNAMIC_ICONV) /* Pointers to functions and variables to be loaded at runtime */ EXTERN size_t (*iconv)(iconv_t cd, const char **inbuf, size_t *inbytesleft, diff --git a/src/nvim/keymap.c b/src/nvim/keymap.c index a75fe793ac..6ed54464e8 100644 --- a/src/nvim/keymap.c +++ b/src/nvim/keymap.c @@ -24,12 +24,11 @@ * Some useful tables. */ -static struct modmasktable { - short mod_mask; /* Bit-mask for particular key modifier */ - short mod_flag; /* Bit(s) for particular key modifier */ - char_u name; /* Single letter name of modifier */ -} mod_mask_table[] = -{ +static const struct modmasktable { + short mod_mask; ///< Bit-mask for particular key modifier. + short mod_flag; ///< Bit(s) for particular key modifier. + char_u name; ///< Single letter name of modifier. +} mod_mask_table[] = { {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'M'}, {MOD_MASK_META, MOD_MASK_META, (char_u)'T'}, {MOD_MASK_CTRL, MOD_MASK_CTRL, (char_u)'C'}, @@ -139,11 +138,10 @@ static char_u modifier_keys_table[] = NUL }; -static struct key_name_entry { +static const struct key_name_entry { int key; // Special key code or ascii value - char *name; // Name of key -} key_names_table[] = -{ + const char *name; // Name of key +} key_names_table[] = { { ' ', "Space" }, { TAB, "Tab" }, { K_TAB, "Tab" }, @@ -318,73 +316,73 @@ static struct mousetable { {0, 0, 0, 0}, }; -/* - * Return the modifier mask bit (MOD_MASK_*) which corresponds to the given - * modifier name ('S' for Shift, 'C' for Ctrl etc). - */ +/// Return the modifier mask bit (#MOD_MASK_*) corresponding to mod name +/// +/// E.g. 'S' for shift, 'C' for ctrl. int name_to_mod_mask(int c) + FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT { - int i; - c = TOUPPER_ASC(c); - for (i = 0; mod_mask_table[i].mod_mask != 0; i++) - if (c == mod_mask_table[i].name) + for (size_t i = 0; mod_mask_table[i].mod_mask != 0; i++) { + if (c == mod_mask_table[i].name) { return mod_mask_table[i].mod_flag; + } + } return 0; } -/* - * Check if if there is a special key code for "key" that includes the - * modifiers specified. - */ -int simplify_key(int key, int *modifiers) +/// Check if there is a special key code for "key" with specified modifiers +/// +/// @param[in] key Initial key code. +/// @param[in,out] modifiers Initial modifiers, is adjusted to have simplified +/// modifiers. +/// +/// @return Simplified key code. +int simplify_key(const int key, int *modifiers) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - int i; - int key0; - int key1; - if (*modifiers & (MOD_MASK_SHIFT | MOD_MASK_CTRL | MOD_MASK_ALT)) { - /* TAB is a special case */ + // TAB is a special case. if (key == TAB && (*modifiers & MOD_MASK_SHIFT)) { *modifiers &= ~MOD_MASK_SHIFT; return K_S_TAB; } - key0 = KEY2TERMCAP0(key); - key1 = KEY2TERMCAP1(key); - for (i = 0; modifier_keys_table[i] != NUL; i += MOD_KEYS_ENTRY_SIZE) + const int key0 = KEY2TERMCAP0(key); + const int key1 = KEY2TERMCAP1(key); + for (int i = 0; modifier_keys_table[i] != NUL; i += MOD_KEYS_ENTRY_SIZE) { if (key0 == modifier_keys_table[i + 3] && key1 == modifier_keys_table[i + 4] && (*modifiers & modifier_keys_table[i])) { *modifiers &= ~modifier_keys_table[i]; return TERMCAP2KEY(modifier_keys_table[i + 1], - modifier_keys_table[i + 2]); + modifier_keys_table[i + 2]); } + } } return key; } -/* - * Change <xHome> to <Home>, <xUp> to <Up>, etc. - */ -int handle_x_keys(int key) +/// Change <xKey> to <Key> +int handle_x_keys(const int key) + FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT { switch (key) { - case K_XUP: return K_UP; - case K_XDOWN: return K_DOWN; - case K_XLEFT: return K_LEFT; - case K_XRIGHT: return K_RIGHT; - case K_XHOME: return K_HOME; - case K_ZHOME: return K_HOME; - case K_XEND: return K_END; - case K_ZEND: return K_END; - case K_XF1: return K_F1; - case K_XF2: return K_F2; - case K_XF3: return K_F3; - case K_XF4: return K_F4; - case K_S_XF1: return K_S_F1; - case K_S_XF2: return K_S_F2; - case K_S_XF3: return K_S_F3; - case K_S_XF4: return K_S_F4; + case K_XUP: return K_UP; + case K_XDOWN: return K_DOWN; + case K_XLEFT: return K_LEFT; + case K_XRIGHT: return K_RIGHT; + case K_XHOME: return K_HOME; + case K_ZHOME: return K_HOME; + case K_XEND: return K_END; + case K_ZEND: return K_END; + case K_XF1: return K_F1; + case K_XF2: return K_F2; + case K_XF3: return K_F3; + case K_XF4: return K_F4; + case K_S_XF1: return K_S_F1; + case K_S_XF2: return K_S_F2; + case K_S_XF3: return K_S_F3; + case K_S_XF4: return K_S_F4; } return key; } @@ -508,7 +506,7 @@ unsigned int trans_special(const char_u **srcp, const size_t src_len, return 0; } - /* Put the appropriate modifier in a string */ + // Put the appropriate modifier in a string. if (modifiers != 0) { dst[dlen++] = K_SPECIAL; dst[dlen++] = KS_MODIFIER; @@ -573,11 +571,7 @@ int find_special_key(const char_u **srcp, const size_t src_len, int *const modp, if (*bp == '-') { last_dash = bp; if (bp + 1 <= end) { - if (has_mbyte) { - l = mb_ptr2len_len(bp + 1, (int) (end - bp) + 1); - } else { - l = 1; - } + l = utfc_ptr2len_len(bp + 1, (int)(end - bp) + 1); // Anything accepted, like <C-?>. // <C-"> or <M-"> are not special in strings as " is // the string delimiter. With a backslash it works: <M-\"> @@ -702,33 +696,39 @@ int find_special_key_in_table(int c) { int i; - for (i = 0; key_names_table[i].name != NULL; i++) - if (c == key_names_table[i].key) + for (i = 0; key_names_table[i].name != NULL; i++) { + if (c == key_names_table[i].key) { break; - if (key_names_table[i].name == NULL) + } + } + if (key_names_table[i].name == NULL) { i = -1; + } return i; } -/* - * Find the special key with the given name (the given string does not have to - * end with NUL, the name is assumed to end before the first non-idchar). - * If the name starts with "t_" the next two characters are interpreted as a - * termcap name. - * Return the key code, or 0 if not found. - */ +/// Find the special key with the given name +/// +/// @param[in] name Name of the special. Does not have to end with NUL, it is +/// assumed to end before the first non-idchar. If name starts +/// with "t_" the next two characters are interpreted as +/// a termcap name. +/// +/// @return Key code or 0 if ton found. int get_special_key_code(const char_u *name) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT { - char *table_name; - int i, j; - - for (i = 0; key_names_table[i].name != NULL; i++) { - table_name = key_names_table[i].name; - for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) - if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) + for (int i = 0; key_names_table[i].name != NULL; i++) { + const char *const table_name = key_names_table[i].name; + int j; + for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) { + if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) { break; - if (!vim_isIDc(name[j]) && table_name[j] == NUL) + } + } + if (!vim_isIDc(name[j]) && table_name[j] == NUL) { return key_names_table[i].key; + } } return 0; diff --git a/src/nvim/lib/kvec.h b/src/nvim/lib/kvec.h index 584282d773..ee1b890cb9 100644 --- a/src/nvim/lib/kvec.h +++ b/src/nvim/lib/kvec.h @@ -62,7 +62,16 @@ #define kv_pop(v) ((v).items[--(v).size]) #define kv_size(v) ((v).size) #define kv_max(v) ((v).capacity) -#define kv_last(v) kv_A(v, kv_size(v) - 1) +#define kv_Z(v, i) kv_A(v, kv_size(v) - (i) - 1) +#define kv_last(v) kv_Z(v, 0) + +/// Drop last n items from kvec without resizing +/// +/// Previously spelled as `(void)kv_pop(v)`, repeated n times. +/// +/// @param[out] v Kvec to drop items from. +/// @param[in] n Number of elements to drop. +#define kv_drop(v, n) ((v).size -= (n)) #define kv_resize(v, s) \ ((v).capacity = (s), \ diff --git a/src/nvim/lib/ringbuf.h b/src/nvim/lib/ringbuf.h index 12b75ec65a..e63eae70b0 100644 --- a/src/nvim/lib/ringbuf.h +++ b/src/nvim/lib/ringbuf.h @@ -15,6 +15,7 @@ #ifndef NVIM_LIB_RINGBUF_H #define NVIM_LIB_RINGBUF_H +#include <stddef.h> #include <string.h> #include <assert.h> #include <stdint.h> @@ -73,6 +74,32 @@ typedef struct { \ RBType *buf_end; \ } TypeName##RingBuffer; +/// Dummy item free macros, for use in RINGBUF_INIT +/// +/// This macros actually does nothing. +/// +/// @param[in] item Item to be freed. +#define RINGBUF_DUMMY_FREE(item) + +/// Static ring buffer +/// +/// @warning Ring buffers created with this macros must neither be freed nor +/// deallocated. +/// +/// @param scope Ring buffer scope. +/// @param TypeName Ring buffer type name. +/// @param RBType Type of the single ring buffer element. +/// @param varname Variable name. +/// @param rbsize Ring buffer size. +#define RINGBUF_STATIC(scope, TypeName, RBType, varname, rbsize) \ +static RBType _##varname##_buf[rbsize]; \ +scope TypeName##RingBuffer varname = { \ + .buf = _##varname##_buf, \ + .next = _##varname##_buf, \ + .first = NULL, \ + .buf_end = _##varname##_buf + rbsize - 1, \ +}; + /// Initialize a new ring buffer /// /// @param TypeName Ring buffer type name. Actual type name will be diff --git a/src/nvim/mbyte.c b/src/nvim/mbyte.c index b24770a409..843007b97b 100644 --- a/src/nvim/mbyte.c +++ b/src/nvim/mbyte.c @@ -72,19 +72,41 @@ struct interval { # include "unicode_tables.generated.h" #endif -/* - * Like utf8len_tab above, but using a zero for illegal lead bytes. - */ -const uint8_t utf8len_tab_zero[256] = -{ - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0, +// To speed up BYTELEN(); keep a lookup table to quickly get the length in +// bytes of a UTF-8 character from the first byte of a UTF-8 string. Bytes +// which are illegal when used as the first byte have a 1. The NUL byte has +// length 1. +const uint8_t utf8len_tab[] = { + // ?1 ?2 ?3 ?4 ?5 ?6 ?7 ?8 ?9 ?A ?B ?C ?D ?E ?F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D? + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E? + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, // F? +}; + +// Like utf8len_tab above, but using a zero for illegal lead bytes. +const uint8_t utf8len_tab_zero[] = { + //1 2 3 4 5 6 7 8 9 A B C D E F 0 1 2 3 4 5 6 7 8 9 A B C D E F + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 2 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 4 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 6 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 8 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // A + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // C + 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0, // E }; /* @@ -528,45 +550,52 @@ int utf_off2cells(unsigned off, unsigned max_off) return (off + 1 < max_off && ScreenLines[off + 1] == 0) ? 2 : 1; } -/* - * Convert a UTF-8 byte sequence to a wide character. - * If the sequence is illegal or truncated by a NUL the first byte is - * returned. - * Does not include composing characters, of course. - */ -int utf_ptr2char(const char_u *p) +/// Convert a UTF-8 byte sequence to a wide character +/// +/// If the sequence is illegal or truncated by a NUL then the first byte is +/// returned. Does not include composing characters for obvious reasons. +/// +/// @param[in] p String to convert. +/// +/// @return Unicode codepoint or byte value. +int utf_ptr2char(const char_u *const p) + FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT { - uint8_t len; - - if (p[0] < 0x80) /* be quick for ASCII */ + if (p[0] < 0x80) { // Be quick for ASCII. return p[0]; + } - len = utf8len_tab_zero[p[0]]; + const uint8_t len = utf8len_tab_zero[p[0]]; if (len > 1 && (p[1] & 0xc0) == 0x80) { - if (len == 2) + if (len == 2) { return ((p[0] & 0x1f) << 6) + (p[1] & 0x3f); + } if ((p[2] & 0xc0) == 0x80) { - if (len == 3) - return ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) - + (p[2] & 0x3f); + if (len == 3) { + return (((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + + (p[2] & 0x3f)); + } if ((p[3] & 0xc0) == 0x80) { - if (len == 4) - return ((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12) - + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f); + if (len == 4) { + return (((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12) + + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f)); + } if ((p[4] & 0xc0) == 0x80) { - if (len == 5) - return ((p[0] & 0x03) << 24) + ((p[1] & 0x3f) << 18) - + ((p[2] & 0x3f) << 12) + ((p[3] & 0x3f) << 6) - + (p[4] & 0x3f); - if ((p[5] & 0xc0) == 0x80 && len == 6) - return ((p[0] & 0x01) << 30) + ((p[1] & 0x3f) << 24) - + ((p[2] & 0x3f) << 18) + ((p[3] & 0x3f) << 12) - + ((p[4] & 0x3f) << 6) + (p[5] & 0x3f); + if (len == 5) { + return (((p[0] & 0x03) << 24) + ((p[1] & 0x3f) << 18) + + ((p[2] & 0x3f) << 12) + ((p[3] & 0x3f) << 6) + + (p[4] & 0x3f)); + } + if ((p[5] & 0xc0) == 0x80 && len == 6) { + return (((p[0] & 0x01) << 30) + ((p[1] & 0x3f) << 24) + + ((p[2] & 0x3f) << 18) + ((p[3] & 0x3f) << 12) + + ((p[4] & 0x3f) << 6) + (p[5] & 0x3f)); + } } } } } - /* Illegal value, just return the first byte */ + // Illegal value: just return the first byte. return p[0]; } @@ -767,23 +796,24 @@ int utfc_char2bytes(int off, char_u *buf) return len; } -/* - * Get the length of a UTF-8 byte sequence, not including any following - * composing characters. - * Returns 0 for "". - * Returns 1 for an illegal byte sequence. - */ -int utf_ptr2len(const char_u *p) +/// Get the length of a UTF-8 byte sequence representing a single codepoint +/// +/// @param[in] p UTF-8 string. +/// +/// @return Sequence length, 0 for empty string and 1 for non-UTF-8 byte +/// sequence. +int utf_ptr2len(const char_u *const p) + FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - int len; - int i; - - if (*p == NUL) + if (*p == NUL) { return 0; - len = utf8len_tab[*p]; - for (i = 1; i < len; ++i) - if ((p[i] & 0xc0) != 0x80) + } + const int len = utf8len_tab[*p]; + for (int i = 1; i < len; i++) { + if ((p[i] & 0xc0) != 0x80) { return 1; + } + } return len; } @@ -824,38 +854,38 @@ int utf_ptr2len_len(const char_u *p, int size) return len; } -/* - * Return the number of bytes the UTF-8 encoding of the character at "p" takes. - * This includes following composing characters. - */ -int utfc_ptr2len(const char_u *p) +/// Return the number of bytes occupied by a UTF-8 character in a string +/// +/// This includes following composing characters. +int utfc_ptr2len(const char_u *const p) + FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - int len; - int b0 = *p; - int prevlen; + uint8_t b0 = (uint8_t)(*p); - if (b0 == NUL) + if (b0 == NUL) { return 0; - if (b0 < 0x80 && p[1] < 0x80) /* be quick for ASCII */ + } + if (b0 < 0x80 && p[1] < 0x80) { // be quick for ASCII return 1; + } - /* Skip over first UTF-8 char, stopping at a NUL byte. */ - len = utf_ptr2len(p); + // Skip over first UTF-8 char, stopping at a NUL byte. + int len = utf_ptr2len(p); - /* Check for illegal byte. */ - if (len == 1 && b0 >= 0x80) + // Check for illegal byte. + if (len == 1 && b0 >= 0x80) { return 1; + } - /* - * Check for composing characters. We can handle only the first six, but - * skip all of them (otherwise the cursor would get stuck). - */ - prevlen = 0; - for (;; ) { - if (p[len] < 0x80 || !UTF_COMPOSINGLIKE(p + prevlen, p + len)) + // Check for composing characters. We can handle only the first six, but + // skip all of them (otherwise the cursor would get stuck). + int prevlen = 0; + for (;;) { + if (p[len] < 0x80 || !UTF_COMPOSINGLIKE(p + prevlen, p + len)) { return len; + } - /* Skip over composing char */ + // Skip over composing char. prevlen = len; len += utf_ptr2len(p + len); } @@ -913,23 +943,22 @@ int utfc_ptr2len_len(const char_u *p, int size) return len; } -/* - * Return the number of bytes the UTF-8 encoding of character "c" takes. - * This does not include composing characters. - */ -int utf_char2len(int c) +/// Determine how many bytes certain unicode codepoint will occupy +int utf_char2len(const int c) { - if (c < 0x80) + if (c < 0x80) { return 1; - if (c < 0x800) + } else if (c < 0x800) { return 2; - if (c < 0x10000) + } else if (c < 0x10000) { return 3; - if (c < 0x200000) + } else if (c < 0x200000) { return 4; - if (c < 0x4000000) + } else if (c < 0x4000000) { return 5; - return 6; + } else { + return 6; + } } /// Convert Unicode character to UTF-8 string @@ -937,46 +966,42 @@ int utf_char2len(int c) /// @param c character to convert to \p buf /// @param[out] buf UTF-8 string generated from \p c, does not add \0 /// @return Number of bytes (1-6). Does not include composing characters. -int utf_char2bytes(int c, char_u *const buf) +int utf_char2bytes(const int c, char_u *const buf) { - if (c < 0x80) { /* 7 bits */ + if (c < 0x80) { // 7 bits buf[0] = c; return 1; - } - if (c < 0x800) { /* 11 bits */ + } else if (c < 0x800) { // 11 bits buf[0] = 0xc0 + ((unsigned)c >> 6); buf[1] = 0x80 + (c & 0x3f); return 2; - } - if (c < 0x10000) { /* 16 bits */ + } else if (c < 0x10000) { // 16 bits buf[0] = 0xe0 + ((unsigned)c >> 12); buf[1] = 0x80 + (((unsigned)c >> 6) & 0x3f); buf[2] = 0x80 + (c & 0x3f); return 3; - } - if (c < 0x200000) { /* 21 bits */ + } else if (c < 0x200000) { // 21 bits buf[0] = 0xf0 + ((unsigned)c >> 18); buf[1] = 0x80 + (((unsigned)c >> 12) & 0x3f); buf[2] = 0x80 + (((unsigned)c >> 6) & 0x3f); buf[3] = 0x80 + (c & 0x3f); return 4; - } - if (c < 0x4000000) { /* 26 bits */ + } else if (c < 0x4000000) { // 26 bits buf[0] = 0xf8 + ((unsigned)c >> 24); buf[1] = 0x80 + (((unsigned)c >> 18) & 0x3f); buf[2] = 0x80 + (((unsigned)c >> 12) & 0x3f); buf[3] = 0x80 + (((unsigned)c >> 6) & 0x3f); buf[4] = 0x80 + (c & 0x3f); return 5; + } else { // 31 bits + buf[0] = 0xfc + ((unsigned)c >> 30); + buf[1] = 0x80 + (((unsigned)c >> 24) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 18) & 0x3f); + buf[3] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[4] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[5] = 0x80 + (c & 0x3f); + return 6; } - /* 31 bits */ - buf[0] = 0xfc + ((unsigned)c >> 30); - buf[1] = 0x80 + (((unsigned)c >> 24) & 0x3f); - buf[2] = 0x80 + (((unsigned)c >> 18) & 0x3f); - buf[3] = 0x80 + (((unsigned)c >> 12) & 0x3f); - buf[4] = 0x80 + (((unsigned)c >> 6) & 0x3f); - buf[5] = 0x80 + (c & 0x3f); - return 6; } /* @@ -1513,14 +1538,15 @@ int utf_head_off(const char_u *base, const char_u *p) return (int)(p - q); } -/* - * Copy a character from "*fp" to "*tp" and advance the pointers. - */ -void mb_copy_char(const char_u **fp, char_u **tp) +/// Copy a character, advancing the pointers +/// +/// @param[in,out] fp Source of the character to copy. +/// @param[in,out] tp Destination to copy to. +void mb_copy_char(const char_u **const fp, char_u **const tp) { - int l = (*mb_ptr2len)(*fp); + const size_t l = (size_t)utfc_ptr2len(*fp); - memmove(*tp, *fp, (size_t)l); + memmove(*tp, *fp, l); *tp += l; *fp += l; } @@ -2262,9 +2288,7 @@ int convert_setup_ext(vimconv_T *vcp, char_u *from, bool from_unicode_is_utf8, if (vcp->vc_type == CONV_ICONV && vcp->vc_fd != (iconv_t)-1) iconv_close(vcp->vc_fd); # endif - vcp->vc_type = CONV_NONE; - vcp->vc_factor = 1; - vcp->vc_fail = false; + *vcp = (vimconv_T)MBYTE_NONE_CONV; /* No conversion when one of the names is empty or they are equal. */ if (from == NULL || *from == NUL || to == NULL || *to == NUL diff --git a/src/nvim/mbyte.h b/src/nvim/mbyte.h index bf6ccb13a5..a5ce1b0a15 100644 --- a/src/nvim/mbyte.h +++ b/src/nvim/mbyte.h @@ -60,6 +60,12 @@ typedef enum { CONV_ICONV = 5, } ConvFlags; +#define MBYTE_NONE_CONV { \ + .vc_type = CONV_NONE, \ + .vc_factor = 1, \ + .vc_fail = false, \ +} + /// Structure used for string conversions typedef struct { int vc_type; ///< Zero or more ConvFlags. @@ -73,6 +79,8 @@ typedef struct { extern const uint8_t utf8len_tab_zero[256]; +extern const uint8_t utf8len_tab[256]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "mbyte.h.generated.h" #endif diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index 70bda42d83..d787790bc3 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -5929,8 +5929,7 @@ static void syntime_report(void) // // When making changes here, also change runtime/colors/default.vim! -static char *highlight_init_both[] = -{ +static const char *highlight_init_both[] = { "Conceal ctermbg=DarkGrey ctermfg=LightGrey guibg=DarkGrey guifg=LightGrey", "Cursor guibg=fg guifg=bg", "lCursor guibg=fg guifg=bg", @@ -5954,8 +5953,7 @@ static char *highlight_init_both[] = NULL }; -static char *highlight_init_light[] = -{ +static const char *highlight_init_light[] = { "ColorColumn ctermbg=LightRed guibg=LightRed", "CursorColumn ctermbg=LightGrey guibg=Grey90", "CursorLine cterm=underline guibg=Grey90", @@ -5988,8 +5986,7 @@ static char *highlight_init_light[] = NULL }; -static char *highlight_init_dark[] = -{ +static const char *highlight_init_dark[] = { "ColorColumn ctermbg=DarkRed guibg=DarkRed", "CursorColumn ctermbg=DarkGrey guibg=Grey40", "CursorLine cterm=underline guibg=Grey40", @@ -6022,6 +6019,192 @@ static char *highlight_init_dark[] = NULL }; +static const char *highlight_init_cmdline[] = { + // NVimInternalError should appear only when highlighter has a bug. + "NVimInternalError ctermfg=Red ctermbg=Red guifg=Red guibg=Red", + + // Highlight groups (links) used by parser: + + "default link NVimOperator Operator", + + "default link NVimUnaryOperator NVimOperator", + "default link NVimUnaryPlus NVimUnaryOperator", + "default link NVimUnaryMinus NVimUnaryOperator", + "default link NVimNot NVimUnaryOperator", + + "default link NVimBinaryOperator NVimOperator", + "default link NVimComparison NVimBinaryOperator", + "default link NVimComparisonModifier NVimComparison", + "default link NVimBinaryPlus NVimBinaryOperator", + "default link NVimConcat NVimBinaryOperator", + "default link NVimConcatOrSubscript NVimConcat", + "default link NVimOr NVimBinaryOperator", + "default link NVimAnd NVimBinaryOperator", + "default link NVimMultiplication NVimBinaryOperator", + "default link NVimDivision NVimBinaryOperator", + "default link NVimMod NVimBinaryOperator", + + "default link NVimTernary NVimOperator", + "default link NVimTernaryColon NVimTernary", + + "default link NVimParenthesis Delimiter", + "default link NVimLambda NVimParenthesis", + "default link NVimNestingParenthesis NVimParenthesis", + "default link NVimCallingParenthesis NVimParenthesis", + + "default link NVimSubscript NVimParenthesis", + "default link NVimSubscriptBracket NVimSubscript", + "default link NVimSubscriptColon NVimSubscript", + "default link NVimSubscriptColon NVimSubscript", + "default link NVimCurly NVimSubscript", + + "default link NVimContainer NVimParenthesis", + "default link NVimDict NVimContainer", + "default link NVimList NVimContainer", + + "default link NVimIdentifier Identifier", + "default link NVimIdentifierScope NVimIdentifier", + "default link NVimIdentifierScopeDelimiter NVimIdentifier", + "default link NVimIdentifierName NVimIdentifier", + "default link NVimIdentifierKey NVimIdentifier", + + "default link NVimColon Delimiter", + "default link NVimComma Delimiter", + "default link NVimArrow Delimiter", + + "default link NVimRegister SpecialChar", + "default link NVimNumber Number", + "default link NVimFloat NVimNumber", + "default link NVimNumberPrefix Type", + + "default link NVimOptionSigil Type", + "default link NVimOptionName NVimIdentifier", + "default link NVimOptionScope NVimIdentifierScope", + "default link NVimOptionScopeDelimiter NVimIdentifierScopeDelimiter", + + "default link NVimEnvironmentSigil NVimOptionSigil", + "default link NVimEnvironmentName NVimIdentifier", + + "default link NVimString String", + "default link NVimStringBody NVimString", + "default link NVimStringQuote NVimString", + "default link NVimStringSpecial SpecialChar", + + "default link NVimSingleQuote NVimStringQuote", + "default link NVimSingleQuotedBody NVimStringBody", + "default link NVimSingleQuotedQuote NVimStringSpecial", + + "default link NVimDoubleQuote NVimStringQuote", + "default link NVimDoubleQuotedBody NVimStringBody", + "default link NVimDoubleQuotedEscape NVimStringSpecial", + // Not actually invalid, but we highlight user that he is doing something + // wrong. + "default link NVimDoubleQuotedUnknownEscape NVimInvalidValue", + + "default link NVimFigureBrace NVimInternalError", + "default link NVimSingleQuotedUnknownEscape NVimInternalError", + + "default link NVimSpacing Normal", + + // NVimInvalid groups: + + "default link NVimInvalidSingleQuotedUnknownEscape NVimInternalError", + + "default link NVimInvalid Error", + + "default link NVimInvalidOperator NVimInvalid", + + "default link NVimInvalidUnaryOperator NVimInvalidOperator", + "default link NVimInvalidUnaryPlus NVimInvalidUnaryOperator", + "default link NVimInvalidUnaryMinus NVimInvalidUnaryOperator", + "default link NVimInvalidNot NVimInvalidUnaryOperator", + + "default link NVimInvalidBinaryOperator NVimInvalidOperator", + "default link NVimInvalidComparison NVimInvalidBinaryOperator", + "default link NVimInvalidComparisonModifier NVimInvalidComparison", + "default link NVimInvalidBinaryPlus NVimInvalidBinaryOperator", + "default link NVimInvalidConcat NVimInvalidBinaryOperator", + "default link NVimInvalidConcatOrSubscript NVimInvalidConcat", + "default link NVimInvalidOr NVimInvalidBinaryOperator", + "default link NVimInvalidAnd NVimInvalidBinaryOperator", + "default link NVimInvalidMultiplication NVimInvalidBinaryOperator", + "default link NVimInvalidDivision NVimInvalidBinaryOperator", + "default link NVimInvalidMod NVimInvalidBinaryOperator", + + "default link NVimInvalidTernary NVimInvalidOperator", + "default link NVimInvalidTernaryColon NVimInvalidTernary", + + "default link NVimInvalidDelimiter NVimInvalid", + + "default link NVimInvalidParenthesis NVimInvalidDelimiter", + "default link NVimInvalidLambda NVimInvalidParenthesis", + "default link NVimInvalidNestingParenthesis NVimInvalidParenthesis", + "default link NVimInvalidCallingParenthesis NVimInvalidParenthesis", + + "default link NVimInvalidSubscript NVimInvalidParenthesis", + "default link NVimInvalidSubscriptBracket NVimInvalidSubscript", + "default link NVimInvalidSubscriptColon NVimInvalidSubscript", + "default link NVimInvalidSubscriptColon NVimInvalidSubscript", + "default link NVimInvalidCurly NVimInvalidSubscript", + + "default link NVimInvalidContainer NVimInvalidParenthesis", + "default link NVimInvalidDict NVimInvalidContainer", + "default link NVimInvalidList NVimInvalidContainer", + + "default link NVimInvalidIdentifier Identifier", + "default link NVimInvalidIdentifierScope NVimIdentifier", + "default link NVimInvalidIdentifierScopeDelimiter NVimIdentifier", + "default link NVimInvalidIdentifierName NVimIdentifier", + "default link NVimInvalidIdentifierKey NVimIdentifier", + + "default link NVimInvalidColon NVimInvalidDelimiter", + "default link NVimInvalidComma NVimInvalidDelimiter", + "default link NVimInvalidArrow NVimInvalidDelimiter", + + "default link NVimInvalidValue NVimInvalid", + + "default link NVimInvalidRegister NVimInvalidValue", + "default link NVimInvalidNumber NVimInvalidValue", + "default link NVimInvalidFloat NVimInvalidNumber", + "default link NVimInvalidNumberPrefix NVimInvalidNumber", + + "default link NVimInvalidOptionSigil NVimInvalidIdentifier", + "default link NVimInvalidOptionName NVimInvalidIdentifier", + "default link NVimInvalidOptionScope NVimInvalidIdentifierScope", + "default link NVimInvalidOptionScopeDelimiter " + "NVimInvalidIdentifierScopeDelimiter", + + "default link NVimInvalidEnvironmentSigil NVimInvalidOptionSigil", + "default link NVimInvalidEnvironmentName NVimInvalidIdentifier", + + // Invalid string bodies and specials are still highlighted as valid ones to + // minimize the red area. + "default link NVimInvalidString NVimInvalidValue", + "default link NVimInvalidStringBody NVimString", + "default link NVimInvalidStringQuote NVimInvalidString", + "default link NVimInvalidStringSpecial NVimStringSpecial", + + "default link NVimInvalidSingleQuote NVimInvalidStringQuote", + "default link NVimInvalidSingleQuotedBody NVimInvalidStringBody", + "default link NVimInvalidSingleQuotedQuote NVimInvalidStringSpecial", + + "default link NVimInvalidDoubleQuote NVimInvalidStringQuote", + "default link NVimInvalidDoubleQuotedBody NVimInvalidStringBody", + "default link NVimInvalidDoubleQuotedEscape NVimInvalidStringSpecial", + "default link NVimInvalidDoubleQuotedUnknownEscape NVimInvalidValue", + + "default link NVimInvalidFigureBrace NVimInvalidDelimiter", + + "default link NVimInvalidSpacing ErrorMsg", +}; + +/// Create default links for NVim* highlight groups used for cmdline coloring +void syn_init_cmdline_highlight(bool reset, bool init) +{ + for (size_t i = 0 ; i < ARRAY_SIZE(highlight_init_cmdline) ; i++) { + do_highlight(highlight_init_cmdline[i], reset, init); + } +} /// Load colors from a file if "g:colors_name" is set, otherwise load builtin /// colors @@ -6032,7 +6215,6 @@ void init_highlight(int both, int reset) { int i; - char **pp; static int had_both = FALSE; // Try finding the color scheme file. Used when a color file was loaded @@ -6054,9 +6236,9 @@ init_highlight(int both, int reset) */ if (both) { had_both = TRUE; - pp = highlight_init_both; + const char *const *const pp = highlight_init_both; for (i = 0; pp[i] != NULL; i++) { - do_highlight((char_u *)pp[i], reset, true); + do_highlight(pp[i], reset, true); } } else if (!had_both) { // Don't do anything before the call with both == TRUE from main(). @@ -6065,10 +6247,11 @@ init_highlight(int both, int reset) return; } - pp = (*p_bg == 'l') ? highlight_init_light : highlight_init_dark; - + const char *const *const pp = ((*p_bg == 'l') + ? highlight_init_light + : highlight_init_dark); for (i = 0; pp[i] != NULL; i++) { - do_highlight((char_u *)pp[i], reset, true); + do_highlight(pp[i], reset, true); } /* Reverse looks ugly, but grey may not work for 8 colors. Thus let it @@ -6078,15 +6261,13 @@ init_highlight(int both, int reset) * Clear the attributes, needed when changing the t_Co value. */ if (t_colors > 8) { do_highlight( - (char_u *)(*p_bg == 'l' - ? "Visual cterm=NONE ctermbg=LightGrey" - : "Visual cterm=NONE ctermbg=DarkGrey"), false, - true); + (*p_bg == 'l' + ? "Visual cterm=NONE ctermbg=LightGrey" + : "Visual cterm=NONE ctermbg=DarkGrey"), false, true); } else { - do_highlight((char_u *)"Visual cterm=reverse ctermbg=NONE", - FALSE, TRUE); + do_highlight("Visual cterm=reverse ctermbg=NONE", false, true); if (*p_bg == 'l') - do_highlight((char_u *)"Search ctermfg=black", FALSE, TRUE); + do_highlight("Search ctermfg=black", false, true); } /* @@ -6103,6 +6284,7 @@ init_highlight(int both, int reset) recursive--; } } + syn_init_cmdline_highlight(false, false); } /* @@ -6137,17 +6319,22 @@ int load_colors(char_u *name) } -/// Handle the ":highlight .." command. -/// When using ":hi clear" this is called recursively for each group with -/// "forceit" and "init" both TRUE. -/// @param init TRUE when called for initializing -void -do_highlight(char_u *line, int forceit, int init) { - char_u *name_end; - char_u *linep; - char_u *key_start; - char_u *arg_start; - char_u *key = NULL, *arg = NULL; +/// Handle ":highlight" command +/// +/// When using ":highlight clear" this is called recursively for each group with +/// forceit and init being both true. +/// +/// @param[in] line Command arguments. +/// @param[in] forceit True when bang is given, allows to link group even if +/// it has its own settings. +/// @param[in] init True when initializing. +void do_highlight(const char *line, const bool forceit, const bool init) + FUNC_ATTR_NONNULL_ALL +{ + const char *name_end; + const char *linep; + const char *key_start; + const char *arg_start; long i; int off; int len; @@ -6161,94 +6348,87 @@ do_highlight(char_u *line, int forceit, int init) { int color; bool is_normal_group = false; // "Normal" group - /* - * If no argument, list current highlighting. - */ - if (ends_excmd(*line)) { + // If no argument, list current highlighting. + if (ends_excmd((uint8_t)(*line))) { for (int i = 1; i <= highlight_ga.ga_len && !got_int; i++) { - // todo(vim): only call when the group has attributes set + // TODO(brammool): only call when the group has attributes set highlight_list_one(i); } return; } - /* - * Isolate the name. - */ - name_end = skiptowhite(line); - linep = skipwhite(name_end); + // Isolate the name. + name_end = (const char *)skiptowhite((const char_u *)line); + linep = (const char *)skipwhite((const char_u *)name_end); - /* - * Check for "default" argument. - */ - if (STRNCMP(line, "default", name_end - line) == 0) { - dodefault = TRUE; + // Check for "default" argument. + if (strncmp(line, "default", name_end - line) == 0) { + dodefault = true; line = linep; - name_end = skiptowhite(line); - linep = skipwhite(name_end); + name_end = (const char *)skiptowhite((const char_u *)line); + linep = (const char *)skipwhite((const char_u *)name_end); } - /* - * Check for "clear" or "link" argument. - */ - if (STRNCMP(line, "clear", name_end - line) == 0) - doclear = TRUE; - if (STRNCMP(line, "link", name_end - line) == 0) - dolink = TRUE; + // Check for "clear" or "link" argument. + if (strncmp(line, "clear", name_end - line) == 0) { + doclear = true; + } else if (strncmp(line, "link", name_end - line) == 0) { + dolink = true; + } - /* - * ":highlight {group-name}": list highlighting for one group. - */ - if (!doclear && !dolink && ends_excmd(*linep)) { - id = syn_namen2id(line, (int)(name_end - line)); - if (id == 0) - EMSG2(_("E411: highlight group not found: %s"), line); - else + // ":highlight {group-name}": list highlighting for one group. + if (!doclear && !dolink && ends_excmd((uint8_t)(*linep))) { + id = syn_namen2id((const char_u *)line, (int)(name_end - line)); + if (id == 0) { + emsgf(_("E411: highlight group not found: %s"), line); + } else { highlight_list_one(id); + } return; } - /* - * Handle ":highlight link {from} {to}" command. - */ + // Handle ":highlight link {from} {to}" command. if (dolink) { - char_u *from_start = linep; - char_u *from_end; - char_u *to_start; - char_u *to_end; + const char *from_start = linep; + const char *from_end; + const char *to_start; + const char *to_end; int from_id; int to_id; - from_end = skiptowhite(from_start); - to_start = skipwhite(from_end); - to_end = skiptowhite(to_start); + from_end = (const char *)skiptowhite((const char_u *)from_start); + to_start = (const char *)skipwhite((const char_u *)from_end); + to_end = (const char *)skiptowhite((const char_u *)to_start); - if (ends_excmd(*from_start) || ends_excmd(*to_start)) { - EMSG2(_("E412: Not enough arguments: \":highlight link %s\""), - from_start); + if (ends_excmd((uint8_t)(*from_start)) + || ends_excmd((uint8_t)(*to_start))) { + emsgf(_("E412: Not enough arguments: \":highlight link %s\""), + from_start); return; } - if (!ends_excmd(*skipwhite(to_end))) { - EMSG2(_("E413: Too many arguments: \":highlight link %s\""), from_start); + if (!ends_excmd(*skipwhite((const char_u *)to_end))) { + emsgf(_("E413: Too many arguments: \":highlight link %s\""), from_start); return; } - from_id = syn_check_group(from_start, (int)(from_end - from_start)); - if (STRNCMP(to_start, "NONE", 4) == 0) + from_id = syn_check_group((const char_u *)from_start, + (int)(from_end - from_start)); + if (strncmp(to_start, "NONE", 4) == 0) { to_id = 0; - else - to_id = syn_check_group(to_start, (int)(to_end - to_start)); + } else { + to_id = syn_check_group((const char_u *)to_start, + (int)(to_end - to_start)); + } if (from_id > 0 && (!init || HL_TABLE()[from_id - 1].sg_set == 0)) { - /* - * Don't allow a link when there already is some highlighting - * for the group, unless '!' is used - */ + // Don't allow a link when there already is some highlighting + // for the group, unless '!' is used if (to_id > 0 && !forceit && !init && hl_has_settings(from_id - 1, dodefault)) { - if (sourcing_name == NULL && !dodefault) + if (sourcing_name == NULL && !dodefault) { EMSG(_("E414: group has settings, highlight link ignored")); + } } else { if (!init) HL_TABLE()[from_id - 1].sg_set |= SG_LINK; @@ -6258,43 +6438,38 @@ do_highlight(char_u *line, int forceit, int init) { } } - /* Only call highlight_changed() once, after sourcing a syntax file */ - need_highlight_changed = TRUE; + // Only call highlight_changed() once, after sourcing a syntax file. + need_highlight_changed = true; return; } if (doclear) { - /* - * ":highlight clear [group]" command. - */ + // ":highlight clear [group]" command. line = linep; - if (ends_excmd(*line)) { + if (ends_excmd((uint8_t)(*line))) { do_unlet(S_LEN("colors_name"), true); restore_cterm_colors(); - /* - * Clear all default highlight groups and load the defaults. - */ + // Clear all default highlight groups and load the defaults. for (int idx = 0; idx < highlight_ga.ga_len; ++idx) { highlight_clear(idx); } - init_highlight(TRUE, TRUE); + init_highlight(true, true); highlight_changed(); redraw_later_clear(); return; } - name_end = skiptowhite(line); - linep = skipwhite(name_end); + name_end = (const char *)skiptowhite((const char_u *)line); + linep = (const char *)skipwhite((const char_u *)name_end); } - /* - * Find the group name in the table. If it does not exist yet, add it. - */ - id = syn_check_group(line, (int)(name_end - line)); - if (id == 0) /* failed (out of memory) */ + // Find the group name in the table. If it does not exist yet, add it. + id = syn_check_group((const char_u *)line, (int)(name_end - line)); + if (id == 0) { // Failed (out of memory). return; - idx = id - 1; /* index is ID minus one */ + } + idx = id - 1; // Index is ID minus one. // Return if "default" was used and the group already has settings if (dodefault && hl_has_settings(idx, true)) { @@ -6303,19 +6478,21 @@ do_highlight(char_u *line, int forceit, int init) { is_normal_group = (STRCMP(HL_TABLE()[idx].sg_name_u, "NORMAL") == 0); - /* Clear the highlighting for ":hi clear {group}" and ":hi clear". */ + // Clear the highlighting for ":hi clear {group}" and ":hi clear". if (doclear || (forceit && init)) { highlight_clear(idx); if (!doclear) HL_TABLE()[idx].sg_set = 0; } + char *key = NULL; + char *arg = NULL; if (!doclear) { - while (!ends_excmd(*linep)) { + while (!ends_excmd((uint8_t)(*linep))) { key_start = linep; if (*linep == '=') { - EMSG2(_("E415: unexpected equal sign: %s"), key_start); - error = TRUE; + emsgf(_("E415: unexpected equal sign: %s"), key_start); + error = true; break; } @@ -6325,61 +6502,58 @@ do_highlight(char_u *line, int forceit, int init) { linep++; } xfree(key); - key = vim_strnsave_up(key_start, (int)(linep - key_start)); - linep = skipwhite(linep); + key = (char *)vim_strnsave_up((const char_u *)key_start, + (int)(linep - key_start)); + linep = (const char *)skipwhite((const char_u *)linep); - if (STRCMP(key, "NONE") == 0) { + if (strcmp(key, "NONE") == 0) { if (!init || HL_TABLE()[idx].sg_set == 0) { - if (!init) + if (!init) { HL_TABLE()[idx].sg_set |= SG_CTERM+SG_GUI; + } highlight_clear(idx); } continue; } - /* - * Check for the equal sign. - */ + // Check for the equal sign. if (*linep != '=') { - EMSG2(_("E416: missing equal sign: %s"), key_start); - error = TRUE; + emsgf(_("E416: missing equal sign: %s"), key_start); + error = true; break; } - ++linep; + linep++; - /* - * Isolate the argument. - */ - linep = skipwhite(linep); - if (*linep == '\'') { /* guifg='color name' */ + // Isolate the argument. + linep = (const char *)skipwhite((const char_u *)linep); + if (*linep == '\'') { // guifg='color name' arg_start = ++linep; - linep = vim_strchr(linep, '\''); + linep = strchr(linep, '\''); if (linep == NULL) { - EMSG2(_(e_invarg2), key_start); - error = TRUE; + emsgf(_(e_invarg2), key_start); + error = true; break; } } else { arg_start = linep; - linep = skiptowhite(linep); + linep = (const char *)skiptowhite((const char_u *)linep); } if (linep == arg_start) { - EMSG2(_("E417: missing argument: %s"), key_start); - error = TRUE; + emsgf(_("E417: missing argument: %s"), key_start); + error = true; break; } xfree(arg); - arg = vim_strnsave(arg_start, (int)(linep - arg_start)); + arg = xstrndup(arg_start, (size_t)(linep - arg_start)); - if (*linep == '\'') - ++linep; + if (*linep == '\'') { + linep++; + } - /* - * Store the argument. - */ - if ( STRCMP(key, "TERM") == 0 - || STRCMP(key, "CTERM") == 0 - || STRCMP(key, "GUI") == 0) { + // Store the argument. + if (strcmp(key, "TERM") == 0 + || strcmp(key, "CTERM") == 0 + || strcmp(key, "GUI") == 0) { attr = 0; off = 0; while (arg[off] != NUL) { @@ -6392,26 +6566,30 @@ do_highlight(char_u *line, int forceit, int init) { } } if (i < 0) { - EMSG2(_("E418: Illegal value: %s"), arg); - error = TRUE; + emsgf(_("E418: Illegal value: %s"), arg); + error = true; break; } - if (arg[off] == ',') /* another one follows */ - ++off; + if (arg[off] == ',') { // Another one follows. + off++; + } } - if (error) + if (error) { break; + } if (*key == 'C') { if (!init || !(HL_TABLE()[idx].sg_set & SG_CTERM)) { - if (!init) + if (!init) { HL_TABLE()[idx].sg_set |= SG_CTERM; + } HL_TABLE()[idx].sg_cterm = attr; HL_TABLE()[idx].sg_cterm_bold = FALSE; } } else if (*key == 'G') { if (!init || !(HL_TABLE()[idx].sg_set & SG_GUI)) { - if (!init) + if (!init) { HL_TABLE()[idx].sg_set |= SG_GUI; + } HL_TABLE()[idx].sg_gui = attr; } } @@ -6430,14 +6608,14 @@ do_highlight(char_u *line, int forceit, int init) { HL_TABLE()[idx].sg_cterm_bold = FALSE; } - if (ascii_isdigit(*arg)) + if (ascii_isdigit(*arg)) { color = atoi((char *)arg); - else if (STRICMP(arg, "fg") == 0) { - if (cterm_normal_fg_color) + } else if (STRICMP(arg, "fg") == 0) { + if (cterm_normal_fg_color) { color = cterm_normal_fg_color - 1; - else { + } else { EMSG(_("E419: FG color unknown")); - error = TRUE; + error = true; break; } } else if (STRICMP(arg, "bg") == 0) { @@ -6445,70 +6623,85 @@ do_highlight(char_u *line, int forceit, int init) { color = cterm_normal_bg_color - 1; else { EMSG(_("E420: BG color unknown")); - error = TRUE; + error = true; break; } } else { - static char *(color_names[28]) = { + static const char *color_names[] = { "Black", "DarkBlue", "DarkGreen", "DarkCyan", "DarkRed", "DarkMagenta", "Brown", "DarkYellow", "Gray", "Grey", "LightGray", "LightGrey", "DarkGray", "DarkGrey", "Blue", "LightBlue", "Green", "LightGreen", "Cyan", "LightCyan", "Red", "LightRed", "Magenta", - "LightMagenta", "Yellow", "LightYellow", "White", "NONE" + "LightMagenta", "Yellow", "LightYellow", "White", + "NONE" + }; + static const int color_numbers_16[] = { + 0, 1, 2, 3, + 4, 5, 6, 6, + 7, 7, + 7, 7, 8, 8, + 9, 9, 10, 10, + 11, 11, 12, 12, 13, + 13, 14, 14, 15, + -1 + }; + // For xterm with 88 colors: + static int color_numbers_88[] = { + 0, 4, 2, 6, + 1, 5, 32, 72, + 84, 84, + 7, 7, 82, 82, + 12, 43, 10, 61, + 14, 63, 9, 74, 13, + 75, 11, 78, 15, + -1 }; - static int color_numbers_16[28] = {0, 1, 2, 3, - 4, 5, 6, 6, - 7, 7, - 7, 7, 8, 8, - 9, 9, 10, 10, - 11, 11, 12, 12, 13, - 13, 14, 14, 15, -1}; - /* for xterm with 88 colors... */ - static int color_numbers_88[28] = {0, 4, 2, 6, - 1, 5, 32, 72, - 84, 84, - 7, 7, 82, 82, - 12, 43, 10, 61, - 14, 63, 9, 74, 13, - 75, 11, 78, 15, -1}; - /* for xterm with 256 colors... */ - static int color_numbers_256[28] = {0, 4, 2, 6, - 1, 5, 130, 130, - 248, 248, - 7, 7, 242, 242, - 12, 81, 10, 121, - 14, 159, 9, 224, 13, - 225, 11, 229, 15, -1}; - /* for terminals with less than 16 colors... */ - static int color_numbers_8[28] = {0, 4, 2, 6, - 1, 5, 3, 3, - 7, 7, - 7, 7, 0+8, 0+8, - 4+8, 4+8, 2+8, 2+8, - 6+8, 6+8, 1+8, 1+8, 5+8, - 5+8, 3+8, 3+8, 7+8, -1}; - - /* reduce calls to STRICMP a bit, it can be slow */ + // For xterm with 256 colors: + static int color_numbers_256[] = { + 0, 4, 2, 6, + 1, 5, 130, 130, + 248, 248, + 7, 7, 242, 242, + 12, 81, 10, 121, + 14, 159, 9, 224, 13, + 225, 11, 229, 15, + -1 + }; + // For terminals with less than 16 colors: + static int color_numbers_8[28] = { + 0, 4, 2, 6, + 1, 5, 3, 3, + 7, 7, + 7, 7, 0+8, 0+8, + 4+8, 4+8, 2+8, 2+8, + 6+8, 6+8, 1+8, 1+8, 5+8, + 5+8, 3+8, 3+8, 7+8, + -1 + }; + + // Reduce calls to STRICMP a bit, it can be slow. off = TOUPPER_ASC(*arg); - for (i = ARRAY_SIZE(color_names); --i >= 0; ) + for (i = ARRAY_SIZE(color_names); --i >= 0; ) { if (off == color_names[i][0] - && STRICMP(arg + 1, color_names[i] + 1) == 0) + && STRICMP(arg + 1, color_names[i] + 1) == 0) { break; + } + } if (i < 0) { - EMSG2(_( + emsgf(_( "E421: Color name or number not recognized: %s"), key_start); - error = TRUE; + error = true; break; } - /* Use the _16 table to check if its a valid color name. */ + // Use the _16 table to check if its a valid color name. color = color_numbers_16[i]; if (color >= 0) { if (t_colors == 8) { - /* t_Co is 8: use the 8 colors table */ + // t_Co is 8: use the 8 colors table. color = color_numbers_8[i]; if (key[5] == 'F') { /* set/reset bold attribute to get light foreground @@ -6532,16 +6725,14 @@ do_highlight(char_u *line, int forceit, int init) { } } } - /* Add one to the argument, to avoid zero. Zero is used for - * "NONE", then "color" is -1. */ + // Add one to the argument, to avoid zero. Zero is used for + // "NONE", then "color" is -1. if (key[5] == 'F') { HL_TABLE()[idx].sg_cterm_fg = color + 1; if (is_normal_group) { cterm_normal_fg_color = color + 1; cterm_normal_fg_bold = (HL_TABLE()[idx].sg_cterm & HL_BOLD); - { - must_redraw = CLEAR; - } + must_redraw = CLEAR; } } else { HL_TABLE()[idx].sg_cterm_bg = color + 1; @@ -6565,15 +6756,15 @@ do_highlight(char_u *line, int forceit, int init) { } } } - } else if (STRCMP(key, "GUIFG") == 0) { + } else if (strcmp(key, "GUIFG") == 0) { if (!init || !(HL_TABLE()[idx].sg_set & SG_GUI)) { if (!init) HL_TABLE()[idx].sg_set |= SG_GUI; xfree(HL_TABLE()[idx].sg_rgb_fg_name); - if (STRCMP(arg, "NONE")) { - HL_TABLE()[idx].sg_rgb_fg_name = (uint8_t *)xstrdup((char *)arg); - HL_TABLE()[idx].sg_rgb_fg = name_to_color(arg); + if (strcmp(arg, "NONE")) { + HL_TABLE()[idx].sg_rgb_fg_name = (char_u *)xstrdup((char *)arg); + HL_TABLE()[idx].sg_rgb_fg = name_to_color((const char_u *)arg); } else { HL_TABLE()[idx].sg_rgb_fg_name = NULL; HL_TABLE()[idx].sg_rgb_fg = -1; @@ -6590,8 +6781,8 @@ do_highlight(char_u *line, int forceit, int init) { xfree(HL_TABLE()[idx].sg_rgb_bg_name); if (STRCMP(arg, "NONE") != 0) { - HL_TABLE()[idx].sg_rgb_bg_name = (uint8_t *)xstrdup((char *)arg); - HL_TABLE()[idx].sg_rgb_bg = name_to_color(arg); + HL_TABLE()[idx].sg_rgb_bg_name = (char_u *)xstrdup((char *)arg); + HL_TABLE()[idx].sg_rgb_bg = name_to_color((const char_u *)arg); } else { HL_TABLE()[idx].sg_rgb_bg_name = NULL; HL_TABLE()[idx].sg_rgb_bg = -1; @@ -6601,15 +6792,15 @@ do_highlight(char_u *line, int forceit, int init) { if (is_normal_group) { normal_bg = HL_TABLE()[idx].sg_rgb_bg; } - } else if (STRCMP(key, "GUISP") == 0) { + } else if (strcmp(key, "GUISP") == 0) { if (!init || !(HL_TABLE()[idx].sg_set & SG_GUI)) { if (!init) HL_TABLE()[idx].sg_set |= SG_GUI; xfree(HL_TABLE()[idx].sg_rgb_sp_name); - if (STRCMP(arg, "NONE") != 0) { - HL_TABLE()[idx].sg_rgb_sp_name = (uint8_t *)xstrdup((char *)arg); - HL_TABLE()[idx].sg_rgb_sp = name_to_color(arg); + if (strcmp(arg, "NONE") != 0) { + HL_TABLE()[idx].sg_rgb_sp_name = (char_u *)xstrdup((char *)arg); + HL_TABLE()[idx].sg_rgb_sp = name_to_color((const char_u *)arg); } else { HL_TABLE()[idx].sg_rgb_sp_name = NULL; HL_TABLE()[idx].sg_rgb_sp = -1; @@ -6619,31 +6810,25 @@ do_highlight(char_u *line, int forceit, int init) { if (is_normal_group) { normal_sp = HL_TABLE()[idx].sg_rgb_sp; } - } else if (STRCMP(key, "START") == 0 || STRCMP(key, "STOP") == 0) { + } else if (strcmp(key, "START") == 0 || strcmp(key, "STOP") == 0) { // Ignored for now } else { - EMSG2(_("E423: Illegal argument: %s"), key_start); - error = TRUE; + emsgf(_("E423: Illegal argument: %s"), key_start); + error = true; break; } - /* - * When highlighting has been given for a group, don't link it. - */ + // When highlighting has been given for a group, don't link it. if (!init || !(HL_TABLE()[idx].sg_set & SG_LINK)) { HL_TABLE()[idx].sg_link = 0; } - /* - * Continue with next argument. - */ - linep = skipwhite(linep); + // Continue with next argument. + linep = (const char *)skipwhite((const char_u *)linep); } } - /* - * If there is an error, and it's a new entry, remove it from the table. - */ + // If there is an error, and it's a new entry, remove it from the table. if (error && idx == highlight_ga.ga_len) { syn_unadd_group(); } else { @@ -7201,7 +7386,7 @@ char_u *syn_id2name(int id) /* * Like syn_name2id(), but take a pointer + length argument. */ -int syn_namen2id(char_u *linep, int len) +int syn_namen2id(const char_u *linep, int len) { char_u *name = vim_strnsave(linep, len); int id = syn_name2id(name); @@ -7217,7 +7402,7 @@ int syn_namen2id(char_u *linep, int len) /// @param len length of \p pp /// /// @return 0 for failure else the id of the group -int syn_check_group(char_u *pp, int len) +int syn_check_group(const char_u *pp, int len) { char_u *name = vim_strnsave(pp, len); int id = syn_name2id(name); @@ -8220,7 +8405,7 @@ color_name_table_T color_name_table[] = { /// /// @param[in] name string value to convert to RGB /// return the hex value or -1 if could not find a correct value -RgbValue name_to_color(const uint8_t *name) +RgbValue name_to_color(const char_u *name) { if (name[0] == '#' && isxdigit(name[1]) && isxdigit(name[2]) diff --git a/src/nvim/version.c b/src/nvim/version.c index a31381eddf..17f60f309c 100644 --- a/src/nvim/version.c +++ b/src/nvim/version.c @@ -78,6 +78,8 @@ static char *features[] = { // clang-format off static const int included_patches[] = { + 1229, + 1230, // 1026, 1025, 1024, diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c new file mode 100644 index 0000000000..f5bc547d54 --- /dev/null +++ b/src/nvim/viml/parser/expressions.c @@ -0,0 +1,2859 @@ +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +/// VimL expression parser + +#include <stdbool.h> +#include <stddef.h> +#include <assert.h> +#include <string.h> + +#include "nvim/vim.h" +#include "nvim/memory.h" +#include "nvim/types.h" +#include "nvim/charset.h" +#include "nvim/ascii.h" +#include "nvim/assert.h" +#include "nvim/lib/kvec.h" +#include "nvim/eval/typval.h" + +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" + +#define vim_str2nr(s, ...) vim_str2nr((const char_u *)(s), __VA_ARGS__) + +typedef kvec_withinit_t(ExprASTNode **, 16) ExprASTStack; + +/// Which nodes may be wanted +typedef enum { + /// Operators: function call, subscripts, binary operators, … + /// + /// For unrestricted expressions. + kENodeOperator, + /// Values: literals, variables, nested expressions, unary operators. + /// + /// For unrestricted expressions as well, implies that top item in AST stack + /// points to NULL. + kENodeValue, + /// Argument: only allows simple argument names. + kENodeArgument, + /// Argument separator: only allows commas. + kENodeArgumentSeparator, +} ExprASTWantedNode; + +/// Operator priority level +typedef enum { + kEOpLvlInvalid = 0, + kEOpLvlComplexIdentifier, + kEOpLvlParens, + kEOpLvlArrow, + kEOpLvlComma, + kEOpLvlColon, + kEOpLvlTernaryValue, + kEOpLvlTernary, + kEOpLvlOr, + kEOpLvlAnd, + kEOpLvlComparison, + kEOpLvlAddition, ///< Addition, subtraction and concatenation. + kEOpLvlMultiplication, ///< Multiplication, division and modulo. + kEOpLvlUnary, ///< Unary operations: not, minus, plus. + kEOpLvlSubscript, ///< Subscripts. + kEOpLvlValue, ///< Values: literals, variables, nested expressions, … +} ExprOpLvl; + +/// Operator associativity +typedef enum { + kEOpAssNo= 'n', ///< Not associative / not applicable. + kEOpAssLeft = 'l', ///< Left associativity. + kEOpAssRight = 'r', ///< Right associativity. +} ExprOpAssociativity; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/expressions.c.generated.h" +#endif + +/// Character used as a separator in autoload function/variable names. +#define AUTOLOAD_CHAR '#' + +/// Scale number by a given factor +/// +/// Used to apply exponent to a number. Idea taken from uClibc. +/// +/// @param[in] num Number to scale. Does not bother doing anything if it is +/// zero. +/// @param[in] base Base, should be 10 since non-decimal floating-point +/// numbers are not supported. +/// @param[in] exponent Exponent to scale by. +/// @param[in] exponent_negative True if exponent is negative. +static inline float_T scale_number(const float_T num, + const uint8_t base, + const uvarnumber_T exponent, + const bool exponent_negative) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_CONST +{ + if (num == 0 || exponent == 0) { + return num; + } + assert(base); + uvarnumber_T exp = exponent; + float_T p_base = (float_T)base; + float_T ret = num; + while (exp) { + if (exp & 1) { + if (exponent_negative) { + ret /= p_base; + } else { + ret *= p_base; + } + } + exp >>= 1; + p_base *= p_base; + } + return ret; +} + +/// Get next token for the VimL expression input +/// +/// @param pstate Parser state. +/// @param[in] flags Flags, @see LexExprFlags. +/// +/// @return Next token. +LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL +{ + LexExprToken ret = { + .type = kExprLexInvalid, + .start = pstate->pos, + }; + ParserLine pline; + if (!viml_parser_get_remaining_line(pstate, &pline)) { + ret.type = kExprLexEOC; + return ret; + } + if (pline.size <= 0) { + ret.len = 0; + ret.type = kExprLexEOC; + goto viml_pexpr_next_token_adv_return; + } + ret.len = 1; + const uint8_t schar = (uint8_t)pline.data[0]; +#define GET_CCS(ret, pline) \ + do { \ + if (ret.len < pline.size \ + && strchr("?#", pline.data[ret.len]) != NULL) { \ + ret.data.cmp.ccs = \ + (ExprCaseCompareStrategy)pline.data[ret.len]; \ + ret.len++; \ + } else { \ + ret.data.cmp.ccs = kCCStrategyUseOption; \ + } \ + } while (0) + switch (schar) { + // Paired brackets. +#define BRACKET(typ, opning, clsing) \ + case opning: \ + case clsing: { \ + ret.type = typ; \ + ret.data.brc.closing = (schar == clsing); \ + break; \ + } + BRACKET(kExprLexParenthesis, '(', ')') + BRACKET(kExprLexBracket, '[', ']') + BRACKET(kExprLexFigureBrace, '{', '}') +#undef BRACKET + + // Single character tokens without data. +#define CHAR(typ, ch) \ + case ch: { \ + ret.type = typ; \ + break; \ + } + CHAR(kExprLexQuestion, '?') + CHAR(kExprLexColon, ':') + CHAR(kExprLexDot, '.') + CHAR(kExprLexPlus, '+') + CHAR(kExprLexComma, ',') +#undef CHAR + + // Multiplication/division/modulo. +#define MUL(mul_type, ch) \ + case ch: { \ + ret.type = kExprLexMultiplication; \ + ret.data.mul.type = mul_type; \ + break; \ + } + MUL(kExprLexMulMul, '*') + MUL(kExprLexMulDiv, '/') + MUL(kExprLexMulMod, '%') +#undef MUL + +#define CHARREG(typ, cond) \ + do { \ + ret.type = typ; \ + for (; (ret.len < pline.size \ + && cond(pline.data[ret.len])) \ + ; ret.len++) { \ + } \ + } while (0) + + // Whitespace. + case ' ': + case TAB: { + CHARREG(kExprLexSpacing, ascii_iswhite); + break; + } + + // Control character, except for NUL, NL and TAB. + case Ctrl_A: case Ctrl_B: case Ctrl_C: case Ctrl_D: case Ctrl_E: + case Ctrl_F: case Ctrl_G: case Ctrl_H: + + case Ctrl_K: case Ctrl_L: case Ctrl_M: case Ctrl_N: case Ctrl_O: + case Ctrl_P: case Ctrl_Q: case Ctrl_R: case Ctrl_S: case Ctrl_T: + case Ctrl_U: case Ctrl_V: case Ctrl_W: case Ctrl_X: case Ctrl_Y: + case Ctrl_Z: { +#define ISCTRL(schar) (schar < ' ') + CHARREG(kExprLexInvalid, ISCTRL); + ret.data.err.type = kExprLexSpacing; + ret.data.err.msg = + _("E15: Invalid control character present in input: %.*s"); + break; +#undef ISCTRL + } + + // Number. + case '0': case '1': case '2': case '3': case '4': case '5': case '6': + case '7': case '8': case '9': { + ret.data.num.is_float = false; + ret.data.num.base = 10; + size_t frac_start = 0; + size_t exp_start = 0; + size_t frac_end = 0; + bool exp_negative = false; + CHARREG(kExprLexNumber, ascii_isdigit); + if (flags & kELFlagAllowFloat) { + const LexExprToken non_float_ret = ret; + if (pline.size > ret.len + 1 + && pline.data[ret.len] == '.' + && ascii_isdigit(pline.data[ret.len + 1])) { + ret.len++; + frac_start = ret.len; + frac_end = ret.len; + ret.data.num.is_float = true; + for (; ret.len < pline.size && ascii_isdigit(pline.data[ret.len]) + ; ret.len++) { + // A small optimization: trailing zeroes in fractional part do not + // add anything to significand, so it is useless to include them in + // frac_end. + if (pline.data[ret.len] != '0') { + frac_end = ret.len + 1; + } + } + if (pline.size > ret.len + 1 + && (pline.data[ret.len] == 'e' + || pline.data[ret.len] == 'E') + && ((pline.size > ret.len + 2 + && (pline.data[ret.len + 1] == '+' + || pline.data[ret.len + 1] == '-') + && ascii_isdigit(pline.data[ret.len + 2])) + || ascii_isdigit(pline.data[ret.len + 1]))) { + ret.len++; + if (pline.data[ret.len] == '+' + || (exp_negative = (pline.data[ret.len] == '-'))) { + ret.len++; + } + exp_start = ret.len; + CHARREG(kExprLexNumber, ascii_isdigit); + } + } + if (pline.size > ret.len + && (pline.data[ret.len] == '.' + || ASCII_ISALPHA(pline.data[ret.len]))) { + ret = non_float_ret; + } + } + // TODO(ZyX-I): detect overflows + if (ret.data.num.is_float) { + // Vim used to use string2float here which in turn uses strtod(). There + // are two problems with this approach: + // 1. strtod() is locale-dependent. Not sure how it is worked around so + // that I do not see relevant bugs, but it still does not look like + // a good idea. + // 2. strtod() does not accept length argument. + // + // The below variant of parsing floats was recognized as acceptable + // because it is basically how uClibc does the thing: it generates + // a number ignoring decimal point (but recording its position), then + // uses recorded position to scale number down when processing exponent. + float_T significand_part = 0; + uvarnumber_T exp_part = 0; + const size_t frac_size = (size_t)(frac_end - frac_start); + for (size_t i = 0; i < frac_end; i++) { + if (i == frac_start - 1) { + continue; + } + significand_part = significand_part * 10 + (pline.data[i] - '0'); + } + if (exp_start) { + vim_str2nr(pline.data + exp_start, NULL, NULL, 0, NULL, &exp_part, + (int)(ret.len - exp_start)); + } + if (exp_negative) { + exp_part += frac_size; + } else { + if (exp_part < frac_size) { + exp_negative = true; + exp_part = frac_size - exp_part; + } else { + exp_part -= frac_size; + } + } + ret.data.num.val.floating = scale_number(significand_part, 10, exp_part, + exp_negative); + } else { + int len; + int prep; + vim_str2nr(pline.data, &prep, &len, STR2NR_ALL, NULL, + &ret.data.num.val.integer, (int)pline.size); + ret.len = (size_t)len; + const uint8_t bases[] = { + [0] = 10, + ['0'] = 8, + ['x'] = 16, ['X'] = 16, + ['b'] = 2, ['B'] = 2, + }; + ret.data.num.base = bases[prep]; + } + break; + } + + // Environment variable. + case '$': { + // FIXME: Parser function can’t be thread-safe with vim_isIDc. + CHARREG(kExprLexEnv, vim_isIDc); + break; + } + + // Normal variable/function name. + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': + case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': + case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': + case 'v': case 'w': case 'x': case 'y': case 'z': + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': + case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': + case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': + case 'V': case 'W': case 'X': case 'Y': case 'Z': + case '_': { +#define ISWORD_OR_AUTOLOAD(x) \ + (ASCII_ISALNUM(x) || (x) == AUTOLOAD_CHAR || (x) == '_') +#define ISWORD(x) \ + (ASCII_ISALNUM(x) || (x) == '_') + ret.data.var.scope = 0; + ret.data.var.autoload = false; + CHARREG(kExprLexPlainIdentifier, ISWORD); + // "is" and "isnot" operators. + if (!(flags & kELFlagIsNotCmp) + && ((ret.len == 2 && memcmp(pline.data, "is", 2) == 0) + || (ret.len == 5 && memcmp(pline.data, "isnot", 5) == 0))) { + ret.type = kExprLexComparison; + ret.data.cmp.type = kExprCmpIdentical; + ret.data.cmp.inv = (ret.len == 5); + GET_CCS(ret, pline); + // Scope: `s:`, etc. + } else if (ret.len == 1 + && pline.size > 1 + && memchr(EXPR_VAR_SCOPE_LIST, schar, + sizeof(EXPR_VAR_SCOPE_LIST)) != NULL + && pline.data[ret.len] == ':' + && !(flags & kELFlagForbidScope)) { + ret.len++; + ret.data.var.scope = (ExprVarScope)schar; + CHARREG(kExprLexPlainIdentifier, ISWORD_OR_AUTOLOAD); + ret.data.var.autoload = ( + memchr(pline.data + 2, AUTOLOAD_CHAR, ret.len - 2) + != NULL); + // Previous CHARREG stopped at autoload character in order to make it + // possible to detect `is#`. Continue now with autoload characters + // included. + // + // Warning: there is ambiguity for the lexer: `is#Foo(1)` is a call of + // function `is#Foo()`, `1is#Foo(1)` is a comparison `1 is# Foo(1)`. This + // needs to be resolved on the higher level where context is available. + } else if (pline.size > ret.len + && pline.data[ret.len] == AUTOLOAD_CHAR) { + ret.data.var.autoload = true; + CHARREG(kExprLexPlainIdentifier, ISWORD_OR_AUTOLOAD); + } + break; +#undef ISWORD_OR_AUTOLOAD +#undef ISWORD + } +#undef CHARREG + + // Option. + case '&': { +#define OPTNAMEMISS(ret) \ + do { \ + ret.type = kExprLexInvalid; \ + ret.data.err.type = kExprLexOption; \ + ret.data.err.msg = _("E112: Option name missing: %.*s"); \ + } while (0) + if (pline.size > 1 && pline.data[1] == '&') { + ret.type = kExprLexAnd; + ret.len++; + break; + } + if (pline.size == 1 || !ASCII_ISALPHA(pline.data[1])) { + OPTNAMEMISS(ret); + break; + } + ret.type = kExprLexOption; + if (pline.size > 2 + && pline.data[2] == ':' + && memchr(EXPR_OPT_SCOPE_LIST, pline.data[1], + sizeof(EXPR_OPT_SCOPE_LIST)) != NULL) { + ret.len += 2; + ret.data.opt.scope = (ExprOptScope)pline.data[1]; + ret.data.opt.name = pline.data + 3; + } else { + ret.data.opt.scope = kExprOptScopeUnspecified; + ret.data.opt.name = pline.data + 1; + } + const char *p = ret.data.opt.name; + const char *const e = pline.data + pline.size; + if (e - p >= 4 && p[0] == 't' && p[1] == '_') { + ret.data.opt.len = 4; + ret.len += 4; + } else { + for (; p < e && ASCII_ISALPHA(*p); p++) { + } + ret.data.opt.len = (size_t)(p - ret.data.opt.name); + if (ret.data.opt.len == 0) { + OPTNAMEMISS(ret); + } else { + ret.len += ret.data.opt.len; + } + } + break; +#undef OPTNAMEMISS + } + + // Register. + case '@': { + ret.type = kExprLexRegister; + if (pline.size > 1) { + ret.len++; + ret.data.reg.name = (uint8_t)pline.data[1]; + } else { + ret.data.reg.name = -1; + } + break; + } + + // Single quoted string. + case '\'': { + ret.type = kExprLexSingleQuotedString; + ret.data.str.closed = false; + for (; ret.len < pline.size && !ret.data.str.closed; ret.len++) { + if (pline.data[ret.len] == '\'') { + if (ret.len + 1 < pline.size && pline.data[ret.len + 1] == '\'') { + ret.len++; + } else { + ret.data.str.closed = true; + } + } + } + break; + } + + // Double quoted string. + case '"': { + ret.type = kExprLexDoubleQuotedString; + ret.data.str.closed = false; + for (; ret.len < pline.size && !ret.data.str.closed; ret.len++) { + if (pline.data[ret.len] == '\\') { + if (ret.len + 1 < pline.size) { + ret.len++; + } + } else if (pline.data[ret.len] == '"') { + ret.data.str.closed = true; + } + } + break; + } + + // Unary not, (un)equality and regex (not) match comparison operators. + case '!': + case '=': { + if (pline.size == 1) { +viml_pexpr_next_token_invalid_comparison: + ret.type = (schar == '!' ? kExprLexNot : kExprLexInvalid); + if (ret.type == kExprLexInvalid) { + ret.data.err.msg = _("E15: Expected == or =~: %.*s"); + ret.data.err.type = kExprLexComparison; + } + break; + } + ret.type = kExprLexComparison; + ret.data.cmp.inv = (schar == '!'); + if (pline.data[1] == '=') { + ret.data.cmp.type = kExprCmpEqual; + ret.len++; + } else if (pline.data[1] == '~') { + ret.data.cmp.type = kExprCmpMatches; + ret.len++; + } else { + goto viml_pexpr_next_token_invalid_comparison; + } + GET_CCS(ret, pline); + break; + } + + // Less/greater [or equal to] comparison operators. + case '>': + case '<': { + ret.type = kExprLexComparison; + const bool haseqsign = (pline.size > 1 && pline.data[1] == '='); + if (haseqsign) { + ret.len++; + } + GET_CCS(ret, pline); + ret.data.cmp.inv = (schar == '<'); + ret.data.cmp.type = ((ret.data.cmp.inv ^ haseqsign) + ? kExprCmpGreaterOrEqual + : kExprCmpGreater); + break; + } + + // Minus sign or arrow from lambdas. + case '-': { + if (pline.size > 1 && pline.data[1] == '>') { + ret.len++; + ret.type = kExprLexArrow; + } else { + ret.type = kExprLexMinus; + } + break; + } + + // Expression end because Ex command ended. + case NUL: + case NL: { + if (flags & kELFlagForbidEOC) { + ret.type = kExprLexInvalid; + ret.data.err.msg = _("E15: Unexpected EOC character: %.*s"); + ret.data.err.type = kExprLexSpacing; + } else { + ret.type = kExprLexEOC; + } + break; + } + + case '|': { + if (pline.size >= 2 && pline.data[ret.len] == '|') { + // "||" is or. + ret.len++; + ret.type = kExprLexOr; + } else if (flags & kELFlagForbidEOC) { + // Note: `<C-r>=1 | 2<CR>` actually yields 1 in Vim without any + // errors. This will be changed here. + ret.type = kExprLexInvalid; + ret.data.err.msg = _("E15: Unexpected EOC character: %.*s"); + ret.data.err.type = kExprLexOr; + } else { + ret.type = kExprLexEOC; + } + break; + } + + // Everything else is not valid. + default: { + ret.len = (size_t)utfc_ptr2len_len((const char_u *)pline.data, + (int)pline.size); + ret.type = kExprLexInvalid; + ret.data.err.type = kExprLexPlainIdentifier; + ret.data.err.msg = _("E15: Unidentified character: %.*s"); + break; + } + } +#undef GET_CCS +viml_pexpr_next_token_adv_return: + if (!(flags & kELFlagPeek)) { + viml_parser_advance(pstate, ret.len); + } + return ret; +} + +static const char *const eltkn_type_tab[] = { + [kExprLexInvalid] = "Invalid", + [kExprLexMissing] = "Missing", + [kExprLexSpacing] = "Spacing", + [kExprLexEOC] = "EOC", + + [kExprLexQuestion] = "Question", + [kExprLexColon] = "Colon", + [kExprLexOr] = "Or", + [kExprLexAnd] = "And", + [kExprLexComparison] = "Comparison", + [kExprLexPlus] = "Plus", + [kExprLexMinus] = "Minus", + [kExprLexDot] = "Dot", + [kExprLexMultiplication] = "Multiplication", + + [kExprLexNot] = "Not", + + [kExprLexNumber] = "Number", + [kExprLexSingleQuotedString] = "SingleQuotedString", + [kExprLexDoubleQuotedString] = "DoubleQuotedString", + [kExprLexOption] = "Option", + [kExprLexRegister] = "Register", + [kExprLexEnv] = "Env", + [kExprLexPlainIdentifier] = "PlainIdentifier", + + [kExprLexBracket] = "Bracket", + [kExprLexFigureBrace] = "FigureBrace", + [kExprLexParenthesis] = "Parenthesis", + [kExprLexComma] = "Comma", + [kExprLexArrow] = "Arrow", +}; + +static const char *const eltkn_cmp_type_tab[] = { + [kExprCmpEqual] = "Equal", + [kExprCmpMatches] = "Matches", + [kExprCmpGreater] = "Greater", + [kExprCmpGreaterOrEqual] = "GreaterOrEqual", + [kExprCmpIdentical] = "Identical", +}; + +static const char *const ccs_tab[] = { + [kCCStrategyUseOption] = "UseOption", + [kCCStrategyMatchCase] = "MatchCase", + [kCCStrategyIgnoreCase] = "IgnoreCase", +}; + +static const char *const eltkn_mul_type_tab[] = { + [kExprLexMulMul] = "Mul", + [kExprLexMulDiv] = "Div", + [kExprLexMulMod] = "Mod", +}; + +static const char *const eltkn_opt_scope_tab[] = { + [kExprOptScopeUnspecified] = "Unspecified", + [kExprOptScopeGlobal] = "Global", + [kExprOptScopeLocal] = "Local", +}; + +/// Represent token as a string +/// +/// Intended for testing and debugging purposes. +/// +/// @param[in] pstate Parser state, needed to get token string from it. May be +/// NULL, in which case in place of obtaining part of the +/// string represented by token only token length is +/// returned. +/// @param[in] token Token to represent. +/// @param[out] ret_size Return string size, for cases like NULs inside +/// a string. May be NULL. +/// +/// @return Token represented in a string form, in a static buffer (overwritten +/// on each call). +const char *viml_pexpr_repr_token(const ParserState *const pstate, + const LexExprToken token, + size_t *const ret_size) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + static char ret[1024]; + char *p = ret; + const char *const e = &ret[1024] - 1; +#define ADDSTR(...) \ + do { \ + p += snprintf(p, (size_t)(sizeof(ret) - (size_t)(p - ret)), __VA_ARGS__); \ + if (p >= e) { \ + goto viml_pexpr_repr_token_end; \ + } \ + } while (0) + ADDSTR("%zu:%zu:%s", token.start.line, token.start.col, + eltkn_type_tab[token.type]); + switch (token.type) { +#define TKNARGS(tkn_type, ...) \ + case tkn_type: { \ + ADDSTR(__VA_ARGS__); \ + break; \ + } + TKNARGS(kExprLexComparison, "(type=%s,ccs=%s,inv=%i)", + eltkn_cmp_type_tab[token.data.cmp.type], + ccs_tab[token.data.cmp.ccs], + (int)token.data.cmp.inv) + TKNARGS(kExprLexMultiplication, "(type=%s)", + eltkn_mul_type_tab[token.data.mul.type]) + TKNARGS(kExprLexRegister, "(name=%s)", intchar2str(token.data.reg.name)) + case kExprLexDoubleQuotedString: + TKNARGS(kExprLexSingleQuotedString, "(closed=%i)", + (int)token.data.str.closed) + TKNARGS(kExprLexOption, "(scope=%s,name=%.*s)", + eltkn_opt_scope_tab[token.data.opt.scope], + (int)token.data.opt.len, token.data.opt.name) + TKNARGS(kExprLexPlainIdentifier, "(scope=%s,autoload=%i)", + intchar2str(token.data.var.scope), (int)token.data.var.autoload) + TKNARGS(kExprLexNumber, "(is_float=%i,base=%i,val=%lg)", + (int)token.data.num.is_float, + (int)token.data.num.base, + (double)(token.data.num.is_float + ? (double)token.data.num.val.floating + : (double)token.data.num.val.integer)) + TKNARGS(kExprLexInvalid, "(msg=%s)", token.data.err.msg) + default: { + // No additional arguments. + break; + } +#undef TKNARGS + } + if (pstate == NULL) { + ADDSTR("::%zu", token.len); + } else { + *p++ = ':'; + memmove( + p, &pstate->reader.lines.items[token.start.line].data[token.start.col], + token.len); + p += token.len; + *p = NUL; + } +#undef ADDSTR +viml_pexpr_repr_token_end: + if (ret_size != NULL) { + *ret_size = (size_t)(p - ret); + } + return ret; +} + +#ifdef UNIT_TESTING +static const char *const east_node_type_tab[] = { + [kExprNodeMissing] = "Missing", + [kExprNodeOpMissing] = "OpMissing", + [kExprNodeTernary] = "Ternary", + [kExprNodeTernaryValue] = "TernaryValue", + [kExprNodeRegister] = "Register", + [kExprNodeSubscript] = "Subscript", + [kExprNodeListLiteral] = "ListLiteral", + [kExprNodeUnaryPlus] = "UnaryPlus", + [kExprNodeBinaryPlus] = "BinaryPlus", + [kExprNodeNested] = "Nested", + [kExprNodeCall] = "Call", + [kExprNodePlainIdentifier] = "PlainIdentifier", + [kExprNodePlainKey] = "PlainKey", + [kExprNodeComplexIdentifier] = "ComplexIdentifier", + [kExprNodeUnknownFigure] = "UnknownFigure", + [kExprNodeLambda] = "Lambda", + [kExprNodeDictLiteral] = "DictLiteral", + [kExprNodeCurlyBracesIdentifier] = "CurlyBracesIdentifier", + [kExprNodeComma] = "Comma", + [kExprNodeColon] = "Colon", + [kExprNodeArrow] = "Arrow", + [kExprNodeComparison] = "Comparison", + [kExprNodeConcat] = "Concat", + [kExprNodeConcatOrSubscript] = "ConcatOrSubscript", + [kExprNodeInteger] = "Integer", + [kExprNodeFloat] = "Float", + [kExprNodeSingleQuotedString] = "SingleQuotedString", + [kExprNodeDoubleQuotedString] = "DoubleQuotedString", + [kExprNodeOr] = "Or", + [kExprNodeAnd] = "And", + [kExprNodeUnaryMinus] = "UnaryMinus", + [kExprNodeBinaryMinus] = "BinaryMinus", + [kExprNodeNot] = "Not", + [kExprNodeMultiplication] = "Multiplication", + [kExprNodeDivision] = "Division", + [kExprNodeMod] = "Mod", + [kExprNodeOption] = "Option", + [kExprNodeEnvironment] = "Environment", +}; +#endif + +/// Represent `int` character as a string +/// +/// Converts +/// - ASCII digits into '{digit}' +/// - ASCII printable characters into a single-character strings +/// - everything else to numbers. +/// +/// @param[in] ch Character to convert. +/// +/// @return Converted string, stored in a static buffer (overriden after each +/// call). +static const char *intchar2str(const int ch) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + static char buf[sizeof(int) * 3 + 1]; + if (' ' <= ch && ch < 0x7f) { + if (ascii_isdigit(ch)) { + buf[0] = '\''; + buf[1] = (char)ch; + buf[2] = '\''; + buf[3] = NUL; + } else { + buf[0] = (char)ch; + buf[1] = NUL; + } + } else { + snprintf(buf, sizeof(buf), "%i", ch); + } + return buf; +} + +#ifdef UNIT_TESTING +#include <stdio.h> + +REAL_FATTR_UNUSED +static inline void viml_pexpr_debug_print_ast_node( + const ExprASTNode *const *const eastnode_p, + const char *const prefix) +{ + if (*eastnode_p == NULL) { + fprintf(stderr, "%s %p : NULL\n", prefix, (void *)eastnode_p); + } else { + fprintf(stderr, "%s %p : %p : %s : %zu:%zu:%zu\n", + prefix, (void *)eastnode_p, (void *)(*eastnode_p), + east_node_type_tab[(*eastnode_p)->type], (*eastnode_p)->start.line, + (*eastnode_p)->start.col, (*eastnode_p)->len); + } +} + +REAL_FATTR_UNUSED +static inline void viml_pexpr_debug_print_ast_stack( + const ExprASTStack *const ast_stack, + const char *const msg) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE +{ + fprintf(stderr, "\n%sstack: %zu:\n", msg, kv_size(*ast_stack)); + for (size_t i = 0; i < kv_size(*ast_stack); i++) { + viml_pexpr_debug_print_ast_node( + (const ExprASTNode *const *)kv_A(*ast_stack, i), + "-"); + } +} + +REAL_FATTR_UNUSED +static inline void viml_pexpr_debug_print_token( + const ParserState *const pstate, const LexExprToken token) + FUNC_ATTR_ALWAYS_INLINE +{ + fprintf(stderr, "\ntkn: %s\n", viml_pexpr_repr_token(pstate, token, NULL)); +} +#define PSTACK(msg) \ + viml_pexpr_debug_print_ast_stack(&ast_stack, #msg) +#define PSTACK_P(msg) \ + viml_pexpr_debug_print_ast_stack(ast_stack, #msg) +#define PNODE_P(eastnode_p, msg) \ + viml_pexpr_debug_print_ast_node((const ExprASTNode *const *)eastnode_p, \ + (#msg)) +#define PTOKEN(tkn) \ + viml_pexpr_debug_print_token(pstate, tkn) +#endif + +const uint8_t node_maxchildren[] = { + [kExprNodeMissing] = 0, + [kExprNodeOpMissing] = 2, + [kExprNodeTernary] = 2, + [kExprNodeTernaryValue] = 2, + [kExprNodeRegister] = 0, + [kExprNodeSubscript] = 2, + [kExprNodeListLiteral] = 1, + [kExprNodeUnaryPlus] = 1, + [kExprNodeBinaryPlus] = 2, + [kExprNodeNested] = 1, + [kExprNodeCall] = 2, + [kExprNodePlainIdentifier] = 0, + [kExprNodePlainKey] = 0, + [kExprNodeComplexIdentifier] = 2, + [kExprNodeUnknownFigure] = 1, + [kExprNodeLambda] = 2, + [kExprNodeDictLiteral] = 1, + [kExprNodeCurlyBracesIdentifier] = 1, + [kExprNodeComma] = 2, + [kExprNodeColon] = 2, + [kExprNodeArrow] = 2, + [kExprNodeComparison] = 2, + [kExprNodeConcat] = 2, + [kExprNodeConcatOrSubscript] = 2, + [kExprNodeInteger] = 0, + [kExprNodeFloat] = 0, + [kExprNodeSingleQuotedString] = 0, + [kExprNodeDoubleQuotedString] = 0, + [kExprNodeOr] = 2, + [kExprNodeAnd] = 2, + [kExprNodeUnaryMinus] = 1, + [kExprNodeBinaryMinus] = 2, + [kExprNodeNot] = 1, + [kExprNodeMultiplication] = 2, + [kExprNodeDivision] = 2, + [kExprNodeMod] = 2, + [kExprNodeOption] = 0, + [kExprNodeEnvironment] = 0, +}; + +/// Free memory occupied by AST +/// +/// @param ast AST stack to free. +void viml_pexpr_free_ast(ExprAST ast) +{ + ExprASTStack ast_stack; + kvi_init(ast_stack); + kvi_push(ast_stack, &ast.root); + while (kv_size(ast_stack)) { + ExprASTNode **const cur_node = kv_last(ast_stack); +#ifndef NDEBUG + // Explicitly check for AST recursiveness. + for (size_t i = 0 ; i < kv_size(ast_stack) - 1 ; i++) { + assert(*kv_A(ast_stack, i) != *cur_node); + } +#endif + if (*cur_node == NULL) { + assert(kv_size(ast_stack) == 1); + kv_drop(ast_stack, 1); + } else if ((*cur_node)->children != NULL) { +#ifndef NDEBUG + const uint8_t maxchildren = node_maxchildren[(*cur_node)->type]; + assert(maxchildren > 0); + assert(maxchildren <= 2); + assert(maxchildren == 1 + ? (*cur_node)->children->next == NULL + : ((*cur_node)->children->next == NULL + || (*cur_node)->children->next->next == NULL)); +#endif + kvi_push(ast_stack, &(*cur_node)->children); + } else if ((*cur_node)->next != NULL) { + kvi_push(ast_stack, &(*cur_node)->next); + } else if (*cur_node != NULL) { + kv_drop(ast_stack, 1); + switch ((*cur_node)->type) { + case kExprNodeDoubleQuotedString: + case kExprNodeSingleQuotedString: { + xfree((*cur_node)->data.str.value); + break; + } + case kExprNodeMissing: + case kExprNodeOpMissing: + case kExprNodeTernary: + case kExprNodeTernaryValue: + case kExprNodeRegister: + case kExprNodeSubscript: + case kExprNodeListLiteral: + case kExprNodeUnaryPlus: + case kExprNodeBinaryPlus: + case kExprNodeNested: + case kExprNodeCall: + case kExprNodePlainIdentifier: + case kExprNodePlainKey: + case kExprNodeComplexIdentifier: + case kExprNodeUnknownFigure: + case kExprNodeLambda: + case kExprNodeDictLiteral: + case kExprNodeCurlyBracesIdentifier: + case kExprNodeComma: + case kExprNodeColon: + case kExprNodeArrow: + case kExprNodeComparison: + case kExprNodeConcat: + case kExprNodeConcatOrSubscript: + case kExprNodeInteger: + case kExprNodeFloat: + case kExprNodeOr: + case kExprNodeAnd: + case kExprNodeUnaryMinus: + case kExprNodeBinaryMinus: + case kExprNodeNot: + case kExprNodeMultiplication: + case kExprNodeDivision: + case kExprNodeMod: + case kExprNodeOption: + case kExprNodeEnvironment: { + break; + } + } + xfree(*cur_node); + *cur_node = NULL; + } + } + kvi_destroy(ast_stack); +} + +// Binary operator precedence and associativity: +// +// Operator | Precedence | Associativity +// ---------+------------+----------------- +// || | 2 | left +// && | 3 | left +// cmp* | 4 | not associative +// + - . | 5 | left +// * / % | 6 | left +// +// * comparison operators: +// +// == ==# ==? != !=# !=? +// =~ =~# =~? !~ !~# !~? +// > ># >? <= <=# <=? +// < <# <? >= >=# >=? +// is is# is? isnot isnot# isnot? + +/// Allocate a new node and set some of the values +/// +/// @param[in] type Node type to allocate. +/// @param[in] level Node level to allocate +static inline ExprASTNode *viml_pexpr_new_node(const ExprASTNodeType type) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_MALLOC +{ + ExprASTNode *ret = xmalloc(sizeof(*ret)); + ret->type = type; + ret->children = NULL; + ret->next = NULL; + return ret; +} + +static struct { + ExprOpLvl lvl; + ExprOpAssociativity ass; +} node_type_to_node_props[] = { + [kExprNodeMissing] = { kEOpLvlInvalid, kEOpAssNo, }, + [kExprNodeOpMissing] = { kEOpLvlMultiplication, kEOpAssNo }, + + [kExprNodeNested] = { kEOpLvlParens, kEOpAssNo }, + // Note: below nodes are kEOpLvlSubscript for “binary operator” itself, but + // kEOpLvlParens when it comes to inside the parenthesis. + [kExprNodeCall] = { kEOpLvlParens, kEOpAssNo }, + [kExprNodeSubscript] = { kEOpLvlParens, kEOpAssNo }, + + [kExprNodeUnknownFigure] = { kEOpLvlParens, kEOpAssLeft }, + [kExprNodeLambda] = { kEOpLvlParens, kEOpAssNo }, + [kExprNodeDictLiteral] = { kEOpLvlParens, kEOpAssNo }, + [kExprNodeListLiteral] = { kEOpLvlParens, kEOpAssNo }, + + [kExprNodeArrow] = { kEOpLvlArrow, kEOpAssNo }, + + // Right associativity for comma because this means easier access to arguments + // list, etc: for "[a, b, c, d]" you can access "a" in one step if it is + // represented as "list(comma(a, comma(b, comma(c, d))))" then if it is + // "list(comma(comma(comma(a, b), c), d))" in which case you will need to + // traverse all three comma() structures. And with comma operator (including + // actual comma operator from C which is not present in VimL) nobody cares + // about associativity, only about order of execution. + [kExprNodeComma] = { kEOpLvlComma, kEOpAssRight }, + + // Colons are not eligible for chaining, so nobody cares about associativity. + [kExprNodeColon] = { kEOpLvlColon, kEOpAssNo }, + + [kExprNodeTernary] = { kEOpLvlTernary, kEOpAssRight }, + + [kExprNodeOr] = { kEOpLvlOr, kEOpAssLeft }, + + [kExprNodeAnd] = { kEOpLvlAnd, kEOpAssLeft }, + + [kExprNodeTernaryValue] = { kEOpLvlTernaryValue, kEOpAssRight }, + + [kExprNodeComparison] = { kEOpLvlComparison, kEOpAssRight }, + + [kExprNodeBinaryPlus] = { kEOpLvlAddition, kEOpAssLeft }, + [kExprNodeBinaryMinus] = { kEOpLvlAddition, kEOpAssLeft }, + [kExprNodeConcat] = { kEOpLvlAddition, kEOpAssLeft }, + + [kExprNodeMultiplication] = { kEOpLvlMultiplication, kEOpAssLeft }, + [kExprNodeDivision] = { kEOpLvlMultiplication, kEOpAssLeft }, + [kExprNodeMod] = { kEOpLvlMultiplication, kEOpAssLeft }, + + [kExprNodeUnaryPlus] = { kEOpLvlUnary, kEOpAssNo }, + [kExprNodeUnaryMinus] = { kEOpLvlUnary, kEOpAssNo }, + [kExprNodeNot] = { kEOpLvlUnary, kEOpAssNo }, + + [kExprNodeConcatOrSubscript] = { kEOpLvlSubscript, kEOpAssLeft }, + + [kExprNodeCurlyBracesIdentifier] = { kEOpLvlComplexIdentifier, kEOpAssLeft }, + + [kExprNodeComplexIdentifier] = { kEOpLvlValue, kEOpAssLeft }, + + [kExprNodePlainIdentifier] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodePlainKey] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeRegister] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeInteger] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeFloat] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeDoubleQuotedString] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeSingleQuotedString] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeOption] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeEnvironment] = { kEOpLvlValue, kEOpAssNo }, +}; + +/// Get AST node priority level +/// +/// Used primary to reduce line length, so keep the name short. +/// +/// @param[in] node Node to get priority for. +/// +/// @return Node priority level. +static inline ExprOpLvl node_lvl(const ExprASTNode node) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + return node_type_to_node_props[node.type].lvl; +} + +/// Get AST node associativity, to be used for operator nodes primary +/// +/// Used primary to reduce line length, so keep the name short. +/// +/// @param[in] node Node to get priority for. +/// +/// @return Node associativity. +static inline ExprOpAssociativity node_ass(const ExprASTNode node) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + return node_type_to_node_props[node.type].ass; +} + +/// Handle binary operator +/// +/// This function is responsible for handling priority levels as well. +/// +/// @param[in] pstate Parser state, used for error reporting. +/// @param ast_stack AST stack. May be popped of some values and will +/// definitely receive new ones. +/// @param bop_node New node to handle. +/// @param[out] want_node_p New value of want_node. +/// @param[out] ast_err Location where error is saved, if any. +/// +/// @return True if no errors occurred, false otherwise. +static bool viml_pexpr_handle_bop(const ParserState *const pstate, + ExprASTStack *const ast_stack, + ExprASTNode *const bop_node, + ExprASTWantedNode *const want_node_p, + ExprASTError *const ast_err) + FUNC_ATTR_NONNULL_ALL +{ + bool ret = true; + ExprASTNode **top_node_p = NULL; + ExprASTNode *top_node; + ExprOpLvl top_node_lvl; + ExprOpAssociativity top_node_ass; + assert(kv_size(*ast_stack)); + const ExprOpLvl bop_node_lvl = ((bop_node->type == kExprNodeCall + || bop_node->type == kExprNodeSubscript) + ? kEOpLvlSubscript + : node_lvl(*bop_node)); +#ifndef NDEBUG + const ExprOpAssociativity bop_node_ass = ( + (bop_node->type == kExprNodeCall + || bop_node->type == kExprNodeSubscript) + ? kEOpAssLeft + : node_ass(*bop_node)); +#endif + do { + ExprASTNode **new_top_node_p = kv_last(*ast_stack); + ExprASTNode *new_top_node = *new_top_node_p; + assert(new_top_node != NULL); + const ExprOpLvl new_top_node_lvl = node_lvl(*new_top_node); + const ExprOpAssociativity new_top_node_ass = node_ass(*new_top_node); + assert(bop_node_lvl != new_top_node_lvl + || bop_node_ass == new_top_node_ass); + if (top_node_p != NULL + && ((bop_node_lvl > new_top_node_lvl + || (bop_node_lvl == new_top_node_lvl + && new_top_node_ass == kEOpAssNo)))) { + break; + } + kv_drop(*ast_stack, 1); + top_node_p = new_top_node_p; + top_node = new_top_node; + top_node_lvl = new_top_node_lvl; + top_node_ass = new_top_node_ass; + if (bop_node_lvl == top_node_lvl && top_node_ass == kEOpAssRight) { + break; + } + } while (kv_size(*ast_stack)); + if (top_node_ass == kEOpAssLeft || top_node_lvl != bop_node_lvl) { + // outer(op(x,y)) -> outer(new_op(op(x,y),*)) + // + // Before: top_node_p = outer(*), points to op(x,y) + // Other stack elements unknown + // + // After: top_node_p = outer(*), points to new_op(op(x,y)) + // &bop_node->children->next = new_op(op(x,y),*), points to NULL + *top_node_p = bop_node; + bop_node->children = top_node; + assert(bop_node->children->next == NULL); + kvi_push(*ast_stack, top_node_p); + kvi_push(*ast_stack, &bop_node->children->next); + } else { + assert(top_node_lvl == bop_node_lvl && top_node_ass == kEOpAssRight); + assert(top_node->children != NULL && top_node->children->next != NULL); + // outer(op(x,y)) -> outer(op(x,new_op(y,*))) + // + // Before: top_node_p = outer(*), points to op(x,y) + // Other stack elements unknown + // + // After: top_node_p = outer(*), points to op(x,new_op(y)) + // &top_node->children->next = op(x,*), points to new_op(y) + // &bop_node->children->next = new_op(y,*), points to NULL + bop_node->children = top_node->children->next; + top_node->children->next = bop_node; + assert(bop_node->children->next == NULL); + kvi_push(*ast_stack, top_node_p); + kvi_push(*ast_stack, &top_node->children->next); + kvi_push(*ast_stack, &bop_node->children->next); + // TODO(ZyX-I): Make this not error, but treat like Python does + if (bop_node->type == kExprNodeComparison) { + east_set_error(pstate, ast_err, + _("E15: Operator is not associative: %.*s"), + bop_node->start); + ret = false; + } + } + *want_node_p = (*want_node_p == kENodeArgumentSeparator + ? kENodeArgument + : kENodeValue); + return ret; +} + +/// ParserPosition literal based on ParserPosition pos with columns shifted +/// +/// Function does not check whether resulting position is valid. +/// +/// @param[in] pos Position to shift. +/// @param[in] shift Number of bytes to shift. +/// +/// @return Shifted position. +static inline ParserPosition shifted_pos(const ParserPosition pos, + const size_t shift) + FUNC_ATTR_CONST FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT +{ + return (ParserPosition) { .line = pos.line, .col = pos.col + shift }; +} + +/// ParserPosition literal based on ParserPosition pos with specified column +/// +/// Function does not check whether remaining position is valid. +/// +/// @param[in] pos Position to adjust. +/// @param[in] new_col New column. +/// +/// @return Shifted position. +static inline ParserPosition recol_pos(const ParserPosition pos, + const size_t new_col) + FUNC_ATTR_CONST FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT +{ + return (ParserPosition) { .line = pos.line, .col = new_col }; +} + +/// Get highlight group name +#define HL(g) (is_invalid ? "NVimInvalid" #g : "NVim" #g) + +/// Highlight current token with the given group +#define HL_CUR_TOKEN(g) \ + viml_parser_highlight(pstate, cur_token.start, cur_token.len, \ + HL(g)) + +/// Allocate new node, saving some values +#define NEW_NODE(type) \ + viml_pexpr_new_node(type) + +/// Set position of the given node to position from the given token +/// +/// @param cur_node Node to modify. +/// @param cur_token Token to set position from. +#define POS_FROM_TOKEN(cur_node, cur_token) \ + do { \ + (cur_node)->start = cur_token.start; \ + (cur_node)->len = cur_token.len; \ + } while (0) + +/// Allocate new node and set its position from the current token +/// +/// If previous token happened to contain spacing then it will be included. +/// +/// @param cur_node Variable to save allocated node to. +/// @param typ Node type. +#define NEW_NODE_WITH_CUR_POS(cur_node, typ) \ + do { \ + (cur_node) = NEW_NODE(typ); \ + POS_FROM_TOKEN((cur_node), cur_token); \ + if (prev_token.type == kExprLexSpacing) { \ + (cur_node)->start = prev_token.start; \ + (cur_node)->len += prev_token.len; \ + } \ + } while (0) + +// TODO(ZyX-I): actual condition +/// Check whether it is possible to have next expression after current +/// +/// For :echo: `:echo @a @a` is a valid expression. `:echo (@a @a)` is not. +#define MAY_HAVE_NEXT_EXPR \ + (kv_size(ast_stack) == 1) + +/// Add operator node +/// +/// @param[in] cur_node Node to add. +#define ADD_OP_NODE(cur_node) \ + is_invalid |= !viml_pexpr_handle_bop(pstate, &ast_stack, cur_node, \ + &want_node, &ast.err) + +/// Record missing operator: for things like +/// +/// :echo @a @a +/// +/// (allowed) or +/// +/// :echo (@a @a) +/// +/// (parsed as OpMissing(@a, @a)). +#define OP_MISSING \ + do { \ + if (flags & kExprFlagsMulti && MAY_HAVE_NEXT_EXPR) { \ + /* Multiple expressions allowed, return without calling */ \ + /* viml_parser_advance(). */ \ + goto viml_pexpr_parse_end; \ + } else { \ + assert(*top_node_p != NULL); \ + ERROR_FROM_TOKEN_AND_MSG(cur_token, _("E15: Missing operator: %.*s")); \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeOpMissing); \ + cur_node->len = 0; \ + ADD_OP_NODE(cur_node); \ + goto viml_pexpr_parse_process_token; \ + } \ + } while (0) + +/// Record missing value: for things like "* 5" +/// +/// @param[in] msg Error message. +#define ADD_VALUE_IF_MISSING(msg) \ + do { \ + if (want_node == kENodeValue) { \ + ERROR_FROM_TOKEN_AND_MSG(cur_token, (msg)); \ + NEW_NODE_WITH_CUR_POS((*top_node_p), kExprNodeMissing); \ + (*top_node_p)->len = 0; \ + want_node = kENodeOperator; \ + } \ + } while (0) + +/// Set AST error, unless AST already is not correct +/// +/// @param[out] ret_ast AST to set error in. +/// @param[in] pstate Parser state, used to get error message argument. +/// @param[in] msg Error message, assumed to be already translated and +/// containing a single %token "%.*s". +/// @param[in] start Position at which error occurred. +static inline void east_set_error(const ParserState *const pstate, + ExprASTError *const ret_ast_err, + const char *const msg, + const ParserPosition start) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE +{ + if (ret_ast_err->msg != NULL) { + return; + } + const ParserLine pline = pstate->reader.lines.items[start.line]; + ret_ast_err->msg = msg; + ret_ast_err->arg_len = (int)(pline.size - start.col); + ret_ast_err->arg = pline.data + start.col; +} + +/// Set error from the given token and given message +#define ERROR_FROM_TOKEN_AND_MSG(cur_token, msg) \ + do { \ + is_invalid = true; \ + east_set_error(pstate, &ast.err, msg, cur_token.start); \ + } while (0) + +/// Like #ERROR_FROM_TOKEN_AND_MSG, but gets position from a node +#define ERROR_FROM_NODE_AND_MSG(node, msg) \ + do { \ + is_invalid = true; \ + east_set_error(pstate, &ast.err, msg, node->start); \ + } while (0) + +/// Set error from the given kExprLexInvalid token +#define ERROR_FROM_TOKEN(cur_token) \ + ERROR_FROM_TOKEN_AND_MSG(cur_token, cur_token.data.err.msg) + +/// Select figure brace type, altering highlighting as well if needed +/// +/// @param[out] node Node to modify type. +/// @param[in] new_type New type, one of ExprASTNodeType values without +/// kExprNode prefix. +/// @param[in] hl Corresponding highlighting, passed as an argument to #HL. +#define SELECT_FIGURE_BRACE_TYPE(node, new_type, hl) \ + do { \ + ExprASTNode *const node_ = (node); \ + assert(node_->type == kExprNodeUnknownFigure \ + || node_->type == kExprNode##new_type); \ + node_->type = kExprNode##new_type; \ + if (pstate->colors) { \ + kv_A(*pstate->colors, node_->data.fig.opening_hl_idx).group = \ + HL(hl); \ + } \ + } while (0) + +/// Add identifier which should constitute complex identifier node +/// +/// This one is to be called only in case want_node is kENodeOperator. +/// +/// @param new_ident_node_code Code used to create a new identifier node and +/// update want_node and ast_stack, without +/// a trailing semicolon. +/// @param hl Highlighting name to use, passed as an argument to #HL. +#define ADD_IDENT(new_ident_node_code, hl) \ + do { \ + assert(want_node == kENodeOperator); \ + /* Operator: may only be curly braces name, but only under certain */ \ + /* conditions. */ \ +\ + /* First condition is that there is no space before a part of complex */ \ + /* identifier. */ \ + if (prev_token.type == kExprLexSpacing) { \ + OP_MISSING; \ + } \ + switch ((*top_node_p)->type) { \ + /* Second is that previous node is one of the identifiers: */ \ + /* complex, plain, curly braces. */ \ +\ + /* TODO(ZyX-I): Extend syntax to allow ${expr}. This is needed to */ \ + /* handle environment variables like those bash uses for */ \ + /* `export -f`: their names consist not only of alphanumeric */ \ + /* characetrs. */ \ + case kExprNodeComplexIdentifier: \ + case kExprNodePlainIdentifier: \ + case kExprNodeCurlyBracesIdentifier: { \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComplexIdentifier); \ + cur_node->len = 0; \ + cur_node->children = *top_node_p; \ + *top_node_p = cur_node; \ + kvi_push(ast_stack, &cur_node->children->next); \ + ExprASTNode **const new_top_node_p = kv_last(ast_stack); \ + assert(*new_top_node_p == NULL); \ + new_ident_node_code; \ + *new_top_node_p = cur_node; \ + HL_CUR_TOKEN(hl); \ + break; \ + } \ + default: { \ + OP_MISSING; \ + break; \ + } \ + } \ + } while (0) + +/// Structure used to define “string shifts” necessary to map string +/// highlighting to actual strings. +typedef struct { + size_t start; ///< Where special character starts in original string. + size_t orig_len; ///< Length of orininal string (e.g. 4 for "\x80"). + size_t act_len; ///< Length of resulting character(s) (e.g. 1 for "\x80"). + bool escape_not_known; ///< True if escape sequence in original is not known. +} StringShift; + +/// Parse and highlight single- or double-quoted string +/// +/// Function is supposed to detect and highlight regular expressions (but does +/// not do now). +/// +/// @param[out] pstate Parser state which also contains a place where +/// highlighting is saved. +/// @param[out] node Node where string parsing results are saved. +/// @param[in] token Token to highlight. +/// @param[in] ast_stack Parser AST stack, used to detect whether current +/// string is a regex. +/// @param[in] is_invalid Whether currently processed token is not valid. +static void parse_quoted_string(ParserState *const pstate, + ExprASTNode *const node, + const LexExprToken token, + const ExprASTStack ast_stack, + const bool is_invalid) + FUNC_ATTR_NONNULL_ALL +{ + const ParserLine pline = pstate->reader.lines.items[token.start.line]; + const char *const s = pline.data + token.start.col; + const char *const e = s + token.len - token.data.str.closed; + const char *p = s + 1; + const bool is_double = (token.type == kExprLexDoubleQuotedString); + size_t size = token.len - token.data.str.closed - 1; + kvec_withinit_t(StringShift, 16) shifts; + kvi_init(shifts); + if (!is_double) { + viml_parser_highlight(pstate, token.start, 1, HL(SingleQuote)); + while (p < e) { + const char *const chunk_e = memchr(p, '\'', (size_t)(e - p)); + if (chunk_e == NULL) { + break; + } + size--; + p = chunk_e + 2; + if (pstate->colors) { + kvi_push(shifts, ((StringShift) { + .start = token.start.col + (size_t)(chunk_e - s), + .orig_len = 2, + .act_len = 1, + .escape_not_known = false, + })); + } + } + node->data.str.size = size; + if (size == 0) { + node->data.str.value = NULL; + } else { + char *v_p; + v_p = node->data.str.value = xmalloc(size); + p = s + 1; + while (p < e) { + const char *const chunk_e = memchr(p, '\'', (size_t)(e - p)); + if (chunk_e == NULL) { + memcpy(v_p, p, (size_t)(e - p)); + break; + } + memcpy(v_p, p, (size_t)(chunk_e - p)); + v_p += (size_t)(chunk_e - p) + 1; + v_p[-1] = '\''; + p = chunk_e + 2; + } + } + } else { + viml_parser_highlight(pstate, token.start, 1, HL(DoubleQuote)); + for (p = s + 1; p < e; p++) { + if (*p == '\\' && p + 1 < e) { + p++; + if (p + 1 == e) { + size--; + break; + } + switch (*p) { + // A "\<x>" form occupies at least 4 characters, and produces up to + // 6 characters: reserve space for 2 extra, but do not compute actual + // length just now, it would be costy. + case '<': { + size += 2; + break; + } + // Hexadecimal, always single byte, but at least three bytes each. + case 'x': case 'X': { + size--; + if (ascii_isxdigit(p[1])) { + size--; + if (p + 2 < e && ascii_isxdigit(p[2])) { + size--; + } + } + break; + } + // Unicode + // + // \uF takes 1 byte which is 2 bytes less then escape sequence. + // \uFF: 2 bytes, 2 bytes less. + // \uFFF: 3 bytes, 2 bytes less. + // \uFFFF: 3 bytes, 3 bytes less. + // \UFFFFF: 4 bytes, 3 bytes less. + // \UFFFFFF: 5 bytes, 3 bytes less. + // \UFFFFFFF: 6 bytes, 3 bytes less. + // \U7FFFFFFF: 6 bytes, 4 bytes less. + case 'u': case 'U': { + const char *const esc_start = p; + size_t n = (*p == 'u' ? 4 : 8); + int nr = 0; + p++; + while (p + 1 < e && n-- && ascii_isxdigit(p[1])) { + p++; + nr = (nr << 4) + hex2nr(*p); + } + // Escape length: (esc_start - 1) points to "\\", esc_start to "u" + // or "U", p to the byte after last byte. So escape sequence + // occupies p - (esc_start - 1), but it stands for a utf_char2len + // bytes. + size -= (size_t)((p - (esc_start - 1)) - utf_char2len(nr)); + p--; + break; + } + // Octal, always single byte, but at least two bytes each. + case '0': case '1': case '2': case '3': case '4': case '5': case '6': + case '7': { + size--; + p++; + if (*p >= '0' && *p <= '7') { + size--; + p++; + if (p < e && *p >= '0' && *p <= '7') { + size--; + p++; + } + } + break; + } + default: { + size--; + break; + } + } + } + } + if (size == 0) { + node->data.str.value = NULL; + node->data.str.size = 0; + } else { + char *v_p; + v_p = node->data.str.value = xmalloc(size); + p = s + 1; + while (p < e) { + const char *const chunk_e = memchr(p, '\\', (size_t)(e - p)); + if (chunk_e == NULL) { + memcpy(v_p, p, (size_t)(e - p)); + v_p += e - p; + break; + } + memcpy(v_p, p, (size_t)(chunk_e - p)); + v_p += (size_t)(chunk_e - p); + p = chunk_e + 1; + if (p == e) { + *v_p++ = '\\'; + break; + } + bool is_unknown = false; + const char *const v_p_start = v_p; + switch (*p) { +#define SINGLE_CHAR_ESC(ch, real_ch) \ + case ch: { \ + *v_p++ = real_ch; \ + p++; \ + break; \ + } + SINGLE_CHAR_ESC('b', BS) + SINGLE_CHAR_ESC('e', ESC) + SINGLE_CHAR_ESC('f', FF) + SINGLE_CHAR_ESC('n', NL) + SINGLE_CHAR_ESC('r', CAR) + SINGLE_CHAR_ESC('t', TAB) + SINGLE_CHAR_ESC('"', '"') + SINGLE_CHAR_ESC('\\', '\\') +#undef SINGLE_CHAR_ESC + + // Hexadecimal or unicode. + case 'X': case 'x': case 'u': case 'U': { + if (p + 1 < e && ascii_isxdigit(p[1])) { + size_t n; + int nr; + bool is_hex = (*p == 'x' || *p == 'X'); + + if (is_hex) { + n = 2; + } else if (*p == 'u') { + n = 4; + } else { + n = 8; + } + nr = 0; + while (p + 1 < e && n-- && ascii_isxdigit(p[1])) { + p++; + nr = (nr << 4) + hex2nr(*p); + } + p++; + if (is_hex) { + *v_p++ = (char)nr; + } else { + v_p += utf_char2bytes(nr, (char_u *)v_p); + } + } else { + is_unknown = true; + *v_p++ = *p; + p++; + } + break; + } + // Octal: "\1", "\12", "\123". + case '0': case '1': case '2': case '3': case '4': case '5': case '6': + case '7': { + uint8_t ch = (uint8_t)(*p++ - '0'); + if (p < e && *p >= '0' && *p <= '7') { + ch = (uint8_t)((ch << 3) + *p++ - '0'); + if (p < e && *p >= '0' && *p <= '7') { + ch = (uint8_t)((ch << 3) + *p++ - '0'); + } + } + *v_p++ = (char)ch; + break; + } + // Special key, e.g.: "\<C-W>" + case '<': { + const size_t special_len = ( + trans_special((const char_u **)&p, (size_t)(e - p), + (char_u *)v_p, true, true)); + if (special_len != 0) { + v_p += special_len; + } else { + is_unknown = true; + mb_copy_char((const char_u **)&p, (char_u **)&v_p); + } + break; + } + default: { + is_unknown = true; + mb_copy_char((const char_u **)&p, (char_u **)&v_p); + break; + } + } + if (pstate->colors) { + kvi_push(shifts, ((StringShift) { + .start = token.start.col + (size_t)(chunk_e - s), + .orig_len = (size_t)(p - chunk_e), + .act_len = (size_t)(v_p - (char *)v_p_start), + .escape_not_known = is_unknown, + })); + } + } + node->data.str.size = (size_t)(v_p - node->data.str.value); + } + } + if (pstate->colors) { + // TODO(ZyX-I): use ast_stack to determine and highlight regular expressions + // TODO(ZyX-I): use ast_stack to determine and highlight printf format str + // TODO(ZyX-I): use ast_stack to determine and highlight expression strings + size_t next_col = token.start.col + 1; + const char *const body_str = (is_double + ? HL(DoubleQuotedBody) + : HL(SingleQuotedBody)); + const char *const esc_str = (is_double + ? HL(DoubleQuotedEscape) + : HL(SingleQuotedQuote)); + const char *const ukn_esc_str = (is_double + ? HL(DoubleQuotedUnknownEscape) + : HL(SingleQuotedUnknownEscape)); + for (size_t i = 0; i < kv_size(shifts); i++) { + const StringShift cur_shift = kv_A(shifts, i); + if (cur_shift.start > next_col) { + viml_parser_highlight(pstate, recol_pos(token.start, next_col), + cur_shift.start - next_col, + body_str); + } + viml_parser_highlight(pstate, recol_pos(token.start, cur_shift.start), + cur_shift.orig_len, + (cur_shift.escape_not_known + ? ukn_esc_str + : esc_str)); + next_col = cur_shift.start + cur_shift.orig_len; + } + if (next_col - token.start.col < token.len - token.data.str.closed) { + viml_parser_highlight(pstate, recol_pos(token.start, next_col), + (token.start.col + + token.len + - token.data.str.closed + - next_col), + body_str); + } + } + if (token.data.str.closed) { + if (is_double) { + viml_parser_highlight(pstate, shifted_pos(token.start, token.len - 1), + 1, HL(DoubleQuote)); + } else { + viml_parser_highlight(pstate, shifted_pos(token.start, token.len - 1), + 1, HL(SingleQuote)); + } + } + kvi_destroy(shifts); +} + +/// Additional flags to pass to lexer depending on want_node +static const int want_node_to_lexer_flags[] = { + [kENodeValue] = kELFlagIsNotCmp, + [kENodeOperator] = kELFlagForbidScope, + [kENodeArgument] = kELFlagIsNotCmp, + [kENodeArgumentSeparator] = kELFlagForbidScope, +}; + +/// Number of characters to highlight as NumberPrefix depending on the base +static const uint8_t base_to_prefix_length[] = { + [2] = 2, + [8] = 1, + [10] = 0, + [16] = 2, +}; + +/// Parse one VimL expression +/// +/// @param pstate Parser state. +/// @param[in] flags Additional flags, see ExprParserFlags +/// +/// @return Parsed AST. +ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL +{ + ExprAST ast = { + .err = { + .msg = NULL, + .arg_len = 0, + .arg = NULL, + }, + .root = NULL, + }; + // Expression stack contains current branch in AST tree: that is + // - Stack item 0 contains root of the tree, i.e. &ast->root. + // - Stack item i points to the previous stack items’ last child. + // + // When parser expects “value” node that is something like identifier or "[" + // (list start) last stack item contains NULL. Otherwise last stack item is + // supposed to contain last “finished” value: e.g. "1" or "+(1, 1)" (node + // representing "1+1"). + // + // Both kENodeValue and kENodeArgument stand for “value” nodes. + ExprASTStack ast_stack; + kvi_init(ast_stack); + kvi_push(ast_stack, &ast.root); + ExprASTWantedNode want_node = kENodeValue; + LexExprToken prev_token = { .type = kExprLexMissing }; + bool highlighted_prev_spacing = false; + // Lambda node, valid when parsing lambda arguments only. + ExprASTNode *lambda_node = NULL; + do { + const bool is_concat_or_subscript = ( + want_node == kENodeValue + && kv_size(ast_stack) > 1 + && (*kv_Z(ast_stack, 1))->type == kExprNodeConcatOrSubscript); + const int lexer_additional_flags = ( + kELFlagPeek + | ((flags & kExprFlagsDisallowEOC) ? kELFlagForbidEOC : 0) + | ((want_node == kENodeValue + && (kv_size(ast_stack) == 1 + || ((*kv_Z(ast_stack, 1))->type != kExprNodeConcat + && ((*kv_Z(ast_stack, 1))->type + != kExprNodeConcatOrSubscript)))) + ? kELFlagAllowFloat + : 0)); + LexExprToken cur_token = viml_pexpr_next_token( + pstate, want_node_to_lexer_flags[want_node] | lexer_additional_flags); + if (cur_token.type == kExprLexEOC) { + break; + } + LexExprTokenType tok_type = cur_token.type; + const bool token_invalid = (tok_type == kExprLexInvalid); + bool is_invalid = token_invalid; +viml_pexpr_parse_process_token: + // May use different flags this time. + cur_token = viml_pexpr_next_token( + pstate, want_node_to_lexer_flags[want_node] | lexer_additional_flags); + if (tok_type == kExprLexSpacing) { + if (is_invalid) { + HL_CUR_TOKEN(Spacing); + } else { + // Do not do anything: let regular spacing be highlighted as normal. + // This also allows later to highlight spacing as invalid. + } + goto viml_pexpr_parse_cycle_end; + } else if (is_invalid && prev_token.type == kExprLexSpacing + && !highlighted_prev_spacing) { + viml_parser_highlight(pstate, prev_token.start, prev_token.len, + HL(Spacing)); + is_invalid = false; + highlighted_prev_spacing = true; + } + const ParserLine pline = pstate->reader.lines.items[cur_token.start.line]; + ExprASTNode **const top_node_p = kv_last(ast_stack); + assert(kv_size(ast_stack) >= 1); + ExprASTNode *cur_node = NULL; +#ifndef NDEBUG + const bool want_value = ( + want_node == kENodeValue || want_node == kENodeArgument); + assert(want_value == (*top_node_p == NULL)); + assert(kv_A(ast_stack, 0) == &ast.root); + // Check that stack item i + 1 points to stack items’ i *last* child. + for (size_t i = 0; i + 1 < kv_size(ast_stack); i++) { + const bool item_null = (want_value && i + 2 == kv_size(ast_stack)); + assert((&(*kv_A(ast_stack, i))->children == kv_A(ast_stack, i + 1) + && (item_null + ? (*kv_A(ast_stack, i))->children == NULL + : (*kv_A(ast_stack, i))->children->next == NULL)) + || ((&(*kv_A(ast_stack, i))->children->next + == kv_A(ast_stack, i + 1)) + && (item_null + ? (*kv_A(ast_stack, i))->children->next == NULL + : (*kv_A(ast_stack, i))->children->next->next == NULL))); + } +#endif + // Note: in Vim whether expression "cond?d.a:2" is valid depends both on + // "cond" and whether "d" is a dictionary: expression is valid if condition + // is true and "d" is a dictionary (with "a" key or it will complain about + // missing one, but this is not relevant); if any of the requirements is + // broken then this thing is parsed as "d . a:2" yielding missing colon + // error. This parser does not allow such ambiguity, especially because it + // simply can’t: whether "d" is a dictionary is not known at the parsing + // time. + // + // Here example will always contain a concat with "a:2" sucking colon, + // making expression invalid both because there is no longer a spare colon + // for ternary and because concatenating dictionary with anything is not + // valid. There are more cases when this will make a difference though. + const bool node_is_key = ( + is_concat_or_subscript + && (cur_token.type == kExprLexPlainIdentifier + ? (!cur_token.data.var.autoload + && cur_token.data.var.scope == kExprVarScopeMissing) + : (cur_token.type == kExprLexNumber)) + && prev_token.type != kExprLexSpacing); + if (is_concat_or_subscript && !node_is_key) { + // Note: in Vim "d. a" (this is the reason behind `prev_token.type != + // kExprLexSpacing` part of the condition) as well as any other "d.{expr}" + // where "{expr}" does not look like a key is invalid whenever "d" happens + // to be a dictionary. Since parser has no idea whether preceding + // expression is actually a dictionary it can’t outright reject anything, + // so it turns kExprNodeConcatOrSubscript into kExprNodeConcat instead, + // which will yield different errors then Vim does in a number of + // circumstances, and in any case runtime and not parse time errors. + (*kv_Z(ast_stack, 1))->type = kExprNodeConcat; + } + if ((want_node == kENodeArgumentSeparator + && tok_type != kExprLexComma + && tok_type != kExprLexArrow) + || (want_node == kENodeArgument + && !(cur_token.type == kExprLexPlainIdentifier + && cur_token.data.var.scope == kExprVarScopeMissing + && !cur_token.data.var.autoload) + && tok_type != kExprLexArrow)) { + lambda_node->data.fig.type_guesses.allow_lambda = false; + if (lambda_node->children != NULL + && lambda_node->children->type == kExprNodeComma) { + // If lambda has comma child this means that parser has already seen at + // least "{arg1,", so node cannot possibly be anything, but lambda. + + // Vim may give E121 or E720 in this case, but it does not look right to + // have either because both are results of reevaluation possibly-lambda + // node as a dictionary and here this is not going to happen. + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Expected lambda arguments list or arrow: %.*s")); + } else { + // Else it may appear that possibly-lambda node is actually a dictionary + // or curly-braces-name identifier. + lambda_node = NULL; + if (want_node == kENodeArgumentSeparator) { + want_node = kENodeOperator; + } else { + want_node = kENodeValue; + } + } + } + assert(lambda_node == NULL + || want_node == kENodeArgumentSeparator + || want_node == kENodeArgument); + switch (tok_type) { + case kExprLexMissing: + case kExprLexSpacing: + case kExprLexEOC: { + assert(false); + } + case kExprLexInvalid: { + ERROR_FROM_TOKEN(cur_token); + tok_type = cur_token.data.err.type; + goto viml_pexpr_parse_process_token; + } + case kExprLexRegister: { + if (want_node == kENodeOperator) { + // Register in operator position: e.g. @a @a + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeRegister); + cur_node->data.reg.name = cur_token.data.reg.name; + *top_node_p = cur_node; + want_node = kENodeOperator; + HL_CUR_TOKEN(Register); + break; + } +#define SIMPLE_UB_OP(op) \ + case kExprLex##op: { \ + if (want_node == kENodeValue) { \ + /* Value level: assume unary operator. */ \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnary##op); \ + *top_node_p = cur_node; \ + kvi_push(ast_stack, &cur_node->children); \ + HL_CUR_TOKEN(Unary##op); \ + } else { \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeBinary##op); \ + ADD_OP_NODE(cur_node); \ + HL_CUR_TOKEN(Binary##op); \ + } \ + want_node = kENodeValue; \ + break; \ + } + SIMPLE_UB_OP(Plus) + SIMPLE_UB_OP(Minus) +#undef SIMPLE_UB_OP +#define SIMPLE_B_OP(op, msg) \ + case kExprLex##op: { \ + ADD_VALUE_IF_MISSING(_("E15: Unexpected " msg ": %.*s")); \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNode##op); \ + HL_CUR_TOKEN(op); \ + ADD_OP_NODE(cur_node); \ + break; \ + } + SIMPLE_B_OP(Or, "or operator") + SIMPLE_B_OP(And, "and operator") +#undef SIMPLE_B_OP + case kExprLexMultiplication: { + ADD_VALUE_IF_MISSING( + _("E15: Unexpected multiplication-like operator: %.*s")); + switch (cur_token.data.mul.type) { +#define MUL_OP(lex_op_tail, node_op_tail) \ + case kExprLexMul##lex_op_tail: { \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNode##node_op_tail); \ + HL_CUR_TOKEN(node_op_tail); \ + break; \ + } + MUL_OP(Mul, Multiplication) + MUL_OP(Div, Division) + MUL_OP(Mod, Mod) +#undef MUL_OP + } + ADD_OP_NODE(cur_node); + break; + } + case kExprLexOption: { + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeOption); + if (cur_token.type == kExprLexInvalid) { + assert(cur_token.len == 1 + || (cur_token.len == 3 + && pline.data[cur_token.start.col + 2] == ':')); + cur_node->data.opt.ident = ( + pline.data + cur_token.start.col + cur_token.len); + cur_node->data.opt.ident_len = 0; + cur_node->data.opt.scope = ( + cur_token.len == 3 + ? (ExprOptScope)pline.data[cur_token.start.col + 1] + : kExprOptScopeUnspecified); + } else { + cur_node->data.opt.ident = cur_token.data.opt.name; + cur_node->data.opt.ident_len = cur_token.data.opt.len; + cur_node->data.opt.scope = cur_token.data.opt.scope; + } + *top_node_p = cur_node; + want_node = kENodeOperator; + viml_parser_highlight(pstate, cur_token.start, 1, HL(OptionSigil)); + const size_t scope_shift = ( + cur_token.data.opt.scope == kExprOptScopeUnspecified ? 0 : 2); + if (scope_shift) { + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), 1, + HL(OptionScope)); + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 2), 1, + HL(OptionScopeDelimiter)); + } + viml_parser_highlight( + pstate, shifted_pos(cur_token.start, scope_shift + 1), + cur_token.len - (scope_shift + 1), HL(OptionName)); + break; + } + case kExprLexEnv: { + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeEnvironment); + cur_node->data.env.ident = pline.data + cur_token.start.col + 1; + cur_node->data.env.ident_len = cur_token.len - 1; + if (cur_node->data.env.ident_len == 0) { + ERROR_FROM_TOKEN_AND_MSG(cur_token, + _("E15: Environment variable name missing")); + } + *top_node_p = cur_node; + want_node = kENodeOperator; + viml_parser_highlight(pstate, cur_token.start, 1, HL(EnvironmentSigil)); + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), + cur_token.len - 1, HL(EnvironmentName)); + break; + } + case kExprLexNot: { + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeNot); + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + HL_CUR_TOKEN(Not); + break; + } + case kExprLexComparison: { + ADD_VALUE_IF_MISSING( + _("E15: Expected value, got comparison operator: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComparison); + if (cur_token.type == kExprLexInvalid) { + cur_node->data.cmp.ccs = kCCStrategyUseOption; + cur_node->data.cmp.type = kExprCmpEqual; + cur_node->data.cmp.inv = false; + } else { + cur_node->data.cmp.ccs = cur_token.data.cmp.ccs; + cur_node->data.cmp.type = cur_token.data.cmp.type; + cur_node->data.cmp.inv = cur_token.data.cmp.inv; + } + ADD_OP_NODE(cur_node); + if (cur_token.data.cmp.ccs != kCCStrategyUseOption) { + viml_parser_highlight(pstate, cur_token.start, cur_token.len - 1, + HL(Comparison)); + viml_parser_highlight( + pstate, shifted_pos(cur_token.start, cur_token.len - 1), 1, + HL(ComparisonModifier)); + } else { + HL_CUR_TOKEN(Comparison); + } + want_node = kENodeValue; + break; + } + case kExprLexComma: { + assert(want_node != kENodeArgument); + if (want_node == kENodeValue) { + // Value level: comma appearing here is not valid. + // Note: in Vim string(,x) will give E116, this is not the case here. + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Expected value, got comma: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeMissing); + cur_node->len = 0; + *top_node_p = cur_node; + want_node = (want_node == kENodeArgument + ? kENodeArgumentSeparator + : kENodeOperator); + } + if (want_node == kENodeArgumentSeparator) { + assert(lambda_node->data.fig.type_guesses.allow_lambda); + assert(lambda_node != NULL); + SELECT_FIGURE_BRACE_TYPE(lambda_node, Lambda, Lambda); + } + if (kv_size(ast_stack) < 2) { + goto viml_pexpr_parse_invalid_comma; + } + for (size_t i = 1; i < kv_size(ast_stack); i++) { + ExprASTNode *const *const eastnode_p = + (ExprASTNode *const *)kv_Z(ast_stack, i); + const ExprASTNodeType eastnode_type = (*eastnode_p)->type; + const ExprOpLvl eastnode_lvl = node_lvl(**eastnode_p); + if (eastnode_type == kExprNodeLambda) { + assert(want_node == kENodeArgumentSeparator); + break; + } else if (eastnode_type == kExprNodeDictLiteral + || eastnode_type == kExprNodeListLiteral + || eastnode_type == kExprNodeCall) { + break; + } else if (eastnode_type == kExprNodeComma + || eastnode_type == kExprNodeColon + || eastnode_lvl > kEOpLvlComma) { + // Do nothing + } else { +viml_pexpr_parse_invalid_comma: + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Comma outside of call, lambda or literal: %.*s")); + break; + } + if (i == kv_size(ast_stack) - 1) { + goto viml_pexpr_parse_invalid_comma; + } + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComma); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(Comma); + break; + } +#define EXP_VAL_COLON "E15: Expected value, got colon: %.*s" + case kExprLexColon: { + if (kv_size(ast_stack) < 2) { + goto viml_pexpr_parse_invalid_colon; + } + bool is_ternary = false; + bool can_be_ternary = true; + bool is_subscript = false; + for (size_t i = 1; i < kv_size(ast_stack); i++) { + ExprASTNode *const *const eastnode_p = + (ExprASTNode *const *)kv_Z(ast_stack, i); + const ExprASTNodeType eastnode_type = (*eastnode_p)->type; + const ExprOpLvl eastnode_lvl = node_lvl(**eastnode_p); + STATIC_ASSERT(kEOpLvlTernary > kEOpLvlComma, + "Unexpected operator priorities"); + if (can_be_ternary && eastnode_type == kExprNodeTernaryValue + && !(*eastnode_p)->data.ter.got_colon) { + kv_drop(ast_stack, i); + (*eastnode_p)->start = cur_token.start; + (*eastnode_p)->len = cur_token.len; + if (prev_token.type == kExprLexSpacing) { + (*eastnode_p)->start = prev_token.start; + (*eastnode_p)->len += prev_token.len; + } + is_ternary = true; + (*eastnode_p)->data.ter.got_colon = true; + ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); + assert((*eastnode_p)->children != NULL); + assert((*eastnode_p)->children->next == NULL); + kvi_push(ast_stack, &(*eastnode_p)->children->next); + break; + } else if (eastnode_type == kExprNodeUnknownFigure) { + SELECT_FIGURE_BRACE_TYPE(*eastnode_p, DictLiteral, Dict); + break; + } else if (eastnode_type == kExprNodeDictLiteral) { + break; + } else if (eastnode_type == kExprNodeSubscript) { + is_subscript = true; + can_be_ternary = false; + assert(!is_ternary); + break; + } else if (eastnode_type == kExprNodeColon) { + goto viml_pexpr_parse_invalid_colon; + } else if (eastnode_lvl >= kEOpLvlTernaryValue) { + // Do nothing + } else if (eastnode_lvl >= kEOpLvlComma) { + can_be_ternary = false; + } else { + goto viml_pexpr_parse_invalid_colon; + } + if (i == kv_size(ast_stack) - 1) { + goto viml_pexpr_parse_invalid_colon; + } + } + if (is_subscript) { + assert(kv_size(ast_stack) > 1); + // Colon immediately following subscript start: it is empty subscript + // part like a[:2]. + if (want_node == kENodeValue + && (*kv_Z(ast_stack, 1))->type == kExprNodeSubscript) { + NEW_NODE_WITH_CUR_POS(*top_node_p, kExprNodeMissing); + (*top_node_p)->len = 0; + want_node = kENodeOperator; + } else { + ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(SubscriptColon); + } else { + goto viml_pexpr_parse_valid_colon; +viml_pexpr_parse_invalid_colon: + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Colon outside of dictionary or ternary operator: %.*s")); +viml_pexpr_parse_valid_colon: + ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); + if (is_ternary) { + HL_CUR_TOKEN(TernaryColon); + } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(Colon); + } + } + want_node = kENodeValue; + break; + } +#undef EXP_VAL_COLON + case kExprLexBracket: { + if (cur_token.data.brc.closing) { + ExprASTNode **new_top_node_p = NULL; + // Always drop the topmost value: + // + // 1. When want_node != kENodeValue topmost item on stack is + // a *finished* left operand, which may as well be "{@a}" which + // needs not be finished again. + // 2. Otherwise it is pointing to NULL what nobody wants. + kv_drop(ast_stack, 1); + if (!kv_size(ast_stack)) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeListLiteral); + cur_node->len = 0; + if (want_node != kENodeValue) { + cur_node->children = *top_node_p; + } + *top_node_p = cur_node; + goto viml_pexpr_parse_bracket_closing_error; + } + if (want_node == kENodeValue) { + // It is OK to want value if + // + // 1. It is empty list literal, in which case top node will be + // ListLiteral. + // 2. It is list literal with trailing comma, in which case top node + // will be that comma. + // 3. It is subscript with colon, but without one of the values: + // e.g. "a[:]", "a[1:]", top node will be colon in this case. + if ((*kv_last(ast_stack))->type != kExprNodeListLiteral + && (*kv_last(ast_stack))->type != kExprNodeComma + && (*kv_last(ast_stack))->type != kExprNodeColon) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected value, got closing bracket: %.*s")); + } + } else { + if (!kv_size(ast_stack)) { + new_top_node_p = top_node_p; + goto viml_pexpr_parse_bracket_closing_error; + } + } + do { + new_top_node_p = kv_pop(ast_stack); + } while (kv_size(ast_stack) + && (new_top_node_p == NULL + || ((*new_top_node_p)->type != kExprNodeListLiteral + && (*new_top_node_p)->type != kExprNodeSubscript))); + ExprASTNode *new_top_node = *new_top_node_p; + switch (new_top_node->type) { + case kExprNodeListLiteral: { + HL_CUR_TOKEN(List); + break; + } + case kExprNodeSubscript: { + HL_CUR_TOKEN(SubscriptBracket); + break; + } + default: { +viml_pexpr_parse_bracket_closing_error: + assert(!kv_size(ast_stack)); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Unexpected closing figure brace: %.*s")); + HL_CUR_TOKEN(List); + break; + } + } + kvi_push(ast_stack, new_top_node_p); + want_node = kENodeOperator; + } else { + if (want_node == kENodeValue) { + // Value means list literal. + HL_CUR_TOKEN(List); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeListLiteral); + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + want_node = kENodeValue; + } else { + if (prev_token.type == kExprLexSpacing) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeSubscript); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(SubscriptBracket); + } + } + break; + } + case kExprLexFigureBrace: { + if (cur_token.data.brc.closing) { + ExprASTNode **new_top_node_p = NULL; + // Always drop the topmost value: + // + // 1. When want_node != kENodeValue topmost item on stack is + // a *finished* left operand, which may as well be "{@a}" which + // needs not be finished again. + // 2. Otherwise it is pointing to NULL what nobody wants. + kv_drop(ast_stack, 1); + if (!kv_size(ast_stack)) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnknownFigure); + cur_node->data.fig.type_guesses.allow_lambda = false; + cur_node->data.fig.type_guesses.allow_dict = false; + cur_node->data.fig.type_guesses.allow_ident = false; + cur_node->len = 0; + if (want_node != kENodeValue) { + cur_node->children = *top_node_p; + } + *top_node_p = cur_node; + new_top_node_p = top_node_p; + goto viml_pexpr_parse_figure_brace_closing_error; + } + if (want_node == kENodeValue) { + if ((*kv_last(ast_stack))->type != kExprNodeUnknownFigure + && (*kv_last(ast_stack))->type != kExprNodeComma) { + // kv_last being UnknownFigure may occur for empty dictionary + // literal, while Comma is expected in case of non-empty one. + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected value, got closing figure brace: %.*s")); + } + } else { + if (!kv_size(ast_stack)) { + new_top_node_p = top_node_p; + goto viml_pexpr_parse_figure_brace_closing_error; + } + } + do { + new_top_node_p = kv_pop(ast_stack); + } while (kv_size(ast_stack) + && (new_top_node_p == NULL + || ((*new_top_node_p)->type != kExprNodeUnknownFigure + && (*new_top_node_p)->type != kExprNodeDictLiteral + && ((*new_top_node_p)->type + != kExprNodeCurlyBracesIdentifier) + && (*new_top_node_p)->type != kExprNodeLambda))); + ExprASTNode *new_top_node = *new_top_node_p; + switch (new_top_node->type) { + case kExprNodeUnknownFigure: { + if (new_top_node->children == NULL) { + // No children of curly braces node indicates empty dictionary. + + // Should actually be kENodeArgument, but that was changed + // earlier. + assert(want_node == kENodeValue); + assert(new_top_node->data.fig.type_guesses.allow_dict); + SELECT_FIGURE_BRACE_TYPE(new_top_node, DictLiteral, Dict); + HL_CUR_TOKEN(Dict); + } else if (new_top_node->data.fig.type_guesses.allow_ident) { + SELECT_FIGURE_BRACE_TYPE(new_top_node, CurlyBracesIdentifier, + Curly); + HL_CUR_TOKEN(Curly); + } else { + // If by this time type of the node has not already been + // guessed, but it definitely is not a curly braces name then + // it is invalid for sure. + ERROR_FROM_NODE_AND_MSG( + new_top_node, + _("E15: Don't know what figure brace means: %.*s")); + if (pstate->colors) { + // Will reset to NVimInvalidFigureBrace. + kv_A(*pstate->colors, + new_top_node->data.fig.opening_hl_idx).group = ( + HL(FigureBrace)); + } + HL_CUR_TOKEN(FigureBrace); + } + break; + } + case kExprNodeDictLiteral: { + HL_CUR_TOKEN(Dict); + break; + } + case kExprNodeCurlyBracesIdentifier: { + HL_CUR_TOKEN(Curly); + break; + } + case kExprNodeLambda: { + HL_CUR_TOKEN(Lambda); + break; + } + default: { +viml_pexpr_parse_figure_brace_closing_error: + assert(!kv_size(ast_stack)); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Unexpected closing figure brace: %.*s")); + HL_CUR_TOKEN(FigureBrace); + break; + } + } + kvi_push(ast_stack, new_top_node_p); + want_node = kENodeOperator; + } else { + if (want_node == kENodeValue) { + HL_CUR_TOKEN(FigureBrace); + // Value: may be any of lambda, dictionary literal and curly braces + // name. + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnknownFigure); + cur_node->data.fig.type_guesses.allow_lambda = true; + cur_node->data.fig.type_guesses.allow_dict = true; + cur_node->data.fig.type_guesses.allow_ident = true; + if (pstate->colors) { + cur_node->data.fig.opening_hl_idx = kv_size(*pstate->colors) - 1; + } + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + want_node = kENodeArgument; + lambda_node = cur_node; + } else { + ADD_IDENT( + do { + NEW_NODE_WITH_CUR_POS(cur_node, + kExprNodeCurlyBracesIdentifier); + cur_node->data.fig.opening_hl_idx = kv_size(*pstate->colors); + cur_node->data.fig.type_guesses.allow_lambda = false; + cur_node->data.fig.type_guesses.allow_dict = false; + cur_node->data.fig.type_guesses.allow_ident = true; + kvi_push(ast_stack, &cur_node->children); + want_node = kENodeValue; + } while (0), + Curly); + } + } + break; + } + case kExprLexArrow: { + if (want_node == kENodeArgumentSeparator + || want_node == kENodeArgument) { + if (want_node == kENodeArgument) { + kv_drop(ast_stack, 1); + } + assert(kv_size(ast_stack) >= 1); + while ((*kv_last(ast_stack))->type != kExprNodeLambda + && (*kv_last(ast_stack))->type != kExprNodeUnknownFigure) { + kv_drop(ast_stack, 1); + } + assert((*kv_last(ast_stack)) == lambda_node); + SELECT_FIGURE_BRACE_TYPE(lambda_node, Lambda, Lambda); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeArrow); + if (lambda_node->children == NULL) { + assert(want_node == kENodeArgument); + lambda_node->children = cur_node; + kvi_push(ast_stack, &lambda_node->children); + } else { + assert(lambda_node->children->next == NULL); + lambda_node->children->next = cur_node; + kvi_push(ast_stack, &lambda_node->children->next); + } + kvi_push(ast_stack, &cur_node->children); + lambda_node = NULL; + } else { + // Only first branch is valid. + ADD_VALUE_IF_MISSING(_("E15: Unexpected arrow: %.*s")); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Arrow outside of lambda: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeArrow); + ADD_OP_NODE(cur_node); + } + want_node = kENodeValue; + HL_CUR_TOKEN(Arrow); + break; + } + case kExprLexPlainIdentifier: { + const ExprVarScope scope = (cur_token.type == kExprLexInvalid + ? kExprVarScopeMissing + : cur_token.data.var.scope); + if (want_node == kENodeValue || want_node == kENodeArgument) { + want_node = (want_node == kENodeArgument + ? kENodeArgumentSeparator + : kENodeOperator); + NEW_NODE_WITH_CUR_POS(cur_node, + (node_is_key + ? kExprNodePlainKey + : kExprNodePlainIdentifier)); + cur_node->data.var.scope = scope; + const size_t scope_shift = (scope == kExprVarScopeMissing ? 0 : 2); + cur_node->data.var.ident = (pline.data + cur_token.start.col + + scope_shift); + cur_node->data.var.ident_len = cur_token.len - scope_shift; + *top_node_p = cur_node; + if (scope_shift) { + assert(!node_is_key); + viml_parser_highlight(pstate, cur_token.start, 1, + HL(IdentifierScope)); + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), 1, + HL(IdentifierScopeDelimiter)); + } + viml_parser_highlight(pstate, shifted_pos(cur_token.start, + scope_shift), + cur_token.len - scope_shift, + (node_is_key + ? HL(IdentifierKey) + : HL(IdentifierName))); + } else { + if (scope == kExprVarScopeMissing) { + ADD_IDENT( + do { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainIdentifier); + cur_node->data.var.scope = scope; + cur_node->data.var.ident = pline.data + cur_token.start.col; + cur_node->data.var.ident_len = cur_token.len; + want_node = kENodeOperator; + } while (0), + IdentifierName); + } else { + OP_MISSING; + } + } + break; + } + case kExprLexNumber: { + if (want_node != kENodeValue) { + OP_MISSING; + } + if (node_is_key) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainKey); + cur_node->data.var.ident = pline.data + cur_token.start.col; + cur_node->data.var.ident_len = cur_token.len; + HL_CUR_TOKEN(IdentifierKey); + } else if (cur_token.data.num.is_float) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeFloat); + cur_node->data.flt.value = cur_token.data.num.val.floating; + HL_CUR_TOKEN(Float); + } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeInteger); + cur_node->data.num.value = cur_token.data.num.val.integer; + const uint8_t prefix_length = base_to_prefix_length[ + cur_token.data.num.base]; + viml_parser_highlight(pstate, cur_token.start, prefix_length, + HL(NumberPrefix)); + viml_parser_highlight( + pstate, shifted_pos(cur_token.start, prefix_length), + cur_token.len - prefix_length, HL(Number)); + } + want_node = kENodeOperator; + *top_node_p = cur_node; + break; + } + case kExprLexDot: { + ADD_VALUE_IF_MISSING(_("E15: Unexpected dot: %.*s")); + if (prev_token.type == kExprLexSpacing) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeConcat); + HL_CUR_TOKEN(Concat); + } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeConcatOrSubscript); + HL_CUR_TOKEN(ConcatOrSubscript); + } + ADD_OP_NODE(cur_node); + break; + } + case kExprLexParenthesis: { + if (cur_token.data.brc.closing) { + if (want_node == kENodeValue) { + if (kv_size(ast_stack) > 1) { + const ExprASTNode *const prev_top_node = *kv_Z(ast_stack, 1); + if (prev_top_node->type == kExprNodeCall) { + // Function call without arguments, this is not an error. + // But further code does not expect NULL nodes. + kv_drop(ast_stack, 1); + goto viml_pexpr_parse_no_paren_closing_error; + } + } + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Expected value, got parenthesis: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeMissing); + cur_node->len = 0; + *top_node_p = cur_node; + } else { + // Always drop the topmost value: when want_node != kENodeValue + // topmost item on stack is a *finished* left operand, which may as + // well be "(@a)" which needs not be finished again. + kv_drop(ast_stack, 1); + } +viml_pexpr_parse_no_paren_closing_error: {} + ExprASTNode **new_top_node_p = NULL; + while (kv_size(ast_stack) + && (new_top_node_p == NULL + || ((*new_top_node_p)->type != kExprNodeNested + && (*new_top_node_p)->type != kExprNodeCall))) { + new_top_node_p = kv_pop(ast_stack); + } + if (new_top_node_p != NULL + && ((*new_top_node_p)->type == kExprNodeNested + || (*new_top_node_p)->type == kExprNodeCall)) { + if ((*new_top_node_p)->type == kExprNodeNested) { + HL_CUR_TOKEN(NestingParenthesis); + } else { + HL_CUR_TOKEN(CallingParenthesis); + } + } else { + // “Always drop the topmost value” branch has got rid of the single + // value stack had, so there is nothing known to enclose. Correct + // this. + if (new_top_node_p == NULL) { + new_top_node_p = top_node_p; + } + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Unexpected closing parenthesis: %.*s")); + HL_CUR_TOKEN(NestingParenthesis); + cur_node = NEW_NODE(kExprNodeNested); + cur_node->start = cur_token.start; + cur_node->len = 0; + // Unexpected closing parenthesis, assume that it was wanted to + // enclose everything in (). + cur_node->children = *new_top_node_p; + *new_top_node_p = cur_node; + assert(cur_node->next == NULL); + } + kvi_push(ast_stack, new_top_node_p); + want_node = kENodeOperator; + } else { + if (want_node == kENodeValue) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeNested); + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + HL_CUR_TOKEN(NestingParenthesis); + } else if (want_node == kENodeOperator) { + if (prev_token.type == kExprLexSpacing) { + // For some reason "function (args)" is a function call, but + // "(funcref) (args)" is not. AFAIR this somehow involves + // compatibility and Bram was commenting that this is + // intentionally inconsistent and he is not very happy with the + // situation himself. + if ((*top_node_p)->type != kExprNodePlainIdentifier + && (*top_node_p)->type != kExprNodeComplexIdentifier + && (*top_node_p)->type != kExprNodeCurlyBracesIdentifier) { + OP_MISSING; + } + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeCall); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(CallingParenthesis); + } else { + // Currently it is impossible to reach this. + assert(false); + } + want_node = kENodeValue; + } + break; + } + case kExprLexQuestion: { + ADD_VALUE_IF_MISSING(_("E15: Expected value, got question mark: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeTernary); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(Ternary); + ExprASTNode *ter_val_node; + NEW_NODE_WITH_CUR_POS(ter_val_node, kExprNodeTernaryValue); + ter_val_node->data.ter.got_colon = false; + assert(cur_node->children != NULL); + assert(cur_node->children->next == NULL); + assert(kv_last(ast_stack) == &cur_node->children->next); + *kv_last(ast_stack) = ter_val_node; + kvi_push(ast_stack, &ter_val_node->children); + break; + } + case kExprLexDoubleQuotedString: + case kExprLexSingleQuotedString: { + const bool is_double = (tok_type == kExprLexDoubleQuotedString); + if (!cur_token.data.str.closed) { + // It is weird, but Vim has two identical errors messages with + // different error numbers: "E114: Missing quote" and + // "E115: Missing quote". + ERROR_FROM_TOKEN_AND_MSG( + cur_token, (is_double + ? _("E114: Missing double quote: %.*s") + : _("E115: Missing single quote: %.*s"))); + } + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS( + cur_node, (is_double + ? kExprNodeDoubleQuotedString + : kExprNodeSingleQuotedString)); + *top_node_p = cur_node; + parse_quoted_string(pstate, cur_node, cur_token, ast_stack, is_invalid); + want_node = kENodeOperator; + break; + } + } +viml_pexpr_parse_cycle_end: + prev_token = cur_token; + highlighted_prev_spacing = false; + viml_parser_advance(pstate, cur_token.len); + } while (true); +viml_pexpr_parse_end: + if (want_node == kENodeValue) { + east_set_error(pstate, &ast.err, _("E15: Expected value, got EOC: %.*s"), + pstate->pos); + } else if (kv_size(ast_stack) != 1) { + // Something may be wrong, check whether it really is. + + // Pointer to ast.root must never be dropped, so “!= 1” is expected to be + // the same as “> 1”. + assert(kv_size(ast_stack)); + // Topmost stack item must be a *finished* value, so it must not be + // analyzed. E.g. it may contain an already finished nested expression. + kv_drop(ast_stack, 1); + while (ast.err.msg == NULL && kv_size(ast_stack)) { + const ExprASTNode *const cur_node = (*kv_pop(ast_stack)); + // This should only happen when want_node == kENodeValue. + assert(cur_node != NULL); + // TODO(ZyX-I): Rehighlight as invalid? + switch (cur_node->type) { + case kExprNodeOpMissing: + case kExprNodeMissing: { + // Error should’ve been already reported. + break; + } + case kExprNodeCall: { + east_set_error( + pstate, &ast.err, + _("E116: Missing closing parenthesis for function call: %.*s"), + cur_node->start); + break; + } + case kExprNodeNested: { + east_set_error( + pstate, &ast.err, + _("E110: Missing closing parenthesis for nested expression" + ": %.*s"), + cur_node->start); + break; + } + case kExprNodeListLiteral: { + // For whatever reason "[1" yields "E696: Missing comma in list" error + // in Vim while "[1," yields E697. + east_set_error( + pstate, &ast.err, + _("E697: Missing end of List ']': %.*s"), + cur_node->start); + break; + } + case kExprNodeDictLiteral: { + // Same problem like with list literal with E722 (missing comma) vs + // E723, but additionally just "{" yields only E15. + east_set_error( + pstate, &ast.err, + _("E723: Missing end of Dictionary '}': %.*s"), + cur_node->start); + break; + } + case kExprNodeUnknownFigure: { + east_set_error( + pstate, &ast.err, + _("E15: Missing closing figure brace: %.*s"), + cur_node->start); + break; + } + case kExprNodeLambda: { + east_set_error( + pstate, &ast.err, + _("E15: Missing closing figure brace for lambda: %.*s"), + cur_node->start); + break; + } + case kExprNodeCurlyBracesIdentifier: { + // Until trailing "}" it is impossible to distinguish curly braces + // identifier and dictionary, so it must not appear in the stack like + // this. + assert(false); + } + case kExprNodeInteger: + case kExprNodeFloat: + case kExprNodeSingleQuotedString: + case kExprNodeDoubleQuotedString: + case kExprNodeOption: + case kExprNodeEnvironment: + case kExprNodeRegister: + case kExprNodePlainIdentifier: + case kExprNodePlainKey: { + // These are plain values and not containers, for them it should only + // be possible to show up in the topmost stack element, but it was + // unconditionally popped at the start. + assert(false); + } + case kExprNodeComma: + case kExprNodeColon: + case kExprNodeArrow: { + // It is actually only valid inside something else, but everything + // where one of the above is valid requires to be closed and thus is + // to be caught later. + break; + } + case kExprNodeConcatOrSubscript: + case kExprNodeComplexIdentifier: + case kExprNodeSubscript: { + // FIXME: Investigate whether above are OK to be present in the stack. + break; + } + case kExprNodeMod: + case kExprNodeDivision: + case kExprNodeMultiplication: + case kExprNodeNot: + case kExprNodeAnd: + case kExprNodeOr: + case kExprNodeConcat: + case kExprNodeComparison: + case kExprNodeUnaryMinus: + case kExprNodeUnaryPlus: + case kExprNodeBinaryMinus: + case kExprNodeTernary: + case kExprNodeBinaryPlus: { + // It is OK to see these in the stack. + break; + } + case kExprNodeTernaryValue: { + if (!cur_node->data.ter.got_colon) { + // Actually Vim throws E109 in more cases. + east_set_error( + pstate, &ast.err, _("E109: Missing ':' after '?': %.*s"), + cur_node->start); + } + break; + } + } + } + } + kvi_destroy(ast_stack); + return ast; +} + +#undef NEW_NODE +#undef HL diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h new file mode 100644 index 0000000000..d00d4855f3 --- /dev/null +++ b/src/nvim/viml/parser/expressions.h @@ -0,0 +1,348 @@ +#ifndef NVIM_VIML_PARSER_EXPRESSIONS_H +#define NVIM_VIML_PARSER_EXPRESSIONS_H + +#include <stddef.h> +#include <stdint.h> +#include <stdbool.h> + +#include "nvim/types.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/eval/typval.h" + +// Defines whether to ignore case: +// == kCCStrategyUseOption +// ==# kCCStrategyMatchCase +// ==? kCCStrategyIgnoreCase +typedef enum { + kCCStrategyUseOption = 0, // 0 for xcalloc + kCCStrategyMatchCase = '#', + kCCStrategyIgnoreCase = '?', +} ExprCaseCompareStrategy; + +/// Lexer token type +typedef enum { + kExprLexInvalid = 0, ///< Invalid token, indicaten an error. + kExprLexMissing, ///< Missing token, for use in parser. + kExprLexSpacing, ///< Spaces, tabs, newlines, etc. + kExprLexEOC, ///< End of command character: NL, |, just end of stream. + + kExprLexQuestion, ///< Question mark, for use in ternary. + kExprLexColon, ///< Colon, for use in ternary. + kExprLexOr, ///< Logical or operator. + kExprLexAnd, ///< Logical and operator. + kExprLexComparison, ///< One of the comparison operators. + kExprLexPlus, ///< Plus sign. + kExprLexMinus, ///< Minus sign. + kExprLexDot, ///< Dot: either concat or subscript, also part of the float. + kExprLexMultiplication, ///< Multiplication, division or modulo operator. + + kExprLexNot, ///< Not: !. + + kExprLexNumber, ///< Integer number literal, or part of a float. + kExprLexSingleQuotedString, ///< Single quoted string literal. + kExprLexDoubleQuotedString, ///< Double quoted string literal. + kExprLexOption, ///< &optionname option value. + kExprLexRegister, ///< @r register value. + kExprLexEnv, ///< Environment $variable value. + kExprLexPlainIdentifier, ///< Identifier without scope: `abc`, `foo#bar`. + + kExprLexBracket, ///< Bracket, either opening or closing. + kExprLexFigureBrace, ///< Figure brace, either opening or closing. + kExprLexParenthesis, ///< Parenthesis, either opening or closing. + kExprLexComma, ///< Comma. + kExprLexArrow, ///< Arrow, like from lambda expressions. +} LexExprTokenType; + +typedef enum { + kExprCmpEqual, ///< Equality, unequality. + kExprCmpMatches, ///< Matches regex, not matches regex. + kExprCmpGreater, ///< `>` or `<=` + kExprCmpGreaterOrEqual, ///< `>=` or `<`. + kExprCmpIdentical, ///< `is` or `isnot` +} ExprComparisonType; + +/// All possible option scopes +typedef enum { + kExprOptScopeUnspecified = 0, + kExprOptScopeGlobal = 'g', + kExprOptScopeLocal = 'l', +} ExprOptScope; + +#define EXPR_OPT_SCOPE_LIST \ + ((char[]){ kExprOptScopeGlobal, kExprOptScopeLocal }) + +/// All possible variable scopes +typedef enum { + kExprVarScopeMissing = 0, + kExprVarScopeScript = 's', + kExprVarScopeGlobal = 'g', + kExprVarScopeVim = 'v', + kExprVarScopeBuffer = 'b', + kExprVarScopeWindow = 'w', + kExprVarScopeTabpage = 't', + kExprVarScopeLocal = 'l', + kExprVarScopeArguments = 'a', +} ExprVarScope; + +#define EXPR_VAR_SCOPE_LIST \ + ((char[]) { \ + kExprVarScopeScript, kExprVarScopeGlobal, kExprVarScopeVim, \ + kExprVarScopeBuffer, kExprVarScopeWindow, kExprVarScopeTabpage, \ + kExprVarScopeLocal, kExprVarScopeBuffer, kExprVarScopeArguments, \ + }) + +/// Lexer token +typedef struct { + ParserPosition start; + size_t len; + LexExprTokenType type; + union { + struct { + ExprComparisonType type; ///< Comparison type. + ExprCaseCompareStrategy ccs; ///< Case comparison strategy. + bool inv; ///< True if comparison is to be inverted. + } cmp; ///< For kExprLexComparison. + + struct { + enum { + kExprLexMulMul, ///< Real multiplication. + kExprLexMulDiv, ///< Division. + kExprLexMulMod, ///< Modulo. + } type; ///< Multiplication type. + } mul; ///< For kExprLexMultiplication. + + struct { + bool closing; ///< True if bracket/etc is a closing one. + } brc; ///< For brackets/braces/parenthesis. + + struct { + int name; ///< Register name, may be -1 if name not present. + } reg; ///< For kExprLexRegister. + + struct { + bool closed; ///< True if quote was closed. + } str; ///< For kExprLexSingleQuotedString and kExprLexDoubleQuotedString. + + struct { + const char *name; ///< Option name start. + size_t len; ///< Option name length. + ExprOptScope scope; ///< Option scope: &l:, &g: or not specified. + } opt; ///< Option properties. + + struct { + ExprVarScope scope; ///< Scope character or 0 if not present. + bool autoload; ///< Has autoload characters. + } var; ///< For kExprLexPlainIdentifier + + struct { + LexExprTokenType type; ///< Suggested type for parsing incorrect code. + const char *msg; ///< Error message. + } err; ///< For kExprLexInvalid + + struct { + union { + float_T floating; + uvarnumber_T integer; + } val; ///< Number value. + uint8_t base; ///< Base: 2, 8, 10 or 16. + bool is_float; ///< True if number is a floating-point. + } num; ///< For kExprLexNumber + } data; ///< Additional data, if needed. +} LexExprToken; + +typedef enum { + /// If set, “pointer” to the current byte in pstate will not be shifted + kELFlagPeek = (1 << 0), + /// Determines whether scope is allowed to come before the identifier + kELFlagForbidScope = (1 << 1), + /// Determines whether floating-point numbers are allowed + /// + /// I.e. whether dot is a decimal point separator or is not a part of + /// a number at all. + kELFlagAllowFloat = (1 << 2), + /// Determines whether `is` and `isnot` are seen as comparison operators + /// + /// If set they are supposed to be just regular identifiers. + kELFlagIsNotCmp = (1 << 3), + /// Determines whether EOC tokens are allowed + /// + /// If set then it will yield Invalid token with E15 in place of EOC one if + /// “EOC” is something like "|". It is fine with emitting EOC at the end of + /// string still, with or without this flag set. + kELFlagForbidEOC = (1 << 4), + // WARNING: whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_lexer.c. +} LexExprFlags; + +/// Expression AST node type +typedef enum { + kExprNodeMissing = 0, + kExprNodeOpMissing, + kExprNodeTernary, ///< Ternary operator. + kExprNodeTernaryValue, ///< Ternary operator, colon. + kExprNodeRegister, ///< Register. + kExprNodeSubscript, ///< Subscript. + kExprNodeListLiteral, ///< List literal. + kExprNodeUnaryPlus, + kExprNodeBinaryPlus, + kExprNodeNested, ///< Nested parenthesised expression. + kExprNodeCall, ///< Function call. + /// Plain identifier: simple variable/function name + /// + /// Looks like "string", "g:Foo", etc: consists from a single + /// kExprLexPlainIdentifier token. + kExprNodePlainIdentifier, + /// Plain dictionary key, for use with kExprNodeConcatOrSubscript + kExprNodePlainKey, + /// Complex identifier: variable/function name with curly braces + kExprNodeComplexIdentifier, + /// Figure brace expression which is not yet known + /// + /// May resolve to any of kExprNodeDictLiteral, kExprNodeLambda or + /// kExprNodeCurlyBracesIdentifier. + kExprNodeUnknownFigure, + kExprNodeLambda, ///< Lambda. + kExprNodeDictLiteral, ///< Dictionary literal. + kExprNodeCurlyBracesIdentifier, ///< Part of the curly braces name. + kExprNodeComma, ///< Comma “operator”. + kExprNodeColon, ///< Colon “operator”. + kExprNodeArrow, ///< Arrow “operator”. + kExprNodeComparison, ///< Various comparison operators. + /// Concat operator + /// + /// To be only used in cases when it is known for sure it is not a subscript. + kExprNodeConcat, + /// Concat or subscript operator + /// + /// For cases when it is not obvious whether expression is a concat or + /// a subscript. May only have either number or plain identifier as the second + /// child. To make it easier to avoid curly braces in place of + /// kExprNodePlainIdentifier node kExprNodePlainKey is used. + kExprNodeConcatOrSubscript, + kExprNodeInteger, ///< Integral number. + kExprNodeFloat, ///< Floating-point number. + kExprNodeSingleQuotedString, + kExprNodeDoubleQuotedString, + kExprNodeOr, + kExprNodeAnd, + kExprNodeUnaryMinus, + kExprNodeBinaryMinus, + kExprNodeNot, + kExprNodeMultiplication, + kExprNodeDivision, + kExprNodeMod, + kExprNodeOption, + kExprNodeEnvironment, +} ExprASTNodeType; + +typedef struct expr_ast_node ExprASTNode; + +/// Structure representing one AST node +struct expr_ast_node { + ExprASTNodeType type; ///< Node type. + /// Node children: e.g. for 1 + 2 nodes 1 and 2 will be children of +. + ExprASTNode *children; + /// Next node: e.g. for 1 + 2 child nodes 1 and 2 are put into a single-linked + /// list: `(+)->children` references only node 1, node 2 is in + /// `(+)->children->next`. + ExprASTNode *next; + ParserPosition start; + size_t len; + union { + struct { + int name; ///< Register name, may be -1 if name not present. + } reg; ///< For kExprNodeRegister. + struct { + /// Which nodes UnknownFigure can’t possibly represent. + struct { + /// True if UnknownFigure may actually represent dictionary literal. + bool allow_dict; + /// True if UnknownFigure may actually represent lambda. + bool allow_lambda; + /// True if UnknownFigure may actually be part of curly braces name. + bool allow_ident; + } type_guesses; + /// Highlight chunk index, used for rehighlighting if needed + size_t opening_hl_idx; + } fig; ///< For kExprNodeUnknownFigure. + struct { + ExprVarScope scope; ///< Scope character or 0 if not present. + /// Actual identifier without scope. + /// + /// Points to inside parser reader state. + const char *ident; + size_t ident_len; ///< Actual identifier length. + } var; ///< For kExprNodePlainIdentifier and kExprNodePlainKey. + struct { + bool got_colon; ///< True if colon was seen. + } ter; ///< For kExprNodeTernaryValue. + struct { + ExprComparisonType type; ///< Comparison type. + ExprCaseCompareStrategy ccs; ///< Case comparison strategy. + bool inv; ///< True if comparison is to be inverted. + } cmp; ///< For kExprNodeComparison. + struct { + uvarnumber_T value; + } num; ///< For kExprNodeInteger. + struct { + float_T value; + } flt; ///< For kExprNodeFloat. + struct { + char *value; + size_t size; + } str; ///< For kExprNodeSingleQuotedString and + ///< kExprNodeDoubleQuotedString. + struct { + const char *ident; ///< Option name start. + size_t ident_len; ///< Option name length. + ExprOptScope scope; ///< Option scope: &l:, &g: or not specified. + } opt; ///< For kExprNodeOption. + struct { + const char *ident; ///< Environment variable name start. + size_t ident_len; ///< Environment variable name length. + } env; ///< For kExprNodeEnvironment. + } data; +}; + +enum { + /// Allow multiple expressions in a row: e.g. for :echo + /// + /// Parser will still parse only one of them though. + kExprFlagsMulti = (1 << 0), + /// Allow NL, NUL and bar to be EOC + /// + /// When parsing expressions input by user bar is assumed to be a binary + /// operator and other two are spacings. + kExprFlagsDisallowEOC = (1 << 1), + // WARNING: whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_parser.c. +} ExprParserFlags; + +/// AST error definition +typedef struct { + /// Error message. Must contain a single printf format atom: %.*s. + const char *msg; + /// Error message argument: points to the location of the error. + const char *arg; + /// Message argument length: length till the end of string. + int arg_len; +} ExprASTError; + +/// Structure representing complety AST for one expression +typedef struct { + /// When AST is not correct this message will be printed. + /// + /// Uses `emsgf(msg, arg_len, arg);`, `msg` is assumed to contain only `%.*s`. + ExprASTError err; + /// Root node of the AST. + ExprASTNode *root; +} ExprAST; + +/// Array mapping ExprASTNodeType to maximum amount of children node may have +extern const uint8_t node_maxchildren[]; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/expressions.h.generated.h" +#endif + +#endif // NVIM_VIML_PARSER_EXPRESSIONS_H diff --git a/src/nvim/viml/parser/parser.c b/src/nvim/viml/parser/parser.c new file mode 100644 index 0000000000..08d8846018 --- /dev/null +++ b/src/nvim/viml/parser/parser.c @@ -0,0 +1,13 @@ +#include "nvim/viml/parser/parser.h" + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/parser.c.generated.h" +#endif + + +void parser_simple_get_line(void *cookie, ParserLine *ret_pline) +{ + ParserLine **plines_p = (ParserLine **)cookie; + *ret_pline = **plines_p; + (*plines_p)++; +} diff --git a/src/nvim/viml/parser/parser.h b/src/nvim/viml/parser/parser.h new file mode 100644 index 0000000000..7ac49709d8 --- /dev/null +++ b/src/nvim/viml/parser/parser.h @@ -0,0 +1,244 @@ +#ifndef NVIM_VIML_PARSER_PARSER_H +#define NVIM_VIML_PARSER_PARSER_H + +#include <stdbool.h> +#include <stddef.h> +#include <assert.h> + +#include "nvim/lib/kvec.h" +#include "nvim/func_attr.h" +#include "nvim/mbyte.h" +#include "nvim/memory.h" + +/// One parsed line +typedef struct { + const char *data; ///< Parsed line pointer + size_t size; ///< Parsed line size + bool allocated; ///< True if line may be freed. +} ParserLine; + +/// Line getter type for parser +/// +/// Line getter must return {NULL, 0} for EOF. +typedef void (*ParserLineGetter)(void *cookie, ParserLine *ret_pline); + +/// Parser position in the input +typedef struct { + size_t line; ///< Line index in ParserInputReader.lines. + size_t col; ///< Byte index in the line. +} ParserPosition; + +/// Parser state item. +typedef struct { + enum { + kPTopStateParsingCommand = 0, + kPTopStateParsingExpression, + } type; + union { + struct { + enum { + kExprUnknown = 0, + } type; + } expr; + } data; +} ParserStateItem; + +/// Structure defining input reader +typedef struct { + /// Function used to get next line. + ParserLineGetter get_line; + /// Data for get_line function. + void *cookie; + /// All lines obtained by get_line. + kvec_withinit_t(ParserLine, 4) lines; + /// Conversion, for :scriptencoding. + vimconv_T conv; +} ParserInputReader; + +/// Highlighted region definition +/// +/// Note: one chunk may highlight only one line. +typedef struct { + ParserPosition start; ///< Start of the highlight: line and column. + size_t end_col; ///< End column, points to the start of the next character. + const char *group; ///< Highlight group. +} ParserHighlightChunk; + +/// Highlighting defined by a parser +typedef kvec_withinit_t(ParserHighlightChunk, 16) ParserHighlight; + +/// Structure defining parser state +typedef struct { + /// Line reader. + ParserInputReader reader; + /// Position up to which input was parsed. + ParserPosition pos; + /// Parser state stack. + kvec_withinit_t(ParserStateItem, 16) stack; + /// Highlighting support. + ParserHighlight *colors; + /// True if line continuation can be used. + bool can_continuate; +} ParserState; + +static inline void viml_parser_init( + ParserState *const ret_pstate, + const ParserLineGetter get_line, void *const cookie, + ParserHighlight *const colors) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ARG(1, 2); + +/// Initialize a new parser state instance +/// +/// @param[out] ret_pstate Parser state to initialize. +/// @param[in] get_line Line getter function. +/// @param[in] cookie Argument for the get_line function. +/// @param[in] colors Where to save highlighting. May be NULL if it is not +/// needed. +static inline void viml_parser_init( + ParserState *const ret_pstate, + const ParserLineGetter get_line, void *const cookie, + ParserHighlight *const colors) +{ + *ret_pstate = (ParserState) { + .reader = { + .get_line = get_line, + .cookie = cookie, + .conv = MBYTE_NONE_CONV, + }, + .pos = { 0, 0 }, + .colors = colors, + .can_continuate = false, + }; + kvi_init(ret_pstate->reader.lines); + kvi_init(ret_pstate->stack); +} + +static inline void viml_parser_destroy(ParserState *const pstate) + REAL_FATTR_NONNULL_ALL REAL_FATTR_ALWAYS_INLINE; + +/// Free all memory allocated by the parser on heap +/// +/// @param pstate Parser state to free. +static inline void viml_parser_destroy(ParserState *const pstate) +{ + for (size_t i = 0; i < kv_size(pstate->reader.lines); i++) { + ParserLine pline = kv_A(pstate->reader.lines, i); + if (pline.allocated) { + xfree((void *)pline.data); + } + } + kvi_destroy(pstate->reader.lines); + kvi_destroy(pstate->stack); +} + +static inline void viml_preader_get_line(ParserInputReader *const preader, + ParserLine *const ret_pline) + REAL_FATTR_NONNULL_ALL; + +/// Get one line from ParserInputReader +static inline void viml_preader_get_line(ParserInputReader *const preader, + ParserLine *const ret_pline) +{ + ParserLine pline; + preader->get_line(preader->cookie, &pline); + if (preader->conv.vc_type != CONV_NONE && pline.size) { + ParserLine cpline = { + .allocated = true, + .size = pline.size, + }; + cpline.data = (char *)string_convert(&preader->conv, + (char_u *)pline.data, + &cpline.size); + if (pline.allocated) { + xfree((void *)pline.data); + } + pline = cpline; + } + kvi_push(preader->lines, pline); + *ret_pline = pline; +} + +static inline bool viml_parser_get_remaining_line(ParserState *const pstate, + ParserLine *const ret_pline) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_WARN_UNUSED_RESULT REAL_FATTR_NONNULL_ALL; + +/// Get currently parsed line, shifted to pstate->pos.col +/// +/// @param pstate Parser state to operate on. +/// +/// @return True if there is a line, false in case of EOF. +static inline bool viml_parser_get_remaining_line(ParserState *const pstate, + ParserLine *const ret_pline) +{ + const size_t num_lines = kv_size(pstate->reader.lines); + if (pstate->pos.line == num_lines) { + viml_preader_get_line(&pstate->reader, ret_pline); + } else { + *ret_pline = kv_last(pstate->reader.lines); + } + assert(pstate->pos.line == kv_size(pstate->reader.lines) - 1); + if (ret_pline->data != NULL) { + ret_pline->data += pstate->pos.col; + ret_pline->size -= pstate->pos.col; + } + return ret_pline->data != NULL; +} + +static inline void viml_parser_advance(ParserState *const pstate, + const size_t len) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ALL; + +/// Advance position by a given number of bytes +/// +/// At maximum advances to the next line. +/// +/// @param pstate Parser state to advance. +/// @param[in] len Number of bytes to advance. +static inline void viml_parser_advance(ParserState *const pstate, + const size_t len) +{ + assert(pstate->pos.line == kv_size(pstate->reader.lines) - 1); + const ParserLine pline = kv_last(pstate->reader.lines); + if (pstate->pos.col + len >= pline.size) { + pstate->pos.line++; + pstate->pos.col = 0; + } else { + pstate->pos.col += len; + } +} + +static inline void viml_parser_highlight(ParserState *const pstate, + const ParserPosition start, + const size_t end_col, + const char *const group) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ALL; + +/// Record highlighting of some region of text +/// +/// @param pstate Parser state to work with. +/// @param[in] start Start position of the highlight. +/// @param[in] len Highlighting chunk length. +/// @param[in] group Highlight group. +static inline void viml_parser_highlight(ParserState *const pstate, + const ParserPosition start, + const size_t len, + const char *const group) +{ + if (pstate->colors == NULL || len == 0) { + return; + } + assert(kv_size(*pstate->colors) == 0 + || kv_Z(*pstate->colors, 0).start.line < start.line + || kv_Z(*pstate->colors, 0).end_col <= start.col); + kvi_push(*pstate->colors, ((ParserHighlightChunk) { + .start = start, + .end_col = start.col + len, + .group = group, + })); +} + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/parser.h.generated.h" +#endif + +#endif // NVIM_VIML_PARSER_PARSER_H diff --git a/test/README.md b/test/README.md index 01db5960cd..15041b74e8 100644 --- a/test/README.md +++ b/test/README.md @@ -110,3 +110,6 @@ disables tracing (the fastest, but you get no data if tests crash and there was no core dump generated), `1` or empty/undefined leaves only C function cals and returns in the trace (faster then recording everything), `2` records all function calls, returns and lua source lines exuecuted. + +`NVIM_TEST_TRACE_ON_ERROR` (U) (1): makes unit tests yield trace on error in +addition to regular error message. diff --git a/test/functional/ui/cmdline_highlight_spec.lua b/test/functional/ui/cmdline_highlight_spec.lua index d87ce72599..60a4a815e7 100644 --- a/test/functional/ui/cmdline_highlight_spec.lua +++ b/test/functional/ui/cmdline_highlight_spec.lua @@ -144,7 +144,9 @@ before_each(function() EOB={bold = true, foreground = Screen.colors.Blue1}, ERR={foreground = Screen.colors.Grey100, background = Screen.colors.Red}, SK={foreground = Screen.colors.Blue}, - PE={bold = true, foreground = Screen.colors.SeaGreen4} + PE={bold = true, foreground = Screen.colors.SeaGreen4}, + NUM={foreground = Screen.colors.Blue2}, + NPAR={foreground = Screen.colors.Yellow}, }) end) @@ -863,7 +865,10 @@ describe('Ex commands coloring support', function() end) describe('Expressions coloring support', function() it('works', function() - meths.set_var('Nvim_color_expr', 'RainBowParens') + meths.command('hi clear NVimNumber') + meths.command('hi clear NVimNestingParenthesis') + meths.command('hi NVimNumber guifg=Blue2') + meths.command('hi NVimNestingParenthesis guifg=Yellow') feed(':echo <C-r>=(((1)))') screen:expect([[ | @@ -873,21 +878,24 @@ describe('Expressions coloring support', function() {EOB:~ }| {EOB:~ }| {EOB:~ }| - ={RBP1:(}{RBP2:(}{RBP3:(}1{RBP3:)}{RBP2:)}{RBP1:)}^ | + ={NPAR:(((}{NUM:1}{NPAR:)))}^ | ]]) end) - it('errors out when failing to get callback', function() + it('does not use Nvim_color_expr', function() meths.set_var('Nvim_color_expr', 42) + -- Used to error out due to failing to get callback. + meths.command('hi clear NVimNumber') + meths.command('hi NVimNumber guifg=Blue2') feed(':<C-r>=1') screen:expect([[ + | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| {EOB:~ }| {EOB:~ }| {EOB:~ }| - = | - {ERR:E5409: Unable to get g:Nvim_color_expr c}| - {ERR:allback: Vim:E6000: Argument is not a fu}| - {ERR:nction or function name} | - =1^ | + ={NUM:1}^ | ]]) end) end) diff --git a/test/helpers.lua b/test/helpers.lua index 260f10002e..83e78ba059 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -307,6 +307,93 @@ local function dedent(str, leave_indent) return str end +local SUBTBL = { + '\\000', '\\001', '\\002', '\\003', '\\004', + '\\005', '\\006', '\\007', '\\008', '\\t', + '\\n', '\\011', '\\012', '\\r', '\\014', + '\\015', '\\016', '\\017', '\\018', '\\019', + '\\020', '\\021', '\\022', '\\023', '\\024', + '\\025', '\\026', '\\027', '\\028', '\\029', + '\\030', '\\031', +} + +local format_luav + +format_luav = function(v, indent) + local linesep = '\n' + local next_indent = nil + if indent == nil then + indent = '' + linesep = '' + else + next_indent = indent .. ' ' + end + local ret = '' + if type(v) == 'string' then + ret = tostring(v):gsub('[\'\\]', '\\%0'):gsub('[%z\1-\31]', function(match) + return SUBTBL[match:byte() + 1] + end) + ret = '\'' .. ret .. '\'' + elseif type(v) == 'table' then + local processed_keys = {} + ret = '{' .. linesep + for i, subv in ipairs(v) do + ret = ret .. (next_indent or '') .. format_luav(subv, next_indent) .. ',\n' + processed_keys[i] = true + end + for k, subv in pairs(v) do + if not processed_keys[k] then + if type(k) == 'string' and k:match('^[a-zA-Z_][a-zA-Z0-9_]*$') then + ret = ret .. next_indent .. k .. ' = ' + else + ret = ret .. next_indent .. '[' .. format_luav(k) .. '] = ' + end + ret = ret .. format_luav(subv, next_indent) .. ',\n' + end + end + ret = ret .. indent .. '}' + elseif type(v) == 'number' then + if v % 1 == 0 then + ret = ('%d'):format(v) + else + ret = ('%e'):format(v) + end + elseif type(v) == 'nil' then + ret = 'nil' + else + print(type(v)) + -- Not implemented yet + assert(false) + end + return ret +end + +local function format_string(fmt, ...) + local i = 0 + local args = {...} + local function getarg() + i = i + 1 + return args[i] + end + local ret = fmt:gsub('%%[0-9*]*%.?[0-9*]*[cdEefgGiouXxqsr%%]', function(match) + local subfmt = match:gsub('%*', function(match) + return getarg() + end) + local arg = nil + if subfmt:sub(-1) ~= '%' then + arg = getarg() + end + if subfmt:sub(-1) == 'r' then + -- %r is like %q, but it is supposed to single-quote strings and not + -- double-quote them, and also work not only for strings. + subfmt = subfmt:sub(1, -2) .. 's' + arg = format_luav(arg) + end + return subfmt:format(arg) + end) + return ret +end + return { eq = eq, neq = neq, @@ -323,4 +410,6 @@ return { shallowcopy = shallowcopy, concat_tables = concat_tables, dedent = dedent, + format_luav = format_luav, + format_string = format_string, } diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c new file mode 100644 index 0000000000..f3a218949f --- /dev/null +++ b/test/symbolic/klee/nvim/charset.c @@ -0,0 +1,138 @@ +#include <stdbool.h> + +#include "nvim/ascii.h" +#include "nvim/macros.h" +#include "nvim/charset.h" +#include "nvim/eval/typval.h" +#include "nvim/vim.h" + +bool vim_isIDc(int c) +{ + return ASCII_ISALNUM(c); +} + +int hex2nr(int c) +{ + if ((c >= 'a') && (c <= 'f')) { + return c - 'a' + 10; + } + + if ((c >= 'A') && (c <= 'F')) { + return c - 'A' + 10; + } + return c - '0'; +} + +void vim_str2nr(const char_u *const start, int *const prep, int *const len, + const int what, varnumber_T *const nptr, + uvarnumber_T *const unptr, const int maxlen) +{ + const char *ptr = (const char *)start; +#define STRING_ENDED(ptr) \ + (!(maxlen == 0 || (int)((ptr) - (const char *)start) < maxlen)) + int pre = 0; // default is decimal + bool negative = false; + + if (ptr[0] == '-') { + negative = true; + ptr++; + } + + // Recognize hex, octal and bin. + if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) + && !STRING_ENDED(ptr + 1) + && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { + pre = ptr[1]; + + if ((what & STR2NR_HEX) + && !STRING_ENDED(ptr + 2) + && (pre == 'X' || pre == 'x') + && ascii_isxdigit(ptr[2])) { + // hexadecimal + ptr += 2; + } else if ((what & STR2NR_BIN) + && !STRING_ENDED(ptr + 2) + && (pre == 'B' || pre == 'b') + && ascii_isbdigit(ptr[2])) { + // binary + ptr += 2; + } else { + // decimal or octal, default is decimal + pre = 0; + + if (what & STR2NR_OCT + && !STRING_ENDED(ptr + 1) + && ('0' <= ptr[1] && ptr[1] <= '7')) { + // Assume octal now: what we already know is that string starts with + // zero and some octal digit. + pre = '0'; + // Don’t interpret "0", "008" or "0129" as octal. + for (int i = 2; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { + if (ptr[i] > '7') { + // Can’t be octal. + pre = 0; + break; + } + } + } + } + } + + // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. + uvarnumber_T un = 0; +#define PARSE_NUMBER(base, cond, conv) \ + do { \ + while (!STRING_ENDED(ptr) && (cond)) { \ + /* avoid ubsan error for overflow */ \ + if (un < UVARNUMBER_MAX / base) { \ + un = base * un + (uvarnumber_T)(conv); \ + } else { \ + un = UVARNUMBER_MAX; \ + } \ + ptr++; \ + } \ + } while (0) + if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { + // Binary number. + PARSE_NUMBER(2, (*ptr == '0' || *ptr == '1'), (*ptr - '0')); + } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { + // Octal number. + PARSE_NUMBER(8, ('0' <= *ptr && *ptr <= '7'), (*ptr - '0')); + } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { + // Hexadecimal number. + PARSE_NUMBER(16, (ascii_isxdigit(*ptr)), (hex2nr(*ptr))); + } else { + // Decimal number. + PARSE_NUMBER(10, (ascii_isdigit(*ptr)), (*ptr - '0')); + } +#undef PARSE_NUMBER + + if (prep != NULL) { + *prep = pre; + } + + if (len != NULL) { + *len = (int)(ptr - (const char *)start); + } + + if (nptr != NULL) { + if (negative) { // account for leading '-' for decimal numbers + // avoid ubsan error for overflow + if (un > VARNUMBER_MAX) { + *nptr = VARNUMBER_MIN; + } else { + *nptr = -(varnumber_T)un; + } + } else { + if (un > VARNUMBER_MAX) { + un = VARNUMBER_MAX; + } + *nptr = (varnumber_T)un; + } + } + + if (unptr != NULL) { + *unptr = un; + } +#undef STRING_ENDED +} diff --git a/test/symbolic/klee/nvim/garray.c b/test/symbolic/klee/nvim/garray.c new file mode 100644 index 0000000000..260870c3c2 --- /dev/null +++ b/test/symbolic/klee/nvim/garray.c @@ -0,0 +1,195 @@ +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +/// @file garray.c +/// +/// Functions for handling growing arrays. + +#include <string.h> +#include <inttypes.h> + +#include "nvim/vim.h" +#include "nvim/ascii.h" +#include "nvim/log.h" +#include "nvim/memory.h" +#include "nvim/path.h" +#include "nvim/garray.h" +#include "nvim/strings.h" + +// #include "nvim/globals.h" +#include "nvim/memline.h" + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "garray.c.generated.h" +#endif + +/// Clear an allocated growing array. +void ga_clear(garray_T *gap) +{ + xfree(gap->ga_data); + + // Initialize growing array without resetting itemsize or growsize + gap->ga_data = NULL; + gap->ga_maxlen = 0; + gap->ga_len = 0; +} + +/// Clear a growing array that contains a list of strings. +/// +/// @param gap +void ga_clear_strings(garray_T *gap) +{ + GA_DEEP_CLEAR_PTR(gap); +} + +/// Initialize a growing array. +/// +/// @param gap +/// @param itemsize +/// @param growsize +void ga_init(garray_T *gap, int itemsize, int growsize) +{ + gap->ga_data = NULL; + gap->ga_maxlen = 0; + gap->ga_len = 0; + gap->ga_itemsize = itemsize; + ga_set_growsize(gap, growsize); +} + +/// A setter for the growsize that guarantees it will be at least 1. +/// +/// @param gap +/// @param growsize +void ga_set_growsize(garray_T *gap, int growsize) +{ + if (growsize < 1) { + WLOG("trying to set an invalid ga_growsize: %d", growsize); + gap->ga_growsize = 1; + } else { + gap->ga_growsize = growsize; + } +} + +/// Make room in growing array "gap" for at least "n" items. +/// +/// @param gap +/// @param n +void ga_grow(garray_T *gap, int n) +{ + if (gap->ga_maxlen - gap->ga_len >= n) { + // the garray still has enough space, do nothing + return; + } + + if (gap->ga_growsize < 1) { + WLOG("ga_growsize(%d) is less than 1", gap->ga_growsize); + } + + // the garray grows by at least growsize + if (n < gap->ga_growsize) { + n = gap->ga_growsize; + } + int new_maxlen = gap->ga_len + n; + + size_t new_size = (size_t)(gap->ga_itemsize * new_maxlen); + size_t old_size = (size_t)(gap->ga_itemsize * gap->ga_maxlen); + + // reallocate and clear the new memory + char *pp = xrealloc(gap->ga_data, new_size); + memset(pp + old_size, 0, new_size - old_size); + + gap->ga_maxlen = new_maxlen; + gap->ga_data = pp; +} + +/// For a growing array that contains a list of strings: concatenate all the +/// strings with sep as separator. +/// +/// @param gap +/// @param sep +/// +/// @returns the concatenated strings +char_u *ga_concat_strings_sep(const garray_T *gap, const char *sep) + FUNC_ATTR_NONNULL_RET +{ + const size_t nelem = (size_t) gap->ga_len; + const char **strings = gap->ga_data; + + if (nelem == 0) { + return (char_u *) xstrdup(""); + } + + size_t len = 0; + for (size_t i = 0; i < nelem; i++) { + len += strlen(strings[i]); + } + + // add some space for the (num - 1) separators + len += (nelem - 1) * strlen(sep); + char *const ret = xmallocz(len); + + char *s = ret; + for (size_t i = 0; i < nelem - 1; i++) { + s = xstpcpy(s, strings[i]); + s = xstpcpy(s, sep); + } + strcpy(s, strings[nelem - 1]); + + return (char_u *) ret; +} + +/// For a growing array that contains a list of strings: concatenate all the +/// strings with a separating comma. +/// +/// @param gap +/// +/// @returns the concatenated strings +char_u* ga_concat_strings(const garray_T *gap) FUNC_ATTR_NONNULL_RET +{ + return ga_concat_strings_sep(gap, ","); +} + +/// Concatenate a string to a growarray which contains characters. +/// When "s" is NULL does not do anything. +/// +/// WARNING: +/// - Does NOT copy the NUL at the end! +/// - The parameter may not overlap with the growing array +/// +/// @param gap +/// @param s +void ga_concat(garray_T *gap, const char_u *restrict s) +{ + if (s == NULL) { + return; + } + + ga_concat_len(gap, (const char *restrict) s, strlen((char *) s)); +} + +/// Concatenate a string to a growarray which contains characters +/// +/// @param[out] gap Growarray to modify. +/// @param[in] s String to concatenate. +/// @param[in] len String length. +void ga_concat_len(garray_T *const gap, const char *restrict s, + const size_t len) + FUNC_ATTR_NONNULL_ALL +{ + if (len) { + ga_grow(gap, (int) len); + char *data = gap->ga_data; + memcpy(data + gap->ga_len, s, len); + gap->ga_len += (int) len; + } +} + +/// Append one byte to a growarray which contains bytes. +/// +/// @param gap +/// @param c +void ga_append(garray_T *gap, char c) +{ + GA_APPEND(char, gap, c); +} + diff --git a/test/symbolic/klee/nvim/gettext.c b/test/symbolic/klee/nvim/gettext.c new file mode 100644 index 0000000000..b9cc98d770 --- /dev/null +++ b/test/symbolic/klee/nvim/gettext.c @@ -0,0 +1,4 @@ +char *gettext(const char *s) +{ + return (char *)s; +} diff --git a/test/symbolic/klee/nvim/keymap.c b/test/symbolic/klee/nvim/keymap.c new file mode 100644 index 0000000000..ff5e46e75b --- /dev/null +++ b/test/symbolic/klee/nvim/keymap.c @@ -0,0 +1,538 @@ +#include <stdbool.h> + +#include "nvim/types.h" +#include "nvim/keymap.h" +#include "nvim/eval/typval.h" + +#define MOD_KEYS_ENTRY_SIZE 5 + +static char_u modifier_keys_table[] = +{ + MOD_MASK_SHIFT, '&', '9', '@', '1', + MOD_MASK_SHIFT, '&', '0', '@', '2', + MOD_MASK_SHIFT, '*', '1', '@', '4', + MOD_MASK_SHIFT, '*', '2', '@', '5', + MOD_MASK_SHIFT, '*', '3', '@', '6', + MOD_MASK_SHIFT, '*', '4', 'k', 'D', + MOD_MASK_SHIFT, '*', '5', 'k', 'L', + MOD_MASK_SHIFT, '*', '7', '@', '7', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_END, '@', '7', + MOD_MASK_SHIFT, '*', '9', '@', '9', + MOD_MASK_SHIFT, '*', '0', '@', '0', + MOD_MASK_SHIFT, '#', '1', '%', '1', + MOD_MASK_SHIFT, '#', '2', 'k', 'h', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_HOME, 'k', 'h', + MOD_MASK_SHIFT, '#', '3', 'k', 'I', + MOD_MASK_SHIFT, '#', '4', 'k', 'l', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_LEFT, 'k', 'l', + MOD_MASK_SHIFT, '%', 'a', '%', '3', + MOD_MASK_SHIFT, '%', 'b', '%', '4', + MOD_MASK_SHIFT, '%', 'c', '%', '5', + MOD_MASK_SHIFT, '%', 'd', '%', '7', + MOD_MASK_SHIFT, '%', 'e', '%', '8', + MOD_MASK_SHIFT, '%', 'f', '%', '9', + MOD_MASK_SHIFT, '%', 'g', '%', '0', + MOD_MASK_SHIFT, '%', 'h', '&', '3', + MOD_MASK_SHIFT, '%', 'i', 'k', 'r', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_RIGHT, 'k', 'r', + MOD_MASK_SHIFT, '%', 'j', '&', '5', + MOD_MASK_SHIFT, '!', '1', '&', '6', + MOD_MASK_SHIFT, '!', '2', '&', '7', + MOD_MASK_SHIFT, '!', '3', '&', '8', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_UP, 'k', 'u', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_DOWN, 'k', 'd', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF1, KS_EXTRA, (int)KE_XF1, + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF2, KS_EXTRA, (int)KE_XF2, + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF3, KS_EXTRA, (int)KE_XF3, + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF4, KS_EXTRA, (int)KE_XF4, + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F1, 'k', '1', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F2, 'k', '2', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F3, 'k', '3', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F4, 'k', '4', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F5, 'k', '5', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F6, 'k', '6', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F7, 'k', '7', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F8, 'k', '8', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F9, 'k', '9', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F10, 'k', ';', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F11, 'F', '1', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F12, 'F', '2', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F13, 'F', '3', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F14, 'F', '4', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F15, 'F', '5', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F16, 'F', '6', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F17, 'F', '7', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F18, 'F', '8', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F19, 'F', '9', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F20, 'F', 'A', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F21, 'F', 'B', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F22, 'F', 'C', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F23, 'F', 'D', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F24, 'F', 'E', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F25, 'F', 'F', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F26, 'F', 'G', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F27, 'F', 'H', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F28, 'F', 'I', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F29, 'F', 'J', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F30, 'F', 'K', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F31, 'F', 'L', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F32, 'F', 'M', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F33, 'F', 'N', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F34, 'F', 'O', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F35, 'F', 'P', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F36, 'F', 'Q', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F37, 'F', 'R', + + MOD_MASK_SHIFT, 'k', 'B', KS_EXTRA, (int)KE_TAB, + + NUL +}; + +int simplify_key(const int key, int *modifiers) +{ + if (*modifiers & (MOD_MASK_SHIFT | MOD_MASK_CTRL | MOD_MASK_ALT)) { + // TAB is a special case. + if (key == TAB && (*modifiers & MOD_MASK_SHIFT)) { + *modifiers &= ~MOD_MASK_SHIFT; + return K_S_TAB; + } + const int key0 = KEY2TERMCAP0(key); + const int key1 = KEY2TERMCAP1(key); + for (int i = 0; modifier_keys_table[i] != NUL; i += MOD_KEYS_ENTRY_SIZE) { + if (key0 == modifier_keys_table[i + 3] + && key1 == modifier_keys_table[i + 4] + && (*modifiers & modifier_keys_table[i])) { + *modifiers &= ~modifier_keys_table[i]; + return TERMCAP2KEY(modifier_keys_table[i + 1], + modifier_keys_table[i + 2]); + } + } + } + return key; +} + +int handle_x_keys(const int key) + FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + switch (key) { + case K_XUP: return K_UP; + case K_XDOWN: return K_DOWN; + case K_XLEFT: return K_LEFT; + case K_XRIGHT: return K_RIGHT; + case K_XHOME: return K_HOME; + case K_ZHOME: return K_HOME; + case K_XEND: return K_END; + case K_ZEND: return K_END; + case K_XF1: return K_F1; + case K_XF2: return K_F2; + case K_XF3: return K_F3; + case K_XF4: return K_F4; + case K_S_XF1: return K_S_F1; + case K_S_XF2: return K_S_F2; + case K_S_XF3: return K_S_F3; + case K_S_XF4: return K_S_F4; + } + return key; +} + +static const struct key_name_entry { + int key; // Special key code or ascii value + const char *name; // Name of key +} key_names_table[] = { + { ' ', "Space" }, + { TAB, "Tab" }, + { K_TAB, "Tab" }, + { NL, "NL" }, + { NL, "NewLine" }, // Alternative name + { NL, "LineFeed" }, // Alternative name + { NL, "LF" }, // Alternative name + { CAR, "CR" }, + { CAR, "Return" }, // Alternative name + { CAR, "Enter" }, // Alternative name + { K_BS, "BS" }, + { K_BS, "BackSpace" }, // Alternative name + { ESC, "Esc" }, + { CSI, "CSI" }, + { K_CSI, "xCSI" }, + { '|', "Bar" }, + { '\\', "Bslash" }, + { K_DEL, "Del" }, + { K_DEL, "Delete" }, // Alternative name + { K_KDEL, "kDel" }, + { K_UP, "Up" }, + { K_DOWN, "Down" }, + { K_LEFT, "Left" }, + { K_RIGHT, "Right" }, + { K_XUP, "xUp" }, + { K_XDOWN, "xDown" }, + { K_XLEFT, "xLeft" }, + { K_XRIGHT, "xRight" }, + + { K_F1, "F1" }, + { K_F2, "F2" }, + { K_F3, "F3" }, + { K_F4, "F4" }, + { K_F5, "F5" }, + { K_F6, "F6" }, + { K_F7, "F7" }, + { K_F8, "F8" }, + { K_F9, "F9" }, + { K_F10, "F10" }, + + { K_F11, "F11" }, + { K_F12, "F12" }, + { K_F13, "F13" }, + { K_F14, "F14" }, + { K_F15, "F15" }, + { K_F16, "F16" }, + { K_F17, "F17" }, + { K_F18, "F18" }, + { K_F19, "F19" }, + { K_F20, "F20" }, + + { K_F21, "F21" }, + { K_F22, "F22" }, + { K_F23, "F23" }, + { K_F24, "F24" }, + { K_F25, "F25" }, + { K_F26, "F26" }, + { K_F27, "F27" }, + { K_F28, "F28" }, + { K_F29, "F29" }, + { K_F30, "F30" }, + + { K_F31, "F31" }, + { K_F32, "F32" }, + { K_F33, "F33" }, + { K_F34, "F34" }, + { K_F35, "F35" }, + { K_F36, "F36" }, + { K_F37, "F37" }, + + { K_XF1, "xF1" }, + { K_XF2, "xF2" }, + { K_XF3, "xF3" }, + { K_XF4, "xF4" }, + + { K_HELP, "Help" }, + { K_UNDO, "Undo" }, + { K_INS, "Insert" }, + { K_INS, "Ins" }, // Alternative name + { K_KINS, "kInsert" }, + { K_HOME, "Home" }, + { K_KHOME, "kHome" }, + { K_XHOME, "xHome" }, + { K_ZHOME, "zHome" }, + { K_END, "End" }, + { K_KEND, "kEnd" }, + { K_XEND, "xEnd" }, + { K_ZEND, "zEnd" }, + { K_PAGEUP, "PageUp" }, + { K_PAGEDOWN, "PageDown" }, + { K_KPAGEUP, "kPageUp" }, + { K_KPAGEDOWN, "kPageDown" }, + + { K_KPLUS, "kPlus" }, + { K_KMINUS, "kMinus" }, + { K_KDIVIDE, "kDivide" }, + { K_KMULTIPLY, "kMultiply" }, + { K_KENTER, "kEnter" }, + { K_KPOINT, "kPoint" }, + + { K_K0, "k0" }, + { K_K1, "k1" }, + { K_K2, "k2" }, + { K_K3, "k3" }, + { K_K4, "k4" }, + { K_K5, "k5" }, + { K_K6, "k6" }, + { K_K7, "k7" }, + { K_K8, "k8" }, + { K_K9, "k9" }, + + { '<', "lt" }, + + { K_MOUSE, "Mouse" }, + { K_LEFTMOUSE, "LeftMouse" }, + { K_LEFTMOUSE_NM, "LeftMouseNM" }, + { K_LEFTDRAG, "LeftDrag" }, + { K_LEFTRELEASE, "LeftRelease" }, + { K_LEFTRELEASE_NM, "LeftReleaseNM" }, + { K_MIDDLEMOUSE, "MiddleMouse" }, + { K_MIDDLEDRAG, "MiddleDrag" }, + { K_MIDDLERELEASE, "MiddleRelease" }, + { K_RIGHTMOUSE, "RightMouse" }, + { K_RIGHTDRAG, "RightDrag" }, + { K_RIGHTRELEASE, "RightRelease" }, + { K_MOUSEDOWN, "ScrollWheelUp" }, + { K_MOUSEUP, "ScrollWheelDown" }, + { K_MOUSELEFT, "ScrollWheelRight" }, + { K_MOUSERIGHT, "ScrollWheelLeft" }, + { K_MOUSEDOWN, "MouseDown" }, // OBSOLETE: Use + { K_MOUSEUP, "MouseUp" }, // ScrollWheelXXX instead + { K_X1MOUSE, "X1Mouse" }, + { K_X1DRAG, "X1Drag" }, + { K_X1RELEASE, "X1Release" }, + { K_X2MOUSE, "X2Mouse" }, + { K_X2DRAG, "X2Drag" }, + { K_X2RELEASE, "X2Release" }, + { K_DROP, "Drop" }, + { K_ZERO, "Nul" }, + { K_SNR, "SNR" }, + { K_PLUG, "Plug" }, + { K_PASTE, "Paste" }, + { 0, NULL } +}; + +int get_special_key_code(const char_u *name) +{ + for (int i = 0; key_names_table[i].name != NULL; i++) { + const char *const table_name = key_names_table[i].name; + int j; + for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) { + if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) { + break; + } + } + if (!vim_isIDc(name[j]) && table_name[j] == NUL) { + return key_names_table[i].key; + } + } + + return 0; +} + + +static const struct modmasktable { + short mod_mask; ///< Bit-mask for particular key modifier. + short mod_flag; ///< Bit(s) for particular key modifier. + char_u name; ///< Single letter name of modifier. +} mod_mask_table[] = { + {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'M'}, + {MOD_MASK_META, MOD_MASK_META, (char_u)'T'}, + {MOD_MASK_CTRL, MOD_MASK_CTRL, (char_u)'C'}, + {MOD_MASK_SHIFT, MOD_MASK_SHIFT, (char_u)'S'}, + {MOD_MASK_MULTI_CLICK, MOD_MASK_2CLICK, (char_u)'2'}, + {MOD_MASK_MULTI_CLICK, MOD_MASK_3CLICK, (char_u)'3'}, + {MOD_MASK_MULTI_CLICK, MOD_MASK_4CLICK, (char_u)'4'}, + {MOD_MASK_CMD, MOD_MASK_CMD, (char_u)'D'}, + // 'A' must be the last one + {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'A'}, + {0, 0, NUL} +}; + +int name_to_mod_mask(int c) +{ + c = TOUPPER_ASC(c); + for (size_t i = 0; mod_mask_table[i].mod_mask != 0; i++) { + if (c == mod_mask_table[i].name) { + return mod_mask_table[i].mod_flag; + } + } + return 0; +} + +static int extract_modifiers(int key, int *modp) +{ + int modifiers = *modp; + + if (!(modifiers & MOD_MASK_CMD)) { // Command-key is special + if ((modifiers & MOD_MASK_SHIFT) && ASCII_ISALPHA(key)) { + key = TOUPPER_ASC(key); + modifiers &= ~MOD_MASK_SHIFT; + } + } + if ((modifiers & MOD_MASK_CTRL) + && ((key >= '?' && key <= '_') || ASCII_ISALPHA(key))) { + key = Ctrl_chr(key); + modifiers &= ~MOD_MASK_CTRL; + if (key == 0) { // <C-@> is <Nul> + key = K_ZERO; + } + } + + *modp = modifiers; + return key; +} + +int find_special_key(const char_u **srcp, const size_t src_len, int *const modp, + const bool keycode, const bool keep_x_key, + const bool in_string) +{ + const char_u *last_dash; + const char_u *end_of_name; + const char_u *src; + const char_u *bp; + const char_u *const end = *srcp + src_len - 1; + int modifiers; + int bit; + int key; + uvarnumber_T n; + int l; + + if (src_len == 0) { + return 0; + } + + src = *srcp; + if (src[0] != '<') { + return 0; + } + + // Find end of modifier list + last_dash = src; + for (bp = src + 1; bp <= end && (*bp == '-' || vim_isIDc(*bp)); bp++) { + if (*bp == '-') { + last_dash = bp; + if (bp + 1 <= end) { + l = utfc_ptr2len_len(bp + 1, (int)(end - bp) + 1); + // Anything accepted, like <C-?>. + // <C-"> or <M-"> are not special in strings as " is + // the string delimiter. With a backslash it works: <M-\"> + if (end - bp > l && !(in_string && bp[1] == '"') && bp[2] == '>') { + bp += l; + } else if (end - bp > 2 && in_string && bp[1] == '\\' + && bp[2] == '"' && bp[3] == '>') { + bp += 2; + } + } + } + if (end - bp > 3 && bp[0] == 't' && bp[1] == '_') { + bp += 3; // skip t_xx, xx may be '-' or '>' + } else if (end - bp > 4 && STRNICMP(bp, "char-", 5) == 0) { + vim_str2nr(bp + 5, NULL, &l, STR2NR_ALL, NULL, NULL, 0); + bp += l + 5; + break; + } + } + + if (bp <= end && *bp == '>') { // found matching '>' + end_of_name = bp + 1; + + /* Which modifiers are given? */ + modifiers = 0x0; + for (bp = src + 1; bp < last_dash; bp++) { + if (*bp != '-') { + bit = name_to_mod_mask(*bp); + if (bit == 0x0) { + break; // Illegal modifier name + } + modifiers |= bit; + } + } + + // Legal modifier name. + if (bp >= last_dash) { + if (STRNICMP(last_dash + 1, "char-", 5) == 0 + && ascii_isdigit(last_dash[6])) { + // <Char-123> or <Char-033> or <Char-0x33> + vim_str2nr(last_dash + 6, NULL, NULL, STR2NR_ALL, NULL, &n, 0); + key = (int)n; + } else { + int off = 1; + + // Modifier with single letter, or special key name. + if (in_string && last_dash[1] == '\\' && last_dash[2] == '"') { + off = 2; + } + l = mb_ptr2len(last_dash + 1); + if (modifiers != 0 && last_dash[l + 1] == '>') { + key = PTR2CHAR(last_dash + off); + } else { + key = get_special_key_code(last_dash + off); + if (!keep_x_key) { + key = handle_x_keys(key); + } + } + } + + // get_special_key_code() may return NUL for invalid + // special key name. + if (key != NUL) { + // Only use a modifier when there is no special key code that + // includes the modifier. + key = simplify_key(key, &modifiers); + + if (!keycode) { + // don't want keycode, use single byte code + if (key == K_BS) { + key = BS; + } else if (key == K_DEL || key == K_KDEL) { + key = DEL; + } + } + + // Normal Key with modifier: + // Try to make a single byte code (except for Alt/Meta modifiers). + if (!IS_SPECIAL(key)) { + key = extract_modifiers(key, &modifiers); + } + + *modp = modifiers; + *srcp = end_of_name; + return key; + } + } + } + return 0; +} + +char_u *add_char2buf(int c, char_u *s) +{ + char_u temp[MB_MAXBYTES + 1]; + const int len = utf_char2bytes(c, temp); + for (int i = 0; i < len; ++i) { + c = temp[i]; + // Need to escape K_SPECIAL and CSI like in the typeahead buffer. + if (c == K_SPECIAL) { + *s++ = K_SPECIAL; + *s++ = KS_SPECIAL; + *s++ = KE_FILLER; + } else { + *s++ = c; + } + } + return s; +} + +unsigned int trans_special(const char_u **srcp, const size_t src_len, + char_u *const dst, const bool keycode, + const bool in_string) +{ + int modifiers = 0; + int key; + unsigned int dlen = 0; + + key = find_special_key(srcp, src_len, &modifiers, keycode, false, in_string); + if (key == 0) { + return 0; + } + + // Put the appropriate modifier in a string. + if (modifiers != 0) { + dst[dlen++] = K_SPECIAL; + dst[dlen++] = KS_MODIFIER; + dst[dlen++] = (char_u)modifiers; + } + + if (IS_SPECIAL(key)) { + dst[dlen++] = K_SPECIAL; + dst[dlen++] = (char_u)KEY2TERMCAP0(key); + dst[dlen++] = KEY2TERMCAP1(key); + } else if (has_mbyte && !keycode) { + dlen += (unsigned int)(*mb_char2bytes)(key, dst + dlen); + } else if (keycode) { + char_u *after = add_char2buf(key, dst + dlen); + assert(after >= dst && (uintmax_t)(after - dst) <= UINT_MAX); + dlen = (unsigned int)(after - dst); + } else { + dst[dlen++] = (char_u)key; + } + + return dlen; +} diff --git a/test/symbolic/klee/nvim/mbyte.c b/test/symbolic/klee/nvim/mbyte.c new file mode 100644 index 0000000000..f98a531206 --- /dev/null +++ b/test/symbolic/klee/nvim/mbyte.c @@ -0,0 +1,266 @@ +#include <stddef.h> +#include <inttypes.h> +#include <assert.h> +#include <stdbool.h> + +#include "nvim/types.h" +#include "nvim/mbyte.h" +#include "nvim/ascii.h" + +const uint8_t utf8len_tab_zero[] = { + //1 2 3 4 5 6 7 8 9 A B C D E F 0 1 2 3 4 5 6 7 8 9 A B C D E F + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 2 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 4 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 6 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 8 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // A + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // C + 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0, // E +}; + +const uint8_t utf8len_tab[] = { + // ?1 ?2 ?3 ?4 ?5 ?6 ?7 ?8 ?9 ?A ?B ?C ?D ?E ?F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D? + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E? + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, // F? +}; + +int utf_ptr2char(const char_u *const p) +{ + if (p[0] < 0x80) { // Be quick for ASCII. + return p[0]; + } + + const uint8_t len = utf8len_tab_zero[p[0]]; + if (len > 1 && (p[1] & 0xc0) == 0x80) { + if (len == 2) { + return ((p[0] & 0x1f) << 6) + (p[1] & 0x3f); + } + if ((p[2] & 0xc0) == 0x80) { + if (len == 3) { + return (((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + + (p[2] & 0x3f)); + } + if ((p[3] & 0xc0) == 0x80) { + if (len == 4) { + return (((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12) + + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f)); + } + if ((p[4] & 0xc0) == 0x80) { + if (len == 5) { + return (((p[0] & 0x03) << 24) + ((p[1] & 0x3f) << 18) + + ((p[2] & 0x3f) << 12) + ((p[3] & 0x3f) << 6) + + (p[4] & 0x3f)); + } + if ((p[5] & 0xc0) == 0x80 && len == 6) { + return (((p[0] & 0x01) << 30) + ((p[1] & 0x3f) << 24) + + ((p[2] & 0x3f) << 18) + ((p[3] & 0x3f) << 12) + + ((p[4] & 0x3f) << 6) + (p[5] & 0x3f)); + } + } + } + } + } + // Illegal value: just return the first byte. + return p[0]; +} + +bool utf_composinglike(const char_u *p1, const char_u *p2) +{ + return false; +} + +char_u *string_convert(const vimconv_T *conv, char_u *data, size_t *size) +{ + return NULL; +} + +int utf_ptr2len_len(const char_u *p, int size) +{ + int len; + int i; + int m; + + len = utf8len_tab[*p]; + if (len == 1) + return 1; /* NUL, ascii or illegal lead byte */ + if (len > size) + m = size; /* incomplete byte sequence. */ + else + m = len; + for (i = 1; i < m; ++i) + if ((p[i] & 0xc0) != 0x80) + return 1; + return len; +} + +int utfc_ptr2len_len(const char_u *p, int size) +{ + int len; + int prevlen; + + if (size < 1 || *p == NUL) + return 0; + if (p[0] < 0x80 && (size == 1 || p[1] < 0x80)) /* be quick for ASCII */ + return 1; + + /* Skip over first UTF-8 char, stopping at a NUL byte. */ + len = utf_ptr2len_len(p, size); + + /* Check for illegal byte and incomplete byte sequence. */ + if ((len == 1 && p[0] >= 0x80) || len > size) + return 1; + + /* + * Check for composing characters. We can handle only the first six, but + * skip all of them (otherwise the cursor would get stuck). + */ + prevlen = 0; + while (len < size) { + int len_next_char; + + if (p[len] < 0x80) + break; + + /* + * Next character length should not go beyond size to ensure that + * UTF_COMPOSINGLIKE(...) does not read beyond size. + */ + len_next_char = utf_ptr2len_len(p + len, size - len); + if (len_next_char > size - len) + break; + + if (!UTF_COMPOSINGLIKE(p + prevlen, p + len)) + break; + + /* Skip over composing char */ + prevlen = len; + len += len_next_char; + } + return len; +} + +int utf_char2len(const int c) +{ + if (c < 0x80) { + return 1; + } else if (c < 0x800) { + return 2; + } else if (c < 0x10000) { + return 3; + } else if (c < 0x200000) { + return 4; + } else if (c < 0x4000000) { + return 5; + } else { + return 6; + } +} + +int utf_char2bytes(const int c, char_u *const buf) +{ + if (c < 0x80) { // 7 bits + buf[0] = c; + return 1; + } else if (c < 0x800) { // 11 bits + buf[0] = 0xc0 + ((unsigned)c >> 6); + buf[1] = 0x80 + (c & 0x3f); + return 2; + } else if (c < 0x10000) { // 16 bits + buf[0] = 0xe0 + ((unsigned)c >> 12); + buf[1] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[2] = 0x80 + (c & 0x3f); + return 3; + } else if (c < 0x200000) { // 21 bits + buf[0] = 0xf0 + ((unsigned)c >> 18); + buf[1] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[3] = 0x80 + (c & 0x3f); + return 4; + } else if (c < 0x4000000) { // 26 bits + buf[0] = 0xf8 + ((unsigned)c >> 24); + buf[1] = 0x80 + (((unsigned)c >> 18) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[3] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[4] = 0x80 + (c & 0x3f); + return 5; + } else { // 31 bits + buf[0] = 0xfc + ((unsigned)c >> 30); + buf[1] = 0x80 + (((unsigned)c >> 24) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 18) & 0x3f); + buf[3] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[4] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[5] = 0x80 + (c & 0x3f); + return 6; + } +} + +int utf_ptr2len(const char_u *const p) +{ + if (*p == NUL) { + return 0; + } + const int len = utf8len_tab[*p]; + for (int i = 1; i < len; i++) { + if ((p[i] & 0xc0) != 0x80) { + return 1; + } + } + return len; +} + +int utfc_ptr2len(const char_u *const p) +{ + uint8_t b0 = (uint8_t)(*p); + + if (b0 == NUL) { + return 0; + } + if (b0 < 0x80 && p[1] < 0x80) { // be quick for ASCII + return 1; + } + + // Skip over first UTF-8 char, stopping at a NUL byte. + int len = utf_ptr2len(p); + + // Check for illegal byte. + if (len == 1 && b0 >= 0x80) { + return 1; + } + + // Check for composing characters. We can handle only the first six, but + // skip all of them (otherwise the cursor would get stuck). + int prevlen = 0; + for (;;) { + if (p[len] < 0x80 || !UTF_COMPOSINGLIKE(p + prevlen, p + len)) { + return len; + } + + // Skip over composing char. + prevlen = len; + len += utf_ptr2len(p + len); + } +} + +void mb_copy_char(const char_u **fp, char_u **tp) +{ + const size_t l = utfc_ptr2len(*fp); + + memmove(*tp, *fp, (size_t)l); + *tp += l; + *fp += l; +} diff --git a/test/symbolic/klee/nvim/memory.c b/test/symbolic/klee/nvim/memory.c new file mode 100644 index 0000000000..df422cea3e --- /dev/null +++ b/test/symbolic/klee/nvim/memory.c @@ -0,0 +1,101 @@ +#include <stdlib.h> +#include <assert.h> + +#include "nvim/lib/ringbuf.h" + +enum { RB_SIZE = 1024 }; + +typedef struct { + void *ptr; + size_t size; +} AllocRecord; + +RINGBUF_TYPEDEF(AllocRecords, AllocRecord) +RINGBUF_INIT(AllocRecords, arecs, AllocRecord, RINGBUF_DUMMY_FREE) +RINGBUF_STATIC(static, AllocRecords, AllocRecord, arecs, RB_SIZE) + +size_t allocated_memory = 0; +size_t ever_allocated_memory = 0; + +size_t allocated_memory_limit = SIZE_MAX; + +void *xmalloc(const size_t size) +{ + void *ret = malloc(size); + allocated_memory += size; + ever_allocated_memory += size; + assert(allocated_memory <= allocated_memory_limit); + assert(arecs_rb_length(&arecs) < RB_SIZE); + arecs_rb_push(&arecs, (AllocRecord) { + .ptr = ret, + .size = size, + }); + return ret; +} + +void xfree(void *const p) +{ + if (p == NULL) { + return; + } + RINGBUF_FORALL(&arecs, AllocRecord, arec) { + if (arec->ptr == p) { + allocated_memory -= arec->size; + arecs_rb_remove(&arecs, arecs_rb_find_idx(&arecs, arec)); + return; + } + } + assert(false); +} + +void *xrealloc(void *const p, size_t new_size) +{ + void *ret = realloc(p, new_size); + RINGBUF_FORALL(&arecs, AllocRecord, arec) { + if (arec->ptr == p) { + allocated_memory -= arec->size; + allocated_memory += new_size; + if (new_size > arec->size) { + ever_allocated_memory += (new_size - arec->size); + } + arec->ptr = ret; + arec->size = new_size; + return ret; + } + } + assert(false); + return (void *)(intptr_t)1; +} + +char *xstrdup(const char *str) + FUNC_ATTR_MALLOC FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_RET + FUNC_ATTR_NONNULL_ALL +{ + return xmemdupz(str, strlen(str)); +} + +void *xmallocz(size_t size) + FUNC_ATTR_MALLOC FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT +{ + size_t total_size = size + 1; + assert(total_size > size); + + void *ret = xmalloc(total_size); + ((char *)ret)[size] = 0; + + return ret; +} + +char *xstpcpy(char *restrict dst, const char *restrict src) + FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL +{ + const size_t len = strlen(src); + return (char *)memcpy(dst, src, len + 1) + len; +} + +void *xmemdupz(const void *data, size_t len) + FUNC_ATTR_MALLOC FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT + FUNC_ATTR_NONNULL_ALL +{ + return memcpy(xmallocz(len), data, len); +} diff --git a/test/symbolic/klee/run.sh b/test/symbolic/klee/run.sh new file mode 100755 index 0000000000..0234a935b5 --- /dev/null +++ b/test/symbolic/klee/run.sh @@ -0,0 +1,102 @@ +#!/bin/sh + +set -e +set -x +test -z "$POSH_VERSION" && set -u + +PROJECT_SOURCE_DIR=. +PROJECT_BINARY_DIR="$PROJECT_SOURCE_DIR/build" +KLEE_TEST_DIR="$PROJECT_SOURCE_DIR/test/symbolic/klee" +KLEE_BIN_DIR="$PROJECT_BINARY_DIR/klee" +KLEE_OUT_DIR="$KLEE_BIN_DIR/out" + +help() { + echo "Usage:" + echo + echo " $0 -c fname" + echo " $0 fname" + echo " $0 -s" + echo + echo "First form compiles executable out of test/symbolic/klee/{fname}.c." + echo "Compiled executable is placed into build/klee/{fname}. Must first" + echo "successfully compile Neovim in order to generate declarations." + echo + echo "Second form runs KLEE in a docker container using file " + echo "test/symbolic/klee/{fname.c}. Bitcode is placed into build/klee/a.bc," + echo "results are placed into build/klee/out/. The latter is first deleted if" + echo "it exists." + echo + echo "Third form runs ktest-tool which prints errors found by KLEE via " + echo "the same container used to run KLEE." +} + +main() { + local compile= + local print_errs= + if test "$1" = "--help" ; then + help + return + fi + if test "$1" = "-s" ; then + print_errs=1 + shift + elif test "$1" = "-c" ; then + compile=1 + shift + fi + if test -z "$print_errs" ; then + local test="$1" ; shift + fi + + local includes= + includes="$includes -I$KLEE_TEST_DIR" + includes="$includes -I/home/klee/klee_src/include" + includes="$includes -I$PROJECT_SOURCE_DIR/src" + includes="$includes -I$PROJECT_BINARY_DIR/src/nvim/auto" + includes="$includes -I$PROJECT_BINARY_DIR/include" + includes="$includes -I$PROJECT_BINARY_DIR/config" + includes="$includes -I/host-includes" + + local defines= + defines="$defines -DMIN_LOG_LEVEL=9999" + defines="$defines -DINCLUDE_GENERATED_DECLARATIONS" + + test -z "$compile" && defines="$defines -DUSE_KLEE" + + test -d "$KLEE_BIN_DIR" || mkdir -p "$KLEE_BIN_DIR" + + if test -z "$compile" ; then + local line1='cd /image' + if test -z "$print_errs" ; then + test -d "$KLEE_OUT_DIR" && rm -r "$KLEE_OUT_DIR" + + line1="$line1 && $(echo clang \ + $includes $defines \ + -o "$KLEE_BIN_DIR/a.bc" -emit-llvm -g -c \ + "$KLEE_TEST_DIR/$test.c")" + line1="$line1 && klee --libc=uclibc --posix-runtime " + line1="$line1 '--output-dir=$KLEE_OUT_DIR' '$KLEE_BIN_DIR/a.bc'" + fi + local line2="for t in '$KLEE_OUT_DIR'/*.err" + line2="$line2 ; do ktest-tool --write-ints" + line2="$line2 \"\$(printf '%s' \"\$t\" | sed -e 's@\\.[^/]*\$@.ktest@')\"" + line2="$line2 ; done" + printf '%s\n%s\n' "$line1" "$line2" | \ + docker run \ + --volume "$(cd "$PROJECT_SOURCE_DIR" && pwd)":/image \ + --volume "/usr/include":/host-includes \ + --interactive \ + --rm \ + --ulimit='stack=-1:-1' \ + klee/klee \ + /bin/sh -x + else + clang \ + $includes $defines \ + -o "$KLEE_BIN_DIR/$test" \ + -O0 -g \ + "$KLEE_TEST_DIR/$test.c" + fi +} + +main "$@" diff --git a/test/symbolic/klee/viml_expressions_lexer.c b/test/symbolic/klee/viml_expressions_lexer.c new file mode 100644 index 0000000000..ee7dc312e9 --- /dev/null +++ b/test/symbolic/klee/viml_expressions_lexer.c @@ -0,0 +1,105 @@ +#ifdef USE_KLEE +# include <klee/klee.h> +#else +# include <string.h> +# include <stdio.h> +#endif +#include <stddef.h> +#include <stdint.h> +#include <assert.h> + +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/mbyte.h" + +#include "nvim/memory.c" +#include "nvim/mbyte.c" +#include "nvim/charset.c" +#include "nvim/garray.c" +#include "nvim/gettext.c" +#include "nvim/keymap.c" +#include "nvim/viml/parser/expressions.c" + +#define INPUT_SIZE 7 + +uint8_t avoid_optimizing_out; + +void simple_get_line(void *cookie, ParserLine *ret_pline) +{ + ParserLine **plines_p = (ParserLine **)cookie; + *ret_pline = **plines_p; + (*plines_p)++; +} + +int main(const int argc, const char *const *const argv, + const char *const *const environ) +{ + char input[INPUT_SIZE]; + uint8_t shift; + int flags; + avoid_optimizing_out = argc; + +#ifndef USE_KLEE + sscanf(argv[2], "%d", &flags); +#endif + +#ifdef USE_KLEE + klee_make_symbolic(input, sizeof(input), "input"); + klee_make_symbolic(&shift, sizeof(shift), "shift"); + klee_make_symbolic(&flags, sizeof(flags), "flags"); + klee_assume(shift < INPUT_SIZE); + klee_assume(flags <= (kELFlagPeek|kELFlagAllowFloat|kELFlagForbidEOC + |kELFlagForbidScope|kELFlagIsNotCmp)); +#endif + + ParserLine plines[] = { + { +#ifdef USE_KLEE + .data = &input[shift], + .size = sizeof(input) - shift, +#else + .data = (const char *)argv[1], + .size = strlen(argv[1]), +#endif + .allocated = false, + }, + { + .data = NULL, + .size = 0, + .allocated = false, + }, + }; +#ifdef USE_KLEE + assert(plines[0].size <= INPUT_SIZE); + assert((plines[0].data[0] != 5) | (plines[0].data[0] != argc)); +#endif + ParserLine *cur_pline = &plines[0]; + + ParserState pstate = { + .reader = { + .get_line = simple_get_line, + .cookie = &cur_pline, + .lines = KV_INITIAL_VALUE, + .conv.vc_type = CONV_NONE, + }, + .pos = { 0, 0 }, + .colors = NULL, + .can_continuate = false, + }; + kvi_init(pstate.reader.lines); + + allocated_memory_limit = 0; + LexExprToken token = viml_pexpr_next_token(&pstate, flags); + if (flags & kELFlagPeek) { + assert(pstate.pos.line == 0 && pstate.pos.col == 0); + } else { + assert((pstate.pos.line == 0) + ? (pstate.pos.col > 0) + : (pstate.pos.line == 1 && pstate.pos.col == 0)); + } + assert(allocated_memory == 0); + assert(ever_allocated_memory == 0); +#ifndef USE_KLEE + fprintf(stderr, "tkn: %s\n", viml_pexpr_repr_token(&pstate, token, NULL)); +#endif +} diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c new file mode 100644 index 0000000000..9448937430 --- /dev/null +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -0,0 +1,102 @@ +#ifdef USE_KLEE +# include <klee/klee.h> +#else +# include <string.h> +#endif +#include <stddef.h> +#include <stdint.h> +#include <assert.h> + +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/mbyte.h" + +#include "nvim/memory.c" +#include "nvim/mbyte.c" +#include "nvim/charset.c" +#include "nvim/garray.c" +#include "nvim/gettext.c" +#include "nvim/viml/parser/expressions.c" +#include "nvim/keymap.c" + +#define INPUT_SIZE 50 + +uint8_t avoid_optimizing_out; + +void simple_get_line(void *cookie, ParserLine *ret_pline) +{ + ParserLine **plines_p = (ParserLine **)cookie; + *ret_pline = **plines_p; + (*plines_p)++; +} + +int main(const int argc, const char *const *const argv, + const char *const *const environ) +{ + char input[INPUT_SIZE]; + uint8_t shift; + unsigned flags; + const bool peek = false; + avoid_optimizing_out = argc; + +#ifndef USE_KLEE + sscanf(argv[2], "%d", &flags); +#endif + +#ifdef USE_KLEE + klee_make_symbolic(input, sizeof(input), "input"); + klee_make_symbolic(&shift, sizeof(shift), "shift"); + klee_make_symbolic(&flags, sizeof(flags), "flags"); + klee_assume(shift < INPUT_SIZE); + klee_assume(flags <= (kExprFlagsMulti|kExprFlagsDisallowEOC)); +#endif + + ParserLine plines[] = { + { +#ifdef USE_KLEE + .data = &input[shift], + .size = sizeof(input) - shift, +#else + .data = argv[1], + .size = strlen(argv[1]), +#endif + .allocated = false, + }, + { + .data = NULL, + .size = 0, + .allocated = false, + }, + }; +#ifdef USE_KLEE + assert(plines[0].size <= INPUT_SIZE); + assert((plines[0].data[0] != 5) | (plines[0].data[0] != argc)); +#endif + ParserLine *cur_pline = &plines[0]; + + ParserHighlight colors; + kvi_init(colors); + + ParserState pstate = { + .reader = { + .get_line = simple_get_line, + .cookie = &cur_pline, + .lines = KV_INITIAL_VALUE, + .conv.vc_type = CONV_NONE, + }, + .pos = { 0, 0 }, + .colors = &colors, + .can_continuate = false, + }; + kvi_init(pstate.reader.lines); + + const ExprAST ast = viml_pexpr_parse(&pstate, (int)flags); + assert(ast.root != NULL || ast.err.msg); + // Can’t possibly have more highlight tokens then there are bytes in string. + assert(kv_size(colors) <= INPUT_SIZE - shift); + kvi_destroy(colors); + // Not destroying pstate.reader.lines because there is no way it could exceed + // its limits in the current circumstances. + viml_pexpr_free_ast(ast); + assert(allocated_memory == 0); +} diff --git a/test/unit/eval/helpers.lua b/test/unit/eval/helpers.lua index 5bc482216e..d7399182f7 100644 --- a/test/unit/eval/helpers.lua +++ b/test/unit/eval/helpers.lua @@ -1,5 +1,6 @@ local helpers = require('test.unit.helpers')(nil) +local ptr2key = helpers.ptr2key local cimport = helpers.cimport local to_cstr = helpers.to_cstr local ffi = helpers.ffi @@ -91,10 +92,6 @@ local function populate_partial(pt, lua_pt, processed) return pt end -local ptr2key = function(ptr) - return tostring(ptr) -end - local lst2tbl local dct2tbl diff --git a/test/unit/formatc.lua b/test/unit/formatc.lua index e288081960..2fd37c599a 100644 --- a/test/unit/formatc.lua +++ b/test/unit/formatc.lua @@ -65,11 +65,12 @@ local tokens = P { "tokens"; identifier = Ct(C(R("az","AZ","__") * R("09","az","AZ","__")^0) * Cc"identifier"), -- Single character in a string - string_char = R("az","AZ","09") + S"$%^&*()_-+={[}]:;@~#<,>.!?/ \t" + (P"\\" * S[[ntvbrfa\?'"0x]]), + sstring_char = R("\001&","([","]\255") + (P"\\" * S[[ntvbrfa\?'"0x]]), + dstring_char = R("\001!","#[","]\255") + (P"\\" * S[[ntvbrfa\?'"0x]]), -- String literal - string = Ct(C(P"'" * (V"string_char" + P'"')^0 * P"'" + - P'"' * (V"string_char" + P"'")^0 * P'"') * Cc"string"), + string = Ct(C(P"'" * (V"sstring_char" + P'"')^0 * P"'" + + P'"' * (V"dstring_char" + P"'")^0 * P'"') * Cc"string"), -- Operator operator = Ct(C(P">>=" + P"<<=" + P"..." + diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index 4b9f185156..68ce9eed62 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -315,7 +315,7 @@ local function alloc_log_new() eq(exp, self.log) self:clear() end - function log:clear_tmp_allocs() + function log:clear_tmp_allocs(clear_null_frees) local toremove = {} local allocs = {} for i, v in ipairs(self.log) do @@ -327,6 +327,8 @@ local function alloc_log_new() if v.func == 'free' then toremove[#toremove + 1] = i end + elseif clear_null_frees and v.args[1] == self.null then + toremove[#toremove + 1] = i end if v.func == 'realloc' then allocs[tostring(v.ret)] = i @@ -698,7 +700,14 @@ local function check_child_err(rd) local len_s = sc.read(rd, 5) local len = tonumber(len_s) neq(0, len) - local err = sc.read(rd, len + 1) + local err = '' + if os.getenv('NVIM_TEST_TRACE_ON_ERROR') == '1' and #trace ~= 0 then + err = '\nTest failed, trace:\n' .. tracehelp + for _, traceline in ipairs(trace) do + err = err .. traceline + end + end + err = err .. sc.read(rd, len + 1) assert.just_fail(err) end @@ -753,6 +762,60 @@ end cimport('./src/nvim/types.h', './src/nvim/main.h', './src/nvim/os/time.h') +local function conv_enum(etab, eval) + local n = tonumber(eval) + return etab[n] or n +end + +local function array_size(arr) + return ffi.sizeof(arr) / ffi.sizeof(arr[0]) +end + +local function kvi_size(kvi) + return array_size(kvi.init_array) +end + +local function kvi_init(kvi) + kvi.capacity = kvi_size(kvi) + kvi.items = kvi.init_array + return kvi +end + +local function kvi_destroy(kvi) + if kvi.items ~= kvi.init_array then + lib.xfree(kvi.items) + end +end + +local function kvi_new(ct) + return kvi_init(ffi.new(ct)) +end + +local function make_enum_conv_tab(lib, values, skip_pref, set_cb) + child_call_once(function() + local ret = {} + for _, v in ipairs(values) do + local str_v = v + if v:sub(1, #skip_pref) == skip_pref then + str_v = v:sub(#skip_pref + 1) + end + ret[tonumber(lib[v])] = str_v + end + set_cb(ret) + end) +end + +local function ptr2addr(ptr) + return tonumber(ffi.cast('intptr_t', ffi.cast('void *', ptr))) +end + +local s = ffi.new('char[64]', {0}) + +local function ptr2key(ptr) + ffi.C.snprintf(s, ffi.sizeof(s), '%p', ffi.cast('void *', ptr)) + return ffi.string(s) +end + local module = { cimport = cimport, cppimport = cppimport, @@ -773,6 +836,15 @@ local module = { child_call_once = child_call_once, child_cleanup_once = child_cleanup_once, sc = sc, + conv_enum = conv_enum, + array_size = array_sive, + kvi_destroy = kvi_destroy, + kvi_size = kvi_size, + kvi_init = kvi_init, + kvi_new = kvi_new, + make_enum_conv_tab = make_enum_conv_tab, + ptr2addr = ptr2addr, + ptr2key = ptr2key, } return function() return module diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua new file mode 100644 index 0000000000..5910468017 --- /dev/null +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -0,0 +1,421 @@ +local helpers = require('test.unit.helpers')(after_each) +local global_helpers = require('test.helpers') +local itp = helpers.gen_itp(it) +local viml_helpers = require('test.unit.viml.helpers') + +local child_call_once = helpers.child_call_once +local conv_enum = helpers.conv_enum +local cimport = helpers.cimport +local ffi = helpers.ffi +local eq = helpers.eq + +local conv_ccs = viml_helpers.conv_ccs +local pline2lua = viml_helpers.pline2lua +local new_pstate = viml_helpers.new_pstate +local intchar2lua = viml_helpers.intchar2lua +local conv_cmp_type = viml_helpers.conv_cmp_type +local pstate_set_str = viml_helpers.pstate_set_str + +local shallowcopy = global_helpers.shallowcopy + +local lib = cimport('./src/nvim/viml/parser/expressions.h') + +local eltkn_type_tab, eltkn_mul_type_tab, eltkn_opt_scope_tab +child_call_once(function() + eltkn_type_tab = { + [tonumber(lib.kExprLexInvalid)] = 'Invalid', + [tonumber(lib.kExprLexMissing)] = 'Missing', + [tonumber(lib.kExprLexSpacing)] = 'Spacing', + [tonumber(lib.kExprLexEOC)] = 'EOC', + + [tonumber(lib.kExprLexQuestion)] = 'Question', + [tonumber(lib.kExprLexColon)] = 'Colon', + [tonumber(lib.kExprLexOr)] = 'Or', + [tonumber(lib.kExprLexAnd)] = 'And', + [tonumber(lib.kExprLexComparison)] = 'Comparison', + [tonumber(lib.kExprLexPlus)] = 'Plus', + [tonumber(lib.kExprLexMinus)] = 'Minus', + [tonumber(lib.kExprLexDot)] = 'Dot', + [tonumber(lib.kExprLexMultiplication)] = 'Multiplication', + + [tonumber(lib.kExprLexNot)] = 'Not', + + [tonumber(lib.kExprLexNumber)] = 'Number', + [tonumber(lib.kExprLexSingleQuotedString)] = 'SingleQuotedString', + [tonumber(lib.kExprLexDoubleQuotedString)] = 'DoubleQuotedString', + [tonumber(lib.kExprLexOption)] = 'Option', + [tonumber(lib.kExprLexRegister)] = 'Register', + [tonumber(lib.kExprLexEnv)] = 'Env', + [tonumber(lib.kExprLexPlainIdentifier)] = 'PlainIdentifier', + + [tonumber(lib.kExprLexBracket)] = 'Bracket', + [tonumber(lib.kExprLexFigureBrace)] = 'FigureBrace', + [tonumber(lib.kExprLexParenthesis)] = 'Parenthesis', + [tonumber(lib.kExprLexComma)] = 'Comma', + [tonumber(lib.kExprLexArrow)] = 'Arrow', + } + + eltkn_mul_type_tab = { + [tonumber(lib.kExprLexMulMul)] = 'Mul', + [tonumber(lib.kExprLexMulDiv)] = 'Div', + [tonumber(lib.kExprLexMulMod)] = 'Mod', + } + + eltkn_opt_scope_tab = { + [tonumber(lib.kExprOptScopeUnspecified)] = 'Unspecified', + [tonumber(lib.kExprOptScopeGlobal)] = 'Global', + [tonumber(lib.kExprOptScopeLocal)] = 'Local', + } +end) + +local function conv_eltkn_type(typ) + return conv_enum(eltkn_type_tab, typ) +end + +local bracket_types = { + Bracket = true, + FigureBrace = true, + Parenthesis = true, +} + +local function eltkn2lua(pstate, tkn) + local ret = { + type = conv_eltkn_type(tkn.type), + } + pstate_set_str(pstate, tkn.start, tkn.len, ret) + if not ret.error and (#(ret.str) ~= ret.len) then + ret.error = '#str /= len' + end + if ret.type == 'Comparison' then + ret.data = { + type = conv_cmp_type(tkn.data.cmp.type), + ccs = conv_ccs(tkn.data.cmp.ccs), + inv = (not not tkn.data.cmp.inv), + } + elseif ret.type == 'Multiplication' then + ret.data = { type = conv_enum(eltkn_mul_type_tab, tkn.data.mul.type) } + elseif bracket_types[ret.type] then + ret.data = { closing = (not not tkn.data.brc.closing) } + elseif ret.type == 'Register' then + ret.data = { name = intchar2lua(tkn.data.reg.name) } + elseif (ret.type == 'SingleQuotedString' + or ret.type == 'DoubleQuotedString') then + ret.data = { closed = (not not tkn.data.str.closed) } + elseif ret.type == 'Option' then + ret.data = { + scope = conv_enum(eltkn_opt_scope_tab, tkn.data.opt.scope), + name = ffi.string(tkn.data.opt.name, tkn.data.opt.len), + } + elseif ret.type == 'PlainIdentifier' then + ret.data = { + scope = intchar2lua(tkn.data.var.scope), + autoload = (not not tkn.data.var.autoload), + } + elseif ret.type == 'Number' then + ret.data = { + is_float = (not not tkn.data.num.is_float), + base = tonumber(tkn.data.num.base), + } + ret.data.val = tonumber(tkn.data.num.is_float + and tkn.data.num.val.floating + or tkn.data.num.val.integer) + elseif ret.type == 'Invalid' then + ret.data = { error = ffi.string(tkn.data.err.msg) } + end + return ret, tkn +end + +local function next_eltkn(pstate, flags) + return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, flags)) +end + +describe('Expressions lexer', function() + local flags = 0 + local should_advance = true + local function check_advance(pstate, bytes_to_advance, initial_col) + local tgt = initial_col + bytes_to_advance + if should_advance then + if pstate.reader.lines.items[0].size == tgt then + eq(1, pstate.pos.line) + eq(0, pstate.pos.col) + else + eq(0, pstate.pos.line) + eq(tgt, pstate.pos.col) + end + else + eq(0, pstate.pos.line) + eq(initial_col, pstate.pos.col) + end + end + local function singl_eltkn_test(typ, str, data) + local pstate = new_pstate({str}) + eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, + next_eltkn(pstate, flags)) + check_advance(pstate, #str, 0) + if not ( + typ == 'Spacing' + or (typ == 'Register' and str == '@') + or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString') + and not data.closed) + ) then + pstate = new_pstate({str .. ' '}) + eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, + next_eltkn(pstate, flags)) + check_advance(pstate, #str, 0) + end + pstate = new_pstate({'x' .. str}) + pstate.pos.col = 1 + eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ}, + next_eltkn(pstate, flags)) + check_advance(pstate, #str, 1) + end + local function scope_test(scope) + singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope}) + singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope}) + end + local function comparison_test(op, inv_op, cmp_type) + singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'}) + singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'}) + singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'}) + singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'}) + singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'}) + singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'}) + end + local function simple_test(pstate_arg, exp_type, exp_len, exp) + local pstate = new_pstate(pstate_arg) + local exp = shallowcopy(exp) + exp.type = exp_type + exp.len = exp_len or #(pstate_arg[0]) + exp.start = { col = 0, line = 0 } + eq(exp, next_eltkn(pstate, flags)) + end + local function stable_tests() + singl_eltkn_test('Parenthesis', '(', {closing=false}) + singl_eltkn_test('Parenthesis', ')', {closing=true}) + singl_eltkn_test('Bracket', '[', {closing=false}) + singl_eltkn_test('Bracket', ']', {closing=true}) + singl_eltkn_test('FigureBrace', '{', {closing=false}) + singl_eltkn_test('FigureBrace', '}', {closing=true}) + singl_eltkn_test('Question', '?') + singl_eltkn_test('Colon', ':') + singl_eltkn_test('Dot', '.') + singl_eltkn_test('Plus', '+') + singl_eltkn_test('Comma', ',') + singl_eltkn_test('Multiplication', '*', {type='Mul'}) + singl_eltkn_test('Multiplication', '/', {type='Div'}) + singl_eltkn_test('Multiplication', '%', {type='Mod'}) + singl_eltkn_test('Spacing', ' \t\t \t\t') + singl_eltkn_test('Spacing', ' ') + singl_eltkn_test('Spacing', '\t') + singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'}) + singl_eltkn_test('Number', '0123', {is_float=false, base=8, val=83}) + singl_eltkn_test('Number', '01234567', {is_float=false, base=8, val=342391}) + singl_eltkn_test('Number', '012345678', {is_float=false, base=10, val=12345678}) + singl_eltkn_test('Number', '0x123', {is_float=false, base=16, val=291}) + singl_eltkn_test('Number', '0x56FF', {is_float=false, base=16, val=22271}) + singl_eltkn_test('Number', '0xabcdef', {is_float=false, base=16, val=11259375}) + singl_eltkn_test('Number', '0xABCDEF', {is_float=false, base=16, val=11259375}) + singl_eltkn_test('Number', '0x0', {is_float=false, base=16, val=0}) + singl_eltkn_test('Number', '00', {is_float=false, base=8, val=0}) + singl_eltkn_test('Number', '0b0', {is_float=false, base=2, val=0}) + singl_eltkn_test('Number', '0b010111', {is_float=false, base=2, val=23}) + singl_eltkn_test('Number', '0b100111', {is_float=false, base=2, val=39}) + singl_eltkn_test('Number', '0', {is_float=false, base=10, val=0}) + singl_eltkn_test('Number', '9', {is_float=false, base=10, val=9}) + singl_eltkn_test('Env', '$abc') + singl_eltkn_test('Env', '$') + singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', '_test', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', '_test_foo', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 't', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 'test5', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 't0', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 'test#var', {autoload=true, scope=0}) + singl_eltkn_test('PlainIdentifier', 'test#var#val###', {autoload=true, scope=0}) + singl_eltkn_test('PlainIdentifier', 't#####', {autoload=true, scope=0}) + singl_eltkn_test('And', '&&') + singl_eltkn_test('Or', '||') + singl_eltkn_test('Invalid', '&', {error='E112: Option name missing: %.*s'}) + singl_eltkn_test('Option', '&opt', {scope='Unspecified', name='opt'}) + singl_eltkn_test('Option', '&t_xx', {scope='Unspecified', name='t_xx'}) + singl_eltkn_test('Option', '&t_\r\r', {scope='Unspecified', name='t_\r\r'}) + singl_eltkn_test('Option', '&t_\t\t', {scope='Unspecified', name='t_\t\t'}) + singl_eltkn_test('Option', '&t_ ', {scope='Unspecified', name='t_ '}) + singl_eltkn_test('Option', '&g:opt', {scope='Global', name='opt'}) + singl_eltkn_test('Option', '&l:opt', {scope='Local', name='opt'}) + singl_eltkn_test('Invalid', '&l:', {error='E112: Option name missing: %.*s'}) + singl_eltkn_test('Invalid', '&g:', {error='E112: Option name missing: %.*s'}) + singl_eltkn_test('Register', '@', {name=-1}) + singl_eltkn_test('Register', '@a', {name='a'}) + singl_eltkn_test('Register', '@\r', {name=13}) + singl_eltkn_test('Register', '@ ', {name=' '}) + singl_eltkn_test('Register', '@\t', {name=9}) + singl_eltkn_test('SingleQuotedString', '\'test', {closed=false}) + singl_eltkn_test('SingleQuotedString', '\'test\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'\'\'\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'x\'\'\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'\'\'x\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'\'\'', {closed=false}) + singl_eltkn_test('SingleQuotedString', '\'x\'\'', {closed=false}) + singl_eltkn_test('SingleQuotedString', '\'\'\'x', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"test', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"test"', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"\\""', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"x\\""', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"\\"x"', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"\\"', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"x\\"', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"\\"x', {closed=false}) + singl_eltkn_test('Not', '!') + singl_eltkn_test('Invalid', '=', {error='E15: Expected == or =~: %.*s'}) + comparison_test('==', '!=', 'Equal') + comparison_test('=~', '!~', 'Matches') + comparison_test('>', '<=', 'Greater') + comparison_test('>=', '<', 'GreaterOrEqual') + singl_eltkn_test('Minus', '-') + singl_eltkn_test('Arrow', '->') + singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'}) + simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'}) + simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'}) + simple_test({'2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'0b102'}, 'Number', 4, {data={is_float=false, base=2, val=2}, str='0b10'}) + simple_test({'10F'}, 'Number', 2, {data={is_float=false, base=10, val=10}, str='10'}) + simple_test({'0x0123456789ABCDEFG'}, 'Number', 18, {data={is_float=false, base=16, val=81985529216486895}, str='0x0123456789ABCDEF'}) + simple_test({{data='00', size=2}}, 'Number', 2, {data={is_float=false, base=8, val=0}, str='00'}) + simple_test({{data='009', size=2}}, 'Number', 2, {data={is_float=false, base=8, val=0}, str='00'}) + simple_test({{data='01', size=1}}, 'Number', 1, {data={is_float=false, base=10, val=0}, str='0'}) + end + + local function regular_scope_tests() + scope_test('s') + scope_test('g') + scope_test('v') + scope_test('b') + scope_test('w') + scope_test('t') + scope_test('l') + scope_test('a') + + simple_test({'g:'}, 'PlainIdentifier', 2, {data={scope='g', autoload=false}, str='g:'}) + simple_test({'g:is#foo'}, 'PlainIdentifier', 8, {data={scope='g', autoload=true}, str='g:is#foo'}) + simple_test({'g:isnot#foo'}, 'PlainIdentifier', 11, {data={scope='g', autoload=true}, str='g:isnot#foo'}) + end + + local function regular_is_tests() + comparison_test('is', 'isnot', 'Identical') + + simple_test({'is'}, 'Comparison', 2, {data={type='Identical', inv=false, ccs='UseOption'}, str='is'}) + simple_test({'isnot'}, 'Comparison', 5, {data={type='Identical', inv=true, ccs='UseOption'}, str='isnot'}) + simple_test({'is?'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='IgnoreCase'}, str='is?'}) + simple_test({'isnot?'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='IgnoreCase'}, str='isnot?'}) + simple_test({'is#'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'}) + simple_test({'isnot#'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'}) + simple_test({'is#foo'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'}) + simple_test({'isnot#foo'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'}) + end + + local function regular_number_tests() + simple_test({'2.0'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + end + + local function regular_eoc_tests() + singl_eltkn_test('EOC', '|') + singl_eltkn_test('EOC', '\0') + singl_eltkn_test('EOC', '\n') + end + + itp('works (single tokens, zero flags)', function() + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_is_tests() + regular_number_tests() + end) + itp('peeks', function() + flags = tonumber(lib.kELFlagPeek) + should_advance = false + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_is_tests() + regular_number_tests() + end) + itp('forbids scope', function() + flags = tonumber(lib.kELFlagForbidScope) + stable_tests() + + regular_eoc_tests() + regular_is_tests() + regular_number_tests() + + simple_test({'g:'}, 'PlainIdentifier', 1, {data={scope=0, autoload=false}, str='g'}) + end) + itp('allows floats', function() + flags = tonumber(lib.kELFlagAllowFloat) + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_is_tests() + + simple_test({'2.2'}, 'Number', 3, {data={is_float=true, base=10, val=2.2}, str='2.2'}) + simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true, base=10, val=2e5}, str='2.0e5'}) + simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true, base=10, val=2e5}, str='2.0e+5'}) + simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true, base=10, val=2e-5}, str='2.0e-5'}) + simple_test({'2.500000e-5'}, 'Number', 11, {data={is_float=true, base=10, val=2.5e-5}, str='2.500000e-5'}) + simple_test({'2.5555e2'}, 'Number', 8, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e2'}) + simple_test({'2.5555e+2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e+2'}) + simple_test({'2.5555e-2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e-2}, str='2.5555e-2'}) + simple_test({{data='2.5e-5', size=3}}, + 'Number', 3, {data={is_float=true, base=10, val=2.5}, str='2.5'}) + simple_test({{data='2.5e5', size=4}}, + 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({{data='2.5e-50', size=6}}, + 'Number', 6, {data={is_float=true, base=10, val=2.5e-5}, str='2.5e-5'}) + end) + itp('treats `is` as an identifier', function() + flags = tonumber(lib.kELFlagIsNotCmp) + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_number_tests() + + simple_test({'is'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'}) + simple_test({'isnot'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'}) + simple_test({'is?'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'}) + simple_test({'isnot?'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'}) + simple_test({'is#'}, 'PlainIdentifier', 3, {data={scope=0, autoload=true}, str='is#'}) + simple_test({'isnot#'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='isnot#'}) + simple_test({'is#foo'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='is#foo'}) + simple_test({'isnot#foo'}, 'PlainIdentifier', 9, {data={scope=0, autoload=true}, str='isnot#foo'}) + end) + itp('forbids EOC', function() + flags = tonumber(lib.kELFlagForbidEOC) + stable_tests() + + regular_scope_tests() + regular_is_tests() + regular_number_tests() + + singl_eltkn_test('Invalid', '|', {error='E15: Unexpected EOC character: %.*s'}) + singl_eltkn_test('Invalid', '\0', {error='E15: Unexpected EOC character: %.*s'}) + singl_eltkn_test('Invalid', '\n', {error='E15: Unexpected EOC character: %.*s'}) + end) +end) diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua new file mode 100644 index 0000000000..019fe69046 --- /dev/null +++ b/test/unit/viml/expressions/parser_spec.lua @@ -0,0 +1,7199 @@ +local helpers = require('test.unit.helpers')(after_each) +local global_helpers = require('test.helpers') +local itp = helpers.gen_itp(it) +local viml_helpers = require('test.unit.viml.helpers') + +local make_enum_conv_tab = helpers.make_enum_conv_tab +local child_call_once = helpers.child_call_once +local alloc_log_new = helpers.alloc_log_new +local kvi_destroy = helpers.kvi_destroy +local conv_enum = helpers.conv_enum +local ptr2key = helpers.ptr2key +local cimport = helpers.cimport +local ffi = helpers.ffi +local eq = helpers.eq + +local conv_ccs = viml_helpers.conv_ccs +local pline2lua = viml_helpers.pline2lua +local new_pstate = viml_helpers.new_pstate +local intchar2lua = viml_helpers.intchar2lua +local conv_cmp_type = viml_helpers.conv_cmp_type +local pstate_set_str = viml_helpers.pstate_set_str + +local format_string = global_helpers.format_string +local format_luav = global_helpers.format_luav + +local lib = cimport('./src/nvim/viml/parser/expressions.h') + +local alloc_log = alloc_log_new() + +local function format_check(expr, flags, ast, hls) + -- That forces specific order. + print( format_string('\ncheck_parsing(%r, %u, {', expr, flags)) + local digits = ' -- ' + local digits2 = ' -- ' + for i = 0, #expr - 1 do + if i % 10 == 0 then + digits2 = ('%s%10u'):format(digits2, i / 10) + end + digits = ('%s%u'):format(digits, i % 10) + end + print(digits) + if #expr > 10 then + print(digits2) + end + print(' ast = ' .. format_luav(ast.ast, ' ') .. ',') + if ast.err then + print(' err = {') + print(' arg = ' .. format_luav(ast.err.arg) .. ',') + print(' msg = ' .. format_luav(ast.err.msg) .. ',') + print(' },') + end + print('}, {') + local next_col = 0 + for _, v in ipairs(hls) do + local group, line, col, str = v:match('NVim([a-zA-Z]+):(%d+):(%d+):(.*)') + col = tonumber(col) + line = tonumber(line) + assert(line == 0) + local col_shift = col - next_col + assert(col_shift >= 0) + next_col = col + #str + print(format_string(' hl(%r, %r%s),', + group, + str, + (col_shift == 0 and '' or (', %u'):format(col_shift)))) + end + print('})') +end + +local east_node_type_tab +make_enum_conv_tab(lib, { + 'kExprNodeMissing', + 'kExprNodeOpMissing', + 'kExprNodeTernary', + 'kExprNodeTernaryValue', + 'kExprNodeRegister', + 'kExprNodeSubscript', + 'kExprNodeListLiteral', + 'kExprNodeUnaryPlus', + 'kExprNodeBinaryPlus', + 'kExprNodeNested', + 'kExprNodeCall', + 'kExprNodePlainIdentifier', + 'kExprNodePlainKey', + 'kExprNodeComplexIdentifier', + 'kExprNodeUnknownFigure', + 'kExprNodeLambda', + 'kExprNodeDictLiteral', + 'kExprNodeCurlyBracesIdentifier', + 'kExprNodeComma', + 'kExprNodeColon', + 'kExprNodeArrow', + 'kExprNodeComparison', + 'kExprNodeConcat', + 'kExprNodeConcatOrSubscript', + 'kExprNodeInteger', + 'kExprNodeFloat', + 'kExprNodeSingleQuotedString', + 'kExprNodeDoubleQuotedString', + 'kExprNodeOr', + 'kExprNodeAnd', + 'kExprNodeUnaryMinus', + 'kExprNodeBinaryMinus', + 'kExprNodeNot', + 'kExprNodeMultiplication', + 'kExprNodeDivision', + 'kExprNodeMod', + 'kExprNodeOption', + 'kExprNodeEnvironment', +}, 'kExprNode', function(ret) east_node_type_tab = ret end) + +local function conv_east_node_type(typ) + return conv_enum(east_node_type_tab, typ) +end + +local eastnodelist2lua + +local function eastnode2lua(pstate, eastnode, checked_nodes) + local key = ptr2key(eastnode) + if checked_nodes[key] then + checked_nodes[key].duplicate_key = key + return { duplicate = key } + end + local typ = conv_east_node_type(eastnode.type) + local ret = {} + checked_nodes[key] = ret + ret.children = eastnodelist2lua(pstate, eastnode.children, checked_nodes) + local str = pstate_set_str(pstate, eastnode.start, eastnode.len) + local ret_str + if str.error then + ret_str = 'error:' .. str.error + else + ret_str = ('%u:%u:%s'):format(str.start.line, str.start.col, str.str) + end + if typ == 'Register' then + typ = typ .. ('(name=%s)'):format( + tostring(intchar2lua(eastnode.data.reg.name))) + elseif typ == 'PlainIdentifier' then + typ = typ .. ('(scope=%s,ident=%s)'):format( + tostring(intchar2lua(eastnode.data.var.scope)), + ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len)) + elseif typ == 'PlainKey' then + typ = typ .. ('(key=%s)'):format( + ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len)) + elseif (typ == 'UnknownFigure' or typ == 'DictLiteral' + or typ == 'CurlyBracesIdentifier' or typ == 'Lambda') then + typ = typ .. ('(%s)'):format( + (eastnode.data.fig.type_guesses.allow_lambda and '\\' or '-') + .. (eastnode.data.fig.type_guesses.allow_dict and 'd' or '-') + .. (eastnode.data.fig.type_guesses.allow_ident and 'i' or '-')) + elseif typ == 'Comparison' then + typ = typ .. ('(type=%s,inv=%u,ccs=%s)'):format( + conv_cmp_type(eastnode.data.cmp.type), eastnode.data.cmp.inv and 1 or 0, + conv_ccs(eastnode.data.cmp.ccs)) + elseif typ == 'Integer' then + typ = typ .. ('(val=%u)'):format(tonumber(eastnode.data.num.value)) + elseif typ == 'Float' then + typ = typ .. ('(val=%e)'):format(tonumber(eastnode.data.flt.value)) + elseif typ == 'SingleQuotedString' or typ == 'DoubleQuotedString' then + if eastnode.data.str.value == nil then + typ = typ .. '(val=NULL)' + else + local s = ffi.string(eastnode.data.str.value, eastnode.data.str.size) + typ = format_string('%s(val=%q)', typ, s) + end + elseif typ == 'Option' then + typ = ('%s(scope=%s,ident=%s)'):format( + typ, + tostring(intchar2lua(eastnode.data.opt.scope)), + ffi.string(eastnode.data.opt.ident, eastnode.data.opt.ident_len)) + elseif typ == 'Environment' then + typ = ('%s(ident=%s)'):format( + typ, + ffi.string(eastnode.data.env.ident, eastnode.data.env.ident_len)) + end + ret_str = typ .. ':' .. ret_str + local can_simplify = true + for k, v in pairs(ret) do + can_simplify = false + end + if can_simplify then + ret = ret_str + else + ret[1] = ret_str + end + return ret +end + +eastnodelist2lua = function(pstate, eastnode, checked_nodes) + local ret = {} + while eastnode ~= nil do + ret[#ret + 1] = eastnode2lua(pstate, eastnode, checked_nodes) + eastnode = eastnode.next + end + if #ret == 0 then + ret = nil + end + return ret +end + +local function east2lua(pstate, east) + local checked_nodes = {} + return { + err = east.err.msg ~= nil and { + msg = ffi.string(east.err.msg), + arg = ('%s'):format( + ffi.string(east.err.arg, east.err.arg_len)), + } or nil, + ast = eastnodelist2lua(pstate, east.root, checked_nodes), + } +end + +local function phl2lua(pstate) + local ret = {} + for i = 0, (tonumber(pstate.colors.size) - 1) do + local chunk = pstate.colors.items[i] + local chunk_tbl = pstate_set_str( + pstate, chunk.start, chunk.end_col - chunk.start.col, { + group = ffi.string(chunk.group), + }) + chunk_str = ('%s:%u:%u:%s'):format( + chunk_tbl.group, + chunk_tbl.start.line, + chunk_tbl.start.col, + chunk_tbl.str) + ret[i + 1] = chunk_str + end + return ret +end + +child_call_once(function() + assert:set_parameter('TableFormatLevel', 1000000) +end) + +describe('Expressions parser', function() + local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) + local err, msg = pcall(function() + flags = flags or 0 + + if os.getenv('NVIM_TEST_PARSER_SPEC_PRINT_TEST_CASE') == '1' then + print(str, flags) + end + alloc_log:check({}) + + local pstate = new_pstate({str}) + local east = lib.viml_pexpr_parse(pstate, flags) + local ast = east2lua(pstate, east) + local hls = phl2lua(pstate) + if exp_ast == nil then + format_check(str, flags, ast, hls) + else + eq(exp_ast, ast) + if exp_highlighting_fs then + local exp_highlighting = {} + local next_col = 0 + for i, h in ipairs(exp_highlighting_fs) do + exp_highlighting[i], next_col = h(next_col) + end + eq(exp_highlighting, hls) + end + end + lib.viml_pexpr_free_ast(east) + kvi_destroy(pstate.colors) + alloc_log:clear_tmp_allocs(true) + alloc_log:check({}) + end) + if not err then + msg = format_string('Error while processing test (%r, %u):\n%s', str, flags, msg) + error(msg) + end + end + local function hl(group, str, shift) + return function(next_col) + local col = next_col + (shift or 0) + return (('%s:%u:%u:%s'):format( + 'NVim' .. group, + 0, + col, + str)), (col + #str) + end + end + itp('works with + and @a', function() + check_parsing('@a', 0, { + ast = { + 'Register(name=a):0:0:@a', + }, + }, { + hl('Register', '@a'), + }) + check_parsing('+@a', 0, { + ast = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + }) + check_parsing('@a+@b', 0, { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a+@b+@c', 0, { + ast = { + { + 'BinaryPlus:0:5:+', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + 'Register(name=c):0:6:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('BinaryPlus', '+'), + hl('Register', '@c'), + }) + check_parsing('+@a+@b', 0, { + ast = { + { + 'BinaryPlus:0:3:+', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:4:@b', + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('+@a++@b', 0, { + ast = { + { + 'BinaryPlus:0:3:+', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'UnaryPlus:0:4:+', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a@b', 0, { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:2:@b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidRegister', '@b'), + }) + check_parsing(' @a \t @b', 0, { + ast = { + { + 'OpMissing:0:3:', + children = { + 'Register(name=a):0:0: @a', + 'Register(name=b):0:3: \t @b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a', 1), + hl('InvalidSpacing', ' \t '), + hl('Register', '@b'), + }) + check_parsing('+', 0, { + ast = { + 'UnaryPlus:0:0:+', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryPlus', '+'), + }) + check_parsing(' +', 0, { + ast = { + 'UnaryPlus:0:0: +', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryPlus', '+', 1), + }) + check_parsing('@a+ ', 0, { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + }) + end) + itp('works with @a, + and parenthesis', function() + check_parsing('(@a)', 0, { + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + }) + check_parsing('()', 0, { + ast = { + { + 'Nested:0:0:(', + children = { + 'Missing:0:1:', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing(')', 0, { + ast = { + { + 'Nested:0:0:', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('+)', 0, { + ast = { + { + 'Nested:0:1:', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('UnaryPlus', '+'), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('+@a(@b)', 0, { + ast = { + { + 'UnaryPlus:0:0:+', + children = { + { + 'Call:0:3:(', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+@b(@c)', 0, { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + 'Register(name=c):0:6:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a()', 0, { + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a ()', 0, { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:2: (', + children = { + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = '()', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidSpacing', ' '), + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing( + '@a + (@b)', 0, { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing( + '@a + (+@b)', 0, { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'UnaryPlus:0:6:+', + children = { + 'Register(name=b):0:7:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing( + '@a + (@b + @c)', 0, { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + }) + check_parsing('(@a)+@b', 0, { + ast = { + { + 'BinaryPlus:0:4:+', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:5:@b', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a+(@b)(@c)', 0, { + -- 01234567890 + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:7:(', + children = { + { + 'Nested:0:3:(', + children = { 'Register(name=b):0:4:@b' }, + }, + 'Register(name=c):0:8:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+((@b))(@c)', 0, { + -- 01234567890123456890123456789 + -- 0 1 2 + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:9:(', + children = { + { + 'Nested:0:3:(', + children = { + { + 'Nested:0:4:(', + children = { 'Register(name=b):0:5:@b' } + }, + }, + }, + 'Register(name=c):0:10:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+((@b))+@c', 0, { + -- 01234567890123456890123456789 + -- 0 1 2 + ast = { + { + 'BinaryPlus:0:9:+', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:3:(', + children = { + { + 'Nested:0:4:(', + children = { 'Register(name=b):0:5:@b' } + }, + }, + }, + }, + }, + 'Register(name=c):0:10:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+'), + hl('Register', '@c'), + }) + check_parsing( + '@a + (@b + @c) + @d(@e) + (+@f) + ((+@g(@h))(@j)(@k))(@l)', 0, {--[[ + | | | | | | | | || | | || | | ||| || || || || + 000000000011111111112222222222333333333344444444445555555 + 012345678901234567890123456789012345678901234567890123456 + ]] + ast = {{ + 'BinaryPlus:0:31: +', + children = { + { + 'BinaryPlus:0:23: +', + children = { + { + 'BinaryPlus:0:14: +', + children = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', + }, + }, + }, + }, + }, + }, + { + 'Call:0:19:(', + children = { + 'Register(name=d):0:16: @d', + 'Register(name=e):0:20:@e', + }, + }, + }, + }, + { + 'Nested:0:25: (', + children = { + { + 'UnaryPlus:0:27:+', + children = { + 'Register(name=f):0:28:@f', + }, + }, + }, + }, + }, + }, + { + 'Call:0:53:(', + children = { + { + 'Nested:0:33: (', + children = { + { + 'Call:0:48:(', + children = { + { + 'Call:0:44:(', + children = { + { + 'Nested:0:35:(', + children = { + { + 'UnaryPlus:0:36:+', + children = { + { + 'Call:0:39:(', + children = { + 'Register(name=g):0:37:@g', + 'Register(name=h):0:40:@h', + }, + }, + }, + }, + }, + }, + 'Register(name=j):0:45:@j', + }, + }, + 'Register(name=k):0:49:@k', + }, + }, + }, + }, + 'Register(name=l):0:54:@l', + }, + }, + }, + }}, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('Register', '@d', 1), + hl('CallingParenthesis', '('), + hl('Register', '@e'), + hl('CallingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@f'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('NestingParenthesis', '('), + hl('UnaryPlus', '+'), + hl('Register', '@g'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@j'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@k'), + hl('CallingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@l'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a)', 0, { + -- 012 + ast = { + { + 'Nested:0:2:', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Unexpected closing parenthesis: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('(@a', 0, { + -- 012 + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + err = { + arg = '(@a', + msg = 'E110: Missing closing parenthesis for nested expression: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + }) + check_parsing('@a(@b', 0, { + -- 01234 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + err = { + arg = '(@b', + msg = 'E116: Missing closing parenthesis for function call: %.*s', + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + }) + check_parsing('@a(@b, @c, @d, @e)', 0, { + -- 012345678901234567 + -- 0 1 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Comma:0:5:,', + children = { + 'Register(name=b):0:3:@b', + { + 'Comma:0:9:,', + children = { + 'Register(name=c):0:6: @c', + { + 'Comma:0:13:,', + children = { + 'Register(name=d):0:10: @d', + 'Register(name=e):0:14: @e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c', 1), + hl('Comma', ','), + hl('Register', '@d', 1), + hl('Comma', ','), + hl('Register', '@e', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a(@b(@c))', 0, { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + 'Register(name=c):0:6:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a(@b(@c(@d(@e), @f(@g(@h), @i(@j)))))', 0, { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + { + 'Call:0:8:(', + children = { + 'Register(name=c):0:6:@c', + { + 'Comma:0:15:,', + children = { + { + 'Call:0:11:(', + children = { + 'Register(name=d):0:9:@d', + 'Register(name=e):0:12:@e', + }, + }, + { + 'Call:0:19:(', + children = { + 'Register(name=f):0:16: @f', + { + 'Comma:0:26:,', + children = { + { + 'Call:0:22:(', + children = { + 'Register(name=g):0:20:@g', + 'Register(name=h):0:23:@h', + }, + }, + { + 'Call:0:30:(', + children = { + 'Register(name=i):0:27: @i', + 'Register(name=j):0:31:@j', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', '('), + hl('Register', '@d'), + hl('CallingParenthesis', '('), + hl('Register', '@e'), + hl('CallingParenthesis', ')'), + hl('Comma', ','), + hl('Register', '@f', 1), + hl('CallingParenthesis', '('), + hl('Register', '@g'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('Comma', ','), + hl('Register', '@i', 1), + hl('CallingParenthesis', '('), + hl('Register', '@j'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + }) + end) + itp('works with variable names, including curly braces ones', function() + check_parsing('var', 0, { + ast = { + 'PlainIdentifier(scope=0,ident=var):0:0:var', + }, + }, { + hl('IdentifierName', 'var'), + }) + check_parsing('g:var', 0, { + ast = { + 'PlainIdentifier(scope=g,ident=var):0:0:g:var', + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'var'), + }) + check_parsing('g:', 0, { + ast = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + }) + check_parsing('{a}', 0, { + -- 012 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + }) + check_parsing('{a:b}', 0, { + -- 012 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'PlainIdentifier(scope=a,ident=b):0:1:a:b', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + }) + check_parsing('{a:@b}', 0, { + -- 012345 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'OpMissing:0:3:', + children={ + 'PlainIdentifier(scope=a,ident=):0:1:a:', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, + }, + err = { + arg = '@b}', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('InvalidRegister', '@b'), + hl('Curly', '}'), + }) + check_parsing('{@a}', 0, { + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('{@a}{@b}', 0, { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:4:', + children = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'CurlyBracesIdentifier(--i):0:4:{', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('Register', '@b'), + hl('Curly', '}'), + }) + check_parsing('g:{@a}', 0, { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('{@a}_test', 0, { + -- 012345678 + ast = { + { + 'ComplexIdentifier:0:4:', + children = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:4:_test', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + }) + check_parsing('g:{@a}_test', 0, { + -- 01234567890 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:6:_test', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + }) + check_parsing('g:{@a}_test()', 0, { + -- 0123456789012 + ast = { + { + 'Call:0:11:(', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:6:_test', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('{@a} ()', 0, { + -- 0123456789012 + ast = { + { + 'Call:0:4: (', + children = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('g:{@a} ()', 0, { + -- 0123456789012 + ast = { + { + 'Call:0:6: (', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('{@a', 0, { + -- 012 + ast = { + { + 'UnknownFigure(-di):0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + err = { + arg = '{@a', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + hl('Register', '@a'), + }) + end) + itp('works with lambdas and dictionaries', function() + check_parsing('{}', 0, { + ast = { + 'DictLiteral(-di):0:0:{', + }, + }, { + hl('Dict', '{'), + hl('Dict', '}'), + }) + check_parsing('{->@a}', 0, { + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Arrow:0:1:->', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{->@a+@b}', 0, { + -- 012345678 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Arrow:0:1:->', + children = { + { + 'BinaryPlus:0:5:+', + children = { + 'Register(name=a):0:3:@a', + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('Lambda', '}'), + }) + check_parsing('{a->@a}', 0, { + -- 012345678 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Arrow:0:2:->', + children = { + 'Register(name=a):0:4:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->@a}', 0, { + -- 012345678 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + 'Register(name=a):0:6:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c->@a}', 0, { + -- 01234567890 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + { + 'Arrow:0:6:->', + children = { + 'Register(name=a):0:8:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c,d->@a}', 0, { + -- 0123456789012 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + { + 'Comma:0:6:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:5:c', + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + { + 'Arrow:0:8:->', + children = { + 'Register(name=a):0:10:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c,d,->@a}', 0, { + -- 01234567890123 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + { + 'Comma:0:6:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:5:c', + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + }, + }, + { + 'Arrow:0:9:->', + children = { + 'Register(name=a):0:11:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Comma', ','), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->{c,d->{e,f->@a}}}', 0, { + -- 01234567890123456789012 + -- 0 1 2 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + { + 'Lambda(\\di):0:6:{', + children = { + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7:c', + 'PlainIdentifier(scope=0,ident=d):0:9:d', + }, + }, + { + 'Arrow:0:10:->', + children = { + { + 'Lambda(\\di):0:12:{', + children = { + { + 'Comma:0:14:,', + children = { + 'PlainIdentifier(scope=0,ident=e):0:13:e', + 'PlainIdentifier(scope=0,ident=f):0:15:f', + }, + }, + { + 'Arrow:0:16:->', + children = { + 'Register(name=a):0:18:@a', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('Lambda', '{'), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Arrow', '->'), + hl('Lambda', '{'), + hl('IdentifierName', 'e'), + hl('Comma', ','), + hl('IdentifierName', 'f'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + hl('Lambda', '}'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->c,d}', 0, { + -- 0123456789 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + { + 'Comma:0:7:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6:c', + 'PlainIdentifier(scope=0,ident=d):0:8:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',d}', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + hl('Lambda', '}'), + }) + check_parsing('a,b,c,d', 0, { + -- 0123456789 + ast = { + { + 'Comma:0:1:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comma:0:3:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + 'PlainIdentifier(scope=0,ident=d):0:6:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',b,c,d', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidComma', ','), + hl('IdentifierName', 'b'), + hl('InvalidComma', ','), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + }) + check_parsing('a,b,c,d,', 0, { + -- 0123456789 + ast = { + { + 'Comma:0:1:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comma:0:3:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + { + 'Comma:0:7:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:6:d', + }, + }, + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',b,c,d,', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidComma', ','), + hl('IdentifierName', 'b'), + hl('InvalidComma', ','), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + hl('InvalidComma', ','), + }) + check_parsing(',', 0, { + -- 0123456789 + ast = { + { + 'Comma:0:0:,', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ',', + msg = 'E15: Expected value, got comma: %.*s', + }, + }, { + hl('InvalidComma', ','), + }) + check_parsing('{,a->@a}', 0, { + -- 0123456789 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'Arrow:0:3:->', + children = { + { + 'Comma:0:1:,', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:2:a', + }, + }, + 'Register(name=a):0:5:@a', + }, + }, + }, + }, + }, + err = { + arg = ',a->@a}', + msg = 'E15: Expected value, got comma: %.*s', + }, + }, { + hl('Curly', '{'), + hl('InvalidComma', ','), + hl('IdentifierName', 'a'), + hl('InvalidArrow', '->'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('}', 0, { + -- 0123456789 + ast = { + 'UnknownFigure(---):0:0:', + }, + err = { + arg = '}', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidFigureBrace', '}'), + }) + check_parsing('{->}', 0, { + -- 0123456789 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + 'Arrow:0:1:->', + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected value, got closing figure brace: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('InvalidLambda', '}'), + }) + check_parsing('{a,b}', 0, { + -- 0123456789 + ast = { + { + 'Lambda(-di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected lambda arguments list or arrow: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('InvalidLambda', '}'), + }) + check_parsing('{a,}', 0, { + -- 0123456789 + ast = { + { + 'Lambda(-di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected lambda arguments list or arrow: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('InvalidLambda', '}'), + }) + check_parsing('{@a:@b}', 0, { + -- 0123456789 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d}', 0, { + -- 0123456789012 + -- 0 1 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d,@e:@f,}', 0, { + -- 01234567890123456789 + -- 0 1 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Comma:0:12:,', + children = { + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:15::', + children = { + 'Register(name=e):0:13:@e', + 'Register(name=f):0:16:@f', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Comma', ','), + hl('Register', '@e'), + hl('Colon', ':'), + hl('Register', '@f'), + hl('Comma', ','), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', 0, { + -- 01234567890123456789012 + -- 0 1 2 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Comma:0:12:,', + children = { + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:15::', + children = { + 'Register(name=e):0:13:@e', + 'Register(name=f):0:16:@f', + }, + }, + { + 'Colon:0:21::', + children = { + 'Register(name=g):0:19:@g', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected value, got closing figure brace: %.*s', + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Comma', ','), + hl('Register', '@e'), + hl('Colon', ':'), + hl('Register', '@f'), + hl('Comma', ','), + hl('Register', '@g'), + hl('Colon', ':'), + hl('InvalidDict', '}'), + }) + check_parsing('{@a:@b,}', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Dict', '}'), + }) + check_parsing('{({f -> g})(@h)(@i)}', 0, { + -- 01234567890123456789 + -- 0 1 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'Call:0:15:(', + children = { + { + 'Call:0:11:(', + children = { + { + 'Nested:0:1:(', + children = { + { + 'Lambda(\\di):0:2:{', + children = { + 'PlainIdentifier(scope=0,ident=f):0:3:f', + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=0,ident=g):0:7: g', + }, + }, + }, + }, + }, + }, + 'Register(name=h):0:12:@h', + }, + }, + 'Register(name=i):0:16:@i', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('NestingParenthesis', '('), + hl('Lambda', '{'), + hl('IdentifierName', 'f'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'g', 1), + hl('Lambda', '}'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@i'), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + }) + check_parsing('a:{b()}c', 0, { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=a,ident=):0:0:a:', + { + 'ComplexIdentifier:0:7:', + children = { + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + { + 'Call:0:4:(', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:7:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + }) + check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', 0, { + -- 01234567890123456789012345678901234567890123456 + -- 0 1 2 3 4 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=a,ident=):0:0:a:', + { + 'ComplexIdentifier:0:42:', + children = { + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + { + 'Call:0:37:(', + children = { + { + 'Lambda(\\di):0:3:{', + children = { + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + { + 'Arrow:0:8: ->', + children = { + { + 'BinaryPlus:0:19: +', + children = { + { + 'BinaryPlus:0:14: +', + children = { + 'Register(name=d):0:11: @d', + 'Register(name=e):0:16: @e', + }, + }, + { + 'Call:0:32:(', + children = { + { + 'Nested:0:21: (', + children = { + { + 'Lambda(\\di):0:23:{', + children = { + 'PlainIdentifier(scope=0,ident=f):0:24:f', + { + 'Arrow:0:25: ->', + children = { + 'PlainIdentifier(scope=0,ident=g):0:28: g', + }, + }, + }, + }, + }, + }, + 'Register(name=h):0:33:@h', + }, + }, + }, + }, + }, + }, + }, + }, + 'Register(name=i):0:38:@i', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=j):0:42:j', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Lambda', '{'), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('Arrow', '->', 1), + hl('Register', '@d', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@e', 1), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Lambda', '{'), + hl('IdentifierName', 'f'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'g', 1), + hl('Lambda', '}'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('Register', '@i'), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + hl('IdentifierName', 'j'), + }) + check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', 0, { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:8: :', + children = { + { + 'BinaryPlus:0:3: +', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:5: @b', + }, + }, + { + 'BinaryPlus:0:13: +', + children = { + 'Register(name=c):0:10: @c', + 'Register(name=d):0:15: @d', + }, + }, + }, + }, + { + 'Colon:0:27: :', + children = { + { + 'BinaryPlus:0:22: +', + children = { + 'Register(name=e):0:19: @e', + 'Register(name=f):0:24: @f', + }, + }, + { + 'BinaryPlus:0:32: +', + children = { + 'Register(name=g):0:29: @g', + 'Register(name=i):0:34: @i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('Register', '@b', 1), + hl('Colon', ':', 1), + hl('Register', '@c', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@d', 1), + hl('Comma', ','), + hl('Register', '@e', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@f', 1), + hl('Colon', ':', 1), + hl('Register', '@g', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@i', 1), + hl('Dict', '}'), + }) + check_parsing('-> -> ->', 0, { + -- 01234567 + ast = { + { + 'Arrow:0:0:->', + children = { + 'Missing:0:0:', + { + 'Arrow:0:2: ->', + children = { + 'Missing:0:2:', + { + 'Arrow:0:5: ->', + children = { + 'Missing:0:5:', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> -> ->', + msg = 'E15: Unexpected arrow: %.*s', + }, + }, { + hl('InvalidArrow', '->'), + hl('InvalidArrow', '->', 1), + hl('InvalidArrow', '->', 1), + }) + check_parsing('a -> b -> c -> d', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Arrow:0:1: ->', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Arrow:0:6: ->', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4: b', + { + 'Arrow:0:11: ->', + children = { + 'PlainIdentifier(scope=0,ident=c):0:9: c', + 'PlainIdentifier(scope=0,ident=d):0:14: d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> b -> c -> d', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'c', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'd', 1), + }) + check_parsing('{a -> b -> c}', 0, { + -- 0123456789012 + -- 0 1 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Arrow:0:2: ->', + children = { + { + 'Arrow:0:7: ->', + children = { + 'PlainIdentifier(scope=0,ident=b):0:5: b', + 'PlainIdentifier(scope=0,ident=c):0:10: c', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> c}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'c', 1), + hl('Lambda', '}'), + }) + check_parsing('{a: -> b}', 0, { + -- 012345678 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'Arrow:0:3: ->', + children = { + 'PlainIdentifier(scope=a,ident=):0:1:a:', + 'PlainIdentifier(scope=0,ident=b):0:6: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + + check_parsing('{a:b -> b}', 0, { + -- 0123456789 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=a,ident=b):0:1:a:b', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'b'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + + check_parsing('{a#b -> b}', 0, { + -- 0123456789 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=0,ident=a#b):0:1:a#b', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a#b'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + check_parsing('{a : b : c}', 0, { + -- 01234567890 + -- 0 1 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Colon:0:6: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4: b', + 'PlainIdentifier(scope=0,ident=c):0:8: c', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ': c}', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('Dict', '}'), + }) + check_parsing('{', 0, { + -- 0 + ast = { + 'UnknownFigure(\\di):0:0:{', + }, + err = { + arg = '{', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + }) + check_parsing('{a', 0, { + -- 01 + ast = { + { + 'UnknownFigure(\\di):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + err = { + arg = '{a', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + }) + check_parsing('{a,b', 0, { + -- 0123 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + }, + err = { + arg = '{a,b', + msg = 'E15: Missing closing figure brace for lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + }) + check_parsing('{a,b->', 0, { + -- 012345 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + 'Arrow:0:4:->', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + }) + check_parsing('{a,b->c', 0, { + -- 0123456 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6:c', + }, + }, + }, + }, + }, + err = { + arg = '{a,b->c', + msg = 'E15: Missing closing figure brace for lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('IdentifierName', 'c'), + }) + check_parsing('{a : b', 0, { + -- 012345 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, + }, + err = { + arg = '{a : b', + msg = 'E723: Missing end of Dictionary \'}\': %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + }) + check_parsing('{a : b,', 0, { + -- 0123456 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + }) + end) + itp('works with ternary operator', function() + check_parsing('a ? b : c', 0, { + -- 012345678 + ast = { + { + 'Ternary:0:1: ?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:5: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?', 1), + hl('IdentifierName', 'b', 1), + hl('TernaryColon', ':', 1), + hl('IdentifierName', 'c', 1), + }) + check_parsing('@a?@b?@c:@d:@e', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:11::', + children = { + { + 'Ternary:0:5:?', + children = { + 'Register(name=b):0:3:@b', + { + 'TernaryValue:0:8::', + children = { + 'Register(name=c):0:6:@c', + 'Register(name=d):0:9:@d', + }, + }, + }, + }, + 'Register(name=e):0:12:@e', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('Ternary', '?'), + hl('Register', '@c'), + hl('TernaryColon', ':'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + check_parsing('@a?@b:@c?@d:@e', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:5::', + children = { + 'Register(name=b):0:3:@b', + { + 'Ternary:0:8:?', + children = { + 'Register(name=c):0:6:@c', + { + 'TernaryValue:0:11::', + children = { + 'Register(name=d):0:9:@d', + 'Register(name=e):0:12:@e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', 0, { + -- 01234567890123456789012345678901 + -- 0 1 2 3 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:29::', + children = { + { + 'Ternary:0:5:?', + children = { + 'Register(name=b):0:3:@b', + { + 'TernaryValue:0:20::', + children = { + { + 'Ternary:0:8:?', + children = { + 'Register(name=c):0:6:@c', + { + 'TernaryValue:0:11::', + children = { + 'Register(name=d):0:9:@d', + { + 'Ternary:0:14:?', + children = { + 'Register(name=e):0:12:@e', + { + 'TernaryValue:0:17::', + children = { + 'Register(name=f):0:15:@f', + 'Register(name=g):0:18:@g', + }, + }, + }, + }, + }, + }, + }, + }, + { + 'Ternary:0:23:?', + children = { + 'Register(name=h):0:21:@h', + { + 'TernaryValue:0:26::', + children = { + 'Register(name=i):0:24:@i', + 'Register(name=j):0:27:@j', + }, + }, + }, + }, + }, + }, + }, + }, + 'Register(name=k):0:30:@k', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('Ternary', '?'), + hl('Register', '@c'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + hl('Ternary', '?'), + hl('Register', '@f'), + hl('TernaryColon', ':'), + hl('Register', '@g'), + hl('TernaryColon', ':'), + hl('Register', '@h'), + hl('Ternary', '?'), + hl('Register', '@i'), + hl('TernaryColon', ':'), + hl('Register', '@j'), + hl('TernaryColon', ':'), + hl('Register', '@k'), + }) + check_parsing('?', 0, { + -- 0 + ast = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + 'TernaryValue:0:0:?', + }, + }, + }, + err = { + arg = '?', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + }) + + check_parsing('?:', 0, { + -- 01 + ast = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + { + 'TernaryValue:0:1::', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = '?:', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + hl('InvalidTernaryColon', ':'), + }) + + check_parsing('?::', 0, { + -- 012 + ast = { + { + 'Colon:0:2::', + children = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + { + 'TernaryValue:0:1::', + children = { + 'Missing:0:1:', + 'Missing:0:2:', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '?::', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + hl('InvalidTernaryColon', ':'), + hl('InvalidColon', ':'), + }) + + check_parsing('a?b', 0, { + -- 012 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + err = { + arg = '?b', + msg = 'E109: Missing \':\' after \'?\': %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + }) + check_parsing('a?b:', 0, { + -- 0123 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:1:?', + children = { + 'PlainIdentifier(scope=b,ident=):0:2:b:', + }, + }, + }, + }, + }, + err = { + arg = '?b:', + msg = 'E109: Missing \':\' after \'?\': %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + }) + + check_parsing('a?b::c', 0, { + -- 012345 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:4::', + children = { + 'PlainIdentifier(scope=b,ident=):0:2:b:', + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('TernaryColon', ':'), + hl('IdentifierName', 'c'), + }) + + check_parsing('a?b :', 0, { + -- 01234 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + hl('TernaryColon', ':', 1), + }) + + check_parsing('(@a?@b:@c)?@d:@e', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:13::', + children = { + 'Register(name=d):0:11:@d', + 'Register(name=e):0:14:@e', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + + check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', 0, { + -- 01234567890123456789012345678901 + -- 0 1 2 3 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:21::', + children = { + { + 'Nested:0:11:(', + children = { + { + 'Ternary:0:14:?', + children = { + 'Register(name=d):0:12:@d', + { + 'TernaryValue:0:17::', + children = { + 'Register(name=e):0:15:@e', + 'Register(name=f):0:18:@f', + }, + }, + }, + }, + }, + }, + { + 'Nested:0:22:(', + children = { + { + 'Ternary:0:25:?', + children = { + 'Register(name=g):0:23:@g', + { + 'TernaryValue:0:28::', + children = { + 'Register(name=h):0:26:@h', + 'Register(name=i):0:29:@i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('NestingParenthesis', '('), + hl('Register', '@d'), + hl('Ternary', '?'), + hl('Register', '@e'), + hl('TernaryColon', ':'), + hl('Register', '@f'), + hl('NestingParenthesis', ')'), + hl('TernaryColon', ':'), + hl('NestingParenthesis', '('), + hl('Register', '@g'), + hl('Ternary', '?'), + hl('Register', '@h'), + hl('TernaryColon', ':'), + hl('Register', '@i'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', 0, { + -- 0123456789012345678901234567 + -- 0 1 2 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:19::', + children = { + { + 'Ternary:0:13:?', + children = { + 'Register(name=d):0:11:@d', + { + 'TernaryValue:0:16::', + children = { + 'Register(name=e):0:14:@e', + 'Register(name=f):0:17:@f', + }, + }, + }, + }, + { + 'Ternary:0:22:?', + children = { + 'Register(name=g):0:20:@g', + { + 'TernaryValue:0:25::', + children = { + 'Register(name=h):0:23:@h', + 'Register(name=i):0:26:@i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('Ternary', '?'), + hl('Register', '@e'), + hl('TernaryColon', ':'), + hl('Register', '@f'), + hl('TernaryColon', ':'), + hl('Register', '@g'), + hl('Ternary', '?'), + hl('Register', '@h'), + hl('TernaryColon', ':'), + hl('Register', '@i'), + }) + check_parsing('a?b{cdef}g:h', 0, { + -- 012345678901 + -- 0 1 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:10::', + children = { + { + 'ComplexIdentifier:0:3:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'ComplexIdentifier:0:9:', + children = { + { + 'CurlyBracesIdentifier(--i):0:3:{', + children = { + 'PlainIdentifier(scope=0,ident=cdef):0:4:cdef', + }, + }, + 'PlainIdentifier(scope=0,ident=g):0:9:g', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=h):0:11:h', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + hl('Curly', '{'), + hl('IdentifierName', 'cdef'), + hl('Curly', '}'), + hl('IdentifierName', 'g'), + hl('TernaryColon', ':'), + hl('IdentifierName', 'h'), + }) + check_parsing('a ? b : c : d', 0, { + -- 0123456789012 + -- 0 1 + ast = { + { + 'Colon:0:9: :', + children = { + { + 'Ternary:0:1: ?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:5: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:11: d', + }, + }, + }, + err = { + arg = ': d', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?', 1), + hl('IdentifierName', 'b', 1), + hl('TernaryColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'd', 1), + }) + end) + itp('works with comparison operators', function() + check_parsing('a == b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:1: ==', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ==? b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=IgnoreCase):0:1: ==?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('ComparisonModifier', '?'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ==# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=MatchCase):0:1: ==#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a !=# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=1,ccs=MatchCase):0:1: !=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '!=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a <=# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Greater,inv=1,ccs=MatchCase):0:1: <=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a >=# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=0,ccs=MatchCase):0:1: >=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '>=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ># b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Greater,inv=0,ccs=MatchCase):0:1: >#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '>', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a <# b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:1: <#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a is#b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Identical,inv=0,ccs=MatchCase):0:1: is#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'is', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b'), + }) + + check_parsing('a is?b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Identical,inv=0,ccs=IgnoreCase):0:1: is?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'is', 1), + hl('ComparisonModifier', '?'), + hl('IdentifierName', 'b'), + }) + + check_parsing('a isnot b', 0, { + -- 012345678 + ast = { + { + 'Comparison(type=Identical,inv=1,ccs=UseOption):0:1: isnot', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'isnot', 1), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a < b < c', 0, { + -- 012345678 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:5: <', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + }, + err = { + arg = ' < c', + msg = 'E15: Operator is not associative: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidComparison', '<', 1), + hl('IdentifierName', 'c', 1), + }) + + check_parsing('a < b <# c', 0, { + -- 012345678 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:5: <#', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:8: c', + }, + }, + }, + }, + }, + err = { + arg = ' <# c', + msg = 'E15: Operator is not associative: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidComparison', '<', 1), + hl('InvalidComparisonModifier', '#'), + hl('IdentifierName', 'c', 1), + }) + + check_parsing('a += b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3:=', + children = { + { + 'BinaryPlus:0:1: +', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Missing:0:3:', + }, + }, + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + err = { + arg = '= b', + msg = 'E15: Expected == or =~: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('BinaryPlus', '+', 1), + hl('InvalidComparison', '='), + hl('IdentifierName', 'b', 1), + }) + check_parsing('a + b == c + d', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:5: ==', + children = { + { + 'BinaryPlus:0:1: +', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + { + 'BinaryPlus:0:10: +', + children = { + 'PlainIdentifier(scope=0,ident=c):0:8: c', + 'PlainIdentifier(scope=0,ident=d):0:12: d', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'b', 1), + hl('Comparison', '==', 1), + hl('IdentifierName', 'c', 1), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'd', 1), + }) + check_parsing('+ a == + b', 0, { + -- 0123456789 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3: ==', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1: a', + }, + }, + { + 'UnaryPlus:0:6: +', + children = { + 'PlainIdentifier(scope=0,ident=b):0:8: b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('IdentifierName', 'a', 1), + hl('Comparison', '==', 1), + hl('UnaryPlus', '+', 1), + hl('IdentifierName', 'b', 1), + }) + end) + itp('works with concat/subscript', function() + check_parsing('.', 0, { + -- 0 + ast = { + { + 'ConcatOrSubscript:0:0:.', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = '.', + msg = 'E15: Unexpected dot: %.*s', + }, + }, { + hl('InvalidConcatOrSubscript', '.'), + }) + + check_parsing('a.', 0, { + -- 01 + ast = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + }) + + check_parsing('a.b', 0, { + -- 012 + ast = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=b):0:2:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', 'b'), + }) + + check_parsing('1.2', 0, { + -- 012 + ast = { + 'Float(val=1.200000e+00):0:0:1.2', + }, + }, { + hl('Float', '1.2'), + }) + + check_parsing('1.2 + 1.3e-5', 0, { + -- 012345678901 + -- 0 1 + ast = { + { + 'BinaryPlus:0:3: +', + children = { + 'Float(val=1.200000e+00):0:0:1.2', + 'Float(val=1.300000e-05):0:5: 1.3e-5', + }, + }, + }, + }, { + hl('Float', '1.2'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.3e-5', 1), + }) + + check_parsing('a . 1.2 + 1.3e-5', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'BinaryPlus:0:7: +', + children = { + { + 'Concat:0:1: .', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ConcatOrSubscript:0:5:.', + children = { + 'Integer(val=1):0:3: 1', + 'PlainKey(key=2):0:6:2', + }, + }, + }, + }, + 'Float(val=1.300000e-05):0:9: 1.3e-5', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.3e-5', 1), + }) + + check_parsing('1.3e-5 + 1.2 . a', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Concat:0:12: .', + children = { + { + 'BinaryPlus:0:6: +', + children = { + 'Float(val=1.300000e-05):0:0:1.3e-5', + 'Float(val=1.200000e+00):0:8: 1.2', + }, + }, + 'PlainIdentifier(scope=0,ident=a):0:14: a', + }, + }, + }, + }, { + hl('Float', '1.3e-5'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.2', 1), + hl('Concat', '.', 1), + hl('IdentifierName', 'a', 1), + }) + + check_parsing('1.3e-5 + a . 1.2', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Concat:0:10: .', + children = { + { + 'BinaryPlus:0:6: +', + children = { + 'Float(val=1.300000e-05):0:0:1.3e-5', + 'PlainIdentifier(scope=0,ident=a):0:8: a', + }, + }, + { + 'ConcatOrSubscript:0:14:.', + children = { + 'Integer(val=1):0:12: 1', + 'PlainKey(key=2):0:15:2', + }, + }, + }, + }, + }, + }, { + hl('Float', '1.3e-5'), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'a', 1), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('1.2.3', 0, { + -- 01234 + ast = { + { + 'ConcatOrSubscript:0:3:.', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'Integer(val=1):0:0:1', + 'PlainKey(key=2):0:2:2', + }, + }, + 'PlainKey(key=3):0:4:3', + }, + }, + }, + }, { + hl('Number', '1'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '3'), + }) + + check_parsing('a.1.2', 0, { + -- 01234 + ast = { + { + 'ConcatOrSubscript:0:3:.', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=1):0:2:1', + }, + }, + 'PlainKey(key=2):0:4:2', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '1'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('a . 1.2', 0, { + -- 0123456 + ast = { + { + 'Concat:0:1: .', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ConcatOrSubscript:0:5:.', + children = { + 'Integer(val=1):0:3: 1', + 'PlainKey(key=2):0:6:2', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('+a . +b', 0, { + -- 0123456 + ast = { + { + 'Concat:0:2: .', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'UnaryPlus:0:4: +', + children = { + 'PlainIdentifier(scope=0,ident=b):0:6:b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('UnaryPlus', '+', 1), + hl('IdentifierName', 'b'), + }) + + check_parsing('a. b', 0, { + -- 0123 + ast = { + { + 'Concat:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a. 1', 0, { + -- 0123 + ast = { + { + 'Concat:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Integer(val=1):0:2: 1', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('Number', '1', 1), + }) + end) + itp('works with bracket subscripts', function() + check_parsing(':', 0, { + -- 0 + ast = { + { + 'Colon:0:0::', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ':', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('InvalidColon', ':'), + }) + check_parsing('a[]', 0, { + -- 012 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Expected value, got closing bracket: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('InvalidSubscriptBracket', ']'), + }) + check_parsing('a[b:]', 0, { + -- 01234 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=b,ident=):0:2:b:', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[b:c]', 0, { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=b,ident=c):0:2:b:c', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', ']'), + }) + check_parsing('a[b : c]', 0, { + -- 01234567 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + 'PlainIdentifier(scope=0,ident=c):0:5: c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[: b]', 0, { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:2::', + children = { + 'Missing:0:2:', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('SubscriptColon', ':'), + hl('IdentifierName', 'b', 1), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[b :]', 0, { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptColon', ':', 1), + hl('SubscriptBracket', ']'), + }) + check_parsing('a[b][c][d](e)(f)(g)', 0, { + -- 0123456789012345678 + -- 0 1 + ast = { + { + 'Call:0:16:(', + children = { + { + 'Call:0:13:(', + children = { + { + 'Call:0:10:(', + children = { + { + 'Subscript:0:7:[', + children = { + { + 'Subscript:0:4:[', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:8:d', + }, + }, + 'PlainIdentifier(scope=0,ident=e):0:11:e', + }, + }, + 'PlainIdentifier(scope=0,ident=f):0:14:f', + }, + }, + 'PlainIdentifier(scope=0,ident=g):0:17:g', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'd'), + hl('SubscriptBracket', ']'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'e'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'f'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'g'), + hl('CallingParenthesis', ')'), + }) + check_parsing('{a}{b}{c}[d][e][f]', 0, { + -- 012345678901234567 + -- 0 1 + ast = { + { + 'Subscript:0:15:[', + children = { + { + 'Subscript:0:12:[', + children = { + { + 'Subscript:0:9:[', + children = { + { + 'ComplexIdentifier:0:3:', + children = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier(--i):0:3:{', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + }, + }, + { + 'CurlyBracesIdentifier(--i):0:6:{', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7:c', + }, + }, + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:10:d', + }, + }, + 'PlainIdentifier(scope=0,ident=e):0:13:e', + }, + }, + 'PlainIdentifier(scope=0,ident=f):0:16:f', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'c'), + hl('Curly', '}'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'd'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'e'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'f'), + hl('SubscriptBracket', ']'), + }) + end) + itp('supports list literals', function() + check_parsing('[]', 0, { + -- 01 + ast = { + 'ListLiteral:0:0:[', + }, + }, { + hl('List', '['), + hl('List', ']'), + }) + + check_parsing('[a]', 0, { + -- 012 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('List', ']'), + }) + + check_parsing('[a, b]', 0, { + -- 012345 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('List', ']'), + }) + + check_parsing('[a, b, c]', 0, { + -- 012345678 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('List', ']'), + }) + + check_parsing('[a, b, c, ]', 0, { + -- 01234567890 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('Comma', ','), + hl('List', ']', 1), + }) + + check_parsing('[a : b, c : d]', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + { + 'Colon:0:9: :', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7: c', + 'PlainIdentifier(scope=0,ident=d):0:11: d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ': b, c : d]', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'd', 1), + hl('List', ']'), + }) + + check_parsing(']', 0, { + -- 0 + ast = { + 'ListLiteral:0:0:', + }, + err = { + arg = ']', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidList', ']'), + }) + + check_parsing('a]', 0, { + -- 01 + ast = { + { + 'ListLiteral:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidList', ']'), + }) + + check_parsing('[] []', 0, { + -- 01234 + ast = { + { + 'OpMissing:0:2:', + children = { + 'ListLiteral:0:0:[', + 'ListLiteral:0:2: [', + }, + }, + }, + err = { + arg = '[]', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('List', '['), + hl('List', ']'), + hl('InvalidSpacing', ' '), + hl('List', '['), + hl('List', ']'), + }) + + check_parsing('[][]', 0, { + -- 0123 + ast = { + { + 'Subscript:0:2:[', + children = { + 'ListLiteral:0:0:[', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Expected value, got closing bracket: %.*s', + }, + }, { + hl('List', '['), + hl('List', ']'), + hl('SubscriptBracket', '['), + hl('InvalidSubscriptBracket', ']'), + }) + + check_parsing('[', 0, { + -- 0 + ast = { + 'ListLiteral:0:0:[', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('List', '['), + }) + + check_parsing('[1', 0, { + -- 01 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'Integer(val=1):0:1:1', + }, + }, + }, + err = { + arg = '[1', + msg = 'E697: Missing end of List \']\': %.*s', + }, + }, { + hl('List', '['), + hl('Number', '1'), + }) + end) + itp('works with strings', function() + check_parsing('\'abc\'', 0, { + -- 01234 + ast = { + 'SingleQuotedString(val="abc"):0:0:\'abc\'', + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedBody', 'abc'), + hl('SingleQuote', '\''), + }) + check_parsing('"abc"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="abc"):0:0:"abc"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', 'abc'), + hl('DoubleQuote', '"'), + }) + check_parsing('\'\'', 0, { + -- 01 + ast = { + 'SingleQuotedString(val=NULL):0:0:\'\'', + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuote', '\''), + }) + check_parsing('""', 0, { + -- 01 + ast = { + 'DoubleQuotedString(val=NULL):0:0:""', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuote', '"'), + }) + check_parsing('"', 0, { + -- 0 + ast = { + 'DoubleQuotedString(val=NULL):0:0:"', + }, + err = { + arg = '"', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + }) + check_parsing('\'', 0, { + -- 0 + ast = { + 'SingleQuotedString(val=NULL):0:0:\'', + }, + err = { + arg = '\'', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + }) + check_parsing('"a', 0, { + -- 01 + ast = { + 'DoubleQuotedString(val="a"):0:0:"a', + }, + err = { + arg = '"a', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedBody', 'a'), + }) + check_parsing('\'a', 0, { + -- 01 + ast = { + 'SingleQuotedString(val="a"):0:0:\'a', + }, + err = { + arg = '\'a', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + hl('InvalidSingleQuotedBody', 'a'), + }) + check_parsing('\'abc\'\'def\'', 0, { + -- 0123456789 + ast = { + 'SingleQuotedString(val="abc\'def"):0:0:\'abc\'\'def\'', + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedBody', 'abc'), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'def'), + hl('SingleQuote', '\''), + }) + check_parsing('\'abc\'\'', 0, { + -- 012345 + ast = { + 'SingleQuotedString(val="abc\'"):0:0:\'abc\'\'', + }, + err = { + arg = '\'abc\'\'', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + hl('InvalidSingleQuotedBody', 'abc'), + hl('InvalidSingleQuotedQuote', '\'\''), + }) + check_parsing('\'\'\'\'\'\'\'\'', 0, { + -- 01234567 + ast = { + 'SingleQuotedString(val="\'\'\'"):0:0:\'\'\'\'\'\'\'\'', + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuote', '\''), + }) + check_parsing('\'\'\'a\'\'\'\'bc\'', 0, { + -- 01234567890 + -- 0 1 + ast = { + 'SingleQuotedString(val="\'a\'\'bc"):0:0:\'\'\'a\'\'\'\'bc\'', + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'a'), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'bc'), + hl('SingleQuote', '\''), + }) + check_parsing('"\\"\\"\\"\\""', 0, { + -- 0123456789 + ast = { + 'DoubleQuotedString(val="\\"\\"\\"\\""):0:0:"\\"\\"\\"\\""', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuote', '"'), + }) + check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', 0, { + -- 0123456789012345678901234 + -- 0 1 2 + ast = { + 'DoubleQuotedString(val="abc\\"def\\"ghi\\"jkl\\"mno"):0:0:"abc\\"def\\"ghi\\"jkl\\"mno"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', 'abc'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'def'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'ghi'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'jkl'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'mno'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\b\\e\\f\\r\\t\\\\"', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + [[DoubleQuotedString(val="\8\27\12\13\9\\"):0:0:"\b\e\f\r\t\\"]], + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\b'), + hl('DoubleQuotedEscape', '\\e'), + hl('DoubleQuotedEscape', '\\f'), + hl('DoubleQuotedEscape', '\\r'), + hl('DoubleQuotedEscape', '\\t'), + hl('DoubleQuotedEscape', '\\\\'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\n\n"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="\\\n\\\n"):0:0:"\\n\n"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\n'), + hl('DoubleQuotedBody', '\n'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\x00"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\x00"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\xFF"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\255"):0:0:"\\xFF"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xFF'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\xF"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\15"):0:0:"\\xF"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xF'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\u00AB"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="«"):0:0:"\\u00AB"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u00AB'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\U000000AB"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="«"):0:0:"\\U000000AB"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000000AB'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\x"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="x"):0:0:"\\x"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\x'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="x"):0:0:"\\x', + }, + err = { + arg = '"\\x', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\x'), + }) + + check_parsing('"\\xF', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="\\15"):0:0:"\\xF', + }, + err = { + arg = '"\\xF', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedEscape', '\\xF'), + }) + + check_parsing('"\\u"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="u"):0:0:"\\u"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\u'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="u"):0:0:"\\u', + }, + err = { + arg = '"\\u', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\u'), + }) + + check_parsing('"\\U', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="U"):0:0:"\\U', + }, + err = { + arg = '"\\U', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + }) + + check_parsing('"\\U"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="U"):0:0:"\\U"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\U'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\xFX"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\15X"):0:0:"\\xFX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xF'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\XFX"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\15X"):0:0:"\\XFX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\XF'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\xX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="xX"):0:0:"\\xX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\x'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\XX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="XX"):0:0:"\\XX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\X'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\uX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="uX"):0:0:"\\uX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\u'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\UX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="UX"):0:0:"\\UX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\U'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\x0X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\X0X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u0X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U0X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\x00X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\X00X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u00X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U00X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u0000X"', 0, { + -- 012345678 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u0000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0000X"', 0, { + -- 012345678 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00000X"', 0, { + -- 0123456789 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000000X"', 0, { + -- 01234567890 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U000000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0000000X"', 0, { + -- 012345678901 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00000000X"', 0, { + -- 0123456789012 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\x000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\X000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X00'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u00000X"', 0, { + -- 0123456789 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\u00000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0000'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000000000X"', 0, { + -- 01234567890123 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\U000000000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000000'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\0"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\0'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\00"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\00"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\00'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\000"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\000"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0000"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0000"):0:0:"\\0000"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedBody', '0'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\8"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="8"):0:0:"\\8"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\08"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="\\0008"):0:0:"\\08"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\0'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\008"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0008"):0:0:"\\008"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\00'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0008"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0008"):0:0:"\\0008"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\777"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\255"):0:0:"\\777"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\777'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\050"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\40"):0:0:"\\050"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\050'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\<C-u>"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\21"):0:0:"\\<C-u>"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\<C-u>'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\<', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="<"):0:0:"\\<', + }, + err = { + arg = '"\\<', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\<'), + }) + + check_parsing('"\\<"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="<"):0:0:"\\<"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\<'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\<C-u"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="<C-u"):0:0:"\\<C-u"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\<'), + hl('DoubleQuotedBody', 'C-u'), + hl('DoubleQuote', '"'), + }) + end) + itp('works with multiplication-like operators', function() + check_parsing('2+2*2', 0, { + -- 01234 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Multiplication:0:3:*', + children = { + 'Integer(val=2):0:2:2', + 'Integer(val=2):0:4:2', + }, + }, + }, + }, + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Multiplication', '*'), + hl('Number', '2'), + }) + + check_parsing('2+2*', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Multiplication:0:3:*', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Multiplication', '*'), + }) + + check_parsing('2+*2', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Multiplication:0:2:*', + children = { + 'Missing:0:2:', + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + err = { + arg = '*2', + msg = 'E15: Unexpected multiplication-like operator: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('InvalidMultiplication', '*'), + hl('Number', '2'), + }) + + check_parsing('2+2/2', 0, { + -- 01234 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Division:0:3:/', + children = { + 'Integer(val=2):0:2:2', + 'Integer(val=2):0:4:2', + }, + }, + }, + }, + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Division', '/'), + hl('Number', '2'), + }) + + check_parsing('2+2/', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Division:0:3:/', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Division', '/'), + }) + + check_parsing('2+/2', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Division:0:2:/', + children = { + 'Missing:0:2:', + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + err = { + arg = '/2', + msg = 'E15: Unexpected multiplication-like operator: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('InvalidDivision', '/'), + hl('Number', '2'), + }) + + check_parsing('2+2%2', 0, { + -- 01234 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Mod:0:3:%', + children = { + 'Integer(val=2):0:2:2', + 'Integer(val=2):0:4:2', + }, + }, + }, + }, + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Mod', '%'), + hl('Number', '2'), + }) + + check_parsing('2+2%', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Mod:0:3:%', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Mod', '%'), + }) + + check_parsing('2+%2', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Mod:0:2:%', + children = { + 'Missing:0:2:', + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + err = { + arg = '%2', + msg = 'E15: Unexpected multiplication-like operator: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('InvalidMod', '%'), + hl('Number', '2'), + }) + end) + itp('works with -', function() + check_parsing('@a', 0, { + ast = { + 'Register(name=a):0:0:@a', + }, + }, { + hl('Register', '@a'), + }) + check_parsing('-@a', 0, { + ast = { + { + 'UnaryMinus:0:0:-', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('UnaryMinus', '-'), + hl('Register', '@a'), + }) + check_parsing('@a-@b', 0, { + ast = { + { + 'BinaryMinus:0:2:-', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('Register', '@b'), + }) + check_parsing('@a-@b-@c', 0, { + ast = { + { + 'BinaryMinus:0:5:-', + children = { + { + 'BinaryMinus:0:2:-', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + 'Register(name=c):0:6:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('Register', '@b'), + hl('BinaryMinus', '-'), + hl('Register', '@c'), + }) + check_parsing('-@a-@b', 0, { + ast = { + { + 'BinaryMinus:0:3:-', + children = { + { + 'UnaryMinus:0:0:-', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:4:@b', + }, + }, + }, + }, { + hl('UnaryMinus', '-'), + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('Register', '@b'), + }) + check_parsing('-@a--@b', 0, { + ast = { + { + 'BinaryMinus:0:3:-', + children = { + { + 'UnaryMinus:0:0:-', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'UnaryMinus:0:4:-', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryMinus', '-'), + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('UnaryMinus', '-'), + hl('Register', '@b'), + }) + check_parsing('@a@b', 0, { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:2:@b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidRegister', '@b'), + }) + check_parsing(' @a \t @b', 0, { + ast = { + { + 'OpMissing:0:3:', + children = { + 'Register(name=a):0:0: @a', + 'Register(name=b):0:3: \t @b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a', 1), + hl('InvalidSpacing', ' \t '), + hl('Register', '@b'), + }) + check_parsing('-', 0, { + ast = { + 'UnaryMinus:0:0:-', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryMinus', '-'), + }) + check_parsing(' -', 0, { + ast = { + 'UnaryMinus:0:0: -', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryMinus', '-', 1), + }) + check_parsing('@a- ', 0, { + ast = { + { + 'BinaryMinus:0:2:-', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Register', '@a'), + hl('BinaryMinus', '-'), + }) + end) + itp('works with logical operators', function() + check_parsing('a && b || c && d', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Or:0:6: ||', + children = { + { + 'And:0:1: &&', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + { + 'And:0:11: &&', + children = { + 'PlainIdentifier(scope=0,ident=c):0:9: c', + 'PlainIdentifier(scope=0,ident=d):0:14: d', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('And', '&&', 1), + hl('IdentifierName', 'b', 1), + hl('Or', '||', 1), + hl('IdentifierName', 'c', 1), + hl('And', '&&', 1), + hl('IdentifierName', 'd', 1), + }) + + check_parsing('&& a', 0, { + -- 0123 + ast = { + { + 'And:0:0:&&', + children = { + 'Missing:0:0:', + 'PlainIdentifier(scope=0,ident=a):0:2: a', + }, + }, + }, + err = { + arg = '&& a', + msg = 'E15: Unexpected and operator: %.*s', + }, + }, { + hl('InvalidAnd', '&&'), + hl('IdentifierName', 'a', 1), + }) + + check_parsing('|| a', 0, { + -- 0123 + ast = { + { + 'Or:0:0:||', + children = { + 'Missing:0:0:', + 'PlainIdentifier(scope=0,ident=a):0:2: a', + }, + }, + }, + err = { + arg = '|| a', + msg = 'E15: Unexpected or operator: %.*s', + }, + }, { + hl('InvalidOr', '||'), + hl('IdentifierName', 'a', 1), + }) + + check_parsing('a||', 0, { + -- 012 + ast = { + { + 'Or:0:1:||', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Or', '||'), + }) + + check_parsing('a&&', 0, { + -- 012 + ast = { + { + 'And:0:1:&&', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('And', '&&'), + }) + + check_parsing('(&&)', 0, { + -- 0123 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'And:0:1:&&', + children = { + 'Missing:0:1:', + 'Missing:0:3:', + }, + }, + }, + }, + }, + err = { + arg = '&&)', + msg = 'E15: Unexpected and operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidAnd', '&&'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(||)', 0, { + -- 0123 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Or:0:1:||', + children = { + 'Missing:0:1:', + 'Missing:0:3:', + }, + }, + }, + }, + }, + err = { + arg = '||)', + msg = 'E15: Unexpected or operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidOr', '||'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(a||)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Or:0:2:||', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('IdentifierName', 'a'), + hl('Or', '||'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(a&&)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'And:0:2:&&', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('IdentifierName', 'a'), + hl('And', '&&'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(&&a)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'And:0:1:&&', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:3:a', + }, + }, + }, + }, + }, + err = { + arg = '&&a)', + msg = 'E15: Unexpected and operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidAnd', '&&'), + hl('IdentifierName', 'a'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(||a)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Or:0:1:||', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:3:a', + }, + }, + }, + }, + }, + err = { + arg = '||a)', + msg = 'E15: Unexpected or operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidOr', '||'), + hl('IdentifierName', 'a'), + hl('NestingParenthesis', ')'), + }) + end) + itp('works with &opt', function() + check_parsing('&', 0, { + -- 0 + ast = { + 'Option(scope=0,ident=):0:0:&', + }, + err = { + arg = '&', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('InvalidOptionSigil', '&'), + }) + + check_parsing('&opt', 0, { + -- 0123 + ast = { + 'Option(scope=0,ident=opt):0:0:&opt', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 'opt'), + }) + + check_parsing('&l:opt', 0, { + -- 012345 + ast = { + 'Option(scope=l,ident=opt):0:0:&l:opt', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionScope', 'l'), + hl('OptionScopeDelimiter', ':'), + hl('OptionName', 'opt'), + }) + + check_parsing('&g:opt', 0, { + -- 012345 + ast = { + 'Option(scope=g,ident=opt):0:0:&g:opt', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionScope', 'g'), + hl('OptionScopeDelimiter', ':'), + hl('OptionName', 'opt'), + }) + + check_parsing('&s:opt', 0, { + -- 012345 + ast = { + { + 'Colon:0:2::', + children = { + 'Option(scope=0,ident=s):0:0:&s', + 'PlainIdentifier(scope=0,ident=opt):0:3:opt', + }, + }, + }, + err = { + arg = ':opt', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 's'), + hl('InvalidColon', ':'), + hl('IdentifierName', 'opt'), + }) + + check_parsing('& ', 0, { + -- 01 + ast = { + 'Option(scope=0,ident=):0:0:&', + }, + err = { + arg = '& ', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('InvalidOptionSigil', '&'), + }) + + check_parsing('&-', 0, { + -- 01 + ast = { + { + 'BinaryMinus:0:1:-', + children = { + 'Option(scope=0,ident=):0:0:&', + }, + }, + }, + err = { + arg = '&-', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('InvalidOptionSigil', '&'), + hl('BinaryMinus', '-'), + }) + + check_parsing('&A', 0, { + -- 01 + ast = { + 'Option(scope=0,ident=A):0:0:&A', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 'A'), + }) + + check_parsing('&xxx_yyy', 0, { + -- 01234567 + ast = { + { + 'OpMissing:0:4:', + children = { + 'Option(scope=0,ident=xxx):0:0:&xxx', + 'PlainIdentifier(scope=0,ident=_yyy):0:4:_yyy', + }, + }, + }, + err = { + arg = '_yyy', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 'xxx'), + hl('InvalidIdentifierName', '_yyy'), + }) + + check_parsing('(1+&)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Integer(val=1):0:1:1', + 'Option(scope=0,ident=):0:3:&', + }, + }, + }, + }, + }, + err = { + arg = '&)', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Number', '1'), + hl('BinaryPlus', '+'), + hl('InvalidOptionSigil', '&'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(&+1)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Option(scope=0,ident=):0:1:&', + 'Integer(val=1):0:3:1', + }, + }, + }, + }, + }, + err = { + arg = '&+1)', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidOptionSigil', '&'), + hl('BinaryPlus', '+'), + hl('Number', '1'), + hl('NestingParenthesis', ')'), + }) + end) + itp('works with $ENV', function() + check_parsing('$', 0, { + -- 0 + ast = { + 'Environment(ident=):0:0:$', + }, + err = { + arg = '$', + msg = 'E15: Environment variable name missing', + }, + }, { + hl('InvalidEnvironmentSigil', '$'), + }) + + check_parsing('$g:A', 0, { + -- 0123 + ast = { + { + 'Colon:0:2::', + children = { + 'Environment(ident=g):0:0:$g', + 'PlainIdentifier(scope=0,ident=A):0:3:A', + }, + }, + }, + err = { + arg = ':A', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'g'), + hl('InvalidColon', ':'), + hl('IdentifierName', 'A'), + }) + + check_parsing('$A', 0, { + -- 01 + ast = { + 'Environment(ident=A):0:0:$A', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'A'), + }) + + check_parsing('$ABC', 0, { + -- 0123 + ast = { + 'Environment(ident=ABC):0:0:$ABC', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'ABC'), + }) + + check_parsing('(1+$)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Integer(val=1):0:1:1', + 'Environment(ident=):0:3:$', + }, + }, + }, + }, + }, + err = { + arg = '$)', + msg = 'E15: Environment variable name missing', + }, + }, { + hl('NestingParenthesis', '('), + hl('Number', '1'), + hl('BinaryPlus', '+'), + hl('InvalidEnvironmentSigil', '$'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('($+1)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Environment(ident=):0:1:$', + 'Integer(val=1):0:3:1', + }, + }, + }, + }, + }, + err = { + arg = '$+1)', + msg = 'E15: Environment variable name missing', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidEnvironmentSigil', '$'), + hl('BinaryPlus', '+'), + hl('Number', '1'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('$_ABC', 0, { + -- 01234 + ast = { + 'Environment(ident=_ABC):0:0:$_ABC', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', '_ABC'), + }) + + check_parsing('$_', 0, { + -- 01 + ast = { + 'Environment(ident=_):0:0:$_', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', '_'), + }) + + check_parsing('$ABC_DEF', 0, { + -- 01234567 + ast = { + 'Environment(ident=ABC_DEF):0:0:$ABC_DEF', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'ABC_DEF'), + }) + end) + itp('works with unary !', function() + check_parsing('!', 0, { + -- 0 + ast = { + 'Not:0:0:!', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Not', '!'), + }) + + check_parsing('!!', 0, { + -- 01 + ast = { + { + 'Not:0:0:!', + children = { + 'Not:0:1:!', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Not', '!'), + hl('Not', '!'), + }) + + check_parsing('!!1', 0, { + -- 012 + ast = { + { + 'Not:0:0:!', + children = { + { + 'Not:0:1:!', + children = { + 'Integer(val=1):0:2:1', + }, + }, + }, + }, + }, + }, { + hl('Not', '!'), + hl('Not', '!'), + hl('Number', '1'), + }) + + check_parsing('!1', 0, { + -- 01 + ast = { + { + 'Not:0:0:!', + children = { + 'Integer(val=1):0:1:1', + }, + }, + }, + }, { + hl('Not', '!'), + hl('Number', '1'), + }) + + check_parsing('(!1)', 0, { + -- 0123 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Not:0:1:!', + children = { + 'Integer(val=1):0:2:1', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Not', '!'), + hl('Number', '1'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(!)', 0, { + -- 012 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Not:0:1:!', + children = { + 'Missing:0:2:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Not', '!'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(1!2)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'OpMissing:0:2:', + children = { + 'Integer(val=1):0:1:1', + { + 'Not:0:2:!', + children = { + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '!2)', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Number', '1'), + hl('InvalidNot', '!'), + hl('Number', '2'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('1!2', 0, { + -- 012 + ast = { + { + 'OpMissing:0:1:', + children = { + 'Integer(val=1):0:0:1', + { + 'Not:0:1:!', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '!2', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Number', '1'), + hl('InvalidNot', '!'), + hl('Number', '2'), + }) + end) + itp('highlights numbers with prefix', function() + check_parsing('0xABCDEF', 0, { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0xABCDEF', + }, + }, { + hl('NumberPrefix', '0x'), + hl('Number', 'ABCDEF'), + }) + + check_parsing('0Xabcdef', 0, { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0Xabcdef', + }, + }, { + hl('NumberPrefix', '0X'), + hl('Number', 'abcdef'), + }) + + check_parsing('0XABCDEF', 0, { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0XABCDEF', + }, + }, { + hl('NumberPrefix', '0X'), + hl('Number', 'ABCDEF'), + }) + + check_parsing('0xabcdef', 0, { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0xabcdef', + }, + }, { + hl('NumberPrefix', '0x'), + hl('Number', 'abcdef'), + }) + + check_parsing('0b001', 0, { + -- 01234 + ast = { + 'Integer(val=1):0:0:0b001', + }, + }, { + hl('NumberPrefix', '0b'), + hl('Number', '001'), + }) + + check_parsing('0B001', 0, { + -- 01234 + ast = { + 'Integer(val=1):0:0:0B001', + }, + }, { + hl('NumberPrefix', '0B'), + hl('Number', '001'), + }) + + check_parsing('0B00', 0, { + -- 0123 + ast = { + 'Integer(val=0):0:0:0B00', + }, + }, { + hl('NumberPrefix', '0B'), + hl('Number', '00'), + }) + + check_parsing('00', 0, { + -- 01 + ast = { + 'Integer(val=0):0:0:00', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '0'), + }) + + check_parsing('001', 0, { + -- 012 + ast = { + 'Integer(val=1):0:0:001', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '01'), + }) + + check_parsing('01', 0, { + -- 01 + ast = { + 'Integer(val=1):0:0:01', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '1'), + }) + + check_parsing('1', 0, { + -- 0 + ast = { + 'Integer(val=1):0:0:1', + }, + }, { + hl('Number', '1'), + }) + end) + itp('works (KLEE tests)', function() + check_parsing('\0002&A:\000', 0, { + ast = nil, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + }) + check_parsing({data='01', size=1}, 0, { + ast = { + 'Integer(val=0):0:0:0', + }, + }, { + hl('Number', '0'), + }) + check_parsing({data='001', size=2}, 0, { + ast = { + 'Integer(val=0):0:0:00', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '0'), + }) + check_parsing('"\\U\\', 0, { + -- 0123 + ast = { + [[DoubleQuotedString(val="U\\"):0:0:"\U\]], + }, + err = { + arg = '"\\U\\', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + hl('InvalidDoubleQuotedBody', '\\'), + }) + check_parsing('"\\U', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="U"):0:0:"\\U', + }, + err = { + arg = '"\\U', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + }) + check_parsing('|"\\U\\', 2, { + -- 01234 + ast = { + { + 'Or:0:0:|', + children = { + 'Missing:0:0:', + 'DoubleQuotedString(val="U\\\\"):0:1:"\\U\\', + }, + }, + }, + err = { + arg = '|"\\U\\', + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, { + hl('InvalidOr', '|'), + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + hl('InvalidDoubleQuotedBody', '\\'), + }) + check_parsing('|"\\e"', 2, { + -- 01234 + ast = { + { + 'Or:0:0:|', + children = { + 'Missing:0:0:', + 'DoubleQuotedString(val="\\27"):0:1:"\\e"', + }, + }, + }, + err = { + arg = '|"\\e"', + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, { + hl('InvalidOr', '|'), + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\e'), + hl('DoubleQuote', '"'), + }) + check_parsing('|\029', 2, { + -- 01 + ast = { + { + 'Or:0:0:|', + children = { + 'Missing:0:0:', + 'PlainIdentifier(scope=0,ident=\029):0:1:\029', + }, + }, + }, + err = { + arg = '|\029', + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, { + hl('InvalidOr', '|'), + hl('InvalidIdentifierName', '\029'), + }) + check_parsing('"\\<', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="<"):0:0:"\\<', + }, + err = { + arg = '"\\<', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\<'), + }) + check_parsing('"\\1', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="\\1"):0:0:"\\1', + }, + err = { + arg = '"\\1', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedEscape', '\\1'), + }) + check_parsing('}l', 0, { + -- 01 + ast = { + { + 'OpMissing:0:1:', + children = { + 'UnknownFigure(---):0:0:', + 'PlainIdentifier(scope=0,ident=l):0:1:l', + }, + }, + }, + err = { + arg = '}l', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidFigureBrace', '}'), + hl('InvalidIdentifierName', 'l'), + }) + check_parsing(':?\000\000\000\000\000\000\000', 0, { + ast = { + { + 'Colon:0:0::', + children = { + 'Missing:0:0:', + { + 'Ternary:0:1:?', + children = { + 'Missing:0:1:', + 'TernaryValue:0:1:?', + }, + }, + }, + }, + }, + err = { + arg = ':?', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('InvalidColon', ':'), + hl('InvalidTernary', '?'), + }) + end) + -- FIXME: check flag effects +end) diff --git a/test/unit/viml/helpers.lua b/test/unit/viml/helpers.lua new file mode 100644 index 0000000000..c965cacb29 --- /dev/null +++ b/test/unit/viml/helpers.lua @@ -0,0 +1,129 @@ +local helpers = require('test.unit.helpers')(nil) + +local ffi = helpers.ffi +local cimport = helpers.cimport +local kvi_new = helpers.kvi_new +local kvi_init = helpers.kvi_init +local conv_enum = helpers.conv_enum +local child_call_once = helpers.child_call_once +local make_enum_conv_tab = helpers.make_enum_conv_tab + +local lib = cimport('./src/nvim/viml/parser/expressions.h') + +local function new_pstate(strings) + local strings_idx = 0 + local function get_line(_, ret_pline) + strings_idx = strings_idx + 1 + local str = strings[strings_idx] + local data, size + if type(str) == 'string' then + data = str + size = #str + elseif type(str) == 'nil' then + data = nil + size = 0 + elseif type(str) == 'table' then + data = str.data + size = str.size + elseif type(str) == 'function' then + data, size = str() + size = size or 0 + end + ret_pline.data = data + ret_pline.size = size + ret_pline.allocated = false + end + local pline_init = { + data = nil, + size = 0, + allocated = false, + } + local state = { + reader = { + get_line = get_line, + cookie = nil, + conv = { + vc_type = 0, + vc_factor = 1, + vc_fail = false, + }, + }, + pos = { line = 0, col = 0 }, + colors = kvi_new('ParserHighlight'), + can_continuate = false, + } + local ret = ffi.new('ParserState', state) + kvi_init(ret.reader.lines) + kvi_init(ret.stack) + return ret +end + +local function intchar2lua(ch) + ch = tonumber(ch) + return (20 <= ch and ch < 127) and ('%c'):format(ch) or ch +end + +local function pline2lua(pline) + return ffi.string(pline.data, pline.size) +end + +local function pstate_str(pstate, start, len) + local str = nil + local err = nil + if start.line < pstate.reader.lines.size then + local pstr = pline2lua(pstate.reader.lines.items[start.line]) + if start.col >= #pstr then + err = 'start.col >= #pstr' + else + str = pstr:sub(tonumber(start.col) + 1, tonumber(start.col + len)) + end + else + err = 'start.line >= pstate.reader.lines.size' + end + return str, err +end + +local function pstate_set_str(pstate, start, len, ret) + ret = ret or {} + ret.start = { + line = tonumber(start.line), + col = tonumber(start.col) + } + ret.len = tonumber(len) + ret.str, ret.error = pstate_str(pstate, start, len) + return ret +end + +local eltkn_cmp_type_tab +make_enum_conv_tab(lib, { + 'kExprCmpEqual', + 'kExprCmpMatches', + 'kExprCmpGreater', + 'kExprCmpGreaterOrEqual', + 'kExprCmpIdentical', +}, 'kExprCmp', function(ret) eltkn_cmp_type_tab = ret end) + +local function conv_cmp_type(typ) + return conv_enum(eltkn_cmp_type_tab, typ) +end + +local ccs_tab +make_enum_conv_tab(lib, { + 'kCCStrategyUseOption', + 'kCCStrategyMatchCase', + 'kCCStrategyIgnoreCase', +}, 'kCCStrategy', function(ret) ccs_tab = ret end) + +local function conv_ccs(ccs) + return conv_enum(ccs_tab, ccs) +end + +return { + conv_ccs = conv_ccs, + pline2lua = pline2lua, + pstate_str = pstate_str, + new_pstate = new_pstate, + intchar2lua = intchar2lua, + conv_cmp_type = conv_cmp_type, + pstate_set_str = pstate_set_str, +} |