diff options
author | Justin M. Keyes <justinkz@gmail.com> | 2017-12-09 18:47:34 +0100 |
---|---|---|
committer | Justin M. Keyes <justinkz@gmail.com> | 2017-12-09 18:47:34 +0100 |
commit | 3cc7ebf8107bddb42a566ede4c2a51818ff623c5 (patch) | |
tree | e2f7a79ff73ee7a8e2a292f77fdb0df67485e555 | |
parent | afae4b514183c490737a28f2946def717e78a11c (diff) | |
parent | fbdc3ac4efbc97e2965b6083d429beabe261461c (diff) | |
download | rneovim-3cc7ebf8107bddb42a566ede4c2a51818ff623c5.tar.gz rneovim-3cc7ebf8107bddb42a566ede4c2a51818ff623c5.tar.bz2 rneovim-3cc7ebf8107bddb42a566ede4c2a51818ff623c5.zip |
Merge #7234 'built-in expression parser'
49 files changed, 17464 insertions, 771 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index cad3ea6786..fb0cc262b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,9 +68,9 @@ set(NVIM_VERSION_PATCH 3) set(NVIM_VERSION_PRERELEASE "-dev") # for package maintainers # API level -set(NVIM_API_LEVEL 3) # Bump this after any API change. +set(NVIM_API_LEVEL 4) # Bump this after any API change. set(NVIM_API_LEVEL_COMPAT 0) # Adjust this after a _breaking_ API change. -set(NVIM_API_PRERELEASE false) +set(NVIM_API_PRERELEASE true) file(TO_CMAKE_PATH ${CMAKE_CURRENT_LIST_DIR}/.git FORCED_GIT_DIR) include(GetGitRevisionDescription) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 71551e38e9..e337c5d6d5 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -1559,10 +1559,10 @@ v:exception The value of the exception most recently caught and not < Output: "caught oops". *v:false* *false-variable* -v:false Special value used to put "false" in JSON and msgpack. See - |json_encode()|. This value is converted to "v:false" when used - as a String (e.g. in |expr5| with string concatenation - operator) and to zero when used as a Number (e.g. in |expr5| +v:false Special value used to put "false" in JSON and msgpack. See + |json_encode()|. This value is converted to "v:false" when used + as a String (e.g. in |expr5| with string concatenation + operator) and to zero when used as a Number (e.g. in |expr5| or |expr7| when used with numeric operators). Read-only. *v:fcs_reason* *fcs_reason-variable* @@ -1703,16 +1703,16 @@ v:mouse_col Column number for a mouse click obtained with |getchar()|. value is zero when there was no mouse button click. *v:msgpack_types* *msgpack_types-variable* -v:msgpack_types Dictionary containing msgpack types used by |msgpackparse()| - and |msgpackdump()|. All types inside dictionary are fixed - (not editable) empty lists. To check whether some list is one +v:msgpack_types Dictionary containing msgpack types used by |msgpackparse()| + and |msgpackdump()|. All types inside dictionary are fixed + (not editable) empty lists. To check whether some list is one of msgpack types, use |is| operator. *v:null* *null-variable* -v:null Special value used to put "null" in JSON and NIL in msgpack. - See |json_encode()|. This value is converted to "v:null" when - used as a String (e.g. in |expr5| with string concatenation - operator) and to zero when used as a Number (e.g. in |expr5| +v:null Special value used to put "null" in JSON and NIL in msgpack. + See |json_encode()|. This value is converted to "v:null" when + used as a String (e.g. in |expr5| with string concatenation + operator) and to zero when used as a Number (e.g. in |expr5| or |expr7| when used with numeric operators). Read-only. *v:oldfiles* *oldfiles-variable* @@ -1903,10 +1903,10 @@ v:throwpoint The point where the exception most recently caught and not < Output: "Exception from test.vim, line 2" *v:true* *true-variable* -v:true Special value used to put "true" in JSON and msgpack. See - |json_encode()|. This value is converted to "v:true" when used - as a String (e.g. in |expr5| with string concatenation - operator) and to one when used as a Number (e.g. in |expr5| or +v:true Special value used to put "true" in JSON and msgpack. See + |json_encode()|. This value is converted to "v:true" when used + as a String (e.g. in |expr5| with string concatenation + operator) and to one when used as a Number (e.g. in |expr5| or |expr7| when used with numeric operators). Read-only. *v:val* *val-variable* @@ -2492,7 +2492,7 @@ assert_fails({cmd} [, {error}]) *assert_fails()* assert_false({actual} [, {msg}]) *assert_false()* When {actual} is not false an error message is added to |v:errors|, like with |assert_equal()|. - A value is false when it is zero or |v:false|. When "{actual}" + A value is false when it is zero or |v:false|. When "{actual}" is not a number or |v:false| the assert fails. When {msg} is omitted an error in the form "Expected False but got {actual}" is produced. @@ -3182,7 +3182,7 @@ diff_hlID({lnum}, {col}) *diff_hlID()* empty({expr}) *empty()* Return the Number 1 if {expr} is empty, zero otherwise. A |List| or |Dictionary| is empty when it does not have any - items. A Number is empty when its value is zero. Special + items. A Number is empty when its value is zero. Special variable is empty when it is |v:false| or |v:null|. escape({string}, {chars}) *escape()* @@ -4749,7 +4749,7 @@ input({opts}) string, or a blank string (for no prompt). A '\n' can be used in the prompt to start a new line. - In the second form it accepts a single dictionary with the + In the second form it accepts a single dictionary with the following keys, any of which may be omitted: Key Default Description ~ @@ -4757,7 +4757,7 @@ input({opts}) default "" Same as {text} in the first form. completion nothing Same as {completion} in the first form. cancelreturn "" Same as {cancelreturn} from - |inputdialog()|. Also works with + |inputdialog()|. Also works with input(). highlight nothing Highlight handler: |Funcref|. @@ -4833,8 +4833,8 @@ input({opts}) modifier. If the function causes any errors, it will be skipped for the duration of the current input() call. - Currently coloring is disabled when command-line contains - arabic characters. + Highlighting is disabled if command-line contains arabic + characters. NOTE: This function must not be used in a startup file, for the versions that only run in GUI mode (e.g., the Win32 GUI). @@ -4948,19 +4948,19 @@ islocked({expr}) *islocked()* *E786* message. Use |exists()| to check for existence. id({expr}) *id()* - Returns a |String| which is a unique identifier of the - container type (|List|, |Dict| and |Partial|). It is - guaranteed that for the mentioned types `id(v1) ==# id(v2)` - returns true iff `type(v1) == type(v2) && v1 is v2` (note: - |v:_null_list| and |v:_null_dict| have the same `id()` with - different types because they are internally represented as - a NULL pointers). Currently `id()` returns a hexadecimal - representanion of the pointers to the containers (i.e. like - `0x994a40`), same as `printf("%p", {expr})`, but it is advised + Returns a |String| which is a unique identifier of the + container type (|List|, |Dict| and |Partial|). It is + guaranteed that for the mentioned types `id(v1) ==# id(v2)` + returns true iff `type(v1) == type(v2) && v1 is v2` (note: + |v:_null_list| and |v:_null_dict| have the same `id()` with + different types because they are internally represented as + a NULL pointers). Currently `id()` returns a hexadecimal + representanion of the pointers to the containers (i.e. like + `0x994a40`), same as `printf("%p", {expr})`, but it is advised against counting on exact format of return value. - It is not guaranteed that `id(no_longer_existing_container)` - will not be equal to some other `id()`: new containers may + It is not guaranteed that `id(no_longer_existing_container)` + will not be equal to some other `id()`: new containers may reuse identifiers of the garbage-collected ones. items({dict}) *items()* @@ -5071,14 +5071,14 @@ join({list} [, {sep}]) *join()* The opposite function is |split()|. json_decode({expr}) *json_decode()* - Convert {expr} from JSON object. Accepts |readfile()|-style - list as the input, as well as regular string. May output any + Convert {expr} from JSON object. Accepts |readfile()|-style + list as the input, as well as regular string. May output any Vim value. In the following cases it will output |msgpack-special-dict|: 1. Dictionary contains duplicate key. 2. Dictionary contains empty key. - 3. String contains NUL byte. Two special dictionaries: for - dictionary and for string will be emitted in case string + 3. String contains NUL byte. Two special dictionaries: for + dictionary and for string will be emitted in case string with NUL byte was a dictionary key. Note: function treats its input as UTF-8 always. The JSON @@ -5087,14 +5087,14 @@ json_decode({expr}) *json_decode()* Non-UTF-8 characters are an error. json_encode({expr}) *json_encode()* - Convert {expr} into a JSON string. Accepts - |msgpack-special-dict| as the input. Will not convert - |Funcref|s, mappings with non-string keys (can be created as - |msgpack-special-dict|), values with self-referencing - containers, strings which contain non-UTF-8 characters, - pseudo-UTF-8 strings which contain codepoints reserved for - surrogate pairs (such strings are not valid UTF-8 strings). - Non-printable characters are converted into "\u1234" escapes + Convert {expr} into a JSON string. Accepts + |msgpack-special-dict| as the input. Will not convert + |Funcref|s, mappings with non-string keys (can be created as + |msgpack-special-dict|), values with self-referencing + containers, strings which contain non-UTF-8 characters, + pseudo-UTF-8 strings which contain codepoints reserved for + surrogate pairs (such strings are not valid UTF-8 strings). + Non-printable characters are converted into "\u1234" escapes or special escapes like "\t", other are dumped as-is. keys({dict}) *keys()* @@ -5193,7 +5193,7 @@ line({expr}) The result is a Number, which is the line number of the file This autocommand jumps to the last known position in a file just after opening it, if the '" mark is set: > :au BufReadPost * - \ if line("'\"") > 1 && line("'\"") <= line("$") && &ft !~# 'commit' + \ if line("'\"") > 1 && line("'\"") <= line("$") && &ft !~# 'commit' \ | exe "normal! g`\"" \ | endif @@ -5244,7 +5244,7 @@ log10({expr}) *log10()* < -2.0 luaeval({expr}[, {expr}]) - Evaluate Lua expression {expr} and return its result converted + Evaluate Lua expression {expr} and return its result converted to Vim data structures. See |lua-luaeval| for more details. map({expr1}, {expr2}) *map()* @@ -5675,7 +5675,7 @@ mkdir({name} [, {path} [, {prot}]]) :call mkdir($HOME . "/tmp/foo/bar", "p", 0700) < This function is not available in the |sandbox|. - If you try to create an existing directory with {path} set to + If you try to create an existing directory with {path} set to "p" mkdir() will silently exit. *mode()* @@ -5728,78 +5728,76 @@ msgpackdump({list}) {Nvim} *msgpackdump()* 5. Points 3. and 4. do not apply to |msgpack-special-dict|s. msgpackparse({list}) {Nvim} *msgpackparse()* - Convert a |readfile()|-style list to a list of VimL objects. + Convert a |readfile()|-style list to a list of VimL objects. Example: > let fname = expand('~/.config/nvim/shada/main.shada') let mpack = readfile(fname, 'b') let shada_objects = msgpackparse(mpack) -< This will read ~/.config/nvim/shada/main.shada file to +< This will read ~/.config/nvim/shada/main.shada file to `shada_objects` list. Limitations: - 1. Mapping ordering is not preserved unless messagepack - mapping is dumped using generic mapping + 1. Mapping ordering is not preserved unless messagepack + mapping is dumped using generic mapping (|msgpack-special-map|). - 2. Since the parser aims to preserve all data untouched - (except for 1.) some strings are parsed to - |msgpack-special-dict| format which is not convenient to + 2. Since the parser aims to preserve all data untouched + (except for 1.) some strings are parsed to + |msgpack-special-dict| format which is not convenient to use. *msgpack-special-dict* - Some messagepack strings may be parsed to special + Some messagepack strings may be parsed to special dictionaries. Special dictionaries are dictionaries which 1. Contain exactly two keys: `_TYPE` and `_VAL`. - 2. `_TYPE` key is one of the types found in |v:msgpack_types| + 2. `_TYPE` key is one of the types found in |v:msgpack_types| variable. - 3. Value for `_VAL` has the following format (Key column + 3. Value for `_VAL` has the following format (Key column contains name of the key from |v:msgpack_types|): Key Value ~ - nil Zero, ignored when dumping. This value cannot - possibly appear in |msgpackparse()| output in Neovim - versions which have |v:null|. - boolean One or zero. When dumping it is only checked that - value is a |Number|. This value cannot possibly - appear in |msgpackparse()| output in Neovim versions - which have |v:true| and |v:false|. - integer |List| with four numbers: sign (-1 or 1), highest two - bits, number with bits from 62nd to 31st, lowest 31 - bits. I.e. to get actual number one will need to use + nil Zero, ignored when dumping. Not returned by + |msgpackparse()| since |v:null| was introduced. + boolean One or zero. When dumping it is only checked that + value is a |Number|. Not returned by |msgpackparse()| + since |v:true| and |v:false| were introduced. + integer |List| with four numbers: sign (-1 or 1), highest two + bits, number with bits from 62nd to 31st, lowest 31 + bits. I.e. to get actual number one will need to use code like > _VAL[0] * ((_VAL[1] << 62) & (_VAL[2] << 31) & _VAL[3]) -< Special dictionary with this type will appear in - |msgpackparse()| output under one of the following +< Special dictionary with this type will appear in + |msgpackparse()| output under one of the following circumstances: - 1. |Number| is 32-bit and value is either above + 1. |Number| is 32-bit and value is either above INT32_MAX or below INT32_MIN. - 2. |Number| is 64-bit and value is above INT64_MAX. It - cannot possibly be below INT64_MIN because msgpack + 2. |Number| is 64-bit and value is above INT64_MAX. It + cannot possibly be below INT64_MIN because msgpack C parser does not support such values. - float |Float|. This value cannot possibly appear in + float |Float|. This value cannot possibly appear in |msgpackparse()| output. - string |readfile()|-style list of strings. This value will - appear in |msgpackparse()| output if string contains - zero byte or if string is a mapping key and mapping is - being represented as special dictionary for other + string |readfile()|-style list of strings. This value will + appear in |msgpackparse()| output if string contains + zero byte or if string is a mapping key and mapping is + being represented as special dictionary for other reasons. - binary |readfile()|-style list of strings. This value will - appear in |msgpackparse()| output if binary string + binary |readfile()|-style list of strings. This value will + appear in |msgpackparse()| output if binary string contains zero byte. - array |List|. This value cannot appear in |msgpackparse()| + array |List|. This value cannot appear in |msgpackparse()| output. *msgpack-special-map* - map |List| of |List|s with two items (key and value) each. - This value will appear in |msgpackparse()| output if + map |List| of |List|s with two items (key and value) each. + This value will appear in |msgpackparse()| output if parsed mapping contains one of the following keys: - 1. Any key that is not a string (including keys which + 1. Any key that is not a string (including keys which are binary strings). 2. String with NUL byte inside. 3. Duplicate key. 4. Empty key. - ext |List| with two values: first is a signed integer - representing extension type. Second is + ext |List| with two values: first is a signed integer + representing extension type. Second is |readfile()|-style list of strings. nextnonblank({lnum}) *nextnonblank()* @@ -6073,11 +6071,11 @@ pumvisible() *pumvisible()* py3eval({expr}) *py3eval()* Evaluate Python expression {expr} and return its result converted to Vim data structures. - Numbers and strings are returned as they are (strings are - copied though, Unicode strings are additionally converted to + Numbers and strings are returned as they are (strings are + copied though, Unicode strings are additionally converted to UTF-8). Lists are represented as Vim |List| type. - Dictionaries are represented as Vim |Dictionary| type with + Dictionaries are represented as Vim |Dictionary| type with keys converted to strings. {only available when compiled with the |+python3| feature} @@ -6145,7 +6143,7 @@ readfile({fname} [, {binary} [, {max}]]) reltime([{start} [, {end}]]) *reltime()* Return an item that represents a time value. The format of the item depends on the system. It can be passed to - |reltimestr()| to convert it to a string or |reltimefloat()| + |reltimestr()| to convert it to a string or |reltimefloat()| to convert to a float. Without an argument it returns the current "relative time", an @@ -6929,9 +6927,9 @@ setreg({regname}, {value} [, {options}]) :call setreg('a', "1\n2\n3", 'b5') < This example shows using the functions to save and restore a - register (note: you may not reliably restore register value - without using the third argument to |getreg()| as without it - newlines are represented as newlines AND Nul bytes are + register (note: you may not reliably restore register value + without using the third argument to |getreg()| as without it + newlines are represented as newlines AND Nul bytes are represented as newlines as well, see |NL-used-for-Nul|). > :let var_a = getreg('a', 1, 1) :let var_amode = getregtype('a') @@ -7369,20 +7367,20 @@ string({expr}) Return {expr} converted to a String. If {expr} is a Number, {expr} type result ~ String 'string' Number 123 - Float 123.123456 or 1.123456e8 or + Float 123.123456 or 1.123456e8 or `str2float('inf')` Funcref `function('name')` List [item, item] Dictionary {key: value, key: value} Note that in String values the ' character is doubled. Also see |strtrans()|. - Note 2: Output format is mostly compatible with YAML, except - for infinite and NaN floating-point values representations - which use |str2float()|. Strings are also dumped literally, - only single quote is escaped, which does not allow using YAML - for parsing back binary strings. |eval()| should always work for - strings and floats though and this is the only official - method, use |msgpackdump()| or |json_encode()| if you need to + Note 2: Output format is mostly compatible with YAML, except + for infinite and NaN floating-point values representations + which use |str2float()|. Strings are also dumped literally, + only single quote is escaped, which does not allow using YAML + for parsing back binary strings. |eval()| should always work for + strings and floats though and this is the only official + method, use |msgpackdump()| or |json_encode()| if you need to share data with other application. *strlen()* @@ -7630,10 +7628,10 @@ system({cmd} [, {input}]) *system()* *E677* redirection syntax) before input can reach it. Use |jobstart()| instead. - Note: Use |shellescape()| or |::S| with |expand()| or - |fnamemodify()| to escape special characters in a command - argument. Newlines in {cmd} may cause the command to fail. - The characters in 'shellquote' and 'shellxquote' may also + Note: Use |shellescape()| or |::S| with |expand()| or + |fnamemodify()| to escape special characters in a command + argument. Newlines in {cmd} may cause the command to fail. + The characters in 'shellquote' and 'shellxquote' may also cause trouble. The result is a String. Example: > @@ -7660,9 +7658,9 @@ system({cmd} [, {input}]) *system()* *E677* systemlist({cmd} [, {input} [, {keepempty}]]) *systemlist()* - Same as |system()|, but returns a |List| with lines (parts of - output separated by NL) with NULs transformed into NLs. Output - is the same as |readfile()| will output with {binary} argument + Same as |system()|, but returns a |List| with lines (parts of + output separated by NL) with NULs transformed into NLs. Output + is the same as |readfile()| will output with {binary} argument set to "b", except that a final newline is not preserved, unless {keepempty} is non-zero. Note that on MS-Windows you may get trailing CR characters. @@ -7926,7 +7924,7 @@ type({expr}) *type()* :if type(myvar) == type({}) :if type(myvar) == type(0.0) :if type(myvar) == type(v:true) -< In place of checking for |v:null| type it is better to check +< In place of checking for |v:null| type it is better to check for |v:null| directly as it is the only value of this type: > :if myvar is v:null < To check if the v:t_ variables exist use this: > @@ -8243,10 +8241,10 @@ writefile({list}, {fname} [, {flags}]) :call writefile(["foo"], "event.log", "a") :call writefile(["bar"], "event.log", "a") < - When {flags} contains "S" fsync() call is not used, with "s" - it is used, 'fsync' option applies by default. No fsync() - means that writefile() will finish faster, but writes may be - left in OS buffers and not yet written to disk. Such changes + When {flags} contains "S" fsync() call is not used, with "s" + it is used, 'fsync' option applies by default. No fsync() + means that writefile() will finish faster, but writes may be + left in OS buffers and not yet written to disk. Such changes will disappear if system crashes before OS does writing. All NL characters are replaced with a NUL character. @@ -8728,7 +8726,7 @@ like this: > When such a function is called, and it is not defined yet, Vim will search the "autoload" directories in 'runtimepath' for a script file called -"filename.vim". For example "~/.config/nvim/autoload/filename.vim". That +"filename.vim". For example "~/.config/nvim/autoload/filename.vim". That file should then define the function like this: > function filename#funcname() @@ -8999,8 +8997,8 @@ This does NOT work: > < *E741* *E940* If you try to change a locked variable you get an error message: "E741: Value is locked: {name}". - If you try to lock or unlock a built-in variable you - will get an error message "E940: Cannot lock or unlock + If you try to lock or unlock a built-in variable you + will get an error message "E940: Cannot lock or unlock variable {name}". [depth] is relevant when locking a |List| or @@ -9259,17 +9257,17 @@ This does NOT work: > with the |:redraw| command. Example: > :new | redraw | echo "there is a new window" < *:echo-self-refer* - When printing nested containers echo prints second - occurrence of the self-referencing container using - "[...@level]" (self-referencing |List|) or + When printing nested containers echo prints second + occurrence of the self-referencing container using + "[...@level]" (self-referencing |List|) or "{...@level}" (self-referencing |Dict|): > :let l = [] :call add(l, l) :let l2 = [] :call add(l2, [l2]) :echo l l2 -< echoes "[[...@0]] [[[...@0]]]". Echoing "[l]" will - echo "[[[...@1]]]" because l first occurs at second +< echoes "[[...@0]] [[[...@0]]]". Echoing "[l]" will + echo "[[[...@1]]]" because l first occurs at second level. *:echon* @@ -10558,7 +10556,7 @@ option will still be marked as it was set in the sandbox. In a few situations it is not allowed to change the text in the buffer, jump to another window and some other things that might confuse or break what Vim is currently doing. This mostly applies to things that happen when Vim is -actually doing something else. For example, evaluating the 'balloonexpr' may +actually doing something else. For example, evaluating the 'balloonexpr' may happen any moment the mouse cursor is resting at some position. This is not allowed when the textlock is active: @@ -10568,5 +10566,123 @@ This is not allowed when the textlock is active: - closing a window or quitting Vim - etc. +============================================================================== +13. Command-line expressions highlighting *expr-highlight* + +Expressions entered by the user in |i_CTRL-R_=|, |c_CTRL-\_e|, |quote=| are +highlighted by the built-in expressions parser. It uses highlight groups +described in the table below, which may be overriden by colorschemes. + *hl-NvimInvalid* +Besides the "Nvim"-prefixed highlight groups described below, there are +"NvimInvalid"-prefixed highlight groups which have the same meaning but +indicate that the token contains an error or that an error occurred just +before it. They have mostly the same hierarchy, except that (by default) in +place of any non-Nvim-prefixed group NvimInvalid linking to `Error` is used +and some other intermediate groups are present. + +Group Default link Colored expression ~ +*hl-NvimInternalError* None, red/red Parser bug + +*hl-NvimAssignment* Operator Generic assignment +*hl-NvimPlainAssignment* NvimAssignment `=` in |:let| +*hl-NvimAugmentedAssignment* NvimAssignment Generic, `+=`/`-=`/`.=` +*hl-NvimAssignmentWithAddition* NvimAugmentedAssignment `+=` in |:let+=| +*hl-NvimAssignmentWithSubtraction* NvimAugmentedAssignment `-=` in |:let-=| +*hl-NvimAssignmentWithConcatenation* NvimAugmentedAssignment `.=` in |:let.=| + +*hl-NvimOperator* Operator Generic operator + +*hl-NvimUnaryOperator* NvimOperator Generic unary op +*hl-NvimUnaryPlus* NvimUnaryOperator |expr-unary-+| +*hl-NvimUnaryMinus* NvimUnaryOperator |expr-unary--| +*hl-NvimNot* NvimUnaryOperator |expr-!| + +*hl-NvimBinaryOperator* NvimOperator Generic binary op +*hl-NvimComparison* NvimBinaryOperator Any |expr4| operator +*hl-NvimComparisonModifier* NvimComparison `#`/`?` near |expr4| op +*hl-NvimBinaryPlus* NvimBinaryOperator |expr-+| +*hl-NvimBinaryMinus* NvimBinaryOperator |expr--| +*hl-NvimConcat* NvimBinaryOperator |expr-.| +*hl-NvimConcatOrSubscript* NvimConcat |expr-.| or |expr-entry| +*hl-NvimOr* NvimBinaryOperator |expr-barbar| +*hl-NvimAnd* NvimBinaryOperator |expr-&&| +*hl-NvimMultiplication* NvimBinaryOperator |expr-star| +*hl-NvimDivision* NvimBinaryOperator |expr-/| +*hl-NvimMod* NvimBinaryOperator |expr-%| + +*hl-NvimTernary* NvimOperator `?` in |expr1| +*hl-NvimTernaryColon* NvimTernary `:` in |expr1| + +*hl-NvimParenthesis* Delimiter Generic bracket +*hl-NvimLambda* NvimParenthesis `{`/`}` in |lambda| +*hl-NvimNestingParenthesis* NvimParenthesis `(`/`)` in |expr-nesting| +*hl-NvimCallingParenthesis* NvimParenthesis `(`/`)` in |expr-function| + +*hl-NvimSubscript* NvimParenthesis Generic subscript +*hl-NvimSubscriptBracket* NvimSubscript `[`/`]` in |expr-[]| +*hl-NvimSubscriptColon* NvimSubscript `:` in |expr-[:]| +*hl-NvimCurly* NvimSubscript `{`/`}` in + |curly-braces-names| + +*hl-NvimContainer* NvimParenthesis Generic container +*hl-NvimDict* NvimContainer `{`/`}` in |dict| literal +*hl-NvimList* NvimContainer `[`/`]` in |list| literal + +*hl-NvimIdentifier* Identifier Generic identifier +*hl-NvimIdentifierScope* NvimIdentifier Namespace: letter + before `:` in + |internal-variables| +*hl-NvimIdentifierScopeDelimiter* NvimIdentifier `:` after namespace + letter +*hl-NvimIdentifierName* NvimIdentifier Rest of the ident +*hl-NvimIdentifierKey* NvimIdentifier Identifier after + |expr-entry| + +*hl-NvimColon* Delimiter `:` in |dict| literal +*hl-NvimComma* Delimiter `,` in |dict|/|list| + literal or + |expr-function| +*hl-NvimArrow* Delimiter `->` in |lambda| + +*hl-NvimRegister* SpecialChar |expr-register| +*hl-NvimNumber* Number Non-prefix digits + in integer + |expr-number| +*hl-NvimNumberPrefix* Type `0` for |octal-number| + `0x` for |hex-number| + `0b` for |binary-number| +*hl-NvimFloat* NvimNumber Floating-point + number + +*hl-NvimOptionSigil* Type `&` in |expr-option| +*hl-NvimOptionScope* NvimIdentifierScope Option scope if any +*hl-NvimOptionScopeDelimiter* NvimIdentifierScopeDelimiter + `:` after option scope +*hl-NvimOptionName* NvimIdentifier Option name + +*hl-NvimEnvironmentSigil* NvimOptionSigil `$` in |expr-env| +*hl-NvimEnvironmentName* NvimIdentifier Env variable name + +*hl-NvimString* String Generic string +*hl-NvimStringBody* NvimString Generic string + literal body +*hl-NvimStringQuote* NvimString Generic string quote +*hl-NvimStringSpecial* SpecialChar Generic string + non-literal body + +*hl-NvimSingleQuote* NvimStringQuote `'` in |expr-'| +*hl-NvimSingleQuotedBody* NvimStringBody Literal part of + |expr-'| string body +*hl-NvimSingleQuotedQuote* NvimStringSpecial `''` inside |expr-'| + string body + +*hl-NvimDoubleQuote* NvimStringQuote `"` in |expr-quote| +*hl-NvimDoubleQuotedBody* NvimStringBody Literal part of + |expr-quote| body +*hl-NvimDoubleQuotedEscape* NvimStringSpecial Valid |expr-quote| + escape sequence +*hl-NvimDoubleQuotedUnknownEscape* NvimInvalidValue Unrecognized + |expr-quote| escape + sequence vim:tw=78:ts=8:noet:ft=help:norl: diff --git a/runtime/doc/vim_diff.txt b/runtime/doc/vim_diff.txt index 45c88f6fe9..1a4a66ed89 100644 --- a/runtime/doc/vim_diff.txt +++ b/runtime/doc/vim_diff.txt @@ -165,14 +165,17 @@ Highlight groups: |hl-TermCursor| |hl-TermCursorNC| |hl-Whitespace| highlights 'listchars' whitespace - -UI: - *E5408* *E5409* *g:Nvim_color_expr* *g:Nvim_color_cmdline* - Command-line coloring is supported. Only |input()| and |inputdialog()| may - be colored. For testing purposes expressions (e.g. |i_CTRL-R_=|) and regular - command-line (|:|) are colored by callbacks defined in `g:Nvim_color_expr` - and `g:Nvim_color_cmdline` respectively (these callbacks are for testing - only, and will be removed in a future version). + |expr-highlight| highlight groups (prefixed with "Nvim") + +Command-line highlighting: + The expression prompt (|@=|, |c_CTRL-R_=|, |i_CTRL-R_=|) is highlighted + using a built-in VimL expression parser. |expr-highlight| + *E5408* *E5409* + |input()|, |inputdialog()| support custom highlighting. |input()-highlight| + *g:Nvim_color_cmdline* + (Experimental) Command-line (|:|) is colored by callback defined in + `g:Nvim_color_cmdline` (this callback is for testing only, and will be + removed in the future). ============================================================================== 4. Changed features *nvim-features-changed* diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index a166ee6c02..d3e07326bf 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -86,6 +86,8 @@ foreach(subdir event eval lua + viml + viml/parser ) if(${subdir} MATCHES "tui" AND NOT FEAT_TUI) continue() diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index f4ccf07bec..416e7d22d2 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -33,6 +33,8 @@ #include "nvim/syntax.h" #include "nvim/getchar.h" #include "nvim/os/input.h" +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" #define LINE_BUFFER_SIZE 4096 @@ -889,6 +891,458 @@ theend: return rv; } +typedef struct { + ExprASTNode **node_p; + Object *ret_node_p; +} ExprASTConvStackItem; + +typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack; + +/// Parse a VimL expression +/// +/// @param[in] expr Expression to parse. Is always treated as a single line. +/// @param[in] flags Flags: +/// +/// - "m" if multiple expressions in a row are allowed (only +/// the first one will be parsed), +/// - "E" if EOC tokens are not allowed (determines whether +/// they will stop parsing process or be recognized as an +/// operator/space, though also yielding an error). +/// - "l" when needing to start parsing with lvalues for +/// ":let" or ":for". +/// +/// Common flag sets: +/// - "m" to parse like for ":echo". +/// - "E" to parse like for "<C-r>=". +/// - empty string for ":call". +/// - "lm" to parse for ":let". +/// @param[in] highlight If true, return value will also include "highlight" +/// key containing array of 4-tuples (arrays) (Integer, +/// Integer, Integer, String), where first three numbers +/// define the highlighted region and represent line, +/// starting column and ending column (latter exclusive: +/// one should highlight region [start_col, end_col)). +/// +/// @return AST: top-level dictionary holds keys +/// +/// "error": Dictionary with error, present only if parser saw some +/// error. Contains the following keys: +/// +/// "message": String, error message in printf format, translated. +/// Must contain exactly one "%.*s". +/// "arg": String, error message argument. +/// +/// "len": Amount of bytes successfully parsed. With flags equal to "" +/// that should be equal to the length of expr string. +/// +/// @note: “Sucessfully parsed” here means “participated in AST +/// creation”, not “till the first error”. +/// +/// "ast": AST, either nil or a dictionary with these keys: +/// +/// "type": node type, one of the value names from ExprASTNodeType +/// stringified without "kExprNode" prefix. +/// "start": a pair [line, column] describing where node is “started” +/// where "line" is always 0 (will not be 0 if you will be +/// using nvim_parse_viml() on e.g. ":let", but that is not +/// present yet). Both elements are Integers. +/// "len": “length” of the node. This and "start" are there for +/// debugging purposes primary (debugging parser and providing +/// debug information). +/// "children": a list of nodes described in top/"ast". There always +/// is zero, one or two children, key will not be present +/// if node has no children. Maximum number of children +/// may be found in node_maxchildren array. +/// +/// Local values (present only for certain nodes): +/// +/// "scope": a single Integer, specifies scope for "Option" and +/// "PlainIdentifier" nodes. For "Option" it is one of +/// ExprOptScope values, for "PlainIdentifier" it is one of +/// ExprVarScope values. +/// "ident": identifier (without scope, if any), present for "Option", +/// "PlainIdentifier", "PlainKey" and "Environment" nodes. +/// "name": Integer, register name (one character) or -1. Only present +/// for "Register" nodes. +/// "cmp_type": String, comparison type, one of the value names from +/// ExprComparisonType, stringified without "kExprCmp" +/// prefix. Only present for "Comparison" nodes. +/// "ccs_strategy": String, case comparison strategy, one of the +/// value names from ExprCaseCompareStrategy, +/// stringified without "kCCStrategy" prefix. Only +/// present for "Comparison" nodes. +/// "augmentation": String, augmentation type for "Assignment" nodes. +/// Is either an empty string, "Add", "Subtract" or +/// "Concat" for "=", "+=", "-=" or ".=" respectively. +/// "invert": Boolean, true if result of comparison needs to be +/// inverted. Only present for "Comparison" nodes. +/// "ivalue": Integer, integer value for "Integer" nodes. +/// "fvalue": Float, floating-point value for "Float" nodes. +/// "svalue": String, value for "SingleQuotedString" and +/// "DoubleQuotedString" nodes. +Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, + Error *err) + FUNC_API_SINCE(4) FUNC_API_ASYNC +{ + int pflags = 0; + for (size_t i = 0 ; i < flags.size ; i++) { + switch (flags.data[i]) { + case 'm': { pflags |= kExprFlagsMulti; break; } + case 'E': { pflags |= kExprFlagsDisallowEOC; break; } + case 'l': { pflags |= kExprFlagsParseLet; break; } + case NUL: { + api_set_error(err, kErrorTypeValidation, "Invalid flag: '\\0' (%u)", + (unsigned)flags.data[i]); + return (Dictionary)ARRAY_DICT_INIT; + } + default: { + api_set_error(err, kErrorTypeValidation, "Invalid flag: '%c' (%u)", + flags.data[i], (unsigned)flags.data[i]); + return (Dictionary)ARRAY_DICT_INIT; + } + } + } + ParserLine plines[] = { + { + .data = expr.data, + .size = expr.size, + .allocated = false, + }, + { NULL, 0, false }, + }; + ParserLine *plines_p = plines; + ParserHighlight colors; + kvi_init(colors); + ParserHighlight *const colors_p = (highlight ? &colors : NULL); + ParserState pstate; + viml_parser_init( + &pstate, parser_simple_get_line, &plines_p, colors_p); + ExprAST east = viml_pexpr_parse(&pstate, pflags); + + const size_t ret_size = ( + 2 // "ast", "len" + + (size_t)(east.err.msg != NULL) // "error" + + (size_t)highlight // "highlight" + + 0); + Dictionary ret = { + .items = xmalloc(ret_size * sizeof(ret.items[0])), + .size = 0, + .capacity = ret_size, + }; + ret.items[ret.size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ast"), + .value = NIL, + }; + ret.items[ret.size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("len"), + .value = INTEGER_OBJ((Integer)(pstate.pos.line == 1 + ? plines[0].size + : pstate.pos.col)), + }; + if (east.err.msg != NULL) { + Dictionary err_dict = { + .items = xmalloc(2 * sizeof(err_dict.items[0])), + .size = 2, + .capacity = 2, + }; + err_dict.items[0] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("message"), + .value = STRING_OBJ(cstr_to_string(east.err.msg)), + }; + if (east.err.arg == NULL) { + err_dict.items[1] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("arg"), + .value = STRING_OBJ(STRING_INIT), + }; + } else { + err_dict.items[1] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("arg"), + .value = STRING_OBJ(((String) { + .data = xmemdupz(east.err.arg, (size_t)east.err.arg_len), + .size = (size_t)east.err.arg_len, + })), + }; + } + ret.items[ret.size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("error"), + .value = DICTIONARY_OBJ(err_dict), + }; + } + if (highlight) { + Array hl = (Array) { + .items = xmalloc(kv_size(colors) * sizeof(hl.items[0])), + .capacity = kv_size(colors), + .size = kv_size(colors), + }; + for (size_t i = 0 ; i < kv_size(colors) ; i++) { + const ParserHighlightChunk chunk = kv_A(colors, i); + Array chunk_arr = (Array) { + .items = xmalloc(4 * sizeof(chunk_arr.items[0])), + .capacity = 4, + .size = 4, + }; + chunk_arr.items[0] = INTEGER_OBJ((Integer)chunk.start.line); + chunk_arr.items[1] = INTEGER_OBJ((Integer)chunk.start.col); + chunk_arr.items[2] = INTEGER_OBJ((Integer)chunk.end_col); + chunk_arr.items[3] = STRING_OBJ(cstr_to_string(chunk.group)); + hl.items[i] = ARRAY_OBJ(chunk_arr); + } + ret.items[ret.size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("highlight"), + .value = ARRAY_OBJ(hl), + }; + } + kvi_destroy(colors); + + // Walk over the AST, freeing nodes in process. + ExprASTConvStack ast_conv_stack; + kvi_init(ast_conv_stack); + kvi_push(ast_conv_stack, ((ExprASTConvStackItem) { + .node_p = &east.root, + .ret_node_p = &ret.items[0].value, + })); + while (kv_size(ast_conv_stack)) { + ExprASTConvStackItem cur_item = kv_last(ast_conv_stack); + ExprASTNode *const node = *cur_item.node_p; + if (node == NULL) { + assert(kv_size(ast_conv_stack) == 1); + kv_drop(ast_conv_stack, 1); + } else { + if (cur_item.ret_node_p->type == kObjectTypeNil) { + const size_t ret_node_items_size = (size_t)( + 3 // "type", "start" and "len" + + (node->children != NULL) // "children" + + (node->type == kExprNodeOption + || node->type == kExprNodePlainIdentifier) // "scope" + + (node->type == kExprNodeOption + || node->type == kExprNodePlainIdentifier + || node->type == kExprNodePlainKey + || node->type == kExprNodeEnvironment) // "ident" + + (node->type == kExprNodeRegister) // "name" + + (3 // "cmp_type", "ccs_strategy", "invert" + * (node->type == kExprNodeComparison)) + + (node->type == kExprNodeInteger) // "ivalue" + + (node->type == kExprNodeFloat) // "fvalue" + + (node->type == kExprNodeDoubleQuotedString + || node->type == kExprNodeSingleQuotedString) // "svalue" + + (node->type == kExprNodeAssignment) // "augmentation" + + 0); + Dictionary ret_node = { + .items = xmalloc(ret_node_items_size * sizeof(ret_node.items[0])), + .capacity = ret_node_items_size, + .size = 0, + }; + *cur_item.ret_node_p = DICTIONARY_OBJ(ret_node); + } + Dictionary *ret_node = &cur_item.ret_node_p->data.dictionary; + if (node->children != NULL) { + const size_t num_children = 1 + (node->children->next != NULL); + Array children_array = { + .items = xmalloc(num_children * sizeof(children_array.items[0])), + .capacity = num_children, + .size = num_children, + }; + for (size_t i = 0; i < num_children; i++) { + children_array.items[i] = NIL; + } + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("children"), + .value = ARRAY_OBJ(children_array), + }; + kvi_push(ast_conv_stack, ((ExprASTConvStackItem) { + .node_p = &node->children, + .ret_node_p = &children_array.items[0], + })); + } else if (node->next != NULL) { + kvi_push(ast_conv_stack, ((ExprASTConvStackItem) { + .node_p = &node->next, + .ret_node_p = cur_item.ret_node_p + 1, + })); + } else if (node != NULL) { + kv_drop(ast_conv_stack, 1); + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("type"), + .value = STRING_OBJ(cstr_to_string(east_node_type_tab[node->type])), + }; + Array start_array = { + .items = xmalloc(2 * sizeof(start_array.items[0])), + .capacity = 2, + .size = 2, + }; + start_array.items[0] = INTEGER_OBJ((Integer)node->start.line); + start_array.items[1] = INTEGER_OBJ((Integer)node->start.col); + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("start"), + .value = ARRAY_OBJ(start_array), + }; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("len"), + .value = INTEGER_OBJ((Integer)node->len), + }; + switch (node->type) { + case kExprNodeDoubleQuotedString: + case kExprNodeSingleQuotedString: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("svalue"), + .value = STRING_OBJ(((String) { + .data = node->data.str.value, + .size = node->data.str.size, + })), + }; + break; + } + case kExprNodeOption: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("scope"), + .value = INTEGER_OBJ(node->data.opt.scope), + }; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ident"), + .value = STRING_OBJ(((String) { + .data = xmemdupz(node->data.opt.ident, + node->data.opt.ident_len), + .size = node->data.opt.ident_len, + })), + }; + break; + } + case kExprNodePlainIdentifier: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("scope"), + .value = INTEGER_OBJ(node->data.var.scope), + }; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ident"), + .value = STRING_OBJ(((String) { + .data = xmemdupz(node->data.var.ident, + node->data.var.ident_len), + .size = node->data.var.ident_len, + })), + }; + break; + } + case kExprNodePlainKey: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ident"), + .value = STRING_OBJ(((String) { + .data = xmemdupz(node->data.var.ident, + node->data.var.ident_len), + .size = node->data.var.ident_len, + })), + }; + break; + } + case kExprNodeEnvironment: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ident"), + .value = STRING_OBJ(((String) { + .data = xmemdupz(node->data.env.ident, + node->data.env.ident_len), + .size = node->data.env.ident_len, + })), + }; + break; + } + case kExprNodeRegister: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("name"), + .value = INTEGER_OBJ(node->data.reg.name), + }; + break; + } + case kExprNodeComparison: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("cmp_type"), + .value = STRING_OBJ(cstr_to_string( + eltkn_cmp_type_tab[node->data.cmp.type])), + }; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ccs_strategy"), + .value = STRING_OBJ(cstr_to_string( + ccs_tab[node->data.cmp.ccs])), + }; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("invert"), + .value = BOOLEAN_OBJ(node->data.cmp.inv), + }; + break; + } + case kExprNodeFloat: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("fvalue"), + .value = FLOAT_OBJ(node->data.flt.value), + }; + break; + } + case kExprNodeInteger: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ivalue"), + .value = INTEGER_OBJ((Integer)( + node->data.num.value > API_INTEGER_MAX + ? API_INTEGER_MAX + : (Integer)node->data.num.value)), + }; + break; + } + case kExprNodeAssignment: { + const ExprAssignmentType asgn_type = node->data.ass.type; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("augmentation"), + .value = STRING_OBJ( + asgn_type == kExprAsgnPlain + ? (String)STRING_INIT + : cstr_to_string(expr_asgn_type_tab[asgn_type])), + }; + break; + } + case kExprNodeMissing: + case kExprNodeOpMissing: + case kExprNodeTernary: + case kExprNodeTernaryValue: + case kExprNodeSubscript: + case kExprNodeListLiteral: + case kExprNodeUnaryPlus: + case kExprNodeBinaryPlus: + case kExprNodeNested: + case kExprNodeCall: + case kExprNodeComplexIdentifier: + case kExprNodeUnknownFigure: + case kExprNodeLambda: + case kExprNodeDictLiteral: + case kExprNodeCurlyBracesIdentifier: + case kExprNodeComma: + case kExprNodeColon: + case kExprNodeArrow: + case kExprNodeConcat: + case kExprNodeConcatOrSubscript: + case kExprNodeOr: + case kExprNodeAnd: + case kExprNodeUnaryMinus: + case kExprNodeBinaryMinus: + case kExprNodeNot: + case kExprNodeMultiplication: + case kExprNodeDivision: + case kExprNodeMod: { + break; + } + } + assert(cur_item.ret_node_p->data.dictionary.size + == cur_item.ret_node_p->data.dictionary.capacity); + xfree(*cur_item.node_p); + *cur_item.node_p = NULL; + } + } + } + kvi_destroy(ast_conv_stack); + + assert(ret.size == ret.capacity); + // Should be a no-op actually, leaving it in case non-nodes will need to be + // freed later. + viml_pexpr_free_ast(east); + viml_parser_destroy(&pstate); + return ret; +} + /// Writes a message to vim output or error buffer. The string is split /// and flushed after each newline. Incomplete lines are kept for writing diff --git a/src/nvim/ascii.h b/src/nvim/ascii.h index adde91f9ec..9ccb70764d 100644 --- a/src/nvim/ascii.h +++ b/src/nvim/ascii.h @@ -3,6 +3,7 @@ #include <stdbool.h> +#include "nvim/macros.h" #include "nvim/func_attr.h" #include "nvim/os/os_defs.h" @@ -98,6 +99,10 @@ static inline bool ascii_isxdigit(int) REAL_FATTR_CONST REAL_FATTR_ALWAYS_INLINE; +static inline bool ascii_isident(int) + REAL_FATTR_CONST + REAL_FATTR_ALWAYS_INLINE; + static inline bool ascii_isbdigit(int) REAL_FATTR_CONST REAL_FATTR_ALWAYS_INLINE; @@ -138,6 +143,14 @@ static inline bool ascii_isxdigit(int c) || (c >= 'A' && c <= 'F'); } +/// Checks if `c` is an “identifier” character +/// +/// That is, whether it is alphanumeric character or underscore. +static inline bool ascii_isident(const int c) +{ + return ASCII_ISALNUM(c) || c == '_'; +} + /// Checks if `c` is a binary digit, that is, 0-1. /// /// @see {ascii_isdigit} diff --git a/src/nvim/charset.c b/src/nvim/charset.c index 577fc13a31..980b4ed426 100644 --- a/src/nvim/charset.c +++ b/src/nvim/charset.c @@ -1611,141 +1611,145 @@ bool vim_isblankline(char_u *lbuf) /// If maxlen > 0, check at a maximum maxlen chars. /// /// @param start -/// @param prep Returns type of number 0 = decimal, 'x' or 'X' is hex, -/// '0' = octal, 'b' or 'B' is bin +/// @param prep Returns guessed type of number 0 = decimal, 'x' or 'X' is +/// hexadecimal, '0' = octal, 'b' or 'B' is binary. When using +/// STR2NR_FORCE is always zero. /// @param len Returns the detected length of number. -/// @param what Recognizes what number passed. +/// @param what Recognizes what number passed, @see ChStr2NrFlags. /// @param nptr Returns the signed result. /// @param unptr Returns the unsigned result. /// @param maxlen Max length of string to check. void vim_str2nr(const char_u *const start, int *const prep, int *const len, const int what, varnumber_T *const nptr, uvarnumber_T *const unptr, const int maxlen) + FUNC_ATTR_NONNULL_ARG(1) { - const char_u *ptr = start; + const char *ptr = (const char *)start; +#define STRING_ENDED(ptr) \ + (!(maxlen == 0 || (int)((ptr) - (const char *)start) < maxlen)) int pre = 0; // default is decimal - bool negative = false; + const bool negative = (ptr[0] == '-'); uvarnumber_T un = 0; - if (ptr[0] == '-') { - negative = true; + if (negative) { ptr++; } - // Recognize hex, octal and bin. - if ((ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9') - && (maxlen == 0 || maxlen > 1)) { + if (what & STR2NR_FORCE) { + // When forcing main consideration is skipping the prefix. Octal and decimal + // numbers have no prefixes to skip. pre is not set. + switch ((unsigned)what & (~(unsigned)STR2NR_FORCE)) { + case STR2NR_HEX: { + if (!STRING_ENDED(ptr + 2) + && ptr[0] == '0' + && (ptr[1] == 'x' || ptr[1] == 'X') + && ascii_isxdigit(ptr[2])) { + ptr += 2; + } + goto vim_str2nr_hex; + } + case STR2NR_BIN: { + if (!STRING_ENDED(ptr + 2) + && ptr[0] == '0' + && (ptr[1] == 'b' || ptr[1] == 'B') + && ascii_isbdigit(ptr[2])) { + ptr += 2; + } + goto vim_str2nr_bin; + } + case STR2NR_OCT: { + goto vim_str2nr_oct; + } + case 0: { + goto vim_str2nr_dec; + } + default: { + assert(false); + } + } + } else if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) + && !STRING_ENDED(ptr + 1) + && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { pre = ptr[1]; - + // Detect hexadecimal: 0x or 0X follwed by hex digit if ((what & STR2NR_HEX) - && ((pre == 'X') || (pre == 'x')) - && ascii_isxdigit(ptr[2]) - && (maxlen == 0 || maxlen > 2)) { - // hexadecimal + && !STRING_ENDED(ptr + 2) + && (pre == 'X' || pre == 'x') + && ascii_isxdigit(ptr[2])) { ptr += 2; - } else if ((what & STR2NR_BIN) - && ((pre == 'B') || (pre == 'b')) - && ascii_isbdigit(ptr[2]) - && (maxlen == 0 || maxlen > 2)) { - // binary + goto vim_str2nr_hex; + } + // Detect binary: 0b or 0B follwed by 0 or 1 + if ((what & STR2NR_BIN) + && !STRING_ENDED(ptr + 2) + && (pre == 'B' || pre == 'b') + && ascii_isbdigit(ptr[2])) { ptr += 2; - } else { - // decimal or octal, default is decimal - pre = 0; - - if (what & STR2NR_OCT) { - // Don't interpret "0", "08" or "0129" as octal. - for (int n = 1; ascii_isdigit(ptr[n]); ++n) { - if (ptr[n] > '7') { - // can't be octal - pre = 0; - break; - } - if (ptr[n] >= '0') { - // assume octal - pre = '0'; - } - if (n == maxlen) { - break; - } - } + goto vim_str2nr_bin; + } + // Detect octal number: zero followed by octal digits without '8' or '9' + pre = 0; + if (!(what & STR2NR_OCT) + || !('0' <= ptr[1] && ptr[1] <= '7')) { + goto vim_str2nr_dec; + } + for (int i = 2; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { + if (ptr[i] > '7') { + goto vim_str2nr_dec; } } + pre = '0'; + goto vim_str2nr_oct; + } else { + goto vim_str2nr_dec; } // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. - int n = 1; - if ((pre == 'B') || (pre == 'b') || what == STR2NR_BIN + STR2NR_FORCE) { - // bin - if (pre != 0) { - n += 2; // skip over "0b" + assert(false); // Should’ve used goto earlier. +#define PARSE_NUMBER(base, cond, conv) \ + do { \ + while (!STRING_ENDED(ptr) && (cond)) { \ + /* avoid ubsan error for overflow */ \ + if (un < UVARNUMBER_MAX / base) { \ + un = base * un + (uvarnumber_T)(conv); \ + } else { \ + un = UVARNUMBER_MAX; \ + } \ + ptr++; \ + } \ + } while (0) + switch (pre) { + case 'b': + case 'B': { +vim_str2nr_bin: + PARSE_NUMBER(2, (*ptr == '0' || *ptr == '1'), (*ptr - '0')); + break; } - while ('0' <= *ptr && *ptr <= '1') { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 2) { - un = 2 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - if (n++ == maxlen) { - break; - } - } - } else if ((pre == '0') || what == STR2NR_OCT + STR2NR_FORCE) { - // octal - while ('0' <= *ptr && *ptr <= '7') { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 8) { - un = 8 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - if (n++ == maxlen) { - break; - } + case '0': { +vim_str2nr_oct: + PARSE_NUMBER(8, ('0' <= *ptr && *ptr <= '7'), (*ptr - '0')); + break; } - } else if ((pre == 'X') || (pre == 'x') - || what == STR2NR_HEX + STR2NR_FORCE) { - // hex - if (pre != 0) { - n += 2; // skip over "0x" - } - while (ascii_isxdigit(*ptr)) { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 16) { - un = 16 * un + (uvarnumber_T)hex2nr(*ptr); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - if (n++ == maxlen) { - break; - } + case 0: { +vim_str2nr_dec: + PARSE_NUMBER(10, (ascii_isdigit(*ptr)), (*ptr - '0')); + break; } - } else { - // decimal - while (ascii_isdigit(*ptr)) { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 10) { - un = 10 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - if (n++ == maxlen) { - break; - } + case 'x': + case 'X': { +vim_str2nr_hex: + PARSE_NUMBER(16, (ascii_isxdigit(*ptr)), (hex2nr(*ptr))); + break; } } +#undef PARSE_NUMBER if (prep != NULL) { *prep = pre; } if (len != NULL) { - *len = (int)(ptr - start); + *len = (int)(ptr - (const char *)start); } if (nptr != NULL) { @@ -1767,6 +1771,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, if (unptr != NULL) { *unptr = un; } +#undef STRING_ENDED } /// Return the value of a single hex character. diff --git a/src/nvim/charset.h b/src/nvim/charset.h index c69582c4c6..eb64b6128a 100644 --- a/src/nvim/charset.h +++ b/src/nvim/charset.h @@ -4,6 +4,7 @@ #include "nvim/types.h" #include "nvim/pos.h" #include "nvim/buffer_defs.h" +#include "nvim/eval/typval.h" /// Return the folded-case equivalent of the given character /// @@ -15,6 +16,21 @@ ?((int)(uint8_t)(c)) \ :((int)(c))) +/// Flags for vim_str2nr() +typedef enum { + STR2NR_DEC = 0, + STR2NR_BIN = (1 << 0), ///< Allow binary numbers. + STR2NR_OCT = (1 << 1), ///< Allow octal numbers. + STR2NR_HEX = (1 << 2), ///< Allow hexadecimal numbers. + /// Force one of the above variants. + /// + /// STR2NR_FORCE|STR2NR_DEC is actually not different from supplying zero + /// as flags, but still present for completeness. + STR2NR_FORCE = (1 << 3), + /// Recognize all formats vim_str2nr() can recognize. + STR2NR_ALL = STR2NR_BIN | STR2NR_OCT | STR2NR_HEX, +} ChStr2NrFlags; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "charset.h.generated.h" #endif diff --git a/src/nvim/edit.c b/src/nvim/edit.c index 2bafb77fef..859f98d2ad 100644 --- a/src/nvim/edit.c +++ b/src/nvim/edit.c @@ -6066,27 +6066,30 @@ void free_last_insert(void) #endif -/* - * Add character "c" to buffer "s". Escape the special meaning of K_SPECIAL - * and CSI. Handle multi-byte characters. - * Returns a pointer to after the added bytes. - */ +/// Add character "c" to buffer "s" +/// +/// Escapes the special meaning of K_SPECIAL and CSI, handles multi-byte +/// characters. +/// +/// @param[in] c Character to add. +/// @param[out] s Buffer to add to. Must have at least MB_MAXBYTES + 1 bytes. +/// +/// @return Pointer to after the added bytes. char_u *add_char2buf(int c, char_u *s) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT { char_u temp[MB_MAXBYTES + 1]; - int i; - int len; - - len = (*mb_char2bytes)(c, temp); - for (i = 0; i < len; ++i) { + const int len = utf_char2bytes(c, temp); + for (int i = 0; i < len; i++) { c = temp[i]; - /* Need to escape K_SPECIAL and CSI like in the typeahead buffer. */ + // Need to escape K_SPECIAL and CSI like in the typeahead buffer. if (c == K_SPECIAL) { *s++ = K_SPECIAL; *s++ = KS_SPECIAL; *s++ = KE_FILLER; - } else + } else { *s++ = c; + } } return s; } diff --git a/src/nvim/ex_docmd.c b/src/nvim/ex_docmd.c index e11788531b..f6a5f59676 100644 --- a/src/nvim/ex_docmd.c +++ b/src/nvim/ex_docmd.c @@ -5925,9 +5925,10 @@ static void ex_colorscheme(exarg_T *eap) static void ex_highlight(exarg_T *eap) { - if (*eap->arg == NUL && eap->cmd[2] == '!') + if (*eap->arg == NUL && eap->cmd[2] == '!') { MSG(_("Greetings, Vim user!")); - do_highlight(eap->arg, eap->forceit, FALSE); + } + do_highlight((const char *)eap->arg, eap->forceit, false); } diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c index 99330c177c..1adc8325f8 100644 --- a/src/nvim/ex_getln.c +++ b/src/nvim/ex_getln.c @@ -66,6 +66,8 @@ #include "nvim/lib/kvec.h" #include "nvim/api/private/helpers.h" #include "nvim/highlight_defs.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/viml/parser/expressions.h" /// Command-line colors: one chunk /// @@ -2428,6 +2430,63 @@ void free_cmdline_buf(void) enum { MAX_CB_ERRORS = 1 }; +/// Color expression cmdline using built-in expressions parser +/// +/// @param[in] colored_ccline Command-line to color. +/// @param[out] ret_ccline_colors What should be colored. +/// +/// Always colors the whole cmdline. +static void color_expr_cmdline(const CmdlineInfo *const colored_ccline, + ColoredCmdline *const ret_ccline_colors) + FUNC_ATTR_NONNULL_ALL +{ + ParserLine plines[] = { + { + .data = (const char *)colored_ccline->cmdbuff, + .size = STRLEN(colored_ccline->cmdbuff), + .allocated = false, + }, + { NULL, 0, false }, + }; + ParserLine *plines_p = plines; + ParserHighlight colors; + kvi_init(colors); + ParserState pstate; + viml_parser_init( + &pstate, parser_simple_get_line, &plines_p, &colors); + ExprAST east = viml_pexpr_parse(&pstate, kExprFlagsDisallowEOC); + viml_pexpr_free_ast(east); + viml_parser_destroy(&pstate); + kv_resize(ret_ccline_colors->colors, kv_size(colors)); + size_t prev_end = 0; + for (size_t i = 0 ; i < kv_size(colors) ; i++) { + const ParserHighlightChunk chunk = kv_A(colors, i); + if (chunk.start.col != prev_end) { + kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) { + .start = prev_end, + .end = chunk.start.col, + .attr = 0, + })); + } + const int id = syn_name2id((const char_u *)chunk.group); + const int attr = (id == 0 ? 0 : syn_id2attr(id)); + kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) { + .start = chunk.start.col, + .end = chunk.end_col, + .attr = attr, + })); + prev_end = chunk.end_col; + } + if (prev_end < (size_t)colored_ccline->cmdlen) { + kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) { + .start = prev_end, + .end = (size_t)colored_ccline->cmdlen, + .attr = 0, + })); + } + kvi_destroy(colors); +} + /// Color command-line /// /// Should use built-in command parser or user-specified one. Currently only the @@ -2510,13 +2569,7 @@ static bool color_cmdline(CmdlineInfo *colored_ccline) tl_ret = try_leave(&tstate, &err); can_free_cb = true; } else if (colored_ccline->cmdfirstc == '=') { - try_enter(&tstate); - err_errmsg = N_( - "E5409: Unable to get g:Nvim_color_expr callback: %s"); - dgc_ret = tv_dict_get_callback(&globvardict, S_LEN("Nvim_color_expr"), - &color_cb); - tl_ret = try_leave(&tstate, &err); - can_free_cb = true; + color_expr_cmdline(colored_ccline, ccline_colors); } if (!tl_ret || !dgc_ret) { goto color_cmdline_error; diff --git a/src/nvim/generators/gen_declarations.lua b/src/nvim/generators/gen_declarations.lua index e999e53e4a..065c043557 100755 --- a/src/nvim/generators/gen_declarations.lua +++ b/src/nvim/generators/gen_declarations.lua @@ -164,9 +164,40 @@ local pattern = concat( ) if fname == '--help' then - print'Usage:' - print() - print' gendeclarations.lua definitions.c static.h non-static.h preprocessor.i' + print([[ +Usage: + + gendeclarations.lua definitions.c static.h non-static.h definitions.i + +Generates declarations for a C file definitions.c, putting declarations for +static functions into static.h and declarations for non-static functions into +non-static.h. File `definitions.i' should contain an already preprocessed +version of definitions.c and it is the only one which is actually parsed, +definitions.c is needed only to determine functions from which file out of all +functions found in definitions.i are needed. + +Additionally uses the following environment variables: + + NVIM_GEN_DECLARATIONS_LINE_NUMBERS: + If set to 1 then all generated declarations receive a comment with file + name and line number after the declaration. This may be useful for + debugging gen_declarations script, but not much beyond that with + configured development environment (i.e. with ctags/cscope/finding + definitions with clang/etc). + + WARNING: setting this to 1 will cause extensive rebuilds: declarations + generator script will not regenerate non-static.h file if its + contents did not change, but including line numbers will make + contents actually change. + + With contents changed timestamp of the file is regenerated even + when no real changes were made (e.g. a few lines were added to + a function which is not at the bottom of the file). + + With changed timestamp build system will assume that header + changed, triggering rebuilds of all C files which depend on the + "changed" header. +]]) os.exit() end @@ -249,8 +280,10 @@ while init ~= nil do declaration = declaration:gsub(' $', '') declaration = declaration:gsub('^ ', '') declaration = declaration .. ';' - declaration = declaration .. (' // %s/%s:%u'):format( - curdir, curfile, declline) + if os.getenv('NVIM_GEN_DECLARATIONS_LINE_NUMBERS') == '1' then + declaration = declaration .. (' // %s/%s:%u'):format( + curdir, curfile, declline) + end declaration = declaration .. '\n' if declaration:sub(1, 6) == 'static' then static = static .. declaration diff --git a/src/nvim/globals.h b/src/nvim/globals.h index dcb8b40973..ef56954aa0 100644 --- a/src/nvim/globals.h +++ b/src/nvim/globals.h @@ -729,29 +729,6 @@ EXTERN int vr_lines_changed INIT(= 0); /* #Lines changed by "gR" so far */ /// Encoding used when 'fencs' is set to "default" EXTERN char_u *fenc_default INIT(= NULL); -// To speed up BYTELEN(); keep a lookup table to quickly get the length in -// bytes of a UTF-8 character from the first byte of a UTF-8 string. Bytes -// which are illegal when used as the first byte have a 1. The NUL byte has -// length 1. -EXTERN char utf8len_tab[256] INIT(= { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, -}); - # if defined(USE_ICONV) && defined(DYNAMIC_ICONV) /* Pointers to functions and variables to be loaded at runtime */ EXTERN size_t (*iconv)(iconv_t cd, const char **inbuf, size_t *inbytesleft, diff --git a/src/nvim/keymap.c b/src/nvim/keymap.c index a75fe793ac..0eb5a41b1e 100644 --- a/src/nvim/keymap.c +++ b/src/nvim/keymap.c @@ -24,23 +24,22 @@ * Some useful tables. */ -static struct modmasktable { - short mod_mask; /* Bit-mask for particular key modifier */ - short mod_flag; /* Bit(s) for particular key modifier */ - char_u name; /* Single letter name of modifier */ -} mod_mask_table[] = -{ - {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'M'}, - {MOD_MASK_META, MOD_MASK_META, (char_u)'T'}, - {MOD_MASK_CTRL, MOD_MASK_CTRL, (char_u)'C'}, - {MOD_MASK_SHIFT, MOD_MASK_SHIFT, (char_u)'S'}, - {MOD_MASK_MULTI_CLICK, MOD_MASK_2CLICK, (char_u)'2'}, - {MOD_MASK_MULTI_CLICK, MOD_MASK_3CLICK, (char_u)'3'}, - {MOD_MASK_MULTI_CLICK, MOD_MASK_4CLICK, (char_u)'4'}, - {MOD_MASK_CMD, MOD_MASK_CMD, (char_u)'D'}, +static const struct modmasktable { + uint16_t mod_mask; ///< Bit-mask for particular key modifier. + uint16_t mod_flag; ///< Bit(s) for particular key modifier. + char_u name; ///< Single letter name of modifier. +} mod_mask_table[] = { + { MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'M' }, + { MOD_MASK_META, MOD_MASK_META, (char_u)'T' }, + { MOD_MASK_CTRL, MOD_MASK_CTRL, (char_u)'C' }, + { MOD_MASK_SHIFT, MOD_MASK_SHIFT, (char_u)'S' }, + { MOD_MASK_MULTI_CLICK, MOD_MASK_2CLICK, (char_u)'2' }, + { MOD_MASK_MULTI_CLICK, MOD_MASK_3CLICK, (char_u)'3' }, + { MOD_MASK_MULTI_CLICK, MOD_MASK_4CLICK, (char_u)'4' }, + { MOD_MASK_CMD, MOD_MASK_CMD, (char_u)'D' }, // 'A' must be the last one - {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'A'}, - {0, 0, NUL} + { MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'A' }, + { 0, 0, NUL } }; /* @@ -139,11 +138,10 @@ static char_u modifier_keys_table[] = NUL }; -static struct key_name_entry { +static const struct key_name_entry { int key; // Special key code or ascii value - char *name; // Name of key -} key_names_table[] = -{ + const char *name; // Name of key +} key_names_table[] = { { ' ', "Space" }, { TAB, "Tab" }, { K_TAB, "Tab" }, @@ -318,73 +316,73 @@ static struct mousetable { {0, 0, 0, 0}, }; -/* - * Return the modifier mask bit (MOD_MASK_*) which corresponds to the given - * modifier name ('S' for Shift, 'C' for Ctrl etc). - */ +/// Return the modifier mask bit (#MOD_MASK_*) corresponding to mod name +/// +/// E.g. 'S' for shift, 'C' for ctrl. int name_to_mod_mask(int c) + FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT { - int i; - c = TOUPPER_ASC(c); - for (i = 0; mod_mask_table[i].mod_mask != 0; i++) - if (c == mod_mask_table[i].name) + for (size_t i = 0; mod_mask_table[i].mod_mask != 0; i++) { + if (c == mod_mask_table[i].name) { return mod_mask_table[i].mod_flag; + } + } return 0; } -/* - * Check if if there is a special key code for "key" that includes the - * modifiers specified. - */ -int simplify_key(int key, int *modifiers) +/// Check if there is a special key code for "key" with specified modifiers +/// +/// @param[in] key Initial key code. +/// @param[in,out] modifiers Initial modifiers, is adjusted to have simplified +/// modifiers. +/// +/// @return Simplified key code. +int simplify_key(const int key, int *modifiers) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - int i; - int key0; - int key1; - if (*modifiers & (MOD_MASK_SHIFT | MOD_MASK_CTRL | MOD_MASK_ALT)) { - /* TAB is a special case */ + // TAB is a special case. if (key == TAB && (*modifiers & MOD_MASK_SHIFT)) { *modifiers &= ~MOD_MASK_SHIFT; return K_S_TAB; } - key0 = KEY2TERMCAP0(key); - key1 = KEY2TERMCAP1(key); - for (i = 0; modifier_keys_table[i] != NUL; i += MOD_KEYS_ENTRY_SIZE) + const int key0 = KEY2TERMCAP0(key); + const int key1 = KEY2TERMCAP1(key); + for (int i = 0; modifier_keys_table[i] != NUL; i += MOD_KEYS_ENTRY_SIZE) { if (key0 == modifier_keys_table[i + 3] && key1 == modifier_keys_table[i + 4] && (*modifiers & modifier_keys_table[i])) { *modifiers &= ~modifier_keys_table[i]; return TERMCAP2KEY(modifier_keys_table[i + 1], - modifier_keys_table[i + 2]); + modifier_keys_table[i + 2]); } + } } return key; } -/* - * Change <xHome> to <Home>, <xUp> to <Up>, etc. - */ -int handle_x_keys(int key) +/// Change <xKey> to <Key> +int handle_x_keys(const int key) + FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT { switch (key) { - case K_XUP: return K_UP; - case K_XDOWN: return K_DOWN; - case K_XLEFT: return K_LEFT; - case K_XRIGHT: return K_RIGHT; - case K_XHOME: return K_HOME; - case K_ZHOME: return K_HOME; - case K_XEND: return K_END; - case K_ZEND: return K_END; - case K_XF1: return K_F1; - case K_XF2: return K_F2; - case K_XF3: return K_F3; - case K_XF4: return K_F4; - case K_S_XF1: return K_S_F1; - case K_S_XF2: return K_S_F2; - case K_S_XF3: return K_S_F3; - case K_S_XF4: return K_S_F4; + case K_XUP: return K_UP; + case K_XDOWN: return K_DOWN; + case K_XLEFT: return K_LEFT; + case K_XRIGHT: return K_RIGHT; + case K_XHOME: return K_HOME; + case K_ZHOME: return K_HOME; + case K_XEND: return K_END; + case K_ZEND: return K_END; + case K_XF1: return K_F1; + case K_XF2: return K_F2; + case K_XF3: return K_F3; + case K_XF4: return K_F4; + case K_S_XF1: return K_S_F1; + case K_S_XF2: return K_S_F2; + case K_S_XF3: return K_S_F3; + case K_S_XF4: return K_S_F4; } return key; } @@ -508,7 +506,7 @@ unsigned int trans_special(const char_u **srcp, const size_t src_len, return 0; } - /* Put the appropriate modifier in a string */ + // Put the appropriate modifier in a string. if (modifiers != 0) { dst[dlen++] = K_SPECIAL; dst[dlen++] = KS_MODIFIER; @@ -569,15 +567,11 @@ int find_special_key(const char_u **srcp, const size_t src_len, int *const modp, // Find end of modifier list last_dash = src; - for (bp = src + 1; bp <= end && (*bp == '-' || vim_isIDc(*bp)); bp++) { + for (bp = src + 1; bp <= end && (*bp == '-' || ascii_isident(*bp)); bp++) { if (*bp == '-') { last_dash = bp; if (bp + 1 <= end) { - if (has_mbyte) { - l = mb_ptr2len_len(bp + 1, (int) (end - bp) + 1); - } else { - l = 1; - } + l = utfc_ptr2len_len(bp + 1, (int)(end - bp) + 1); // Anything accepted, like <C-?>. // <C-"> or <M-"> are not special in strings as " is // the string delimiter. With a backslash it works: <M-\"> @@ -702,33 +696,39 @@ int find_special_key_in_table(int c) { int i; - for (i = 0; key_names_table[i].name != NULL; i++) - if (c == key_names_table[i].key) + for (i = 0; key_names_table[i].name != NULL; i++) { + if (c == key_names_table[i].key) { break; - if (key_names_table[i].name == NULL) + } + } + if (key_names_table[i].name == NULL) { i = -1; + } return i; } -/* - * Find the special key with the given name (the given string does not have to - * end with NUL, the name is assumed to end before the first non-idchar). - * If the name starts with "t_" the next two characters are interpreted as a - * termcap name. - * Return the key code, or 0 if not found. - */ +/// Find the special key with the given name +/// +/// @param[in] name Name of the special. Does not have to end with NUL, it is +/// assumed to end before the first non-idchar. If name starts +/// with "t_" the next two characters are interpreted as +/// a termcap name. +/// +/// @return Key code or 0 if not found. int get_special_key_code(const char_u *name) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT { - char *table_name; - int i, j; - - for (i = 0; key_names_table[i].name != NULL; i++) { - table_name = key_names_table[i].name; - for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) - if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) + for (int i = 0; key_names_table[i].name != NULL; i++) { + const char *const table_name = key_names_table[i].name; + int j; + for (j = 0; ascii_isident(name[j]) && table_name[j] != NUL; j++) { + if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) { break; - if (!vim_isIDc(name[j]) && table_name[j] == NUL) + } + } + if (!ascii_isident(name[j]) && table_name[j] == NUL) { return key_names_table[i].key; + } } return 0; diff --git a/src/nvim/lib/kvec.h b/src/nvim/lib/kvec.h index 584282d773..ee1b890cb9 100644 --- a/src/nvim/lib/kvec.h +++ b/src/nvim/lib/kvec.h @@ -62,7 +62,16 @@ #define kv_pop(v) ((v).items[--(v).size]) #define kv_size(v) ((v).size) #define kv_max(v) ((v).capacity) -#define kv_last(v) kv_A(v, kv_size(v) - 1) +#define kv_Z(v, i) kv_A(v, kv_size(v) - (i) - 1) +#define kv_last(v) kv_Z(v, 0) + +/// Drop last n items from kvec without resizing +/// +/// Previously spelled as `(void)kv_pop(v)`, repeated n times. +/// +/// @param[out] v Kvec to drop items from. +/// @param[in] n Number of elements to drop. +#define kv_drop(v, n) ((v).size -= (n)) #define kv_resize(v, s) \ ((v).capacity = (s), \ diff --git a/src/nvim/lib/ringbuf.h b/src/nvim/lib/ringbuf.h index 12b75ec65a..e63eae70b0 100644 --- a/src/nvim/lib/ringbuf.h +++ b/src/nvim/lib/ringbuf.h @@ -15,6 +15,7 @@ #ifndef NVIM_LIB_RINGBUF_H #define NVIM_LIB_RINGBUF_H +#include <stddef.h> #include <string.h> #include <assert.h> #include <stdint.h> @@ -73,6 +74,32 @@ typedef struct { \ RBType *buf_end; \ } TypeName##RingBuffer; +/// Dummy item free macros, for use in RINGBUF_INIT +/// +/// This macros actually does nothing. +/// +/// @param[in] item Item to be freed. +#define RINGBUF_DUMMY_FREE(item) + +/// Static ring buffer +/// +/// @warning Ring buffers created with this macros must neither be freed nor +/// deallocated. +/// +/// @param scope Ring buffer scope. +/// @param TypeName Ring buffer type name. +/// @param RBType Type of the single ring buffer element. +/// @param varname Variable name. +/// @param rbsize Ring buffer size. +#define RINGBUF_STATIC(scope, TypeName, RBType, varname, rbsize) \ +static RBType _##varname##_buf[rbsize]; \ +scope TypeName##RingBuffer varname = { \ + .buf = _##varname##_buf, \ + .next = _##varname##_buf, \ + .first = NULL, \ + .buf_end = _##varname##_buf + rbsize - 1, \ +}; + /// Initialize a new ring buffer /// /// @param TypeName Ring buffer type name. Actual type name will be diff --git a/src/nvim/mbyte.c b/src/nvim/mbyte.c index b24770a409..008bce6df6 100644 --- a/src/nvim/mbyte.c +++ b/src/nvim/mbyte.c @@ -72,19 +72,49 @@ struct interval { # include "unicode_tables.generated.h" #endif -/* - * Like utf8len_tab above, but using a zero for illegal lead bytes. - */ -const uint8_t utf8len_tab_zero[256] = -{ - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0, +// To speed up BYTELEN(); keep a lookup table to quickly get the length in +// bytes of a UTF-8 character from the first byte of a UTF-8 string. Bytes +// which are illegal when used as the first byte have a 1. The NUL byte has +// length 1. +const uint8_t utf8len_tab[] = { + // ?1 ?2 ?3 ?4 ?5 ?6 ?7 ?8 ?9 ?A ?B ?C ?D ?E ?F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D? + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E? + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, // F? +}; + +// Like utf8len_tab above, but using a zero for illegal lead bytes. +const uint8_t utf8len_tab_zero[] = { + // ?1 ?2 ?3 ?4 ?5 ?6 ?7 ?8 ?9 ?A ?B ?C ?D ?E ?F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7? + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 8? + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 9? + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // A? + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // B? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D? + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E? + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0, // F? }; /* @@ -528,45 +558,52 @@ int utf_off2cells(unsigned off, unsigned max_off) return (off + 1 < max_off && ScreenLines[off + 1] == 0) ? 2 : 1; } -/* - * Convert a UTF-8 byte sequence to a wide character. - * If the sequence is illegal or truncated by a NUL the first byte is - * returned. - * Does not include composing characters, of course. - */ -int utf_ptr2char(const char_u *p) +/// Convert a UTF-8 byte sequence to a wide character +/// +/// If the sequence is illegal or truncated by a NUL then the first byte is +/// returned. Does not include composing characters for obvious reasons. +/// +/// @param[in] p String to convert. +/// +/// @return Unicode codepoint or byte value. +int utf_ptr2char(const char_u *const p) + FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT { - uint8_t len; - - if (p[0] < 0x80) /* be quick for ASCII */ + if (p[0] < 0x80) { // Be quick for ASCII. return p[0]; + } - len = utf8len_tab_zero[p[0]]; + const uint8_t len = utf8len_tab_zero[p[0]]; if (len > 1 && (p[1] & 0xc0) == 0x80) { - if (len == 2) + if (len == 2) { return ((p[0] & 0x1f) << 6) + (p[1] & 0x3f); + } if ((p[2] & 0xc0) == 0x80) { - if (len == 3) - return ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) - + (p[2] & 0x3f); + if (len == 3) { + return (((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + + (p[2] & 0x3f)); + } if ((p[3] & 0xc0) == 0x80) { - if (len == 4) - return ((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12) - + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f); + if (len == 4) { + return (((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12) + + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f)); + } if ((p[4] & 0xc0) == 0x80) { - if (len == 5) - return ((p[0] & 0x03) << 24) + ((p[1] & 0x3f) << 18) - + ((p[2] & 0x3f) << 12) + ((p[3] & 0x3f) << 6) - + (p[4] & 0x3f); - if ((p[5] & 0xc0) == 0x80 && len == 6) - return ((p[0] & 0x01) << 30) + ((p[1] & 0x3f) << 24) - + ((p[2] & 0x3f) << 18) + ((p[3] & 0x3f) << 12) - + ((p[4] & 0x3f) << 6) + (p[5] & 0x3f); + if (len == 5) { + return (((p[0] & 0x03) << 24) + ((p[1] & 0x3f) << 18) + + ((p[2] & 0x3f) << 12) + ((p[3] & 0x3f) << 6) + + (p[4] & 0x3f)); + } + if ((p[5] & 0xc0) == 0x80 && len == 6) { + return (((p[0] & 0x01) << 30) + ((p[1] & 0x3f) << 24) + + ((p[2] & 0x3f) << 18) + ((p[3] & 0x3f) << 12) + + ((p[4] & 0x3f) << 6) + (p[5] & 0x3f)); + } } } } } - /* Illegal value, just return the first byte */ + // Illegal value: just return the first byte. return p[0]; } @@ -767,23 +804,24 @@ int utfc_char2bytes(int off, char_u *buf) return len; } -/* - * Get the length of a UTF-8 byte sequence, not including any following - * composing characters. - * Returns 0 for "". - * Returns 1 for an illegal byte sequence. - */ -int utf_ptr2len(const char_u *p) +/// Get the length of a UTF-8 byte sequence representing a single codepoint +/// +/// @param[in] p UTF-8 string. +/// +/// @return Sequence length, 0 for empty string and 1 for non-UTF-8 byte +/// sequence. +int utf_ptr2len(const char_u *const p) + FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - int len; - int i; - - if (*p == NUL) + if (*p == NUL) { return 0; - len = utf8len_tab[*p]; - for (i = 1; i < len; ++i) - if ((p[i] & 0xc0) != 0x80) + } + const int len = utf8len_tab[*p]; + for (int i = 1; i < len; i++) { + if ((p[i] & 0xc0) != 0x80) { return 1; + } + } return len; } @@ -824,38 +862,38 @@ int utf_ptr2len_len(const char_u *p, int size) return len; } -/* - * Return the number of bytes the UTF-8 encoding of the character at "p" takes. - * This includes following composing characters. - */ -int utfc_ptr2len(const char_u *p) +/// Return the number of bytes occupied by a UTF-8 character in a string +/// +/// This includes following composing characters. +int utfc_ptr2len(const char_u *const p) + FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - int len; - int b0 = *p; - int prevlen; + uint8_t b0 = (uint8_t)(*p); - if (b0 == NUL) + if (b0 == NUL) { return 0; - if (b0 < 0x80 && p[1] < 0x80) /* be quick for ASCII */ + } + if (b0 < 0x80 && p[1] < 0x80) { // be quick for ASCII return 1; + } - /* Skip over first UTF-8 char, stopping at a NUL byte. */ - len = utf_ptr2len(p); + // Skip over first UTF-8 char, stopping at a NUL byte. + int len = utf_ptr2len(p); - /* Check for illegal byte. */ - if (len == 1 && b0 >= 0x80) + // Check for illegal byte. + if (len == 1 && b0 >= 0x80) { return 1; + } - /* - * Check for composing characters. We can handle only the first six, but - * skip all of them (otherwise the cursor would get stuck). - */ - prevlen = 0; - for (;; ) { - if (p[len] < 0x80 || !UTF_COMPOSINGLIKE(p + prevlen, p + len)) + // Check for composing characters. We can handle only the first six, but + // skip all of them (otherwise the cursor would get stuck). + int prevlen = 0; + for (;;) { + if (p[len] < 0x80 || !UTF_COMPOSINGLIKE(p + prevlen, p + len)) { return len; + } - /* Skip over composing char */ + // Skip over composing char. prevlen = len; len += utf_ptr2len(p + len); } @@ -913,23 +951,22 @@ int utfc_ptr2len_len(const char_u *p, int size) return len; } -/* - * Return the number of bytes the UTF-8 encoding of character "c" takes. - * This does not include composing characters. - */ -int utf_char2len(int c) +/// Determine how many bytes certain unicode codepoint will occupy +int utf_char2len(const int c) { - if (c < 0x80) + if (c < 0x80) { return 1; - if (c < 0x800) + } else if (c < 0x800) { return 2; - if (c < 0x10000) + } else if (c < 0x10000) { return 3; - if (c < 0x200000) + } else if (c < 0x200000) { return 4; - if (c < 0x4000000) + } else if (c < 0x4000000) { return 5; - return 6; + } else { + return 6; + } } /// Convert Unicode character to UTF-8 string @@ -937,46 +974,42 @@ int utf_char2len(int c) /// @param c character to convert to \p buf /// @param[out] buf UTF-8 string generated from \p c, does not add \0 /// @return Number of bytes (1-6). Does not include composing characters. -int utf_char2bytes(int c, char_u *const buf) +int utf_char2bytes(const int c, char_u *const buf) { - if (c < 0x80) { /* 7 bits */ + if (c < 0x80) { // 7 bits buf[0] = c; return 1; - } - if (c < 0x800) { /* 11 bits */ + } else if (c < 0x800) { // 11 bits buf[0] = 0xc0 + ((unsigned)c >> 6); buf[1] = 0x80 + (c & 0x3f); return 2; - } - if (c < 0x10000) { /* 16 bits */ + } else if (c < 0x10000) { // 16 bits buf[0] = 0xe0 + ((unsigned)c >> 12); buf[1] = 0x80 + (((unsigned)c >> 6) & 0x3f); buf[2] = 0x80 + (c & 0x3f); return 3; - } - if (c < 0x200000) { /* 21 bits */ + } else if (c < 0x200000) { // 21 bits buf[0] = 0xf0 + ((unsigned)c >> 18); buf[1] = 0x80 + (((unsigned)c >> 12) & 0x3f); buf[2] = 0x80 + (((unsigned)c >> 6) & 0x3f); buf[3] = 0x80 + (c & 0x3f); return 4; - } - if (c < 0x4000000) { /* 26 bits */ + } else if (c < 0x4000000) { // 26 bits buf[0] = 0xf8 + ((unsigned)c >> 24); buf[1] = 0x80 + (((unsigned)c >> 18) & 0x3f); buf[2] = 0x80 + (((unsigned)c >> 12) & 0x3f); buf[3] = 0x80 + (((unsigned)c >> 6) & 0x3f); buf[4] = 0x80 + (c & 0x3f); return 5; + } else { // 31 bits + buf[0] = 0xfc + ((unsigned)c >> 30); + buf[1] = 0x80 + (((unsigned)c >> 24) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 18) & 0x3f); + buf[3] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[4] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[5] = 0x80 + (c & 0x3f); + return 6; } - /* 31 bits */ - buf[0] = 0xfc + ((unsigned)c >> 30); - buf[1] = 0x80 + (((unsigned)c >> 24) & 0x3f); - buf[2] = 0x80 + (((unsigned)c >> 18) & 0x3f); - buf[3] = 0x80 + (((unsigned)c >> 12) & 0x3f); - buf[4] = 0x80 + (((unsigned)c >> 6) & 0x3f); - buf[5] = 0x80 + (c & 0x3f); - return 6; } /* @@ -1513,14 +1546,15 @@ int utf_head_off(const char_u *base, const char_u *p) return (int)(p - q); } -/* - * Copy a character from "*fp" to "*tp" and advance the pointers. - */ -void mb_copy_char(const char_u **fp, char_u **tp) +/// Copy a character, advancing the pointers +/// +/// @param[in,out] fp Source of the character to copy. +/// @param[in,out] tp Destination to copy to. +void mb_copy_char(const char_u **const fp, char_u **const tp) { - int l = (*mb_ptr2len)(*fp); + const size_t l = (size_t)utfc_ptr2len(*fp); - memmove(*tp, *fp, (size_t)l); + memmove(*tp, *fp, l); *tp += l; *fp += l; } @@ -2262,9 +2296,7 @@ int convert_setup_ext(vimconv_T *vcp, char_u *from, bool from_unicode_is_utf8, if (vcp->vc_type == CONV_ICONV && vcp->vc_fd != (iconv_t)-1) iconv_close(vcp->vc_fd); # endif - vcp->vc_type = CONV_NONE; - vcp->vc_factor = 1; - vcp->vc_fail = false; + *vcp = (vimconv_T)MBYTE_NONE_CONV; /* No conversion when one of the names is empty or they are equal. */ if (from == NULL || *from == NUL || to == NULL || *to == NUL diff --git a/src/nvim/mbyte.h b/src/nvim/mbyte.h index bf6ccb13a5..a5ce1b0a15 100644 --- a/src/nvim/mbyte.h +++ b/src/nvim/mbyte.h @@ -60,6 +60,12 @@ typedef enum { CONV_ICONV = 5, } ConvFlags; +#define MBYTE_NONE_CONV { \ + .vc_type = CONV_NONE, \ + .vc_factor = 1, \ + .vc_fail = false, \ +} + /// Structure used for string conversions typedef struct { int vc_type; ///< Zero or more ConvFlags. @@ -73,6 +79,8 @@ typedef struct { extern const uint8_t utf8len_tab_zero[256]; +extern const uint8_t utf8len_tab[256]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "mbyte.h.generated.h" #endif diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index bc7362af72..d1a5f0bd1c 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -5930,9 +5930,9 @@ static void syntime_report(void) // // When making changes here, also change runtime/colors/default.vim! -static char *highlight_init_both[] = -{ - "Conceal ctermbg=DarkGrey ctermfg=LightGrey guibg=DarkGrey guifg=LightGrey", +static const char *highlight_init_both[] = { + "Conceal " + "ctermbg=DarkGrey ctermfg=LightGrey guibg=DarkGrey guifg=LightGrey", "Cursor guibg=fg guifg=bg", "lCursor guibg=fg guifg=bg", "DiffText cterm=bold ctermbg=Red gui=bold guibg=Red", @@ -5955,8 +5955,7 @@ static char *highlight_init_both[] = NULL }; -static char *highlight_init_light[] = -{ +static const char *highlight_init_light[] = { "ColorColumn ctermbg=LightRed guibg=LightRed", "CursorColumn ctermbg=LightGrey guibg=Grey90", "CursorLine cterm=underline guibg=Grey90", @@ -5989,8 +5988,7 @@ static char *highlight_init_light[] = NULL }; -static char *highlight_init_dark[] = -{ +static const char *highlight_init_dark[] = { "ColorColumn ctermbg=DarkRed guibg=DarkRed", "CursorColumn ctermbg=DarkGrey guibg=Grey40", "CursorLine cterm=underline guibg=Grey40", @@ -6023,18 +6021,223 @@ static char *highlight_init_dark[] = NULL }; +const char *const highlight_init_cmdline[] = { + // XXX When modifying a list modify it in both valid and invalid halfs. + // TODO(ZyX-I): merge valid and invalid groups via a macros. + + // NvimInternalError should appear only when highlighter has a bug. + "NvimInternalError ctermfg=Red ctermbg=Red guifg=Red guibg=Red", + + // Highlight groups (links) used by parser: + + "default link NvimAssignment Operator", + "default link NvimPlainAssignment NvimAssignment", + "default link NvimAugmentedAssignment NvimAssignment", + "default link NvimAssignmentWithAddition NvimAugmentedAssignment", + "default link NvimAssignmentWithSubtraction NvimAugmentedAssignment", + "default link NvimAssignmentWithConcatenation NvimAugmentedAssignment", + + "default link NvimOperator Operator", + + "default link NvimUnaryOperator NvimOperator", + "default link NvimUnaryPlus NvimUnaryOperator", + "default link NvimUnaryMinus NvimUnaryOperator", + "default link NvimNot NvimUnaryOperator", + + "default link NvimBinaryOperator NvimOperator", + "default link NvimComparison NvimBinaryOperator", + "default link NvimComparisonModifier NvimComparison", + "default link NvimBinaryPlus NvimBinaryOperator", + "default link NvimBinaryMinus NvimBinaryOperator", + "default link NvimConcat NvimBinaryOperator", + "default link NvimConcatOrSubscript NvimConcat", + "default link NvimOr NvimBinaryOperator", + "default link NvimAnd NvimBinaryOperator", + "default link NvimMultiplication NvimBinaryOperator", + "default link NvimDivision NvimBinaryOperator", + "default link NvimMod NvimBinaryOperator", + + "default link NvimTernary NvimOperator", + "default link NvimTernaryColon NvimTernary", + + "default link NvimParenthesis Delimiter", + "default link NvimLambda NvimParenthesis", + "default link NvimNestingParenthesis NvimParenthesis", + "default link NvimCallingParenthesis NvimParenthesis", + + "default link NvimSubscript NvimParenthesis", + "default link NvimSubscriptBracket NvimSubscript", + "default link NvimSubscriptColon NvimSubscript", + "default link NvimCurly NvimSubscript", + + "default link NvimContainer NvimParenthesis", + "default link NvimDict NvimContainer", + "default link NvimList NvimContainer", + + "default link NvimIdentifier Identifier", + "default link NvimIdentifierScope NvimIdentifier", + "default link NvimIdentifierScopeDelimiter NvimIdentifier", + "default link NvimIdentifierName NvimIdentifier", + "default link NvimIdentifierKey NvimIdentifier", + + "default link NvimColon Delimiter", + "default link NvimComma Delimiter", + "default link NvimArrow Delimiter", + + "default link NvimRegister SpecialChar", + "default link NvimNumber Number", + "default link NvimFloat NvimNumber", + "default link NvimNumberPrefix Type", + + "default link NvimOptionSigil Type", + "default link NvimOptionName NvimIdentifier", + "default link NvimOptionScope NvimIdentifierScope", + "default link NvimOptionScopeDelimiter NvimIdentifierScopeDelimiter", + + "default link NvimEnvironmentSigil NvimOptionSigil", + "default link NvimEnvironmentName NvimIdentifier", + + "default link NvimString String", + "default link NvimStringBody NvimString", + "default link NvimStringQuote NvimString", + "default link NvimStringSpecial SpecialChar", + + "default link NvimSingleQuote NvimStringQuote", + "default link NvimSingleQuotedBody NvimStringBody", + "default link NvimSingleQuotedQuote NvimStringSpecial", + + "default link NvimDoubleQuote NvimStringQuote", + "default link NvimDoubleQuotedBody NvimStringBody", + "default link NvimDoubleQuotedEscape NvimStringSpecial", + + "default link NvimFigureBrace NvimInternalError", + "default link NvimSingleQuotedUnknownEscape NvimInternalError", + + "default link NvimSpacing Normal", + + // NvimInvalid groups: + + "default link NvimInvalidSingleQuotedUnknownEscape NvimInternalError", + + "default link NvimInvalid Error", + + "default link NvimInvalidAssignment NvimInvalid", + "default link NvimInvalidPlainAssignment NvimInvalidAssignment", + "default link NvimInvalidAugmentedAssignment NvimInvalidAssignment", + "default link NvimInvalidAssignmentWithAddition " + "NvimInvalidAugmentedAssignment", + "default link NvimInvalidAssignmentWithSubtraction " + "NvimInvalidAugmentedAssignment", + "default link NvimInvalidAssignmentWithConcatenation " + "NvimInvalidAugmentedAssignment", + + "default link NvimInvalidOperator NvimInvalid", + + "default link NvimInvalidUnaryOperator NvimInvalidOperator", + "default link NvimInvalidUnaryPlus NvimInvalidUnaryOperator", + "default link NvimInvalidUnaryMinus NvimInvalidUnaryOperator", + "default link NvimInvalidNot NvimInvalidUnaryOperator", + + "default link NvimInvalidBinaryOperator NvimInvalidOperator", + "default link NvimInvalidComparison NvimInvalidBinaryOperator", + "default link NvimInvalidComparisonModifier NvimInvalidComparison", + "default link NvimInvalidBinaryPlus NvimInvalidBinaryOperator", + "default link NvimInvalidBinaryMinus NvimInvalidBinaryOperator", + "default link NvimInvalidConcat NvimInvalidBinaryOperator", + "default link NvimInvalidConcatOrSubscript NvimInvalidConcat", + "default link NvimInvalidOr NvimInvalidBinaryOperator", + "default link NvimInvalidAnd NvimInvalidBinaryOperator", + "default link NvimInvalidMultiplication NvimInvalidBinaryOperator", + "default link NvimInvalidDivision NvimInvalidBinaryOperator", + "default link NvimInvalidMod NvimInvalidBinaryOperator", + + "default link NvimInvalidTernary NvimInvalidOperator", + "default link NvimInvalidTernaryColon NvimInvalidTernary", + + "default link NvimInvalidDelimiter NvimInvalid", + + "default link NvimInvalidParenthesis NvimInvalidDelimiter", + "default link NvimInvalidLambda NvimInvalidParenthesis", + "default link NvimInvalidNestingParenthesis NvimInvalidParenthesis", + "default link NvimInvalidCallingParenthesis NvimInvalidParenthesis", + + "default link NvimInvalidSubscript NvimInvalidParenthesis", + "default link NvimInvalidSubscriptBracket NvimInvalidSubscript", + "default link NvimInvalidSubscriptColon NvimInvalidSubscript", + "default link NvimInvalidCurly NvimInvalidSubscript", + + "default link NvimInvalidContainer NvimInvalidParenthesis", + "default link NvimInvalidDict NvimInvalidContainer", + "default link NvimInvalidList NvimInvalidContainer", + + "default link NvimInvalidValue NvimInvalid", + + "default link NvimInvalidIdentifier NvimInvalidValue", + "default link NvimInvalidIdentifierScope NvimInvalidIdentifier", + "default link NvimInvalidIdentifierScopeDelimiter NvimInvalidIdentifier", + "default link NvimInvalidIdentifierName NvimInvalidIdentifier", + "default link NvimInvalidIdentifierKey NvimInvalidIdentifier", + + "default link NvimInvalidColon NvimInvalidDelimiter", + "default link NvimInvalidComma NvimInvalidDelimiter", + "default link NvimInvalidArrow NvimInvalidDelimiter", + + "default link NvimInvalidRegister NvimInvalidValue", + "default link NvimInvalidNumber NvimInvalidValue", + "default link NvimInvalidFloat NvimInvalidNumber", + "default link NvimInvalidNumberPrefix NvimInvalidNumber", + + "default link NvimInvalidOptionSigil NvimInvalidIdentifier", + "default link NvimInvalidOptionName NvimInvalidIdentifier", + "default link NvimInvalidOptionScope NvimInvalidIdentifierScope", + "default link NvimInvalidOptionScopeDelimiter " + "NvimInvalidIdentifierScopeDelimiter", + + "default link NvimInvalidEnvironmentSigil NvimInvalidOptionSigil", + "default link NvimInvalidEnvironmentName NvimInvalidIdentifier", + + // Invalid string bodies and specials are still highlighted as valid ones to + // minimize the red area. + "default link NvimInvalidString NvimInvalidValue", + "default link NvimInvalidStringBody NvimStringBody", + "default link NvimInvalidStringQuote NvimInvalidString", + "default link NvimInvalidStringSpecial NvimStringSpecial", + + "default link NvimInvalidSingleQuote NvimInvalidStringQuote", + "default link NvimInvalidSingleQuotedBody NvimInvalidStringBody", + "default link NvimInvalidSingleQuotedQuote NvimInvalidStringSpecial", + + "default link NvimInvalidDoubleQuote NvimInvalidStringQuote", + "default link NvimInvalidDoubleQuotedBody NvimInvalidStringBody", + "default link NvimInvalidDoubleQuotedEscape NvimInvalidStringSpecial", + "default link NvimInvalidDoubleQuotedUnknownEscape NvimInvalidValue", + + "default link NvimInvalidFigureBrace NvimInvalidDelimiter", + + "default link NvimInvalidSpacing ErrorMsg", + + // Not actually invalid, but we highlight user that he is doing something + // wrong. + "default link NvimDoubleQuotedUnknownEscape NvimInvalidValue", + NULL, +}; + +/// Create default links for Nvim* highlight groups used for cmdline coloring +void syn_init_cmdline_highlight(bool reset, bool init) +{ + for (size_t i = 0 ; highlight_init_cmdline[i] != NULL ; i++) { + do_highlight(highlight_init_cmdline[i], reset, init); + } +} /// Load colors from a file if "g:colors_name" is set, otherwise load builtin /// colors /// /// @param both include groups where 'bg' doesn't matter /// @param reset clear groups first -void -init_highlight(int both, int reset) +void init_highlight(bool both, bool reset) { - int i; - char **pp; - static int had_both = FALSE; + static int had_both = false; // Try finding the color scheme file. Used when a color file was loaded // and 'background' or 't_Co' is changed. @@ -6054,10 +6257,10 @@ init_highlight(int both, int reset) * Didn't use a color file, use the compiled-in colors. */ if (both) { - had_both = TRUE; - pp = highlight_init_both; - for (i = 0; pp[i] != NULL; i++) { - do_highlight((char_u *)pp[i], reset, true); + had_both = true; + const char *const *const pp = highlight_init_both; + for (size_t i = 0; pp[i] != NULL; i++) { + do_highlight(pp[i], reset, true); } } else if (!had_both) { // Don't do anything before the call with both == TRUE from main(). @@ -6066,10 +6269,11 @@ init_highlight(int both, int reset) return; } - pp = (*p_bg == 'l') ? highlight_init_light : highlight_init_dark; - - for (i = 0; pp[i] != NULL; i++) { - do_highlight((char_u *)pp[i], reset, true); + const char *const *const pp = ((*p_bg == 'l') + ? highlight_init_light + : highlight_init_dark); + for (size_t i = 0; pp[i] != NULL; i++) { + do_highlight(pp[i], reset, true); } /* Reverse looks ugly, but grey may not work for 8 colors. Thus let it @@ -6079,15 +6283,14 @@ init_highlight(int both, int reset) * Clear the attributes, needed when changing the t_Co value. */ if (t_colors > 8) { do_highlight( - (char_u *)(*p_bg == 'l' - ? "Visual cterm=NONE ctermbg=LightGrey" - : "Visual cterm=NONE ctermbg=DarkGrey"), false, - true); + (*p_bg == 'l' + ? "Visual cterm=NONE ctermbg=LightGrey" + : "Visual cterm=NONE ctermbg=DarkGrey"), false, true); } else { - do_highlight((char_u *)"Visual cterm=reverse ctermbg=NONE", - FALSE, TRUE); - if (*p_bg == 'l') - do_highlight((char_u *)"Search ctermfg=black", FALSE, TRUE); + do_highlight("Visual cterm=reverse ctermbg=NONE", false, true); + if (*p_bg == 'l') { + do_highlight("Search ctermfg=black", false, true); + } } /* @@ -6104,6 +6307,7 @@ init_highlight(int both, int reset) recursive--; } } + syn_init_cmdline_highlight(false, false); } /* @@ -6138,17 +6342,22 @@ int load_colors(char_u *name) } -/// Handle the ":highlight .." command. -/// When using ":hi clear" this is called recursively for each group with -/// "forceit" and "init" both TRUE. -/// @param init TRUE when called for initializing -void -do_highlight(char_u *line, int forceit, int init) { - char_u *name_end; - char_u *linep; - char_u *key_start; - char_u *arg_start; - char_u *key = NULL, *arg = NULL; +/// Handle ":highlight" command +/// +/// When using ":highlight clear" this is called recursively for each group with +/// forceit and init being both true. +/// +/// @param[in] line Command arguments. +/// @param[in] forceit True when bang is given, allows to link group even if +/// it has its own settings. +/// @param[in] init True when initializing. +void do_highlight(const char *line, const bool forceit, const bool init) + FUNC_ATTR_NONNULL_ALL +{ + const char *name_end; + const char *linep; + const char *key_start; + const char *arg_start; long i; int off; int len; @@ -6162,94 +6371,87 @@ do_highlight(char_u *line, int forceit, int init) { int color; bool is_normal_group = false; // "Normal" group - /* - * If no argument, list current highlighting. - */ - if (ends_excmd(*line)) { + // If no argument, list current highlighting. + if (ends_excmd((uint8_t)(*line))) { for (int i = 1; i <= highlight_ga.ga_len && !got_int; i++) { - // todo(vim): only call when the group has attributes set + // TODO(brammool): only call when the group has attributes set highlight_list_one(i); } return; } - /* - * Isolate the name. - */ - name_end = skiptowhite(line); - linep = skipwhite(name_end); + // Isolate the name. + name_end = (const char *)skiptowhite((const char_u *)line); + linep = (const char *)skipwhite((const char_u *)name_end); - /* - * Check for "default" argument. - */ - if (STRNCMP(line, "default", name_end - line) == 0) { - dodefault = TRUE; + // Check for "default" argument. + if (strncmp(line, "default", name_end - line) == 0) { + dodefault = true; line = linep; - name_end = skiptowhite(line); - linep = skipwhite(name_end); + name_end = (const char *)skiptowhite((const char_u *)line); + linep = (const char *)skipwhite((const char_u *)name_end); } - /* - * Check for "clear" or "link" argument. - */ - if (STRNCMP(line, "clear", name_end - line) == 0) - doclear = TRUE; - if (STRNCMP(line, "link", name_end - line) == 0) - dolink = TRUE; + // Check for "clear" or "link" argument. + if (strncmp(line, "clear", name_end - line) == 0) { + doclear = true; + } else if (strncmp(line, "link", name_end - line) == 0) { + dolink = true; + } - /* - * ":highlight {group-name}": list highlighting for one group. - */ - if (!doclear && !dolink && ends_excmd(*linep)) { - id = syn_namen2id(line, (int)(name_end - line)); - if (id == 0) - EMSG2(_("E411: highlight group not found: %s"), line); - else + // ":highlight {group-name}": list highlighting for one group. + if (!doclear && !dolink && ends_excmd((uint8_t)(*linep))) { + id = syn_namen2id((const char_u *)line, (int)(name_end - line)); + if (id == 0) { + emsgf(_("E411: highlight group not found: %s"), line); + } else { highlight_list_one(id); + } return; } - /* - * Handle ":highlight link {from} {to}" command. - */ + // Handle ":highlight link {from} {to}" command. if (dolink) { - char_u *from_start = linep; - char_u *from_end; - char_u *to_start; - char_u *to_end; + const char *from_start = linep; + const char *from_end; + const char *to_start; + const char *to_end; int from_id; int to_id; - from_end = skiptowhite(from_start); - to_start = skipwhite(from_end); - to_end = skiptowhite(to_start); + from_end = (const char *)skiptowhite((const char_u *)from_start); + to_start = (const char *)skipwhite((const char_u *)from_end); + to_end = (const char *)skiptowhite((const char_u *)to_start); - if (ends_excmd(*from_start) || ends_excmd(*to_start)) { - EMSG2(_("E412: Not enough arguments: \":highlight link %s\""), - from_start); + if (ends_excmd((uint8_t)(*from_start)) + || ends_excmd((uint8_t)(*to_start))) { + emsgf(_("E412: Not enough arguments: \":highlight link %s\""), + from_start); return; } - if (!ends_excmd(*skipwhite(to_end))) { - EMSG2(_("E413: Too many arguments: \":highlight link %s\""), from_start); + if (!ends_excmd(*skipwhite((const char_u *)to_end))) { + emsgf(_("E413: Too many arguments: \":highlight link %s\""), from_start); return; } - from_id = syn_check_group(from_start, (int)(from_end - from_start)); - if (STRNCMP(to_start, "NONE", 4) == 0) + from_id = syn_check_group((const char_u *)from_start, + (int)(from_end - from_start)); + if (strncmp(to_start, "NONE", 4) == 0) { to_id = 0; - else - to_id = syn_check_group(to_start, (int)(to_end - to_start)); + } else { + to_id = syn_check_group((const char_u *)to_start, + (int)(to_end - to_start)); + } if (from_id > 0 && (!init || HL_TABLE()[from_id - 1].sg_set == 0)) { - /* - * Don't allow a link when there already is some highlighting - * for the group, unless '!' is used - */ + // Don't allow a link when there already is some highlighting + // for the group, unless '!' is used if (to_id > 0 && !forceit && !init && hl_has_settings(from_id - 1, dodefault)) { - if (sourcing_name == NULL && !dodefault) + if (sourcing_name == NULL && !dodefault) { EMSG(_("E414: group has settings, highlight link ignored")); + } } else { if (!init) HL_TABLE()[from_id - 1].sg_set |= SG_LINK; @@ -6259,43 +6461,38 @@ do_highlight(char_u *line, int forceit, int init) { } } - /* Only call highlight_changed() once, after sourcing a syntax file */ - need_highlight_changed = TRUE; + // Only call highlight_changed() once, after sourcing a syntax file. + need_highlight_changed = true; return; } if (doclear) { - /* - * ":highlight clear [group]" command. - */ + // ":highlight clear [group]" command. line = linep; - if (ends_excmd(*line)) { + if (ends_excmd((uint8_t)(*line))) { do_unlet(S_LEN("colors_name"), true); restore_cterm_colors(); - /* - * Clear all default highlight groups and load the defaults. - */ - for (int idx = 0; idx < highlight_ga.ga_len; ++idx) { + // Clear all default highlight groups and load the defaults. + for (int idx = 0; idx < highlight_ga.ga_len; idx++) { highlight_clear(idx); } - init_highlight(TRUE, TRUE); + init_highlight(true, true); highlight_changed(); redraw_later_clear(); return; } - name_end = skiptowhite(line); - linep = skipwhite(name_end); + name_end = (const char *)skiptowhite((const char_u *)line); + linep = (const char *)skipwhite((const char_u *)name_end); } - /* - * Find the group name in the table. If it does not exist yet, add it. - */ - id = syn_check_group(line, (int)(name_end - line)); - if (id == 0) /* failed (out of memory) */ + // Find the group name in the table. If it does not exist yet, add it. + id = syn_check_group((const char_u *)line, (int)(name_end - line)); + if (id == 0) { // Failed (out of memory). return; - idx = id - 1; /* index is ID minus one */ + } + idx = id - 1; // Index is ID minus one. // Return if "default" was used and the group already has settings if (dodefault && hl_has_settings(idx, true)) { @@ -6304,19 +6501,21 @@ do_highlight(char_u *line, int forceit, int init) { is_normal_group = (STRCMP(HL_TABLE()[idx].sg_name_u, "NORMAL") == 0); - /* Clear the highlighting for ":hi clear {group}" and ":hi clear". */ + // Clear the highlighting for ":hi clear {group}" and ":hi clear". if (doclear || (forceit && init)) { highlight_clear(idx); if (!doclear) HL_TABLE()[idx].sg_set = 0; } + char *key = NULL; + char *arg = NULL; if (!doclear) { - while (!ends_excmd(*linep)) { + while (!ends_excmd((uint8_t)(*linep))) { key_start = linep; if (*linep == '=') { - EMSG2(_("E415: unexpected equal sign: %s"), key_start); - error = TRUE; + emsgf(_("E415: unexpected equal sign: %s"), key_start); + error = true; break; } @@ -6326,61 +6525,58 @@ do_highlight(char_u *line, int forceit, int init) { linep++; } xfree(key); - key = vim_strnsave_up(key_start, (int)(linep - key_start)); - linep = skipwhite(linep); + key = (char *)vim_strnsave_up((const char_u *)key_start, + (int)(linep - key_start)); + linep = (const char *)skipwhite((const char_u *)linep); - if (STRCMP(key, "NONE") == 0) { + if (strcmp(key, "NONE") == 0) { if (!init || HL_TABLE()[idx].sg_set == 0) { - if (!init) + if (!init) { HL_TABLE()[idx].sg_set |= SG_CTERM+SG_GUI; + } highlight_clear(idx); } continue; } - /* - * Check for the equal sign. - */ + // Check for the equal sign. if (*linep != '=') { - EMSG2(_("E416: missing equal sign: %s"), key_start); - error = TRUE; + emsgf(_("E416: missing equal sign: %s"), key_start); + error = true; break; } - ++linep; + linep++; - /* - * Isolate the argument. - */ - linep = skipwhite(linep); - if (*linep == '\'') { /* guifg='color name' */ + // Isolate the argument. + linep = (const char *)skipwhite((const char_u *)linep); + if (*linep == '\'') { // guifg='color name' arg_start = ++linep; - linep = vim_strchr(linep, '\''); + linep = strchr(linep, '\''); if (linep == NULL) { - EMSG2(_(e_invarg2), key_start); - error = TRUE; + emsgf(_(e_invarg2), key_start); + error = true; break; } } else { arg_start = linep; - linep = skiptowhite(linep); + linep = (const char *)skiptowhite((const char_u *)linep); } if (linep == arg_start) { - EMSG2(_("E417: missing argument: %s"), key_start); - error = TRUE; + emsgf(_("E417: missing argument: %s"), key_start); + error = true; break; } xfree(arg); - arg = vim_strnsave(arg_start, (int)(linep - arg_start)); + arg = xstrndup(arg_start, (size_t)(linep - arg_start)); - if (*linep == '\'') - ++linep; + if (*linep == '\'') { + linep++; + } - /* - * Store the argument. - */ - if ( STRCMP(key, "TERM") == 0 - || STRCMP(key, "CTERM") == 0 - || STRCMP(key, "GUI") == 0) { + // Store the argument. + if (strcmp(key, "TERM") == 0 + || strcmp(key, "CTERM") == 0 + || strcmp(key, "GUI") == 0) { attr = 0; off = 0; while (arg[off] != NUL) { @@ -6393,26 +6589,30 @@ do_highlight(char_u *line, int forceit, int init) { } } if (i < 0) { - EMSG2(_("E418: Illegal value: %s"), arg); - error = TRUE; + emsgf(_("E418: Illegal value: %s"), arg); + error = true; break; } - if (arg[off] == ',') /* another one follows */ - ++off; + if (arg[off] == ',') { // Another one follows. + off++; + } } - if (error) + if (error) { break; + } if (*key == 'C') { if (!init || !(HL_TABLE()[idx].sg_set & SG_CTERM)) { - if (!init) + if (!init) { HL_TABLE()[idx].sg_set |= SG_CTERM; + } HL_TABLE()[idx].sg_cterm = attr; HL_TABLE()[idx].sg_cterm_bold = FALSE; } } else if (*key == 'G') { if (!init || !(HL_TABLE()[idx].sg_set & SG_GUI)) { - if (!init) + if (!init) { HL_TABLE()[idx].sg_set |= SG_GUI; + } HL_TABLE()[idx].sg_gui = attr; } } @@ -6431,14 +6631,14 @@ do_highlight(char_u *line, int forceit, int init) { HL_TABLE()[idx].sg_cterm_bold = FALSE; } - if (ascii_isdigit(*arg)) + if (ascii_isdigit(*arg)) { color = atoi((char *)arg); - else if (STRICMP(arg, "fg") == 0) { - if (cterm_normal_fg_color) + } else if (STRICMP(arg, "fg") == 0) { + if (cterm_normal_fg_color) { color = cterm_normal_fg_color - 1; - else { + } else { EMSG(_("E419: FG color unknown")); - error = TRUE; + error = true; break; } } else if (STRICMP(arg, "bg") == 0) { @@ -6446,70 +6646,84 @@ do_highlight(char_u *line, int forceit, int init) { color = cterm_normal_bg_color - 1; else { EMSG(_("E420: BG color unknown")); - error = TRUE; + error = true; break; } } else { - static char *(color_names[28]) = { + static const char *color_names[] = { "Black", "DarkBlue", "DarkGreen", "DarkCyan", "DarkRed", "DarkMagenta", "Brown", "DarkYellow", "Gray", "Grey", "LightGray", "LightGrey", "DarkGray", "DarkGrey", "Blue", "LightBlue", "Green", "LightGreen", "Cyan", "LightCyan", "Red", "LightRed", "Magenta", - "LightMagenta", "Yellow", "LightYellow", "White", "NONE" + "LightMagenta", "Yellow", "LightYellow", "White", + "NONE" + }; + static const int color_numbers_16[] = { + 0, 1, 2, 3, + 4, 5, 6, 6, + 7, 7, + 7, 7, 8, 8, + 9, 9, 10, 10, + 11, 11, 12, 12, 13, + 13, 14, 14, 15, + -1 + }; + // For xterm with 88 colors: + static int color_numbers_88[] = { + 0, 4, 2, 6, + 1, 5, 32, 72, + 84, 84, + 7, 7, 82, 82, + 12, 43, 10, 61, + 14, 63, 9, 74, 13, + 75, 11, 78, 15, + -1 }; - static int color_numbers_16[28] = {0, 1, 2, 3, - 4, 5, 6, 6, - 7, 7, - 7, 7, 8, 8, - 9, 9, 10, 10, - 11, 11, 12, 12, 13, - 13, 14, 14, 15, -1}; - /* for xterm with 88 colors... */ - static int color_numbers_88[28] = {0, 4, 2, 6, - 1, 5, 32, 72, - 84, 84, - 7, 7, 82, 82, - 12, 43, 10, 61, - 14, 63, 9, 74, 13, - 75, 11, 78, 15, -1}; - /* for xterm with 256 colors... */ - static int color_numbers_256[28] = {0, 4, 2, 6, - 1, 5, 130, 130, - 248, 248, - 7, 7, 242, 242, - 12, 81, 10, 121, - 14, 159, 9, 224, 13, - 225, 11, 229, 15, -1}; - /* for terminals with less than 16 colors... */ - static int color_numbers_8[28] = {0, 4, 2, 6, - 1, 5, 3, 3, - 7, 7, - 7, 7, 0+8, 0+8, - 4+8, 4+8, 2+8, 2+8, - 6+8, 6+8, 1+8, 1+8, 5+8, - 5+8, 3+8, 3+8, 7+8, -1}; - - /* reduce calls to STRICMP a bit, it can be slow */ + // For xterm with 256 colors: + static int color_numbers_256[] = { + 0, 4, 2, 6, + 1, 5, 130, 130, + 248, 248, + 7, 7, 242, 242, + 12, 81, 10, 121, + 14, 159, 9, 224, 13, + 225, 11, 229, 15, + -1 + }; + // For terminals with less than 16 colors: + static int color_numbers_8[28] = { + 0, 4, 2, 6, + 1, 5, 3, 3, + 7, 7, + 7, 7, 0+8, 0+8, + 4+8, 4+8, 2+8, 2+8, + 6+8, 6+8, 1+8, 1+8, 5+8, + 5+8, 3+8, 3+8, 7+8, + -1 + }; + + // Reduce calls to STRICMP a bit, it can be slow. off = TOUPPER_ASC(*arg); - for (i = ARRAY_SIZE(color_names); --i >= 0; ) + for (i = ARRAY_SIZE(color_names); --i >= 0; ) { if (off == color_names[i][0] - && STRICMP(arg + 1, color_names[i] + 1) == 0) + && STRICMP(arg + 1, color_names[i] + 1) == 0) { break; + } + } if (i < 0) { - EMSG2(_( - "E421: Color name or number not recognized: %s"), - key_start); - error = TRUE; + emsgf(_("E421: Color name or number not recognized: %s"), + key_start); + error = true; break; } - /* Use the _16 table to check if its a valid color name. */ + // Use the _16 table to check if its a valid color name. color = color_numbers_16[i]; if (color >= 0) { if (t_colors == 8) { - /* t_Co is 8: use the 8 colors table */ + // t_Co is 8: use the 8 colors table. color = color_numbers_8[i]; if (key[5] == 'F') { /* set/reset bold attribute to get light foreground @@ -6533,16 +6747,14 @@ do_highlight(char_u *line, int forceit, int init) { } } } - /* Add one to the argument, to avoid zero. Zero is used for - * "NONE", then "color" is -1. */ + // Add one to the argument, to avoid zero. Zero is used for + // "NONE", then "color" is -1. if (key[5] == 'F') { HL_TABLE()[idx].sg_cterm_fg = color + 1; if (is_normal_group) { cterm_normal_fg_color = color + 1; cterm_normal_fg_bold = (HL_TABLE()[idx].sg_cterm & HL_BOLD); - { - must_redraw = CLEAR; - } + must_redraw = CLEAR; } } else { HL_TABLE()[idx].sg_cterm_bg = color + 1; @@ -6566,15 +6778,15 @@ do_highlight(char_u *line, int forceit, int init) { } } } - } else if (STRCMP(key, "GUIFG") == 0) { + } else if (strcmp(key, "GUIFG") == 0) { if (!init || !(HL_TABLE()[idx].sg_set & SG_GUI)) { if (!init) HL_TABLE()[idx].sg_set |= SG_GUI; xfree(HL_TABLE()[idx].sg_rgb_fg_name); - if (STRCMP(arg, "NONE")) { - HL_TABLE()[idx].sg_rgb_fg_name = (uint8_t *)xstrdup((char *)arg); - HL_TABLE()[idx].sg_rgb_fg = name_to_color(arg); + if (strcmp(arg, "NONE")) { + HL_TABLE()[idx].sg_rgb_fg_name = (char_u *)xstrdup((char *)arg); + HL_TABLE()[idx].sg_rgb_fg = name_to_color((const char_u *)arg); } else { HL_TABLE()[idx].sg_rgb_fg_name = NULL; HL_TABLE()[idx].sg_rgb_fg = -1; @@ -6591,8 +6803,8 @@ do_highlight(char_u *line, int forceit, int init) { xfree(HL_TABLE()[idx].sg_rgb_bg_name); if (STRCMP(arg, "NONE") != 0) { - HL_TABLE()[idx].sg_rgb_bg_name = (uint8_t *)xstrdup((char *)arg); - HL_TABLE()[idx].sg_rgb_bg = name_to_color(arg); + HL_TABLE()[idx].sg_rgb_bg_name = (char_u *)xstrdup((char *)arg); + HL_TABLE()[idx].sg_rgb_bg = name_to_color((const char_u *)arg); } else { HL_TABLE()[idx].sg_rgb_bg_name = NULL; HL_TABLE()[idx].sg_rgb_bg = -1; @@ -6602,15 +6814,15 @@ do_highlight(char_u *line, int forceit, int init) { if (is_normal_group) { normal_bg = HL_TABLE()[idx].sg_rgb_bg; } - } else if (STRCMP(key, "GUISP") == 0) { + } else if (strcmp(key, "GUISP") == 0) { if (!init || !(HL_TABLE()[idx].sg_set & SG_GUI)) { if (!init) HL_TABLE()[idx].sg_set |= SG_GUI; xfree(HL_TABLE()[idx].sg_rgb_sp_name); - if (STRCMP(arg, "NONE") != 0) { - HL_TABLE()[idx].sg_rgb_sp_name = (uint8_t *)xstrdup((char *)arg); - HL_TABLE()[idx].sg_rgb_sp = name_to_color(arg); + if (strcmp(arg, "NONE") != 0) { + HL_TABLE()[idx].sg_rgb_sp_name = (char_u *)xstrdup((char *)arg); + HL_TABLE()[idx].sg_rgb_sp = name_to_color((const char_u *)arg); } else { HL_TABLE()[idx].sg_rgb_sp_name = NULL; HL_TABLE()[idx].sg_rgb_sp = -1; @@ -6620,31 +6832,25 @@ do_highlight(char_u *line, int forceit, int init) { if (is_normal_group) { normal_sp = HL_TABLE()[idx].sg_rgb_sp; } - } else if (STRCMP(key, "START") == 0 || STRCMP(key, "STOP") == 0) { + } else if (strcmp(key, "START") == 0 || strcmp(key, "STOP") == 0) { // Ignored for now } else { - EMSG2(_("E423: Illegal argument: %s"), key_start); - error = TRUE; + emsgf(_("E423: Illegal argument: %s"), key_start); + error = true; break; } - /* - * When highlighting has been given for a group, don't link it. - */ + // When highlighting has been given for a group, don't link it. if (!init || !(HL_TABLE()[idx].sg_set & SG_LINK)) { HL_TABLE()[idx].sg_link = 0; } - /* - * Continue with next argument. - */ - linep = skipwhite(linep); + // Continue with next argument. + linep = (const char *)skipwhite((const char_u *)linep); } } - /* - * If there is an error, and it's a new entry, remove it from the table. - */ + // If there is an error, and it's a new entry, remove it from the table. if (error && idx == highlight_ga.ga_len) { syn_unadd_group(); } else { @@ -7202,7 +7408,7 @@ char_u *syn_id2name(int id) /* * Like syn_name2id(), but take a pointer + length argument. */ -int syn_namen2id(char_u *linep, int len) +int syn_namen2id(const char_u *linep, int len) { char_u *name = vim_strnsave(linep, len); int id = syn_name2id(name); @@ -7218,7 +7424,7 @@ int syn_namen2id(char_u *linep, int len) /// @param len length of \p pp /// /// @return 0 for failure else the id of the group -int syn_check_group(char_u *pp, int len) +int syn_check_group(const char_u *pp, int len) { char_u *name = vim_strnsave(pp, len); int id = syn_name2id(name); @@ -8221,7 +8427,7 @@ color_name_table_T color_name_table[] = { /// /// @param[in] name string value to convert to RGB /// return the hex value or -1 if could not find a correct value -RgbValue name_to_color(const uint8_t *name) +RgbValue name_to_color(const char_u *name) { if (name[0] == '#' && isxdigit(name[1]) && isxdigit(name[2]) diff --git a/src/nvim/syntax.h b/src/nvim/syntax.h index bb733ead30..c9b0665ec8 100644 --- a/src/nvim/syntax.h +++ b/src/nvim/syntax.h @@ -45,6 +45,9 @@ typedef struct { } color_name_table_T; extern color_name_table_T color_name_table[]; +/// Array of highlight definitions, used for unit testing +extern const char *const highlight_init_cmdline[]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "syntax.h.generated.h" #endif diff --git a/src/nvim/version.c b/src/nvim/version.c index 8d85855c6e..1a9c9d73e5 100644 --- a/src/nvim/version.c +++ b/src/nvim/version.c @@ -30,8 +30,8 @@ // for ":version", ":intro", and "nvim --version" #ifndef NVIM_VERSION_MEDIUM #define NVIM_VERSION_MEDIUM "v" STR(NVIM_VERSION_MAJOR)\ - "." STR(NVIM_VERSION_MINOR) "." STR(NVIM_VERSION_PATCH)\ - NVIM_VERSION_PRERELEASE +"." STR(NVIM_VERSION_MINOR) "." STR(NVIM_VERSION_PATCH)\ +NVIM_VERSION_PRERELEASE #endif #define NVIM_VERSION_LONG "NVIM " NVIM_VERSION_MEDIUM @@ -47,33 +47,33 @@ char *version_cflags = "Compilation: " NVIM_VERSION_CFLAGS; static char *features[] = { #ifdef HAVE_ACL - "+acl", +"+acl", #else - "-acl", +"-acl", #endif #if (defined(HAVE_ICONV_H) && defined(USE_ICONV)) || defined(DYNAMIC_ICONV) # ifdef DYNAMIC_ICONV - "+iconv/dyn", +"+iconv/dyn", # else - "+iconv", +"+iconv", # endif #else - "-iconv", +"-iconv", #endif #ifdef HAVE_JEMALLOC - "+jemalloc", +"+jemalloc", #else - "-jemalloc", +"-jemalloc", #endif #ifdef FEAT_TUI - "+tui", +"+tui", #else - "-tui", +"-tui", #endif - NULL +NULL }; // clang-format off @@ -205,8 +205,8 @@ static const int included_patches[] = { // 1233, // 1232, // 1231, - // 1230, - // 1229, + 1230, + 1229, // 1228, // 1227, // 1226, diff --git a/src/nvim/vim.h b/src/nvim/vim.h index 7da3c8246f..b535e380d1 100644 --- a/src/nvim/vim.h +++ b/src/nvim/vim.h @@ -26,13 +26,6 @@ /// length of a buffer to store a number in ASCII (64 bits binary + NUL) enum { NUMBUFLEN = 65 }; -// flags for vim_str2nr() -#define STR2NR_BIN 1 -#define STR2NR_OCT 2 -#define STR2NR_HEX 4 -#define STR2NR_ALL (STR2NR_BIN + STR2NR_OCT + STR2NR_HEX) -#define STR2NR_FORCE 8 // only when ONE of the above is used - #define MAX_TYPENR 65535 #define ROOT_UID 0 diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c new file mode 100644 index 0000000000..4196ecb9d2 --- /dev/null +++ b/src/nvim/viml/parser/expressions.c @@ -0,0 +1,3102 @@ +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +/// VimL expression parser + +// Planned incompatibilities (to be included into vim_diff.txt when this parser +// will be an actual part of VimL evaluation process): +// +// 1. Expressions are first fully parsed and only then executed. This means +// that while ":echo [system('touch abc')" will create file "abc" in Vim and +// only then raise syntax error regarding missing comma in list in Neovim +// trying to execute that will immediately raise syntax error regarding +// missing list end without actually executing anything. +// 2. Expressions are first fully parsed, without considering any runtime +// information. This means things like that "d.a" does not change its +// meaning depending on type of "d" (or whether Vim is currently executing or +// skipping). For compatibility reasons the dot thus may either be “concat +// or subscript” operator or just “concat” operator. +// 3. Expressions parser is aware whether it is called for :echo or <C-r>=. +// This means that while "<C-r>=1 | 2<CR>" is equivalent to "<C-r>=1<CR>" +// because "| 2" part is left to be treated as a command separator and then +// ignored in Neovim it is an error. +// 4. Expressions parser has generally better error reporting. But for +// compatibility reasons most errors have error code E15 while error messages +// are significantly different from Vim’s E15. Also some error codes were +// retired because of being harder to emulate or because of them being +// a result of differences in parsing process: e.g. with ":echo {a, b}" Vim +// will attempt to parse expression as lambda, fail, check whether it is +// a curly-braces-name, fail again, and evaluate that as a dictionary, giving +// error regarding undefined variable "a" (or about missing colon). Neovim +// will not try to evaluate anything here: comma right after an argument name +// means that expression may not be anything, but lambda, so the resulting +// error message will never be about missing variable or colon: it will be +// about missing arrow (or a continuation of argument list). +// 5. Failing to parse expression always gives exactly one error message: no +// more stack of error messages like > +// +// :echo [1, +// E697: Missing end of List ']': +// E15: Invalid expression: [1, +// +// < , just exactly one E697 message. +// 6. Some expressions involving calling parenthesis which are treated +// separately by Vim even when not separated by spaces are treated as one +// expression by Neovim: e.g. ":echo (1)(1)" will yield runtime error after +// failing to call "1", while Vim will echo "1 1". Reasoning is the same: +// type of what is in the first expression is generally not known when +// parsing, so to have separate expressions like this separate them with +// spaces. +// 7. 'isident' no longer applies to environment variables, they always include +// ASCII alphanumeric characters and underscore and nothing except this. + +#include <stdbool.h> +#include <stddef.h> +#include <assert.h> +#include <string.h> + +#include "nvim/vim.h" +#include "nvim/memory.h" +#include "nvim/types.h" +#include "nvim/charset.h" +#include "nvim/ascii.h" +#include "nvim/assert.h" +#include "nvim/lib/kvec.h" +#include "nvim/eval/typval.h" + +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" + +#define vim_str2nr(s, ...) vim_str2nr((const char_u *)(s), __VA_ARGS__) + +typedef kvec_withinit_t(ExprASTNode **, 16) ExprASTStack; + +/// Which nodes may be wanted +typedef enum { + /// Operators: function call, subscripts, binary operators, … + /// + /// For unrestricted expressions. + kENodeOperator, + /// Values: literals, variables, nested expressions, unary operators. + /// + /// For unrestricted expressions as well, implies that top item in AST stack + /// points to NULL. + kENodeValue, +} ExprASTWantedNode; + +/// Parse type: what is being parsed currently +typedef enum { + /// Parsing regular VimL expression + kEPTExpr = 0, + /// Parsing lambda arguments + /// + /// Just like parsing function arguments, but it is valid to be ended with an + /// arrow only. + kEPTLambdaArguments, + /// Assignment: parsing for :let + kEPTAssignment, + /// Single assignment: used when lists are not allowed (i.e. when nesting) + kEPTSingleAssignment, +} ExprASTParseType; + +typedef kvec_withinit_t(ExprASTParseType, 4) ExprASTParseTypeStack; + +/// Operator priority level +typedef enum { + kEOpLvlInvalid = 0, + kEOpLvlComplexIdentifier, + kEOpLvlParens, + kEOpLvlAssignment, + kEOpLvlArrow, + kEOpLvlComma, + kEOpLvlColon, + kEOpLvlTernaryValue, + kEOpLvlTernary, + kEOpLvlOr, + kEOpLvlAnd, + kEOpLvlComparison, + kEOpLvlAddition, ///< Addition, subtraction and concatenation. + kEOpLvlMultiplication, ///< Multiplication, division and modulo. + kEOpLvlUnary, ///< Unary operations: not, minus, plus. + kEOpLvlSubscript, ///< Subscripts. + kEOpLvlValue, ///< Values: literals, variables, nested expressions, … +} ExprOpLvl; + +/// Operator associativity +typedef enum { + kEOpAssNo= 'n', ///< Not associative / not applicable. + kEOpAssLeft = 'l', ///< Left associativity. + kEOpAssRight = 'r', ///< Right associativity. +} ExprOpAssociativity; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/expressions.c.generated.h" +#endif + +/// Character used as a separator in autoload function/variable names. +#define AUTOLOAD_CHAR '#' + +/// Scale number by a given factor +/// +/// Used to apply exponent to a number. Idea taken from uClibc. +/// +/// @param[in] num Number to scale. Does not bother doing anything if it is +/// zero. +/// @param[in] base Base, should be 10 since non-decimal floating-point +/// numbers are not supported. +/// @param[in] exponent Exponent to scale by. +/// @param[in] exponent_negative True if exponent is negative. +static inline float_T scale_number(const float_T num, + const uint8_t base, + const uvarnumber_T exponent, + const bool exponent_negative) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_CONST +{ + if (num == 0 || exponent == 0) { + return num; + } + assert(base); + uvarnumber_T exp = exponent; + float_T p_base = (float_T)base; + float_T ret = num; + while (exp) { + if (exp & 1) { + if (exponent_negative) { + ret /= p_base; + } else { + ret *= p_base; + } + } + exp >>= 1; + p_base *= p_base; + } + return ret; +} + +/// Get next token for the VimL expression input +/// +/// @param pstate Parser state. +/// @param[in] flags Flags, @see LexExprFlags. +/// +/// @return Next token. +LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL +{ + LexExprToken ret = { + .type = kExprLexInvalid, + .start = pstate->pos, + }; + ParserLine pline; + if (!viml_parser_get_remaining_line(pstate, &pline)) { + ret.type = kExprLexEOC; + return ret; + } + if (pline.size <= 0) { + ret.len = 0; + ret.type = kExprLexEOC; + goto viml_pexpr_next_token_adv_return; + } + ret.len = 1; + const uint8_t schar = (uint8_t)pline.data[0]; +#define GET_CCS(ret, pline) \ + do { \ + if (ret.len < pline.size \ + && strchr("?#", pline.data[ret.len]) != NULL) { \ + ret.data.cmp.ccs = \ + (ExprCaseCompareStrategy)pline.data[ret.len]; \ + ret.len++; \ + } else { \ + ret.data.cmp.ccs = kCCStrategyUseOption; \ + } \ + } while (0) + switch (schar) { + // Paired brackets. +#define BRACKET(typ, opning, clsing) \ + case opning: \ + case clsing: { \ + ret.type = typ; \ + ret.data.brc.closing = (schar == clsing); \ + break; \ + } + BRACKET(kExprLexParenthesis, '(', ')') + BRACKET(kExprLexBracket, '[', ']') + BRACKET(kExprLexFigureBrace, '{', '}') +#undef BRACKET + + // Single character tokens without data. +#define CHAR(typ, ch) \ + case ch: { \ + ret.type = typ; \ + break; \ + } + CHAR(kExprLexQuestion, '?') + CHAR(kExprLexColon, ':') + CHAR(kExprLexComma, ',') +#undef CHAR + + // Multiplication/division/modulo. +#define MUL(mul_type, ch) \ + case ch: { \ + ret.type = kExprLexMultiplication; \ + ret.data.mul.type = mul_type; \ + break; \ + } + MUL(kExprLexMulMul, '*') + MUL(kExprLexMulDiv, '/') + MUL(kExprLexMulMod, '%') +#undef MUL + +#define CHARREG(typ, cond) \ + do { \ + ret.type = typ; \ + for (; (ret.len < pline.size \ + && cond(pline.data[ret.len])) \ + ; ret.len++) { \ + } \ + } while (0) + + // Whitespace. + case ' ': + case TAB: { + CHARREG(kExprLexSpacing, ascii_iswhite); + break; + } + + // Control character, except for NUL, NL and TAB. + case Ctrl_A: case Ctrl_B: case Ctrl_C: case Ctrl_D: case Ctrl_E: + case Ctrl_F: case Ctrl_G: case Ctrl_H: + + case Ctrl_K: case Ctrl_L: case Ctrl_M: case Ctrl_N: case Ctrl_O: + case Ctrl_P: case Ctrl_Q: case Ctrl_R: case Ctrl_S: case Ctrl_T: + case Ctrl_U: case Ctrl_V: case Ctrl_W: case Ctrl_X: case Ctrl_Y: + case Ctrl_Z: { +#define ISCTRL(schar) (schar < ' ') + CHARREG(kExprLexInvalid, ISCTRL); + ret.data.err.type = kExprLexSpacing; + ret.data.err.msg = + _("E15: Invalid control character present in input: %.*s"); + break; +#undef ISCTRL + } + + // Number. + case '0': case '1': case '2': case '3': case '4': case '5': case '6': + case '7': case '8': case '9': { + ret.data.num.is_float = false; + ret.data.num.base = 10; + size_t frac_start = 0; + size_t exp_start = 0; + size_t frac_end = 0; + bool exp_negative = false; + CHARREG(kExprLexNumber, ascii_isdigit); + if (flags & kELFlagAllowFloat) { + const LexExprToken non_float_ret = ret; + if (pline.size > ret.len + 1 + && pline.data[ret.len] == '.' + && ascii_isdigit(pline.data[ret.len + 1])) { + ret.len++; + frac_start = ret.len; + frac_end = ret.len; + ret.data.num.is_float = true; + for (; ret.len < pline.size && ascii_isdigit(pline.data[ret.len]) + ; ret.len++) { + // A small optimization: trailing zeroes in fractional part do not + // add anything to significand, so it is useless to include them in + // frac_end. + if (pline.data[ret.len] != '0') { + frac_end = ret.len + 1; + } + } + if (pline.size > ret.len + 1 + && (pline.data[ret.len] == 'e' + || pline.data[ret.len] == 'E') + && ((pline.size > ret.len + 2 + && (pline.data[ret.len + 1] == '+' + || pline.data[ret.len + 1] == '-') + && ascii_isdigit(pline.data[ret.len + 2])) + || ascii_isdigit(pline.data[ret.len + 1]))) { + ret.len++; + if (pline.data[ret.len] == '+' + || (exp_negative = (pline.data[ret.len] == '-'))) { + ret.len++; + } + exp_start = ret.len; + CHARREG(kExprLexNumber, ascii_isdigit); + } + } + if (pline.size > ret.len + && (pline.data[ret.len] == '.' + || ASCII_ISALPHA(pline.data[ret.len]))) { + ret = non_float_ret; + } + } + // TODO(ZyX-I): detect overflows + if (ret.data.num.is_float) { + // Vim used to use string2float here which in turn uses strtod(). There + // are two problems with this approach: + // 1. strtod() is locale-dependent. Not sure how it is worked around so + // that I do not see relevant bugs, but it still does not look like + // a good idea. + // 2. strtod() does not accept length argument. + // + // The below variant of parsing floats was recognized as acceptable + // because it is basically how uClibc does the thing: it generates + // a number ignoring decimal point (but recording its position), then + // uses recorded position to scale number down when processing exponent. + float_T significand_part = 0; + uvarnumber_T exp_part = 0; + const size_t frac_size = (size_t)(frac_end - frac_start); + for (size_t i = 0; i < frac_end; i++) { + if (i == frac_start - 1) { + continue; + } + significand_part = significand_part * 10 + (pline.data[i] - '0'); + } + if (exp_start) { + vim_str2nr(pline.data + exp_start, NULL, NULL, 0, NULL, &exp_part, + (int)(ret.len - exp_start)); + } + if (exp_negative) { + exp_part += frac_size; + } else { + if (exp_part < frac_size) { + exp_negative = true; + exp_part = frac_size - exp_part; + } else { + exp_part -= frac_size; + } + } + ret.data.num.val.floating = scale_number(significand_part, 10, exp_part, + exp_negative); + } else { + int len; + int prep; + vim_str2nr(pline.data, &prep, &len, STR2NR_ALL, NULL, + &ret.data.num.val.integer, (int)pline.size); + ret.len = (size_t)len; + const uint8_t bases[] = { + [0] = 10, + ['0'] = 8, + ['x'] = 16, ['X'] = 16, + ['b'] = 2, ['B'] = 2, + }; + ret.data.num.base = bases[prep]; + } + break; + } + +#define ISWORD_OR_AUTOLOAD(x) \ + (ascii_isident(x) || (x) == AUTOLOAD_CHAR) + + // Environment variable. + case '$': { + CHARREG(kExprLexEnv, ascii_isident); + break; + } + + // Normal variable/function name. + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': + case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': + case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': + case 'v': case 'w': case 'x': case 'y': case 'z': + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': + case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': + case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': + case 'V': case 'W': case 'X': case 'Y': case 'Z': + case '_': { + ret.data.var.scope = 0; + ret.data.var.autoload = false; + CHARREG(kExprLexPlainIdentifier, ascii_isident); + // "is" and "isnot" operators. + if (!(flags & kELFlagIsNotCmp) + && ((ret.len == 2 && memcmp(pline.data, "is", 2) == 0) + || (ret.len == 5 && memcmp(pline.data, "isnot", 5) == 0))) { + ret.type = kExprLexComparison; + ret.data.cmp.type = kExprCmpIdentical; + ret.data.cmp.inv = (ret.len == 5); + GET_CCS(ret, pline); + // Scope: `s:`, etc. + } else if (ret.len == 1 + && pline.size > 1 + && memchr(EXPR_VAR_SCOPE_LIST, schar, + sizeof(EXPR_VAR_SCOPE_LIST)) != NULL + && pline.data[ret.len] == ':' + && !(flags & kELFlagForbidScope)) { + ret.len++; + ret.data.var.scope = (ExprVarScope)schar; + CHARREG(kExprLexPlainIdentifier, ISWORD_OR_AUTOLOAD); + ret.data.var.autoload = ( + memchr(pline.data + 2, AUTOLOAD_CHAR, ret.len - 2) + != NULL); + // Previous CHARREG stopped at autoload character in order to make it + // possible to detect `is#`. Continue now with autoload characters + // included. + // + // Warning: there is ambiguity for the lexer: `is#Foo(1)` is a call of + // function `is#Foo()`, `1is#Foo(1)` is a comparison `1 is# Foo(1)`. This + // needs to be resolved on the higher level where context is available. + } else if (pline.size > ret.len + && pline.data[ret.len] == AUTOLOAD_CHAR) { + ret.data.var.autoload = true; + CHARREG(kExprLexPlainIdentifier, ISWORD_OR_AUTOLOAD); + } + break; + } + +#undef ISWORD_OR_AUTOLOAD +#undef CHARREG + + // Option. + case '&': { +#define OPTNAMEMISS(ret) \ + do { \ + ret.type = kExprLexInvalid; \ + ret.data.err.type = kExprLexOption; \ + ret.data.err.msg = _("E112: Option name missing: %.*s"); \ + } while (0) + if (pline.size > 1 && pline.data[1] == '&') { + ret.type = kExprLexAnd; + ret.len++; + break; + } + if (pline.size == 1 || !ASCII_ISALPHA(pline.data[1])) { + OPTNAMEMISS(ret); + break; + } + ret.type = kExprLexOption; + if (pline.size > 2 + && pline.data[2] == ':' + && memchr(EXPR_OPT_SCOPE_LIST, pline.data[1], + sizeof(EXPR_OPT_SCOPE_LIST)) != NULL) { + ret.len += 2; + ret.data.opt.scope = (ExprOptScope)pline.data[1]; + ret.data.opt.name = pline.data + 3; + } else { + ret.data.opt.scope = kExprOptScopeUnspecified; + ret.data.opt.name = pline.data + 1; + } + const char *p = ret.data.opt.name; + const char *const e = pline.data + pline.size; + if (e - p >= 4 && p[0] == 't' && p[1] == '_') { + ret.data.opt.len = 4; + ret.len += 4; + } else { + for (; p < e && ASCII_ISALPHA(*p); p++) { + } + ret.data.opt.len = (size_t)(p - ret.data.opt.name); + if (ret.data.opt.len == 0) { + OPTNAMEMISS(ret); + } else { + ret.len += ret.data.opt.len; + } + } + break; +#undef OPTNAMEMISS + } + + // Register. + case '@': { + ret.type = kExprLexRegister; + if (pline.size > 1) { + ret.len++; + ret.data.reg.name = (uint8_t)pline.data[1]; + } else { + ret.data.reg.name = -1; + } + break; + } + + // Single quoted string. + case '\'': { + ret.type = kExprLexSingleQuotedString; + ret.data.str.closed = false; + for (; ret.len < pline.size && !ret.data.str.closed; ret.len++) { + if (pline.data[ret.len] == '\'') { + if (ret.len + 1 < pline.size && pline.data[ret.len + 1] == '\'') { + ret.len++; + } else { + ret.data.str.closed = true; + } + } + } + break; + } + + // Double quoted string. + case '"': { + ret.type = kExprLexDoubleQuotedString; + ret.data.str.closed = false; + for (; ret.len < pline.size && !ret.data.str.closed; ret.len++) { + if (pline.data[ret.len] == '\\') { + if (ret.len + 1 < pline.size) { + ret.len++; + } + } else if (pline.data[ret.len] == '"') { + ret.data.str.closed = true; + } + } + break; + } + + // Unary not, (un)equality and regex (not) match comparison operators. + case '!': + case '=': { + if (pline.size == 1) { + ret.type = (schar == '!' ? kExprLexNot : kExprLexAssignment); + ret.data.ass.type = kExprAsgnPlain; + break; + } + ret.type = kExprLexComparison; + ret.data.cmp.inv = (schar == '!'); + if (pline.data[1] == '=') { + ret.data.cmp.type = kExprCmpEqual; + ret.len++; + } else if (pline.data[1] == '~') { + ret.data.cmp.type = kExprCmpMatches; + ret.len++; + } else if (schar == '!') { + ret.type = kExprLexNot; + } else { + ret.type = kExprLexAssignment; + ret.data.ass.type = kExprAsgnPlain; + } + GET_CCS(ret, pline); + break; + } + + // Less/greater [or equal to] comparison operators. + case '>': + case '<': { + ret.type = kExprLexComparison; + const bool haseqsign = (pline.size > 1 && pline.data[1] == '='); + if (haseqsign) { + ret.len++; + } + GET_CCS(ret, pline); + ret.data.cmp.inv = (schar == '<'); + ret.data.cmp.type = ((ret.data.cmp.inv ^ haseqsign) + ? kExprCmpGreaterOrEqual + : kExprCmpGreater); + break; + } + + // Minus sign, arrow from lambdas or augmented assignment. + case '-': { + if (pline.size > 1 && pline.data[1] == '>') { + ret.len++; + ret.type = kExprLexArrow; + } else if (pline.size > 1 && pline.data[1] == '=') { + ret.len++; + ret.type = kExprLexAssignment; + ret.data.ass.type = kExprAsgnSubtract; + } else { + ret.type = kExprLexMinus; + } + break; + } + + // Sign or augmented assignment. +#define CHAR_OR_ASSIGN(ch, ch_type, ass_type) \ + case ch: { \ + if (pline.size > 1 && pline.data[1] == '=') { \ + ret.len++; \ + ret.type = kExprLexAssignment; \ + ret.data.ass.type = ass_type; \ + } else { \ + ret.type = ch_type; \ + } \ + break; \ + } + CHAR_OR_ASSIGN('+', kExprLexPlus, kExprAsgnAdd) + CHAR_OR_ASSIGN('.', kExprLexDot, kExprAsgnConcat) +#undef CHAR_OR_ASSIGN + + // Expression end because Ex command ended. + case NUL: + case NL: { + if (flags & kELFlagForbidEOC) { + ret.type = kExprLexInvalid; + ret.data.err.msg = _("E15: Unexpected EOC character: %.*s"); + ret.data.err.type = kExprLexSpacing; + } else { + ret.type = kExprLexEOC; + } + break; + } + + case '|': { + if (pline.size >= 2 && pline.data[ret.len] == '|') { + // "||" is or. + ret.len++; + ret.type = kExprLexOr; + } else if (flags & kELFlagForbidEOC) { + // Note: `<C-r>=1 | 2<CR>` actually yields 1 in Vim without any + // errors. This will be changed here. + ret.type = kExprLexInvalid; + ret.data.err.msg = _("E15: Unexpected EOC character: %.*s"); + ret.data.err.type = kExprLexOr; + } else { + ret.type = kExprLexEOC; + } + break; + } + + // Everything else is not valid. + default: { + ret.len = (size_t)utfc_ptr2len_len((const char_u *)pline.data, + (int)pline.size); + ret.type = kExprLexInvalid; + ret.data.err.type = kExprLexPlainIdentifier; + ret.data.err.msg = _("E15: Unidentified character: %.*s"); + break; + } + } +#undef GET_CCS +viml_pexpr_next_token_adv_return: + if (!(flags & kELFlagPeek)) { + viml_parser_advance(pstate, ret.len); + } + return ret; +} + +static const char *const eltkn_type_tab[] = { + [kExprLexInvalid] = "Invalid", + [kExprLexMissing] = "Missing", + [kExprLexSpacing] = "Spacing", + [kExprLexEOC] = "EOC", + + [kExprLexQuestion] = "Question", + [kExprLexColon] = "Colon", + [kExprLexOr] = "Or", + [kExprLexAnd] = "And", + [kExprLexComparison] = "Comparison", + [kExprLexPlus] = "Plus", + [kExprLexMinus] = "Minus", + [kExprLexDot] = "Dot", + [kExprLexMultiplication] = "Multiplication", + + [kExprLexNot] = "Not", + + [kExprLexNumber] = "Number", + [kExprLexSingleQuotedString] = "SingleQuotedString", + [kExprLexDoubleQuotedString] = "DoubleQuotedString", + [kExprLexOption] = "Option", + [kExprLexRegister] = "Register", + [kExprLexEnv] = "Env", + [kExprLexPlainIdentifier] = "PlainIdentifier", + + [kExprLexBracket] = "Bracket", + [kExprLexFigureBrace] = "FigureBrace", + [kExprLexParenthesis] = "Parenthesis", + [kExprLexComma] = "Comma", + [kExprLexArrow] = "Arrow", + [kExprLexAssignment] = "Assignment", +}; + +const char *const eltkn_cmp_type_tab[] = { + [kExprCmpEqual] = "Equal", + [kExprCmpMatches] = "Matches", + [kExprCmpGreater] = "Greater", + [kExprCmpGreaterOrEqual] = "GreaterOrEqual", + [kExprCmpIdentical] = "Identical", +}; + +const char *const expr_asgn_type_tab[] = { + [kExprAsgnPlain] = "Plain", + [kExprAsgnAdd] = "Add", + [kExprAsgnSubtract] = "Subtract", + [kExprAsgnConcat] = "Concat", +}; + +const char *const ccs_tab[] = { + [kCCStrategyUseOption] = "UseOption", + [kCCStrategyMatchCase] = "MatchCase", + [kCCStrategyIgnoreCase] = "IgnoreCase", +}; + +static const char *const eltkn_mul_type_tab[] = { + [kExprLexMulMul] = "Mul", + [kExprLexMulDiv] = "Div", + [kExprLexMulMod] = "Mod", +}; + +static const char *const eltkn_opt_scope_tab[] = { + [kExprOptScopeUnspecified] = "Unspecified", + [kExprOptScopeGlobal] = "Global", + [kExprOptScopeLocal] = "Local", +}; + +/// Represent token as a string +/// +/// Intended for testing and debugging purposes. +/// +/// @param[in] pstate Parser state, needed to get token string from it. May be +/// NULL, in which case in place of obtaining part of the +/// string represented by token only token length is +/// returned. +/// @param[in] token Token to represent. +/// @param[out] ret_size Return string size, for cases like NULs inside +/// a string. May be NULL. +/// +/// @return Token represented in a string form, in a static buffer (overwritten +/// on each call). +const char *viml_pexpr_repr_token(const ParserState *const pstate, + const LexExprToken token, + size_t *const ret_size) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + static char ret[1024]; + char *p = ret; + const char *const e = &ret[1024] - 1; +#define ADDSTR(...) \ + do { \ + p += snprintf(p, (size_t)(sizeof(ret) - (size_t)(p - ret)), __VA_ARGS__); \ + if (p >= e) { \ + goto viml_pexpr_repr_token_end; \ + } \ + } while (0) + ADDSTR("%zu:%zu:%s", token.start.line, token.start.col, + eltkn_type_tab[token.type]); + switch (token.type) { +#define TKNARGS(tkn_type, ...) \ + case tkn_type: { \ + ADDSTR(__VA_ARGS__); \ + break; \ + } + TKNARGS(kExprLexComparison, "(type=%s,ccs=%s,inv=%i)", + eltkn_cmp_type_tab[token.data.cmp.type], + ccs_tab[token.data.cmp.ccs], + (int)token.data.cmp.inv) + TKNARGS(kExprLexMultiplication, "(type=%s)", + eltkn_mul_type_tab[token.data.mul.type]) + TKNARGS(kExprLexAssignment, "(type=%s)", + expr_asgn_type_tab[token.data.ass.type]) + TKNARGS(kExprLexRegister, "(name=%s)", intchar2str(token.data.reg.name)) + case kExprLexDoubleQuotedString: + TKNARGS(kExprLexSingleQuotedString, "(closed=%i)", + (int)token.data.str.closed) + TKNARGS(kExprLexOption, "(scope=%s,name=%.*s)", + eltkn_opt_scope_tab[token.data.opt.scope], + (int)token.data.opt.len, token.data.opt.name) + TKNARGS(kExprLexPlainIdentifier, "(scope=%s,autoload=%i)", + intchar2str(token.data.var.scope), (int)token.data.var.autoload) + TKNARGS(kExprLexNumber, "(is_float=%i,base=%i,val=%lg)", + (int)token.data.num.is_float, + (int)token.data.num.base, + (double)(token.data.num.is_float + ? (double)token.data.num.val.floating + : (double)token.data.num.val.integer)) + TKNARGS(kExprLexInvalid, "(msg=%s)", token.data.err.msg) + default: { + // No additional arguments. + break; + } +#undef TKNARGS + } + if (pstate == NULL) { + ADDSTR("::%zu", token.len); + } else { + *p++ = ':'; + memmove( + p, &pstate->reader.lines.items[token.start.line].data[token.start.col], + token.len); + p += token.len; + *p = NUL; + } +#undef ADDSTR +viml_pexpr_repr_token_end: + if (ret_size != NULL) { + *ret_size = (size_t)(p - ret); + } + return ret; +} + +const char *const east_node_type_tab[] = { + [kExprNodeMissing] = "Missing", + [kExprNodeOpMissing] = "OpMissing", + [kExprNodeTernary] = "Ternary", + [kExprNodeTernaryValue] = "TernaryValue", + [kExprNodeRegister] = "Register", + [kExprNodeSubscript] = "Subscript", + [kExprNodeListLiteral] = "ListLiteral", + [kExprNodeUnaryPlus] = "UnaryPlus", + [kExprNodeBinaryPlus] = "BinaryPlus", + [kExprNodeNested] = "Nested", + [kExprNodeCall] = "Call", + [kExprNodePlainIdentifier] = "PlainIdentifier", + [kExprNodePlainKey] = "PlainKey", + [kExprNodeComplexIdentifier] = "ComplexIdentifier", + [kExprNodeUnknownFigure] = "UnknownFigure", + [kExprNodeLambda] = "Lambda", + [kExprNodeDictLiteral] = "DictLiteral", + [kExprNodeCurlyBracesIdentifier] = "CurlyBracesIdentifier", + [kExprNodeComma] = "Comma", + [kExprNodeColon] = "Colon", + [kExprNodeArrow] = "Arrow", + [kExprNodeComparison] = "Comparison", + [kExprNodeConcat] = "Concat", + [kExprNodeConcatOrSubscript] = "ConcatOrSubscript", + [kExprNodeInteger] = "Integer", + [kExprNodeFloat] = "Float", + [kExprNodeSingleQuotedString] = "SingleQuotedString", + [kExprNodeDoubleQuotedString] = "DoubleQuotedString", + [kExprNodeOr] = "Or", + [kExprNodeAnd] = "And", + [kExprNodeUnaryMinus] = "UnaryMinus", + [kExprNodeBinaryMinus] = "BinaryMinus", + [kExprNodeNot] = "Not", + [kExprNodeMultiplication] = "Multiplication", + [kExprNodeDivision] = "Division", + [kExprNodeMod] = "Mod", + [kExprNodeOption] = "Option", + [kExprNodeEnvironment] = "Environment", + [kExprNodeAssignment] = "Assignment", +}; + +/// Represent `int` character as a string +/// +/// Converts +/// - ASCII digits into '{digit}' +/// - ASCII printable characters into a single-character strings +/// - everything else to numbers. +/// +/// @param[in] ch Character to convert. +/// +/// @return Converted string, stored in a static buffer (overriden after each +/// call). +static const char *intchar2str(const int ch) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + static char buf[sizeof(int) * 3 + 1]; + if (' ' <= ch && ch < 0x7f) { + if (ascii_isdigit(ch)) { + buf[0] = '\''; + buf[1] = (char)ch; + buf[2] = '\''; + buf[3] = NUL; + } else { + buf[0] = (char)ch; + buf[1] = NUL; + } + } else { + snprintf(buf, sizeof(buf), "%i", ch); + } + return buf; +} + +#ifdef UNIT_TESTING +#include <stdio.h> + +REAL_FATTR_UNUSED +static inline void viml_pexpr_debug_print_ast_node( + const ExprASTNode *const *const eastnode_p, + const char *const prefix) +{ + if (*eastnode_p == NULL) { + fprintf(stderr, "%s %p : NULL\n", prefix, (void *)eastnode_p); + } else { + fprintf(stderr, "%s %p : %p : %s : %zu:%zu:%zu\n", + prefix, (void *)eastnode_p, (void *)(*eastnode_p), + east_node_type_tab[(*eastnode_p)->type], (*eastnode_p)->start.line, + (*eastnode_p)->start.col, (*eastnode_p)->len); + } +} + +REAL_FATTR_UNUSED +static inline void viml_pexpr_debug_print_ast_stack( + const ExprASTStack *const ast_stack, + const char *const msg) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE +{ + fprintf(stderr, "\n%sstack: %zu:\n", msg, kv_size(*ast_stack)); + for (size_t i = 0; i < kv_size(*ast_stack); i++) { + viml_pexpr_debug_print_ast_node( + (const ExprASTNode *const *)kv_A(*ast_stack, i), + "-"); + } +} + +REAL_FATTR_UNUSED +static inline void viml_pexpr_debug_print_token( + const ParserState *const pstate, const LexExprToken token) + FUNC_ATTR_ALWAYS_INLINE +{ + fprintf(stderr, "\ntkn: %s\n", viml_pexpr_repr_token(pstate, token, NULL)); +} +#define PSTACK(msg) \ + viml_pexpr_debug_print_ast_stack(&ast_stack, #msg) +#define PSTACK_P(msg) \ + viml_pexpr_debug_print_ast_stack(ast_stack, #msg) +#define PNODE_P(eastnode_p, msg) \ + viml_pexpr_debug_print_ast_node((const ExprASTNode *const *)eastnode_p, \ + (#msg)) +#define PTOKEN(tkn) \ + viml_pexpr_debug_print_token(pstate, tkn) +#endif + +const uint8_t node_maxchildren[] = { + [kExprNodeMissing] = 0, + [kExprNodeOpMissing] = 2, + [kExprNodeTernary] = 2, + [kExprNodeTernaryValue] = 2, + [kExprNodeRegister] = 0, + [kExprNodeSubscript] = 2, + [kExprNodeListLiteral] = 1, + [kExprNodeUnaryPlus] = 1, + [kExprNodeBinaryPlus] = 2, + [kExprNodeNested] = 1, + [kExprNodeCall] = 2, + [kExprNodePlainIdentifier] = 0, + [kExprNodePlainKey] = 0, + [kExprNodeComplexIdentifier] = 2, + [kExprNodeUnknownFigure] = 1, + [kExprNodeLambda] = 2, + [kExprNodeDictLiteral] = 1, + [kExprNodeCurlyBracesIdentifier] = 1, + [kExprNodeComma] = 2, + [kExprNodeColon] = 2, + [kExprNodeArrow] = 2, + [kExprNodeComparison] = 2, + [kExprNodeConcat] = 2, + [kExprNodeConcatOrSubscript] = 2, + [kExprNodeInteger] = 0, + [kExprNodeFloat] = 0, + [kExprNodeSingleQuotedString] = 0, + [kExprNodeDoubleQuotedString] = 0, + [kExprNodeOr] = 2, + [kExprNodeAnd] = 2, + [kExprNodeUnaryMinus] = 1, + [kExprNodeBinaryMinus] = 2, + [kExprNodeNot] = 1, + [kExprNodeMultiplication] = 2, + [kExprNodeDivision] = 2, + [kExprNodeMod] = 2, + [kExprNodeOption] = 0, + [kExprNodeEnvironment] = 0, + [kExprNodeAssignment] = 2, +}; + +/// Free memory occupied by AST +/// +/// @param ast AST stack to free. +void viml_pexpr_free_ast(ExprAST ast) +{ + ExprASTStack ast_stack; + kvi_init(ast_stack); + kvi_push(ast_stack, &ast.root); + while (kv_size(ast_stack)) { + ExprASTNode **const cur_node = kv_last(ast_stack); +#ifndef NDEBUG + // Explicitly check for AST recursiveness. + for (size_t i = 0 ; i < kv_size(ast_stack) - 1 ; i++) { + assert(*kv_A(ast_stack, i) != *cur_node); + } +#endif + if (*cur_node == NULL) { + assert(kv_size(ast_stack) == 1); + kv_drop(ast_stack, 1); + } else if ((*cur_node)->children != NULL) { +#ifndef NDEBUG + const uint8_t maxchildren = node_maxchildren[(*cur_node)->type]; + assert(maxchildren > 0); + assert(maxchildren <= 2); + assert(maxchildren == 1 + ? (*cur_node)->children->next == NULL + : ((*cur_node)->children->next == NULL + || (*cur_node)->children->next->next == NULL)); +#endif + kvi_push(ast_stack, &(*cur_node)->children); + } else if ((*cur_node)->next != NULL) { + kvi_push(ast_stack, &(*cur_node)->next); + } else if (*cur_node != NULL) { + kv_drop(ast_stack, 1); + switch ((*cur_node)->type) { + case kExprNodeDoubleQuotedString: + case kExprNodeSingleQuotedString: { + xfree((*cur_node)->data.str.value); + break; + } + case kExprNodeMissing: + case kExprNodeOpMissing: + case kExprNodeTernary: + case kExprNodeTernaryValue: + case kExprNodeRegister: + case kExprNodeSubscript: + case kExprNodeListLiteral: + case kExprNodeUnaryPlus: + case kExprNodeBinaryPlus: + case kExprNodeNested: + case kExprNodeCall: + case kExprNodePlainIdentifier: + case kExprNodePlainKey: + case kExprNodeComplexIdentifier: + case kExprNodeUnknownFigure: + case kExprNodeLambda: + case kExprNodeDictLiteral: + case kExprNodeCurlyBracesIdentifier: + case kExprNodeAssignment: + case kExprNodeComma: + case kExprNodeColon: + case kExprNodeArrow: + case kExprNodeComparison: + case kExprNodeConcat: + case kExprNodeConcatOrSubscript: + case kExprNodeInteger: + case kExprNodeFloat: + case kExprNodeOr: + case kExprNodeAnd: + case kExprNodeUnaryMinus: + case kExprNodeBinaryMinus: + case kExprNodeNot: + case kExprNodeMultiplication: + case kExprNodeDivision: + case kExprNodeMod: + case kExprNodeOption: + case kExprNodeEnvironment: { + break; + } + } + xfree(*cur_node); + *cur_node = NULL; + } + } + kvi_destroy(ast_stack); +} + +// Binary operator precedence and associativity: +// +// Operator | Precedence | Associativity +// ---------+------------+----------------- +// || | 2 | left +// && | 3 | left +// cmp* | 4 | not associative +// + - . | 5 | left +// * / % | 6 | left +// +// * comparison operators: +// +// == ==# ==? != !=# !=? +// =~ =~# =~? !~ !~# !~? +// > ># >? <= <=# <=? +// < <# <? >= >=# >=? +// is is# is? isnot isnot# isnot? + +/// Allocate a new node and set some of the values +/// +/// @param[in] type Node type to allocate. +/// @param[in] level Node level to allocate +static inline ExprASTNode *viml_pexpr_new_node(const ExprASTNodeType type) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_MALLOC +{ + ExprASTNode *ret = xmalloc(sizeof(*ret)); + ret->type = type; + ret->children = NULL; + ret->next = NULL; + return ret; +} + +static struct { + ExprOpLvl lvl; + ExprOpAssociativity ass; +} node_type_to_node_props[] = { + [kExprNodeMissing] = { kEOpLvlInvalid, kEOpAssNo, }, + [kExprNodeOpMissing] = { kEOpLvlMultiplication, kEOpAssNo }, + + [kExprNodeNested] = { kEOpLvlParens, kEOpAssNo }, + // Note: below nodes are kEOpLvlSubscript for “binary operator” itself, but + // kEOpLvlParens when it comes to inside the parenthesis. + [kExprNodeCall] = { kEOpLvlParens, kEOpAssNo }, + [kExprNodeSubscript] = { kEOpLvlParens, kEOpAssNo }, + + [kExprNodeUnknownFigure] = { kEOpLvlParens, kEOpAssLeft }, + [kExprNodeLambda] = { kEOpLvlParens, kEOpAssNo }, + [kExprNodeDictLiteral] = { kEOpLvlParens, kEOpAssNo }, + [kExprNodeListLiteral] = { kEOpLvlParens, kEOpAssNo }, + + [kExprNodeArrow] = { kEOpLvlArrow, kEOpAssNo }, + + // Right associativity for comma because this means easier access to arguments + // list, etc: for "[a, b, c, d]" you can access "a" in one step if it is + // represented as "list(comma(a, comma(b, comma(c, d))))" then if it is + // "list(comma(comma(comma(a, b), c), d))" in which case you will need to + // traverse all three comma() structures. And with comma operator (including + // actual comma operator from C which is not present in VimL) nobody cares + // about associativity, only about order of execution. + [kExprNodeComma] = { kEOpLvlComma, kEOpAssRight }, + + // Colons are not eligible for chaining, so nobody cares about associativity. + [kExprNodeColon] = { kEOpLvlColon, kEOpAssNo }, + + [kExprNodeTernary] = { kEOpLvlTernary, kEOpAssRight }, + + [kExprNodeOr] = { kEOpLvlOr, kEOpAssLeft }, + + [kExprNodeAnd] = { kEOpLvlAnd, kEOpAssLeft }, + + [kExprNodeTernaryValue] = { kEOpLvlTernaryValue, kEOpAssRight }, + + [kExprNodeComparison] = { kEOpLvlComparison, kEOpAssRight }, + + [kExprNodeBinaryPlus] = { kEOpLvlAddition, kEOpAssLeft }, + [kExprNodeBinaryMinus] = { kEOpLvlAddition, kEOpAssLeft }, + [kExprNodeConcat] = { kEOpLvlAddition, kEOpAssLeft }, + + [kExprNodeMultiplication] = { kEOpLvlMultiplication, kEOpAssLeft }, + [kExprNodeDivision] = { kEOpLvlMultiplication, kEOpAssLeft }, + [kExprNodeMod] = { kEOpLvlMultiplication, kEOpAssLeft }, + + [kExprNodeUnaryPlus] = { kEOpLvlUnary, kEOpAssNo }, + [kExprNodeUnaryMinus] = { kEOpLvlUnary, kEOpAssNo }, + [kExprNodeNot] = { kEOpLvlUnary, kEOpAssNo }, + + [kExprNodeConcatOrSubscript] = { kEOpLvlSubscript, kEOpAssLeft }, + + [kExprNodeCurlyBracesIdentifier] = { kEOpLvlComplexIdentifier, kEOpAssLeft }, + + [kExprNodeAssignment] = { kEOpLvlAssignment, kEOpAssLeft }, + + [kExprNodeComplexIdentifier] = { kEOpLvlValue, kEOpAssLeft }, + + [kExprNodePlainIdentifier] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodePlainKey] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeRegister] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeInteger] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeFloat] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeDoubleQuotedString] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeSingleQuotedString] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeOption] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeEnvironment] = { kEOpLvlValue, kEOpAssNo }, +}; + +/// Get AST node priority level +/// +/// Used primary to reduce line length, so keep the name short. +/// +/// @param[in] node Node to get priority for. +/// +/// @return Node priority level. +static inline ExprOpLvl node_lvl(const ExprASTNode node) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + return node_type_to_node_props[node.type].lvl; +} + +/// Get AST node associativity, to be used for operator nodes primary +/// +/// Used primary to reduce line length, so keep the name short. +/// +/// @param[in] node Node to get priority for. +/// +/// @return Node associativity. +static inline ExprOpAssociativity node_ass(const ExprASTNode node) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + return node_type_to_node_props[node.type].ass; +} + +/// Handle binary operator +/// +/// This function is responsible for handling priority levels as well. +/// +/// @param[in] pstate Parser state, used for error reporting. +/// @param ast_stack AST stack. May be popped of some values and will +/// definitely receive new ones. +/// @param bop_node New node to handle. +/// @param[out] want_node_p New value of want_node. +/// @param[out] ast_err Location where error is saved, if any. +/// +/// @return True if no errors occurred, false otherwise. +static bool viml_pexpr_handle_bop(const ParserState *const pstate, + ExprASTStack *const ast_stack, + ExprASTNode *const bop_node, + ExprASTWantedNode *const want_node_p, + ExprASTError *const ast_err) + FUNC_ATTR_NONNULL_ALL +{ + bool ret = true; + ExprASTNode **top_node_p = NULL; + ExprASTNode *top_node; + ExprOpLvl top_node_lvl; + ExprOpAssociativity top_node_ass; + assert(kv_size(*ast_stack)); + const ExprOpLvl bop_node_lvl = ((bop_node->type == kExprNodeCall + || bop_node->type == kExprNodeSubscript) + ? kEOpLvlSubscript + : node_lvl(*bop_node)); +#ifndef NDEBUG + const ExprOpAssociativity bop_node_ass = ( + (bop_node->type == kExprNodeCall + || bop_node->type == kExprNodeSubscript) + ? kEOpAssLeft + : node_ass(*bop_node)); +#endif + do { + ExprASTNode **new_top_node_p = kv_last(*ast_stack); + ExprASTNode *new_top_node = *new_top_node_p; + assert(new_top_node != NULL); + const ExprOpLvl new_top_node_lvl = node_lvl(*new_top_node); + const ExprOpAssociativity new_top_node_ass = node_ass(*new_top_node); + assert(bop_node_lvl != new_top_node_lvl + || bop_node_ass == new_top_node_ass); + if (top_node_p != NULL + && ((bop_node_lvl > new_top_node_lvl + || (bop_node_lvl == new_top_node_lvl + && new_top_node_ass == kEOpAssNo)))) { + break; + } + kv_drop(*ast_stack, 1); + top_node_p = new_top_node_p; + top_node = new_top_node; + top_node_lvl = new_top_node_lvl; + top_node_ass = new_top_node_ass; + if (bop_node_lvl == top_node_lvl && top_node_ass == kEOpAssRight) { + break; + } + } while (kv_size(*ast_stack)); + if (top_node_ass == kEOpAssLeft || top_node_lvl != bop_node_lvl) { + // outer(op(x,y)) -> outer(new_op(op(x,y),*)) + // + // Before: top_node_p = outer(*), points to op(x,y) + // Other stack elements unknown + // + // After: top_node_p = outer(*), points to new_op(op(x,y)) + // &bop_node->children->next = new_op(op(x,y),*), points to NULL + *top_node_p = bop_node; + bop_node->children = top_node; + assert(bop_node->children->next == NULL); + kvi_push(*ast_stack, top_node_p); + kvi_push(*ast_stack, &bop_node->children->next); + } else { + assert(top_node_lvl == bop_node_lvl && top_node_ass == kEOpAssRight); + assert(top_node->children != NULL && top_node->children->next != NULL); + // outer(op(x,y)) -> outer(op(x,new_op(y,*))) + // + // Before: top_node_p = outer(*), points to op(x,y) + // Other stack elements unknown + // + // After: top_node_p = outer(*), points to op(x,new_op(y)) + // &top_node->children->next = op(x,*), points to new_op(y) + // &bop_node->children->next = new_op(y,*), points to NULL + bop_node->children = top_node->children->next; + top_node->children->next = bop_node; + assert(bop_node->children->next == NULL); + kvi_push(*ast_stack, top_node_p); + kvi_push(*ast_stack, &top_node->children->next); + kvi_push(*ast_stack, &bop_node->children->next); + // TODO(ZyX-I): Make this not error, but treat like Python does + if (bop_node->type == kExprNodeComparison) { + east_set_error(pstate, ast_err, + _("E15: Operator is not associative: %.*s"), + bop_node->start); + ret = false; + } + } + *want_node_p = kENodeValue; + return ret; +} + +/// ParserPosition literal based on ParserPosition pos with columns shifted +/// +/// Function does not check whether resulting position is valid. +/// +/// @param[in] pos Position to shift. +/// @param[in] shift Number of bytes to shift. +/// +/// @return Shifted position. +static inline ParserPosition shifted_pos(const ParserPosition pos, + const size_t shift) + FUNC_ATTR_CONST FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT +{ + return (ParserPosition) { .line = pos.line, .col = pos.col + shift }; +} + +/// ParserPosition literal based on ParserPosition pos with specified column +/// +/// Function does not check whether remaining position is valid. +/// +/// @param[in] pos Position to adjust. +/// @param[in] new_col New column. +/// +/// @return Shifted position. +static inline ParserPosition recol_pos(const ParserPosition pos, + const size_t new_col) + FUNC_ATTR_CONST FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT +{ + return (ParserPosition) { .line = pos.line, .col = new_col }; +} + +/// Get highlight group name +#define HL(g) (is_invalid ? "NvimInvalid" #g : "Nvim" #g) + +/// Highlight current token with the given group +#define HL_CUR_TOKEN(g) \ + viml_parser_highlight(pstate, cur_token.start, cur_token.len, \ + HL(g)) + +/// Allocate new node, saving some values +#define NEW_NODE(type) \ + viml_pexpr_new_node(type) + +/// Set position of the given node to position from the given token +/// +/// @param cur_node Node to modify. +/// @param cur_token Token to set position from. +#define POS_FROM_TOKEN(cur_node, cur_token) \ + do { \ + (cur_node)->start = cur_token.start; \ + (cur_node)->len = cur_token.len; \ + } while (0) + +/// Allocate new node and set its position from the current token +/// +/// If previous token happened to contain spacing then it will be included. +/// +/// @param cur_node Variable to save allocated node to. +/// @param typ Node type. +#define NEW_NODE_WITH_CUR_POS(cur_node, typ) \ + do { \ + (cur_node) = NEW_NODE(typ); \ + POS_FROM_TOKEN((cur_node), cur_token); \ + if (prev_token.type == kExprLexSpacing) { \ + (cur_node)->start = prev_token.start; \ + (cur_node)->len += prev_token.len; \ + } \ + } while (0) + +/// Check whether it is possible to have next expression after current +/// +/// For :echo: `:echo @a @a` is a valid expression. `:echo (@a @a)` is not. +#define MAY_HAVE_NEXT_EXPR \ + (kv_size(ast_stack) == 1) + +/// Add operator node +/// +/// @param[in] cur_node Node to add. +#define ADD_OP_NODE(cur_node) \ + is_invalid |= !viml_pexpr_handle_bop(pstate, &ast_stack, cur_node, \ + &want_node, &ast.err) + +/// Record missing operator: for things like +/// +/// :echo @a @a +/// +/// (allowed) or +/// +/// :echo (@a @a) +/// +/// (parsed as OpMissing(@a, @a)). +#define OP_MISSING \ + do { \ + if (flags & kExprFlagsMulti && MAY_HAVE_NEXT_EXPR) { \ + /* Multiple expressions allowed, return without calling */ \ + /* viml_parser_advance(). */ \ + goto viml_pexpr_parse_end; \ + } else { \ + assert(*top_node_p != NULL); \ + ERROR_FROM_TOKEN_AND_MSG(cur_token, _("E15: Missing operator: %.*s")); \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeOpMissing); \ + cur_node->len = 0; \ + ADD_OP_NODE(cur_node); \ + goto viml_pexpr_parse_process_token; \ + } \ + } while (0) + +/// Record missing value: for things like "* 5" +/// +/// @param[in] msg Error message. +#define ADD_VALUE_IF_MISSING(msg) \ + do { \ + if (want_node == kENodeValue) { \ + ERROR_FROM_TOKEN_AND_MSG(cur_token, (msg)); \ + NEW_NODE_WITH_CUR_POS((*top_node_p), kExprNodeMissing); \ + (*top_node_p)->len = 0; \ + want_node = kENodeOperator; \ + } \ + } while (0) + +/// Set AST error, unless AST already is not correct +/// +/// @param[out] ret_ast AST to set error in. +/// @param[in] pstate Parser state, used to get error message argument. +/// @param[in] msg Error message, assumed to be already translated and +/// containing a single %token "%.*s". +/// @param[in] start Position at which error occurred. +static inline void east_set_error(const ParserState *const pstate, + ExprASTError *const ret_ast_err, + const char *const msg, + const ParserPosition start) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE +{ + if (ret_ast_err->msg != NULL) { + return; + } + const ParserLine pline = pstate->reader.lines.items[start.line]; + ret_ast_err->msg = msg; + ret_ast_err->arg_len = (int)(pline.size - start.col); + ret_ast_err->arg = pline.data + start.col; +} + +/// Set error from the given token and given message +#define ERROR_FROM_TOKEN_AND_MSG(cur_token, msg) \ + do { \ + is_invalid = true; \ + east_set_error(pstate, &ast.err, msg, cur_token.start); \ + } while (0) + +/// Like #ERROR_FROM_TOKEN_AND_MSG, but gets position from a node +#define ERROR_FROM_NODE_AND_MSG(node, msg) \ + do { \ + is_invalid = true; \ + east_set_error(pstate, &ast.err, msg, node->start); \ + } while (0) + +/// Set error from the given kExprLexInvalid token +#define ERROR_FROM_TOKEN(cur_token) \ + ERROR_FROM_TOKEN_AND_MSG(cur_token, cur_token.data.err.msg) + +/// Select figure brace type, altering highlighting as well if needed +/// +/// @param[out] node Node to modify type. +/// @param[in] new_type New type, one of ExprASTNodeType values without +/// kExprNode prefix. +/// @param[in] hl Corresponding highlighting, passed as an argument to #HL. +#define SELECT_FIGURE_BRACE_TYPE(node, new_type, hl) \ + do { \ + ExprASTNode *const node_ = (node); \ + assert(node_->type == kExprNodeUnknownFigure \ + || node_->type == kExprNode##new_type); \ + node_->type = kExprNode##new_type; \ + if (pstate->colors) { \ + kv_A(*pstate->colors, node_->data.fig.opening_hl_idx).group = \ + HL(hl); \ + } \ + } while (0) + +/// Add identifier which should constitute complex identifier node +/// +/// This one is to be called only in case want_node is kENodeOperator. +/// +/// @param new_ident_node_code Code used to create a new identifier node and +/// update want_node and ast_stack, without +/// a trailing semicolon. +/// @param hl Highlighting name to use, passed as an argument to #HL. +#define ADD_IDENT(new_ident_node_code, hl) \ + do { \ + assert(want_node == kENodeOperator); \ + /* Operator: may only be curly braces name, but only under certain */ \ + /* conditions. */ \ +\ + /* First condition is that there is no space before a part of complex */ \ + /* identifier. */ \ + if (prev_token.type == kExprLexSpacing) { \ + OP_MISSING; \ + } \ + switch ((*top_node_p)->type) { \ + /* Second is that previous node is one of the identifiers: */ \ + /* complex, plain, curly braces. */ \ +\ + /* TODO(ZyX-I): Extend syntax to allow ${expr}. This is needed to */ \ + /* handle environment variables like those bash uses for */ \ + /* `export -f`: their names consist not only of alphanumeric */ \ + /* characetrs. */ \ + case kExprNodeComplexIdentifier: \ + case kExprNodePlainIdentifier: \ + case kExprNodeCurlyBracesIdentifier: { \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComplexIdentifier); \ + cur_node->len = 0; \ + cur_node->children = *top_node_p; \ + *top_node_p = cur_node; \ + kvi_push(ast_stack, &cur_node->children->next); \ + ExprASTNode **const new_top_node_p = kv_last(ast_stack); \ + assert(*new_top_node_p == NULL); \ + new_ident_node_code; \ + *new_top_node_p = cur_node; \ + HL_CUR_TOKEN(hl); \ + break; \ + } \ + default: { \ + OP_MISSING; \ + break; \ + } \ + } \ + } while (0) + +/// Determine whether given parse type is an assignment +/// +/// @param[in] pt Checked parse type. +/// +/// @return true if parsing an assignment, false otherwise. +static inline bool pt_is_assignment(const ExprASTParseType pt) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + return (pt == kEPTAssignment || pt == kEPTSingleAssignment); +} + +/// Structure used to define “string shifts” necessary to map string +/// highlighting to actual strings. +typedef struct { + size_t start; ///< Where special character starts in original string. + size_t orig_len; ///< Length of orininal string (e.g. 4 for "\x80"). + size_t act_len; ///< Length of resulting character(s) (e.g. 1 for "\x80"). + bool escape_not_known; ///< True if escape sequence in original is not known. +} StringShift; + +/// Parse and highlight single- or double-quoted string +/// +/// Function is supposed to detect and highlight regular expressions (but does +/// not do now). +/// +/// @param[out] pstate Parser state which also contains a place where +/// highlighting is saved. +/// @param[out] node Node where string parsing results are saved. +/// @param[in] token Token to highlight. +/// @param[in] ast_stack Parser AST stack, used to detect whether current +/// string is a regex. +/// @param[in] is_invalid Whether currently processed token is not valid. +static void parse_quoted_string(ParserState *const pstate, + ExprASTNode *const node, + const LexExprToken token, + const ExprASTStack ast_stack, + const bool is_invalid) + FUNC_ATTR_NONNULL_ALL +{ + const ParserLine pline = pstate->reader.lines.items[token.start.line]; + const char *const s = pline.data + token.start.col; + const char *const e = s + token.len - token.data.str.closed; + const char *p = s + 1; + const bool is_double = (token.type == kExprLexDoubleQuotedString); + size_t size = token.len - token.data.str.closed - 1; + kvec_withinit_t(StringShift, 16) shifts; + kvi_init(shifts); + if (!is_double) { + viml_parser_highlight(pstate, token.start, 1, HL(SingleQuote)); + while (p < e) { + const char *const chunk_e = memchr(p, '\'', (size_t)(e - p)); + if (chunk_e == NULL) { + break; + } + size--; + p = chunk_e + 2; + if (pstate->colors) { + kvi_push(shifts, ((StringShift) { + .start = token.start.col + (size_t)(chunk_e - s), + .orig_len = 2, + .act_len = 1, + .escape_not_known = false, + })); + } + } + node->data.str.size = size; + if (size == 0) { + node->data.str.value = NULL; + } else { + char *v_p; + v_p = node->data.str.value = xmallocz(size); + p = s + 1; + while (p < e) { + const char *const chunk_e = memchr(p, '\'', (size_t)(e - p)); + if (chunk_e == NULL) { + memcpy(v_p, p, (size_t)(e - p)); + break; + } + memcpy(v_p, p, (size_t)(chunk_e - p)); + v_p += (size_t)(chunk_e - p) + 1; + v_p[-1] = '\''; + p = chunk_e + 2; + } + } + } else { + viml_parser_highlight(pstate, token.start, 1, HL(DoubleQuote)); + for (p = s + 1; p < e; p++) { + if (*p == '\\' && p + 1 < e) { + p++; + if (p + 1 == e) { + size--; + break; + } + switch (*p) { + // A "\<x>" form occupies at least 4 characters, and produces up to + // 6 characters: reserve space for 2 extra, but do not compute actual + // length just now, it would be costy. + case '<': { + size += 2; + break; + } + // Hexadecimal, always single byte, but at least three bytes each. + case 'x': case 'X': { + size--; + if (ascii_isxdigit(p[1])) { + size--; + if (p + 2 < e && ascii_isxdigit(p[2])) { + size--; + } + } + break; + } + // Unicode + // + // \uF takes 1 byte which is 2 bytes less then escape sequence. + // \uFF: 2 bytes, 2 bytes less. + // \uFFF: 3 bytes, 2 bytes less. + // \uFFFF: 3 bytes, 3 bytes less. + // \UFFFFF: 4 bytes, 3 bytes less. + // \UFFFFFF: 5 bytes, 3 bytes less. + // \UFFFFFFF: 6 bytes, 3 bytes less. + // \U7FFFFFFF: 6 bytes, 4 bytes less. + case 'u': case 'U': { + const char *const esc_start = p; + size_t n = (*p == 'u' ? 4 : 8); + int nr = 0; + p++; + while (p + 1 < e && n-- && ascii_isxdigit(p[1])) { + p++; + nr = (nr << 4) + hex2nr(*p); + } + // Escape length: (esc_start - 1) points to "\\", esc_start to "u" + // or "U", p to the byte after last byte. So escape sequence + // occupies p - (esc_start - 1), but it stands for a utf_char2len + // bytes. + size -= (size_t)((p - (esc_start - 1)) - utf_char2len(nr)); + p--; + break; + } + // Octal, always single byte, but at least two bytes each. + case '0': case '1': case '2': case '3': case '4': case '5': case '6': + case '7': { + size--; + p++; + if (*p >= '0' && *p <= '7') { + size--; + p++; + if (p < e && *p >= '0' && *p <= '7') { + size--; + p++; + } + } + break; + } + default: { + size--; + break; + } + } + } + } + if (size == 0) { + node->data.str.value = NULL; + node->data.str.size = 0; + } else { + char *v_p; + v_p = node->data.str.value = xmalloc(size); + p = s + 1; + while (p < e) { + const char *const chunk_e = memchr(p, '\\', (size_t)(e - p)); + if (chunk_e == NULL) { + memcpy(v_p, p, (size_t)(e - p)); + v_p += e - p; + break; + } + memcpy(v_p, p, (size_t)(chunk_e - p)); + v_p += (size_t)(chunk_e - p); + p = chunk_e + 1; + if (p == e) { + *v_p++ = '\\'; + break; + } + bool is_unknown = false; + const char *const v_p_start = v_p; + switch (*p) { +#define SINGLE_CHAR_ESC(ch, real_ch) \ + case ch: { \ + *v_p++ = real_ch; \ + p++; \ + break; \ + } + SINGLE_CHAR_ESC('b', BS) + SINGLE_CHAR_ESC('e', ESC) + SINGLE_CHAR_ESC('f', FF) + SINGLE_CHAR_ESC('n', NL) + SINGLE_CHAR_ESC('r', CAR) + SINGLE_CHAR_ESC('t', TAB) + SINGLE_CHAR_ESC('"', '"') + SINGLE_CHAR_ESC('\\', '\\') +#undef SINGLE_CHAR_ESC + + // Hexadecimal or unicode. + case 'X': case 'x': case 'u': case 'U': { + if (p + 1 < e && ascii_isxdigit(p[1])) { + size_t n; + int nr; + bool is_hex = (*p == 'x' || *p == 'X'); + + if (is_hex) { + n = 2; + } else if (*p == 'u') { + n = 4; + } else { + n = 8; + } + nr = 0; + while (p + 1 < e && n-- && ascii_isxdigit(p[1])) { + p++; + nr = (nr << 4) + hex2nr(*p); + } + p++; + if (is_hex) { + *v_p++ = (char)nr; + } else { + v_p += utf_char2bytes(nr, (char_u *)v_p); + } + } else { + is_unknown = true; + *v_p++ = *p; + p++; + } + break; + } + // Octal: "\1", "\12", "\123". + case '0': case '1': case '2': case '3': case '4': case '5': case '6': + case '7': { + uint8_t ch = (uint8_t)(*p++ - '0'); + if (p < e && *p >= '0' && *p <= '7') { + ch = (uint8_t)((ch << 3) + *p++ - '0'); + if (p < e && *p >= '0' && *p <= '7') { + ch = (uint8_t)((ch << 3) + *p++ - '0'); + } + } + *v_p++ = (char)ch; + break; + } + // Special key, e.g.: "\<C-W>" + case '<': { + const size_t special_len = ( + trans_special((const char_u **)&p, (size_t)(e - p), + (char_u *)v_p, true, true)); + if (special_len != 0) { + v_p += special_len; + } else { + is_unknown = true; + mb_copy_char((const char_u **)&p, (char_u **)&v_p); + } + break; + } + default: { + is_unknown = true; + mb_copy_char((const char_u **)&p, (char_u **)&v_p); + break; + } + } + if (pstate->colors) { + kvi_push(shifts, ((StringShift) { + .start = token.start.col + (size_t)(chunk_e - s), + .orig_len = (size_t)(p - chunk_e), + .act_len = (size_t)(v_p - (char *)v_p_start), + .escape_not_known = is_unknown, + })); + } + } + node->data.str.size = (size_t)(v_p - node->data.str.value); + } + } + if (pstate->colors) { + // TODO(ZyX-I): use ast_stack to determine and highlight regular expressions + // TODO(ZyX-I): use ast_stack to determine and highlight printf format str + // TODO(ZyX-I): use ast_stack to determine and highlight expression strings + size_t next_col = token.start.col + 1; + const char *const body_str = (is_double + ? HL(DoubleQuotedBody) + : HL(SingleQuotedBody)); + const char *const esc_str = (is_double + ? HL(DoubleQuotedEscape) + : HL(SingleQuotedQuote)); + const char *const ukn_esc_str = (is_double + ? HL(DoubleQuotedUnknownEscape) + : HL(SingleQuotedUnknownEscape)); + for (size_t i = 0; i < kv_size(shifts); i++) { + const StringShift cur_shift = kv_A(shifts, i); + if (cur_shift.start > next_col) { + viml_parser_highlight(pstate, recol_pos(token.start, next_col), + cur_shift.start - next_col, + body_str); + } + viml_parser_highlight(pstate, recol_pos(token.start, cur_shift.start), + cur_shift.orig_len, + (cur_shift.escape_not_known + ? ukn_esc_str + : esc_str)); + next_col = cur_shift.start + cur_shift.orig_len; + } + if (next_col - token.start.col < token.len - token.data.str.closed) { + viml_parser_highlight(pstate, recol_pos(token.start, next_col), + (token.start.col + + token.len + - token.data.str.closed + - next_col), + body_str); + } + } + if (token.data.str.closed) { + if (is_double) { + viml_parser_highlight(pstate, shifted_pos(token.start, token.len - 1), + 1, HL(DoubleQuote)); + } else { + viml_parser_highlight(pstate, shifted_pos(token.start, token.len - 1), + 1, HL(SingleQuote)); + } + } + kvi_destroy(shifts); +} + +/// Additional flags to pass to lexer depending on want_node +static const int want_node_to_lexer_flags[] = { + [kENodeValue] = kELFlagIsNotCmp, + [kENodeOperator] = kELFlagForbidScope, +}; + +/// Number of characters to highlight as NumberPrefix depending on the base +static const uint8_t base_to_prefix_length[] = { + [2] = 2, + [8] = 1, + [10] = 0, + [16] = 2, +}; + +/// Parse one VimL expression +/// +/// @param pstate Parser state. +/// @param[in] flags Additional flags, see ExprParserFlags +/// +/// @return Parsed AST. +ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL +{ + ExprAST ast = { + .err = { + .msg = NULL, + .arg_len = 0, + .arg = NULL, + }, + .root = NULL, + }; + // Expression stack contains current branch in AST tree: that is + // - Stack item 0 contains root of the tree, i.e. &ast->root. + // - Stack item i points to the previous stack items’ last child. + // + // When parser expects “value” node that is something like identifier or "[" + // (list start) last stack item contains NULL. Otherwise last stack item is + // supposed to contain last “finished” value: e.g. "1" or "+(1, 1)" (node + // representing "1+1"). + ExprASTStack ast_stack; + kvi_init(ast_stack); + kvi_push(ast_stack, &ast.root); + ExprASTWantedNode want_node = kENodeValue; + ExprASTParseTypeStack pt_stack; + kvi_init(pt_stack); + kvi_push(pt_stack, kEPTExpr); + if (flags & kExprFlagsParseLet) { + kvi_push(pt_stack, kEPTAssignment); + } + LexExprToken prev_token = { .type = kExprLexMissing }; + bool highlighted_prev_spacing = false; + // Lambda node, valid when parsing lambda arguments only. + ExprASTNode *lambda_node = NULL; + size_t asgn_level = 0; + do { + const bool is_concat_or_subscript = ( + want_node == kENodeValue + && kv_size(ast_stack) > 1 + && (*kv_Z(ast_stack, 1))->type == kExprNodeConcatOrSubscript); + const int lexer_additional_flags = ( + kELFlagPeek + | ((flags & kExprFlagsDisallowEOC) ? kELFlagForbidEOC : 0) + | ((want_node == kENodeValue + && (kv_size(ast_stack) == 1 + || ((*kv_Z(ast_stack, 1))->type != kExprNodeConcat + && ((*kv_Z(ast_stack, 1))->type + != kExprNodeConcatOrSubscript)))) + ? kELFlagAllowFloat + : 0)); + LexExprToken cur_token = viml_pexpr_next_token( + pstate, want_node_to_lexer_flags[want_node] | lexer_additional_flags); + if (cur_token.type == kExprLexEOC) { + break; + } + LexExprTokenType tok_type = cur_token.type; + const bool token_invalid = (tok_type == kExprLexInvalid); + bool is_invalid = token_invalid; +viml_pexpr_parse_process_token: + // May use different flags this time. + cur_token = viml_pexpr_next_token( + pstate, want_node_to_lexer_flags[want_node] | lexer_additional_flags); + if (tok_type == kExprLexSpacing) { + if (is_invalid) { + HL_CUR_TOKEN(Spacing); + } else { + // Do not do anything: let regular spacing be highlighted as normal. + // This also allows later to highlight spacing as invalid. + } + goto viml_pexpr_parse_cycle_end; + } else if (is_invalid && prev_token.type == kExprLexSpacing + && !highlighted_prev_spacing) { + viml_parser_highlight(pstate, prev_token.start, prev_token.len, + HL(Spacing)); + is_invalid = false; + highlighted_prev_spacing = true; + } + const ParserLine pline = pstate->reader.lines.items[cur_token.start.line]; + ExprASTNode **const top_node_p = kv_last(ast_stack); + assert(kv_size(ast_stack) >= 1); + ExprASTNode *cur_node = NULL; +#ifndef NDEBUG + const bool want_value = (want_node == kENodeValue); + assert(want_value == (*top_node_p == NULL)); + assert(kv_A(ast_stack, 0) == &ast.root); + // Check that stack item i + 1 points to stack items’ i *last* child. + for (size_t i = 0; i + 1 < kv_size(ast_stack); i++) { + const bool item_null = (want_value && i + 2 == kv_size(ast_stack)); + assert((&(*kv_A(ast_stack, i))->children == kv_A(ast_stack, i + 1) + && (item_null + ? (*kv_A(ast_stack, i))->children == NULL + : (*kv_A(ast_stack, i))->children->next == NULL)) + || ((&(*kv_A(ast_stack, i))->children->next + == kv_A(ast_stack, i + 1)) + && (item_null + ? (*kv_A(ast_stack, i))->children->next == NULL + : (*kv_A(ast_stack, i))->children->next->next == NULL))); + } +#endif + // Note: in Vim whether expression "cond?d.a:2" is valid depends both on + // "cond" and whether "d" is a dictionary: expression is valid if condition + // is true and "d" is a dictionary (with "a" key or it will complain about + // missing one, but this is not relevant); if any of the requirements is + // broken then this thing is parsed as "d . a:2" yielding missing colon + // error. This parser does not allow such ambiguity, especially because it + // simply can’t: whether "d" is a dictionary is not known at the parsing + // time. + // + // Here example will always contain a concat with "a:2" sucking colon, + // making expression invalid both because there is no longer a spare colon + // for ternary and because concatenating dictionary with anything is not + // valid. There are more cases when this will make a difference though. + const bool node_is_key = ( + is_concat_or_subscript + && (cur_token.type == kExprLexPlainIdentifier + ? (!cur_token.data.var.autoload + && cur_token.data.var.scope == kExprVarScopeMissing) + : (cur_token.type == kExprLexNumber)) + && prev_token.type != kExprLexSpacing); + if (is_concat_or_subscript && !node_is_key) { + // Note: in Vim "d. a" (this is the reason behind `prev_token.type != + // kExprLexSpacing` part of the condition) as well as any other "d.{expr}" + // where "{expr}" does not look like a key is invalid whenever "d" happens + // to be a dictionary. Since parser has no idea whether preceding + // expression is actually a dictionary it can’t outright reject anything, + // so it turns kExprNodeConcatOrSubscript into kExprNodeConcat instead, + // which will yield different errors then Vim does in a number of + // circumstances, and in any case runtime and not parse time errors. + (*kv_Z(ast_stack, 1))->type = kExprNodeConcat; + } + // Pop some stack pt_stack items in case of misplaced nodes. + const bool is_single_assignment = kv_last(pt_stack) == kEPTSingleAssignment; + switch (kv_last(pt_stack)) { + case kEPTExpr: { + break; + } + case kEPTLambdaArguments: { + if ((want_node == kENodeOperator + && tok_type != kExprLexComma + && tok_type != kExprLexArrow) + || (want_node == kENodeValue + && !(cur_token.type == kExprLexPlainIdentifier + && cur_token.data.var.scope == kExprVarScopeMissing + && !cur_token.data.var.autoload) + && tok_type != kExprLexArrow)) { + lambda_node->data.fig.type_guesses.allow_lambda = false; + if (lambda_node->children != NULL + && lambda_node->children->type == kExprNodeComma) { + // If lambda has comma child this means that parser has already seen + // at least "{arg1,", so node cannot possibly be anything, but + // lambda. + + // Vim may give E121 or E720 in this case, but it does not look + // right to have either because both are results of reevaluation + // possibly-lambda node as a dictionary and here this is not going + // to happen. + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected lambda arguments list or arrow: %.*s")); + } else { + // Else it may appear that possibly-lambda node is actually + // a dictionary or curly-braces-name identifier. + lambda_node = NULL; + kv_drop(pt_stack, 1); + } + } + break; + } + case kEPTSingleAssignment: + case kEPTAssignment: { + if (want_node == kENodeValue + && tok_type != kExprLexBracket + && tok_type != kExprLexPlainIdentifier + && (tok_type != kExprLexFigureBrace || cur_token.data.brc.closing) + && !(node_is_key && tok_type == kExprLexNumber) + && tok_type != kExprLexEnv + && tok_type != kExprLexOption + && tok_type != kExprLexRegister) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected value part of assignment lvalue: %.*s")); + kv_drop(pt_stack, 1); + } else if (want_node == kENodeOperator + && tok_type != kExprLexBracket + && (tok_type != kExprLexFigureBrace + || cur_token.data.brc.closing) + && tok_type != kExprLexDot + && (tok_type != kExprLexComma || !is_single_assignment) + && tok_type != kExprLexAssignment + // Curly brace identifiers: will contain plain identifier or + // another curly brace in position where operator is wanted. + && !((tok_type == kExprLexPlainIdentifier + || (tok_type == kExprLexFigureBrace + && !cur_token.data.brc.closing)) + && prev_token.type != kExprLexSpacing)) { + if (flags & kExprFlagsMulti && MAY_HAVE_NEXT_EXPR) { + goto viml_pexpr_parse_end; + } + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected assignment operator or subscript: %.*s")); + kv_drop(pt_stack, 1); + } + assert(kv_size(pt_stack)); + break; + } + } + assert(kv_size(pt_stack)); + const ExprASTParseType cur_pt = kv_last(pt_stack); + assert(lambda_node == NULL || cur_pt == kEPTLambdaArguments); + switch (tok_type) { + case kExprLexMissing: + case kExprLexSpacing: + case kExprLexEOC: { + assert(false); + } + case kExprLexInvalid: { + ERROR_FROM_TOKEN(cur_token); + tok_type = cur_token.data.err.type; + goto viml_pexpr_parse_process_token; + } + case kExprLexRegister: { + if (want_node == kENodeOperator) { + // Register in operator position: e.g. @a @a + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeRegister); + cur_node->data.reg.name = cur_token.data.reg.name; + *top_node_p = cur_node; + want_node = kENodeOperator; + HL_CUR_TOKEN(Register); + break; + } +#define SIMPLE_UB_OP(op) \ + case kExprLex##op: { \ + if (want_node == kENodeValue) { \ + /* Value level: assume unary operator. */ \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnary##op); \ + *top_node_p = cur_node; \ + kvi_push(ast_stack, &cur_node->children); \ + HL_CUR_TOKEN(Unary##op); \ + } else { \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeBinary##op); \ + ADD_OP_NODE(cur_node); \ + HL_CUR_TOKEN(Binary##op); \ + } \ + want_node = kENodeValue; \ + break; \ + } + SIMPLE_UB_OP(Plus) + SIMPLE_UB_OP(Minus) +#undef SIMPLE_UB_OP +#define SIMPLE_B_OP(op, msg) \ + case kExprLex##op: { \ + ADD_VALUE_IF_MISSING(_("E15: Unexpected " msg ": %.*s")); \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNode##op); \ + HL_CUR_TOKEN(op); \ + ADD_OP_NODE(cur_node); \ + break; \ + } + SIMPLE_B_OP(Or, "or operator") + SIMPLE_B_OP(And, "and operator") +#undef SIMPLE_B_OP + case kExprLexMultiplication: { + ADD_VALUE_IF_MISSING( + _("E15: Unexpected multiplication-like operator: %.*s")); + switch (cur_token.data.mul.type) { +#define MUL_OP(lex_op_tail, node_op_tail) \ + case kExprLexMul##lex_op_tail: { \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNode##node_op_tail); \ + HL_CUR_TOKEN(node_op_tail); \ + break; \ + } + MUL_OP(Mul, Multiplication) + MUL_OP(Div, Division) + MUL_OP(Mod, Mod) +#undef MUL_OP + } + ADD_OP_NODE(cur_node); + break; + } + case kExprLexOption: { + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeOption); + if (cur_token.type == kExprLexInvalid) { + assert(cur_token.len == 1 + || (cur_token.len == 3 + && pline.data[cur_token.start.col + 2] == ':')); + cur_node->data.opt.ident = ( + pline.data + cur_token.start.col + cur_token.len); + cur_node->data.opt.ident_len = 0; + cur_node->data.opt.scope = ( + cur_token.len == 3 + ? (ExprOptScope)pline.data[cur_token.start.col + 1] + : kExprOptScopeUnspecified); + } else { + cur_node->data.opt.ident = cur_token.data.opt.name; + cur_node->data.opt.ident_len = cur_token.data.opt.len; + cur_node->data.opt.scope = cur_token.data.opt.scope; + } + *top_node_p = cur_node; + want_node = kENodeOperator; + viml_parser_highlight(pstate, cur_token.start, 1, HL(OptionSigil)); + const size_t scope_shift = ( + cur_token.data.opt.scope == kExprOptScopeUnspecified ? 0 : 2); + if (scope_shift) { + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), 1, + HL(OptionScope)); + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 2), 1, + HL(OptionScopeDelimiter)); + } + viml_parser_highlight( + pstate, shifted_pos(cur_token.start, scope_shift + 1), + cur_token.len - (scope_shift + 1), HL(OptionName)); + break; + } + case kExprLexEnv: { + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeEnvironment); + cur_node->data.env.ident = pline.data + cur_token.start.col + 1; + cur_node->data.env.ident_len = cur_token.len - 1; + if (cur_node->data.env.ident_len == 0) { + ERROR_FROM_TOKEN_AND_MSG(cur_token, + _("E15: Environment variable name missing")); + } + *top_node_p = cur_node; + want_node = kENodeOperator; + viml_parser_highlight(pstate, cur_token.start, 1, HL(EnvironmentSigil)); + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), + cur_token.len - 1, HL(EnvironmentName)); + break; + } + case kExprLexNot: { + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeNot); + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + HL_CUR_TOKEN(Not); + break; + } + case kExprLexComparison: { + ADD_VALUE_IF_MISSING( + _("E15: Expected value, got comparison operator: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComparison); + if (cur_token.type == kExprLexInvalid) { + cur_node->data.cmp.ccs = kCCStrategyUseOption; + cur_node->data.cmp.type = kExprCmpEqual; + cur_node->data.cmp.inv = false; + } else { + cur_node->data.cmp.ccs = cur_token.data.cmp.ccs; + cur_node->data.cmp.type = cur_token.data.cmp.type; + cur_node->data.cmp.inv = cur_token.data.cmp.inv; + } + ADD_OP_NODE(cur_node); + if (cur_token.data.cmp.ccs != kCCStrategyUseOption) { + viml_parser_highlight(pstate, cur_token.start, cur_token.len - 1, + HL(Comparison)); + viml_parser_highlight( + pstate, shifted_pos(cur_token.start, cur_token.len - 1), 1, + HL(ComparisonModifier)); + } else { + HL_CUR_TOKEN(Comparison); + } + want_node = kENodeValue; + break; + } + case kExprLexComma: { + assert(!(want_node == kENodeValue && cur_pt == kEPTLambdaArguments)); + if (want_node == kENodeValue) { + // Value level: comma appearing here is not valid. + // Note: in Vim string(,x) will give E116, this is not the case here. + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Expected value, got comma: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeMissing); + cur_node->len = 0; + *top_node_p = cur_node; + want_node = kENodeOperator; + } + if (cur_pt == kEPTLambdaArguments) { + assert(lambda_node != NULL); + assert(lambda_node->data.fig.type_guesses.allow_lambda); + SELECT_FIGURE_BRACE_TYPE(lambda_node, Lambda, Lambda); + } + if (kv_size(ast_stack) < 2) { + goto viml_pexpr_parse_invalid_comma; + } + for (size_t i = 1; i < kv_size(ast_stack); i++) { + ExprASTNode *const *const eastnode_p = + (ExprASTNode *const *)kv_Z(ast_stack, i); + const ExprASTNodeType eastnode_type = (*eastnode_p)->type; + const ExprOpLvl eastnode_lvl = node_lvl(**eastnode_p); + if (eastnode_type == kExprNodeLambda) { + assert(cur_pt == kEPTLambdaArguments + && want_node == kENodeOperator); + break; + } else if (eastnode_type == kExprNodeDictLiteral + || eastnode_type == kExprNodeListLiteral + || eastnode_type == kExprNodeCall) { + break; + } else if (eastnode_type == kExprNodeComma + || eastnode_type == kExprNodeColon + || eastnode_lvl > kEOpLvlComma) { + // Do nothing + } else { +viml_pexpr_parse_invalid_comma: + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Comma outside of call, lambda or literal: %.*s")); + break; + } + if (i == kv_size(ast_stack) - 1) { + goto viml_pexpr_parse_invalid_comma; + } + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComma); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(Comma); + break; + } +#define EXP_VAL_COLON "E15: Expected value, got colon: %.*s" + case kExprLexColon: { + bool is_ternary = false; + if (kv_size(ast_stack) < 2) { + goto viml_pexpr_parse_invalid_colon; + } + bool can_be_ternary = true; + bool is_subscript = false; + for (size_t i = 1; i < kv_size(ast_stack); i++) { + ExprASTNode *const *const eastnode_p = + (ExprASTNode *const *)kv_Z(ast_stack, i); + const ExprASTNodeType eastnode_type = (*eastnode_p)->type; + const ExprOpLvl eastnode_lvl = node_lvl(**eastnode_p); + STATIC_ASSERT(kEOpLvlTernary > kEOpLvlComma, + "Unexpected operator priorities"); + if (can_be_ternary && eastnode_type == kExprNodeTernaryValue + && !(*eastnode_p)->data.ter.got_colon) { + kv_drop(ast_stack, i); + (*eastnode_p)->start = cur_token.start; + (*eastnode_p)->len = cur_token.len; + if (prev_token.type == kExprLexSpacing) { + (*eastnode_p)->start = prev_token.start; + (*eastnode_p)->len += prev_token.len; + } + is_ternary = true; + (*eastnode_p)->data.ter.got_colon = true; + ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); + assert((*eastnode_p)->children != NULL); + assert((*eastnode_p)->children->next == NULL); + kvi_push(ast_stack, &(*eastnode_p)->children->next); + break; + } else if (eastnode_type == kExprNodeUnknownFigure) { + SELECT_FIGURE_BRACE_TYPE(*eastnode_p, DictLiteral, Dict); + break; + } else if (eastnode_type == kExprNodeDictLiteral) { + break; + } else if (eastnode_type == kExprNodeSubscript) { + is_subscript = true; + can_be_ternary = false; + assert(!is_ternary); + break; + } else if (eastnode_type == kExprNodeColon) { + goto viml_pexpr_parse_invalid_colon; + } else if (eastnode_lvl >= kEOpLvlTernaryValue) { + // Do nothing + } else if (eastnode_lvl >= kEOpLvlComma) { + can_be_ternary = false; + } else { + goto viml_pexpr_parse_invalid_colon; + } + if (i == kv_size(ast_stack) - 1) { + goto viml_pexpr_parse_invalid_colon; + } + } + if (is_subscript) { + assert(kv_size(ast_stack) > 1); + // Colon immediately following subscript start: it is empty subscript + // part like a[:2]. + if (want_node == kENodeValue + && (*kv_Z(ast_stack, 1))->type == kExprNodeSubscript) { + NEW_NODE_WITH_CUR_POS(*top_node_p, kExprNodeMissing); + (*top_node_p)->len = 0; + want_node = kENodeOperator; + } else { + ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(SubscriptColon); + } else { + goto viml_pexpr_parse_valid_colon; +viml_pexpr_parse_invalid_colon: + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Colon outside of dictionary or ternary operator: %.*s")); +viml_pexpr_parse_valid_colon: + ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); + if (is_ternary) { + HL_CUR_TOKEN(TernaryColon); + } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(Colon); + } + } + want_node = kENodeValue; + break; + } +#undef EXP_VAL_COLON + case kExprLexBracket: { + if (cur_token.data.brc.closing) { + ExprASTNode **new_top_node_p = NULL; + // Always drop the topmost value: + // + // 1. When want_node != kENodeValue topmost item on stack is + // a *finished* left operand, which may as well be "{@a}" which + // needs not be finished again. + // 2. Otherwise it is pointing to NULL what nobody wants. + kv_drop(ast_stack, 1); + if (!kv_size(ast_stack)) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeListLiteral); + cur_node->len = 0; + if (want_node != kENodeValue) { + cur_node->children = *top_node_p; + } + *top_node_p = cur_node; + goto viml_pexpr_parse_bracket_closing_error; + } + if (want_node == kENodeValue) { + // It is OK to want value if + // + // 1. It is empty list literal, in which case top node will be + // ListLiteral. + // 2. It is list literal with trailing comma, in which case top node + // will be that comma. + // 3. It is subscript with colon, but without one of the values: + // e.g. "a[:]", "a[1:]", top node will be colon in this case. + if ((*kv_last(ast_stack))->type != kExprNodeListLiteral + && (*kv_last(ast_stack))->type != kExprNodeComma + && (*kv_last(ast_stack))->type != kExprNodeColon) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected value, got closing bracket: %.*s")); + } + } else { + if (!kv_size(ast_stack)) { + new_top_node_p = top_node_p; + goto viml_pexpr_parse_bracket_closing_error; + } + } + do { + new_top_node_p = kv_pop(ast_stack); + } while (kv_size(ast_stack) + && (new_top_node_p == NULL + || ((*new_top_node_p)->type != kExprNodeListLiteral + && (*new_top_node_p)->type != kExprNodeSubscript))); + ExprASTNode *new_top_node = *new_top_node_p; + switch (new_top_node->type) { + case kExprNodeListLiteral: { + if (pt_is_assignment(cur_pt) && new_top_node->children == NULL) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E475: Unable to assign to empty list: %.*s")); + } + HL_CUR_TOKEN(List); + break; + } + case kExprNodeSubscript: { + HL_CUR_TOKEN(SubscriptBracket); + break; + } + default: { +viml_pexpr_parse_bracket_closing_error: + assert(!kv_size(ast_stack)); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Unexpected closing figure brace: %.*s")); + HL_CUR_TOKEN(List); + break; + } + } + kvi_push(ast_stack, new_top_node_p); + want_node = kENodeOperator; + if (kv_size(ast_stack) <= asgn_level) { + assert(kv_size(ast_stack) == asgn_level); + asgn_level = 0; + if (cur_pt == kEPTAssignment) { + assert(ast.err.msg); + } else if (cur_pt == kEPTExpr + && kv_size(pt_stack) > 1 + && pt_is_assignment(kv_Z(pt_stack, 1))) { + kv_drop(pt_stack, 1); + } + } + if (cur_pt == kEPTSingleAssignment && kv_size(ast_stack) == 1) { + kv_drop(pt_stack, 1); + } + } else { + if (want_node == kENodeValue) { + // Value means list literal or list assignment. + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeListLiteral); + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + want_node = kENodeValue; + if (cur_pt == kEPTAssignment) { + // Additional assignment parse type allows to easily forbid nested + // lists. + kvi_push(pt_stack, kEPTSingleAssignment); + } else if (cur_pt == kEPTSingleAssignment) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E475: Nested lists not allowed when assigning: %.*s")); + } + HL_CUR_TOKEN(List); + } else { + // Operator means subscript, also in assignment. But in assignment + // subscript may be pretty much any expression, so need to push + // kEPTExpr. + if (prev_token.type == kExprLexSpacing) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeSubscript); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(SubscriptBracket); + if (pt_is_assignment(cur_pt)) { + assert(want_node == kENodeValue); // Subtract 1 for NULL at top. + asgn_level = kv_size(ast_stack) - 1; + kvi_push(pt_stack, kEPTExpr); + } + } + } + break; + } + case kExprLexFigureBrace: { + if (cur_token.data.brc.closing) { + ExprASTNode **new_top_node_p = NULL; + // Always drop the topmost value: + // + // 1. When want_node != kENodeValue topmost item on stack is + // a *finished* left operand, which may as well be "{@a}" which + // needs not be finished again. + // 2. Otherwise it is pointing to NULL what nobody wants. + kv_drop(ast_stack, 1); + if (!kv_size(ast_stack)) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnknownFigure); + cur_node->data.fig.type_guesses.allow_lambda = false; + cur_node->data.fig.type_guesses.allow_dict = false; + cur_node->data.fig.type_guesses.allow_ident = false; + cur_node->len = 0; + if (want_node != kENodeValue) { + cur_node->children = *top_node_p; + } + *top_node_p = cur_node; + new_top_node_p = top_node_p; + goto viml_pexpr_parse_figure_brace_closing_error; + } + if (want_node == kENodeValue) { + if ((*kv_last(ast_stack))->type != kExprNodeUnknownFigure + && (*kv_last(ast_stack))->type != kExprNodeComma) { + // kv_last being UnknownFigure may occur for empty dictionary + // literal, while Comma is expected in case of non-empty one. + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected value, got closing figure brace: %.*s")); + } + } else { + if (!kv_size(ast_stack)) { + new_top_node_p = top_node_p; + goto viml_pexpr_parse_figure_brace_closing_error; + } + } + do { + new_top_node_p = kv_pop(ast_stack); + } while (kv_size(ast_stack) + && (new_top_node_p == NULL + || ((*new_top_node_p)->type != kExprNodeUnknownFigure + && (*new_top_node_p)->type != kExprNodeDictLiteral + && ((*new_top_node_p)->type + != kExprNodeCurlyBracesIdentifier) + && (*new_top_node_p)->type != kExprNodeLambda))); + ExprASTNode *new_top_node = *new_top_node_p; + switch (new_top_node->type) { + case kExprNodeUnknownFigure: { + if (new_top_node->children == NULL) { + // No children of curly braces node indicates empty dictionary. + assert(want_node == kENodeValue); + assert(new_top_node->data.fig.type_guesses.allow_dict); + SELECT_FIGURE_BRACE_TYPE(new_top_node, DictLiteral, Dict); + HL_CUR_TOKEN(Dict); + } else if (new_top_node->data.fig.type_guesses.allow_ident) { + SELECT_FIGURE_BRACE_TYPE(new_top_node, CurlyBracesIdentifier, + Curly); + HL_CUR_TOKEN(Curly); + } else { + // If by this time type of the node has not already been + // guessed, but it definitely is not a curly braces name then + // it is invalid for sure. + ERROR_FROM_NODE_AND_MSG( + new_top_node, + _("E15: Don't know what figure brace means: %.*s")); + if (pstate->colors) { + // Will reset to NvimInvalidFigureBrace. + kv_A(*pstate->colors, + new_top_node->data.fig.opening_hl_idx).group = ( + HL(FigureBrace)); + } + HL_CUR_TOKEN(FigureBrace); + } + break; + } + case kExprNodeDictLiteral: { + HL_CUR_TOKEN(Dict); + break; + } + case kExprNodeCurlyBracesIdentifier: { + HL_CUR_TOKEN(Curly); + break; + } + case kExprNodeLambda: { + HL_CUR_TOKEN(Lambda); + break; + } + default: { +viml_pexpr_parse_figure_brace_closing_error: + assert(!kv_size(ast_stack)); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Unexpected closing figure brace: %.*s")); + HL_CUR_TOKEN(FigureBrace); + break; + } + } + kvi_push(ast_stack, new_top_node_p); + want_node = kENodeOperator; + if (kv_size(ast_stack) <= asgn_level) { + assert(kv_size(ast_stack) == asgn_level); + if (cur_pt == kEPTExpr + && kv_size(pt_stack) > 1 + && pt_is_assignment(kv_Z(pt_stack, 1))) { + kv_drop(pt_stack, 1); + asgn_level = 0; + } + } + } else { + if (want_node == kENodeValue) { + HL_CUR_TOKEN(FigureBrace); + // Value: may be any of lambda, dictionary literal and curly braces + // name. + + // Though if we are in an assignment this may only be a curly braces + // name. + if (pt_is_assignment(cur_pt)) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeCurlyBracesIdentifier); + cur_node->data.fig.type_guesses.allow_lambda = false; + cur_node->data.fig.type_guesses.allow_dict = false; + cur_node->data.fig.type_guesses.allow_ident = true; + kvi_push(pt_stack, kEPTExpr); + } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnknownFigure); + cur_node->data.fig.type_guesses.allow_lambda = true; + cur_node->data.fig.type_guesses.allow_dict = true; + cur_node->data.fig.type_guesses.allow_ident = true; + } + if (pstate->colors) { + cur_node->data.fig.opening_hl_idx = kv_size(*pstate->colors) - 1; + } + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + kvi_push(pt_stack, kEPTLambdaArguments); + lambda_node = cur_node; + } else { + ADD_IDENT( + do { + NEW_NODE_WITH_CUR_POS(cur_node, + kExprNodeCurlyBracesIdentifier); + cur_node->data.fig.opening_hl_idx = kv_size(*pstate->colors); + cur_node->data.fig.type_guesses.allow_lambda = false; + cur_node->data.fig.type_guesses.allow_dict = false; + cur_node->data.fig.type_guesses.allow_ident = true; + kvi_push(ast_stack, &cur_node->children); + if (pt_is_assignment(cur_pt)) { + kvi_push(pt_stack, kEPTExpr); + } + want_node = kENodeValue; + } while (0), + Curly); + } + if (pt_is_assignment(cur_pt) + && !pt_is_assignment(kv_last(pt_stack))) { + assert(want_node == kENodeValue); // Subtract 1 for NULL at top. + asgn_level = kv_size(ast_stack) - 1; + } + } + break; + } + case kExprLexArrow: { + if (cur_pt == kEPTLambdaArguments) { + kv_drop(pt_stack, 1); + assert(kv_size(pt_stack)); + if (want_node == kENodeValue) { + // Wanting value means trailing comma and NULL at the top of the + // stack. + kv_drop(ast_stack, 1); + } + assert(kv_size(ast_stack) >= 1); + while ((*kv_last(ast_stack))->type != kExprNodeLambda + && (*kv_last(ast_stack))->type != kExprNodeUnknownFigure) { + kv_drop(ast_stack, 1); + } + assert((*kv_last(ast_stack)) == lambda_node); + SELECT_FIGURE_BRACE_TYPE(lambda_node, Lambda, Lambda); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeArrow); + if (lambda_node->children == NULL) { + assert(want_node == kENodeValue); + lambda_node->children = cur_node; + kvi_push(ast_stack, &lambda_node->children); + } else { + assert(lambda_node->children->next == NULL); + lambda_node->children->next = cur_node; + kvi_push(ast_stack, &lambda_node->children->next); + } + kvi_push(ast_stack, &cur_node->children); + lambda_node = NULL; + } else { + // Only first branch is valid. + ADD_VALUE_IF_MISSING(_("E15: Unexpected arrow: %.*s")); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Arrow outside of lambda: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeArrow); + ADD_OP_NODE(cur_node); + } + want_node = kENodeValue; + HL_CUR_TOKEN(Arrow); + break; + } + case kExprLexPlainIdentifier: { + const ExprVarScope scope = (cur_token.type == kExprLexInvalid + ? kExprVarScopeMissing + : cur_token.data.var.scope); + if (want_node == kENodeValue) { + want_node = kENodeOperator; + NEW_NODE_WITH_CUR_POS(cur_node, + (node_is_key + ? kExprNodePlainKey + : kExprNodePlainIdentifier)); + cur_node->data.var.scope = scope; + const size_t scope_shift = (scope == kExprVarScopeMissing ? 0 : 2); + cur_node->data.var.ident = (pline.data + cur_token.start.col + + scope_shift); + cur_node->data.var.ident_len = cur_token.len - scope_shift; + *top_node_p = cur_node; + if (scope_shift) { + assert(!node_is_key); + viml_parser_highlight(pstate, cur_token.start, 1, + HL(IdentifierScope)); + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), 1, + HL(IdentifierScopeDelimiter)); + } + viml_parser_highlight(pstate, shifted_pos(cur_token.start, + scope_shift), + cur_token.len - scope_shift, + (node_is_key + ? HL(IdentifierKey) + : HL(IdentifierName))); + } else { + if (scope == kExprVarScopeMissing) { + ADD_IDENT( + do { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainIdentifier); + cur_node->data.var.scope = scope; + cur_node->data.var.ident = pline.data + cur_token.start.col; + cur_node->data.var.ident_len = cur_token.len; + want_node = kENodeOperator; + } while (0), + IdentifierName); + } else { + OP_MISSING; + } + } + break; + } + case kExprLexNumber: { + if (want_node != kENodeValue) { + OP_MISSING; + } + if (node_is_key) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainKey); + cur_node->data.var.ident = pline.data + cur_token.start.col; + cur_node->data.var.ident_len = cur_token.len; + HL_CUR_TOKEN(IdentifierKey); + } else if (cur_token.data.num.is_float) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeFloat); + cur_node->data.flt.value = cur_token.data.num.val.floating; + HL_CUR_TOKEN(Float); + } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeInteger); + cur_node->data.num.value = cur_token.data.num.val.integer; + const uint8_t prefix_length = base_to_prefix_length[ + cur_token.data.num.base]; + viml_parser_highlight(pstate, cur_token.start, prefix_length, + HL(NumberPrefix)); + viml_parser_highlight( + pstate, shifted_pos(cur_token.start, prefix_length), + cur_token.len - prefix_length, HL(Number)); + } + want_node = kENodeOperator; + *top_node_p = cur_node; + break; + } + case kExprLexDot: { + ADD_VALUE_IF_MISSING(_("E15: Unexpected dot: %.*s")); + if (prev_token.type == kExprLexSpacing) { + if (cur_pt == kEPTAssignment) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Cannot concatenate in assignments: %.*s")); + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeConcat); + HL_CUR_TOKEN(Concat); + } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeConcatOrSubscript); + HL_CUR_TOKEN(ConcatOrSubscript); + } + ADD_OP_NODE(cur_node); + break; + } + case kExprLexParenthesis: { + if (cur_token.data.brc.closing) { + if (want_node == kENodeValue) { + if (kv_size(ast_stack) > 1) { + const ExprASTNode *const prev_top_node = *kv_Z(ast_stack, 1); + if (prev_top_node->type == kExprNodeCall) { + // Function call without arguments, this is not an error. + // But further code does not expect NULL nodes. + kv_drop(ast_stack, 1); + goto viml_pexpr_parse_no_paren_closing_error; + } + } + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Expected value, got parenthesis: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeMissing); + cur_node->len = 0; + *top_node_p = cur_node; + } else { + // Always drop the topmost value: when want_node != kENodeValue + // topmost item on stack is a *finished* left operand, which may as + // well be "(@a)" which needs not be finished again. + kv_drop(ast_stack, 1); + } +viml_pexpr_parse_no_paren_closing_error: {} + ExprASTNode **new_top_node_p = NULL; + while (kv_size(ast_stack) + && (new_top_node_p == NULL + || ((*new_top_node_p)->type != kExprNodeNested + && (*new_top_node_p)->type != kExprNodeCall))) { + new_top_node_p = kv_pop(ast_stack); + } + if (new_top_node_p != NULL + && ((*new_top_node_p)->type == kExprNodeNested + || (*new_top_node_p)->type == kExprNodeCall)) { + if ((*new_top_node_p)->type == kExprNodeNested) { + HL_CUR_TOKEN(NestingParenthesis); + } else { + HL_CUR_TOKEN(CallingParenthesis); + } + } else { + // “Always drop the topmost value” branch has got rid of the single + // value stack had, so there is nothing known to enclose. Correct + // this. + if (new_top_node_p == NULL) { + new_top_node_p = top_node_p; + } + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Unexpected closing parenthesis: %.*s")); + HL_CUR_TOKEN(NestingParenthesis); + cur_node = NEW_NODE(kExprNodeNested); + cur_node->start = cur_token.start; + cur_node->len = 0; + // Unexpected closing parenthesis, assume that it was wanted to + // enclose everything in (). + cur_node->children = *new_top_node_p; + *new_top_node_p = cur_node; + assert(cur_node->next == NULL); + } + kvi_push(ast_stack, new_top_node_p); + want_node = kENodeOperator; + } else { + if (want_node == kENodeValue) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeNested); + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + HL_CUR_TOKEN(NestingParenthesis); + } else if (want_node == kENodeOperator) { + if (prev_token.type == kExprLexSpacing) { + // For some reason "function (args)" is a function call, but + // "(funcref) (args)" is not. AFAIR this somehow involves + // compatibility and Bram was commenting that this is + // intentionally inconsistent and he is not very happy with the + // situation himself. + if ((*top_node_p)->type != kExprNodePlainIdentifier + && (*top_node_p)->type != kExprNodeComplexIdentifier + && (*top_node_p)->type != kExprNodeCurlyBracesIdentifier) { + OP_MISSING; + } + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeCall); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(CallingParenthesis); + } else { + // Currently it is impossible to reach this. + assert(false); + } + want_node = kENodeValue; + } + break; + } + case kExprLexQuestion: { + ADD_VALUE_IF_MISSING(_("E15: Expected value, got question mark: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeTernary); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(Ternary); + ExprASTNode *ter_val_node; + NEW_NODE_WITH_CUR_POS(ter_val_node, kExprNodeTernaryValue); + ter_val_node->data.ter.got_colon = false; + assert(cur_node->children != NULL); + assert(cur_node->children->next == NULL); + assert(kv_last(ast_stack) == &cur_node->children->next); + *kv_last(ast_stack) = ter_val_node; + kvi_push(ast_stack, &ter_val_node->children); + break; + } + case kExprLexDoubleQuotedString: + case kExprLexSingleQuotedString: { + const bool is_double = (tok_type == kExprLexDoubleQuotedString); + if (!cur_token.data.str.closed) { + // It is weird, but Vim has two identical errors messages with + // different error numbers: "E114: Missing quote" and + // "E115: Missing quote". + ERROR_FROM_TOKEN_AND_MSG( + cur_token, (is_double + ? _("E114: Missing double quote: %.*s") + : _("E115: Missing single quote: %.*s"))); + } + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS( + cur_node, (is_double + ? kExprNodeDoubleQuotedString + : kExprNodeSingleQuotedString)); + *top_node_p = cur_node; + parse_quoted_string(pstate, cur_node, cur_token, ast_stack, is_invalid); + want_node = kENodeOperator; + break; + } + case kExprLexAssignment: { + if (cur_pt == kEPTAssignment) { + kv_drop(pt_stack, 1); + } else if (cur_pt == kEPTSingleAssignment) { + kv_drop(pt_stack, 2); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E475: Expected closing bracket to end list assignment " + "lvalue: %.*s")); + } else { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Misplaced assignment: %.*s")); + } + assert(kv_size(pt_stack)); + assert(kv_last(pt_stack) == kEPTExpr); + ADD_VALUE_IF_MISSING(_("E15: Unexpected assignment: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeAssignment); + cur_node->data.ass.type = cur_token.data.ass.type; + switch (cur_token.data.ass.type) { +#define HL_ASGN(asgn, hl) \ + case kExprAsgn##asgn: { HL_CUR_TOKEN(hl); break; } + HL_ASGN(Plain, PlainAssignment) + HL_ASGN(Add, AssignmentWithAddition) + HL_ASGN(Subtract, AssignmentWithSubtraction) + HL_ASGN(Concat, AssignmentWithConcatenation) +#undef HL_ASGN + } + ADD_OP_NODE(cur_node); + break; + } + } +viml_pexpr_parse_cycle_end: + prev_token = cur_token; + highlighted_prev_spacing = false; + viml_parser_advance(pstate, cur_token.len); + } while (true); +viml_pexpr_parse_end: + assert(kv_size(pt_stack)); + assert(kv_size(ast_stack)); + if (want_node == kENodeValue + // Blacklist some parse type entries as their presence means better error + // message in the other branch. + && kv_last(pt_stack) != kEPTLambdaArguments) { + east_set_error(pstate, &ast.err, _("E15: Expected value, got EOC: %.*s"), + pstate->pos); + } else if (kv_size(ast_stack) != 1) { + // Something may be wrong, check whether it really is. + + // Pointer to ast.root must never be dropped, so “!= 1” is expected to be + // the same as “> 1”. + assert(kv_size(ast_stack)); + // Topmost stack item must be a *finished* value, so it must not be + // analyzed. E.g. it may contain an already finished nested expression. + kv_drop(ast_stack, 1); + while (ast.err.msg == NULL && kv_size(ast_stack)) { + const ExprASTNode *const cur_node = (*kv_pop(ast_stack)); + // This should only happen when want_node == kENodeValue. + assert(cur_node != NULL); + // TODO(ZyX-I): Rehighlight as invalid? + switch (cur_node->type) { + case kExprNodeOpMissing: + case kExprNodeMissing: { + // Error should’ve been already reported. + break; + } + case kExprNodeCall: { + east_set_error( + pstate, &ast.err, + _("E116: Missing closing parenthesis for function call: %.*s"), + cur_node->start); + break; + } + case kExprNodeNested: { + east_set_error( + pstate, &ast.err, + _("E110: Missing closing parenthesis for nested expression" + ": %.*s"), + cur_node->start); + break; + } + case kExprNodeListLiteral: { + // For whatever reason "[1" yields "E696: Missing comma in list" error + // in Vim while "[1," yields E697. + east_set_error( + pstate, &ast.err, + _("E697: Missing end of List ']': %.*s"), + cur_node->start); + break; + } + case kExprNodeDictLiteral: { + // Same problem like with list literal with E722 (missing comma) vs + // E723, but additionally just "{" yields only E15. + east_set_error( + pstate, &ast.err, + _("E723: Missing end of Dictionary '}': %.*s"), + cur_node->start); + break; + } + case kExprNodeUnknownFigure: { + east_set_error( + pstate, &ast.err, + _("E15: Missing closing figure brace: %.*s"), + cur_node->start); + break; + } + case kExprNodeLambda: { + east_set_error( + pstate, &ast.err, + _("E15: Missing closing figure brace for lambda: %.*s"), + cur_node->start); + break; + } + case kExprNodeCurlyBracesIdentifier: { + // Until trailing "}" it is impossible to distinguish curly braces + // identifier and dictionary, so it must not appear in the stack like + // this. + assert(false); + } + case kExprNodeInteger: + case kExprNodeFloat: + case kExprNodeSingleQuotedString: + case kExprNodeDoubleQuotedString: + case kExprNodeOption: + case kExprNodeEnvironment: + case kExprNodeRegister: + case kExprNodePlainIdentifier: + case kExprNodePlainKey: { + // These are plain values and not containers, for them it should only + // be possible to show up in the topmost stack element, but it was + // unconditionally popped at the start. + assert(false); + } + case kExprNodeComma: + case kExprNodeColon: + case kExprNodeArrow: { + // It is actually only valid inside something else, but everything + // where one of the above is valid requires to be closed and thus is + // to be caught later. + break; + } + case kExprNodeSubscript: + case kExprNodeConcatOrSubscript: + case kExprNodeComplexIdentifier: + case kExprNodeAssignment: + case kExprNodeMod: + case kExprNodeDivision: + case kExprNodeMultiplication: + case kExprNodeNot: + case kExprNodeAnd: + case kExprNodeOr: + case kExprNodeConcat: + case kExprNodeComparison: + case kExprNodeUnaryMinus: + case kExprNodeUnaryPlus: + case kExprNodeBinaryMinus: + case kExprNodeTernary: + case kExprNodeBinaryPlus: { + // It is OK to see these in the stack. + break; + } + case kExprNodeTernaryValue: { + if (!cur_node->data.ter.got_colon) { + // Actually Vim throws E109 in more cases. + east_set_error( + pstate, &ast.err, _("E109: Missing ':' after '?': %.*s"), + cur_node->start); + } + break; + } + } + } + } + kvi_destroy(ast_stack); + return ast; +} // NOLINT(readability/fn_size) + +#undef NEW_NODE +#undef HL diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h new file mode 100644 index 0000000000..23e172da75 --- /dev/null +++ b/src/nvim/viml/parser/expressions.h @@ -0,0 +1,389 @@ +#ifndef NVIM_VIML_PARSER_EXPRESSIONS_H +#define NVIM_VIML_PARSER_EXPRESSIONS_H + +#include <stddef.h> +#include <stdint.h> +#include <stdbool.h> + +#include "nvim/types.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/eval/typval.h" + +// Defines whether to ignore case: +// == kCCStrategyUseOption +// ==# kCCStrategyMatchCase +// ==? kCCStrategyIgnoreCase +typedef enum { + kCCStrategyUseOption = 0, // 0 for xcalloc + kCCStrategyMatchCase = '#', + kCCStrategyIgnoreCase = '?', +} ExprCaseCompareStrategy; + +/// Lexer token type +typedef enum { + kExprLexInvalid = 0, ///< Invalid token, indicaten an error. + kExprLexMissing, ///< Missing token, for use in parser. + kExprLexSpacing, ///< Spaces, tabs, newlines, etc. + kExprLexEOC, ///< End of command character: NL, |, just end of stream. + + kExprLexQuestion, ///< Question mark, for use in ternary. + kExprLexColon, ///< Colon, for use in ternary. + kExprLexOr, ///< Logical or operator. + kExprLexAnd, ///< Logical and operator. + kExprLexComparison, ///< One of the comparison operators. + kExprLexPlus, ///< Plus sign. + kExprLexMinus, ///< Minus sign. + kExprLexDot, ///< Dot: either concat or subscript, also part of the float. + kExprLexMultiplication, ///< Multiplication, division or modulo operator. + + kExprLexNot, ///< Not: !. + + kExprLexNumber, ///< Integer number literal, or part of a float. + kExprLexSingleQuotedString, ///< Single quoted string literal. + kExprLexDoubleQuotedString, ///< Double quoted string literal. + kExprLexOption, ///< &optionname option value. + kExprLexRegister, ///< @r register value. + kExprLexEnv, ///< Environment $variable value. + kExprLexPlainIdentifier, ///< Identifier without scope: `abc`, `foo#bar`. + + kExprLexBracket, ///< Bracket, either opening or closing. + kExprLexFigureBrace, ///< Figure brace, either opening or closing. + kExprLexParenthesis, ///< Parenthesis, either opening or closing. + kExprLexComma, ///< Comma. + kExprLexArrow, ///< Arrow, like from lambda expressions. + kExprLexAssignment, ///< Assignment: `=` or `{op}=`. + // XXX When modifying this enum you need to also modify eltkn_type_tab in + // expressions.c and tests and, possibly, viml_pexpr_repr_token. +} LexExprTokenType; + +typedef enum { + kExprCmpEqual, ///< Equality, unequality. + kExprCmpMatches, ///< Matches regex, not matches regex. + kExprCmpGreater, ///< `>` or `<=` + kExprCmpGreaterOrEqual, ///< `>=` or `<`. + kExprCmpIdentical, ///< `is` or `isnot` +} ExprComparisonType; + +/// All possible option scopes +typedef enum { + kExprOptScopeUnspecified = 0, + kExprOptScopeGlobal = 'g', + kExprOptScopeLocal = 'l', +} ExprOptScope; + +/// All possible assignment types: `=` and `{op}=`. +typedef enum { + kExprAsgnPlain = 0, ///< Plain assignment: `=`. + kExprAsgnAdd, ///< Assignment augmented with addition: `+=`. + kExprAsgnSubtract, ///< Assignment augmented with subtraction: `-=`. + kExprAsgnConcat, ///< Assignment augmented with concatenation: `.=`. +} ExprAssignmentType; + +#define EXPR_OPT_SCOPE_LIST \ + ((char[]){ kExprOptScopeGlobal, kExprOptScopeLocal }) + +/// All possible variable scopes +typedef enum { + kExprVarScopeMissing = 0, + kExprVarScopeScript = 's', + kExprVarScopeGlobal = 'g', + kExprVarScopeVim = 'v', + kExprVarScopeBuffer = 'b', + kExprVarScopeWindow = 'w', + kExprVarScopeTabpage = 't', + kExprVarScopeLocal = 'l', + kExprVarScopeArguments = 'a', +} ExprVarScope; + +#define EXPR_VAR_SCOPE_LIST \ + ((char[]) { \ + kExprVarScopeScript, kExprVarScopeGlobal, kExprVarScopeVim, \ + kExprVarScopeBuffer, kExprVarScopeWindow, kExprVarScopeTabpage, \ + kExprVarScopeLocal, kExprVarScopeBuffer, kExprVarScopeArguments, \ + }) + +/// Lexer token +typedef struct { + ParserPosition start; + size_t len; + LexExprTokenType type; + union { + struct { + ExprComparisonType type; ///< Comparison type. + ExprCaseCompareStrategy ccs; ///< Case comparison strategy. + bool inv; ///< True if comparison is to be inverted. + } cmp; ///< For kExprLexComparison. + + struct { + enum { + kExprLexMulMul, ///< Real multiplication. + kExprLexMulDiv, ///< Division. + kExprLexMulMod, ///< Modulo. + } type; ///< Multiplication type. + } mul; ///< For kExprLexMultiplication. + + struct { + bool closing; ///< True if bracket/etc is a closing one. + } brc; ///< For brackets/braces/parenthesis. + + struct { + int name; ///< Register name, may be -1 if name not present. + } reg; ///< For kExprLexRegister. + + struct { + bool closed; ///< True if quote was closed. + } str; ///< For kExprLexSingleQuotedString and kExprLexDoubleQuotedString. + + struct { + const char *name; ///< Option name start. + size_t len; ///< Option name length. + ExprOptScope scope; ///< Option scope: &l:, &g: or not specified. + } opt; ///< Option properties. + + struct { + ExprVarScope scope; ///< Scope character or 0 if not present. + bool autoload; ///< Has autoload characters. + } var; ///< For kExprLexPlainIdentifier + + struct { + LexExprTokenType type; ///< Suggested type for parsing incorrect code. + const char *msg; ///< Error message. + } err; ///< For kExprLexInvalid + + struct { + union { + float_T floating; + uvarnumber_T integer; + } val; ///< Number value. + uint8_t base; ///< Base: 2, 8, 10 or 16. + bool is_float; ///< True if number is a floating-point. + } num; ///< For kExprLexNumber + + struct { + ExprAssignmentType type; + } ass; ///< For kExprLexAssignment + } data; ///< Additional data, if needed. +} LexExprToken; + +typedef enum { + /// If set, “pointer” to the current byte in pstate will not be shifted + kELFlagPeek = (1 << 0), + /// Determines whether scope is allowed to come before the identifier + kELFlagForbidScope = (1 << 1), + /// Determines whether floating-point numbers are allowed + /// + /// I.e. whether dot is a decimal point separator or is not a part of + /// a number at all. + kELFlagAllowFloat = (1 << 2), + /// Determines whether `is` and `isnot` are seen as comparison operators + /// + /// If set they are supposed to be just regular identifiers. + kELFlagIsNotCmp = (1 << 3), + /// Determines whether EOC tokens are allowed + /// + /// If set then it will yield Invalid token with E15 in place of EOC one if + /// “EOC” is something like "|". It is fine with emitting EOC at the end of + /// string still, with or without this flag set. + kELFlagForbidEOC = (1 << 4), + // XXX Whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_lexer.c. +} LexExprFlags; + +/// Expression AST node type +typedef enum { + kExprNodeMissing = 0, + kExprNodeOpMissing, + kExprNodeTernary, ///< Ternary operator. + kExprNodeTernaryValue, ///< Ternary operator, colon. + kExprNodeRegister, ///< Register. + kExprNodeSubscript, ///< Subscript. + kExprNodeListLiteral, ///< List literal. + kExprNodeUnaryPlus, + kExprNodeBinaryPlus, + kExprNodeNested, ///< Nested parenthesised expression. + kExprNodeCall, ///< Function call. + /// Plain identifier: simple variable/function name + /// + /// Looks like "string", "g:Foo", etc: consists from a single + /// kExprLexPlainIdentifier token. + kExprNodePlainIdentifier, + /// Plain dictionary key, for use with kExprNodeConcatOrSubscript + kExprNodePlainKey, + /// Complex identifier: variable/function name with curly braces + kExprNodeComplexIdentifier, + /// Figure brace expression which is not yet known + /// + /// May resolve to any of kExprNodeDictLiteral, kExprNodeLambda or + /// kExprNodeCurlyBracesIdentifier. + kExprNodeUnknownFigure, + kExprNodeLambda, ///< Lambda. + kExprNodeDictLiteral, ///< Dictionary literal. + kExprNodeCurlyBracesIdentifier, ///< Part of the curly braces name. + kExprNodeComma, ///< Comma “operator”. + kExprNodeColon, ///< Colon “operator”. + kExprNodeArrow, ///< Arrow “operator”. + kExprNodeComparison, ///< Various comparison operators. + /// Concat operator + /// + /// To be only used in cases when it is known for sure it is not a subscript. + kExprNodeConcat, + /// Concat or subscript operator + /// + /// For cases when it is not obvious whether expression is a concat or + /// a subscript. May only have either number or plain identifier as the second + /// child. To make it easier to avoid curly braces in place of + /// kExprNodePlainIdentifier node kExprNodePlainKey is used. + kExprNodeConcatOrSubscript, + kExprNodeInteger, ///< Integral number. + kExprNodeFloat, ///< Floating-point number. + kExprNodeSingleQuotedString, + kExprNodeDoubleQuotedString, + kExprNodeOr, + kExprNodeAnd, + kExprNodeUnaryMinus, + kExprNodeBinaryMinus, + kExprNodeNot, + kExprNodeMultiplication, + kExprNodeDivision, + kExprNodeMod, + kExprNodeOption, + kExprNodeEnvironment, + kExprNodeAssignment, + // XXX When modifying this list also modify east_node_type_tab both in parser + // and in tests, and you most likely will also have to alter list of + // highlight groups stored in highlight_init_cmdline variable. +} ExprASTNodeType; + +typedef struct expr_ast_node ExprASTNode; + +/// Structure representing one AST node +struct expr_ast_node { + ExprASTNodeType type; ///< Node type. + /// Node children: e.g. for 1 + 2 nodes 1 and 2 will be children of +. + ExprASTNode *children; + /// Next node: e.g. for 1 + 2 child nodes 1 and 2 are put into a single-linked + /// list: `(+)->children` references only node 1, node 2 is in + /// `(+)->children->next`. + ExprASTNode *next; + ParserPosition start; + size_t len; + union { + struct { + int name; ///< Register name, may be -1 if name not present. + } reg; ///< For kExprNodeRegister. + struct { + /// Which nodes UnknownFigure can’t possibly represent. + struct { + /// True if UnknownFigure may actually represent dictionary literal. + bool allow_dict; + /// True if UnknownFigure may actually represent lambda. + bool allow_lambda; + /// True if UnknownFigure may actually be part of curly braces name. + bool allow_ident; + } type_guesses; + /// Highlight chunk index, used for rehighlighting if needed + size_t opening_hl_idx; + } fig; ///< For kExprNodeUnknownFigure. + struct { + ExprVarScope scope; ///< Scope character or 0 if not present. + /// Actual identifier without scope. + /// + /// Points to inside parser reader state. + const char *ident; + size_t ident_len; ///< Actual identifier length. + } var; ///< For kExprNodePlainIdentifier and kExprNodePlainKey. + struct { + bool got_colon; ///< True if colon was seen. + } ter; ///< For kExprNodeTernaryValue. + struct { + ExprComparisonType type; ///< Comparison type. + ExprCaseCompareStrategy ccs; ///< Case comparison strategy. + bool inv; ///< True if comparison is to be inverted. + } cmp; ///< For kExprNodeComparison. + struct { + uvarnumber_T value; + } num; ///< For kExprNodeInteger. + struct { + float_T value; + } flt; ///< For kExprNodeFloat. + struct { + char *value; + size_t size; + } str; ///< For kExprNodeSingleQuotedString and + ///< kExprNodeDoubleQuotedString. + struct { + const char *ident; ///< Option name start. + size_t ident_len; ///< Option name length. + ExprOptScope scope; ///< Option scope: &l:, &g: or not specified. + } opt; ///< For kExprNodeOption. + struct { + const char *ident; ///< Environment variable name start. + size_t ident_len; ///< Environment variable name length. + } env; ///< For kExprNodeEnvironment. + struct { + ExprAssignmentType type; + } ass; ///< For kExprNodeAssignment + } data; +}; + +enum { + /// Allow multiple expressions in a row: e.g. for :echo + /// + /// Parser will still parse only one of them though. + kExprFlagsMulti = (1 << 0), + /// Allow NL, NUL and bar to be EOC + /// + /// When parsing expressions input by user bar is assumed to be a binary + /// operator and other two are spacings. + kExprFlagsDisallowEOC = (1 << 1), + /// Parse :let argument + /// + /// That mean that top level node must be an assignment and first nodes + /// belong to lvalues. + kExprFlagsParseLet = (1 << 2), + // XXX whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_parser.c, nvim_parse_expression() flags parsing + // alongside with its documentation and flag sets in check_parsing() + // function in expressions parser functional and unit tests. +} ExprParserFlags; + +/// AST error definition +typedef struct { + /// Error message. Must contain a single printf format atom: %.*s. + const char *msg; + /// Error message argument: points to the location of the error. + const char *arg; + /// Message argument length: length till the end of string. + int arg_len; +} ExprASTError; + +/// Structure representing complety AST for one expression +typedef struct { + /// When AST is not correct this message will be printed. + /// + /// Uses `emsgf(msg, arg_len, arg);`, `msg` is assumed to contain only `%.*s`. + ExprASTError err; + /// Root node of the AST. + ExprASTNode *root; +} ExprAST; + +/// Array mapping ExprASTNodeType to maximum amount of children node may have +extern const uint8_t node_maxchildren[]; + +/// Array mapping ExprASTNodeType values to their stringified versions +extern const char *const east_node_type_tab[]; + +/// Array mapping ExprComparisonType values to their stringified versions +extern const char *const eltkn_cmp_type_tab[]; + +/// Array mapping ExprCaseCompareStrategy values to their stringified versions +extern const char *const ccs_tab[]; + +/// Array mapping ExprAssignmentType values to their stringified versions +extern const char *const expr_asgn_type_tab[]; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/expressions.h.generated.h" +#endif + +#endif // NVIM_VIML_PARSER_EXPRESSIONS_H diff --git a/src/nvim/viml/parser/parser.c b/src/nvim/viml/parser/parser.c new file mode 100644 index 0000000000..08d8846018 --- /dev/null +++ b/src/nvim/viml/parser/parser.c @@ -0,0 +1,13 @@ +#include "nvim/viml/parser/parser.h" + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/parser.c.generated.h" +#endif + + +void parser_simple_get_line(void *cookie, ParserLine *ret_pline) +{ + ParserLine **plines_p = (ParserLine **)cookie; + *ret_pline = **plines_p; + (*plines_p)++; +} diff --git a/src/nvim/viml/parser/parser.h b/src/nvim/viml/parser/parser.h new file mode 100644 index 0000000000..7ac49709d8 --- /dev/null +++ b/src/nvim/viml/parser/parser.h @@ -0,0 +1,244 @@ +#ifndef NVIM_VIML_PARSER_PARSER_H +#define NVIM_VIML_PARSER_PARSER_H + +#include <stdbool.h> +#include <stddef.h> +#include <assert.h> + +#include "nvim/lib/kvec.h" +#include "nvim/func_attr.h" +#include "nvim/mbyte.h" +#include "nvim/memory.h" + +/// One parsed line +typedef struct { + const char *data; ///< Parsed line pointer + size_t size; ///< Parsed line size + bool allocated; ///< True if line may be freed. +} ParserLine; + +/// Line getter type for parser +/// +/// Line getter must return {NULL, 0} for EOF. +typedef void (*ParserLineGetter)(void *cookie, ParserLine *ret_pline); + +/// Parser position in the input +typedef struct { + size_t line; ///< Line index in ParserInputReader.lines. + size_t col; ///< Byte index in the line. +} ParserPosition; + +/// Parser state item. +typedef struct { + enum { + kPTopStateParsingCommand = 0, + kPTopStateParsingExpression, + } type; + union { + struct { + enum { + kExprUnknown = 0, + } type; + } expr; + } data; +} ParserStateItem; + +/// Structure defining input reader +typedef struct { + /// Function used to get next line. + ParserLineGetter get_line; + /// Data for get_line function. + void *cookie; + /// All lines obtained by get_line. + kvec_withinit_t(ParserLine, 4) lines; + /// Conversion, for :scriptencoding. + vimconv_T conv; +} ParserInputReader; + +/// Highlighted region definition +/// +/// Note: one chunk may highlight only one line. +typedef struct { + ParserPosition start; ///< Start of the highlight: line and column. + size_t end_col; ///< End column, points to the start of the next character. + const char *group; ///< Highlight group. +} ParserHighlightChunk; + +/// Highlighting defined by a parser +typedef kvec_withinit_t(ParserHighlightChunk, 16) ParserHighlight; + +/// Structure defining parser state +typedef struct { + /// Line reader. + ParserInputReader reader; + /// Position up to which input was parsed. + ParserPosition pos; + /// Parser state stack. + kvec_withinit_t(ParserStateItem, 16) stack; + /// Highlighting support. + ParserHighlight *colors; + /// True if line continuation can be used. + bool can_continuate; +} ParserState; + +static inline void viml_parser_init( + ParserState *const ret_pstate, + const ParserLineGetter get_line, void *const cookie, + ParserHighlight *const colors) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ARG(1, 2); + +/// Initialize a new parser state instance +/// +/// @param[out] ret_pstate Parser state to initialize. +/// @param[in] get_line Line getter function. +/// @param[in] cookie Argument for the get_line function. +/// @param[in] colors Where to save highlighting. May be NULL if it is not +/// needed. +static inline void viml_parser_init( + ParserState *const ret_pstate, + const ParserLineGetter get_line, void *const cookie, + ParserHighlight *const colors) +{ + *ret_pstate = (ParserState) { + .reader = { + .get_line = get_line, + .cookie = cookie, + .conv = MBYTE_NONE_CONV, + }, + .pos = { 0, 0 }, + .colors = colors, + .can_continuate = false, + }; + kvi_init(ret_pstate->reader.lines); + kvi_init(ret_pstate->stack); +} + +static inline void viml_parser_destroy(ParserState *const pstate) + REAL_FATTR_NONNULL_ALL REAL_FATTR_ALWAYS_INLINE; + +/// Free all memory allocated by the parser on heap +/// +/// @param pstate Parser state to free. +static inline void viml_parser_destroy(ParserState *const pstate) +{ + for (size_t i = 0; i < kv_size(pstate->reader.lines); i++) { + ParserLine pline = kv_A(pstate->reader.lines, i); + if (pline.allocated) { + xfree((void *)pline.data); + } + } + kvi_destroy(pstate->reader.lines); + kvi_destroy(pstate->stack); +} + +static inline void viml_preader_get_line(ParserInputReader *const preader, + ParserLine *const ret_pline) + REAL_FATTR_NONNULL_ALL; + +/// Get one line from ParserInputReader +static inline void viml_preader_get_line(ParserInputReader *const preader, + ParserLine *const ret_pline) +{ + ParserLine pline; + preader->get_line(preader->cookie, &pline); + if (preader->conv.vc_type != CONV_NONE && pline.size) { + ParserLine cpline = { + .allocated = true, + .size = pline.size, + }; + cpline.data = (char *)string_convert(&preader->conv, + (char_u *)pline.data, + &cpline.size); + if (pline.allocated) { + xfree((void *)pline.data); + } + pline = cpline; + } + kvi_push(preader->lines, pline); + *ret_pline = pline; +} + +static inline bool viml_parser_get_remaining_line(ParserState *const pstate, + ParserLine *const ret_pline) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_WARN_UNUSED_RESULT REAL_FATTR_NONNULL_ALL; + +/// Get currently parsed line, shifted to pstate->pos.col +/// +/// @param pstate Parser state to operate on. +/// +/// @return True if there is a line, false in case of EOF. +static inline bool viml_parser_get_remaining_line(ParserState *const pstate, + ParserLine *const ret_pline) +{ + const size_t num_lines = kv_size(pstate->reader.lines); + if (pstate->pos.line == num_lines) { + viml_preader_get_line(&pstate->reader, ret_pline); + } else { + *ret_pline = kv_last(pstate->reader.lines); + } + assert(pstate->pos.line == kv_size(pstate->reader.lines) - 1); + if (ret_pline->data != NULL) { + ret_pline->data += pstate->pos.col; + ret_pline->size -= pstate->pos.col; + } + return ret_pline->data != NULL; +} + +static inline void viml_parser_advance(ParserState *const pstate, + const size_t len) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ALL; + +/// Advance position by a given number of bytes +/// +/// At maximum advances to the next line. +/// +/// @param pstate Parser state to advance. +/// @param[in] len Number of bytes to advance. +static inline void viml_parser_advance(ParserState *const pstate, + const size_t len) +{ + assert(pstate->pos.line == kv_size(pstate->reader.lines) - 1); + const ParserLine pline = kv_last(pstate->reader.lines); + if (pstate->pos.col + len >= pline.size) { + pstate->pos.line++; + pstate->pos.col = 0; + } else { + pstate->pos.col += len; + } +} + +static inline void viml_parser_highlight(ParserState *const pstate, + const ParserPosition start, + const size_t end_col, + const char *const group) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ALL; + +/// Record highlighting of some region of text +/// +/// @param pstate Parser state to work with. +/// @param[in] start Start position of the highlight. +/// @param[in] len Highlighting chunk length. +/// @param[in] group Highlight group. +static inline void viml_parser_highlight(ParserState *const pstate, + const ParserPosition start, + const size_t len, + const char *const group) +{ + if (pstate->colors == NULL || len == 0) { + return; + } + assert(kv_size(*pstate->colors) == 0 + || kv_Z(*pstate->colors, 0).start.line < start.line + || kv_Z(*pstate->colors, 0).end_col <= start.col); + kvi_push(*pstate->colors, ((ParserHighlightChunk) { + .start = start, + .end_col = start.col + len, + .group = group, + })); +} + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/parser.h.generated.h" +#endif + +#endif // NVIM_VIML_PARSER_PARSER_H diff --git a/test/README.md b/test/README.md index 010a2c9c12..1cb814d85b 100644 --- a/test/README.md +++ b/test/README.md @@ -341,3 +341,9 @@ disables tracing (the fastest, but you get no data if tests crash and there was no core dump generated), `1` or empty/undefined leaves only C function cals and returns in the trace (faster then recording everything), `2` records all function calls, returns and lua source lines exuecuted. + +`NVIM_TEST_TRACE_ON_ERROR` (U) (1): makes unit tests yield trace on error in +addition to regular error message. + +`NVIM_TEST_MAXTRACE` (U) (N): specifies maximum number of trace lines to keep. +Default is 1024. diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index b849304d45..ff28e3d133 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -1,5 +1,7 @@ local helpers = require('test.functional.helpers')(after_each) local Screen = require('test.functional.ui.screen') +local global_helpers = require('test.helpers') + local NIL = helpers.NIL local clear, nvim, eq, neq = helpers.clear, helpers.nvim, helpers.eq, helpers.neq local ok, nvim_async, feed = helpers.ok, helpers.nvim_async, helpers.feed @@ -10,6 +12,10 @@ local request = helpers.request local meth_pcall = helpers.meth_pcall local command = helpers.command +local intchar2lua = global_helpers.intchar2lua +local format_string = global_helpers.format_string +local mergedicts_copy = global_helpers.mergedicts_copy + describe('api', function() before_each(clear) @@ -710,4 +716,197 @@ describe('api', function() ok(err:match(': Wrong type for argument 1, expecting String') ~= nil) end) + describe('nvim_parse_expression', function() + before_each(function() + meths.set_option('isident', '') + end) + local function simplify_east_api_node(line, east_api_node) + if east_api_node == NIL then + return nil + end + if east_api_node.children then + for k, v in pairs(east_api_node.children) do + east_api_node.children[k] = simplify_east_api_node(line, v) + end + end + local typ = east_api_node.type + if typ == 'Register' then + typ = typ .. ('(name=%s)'):format( + tostring(intchar2lua(east_api_node.name))) + east_api_node.name = nil + elseif typ == 'PlainIdentifier' then + typ = typ .. ('(scope=%s,ident=%s)'):format( + tostring(intchar2lua(east_api_node.scope)), east_api_node.ident) + east_api_node.scope = nil + east_api_node.ident = nil + elseif typ == 'PlainKey' then + typ = typ .. ('(key=%s)'):format(east_api_node.ident) + east_api_node.ident = nil + elseif typ == 'Comparison' then + typ = typ .. ('(type=%s,inv=%u,ccs=%s)'):format( + east_api_node.cmp_type, east_api_node.invert and 1 or 0, + east_api_node.ccs_strategy) + east_api_node.ccs_strategy = nil + east_api_node.cmp_type = nil + east_api_node.invert = nil + elseif typ == 'Integer' then + typ = typ .. ('(val=%u)'):format(east_api_node.ivalue) + east_api_node.ivalue = nil + elseif typ == 'Float' then + typ = typ .. format_string('(val=%e)', east_api_node.fvalue) + east_api_node.fvalue = nil + elseif typ == 'SingleQuotedString' or typ == 'DoubleQuotedString' then + typ = format_string('%s(val=%q)', typ, east_api_node.svalue) + east_api_node.svalue = nil + elseif typ == 'Option' then + typ = ('%s(scope=%s,ident=%s)'):format( + typ, + tostring(intchar2lua(east_api_node.scope)), + east_api_node.ident) + east_api_node.ident = nil + east_api_node.scope = nil + elseif typ == 'Environment' then + typ = ('%s(ident=%s)'):format(typ, east_api_node.ident) + east_api_node.ident = nil + elseif typ == 'Assignment' then + local aug = east_api_node.augmentation + if aug == '' then aug = 'Plain' end + typ = ('%s(%s)'):format(typ, aug) + east_api_node.augmentation = nil + end + typ = ('%s:%u:%u:%s'):format( + typ, east_api_node.start[1], east_api_node.start[2], + line:sub(east_api_node.start[2] + 1, + east_api_node.start[2] + 1 + east_api_node.len - 1)) + assert(east_api_node.start[2] + east_api_node.len - 1 <= #line) + for k, _ in pairs(east_api_node.start) do + assert(({true, true})[k]) + end + east_api_node.start = nil + east_api_node.type = nil + east_api_node.len = nil + local can_simplify = true + for _, _ in pairs(east_api_node) do + if can_simplify then can_simplify = false end + end + if can_simplify then + return typ + else + east_api_node[1] = typ + return east_api_node + end + end + local function simplify_east_api(line, east_api) + if east_api.error then + east_api.err = east_api.error + east_api.error = nil + east_api.err.msg = east_api.err.message + east_api.err.message = nil + end + if east_api.ast then + east_api.ast = {simplify_east_api_node(line, east_api.ast)} + if #east_api.ast == 0 then + east_api.ast = nil + end + end + if east_api.len == #line then + east_api.len = nil + end + return east_api + end + local function simplify_east_hl(line, east_hl) + for i, v in ipairs(east_hl) do + east_hl[i] = ('%s:%u:%u:%s'):format( + v[4], + v[1], + v[2], + line:sub(v[2] + 1, v[3])) + end + return east_hl + end + local FLAGS_TO_STR = { + [0] = "", + [1] = "m", + [2] = "E", + [3] = "mE", + [4] = "l", + [5] = "lm", + [6] = "lE", + [7] = "lmE", + } + local function _check_parsing(opts, str, exp_ast, exp_highlighting_fs, + nz_flags_exps) + if type(str) ~= 'string' then + return + end + local zflags = opts.flags[1] + nz_flags_exps = nz_flags_exps or {} + for _, flags in ipairs(opts.flags) do + local err, msg = pcall(function() + local east_api = meths.parse_expression(str, FLAGS_TO_STR[flags], true) + local east_hl = east_api.highlight + east_api.highlight = nil + local ast = simplify_east_api(str, east_api) + local hls = simplify_east_hl(str, east_hl) + local exps = { + ast = exp_ast, + hl_fs = exp_highlighting_fs, + } + local add_exps = nz_flags_exps[flags] + if not add_exps and flags == 3 + zflags then + add_exps = nz_flags_exps[1 + zflags] or nz_flags_exps[2 + zflags] + end + if add_exps then + if add_exps.ast then + exps.ast = mergedicts_copy(exps.ast, add_exps.ast) + end + if add_exps.hl_fs then + exps.hl_fs = mergedicts_copy(exps.hl_fs, add_exps.hl_fs) + end + end + eq(exps.ast, ast) + if exp_highlighting_fs then + local exp_highlighting = {} + local next_col = 0 + for i, h in ipairs(exps.hl_fs) do + exp_highlighting[i], next_col = h(next_col) + end + eq(exp_highlighting, hls) + end + end) + if not err then + msg = format_string('Error while processing test (%r, %s):\n%s', + str, FLAGS_TO_STR[flags], msg) + error(msg) + end + end + end + local function hl(group, str, shift) + return function(next_col) + local col = next_col + (shift or 0) + return (('%s:%u:%u:%s'):format( + 'Nvim' .. group, + 0, + col, + str)), (col + #str) + end + end + local function fmtn(typ, args, rest) + if (typ == 'UnknownFigure' + or typ == 'DictLiteral' + or typ == 'CurlyBracesIdentifier' + or typ == 'Lambda') then + return ('%s%s'):format(typ, rest) + elseif typ == 'DoubleQuotedString' or typ == 'SingleQuotedString' then + if args:sub(-4) == 'NULL' then + args = args:sub(1, -5) .. '""' + end + return ('%s(%s)%s'):format(typ, args, rest) + end + end + assert:set_parameter('TableFormatLevel', 1000000) + require('test.unit.viml.expressions.parser_tests')( + it, _check_parsing, hl, fmtn) + end) + end) diff --git a/test/functional/ex_cmds/map_spec.lua b/test/functional/ex_cmds/map_spec.lua new file mode 100644 index 0000000000..b46f83405e --- /dev/null +++ b/test/functional/ex_cmds/map_spec.lua @@ -0,0 +1,21 @@ +local helpers = require("test.functional.helpers")(after_each) + +local eq = helpers.eq +local feed = helpers.feed +local meths = helpers.meths +local clear = helpers.clear +local command = helpers.command + +describe(':*map', function() + before_each(clear) + + it('are not affected by &isident', function() + meths.set_var('counter', 0) + command('nnoremap <C-x> :let counter+=1<CR>') + meths.set_option('isident', ('%u'):format(('>'):byte())) + command('nnoremap <C-y> :let counter+=1<CR>') + -- &isident used to disable keycode parsing here as well + feed('\24\25<C-x><C-y>') + eq(4, meths.get_var('counter')) + end) +end) diff --git a/test/functional/ui/cmdline_highlight_spec.lua b/test/functional/ui/cmdline_highlight_spec.lua index d87ce72599..73fe94c056 100644 --- a/test/functional/ui/cmdline_highlight_spec.lua +++ b/test/functional/ui/cmdline_highlight_spec.lua @@ -144,7 +144,13 @@ before_each(function() EOB={bold = true, foreground = Screen.colors.Blue1}, ERR={foreground = Screen.colors.Grey100, background = Screen.colors.Red}, SK={foreground = Screen.colors.Blue}, - PE={bold = true, foreground = Screen.colors.SeaGreen4} + PE={bold = true, foreground = Screen.colors.SeaGreen4}, + NUM={foreground = Screen.colors.Blue2}, + NPAR={foreground = Screen.colors.Yellow}, + SQ={foreground = Screen.colors.Blue3}, + SB={foreground = Screen.colors.Blue4}, + E={foreground = Screen.colors.Red, background = Screen.colors.Blue}, + M={bold = true}, }) end) @@ -863,7 +869,10 @@ describe('Ex commands coloring support', function() end) describe('Expressions coloring support', function() it('works', function() - meths.set_var('Nvim_color_expr', 'RainBowParens') + meths.command('hi clear NvimNumber') + meths.command('hi clear NvimNestingParenthesis') + meths.command('hi NvimNumber guifg=Blue2') + meths.command('hi NvimNestingParenthesis guifg=Yellow') feed(':echo <C-r>=(((1)))') screen:expect([[ | @@ -873,21 +882,103 @@ describe('Expressions coloring support', function() {EOB:~ }| {EOB:~ }| {EOB:~ }| - ={RBP1:(}{RBP2:(}{RBP3:(}1{RBP3:)}{RBP2:)}{RBP1:)}^ | + ={NPAR:(((}{NUM:1}{NPAR:)))}^ | ]]) end) - it('errors out when failing to get callback', function() + it('does not use Nvim_color_expr', function() meths.set_var('Nvim_color_expr', 42) + -- Used to error out due to failing to get callback. + meths.command('hi clear NvimNumber') + meths.command('hi NvimNumber guifg=Blue2') feed(':<C-r>=1') screen:expect([[ + | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + ={NUM:1}^ | + ]]) + end) + it('works correctly with non-ASCII and control characters', function() + meths.command('hi clear NvimStringBody') + meths.command('hi clear NvimStringQuote') + meths.command('hi clear NvimInvalid') + meths.command('hi NvimStringQuote guifg=Blue3') + meths.command('hi NvimStringBody guifg=Blue4') + meths.command('hi NvimInvalid guifg=Red guibg=Blue') + feed('i<C-r>="«»"«»') + screen:expect([[ + | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + ={SQ:"}{SB:«»}{SQ:"}{E:«»}^ | + ]]) + feed('<C-c>') + screen:expect([[ + ^ | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {M:-- INSERT --} | + ]]) + feed('<Esc>') + screen:expect([[ + ^ | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + | + ]]) + feed(':<C-\\>e"<C-v><C-x>"<C-v><C-x>') + -- TODO(ZyX-I): Parser highlighting should not override special character + -- highlighting. + screen:expect([[ + | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + ={SQ:"}{SB:^X}{SQ:"}{ERR:^X}^ | + ]]) + feed('<C-c>') + screen:expect([[ + | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + :^ | + ]]) + funcs.setreg('a', {'\192'}) + feed('<C-r>="<C-r><C-r>a"<C-r><C-r>a"foo"') + -- TODO(ZyX-I): Parser highlighting should not override special character + -- highlighting. + screen:expect([[ + | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| {EOB:~ }| {EOB:~ }| {EOB:~ }| - = | - {ERR:E5409: Unable to get g:Nvim_color_expr c}| - {ERR:allback: Vim:E6000: Argument is not a fu}| - {ERR:nction or function name} | - =1^ | + ={SQ:"}{SB:<c0>}{SQ:"}{E:<c0>"}{SB:foo}{E:"}^ | ]]) end) end) diff --git a/test/functional/ui/cmdline_spec.lua b/test/functional/ui/cmdline_spec.lua index c9ee61fb1e..d15afc81bb 100644 --- a/test/functional/ui/cmdline_spec.lua +++ b/test/functional/ui/cmdline_spec.lua @@ -239,7 +239,11 @@ describe('external cmdline', function() prompt = "", special = {'"', true}, },{ - content = { { {}, "1+2" } }, + content = { + { {}, "1" }, + { {}, "+" }, + { {}, "2" }, + }, firstc = "=", indent = 0, pos = 3, @@ -316,7 +320,7 @@ describe('external cmdline', function() pos = 0, prompt = "", }}, cmdline) - eq({{{{}, 'function Foo()'}}}, block) + eq({ { { {}, 'function Foo()'} } }, block) end) feed('line1<cr>') @@ -327,8 +331,8 @@ describe('external cmdline', function() {1:~ }| | ]], nil, nil, function() - eq({{{{}, 'function Foo()'}}, - {{{}, ' line1'}}}, block) + eq({ { { {}, 'function Foo()'} }, + { { {}, ' line1'} } }, block) end) block = {} @@ -340,8 +344,8 @@ describe('external cmdline', function() {1:~ }| ^ | ]], nil, nil, function() - eq({{{{}, 'function Foo()'}}, - {{{}, ' line1'}}}, block) + eq({ { { {}, 'function Foo()'} }, + { { {}, ' line1'} } }, block) end) diff --git a/test/helpers.lua b/test/helpers.lua index 123dd111ee..faf5c8e7f2 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -251,6 +251,9 @@ local function which(exe) end local function shallowcopy(orig) + if type(orig) ~= 'table' then + return orig + end local copy = {} for orig_key, orig_value in pairs(orig) do copy[orig_key] = orig_value @@ -258,6 +261,92 @@ local function shallowcopy(orig) return copy end +local deepcopy + +local function id(v) + return v +end + +local deepcopy_funcs = { + table = function(orig) + local copy = {} + for k, v in pairs(orig) do + copy[deepcopy(k)] = deepcopy(v) + end + return copy + end, + number = id, + string = id, + ['nil'] = id, + boolean = id, +} + +deepcopy = function(orig) + return deepcopy_funcs[type(orig)](orig) +end + +local REMOVE_THIS = {} + +local function mergedicts_copy(d1, d2) + local ret = shallowcopy(d1) + for k, v in pairs(d2) do + if d2[k] == REMOVE_THIS then + ret[k] = nil + elseif type(d1[k]) == 'table' and type(v) == 'table' then + ret[k] = mergedicts_copy(d1[k], v) + else + ret[k] = v + end + end + return ret +end + +-- dictdiff: find a diff so that mergedicts_copy(d1, diff) is equal to d2 +-- +-- Note: does not do copies of d2 values used. +local function dictdiff(d1, d2) + local ret = {} + local hasdiff = false + for k, v in pairs(d1) do + if d2[k] == nil then + hasdiff = true + ret[k] = REMOVE_THIS + elseif type(v) == type(d2[k]) then + if type(v) == 'table' then + local subdiff = dictdiff(v, d2[k]) + if subdiff ~= nil then + hasdiff = true + ret[k] = subdiff + end + elseif v ~= d2[k] then + ret[k] = d2[k] + hasdiff = true + end + else + ret[k] = d2[k] + hasdiff = true + end + end + for k, v in pairs(d2) do + if d1[k] == nil then + ret[k] = shallowcopy(v) + hasdiff = true + end + end + if hasdiff then + return ret + else + return nil + end +end + +local function updated(d, d2) + for k, v in pairs(d2) do + d[k] = v + end + return d +end + local function concat_tables(...) local ret = {} for i = 1, select('#', ...) do @@ -294,6 +383,152 @@ local function dedent(str, leave_indent) return str end +local function format_float(v) + -- On windows exponent appears to have three digits and not two + local ret = ('%.6e'):format(v) + local l, f, es, e = ret:match('^(%-?%d)%.(%d+)e([+%-])0*(%d%d+)$') + return l .. '.' .. f .. 'e' .. es .. e +end + +local SUBTBL = { + '\\000', '\\001', '\\002', '\\003', '\\004', + '\\005', '\\006', '\\007', '\\008', '\\t', + '\\n', '\\011', '\\012', '\\r', '\\014', + '\\015', '\\016', '\\017', '\\018', '\\019', + '\\020', '\\021', '\\022', '\\023', '\\024', + '\\025', '\\026', '\\027', '\\028', '\\029', + '\\030', '\\031', +} + +local format_luav + +format_luav = function(v, indent, opts) + opts = opts or {} + local linesep = '\n' + local next_indent_arg = nil + local indent_shift = opts.indent_shift or ' ' + local next_indent + local nl = '\n' + if indent == nil then + indent = '' + linesep = '' + next_indent = '' + nl = ' ' + else + next_indent_arg = indent .. indent_shift + next_indent = indent .. indent_shift + end + local ret = '' + if type(v) == 'string' then + if opts.literal_strings then + ret = v + else + ret = tostring(v):gsub('[\'\\]', '\\%0'):gsub( + '[%z\1-\31]', function(match) + return SUBTBL[match:byte() + 1] + end) + ret = '\'' .. ret .. '\'' + end + elseif type(v) == 'table' then + if v == REMOVE_THIS then + ret = 'REMOVE_THIS' + else + local processed_keys = {} + ret = '{' .. linesep + local non_empty = false + for i, subv in ipairs(v) do + ret = ('%s%s%s,%s'):format(ret, next_indent, + format_luav(subv, next_indent_arg, opts), nl) + processed_keys[i] = true + non_empty = true + end + for k, subv in pairs(v) do + if not processed_keys[k] then + if type(k) == 'string' and k:match('^[a-zA-Z_][a-zA-Z0-9_]*$') then + ret = ret .. next_indent .. k .. ' = ' + else + ret = ('%s%s[%s] = '):format(ret, next_indent, + format_luav(k, nil, opts)) + end + ret = ret .. format_luav(subv, next_indent_arg, opts) .. ',' .. nl + non_empty = true + end + end + if nl == ' ' and non_empty then + ret = ret:sub(1, -3) + end + ret = ret .. indent .. '}' + end + elseif type(v) == 'number' then + if v % 1 == 0 then + ret = ('%d'):format(v) + else + ret = format_float(v) + end + elseif type(v) == 'nil' then + ret = 'nil' + else + print(type(v)) + -- Not implemented yet + assert(false) + end + return ret +end + +local function format_string(fmt, ...) + local i = 0 + local args = {...} + local function getarg() + i = i + 1 + return args[i] + end + local ret = fmt:gsub('%%[0-9*]*%.?[0-9*]*[cdEefgGiouXxqsr%%]', function(match) + local subfmt = match:gsub('%*', function() + return tostring(getarg()) + end) + local arg = nil + if subfmt:sub(-1) ~= '%' then + arg = getarg() + end + if subfmt:sub(-1) == 'r' then + -- %r is like %q, but it is supposed to single-quote strings and not + -- double-quote them, and also work not only for strings. + subfmt = subfmt:sub(1, -2) .. 's' + arg = format_luav(arg) + end + if subfmt == '%e' then + return format_float(arg) + else + return subfmt:format(arg) + end + end) + return ret +end + +local function intchar2lua(ch) + ch = tonumber(ch) + return (20 <= ch and ch < 127) and ('%c'):format(ch) or ch +end + +local fixtbl_metatable = { + __newindex = function() + assert(false) + end, +} + +local function fixtbl(tbl) + return setmetatable(tbl, fixtbl_metatable) +end + +local function fixtbl_rec(tbl) + for _, v in pairs(tbl) do + if type(v) == 'table' then + fixtbl_rec(v) + end + end + return fixtbl(tbl) +end + return { eq = eq, neq = neq, @@ -308,6 +543,16 @@ return { hasenv = hasenv, which = which, shallowcopy = shallowcopy, + deepcopy = deepcopy, + mergedicts_copy = mergedicts_copy, + dictdiff = dictdiff, + REMOVE_THIS = REMOVE_THIS, concat_tables = concat_tables, dedent = dedent, + format_luav = format_luav, + format_string = format_string, + intchar2lua = intchar2lua, + updated = updated, + fixtbl = fixtbl, + fixtbl_rec = fixtbl_rec, } diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c new file mode 100644 index 0000000000..95853a6834 --- /dev/null +++ b/test/symbolic/klee/nvim/charset.c @@ -0,0 +1,172 @@ +#include <stdbool.h> + +#include "nvim/ascii.h" +#include "nvim/macros.h" +#include "nvim/charset.h" +#include "nvim/eval/typval.h" +#include "nvim/vim.h" + +int hex2nr(int c) +{ + if ((c >= 'a') && (c <= 'f')) { + return c - 'a' + 10; + } + + if ((c >= 'A') && (c <= 'F')) { + return c - 'A' + 10; + } + return c - '0'; +} + +void vim_str2nr(const char_u *const start, int *const prep, int *const len, + const int what, varnumber_T *const nptr, + uvarnumber_T *const unptr, const int maxlen) +{ + const char *ptr = (const char *)start; +#define STRING_ENDED(ptr) \ + (!(maxlen == 0 || (int)((ptr) - (const char *)start) < maxlen)) + int pre = 0; // default is decimal + const bool negative = (ptr[0] == '-'); + uvarnumber_T un = 0; + + if (negative) { + ptr++; + } + + if (what & STR2NR_FORCE) { + // When forcing main consideration is skipping the prefix. Octal and decimal + // numbers have no prefixes to skip. pre is not set. + switch ((unsigned)what & (~(unsigned)STR2NR_FORCE)) { + case STR2NR_HEX: { + if (!STRING_ENDED(ptr + 2) + && ptr[0] == '0' + && (ptr[1] == 'x' || ptr[1] == 'X') + && ascii_isxdigit(ptr[2])) { + ptr += 2; + } + goto vim_str2nr_hex; + } + case STR2NR_BIN: { + if (!STRING_ENDED(ptr + 2) + && ptr[0] == '0' + && (ptr[1] == 'b' || ptr[1] == 'B') + && ascii_isbdigit(ptr[2])) { + ptr += 2; + } + goto vim_str2nr_bin; + } + case STR2NR_OCT: { + goto vim_str2nr_oct; + } + case 0: { + goto vim_str2nr_dec; + } + default: { + assert(false); + } + } + } else if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) + && !STRING_ENDED(ptr + 1) + && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { + pre = ptr[1]; + // Detect hexadecimal: 0x or 0X follwed by hex digit + if ((what & STR2NR_HEX) + && !STRING_ENDED(ptr + 2) + && (pre == 'X' || pre == 'x') + && ascii_isxdigit(ptr[2])) { + ptr += 2; + goto vim_str2nr_hex; + } + // Detect binary: 0b or 0B follwed by 0 or 1 + if ((what & STR2NR_BIN) + && !STRING_ENDED(ptr + 2) + && (pre == 'B' || pre == 'b') + && ascii_isbdigit(ptr[2])) { + ptr += 2; + goto vim_str2nr_bin; + } + // Detect octal number: zero followed by octal digits without '8' or '9' + pre = 0; + if (!(what & STR2NR_OCT)) { + goto vim_str2nr_dec; + } + for (int i = 2; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { + if (ptr[i] > '7') { + goto vim_str2nr_dec; + } + } + pre = '0'; + goto vim_str2nr_oct; + } else { + goto vim_str2nr_dec; + } + + // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. + assert(false); // Should’ve used goto earlier. +#define PARSE_NUMBER(base, cond, conv) \ + do { \ + while (!STRING_ENDED(ptr) && (cond)) { \ + /* avoid ubsan error for overflow */ \ + if (un < UVARNUMBER_MAX / base) { \ + un = base * un + (uvarnumber_T)(conv); \ + } else { \ + un = UVARNUMBER_MAX; \ + } \ + ptr++; \ + } \ + } while (0) + switch (pre) { + case 'b': + case 'B': { +vim_str2nr_bin: + PARSE_NUMBER(2, (*ptr == '0' || *ptr == '1'), (*ptr - '0')); + break; + } + case '0': { +vim_str2nr_oct: + PARSE_NUMBER(8, ('0' <= *ptr && *ptr <= '7'), (*ptr - '0')); + break; + } + case 0: { +vim_str2nr_dec: + PARSE_NUMBER(10, (ascii_isdigit(*ptr)), (*ptr - '0')); + break; + } + case 'x': + case 'X': { +vim_str2nr_hex: + PARSE_NUMBER(16, (ascii_isxdigit(*ptr)), (hex2nr(*ptr))); + break; + } + } +#undef PARSE_NUMBER + + if (prep != NULL) { + *prep = pre; + } + + if (len != NULL) { + *len = (int)(ptr - (const char *)start); + } + + if (nptr != NULL) { + if (negative) { // account for leading '-' for decimal numbers + // avoid ubsan error for overflow + if (un > VARNUMBER_MAX) { + *nptr = VARNUMBER_MIN; + } else { + *nptr = -(varnumber_T)un; + } + } else { + if (un > VARNUMBER_MAX) { + un = VARNUMBER_MAX; + } + *nptr = (varnumber_T)un; + } + } + + if (unptr != NULL) { + *unptr = un; + } +#undef STRING_ENDED +} diff --git a/test/symbolic/klee/nvim/garray.c b/test/symbolic/klee/nvim/garray.c new file mode 100644 index 0000000000..260870c3c2 --- /dev/null +++ b/test/symbolic/klee/nvim/garray.c @@ -0,0 +1,195 @@ +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +/// @file garray.c +/// +/// Functions for handling growing arrays. + +#include <string.h> +#include <inttypes.h> + +#include "nvim/vim.h" +#include "nvim/ascii.h" +#include "nvim/log.h" +#include "nvim/memory.h" +#include "nvim/path.h" +#include "nvim/garray.h" +#include "nvim/strings.h" + +// #include "nvim/globals.h" +#include "nvim/memline.h" + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "garray.c.generated.h" +#endif + +/// Clear an allocated growing array. +void ga_clear(garray_T *gap) +{ + xfree(gap->ga_data); + + // Initialize growing array without resetting itemsize or growsize + gap->ga_data = NULL; + gap->ga_maxlen = 0; + gap->ga_len = 0; +} + +/// Clear a growing array that contains a list of strings. +/// +/// @param gap +void ga_clear_strings(garray_T *gap) +{ + GA_DEEP_CLEAR_PTR(gap); +} + +/// Initialize a growing array. +/// +/// @param gap +/// @param itemsize +/// @param growsize +void ga_init(garray_T *gap, int itemsize, int growsize) +{ + gap->ga_data = NULL; + gap->ga_maxlen = 0; + gap->ga_len = 0; + gap->ga_itemsize = itemsize; + ga_set_growsize(gap, growsize); +} + +/// A setter for the growsize that guarantees it will be at least 1. +/// +/// @param gap +/// @param growsize +void ga_set_growsize(garray_T *gap, int growsize) +{ + if (growsize < 1) { + WLOG("trying to set an invalid ga_growsize: %d", growsize); + gap->ga_growsize = 1; + } else { + gap->ga_growsize = growsize; + } +} + +/// Make room in growing array "gap" for at least "n" items. +/// +/// @param gap +/// @param n +void ga_grow(garray_T *gap, int n) +{ + if (gap->ga_maxlen - gap->ga_len >= n) { + // the garray still has enough space, do nothing + return; + } + + if (gap->ga_growsize < 1) { + WLOG("ga_growsize(%d) is less than 1", gap->ga_growsize); + } + + // the garray grows by at least growsize + if (n < gap->ga_growsize) { + n = gap->ga_growsize; + } + int new_maxlen = gap->ga_len + n; + + size_t new_size = (size_t)(gap->ga_itemsize * new_maxlen); + size_t old_size = (size_t)(gap->ga_itemsize * gap->ga_maxlen); + + // reallocate and clear the new memory + char *pp = xrealloc(gap->ga_data, new_size); + memset(pp + old_size, 0, new_size - old_size); + + gap->ga_maxlen = new_maxlen; + gap->ga_data = pp; +} + +/// For a growing array that contains a list of strings: concatenate all the +/// strings with sep as separator. +/// +/// @param gap +/// @param sep +/// +/// @returns the concatenated strings +char_u *ga_concat_strings_sep(const garray_T *gap, const char *sep) + FUNC_ATTR_NONNULL_RET +{ + const size_t nelem = (size_t) gap->ga_len; + const char **strings = gap->ga_data; + + if (nelem == 0) { + return (char_u *) xstrdup(""); + } + + size_t len = 0; + for (size_t i = 0; i < nelem; i++) { + len += strlen(strings[i]); + } + + // add some space for the (num - 1) separators + len += (nelem - 1) * strlen(sep); + char *const ret = xmallocz(len); + + char *s = ret; + for (size_t i = 0; i < nelem - 1; i++) { + s = xstpcpy(s, strings[i]); + s = xstpcpy(s, sep); + } + strcpy(s, strings[nelem - 1]); + + return (char_u *) ret; +} + +/// For a growing array that contains a list of strings: concatenate all the +/// strings with a separating comma. +/// +/// @param gap +/// +/// @returns the concatenated strings +char_u* ga_concat_strings(const garray_T *gap) FUNC_ATTR_NONNULL_RET +{ + return ga_concat_strings_sep(gap, ","); +} + +/// Concatenate a string to a growarray which contains characters. +/// When "s" is NULL does not do anything. +/// +/// WARNING: +/// - Does NOT copy the NUL at the end! +/// - The parameter may not overlap with the growing array +/// +/// @param gap +/// @param s +void ga_concat(garray_T *gap, const char_u *restrict s) +{ + if (s == NULL) { + return; + } + + ga_concat_len(gap, (const char *restrict) s, strlen((char *) s)); +} + +/// Concatenate a string to a growarray which contains characters +/// +/// @param[out] gap Growarray to modify. +/// @param[in] s String to concatenate. +/// @param[in] len String length. +void ga_concat_len(garray_T *const gap, const char *restrict s, + const size_t len) + FUNC_ATTR_NONNULL_ALL +{ + if (len) { + ga_grow(gap, (int) len); + char *data = gap->ga_data; + memcpy(data + gap->ga_len, s, len); + gap->ga_len += (int) len; + } +} + +/// Append one byte to a growarray which contains bytes. +/// +/// @param gap +/// @param c +void ga_append(garray_T *gap, char c) +{ + GA_APPEND(char, gap, c); +} + diff --git a/test/symbolic/klee/nvim/gettext.c b/test/symbolic/klee/nvim/gettext.c new file mode 100644 index 0000000000..b9cc98d770 --- /dev/null +++ b/test/symbolic/klee/nvim/gettext.c @@ -0,0 +1,4 @@ +char *gettext(const char *s) +{ + return (char *)s; +} diff --git a/test/symbolic/klee/nvim/keymap.c b/test/symbolic/klee/nvim/keymap.c new file mode 100644 index 0000000000..a341a73689 --- /dev/null +++ b/test/symbolic/klee/nvim/keymap.c @@ -0,0 +1,539 @@ +#include <stdbool.h> + +#include "nvim/types.h" +#include "nvim/keymap.h" +#include "nvim/ascii.h" +#include "nvim/eval/typval.h" + +#define MOD_KEYS_ENTRY_SIZE 5 + +static char_u modifier_keys_table[] = +{ + MOD_MASK_SHIFT, '&', '9', '@', '1', + MOD_MASK_SHIFT, '&', '0', '@', '2', + MOD_MASK_SHIFT, '*', '1', '@', '4', + MOD_MASK_SHIFT, '*', '2', '@', '5', + MOD_MASK_SHIFT, '*', '3', '@', '6', + MOD_MASK_SHIFT, '*', '4', 'k', 'D', + MOD_MASK_SHIFT, '*', '5', 'k', 'L', + MOD_MASK_SHIFT, '*', '7', '@', '7', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_END, '@', '7', + MOD_MASK_SHIFT, '*', '9', '@', '9', + MOD_MASK_SHIFT, '*', '0', '@', '0', + MOD_MASK_SHIFT, '#', '1', '%', '1', + MOD_MASK_SHIFT, '#', '2', 'k', 'h', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_HOME, 'k', 'h', + MOD_MASK_SHIFT, '#', '3', 'k', 'I', + MOD_MASK_SHIFT, '#', '4', 'k', 'l', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_LEFT, 'k', 'l', + MOD_MASK_SHIFT, '%', 'a', '%', '3', + MOD_MASK_SHIFT, '%', 'b', '%', '4', + MOD_MASK_SHIFT, '%', 'c', '%', '5', + MOD_MASK_SHIFT, '%', 'd', '%', '7', + MOD_MASK_SHIFT, '%', 'e', '%', '8', + MOD_MASK_SHIFT, '%', 'f', '%', '9', + MOD_MASK_SHIFT, '%', 'g', '%', '0', + MOD_MASK_SHIFT, '%', 'h', '&', '3', + MOD_MASK_SHIFT, '%', 'i', 'k', 'r', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_RIGHT, 'k', 'r', + MOD_MASK_SHIFT, '%', 'j', '&', '5', + MOD_MASK_SHIFT, '!', '1', '&', '6', + MOD_MASK_SHIFT, '!', '2', '&', '7', + MOD_MASK_SHIFT, '!', '3', '&', '8', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_UP, 'k', 'u', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_DOWN, 'k', 'd', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF1, KS_EXTRA, (int)KE_XF1, + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF2, KS_EXTRA, (int)KE_XF2, + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF3, KS_EXTRA, (int)KE_XF3, + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF4, KS_EXTRA, (int)KE_XF4, + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F1, 'k', '1', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F2, 'k', '2', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F3, 'k', '3', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F4, 'k', '4', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F5, 'k', '5', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F6, 'k', '6', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F7, 'k', '7', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F8, 'k', '8', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F9, 'k', '9', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F10, 'k', ';', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F11, 'F', '1', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F12, 'F', '2', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F13, 'F', '3', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F14, 'F', '4', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F15, 'F', '5', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F16, 'F', '6', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F17, 'F', '7', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F18, 'F', '8', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F19, 'F', '9', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F20, 'F', 'A', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F21, 'F', 'B', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F22, 'F', 'C', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F23, 'F', 'D', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F24, 'F', 'E', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F25, 'F', 'F', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F26, 'F', 'G', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F27, 'F', 'H', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F28, 'F', 'I', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F29, 'F', 'J', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F30, 'F', 'K', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F31, 'F', 'L', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F32, 'F', 'M', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F33, 'F', 'N', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F34, 'F', 'O', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F35, 'F', 'P', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F36, 'F', 'Q', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F37, 'F', 'R', + + MOD_MASK_SHIFT, 'k', 'B', KS_EXTRA, (int)KE_TAB, + + NUL +}; + +int simplify_key(const int key, int *modifiers) +{ + if (*modifiers & (MOD_MASK_SHIFT | MOD_MASK_CTRL | MOD_MASK_ALT)) { + // TAB is a special case. + if (key == TAB && (*modifiers & MOD_MASK_SHIFT)) { + *modifiers &= ~MOD_MASK_SHIFT; + return K_S_TAB; + } + const int key0 = KEY2TERMCAP0(key); + const int key1 = KEY2TERMCAP1(key); + for (int i = 0; modifier_keys_table[i] != NUL; i += MOD_KEYS_ENTRY_SIZE) { + if (key0 == modifier_keys_table[i + 3] + && key1 == modifier_keys_table[i + 4] + && (*modifiers & modifier_keys_table[i])) { + *modifiers &= ~modifier_keys_table[i]; + return TERMCAP2KEY(modifier_keys_table[i + 1], + modifier_keys_table[i + 2]); + } + } + } + return key; +} + +int handle_x_keys(const int key) + FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + switch (key) { + case K_XUP: return K_UP; + case K_XDOWN: return K_DOWN; + case K_XLEFT: return K_LEFT; + case K_XRIGHT: return K_RIGHT; + case K_XHOME: return K_HOME; + case K_ZHOME: return K_HOME; + case K_XEND: return K_END; + case K_ZEND: return K_END; + case K_XF1: return K_F1; + case K_XF2: return K_F2; + case K_XF3: return K_F3; + case K_XF4: return K_F4; + case K_S_XF1: return K_S_F1; + case K_S_XF2: return K_S_F2; + case K_S_XF3: return K_S_F3; + case K_S_XF4: return K_S_F4; + } + return key; +} + +static const struct key_name_entry { + int key; // Special key code or ascii value + const char *name; // Name of key +} key_names_table[] = { + { ' ', "Space" }, + { TAB, "Tab" }, + { K_TAB, "Tab" }, + { NL, "NL" }, + { NL, "NewLine" }, // Alternative name + { NL, "LineFeed" }, // Alternative name + { NL, "LF" }, // Alternative name + { CAR, "CR" }, + { CAR, "Return" }, // Alternative name + { CAR, "Enter" }, // Alternative name + { K_BS, "BS" }, + { K_BS, "BackSpace" }, // Alternative name + { ESC, "Esc" }, + { CSI, "CSI" }, + { K_CSI, "xCSI" }, + { '|', "Bar" }, + { '\\', "Bslash" }, + { K_DEL, "Del" }, + { K_DEL, "Delete" }, // Alternative name + { K_KDEL, "kDel" }, + { K_UP, "Up" }, + { K_DOWN, "Down" }, + { K_LEFT, "Left" }, + { K_RIGHT, "Right" }, + { K_XUP, "xUp" }, + { K_XDOWN, "xDown" }, + { K_XLEFT, "xLeft" }, + { K_XRIGHT, "xRight" }, + + { K_F1, "F1" }, + { K_F2, "F2" }, + { K_F3, "F3" }, + { K_F4, "F4" }, + { K_F5, "F5" }, + { K_F6, "F6" }, + { K_F7, "F7" }, + { K_F8, "F8" }, + { K_F9, "F9" }, + { K_F10, "F10" }, + + { K_F11, "F11" }, + { K_F12, "F12" }, + { K_F13, "F13" }, + { K_F14, "F14" }, + { K_F15, "F15" }, + { K_F16, "F16" }, + { K_F17, "F17" }, + { K_F18, "F18" }, + { K_F19, "F19" }, + { K_F20, "F20" }, + + { K_F21, "F21" }, + { K_F22, "F22" }, + { K_F23, "F23" }, + { K_F24, "F24" }, + { K_F25, "F25" }, + { K_F26, "F26" }, + { K_F27, "F27" }, + { K_F28, "F28" }, + { K_F29, "F29" }, + { K_F30, "F30" }, + + { K_F31, "F31" }, + { K_F32, "F32" }, + { K_F33, "F33" }, + { K_F34, "F34" }, + { K_F35, "F35" }, + { K_F36, "F36" }, + { K_F37, "F37" }, + + { K_XF1, "xF1" }, + { K_XF2, "xF2" }, + { K_XF3, "xF3" }, + { K_XF4, "xF4" }, + + { K_HELP, "Help" }, + { K_UNDO, "Undo" }, + { K_INS, "Insert" }, + { K_INS, "Ins" }, // Alternative name + { K_KINS, "kInsert" }, + { K_HOME, "Home" }, + { K_KHOME, "kHome" }, + { K_XHOME, "xHome" }, + { K_ZHOME, "zHome" }, + { K_END, "End" }, + { K_KEND, "kEnd" }, + { K_XEND, "xEnd" }, + { K_ZEND, "zEnd" }, + { K_PAGEUP, "PageUp" }, + { K_PAGEDOWN, "PageDown" }, + { K_KPAGEUP, "kPageUp" }, + { K_KPAGEDOWN, "kPageDown" }, + + { K_KPLUS, "kPlus" }, + { K_KMINUS, "kMinus" }, + { K_KDIVIDE, "kDivide" }, + { K_KMULTIPLY, "kMultiply" }, + { K_KENTER, "kEnter" }, + { K_KPOINT, "kPoint" }, + + { K_K0, "k0" }, + { K_K1, "k1" }, + { K_K2, "k2" }, + { K_K3, "k3" }, + { K_K4, "k4" }, + { K_K5, "k5" }, + { K_K6, "k6" }, + { K_K7, "k7" }, + { K_K8, "k8" }, + { K_K9, "k9" }, + + { '<', "lt" }, + + { K_MOUSE, "Mouse" }, + { K_LEFTMOUSE, "LeftMouse" }, + { K_LEFTMOUSE_NM, "LeftMouseNM" }, + { K_LEFTDRAG, "LeftDrag" }, + { K_LEFTRELEASE, "LeftRelease" }, + { K_LEFTRELEASE_NM, "LeftReleaseNM" }, + { K_MIDDLEMOUSE, "MiddleMouse" }, + { K_MIDDLEDRAG, "MiddleDrag" }, + { K_MIDDLERELEASE, "MiddleRelease" }, + { K_RIGHTMOUSE, "RightMouse" }, + { K_RIGHTDRAG, "RightDrag" }, + { K_RIGHTRELEASE, "RightRelease" }, + { K_MOUSEDOWN, "ScrollWheelUp" }, + { K_MOUSEUP, "ScrollWheelDown" }, + { K_MOUSELEFT, "ScrollWheelRight" }, + { K_MOUSERIGHT, "ScrollWheelLeft" }, + { K_MOUSEDOWN, "MouseDown" }, // OBSOLETE: Use + { K_MOUSEUP, "MouseUp" }, // ScrollWheelXXX instead + { K_X1MOUSE, "X1Mouse" }, + { K_X1DRAG, "X1Drag" }, + { K_X1RELEASE, "X1Release" }, + { K_X2MOUSE, "X2Mouse" }, + { K_X2DRAG, "X2Drag" }, + { K_X2RELEASE, "X2Release" }, + { K_DROP, "Drop" }, + { K_ZERO, "Nul" }, + { K_SNR, "SNR" }, + { K_PLUG, "Plug" }, + { K_PASTE, "Paste" }, + { 0, NULL } +}; + +int get_special_key_code(const char_u *name) +{ + for (int i = 0; key_names_table[i].name != NULL; i++) { + const char *const table_name = key_names_table[i].name; + int j; + for (j = 0; ascii_isident(name[j]) && table_name[j] != NUL; j++) { + if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) { + break; + } + } + if (!ascii_isident(name[j]) && table_name[j] == NUL) { + return key_names_table[i].key; + } + } + + return 0; +} + + +static const struct modmasktable { + short mod_mask; ///< Bit-mask for particular key modifier. + short mod_flag; ///< Bit(s) for particular key modifier. + char_u name; ///< Single letter name of modifier. +} mod_mask_table[] = { + {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'M'}, + {MOD_MASK_META, MOD_MASK_META, (char_u)'T'}, + {MOD_MASK_CTRL, MOD_MASK_CTRL, (char_u)'C'}, + {MOD_MASK_SHIFT, MOD_MASK_SHIFT, (char_u)'S'}, + {MOD_MASK_MULTI_CLICK, MOD_MASK_2CLICK, (char_u)'2'}, + {MOD_MASK_MULTI_CLICK, MOD_MASK_3CLICK, (char_u)'3'}, + {MOD_MASK_MULTI_CLICK, MOD_MASK_4CLICK, (char_u)'4'}, + {MOD_MASK_CMD, MOD_MASK_CMD, (char_u)'D'}, + // 'A' must be the last one + {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'A'}, + {0, 0, NUL} +}; + +int name_to_mod_mask(int c) +{ + c = TOUPPER_ASC(c); + for (size_t i = 0; mod_mask_table[i].mod_mask != 0; i++) { + if (c == mod_mask_table[i].name) { + return mod_mask_table[i].mod_flag; + } + } + return 0; +} + +static int extract_modifiers(int key, int *modp) +{ + int modifiers = *modp; + + if (!(modifiers & MOD_MASK_CMD)) { // Command-key is special + if ((modifiers & MOD_MASK_SHIFT) && ASCII_ISALPHA(key)) { + key = TOUPPER_ASC(key); + modifiers &= ~MOD_MASK_SHIFT; + } + } + if ((modifiers & MOD_MASK_CTRL) + && ((key >= '?' && key <= '_') || ASCII_ISALPHA(key))) { + key = Ctrl_chr(key); + modifiers &= ~MOD_MASK_CTRL; + if (key == 0) { // <C-@> is <Nul> + key = K_ZERO; + } + } + + *modp = modifiers; + return key; +} + +int find_special_key(const char_u **srcp, const size_t src_len, int *const modp, + const bool keycode, const bool keep_x_key, + const bool in_string) +{ + const char_u *last_dash; + const char_u *end_of_name; + const char_u *src; + const char_u *bp; + const char_u *const end = *srcp + src_len - 1; + int modifiers; + int bit; + int key; + uvarnumber_T n; + int l; + + if (src_len == 0) { + return 0; + } + + src = *srcp; + if (src[0] != '<') { + return 0; + } + + // Find end of modifier list + last_dash = src; + for (bp = src + 1; bp <= end && (*bp == '-' || ascii_isident(*bp)); bp++) { + if (*bp == '-') { + last_dash = bp; + if (bp + 1 <= end) { + l = utfc_ptr2len_len(bp + 1, (int)(end - bp) + 1); + // Anything accepted, like <C-?>. + // <C-"> or <M-"> are not special in strings as " is + // the string delimiter. With a backslash it works: <M-\"> + if (end - bp > l && !(in_string && bp[1] == '"') && bp[2] == '>') { + bp += l; + } else if (end - bp > 2 && in_string && bp[1] == '\\' + && bp[2] == '"' && bp[3] == '>') { + bp += 2; + } + } + } + if (end - bp > 3 && bp[0] == 't' && bp[1] == '_') { + bp += 3; // skip t_xx, xx may be '-' or '>' + } else if (end - bp > 4 && STRNICMP(bp, "char-", 5) == 0) { + vim_str2nr(bp + 5, NULL, &l, STR2NR_ALL, NULL, NULL, 0); + bp += l + 5; + break; + } + } + + if (bp <= end && *bp == '>') { // found matching '>' + end_of_name = bp + 1; + + /* Which modifiers are given? */ + modifiers = 0x0; + for (bp = src + 1; bp < last_dash; bp++) { + if (*bp != '-') { + bit = name_to_mod_mask(*bp); + if (bit == 0x0) { + break; // Illegal modifier name + } + modifiers |= bit; + } + } + + // Legal modifier name. + if (bp >= last_dash) { + if (STRNICMP(last_dash + 1, "char-", 5) == 0 + && ascii_isdigit(last_dash[6])) { + // <Char-123> or <Char-033> or <Char-0x33> + vim_str2nr(last_dash + 6, NULL, NULL, STR2NR_ALL, NULL, &n, 0); + key = (int)n; + } else { + int off = 1; + + // Modifier with single letter, or special key name. + if (in_string && last_dash[1] == '\\' && last_dash[2] == '"') { + off = 2; + } + l = mb_ptr2len(last_dash + 1); + if (modifiers != 0 && last_dash[l + 1] == '>') { + key = PTR2CHAR(last_dash + off); + } else { + key = get_special_key_code(last_dash + off); + if (!keep_x_key) { + key = handle_x_keys(key); + } + } + } + + // get_special_key_code() may return NUL for invalid + // special key name. + if (key != NUL) { + // Only use a modifier when there is no special key code that + // includes the modifier. + key = simplify_key(key, &modifiers); + + if (!keycode) { + // don't want keycode, use single byte code + if (key == K_BS) { + key = BS; + } else if (key == K_DEL || key == K_KDEL) { + key = DEL; + } + } + + // Normal Key with modifier: + // Try to make a single byte code (except for Alt/Meta modifiers). + if (!IS_SPECIAL(key)) { + key = extract_modifiers(key, &modifiers); + } + + *modp = modifiers; + *srcp = end_of_name; + return key; + } + } + } + return 0; +} + +char_u *add_char2buf(int c, char_u *s) +{ + char_u temp[MB_MAXBYTES + 1]; + const int len = utf_char2bytes(c, temp); + for (int i = 0; i < len; ++i) { + c = temp[i]; + // Need to escape K_SPECIAL and CSI like in the typeahead buffer. + if (c == K_SPECIAL) { + *s++ = K_SPECIAL; + *s++ = KS_SPECIAL; + *s++ = KE_FILLER; + } else { + *s++ = c; + } + } + return s; +} + +unsigned int trans_special(const char_u **srcp, const size_t src_len, + char_u *const dst, const bool keycode, + const bool in_string) +{ + int modifiers = 0; + int key; + unsigned int dlen = 0; + + key = find_special_key(srcp, src_len, &modifiers, keycode, false, in_string); + if (key == 0) { + return 0; + } + + // Put the appropriate modifier in a string. + if (modifiers != 0) { + dst[dlen++] = K_SPECIAL; + dst[dlen++] = KS_MODIFIER; + dst[dlen++] = (char_u)modifiers; + } + + if (IS_SPECIAL(key)) { + dst[dlen++] = K_SPECIAL; + dst[dlen++] = (char_u)KEY2TERMCAP0(key); + dst[dlen++] = KEY2TERMCAP1(key); + } else if (has_mbyte && !keycode) { + dlen += (unsigned int)(*mb_char2bytes)(key, dst + dlen); + } else if (keycode) { + char_u *after = add_char2buf(key, dst + dlen); + assert(after >= dst && (uintmax_t)(after - dst) <= UINT_MAX); + dlen = (unsigned int)(after - dst); + } else { + dst[dlen++] = (char_u)key; + } + + return dlen; +} diff --git a/test/symbolic/klee/nvim/mbyte.c b/test/symbolic/klee/nvim/mbyte.c new file mode 100644 index 0000000000..f98a531206 --- /dev/null +++ b/test/symbolic/klee/nvim/mbyte.c @@ -0,0 +1,266 @@ +#include <stddef.h> +#include <inttypes.h> +#include <assert.h> +#include <stdbool.h> + +#include "nvim/types.h" +#include "nvim/mbyte.h" +#include "nvim/ascii.h" + +const uint8_t utf8len_tab_zero[] = { + //1 2 3 4 5 6 7 8 9 A B C D E F 0 1 2 3 4 5 6 7 8 9 A B C D E F + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 2 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 4 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 6 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 8 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // A + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // C + 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0, // E +}; + +const uint8_t utf8len_tab[] = { + // ?1 ?2 ?3 ?4 ?5 ?6 ?7 ?8 ?9 ?A ?B ?C ?D ?E ?F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D? + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E? + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, // F? +}; + +int utf_ptr2char(const char_u *const p) +{ + if (p[0] < 0x80) { // Be quick for ASCII. + return p[0]; + } + + const uint8_t len = utf8len_tab_zero[p[0]]; + if (len > 1 && (p[1] & 0xc0) == 0x80) { + if (len == 2) { + return ((p[0] & 0x1f) << 6) + (p[1] & 0x3f); + } + if ((p[2] & 0xc0) == 0x80) { + if (len == 3) { + return (((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + + (p[2] & 0x3f)); + } + if ((p[3] & 0xc0) == 0x80) { + if (len == 4) { + return (((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12) + + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f)); + } + if ((p[4] & 0xc0) == 0x80) { + if (len == 5) { + return (((p[0] & 0x03) << 24) + ((p[1] & 0x3f) << 18) + + ((p[2] & 0x3f) << 12) + ((p[3] & 0x3f) << 6) + + (p[4] & 0x3f)); + } + if ((p[5] & 0xc0) == 0x80 && len == 6) { + return (((p[0] & 0x01) << 30) + ((p[1] & 0x3f) << 24) + + ((p[2] & 0x3f) << 18) + ((p[3] & 0x3f) << 12) + + ((p[4] & 0x3f) << 6) + (p[5] & 0x3f)); + } + } + } + } + } + // Illegal value: just return the first byte. + return p[0]; +} + +bool utf_composinglike(const char_u *p1, const char_u *p2) +{ + return false; +} + +char_u *string_convert(const vimconv_T *conv, char_u *data, size_t *size) +{ + return NULL; +} + +int utf_ptr2len_len(const char_u *p, int size) +{ + int len; + int i; + int m; + + len = utf8len_tab[*p]; + if (len == 1) + return 1; /* NUL, ascii or illegal lead byte */ + if (len > size) + m = size; /* incomplete byte sequence. */ + else + m = len; + for (i = 1; i < m; ++i) + if ((p[i] & 0xc0) != 0x80) + return 1; + return len; +} + +int utfc_ptr2len_len(const char_u *p, int size) +{ + int len; + int prevlen; + + if (size < 1 || *p == NUL) + return 0; + if (p[0] < 0x80 && (size == 1 || p[1] < 0x80)) /* be quick for ASCII */ + return 1; + + /* Skip over first UTF-8 char, stopping at a NUL byte. */ + len = utf_ptr2len_len(p, size); + + /* Check for illegal byte and incomplete byte sequence. */ + if ((len == 1 && p[0] >= 0x80) || len > size) + return 1; + + /* + * Check for composing characters. We can handle only the first six, but + * skip all of them (otherwise the cursor would get stuck). + */ + prevlen = 0; + while (len < size) { + int len_next_char; + + if (p[len] < 0x80) + break; + + /* + * Next character length should not go beyond size to ensure that + * UTF_COMPOSINGLIKE(...) does not read beyond size. + */ + len_next_char = utf_ptr2len_len(p + len, size - len); + if (len_next_char > size - len) + break; + + if (!UTF_COMPOSINGLIKE(p + prevlen, p + len)) + break; + + /* Skip over composing char */ + prevlen = len; + len += len_next_char; + } + return len; +} + +int utf_char2len(const int c) +{ + if (c < 0x80) { + return 1; + } else if (c < 0x800) { + return 2; + } else if (c < 0x10000) { + return 3; + } else if (c < 0x200000) { + return 4; + } else if (c < 0x4000000) { + return 5; + } else { + return 6; + } +} + +int utf_char2bytes(const int c, char_u *const buf) +{ + if (c < 0x80) { // 7 bits + buf[0] = c; + return 1; + } else if (c < 0x800) { // 11 bits + buf[0] = 0xc0 + ((unsigned)c >> 6); + buf[1] = 0x80 + (c & 0x3f); + return 2; + } else if (c < 0x10000) { // 16 bits + buf[0] = 0xe0 + ((unsigned)c >> 12); + buf[1] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[2] = 0x80 + (c & 0x3f); + return 3; + } else if (c < 0x200000) { // 21 bits + buf[0] = 0xf0 + ((unsigned)c >> 18); + buf[1] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[3] = 0x80 + (c & 0x3f); + return 4; + } else if (c < 0x4000000) { // 26 bits + buf[0] = 0xf8 + ((unsigned)c >> 24); + buf[1] = 0x80 + (((unsigned)c >> 18) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[3] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[4] = 0x80 + (c & 0x3f); + return 5; + } else { // 31 bits + buf[0] = 0xfc + ((unsigned)c >> 30); + buf[1] = 0x80 + (((unsigned)c >> 24) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 18) & 0x3f); + buf[3] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[4] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[5] = 0x80 + (c & 0x3f); + return 6; + } +} + +int utf_ptr2len(const char_u *const p) +{ + if (*p == NUL) { + return 0; + } + const int len = utf8len_tab[*p]; + for (int i = 1; i < len; i++) { + if ((p[i] & 0xc0) != 0x80) { + return 1; + } + } + return len; +} + +int utfc_ptr2len(const char_u *const p) +{ + uint8_t b0 = (uint8_t)(*p); + + if (b0 == NUL) { + return 0; + } + if (b0 < 0x80 && p[1] < 0x80) { // be quick for ASCII + return 1; + } + + // Skip over first UTF-8 char, stopping at a NUL byte. + int len = utf_ptr2len(p); + + // Check for illegal byte. + if (len == 1 && b0 >= 0x80) { + return 1; + } + + // Check for composing characters. We can handle only the first six, but + // skip all of them (otherwise the cursor would get stuck). + int prevlen = 0; + for (;;) { + if (p[len] < 0x80 || !UTF_COMPOSINGLIKE(p + prevlen, p + len)) { + return len; + } + + // Skip over composing char. + prevlen = len; + len += utf_ptr2len(p + len); + } +} + +void mb_copy_char(const char_u **fp, char_u **tp) +{ + const size_t l = utfc_ptr2len(*fp); + + memmove(*tp, *fp, (size_t)l); + *tp += l; + *fp += l; +} diff --git a/test/symbolic/klee/nvim/memory.c b/test/symbolic/klee/nvim/memory.c new file mode 100644 index 0000000000..df422cea3e --- /dev/null +++ b/test/symbolic/klee/nvim/memory.c @@ -0,0 +1,101 @@ +#include <stdlib.h> +#include <assert.h> + +#include "nvim/lib/ringbuf.h" + +enum { RB_SIZE = 1024 }; + +typedef struct { + void *ptr; + size_t size; +} AllocRecord; + +RINGBUF_TYPEDEF(AllocRecords, AllocRecord) +RINGBUF_INIT(AllocRecords, arecs, AllocRecord, RINGBUF_DUMMY_FREE) +RINGBUF_STATIC(static, AllocRecords, AllocRecord, arecs, RB_SIZE) + +size_t allocated_memory = 0; +size_t ever_allocated_memory = 0; + +size_t allocated_memory_limit = SIZE_MAX; + +void *xmalloc(const size_t size) +{ + void *ret = malloc(size); + allocated_memory += size; + ever_allocated_memory += size; + assert(allocated_memory <= allocated_memory_limit); + assert(arecs_rb_length(&arecs) < RB_SIZE); + arecs_rb_push(&arecs, (AllocRecord) { + .ptr = ret, + .size = size, + }); + return ret; +} + +void xfree(void *const p) +{ + if (p == NULL) { + return; + } + RINGBUF_FORALL(&arecs, AllocRecord, arec) { + if (arec->ptr == p) { + allocated_memory -= arec->size; + arecs_rb_remove(&arecs, arecs_rb_find_idx(&arecs, arec)); + return; + } + } + assert(false); +} + +void *xrealloc(void *const p, size_t new_size) +{ + void *ret = realloc(p, new_size); + RINGBUF_FORALL(&arecs, AllocRecord, arec) { + if (arec->ptr == p) { + allocated_memory -= arec->size; + allocated_memory += new_size; + if (new_size > arec->size) { + ever_allocated_memory += (new_size - arec->size); + } + arec->ptr = ret; + arec->size = new_size; + return ret; + } + } + assert(false); + return (void *)(intptr_t)1; +} + +char *xstrdup(const char *str) + FUNC_ATTR_MALLOC FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_RET + FUNC_ATTR_NONNULL_ALL +{ + return xmemdupz(str, strlen(str)); +} + +void *xmallocz(size_t size) + FUNC_ATTR_MALLOC FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT +{ + size_t total_size = size + 1; + assert(total_size > size); + + void *ret = xmalloc(total_size); + ((char *)ret)[size] = 0; + + return ret; +} + +char *xstpcpy(char *restrict dst, const char *restrict src) + FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL +{ + const size_t len = strlen(src); + return (char *)memcpy(dst, src, len + 1) + len; +} + +void *xmemdupz(const void *data, size_t len) + FUNC_ATTR_MALLOC FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT + FUNC_ATTR_NONNULL_ALL +{ + return memcpy(xmallocz(len), data, len); +} diff --git a/test/symbolic/klee/run.sh b/test/symbolic/klee/run.sh new file mode 100755 index 0000000000..0234a935b5 --- /dev/null +++ b/test/symbolic/klee/run.sh @@ -0,0 +1,102 @@ +#!/bin/sh + +set -e +set -x +test -z "$POSH_VERSION" && set -u + +PROJECT_SOURCE_DIR=. +PROJECT_BINARY_DIR="$PROJECT_SOURCE_DIR/build" +KLEE_TEST_DIR="$PROJECT_SOURCE_DIR/test/symbolic/klee" +KLEE_BIN_DIR="$PROJECT_BINARY_DIR/klee" +KLEE_OUT_DIR="$KLEE_BIN_DIR/out" + +help() { + echo "Usage:" + echo + echo " $0 -c fname" + echo " $0 fname" + echo " $0 -s" + echo + echo "First form compiles executable out of test/symbolic/klee/{fname}.c." + echo "Compiled executable is placed into build/klee/{fname}. Must first" + echo "successfully compile Neovim in order to generate declarations." + echo + echo "Second form runs KLEE in a docker container using file " + echo "test/symbolic/klee/{fname.c}. Bitcode is placed into build/klee/a.bc," + echo "results are placed into build/klee/out/. The latter is first deleted if" + echo "it exists." + echo + echo "Third form runs ktest-tool which prints errors found by KLEE via " + echo "the same container used to run KLEE." +} + +main() { + local compile= + local print_errs= + if test "$1" = "--help" ; then + help + return + fi + if test "$1" = "-s" ; then + print_errs=1 + shift + elif test "$1" = "-c" ; then + compile=1 + shift + fi + if test -z "$print_errs" ; then + local test="$1" ; shift + fi + + local includes= + includes="$includes -I$KLEE_TEST_DIR" + includes="$includes -I/home/klee/klee_src/include" + includes="$includes -I$PROJECT_SOURCE_DIR/src" + includes="$includes -I$PROJECT_BINARY_DIR/src/nvim/auto" + includes="$includes -I$PROJECT_BINARY_DIR/include" + includes="$includes -I$PROJECT_BINARY_DIR/config" + includes="$includes -I/host-includes" + + local defines= + defines="$defines -DMIN_LOG_LEVEL=9999" + defines="$defines -DINCLUDE_GENERATED_DECLARATIONS" + + test -z "$compile" && defines="$defines -DUSE_KLEE" + + test -d "$KLEE_BIN_DIR" || mkdir -p "$KLEE_BIN_DIR" + + if test -z "$compile" ; then + local line1='cd /image' + if test -z "$print_errs" ; then + test -d "$KLEE_OUT_DIR" && rm -r "$KLEE_OUT_DIR" + + line1="$line1 && $(echo clang \ + $includes $defines \ + -o "$KLEE_BIN_DIR/a.bc" -emit-llvm -g -c \ + "$KLEE_TEST_DIR/$test.c")" + line1="$line1 && klee --libc=uclibc --posix-runtime " + line1="$line1 '--output-dir=$KLEE_OUT_DIR' '$KLEE_BIN_DIR/a.bc'" + fi + local line2="for t in '$KLEE_OUT_DIR'/*.err" + line2="$line2 ; do ktest-tool --write-ints" + line2="$line2 \"\$(printf '%s' \"\$t\" | sed -e 's@\\.[^/]*\$@.ktest@')\"" + line2="$line2 ; done" + printf '%s\n%s\n' "$line1" "$line2" | \ + docker run \ + --volume "$(cd "$PROJECT_SOURCE_DIR" && pwd)":/image \ + --volume "/usr/include":/host-includes \ + --interactive \ + --rm \ + --ulimit='stack=-1:-1' \ + klee/klee \ + /bin/sh -x + else + clang \ + $includes $defines \ + -o "$KLEE_BIN_DIR/$test" \ + -O0 -g \ + "$KLEE_TEST_DIR/$test.c" + fi +} + +main "$@" diff --git a/test/symbolic/klee/viml_expressions_lexer.c b/test/symbolic/klee/viml_expressions_lexer.c new file mode 100644 index 0000000000..ee7dc312e9 --- /dev/null +++ b/test/symbolic/klee/viml_expressions_lexer.c @@ -0,0 +1,105 @@ +#ifdef USE_KLEE +# include <klee/klee.h> +#else +# include <string.h> +# include <stdio.h> +#endif +#include <stddef.h> +#include <stdint.h> +#include <assert.h> + +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/mbyte.h" + +#include "nvim/memory.c" +#include "nvim/mbyte.c" +#include "nvim/charset.c" +#include "nvim/garray.c" +#include "nvim/gettext.c" +#include "nvim/keymap.c" +#include "nvim/viml/parser/expressions.c" + +#define INPUT_SIZE 7 + +uint8_t avoid_optimizing_out; + +void simple_get_line(void *cookie, ParserLine *ret_pline) +{ + ParserLine **plines_p = (ParserLine **)cookie; + *ret_pline = **plines_p; + (*plines_p)++; +} + +int main(const int argc, const char *const *const argv, + const char *const *const environ) +{ + char input[INPUT_SIZE]; + uint8_t shift; + int flags; + avoid_optimizing_out = argc; + +#ifndef USE_KLEE + sscanf(argv[2], "%d", &flags); +#endif + +#ifdef USE_KLEE + klee_make_symbolic(input, sizeof(input), "input"); + klee_make_symbolic(&shift, sizeof(shift), "shift"); + klee_make_symbolic(&flags, sizeof(flags), "flags"); + klee_assume(shift < INPUT_SIZE); + klee_assume(flags <= (kELFlagPeek|kELFlagAllowFloat|kELFlagForbidEOC + |kELFlagForbidScope|kELFlagIsNotCmp)); +#endif + + ParserLine plines[] = { + { +#ifdef USE_KLEE + .data = &input[shift], + .size = sizeof(input) - shift, +#else + .data = (const char *)argv[1], + .size = strlen(argv[1]), +#endif + .allocated = false, + }, + { + .data = NULL, + .size = 0, + .allocated = false, + }, + }; +#ifdef USE_KLEE + assert(plines[0].size <= INPUT_SIZE); + assert((plines[0].data[0] != 5) | (plines[0].data[0] != argc)); +#endif + ParserLine *cur_pline = &plines[0]; + + ParserState pstate = { + .reader = { + .get_line = simple_get_line, + .cookie = &cur_pline, + .lines = KV_INITIAL_VALUE, + .conv.vc_type = CONV_NONE, + }, + .pos = { 0, 0 }, + .colors = NULL, + .can_continuate = false, + }; + kvi_init(pstate.reader.lines); + + allocated_memory_limit = 0; + LexExprToken token = viml_pexpr_next_token(&pstate, flags); + if (flags & kELFlagPeek) { + assert(pstate.pos.line == 0 && pstate.pos.col == 0); + } else { + assert((pstate.pos.line == 0) + ? (pstate.pos.col > 0) + : (pstate.pos.line == 1 && pstate.pos.col == 0)); + } + assert(allocated_memory == 0); + assert(ever_allocated_memory == 0); +#ifndef USE_KLEE + fprintf(stderr, "tkn: %s\n", viml_pexpr_repr_token(&pstate, token, NULL)); +#endif +} diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c new file mode 100644 index 0000000000..9a876ed3fa --- /dev/null +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -0,0 +1,117 @@ +#ifdef USE_KLEE +# include <klee/klee.h> +#else +# include <string.h> +#endif +#include <stddef.h> +#include <stdint.h> +#include <assert.h> + +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/mbyte.h" + +#include "nvim/memory.c" +#include "nvim/mbyte.c" +#include "nvim/charset.c" +#include "nvim/garray.c" +#include "nvim/gettext.c" +#include "nvim/viml/parser/expressions.c" +#include "nvim/keymap.c" + +#define INPUT_SIZE 50 + +uint8_t avoid_optimizing_out; + +void simple_get_line(void *cookie, ParserLine *ret_pline) +{ + ParserLine **plines_p = (ParserLine **)cookie; + *ret_pline = **plines_p; + (*plines_p)++; +} + +int main(const int argc, const char *const *const argv, + const char *const *const environ) +{ + char input[INPUT_SIZE]; + uint8_t shift; + unsigned flags; + const bool peek = false; + avoid_optimizing_out = argc; + +#ifndef USE_KLEE + sscanf(argv[2], "%d", &flags); +#endif + +#ifdef USE_KLEE + klee_make_symbolic(input, sizeof(input), "input"); + klee_make_symbolic(&shift, sizeof(shift), "shift"); + klee_make_symbolic(&flags, sizeof(flags), "flags"); + klee_assume(shift < INPUT_SIZE); + klee_assume( + flags <= (kExprFlagsMulti|kExprFlagsDisallowEOC|kExprFlagsParseLet)); +#endif + + ParserLine plines[] = { + { +#ifdef USE_KLEE + .data = &input[shift], + .size = sizeof(input) - shift, +#else + .data = argv[1], + .size = strlen(argv[1]), +#endif + .allocated = false, + }, + { + .data = NULL, + .size = 0, + .allocated = false, + }, + }; +#ifdef USE_KLEE + assert(plines[0].size <= INPUT_SIZE); + assert((plines[0].data[0] != 5) | (plines[0].data[0] != argc)); +#endif + ParserLine *cur_pline = &plines[0]; + + ParserHighlight colors; + kvi_init(colors); + + ParserState pstate = { + .reader = { + .get_line = simple_get_line, + .cookie = &cur_pline, + .lines = KV_INITIAL_VALUE, + .conv.vc_type = CONV_NONE, + }, + .pos = { 0, 0 }, + .colors = &colors, + .can_continuate = false, + }; + kvi_init(pstate.reader.lines); + + const ExprAST ast = viml_pexpr_parse(&pstate, (int)flags); + assert(ast.root != NULL || ast.err.msg); + if (flags & kExprFlagsParseLet) { + assert(ast.err.msg != NULL + || ast.root->type == kExprNodeAssignment + || (ast.root->type == kExprNodeListLiteral + && ast.root->children != NULL) + || ast.root->type == kExprNodeComplexIdentifier + || ast.root->type == kExprNodeCurlyBracesIdentifier + || ast.root->type == kExprNodePlainIdentifier + || ast.root->type == kExprNodeRegister + || ast.root->type == kExprNodeEnvironment + || ast.root->type == kExprNodeOption + || ast.root->type == kExprNodeSubscript + || ast.root->type == kExprNodeConcatOrSubscript); + } + // Can’t possibly have more highlight tokens then there are bytes in string. + assert(kv_size(colors) <= INPUT_SIZE - shift); + kvi_destroy(colors); + // Not destroying pstate.reader.lines because there is no way it could exceed + // its limits in the current circumstances. + viml_pexpr_free_ast(ast); + assert(allocated_memory == 0); +} diff --git a/test/unit/charset/vim_str2nr_spec.lua b/test/unit/charset/vim_str2nr_spec.lua new file mode 100644 index 0000000000..891e6def09 --- /dev/null +++ b/test/unit/charset/vim_str2nr_spec.lua @@ -0,0 +1,320 @@ +local helpers = require("test.unit.helpers")(after_each) +local bit = require('bit') + +local itp = helpers.gen_itp(it) + +local child_call_once = helpers.child_call_once +local cimport = helpers.cimport +local ffi = helpers.ffi + +local lib = cimport('./src/nvim/charset.h') + +local ARGTYPES + +child_call_once(function() + ARGTYPES = { + num = ffi.typeof('varnumber_T[1]'), + unum = ffi.typeof('uvarnumber_T[1]'), + pre = ffi.typeof('int[1]'), + len = ffi.typeof('int[1]'), + } +end) + +local icnt = -42 +local ucnt = 4242 + +local function arginit(arg) + if arg == 'unum' then + ucnt = ucnt + 1 + return ARGTYPES[arg]({ucnt}) + else + icnt = icnt - 1 + return ARGTYPES[arg]({icnt}) + end +end + +local function argreset(arg, args) + if arg == 'unum' then + ucnt = ucnt + 1 + args[arg][0] = ucnt + else + icnt = icnt - 1 + args[arg][0] = icnt + end +end + +local function test_vim_str2nr(s, what, exp, maxlen) + local bits = {} + for k, _ in pairs(exp) do + bits[#bits + 1] = k + end + maxlen = maxlen or #s + local args = {} + for k, _ in pairs(ARGTYPES) do + args[k] = arginit(k) + end + for case = 0, ((2 ^ (#bits)) - 1) do + local cv = {} + for b = 0, (#bits - 1) do + if bit.band(case, (2 ^ b)) == 0 then + local k = bits[b + 1] + argreset(k, args) + cv[k] = args[k] + end + end + lib.vim_str2nr(s, cv.pre, cv.len, what, cv.num, cv.unum, maxlen) + for cck, ccv in pairs(cv) do + if exp[cck] ~= tonumber(ccv[0]) then + error(('Failed check (%s = %d) in test (s=%s, w=%u, m=%d): %d'):format( + cck, exp[cck], s, tonumber(what), maxlen, tonumber(ccv[0]) + )) + end + end + end +end + +local _itp = itp +itp = function(...) + collectgarbage('restart') + _itp(...) +end + +describe('vim_str2nr()', function() + itp('works fine when it has nothing to do', function() + test_vim_str2nr('', 0, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_ALL, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_BIN, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_OCT, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_HEX, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_FORCE + lib.STR2NR_DEC, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_FORCE + lib.STR2NR_BIN, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_FORCE + lib.STR2NR_OCT, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_FORCE + lib.STR2NR_HEX, {len = 0, num = 0, unum = 0, pre = 0}, 0) + end) + itp('works with decimal numbers', function() + for _, flags in ipairs({ + 0, + lib.STR2NR_BIN, + lib.STR2NR_OCT, + lib.STR2NR_HEX, + lib.STR2NR_BIN + lib.STR2NR_OCT, + lib.STR2NR_BIN + lib.STR2NR_HEX, + lib.STR2NR_OCT + lib.STR2NR_HEX, + lib.STR2NR_ALL, + lib.STR2NR_FORCE + lib.STR2NR_DEC, + }) do + -- Check that all digits are recognized + test_vim_str2nr( '12345', flags, {len = 5, num = 12345, unum = 12345, pre = 0}, 0) + test_vim_str2nr( '67890', flags, {len = 5, num = 67890, unum = 67890, pre = 0}, 0) + test_vim_str2nr( '12345A', flags, {len = 5, num = 12345, unum = 12345, pre = 0}, 0) + test_vim_str2nr( '67890A', flags, {len = 5, num = 67890, unum = 67890, pre = 0}, 0) + + test_vim_str2nr( '42', flags, {len = 2, num = 42, unum = 42, pre = 0}, 0) + test_vim_str2nr( '42', flags, {len = 1, num = 4, unum = 4, pre = 0}, 1) + test_vim_str2nr( '42', flags, {len = 2, num = 42, unum = 42, pre = 0}, 2) + test_vim_str2nr( '42', flags, {len = 2, num = 42, unum = 42, pre = 0}, 3) -- includes NUL byte in maxlen + + test_vim_str2nr( '42x', flags, {len = 2, num = 42, unum = 42, pre = 0}, 0) + test_vim_str2nr( '42x', flags, {len = 2, num = 42, unum = 42, pre = 0}, 3) + + test_vim_str2nr('-42', flags, {len = 3, num = -42, unum = 42, pre = 0}, 3) + test_vim_str2nr('-42', flags, {len = 1, num = 0, unum = 0, pre = 0}, 1) + + test_vim_str2nr('-42x', flags, {len = 3, num = -42, unum = 42, pre = 0}, 0) + test_vim_str2nr('-42x', flags, {len = 3, num = -42, unum = 42, pre = 0}, 4) + end + end) + itp('works with binary numbers', function() + for _, flags in ipairs({ + lib.STR2NR_BIN, + lib.STR2NR_BIN + lib.STR2NR_OCT, + lib.STR2NR_BIN + lib.STR2NR_HEX, + lib.STR2NR_ALL, + lib.STR2NR_FORCE + lib.STR2NR_BIN, + }) do + local bin + local BIN + if flags > lib.STR2NR_FORCE then + bin = 0 + BIN = 0 + else + bin = ('b'):byte() + BIN = ('B'):byte() + end + + test_vim_str2nr( '0b101', flags, {len = 5, num = 5, unum = 5, pre = bin}, 0) + test_vim_str2nr( '0b101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr( '0b101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr( '0b101', flags, {len = 3, num = 1, unum = 1, pre = bin}, 3) + test_vim_str2nr( '0b101', flags, {len = 4, num = 2, unum = 2, pre = bin}, 4) + test_vim_str2nr( '0b101', flags, {len = 5, num = 5, unum = 5, pre = bin}, 5) + test_vim_str2nr( '0b101', flags, {len = 5, num = 5, unum = 5, pre = bin}, 6) + + test_vim_str2nr( '0b1012', flags, {len = 5, num = 5, unum = 5, pre = bin}, 0) + test_vim_str2nr( '0b1012', flags, {len = 5, num = 5, unum = 5, pre = bin}, 6) + + test_vim_str2nr('-0b101', flags, {len = 6, num = -5, unum = 5, pre = bin}, 0) + test_vim_str2nr('-0b101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr('-0b101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr('-0b101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 3) + test_vim_str2nr('-0b101', flags, {len = 4, num = -1, unum = 1, pre = bin}, 4) + test_vim_str2nr('-0b101', flags, {len = 5, num = -2, unum = 2, pre = bin}, 5) + test_vim_str2nr('-0b101', flags, {len = 6, num = -5, unum = 5, pre = bin}, 6) + test_vim_str2nr('-0b101', flags, {len = 6, num = -5, unum = 5, pre = bin}, 7) + + test_vim_str2nr('-0b1012', flags, {len = 6, num = -5, unum = 5, pre = bin}, 0) + test_vim_str2nr('-0b1012', flags, {len = 6, num = -5, unum = 5, pre = bin}, 7) + + test_vim_str2nr( '0B101', flags, {len = 5, num = 5, unum = 5, pre = BIN}, 0) + test_vim_str2nr( '0B101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr( '0B101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr( '0B101', flags, {len = 3, num = 1, unum = 1, pre = BIN}, 3) + test_vim_str2nr( '0B101', flags, {len = 4, num = 2, unum = 2, pre = BIN}, 4) + test_vim_str2nr( '0B101', flags, {len = 5, num = 5, unum = 5, pre = BIN}, 5) + test_vim_str2nr( '0B101', flags, {len = 5, num = 5, unum = 5, pre = BIN}, 6) + + test_vim_str2nr( '0B1012', flags, {len = 5, num = 5, unum = 5, pre = BIN}, 0) + test_vim_str2nr( '0B1012', flags, {len = 5, num = 5, unum = 5, pre = BIN}, 6) + + test_vim_str2nr('-0B101', flags, {len = 6, num = -5, unum = 5, pre = BIN}, 0) + test_vim_str2nr('-0B101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr('-0B101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr('-0B101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 3) + test_vim_str2nr('-0B101', flags, {len = 4, num = -1, unum = 1, pre = BIN}, 4) + test_vim_str2nr('-0B101', flags, {len = 5, num = -2, unum = 2, pre = BIN}, 5) + test_vim_str2nr('-0B101', flags, {len = 6, num = -5, unum = 5, pre = BIN}, 6) + test_vim_str2nr('-0B101', flags, {len = 6, num = -5, unum = 5, pre = BIN}, 7) + + test_vim_str2nr('-0B1012', flags, {len = 6, num = -5, unum = 5, pre = BIN}, 0) + test_vim_str2nr('-0B1012', flags, {len = 6, num = -5, unum = 5, pre = BIN}, 7) + + if flags > lib.STR2NR_FORCE then + test_vim_str2nr('-101', flags, {len = 4, num = -5, unum = 5, pre = 0}, 0) + end + end + end) + itp('works with octal numbers', function() + for _, flags in ipairs({ + lib.STR2NR_OCT, + lib.STR2NR_OCT + lib.STR2NR_BIN, + lib.STR2NR_OCT + lib.STR2NR_HEX, + lib.STR2NR_ALL, + lib.STR2NR_FORCE + lib.STR2NR_OCT, + }) do + local oct + if flags > lib.STR2NR_FORCE then + oct = 0 + else + oct = ('0'):byte() + end + + -- Check that all digits are recognized + test_vim_str2nr( '012345670', flags, {len = 9, num = 2739128, unum = 2739128, pre = oct}, 0) + + test_vim_str2nr( '054', flags, {len = 3, num = 44, unum = 44, pre = oct}, 0) + test_vim_str2nr( '054', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr( '054', flags, {len = 2, num = 5, unum = 5, pre = oct}, 2) + test_vim_str2nr( '054', flags, {len = 3, num = 44, unum = 44, pre = oct}, 3) + test_vim_str2nr( '0548', flags, {len = 3, num = 44, unum = 44, pre = oct}, 3) + test_vim_str2nr( '054', flags, {len = 3, num = 44, unum = 44, pre = oct}, 4) + + test_vim_str2nr( '054x', flags, {len = 3, num = 44, unum = 44, pre = oct}, 4) + test_vim_str2nr( '054x', flags, {len = 3, num = 44, unum = 44, pre = oct}, 0) + + test_vim_str2nr('-054', flags, {len = 4, num = -44, unum = 44, pre = oct}, 0) + test_vim_str2nr('-054', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr('-054', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr('-054', flags, {len = 3, num = -5, unum = 5, pre = oct}, 3) + test_vim_str2nr('-054', flags, {len = 4, num = -44, unum = 44, pre = oct}, 4) + test_vim_str2nr('-0548', flags, {len = 4, num = -44, unum = 44, pre = oct}, 4) + test_vim_str2nr('-054', flags, {len = 4, num = -44, unum = 44, pre = oct}, 5) + + test_vim_str2nr('-054x', flags, {len = 4, num = -44, unum = 44, pre = oct}, 5) + test_vim_str2nr('-054x', flags, {len = 4, num = -44, unum = 44, pre = oct}, 0) + + if flags > lib.STR2NR_FORCE then + test_vim_str2nr('-54', flags, {len = 3, num = -44, unum = 44, pre = 0}, 0) + test_vim_str2nr('-0548', flags, {len = 4, num = -44, unum = 44, pre = 0}, 5) + test_vim_str2nr('-0548', flags, {len = 4, num = -44, unum = 44, pre = 0}, 0) + else + test_vim_str2nr('-0548', flags, {len = 5, num = -548, unum = 548, pre = 0}, 5) + test_vim_str2nr('-0548', flags, {len = 5, num = -548, unum = 548, pre = 0}, 0) + end + end + end) + itp('works with hexadecimal numbers', function() + for _, flags in ipairs({ + lib.STR2NR_HEX, + lib.STR2NR_HEX + lib.STR2NR_BIN, + lib.STR2NR_HEX + lib.STR2NR_OCT, + lib.STR2NR_ALL, + lib.STR2NR_FORCE + lib.STR2NR_HEX, + }) do + local hex + local HEX + if flags > lib.STR2NR_FORCE then + hex = 0 + HEX = 0 + else + hex = ('x'):byte() + HEX = ('X'):byte() + end + + -- Check that all digits are recognized + test_vim_str2nr('0x12345', flags, {len = 7, num = 74565, unum = 74565, pre = hex}, 0) + test_vim_str2nr('0x67890', flags, {len = 7, num = 424080, unum = 424080, pre = hex}, 0) + test_vim_str2nr('0xABCDEF', flags, {len = 8, num = 11259375, unum = 11259375, pre = hex}, 0) + test_vim_str2nr('0xabcdef', flags, {len = 8, num = 11259375, unum = 11259375, pre = hex}, 0) + + test_vim_str2nr( '0x101', flags, {len = 5, num = 257, unum =257, pre = hex}, 0) + test_vim_str2nr( '0x101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr( '0x101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr( '0x101', flags, {len = 3, num = 1, unum = 1, pre = hex}, 3) + test_vim_str2nr( '0x101', flags, {len = 4, num = 16, unum = 16, pre = hex}, 4) + test_vim_str2nr( '0x101', flags, {len = 5, num = 257, unum =257, pre = hex}, 5) + test_vim_str2nr( '0x101', flags, {len = 5, num = 257, unum =257, pre = hex}, 6) + + test_vim_str2nr( '0x101G', flags, {len = 5, num = 257, unum =257, pre = hex}, 0) + test_vim_str2nr( '0x101G', flags, {len = 5, num = 257, unum =257, pre = hex}, 6) + + test_vim_str2nr('-0x101', flags, {len = 6, num =-257, unum =257, pre = hex}, 0) + test_vim_str2nr('-0x101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr('-0x101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr('-0x101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 3) + test_vim_str2nr('-0x101', flags, {len = 4, num = -1, unum = 1, pre = hex}, 4) + test_vim_str2nr('-0x101', flags, {len = 5, num = -16, unum = 16, pre = hex}, 5) + test_vim_str2nr('-0x101', flags, {len = 6, num =-257, unum =257, pre = hex}, 6) + test_vim_str2nr('-0x101', flags, {len = 6, num =-257, unum =257, pre = hex}, 7) + + test_vim_str2nr('-0x101G', flags, {len = 6, num =-257, unum =257, pre = hex}, 0) + test_vim_str2nr('-0x101G', flags, {len = 6, num =-257, unum =257, pre = hex}, 7) + + test_vim_str2nr( '0X101', flags, {len = 5, num = 257, unum =257, pre = HEX}, 0) + test_vim_str2nr( '0X101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr( '0X101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr( '0X101', flags, {len = 3, num = 1, unum = 1, pre = HEX}, 3) + test_vim_str2nr( '0X101', flags, {len = 4, num = 16, unum = 16, pre = HEX}, 4) + test_vim_str2nr( '0X101', flags, {len = 5, num = 257, unum =257, pre = HEX}, 5) + test_vim_str2nr( '0X101', flags, {len = 5, num = 257, unum =257, pre = HEX}, 6) + + test_vim_str2nr( '0X101G', flags, {len = 5, num = 257, unum =257, pre = HEX}, 0) + test_vim_str2nr( '0X101G', flags, {len = 5, num = 257, unum =257, pre = HEX}, 6) + + test_vim_str2nr('-0X101', flags, {len = 6, num =-257, unum =257, pre = HEX}, 0) + test_vim_str2nr('-0X101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr('-0X101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr('-0X101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 3) + test_vim_str2nr('-0X101', flags, {len = 4, num = -1, unum = 1, pre = HEX}, 4) + test_vim_str2nr('-0X101', flags, {len = 5, num = -16, unum = 16, pre = HEX}, 5) + test_vim_str2nr('-0X101', flags, {len = 6, num =-257, unum =257, pre = HEX}, 6) + test_vim_str2nr('-0X101', flags, {len = 6, num =-257, unum =257, pre = HEX}, 7) + + test_vim_str2nr('-0X101G', flags, {len = 6, num =-257, unum =257, pre = HEX}, 0) + test_vim_str2nr('-0X101G', flags, {len = 6, num =-257, unum =257, pre = HEX}, 7) + + if flags > lib.STR2NR_FORCE then + test_vim_str2nr('-101', flags, {len = 4, num = -257, unum = 257, pre = 0}, 0) + end + end + end) +end) diff --git a/test/unit/eval/helpers.lua b/test/unit/eval/helpers.lua index 5bc482216e..d7399182f7 100644 --- a/test/unit/eval/helpers.lua +++ b/test/unit/eval/helpers.lua @@ -1,5 +1,6 @@ local helpers = require('test.unit.helpers')(nil) +local ptr2key = helpers.ptr2key local cimport = helpers.cimport local to_cstr = helpers.to_cstr local ffi = helpers.ffi @@ -91,10 +92,6 @@ local function populate_partial(pt, lua_pt, processed) return pt end -local ptr2key = function(ptr) - return tostring(ptr) -end - local lst2tbl local dct2tbl diff --git a/test/unit/formatc.lua b/test/unit/formatc.lua index e288081960..2fd37c599a 100644 --- a/test/unit/formatc.lua +++ b/test/unit/formatc.lua @@ -65,11 +65,12 @@ local tokens = P { "tokens"; identifier = Ct(C(R("az","AZ","__") * R("09","az","AZ","__")^0) * Cc"identifier"), -- Single character in a string - string_char = R("az","AZ","09") + S"$%^&*()_-+={[}]:;@~#<,>.!?/ \t" + (P"\\" * S[[ntvbrfa\?'"0x]]), + sstring_char = R("\001&","([","]\255") + (P"\\" * S[[ntvbrfa\?'"0x]]), + dstring_char = R("\001!","#[","]\255") + (P"\\" * S[[ntvbrfa\?'"0x]]), -- String literal - string = Ct(C(P"'" * (V"string_char" + P'"')^0 * P"'" + - P'"' * (V"string_char" + P"'")^0 * P'"') * Cc"string"), + string = Ct(C(P"'" * (V"sstring_char" + P'"')^0 * P"'" + + P'"' * (V"dstring_char" + P"'")^0 * P'"') * Cc"string"), -- Operator operator = Ct(C(P">>=" + P"<<=" + P"..." + diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index ac5e394a54..b1e709c444 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -316,7 +316,7 @@ local function alloc_log_new() eq(exp, self.log) self:clear() end - function log:clear_tmp_allocs() + function log:clear_tmp_allocs(clear_null_frees) local toremove = {} local allocs = {} for i, v in ipairs(self.log) do @@ -328,6 +328,8 @@ local function alloc_log_new() if v.func == 'free' then toremove[#toremove + 1] = i end + elseif clear_null_frees and v.args[1] == self.null then + toremove[#toremove + 1] = i end if v.func == 'realloc' then allocs[tostring(v.ret)] = i @@ -528,9 +530,13 @@ local hook_numlen = 5 local hook_msglen = 1 + 1 + 1 + (1 + hook_fnamelen) + (1 + hook_sfnamelen) + (1 + hook_numlen) + 1 local tracehelp = dedent([[ + Trace: either in the format described below or custom debug output starting + with `>`. Latter lines still have the same width in byte. + ┌ Trace type: _r_eturn from function , function _c_all, _l_ine executed, │ _t_ail return, _C_ount (should not actually appear), - │ _s_aved from previous run for reference. + │ _s_aved from previous run for reference, _>_ for custom debug + │ output. │┏ Function type: _L_ua function, _C_ function, _m_ain part of chunk, │┃ function that did _t_ail call. │┃┌ Function name type: _g_lobal, _l_ocal, _m_ethod, _f_ield, _u_pvalue, @@ -628,14 +634,26 @@ end local trace_end_msg = ('E%s\n'):format((' '):rep(hook_msglen - 2)) +local _debug_log + +local debug_log = only_separate(function(...) + return _debug_log(...) +end) + local function itp_child(wr, func) - init() - collectgarbage('stop') - child_sethook(wr) - local err, emsg = pcall(func) - collectgarbage('restart') - collectgarbage() - debug.sethook() + _debug_log = function(s) + s = s:sub(1, hook_msglen - 2) + sc.write(wr, '>' .. s .. (' '):rep(hook_msglen - 2 - #s) .. '\n') + end + local err, emsg = pcall(init) + if err then + collectgarbage('stop') + child_sethook(wr) + err, emsg = pcall(func) + collectgarbage('restart') + collectgarbage() + debug.sethook() + end emsg = tostring(emsg) sc.write(wr, trace_end_msg) if not err then @@ -657,6 +675,7 @@ end local function check_child_err(rd) local trace = {} local did_traceline = false + local maxtrace = tonumber(os.getenv('NVIM_TEST_MAXTRACE')) or 1024 while true do local traceline = sc.read(rd, hook_msglen) if #traceline ~= hook_msglen then @@ -671,6 +690,7 @@ local function check_child_err(rd) break end trace[#trace + 1] = traceline + table.remove(trace, maxtrace + 1) end local res = sc.read(rd, 2) if #res ~= 2 then @@ -699,7 +719,14 @@ local function check_child_err(rd) local len_s = sc.read(rd, 5) local len = tonumber(len_s) neq(0, len) - local err = sc.read(rd, len + 1) + local err = '' + if os.getenv('NVIM_TEST_TRACE_ON_ERROR') == '1' and #trace ~= 0 then + err = '\nTest failed, trace:\n' .. tracehelp + for _, traceline in ipairs(trace) do + err = err .. traceline + end + end + err = err .. sc.read(rd, len + 1) assert.just_fail(err) end @@ -754,6 +781,60 @@ end cimport('./src/nvim/types.h', './src/nvim/main.h', './src/nvim/os/time.h') +local function conv_enum(etab, eval) + local n = tonumber(eval) + return etab[n] or n +end + +local function array_size(arr) + return ffi.sizeof(arr) / ffi.sizeof(arr[0]) +end + +local function kvi_size(kvi) + return array_size(kvi.init_array) +end + +local function kvi_init(kvi) + kvi.capacity = kvi_size(kvi) + kvi.items = kvi.init_array + return kvi +end + +local function kvi_destroy(kvi) + if kvi.items ~= kvi.init_array then + lib.xfree(kvi.items) + end +end + +local function kvi_new(ct) + return kvi_init(ffi.new(ct)) +end + +local function make_enum_conv_tab(m, values, skip_pref, set_cb) + child_call_once(function() + local ret = {} + for _, v in ipairs(values) do + local str_v = v + if v:sub(1, #skip_pref) == skip_pref then + str_v = v:sub(#skip_pref + 1) + end + ret[tonumber(m[v])] = str_v + end + set_cb(ret) + end) +end + +local function ptr2addr(ptr) + return tonumber(ffi.cast('intptr_t', ffi.cast('void *', ptr))) +end + +local s = ffi.new('char[64]', {0}) + +local function ptr2key(ptr) + ffi.C.snprintf(s, ffi.sizeof(s), '%p', ffi.cast('void *', ptr)) + return ffi.string(s) +end + local module = { cimport = cimport, cppimport = cppimport, @@ -774,6 +855,16 @@ local module = { child_call_once = child_call_once, child_cleanup_once = child_cleanup_once, sc = sc, + conv_enum = conv_enum, + array_size = array_size, + kvi_destroy = kvi_destroy, + kvi_size = kvi_size, + kvi_init = kvi_init, + kvi_new = kvi_new, + make_enum_conv_tab = make_enum_conv_tab, + ptr2addr = ptr2addr, + ptr2key = ptr2key, + debug_log = debug_log, } return function() return module diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua new file mode 100644 index 0000000000..1b57a24ad5 --- /dev/null +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -0,0 +1,428 @@ +local helpers = require('test.unit.helpers')(after_each) +local global_helpers = require('test.helpers') +local itp = helpers.gen_itp(it) +local viml_helpers = require('test.unit.viml.helpers') + +local child_call_once = helpers.child_call_once +local conv_enum = helpers.conv_enum +local cimport = helpers.cimport +local ffi = helpers.ffi +local eq = helpers.eq + +local conv_ccs = viml_helpers.conv_ccs +local new_pstate = viml_helpers.new_pstate +local conv_cmp_type = viml_helpers.conv_cmp_type +local pstate_set_str = viml_helpers.pstate_set_str +local conv_expr_asgn_type = viml_helpers.conv_expr_asgn_type + +local shallowcopy = global_helpers.shallowcopy +local intchar2lua = global_helpers.intchar2lua + +local lib = cimport('./src/nvim/viml/parser/expressions.h') + +local eltkn_type_tab, eltkn_mul_type_tab, eltkn_opt_scope_tab +child_call_once(function() + eltkn_type_tab = { + [tonumber(lib.kExprLexInvalid)] = 'Invalid', + [tonumber(lib.kExprLexMissing)] = 'Missing', + [tonumber(lib.kExprLexSpacing)] = 'Spacing', + [tonumber(lib.kExprLexEOC)] = 'EOC', + + [tonumber(lib.kExprLexQuestion)] = 'Question', + [tonumber(lib.kExprLexColon)] = 'Colon', + [tonumber(lib.kExprLexOr)] = 'Or', + [tonumber(lib.kExprLexAnd)] = 'And', + [tonumber(lib.kExprLexComparison)] = 'Comparison', + [tonumber(lib.kExprLexPlus)] = 'Plus', + [tonumber(lib.kExprLexMinus)] = 'Minus', + [tonumber(lib.kExprLexDot)] = 'Dot', + [tonumber(lib.kExprLexMultiplication)] = 'Multiplication', + + [tonumber(lib.kExprLexNot)] = 'Not', + + [tonumber(lib.kExprLexNumber)] = 'Number', + [tonumber(lib.kExprLexSingleQuotedString)] = 'SingleQuotedString', + [tonumber(lib.kExprLexDoubleQuotedString)] = 'DoubleQuotedString', + [tonumber(lib.kExprLexOption)] = 'Option', + [tonumber(lib.kExprLexRegister)] = 'Register', + [tonumber(lib.kExprLexEnv)] = 'Env', + [tonumber(lib.kExprLexPlainIdentifier)] = 'PlainIdentifier', + + [tonumber(lib.kExprLexBracket)] = 'Bracket', + [tonumber(lib.kExprLexFigureBrace)] = 'FigureBrace', + [tonumber(lib.kExprLexParenthesis)] = 'Parenthesis', + [tonumber(lib.kExprLexComma)] = 'Comma', + [tonumber(lib.kExprLexArrow)] = 'Arrow', + + [tonumber(lib.kExprLexAssignment)] = 'Assignment', + } + + eltkn_mul_type_tab = { + [tonumber(lib.kExprLexMulMul)] = 'Mul', + [tonumber(lib.kExprLexMulDiv)] = 'Div', + [tonumber(lib.kExprLexMulMod)] = 'Mod', + } + + eltkn_opt_scope_tab = { + [tonumber(lib.kExprOptScopeUnspecified)] = 'Unspecified', + [tonumber(lib.kExprOptScopeGlobal)] = 'Global', + [tonumber(lib.kExprOptScopeLocal)] = 'Local', + } +end) + +local function conv_eltkn_type(typ) + return conv_enum(eltkn_type_tab, typ) +end + +local bracket_types = { + Bracket = true, + FigureBrace = true, + Parenthesis = true, +} + +local function eltkn2lua(pstate, tkn) + local ret = { + type = conv_eltkn_type(tkn.type), + } + pstate_set_str(pstate, tkn.start, tkn.len, ret) + if not ret.error and (#(ret.str) ~= ret.len) then + ret.error = '#str /= len' + end + if ret.type == 'Comparison' then + ret.data = { + type = conv_cmp_type(tkn.data.cmp.type), + ccs = conv_ccs(tkn.data.cmp.ccs), + inv = (not not tkn.data.cmp.inv), + } + elseif ret.type == 'Multiplication' then + ret.data = { type = conv_enum(eltkn_mul_type_tab, tkn.data.mul.type) } + elseif bracket_types[ret.type] then + ret.data = { closing = (not not tkn.data.brc.closing) } + elseif ret.type == 'Register' then + ret.data = { name = intchar2lua(tkn.data.reg.name) } + elseif (ret.type == 'SingleQuotedString' + or ret.type == 'DoubleQuotedString') then + ret.data = { closed = (not not tkn.data.str.closed) } + elseif ret.type == 'Option' then + ret.data = { + scope = conv_enum(eltkn_opt_scope_tab, tkn.data.opt.scope), + name = ffi.string(tkn.data.opt.name, tkn.data.opt.len), + } + elseif ret.type == 'PlainIdentifier' then + ret.data = { + scope = intchar2lua(tkn.data.var.scope), + autoload = (not not tkn.data.var.autoload), + } + elseif ret.type == 'Number' then + ret.data = { + is_float = (not not tkn.data.num.is_float), + base = tonumber(tkn.data.num.base), + } + ret.data.val = tonumber(tkn.data.num.is_float + and tkn.data.num.val.floating + or tkn.data.num.val.integer) + elseif ret.type == 'Assignment' then + ret.data = { type = conv_expr_asgn_type(tkn.data.ass.type) } + elseif ret.type == 'Invalid' then + ret.data = { error = ffi.string(tkn.data.err.msg) } + end + return ret, tkn +end + +local function next_eltkn(pstate, flags) + return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, flags)) +end + +describe('Expressions lexer', function() + local flags = 0 + local should_advance = true + local function check_advance(pstate, bytes_to_advance, initial_col) + local tgt = initial_col + bytes_to_advance + if should_advance then + if pstate.reader.lines.items[0].size == tgt then + eq(1, pstate.pos.line) + eq(0, pstate.pos.col) + else + eq(0, pstate.pos.line) + eq(tgt, pstate.pos.col) + end + else + eq(0, pstate.pos.line) + eq(initial_col, pstate.pos.col) + end + end + local function singl_eltkn_test(typ, str, data) + local pstate = new_pstate({str}) + eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, + next_eltkn(pstate, flags)) + check_advance(pstate, #str, 0) + if not ( + typ == 'Spacing' + or (typ == 'Register' and str == '@') + or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString') + and not data.closed) + ) then + pstate = new_pstate({str .. ' '}) + eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, + next_eltkn(pstate, flags)) + check_advance(pstate, #str, 0) + end + pstate = new_pstate({'x' .. str}) + pstate.pos.col = 1 + eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ}, + next_eltkn(pstate, flags)) + check_advance(pstate, #str, 1) + end + local function scope_test(scope) + singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope}) + singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope}) + end + local function comparison_test(op, inv_op, cmp_type) + singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'}) + singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'}) + singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'}) + singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'}) + singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'}) + singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'}) + end + local function simple_test(pstate_arg, exp_type, exp_len, exp) + local pstate = new_pstate(pstate_arg) + exp = shallowcopy(exp) + exp.type = exp_type + exp.len = exp_len or #(pstate_arg[0]) + exp.start = { col = 0, line = 0 } + eq(exp, next_eltkn(pstate, flags)) + end + local function stable_tests() + singl_eltkn_test('Parenthesis', '(', {closing=false}) + singl_eltkn_test('Parenthesis', ')', {closing=true}) + singl_eltkn_test('Bracket', '[', {closing=false}) + singl_eltkn_test('Bracket', ']', {closing=true}) + singl_eltkn_test('FigureBrace', '{', {closing=false}) + singl_eltkn_test('FigureBrace', '}', {closing=true}) + singl_eltkn_test('Question', '?') + singl_eltkn_test('Colon', ':') + singl_eltkn_test('Dot', '.') + singl_eltkn_test('Assignment', '.=', {type='Concat'}) + singl_eltkn_test('Plus', '+') + singl_eltkn_test('Assignment', '+=', {type='Add'}) + singl_eltkn_test('Comma', ',') + singl_eltkn_test('Multiplication', '*', {type='Mul'}) + singl_eltkn_test('Multiplication', '/', {type='Div'}) + singl_eltkn_test('Multiplication', '%', {type='Mod'}) + singl_eltkn_test('Spacing', ' \t\t \t\t') + singl_eltkn_test('Spacing', ' ') + singl_eltkn_test('Spacing', '\t') + singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'}) + singl_eltkn_test('Number', '0123', {is_float=false, base=8, val=83}) + singl_eltkn_test('Number', '01234567', {is_float=false, base=8, val=342391}) + singl_eltkn_test('Number', '012345678', {is_float=false, base=10, val=12345678}) + singl_eltkn_test('Number', '0x123', {is_float=false, base=16, val=291}) + singl_eltkn_test('Number', '0x56FF', {is_float=false, base=16, val=22271}) + singl_eltkn_test('Number', '0xabcdef', {is_float=false, base=16, val=11259375}) + singl_eltkn_test('Number', '0xABCDEF', {is_float=false, base=16, val=11259375}) + singl_eltkn_test('Number', '0x0', {is_float=false, base=16, val=0}) + singl_eltkn_test('Number', '00', {is_float=false, base=8, val=0}) + singl_eltkn_test('Number', '0b0', {is_float=false, base=2, val=0}) + singl_eltkn_test('Number', '0b010111', {is_float=false, base=2, val=23}) + singl_eltkn_test('Number', '0b100111', {is_float=false, base=2, val=39}) + singl_eltkn_test('Number', '0', {is_float=false, base=10, val=0}) + singl_eltkn_test('Number', '9', {is_float=false, base=10, val=9}) + singl_eltkn_test('Env', '$abc') + singl_eltkn_test('Env', '$') + singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', '_test', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', '_test_foo', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 't', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 'test5', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 't0', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 'test#var', {autoload=true, scope=0}) + singl_eltkn_test('PlainIdentifier', 'test#var#val###', {autoload=true, scope=0}) + singl_eltkn_test('PlainIdentifier', 't#####', {autoload=true, scope=0}) + singl_eltkn_test('And', '&&') + singl_eltkn_test('Or', '||') + singl_eltkn_test('Invalid', '&', {error='E112: Option name missing: %.*s'}) + singl_eltkn_test('Option', '&opt', {scope='Unspecified', name='opt'}) + singl_eltkn_test('Option', '&t_xx', {scope='Unspecified', name='t_xx'}) + singl_eltkn_test('Option', '&t_\r\r', {scope='Unspecified', name='t_\r\r'}) + singl_eltkn_test('Option', '&t_\t\t', {scope='Unspecified', name='t_\t\t'}) + singl_eltkn_test('Option', '&t_ ', {scope='Unspecified', name='t_ '}) + singl_eltkn_test('Option', '&g:opt', {scope='Global', name='opt'}) + singl_eltkn_test('Option', '&l:opt', {scope='Local', name='opt'}) + singl_eltkn_test('Invalid', '&l:', {error='E112: Option name missing: %.*s'}) + singl_eltkn_test('Invalid', '&g:', {error='E112: Option name missing: %.*s'}) + singl_eltkn_test('Register', '@', {name=-1}) + singl_eltkn_test('Register', '@a', {name='a'}) + singl_eltkn_test('Register', '@\r', {name=13}) + singl_eltkn_test('Register', '@ ', {name=' '}) + singl_eltkn_test('Register', '@\t', {name=9}) + singl_eltkn_test('SingleQuotedString', '\'test', {closed=false}) + singl_eltkn_test('SingleQuotedString', '\'test\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'\'\'\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'x\'\'\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'\'\'x\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'\'\'', {closed=false}) + singl_eltkn_test('SingleQuotedString', '\'x\'\'', {closed=false}) + singl_eltkn_test('SingleQuotedString', '\'\'\'x', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"test', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"test"', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"\\""', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"x\\""', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"\\"x"', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"\\"', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"x\\"', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"\\"x', {closed=false}) + singl_eltkn_test('Not', '!') + singl_eltkn_test('Assignment', '=', {type='Plain'}) + comparison_test('==', '!=', 'Equal') + comparison_test('=~', '!~', 'Matches') + comparison_test('>', '<=', 'Greater') + comparison_test('>=', '<', 'GreaterOrEqual') + singl_eltkn_test('Minus', '-') + singl_eltkn_test('Assignment', '-=', {type='Subtract'}) + singl_eltkn_test('Arrow', '->') + singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'}) + simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'}) + simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'}) + simple_test({'2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'0b102'}, 'Number', 4, {data={is_float=false, base=2, val=2}, str='0b10'}) + simple_test({'10F'}, 'Number', 2, {data={is_float=false, base=10, val=10}, str='10'}) + simple_test({'0x0123456789ABCDEFG'}, 'Number', 18, {data={is_float=false, base=16, val=81985529216486895}, str='0x0123456789ABCDEF'}) + simple_test({{data='00', size=2}}, 'Number', 2, {data={is_float=false, base=8, val=0}, str='00'}) + simple_test({{data='009', size=2}}, 'Number', 2, {data={is_float=false, base=8, val=0}, str='00'}) + simple_test({{data='01', size=1}}, 'Number', 1, {data={is_float=false, base=10, val=0}, str='0'}) + end + + local function regular_scope_tests() + scope_test('s') + scope_test('g') + scope_test('v') + scope_test('b') + scope_test('w') + scope_test('t') + scope_test('l') + scope_test('a') + + simple_test({'g:'}, 'PlainIdentifier', 2, {data={scope='g', autoload=false}, str='g:'}) + simple_test({'g:is#foo'}, 'PlainIdentifier', 8, {data={scope='g', autoload=true}, str='g:is#foo'}) + simple_test({'g:isnot#foo'}, 'PlainIdentifier', 11, {data={scope='g', autoload=true}, str='g:isnot#foo'}) + end + + local function regular_is_tests() + comparison_test('is', 'isnot', 'Identical') + + simple_test({'is'}, 'Comparison', 2, {data={type='Identical', inv=false, ccs='UseOption'}, str='is'}) + simple_test({'isnot'}, 'Comparison', 5, {data={type='Identical', inv=true, ccs='UseOption'}, str='isnot'}) + simple_test({'is?'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='IgnoreCase'}, str='is?'}) + simple_test({'isnot?'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='IgnoreCase'}, str='isnot?'}) + simple_test({'is#'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'}) + simple_test({'isnot#'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'}) + simple_test({'is#foo'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'}) + simple_test({'isnot#foo'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'}) + end + + local function regular_number_tests() + simple_test({'2.0'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + end + + local function regular_eoc_tests() + singl_eltkn_test('EOC', '|') + singl_eltkn_test('EOC', '\0') + singl_eltkn_test('EOC', '\n') + end + + itp('works (single tokens, zero flags)', function() + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_is_tests() + regular_number_tests() + end) + itp('peeks', function() + flags = tonumber(lib.kELFlagPeek) + should_advance = false + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_is_tests() + regular_number_tests() + end) + itp('forbids scope', function() + flags = tonumber(lib.kELFlagForbidScope) + stable_tests() + + regular_eoc_tests() + regular_is_tests() + regular_number_tests() + + simple_test({'g:'}, 'PlainIdentifier', 1, {data={scope=0, autoload=false}, str='g'}) + end) + itp('allows floats', function() + flags = tonumber(lib.kELFlagAllowFloat) + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_is_tests() + + simple_test({'2.2'}, 'Number', 3, {data={is_float=true, base=10, val=2.2}, str='2.2'}) + simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true, base=10, val=2e5}, str='2.0e5'}) + simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true, base=10, val=2e5}, str='2.0e+5'}) + simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true, base=10, val=2e-5}, str='2.0e-5'}) + simple_test({'2.500000e-5'}, 'Number', 11, {data={is_float=true, base=10, val=2.5e-5}, str='2.500000e-5'}) + simple_test({'2.5555e2'}, 'Number', 8, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e2'}) + simple_test({'2.5555e+2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e+2'}) + simple_test({'2.5555e-2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e-2}, str='2.5555e-2'}) + simple_test({{data='2.5e-5', size=3}}, + 'Number', 3, {data={is_float=true, base=10, val=2.5}, str='2.5'}) + simple_test({{data='2.5e5', size=4}}, + 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({{data='2.5e-50', size=6}}, + 'Number', 6, {data={is_float=true, base=10, val=2.5e-5}, str='2.5e-5'}) + end) + itp('treats `is` as an identifier', function() + flags = tonumber(lib.kELFlagIsNotCmp) + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_number_tests() + + simple_test({'is'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'}) + simple_test({'isnot'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'}) + simple_test({'is?'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'}) + simple_test({'isnot?'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'}) + simple_test({'is#'}, 'PlainIdentifier', 3, {data={scope=0, autoload=true}, str='is#'}) + simple_test({'isnot#'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='isnot#'}) + simple_test({'is#foo'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='is#foo'}) + simple_test({'isnot#foo'}, 'PlainIdentifier', 9, {data={scope=0, autoload=true}, str='isnot#foo'}) + end) + itp('forbids EOC', function() + flags = tonumber(lib.kELFlagForbidEOC) + stable_tests() + + regular_scope_tests() + regular_is_tests() + regular_number_tests() + + singl_eltkn_test('Invalid', '|', {error='E15: Unexpected EOC character: %.*s'}) + singl_eltkn_test('Invalid', '\0', {error='E15: Unexpected EOC character: %.*s'}) + singl_eltkn_test('Invalid', '\n', {error='E15: Unexpected EOC character: %.*s'}) + end) +end) diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua new file mode 100644 index 0000000000..73388e5dd2 --- /dev/null +++ b/test/unit/viml/expressions/parser_spec.lua @@ -0,0 +1,540 @@ +local helpers = require('test.unit.helpers')(after_each) +local global_helpers = require('test.helpers') +local itp = helpers.gen_itp(it) +local viml_helpers = require('test.unit.viml.helpers') + +local make_enum_conv_tab = helpers.make_enum_conv_tab +local child_call_once = helpers.child_call_once +local alloc_log_new = helpers.alloc_log_new +local kvi_destroy = helpers.kvi_destroy +local conv_enum = helpers.conv_enum +local debug_log = helpers.debug_log +local ptr2key = helpers.ptr2key +local cimport = helpers.cimport +local ffi = helpers.ffi +local neq = helpers.neq +local eq = helpers.eq + +local conv_ccs = viml_helpers.conv_ccs +local new_pstate = viml_helpers.new_pstate +local conv_cmp_type = viml_helpers.conv_cmp_type +local pstate_set_str = viml_helpers.pstate_set_str +local conv_expr_asgn_type = viml_helpers.conv_expr_asgn_type + +local mergedicts_copy = global_helpers.mergedicts_copy +local format_string = global_helpers.format_string +local format_luav = global_helpers.format_luav +local intchar2lua = global_helpers.intchar2lua +local dictdiff = global_helpers.dictdiff + +local lib = cimport('./src/nvim/viml/parser/expressions.h', + './src/nvim/syntax.h') + +local alloc_log = alloc_log_new() + +local predefined_hl_defs = { + -- From highlight_init_both + Conceal=true, + Cursor=true, + lCursor=true, + DiffText=true, + ErrorMsg=true, + IncSearch=true, + ModeMsg=true, + NonText=true, + PmenuSbar=true, + StatusLine=true, + StatusLineNC=true, + TabLineFill=true, + TabLineSel=true, + TermCursor=true, + VertSplit=true, + WildMenu=true, + EndOfBuffer=true, + QuickFixLine=true, + Substitute=true, + Whitespace=true, + + -- From highlight_init_(dark|light) + ColorColumn=true, + CursorColumn=true, + CursorLine=true, + CursorLineNr=true, + DiffAdd=true, + DiffChange=true, + DiffDelete=true, + Directory=true, + FoldColumn=true, + Folded=true, + LineNr=true, + MatchParen=true, + MoreMsg=true, + Pmenu=true, + PmenuSel=true, + PmenuThumb=true, + Question=true, + Search=true, + SignColumn=true, + SpecialKey=true, + SpellBad=true, + SpellCap=true, + SpellLocal=true, + SpellRare=true, + TabLine=true, + Title=true, + Visual=true, + WarningMsg=true, + Normal=true, + + -- From syncolor.vim, if &background + Comment=true, + Constant=true, + Special=true, + Identifier=true, + Statement=true, + PreProc=true, + Type=true, + Underlined=true, + Ignore=true, + + -- From syncolor.vim, below if &background + Error=true, + Todo=true, + + -- From syncolor.vim, links at the bottom + String=true, + Character=true, + Number=true, + Boolean=true, + Float=true, + Function=true, + Conditional=true, + Repeat=true, + Label=true, + Operator=true, + Keyword=true, + Exception=true, + Include=true, + Define=true, + Macro=true, + PreCondit=true, + StorageClass=true, + Structure=true, + Typedef=true, + Tag=true, + SpecialChar=true, + Delimiter=true, + SpecialComment=true, + Debug=true, +} + +local nvim_hl_defs = {} + +child_call_once(function() + local i = 0 + while lib.highlight_init_cmdline[i] ~= nil do + local hl_args = lib.highlight_init_cmdline[i] + local s = ffi.string(hl_args) + local err, msg = pcall(function() + if s:sub(1, 13) == 'default link ' then + local new_grp, grp_link = s:match('^default link (%w+) (%w+)$') + neq(nil, new_grp) + -- Note: group to link to must be already defined at the time of + -- linking, otherwise it will be created as cleared. So existence + -- of the group is checked here and not in the next pass over + -- nvim_hl_defs. + eq(true, not not (nvim_hl_defs[grp_link] + or predefined_hl_defs[grp_link])) + eq(false, not not (nvim_hl_defs[new_grp] + or predefined_hl_defs[new_grp])) + nvim_hl_defs[new_grp] = {'link', grp_link} + else + local new_grp, grp_args = s:match('^(%w+) (.*)') + neq(nil, new_grp) + eq(false, not not (nvim_hl_defs[new_grp] + or predefined_hl_defs[new_grp])) + nvim_hl_defs[new_grp] = {'definition', grp_args} + end + end) + if not err then + msg = format_string( + 'Error while processing string %s at position %u:\n%s', s, i, msg) + error(msg) + end + i = i + 1 + end + for k, _ in ipairs(nvim_hl_defs) do + eq('Nvim', k:sub(1, 4)) + -- NvimInvalid + -- 12345678901 + local err, msg = pcall(function() + if k:sub(5, 11) == 'Invalid' then + neq(nil, nvim_hl_defs['Nvim' .. k:sub(12)]) + else + neq(nil, nvim_hl_defs['NvimInvalid' .. k:sub(5)]) + end + end) + if not err then + msg = format_string('Error while processing group %s:\n%s', k, msg) + error(msg) + end + end +end) + +local function hls_to_hl_fs(hls) + local ret = {} + local next_col = 0 + for i, v in ipairs(hls) do + local group, line, col, str = v:match('^Nvim([a-zA-Z]+):(%d+):(%d+):(.*)$') + col = tonumber(col) + line = tonumber(line) + assert(line == 0) + local col_shift = col - next_col + assert(col_shift >= 0) + next_col = col + #str + ret[i] = format_string('hl(%r, %r%s)', + group, + str, + (col_shift == 0 + and '' + or (', %u'):format(col_shift))) + end + return ret +end + +local function format_check(expr, format_check_data, opts) + -- That forces specific order. + local zflags = opts.flags[1] + local zdata = format_check_data[zflags] + local dig_len + if opts.funcname then + print(format_string('\n%s(%r, {', opts.funcname, expr)) + dig_len = #opts.funcname + 2 + else + print(format_string('\n_check_parsing(%r, %r, {', opts, expr)) + dig_len = #('_check_parsing(, \'') + #(format_string('%r', opts)) + end + local digits = ' --' .. (' '):rep(dig_len - #(' --')) + local digits2 = digits:sub(1, -10) + for i = 0, #expr - 1 do + if i % 10 == 0 then + digits2 = ('%s%10u'):format(digits2, i / 10) + end + digits = ('%s%u'):format(digits, i % 10) + end + print(digits) + if #expr > 10 then + print(digits2) + end + if zdata.ast.len then + print((' len = %u,'):format(zdata.ast.len)) + end + print(' ast = ' .. format_luav(zdata.ast.ast, ' ') .. ',') + if zdata.ast.err then + print(' err = {') + print(' arg = ' .. format_luav(zdata.ast.err.arg) .. ',') + print(' msg = ' .. format_luav(zdata.ast.err.msg) .. ',') + print(' },') + end + print('}, {') + for _, v in ipairs(zdata.hl_fs) do + print(' ' .. v .. ',') + end + local diffs = {} + local diffs_num = 0 + for flags, v in pairs(format_check_data) do + if flags ~= zflags then + diffs[flags] = dictdiff(zdata, v) + if diffs[flags] then + if flags == 3 + zflags then + if (dictdiff(format_check_data[1 + zflags], + format_check_data[3 + zflags]) == nil + or dictdiff(format_check_data[2 + zflags], + format_check_data[3 + zflags]) == nil) + then + diffs[flags] = nil + else + diffs_num = diffs_num + 1 + end + else + diffs_num = diffs_num + 1 + end + end + end + end + if diffs_num ~= 0 then + print('}, {') + local flags = 1 + while diffs_num ~= 0 do + if diffs[flags] then + diffs_num = diffs_num - 1 + local diff = diffs[flags] + print((' [%u] = {'):format(flags)) + if diff.ast then + print(' ast = ' .. format_luav(diff.ast, ' ') .. ',') + end + if diff.hl_fs then + print(' hl_fs = ' .. format_luav(diff.hl_fs, ' ', { + literal_strings=true + }) .. ',') + end + print(' },') + end + flags = flags + 1 + end + end + print('})') +end + +local east_node_type_tab +make_enum_conv_tab(lib, { + 'kExprNodeMissing', + 'kExprNodeOpMissing', + 'kExprNodeTernary', + 'kExprNodeTernaryValue', + 'kExprNodeRegister', + 'kExprNodeSubscript', + 'kExprNodeListLiteral', + 'kExprNodeUnaryPlus', + 'kExprNodeBinaryPlus', + 'kExprNodeNested', + 'kExprNodeCall', + 'kExprNodePlainIdentifier', + 'kExprNodePlainKey', + 'kExprNodeComplexIdentifier', + 'kExprNodeUnknownFigure', + 'kExprNodeLambda', + 'kExprNodeDictLiteral', + 'kExprNodeCurlyBracesIdentifier', + 'kExprNodeComma', + 'kExprNodeColon', + 'kExprNodeArrow', + 'kExprNodeComparison', + 'kExprNodeConcat', + 'kExprNodeConcatOrSubscript', + 'kExprNodeInteger', + 'kExprNodeFloat', + 'kExprNodeSingleQuotedString', + 'kExprNodeDoubleQuotedString', + 'kExprNodeOr', + 'kExprNodeAnd', + 'kExprNodeUnaryMinus', + 'kExprNodeBinaryMinus', + 'kExprNodeNot', + 'kExprNodeMultiplication', + 'kExprNodeDivision', + 'kExprNodeMod', + 'kExprNodeOption', + 'kExprNodeEnvironment', + 'kExprNodeAssignment', +}, 'kExprNode', function(ret) east_node_type_tab = ret end) + +local function conv_east_node_type(typ) + return conv_enum(east_node_type_tab, typ) +end + +local eastnodelist2lua + +local function eastnode2lua(pstate, eastnode, checked_nodes) + local key = ptr2key(eastnode) + if checked_nodes[key] then + checked_nodes[key].duplicate_key = key + return { duplicate = key } + end + local typ = conv_east_node_type(eastnode.type) + local ret = {} + checked_nodes[key] = ret + ret.children = eastnodelist2lua(pstate, eastnode.children, checked_nodes) + local str = pstate_set_str(pstate, eastnode.start, eastnode.len) + local ret_str + if str.error then + ret_str = 'error:' .. str.error + else + ret_str = ('%u:%u:%s'):format(str.start.line, str.start.col, str.str) + end + if typ == 'Register' then + typ = typ .. ('(name=%s)'):format( + tostring(intchar2lua(eastnode.data.reg.name))) + elseif typ == 'PlainIdentifier' then + typ = typ .. ('(scope=%s,ident=%s)'):format( + tostring(intchar2lua(eastnode.data.var.scope)), + ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len)) + elseif typ == 'PlainKey' then + typ = typ .. ('(key=%s)'):format( + ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len)) + elseif (typ == 'UnknownFigure' or typ == 'DictLiteral' + or typ == 'CurlyBracesIdentifier' or typ == 'Lambda') then + typ = typ .. ('(%s)'):format( + (eastnode.data.fig.type_guesses.allow_lambda and '\\' or '-') + .. (eastnode.data.fig.type_guesses.allow_dict and 'd' or '-') + .. (eastnode.data.fig.type_guesses.allow_ident and 'i' or '-')) + elseif typ == 'Comparison' then + typ = typ .. ('(type=%s,inv=%u,ccs=%s)'):format( + conv_cmp_type(eastnode.data.cmp.type), eastnode.data.cmp.inv and 1 or 0, + conv_ccs(eastnode.data.cmp.ccs)) + elseif typ == 'Integer' then + typ = typ .. ('(val=%u)'):format(tonumber(eastnode.data.num.value)) + elseif typ == 'Float' then + typ = typ .. format_string('(val=%e)', tonumber(eastnode.data.flt.value)) + elseif typ == 'SingleQuotedString' or typ == 'DoubleQuotedString' then + if eastnode.data.str.value == nil then + typ = typ .. '(val=NULL)' + else + local s = ffi.string(eastnode.data.str.value, eastnode.data.str.size) + typ = format_string('%s(val=%q)', typ, s) + end + elseif typ == 'Option' then + typ = ('%s(scope=%s,ident=%s)'):format( + typ, + tostring(intchar2lua(eastnode.data.opt.scope)), + ffi.string(eastnode.data.opt.ident, eastnode.data.opt.ident_len)) + elseif typ == 'Environment' then + typ = ('%s(ident=%s)'):format( + typ, + ffi.string(eastnode.data.env.ident, eastnode.data.env.ident_len)) + elseif typ == 'Assignment' then + typ = ('%s(%s)'):format(typ, conv_expr_asgn_type(eastnode.data.ass.type)) + end + ret_str = typ .. ':' .. ret_str + local can_simplify = not ret.children + if can_simplify then + ret = ret_str + else + ret[1] = ret_str + end + return ret +end + +eastnodelist2lua = function(pstate, eastnode, checked_nodes) + local ret = {} + while eastnode ~= nil do + ret[#ret + 1] = eastnode2lua(pstate, eastnode, checked_nodes) + eastnode = eastnode.next + end + if #ret == 0 then + ret = nil + end + return ret +end + +local function east2lua(str, pstate, east) + local checked_nodes = {} + local len = tonumber(pstate.pos.col) + if pstate.pos.line == 1 then + len = tonumber(pstate.reader.lines.items[0].size) + end + if type(str) == 'string' and len == #str then + len = nil + end + return { + err = east.err.msg ~= nil and { + msg = ffi.string(east.err.msg), + arg = ffi.string(east.err.arg, east.err.arg_len), + } or nil, + len = len, + ast = eastnodelist2lua(pstate, east.root, checked_nodes), + } +end + +local function phl2lua(pstate) + local ret = {} + for i = 0, (tonumber(pstate.colors.size) - 1) do + local chunk = pstate.colors.items[i] + local chunk_tbl = pstate_set_str( + pstate, chunk.start, chunk.end_col - chunk.start.col, { + group = ffi.string(chunk.group), + }) + ret[i + 1] = ('%s:%u:%u:%s'):format( + chunk_tbl.group, + chunk_tbl.start.line, + chunk_tbl.start.col, + chunk_tbl.str) + end + return ret +end + +child_call_once(function() + assert:set_parameter('TableFormatLevel', 1000000) +end) + +describe('Expressions parser', function() + local function _check_parsing(opts, str, exp_ast, exp_highlighting_fs, + nz_flags_exps) + local zflags = opts.flags[1] + nz_flags_exps = nz_flags_exps or {} + local format_check_data = {} + for _, flags in ipairs(opts.flags) do + debug_log(('Running test case (%s, %u)'):format(str, flags)) + local err, msg = pcall(function() + if os.getenv('NVIM_TEST_PARSER_SPEC_PRINT_TEST_CASE') == '1' then + print(str, flags) + end + alloc_log:check({}) + + local pstate = new_pstate({str}) + local east = lib.viml_pexpr_parse(pstate, flags) + local ast = east2lua(str, pstate, east) + local hls = phl2lua(pstate) + if exp_ast == nil then + format_check_data[flags] = {ast=ast, hl_fs=hls_to_hl_fs(hls)} + else + local exps = { + ast = exp_ast, + hl_fs = exp_highlighting_fs, + } + local add_exps = nz_flags_exps[flags] + if not add_exps and flags == 3 + zflags then + add_exps = nz_flags_exps[1 + zflags] or nz_flags_exps[2 + zflags] + end + if add_exps then + if add_exps.ast then + exps.ast = mergedicts_copy(exps.ast, add_exps.ast) + end + if add_exps.hl_fs then + exps.hl_fs = mergedicts_copy(exps.hl_fs, add_exps.hl_fs) + end + end + eq(exps.ast, ast) + if exp_highlighting_fs then + local exp_highlighting = {} + local next_col = 0 + for i, h in ipairs(exps.hl_fs) do + exp_highlighting[i], next_col = h(next_col) + end + eq(exp_highlighting, hls) + end + end + lib.viml_pexpr_free_ast(east) + kvi_destroy(pstate.colors) + alloc_log:clear_tmp_allocs(true) + alloc_log:check({}) + end) + if not err then + msg = format_string('Error while processing test (%r, %u):\n%s', + str, flags, msg) + error(msg) + end + end + if exp_ast == nil then + format_check(str, format_check_data, opts) + end + end + local function hl(group, str, shift) + return function(next_col) + if nvim_hl_defs['Nvim' .. group] == nil then + error(('Unknown group: Nvim%s'):format(group)) + end + local col = next_col + (shift or 0) + return (('%s:%u:%u:%s'):format( + 'Nvim' .. group, + 0, + col, + str)), (col + #str) + end + end + local function fmtn(typ, args, rest) + return ('%s(%s)%s'):format(typ, args, rest) + end + require('test.unit.viml.expressions.parser_tests')( + itp, _check_parsing, hl, fmtn) +end) diff --git a/test/unit/viml/expressions/parser_tests.lua b/test/unit/viml/expressions/parser_tests.lua new file mode 100644 index 0000000000..e085d7e932 --- /dev/null +++ b/test/unit/viml/expressions/parser_tests.lua @@ -0,0 +1,8317 @@ +local global_helpers = require('test.helpers') + +local REMOVE_THIS = global_helpers.REMOVE_THIS + +return function(itp, _check_parsing, hl, fmtn) + local function check_parsing(...) + return _check_parsing({flags={0, 1, 2, 3}, funcname='check_parsing'}, ...) + end + local function check_asgn_parsing(...) + return _check_parsing({ + flags={4, 5, 6, 7}, + funcname='check_asgn_parsing', + }, ...) + end + itp('works with + and @a', function() + check_parsing('@a', { + ast = { + 'Register(name=a):0:0:@a', + }, + }, { + hl('Register', '@a'), + }) + check_parsing('+@a', { + ast = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + }) + check_parsing('@a+@b', { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a+@b+@c', { + ast = { + { + 'BinaryPlus:0:5:+', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + 'Register(name=c):0:6:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('BinaryPlus', '+'), + hl('Register', '@c'), + }) + check_parsing('+@a+@b', { + ast = { + { + 'BinaryPlus:0:3:+', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:4:@b', + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('+@a++@b', { + ast = { + { + 'BinaryPlus:0:3:+', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'UnaryPlus:0:4:+', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a@b', { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:2:@b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidRegister', '@b'), + }, { + [1] = { + ast = { + len = 2, + err = REMOVE_THIS, + ast = { + 'Register(name=a):0:0:@a' + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + }, + }, + }) + check_parsing(' @a \t @b', { + ast = { + { + 'OpMissing:0:3:', + children = { + 'Register(name=a):0:0: @a', + 'Register(name=b):0:3: \t @b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a', 1), + hl('InvalidSpacing', ' \t '), + hl('Register', '@b'), + }, { + [1] = { + ast = { + len = 6, + err = REMOVE_THIS, + ast = { + 'Register(name=a):0:0: @a' + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + [3] = REMOVE_THIS, + }, + }, + }) + check_parsing('+', { + ast = { + 'UnaryPlus:0:0:+', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryPlus', '+'), + }) + check_parsing(' +', { + ast = { + 'UnaryPlus:0:0: +', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryPlus', '+', 1), + }) + check_parsing('@a+ ', { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + }) + end) + itp('works with @a, + and parenthesis', function() + check_parsing('(@a)', { + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + }) + check_parsing('()', { + ast = { + { + 'Nested:0:0:(', + children = { + 'Missing:0:1:', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing(')', { + ast = { + { + 'Nested:0:0:', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('+)', { + ast = { + { + 'Nested:0:1:', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('UnaryPlus', '+'), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('+@a(@b)', { + ast = { + { + 'UnaryPlus:0:0:+', + children = { + { + 'Call:0:3:(', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+@b(@c)', { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + 'Register(name=c):0:6:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a()', { + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a ()', { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:2: (', + children = { + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = '()', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidSpacing', ' '), + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + }, { + [1] = { + ast = { + len = 3, + err = REMOVE_THIS, + ast = { + 'Register(name=a):0:0:@a', + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + [3] = REMOVE_THIS, + [4] = REMOVE_THIS, + }, + }, + }) + check_parsing('@a + (@b)', { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing('@a + (+@b)', { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'UnaryPlus:0:6:+', + children = { + 'Register(name=b):0:7:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing('@a + (@b + @c)', { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + }) + check_parsing('(@a)+@b', { + ast = { + { + 'BinaryPlus:0:4:+', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:5:@b', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a+(@b)(@c)', { + -- 01234567890 + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:7:(', + children = { + { + 'Nested:0:3:(', + children = { 'Register(name=b):0:4:@b' }, + }, + 'Register(name=c):0:8:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+((@b))(@c)', { + -- 01234567890123456890123456789 + -- 0 1 2 + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:9:(', + children = { + { + 'Nested:0:3:(', + children = { + { + 'Nested:0:4:(', + children = { 'Register(name=b):0:5:@b' } + }, + }, + }, + 'Register(name=c):0:10:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+((@b))+@c', { + -- 01234567890123456890123456789 + -- 0 1 2 + ast = { + { + 'BinaryPlus:0:9:+', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:3:(', + children = { + { + 'Nested:0:4:(', + children = { 'Register(name=b):0:5:@b' } + }, + }, + }, + }, + }, + 'Register(name=c):0:10:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+'), + hl('Register', '@c'), + }) + check_parsing( + '@a + (@b + @c) + @d(@e) + (+@f) + ((+@g(@h))(@j)(@k))(@l)', {--[[ + | | | | | | | | || | | || | | ||| || || || || + 000000000011111111112222222222333333333344444444445555555 + 012345678901234567890123456789012345678901234567890123456 + ]] + ast = {{ + 'BinaryPlus:0:31: +', + children = { + { + 'BinaryPlus:0:23: +', + children = { + { + 'BinaryPlus:0:14: +', + children = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', + }, + }, + }, + }, + }, + }, + { + 'Call:0:19:(', + children = { + 'Register(name=d):0:16: @d', + 'Register(name=e):0:20:@e', + }, + }, + }, + }, + { + 'Nested:0:25: (', + children = { + { + 'UnaryPlus:0:27:+', + children = { + 'Register(name=f):0:28:@f', + }, + }, + }, + }, + }, + }, + { + 'Call:0:53:(', + children = { + { + 'Nested:0:33: (', + children = { + { + 'Call:0:48:(', + children = { + { + 'Call:0:44:(', + children = { + { + 'Nested:0:35:(', + children = { + { + 'UnaryPlus:0:36:+', + children = { + { + 'Call:0:39:(', + children = { + 'Register(name=g):0:37:@g', + 'Register(name=h):0:40:@h', + }, + }, + }, + }, + }, + }, + 'Register(name=j):0:45:@j', + }, + }, + 'Register(name=k):0:49:@k', + }, + }, + }, + }, + 'Register(name=l):0:54:@l', + }, + }, + }, + }}, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('Register', '@d', 1), + hl('CallingParenthesis', '('), + hl('Register', '@e'), + hl('CallingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@f'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('NestingParenthesis', '('), + hl('UnaryPlus', '+'), + hl('Register', '@g'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@j'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@k'), + hl('CallingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@l'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a)', { + -- 012 + ast = { + { + 'Nested:0:2:', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Unexpected closing parenthesis: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('(@a', { + -- 012 + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + err = { + arg = '(@a', + msg = 'E110: Missing closing parenthesis for nested expression: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + }) + check_parsing('@a(@b', { + -- 01234 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + err = { + arg = '(@b', + msg = 'E116: Missing closing parenthesis for function call: %.*s', + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + }) + check_parsing('@a(@b, @c, @d, @e)', { + -- 012345678901234567 + -- 0 1 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Comma:0:5:,', + children = { + 'Register(name=b):0:3:@b', + { + 'Comma:0:9:,', + children = { + 'Register(name=c):0:6: @c', + { + 'Comma:0:13:,', + children = { + 'Register(name=d):0:10: @d', + 'Register(name=e):0:14: @e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c', 1), + hl('Comma', ','), + hl('Register', '@d', 1), + hl('Comma', ','), + hl('Register', '@e', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a(@b(@c))', { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + 'Register(name=c):0:6:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a(@b(@c(@d(@e), @f(@g(@h), @i(@j)))))', { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + { + 'Call:0:8:(', + children = { + 'Register(name=c):0:6:@c', + { + 'Comma:0:15:,', + children = { + { + 'Call:0:11:(', + children = { + 'Register(name=d):0:9:@d', + 'Register(name=e):0:12:@e', + }, + }, + { + 'Call:0:19:(', + children = { + 'Register(name=f):0:16: @f', + { + 'Comma:0:26:,', + children = { + { + 'Call:0:22:(', + children = { + 'Register(name=g):0:20:@g', + 'Register(name=h):0:23:@h', + }, + }, + { + 'Call:0:30:(', + children = { + 'Register(name=i):0:27: @i', + 'Register(name=j):0:31:@j', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', '('), + hl('Register', '@d'), + hl('CallingParenthesis', '('), + hl('Register', '@e'), + hl('CallingParenthesis', ')'), + hl('Comma', ','), + hl('Register', '@f', 1), + hl('CallingParenthesis', '('), + hl('Register', '@g'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('Comma', ','), + hl('Register', '@i', 1), + hl('CallingParenthesis', '('), + hl('Register', '@j'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + }) + check_parsing('()()', { + -- 0123 + ast = { + { + 'Call:0:2:(', + children = { + { + 'Nested:0:0:(', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = ')()', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('(@a)()', { + -- 012345 + ast = { + { + 'Call:0:4:(', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('(@a)(@b)', { + -- 01234567 + ast = { + { + 'Call:0:4:(', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:5:@b', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', ')'), + }) + check_parsing('(@a) (@b)', { + -- 012345678 + ast = { + { + 'OpMissing:0:4:', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'Nested:0:4: (', + children = { + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + err = { + arg = '(@b)', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('InvalidSpacing', ' '), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }, { + [1] = { + ast = { + len = 5, + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + REMOVE_THIS, + }, + }, + }, + err = REMOVE_THIS, + }, + hl_fs = { + [4] = REMOVE_THIS, + [5] = REMOVE_THIS, + [6] = REMOVE_THIS, + [7] = REMOVE_THIS, + }, + }, + }) + end) + itp('works with variable names, including curly braces ones', function() + check_parsing('var', { + ast = { + 'PlainIdentifier(scope=0,ident=var):0:0:var', + }, + }, { + hl('IdentifierName', 'var'), + }) + check_parsing('g:var', { + ast = { + 'PlainIdentifier(scope=g,ident=var):0:0:g:var', + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'var'), + }) + check_parsing('g:', { + ast = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + }) + check_parsing('{a}', { + -- 012 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + }) + check_parsing('{a:b}', { + -- 012 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=a,ident=b):0:1:a:b', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + }) + check_parsing('{a:@b}', { + -- 012345 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'OpMissing:0:3:', + children={ + 'PlainIdentifier(scope=a,ident=):0:1:a:', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, + }, + err = { + arg = '@b}', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('InvalidRegister', '@b'), + hl('Curly', '}'), + }) + check_parsing('{@a}', { + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('{@a}{@b}', { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:4:', + children = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + fmtn('CurlyBracesIdentifier', '--i', ':0:4:{'), + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('Register', '@b'), + hl('Curly', '}'), + }) + check_parsing('g:{@a}', { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('{@a}_test', { + -- 012345678 + ast = { + { + 'ComplexIdentifier:0:4:', + children = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:4:_test', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + }) + check_parsing('g:{@a}_test', { + -- 01234567890 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + 'Register(name=a):0:3:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:6:_test', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + }) + check_parsing('g:{@a}_test()', { + -- 0123456789012 + ast = { + { + 'Call:0:11:(', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + 'Register(name=a):0:3:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:6:_test', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('{@a} ()', { + -- 0123456789012 + ast = { + { + 'Call:0:4: (', + children = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('g:{@a} ()', { + -- 0123456789012 + ast = { + { + 'Call:0:6: (', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('{@a', { + -- 012 + ast = { + { + fmtn('UnknownFigure', '-di', ':0:0:{'), + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + err = { + arg = '{@a', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + hl('Register', '@a'), + }) + check_parsing('a ()', { + -- 0123 + ast = { + { + 'Call:0:1: (', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + end) + itp('works with lambdas and dictionaries', function() + check_parsing('{}', { + ast = { + fmtn('DictLiteral', '-di', ':0:0:{'), + }, + }, { + hl('Dict', '{'), + hl('Dict', '}'), + }) + check_parsing('{->@a}', { + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Arrow:0:1:->', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{->@a+@b}', { + -- 012345678 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Arrow:0:1:->', + children = { + { + 'BinaryPlus:0:5:+', + children = { + 'Register(name=a):0:3:@a', + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('Lambda', '}'), + }) + check_parsing('{a->@a}', { + -- 012345678 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Arrow:0:2:->', + children = { + 'Register(name=a):0:4:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->@a}', { + -- 012345678 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + 'Register(name=a):0:6:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c->@a}', { + -- 01234567890 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + { + 'Arrow:0:6:->', + children = { + 'Register(name=a):0:8:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c,d->@a}', { + -- 0123456789012 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + { + 'Comma:0:6:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:5:c', + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + { + 'Arrow:0:8:->', + children = { + 'Register(name=a):0:10:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c,d,->@a}', { + -- 01234567890123 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + { + 'Comma:0:6:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:5:c', + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + }, + }, + { + 'Arrow:0:9:->', + children = { + 'Register(name=a):0:11:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Comma', ','), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->{c,d->{e,f->@a}}}', { + -- 01234567890123456789012 + -- 0 1 2 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + { + fmtn('Lambda', '\\di', ':0:6:{'), + children = { + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7:c', + 'PlainIdentifier(scope=0,ident=d):0:9:d', + }, + }, + { + 'Arrow:0:10:->', + children = { + { + fmtn('Lambda', '\\di', ':0:12:{'), + children = { + { + 'Comma:0:14:,', + children = { + 'PlainIdentifier(scope=0,ident=e):0:13:e', + 'PlainIdentifier(scope=0,ident=f):0:15:f', + }, + }, + { + 'Arrow:0:16:->', + children = { + 'Register(name=a):0:18:@a', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('Lambda', '{'), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Arrow', '->'), + hl('Lambda', '{'), + hl('IdentifierName', 'e'), + hl('Comma', ','), + hl('IdentifierName', 'f'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + hl('Lambda', '}'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->c,d}', { + -- 0123456789 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + { + 'Comma:0:7:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6:c', + 'PlainIdentifier(scope=0,ident=d):0:8:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',d}', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + hl('Lambda', '}'), + }) + check_parsing('a,b,c,d', { + -- 0123456789 + ast = { + { + 'Comma:0:1:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comma:0:3:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + 'PlainIdentifier(scope=0,ident=d):0:6:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',b,c,d', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidComma', ','), + hl('IdentifierName', 'b'), + hl('InvalidComma', ','), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + }) + check_parsing('a,b,c,d,', { + -- 0123456789 + ast = { + { + 'Comma:0:1:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comma:0:3:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + { + 'Comma:0:7:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:6:d', + }, + }, + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',b,c,d,', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidComma', ','), + hl('IdentifierName', 'b'), + hl('InvalidComma', ','), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + hl('InvalidComma', ','), + }) + check_parsing(',', { + -- 0123456789 + ast = { + { + 'Comma:0:0:,', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ',', + msg = 'E15: Expected value, got comma: %.*s', + }, + }, { + hl('InvalidComma', ','), + }) + check_parsing('{,a->@a}', { + -- 0123456789 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'Arrow:0:3:->', + children = { + { + 'Comma:0:1:,', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:2:a', + }, + }, + 'Register(name=a):0:5:@a', + }, + }, + }, + }, + }, + err = { + arg = ',a->@a}', + msg = 'E15: Expected value, got comma: %.*s', + }, + }, { + hl('Curly', '{'), + hl('InvalidComma', ','), + hl('IdentifierName', 'a'), + hl('InvalidArrow', '->'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('}', { + -- 0123456789 + ast = { + fmtn('UnknownFigure', '---', ':0:0:'), + }, + err = { + arg = '}', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidFigureBrace', '}'), + }) + check_parsing('{->}', { + -- 0123456789 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + 'Arrow:0:1:->', + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected value, got closing figure brace: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('InvalidLambda', '}'), + }) + check_parsing('{a,b}', { + -- 0123456789 + ast = { + { + fmtn('Lambda', '-di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected lambda arguments list or arrow: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('InvalidLambda', '}'), + }) + check_parsing('{a,}', { + -- 0123456789 + ast = { + { + fmtn('Lambda', '-di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected lambda arguments list or arrow: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('InvalidLambda', '}'), + }) + check_parsing('{@a:@b}', { + -- 0123456789 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d}', { + -- 0123456789012 + -- 0 1 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d,@e:@f,}', { + -- 01234567890123456789 + -- 0 1 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Comma:0:12:,', + children = { + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:15::', + children = { + 'Register(name=e):0:13:@e', + 'Register(name=f):0:16:@f', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Comma', ','), + hl('Register', '@e'), + hl('Colon', ':'), + hl('Register', '@f'), + hl('Comma', ','), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', { + -- 01234567890123456789012 + -- 0 1 2 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Comma:0:12:,', + children = { + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:15::', + children = { + 'Register(name=e):0:13:@e', + 'Register(name=f):0:16:@f', + }, + }, + { + 'Colon:0:21::', + children = { + 'Register(name=g):0:19:@g', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected value, got closing figure brace: %.*s', + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Comma', ','), + hl('Register', '@e'), + hl('Colon', ':'), + hl('Register', '@f'), + hl('Comma', ','), + hl('Register', '@g'), + hl('Colon', ':'), + hl('InvalidDict', '}'), + }) + check_parsing('{@a:@b,}', { + -- 01234567890123 + -- 0 1 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Dict', '}'), + }) + check_parsing('{({f -> g})(@h)(@i)}', { + -- 01234567890123456789 + -- 0 1 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'Call:0:15:(', + children = { + { + 'Call:0:11:(', + children = { + { + 'Nested:0:1:(', + children = { + { + fmtn('Lambda', '\\di', ':0:2:{'), + children = { + 'PlainIdentifier(scope=0,ident=f):0:3:f', + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=0,ident=g):0:7: g', + }, + }, + }, + }, + }, + }, + 'Register(name=h):0:12:@h', + }, + }, + 'Register(name=i):0:16:@i', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('NestingParenthesis', '('), + hl('Lambda', '{'), + hl('IdentifierName', 'f'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'g', 1), + hl('Lambda', '}'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@i'), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + }) + check_parsing('a:{b()}c', { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=a,ident=):0:0:a:', + { + 'ComplexIdentifier:0:7:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + { + 'Call:0:4:(', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:7:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + }) + check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', { + -- 01234567890123456789012345678901234567890123456 + -- 0 1 2 3 4 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=a,ident=):0:0:a:', + { + 'ComplexIdentifier:0:42:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + { + 'Call:0:37:(', + children = { + { + fmtn('Lambda', '\\di', ':0:3:{'), + children = { + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + { + 'Arrow:0:8: ->', + children = { + { + 'BinaryPlus:0:19: +', + children = { + { + 'BinaryPlus:0:14: +', + children = { + 'Register(name=d):0:11: @d', + 'Register(name=e):0:16: @e', + }, + }, + { + 'Call:0:32:(', + children = { + { + 'Nested:0:21: (', + children = { + { + fmtn('Lambda', '\\di', ':0:23:{'), + children = { + 'PlainIdentifier(scope=0,ident=f):0:24:f', + { + 'Arrow:0:25: ->', + children = { + 'PlainIdentifier(scope=0,ident=g):0:28: g', + }, + }, + }, + }, + }, + }, + 'Register(name=h):0:33:@h', + }, + }, + }, + }, + }, + }, + }, + }, + 'Register(name=i):0:38:@i', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=j):0:42:j', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Lambda', '{'), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('Arrow', '->', 1), + hl('Register', '@d', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@e', 1), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Lambda', '{'), + hl('IdentifierName', 'f'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'g', 1), + hl('Lambda', '}'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('Register', '@i'), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + hl('IdentifierName', 'j'), + }) + check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:8: :', + children = { + { + 'BinaryPlus:0:3: +', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:5: @b', + }, + }, + { + 'BinaryPlus:0:13: +', + children = { + 'Register(name=c):0:10: @c', + 'Register(name=d):0:15: @d', + }, + }, + }, + }, + { + 'Colon:0:27: :', + children = { + { + 'BinaryPlus:0:22: +', + children = { + 'Register(name=e):0:19: @e', + 'Register(name=f):0:24: @f', + }, + }, + { + 'BinaryPlus:0:32: +', + children = { + 'Register(name=g):0:29: @g', + 'Register(name=i):0:34: @i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('Register', '@b', 1), + hl('Colon', ':', 1), + hl('Register', '@c', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@d', 1), + hl('Comma', ','), + hl('Register', '@e', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@f', 1), + hl('Colon', ':', 1), + hl('Register', '@g', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@i', 1), + hl('Dict', '}'), + }) + check_parsing('-> -> ->', { + -- 01234567 + ast = { + { + 'Arrow:0:0:->', + children = { + 'Missing:0:0:', + { + 'Arrow:0:2: ->', + children = { + 'Missing:0:2:', + { + 'Arrow:0:5: ->', + children = { + 'Missing:0:5:', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> -> ->', + msg = 'E15: Unexpected arrow: %.*s', + }, + }, { + hl('InvalidArrow', '->'), + hl('InvalidArrow', '->', 1), + hl('InvalidArrow', '->', 1), + }) + check_parsing('a -> b -> c -> d', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Arrow:0:1: ->', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Arrow:0:6: ->', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4: b', + { + 'Arrow:0:11: ->', + children = { + 'PlainIdentifier(scope=0,ident=c):0:9: c', + 'PlainIdentifier(scope=0,ident=d):0:14: d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> b -> c -> d', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'c', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'd', 1), + }) + check_parsing('{a -> b -> c}', { + -- 0123456789012 + -- 0 1 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Arrow:0:2: ->', + children = { + { + 'Arrow:0:7: ->', + children = { + 'PlainIdentifier(scope=0,ident=b):0:5: b', + 'PlainIdentifier(scope=0,ident=c):0:10: c', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> c}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'c', 1), + hl('Lambda', '}'), + }) + check_parsing('{a: -> b}', { + -- 012345678 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'Arrow:0:3: ->', + children = { + 'PlainIdentifier(scope=a,ident=):0:1:a:', + 'PlainIdentifier(scope=0,ident=b):0:6: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + + check_parsing('{a:b -> b}', { + -- 0123456789 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=a,ident=b):0:1:a:b', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'b'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + + check_parsing('{a#b -> b}', { + -- 0123456789 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=0,ident=a#b):0:1:a#b', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a#b'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + check_parsing('{a : b : c}', { + -- 01234567890 + -- 0 1 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Colon:0:6: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4: b', + 'PlainIdentifier(scope=0,ident=c):0:8: c', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ': c}', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('Dict', '}'), + }) + check_parsing('{', { + -- 0 + ast = { + fmtn('UnknownFigure', '\\di', ':0:0:{'), + }, + err = { + arg = '{', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + }) + check_parsing('{a', { + -- 01 + ast = { + { + fmtn('UnknownFigure', '\\di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + err = { + arg = '{a', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + }) + check_parsing('{a,b', { + -- 0123 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + }, + err = { + arg = '{a,b', + msg = 'E15: Missing closing figure brace for lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + }) + check_parsing('{a,b->', { + -- 012345 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + 'Arrow:0:4:->', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + }) + check_parsing('{a,b->c', { + -- 0123456 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6:c', + }, + }, + }, + }, + }, + err = { + arg = '{a,b->c', + msg = 'E15: Missing closing figure brace for lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('IdentifierName', 'c'), + }) + check_parsing('{a : b', { + -- 012345 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, + }, + err = { + arg = '{a : b', + msg = 'E723: Missing end of Dictionary \'}\': %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + }) + check_parsing('{a : b,', { + -- 0123456 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + }) + end) + itp('works with ternary operator', function() + check_parsing('a ? b : c', { + -- 012345678 + ast = { + { + 'Ternary:0:1: ?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:5: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?', 1), + hl('IdentifierName', 'b', 1), + hl('TernaryColon', ':', 1), + hl('IdentifierName', 'c', 1), + }) + check_parsing('@a?@b?@c:@d:@e', { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:11::', + children = { + { + 'Ternary:0:5:?', + children = { + 'Register(name=b):0:3:@b', + { + 'TernaryValue:0:8::', + children = { + 'Register(name=c):0:6:@c', + 'Register(name=d):0:9:@d', + }, + }, + }, + }, + 'Register(name=e):0:12:@e', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('Ternary', '?'), + hl('Register', '@c'), + hl('TernaryColon', ':'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + check_parsing('@a?@b:@c?@d:@e', { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:5::', + children = { + 'Register(name=b):0:3:@b', + { + 'Ternary:0:8:?', + children = { + 'Register(name=c):0:6:@c', + { + 'TernaryValue:0:11::', + children = { + 'Register(name=d):0:9:@d', + 'Register(name=e):0:12:@e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', { + -- 01234567890123456789012345678901 + -- 0 1 2 3 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:29::', + children = { + { + 'Ternary:0:5:?', + children = { + 'Register(name=b):0:3:@b', + { + 'TernaryValue:0:20::', + children = { + { + 'Ternary:0:8:?', + children = { + 'Register(name=c):0:6:@c', + { + 'TernaryValue:0:11::', + children = { + 'Register(name=d):0:9:@d', + { + 'Ternary:0:14:?', + children = { + 'Register(name=e):0:12:@e', + { + 'TernaryValue:0:17::', + children = { + 'Register(name=f):0:15:@f', + 'Register(name=g):0:18:@g', + }, + }, + }, + }, + }, + }, + }, + }, + { + 'Ternary:0:23:?', + children = { + 'Register(name=h):0:21:@h', + { + 'TernaryValue:0:26::', + children = { + 'Register(name=i):0:24:@i', + 'Register(name=j):0:27:@j', + }, + }, + }, + }, + }, + }, + }, + }, + 'Register(name=k):0:30:@k', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('Ternary', '?'), + hl('Register', '@c'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + hl('Ternary', '?'), + hl('Register', '@f'), + hl('TernaryColon', ':'), + hl('Register', '@g'), + hl('TernaryColon', ':'), + hl('Register', '@h'), + hl('Ternary', '?'), + hl('Register', '@i'), + hl('TernaryColon', ':'), + hl('Register', '@j'), + hl('TernaryColon', ':'), + hl('Register', '@k'), + }) + check_parsing('?', { + -- 0 + ast = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + 'TernaryValue:0:0:?', + }, + }, + }, + err = { + arg = '?', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + }) + + check_parsing('?:', { + -- 01 + ast = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + { + 'TernaryValue:0:1::', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = '?:', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + hl('InvalidTernaryColon', ':'), + }) + + check_parsing('?::', { + -- 012 + ast = { + { + 'Colon:0:2::', + children = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + { + 'TernaryValue:0:1::', + children = { + 'Missing:0:1:', + 'Missing:0:2:', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '?::', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + hl('InvalidTernaryColon', ':'), + hl('InvalidColon', ':'), + }) + + check_parsing('a?b', { + -- 012 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + err = { + arg = '?b', + msg = 'E109: Missing \':\' after \'?\': %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + }) + check_parsing('a?b:', { + -- 0123 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:1:?', + children = { + 'PlainIdentifier(scope=b,ident=):0:2:b:', + }, + }, + }, + }, + }, + err = { + arg = '?b:', + msg = 'E109: Missing \':\' after \'?\': %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + }) + + check_parsing('a?b::c', { + -- 012345 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:4::', + children = { + 'PlainIdentifier(scope=b,ident=):0:2:b:', + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('TernaryColon', ':'), + hl('IdentifierName', 'c'), + }) + + check_parsing('a?b :', { + -- 01234 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + hl('TernaryColon', ':', 1), + }) + + check_parsing('(@a?@b:@c)?@d:@e', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:13::', + children = { + 'Register(name=d):0:11:@d', + 'Register(name=e):0:14:@e', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + + check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', { + -- 01234567890123456789012345678901 + -- 0 1 2 3 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:21::', + children = { + { + 'Nested:0:11:(', + children = { + { + 'Ternary:0:14:?', + children = { + 'Register(name=d):0:12:@d', + { + 'TernaryValue:0:17::', + children = { + 'Register(name=e):0:15:@e', + 'Register(name=f):0:18:@f', + }, + }, + }, + }, + }, + }, + { + 'Nested:0:22:(', + children = { + { + 'Ternary:0:25:?', + children = { + 'Register(name=g):0:23:@g', + { + 'TernaryValue:0:28::', + children = { + 'Register(name=h):0:26:@h', + 'Register(name=i):0:29:@i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('NestingParenthesis', '('), + hl('Register', '@d'), + hl('Ternary', '?'), + hl('Register', '@e'), + hl('TernaryColon', ':'), + hl('Register', '@f'), + hl('NestingParenthesis', ')'), + hl('TernaryColon', ':'), + hl('NestingParenthesis', '('), + hl('Register', '@g'), + hl('Ternary', '?'), + hl('Register', '@h'), + hl('TernaryColon', ':'), + hl('Register', '@i'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', { + -- 0123456789012345678901234567 + -- 0 1 2 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:19::', + children = { + { + 'Ternary:0:13:?', + children = { + 'Register(name=d):0:11:@d', + { + 'TernaryValue:0:16::', + children = { + 'Register(name=e):0:14:@e', + 'Register(name=f):0:17:@f', + }, + }, + }, + }, + { + 'Ternary:0:22:?', + children = { + 'Register(name=g):0:20:@g', + { + 'TernaryValue:0:25::', + children = { + 'Register(name=h):0:23:@h', + 'Register(name=i):0:26:@i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('Ternary', '?'), + hl('Register', '@e'), + hl('TernaryColon', ':'), + hl('Register', '@f'), + hl('TernaryColon', ':'), + hl('Register', '@g'), + hl('Ternary', '?'), + hl('Register', '@h'), + hl('TernaryColon', ':'), + hl('Register', '@i'), + }) + check_parsing('a?b{cdef}g:h', { + -- 012345678901 + -- 0 1 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:10::', + children = { + { + 'ComplexIdentifier:0:3:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'ComplexIdentifier:0:9:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:3:{'), + children = { + 'PlainIdentifier(scope=0,ident=cdef):0:4:cdef', + }, + }, + 'PlainIdentifier(scope=0,ident=g):0:9:g', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=h):0:11:h', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + hl('Curly', '{'), + hl('IdentifierName', 'cdef'), + hl('Curly', '}'), + hl('IdentifierName', 'g'), + hl('TernaryColon', ':'), + hl('IdentifierName', 'h'), + }) + check_parsing('a ? b : c : d', { + -- 0123456789012 + -- 0 1 + ast = { + { + 'Colon:0:9: :', + children = { + { + 'Ternary:0:1: ?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:5: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:11: d', + }, + }, + }, + err = { + arg = ': d', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?', 1), + hl('IdentifierName', 'b', 1), + hl('TernaryColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'd', 1), + }) + end) + itp('works with comparison operators', function() + check_parsing('a == b', { + -- 012345 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:1: ==', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ==? b', { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=IgnoreCase):0:1: ==?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('ComparisonModifier', '?'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ==# b', { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=MatchCase):0:1: ==#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a !=# b', { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=1,ccs=MatchCase):0:1: !=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '!=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a <=# b', { + -- 0123456 + ast = { + { + 'Comparison(type=Greater,inv=1,ccs=MatchCase):0:1: <=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a >=# b', { + -- 0123456 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=0,ccs=MatchCase):0:1: >=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '>=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ># b', { + -- 012345 + ast = { + { + 'Comparison(type=Greater,inv=0,ccs=MatchCase):0:1: >#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '>', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a <# b', { + -- 012345 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:1: <#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a is#b', { + -- 012345 + ast = { + { + 'Comparison(type=Identical,inv=0,ccs=MatchCase):0:1: is#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'is', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b'), + }) + + check_parsing('a is?b', { + -- 012345 + ast = { + { + 'Comparison(type=Identical,inv=0,ccs=IgnoreCase):0:1: is?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'is', 1), + hl('ComparisonModifier', '?'), + hl('IdentifierName', 'b'), + }) + + check_parsing('a isnot b', { + -- 012345678 + ast = { + { + 'Comparison(type=Identical,inv=1,ccs=UseOption):0:1: isnot', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'isnot', 1), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a < b < c', { + -- 012345678 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:5: <', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + }, + err = { + arg = ' < c', + msg = 'E15: Operator is not associative: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidComparison', '<', 1), + hl('IdentifierName', 'c', 1), + }) + + check_parsing('a < b <# c', { + -- 012345678 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:5: <#', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:8: c', + }, + }, + }, + }, + }, + err = { + arg = ' <# c', + msg = 'E15: Operator is not associative: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidComparison', '<', 1), + hl('InvalidComparisonModifier', '#'), + hl('IdentifierName', 'c', 1), + }) + + check_parsing('a += b', { + -- 012345 + ast = { + { + 'Assignment(Add):0:1: +=', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + err = { + arg = '+= b', + msg = 'E15: Misplaced assignment: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidAssignmentWithAddition', '+=', 1), + hl('IdentifierName', 'b', 1), + }) + check_parsing('a + b == c + d', { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:5: ==', + children = { + { + 'BinaryPlus:0:1: +', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + { + 'BinaryPlus:0:10: +', + children = { + 'PlainIdentifier(scope=0,ident=c):0:8: c', + 'PlainIdentifier(scope=0,ident=d):0:12: d', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'b', 1), + hl('Comparison', '==', 1), + hl('IdentifierName', 'c', 1), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'd', 1), + }) + check_parsing('+ a == + b', { + -- 0123456789 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3: ==', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1: a', + }, + }, + { + 'UnaryPlus:0:6: +', + children = { + 'PlainIdentifier(scope=0,ident=b):0:8: b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('IdentifierName', 'a', 1), + hl('Comparison', '==', 1), + hl('UnaryPlus', '+', 1), + hl('IdentifierName', 'b', 1), + }) + end) + itp('works with concat/subscript', function() + check_parsing('.', { + -- 0 + ast = { + { + 'ConcatOrSubscript:0:0:.', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = '.', + msg = 'E15: Unexpected dot: %.*s', + }, + }, { + hl('InvalidConcatOrSubscript', '.'), + }) + + check_parsing('a.', { + -- 01 + ast = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + }) + + check_parsing('a.b', { + -- 012 + ast = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=b):0:2:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', 'b'), + }) + + check_parsing('1.2', { + -- 012 + ast = { + 'Float(val=1.200000e+00):0:0:1.2', + }, + }, { + hl('Float', '1.2'), + }) + + check_parsing('1.2 + 1.3e-5', { + -- 012345678901 + -- 0 1 + ast = { + { + 'BinaryPlus:0:3: +', + children = { + 'Float(val=1.200000e+00):0:0:1.2', + 'Float(val=1.300000e-05):0:5: 1.3e-5', + }, + }, + }, + }, { + hl('Float', '1.2'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.3e-5', 1), + }) + + check_parsing('a . 1.2 + 1.3e-5', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'BinaryPlus:0:7: +', + children = { + { + 'Concat:0:1: .', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ConcatOrSubscript:0:5:.', + children = { + 'Integer(val=1):0:3: 1', + 'PlainKey(key=2):0:6:2', + }, + }, + }, + }, + 'Float(val=1.300000e-05):0:9: 1.3e-5', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.3e-5', 1), + }) + + check_parsing('1.3e-5 + 1.2 . a', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Concat:0:12: .', + children = { + { + 'BinaryPlus:0:6: +', + children = { + 'Float(val=1.300000e-05):0:0:1.3e-5', + 'Float(val=1.200000e+00):0:8: 1.2', + }, + }, + 'PlainIdentifier(scope=0,ident=a):0:14: a', + }, + }, + }, + }, { + hl('Float', '1.3e-5'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.2', 1), + hl('Concat', '.', 1), + hl('IdentifierName', 'a', 1), + }) + + check_parsing('1.3e-5 + a . 1.2', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Concat:0:10: .', + children = { + { + 'BinaryPlus:0:6: +', + children = { + 'Float(val=1.300000e-05):0:0:1.3e-5', + 'PlainIdentifier(scope=0,ident=a):0:8: a', + }, + }, + { + 'ConcatOrSubscript:0:14:.', + children = { + 'Integer(val=1):0:12: 1', + 'PlainKey(key=2):0:15:2', + }, + }, + }, + }, + }, + }, { + hl('Float', '1.3e-5'), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'a', 1), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('1.2.3', { + -- 01234 + ast = { + { + 'ConcatOrSubscript:0:3:.', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'Integer(val=1):0:0:1', + 'PlainKey(key=2):0:2:2', + }, + }, + 'PlainKey(key=3):0:4:3', + }, + }, + }, + }, { + hl('Number', '1'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '3'), + }) + + check_parsing('a.1.2', { + -- 01234 + ast = { + { + 'ConcatOrSubscript:0:3:.', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=1):0:2:1', + }, + }, + 'PlainKey(key=2):0:4:2', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '1'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('a . 1.2', { + -- 0123456 + ast = { + { + 'Concat:0:1: .', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ConcatOrSubscript:0:5:.', + children = { + 'Integer(val=1):0:3: 1', + 'PlainKey(key=2):0:6:2', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('+a . +b', { + -- 0123456 + ast = { + { + 'Concat:0:2: .', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'UnaryPlus:0:4: +', + children = { + 'PlainIdentifier(scope=0,ident=b):0:6:b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('UnaryPlus', '+', 1), + hl('IdentifierName', 'b'), + }) + + check_parsing('a. b', { + -- 0123 + ast = { + { + 'Concat:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a. 1', { + -- 0123 + ast = { + { + 'Concat:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Integer(val=1):0:2: 1', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('Number', '1', 1), + }) + + check_parsing('a[1][2][3[4', { + -- 01234567890 + -- 0 1 + ast = { + { + 'Subscript:0:7:[', + children = { + { + 'Subscript:0:4:[', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Integer(val=1):0:2:1', + }, + }, + 'Integer(val=2):0:5:2', + }, + }, + { + 'Subscript:0:9:[', + children = { + 'Integer(val=3):0:8:3', + 'Integer(val=4):0:10:4', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('Number', '1'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('Number', '2'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('Number', '3'), + hl('SubscriptBracket', '['), + hl('Number', '4'), + }) + end) + itp('works with bracket subscripts', function() + check_parsing(':', { + -- 0 + ast = { + { + 'Colon:0:0::', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ':', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('InvalidColon', ':'), + }) + check_parsing('a[]', { + -- 012 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Expected value, got closing bracket: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('InvalidSubscriptBracket', ']'), + }) + check_parsing('a[b:]', { + -- 01234 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=b,ident=):0:2:b:', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[b:c]', { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=b,ident=c):0:2:b:c', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', ']'), + }) + check_parsing('a[b : c]', { + -- 01234567 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + 'PlainIdentifier(scope=0,ident=c):0:5: c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[: b]', { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:2::', + children = { + 'Missing:0:2:', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('SubscriptColon', ':'), + hl('IdentifierName', 'b', 1), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[b :]', { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptColon', ':', 1), + hl('SubscriptBracket', ']'), + }) + check_parsing('a[b][c][d](e)(f)(g)', { + -- 0123456789012345678 + -- 0 1 + ast = { + { + 'Call:0:16:(', + children = { + { + 'Call:0:13:(', + children = { + { + 'Call:0:10:(', + children = { + { + 'Subscript:0:7:[', + children = { + { + 'Subscript:0:4:[', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:8:d', + }, + }, + 'PlainIdentifier(scope=0,ident=e):0:11:e', + }, + }, + 'PlainIdentifier(scope=0,ident=f):0:14:f', + }, + }, + 'PlainIdentifier(scope=0,ident=g):0:17:g', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'd'), + hl('SubscriptBracket', ']'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'e'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'f'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'g'), + hl('CallingParenthesis', ')'), + }) + check_parsing('{a}{b}{c}[d][e][f]', { + -- 012345678901234567 + -- 0 1 + ast = { + { + 'Subscript:0:15:[', + children = { + { + 'Subscript:0:12:[', + children = { + { + 'Subscript:0:9:[', + children = { + { + 'ComplexIdentifier:0:3:', + children = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'ComplexIdentifier:0:6:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:3:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + }, + }, + { + fmtn('CurlyBracesIdentifier', '--i', ':0:6:{'), + children = { + 'PlainIdentifier(scope=0,ident=c):0:7:c', + }, + }, + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:10:d', + }, + }, + 'PlainIdentifier(scope=0,ident=e):0:13:e', + }, + }, + 'PlainIdentifier(scope=0,ident=f):0:16:f', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'c'), + hl('Curly', '}'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'd'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'e'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'f'), + hl('SubscriptBracket', ']'), + }) + end) + itp('supports list literals', function() + check_parsing('[]', { + -- 01 + ast = { + 'ListLiteral:0:0:[', + }, + }, { + hl('List', '['), + hl('List', ']'), + }) + + check_parsing('[a]', { + -- 012 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('List', ']'), + }) + + check_parsing('[a, b]', { + -- 012345 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('List', ']'), + }) + + check_parsing('[a, b, c]', { + -- 012345678 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('List', ']'), + }) + + check_parsing('[a, b, c, ]', { + -- 01234567890 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('Comma', ','), + hl('List', ']', 1), + }) + + check_parsing('[a : b, c : d]', { + -- 01234567890123 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + { + 'Colon:0:9: :', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7: c', + 'PlainIdentifier(scope=0,ident=d):0:11: d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ': b, c : d]', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'd', 1), + hl('List', ']'), + }) + + check_parsing(']', { + -- 0 + ast = { + 'ListLiteral:0:0:', + }, + err = { + arg = ']', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidList', ']'), + }) + + check_parsing('a]', { + -- 01 + ast = { + { + 'ListLiteral:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidList', ']'), + }) + + check_parsing('[] []', { + -- 01234 + ast = { + { + 'OpMissing:0:2:', + children = { + 'ListLiteral:0:0:[', + 'ListLiteral:0:2: [', + }, + }, + }, + err = { + arg = '[]', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('List', '['), + hl('List', ']'), + hl('InvalidSpacing', ' '), + hl('List', '['), + hl('List', ']'), + }, { + [1] = { + ast = { + len = 3, + err = REMOVE_THIS, + ast = { + 'ListLiteral:0:0:[', + }, + }, + hl_fs = { + [3] = REMOVE_THIS, + [4] = REMOVE_THIS, + [5] = REMOVE_THIS, + }, + }, + }) + + check_parsing('[][]', { + -- 0123 + ast = { + { + 'Subscript:0:2:[', + children = { + 'ListLiteral:0:0:[', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Expected value, got closing bracket: %.*s', + }, + }, { + hl('List', '['), + hl('List', ']'), + hl('SubscriptBracket', '['), + hl('InvalidSubscriptBracket', ']'), + }) + + check_parsing('[', { + -- 0 + ast = { + 'ListLiteral:0:0:[', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('List', '['), + }) + + check_parsing('[1', { + -- 01 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'Integer(val=1):0:1:1', + }, + }, + }, + err = { + arg = '[1', + msg = 'E697: Missing end of List \']\': %.*s', + }, + }, { + hl('List', '['), + hl('Number', '1'), + }) + end) + itp('works with strings', function() + check_parsing('\'abc\'', { + -- 01234 + ast = { + fmtn('SingleQuotedString', 'val="abc"', ':0:0:\'abc\''), + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedBody', 'abc'), + hl('SingleQuote', '\''), + }) + check_parsing('"abc"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="abc"', ':0:0:"abc"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', 'abc'), + hl('DoubleQuote', '"'), + }) + check_parsing('\'\'', { + -- 01 + ast = { + fmtn('SingleQuotedString', 'val=NULL', ':0:0:\'\''), + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuote', '\''), + }) + check_parsing('""', { + -- 01 + ast = { + fmtn('DoubleQuotedString', 'val=NULL', ':0:0:""'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuote', '"'), + }) + check_parsing('"', { + -- 0 + ast = { + fmtn('DoubleQuotedString', 'val=NULL', ':0:0:"'), + }, + err = { + arg = '"', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + }) + check_parsing('\'', { + -- 0 + ast = { + fmtn('SingleQuotedString', 'val=NULL', ':0:0:\''), + }, + err = { + arg = '\'', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + }) + check_parsing('"a', { + -- 01 + ast = { + fmtn('DoubleQuotedString', 'val="a"', ':0:0:"a'), + }, + err = { + arg = '"a', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedBody', 'a'), + }) + check_parsing('\'a', { + -- 01 + ast = { + fmtn('SingleQuotedString', 'val="a"', ':0:0:\'a'), + }, + err = { + arg = '\'a', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + hl('InvalidSingleQuotedBody', 'a'), + }) + check_parsing('\'abc\'\'def\'', { + -- 0123456789 + ast = { + fmtn('SingleQuotedString', 'val="abc\'def"', ':0:0:\'abc\'\'def\''), + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedBody', 'abc'), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'def'), + hl('SingleQuote', '\''), + }) + check_parsing('\'abc\'\'', { + -- 012345 + ast = { + fmtn('SingleQuotedString', 'val="abc\'"', ':0:0:\'abc\'\''), + }, + err = { + arg = '\'abc\'\'', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + hl('InvalidSingleQuotedBody', 'abc'), + hl('InvalidSingleQuotedQuote', '\'\''), + }) + check_parsing('\'\'\'\'\'\'\'\'', { + -- 01234567 + ast = { + fmtn('SingleQuotedString', 'val="\'\'\'"', ':0:0:\'\'\'\'\'\'\'\''), + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuote', '\''), + }) + check_parsing('\'\'\'a\'\'\'\'bc\'', { + -- 01234567890 + -- 0 1 + ast = { + fmtn('SingleQuotedString', 'val="\'a\'\'bc"', ':0:0:\'\'\'a\'\'\'\'bc\''), + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'a'), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'bc'), + hl('SingleQuote', '\''), + }) + check_parsing('"\\"\\"\\"\\""', { + -- 0123456789 + ast = { + fmtn('DoubleQuotedString', 'val="\\"\\"\\"\\""', ':0:0:"\\"\\"\\"\\""'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuote', '"'), + }) + check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', { + -- 0123456789012345678901234 + -- 0 1 2 + ast = { + fmtn('DoubleQuotedString', 'val="abc\\"def\\"ghi\\"jkl\\"mno"', ':0:0:"abc\\"def\\"ghi\\"jkl\\"mno"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', 'abc'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'def'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'ghi'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'jkl'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'mno'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\b\\e\\f\\r\\t\\\\"', { + -- 0123456789012345 + -- 0 1 + ast = { + [[DoubleQuotedString(val="\8\27\12\13\9\\"):0:0:"\b\e\f\r\t\\"]], + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\b'), + hl('DoubleQuotedEscape', '\\e'), + hl('DoubleQuotedEscape', '\\f'), + hl('DoubleQuotedEscape', '\\r'), + hl('DoubleQuotedEscape', '\\t'), + hl('DoubleQuotedEscape', '\\\\'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\n\n"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="\\\n\\\n"', ':0:0:"\\n\n"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\n'), + hl('DoubleQuotedBody', '\n'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\x00"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0"', ':0:0:"\\x00"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\xFF"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\255"', ':0:0:"\\xFF"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xFF'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\xF"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\15"', ':0:0:"\\xF"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xF'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\u00AB"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="«"', ':0:0:"\\u00AB"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u00AB'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\U000000AB"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="«"', ':0:0:"\\U000000AB"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000000AB'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\x"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="x"', ':0:0:"\\x"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\x'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="x"', ':0:0:"\\x'), + }, + err = { + arg = '"\\x', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\x'), + }) + + check_parsing('"\\xF', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="\\15"', ':0:0:"\\xF'), + }, + err = { + arg = '"\\xF', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedEscape', '\\xF'), + }) + + check_parsing('"\\u"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="u"', ':0:0:"\\u"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\u'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="u"', ':0:0:"\\u'), + }, + err = { + arg = '"\\u', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\u'), + }) + + check_parsing('"\\U', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="U"', ':0:0:"\\U'), + }, + err = { + arg = '"\\U', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + }) + + check_parsing('"\\U"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="U"', ':0:0:"\\U"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\U'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\xFX"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\15X"', ':0:0:"\\xFX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xF'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\XFX"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\15X"', ':0:0:"\\XFX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\XF'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\xX"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="xX"', ':0:0:"\\xX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\x'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\XX"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="XX"', ':0:0:"\\XX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\X'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\uX"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="uX"', ':0:0:"\\uX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\u'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\UX"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="UX"', ':0:0:"\\UX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\U'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x0X"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\x0X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X0X"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\X0X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u0X"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\u0X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0X"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U0X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x00X"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\x00X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X00X"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\X00X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u00X"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\u00X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00X"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U00X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u000X"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\u000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000X"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u0000X"', { + -- 012345678 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\u0000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0000X"', { + -- 012345678 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U0000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00000X"', { + -- 0123456789 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U00000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000000X"', { + -- 01234567890 + -- 0 1 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U000000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0000000X"', { + -- 012345678901 + -- 0 1 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U0000000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00000000X"', { + -- 0123456789012 + -- 0 1 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U00000000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x000X"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="\\0000X"', ':0:0:"\\x000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X000X"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="\\0000X"', ':0:0:"\\X000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X00'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u00000X"', { + -- 0123456789 + ast = { + fmtn('DoubleQuotedString', 'val="\\0000X"', ':0:0:"\\u00000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0000'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000000000X"', { + -- 01234567890123 + -- 0 1 + ast = { + fmtn('DoubleQuotedString', 'val="\\0000X"', ':0:0:"\\U000000000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000000'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="\\0"', ':0:0:"\\0"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\0'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\00"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="\\0"', ':0:0:"\\00"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\00'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\000"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0"', ':0:0:"\\000"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0000"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0000"', ':0:0:"\\0000"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedBody', '0'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\8"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="8"', ':0:0:"\\8"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\08"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="\\0008"', ':0:0:"\\08"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\0'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\008"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0008"', ':0:0:"\\008"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\00'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0008"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0008"', ':0:0:"\\0008"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\777"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\255"', ':0:0:"\\777"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\777'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\050"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\40"', ':0:0:"\\050"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\050'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\<C-u>"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\21"', ':0:0:"\\<C-u>"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\<C-u>'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\<', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="<"', ':0:0:"\\<'), + }, + err = { + arg = '"\\<', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\<'), + }) + + check_parsing('"\\<"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="<"', ':0:0:"\\<"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\<'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\<C-u"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="<C-u"', ':0:0:"\\<C-u"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\<'), + hl('DoubleQuotedBody', 'C-u'), + hl('DoubleQuote', '"'), + }) + end) + itp('works with multiplication-like operators', function() + check_parsing('2+2*2', { + -- 01234 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Multiplication:0:3:*', + children = { + 'Integer(val=2):0:2:2', + 'Integer(val=2):0:4:2', + }, + }, + }, + }, + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Multiplication', '*'), + hl('Number', '2'), + }) + + check_parsing('2+2*', { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Multiplication:0:3:*', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Multiplication', '*'), + }) + + check_parsing('2+*2', { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Multiplication:0:2:*', + children = { + 'Missing:0:2:', + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + err = { + arg = '*2', + msg = 'E15: Unexpected multiplication-like operator: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('InvalidMultiplication', '*'), + hl('Number', '2'), + }) + + check_parsing('2+2/2', { + -- 01234 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Division:0:3:/', + children = { + 'Integer(val=2):0:2:2', + 'Integer(val=2):0:4:2', + }, + }, + }, + }, + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Division', '/'), + hl('Number', '2'), + }) + + check_parsing('2+2/', { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Division:0:3:/', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Division', '/'), + }) + + check_parsing('2+/2', { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Division:0:2:/', + children = { + 'Missing:0:2:', + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + err = { + arg = '/2', + msg = 'E15: Unexpected multiplication-like operator: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('InvalidDivision', '/'), + hl('Number', '2'), + }) + + check_parsing('2+2%2', { + -- 01234 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Mod:0:3:%', + children = { + 'Integer(val=2):0:2:2', + 'Integer(val=2):0:4:2', + }, + }, + }, + }, + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Mod', '%'), + hl('Number', '2'), + }) + + check_parsing('2+2%', { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Mod:0:3:%', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Mod', '%'), + }) + + check_parsing('2+%2', { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Mod:0:2:%', + children = { + 'Missing:0:2:', + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + err = { + arg = '%2', + msg = 'E15: Unexpected multiplication-like operator: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('InvalidMod', '%'), + hl('Number', '2'), + }) + end) + itp('works with -', function() + check_parsing('@a', { + ast = { + 'Register(name=a):0:0:@a', + }, + }, { + hl('Register', '@a'), + }) + check_parsing('-@a', { + ast = { + { + 'UnaryMinus:0:0:-', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('UnaryMinus', '-'), + hl('Register', '@a'), + }) + check_parsing('@a-@b', { + ast = { + { + 'BinaryMinus:0:2:-', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('Register', '@b'), + }) + check_parsing('@a-@b-@c', { + ast = { + { + 'BinaryMinus:0:5:-', + children = { + { + 'BinaryMinus:0:2:-', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + 'Register(name=c):0:6:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('Register', '@b'), + hl('BinaryMinus', '-'), + hl('Register', '@c'), + }) + check_parsing('-@a-@b', { + ast = { + { + 'BinaryMinus:0:3:-', + children = { + { + 'UnaryMinus:0:0:-', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:4:@b', + }, + }, + }, + }, { + hl('UnaryMinus', '-'), + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('Register', '@b'), + }) + check_parsing('-@a--@b', { + ast = { + { + 'BinaryMinus:0:3:-', + children = { + { + 'UnaryMinus:0:0:-', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'UnaryMinus:0:4:-', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryMinus', '-'), + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('UnaryMinus', '-'), + hl('Register', '@b'), + }) + check_parsing('-', { + ast = { + 'UnaryMinus:0:0:-', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryMinus', '-'), + }) + check_parsing(' -', { + ast = { + 'UnaryMinus:0:0: -', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryMinus', '-', 1), + }) + check_parsing('@a- ', { + ast = { + { + 'BinaryMinus:0:2:-', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Register', '@a'), + hl('BinaryMinus', '-'), + }) + end) + itp('works with logical operators', function() + check_parsing('a && b || c && d', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Or:0:6: ||', + children = { + { + 'And:0:1: &&', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + { + 'And:0:11: &&', + children = { + 'PlainIdentifier(scope=0,ident=c):0:9: c', + 'PlainIdentifier(scope=0,ident=d):0:14: d', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('And', '&&', 1), + hl('IdentifierName', 'b', 1), + hl('Or', '||', 1), + hl('IdentifierName', 'c', 1), + hl('And', '&&', 1), + hl('IdentifierName', 'd', 1), + }) + + check_parsing('&& a', { + -- 0123 + ast = { + { + 'And:0:0:&&', + children = { + 'Missing:0:0:', + 'PlainIdentifier(scope=0,ident=a):0:2: a', + }, + }, + }, + err = { + arg = '&& a', + msg = 'E15: Unexpected and operator: %.*s', + }, + }, { + hl('InvalidAnd', '&&'), + hl('IdentifierName', 'a', 1), + }) + + check_parsing('|| a', { + -- 0123 + ast = { + { + 'Or:0:0:||', + children = { + 'Missing:0:0:', + 'PlainIdentifier(scope=0,ident=a):0:2: a', + }, + }, + }, + err = { + arg = '|| a', + msg = 'E15: Unexpected or operator: %.*s', + }, + }, { + hl('InvalidOr', '||'), + hl('IdentifierName', 'a', 1), + }) + + check_parsing('a||', { + -- 012 + ast = { + { + 'Or:0:1:||', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Or', '||'), + }) + + check_parsing('a&&', { + -- 012 + ast = { + { + 'And:0:1:&&', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('And', '&&'), + }) + + check_parsing('(&&)', { + -- 0123 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'And:0:1:&&', + children = { + 'Missing:0:1:', + 'Missing:0:3:', + }, + }, + }, + }, + }, + err = { + arg = '&&)', + msg = 'E15: Unexpected and operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidAnd', '&&'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(||)', { + -- 0123 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Or:0:1:||', + children = { + 'Missing:0:1:', + 'Missing:0:3:', + }, + }, + }, + }, + }, + err = { + arg = '||)', + msg = 'E15: Unexpected or operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidOr', '||'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(a||)', { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Or:0:2:||', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('IdentifierName', 'a'), + hl('Or', '||'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(a&&)', { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'And:0:2:&&', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('IdentifierName', 'a'), + hl('And', '&&'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(&&a)', { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'And:0:1:&&', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:3:a', + }, + }, + }, + }, + }, + err = { + arg = '&&a)', + msg = 'E15: Unexpected and operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidAnd', '&&'), + hl('IdentifierName', 'a'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(||a)', { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Or:0:1:||', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:3:a', + }, + }, + }, + }, + }, + err = { + arg = '||a)', + msg = 'E15: Unexpected or operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidOr', '||'), + hl('IdentifierName', 'a'), + hl('NestingParenthesis', ')'), + }) + end) + itp('works with &opt', function() + check_parsing('&', { + -- 0 + ast = { + 'Option(scope=0,ident=):0:0:&', + }, + err = { + arg = '&', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('InvalidOptionSigil', '&'), + }) + + check_parsing('&opt', { + -- 0123 + ast = { + 'Option(scope=0,ident=opt):0:0:&opt', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 'opt'), + }) + + check_parsing('&l:opt', { + -- 012345 + ast = { + 'Option(scope=l,ident=opt):0:0:&l:opt', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionScope', 'l'), + hl('OptionScopeDelimiter', ':'), + hl('OptionName', 'opt'), + }) + + check_parsing('&g:opt', { + -- 012345 + ast = { + 'Option(scope=g,ident=opt):0:0:&g:opt', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionScope', 'g'), + hl('OptionScopeDelimiter', ':'), + hl('OptionName', 'opt'), + }) + + check_parsing('&s:opt', { + -- 012345 + ast = { + { + 'Colon:0:2::', + children = { + 'Option(scope=0,ident=s):0:0:&s', + 'PlainIdentifier(scope=0,ident=opt):0:3:opt', + }, + }, + }, + err = { + arg = ':opt', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 's'), + hl('InvalidColon', ':'), + hl('IdentifierName', 'opt'), + }) + + check_parsing('& ', { + -- 01 + ast = { + 'Option(scope=0,ident=):0:0:&', + }, + err = { + arg = '& ', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('InvalidOptionSigil', '&'), + }) + + check_parsing('&-', { + -- 01 + ast = { + { + 'BinaryMinus:0:1:-', + children = { + 'Option(scope=0,ident=):0:0:&', + }, + }, + }, + err = { + arg = '&-', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('InvalidOptionSigil', '&'), + hl('BinaryMinus', '-'), + }) + + check_parsing('&A', { + -- 01 + ast = { + 'Option(scope=0,ident=A):0:0:&A', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 'A'), + }) + + check_parsing('&xxx_yyy', { + -- 01234567 + ast = { + { + 'OpMissing:0:4:', + children = { + 'Option(scope=0,ident=xxx):0:0:&xxx', + 'PlainIdentifier(scope=0,ident=_yyy):0:4:_yyy', + }, + }, + }, + err = { + arg = '_yyy', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 'xxx'), + hl('InvalidIdentifierName', '_yyy'), + }, { + [1] = { + ast = { + len = 4, + err = REMOVE_THIS, + ast = { + 'Option(scope=0,ident=xxx):0:0:&xxx', + }, + }, + hl_fs = { + [3] = REMOVE_THIS, + }, + }, + }) + + check_parsing('(1+&)', { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Integer(val=1):0:1:1', + 'Option(scope=0,ident=):0:3:&', + }, + }, + }, + }, + }, + err = { + arg = '&)', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Number', '1'), + hl('BinaryPlus', '+'), + hl('InvalidOptionSigil', '&'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(&+1)', { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Option(scope=0,ident=):0:1:&', + 'Integer(val=1):0:3:1', + }, + }, + }, + }, + }, + err = { + arg = '&+1)', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidOptionSigil', '&'), + hl('BinaryPlus', '+'), + hl('Number', '1'), + hl('NestingParenthesis', ')'), + }) + end) + itp('works with $ENV', function() + check_parsing('$', { + -- 0 + ast = { + 'Environment(ident=):0:0:$', + }, + err = { + arg = '$', + msg = 'E15: Environment variable name missing', + }, + }, { + hl('InvalidEnvironmentSigil', '$'), + }) + + check_parsing('$g:A', { + -- 0123 + ast = { + { + 'Colon:0:2::', + children = { + 'Environment(ident=g):0:0:$g', + 'PlainIdentifier(scope=0,ident=A):0:3:A', + }, + }, + }, + err = { + arg = ':A', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'g'), + hl('InvalidColon', ':'), + hl('IdentifierName', 'A'), + }) + + check_parsing('$A', { + -- 01 + ast = { + 'Environment(ident=A):0:0:$A', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'A'), + }) + + check_parsing('$ABC', { + -- 0123 + ast = { + 'Environment(ident=ABC):0:0:$ABC', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'ABC'), + }) + + check_parsing('(1+$)', { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Integer(val=1):0:1:1', + 'Environment(ident=):0:3:$', + }, + }, + }, + }, + }, + err = { + arg = '$)', + msg = 'E15: Environment variable name missing', + }, + }, { + hl('NestingParenthesis', '('), + hl('Number', '1'), + hl('BinaryPlus', '+'), + hl('InvalidEnvironmentSigil', '$'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('($+1)', { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Environment(ident=):0:1:$', + 'Integer(val=1):0:3:1', + }, + }, + }, + }, + }, + err = { + arg = '$+1)', + msg = 'E15: Environment variable name missing', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidEnvironmentSigil', '$'), + hl('BinaryPlus', '+'), + hl('Number', '1'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('$_ABC', { + -- 01234 + ast = { + 'Environment(ident=_ABC):0:0:$_ABC', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', '_ABC'), + }) + + check_parsing('$_', { + -- 01 + ast = { + 'Environment(ident=_):0:0:$_', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', '_'), + }) + + check_parsing('$ABC_DEF', { + -- 01234567 + ast = { + 'Environment(ident=ABC_DEF):0:0:$ABC_DEF', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'ABC_DEF'), + }) + end) + itp('works with unary !', function() + check_parsing('!', { + -- 0 + ast = { + 'Not:0:0:!', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Not', '!'), + }) + + check_parsing('!!', { + -- 01 + ast = { + { + 'Not:0:0:!', + children = { + 'Not:0:1:!', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Not', '!'), + hl('Not', '!'), + }) + + check_parsing('!!1', { + -- 012 + ast = { + { + 'Not:0:0:!', + children = { + { + 'Not:0:1:!', + children = { + 'Integer(val=1):0:2:1', + }, + }, + }, + }, + }, + }, { + hl('Not', '!'), + hl('Not', '!'), + hl('Number', '1'), + }) + + check_parsing('!1', { + -- 01 + ast = { + { + 'Not:0:0:!', + children = { + 'Integer(val=1):0:1:1', + }, + }, + }, + }, { + hl('Not', '!'), + hl('Number', '1'), + }) + + check_parsing('(!1)', { + -- 0123 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Not:0:1:!', + children = { + 'Integer(val=1):0:2:1', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Not', '!'), + hl('Number', '1'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(!)', { + -- 012 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Not:0:1:!', + children = { + 'Missing:0:2:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Not', '!'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(1!2)', { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'OpMissing:0:2:', + children = { + 'Integer(val=1):0:1:1', + { + 'Not:0:2:!', + children = { + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '!2)', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Number', '1'), + hl('InvalidNot', '!'), + hl('Number', '2'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('1!2', { + -- 012 + ast = { + { + 'OpMissing:0:1:', + children = { + 'Integer(val=1):0:0:1', + { + 'Not:0:1:!', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '!2', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Number', '1'), + hl('InvalidNot', '!'), + hl('Number', '2'), + }, { + [1] = { + ast = { + len = 1, + err = REMOVE_THIS, + ast = { + 'Integer(val=1):0:0:1', + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + [3] = REMOVE_THIS, + }, + }, + }) + end) + itp('highlights numbers with prefix', function() + check_parsing('0xABCDEF', { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0xABCDEF', + }, + }, { + hl('NumberPrefix', '0x'), + hl('Number', 'ABCDEF'), + }) + + check_parsing('0Xabcdef', { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0Xabcdef', + }, + }, { + hl('NumberPrefix', '0X'), + hl('Number', 'abcdef'), + }) + + check_parsing('0XABCDEF', { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0XABCDEF', + }, + }, { + hl('NumberPrefix', '0X'), + hl('Number', 'ABCDEF'), + }) + + check_parsing('0xabcdef', { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0xabcdef', + }, + }, { + hl('NumberPrefix', '0x'), + hl('Number', 'abcdef'), + }) + + check_parsing('0b001', { + -- 01234 + ast = { + 'Integer(val=1):0:0:0b001', + }, + }, { + hl('NumberPrefix', '0b'), + hl('Number', '001'), + }) + + check_parsing('0B001', { + -- 01234 + ast = { + 'Integer(val=1):0:0:0B001', + }, + }, { + hl('NumberPrefix', '0B'), + hl('Number', '001'), + }) + + check_parsing('0B00', { + -- 0123 + ast = { + 'Integer(val=0):0:0:0B00', + }, + }, { + hl('NumberPrefix', '0B'), + hl('Number', '00'), + }) + + check_parsing('00', { + -- 01 + ast = { + 'Integer(val=0):0:0:00', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '0'), + }) + + check_parsing('001', { + -- 012 + ast = { + 'Integer(val=1):0:0:001', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '01'), + }) + + check_parsing('01', { + -- 01 + ast = { + 'Integer(val=1):0:0:01', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '1'), + }) + + check_parsing('1', { + -- 0 + ast = { + 'Integer(val=1):0:0:1', + }, + }, { + hl('Number', '1'), + }) + end) + itp('works (KLEE tests)', function() + check_parsing('\0002&A:\000', { + len = 0, + ast = nil, + err = { + arg = '\0002&A:\000', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + }, { + [2] = { + ast = { + len = REMOVE_THIS, + ast = { + { + 'Colon:0:4::', + children = { + { + 'OpMissing:0:2:', + children = { + 'Integer(val=2):0:1:2', + 'Option(scope=0,ident=A):0:2:&A', + }, + }, + }, + }, + }, + err = { + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, + hl_fs = { + hl('InvalidSpacing', '\0'), + hl('Number', '2'), + hl('InvalidOptionSigil', '&'), + hl('InvalidOptionName', 'A'), + hl('InvalidColon', ':'), + hl('InvalidSpacing', '\0'), + }, + }, + [3] = { + ast = { + len = 2, + ast = { + 'Integer(val=2):0:1:2', + }, + err = { + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, + hl_fs = { + hl('InvalidSpacing', '\0'), + hl('Number', '2'), + }, + }, + }) + check_parsing({data='01', size=1}, { + len = 1, + ast = { + 'Integer(val=0):0:0:0', + }, + }, { + hl('Number', '0'), + }) + check_parsing({data='001', size=2}, { + len = 2, + ast = { + 'Integer(val=0):0:0:00', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '0'), + }) + check_parsing('"\\U\\', { + -- 0123 + ast = { + [[DoubleQuotedString(val="U\\"):0:0:"\U\]], + }, + err = { + arg = '"\\U\\', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + hl('InvalidDoubleQuotedBody', '\\'), + }) + check_parsing('"\\U', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="U"', ':0:0:"\\U'), + }, + err = { + arg = '"\\U', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + }) + check_parsing('|"\\U\\', { + -- 01234 + len = 0, + err = { + arg = '|"\\U\\', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + }, { + [2] = { + ast = { + len = REMOVE_THIS, + ast = { + { + 'Or:0:0:|', + children = { + 'Missing:0:0:', + fmtn('DoubleQuotedString', 'val="U\\\\"', ':0:1:"\\U\\'), + }, + }, + }, + err = { + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, + hl_fs = { + hl('InvalidOr', '|'), + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + hl('InvalidDoubleQuotedBody', '\\'), + }, + }, + }) + check_parsing('|"\\e"', { + -- 01234 + len = 0, + err = { + arg = '|"\\e"', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + }, { + [2] = { + ast = { + len = REMOVE_THIS, + ast = { + { + 'Or:0:0:|', + children = { + 'Missing:0:0:', + fmtn('DoubleQuotedString', 'val="\\27"', ':0:1:"\\e"'), + }, + }, + }, + err = { + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, + hl_fs = { + hl('InvalidOr', '|'), + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\e'), + hl('DoubleQuote', '"'), + }, + }, + }) + check_parsing('|\029', { + -- 01 + len = 0, + err = { + arg = '|\029', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + }, { + [2] = { + ast = { + len = REMOVE_THIS, + ast = { + { + 'Or:0:0:|', + children = { + 'Missing:0:0:', + 'PlainIdentifier(scope=0,ident=\029):0:1:\029', + }, + }, + }, + err = { + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, + hl_fs = { + hl('InvalidOr', '|'), + hl('InvalidIdentifierName', '\029'), + }, + }, + }) + check_parsing('"\\<', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="<"', ':0:0:"\\<'), + }, + err = { + arg = '"\\<', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\<'), + }) + check_parsing('"\\1', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="\\1"', ':0:0:"\\1'), + }, + err = { + arg = '"\\1', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedEscape', '\\1'), + }) + check_parsing('}l', { + -- 01 + ast = { + { + 'OpMissing:0:1:', + children = { + fmtn('UnknownFigure', '---', ':0:0:'), + 'PlainIdentifier(scope=0,ident=l):0:1:l', + }, + }, + }, + err = { + arg = '}l', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidFigureBrace', '}'), + hl('InvalidIdentifierName', 'l'), + }, { + [1] = { + ast = { + len = 1, + ast = { + fmtn('UnknownFigure', '---', ':0:0:'), + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + }, + }, + }) + check_parsing(':?\000\000\000\000\000\000\000', { + len = 2, + ast = { + { + 'Colon:0:0::', + children = { + 'Missing:0:0:', + { + 'Ternary:0:1:?', + children = { + 'Missing:0:1:', + 'TernaryValue:0:1:?', + }, + }, + }, + }, + }, + err = { + arg = ':?\000\000\000\000\000\000\000', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('InvalidColon', ':'), + hl('InvalidTernary', '?'), + }, { + [2] = { + ast = { + len = REMOVE_THIS, + }, + hl_fs = { + [3] = hl('InvalidSpacing', '\0'), + [4] = hl('InvalidSpacing', '\0'), + [5] = hl('InvalidSpacing', '\0'), + [6] = hl('InvalidSpacing', '\0'), + [7] = hl('InvalidSpacing', '\0'), + [8] = hl('InvalidSpacing', '\0'), + [9] = hl('InvalidSpacing', '\0'), + }, + }, + }) + end) + itp('works with assignments', function() + check_asgn_parsing('a=b', { + -- 012 + ast = { + { + 'Assignment(Plain):0:1:=', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('PlainAssignment', '='), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a+=b', { + -- 0123 + ast = { + { + 'Assignment(Add):0:1:+=', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('AssignmentWithAddition', '+='), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a-=b', { + -- 0123 + ast = { + { + 'Assignment(Subtract):0:1:-=', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('AssignmentWithSubtraction', '-='), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a.=b', { + -- 0123 + ast = { + { + 'Assignment(Concat):0:1:.=', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('AssignmentWithConcatenation', '.='), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a', { + -- 0 + ast = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, { + hl('IdentifierName', 'a'), + }) + + check_asgn_parsing('a b', { + -- 012 + ast = { + { + 'OpMissing:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:1: b', + }, + }, + }, + err = { + arg = 'b', + msg = 'E15: Expected assignment operator or subscript: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidSpacing', ' '), + hl('IdentifierName', 'b'), + }, { + [5] = { + ast = { + len = 2, + ast = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + err = REMOVE_THIS, + }, + hl_fs = { + [2] = REMOVE_THIS, + [3] = REMOVE_THIS, + } + }, + }) + + check_asgn_parsing('[a, b, c]', { + -- 012345678 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('List', ']'), + }) + + check_asgn_parsing('[a, b]', { + -- 012345 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('List', ']'), + }) + + check_asgn_parsing('[a]', { + -- 012 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('List', ']'), + }) + + check_asgn_parsing('[]', { + -- 01 + ast = { + 'ListLiteral:0:0:[', + }, + err = { + arg = ']', + msg = 'E475: Unable to assign to empty list: %.*s', + }, + }, { + hl('List', '['), + hl('InvalidList', ']'), + }) + + check_asgn_parsing('a[1] += 3', { + -- 012345678 + ast = { + { + 'Assignment(Add):0:4: +=', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Integer(val=1):0:2:1', + }, + }, + 'Integer(val=3):0:7: 3', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('Number', '1'), + hl('SubscriptBracket', ']'), + hl('AssignmentWithAddition', '+=', 1), + hl('Number', '3', 1), + }) + + check_asgn_parsing('a[1 + 2] += 3', { + -- 0123456789012 + -- 0 1 + ast = { + { + 'Assignment(Add):0:8: +=', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'BinaryPlus:0:3: +', + children = { + 'Integer(val=1):0:2:1', + 'Integer(val=2):0:5: 2', + }, + }, + }, + }, + 'Integer(val=3):0:11: 3', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('Number', '1'), + hl('BinaryPlus', '+', 1), + hl('Number', '2', 1), + hl('SubscriptBracket', ']'), + hl('AssignmentWithAddition', '+=', 1), + hl('Number', '3', 1), + }) + + check_asgn_parsing('a[{-> {b{3}: 4}[5]}()] += 6', { + -- 012345678901234567890123456 + -- 0 1 2 + ast = { + { + 'Assignment(Add):0:22: +=', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Call:0:19:(', + children = { + { + fmtn('Lambda', '\\di', ':0:2:{'), + children = { + { + 'Arrow:0:3:->', + children = { + { + 'Subscript:0:15:[', + children = { + { + fmtn('DictLiteral', '-di', ':0:5: {'), + children = { + { + 'Colon:0:11::', + children = { + { + 'ComplexIdentifier:0:8:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:7:b', + { + fmtn('CurlyBracesIdentifier', '--i', ':0:8:{'), + children = { + 'Integer(val=3):0:9:3', + }, + }, + }, + }, + 'Integer(val=4):0:12: 4', + }, + }, + }, + }, + 'Integer(val=5):0:16:5', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + 'Integer(val=6):0:25: 6', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Dict', '{', 1), + hl('IdentifierName', 'b'), + hl('Curly', '{'), + hl('Number', '3'), + hl('Curly', '}'), + hl('Colon', ':'), + hl('Number', '4', 1), + hl('Dict', '}'), + hl('SubscriptBracket', '['), + hl('Number', '5'), + hl('SubscriptBracket', ']'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('SubscriptBracket', ']'), + hl('AssignmentWithAddition', '+=', 1), + hl('Number', '6', 1), + }) + + check_asgn_parsing('a{1}.2[{-> {b{3}: 4}[5]}()]', { + -- 012345678901234567890123456 + -- 0 1 2 + ast = { + { + 'Subscript:0:6:[', + children = { + { + 'ConcatOrSubscript:0:4:.', + children = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + fmtn('CurlyBracesIdentifier', '--i', ':0:1:{'), + children = { + 'Integer(val=1):0:2:1', + }, + }, + }, + }, + 'PlainKey(key=2):0:5:2', + }, + }, + { + 'Call:0:24:(', + children = { + { + fmtn('Lambda', '\\di', ':0:7:{'), + children = { + { + 'Arrow:0:8:->', + children = { + { + 'Subscript:0:20:[', + children = { + { + fmtn('DictLiteral', '-di', ':0:10: {'), + children = { + { + 'Colon:0:16::', + children = { + { + 'ComplexIdentifier:0:13:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:12:b', + { + fmtn('CurlyBracesIdentifier', '--i', ':0:13:{'), + children = { + 'Integer(val=3):0:14:3', + }, + }, + }, + }, + 'Integer(val=4):0:17: 4', + }, + }, + }, + }, + 'Integer(val=5):0:21:5', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('Number', '1'), + hl('Curly', '}'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('SubscriptBracket', '['), + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Dict', '{', 1), + hl('IdentifierName', 'b'), + hl('Curly', '{'), + hl('Number', '3'), + hl('Curly', '}'), + hl('Colon', ':'), + hl('Number', '4', 1), + hl('Dict', '}'), + hl('SubscriptBracket', '['), + hl('Number', '5'), + hl('SubscriptBracket', ']'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('SubscriptBracket', ']'), + }) + + check_asgn_parsing('a', { + -- 0 + ast = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, { + hl('IdentifierName', 'a'), + }) + + check_asgn_parsing('{a}', { + -- 012 + ast = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + }) + + check_asgn_parsing('{a}b', { + -- 0123 + ast = { + { + 'ComplexIdentifier:0:3:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a{b}c', { + -- 01234 + ast = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ComplexIdentifier:0:4:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:1:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + }) + + check_asgn_parsing('a{b}c[0]', { + -- 01234567 + ast = { + { + 'Subscript:0:5:[', + children = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ComplexIdentifier:0:4:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:1:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + }, + }, + 'Integer(val=0):0:6:0', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', '['), + hl('Number', '0'), + hl('SubscriptBracket', ']'), + }) + + check_asgn_parsing('a{b}c.0', { + -- 0123456 + ast = { + { + 'ConcatOrSubscript:0:5:.', + children = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ComplexIdentifier:0:4:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:1:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + }, + }, + 'PlainKey(key=0):0:6:0', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '0'), + }) + + check_asgn_parsing('[a{b}c[0].0]', { + -- 012345678901 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'ConcatOrSubscript:0:9:.', + children = { + { + 'Subscript:0:6:[', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'ComplexIdentifier:0:5:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + 'Integer(val=0):0:7:0', + }, + }, + 'PlainKey(key=0):0:10:0', + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', '['), + hl('Number', '0'), + hl('SubscriptBracket', ']'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '0'), + hl('List', ']'), + }) + + check_asgn_parsing('{a}{b}', { + -- 012345 + ast = { + { + 'ComplexIdentifier:0:3:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + fmtn('CurlyBracesIdentifier', '--i', ':0:3:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + }, + }, + }, + }, + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + }) + + check_asgn_parsing('a.b{c}{d}', { + -- 012345678 + ast = { + { + 'OpMissing:0:3:', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=b):0:2:b', + }, + }, + { + 'ComplexIdentifier:0:6:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:3:{'), + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + { + fmtn('CurlyBracesIdentifier', '--i', ':0:6:{'), + children = { + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '{c}{d}', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', 'b'), + hl('InvalidFigureBrace', '{'), + hl('IdentifierName', 'c'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'd'), + hl('Curly', '}'), + }) + + check_asgn_parsing('[a] = 1', { + -- 0123456 + ast = { + { + 'Assignment(Plain):0:3: =', + children = { + { + 'ListLiteral:0:0:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + 'Integer(val=1):0:5: 1', + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('List', ']'), + hl('PlainAssignment', '=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('[a[b], [c, [d, [e]]]] = 1', { + -- 0123456789012345678901234 + -- 0 1 2 + ast = { + { + 'Assignment(Plain):0:21: =', + children = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:5:,', + children = { + { + 'Subscript:0:2:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'ListLiteral:0:6: [', + children = { + { + 'Comma:0:9:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:8:c', + { + 'ListLiteral:0:10: [', + children = { + { + 'Comma:0:13:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:12:d', + { + 'ListLiteral:0:14: [', + children = { + 'PlainIdentifier(scope=0,ident=e):0:16:e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + 'Integer(val=1):0:23: 1', + }, + }, + }, + err = { + arg = '[c, [d, [e]]]] = 1', + msg = 'E475: Nested lists not allowed when assigning: %.*s', + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptBracket', ']'), + hl('Comma', ','), + hl('InvalidList', '[', 1), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('InvalidList', '[', 1), + hl('IdentifierName', 'd'), + hl('Comma', ','), + hl('InvalidList', '[', 1), + hl('IdentifierName', 'e'), + hl('List', ']'), + hl('List', ']'), + hl('List', ']'), + hl('List', ']'), + hl('PlainAssignment', '=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('$X += 1', { + -- 0123456 + ast = { + { + 'Assignment(Add):0:2: +=', + children = { + 'Environment(ident=X):0:0:$X', + 'Integer(val=1):0:5: 1', + }, + }, + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'X'), + hl('AssignmentWithAddition', '+=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('@a .= 1', { + -- 0123456 + ast = { + { + 'Assignment(Concat):0:2: .=', + children = { + 'Register(name=a):0:0:@a', + 'Integer(val=1):0:5: 1', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('AssignmentWithConcatenation', '.=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('&option -= 1', { + -- 012345678901 + -- 0 1 + ast = { + { + 'Assignment(Subtract):0:7: -=', + children = { + 'Option(scope=0,ident=option):0:0:&option', + 'Integer(val=1):0:10: 1', + }, + }, + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 'option'), + hl('AssignmentWithSubtraction', '-=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('[$X, @a, &l:option] = [1, 2, 3]', { + -- 0123456789012345678901234567890 + -- 0 1 2 3 + ast = { + { + 'Assignment(Plain):0:19: =', + children = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:3:,', + children = { + 'Environment(ident=X):0:1:$X', + { + 'Comma:0:7:,', + children = { + 'Register(name=a):0:4: @a', + 'Option(scope=l,ident=option):0:8: &l:option', + }, + }, + }, + }, + }, + }, + { + 'ListLiteral:0:21: [', + children = { + { + 'Comma:0:24:,', + children = { + 'Integer(val=1):0:23:1', + { + 'Comma:0:27:,', + children = { + 'Integer(val=2):0:25: 2', + 'Integer(val=3):0:28: 3', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'X'), + hl('Comma', ','), + hl('Register', '@a', 1), + hl('Comma', ','), + hl('OptionSigil', '&', 1), + hl('OptionScope', 'l'), + hl('OptionScopeDelimiter', ':'), + hl('OptionName', 'option'), + hl('List', ']'), + hl('PlainAssignment', '=', 1), + hl('List', '[', 1), + hl('Number', '1'), + hl('Comma', ','), + hl('Number', '2', 1), + hl('Comma', ','), + hl('Number', '3', 1), + hl('List', ']'), + }) + end) + itp('works with non-ASCII characters', function() + check_parsing('"«»"«»', { + -- 013568 + ast = { + { + 'OpMissing:0:6:', + children = { + 'DoubleQuotedString(val="«»"):0:0:"«»"', + { + 'ComplexIdentifier:0:8:', + children = { + 'PlainIdentifier(scope=0,ident=«):0:6:«', + 'PlainIdentifier(scope=0,ident=»):0:8:»', + }, + }, + }, + }, + }, + err = { + arg = '«»', + msg = 'E15: Unidentified character: %.*s', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', '«»'), + hl('DoubleQuote', '"'), + hl('InvalidIdentifierName', '«'), + hl('InvalidIdentifierName', '»'), + }, { + [1] = { + ast = { + ast = { + 'DoubleQuotedString(val="«»"):0:0:"«»"', + }, + len = 6, + }, + hl_fs = { + [5] = REMOVE_THIS, + [4] = REMOVE_THIS, + }, + }, + }) + check_parsing('"\192"\192"foo"', { + -- 01 23 45678 + ast = { + { + 'OpMissing:0:3:', + children = { + 'DoubleQuotedString(val="\192"):0:0:"\192"', + { + 'OpMissing:0:4:', + children = { + 'PlainIdentifier(scope=0,ident=\192):0:3:\192', + 'DoubleQuotedString(val="foo"):0:4:"foo"', + }, + }, + }, + }, + }, + err = { + arg = '\192"foo"', + msg = 'E15: Unidentified character: %.*s', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', '\192'), + hl('DoubleQuote', '"'), + hl('InvalidIdentifierName', '\192'), + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedBody', 'foo'), + hl('InvalidDoubleQuote', '"'), + }, { + [1] = { + ast = { + ast = { + 'DoubleQuotedString(val="\192"):0:0:"\192"', + }, + len = 3, + }, + hl_fs = { + [4] = REMOVE_THIS, + [5] = REMOVE_THIS, + [6] = REMOVE_THIS, + [7] = REMOVE_THIS, + }, + }, + }) + end) +end diff --git a/test/unit/viml/helpers.lua b/test/unit/viml/helpers.lua new file mode 100644 index 0000000000..9d8102e023 --- /dev/null +++ b/test/unit/viml/helpers.lua @@ -0,0 +1,130 @@ +local helpers = require('test.unit.helpers')(nil) + +local ffi = helpers.ffi +local cimport = helpers.cimport +local kvi_new = helpers.kvi_new +local kvi_init = helpers.kvi_init +local conv_enum = helpers.conv_enum +local make_enum_conv_tab = helpers.make_enum_conv_tab + +local lib = cimport('./src/nvim/viml/parser/expressions.h') + +local function new_pstate(strings) + local strings_idx = 0 + local function get_line(_, ret_pline) + strings_idx = strings_idx + 1 + local str = strings[strings_idx] + local data, size + if type(str) == 'string' then + data = str + size = #str + elseif type(str) == 'nil' then + data = nil + size = 0 + elseif type(str) == 'table' then + data = str.data + size = str.size + elseif type(str) == 'function' then + data, size = str() + size = size or 0 + end + ret_pline.data = data + ret_pline.size = size + ret_pline.allocated = false + end + local state = { + reader = { + get_line = get_line, + cookie = nil, + conv = { + vc_type = 0, + vc_factor = 1, + vc_fail = false, + }, + }, + pos = { line = 0, col = 0 }, + colors = kvi_new('ParserHighlight'), + can_continuate = false, + } + local ret = ffi.new('ParserState', state) + kvi_init(ret.reader.lines) + kvi_init(ret.stack) + return ret +end + +local function pline2lua(pline) + return ffi.string(pline.data, pline.size) +end + +local function pstate_str(pstate, start, len) + local str = nil + local err = nil + if start.line < pstate.reader.lines.size then + local pstr = pline2lua(pstate.reader.lines.items[start.line]) + if start.col >= #pstr then + err = 'start.col >= #pstr' + else + str = pstr:sub(tonumber(start.col) + 1, tonumber(start.col + len)) + end + else + err = 'start.line >= pstate.reader.lines.size' + end + return str, err +end + +local function pstate_set_str(pstate, start, len, ret) + ret = ret or {} + ret.start = { + line = tonumber(start.line), + col = tonumber(start.col) + } + ret.len = tonumber(len) + ret.str, ret.error = pstate_str(pstate, start, len) + return ret +end + +local eltkn_cmp_type_tab +make_enum_conv_tab(lib, { + 'kExprCmpEqual', + 'kExprCmpMatches', + 'kExprCmpGreater', + 'kExprCmpGreaterOrEqual', + 'kExprCmpIdentical', +}, 'kExprCmp', function(ret) eltkn_cmp_type_tab = ret end) + +local function conv_cmp_type(typ) + return conv_enum(eltkn_cmp_type_tab, typ) +end + +local ccs_tab +make_enum_conv_tab(lib, { + 'kCCStrategyUseOption', + 'kCCStrategyMatchCase', + 'kCCStrategyIgnoreCase', +}, 'kCCStrategy', function(ret) ccs_tab = ret end) + +local function conv_ccs(ccs) + return conv_enum(ccs_tab, ccs) +end + +local expr_asgn_type_tab +make_enum_conv_tab(lib, { + 'kExprAsgnPlain', + 'kExprAsgnAdd', + 'kExprAsgnSubtract', + 'kExprAsgnConcat', +}, 'kExprAsgn', function(ret) expr_asgn_type_tab = ret end) + +local function conv_expr_asgn_type(expr_asgn_type) + return conv_enum(expr_asgn_type_tab, expr_asgn_type) +end + +return { + conv_ccs = conv_ccs, + pline2lua = pline2lua, + pstate_str = pstate_str, + new_pstate = new_pstate, + conv_cmp_type = conv_cmp_type, + pstate_set_str = pstate_set_str, + conv_expr_asgn_type = conv_expr_asgn_type, +} |