From 0300c4d10991fb6ce218d45c4fe6d71a73f07d62 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 20 Aug 2017 18:40:22 +0300 Subject: viml/expressions: Add lexer with some basic tests --- src/nvim/viml/parser/expressions.h | 118 +++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 src/nvim/viml/parser/expressions.h (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h new file mode 100644 index 0000000000..52354760a5 --- /dev/null +++ b/src/nvim/viml/parser/expressions.h @@ -0,0 +1,118 @@ +#ifndef NVIM_VIML_PARSER_EXPRESSIONS_H +#define NVIM_VIML_PARSER_EXPRESSIONS_H + +#include +#include + +#include "nvim/types.h" +#include "nvim/viml/parser/parser.h" + +// Defines whether to ignore case: +// == kCCStrategyUseOption +// ==# kCCStrategyMatchCase +// ==? kCCStrategyIgnoreCase +typedef enum { + kCCStrategyUseOption = 0, // 0 for xcalloc + kCCStrategyMatchCase = '#', + kCCStrategyIgnoreCase = '?', +} CaseCompareStrategy; + +/// Lexer token type +typedef enum { + kExprLexInvalid = 0, ///< Invalid token, indicaten an error. + kExprLexMissing, ///< Missing token, for use in parser. + kExprLexSpacing, ///< Spaces, tabs, newlines, etc. + kExprLexEOC, ///< End of command character: NL, |, just end of stream. + + kExprLexQuestion, ///< Question mark, for use in ternary. + kExprLexColon, ///< Colon, for use in ternary. + kExprLexOr, ///< Logical or operator. + kExprLexAnd, ///< Logical and operator. + kExprLexComparison, ///< One of the comparison operators. + kExprLexPlus, ///< Plus sign. + kExprLexMinus, ///< Minus sign. + kExprLexDot, ///< Dot: either concat or subscript, also part of the float. + kExprLexMultiplication, ///< Multiplication, division or modulo operator. + + kExprLexNot, ///< Not: !. + + kExprLexNumber, ///< Integer number literal, or part of a float. + kExprLexSingleQuotedString, ///< Single quoted string literal. + kExprLexDoubleQuotedString, ///< Double quoted string literal. + kExprLexOption, ///< &optionname option value. + kExprLexRegister, ///< @r register value. + kExprLexEnv, ///< Environment $variable value. + kExprLexPlainIdentifier, ///< Identifier without scope: `abc`, `foo#bar`. + + kExprLexBracket, ///< Bracket, either opening or closing. + kExprLexFigureBrace, ///< Figure brace, either opening or closing. + kExprLexParenthesis, ///< Parenthesis, either opening or closing. + kExprLexComma, ///< Comma. + kExprLexArrow, ///< Arrow, like from lambda expressions. +} LexExprTokenType; + +/// Lexer token +typedef struct { + ParserPosition start; + size_t len; + LexExprTokenType type; + union { + struct { + enum { + kExprLexCmpEqual, ///< Equality, unequality. + kExprLexCmpMatches, ///< Matches regex, not matches regex. + kExprLexCmpGreater, ///< `>` or `<=` + kExprLexCmpGreaterOrEqual, ///< `>=` or `<`. + kExprLexCmpIdentical, ///< `is` or `isnot` + } type; ///< Comparison type. + CaseCompareStrategy ccs; ///< Case comparison strategy. + bool inv; ///< True if comparison is to be inverted. + } cmp; ///< For kExprLexComparison. + + struct { + enum { + kExprLexMulMul, ///< Real multiplication. + kExprLexMulDiv, ///< Division. + kExprLexMulMod, ///< Modulo. + } type; ///< Multiplication type. + } mul; ///< For kExprLexMultiplication. + + struct { + bool closing; ///< True if bracket/etc is a closing one. + } brc; ///< For brackets/braces/parenthesis. + + struct { + int name; ///< Register name, may be -1 if name not present. + } reg; ///< For kExprLexRegister. + + struct { + bool closed; ///< True if quote was closed. + } str; ///< For kExprLexSingleQuotedString and kExprLexDoubleQuotedString. + + struct { + const char *name; ///< Option name start. + size_t len; ///< Option name length. + enum { + kExprLexOptUnspecified = 0, + kExprLexOptGlobal = 1, + kExprLexOptLocal = 2, + } scope; ///< Option scope: &l:, &g: or not specified. + } opt; ///< Option properties. + + struct { + int scope; ///< Scope character or 0 if not present. + bool autoload; ///< Has autoload characters. + } var; ///< For kExprLexPlainIdentifier + + struct { + LexExprTokenType type; ///< Suggested type for parsing incorrect code. + const char *msg; ///< Error message. + } err; ///< For kExprLexInvalid + } data; ///< Additional data, if needed. +} LexExprToken; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/expressions.h.generated.h" +#endif + +#endif // NVIM_VIML_PARSER_EXPRESSIONS_H -- cgit From 430e516d3ac1235c1ee3009a8a36089bf278440e Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 3 Sep 2017 21:58:16 +0300 Subject: viml/parser/expressions: Start creating expressions parser Currently supported nodes: - Register as it is one of the simplest value nodes (even numbers are not that simple with that dot handling). - Plus, both unary and binary. - Parenthesis, both nesting and calling. Note regarding unit tests: it stores data for AST in highlighting in strings in place of tables because luassert fails to do a good job at representing big tables. Squashing a bunch of data into a single string simply yields more readable result. --- src/nvim/viml/parser/expressions.h | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 52354760a5..13888562df 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -111,6 +111,80 @@ typedef struct { } data; ///< Additional data, if needed. } LexExprToken; +/// Expression AST node type +typedef enum { + kExprNodeMissing = 'X', + kExprNodeOpMissing = '_', + kExprNodeTernary = '?', ///< Ternary operator, valid one has three children. + kExprNodeRegister = '@', ///< Register, no children. + kExprNodeSubscript = 's', ///< Subscript, should have two or three children. + kExprNodeListLiteral = 'l', ///< List literal, any number of children. + kExprNodeUnaryPlus = 'p', + kExprNodeBinaryPlus = '+', + kExprNodeNested = 'e', ///< Nested parenthesised expression. + kExprNodeCall = 'c', ///< Function call. + /// Plain identifier: simple variable/function name + /// + /// Looks like "string", "g:Foo", etc: consists from a single + /// kExprLexPlainIdentifier token. + kExprNodePlainIdentifier = 'i', + /// Complex identifier: variable/function name with curly braces + kExprNodeComplexIdentifier = 'I', +} ExprASTNodeType; + +typedef struct expr_ast_node ExprASTNode; + +/// Structure representing one AST node +struct expr_ast_node { + ExprASTNodeType type; ///< Node type. + /// Node children: e.g. for 1 + 2 nodes 1 and 2 will be children of +. + ExprASTNode *children; + /// Next node: e.g. for 1 + 2 child nodes 1 and 2 are put into a single-linked + /// list: `(+)->children` references only node 1, node 2 is in + /// `(+)->children->next`. + ExprASTNode *next; + ParserPosition start; + size_t len; + union { + struct { + int name; ///< Register name, may be -1 if name not present. + } reg; ///< For kExprNodeRegister. + } data; +}; + +enum { + /// Allow multiple expressions in a row: e.g. for :echo + /// + /// Parser will still parse only one of them though. + kExprFlagsMulti = (1 << 0), + /// Allow NL, NUL and bar to be EOC + /// + /// When parsing expressions input by user bar is assumed to be a binary + /// operator and other two are spacings. + kExprFlagsDisallowEOC = (1 << 1), + /// Print errors when encountered + /// + /// Without the flag they are only taken into account when parsing. + kExprFlagsPrintError = (1 << 2), +} ExprParserFlags; + +/// Structure representing complety AST for one expression +typedef struct { + /// True if represented AST is correct and can be executed. Incorrect ones may + /// still be used for completion, or in linters. + bool correct; + /// When AST is not correct this message will be printed. + /// + /// Uses `emsgf(msg, arg_len, arg);`, `msg` is assumed to contain only `%.*s`. + struct { + const char *msg; + int arg_len; + const char *arg; + } err; + /// Root node of the AST. + ExprASTNode *root; +} ExprAST; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.h.generated.h" #endif -- cgit From 7980614650f0aedb39bf88466e5bd3ce90429cc1 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 17 Sep 2017 17:33:03 +0300 Subject: viml/parser/expressions: Add support for figure braces (three kinds) --- src/nvim/viml/parser/expressions.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 13888562df..13640ec137 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -2,6 +2,7 @@ #define NVIM_VIML_PARSER_EXPRESSIONS_H #include +#include #include #include "nvim/types.h" @@ -130,6 +131,17 @@ typedef enum { kExprNodePlainIdentifier = 'i', /// Complex identifier: variable/function name with curly braces kExprNodeComplexIdentifier = 'I', + /// Figure brace expression which is not yet known + /// + /// May resolve to any of kExprNodeDictLiteral, kExprNodeLambda or + /// kExprNodeCurlyBracesIdentifier. + kExprNodeUnknownFigure = '{', + kExprNodeLambda = '\\', ///< Lambda. + kExprNodeDictLiteral = 'd', ///< Dictionary literal. + kExprNodeCurlyBracesIdentifier= '}', ///< Part of the curly braces name. + kExprNodeComma = ',', ///< Comma “operator”. + kExprNodeColon = ':', ///< Colon “operator”. + kExprNodeArrow = '>', ///< Arrow “operator”. } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -149,6 +161,27 @@ struct expr_ast_node { struct { int name; ///< Register name, may be -1 if name not present. } reg; ///< For kExprNodeRegister. + struct { + /// Which nodes UnknownFigure can’t possibly represent. + struct { + /// True if UnknownFigure may actually represent dictionary literal. + bool allow_dict; + /// True if UnknownFigure may actually represent lambda. + bool allow_lambda; + /// True if UnknownFigure may actually be part of curly braces name. + bool allow_ident; + } type_guesses; + /// Highlight chunk index, used for rehighlighting if needed + size_t opening_hl_idx; + } fig; ///< For kExprNodeUnknownFigure. + struct { + int scope; ///< Scope character or 0 if not present. + /// Actual identifier without scope. + /// + /// Points to inside parser reader state. + const char *ident; + size_t ident_len; ///< Actual identifier length. + } var; } data; }; @@ -166,6 +199,8 @@ enum { /// /// Without the flag they are only taken into account when parsing. kExprFlagsPrintError = (1 << 2), + // WARNING: whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_parser.c. } ExprParserFlags; /// Structure representing complety AST for one expression -- cgit From 9fa8f7fc0a24371f7956450d840bdae8a2fc9a51 Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 28 Sep 2017 00:40:25 +0300 Subject: viml/parser/expressions: Add a way to adjust lexer It also adds support for kExprLexOr which for some reason was forgotten. It was only made sure that KLEE test compiles in non-KLEE mode, not that something works or that KLEE is able to run tests. --- src/nvim/viml/parser/expressions.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 13640ec137..64abab9e41 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -109,9 +109,37 @@ typedef struct { LexExprTokenType type; ///< Suggested type for parsing incorrect code. const char *msg; ///< Error message. } err; ///< For kExprLexInvalid + + struct { + bool is_float; ///< True if number is a floating-point. + } num; ///< For kExprLexNumber } data; ///< Additional data, if needed. } LexExprToken; +typedef enum { + /// If set, “pointer” to the current byte in pstate will not be shifted + kELFlagPeek = (1 << 0), + /// Determines whether scope is allowed to come before the identifier + kELFlagForbidScope = (1 << 1), + /// Determines whether floating-point numbers are allowed + /// + /// I.e. whether dot is a decimal point separator or is not a part of + /// a number at all. + kELFlagAllowFloat = (1 << 2), + /// Determines whether `is` and `isnot` are seen as comparison operators + /// + /// If set they are supposed to be just regular identifiers. + kELFlagIsNotCmp = (1 << 3), + /// Determines whether EOC tokens are allowed + /// + /// If set then it will yield Invalid token with E15 in place of EOC one if + /// “EOC” is something like "|". It is fine with emitting EOC at the end of + /// string still, with or without this flag set. + kELFlagForbidEOC = (1 << 4), + // WARNING: whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_lexer.c. +} LexExprFlags; + /// Expression AST node type typedef enum { kExprNodeMissing = 'X', -- cgit From 9e721031d597bfa435da03597939191970f7a918 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 1 Oct 2017 16:50:46 +0300 Subject: viml/parser/expressions: Fix determining invalid commas/colons --- src/nvim/viml/parser/expressions.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 64abab9e41..01a51e4eda 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -144,10 +144,10 @@ typedef enum { typedef enum { kExprNodeMissing = 'X', kExprNodeOpMissing = '_', - kExprNodeTernary = '?', ///< Ternary operator, valid one has three children. - kExprNodeRegister = '@', ///< Register, no children. - kExprNodeSubscript = 's', ///< Subscript, should have two or three children. - kExprNodeListLiteral = 'l', ///< List literal, any number of children. + kExprNodeTernary = '?', ///< Ternary operator. + kExprNodeRegister = '@', ///< Register. + kExprNodeSubscript = 's', ///< Subscript. + kExprNodeListLiteral = 'l', ///< List literal. kExprNodeUnaryPlus = 'p', kExprNodeBinaryPlus = '+', kExprNodeNested = 'e', ///< Nested parenthesised expression. -- cgit From 6144e26eb920a90b0db22bd7afcac0b9e0734ed6 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 1 Oct 2017 22:35:41 +0300 Subject: viml/parser/expressions: Add support for ternary operator --- src/nvim/viml/parser/expressions.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 01a51e4eda..cf0907850a 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -145,6 +145,7 @@ typedef enum { kExprNodeMissing = 'X', kExprNodeOpMissing = '_', kExprNodeTernary = '?', ///< Ternary operator. + kExprNodeTernaryValue = 'C', ///< Ternary operator, colon. kExprNodeRegister = '@', ///< Register. kExprNodeSubscript = 's', ///< Subscript. kExprNodeListLiteral = 'l', ///< List literal. @@ -209,7 +210,10 @@ struct expr_ast_node { /// Points to inside parser reader state. const char *ident; size_t ident_len; ///< Actual identifier length. - } var; + } var; ///< For kExprNodePlainIdentifier. + struct { + bool got_colon; ///< True if colon was seen. + } ter; ///< For kExprNodeTernaryValue. } data; }; -- cgit From 6168e1127c1c80a3810854649b0776146545043b Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 2 Oct 2017 02:41:55 +0300 Subject: viml/parser/expressions: Add support for comparison operators --- src/nvim/viml/parser/expressions.h | 45 ++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 17 deletions(-) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index cf0907850a..8ca3ceacb9 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -16,7 +16,7 @@ typedef enum { kCCStrategyUseOption = 0, // 0 for xcalloc kCCStrategyMatchCase = '#', kCCStrategyIgnoreCase = '?', -} CaseCompareStrategy; +} ExprCaseCompareStrategy; /// Lexer token type typedef enum { @@ -52,6 +52,14 @@ typedef enum { kExprLexArrow, ///< Arrow, like from lambda expressions. } LexExprTokenType; +typedef enum { + kExprCmpEqual, ///< Equality, unequality. + kExprCmpMatches, ///< Matches regex, not matches regex. + kExprCmpGreater, ///< `>` or `<=` + kExprCmpGreaterOrEqual, ///< `>=` or `<`. + kExprCmpIdentical, ///< `is` or `isnot` +} ExprComparisonType; + /// Lexer token typedef struct { ParserPosition start; @@ -59,14 +67,8 @@ typedef struct { LexExprTokenType type; union { struct { - enum { - kExprLexCmpEqual, ///< Equality, unequality. - kExprLexCmpMatches, ///< Matches regex, not matches regex. - kExprLexCmpGreater, ///< `>` or `<=` - kExprLexCmpGreaterOrEqual, ///< `>=` or `<`. - kExprLexCmpIdentical, ///< `is` or `isnot` - } type; ///< Comparison type. - CaseCompareStrategy ccs; ///< Case comparison strategy. + ExprComparisonType type; ///< Comparison type. + ExprCaseCompareStrategy ccs; ///< Case comparison strategy. bool inv; ///< True if comparison is to be inverted. } cmp; ///< For kExprLexComparison. @@ -171,6 +173,7 @@ typedef enum { kExprNodeComma = ',', ///< Comma “operator”. kExprNodeColon = ':', ///< Colon “operator”. kExprNodeArrow = '>', ///< Arrow “operator”. + kExprNodeComparison = '=', ///< Various comparison operators. } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -214,6 +217,11 @@ struct expr_ast_node { struct { bool got_colon; ///< True if colon was seen. } ter; ///< For kExprNodeTernaryValue. + struct { + ExprComparisonType type; ///< Comparison type. + ExprCaseCompareStrategy ccs; ///< Case comparison strategy. + bool inv; ///< True if comparison is to be inverted. + } cmp; ///< For kExprNodeComparison. } data; }; @@ -235,19 +243,22 @@ enum { // viml_expressions_parser.c. } ExprParserFlags; +/// AST error definition +typedef struct { + /// Error message. Must contain a single printf format atom: %.*s. + const char *msg; + /// Error message argument: points to the location of the error. + const char *arg; + /// Message argument length: length till the end of string. + int arg_len; +} ExprASTError; + /// Structure representing complety AST for one expression typedef struct { - /// True if represented AST is correct and can be executed. Incorrect ones may - /// still be used for completion, or in linters. - bool correct; /// When AST is not correct this message will be printed. /// /// Uses `emsgf(msg, arg_len, arg);`, `msg` is assumed to contain only `%.*s`. - struct { - const char *msg; - int arg_len; - const char *arg; - } err; + ExprASTError err; /// Root node of the AST. ExprASTNode *root; } ExprAST; -- cgit From 163792e9b9854fe046ada3233dec0fd0f6c55737 Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 6 Oct 2017 01:19:43 +0300 Subject: viml/parser/expressions: Make lexer parse numbers, support non-decimal --- src/nvim/viml/parser/expressions.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 8ca3ceacb9..29903490bb 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -7,6 +7,7 @@ #include "nvim/types.h" #include "nvim/viml/parser/parser.h" +#include "nvim/eval/typval.h" // Defines whether to ignore case: // == kCCStrategyUseOption @@ -113,6 +114,11 @@ typedef struct { } err; ///< For kExprLexInvalid struct { + union { + float_T floating; + uvarnumber_T integer; + } val; ///< Number value. + uint8_t base; ///< Base: 2, 8, 10 or 16. bool is_float; ///< True if number is a floating-point. } num; ///< For kExprLexNumber } data; ///< Additional data, if needed. -- cgit From 21a5ce033c5a853bed3204ea9f0f7a3cfc1d164f Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 3 Oct 2017 01:30:02 +0300 Subject: viml/parser/expressions: Add support for the dot operator and numbers --- src/nvim/viml/parser/expressions.h | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 29903490bb..0d496c87ba 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -166,6 +166,8 @@ typedef enum { /// Looks like "string", "g:Foo", etc: consists from a single /// kExprLexPlainIdentifier token. kExprNodePlainIdentifier = 'i', + /// Plain dictionary key, for use with kExprNodeConcatOrSubscript + kExprNodePlainKey = 'k', /// Complex identifier: variable/function name with curly braces kExprNodeComplexIdentifier = 'I', /// Figure brace expression which is not yet known @@ -180,6 +182,19 @@ typedef enum { kExprNodeColon = ':', ///< Colon “operator”. kExprNodeArrow = '>', ///< Arrow “operator”. kExprNodeComparison = '=', ///< Various comparison operators. + /// Concat operator + /// + /// To be only used in cases when it is known for sure it is not a subscript. + kExprNodeConcat = '.', + /// Concat or subscript operator + /// + /// For cases when it is not obvious whether expression is a concat or + /// a subscript. May only have either number or plain identifier as the second + /// child. To make it easier to avoid curly braces in place of + /// kExprNodePlainIdentifier node kExprNodePlainKey is used. + kExprNodeConcatOrSubscript = 'S', + kExprNodeInteger = '0', ///< Integral number. + kExprNodeFloat = '1', ///< Floating-point number. } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -219,7 +234,7 @@ struct expr_ast_node { /// Points to inside parser reader state. const char *ident; size_t ident_len; ///< Actual identifier length. - } var; ///< For kExprNodePlainIdentifier. + } var; ///< For kExprNodePlainIdentifier and kExprNodePlainKey. struct { bool got_colon; ///< True if colon was seen. } ter; ///< For kExprNodeTernaryValue. @@ -228,6 +243,12 @@ struct expr_ast_node { ExprCaseCompareStrategy ccs; ///< Case comparison strategy. bool inv; ///< True if comparison is to be inverted. } cmp; ///< For kExprNodeComparison. + struct { + uvarnumber_T value; + } num; ///< For kExprNodeInteger. + struct { + float_T value; + } flt; ///< For kExprNodeFloat. } data; }; -- cgit From af38cea133f5ebb67208cedd289e408cd1dad15a Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 8 Oct 2017 21:52:38 +0300 Subject: viml/parser/expressions: Add support for string parsing --- src/nvim/viml/parser/expressions.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 0d496c87ba..a09cdde4c0 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -195,6 +195,8 @@ typedef enum { kExprNodeConcatOrSubscript = 'S', kExprNodeInteger = '0', ///< Integral number. kExprNodeFloat = '1', ///< Floating-point number. + kExprNodeSingleQuotedString = '\'', + kExprNodeDoubleQuotedString = '"', } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -249,6 +251,11 @@ struct expr_ast_node { struct { float_T value; } flt; ///< For kExprNodeFloat. + struct { + char *value; + size_t size; + } str; ///< For kExprNodeSingleQuotedString and + ///< kExprNodeDoubleQuotedString. } data; }; -- cgit From fa3cfc0dd54df125a1dbabccda47a5f45dc483ae Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 9 Oct 2017 02:55:56 +0300 Subject: viml/parser/expressions: Finish parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note: formatc.lua was unable to swallow some newer additions to ExprASTNodeType (specifically `kExprNodeOr = '|'` and probably something else), so all `= …` were dropped: in any case they only were there in order to not bother updating viml_pexpr_debug_print_ast_node and since it is now known all nodes which will be present it is not much of an issue. --- src/nvim/viml/parser/expressions.h | 115 ++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 35 deletions(-) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index a09cdde4c0..0198852bed 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -61,6 +61,36 @@ typedef enum { kExprCmpIdentical, ///< `is` or `isnot` } ExprComparisonType; +/// All possible option scopes +typedef enum { + kExprOptScopeUnspecified = 0, + kExprOptScopeGlobal = 'g', + kExprOptScopeLocal = 'l', +} ExprOptScope; + +#define EXPR_OPT_SCOPE_LIST \ + ((char *)(char[]){ kExprOptScopeGlobal, kExprOptScopeLocal }) + +/// All possible variable scopes +typedef enum { + kExprVarScopeMissing = 0, + kExprVarScopeScript = 's', + kExprVarScopeGlobal = 'g', + kExprVarScopeVim = 'v', + kExprVarScopeBuffer = 'b', + kExprVarScopeWindow = 'w', + kExprVarScopeTabpage = 't', + kExprVarScopeLocal = 'l', + kExprVarScopeArguments = 'a', +} ExprVarScope; + +#define EXPR_VAR_SCOPE_LIST \ + ((char[]) { \ + kExprVarScopeScript, kExprVarScopeGlobal, kExprVarScopeVim, \ + kExprVarScopeBuffer, kExprVarScopeWindow, kExprVarScopeTabpage, \ + kExprVarScopeLocal, kExprVarScopeBuffer, kExprVarScopeArguments, \ + }) + /// Lexer token typedef struct { ParserPosition start; @@ -96,15 +126,11 @@ typedef struct { struct { const char *name; ///< Option name start. size_t len; ///< Option name length. - enum { - kExprLexOptUnspecified = 0, - kExprLexOptGlobal = 1, - kExprLexOptLocal = 2, - } scope; ///< Option scope: &l:, &g: or not specified. + ExprOptScope scope; ///< Option scope: &l:, &g: or not specified. } opt; ///< Option properties. struct { - int scope; ///< Scope character or 0 if not present. + ExprVarScope scope; ///< Scope character or 0 if not present. bool autoload; ///< Has autoload characters. } var; ///< For kExprLexPlainIdentifier @@ -150,53 +176,63 @@ typedef enum { /// Expression AST node type typedef enum { - kExprNodeMissing = 'X', - kExprNodeOpMissing = '_', - kExprNodeTernary = '?', ///< Ternary operator. - kExprNodeTernaryValue = 'C', ///< Ternary operator, colon. - kExprNodeRegister = '@', ///< Register. - kExprNodeSubscript = 's', ///< Subscript. - kExprNodeListLiteral = 'l', ///< List literal. - kExprNodeUnaryPlus = 'p', - kExprNodeBinaryPlus = '+', - kExprNodeNested = 'e', ///< Nested parenthesised expression. - kExprNodeCall = 'c', ///< Function call. + kExprNodeMissing = 0, + kExprNodeOpMissing, + kExprNodeTernary, ///< Ternary operator. + kExprNodeTernaryValue, ///< Ternary operator, colon. + kExprNodeRegister, ///< Register. + kExprNodeSubscript, ///< Subscript. + kExprNodeListLiteral, ///< List literal. + kExprNodeUnaryPlus, + kExprNodeBinaryPlus, + kExprNodeNested, ///< Nested parenthesised expression. + kExprNodeCall, ///< Function call. /// Plain identifier: simple variable/function name /// /// Looks like "string", "g:Foo", etc: consists from a single /// kExprLexPlainIdentifier token. - kExprNodePlainIdentifier = 'i', + kExprNodePlainIdentifier, /// Plain dictionary key, for use with kExprNodeConcatOrSubscript - kExprNodePlainKey = 'k', + kExprNodePlainKey, /// Complex identifier: variable/function name with curly braces - kExprNodeComplexIdentifier = 'I', + kExprNodeComplexIdentifier, /// Figure brace expression which is not yet known /// /// May resolve to any of kExprNodeDictLiteral, kExprNodeLambda or /// kExprNodeCurlyBracesIdentifier. - kExprNodeUnknownFigure = '{', - kExprNodeLambda = '\\', ///< Lambda. - kExprNodeDictLiteral = 'd', ///< Dictionary literal. - kExprNodeCurlyBracesIdentifier= '}', ///< Part of the curly braces name. - kExprNodeComma = ',', ///< Comma “operator”. - kExprNodeColon = ':', ///< Colon “operator”. - kExprNodeArrow = '>', ///< Arrow “operator”. - kExprNodeComparison = '=', ///< Various comparison operators. + kExprNodeUnknownFigure, + kExprNodeLambda, ///< Lambda. + kExprNodeDictLiteral, ///< Dictionary literal. + kExprNodeCurlyBracesIdentifier, ///< Part of the curly braces name. + kExprNodeComma, ///< Comma “operator”. + kExprNodeColon, ///< Colon “operator”. + kExprNodeArrow, ///< Arrow “operator”. + kExprNodeComparison, ///< Various comparison operators. /// Concat operator /// /// To be only used in cases when it is known for sure it is not a subscript. - kExprNodeConcat = '.', + kExprNodeConcat, /// Concat or subscript operator /// /// For cases when it is not obvious whether expression is a concat or /// a subscript. May only have either number or plain identifier as the second /// child. To make it easier to avoid curly braces in place of /// kExprNodePlainIdentifier node kExprNodePlainKey is used. - kExprNodeConcatOrSubscript = 'S', - kExprNodeInteger = '0', ///< Integral number. - kExprNodeFloat = '1', ///< Floating-point number. - kExprNodeSingleQuotedString = '\'', - kExprNodeDoubleQuotedString = '"', + kExprNodeConcatOrSubscript, + kExprNodeInteger, ///< Integral number. + kExprNodeFloat, ///< Floating-point number. + kExprNodeSingleQuotedString, + kExprNodeDoubleQuotedString, + kExprNodeOr, + kExprNodeAnd, + kExprNodeUnaryMinus, + kExprNodeBinaryMinus, + kExprNodeNot, + kExprNodeMultiplication, + kExprNodeDivision, + kExprNodeMod, + kExprNodeOption, + kExprNodeEnvironment, } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -230,7 +266,7 @@ struct expr_ast_node { size_t opening_hl_idx; } fig; ///< For kExprNodeUnknownFigure. struct { - int scope; ///< Scope character or 0 if not present. + ExprVarScope scope; ///< Scope character or 0 if not present. /// Actual identifier without scope. /// /// Points to inside parser reader state. @@ -256,6 +292,15 @@ struct expr_ast_node { size_t size; } str; ///< For kExprNodeSingleQuotedString and ///< kExprNodeDoubleQuotedString. + struct { + const char *ident; ///< Option name start. + size_t ident_len; ///< Option name length. + ExprOptScope scope; ///< Option scope: &l:, &g: or not specified. + } opt; ///< For kExprNodeOption. + struct { + const char *ident; ///< Environment variable name start. + size_t ident_len; ///< Environment variable name length. + } env; ///< For kExprNodeEnvironment. } data; }; -- cgit From 3aa2c0d63ae488e302a89fdcdd650404cb2670fd Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 15 Oct 2017 21:11:00 +0300 Subject: viml/parser/expressions,klee: Fix some problems found by KLEE run --- src/nvim/viml/parser/expressions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 0198852bed..025f0f766e 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -69,7 +69,7 @@ typedef enum { } ExprOptScope; #define EXPR_OPT_SCOPE_LIST \ - ((char *)(char[]){ kExprOptScopeGlobal, kExprOptScopeLocal }) + ((char[]){ kExprOptScopeGlobal, kExprOptScopeLocal }) /// All possible variable scopes typedef enum { -- cgit From c9f511d24a64da135bef4b9874c7bec04d9330e4 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 09:06:05 +0300 Subject: viml/parser/expressions: Remove unused flag --- src/nvim/viml/parser/expressions.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 025f0f766e..d783518b3a 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -314,10 +314,6 @@ enum { /// When parsing expressions input by user bar is assumed to be a binary /// operator and other two are spacings. kExprFlagsDisallowEOC = (1 << 1), - /// Print errors when encountered - /// - /// Without the flag they are only taken into account when parsing. - kExprFlagsPrintError = (1 << 2), // WARNING: whenever you add a new flag, alter klee_assume() statement in // viml_expressions_parser.c. } ExprParserFlags; -- cgit From 22d161a5dd1c519f998916f45d61be92662fbb44 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 20:11:44 +0300 Subject: api/vim: Add nvim_parse_expression function --- src/nvim/viml/parser/expressions.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index d783518b3a..d00d4855f3 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -338,6 +338,9 @@ typedef struct { ExprASTNode *root; } ExprAST; +/// Array mapping ExprASTNodeType to maximum amount of children node may have +extern const uint8_t node_maxchildren[]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.h.generated.h" #endif -- cgit From a9b203d23fcc6ba8c4e298b8e082db990e1ec04f Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 30 Oct 2017 01:32:10 +0300 Subject: *: Fix linter errors Big function in expressions.c may be refactored, if I ever catch the idea how to split it right. --- src/nvim/viml/parser/expressions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index d00d4855f3..668c2a4c84 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -189,7 +189,7 @@ typedef enum { kExprNodeCall, ///< Function call. /// Plain identifier: simple variable/function name /// - /// Looks like "string", "g:Foo", etc: consists from a single + /// Looks like "string", "g:Foo", etc: consists from a single /// kExprLexPlainIdentifier token. kExprNodePlainIdentifier, /// Plain dictionary key, for use with kExprNodeConcatOrSubscript -- cgit From 07ec709141886c6db4f944665e07a36ef7302eb4 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 5 Nov 2017 01:33:44 +0300 Subject: vim/api: Actually dump AST, fix some bugs in nvim_parse_expression --- src/nvim/viml/parser/expressions.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 668c2a4c84..648f8cbc1f 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -341,6 +341,15 @@ typedef struct { /// Array mapping ExprASTNodeType to maximum amount of children node may have extern const uint8_t node_maxchildren[]; +/// Array mapping ExprASTNodeType values to their stringified versions +extern const char *const east_node_type_tab[]; + +/// Array mapping ExprComparisonType values to their stringified versions +extern const char *const eltkn_cmp_type_tab[]; + +/// Array mapping ExprCaseCompareStrategy values to their stringified versions +extern const char *const ccs_tab[]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.h.generated.h" #endif -- cgit From c7495ebcc0918ffd682083408895451318e41d1f Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 12 Nov 2017 02:18:43 +0300 Subject: viml/parser/expressions: Add support for parsing assignments --- src/nvim/viml/parser/expressions.h | 40 ++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) (limited to 'src/nvim/viml/parser/expressions.h') diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 648f8cbc1f..23e172da75 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -51,6 +51,9 @@ typedef enum { kExprLexParenthesis, ///< Parenthesis, either opening or closing. kExprLexComma, ///< Comma. kExprLexArrow, ///< Arrow, like from lambda expressions. + kExprLexAssignment, ///< Assignment: `=` or `{op}=`. + // XXX When modifying this enum you need to also modify eltkn_type_tab in + // expressions.c and tests and, possibly, viml_pexpr_repr_token. } LexExprTokenType; typedef enum { @@ -68,6 +71,14 @@ typedef enum { kExprOptScopeLocal = 'l', } ExprOptScope; +/// All possible assignment types: `=` and `{op}=`. +typedef enum { + kExprAsgnPlain = 0, ///< Plain assignment: `=`. + kExprAsgnAdd, ///< Assignment augmented with addition: `+=`. + kExprAsgnSubtract, ///< Assignment augmented with subtraction: `-=`. + kExprAsgnConcat, ///< Assignment augmented with concatenation: `.=`. +} ExprAssignmentType; + #define EXPR_OPT_SCOPE_LIST \ ((char[]){ kExprOptScopeGlobal, kExprOptScopeLocal }) @@ -147,6 +158,10 @@ typedef struct { uint8_t base; ///< Base: 2, 8, 10 or 16. bool is_float; ///< True if number is a floating-point. } num; ///< For kExprLexNumber + + struct { + ExprAssignmentType type; + } ass; ///< For kExprLexAssignment } data; ///< Additional data, if needed. } LexExprToken; @@ -170,8 +185,8 @@ typedef enum { /// “EOC” is something like "|". It is fine with emitting EOC at the end of /// string still, with or without this flag set. kELFlagForbidEOC = (1 << 4), - // WARNING: whenever you add a new flag, alter klee_assume() statement in - // viml_expressions_lexer.c. + // XXX Whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_lexer.c. } LexExprFlags; /// Expression AST node type @@ -233,6 +248,10 @@ typedef enum { kExprNodeMod, kExprNodeOption, kExprNodeEnvironment, + kExprNodeAssignment, + // XXX When modifying this list also modify east_node_type_tab both in parser + // and in tests, and you most likely will also have to alter list of + // highlight groups stored in highlight_init_cmdline variable. } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -301,6 +320,9 @@ struct expr_ast_node { const char *ident; ///< Environment variable name start. size_t ident_len; ///< Environment variable name length. } env; ///< For kExprNodeEnvironment. + struct { + ExprAssignmentType type; + } ass; ///< For kExprNodeAssignment } data; }; @@ -314,8 +336,15 @@ enum { /// When parsing expressions input by user bar is assumed to be a binary /// operator and other two are spacings. kExprFlagsDisallowEOC = (1 << 1), - // WARNING: whenever you add a new flag, alter klee_assume() statement in - // viml_expressions_parser.c. + /// Parse :let argument + /// + /// That mean that top level node must be an assignment and first nodes + /// belong to lvalues. + kExprFlagsParseLet = (1 << 2), + // XXX whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_parser.c, nvim_parse_expression() flags parsing + // alongside with its documentation and flag sets in check_parsing() + // function in expressions parser functional and unit tests. } ExprParserFlags; /// AST error definition @@ -350,6 +379,9 @@ extern const char *const eltkn_cmp_type_tab[]; /// Array mapping ExprCaseCompareStrategy values to their stringified versions extern const char *const ccs_tab[]; +/// Array mapping ExprAssignmentType values to their stringified versions +extern const char *const expr_asgn_type_tab[]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.h.generated.h" #endif -- cgit