From 3bddf050230635febc16aabe0ba4f73abeed6663 Mon Sep 17 00:00:00 2001 From: Björn Linse Date: Thu, 6 Jun 2019 10:34:01 +0200 Subject: tree-sitter: vendor tree-sitter runtime tree-sitter/tree-sitter commit 7685b7861ca475664b6ef57e14d1da9acf741275 Included files are: lib/include/tree-sitter/*.h lib/src/*.[ch] LICENSE --- src/tree_sitter/parser.h | 220 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 src/tree_sitter/parser.h (limited to 'src/tree_sitter/parser.h') diff --git a/src/tree_sitter/parser.h b/src/tree_sitter/parser.h new file mode 100644 index 0000000000..974a7ca52f --- /dev/null +++ b/src/tree_sitter/parser.h @@ -0,0 +1,220 @@ +#ifndef TREE_SITTER_PARSER_H_ +#define TREE_SITTER_PARSER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#define ts_builtin_sym_error ((TSSymbol)-1) +#define ts_builtin_sym_end 0 +#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 + +#ifndef TREE_SITTER_API_H_ +typedef uint16_t TSSymbol; +typedef uint16_t TSFieldId; +typedef struct TSLanguage TSLanguage; +#endif + +typedef struct { + TSFieldId field_id; + uint8_t child_index; + bool inherited; +} TSFieldMapEntry; + +typedef struct { + uint16_t index; + uint16_t length; +} TSFieldMapSlice; + +typedef uint16_t TSStateId; + +typedef struct { + bool visible : 1; + bool named : 1; +} TSSymbolMetadata; + +typedef struct TSLexer TSLexer; + +struct TSLexer { + int32_t lookahead; + TSSymbol result_symbol; + void (*advance)(TSLexer *, bool); + void (*mark_end)(TSLexer *); + uint32_t (*get_column)(TSLexer *); + bool (*is_at_included_range_start)(TSLexer *); +}; + +typedef enum { + TSParseActionTypeShift, + TSParseActionTypeReduce, + TSParseActionTypeAccept, + TSParseActionTypeRecover, +} TSParseActionType; + +typedef struct { + union { + struct { + TSStateId state; + bool extra : 1; + bool repetition : 1; + }; + struct { + TSSymbol symbol; + int16_t dynamic_precedence; + uint8_t child_count; + uint8_t production_id; + }; + } params; + TSParseActionType type : 4; +} TSParseAction; + +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; +} TSLexMode; + +typedef union { + TSParseAction action; + struct { + uint8_t count; + bool reusable : 1; + }; +} TSParseActionEntry; + +struct TSLanguage { + uint32_t version; + uint32_t symbol_count; + uint32_t alias_count; + uint32_t token_count; + uint32_t external_token_count; + const char **symbol_names; + const TSSymbolMetadata *symbol_metadata; + const uint16_t *parse_table; + const TSParseActionEntry *parse_actions; + const TSLexMode *lex_modes; + const TSSymbol *alias_sequences; + uint16_t max_alias_sequence_length; + bool (*lex_fn)(TSLexer *, TSStateId); + bool (*keyword_lex_fn)(TSLexer *, TSStateId); + TSSymbol keyword_capture_token; + struct { + const bool *states; + const TSSymbol *symbol_map; + void *(*create)(void); + void (*destroy)(void *); + bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); + unsigned (*serialize)(void *, char *); + void (*deserialize)(void *, const char *, unsigned); + } external_scanner; + uint32_t field_count; + const TSFieldMapSlice *field_map_slices; + const TSFieldMapEntry *field_map_entries; + const char **field_names; + uint32_t large_state_count; + const uint16_t *small_parse_table; + const uint32_t *small_parse_table_map; +}; + +/* + * Lexer Macros + */ + +#define START_LEXER() \ + bool result = false; \ + bool skip = false; \ + int32_t lookahead; \ + goto start; \ + next_state: \ + lexer->advance(lexer, skip); \ + start: \ + skip = false; \ + lookahead = lexer->lookahead; + +#define ADVANCE(state_value) \ + { \ + state = state_value; \ + goto next_state; \ + } + +#define SKIP(state_value) \ + { \ + skip = true; \ + state = state_value; \ + goto next_state; \ + } + +#define ACCEPT_TOKEN(symbol_value) \ + result = true; \ + lexer->result_symbol = symbol_value; \ + lexer->mark_end(lexer); + +#define END_STATE() return result; + +/* + * Parse Table Macros + */ + +#define SMALL_STATE(id) id - LARGE_STATE_COUNT + +#define STATE(id) id + +#define ACTIONS(id) id + +#define SHIFT(state_value) \ + { \ + { \ + .type = TSParseActionTypeShift, \ + .params = {.state = state_value}, \ + } \ + } + +#define SHIFT_REPEAT(state_value) \ + { \ + { \ + .type = TSParseActionTypeShift, \ + .params = { \ + .state = state_value, \ + .repetition = true \ + }, \ + } \ + } + +#define RECOVER() \ + { \ + { .type = TSParseActionTypeRecover } \ + } + +#define SHIFT_EXTRA() \ + { \ + { \ + .type = TSParseActionTypeShift, \ + .params = {.extra = true} \ + } \ + } + +#define REDUCE(symbol_val, child_count_val, ...) \ + { \ + { \ + .type = TSParseActionTypeReduce, \ + .params = { \ + .symbol = symbol_val, \ + .child_count = child_count_val, \ + __VA_ARGS__ \ + } \ + } \ + } + +#define ACCEPT_INPUT() \ + { \ + { .type = TSParseActionTypeAccept } \ + } + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_PARSER_H_ -- cgit From 79bd8d2ab6cae1c0be3233a9a7551d0b7bcc5944 Mon Sep 17 00:00:00 2001 From: Björn Linse Date: Sat, 28 Sep 2019 18:41:49 +0200 Subject: tree-sitter: update vendored tree-sitter runtime tree-sitter/tree-sitter commit edb569310005c66838b7d69fa60850acac6abeee Included files are: lib/include/tree-sitter/*.h lib/src/*.[ch] lib/src/unicode/* LICENSE --- src/tree_sitter/parser.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/tree_sitter/parser.h') diff --git a/src/tree_sitter/parser.h b/src/tree_sitter/parser.h index 974a7ca52f..9df91f8c3c 100644 --- a/src/tree_sitter/parser.h +++ b/src/tree_sitter/parser.h @@ -45,7 +45,8 @@ struct TSLexer { void (*advance)(TSLexer *, bool); void (*mark_end)(TSLexer *); uint32_t (*get_column)(TSLexer *); - bool (*is_at_included_range_start)(TSLexer *); + bool (*is_at_included_range_start)(const TSLexer *); + bool (*eof)(const TSLexer *); }; typedef enum { @@ -117,6 +118,7 @@ struct TSLanguage { uint32_t large_state_count; const uint16_t *small_parse_table; const uint32_t *small_parse_table_map; + const TSSymbol *public_symbol_map; }; /* @@ -126,6 +128,7 @@ struct TSLanguage { #define START_LEXER() \ bool result = false; \ bool skip = false; \ + bool eof = false; \ int32_t lookahead; \ goto start; \ next_state: \ -- cgit From 8349192503450d645bad6a2b30a72c67fd97f7c8 Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Fri, 15 May 2020 12:23:26 +0200 Subject: treesitter: update runtime Since tree-sitter PR 615, predicates are not parsed the same. "Old" way of writing predicates is still supported. --- src/tree_sitter/parser.h | 50 ++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 19 deletions(-) (limited to 'src/tree_sitter/parser.h') diff --git a/src/tree_sitter/parser.h b/src/tree_sitter/parser.h index 9df91f8c3c..11bf4fc42a 100644 --- a/src/tree_sitter/parser.h +++ b/src/tree_sitter/parser.h @@ -62,13 +62,13 @@ typedef struct { TSStateId state; bool extra : 1; bool repetition : 1; - }; + } shift; struct { TSSymbol symbol; int16_t dynamic_precedence; uint8_t child_count; uint8_t production_id; - }; + } reduce; } params; TSParseActionType type : 4; } TSParseAction; @@ -83,7 +83,7 @@ typedef union { struct { uint8_t count; bool reusable : 1; - }; + } entry; } TSParseActionEntry; struct TSLanguage { @@ -167,22 +167,28 @@ struct TSLanguage { #define ACTIONS(id) id -#define SHIFT(state_value) \ - { \ - { \ - .type = TSParseActionTypeShift, \ - .params = {.state = state_value}, \ - } \ +#define SHIFT(state_value) \ + { \ + { \ + .params = { \ + .shift = { \ + .state = state_value \ + } \ + }, \ + .type = TSParseActionTypeShift \ + } \ } #define SHIFT_REPEAT(state_value) \ { \ { \ - .type = TSParseActionTypeShift, \ .params = { \ - .state = state_value, \ - .repetition = true \ + .shift = { \ + .state = state_value, \ + .repetition = true \ + } \ }, \ + .type = TSParseActionTypeShift \ } \ } @@ -194,20 +200,26 @@ struct TSLanguage { #define SHIFT_EXTRA() \ { \ { \ - .type = TSParseActionTypeShift, \ - .params = {.extra = true} \ + .params = { \ + .shift = { \ + .extra = true \ + } \ + }, \ + .type = TSParseActionTypeShift \ } \ } #define REDUCE(symbol_val, child_count_val, ...) \ { \ { \ - .type = TSParseActionTypeReduce, \ .params = { \ - .symbol = symbol_val, \ - .child_count = child_count_val, \ - __VA_ARGS__ \ - } \ + .reduce = { \ + .symbol = symbol_val, \ + .child_count = child_count_val, \ + __VA_ARGS__ \ + }, \ + }, \ + .type = TSParseActionTypeReduce \ } \ } -- cgit