Merge pull request #24219 from clason/ts-hoon

feat(treesitter): add python, bash parser and queries

7 files changed, 533 insertions, 2 deletions

diff --git a/cmake.deps/cmake/BuildTreesitterParsers.cmake b/cmake.deps/cmake/BuildTreesitterParsers.cmake
index 89c1ec5cc9..e54544740e 100644
--- a/cmake.deps/cmake/BuildTreesitterParsers.cmake
+++ b/cmake.deps/cmake/BuildTreesitterParsers.cmake
@@ -34,7 +34,7 @@ function(BuildTSParser)
     CMAKE_CACHE_ARGS ${DEPS_CMAKE_CACHE_ARGS})
 endfunction()
 
-foreach(lang c lua vim vimdoc query)
+foreach(lang c lua vim vimdoc query python bash)
   BuildTSParser(LANG ${lang})
 endforeach()
 BuildTSParser(LANG markdown CMAKE_FILE MarkdownParserCMakeLists.txt)
diff --git a/cmake.deps/deps.txt b/cmake.deps/deps.txt
index fd0b4aa400..a62bad20c6 100644
--- a/cmake.deps/deps.txt
+++ b/cmake.deps/deps.txt
@@ -54,6 +54,10 @@ TREESITTER_VIMDOC_URL https://github.com/neovim/tree-sitter-vimdoc/archive/v2.0.
 TREESITTER_VIMDOC_SHA256 61e165df29778dc0c9277c2a7bc67447cc4e1bed36ca916a2f476dd25ce3260e
 TREESITTER_QUERY_URL https://github.com/nvim-treesitter/tree-sitter-query/archive/v0.1.0.tar.gz
 TREESITTER_QUERY_SHA256 e2b806f80e8bf1c4f4e5a96248393fe6622fc1fc6189d6896d269658f67f914c
+TREESITTER_PYTHON_URL https://github.com/tree-sitter/tree-sitter-python/archive/36f9e33d52b7572536ac1a8af8d7e78363ad52c3.tar.gz
+TREESITTER_PYTHON_SHA256 ac92b4759c363fe284a1bfd9df2d204b5efa166042a1484eefdf8e03b3f37d57
+TREESITTER_BASH_URL https://github.com/tree-sitter/tree-sitter-bash/archive/493646764e7ad61ce63ce3b8c59ebeb37f71b841.tar.gz
+TREESITTER_BASH_SHA256 99ebe9f2886efecc1a5e9e1360d804a1b49ad89976a66bb5c3871539cca5fb7e
 TREESITTER_MARKDOWN_URL https://github.com/MDeiml/tree-sitter-markdown/archive/936cc84289f6de83c263ae8e659fb342867ceb16.tar.gz
 TREESITTER_MARKDOWN_SHA256 4f2315930dc2c1bd42971a0b728cf4dafc57830c61f8abe3e2548cf230968713
 TREESITTER_URL https://github.com/tree-sitter/tree-sitter/archive/91e4d940169a0c0b024560632ef53c4f119117ca.tar.gz
diff --git a/runtime/doc/news.txt b/runtime/doc/news.txt
index 5ee9cb7a14..58c338b225 100644
--- a/runtime/doc/news.txt
+++ b/runtime/doc/news.txt
@@ -101,7 +101,8 @@ The following new APIs and features were added.
 • Implemented LSP inlay hints: |vim.lsp.inlay_hint()|
   https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_inlayHint
 
-• Bundled Markdown parser for treesitter highlighting and folding.
+• Bundled treesitter parser and queries (highlight, folds) for Markdown,
+Python, and Bash.
 
 ==============================================================================
 CHANGED FEATURES                                                 *news-changed*
diff --git a/runtime/queries/bash/folds.scm b/runtime/queries/bash/folds.scm
new file mode 100644
index 0000000000..851c67eed4
--- /dev/null
+++ b/runtime/queries/bash/folds.scm
@@ -0,0 +1,8 @@
+[
+  (function_definition)
+  (if_statement)
+  (case_statement)
+  (for_statement)
+  (while_statement)
+  (c_style_for_statement)
+] @fold
diff --git a/runtime/queries/bash/highlights.scm b/runtime/queries/bash/highlights.scm
new file mode 100644
index 0000000000..23bf03e697
--- /dev/null
+++ b/runtime/queries/bash/highlights.scm
@@ -0,0 +1,145 @@
+(simple_expansion) @none
+(expansion
+  "${" @punctuation.special
+  "}" @punctuation.special) @none
+[
+ "("
+ ")"
+ "(("
+ "))"
+ "{"
+ "}"
+ "["
+ "]"
+ "[["
+ "]]"
+ ] @punctuation.bracket
+
+[
+ ";"
+ ";;"
+ (heredoc_start)
+ ] @punctuation.delimiter
+
+[
+ "$"
+] @punctuation.special
+
+[
+ ">"
+ ">>"
+ "<"
+ "<<"
+ "&"
+ "&&"
+ "|"
+ "||"
+ "="
+ "=~"
+ "=="
+ "!="
+ ] @operator
+
+[
+ (string)
+ (raw_string)
+ (ansi_c_string)
+ (heredoc_body)
+] @string @spell
+
+(variable_assignment (word) @string)
+
+[
+ "if"
+ "then"
+ "else"
+ "elif"
+ "fi"
+ "case"
+ "in"
+ "esac"
+ ] @conditional
+
+[
+ "for"
+ "do"
+ "done"
+ "select"
+ "until"
+ "while"
+ ] @repeat
+
+[
+ "declare"
+ "export"
+ "local"
+ "readonly"
+ "unset"
+ ] @keyword
+
+"function" @keyword.function
+
+(special_variable_name) @constant
+
+; trap -l
+((word) @constant.builtin
+ (#match? @constant.builtin "^SIG(HUP|INT|QUIT|ILL|TRAP|ABRT|BUS|FPE|KILL|USR[12]|SEGV|PIPE|ALRM|TERM|STKFLT|CHLD|CONT|STOP|TSTP|TT(IN|OU)|URG|XCPU|XFSZ|VTALRM|PROF|WINCH|IO|PWR|SYS|RTMIN([+]([1-9]|1[0-5]))?|RTMAX(-([1-9]|1[0-4]))?)$"))
+
+((word) @boolean
+  (#any-of? @boolean "true" "false"))
+
+(comment) @comment @spell
+(test_operator) @string
+
+(command_substitution
+  [ "$(" ")" ] @punctuation.bracket)
+
+(process_substitution
+  [ "<(" ")" ] @punctuation.bracket)
+
+
+(function_definition
+  name: (word) @function)
+
+(command_name (word) @function.call)
+
+((command_name (word) @function.builtin)
+ (#any-of? @function.builtin
+    "alias" "bg" "bind" "break" "builtin" "caller" "cd"
+    "command" "compgen" "complete" "compopt" "continue"
+    "coproc" "dirs" "disown" "echo" "enable" "eval"
+    "exec" "exit" "fc" "fg" "getopts" "hash" "help"
+    "history" "jobs" "kill" "let" "logout" "mapfile"
+    "popd" "printf" "pushd" "pwd" "read" "readarray"
+    "return" "set" "shift" "shopt" "source" "suspend"
+    "test" "time" "times" "trap" "type" "typeset"
+    "ulimit" "umask" "unalias" "wait"))
+
+(command
+  argument: [
+             (word) @parameter
+             (concatenation (word) @parameter)
+             ])
+
+((word) @number
+  (#lua-match? @number "^[0-9]+$"))
+
+(file_redirect
+  descriptor: (file_descriptor) @operator
+  destination: (word) @parameter)
+
+(expansion
+  [ "${" "}" ] @punctuation.bracket)
+
+(variable_name) @variable
+
+((variable_name) @constant
+ (#lua-match? @constant "^[A-Z][A-Z_0-9]*$"))
+
+(case_item
+  value: (word) @parameter)
+
+(regex) @string.regex
+
+((program . (comment) @preproc)
+  (#lua-match? @preproc "^#!/"))
diff --git a/runtime/queries/python/folds.scm b/runtime/queries/python/folds.scm
new file mode 100644
index 0000000000..78e1e2c00d
--- /dev/null
+++ b/runtime/queries/python/folds.scm
@@ -0,0 +1,28 @@
+[
+  (function_definition)
+  (class_definition)
+
+  (while_statement)
+  (for_statement)
+  (if_statement)
+  (with_statement)
+  (try_statement)
+  (match_statement)
+
+  (import_from_statement)
+  (parameters)
+  (argument_list)
+
+  (parenthesized_expression)
+  (generator_expression)
+  (list_comprehension)
+  (set_comprehension)
+  (dictionary_comprehension)
+
+  (tuple)
+  (list)
+  (set)
+  (dictionary)
+
+  (string)
+] @fold
diff --git a/runtime/queries/python/highlights.scm b/runtime/queries/python/highlights.scm
new file mode 100644
index 0000000000..c18b748674
--- /dev/null
+++ b/runtime/queries/python/highlights.scm
@@ -0,0 +1,345 @@
+;; From tree-sitter-python licensed under MIT License
+; Copyright (c) 2016 Max Brunsfeld
+
+; Variables
+(identifier) @variable
+
+; Reset highlighting in f-string interpolations
+(interpolation) @none
+
+;; Identifier naming conventions
+((identifier) @type
+ (#lua-match? @type "^[A-Z].*[a-z]"))
+((identifier) @constant
+ (#lua-match? @constant "^[A-Z][A-Z_0-9]*$"))
+
+((identifier) @constant.builtin
+ (#lua-match? @constant.builtin "^__[a-zA-Z0-9_]*__$"))
+
+((identifier) @constant.builtin
+ (#any-of? @constant.builtin
+           ;; https://docs.python.org/3/library/constants.html
+           "NotImplemented"
+           "Ellipsis"
+           "quit"
+           "exit"
+           "copyright"
+           "credits"
+           "license"))
+
+((attribute
+    attribute: (identifier) @field)
+ (#lua-match? @field "^[%l_].*$"))
+
+((assignment
+  left: (identifier) @type.definition
+  (type (identifier) @_annotation))
+ (#eq? @_annotation "TypeAlias"))
+
+((assignment
+  left: (identifier) @type.definition
+  right: (call
+    function: (identifier) @_func))
+ (#any-of? @_func "TypeVar" "NewType"))
+
+; Function calls
+
+(call
+  function: (identifier) @function.call)
+
+(call
+  function: (attribute
+              attribute: (identifier) @method.call))
+
+((call
+   function: (identifier) @constructor)
+ (#lua-match? @constructor "^%u"))
+
+((call
+  function: (attribute
+              attribute: (identifier) @constructor))
+ (#lua-match? @constructor "^%u"))
+
+;; Decorators
+
+((decorator "@" @attribute)
+ (#set! "priority" 101))
+
+(decorator
+  (identifier) @attribute)
+(decorator
+  (attribute
+    attribute: (identifier) @attribute))
+(decorator
+  (call (identifier) @attribute))
+(decorator
+  (call (attribute
+          attribute: (identifier) @attribute)))
+
+((decorator
+  (identifier) @attribute.builtin)
+ (#any-of? @attribute.builtin "classmethod" "property"))
+
+;; Builtin functions
+
+((call
+  function: (identifier) @function.builtin)
+ (#any-of? @function.builtin
+          "abs" "all" "any" "ascii" "bin" "bool" "breakpoint" "bytearray" "bytes" "callable" "chr" "classmethod"
+          "compile" "complex" "delattr" "dict" "dir" "divmod" "enumerate" "eval" "exec" "filter" "float" "format"
+          "frozenset" "getattr" "globals" "hasattr" "hash" "help" "hex" "id" "input" "int" "isinstance" "issubclass"
+          "iter" "len" "list" "locals" "map" "max" "memoryview" "min" "next" "object" "oct" "open" "ord" "pow"
+          "print" "property" "range" "repr" "reversed" "round" "set" "setattr" "slice" "sorted" "staticmethod" "str"
+          "sum" "super" "tuple" "type" "vars" "zip" "__import__"))
+
+;; Function definitions
+
+(function_definition
+  name: (identifier) @function)
+
+(type (identifier) @type)
+(type
+  (subscript
+    (identifier) @type)) ; type subscript: Tuple[int]
+
+((call
+  function: (identifier) @_isinstance
+  arguments: (argument_list
+    (_)
+    (identifier) @type))
+ (#eq? @_isinstance "isinstance"))
+
+;; Normal parameters
+(parameters
+  (identifier) @parameter)
+;; Lambda parameters
+(lambda_parameters
+  (identifier) @parameter)
+(lambda_parameters
+  (tuple_pattern
+    (identifier) @parameter))
+; Default parameters
+(keyword_argument
+  name: (identifier) @parameter)
+; Naming parameters on call-site
+(default_parameter
+  name: (identifier) @parameter)
+(typed_parameter
+  (identifier) @parameter)
+(typed_default_parameter
+  (identifier) @parameter)
+; Variadic parameters *args, **kwargs
+(parameters
+  (list_splat_pattern ; *args
+    (identifier) @parameter))
+(parameters
+  (dictionary_splat_pattern ; **kwargs
+    (identifier) @parameter))
+
+
+;; Literals
+
+(none) @constant.builtin
+[(true) (false)] @boolean
+((identifier) @variable.builtin
+ (#eq? @variable.builtin "self"))
+((identifier) @variable.builtin
+ (#eq? @variable.builtin "cls"))
+
+(integer) @number
+(float) @float
+
+(comment) @comment @spell
+
+((module . (comment) @preproc)
+  (#lua-match? @preproc "^#!/"))
+
+(string) @string
+(escape_sequence) @string.escape
+
+; doc-strings
+
+(module . (expression_statement (string) @string.documentation @spell))
+
+(class_definition
+  body:
+    (block
+      . (expression_statement (string) @string.documentation @spell)))
+
+(function_definition
+  body:
+    (block
+      . (expression_statement (string) @string.documentation @spell)))
+
+; Tokens
+
+[
+  "-"
+  "-="
+  ":="
+  "!="
+  "*"
+  "**"
+  "**="
+  "*="
+  "/"
+  "//"
+  "//="
+  "/="
+  "&"
+  "&="
+  "%"
+  "%="
+  "^"
+  "^="
+  "+"
+  "+="
+  "<"
+  "<<"
+  "<<="
+  "<="
+  "<>"
+  "="
+  "=="
+  ">"
+  ">="
+  ">>"
+  ">>="
+  "@"
+  "@="
+  "|"
+  "|="
+  "~"
+  "->"
+] @operator
+
+; Keywords
+[
+  "and"
+  "in"
+  "is"
+  "not"
+  "or"
+  "is not"
+  "not in"
+
+  "del"
+] @keyword.operator
+
+[
+  "def"
+  "lambda"
+] @keyword.function
+
+[
+  "assert"
+  "class"
+  "exec"
+  "global"
+  "nonlocal"
+  "pass"
+  "print"
+  "with"
+  "as"
+] @keyword
+
+[
+  "async"
+  "await"
+] @keyword.coroutine
+
+[
+  "return"
+  "yield"
+] @keyword.return
+(yield "from" @keyword.return)
+
+(future_import_statement
+  "from" @include
+  "__future__" @constant.builtin)
+(import_from_statement "from" @include)
+"import" @include
+
+(aliased_import "as" @include)
+
+["if" "elif" "else" "match" "case"] @conditional
+
+["for" "while" "break" "continue"] @repeat
+
+[
+  "try"
+  "except"
+  "except*"
+  "raise"
+  "finally"
+] @exception
+
+(raise_statement "from" @exception)
+
+(try_statement
+  (else_clause
+    "else" @exception))
+
+["(" ")" "[" "]" "{" "}"] @punctuation.bracket
+
+(interpolation
+  "{" @punctuation.special
+  "}" @punctuation.special)
+
+(type_conversion) @function.macro
+
+["," "." ":" ";" (ellipsis)] @punctuation.delimiter
+
+;; Class definitions
+
+(class_definition name: (identifier) @type)
+
+(class_definition
+  body: (block
+          (function_definition
+            name: (identifier) @method)))
+
+(class_definition
+  superclasses: (argument_list
+                  (identifier) @type))
+
+((class_definition
+  body: (block
+          (expression_statement
+            (assignment
+              left: (identifier) @field))))
+ (#lua-match? @field "^%l.*$"))
+((class_definition
+  body: (block
+          (expression_statement
+            (assignment
+              left: (_
+                     (identifier) @field)))))
+ (#lua-match? @field "^%l.*$"))
+
+((class_definition
+  (block
+    (function_definition
+      name: (identifier) @constructor)))
+ (#any-of? @constructor "__new__" "__init__"))
+
+((identifier) @type.builtin
+ (#any-of? @type.builtin
+              ;; https://docs.python.org/3/library/exceptions.html
+              "BaseException" "Exception" "ArithmeticError" "BufferError" "LookupError" "AssertionError" "AttributeError"
+              "EOFError" "FloatingPointError" "GeneratorExit" "ImportError" "ModuleNotFoundError" "IndexError" "KeyError"
+              "KeyboardInterrupt" "MemoryError" "NameError" "NotImplementedError" "OSError" "OverflowError" "RecursionError"
+              "ReferenceError" "RuntimeError" "StopIteration" "StopAsyncIteration" "SyntaxError" "IndentationError" "TabError"
+              "SystemError" "SystemExit" "TypeError" "UnboundLocalError" "UnicodeError" "UnicodeEncodeError" "UnicodeDecodeError"
+              "UnicodeTranslateError" "ValueError" "ZeroDivisionError" "EnvironmentError" "IOError" "WindowsError"
+              "BlockingIOError" "ChildProcessError" "ConnectionError" "BrokenPipeError" "ConnectionAbortedError"
+              "ConnectionRefusedError" "ConnectionResetError" "FileExistsError" "FileNotFoundError" "InterruptedError"
+              "IsADirectoryError" "NotADirectoryError" "PermissionError" "ProcessLookupError" "TimeoutError" "Warning"
+              "UserWarning" "DeprecationWarning" "PendingDeprecationWarning" "SyntaxWarning" "RuntimeWarning"
+              "FutureWarning" "ImportWarning" "UnicodeWarning" "BytesWarning" "ResourceWarning"
+              ;; https://docs.python.org/3/library/stdtypes.html
+              "bool" "int" "float" "complex" "list" "tuple" "range" "str"
+              "bytes" "bytearray" "memoryview" "set" "frozenset" "dict" "type" "object"))
+
+;; Error
+(ERROR) @error