tree-sitter: implement query functionality and highlighting prototype [skip.lint]

3 files changed, 350 insertions, 8 deletions

diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt
index c0da06ffe3..1c3a7f70c9 100644
--- a/runtime/doc/lua.txt
+++ b/runtime/doc/lua.txt
@@ -594,6 +594,102 @@ tsnode:named_descendant_for_range(start_row, start_col, end_row, end_col)
 	Get the smallest named node within this node that spans the given
 	range of (row, column) positions
 
+Query methods						*lua-treesitter-query*
+
+Tree-sitter queries are supported, with some limitations. Currently, the only
+supported match predicate is `eq?` (both comparing a capture against a string
+and two captures against each other).
+
+vim.treesitter.parse_query(lang, query)
+						*vim.treesitter.parse_query(()*
+	Parse the query as a string. (If the query is in a file, the caller
+        should read the contents into a string before calling).
+
+query:iter_captures(node, bufnr, start_row, end_row)
+							*query:iter_captures()*
+	Iterate over all captures from all matches inside a `node`.
+	`bufnr` is needed if the query contains predicates, then the caller
+	must ensure to use a freshly parsed tree consistent with the current
+	text of the buffer. `start_row` and `end_row` can be used to limit
+	matches inside a row range (this is typically used with root node
+	as the node, i e to get syntax highlight matches in the current
+	viewport)
+
+	The iterator returns two values, a numeric id identifying the capture
+	and the captured node. The following example shows how to get captures
+	by name:
+>
+	for id, node in query:iter_captures(tree:root(), bufnr, first, last) do
+	  local name = query.captures[id] -- name of the capture in the query
+	  -- typically useful info about the node:
+	  local type = node:type() -- type of the captured node
+	  local row1, col1, row2, col2 = node:range() -- range of the capture
+	  ... use the info here ...
+	end
+<
+query:iter_matches(node, bufnr, start_row, end_row)
+							*query:iter_matches()*
+	Iterate over all matches within a node. The arguments are the same as
+	for |query:iter_captures()| but the iterated values are different:
+	an (1-based) index of the pattern in the query, and a table mapping
+	capture indices to nodes. If the query has more than one pattern
+	the capture table might be sparse, and e.g. `pairs` should be used and not
+	`ipairs`. Here an example iterating over all captures in
+	every match:
+>
+	for pattern, match in cquery:iter_matches(tree:root(), bufnr, first, last) do
+	  for id,node in pairs(match) do
+	    local name = query.captures[id]
+	    -- `node` was captured by the `name` capture in the match
+	    ... use the info here ...
+	  end
+	end
+>
+Treesitter syntax highlighting (WIP)		    *lua-treesitter-highlight*
+
+NOTE: This is a partially implemented feature, and not usable as a default
+solution yet. What is documented here is a temporary interface indented
+for those who want to experiment with this feature and contribute to
+its development.
+
+Highlights are defined in the same query format as in the tree-sitter highlight
+crate, which some limitations and additions. Set a highlight query for a
+buffer with this code: >
+
+    local query = [[
+      "for" @keyword
+      "if" @keyword
+      "return" @keyword
+
+      (string_literal) @string
+      (number_literal) @number
+      (comment) @comment
+
+      (preproc_function_def name: (identifier) @function)
+
+      ; ... more definitions
+    ]]
+
+    highlighter = vim.treesitter.TSHighlighter.new(query, bufnr, lang)
+    -- alternatively, to use the current buffer and its filetype:
+    -- highlighter = vim.treesitter.TSHighlighter.new(query)
+
+    -- Don't recreate the highlighter for the same buffer, instead
+    -- modify the query like this:
+    local query2 = [[ ... ]]
+    highlighter:set_query(query2)
+
+As mentioned above the supported predicate is currently only `eq?`. `match?`
+predicates behave like matching always fails. As an addition a capture which
+begin with an upper-case letter like `@WarningMsg` will map directly to this
+highlight group, if defined. Also if the predicate begins with upper-case and
+contains a dot only the part before the first will be interpreted as the
+highlight group. As an example, this warns of a binary expression with two
+identical identifiers, highlighting both as |hl-WarningMsg|: >
+
+    ((binary_expression left: (identifier) @WarningMsg.left right: (identifier) @WarningMsg.right)
+     (eq? @WarningMsg.left @WarningMsg.right))
+
 ------------------------------------------------------------------------------
 VIM							*lua-builtin*
 
diff --git a/runtime/lua/vim/treesitter.lua b/runtime/lua/vim/treesitter.lua
index e0202927bb..aa8b8fcdd1 100644
--- a/runtime/lua/vim/treesitter.lua
+++ b/runtime/lua/vim/treesitter.lua
@@ -12,9 +12,13 @@ function Parser:parse()
   if self.valid then
     return self.tree
   end
-  self.tree = self._parser:parse_buf(self.bufnr)
+  local changes
+  self.tree, changes = self._parser:parse_buf(self.bufnr)
   self.valid = true
-  return self.tree
+  for _, cb in ipairs(self.change_cbs) do
+    cb(changes)
+  end
+  return self.tree, changes
 end
 
 function Parser:_on_lines(bufnr, _, start_row, old_stop_row, stop_row, old_byte_size)
@@ -26,17 +30,28 @@ function Parser:_on_lines(bufnr, _, start_row, old_stop_row, stop_row, old_byte_
   self.valid = false
 end
 
-local module = {
+local M = {
   add_language=vim._ts_add_language,
   inspect_language=vim._ts_inspect_language,
+  parse_query = vim._ts_parse_query,
 }
 
-function module.create_parser(bufnr, ft, id)
+setmetatable(M, {
+  __index = function (t, k)
+      if k == "TSHighlighter" then
+        t[k] = require'vim.tshighlighter'
+        return t[k]
+      end
+   end
+ })
+
+function M.create_parser(bufnr, ft, id)
   if bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
-  local self = setmetatable({bufnr=bufnr, valid=false}, Parser)
+  local self = setmetatable({bufnr=bufnr, lang=ft, valid=false}, Parser)
   self._parser = vim._create_ts_parser(ft)
+  self.change_cbs = {}
   self:parse()
     -- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is
     -- using it.
@@ -55,7 +70,7 @@ function module.create_parser(bufnr, ft, id)
   return self
 end
 
-function module.get_parser(bufnr, ft)
+function M.get_parser(bufnr, ft, cb)
   if bufnr == nil or bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
@@ -65,9 +80,98 @@ function module.get_parser(bufnr, ft)
   local id = tostring(bufnr)..'_'..ft
 
   if parsers[id] == nil then
-    parsers[id] = module.create_parser(bufnr, ft, id)
+    parsers[id] = M.create_parser(bufnr, ft, id)
+  end
+  if cb ~= nil then
+    table.insert(parsers[id].change_cbs, cb)
   end
   return parsers[id]
 end
 
-return module
+-- query: pattern matching on trees
+-- predicate matching is implemented in lua
+local Query = {}
+Query.__index = Query
+
+function M.parse_query(lang, query)
+  local self = setmetatable({}, Query)
+  self.query = vim._ts_parse_query(lang, query)
+  self.info = self.query:inspect()
+  self.captures = self.info.captures
+  return self
+end
+
+local function get_node_text(node, bufnr)
+  local start_row, start_col, end_row, end_col = node:range()
+  if start_row ~= end_row then
+    return nil
+  end
+  local line = a.nvim_buf_get_lines(bufnr, start_row, start_row+1, true)[1]
+  return string.sub(line, start_col+1, end_col)
+end
+
+local function match_preds(match, preds, bufnr)
+  for _, pred in pairs(preds) do
+    if pred[1] == "eq?" then
+      local node = match[pred[2]]
+      local node_text = get_node_text(node, bufnr)
+
+      local str
+      if type(pred[3]) == "string" then
+        -- (eq? @aa "foo")
+        str = pred[3]
+      else
+        -- (eq? @aa @bb)
+        str = get_node_text(match[pred[3]], bufnr)
+      end
+
+      if node_text ~= str or str == nil then
+        return false
+      end
+    else
+      return false
+    end
+  end
+  return true
+end
+
+function Query:iter_captures(node, bufnr, start, stop)
+  if bufnr == 0 then
+    bufnr = vim.api.nvim_get_current_buf()
+  end
+  local raw_iter = node:_rawquery(self.query,true,start,stop)
+  local function iter()
+    local capture, captured_node, match = raw_iter()
+    if match ~= nil then
+      local preds = self.info.patterns[match.pattern]
+      local active = match_preds(match, preds, bufnr)
+      match.active = active
+      if not active then
+        return iter() -- tail call: try next match
+      end
+    end
+    return capture, captured_node
+  end
+  return iter
+end
+
+function Query:iter_matches(node, bufnr, start, stop)
+  if bufnr == 0 then
+    bufnr = vim.api.nvim_get_current_buf()
+  end
+  local raw_iter = node:_rawquery(self.query,false,start,stop)
+  local function iter()
+    local pattern, match = raw_iter()
+    if match ~= nil then
+      local preds = self.info.patterns[pattern]
+      local active = (not preds) or match_preds(match, preds, bufnr)
+      if not active then
+        return iter() -- tail call: try next match
+      end
+    end
+    return pattern, match
+  end
+  return iter
+end
+
+return M
diff --git a/runtime/lua/vim/tshighlighter.lua b/runtime/lua/vim/tshighlighter.lua
new file mode 100644
index 0000000000..1544ecbf49
--- /dev/null
+++ b/runtime/lua/vim/tshighlighter.lua
@@ -0,0 +1,142 @@
+local a = vim.api
+
+-- support reload for quick experimentation
+local TSHighlighter = rawget(vim.treesitter, 'TSHighlighter') or {}
+TSHighlighter.__index = TSHighlighter
+
+-- These are conventions defined by tree-sitter, though it
+-- needs to be user extensible also.
+-- TODO(bfredl): this is very much incomplete, we will need to
+-- go through a few tree-sitter provided queries and decide
+-- on translations that makes the most sense.
+TSHighlighter.hl_map = {
+    keyword="Keyword",
+    string="String",
+    type="Type",
+    comment="Comment",
+    constant="Constant",
+    operator="Operator",
+    number="Number",
+    label="Label",
+    ["function"]="Function",
+    ["function.special"]="Function",
+}
+
+function TSHighlighter.new(query, bufnr, ft)
+  local self = setmetatable({}, TSHighlighter)
+  self.parser = vim.treesitter.get_parser(bufnr, ft, function(...) self:on_change(...) end)
+  self.buf = self.parser.bufnr
+  -- TODO(bfredl): perhaps on_start should be called uncondionally, instead for only on mod?
+  local tree = self.parser:parse()
+  self.root = tree:root()
+  self:set_query(query)
+  self.edit_count = 0
+  self.redraw_count = 0
+  self.line_count = {}
+  a.nvim_buf_set_option(self.buf, "syntax", "")
+  a.nvim__buf_set_luahl(self.buf, {
+    on_start=function(...) return self:on_start(...) end,
+    on_window=function(...) return self:on_window(...) end,
+    on_line=function(...) return self:on_line(...) end,
+  })
+
+  -- Tricky: if syntax hasn't been enabled, we need to reload color scheme
+  -- but use synload.vim rather than syntax.vim to not enable
+  -- syntax FileType autocmds. Later on we should integrate with the
+  -- `:syntax` and `set syntax=...` machinery properly.
+  if vim.g.syntax_on ~= 1 then
+    vim.api.nvim_command("runtime! syntax/synload.vim")
+  end
+  return self
+end
+
+function TSHighlighter:set_query(query)
+  if type(query) == "string" then
+    query = vim.treesitter.parse_query(self.parser.lang, query)
+  end
+  self.query = query
+
+  self.id_map = {}
+  for i, capture in ipairs(self.query.captures) do
+    local hl = 0
+    local firstc = string.sub(capture, 1, 1)
+    local hl_group = self.hl_map[capture]
+    if firstc ~= string.lower(firstc) then
+      hl_group = vim.split(capture, '.', true)[1]
+    end
+    if hl_group then
+      hl = a.nvim_get_hl_id_by_name(hl_group)
+    end
+    self.id_map[i] = hl
+  end
+end
+
+function TSHighlighter:on_change(changes)
+  for _, ch in ipairs(changes or {}) do
+    a.nvim__buf_redraw_range(self.buf, ch[1], ch[3]+1)
+  end
+  self.edit_count = self.edit_count + 1
+end
+
+function TSHighlighter:on_start(_, _buf, _tick)
+  local tree = self.parser:parse()
+  self.root = tree:root()
+end
+
+function TSHighlighter:on_window(_, _win, _buf, _topline, botline)
+  self.iter = nil
+  self.active_nodes = {}
+  self.nextrow = 0
+  self.botline = botline
+  self.redraw_count = self.redraw_count + 1
+end
+
+function TSHighlighter:on_line(_, _win, buf, line)
+  if self.iter == nil then
+    self.iter = self.query:iter_captures(self.root,buf,line,self.botline)
+  end
+  while line >= self.nextrow do
+    local capture, node, match = self.iter()
+    local active = true
+    if capture == nil then
+      break
+    end
+    if match ~= nil then
+      active = self:run_pred(match)
+      match.active = active
+    end
+    local start_row, start_col, end_row, end_col = node:range()
+    local hl = self.id_map[capture]
+    if hl > 0 and active then
+      if start_row == line and end_row == line then
+        a.nvim__put_attr(hl, start_col, end_col)
+      elseif end_row >= line then
+        -- TODO(bfredl): this is quite messy. Togheter with multiline bufhl we should support
+        -- luahl generating multiline highlights (and other kinds of annotations)
+        self.active_nodes[{hl=hl, start_row=start_row, start_col=start_col, end_row=end_row, end_col=end_col}] = true
+      end
+    end
+    if start_row > line then
+      self.nextrow = start_row
+    end
+  end
+  for node,_ in pairs(self.active_nodes) do
+    if node.start_row <= line and node.end_row >= line then
+      local start_col, end_col = node.start_col, node.end_col
+      if node.start_row < line then
+        start_col = 0
+      end
+      if node.end_row > line then
+        end_col = 9000
+      end
+      a.nvim__put_attr(node.hl, start_col, end_col)
+    end
+    if node.end_row <= line then
+      self.active_nodes[node] = nil
+    end
+  end
+  self.line_count[line] = (self.line_count[line] or 0) + 1
+  --return tostring(self.line_count[line])
+end
+
+return TSHighlighter