From 613068071e02ddf5958fa82974373c370112c5e6 Mon Sep 17 00:00:00 2001
From: Thomas Vigouroux <tomvig38@gmail.com>
Date: Tue, 14 Jul 2020 21:50:57 +0200
Subject: treesitter: refactor and use lua regexes

---
 runtime/lua/vim/treesitter.lua             | 197 ++++++-----------------------
 runtime/lua/vim/treesitter/highlighter.lua | 160 +++++++++++++++++++++++
 runtime/lua/vim/treesitter/language.lua    |  26 ++++
 runtime/lua/vim/treesitter/query.lua       | 133 +++++++++++++++++++
 runtime/lua/vim/tshighlighter.lua          | 116 -----------------
 5 files changed, 357 insertions(+), 275 deletions(-)
 create mode 100644 runtime/lua/vim/treesitter/highlighter.lua
 create mode 100644 runtime/lua/vim/treesitter/language.lua
 create mode 100644 runtime/lua/vim/treesitter/query.lua
 delete mode 100644 runtime/lua/vim/tshighlighter.lua

(limited to 'runtime/lua/vim')

diff --git a/runtime/lua/vim/treesitter.lua b/runtime/lua/vim/treesitter.lua
index 927456708c..f43c8a872d 100644
--- a/runtime/lua/vim/treesitter.lua
+++ b/runtime/lua/vim/treesitter.lua
@@ -1,4 +1,6 @@
 local a = vim.api
+local query = require'vim.treesitter.query'
+local language = require'vim.treesitter.language'
 
 -- TODO(bfredl): currently we retain parsers for the lifetime of the buffer.
 -- Consider use weak references to release parser if all plugins are done with
@@ -44,42 +46,30 @@ function Parser:set_included_ranges(ranges)
   self.valid = false
 end
 
-local M = {
-  parse_query = vim._ts_parse_query,
-}
+local M = vim.tbl_extend("error", query, language)
 
 setmetatable(M, {
   __index = function (t, k)
       if k == "TSHighlighter" then
-        t[k] = require'vim.tshighlighter'
+        a.nvim_err_writeln("vim.TSHighlighter is deprecated, please use vim.treesitter.highlighter")
+        t[k] = require'vim.treesitter.highlighter'
+        return t[k]
+      elseif k == "highlighter" then
+        t[k] = require'vim.treesitter.highlighter'
         return t[k]
       end
    end
  })
 
-function M.require_language(lang, path)
-  if vim._ts_has_language(lang) then
-    return true
-  end
-  if path == nil then
-    local fname = 'parser/' .. lang .. '.*'
-    local paths = a.nvim_get_runtime_file(fname, false)
-    if #paths == 0 then
-      -- TODO(bfredl): help tag?
-      error("no parser for '"..lang.."' language")
-    end
-    path = paths[1]
-  end
-  vim._ts_add_language(path, lang)
-end
-
-function M.inspect_language(lang)
-  M.require_language(lang)
-  return vim._ts_inspect_language(lang)
-end
-
-function M.create_parser(bufnr, lang, id)
-  M.require_language(lang)
+--- Creates a new parser.
+--
+-- It is not recommended to use this, use vim.treesitter.get_parser() instead.
+--
+-- @param bufnr The buffer the parser will be tied to
+-- @param lang The language of the parser.
+-- @param id The id the parser will have
+function M._create_parser(bufnr, lang, id)
+  language.require_language(lang)
   if bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
@@ -91,8 +81,8 @@ function M.create_parser(bufnr, lang, id)
   self.changedtree_cbs = {}
   self.lines_cbs = {}
   self:parse()
-    -- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is
-    -- using it.
+  -- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is
+  -- using it.
   local function lines_cb(_, ...)
     return self:_on_lines(...)
   end
@@ -108,17 +98,31 @@ function M.create_parser(bufnr, lang, id)
   return self
 end
 
-function M.get_parser(bufnr, ft, buf_attach_cbs)
+--- Gets the parser for this bufnr / ft combination.
+--
+-- If needed this will create the parser.
+-- Unconditionnally attach the provided callback
+--
+-- @param bufnr The buffer the parser should be tied to
+-- @param ft The filetype of this parser
+-- @param buf_attach_cbs An `nvim_buf_attach`-like table argument with the following keys :
+--  `on_lines` : see `nvim_buf_attach`, but this will be called _after_ the parsers callback.
+--  `on_changedtree` : a callback that will be called everytime the tree has syntactical changes.
+--      it will only be passed one argument, that is a table of the ranges (as node ranges) that
+--      changed.
+--
+-- @returns The parser
+function M.get_parser(bufnr, lang, buf_attach_cbs)
   if bufnr == nil or bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
-  if ft == nil then
-    ft = a.nvim_buf_get_option(bufnr, "filetype")
+  if lang == nil then
+    lang = a.nvim_buf_get_option(bufnr, "filetype")
   end
-  local id = tostring(bufnr)..'_'..ft
+  local id = tostring(bufnr)..'_'..lang
 
   if parsers[id] == nil then
-    parsers[id] = M.create_parser(bufnr, ft, id)
+    parsers[id] = M._create_parser(bufnr, lang, id)
   end
 
   if buf_attach_cbs and buf_attach_cbs.on_changedtree then
@@ -132,129 +136,4 @@ function M.get_parser(bufnr, ft, buf_attach_cbs)
   return parsers[id]
 end
 
--- query: pattern matching on trees
--- predicate matching is implemented in lua
-local Query = {}
-Query.__index = Query
-
-local magic_prefixes = {['\\v']=true, ['\\m']=true, ['\\M']=true, ['\\V']=true}
-local function check_magic(str)
-  if string.len(str) < 2 or magic_prefixes[string.sub(str,1,2)] then
-    return str
-  end
-  return '\\v'..str
-end
-
-function M.parse_query(lang, query)
-  M.require_language(lang)
-  local self = setmetatable({}, Query)
-  self.query = vim._ts_parse_query(lang, vim.fn.escape(query,'\\'))
-  self.info = self.query:inspect()
-  self.captures = self.info.captures
-  self.regexes = {}
-  for id,preds in pairs(self.info.patterns) do
-    local regexes = {}
-    for i, pred in ipairs(preds) do
-      if (pred[1] == "match?" and type(pred[2]) == "number"
-          and type(pred[3]) == "string") then
-        regexes[i] = vim.regex(check_magic(pred[3]))
-      end
-    end
-    if next(regexes) then
-      self.regexes[id] = regexes
-    end
-  end
-  return self
-end
-
-local function get_node_text(node, bufnr)
-  local start_row, start_col, end_row, end_col = node:range()
-  if start_row ~= end_row then
-    return nil
-  end
-  local line = a.nvim_buf_get_lines(bufnr, start_row, start_row+1, true)[1]
-  return string.sub(line, start_col+1, end_col)
-end
-
-function Query:match_preds(match, pattern, bufnr)
-  local preds = self.info.patterns[pattern]
-  if not preds then
-    return true
-  end
-  local regexes = self.regexes[pattern]
-  for i, pred in pairs(preds) do
-    -- Here we only want to return if a predicate DOES NOT match, and
-    -- continue on the other case. This way unknown predicates will not be considered,
-    -- which allows some testing and easier user extensibility (#12173).
-    -- Also, tree-sitter strips the leading # from predicates for us.
-    if pred[1] == "eq?" then
-      local node = match[pred[2]]
-      local node_text = get_node_text(node, bufnr)
-
-      local str
-      if type(pred[3]) == "string" then
-        -- (#eq? @aa "foo")
-        str = pred[3]
-      else
-        -- (#eq? @aa @bb)
-        str = get_node_text(match[pred[3]], bufnr)
-      end
-
-      if node_text ~= str or str == nil then
-        return false
-      end
-    elseif pred[1] == "match?" then
-      if not regexes or not regexes[i] then
-        return false
-      end
-      local node = match[pred[2]]
-      local start_row, start_col, end_row, end_col = node:range()
-      if start_row ~= end_row then
-        return false
-      end
-      if not regexes[i]:match_line(bufnr, start_row, start_col, end_col) then
-        return false
-      end
-    end
-  end
-  return true
-end
-
-function Query:iter_captures(node, bufnr, start, stop)
-  if bufnr == 0 then
-    bufnr = vim.api.nvim_get_current_buf()
-  end
-  local raw_iter = node:_rawquery(self.query,true,start,stop)
-  local function iter()
-    local capture, captured_node, match = raw_iter()
-    if match ~= nil then
-      local active = self:match_preds(match, match.pattern, bufnr)
-      match.active = active
-      if not active then
-        return iter() -- tail call: try next match
-      end
-    end
-    return capture, captured_node
-  end
-  return iter
-end
-
-function Query:iter_matches(node, bufnr, start, stop)
-  if bufnr == 0 then
-    bufnr = vim.api.nvim_get_current_buf()
-  end
-  local raw_iter = node:_rawquery(self.query,false,start,stop)
-  local function iter()
-    local pattern, match = raw_iter()
-    if match ~= nil then
-      local active = self:match_preds(match, pattern, bufnr)
-      if not active then
-        return iter() -- tail call: try next match
-      end
-    end
-    return pattern, match
-  end
-  return iter
-end
-
 return M
diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
new file mode 100644
index 0000000000..b410f01092
--- /dev/null
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -0,0 +1,160 @@
+local a = vim.api
+
+-- support reload for quick experimentation
+local TSHighlighter = rawget(vim.treesitter, 'TSHighlighter') or {}
+TSHighlighter.__index = TSHighlighter
+local ts_hs_ns = a.nvim_create_namespace("treesitter_hl")
+
+-- These are conventions defined by tree-sitter, though it
+-- needs to be user extensible also.
+-- TODO(bfredl): this is very much incomplete, we will need to
+-- go through a few tree-sitter provided queries and decide
+-- on translations that makes the most sense.
+TSHighlighter.hl_map = {
+    ["error"] = "Error",
+
+-- Miscs
+    ["comment"] = "Comment",
+    ["punctuation.delimiter"] = "Delimiter",
+    ["punctuation.bracket"] = "Delimiter",
+    ["punctuation.special"] = "Delimiter",
+
+-- Constants
+    ["constant"] = "Constant",
+    ["constant.builtin"] = "Special",
+    ["constant.macro"] = "Define",
+    ["string"] = "String",
+    ["string.regex"] = "String",
+    ["string.escape"] = "SpecialChar",
+    ["character"] = "Character",
+    ["number"] = "Number",
+    ["boolean"] = "Boolean",
+    ["float"] = "Float",
+
+-- Functions
+    ["function"] = "Function",
+    ["function.special"] = "Function",
+    ["function.builtin"] = "Special",
+    ["function.macro"] = "Macro",
+    ["parameter"] = "Identifier",
+    ["method"] = "Function",
+    ["field"] = "Identifier",
+    ["property"] = "Identifier",
+    ["constructor"] = "Special",
+
+-- Keywords
+    ["conditional"] = "Conditional",
+    ["repeat"] = "Repeat",
+    ["label"] = "Label",
+    ["operator"] = "Operator",
+    ["keyword"] = "Keyword",
+    ["exception"] = "Exception",
+
+    ["type"] = "Type",
+    ["type.builtin"] = "Type",
+    ["structure"] = "Structure",
+    ["include"] = "Include",
+}
+
+function TSHighlighter.new(query, bufnr, ft)
+  local self = setmetatable({}, TSHighlighter)
+  self.parser = vim.treesitter.get_parser(
+    bufnr,
+    ft,
+    {
+      on_changedtree = function(...) self:on_changedtree(...) end,
+      on_lines = function() self.root = self.parser:parse():root() end
+    }
+  )
+
+  self.buf = self.parser.bufnr
+
+  local tree = self.parser:parse()
+  self.root = tree:root()
+  self:set_query(query)
+  self.edit_count = 0
+  self.redraw_count = 0
+  self.line_count = {}
+  a.nvim_buf_set_option(self.buf, "syntax", "")
+
+  -- Tricky: if syntax hasn't been enabled, we need to reload color scheme
+  -- but use synload.vim rather than syntax.vim to not enable
+  -- syntax FileType autocmds. Later on we should integrate with the
+  -- `:syntax` and `set syntax=...` machinery properly.
+  if vim.g.syntax_on ~= 1 then
+    vim.api.nvim_command("runtime! syntax/synload.vim")
+  end
+  return self
+end
+
+local function is_highlight_name(capture_name)
+  local firstc = string.sub(capture_name, 1, 1)
+  return firstc ~= string.lower(firstc)
+end
+
+function TSHighlighter:get_hl_from_capture(capture)
+
+  local name = self.query.captures[capture]
+
+  if is_highlight_name(name) then
+    -- From "Normal.left" only keep "Normal"
+    return vim.split(name, '.', true)[1]
+  else
+    -- Default to false to avoid recomputing
+    return TSHighlighter.hl_map[name]
+  end
+end
+
+function TSHighlighter:set_query(query)
+  if type(query) == "string" then
+    query = vim.treesitter.parse_query(self.parser.lang, query)
+  elseif query == nil then
+    query = vim.treesitter.get_query(self.parser.lang, 'highlights')
+
+    if query == nil then
+      a.err_writeln("No highlights.scm query found for " .. self.parser.lang)
+
+      if query == nil then
+        query = vim.treesitter.parse_query(self.parser.lang, "")
+      end
+    end
+  end
+
+  self.query = query
+
+  self.hl_cache = setmetatable({}, {
+    __index = function(table, capture)
+      local hl = self:get_hl_from_capture(capture)
+      rawset(table, capture, hl)
+
+      return hl
+    end
+  })
+
+  self:on_changedtree({{self.root:range()}})
+end
+
+function TSHighlighter:on_changedtree(changes)
+  -- Get a fresh root
+  self.root = self.parser.tree:root()
+
+  for _, ch in ipairs(changes or {}) do
+    -- Try to be as exact as possible
+    local changed_node = self.root:descendant_for_range(ch[1], ch[2], ch[3], ch[4])
+
+    a.nvim_buf_clear_namespace(self.buf, ts_hs_ns, ch[1], ch[3])
+
+    for capture, node in self.query:iter_captures(changed_node, self.buf, ch[1], ch[3] + 1) do
+      local start_row, start_col, end_row, end_col = node:range()
+      local hl = self.hl_cache[capture]
+      if hl then
+        a.nvim__buf_add_decoration(self.buf, ts_hs_ns, hl,
+          start_row, start_col,
+          end_row, end_col,
+          {})
+      end
+    end
+  end
+end
+
+return TSHighlighter
diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua
new file mode 100644
index 0000000000..b4817de91e
--- /dev/null
+++ b/runtime/lua/vim/treesitter/language.lua
@@ -0,0 +1,26 @@
+local a = vim.api
+
+local M = {}
+
+function M.require_language(lang, path)
+  if vim._ts_has_language(lang) then
+    return true
+  end
+  if path == nil then
+    local fname = 'parser/' .. lang .. '.*'
+    local paths = a.nvim_get_runtime_file(fname, false)
+    if #paths == 0 then
+      -- TODO(bfredl): help tag?
+      error("no parser for '"..lang.."' language")
+    end
+    path = paths[1]
+  end
+  vim._ts_add_language(path, lang)
+end
+
+function M.inspect_language(lang)
+  M.require_language(lang)
+  return vim._ts_inspect_language(lang)
+end
+
+return M
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
new file mode 100644
index 0000000000..914c266426
--- /dev/null
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -0,0 +1,133 @@
+local a = vim.api
+local language = require'vim.treesitter.language'
+
+-- query: pattern matching on trees
+-- predicate matching is implemented in lua
+local Query = {}
+Query.__index = Query
+
+local M = {}
+
+--- Parses a query.
+--
+-- @param language The language
+-- @param query A string containing the query (s-expr syntax)
+--
+-- @returns The query
+function M.parse_query(lang, query)
+  language.require_language(lang)
+  local self = setmetatable({}, Query)
+  self.query = vim._ts_parse_query(lang, vim.fn.escape(query,'\\'))
+  self.info = self.query:inspect()
+  self.captures = self.info.captures
+  return self
+end
+
+-- TODO(vigoux): support multiline nodes too
+local function get_node_text(node, bufnr)
+  local start_row, start_col, end_row, end_col = node:range()
+  if start_row ~= end_row then
+    return nil
+  end
+  local line = a.nvim_buf_get_lines(bufnr, start_row, start_row+1, true)[1]
+  return string.sub(line, start_col+1, end_col)
+end
+
+-- Predicate handler receive the following arguments
+-- (match, pattern, bufnr, regexes, index, predicate)
+local predicate_handlers = {
+  ["eq?"] = function(match, _, bufnr, predicate)
+      local node = match[predicate[2]]
+      local node_text = get_node_text(node, bufnr)
+
+      local str
+      if type(predicate[3]) == "string" then
+        -- (#eq? @aa "foo")
+        str = predicate[3]
+      else
+        -- (#eq? @aa @bb)
+        str = get_node_text(match[predicate[3]], bufnr)
+      end
+
+      if node_text ~= str or str == nil then
+        return false
+      end
+
+      return true
+  end,
+  ["match?"] = function(match, _, bufnr, predicate)
+      local node = match[predicate[2]]
+      local regex = predicate[3]
+      local start_row, _, end_row, _ = node:range()
+      if start_row ~= end_row then
+        return false
+      end
+
+      return string.find(get_node_text(node, bufnr), regex)
+  end,
+}
+
+function M.add_predicate(name, handler)
+  if predicate_handlers[name] then
+    a.nvim_err_writeln("It is recomended to not overwrite predicates.")
+  end
+
+  predicate_handlers[name] = handler
+end
+
+function Query:match_preds(match, pattern, bufnr)
+  local preds = self.info.patterns[pattern]
+  if not preds then
+    return true
+  end
+  for _, pred in pairs(preds) do
+    -- Here we only want to return if a predicate DOES NOT match, and
+    -- continue on the other case. This way unknown predicates will not be considered,
+    -- which allows some testing and easier user extensibility (#12173).
+    -- Also, tree-sitter strips the leading # from predicates for us.
+    if predicate_handlers[pred[1]] and
+      not predicate_handlers[pred[1]](match, pattern, bufnr, pred) then
+      return false
+    end
+  end
+  return true
+end
+
+function Query:iter_captures(node, bufnr, start, stop)
+  if bufnr == 0 then
+    bufnr = vim.api.nvim_get_current_buf()
+  end
+  local raw_iter = node:_rawquery(self.query, true, start, stop)
+  local function iter()
+    local capture, captured_node, match = raw_iter()
+    if match ~= nil then
+      local active = self:match_preds(match, match.pattern, bufnr)
+      match.active = active
+      if not active then
+        return iter() -- tail call: try next match
+      end
+    end
+    return capture, captured_node
+  end
+  return iter
+end
+
+function Query:iter_matches(node, bufnr, start, stop)
+  if bufnr == 0 then
+    bufnr = vim.api.nvim_get_current_buf()
+  end
+  local raw_iter = node:_rawquery(self.query, false, start, stop)
+  local function iter()
+    local pattern, match = raw_iter()
+    if match ~= nil then
+      local active = self:match_preds(match, pattern, bufnr)
+      if not active then
+        return iter() -- tail call: try next match
+      end
+    end
+    return pattern, match
+  end
+  return iter
+end
+
+return M
diff --git a/runtime/lua/vim/tshighlighter.lua b/runtime/lua/vim/tshighlighter.lua
deleted file mode 100644
index 6465751ae8..0000000000
--- a/runtime/lua/vim/tshighlighter.lua
+++ /dev/null
@@ -1,116 +0,0 @@
-local a = vim.api
-
--- support reload for quick experimentation
-local TSHighlighter = rawget(vim.treesitter, 'TSHighlighter') or {}
-TSHighlighter.__index = TSHighlighter
-local ts_hs_ns = a.nvim_create_namespace("treesitter_hl")
-
--- These are conventions defined by tree-sitter, though it
--- needs to be user extensible also.
--- TODO(bfredl): this is very much incomplete, we will need to
--- go through a few tree-sitter provided queries and decide
--- on translations that makes the most sense.
-TSHighlighter.hl_map = {
-    keyword="Keyword",
-    string="String",
-    type="Type",
-    comment="Comment",
-    constant="Constant",
-    operator="Operator",
-    number="Number",
-    label="Label",
-    ["function"]="Function",
-    ["function.special"]="Function",
-}
-
-function TSHighlighter.new(query, bufnr, ft)
-  local self = setmetatable({}, TSHighlighter)
-  self.parser = vim.treesitter.get_parser(
-    bufnr,
-    ft,
-    {
-      on_changedtree = function(...) self:on_changedtree(...) end,
-      on_lines = function() self.root = self.parser:parse():root() end
-    }
-  )
-
-  self.buf = self.parser.bufnr
-
-  local tree = self.parser:parse()
-  self.root = tree:root()
-  self:set_query(query)
-  self.edit_count = 0
-  self.redraw_count = 0
-  self.line_count = {}
-  a.nvim_buf_set_option(self.buf, "syntax", "")
-
-  -- Tricky: if syntax hasn't been enabled, we need to reload color scheme
-  -- but use synload.vim rather than syntax.vim to not enable
-  -- syntax FileType autocmds. Later on we should integrate with the
-  -- `:syntax` and `set syntax=...` machinery properly.
-  if vim.g.syntax_on ~= 1 then
-    vim.api.nvim_command("runtime! syntax/synload.vim")
-  end
-  return self
-end
-
-local function is_highlight_name(capture_name)
-  local firstc = string.sub(capture_name, 1, 1)
-  return firstc ~= string.lower(firstc)
-end
-
-function TSHighlighter:get_hl_from_capture(capture)
-
-  local name = self.query.captures[capture]
-
-  if is_highlight_name(name) then
-    -- From "Normal.left" only keep "Normal"
-    return vim.split(name, '.', true)[1]
-  else
-    -- Default to false to avoid recomputing
-    return TSHighlighter.hl_map[name]
-  end
-end
-
-function TSHighlighter:set_query(query)
-  if type(query) == "string" then
-    query = vim.treesitter.parse_query(self.parser.lang, query)
-  end
-  self.query = query
-
-  self.hl_cache = setmetatable({}, {
-    __index = function(table, capture)
-      local hl = self:get_hl_from_capture(capture)
-      rawset(table, capture, hl)
-
-      return hl
-    end
-  })
-
-  self:on_changedtree({{self.root:range()}})
-end
-
-function TSHighlighter:on_changedtree(changes)
-  -- Get a fresh root
-  self.root = self.parser.tree:root()
-
-  for _, ch in ipairs(changes or {}) do
-    -- Try to be as exact as possible
-    local changed_node = self.root:descendant_for_range(ch[1], ch[2], ch[3], ch[4])
-
-    a.nvim_buf_clear_namespace(self.buf, ts_hs_ns, ch[1], ch[3])
-
-    for capture, node in self.query:iter_captures(changed_node, self.buf, ch[1], ch[3] + 1) do
-      local start_row, start_col, end_row, end_col = node:range()
-      local hl = self.hl_cache[capture]
-      if hl then
-        a.nvim__buf_add_decoration(self.buf, ts_hs_ns, hl,
-          start_row, start_col,
-          end_row, end_col,
-          {})
-      end
-    end
-  end
-end
-
-return TSHighlighter
-- 
cgit 


From 18c0e775286aa6621d9190db8cf02e65a3f3d61f Mon Sep 17 00:00:00 2001
From: Thomas Vigouroux <tomvig38@gmail.com>
Date: Thu, 16 Jul 2020 17:17:42 +0200
Subject: treesitter(docs): update and refresh docs

---
 runtime/lua/vim/treesitter.lua             |  9 +++++++++
 runtime/lua/vim/treesitter/highlighter.lua | 10 ++--------
 runtime/lua/vim/treesitter/language.lua    | 13 ++++++++++++-
 runtime/lua/vim/treesitter/query.lua       | 23 +++++++++++++++++++++++
 4 files changed, 46 insertions(+), 9 deletions(-)

(limited to 'runtime/lua/vim')

diff --git a/runtime/lua/vim/treesitter.lua b/runtime/lua/vim/treesitter.lua
index f43c8a872d..550dee1e3f 100644
--- a/runtime/lua/vim/treesitter.lua
+++ b/runtime/lua/vim/treesitter.lua
@@ -10,6 +10,12 @@ local parsers = {}
 local Parser = {}
 Parser.__index = Parser
 
+--- Parses the buffer if needed and returns a tree.
+--
+-- Calling this will call the on_changedtree callbacks if the tree has changed.
+--
+-- @returns An up to date tree
+-- @returns If the tree changed with this call, the changed ranges
 function Parser:parse()
   if self.valid then
     return self.tree
@@ -40,6 +46,9 @@ function Parser:_on_lines(bufnr, changed_tick, start_row, old_stop_row, stop_row
   end
 end
 
+--- Sets the included ranges for the current parser
+--
+-- @param ranges A table of nodes that will be used as the ranges the parser should include.
 function Parser:set_included_ranges(ranges)
   self._parser:set_included_ranges(ranges)
   -- The buffer will need to be parsed again later
diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index b410f01092..681d2c6324 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -7,9 +7,6 @@ local ts_hs_ns = a.nvim_create_namespace("treesitter_hl")
 
 -- These are conventions defined by tree-sitter, though it
 -- needs to be user extensible also.
--- TODO(bfredl): this is very much incomplete, we will need to
--- go through a few tree-sitter provided queries and decide
--- on translations that makes the most sense.
 TSHighlighter.hl_map = {
     ["error"] = "Error",
 
@@ -112,11 +109,8 @@ function TSHighlighter:set_query(query)
     query = vim.treesitter.get_query(self.parser.lang, 'highlights')
 
     if query == nil then
-      a.err_writeln("No highlights.scm query found for " .. self.parser.lang)
-
-      if query == nil then
-        query = vim.treesitter.parse_query(self.parser.lang, "")
-      end
+      a.nvim_err_writeln("No highlights.scm query found for " .. self.parser.lang)
+      query = vim.treesitter.parse_query(self.parser.lang, "")
     end
   end
 
diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua
index b4817de91e..a7e36a0b89 100644
--- a/runtime/lua/vim/treesitter/language.lua
+++ b/runtime/lua/vim/treesitter/language.lua
@@ -2,6 +2,12 @@ local a = vim.api
 
 local M = {}
 
+--- Asserts that the provided language is installed, and optionnaly provide a path for the parser
+--
+-- Parsers are searched in the `parser` runtime directory.
+--
+-- @param lang The language the parser should parse
+-- @param path Optionnal path the parser is located at
 function M.require_language(lang, path)
   if vim._ts_has_language(lang) then
     return true
@@ -11,13 +17,18 @@ function M.require_language(lang, path)
     local paths = a.nvim_get_runtime_file(fname, false)
     if #paths == 0 then
       -- TODO(bfredl): help tag?
-      error("no parser for '"..lang.."' language")
+      error("no parser for '"..lang.."' language, see :help treesitter-parsers")
     end
     path = paths[1]
   end
   vim._ts_add_language(path, lang)
 end
 
+--- Inspects the provided language.
+--
+-- Inspecting provides some useful informations on the language like node names, ...
+--
+-- @param lang The language.
 function M.inspect_language(lang)
   M.require_language(lang)
   return vim._ts_inspect_language(lang)
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 914c266426..d16e1c3662 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -67,6 +67,11 @@ local predicate_handlers = {
   end,
 }
 
+--- Adds a new predicates to be used in queries
+--
+-- @param name the name of the predicate, without leading #
+-- @param handler the handler function to be used
+--    signature will be (match, pattern, bufnr, predicate)
 function M.add_predicate(name, handler)
   if predicate_handlers[name] then
     a.nvim_err_writeln("It is recomended to not overwrite predicates.")
@@ -93,6 +98,15 @@ function Query:match_preds(match, pattern, bufnr)
   return true
 end
 
+--- Iterates of the captures of self on a given range.
+--
+-- @param node The node under witch the search will occur
+-- @param buffer The source buffer to search
+-- @param start The starting line of the search
+-- @param stop The stoping line of the search (end-exclusive)
+--
+-- @returns The matching capture id
+-- @returns The captured node
 function Query:iter_captures(node, bufnr, start, stop)
   if bufnr == 0 then
     bufnr = vim.api.nvim_get_current_buf()
@@ -112,6 +126,15 @@ function Query:iter_captures(node, bufnr, start, stop)
   return iter
 end
 
+--- Iterates of the matches of self on a given range.
+--
+-- @param node The node under witch the search will occur
+-- @param buffer The source buffer to search
+-- @param start The starting line of the search
+-- @param stop The stoping line of the search (end-exclusive)
+--
+-- @returns The matching pattern id
+-- @returns The matching match
 function Query:iter_matches(node, bufnr, start, stop)
   if bufnr == 0 then
     bufnr = vim.api.nvim_get_current_buf()
-- 
cgit 


From 58e37d7df8ab3afc4d77e6ff1248d26a2559399e Mon Sep 17 00:00:00 2001
From: Thomas Vigouroux <tomvig38@gmail.com>
Date: Mon, 10 Aug 2020 09:41:57 +0200
Subject: treesitter: add contains? predicate

---
 runtime/lua/vim/treesitter/query.lua | 13 +++++++++++++
 1 file changed, 13 insertions(+)

(limited to 'runtime/lua/vim')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index d16e1c3662..b30bf5fb6b 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -65,6 +65,19 @@ local predicate_handlers = {
 
       return string.find(get_node_text(node, bufnr), regex)
   end,
+
+  ["contains?"] = function(match, _, bufnr, predicate)
+    local node = match[predicate[2]]
+    local node_text = get_node_text(node, bufnr)
+
+    for i=3,#predicate do
+      if string.find(node_text, predicate[i], 1, true) then
+        return true
+      end
+    end
+
+    return false
+  end
 }
 
 --- Adds a new predicates to be used in queries
-- 
cgit 


From d7b12e58dfc7303dbc06381a9bedd5c3539d5413 Mon Sep 17 00:00:00 2001
From: Thomas Vigouroux <tomvig38@gmail.com>
Date: Mon, 10 Aug 2020 18:25:52 +0200
Subject: treesitter: add and test vim-match? predicate

---
 runtime/lua/vim/treesitter/query.lua | 46 +++++++++++++++++++++++++++++++-----
 1 file changed, 40 insertions(+), 6 deletions(-)

(limited to 'runtime/lua/vim')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index b30bf5fb6b..b43c28b0ab 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -24,7 +24,11 @@ function M.parse_query(lang, query)
 end
 
 -- TODO(vigoux): support multiline nodes too
-local function get_node_text(node, bufnr)
+
+--- Gets the text corresponding to a given node
+-- @param node the node
+-- @param bufnr the buffer from which the node in extracted.
+function M.get_node_text(node, bufnr)
   local start_row, start_col, end_row, end_col = node:range()
   if start_row ~= end_row then
     return nil
@@ -34,11 +38,11 @@ local function get_node_text(node, bufnr)
 end
 
 -- Predicate handler receive the following arguments
--- (match, pattern, bufnr, regexes, index, predicate)
+-- (match, pattern, bufnr, predicate)
 local predicate_handlers = {
   ["eq?"] = function(match, _, bufnr, predicate)
       local node = match[predicate[2]]
-      local node_text = get_node_text(node, bufnr)
+      local node_text = M.get_node_text(node, bufnr)
 
       local str
       if type(predicate[3]) == "string" then
@@ -46,7 +50,7 @@ local predicate_handlers = {
         str = predicate[3]
       else
         -- (#eq? @aa @bb)
-        str = get_node_text(match[predicate[3]], bufnr)
+        str = M.get_node_text(match[predicate[3]], bufnr)
       end
 
       if node_text ~= str or str == nil then
@@ -63,12 +67,42 @@ local predicate_handlers = {
         return false
       end
 
-      return string.find(get_node_text(node, bufnr), regex)
+      return string.find(M.get_node_text(node, bufnr), regex)
   end,
 
+  ["vim-match?"] = (function()
+
+    local magic_prefixes = {['\\v']=true, ['\\m']=true, ['\\M']=true, ['\\V']=true}
+    local function check_magic(str)
+      if string.len(str) < 2 or magic_prefixes[string.sub(str,1,2)] then
+        return str
+      end
+      return '\\v'..str
+    end
+
+    local compiled_vim_regexes = setmetatable({}, {
+      __index = function(t, pattern)
+        local res = vim.regex(check_magic(pattern))
+        rawset(t, pattern, res)
+        return res
+      end
+    })
+
+    return function(match, _, bufnr, pred)
+      local node = match[pred[2]]
+      local start_row, start_col, end_row, end_col = node:range()
+      if start_row ~= end_row then
+        return false
+      end
+
+      local regex = compiled_vim_regexes[pred[3]]
+      return regex:match_line(bufnr, start_row, start_col, end_col)
+    end
+  end)(),
+
   ["contains?"] = function(match, _, bufnr, predicate)
     local node = match[predicate[2]]
-    local node_text = get_node_text(node, bufnr)
+    local node_text = M.get_node_text(node, bufnr)
 
     for i=3,#predicate do
       if string.find(node_text, predicate[i], 1, true) then
-- 
cgit 


From 9564803d1ad2c618176a917f974fb47ce1c598a9 Mon Sep 17 00:00:00 2001
From: Thomas Vigouroux <tomvig38@gmail.com>
Date: Tue, 11 Aug 2020 23:21:15 +0200
Subject: treesitter: add predicate negation

---
 runtime/lua/vim/treesitter/query.lua | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

(limited to 'runtime/lua/vim')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index b43c28b0ab..0665a837e9 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -59,6 +59,7 @@ local predicate_handlers = {
 
       return true
   end,
+
   ["match?"] = function(match, _, bufnr, predicate)
       local node = match[predicate[2]]
       local regex = predicate[3]
@@ -71,7 +72,6 @@ local predicate_handlers = {
   end,
 
   ["vim-match?"] = (function()
-
     local magic_prefixes = {['\\v']=true, ['\\m']=true, ['\\M']=true, ['\\V']=true}
     local function check_magic(str)
       if string.len(str) < 2 or magic_prefixes[string.sub(str,1,2)] then
@@ -121,7 +121,7 @@ local predicate_handlers = {
 --    signature will be (match, pattern, bufnr, predicate)
 function M.add_predicate(name, handler)
   if predicate_handlers[name] then
-    a.nvim_err_writeln("It is recomended to not overwrite predicates.")
+    a.nvim_err_writeln(string.format("Overriding %s", name))
   end
 
   predicate_handlers[name] = handler
@@ -137,7 +137,14 @@ function Query:match_preds(match, pattern, bufnr)
     -- continue on the other case. This way unknown predicates will not be considered,
     -- which allows some testing and easier user extensibility (#12173).
     -- Also, tree-sitter strips the leading # from predicates for us.
-    if predicate_handlers[pred[1]] and
+    if string.sub(pred[1], 1, 4) == "not-" then
+      local pred_name = string.sub(pred[1], 5)
+      if predicate_handlers[pred_name] and
+        predicate_handlers[pred_name](match, pattern, bufnr, pred) then
+        return false
+      end
+
+    elseif predicate_handlers[pred[1]] and
       not predicate_handlers[pred[1]](match, pattern, bufnr, pred) then
       return false
     end
-- 
cgit 


From 6a8dcfab4b2bada9c68379ee17235974fa8ad411 Mon Sep 17 00:00:00 2001
From: Thomas Vigouroux <tomvig38@gmail.com>
Date: Thu, 13 Aug 2020 20:40:40 +0200
Subject: treesitter: allow to force predicate addition

---
 runtime/lua/vim/treesitter/query.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'runtime/lua/vim')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 0665a837e9..17f61b24f1 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -119,8 +119,8 @@ local predicate_handlers = {
 -- @param name the name of the predicate, without leading #
 -- @param handler the handler function to be used
 --    signature will be (match, pattern, bufnr, predicate)
-function M.add_predicate(name, handler)
-  if predicate_handlers[name] then
+function M.add_predicate(name, handler, force)
+  if predicate_handlers[name] and not force then
     a.nvim_err_writeln(string.format("Overriding %s", name))
   end
 
-- 
cgit