feat(treesitter)!: incremental injection parsing

Problem:

Treesitter highlighting is slow for large files with lots of injections.

Solution:

Only parse injections we are going to render during a redraw cycle.

---

- `LanguageTree:parse()` will no longer parse injections by default and
  now requires an explicit range argument to be passed.

- `TSHighlighter` now parses injections incrementally during on_win
  callbacks for the line range being rendered.

- Plugins which require certain injections to be parsed must run
  `parser:parse({ start_row, end_row })` before using the tree.

2 files changed, 80 insertions, 19 deletions

diff --git a/test/functional/treesitter/parser_spec.lua b/test/functional/treesitter/parser_spec.lua
index 834998bae7..cc833a67cc 100644
--- a/test/functional/treesitter/parser_spec.lua
+++ b/test/functional/treesitter/parser_spec.lua
@@ -1,6 +1,7 @@
 local helpers = require('test.functional.helpers')(after_each)
 
 local clear = helpers.clear
+local dedent = helpers.dedent
 local eq = helpers.eq
 local insert = helpers.insert
 local exec_lua = helpers.exec_lua
@@ -502,22 +503,12 @@ end]]
     local root = parser:parse()[1]:root()
     parser:set_included_regions({{root:child(0)}})
     parser:invalidate()
-    return { parser:parse()[1]:root():range() }
+    return { parser:parse(true)[1]:root():range() }
     ]]
 
     eq({0, 0, 18, 1}, res2)
 
-    local range = exec_lua [[
-      local res = {}
-      for _, region in ipairs(parser:included_regions()) do
-        for _, node in ipairs(region) do
-          table.insert(res, {node:range()})
-        end
-      end
-      return res
-    ]]
-
-    eq(range, { { 0, 0, 18, 1 } })
+    eq({ { { 0, 0, 0, 18, 1, 512 } } }, exec_lua [[ return parser:included_regions() ]])
 
     local range_tbl = exec_lua [[
       parser:set_included_regions { { { 0, 0, 17, 1 } } }
@@ -542,7 +533,7 @@ end]]
 
     parser:set_included_regions({nodes})
 
-    local root = parser:parse()[1]:root()
+    local root = parser:parse(true)[1]:root()
 
     local res = {}
     for i=0,(root:named_child_count() - 1) do
@@ -638,9 +629,11 @@ int x = INT_MAX;
     describe("when parsing regions independently", function()
       it("should inject a language", function()
         exec_lua([[
+        vim.g.__ts_debug = 1
         parser = vim.treesitter.get_parser(0, "c", {
           injections = {
             c = "(preproc_def (preproc_arg) @c) (preproc_function_def value: (preproc_arg) @c)"}})
+        parser:parse(true)
         ]])
 
         eq("table", exec_lua("return type(parser:children().c)"))
@@ -673,6 +666,7 @@ int x = INT_MAX;
         parser = vim.treesitter.get_parser(0, "c", {
           injections = {
             c = "(preproc_def (preproc_arg) @c @combined) (preproc_function_def value: (preproc_arg) @c @combined)"}})
+        parser:parse(true)
         ]])
 
         eq("table", exec_lua("return type(parser:children().c)"))
@@ -713,6 +707,7 @@ int x = INT_MAX;
           injections = {
             c = "(preproc_def ((preproc_arg) @_c (#inject-clang! @_c)))" ..
                 "(preproc_function_def value: ((preproc_arg) @_a (#inject-clang! @_a)))"}})
+        parser:parse(true)
         ]=])
 
         eq("table", exec_lua("return type(parser:children().c)"))
@@ -760,6 +755,7 @@ int x = INT_MAX;
         parser = vim.treesitter.get_parser(0, "c", {
           injections = {
             c = "(preproc_def ((preproc_arg) @c (#offset! @c 0 2 0 -1))) (preproc_function_def value: (preproc_arg) @c)"}})
+        parser:parse(true)
         ]])
 
         eq("table", exec_lua("return type(parser:children().c)"))
@@ -800,6 +796,7 @@ int x = INT_MAX;
       local result = exec_lua([[
       parser = vim.treesitter.get_parser(0, "c", {
         injections = { c = "(preproc_def (preproc_arg) @c)"}})
+      parser:parse(true)
 
       local sub_tree = parser:language_for_range({1, 18, 1, 19})
 
@@ -951,7 +948,7 @@ int x = INT_MAX;
 
     local r = exec_lua([[
       local parser = vim.treesitter.get_string_parser(..., 'lua')
-      parser:parse()
+      parser:parse(true)
       local ranges = {}
       parser:for_each_tree(function(tstree, tree)
         ranges[tree:lang()] = { tstree:root():range(true) }
@@ -997,7 +994,7 @@ int x = INT_MAX;
           vimdoc = "((codeblock (language) @injection.language (code) @injection.content))"
         }
       })
-      parser1:parse()
+      parser1:parse(true)
     ]]
 
     eq(0, exec_lua("return #vim.tbl_keys(parser1:children())"))
@@ -1008,7 +1005,7 @@ int x = INT_MAX;
           vimdoc = "((codeblock (language) @injection.language (code) @injection.content) (#set! injection.include-children))"
         }
       })
-      parser2:parse()
+      parser2:parse(true)
     ]]
 
     eq(1, exec_lua("return #vim.tbl_keys(parser2:children())"))
@@ -1016,4 +1013,66 @@ int x = INT_MAX;
 
   end)
 
+  it("parsers injections incrementally", function()
+    insert(dedent[[
+      >lua
+        local a = {}
+      <
+
+      >lua
+        local b = {}
+      <
+
+      >lua
+        local c = {}
+      <
+
+      >lua
+        local d = {}
+      <
+
+      >lua
+        local e = {}
+      <
+
+      >lua
+        local f = {}
+      <
+
+      >lua
+        local g = {}
+      <
+    ]])
+
+    exec_lua [[
+      parser = require('vim.treesitter.languagetree').new(0, "vimdoc", {
+        injections = {
+          vimdoc = "((codeblock (language) @injection.language (code) @injection.content) (#set! injection.include-children))"
+        }
+      })
+    ]]
+
+    --- Do not parse injections by default
+    eq(0, exec_lua [[
+      parser:parse()
+      return #vim.tbl_keys(parser:children())
+    ]])
+
+    --- Only parse injections between lines 0, 2
+    eq(1, exec_lua [[
+      parser:parse({0, 2})
+      return #parser:children().lua:trees()
+    ]])
+
+    eq(2, exec_lua [[
+      parser:parse({2, 6})
+      return #parser:children().lua:trees()
+    ]])
+
+    eq(7, exec_lua [[
+      parser:parse(true)
+      return #parser:children().lua:trees()
+    ]])
+  end)
+
 end)
diff --git a/test/helpers.lua b/test/helpers.lua
index 8f06311a3c..f0e8576a3a 100644
--- a/test/helpers.lua
+++ b/test/helpers.lua
@@ -570,21 +570,23 @@ function module.concat_tables(...)
 end
 
 --- @param str string
---- @param leave_indent? boolean
+--- @param leave_indent? integer
 --- @return string
 function module.dedent(str, leave_indent)
   -- find minimum common indent across lines
-  local indent = nil
+  local indent --- @type string?
   for line in str:gmatch('[^\n]+') do
     local line_indent = line:match('^%s+') or ''
     if indent == nil or #line_indent < #indent then
       indent = line_indent
     end
   end
-  if indent == nil or #indent == 0 then
+
+  if not indent or #indent == 0 then
     -- no minimum common indent
     return str
   end
+
   local left_indent = (' '):rep(leave_indent or 0)
   -- create a pattern for the indent
   indent = indent:gsub('%s', '[ \t]')