feat(treesitter): add language tree

Implement the LanguageTree structure to enable language injection.

This is done be removing the old Parser metatable and replacing by the
new structure, with the same API (almost).

Some noticeable differences :
  - `parser:parse()` now returns a table of trees
  - There is no incremental parsing for child (injected) languages

Co-authored-by: Thomas Vigouroux <tomvig38@gmail.com>

5 files changed, 1013 insertions, 485 deletions

diff --git a/runtime/doc/treesitter.txt b/runtime/doc/treesitter.txt
index 58cd535e98..b6a238f158 100644
--- a/runtime/doc/treesitter.txt
+++ b/runtime/doc/treesitter.txt
@@ -59,15 +59,16 @@ shouldn't be done directly in the change callback anyway as they will be very
 frequent. Rather a plugin that does any kind of analysis on a tree should use
 a timer to throttle too frequent updates.
 
-tsparser:set_included_ranges({ranges})			*tsparser:set_included_ranges()*
-	Changes the ranges the parser should consider. This is used for
-	language injection.  {ranges} should be of the form (all zero-based): >
+tsparser:set_included_regions({region_list})			*tsparser:set_included_regions()*
+	Changes the regions the parser should consider. This is used for
+	language injection.  {region_list} should be of the form (all zero-based): >
 	{
-		{start_node, end_node},
+		{node1, node2},
 		...
 	}
 <
-	NOTE: `start_node` and `end_node` are both inclusive.
+	`node1` and `node2` are both considered part of the same region and
+	will be parsed together with the parser in the same context.
 
 Tree methods						*lua-treesitter-tree*
 
@@ -253,7 +254,7 @@ Here is a list of built-in predicates :
 	`lua-match?`					*ts-predicate-lua-match?*
 		This will match the same way than |match?| but using lua
 		regexes.
-	
+
 	`contains?`					*ts-predicate-contains?*
 		Will check if any of the following arguments appears in the
 		text corresponding to the node : >
diff --git a/runtime/lua/vim/treesitter.lua b/runtime/lua/vim/treesitter.lua
index 19ef148afc..6886f0c178 100644
--- a/runtime/lua/vim/treesitter.lua
+++ b/runtime/lua/vim/treesitter.lua
@@ -1,98 +1,13 @@
 local a = vim.api
 local query = require'vim.treesitter.query'
 local language = require'vim.treesitter.language'
+local LanguageTree = require'vim.treesitter.languagetree'
 
 -- TODO(bfredl): currently we retain parsers for the lifetime of the buffer.
 -- Consider use weak references to release parser if all plugins are done with
 -- it.
 local parsers = {}
 
-local Parser = {}
-Parser.__index = Parser
-
---- Parses the buffer if needed and returns a tree.
---
--- Calling this will call the on_changedtree callbacks if the tree has changed.
---
--- @returns An up to date tree
--- @returns If the tree changed with this call, the changed ranges
-function Parser:parse()
-  if self.valid then
-    return self._tree_immutable
-  end
-  local changes
-
-  self._tree, changes = self._parser:parse(self._tree, self:input_source())
-
-  self._tree_immutable = self._tree:copy()
-
-  self.valid = true
-
-  if not vim.tbl_isempty(changes) then
-    for _, cb in ipairs(self.changedtree_cbs) do
-      cb(changes)
-    end
-  end
-
-  return self._tree_immutable, changes
-end
-
-function Parser:input_source()
-  return self.bufnr or self.str
-end
-
-function Parser:_on_bytes(bufnr, changed_tick,
-                          start_row, start_col, start_byte,
-                          old_row, old_col, old_byte,
-                          new_row, new_col, new_byte)
-  local old_end_col = old_col + ((old_row == 0) and start_col or 0)
-  local new_end_col = new_col + ((new_row == 0) and start_col or 0)
-  self._tree:edit(start_byte,start_byte+old_byte,start_byte+new_byte,
-                    start_row, start_col,
-                    start_row+old_row, old_end_col,
-                    start_row+new_row, new_end_col)
-  self.valid = false
-
-  for _, cb in ipairs(self.bytes_cbs) do
-    cb(bufnr, changed_tick,
-      start_row, start_col, start_byte,
-      old_row, old_col, old_byte,
-      new_row, new_col, new_byte)
-  end
-end
-
---- Registers callbacks for the parser
--- @param cbs An `nvim_buf_attach`-like table argument with the following keys :
---  `on_bytes` : see `nvim_buf_attach`, but this will be called _after_ the parsers callback.
---  `on_changedtree` : a callback that will be called everytime the tree has syntactical changes.
---      it will only be passed one argument, that is a table of the ranges (as node ranges) that
---      changed.
-function Parser:register_cbs(cbs)
-  if not cbs then return end
-
-  if cbs.on_changedtree then
-    table.insert(self.changedtree_cbs, cbs.on_changedtree)
-  end
-
-  if cbs.on_bytes then
-    table.insert(self.bytes_cbs, cbs.on_bytes)
-  end
-end
-
---- Sets the included ranges for the current parser
---
--- @param ranges A table of nodes that will be used as the ranges the parser should include.
-function Parser:set_included_ranges(ranges)
-  self._parser:set_included_ranges(ranges)
-  -- The buffer will need to be parsed again later
-  self.valid = false
-end
-
---- Gets the included ranges for the parsers
-function Parser:included_ranges()
-  return self._parser:included_ranges()
-end
-
 local M = vim.tbl_extend("error", query, language)
 
 setmetatable(M, {
@@ -113,9 +28,9 @@ setmetatable(M, {
 -- It is not recommended to use this, use vim.treesitter.get_parser() instead.
 --
 -- @param bufnr The buffer the parser will be tied to
--- @param lang The language of the parser.
--- @param id The id the parser will have
-function M._create_parser(bufnr, lang, id)
+-- @param lang The language of the parser
+-- @param opts Options to pass to the language tree
+function M._create_parser(bufnr, lang, opts)
   language.require_language(lang)
   if bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
@@ -123,25 +38,22 @@ function M._create_parser(bufnr, lang, id)
 
   vim.fn.bufload(bufnr)
 
-  local self = setmetatable({bufnr=bufnr, lang=lang, valid=false}, Parser)
-  self._parser = vim._create_ts_parser(lang)
-  self.changedtree_cbs = {}
-  self.bytes_cbs = {}
-  self:parse()
-    -- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is
-    -- using it.
+  local self = LanguageTree.new(bufnr, lang, opts)
+
   local function bytes_cb(_, ...)
-    return self:_on_bytes(...)
+    self:_on_bytes(...)
   end
-  local detach_cb = nil
-  if id ~= nil then
-    detach_cb = function()
-      if parsers[id] == self then
-        parsers[id] = nil
-      end
+
+  local function detach_cb()
+    if parsers[bufnr] == self then
+      parsers[bufnr] = nil
     end
   end
+
   a.nvim_buf_attach(self.bufnr, false, {on_bytes=bytes_cb, on_detach=detach_cb})
+
+  self:parse()
+
   return self
 end
 
@@ -152,39 +64,36 @@ end
 --
 -- @param bufnr The buffer the parser should be tied to
 -- @param ft The filetype of this parser
--- @param buf_attach_cbs See Parser:register_cbs
+-- @param opts Options object to pass to the parser
 --
 -- @returns The parser
-function M.get_parser(bufnr, lang, buf_attach_cbs)
+function M.get_parser(bufnr, lang, opts)
+  opts = opts or {}
+
   if bufnr == nil or bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
   if lang == nil then
     lang = a.nvim_buf_get_option(bufnr, "filetype")
   end
-  local id = tostring(bufnr)..'_'..lang
 
-  if parsers[id] == nil then
-    parsers[id] = M._create_parser(bufnr, lang, id)
+  if parsers[bufnr] == nil then
+    parsers[bufnr] = M._create_parser(bufnr, lang, opts)
   end
 
-  parsers[id]:register_cbs(buf_attach_cbs)
+  parsers[bufnr]:register_cbs(opts.buf_attach_cbs)
 
-  return parsers[id]
+  return parsers[bufnr]
 end
 
-function M.get_string_parser(str, lang)
+function M.get_string_parser(str, lang, opts)
   vim.validate {
     str = { str, 'string' },
     lang = { lang, 'string' }
   }
   language.require_language(lang)
 
-  local self = setmetatable({str=str, lang=lang, valid=false}, Parser)
-  self._parser = vim._create_ts_parser(lang)
-  self:parse()
-
-  return self
+  return LanguageTree.new(str, lang, opts)
 end
 
 return M
diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index 6714bb6354..60db7f24cf 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -1,4 +1,5 @@
 local a = vim.api
+local query = require"vim.treesitter.query"
 
 -- support reload for quick experimentation
 local TSHighlighter = rawget(vim.treesitter, 'TSHighlighter') or {}
@@ -6,6 +7,9 @@ TSHighlighter.__index = TSHighlighter
 
 TSHighlighter.active = TSHighlighter.active or {}
 
+local TSHighlighterQuery = {}
+TSHighlighterQuery.__index = TSHighlighterQuery
+
 local ns = a.nvim_create_namespace("treesitter/highlighter")
 
 -- These are conventions defined by nvim-treesitter, though it
@@ -56,27 +60,83 @@ TSHighlighter.hl_map = {
     ["include"] = "Include",
 }
 
-function TSHighlighter.new(parser, query)
+local function is_highlight_name(capture_name)
+  local firstc = string.sub(capture_name, 1, 1)
+  return firstc ~= string.lower(firstc)
+end
+
+function TSHighlighterQuery.new(lang, query_string)
+  local self = setmetatable({}, { __index = TSHighlighterQuery })
+
+  self.hl_cache = setmetatable({}, {
+    __index = function(table, capture)
+      local hl = self:get_hl_from_capture(capture)
+      rawset(table, capture, hl)
+
+      return hl
+    end
+  })
+
+  if query_string then
+    self._query = query.parse_query(lang, query_string)
+  else
+    self._query = query.get_query(lang, "highlights")
+  end
+
+  return self
+end
+
+function TSHighlighterQuery:query()
+  return self._query
+end
+
+function TSHighlighterQuery:get_hl_from_capture(capture)
+  local name = self._query.captures[capture]
+
+  if is_highlight_name(name) then
+    -- From "Normal.left" only keep "Normal"
+    return vim.split(name, '.', true)[1]
+  else
+    -- Default to false to avoid recomputing
+    local hl = TSHighlighter.hl_map[name]
+    return hl and a.nvim_get_hl_id_by_name(hl) or 0
+  end
+end
+
+function TSHighlighter.new(tree, opts)
   local self = setmetatable({}, TSHighlighter)
 
-  self.parser = parser
-  parser:register_cbs {
-    on_changedtree = function(...) self:on_changedtree(...) end
+  if type(tree:source()) ~= "number" then
+    error("TSHighlighter can not be used with a string parser source.")
+  end
+
+  opts = opts or {}
+  self.tree = tree
+  tree:register_cbs {
+    on_changedtree = function(...) self:on_changedtree(...) end,
+    on_bytes = function(...) self:on_bytes(...) end
   }
 
-  self:set_query(query)
+  self.bufnr = tree:source()
   self.edit_count = 0
   self.redraw_count = 0
   self.line_count = {}
-  self.root = self.parser:parse():root()
-  a.nvim_buf_set_option(self.buf, "syntax", "")
-
-  -- TODO(bfredl): can has multiple highlighters per buffer????
-  if not TSHighlighter.active[parser.bufnr] then
-    TSHighlighter.active[parser.bufnr] = {}
+  -- A map of highlight states.
+  -- This state is kept during rendering across each line update.
+  self._highlight_states = {}
+  self._queries = {}
+
+  -- Queries for a specific language can be overridden by a custom
+  -- string query... if one is not provided it will be looked up by file.
+  if opts.queries then
+    for lang, query_string in pairs(opts.queries) do
+      self._queries[lang] = TSHighlighterQuery.new(lang, query_string)
+    end
   end
 
-  TSHighlighter.active[parser.bufnr][parser.lang] = self
+  a.nvim_buf_set_option(self.bufnr, "syntax", "")
+
+  TSHighlighter.active[self.bufnr] = self
 
   -- Tricky: if syntax hasn't been enabled, we need to reload color scheme
   -- but use synload.vim rather than syntax.vim to not enable
@@ -85,119 +145,112 @@ function TSHighlighter.new(parser, query)
   if vim.g.syntax_on ~= 1 then
     vim.api.nvim_command("runtime! syntax/synload.vim")
   end
+
+  self.tree:parse()
+
   return self
 end
 
-local function is_highlight_name(capture_name)
-  local firstc = string.sub(capture_name, 1, 1)
-  return firstc ~= string.lower(firstc)
+function TSHighlighter:destroy()
+  if TSHighlighter.active[self.bufnr] then
+    TSHighlighter.active[self.bufnr] = nil
+  end
 end
 
-function TSHighlighter:get_hl_from_capture(capture)
+function TSHighlighter:get_highlight_state(tstree)
+  if not self._highlight_states[tstree] then
+    self._highlight_states[tstree] = {
+      next_row = 0,
+      iter = nil
+    }
+  end
 
-  local name = self.query.captures[capture]
+  return self._highlight_states[tstree]
+end
 
-  if is_highlight_name(name) then
-    -- From "Normal.left" only keep "Normal"
-    return vim.split(name, '.', true)[1]
-  else
-    -- Default to false to avoid recomputing
-    local hl = TSHighlighter.hl_map[name]
-    return hl and a.nvim_get_hl_id_by_name(hl) or 0
-  end
+function TSHighlighter:reset_highlight_state()
+  self._highlight_states = {}
+end
+
+function TSHighlighter:on_bytes(_, _, start_row, _, _, _, _, _, new_end)
+  a.nvim__buf_redraw_range(self.bufnr, start_row, start_row + new_end + 1)
 end
 
 function TSHighlighter:on_changedtree(changes)
   for _, ch in ipairs(changes or {}) do
-    a.nvim__buf_redraw_range(self.buf, ch[1], ch[3]+1)
+    a.nvim__buf_redraw_range(self.bufnr, ch[1], ch[3]+1)
   end
 end
 
-function TSHighlighter:set_query(query)
-  if type(query) == "string" then
-    query = vim.treesitter.parse_query(self.parser.lang, query)
+function TSHighlighter:get_query(lang)
+  if not self._queries[lang] then
+    self._queries[lang] = TSHighlighterQuery.new(lang)
   end
 
-  self.query = query
+  return self._queries[lang]
+end
 
-  self.hl_cache = setmetatable({}, {
-    __index = function(table, capture)
-      local hl = self:get_hl_from_capture(capture)
-      rawset(table, capture, hl)
+local function on_line_impl(self, buf, line)
+  self.tree:for_each_tree(function(tstree, tree)
+    if not tstree then return end
 
-      return hl
+    local root_node = tstree:root()
+    local root_start_row, _, root_end_row, _ = root_node:range()
+
+    -- Only worry about trees within the line range
+    if root_start_row > line or root_end_row < line then return end
+
+    local state = self:get_highlight_state(tstree)
+    local highlighter_query = self:get_query(tree:lang())
+
+    if state.iter == nil then
+      state.iter = highlighter_query:query():iter_captures(root_node, self.bufnr, line, root_end_row + 1)
     end
-  })
 
-  a.nvim__buf_redraw_range(self.parser.bufnr, 0, a.nvim_buf_line_count(self.parser.bufnr))
-end
+    while line >= state.next_row do
+      local capture, node = state.iter()
 
-local function iter_active_tshl(buf, fn)
-  for _, hl in pairs(TSHighlighter.active[buf] or {}) do
-    fn(hl)
-  end
-end
+      if capture == nil then break end
 
-local function on_line_impl(self, buf, line)
-  if self.root == nil then
-    return -- parser bought the farm already
-  end
+      local start_row, start_col, end_row, end_col = node:range()
+      local hl = highlighter_query.hl_cache[capture]
 
-  if self.iter == nil then
-    self.iter = self.query:iter_captures(self.root,buf,line,self.botline)
-  end
-  while line >= self.nextrow do
-    local capture, node = self.iter()
-    if capture == nil then
-      break
+      if hl and end_row >= line then
+        a.nvim_buf_set_extmark(buf, ns, start_row, start_col,
+                               { end_line = end_row, end_col = end_col,
+                                 hl_group = hl,
+                                 ephemeral = true
+                                })
+      end
+      if start_row > line then
+        state.next_row = start_row
+      end
     end
-    local start_row, start_col, end_row, end_col = node:range()
-    local hl = self.hl_cache[capture]
-    if hl and end_row >= line then
-      a.nvim_buf_set_extmark(buf, ns, start_row, start_col,
-                             { end_line = end_row, end_col = end_col,
-                               hl_group = hl,
-                               ephemeral = true,
-                              })
-    end
-    if start_row > line then
-      self.nextrow = start_row
-    end
-  end
+  end, true)
 end
 
-function TSHighlighter._on_line(_, _win, buf, line, highlighter)
-  -- on_line is only called when this is non-nil
-  if highlighter then
-    on_line_impl(highlighter, buf, line)
-  else
-    iter_active_tshl(buf, function(self)
-      on_line_impl(self, buf, line)
-    end)
-  end
+function TSHighlighter._on_line(_, _win, buf, line, _)
+  local self = TSHighlighter.active[buf]
+  if not self then return end
+
+  on_line_impl(self, buf, line)
 end
 
 function TSHighlighter._on_buf(_, buf)
-  iter_active_tshl(buf, function(self)
-    if self then
-      local tree = self.parser:parse()
-      self.root = (tree and tree:root()) or nil
-    end
-  end)
+  local self = TSHighlighter.active[buf]
+  if self then
+    self.tree:parse()
+  end
 end
 
-function TSHighlighter._on_win(_, _win, buf, _topline, botline)
-  iter_active_tshl(buf, function(self)
-    if not self then
-      return false
-    end
+function TSHighlighter._on_win(_, _win, buf, _topline)
+  local self = TSHighlighter.active[buf]
+  if not self then
+    return false
+  end
 
-    self.iter = nil
-    self.nextrow = 0
-    self.botline = botline
-    self.redraw_count = self.redraw_count + 1
-    return true
-  end)
+  self:reset_highlight_state()
+  self.redraw_count = self.redraw_count + 1
   return true
 end
 
diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
new file mode 100644
index 0000000000..ed07e73a55
--- /dev/null
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -0,0 +1,435 @@
+local query = require'vim.treesitter.query'
+local language = require'vim.treesitter.language'
+
+local LanguageTree = {}
+LanguageTree.__index = LanguageTree
+
+-- Represents a single treesitter parser for a language.
+-- The language can contain child languages with in it's range,
+-- hence the tree.
+--
+-- @param source Can be a bufnr or a string of text to parse
+-- @param lang The language this tree represents
+-- @param opts Options table
+-- @param opts.queries A table of language to injection query strings
+--                     This is useful for overridding the built in runtime file
+--                     searching for the injection language query per language.
+function LanguageTree.new(source, lang, opts)
+  language.require_language(lang)
+  opts = opts or {}
+
+  local custom_queries = opts.queries or {}
+  local self = setmetatable({
+    _source=source,
+    _lang=lang,
+    _children = {},
+    _regions = {},
+    _trees = {},
+    _opts = opts,
+    _injection_query = custom_queries[lang]
+      and query.parse_query(lang, custom_queries[lang])
+      or query.get_query(lang, "injections"),
+    _valid = false,
+    _parser = vim._create_ts_parser(lang),
+    _callbacks = {
+      changedtree = {},
+      bytes = {},
+      child_added = {},
+      child_removed = {}
+    },
+  }, LanguageTree)
+
+
+  return self
+end
+
+-- Invalidates this parser and all it's children
+function LanguageTree:invalidate()
+  self._valid = false
+
+  for _, child in ipairs(self._children) do
+    child:invalidate()
+  end
+end
+
+-- Returns all trees this language tree contains.
+-- Does not include child languages.
+function LanguageTree:trees()
+  return self._trees
+end
+
+-- Gets the language of this tree layer.
+function LanguageTree:lang()
+  return self._lang
+end
+
+-- Determines whether this tree is valid.
+-- If the tree is invalid, `parse()` must be called
+-- to get the an updated tree.
+function LanguageTree:is_valid()
+  return self._valid
+end
+
+-- Returns a map of language to child tree.
+function LanguageTree:children()
+  return self._children
+end
+
+-- Returns the source content of the language tree (bufnr or string).
+function LanguageTree:source()
+  return self._source
+end
+
+-- Parses all defined regions using a treesitter parser
+-- for the language this tree represents.
+-- This will run the injection query for this language to
+-- determine if any child languages should be created.
+function LanguageTree:parse()
+  if self._valid then
+    return self._trees
+  end
+
+  local parser = self._parser
+  local changes = {}
+
+  local old_trees = self._trees
+  self._trees = {}
+
+  -- If there are no ranges, set to an empty list
+  -- so the included ranges in the parser ar cleared.
+  if self._regions and #self._regions > 0 then
+    for i, ranges in ipairs(self._regions) do
+      local old_tree = old_trees[i]
+      parser:set_included_ranges(ranges)
+
+      local tree, tree_changes = parser:parse(old_tree, self._source)
+
+      table.insert(self._trees, tree)
+      vim.list_extend(changes, tree_changes)
+    end
+  else
+    local tree, tree_changes = parser:parse(old_trees[1], self._source)
+
+    table.insert(self._trees, tree)
+    vim.list_extend(changes, tree_changes)
+  end
+
+  local injections_by_lang = self:_get_injections()
+  local seen_langs = {}
+
+  for lang, injection_ranges in pairs(injections_by_lang) do
+    local child = self._children[lang]
+
+    if not child then
+      child = self:add_child(lang)
+    end
+
+    child:set_included_regions(injection_ranges)
+
+    local _, child_changes = child:parse()
+
+    -- Propagate any child changes so they are included in the
+    -- the change list for the callback.
+    if child_changes then
+      vim.list_extend(changes, child_changes)
+    end
+
+    seen_langs[lang] = true
+  end
+
+  for lang, _ in pairs(self._children) do
+    if not seen_langs[lang] then
+      self:remove_child(lang)
+    end
+  end
+
+  self._valid = true
+
+  self:_do_callback('changedtree', changes)
+  return self._trees, changes
+end
+
+-- Invokes the callback for each LanguageTree and it's children recursively
+-- @param fn The function to invoke. This is invoked with arguments (tree: LanguageTree, lang: string)
+-- @param include_self Whether to include the invoking tree in the results.
+function LanguageTree:for_each_child(fn, include_self)
+  if include_self then
+    fn(self, self._lang)
+  end
+
+  for _, child in pairs(self._children) do
+    child:for_each_child(fn, true)
+  end
+end
+
+-- Invokes the callback for each treesitter trees recursively.
+-- Note, this includes the invoking language tree's trees as well.
+-- @param fn The callback to invoke. The callback is invoked with arguments
+--           (tree: TSTree, languageTree: LanguageTree)
+function LanguageTree:for_each_tree(fn)
+  for _, tree in ipairs(self._trees) do
+    fn(tree, self)
+  end
+
+  for _, child in pairs(self._children) do
+    child:for_each_tree(fn)
+  end
+end
+
+-- Adds a child language to this tree.
+-- If the language already exists as a child, it will first be removed.
+-- @param lang The language to add.
+function LanguageTree:add_child(lang)
+  if self._children[lang] then
+    self:remove_child(lang)
+  end
+
+  self._children[lang] = LanguageTree.new(self._source, lang, self._opts)
+
+  self:invalidate()
+  self:_do_callback('child_added', self._children[lang])
+
+  return self._children[lang]
+end
+
+-- Removes a child language from this tree.
+-- @param lang The language to remove.
+function LanguageTree:remove_child(lang)
+  local child = self._children[lang]
+
+  if child then
+    self._children[lang] = nil
+    child:destroy()
+    self:invalidate()
+    self:_do_callback('child_removed', child)
+  end
+end
+
+-- Destroys this language tree and all it's children.
+-- Any cleanup logic should be performed here.
+-- Note, this DOES NOT remove this tree from a parent.
+-- `remove_child` must be called on the parent to remove it.
+function LanguageTree:destroy()
+  -- Cleanup here
+  for _, child in ipairs(self._children) do
+    child:destroy()
+  end
+end
+
+-- Sets the included regions that should be parsed by this parser.
+-- A region is a set of nodes and/or ranges that will be parsed in the same context.
+--
+-- For example, `{ { node1 }, { node2} }` is two separate regions.
+-- This will be parsed by the parser in two different contexts... thus resulting
+-- in two separate trees.
+--
+-- `{ { node1, node2 } }` is a single region consisting of two nodes.
+-- This will be parsed by the parser in a single context... thus resulting
+-- in a single tree.
+--
+-- This allows for embedded languages to be parsed together across different
+-- nodes, which is useful for templating languages like ERB and EJS.
+--
+-- Note, this call invalidates the tree and requires it to be parsed again.
+--
+-- @param regions A list of regions this tree should manange and parse.
+function LanguageTree:set_included_regions(regions)
+  self._regions = regions
+  -- Trees are no longer valid now that we have changed regions.
+  -- TODO(vigoux,steelsojka): Look into doing this smarter so we can use some of the
+  --                          old trees for incremental parsing. Currently, this only
+  --                          effects injected languages.
+  self._trees = {}
+  self:invalidate()
+end
+
+-- Gets the set of included regions
+function LanguageTree:included_regions()
+  return self._regions
+end
+
+-- Gets language injection points by language.
+-- This is where most of the injection processing occurs.
+-- TODO: Allow for an offset predicate to tailor the injection range
+--       instead of using the entire nodes range.
+-- @private
+function LanguageTree:_get_injections()
+  if not self._injection_query then return {} end
+
+  local injections = {}
+
+  for tree_index, tree in ipairs(self._trees) do
+    local root_node = tree:root()
+    local start_line, _, end_line, _ = root_node:range()
+
+    for pattern, match in self._injection_query:iter_matches(root_node, self._source, start_line, end_line+1) do
+      local lang = nil
+      local injection_node = nil
+      local combined = false
+
+      -- You can specify the content and language together
+      -- using a tag with the language, for example
+      -- @javascript
+      for id, node in pairs(match) do
+        local name = self._injection_query.captures[id]
+        -- TODO add a way to offset the content passed to the parser.
+        -- Needed to shave off leading quotes and things of that nature.
+
+        -- Lang should override any other language tag
+        if name == "language" then
+          lang = query.get_node_text(node, self._source)
+        elseif name == "combined" then
+          combined = true
+        elseif name == "content" then
+          injection_node = node
+        -- Ignore any tags that start with "_"
+        -- Allows for other tags to be used in matches
+        elseif string.sub(name, 1, 1) ~= "_" then
+          if lang == nil then
+            lang = name
+          end
+
+          if not injection_node then
+            injection_node = node
+          end
+        end
+      end
+
+      -- Each tree index should be isolated from the other nodes.
+      if not injections[tree_index] then
+        injections[tree_index] = {}
+      end
+
+      if not injections[tree_index][lang] then
+        injections[tree_index][lang] = {}
+      end
+
+      -- Key by pattern so we can either combine each node to parse in the same
+      -- context or treat each node independently.
+      if not injections[tree_index][lang][pattern] then
+        injections[tree_index][lang][pattern] = { combined = combined, nodes = {} }
+      end
+
+      table.insert(injections[tree_index][lang][pattern].nodes, injection_node)
+    end
+  end
+
+  local result = {}
+
+  -- Generate a map by lang of node lists.
+  -- Each list is a set of ranges that should be parsed
+  -- together.
+  for _, lang_map in ipairs(injections) do
+    for lang, patterns in pairs(lang_map) do
+      if not result[lang] then
+        result[lang] = {}
+      end
+
+      for _, entry in pairs(patterns) do
+        if entry.combined then
+          table.insert(result[lang], entry.nodes)
+        else
+          for _, node in ipairs(entry.nodes) do
+            table.insert(result[lang], {node})
+          end
+        end
+      end
+    end
+  end
+
+  return result
+end
+
+function LanguageTree:_do_callback(cb_name, ...)
+  for _, cb in ipairs(self._callbacks[cb_name]) do
+    cb(...)
+  end
+end
+
+function LanguageTree:_on_bytes(bufnr, changed_tick,
+                          start_row, start_col, start_byte,
+                          old_row, old_col, old_byte,
+                          new_row, new_col, new_byte)
+  self:invalidate()
+
+  local old_end_col = old_col + ((old_row == 0) and start_col or 0)
+  local new_end_col = new_col + ((new_row == 0) and start_col or 0)
+
+  -- Edit all trees recursively, together BEFORE emitting a bytes callback.
+  -- In most cases this callback should only be called from the root tree.
+  self:for_each_tree(function(tree)
+    tree:edit(start_byte,start_byte+old_byte,start_byte+new_byte,
+      start_row, start_col,
+      start_row+old_row, old_end_col,
+      start_row+new_row, new_end_col)
+  end)
+
+  self:_do_callback('bytes', bufnr, changed_tick,
+      start_row, start_col, start_byte,
+      old_row, old_col, old_byte,
+      new_row, new_col, new_byte)
+end
+
+--- Registers callbacks for the parser
+-- @param cbs An `nvim_buf_attach`-like table argument with the following keys :
+--  `on_bytes` : see `nvim_buf_attach`, but this will be called _after_ the parsers callback.
+--  `on_changedtree` : a callback that will be called everytime the tree has syntactical changes.
+--      it will only be passed one argument, that is a table of the ranges (as node ranges) that
+--      changed.
+--  `on_child_added` : emitted when a child is added to the tree.
+--  `on_child_removed` : emitted when a child is remvoed from the tree.
+function LanguageTree:register_cbs(cbs)
+  if not cbs then return end
+
+  if cbs.on_changedtree then
+    table.insert(self._callbacks.changedtree, cbs.on_changedtree)
+  end
+
+  if cbs.on_bytes then
+    table.insert(self._callbacks.bytes, cbs.on_bytes)
+  end
+
+  if cbs.on_child_added then
+    table.insert(self._callbacks.child_added, cbs.on_child_added)
+  end
+
+  if cbs.on_child_removed then
+    table.insert(self._callbacks.child_removed, cbs.on_child_removed)
+  end
+end
+
+local function region_contains(region, range)
+  for _, node in ipairs(region) do
+    local start_row, start_col, end_row, end_col = node:range()
+    local start_fits = start_row < range[1] or (start_row == range[1] and start_col <= range[2])
+    local end_fits = end_row > range[3] or (end_row == range[3] and end_col >= range[4])
+
+    if start_fits and end_fits then
+      return true
+    end
+  end
+
+  return false
+end
+
+function LanguageTree:contains(range)
+  for _, region in pairs(self._region) do
+    if region_contains(region, range) then
+      return true
+    end
+  end
+
+  return false
+end
+
+function LanguageTree:language_for_range(range)
+  for _, child in pairs(self._children) do
+    if child:contains(range) then
+      return child:node_for_range(range)
+    end
+  end
+
+  return self
+end
+
+return LanguageTree
diff --git a/test/functional/lua/treesitter_spec.lua b/test/functional/lua/treesitter_spec.lua
index 4c1083c386..273a5119cb 100644
--- a/test/functional/lua/treesitter_spec.lua
+++ b/test/functional/lua/treesitter_spec.lua
@@ -50,7 +50,7 @@ describe('treesitter API with C parser', function()
 
     exec_lua([[
       parser = vim.treesitter.get_parser(0, "c")
-      tree = parser:parse()
+      tree = parser:parse()[1]
       root = tree:root()
       lang = vim.treesitter.inspect_language('c')
     ]])
@@ -82,7 +82,7 @@ describe('treesitter API with C parser', function()
 
     feed("2G7|ay")
     exec_lua([[
-      tree2 = parser:parse()
+      tree2 = parser:parse()[1]
       root2 = tree2:root()
       descendant2 = root2:descendant_for_range(1,2,1,13)
     ]])
@@ -106,11 +106,8 @@ describe('treesitter API with C parser', function()
     eq(false, exec_lua("return child:id() == nil"))
     eq(false, exec_lua("return child:id() == tree"))
 
-    -- orginal tree did not change
-    eq({1,2,1,12}, exec_lua("return {descendant:range()}"))
-
     -- unchanged buffer: return the same tree
-    eq(true, exec_lua("return parser:parse() == tree2"))
+    eq(true, exec_lua("return parser:parse()[1] == tree2"))
   end)
 
     local test_text = [[
@@ -142,7 +139,7 @@ void ui_refresh(void)
     local res = exec_lua([[
       parser = vim.treesitter.get_parser(0, "c")
 
-      func_node = parser:parse():root():child(0)
+      func_node = parser:parse()[1]:root():child(0)
 
       res = {}
       for node, field in func_node:iter_children() do
@@ -166,7 +163,7 @@ void ui_refresh(void)
     local res = exec_lua([[
       parser = vim.treesitter.get_parser(0, "c")
 
-      func_node = parser:parse():root():child(0)
+      func_node = parser:parse()[1]:root():child(0)
 
       local res = {}
       for _, node in ipairs(func_node:field("type")) do
@@ -211,7 +208,7 @@ void ui_refresh(void)
     local res = exec_lua([[
       cquery = vim.treesitter.parse_query("c", ...)
       parser = vim.treesitter.get_parser(0, "c")
-      tree = parser:parse()
+      tree = parser:parse()[1]
       res = {}
       for cid, node in cquery:iter_captures(tree:root(), 0, 7, 14) do
         -- can't transmit node over RPC. just check the name and range
@@ -242,7 +239,7 @@ void ui_refresh(void)
     local res = exec_lua([[
       cquery = vim.treesitter.parse_query("c", ...)
       parser = vim.treesitter.get_parser(0, "c")
-      tree = parser:parse()
+      tree = parser:parse()[1]
       res = {}
       for pattern, match in cquery:iter_matches(tree:root(), 0, 7, 14) do
         -- can't transmit node over RPC. just check the name and range
@@ -275,7 +272,7 @@ void ui_refresh(void)
     local res = exec_lua([[
       cquery = vim.treesitter.parse_query("c", '((_) @quote (vim-match? @quote "^\\"$")) ((_) @quote (lua-match? @quote "^\\"$"))')
       parser = vim.treesitter.get_parser(0, "c")
-      tree = parser:parse()
+      tree = parser:parse()[1]
       res = {}
       for pattern, match in cquery:iter_matches(tree:root(), 0, 0, 1) do
         -- can't transmit node over RPC. just check the name and range
@@ -321,7 +318,7 @@ void ui_refresh(void)
     local query = query.parse_query("c", ...)
 
     local nodes = {}
-    for _, node in query:iter_captures(parser:parse():root(), 0, 0, 19) do
+    for _, node in query:iter_captures(parser:parse()[1]:root(), 0, 0, 19) do
       table.insert(nodes, {node:range()})
     end
 
@@ -343,10 +340,7 @@ void ui_refresh(void)
     eq({ 'contains?', 'eq?', 'is-main?', 'lua-match?', 'match?', 'vim-match?' }, res_list)
   end)
 
-  it('supports highlighting', function()
-    if not check_parser() then return end
-
-    local hl_text = [[
+  local hl_text = [[
 /// Schedule Lua callback on main loop's event queue
 static int nlua_schedule(lua_State *const lstate)
 {
@@ -363,7 +357,7 @@ static int nlua_schedule(lua_State *const lstate)
   return 0;
 }]]
 
-    local hl_query = [[
+local hl_query = [[
 (ERROR) @ErrorMsg
 
 "if" @keyword
@@ -398,249 +392,319 @@ static int nlua_schedule(lua_State *const lstate)
 (comment) @comment
 ]]
 
-    local screen = Screen.new(65, 18)
-    screen:attach()
-    screen:set_default_attr_ids({
-      [1] = {bold = true, foreground = Screen.colors.Blue1},
-      [2] = {foreground = Screen.colors.Blue1},
-      [3] = {bold = true, foreground = Screen.colors.SeaGreen4},
-      [4] = {bold = true, foreground = Screen.colors.Brown},
-      [5] = {foreground = Screen.colors.Magenta},
-      [6] = {foreground = Screen.colors.Red},
-      [7] = {bold = true, foreground = Screen.colors.SlateBlue},
-      [8] = {foreground = Screen.colors.Grey100, background = Screen.colors.Red},
-      [9] = {foreground = Screen.colors.Magenta, background = Screen.colors.Red},
-      [10] = {foreground = Screen.colors.Red, background = Screen.colors.Red},
-      [11] = {foreground = Screen.colors.Cyan4},
-    })
-
-    insert(hl_text)
-    screen:expect{grid=[[
-      /// Schedule Lua callback on main loop's event queue             |
-      static int nlua_schedule(lua_State *const lstate)                |
-      {                                                                |
-        if (lua_type(lstate, 1) != LUA_TFUNCTION                       |
-            || lstate != lstate) {                                     |
-          lua_pushliteral(lstate, "vim.schedule: expected function");  |
-          return lua_error(lstate);                                    |
+  describe('when highlighting', function()
+    local screen
+
+    before_each(function()
+      screen = Screen.new(65, 18)
+      screen:attach()
+      screen:set_default_attr_ids({
+        [1] = {bold = true, foreground = Screen.colors.Blue1},
+        [2] = {foreground = Screen.colors.Blue1},
+        [3] = {bold = true, foreground = Screen.colors.SeaGreen4},
+        [4] = {bold = true, foreground = Screen.colors.Brown},
+        [5] = {foreground = Screen.colors.Magenta},
+        [6] = {foreground = Screen.colors.Red},
+        [7] = {bold = true, foreground = Screen.colors.SlateBlue},
+        [8] = {foreground = Screen.colors.Grey100, background = Screen.colors.Red},
+        [9] = {foreground = Screen.colors.Magenta, background = Screen.colors.Red},
+        [10] = {foreground = Screen.colors.Red, background = Screen.colors.Red},
+        [11] = {foreground = Screen.colors.Cyan4},
+      })
+    end)
+
+    it('supports highlighting', function()
+      if not check_parser() then return end
+
+      insert(hl_text)
+      screen:expect{grid=[[
+        /// Schedule Lua callback on main loop's event queue             |
+        static int nlua_schedule(lua_State *const lstate)                |
+        {                                                                |
+          if (lua_type(lstate, 1) != LUA_TFUNCTION                       |
+              || lstate != lstate) {                                     |
+            lua_pushliteral(lstate, "vim.schedule: expected function");  |
+            return lua_error(lstate);                                    |
+          }                                                              |
+                                                                         |
+          LuaRef cb = nlua_ref(lstate, 1);                               |
+                                                                         |
+          multiqueue_put(main_loop.events, nlua_schedule_event,          |
+                         1, (void *)(ptrdiff_t)cb);                      |
+          return 0;                                                      |
+        ^}                                                                |
+        {1:~                                                                }|
+        {1:~                                                                }|
+                                                                         |
+      ]]}
+
+      exec_lua([[
+        local parser = vim.treesitter.get_parser(0, "c")
+        local highlighter = vim.treesitter.highlighter
+        local query = ...
+        test_hl = highlighter.new(parser, {queries = {c = query}})
+      ]], hl_query)
+      screen:expect{grid=[[
+        {2:/// Schedule Lua callback on main loop's event queue}             |
+        {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
+        {                                                                |
+          {4:if} ({11:lua_type}(lstate, {5:1}) != {5:LUA_TFUNCTION}                       |
+              || {6:lstate} != {6:lstate}) {                                     |
+            {11:lua_pushliteral}(lstate, {5:"vim.schedule: expected function"});  |
+            {4:return} {11:lua_error}(lstate);                                    |
+          }                                                              |
+                                                                         |
+          {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
+                                                                         |
+          multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
+                         {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
+          {4:return} {5:0};                                                      |
+        ^}                                                                |
+        {1:~                                                                }|
+        {1:~                                                                }|
+                                                                         |
+      ]]}
+
+      feed("5Goc<esc>dd")
+
+      screen:expect{grid=[[
+        {2:/// Schedule Lua callback on main loop's event queue}             |
+        {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
+        {                                                                |
+          {4:if} ({11:lua_type}(lstate, {5:1}) != {5:LUA_TFUNCTION}                       |
+              || {6:lstate} != {6:lstate}) {                                     |
+            {11:^lua_pushliteral}(lstate, {5:"vim.schedule: expected function"});  |
+            {4:return} {11:lua_error}(lstate);                                    |
+          }                                                              |
+                                                                         |
+          {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
+                                                                         |
+          multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
+                         {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
+          {4:return} {5:0};                                                      |
+        }                                                                |
+        {1:~                                                                }|
+        {1:~                                                                }|
+                                                                         |
+      ]]}
+
+      feed('7Go*/<esc>')
+      screen:expect{grid=[[
+        {2:/// Schedule Lua callback on main loop's event queue}             |
+        {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
+        {                                                                |
+          {4:if} ({11:lua_type}(lstate, {5:1}) != {5:LUA_TFUNCTION}                       |
+              || {6:lstate} != {6:lstate}) {                                     |
+            {11:lua_pushliteral}(lstate, {5:"vim.schedule: expected function"});  |
+            {4:return} {11:lua_error}(lstate);                                    |
+        {8:*^/}                                                               |
+          }                                                              |
+                                                                         |
+          {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
+                                                                         |
+          multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
+                         {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
+          {4:return} {5:0};                                                      |
+        }                                                                |
+        {1:~                                                                }|
+                                                                         |
+      ]]}
+
+      feed('3Go/*<esc>')
+      screen:expect{grid=[[
+        {2:/// Schedule Lua callback on main loop's event queue}             |
+        {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
+        {                                                                |
+        {2:/^*}                                                               |
+        {2:  if (lua_type(lstate, 1) != LUA_TFUNCTION}                       |
+        {2:      || lstate != lstate) {}                                     |
+        {2:    lua_pushliteral(lstate, "vim.schedule: expected function");}  |
+        {2:    return lua_error(lstate);}                                    |
+        {2:*/}                                                               |
+          }                                                              |
+                                                                         |
+          {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
+                                                                         |
+          multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
+                         {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
+          {4:return} {5:0};                                                      |
+        {8:}}                                                                |
+                                                                         |
+      ]]}
+
+      feed("gg$")
+      feed("~")
+      screen:expect{grid=[[
+        {2:/// Schedule Lua callback on main loop's event queu^E}             |
+        {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
+        {                                                                |
+        {2:/*}                                                               |
+        {2:  if (lua_type(lstate, 1) != LUA_TFUNCTION}                       |
+        {2:      || lstate != lstate) {}                                     |
+        {2:    lua_pushliteral(lstate, "vim.schedule: expected function");}  |
+        {2:    return lua_error(lstate);}                                    |
+        {2:*/}                                                               |
+          }                                                              |
+                                                                         |
+          {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
+                                                                         |
+          multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
+                         {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
+          {4:return} {5:0};                                                      |
+        {8:}}                                                                |
+                                                                         |
+      ]]}
+
+
+      feed("re")
+      screen:expect{grid=[[
+        {2:/// Schedule Lua callback on main loop's event queu^e}             |
+        {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
+        {                                                                |
+        {2:/*}                                                               |
+        {2:  if (lua_type(lstate, 1) != LUA_TFUNCTION}                       |
+        {2:      || lstate != lstate) {}                                     |
+        {2:    lua_pushliteral(lstate, "vim.schedule: expected function");}  |
+        {2:    return lua_error(lstate);}                                    |
+        {2:*/}                                                               |
+          }                                                              |
+                                                                         |
+          {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
+                                                                         |
+          multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
+                         {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
+          {4:return} {5:0};                                                      |
+        {8:}}                                                                |
+                                                                         |
+      ]]}
+    end)
+
+    it("supports highlighting with custom parser", function()
+      if not check_parser() then return end
+
+      screen:set_default_attr_ids({ {bold = true, foreground = Screen.colors.SeaGreen4} })
+
+      insert(test_text)
+
+      screen:expect{ grid= [[
+        int width = INT_MAX, height = INT_MAX;                         |
+        bool ext_widgets[kUIExtCount];                                 |
+        for (UIExtension i = 0; (int)i < kUIExtCount; i++) {           |
+          ext_widgets[i] = true;                                       |
         }                                                              |
                                                                        |
-        LuaRef cb = nlua_ref(lstate, 1);                               |
-                                                                       |
-        multiqueue_put(main_loop.events, nlua_schedule_event,          |
-                       1, (void *)(ptrdiff_t)cb);                      |
-        return 0;                                                      |
-      ^}                                                                |
-      {1:~                                                                }|
-      {1:~                                                                }|
-                                                                       |
-    ]]}
-
-    exec_lua([[
-      local parser = vim.treesitter.get_parser(0, "c")
-      local highlighter = vim.treesitter.highlighter
-      local query = ...
-      test_hl = highlighter.new(parser, query)
-    ]], hl_query)
-    screen:expect{grid=[[
-      {2:/// Schedule Lua callback on main loop's event queue}             |
-      {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
-      {                                                                |
-        {4:if} ({11:lua_type}(lstate, {5:1}) != {5:LUA_TFUNCTION}                       |
-            || {6:lstate} != {6:lstate}) {                                     |
-          {11:lua_pushliteral}(lstate, {5:"vim.schedule: expected function"});  |
-          {4:return} {11:lua_error}(lstate);                                    |
+        bool inclusive = ui_override();                                |
+        for (size_t i = 0; i < ui_count; i++) {                        |
+          UI *ui = uis[i];                                             |
+          width = MIN(ui->width, width);                               |
+          height = MIN(ui->height, height);                            |
+          foo = BAR(ui->bazaar, bazaar);                               |
+          for (UIExtension j = 0; (int)j < kUIExtCount; j++) {         |
+            ext_widgets[j] &= (ui->ui_ext[j] || inclusive);            |
+          }                                                            |
         }                                                              |
-                                                                       |
-        {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
-                                                                       |
-        multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
-                       {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
-        {4:return} {5:0};                                                      |
       ^}                                                                |
-      {1:~                                                                }|
-      {1:~                                                                }|
-                                                                       |
-    ]]}
-
-    feed("5Goc<esc>dd")
-
-    screen:expect{grid=[[
-      {2:/// Schedule Lua callback on main loop's event queue}             |
-      {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
-      {                                                                |
-        {4:if} ({11:lua_type}(lstate, {5:1}) != {5:LUA_TFUNCTION}                       |
-            || {6:lstate} != {6:lstate}) {                                     |
-          {11:^lua_pushliteral}(lstate, {5:"vim.schedule: expected function"});  |
-          {4:return} {11:lua_error}(lstate);                                    |
-        }                                                              |
-                                                                       |
-        {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
-                                                                       |
-        multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
-                       {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
-        {4:return} {5:0};                                                      |
-      }                                                                |
-      {1:~                                                                }|
-      {1:~                                                                }|
-                                                                       |
-    ]]}
-
-    feed('7Go*/<esc>')
-    screen:expect{grid=[[
-      {2:/// Schedule Lua callback on main loop's event queue}             |
-      {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
-      {                                                                |
-        {4:if} ({11:lua_type}(lstate, {5:1}) != {5:LUA_TFUNCTION}                       |
-            || {6:lstate} != {6:lstate}) {                                     |
-          {11:lua_pushliteral}(lstate, {5:"vim.schedule: expected function"});  |
-          {4:return} {11:lua_error}(lstate);                                    |
-      {8:*^/}                                                               |
-        }                                                              |
-                                                                       |
-        {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
-                                                                       |
-        multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
-                       {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
-        {4:return} {5:0};                                                      |
-      }                                                                |
-      {1:~                                                                }|
-                                                                       |
-    ]]}
-
-    feed('3Go/*<esc>')
-    screen:expect{grid=[[
-      {2:/// Schedule Lua callback on main loop's event queue}             |
-      {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
-      {                                                                |
-      {2:/^*}                                                               |
-      {2:  if (lua_type(lstate, 1) != LUA_TFUNCTION}                       |
-      {2:      || lstate != lstate) {}                                     |
-      {2:    lua_pushliteral(lstate, "vim.schedule: expected function");}  |
-      {2:    return lua_error(lstate);}                                    |
-      {2:*/}                                                               |
-        }                                                              |
-                                                                       |
-        {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
-                                                                       |
-        multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
-                       {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
-        {4:return} {5:0};                                                      |
-      {8:}}                                                                |
-                                                                       |
-    ]]}
-
-    feed("gg$")
-    feed("~")
-    screen:expect{grid=[[
-      {2:/// Schedule Lua callback on main loop's event queu^E}             |
-      {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
-      {                                                                |
-      {2:/*}                                                               |
-      {2:  if (lua_type(lstate, 1) != LUA_TFUNCTION}                       |
-      {2:      || lstate != lstate) {}                                     |
-      {2:    lua_pushliteral(lstate, "vim.schedule: expected function");}  |
-      {2:    return lua_error(lstate);}                                    |
-      {2:*/}                                                               |
-        }                                                              |
-                                                                       |
-        {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
-                                                                       |
-        multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
-                       {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
-        {4:return} {5:0};                                                      |
-      {8:}}                                                                |
-                                                                       |
-    ]]}
-
-
-    feed("re")
-    screen:expect{grid=[[
-      {2:/// Schedule Lua callback on main loop's event queu^e}             |
-      {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
-      {                                                                |
-      {2:/*}                                                               |
-      {2:  if (lua_type(lstate, 1) != LUA_TFUNCTION}                       |
-      {2:      || lstate != lstate) {}                                     |
-      {2:    lua_pushliteral(lstate, "vim.schedule: expected function");}  |
-      {2:    return lua_error(lstate);}                                    |
-      {2:*/}                                                               |
-        }                                                              |
-                                                                       |
-        {7:LuaRef} cb = {11:nlua_ref}(lstate, {5:1});                               |
-                                                                       |
-        multiqueue_put(main_loop.events, {11:nlua_schedule_event},          |
-                       {5:1}, ({3:void} *)({3:ptrdiff_t})cb);                      |
-        {4:return} {5:0};                                                      |
-      {8:}}                                                                |
                                                                        |
-    ]]}
-  end)
-
-  it("supports highlighting with custom parser", function()
-    if not check_parser() then return end
-
-    local screen = Screen.new(65, 18)
-    screen:attach()
-    screen:set_default_attr_ids({ {bold = true, foreground = Screen.colors.SeaGreen4} })
+      ]] }
 
-    insert(test_text)
-
-    screen:expect{ grid= [[
-      int width = INT_MAX, height = INT_MAX;                         |
-      bool ext_widgets[kUIExtCount];                                 |
-      for (UIExtension i = 0; (int)i < kUIExtCount; i++) {           |
-        ext_widgets[i] = true;                                       |
-      }                                                              |
-                                                                     |
-      bool inclusive = ui_override();                                |
-      for (size_t i = 0; i < ui_count; i++) {                        |
-        UI *ui = uis[i];                                             |
-        width = MIN(ui->width, width);                               |
-        height = MIN(ui->height, height);                            |
-        foo = BAR(ui->bazaar, bazaar);                               |
-        for (UIExtension j = 0; (int)j < kUIExtCount; j++) {         |
-          ext_widgets[j] &= (ui->ui_ext[j] || inclusive);            |
-        }                                                            |
-      }                                                              |
-    ^}                                                                |
-                                                                     |
-    ]] }
-
-    exec_lua([[
-    parser = vim.treesitter.get_parser(0, "c")
-    query = vim.treesitter.parse_query("c", "(declaration) @decl")
+      exec_lua([[
+      parser = vim.treesitter.get_parser(0, "c")
+      query = vim.treesitter.parse_query("c", "(declaration) @decl")
 
-    local nodes = {}
-    for _, node in query:iter_captures(parser:parse():root(), 0, 0, 19) do
-      table.insert(nodes, node)
-    end
+      local nodes = {}
+      for _, node in query:iter_captures(parser:parse()[1]:root(), 0, 0, 19) do
+        table.insert(nodes, node)
+      end
 
-    parser:set_included_ranges(nodes)
+      parser:set_included_regions({nodes})
 
-    local hl = vim.treesitter.highlighter.new(parser, "(identifier) @type")
-    ]])
+      local hl = vim.treesitter.highlighter.new(parser, {queries = {c = "(identifier) @type"}})
+      ]])
 
-    screen:expect{ grid = [[
-      int {1:width} = {1:INT_MAX}, {1:height} = {1:INT_MAX};                         |
-      bool {1:ext_widgets}[{1:kUIExtCount}];                                 |
-      for (UIExtension {1:i} = 0; (int)i < kUIExtCount; i++) {           |
-        ext_widgets[i] = true;                                       |
-      }                                                              |
-                                                                     |
-      bool {1:inclusive} = {1:ui_override}();                                |
-      for (size_t {1:i} = 0; i < ui_count; i++) {                        |
-        UI *{1:ui} = {1:uis}[{1:i}];                                             |
-        width = MIN(ui->width, width);                               |
-        height = MIN(ui->height, height);                            |
-        foo = BAR(ui->bazaar, bazaar);                               |
-        for (UIExtension {1:j} = 0; (int)j < kUIExtCount; j++) {         |
-          ext_widgets[j] &= (ui->ui_ext[j] || inclusive);            |
-        }                                                            |
-      }                                                              |
-    ^}                                                                |
-                                                                     |
-    ]] }
+      screen:expect{ grid = [[
+        int {1:width} = {1:INT_MAX}, {1:height} = {1:INT_MAX};                         |
+        bool {1:ext_widgets}[{1:kUIExtCount}];                                 |
+        for (UIExtension {1:i} = 0; (int)i < kUIExtCount; i++) {           |
+          ext_widgets[i] = true;                                       |
+        }                                                              |
+                                                                       |
+        bool {1:inclusive} = {1:ui_override}();                                |
+        for (size_t {1:i} = 0; i < ui_count; i++) {                        |
+          UI *{1:ui} = {1:uis}[{1:i}];                                             |
+          width = MIN(ui->width, width);                               |
+          height = MIN(ui->height, height);                            |
+          foo = BAR(ui->bazaar, bazaar);                               |
+          for (UIExtension {1:j} = 0; (int)j < kUIExtCount; j++) {         |
+            ext_widgets[j] &= (ui->ui_ext[j] || inclusive);            |
+          }                                                            |
+        }                                                              |
+      ^}                                                                |
+                                                                       |
+      ]] }
+    end)
+
+    it("supports highlighting injected languages", function()
+      if not check_parser() then return end
+
+      insert([[
+      int x = INT_MAX;
+      #define READ_STRING(x, y) (char_u *)read_string((x), (size_t)(y))
+      #define foo void main() { \
+                    return 42;  \
+                  }
+      ]])
+
+      screen:expect{grid=[[
+        int x = INT_MAX;                                                 |
+        #define READ_STRING(x, y) (char_u *)read_string((x), (size_t)(y))|
+        #define foo void main() { \                                      |
+                      return 42;  \                                      |
+                    }                                                    |
+        ^                                                                 |
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+                                                                         |
+      ]]}
+
+      exec_lua([[
+        local parser = vim.treesitter.get_parser(0, "c", {
+          queries = {c = "(preproc_def (preproc_arg) @c) (preproc_function_def value: (preproc_arg) @c)"}
+        })
+        local highlighter = vim.treesitter.highlighter
+        local query = ...
+        test_hl = highlighter.new(parser, {queries = {c = query}})
+      ]], hl_query)
+
+      screen:expect{grid=[[
+        {3:int} x = {5:INT_MAX};                                                 |
+        #define {5:READ_STRING}(x, y) ({3:char_u} *)read_string((x), ({3:size_t})(y))|
+        #define foo {3:void} main() { \                                      |
+                      {4:return} {5:42};  \                                      |
+                    }                                                    |
+        ^                                                                 |
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+        {1:~                                                                }|
+                                                                         |
+      ]]}
+    end)
   end)
 
   it('inspects language', function()
@@ -690,7 +754,7 @@ static int nlua_schedule(lua_State *const lstate)
 
     local res = exec_lua [[
     parser = vim.treesitter.get_parser(0, "c")
-    return { parser:parse():root():range() }
+    return { parser:parse()[1]:root():range() }
     ]]
 
     eq({0, 0, 19, 0}, res)
@@ -698,16 +762,22 @@ static int nlua_schedule(lua_State *const lstate)
     -- The following sets the included ranges for the current parser
     -- As stated here, this only includes the function (thus the whole buffer, without the last line)
     local res2 = exec_lua [[
-    local root = parser:parse():root()
-    parser:set_included_ranges({root:child(0)})
-    parser.valid = false
-    return { parser:parse():root():range() }
+    local root = parser:parse()[1]:root()
+    parser:set_included_regions({{root:child(0)}})
+    parser:invalidate()
+    return { parser:parse()[1]:root():range() }
     ]]
 
     eq({0, 0, 18, 1}, res2)
 
     local range = exec_lua [[
-      return parser:included_ranges()
+      local res = {}
+      for _, region in ipairs(parser:included_regions()) do
+        for _, node in ipairs(region) do
+          table.insert(res, {node:range()})
+        end
+      end
+      return res
     ]]
 
     eq(range, { { 0, 0, 18, 1 } })
@@ -717,19 +787,18 @@ static int nlua_schedule(lua_State *const lstate)
 
     insert(test_text)
 
-
     local res = exec_lua [[
     parser = vim.treesitter.get_parser(0, "c")
     query = vim.treesitter.parse_query("c", "(declaration) @decl")
 
     local nodes = {}
-    for _, node in query:iter_captures(parser:parse():root(), 0, 0, 19) do
+    for _, node in query:iter_captures(parser:parse()[1]:root(), 0, 0, 19) do
       table.insert(nodes, node)
     end
 
-    parser:set_included_ranges(nodes)
+    parser:set_included_regions({nodes})
 
-    local root = parser:parse():root()
+    local root = parser:parse()[1]:root()
 
     local res = {}
     for i=0,(root:named_child_count() - 1) do
@@ -740,21 +809,18 @@ static int nlua_schedule(lua_State *const lstate)
 
     eq({
       { 2, 2, 2, 40 },
-      { 3, 3, 3, 32 },
-      { 4, 7, 4, 8 },
-      { 4, 8, 4, 25 },
-      { 8, 2, 8, 6 },
-      { 8, 7, 8, 33 },
-      { 9, 8, 9, 20 },
-      { 10, 4, 10, 5 },
-      { 10, 5, 10, 20 },
+      { 3, 2, 3, 32 },
+      { 4, 7, 4, 25 },
+      { 8, 2, 8, 33 },
+      { 9, 7, 9, 20 },
+      { 10, 4, 10, 20 },
       { 14, 9, 14, 27 } }, res)
   end)
 
   it("allows to create string parsers", function()
     local ret = exec_lua [[
       local parser = vim.treesitter.get_string_parser("int foo = 42;", "c")
-      return { parser:parse():root():range() }
+      return { parser:parse()[1]:root():range() }
     ]]
 
     eq({ 0, 0, 0, 13 }, ret)
@@ -773,7 +839,7 @@ static int nlua_schedule(lua_State *const lstate)
     local nodes = {}
     local query = vim.treesitter.parse_query("c", '((identifier) @id (eq? @id "foo"))')
 
-    for _, node in query:iter_captures(parser:parse():root(), str, 0, 2) do
+    for _, node in query:iter_captures(parser:parse()[1]:root(), str, 0, 2) do
       table.insert(nodes, { node:range() })
     end
 
@@ -781,4 +847,68 @@ static int nlua_schedule(lua_State *const lstate)
 
     eq({ {0, 10, 0, 13} }, ret)
   end)
+
+  describe("when creating a language tree", function()
+    local function get_ranges()
+      return exec_lua([[
+      local result = {}
+      parser:for_each_tree(function(tree) table.insert(result, {tree:root():range()}) end)
+      return result
+      ]])
+    end
+
+    before_each(function()
+      insert([[
+        int x = INT_MAX;
+        #define READ_STRING(x, y) (char_u *)read_string((x), (size_t)(y))
+        #define READ_STRING_OK(x, y) (char_u *)read_string((x), (size_t)(y))
+        #define VALUE 0
+        #define VALUE1 1
+        #define VALUE2 2
+      ]])
+    end)
+
+    describe("when parsing regions independently", function()
+      it("should inject a language", function()
+        exec_lua([[
+        parser = vim.treesitter.get_parser(0, "c", {
+          queries = {
+            c = "(preproc_def (preproc_arg) @c) (preproc_function_def value: (preproc_arg) @c)"}})
+        ]])
+
+        eq("table", exec_lua("return type(parser:children().c)"))
+        eq(5, exec_lua("return #parser:children().c:trees()"))
+        eq({
+          {0, 2, 7, 0},   -- root tree
+          {3, 16, 3, 17}, -- VALUE 0
+          {4, 17, 4, 18}, -- VALUE1 1
+          {5, 17, 5, 18}, -- VALUE2 2
+          {1, 28, 1, 67}, -- READ_STRING(x, y) (char_u *)read_string((x), (size_t)(y))
+          {2, 31, 2, 70}  -- READ_STRING_OK(x, y) (char_u *)read_string((x), (size_t)(y))
+        }, get_ranges())
+      end)
+    end)
+
+    describe("when parsing regions combined", function()
+      it("should inject a language", function()
+        exec_lua([[
+        parser = vim.treesitter.get_parser(0, "c", {
+          queries = {
+            c = "(preproc_def (preproc_arg) @c @combined) (preproc_function_def value: (preproc_arg) @c @combined)"}})
+        ]])
+
+        eq("table", exec_lua("return type(parser:children().c)"))
+        eq(2, exec_lua("return #parser:children().c:trees()"))
+        eq({
+          {0, 2, 7, 0},   -- root tree
+          {3, 16, 5, 18}, -- VALUE 0
+                          -- VALUE1 1
+                          -- VALUE2 2
+          {1, 28, 2, 70}  -- READ_STRING(x, y) (char_u *)read_string((x), (size_t)(y))
+                          -- READ_STRING_OK(x, y) (char_u *)read_string((x), (size_t)(y))
+        }, get_ranges())
+      end)
+    end)
+  end)
+
 end)