vim-patch:9.1.1077: included syntax items do not understand contains=TOP (#32343)

Problem:  Syntax engine interpreted contains=TOP as matching nothing
          inside included files, since :syn-include forces HL_CONTAINED
          on for every included item. After 8.2.2761, interprets
          contains=TOP as contains=@INCLUDED, which is also not correct
          since it doesn't respect exclusions, and doesn't work if there
          is no @INCLUDED cluster.
Solution: revert patch 8.2.2761, instead track groups that have had
          HL_CONTAINED forced, and interpret contains=TOP and
          contains=CONTAINED using this. (Theodore Dubois)

fixes: vim/vim#11277
closes: vim/vim#16571

https://github.com/vim/vim/commit/f50d5364d790619a3b982a3ad3658b5a10daf511

Co-authored-by: Theodore Dubois <tblodt@icloud.com>

3 files changed, 52 insertions, 38 deletions

diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c
index 7160c757bb..6a41c2ad05 100644
--- a/src/nvim/syntax.c
+++ b/src/nvim/syntax.c
@@ -1644,7 +1644,7 @@ static int syn_current_attr(const bool syncing, const bool displaying, bool *con
                        ? !(spp->sp_flags & HL_CONTAINED)
                        : in_id_list(cur_si,
                                     cur_si->si_cont_list, &spp->sp_syn,
-                                    spp->sp_flags & HL_CONTAINED)))) {
+                                    spp->sp_flags)))) {
               // If we already tried matching in this line, and
               // there isn't a match before next_match_col, skip
               // this item.
@@ -2775,7 +2775,7 @@ static keyentry_T *match_keyword(char *keyword, hashtab_T *ht, stateitem_T *cur_
           : (cur_si == NULL
              ? !(kp->flags & HL_CONTAINED)
              : in_id_list(cur_si, cur_si->si_cont_list,
-                          &kp->k_syn, kp->flags & HL_CONTAINED))) {
+                          &kp->k_syn, kp->flags))) {
         return kp;
       }
     }
@@ -3931,7 +3931,7 @@ static void syn_incl_toplevel(int id, int *flagsp)
   if ((*flagsp & HL_CONTAINED) || curwin->w_s->b_syn_topgrp == 0) {
     return;
   }
-  *flagsp |= HL_CONTAINED;
+  *flagsp |= HL_CONTAINED | HL_INCLUDED_TOPLEVEL;
   if (curwin->w_s->b_syn_topgrp >= SYNID_CLUSTER) {
     // We have to alloc this, because syn_combine_list() will free it.
     int16_t *grp_list = xmalloc(2 * sizeof(*grp_list));
@@ -4977,16 +4977,13 @@ static int get_id_list(char **const arg, const int keylen, int16_t **const list,
           break;
         }
         if (name[1] == 'A') {
-          id = SYNID_ALLBUT + current_syn_inc_tag;
+          id = SYNID_ALLBUT;
         } else if (name[1] == 'T') {
-          if (curwin->w_s->b_syn_topgrp >= SYNID_CLUSTER) {
-            id = curwin->w_s->b_syn_topgrp;
-          } else {
-            id = SYNID_TOP + current_syn_inc_tag;
-          }
+          id = SYNID_TOP;
         } else {
-          id = SYNID_CONTAINED + current_syn_inc_tag;
+          id = SYNID_CONTAINED;
         }
+        id += current_syn_inc_tag;
       } else if (name[1] == '@') {
         if (skip) {
           id = -1;
@@ -5104,8 +5101,8 @@ static int16_t *copy_id_list(const int16_t *const list)
 /// @param cur_si     current item or NULL
 /// @param list       id list
 /// @param ssp        group id and ":syn include" tag of group
-/// @param contained  group id is contained
-static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, int contained)
+/// @param flags      group flags
+static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, int flags)
 {
   int retval;
   int16_t id = ssp->id;
@@ -5123,8 +5120,7 @@ static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, in
     // cur_si->si_idx is -1 for keywords, these never contain anything.
     if (cur_si->si_idx >= 0 && in_id_list(NULL, ssp->cont_in_list,
                                           &(SYN_ITEMS(syn_block)[cur_si->si_idx].sp_syn),
-                                          SYN_ITEMS(syn_block)[cur_si->si_idx].sp_flags &
-                                          HL_CONTAINED)) {
+                                          SYN_ITEMS(syn_block)[cur_si->si_idx].sp_flags)) {
       return true;
     }
   }
@@ -5136,9 +5132,14 @@ static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, in
   // If list is ID_LIST_ALL, we are in a transparent item that isn't
   // inside anything.  Only allow not-contained groups.
   if (list == ID_LIST_ALL) {
-    return !contained;
+    return !(flags & HL_CONTAINED);
   }
 
+  // Is this top-level (i.e. not 'contained') in the file it was declared in?
+  // For included files, this is different from HL_CONTAINED, which is set
+  // unconditionally.
+  bool toplevel = !(flags & HL_CONTAINED) || (flags & HL_INCLUDED_TOPLEVEL);
+
   // If the first item is "ALLBUT", return true if "id" is NOT in the
   // contains list.  We also require that "id" is at the same ":syn include"
   // level as the list.
@@ -5151,12 +5152,12 @@ static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, in
       }
     } else if (item < SYNID_CONTAINED) {
       // TOP: accept all not-contained groups in the same file
-      if (item - SYNID_TOP != ssp->inc_tag || contained) {
+      if (item - SYNID_TOP != ssp->inc_tag || !toplevel) {
         return false;
       }
     } else {
       // CONTAINED: accept all contained groups in the same file
-      if (item - SYNID_CONTAINED != ssp->inc_tag || !contained) {
+      if (item - SYNID_CONTAINED != ssp->inc_tag || toplevel) {
         return false;
       }
     }
@@ -5177,7 +5178,7 @@ static int in_id_list(stateitem_T *cur_si, int16_t *list, struct sp_syn *ssp, in
       // cluster that includes itself (indirectly)
       if (scl_list != NULL && depth < 30) {
         depth++;
-        int r = in_id_list(NULL, scl_list, ssp, contained);
+        int r = in_id_list(NULL, scl_list, ssp, flags);
         depth--;
         if (r) {
           return retval;
diff --git a/src/nvim/syntax.h b/src/nvim/syntax.h
index 69fc1a74b8..a4a15add6d 100644
--- a/src/nvim/syntax.h
+++ b/src/nvim/syntax.h
@@ -7,25 +7,26 @@
 #include "nvim/types_defs.h"  // IWYU pragma: keep
 
 enum {
-  HL_CONTAINED   = 0x01,     ///< not used on toplevel
-  HL_TRANSP      = 0x02,     ///< has no highlighting
-  HL_ONELINE     = 0x04,     ///< match within one line only
-  HL_HAS_EOL     = 0x08,     ///< end pattern that matches with $
-  HL_SYNC_HERE   = 0x10,     ///< sync point after this item (syncing only)
-  HL_SYNC_THERE  = 0x20,     ///< sync point at current line (syncing only)
-  HL_MATCH       = 0x40,     ///< use match ID instead of item ID
-  HL_SKIPNL      = 0x80,     ///< nextgroup can skip newlines
-  HL_SKIPWHITE   = 0x100,    ///< nextgroup can skip white space
-  HL_SKIPEMPTY   = 0x200,    ///< nextgroup can skip empty lines
-  HL_KEEPEND     = 0x400,    ///< end match always kept
-  HL_EXCLUDENL   = 0x800,    ///< exclude NL from match
-  HL_DISPLAY     = 0x1000,   ///< only used for displaying, not syncing
-  HL_FOLD        = 0x2000,   ///< define fold
-  HL_EXTEND      = 0x4000,   ///< ignore a keepend
-  HL_MATCHCONT   = 0x8000,   ///< match continued from previous line
-  HL_TRANS_CONT  = 0x10000,  ///< transparent item without contains arg
-  HL_CONCEAL     = 0x20000,  ///< can be concealed
-  HL_CONCEALENDS = 0x40000,  ///< can be concealed
+  HL_CONTAINED         = 0x01,     ///< not used on toplevel
+  HL_TRANSP            = 0x02,     ///< has no highlighting
+  HL_ONELINE           = 0x04,     ///< match within one line only
+  HL_HAS_EOL           = 0x08,     ///< end pattern that matches with $
+  HL_SYNC_HERE         = 0x10,     ///< sync point after this item (syncing only)
+  HL_SYNC_THERE        = 0x20,     ///< sync point at current line (syncing only)
+  HL_MATCH             = 0x40,     ///< use match ID instead of item ID
+  HL_SKIPNL            = 0x80,     ///< nextgroup can skip newlines
+  HL_SKIPWHITE         = 0x100,    ///< nextgroup can skip white space
+  HL_SKIPEMPTY         = 0x200,    ///< nextgroup can skip empty lines
+  HL_KEEPEND           = 0x400,    ///< end match always kept
+  HL_EXCLUDENL         = 0x800,    ///< exclude NL from match
+  HL_DISPLAY           = 0x1000,   ///< only used for displaying, not syncing
+  HL_FOLD              = 0x2000,   ///< define fold
+  HL_EXTEND            = 0x4000,   ///< ignore a keepend
+  HL_MATCHCONT         = 0x8000,   ///< match continued from previous line
+  HL_TRANS_CONT        = 0x10000,  ///< transparent item without contains arg
+  HL_CONCEAL           = 0x20000,  ///< can be concealed
+  HL_CONCEALENDS       = 0x40000,  ///< can be concealed
+  HL_INCLUDED_TOPLEVEL = 0x80000,  ///< toplevel item in included syntax, allowed by contains=TOP
 };
 
 #define SYN_GROUP_STATIC(s) syn_check_group(S_LEN(s))
diff --git a/test/old/testdir/test_syntax.vim b/test/old/testdir/test_syntax.vim
index 207efb6223..6db5d3c04d 100644
--- a/test/old/testdir/test_syntax.vim
+++ b/test/old/testdir/test_syntax.vim
@@ -944,7 +944,7 @@ func Test_syn_contained_transparent()
 endfunc
 
 func Test_syn_include_contains_TOP()
-  let l:case = "TOP in included syntax means its group list name"
+  let l:case = "TOP in included syntax refers to top level of that included syntax"
   new
   syntax include @INCLUDED syntax/c.vim
   syntax region FencedCodeBlockC start=/```c/ end=/```/ contains=@INCLUDED
@@ -959,6 +959,18 @@ func Test_syn_include_contains_TOP()
   bw!
 endfunc
 
+func Test_syn_include_contains_TOP_excluding()
+  new
+  syntax include @INCLUDED syntax/c.vim
+  syntax region FencedCodeBlockC start=/```c/ end=/```/ contains=@INCLUDED
+
+  call setline(1,  ['```c', '#if 0', 'int', '#else', 'int', '#if', '#endif', '```' ])
+  let l:expected = ["cCppOutElse", "cConditional"]
+  eval AssertHighlightGroups(6, 1, l:expected, 1)
+  syntax clear
+  bw!
+endfunc
+
 " This was using freed memory
 func Test_WinEnter_synstack_synID()
   autocmd WinEnter * call synstack(line("."), col("."))