ui: fix problem with sattr_T overflow

sattr_T was defined as uint16_t. But this is not enough to handle the
24-bit colors of the terminal. To solve this problem, change it to int.
In 32bit, int may overflow. So, if it overflows, change it to ignore it
without adding more attr_entries.

fixes #12366

2 files changed, 7 insertions, 2 deletions

diff --git a/src/nvim/grid_defs.h b/src/nvim/grid_defs.h
index c6687c8da9..e14aae73d8 100644
--- a/src/nvim/grid_defs.h
+++ b/src/nvim/grid_defs.h
@@ -11,7 +11,7 @@
 
 // The characters and attributes drawn on grids.
 typedef char_u schar_T[(MAX_MCO+1) * 4 + 1];
-typedef int16_t sattr_T;
+typedef int sattr_T;
 
 /// ScreenGrid represents a resizable rectuangular grid displayed by UI clients.
 ///
diff --git a/src/nvim/highlight.c b/src/nvim/highlight.c
index c0cae54572..262afba07a 100644
--- a/src/nvim/highlight.c
+++ b/src/nvim/highlight.c
@@ -90,7 +90,12 @@ static int get_attr_entry(HlEntry entry)
     }
   }
 
-  id = (int)kv_size(attr_entries);
+  size_t next_id = kv_size(attr_entries);
+  if (next_id > INT_MAX) {
+    ELOG("The index on attr_entries has overflowed");
+    return 0;
+  }
+  id = (int)next_id;
   kv_push(attr_entries, entry);
 
   map_put(HlEntry, int)(attr_entry_ids, entry, id);