Reduce InstanceData footprint

The InstanceData type in the rendering subsystem was previously 17 f32s
plus one u8 which occupied a total of 72 bytes per instance. This meant
that for every character or background cell drawn, 72 bytes were sent to
the GPU. In the case of a 400x100 cell grid, a total of 2.9MB would be
sent.

This patch reduces InstanceData's size to 36 bytes, a 50% improvement!
Using the above example for comparison, a worst case of 1.44MB would be
transferred.

The motivation for this patch comes from macOS. Once the terminal grid
would reach a certain size, performance experienced a sharp and dramatic
drop (render times would go from ~3ms to ~16ms). I don't want to
speculate too much on the underlying issue, but suffice it to say that
this patch alleviates the problem in my testing.

While the performance impact was most significant on macOS, with
rendering times cut by more than 50% in some cases, this also results in
a measurable performance difference on other systems with high density
grids.

Co-authored-by: Christian Duerr <contact@christianduerr.com>

1 files changed, 91 insertions, 114 deletions

diff --git a/alacritty/src/renderer/mod.rs b/alacritty/src/renderer/mod.rs
index 79df6b34..6b27a837 100644
--- a/alacritty/src/renderer/mod.rs
+++ b/alacritty/src/renderer/mod.rs
@@ -124,11 +124,11 @@ pub struct RectShaderProgram {
 #[derive(Copy, Debug, Clone)]
 pub struct Glyph {
     tex_id: GLuint,
-    colored: bool,
-    top: f32,
-    left: f32,
-    width: f32,
-    height: f32,
+    multicolor: u8,
+    top: i16,
+    left: i16,
+    width: i16,
+    height: i16,
     uv_bot: f32,
     uv_left: f32,
     uv_width: f32,
@@ -392,31 +392,31 @@ impl GlyphCache {
 #[repr(C)]
 struct InstanceData {
     // Coords.
-    col: f32,
-    row: f32,
+    col: u16,
+    row: u16,
     // Glyph offset.
-    left: f32,
-    top: f32,
-    // Glyph scale.
-    width: f32,
-    height: f32,
-    // uv offset.
+    left: i16,
+    top: i16,
+    // Glyph size.
+    width: i16,
+    height: i16,
+    // UV offset.
     uv_left: f32,
     uv_bot: f32,
     // uv scale.
     uv_width: f32,
     uv_height: f32,
     // Color.
-    r: f32,
-    g: f32,
-    b: f32,
-    // Background color.
-    bg_r: f32,
-    bg_g: f32,
-    bg_b: f32,
-    bg_a: f32,
-    // Flag indicating that glyph uses multiple colors, like an Emoji.
+    r: u8,
+    g: u8,
+    b: u8,
+    // Flag indicating that a glyph uses multiple colors; like an Emoji.
     multicolor: u8,
+    // Background color.
+    bg_r: u8,
+    bg_g: u8,
+    bg_b: u8,
+    bg_a: u8,
 }
 
 #[derive(Debug)]
@@ -471,8 +471,8 @@ impl Batch {
         }
 
         self.instances.push(InstanceData {
-            col: cell.column.0 as f32,
-            row: cell.line.0 as f32,
+            col: cell.column.0 as u16,
+            row: cell.line.0 as u16,
 
             top: glyph.top,
             left: glyph.left,
@@ -484,15 +484,15 @@ impl Batch {
             uv_width: glyph.uv_width,
             uv_height: glyph.uv_height,
 
-            r: f32::from(cell.fg.r),
-            g: f32::from(cell.fg.g),
-            b: f32::from(cell.fg.b),
+            r: cell.fg.r,
+            g: cell.fg.g,
+            b: cell.fg.b,
 
-            bg_r: f32::from(cell.bg.r),
-            bg_g: f32::from(cell.bg.g),
-            bg_b: f32::from(cell.bg.b),
-            bg_a: cell.bg_alpha,
-            multicolor: glyph.colored as u8,
+            bg_r: cell.bg.r,
+            bg_g: cell.bg.g,
+            bg_b: cell.bg.b,
+            bg_a: (cell.bg_alpha * 255.0) as u8,
+            multicolor: glyph.multicolor,
         });
     }
 
@@ -581,72 +581,49 @@ impl QuadRenderer {
                 ptr::null(),
                 gl::STREAM_DRAW,
             );
+
+            let mut index = 0;
+            let mut size = 0;
+
+            macro_rules! add_attr {
+                ($count:expr, $gl_type:expr, $type:ty) => {
+                    gl::VertexAttribPointer(
+                        index,
+                        $count,
+                        $gl_type,
+                        gl::FALSE,
+                        size_of::<InstanceData>() as i32,
+                        size as *const _,
+                    );
+                    gl::EnableVertexAttribArray(index);
+                    gl::VertexAttribDivisor(index, 1);
+
+                    #[allow(unused_assignments)]
+                    {
+                        size += $count * size_of::<$type>();
+                        index += 1;
+                    }
+                };
+            }
+
             // Coords.
-            gl::VertexAttribPointer(
-                0,
-                2,
-                gl::FLOAT,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                ptr::null(),
-            );
-            gl::EnableVertexAttribArray(0);
-            gl::VertexAttribDivisor(0, 1);
-            // Glyph offset.
-            gl::VertexAttribPointer(
-                1,
-                4,
-                gl::FLOAT,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                (2 * size_of::<f32>()) as *const _,
-            );
-            gl::EnableVertexAttribArray(1);
-            gl::VertexAttribDivisor(1, 1);
-            // uv.
-            gl::VertexAttribPointer(
-                2,
-                4,
-                gl::FLOAT,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                (6 * size_of::<f32>()) as *const _,
-            );
-            gl::EnableVertexAttribArray(2);
-            gl::VertexAttribDivisor(2, 1);
-            // Color.
-            gl::VertexAttribPointer(
-                3,
-                3,
-                gl::FLOAT,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                (10 * size_of::<f32>()) as *const _,
-            );
-            gl::EnableVertexAttribArray(3);
-            gl::VertexAttribDivisor(3, 1);
+            add_attr!(2, gl::UNSIGNED_SHORT, u16);
+
+            // Glyph offset and size.
+            add_attr!(4, gl::SHORT, i16);
+
+            // UV offset.
+            add_attr!(4, gl::FLOAT, f32);
+
+            // Color and multicolor flag.
+            //
+            // These are packed together because of an OpenGL driver issue on macOS, which caused a
+            // `vec3(u8)` text color and a `u8` multicolor flag to increase the rendering time by a
+            // huge margin.
+            add_attr!(4, gl::UNSIGNED_BYTE, u8);
+
             // Background color.
-            gl::VertexAttribPointer(
-                4,
-                4,
-                gl::FLOAT,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                (13 * size_of::<f32>()) as *const _,
-            );
-            gl::EnableVertexAttribArray(4);
-            gl::VertexAttribDivisor(4, 1);
-            // Multicolor flag.
-            gl::VertexAttribPointer(
-                5,
-                1,
-                gl::BYTE,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                (17 * size_of::<f32>()) as *const _,
-            );
-            gl::EnableVertexAttribArray(5);
-            gl::VertexAttribDivisor(5, 1);
+            add_attr!(4, gl::UNSIGNED_BYTE, u8);
 
             // Rectangle setup.
             gl::GenVertexArrays(1, &mut rect_vao);
@@ -1089,7 +1066,7 @@ impl<'a> RenderApi<'a> {
             // right side of the preceding character. Since we render the
             // zero-width characters inside the preceding character, the
             // anchor has been moved to the right by one cell.
-            glyph.left += glyph_cache.metrics.average_advance as f32;
+            glyph.left += glyph_cache.metrics.average_advance as i16;
 
             self.add_render_item(cell, &glyph);
         }
@@ -1121,15 +1098,15 @@ fn load_glyph(
         },
         Err(AtlasInsertError::GlyphTooLarge) => Glyph {
             tex_id: atlas[*current_atlas].id,
-            colored: false,
-            top: 0.0,
-            left: 0.0,
-            width: 0.0,
-            height: 0.0,
-            uv_bot: 0.0,
-            uv_left: 0.0,
-            uv_width: 0.0,
-            uv_height: 0.0,
+            multicolor: 0,
+            top: 0,
+            left: 0,
+            width: 0,
+            height: 0,
+            uv_bot: 0.,
+            uv_left: 0.,
+            uv_width: 0.,
+            uv_height: 0.,
         },
     }
 }
@@ -1590,7 +1567,7 @@ impl Atlas {
         let offset_x = self.row_extent;
         let height = glyph.height as i32;
         let width = glyph.width as i32;
-        let colored;
+        let multicolor;
 
         unsafe {
             gl::BindTexture(gl::TEXTURE_2D, self.id);
@@ -1598,11 +1575,11 @@ impl Atlas {
             // Load data into OpenGL.
             let (format, buf) = match &glyph.buf {
                 BitmapBuffer::RGB(buf) => {
-                    colored = false;
+                    multicolor = false;
                     (gl::RGB, buf)
                 },
                 BitmapBuffer::RGBA(buf) => {
-                    colored = true;
+                    multicolor = true;
                     (gl::RGBA, buf)
                 },
             };
@@ -1637,11 +1614,11 @@ impl Atlas {
 
         Glyph {
             tex_id: self.id,
-            colored,
-            top: glyph.top as f32,
-            width: width as f32,
-            height: height as f32,
-            left: glyph.left as f32,
+            multicolor: multicolor as u8,
+            top: glyph.top as i16,
+            left: glyph.left as i16,
+            width: width as i16,
+            height: height as i16,
             uv_bot,
             uv_left,
             uv_width,