refactor(grid): do arabic shaping in one place

The 'arabicshape' feature of vim is a transformation of unicode text to
make arabic and some related scripts look better at display time. In
particular the content of a cell will be adjusted depending on the
(original) content of the cells just before and after it.

This is implemented by the arabic_shape() function in nvim. Before this
commit, shaping was invoked in four different contexts:

- when rendering buffer text in win_line()
- in line_putchar() for rendering virtual text
- as part of grid_line_puts, used by messages and statuslines and
  similar
- as part of draw_cmdline() for drawing the cmdline

This replaces all these with a post-processing step in grid_put_linebuf(),
which has become the entry point for all text rendering after recent
refactors.

An aim of this is to make the handling of multibyte text yet simpler.
One of the main reasons multibyte chars needs to be "parsed" into
codepoint arrays of composing chars is so that these could be inspected
for the purpose of shaping. This can likely be vastly simplified in many
contexts where only the total length (in bytes) and width of composed
char is needed.

1 files changed, 11 insertions, 70 deletions

diff --git a/src/nvim/drawline.c b/src/nvim/drawline.c
index e5d2ba7550..2d450087fd 100644
--- a/src/nvim/drawline.c
+++ b/src/nvim/drawline.c
@@ -12,7 +12,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "nvim/arabic.h"
 #include "nvim/ascii.h"
 #include "nvim/buffer.h"
 #include "nvim/charset.h"
@@ -141,15 +140,6 @@ typedef struct {
                              ///< to be added to wlv.vcol later
 } winlinevars_T;
 
-/// for line_putchar. Contains the state that needs to be remembered from
-/// putting one character to the next.
-typedef struct {
-  const char *p;
-  int prev_c;   ///< previous Arabic character
-  int prev_c1;  ///< first composing char for prev_c
-} LineState;
-#define LINE_STATE(p) { p, 0, 0 }
-
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "drawline.c.generated.h"
 #endif
@@ -220,10 +210,10 @@ static void line_check_overwrite(schar_T *dest, int cells, int maxcells, bool rl
 
 /// Put a single char from an UTF-8 buffer into a line buffer.
 ///
-/// Handles composing chars and arabic shaping state.
-static int line_putchar(buf_T *buf, LineState *s, schar_T *dest, int maxcells, bool rl, int vcol)
+/// Handles composing chars
+static int line_putchar(buf_T *buf, const char **pp, schar_T *dest, int maxcells, bool rl, int vcol)
 {
-  const char *p = s->p;
+  const char *p = *pp;
   int cells = utf_ptr2cells(p);
   int c_len = utfc_ptr2len(p);
   int u8c, u8cc[MAX_MCO];
@@ -244,39 +234,14 @@ static int line_putchar(buf_T *buf, LineState *s, schar_T *dest, int maxcells, b
     goto done;
   } else if ((uint8_t)(*p) < 0x80 && u8cc[0] == 0) {
     dest[0] = schar_from_ascii(*p);
-    s->prev_c = u8c;
   } else {
-    if (p_arshape && !p_tbidi && ARABIC_CHAR(u8c)) {
-      // Do Arabic shaping.
-      int pc, pc1, nc;
-      int pcc[MAX_MCO];
-      int firstbyte = (uint8_t)(*p);
-
-      // The idea of what is the previous and next
-      // character depends on 'rightleft'.
-      if (rl) {
-        pc = s->prev_c;
-        pc1 = s->prev_c1;
-        nc = utf_ptr2char(p + c_len);
-        s->prev_c1 = u8cc[0];
-      } else {
-        pc = utfc_ptr2char(p + c_len, pcc);
-        nc = s->prev_c;
-        pc1 = pcc[0];
-      }
-      s->prev_c = u8c;
-
-      u8c = arabic_shape(u8c, &firstbyte, &u8cc[0], pc, pc1, nc);
-    } else {
-      s->prev_c = u8c;
-    }
     dest[0] = schar_from_cc(u8c, u8cc);
   }
   if (cells > 1) {
     dest[rl ? -1 : 1] = 0;
   }
 done:
-  s->p += c_len;
+  *pp += c_len;
   return cells;
 }
 
@@ -344,22 +309,22 @@ static void draw_virt_text(win_T *wp, buf_T *buf, int col_off, int *end_col, int
 static int draw_virt_text_item(buf_T *buf, int col, VirtText vt, HlMode hl_mode, int max_col,
                                int vcol, bool rl)
 {
-  LineState s = LINE_STATE("");
+  const char *p = "";
   int virt_attr = 0;
   size_t virt_pos = 0;
 
   while (rl ? col > max_col : col < max_col) {
-    if (*s.p == NUL) {
+    if (!*p) {
       if (virt_pos >= kv_size(vt)) {
         break;
       }
       virt_attr = 0;
-      s.p = next_virt_text_chunk(vt, &virt_pos, &virt_attr);
-      if (s.p == NULL) {
+      p = next_virt_text_chunk(vt, &virt_pos, &virt_attr);
+      if (p == NULL) {
         break;
       }
     }
-    if (*s.p == NUL) {
+    if (*p == NUL) {
       continue;
     }
     int attr;
@@ -367,14 +332,14 @@ static int draw_virt_text_item(buf_T *buf, int col, VirtText vt, HlMode hl_mode,
     if (hl_mode == kHlModeCombine) {
       attr = hl_combine_attr(linebuf_attr[col], virt_attr);
     } else if (hl_mode == kHlModeBlend) {
-      through = (*s.p == ' ');
+      through = (*p == ' ');
       attr = hl_blend_attrs(linebuf_attr[col], virt_attr, &through);
     } else {
       attr = virt_attr;
     }
     schar_T dummy[2];
     int maxcells = rl ? col - max_col : max_col - col;
-    int cells = line_putchar(buf, &s, through ? dummy : &linebuf_char[col],
+    int cells = line_putchar(buf, &p, through ? dummy : &linebuf_char[col],
                              maxcells, rl, vcol);
     // If we failed to emit a char, we still need to put a space and advance.
     if (cells < 1) {
@@ -1171,8 +1136,6 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
   int multispace_pos = 0;               // position in lcs-multispace string
   int line_attr_save;
   int line_attr_lowprio_save;
-  int prev_c = 0;                       // previous Arabic character
-  int prev_c1 = 0;                      // first composing char for prev_c
 
   bool search_attr_from_match = false;  // if search_attr is from :match
   bool has_decor = false;               // this buffer has decoration
@@ -2160,28 +2123,6 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
         }
       } else if (mb_l == 0) {        // at the NUL at end-of-line
         mb_l = 1;
-      } else if (p_arshape && !p_tbidi && ARABIC_CHAR(mb_c)) {
-        // Do Arabic shaping.
-        int pc, pc1, nc;
-        int pcc[MAX_MCO];
-
-        // The idea of what is the previous and next
-        // character depends on 'rightleft'.
-        if (wp->w_p_rl) {
-          pc = prev_c;
-          pc1 = prev_c1;
-          nc = utf_ptr2char(ptr + mb_l);
-          prev_c1 = u8cc[0];
-        } else {
-          pc = utfc_ptr2char(ptr + mb_l, pcc);
-          nc = prev_c;
-          pc1 = pcc[0];
-        }
-        prev_c = mb_c;
-
-        mb_c = arabic_shape(mb_c, &c, &u8cc[0], pc, pc1, nc);
-      } else {
-        prev_c = mb_c;
       }
       // If a double-width char doesn't fit display a '>' in the
       // last column; the character is displayed at the start of the