ex_getln: Make use of new parser to color expressions

Retires g:Nvim_color_expr callback.

6 files changed, 152 insertions, 19 deletions

diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c
index 54e5bcb9ff..386e9e81aa 100644
--- a/src/nvim/ex_getln.c
+++ b/src/nvim/ex_getln.c
@@ -66,6 +66,8 @@
 #include "nvim/lib/kvec.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/highlight_defs.h"
+#include "nvim/viml/parser/parser.h"
+#include "nvim/viml/parser/expressions.h"
 
 /*
  * Variables shared between getcmdline(), redrawcmdline() and others.
@@ -2341,6 +2343,62 @@ void free_cmdline_buf(void)
 
 enum { MAX_CB_ERRORS = 1 };
 
+/// Color expression cmdline using built-in expressions parser
+///
+/// @param[in]  colored_ccline  Command-line to color.
+/// @param[out]  ret_ccline_colors  What should be colored.
+///
+/// Always colors the whole cmdline.
+static void color_expr_cmdline(const CmdlineInfo *const colored_ccline,
+                               ColoredCmdline *const ret_ccline_colors)
+  FUNC_ATTR_NONNULL_ALL
+{
+  ParserLine plines[] = {
+    {
+      .data = (const char *)colored_ccline->cmdbuff,
+      .size = STRLEN(colored_ccline->cmdbuff),
+      .allocated = false,
+    },
+    { NULL, 0, false },
+  };
+  ParserLine *plines_p = plines;
+  ParserHighlight colors;
+  kvi_init(colors);
+  ParserState pstate;
+  viml_parser_init(
+      &pstate, parser_simple_get_line, &plines_p, &colors);
+  ExprAST east = viml_pexpr_parse(&pstate, kExprFlagsDisallowEOC);
+  viml_pexpr_free_ast(east);
+  viml_parser_destroy(&pstate);
+  kv_resize(ret_ccline_colors->colors, kv_size(colors));
+  size_t prev_end = 0;
+  for (size_t i = 0 ; i < kv_size(colors) ; i++) {
+    const ParserHighlightChunk chunk = kv_A(colors, i);
+    if (chunk.start.col != prev_end) {
+      kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) {
+        .start = prev_end,
+        .end = chunk.start.col,
+        .attr = 0,
+      }));
+    }
+    const int id = syn_name2id((const char_u *)chunk.group);
+    const int attr = (id == 0 ? 0 : syn_id2attr(id));
+    kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) {
+        .start = chunk.start.col,
+        .end = chunk.end_col,
+        .attr = attr,
+    }));
+    prev_end = chunk.end_col;
+  }
+  if (prev_end < (size_t)colored_ccline->cmdlen) {
+    kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) {
+      .start = prev_end,
+      .end = (size_t)colored_ccline->cmdlen,
+      .attr = 0,
+    }));
+  }
+}
+
 /// Color command-line
 ///
 /// Should use built-in command parser or user-specified one. Currently only the
@@ -2422,13 +2480,8 @@ static bool color_cmdline(const CmdlineInfo *const colored_ccline,
     tl_ret = try_leave(&tstate, &err);
     can_free_cb = true;
   } else if (colored_ccline->cmdfirstc == '=') {
-    try_enter(&tstate);
-    err_errmsg = N_(
-        "E5409: Unable to get g:Nvim_color_expr callback: %s");
-    dgc_ret = tv_dict_get_callback(&globvardict, S_LEN("Nvim_color_expr"),
-                                   &color_cb);
-    tl_ret = try_leave(&tstate, &err);
-    can_free_cb = true;
+    color_expr_cmdline(colored_ccline, ret_ccline_colors);
+    can_free_cb = false;
   }
   if (!tl_ret || !dgc_ret) {
     goto color_cmdline_error;
diff --git a/src/nvim/mbyte.c b/src/nvim/mbyte.c
index f65d7a6b13..843007b97b 100644
--- a/src/nvim/mbyte.c
+++ b/src/nvim/mbyte.c
@@ -2288,9 +2288,7 @@ int convert_setup_ext(vimconv_T *vcp, char_u *from, bool from_unicode_is_utf8,
   if (vcp->vc_type == CONV_ICONV && vcp->vc_fd != (iconv_t)-1)
     iconv_close(vcp->vc_fd);
 # endif
-  vcp->vc_type = CONV_NONE;
-  vcp->vc_factor = 1;
-  vcp->vc_fail = false;
+  *vcp = (vimconv_T)MBYTE_NONE_CONV;
 
   /* No conversion when one of the names is empty or they are equal. */
   if (from == NULL || *from == NUL || to == NULL || *to == NUL
diff --git a/src/nvim/mbyte.h b/src/nvim/mbyte.h
index fce600d0a9..a5ce1b0a15 100644
--- a/src/nvim/mbyte.h
+++ b/src/nvim/mbyte.h
@@ -60,6 +60,12 @@ typedef enum {
   CONV_ICONV     = 5,
 } ConvFlags;
 
+#define MBYTE_NONE_CONV { \
+  .vc_type = CONV_NONE, \
+  .vc_factor = 1, \
+  .vc_fail = false, \
+}
+
 /// Structure used for string conversions
 typedef struct {
   int vc_type;  ///< Zero or more ConvFlags.
diff --git a/src/nvim/viml/parser/parser.c b/src/nvim/viml/parser/parser.c
new file mode 100644
index 0000000000..08d8846018
--- /dev/null
+++ b/src/nvim/viml/parser/parser.c
@@ -0,0 +1,13 @@
+#include "nvim/viml/parser/parser.h"
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "viml/parser/parser.c.generated.h"
+#endif
+
+
+void parser_simple_get_line(void *cookie, ParserLine *ret_pline)
+{
+  ParserLine **plines_p = (ParserLine **)cookie;
+  *ret_pline = **plines_p;
+  (*plines_p)++;
+}
diff --git a/src/nvim/viml/parser/parser.h b/src/nvim/viml/parser/parser.h
index fbc5ba5f07..7ac49709d8 100644
--- a/src/nvim/viml/parser/parser.h
+++ b/src/nvim/viml/parser/parser.h
@@ -8,6 +8,7 @@
 #include "nvim/lib/kvec.h"
 #include "nvim/func_attr.h"
 #include "nvim/mbyte.h"
+#include "nvim/memory.h"
 
 /// One parsed line
 typedef struct {
@@ -80,6 +81,56 @@ typedef struct {
   bool can_continuate;
 } ParserState;
 
+static inline void viml_parser_init(
+    ParserState *const ret_pstate,
+    const ParserLineGetter get_line, void *const cookie,
+    ParserHighlight *const colors)
+  REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ARG(1, 2);
+
+/// Initialize a new parser state instance
+///
+/// @param[out]  ret_pstate  Parser state to initialize.
+/// @param[in]  get_line  Line getter function.
+/// @param[in]  cookie  Argument for the get_line function.
+/// @param[in]  colors  Where to save highlighting. May be NULL if it is not
+///                     needed.
+static inline void viml_parser_init(
+    ParserState *const ret_pstate,
+    const ParserLineGetter get_line, void *const cookie,
+    ParserHighlight *const colors)
+{
+  *ret_pstate = (ParserState) {
+    .reader = {
+      .get_line = get_line,
+      .cookie = cookie,
+      .conv = MBYTE_NONE_CONV,
+    },
+    .pos = { 0, 0 },
+    .colors = colors,
+    .can_continuate = false,
+  };
+  kvi_init(ret_pstate->reader.lines);
+  kvi_init(ret_pstate->stack);
+}
+
+static inline void viml_parser_destroy(ParserState *const pstate)
+  REAL_FATTR_NONNULL_ALL REAL_FATTR_ALWAYS_INLINE;
+
+/// Free all memory allocated by the parser on heap
+///
+/// @param  pstate  Parser state to free.
+static inline void viml_parser_destroy(ParserState *const pstate)
+{
+  for (size_t i = 0; i < kv_size(pstate->reader.lines); i++) {
+    ParserLine pline = kv_A(pstate->reader.lines, i);
+    if (pline.allocated) {
+      xfree((void *)pline.data);
+    }
+  }
+  kvi_destroy(pstate->reader.lines);
+  kvi_destroy(pstate->stack);
+}
+
 static inline void viml_preader_get_line(ParserInputReader *const preader,
                                          ParserLine *const ret_pline)
   REAL_FATTR_NONNULL_ALL;
@@ -186,4 +237,8 @@ static inline void viml_parser_highlight(ParserState *const pstate,
   }));
 }
 
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "viml/parser/parser.h.generated.h"
+#endif
+
 #endif  // NVIM_VIML_PARSER_PARSER_H
diff --git a/test/functional/ui/cmdline_highlight_spec.lua b/test/functional/ui/cmdline_highlight_spec.lua
index d87ce72599..60a4a815e7 100644
--- a/test/functional/ui/cmdline_highlight_spec.lua
+++ b/test/functional/ui/cmdline_highlight_spec.lua
@@ -144,7 +144,9 @@ before_each(function()
     EOB={bold = true, foreground = Screen.colors.Blue1},
     ERR={foreground = Screen.colors.Grey100, background = Screen.colors.Red},
     SK={foreground = Screen.colors.Blue},
-    PE={bold = true, foreground = Screen.colors.SeaGreen4}
+    PE={bold = true, foreground = Screen.colors.SeaGreen4},
+    NUM={foreground = Screen.colors.Blue2},
+    NPAR={foreground = Screen.colors.Yellow},
   })
 end)
 
@@ -863,7 +865,10 @@ describe('Ex commands coloring support', function()
 end)
 describe('Expressions coloring support', function()
   it('works', function()
-    meths.set_var('Nvim_color_expr', 'RainBowParens')
+    meths.command('hi clear NVimNumber')
+    meths.command('hi clear NVimNestingParenthesis')
+    meths.command('hi NVimNumber guifg=Blue2')
+    meths.command('hi NVimNestingParenthesis guifg=Yellow')
     feed(':echo <C-r>=(((1)))')
     screen:expect([[
                                               |
@@ -873,21 +878,24 @@ describe('Expressions coloring support', function()
       {EOB:~                                       }|
       {EOB:~                                       }|
       {EOB:~                                       }|
-      ={RBP1:(}{RBP2:(}{RBP3:(}1{RBP3:)}{RBP2:)}{RBP1:)}^                                |
+      ={NPAR:(((}{NUM:1}{NPAR:)))}^                                |
     ]])
   end)
-  it('errors out when failing to get callback', function()
+  it('does not use Nvim_color_expr', function()
     meths.set_var('Nvim_color_expr', 42)
+    -- Used to error out due to failing to get callback.
+    meths.command('hi clear NVimNumber')
+    meths.command('hi NVimNumber guifg=Blue2')
     feed(':<C-r>=1')
     screen:expect([[
+                                              |
+      {EOB:~                                       }|
+      {EOB:~                                       }|
+      {EOB:~                                       }|
       {EOB:~                                       }|
       {EOB:~                                       }|
       {EOB:~                                       }|
-      =                                       |
-      {ERR:E5409: Unable to get g:Nvim_color_expr c}|
-      {ERR:allback: Vim:E6000: Argument is not a fu}|
-      {ERR:nction or function name}                 |
-      =1^                                      |
+      ={NUM:1}^                                      |
     ]])
   end)
 end)