Inccommand: Multiline substitutions, highlighting, multibyte.

Make inccomand work with multiline patterns and substitutions. Also care
for proper highlighting and multibyte characters.

2 files changed, 193 insertions, 68 deletions

diff --git a/src/nvim/buffer.c b/src/nvim/buffer.c
index 9d42fbc1b3..3080d4960d 100644
--- a/src/nvim/buffer.c
+++ b/src/nvim/buffer.c
@@ -5278,6 +5278,43 @@ int bufhl_add_hl(buf_T *buf,
   return src_id;
 }
 
+/// Add highlighting to a buffer, bounded by two cursor positions,
+/// with an offset
+/// @param buf The buffer to add highlights to
+/// @param src_id src_id to use or 0 to use a new src_id group,
+///               or -1 for ungrouped highlight.
+/// @param hl_id Id of the highlight group to use
+/// @param lpos_start Cursor position to start the hightlighting at
+/// @param lpos_end Cursor position to end the highlighting at
+/// @param offset Move the whole highlighting this many columns to the right
+void bufhl_add_hl_pos_offset(buf_T *buf,
+                             int src_id,
+                             int hl_id,
+                             lpos_T pos_start,
+                             lpos_T pos_end,
+                             colnr_T offset)
+{
+  colnr_T hl_start = 0;
+  colnr_T hl_end = 0;
+
+  for (linenr_T lnum = pos_start.lnum; lnum <= pos_end.lnum; lnum ++) {
+    if (pos_start.lnum < lnum && lnum < pos_end.lnum) {
+      hl_start = offset;
+      hl_end = MAXCOL;
+    } else if (lnum == pos_start.lnum && lnum < pos_end.lnum) {
+      hl_start = pos_start.col + offset + 1;
+      hl_end = MAXCOL;
+    } else if (pos_start.lnum < lnum && lnum == pos_end.lnum) {
+      hl_start = offset;
+      hl_end = pos_end.col + offset;
+    } else if (pos_start.lnum == lnum && pos_end.lnum == lnum) {
+      hl_start = pos_start.col + offset + 1;
+      hl_end = pos_end.col + offset;
+    }
+    (void)bufhl_add_hl(buf, src_id, hl_id, lnum, hl_start, hl_end);
+  }
+}
+
 /// Clear bufhl highlights from a given source group and range of lines.
 ///
 /// @param buf The buffer to remove highlights from
diff --git a/src/nvim/ex_cmds.c b/src/nvim/ex_cmds.c
index 918e7a0c91..fe0c662898 100644
--- a/src/nvim/ex_cmds.c
+++ b/src/nvim/ex_cmds.c
@@ -90,14 +90,20 @@ typedef struct {
   SubIgnoreType do_ic;  ///< ignore case flag
 } subflags_T;
 
-/// Lines matched during :substitute.
+/// Partial result of a substitution during :substitute.
+/// Numbers refer to the buffer _after_ substitution
 typedef struct {
-  linenr_T lnum;
-  long nmatch;
-  char_u *line;
-  kvec_t(colnr_T) cols;  ///< columns of in-line matches
-} MatchedLine;
-typedef kvec_t(MatchedLine) MatchedLineVec;
+  lpos_T start;  // start of the match
+  lpos_T end;    // end of the match
+  linenr_T pre_match;  // where to begin showing lines before the match
+} SubResult;
+
+// Collected results of a substitution for showing them in
+// the preview window
+typedef struct {    
+  kvec_t(SubResult) subresults;
+  linenr_T lines_needed; // lines neede in the preview window
+} PreviewLines;
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "ex_cmds.c.generated.h"
@@ -3168,7 +3174,10 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout)
   linenr_T old_line_count = curbuf->b_ml.ml_line_count;
   char_u *sub_firstline;    // allocated copy of first sub line
   bool endcolumn = false;   // cursor in last column when done
-  MatchedLineVec matched_lines = KV_INITIAL_VALUE;
+  PreviewLines preview_lines = { KV_INITIAL_VALUE, 0 };
+  static int pre_src_id = 0;  // Source id for the preview highlight
+  static int pre_hl_id = 0;
+  buf_T *orig_buf = curbuf;  // save to reset highlighting
   pos_T old_cursor = curwin->w_cursor;
   int start_nsubs;
   int save_ma = 0;
@@ -3336,7 +3345,7 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout)
   linenr_T line2 = eap->line2;
   for (linenr_T lnum = eap->line1;
        lnum <= line2 && !got_quit && !aborting()
-       && (!preview || matched_lines.size < (size_t)p_cwh
+       && (!preview || preview_lines.lines_needed <= (linenr_T)p_cwh
            || lnum <= curwin->w_botline);
        lnum++) {
     long nmatch = vim_regexec_multi(&regmatch, curwin, curbuf, lnum,
@@ -3401,8 +3410,6 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout)
       sub_firstlnum = lnum;
       copycol = 0;
       matchcol = 0;
-      // the current match
-      MatchedLine matched_line = { 0, 0, NULL, KV_INITIAL_VALUE };
 
       /* At first match, remember current cursor position. */
       if (!got_match) {
@@ -3419,10 +3426,19 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout)
        * 5. break if there isn't another match in this line
        */
       for (;; ) {
+        SubResult current_match = {
+          .start = { 0, 0 },
+          .end   = { 0, 0 },
+          .pre_match = 0,
+        };
+        // lnum is where the match start, but maybe not the pattern match,
+        // since we can have \n before \zs in the pattern
+
         /* Advance "lnum" to the line where the match starts.  The
          * match does not start in the first line when there is a line
          * break before \zs. */
         if (regmatch.startpos[0].lnum > 0) {
+          current_match.pre_match = lnum;
           lnum += regmatch.startpos[0].lnum;
           sub_firstlnum += regmatch.startpos[0].lnum;
           nmatch -= regmatch.startpos[0].lnum;
@@ -3430,6 +3446,10 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout)
           sub_firstline = NULL;
         }
 
+        // Now we're at the line where the pattern match starts
+        // Note: If not first match on a line, column can't be known here
+        current_match.start.lnum = sub_firstlnum;
+
         if (sub_firstline == NULL) {
           sub_firstline = vim_strsave(ml_get(sub_firstlnum));
         }
@@ -3439,12 +3459,6 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout)
         curwin->w_cursor.lnum = lnum;
         do_again = FALSE;
 
-        if (preview) {
-          // Increment the in-line match count and store the column.
-          matched_line.nmatch++;
-          kv_push(matched_line.cols, regmatch.startpos[0].col);
-        }
-
         /*
          * 1. Match empty string does not count, except for first
          * match.  This reproduces the strange vi behaviour.
@@ -3696,8 +3710,17 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout)
           } \
         } while (0)
 
+        // Save the line numbers for the preview buffer
+        // NOTE: If the pattern matches a final newline, the next line will
+        // be shown also, but should not be highlighted. Intentional for now.
         if (preview && !has_second_delim) {
+          current_match.start.col = regmatch.startpos[0].col;
+          current_match.end.lnum = sub_firstlnum + nmatch - 1;
+          current_match.end.col  = regmatch.endpos[0].col;
+
           ADJUST_SUB_FIRSTLNUM();
+          lnum += nmatch - 1;
+
           goto skip;
         }
 
@@ -3747,6 +3770,10 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout)
           memmove(new_end, sub_firstline + copycol, (size_t)copy_len);
           new_end += copy_len;
 
+          // Finally, at this point we can know where the match actually will
+          // start in the new text
+          current_match.start.col = new_end - new_start;
+
           (void)vim_regsub_multi(&regmatch,
                                  sub_firstlnum - regmatch.startpos[0].lnum,
                                  sub, new_end, true, p_magic, true);
@@ -3799,6 +3826,8 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout)
               p1 += (*mb_ptr2len)(p1) - 1;
             }
           }
+          current_match.end.col = STRLEN(new_start);
+          current_match.end.lnum = lnum;
         }
 
         // 4. If subflags.do_all is set, find next match.
@@ -3907,9 +3936,35 @@ skip:
              * found the match. */
             if (nmatch == -1)
               lnum -= regmatch.startpos[0].lnum;
+
+            // Push the match to preview_lines
+            // TODO(KillTheMule): Code duplication at line 3961
+            linenr_T match_lines = current_match.end.lnum
+                                   - current_match.start.lnum +1;
+            if (preview_lines.subresults.size > 0) {
+              linenr_T last_lnum = kv_last(preview_lines.subresults).end.lnum;
+              if (last_lnum == current_match.start.lnum) {
+                preview_lines.lines_needed += match_lines - 1;
+              }
+            } else {
+              preview_lines.lines_needed += match_lines;
+            }
+            kv_push(preview_lines.subresults, current_match);
             break;
           }
         }
+        // Push the match to preview_lines
+        linenr_T match_lines = current_match.end.lnum
+                               - current_match.start.lnum +1;
+        if (preview_lines.subresults.size > 0) {
+          linenr_T last_lnum = kv_last(preview_lines.subresults).end.lnum;
+          if (last_lnum == current_match.start.lnum) {
+            preview_lines.lines_needed += match_lines - 1;
+          }
+        } else {
+          preview_lines.lines_needed += match_lines;
+        }
+        kv_push(preview_lines.subresults, current_match);
 
         line_breakcheck();
       }
@@ -3919,12 +3974,6 @@ skip:
       xfree(new_start);              /* for when substitute was cancelled */
       xfree(sub_firstline);          /* free the copy of the original line */
       sub_firstline = NULL;
-
-      if (preview) {
-        matched_line.lnum = lnum;
-        matched_line.line = vim_strsave(ml_get(lnum));
-        kv_push(matched_lines, matched_line);
-      }
     }
 
     line_breakcheck();
@@ -4003,18 +4052,23 @@ skip:
     if (got_quit) {  // Substitution is too slow, disable 'inccommand'.
       set_string_option_direct((char_u *)"icm", -1, (char_u *)"", OPT_FREE,
                                SID_NONE);
-    } else if (*p_icm != NUL && matched_lines.size != 0 && pat != NULL) {
+    } else if (*p_icm != NUL && preview_lines.subresults.size != 0 && pat != NULL) {
+      if (pre_src_id == 0) {
+        // Get a unique new src_id, saved in a static
+        pre_src_id = bufhl_add_hl(NULL, 0, -1, 0, 0, 0);
+      }
+      if (pre_hl_id == 0) {
+        pre_hl_id = syn_check_group((char_u *)"Substitute", 13);
+      }
       curbuf->b_changed = save_b_changed;  // preserve 'modified' during preview
-      preview_buf = show_sub(eap, old_cursor, pat, sub, &matched_lines);
+      preview_buf = show_sub(eap, old_cursor, pat, sub, &preview_lines,
+                             has_second_delim, pre_hl_id, pre_src_id);
+      bufhl_clear_line_range(orig_buf, pre_src_id, eap->line1,
+                             kv_last(preview_lines.subresults).end.lnum);
     }
   }
 
-  for (MatchedLine m; kv_size(matched_lines);) {
-    m = kv_pop(matched_lines);
-    xfree(m.line);
-    kv_destroy(m.cols);
-  }
-  kv_destroy(matched_lines);
+  kv_destroy(preview_lines.subresults);
 
   return preview_buf;
 #undef ADJUST_SUB_FIRSTLNUM
@@ -6018,7 +6072,8 @@ void set_context_in_sign_cmd(expand_T *xp, char_u *arg)
 /// Shows the effects of the :substitute command being typed ('inccommand').
 /// If inccommand=split, shows a preview window and later restores the layout.
 static buf_T *show_sub(exarg_T *eap, pos_T old_cusr, char_u *pat, char_u *sub,
-                       MatchedLineVec *matched_lines)
+                       PreviewLines *preview_lines, bool show_hl, int hl_id,
+                       int src_id)
   FUNC_ATTR_NONNULL_ALL
 {
   static handle_T bufnr = 0;  // special buffer, re-used on each visit
@@ -6028,6 +6083,8 @@ static buf_T *show_sub(exarg_T *eap, pos_T old_cusr, char_u *pat, char_u *sub,
   char_u *save_shm_p = vim_strsave(p_shm);
   size_t sub_size = mb_string2cells(sub);
   size_t pat_size = mb_string2cells(pat);
+  PreviewLines lines = *preview_lines;
+  buf_T *orig_buf = curbuf;
 
   // We keep a special-purpose buffer around, but don't assume it exists.
   buf_T *preview_buf = bufnr ? buflist_findnr(bufnr) : 0;
@@ -6046,15 +6103,19 @@ static buf_T *show_sub(exarg_T *eap, pos_T old_cusr, char_u *pat, char_u *sub,
   }
 
   // Place cursor on nearest matching line, to undo do_sub() cursor placement.
-  for (size_t i = 0; i < matched_lines->size; i++) {
-    MatchedLine curmatch = matched_lines->items[i];
-    if (curmatch.lnum >= old_cusr.lnum) {
-      curwin->w_cursor.lnum = curmatch.lnum;
-      curwin->w_cursor.col = curmatch.cols.items[0];
+  for (size_t i = 0; i < lines.subresults.size; i++) {
+    SubResult curres = lines.subresults.items[i];
+    if (curres.start.lnum >= old_cusr.lnum) {
+      curwin->w_cursor.lnum = curres.start.lnum;
+      curwin->w_cursor.col = curres.start.col;
       break;
     }  // Else: All matches are above, do_sub() already placed cursor.
   }
 
+  // Width of the "| lnum|..." column which displays the line numbers.
+  linenr_T highest_num_line = 0;
+  int col_width = 0;
+
   if (split && win_split((int)p_cwh, WSP_BOT) != FAIL) {
     buf_open_scratch(preview_buf ? bufnr : 0, "[Preview]");
     buf_clear();
@@ -6070,43 +6131,70 @@ static buf_T *show_sub(exarg_T *eap, pos_T old_cusr, char_u *pat, char_u *sub,
     curwin->w_p_fen = false;
 
     // Width of the "| lnum|..." column which displays the line numbers.
-    linenr_T highest_num_line = kv_last(*matched_lines).lnum;
-    int col_width = log10(highest_num_line) + 1 + 3;
-
-    char *str = NULL;
-    size_t old_line_size = 0;
-    size_t line_size;
-    int src_id_highlight = 0;
-    int hl_id = syn_check_group((char_u *)"Substitute", 13);
-
-    // Dump the lines into the preview buffer.
-    for (size_t line = 0; line < matched_lines->size; line++) {
-      MatchedLine mat = matched_lines->items[line];
-      line_size = mb_string2cells(mat.line) + col_width + 1;
-
-      // Reallocate if str not long enough
-      if (line_size > old_line_size) {
-        str = xrealloc(str, line_size * sizeof(char));
-        old_line_size = line_size;
+    highest_num_line = kv_last(lines.subresults).end.lnum;
+    col_width = log10(highest_num_line) + 1 + 3;
+  }
+
+  char *str = NULL;  // construct the line to show in here
+  size_t old_line_size = 0;
+  size_t line_size = 0;
+  linenr_T linenr_preview = 0;  // # of last line added to preview buffer
+  linenr_T linenr_origbuf = 0;  // # of last line added to original number
+  linenr_T next_linenr = 0;  // # of the next line to show for the match
+
+  for (size_t matchidx = 0; matchidx < lines.subresults.size; matchidx++) {
+    SubResult match = lines.subresults.items[matchidx];
+
+    if (split && preview_buf) {
+      lpos_T p_start = { 0, match.start.col };  // match starts here in preview
+      lpos_T p_end   = { 0, match.end.col };    // ... and ends here
+
+      if (match.pre_match == 0) {
+        next_linenr = match.start.lnum;
+      } else {
+        next_linenr = match.pre_match;
       }
+      // Don't add a line twice
+      if (next_linenr == linenr_origbuf) {
+        next_linenr++;
+        p_start.lnum = linenr_preview;  // might be redefined below
+        p_end.lnum = linenr_preview;  // might be redefined below
+      }
+
+      for (; next_linenr <= match.end.lnum; next_linenr++) {
+        if (next_linenr == match.start.lnum) {
+          p_start.lnum = linenr_preview + 1;
+        }
+        if (next_linenr == match.end.lnum) {
+          p_end.lnum = linenr_preview + 1;
+        }
+        char_u *line = ml_get_buf(orig_buf, next_linenr, false);
+        line_size = STRLEN(line) + col_width + 1;
 
-      // Put "|lnum| line" into `str` and append it to the preview buffer.
-      snprintf(str, line_size, "|%*ld| %s", col_width - 3, mat.lnum, mat.line);
-      ml_append(line, (char_u *)str, (colnr_T)line_size, false);
-
-      // highlight the replaced part
-      if (sub_size > 0) {
-        for (size_t i = 0; i < mat.cols.size; i++) {
-          colnr_T col_start = mat.cols.items[i] + col_width
-            + i * (sub_size - pat_size) + 1;
-          colnr_T col_end = col_start - 1 + sub_size;
-          src_id_highlight = bufhl_add_hl(curbuf, src_id_highlight, hl_id,
-                                          line + 1, col_start, col_end);
+        // Reallocate if line not long enough
+        if (line_size > old_line_size) {
+          str = xrealloc(str, line_size * sizeof(char));
+          old_line_size = line_size;
         }
+        // Put "|lnum| line" into `str` and append it to the preview buffer.
+        snprintf(str, line_size, "|%*ld| %s", col_width - 3,
+                 next_linenr, line);
+        ml_append(linenr_preview, (char_u *)str, (colnr_T)line_size, false);
+        linenr_preview += 1;
+      }
+      linenr_origbuf = match.end.lnum;
+
+      if (show_hl) {
+        bufhl_add_hl_pos_offset(preview_buf, src_id, hl_id, p_start,
+                                p_end, col_width);
       }
     }
-    xfree(str);
+    if (show_hl) {
+      bufhl_add_hl_pos_offset(orig_buf, src_id, hl_id, match.start,
+                              match.end, 0);
+    }
   }
+  xfree(str);
 
   redraw_later(SOME_VALID);
   win_enter(save_curwin, false);  // Return to original window