vim-patch:9.1.1243: diff mode is lacking for changes within lines

Problem:  Diff mode's inline highlighting is lackluster. It only
          performs a line-by-line comparison, and calculates a single
          shortest range within a line that could encompass all the
          changes. In lines with multiple changes, or those that span
          multiple lines, this approach tends to end up highlighting
          much more than necessary.

Solution: Implement new inline highlighting modes by doing per-character
          or per-word diff within the diff block, and highlight only the
          relevant parts, add "inline:simple" to the defaults (which is
          the old behaviour)

This change introduces a new diffopt option "inline:<type>". Setting to
"none" will disable all inline highlighting, "simple" (the default) will
use the old behavior, "char" / "word" will perform a character/word-wise
diff of the texts within each diff block and only highlight the
differences.

The new char/word inline diff only use the internal xdiff, and will
respect diff options such as algorithm choice, icase, and misc iwhite
options. indent-heuristics is always on to perform better sliding.

For character highlight, a post-process of the diff results is first
applied before we show the highlight. This is because a naive diff will
create a result with a lot of small diff chunks and gaps, due to the
repetitive nature of individual characters. The post-process is a
heuristic-based refinement that attempts to merge adjacent diff blocks
if they are separated by a short gap (1-3 characters), and can be
further tuned in the future for better results. This process results in
more characters than necessary being highlighted but overall less visual
noise.

For word highlight, always use first buffer's iskeyword definition.
Otherwise if each buffer has different iskeyword settings we would not
be able to group words properly.

The char/word diffing is always per-diff block, not per line, meaning
that changes that span multiple lines will show up correctly.
Added/removed newlines are not shown by default, but if the user has
'list' set (with "eol" listchar defined), the eol character will be be
highlighted correctly for the specific newline characters.

Also, add a new "DiffTextAdd" highlight group linked to "DiffText" by
default. It allows color schemes to use different colors for texts that
have been added within a line versus modified.

This doesn't interact with linematch perfectly currently. The linematch
feature splits up diff blocks into multiple smaller blocks for better
visual matching, which makes inline highlight less useful especially for
multi-line change (e.g. a line is broken into two lines). This could be
addressed in the future.

As a side change, this also removes the bounds checking introduced to
diff_read() as they were added to mask existing logic bugs that were
properly fixed in vim/vim#16768.

closes: vim/vim#16881

https://github.com/vim/vim/commit/9943d4790e42721a6777da9e12637aa595ba4965

Co-authored-by: Yee Cheng Chin <ychin.git@gmail.com>

4 files changed, 207 insertions, 9 deletions

diff --git a/test/old/testdir/gen_opt_test.vim b/test/old/testdir/gen_opt_test.vim
index 2127945624..fb0af3de0a 100644
--- a/test/old/testdir/gen_opt_test.vim
+++ b/test/old/testdir/gen_opt_test.vim
@@ -214,10 +214,12 @@ let test_values = {
       \		'closeoff', 'hiddenoff', 'foldcolumn:0', 'foldcolumn:12',
       \		'followwrap', 'internal', 'indent-heuristic', 'algorithm:myers',
       \		'icase,iwhite', 'algorithm:minimal', 'algorithm:patience',
-      \		'algorithm:histogram', 'linematch:5'],
+      \		'algorithm:histogram', 'inline:none', 'inline:simple',
+      \		'inline:char', 'inline:word', 'inline:char,inline:word', 'linematch:5'],
       \		['xxx', 'foldcolumn:', 'foldcolumn:x', 'foldcolumn:xxx',
       \		'linematch:', 'linematch:x', 'linematch:xxx', 'algorithm:',
-      \		'algorithm:xxx', 'context:', 'context:x', 'context:xxx']],
+      \		'algorithm:xxx', 'context:', 'context:x', 'context:xxx',
+      \		'inline:xxx']],
       \ 'display': [['', 'lastline', 'truncate', 'uhex', 'lastline,uhex'],
       \		['xxx']],
       \ 'eadirection': [['both', 'ver', 'hor'], ['xxx', 'ver,hor']],
diff --git a/test/old/testdir/setup.vim b/test/old/testdir/setup.vim
index 87287a57ff..c077eb021a 100644
--- a/test/old/testdir/setup.vim
+++ b/test/old/testdir/setup.vim
@@ -3,7 +3,7 @@ if exists('s:did_load')
   set commentstring=/*\ %s\ */
   set complete=.,w,b,u,t,i
   set define=^\\s*#\\s*define
-  set diffopt=internal,filler,closeoff
+  set diffopt=internal,filler,closeoff,inline:simple
   set directory^=.
   set display=
   set fillchars=vert:\|,foldsep:\|,fold:-
diff --git a/test/old/testdir/test_diffmode.vim b/test/old/testdir/test_diffmode.vim
index 0d20085360..b5bfef1f1d 100644
--- a/test/old/testdir/test_diffmode.vim
+++ b/test/old/testdir/test_diffmode.vim
@@ -430,13 +430,13 @@ endfunc
 
 func Common_icase_test()
   edit one
-  call setline(1, ['One', 'Two', 'Three', 'Four', 'Fi#ve'])
+  call setline(1, ['One', 'Two', 'Three', 'Four', 'Fi#vϵ', 'Si⃗x', 'Se⃗ve⃗n'])
   redraw
   let normattr = screenattr(1, 1)
   diffthis
 
   botright vert new two
-  call setline(1, ['one', 'TWO', 'Three ', 'Four', 'fI=VE'])
+  call setline(1, ['one', 'TWO', 'Three ', 'Four', 'fI=VΕ', 'SI⃗x', 'SEvE⃗n'])
   diffthis
 
   redraw
@@ -444,10 +444,13 @@ func Common_icase_test()
   call assert_equal(normattr, screenattr(2, 1))
   call assert_notequal(normattr, screenattr(3, 1))
   call assert_equal(normattr, screenattr(4, 1))
+  call assert_equal(normattr, screenattr(6, 2))
+  call assert_notequal(normattr, screenattr(7, 2))
 
   let dtextattr = screenattr(5, 3)
   call assert_notequal(dtextattr, screenattr(5, 1))
   call assert_notequal(dtextattr, screenattr(5, 5))
+  call assert_notequal(dtextattr, screenattr(7, 4))
 
   diffoff!
   %bwipe!
@@ -779,10 +782,10 @@ endfunc
 
 func Test_diff_hlID()
   new
-  call setline(1, [1, 2, 3])
+  call setline(1, [1, 2, 3, 'Yz', 'a dxxg',])
   diffthis
   vnew
-  call setline(1, ['1x', 2, 'x', 3])
+  call setline(1, ['1x', 2, 'x', 3, 'yx', 'abc defg'])
   diffthis
   redraw
 
@@ -793,6 +796,26 @@ func Test_diff_hlID()
   call diff_hlID(2, 1)->synIDattr("name")->assert_equal("")
   call diff_hlID(3, 1)->synIDattr("name")->assert_equal("DiffAdd")
   eval 4->diff_hlID(1)->synIDattr("name")->assert_equal("")
+  call diff_hlID(5, 1)->synIDattr("name")->assert_equal("DiffText")
+  call diff_hlID(5, 2)->synIDattr("name")->assert_equal("DiffText")
+
+  set diffopt+=icase " test that caching is invalidated by diffopt change
+  call diff_hlID(5, 1)->synIDattr("name")->assert_equal("DiffChange")
+  set diffopt-=icase
+  call diff_hlID(5, 1)->synIDattr("name")->assert_equal("DiffText")
+
+  call diff_hlID(6, 1)->synIDattr("name")->assert_equal("DiffChange")
+  call diff_hlID(6, 2)->synIDattr("name")->assert_equal("DiffText")
+  call diff_hlID(6, 4)->synIDattr("name")->assert_equal("DiffText")
+  call diff_hlID(6, 7)->synIDattr("name")->assert_equal("DiffText")
+  call diff_hlID(6, 8)->synIDattr("name")->assert_equal("DiffChange")
+  set diffopt+=inline:char
+  call diff_hlID(6, 1)->synIDattr("name")->assert_equal("DiffChange")
+  call diff_hlID(6, 2)->synIDattr("name")->assert_equal("DiffTextAdd")
+  call diff_hlID(6, 4)->synIDattr("name")->assert_equal("DiffChange")
+  call diff_hlID(6, 7)->synIDattr("name")->assert_equal("DiffText")
+  call diff_hlID(6, 8)->synIDattr("name")->assert_equal("DiffChange")
+  set diffopt-=inline:char
 
   wincmd w
   call assert_equal(synIDattr(diff_hlID(1, 1), "name"), "DiffChange")
@@ -2099,6 +2122,178 @@ func Test_diff_topline_noscroll()
   call StopVimInTerminal(buf)
 endfunc
 
+" Test inline highlighting which shows what's different within each diff block
+func Test_diff_inline()
+  CheckScreendump
+
+  call WriteDiffFiles(0, [], [])
+  let buf = RunVimInTerminal('-d Xdifile1 Xdifile2', {})
+  call term_sendkeys(buf, ":set autoread\<CR>\<c-w>w:set autoread\<CR>\<c-w>w")
+
+  call WriteDiffFiles(buf, ["abcdef ghi jk n", "x", "y"], ["aBcef gHi lm n", "y", "z"])
+  call VerifyInternal(buf, "Test_diff_inline_01", "")
+  call VerifyInternal(buf, "Test_diff_inline_02", " diffopt+=inline:none")
+
+  " inline:simple is the same as default
+  call VerifyInternal(buf, "Test_diff_inline_01", " diffopt+=inline:simple")
+
+  call VerifyInternal(buf, "Test_diff_inline_03", " diffopt+=inline:char")
+  call VerifyInternal(buf, "Test_diff_inline_04", " diffopt+=inline:word")
+
+  " multiple inline values will the last one
+  call VerifyInternal(buf, "Test_diff_inline_01", " diffopt+=inline:none,inline:char,inline:simple")
+  call VerifyInternal(buf, "Test_diff_inline_02", " diffopt+=inline:simple,inline:word,inline:none")
+  call VerifyInternal(buf, "Test_diff_inline_03", " diffopt+=inline:simple,inline:word,inline:char")
+
+  " DiffTextAdd highlight
+  call term_sendkeys(buf, ":hi DiffTextAdd ctermbg=blue\<CR>")
+  call VerifyInternal(buf, "Test_diff_inline_05", " diffopt+=inline:char")
+
+  " Live update in insert mode
+  call term_sendkeys(buf, "\<Esc>isometext")
+  call VerifyScreenDump(buf, "Test_diff_inline_06", {})
+  call term_sendkeys(buf, "\<Esc>u")
+
+  " icase simple scenarios
+  call VerifyInternal(buf, "Test_diff_inline_07", " diffopt+=inline:simple,icase")
+  call VerifyInternal(buf, "Test_diff_inline_08", " diffopt+=inline:char,icase")
+  call VerifyInternal(buf, "Test_diff_inline_09", " diffopt+=inline:word,icase")
+
+  " diff algorithms should affect highlight
+  call WriteDiffFiles(buf, ["apples and oranges"], ["oranges and apples"])
+  call VerifyInternal(buf, "Test_diff_inline_10", " diffopt+=inline:char")
+  call VerifyInternal(buf, "Test_diff_inline_11", " diffopt+=inline:char,algorithm:patience")
+
+  " icase: composing chars and Unicode fold case edge cases
+  call WriteDiffFiles(buf,
+        \ ["1 - sigma in 6σ and Ὀδυσσεύς", "1 - angstrom in åå", "1 - composing: ii⃗I⃗"],
+        \ ["2 - Sigma in 6Σ and ὈΔΥΣΣΕΎΣ", "2 - Angstrom in ÅÅ", "2 - Composing: i⃗I⃗I⃗"])
+  call VerifyInternal(buf, "Test_diff_inline_12", " diffopt+=inline:char")
+  call VerifyInternal(buf, "Test_diff_inline_13", " diffopt+=inline:char,icase")
+
+  " wide chars
+  call WriteDiffFiles(buf, ["abc😅xde一", "f🚀g"], ["abcy😢de", "二f🚀g"])
+  call VerifyInternal(buf, "Test_diff_inline_14", " diffopt+=inline:char,icase")
+
+  " NUL char (\n below is internally substituted as NUL)
+  call WriteDiffFiles(buf, ["1\n34\n5\n6"], ["1234\n5", "6"])
+  call VerifyInternal(buf, "Test_diff_inline_15", " diffopt+=inline:char")
+
+  " word diff: always use first buffer's iskeyword and ignore others' for consistency
+  call WriteDiffFiles(buf, ["foo+bar test"], ["foo+baz test"])
+  call VerifyInternal(buf, "Test_diff_inline_word_01", " diffopt+=inline:word")
+
+  call term_sendkeys(buf, ":set iskeyword+=+\<CR>:diffupdate\<CR>")
+  call VerifyInternal(buf, "Test_diff_inline_word_02", " diffopt+=inline:word")
+
+  call term_sendkeys(buf, ":set iskeyword&\<CR>:wincmd w\<CR>")
+  call term_sendkeys(buf, ":set iskeyword+=+\<CR>:wincmd w\<CR>:diffupdate\<CR>")
+  " Use the previous screen dump as 2nd buffer's iskeyword does not matter
+  call VerifyInternal(buf, "Test_diff_inline_word_01", " diffopt+=inline:word")
+
+  call term_sendkeys(buf, ":windo set iskeyword&\<CR>:1wincmd w\<CR>")
+
+  " char diff: should slide highlight to whitespace boundary if possible for
+  " better readability (by using forced indent-heuristics). A wrong result
+  " would be if the highlight is "Bar, prefix". It should be "prefixBar, "
+  " instead.
+  call WriteDiffFiles(buf, ["prefixFoo, prefixEnd"], ["prefixFoo, prefixBar, prefixEnd"])
+  call VerifyInternal(buf, "Test_diff_inline_char_01", " diffopt+=inline:char")
+
+  " char diff: small gaps between inline diff blocks will be merged during refine step
+  " - first segment: test that we iteratively merge small gaps after we merged
+  "   adjacent blocks, but only with limited number (set to 4) of iterations.
+  " - second and third segments: show that we need a large enough adjacent block to
+  "   trigger a merge.
+  " - fourth segment: small gaps are not merged when adjacent large block is
+  "   on a different line.
+  call WriteDiffFiles(buf,
+        \ ["abcdefghijklmno", "anchor1",
+        \  "abcdefghijklmno", "anchor2",
+        \  "abcdefghijklmno", "anchor3",
+        \  "test", "multiline"],
+        \ ["a?c?e?g?i?k???o",  "anchor1",
+        \  "a??de?????klmno",  "anchor2",
+        \  "a??de??????lmno", "anchor3",
+        \  "t?s?", "??????i?e"])
+  call VerifyInternal(buf, "Test_diff_inline_char_02", " diffopt+=inline:char")
+
+  " Test multi-line blocks and whitespace
+  call WriteDiffFiles(buf,
+        \ ["this   is   ", "sometest text foo", "baz abc def ", "one", "word another word", "additional line"],
+        \ ["this is some test", "texts", "foo bar abX Yef     ", "oneword another word"])
+  call VerifyInternal(buf, "Test_diff_inline_multiline_01", " diffopt+=inline:char,iwhite")
+  call VerifyInternal(buf, "Test_diff_inline_multiline_02", " diffopt+=inline:word,iwhite")
+  call VerifyInternal(buf, "Test_diff_inline_multiline_03", " diffopt+=inline:char,iwhiteeol")
+  call VerifyInternal(buf, "Test_diff_inline_multiline_04", " diffopt+=inline:word,iwhiteeol")
+  call VerifyInternal(buf, "Test_diff_inline_multiline_05", " diffopt+=inline:char,iwhiteall")
+  call VerifyInternal(buf, "Test_diff_inline_multiline_06", " diffopt+=inline:word,iwhiteall")
+
+  " newline should be highlighted too when 'list' is set
+  call term_sendkeys(buf, ":windo set list\<CR>")
+  call VerifyInternal(buf, "Test_diff_inline_multiline_07", " diffopt+=inline:char")
+  call VerifyInternal(buf, "Test_diff_inline_multiline_08", " diffopt+=inline:char,iwhite")
+  call VerifyInternal(buf, "Test_diff_inline_multiline_09", " diffopt+=inline:char,iwhiteeol")
+  call VerifyInternal(buf, "Test_diff_inline_multiline_10", " diffopt+=inline:char,iwhiteall")
+  call term_sendkeys(buf, ":windo set nolist\<CR>")
+
+  call StopVimInTerminal(buf)
+endfunc
+
+func Test_diff_inline_multibuffer()
+  CheckScreendump
+
+  call WriteDiffFiles3(0, [], [], [])
+  let buf = RunVimInTerminal('-d Xdifile1 Xdifile2 Xdifile3', {})
+  call term_sendkeys(buf, ":windo set autoread\<CR>:1wincmd w\<CR>")
+  call term_sendkeys(buf, ":hi DiffTextAdd ctermbg=blue\<CR>")
+
+  call WriteDiffFiles3(buf,
+        \ ["That is buffer1.", "anchor", "Some random text", "anchor"],
+        \ ["This is buffer2.", "anchor", "Some text", "anchor", "buffer2/3"],
+        \ ["This is buffer3. Last.", "anchor", "Some more", "text here.", "anchor", "only in buffer2/3", "not in buffer1"])
+  call VerifyInternal(buf, "Test_diff_inline_multibuffer_01", " diffopt+=inline:char")
+
+  " Close one of the buffer and make sure it updates correctly
+  call term_sendkeys(buf, ":diffoff\<CR>")
+  call VerifyInternal(buf, "Test_diff_inline_multibuffer_02", " diffopt+=inline:char")
+
+  " Update text in the non-diff buffer and nothing should be changed
+  call term_sendkeys(buf, "\<Esc>isometext")
+  call VerifyScreenDump(buf, "Test_diff_inline_multibuffer_03", {})
+  call term_sendkeys(buf, "\<Esc>u")
+
+  call term_sendkeys(buf, ":diffthis\<CR>")
+  call VerifyInternal(buf, "Test_diff_inline_multibuffer_01", " diffopt+=inline:char")
+
+  " Test that removing first buffer from diff will in turn use the next
+  " earliest buffer's iskeyword during word diff.
+  call WriteDiffFiles3(buf,
+        \ ["This+is=a-setence"],
+        \ ["This+is=another-setence"],
+        \ ["That+is=a-setence"])
+  call term_sendkeys(buf, ":set iskeyword+=+\<CR>:2wincmd w\<CR>:set iskeyword+=-\<CR>:1wincmd w\<CR>")
+  call VerifyInternal(buf, "Test_diff_inline_multibuffer_04", " diffopt+=inline:word")
+  call term_sendkeys(buf, ":diffoff\<CR>")
+  call VerifyInternal(buf, "Test_diff_inline_multibuffer_05", " diffopt+=inline:word")
+  call term_sendkeys(buf, ":diffthis\<CR>")
+  call VerifyInternal(buf, "Test_diff_inline_multibuffer_04", " diffopt+=inline:word")
+
+  " Test multi-buffer char diff refinement, and that removing a buffer from
+  " diff will update the others properly.
+  call WriteDiffFiles3(buf,
+        \ ["abcdefghijkYmYYY"],
+        \ ["aXXdXXghijklmnop"],
+        \ ["abcdefghijkYmYop"])
+  call VerifyInternal(buf, "Test_diff_inline_multibuffer_06", " diffopt+=inline:char")
+  call term_sendkeys(buf, ":diffoff\<CR>")
+  call VerifyInternal(buf, "Test_diff_inline_multibuffer_07", " diffopt+=inline:char")
+  call term_sendkeys(buf, ":diffthis\<CR>")
+  call VerifyInternal(buf, "Test_diff_inline_multibuffer_06", " diffopt+=inline:char")
+
+  call StopVimInTerminal(buf)
+endfunc
+
 func Test_diffget_diffput_linematch()
   CheckScreendump
   call delete('.Xdifile1.swp')
diff --git a/test/old/testdir/test_options.vim b/test/old/testdir/test_options.vim
index 9d8122cd8b..f3dcb3bbb7 100644
--- a/test/old/testdir/test_options.vim
+++ b/test/old/testdir/test_options.vim
@@ -629,10 +629,11 @@ func Test_set_completion_string_values()
   " call assert_equal([], getcompletion('set completepopup=bogusname:', 'cmdline'))
   " set previewpopup& completepopup&
 
-  " diffopt: special handling of algorithm:<alg_list>
+  " diffopt: special handling of algorithm:<alg_list> and inline:<inline_type>
   call assert_equal('filler', getcompletion('set diffopt+=', 'cmdline')[0])
   call assert_equal([], getcompletion('set diffopt+=iblank,foldcolumn:', 'cmdline'))
   call assert_equal('patience', getcompletion('set diffopt+=iblank,algorithm:pat*', 'cmdline')[0])
+  call assert_equal('char', getcompletion('set diffopt+=iwhite,inline:ch*', 'cmdline')[0])
 
   " highlight: special parsing, including auto-completing highlight groups
   " after ':'
@@ -722,7 +723,7 @@ func Test_set_completion_string_values()
   call assert_equal([], getcompletion('set diffopt-=', 'cmdline'))
   " Test all possible values
   call assert_equal(['filler', 'context:', 'iblank', 'icase', 'iwhite', 'iwhiteall', 'iwhiteeol', 'horizontal',
-        \ 'vertical', 'closeoff', 'hiddenoff', 'foldcolumn:', 'followwrap', 'internal', 'indent-heuristic', 'algorithm:', 'linematch:'],
+        \ 'vertical', 'closeoff', 'hiddenoff', 'foldcolumn:', 'followwrap', 'internal', 'indent-heuristic', 'algorithm:', 'inline:', 'linematch:'],
         \ getcompletion('set diffopt=', 'cmdline'))
   set diffopt&