Merge #7623 'man.vim: highlight bold, underlined text'

6 files changed, 314 insertions, 8 deletions

diff --git a/runtime/autoload/man.vim b/runtime/autoload/man.vim
index af5c4dbd60..00ce1c77d7 100644
--- a/runtime/autoload/man.vim
+++ b/runtime/autoload/man.vim
@@ -148,7 +148,8 @@ function! s:get_page(path) abort
   let manwidth = empty($MANWIDTH) ? winwidth(0) : $MANWIDTH
   " Force MANPAGER=cat to ensure Vim is not recursively invoked (by man-db).
   " http://comments.gmane.org/gmane.editors.vim.devel/29085
-  let cmd = ['env', 'MANPAGER=cat', 'MANWIDTH='.manwidth, 'man']
+  " Set MAN_KEEP_FORMATTING so Debian man doesn't discard backspaces.
+  let cmd = ['env', 'MANPAGER=cat', 'MANWIDTH='.manwidth, 'MAN_KEEP_FORMATTING=1', 'man']
   return s:system(cmd + (s:localfile_arg ? ['-l', a:path] : [a:path]))
 endfunction
 
@@ -157,11 +158,10 @@ function! s:put_page(page) abort
   setlocal noreadonly
   silent keepjumps %delete _
   silent put =a:page
-  " Remove all backspaced/escape characters.
-  execute 'silent keeppatterns keepjumps %substitute,.\b\|\e\[\d\+m,,e'.(&gdefault?'':'g')
   while getline(1) =~# '^\s*$'
     silent keepjumps 1delete _
   endwhile
+  lua require("man").highlight_man_page()
   setlocal filetype=man
 endfunction
 
@@ -370,13 +370,12 @@ function! s:format_candidate(path, psect) abort
 endfunction
 
 function! man#init_pager() abort
-  " Remove all backspaced/escape characters.
-  execute 'silent keeppatterns keepjumps %substitute,.\b\|\e\[\d\+m,,e'.(&gdefault?'':'g')
   if getline(1) =~# '^\s*$'
     silent keepjumps 1delete _
   else
     keepjumps 1
   endif
+  lua require("man").highlight_man_page()
   " This is not perfect. See `man glDrawArraysInstanced`. Since the title is
   " all caps it is impossible to tell what the original capitilization was.
   let ref = substitute(matchstr(getline(1), '^[^)]\+)'), ' ', '_', 'g')
diff --git a/runtime/lua/man.lua b/runtime/lua/man.lua
new file mode 100644
index 0000000000..baa522f343
--- /dev/null
+++ b/runtime/lua/man.lua
@@ -0,0 +1,168 @@
+local buf_hls = {}
+
+local function highlight_line(line, linenr)
+  local chars = {}
+  local prev_char = ''
+  local overstrike, escape = false, false
+  local hls = {} -- Store highlight groups as { attr, start, final }
+  local NONE, BOLD, UNDERLINE, ITALIC = 0, 1, 2, 3
+  local hl_groups = {[BOLD]="manBold", [UNDERLINE]="manUnderline", [ITALIC]="manItalic"}
+  local attr = NONE
+  local byte = 0 -- byte offset
+
+  local function end_attr_hl(attr)
+    for i, hl in ipairs(hls) do
+      if hl.attr == attr and hl.final == -1 then
+        hl.final = byte
+        hls[i] = hl
+      end
+    end
+  end
+
+  local function add_attr_hl(code)
+    local continue_hl = true
+    if code == 0 then
+      attr = NONE
+      continue_hl = false
+    elseif code == 1 then
+      attr = BOLD
+    elseif code == 22 then
+      attr = BOLD
+      continue_hl = false
+    elseif code == 3 then
+      attr = ITALIC
+    elseif code == 23 then
+      attr = ITALIC
+      continue_hl = false
+    elseif code == 4 then
+      attr = UNDERLINE
+    elseif code == 24 then
+      attr = UNDERLINE
+      continue_hl = false
+    else
+      attr = NONE
+      return
+    end
+
+    if continue_hl then
+      hls[#hls + 1] = {attr=attr, start=byte, final=-1}
+    else
+      if attr == NONE then
+        for a, _ in pairs(hl_groups) do
+          end_attr_hl(a)
+        end
+      else
+        end_attr_hl(attr)
+      end
+    end
+  end
+
+  -- Break input into UTF8 code points. ASCII code points (from 0x00 to 0x7f)
+  -- can be represented in one byte. Any code point above that is represented by
+  -- a leading byte (0xc0 and above) and continuation bytes (0x80 to 0xbf, or
+  -- decimal 128 to 191).
+  for char in line:gmatch("[^\128-\191][\128-\191]*") do
+    if overstrike then
+      local last_hl = hls[#hls]
+      if char == prev_char then
+        if char == '_' and attr == UNDERLINE and last_hl and last_hl.final == byte then
+          -- This underscore is in the middle of an underlined word
+          attr = UNDERLINE
+        else
+          attr = BOLD
+        end
+      elseif prev_char == '_' then
+        -- char is underlined
+        attr = UNDERLINE
+      elseif prev_char == '+' and char == 'o' then
+        -- bullet (overstrike text '+^Ho')
+        attr = BOLD
+        char = '·'
+      elseif prev_char == '·' and char == 'o' then
+        -- bullet (additional handling for '+^H+^Ho^Ho')
+        attr = BOLD
+        char = '·'
+      else
+        -- use plain char
+        attr = NONE
+      end
+
+      -- Grow the previous highlight group if possible
+      if last_hl and last_hl.attr == attr and last_hl.final == byte then
+        last_hl.final = byte + #char
+      else
+        hls[#hls + 1] = {attr=attr, start=byte, final=byte + #char}
+      end
+
+      overstrike = false
+      prev_char = ''
+      byte = byte + #char
+      chars[#chars + 1] = char
+    elseif escape then
+      -- Use prev_char to store the escape sequence
+      prev_char = prev_char .. char
+      -- We only want to match against SGR sequences, which consist of ESC
+      -- followed by '[', then a series of parameter and intermediate bytes in
+      -- the range 0x20 - 0x3f, then 'm'. (See ECMA-48, sections 5.4 & 8.3.117)
+      local sgr = prev_char:match("^%[([\032-\063]*)m$")
+      if sgr then
+        local match = ''
+        while sgr and #sgr > 0 do
+          -- Match against SGR parameters, which may be separated by ';'
+          match, sgr = sgr:match("^(%d*);?(.*)")
+          add_attr_hl(match + 0) -- coerce to number
+        end
+        escape = false
+      elseif not prev_char:match("^%[[\032-\063]*$") then
+        -- Stop looking if this isn't a partial CSI sequence
+        escape = false
+      end
+    elseif char == "\027" then
+      escape = true
+      prev_char = ''
+    elseif char == "\b" then
+      overstrike = true
+      prev_char = chars[#chars]
+      byte = byte - #prev_char
+      chars[#chars] = nil
+    else
+      byte = byte + #char
+      chars[#chars + 1] = char
+    end
+  end
+
+  for _, hl in ipairs(hls) do
+    if hl.attr ~= NONE then
+      buf_hls[#buf_hls + 1] = {
+        0,
+        -1,
+        hl_groups[hl.attr],
+        linenr - 1,
+        hl.start,
+        hl.final
+      }
+    end
+  end
+
+  return table.concat(chars, '')
+end
+
+local function highlight_man_page()
+  local mod = vim.api.nvim_eval("&modifiable")
+  vim.api.nvim_command("set modifiable")
+
+  local lines = vim.api.nvim_buf_get_lines(0, 0, -1, false)
+  for i, line in ipairs(lines) do
+    lines[i] = highlight_line(line, i)
+  end
+  vim.api.nvim_buf_set_lines(0, 0, -1, false, lines)
+
+  for _, args in ipairs(buf_hls) do
+    vim.api.nvim_buf_add_highlight(unpack(args))
+  end
+  buf_hls = {}
+
+  vim.api.nvim_command("let &modifiable = "..mod)
+end
+
+return { highlight_man_page = highlight_man_page }
diff --git a/runtime/syntax/man.vim b/runtime/syntax/man.vim
index 0975b160ae..b8e605cb9a 100644
--- a/runtime/syntax/man.vim
+++ b/runtime/syntax/man.vim
@@ -18,6 +18,10 @@ highlight default link manOptionDesc     Constant
 highlight default link manReference      PreProc
 highlight default link manSubHeading     Function
 
+highlight default manUnderline cterm=underline gui=underline
+highlight default manBold      cterm=bold      gui=bold
+highlight default manItalic    cterm=italic    gui=italic
+
 if &filetype != 'man'
   " May have been included by some other filetype.
   finish
diff --git a/test/functional/plugin/man_spec.lua b/test/functional/plugin/man_spec.lua
new file mode 100644
index 0000000000..dc189b8f8e
--- /dev/null
+++ b/test/functional/plugin/man_spec.lua
@@ -0,0 +1,135 @@
+local helpers = require('test.functional.helpers')(after_each)
+local plugin_helpers = require('test.functional.plugin.helpers')
+
+local Screen = require('test.functional.ui.screen')
+
+local command, eval, rawfeed = helpers.command, helpers.eval, helpers.rawfeed
+
+before_each(function()
+  plugin_helpers.reset()
+  helpers.clear()
+  command('syntax on')
+  command('set filetype=man')
+end)
+
+describe(':Man', function()
+  describe('man.lua: highlight_line()', function()
+    local screen
+
+    before_each(function()
+      command('syntax off') -- Ignore syntax groups
+      screen = Screen.new(52, 5)
+      screen:set_default_attr_ids({
+        b = { bold = true },
+        i = { italic = true },
+        u = { underline = true },
+        bi = { bold = true, italic = true },
+        biu = { bold = true, italic = true, underline = true },
+      })
+      screen:set_default_attr_ignore({
+        { foreground = Screen.colors.Blue }, -- control chars
+        { bold = true, foreground = Screen.colors.Blue } -- empty line '~'s
+      })
+      screen:attach()
+    end)
+
+    after_each(function()
+      screen:detach()
+    end)
+
+    it('clears backspaces from text and adds highlights', function()
+      rawfeed([[
+        ithis i<C-v><C-h>is<C-v><C-h>s a<C-v><C-h>a test
+        with _<C-v><C-h>o_<C-v><C-h>v_<C-v><C-h>e_<C-v><C-h>r_<C-v><C-h>s_<C-v><C-h>t_<C-v><C-h>r_<C-v><C-h>u_<C-v><C-h>c_<C-v><C-h>k text<ESC>]])
+
+      screen:expect([[
+      this i^His^Hs a^Ha test                             |
+      with _^Ho_^Hv_^He_^Hr_^Hs_^Ht_^Hr_^Hu_^Hc_^Hk tex^t  |
+      ~                                                   |
+      ~                                                   |
+                                                          |
+      ]])
+
+      eval('man#init_pager()')
+
+      screen:expect([[
+      ^this {b:is} {b:a} test                                      |
+      with {u:overstruck} text                                |
+      ~                                                   |
+      ~                                                   |
+                                                          |
+      ]])
+    end)
+
+    it('clears escape sequences from text and adds highlights', function()
+      rawfeed([[
+        ithis <C-v><ESC>[1mis <C-v><ESC>[3ma <C-v><ESC>[4mtest<C-v><ESC>[0m
+        <C-v><ESC>[4mwith<C-v><ESC>[24m <C-v><ESC>[4mescaped<C-v><ESC>[24m <C-v><ESC>[4mtext<C-v><ESC>[24m<ESC>]])
+
+      screen:expect([[
+      this ^[[1mis ^[[3ma ^[[4mtest^[[0m                  |
+      ^[[4mwith^[[24m ^[[4mescaped^[[24m ^[[4mtext^[[24^m  |
+      ~                                                   |
+      ~                                                   |
+                                                          |
+      ]])
+
+      eval('man#init_pager()')
+
+      screen:expect([[
+      ^this {b:is }{bi:a }{biu:test}                                      |
+      {u:with} {u:escaped} {u:text}                                   |
+      ~                                                   |
+      ~                                                   |
+                                                          |
+      ]])
+    end)
+
+    it('highlights multibyte text', function()
+      rawfeed([[
+        ithis i<C-v><C-h>is<C-v><C-h>s あ<C-v><C-h>あ test
+        with _<C-v><C-h>ö_<C-v><C-h>v_<C-v><C-h>e_<C-v><C-h>r_<C-v><C-h>s_<C-v><C-h>t_<C-v><C-h>r_<C-v><C-h>u_<C-v><C-h>̃_<C-v><C-h>c_<C-v><C-h>k te<C-v><ESC>[3mxt¶<C-v><ESC>[0m<ESC>]])
+      eval('man#init_pager()')
+
+      screen:expect([[
+      ^this {b:is} {b:あ} test                                     |
+      with {u:överstrũck} te{i:xt¶}                               |
+      ~                                                   |
+      ~                                                   |
+                                                          |
+      ]])
+    end)
+
+    it('highlights underscores based on context', function()
+      rawfeed([[
+        i_<C-v><C-h>_b<C-v><C-h>be<C-v><C-h>eg<C-v><C-h>gi<C-v><C-h>in<C-v><C-h>ns<C-v><C-h>s
+        m<C-v><C-h>mi<C-v><C-h>id<C-v><C-h>d_<C-v><C-h>_d<C-v><C-h>dl<C-v><C-h>le<C-v><C-h>e
+        _<C-v><C-h>m_<C-v><C-h>i_<C-v><C-h>d_<C-v><C-h>__<C-v><C-h>d_<C-v><C-h>l_<C-v><C-h>e<ESC>]])
+      eval('man#init_pager()')
+
+      screen:expect([[
+      {b:^_begins}                                             |
+      {b:mid_dle}                                             |
+      {u:mid_dle}                                             |
+      ~                                                   |
+                                                          |
+      ]])
+    end)
+
+    it('highlights various bullet formats', function()
+      rawfeed([[
+        i· ·<C-v><C-h>·
+        +<C-v><C-h>o
+        +<C-v><C-h>+<C-v><C-h>o<C-v><C-h>o double<ESC>]])
+      eval('man#init_pager()')
+
+      screen:expect([[
+      ^· {b:·}                                                 |
+      {b:·}                                                   |
+      {b:·} double                                            |
+      ~                                                   |
+                                                          |
+      ]])
+    end)
+  end)
+end)
diff --git a/test/functional/plugin/msgpack_spec.lua b/test/functional/plugin/msgpack_spec.lua
index 5ba19708cf..4b014cbc73 100644
--- a/test/functional/plugin/msgpack_spec.lua
+++ b/test/functional/plugin/msgpack_spec.lua
@@ -8,7 +8,7 @@ local NIL = helpers.NIL
 local plugin_helpers = require('test.functional.plugin.helpers')
 local reset = plugin_helpers.reset
 
-describe('In autoload/msgpack.vim', function()
+describe('autoload/msgpack.vim', function()
   before_each(reset)
 
   local sp = function(typ, val)
diff --git a/test/functional/plugin/shada_spec.lua b/test/functional/plugin/shada_spec.lua
index 57891a8229..5a064a759f 100644
--- a/test/functional/plugin/shada_spec.lua
+++ b/test/functional/plugin/shada_spec.lua
@@ -43,7 +43,7 @@ local wshada, _, fname = get_shada_rw('Xtest-functional-plugin-shada.shada')
 local wshada_tmp, _, fname_tmp =
   get_shada_rw('Xtest-functional-plugin-shada.shada.tmp.f')
 
-describe('In autoload/shada.vim', function()
+describe('autoload/shada.vim', function()
   local epoch = os.date('%Y-%m-%dT%H:%M:%S', 0)
   before_each(function()
     reset()
@@ -2136,7 +2136,7 @@ describe('In autoload/shada.vim', function()
   end)
 end)
 
-describe('In plugin/shada.vim', function()
+describe('plugin/shada.vim', function()
   local epoch = os.date('%Y-%m-%dT%H:%M:%S', 0)
   local eol = helpers.iswin() and '\r\n' or '\n'
   before_each(function()