feat(docs): nested lists in HTML, update :help parser

- Docs HTML: improvements in https://github.com/neovim/tree-sitter-vimdoc
  allow us to many hacks in `gen_help_html.lua`.
- Docs HTML: support nested lists.
- Docs HTML: avoid extra newlines (too much whitespace) in old
  (preformatted) layout.
- Docs HTML: disable golden-grid for narrow viewport.
- Workaround for https://github.com/neovim/neovim/issues/20404

closes https://github.com/neovim/neovim/issues/20404

1 files changed, 80 insertions, 44 deletions

diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua
index 3334dbabd2..3141ff3cdf 100644
--- a/scripts/gen_help_html.lua
+++ b/scripts/gen_help_html.lua
@@ -67,7 +67,7 @@ local exclude_invalid = {
   ["v:_null_string"] = "builtin.txt",
   ["vim.lsp.buf_request()"] = "lsp.txt",
   ["vim.lsp.util.get_progress_messages()"] = "lsp.txt",
-  ["vim.treesitter.start()"] = "treesitter.txt"
+  ["vim.treesitter.start()"] = "treesitter.txt",
 }
 
 local function tofile(fname, text)
@@ -96,11 +96,6 @@ local function url_encode(s)
     'g')
 end
 
--- Removes the ">" and "<" chars that delineate a codeblock in Vim :help files.
-local function trim_gt_lt(s)
-  return s:gsub('^%s*>%s*\n', ''):gsub('\n<', '')
-end
-
 local function expandtabs(s)
   return s:gsub('\t', (' '):rep(8))
 end
@@ -123,15 +118,29 @@ local function basename_noext(f)
 end
 
 local function is_blank(s)
-  return not not s:find('^%s*$')
+  return not not s:find([[^[\t ]*$]])
 end
 
-local function trim(s)
-  return vim.trim(s)
+local function trim(s, dir)
+  return vim.fn.trim(s, '\r\t\n ', dir or 0)
 end
 
-local function trim_bullet(s)
-  return s:gsub('^%s*[-*•]%s', '')
+-- Remove common punctuation from URLs.
+--
+-- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc
+--
+-- @returns (fixed_url, removed_chars) where `removed_chars` is in the order found in the input.
+local function fix_url(url)
+  local removed_chars = ''
+  local fixed_url = url
+  -- Remove up to one of each char from end of the URL, in this order.
+  for _, c in ipairs({ '.', ')', }) do
+    if fixed_url:sub(-1) == c then
+      removed_chars = c .. removed_chars
+      fixed_url = fixed_url:sub(1, -2)
+    end
+  end
+  return fixed_url, removed_chars
 end
 
 -- Checks if a given line is a "noise" line that doesn't look good in HTML form.
@@ -357,7 +366,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
   if node_name == 'help_file' then  -- root node
     return text
   elseif node_name == 'url' then
-    return ('%s<a href="%s">%s</a>'):format(ws(), trimmed, trimmed)
+    local fixed_url, removed_chars = fix_url(trimmed)
+    return ('%s<a href="%s">%s</a>%s'):format(ws(), fixed_url, fixed_url, removed_chars)
   elseif node_name == 'word' or node_name == 'uppercase_name' then
     return html_esc(text)
   elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then
@@ -383,38 +393,37 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       return ''
     end
     if opt.old then
-      -- XXX: Treat old docs as preformatted; random indentation is used for layout there.
-      return ('<div class="old-help-para">%s</div>\n'):format(text)
+      -- XXX: Treat "old" docs as preformatted: they use indentation for layout.
+      --      Trim trailing newlines to avoid too much whitespace between divs.
+      return ('<div class="old-help-para">%s</div>\n'):format(trim(text, 2))
     end
     return string.format('<div class="help-para">\n%s\n</div>\n', text)
   elseif node_name == 'line' then
-    local sib = root:prev_sibling()
-    local sib_last = sib and sib:named_child(sib:named_child_count() - 1)
-    local in_li = false
-
-    -- XXX: parser bug: (codeblock) without terminating "<" consumes first char of the next (line). Recover it here.
-    local recovered = (sib_last and sib_last:type() == 'codeblock') and node_text(root:prev_sibling()):sub(-1) or ''
-    recovered = recovered == '<' and '' or html_esc(recovered)
-
-    -- XXX: see if we are currently "in" a listitem.
-    while sib ~= nil and not in_li do
-      in_li = (sib:type() == 'line_li')
-      sib = sib:prev_sibling()
+    if parent ~= 'codeblock' and (is_blank(text) or is_noise(text, stats.noise_lines)) then
+      return ''  -- Discard common "noise" lines.
     end
+    -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout.
+    local div = opt.old and root:child(0) and vim.tbl_contains({'column_heading', 'h1', 'h2', 'h3'}, root:child(0):type())
+    return string.format('%s%s', div and trim(text) or text, div and '' or '\n')
+  elseif node_name == 'line_li' then
+    local sib = root:prev_sibling()
+    local prev_li = sib and sib:type() == 'line_li'
 
-    -- Close the current listitem.
-    local close = (in_li and next_ ~= 'line') and '</div>' or ''
-
-    if is_blank(text) or is_noise(text, stats.noise_lines) then
-      return close  -- Discard common "noise" lines.
+    if not prev_li then
+      opt.indent = 1
+    else
+      -- The previous listitem _sibling_ is _logically_ the _parent_ if it is indented less.
+      local parent_indent = get_indent(node_text(sib))
+      local this_indent = get_indent(node_text())
+      if this_indent > parent_indent then
+        opt.indent = opt.indent + 1
+      elseif this_indent < parent_indent then
+        opt.indent = math.max(1, opt.indent - 1)
+      end
     end
+    local margin = opt.indent == 1 and '' or ('margin-left: %drem;'):format((1.5 * opt.indent))
 
-    local div = (root:child(0) and root:child(0):type() == 'column_heading') or close ~= ''
-    return string.format('%s%s%s%s', recovered, div and trim(text) or text, div and '' or '\n', close)
-  elseif node_name == 'line_li' then
-    -- Close the listitem immediately if the next sibling is not a line.
-    local close = (next_ ~= 'line') and '</div>' or ''
-    return string.format('<div class="help-li">%s %s', trim_bullet(text), close)
+    return string.format('<div class="help-li" style="%s">%s</div>', margin, text)
   elseif node_name == 'taglink' or node_name == 'optionlink' then
     if root:has_error() then
       return text
@@ -429,7 +438,10 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
   elseif node_name == 'argument' then
     return ('%s<code>{%s}</code>'):format(ws(), text)
   elseif node_name == 'codeblock' then
-    return ('<pre>%s</pre>'):format(html_esc(trim_indent(trim_gt_lt(text))))
+    if is_blank(text) then
+      return ''
+    end
+    return ('<pre>%s</pre>'):format(html_esc(trim(trim_indent(text), 2)))
   elseif node_name == 'tag' then  -- anchor
     if root:has_error() then
       return text
@@ -630,7 +642,9 @@ local function gen_one(fname, to_fname, old, commit)
 
   local main = ''
   for _, tree in ipairs(lang_tree:trees()) do
-    main = main .. (visit_node(tree:root(), 0, tree, headings, { buf = buf, old = old, fname = fname, to_fname = to_fname }, stats))
+    main = main .. (visit_node(tree:root(), 0, tree, headings,
+      { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1, },
+      stats))
   end
 
   main = ([[
@@ -718,6 +732,17 @@ local function gen_css(fname)
         position: fixed;
         left: 67%;
       }
+      .golden-grid {
+          display: grid;
+          grid-template-columns: 65% auto;
+          grid-gap: 1em;
+      }
+    }
+    @media (max-width: 40em) {
+      .golden-grid {
+        /* Disable grid for narrow viewport (mobile phone). */
+        display: block;
+      }
     }
     @media (prefers-color-scheme: dark) {
       :root {
@@ -831,11 +856,6 @@ local function gen_css(fname)
       color: gray;
       font-size: smaller;
     }
-    .golden-grid {
-        display: grid;
-        grid-template-columns: 65% auto;
-        grid-gap: 1em;
-    }
   ]]
   tofile(fname, css)
 end
@@ -869,6 +889,22 @@ function M._test()
   eq(5, get_indent('     a\n      \n        b\n      c\n      d\n      e\n'))
   eq('a\n        \n   b\n c\n d\n e\n', trim_indent('     a\n             \n        b\n      c\n      d\n      e\n'))
 
+  local fixed_url, removed_chars = fix_url('https://example.com).')
+  eq('https://example.com', fixed_url)
+  eq(').', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com.)')
+  eq('https://example.com.', fixed_url)
+  eq(')', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com.')
+  eq('https://example.com', fixed_url)
+  eq('.', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com)')
+  eq('https://example.com', fixed_url)
+  eq(')', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com')
+  eq('https://example.com', fixed_url)
+  eq('', removed_chars)
+
   print('all tests passed')
 end