ci: check URL reachability #35593

Problem:
scripts/check_urls.vim manually matches urls in the help pages and then
synchronously checks them via curl/wget/powershell. This is extremely
slow (~5 minutes for Nvims runtime on my machine) and prone to errors in
how the urls are matched.

Solution:
- Use Tree-sitter to find the urls in the help pages and `vim.net.request` to
  check the responses.
- Add a `lintdocurls` build task and check it in CI (every Friday).
  - Reopens a dedicated issue if it finds unreachable URLs.
- Drop the old check_urls.vim script.

1 files changed, 42 insertions, 15 deletions

diff --git a/src/gen/gen_help_html.lua b/src/gen/gen_help_html.lua
index 08ab301eed..96f142ecc3 100644
--- a/src/gen/gen_help_html.lua
+++ b/src/gen/gen_help_html.lua
@@ -15,6 +15,8 @@
 -- USAGE (VALIDATE):
 --   1. nvim -V1 -es +"lua require('src.gen.gen_help_html').validate('./runtime/doc')" +q
 --      - validate() is 10x faster than gen(), so it is used in CI.
+--   2. Check for unreachable URLs:
+--      nvim -V1 -es +"lua require('src.gen.gen_help_html').validate('./runtime/doc', true)" +q
 --
 -- SELF-TEST MODE:
 --   1. nvim -V1 -es +"lua require('src.gen.gen_help_html')._test()" +q
@@ -28,6 +30,7 @@
 --   * visit_validate() is the core function used by validate().
 --   * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout.
 
+local pending_urls = 0
 local tagmap = nil ---@type table<string, string>
 local helpfiles = nil ---@type string[]
 local invalid_links = {} ---@type table<string, any>
@@ -383,15 +386,26 @@ local function validate_link(node, bufnr, fname)
   return helppage, tagname, ignored
 end
 
---- TODO: port the logic from scripts/check_urls.vim
-local function validate_url(text, fname)
-  local ignored = false
-  if ignore_errors[vim.fs.basename(fname)] then
-    ignored = true
-  elseif text:find('http%:') and not exclude_invalid_urls[text] then
-    invalid_urls[text] = vim.fs.basename(fname)
+local function validate_url(text, fname, check_unreachable)
+  fname = vim.fs.basename(fname)
+  local ignored = ignore_errors[fname] or exclude_invalid_urls[text]
+  if ignored then
+    return true
   end
-  return ignored
+  if check_unreachable then
+    vim.net.request(text, { retry = 2 }, function(err, _)
+      if err then
+        invalid_urls[text] = fname
+      end
+      pending_urls = pending_urls - 1
+    end)
+    pending_urls = pending_urls + 1
+  else
+    if text:find('http%:') then
+      invalid_urls[text] = fname
+    end
+  end
+  return false
 end
 
 --- Traverses the tree at `root` and checks that |tag| links point to valid helptags.
@@ -449,7 +463,7 @@ local function visit_validate(root, level, lang_tree, opt, stats)
     end
   elseif node_name == 'url' then
     local fixed_url, _ = fix_url(trim(text))
-    validate_url(fixed_url, opt.fname)
+    validate_url(fixed_url, opt.fname, opt.request_urls)
   elseif node_name == 'taglink' or node_name == 'optionlink' then
     local _, _, _ = validate_link(root, opt.buf, opt.fname)
   end
@@ -811,14 +825,19 @@ end
 ---
 --- @param fname string help file to validate
 --- @param parser_path string? path to non-default vimdoc.so
+--- @param request_urls boolean? whether to make requests to the URLs
 --- @return { invalid_links: number, parse_errors: string[] }
-local function validate_one(fname, parser_path)
+local function validate_one(fname, parser_path, request_urls)
   local stats = {
     parse_errors = {},
   }
   local lang_tree, buf = parse_buf(fname, nil, parser_path)
   for _, tree in ipairs(lang_tree:trees()) do
-    visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname }, stats)
+    visit_validate(tree:root(), 0, tree, {
+      buf = buf,
+      fname = fname,
+      request_urls = request_urls,
+    }, stats)
   end
   lang_tree:destroy()
   vim.cmd.close()
@@ -1484,7 +1503,7 @@ end
 --- This is 10x faster than gen(), for use in CI.
 ---
 --- @return nvim.gen_help_html.validate_result result
-function M.validate(help_dir, include, parser_path)
+function M.validate(help_dir, include, parser_path, request_urls)
   vim.validate('help_dir', help_dir, function(d)
     return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
   end, 'valid directory')
@@ -1501,7 +1520,7 @@ function M.validate(help_dir, include, parser_path)
 
   for _, f in ipairs(helpfiles) do
     local helpfile = vim.fs.basename(f)
-    local rv = validate_one(f, parser_path)
+    local rv = validate_one(f, parser_path, request_urls)
     print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile))
     if #rv.parse_errors > 0 then
       files_to_errors[helpfile] = rv.parse_errors
@@ -1512,6 +1531,13 @@ function M.validate(help_dir, include, parser_path)
     err_count = err_count + #rv.parse_errors
   end
 
+  -- Requests are async, wait for them to finish.
+  -- TODO(yochem): `:cancel()` tasks after #36146
+  vim.wait(20000, function()
+    return pending_urls <= 0
+  end)
+  ok(pending_urls <= 0, 'pending url checks', pending_urls)
+
   ---@type nvim.gen_help_html.validate_result
   return {
     helpfiles = #helpfiles,
@@ -1531,11 +1557,12 @@ end
 --- 3. File a parser bug, and adjust the tolerance of this test in the meantime.
 ---
 --- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
-function M.run_validate(help_dir)
+--- @param request_urls? boolean make network requests to check if the URLs are reachable.
+function M.run_validate(help_dir, request_urls)
   help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
   print('doc path = ' .. vim.uv.fs_realpath(help_dir))
 
-  local rv = M.validate(help_dir)
+  local rv = M.validate(help_dir, nil, nil, request_urls)
 
   -- Check that we actually found helpfiles.
   ok(rv.helpfiles > 100, '>100 :help files', rv.helpfiles)