Merge remote-tracking branch 'upstream/master' into mix_20240309

15 files changed, 196 insertions, 633 deletions

diff --git a/scripts/check_urls.vim b/scripts/check_urls.vim
index 3580b79475..b75dc29c48 100644
--- a/scripts/check_urls.vim
+++ b/scripts/check_urls.vim
@@ -6,21 +6,41 @@
 " Written by Christian Brabandt.
 
 func Test_check_URLs()
+"20.10.23, added by Restorer
   if has("win32")
-    echoerr "Doesn't work on MS-Windows"
-    return
+    let s:outdev = 'nul'
+  else
+    let s:outdev = '/dev/null'
   endif
+" Restorer: For Windows users. If "curl" or "wget" is installed on the system
+" but not in %PATH%, add the full path to them to %PATH% environment variable.
   if executable('curl')
     " Note: does not follow redirects!
-    let s:command = 'curl --silent --fail --output /dev/null --head '
+    let s:command1 = 'curl --silent --max-time 5 --fail --output ' ..s:outdev.. ' --head '
+    let s:command2 = ""
   elseif executable('wget')
     " Note: only allow a couple of redirects
-    let s:command = 'wget --quiet -S --spider --max-redirect=2 --timeout=5 --tries=2 -O /dev/null '
+    let s:command1 = 'wget --quiet -S --spider --max-redirect=2 --timeout=5 --tries=2 -O ' ..s:outdev.. ' '
+    let s:command2 = ""
+  elseif has("win32") "20.10.23, added by Restorer
+    if executable('powershell')
+      if 2 == system('powershell -nologo -noprofile "$psversiontable.psversion.major"')
+        echoerr 'To work in OS Windows requires the program "PowerShell" version 3.0 or higher'
+        return
+      endif
+      let s:command1 = 
+            \ "powershell -nologo -noprofile \"{[Net.ServicePointManager]::SecurityProtocol = 'Tls12, Tls11, Tls, Ssl3'};try{(Invoke-WebRequest -MaximumRedirection 2 -TimeoutSec 5 -Uri "
+      let s:command2 = ').StatusCode}catch{exit [int]$Error[0].Exception.Status}"'
+    endif
   else
-    echoerr 'Only works when "curl" or "wget" is available'
+    echoerr 'Only works when "curl" or "wget", or "powershell" is available'
     return
   endif
 
+  " Do the testing.
+  set report =999
+  set nomore shm +=s
+
   let pat='\(https\?\|ftp\)://[^\t* ]\+'
   exe 'helpgrep' pat
   helpclose
@@ -36,22 +56,21 @@ func Test_check_URLs()
   put =urls
   " remove some more invalid items
   " empty lines
-  v/./d
+  "20.10.23, Restorer: '_' is a little faster, see `:h global`
+  v/./d _
   " remove # anchors
   %s/#.*$//e
   " remove trailing stuff (parenthesis, dot, comma, quotes), but only for HTTP
   " links
-  g/^h/s#[.,)'"/>][:.]\?$##
-  g#^[hf]t\?tp:/\(/\?\.*\)$#d
-  silent! g/ftp://,$/d
-  silent! g/=$/d
+  g/^h/s#[.),'"`/>][:.,]\?$##
+  g#^[hf]t\?tp:/\(/\?\.*\)$#d _
+  silent! g/ftp://,$/d _
+  silent! g/=$/d _
   let a = getline(1,'$')
   let a = uniq(sort(a))
-  %d
+  %d _
   call setline(1, a)
 
-  " Do the testing.
-  set nomore
   %s/.*/\=TestURL(submatch(0))/
 
   " highlight the failures
@@ -61,8 +80,10 @@ endfunc
 func TestURL(url)
   " Relies on the return code to determine whether a page is valid
   echom printf("Testing URL: %d/%d %s", line('.'), line('$'), a:url)
-  call system(s:command . shellescape(a:url))
+  call system(s:command1 .. shellescape(a:url) .. s:command2)
   return printf("%s %d", a:url, v:shell_error)
 endfunc
 
 call Test_check_URLs()
+
+" vim: sw=2 sts=2 et
diff --git a/scripts/cliff.toml b/scripts/cliff.toml
index 3fc10e5d16..51725cacfc 100644
--- a/scripts/cliff.toml
+++ b/scripts/cliff.toml
@@ -1,39 +1,38 @@
-# configuration file for git-cliff (0.1.0)
+# configuration file for git-cliff
 
 [changelog]
 # changelog header
 header = """
 # Changelog\n
-All notable changes to this project will be documented in this file.\n
+For notable changes, see runtime/doc/news.txt (or `:help news` in Nvim).\n
+Following is a list of fixes/features commits.\n
 """
 # template for the changelog body
-# https://tera.netlify.app/docs/#introduction
+# https://github.com/Keats/tera
+# https://keats.github.io/tera/docs/
 body = """
 {% if version %}\
-    ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
+    # [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
 {% else %}\
-    ## [unreleased]
+    # [unreleased]
 {% endif %}\
 {% for group, commits in commits | group_by(attribute="group") %}
-    ### {{ group | upper_first }}
-    {% for commit in commits%}\
+    {{ group | striptags | upper_first }}
+    --------------------------------------------------------------------------------
+    {% for commit in commits | sort(attribute="message")%}\
         {% if not commit.scope %}\
-            - {{ commit.message | upper_first }}
+            - {{ commit.id | truncate(length=12, end="") }} {{ commit.message }}
         {% endif %}\
     {% endfor %}\
     {% for group, commits in commits | group_by(attribute="scope") %}\
-        {% for commit in commits %}\
-            - **{{commit.scope}}**: {{ commit.message | upper_first }}
+        {% for commit in commits | sort(attribute="message") %}\
+            - {{ commit.id | truncate(length=12, end="") }} {{commit.scope}}: {{ commit.message }}
         {% endfor %}\
     {% endfor %}
 {% endfor %}\n
 """
 # remove the leading and trailing whitespace from the template
 trim = true
-# changelog footer
-footer = """
-<!-- generated by git-cliff -->
-"""
 
 [git]
 # parse the commits based on https://www.conventionalcommits.org
@@ -48,16 +47,18 @@ commit_preprocessors = [
 ]
 # regex for parsing and grouping commits
 commit_parsers = [
-    { message = "!:", group = "Breaking"},
-    { message = "^feat", group = "Features"},
-    { message = "^fix", group = "Bug Fixes"},
-    { message = "^doc", group = "Documentation"},
-    { message = "^perf", group = "Performance"},
-    { message = "^refactor", group = "Refactor"},
-    { message = "^test", group = "Testing"},
-    { message = "^chore", group = "Miscellaneous Tasks"},
-    { message = "^build", group = "Build System"},
-    { message = "^Revert", group = "Reverted Changes"},
+    { message = "!:", group = "<!-- 0 -->BREAKING"},
+    { message = "^feat", group = "<!-- 1 -->FEATURES"},
+    { message = "^fix", group = "<!-- 2 -->FIXES"},
+    { message = "^perf", group = "<!-- 3 -->PERFORMANCE"},
+    { message = "^build", group = "<!-- 4 -->BUILD"},
+    { message = "^vim-patch", group = "<!-- 5 -->VIM PATCHES"},
+    { message = "^refactor", group = "<!-- 6 -->REFACTOR" },
+    { message = "^ci", group = "<!-- 8 -->CI" },
+    { message = "^test", group = "<!-- 9 -->TESTING" },
+    { message = "^docs", group = "<!-- 99 -->DOCUMENTATION" },
+    { message = "^revert", group = "<!-- 999 -->REVERTED CHANGES" },
+    { message = ".*", group = "<!-- 9999 -->OTHER"},
 ]
 # filter out the commits that are not matched by commit parsers
 filter_commits = true
diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua
index 43040151eb..cdfb85bde6 100644
--- a/scripts/gen_help_html.lua
+++ b/scripts/gen_help_html.lua
@@ -50,6 +50,7 @@ local spell_dict = {
 local spell_ignore_files = {
   ['backers.txt'] = true,
   ['news.txt'] = { 'tree-sitter' }, -- in news, may refer to the upstream "tree-sitter" library
+  ['news-0.10.txt'] = { 'tree-sitter' },
 }
 local language = nil
 
@@ -63,13 +64,20 @@ local new_layout = {
   ['channel.txt'] = true,
   ['deprecated.txt'] = true,
   ['develop.txt'] = true,
+  ['dev_style.txt'] = true,
+  ['dev_theme.txt'] = true,
+  ['dev_tools.txt'] = true,
+  ['dev_vimpatch.txt'] = true,
+  ['faq.txt'] = true,
   ['lua.txt'] = true,
   ['luaref.txt'] = true,
   ['news.txt'] = true,
+  ['news-0.9.txt'] = true,
+  ['news-0.10.txt'] = true,
   ['nvim.txt'] = true,
-  ['pi_health.txt'] = true,
   ['provider.txt'] = true,
   ['ui.txt'] = true,
+  ['vim_diff.txt'] = true,
 }
 
 -- TODO: These known invalid |links| require an update to the relevant docs.
@@ -523,6 +531,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     return ('%s<a href="%s">%s</a>%s'):format(ws(), fixed_url, fixed_url, removed_chars)
   elseif node_name == 'word' or node_name == 'uppercase_name' then
     return text
+  elseif node_name == 'note' then
+    return ('<b>%s</b>'):format(text)
   elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then
     if is_noise(text, stats.noise_lines) then
       return '' -- Discard common "noise" lines.
@@ -685,6 +695,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       return string.format('%s</span>', s)
     end
     return s
+  elseif node_name == 'modeline' then
+    return ''
   elseif node_name == 'ERROR' then
     if ignore_parse_error(opt.fname, trimmed) then
       return text
@@ -824,8 +836,7 @@ local function gen_one(fname, to_fname, old, commit, parser_path)
     <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@docsearch/css@3" />
     <link rel="preconnect" href="https://X185E15FPG-dsn.algolia.net" crossorigin />
 
-    <link href="/css/normalize.min.css" rel="stylesheet">
-    <link href="/css/bootstrap.css" rel="stylesheet">
+    <link href="/css/bootstrap.min.css" rel="stylesheet">
     <link href="/css/main.css" rel="stylesheet">
     <link href="help.css" rel="stylesheet">
     <link href="/highlight/styles/neovim.min.css" rel="stylesheet">
@@ -1086,14 +1097,19 @@ local function gen_css(fname)
       padding-bottom: 10px;
       /* Tabs are used for alignment in old docs, so we must match Vim's 8-char expectation. */
       tab-size: 8;
-      white-space: pre;
+      white-space: pre-wrap;
       font-size: 16px;
       font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace;
+      word-wrap: break-word;
     }
-    .old-help-para pre {
-      /* All text in .old-help-para is formatted as "white-space:pre" so text following <pre> is
-         already visually separated by the linebreak. */
+    .old-help-para pre, .old-help-para pre:hover {
+      /* Text following <pre> is already visually separated by the linebreak. */
       margin-bottom: 0;
+      /* Long lines that exceed the textwidth should not be wrapped (no "pre-wrap").
+         Since text may overflow horizontally, we make the contents to be scrollable
+         (only if necessary) to prevent overlapping with the navigation bar at the right. */
+      white-space: pre;
+      overflow-x: auto;
     }
 
     /* TODO: should this rule be deleted? help tags are rendered as <code> or <span>, not <a> */
@@ -1422,9 +1438,8 @@ function M.test_gen(help_dir)
     help_dir,
     tmpdir,
     -- Because gen() is slow (~30s), this test is limited to a few files.
-    { 'pi_health.txt', 'help.txt', 'index.txt', 'nvim.txt' }
+    { 'help.txt', 'index.txt', 'nvim.txt' }
   )
-  eq(4, #rv.helpfiles)
   eq(0, rv.err_count, 'parse errors in :help docs')
   eq({}, rv.invalid_links, 'invalid tags in :help docs')
 end
diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua
index 19fad7bab4..04d19f22e6 100644
--- a/scripts/gen_lsp.lua
+++ b/scripts/gen_lsp.lua
@@ -259,10 +259,13 @@ function M.gen(opt)
       if prefix then
         anonymous_classname = anonymous_classname .. '.' .. prefix
       end
-      local anonym = vim.tbl_flatten { -- remove nil
-        anonymous_num > 1 and '' or nil,
-        '---@class ' .. anonymous_classname,
-      }
+      local anonym = vim
+        .iter({
+          (anonymous_num > 1 and { '' } or {}),
+          { '---@class ' .. anonymous_classname },
+        })
+        :flatten()
+        :totable()
 
       --- @class vim._gen_lsp.StructureLiteral translated to anonymous @class.
       --- @field deprecated? string
diff --git a/scripts/gen_vimdoc.lua b/scripts/gen_vimdoc.lua
index 22df411a35..9c6225efc3 100755
--- a/scripts/gen_vimdoc.lua
+++ b/scripts/gen_vimdoc.lua
@@ -94,12 +94,12 @@ end
 local function fn_helptag_fmt_common(fun)
   local fn_sfx = fun.table and '' or '()'
   if fun.classvar then
-    return fmt('*%s:%s%s*', fun.classvar, fun.name, fn_sfx)
+    return fmt('%s:%s%s', fun.classvar, fun.name, fn_sfx)
   end
   if fun.module then
-    return fmt('*%s.%s%s*', fun.module, fun.name, fn_sfx)
+    return fmt('%s.%s%s', fun.module, fun.name, fn_sfx)
   end
-  return fmt('*%s%s*', fun.name, fn_sfx)
+  return fun.name .. fn_sfx
 end
 
 --- @type table<string,nvim.gen_vimdoc.Config>
@@ -129,7 +129,7 @@ local config = {
       return name .. ' Functions'
     end,
     helptag_fmt = function(name)
-      return fmt('*api-%s*', name:lower())
+      return fmt('api-%s', name:lower())
     end,
   },
   lua = {
@@ -241,22 +241,22 @@ local config = {
     end,
     helptag_fmt = function(name)
       if name == '_editor' then
-        return '*lua-vim*'
+        return 'lua-vim'
       elseif name == '_options' then
-        return '*lua-vimscript*'
+        return 'lua-vimscript'
       elseif name == 'tohtml' then
-        return '*tohtml*'
+        return 'tohtml'
       end
-      return '*vim.' .. name:lower() .. '*'
+      return 'vim.' .. name:lower()
     end,
     fn_helptag_fmt = function(fun)
       local name = fun.name
 
       if vim.startswith(name, 'vim.') then
         local fn_sfx = fun.table and '' or '()'
-        return fmt('*%s%s*', name, fn_sfx)
+        return name .. fn_sfx
       elseif fun.classvar == 'Option' then
-        return fmt('*vim.opt:%s()*', name)
+        return fmt('vim.opt:%s()', name)
       end
 
       return fn_helptag_fmt_common(fun)
@@ -297,9 +297,9 @@ local config = {
     end,
     helptag_fmt = function(name)
       if name:lower() == 'lsp' then
-        return '*lsp-core*'
+        return 'lsp-core'
       end
-      return fmt('*lsp-%s*', name:lower())
+      return fmt('lsp-%s', name:lower())
     end,
   },
   diagnostic = {
@@ -312,7 +312,7 @@ local config = {
       return 'Lua module: vim.diagnostic'
     end,
     helptag_fmt = function()
-      return '*diagnostic-api*'
+      return 'diagnostic-api'
     end,
   },
   treesitter = {
@@ -337,9 +337,43 @@ local config = {
     end,
     helptag_fmt = function(name)
       if name:lower() == 'treesitter' then
-        return '*lua-treesitter-core*'
+        return 'lua-treesitter-core'
       end
-      return '*lua-treesitter-' .. name:lower() .. '*'
+      return 'lua-treesitter-' .. name:lower()
+    end,
+  },
+  editorconfig = {
+    filename = 'editorconfig.txt',
+    files = {
+      'runtime/lua/editorconfig.lua',
+    },
+    section_order = {
+      'editorconfig.lua',
+    },
+    section_fmt = function(_name)
+      return 'EditorConfig integration'
+    end,
+    helptag_fmt = function(name)
+      return name:lower()
+    end,
+    fn_xform = function(fun)
+      fun.table = true
+      fun.name = vim.split(fun.name, '.', { plain = true })[2]
+    end,
+  },
+  health = {
+    filename = 'health.txt',
+    files = {
+      'runtime/lua/vim/health.lua',
+    },
+    section_order = {
+      'health.lua',
+    },
+    section_fmt = function(_name)
+      return 'Checkhealth'
+    end,
+    helptag_fmt = function(name)
+      return name:lower()
     end,
   },
 }
@@ -469,10 +503,12 @@ local function inline_type(obj, classes)
 
   local desc_append = {}
   for _, f in ipairs(cls.fields) do
-    local fdesc, default = get_default(f.desc)
-    local fty = render_type(f.type, nil, default)
-    local fnm = fmt_field_name(f.name)
-    table.insert(desc_append, table.concat({ '-', fnm, fty, fdesc }, ' '))
+    if not f.access then
+      local fdesc, default = get_default(f.desc)
+      local fty = render_type(f.type, nil, default)
+      local fnm = fmt_field_name(f.name)
+      table.insert(desc_append, table.concat({ '-', fnm, fty, fdesc }, ' '))
+    end
   end
 
   desc = desc .. '\n' .. table.concat(desc_append, '\n')
@@ -593,6 +629,12 @@ local function render_fun_header(fun, cfg)
   if fun.classvar then
     nm = fmt('%s:%s', fun.classvar, nm)
   end
+  if nm == 'vim.bo' then
+    nm = 'vim.bo[{bufnr}]'
+  end
+  if nm == 'vim.wo' then
+    nm = 'vim.wo[{winid}][{bufnr}]'
+  end
 
   local proto = fun.table and nm or nm .. '(' .. table.concat(args, ', ') .. ')'
 
@@ -600,7 +642,7 @@ local function render_fun_header(fun, cfg)
     cfg.fn_helptag_fmt = fn_helptag_fmt_common
   end
 
-  local tag = cfg.fn_helptag_fmt(fun)
+  local tag = '*' .. cfg.fn_helptag_fmt(fun) .. '*'
 
   if #proto + #tag > TEXT_WIDTH - 8 then
     table.insert(ret, fmt('%78s\n', tag))
@@ -747,10 +789,17 @@ local function render_funs(funs, classes, cfg)
     ret[#ret + 1] = render_fun(f, classes, cfg)
   end
 
-  -- Sort via prototype
+  -- Sort via prototype. Experimental API functions ("nvim__") sort last.
   table.sort(ret, function(a, b)
     local a1 = ('\n' .. a):match('\n[a-zA-Z_][^\n]+\n')
     local b1 = ('\n' .. b):match('\n[a-zA-Z_][^\n]+\n')
+
+    local a1__ = a1:find('^%s*nvim__') and 1 or 0
+    local b1__ = b1:find('^%s*nvim__') and 1 or 0
+    if a1__ ~= b1__ then
+      return a1__ < b1__
+    end
+
     return a1:lower() < b1:lower()
   end)
 
@@ -816,7 +865,7 @@ local function make_section(filename, cfg, section_docs, funs_txt)
   local sectname = cfg.section_name and cfg.section_name[filename] or mktitle(name)
 
   -- section tag: e.g., "*api-autocmd*"
-  local help_tag = cfg.helptag_fmt(sectname)
+  local help_tag = '*' .. cfg.helptag_fmt(sectname) .. '*'
 
   if funs_txt == '' and #section_docs == 0 then
     return
@@ -845,9 +894,9 @@ local function render_section(section, add_header)
     })
   end
 
-  if section.doc and #section.doc > 0 then
-    table.insert(doc, '\n\n')
-    vim.list_extend(doc, section.doc)
+  local sdoc = '\n\n' .. table.concat(section.doc or {}, '\n')
+  if sdoc:find('[^%s]') then
+    doc[#doc + 1] = sdoc
   end
 
   if section.funs_txt then
@@ -880,6 +929,7 @@ end
 
 --- @param cfg nvim.gen_vimdoc.Config
 local function gen_target(cfg)
+  print('Target:', cfg.filename)
   local sections = {} --- @type table<string,nvim.gen_vimdoc.Section>
 
   expand_files(cfg.files)
@@ -891,7 +941,7 @@ local function gen_target(cfg)
   local all_classes = {}
 
   --- First pass so we can collect all classes
-  for _, f in pairs(cfg.files) do
+  for _, f in vim.spairs(cfg.files) do
     local ext = assert(f:match('%.([^.]+)$')) --[[@as 'h'|'c'|'lua']]
     local parser = assert(parsers[ext])
     local classes, funs, briefs = parser(f)
@@ -899,13 +949,14 @@ local function gen_target(cfg)
     all_classes = vim.tbl_extend('error', all_classes, classes)
   end
 
-  for f, r in pairs(file_results) do
+  for f, r in vim.spairs(file_results) do
     local classes, funs, briefs = r[1], r[2], r[3]
 
     local briefs_txt = {} --- @type string[]
     for _, b in ipairs(briefs) do
       briefs_txt[#briefs_txt + 1] = md_to_vimdoc(b, 0, 0, TEXT_WIDTH)
     end
+    print('    Processing file:', f)
     local funs_txt = render_funs(funs, all_classes, cfg)
     if next(classes) then
       local classes_txt = render_classes(classes)
@@ -923,8 +974,9 @@ local function gen_target(cfg)
   for _, f in ipairs(cfg.section_order) do
     local section = sections[f]
     if section then
+      print(string.format("    Rendering section: '%s'", section.title))
       local add_sep_and_header = not vim.tbl_contains(cfg.append_only or {}, f)
-      table.insert(docs, render_section(section, add_sep_and_header))
+      docs[#docs + 1] = render_section(section, add_sep_and_header)
     end
   end
 
@@ -945,7 +997,7 @@ local function gen_target(cfg)
 end
 
 local function run()
-  for _, cfg in pairs(config) do
+  for _, cfg in vim.spairs(config) do
     gen_target(cfg)
   end
 end
diff --git a/scripts/git-log-pretty-since.sh b/scripts/git-log-pretty-since.sh
deleted file mode 100755
index 95dcee23f5..0000000000
--- a/scripts/git-log-pretty-since.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-
-# Prints a nicely-formatted commit history.
-#   - Commits are grouped below their merge-commit.
-#   - Issue numbers are moved next to the commit-id.
-#
-# Parameters:
-#   $1    "since" commit
-#   $2    "inverse match" regex pattern
-
-set -e
-set -u
-set -o pipefail
-
-__SINCE=$1
-__INVMATCH=$2
-
-is_merge_commit() {
-  git rev-parse "$1" >/dev/null 2>&1 \
-    || { echo "ERROR: invalid commit: $1"; exit 1; }
-  git log "$1"^2 >/dev/null 2>&1 && return 0 || return 1
-}
-
-# Removes parens from issue/ticket/PR numbers.
-#
-# Example:
-#   in:   3340e08becbf foo (#9423)
-#   out:  3340e08becbf foo #9423
-_deparen() {
-  sed 's/(\(\#[0-9]\{3,\}\))/\1/g'
-}
-
-# Cleans up issue/ticket/PR numbers in the commit descriptions.
-#
-# Example:
-#   in:   3340e08becbf foo (#9423)
-#   out:  3340e08becbf #9423 foo
-_format_ticketnums() {
-  nvim -Es +'g/\v(#[0-9]{3,})/norm! ngEldE0ep' +'%p' | _deparen
-}
-
-for commit in $(git log --format='%H' --first-parent "$__SINCE"..HEAD); do
-  if is_merge_commit "${commit}" ; then
-      if [ -z "$__INVMATCH" ] || ! git log --oneline "${commit}^1..${commit}^2" \
-           | >/dev/null 2>&1 grep -E "$__INVMATCH" ; then
-        git log -1 --oneline "${commit}"
-        git log --format='    %h %s' "${commit}^1..${commit}^2"
-      fi
-  else
-    git log -1 --oneline "${commit}"
-  fi
-done | _format_ticketnums
diff --git a/scripts/legacy2luatest.pl b/scripts/legacy2luatest.pl
deleted file mode 100755
index 8155353fc7..0000000000
--- a/scripts/legacy2luatest.pl
+++ /dev/null
@@ -1,300 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use warnings;
-use 5.010;
-use autodie;
-
-use File::Basename;
-use File::Spec::Functions;
-
-sub read_in_file {
-  my $in_file = $_[0];
-
-  # Will contain lines before first STARTTEST
-  # as Lua comments.
-  my @description_lines = ();
-
-  # Will contain alternating blocks of lines of textual input
-  # (text between ENDTEST and EOF/next STARTTEST) and test commands
-  # (commands between STARTTEST and ENDTEST) as Lua code.
-  my @test_body_lines = ();
-
-  # Will contain current command block, i.e. lines
-  # between STARTTEST and ENDTEST.
-  my @command_lines = ();
-
-  # Will contain current input block, i.e. lines
-  # between ENDTEST and STARTTEST.
-  my @input_lines = ();
-
-  open my $in_file_handle, '<', $in_file;
-
-  use constant EMIT_DESCRIPTION => 0;
-  use constant EMIT_COMMAND => 1;
-  use constant EMIT_INPUT => 2;
-
-  # Push lines from current input and
-  # command blocks into @test_body_lines
-  # in the correct order.
-  sub end_input {
-    my $input_lines = $_[0];
-    my $command_lines = $_[1];
-    my $test_body_lines = $_[2];
-
-    # If there are input lines, wrap with an `insert`
-    # command and add before the previous command block.
-    if (@{$input_lines}) {
-      my $last_input_line = pop @{$input_lines};
-      unshift @{$command_lines}, '';
-      unshift @{$command_lines}, $last_input_line . ']=])';
-      unshift @{$command_lines}, @{$input_lines};
-      unshift @{$command_lines}, "insert([=[";
-
-      @{$input_lines} = ();
-    }
-
-    # Output remaining command lines.
-    push @{$test_body_lines}, @{$command_lines};
-    @{$command_lines} = ();
-  }
-
-  sub format_comment {
-    # Handle empty comments.
-    if (/^$/) {
-      return '';
-    }
-
-    # Capitalize first character and emit as Lua comment.
-    my $comment = '-- ' . ucfirst $_;
-
-    # Add trailing dot if not already there.
-    $comment .= '.' unless $comment =~ /\.$/;
-
-    return $comment;
-  }
-
-  my %states = (
-    # Add test description to @description_lines.
-    EMIT_DESCRIPTION() => sub {
-      if (/^STARTTEST/) {
-        return EMIT_COMMAND;
-      }
-
-      # If not an empty line, emit as Lua comment.
-      if (!/^$/) {
-        # Remove modeline
-        s/vim:.*set f\w+=vim//g;
-        # Remove trailing ":"
-        s/\s*:\s*$//g;
-        push @description_lines, '-- ' . $_;
-      }
-
-      return EMIT_DESCRIPTION;
-    },
-    # Add test commands to @command_lines.
-    EMIT_COMMAND() => sub {
-      if (/^ENDTEST/) {
-        return EMIT_INPUT;
-      }
-
-      # If line starts with ':"', emit a comment.
-      if (/^:"/) {
-        # Remove Vim comment prefix.
-        s/^:"\s*//;
-
-        push @command_lines, format_comment $_;
-
-        return EMIT_COMMAND;
-      }
-
-      # Extract possible inline comment.
-      if (/^[^"]*"[^"]*$/) {
-        # Remove command part and prepended whitespace.
-        s/^(.*?)\s*"\s*//;
-
-        push @command_lines, format_comment $_;
-
-        # Set implicit variable to command without comment.
-        $_ = $1;
-      }
-
-      # Only continue if remaining command is not empty.
-      if (!/^:?\s*$/) {
-        # Replace terminal escape characters with <esc>.
-        s/\e/<esc>/g;
-
-        my $startstr = "'";
-        my $endstr = "'";
-
-        # If line contains single quotes or backslashes, use double
-        # square brackets to wrap string.
-        if (/'/ || /\\/) {
-            # If the line contains a closing square bracket,
-            # wrap it with [=[...]=].
-            if (/\]/) {
-              $startstr = '[=[';
-              $endstr = ']=]';
-            } else {
-              $startstr = '[[';
-              $endstr = ']]';
-            }
-        }
-
-        # Emit 'feed' if not a search ('/') or ex (':') command.
-        if (!/^\// && !/^:/) {
-          # If command does not end with <esc>, insert trailing <cr>.
-          my $command = 'feed(' . $startstr . $_;
-          $command .= '<cr>' unless /<esc>$/;
-          $command .= $endstr . ')';
-
-          push @command_lines, $command;
-        } else {
-          # Remove prepending ':'.
-          s/^://;
-          push @command_lines, 'execute(' . $startstr . $_ . $endstr . ')';
-        }
-      }
-
-      return EMIT_COMMAND;
-    },
-    # Add input to @input_lines.
-    EMIT_INPUT() => sub {
-      if (/^STARTTEST/) {
-        end_input \@input_lines, \@command_lines, \@test_body_lines;
-        return EMIT_COMMAND;
-      }
-
-      # Skip initial lines if they are empty.
-      if (@input_lines or !/^$/) {
-        push @input_lines, '  ' . $_;
-      }
-      return EMIT_INPUT;
-    },
-  );
-
-  my $state = EMIT_DESCRIPTION;
-
-  while (<$in_file_handle>) {
-    # Remove trailing newline character and process line.
-    chomp;
-    $state = $states{$state}->($_);
-  }
-
-  # If not all lines have been processed yet,
-  # do it now.
-  end_input \@input_lines, \@command_lines, \@test_body_lines;
-
-  close $in_file_handle;
-
-  return (\@description_lines, \@test_body_lines);
-}
-
-sub read_ok_file {
-  my $ok_file = $_[0];
-  my @assertions = ();
-
-  if (-f $ok_file) {
-    push @assertions, '';
-    push @assertions, "-- Assert buffer contents.";
-    push @assertions, "expect([=[";
-
-    open my $ok_file_handle, '<', $ok_file;
-
-    while (<$ok_file_handle>) {
-      # Remove trailing newline character and process line.
-      chomp;
-      push @assertions, '  ' . $_;
-    }
-
-    close $ok_file_handle;
-
-    $assertions[-1] .= "]=])";
-  }
-
-  return \@assertions;
-}
-
-my $legacy_testfile = $ARGV[0];
-my $out_dir = $ARGV[1];
-
-if ($#ARGV != 1) {
-  say "Convert a legacy Vim test to a Neovim lua spec.";
-  say '';
-  say "Usage: $0 legacy-testfile output-directory";
-  say '';
-  say "legacy-testfile:  Path to .in or .ok file.";
-  say "output-directory: Directory where Lua spec will be saved to.";
-  say '';
-  say "Note: Only works reliably for fairly simple tests.";
-  say "      Manual adjustments to generated spec files are required.";
-  exit 1;
-}
-
-my @legacy_suffixes = ('.in', '.ok');
-my ($base_name, $base_path, $suffix) = fileparse($legacy_testfile, @legacy_suffixes);
-my $in_file = catfile($base_path, $base_name . '.in');
-my $ok_file = catfile($base_path, $base_name . '.ok');
-
-# Remove leading 'test'.
-my $test_name = $base_name;
-$test_name =~ s/^test_?//;
-
-my $spec_file = do {
-  if ($test_name =~ /^([0-9]+)/) {
-    catfile($out_dir,  sprintf('%03d', $1) . '_spec.lua')
-  } else {
-    catfile($out_dir,  $test_name . '_spec.lua')
-  }
-};
-
-if (! -f $in_file) {
-  say "Test input file $in_file not found.";
-  exit 2;
-}
-
-if (! -d $out_dir) {
-  say "Output directory $out_dir does not exist.";
-  exit 3;
-}
-
-if (-f $spec_file) {
-  say "Output file $spec_file already exists.";
-  print "Overwrite (Y/n)? ";
-  my $input = <STDIN>;
-  chomp($input);
-  unless ($input =~ /^y|Y/) {
-    say "Aborting.";
-    exit 4;
-  }
-}
-
-# Read .in and .ok files.
-my ($description_lines, $test_body_lines) = read_in_file $in_file;
-my $assertion_lines = read_ok_file $ok_file;
-
-# Append assertions to test body.
-push @{$test_body_lines}, @{$assertion_lines} if @{$assertion_lines};
-
-# Write spec file.
-open my $spec_file_handle, ">", $spec_file;
-
-print $spec_file_handle <<"EOS";
-@{[join "\n", @{$description_lines}]}
-
-local helpers = require('test.functional.helpers')
-local feed, insert, source = helpers.feed, helpers.insert, helpers.source
-local clear, execute, expect = helpers.clear, helpers.execute, helpers.expect
-
-describe('$test_name', function()
-  before_each(clear)
-
-  it('is working', function()
-@{[join "\n", map { /^$/ ? '' : '    ' . $_ } @{$test_body_lines}]}
-  end)
-end)
-EOS
-
-close $spec_file_handle;
-
-say "Written to $spec_file."
diff --git a/scripts/luacats_grammar.lua b/scripts/luacats_grammar.lua
index ca26c70156..29f3bda5aa 100644
--- a/scripts/luacats_grammar.lua
+++ b/scripts/luacats_grammar.lua
@@ -170,7 +170,7 @@ local grammar = P {
   ltype = parenOpt(v.ty_union),
 
   ty_union = v.ty_opt * rep(Pf('|') * v.ty_opt),
-  ty = v.ty_fun + ident + v.ty_table + literal + paren(v.ty),
+  ty = v.ty_fun + ident + v.ty_table + literal + paren(v.ty) + v.ty_generic,
   ty_param = Pf('<') * comma1(v.ltype) * fill * P('>'),
   ty_opt = v.ty * opt(v.ty_param) * opt(P('[]')) * opt(P('?')),
   ty_index = (Pf('[') * (v.ltype + ident + rep1(num)) * fill * P(']')),
@@ -179,6 +179,7 @@ local grammar = P {
   ty_table = Pf('{') * comma1(v.table_elem) * fill * P('}'),
   fun_param = lname * opt(colon * v.ltype),
   ty_fun = Pf('fun') * paren(comma(lname * opt(colon * v.ltype))) * opt(colon * comma1(v.ltype)),
+  ty_generic = P('`') * le