diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/bump-deps.sh | 104 | ||||
| -rwxr-xr-x[-rw-r--r--] | scripts/bump_deps.lua | 601 | ||||
| -rwxr-xr-x | scripts/check-includes.py | 64 | ||||
| -rw-r--r-- | scripts/cliff.toml | 73 | ||||
| -rwxr-xr-x | scripts/download-unicode-files.sh | 19 | ||||
| -rw-r--r-- | scripts/gen_help_html.lua | 1110 | ||||
| -rw-r--r-- | scripts/gen_help_html.py | 389 | ||||
| -rwxr-xr-x | scripts/gen_vimdoc.py | 94 | ||||
| -rwxr-xr-x | scripts/genappimage.sh | 12 | ||||
| -rw-r--r-- | scripts/genvimvim.lua | 6 | ||||
| -rwxr-xr-x | scripts/git-log-pretty-since.sh | 14 | ||||
| -rw-r--r-- | scripts/lintcommit.lua | 21 | ||||
| -rw-r--r-- | scripts/lua2dox.lua | 372 | ||||
| -rwxr-xr-x | scripts/lua2dox_filter | 88 | ||||
| -rwxr-xr-x | scripts/pvscheck.sh | 2 | ||||
| -rwxr-xr-x | scripts/release.sh | 4 | ||||
| -rwxr-xr-x | scripts/update_terminfo.sh | 2 | ||||
| -rwxr-xr-x | scripts/vim-patch.sh | 68 | ||||
| -rwxr-xr-x | scripts/vimpatch.lua | 7 |
19 files changed, 1860 insertions, 1190 deletions
diff --git a/scripts/bump-deps.sh b/scripts/bump-deps.sh deleted file mode 100755 index 85c7f72700..0000000000 --- a/scripts/bump-deps.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -set -e -set -u -# Use privileged mode, which e.g. skips using CDPATH. -set -p - -# Ensure that the user has a bash that supports -A -if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then - echo >&2 "error: script requires bash 4+ (you have ${BASH_VERSION})." - exit 1 -fi - -readonly NVIM_SOURCE_DIR="${NVIM_SOURCE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" -readonly VIM_SOURCE_DIR_DEFAULT="${NVIM_SOURCE_DIR}/.vim-src" -readonly VIM_SOURCE_DIR="${VIM_SOURCE_DIR:-${VIM_SOURCE_DIR_DEFAULT}}" -BASENAME="$(basename "${0}")" -readonly BASENAME - -usage() { - echo "Bump Neovim dependencies" - echo - echo "Usage: ${BASENAME} [ -h | --pr | --branch=<dep> | --dep=<dependency> ]" - echo - echo "Options:" - echo " -h show this message and exit." - echo " --pr submit pr for bumping deps." - echo " --branch=<dep> create a branch bump-<dep> from current branch." - echo " --dep=<dependency> bump to a specific release or tag." - echo - echo "Dependency Options:" - echo " --version=<tag> bump to a specific release or tag." - echo " --commit=<hash> bump to a specific commit." - echo " --HEAD bump to a current head." - echo - echo " <dependency> is one of:" - echo " \"LuaJIT\", \"libuv\", \"Luv\", \"tree-sitter\"" -} - -# Checks if a program is in the user's PATH, and is executable. -check_executable() { - test -x "$(command -v "${1}")" -} - -require_executable() { - if ! check_executable "${1}"; then - echo >&2 "${BASENAME}: '${1}' not found in PATH or not executable." - exit 1 - fi -} - -require_executable "nvim" - -if [ $# -eq 0 ]; then - usage - exit 1 -fi - -PARSED_ARGS=$(getopt -a -n "$BASENAME" -o h --long pr,branch:,dep:,version:,commit:,HEAD -- "$@") - -DEPENDENCY="" -eval set -- "$PARSED_ARGS" -while :; do - case "$1" in - -h) - usage - exit 0 - ;; - --pr) - nvim -es +"lua require('scripts.bump_deps').submit_pr()" - exit 0 - ;; - --branch) - DEP=$2 - nvim -es +"lua require('scripts.bump_deps').create_branch('$DEP')" - exit 0 - ;; - --dep) - DEPENDENCY=$2 - shift 2 - ;; - --version) - VERSION=$2 - nvim -es +"lua require('scripts.bump_deps').version('$DEPENDENCY', '$VERSION')" - exit 0 - ;; - --commit) - COMMIT=$2 - nvim -es +"lua require('scripts.bump_deps').commit('$DEPENDENCY', '$COMMIT')" - exit 0 - ;; - --HEAD) - nvim -es +"lua require('scripts.bump_deps').head('$DEPENDENCY')" - exit 0 - ;; - *) - break - ;; - esac -done - -usage -exit 1 - -# vim: et sw=2 diff --git a/scripts/bump_deps.lua b/scripts/bump_deps.lua index 17e3fd35d6..1873c3cd0d 100644..100755 --- a/scripts/bump_deps.lua +++ b/scripts/bump_deps.lua @@ -1,35 +1,24 @@ +#!/usr/bin/env -S nvim -l + -- Usage: --- # bump to version --- nvim -es +"lua require('scripts.bump_deps').version(dependency, version_tag)" --- --- # bump to commit --- nvim -es +"lua require('scripts.bump_deps').commit(dependency, commit_hash)" --- --- # bump to HEAD --- nvim -es +"lua require('scripts.bump_deps').head(dependency)" --- --- # submit PR --- nvim -es +"lua require('scripts.bump_deps').submit_pr()" --- --- # create branch --- nvim -es +"lua require('scripts.bump_deps').create_branch()" +-- ./scripts/bump_deps.lua -h local M = {} local _trace = false -local required_branch_prefix = "bump-" -local commit_prefix = "build(deps): " +local required_branch_prefix = 'bump-' +local commit_prefix = 'build(deps): ' -- Print message local function p(s) - vim.cmd("set verbose=1") - vim.api.nvim_echo({ { s, "" } }, false, {}) - vim.cmd("set verbose=0") + vim.cmd('set verbose=1') + vim.api.nvim_echo({ { s, '' } }, false, {}) + vim.cmd('set verbose=0') end local function die() - p("") - vim.cmd("cquit 1") + p('') + vim.cmd('cquit 1') end -- Executes and returns the output of `cmd`, or nil on failure. @@ -37,307 +26,421 @@ end -- -- Prints `cmd` if `trace` is enabled. local function _run(cmd, die_on_fail, die_msg) - if _trace then - p("run: " .. vim.inspect(cmd)) - end - local rv = vim.trim(vim.fn.system(cmd)) or "" - if vim.v.shell_error ~= 0 then - if die_on_fail then - if _trace then - p(rv) - end - p(die_msg) - die() - end - return nil - end - return rv + if _trace then + p('run: ' .. vim.inspect(cmd)) + end + local rv = vim.trim(vim.fn.system(cmd)) or '' + if vim.v.shell_error ~= 0 then + if die_on_fail then + if _trace then + p(rv) + end + p(die_msg) + die() + end + return nil + end + return rv end -- Run a command, return nil on failure local function run(cmd) - return _run(cmd, false, "") + return _run(cmd, false, '') end -- Run a command, die on failure with err_msg local function run_die(cmd, err_msg) - return _run(cmd, true, err_msg) + return _run(cmd, true, err_msg) end local function require_executable(cmd) - local cmd_path = run_die({ "command", "-v", cmd }, cmd .. " not found!") - run_die({ "test", "-x", cmd_path }, cmd .. " is not executable") + local cmd_path = run_die({ 'command', '-v', cmd }, cmd .. ' not found!') + run_die({ 'test', '-x', cmd_path }, cmd .. ' is not executable') end local function rm_file_if_present(path_to_file) - run({ "rm", "-f", path_to_file }) + run({ 'rm', '-f', path_to_file }) end local nvim_src_dir = vim.fn.getcwd() -local temp_dir = nvim_src_dir .. "/tmp" -run({ "mkdir", "-p", temp_dir }) +local temp_dir = nvim_src_dir .. '/tmp' +run({ 'mkdir', '-p', temp_dir }) local function get_dependency(dependency_name) - local dependency_table = { - ["LuaJIT"] = { - repo = "LuaJIT/LuaJIT", - symbol = "LUAJIT", - }, - ["libuv"] = { - repo = "libuv/libuv", - symbol = "LIBUV", - }, - ["Luv"] = { - repo = "luvit/luv", - symbol = "LUV", - }, - ["tree-sitter"] = { - repo = "tree-sitter/tree-sitter", - symbol = "TREESITTER", - }, - } - local dependency = dependency_table[dependency_name] - if dependency == nil then - p("Not a dependency: " .. dependency_name) - die() - end - dependency.name = dependency_name - return dependency + local dependency_table = { + ['LuaJIT'] = { + repo = 'LuaJIT/LuaJIT', + symbol = 'LUAJIT', + }, + ['libuv'] = { + repo = 'libuv/libuv', + symbol = 'LIBUV', + }, + ['Luv'] = { + repo = 'luvit/luv', + symbol = 'LUV', + }, + ['tree-sitter'] = { + repo = 'tree-sitter/tree-sitter', + symbol = 'TREESITTER', + }, + } + local dependency = dependency_table[dependency_name] + if dependency == nil then + p('Not a dependency: ' .. dependency_name) + die() + end + dependency.name = dependency_name + return dependency end local function get_gh_commit_sha(repo, ref) - require_executable("gh") + require_executable('gh') - local sha = run_die( - { "gh", "api", "repos/" .. repo .. "/commits/" .. ref, "--jq", ".sha" }, - "Failed to get commit hash from GitHub. Not a valid ref?" - ) - return sha + local sha = run_die( + { 'gh', 'api', 'repos/' .. repo .. '/commits/' .. ref, '--jq', '.sha' }, + 'Failed to get commit hash from GitHub. Not a valid ref?' + ) + return sha end local function get_archive_info(repo, ref) - require_executable("curl") - - local archive_name = ref .. ".tar.gz" - local archive_path = temp_dir .. "/" .. archive_name - local archive_url = "https://github.com/" .. repo .. "/archive/" .. archive_name - - rm_file_if_present(archive_path) - run_die({ "curl", "-sL", archive_url, "-o", archive_path }, "Failed to download archive from GitHub") - - local archive_sha = run({ "sha256sum", archive_path }):gmatch("%w+")() - return { url = archive_url, sha = archive_sha } + require_executable('curl') + + local archive_name = ref .. '.tar.gz' + local archive_path = temp_dir .. '/' .. archive_name + local archive_url = 'https://github.com/' .. repo .. '/archive/' .. archive_name + + rm_file_if_present(archive_path) + run_die( + { 'curl', '-sL', archive_url, '-o', archive_path }, + 'Failed to download archive from GitHub' + ) + + local shacmd = (vim.fn.executable('sha256sum') == 1 + and{ 'sha256sum', archive_path } + or { 'shasum', '-a', '256', archive_path }) + local archive_sha = run(shacmd):gmatch('%w+')() + return { url = archive_url, sha = archive_sha } end local function write_cmakelists_line(symbol, kind, value) - require_executable("sed") - - local cmakelists_path = nvim_src_dir .. "/" .. "cmake.deps/CMakeLists.txt" - run_die({ - "sed", - "-i", - "-e", - "s/set(" .. symbol .. "_" .. kind .. ".*$" .. "/set(" .. symbol .. "_" .. kind .. " " .. value .. ")" .. "/", - cmakelists_path, - }, "Failed to write " .. cmakelists_path) + require_executable('sed') + + local cmakelists_path = nvim_src_dir .. '/' .. 'cmake.deps/CMakeLists.txt' + run_die({ + 'sed', + '-i', + '-e', + 's/set(' + .. symbol + .. '_' + .. kind + .. '.*$' + .. '/set(' + .. symbol + .. '_' + .. kind + .. ' ' + .. value + .. ')' + .. '/', + cmakelists_path, + }, 'Failed to write ' .. cmakelists_path) end local function explicit_create_branch(dep) - require_executable("git") - - local checked_out_branch = run({ "git", "rev-parse", "--abbrev-ref", "HEAD" }) - if checked_out_branch ~= "master" then - p("Not on master!") - die() - end - run_die({ "git", "checkout", "-b", "bump-" .. dep }, "git failed to create branch") + require_executable('git') + + local checked_out_branch = run({ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' }) + if checked_out_branch ~= 'master' then + p('Not on master!') + die() + end + run_die({ 'git', 'checkout', '-b', 'bump-' .. dep }, 'git failed to create branch') end local function verify_branch(new_branch_suffix) - require_executable("git") - - local checked_out_branch = run({ "git", "rev-parse", "--abbrev-ref", "HEAD" }) - if not checked_out_branch:match("^" .. required_branch_prefix) then - p("Current branch '" .. checked_out_branch .. "' doesn't seem to start with " .. required_branch_prefix) - p("Checking out to bump-" .. new_branch_suffix) - explicit_create_branch(new_branch_suffix) - end + require_executable('git') + + local checked_out_branch = assert(run({ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' })) + if not checked_out_branch:match('^' .. required_branch_prefix) then + p( + "Current branch '" + .. checked_out_branch + .. "' doesn't seem to start with " + .. required_branch_prefix + ) + p('Checking out to bump-' .. new_branch_suffix) + explicit_create_branch(new_branch_suffix) + end end local function update_cmakelists(dependency, archive, comment) - require_executable("git") - - verify_branch(dependency.name) - - local changed_file = nvim_src_dir .. "/" .. "cmake.deps/CMakeLists.txt" - - p("Updating " .. dependency.name .. " to " .. archive.url .. "\n") - write_cmakelists_line(dependency.symbol, "URL", archive.url:gsub("/", "\\/")) - write_cmakelists_line(dependency.symbol, "SHA256", archive.sha) - run_die( - { "git", "commit", changed_file, "-m", commit_prefix .. "bump " .. dependency.name .. " to " .. comment }, - "git failed to commit" - ) + require_executable('git') + + verify_branch(dependency.name) + + local changed_file = nvim_src_dir .. '/' .. 'cmake.deps/CMakeLists.txt' + + p('Updating ' .. dependency.name .. ' to ' .. archive.url .. '\n') + write_cmakelists_line(dependency.symbol, 'URL', archive.url:gsub('/', '\\/')) + write_cmakelists_line(dependency.symbol, 'SHA256', archive.sha) + run_die( + { + 'git', + 'commit', + changed_file, + '-m', + commit_prefix .. 'bump ' .. dependency.name .. ' to ' .. comment, + }, + 'git failed to commit' + ) end local function verify_cmakelists_committed() - require_executable("git") + require_executable('git') - local cmakelists_path = nvim_src_dir .. "/" .. "cmake.deps/CMakeLists.txt" - run_die({ "git", "diff", "--quiet", "HEAD", "--", cmakelists_path }, cmakelists_path .. " has uncommitted changes") + local cmakelists_path = nvim_src_dir .. '/' .. 'cmake.deps/CMakeLists.txt' + run_die( + { 'git', 'diff', '--quiet', 'HEAD', '--', cmakelists_path }, + cmakelists_path .. ' has uncommitted changes' + ) end local function warn_luv_symbol() - p("warning: " .. get_dependency("Luv").symbol .. "_VERSION will not be updated") + p('warning: ' .. get_dependency('Luv').symbol .. '_VERSION will not be updated') end -- return first 9 chars of commit local function short_commit(commit) - return string.sub(commit, 1, 9) + return string.sub(commit, 1, 9) end -- TODO: remove hardcoded fork local function gh_pr(pr_title, pr_body) - require_executable("gh") - - local pr_url = run_die({ - "gh", - "pr", - "create", - "--title", - pr_title, - "--body", - pr_body, - }, "Failed to create PR") - return pr_url + require_executable('gh') + + local pr_url = run_die({ + 'gh', + 'pr', + 'create', + '--title', + pr_title, + '--body', + pr_body, + }, 'Failed to create PR') + return pr_url end local function find_git_remote(fork) - require_executable("git") - - local remotes = run({ "git", "remote", "-v" }) - local git_remote = "" - for remote in remotes:gmatch("[^\r\n]+") do - local words = {} - for word in remote:gmatch("%w+") do - table.insert(words, word) - end - local match = words[1]:match("/github.com[:/]neovim/neovim/") - if fork == "fork" then - match = not match - end - if match and words[3] == "(fetch)" then - git_remote = words[0] - break - end - end - if git_remote == "" then - git_remote = "origin" - end - return git_remote + require_executable('git') + + local remotes = assert(run({ 'git', 'remote', '-v' })) + local git_remote = '' + for remote in remotes:gmatch('[^\r\n]+') do + local words = {} + for word in remote:gmatch('%w+') do + table.insert(words, word) + end + local match = words[1]:match('/github.com[:/]neovim/neovim/') + if fork == 'fork' then + match = not match + end + if match and words[3] == '(fetch)' then + git_remote = words[0] + break + end + end + if git_remote == '' then + git_remote = 'origin' + end + return git_remote end local function create_pr(pr_title, pr_body) - require_executable("git") - - local push_first = true - - local checked_out_branch = run({ "git", "rev-parse", "--abbrev-ref", "HEAD" }) - if push_first then - local push_remote = run({ "git", "config", "--get", "branch." .. checked_out_branch .. ".pushRemote" }) - if push_remote == nil then - push_remote = run({ "git", "config", "--get", "remote.pushDefault" }) - if push_remote == nil then - push_remote = run({ "git", "config", "--get", "branch." .. checked_out_branch .. ".remote" }) - if push_remote == nil or push_remote == find_git_remote(nil) then - push_remote = find_git_remote("fork") - end - end - end - - p("Pushing to " .. push_remote .. "/" .. checked_out_branch) - run_die({ "git", "push", push_remote, checked_out_branch }, "Git failed to push") - end - - local pr_url = gh_pr(pr_title, pr_body) - p("\nCreated PR: " .. pr_url .. "\n") + require_executable('git') + + local push_first = true + + local checked_out_branch = run({ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' }) + if push_first then + local push_remote = + run({ 'git', 'config', '--get', 'branch.' .. checked_out_branch .. '.pushRemote' }) + if push_remote == nil then + push_remote = run({ 'git', 'config', '--get', 'remote.pushDefault' }) + if push_remote == nil then + push_remote = + run({ 'git', 'config', '--get', 'branch.' .. checked_out_branch .. '.remote' }) + if push_remote == nil or push_remote == find_git_remote(nil) then + push_remote = find_git_remote('fork') + end + end + end + + p('Pushing to ' .. push_remote .. '/' .. checked_out_branch) + run_die({ 'git', 'push', push_remote, checked_out_branch }, 'Git failed to push') + end + + local pr_url = gh_pr(pr_title, pr_body) + p('\nCreated PR: ' .. pr_url .. '\n') end function M.commit(dependency_name, commit) - local dependency = get_dependency(dependency_name) - verify_cmakelists_committed() - local commit_sha = get_gh_commit_sha(dependency.repo, commit) - if commit_sha ~= commit then - p("Not a commit: " .. commit .. ". Did you mean version?") - die() - end - local archive = get_archive_info(dependency.repo, commit) - if dependency_name == "Luv" then - warn_luv_symbol() - end - update_cmakelists(dependency, archive, short_commit(commit)) + local dependency = assert(get_dependency(dependency_name)) + verify_cmakelists_committed() + local commit_sha = get_gh_commit_sha(dependency.repo, commit) + if commit_sha ~= commit then + p('Not a commit: ' .. commit .. '. Did you mean version?') + die() + end + local archive = get_archive_info(dependency.repo, commit) + if dependency_name == 'Luv' then + warn_luv_symbol() + end + update_cmakelists(dependency, archive, short_commit(commit)) end function M.version(dependency_name, version) - local dependency = get_dependency(dependency_name) - verify_cmakelists_committed() - local commit_sha = get_gh_commit_sha(dependency.repo, version) - if commit_sha == version then - p("Not a version: " .. version .. ". Did you mean commit?") - die() - end - local archive = get_archive_info(dependency.repo, version) - if dependency_name == "Luv" then - write_cmakelists_line(dependency.symbol, "VERSION", version) - end - update_cmakelists(dependency, archive, version) + vim.validate{ + dependency_name={dependency_name,'s'}, + version={version,'s'}, + } + local dependency = assert(get_dependency(dependency_name)) + verify_cmakelists_committed() + local commit_sha = get_gh_commit_sha(dependency.repo, version) + if commit_sha == version then + p('Not a version: ' .. version .. '. Did you mean commit?') + die() + end + local archive = get_archive_info(dependency.repo, version) + if dependency_name == 'Luv' then + write_cmakelists_line(dependency.symbol, 'VERSION', version) + end + update_cmakelists(dependency, archive, version) end function M.head(dependency_name) - local dependency = get_dependency(dependency_name) - verify_cmakelists_committed() - local commit_sha = get_gh_commit_sha(dependency.repo, "HEAD") - local archive = get_archive_info(dependency.repo, commit_sha) - if dependency_name == "Luv" then - warn_luv_symbol() - end - update_cmakelists(dependency, archive, "HEAD - " .. short_commit(commit_sha)) + local dependency = assert(get_dependency(dependency_name)) + verify_cmakelists_committed() + local commit_sha = get_gh_commit_sha(dependency.repo, 'HEAD') + local archive = get_archive_info(dependency.repo, commit_sha) + if dependency_name == 'Luv' then + warn_luv_symbol() + end + update_cmakelists(dependency, archive, 'HEAD - ' .. short_commit(commit_sha)) end function M.create_branch(dep) - explicit_create_branch(dep) + explicit_create_branch(dep) end function M.submit_pr() - require_executable("git") - - verify_branch("deps") - - local nvim_remote = find_git_remote(nil) - local relevant_commit = run_die({ - "git", - "log", - "--grep=" .. commit_prefix, - "--reverse", - "--format='%s'", - nvim_remote .. "/master..HEAD", - "-1", - }, "Failed to fetch commits") - - local pr_title - local pr_body - - if relevant_commit == "" then - pr_title = commit_prefix .. "bump some dependencies" - pr_body = "bump some dependencies" - else - relevant_commit = relevant_commit:gsub("'", "") - pr_title = relevant_commit - pr_body = relevant_commit:gsub(commit_prefix:gsub("%(", "%%("):gsub("%)", "%%)"), "") - end - pr_body = pr_body .. "\n\n(add explanations if needed)" - p(pr_title .. "\n" .. pr_body .. "\n") - create_pr(pr_title, pr_body) + require_executable('git') + + verify_branch('deps') + + local nvim_remote = find_git_remote(nil) + local relevant_commit = assert(run_die({ + 'git', + 'log', + '--grep=' .. commit_prefix, + '--reverse', + "--format='%s'", + nvim_remote .. '/master..HEAD', + '-1', + }, 'Failed to fetch commits')) + + local pr_title + local pr_body + + if relevant_commit == '' then + pr_title = commit_prefix .. 'bump some dependencies' + pr_body = 'bump some dependencies' + else + relevant_commit = relevant_commit:gsub("'", '') + pr_title = relevant_commit + pr_body = relevant_commit:gsub(commit_prefix:gsub('%(', '%%('):gsub('%)', '%%)'), '') + end + pr_body = pr_body .. '\n\n(add explanations if needed)' + p(pr_title .. '\n' .. pr_body .. '\n') + create_pr(pr_title, pr_body) +end + +local function usage() + local this_script = _G.arg[0]:match("[^/]*.lua$") + print(([=[ + Bump Nvim dependencies + + Usage: nvim -l %s [options] + Bump to HEAD, tagged version, commit, or branch: + nvim -l %s --dep Luv --head + nvim -l %s --dep Luv --version 1.43.0-0 + nvim -l %s --dep Luv --commit abc123 + nvim -l %s --dep Luv --branch + Create a PR: + nvim -l %s --pr + + Options: + -h show this message and exit. + --pr submit pr for bumping deps. + --branch <dep> create a branch bump-<dep> from current branch. + --dep <dependency> bump to a specific release or tag. + + Dependency Options: + --version <tag> bump to a specific release or tag. + --commit <hash> bump to a specific commit. + --HEAD bump to a current head. + + <dependency> is one of: + "LuaJIT", "libuv", "Luv", "tree-sitter" + ]=]):format(this_script, this_script, this_script, this_script, this_script, this_script)) +end + +local function parseargs() + local args = {} + for i = 1, #_G.arg do + if _G.arg[i] == '-h' then + args.h = true + elseif _G.arg[i] == '--pr' then + args.pr = true + elseif _G.arg[i] == '--branch' then + args.branch = _G.arg[i+1] + elseif _G.arg[i] == '--dep' then + args.dep = _G.arg[i+1] + elseif _G.arg[i] == '--version' then + args.version = _G.arg[i+1] + elseif _G.arg[i] == '--commit' then + args.commit = _G.arg[i+1] + elseif _G.arg[i] == '--head' then + args.head = true + end + end + return args end -return M +local is_main = _G.arg[0]:match('bump_deps.lua') + +if is_main then + local args = parseargs() + if args.h then + usage() + elseif args.pr then + M.submit_pr() + elseif args.head then + M.head(args.dep) + elseif args.branch then + M.create_branch(args.dep) + elseif args.version then + M.version(args.dep, args.version) + elseif args.commit then + M.commit(args.dep, args.commit) + elseif args.pr then + M.submit_pr() + else + print('missing required arg\n') + os.exit(1) + end +else + return M +end diff --git a/scripts/check-includes.py b/scripts/check-includes.py deleted file mode 100755 index ed1fe407c5..0000000000 --- a/scripts/check-includes.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python - -import sys -import re -import os - -from subprocess import Popen, PIPE -from argparse import ArgumentParser - - -GENERATED_INCLUDE_RE = re.compile( - r'^\s*#\s*include\s*"([/a-z_0-9.]+\.generated\.h)"(\s+//.*)?$') - - -def main(argv): - argparser = ArgumentParser() - argparser.add_argument('--generated-includes-dir', action='append', - help='Directory where generated includes are located.') - argparser.add_argument('--file', type=open, help='File to check.') - argparser.add_argument('iwyu_args', nargs='*', - help='IWYU arguments, must go after --.') - args = argparser.parse_args(argv) - - with args.file: - iwyu = Popen(['include-what-you-use', '-xc'] + args.iwyu_args + ['/dev/stdin'], - stdin=PIPE, stdout=PIPE, stderr=PIPE) - - for line in args.file: - match = GENERATED_INCLUDE_RE.match(line) - if match: - for d in args.generated_includes_dir: - try: - f = open(os.path.join(d, match.group(1))) - except IOError: - continue - else: - with f: - for generated_line in f: - iwyu.stdin.write(generated_line) - break - else: - raise IOError('Failed to find {0}'.format(match.group(1))) - else: - iwyu.stdin.write(line) - - iwyu.stdin.close() - - out = iwyu.stdout.read() - err = iwyu.stderr.read() - - ret = iwyu.wait() - - if ret != 2: - print('IWYU failed with exit code {0}:'.format(ret)) - print('{0} stdout {0}'.format('=' * ((80 - len(' stdout ')) // 2))) - print(out) - print('{0} stderr {0}'.format('=' * ((80 - len(' stderr ')) // 2))) - print(err) - return 1 - return 0 - - -if __name__ == '__main__': - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/cliff.toml b/scripts/cliff.toml new file mode 100644 index 0000000000..3fc10e5d16 --- /dev/null +++ b/scripts/cliff.toml @@ -0,0 +1,73 @@ +# configuration file for git-cliff (0.1.0) + +[changelog] +# changelog header +header = """ +# Changelog\n +All notable changes to this project will be documented in this file.\n +""" +# template for the changelog body +# https://tera.netlify.app/docs/#introduction +body = """ +{% if version %}\ + ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} +{% else %}\ + ## [unreleased] +{% endif %}\ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | upper_first }} + {% for commit in commits%}\ + {% if not commit.scope %}\ + - {{ commit.message | upper_first }} + {% endif %}\ + {% endfor %}\ + {% for group, commits in commits | group_by(attribute="scope") %}\ + {% for commit in commits %}\ + - **{{commit.scope}}**: {{ commit.message | upper_first }} + {% endfor %}\ + {% endfor %} +{% endfor %}\n +""" +# remove the leading and trailing whitespace from the template +trim = true +# changelog footer +footer = """ +<!-- generated by git-cliff --> +""" + +[git] +# parse the commits based on https://www.conventionalcommits.org +conventional_commits = true +# filter out the commits that are not conventional +filter_unconventional = true +# process each line of a commit as an individual commit +split_commits = false +# regex for preprocessing the commit messages +commit_preprocessors = [ +# { pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](https://github.com/neovim/neovim/issues/${2}))"}, +] +# regex for parsing and grouping commits +commit_parsers = [ + { message = "!:", group = "Breaking"}, + { message = "^feat", group = "Features"}, + { message = "^fix", group = "Bug Fixes"}, + { message = "^doc", group = "Documentation"}, + { message = "^perf", group = "Performance"}, + { message = "^refactor", group = "Refactor"}, + { message = "^test", group = "Testing"}, + { message = "^chore", group = "Miscellaneous Tasks"}, + { message = "^build", group = "Build System"}, + { message = "^Revert", group = "Reverted Changes"}, +] +# filter out the commits that are not matched by commit parsers +filter_commits = true +# glob pattern for matching git tags +tag_pattern = "v[0-9]*" +# regex for skipping tags +skip_tags = "v0.1.0-beta.1" +# regex for ignoring tags +ignore_tags = "" +# sort the tags chronologically +date_order = false +# sort the commits inside sections by oldest/newest order +sort_commits = "oldest" diff --git a/scripts/download-unicode-files.sh b/scripts/download-unicode-files.sh index 4482cefa34..f0fd4c66ea 100755 --- a/scripts/download-unicode-files.sh +++ b/scripts/download-unicode-files.sh @@ -3,11 +3,12 @@ set -e data_files="UnicodeData.txt CaseFolding.txt EastAsianWidth.txt" emoji_files="emoji-data.txt" +files="'$data_files $emoji_files'" UNIDIR_DEFAULT=src/unicode DOWNLOAD_URL_BASE_DEFAULT='http://unicode.org/Public' -if test x$1 = 'x--help' ; then +if test "$1" = '--help' ; then echo 'Usage:' echo " $0[ TARGET_DIRECTORY[ URL_BASE]]" echo @@ -16,6 +17,7 @@ if test x$1 = 'x--help' ; then echo echo "Default target directory is $PWD/${UNIDIR_DEFAULT}." echo "Default URL base is ${DOWNLOAD_URL_BASE_DEFAULT}." + exit 0 fi UNIDIR=${1:-$UNIDIR_DEFAULT} @@ -23,21 +25,12 @@ DOWNLOAD_URL_BASE=${2:-$DOWNLOAD_URL_BASE_DEFAULT} for filename in $data_files ; do curl -L -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/UNIDATA/$filename" - ( - cd "$UNIDIR" - git add $filename - ) + git -C "$UNIDIR" add "$filename" done for filename in $emoji_files ; do curl -L -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/UNIDATA/emoji/$filename" - ( - cd "$UNIDIR" - git add $filename - ) + git -C "$UNIDIR" add "$filename" done -( - cd "$UNIDIR" - git commit -m "feat: update unicode tables" -- $files -) +git -C "$UNIDIR" commit -m "feat: update unicode tables" . diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua new file mode 100644 index 0000000000..2563f2f410 --- /dev/null +++ b/scripts/gen_help_html.lua @@ -0,0 +1,1110 @@ +-- Converts Vim :help files to HTML. Validates |tag| links and document syntax (parser errors). +-- +-- NOTE: :helptags checks for duplicate tags, whereas this script checks _links_ (to tags). +-- +-- USAGE (GENERATE HTML): +-- 1. Run `make helptags` first; this script depends on vim.fn.taglist(). +-- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')" +-- - Read the docstring at gen(). +-- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 +-- 4. Visit http://localhost:4000/…/help.txt.html +-- +-- USAGE (VALIDATE): +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate()" +-- - validate() is 10x faster than gen(), so it is used in CI. +-- +-- SELF-TEST MODE: +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +-- +-- NOTES: +-- * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too +-- slow (~1 min) to run in per-commit CI. +-- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. +-- * visit_validate() is the core function used by validate(). +-- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. + +local tagmap = nil +local helpfiles = nil +local invalid_links = {} +local invalid_urls = {} +local invalid_spelling = {} +local spell_dict = { + Neovim = 'Nvim', + NeoVim = 'Nvim', + neovim = 'Nvim', + lua = 'Lua', + VimL = 'Vimscript', +} +local language = nil + +local M = {} + +-- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). +-- All other files are "legacy" files which require fixed-width layout. +local new_layout = { + ['api.txt'] = true, + ['channel.txt'] = true, + ['deprecated.txt'] = true, + ['develop.txt'] = true, + ['lua.txt'] = true, + ['luaref.txt'] = true, + ['news.txt'] = true, + ['nvim.txt'] = true, + ['pi_health.txt'] = true, + ['provider.txt'] = true, + ['ui.txt'] = true, +} + +-- TODO: These known invalid |links| require an update to the relevant docs. +local exclude_invalid = { + ["'previewpopup'"] = "quickref.txt", + ["'pvp'"] = "quickref.txt", + ["'string'"] = "eval.txt", + Query = 'treesitter.txt', + ['eq?'] = 'treesitter.txt', + ['lsp-request'] = 'lsp.txt', + matchit = 'vim_diff.txt', + ['matchit.txt'] = 'help.txt', + ["set!"] = "treesitter.txt", + ['v:_null_blob'] = 'builtin.txt', + ['v:_null_dict'] = 'builtin.txt', + ['v:_null_list'] = 'builtin.txt', + ['v:_null_string'] = 'builtin.txt', + ['vim.lsp.buf_request()'] = 'lsp.txt', + ['vim.lsp.util.get_progress_messages()'] = 'lsp.txt', +} + +-- False-positive "invalid URLs". +local exclude_invalid_urls = { + ["http://"] = "usr_23.txt", + ["http://."] = "usr_23.txt", + ["http://aspell.net/man-html/Affix-Compression.html"] = "spell.txt", + ["http://aspell.net/man-html/Phonetic-Code.html"] = "spell.txt", + ["http://canna.sourceforge.jp/"] = "mbyte.txt", + ["http://gnuada.sourceforge.net"] = "ft_ada.txt", + ["http://lua-users.org/wiki/StringLibraryTutorial"] = "lua.txt", + ["http://michael.toren.net/code/"] = "pi_tar.txt", + ["http://papp.plan9.de"] = "syntax.txt", + ["http://wiki.services.openoffice.org/wiki/Dictionaries"] = "spell.txt", + ["http://www.adapower.com"] = "ft_ada.txt", + ["http://www.jclark.com/"] = "quickfix.txt", +} + +local function tofile(fname, text) + local f = io.open(fname, 'w') + if not f then + error(('failed to write: %s'):format(f)) + else + f:write(text) + f:close() + end +end + +local function html_esc(s) + return s:gsub( + '&', '&'):gsub( + '<', '<'):gsub( + '>', '>') +end + +local function url_encode(s) + -- Credit: tpope / vim-unimpaired + -- NOTE: these chars intentionally *not* escaped: ' ( ) + return vim.fn.substitute(vim.fn.iconv(s, 'latin1', 'utf-8'), + [=[[^A-Za-z0-9()'_.~-]]=], + [=[\="%".printf("%02X",char2nr(submatch(0)))]=], + 'g') +end + +local function expandtabs(s) + return s:gsub('\t', (' '):rep(8)) +end + +local function to_titlecase(s) + local text = '' + for w in vim.gsplit(s, '[ \t]+') do + text = ('%s %s%s'):format(text, vim.fn.toupper(w:sub(1, 1)), w:sub(2)) + end + return text +end + +local function to_heading_tag(text) + -- Prepend "_" to avoid conflicts with actual :help tags. + return text and string.format('_%s', vim.fn.tolower((text:gsub('%s+', '-')))) or 'unknown' +end + +local function basename_noext(f) + return vim.fs.basename(f:gsub('%.txt', '')) +end + +local function is_blank(s) + return not not s:find([[^[\t ]*$]]) +end + +local function trim(s, dir) + return vim.fn.trim(s, '\r\t\n ', dir or 0) +end + +-- Remove common punctuation from URLs. +-- +-- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc +-- +-- @returns (fixed_url, removed_chars) where `removed_chars` is in the order found in the input. +local function fix_url(url) + local removed_chars = '' + local fixed_url = url + -- Remove up to one of each char from end of the URL, in this order. + for _, c in ipairs({ '.', ')', }) do + if fixed_url:sub(-1) == c then + removed_chars = c .. removed_chars + fixed_url = fixed_url:sub(1, -2) + end + end + return fixed_url, removed_chars +end + +-- Checks if a given line is a "noise" line that doesn't look good in HTML form. +local function is_noise(line, noise_lines) + if ( + -- First line is always noise. + (noise_lines ~= nil and vim.tbl_count(noise_lines) == 0) + or line:find('Type .*gO.* to see the table of contents') + -- Title line of traditional :help pages. + -- Example: "NVIM REFERENCE MANUAL by ..." + or line:find([[^%s*N?VIM[ \t]*REFERENCE[ \t]*MANUAL]]) + -- First line of traditional :help pages. + -- Example: "*api.txt* Nvim" + or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') + -- modeline + -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" + or line:find('^%s*vim?%:.*ft=help') + or line:find('^%s*vim?%:.*filetype=help') + or line:find('[*>]local%-additions[*<]') + ) then + -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) + table.insert(noise_lines or {}, line) + return true + end + return false +end + +-- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content. +local function get_bug_url_vimdoc(fname, to_fname, sample_text) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ('https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + ..vim.fs.basename(fname) + ..'+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' + ..this_url + ..'%0D%0DContext%3A%0D%0D%60%60%60%0D' + ..url_encode(sample_text) + ..'%0D%60%60%60') + return bug_url +end + +-- Creates a github issue URL at neovim/neovim with prefilled content. +local function get_bug_url_nvim(fname, to_fname, sample_text, token_name) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ('https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' + ..vim.fs.basename(fname) + ..'+&body=%60gen_help_html.lua%60+problem+at%3A+' + ..this_url + ..'%0D' + ..(token_name and '+unhandled+token%3A+%60'..token_name..'%60' or '') + ..'%0DContext%3A%0D%0D%60%60%60%0D' + ..url_encode(sample_text) + ..'%0D%60%60%60') + return bug_url +end + +-- Gets a "foo.html" name from a "foo.txt" helpfile name. +local function get_helppage(f) + if not f then + return nil + end + -- Special case: help.txt is the "main landing page" of :help files, not index.txt. + if f == 'index.txt' then + return 'vimindex.html' + elseif f == 'help.txt' then + return 'index.html' + end + + return (f:gsub('%.txt$', '.html')) +end + +-- Counts leading spaces (tab=8) to decide the indent size of multiline text. +-- +-- Blank lines (empty or whitespace-only) are ignored. +local function get_indent(s) + local min_indent = nil + for line in vim.gsplit(s, '\n') do + if line and not is_blank(line) then + local ws = expandtabs(line:match('^%s+') or '') + min_indent = (not min_indent or ws:len() < min_indent) and ws:len() or min_indent + end + end + return min_indent or 0 +end + +-- Removes the common indent level, after expanding tabs to 8 spaces. +local function trim_indent(s) + local indent_size = get_indent(s) + local trimmed = '' + for line in vim.gsplit(s, '\n') do + line = expandtabs(line) + trimmed = ('%s%s\n'):format(trimmed, line:sub(indent_size + 1)) + end + return trimmed:sub(1, -2) +end + +-- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string. +local function getbuflinestr(node, bufnr, offset) + local line1, _, line2, _ = node:range() + line1 = line1 - offset + line2 = line2 + offset + local lines = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1) + return table.concat(lines, '\n') +end + +-- Gets the whitespace just before `node` from the raw buffer text. +-- Needed for preformatted `old` lines. +local function getws(node, bufnr) + local line1, c1, line2, _ = node:range() + local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] + local text_before = raw:sub(1, c1) + local leading_ws = text_before:match('%s+$') or '' + return leading_ws +end + +local function get_tagname(node, bufnr) + local text = vim.treesitter.get_node_text(node, bufnr) + local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(text) or text + local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" + return helppage, tag +end + +-- Returns true if the given invalid tagname is a false positive. +local function ignore_invalid(s) + return not not ( + exclude_invalid[s] + -- Strings like |~/====| appear in various places and the parser thinks they are links, but they + -- are just table borders. + or s:find('===') + or s:find('---') + ) +end + +local function ignore_parse_error(s) + return ( + -- Ignore parse errors for unclosed tag. + -- This is common in vimdocs and is treated as plaintext by :help. + s:find("^[`'|*]") + ) +end + +local function has_ancestor(node, ancestor_name) + local p = node + while true do + p = p:parent() + if not p or p:type() == 'help_file' then + break + elseif p:type() == ancestor_name then + return true + end + end + return false +end + +-- Gets the first matching child node matching `name`. +local function first(node, name) + for c, _ in node:iter_children() do + if c:named() and c:type() == name then + return c + end + end + return nil +end + +local function validate_link(node, bufnr, fname) + local helppage, tagname = get_tagname(node:child(1), bufnr) + local ignored = false + if not tagmap[tagname] then + ignored = has_ancestor(node, 'column_heading') or node:has_error() or ignore_invalid(tagname) + if not ignored then + invalid_links[tagname] = vim.fs.basename(fname) + end + end + return helppage, tagname, ignored +end + +-- TODO: port the logic from scripts/check_urls.vim +local function validate_url(text, fname) + local ignored = false + if vim.fs.basename(fname) == 'pi_netrw.txt' then + ignored = true + elseif text:find('http%:') and not exclude_invalid_urls[text] then + invalid_urls[text] = vim.fs.basename(fname) + end + return ignored +end + +-- Traverses the tree at `root` and checks that |tag| links point to valid helptags. +local function visit_validate(root, level, lang_tree, opt, stats) + level = level or 0 + local node_name = (root.named and root:named()) and root:type() or nil + local toplevel = level < 1 + local function node_text(node) + return vim.treesitter.get_node_text(node or root, opt.buf) + end + local text = trim(node_text()) + + if root:child_count() > 0 then + for node, _ in root:iter_children() do + if node:named() then + visit_validate(node, level + 1, lang_tree, opt, stats) + end + end + end + + if node_name == 'ERROR' then + if ignore_parse_error(text) then + return + end + -- Store the raw text to give context to the error report. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + table.insert(stats.parse_errors, sample_text) + elseif node_name == 'word' or node_name == 'uppercase_name' then + if spell_dict[text] then + if not invalid_spelling[text] then + invalid_spelling[text] = { vim.fs.basename(opt.fname) } + else + table.insert(invalid_spelling[text], vim.fs.basename(opt.fname)) + end + end + elseif node_name == 'url' then + local fixed_url, _ = fix_url(trim(text)) + validate_url(fixed_url, opt.fname) + elseif node_name == 'taglink' or node_name == 'optionlink' then + local _, _, _ = validate_link(root, opt.buf, opt.fname) + end +end + +-- Fix tab alignment issues caused by concealed characters like |, `, * in tags +-- and code blocks. +local function fix_tab_after_conceal(text, next_node_text) + -- Vim tabs take into account the two concealed characters even though they + -- are invisible, so we need to add back in the two spaces if this is + -- followed by a tab to make the tab alignment to match Vim's behavior. + if string.sub(next_node_text,1,1) == '\t' then + text = text .. ' ' + end + return text +end + +-- Generates HTML from node `root` recursively. +local function visit_node(root, level, lang_tree, headings, opt, stats) + level = level or 0 + + local node_name = (root.named and root:named()) and root:type() or nil + -- Previous sibling kind (string). + local prev = root:prev_sibling() and (root:prev_sibling().named and root:prev_sibling():named()) and root:prev_sibling():type() or nil + -- Next sibling kind (string). + local next_ = root:next_sibling() and (root:next_sibling().named and root:next_sibling():named()) and root:next_sibling():type() or nil + -- Parent kind (string). + local parent = root:parent() and root:parent():type() or nil + local text = '' + local trimmed + -- Gets leading whitespace of `node`. + local function ws(node) + node = node or root + local ws_ = getws(node, opt.buf) + -- XXX: first node of a (line) includes whitespace, even after + -- https://github.com/neovim/tree-sitter-vimdoc/pull/31 ? + if ws_ == '' then + ws_ = vim.treesitter.get_node_text(node, opt.buf):match('^%s+') or '' + end + return ws_ + end + local function node_text(node, ws_) + node = node or root + ws_ = (ws_ == nil or ws_ == true) and getws(node, opt.buf) or '' + return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf)) + end + + if root:named_child_count() == 0 or node_name == 'ERROR' then + text = node_text() + trimmed = html_esc(trim(text)) + text = html_esc(text) + else + -- Process children and join them with whitespace. + for node, _ in root:iter_children() do + if node:named() then + local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) + text = string.format('%s%s', text, r) + end + end + trimmed = trim(text) + end + + if node_name == 'help_file' then -- root node + return text + elseif node_name == 'url' then + local fixed_url, removed_chars = fix_url(trimmed) + return ('%s<a href="%s">%s</a>%s'):format(ws(), fixed_url, fixed_url, removed_chars) + elseif node_name == 'word' or node_name == 'uppercase_name' then + return text + elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then + if is_noise(text, stats.noise_lines) then + return '' -- Discard common "noise" lines. + end + -- Remove "===" and tags from ToC text. + local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', '')) + -- Use the first *tag* node as the heading anchor, if any. + local tagnode = first(root, 'tag') + local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) or to_heading_tag(hname) + if node_name == 'h1' or #headings == 0 then + table.insert(headings, { name = hname, subheadings = {}, tag = tagname }) + else + table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, tag = tagname }) + end + local el = node_name == 'h1' and 'h2' or 'h3' + -- If we are re-using the *tag*, this heading anchor is redundant. + local a = tagnode and '' or ('<a name="%s"></a>'):format(tagname) + return ('%s<%s class="help-heading">%s</%s>\n'):format(a, el, text, el) + elseif node_name == 'column_heading' or node_name == 'column_name' then + if root:has_error() then + return text + end + return ('<div class="help-column_heading">%s</div>'):format(text) + elseif node_name == 'block' then + if is_blank(text) then + return '' + end + if opt.old then + -- XXX: Treat "old" docs as preformatted: they use indentation for layout. + -- Trim trailing newlines to avoid too much whitespace between divs. + return ('<div class="old-help-para">%s</div>\n'):format(trim(text, 2)) + end + return string.format('<div class="help-para">\n%s\n</div>\n', text) + elseif node_name == 'line' then + if (parent ~= 'codeblock' or parent ~= 'code') and (is_blank(text) or is_noise(text, stats.noise_lines)) then + return '' -- Discard common "noise" lines. + end + -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout. + local div = opt.old and root:child(0) and vim.tbl_contains({'column_heading', 'h1', 'h2', 'h3'}, root:child(0):type()) + return string.format('%s%s', div and trim(text) or text, div and '' or '\n') + elseif node_name == 'line_li' then + local sib = root:prev_sibling() + local prev_li = sib and sib:type() == 'line_li' + + if not prev_li then + opt.indent = 1 + else + -- The previous listitem _sibling_ is _logically_ the _parent_ if it is indented less. + local parent_indent = get_indent(node_text(sib)) + local this_indent = get_indent(node_text()) + if this_indent > parent_indent then + opt.indent = opt.indent + 1 + elseif this_indent < parent_indent then + opt.indent = math.max(1, opt.indent - 1) + end + end + local margin = opt.indent == 1 and '' or ('margin-left: %drem;'):format((1.5 * opt.indent)) + + return string.format('<div class="help-li" style="%s">%s</div>', margin, text) + elseif node_name == 'taglink' or node_name == 'optionlink' then + local helppage, tagname, ignored = validate_link(root, opt.buf, opt.fname) + if ignored then + return text + end + local s = ('%s<a href="%s#%s">%s</a>'):format(ws(), helppage, url_encode(tagname), html_esc(tagname)) + if opt.old and node_name == 'taglink' then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + return s + elseif vim.tbl_contains({'codespan', 'keycode'}, node_name) then + if root:has_error() then + return text + end + local s = ('%s<code>%s</code>'):format(ws(), trimmed) + if opt.old and node_name == 'codespan' then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + return s + elseif node_name == 'argument' then + return ('%s<code>{%s}</code>'):format(ws(), text) + elseif node_name == 'codeblock' then + return text + elseif node_name == 'language' then + language = node_text(root) + return '' + elseif node_name == 'code' then + if is_blank(text) then + return '' + end + local code + if language then + code = ('<pre><code class="language-%s">%s</code></pre>'):format(language,trim(trim_indent(text), 2)) + language = nil + else + code = ('<pre>%s</pre>'):format(trim(trim_indent(text), 2)) + end + return code + elseif node_name == 'tag' then -- anchor + if root:has_error() then + return text + end + local in_heading = vim.tbl_contains({'h1', 'h2', 'h3'}, parent) + local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' + local tagname = node_text(root:child(1), false) + if vim.tbl_count(stats.first_tags) < 2 then + -- Force the first 2 tags in the doc to be anchored at the main heading. + table.insert(stats.first_tags, tagname) + return '' + end + local el = in_heading and 'span' or 'code' + local s = ('%s<a name="%s"></a><%s class="%s">%s</%s>'):format(ws(), url_encode(tagname), el, cssclass, trimmed, el) + if opt.old then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + + if in_heading and prev ~= 'tag' then + -- Start the <span> container for tags in a heading. + -- This makes "justify-content:space-between" right-align the tags. + -- <h2>foo bar<span>tag1 tag2</span></h2> + return string.format('<span class="help-heading-tags">%s', s) + elseif in_heading and next_ == nil then + -- End the <span> container for tags in a heading. + return string.format('%s</span>', s) + end + return s + elseif node_name == 'ERROR' then + if ignore_parse_error(trimmed) then + return text + end + + -- Store the raw text to give context to the bug report. + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' + table.insert(stats.parse_errors, sample_text) + return ('<a class="parse-error" target="_blank" title="Report bug... (parse error)" href="%s">%s</a>'):format( + get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) + else -- Unknown token. + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' + return ('<a class="unknown-token" target="_blank" title="Report bug... (unhandled token "%s")" href="%s">%s</a>'):format( + node_name, get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), trimmed), ('unknown-token:"%s"'):format(node_name) + end +end + +local function get_helpfiles(include) + local dir = './build/runtime/doc' + local rv = {} + for f, type in vim.fs.dir(dir) do + if (vim.endswith(f, '.txt') + and type == 'file' + and (not include or vim.tbl_contains(include, f))) then + local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') + table.insert(rv, fullpath) + end + end + return rv +end + +-- Populates the helptags map. +local function get_helptags(help_dir) + local m = {} + -- Load a random help file to convince taglist() to do its job. + vim.cmd(string.format('split %s/api.txt', help_dir)) + vim.cmd('lcd %:p:h') + for _, item in ipairs(vim.fn.taglist('.*')) do + if vim.endswith(item.filename, '.txt') then + m[item.name] = item.filename + end + end + vim.cmd('q!') + return m +end + +-- Use the help.so parser defined in the build, not whatever happens to be installed on the system. +local function ensure_runtimepath() + if not vim.o.runtimepath:find('build/lib/nvim/') then + vim.cmd[[set runtimepath^=./build/lib/nvim/]] + end +end + +-- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents. +-- +-- @returns lang_tree, bufnr +local function parse_buf(fname) + local buf + if type(fname) == 'string' then + vim.cmd('split '..vim.fn.fnameescape(fname)) -- Filename. + buf = vim.api.nvim_get_current_buf() + else + buf = fname + vim.cmd('sbuffer '..tostring(fname)) -- Buffer number. + end + -- vim.treesitter.require_language('help', './build/lib/nvim/parser/help.so') + local lang_tree = vim.treesitter.get_parser(buf, 'help') + return lang_tree, buf +end + +-- Validates one :help file `fname`: +-- - checks that |tag| links point to valid helptags. +-- - recursively counts parse errors ("ERROR" nodes) +-- +-- @returns { invalid_links: number, parse_errors: number } +local function validate_one(fname) + local stats = { + parse_errors = {}, + } + local lang_tree, buf = parse_buf(fname) + for _, tree in ipairs(lang_tree:trees()) do + visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname, }, stats) + end + lang_tree:destroy() + vim.cmd.close() + return stats +end + +-- Generates HTML from one :help file `fname` and writes the result to `to_fname`. +-- +-- @param fname Source :help file +-- @param to_fname Destination .html file +-- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) +-- +-- @returns html, stats +local function gen_one(fname, to_fname, old, commit) + local stats = { + noise_lines = {}, + parse_errors = {}, + first_tags = {}, -- Track the first few tags in doc. + } + local lang_tree, buf = parse_buf(fname) + local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. + local title = to_titlecase(basename_noext(fname)) + + local html = ([[ + <!DOCTYPE html> + <html> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="description" content="Neovim user documentation"> + <link href="/css/normalize.min.css" rel="stylesheet"> + <link href="/css/bootstrap.css" rel="stylesheet"> + <link href="/css/main.css" rel="stylesheet"> + <link href="help.css" rel="stylesheet"> + <link href="/highlight/styles/neovim.min.css" rel="stylesheet"> + <script src="/highlight/highlight.min.js"></script> + <script>hljs.highlightAll();</script> + <title>%s - Neovim docs</title> + </head> + <body> + ]]):format(title) + + local logo_svg = [[ + <svg xmlns="http://www.w3.org/2000/svg" role="img" width="173" height="50" viewBox="0 0 742 214" aria-label="Neovim"> + <title>Neovim</title> + <defs> + <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="a"> + <stop stop-color="#16B0ED" stop-opacity=".8" offset="0%" /> + <stop stop-color="#0F59B2" stop-opacity=".837" offset="100%" /> + </linearGradient> + <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="b"> + <stop stop-color="#7DB643" offset="0%" /> + <stop stop-color="#367533" offset="100%" /> + </linearGradient> + <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="c"> + <stop stop-color="#88C649" stop-opacity=".8" offset="0%" /> + <stop stop-color="#439240" stop-opacity=".84" offset="100%" /> + </linearGradient> + </defs> + <g fill="none" fill-rule="evenodd"> + <path + d="M.027 45.459L45.224-.173v212.171L.027 166.894V45.459z" + fill="url(#a)" + transform="translate(1 1)" + /> + <path + d="M129.337 45.89L175.152-.149l-.928 212.146-45.197-45.104.31-121.005z" + fill="url(#b)" + transform="matrix(-1 0 0 1 305 1)" + /> + <path + d="M45.194-.137L162.7 179.173l-32.882 32.881L12.25 33.141 45.194-.137z" + fill="url(#c)" + transform="translate(1 1)" + /> + <path + d="M46.234 84.032l-.063 7.063-36.28-53.563 3.36-3.422 32.983 49.922z" + fill-opacity=".13" + fill="#000" + /> + <g fill="#444"> + <path + d="M227 154V64.44h4.655c1.55 0 2.445.75 2.685 2.25l.806 13.502c4.058-5.16 8.786-9.316 14.188-12.466 5.4-3.15 11.413-4.726 18.037-4.726 4.893 0 9.205.781 12.935 2.34 3.729 1.561 6.817 3.811 9.264 6.751 2.448 2.942 4.297 6.48 5.55 10.621 1.253 4.14 1.88 8.821 1.88 14.042V154h-8.504V96.754c0-8.402-1.91-14.987-5.729-19.757-3.82-4.771-9.667-7.156-17.544-7.156-5.851 0-11.28 1.516-16.292 4.545-5.013 3.032-9.489 7.187-13.427 12.467V154H227zM350.624 63c5.066 0 9.755.868 14.069 2.605 4.312 1.738 8.052 4.268 11.219 7.592s5.638 7.412 7.419 12.264C385.11 90.313 386 95.883 386 102.17c0 1.318-.195 2.216-.588 2.696-.393.48-1.01.719-1.851.719h-64.966v1.70c0 6.708.784 12.609 2.353 17.7 1.567 5.09 3.8 9.357 6.695 12.802 2.895 3.445 6.393 6.034 10.495 7.771 4.1 1.738 8.686 2.606 13.752 2.606 4.524 0 8.446-.494 11.762-1.483 3.317-.988 6.108-2.097 8.37-3.324 2.261-1.227 4.056-2.336 5.383-3.324 1.326-.988 2.292-1.482 2.895-1.482.784 0 1.388.3 1.81.898l2.352 2.875c-1.448 1.797-3.362 3.475-5.745 5.031-2.383 1.558-5.038 2.891-7.962 3.998-2.926 1.109-6.062 1.991-9.41 2.65a52.21 52.21 0 01-10.088.989c-6.152 0-11.762-1.064-16.828-3.19-5.067-2.125-9.415-5.225-13.043-9.298-3.63-4.074-6.435-9.06-8.415-14.96C310.99 121.655 310 114.9 310 107.294c0-6.408.92-12.323 2.76-17.744 1.84-5.421 4.493-10.093 7.961-14.016 3.467-3.922 7.72-6.991 12.758-9.209C338.513 64.11 344.229 63 350.624 63zm.573 6c-4.696 0-8.904.702-12.623 2.105-3.721 1.404-6.936 3.421-9.65 6.053-2.713 2.631-4.908 5.79-6.586 9.474S319.55 94.439 319 99h60c0-4.679-.672-8.874-2.013-12.588-1.343-3.712-3.232-6.856-5.67-9.43-2.44-2.571-5.367-4.545-8.782-5.92-3.413-1.374-7.192-2.062-11.338-2.062zM435.546 63c6.526 0 12.368 1.093 17.524 3.28 5.154 2.186 9.5 5.286 13.04 9.298 3.538 4.013 6.238 8.85 8.099 14.51 1.861 5.66 2.791 11.994 2.791 19.002 0 7.008-.932 13.327-2.791 18.957-1.861 5.631-4.561 10.452-8.099 14.465-3.54 4.012-7.886 7.097-13.04 9.254-5.156 2.156-10.998 3.234-17.524 3.234-6.529 0-12.369-1.078-17.525-3.234-5.155-2.157-9.517-5.242-13.085-9.254-3.57-4.013-6.285-8.836-8.145-14.465-1.861-5.63-2.791-11.95-2.791-18.957 0-7.008.93-13.342 2.791-19.002 1.861-5.66 4.576-10.496 8.145-14.51 3.568-4.012 7.93-7.112 13.085-9.299C423.177 64.094 429.017 63 435.546 63zm-.501 86c5.341 0 10.006-.918 13.997-2.757 3.99-1.838 7.32-4.474 9.992-7.909 2.67-3.435 4.664-7.576 5.986-12.428 1.317-4.85 1.98-10.288 1.98-16.316 0-5.965-.66-11.389-1.98-16.27-1.322-4.88-3.316-9.053-5.986-12.519-2.67-3.463-6-6.13-9.992-7.999-3.991-1.867-8.657-2.802-13.997-2.802s-10.008.935-13.997 2.802c-3.991 1.87-7.322 4.536-9.992 8-2.671 3.465-4.68 7.637-6.03 12.518-1.35 4.881-2.026 10.305-2.026 16.27 0 6.026.675 11.465 2.025 16.316 1.35 4.852 3.36 8.993 6.031 12.428 2.67 3.435 6 6.07 9.992 7.91 3.99 1.838 8.656 2.756 13.997 2.756z" + fill="currentColor" + /> + <path + d="M530.57 152h-20.05L474 60h18.35c1.61 0 2.967.39 4.072 1.166 1.103.778 1.865 1.763 2.283 2.959l17.722 49.138a92.762 92.762 0 012.551 8.429c.686 2.751 1.298 5.5 1.835 8.25.537-2.75 1.148-5.499 1.835-8.25a77.713 77.713 0 012.64-8.429l18.171-49.138c.417-1.196 1.164-2.181 2.238-2.96 1.074-.776 2.356-1.165 3.849-1.165H567l-36.43 92zM572 61h23v92h-23zM610 153V60.443h13.624c2.887 0 4.78 1.354 5.682 4.06l1.443 6.856a52.7 52.7 0 015.097-4.962 32.732 32.732 0 015.683-3.879 30.731 30.731 0 016.496-2.57c2.314-.632 4.855-.948 7.624-.948 5.832 0 10.63 1.579 14.39 4.736 3.758 3.157 6.57 7.352 8.434 12.585 1.444-3.068 3.248-5.698 5.413-7.894 2.165-2.194 4.541-3.984 7.127-5.367a32.848 32.848 0 018.254-3.068 39.597 39.597 0 018.796-.992c5.111 0 9.653.783 13.622 2.345 3.97 1.565 7.307 3.849 10.014 6.857 2.706 3.007 4.766 6.675 6.18 11.005C739.29 83.537 740 88.5 740 94.092V153h-22.284V94.092c0-5.894-1.294-10.329-3.878-13.306-2.587-2.977-6.376-4.465-11.368-4.465-2.286 0-4.404.391-6.358 1.172a15.189 15.189 0 00-5.144 3.383c-1.473 1.474-2.631 3.324-3.474 5.548-.842 2.225-1.263 4.781-1.263 7.668V153h-22.37V94.092c0-6.194-1.249-10.704-3.744-13.532-2.497-2.825-6.18-4.24-11.051-4.24-3.19 0-6.18.798-8.976 2.391-2.799 1.593-5.399 3.775-7.804 6.54V153H610zM572 30h23v19h-23z" + fill="currentColor" + fill-opacity=".8" + /> + </g> + </g> + </svg> + ]] + + local main = '' + for _, tree in ipairs(lang_tree:trees()) do + main = main .. (visit_node(tree:root(), 0, tree, headings, + { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1, }, + stats)) + end + + main = ([[ + <header class="container"> + <nav class="navbar navbar-expand-lg"> + <div> + <a href="/" class="navbar-brand" aria-label="logo"> + <!--TODO: use <img src="….svg"> here instead. Need one that has green lettering instead of gray. --> + %s + <!--<img src="https://neovim.io/logos/neovim-logo.svg" width="173" height="50" alt="Neovim" />--> + </a> + </div> + </nav> + </header> + + <div class="container golden-grid help-body"> + <div class="col-wide"> + <a name="%s"></a><a name="%s"></a><h1>%s</h1> + <p> + <i> + Nvim <code>:help</code> pages, <a href="https://github.com/neovim/neovim/blob/master/scripts/gen_help_html.lua">generated</a> + from <a href="https://github.com/neovim/neovim/blob/master/runtime/doc/%s">source</a> + using the <a href="https://github.com/neovim/tree-sitter-vimdoc">tree-sitter-vimdoc</a> parser. + </i> + </p> + <hr/> + %s + </div> + ]]):format(logo_svg, stats.first_tags[1] or '', stats.first_tags[2] or '', title, vim.fs.basename(fname), main) + + local toc = [[ + <div class="col-narrow toc"> + <div><a href="index.html">Main</a></div> + <div><a href="vimindex.html">Commands index</a></div> + <div><a href="quickref.html">Quick reference</a></div> + <hr/> + ]] + + local n = 0 -- Count of all headings + subheadings. + for _, h1 in ipairs(headings) do n = n + 1 + #h1.subheadings end + for _, h1 in ipairs(headings) do + toc = toc .. ('<div class="help-toc-h1"><a href="#%s">%s</a>\n'):format(h1.tag, h1.name) + if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. + for _, h2 in ipairs(h1.subheadings) do + toc = toc .. ('<div class="help-toc-h2"><a href="#%s">%s</a></div>\n'):format(h2.tag, h2.name) + end + end + toc = toc .. '</div>' + end + toc = toc .. '</div>\n' + + local bug_url = get_bug_url_nvim(fname, to_fname, 'TODO', nil) + local bug_link = string.format('(<a href="%s" target="_blank">report docs bug...</a>)', bug_url) + + local footer = ([[ + <footer> + <div class="container flex"> + <div class="generator-stats"> + Generated at %s from <code><a href="https://github.com/neovim/neovim/commit/%s">%s</a></code> + </div> + <div class="generator-stats"> + parse_errors: %d %s | <span title="%s">noise_lines: %d</span> + </div> + <div> + </footer> + ]]):format( + os.date('%Y-%m-%d %H:%M'), commit, commit:sub(1, 7), #stats.parse_errors, bug_link, + html_esc(table.concat(stats.noise_lines, '\n')), #stats.noise_lines) + + html = ('%s%s%s</div>\n%s</body>\n</html>\n'):format( + html, main, toc, footer) + vim.cmd('q!') + lang_tree:destroy() + return html, stats +end + +local function gen_css(fname) + local css = [[ + :root { + --code-color: #004b4b; + --tag-color: #095943; + } + @media (prefers-color-scheme: dark) { + :root { + --code-color: #00c243; + --tag-color: #00b7b7; + } + } + @media (min-width: 40em) { + .toc { + position: fixed; + left: 67%; + } + .golden-grid { + display: grid; + grid-template-columns: 65% auto; + grid-gap: 1em; + } + } + @media (max-width: 40em) { + .golden-grid { + /* Disable grid for narrow viewport (mobile phone). */ + display: block; + } + } + .toc { + /* max-width: 12rem; */ + height: 85%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + } + .toc > div { + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + } + html { + scroll-behavior: auto; + } + body { + font-size: 18px; + line-height: 1.5; + } + h1, h2, h3, h4, h5 { + font-family: sans-serif; + border-bottom: 1px solid var(--tag-color); /*rgba(0, 0, 0, .9);*/ + } + h3, h4, h5 { + border-bottom-style: dashed; + } + .help-column_heading { + color: var(--code-color); + } + .help-body { + padding-bottom: 2em; + } + .help-line { + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + } + .help-li { + white-space: normal; + display: list-item; + margin-left: 1.5rem; /* padding-left: 1rem; */ + } + .help-para { + padding-top: 10px; + padding-bottom: 10px; + } + .old-help-para { + padding-top: 10px; + padding-bottom: 10px; + /* Tabs are used for alignment in old docs, so we must match Vim's 8-char expectation. */ + tab-size: 8; + white-space: pre; + font-size: 16px; + font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; + } + a.help-tag, a.help-tag:focus, a.help-tag:hover { + color: inherit; + text-decoration: none; + } + .help-tag { + color: var(--tag-color); + } + /* Tag pseudo-header common in :help docs. */ + .help-tag-right { + color: var(--tag-color); + } + h1 .help-tag, h2 .help-tag, h3 .help-tag { + font-size: smaller; + } + .help-heading { + overflow: hidden; + white-space: nowrap; + display: flex; + justify-content: space-between; + } + /* The (right-aligned) "tags" part of a section heading. */ + .help-heading-tags { + margin-right: 10px; + } + .help-toc-h1 { + } + .help-toc-h2 { + margin-left: 1em; + } + .parse-error { + background-color: red; + } + .unknown-token { + color: black; + background-color: yellow; + } + code { + color: var(--code-color); + font-size: 16px; + } + pre { + /* Tabs are used in codeblocks only for indentation, not alignment, so we can aggressively shrink them. */ + tab-size: 2; + white-space: pre-wrap; + line-height: 1.3; /* Important for ascii art. */ + overflow: visible; + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + font-size: 16px; + margin-top: 10px; + } + pre:hover, + .help-heading:hover { + overflow: visible; + } + .generator-stats { + color: gray; + font-size: smaller; + } + ]] + tofile(fname, css) +end + +function M._test() + tagmap = get_helptags('./build/runtime/doc') + helpfiles = get_helpfiles() + + local function ok(cond, expected, actual) + assert((not expected and not actual) or (expected and actual), 'if "expected" is given, "actual" is also required') + if expected then + return assert(cond, ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual))) + else + return assert(cond) + end + end + local function eq(expected, actual) + return ok(expected == actual, expected, actual) + end + + ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap)) + ok(vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), tagmap['vim.diagnostic.set()'], 'diagnostic.txt') + ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt') + ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]])) + ok(is_noise([[ NVIM REFERENCE MANUAL by Thiago de Arruda ]])) + ok(not is_noise([[vim:tw=78]])) + + eq(0, get_indent('a')) + eq(1, get_indent(' a')) + eq(2, get_indent(' a\n b\n c\n')) + eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) + eq('a\n \n b\n c\n d\n e\n', trim_indent(' a\n \n b\n c\n d\n e\n')) + + local fixed_url, removed_chars = fix_url('https://example.com).') + eq('https://example.com', fixed_url) + eq(').', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com.)') + eq('https://example.com.', fixed_url) + eq(')', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com.') + eq('https://example.com', fixed_url) + eq('.', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com)') + eq('https://example.com', fixed_url) + eq(')', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com') + eq('https://example.com', fixed_url) + eq('', removed_chars) + + print('all tests passed') +end + +--- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`. +--- +--- Example: +--- +--- gen('./build/runtime/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil) +--- +--- @param help_dir string Source directory containing the :help files. Must run `make helptags` first. +--- @param to_dir string Target directory where the .html files will be written. +--- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} +--- +--- @returns info dict +function M.gen(help_dir, to_dir, include, commit) + vim.validate{ + help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, + to_dir={to_dir, 's'}, + include={include, 't', true}, + commit={commit, 's', true}, + } + + local err_count = 0 + ensure_runtimepath() + tagmap = get_helptags(help_dir) + helpfiles = get_helpfiles(include) + + print(('output dir: %s'):format(to_dir)) + vim.fn.mkdir(to_dir, 'p') + gen_css(('%s/help.css'):format(to_dir)) + + for _, f in ipairs(helpfiles) do + local helpfile = vim.fs.basename(f) + local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) + local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?') + tofile(to_fname, html) + print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname))) + err_count = err_count + #stats.parse_errors + end + print(('generated %d html pages'):format(#helpfiles)) + print(('total errors: %d'):format(err_count)) + print(('invalid tags:\n%s'):format(vim.inspect(invalid_links))) + + return { + helpfiles = helpfiles, + err_count = err_count, + invalid_links = invalid_links, + } +end + +-- Validates all :help files found in `help_dir`: +-- - checks that |tag| links point to valid helptags. +-- - recursively counts parse errors ("ERROR" nodes) +-- +-- This is 10x faster than gen(), for use in CI. +-- +-- @returns results dict +function M.validate(help_dir, include) + vim.validate{ + help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, + include={include, 't', true}, + } + local err_count = 0 + ensure_runtimepath() + tagmap = get_helptags(help_dir) + helpfiles = get_helpfiles(include) + + for _, f in ipairs(helpfiles) do + local helpfile = vim.fs.basename(f) + local rv = validate_one(f) + print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile)) + err_count = err_count + #rv.parse_errors + end + + return { + helpfiles = #helpfiles, + err_count = err_count, + invalid_links = invalid_links, + invalid_urls = invalid_urls, + invalid_spelling = invalid_spelling, + } +end + +return M diff --git a/scripts/gen_help_html.py b/scripts/gen_help_html.py deleted file mode 100644 index 0b8e77ac22..0000000000 --- a/scripts/gen_help_html.py +++ /dev/null @@ -1,389 +0,0 @@ -# Converts Vim/Nvim documentation to HTML. -# -# USAGE: -# 1. python3 scripts/gen_help_html.py runtime/doc/ ~/neovim.github.io/t/ -# 3. cd ~/neovim.github.io/ && jekyll serve --host 0.0.0.0 -# 2. Visit http://localhost:4000/t/help.txt.html -# -# Adapted from https://github.com/c4rlo/vimhelp/ -# License: MIT -# -# Copyright (c) 2016 Carlo Teubner -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import re -import urllib.parse -import datetime -import sys -from itertools import chain - -HEAD = """\ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" - "http://www.w3.org/TR/html4/loose.dtd"> -<html> -<head> -<meta http-equiv="Content-type" content="text/html; charset={encoding}"/> -<style> -.h {{ - font-weight: bold; -}} -h1 {{ - font-family: sans-serif; -}} -pre {{ - font-family: sans-serif; -}} -</style> -<title>Nvim: {filename}</title> -""" - -HEAD_END = '</head>\n<body>\n' - -INTRO = """ -<h1>Nvim help files</h1> -<p> -<a href="https://neovim.io/">Nvim</a> help pages{vers-note}. -Updated <a href="https://github.com/neovim/bot-ci" class="d">automatically</a> -from the <a href="https://github.com/neovim/neovim" class="d">Nvim source</a>. -</p> -""" - -VERSION_NOTE = ", current as of Nvim {version}" - -SITENAVI_LINKS = """ -<a href="quickref.txt.html">Quick reference</a> · -<a href="usr_toc.txt.html">User manual</a> · -<a href="{helptxt}#reference_toc">Reference manual</a> · -""" - -SITENAVI_LINKS_PLAIN = SITENAVI_LINKS.format(helptxt='help.txt.html') -SITENAVI_LINKS_WEB = SITENAVI_LINKS.format(helptxt='/') - -SITENAVI_PLAIN = '<p>' + SITENAVI_LINKS_PLAIN + '</p>' -SITENAVI_WEB = '<p>' + SITENAVI_LINKS_WEB + '</p>' - -SITENAVI_SEARCH = '<table width="100%"><tbody><tr><td>' + SITENAVI_LINKS_WEB + \ - '</td><td style="text-align: right; max-width: 25vw"><div class="gcse-searchbox">' \ - '</div></td></tr></tbody></table><div class="gcse-searchresults"></div>' - -TEXTSTART = """ -<div id="d1"> -<pre id="sp">""" + (" " * 80) + """</pre> -<div id="d2"> -<pre> -""" - -FOOTER = '</pre>' - -FOOTER2 = """ -<p id="footer">Generated {generated_date} from <code>{commit}</code></p> -</div> -</div> -</body> -</html> -""".format( - generated_date='{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()), - commit='?') - -RE_TAGLINE = re.compile(r'(\S+)\s+(\S+)') - -PAT_WORDCHAR = '[!#-)+-{}~\xC0-\xFF]' - -PAT_HEADER = r'(^.*~$)' -PAT_GRAPHIC = r'(^.* `$)' -PAT_PIPEWORD = r'(?<!\\)\|([#-)!+-~]+)\|' -PAT_STARWORD = r'\*([#-)!+-~]+)\*(?:(?=\s)|$)' -PAT_COMMAND = r'`([^` ]+)`' -PAT_OPTWORD = r"('(?:[a-z]{2,}|t_..)')" -PAT_CTRL = r'(CTRL-(?:W_)?(?:\{char\}|<[A-Za-z]+?>|.)?)' -PAT_SPECIAL = r'(<.+?>|\{.+?}|' \ - r'\[(?:range|line|count|offset|\+?cmd|[-+]?num|\+\+opt|' \ - r'arg|arguments|ident|addr|group)]|' \ - r'(?<=\s)\[[-a-z^A-Z0-9_]{2,}])' -PAT_TITLE = r'(Vim version [0-9.a-z]+|VIM REFERENCE.*)' -PAT_NOTE = r'((?<!' + PAT_WORDCHAR + r')(?:note|NOTE|Notes?):?' \ - r'(?!' + PAT_WORDCHAR + r'))' -PAT_URL = r'((?:https?|ftp)://[^\'"<> \t]+[a-zA-Z0-9/])' -PAT_WORD = r'((?<!' + PAT_WORDCHAR + r')' + PAT_WORDCHAR + r'+' \ - r'(?!' + PAT_WORDCHAR + r'))' - -RE_LINKWORD = re.compile( - PAT_OPTWORD + '|' + - PAT_CTRL + '|' + - PAT_SPECIAL) -RE_TAGWORD = re.compile( - PAT_HEADER + '|' + - PAT_GRAPHIC + '|' + - PAT_PIPEWORD + '|' + - PAT_STARWORD + '|' + - PAT_COMMAND + '|' + - PAT_OPTWORD + '|' + - PAT_CTRL + '|' + - PAT_SPECIAL + '|' + - PAT_TITLE + '|' + - PAT_NOTE + '|' + - PAT_URL + '|' + - PAT_WORD) -RE_NEWLINE = re.compile(r'[\r\n]') -# H1 header "=====…" -# H2 header "-----…" -RE_HRULE = re.compile(r'[-=]{3,}.*[-=]{3,3}$') -RE_EG_START = re.compile(r'(?:.* )?>$') -RE_EG_END = re.compile(r'\S') -RE_SECTION = re.compile(r'[-A-Z .][-A-Z0-9 .()]*(?=\s+\*)') -RE_STARTAG = re.compile(r'\s\*([^ \t|]+)\*(?:\s|$)') -RE_LOCAL_ADD = re.compile(r'LOCAL ADDITIONS:\s+\*local-additions\*$') - - -class Link(object): - __slots__ = 'link_plain_same', 'link_pipe_same', \ - 'link_plain_foreign', 'link_pipe_foreign', \ - 'filename' - - def __init__(self, link_plain_same, link_plain_foreign, - link_pipe_same, link_pipe_foreign, filename): - self.link_plain_same = link_plain_same - self.link_plain_foreign = link_plain_foreign - self.link_pipe_same = link_pipe_same - self.link_pipe_foreign = link_pipe_foreign - self.filename = filename - - -class VimH2H(object): - def __init__(self, tags, version=None, is_web_version=True): - self._urls = {} - self._version = version - self._is_web_version = is_web_version - for line in RE_NEWLINE.split(tags): - m = RE_TAGLINE.match(line) - if m: - tag, filename = m.group(1, 2) - self.do_add_tag(filename, tag) - - def add_tags(self, filename, contents): - for match in RE_STARTAG.finditer(contents): - tag = match.group(1).replace('\\', '\\\\').replace('/', '\\/') - self.do_add_tag(str(filename), tag) - - def do_add_tag(self, filename, tag): - tag_quoted = urllib.parse.quote_plus(tag) - - def mkpart1(doc): - return '<a href="' + doc + '#' + tag_quoted + '" class="' - part1_same = mkpart1('') - if self._is_web_version and filename == 'help.txt': - doc = '/' - else: - doc = filename + '.html' - part1_foreign = mkpart1(doc) - part2 = '">' + html_escape[tag] + '</a>' - - def mklinks(cssclass): - return (part1_same + cssclass + part2, - part1_foreign + cssclass + part2) - cssclass_plain = 'd' - m = RE_LINKWORD.match(tag) - if m: - opt, ctrl, special = m.groups() - if opt is not None: - cssclass_plain = 'o' - elif ctrl is not None: - cssclass_plain = 'k' - elif special is not None: - cssclass_plain = 's' - links_plain = mklinks(cssclass_plain) - links_pipe = mklinks('l') - self._urls[tag] = Link( - links_plain[0], links_plain[1], - links_pipe[0], links_pipe[1], - filename) - - def maplink(self, tag, curr_filename, css_class=None): - links = self._urls.get(tag) - if links is not None: - if links.filename == curr_filename: - if css_class == 'l': - return links.link_pipe_same - else: - return links.link_plain_same - else: - if css_class == 'l': - return links.link_pipe_foreign - else: - return links.link_plain_foreign - elif css_class is not None: - return '<span class="' + css_class + '">' + html_escape[tag] + \ - '</span>' - else: - return html_escape[tag] - - def to_html(self, filename, contents, encoding): - out = [] - - inexample = 0 - filename = str(filename) - is_help_txt = (filename == 'help.txt') - last = '' - for line in RE_NEWLINE.split(contents): - line = line.rstrip('\r\n') - line_tabs = line - line = line.expandtabs() - if last == 'h1': - out.extend(('</pre>')) # XXX - out.extend(('<h1>', line.rstrip(), '</h1>\n')) - out.extend(('<pre>')) - last = '' - continue - if RE_HRULE.match(line): - # out.extend(('<span class="h">', line, '</span>\n')) - last = 'h1' - continue - if inexample == 2: - if RE_EG_END.match(line): - inexample = 0 - if line[0] == '<': - line = line[1:] - else: - out.extend(('<span class="e">', html_escape[line], - '</span>\n')) - continue - if RE_EG_START.match(line_tabs): - inexample = 1 - line = line[0:-1] - if RE_SECTION.match(line_tabs): - m = RE_SECTION.match(line) - out.extend((r'<span class="c">', m.group(0), r'</span>')) - line = line[m.end():] - lastpos = 0 - for match in RE_TAGWORD.finditer(line): - pos = match.start() - if pos > lastpos: - out.append(html_escape[line[lastpos:pos]]) - lastpos = match.end() - header, graphic, pipeword, starword, command, opt, ctrl, \ - special, title, note, url, word = match.groups() - if pipeword is not None: - out.append(self.maplink(pipeword, filename, 'l')) - elif starword is not None: - out.extend(('<a name="', urllib.parse.quote_plus(starword), - '" class="t">', html_escape[starword], '</a>')) - elif command is not None: - out.extend(('<span class="e">', html_escape[command], - '</span>')) - elif opt is not None: - out.append(self.maplink(opt, filename, 'o')) - elif ctrl is not None: - out.append(self.maplink(ctrl, filename, 'k')) - elif special is not None: - out.append(self.maplink(special, filename, 's')) - elif title is not None: - out.extend(('<span class="i">', html_escape[title], - '</span>')) - elif note is not None: - out.extend(('<span class="n">', html_escape[note], - '</span>')) - elif header is not None: - out.extend(('<span class="h">', html_escape[header[:-1]], - '</span>')) - elif graphic is not None: - out.append(html_escape[graphic[:-2]]) - elif url is not None: - out.extend(('<a class="u" href="', url, '">' + - html_escape[url], '</a>')) - elif word is not None: - out.append(self.maplink(word, filename)) - if lastpos < len(line): - out.append(html_escape[line[lastpos:]]) - out.append('\n') - if inexample == 1: - inexample = 2 - - header = [] - header.append(HEAD.format(encoding=encoding, filename=filename)) - header.append(HEAD_END) - if self._is_web_version and is_help_txt: - vers_note = VERSION_NOTE.replace('{version}', self._version) \ - if self._version else '' - header.append(INTRO.replace('{vers-note}', vers_note)) - if self._is_web_version: - header.append(SITENAVI_SEARCH) - sitenavi_footer = SITENAVI_WEB - else: - header.append(SITENAVI_PLAIN) - sitenavi_footer = SITENAVI_PLAIN - header.append(TEXTSTART) - return ''.join(chain(header, out, (FOOTER, sitenavi_footer, FOOTER2))) - - -class HtmlEscCache(dict): - def __missing__(self, key): - r = key.replace('&', '&') \ - .replace('<', '<') \ - .replace('>', '>') - self[key] = r - return r - - -html_escape = HtmlEscCache() - - -def slurp(filename): - try: - with open(filename, encoding='UTF-8') as f: - return f.read(), 'UTF-8' - except UnicodeError: - # 'ISO-8859-1' ? - with open(filename, encoding='latin-1') as f: - return f.read(), 'latin-1' - - -def usage(): - return "usage: " + sys.argv[0] + " IN_DIR OUT_DIR [BASENAMES...]" - - -def main(): - if len(sys.argv) < 3: - sys.exit(usage()) - - in_dir = sys.argv[1] - out_dir = sys.argv[2] - basenames = sys.argv[3:] - - print("Processing tags...") - h2h = VimH2H(slurp(os.path.join(in_dir, 'tags'))[0], is_web_version=False) - - if len(basenames) == 0: - basenames = os.listdir(in_dir) - - for basename in basenames: - if os.path.splitext(basename)[1] != '.txt' and basename != 'tags': - print("Ignoring " + basename) - continue - print("Processing " + basename + "...") - path = os.path.join(in_dir, basename) - text, encoding = slurp(path) - outpath = os.path.join(out_dir, basename + '.html') - of = open(outpath, 'w') - of.write(h2h.to_html(basename, text, encoding)) - of.close() - - -main() diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index bca3dd816f..8e1d6ef80a 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -2,9 +2,7 @@ """Generates Nvim :help docs from C/Lua docstrings, using Doxygen. Also generates *.mpack files. To inspect the *.mpack structure: - - :new | put=v:lua.vim.inspect(msgpackparse(readfile('runtime/doc/api.mpack'))) - + :new | put=v:lua.vim.inspect(v:lua.vim.mpack.unpack(readfile('runtime/doc/api.mpack','B'))) Flow: main @@ -14,15 +12,10 @@ Flow: update_params_map / render_node -This would be easier using lxml and XSLT, but: - - 1. This should avoid needing Python dependencies, especially ones that are - C modules that have library dependencies (lxml requires libxml and - libxslt). - 2. I wouldn't know how to deal with nested indentation in <para> tags using - XSLT. +TODO: eliminate this script and use Lua+treesitter (requires parsers for C and +Lua markdown-style docstrings). -Each function :help block is formatted as follows: +The generated :help text for each function is formatted as follows: - Max width of 78 columns (`text_width`). - Indent with spaces (not tabs). @@ -62,11 +55,19 @@ if sys.version_info < MIN_PYTHON_VERSION: doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"], universal_newlines=True).split()[0].split('.'))) +# Until 0.9 is released, need this hacky way to check that "nvim -l foo.lua" works. +nvim_version = list(line for line in subprocess.check_output(['nvim', '-h'], universal_newlines=True).split('\n') + if '-l ' in line) + if doxygen_version < MIN_DOXYGEN_VERSION: print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION)) print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version)) sys.exit(1) +if len(nvim_version) == 0: + print("\nRequires 'nvim -l' feature, see https://github.com/neovim/neovim/pull/18706") + sys.exit(1) + # DEBUG = ('DEBUG' in os.environ) INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) @@ -86,7 +87,7 @@ base_dir = os.path.dirname(os.path.dirname(script_path)) out_dir = os.path.join(base_dir, 'tmp-{target}-doc') filter_cmd = '%s %s' % (sys.executable, script_path) msgs = [] # Messages to show on exit. -lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter') +lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua') CONFIG = { 'api': { @@ -132,12 +133,14 @@ CONFIG = { 'filename': 'lua.txt', 'section_order': [ '_editor.lua', + '_inspector.lua', 'shared.lua', 'uri.lua', 'ui.lua', 'filetype.lua', 'keymap.lua', 'fs.lua', + 'secure.lua', ], 'files': [ 'runtime/lua/vim/_editor.lua', @@ -147,11 +150,14 @@ CONFIG = { 'runtime/lua/vim/filetype.lua', 'runtime/lua/vim/keymap.lua', 'runtime/lua/vim/fs.lua', + 'runtime/lua/vim/secure.lua', + 'runtime/lua/vim/_inspector.lua', ], 'file_patterns': '*.lua', 'fn_name_prefix': '', 'section_name': { 'lsp.lua': 'core', + '_inspector.lua': 'inspector', }, 'section_fmt': lambda name: ( 'Lua module: vim' @@ -168,11 +174,13 @@ CONFIG = { 'module_override': { # `shared` functions are exposed on the `vim` module. 'shared': 'vim', + '_inspector': 'vim', 'uri': 'vim', 'ui': 'vim.ui', 'filetype': 'vim.filetype', 'keymap': 'vim.keymap', 'fs': 'vim.fs', + 'secure': 'vim.secure', }, 'append_only': [ 'shared.lua', @@ -187,6 +195,7 @@ CONFIG = { 'diagnostic.lua', 'codelens.lua', 'tagfunc.lua', + 'semantic_tokens.lua', 'handlers.lua', 'util.lua', 'log.lua', @@ -260,19 +269,13 @@ CONFIG = { 'helptag_fmt': lambda name: ( '*lua-treesitter-core*' if name.lower() == 'treesitter' - else f'*treesitter-{name.lower()}*'), + else f'*lua-treesitter-{name.lower()}*'), 'fn_helptag_fmt': lambda fstem, name: ( - f'*{name}()*' - if name != 'new' - else f'*{fstem}.{name}()*'), - # 'fn_helptag_fmt': lambda fstem, name: ( - # f'*vim.treesitter.{name}()*' - # if fstem == 'treesitter' - # else ( - # '*vim.lsp.client*' - # # HACK. TODO(justinmk): class/structure support in lua2dox - # if 'lsp.client' == f'{fstem}.{name}' - # else f'*vim.lsp.{fstem}.{name}()*')), + f'*vim.{fstem}.{name}()*' + if fstem == 'treesitter' + else f'*{name}()*' + if name[0].isupper() + else f'*vim.treesitter.{fstem}.{name}()*'), 'module_override': {}, 'append_only': [], } @@ -287,7 +290,7 @@ annotation_map = { 'FUNC_API_FAST': '|api-fast|', 'FUNC_API_CHECK_TEXTLOCK': 'not allowed when |textlock| is active', 'FUNC_API_REMOTE_ONLY': '|RPC| only', - 'FUNC_API_LUA_ONLY': '|vim.api| only', + 'FUNC_API_LUA_ONLY': 'Lua |vim.api| only', } @@ -295,14 +298,16 @@ annotation_map = { # or if `cond()` is callable and returns True. def debug_this(o, cond=True): name = '' + if cond is False: + return if not isinstance(o, str): try: name = o.nodeName o = o.toprettyxml(indent=' ', newl='\n') except Exception: pass - if ((callable(cond) and cond()) - or (not callable(cond) and cond) + if (cond is True + or (callable(cond) and cond()) or (not callable(cond) and cond in o)): raise RuntimeError('xxx: {}\n{}'.format(name, o)) @@ -353,6 +358,17 @@ def self_or_child(n): return n.childNodes[0] +def align_tags(line): + tag_regex = r"\s(\*.+?\*)(?:\s|$)" + tags = re.findall(tag_regex, line) + + if len(tags) > 0: + line = re.sub(tag_regex, "", line) + tags = " " + " ".join(tags) + line = line + (" " * (78 - len(line) - len(tags))) + tags + return line + + def clean_lines(text): """Removes superfluous lines. @@ -498,7 +514,12 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation, if n.nodeName == 'preformatted': o = get_text(n, preformatted=True) ensure_nl = '' if o[-1] == '\n' else '\n' - text += '>{}{}\n<'.format(ensure_nl, o) + if o[0:4] == 'lua\n': + text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1]) + elif o[0:4] == 'vim\n': + text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1]) + else: + text += '>{}{}\n<'.format(ensure_nl, o) elif is_inline(n): text = doc_wrap(get_text(n), indent=indent, width=width) @@ -671,7 +692,7 @@ def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='', max_name_len = max_name(m.keys()) + 4 out = '' for name, desc in m.items(): - name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len)) + name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) out += '{}{}\n'.format(name, desc) return out.rstrip() @@ -801,7 +822,8 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): prefix = '%s(' % name suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error')) + if a[0] not in ('void', 'Error', 'Arena', + 'lua_State')) if not fmt_vimhelp: c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) @@ -887,7 +909,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): def fmt_doxygen_xml_as_vimhelp(filename, target): """Entrypoint for generating Vim :help from from Doxygen XML. - Returns 3 items: + Returns 2 items: 1. Vim help text for functions found in `filename`. 2. Vim help text for deprecated functions. """ @@ -951,7 +973,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): start = end - func_doc = "\n".join(split_lines) + func_doc = "\n".join(map(align_tags, split_lines)) if (name.startswith(CONFIG[target]['fn_name_prefix']) and name != "nvim_error_event"): @@ -979,7 +1001,7 @@ def delete_lines_below(filename, tokenstr): fp.writelines(lines[0:i]) -def main(config, args): +def main(doxygen_config, args): """Generates: 1. Vim :help docs @@ -1007,7 +1029,7 @@ def main(config, args): # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL)) p.communicate( - config.format( + doxygen_config.format( input=' '.join( [f'"{file}"' for file in CONFIG[target]['files']]), output=output_dir, @@ -1094,7 +1116,7 @@ def main(config, args): fn_map_full.update(fn_map) if len(sections) == 0: - fail(f'no sections for target: {target}') + fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') if len(sections) > len(CONFIG[target]['section_order']): raise RuntimeError( 'found new modules "{}"; update the "section_order" map'.format( @@ -1141,7 +1163,7 @@ def main(config, args): def filter_source(filename): name, extension = os.path.splitext(filename) if extension == '.lua': - p = subprocess.run([lua2dox_filter, filename], stdout=subprocess.PIPE) + p = subprocess.run(['nvim', '-l', lua2dox, filename], stdout=subprocess.PIPE) op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8')) print(op) else: diff --git a/scripts/genappimage.sh b/scripts/genappimage.sh index cc88ab5559..9944b5eb31 100755 --- a/scripts/genappimage.sh +++ b/scripts/genappimage.sh @@ -8,7 +8,8 @@ # App arch, used by generate_appimage. if [ -z "$ARCH" ]; then - export ARCH="$(arch)" + ARCH="$(arch)" + export ARCH fi TAG=$1 @@ -34,8 +35,9 @@ make install # App version, used by generate_appimage. VERSION=$("$ROOT_DIR"/build/bin/nvim --version | head -n 1 | grep -o 'v.*') +export VERSION -cd "$APP_BUILD_DIR" +cd "$APP_BUILD_DIR" || exit # Only downloads linuxdeploy if the remote file is different from local if [ -e "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage ]; then @@ -53,7 +55,7 @@ chmod +x "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage mkdir "$APP_DIR/usr/share/metainfo/" cp "$ROOT_DIR/runtime/nvim.appdata.xml" "$APP_DIR/usr/share/metainfo/" -cd "$APP_DIR" +cd "$APP_DIR" || exit ######################################################################## # AppDir complete. Now package it as an AppImage. @@ -71,7 +73,7 @@ exec "$(dirname "$(readlink -f "${0}")")/usr/bin/nvim" ${@+"$@"} EOF chmod 755 AppRun -cd "$APP_BUILD_DIR" # Get out of AppImage directory. +cd "$APP_BUILD_DIR" || exit # Get out of AppImage directory. # Set the name of the file generated by appimage export OUTPUT=nvim.appimage @@ -85,7 +87,7 @@ fi # - Expects: $ARCH, $APP, $VERSION env vars # - Expects: ./$APP.AppDir/ directory # - Produces: ./nvim.appimage -./linuxdeploy-x86_64.AppImage --appdir $APP.AppDir -d $ROOT_DIR/runtime/nvim.desktop -i \ +./linuxdeploy-x86_64.AppImage --appdir $APP.AppDir -d "$ROOT_DIR"/runtime/nvim.desktop -i \ "$ROOT_DIR/runtime/nvim.png" --output appimage # Moving the final executable to a different folder so it isn't in the diff --git a/scripts/genvimvim.lua b/scripts/genvimvim.lua index 868084a583..3e9e7077be 100644 --- a/scripts/genvimvim.lua +++ b/scripts/genvimvim.lua @@ -11,6 +11,8 @@ local funcs_file = arg[3] package.path = nvimsrcdir .. '/?.lua;' .. package.path +_G.vim = loadfile(nvimsrcdir..'/../../runtime/lua/vim/shared.lua')() + local lld = {} local syn_fd = io.open(syntax_file, 'w') lld.line_length = 0 @@ -115,7 +117,7 @@ end local nvimau_start = 'syn keyword nvimAutoEvent contained ' w('\n\n' .. nvimau_start) -for au, _ in pairs(auevents.nvim_specific) do +for au, _ in vim.spairs(auevents.nvim_specific) do if lld.line_length > 850 then w('\n' .. nvimau_start) end @@ -126,7 +128,7 @@ w('\n\nsyn case match') local vimfun_start = 'syn keyword vimFuncName contained ' w('\n\n' .. vimfun_start) local funcs = mpack.unpack(io.open(funcs_file, 'rb'):read("*all")) -for name, _ in pairs(funcs) do +for _, name in ipairs(funcs) do if name then if lld.line_length > 850 then w('\n' .. vimfun_start) diff --git a/scripts/git-log-pretty-since.sh b/scripts/git-log-pretty-since.sh index a0aa4354b6..95dcee23f5 100755 --- a/scripts/git-log-pretty-since.sh +++ b/scripts/git-log-pretty-since.sh @@ -16,9 +16,9 @@ __SINCE=$1 __INVMATCH=$2 is_merge_commit() { - git rev-parse $1 >/dev/null 2>&1 \ + git rev-parse "$1" >/dev/null 2>&1 \ || { echo "ERROR: invalid commit: $1"; exit 1; } - git log $1^2 >/dev/null 2>&1 && return 0 || return 1 + git log "$1"^2 >/dev/null 2>&1 && return 0 || return 1 } # Removes parens from issue/ticket/PR numbers. @@ -40,13 +40,13 @@ _format_ticketnums() { } for commit in $(git log --format='%H' --first-parent "$__SINCE"..HEAD); do - if is_merge_commit ${commit} ; then - if [ -z "$__INVMATCH" ] || ! git log --oneline ${commit}^1..${commit}^2 \ + if is_merge_commit "${commit}" ; then + if [ -z "$__INVMATCH" ] || ! git log --oneline "${commit}^1..${commit}^2" \ | >/dev/null 2>&1 grep -E "$__INVMATCH" ; then - git log -1 --oneline ${commit} - git log --format=' %h %s' ${commit}^1..${commit}^2 + git log -1 --oneline "${commit}" + git log --format=' %h %s' "${commit}^1..${commit}^2" fi else - git log -1 --oneline ${commit} + git log -1 --oneline "${commit}" fi done | _format_ticketnums diff --git a/scripts/lintcommit.lua b/scripts/lintcommit.lua index 16326cfe66..87a8e62503 100644 --- a/scripts/lintcommit.lua +++ b/scripts/lintcommit.lua @@ -86,11 +86,28 @@ local function validate_commit(commit_message) vim.inspect(allowed_types)) end - -- Check if scope is empty + -- Check if scope is appropriate if before_colon:match("%(") then local scope = vim.trim(before_colon:match("%((.*)%)")) + if scope == '' then - return [[Scope can't be empty.]] + return [[Scope can't be empty]] + end + + if vim.startswith(scope, "nvim_") then + return [[Scope should be "api" instead of "nvim_..."]] + end + + local alternative_scope = { + ['filetype.vim'] = 'filetype', + ['filetype.lua'] = 'filetype', + ['tree-sitter'] = 'treesitter', + ['ts'] = 'treesitter', + ['hl'] = 'highlight', + } + + if alternative_scope[scope] then + return ('Scope should be "%s" instead of "%s"'):format(alternative_scope[scope], scope) end end diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 6a206066b8..19f8f8141d 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -27,14 +27,13 @@ http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm Running ------- -This file "lua2dox.lua" gets called by "lua2dox_filter" (bash). +This script "lua2dox.lua" gets called by "gen_vimdoc.py". Doxygen must be on your system. You can experiment like so: - Run "doxygen -g" to create a default Doxyfile. -- Then alter it to let it recognise lua. Add the two following lines: +- Then alter it to let it recognise lua. Add the following line: FILE_PATTERNS = *.lua - FILTER_PATTERNS = *.lua=lua2dox_filter - Then run "doxygen". The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language. @@ -50,108 +49,46 @@ However I have put in a hack that will insert the "missing" close paren. The effect is that you will get the function documented, but not with the parameter list you might expect. ]] -local function class(BaseClass, ClassInitialiser) - local newClass = {} -- a new class newClass - if not ClassInitialiser and type(BaseClass) == 'function' then - ClassInitialiser = BaseClass - BaseClass = nil - elseif type(BaseClass) == 'table' then - -- our new class is a shallow copy of the base class! - for i,v in pairs(BaseClass) do - newClass[i] = v - end - newClass._base = BaseClass - end +local function class() + local newClass = {} -- a new class newClass -- the class will be the metatable for all its newInstanceects, -- and they will look up their methods in it. newClass.__index = newClass -- expose a constructor which can be called by <classname>(<args>) - local classMetatable = {} - classMetatable.__call = function(class_tbl, ...) - local newInstance = {} - setmetatable(newInstance,newClass) - --if init then - -- init(newInstance,...) - if class_tbl.init then - class_tbl.init(newInstance,...) - else - -- make sure that any stuff from the base class is initialized! - if BaseClass and BaseClass.init then - BaseClass.init(newInstance, ...) + setmetatable(newClass, { + __call = function(class_tbl, ...) + local newInstance = {} + setmetatable(newInstance, newClass) + --if init then + -- init(newInstance,...) + if class_tbl.init then + class_tbl.init(newInstance, ...) end + return newInstance end - return newInstance - end - newClass.init = ClassInitialiser - newClass.is_a = function(this, klass) - local thisMetatable = getmetatable(this) - while thisMetatable do - if thisMetatable == klass then - return true - end - thisMetatable = thisMetatable._base - end - return false - end - setmetatable(newClass, classMetatable) + }) return newClass end ---! \class TCore_Clock ---! \brief a clock -local TCore_Clock = class() - ---! \brief get the current time -function TCore_Clock.GetTimeNow() - local gettimeofday = os.gettimeofday -- luacheck: ignore 143 Accessing an undefined field of a global variable. - if gettimeofday then - return gettimeofday() - else - return os.time() - end -end - ---! \brief constructor -function TCore_Clock.init(this,T0) - if T0 then - this.t0 = T0 - else - this.t0 = TCore_Clock.GetTimeNow() - end -end - ---! \brief get time string -function TCore_Clock.getTimeStamp(this,T0) - local t0 - if T0 then - t0 = T0 - else - t0 = this.t0 - end - return os.date('%c %Z',t0) -end - - --! \brief write to stdout local function TCore_IO_write(Str) - if (Str) then + if Str then io.write(Str) end end --! \brief write to stdout local function TCore_IO_writeln(Str) - if (Str) then + if Str then io.write(Str) end - io.write("\n") + io.write('\n') end - --! \brief trims a string local function string_trim(Str) - return Str:match("^%s*(.-)%s*$") + return Str:match('^%s*(.-)%s*$') end --! \brief split a string @@ -159,46 +96,26 @@ end --! \param Str --! \param Pattern --! \returns table of string fragments +---@return string[] local function string_split(Str, Pattern) local splitStr = {} - local fpat = "(.-)" .. Pattern + local fpat = '(.-)' .. Pattern local last_end = 1 - local str, e, cap = string.find(Str,fpat, 1) + local str, e, cap = string.find(Str, fpat, 1) while str do - if str ~= 1 or cap ~= "" then - table.insert(splitStr,cap) + if str ~= 1 or cap ~= '' then + table.insert(splitStr, cap) end - last_end = e+1 - str, e, cap = string.find(Str,fpat, last_end) + last_end = e + 1 + str, e, cap = string.find(Str, fpat, last_end) end if last_end <= #Str then - cap = string.sub(Str,last_end) + cap = string.sub(Str, last_end) table.insert(splitStr, cap) end return splitStr end - ---! \class TCore_Commandline ---! \brief reads/parses commandline -local TCore_Commandline = class() - ---! \brief constructor -function TCore_Commandline.init(this) - this.argv = arg - this.parsed = {} - this.params = {} -end - ---! \brief get value -function TCore_Commandline.getRaw(this,Key,Default) - local val = this.argv[Key] - if not val then - val = Default - end - return val -end - ------------------------------- --! \brief file buffer --! @@ -208,14 +125,13 @@ local TStream_Read = class() --! \brief get contents of file --! --! \param Filename name of file to read (or nil == stdin) -function TStream_Read.getContents(this,Filename) - assert(Filename) +function TStream_Read.getContents(this, Filename) + assert(Filename, ('invalid file: %s'):format(Filename)) -- get lines from file -- syphon lines to our table - --TCore_Debug_show_var('Filename',Filename) - local filecontents={} + local filecontents = {} for line in io.lines(Filename) do - table.insert(filecontents,line) + table.insert(filecontents, line) end if filecontents then @@ -240,7 +156,7 @@ function TStream_Read.getLine(this) this.currentLine = nil else -- get line - if this.currentLineNo<=this.contentsLen then + if this.currentLineNo <= this.contentsLen then line = this.filecontents[this.currentLineNo] this.currentLineNo = this.currentLineNo + 1 else @@ -251,13 +167,13 @@ function TStream_Read.getLine(this) end --! \brief save line fragment -function TStream_Read.ungetLine(this,LineFrag) +function TStream_Read.ungetLine(this, LineFrag) this.currentLine = LineFrag end --! \brief is it eof? function TStream_Read.eof(this) - if this.currentLine or this.currentLineNo<=this.contentsLen then + if this.currentLine or this.currentLineNo <= this.contentsLen then return false end return true @@ -272,32 +188,32 @@ function TStream_Write.init(this) end --! \brief write immediately -function TStream_Write.write(_,Str) +function TStream_Write.write(_, Str) TCore_IO_write(Str) end --! \brief write immediately -function TStream_Write.writeln(_,Str) +function TStream_Write.writeln(_, Str) TCore_IO_writeln(Str) end --! \brief write immediately -function TStream_Write.writelnComment(_,Str) +function TStream_Write.writelnComment(_, Str) TCore_IO_write('// ZZ: ') TCore_IO_writeln(Str) end --! \brief write to tail -function TStream_Write.writelnTail(this,Line) +function TStream_Write.writelnTail(this, Line) if not Line then Line = '' end - table.insert(this.tailLine,Line) + table.insert(this.tailLine, Line) end --! \brief output tail lines function TStream_Write.write_tailLines(this) - for _,line in ipairs(this.tailLine) do + for _, line in ipairs(this.tailLine) do TCore_IO_writeln(line) end TCore_IO_write('// Lua2DoX new eof') @@ -307,9 +223,9 @@ end local TLua2DoX_filter = class() --! \brief allow us to do errormessages -function TLua2DoX_filter.warning(this,Line,LineNo,Legend) +function TLua2DoX_filter.warning(this, Line, LineNo, Legend) this.outStream:writelnTail( - '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"' + '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"' ) end @@ -318,47 +234,47 @@ end --! If the string has a comment on the end, this trims it off. --! local function TString_removeCommentFromLine(Line) - local pos_comment = string.find(Line,'%-%-') + local pos_comment = string.find(Line, '%-%-') local tailComment if pos_comment then - Line = string.sub(Line,1,pos_comment-1) - tailComment = string.sub(Line,pos_comment) + Line = string.sub(Line, 1, pos_comment - 1) + tailComment = string.sub(Line, pos_comment) end - return Line,tailComment + return Line, tailComment end --! \brief get directive from magic local function getMagicDirective(Line) - local macro,tail + local macro, tail local macroStr = '[\\@]' - local pos_macro = string.find(Line,macroStr) + local pos_macro = string.find(Line, macroStr) if pos_macro then --! ....\\ macro...stuff --! ....\@ macro...stuff - local line = string.sub(Line,pos_macro+1) - local space = string.find(line,'%s+') + local line = string.sub(Line, pos_macro + 1) + local space = string.find(line, '%s+') if space then - macro = string.sub(line,1,space-1) - tail = string_trim(string.sub(line,space+1)) + macro = string.sub(line, 1, space - 1) + tail = string_trim(string.sub(line, space + 1)) else macro = line - tail = '' + tail = '' end end - return macro,tail + return macro, tail end --! \brief check comment for fn -local function checkComment4fn(Fn_magic,MagicLines) +local function checkComment4fn(Fn_magic, MagicLines) local fn_magic = Fn_magic -- TCore_IO_writeln('// checkComment4fn "' .. MagicLines .. '"') - local magicLines = string_split(MagicLines,'\n') + local magicLines = string_split(MagicLines, '\n') - local macro,tail + local macro, tail for _, line in ipairs(magicLines) do - macro,tail = getMagicDirective(line) + macro, tail = getMagicDirective(line) if macro == 'fn' then fn_magic = tail -- TCore_IO_writeln('// found fn "' .. fn_magic .. '"') @@ -369,13 +285,16 @@ local function checkComment4fn(Fn_magic,MagicLines) return fn_magic end + +local types = { 'number', 'string', 'table', 'list', 'boolean', 'function' } + --! \brief run the filter -function TLua2DoX_filter.readfile(this,AppStamp,Filename) +function TLua2DoX_filter.readfile(this, AppStamp, Filename) local inStream = TStream_Read() local outStream = TStream_Write() this.outStream = outStream -- save to this obj - if (inStream:getContents(Filename)) then + if inStream:getContents(Filename) then -- output the file local line local fn_magic -- function name/def from magic comment @@ -385,13 +304,14 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) outStream:writelnTail('// #######################') outStream:writelnTail() - local state = '' -- luacheck: ignore 231 variable is set but never accessed. + local state = '' -- luacheck: ignore 231 variable is set but never accessed. local offset = 0 + local generic = {} + local l = 0 while not (inStream:eof()) do line = string_trim(inStream:getLine()) - -- TCore_Debug_show_var('inStream',inStream) - -- TCore_Debug_show_var('line',line ) - if string.sub(line,1,2) == '--' then -- it's a comment + l = l + 1 + if string.sub(line, 1, 2) == '--' then -- it's a comment -- Allow people to write style similar to EmmyLua (since they are basically the same) -- instead of silently skipping things that start with --- if string.sub(line, 3, 3) == '@' then -- it's a magic comment @@ -405,55 +325,87 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local magic = string.sub(line, 4 + offset) local magic_split = string_split(magic, ' ') - - local type_index = 2 if magic_split[1] == 'param' then - type_index = type_index + 1 + for _, type in ipairs(types) do + magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2') + magic = + magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2') + end + magic_split = string_split(magic, ' ') + elseif magic_split[1] == 'return' then + for _, type in ipairs(types) do + magic = magic:gsub('^return%s+.*%((' .. type .. ')%)', 'return %1') + magic = magic:gsub('^return%s+.*%((' .. type .. '|nil)%)', 'return %1') + end + magic_split = string_split(magic, ' ') end - if magic_split[type_index] == 'number' or - magic_split[type_index] == 'number|nil' or - magic_split[type_index] == 'string' or - magic_split[type_index] == 'string|nil' or - magic_split[type_index] == 'table' or - magic_split[type_index] == 'table|nil' or - magic_split[type_index] == 'boolean' or - magic_split[type_index] == 'boolean|nil' or - magic_split[type_index] == 'function' or - magic_split[type_index] == 'function|nil' - then - magic_split[type_index] = '(' .. magic_split[type_index] .. ')' + if magic_split[1] == 'generic' then + local generic_name, generic_type = line:match('@generic%s*(%w+)%s*:?%s*(.*)') + if generic_type == '' then + generic_type = 'any' + end + generic[generic_name] = generic_type + else + local type_index = 2 + if magic_split[1] == 'param' then + type_index = type_index + 1 + end + + if magic_split[type_index] then + -- fix optional parameters + if magic_split[type_index] and magic_split[2]:find('%?$') then + if not magic_split[type_index]:find('nil') then + magic_split[type_index] = magic_split[type_index] .. '|nil' + end + magic_split[2] = magic_split[2]:sub(1, -2) + end + -- replace generic types + if magic_split[type_index] then + for k, v in pairs(generic) do + magic_split[type_index] = magic_split[type_index]:gsub(k, v) + end + end + -- surround some types by () + for _, type in ipairs(types) do + magic_split[type_index] = + magic_split[type_index]:gsub('^(' .. type .. '|nil):?$', '(%1)') + magic_split[type_index] = + magic_split[type_index]:gsub('^(' .. type .. '):?$', '(%1)') + end + end + + magic = table.concat(magic_split, ' ') + + outStream:writeln('/// @' .. magic) + fn_magic = checkComment4fn(fn_magic, magic) end - magic = table.concat(magic_split, ' ') - - outStream:writeln('/// @' .. magic) - fn_magic = checkComment4fn(fn_magic,magic) - elseif string.sub(line,3,3)=='-' then -- it's a nonmagic doc comment - local comment = string.sub(line,4) - outStream:writeln('/// '.. comment) - elseif string.sub(line,3,4)=='[[' then -- it's a long comment - line = string.sub(line,5) -- nibble head + elseif string.sub(line, 3, 3) == '-' then -- it's a nonmagic doc comment + local comment = string.sub(line, 4) + outStream:writeln('/// ' .. comment) + elseif string.sub(line, 3, 4) == '[[' then -- it's a long comment + line = string.sub(line, 5) -- nibble head local comment = '' - local closeSquare,hitend,thisComment - while (not hitend) and (not inStream:eof()) do - closeSquare = string.find(line,']]') + local closeSquare, hitend, thisComment + while not hitend and (not inStream:eof()) do + closeSquare = string.find(line, ']]') if not closeSquare then -- need to look on another line thisComment = line .. '\n' line = inStream:getLine() else - thisComment = string.sub(line,1,closeSquare-1) + thisComment = string.sub(line, 1, closeSquare - 1) hitend = true -- unget the tail of the line -- in most cases it's empty. This may make us less efficient but -- easier to program - inStream:ungetLine(string_trim(string.sub(line,closeSquare+2))) + inStream:ungetLine(string_trim(string.sub(line, closeSquare + 2))) end comment = comment .. thisComment end - if string.sub(comment,1,1)=='@' then -- it's a long magic comment + if string.sub(comment, 1, 1) == '@' then -- it's a long magic comment outStream:write('/*' .. comment .. '*/ ') - fn_magic = checkComment4fn(fn_magic,comment) + fn_magic = checkComment4fn(fn_magic, comment) else -- discard outStream:write('/* zz:' .. comment .. '*/ ') fn_magic = nil @@ -467,76 +419,74 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) fn_magic = nil end elseif string.find(line, '^function') or string.find(line, '^local%s+function') then - state = 'in_function' -- it's a function - local pos_fn = string.find(line,'function') + generic = {} + state = 'in_function' -- it's a function + local pos_fn = string.find(line, 'function') -- function -- ....v... if pos_fn then -- we've got a function - local fn_type - if string.find(line,'^local%s+') then - fn_type = ''--'static ' -- static functions seem to be excluded - else - fn_type = '' - end - local fn = TString_removeCommentFromLine(string_trim(string.sub(line,pos_fn+8))) + local fn = TString_removeCommentFromLine(string_trim(string.sub(line, pos_fn + 8))) if fn_magic then fn = fn_magic end - if string.sub(fn,1,1)=='(' then + if string.sub(fn, 1, 1) == '(' then -- it's an anonymous function outStream:writelnComment(line) else -- fn has a name, so is interesting -- want to fix for iffy declarations - local open_paren = string.find(fn,'[%({]') + local open_paren = string.find(fn, '[%({]') if open_paren then -- we might have a missing close paren - if not string.find(fn,'%)') then + if not string.find(fn, '%)') then fn = fn .. ' ___MissingCloseParenHere___)' end end -- Big hax - if string.find(fn, ":") then + if string.find(fn, ':') then -- TODO: We need to add a first parameter of "SELF" here -- local colon_place = string.find(fn, ":") -- local name = string.sub(fn, 1, colon_place) - fn = fn:gsub(":", ".", 1) - outStream:writeln("/// @param self") + fn = fn:gsub(':', '.', 1) + outStream:writeln('/// @param self') - local paren_start = string.find(fn, "(", 1, true) - local paren_finish = string.find(fn, ")", 1, true) + local paren_start = string.find(fn, '(', 1, true) + local paren_finish = string.find(fn, ')', 1, true) -- Nothing in between the parens local comma if paren_finish == paren_start + 1 then - comma = "" + comma = '' else - comma = ", " + comma = ', ' end - fn = string.sub(fn, 1, paren_start) .. "self" .. comma .. string.sub(fn, paren_start + 1) + fn = string.sub(fn, 1, paren_start) + .. 'self' + .. comma + .. string.sub(fn, paren_start + 1) end -- add vanilla function - outStream:writeln(fn_type .. 'function ' .. fn .. '{}') + outStream:writeln('function ' .. fn .. '{}') end else - this:warning(inStream:getLineNo(),'something weird here') + this:warning(inStream:getLineNo(), 'something weird here') end - fn_magic = nil -- mustn't indavertently use it again + fn_magic = nil -- mustn't inadvertently use it again -- TODO: If we can make this learn how to generate these, that would be helpful. -- elseif string.find(line, "^M%['.*'%] = function") then -- state = 'in_function' -- it's a function -- outStream:writeln("function textDocument/publishDiagnostics(...){}") - -- fn_magic = nil -- mustn't indavertently use it again + -- fn_magic = nil -- mustn't inadvertently use it again else - state = '' -- unknown - if #line>0 then -- we don't know what this line means, so just comment it out + state = '' -- unknown + if #line > 0 then -- we don't know what this line means, so just comment it out outStream:writeln('// zz: ' .. line) else outStream:writeln() -- keep this line blank @@ -556,16 +506,14 @@ local TApp = class() --! \brief constructor function TApp.init(this) - local t0 = TCore_Clock() - this.timestamp = t0:getTimeStamp() + this.timestamp = os.date('%c %Z', os.time()) this.name = 'Lua2DoX' this.version = '0.2 20130128' this.copyright = 'Copyright (c) Simon Dales 2012-13' end function TApp.getRunStamp(this) - return this.name .. ' (' .. this.version .. ') ' - .. this.timestamp + return this.name .. ' (' .. this.version .. ') ' .. this.timestamp end function TApp.getVersion(this) @@ -579,15 +527,14 @@ end local This_app = TApp() --main -local cl = TCore_Commandline() -local argv1 = cl:getRaw(2) +local argv1 = arg[1] if argv1 == '--help' then TCore_IO_writeln(This_app:getVersion()) TCore_IO_writeln(This_app:getCopyright()) TCore_IO_writeln([[ run as: - lua2dox_filter <param> + nvim -l scripts/lua2dox.lua <param> -------------- Param: <filename> : interprets filename @@ -602,8 +549,7 @@ else local filename = argv1 local filter = TLua2DoX_filter() - filter:readfile(appStamp,filename) + filter:readfile(appStamp, filename) end - --eof diff --git a/scripts/lua2dox_filter b/scripts/lua2dox_filter deleted file mode 100755 index 22484a807f..0000000000 --- a/scripts/lua2dox_filter +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -########################################################################### -# Copyright (C) 2012 by Simon Dales # -# simon@purrsoft.co.uk # -# # -# This program is free software; you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation; either version 2 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program; if not, write to the # -# Free Software Foundation, Inc., # -# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -########################################################################### -LANG="" - -##! \brief test executable to see if it exists -test_executable(){ - P_EXE="$1" - ######### - WHICH=`which ${P_EXE}` - if test -z "${WHICH}" - then - echo "not found \"${P_EXE}\"" - else - EXE="${P_EXE}" - fi - } - -##! \brief sets the lua interpreter -set_lua(){ - if test -z "${EXE}"; then - test_executable 'luajit' - fi - - if test -z "${EXE}"; then - test_executable 'lua' - fi -} - -##! \brief makes canonical name of file -##! -##! Note that "readlink -f" doesn't work in MacOSX -##! -do_readlink(){ - pushd . > /dev/null - TARGET_FILE=$1 - - cd `dirname $TARGET_FILE` - TARGET_FILE=`basename $TARGET_FILE` - - # Iterate down a (possible) chain of symlinks - while [ -L "$TARGET_FILE" ] - do - TARGET_FILE=`readlink $TARGET_FILE` - cd `dirname $TARGET_FILE` - TARGET_FILE=`basename $TARGET_FILE` - done - - PHYS_DIR=`pwd -P` - RESULT=$PHYS_DIR - popd > /dev/null - } - -##main -set_lua -if test -z "${EXE}" -then - echo "no lua interpreter available" -else - BASENAME=`basename "$0"` - do_readlink "$0" - DIRNAME="${RESULT}" - - LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}" - #echo "lua[${LUASCRIPT}]" - - ${EXE} ${LUASCRIPT} $@ -fi -# -##eof diff --git a/scripts/pvscheck.sh b/scripts/pvscheck.sh index 610c20eb48..97757c0848 100755 --- a/scripts/pvscheck.sh +++ b/scripts/pvscheck.sh @@ -328,7 +328,7 @@ realdir() {( patch_sources() {( local tgt="$1" ; shift - local only_bulid="${1}" ; shift + local only_build="${1}" ; shift get_pvs_comment "$tgt" diff --git a/scripts/release.sh b/scripts/release.sh index 380503662d..4321d96f62 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -59,8 +59,8 @@ _do_release_commit() { $__sed -i.bk 's/(NVIM_VERSION_PRERELEASE) "-dev"/\1 ""/' CMakeLists.txt if grep '(NVIM_API_PRERELEASE true)' CMakeLists.txt > /dev/null; then $__sed -i.bk 's/(NVIM_API_PRERELEASE) true/\1 false/' CMakeLists.txt - build/bin/nvim --api-info > test/functional/fixtures/api_level_$__API_LEVEL.mpack - git add test/functional/fixtures/api_level_$__API_LEVEL.mpack + build/bin/nvim --api-info > "test/functional/fixtures/api_level_$__API_LEVEL.mpack" + git add "test/functional/fixtures/api_level_${__API_LEVEL}.mpack" fi $__sed -i.bk 's,(<releases>),\1\ diff --git a/scripts/update_terminfo.sh b/scripts/update_terminfo.sh index 8a0937cc8c..775048f246 100755 --- a/scripts/update_terminfo.sh +++ b/scripts/update_terminfo.sh @@ -35,7 +35,7 @@ readonly -A entries=( db="$(mktemp -du)" print_bold() { - printf "\\e[1m$*\\e[0m" + printf "\\e[1m%b\\e[0m" "$*" } cd "$(git rev-parse --show-toplevel)" diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index ad1973603e..f9f7330097 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -35,7 +35,7 @@ usage() { echo " -m {vim-revision} List previous (older) missing Vim patches." echo " -M List all merged patch-numbers (at current v:version)." echo " -p {vim-revision} Download and generate a Vim patch. vim-revision" - echo " can be a Vim version (8.0.xxx) or a Git hash." + echo " can be a Vim version (8.1.xxx) or a Git hash." echo " -P {vim-revision} Download, generate and apply a Vim patch." echo " -g {vim-revision} Download a Vim patch." echo " -s [pr args] Create a vim-patch pull request." @@ -120,9 +120,9 @@ get_vim_sources() { commit_message() { if [[ -n "$vim_tag" ]]; then - printf '%s\n%s' "${vim_message}" "${vim_commit_url}" + printf '%s\n\n%s\n\n%s' "${vim_message}" "${vim_commit_url}" "${vim_coauthor}" else - printf 'vim-patch:%s\n\n%s\n%s' "$vim_version" "$vim_message" "$vim_commit_url" + printf 'vim-patch:%s\n\n%s\n\n%s\n\n%s' "$vim_version" "$vim_message" "$vim_commit_url" "$vim_coauthor" fi } @@ -175,6 +175,7 @@ assign_commit_details() { vim_commit_url="https://github.com/vim/vim/commit/${vim_commit}" vim_message="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:%B' "${vim_commit}" \ | sed -e 's/\(#[0-9]\{1,\}\)/vim\/vim\1/g')" + vim_coauthor="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:Co-authored-by: %an <%ae>' "${vim_commit}")" if [[ ${munge_commit_line} == "true" ]]; then # Remove first line of commit message. vim_message="$(echo "${vim_message}" | sed -e '1s/^patch /vim-patch:/')" @@ -192,11 +193,15 @@ preprocess_patch() { 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/\<\%('"${na_files}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" # Remove *.proto, Make*, INSTALL*, gui_*, beval.*, some if_*, gvim, libvterm, tee, VisVim, xpm, xxd - local na_src='auto\|configure.*\|GvimExt\|libvterm\|proto\|tee\|VisVim\|xpm\|xxd\|Make.*\|INSTALL.*\|beval.*\|gui.*\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv' + local na_src='auto\|configure.*\|GvimExt\|hardcopy.*\|libvterm\|proto\|tee\|VisVim\|xpm\|xxd\|Make.*\|INSTALL.*\|beval.*\|gui.*\|if_cscop\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\S*\<\%(testdir/\)\@<!\%('"${na_src}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" + # Remove runtime files ported to Lua. + local na_rt='filetype\.vim\|scripts\.vim\|autoload\/ft\/dist\.vim\|print\/.*' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/\<\%('"${na_rt}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" + # Remove unwanted Vim doc files. - local na_doc='channel\.txt\|netbeans\.txt\|os_\w\+\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|vim9\.txt\|sponsor\.txt\|intro\.txt\|tags' + local na_doc='channel\.txt\|if_cscop\.txt\|netbeans\.txt\|os_\w\+\.txt\|print\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|vim9\.txt\|sponsor\.txt\|intro\.txt\|tags' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\<\%('"${na_doc}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" # Remove "Last change ..." changes in doc files. @@ -207,7 +212,7 @@ preprocess_patch() { 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\<\%('"${na_src_testdir}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" # Remove testdir/test_*.vim files - local na_src_testdir='balloon.*\|channel.*\|crypt\.vim\|gui.*\|job_fails\.vim\|json\.vim\|mzscheme\.vim\|netbeans.*\|paste\.vim\|popupwin.*\|restricted\.vim\|shortpathname\.vim\|tcl\.vim\|terminal.*\|xxd\.vim' + local na_src_testdir='balloon.*\|channel.*\|crypt\.vim\|cscope\.vim\|gui.*\|hardcopy\.vim\|job_fails\.vim\|json\.vim\|mzscheme\.vim\|netbeans.*\|paste\.vim\|popupwin.*\|restricted\.vim\|shortpathname\.vim\|tcl\.vim\|terminal.*\|xxd\.vim' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\<test_\%('"${na_src_testdir}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" # Remove version.c #7555 @@ -237,6 +242,14 @@ preprocess_patch() { LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/userfunc\.c/\1\/eval\/userfunc\.c/g' \ "$file" > "$file".tmp && mv "$file".tmp "$file" + # Rename evalbuffer.c to eval/buffer.c + LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/evalbuffer\.c/\1\/eval\/buffer\.c/g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" + + # Rename evalwindow.c to eval/window.c + LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/evalwindow\.c/\1\/eval\/window\.c/g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" + # Rename map.c to mapping.c LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/map\(\.[ch]\)/\1\/mapping\2/g' \ "$file" > "$file".tmp && mv "$file".tmp "$file" @@ -274,6 +287,35 @@ preprocess_patch() { "$file" > "$file".tmp && mv "$file".tmp "$file" } +uncrustify_patch() { + git diff --quiet || { + >&2 echo 'Vim source working tree dirty, aborting.' + exit 1 + } + + local patch_path=$NVIM_SOURCE_DIR/build/vim_patch + rm -rf "$patch_path" + mkdir -p "$patch_path"/{a,b} + + local commit="$1" + for file in $(git diff-tree --name-only --no-commit-id -r --diff-filter=a "$commit"); do + git --work-tree="$patch_path"/a checkout --quiet "$commit"~ -- "$file" + done + for file in $(git diff-tree --name-only --no-commit-id -r --diff-filter=d "$commit"); do + git --work-tree="$patch_path"/b checkout --quiet "$commit" -- "$file" + done + git reset --quiet --hard HEAD + + # If the difference are drastic enough uncrustify may need to be used more + # than once. This is obviously a bug that needs to be fixed on uncrustify's + # end, but in the meantime this workaround is sufficient. + for _ in {1..2}; do + uncrustify -c "$NVIM_SOURCE_DIR"/src/uncrustify.cfg -q --replace --no-backup "$patch_path"/{a,b}/src/*.[ch] + done + + (cd "$patch_path" && (git --no-pager diff --no-index --no-prefix --patch --unified=5 --color=never a/ b/ || true)) +} + get_vimpatch() { get_vim_sources @@ -282,7 +324,11 @@ get_vimpatch() { msg_ok "Found Vim revision '${vim_commit}'." local patch_content - patch_content="$(git --no-pager show --unified=5 --color=never -1 --pretty=medium "${vim_commit}")" + if check_executable uncrustify; then + patch_content="$(uncrustify_patch "${vim_commit}")" + else + patch_content="$(git --no-pager show --unified=5 --color=never -1 --pretty=medium "${vim_commit}")" + fi cd "${NVIM_SOURCE_DIR}" @@ -466,7 +512,7 @@ submit_pr() { # Gets all Vim commits since the "start" commit. list_vim_commits() { ( - cd "${VIM_SOURCE_DIR}" && git log --reverse v8.0.0000..HEAD "$@" + cd "${VIM_SOURCE_DIR}" && git log --reverse v8.1.0000..HEAD "$@" ) } # Prints all (sorted) "vim-patch:xxx" tokens found in the Nvim git log. @@ -484,7 +530,7 @@ list_vimpatch_tokens() { list_vimpatch_numbers() { # Transform "vim-patch:X.Y.ZZZZ" to "ZZZZ". list_vimpatch_tokens | while read -r vimpatch_token; do - echo "$vimpatch_token" | grep '8\.0\.' | sed 's/.*vim-patch:8\.0\.\([0-9a-z]\+\).*/\1/' + echo "$vimpatch_token" | grep '8\.1\.' | sed -E 's/.*vim-patch:8\.1\.([0-9a-z]+).*/\1/' done } @@ -581,7 +627,7 @@ _set_missing_vimpatches() { else info=${line#* } if [[ -n $info ]]; then - # Remove any "patch 8.0.0902: " prefixes, and prefix with ": ". + # Remove any "patch 8.1.0902: " prefixes, and prefix with ": ". info=": ${info#patch*: }" fi fi @@ -624,7 +670,7 @@ show_vimpatches() { Instructions: To port one of the above patches to Neovim, execute this script with the patch revision as argument and follow the instructions, e.g. - '${BASENAME} -p v8.0.1234', or '${BASENAME} -P v8.0.1234' + '${BASENAME} -p v8.1.1234', or '${BASENAME} -P v8.1.1234' NOTE: Please port the _oldest_ patch if you possibly can. You can use '${BASENAME} -l path/to/file' to see what patches are missing for a file. diff --git a/scripts/vimpatch.lua b/scripts/vimpatch.lua index 11eb285462..836f672f6e 100755 --- a/scripts/vimpatch.lua +++ b/scripts/vimpatch.lua @@ -1,7 +1,7 @@ -- Updates version.c list of applied Vim patches. -- -- Usage: --- VIM_SOURCE_DIR=~/neovim/.vim-src/ nvim -i NONE -u NONE --headless +'luafile ./scripts/vimpatch.lua' +q +-- VIM_SOURCE_DIR=~/neovim/.vim-src/ nvim -V1 -es -i NONE +'luafile ./scripts/vimpatch.lua' +q local nvim = vim.api @@ -22,12 +22,13 @@ end -- Generates the lines to be inserted into the src/version.c -- `included_patches[]` definition. local function gen_version_c_lines() - -- Set of merged Vim 8.0.zzzz patch numbers. + -- Set of merged Vim 8.1.zzzz patch numbers. local merged_patch_numbers = {} local highest = 0 for _, n in ipairs(vimpatch_sh_list_numbers()) do + n = tonumber(n) if n then - merged_patch_numbers[tonumber(n)] = true + merged_patch_numbers[n] = true highest = math.max(highest, n) end end |