From 319563725243485d11bc980efd3821ae9cef4ddd Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 15 Sep 2019 16:34:31 -0700 Subject: release.sh [ci skip] --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/release.sh b/scripts/release.sh index 29d61370ce..7ac80b7e01 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -80,7 +80,7 @@ _do_bump_commit() { ,' runtime/nvim.appdata.xml rm CMakeLists.txt.bk rm runtime/nvim.appdata.xml.bk - nvim +'/NVIM_VERSION' +1new +'exe "norm! iUpdate version numbers!!!\"' \ + nvim +'/NVIM_VERSION' +1new +'exe "norm! iUpdate version numbers!!!"' \ -O CMakeLists.txt runtime/nvim.appdata.xml git add CMakeLists.txt runtime/nvim.appdata.xml -- cgit From 8f3d0276ee8ff0d35e07d3767e7dc5ace9d2b2d0 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 15 Sep 2019 23:08:40 -0700 Subject: release.sh [ci skip] --- scripts/release.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/release.sh b/scripts/release.sh index 7ac80b7e01..67268ba9bf 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -93,8 +93,7 @@ fi _do_bump_commit echo " Next steps: - - Double-check NVIM_VERSION_* in CMakeLists.txt - - Double-check runtime/nvim.appdata.xml + - Run tests/CI (version_spec.lua)! - Push the tag: git push --follow-tags - Update the 'stable' tag: -- cgit From b18b84df5eab9829ecbef644ef0af226becf881d Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Mon, 30 Sep 2019 00:10:29 +0200 Subject: build: run git-describe for dev version during build (#11117) This avoids invoking CMake after a new commit, which might take 15s on some systems. Skipped on CMake < 3.2.0 (missing BYPRODUCTS support). Co-Authored-By: Justin M. Keyes --- scripts/update_version_stamp.lua | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 scripts/update_version_stamp.lua (limited to 'scripts') diff --git a/scripts/update_version_stamp.lua b/scripts/update_version_stamp.lua new file mode 100644 index 0000000000..f01642043a --- /dev/null +++ b/scripts/update_version_stamp.lua @@ -0,0 +1,45 @@ +#!/usr/bin/env lua +-- +-- Script to update the Git version stamp during build. +-- This is called via the custom update_version_stamp target in +-- src/nvim/CMakeLists.txt. +-- +-- arg[1]: file containing the last git-describe output +-- arg[2]: file in which to update the version string + +local function die(msg) + print(string.format('%s: %s', arg[0], msg)) + -- No error, fall back to using generated "-dev" version. + os.exit(0) +end + +if #arg ~= 2 then + die(string.format("Expected two args, got %d", #arg)) +end + +local stampfile = arg[1] +local stamp = io.open(stampfile, 'r') +if stamp then + stamp = stamp:read('*l') +end + +local current = io.popen('git describe --dirty'):read('*l') +if not current then + die('git-describe failed') +end + +if stamp ~= current then + if stamp then + print(string.format('git version changed: %s -> %s', stamp, current)) + end + local new_lines = {} + local versiondeffile = arg[2] + for line in io.lines(versiondeffile) do + if line:match("NVIM_VERSION_MEDIUM") then + line = '#define NVIM_VERSION_MEDIUM "'..current..'"' + end + new_lines[#new_lines + 1] = line + end + io.open(versiondeffile, 'w'):write(table.concat(new_lines, '\n') .. '\n') + io.open(stampfile, 'w'):write(current) +end -- cgit From 30ae60e7cac7e77005aa429bc13f8ffa3ce64eb1 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 2 Oct 2019 03:45:59 +0200 Subject: Fix/revisit git-describe enhancement (#11124) * Fix/keep massaging git-describe result Ref: https://github.com/neovim/neovim/pull/11117#issuecomment-536416223 * build: revisit generation of version from Git Fixes "make clean && make", where "auto/versiondef.h" would be missing since b18b84d - because BYPRODUCTS are apparently removed when cleaning. This includes the following improvements/changes: - do not run git-describe during CMake's configure phase just for reporting - do not print with changed Git version (too noisy, simplifies code) * Move to src/nvim (included before config) for easier flow * fallback to describe always, write empty include file * update_version_stamp.lua: use prefix always --- scripts/update_version_stamp.lua | 53 ++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 24 deletions(-) mode change 100644 => 100755 scripts/update_version_stamp.lua (limited to 'scripts') diff --git a/scripts/update_version_stamp.lua b/scripts/update_version_stamp.lua old mode 100644 new mode 100755 index f01642043a..509e1f6fad --- a/scripts/update_version_stamp.lua +++ b/scripts/update_version_stamp.lua @@ -4,11 +4,11 @@ -- This is called via the custom update_version_stamp target in -- src/nvim/CMakeLists.txt. -- --- arg[1]: file containing the last git-describe output --- arg[2]: file in which to update the version string +-- arg[1]: file in which to update the version string +-- arg[2]: prefix to use always ("vX.Y.Z") local function die(msg) - print(string.format('%s: %s', arg[0], msg)) + io.stderr:write(string.format('%s: %s\n', arg[0], msg)) -- No error, fall back to using generated "-dev" version. os.exit(0) end @@ -17,29 +17,34 @@ if #arg ~= 2 then die(string.format("Expected two args, got %d", #arg)) end -local stampfile = arg[1] -local stamp = io.open(stampfile, 'r') -if stamp then - stamp = stamp:read('*l') +local versiondeffile = arg[1] +local prefix = arg[2] + +local described = io.popen('git describe --dirty'):read('*l') +if not described then + described = io.popen('git describe --tags --always --dirty'):read('*l') +end +if not described then + io.open(versiondeffile, 'w'):write('\n') + die('git-describe failed, using empty include file.') +end + +-- `git describe` annotates the most recent tagged release; for pre-release +-- builds we must replace that with the unreleased version. +local with_prefix = described:gsub("^v%d+%.%d+%.%d+", prefix) +if described == with_prefix then + -- Prepend the prefix always, e.g. with "nightly-12208-g4041b62b9". + with_prefix = prefix .. "-" .. described end -local current = io.popen('git describe --dirty'):read('*l') -if not current then - die('git-describe failed') +-- Read existing include file. +local current = io.open(versiondeffile, 'r') +if current then + current = current:read('*l') end -if stamp ~= current then - if stamp then - print(string.format('git version changed: %s -> %s', stamp, current)) - end - local new_lines = {} - local versiondeffile = arg[2] - for line in io.lines(versiondeffile) do - if line:match("NVIM_VERSION_MEDIUM") then - line = '#define NVIM_VERSION_MEDIUM "'..current..'"' - end - new_lines[#new_lines + 1] = line - end - io.open(versiondeffile, 'w'):write(table.concat(new_lines, '\n') .. '\n') - io.open(stampfile, 'w'):write(current) +-- Write new include file, if different. +local new = '#define NVIM_VERSION_MEDIUM "'..with_prefix..'"' +if current ~= new then + io.open(versiondeffile, 'w'):write(new .. '\n') end -- cgit From 6768c43e2129af85923ed0b4ed9f6f47be42b4fb Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Thu, 10 Oct 2019 10:38:15 +0200 Subject: update_version_stamp: redirect stderr on first try, --first-parent #11186 Avoid noise during builds: > fatal: No annotated tags can describe '417449f468c4ba186954f6295b3338fb55ee7b4a'. > However, there were unannotated tags: try --tags. This might be useful in general, but is expected to not happen - and falling back is OK then. The fallback command would still display errors then. It also uses `--first-parent`, which is important for when a release branch gets merged back. --- scripts/update_version_stamp.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/update_version_stamp.lua b/scripts/update_version_stamp.lua index 509e1f6fad..394c4f7694 100755 --- a/scripts/update_version_stamp.lua +++ b/scripts/update_version_stamp.lua @@ -20,9 +20,9 @@ end local versiondeffile = arg[1] local prefix = arg[2] -local described = io.popen('git describe --dirty'):read('*l') +local described = io.popen('git describe --first-parent --dirty 2>/dev/null'):read('*l') if not described then - described = io.popen('git describe --tags --always --dirty'):read('*l') + described = io.popen('git describe --first-parent --tags --always --dirty'):read('*l') end if not described then io.open(versiondeffile, 'w'):write('\n') -- cgit From 5cf6beb22178a844bc1b6e75e744eb04508b865c Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Sun, 13 Oct 2019 23:09:09 +0200 Subject: scripts/vim-patch.sh -l: display commit subjects Closes https://github.com/neovim/neovim/pull/11182. --- scripts/vim-patch.sh | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 2cc32f0dd0..4efe509008 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -366,7 +366,7 @@ submit_pr() { # Gets all Vim commits since the "start" commit. list_vim_commits() { ( - cd "${VIM_SOURCE_DIR}" && git log --reverse --format='%H' v8.0.0000..HEAD "$@" + cd "${VIM_SOURCE_DIR}" && git log --reverse v8.0.0000..HEAD "$@" ) } # Prints all (sorted) "vim-patch:xxx" tokens found in the Nvim git log. @@ -389,6 +389,7 @@ list_vimpatch_numbers() { } # Prints a newline-delimited list of Vim commits, for use by scripts. +# "$1": use extended format? # "$@" is passed to list_vim_commits, as extra arguments to git-log. list_missing_vimpatches() { local token vim_commit vim_tag patch_number @@ -396,6 +397,13 @@ list_missing_vimpatches() { declare -A vim_commit_tags declare -a git_log_args + local extended_format=$1; shift + if [[ "$extended_format" == 1 ]]; then + git_log_args=("--format=%H %s") + else + git_log_args=("--format=%H") + fi + # Massage arguments for git-log. declare -A git_log_replacements=( [^\(.*/\)?src/nvim/\(.*\)]="\${BASH_REMATCH[1]}src/\${BASH_REMATCH[2]}" @@ -431,13 +439,27 @@ list_missing_vimpatches() { # Get missing Vim commits set +u # Avoid "unbound variable" with bash < 4.4 below. - for vim_commit in $(list_vim_commits "${git_log_args[@]}"); do + local vim_commit info + while IFS=' ' read -r line; do # Check for vim-patch: (usually runtime updates). - token="vim-patch:${vim_commit:0:7}" + token="vim-patch:${line:0:7}" if [[ "${tokens[$token]-}" ]]; then continue fi + # Get commit hash, and optional info from line. This is used in + # extended mode, and when using e.g. '--format' manually. + vim_commit=${line%% *} + if [[ "$vim_commit" == "$line" ]]; then + info= + else + info=${line#* } + if [[ -n $info ]]; then + # Remove any "patch 8.0.0902: " prefixes, and prefix with ": ". + info=": ${info#patch*: }" + fi + fi + vim_tag="${vim_commit_tags[$vim_commit]-}" if [[ -n "$vim_tag" ]]; then # Check for vim-patch: (not commit hash). @@ -445,11 +467,11 @@ list_missing_vimpatches() { if [[ "${tokens[$patch_number]-}" ]]; then continue fi - echo "$vim_tag" + printf '%s%s\n' "$vim_tag" "$info" else - echo "$vim_commit" + printf '%s%s\n' "$vim_commit" "$info" fi - done + done < <(list_vim_commits "${git_log_args[@]}") set -u } @@ -464,7 +486,7 @@ show_vimpatches() { runtime_commits[$commit]=1 done - list_missing_vimpatches "$@" | while read -r vim_commit; do + list_missing_vimpatches 1 "$@" | while read -r vim_commit; do if [[ "${runtime_commits[$vim_commit]-}" ]]; then printf ' • %s (+runtime)\n' "${vim_commit}" else @@ -596,7 +618,7 @@ while getopts "hlLMVp:P:g:r:s" opt; do ;; L) shift # remove opt - list_missing_vimpatches "$@" + list_missing_vimpatches 0 "$@" exit 0 ;; M) -- cgit From c5c06665ed302e774dc1a01bed949815b4b7c8c3 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 16 Oct 2019 15:44:38 +0200 Subject: scripts/vim-patch.sh: lazily update Vim source (#11207) This gets done only initially, for `-l`, and when a commit cannot be found. Also provide more compact instructions with `-l` / `show_vimpatches`. --- scripts/vim-patch.sh | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 4efe509008..2a04805606 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -89,7 +89,7 @@ get_vim_sources() { echo "Cloning Vim into: ${VIM_SOURCE_DIR}" git clone https://github.com/vim/vim.git "${VIM_SOURCE_DIR}" cd "${VIM_SOURCE_DIR}" - else + elif [[ "${1-}" == update ]]; then cd "${VIM_SOURCE_DIR}" if ! [ -d ".git" ] \ && ! [ "$(git rev-parse --show-toplevel)" = "${VIM_SOURCE_DIR}" ]; then @@ -103,6 +103,8 @@ get_vim_sources() { else msg_err "Could not update Vim sources; ignoring error." fi + else + cd "${VIM_SOURCE_DIR}" fi } @@ -124,7 +126,7 @@ find_git_remote() { } # Assign variables for a given Vim tag, patch version, or commit. -# Might exit in case it cannot be found. +# Might exit in case it cannot be found, after updating Vim sources. assign_commit_details() { local vim_commit_ref if [[ ${1} =~ v?[0-9]\.[0-9]\.[0-9]{3,4} ]]; then @@ -146,9 +148,14 @@ assign_commit_details() { local munge_commit_line=false fi - vim_commit=$(git -C "${VIM_SOURCE_DIR}" log -1 --format="%H" "${vim_commit_ref}" --) || { - >&2 msg_err "Couldn't find Vim revision '${vim_commit_ref}'." - exit 3 + local get_vim_commit_cmd="git -C ${VIM_SOURCE_DIR} log -1 --format=%H ${vim_commit_ref} --" + vim_commit=$($get_vim_commit_cmd 2>&1) || { + # Update Vim sources. + get_vim_sources update + vim_commit=$($get_vim_commit_cmd 2>&1) || { + >&2 msg_err "Couldn't find Vim revision '${vim_commit_ref}': git error: ${vim_commit}." + exit 3 + } } vim_commit_url="https://github.com/vim/vim/commit/${vim_commit}" @@ -478,8 +485,8 @@ list_missing_vimpatches() { # Prints a human-formatted list of Vim commits, with instructional messages. # Passes "$@" onto list_missing_vimpatches (args for git-log). show_vimpatches() { - get_vim_sources - printf "\nVim patches missing from Neovim:\n" + get_vim_sources update + printf "Vim patches missing from Neovim:\n" local -A runtime_commits for commit in $(git -C "${VIM_SOURCE_DIR}" log --format="%H %D" -- runtime | sed 's/,\? tag: / /g'); do @@ -494,17 +501,12 @@ show_vimpatches() { fi done - printf "Instructions: - - To port one of the above patches to Neovim, execute - this script with the patch revision as argument and - follow the instructions. - - Examples: '%s -p 7.4.487' - '%s -p 1e8ebf870720e7b671f98f22d653009826304c4f' + printf "\nInstructions: + To port one of the above patches to Neovim, execute this script with the patch revision as argument and follow the instructions, e.g. + '%s -p v8.0.1234', or '%s -P v8.0.1234' NOTE: Please port the _oldest_ patch if you possibly can. - Out-of-order patches increase the possibility of bugs. + You can use '%s -l path/to/file' to see what patches are missing for a file. " "${BASENAME}" "${BASENAME}" } @@ -646,7 +648,7 @@ while getopts "hlLMVp:P:g:r:s" opt; do exit 0 ;; V) - get_vim_sources + get_vim_sources update exit 0 ;; *) -- cgit From 46bde66147ead1bb234932a48e2c56fc4b698ec0 Mon Sep 17 00:00:00 2001 From: smolck <46855713+smolck@users.noreply.github.com> Date: Sat, 26 Oct 2019 14:54:54 -0500 Subject: gen_vimdoc.py: dump API docs to msgpack #11296 Convenient for API clients who want to reuse the API docs in their own docs. Could be used e.g. to eliminate nvim.net's own doxygen parser: https://github.com/neovim/nvim.net/tree/3a736232a4e7b7a2a1eff4bded24d2bf27a918c2/src/NvimClient.APIGenerator/Docs TODO: currently the result values are formatted as Vim help docs. We should change the values to have structure, something like this: [{ 'nvim_win_get_var': [ 'line1, 'line2', [ 'item1', 'item2', ... ] ], 'nvim_win_set_var': [ ... ], ... }] close #11296 --- scripts/gen_vimdoc.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 373a58d11e..aa50132d4c 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -36,11 +36,12 @@ import shutil import textwrap import subprocess import collections +import msgpack from xml.dom import minidom -if sys.version_info[0] < 3: - print("use Python 3") +if sys.version_info[0] < 3 or sys.version_info[1] < 5: + print("requires Python 3.5+") sys.exit(1) DEBUG = ('DEBUG' in os.environ) @@ -453,7 +454,7 @@ def parse_source_xml(filename, mode): """ global xrefs xrefs = set() - functions = [] + functions = {} # Map of func_name:docstring. deprecated_functions = [] dom = minidom.parse(filename) @@ -577,11 +578,11 @@ def parse_source_xml(filename, mode): if 'Deprecated' in xrefs: deprecated_functions.append(func_doc) elif name.startswith(CONFIG[mode]['func_name_prefix']): - functions.append(func_doc) + functions[name] = func_doc xrefs.clear() - return '\n\n'.join(functions), '\n\n'.join(deprecated_functions) + return '\n\n'.join(list(functions.values())), '\n\n'.join(deprecated_functions), functions def delete_lines_below(filename, tokenstr): @@ -604,6 +605,12 @@ def gen_docs(config): Doxygen is called and configured through stdin. """ for mode in CONFIG: + functions = {} # Map of func_name:docstring. + mpack_file = os.path.join(base_dir, 'runtime', 'doc', + CONFIG[mode]['filename'].replace('.txt', '.mpack')) + if os.path.exists(mpack_file): + os.remove(mpack_file) + output_dir = out_dir.format(mode=mode) p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE) p.communicate( @@ -645,14 +652,15 @@ def gen_docs(config): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): - functions, deprecated = parse_source_xml( + functions_text, deprecated_text, fns = parse_source_xml( os.path.join(base, '%s.xml' % compound.getAttribute('refid')), mode) + # Collect functions from all modules (for the current `mode`). + functions = {**functions, **fns} - if not functions and not deprecated: + if not functions_text and not deprecated_text: continue - - if functions or deprecated: + else: name = os.path.splitext(os.path.basename(filename))[0] if name == 'ui': name = name.upper() @@ -665,12 +673,12 @@ def gen_docs(config): if intro: doc += '\n\n' + intro - if functions: - doc += '\n\n' + functions + if functions_text: + doc += '\n\n' + functions_text - if INCLUDE_DEPRECATED and deprecated: + if INCLUDE_DEPRECATED and deprecated_text: doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name - doc += deprecated + doc += deprecated_text if doc: filename = os.path.basename(filename) @@ -713,6 +721,8 @@ def gen_docs(config): delete_lines_below(doc_file, CONFIG[mode]['section_start_token']) with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) + with open(mpack_file, 'wb') as fp: + fp.write(msgpack.packb(functions, use_bin_type=True)) shutil.rmtree(output_dir) -- cgit From 6d8fe9b3f40d0d3f4ddeed56f592dfd2be810f7f Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sat, 26 Oct 2019 16:21:22 -0700 Subject: lint --- scripts/gen_vimdoc.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index aa50132d4c..8d0cf54828 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -582,7 +582,9 @@ def parse_source_xml(filename, mode): xrefs.clear() - return '\n\n'.join(list(functions.values())), '\n\n'.join(deprecated_functions), functions + return ('\n\n'.join(list(functions.values())), + '\n\n'.join(deprecated_functions), + functions) def delete_lines_below(filename, tokenstr): @@ -606,7 +608,8 @@ def gen_docs(config): """ for mode in CONFIG: functions = {} # Map of func_name:docstring. - mpack_file = os.path.join(base_dir, 'runtime', 'doc', + mpack_file = os.path.join( + base_dir, 'runtime', 'doc', CONFIG[mode]['filename'].replace('.txt', '.mpack')) if os.path.exists(mpack_file): os.remove(mpack_file) @@ -653,8 +656,8 @@ def gen_docs(config): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): functions_text, deprecated_text, fns = parse_source_xml( - os.path.join(base, '%s.xml' % - compound.getAttribute('refid')), mode) + os.path.join(base, '{}.xml'.format( + compound.getAttribute('refid'))), mode) # Collect functions from all modules (for the current `mode`). functions = {**functions, **fns} -- cgit From f9da2673737d8585c573f04099f1efe0dd30f526 Mon Sep 17 00:00:00 2001 From: Hirokazu Hata Date: Sun, 27 Oct 2019 11:58:28 +0900 Subject: scripts/lua2dox.lua: Remove class declaration block Judging class definitions in the form "string.find (line, '=% s * class% (')" must force writing class definitions in this format, but such a mechanism is Absent. Also, Lua has no formal class in the language specification, and implements inheritance with setmetadable. To detect this, we should have a parser for it, not a simple regular expression. --- scripts/lua2dox.lua | 34 ++-------------------------------- 1 file changed, 2 insertions(+), 32 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 77cdabcc4b..438f734917 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -543,7 +543,6 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local fn = TString_removeCommentFromLine(string_trim(string.sub(line,pos_fn+8))) if fn_magic then fn = fn_magic - fn_magic = nil end if string.sub(fn,1,1)=='(' then @@ -554,49 +553,20 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) -- want to fix for iffy declarations local open_paren = string.find(fn,'[%({]') - local fn0 = fn if open_paren then - fn0 = string.sub(fn,1,open_paren-1) -- we might have a missing close paren if not string.find(fn,'%)') then fn = fn .. ' ___MissingCloseParenHere___)' end end - local dot = string.find(fn0,'[%.:]') - if dot then -- it's a method - local klass = string.sub(fn,1,dot-1) - local method = string.sub(fn,dot+1) - --TCore_IO_writeln('function ' .. klass .. '::' .. method .. ftail .. '{}') - --TCore_IO_writeln(klass .. '::' .. method .. ftail .. '{}') - outStream:writeln( - '/*! \\memberof ' .. klass .. ' */ ' - .. method .. '{}' - ) - else - -- add vanilla function - - outStream:writeln(fn_type .. 'function ' .. fn .. '{}') - end + -- add vanilla function + outStream:writeln(fn_type .. 'function ' .. fn .. '{}') end else this:warning(inStream:getLineNo(),'something weird here') end fn_magic = nil -- mustn't indavertently use it again - elseif string.find(line,'=%s*class%(') then - state = 'in_class' -- it's a class declaration - local tailComment - line,tailComment = TString_removeCommentFromLine(line) - local equals = string.find(line,'=') - local klass = string_trim(string.sub(line,1,equals-1)) - local tail = string_trim(string.sub(line,equals+1)) - -- class(wibble wibble) - -- ....v. - local parent = string.sub(tail,7,-2) - if #parent>0 then - parent = ' :public ' .. parent - end - outStream:writeln('class ' .. klass .. parent .. '{};') else state = '' -- unknown if #line>0 then -- we don't know what this line means, so just comment it out -- cgit From 90981f5861e4c23c6a23a0082261b074ba48ea47 Mon Sep 17 00:00:00 2001 From: erw7 Date: Sat, 2 Nov 2019 04:54:59 +0900 Subject: update_version_stamp.lua: Use NUL on Windows #11323 --- scripts/update_version_stamp.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/update_version_stamp.lua b/scripts/update_version_stamp.lua index 394c4f7694..11b521fab6 100755 --- a/scripts/update_version_stamp.lua +++ b/scripts/update_version_stamp.lua @@ -13,6 +13,10 @@ local function die(msg) os.exit(0) end +local function iswin() + return package.config:sub(1,1) == '\\' +end + if #arg ~= 2 then die(string.format("Expected two args, got %d", #arg)) end @@ -20,7 +24,8 @@ end local versiondeffile = arg[1] local prefix = arg[2] -local described = io.popen('git describe --first-parent --dirty 2>/dev/null'):read('*l') +local dev_null = iswin() and 'NUL' or '/dev/null' +local described = io.popen('git describe --first-parent --dirty 2>'..dev_null):read('*l') if not described then described = io.popen('git describe --first-parent --tags --always --dirty'):read('*l') end -- cgit From 2ba212e8c25794184ce465d468eb62714ea7ba03 Mon Sep 17 00:00:00 2001 From: Marco Hinz Date: Fri, 8 Nov 2019 15:17:55 +0100 Subject: vim-patch.sh: add missing argument [skip ci] --- scripts/vim-patch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 2a04805606..b946266599 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -507,7 +507,7 @@ show_vimpatches() { NOTE: Please port the _oldest_ patch if you possibly can. You can use '%s -l path/to/file' to see what patches are missing for a file. -" "${BASENAME}" "${BASENAME}" +" "${BASENAME}" "${BASENAME}" "${BASENAME}" } review_commit() { -- cgit From 9fb278ddea492cdae53be6fd948c75498df2f712 Mon Sep 17 00:00:00 2001 From: Marco Hinz Date: Fri, 8 Nov 2019 20:47:58 +0100 Subject: vim-patch.sh: multiline printf -> heredoc (#11351) The following script is cut out from vim-patch.sh: ```sh #!/usr/bin/env bash BASENAME=vim-patch.sh printf "\nInstructions: To port one of the above patches to Neovim, execute this script with the patch revision as argument and follow the instructions, e.g. '%s -p v8.0.1234', or '%s -P v8.0.1234' NOTE: Please port the _oldest_ patch if you possibly can. You can use '%s -l path/to/file' to see what patches are missing for a file. " "${BASENAME}" "${BASENAME}" "${BASENAME}" ``` The code itself should be correct, but shellcheck 0.7.0 says: ``` In /tmp/test.sh line 5: printf "\nInstructions: ^-- SC2183: This format string has 2 variables, but is passed 3 arguments. ``` We also had a problem before that a `%s` was added, but the accompanying argument to printf was forgotten. Using a heredoc is less error-prone, since we insert variables directly. --- scripts/vim-patch.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index b946266599..b6a0df4649 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -501,13 +501,15 @@ show_vimpatches() { fi done - printf "\nInstructions: + cat << EOF + +Instructions: To port one of the above patches to Neovim, execute this script with the patch revision as argument and follow the instructions, e.g. - '%s -p v8.0.1234', or '%s -P v8.0.1234' + '${BASENAME} -p v8.0.1234', or '${BASENAME} -P v8.0.1234' NOTE: Please port the _oldest_ patch if you possibly can. - You can use '%s -l path/to/file' to see what patches are missing for a file. -" "${BASENAME}" "${BASENAME}" "${BASENAME}" + You can use '${BASENAME} -l path/to/file' to see what patches are missing for a file. +EOF } review_commit() { -- cgit From 4bd7881fce0c7183b97e7541b7a7ac872e08bbda Mon Sep 17 00:00:00 2001 From: smolck <46855713+smolck@users.noreply.github.com> Date: Wed, 6 Nov 2019 16:22:20 -0600 Subject: [scripts/gen_vimdoc.py] Generate better-formatted mpack Changes the generated msgpack result values in the runtime/doc/*.mpack files to be formatted like this (instead of being formatted like Vim help text): [ 'nvim_win_get_var': { 'signature': 'nvim_win_get_var({window}, {name}, {err})', 'parameters': [('Window', 'window'), ('String', 'name'), ('Error *', 'err')], 'parameters_doc': [{'window': 'Window handle, or 0 for current window', 'name': 'Variable name'}], 'doc': ['Gets a window-scoped (w:) variable'], 'return': ['Variable value'], 'seealso': [] } }, ... ] --- scripts/gen_vimdoc.py | 259 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 254 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 8d0cf54828..df06644118 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -262,6 +262,36 @@ def has_nonexcluded_params(nodes): return True +def params_as_map(parent, width=62): + """Returns a Doxygen as a map of + :""" + items = [] + for node in parent.childNodes: + if node.nodeType == node.TEXT_NODE: + continue + + name_node = find_first(node, 'parametername') + if name_node.getAttribute('direction') == 'out': + continue + + name = get_text(name_node) + if name in param_exclude: + continue + + items.append((name.strip(), node)) + + out = {} # Map of name:desc + for name, node in items: + desc = '' + desc_node = get_child(node, 'parameterdescription') + if desc_node: + desc = parse_parblock(desc_node, width=width, + indent=(' ' * len(name))) + out[name] = desc + + return out + + def render_params(parent, width=62): """Renders Doxygen tag as Vim help text.""" name_length = 0 @@ -365,6 +395,86 @@ def render_node(n, text, prefix='', indent='', width=62): return text +def para_as_map(parent, indent='', width=62): + """Parses XML as a map + + The map returned may or may not contain + all of the following keys: + 'text': Text in the para tag + 'params': Information from 's + 'return': TODO + 'seealso': TODO + 'xrefs': TODO + """ + if is_inline(parent): + return { + 'text': clean_lines(doc_wrap(render_node(parent, ''), + indent=indent, width=width).strip()) + } + + # Ordered dict of ordered lists. + groups = collections.OrderedDict([ + ('params', []), + ('return', []), + ('seealso', []), + ('xrefs', []), + ]) + + # Gather nodes into groups. Mostly this is because we want "parameterlist" + # nodes to appear together. + text = '' + kind = '' + last = '' + for child in parent.childNodes: + if child.nodeName == 'parameterlist': + groups['params'].append(child) + elif child.nodeName == 'xrefsect': + groups['xrefs'].append(child) + elif child.nodeName == 'simplesect': + last = kind + kind = child.getAttribute('kind') + if kind == 'return' or (kind == 'note' and last == 'return'): + groups['return'].append(child) + elif kind == 'see': + groups['seealso'].append(child) + elif kind in ('note', 'warning'): + text += render_node(child, text, indent=indent, width=width) + else: + raise RuntimeError('unhandled simplesect: {}\n{}'.format( + child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) + else: + text += render_node(child, text, indent=indent, width=width) + + chunks = { + 'text': text, + 'params': [], + 'return': [], + 'seealso': [], + 'xrefs': [] + } + + # Generate text from the gathered items. + if len(groups['params']) > 0 and has_nonexcluded_params(groups['params']): + for child in groups['params']: + chunks['params'].append(params_as_map(child, width=width)) + if len(groups['return']) > 0: + for child in groups['return']: + chunks['return'].append(render_node( + child, '', indent=indent, width=width).lstrip()) + if len(groups['seealso']) > 0: + for child in groups['seealso']: + chunks['seealso'].append(render_node( + child, '', indent=indent, width=width)) + for child in groups['xrefs']: + title = get_text(get_child(child, 'xreftitle')) + xrefs.add(title) + xrefdesc = render_para(get_child(child, 'xrefdescription'), width=width) + chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title), + width=width) + '\n') + + return chunks + + def render_para(parent, indent='', width=62): """Renders Doxygen containing arbitrary nodes. @@ -433,6 +543,16 @@ def render_para(parent, indent='', width=62): return clean_lines('\n'.join(chunks).strip()) +def parse_parblock_as_array(parent, prefix='', width=62, indent=''): + """Parses a nested block of tags as an array of maps""" + lines = [] + for child in parent.childNodes: + rendered = para_as_map(child, width=width, indent=indent) + lines.append(rendered) + + return lines + + def parse_parblock(parent, prefix='', width=62, indent=''): """Renders a nested block of tags as Vim help text.""" paragraphs = [] @@ -443,13 +563,136 @@ def parse_parblock(parent, prefix='', width=62, indent=''): # }}} +def parse_source_xml_for_mpack(filename, mode): + """Collects API functions to be packed as msgpack. + + Returns two maps: + 1. API functions + 2. Deprecated API functions + + Caller decides what to do with the deprecated documentation. + """ + global xrefs + xrefs = set() + functions = {} # Map of func_name:docstring. + deprecated_functions = {} # Map of func_name:docstring. + + dom = minidom.parse(filename) + for member in dom.getElementsByTagName('memberdef'): + if member.getAttribute('static') == 'yes' or \ + member.getAttribute('kind') != 'function' or \ + member.getAttribute('prot') == 'private' or \ + get_text(get_child(member, 'name')).startswith('_'): + continue + + loc = find_first(member, 'location') + if 'private' in loc.getAttribute('file'): + continue + + return_type = get_text(get_child(member, 'type')) + if return_type == '': + continue + + if return_type.startswith(('ArrayOf', 'DictionaryOf')): + parts = return_type.strip('_').split('_') + return_type = '{}({})'.format(parts[0], ', '.join(parts[1:])) + + name = get_text(get_child(member, 'name')) + + annotations = get_text(get_child(member, 'argsstring')) + if annotations and ')' in annotations: + annotations = annotations.rsplit(')', 1)[-1].strip() + # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of + # non-void functions. Special-case void functions here. + if name == 'nvim_get_mode' and len(annotations) == 0: + annotations += 'FUNC_API_FAST' + annotations = filter(None, map(lambda x: annotation_map.get(x), + annotations.split())) + + params = [] + type_length = 0 + + for param in get_children(member, 'param'): + param_type = get_text(get_child(param, 'type')).strip() + param_name = '' + declname = get_child(param, 'declname') + if declname: + param_name = get_text(declname).strip() + elif mode == 'lua': + # that's how it comes out of lua2dox + param_name = param_type + param_type = '' + + if param_name in param_exclude: + continue + + type_length = max(type_length, len(param_type)) + params.append((param_type, param_name)) + + c_args = [] + for param_type, param_name in params: + c_args.append(( + '%s %s' % (param_type.ljust(type_length), param_name)).strip()) + + c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) + + doc = '' + desc = find_first(member, 'detaileddescription') + if desc: + doc = parse_parblock_as_array(desc) + if DEBUG: + print(textwrap.indent( + re.sub(r'\n\s*\n+', '\n', + desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) + + prefix = '%s(' % name + suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params + if a[0] not in ('void', 'Error')) + + signature = prefix + suffix + + parameters_doc = [] + func_doc = [] + func_return = '' + seealso = '' + for m in doc: + if 'text' in m: + if not m['text'] == '': + func_doc.append(m['text']) + if 'params' in m: + parameters_doc += m['params'] + if 'return' in m: + func_return = m['return'] + if 'seealso' in m: + seealso = m['xrefs'] + + function_map = { + 'signature': signature, + 'parameters': params, + 'parameters_doc': parameters_doc, + 'doc': func_doc, + 'return': func_return, + 'seealso': seealso + } + + if INCLUDE_C_DECL: + function_map['c_decl'] = c_decl + + if 'Deprecated' in xrefs: + deprecated_functions[name] = function_map + elif name.startswith(CONFIG[mode]['func_name_prefix']): + functions[name] = function_map + + xrefs.clear() + + return (functions, deprecated_functions) + + def parse_source_xml(filename, mode): """Collects API functions. - Returns two strings: 1. API functions 2. Deprecated API functions - Caller decides what to do with the deprecated documentation. """ global xrefs @@ -609,8 +852,8 @@ def gen_docs(config): for mode in CONFIG: functions = {} # Map of func_name:docstring. mpack_file = os.path.join( - base_dir, 'runtime', 'doc', - CONFIG[mode]['filename'].replace('.txt', '.mpack')) + base_dir, 'runtime', 'doc', + CONFIG[mode]['filename'].replace('.txt', '.mpack')) if os.path.exists(mpack_file): os.remove(mpack_file) @@ -627,6 +870,7 @@ def gen_docs(config): if p.returncode: sys.exit(p.returncode) + mpacks = {} sections = {} intros = {} sep = '=' * text_width @@ -655,6 +899,9 @@ def gen_docs(config): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): + mpack = parse_source_xml_for_mpack(os.path.join(base, '{}.xml'.format( + compound.getAttribute('refid'))), mode) + functions_text, deprecated_text, fns = parse_source_xml( os.path.join(base, '{}.xml'.format( compound.getAttribute('refid'))), mode) @@ -694,6 +941,7 @@ def gen_docs(config): title = '{} Functions'.format(name) helptag = '*api-{}*'.format(name.lower()) sections[filename] = (title, helptag, doc) + mpacks[filename] = mpack if not sections: return @@ -724,8 +972,9 @@ def gen_docs(config): delete_lines_below(doc_file, CONFIG[mode]['section_start_token']) with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) + with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(functions, use_bin_type=True)) + fp.write(msgpack.packb(mpacks, use_bin_type=True)) shutil.rmtree(output_dir) -- cgit From af53a0c0123338575dd59934449d7fe836835d1c Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 17 Nov 2019 19:06:59 -0800 Subject: doc: Lua [ci skip] #11378 - Rework :help lua-commands - Rename if_lua.txt => lua.txt --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 8d0cf54828..4d71d5e15e 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -85,7 +85,7 @@ CONFIG = { 'append_only': [], }, 'lua': { - 'filename': 'if_lua.txt', + 'filename': 'lua.txt', 'section_start_token': '*lua-vim*', 'section_order': [ 'vim.lua', -- cgit From f460bae441b2a35964c3535396e251c33c997395 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 24 Nov 2019 17:53:33 -0800 Subject: release.sh [ci skip] --- scripts/release.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/release.sh b/scripts/release.sh index 67268ba9bf..5b4902a2d7 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -93,6 +93,7 @@ fi _do_bump_commit echo " Next steps: + - Update runtime/nvim.appdata.xml on _master_ - Run tests/CI (version_spec.lua)! - Push the tag: git push --follow-tags -- cgit From 001e69cd4602e84219fd7cfd8ade62f0cb24097c Mon Sep 17 00:00:00 2001 From: Brian Wignall Date: Tue, 26 Nov 2019 07:15:14 -0500 Subject: doc: fix typos close #11459 --- scripts/lua2dox.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 438f734917..171621e38d 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -65,7 +65,7 @@ FILTER_PATTERNS = *.lua=lua2dox_filter Either add them to the end or find the appropriate entry in Doxyfile. -There are other lines that you might like to alter, but see futher documentation for details. +There are other lines that you might like to alter, but see further documentation for details.
  • When Doxyfile is edited run "doxygen" -- cgit From 01adb81c55c16491372842001f58d810ea2524c2 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 17 Nov 2019 22:49:28 -0800 Subject: gen_vimdoc.py: DRY --- scripts/gen_vimdoc.py | 542 ++++++++++++++++++-------------------------------- 1 file changed, 198 insertions(+), 344 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index df06644118..cd44667ef3 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -9,25 +9,22 @@ This would be easier using lxml and XSLT, but: 2. I wouldn't know how to deal with nested indentation in tags using XSLT. -Each function documentation is formatted with the following rules: +Each function :help block is formatted as follows: - - Maximum width of 78 characters (`text_width`). - - Spaces for indentation. - - Function signature and helptag are on the same line. - - Helptag is right aligned. + - Max width of 78 columns (`text_width`). + - Indent with spaces (not tabs). + - Indent of 16 columns for body text. + - Function signature and helptag (right-aligned) on the same line. - Signature and helptag must have a minimum of 8 spaces between them. - - If the signature is too long, it is placed on the line after the - helptag. The signature wraps at `text_width - 8` characters with - subsequent lines indented to the open parenthesis. - - Documentation body will be indented by 16 spaces. + - If the signature is too long, it is placed on the line after the helptag. + Signature wraps at `text_width - 8` characters with subsequent + lines indented to the open parenthesis. - Subsection bodies are indented an additional 4 spaces. - - Documentation body consists of the function description, parameter details, - return description, and C declaration. + - Body consists of function description, parameters, return description, and + C declaration (`INCLUDE_C_DECL`). - Parameters are omitted for the `void` and `Error *` types, or if the parameter is marked as [out]. - Each function documentation is separated by a single line. - -The C declaration is added to the end to show actual argument types. """ import os import re @@ -123,14 +120,21 @@ annotation_map = { xrefs = set() -def debug_this(s, n): - o = n if isinstance(n, str) else n.toprettyxml(indent=' ', newl='\n') - name = '' if isinstance(n, str) else n.nodeName - if s in o: +# Raises an error with details about `o`, if `cond` is in object `o`, +# or if `cond()` is callable and returns True. +def debug_this(cond, o): + name = '' + if not isinstance(o, str): + try: + name = o.nodeName + o = o.toprettyxml(indent=' ', newl='\n') + except: + pass + if ((callable(cond) and cond()) + or (not callable(cond) and cond in o)): raise RuntimeError('xxx: {}\n{}'.format(name, o)) -# XML Parsing Utilities {{{ def find_first(parent, name): """Finds the first matching node within parent.""" sub = parent.getElementsByTagName(name) @@ -254,74 +258,42 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None): return result -def has_nonexcluded_params(nodes): - """Returns true if any of the given elements has at least - one non-excluded item.""" - for n in nodes: - if render_params(n) != '': - return True -def params_as_map(parent, width=62): - """Returns a Doxygen as a map of - :""" +def update_params_map(parent, ret_map, width=62): + """Updates `ret_map` with name:desc key-value pairs extracted + from a Doxygen . + """ items = [] for node in parent.childNodes: if node.nodeType == node.TEXT_NODE: continue - name_node = find_first(node, 'parametername') if name_node.getAttribute('direction') == 'out': continue - name = get_text(name_node) if name in param_exclude: continue - items.append((name.strip(), node)) - - out = {} # Map of name:desc + # `ret_map` is a name:desc map. for name, node in items: desc = '' desc_node = get_child(node, 'parameterdescription') if desc_node: desc = parse_parblock(desc_node, width=width, indent=(' ' * len(name))) - out[name] = desc - - return out + ret_map[name] = desc + return ret_map -def render_params(parent, width=62): - """Renders Doxygen tag as Vim help text.""" +def fmt_params_map_as_vimhelp(m, width=62): + """Renders a params map as Vim :help text.""" name_length = 0 - items = [] - for node in parent.childNodes: - if node.nodeType == node.TEXT_NODE: - continue - - name_node = find_first(node, 'parametername') - if name_node.getAttribute('direction') == 'out': - continue - - name = get_text(name_node) - if name in param_exclude: - continue - - name = '{%s}' % name - name_length = max(name_length, len(name) + 2) - items.append((name.strip(), node)) - + for name, desc in m.items(): + name_length = max(name_length, len(name) + 4) out = '' - for name, node in items: - name = ' {}'.format(name.ljust(name_length)) - - desc = '' - desc_node = get_child(node, 'parameterdescription') - if desc_node: - desc = parse_parblock(desc_node, width=width, - indent=(' ' * len(name))) - + for name, desc in m.items(): + name = ' {}'.format('{{{}}}'.format(name).ljust(name_length)) out += '{}{}\n'.format(name, desc) return out.rstrip() @@ -396,21 +368,26 @@ def render_node(n, text, prefix='', indent='', width=62): def para_as_map(parent, indent='', width=62): - """Parses XML as a map - - The map returned may or may not contain - all of the following keys: - 'text': Text in the para tag - 'params': Information from 's - 'return': TODO - 'seealso': TODO - 'xrefs': TODO + """Extracts a Doxygen XML node to a map. + + Keys: + 'text': Text from this element + 'params': map + 'return': List of @return strings + 'seealso': List of @see strings + 'xrefs': ? """ + chunks = { + 'text': '', + 'params': collections.OrderedDict(), + 'return': [], + 'seealso': [], + 'xrefs': [] + } + if is_inline(parent): - return { - 'text': clean_lines(doc_wrap(render_node(parent, ''), - indent=indent, width=width).strip()) - } + chunks['text'] = clean_lines(doc_wrap(render_node(parent, ''), + indent=indent, width=width).strip()) # Ordered dict of ordered lists. groups = collections.OrderedDict([ @@ -445,26 +422,18 @@ def para_as_map(parent, indent='', width=62): else: text += render_node(child, text, indent=indent, width=width) - chunks = { - 'text': text, - 'params': [], - 'return': [], - 'seealso': [], - 'xrefs': [] - } + chunks['text'] = text - # Generate text from the gathered items. - if len(groups['params']) > 0 and has_nonexcluded_params(groups['params']): + # Generate map from the gathered items. + if len(groups['params']) > 0: for child in groups['params']: - chunks['params'].append(params_as_map(child, width=width)) - if len(groups['return']) > 0: - for child in groups['return']: - chunks['return'].append(render_node( - child, '', indent=indent, width=width).lstrip()) - if len(groups['seealso']) > 0: - for child in groups['seealso']: - chunks['seealso'].append(render_node( - child, '', indent=indent, width=width)) + update_params_map(child, ret_map=chunks['params'], width=width) + for child in groups['return']: + chunks['return'].append(render_node( + child, '', indent=indent, width=width).lstrip()) + for child in groups['seealso']: + chunks['seealso'].append(render_node( + child, '', indent=indent, width=width)) for child in groups['xrefs']: title = get_text(get_child(child, 'xreftitle')) xrefs.add(title) @@ -480,79 +449,33 @@ def render_para(parent, indent='', width=62): NB: Blank lines in a docstring manifest as tags. """ - if is_inline(parent): - return clean_lines(doc_wrap(render_node(parent, ''), - indent=indent, width=width).strip()) + para = para_as_map(parent, indent, width) - # Ordered dict of ordered lists. - groups = collections.OrderedDict([ - ('params', []), - ('return', []), - ('seealso', []), - ('xrefs', []), - ]) - - # Gather nodes into groups. Mostly this is because we want "parameterlist" - # nodes to appear together. - text = '' - kind = '' - last = '' - for child in parent.childNodes: - if child.nodeName == 'parameterlist': - groups['params'].append(child) - elif child.nodeName == 'xrefsect': - groups['xrefs'].append(child) - elif child.nodeName == 'simplesect': - last = kind - kind = child.getAttribute('kind') - if kind == 'return' or (kind == 'note' and last == 'return'): - groups['return'].append(child) - elif kind == 'see': - groups['seealso'].append(child) - elif kind in ('note', 'warning'): - text += render_node(child, text, indent=indent, width=width) - else: - raise RuntimeError('unhandled simplesect: {}\n{}'.format( - child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) - else: - text += render_node(child, text, indent=indent, width=width) + def has_nonexcluded_params(m): + """Returns true if any of the given params has at least + one non-excluded item.""" + if fmt_params_map_as_vimhelp(m) != '': + return True - chunks = [text] # Generate text from the gathered items. - if len(groups['params']) > 0 and has_nonexcluded_params(groups['params']): + chunks = [para['text']] + if len(para['params']) > 0 and has_nonexcluded_params(para['params']): chunks.append('\nParameters: ~') - for child in groups['params']: - chunks.append(render_params(child, width=width)) - if len(groups['return']) > 0: + chunks.append(fmt_params_map_as_vimhelp(para['params'], width=width)) + if len(para['return']) > 0: chunks.append('\nReturn: ~') - for child in groups['return']: - chunks.append(render_node( - child, chunks[-1][-1], indent=indent, width=width)) - if len(groups['seealso']) > 0: + for s in para['return']: + chunks.append(s) + if len(para['seealso']) > 0: chunks.append('\nSee also: ~') - for child in groups['seealso']: - chunks.append(render_node( - child, chunks[-1][-1], indent=indent, width=width)) - for child in groups['xrefs']: - title = get_text(get_child(child, 'xreftitle')) - xrefs.add(title) - xrefdesc = render_para(get_child(child, 'xrefdescription'), width=width) - chunks.append(doc_wrap(xrefdesc, prefix='{}: '.format(title), - width=width) + '\n') + for s in para['seealso']: + chunks.append(s) + for s in para['xrefs']: + chunks.append(s) return clean_lines('\n'.join(chunks).strip()) -def parse_parblock_as_array(parent, prefix='', width=62, indent=''): - """Parses a nested block of tags as an array of maps""" - lines = [] - for child in parent.childNodes: - rendered = para_as_map(child, width=width, indent=indent) - lines.append(rendered) - - return lines - - def parse_parblock(parent, prefix='', width=62, indent=''): """Renders a nested block of tags as Vim help text.""" paragraphs = [] @@ -560,17 +483,17 @@ def parse_parblock(parent, prefix='', width=62, indent=''): paragraphs.append(render_para(child, width=width, indent=indent)) paragraphs.append('') return clean_lines('\n'.join(paragraphs).strip()) -# }}} -def parse_source_xml_for_mpack(filename, mode): - """Collects API functions to be packed as msgpack. +def extract_from_xml(filename, mode, fmt_vimhelp): + """Extracts Doxygen info as maps without formatting the text. Returns two maps: - 1. API functions - 2. Deprecated API functions + 1. Functions + 2. Deprecated functions - Caller decides what to do with the deprecated documentation. + The `fmt_vimhelp` parameter controls some special cases for use by + fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) """ global xrefs xrefs = set() @@ -578,6 +501,7 @@ def parse_source_xml_for_mpack(filename, mode): deprecated_functions = {} # Map of func_name:docstring. dom = minidom.parse(filename) + compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) for member in dom.getElementsByTagName('memberdef'): if member.getAttribute('static') == 'yes' or \ member.getAttribute('kind') != 'function' or \ @@ -609,6 +533,15 @@ def parse_source_xml_for_mpack(filename, mode): annotations = filter(None, map(lambda x: annotation_map.get(x), annotations.split())) + if not fmt_vimhelp: + pass + elif mode == 'lua': + fstem = compoundname.split('.')[0] + fstem = CONFIG[mode]['module_override'].get(fstem, fstem) + vimtag = '*{}.{}()*'.format(fstem, name) + else: + vimtag = '*{}()*'.format(name) + params = [] type_length = 0 @@ -626,180 +559,103 @@ def parse_source_xml_for_mpack(filename, mode): if param_name in param_exclude: continue + if fmt_vimhelp and param_type.endswith('*'): + param_type = param_type.strip('* ') + param_name = '*' + param_name type_length = max(type_length, len(param_type)) params.append((param_type, param_name)) c_args = [] for param_type, param_name in params: - c_args.append(( + c_args.append((' ' if fmt_vimhelp else '') + ( '%s %s' % (param_type.ljust(type_length), param_name)).strip()) - c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) + prefix = '%s(' % name + suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params + if a[0] not in ('void', 'Error')) + if not fmt_vimhelp: + c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) + signature = prefix + suffix + else: + c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, + ',\n'.join(c_args)), + ' ') + + # Minimum 8 chars between signature and vimtag + lhs = (text_width - 8) - len(prefix) + + if len(prefix) + len(suffix) > lhs: + signature = vimtag.rjust(text_width) + '\n' + signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, + func=True) + else: + signature = prefix + suffix + signature += vimtag.rjust(text_width - len(signature)) - doc = '' + paras = [] desc = find_first(member, 'detaileddescription') if desc: - doc = parse_parblock_as_array(desc) + for child in desc.childNodes: + paras.append(para_as_map(child)) #, width=width, indent=indent)) if DEBUG: print(textwrap.indent( re.sub(r'\n\s*\n+', '\n', desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) - prefix = '%s(' % name - suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error')) - - signature = prefix + suffix + fn = { + 'annotations': list(annotations), + 'signature': signature, + 'parameters': params, + 'parameters_doc': [], + 'doc': [], + 'return': '', + 'seealso': '', + } + if fmt_vimhelp: + fn['desc_node'] = desc # HACK :( - parameters_doc = [] - func_doc = [] - func_return = '' - seealso = '' - for m in doc: + for m in paras: if 'text' in m: if not m['text'] == '': - func_doc.append(m['text']) + fn['doc'].append(m['text']) if 'params' in m: - parameters_doc += m['params'] + fn['parameters_doc'] += m['params'] if 'return' in m: - func_return = m['return'] + fn['return'] = m['return'] if 'seealso' in m: - seealso = m['xrefs'] - - function_map = { - 'signature': signature, - 'parameters': params, - 'parameters_doc': parameters_doc, - 'doc': func_doc, - 'return': func_return, - 'seealso': seealso - } + fn['seealso'] = m['xrefs'] if INCLUDE_C_DECL: - function_map['c_decl'] = c_decl + fn['c_decl'] = c_decl if 'Deprecated' in xrefs: - deprecated_functions[name] = function_map + deprecated_functions[name] = fn elif name.startswith(CONFIG[mode]['func_name_prefix']): - functions[name] = function_map + functions[name] = fn xrefs.clear() return (functions, deprecated_functions) -def parse_source_xml(filename, mode): - """Collects API functions. +def fmt_doxygen_xml_as_vimhelp(filename, mode): + """Formats functions from doxygen XML into Vim :help format. + Returns two strings: - 1. API functions - 2. Deprecated API functions - Caller decides what to do with the deprecated documentation. + 1. Functions in Vim :help format + 2. Deprecated functions (handled by caller, or ignored) """ - global xrefs - xrefs = set() functions = {} # Map of func_name:docstring. - deprecated_functions = [] - - dom = minidom.parse(filename) - compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) - for member in dom.getElementsByTagName('memberdef'): - if member.getAttribute('static') == 'yes' or \ - member.getAttribute('kind') != 'function' or \ - member.getAttribute('prot') == 'private' or \ - get_text(get_child(member, 'name')).startswith('_'): - continue - - loc = find_first(member, 'location') - if 'private' in loc.getAttribute('file'): - continue - - return_type = get_text(get_child(member, 'type')) - if return_type == '': - continue - - if return_type.startswith(('ArrayOf', 'DictionaryOf')): - parts = return_type.strip('_').split('_') - return_type = '{}({})'.format(parts[0], ', '.join(parts[1:])) - - name = get_text(get_child(member, 'name')) - - annotations = get_text(get_child(member, 'argsstring')) - if annotations and ')' in annotations: - annotations = annotations.rsplit(')', 1)[-1].strip() - # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of - # non-void functions. Special-case void functions here. - if name == 'nvim_get_mode' and len(annotations) == 0: - annotations += 'FUNC_API_FAST' - annotations = filter(None, map(lambda x: annotation_map.get(x), - annotations.split())) - - if mode == 'lua': - fstem = compoundname.split('.')[0] - fstem = CONFIG[mode]['module_override'].get(fstem, fstem) - vimtag = '*{}.{}()*'.format(fstem, name) - else: - vimtag = '*{}()*'.format(name) - - params = [] - type_length = 0 - - for param in get_children(member, 'param'): - param_type = get_text(get_child(param, 'type')).strip() - param_name = '' - declname = get_child(param, 'declname') - if declname: - param_name = get_text(declname).strip() - elif mode == 'lua': - # that's how it comes out of lua2dox - param_name = param_type - param_type = '' - - if param_name in param_exclude: - continue - - if param_type.endswith('*'): - param_type = param_type.strip('* ') - param_name = '*' + param_name - type_length = max(type_length, len(param_type)) - params.append((param_type, param_name)) - - c_args = [] - for param_type, param_name in params: - c_args.append(' ' + ( - '%s %s' % (param_type.ljust(type_length), param_name)).strip()) - - c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, - ',\n'.join(c_args)), - ' ') - - prefix = '%s(' % name - suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error')) - - # Minimum 8 chars between signature and vimtag - lhs = (text_width - 8) - len(prefix) - - if len(prefix) + len(suffix) > lhs: - signature = vimtag.rjust(text_width) + '\n' - signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, - func=True) - else: - signature = prefix + suffix - signature += vimtag.rjust(text_width - len(signature)) - - doc = '' - desc = find_first(member, 'detaileddescription') - if desc: - doc = parse_parblock(desc) - if DEBUG: - print(textwrap.indent( - re.sub(r'\n\s*\n+', '\n', - desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) + deprecated_functions = {} # Map of func_name:docstring. + fns, deprecated_fns = extract_from_xml(filename, mode, True) + for name, fn in fns.items(): + if fn['desc_node']: + doc = parse_parblock(fn['desc_node']) if not doc: doc = 'TODO: Documentation' - annotations = '\n'.join(annotations) + annotations = '\n'.join(fn['annotations']) if annotations: annotations = ('\n\nAttributes: ~\n' + textwrap.indent(annotations, ' ')) @@ -811,10 +667,10 @@ def parse_source_xml(filename, mode): if INCLUDE_C_DECL: doc += '\n\nC Declaration: ~\n>\n' - doc += c_decl + doc += fn['c_decl'] doc += '\n<' - func_doc = signature + '\n' + func_doc = fn['signature'] + '\n' func_doc += textwrap.indent(clean_lines(doc), ' ' * 16) func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) @@ -826,7 +682,7 @@ def parse_source_xml(filename, mode): xrefs.clear() return ('\n\n'.join(list(functions.values())), - '\n\n'.join(deprecated_functions), + '\n\n'.join(deprecated_fns), functions) @@ -870,7 +726,7 @@ def gen_docs(config): if p.returncode: sys.exit(p.returncode) - mpacks = {} + doc_maps = {} sections = {} intros = {} sep = '=' * text_width @@ -899,10 +755,10 @@ def gen_docs(config): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): - mpack = parse_source_xml_for_mpack(os.path.join(base, '{}.xml'.format( - compound.getAttribute('refid'))), mode) + mpack = extract_from_xml(os.path.join(base, '{}.xml'.format( + compound.getAttribute('refid'))), mode, False) - functions_text, deprecated_text, fns = parse_source_xml( + functions_text, deprecated_text, fns = fmt_doxygen_xml_as_vimhelp( os.path.join(base, '{}.xml'.format( compound.getAttribute('refid'))), mode) # Collect functions from all modules (for the current `mode`). @@ -941,7 +797,7 @@ def gen_docs(config): title = '{} Functions'.format(name) helptag = '*api-{}*'.format(name.lower()) sections[filename] = (title, helptag, doc) - mpacks[filename] = mpack + doc_maps[filename] = mpack if not sections: return @@ -974,7 +830,7 @@ def gen_docs(config): fp.write(docs.encode('utf8')) with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(mpacks, use_bin_type=True)) + fp.write(msgpack.packb(doc_maps, use_bin_type=True)) shutil.rmtree(output_dir) @@ -994,42 +850,40 @@ def filter_source(filename): fp.read(), flags=re.M)) -# Doxygen Config {{{ -Doxyfile = ''' -OUTPUT_DIRECTORY = {output} -INPUT = {input} -INPUT_ENCODING = UTF-8 -FILE_PATTERNS = {file_patterns} -RECURSIVE = YES -INPUT_FILTER = "{filter}" -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = */private/* -EXCLUDE_SYMBOLS = -EXTENSION_MAPPING = lua=C -EXTRACT_PRIVATE = NO - -GENERATE_HTML = NO -GENERATE_DOCSET = NO -GENERATE_HTMLHELP = NO -GENERATE_QHP = NO -GENERATE_TREEVIEW = NO -GENERATE_LATEX = NO -GENERATE_RTF = NO -GENERATE_MAN = NO -GENERATE_DOCBOOK = NO -GENERATE_AUTOGEN_DEF = NO - -GENERATE_XML = YES -XML_OUTPUT = xml -XML_PROGRAMLISTING = NO - -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = YES -EXPAND_ONLY_PREDEF = NO -MARKDOWN_SUPPORT = YES -''' -# }}} +Doxyfile = textwrap.dedent(''' + OUTPUT_DIRECTORY = {output} + INPUT = {input} + INPUT_ENCODING = UTF-8 + FILE_PATTERNS = {file_patterns} + RECURSIVE = YES + INPUT_FILTER = "{filter}" + EXCLUDE = + EXCLUDE_SYMLINKS = NO + EXCLUDE_PATTERNS = */private/* + EXCLUDE_SYMBOLS = + EXTENSION_MAPPING = lua=C + EXTRACT_PRIVATE = NO + + GENERATE_HTML = NO + GENERATE_DOCSET = NO + GENERATE_HTMLHELP = NO + GENERATE_QHP = NO + GENERATE_TREEVIEW = NO + GENERATE_LATEX = NO + GENERATE_RTF = NO + GENERATE_MAN = NO + GENERATE_DOCBOOK = NO + GENERATE_AUTOGEN_DEF = NO + + GENERATE_XML = YES + XML_OUTPUT = xml + XML_PROGRAMLISTING = NO + + ENABLE_PREPROCESSING = YES + MACRO_EXPANSION = YES + EXPAND_ONLY_PREDEF = NO + MARKDOWN_SUPPORT = YES +''') if __name__ == "__main__": if len(sys.argv) > 1: @@ -1037,4 +891,4 @@ if __name__ == "__main__": else: gen_docs(Doxyfile) -# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker : +# vim: set ft=python ts=4 sw=4 tw=79 et : -- cgit From 481da1ce4060fd3045ac6553cef69f4fa4f9829d Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 19 Dec 2019 16:52:37 -0800 Subject: gen_vimdoc.py: fix mpack generator - make parameters_doc a dict intead of a list BEFORE: "parameters_doc": [ { "buffer": "Buffer handle, or 0 for current buffer" } ], AFTER: "parameters_doc": { "buffer": "Buffer handle, or 0 for current buffer" }, - make "return", "seealso", lists instead of strings --- scripts/gen_vimdoc.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index cd44667ef3..368eaf48f1 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -606,10 +606,10 @@ def extract_from_xml(filename, mode, fmt_vimhelp): 'annotations': list(annotations), 'signature': signature, 'parameters': params, - 'parameters_doc': [], + 'parameters_doc': collections.OrderedDict(), 'doc': [], - 'return': '', - 'seealso': '', + 'return': [], + 'seealso': [], } if fmt_vimhelp: fn['desc_node'] = desc # HACK :( @@ -619,11 +619,12 @@ def extract_from_xml(filename, mode, fmt_vimhelp): if not m['text'] == '': fn['doc'].append(m['text']) if 'params' in m: - fn['parameters_doc'] += m['params'] - if 'return' in m: - fn['return'] = m['return'] - if 'seealso' in m: - fn['seealso'] = m['xrefs'] + # Merge OrderedDicts. + fn['parameters_doc'].update(m['params']) + if 'return' in m and len(m['return']) > 0: + fn['return'] += m['return'] + if 'seealso' in m and len(m['xrefs']) > 0: + fn['seealso'].append(str(m['xrefs'])) if INCLUDE_C_DECL: fn['c_decl'] = c_decl -- cgit From 62e365f5776cfe8e460fdd29a1d7fa89a5c9c0bc Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 19 Dec 2019 17:10:34 -0800 Subject: gen_vimdoc.py: mpack: exclude deprecated functions The `mpack` variable was a tuple, which manifests as an array in the generated msgpack structure. - Removes noise from the mpack data (deprecated functions are deprecated). - Eliminates 1 level of nesting. BEFORE: [ { "buffer.c": [ { "nvim__buf_stats": { ... }, ... }, { "buffer_del_line": { ... }, ... }, ], ... } ] AFTER: [ { "buffer.c": { "nvim__buf_stats": { ... }, ... }, ... ] --- scripts/gen_vimdoc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 368eaf48f1..2ca121c299 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -727,7 +727,7 @@ def gen_docs(config): if p.returncode: sys.exit(p.returncode) - doc_maps = {} + fn_maps = {} sections = {} intros = {} sep = '=' * text_width @@ -756,7 +756,7 @@ def gen_docs(config): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): - mpack = extract_from_xml(os.path.join(base, '{}.xml'.format( + fn_map, _ = extract_from_xml(os.path.join(base, '{}.xml'.format( compound.getAttribute('refid'))), mode, False) functions_text, deprecated_text, fns = fmt_doxygen_xml_as_vimhelp( @@ -798,7 +798,7 @@ def gen_docs(config): title = '{} Functions'.format(name) helptag = '*api-{}*'.format(name.lower()) sections[filename] = (title, helptag, doc) - doc_maps[filename] = mpack + fn_maps[filename] = fn_map if not sections: return @@ -831,7 +831,7 @@ def gen_docs(config): fp.write(docs.encode('utf8')) with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(doc_maps, use_bin_type=True)) + fp.write(msgpack.packb(fn_maps, use_bin_type=True)) shutil.rmtree(output_dir) -- cgit From f968dad3bff5dd4e0d8d797b561fcb2c76553a1c Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 19 Dec 2019 17:48:40 -0800 Subject: gen_vimdoc.py: fix "seealso", "xrefs" - Also fix xrefs ("Deprecated" section) - Fix "Deprecated" rendering by a weird hack (see comment). - Eliminate unnecessary use of render_para() --- scripts/gen_vimdoc.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 2ca121c299..676e6b5be9 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -435,9 +435,11 @@ def para_as_map(parent, indent='', width=62): chunks['seealso'].append(render_node( child, '', indent=indent, width=width)) for child in groups['xrefs']: - title = get_text(get_child(child, 'xreftitle')) + # XXX: Add a space (or any char) to `title` here, otherwise xrefs + # ("Deprecated" section) acts very weird... + title = get_text(get_child(child, 'xreftitle')) + ' ' xrefs.add(title) - xrefdesc = render_para(get_child(child, 'xrefdescription'), width=width) + xrefdesc = get_text(get_child(child, 'xrefdescription')) chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title), width=width) + '\n') @@ -496,7 +498,7 @@ def extract_from_xml(filename, mode, fmt_vimhelp): fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) """ global xrefs - xrefs = set() + xrefs.clear() functions = {} # Map of func_name:docstring. deprecated_functions = {} # Map of func_name:docstring. @@ -623,8 +625,8 @@ def extract_from_xml(filename, mode, fmt_vimhelp): fn['parameters_doc'].update(m['params']) if 'return' in m and len(m['return']) > 0: fn['return'] += m['return'] - if 'seealso' in m and len(m['xrefs']) > 0: - fn['seealso'].append(str(m['xrefs'])) + if 'seealso' in m and len(m['seealso']) > 0: + fn['seealso'] += m['seealso'] if INCLUDE_C_DECL: fn['c_decl'] = c_decl -- cgit From 4657819e80e3493564c0dd7cf16864aa9bf73c69 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 19 Dec 2019 18:17:43 -0800 Subject: gen_vimdoc.py: mpack: collect functions in 1 dict All Nvim API, core Vimscript, and core Lua functions are globally unique, so there is no need for per-module nested dicts. BEFORE (generated mpack structure): [ { "buffer.c": { "nvim__buf_stats": { ... }, ... }, "window.c": { "nvim_win_close": { ... }, ... }, ... } ] AFTER (generated mpack structure): [ { "nvim__buf_stats": { ... }, "nvim_buf_attach": { ... }, "nvim_tabpage_set_var": { ... }, "nvim_ui_attach": { ... }, "nvim_win_close": { ... } } ] --- scripts/gen_vimdoc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 676e6b5be9..229a68a462 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -729,7 +729,7 @@ def gen_docs(config): if p.returncode: sys.exit(p.returncode) - fn_maps = {} + fn_map_full = {} # Collects all functions as each module is processed. sections = {} intros = {} sep = '=' * text_width @@ -800,7 +800,7 @@ def gen_docs(config): title = '{} Functions'.format(name) helptag = '*api-{}*'.format(name.lower()) sections[filename] = (title, helptag, doc) - fn_maps[filename] = fn_map + fn_map_full.update(fn_map) if not sections: return @@ -833,7 +833,7 @@ def gen_docs(config): fp.write(docs.encode('utf8')) with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(fn_maps, use_bin_type=True)) + fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) shutil.rmtree(output_dir) -- cgit From 9ea7aeb3086bf1226f673c0736a757f6e6256ec1 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 19 Dec 2019 19:42:24 -0800 Subject: gen_vimdoc.py: rename for clarity - render_para => fmt_node_as_vimhelp - Inline parse_parblock() in fmt_node_as_vimhelp() --- scripts/gen_vimdoc.py | 102 +++++++++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 46 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 229a68a462..50accbdde8 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -1,5 +1,18 @@ #!/usr/bin/env python3 -"""Generates Nvim help docs from C docstrings, by parsing Doxygen XML. +"""Generates Nvim help docs from C/Lua docstrings, using Doxygen. + +Also generates *.mpack files. To inspect the *.mpack structure: + + :new | put=json_encode(msgpackparse(readfile('runtime/doc/api.mpack'))) + +Flow: + gen_docs + extract_from_xml + fmt_node_as_vimhelp + fmt_params_map_as_vimhelp + render_node + para_as_map + render_node This would be easier using lxml and XSLT, but: @@ -262,9 +275,9 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None): def update_params_map(parent, ret_map, width=62): """Updates `ret_map` with name:desc key-value pairs extracted - from a Doxygen . + from Doxygen XML node `parent`. """ - items = [] + params = [] for node in parent.childNodes: if node.nodeType == node.TEXT_NODE: continue @@ -274,13 +287,13 @@ def update_params_map(parent, ret_map, width=62): name = get_text(name_node) if name in param_exclude: continue - items.append((name.strip(), node)) + params.append((name.strip(), node)) # `ret_map` is a name:desc map. - for name, node in items: + for name, node in params: desc = '' desc_node = get_child(node, 'parameterdescription') if desc_node: - desc = parse_parblock(desc_node, width=width, + desc = fmt_node_as_vimhelp(desc_node, width=width, indent=(' ' * len(name))) ret_map[name] = desc return ret_map @@ -288,12 +301,12 @@ def update_params_map(parent, ret_map, width=62): def fmt_params_map_as_vimhelp(m, width=62): """Renders a params map as Vim :help text.""" - name_length = 0 + max_name_len = 0 for name, desc in m.items(): - name_length = max(name_length, len(name) + 4) + max_name_len = max(max_name_len, len(name) + 4) out = '' for name, desc in m.items(): - name = ' {}'.format('{{{}}}'.format(name).ljust(name_length)) + name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len)) out += '{}{}\n'.format(name, desc) return out.rstrip() @@ -446,45 +459,40 @@ def para_as_map(parent, indent='', width=62): return chunks -def render_para(parent, indent='', width=62): - """Renders Doxygen containing arbitrary nodes. +def fmt_node_as_vimhelp(parent, width=62, indent=''): + """Renders (nested) Doxygen nodes as Vim :help text. NB: Blank lines in a docstring manifest as tags. """ - para = para_as_map(parent, indent, width) - - def has_nonexcluded_params(m): - """Returns true if any of the given params has at least - one non-excluded item.""" - if fmt_params_map_as_vimhelp(m) != '': - return True - - # Generate text from the gathered items. - chunks = [para['text']] - if len(para['params']) > 0 and has_nonexcluded_params(para['params']): - chunks.append('\nParameters: ~') - chunks.append(fmt_params_map_as_vimhelp(para['params'], width=width)) - if len(para['return']) > 0: - chunks.append('\nReturn: ~') - for s in para['return']: - chunks.append(s) - if len(para['seealso']) > 0: - chunks.append('\nSee also: ~') - for s in para['seealso']: + rendered_blocks = [] + for child in parent.childNodes: + para = para_as_map(child, indent, width) + + def has_nonexcluded_params(m): + """Returns true if any of the given params has at least + one non-excluded item.""" + if fmt_params_map_as_vimhelp(m) != '': + return True + + # Generate text from the gathered items. + chunks = [para['text']] + if len(para['params']) > 0 and has_nonexcluded_params(para['params']): + chunks.append('\nParameters: ~') + chunks.append(fmt_params_map_as_vimhelp(para['params'], width=width)) + if len(para['return']) > 0: + chunks.append('\nReturn: ~') + for s in para['return']: + chunks.append(s) + if len(para['seealso']) > 0: + chunks.append('\nSee also: ~') + for s in para['seealso']: + chunks.append(s) + for s in para['xrefs']: chunks.append(s) - for s in para['xrefs']: - chunks.append(s) - - return clean_lines('\n'.join(chunks).strip()) - -def parse_parblock(parent, prefix='', width=62, indent=''): - """Renders a nested block of tags as Vim help text.""" - paragraphs = [] - for child in parent.childNodes: - paragraphs.append(render_para(child, width=width, indent=indent)) - paragraphs.append('') - return clean_lines('\n'.join(paragraphs).strip()) + rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) + rendered_blocks.append('') + return clean_lines('\n'.join(rendered_blocks).strip()) def extract_from_xml(filename, mode, fmt_vimhelp): @@ -653,8 +661,9 @@ def fmt_doxygen_xml_as_vimhelp(filename, mode): fns, deprecated_fns = extract_from_xml(filename, mode, True) for name, fn in fns.items(): + # Generate Vim :help for parameters. if fn['desc_node']: - doc = parse_parblock(fn['desc_node']) + doc = fmt_node_as_vimhelp(fn['desc_node']) if not doc: doc = 'TODO: Documentation' @@ -704,7 +713,8 @@ def delete_lines_below(filename, tokenstr): def gen_docs(config): - """Generate documentation. + """Generate formatted Vim :help docs and unformatted *.mpack files for use + by API clients. Doxygen is called and configured through stdin. """ @@ -748,7 +758,7 @@ def gen_docs(config): desc = find_first(minidom.parse(groupxml), 'detaileddescription') if desc: - doc = parse_parblock(desc) + doc = fmt_node_as_vimhelp(desc) if doc: intros[groupname] = doc -- cgit From 2e280dac7df504b0681043647c8cc02abcbcc686 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Mon, 23 Dec 2019 07:08:49 +0100 Subject: gen_vimdoc.py: lint #11593 --- scripts/gen_vimdoc.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 876d46c18f..3ceb5d99e2 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -141,7 +141,7 @@ def debug_this(cond, o): try: name = o.nodeName o = o.toprettyxml(indent=' ', newl='\n') - except: + except Exception: pass if ((callable(cond) and cond()) or (not callable(cond) and cond in o)): @@ -271,8 +271,6 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None): return result - - def update_params_map(parent, ret_map, width=62): """Updates `ret_map` with name:desc key-value pairs extracted from Doxygen XML node `parent`. @@ -293,8 +291,7 @@ def update_params_map(parent, ret_map, width=62): desc = '' desc_node = get_child(node, 'parameterdescription') if desc_node: - desc = fmt_node_as_vimhelp(desc_node, width=width, - indent=(' ' * len(name))) + desc = fmt_node_as_vimhelp(desc_node, width=width, indent=(" " * len(name))) ret_map[name] = desc return ret_map @@ -399,8 +396,9 @@ def para_as_map(parent, indent='', width=62): } if is_inline(parent): - chunks['text'] = clean_lines(doc_wrap(render_node(parent, ''), - indent=indent, width=width).strip()) + chunks["text"] = clean_lines( + doc_wrap(render_node(parent, ""), indent=indent, width=width).strip() + ) # Ordered dict of ordered lists. groups = collections.OrderedDict([ @@ -606,7 +604,7 @@ def extract_from_xml(filename, mode, fmt_vimhelp): desc = find_first(member, 'detaileddescription') if desc: for child in desc.childNodes: - paras.append(para_as_map(child)) #, width=width, indent=indent)) + paras.append(para_as_map(child)) if DEBUG: print(textwrap.indent( re.sub(r'\n\s*\n+', '\n', -- cgit From 34abe8fd23e7760fff4824c08dd8ee7eb4f7dd67 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 24 Dec 2019 08:15:18 +0100 Subject: vim-patch.sh: list related missing Vim patches [ci skip] #11514 * scripts/vim-patch.sh: factor out _set_tokens_and_tags This allows for caching `$tokens` and `$vim_commit_tags`, which will become relevant with the next commit adding `list_missing_previous_vimpatches_for_patch`. --- scripts/vim-patch.sh | 122 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 100 insertions(+), 22 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index b6a0df4649..483264415e 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -395,13 +395,48 @@ list_vimpatch_numbers() { done } +declare -A tokens +declare -A vim_commit_tags + +_set_tokens_and_tags() { + if [[ -n "${tokens[*]}" ]]; then + return + fi + + # Find all "vim-patch:xxx" tokens in the Nvim git log. + for token in $(list_vimpatch_tokens); do + tokens[$token]=1 + done + + # Create an associative array mapping Vim commits to tags. + eval "vim_commit_tags=( + $(git -C "${VIM_SOURCE_DIR}" for-each-ref refs/tags \ + --format '[%(objectname)]=%(refname:strip=2)' \ + --sort='-*authordate' \ + --shell) + )" + # Exit in case of errors from the above eval (empty vim_commit_tags). + if ! (( "${#vim_commit_tags[@]}" )); then + msg_err "Could not get Vim commits/tags." + exit 1 + fi +} + # Prints a newline-delimited list of Vim commits, for use by scripts. # "$1": use extended format? # "$@" is passed to list_vim_commits, as extra arguments to git-log. list_missing_vimpatches() { + local -a missing_vim_patches=() + _set_missing_vimpatches "$@" + for line in "${missing_vim_patches[@]}"; do + printf '%s\n' "$line" + done +} + +# Sets / appends to missing_vim_patches (useful to avoid a subshell when +# used multiple times to cache tokens/vim_commit_tags). +_set_missing_vimpatches() { local token vim_commit vim_tag patch_number - declare -A tokens - declare -A vim_commit_tags declare -a git_log_args local extended_format=$1; shift @@ -416,6 +451,7 @@ list_missing_vimpatches() { [^\(.*/\)?src/nvim/\(.*\)]="\${BASH_REMATCH[1]}src/\${BASH_REMATCH[2]}" [^\(.*/\)?\.vim-src/\(.*\)]="\${BASH_REMATCH[2]}" ) + local i j for i in "$@"; do for j in "${!git_log_replacements[@]}"; do if [[ "$i" =~ $j ]]; then @@ -426,23 +462,7 @@ list_missing_vimpatches() { git_log_args+=("$i") done - # Find all "vim-patch:xxx" tokens in the Nvim git log. - for token in $(list_vimpatch_tokens); do - tokens[$token]=1 - done - - # Create an associative array mapping Vim commits to tags. - eval "declare -A vim_commit_tags=( - $(git -C "${VIM_SOURCE_DIR}" for-each-ref refs/tags \ - --format '[%(objectname)]=%(refname:strip=2)' \ - --sort='-*authordate' \ - --shell) - )" - # Exit in case of errors from the above eval (empty vim_commit_tags). - if ! (( "${#vim_commit_tags[@]}" )); then - msg_err "Could not get Vim commits/tags." - exit 1 - fi + _set_tokens_and_tags # Get missing Vim commits set +u # Avoid "unbound variable" with bash < 4.4 below. @@ -474,9 +494,9 @@ list_missing_vimpatches() { if [[ "${tokens[$patch_number]-}" ]]; then continue fi - printf '%s%s\n' "$vim_tag" "$info" + missing_vim_patches+=("$vim_tag$info") else - printf '%s%s\n' "$vim_commit" "$info" + missing_vim_patches+=("$vim_commit$info") fi done < <(list_vim_commits "${git_log_args[@]}") set -u @@ -512,6 +532,59 @@ Instructions: EOF } +list_missing_previous_vimpatches_for_patch() { + local for_vim_patch="${1}" + local vim_commit vim_tag + assign_commit_details "${for_vim_patch}" + + local file + local -a missing_list + local -a fnames + while IFS= read -r line ; do + fnames+=("$line") + done < <(git -C "${VIM_SOURCE_DIR}" diff-tree --no-commit-id --name-only -r "${vim_commit}") + local i=0 + local n=${#fnames[@]} + printf '=== getting missing patches for %d files ===\n' "$n" + if [[ -z "${vim_tag}" ]]; then + printf 'NOTE: "%s" is not a Vim tag - listing all oldest missing patches\n' "${for_vim_patch}" >&2 + fi + for fname in "${fnames[@]}"; do + i=$(( i+1 )) + printf '[%.*d/%d] %s: ' "${#n}" "$i" "$n" "$fname" + + local -a missing_vim_patches=() + _set_missing_vimpatches 1 -- "${fname}" + + local missing_vim_commit_info="${missing_vim_patches[0]}" + if [[ -z "${missing_vim_commit_info}" ]]; then + printf -- "-\n" + else + local missing_vim_commit="${missing_vim_commit_info%%:*}" + if [[ -z "${vim_tag}" ]] || [[ "${missing_vim_commit}" < "${vim_tag}" ]]; then + printf -- "%s\n" "$missing_vim_commit_info" + missing_list+=("$missing_vim_commit_info") + else + printf -- "-\n" + fi + fi + done + + if [[ -z "${missing_list[*]}" ]]; then + msg_ok 'no missing previous Vim patches' + return 0 + fi + + local -a missing_unique + while IFS= read -r line; do + missing_unique+=("$line") + done < <(printf '%s\n' "${missing_list[@]}" | sort -u) + + msg_err "$(printf '%d missing previous Vim patches:' ${#missing_unique[@]})" + printf ' - %s\n' "${missing_unique[@]}" + return 1 +} + review_commit() { local nvim_commit_url="${1}" local nvim_patch_url="${nvim_commit_url}.patch" @@ -609,7 +682,7 @@ review_pr() { clean_files } -while getopts "hlLMVp:P:g:r:s" opt; do +while getopts "hlLmMVp:P:g:r:s" opt; do case ${opt} in h) usage @@ -629,6 +702,11 @@ while getopts "hlLMVp:P:g:r:s" opt; do list_vimpatch_numbers exit 0 ;; + m) + shift # remove opt + list_missing_previous_vimpatches_for_patch "$@" + exit 0 + ;; p) stage_patch "${OPTARG}" exit -- cgit From 5f1aec5abdb551e5f3035ca054d36580b3233efb Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 25 Dec 2019 13:02:23 +0100 Subject: Fix scripts/vim-patch.sh for Bash 4.3 Ref: https://github.com/neovim/neovim/pull/11514#issuecomment-568780231 --- scripts/vim-patch.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 483264415e..b5806311c9 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -399,9 +399,11 @@ declare -A tokens declare -A vim_commit_tags _set_tokens_and_tags() { + set +u # Avoid "unbound variable" with bash < 4.4 below. if [[ -n "${tokens[*]}" ]]; then return fi + set -u # Find all "vim-patch:xxx" tokens in the Nvim git log. for token in $(list_vimpatch_tokens); do -- cgit From 81a0d10f131ddea0bd5faa50e362deb97c0abb08 Mon Sep 17 00:00:00 2001 From: Jan Edmund Lazo Date: Sat, 28 Dec 2019 00:11:38 -0500 Subject: fixup! vim-patch.sh: list related missing Vim patches [ci skip] #11514 --- scripts/vim-patch.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index b5806311c9..769f0fed58 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -430,9 +430,11 @@ _set_tokens_and_tags() { list_missing_vimpatches() { local -a missing_vim_patches=() _set_missing_vimpatches "$@" + set +u # Avoid "unbound variable" with bash < 4.4 below. for line in "${missing_vim_patches[@]}"; do printf '%s\n' "$line" done + set -u } # Sets / appends to missing_vim_patches (useful to avoid a subshell when -- cgit From 27b678f577eb422ccc67e803727c3b26756787f3 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sat, 28 Dec 2019 02:15:06 -0800 Subject: gen_vimdoc.py: fix deprecated check --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 3ceb5d99e2..93851ed17e 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -637,7 +637,7 @@ def extract_from_xml(filename, mode, fmt_vimhelp): if INCLUDE_C_DECL: fn['c_decl'] = c_decl - if 'Deprecated' in xrefs: + if 'Deprecated' in str(xrefs): deprecated_functions[name] = fn elif name.startswith(CONFIG[mode]['func_name_prefix']): functions[name] = fn -- cgit From b81547ce6d182e2a7518e5a2f80d1ee90c5cdb8f Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Tue, 24 Dec 2019 00:04:14 -0800 Subject: gen_vimdoc.py: better handling of inline (non-block) nodes --- scripts/gen_vimdoc.py | 258 +++++++++++++++++++++++++++----------------------- 1 file changed, 141 insertions(+), 117 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 93851ed17e..891336f571 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -1,18 +1,18 @@ #!/usr/bin/env python3 -"""Generates Nvim help docs from C/Lua docstrings, using Doxygen. +"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen. Also generates *.mpack files. To inspect the *.mpack structure: - :new | put=json_encode(msgpackparse(readfile('runtime/doc/api.mpack'))) + :new | put=v:lua.vim.inspect(msgpackparse(readfile('runtime/doc/api.mpack'))) + Flow: - gen_docs + main extract_from_xml - fmt_node_as_vimhelp - fmt_params_map_as_vimhelp - render_node - para_as_map - render_node + fmt_node_as_vimhelp \ + para_as_map } recursive + update_params_map / + render_node This would be easier using lxml and XSLT, but: @@ -58,6 +58,7 @@ DEBUG = ('DEBUG' in os.environ) INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) +fmt_vimhelp = False # HACK text_width = 78 script_path = os.path.abspath(__file__) base_dir = os.path.dirname(os.path.dirname(script_path)) @@ -144,6 +145,7 @@ def debug_this(cond, o): except Exception: pass if ((callable(cond) and cond()) + or (not callable(cond) and cond == True) or (not callable(cond) and cond in o)): raise RuntimeError('xxx: {}\n{}'.format(name, o)) @@ -156,20 +158,27 @@ def find_first(parent, name): return sub[0] -def get_children(parent, name): - """Yield matching child nodes within parent.""" +def iter_children(parent, name): + """Yields matching child nodes within parent.""" for child in parent.childNodes: if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: yield child def get_child(parent, name): - """Get the first matching child node.""" - for child in get_children(parent, name): + """Gets the first matching child node.""" + for child in iter_children(parent, name): return child return None +def self_or_child(n): + """Gets the first child node, or self.""" + if len(n.childNodes) == 0: + return n + return n.childNodes[0] + + def clean_text(text): """Cleans text. @@ -190,18 +199,21 @@ def is_blank(text): return '' == clean_lines(text) -def get_text(parent, preformatted=False): - """Combine all text in a node.""" - if parent.nodeType == parent.TEXT_NODE: - return parent.data - - out = '' - for node in parent.childNodes: +def get_text(n, preformatted=False): + """Recursively concatenates all text in a node tree.""" + text = '' + if n.nodeType == n.TEXT_NODE: + return n.data + if n.nodeName == 'computeroutput': + for node in n.childNodes: + text += get_text(node) + return '`{}` '.format(text) + for node in n.childNodes: if node.nodeType == node.TEXT_NODE: - out += node.data if preformatted else clean_text(node.data) + text += node.data if preformatted else clean_text(node.data) elif node.nodeType == node.ELEMENT_NODE: - out += ' ' + get_text(node, preformatted) - return out + text += ' ' + get_text(node, preformatted) + return text # Gets the length of the last line in `text`, excluding newline ("\n") char. @@ -221,6 +233,8 @@ def len_lastline_withoutindent(text, indent): # Returns True if node `n` contains only inline (not block-level) elements. def is_inline(n): + # if len(n.childNodes) == 0: + # return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput' for c in n.childNodes: if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput': return False @@ -271,11 +285,17 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None): return result +def max_name(names): + if len(names) == 0: + return 0 + return max(len(name) for name in names) + + def update_params_map(parent, ret_map, width=62): """Updates `ret_map` with name:desc key-value pairs extracted from Doxygen XML node `parent`. """ - params = [] + params = collections.OrderedDict() for node in parent.childNodes: if node.nodeType == node.TEXT_NODE: continue @@ -285,48 +305,34 @@ def update_params_map(parent, ret_map, width=62): name = get_text(name_node) if name in param_exclude: continue - params.append((name.strip(), node)) + params[name.strip()] = node + max_name_len = max_name(params.keys()) + 8 # `ret_map` is a name:desc map. - for name, node in params: + for name, node in params.items(): desc = '' desc_node = get_child(node, 'parameterdescription') if desc_node: - desc = fmt_node_as_vimhelp(desc_node, width=width, indent=(" " * len(name))) + desc = fmt_node_as_vimhelp(desc_node, width=width, + indent=(' ' * max_name_len)) ret_map[name] = desc return ret_map -def fmt_params_map_as_vimhelp(m, width=62): - """Renders a params map as Vim :help text.""" - max_name_len = 0 - for name, desc in m.items(): - max_name_len = max(max_name_len, len(name) + 4) - out = '' - for name, desc in m.items(): - name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len)) - out += '{}{}\n'.format(name, desc) - return out.rstrip() - - def render_node(n, text, prefix='', indent='', width=62): """Renders a node as Vim help text, recursively traversing all descendants.""" + global fmt_vimhelp + def ind(s): + return s if fmt_vimhelp else '' text = '' # space_preceding = (len(text) > 0 and ' ' == text[-1][-1]) # text += (int(not space_preceding) * ' ') - if n.nodeType == n.TEXT_NODE: - # `prefix` is NOT sent to doc_wrap, it was already handled by now. - text += doc_wrap(n.data, indent=indent, width=width) - elif n.nodeName == 'computeroutput': - text += ' `{}` '.format(get_text(n)) - elif n.nodeName == 'preformatted': + if n.nodeName == 'preformatted': o = get_text(n, preformatted=True) ensure_nl = '' if o[-1] == '\n' else '\n' - text += ' >{}{}\n<'.format(ensure_nl, o) + text += '>{}{}\n<'.format(ensure_nl, o) elif is_inline(n): - for c in n.childNodes: - text += render_node(c, text) - text = doc_wrap(text, indent=indent, width=width) + text = doc_wrap(get_text(n), indent=indent, width=width) elif n.nodeName == 'verbatim': # TODO: currently we don't use this. The "[verbatim]" hint is there as # a reminder that we must decide how to format this if we do use it. @@ -341,8 +347,6 @@ def render_node(n, text, prefix='', indent='', width=62): elif n.nodeName in ('para', 'heading'): for c in n.childNodes: text += render_node(c, text, indent=indent, width=width) - if is_inline(n): - text = doc_wrap(text, indent=indent, width=width) elif n.nodeName == 'itemizedlist': for c in n.childNodes: text += '{}\n'.format(render_node(c, text, prefix='• ', @@ -368,7 +372,7 @@ def render_node(n, text, prefix='', indent='', width=62): text += '\n' elif (n.nodeName == 'simplesect' and n.getAttribute('kind') in ('return', 'see')): - text += ' ' + text += ind(' ') for c in n.childNodes: text += render_node(c, text, indent=' ', width=width) else: @@ -395,11 +399,6 @@ def para_as_map(parent, indent='', width=62): 'xrefs': [] } - if is_inline(parent): - chunks["text"] = clean_lines( - doc_wrap(render_node(parent, ""), indent=indent, width=width).strip() - ) - # Ordered dict of ordered lists. groups = collections.OrderedDict([ ('params', []), @@ -413,27 +412,39 @@ def para_as_map(parent, indent='', width=62): text = '' kind = '' last = '' - for child in parent.childNodes: - if child.nodeName == 'parameterlist': - groups['params'].append(child) - elif child.nodeName == 'xrefsect': - groups['xrefs'].append(child) - elif child.nodeName == 'simplesect': - last = kind - kind = child.getAttribute('kind') - if kind == 'return' or (kind == 'note' and last == 'return'): - groups['return'].append(child) - elif kind == 'see': - groups['seealso'].append(child) - elif kind in ('note', 'warning'): - text += render_node(child, text, indent=indent, width=width) + if is_inline(parent): + # Flatten inline text from a tree of non-block nodes. + text = doc_wrap(render_node(parent, ""), indent=indent, width=width) + else: + prev = None # Previous node + for child in parent.childNodes: + if child.nodeName == 'parameterlist': + groups['params'].append(child) + elif child.nodeName == 'xrefsect': + groups['xrefs'].append(child) + elif child.nodeName == 'simplesect': + last = kind + kind = child.getAttribute('kind') + if kind == 'return' or (kind == 'note' and last == 'return'): + groups['return'].append(child) + elif kind == 'see': + groups['seealso'].append(child) + elif kind in ('note', 'warning'): + text += render_node(child, text, indent=indent, width=width) + else: + raise RuntimeError('unhandled simplesect: {}\n{}'.format( + child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) else: - raise RuntimeError('unhandled simplesect: {}\n{}'.format( - child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) - else: - text += render_node(child, text, indent=indent, width=width) + if (prev is not None + and is_inline(self_or_child(prev)) + and is_inline(self_or_child(child)) + and '' != get_text(self_or_child(child)).strip() + and ' ' != text[-1]): + text += ' ' + text += render_node(child, text, indent=indent, width=width) + prev = child - chunks['text'] = text + chunks['text'] += text # Generate map from the gathered items. if len(groups['params']) > 0: @@ -441,7 +452,7 @@ def para_as_map(parent, indent='', width=62): update_params_map(child, ret_map=chunks['params'], width=width) for child in groups['return']: chunks['return'].append(render_node( - child, '', indent=indent, width=width).lstrip()) + child, '', indent=indent, width=width)) for child in groups['seealso']: chunks['seealso'].append(render_node( child, '', indent=indent, width=width)) @@ -463,20 +474,30 @@ def fmt_node_as_vimhelp(parent, width=62, indent=''): NB: Blank lines in a docstring manifest as tags. """ rendered_blocks = [] + + def fmt_param_doc(m): + """Renders a params map as Vim :help text.""" + max_name_len = max_name(m.keys()) + 4 + out = '' + for name, desc in m.items(): + name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len)) + out += '{}{}\n'.format(name, desc) + return out.rstrip() + + def has_nonexcluded_params(m): + """Returns true if any of the given params has at least + one non-excluded item.""" + if fmt_param_doc(m) != '': + return True + for child in parent.childNodes: para = para_as_map(child, indent, width) - def has_nonexcluded_params(m): - """Returns true if any of the given params has at least - one non-excluded item.""" - if fmt_params_map_as_vimhelp(m) != '': - return True - # Generate text from the gathered items. chunks = [para['text']] if len(para['params']) > 0 and has_nonexcluded_params(para['params']): chunks.append('\nParameters: ~') - chunks.append(fmt_params_map_as_vimhelp(para['params'], width=width)) + chunks.append(fmt_param_doc(para['params'])) if len(para['return']) > 0: chunks.append('\nReturn: ~') for s in para['return']: @@ -493,20 +514,21 @@ def fmt_node_as_vimhelp(parent, width=62, indent=''): return clean_lines('\n'.join(rendered_blocks).strip()) -def extract_from_xml(filename, mode, fmt_vimhelp): +def extract_from_xml(filename, mode, width): """Extracts Doxygen info as maps without formatting the text. Returns two maps: 1. Functions 2. Deprecated functions - The `fmt_vimhelp` parameter controls some special cases for use by + The `fmt_vimhelp` global controls some special cases for use by fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) """ global xrefs + global fmt_vimhelp xrefs.clear() - functions = {} # Map of func_name:docstring. - deprecated_functions = {} # Map of func_name:docstring. + fns = {} # Map of func_name:docstring. + deprecated_fns = {} # Map of func_name:docstring. dom = minidom.parse(filename) compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) @@ -553,7 +575,7 @@ def extract_from_xml(filename, mode, fmt_vimhelp): params = [] type_length = 0 - for param in get_children(member, 'param'): + for param in iter_children(member, 'param'): param_type = get_text(get_child(param, 'type')).strip() param_name = '' declname = get_child(param, 'declname') @@ -590,15 +612,15 @@ def extract_from_xml(filename, mode, fmt_vimhelp): ' ') # Minimum 8 chars between signature and vimtag - lhs = (text_width - 8) - len(prefix) + lhs = (width - 8) - len(prefix) if len(prefix) + len(suffix) > lhs: - signature = vimtag.rjust(text_width) + '\n' - signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, + signature = vimtag.rjust(width) + '\n' + signature += doc_wrap(suffix, width=width-8, prefix=prefix, func=True) else: signature = prefix + suffix - signature += vimtag.rjust(text_width - len(signature)) + signature += vimtag.rjust(width - len(signature)) paras = [] desc = find_first(member, 'detaileddescription') @@ -638,25 +660,27 @@ def extract_from_xml(filename, mode, fmt_vimhelp): fn['c_decl'] = c_decl if 'Deprecated' in str(xrefs): - deprecated_functions[name] = fn + deprecated_fns[name] = fn elif name.startswith(CONFIG[mode]['func_name_prefix']): - functions[name] = fn + fns[name] = fn xrefs.clear() - return (functions, deprecated_functions) + return (fns, deprecated_fns) def fmt_doxygen_xml_as_vimhelp(filename, mode): - """Formats functions from doxygen XML into Vim :help format. + """Entrypoint for generating Vim :help from from Doxygen XML. - Returns two strings: - 1. Functions in Vim :help format - 2. Deprecated functions (handled by caller, or ignored) + Returns 3 items: + 1. Vim help text for functions found in `filename`. + 2. Vim help text for deprecated functions. """ - functions = {} # Map of func_name:docstring. - deprecated_functions = {} # Map of func_name:docstring. - fns, deprecated_fns = extract_from_xml(filename, mode, True) + global fmt_vimhelp + fmt_vimhelp = True + fns_txt = {} # Map of func_name:vim-help-text. + deprecated_fns_txt = {} # Map of func_name:vim-help-text. + fns, _ = extract_from_xml(filename, mode, width=text_width) for name, fn in fns.items(): # Generate Vim :help for parameters. @@ -685,15 +709,15 @@ def fmt_doxygen_xml_as_vimhelp(filename, mode): func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) if 'Deprecated' in xrefs: - deprecated_functions.append(func_doc) + deprecated_fns_txt[name] = func_doc elif name.startswith(CONFIG[mode]['func_name_prefix']): - functions[name] = func_doc + fns_txt[name] = func_doc xrefs.clear() - return ('\n\n'.join(list(functions.values())), - '\n\n'.join(deprecated_fns), - functions) + fmt_vimhelp = False + return ('\n\n'.join(list(fns_txt.values())), + '\n\n'.join(list(deprecated_fns_txt.values()))) def delete_lines_below(filename, tokenstr): @@ -710,14 +734,15 @@ def delete_lines_below(filename, tokenstr): fp.writelines(lines[0:i]) -def gen_docs(config): - """Generate formatted Vim :help docs and unformatted *.mpack files for use - by API clients. +def main(config): + """Generates: + + 1. Vim :help docs + 2. *.mpack files for use by API clients Doxygen is called and configured through stdin. """ for mode in CONFIG: - functions = {} # Map of func_name:docstring. mpack_file = os.path.join( base_dir, 'runtime', 'doc', CONFIG[mode]['filename'].replace('.txt', '.mpack')) @@ -766,14 +791,13 @@ def gen_docs(config): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): + # Extract unformatted (*.mpack). fn_map, _ = extract_from_xml(os.path.join(base, '{}.xml'.format( - compound.getAttribute('refid'))), mode, False) - - functions_text, deprecated_text, fns = fmt_doxygen_xml_as_vimhelp( + compound.getAttribute('refid'))), mode, width=9999) + # Extract formatted (:help). + functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( os.path.join(base, '{}.xml'.format( compound.getAttribute('refid'))), mode) - # Collect functions from all modules (for the current `mode`). - functions = {**functions, **fns} if not functions_text and not deprecated_text: continue @@ -900,6 +924,6 @@ if __name__ == "__main__": if len(sys.argv) > 1: filter_source(sys.argv[1]) else: - gen_docs(Doxyfile) + main(Doxyfile) # vim: set ft=python ts=4 sw=4 tw=79 et : -- cgit From c24f8f46b45f097dbe0abf7e6ae686bc9bec5568 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sat, 28 Dec 2019 03:27:25 -0800 Subject: gen_vimdoc.py: sort by name --- scripts/gen_vimdoc.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 891336f571..e30ed92e8f 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -145,7 +145,7 @@ def debug_this(cond, o): except Exception: pass if ((callable(cond) and cond()) - or (not callable(cond) and cond == True) + or (not callable(cond) and cond) or (not callable(cond) and cond in o)): raise RuntimeError('xxx: {}\n{}'.format(name, o)) @@ -312,8 +312,8 @@ def update_params_map(parent, ret_map, width=62): desc = '' desc_node = get_child(node, 'parameterdescription') if desc_node: - desc = fmt_node_as_vimhelp(desc_node, width=width, - indent=(' ' * max_name_len)) + desc = fmt_node_as_vimhelp( + desc_node, width=width, indent=(' ' * max_name_len)) ret_map[name] = desc return ret_map @@ -321,8 +321,10 @@ def update_params_map(parent, ret_map, width=62): def render_node(n, text, prefix='', indent='', width=62): """Renders a node as Vim help text, recursively traversing all descendants.""" global fmt_vimhelp + def ind(s): return s if fmt_vimhelp else '' + text = '' # space_preceding = (len(text) > 0 and ' ' == text[-1][-1]) # text += (int(not space_preceding) * ' ') @@ -666,6 +668,8 @@ def extract_from_xml(filename, mode, width): xrefs.clear() + fns = collections.OrderedDict(sorted(fns.items())) + deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items())) return (fns, deprecated_fns) @@ -864,6 +868,7 @@ def main(config): with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) + fn_map_full = collections.OrderedDict(sorted(fn_map_full.items())) with open(mpack_file, 'wb') as fp: fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) -- cgit From b112fe828fd2457692f556626d7657615e53cb0b Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Tue, 31 Dec 2019 06:52:14 -0800 Subject: gen_vimdoc.py: generate LSP docs --- scripts/gen_vimdoc.py | 126 +++++++++++++++++++++++++++++++++++--------------- scripts/lua2dox.lua | 97 ++++---------------------------------- 2 files changed, 99 insertions(+), 124 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index e30ed92e8f..c1b05e16ff 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -55,6 +55,7 @@ if sys.version_info[0] < 3 or sys.version_info[1] < 5: sys.exit(1) DEBUG = ('DEBUG' in os.environ) +TARGET = os.environ.get('TARGET', None) INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) @@ -69,6 +70,7 @@ lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter') CONFIG = { 'api': { + 'mode': 'c', 'filename': 'api.txt', # String used to find the start of the generated part of the doc. 'section_start_token': '*api-global*', @@ -85,17 +87,24 @@ CONFIG = { # file patterns used by doxygen 'file_patterns': '*.h *.c', # Only function with this prefix are considered - 'func_name_prefix': 'nvim_', + 'fn_name_prefix': 'nvim_', # Section name overrides. 'section_name': { 'vim.c': 'Global', }, + # For generated section names. + 'section_fmt': lambda name: f'{name} Functions', + # Section helptag. + 'helptag_fmt': lambda name: f'*api-{name.lower()}*', + # Per-function helptag. + 'fn_helptag_fmt': lambda fstem, name: f'*{name}()*', # Module name overrides (for Lua). 'module_override': {}, # Append the docs for these modules, do not start a new section. 'append_only': [], }, 'lua': { + 'mode': 'lua', 'filename': 'lua.txt', 'section_start_token': '*lua-vim*', 'section_order': [ @@ -107,8 +116,13 @@ CONFIG = { os.path.join(base_dir, 'runtime/lua/vim/shared.lua'), ]), 'file_patterns': '*.lua', - 'func_name_prefix': '', - 'section_name': {}, + 'fn_name_prefix': '', + 'section_name': { + 'lsp.lua': 'core', + }, + 'section_fmt': lambda name: f'Lua module: {name.lower()}', + 'helptag_fmt': lambda name: f'*lua-{name.lower()}*', + 'fn_helptag_fmt': lambda fstem, name: f'*{fstem}.{name}()*', 'module_override': { # `shared` functions are exposed on the `vim` module. 'shared': 'vim', @@ -117,6 +131,44 @@ CONFIG = { 'shared.lua', ], }, + 'lsp': { + 'mode': 'lua', + 'filename': 'lsp.txt', + 'section_start_token': '*lsp-core*', + 'section_order': [ + 'lsp.lua', + 'protocol.lua', + 'buf.lua', + 'callbacks.lua', + 'log.lua', + 'rpc.lua', + 'util.lua' + ], + 'files': ' '.join([ + os.path.join(base_dir, 'runtime/lua/vim/lsp'), + os.path.join(base_dir, 'runtime/lua/vim/lsp.lua'), + ]), + 'file_patterns': '*.lua', + 'fn_name_prefix': '', + 'section_name': {}, + 'section_fmt': lambda name: ('Lua module: vim.lsp' + if name.lower() == 'lsp' + else f'Lua module: vim.lsp.{name.lower()}'), + 'helptag_fmt': lambda name: ('*lsp-core*' + if name.lower() == 'lsp' + else f'*lsp-{name.lower()}*'), + 'fn_helptag_fmt': lambda fstem, name: (f'*vim.lsp.{name}()*' + if fstem == 'lsp' and name != 'client' + else ('*vim.lsp.client*' + # HACK. TODO(justinmk): class/structure support in lua2dox + if 'lsp.client' == f'{fstem}.{name}' + else f'*vim.lsp.{fstem}.{name}()*')), + 'module_override': { + # Combine are exposed on the `vim` module. + 'shared': 'vim', + }, + 'append_only': [], + }, } param_exclude = ( @@ -567,12 +619,12 @@ def extract_from_xml(filename, mode, width): if not fmt_vimhelp: pass - elif mode == 'lua': - fstem = compoundname.split('.')[0] - fstem = CONFIG[mode]['module_override'].get(fstem, fstem) - vimtag = '*{}.{}()*'.format(fstem, name) else: - vimtag = '*{}()*'.format(name) + fstem = '?' + if '.' in compoundname: + fstem = compoundname.split('.')[0] + fstem = CONFIG[mode]['module_override'].get(fstem, fstem) + vimtag = CONFIG[mode]['fn_helptag_fmt'](fstem, name) params = [] type_length = 0 @@ -583,8 +635,8 @@ def extract_from_xml(filename, mode, width): declname = get_child(param, 'declname') if declname: param_name = get_text(declname).strip() - elif mode == 'lua': - # that's how it comes out of lua2dox + elif CONFIG[mode]['mode'] == 'lua': + # XXX: this is what lua2dox gives us... param_name = param_type param_type = '' @@ -614,7 +666,7 @@ def extract_from_xml(filename, mode, width): ' ') # Minimum 8 chars between signature and vimtag - lhs = (width - 8) - len(prefix) + lhs = (width - 8) - len(vimtag) if len(prefix) + len(suffix) > lhs: signature = vimtag.rjust(width) + '\n' @@ -663,7 +715,7 @@ def extract_from_xml(filename, mode, width): if 'Deprecated' in str(xrefs): deprecated_fns[name] = fn - elif name.startswith(CONFIG[mode]['func_name_prefix']): + elif name.startswith(CONFIG[mode]['fn_name_prefix']): fns[name] = fn xrefs.clear() @@ -714,7 +766,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, mode): if 'Deprecated' in xrefs: deprecated_fns_txt[name] = func_doc - elif name.startswith(CONFIG[mode]['func_name_prefix']): + elif name.startswith(CONFIG[mode]['fn_name_prefix']): fns_txt[name] = func_doc xrefs.clear() @@ -730,9 +782,13 @@ def delete_lines_below(filename, tokenstr): """ lines = open(filename).readlines() i = 0 + found = False for i, line in enumerate(lines, 1): if tokenstr in line: + found = True break + if not found: + raise RuntimeError(f'not found: "{tokenstr}"') i = max(0, i - 2) with open(filename, 'wt') as fp: fp.writelines(lines[0:i]) @@ -747,6 +803,8 @@ def main(config): Doxygen is called and configured through stdin. """ for mode in CONFIG: + if TARGET is not None and mode != TARGET: + continue mpack_file = os.path.join( base_dir, 'runtime', 'doc', CONFIG[mode]['filename'].replace('.txt', '.mpack')) @@ -754,7 +812,10 @@ def main(config): os.remove(mpack_file) output_dir = out_dir.format(mode=mode) - p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE) + p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE, + # silence warnings + # runtime/lua/vim/lsp.lua:209: warning: argument 'trace' of command @param is not found in the argument list + stderr=subprocess.DEVNULL) p.communicate( config.format( input=CONFIG[mode]['files'], @@ -806,15 +867,11 @@ def main(config): if not functions_text and not deprecated_text: continue else: - name = os.path.splitext(os.path.basename(filename))[0] - if name == 'ui': - name = name.upper() - else: - name = name.title() - + name = os.path.splitext( + os.path.basename(filename))[0].lower() + sectname = name.upper() if name == 'ui' else name.title() doc = '' - - intro = intros.get('api-%s' % name.lower()) + intro = intros.get(f'api-{name}') if intro: doc += '\n\n' + intro @@ -822,33 +879,28 @@ def main(config): doc += '\n\n' + functions_text if INCLUDE_DEPRECATED and deprecated_text: - doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name + doc += f'\n\n\nDeprecated {sectname} Functions: ~\n\n' doc += deprecated_text if doc: filename = os.path.basename(filename) - name = CONFIG[mode]['section_name'].get(filename, name) - - if mode == 'lua': - title = 'Lua module: {}'.format(name.lower()) - helptag = '*lua-{}*'.format(name.lower()) - else: - title = '{} Functions'.format(name) - helptag = '*api-{}*'.format(name.lower()) + sectname = CONFIG[mode]['section_name'].get( + filename, sectname) + title = CONFIG[mode]['section_fmt'](sectname) + helptag = CONFIG[mode]['helptag_fmt'](sectname) sections[filename] = (title, helptag, doc) fn_map_full.update(fn_map) - if not sections: - return + assert sections + if len(sections) > len(CONFIG[mode]['section_order']): + raise RuntimeError( + 'found new modules "{}"; update the "section_order" map'.format( + set(sections).difference(CONFIG[mode]['section_order']))) docs = '' i = 0 for filename in CONFIG[mode]['section_order']: - if filename not in sections: - raise RuntimeError( - 'found new module "{}"; update the "section_order" map'.format( - filename)) title, helptag, section_doc = sections.pop(filename) i += 1 if filename not in CONFIG[mode]['append_only']: diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 171621e38d..d4e68f9e45 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -17,61 +17,28 @@ -- Free Software Foundation, Inc., -- -- 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -- ----------------------------------------------------------------------------]] ---[[! -\file -\brief a hack lua2dox converter -]] --[[! -\mainpage - -Introduction ------------- - -A hack lua2dox converter -Version 0.2 +Lua-to-Doxygen converter -This lets us make Doxygen output some documentation to let -us develop this code. - -It is partially cribbed from the functionality of lua2dox -(http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm). -Found on CPAN when looking for something else; kinda handy. - -Improved from lua2dox to make the doxygen output more friendly. -Also it runs faster in lua rather than Perl. - -Because this Perl based system is called "lua2dox"., I have decided to add ".lua" to the name -to keep the two separate. +Partially from lua2dox +http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm Running ------- -
      -
    1. Ensure doxygen is installed on your system and that you are familiar with its use. -Best is to try to make and document some simple C/C++/PHP to see what it produces. -You can experiment with the enclosed example code. - -
    2. Run "doxygen -g" to create a default Doxyfile. +This file "lua2dox.lua" gets called by "lua2dox_filter" (bash). -Then alter it to let it recognise lua. Add the two following lines: +Doxygen must be on your system. You can experiment like so: -\code{.bash} -FILE_PATTERNS = *.lua - -FILTER_PATTERNS = *.lua=lua2dox_filter -\endcode - - -Either add them to the end or find the appropriate entry in Doxyfile. - -There are other lines that you might like to alter, but see further documentation for details. - -
    3. When Doxyfile is edited run "doxygen" +- Run "doxygen -g" to create a default Doxyfile. +- Then alter it to let it recognise lua. Add the two following lines: + FILE_PATTERNS = *.lua + FILTER_PATTERNS = *.lua=lua2dox_filter +- Then run "doxygen". The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language. It only has to be good enough for doxygen to see it as legal. -Therefore our lua interpreter is fairly limited, but "good enough". One limitation is that each line is treated separately (except for long comments). The implication is that class and function declarations must be on the same line. @@ -81,40 +48,8 @@ so it will probably not document accurately if we do do this. However I have put in a hack that will insert the "missing" close paren. The effect is that you will get the function documented, but not with the parameter list you might expect. -
    - -Installation ------------- - -Here for linux or unix-like, for any other OS you need to refer to other documentation. - -This file is "lua2dox.lua". It gets called by "lua2dox_filter"(bash). -Somewhere in your path (e.g. "~/bin" or "/usr/local/bin") put a link to "lua2dox_filter". - -Documentation -------------- - -Read the external documentation that should be part of this package. -For example look for the "README" and some .PDFs. - ]] --- we won't use our library code, so this becomes more portable - --- require 'elijah_fix_require' --- require 'elijah_class' --- ---! \brief ``declare'' as class ---! ---! use as: ---! \code{.lua} ---! TWibble = class() ---! function TWibble.init(this,Str) ---! this.str = Str ---! -- more stuff here ---! end ---! \endcode ---! function class(BaseClass, ClassInitialiser) local newClass = {} -- a new class newClass if not ClassInitialiser and type(BaseClass) == 'function' then @@ -165,8 +100,6 @@ function class(BaseClass, ClassInitialiser) return newClass end --- require 'elijah_clock' - --! \class TCore_Clock --! \brief a clock TCore_Clock = class() @@ -201,9 +134,6 @@ function TCore_Clock.getTimeStamp(this,T0) end ---require 'elijah_io' - ---! \class TCore_IO --! \brief io to console --! --! pseudo class (no methods, just to keep documentation tidy) @@ -225,8 +155,6 @@ function TCore_IO_writeln(Str) end ---require 'elijah_string' - --! \brief trims a string function string_trim(Str) return Str:match("^%s*(.-)%s*$") @@ -257,8 +185,6 @@ function string_split(Str, Pattern) end ---require 'elijah_commandline' - --! \class TCore_Commandline --! \brief reads/parses commandline TCore_Commandline = class() @@ -279,9 +205,6 @@ function TCore_Commandline.getRaw(this,Key,Default) return val end - ---require 'elijah_debug' - ------------------------------- --! \brief file buffer --! -- cgit From 0dcf4bd3ebfec77c5d3816d2053521190080f7e0 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Tue, 31 Dec 2019 07:12:10 -0800 Subject: gen_vimdoc.py: rename `mode` to `target` --- scripts/gen_vimdoc.py | 70 ++++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 32 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index c1b05e16ff..bdfe52addb 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -63,7 +63,7 @@ fmt_vimhelp = False # HACK text_width = 78 script_path = os.path.abspath(__file__) base_dir = os.path.dirname(os.path.dirname(script_path)) -out_dir = os.path.join(base_dir, 'tmp-{mode}-doc') +out_dir = os.path.join(base_dir, 'tmp-{target}-doc') filter_cmd = '%s %s' % (sys.executable, script_path) seen_funcs = set() lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter') @@ -151,15 +151,19 @@ CONFIG = { 'file_patterns': '*.lua', 'fn_name_prefix': '', 'section_name': {}, - 'section_fmt': lambda name: ('Lua module: vim.lsp' + 'section_fmt': lambda name: ( + 'Lua module: vim.lsp' if name.lower() == 'lsp' else f'Lua module: vim.lsp.{name.lower()}'), - 'helptag_fmt': lambda name: ('*lsp-core*' + 'helptag_fmt': lambda name: ( + '*lsp-core*' if name.lower() == 'lsp' else f'*lsp-{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name: (f'*vim.lsp.{name}()*' + 'fn_helptag_fmt': lambda fstem, name: ( + f'*vim.lsp.{name}()*' if fstem == 'lsp' and name != 'client' - else ('*vim.lsp.client*' + else ( + '*vim.lsp.client*' # HACK. TODO(justinmk): class/structure support in lua2dox if 'lsp.client' == f'{fstem}.{name}' else f'*vim.lsp.{fstem}.{name}()*')), @@ -568,7 +572,7 @@ def fmt_node_as_vimhelp(parent, width=62, indent=''): return clean_lines('\n'.join(rendered_blocks).strip()) -def extract_from_xml(filename, mode, width): +def extract_from_xml(filename, target, width): """Extracts Doxygen info as maps without formatting the text. Returns two maps: @@ -623,8 +627,8 @@ def extract_from_xml(filename, mode, width): fstem = '?' if '.' in compoundname: fstem = compoundname.split('.')[0] - fstem = CONFIG[mode]['module_override'].get(fstem, fstem) - vimtag = CONFIG[mode]['fn_helptag_fmt'](fstem, name) + fstem = CONFIG[target]['module_override'].get(fstem, fstem) + vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name) params = [] type_length = 0 @@ -635,7 +639,7 @@ def extract_from_xml(filename, mode, width): declname = get_child(param, 'declname') if declname: param_name = get_text(declname).strip() - elif CONFIG[mode]['mode'] == 'lua': + elif CONFIG[target]['mode'] == 'lua': # XXX: this is what lua2dox gives us... param_name = param_type param_type = '' @@ -715,7 +719,7 @@ def extract_from_xml(filename, mode, width): if 'Deprecated' in str(xrefs): deprecated_fns[name] = fn - elif name.startswith(CONFIG[mode]['fn_name_prefix']): + elif name.startswith(CONFIG[target]['fn_name_prefix']): fns[name] = fn xrefs.clear() @@ -725,7 +729,7 @@ def extract_from_xml(filename, mode, width): return (fns, deprecated_fns) -def fmt_doxygen_xml_as_vimhelp(filename, mode): +def fmt_doxygen_xml_as_vimhelp(filename, target): """Entrypoint for generating Vim :help from from Doxygen XML. Returns 3 items: @@ -736,7 +740,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, mode): fmt_vimhelp = True fns_txt = {} # Map of func_name:vim-help-text. deprecated_fns_txt = {} # Map of func_name:vim-help-text. - fns, _ = extract_from_xml(filename, mode, width=text_width) + fns, _ = extract_from_xml(filename, target, width=text_width) for name, fn in fns.items(): # Generate Vim :help for parameters. @@ -766,7 +770,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, mode): if 'Deprecated' in xrefs: deprecated_fns_txt[name] = func_doc - elif name.startswith(CONFIG[mode]['fn_name_prefix']): + elif name.startswith(CONFIG[target]['fn_name_prefix']): fns_txt[name] = func_doc xrefs.clear() @@ -802,26 +806,28 @@ def main(config): Doxygen is called and configured through stdin. """ - for mode in CONFIG: - if TARGET is not None and mode != TARGET: + for target in CONFIG: + if TARGET is not None and target != TARGET: continue mpack_file = os.path.join( base_dir, 'runtime', 'doc', - CONFIG[mode]['filename'].replace('.txt', '.mpack')) + CONFIG[target]['filename'].replace('.txt', '.mpack')) if os.path.exists(mpack_file): os.remove(mpack_file) - output_dir = out_dir.format(mode=mode) - p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE, + output_dir = out_dir.format(target=target) + p = subprocess.Popen( + ['doxygen', '-'], + stdin=subprocess.PIPE, # silence warnings - # runtime/lua/vim/lsp.lua:209: warning: argument 'trace' of command @param is not found in the argument list + # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found stderr=subprocess.DEVNULL) p.communicate( config.format( - input=CONFIG[mode]['files'], + input=CONFIG[target]['files'], output=output_dir, filter=filter_cmd, - file_patterns=CONFIG[mode]['file_patterns']) + file_patterns=CONFIG[target]['file_patterns']) .encode('utf8') ) if p.returncode: @@ -858,11 +864,11 @@ def main(config): if filename.endswith('.c') or filename.endswith('.lua'): # Extract unformatted (*.mpack). fn_map, _ = extract_from_xml(os.path.join(base, '{}.xml'.format( - compound.getAttribute('refid'))), mode, width=9999) + compound.getAttribute('refid'))), target, width=9999) # Extract formatted (:help). functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( os.path.join(base, '{}.xml'.format( - compound.getAttribute('refid'))), mode) + compound.getAttribute('refid'))), target) if not functions_text and not deprecated_text: continue @@ -884,26 +890,26 @@ def main(config): if doc: filename = os.path.basename(filename) - sectname = CONFIG[mode]['section_name'].get( + sectname = CONFIG[target]['section_name'].get( filename, sectname) - title = CONFIG[mode]['section_fmt'](sectname) - helptag = CONFIG[mode]['helptag_fmt'](sectname) + title = CONFIG[target]['section_fmt'](sectname) + helptag = CONFIG[target]['helptag_fmt'](sectname) sections[filename] = (title, helptag, doc) fn_map_full.update(fn_map) assert sections - if len(sections) > len(CONFIG[mode]['section_order']): + if len(sections) > len(CONFIG[target]['section_order']): raise RuntimeError( 'found new modules "{}"; update the "section_order" map'.format( - set(sections).difference(CONFIG[mode]['section_order']))) + set(sections).difference(CONFIG[target]['section_order']))) docs = '' i = 0 - for filename in CONFIG[mode]['section_order']: + for filename in CONFIG[target]['section_order']: title, helptag, section_doc = sections.pop(filename) i += 1 - if filename not in CONFIG[mode]['append_only']: + if filename not in CONFIG[target]['append_only']: docs += sep docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) @@ -914,9 +920,9 @@ def main(config): docs += ' vim:tw=78:ts=8:ft=help:norl:\n' doc_file = os.path.join(base_dir, 'runtime', 'doc', - CONFIG[mode]['filename']) + CONFIG[target]['filename']) - delete_lines_below(doc_file, CONFIG[mode]['section_start_token']) + delete_lines_below(doc_file, CONFIG[target]['section_start_token']) with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) -- cgit From a4d21f05928b3534cf9df11b0ccc3f86d79d88bc Mon Sep 17 00:00:00 2001 From: Jan Edmund Lazo Date: Sat, 11 Jan 2020 15:09:54 -0500 Subject: vim-patch.sh: fix for bash 4.3 or older #11700 --- scripts/vim-patch.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 769f0fed58..d76c8573c9 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -574,10 +574,12 @@ list_missing_previous_vimpatches_for_patch() { fi done + set +u # Avoid "unbound variable" with bash < 4.4 below. if [[ -z "${missing_list[*]}" ]]; then msg_ok 'no missing previous Vim patches' return 0 fi + set -u local -a missing_unique while IFS= read -r line; do -- cgit From 92316849863bb2661ee5b4bb284f56163fed27ad Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 12 Jan 2020 23:41:55 -0800 Subject: doc [ci skip] #11656 --- scripts/gen_vimdoc.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index bdfe52addb..3c51b2aa81 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -167,10 +167,7 @@ CONFIG = { # HACK. TODO(justinmk): class/structure support in lua2dox if 'lsp.client' == f'{fstem}.{name}' else f'*vim.lsp.{fstem}.{name}()*')), - 'module_override': { - # Combine are exposed on the `vim` module. - 'shared': 'vim', - }, + 'module_override': {}, 'append_only': [], }, } @@ -821,7 +818,7 @@ def main(config): stdin=subprocess.PIPE, # silence warnings # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found - stderr=subprocess.DEVNULL) + stderr=(subprocess.STDOUT if DEBUG else subprocess.DEVNULL)) p.communicate( config.format( input=CONFIG[target]['files'], -- cgit From 4c152be72665808a45f1e957e3fde53eaf9bdc25 Mon Sep 17 00:00:00 2001 From: Jan Edmund Lazo Date: Sun, 19 Jan 2020 10:05:23 -0500 Subject: fixup! fixup! vim-patch.sh: list related missing Vim patches [ci skip] #11514 --- scripts/vim-patch.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index d76c8573c9..ff78cb82f7 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -560,6 +560,7 @@ list_missing_previous_vimpatches_for_patch() { local -a missing_vim_patches=() _set_missing_vimpatches 1 -- "${fname}" + set +u # Avoid "unbound variable" with bash < 4.4 below. local missing_vim_commit_info="${missing_vim_patches[0]}" if [[ -z "${missing_vim_commit_info}" ]]; then printf -- "-\n" @@ -572,6 +573,7 @@ list_missing_previous_vimpatches_for_patch() { printf -- "-\n" fi fi + set -u done set +u # Avoid "unbound variable" with bash < 4.4 below. -- cgit From c6ff23d7a0d5ccf0d8995e3204c18df55d28fc7f Mon Sep 17 00:00:00 2001 From: Chris LaRose Date: Sun, 26 Jan 2020 00:24:42 -0800 Subject: terminal: absolute CWD in term:// URI #11289 This makes it possible to restore the working directory of :terminal buffers when reading those buffers from a session file. Fixes #11288 Co-authored-by: Justin M. Keyes --- scripts/vim-patch.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index ff78cb82f7..f9726e8a87 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -234,7 +234,8 @@ get_vimpatch() { msg_ok "Saved patch to '${NVIM_SOURCE_DIR}/${patch_file}'." } -# shellcheck disable=SC2015 # "Note that A && B || C is not if-then-else." +# shellcheck disable=SC2015 +# ^ "Note that A && B || C is not if-then-else." stage_patch() { get_vimpatch "$1" local try_apply="${2:-}" @@ -305,7 +306,8 @@ git_hub_pr() { git hub pull new -m "$1" } -# shellcheck disable=SC2015 # "Note that A && B || C is not if-then-else." +# shellcheck disable=SC2015 +# ^ "Note that A && B || C is not if-then-else." submit_pr() { require_executable git local push_first -- cgit From f755370682ad6e82f468975b40cf734e5b5f47c9 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Sun, 26 Jan 2020 14:50:25 +0100 Subject: scripts/vim-patch.sh: add -m to usage Ref: https://github.com/neovim/neovim/pull/11514#issuecomment-569476828 --- scripts/vim-patch.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index f9726e8a87..6c9bea2ad3 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -23,6 +23,7 @@ usage() { echo " -h Show this message and exit." echo " -l [git-log opts] List missing Vim patches." echo " -L [git-log opts] List missing Vim patches (for scripts)." + echo " -m {vim-revision} List previous (older) missing Vim patches." echo " -M List all merged patch-numbers (at current v:version)." echo " -p {vim-revision} Download and generate a Vim patch. vim-revision" echo " can be a Vim version (8.0.xxx) or a Git hash." -- cgit From 20f5f448830db553ae9b3bc4cfc1602d66b0aa3e Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Sun, 26 Jan 2020 15:07:41 +0100 Subject: set -u before return Follow up to a4d21f059. --- scripts/vim-patch.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 6c9bea2ad3..7eeb644318 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -582,6 +582,7 @@ list_missing_previous_vimpatches_for_patch() { set +u # Avoid "unbound variable" with bash < 4.4 below. if [[ -z "${missing_list[*]}" ]]; then msg_ok 'no missing previous Vim patches' + set -u return 0 fi set -u -- cgit From 1dc5e5ae9b3c5aa057e21df5fd9978b47ed3492e Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Sun, 26 Jan 2020 15:20:51 +0100 Subject: doc --- scripts/vim-patch.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 7eeb644318..ecaddeacf6 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -428,7 +428,7 @@ _set_tokens_and_tags() { } # Prints a newline-delimited list of Vim commits, for use by scripts. -# "$1": use extended format? +# "$1": use extended format? (with subject) # "$@" is passed to list_vim_commits, as extra arguments to git-log. list_missing_vimpatches() { local -a missing_vim_patches=() @@ -442,6 +442,8 @@ list_missing_vimpatches() { # Sets / appends to missing_vim_patches (useful to avoid a subshell when # used multiple times to cache tokens/vim_commit_tags). +# "$1": use extended format? (with subject) +# "$@": extra arguments to git-log. _set_missing_vimpatches() { local token vim_commit vim_tag patch_number declare -a git_log_args -- cgit From dc7f59e04bf711b296fe0e1f0ce2159d0e97c877 Mon Sep 17 00:00:00 2001 From: James McCoy Date: Tue, 31 Mar 2020 22:40:33 -0400 Subject: vim-patch.sh: Fix creation of commit list for PR review [ci skip] --- scripts/vim-patch.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index f9726e8a87..e50eb307e5 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -670,9 +670,11 @@ review_pr() { echo echo "Downloading data for pull request #${pr}." - local pr_commit_urls=( - "$(curl -Ssf "https://api.github.com/repos/neovim/neovim/pulls/${pr}/commits" \ - | jq -r '.[].html_url')") + local -a pr_commit_urls + while IFS= read -r pr_commit_url; do + pr_commit_urls+=("$pr_commit_url") + done < <(curl -Ssf "https://api.github.com/repos/neovim/neovim/pulls/${pr}/commits" \ + | jq -r '.[].html_url') echo "Found ${#pr_commit_urls[@]} commit(s)." -- cgit From d26a66a2b310100bac47a57da8b92978a22cb785 Mon Sep 17 00:00:00 2001 From: Kerkko Pelttari Date: Thu, 2 Apr 2020 17:56:14 +0300 Subject: Check for bash version in vim-patch.sh --- scripts/vim-patch.sh | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index f9726e8a87..ba0e7f62dd 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -5,6 +5,12 @@ set -u # Use privileged mode, which e.g. skips using CDPATH. set -p +# Ensure that the user has a bash that supports -A +if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then + echo "Update your bash version to one that supports -A syntax (>3)" + exit 1 +fi + readonly NVIM_SOURCE_DIR="${NVIM_SOURCE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" readonly VIM_SOURCE_DIR_DEFAULT="${NVIM_SOURCE_DIR}/.vim-src" readonly VIM_SOURCE_DIR="${VIM_SOURCE_DIR:-${VIM_SOURCE_DIR_DEFAULT}}" -- cgit From f47e574c875c521244cdb30f271714d4bf20294d Mon Sep 17 00:00:00 2001 From: Kerkko Pelttari Date: Tue, 7 Apr 2020 10:31:26 +0300 Subject: Apply suggestions from code review Improve error message for unsupported bash version, use double square bracket operator Co-Authored-By: Daniel Hahler --- scripts/vim-patch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index ba0e7f62dd..e844d2951c 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -6,8 +6,8 @@ set -u set -p # Ensure that the user has a bash that supports -A -if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then - echo "Update your bash version to one that supports -A syntax (>3)" +if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then + echo "This script requires bash version 3 or later (you have ${BASH_VERSION})." >&2 exit 1 fi -- cgit From ed815c61fdc3fa5a0873a3f9005fdf3c5c49e8df Mon Sep 17 00:00:00 2001 From: Poh Zi How Date: Mon, 1 Jun 2020 01:27:05 +0800 Subject: vim-patch.sh: fix bash version-check message #12398 --- scripts/vim-patch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index a8b622d5c4..9c4349abca 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -7,7 +7,7 @@ set -p # Ensure that the user has a bash that supports -A if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then - echo "This script requires bash version 3 or later (you have ${BASH_VERSION})." >&2 + >&2 echo "error: script requires bash 4+ (you have ${BASH_VERSION})." exit 1 fi -- cgit From f5fbe8e3b58b707eadac70de94ca0475071e5da9 Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Tue, 2 Jun 2020 22:16:54 +0200 Subject: treesitter: add update script and update runtime Update treesitter runtime to : 9a82dcc666d06617cbab3061467075019fae0b0d --- scripts/update-ts-runtime.sh | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 scripts/update-ts-runtime.sh (limited to 'scripts') diff --git a/scripts/update-ts-runtime.sh b/scripts/update-ts-runtime.sh new file mode 100755 index 0000000000..6f3cc7ba39 --- /dev/null +++ b/scripts/update-ts-runtime.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# +# This script will update the treesitter runtime to the provided commit. +# Usage : +# $0 + +ts_source_dir="/tmp/tree-sitter" +ts_url="https://github.com/tree-sitter/tree-sitter.git" + +base_dir="$(cd "$(dirname $(dirname $0))" && pwd)" +ts_dest_dir="$base_dir/src/tree_sitter/" + +echo "$ts_dest_dir" + +if [ ! -d "$ts_source_dir" ]; then + echo "Cloning treesitter..." + git clone "$ts_url" "$ts_source_dir" +else + echo "Found a non-empty $ts_source_dir directory..." +fi + +echo "Checking out $1..." +cd "$ts_source_dir" +git -C "$ts_source_dir" checkout $1 + +echo "Removing old files..." +find "$ts_dest_dir" -not -name "LICENSE" -not -type d -delete + +echo "Copying files..." +cp -t "$ts_dest_dir" -r "$ts_source_dir/lib/src"/* +cp -t "$ts_dest_dir" "$ts_source_dir/lib/include/tree_sitter"/* + +cd "$base_dir" +make +TEST_FILE="$base_dir/test/functional/lua/treesitter_spec.lua" make test -- cgit From 662688817b2156edd47019f00f53774e5e00224a Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Tue, 2 Jun 2020 22:32:49 +0200 Subject: treesitter: enhance script and add README The script now updates a `treesitter_commit_sha.txt` file to keep track of which commit/branch/tag we're at. --- scripts/update-ts-runtime.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/update-ts-runtime.sh b/scripts/update-ts-runtime.sh index 6f3cc7ba39..c75c5d2a39 100755 --- a/scripts/update-ts-runtime.sh +++ b/scripts/update-ts-runtime.sh @@ -3,14 +3,16 @@ # This script will update the treesitter runtime to the provided commit. # Usage : # $0 +set -e ts_source_dir="/tmp/tree-sitter" ts_url="https://github.com/tree-sitter/tree-sitter.git" base_dir="$(cd "$(dirname $(dirname $0))" && pwd)" ts_dest_dir="$base_dir/src/tree_sitter/" +ts_current_commit="$ts_dest_dir/treesitter_commit_hash.txt" -echo "$ts_dest_dir" +echo "Updating treesitter runtime from $(cat "$ts_current_commit") to $1..." if [ ! -d "$ts_source_dir" ]; then echo "Cloning treesitter..." @@ -20,16 +22,17 @@ else fi echo "Checking out $1..." -cd "$ts_source_dir" git -C "$ts_source_dir" checkout $1 echo "Removing old files..." -find "$ts_dest_dir" -not -name "LICENSE" -not -type d -delete +find "$ts_dest_dir" -not -name "LICENSE" -not -name "README.md" -not -type d -delete echo "Copying files..." cp -t "$ts_dest_dir" -r "$ts_source_dir/lib/src"/* cp -t "$ts_dest_dir" "$ts_source_dir/lib/include/tree_sitter"/* -cd "$base_dir" +echo "$1" > "$ts_current_commit" + make TEST_FILE="$base_dir/test/functional/lua/treesitter_spec.lua" make test + -- cgit From 6b949211a06e21af67bf4cb3a20c6f87c932ef2a Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Wed, 3 Jun 2020 21:33:34 +0200 Subject: treesitter: update runtime Update to 81d533d2d1b580fdb507accabc91ceddffb5b6f0. --- scripts/update-ts-runtime.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/update-ts-runtime.sh b/scripts/update-ts-runtime.sh index c75c5d2a39..1a947e0ac9 100755 --- a/scripts/update-ts-runtime.sh +++ b/scripts/update-ts-runtime.sh @@ -19,6 +19,7 @@ if [ ! -d "$ts_source_dir" ]; then git clone "$ts_url" "$ts_source_dir" else echo "Found a non-empty $ts_source_dir directory..." + git -C "$ts_source_dir" fetch fi echo "Checking out $1..." -- cgit From 7b529e7912517af078e005dd7b06b3d042be9cb7 Mon Sep 17 00:00:00 2001 From: TJ DeVries Date: Thu, 2 Jul 2020 07:09:17 -0400 Subject: doc: fix scripts and regenerate (#12506) * Fix some small doc issues * doc: fixup * doc: fixup * Fix lint and rebase * Remove bad advice * Ugh, stupid mpack files... * Don't let people include these for now until they specifically want to * Prevent duplicate tag --- scripts/gen_vimdoc.py | 66 +++++++++++++++++++++++++++++++++++++++++++++----- scripts/lua2dox_filter | 11 ++++++++- 2 files changed, 70 insertions(+), 7 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 3c51b2aa81..328a903b46 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -374,6 +374,7 @@ def update_params_map(parent, ret_map, width=62): def render_node(n, text, prefix='', indent='', width=62): """Renders a node as Vim help text, recursively traversing all descendants.""" global fmt_vimhelp + global has_seen_preformatted def ind(s): return s if fmt_vimhelp else '' @@ -386,6 +387,7 @@ def render_node(n, text, prefix='', indent='', width=62): o = get_text(n, preformatted=True) ensure_nl = '' if o[-1] == '\n' else '\n' text += '>{}{}\n<'.format(ensure_nl, o) + elif is_inline(n): text = doc_wrap(get_text(n), indent=indent, width=width) elif n.nodeName == 'verbatim': @@ -394,11 +396,17 @@ def render_node(n, text, prefix='', indent='', width=62): text += ' [verbatim] {}'.format(get_text(n)) elif n.nodeName == 'listitem': for c in n.childNodes: - text += ( - indent - + prefix - + render_node(c, text, indent=indent + (' ' * len(prefix)), width=width) + result = render_node( + c, + text, + indent=indent + (' ' * len(prefix)), + width=width ) + + if is_blank(result): + continue + + text += indent + prefix + result elif n.nodeName in ('para', 'heading'): for c in n.childNodes: text += render_node(c, text, indent=indent, width=width) @@ -433,6 +441,7 @@ def render_node(n, text, prefix='', indent='', width=62): else: raise RuntimeError('unhandled node type: {}\n{}'.format( n.nodeName, n.toprettyxml(indent=' ', newl='\n'))) + return text @@ -496,6 +505,7 @@ def para_as_map(parent, indent='', width=62): and '' != get_text(self_or_child(child)).strip() and ' ' != text[-1]): text += ' ' + text += render_node(child, text, indent=indent, width=width) prev = child @@ -566,6 +576,7 @@ def fmt_node_as_vimhelp(parent, width=62, indent=''): rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) rendered_blocks.append('') + return clean_lines('\n'.join(rendered_blocks).strip()) @@ -678,6 +689,11 @@ def extract_from_xml(filename, target, width): signature += vimtag.rjust(width - len(signature)) paras = [] + brief_desc = find_first(member, 'briefdescription') + if brief_desc: + for child in brief_desc.childNodes: + paras.append(para_as_map(child)) + desc = find_first(member, 'detaileddescription') if desc: for child in desc.childNodes: @@ -763,8 +779,36 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): func_doc = fn['signature'] + '\n' func_doc += textwrap.indent(clean_lines(doc), ' ' * 16) + + # Verbatim handling. func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) + split_lines = func_doc.split('\n') + start = 0 + while True: + try: + start = split_lines.index('>', start) + except ValueError: + break + + try: + end = split_lines.index('<', start) + except ValueError: + break + + split_lines[start + 1:end] = [ + (' ' + x).rstrip() + for x in textwrap.dedent( + "\n".join( + split_lines[start+1:end] + ) + ).split("\n") + ] + + start = end + + func_doc = "\n".join(split_lines) + if 'Deprecated' in xrefs: deprecated_fns_txt[name] = func_doc elif name.startswith(CONFIG[target]['fn_name_prefix']): @@ -847,11 +891,21 @@ def main(config): groupxml = os.path.join(base, '%s.xml' % compound.getAttribute('refid')) - desc = find_first(minidom.parse(groupxml), 'detaileddescription') + group_parsed = minidom.parse(groupxml) + doc_list = [] + brief_desc = find_first(group_parsed, 'briefdescription') + if brief_desc: + for child in brief_desc.childNodes: + doc_list.append(fmt_node_as_vimhelp(child)) + + desc = find_first(group_parsed, 'detaileddescription') if desc: doc = fmt_node_as_vimhelp(desc) + if doc: - intros[groupname] = doc + doc_list.append(doc) + + intros[groupname] = "\n".join(doc_list) for compound in dom.getElementsByTagName('compound'): if compound.getAttribute('kind') != 'file': diff --git a/scripts/lua2dox_filter b/scripts/lua2dox_filter index 6cb16ef060..61577527c4 100755 --- a/scripts/lua2dox_filter +++ b/scripts/lua2dox_filter @@ -36,7 +36,16 @@ test_executable(){ ##! \brief sets the lua interpreter set_lua(){ - test_executable 'texlua' + if test -z "${EXE}" + then + test_executable 'luajit' + fi + + if test -z "${EXE}" + then + test_executable 'texlua' + fi + if test -z "${EXE}" then test_executable 'lua' -- cgit From 44cbf288ec4407c6fddf79b6b39041d5bbee940b Mon Sep 17 00:00:00 2001 From: jnozsc Date: Fri, 24 Jul 2020 13:42:25 -0700 Subject: script: simplify python version check (#12672) --- scripts/gen_vimdoc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 328a903b46..a61690e99f 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -50,8 +50,10 @@ import msgpack from xml.dom import minidom -if sys.version_info[0] < 3 or sys.version_info[1] < 5: - print("requires Python 3.5+") +MIN_PYTHON_VERSION = (3, 5) + +if sys.version_info < MIN_PYTHON_VERSION: + print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION)) sys.exit(1) DEBUG = ('DEBUG' in os.environ) -- cgit