diff options
-rw-r--r-- | .travis.yml | 5 | ||||
-rw-r--r-- | CMakeLists.txt | 1 | ||||
-rwxr-xr-x | ci/before_cache.sh | 5 | ||||
-rw-r--r-- | ci/common/suite.sh | 68 | ||||
-rw-r--r-- | ci/common/test.sh | 62 | ||||
-rwxr-xr-x | ci/run_lint.sh | 5 | ||||
-rwxr-xr-x | ci/run_tests.sh | 6 | ||||
-rw-r--r-- | cmake/InstallClintErrors.cmake | 2 | ||||
-rw-r--r-- | runtime/doc/eval.txt | 4 | ||||
-rw-r--r-- | runtime/doc/options.txt | 4 | ||||
-rw-r--r-- | runtime/doc/starting.txt | 24 | ||||
-rw-r--r-- | src/nvim/CMakeLists.txt | 24 | ||||
-rw-r--r-- | src/nvim/digraph.h | 4 | ||||
-rw-r--r-- | src/nvim/eval/decode.c | 24 | ||||
-rw-r--r-- | src/nvim/eval/encode.c | 55 | ||||
-rw-r--r-- | src/nvim/ex_cmds.h | 4 | ||||
-rw-r--r-- | src/nvim/ex_getln.h | 6 | ||||
-rw-r--r-- | src/nvim/fold.h | 8 | ||||
-rw-r--r-- | src/nvim/iconv.h | 4 | ||||
-rw-r--r-- | src/nvim/memfile_defs.h | 4 | ||||
-rw-r--r-- | src/nvim/move.h | 4 | ||||
-rw-r--r-- | src/nvim/shada.c | 230 | ||||
-rw-r--r-- | src/nvim/vim.h | 17 | ||||
-rw-r--r-- | test/functional/helpers.lua | 7 | ||||
-rw-r--r-- | test/functional/terminal/tui_spec.lua | 106 | ||||
-rw-r--r-- | test/helpers.lua | 40 | ||||
-rw-r--r-- | test/unit/eval/decode_spec.lua | 7 | ||||
-rw-r--r-- | test/unit/eval/helpers.lua | 2 | ||||
-rw-r--r-- | test/unit/eval/typval_spec.lua | 83 | ||||
-rw-r--r-- | test/unit/helpers.lua | 19 |
30 files changed, 341 insertions, 493 deletions
diff --git a/.travis.yml b/.travis.yml index 14899a1289..b8c4c0172f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,11 +43,6 @@ env: # If this file exists, we know that the cache contains compiled # dependencies and we can use it. - CACHE_MARKER="$HOME/.cache/nvim-deps/.travis_cache_marker" - # Test success marker. If this file exists, we know that all tests - # were successful. Required because we only want to update the cache - # if the tests were successful, but don't have this information - # available in before_cache (which is run before after_success). - - SUCCESS_MARKER="$BUILD_DIR/.tests_successful" # default target name for functional tests - FUNCTIONALTEST=functionaltest - CI_TARGET=tests diff --git a/CMakeLists.txt b/CMakeLists.txt index e8956c9074..2232543cc8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -275,7 +275,6 @@ else() endif() add_definitions(-DINCLUDE_GENERATED_DECLARATIONS) -add_definitions(-DHAVE_CONFIG_H) if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_SYSTEM_NAME STREQUAL "Linux") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") diff --git a/ci/before_cache.sh b/ci/before_cache.sh index dd1fcf2bf7..3d7cc0ec5a 100755 --- a/ci/before_cache.sh +++ b/ci/before_cache.sh @@ -3,12 +3,15 @@ set -e set -o pipefail +CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${CI_DIR}/common/suite.sh" + # Don't cache pip's log and selfcheck. rm -rf "${HOME}/.cache/pip/log" rm -f "${HOME}/.cache/pip/selfcheck.json" # Update the third-party dependency cache only if the build was successful. -if [[ -f "${SUCCESS_MARKER}" ]]; then +if ended_successfully; then rm -rf "${HOME}/.cache/nvim-deps" mv "${DEPS_BUILD_DIR}" "${HOME}/.cache/nvim-deps" touch "${CACHE_MARKER}" diff --git a/ci/common/suite.sh b/ci/common/suite.sh index 46207754fa..568d5d5bee 100644 --- a/ci/common/suite.sh +++ b/ci/common/suite.sh @@ -2,11 +2,18 @@ NL="$(printf '\nE')" NL="${NL%E}" -FAILED=0 - FAIL_SUMMARY="" +# Test success marker. If END_MARKER file exists, we know that all tests +# finished. If FAIL_SUMMARY_FILE exists we know that some tests failed, this +# file will contain information about failed tests. Build is considered +# successful if tests ended without any of them failing. +END_MARKER="$BUILD_DIR/.tests_finished" +FAIL_SUMMARY_FILE="$BUILD_DIR/.test_errors" + enter_suite() { + FAILED=0 + rm -f "${END_MARKER}" local suite_name="$1" export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name" } @@ -19,17 +26,16 @@ exit_suite() { export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}" if test "x$1" != "x--continue" ; then exit $FAILED + else + local saved_failed=$FAILED + FAILED=0 + return $saved_failed fi } fail() { - local allow_failure= - if test "x$1" = "x--allow-failure" ; then - shift - allow_failure=A - fi local test_name="$1" - local fail_char="$allow_failure$2" + local fail_char="$2" local message="$3" : ${fail_char:=F} @@ -37,10 +43,9 @@ fail() { local full_msg="$fail_char $NVIM_TEST_CURRENT_SUITE|$test_name :: $message" FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}" + echo "${full_msg}" >> "${FAIL_SUMMARY_FILE}" echo "Failed: $full_msg" - if test "x$allow_failure" = "x" ; then - FAILED=1 - fi + FAILED=1 } run_test() { @@ -55,6 +60,12 @@ run_test() { } run_test_wd() { + local hang_ok= + if test "x$1" = "x--allow-hang" ; then + hang_ok=1 + shift + fi + local timeout="$1" test $# -gt 0 && shift @@ -77,14 +88,13 @@ run_test_wd() { while test $restarts -gt 0 ; do : > "${status_file}" ( - FAILED=0 - if ! ( - set -o pipefail - eval "$cmd" 2>&1 | tee -a "$output_file" - ) ; then - fail "${test_name}" "$@" + set -o pipefail + ret=0 + if ! eval "$cmd" 2>&1 | tee -a "$output_file" ; then + ret=1 fi - echo "$FAILED" > "$status_file" + echo "$ret" > "$status_file" + exit $ret ) & local pid=$! while test "$(stat -c "%s" "$status_file")" -eq 0 ; do @@ -101,7 +111,9 @@ run_test_wd() { # status file not updated, assuming hang kill -KILL $pid if test $restarts -eq 0 ; then - fail "${test_name}" E "Test hang up" + if test "x$hang_ok" = "x" ; then + fail "${test_name}" E "Test hang up" + fi else echo "Test ${test_name} hang up, restarting" eval "$restart_cmd" @@ -116,6 +128,20 @@ run_test_wd() { done } -succeeded() { - return $FAILED +ended_successfully() { + if [[ -f "${FAIL_SUMMARY_FILE}" ]]; then + echo 'Test failed, complete summary:' + cat "${FAIL_SUMMARY_FILE}" + return 1 + fi + if ! [[ -f "${END_MARKER}" ]] ; then + echo 'ended_successfully called before end marker was touched' + return 1 + fi + return 0 +} + +end_tests() { + touch "${END_MARKER}" + ended_successfully } diff --git a/ci/common/test.sh b/ci/common/test.sh index 4936992cfd..d911d9bc18 100644 --- a/ci/common/test.sh +++ b/ci/common/test.sh @@ -1,4 +1,5 @@ source "${CI_DIR}/common/build.sh" +source "${CI_DIR}/common/suite.sh" print_core() { local app="$1" @@ -40,10 +41,9 @@ check_core_dumps() { print_core "$app" "$core" fi done - if test "$app" = quiet ; then - return 0 + if test "$app" != quiet ; then + fail 'cores' E 'Core dumps found' fi - exit 1 } check_logs() { @@ -62,8 +62,7 @@ check_logs() { err=1 done if [[ -n "${err}" ]]; then - echo "Runtime errors detected." - exit 1 + fail 'logs' E 'Runtime errors detected.' fi } @@ -75,50 +74,53 @@ asan_check() { check_logs "${1}" "*san.*" } -run_unittests() { +run_unittests() {( + enter_suite unittests ulimit -c unlimited if ! build_make unittest ; then - check_core_dumps "$(which luajit)" - exit 1 + fail 'unittests' F 'Unit tests failed' fi check_core_dumps "$(which luajit)" -} + exit_suite +)} -run_functionaltests() { +run_functionaltests() {( + enter_suite functionaltests ulimit -c unlimited if ! build_make ${FUNCTIONALTEST}; then - asan_check "${LOG_DIR}" - valgrind_check "${LOG_DIR}" - check_core_dumps - exit 1 + fail 'functionaltests' F 'Functional tests failed' fi asan_check "${LOG_DIR}" valgrind_check "${LOG_DIR}" check_core_dumps -} + exit_suite +)} -run_oldtests() { +run_oldtests() {( + enter_suite oldtests ulimit -c unlimited if ! make -C "${TRAVIS_BUILD_DIR}/src/nvim/testdir"; then reset - asan_check "${LOG_DIR}" - valgrind_check "${LOG_DIR}" - check_core_dumps - exit 1 + fail 'oldtests' F 'Legacy tests failed' fi asan_check "${LOG_DIR}" valgrind_check "${LOG_DIR}" check_core_dumps -} + exit_suite +)} -install_nvim() { - build_make install +install_nvim() {( + enter_suite 'install_nvim' + if ! build_make install ; then + fail 'install' E 'make install failed' + exit_suite + fi "${INSTALL_PREFIX}/bin/nvim" --version "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || { echo "Running ':help' in the installed nvim failed." echo "Maybe the helptags have not been generated properly." - exit 1 + fail 'help' F 'Failed running :help' } local genvimsynf=syntax/vim/generated.vim @@ -127,24 +129,22 @@ install_nvim() { cd runtime ; git ls-files | grep -e '.vim$' -e '.ps$' -e '.dict$' -e '.py$' -e '.tutor$' ) ; do if ! test -e "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then - echo "It appears that $file is not installed." - exit 1 + fail 'runtime-install' F "It appears that $file is not installed." fi done # Check that generated syntax file has function names, #5060. local gpat='syn keyword vimFuncName .*eval' if ! grep -q "$gpat" "${INSTALL_PREFIX}/share/nvim/runtime/$genvimsynf"; then - echo "It appears that $genvimsynf does not contain $gpat." - exit 1 + fail 'funcnames' F "It appears that $genvimsynf does not contain $gpat." fi for file in $( cd runtime ; git ls-files | grep -e '.awk$' -e '.sh$' -e '.bat$' ) ; do if ! test -x "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then - echo "It appears that $file is not installed or is not executable." - exit 1 + fail 'not-exe' F "It appears that $file is not installed or is not executable." fi done -} + exit_suite +)} diff --git a/ci/run_lint.sh b/ci/run_lint.sh index 39a90102e7..5639b4c3db 100755 --- a/ci/run_lint.sh +++ b/ci/run_lint.sh @@ -20,9 +20,10 @@ csi_clean() { run_test 'top_make clint-full' clint run_test 'top_make testlint' testlint CLICOLOR_FORCE=1 run_test_wd \ - 5s \ + --allow-hang \ + 10s \ 'top_make check-single-includes' \ 'csi_clean' \ single-includes -exit_suite +end_tests diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 92cb5a9fd8..4abc9eea9f 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -27,8 +27,4 @@ run_test run_oldtests run_test install_nvim -if succeeded ; then - touch "${SUCCESS_MARKER}" -fi - -exit_suite +end_tests diff --git a/cmake/InstallClintErrors.cmake b/cmake/InstallClintErrors.cmake new file mode 100644 index 0000000000..bd5ca07828 --- /dev/null +++ b/cmake/InstallClintErrors.cmake @@ -0,0 +1,2 @@ +file(GLOB_RECURSE JSON_FILES *.json) +file(COPY ${JSON_FILES} DESTINATION "${TARGET}") diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index e21f5357a2..16f9a2ea6e 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -4944,8 +4944,8 @@ json_decode({expr}) *json_decode()* json_encode({expr}) *json_encode()* Convert {expr} into a JSON string. Accepts - |msgpack-special-dict| as the input. Will not convert |Funcref|s, - mappings with non-string keys (can be created as + |msgpack-special-dict| as the input. Will not convert + |Funcref|s, mappings with non-string keys (can be created as |msgpack-special-dict|), values with self-referencing containers, strings which contain non-UTF-8 characters, pseudo-UTF-8 strings which contain codepoints reserved for diff --git a/runtime/doc/options.txt b/runtime/doc/options.txt index d212e029aa..c30a88f48d 100644 --- a/runtime/doc/options.txt +++ b/runtime/doc/options.txt @@ -5151,8 +5151,8 @@ A jump table for the options with a short description can be found at |Q_op|. saved. When not included, the value of 'history' is used. *shada-c* c Dummy option, kept for compatibility reasons. Has no actual - effect. Current encoding state is described in - |shada-encoding|. + effect: ShaDa always uses UTF-8 and 'encoding' value is fixed + to UTF-8 as well. *shada-f* f Whether file marks need to be stored. If zero, file marks ('0 to '9, 'A to 'Z) are not stored. When not present or when diff --git a/runtime/doc/starting.txt b/runtime/doc/starting.txt index daf6ad9ca2..2d1dd22222 100644 --- a/runtime/doc/starting.txt +++ b/runtime/doc/starting.txt @@ -1097,23 +1097,6 @@ SHADA FILE NAME *shada-file-name* default and the name given with 'shada' or "-i" (unless it's NONE). -CHARACTER ENCODING *shada-encoding* - -The text in the ShaDa file is UTF-8-encoded. Normally you will always work -with the same 'encoding' value, and this works just fine. However, if you -read the ShaDa file with value for 'encoding' different from utf-8 and -'encoding' used when writing ShaDa file, some of the text (non-ASCII -characters) may be invalid as Neovim always attempts to convert the text in -the ShaDa file from the UTF-8 to the current 'encoding' value. Filenames are -never converted, affected elements are: - -- history strings; -- variable values; -- register values; -- last used search and substitute patterns; -- last used substitute replacement string. - - MANUALLY READING AND WRITING *shada-read-write* Two commands can be used to read and write the ShaDa file manually. This @@ -1221,8 +1204,11 @@ exactly four MessagePack objects: 3. Third goes the length of the fourth entry. Unsigned integer as well, used for fast skipping without parsing. 4. Fourth is actual entry data. All currently used ShaDa entries use - containers to hold data: either map or array. Exact format depends on the - entry type: + containers to hold data: either map or array. All string values in those + containers are either binary (applies to filenames) or UTF-8, yet parser + needs to expect that invalid bytes may be present in a UTF-8 string. + + Exact format depends on the entry type: Entry type (name) Entry data ~ 1 (Header) Map containing data that describes the generator diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index e0f4944762..cb003c2026 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -41,7 +41,10 @@ set(LINT_SUPPRESS_URL "${LINT_SUPPRESS_URL_BASE}/errors.json") set(LINT_PRG ${PROJECT_SOURCE_DIR}/src/clint.py) set(DOWNLOAD_SCRIPT ${PROJECT_SOURCE_DIR}/cmake/Download.cmake) set(LINT_SUPPRESSES_ROOT ${PROJECT_BINARY_DIR}/errors) -set(LINT_SUPPRESSES_URL "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint/errors.tar.gz") +set(LINT_SUPPRESSES_URL "${LINT_SUPPRESS_URL_BASE}/errors.tar.gz") +set(LINT_SUPPRESSES_ARCHIVE ${LINT_SUPPRESSES_ROOT}/errors.tar.gz) +set(LINT_SUPPRESSES_TOUCH_FILE "${TOUCHES_DIR}/unpacked-clint-errors-archive") +set(LINT_SUPPRESSES_INSTALL_SCRIPT "${PROJECT_SOURCE_DIR}/cmake/InstallClintErrors.cmake") include_directories(${GENERATED_DIR}) include_directories(${CACHED_GENERATED_DIR}) @@ -50,6 +53,8 @@ include_directories(${GENERATED_INCLUDES_DIR}) file(MAKE_DIRECTORY ${TOUCHES_DIR}) file(MAKE_DIRECTORY ${GENERATED_DIR}) file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}) +file(MAKE_DIRECTORY ${LINT_SUPPRESSES_ROOT}) +file(MAKE_DIRECTORY ${LINT_SUPPRESSES_ROOT}/src) file(GLOB NVIM_SOURCES *.c) file(GLOB NVIM_HEADERS *.h) @@ -486,6 +491,20 @@ function(add_download output url allow_failure) ) endfunction() +add_download(${LINT_SUPPRESSES_ARCHIVE} ${LINT_SUPPRESSES_URL} off) + +add_custom_command( + OUTPUT ${LINT_SUPPRESSES_TOUCH_FILE} + WORKING_DIRECTORY ${LINT_SUPPRESSES_ROOT}/src + COMMAND ${CMAKE_COMMAND} -E tar xfz ${LINT_SUPPRESSES_ARCHIVE} + COMMAND + ${CMAKE_COMMAND} + -DTARGET=${LINT_SUPPRESSES_ROOT} + -P ${LINT_SUPPRESSES_INSTALL_SCRIPT} + DEPENDS + ${LINT_SUPPRESSES_ARCHIVE} ${LINT_SUPPRESSES_INSTALL_SCRIPT} +) + add_download(${LINT_SUPPRESS_FILE} ${LINT_SUPPRESS_URL} off) set(LINT_NVIM_REL_SOURCES) @@ -494,14 +513,13 @@ foreach(sfile ${LINT_NVIM_SOURCES}) set(suppress_file ${LINT_SUPPRESSES_ROOT}/${suffix}.json) set(suppress_url "${LINT_SUPPRESS_URL_BASE}/${suffix}.json") set(rsfile src/nvim/${r}) - add_download(${suppress_file} ${suppress_url} on) set(touch_file "${TOUCHES_DIR}/ran-clint-${suffix}") add_custom_command( OUTPUT ${touch_file} COMMAND ${LINT_PRG} --suppress-errors=${suppress_file} ${rsfile} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${CMAKE_COMMAND} -E touch ${touch_file} - DEPENDS ${LINT_PRG} ${sfile} ${suppress_file} + DEPENDS ${LINT_PRG} ${sfile} ${LINT_SUPPRESSES_TOUCH_FILE} ) list(APPEND LINT_TARGETS ${touch_file}) list(APPEND LINT_NVIM_REL_SOURCES ${rsfile}) diff --git a/src/nvim/digraph.h b/src/nvim/digraph.h index d0f10eaf78..1b73ccaf3f 100644 --- a/src/nvim/digraph.h +++ b/src/nvim/digraph.h @@ -1,8 +1,8 @@ #ifndef NVIM_DIGRAPH_H #define NVIM_DIGRAPH_H -#include "nvim/types.h" // for char_u -#include "nvim/ex_cmds_defs.h" // for exarg_T +#include "nvim/types.h" +#include "nvim/ex_cmds_defs.h" #ifdef INCLUDE_GENERATED_DECLARATIONS # include "digraph.h.generated.h" diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index fb31a65971..a7dc6b205d 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -222,8 +222,6 @@ static inline int json_decoder_pop(ValuesStackItem obj, /// Parse JSON double-quoted string /// -/// @param[in] conv Defines conversion necessary to convert UTF-8 string to -/// &encoding. /// @param[in] buf Buffer being converted. /// @param[in] buf_len Length of the buffer. /// @param[in,out] pp Pointer to the start of the string. Must point to '"'. @@ -240,8 +238,7 @@ static inline int json_decoder_pop(ValuesStackItem obj, /// value when decoder is restarted, otherwise unused. /// /// @return OK in case of success, FAIL in case of error. -static inline int parse_json_string(vimconv_T *const conv, - const char *const buf, const size_t buf_len, +static inline int parse_json_string(const char *const buf, const size_t buf_len, const char **const pp, ValuesStack *const stack, ContainerStack *const container_stack, @@ -416,20 +413,6 @@ static inline int parse_json_string(vimconv_T *const conv, } PUT_FST_IN_PAIR(fst_in_pair, str_end); #undef PUT_FST_IN_PAIR - if (conv->vc_type != CONV_NONE) { - size_t str_len = (size_t) (str_end - str); - char *const new_str = (char *) string_convert(conv, (char_u *) str, - &str_len); - if (new_str == NULL) { - emsgf(_("E474: Failed to convert string \"%.*s\" from UTF-8"), - (int) str_len, str); - xfree(str); - goto parse_json_string_fail; - } - xfree(str); - str = new_str; - str_end = new_str + str_len; - } if (hasnul) { typval_T obj; list_T *const list = tv_list_alloc(); @@ -626,9 +609,6 @@ int json_decode_string(const char *const buf, const size_t buf_len, EMSG(_("E474: Attempt to decode a blank string")); return FAIL; } - vimconv_T conv = { .vc_type = CONV_NONE }; - convert_setup(&conv, (char_u *) "utf-8", p_enc); - conv.vc_fail = true; int ret = OK; ValuesStack stack = KV_INITIAL_VALUE; ContainerStack container_stack = KV_INITIAL_VALUE; @@ -774,7 +754,7 @@ json_decode_string_cycle_start: break; } case '"': { - if (parse_json_string(&conv, buf, buf_len, &p, &stack, &container_stack, + if (parse_json_string(buf, buf_len, &p, &stack, &container_stack, &next_map_special, &didcomma, &didcolon) == FAIL) { // Error message was already given diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 26f9aaa27d..d74913a481 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -11,7 +11,7 @@ #include <math.h> #include "nvim/eval/encode.h" -#include "nvim/buffer_defs.h" // vimconv_T +#include "nvim/buffer_defs.h" #include "nvim/eval.h" #include "nvim/eval/typval.h" #include "nvim/garray.h" @@ -29,10 +29,6 @@ #define utf_ptr2char(b) utf_ptr2char((char_u *)b) #define utf_ptr2len(b) ((size_t)utf_ptr2len((char_u *)b)) #define utf_char2len(b) ((size_t)utf_char2len(b)) -#define string_convert(a, b, c) \ - ((char *)string_convert((vimconv_T *)a, (char_u *)b, c)) -#define convert_setup(vcp, from, to) \ - (convert_setup(vcp, (char_u *)from, (char_u *)to)) const char *const encode_special_var_names[] = { [kSpecialVarNull] = "null", @@ -537,17 +533,6 @@ int encode_read_from_list(ListReaderState *const state, char *const buf, } \ } while (0) -/// Last used p_enc value -/// -/// Generic pointer: it is not used as a string, only pointer comparisons are -/// performed. Must not be freed. -static const void *last_p_enc = NULL; - -/// Conversion setup for converting from last_p_enc to UTF-8 -static vimconv_T p_enc_conv = { - .vc_type = CONV_NONE, -}; - /// Escape sequences used in JSON static const char escapes[][3] = { [BS] = "\\b", @@ -579,33 +564,15 @@ static inline int convert_to_json_string(garray_T *const gap, } else { size_t utf_len = len; char *tofree = NULL; - if (last_p_enc != (const void *) p_enc) { - p_enc_conv.vc_type = CONV_NONE; - convert_setup(&p_enc_conv, p_enc, "utf-8"); - p_enc_conv.vc_fail = true; - last_p_enc = p_enc; - } - if (p_enc_conv.vc_type != CONV_NONE) { - tofree = string_convert(&p_enc_conv, buf, &utf_len); - if (tofree == NULL) { - emsgf(_("E474: Failed to convert string \"%.*s\" to UTF-8"), - utf_len, utf_buf); - return FAIL; - } - utf_buf = tofree; - } size_t str_len = 0; - // Encode character as \u0000 if - // 1. It is an ASCII control character (0x0 .. 0x1F, 0x7F). - // 2. &encoding is not UTF-8 and code point is above 0x7F. - // 3. &encoding is UTF-8 and code point is not printable according to - // utf_printable(). - // This is done to make it possible to :echo values when &encoding is not - // UTF-8. -#define ENCODE_RAW(p_enc_conv, ch) \ - (ch >= 0x20 && (p_enc_conv.vc_type == CONV_NONE \ - ? utf_printable(ch) \ - : ch < 0x7F)) + // Encode character as \uNNNN if + // 1. It is an ASCII control character (0x0 .. 0x1F; 0x7F not + // utf_printable and thus not checked specially). + // 2. Code point is not printable according to utf_printable(). + // This is done to make resulting values displayable on screen also not from + // Neovim. +#define ENCODE_RAW(ch) \ + (ch >= 0x20 && utf_printable(ch)) for (size_t i = 0; i < utf_len;) { const int ch = utf_ptr2char(utf_buf + i); const size_t shift = (ch == 0? 1: utf_ptr2len(utf_buf + i)); @@ -636,7 +603,7 @@ static inline int convert_to_json_string(garray_T *const gap, utf_len - (i - shift), utf_buf + i - shift); xfree(tofree); return FAIL; - } else if (ENCODE_RAW(p_enc_conv, ch)) { + } else if (ENCODE_RAW(ch)) { str_len += shift; } else { str_len += ((sizeof("\\u1234") - 1) @@ -666,7 +633,7 @@ static inline int convert_to_json_string(garray_T *const gap, break; } default: { - if (ENCODE_RAW(p_enc_conv, ch)) { + if (ENCODE_RAW(ch)) { ga_concat_len(gap, utf_buf + i, shift); } else if (ch < SURROGATE_FIRST_CHAR) { ga_concat_len(gap, ((const char[]) { diff --git a/src/nvim/ex_cmds.h b/src/nvim/ex_cmds.h index 792e2f772f..b564cde56c 100644 --- a/src/nvim/ex_cmds.h +++ b/src/nvim/ex_cmds.h @@ -6,8 +6,8 @@ #include "nvim/os/time.h" #include "nvim/pos.h" #include "nvim/eval/typval.h" -#include "nvim/buffer_defs.h" // for buf_T and win_T -#include "nvim/ex_cmds_defs.h" // for exarg_T +#include "nvim/buffer_defs.h" +#include "nvim/ex_cmds_defs.h" // flags for do_ecmd() #define ECMD_HIDE 0x01 // don't free the current buffer diff --git a/src/nvim/ex_getln.h b/src/nvim/ex_getln.h index a29c8297d5..051564fbe1 100644 --- a/src/nvim/ex_getln.h +++ b/src/nvim/ex_getln.h @@ -3,9 +3,9 @@ #include "nvim/eval/typval.h" #include "nvim/ex_cmds.h" -#include "nvim/ex_cmds_defs.h" // for exarg_T -#include "nvim/os/time.h" // for Timestamp -#include "nvim/regexp_defs.h" // for regmatch_T +#include "nvim/ex_cmds_defs.h" +#include "nvim/os/time.h" +#include "nvim/regexp_defs.h" /* Values for nextwild() and ExpandOne(). See ExpandOne() for meaning. */ #define WILD_FREE 1 diff --git a/src/nvim/fold.h b/src/nvim/fold.h index 2393f4ef47..f35b328fb1 100644 --- a/src/nvim/fold.h +++ b/src/nvim/fold.h @@ -1,12 +1,12 @@ #ifndef NVIM_FOLD_H #define NVIM_FOLD_H -#include <stdio.h> // for FILE +#include <stdio.h> #include "nvim/pos.h" -#include "nvim/garray.h" // for garray_T -#include "nvim/types.h" // for char_u -#include "nvim/buffer_defs.h" // for win_T +#include "nvim/garray.h" +#include "nvim/types.h" +#include "nvim/buffer_defs.h" /* * Info used to pass info about a fold from the fold-detection code to the diff --git a/src/nvim/iconv.h b/src/nvim/iconv.h index bf29b15247..d7234090c4 100644 --- a/src/nvim/iconv.h +++ b/src/nvim/iconv.h @@ -10,9 +10,7 @@ // USE_ICONV, or to put the USE_ICONV definition in config.h.in directly. As // it stands, globals.h needs to be included alongside iconv.h. -#ifdef HAVE_CONFIG_H -# include "auto/config.h" -#endif +#include "auto/config.h" // Use iconv() when it's available, either by linking to the library at // compile time or by loading it at runtime. diff --git a/src/nvim/memfile_defs.h b/src/nvim/memfile_defs.h index 57d8abbe30..b3c2f3564c 100644 --- a/src/nvim/memfile_defs.h +++ b/src/nvim/memfile_defs.h @@ -3,10 +3,10 @@ #include <stdint.h> #include <stdbool.h> -#include <stdlib.h> // for size_t +#include <stdlib.h> #include "nvim/types.h" -#include "nvim/pos.h" // for linenr_T +#include "nvim/pos.h" /// A block number. /// diff --git a/src/nvim/move.h b/src/nvim/move.h index 1cdf1f6f52..00fbcc580f 100644 --- a/src/nvim/move.h +++ b/src/nvim/move.h @@ -2,8 +2,8 @@ #define NVIM_MOVE_H #include <stdbool.h> -#include "nvim/buffer_defs.h" // for win_T -#include "nvim/pos.h" // for linenr_T +#include "nvim/buffer_defs.h" +#include "nvim/pos.h" #ifdef INCLUDE_GENERATED_DECLARATIONS # include "move.h.generated.h" diff --git a/src/nvim/shada.c b/src/nvim/shada.c index c7b95958e0..8c5d6dff65 100644 --- a/src/nvim/shada.c +++ b/src/nvim/shada.c @@ -73,15 +73,10 @@ KHASH_SET_INIT_STR(strset) (vim_rename((char_u *)a, (char_u *)b)) #define mb_strnicmp(a, b, c) \ (mb_strnicmp((char_u *)a, (char_u *)b, c)) -#define has_non_ascii(a) (has_non_ascii((char_u *)a)) -#define string_convert(a, b, c) \ - ((char *)string_convert((vimconv_T *)a, (char_u *)b, c)) #define path_shorten_fname_if_possible(b) \ ((char *)path_shorten_fname_if_possible((char_u *)b)) #define buflist_new(ffname, sfname, ...) \ (buflist_new((char_u *)ffname, (char_u *)sfname, __VA_ARGS__)) -#define convert_setup(vcp, from, to) \ - (convert_setup(vcp, (char_u *)from, (char_u *)to)) #define os_isdir(f) (os_isdir((char_u *) f)) #define regtilde(s, m) ((char *) regtilde((char_u *) s, m)) #define path_tail_with_sep(f) ((char *) path_tail_with_sep((char_u *)f)) @@ -413,8 +408,6 @@ typedef struct sd_read_def { const char *error; ///< Error message in case of error. uintmax_t fpos; ///< Current position (amount of bytes read since ///< reader structure initialization). May overflow. - vimconv_T sd_conv; ///< Structure used for converting encodings of some - ///< items. } ShaDaReadDef; struct sd_write_def; @@ -435,8 +428,6 @@ typedef struct sd_write_def { ShaDaWriteCloser close; ///< Close function. void *cookie; ///< Data describing object written to. const char *error; ///< Error message in case of error. - vimconv_T sd_conv; ///< Structure used for converting encodings of some - ///< items. } ShaDaWriteDef; #ifdef INCLUDE_GENERATED_DECLARATIONS @@ -803,7 +794,7 @@ static int open_shada_file_for_reading(const char *const fname, return error; } - convert_setup(&sd_reader->sd_conv, "utf-8", p_enc); + assert(STRCMP(p_enc, "utf-8") == 0); return 0; } @@ -1899,127 +1890,24 @@ shada_pack_entry_error: } #undef PACK_STRING -/// Write single ShaDa entry, converting it if needed +/// Write single ShaDa entry and free it afterwards /// -/// @warning Frees entry after packing. +/// Will not free if entry could not be freed. /// /// @param[in] packer Packer used to write entry. -/// @param[in] sd_conv Conversion definitions. -/// @param[in] entry Entry written. If entry.can_free_entry is false then -/// it assumes that entry was not converted, otherwise it -/// is assumed that entry was already converted. +/// @param[in] entry Entry written. /// @param[in] max_kbyte Maximum size of an item in KiB. Zero means no /// restrictions. -static ShaDaWriteResult shada_pack_encoded_entry(msgpack_packer *const packer, - const vimconv_T *const sd_conv, - PossiblyFreedShadaEntry entry, - const size_t max_kbyte) - FUNC_ATTR_NONNULL_ALL +static inline ShaDaWriteResult shada_pack_pfreed_entry( + msgpack_packer *const packer, PossiblyFreedShadaEntry entry, + const size_t max_kbyte) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE { ShaDaWriteResult ret = kSDWriteSuccessfull; + ret = shada_pack_entry(packer, entry.data, max_kbyte); if (entry.can_free_entry) { - ret = shada_pack_entry(packer, entry.data, max_kbyte); shada_free_shada_entry(&entry.data); - return ret; - } -#define RUN_WITH_CONVERTED_STRING(cstr, code) \ - do { \ - bool did_convert = false; \ - if (sd_conv->vc_type != CONV_NONE && has_non_ascii((cstr))) { \ - char *const converted_string = string_convert(sd_conv, (cstr), NULL); \ - if (converted_string != NULL) { \ - (cstr) = converted_string; \ - did_convert = true; \ - } \ - } \ - code \ - if (did_convert) { \ - xfree((cstr)); \ - } \ - } while (0) - switch (entry.data.type) { - case kSDItemUnknown: - case kSDItemMissing: { - assert(false); - } - case kSDItemSearchPattern: { - RUN_WITH_CONVERTED_STRING(entry.data.data.search_pattern.pat, { - ret = shada_pack_entry(packer, entry.data, max_kbyte); - }); - break; - } - case kSDItemHistoryEntry: { - RUN_WITH_CONVERTED_STRING(entry.data.data.history_item.string, { - ret = shada_pack_entry(packer, entry.data, max_kbyte); - }); - break; - } - case kSDItemSubString: { - RUN_WITH_CONVERTED_STRING(entry.data.data.sub_string.sub, { - ret = shada_pack_entry(packer, entry.data, max_kbyte); - }); - break; - } - case kSDItemVariable: { - if (sd_conv->vc_type != CONV_NONE) { - typval_T tgttv; - var_item_copy(sd_conv, &entry.data.data.global_var.value, &tgttv, - true, 0); - tv_clear(&entry.data.data.global_var.value); - entry.data.data.global_var.value = tgttv; - } - ret = shada_pack_entry(packer, entry.data, max_kbyte); - break; - } - case kSDItemRegister: { - bool did_convert = false; - if (sd_conv->vc_type != CONV_NONE) { - size_t first_non_ascii = 0; - for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) { - if (has_non_ascii(entry.data.data.reg.contents[i])) { - first_non_ascii = i; - did_convert = true; - break; - } - } - if (did_convert) { - entry.data.data.reg.contents = - xmemdup(entry.data.data.reg.contents, - (entry.data.data.reg.contents_size - * sizeof(entry.data.data.reg.contents[0]))); - for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) { - if (i >= first_non_ascii) { - entry.data.data.reg.contents[i] = get_converted_string( - sd_conv, - entry.data.data.reg.contents[i], - strlen(entry.data.data.reg.contents[i])); - } else { - entry.data.data.reg.contents[i] = - xstrdup(entry.data.data.reg.contents[i]); - } - } - } - } - ret = shada_pack_entry(packer, entry.data, max_kbyte); - if (did_convert) { - for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) { - xfree(entry.data.data.reg.contents[i]); - } - xfree(entry.data.data.reg.contents); - } - break; - } - case kSDItemHeader: - case kSDItemGlobalMark: - case kSDItemJump: - case kSDItemBufferList: - case kSDItemLocalMark: - case kSDItemChange: { - ret = shada_pack_entry(packer, entry.data, max_kbyte); - break; - } } -#undef RUN_WITH_CONVERTED_STRING return ret; } @@ -2556,11 +2444,7 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, break; } typval_T tgttv; - if (sd_writer->sd_conv.vc_type != CONV_NONE) { - var_item_copy(&sd_writer->sd_conv, &vartv, &tgttv, true, 0); - } else { - tv_copy(&vartv, &tgttv); - } + tv_copy(&vartv, &tgttv); ShaDaWriteResult spe_ret; if ((spe_ret = shada_pack_entry(packer, (ShadaEntry) { .type = kSDItemVariable, @@ -2811,9 +2695,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, do { \ for (size_t i_ = 0; i_ < ARRAY_SIZE(wms_array); i_++) { \ if (wms_array[i_].data.type != kSDItemMissing) { \ - if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, \ - wms_array[i_], \ - max_kbyte) == kSDWriteFailed) { \ + if (shada_pack_pfreed_entry(packer, wms_array[i_], max_kbyte) \ + == kSDWriteFailed) { \ ret = kSDWriteFailed; \ goto shada_write_exit; \ } \ @@ -2823,8 +2706,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, PACK_WMS_ARRAY(wms->global_marks); PACK_WMS_ARRAY(wms->registers); for (size_t i = 0; i < wms->jumps_size; i++) { - if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, wms->jumps[i], - max_kbyte) == kSDWriteFailed) { + if (shada_pack_pfreed_entry(packer, wms->jumps[i], max_kbyte) + == kSDWriteFailed) { ret = kSDWriteFailed; goto shada_write_exit; } @@ -2832,8 +2715,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, #define PACK_WMS_ENTRY(wms_entry) \ do { \ if (wms_entry.data.type != kSDItemMissing) { \ - if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, wms_entry, \ - max_kbyte) == kSDWriteFailed) { \ + if (shada_pack_pfreed_entry(packer, wms_entry, max_kbyte) \ + == kSDWriteFailed) { \ ret = kSDWriteFailed; \ goto shada_write_exit; \ } \ @@ -2860,9 +2743,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, for (size_t i = 0; i < file_markss_to_dump; i++) { PACK_WMS_ARRAY(all_file_markss[i]->marks); for (size_t j = 0; j < all_file_markss[i]->changes_size; j++) { - if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, - all_file_markss[i]->changes[j], - max_kbyte) == kSDWriteFailed) { + if (shada_pack_pfreed_entry(packer, all_file_markss[i]->changes[j], + max_kbyte) == kSDWriteFailed) { ret = kSDWriteFailed; goto shada_write_exit; } @@ -2886,8 +2768,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer, if (dump_one_history[i]) { hms_insert_whole_neovim_history(&wms->hms[i]); HMS_ITER(&wms->hms[i], cur_entry, { - if (shada_pack_encoded_entry( - packer, &sd_writer->sd_conv, (PossiblyFreedShadaEntry) { + if (shada_pack_pfreed_entry( + packer, (PossiblyFreedShadaEntry) { .data = cur_entry->data, .can_free_entry = cur_entry->can_free_entry, }, max_kbyte) == kSDWriteFailed) { @@ -3038,8 +2920,6 @@ shada_write_file_nomerge: {} verbose_leave(); } - convert_setup(&sd_writer.sd_conv, p_enc, "utf-8"); - const ShaDaWriteResult sw_ret = shada_write(&sd_writer, (nomerge ? NULL : &sd_reader)); @@ -3327,29 +3207,6 @@ static ShaDaReadResult msgpack_read_uint64(ShaDaReadDef *const sd_reader, return kSDReadStatusSuccess; } -/// Convert or copy and return a string -/// -/// @param[in] sd_conv Conversion definition. -/// @param[in] str String to convert. -/// @param[in] len String length. -/// -/// @return [allocated] converted string or copy of the original string. -static inline char *get_converted_string(const vimconv_T *const sd_conv, - const char *const str, - const size_t len) - FUNC_ATTR_NONNULL_ALL FUNC_ATTR_MALLOC FUNC_ATTR_WARN_UNUSED_RESULT -{ - if (!has_non_ascii_len(str, len)) { - return xmemdupz(str, len); - } - size_t new_len = len; - char *const new_str = string_convert(sd_conv, str, &new_len); - if (new_str == NULL) { - return xmemdupz(str, len); - } - return new_str; -} - #define READERR(entry_name, error_desc) \ RERR "Error while reading ShaDa file: " \ entry_name " entry at position %" PRIu64 " " \ @@ -3427,10 +3284,7 @@ static inline char *get_converted_string(const vimconv_T *const sd_conv, sizeof(*unpacked.data.via.map.ptr)); \ ad_ga.ga_len++; \ } -#define CONVERTED(str, len) ( \ - sd_reader->sd_conv.vc_type != CONV_NONE \ - ? get_converted_string(&sd_reader->sd_conv, (str), (len)) \ - : xmemdupz((str), (len))) +#define CONVERTED(str, len) (xmemdupz((str), (len))) #define BIN_CONVERTED(b) CONVERTED(b.ptr, b.size) #define SET_ADDITIONAL_DATA(tgt, name) \ do { \ @@ -3803,30 +3657,14 @@ shada_read_next_item_start: (char) unpacked.data.via.array.ptr[2].via.u64; } size_t strsize; - if (sd_reader->sd_conv.vc_type == CONV_NONE - || !has_non_ascii_len(unpacked.data.via.array.ptr[1].via.bin.ptr, - unpacked.data.via.array.ptr[1].via.bin.size)) { -shada_read_next_item_hist_no_conv: - strsize = ( - unpacked.data.via.array.ptr[1].via.bin.size - + 1 // Zero byte - + 1); // Separator character - entry->data.history_item.string = xmalloc(strsize); - memcpy(entry->data.history_item.string, - unpacked.data.via.array.ptr[1].via.bin.ptr, - unpacked.data.via.array.ptr[1].via.bin.size); - } else { - size_t len = unpacked.data.via.array.ptr[1].via.bin.size; - char *const converted = string_convert( - &sd_reader->sd_conv, unpacked.data.via.array.ptr[1].via.bin.ptr, - &len); - if (converted != NULL) { - strsize = len + 2; - entry->data.history_item.string = xrealloc(converted, strsize); - } else { - goto shada_read_next_item_hist_no_conv; - } - } + strsize = ( + unpacked.data.via.array.ptr[1].via.bin.size + + 1 // Zero byte + + 1); // Separator character + entry->data.history_item.string = xmalloc(strsize); + memcpy(entry->data.history_item.string, + unpacked.data.via.array.ptr[1].via.bin.ptr, + unpacked.data.via.array.ptr[1].via.bin.size); entry->data.history_item.string[strsize - 2] = 0; entry->data.history_item.string[strsize - 1] = entry->data.history_item.sep; @@ -3859,16 +3697,6 @@ shada_read_next_item_hist_no_conv: "be converted to the VimL value")), initial_fpos); goto shada_read_next_item_error; } - if (sd_reader->sd_conv.vc_type != CONV_NONE) { - typval_T tgttv; - var_item_copy(&sd_reader->sd_conv, - &entry->data.global_var.value, - &tgttv, - true, - 0); - tv_clear(&entry->data.global_var.value); - entry->data.global_var.value = tgttv; - } SET_ADDITIONAL_ELEMENTS(unpacked.data.via.array, 2, entry->data.global_var.additional_elements, "variable"); diff --git a/src/nvim/vim.h b/src/nvim/vim.h index cc0587fb88..172e62b039 100644 --- a/src/nvim/vim.h +++ b/src/nvim/vim.h @@ -11,20 +11,16 @@ #define RUNTIME_DIRNAME "runtime" /* end */ -/* ============ the header file puzzle (ca. 50-100 pieces) ========= */ - -#ifdef HAVE_CONFIG_H /* GNU autoconf (or something else) was here */ -# include "auto/config.h" -# define HAVE_PATHDEF +#include "auto/config.h" +#define HAVE_PATHDEF /* * Check if configure correctly managed to find sizeof(int). If this failed, * it becomes zero. This is likely a problem of not being able to run the * test program. Other items from configure may also be wrong then! */ -# if (SIZEOF_INT == 0) -Error: configure did not run properly.Check auto/config.log. -# endif +#if (SIZEOF_INT == 0) +# error Configure did not run properly. #endif #include "nvim/os/os_defs.h" /* bring lots of system header files */ @@ -46,11 +42,6 @@ enum { NUMBUFLEN = 65 }; #include "nvim/keymap.h" #include "nvim/macros.h" - - - -/* ================ end of the header file puzzle =============== */ - #include "nvim/gettext.h" /* special attribute addition: Put message in history */ diff --git a/test/functional/helpers.lua b/test/functional/helpers.lua index 13e6f36510..13b06e7f1b 100644 --- a/test/functional/helpers.lua +++ b/test/functional/helpers.lua @@ -251,12 +251,13 @@ local function retry(max, max_ms, fn) return result end if (max and tries >= max) or (luv.now() - start_time > timeout) then - break + if type(result) == "string" then + result = "\nretry() attempts: "..tostring(tries).."\n"..result + end + error(result) end tries = tries + 1 end - -- Do not use pcall() for the final attempt, let the failure bubble up. - return fn() end local function clear(...) diff --git a/test/functional/terminal/tui_spec.lua b/test/functional/terminal/tui_spec.lua index fb8a61d10d..b14bceecdd 100644 --- a/test/functional/terminal/tui_spec.lua +++ b/test/functional/terminal/tui_spec.lua @@ -2,9 +2,10 @@ -- as a simple way to send keys and assert screen state. local helpers = require('test.functional.helpers')(after_each) local thelpers = require('test.functional.terminal.helpers') -local feed = thelpers.feed_data +local feed_data = thelpers.feed_data local feed_command = helpers.feed_command local nvim_dir = helpers.nvim_dir +local retry = helpers.retry if helpers.pending_win32(pending) then return end @@ -34,7 +35,7 @@ describe('tui', function() end) it('accepts basic utf-8 input', function() - feed('iabc\ntest1\ntest2') + feed_data('iabc\ntest1\ntest2') screen:expect([[ abc | test1 | @@ -44,7 +45,7 @@ describe('tui', function() {3:-- INSERT --} | {3:-- TERMINAL --} | ]]) - feed('\027') + feed_data('\027') screen:expect([[ abc | test1 | @@ -60,7 +61,7 @@ describe('tui', function() local keys = 'dfghjkl' for c in keys:gmatch('.') do feed_command('nnoremap <a-'..c..'> ialt-'..c..'<cr><esc>') - feed('\027'..c) + feed_data('\027'..c) end screen:expect([[ alt-j | @@ -71,7 +72,7 @@ describe('tui', function() | {3:-- TERMINAL --} | ]]) - feed('gg') + feed_data('gg') screen:expect([[ {1:a}lt-d | alt-f | @@ -90,7 +91,7 @@ describe('tui', function() -- Example: for input ALT+j: -- * Vim (Nvim prior to #3982) sets high-bit, inserts "ê". -- * Nvim (after #3982) inserts "j". - feed('i\027j') + feed_data('i\027j') screen:expect([[ j{1: } | {4:~ }| @@ -103,10 +104,10 @@ describe('tui', function() end) it('accepts ascii control sequences', function() - feed('i') - feed('\022\007') -- ctrl+g - feed('\022\022') -- ctrl+v - feed('\022\013') -- ctrl+m + feed_data('i') + feed_data('\022\007') -- ctrl+g + feed_data('\022\022') -- ctrl+v + feed_data('\022\013') -- ctrl+m screen:expect([[ {9:^G^V^M}{1: } | {4:~ }| @@ -119,7 +120,7 @@ describe('tui', function() end) it('automatically sends <Paste> for bracketed paste sequences', function() - feed('i\027[200~') + feed_data('i\027[200~') screen:expect([[ {1: } | {4:~ }| @@ -129,7 +130,7 @@ describe('tui', function() {3:-- INSERT (paste) --} | {3:-- TERMINAL --} | ]]) - feed('pasted from terminal') + feed_data('pasted from terminal') screen:expect([[ pasted from terminal{1: } | {4:~ }| @@ -139,7 +140,7 @@ describe('tui', function() {3:-- INSERT (paste) --} | {3:-- TERMINAL --} | ]]) - feed('\027[201~') + feed_data('\027[201~') screen:expect([[ pasted from terminal{1: } | {4:~ }| @@ -157,7 +158,7 @@ describe('tui', function() for i = 1, 3000 do t[i] = 'item ' .. tostring(i) end - feed('i\027[200~'..table.concat(t, '\n')..'\027[201~') + feed_data('i\027[200~'..table.concat(t, '\n')..'\027[201~') screen:expect([[ item 2997 | item 2998 | @@ -180,7 +181,7 @@ describe('tui with non-tty file descriptors', function() it('can handle pipes as stdout and stderr', function() local screen = thelpers.screen_setup(0, '"'..helpers.nvim_prog ..' -u NONE -i NONE --cmd \'set noswapfile noshowcmd noruler\' --cmd \'normal iabc\' > /dev/null 2>&1 && cat testF && rm testF"') - feed(':w testF\n:q\n') + feed_data(':w testF\n:q\n') screen:expect([[ :w testF | :q | @@ -200,12 +201,13 @@ describe('tui focus event handling', function() helpers.clear() screen = thelpers.screen_setup(0, '["'..helpers.nvim_prog ..'", "-u", "NONE", "-i", "NONE", "--cmd", "set noswapfile noshowcmd noruler"]') - feed_command('autocmd FocusGained * echo "gained"') - feed_command('autocmd FocusLost * echo "lost"') + feed_data(":autocmd FocusGained * echo 'gained'\n") + feed_data(":autocmd FocusLost * echo 'lost'\n") + feed_data("\034\016") -- CTRL-\ CTRL-N end) it('can handle focus events in normal mode', function() - feed('\027[I') + feed_data('\027[I') screen:expect([[ {1: } | {4:~ }| @@ -216,7 +218,7 @@ describe('tui focus event handling', function() {3:-- TERMINAL --} | ]]) - feed('\027[O') + feed_data('\027[O') screen:expect([[ {1: } | {4:~ }| @@ -230,8 +232,8 @@ describe('tui focus event handling', function() it('can handle focus events in insert mode', function() feed_command('set noshowmode') - feed('i') - feed('\027[I') + feed_data('i') + feed_data('\027[I') screen:expect([[ {1: } | {4:~ }| @@ -241,7 +243,7 @@ describe('tui focus event handling', function() gained | {3:-- TERMINAL --} | ]]) - feed('\027[O') + feed_data('\027[O') screen:expect([[ {1: } | {4:~ }| @@ -254,8 +256,8 @@ describe('tui focus event handling', function() end) it('can handle focus events in cmdline mode', function() - feed(':') - feed('\027[I') + feed_data(':') + feed_data('\027[I') screen:expect([[ | {4:~ }| @@ -265,7 +267,7 @@ describe('tui focus event handling', function() g{1:a}ined | {3:-- TERMINAL --} | ]]) - feed('\027[O') + feed_data('\027[O') screen:expect([[ | {4:~ }| @@ -278,30 +280,36 @@ describe('tui focus event handling', function() end) it('can handle focus events in terminal mode', function() - feed_command('set shell='..nvim_dir..'/shell-test') - feed_command('set laststatus=0') - feed_command('set noshowmode') - feed_command('terminal') - feed('\027[I') - screen:expect([[ - ready $ | - [Process exited 0]{1: } | - | - | - | - gained | - {3:-- TERMINAL --} | - ]]) - feed('\027[O') - screen:expect([[ - ready $ | - [Process exited 0]{1: } | - | - | - | - lost | - {3:-- TERMINAL --} | - ]]) + feed_data(':set shell='..nvim_dir..'/shell-test\n') + feed_data(':set noshowmode laststatus=0\n') + + retry(2, 3 * screen.timeout, function() + feed_data(':terminal\n') + feed_data('\027[I') + screen:expect([[ + ready $ | + [Process exited 0]{1: } | + | + | + | + gained | + {3:-- TERMINAL --} | + ]]) + feed_data('\027[O') + screen:expect([[ + ready $ | + [Process exited 0]{1: } | + | + | + | + lost | + {3:-- TERMINAL --} | + ]]) + + -- If retry is needed... + feed_data("\034\016") -- CTRL-\ CTRL-N + feed_data(':bwipeout!\n') + end) end) end) diff --git a/test/helpers.lua b/test/helpers.lua index 3fc10e9e30..d0b1ad552b 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -17,18 +17,28 @@ local ok = function(res) return assert.is_true(res) end +-- initial_path: directory to recurse into +-- re: include pattern (string) +-- exc_re: exclude pattern(s) (string or table) local function glob(initial_path, re, exc_re) + exc_re = type(exc_re) == 'table' and exc_re or { exc_re } local paths_to_check = {initial_path} local ret = {} local checked_files = {} + local function is_excluded(path) + for _, pat in pairs(exc_re) do + if path:match(pat) then return true end + end + return false + end + while #paths_to_check > 0 do local cur_path = paths_to_check[#paths_to_check] paths_to_check[#paths_to_check] = nil for e in lfs.dir(cur_path) do local full_path = cur_path .. '/' .. e local checked_path = full_path:sub(#initial_path + 1) - if ((not exc_re or not checked_path:match(exc_re)) - and e:sub(1, 1) ~= '.') then + if (not is_excluded(checked_path)) and e:sub(1, 1) ~= '.' then local attrs = lfs.attributes(full_path) if attrs then local check_key = attrs.dev .. ':' .. tostring(attrs.ino) @@ -106,13 +116,20 @@ local uname = (function() end) end)() +local function tmpdir_get() + return os.getenv('TMPDIR') and os.getenv('TMPDIR') or os.getenv('TEMP') +end + +-- Is temp directory `dir` defined local to the project workspace? +local function tmpdir_is_local(dir) + return not not (dir and string.find(dir, 'Xtest')) +end + local tmpname = (function() local seq = 0 - local tmpdir = os.getenv('TMPDIR') and os.getenv('TMPDIR') or os.getenv('TEMP') - -- Is $TMPDIR defined local to the project workspace? - local in_workspace = not not (tmpdir and string.find(tmpdir, 'Xtest')) + local tmpdir = tmpdir_get() return (function() - if in_workspace then + if tmpdir_is_local(tmpdir) then -- Cannot control os.tmpname() dir, so hack our own tmpname() impl. seq = seq + 1 local fname = tmpdir..'/nvim-test-lua-'..seq @@ -162,33 +179,34 @@ end local tests_skipped = 0 -local function check_cores(app) +local function check_cores(app, force) app = app or 'build/bin/nvim' local initial_path, re, exc_re local gdb_db_cmd = 'gdb -n -batch -ex "thread apply all bt full" "$_NVIM_TEST_APP" -c "$_NVIM_TEST_CORE"' local lldb_db_cmd = 'lldb -Q -o "bt all" -f "$_NVIM_TEST_APP" -c "$_NVIM_TEST_CORE"' local random_skip = false + local local_tmpdir = tmpdir_is_local(tmpdir_get()) and tmpdir_get() or nil local db_cmd if hasenv('NVIM_TEST_CORE_GLOB_DIRECTORY') then initial_path = os.getenv('NVIM_TEST_CORE_GLOB_DIRECTORY') re = os.getenv('NVIM_TEST_CORE_GLOB_RE') - exc_re = os.getenv('NVIM_TEST_CORE_EXC_RE') + exc_re = { os.getenv('NVIM_TEST_CORE_EXC_RE'), local_tmpdir } db_cmd = os.getenv('NVIM_TEST_CORE_DB_CMD') or gdb_db_cmd random_skip = os.getenv('NVIM_TEST_CORE_RANDOM_SKIP') elseif os.getenv('TRAVIS_OS_NAME') == 'osx' then initial_path = '/cores' re = nil - exc_re = nil + exc_re = { local_tmpdir } db_cmd = lldb_db_cmd else initial_path = '.' re = '/core[^/]*$' - exc_re = '^/%.deps$' + exc_re = { '^/%.deps$', local_tmpdir } db_cmd = gdb_db_cmd random_skip = true end -- Finding cores takes too much time on linux - if random_skip and math.random() < 0.9 then + if not force and random_skip and math.random() < 0.9 then tests_skipped = tests_skipped + 1 return end diff --git a/test/unit/eval/decode_spec.lua b/test/unit/eval/decode_spec.lua index 0b2a423cd6..0c444b33f2 100644 --- a/test/unit/eval/decode_spec.lua +++ b/test/unit/eval/decode_spec.lua @@ -2,7 +2,6 @@ local helpers = require('test.unit.helpers')(after_each) local itp = helpers.gen_itp(it) local cimport = helpers.cimport -local to_cstr = helpers.to_cstr local eq = helpers.eq local neq = helpers.neq local ffi = helpers.ffi @@ -72,7 +71,7 @@ describe('json_decode_string()', function() end itp('does not overflow in error messages', function() - local saved_p_enc = decode.p_enc + collectgarbage('restart') check_failure(']test', 1, 'E474: No container to close: ]') check_failure('[}test', 2, 'E474: Closing list with curly bracket: }') check_failure('{]test', 2, @@ -105,10 +104,6 @@ describe('json_decode_string()', function() check_failure('"\194"test', 3, 'E474: Only UTF-8 strings allowed: \194"') check_failure('"\252\144\128\128\128\128"test', 8, 'E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \252\144\128\128\128\128"') check_failure('"test', 1, 'E474: Expected string end: "') - decode.p_enc = to_cstr('latin1') - check_failure('"\\uABCD"test', 8, - 'E474: Failed to convert string "ꯍ" from UTF-8') - decode.p_enc = saved_p_enc check_failure('-test', 1, 'E474: Missing number after minus sign: -') check_failure('-1.test', 3, 'E474: Missing number after decimal dot: -1.') check_failure('-1.0etest', 5, 'E474: Missing exponent: -1.0e') diff --git a/test/unit/eval/helpers.lua b/test/unit/eval/helpers.lua index fa76113756..5bc482216e 100644 --- a/test/unit/eval/helpers.lua +++ b/test/unit/eval/helpers.lua @@ -468,7 +468,7 @@ local function tbl2callback(tbl) data={funcref=eval.xstrdup(tbl.fref)}}}) elseif tbl.type == 'pt' then local pt = ffi.gc(ffi.cast('partial_T*', - eval.xcalloc(1, ffi.sizeof('partial_T'))), eval.partial_unref) + eval.xcalloc(1, ffi.sizeof('partial_T'))), nil) ret = ffi.new('Callback[1]', {{type=eval.kCallbackPartial, data={partial=populate_partial(pt, tbl.pt, {})}}}) else diff --git a/test/unit/eval/typval_spec.lua b/test/unit/eval/typval_spec.lua index 258b5c4c1f..a1edfcfb7c 100644 --- a/test/unit/eval/typval_spec.lua +++ b/test/unit/eval/typval_spec.lua @@ -47,6 +47,15 @@ local lib = cimport('./src/nvim/eval/typval.h', './src/nvim/memory.h', './src/nvim/eval.h', './src/nvim/vim.h', './src/nvim/globals.h') +local function vimconv_alloc() + return ffi.gc( + ffi.cast('vimconv_T*', lib.xcalloc(1, ffi.sizeof('vimconv_T'))), + function(vc) + lib.convert_setup(vc, nil, nil) + lib.xfree(vc) + end) +end + local function list_watch_alloc(li) return ffi.cast('listwatch_T*', ffi.new('listwatch_T[1]', {{lw_item=li}})) end @@ -237,24 +246,33 @@ describe('typval.c', function() list_watch(l, lis[4]), list_watch(l, lis[7]), } + alloc_log:check({ + a.list(l), + a.li(lis[1]), + a.li(lis[2]), + a.li(lis[3]), + a.li(lis[4]), + a.li(lis[5]), + a.li(lis[6]), + a.li(lis[7]), + }) lib.tv_list_item_remove(l, lis[4]) - ffi.gc(lis[4], lib.tv_list_item_free) + alloc_log:check({a.freed(lis[4])}) eq({lis[1], lis[5], lis[7]}, {lws[1].lw_item, lws[2].lw_item, lws[3].lw_item}) lib.tv_list_item_remove(l, lis[2]) - ffi.gc(lis[2], lib.tv_list_item_free) + alloc_log:check({a.freed(lis[2])}) eq({lis[1], lis[5], lis[7]}, {lws[1].lw_item, lws[2].lw_item, lws[3].lw_item}) lib.tv_list_item_remove(l, lis[7]) - ffi.gc(lis[7], lib.tv_list_item_free) + alloc_log:check({a.freed(lis[7])}) eq({lis[1], lis[5], nil}, {lws[1].lw_item, lws[2].lw_item, lws[3].lw_item == nil and nil}) lib.tv_list_item_remove(l, lis[1]) - ffi.gc(lis[1], lib.tv_list_item_free) + alloc_log:check({a.freed(lis[1])}) eq({lis[3], lis[5], nil}, {lws[1].lw_item, lws[2].lw_item, lws[3].lw_item == nil and nil}) - alloc_log:clear() lib.tv_list_watch_remove(l, lws[2]) lib.tv_list_watch_remove(l, lws[3]) lib.tv_list_watch_remove(l, lws[1]) @@ -460,6 +478,10 @@ describe('typval.c', function() eq(empty_list, typvalt2lua(l_tv)) eq({true, true, true}, {lws[1].lw_item == nil, lws[2].lw_item == nil, lws[3].lw_item == nil}) + lib.tv_list_watch_remove(l, lws[1]) + lib.tv_list_watch_remove(l, lws[2]) + lib.tv_list_watch_remove(l, lws[3]) + alloc_log:check({}) end) end) @@ -730,10 +752,8 @@ describe('typval.c', function() collectgarbage() end) itp('copies list correctly and converts items', function() - local vc = ffi.gc(ffi.new('vimconv_T[1]'), function(vc) - lib.convert_setup(vc, nil, nil) - end) - -- UTF-8 ↔ latin1 conversions need no iconv + local vc = vimconv_alloc() + -- UTF-8 ↔ latin1 conversions needs no iconv eq(OK, lib.convert_setup(vc, to_cstr('utf-8'), to_cstr('latin1'))) local v = {{['«']='»'}, {'„'}, 1, '“', null_string, null_list, null_dict} @@ -1087,12 +1107,16 @@ describe('typval.c', function() end) end) describe('join()', function() - local function list_join(l, sep, ret) + local function list_join(l, sep, join_ret) local ga = ga_alloc() - eq(ret or OK, lib.tv_list_join(ga, l, sep)) - if ga.ga_data == nil then return '' - else return ffi.string(ga.ga_data) + eq(join_ret or OK, lib.tv_list_join(ga, l, sep)) + local ret = '' + if ga.ga_data ~= nil then + ret = ffi.string(ga.ga_data) end + -- For some reason this is not working well in GC + lib.ga_clear(ffi.gc(ga, nil)) + return ret end itp('works', function() local l @@ -1508,7 +1532,7 @@ describe('typval.c', function() eq(s:sub(1, len), ffi.string(di.di_key)) alloc_log:check({a.di(di, len)}) if tv then - di.di_tv = tv + di.di_tv = ffi.gc(tv, nil) else di.di_tv.v_type = lib.VAR_UNKNOWN end @@ -1539,7 +1563,7 @@ describe('typval.c', function() alloc_log:check({a.dict(d)}) local di = ffi.gc(lib.tv_dict_item_alloc(''), nil) local tv = lua2typvalt('test') - di.di_tv = tv + di.di_tv = ffi.gc(tv, nil) alloc_log:check({a.di(di, ''), a.str(tv.vval.v_string, 'test')}) eq(OK, lib.tv_dict_add(d, di)) alloc_log:check({}) @@ -2131,9 +2155,7 @@ describe('typval.c', function() collectgarbage() end) itp('copies dict correctly and converts items', function() - local vc = ffi.gc(ffi.new('vimconv_T[1]'), function(vc) - lib.convert_setup(vc, nil, nil) - end) + local vc = vimconv_alloc() -- UTF-8 ↔ latin1 conversions need no iconv eq(OK, lib.convert_setup(vc, to_cstr('utf-8'), to_cstr('latin1'))) @@ -2659,7 +2681,8 @@ describe('typval.c', function() {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 0}, {lib.VAR_UNKNOWN, nil, 'E685: Internal error: tv_get_number(UNKNOWN)', 0}, }) do - local tv = typvalt(v[1], v[2]) + -- Using to_cstr, cannot free with tv_clear + local tv = ffi.gc(typvalt(v[1], v[2]), nil) alloc_log:check({}) local emsg = v[3] local ret = v[4] @@ -2687,7 +2710,8 @@ describe('typval.c', function() {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 0}, {lib.VAR_UNKNOWN, nil, 'E685: Internal error: tv_get_number(UNKNOWN)', 0}, }) do - local tv = typvalt(v[1], v[2]) + -- Using to_cstr, cannot free with tv_clear + local tv = ffi.gc(typvalt(v[1], v[2]), nil) alloc_log:check({}) local emsg = v[3] local ret = {v[4], not not emsg} @@ -2721,7 +2745,8 @@ describe('typval.c', function() {lib.VAR_UNKNOWN, nil, 'E685: Internal error: tv_get_number(UNKNOWN)', -1}, }) do lib.curwin.w_cursor.lnum = 46 - local tv = typvalt(v[1], v[2]) + -- Using to_cstr, cannot free with tv_clear + local tv = ffi.gc(typvalt(v[1], v[2]), nil) alloc_log:check({}) local emsg = v[3] local ret = v[4] @@ -2749,7 +2774,8 @@ describe('typval.c', function() {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, 'E907: Using a special value as a Float', 0}, {lib.VAR_UNKNOWN, nil, 'E685: Internal error: tv_get_float(UNKNOWN)', 0}, }) do - local tv = typvalt(v[1], v[2]) + -- Using to_cstr, cannot free with tv_clear + local tv = ffi.gc(typvalt(v[1], v[2]), nil) alloc_log:check({}) local emsg = v[3] local ret = v[4] @@ -2780,7 +2806,9 @@ describe('typval.c', function() {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 'false'}, {lib.VAR_UNKNOWN, nil, 'E908: using an invalid value as a String', ''}, }) do - local tv = typvalt(v[1], v[2]) + -- Using to_cstr in place of Neovim allocated string, cannot + -- tv_clear() that. + local tv = ffi.gc(typvalt(v[1], v[2]), nil) alloc_log:check({}) local emsg = v[3] local ret = v[4] @@ -2821,7 +2849,8 @@ describe('typval.c', function() {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 'false'}, {lib.VAR_UNKNOWN, nil, 'E908: using an invalid value as a String', nil}, }) do - local tv = typvalt(v[1], v[2]) + -- Using to_cstr, cannot free with tv_clear + local tv = ffi.gc(typvalt(v[1], v[2]), nil) alloc_log:check({}) local emsg = v[3] local ret = v[4] @@ -2861,7 +2890,8 @@ describe('typval.c', function() {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 'false'}, {lib.VAR_UNKNOWN, nil, 'E908: using an invalid value as a String', ''}, }) do - local tv = typvalt(v[1], v[2]) + -- Using to_cstr, cannot free with tv_clear + local tv = ffi.gc(typvalt(v[1], v[2]), nil) alloc_log:check({}) local emsg = v[3] local ret = v[4] @@ -2902,7 +2932,8 @@ describe('typval.c', function() {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 'false'}, {lib.VAR_UNKNOWN, nil, 'E908: using an invalid value as a String', nil}, }) do - local tv = typvalt(v[1], v[2]) + -- Using to_cstr, cannot free with tv_clear + local tv = ffi.gc(typvalt(v[1], v[2]), nil) alloc_log:check({}) local emsg = v[3] local ret = v[4] diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index 74f214a231..4b9f185156 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -632,8 +632,9 @@ local function itp_child(wr, func) collectgarbage('stop') child_sethook(wr) local err, emsg = pcall(func) - debug.sethook() collectgarbage('restart') + collectgarbage() + debug.sethook() emsg = tostring(emsg) sc.write(wr, trace_end_msg) if not err then @@ -654,6 +655,7 @@ end local function check_child_err(rd) local trace = {} + local did_traceline = false while true do local traceline = sc.read(rd, hook_msglen) if #traceline ~= hook_msglen then @@ -664,6 +666,7 @@ local function check_child_err(rd) end end if traceline == trace_end_msg then + did_traceline = true break end trace[#trace + 1] = traceline @@ -679,6 +682,13 @@ local function check_child_err(rd) error = error .. trace[i] end end + if not did_traceline then + error = error .. '\nNo end of trace occurred' + end + local cc_err, cc_emsg = pcall(check_cores, Paths.test_luajit_prg, true) + if not cc_err then + error = error .. '\ncheck_cores failed: ' .. cc_emsg + end assert.just_fail(error) end if res == '+\n' then @@ -764,11 +774,6 @@ local module = { child_cleanup_once = child_cleanup_once, sc = sc, } -return function(after_each) - if after_each then - after_each(function() - check_cores(Paths.test_luajit_prg) - end) - end +return function() return module end |