From 030c0588a04c7b201ed21ab6bcbc38c040e3d5b6 Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 31 Mar 2017 10:51:41 +0300 Subject: cmake: Add `clint` target to build Makefile Allows linting only modified files and linting multiple files in parallel. In the current state is rather slow because errors.json is a 6 MiB file and needs to be reparsed each time. Results on my system (6-core): # In build dir, actually parallel make -j5 clint 241.24s user 8.39s system 334% cpu 1:14.74 total # In root, one process make -j5 clint 60.69s user 0.37s system 93% cpu 1:05.19 total In both cases download time included. That is not well for travis (though I would keep travis as-is because new variant will fail before checking all files), but already good enough for regular development: total times are nearly identical and this is the *full* build, further `make -C build clint` will check only modified files. --- src/nvim/CMakeLists.txt | 147 ++++++++++++++++++++++++++++++------------------ 1 file changed, 92 insertions(+), 55 deletions(-) (limited to 'src') diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index a3bacaa9d2..68c0339dc5 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -10,6 +10,7 @@ if(USE_GCOV) endif() endif() +set(TOUCHES_DIR ${PROJECT_BINARY_DIR}/touches) set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto) set(DISPATCH_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gendispatch.lua) file(GLOB API_HEADERS api/*.h) @@ -34,11 +35,16 @@ set(UNICODE_TABLES_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genunicodetables.lua) set(UNICODE_DIR ${PROJECT_SOURCE_DIR}/unicode) file(GLOB UNICODE_FILES ${UNICODE_DIR}/*.txt) set(GENERATED_UNICODE_TABLES ${GENERATED_DIR}/unicode_tables.generated.h) +set(LINT_SUPPRESS_FILE ${PROJECT_BINARY_DIR}/errors.json) +set(LINT_SUPPRESS_URL "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint/errors.json") +set(LINT_PRG ${PROJECT_SOURCE_DIR}/src/clint.py) +set(DOWNLOAD_SCRIPT ${PROJECT_SOURCE_DIR}/cmake/Download.cmake) include_directories(${GENERATED_DIR}) include_directories(${CACHED_GENERATED_DIR}) include_directories(${GENERATED_INCLUDES_DIR}) +file(MAKE_DIRECTORY ${TOUCHES_DIR}) file(MAKE_DIRECTORY ${GENERATED_DIR}) file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}) @@ -73,6 +79,8 @@ file(GLOB UNIT_TEST_FIXTURES ${PROJECT_SOURCE_DIR}/test/unit/fixtures/*.c) list(SORT NVIM_SOURCES) list(SORT NVIM_HEADERS) +list(APPEND LINT_NVIM_SOURCES ${NVIM_SOURCES} ${NVIM_HEADERS}) + foreach(sfile ${NVIM_SOURCES}) get_filename_component(f ${sfile} NAME) if(${f} MATCHES "^(regexp_nfa.c)$") @@ -393,76 +401,82 @@ add_library(nvim-test MODULE EXCLUDE_FROM_ALL ${NVIM_GENERATED_FOR_SOURCES} ${NV target_link_libraries(nvim-test ${NVIM_LINK_LIBRARIES}) set_property(TARGET nvim-test APPEND_STRING PROPERTY COMPILE_FLAGS -DUNIT_TESTING) -set(NO_SINGLE_CHECK_HEADERS - cursor_shape - diff - digraph - ex_cmds - ex_getln - file_search - fold - getchar - hardcopy - if_cscope - if_cscope_defs - mark - mbyte - memfile_defs - memline - memline_defs - menu - misc2 - move - msgpack_rpc/server - ops - option - os/shell - os_unix - os/win_defs - popupmnu - quickfix - regexp - regexp_defs - screen - search - sha256 - sign_defs - spell - spellfile - syntax - syntax_defs - tag - terminal - tui/tui - ugrid - ui - ui_bridge - undo - undo_defs - version - window -) -foreach(hfile ${NVIM_HEADERS}) - get_filename_component(full_d ${hfile} PATH) +function(get_test_target prefix sfile relative_path_var target_var) + get_filename_component(full_d "${sfile}" PATH) file(RELATIVE_PATH d "${PROJECT_SOURCE_DIR}/src/nvim" "${full_d}") if(${d} MATCHES "^[.][.]") file(RELATIVE_PATH d "${GENERATED_DIR}" "${full_d}") endif() - get_filename_component(r ${hfile} NAME_WE) + get_filename_component(r "${sfile}" NAME) if(NOT ${d} EQUAL ".") set(r "${d}/${r}") endif() + string(REGEX REPLACE "[/.]" "-" suffix "${r}") + set(${relative_path_var} ${r} PARENT_SCOPE) + set(${target_var} "${prefix}-${suffix}" PARENT_SCOPE) +endfunction() + +set(NO_SINGLE_CHECK_HEADERS + cursor_shape.h + diff.h + digraph.h + ex_cmds.h + ex_getln.h + file_search.h + fold.h + getchar.h + hardcopy.h + if_cscope.h + if_cscope_defs.h + mark.h + mbyte.h + memfile_defs.h + memline.h + memline_defs.h + menu.h + misc2.h + move.h + msgpack_rpc/server.h + ops.h + option.h + os/shell.h + os_unix.h + os/win_defs.h + popupmnu.h + quickfix.h + regexp.h + regexp_defs.h + screen.h + search.h + sha256.h + sign_defs.h + spell.h + spellfile.h + syntax.h + syntax_defs.h + tag.h + terminal.h + tui/tui.h + ugrid.h + ui.h + ui_bridge.h + undo.h + undo_defs.h + version.h + window.h +) +foreach(hfile ${NVIM_HEADERS}) + get_test_target(test-includes "${hfile}" relative_path texe) if(NOT ${hfile} MATCHES "[.]c[.]h$") - set(tsource "${GENERATED_DIR}/${r}.test-include.c") - string(REPLACE "/" "-" texe "test-incl-${r}") + set(tsource "${GENERATED_DIR}/${relative_path}.test-include.c") write_file("${tsource}" "#include \"${hfile}\"\nint main(int argc, char **argv) { return 0; }") add_executable( ${texe} EXCLUDE_FROM_ALL ${tsource} ${NVIM_HEADERS} ${NVIM_GENERATED_FOR_HEADERS}) - list(FIND NO_SINGLE_CHECK_HEADERS "${r}" hfile_exclude_idx) + list(FIND NO_SINGLE_CHECK_HEADERS "${relative_path}" hfile_exclude_idx) if(${hfile_exclude_idx} EQUAL -1) list(APPEND HEADER_CHECK_TARGETS ${texe}) endif() @@ -470,4 +484,27 @@ foreach(hfile ${NVIM_HEADERS}) endforeach() add_custom_target(check-single-includes DEPENDS ${HEADER_CHECK_TARGETS}) +function(add_download output url) + add_custom_command( + OUTPUT "${output}" + COMMAND ${CMAKE_COMMAND} -DURL=${url} -DFILE=${output} -P ${DOWNLOAD_SCRIPT} + DEPENDS ${DOWNLOAD_SCRIPT} + ) +endfunction() + +add_download(${LINT_SUPPRESS_FILE} ${LINT_SUPPRESS_URL}) + +foreach(sfile ${LINT_NVIM_SOURCES}) + get_test_target("${TOUCHES_DIR}/ran-clint" "${sfile}" r touch_file) + add_custom_command( + OUTPUT ${touch_file} + COMMAND ${LINT_PRG} --suppress-errors=${LINT_SUPPRESS_FILE} src/nvim/${r} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E touch ${touch_file} + DEPENDS ${sfile} ${LINT_SUPPRESS_FILE} + ) + list(APPEND LINT_TARGETS ${touch_file}) +endforeach() +add_custom_target(clint DEPENDS ${LINT_TARGETS}) + add_subdirectory(po) -- cgit From 0b528fc4b55a8875579d0a1a5dde852b6c3f55dc Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 31 Mar 2017 11:38:08 +0300 Subject: cmake: Use file-specific supprresses `make -C build clint` time is now make -j5 clint 95.29s user 1.86s system 409% cpu 23.751 total *without* downloading anything (much worse if something was not cached, still a bit better then top-level `make clint`). But since without neovim/bot-ci#95 it is downloading each file one-by-one total time with download (download also parallel!) is make -j5 -B clint 99.29s user 2.98s system 258% cpu 39.634 total Top-level makefile still gives make -j5 clint 59.33s user 0.28s system 95% cpu 1:02.41 total --- src/nvim/CMakeLists.txt | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index 68c0339dc5..b4bc3afc05 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -36,9 +36,12 @@ set(UNICODE_DIR ${PROJECT_SOURCE_DIR}/unicode) file(GLOB UNICODE_FILES ${UNICODE_DIR}/*.txt) set(GENERATED_UNICODE_TABLES ${GENERATED_DIR}/unicode_tables.generated.h) set(LINT_SUPPRESS_FILE ${PROJECT_BINARY_DIR}/errors.json) -set(LINT_SUPPRESS_URL "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint/errors.json") +set(LINT_SUPPRESS_URL_BASE "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint") +set(LINT_SUPPRESS_URL "${LINT_SUPPRESS_URL_BASE}/errors.json") set(LINT_PRG ${PROJECT_SOURCE_DIR}/src/clint.py) set(DOWNLOAD_SCRIPT ${PROJECT_SOURCE_DIR}/cmake/Download.cmake) +set(LINT_SUPPRESSES_ROOT ${PROJECT_BINARY_DIR}/errors) +set(LINT_SUPPRESSES_URL "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint/errors.tar.gz") include_directories(${GENERATED_DIR}) include_directories(${CACHED_GENERATED_DIR}) @@ -404,16 +407,20 @@ set_property(TARGET nvim-test APPEND_STRING PROPERTY COMPILE_FLAGS -DUNIT_TESTIN function(get_test_target prefix sfile relative_path_var target_var) get_filename_component(full_d "${sfile}" PATH) file(RELATIVE_PATH d "${PROJECT_SOURCE_DIR}/src/nvim" "${full_d}") - if(${d} MATCHES "^[.][.]") + if(d MATCHES "^[.][.]") file(RELATIVE_PATH d "${GENERATED_DIR}" "${full_d}") endif() get_filename_component(r "${sfile}" NAME) - if(NOT ${d} EQUAL ".") + if(NOT d MATCHES "^[.]?$") set(r "${d}/${r}") endif() string(REGEX REPLACE "[/.]" "-" suffix "${r}") set(${relative_path_var} ${r} PARENT_SCOPE) - set(${target_var} "${prefix}-${suffix}" PARENT_SCOPE) + if(prefix STREQUAL "") + set(${target_var} "${suffix}" PARENT_SCOPE) + else() + set(${target_var} "${prefix}-${suffix}" PARENT_SCOPE) + endif() endfunction() set(NO_SINGLE_CHECK_HEADERS @@ -495,13 +502,17 @@ endfunction() add_download(${LINT_SUPPRESS_FILE} ${LINT_SUPPRESS_URL}) foreach(sfile ${LINT_NVIM_SOURCES}) - get_test_target("${TOUCHES_DIR}/ran-clint" "${sfile}" r touch_file) + get_test_target("" "${sfile}" r suffix) + set(suppress_file ${LINT_SUPPRESSES_ROOT}/${suffix}.json) + set(suppress_url "${LINT_SUPPRESS_URL_BASE}/${suffix}.json") + add_download(${suppress_file} ${suppress_url}) + set(touch_file "${TOUCHES_DIR}/ran-clint-${suffix}") add_custom_command( OUTPUT ${touch_file} - COMMAND ${LINT_PRG} --suppress-errors=${LINT_SUPPRESS_FILE} src/nvim/${r} + COMMAND ${LINT_PRG} --suppress-errors=${suppress_file} src/nvim/${r} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${CMAKE_COMMAND} -E touch ${touch_file} - DEPENDS ${sfile} ${LINT_SUPPRESS_FILE} + DEPENDS ${sfile} ${suppress_file} ) list(APPEND LINT_TARGETS ${touch_file}) endforeach() -- cgit From 24fd125893203c667789cb574b289fc1c9eaf6bc Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 31 Mar 2017 11:49:18 +0300 Subject: cmake: Allow failing to download small suppress files --- src/nvim/CMakeLists.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index b4bc3afc05..f5f4173879 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -491,21 +491,25 @@ foreach(hfile ${NVIM_HEADERS}) endforeach() add_custom_target(check-single-includes DEPENDS ${HEADER_CHECK_TARGETS}) -function(add_download output url) +function(add_download output url allow_failure) add_custom_command( OUTPUT "${output}" - COMMAND ${CMAKE_COMMAND} -DURL=${url} -DFILE=${output} -P ${DOWNLOAD_SCRIPT} + COMMAND + ${CMAKE_COMMAND} + -DURL=${url} -DFILE=${output} + -DALLOW_FAILURE=${allow_failure} + -P ${DOWNLOAD_SCRIPT} DEPENDS ${DOWNLOAD_SCRIPT} ) endfunction() -add_download(${LINT_SUPPRESS_FILE} ${LINT_SUPPRESS_URL}) +add_download(${LINT_SUPPRESS_FILE} ${LINT_SUPPRESS_URL} off) foreach(sfile ${LINT_NVIM_SOURCES}) get_test_target("" "${sfile}" r suffix) set(suppress_file ${LINT_SUPPRESSES_ROOT}/${suffix}.json) set(suppress_url "${LINT_SUPPRESS_URL_BASE}/${suffix}.json") - add_download(${suppress_file} ${suppress_url}) + add_download(${suppress_file} ${suppress_url} on) set(touch_file "${TOUCHES_DIR}/ran-clint-${suffix}") add_custom_command( OUTPUT ${touch_file} -- cgit From c61858a9978504c645f09de60daf4f9786c2220e Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 31 Mar 2017 12:02:59 +0300 Subject: cmake: Replace RunLint.cmake with code in src/nvim/CMakeLists.txt This also removes LINT_FILE environment variable, other then that functionality is kept. It is expected that developers needing partial linting will use `make lint`, touching interesting file before (if not done already by writing to them). --- src/nvim/CMakeLists.txt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index f5f4173879..b19bdf2326 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -505,21 +505,32 @@ endfunction() add_download(${LINT_SUPPRESS_FILE} ${LINT_SUPPRESS_URL} off) +set(LINT_NVIM_REL_SOURCES) foreach(sfile ${LINT_NVIM_SOURCES}) get_test_target("" "${sfile}" r suffix) set(suppress_file ${LINT_SUPPRESSES_ROOT}/${suffix}.json) set(suppress_url "${LINT_SUPPRESS_URL_BASE}/${suffix}.json") + set(rsfile src/nvim/${r}) add_download(${suppress_file} ${suppress_url} on) set(touch_file "${TOUCHES_DIR}/ran-clint-${suffix}") add_custom_command( OUTPUT ${touch_file} - COMMAND ${LINT_PRG} --suppress-errors=${suppress_file} src/nvim/${r} + COMMAND ${LINT_PRG} --suppress-errors=${suppress_file} ${rsfile} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${CMAKE_COMMAND} -E touch ${touch_file} DEPENDS ${sfile} ${suppress_file} ) list(APPEND LINT_TARGETS ${touch_file}) + list(APPEND LINT_NVIM_REL_SOURCES ${rsfile}) endforeach() add_custom_target(clint DEPENDS ${LINT_TARGETS}) +add_custom_target( + clint-full + COMMAND + ${LINT_PRG} --suppress-errors=${LINT_SUPPRESS_FILE} ${LINT_NVIM_REL_SOURCES} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + DEPENDS ${LINT_NVIM_SOURCES} ${LINT_SUPPRESS_FILE} +) + add_subdirectory(po) -- cgit From 4fc2be490cd075d08ef8532f954424051fe49300 Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 31 Mar 2017 12:27:13 +0300 Subject: clint: Do not report zero errors --- src/clint.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/clint.py b/src/clint.py index 61c53d128e..5174521fb8 100755 --- a/src/clint.py +++ b/src/clint.py @@ -571,7 +571,8 @@ class _CppLintState(object): for category, count in self.errors_by_category.items(): sys.stderr.write('Category \'%s\' errors found: %d\n' % (category, count)) - sys.stderr.write('Total errors found: %d\n' % self.error_count) + if self.error_count: + sys.stderr.write('Total errors found: %d\n' % self.error_count) def SuppressErrorsFrom(self, fname): """Open file and read a list of suppressed errors from it""" -- cgit From 4d0f90f94db6040841b9987d02a021b785cfe0f3 Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 31 Mar 2017 12:28:10 +0300 Subject: cmake: Also depend on LINT_PRG --- src/nvim/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index b19bdf2326..5a5ebc4415 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -518,7 +518,7 @@ foreach(sfile ${LINT_NVIM_SOURCES}) COMMAND ${LINT_PRG} --suppress-errors=${suppress_file} ${rsfile} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${CMAKE_COMMAND} -E touch ${touch_file} - DEPENDS ${sfile} ${suppress_file} + DEPENDS ${LINT_PRG} ${sfile} ${suppress_file} ) list(APPEND LINT_TARGETS ${touch_file}) list(APPEND LINT_NVIM_REL_SOURCES ${rsfile}) @@ -530,7 +530,7 @@ add_custom_target( COMMAND ${LINT_PRG} --suppress-errors=${LINT_SUPPRESS_FILE} ${LINT_NVIM_REL_SOURCES} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - DEPENDS ${LINT_NVIM_SOURCES} ${LINT_SUPPRESS_FILE} + DEPENDS ${LINT_PRG} ${LINT_NVIM_SOURCES} ${LINT_SUPPRESS_FILE} ) add_subdirectory(po) -- cgit