From 410d18ef5c24a325f4d63c9b01991015199194c2 Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 3 Jan 2017 08:46:44 +0300 Subject: unittest: Allow multiple indirect includes Works by saving all preprocessor defines and reusing them on each run. This also saves NVIM_HEADER_H defines. Saving other defines is needed for defines like `Map(foo, bar)` which are sometimes used to declare types or functions. Saving types or function declarations is not needed because they are recorded as luajit state. Fixes #5857 --- test/unit/preprocess.lua | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) (limited to 'test/unit/preprocess.lua') diff --git a/test/unit/preprocess.lua b/test/unit/preprocess.lua index 10ba997758..99537cd66a 100644 --- a/test/unit/preprocess.lua +++ b/test/unit/preprocess.lua @@ -1,8 +1,12 @@ -- helps managing loading different headers into the LuaJIT ffi. Untested on -- windows, will probably need quite a bit of adjustment to run there. +local global_helpers = require('test.helpers') + local ffi = require("ffi") +local tmpname = global_helpers.tmpname + local ccs = {} local env_cc = os.getenv("CC") @@ -61,12 +65,12 @@ end -- will produce a string that represents a meta C header file that includes -- all the passed in headers. I.e.: -- --- headerize({"stdio.h", "math.h", true} +-- headerize({"stdio.h", "math.h"}, true) -- produces: -- #include -- #include -- --- headerize({"vim.h", "memory.h", false} +-- headerize({"vim.h", "memory.h"}, false) -- produces: -- #include "vim.h" -- #include "memory.h" @@ -79,8 +83,7 @@ local function headerize(headers, global) end local formatted = {} - for i = 1, #headers do - local hdr = headers[i] + for i, hdr in ipairs(headers) do formatted[#formatted + 1] = "#include " .. tostring(pre) .. tostring(hdr) .. @@ -111,7 +114,8 @@ local Gcc = { '-D "_Nullable="', '-D "_Nonnull="', '-U__BLOCKS__', - } + }, + added_header_defines = '', } function Gcc:new(obj) @@ -145,21 +149,40 @@ end -- returns a stream representing a preprocessed form of the passed-in headers. -- Don't forget to close the stream by calling the close() method on it. -function Gcc:preprocess_stream(...) +function Gcc:preprocess(...) -- create pseudo-header local pseudoheader = headerize({...}, false) + local pseudoheader_fname = 'tmp_pseudoheader.h' + local pseudoheader_file = io.open(pseudoheader_fname, 'w') + pseudoheader_file:write(self.added_header_defines) + pseudoheader_file:write("\n") + pseudoheader_file:write(pseudoheader) + pseudoheader_file:flush() + pseudoheader_file:close() local defines = table.concat(self.preprocessor_extra_flags, ' ') local cmd = ("echo $hdr | " .. tostring(self.path) .. " " .. tostring(defines) .. - " -std=c99 -P -E -"):gsub('$hdr', shell_quote(pseudoheader)) + " -std=c99 -P -E " .. shell_quote(pseudoheader_fname)) + local def_cmd = ("echo $hdr | " .. + tostring(self.path) .. + " " .. + tostring(defines) .. + " -std=c99 -dM -E " .. shell_quote(pseudoheader_fname)) + local def_stream = io.popen(def_cmd) + self.added_header_defines = def_stream:read('*a') + def_stream:close() -- lfs = require("lfs") -- print("CWD: #{lfs.currentdir!}") -- print("CMD: #{cmd}") -- io.stderr\write("CWD: #{lfs.currentdir!}\n") -- io.stderr\write("CMD: #{cmd}\n") - return io.popen(cmd) + local stream = io.popen(cmd) + local ret = stream:read('*a') + stream:close() + os.remove(pseudoheader_fname) + return ret end local Clang = Gcc:new() @@ -197,8 +220,8 @@ return { includes = function(hdr) return cc:dependencies(hdr) end, - preprocess_stream = function(...) - return cc:preprocess_stream(...) + preprocess = function(...) + return cc:preprocess(...) end, add_to_include_path = function(...) return cc:add_to_include_path(...) -- cgit From 0d7b779cab198c89b70bf9d1e6d42cffc3f28f50 Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 3 Jan 2017 08:57:43 +0300 Subject: unittest: Record previous defines in another place Previous commit made preprocess.lua know how its output will be used. This moves state to cimport, making only it know which is cleaner. --- test/unit/preprocess.lua | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'test/unit/preprocess.lua') diff --git a/test/unit/preprocess.lua b/test/unit/preprocess.lua index 99537cd66a..062432323f 100644 --- a/test/unit/preprocess.lua +++ b/test/unit/preprocess.lua @@ -115,7 +115,6 @@ local Gcc = { '-D "_Nonnull="', '-U__BLOCKS__', }, - added_header_defines = '', } function Gcc:new(obj) @@ -149,12 +148,12 @@ end -- returns a stream representing a preprocessed form of the passed-in headers. -- Don't forget to close the stream by calling the close() method on it. -function Gcc:preprocess(...) +function Gcc:preprocess(previous_defines, ...) -- create pseudo-header local pseudoheader = headerize({...}, false) local pseudoheader_fname = 'tmp_pseudoheader.h' local pseudoheader_file = io.open(pseudoheader_fname, 'w') - pseudoheader_file:write(self.added_header_defines) + pseudoheader_file:write(previous_defines) pseudoheader_file:write("\n") pseudoheader_file:write(pseudoheader) pseudoheader_file:flush() @@ -171,7 +170,7 @@ function Gcc:preprocess(...) tostring(defines) .. " -std=c99 -dM -E " .. shell_quote(pseudoheader_fname)) local def_stream = io.popen(def_cmd) - self.added_header_defines = def_stream:read('*a') + local defines = def_stream:read('*a') def_stream:close() -- lfs = require("lfs") -- print("CWD: #{lfs.currentdir!}") @@ -179,10 +178,10 @@ function Gcc:preprocess(...) -- io.stderr\write("CWD: #{lfs.currentdir!}\n") -- io.stderr\write("CMD: #{cmd}\n") local stream = io.popen(cmd) - local ret = stream:read('*a') + local declarations = stream:read('*a') stream:close() os.remove(pseudoheader_fname) - return ret + return declarations, defines end local Clang = Gcc:new() -- cgit From 2151ddbd73a78fcecc992f58f036b8f765a9113e Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 3 Jan 2017 16:25:04 +0300 Subject: unittest: Move nil checks to Gcc:preprocess --- test/unit/preprocess.lua | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) (limited to 'test/unit/preprocess.lua') diff --git a/test/unit/preprocess.lua b/test/unit/preprocess.lua index 062432323f..236cc58192 100644 --- a/test/unit/preprocess.lua +++ b/test/unit/preprocess.lua @@ -146,6 +146,19 @@ function Gcc:dependencies(hdr) end end +local function repeated_call(cmd) + for _ = 1, 10 do + local stream = io.popen(cmd) + local ret = stream:read('*a') + stream:close() + if ret then + return ret + end + end + print('ERROR: preprocess.lua: Failed to execute ' .. cmd .. ': nil return after 10 attempts') + return nil +end + -- returns a stream representing a preprocessed form of the passed-in headers. -- Don't forget to close the stream by calling the close() method on it. function Gcc:preprocess(previous_defines, ...) @@ -159,28 +172,22 @@ function Gcc:preprocess(previous_defines, ...) pseudoheader_file:flush() pseudoheader_file:close() local defines = table.concat(self.preprocessor_extra_flags, ' ') - local cmd = ("echo $hdr | " .. - tostring(self.path) .. - " " .. - tostring(defines) .. - " -std=c99 -P -E " .. shell_quote(pseudoheader_fname)) - local def_cmd = ("echo $hdr | " .. - tostring(self.path) .. - " " .. - tostring(defines) .. - " -std=c99 -dM -E " .. shell_quote(pseudoheader_fname)) - local def_stream = io.popen(def_cmd) - local defines = def_stream:read('*a') - def_stream:close() + local cmd_base = self.path .. " " .. defines .. " -std=c99" + + local def_cmd = (cmd_base .. " -dM -E " .. shell_quote(pseudoheader_fname)) + local defines = repeated_call(def_cmd) + -- lfs = require("lfs") -- print("CWD: #{lfs.currentdir!}") -- print("CMD: #{cmd}") -- io.stderr\write("CWD: #{lfs.currentdir!}\n") -- io.stderr\write("CMD: #{cmd}\n") - local stream = io.popen(cmd) - local declarations = stream:read('*a') - stream:close() + local decl_cmd = (cmd_base .. " -P -E " .. shell_quote(pseudoheader_fname)) + local declarations = repeated_call(decl_cmd) + os.remove(pseudoheader_fname) + + assert(declarations and defines) return declarations, defines end -- cgit From b38e7254280ea87ab7f5757709a22aa42da16b03 Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 3 Jan 2017 16:56:35 +0300 Subject: unittest: Refactor preprocess.lua Keeps arguments separated and not joined as a single string as long as possible. Abstracts away additional arguments so that Gcc:preprocess should work for compilers with different conventions should they be supported. --- test/unit/preprocess.lua | 117 ++++++++++++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 43 deletions(-) (limited to 'test/unit/preprocess.lua') diff --git a/test/unit/preprocess.lua b/test/unit/preprocess.lua index 236cc58192..a2c6b1712a 100644 --- a/test/unit/preprocess.lua +++ b/test/unit/preprocess.lua @@ -11,22 +11,22 @@ local ccs = {} local env_cc = os.getenv("CC") if env_cc then - table.insert(ccs, {path = "/usr/bin/env " .. tostring(env_cc), type = "gcc"}) + table.insert(ccs, {path = {"/usr/bin/env", env_cc}, type = "gcc"}) end if ffi.os == "Windows" then - table.insert(ccs, {path = "cl", type = "msvc"}) + table.insert(ccs, {path = {"cl"}, type = "msvc"}) end -table.insert(ccs, {path = "/usr/bin/env cc", type = "gcc"}) -table.insert(ccs, {path = "/usr/bin/env gcc", type = "gcc"}) -table.insert(ccs, {path = "/usr/bin/env gcc-4.9", type = "gcc"}) -table.insert(ccs, {path = "/usr/bin/env gcc-4.8", type = "gcc"}) -table.insert(ccs, {path = "/usr/bin/env gcc-4.7", type = "gcc"}) -table.insert(ccs, {path = "/usr/bin/env clang", type = "clang"}) -table.insert(ccs, {path = "/usr/bin/env icc", type = "gcc"}) +table.insert(ccs, {path = {"/usr/bin/env", "cc"}, type = "gcc"}) +table.insert(ccs, {path = {"/usr/bin/env", "gcc"}, type = "gcc"}) +table.insert(ccs, {path = {"/usr/bin/env", "gcc-4.9"}, type = "gcc"}) +table.insert(ccs, {path = {"/usr/bin/env", "gcc-4.8"}, type = "gcc"}) +table.insert(ccs, {path = {"/usr/bin/env", "gcc-4.7"}, type = "gcc"}) +table.insert(ccs, {path = {"/usr/bin/env", "clang"}, type = "clang"}) +table.insert(ccs, {path = {"/usr/bin/env", "icc"}, type = "gcc"}) -local quote_me = '[^%w%+%-%=%@%_%/]' -- complement (needn't quote) +local quote_me = '[^.%w%+%-%@%_%/]' -- complement (needn't quote) local function shell_quote(str) if string.find(str, quote_me) or str == '' then return "'" .. string.gsub(str, "'", [['"'"']]) .. "'" @@ -94,49 +94,78 @@ local function headerize(headers, global) end local Gcc = { + preprocessor_extra_flags = {}, + get_defines_extra_flags = {'-std=c99', '-dM', '-E'}, + get_declarations_extra_flags = {'-std=c99', '-P', '-E'}, +} + +function Gcc:define(name, args, val) + local define = '-D' .. name + local quoted_define = '' + if args ~= nil then + define = define .. '(' .. table.concat(args, ',') .. ')' + end + if val ~= nil then + define = define .. '=' .. val + end + self.preprocessor_extra_flags[#self.preprocessor_extra_flags + 1] = define +end + +function Gcc:undefine(name) + self.preprocessor_extra_flags[#self.preprocessor_extra_flags + 1] = ( + '-U' .. name) +end + +function Gcc:init_defines(name) -- preprocessor flags that will hopefully make the compiler produce C -- declarations that the LuaJIT ffi understands. - preprocessor_extra_flags = { - '-D "aligned(ARGS)="', - '-D "__attribute__(ARGS)="', - '-D "__asm(ARGS)="', - '-D "__asm__(ARGS)="', - '-D "__inline__="', - '-D "EXTERN=extern"', - '-D "INIT(...)="', - '-D_GNU_SOURCE', - '-DINCLUDE_GENERATED_DECLARATIONS', - - -- Needed for FreeBSD - '-D "_Thread_local="', - - -- Needed for macOS Sierra - '-D "_Nullable="', - '-D "_Nonnull="', - '-U__BLOCKS__', - }, -} + self:define('aligned', {'ARGS'}, '') + self:define('__attribute__', {'ARGS'}, '') + self:define('__asm', {'ARGS'}, '') + self:define('__asm__', {'ARGS'}, '') + self:define('__inline__', nil, '') + self:define('EXTERN', nil, 'extern') + self:define('INIT', {'...'}, '') + self:define('_GNU_SOURCE') + self:define('INCLUDE_GENERATED_DECLARATIONS') + -- Needed for FreeBSD + self:define('_Thread_local', nil, '') + -- Needed for macOS Sierra + self:define('_Nullable', nil, '') + self:define('_Nonnull', nil, '') + self:undefine('__BLOCKS__') +end function Gcc:new(obj) obj = obj or {} setmetatable(obj, self) self.__index = self + self:init_defines() return obj end function Gcc:add_to_include_path(...) - local paths = {...} - for i = 1, #paths do - local path = paths[i] - local directive = '-I ' .. '"' .. path .. '"' + for i = 1, select('#', ...) do + local path = select(i, ...) local ef = self.preprocessor_extra_flags - ef[#ef + 1] = directive + ef[#ef + 1] = '-I' .. path end end +local function argss_to_cmd(...) + local cmd = '' + for i = 1, select('#', ...) do + for _, arg in ipairs(select(i, ...)) do + cmd = cmd .. ' ' .. shell_quote(arg) + end + end + return cmd +end + -- returns a list of the headers files upon which this file relies function Gcc:dependencies(hdr) - local out = io.popen(tostring(self.path) .. " -M " .. tostring(hdr) .. " 2>&1") + local cmd = argss_to_cmd(self.path, {'-M', hdr}) .. ' 2>&1' + local out = io.popen(cmd) local deps = out:read("*a") out:close() if deps then @@ -146,7 +175,8 @@ function Gcc:dependencies(hdr) end end -local function repeated_call(cmd) +local function repeated_call(...) + local cmd = argss_to_cmd(...) for _ = 1, 10 do local stream = io.popen(cmd) local ret = stream:read('*a') @@ -171,19 +201,20 @@ function Gcc:preprocess(previous_defines, ...) pseudoheader_file:write(pseudoheader) pseudoheader_file:flush() pseudoheader_file:close() - local defines = table.concat(self.preprocessor_extra_flags, ' ') - local cmd_base = self.path .. " " .. defines .. " -std=c99" - local def_cmd = (cmd_base .. " -dM -E " .. shell_quote(pseudoheader_fname)) - local defines = repeated_call(def_cmd) + local defines = repeated_call(self.path, self.preprocessor_extra_flags, + self.get_defines_extra_flags, + {pseudoheader_fname}) -- lfs = require("lfs") -- print("CWD: #{lfs.currentdir!}") -- print("CMD: #{cmd}") -- io.stderr\write("CWD: #{lfs.currentdir!}\n") -- io.stderr\write("CMD: #{cmd}\n") - local decl_cmd = (cmd_base .. " -P -E " .. shell_quote(pseudoheader_fname)) - local declarations = repeated_call(decl_cmd) + + local declarations = repeated_call(self.path, self.preprocessor_extra_flags, + self.get_declarations_extra_flags, + {pseudoheader_fname}) os.remove(pseudoheader_fname) -- cgit From 937b6fac8fecb701895b2c4151a864861aaf03f4 Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 3 Jan 2017 17:37:18 +0300 Subject: unittest: Fix linter errors --- test/unit/preprocess.lua | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'test/unit/preprocess.lua') diff --git a/test/unit/preprocess.lua b/test/unit/preprocess.lua index a2c6b1712a..8c2a5c73e5 100644 --- a/test/unit/preprocess.lua +++ b/test/unit/preprocess.lua @@ -1,12 +1,8 @@ -- helps managing loading different headers into the LuaJIT ffi. Untested on -- windows, will probably need quite a bit of adjustment to run there. -local global_helpers = require('test.helpers') - local ffi = require("ffi") -local tmpname = global_helpers.tmpname - local ccs = {} local env_cc = os.getenv("CC") @@ -83,7 +79,7 @@ local function headerize(headers, global) end local formatted = {} - for i, hdr in ipairs(headers) do + for _, hdr in ipairs(headers) do formatted[#formatted + 1] = "#include " .. tostring(pre) .. tostring(hdr) .. @@ -101,7 +97,6 @@ local Gcc = { function Gcc:define(name, args, val) local define = '-D' .. name - local quoted_define = '' if args ~= nil then define = define .. '(' .. table.concat(args, ',') .. ')' end @@ -116,7 +111,7 @@ function Gcc:undefine(name) '-U' .. name) end -function Gcc:init_defines(name) +function Gcc:init_defines() -- preprocessor flags that will hopefully make the compiler produce C -- declarations that the LuaJIT ffi understands. self:define('aligned', {'ARGS'}, '') -- cgit