local helpers = require('test.unit.helpers')(after_each) local global_helpers = require('test.helpers') local itp = helpers.gen_itp(it) local viml_helpers = require('test.unit.viml.helpers') local child_call_once = helpers.child_call_once local conv_enum = helpers.conv_enum local cimport = helpers.cimport local ffi = helpers.ffi local eq = helpers.eq local conv_ccs = viml_helpers.conv_ccs local new_pstate = viml_helpers.new_pstate local conv_cmp_type = viml_helpers.conv_cmp_type local pstate_set_str = viml_helpers.pstate_set_str local conv_expr_asgn_type = viml_helpers.conv_expr_asgn_type local shallowcopy = global_helpers.shallowcopy local intchar2lua = global_helpers.intchar2lua local lib = cimport('./src/nvim/viml/parser/expressions.h') local eltkn_type_tab, eltkn_mul_type_tab, eltkn_opt_scope_tab child_call_once(function() eltkn_type_tab = { [tonumber(lib.kExprLexInvalid)] = 'Invalid', [tonumber(lib.kExprLexMissing)] = 'Missing', [tonumber(lib.kExprLexSpacing)] = 'Spacing', [tonumber(lib.kExprLexEOC)] = 'EOC', [tonumber(lib.kExprLexQuestion)] = 'Question', [tonumber(lib.kExprLexColon)] = 'Colon', [tonumber(lib.kExprLexOr)] = 'Or', [tonumber(lib.kExprLexAnd)] = 'And', [tonumber(lib.kExprLexComparison)] = 'Comparison', [tonumber(lib.kExprLexPlus)] = 'Plus', [tonumber(lib.kExprLexMinus)] = 'Minus', [tonumber(lib.kExprLexDot)] = 'Dot', [tonumber(lib.kExprLexMultiplication)] = 'Multiplication', [tonumber(lib.kExprLexNot)] = 'Not', [tonumber(lib.kExprLexNumber)] = 'Number', [tonumber(lib.kExprLexSingleQuotedString)] = 'SingleQuotedString', [tonumber(lib.kExprLexDoubleQuotedString)] = 'DoubleQuotedString', [tonumber(lib.kExprLexOption)] = 'Option', [tonumber(lib.kExprLexRegister)] = 'Register', [tonumber(lib.kExprLexEnv)] = 'Env', [tonumber(lib.kExprLexPlainIdentifier)] = 'PlainIdentifier', [tonumber(lib.kExprLexBracket)] = 'Bracket', [tonumber(lib.kExprLexFigureBrace)] = 'FigureBrace', [tonumber(lib.kExprLexParenthesis)] = 'Parenthesis', [tonumber(lib.kExprLexComma)] = 'Comma', [tonumber(lib.kExprLexArrow)] = 'Arrow', [tonumber(lib.kExprLexAssignment)] = 'Assignment', } eltkn_mul_type_tab = { [tonumber(lib.kExprLexMulMul)] = 'Mul', [tonumber(lib.kExprLexMulDiv)] = 'Div', [tonumber(lib.kExprLexMulMod)] = 'Mod', } eltkn_opt_scope_tab = { [tonumber(lib.kExprOptScopeUnspecified)] = 'Unspecified', [tonumber(lib.kExprOptScopeGlobal)] = 'Global', [tonumber(lib.kExprOptScopeLocal)] = 'Local', } end) local function conv_eltkn_type(typ) return conv_enum(eltkn_type_tab, typ) end local bracket_types = { Bracket = true, FigureBrace = true, Parenthesis = true, } local function eltkn2lua(pstate, tkn) local ret = { type = conv_eltkn_type(tkn.type), } pstate_set_str(pstate, tkn.start, tkn.len, ret) if not ret.error and (#(ret.str) ~= ret.len) then ret.error = '#str /= len' end if ret.type == 'Comparison' then ret.data = { type = conv_cmp_type(tkn.data.cmp.type), ccs = conv_ccs(tkn.data.cmp.ccs), inv = (not not tkn.data.cmp.inv), } elseif ret.type == 'Multiplication' then ret.data = { type = conv_enum(eltkn_mul_type_tab, tkn.data.mul.type) } elseif bracket_types[ret.type] then ret.data = { closing = (not not tkn.data.brc.closing) } elseif ret.type == 'Register' then ret.data = { name = intchar2lua(tkn.data.reg.name) } elseif (ret.type == 'SingleQuotedString' or ret.type == 'DoubleQuotedString') then ret.data = { closed = (not not tkn.data.str.closed) } elseif ret.type == 'Option' then ret.data = { scope = conv_enum(eltkn_opt_scope_tab, tkn.data.opt.scope), name = ffi.string(tkn.data.opt.name, tkn.data.opt.len), } elseif ret.type == 'PlainIdentifier' then ret.data = { scope = intchar2lua(tkn.data.var.scope), autoload = (not not tkn.data.var.autoload), } elseif ret.type == 'Number' then ret.data = { is_float = (not not tkn.data.num.is_float), base = tonumber(tkn.data.num.base), } ret.data.val = tonumber(tkn.data.num.is_float and tkn.data.num.val.floating or tkn.data.num.val.integer) elseif ret.type == 'Assignment' then ret.data = { type = conv_expr_asgn_type(tkn.data.ass.type) } elseif ret.type == 'Invalid' then ret.data = { error = ffi.string(tkn.data.err.msg) } end return ret, tkn end local function next_eltkn(pstate, flags) return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, flags)) end describe('Expressions lexer', function() local flags = 0 local should_advance = true local function check_advance(pstate, bytes_to_advance, initial_col) local tgt = initial_col + bytes_to_advance if should_advance then if pstate.reader.lines.items[0].size == tgt then eq(1, pstate.pos.line) eq(0, pstate.pos.col) else eq(0, pstate.pos.line) eq(tgt, pstate.pos.col) end else eq(0, pstate.pos.line) eq(initial_col, pstate.pos.col) end end local function singl_eltkn_test(typ, str, data) local pstate = new_pstate({str}) eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, next_eltkn(pstate, flags)) check_advance(pstate, #str, 0) if not ( typ == 'Spacing' or (typ == 'Register' and str == '@') or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString') and not data.closed) ) then pstate = new_pstate({str .. ' '}) eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, next_eltkn(pstate, flags)) check_advance(pstate, #str, 0) end pstate = new_pstate({'x' .. str}) pstate.pos.col = 1 eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ}, next_eltkn(pstate, flags)) check_advance(pstate, #str, 1) end local function scope_test(scope) singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope}) singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope}) end local function comparison_test(op, inv_op, cmp_type) singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'}) singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'}) singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'}) singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'}) singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'}) singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'}) end local function simple_test(pstate_arg, exp_type, exp_len, exp) local pstate = new_pstate(pstate_arg) exp = shallowcopy(exp) exp.type = exp_type exp.len = exp_len or #(pstate_arg[0]) exp.start = { col = 0, line = 0 } eq(exp, next_eltkn(pstate, flags)) end local function stable_tests() singl_eltkn_test('Parenthesis', '(', {closing=false}) singl_eltkn_test('Parenthesis', ')', {closing=true}) singl_eltkn_test('Bracket', '[', {closing=false}) singl_eltkn_test('Bracket', ']', {closing=true}) singl_eltkn_test('FigureBrace', '{', {closing=false}) singl_eltkn_test('FigureBrace', '}', {closing=true}) singl_eltkn_test('Question', '?') singl_eltkn_test('Colon', ':') singl_eltkn_test('Dot', '.') singl_eltkn_test('Assignment', '.=', {type='Concat'}) singl_eltkn_test('Plus', '+') singl_eltkn_test('Assignment', '+=', {type='Add'}) singl_eltkn_test('Comma', ',') singl_eltkn_test('Multiplication', '*', {type='Mul'}) singl_eltkn_test('Multiplication', '/', {type='Div'}) singl_eltkn_test('Multiplication', '%', {type='Mod'}) singl_eltkn_test('Spacing', ' \t\t \t\t') singl_eltkn_test('Spacing', ' ') singl_eltkn_test('Spacing', '\t') singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'}) singl_eltkn_test('Number', '0123', {is_float=false, base=8, val=83}) singl_eltkn_test('Number', '01234567', {is_float=false, base=8, val=342391}) singl_eltkn_test('Number', '012345678', {is_float=false, base=10, val=12345678}) singl_eltkn_test('Number', '0x123', {is_float=false, base=16, val=291}) singl_eltkn_test('Number', '0x56FF', {is_float=false, base=16, val=22271}) singl_eltkn_test('Number', '0xabcdef', {is_float=false, base=16, val=11259375}) singl_eltkn_test('Number', '0xABCDEF', {is_float=false, base=16, val=11259375}) singl_eltkn_test('Number', '0x0', {is_float=false, base=16, val=0}) singl_eltkn_test('Number', '00', {is_float=false, base=8, val=0}) singl_eltkn_test('Number', '0b0', {is_float=false, base=2, val=0}) singl_eltkn_test('Number', '0b010111', {is_float=false, base=2, val=23}) singl_eltkn_test('Number', '0b100111', {is_float=false, base=2, val=39}) singl_eltkn_test('Number', '0', {is_float=false, base=10, val=0}) singl_eltkn_test('Number', '9', {is_float=false, base=10, val=9}) singl_eltkn_test('Env', '$abc') singl_eltkn_test('Env', '$') singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0}) singl_eltkn_test('PlainIdentifier', '_test', {autoload=false, scope=0}) singl_eltkn_test('PlainIdentifier', '_test_foo', {autoload=false, scope=0}) singl_eltkn_test('PlainIdentifier', 't', {autoload=false, scope=0}) singl_eltkn_test('PlainIdentifier', 'test5', {autoload=false, scope=0}) singl_eltkn_test('PlainIdentifier', 't0', {autoload=false, scope=0}) singl_eltkn_test('PlainIdentifier', 'test#var', {autoload=true, scope=0}) singl_eltkn_test('PlainIdentifier', 'test#var#val###', {autoload=true, scope=0}) singl_eltkn_test('PlainIdentifier', 't#####', {autoload=true, scope=0}) singl_eltkn_test('And', '&&') singl_eltkn_test('Or', '||') singl_eltkn_test('Invalid', '&', {error='E112: Option name missing: %.*s'}) singl_eltkn_test('Option', '&opt', {scope='Unspecified', name='opt'}) singl_eltkn_test('Option', '&t_xx', {scope='Unspecified', name='t_xx'}) singl_eltkn_test('Option', '&t_\r\r', {scope='Unspecified', name='t_\r\r'}) singl_eltkn_test('Option', '&t_\t\t', {scope='Unspecified', name='t_\t\t'}) singl_eltkn_test('Option', '&t_ ', {scope='Unspecified', name='t_ '}) singl_eltkn_test('Option', '&g:opt', {scope='Global', name='opt'}) singl_eltkn_test('Option', '&l:opt', {scope='Local', name='opt'}) singl_eltkn_test('Invalid', '&l:', {error='E112: Option name missing: %.*s'}) singl_eltkn_test('Invalid', '&g:', {error='E112: Option name missing: %.*s'}) singl_eltkn_test('Register', '@', {name=-1}) singl_eltkn_test('Register', '@a', {name='a'}) singl_eltkn_test('Register', '@\r', {name=13}) singl_eltkn_test('Register', '@ ', {name=' '}) singl_eltkn_test('Register', '@\t', {name=9}) singl_eltkn_test('SingleQuotedString', '\'test', {closed=false}) singl_eltkn_test('SingleQuotedString', '\'test\'', {closed=true}) singl_eltkn_test('SingleQuotedString', '\'\'\'\'', {closed=true}) singl_eltkn_test('SingleQuotedString', '\'x\'\'\'', {closed=true}) singl_eltkn_test('SingleQuotedString', '\'\'\'x\'', {closed=true}) singl_eltkn_test('SingleQuotedString', '\'\'\'', {closed=false}) singl_eltkn_test('SingleQuotedString', '\'x\'\'', {closed=false}) singl_eltkn_test('SingleQuotedString', '\'\'\'x', {closed=false}) singl_eltkn_test('DoubleQuotedString', '"test', {closed=false}) singl_eltkn_test('DoubleQuotedString', '"test"', {closed=true}) singl_eltkn_test('DoubleQuotedString', '"\\""', {closed=true}) singl_eltkn_test('DoubleQuotedString', '"x\\""', {closed=true}) singl_eltkn_test('DoubleQuotedString', '"\\"x"', {closed=true}) singl_eltkn_test('DoubleQuotedString', '"\\"', {closed=false}) singl_eltkn_test('DoubleQuotedString', '"x\\"', {closed=false}) singl_eltkn_test('DoubleQuotedString', '"\\"x', {closed=false}) singl_eltkn_test('Not', '!') singl_eltkn_test('Assignment', '=', {type='Plain'}) comparison_test('==', '!=', 'Equal') comparison_test('=~', '!~', 'Matches') comparison_test('>', '<=', 'Greater') comparison_test('>=', '<', 'GreaterOrEqual') singl_eltkn_test('Minus', '-') singl_eltkn_test('Assignment', '-=', {type='Subtract'}) singl_eltkn_test('Arrow', '->') singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'}) simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'}) simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'}) simple_test({'2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0e'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'0b102'}, 'Number', 4, {data={is_float=false, base=2, val=2}, str='0b10'}) simple_test({'10F'}, 'Number', 2, {data={is_float=false, base=10, val=10}, str='10'}) simple_test({'0x0123456789ABCDEFG'}, 'Number', 18, {data={is_float=false, base=16, val=81985529216486895}, str='0x0123456789ABCDEF'}) simple_test({{data='00', size=2}}, 'Number', 2, {data={is_float=false, base=8, val=0}, str='00'}) simple_test({{data='009', size=2}}, 'Number', 2, {data={is_float=false, base=8, val=0}, str='00'}) simple_test({{data='01', size=1}}, 'Number', 1, {data={is_float=false, base=10, val=0}, str='0'}) end local function regular_scope_tests() scope_test('s') scope_test('g') scope_test('v') scope_test('b') scope_test('w') scope_test('t') scope_test('l') scope_test('a') simple_test({'g:'}, 'PlainIdentifier', 2, {data={scope='g', autoload=false}, str='g:'}) simple_test({'g:is#foo'}, 'PlainIdentifier', 8, {data={scope='g', autoload=true}, str='g:is#foo'}) simple_test({'g:isnot#foo'}, 'PlainIdentifier', 11, {data={scope='g', autoload=true}, str='g:isnot#foo'}) end local function regular_is_tests() comparison_test('is', 'isnot', 'Identical') simple_test({'is'}, 'Comparison', 2, {data={type='Identical', inv=false, ccs='UseOption'}, str='is'}) simple_test({'isnot'}, 'Comparison', 5, {data={type='Identical', inv=true, ccs='UseOption'}, str='isnot'}) simple_test({'is?'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='IgnoreCase'}, str='is?'}) simple_test({'isnot?'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='IgnoreCase'}, str='isnot?'}) simple_test({'is#'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'}) simple_test({'isnot#'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'}) simple_test({'is#foo'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'}) simple_test({'isnot#foo'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'}) end local function regular_number_tests() simple_test({'2.0'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) end local function regular_eoc_tests() singl_eltkn_test('EOC', '|') singl_eltkn_test('EOC', '\0') singl_eltkn_test('EOC', '\n') end itp('works (single tokens, zero flags)', function() stable_tests() regular_eoc_tests() regular_scope_tests() regular_is_tests() regular_number_tests() end) itp('peeks', function() flags = tonumber(lib.kELFlagPeek) should_advance = false stable_tests() regular_eoc_tests() regular_scope_tests() regular_is_tests() regular_number_tests() end) itp('forbids scope', function() flags = tonumber(lib.kELFlagForbidScope) stable_tests() regular_eoc_tests() regular_is_tests() regular_number_tests() simple_test({'g:'}, 'PlainIdentifier', 1, {data={scope=0, autoload=false}, str='g'}) end) itp('allows floats', function() flags = tonumber(lib.kELFlagAllowFloat) stable_tests() regular_eoc_tests() regular_scope_tests() regular_is_tests() simple_test({'2.2'}, 'Number', 3, {data={is_float=true, base=10, val=2.2}, str='2.2'}) simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true, base=10, val=2e5}, str='2.0e5'}) simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true, base=10, val=2e5}, str='2.0e+5'}) simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true, base=10, val=2e-5}, str='2.0e-5'}) simple_test({'2.500000e-5'}, 'Number', 11, {data={is_float=true, base=10, val=2.5e-5}, str='2.500000e-5'}) simple_test({'2.5555e2'}, 'Number', 8, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e2'}) simple_test({'2.5555e+2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e+2'}) simple_test({'2.5555e-2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e-2}, str='2.5555e-2'}) simple_test({{data='2.5e-5', size=3}}, 'Number', 3, {data={is_float=true, base=10, val=2.5}, str='2.5'}) simple_test({{data='2.5e5', size=4}}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) simple_test({{data='2.5e-50', size=6}}, 'Number', 6, {data={is_float=true, base=10, val=2.5e-5}, str='2.5e-5'}) end) itp('treats `is` as an identifier', function() flags = tonumber(lib.kELFlagIsNotCmp) stable_tests() regular_eoc_tests() regular_scope_tests() regular_number_tests() simple_test({'is'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'}) simple_test({'isnot'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'}) simple_test({'is?'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'}) simple_test({'isnot?'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'}) simple_test({'is#'}, 'PlainIdentifier', 3, {data={scope=0, autoload=true}, str='is#'}) simple_test({'isnot#'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='isnot#'}) simple_test({'is#foo'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='is#foo'}) simple_test({'isnot#foo'}, 'PlainIdentifier', 9, {data={scope=0, autoload=true}, str='isnot#foo'}) end) itp('forbids EOC', function() flags = tonumber(lib.kELFlagForbidEOC) stable_tests() regular_scope_tests() regular_is_tests() regular_number_tests() singl_eltkn_test('Invalid', '|', {error='E15: Unexpected EOC character: %.*s'}) singl_eltkn_test('Invalid', '\0', {error='E15: Unexpected EOC character: %.*s'}) singl_eltkn_test('Invalid', '\n', {error='E15: Unexpected EOC character: %.*s'}) end) end)