viml/expressions: Add lexer with some basic tests

1 files changed, 337 insertions, 0 deletions

diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua
new file mode 100644
index 0000000000..bf5afe4eeb
--- /dev/null
+++ b/test/unit/viml/expressions/lexer_spec.lua
@@ -0,0 +1,337 @@
+local helpers = require('test.unit.helpers')(after_each)
+local itp = helpers.gen_itp(it)
+
+local child_call_once = helpers.child_call_once
+local cimport = helpers.cimport
+local ffi = helpers.ffi
+local eq = helpers.eq
+
+local lib = cimport('./src/nvim/viml/parser/expressions.h')
+
+local eltkn_type_tab, eltkn_cmp_type_tab, ccs_tab, eltkn_mul_type_tab
+local eltkn_opt_scope_tab
+child_call_once(function()
+  eltkn_type_tab = {
+    [tonumber(lib.kExprLexInvalid)] = 'Invalid',
+    [tonumber(lib.kExprLexMissing)] = 'Missing',
+    [tonumber(lib.kExprLexSpacing)] = 'Spacing',
+    [tonumber(lib.kExprLexEOC)] = 'EOC',
+
+    [tonumber(lib.kExprLexQuestion)] = 'Question',
+    [tonumber(lib.kExprLexColon)] = 'Colon',
+    [tonumber(lib.kExprLexOr)] = 'Or',
+    [tonumber(lib.kExprLexAnd)] = 'And',
+    [tonumber(lib.kExprLexComparison)] = 'Comparison',
+    [tonumber(lib.kExprLexPlus)] = 'Plus',
+    [tonumber(lib.kExprLexMinus)] = 'Minus',
+    [tonumber(lib.kExprLexDot)] = 'Dot',
+    [tonumber(lib.kExprLexMultiplication)] = 'Multiplication',
+
+    [tonumber(lib.kExprLexNot)] = 'Not',
+
+    [tonumber(lib.kExprLexNumber)] = 'Number',
+    [tonumber(lib.kExprLexSingleQuotedString)] = 'SingleQuotedString',
+    [tonumber(lib.kExprLexDoubleQuotedString)] = 'DoubleQuotedString',
+    [tonumber(lib.kExprLexOption)] = 'Option',
+    [tonumber(lib.kExprLexRegister)] = 'Register',
+    [tonumber(lib.kExprLexEnv)] = 'Env',
+    [tonumber(lib.kExprLexPlainIdentifier)] = 'PlainIdentifier',
+
+    [tonumber(lib.kExprLexBracket)] = 'Bracket',
+    [tonumber(lib.kExprLexFigureBrace)] = 'FigureBrace',
+    [tonumber(lib.kExprLexParenthesis)] = 'Parenthesis',
+    [tonumber(lib.kExprLexComma)] = 'Comma',
+    [tonumber(lib.kExprLexArrow)] = 'Arrow',
+  }
+
+  eltkn_cmp_type_tab = {
+    [tonumber(lib.kExprLexCmpEqual)] = 'Equal',
+    [tonumber(lib.kExprLexCmpMatches)] = 'Matches',
+    [tonumber(lib.kExprLexCmpGreater)] = 'Greater',
+    [tonumber(lib.kExprLexCmpGreaterOrEqual)] = 'GreaterOrEqual',
+    [tonumber(lib.kExprLexCmpIdentical)] = 'Identical',
+  }
+
+  ccs_tab = {
+    [tonumber(lib.kCCStrategyUseOption)] = 'UseOption',
+    [tonumber(lib.kCCStrategyMatchCase)] = 'MatchCase',
+    [tonumber(lib.kCCStrategyIgnoreCase)] = 'IgnoreCase',
+  }
+
+  eltkn_mul_type_tab = {
+    [tonumber(lib.kExprLexMulMul)] = 'Mul',
+    [tonumber(lib.kExprLexMulDiv)] = 'Div',
+    [tonumber(lib.kExprLexMulMod)] = 'Mod',
+  }
+
+  eltkn_opt_scope_tab = {
+    [tonumber(lib.kExprLexOptUnspecified)] = 'Unspecified',
+    [tonumber(lib.kExprLexOptGlobal)] = 'Global',
+    [tonumber(lib.kExprLexOptLocal)] = 'Local',
+  }
+end)
+
+local function array_size(arr)
+  return ffi.sizeof(arr) / ffi.sizeof(arr[0])
+end
+
+local function kvi_size(kvi)
+  return array_size(kvi.init_array)
+end
+
+local function kvi_init(kvi)
+  kvi.capacity = kvi_size(kvi)
+  kvi.items = kvi.init_array
+  return kvi
+end
+
+local function kvi_new(ct)
+  return kvi_init(ffi.new(ct))
+end
+
+local function new_pstate(strings)
+  local strings_idx = 0
+  local function get_line(_, ret_pline)
+    strings_idx = strings_idx + 1
+    local str = strings[strings_idx]
+    local data, size
+    if type(str) == 'string' then
+      data = str
+      size = #str
+    elseif type(str) == 'nil' then
+      data = nil
+      size = 0
+    elseif type(str) == 'table' then
+      data = str.data
+      size = str.size
+    elseif type(str) == 'function' then
+      data, size = str()
+      size = size or 0
+    end
+    ret_pline.data = data
+    ret_pline.size = size
+  end
+  local pline_init = {
+    data = nil,
+    size = 0,
+  }
+  local state = {
+    reader = {
+      get_line = get_line,
+      cookie = nil,
+    },
+    pos = { line = 0, col = 0 },
+    colors = kvi_new('ParserHighlight'),
+    can_continuate = false,
+  }
+  local ret = ffi.new('ParserState', state)
+  kvi_init(ret.reader.lines)
+  kvi_init(ret.stack)
+  return ret
+end
+
+local function conv_enum(etab, eval)
+  local n = tonumber(eval)
+  return etab[n] or n
+end
+
+local function conv_eltkn_type(typ)
+  return conv_enum(eltkn_type_tab, typ)
+end
+
+local function pline2lua(pline)
+  return ffi.string(pline.data, pline.size)
+end
+
+local bracket_types = {
+  Bracket = true,
+  FigureBrace = true,
+  Parenthesis = true,
+}
+
+local function intchar2lua(ch)
+  ch = tonumber(ch)
+  return (20 <= ch and ch < 127) and ('%c'):format(ch) or ch
+end
+
+local function eltkn2lua(pstate, tkn)
+  local ret = {
+    type = conv_eltkn_type(tkn.type),
+    len = tonumber(tkn.len),
+    start = { line = tonumber(tkn.start.line), col = tonumber(tkn.start.col) },
+  }
+  if ret.start.line < pstate.reader.lines.size then
+    local pstr = pline2lua(pstate.reader.lines.items[ret.start.line])
+    if ret.start.col >= #pstr then
+      ret.error = 'start.col >= #pstr'
+    else
+      ret.str = pstr:sub(ret.start.col + 1, ret.start.col + ret.len)
+      if #(ret.str) ~= ret.len then
+        ret.error = '#str /= len'
+      end
+    end
+  else
+    ret.error = 'start.line >= pstate.reader.lines.size'
+  end
+  if ret.type == 'Comparison' then
+    ret.data = {
+      type = conv_enum(eltkn_cmp_type_tab, tkn.data.cmp.type),
+      ccs = conv_enum(ccs_tab, tkn.data.cmp.ccs),
+      inv = (not not tkn.data.cmp.inv),
+    }
+  elseif ret.type == 'Multiplication' then
+    ret.data = { type = conv_enum(eltkn_mul_type_tab, tkn.data.mul.type) }
+  elseif bracket_types[ret.type] then
+    ret.data = { closing = (not not tkn.data.brc.closing) }
+  elseif ret.type == 'Register' then
+    ret.data = { name = intchar2lua(tkn.data.reg.name) }
+  elseif (ret.type == 'SingleQuotedString'
+          or ret.type == 'DoubleQuotedString') then
+    ret.data = { closed = (not not tkn.data.str.closed) }
+  elseif ret.type == 'Option' then
+    ret.data = {
+      scope = conv_enum(eltkn_opt_scope_tab, tkn.data.opt.scope),
+      name = ffi.string(tkn.data.opt.name, tkn.data.opt.len),
+    }
+  elseif ret.type == 'PlainIdentifier' then
+    ret.data = {
+      scope = intchar2lua(tkn.data.var.scope),
+      autoload = (not not tkn.data.var.autoload),
+    }
+  elseif ret.type == 'Invalid' then
+    ret.data = { error = ffi.string(tkn.data.err.msg) }
+  end
+  return ret, tkn
+end
+
+local function next_eltkn(pstate)
+  return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, false))
+end
+
+describe('Expressions lexer', function()
+  itp('works (single tokens)', function()
+    local function singl_eltkn_test(typ, str, data)
+      local pstate = new_pstate({str})
+      eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ},
+         next_eltkn(pstate))
+      if not (
+          typ == 'Spacing'
+          or (typ == 'Register' and str == '@')
+          or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString')
+              and not data.closed)
+      ) then
+        pstate = new_pstate({str .. ' '})
+        eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ},
+           next_eltkn(pstate))
+      end
+      pstate = new_pstate({'x' .. str})
+      pstate.pos.col = 1
+      eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ},
+         next_eltkn(pstate))
+    end
+    singl_eltkn_test('Parenthesis', '(', {closing=false})
+    singl_eltkn_test('Parenthesis', ')', {closing=true})
+    singl_eltkn_test('Bracket', '[', {closing=false})
+    singl_eltkn_test('Bracket', ']', {closing=true})
+    singl_eltkn_test('FigureBrace', '{', {closing=false})
+    singl_eltkn_test('FigureBrace', '}', {closing=true})
+    singl_eltkn_test('Question', '?')
+    singl_eltkn_test('Colon', ':')
+    singl_eltkn_test('Dot', '.')
+    singl_eltkn_test('Plus', '+')
+    singl_eltkn_test('Comma', ',')
+    singl_eltkn_test('Multiplication', '*', {type='Mul'})
+    singl_eltkn_test('Multiplication', '/', {type='Div'})
+    singl_eltkn_test('Multiplication', '%', {type='Mod'})
+    singl_eltkn_test('Spacing', '  \t\t  \t\t')
+    singl_eltkn_test('Spacing', ' ')
+    singl_eltkn_test('Spacing', '\t')
+    singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'})
+    singl_eltkn_test('Number', '0123')
+    singl_eltkn_test('Number', '0')
+    singl_eltkn_test('Number', '9')
+    singl_eltkn_test('Env', '$abc')
+    singl_eltkn_test('Env', '$')
+    singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0})
+    singl_eltkn_test('PlainIdentifier', '_test', {autoload=false, scope=0})
+    singl_eltkn_test('PlainIdentifier', '_test_foo', {autoload=false, scope=0})
+    singl_eltkn_test('PlainIdentifier', 't', {autoload=false, scope=0})
+    singl_eltkn_test('PlainIdentifier', 'test5', {autoload=false, scope=0})
+    singl_eltkn_test('PlainIdentifier', 't0', {autoload=false, scope=0})
+    singl_eltkn_test('PlainIdentifier', 'test#var', {autoload=true, scope=0})
+    singl_eltkn_test('PlainIdentifier', 'test#var#val###', {autoload=true, scope=0})
+    singl_eltkn_test('PlainIdentifier', 't#####', {autoload=true, scope=0})
+    local function scope_test(scope)
+      singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope})
+      singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope})
+    end
+    scope_test('s')
+    scope_test('g')
+    scope_test('v')
+    scope_test('b')
+    scope_test('w')
+    scope_test('t')
+    scope_test('l')
+    scope_test('a')
+    local function comparison_test(op, inv_op, cmp_type)
+      singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'})
+      singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'})
+      singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'})
+      singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'})
+      singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'})
+      singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'})
+    end
+    comparison_test('is', 'isnot', 'Identical')
+    singl_eltkn_test('And', '&&')
+    singl_eltkn_test('Invalid', '&', {error='E112: Option name missing: %.*s'})
+    singl_eltkn_test('Option', '&opt', {scope='Unspecified', name='opt'})
+    singl_eltkn_test('Option', '&t_xx', {scope='Unspecified', name='t_xx'})
+    singl_eltkn_test('Option', '&t_\r\r', {scope='Unspecified', name='t_\r\r'})
+    singl_eltkn_test('Option', '&t_\t\t', {scope='Unspecified', name='t_\t\t'})
+    singl_eltkn_test('Option', '&t_  ', {scope='Unspecified', name='t_  '})
+    singl_eltkn_test('Option', '&g:opt', {scope='Global', name='opt'})
+    singl_eltkn_test('Option', '&l:opt', {scope='Local', name='opt'})
+    singl_eltkn_test('Invalid', '&l:', {error='E112: Option name missing: %.*s'})
+    singl_eltkn_test('Invalid', '&g:', {error='E112: Option name missing: %.*s'})
+    singl_eltkn_test('Register', '@', {name=-1})
+    singl_eltkn_test('Register', '@a', {name='a'})
+    singl_eltkn_test('Register', '@\r', {name=13})
+    singl_eltkn_test('Register', '@ ', {name=' '})
+    singl_eltkn_test('Register', '@\t', {name=9})
+    singl_eltkn_test('SingleQuotedString', '\'test', {closed=false})
+    singl_eltkn_test('SingleQuotedString', '\'test\'', {closed=true})
+    singl_eltkn_test('SingleQuotedString', '\'\'\'\'', {closed=true})
+    singl_eltkn_test('SingleQuotedString', '\'x\'\'\'', {closed=true})
+    singl_eltkn_test('SingleQuotedString', '\'\'\'x\'', {closed=true})
+    singl_eltkn_test('SingleQuotedString', '\'\'\'', {closed=false})
+    singl_eltkn_test('SingleQuotedString', '\'x\'\'', {closed=false})
+    singl_eltkn_test('SingleQuotedString', '\'\'\'x', {closed=false})
+    singl_eltkn_test('DoubleQuotedString', '"test', {closed=false})
+    singl_eltkn_test('DoubleQuotedString', '"test"', {closed=true})
+    singl_eltkn_test('DoubleQuotedString', '"\\""', {closed=true})
+    singl_eltkn_test('DoubleQuotedString', '"x\\""', {closed=true})
+    singl_eltkn_test('DoubleQuotedString', '"\\"x"', {closed=true})
+    singl_eltkn_test('DoubleQuotedString', '"\\"', {closed=false})
+    singl_eltkn_test('DoubleQuotedString', '"x\\"', {closed=false})
+    singl_eltkn_test('DoubleQuotedString', '"\\"x', {closed=false})
+    singl_eltkn_test('Not', '!')
+    singl_eltkn_test('Invalid', '=', {error='E15: Expected == or =~: %.*s'})
+    comparison_test('==', '!=', 'Equal')
+    comparison_test('=~', '!~', 'Matches')
+    comparison_test('>', '<=', 'Greater')
+    comparison_test('>=', '<', 'GreaterOrEqual')
+    singl_eltkn_test('Minus', '-')
+    singl_eltkn_test('Arrow', '->')
+    singl_eltkn_test('EOC', '\0')
+    singl_eltkn_test('EOC', '\n')
+    singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'})
+
+    local pstate = new_pstate({{data=nil, size=0}})
+    eq({len=0, error='start.col >= #pstr', start={col=0, line=0}, type='EOC'},
+       next_eltkn(pstate))
+
+    local pstate = new_pstate({''})
+    eq({len=0, error='start.col >= #pstr', start={col=0, line=0}, type='EOC'},
+       next_eltkn(pstate))
+  end)
+end)