path: root/test/unit/viml/expressions/lexer_spec.lua

local helpers = require('test.unit.helpers')(after_each)
local itp = helpers.gen_itp(it)

local child_call_once = helpers.child_call_once
local cimport = helpers.cimport
local ffi = helpers.ffi
local eq = helpers.eq

local lib = cimport('./src/nvim/viml/parser/expressions.h')

local eltkn_type_tab, eltkn_cmp_type_tab, ccs_tab, eltkn_mul_type_tab
local eltkn_opt_scope_tab
child_call_once(function()
  eltkn_type_tab = {
    [tonumber(lib.kExprLexInvalid)] = 'Invalid',
    [tonumber(lib.kExprLexMissing)] = 'Missing',
    [tonumber(lib.kExprLexSpacing)] = 'Spacing',
    [tonumber(lib.kExprLexEOC)] = 'EOC',

    [tonumber(lib.kExprLexQuestion)] = 'Question',
    [tonumber(lib.kExprLexColon)] = 'Colon',
    [tonumber(lib.kExprLexOr)] = 'Or',
    [tonumber(lib.kExprLexAnd)] = 'And',
    [tonumber(lib.kExprLexComparison)] = 'Comparison',
    [tonumber(lib.kExprLexPlus)] = 'Plus',
    [tonumber(lib.kExprLexMinus)] = 'Minus',
    [tonumber(lib.kExprLexDot)] = 'Dot',
    [tonumber(lib.kExprLexMultiplication)] = 'Multiplication',

    [tonumber(lib.kExprLexNot)] = 'Not',

    [tonumber(lib.kExprLexNumber)] = 'Number',
    [tonumber(lib.kExprLexSingleQuotedString)] = 'SingleQuotedString',
    [tonumber(lib.kExprLexDoubleQuotedString)] = 'DoubleQuotedString',
    [tonumber(lib.kExprLexOption)] = 'Option',
    [tonumber(lib.kExprLexRegister)] = 'Register',
    [tonumber(lib.kExprLexEnv)] = 'Env',
    [tonumber(lib.kExprLexPlainIdentifier)] = 'PlainIdentifier',

    [tonumber(lib.kExprLexBracket)] = 'Bracket',
    [tonumber(lib.kExprLexFigureBrace)] = 'FigureBrace',
    [tonumber(lib.kExprLexParenthesis)] = 'Parenthesis',
    [tonumber(lib.kExprLexComma)] = 'Comma',
    [tonumber(lib.kExprLexArrow)] = 'Arrow',
  }

  eltkn_cmp_type_tab = {
    [tonumber(lib.kExprLexCmpEqual)] = 'Equal',
    [tonumber(lib.kExprLexCmpMatches)] = 'Matches',
    [tonumber(lib.kExprLexCmpGreater)] = 'Greater',
    [tonumber(lib.kExprLexCmpGreaterOrEqual)] = 'GreaterOrEqual',
    [tonumber(lib.kExprLexCmpIdentical)] = 'Identical',
  }

  ccs_tab = {
    [tonumber(lib.kCCStrategyUseOption)] = 'UseOption',
    [tonumber(lib.kCCStrategyMatchCase)] = 'MatchCase',
    [tonumber(lib.kCCStrategyIgnoreCase)] = 'IgnoreCase',
  }

  eltkn_mul_type_tab = {
    [tonumber(lib.kExprLexMulMul)] = 'Mul',
    [tonumber(lib.kExprLexMulDiv)] = 'Div',
    [tonumber(lib.kExprLexMulMod)] = 'Mod',
  }

  eltkn_opt_scope_tab = {
    [tonumber(lib.kExprLexOptUnspecified)] = 'Unspecified',
    [tonumber(lib.kExprLexOptGlobal)] = 'Global',
    [tonumber(lib.kExprLexOptLocal)] = 'Local',
  }
end)

local function array_size(arr)
  return ffi.sizeof(arr) / ffi.sizeof(arr[0])
end

local function kvi_size(kvi)
  return array_size(kvi.init_array)
end

local function kvi_init(kvi)
  kvi.capacity = kvi_size(kvi)
  kvi.items = kvi.init_array
  return kvi
end

local function kvi_new(ct)
  return kvi_init(ffi.new(ct))
end

local function new_pstate(strings)
  local strings_idx = 0
  local function get_line(_, ret_pline)
    strings_idx = strings_idx + 1
    local str = strings[strings_idx]
    local data, size
    if type(str) == 'string' then
      data = str
      size = #str
    elseif type(str) == 'nil' then
      data = nil
      size = 0
    elseif type(str) == 'table' then
      data = str.data
      size = str.size
    elseif type(str) == 'function' then
      data, size = str()
      size = size or 0
    end
    ret_pline.data = data
    ret_pline.size = size
  end
  local pline_init = {
    data = nil,
    size = 0,
  }
  local state = {
    reader = {
      get_line = get_line,
      cookie = nil,
    },
    pos = { line = 0, col = 0 },
    colors = kvi_new('ParserHighlight'),
    can_continuate = false,
  }
  local ret = ffi.new('ParserState', state)
  kvi_init(ret.reader.lines)
  kvi_init(ret.stack)
  return ret
end

local function conv_enum(etab, eval)
  local n = tonumber(eval)
  return etab[n] or n
end

local function conv_eltkn_type(typ)
  return conv_enum(eltkn_type_tab, typ)
end

local function pline2lua(pline)
  return ffi.string(pline.data, pline.size)
end

local bracket_types = {
  Bracket = true,
  FigureBrace = true,
  Parenthesis = true,
}

local function intchar2lua(ch)
  ch = tonumber(ch)
  return (20 <= ch and ch < 127) and ('%c'):format(ch) or ch
end

local function eltkn2lua(pstate, tkn)
  local ret = {
    type = conv_eltkn_type(tkn.type),
    len = tonumber(tkn.len),
    start = { line = tonumber(tkn.start.line), col = tonumber(tkn.start.col) },
  }
  if ret.start.line < pstate.reader.lines.size then
    local pstr = pline2lua(pstate.reader.lines.items[ret.start.line])
    if ret.start.col >= #pstr then
      ret.error = 'start.col >= #pstr'
    else
      ret.str = pstr:sub(ret.start.col + 1, ret.start.col + ret.len)
      if #(ret.str) ~= ret.len then
        ret.error = '#str /= len'
      end
    end
  else
    ret.error = 'start.line >= pstate.reader.lines.size'
  end
  if ret.type == 'Comparison' then
    ret.data = {
      type = conv_enum(eltkn_cmp_type_tab, tkn.data.cmp.type),
      ccs = conv_enum(ccs_tab, tkn.data.cmp.ccs),
      inv = (not not tkn.data.cmp.inv),
    }
  elseif ret.type == 'Multiplication' then
    ret.data = { type = conv_enum(eltkn_mul_type_tab, tkn.data.mul.type) }
  elseif bracket_types[ret.type] then
    ret.data = { closing = (not not tkn.data.brc.closing) }
  elseif ret.type == 'Register' then
    ret.data = { name = intchar2lua(tkn.data.reg.name) }
  elseif (ret.type == 'SingleQuotedString'
          or ret.type == 'DoubleQuotedString') then
    ret.data = { closed = (not not tkn.data.str.closed) }
  elseif ret.type == 'Option' then
    ret.data = {
      scope = conv_enum(eltkn_opt_scope_tab, tkn.data.opt.scope),
      name = ffi.string(tkn.data.opt.name, tkn.data.opt.len),
    }
  elseif ret.type == 'PlainIdentifier' then
    ret.data = {
      scope = intchar2lua(tkn.data.var.scope),
      autoload = (not not tkn.data.var.autoload),
    }
  elseif ret.type == 'Invalid' then
    ret.data = { error = ffi.string(tkn.data.err.msg) }
  end
  return ret, tkn
end

local function next_eltkn(pstate)
  return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, false))
end

describe('Expressions lexer', function()
  itp('works (single tokens)', function()
    local function singl_eltkn_test(typ, str, data)
      local pstate = new_pstate({str})
      eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ},
         next_eltkn(pstate))
      if not (
          typ == 'Spacing'
          or (typ == 'Register' and str == '@')
          or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString')
              and not data.closed)
      ) then
        pstate = new_pstate({str .. ' '})
        eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ},
           next_eltkn(pstate))
      end
      pstate = new_pstate({'x' .. str})
      pstate.pos.col = 1
      eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ},
         next_eltkn(pstate))
    end
    singl_eltkn_test('Parenthesis', '(', {closing=false})
    singl_eltkn_test('Parenthesis', ')', {closing=true})
    singl_eltkn_test('Bracket', '[', {closing=false})
    singl_eltkn_test('Bracket', ']', {closing=true})
    singl_eltkn_test('FigureBrace', '{', {closing=false})
    singl_eltkn_test('FigureBrace', '}', {closing=true})
    singl_eltkn_test('Question', '?')
    singl_eltkn_test('Colon', ':')
    singl_eltkn_test('Dot', '.')
    singl_eltkn_test('Plus', '+')
    singl_eltkn_test('Comma', ',')
    singl_eltkn_test('Multiplication', '*', {type='Mul'})
    singl_eltkn_test('Multiplication', '/', {type='Div'})
    singl_eltkn_test('Multiplication', '%', {type='Mod'})
    singl_eltkn_test('Spacing', '  \t\t  \t\t')
    singl_eltkn_test('Spacing', ' ')
    singl_eltkn_test('Spacing', '\t')
    singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'})
    singl_eltkn_test('Number', '0123')
    singl_eltkn_test('Number', '0')
    singl_eltkn_test('Number', '9')
    singl_eltkn_test('Env', '$abc')
    singl_eltkn_test('Env', '$')
    singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0})
    singl_eltkn_test('PlainIdentifier', '_test', {autoload=false, scope=0})
    singl_eltkn_test('PlainIdentifier', '_test_foo', {autoload=false, scope=0})
    singl_eltkn_test('PlainIdentifier', 't', {autoload=false, scope=0})
    singl_eltkn_test('PlainIdentifier', 'test5', {autoload=false, scope=0})
    singl_eltkn_test('PlainIdentifier', 't0', {autoload=false, scope=0})
    singl_eltkn_test('PlainIdentifier', 'test#var', {autoload=true, scope=0})
    singl_eltkn_test('PlainIdentifier', 'test#var#val###', {autoload=true, scope=0})
    singl_eltkn_test('PlainIdentifier', 't#####', {autoload=true, scope=0})
    local function scope_test(scope)
      singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope})
      singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope})
    end
    scope_test('s')
    scope_test('g')
    scope_test('v')
    scope_test('b')
    scope_test('w')
    scope_test('t')
    scope_test('l')
    scope_test('a')
    local function comparison_test(op, inv_op, cmp_type)
      singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'})
      singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'})
      singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'})
      singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'})
      singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'})
      singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'})
    end
    comparison_test('is', 'isnot', 'Identical')
    singl_eltkn_test('And', '&&')
    singl_eltkn_test('Invalid', '&', {error='E112: Option name missing: %.*s'})
    singl_eltkn_test('Option', '&opt', {scope='Unspecified', name='opt'})
    singl_eltkn_test('Option', '&t_xx', {scope='Unspecified', name='t_xx'})
    singl_eltkn_test('Option', '&t_\r\r', {scope='Unspecified', name='t_\r\r'})
    singl_eltkn_test('Option', '&t_\t\t', {scope='Unspecified', name='t_\t\t'})
    singl_eltkn_test('Option', '&t_  ', {scope='Unspecified', name='t_  '})
    singl_eltkn_test('Option', '&g:opt', {scope='Global', name='opt'})
    singl_eltkn_test('Option', '&l:opt', {scope='Local', name='opt'})
    singl_eltkn_test('Invalid', '&l:', {error='E112: Option name missing: %.*s'})
    singl_eltkn_test('Invalid', '&g:', {error='E112: Option name missing: %.*s'})
    singl_eltkn_test('Register', '@', {name=-1})
    singl_eltkn_test('Register', '@a', {name='a'})
    singl_eltkn_test('Register', '@\r', {name=13})
    singl_eltkn_test('Register', '@ ', {name=' '})
    singl_eltkn_test('Register', '@\t', {name=9})
    singl_eltkn_test('SingleQuotedString', '\'test', {closed=false})
    singl_eltkn_test('SingleQuotedString', '\'test\'', {closed=true})
    singl_eltkn_test('SingleQuotedString', '\'\'\'\'', {closed=true})
    singl_eltkn_test('SingleQuotedString', '\'x\'\'\'', {closed=true})
    singl_eltkn_test('SingleQuotedString', '\'\'\'x\'', {closed=true})
    singl_eltkn_test('SingleQuotedString', '\'\'\'', {closed=false})
    singl_eltkn_test('SingleQuotedString', '\'x\'\'', {closed=false})
    singl_eltkn_test('SingleQuotedString', '\'\'\'x', {closed=false})
    singl_eltkn_test('DoubleQuotedString', '"test', {closed=false})
    singl_eltkn_test('DoubleQuotedString', '"test"', {closed=true})
    singl_eltkn_test('DoubleQuotedString', '"\\""', {closed=true})
    singl_eltkn_test('DoubleQuotedString', '"x\\""', {closed=true})
    singl_eltkn_test('DoubleQuotedString', '"\\"x"', {closed=true})
    singl_eltkn_test('DoubleQuotedString', '"\\"', {closed=false})
    singl_eltkn_test('DoubleQuotedString', '"x\\"', {closed=false})
    singl_eltkn_test('DoubleQuotedString', '"\\"x', {closed=false})
    singl_eltkn_test('Not', '!')
    singl_eltkn_test('Invalid', '=', {error='E15: Expected == or =~: %.*s'})
    comparison_test('==', '!=', 'Equal')
    comparison_test('=~', '!~', 'Matches')
    comparison_test('>', '<=', 'Greater')
    comparison_test('>=', '<', 'GreaterOrEqual')
    singl_eltkn_test('Minus', '-')
    singl_eltkn_test('Arrow', '->')
    singl_eltkn_test('EOC', '\0')
    singl_eltkn_test('EOC', '\n')
    singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'})

    local pstate = new_pstate({{data=nil, size=0}})
    eq({len=0, error='start.col >= #pstr', start={col=0, line=0}, type='EOC'},
       next_eltkn(pstate))

    local pstate = new_pstate({''})
    eq({len=0, error='start.col >= #pstr', start={col=0, line=0}, type='EOC'},
       next_eltkn(pstate))
  end)
end)