viml/parser/expressions: Add a way to adjust lexer

It also adds support for kExprLexOr which for some reason was forgotten.

It was only made sure that KLEE test compiles in non-KLEE mode, not that
something works or that KLEE is able to run tests.

5 files changed, 323 insertions, 89 deletions

diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c
index cabf2dac58..3027c0046b 100644
--- a/src/nvim/viml/parser/expressions.c
+++ b/src/nvim/viml/parser/expressions.c
@@ -47,10 +47,10 @@ typedef enum {
 /// Get next token for the VimL expression input
 ///
 /// @param  pstate  Parser state.
-/// @param[in]  peek  If true, do not advance pstate cursor.
+/// @param[in]  flags  Flags, @see LexExprFlags.
 ///
 /// @return Next token.
-LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek)
+LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags)
   FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
 {
   LexExprToken ret = {
@@ -153,12 +153,33 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek)
     }
 
     // Number.
-    // Note: determining whether dot is (not) a part of a float needs more
-    // context, so lexer does not do this.
-    // FIXME: Resolve ambiguity by additional argument.
     case '0': case '1': case '2': case '3': case '4': case '5': case '6':
     case '7': case '8': case '9': {
+      ret.data.num.is_float = false;
       CHARREG(kExprLexNumber, ascii_isdigit);
+      if (flags & kELFlagAllowFloat) {
+        if (pline.size > ret.len + 1
+            && pline.data[ret.len] == '.'
+            && ascii_isdigit(pline.data[ret.len + 1])) {
+          ret.len++;
+          ret.data.num.is_float = true;
+          CHARREG(kExprLexNumber, ascii_isdigit);
+          if (pline.size > ret.len + 1
+              && (pline.data[ret.len] == 'e'
+                  || pline.data[ret.len] == 'E')
+              && ((pline.size > ret.len + 2
+                   && (pline.data[ret.len + 1] == '+'
+                       || pline.data[ret.len + 1] == '-')
+                   && ascii_isdigit(pline.data[ret.len + 2]))
+                  || ascii_isdigit(pline.data[ret.len + 1]))) {
+            ret.len++;
+            if (pline.data[ret.len] == '+' || pline.data[ret.len] == '-') {
+              ret.len++;
+            }
+            CHARREG(kExprLexNumber, ascii_isdigit);
+          }
+        }
+      }
       break;
     }
 
@@ -187,8 +208,9 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek)
       ret.data.var.autoload = false;
       CHARREG(kExprLexPlainIdentifier, ISWORD);
       // "is" and "isnot" operators.
-      if ((ret.len == 2 && memcmp(pline.data, "is", 2) == 0)
-          || (ret.len == 5 && memcmp(pline.data, "isnot", 5) == 0)) {
+      if (!(flags & kELFlagIsNotCmp)
+          && ((ret.len == 2 && memcmp(pline.data, "is", 2) == 0)
+              || (ret.len == 5 && memcmp(pline.data, "isnot", 5) == 0))) {
         ret.type = kExprLexComparison;
         ret.data.cmp.type = kExprLexCmpIdentical;
         ret.data.cmp.inv = (ret.len == 5);
@@ -197,14 +219,14 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek)
       } else if (ret.len == 1
                  && pline.size > 1
                  && strchr("sgvbwtla", schar) != NULL
-                 && pline.data[ret.len] == ':') {
+                 && pline.data[ret.len] == ':'
+                 && !(flags & kELFlagForbidScope)) {
         ret.len++;
         ret.data.var.scope = schar;
         CHARREG(kExprLexPlainIdentifier, ISWORD_OR_AUTOLOAD);
         ret.data.var.autoload = (
             memchr(pline.data + 2, AUTOLOAD_CHAR, ret.len - 2)
             != NULL);
-      // FIXME: Resolve ambiguity with an argument to the lexer function.
       // Previous CHARREG stopped at autoload character in order to make it
       // possible to detect `is#`. Continue now with autoload characters
       // included.
@@ -373,7 +395,30 @@ viml_pexpr_next_token_invalid_comparison:
     // Expression end because Ex command ended.
     case NUL:
     case NL: {
-      ret.type = kExprLexEOC;
+      if (flags & kELFlagForbidEOC) {
+        ret.type = kExprLexInvalid;
+        ret.data.err.msg = _("E15: Unexpected EOC character: %.*s");
+        ret.data.err.type = kExprLexSpacing;
+      } else {
+        ret.type = kExprLexEOC;
+      }
+      break;
+    }
+
+    case '|': {
+      if (pline.size >= 2 && pline.data[ret.len] == '|') {
+        // "||" is or.
+        ret.len++;
+        ret.type = kExprLexOr;
+      } else if (flags & kELFlagForbidEOC) {
+        // Note: `<C-r>=1 | 2<CR>` actually yields 1 in Vim without any
+        //       errors. This will be changed here.
+        ret.type = kExprLexInvalid;
+        ret.data.err.msg = _("E15: Unexpected EOC character: %.*s");
+        ret.data.err.type = kExprLexOr;
+      } else {
+        ret.type = kExprLexEOC;
+      }
       break;
     }
 
@@ -389,7 +434,7 @@ viml_pexpr_next_token_invalid_comparison:
   }
 #undef GET_CCS
 viml_pexpr_next_token_adv_return:
-  if (!peek) {
+  if (!(flags & kELFlagPeek)) {
     viml_parser_advance(pstate, ret.len);
   }
   return ret;
@@ -990,34 +1035,28 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags)
   // Lambda node, valid when parsing lambda arguments only.
   ExprASTNode *lambda_node = NULL;
   do {
-    LexExprToken cur_token = viml_pexpr_next_token(pstate, true);
+    const int want_node_to_lexer_flags[] = {
+      [kENodeValue] = kELFlagIsNotCmp,
+      [kENodeOperator] = kELFlagForbidScope,
+      [kENodeArgument] = kELFlagIsNotCmp,
+      [kENodeArgumentSeparator] = kELFlagForbidScope,
+    };
+    // FIXME Determine when (not) to allow floating-point numbers.
+    const int lexer_additional_flags = (
+        kELFlagPeek
+        | ((flags & kExprFlagsDisallowEOC) ? kELFlagForbidEOC : 0));
+    LexExprToken cur_token = viml_pexpr_next_token(
+        pstate, want_node_to_lexer_flags[want_node] | lexer_additional_flags);
     if (cur_token.type == kExprLexEOC) {
-      if (flags & kExprFlagsDisallowEOC) {
-        if (cur_token.len == 0) {
-          // It is end of string, break.
-          break;
-        } else {
-          // It is NL, NUL or bar.
-          //
-          // Note: `<C-r>=1 | 2<CR>` actually yields 1 in Vim without any
-          //       errors. This will be changed here.
-          cur_token.type = kExprLexInvalid;
-          cur_token.data.err.msg = _("E15: Unexpected EOC character: %.*s");
-          const ParserLine pline = (
-              pstate->reader.lines.items[cur_token.start.line]);
-          const char eoc_char = pline.data[cur_token.start.col];
-          cur_token.data.err.type = ((eoc_char == NUL || eoc_char == NL)
-                                     ? kExprLexSpacing
-                                     : kExprLexOr);
-        }
-      } else {
-        break;
-      }
+      break;
     }
     LexExprTokenType tok_type = cur_token.type;
     const bool token_invalid = (tok_type == kExprLexInvalid);
     bool is_invalid = token_invalid;
 viml_pexpr_parse_process_token:
+    // May use different flags this time.
+    cur_token = viml_pexpr_next_token(
+        pstate, want_node_to_lexer_flags[want_node] | lexer_additional_flags);
     if (tok_type == kExprLexSpacing) {
       if (is_invalid) {
         HL_CUR_TOKEN(Spacing);
diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h
index 13640ec137..64abab9e41 100644
--- a/src/nvim/viml/parser/expressions.h
+++ b/src/nvim/viml/parser/expressions.h
@@ -109,9 +109,37 @@ typedef struct {
       LexExprTokenType type;  ///< Suggested type for parsing incorrect code.
       const char *msg;  ///< Error message.
     } err;  ///< For kExprLexInvalid
+
+    struct {
+      bool is_float;  ///< True if number is a floating-point.
+    } num;  ///< For kExprLexNumber
   } data;  ///< Additional data, if needed.
 } LexExprToken;
 
+typedef enum {
+  /// If set, “pointer” to the current byte in pstate will not be shifted
+  kELFlagPeek = (1 << 0),
+  /// Determines whether scope is allowed to come before the identifier
+  kELFlagForbidScope = (1 << 1),
+  /// Determines whether floating-point numbers are allowed
+  ///
+  /// I.e. whether dot is a decimal point separator or is not a part of
+  /// a number at all.
+  kELFlagAllowFloat = (1 << 2),
+  /// Determines whether `is` and `isnot` are seen as comparison operators
+  ///
+  /// If set they are supposed to be just regular identifiers.
+  kELFlagIsNotCmp = (1 << 3),
+  /// Determines whether EOC tokens are allowed
+  ///
+  /// If set then it will yield Invalid token with E15 in place of EOC one if
+  /// “EOC” is something like "|". It is fine with emitting EOC at the end of
+  /// string still, with or without this flag set.
+  kELFlagForbidEOC = (1 << 4),
+  // WARNING: whenever you add a new flag, alter klee_assume() statement in
+  // viml_expressions_lexer.c.
+} LexExprFlags;
+
 /// Expression AST node type
 typedef enum {
   kExprNodeMissing = 'X',
diff --git a/test/symbolic/klee/nvim/memory.c b/test/symbolic/klee/nvim/memory.c
index 7e924a410a..1f9cdce6c0 100644
--- a/test/symbolic/klee/nvim/memory.c
+++ b/test/symbolic/klee/nvim/memory.c
@@ -15,11 +15,16 @@ RINGBUF_INIT(AllocRecords, arecs, AllocRecord, RINGBUF_DUMMY_FREE)
 RINGBUF_STATIC(static, AllocRecords, AllocRecord, arecs, RB_SIZE)
 
 size_t allocated_memory = 0;
+size_t ever_allocated_memory = 0;
+
+size_t allocated_memory_limit = SIZE_MAX;
 
 void *xmalloc(const size_t size)
 {
   void *ret = malloc(size);
   allocated_memory += size;
+  ever_allocated_memory += size;
+  assert(allocated_memory <= allocated_memory_limit);
   assert(arecs_rb_length(&arecs) < RB_SIZE);
   arecs_rb_push(&arecs, (AllocRecord) {
     .ptr = ret,
@@ -47,6 +52,9 @@ void *xrealloc(void *const p, size_t new_size)
     if (arec->ptr == p) {
       allocated_memory -= arec->size;
       allocated_memory += new_size;
+      if (new_size > arec->size) {
+        ever_allocated_memory += (new_size - arec->size);
+      }
       arec->ptr = ret;
       arec->size = new_size;
       return ret;
diff --git a/test/symbolic/klee/viml_expressions_lexer.c b/test/symbolic/klee/viml_expressions_lexer.c
index e3b5aa80cc..67f3eb7faa 100644
--- a/test/symbolic/klee/viml_expressions_lexer.c
+++ b/test/symbolic/klee/viml_expressions_lexer.c
@@ -34,13 +34,20 @@ int main(const int argc, const char *const *const argv,
 {
   char input[INPUT_SIZE];
   uint8_t shift;
-  const bool peek = false;
+  int flags;
   avoid_optimizing_out = argc;
 
+#ifndef USE_KLEE
+  sscanf(argv[2], "%d", &flags);
+#endif
+
 #ifdef USE_KLEE
   klee_make_symbolic(input, sizeof(input), "input");
   klee_make_symbolic(&shift, sizeof(shift), "shift");
+  klee_make_symbolic(&flags, sizeof(flags), "flags");
   klee_assume(shift < INPUT_SIZE);
+  klee_assume(flags <= (kELFlagPeek|kELFlagAllowFloat|kELFlagForbidEOC
+                        |kELFlagForbidScope|kELFlagIsNotCmp));
 #endif
 
   ParserLine plines[] = {
@@ -79,8 +86,15 @@ int main(const int argc, const char *const *const argv,
   };
   kvi_init(pstate.reader.lines);
 
-  LexExprToken token = viml_pexpr_next_token(&pstate, peek);
-  assert((pstate.pos.line == 0)
-         ? (pstate.pos.col > 0)
-         : (pstate.pos.line == 1 && pstate.pos.col == 0));
+  allocated_memory_limit = 0;
+  LexExprToken token = viml_pexpr_next_token(&pstate, flags);
+  if (flags & kELFlagPeek) {
+    assert(pstate.pos.line == 0 && pstate.pos.col == 0);
+  } else {
+    assert((pstate.pos.line == 0)
+           ? (pstate.pos.col > 0)
+           : (pstate.pos.line == 1 && pstate.pos.col == 0));
+  }
+  assert(allocated_memory == 0);
+  assert(ever_allocated_memory == 0);
 }
diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua
index 32182f650d..972478c2e5 100644
--- a/test/unit/viml/expressions/lexer_spec.lua
+++ b/test/unit/viml/expressions/lexer_spec.lua
@@ -1,5 +1,6 @@
 local helpers = require('test.unit.helpers')(after_each)
 local viml_helpers = require('test.unit.viml.helpers')
+local global_helpers = require('test.helpers')
 local itp = helpers.gen_itp(it)
 
 local child_call_once = helpers.child_call_once
@@ -13,6 +14,8 @@ local new_pstate = viml_helpers.new_pstate
 local intchar2lua = viml_helpers.intchar2lua
 local pstate_set_str = viml_helpers.pstate_set_str
 
+local shallowcopy = global_helpers.shallowcopy
+
 local lib = cimport('./src/nvim/viml/parser/expressions.h')
 
 local eltkn_type_tab, eltkn_cmp_type_tab, ccs_tab, eltkn_mul_type_tab
@@ -121,37 +124,81 @@ local function eltkn2lua(pstate, tkn)
       scope = intchar2lua(tkn.data.var.scope),
       autoload = (not not tkn.data.var.autoload),
     }
+  elseif ret.type == 'Number' then
+    ret.data = {
+      is_float = (not not tkn.data.num.is_float),
+    }
   elseif ret.type == 'Invalid' then
     ret.data = { error = ffi.string(tkn.data.err.msg) }
   end
   return ret, tkn
 end
 
-local function next_eltkn(pstate)
-  return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, false))
+local function next_eltkn(pstate, flags)
+  return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, flags))
 end
 
 describe('Expressions lexer', function()
-  itp('works (single tokens)', function()
-    local function singl_eltkn_test(typ, str, data)
-      local pstate = new_pstate({str})
-      eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ},
-         next_eltkn(pstate))
-      if not (
-          typ == 'Spacing'
-          or (typ == 'Register' and str == '@')
-          or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString')
-              and not data.closed)
-      ) then
-        pstate = new_pstate({str .. ' '})
-        eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ},
-           next_eltkn(pstate))
+  local flags = 0
+  local should_advance = true
+  local function check_advance(pstate, bytes_to_advance, initial_col)
+    local tgt = initial_col + bytes_to_advance
+    if should_advance then
+      if pstate.reader.lines.items[0].size == tgt then
+        eq(1, pstate.pos.line)
+        eq(0, pstate.pos.col)
+      else
+        eq(0, pstate.pos.line)
+        eq(tgt, pstate.pos.col)
       end
-      pstate = new_pstate({'x' .. str})
-      pstate.pos.col = 1
-      eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ},
-         next_eltkn(pstate))
+    else
+      eq(0, pstate.pos.line)
+      eq(initial_col, pstate.pos.col)
     end
+  end
+  local function singl_eltkn_test(typ, str, data)
+    local pstate = new_pstate({str})
+    eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ},
+       next_eltkn(pstate, flags))
+    check_advance(pstate, #str, 0)
+    if not (
+        typ == 'Spacing'
+        or (typ == 'Register' and str == '@')
+        or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString')
+            and not data.closed)
+    ) then
+      pstate = new_pstate({str .. ' '})
+      eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ},
+         next_eltkn(pstate, flags))
+      check_advance(pstate, #str, 0)
+    end
+    pstate = new_pstate({'x' .. str})
+    pstate.pos.col = 1
+    eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ},
+       next_eltkn(pstate, flags))
+    check_advance(pstate, #str, 1)
+  end
+  local function scope_test(scope)
+    singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope})
+    singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope})
+  end
+  local function comparison_test(op, inv_op, cmp_type)
+    singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'})
+    singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'})
+    singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'})
+    singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'})
+    singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'})
+    singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'})
+  end
+  local function simple_test(pstate_arg, exp_type, exp_len, exp)
+    local pstate = new_pstate(pstate_arg)
+    local exp = shallowcopy(exp)
+    exp.type = exp_type
+    exp.len = exp_len or #(pstate_arg[0])
+    exp.start = { col = 0, line = 0 }
+    eq(exp, next_eltkn(pstate, flags))
+  end
+  local function stable_tests()
     singl_eltkn_test('Parenthesis', '(', {closing=false})
     singl_eltkn_test('Parenthesis', ')', {closing=true})
     singl_eltkn_test('Bracket', '[', {closing=false})
@@ -170,9 +217,9 @@ describe('Expressions lexer', function()
     singl_eltkn_test('Spacing', ' ')
     singl_eltkn_test('Spacing', '\t')
     singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'})
-    singl_eltkn_test('Number', '0123')
-    singl_eltkn_test('Number', '0')
-    singl_eltkn_test('Number', '9')
+    singl_eltkn_test('Number', '0123', {is_float=false})
+    singl_eltkn_test('Number', '0', {is_float=false})
+    singl_eltkn_test('Number', '9', {is_float=false})
     singl_eltkn_test('Env', '$abc')
     singl_eltkn_test('Env', '$')
     singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0})
@@ -184,28 +231,8 @@ describe('Expressions lexer', function()
     singl_eltkn_test('PlainIdentifier', 'test#var', {autoload=true, scope=0})
     singl_eltkn_test('PlainIdentifier', 'test#var#val###', {autoload=true, scope=0})
     singl_eltkn_test('PlainIdentifier', 't#####', {autoload=true, scope=0})
-    local function scope_test(scope)
-      singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope})
-      singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope})
-    end
-    scope_test('s')
-    scope_test('g')
-    scope_test('v')
-    scope_test('b')
-    scope_test('w')
-    scope_test('t')
-    scope_test('l')
-    scope_test('a')
-    local function comparison_test(op, inv_op, cmp_type)
-      singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'})
-      singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'})
-      singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'})
-      singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'})
-      singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'})
-      singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'})
-    end
-    comparison_test('is', 'isnot', 'Identical')
     singl_eltkn_test('And', '&&')
+    singl_eltkn_test('Or', '||')
     singl_eltkn_test('Invalid', '&', {error='E112: Option name missing: %.*s'})
     singl_eltkn_test('Option', '&opt', {scope='Unspecified', name='opt'})
     singl_eltkn_test('Option', '&t_xx', {scope='Unspecified', name='t_xx'})
@@ -245,16 +272,134 @@ describe('Expressions lexer', function()
     comparison_test('>=', '<', 'GreaterOrEqual')
     singl_eltkn_test('Minus', '-')
     singl_eltkn_test('Arrow', '->')
+    singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'})
+    simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'})
+    simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'})
+    simple_test({'2.'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.x'}, 'Number', 1, {data={is_float=false}, str='2'})
+  end
+
+  local function regular_scope_tests()
+    scope_test('s')
+    scope_test('g')
+    scope_test('v')
+    scope_test('b')
+    scope_test('w')
+    scope_test('t')
+    scope_test('l')
+    scope_test('a')
+
+    simple_test({'g:'}, 'PlainIdentifier', 2, {data={scope='g', autoload=false}, str='g:'})
+    simple_test({'g:is#foo'}, 'PlainIdentifier', 8, {data={scope='g', autoload=true}, str='g:is#foo'})
+    simple_test({'g:isnot#foo'}, 'PlainIdentifier', 11, {data={scope='g', autoload=true}, str='g:isnot#foo'})
+  end
+
+  local function regular_is_tests()
+    comparison_test('is', 'isnot', 'Identical')
+
+    simple_test({'is'}, 'Comparison', 2, {data={type='Identical', inv=false, ccs='UseOption'}, str='is'})
+    simple_test({'isnot'}, 'Comparison', 5, {data={type='Identical', inv=true, ccs='UseOption'}, str='isnot'})
+    simple_test({'is?'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='IgnoreCase'}, str='is?'})
+    simple_test({'isnot?'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='IgnoreCase'}, str='isnot?'})
+    simple_test({'is#'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'})
+    simple_test({'isnot#'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'})
+    simple_test({'is#foo'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'})
+    simple_test({'isnot#foo'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'})
+  end
+
+  local function regular_number_tests()
+    simple_test({'2.0'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0x'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0e'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false}, str='2'})
+  end
+
+  local function regular_eoc_tests()
+    singl_eltkn_test('EOC', '|')
     singl_eltkn_test('EOC', '\0')
     singl_eltkn_test('EOC', '\n')
-    singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'})
+  end
+
+  itp('works (single tokens, zero flags)', function()
+    stable_tests()
+
+    regular_eoc_tests()
+    regular_scope_tests()
+    regular_is_tests()
+    regular_number_tests()
+  end)
+  itp('peeks', function()
+    flags = tonumber(lib.kELFlagPeek)
+    should_advance = false
+    stable_tests()
+
+    regular_eoc_tests()
+    regular_scope_tests()
+    regular_is_tests()
+    regular_number_tests()
+  end)
+  itp('forbids scope', function()
+    flags = tonumber(lib.kELFlagForbidScope)
+    stable_tests()
+
+    regular_eoc_tests()
+    regular_is_tests()
+    regular_number_tests()
+
+    simple_test({'g:'}, 'PlainIdentifier', 1, {data={scope=0, autoload=false}, str='g'})
+  end)
+  itp('allows floats', function()
+    flags = tonumber(lib.kELFlagAllowFloat)
+    stable_tests()
+
+    regular_eoc_tests()
+    regular_scope_tests()
+    regular_is_tests()
+
+    simple_test({'2.0'}, 'Number', 3, {data={is_float=true}, str='2.0'})
+    simple_test({'2.0x'}, 'Number', 3, {data={is_float=true}, str='2.0'})
+    simple_test({'2.0e'}, 'Number', 3, {data={is_float=true}, str='2.0'})
+    simple_test({'2.0e+'}, 'Number', 3, {data={is_float=true}, str='2.0'})
+    simple_test({'2.0e-'}, 'Number', 3, {data={is_float=true}, str='2.0'})
+    simple_test({'2.0e+x'}, 'Number', 3, {data={is_float=true}, str='2.0'})
+    simple_test({'2.0e-x'}, 'Number', 3, {data={is_float=true}, str='2.0'})
+    simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true}, str='2.0e5'})
+    simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true}, str='2.0e+5'})
+    simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true}, str='2.0e-5'})
+  end)
+  itp('treats `is` as an identifier', function()
+    flags = tonumber(lib.kELFlagIsNotCmp)
+    stable_tests()
+
+    regular_eoc_tests()
+    regular_scope_tests()
+    regular_number_tests()
+
+    simple_test({'is'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'})
+    simple_test({'isnot'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'})
+    simple_test({'is?'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'})
+    simple_test({'isnot?'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'})
+    simple_test({'is#'}, 'PlainIdentifier', 3, {data={scope=0, autoload=true}, str='is#'})
+    simple_test({'isnot#'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='isnot#'})
+    simple_test({'is#foo'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='is#foo'})
+    simple_test({'isnot#foo'}, 'PlainIdentifier', 9, {data={scope=0, autoload=true}, str='isnot#foo'})
+  end)
+  itp('forbids EOC', function()
+    flags = tonumber(lib.kELFlagForbidEOC)
+    stable_tests()
 
-    local pstate = new_pstate({{data=nil, size=0}})
-    eq({len=0, error='start.col >= #pstr', start={col=0, line=0}, type='EOC'},
-       next_eltkn(pstate))
+    regular_scope_tests()
+    regular_is_tests()
+    regular_number_tests()
 
-    local pstate = new_pstate({''})
-    eq({len=0, error='start.col >= #pstr', start={col=0, line=0}, type='EOC'},
-       next_eltkn(pstate))
+    singl_eltkn_test('Invalid', '|', {error='E15: Unexpected EOC character: %.*s'})
+    singl_eltkn_test('Invalid', '\0', {error='E15: Unexpected EOC character: %.*s'})
+    singl_eltkn_test('Invalid', '\n', {error='E15: Unexpected EOC character: %.*s'})
   end)
 end)