viml/parser/expressions: Make lexer parse numbers, support non-decimal

5 files changed, 362 insertions, 34 deletions

diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c
index 8c95d1db14..5d892fb8f8 100644
--- a/src/nvim/viml/parser/expressions.c
+++ b/src/nvim/viml/parser/expressions.c
@@ -15,10 +15,13 @@
 #include "nvim/ascii.h"
 #include "nvim/assert.h"
 #include "nvim/lib/kvec.h"
+#include "nvim/eval/typval.h"
 
 #include "nvim/viml/parser/expressions.h"
 #include "nvim/viml/parser/parser.h"
 
+#define vim_str2nr(s, ...) vim_str2nr((const char_u *)(s), __VA_ARGS__)
+
 typedef kvec_withinit_t(ExprASTNode **, 16) ExprASTStack;
 
 /// Which nodes may be wanted
@@ -72,6 +75,43 @@ typedef enum {
 /// Character used as a separator in autoload function/variable names.
 #define AUTOLOAD_CHAR '#'
 
+/// Scale number by a given factor
+///
+/// Used to apply exponent to a number. Idea taken from uClibc.
+///
+/// @param[in]  num  Number to scale. Does not bother doing anything if it is
+///                  zero.
+/// @param[in]  base  Base, should be 10 since non-decimal floating-point
+///                   numbers are not supported.
+/// @param[in]  exponent  Exponent to scale by.
+/// @param[in]  exponent_negative  True if exponent is negative.
+static inline float_T scale_number(const float_T num,
+                                   const uint8_t base,
+                                   const uvarnumber_T exponent,
+                                   const bool exponent_negative)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_CONST
+{
+  if (num == 0 || exponent == 0) {
+    return num;
+  }
+  assert(base);
+  uvarnumber_T exp = exponent;
+  float_T p_base = (float_T)base;
+  float_T ret = num;
+  while (exp) {
+    if (exp & 1) {
+      if (exponent_negative) {
+        ret /= p_base;
+      } else {
+        ret *= p_base;
+      }
+    }
+    exp >>= 1;
+    p_base *= p_base;
+  }
+  return ret;
+}
+
 /// Get next token for the VimL expression input
 ///
 /// @param  pstate  Parser state.
@@ -184,6 +224,11 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags)
     case '0': case '1': case '2': case '3': case '4': case '5': case '6':
     case '7': case '8': case '9': {
       ret.data.num.is_float = false;
+      ret.data.num.base = 10;
+      size_t frac_start = 0;
+      size_t exp_start = 0;
+      size_t frac_end = 0;
+      bool exp_negative = false;
       CHARREG(kExprLexNumber, ascii_isdigit);
       if (flags & kELFlagAllowFloat) {
         const LexExprToken non_float_ret = ret;
@@ -191,8 +236,18 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags)
             && pline.data[ret.len] == '.'
             && ascii_isdigit(pline.data[ret.len + 1])) {
           ret.len++;
+          frac_start = ret.len;
+          frac_end = ret.len;
           ret.data.num.is_float = true;
-          CHARREG(kExprLexNumber, ascii_isdigit);
+          for (; ret.len < pline.size && ascii_isdigit(pline.data[ret.len])
+               ; ret.len++) {
+            // A small optimization: trailing zeroes in fractional part do not
+            // add anything to significand, so it is useless to include them in
+            // frac_end.
+            if (pline.data[ret.len] != '0') {
+              frac_end = ret.len + 1;
+            }
+          }
           if (pline.size > ret.len + 1
               && (pline.data[ret.len] == 'e'
                   || pline.data[ret.len] == 'E')
@@ -202,9 +257,11 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags)
                    && ascii_isdigit(pline.data[ret.len + 2]))
                   || ascii_isdigit(pline.data[ret.len + 1]))) {
             ret.len++;
-            if (pline.data[ret.len] == '+' || pline.data[ret.len] == '-') {
+            if (pline.data[ret.len] == '+'
+                || (exp_negative = (pline.data[ret.len] == '-'))) {
               ret.len++;
             }
+            exp_start = ret.len;
             CHARREG(kExprLexNumber, ascii_isdigit);
           }
         }
@@ -214,6 +271,58 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags)
           ret = non_float_ret;
         }
       }
+      // TODO(ZyX-I): detect overflows
+      if (ret.data.num.is_float) {
+        // Vim used to use string2float here which in turn uses strtod(). There
+        // are two problems with this approach:
+        // 1. strtod() is locale-dependent. Not sure how it is worked around so
+        //    that I do not see relevant bugs, but it still does not look like
+        //    a good idea.
+        // 2. strtod() does not accept length argument.
+        //
+        // The below variant of parsing floats was recognized as acceptable
+        // because it is basically how uClibc does the thing: it generates
+        // a number ignoring decimal point (but recording its position), then
+        // uses recorded position to scale number down when processing exponent.
+        float_T significand_part = 0;
+        uvarnumber_T exp_part = 0;
+        const size_t frac_size = (size_t)(frac_end - frac_start);
+        for (size_t i = 0; i < frac_end; i++) {
+          if (i == frac_start - 1) {
+            continue;
+          }
+          significand_part = significand_part * 10 + (pline.data[i] - '0');
+        }
+        if (exp_start) {
+          vim_str2nr(pline.data + exp_start, NULL, NULL, 0, NULL, &exp_part,
+                     (int)(ret.len - exp_start));
+        }
+        if (exp_negative) {
+          exp_part += frac_size;
+        } else {
+          if (exp_part < frac_size) {
+            exp_negative = true;
+            exp_part = frac_size - exp_part;
+          } else {
+            exp_part -= frac_size;
+          }
+        }
+        ret.data.num.val.floating = scale_number(significand_part, 10, exp_part,
+                                                 exp_negative);
+      } else {
+        int len;
+        int prep;
+        vim_str2nr(pline.data, &prep, &len, STR2NR_ALL, NULL,
+                   &ret.data.num.val.integer, (int)pline.size);
+        ret.len = (size_t)len;
+        const uint8_t bases[] = {
+          [0] = 10,
+          ['0'] = 8,
+          ['x'] = 16, ['X'] = 16,
+          ['b'] = 2, ['B'] = 2,
+        };
+        ret.data.num.base = bases[prep];
+      }
       break;
     }
 
@@ -474,7 +583,6 @@ viml_pexpr_next_token_adv_return:
   return ret;
 }
 
-#ifdef UNIT_TESTING
 static const char *const eltkn_type_tab[] = {
   [kExprLexInvalid] = "Invalid",
   [kExprLexMissing] = "Missing",
@@ -617,7 +725,12 @@ const char *viml_pexpr_repr_token(const ParserState *const pstate,
             (int)token.data.opt.len, token.data.opt.name)
     TKNARGS(kExprLexPlainIdentifier, "(scope=%s,autoload=%i)",
             intchar2str(token.data.var.scope), (int)token.data.var.autoload)
-    TKNARGS(kExprLexNumber, "(is_float=%i)", (int)token.data.num.is_float)
+    TKNARGS(kExprLexNumber, "(is_float=%i,base=%i,val=%lg)",
+            (int)token.data.num.is_float,
+            (int)token.data.num.base,
+            (double)(token.data.num.is_float
+                     ? token.data.num.val.floating
+                     : token.data.num.val.integer))
     TKNARGS(kExprLexInvalid, "(msg=%s)", token.data.err.msg)
     default: {
       // No additional arguments.
@@ -642,7 +755,6 @@ viml_pexpr_repr_token_end:
   }
   return ret;
 }
-#endif
 
 #ifdef UNIT_TESTING
 #include <stdio.h>
@@ -776,8 +888,10 @@ static inline void viml_pexpr_debug_print_token(
 // NVimOperator -> Operator
 // NVimUnaryOperator -> NVimOperator
 // NVimBinaryOperator -> NVimOperator
+//
 // NVimComparisonOperator -> NVimBinaryOperator
 // NVimComparisonOperatorModifier -> NVimComparisonOperator
+//
 // NVimTernary -> NVimOperator
 // NVimTernaryColon -> NVimTernary
 //
@@ -795,8 +909,21 @@ static inline void viml_pexpr_debug_print_token(
 // NVimIdentifierScope -> NVimIdentifier
 // NVimIdentifierScopeDelimiter -> NVimIdentifier
 //
+// NVimIdentifierKey -> Identifier
+//
 // NVimFigureBrace -> NVimInternalError
 //
+// NVimUnaryPlus -> NVimUnaryOperator
+// NVimBinaryPlus -> NVimBinaryOperator
+// NVimConcatOrSubscript -> NVimBinaryOperator
+//
+// NVimRegister -> SpecialChar
+// NVimNumber -> Number
+// NVimFloat -> NVimNumber
+//
+// NVimNestingParenthesis -> NVimParenthesis
+// NVimCallingParenthesis -> NVimParenthesis
+//
 // NVimInvalidComma -> NVimInvalidDelimiter
 // NVimInvalidSpacing -> NVimInvalid
 // NVimInvalidTernary -> NVimInvalidOperator
@@ -814,12 +941,9 @@ static inline void viml_pexpr_debug_print_token(
 // NVimInvalidIdentifierScopeDelimiter -> NVimInvalidValue
 // NVimInvalidComparisonOperator -> NVimInvalidOperator
 // NVimInvalidComparisonOperatorModifier -> NVimInvalidComparisonOperator
-//
-// NVimUnaryPlus -> NVimUnaryOperator
-// NVimBinaryPlus -> NVimBinaryOperator
-// NVimRegister -> SpecialChar
-// NVimNestingParenthesis -> NVimParenthesis
-// NVimCallingParenthesis -> NVimParenthesis
+// NVimInvalidNumber -> NVimInvalidValue
+// NVimInvalidFloat -> NVimInvalidValue
+// NVimInvalidIdentifierKey -> NVimInvalidIdentifier
 
 /// Allocate a new node and set some of the values
 ///
diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h
index 8ca3ceacb9..29903490bb 100644
--- a/src/nvim/viml/parser/expressions.h
+++ b/src/nvim/viml/parser/expressions.h
@@ -7,6 +7,7 @@
 
 #include "nvim/types.h"
 #include "nvim/viml/parser/parser.h"
+#include "nvim/eval/typval.h"
 
 // Defines whether to ignore case:
 //    ==   kCCStrategyUseOption
@@ -113,6 +114,11 @@ typedef struct {
     } err;  ///< For kExprLexInvalid
 
     struct {
+      union {
+        float_T floating;
+        uvarnumber_T integer;
+      } val;  ///< Number value.
+      uint8_t base;  ///< Base: 2, 8, 10 or 16.
       bool is_float;  ///< True if number is a floating-point.
     } num;  ///< For kExprLexNumber
   } data;  ///< Additional data, if needed.
diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c
index a40488920e..409d7d443c 100644
--- a/test/symbolic/klee/nvim/charset.c
+++ b/test/symbolic/klee/nvim/charset.c
@@ -3,8 +3,173 @@
 #include "nvim/ascii.h"
 #include "nvim/macros.h"
 #include "nvim/charset.h"
+#include "nvim/eval/typval.h"
+#include "nvim/vim.h"
 
 bool vim_isIDc(int c)
 {
   return ASCII_ISALNUM(c);
 }
+
+int hex2nr(int c)
+{
+  if ((c >= 'a') && (c <= 'f')) {
+    return c - 'a' + 10;
+  }
+
+  if ((c >= 'A') && (c <= 'F')) {
+    return c - 'A' + 10;
+  }
+  return c - '0';
+}
+
+void vim_str2nr(const char_u *const start, int *const prep, int *const len,
+                const int what, varnumber_T *const nptr,
+                uvarnumber_T *const unptr, const int maxlen)
+{
+  const char_u *ptr = start;
+  int pre = 0;  // default is decimal
+  bool negative = false;
+  uvarnumber_T un = 0;
+
+  if (ptr[0] == '-') {
+    negative = true;
+    ptr++;
+  }
+
+  // Recognize hex, octal and bin.
+  if ((ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9')
+      && (maxlen == 0 || maxlen > 1)) {
+    pre = ptr[1];
+
+    if ((what & STR2NR_HEX)
+        && ((pre == 'X') || (pre == 'x'))
+        && ascii_isxdigit(ptr[2])
+        && (maxlen == 0 || maxlen > 2)) {
+      // hexadecimal
+      ptr += 2;
+    } else if ((what & STR2NR_BIN)
+               && ((pre == 'B') || (pre == 'b'))
+               && ascii_isbdigit(ptr[2])
+               && (maxlen == 0 || maxlen > 2)) {
+      // binary
+      ptr += 2;
+    } else {
+      // decimal or octal, default is decimal
+      pre = 0;
+
+      if (what & STR2NR_OCT) {
+        // Don't interpret "0", "08" or "0129" as octal.
+        for (int n = 1; ascii_isdigit(ptr[n]); ++n) {
+          if (ptr[n] > '7') {
+            // can't be octal
+            pre = 0;
+            break;
+          }
+          if (ptr[n] >= '0') {
+            // assume octal
+            pre = '0';
+          }
+          if (n == maxlen) {
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  // Do the string-to-numeric conversion "manually" to avoid sscanf quirks.
+  int n = 1;
+  if ((pre == 'B') || (pre == 'b') || what == STR2NR_BIN + STR2NR_FORCE) {
+    // bin
+    if (pre != 0) {
+      n += 2;  // skip over "0b"
+    }
+    while ('0' <= *ptr && *ptr <= '1') {
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 2) {
+        un = 2 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
+      ptr++;
+      if (n++ == maxlen) {
+        break;
+      }
+    }
+  } else if ((pre == '0') || what == STR2NR_OCT + STR2NR_FORCE) {
+    // octal
+    while ('0' <= *ptr && *ptr <= '7') {
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 8) {
+        un = 8 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
+      ptr++;
+      if (n++ == maxlen) {
+        break;
+      }
+    }
+  } else if ((pre == 'X') || (pre == 'x')
+             || what == STR2NR_HEX + STR2NR_FORCE) {
+    // hex
+    if (pre != 0) {
+      n += 2;  // skip over "0x"
+    }
+    while (ascii_isxdigit(*ptr)) {
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 16) {
+        un = 16 * un + (uvarnumber_T)hex2nr(*ptr);
+      } else {
+        un = UVARNUMBER_MAX;
+      }
+      ptr++;
+      if (n++ == maxlen) {
+        break;
+      }
+    }
+  } else {
+    // decimal
+    while (ascii_isdigit(*ptr)) {
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 10) {
+        un = 10 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
+      ptr++;
+      if (n++ == maxlen) {
+        break;
+      }
+    }
+  }
+
+  if (prep != NULL) {
+    *prep = pre;
+  }
+
+  if (len != NULL) {
+    *len = (int)(ptr - start);
+  }
+
+  if (nptr != NULL) {
+    if (negative) {  // account for leading '-' for decimal numbers
+      // avoid ubsan error for overflow
+      if (un > VARNUMBER_MAX) {
+        *nptr = VARNUMBER_MIN;
+      } else {
+        *nptr = -(varnumber_T)un;
+      }
+    } else {
+      if (un > VARNUMBER_MAX) {
+        un = VARNUMBER_MAX;
+      }
+      *nptr = (varnumber_T)un;
+    }
+  }
+
+  if (unptr != NULL) {
+    *unptr = un;
+  }
+}
diff --git a/test/symbolic/klee/viml_expressions_lexer.c b/test/symbolic/klee/viml_expressions_lexer.c
index 67f3eb7faa..cddc1cb2f1 100644
--- a/test/symbolic/klee/viml_expressions_lexer.c
+++ b/test/symbolic/klee/viml_expressions_lexer.c
@@ -2,6 +2,7 @@
 # include <klee/klee.h>
 #else
 # include <string.h>
+# include <stdio.h>
 #endif
 #include <stddef.h>
 #include <stdint.h>
@@ -56,7 +57,7 @@ int main(const int argc, const char *const *const argv,
       .data = &input[shift],
       .size = sizeof(input) - shift,
 #else
-      .data = (const char *)&argv[1],
+      .data = (const char *)argv[1],
       .size = strlen(argv[1]),
 #endif
       .allocated = false,
@@ -97,4 +98,7 @@ int main(const int argc, const char *const *const argv,
   }
   assert(allocated_memory == 0);
   assert(ever_allocated_memory == 0);
+#ifndef USE_KLEE
+  fprintf(stderr, "tkn: %s\n", viml_pexpr_repr_token(&pstate, token, NULL));
+#endif
 }
diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua
index bd8045632e..f180d8ceff 100644
--- a/test/unit/viml/expressions/lexer_spec.lua
+++ b/test/unit/viml/expressions/lexer_spec.lua
@@ -114,7 +114,11 @@ local function eltkn2lua(pstate, tkn)
   elseif ret.type == 'Number' then
     ret.data = {
       is_float = (not not tkn.data.num.is_float),
+      base = tonumber(tkn.data.num.base),
     }
+    ret.data.val = tonumber(tkn.data.num.is_float
+                            and tkn.data.num.val.floating
+                            or tkn.data.num.val.integer)
   elseif ret.type == 'Invalid' then
     ret.data = { error = ffi.string(tkn.data.err.msg) }
   end
@@ -204,9 +208,20 @@ describe('Expressions lexer', function()
     singl_eltkn_test('Spacing', ' ')
     singl_eltkn_test('Spacing', '\t')
     singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'})
-    singl_eltkn_test('Number', '0123', {is_float=false})
-    singl_eltkn_test('Number', '0', {is_float=false})
-    singl_eltkn_test('Number', '9', {is_float=false})
+    singl_eltkn_test('Number', '0123', {is_float=false, base=8, val=83})
+    singl_eltkn_test('Number', '01234567', {is_float=false, base=8, val=342391})
+    singl_eltkn_test('Number', '012345678', {is_float=false, base=10, val=12345678})
+    singl_eltkn_test('Number', '0x123', {is_float=false, base=16, val=291})
+    singl_eltkn_test('Number', '0x56FF', {is_float=false, base=16, val=22271})
+    singl_eltkn_test('Number', '0xabcdef', {is_float=false, base=16, val=11259375})
+    singl_eltkn_test('Number', '0xABCDEF', {is_float=false, base=16, val=11259375})
+    singl_eltkn_test('Number', '0x0', {is_float=false, base=16, val=0})
+    singl_eltkn_test('Number', '00', {is_float=false, base=8, val=0})
+    singl_eltkn_test('Number', '0b0', {is_float=false, base=2, val=0})
+    singl_eltkn_test('Number', '0b010111', {is_float=false, base=2, val=23})
+    singl_eltkn_test('Number', '0b100111', {is_float=false, base=2, val=39})
+    singl_eltkn_test('Number', '0', {is_float=false, base=10, val=0})
+    singl_eltkn_test('Number', '9', {is_float=false, base=10, val=9})
     singl_eltkn_test('Env', '$abc')
     singl_eltkn_test('Env', '$')
     singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0})
@@ -262,17 +277,21 @@ describe('Expressions lexer', function()
     singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'})
     simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'})
     simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'})
-    simple_test({'2.'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.x'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.2.'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0x'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'0b102'}, 'Number', 4, {data={is_float=false, base=2, val=2}, str='0b10'})
+    simple_test({'10F'}, 'Number', 2, {data={is_float=false, base=10, val=10}, str='10'})
+    simple_test({'0x0123456789ABCDEFG'}, 'Number', 18, {data={is_float=false, base=16, val=81985529216486895}, str='0x0123456789ABCDEF'})
   end
 
   local function regular_scope_tests()
@@ -304,10 +323,10 @@ describe('Expressions lexer', function()
   end
 
   local function regular_number_tests()
-    simple_test({'2.0'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
   end
 
   local function regular_eoc_tests()
@@ -352,10 +371,20 @@ describe('Expressions lexer', function()
     regular_scope_tests()
     regular_is_tests()
 
-    simple_test({'2.0'}, 'Number', 3, {data={is_float=true}, str='2.0'})
-    simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true}, str='2.0e5'})
-    simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true}, str='2.0e+5'})
-    simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true}, str='2.0e-5'})
+    simple_test({'2.2'}, 'Number', 3, {data={is_float=true, base=10, val=2.2}, str='2.2'})
+    simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true, base=10, val=2e5}, str='2.0e5'})
+    simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true, base=10, val=2e5}, str='2.0e+5'})
+    simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true, base=10, val=2e-5}, str='2.0e-5'})
+    simple_test({'2.500000e-5'}, 'Number', 11, {data={is_float=true, base=10, val=2.5e-5}, str='2.500000e-5'})
+    simple_test({'2.5555e2'}, 'Number', 8, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e2'})
+    simple_test({'2.5555e+2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e+2'})
+    simple_test({'2.5555e-2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e-2}, str='2.5555e-2'})
+    simple_test({{data='2.5e-5', size=3}},
+                'Number', 3, {data={is_float=true, base=10, val=2.5}, str='2.5'})
+    simple_test({{data='2.5e5', size=4}},
+                'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({{data='2.5e-50', size=6}},
+                'Number', 6, {data={is_float=true, base=10, val=2.5e-5}, str='2.5e-5'})
   end)
   itp('treats `is` as an identifier', function()
     flags = tonumber(lib.kELFlagIsNotCmp)