viml/parser/expressions: Make lexer parse numbers, support non-decimal

3 files changed, 221 insertions, 23 deletions

diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c
index a40488920e..409d7d443c 100644
--- a/test/symbolic/klee/nvim/charset.c
+++ b/test/symbolic/klee/nvim/charset.c
@@ -3,8 +3,173 @@
 #include "nvim/ascii.h"
 #include "nvim/macros.h"
 #include "nvim/charset.h"
+#include "nvim/eval/typval.h"
+#include "nvim/vim.h"
 
 bool vim_isIDc(int c)
 {
   return ASCII_ISALNUM(c);
 }
+
+int hex2nr(int c)
+{
+  if ((c >= 'a') && (c <= 'f')) {
+    return c - 'a' + 10;
+  }
+
+  if ((c >= 'A') && (c <= 'F')) {
+    return c - 'A' + 10;
+  }
+  return c - '0';
+}
+
+void vim_str2nr(const char_u *const start, int *const prep, int *const len,
+                const int what, varnumber_T *const nptr,
+                uvarnumber_T *const unptr, const int maxlen)
+{
+  const char_u *ptr = start;
+  int pre = 0;  // default is decimal
+  bool negative = false;
+  uvarnumber_T un = 0;
+
+  if (ptr[0] == '-') {
+    negative = true;
+    ptr++;
+  }
+
+  // Recognize hex, octal and bin.
+  if ((ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9')
+      && (maxlen == 0 || maxlen > 1)) {
+    pre = ptr[1];
+
+    if ((what & STR2NR_HEX)
+        && ((pre == 'X') || (pre == 'x'))
+        && ascii_isxdigit(ptr[2])
+        && (maxlen == 0 || maxlen > 2)) {
+      // hexadecimal
+      ptr += 2;
+    } else if ((what & STR2NR_BIN)
+               && ((pre == 'B') || (pre == 'b'))
+               && ascii_isbdigit(ptr[2])
+               && (maxlen == 0 || maxlen > 2)) {
+      // binary
+      ptr += 2;
+    } else {
+      // decimal or octal, default is decimal
+      pre = 0;
+
+      if (what & STR2NR_OCT) {
+        // Don't interpret "0", "08" or "0129" as octal.
+        for (int n = 1; ascii_isdigit(ptr[n]); ++n) {
+          if (ptr[n] > '7') {
+            // can't be octal
+            pre = 0;
+            break;
+          }
+          if (ptr[n] >= '0') {
+            // assume octal
+            pre = '0';
+          }
+          if (n == maxlen) {
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  // Do the string-to-numeric conversion "manually" to avoid sscanf quirks.
+  int n = 1;
+  if ((pre == 'B') || (pre == 'b') || what == STR2NR_BIN + STR2NR_FORCE) {
+    // bin
+    if (pre != 0) {
+      n += 2;  // skip over "0b"
+    }
+    while ('0' <= *ptr && *ptr <= '1') {
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 2) {
+        un = 2 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
+      ptr++;
+      if (n++ == maxlen) {
+        break;
+      }
+    }
+  } else if ((pre == '0') || what == STR2NR_OCT + STR2NR_FORCE) {
+    // octal
+    while ('0' <= *ptr && *ptr <= '7') {
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 8) {
+        un = 8 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
+      ptr++;
+      if (n++ == maxlen) {
+        break;
+      }
+    }
+  } else if ((pre == 'X') || (pre == 'x')
+             || what == STR2NR_HEX + STR2NR_FORCE) {
+    // hex
+    if (pre != 0) {
+      n += 2;  // skip over "0x"
+    }
+    while (ascii_isxdigit(*ptr)) {
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 16) {
+        un = 16 * un + (uvarnumber_T)hex2nr(*ptr);
+      } else {
+        un = UVARNUMBER_MAX;
+      }
+      ptr++;
+      if (n++ == maxlen) {
+        break;
+      }
+    }
+  } else {
+    // decimal
+    while (ascii_isdigit(*ptr)) {
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 10) {
+        un = 10 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
+      ptr++;
+      if (n++ == maxlen) {
+        break;
+      }
+    }
+  }
+
+  if (prep != NULL) {
+    *prep = pre;
+  }
+
+  if (len != NULL) {
+    *len = (int)(ptr - start);
+  }
+
+  if (nptr != NULL) {
+    if (negative) {  // account for leading '-' for decimal numbers
+      // avoid ubsan error for overflow
+      if (un > VARNUMBER_MAX) {
+        *nptr = VARNUMBER_MIN;
+      } else {
+        *nptr = -(varnumber_T)un;
+      }
+    } else {
+      if (un > VARNUMBER_MAX) {
+        un = VARNUMBER_MAX;
+      }
+      *nptr = (varnumber_T)un;
+    }
+  }
+
+  if (unptr != NULL) {
+    *unptr = un;
+  }
+}
diff --git a/test/symbolic/klee/viml_expressions_lexer.c b/test/symbolic/klee/viml_expressions_lexer.c
index 67f3eb7faa..cddc1cb2f1 100644
--- a/test/symbolic/klee/viml_expressions_lexer.c
+++ b/test/symbolic/klee/viml_expressions_lexer.c
@@ -2,6 +2,7 @@
 # include <klee/klee.h>
 #else
 # include <string.h>
+# include <stdio.h>
 #endif
 #include <stddef.h>
 #include <stdint.h>
@@ -56,7 +57,7 @@ int main(const int argc, const char *const *const argv,
       .data = &input[shift],
       .size = sizeof(input) - shift,
 #else
-      .data = (const char *)&argv[1],
+      .data = (const char *)argv[1],
       .size = strlen(argv[1]),
 #endif
       .allocated = false,
@@ -97,4 +98,7 @@ int main(const int argc, const char *const *const argv,
   }
   assert(allocated_memory == 0);
   assert(ever_allocated_memory == 0);
+#ifndef USE_KLEE
+  fprintf(stderr, "tkn: %s\n", viml_pexpr_repr_token(&pstate, token, NULL));
+#endif
 }
diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua
index bd8045632e..f180d8ceff 100644
--- a/test/unit/viml/expressions/lexer_spec.lua
+++ b/test/unit/viml/expressions/lexer_spec.lua
@@ -114,7 +114,11 @@ local function eltkn2lua(pstate, tkn)
   elseif ret.type == 'Number' then
     ret.data = {
       is_float = (not not tkn.data.num.is_float),
+      base = tonumber(tkn.data.num.base),
     }
+    ret.data.val = tonumber(tkn.data.num.is_float
+                            and tkn.data.num.val.floating
+                            or tkn.data.num.val.integer)
   elseif ret.type == 'Invalid' then
     ret.data = { error = ffi.string(tkn.data.err.msg) }
   end
@@ -204,9 +208,20 @@ describe('Expressions lexer', function()
     singl_eltkn_test('Spacing', ' ')
     singl_eltkn_test('Spacing', '\t')
     singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'})
-    singl_eltkn_test('Number', '0123', {is_float=false})
-    singl_eltkn_test('Number', '0', {is_float=false})
-    singl_eltkn_test('Number', '9', {is_float=false})
+    singl_eltkn_test('Number', '0123', {is_float=false, base=8, val=83})
+    singl_eltkn_test('Number', '01234567', {is_float=false, base=8, val=342391})
+    singl_eltkn_test('Number', '012345678', {is_float=false, base=10, val=12345678})
+    singl_eltkn_test('Number', '0x123', {is_float=false, base=16, val=291})
+    singl_eltkn_test('Number', '0x56FF', {is_float=false, base=16, val=22271})
+    singl_eltkn_test('Number', '0xabcdef', {is_float=false, base=16, val=11259375})
+    singl_eltkn_test('Number', '0xABCDEF', {is_float=false, base=16, val=11259375})
+    singl_eltkn_test('Number', '0x0', {is_float=false, base=16, val=0})
+    singl_eltkn_test('Number', '00', {is_float=false, base=8, val=0})
+    singl_eltkn_test('Number', '0b0', {is_float=false, base=2, val=0})
+    singl_eltkn_test('Number', '0b010111', {is_float=false, base=2, val=23})
+    singl_eltkn_test('Number', '0b100111', {is_float=false, base=2, val=39})
+    singl_eltkn_test('Number', '0', {is_float=false, base=10, val=0})
+    singl_eltkn_test('Number', '9', {is_float=false, base=10, val=9})
     singl_eltkn_test('Env', '$abc')
     singl_eltkn_test('Env', '$')
     singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0})
@@ -262,17 +277,21 @@ describe('Expressions lexer', function()
     singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'})
     simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'})
     simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'})
-    simple_test({'2.'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.x'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.2.'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0x'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'0b102'}, 'Number', 4, {data={is_float=false, base=2, val=2}, str='0b10'})
+    simple_test({'10F'}, 'Number', 2, {data={is_float=false, base=10, val=10}, str='10'})
+    simple_test({'0x0123456789ABCDEFG'}, 'Number', 18, {data={is_float=false, base=16, val=81985529216486895}, str='0x0123456789ABCDEF'})
   end
 
   local function regular_scope_tests()
@@ -304,10 +323,10 @@ describe('Expressions lexer', function()
   end
 
   local function regular_number_tests()
-    simple_test({'2.0'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false}, str='2'})
-    simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false}, str='2'})
+    simple_test({'2.0'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
   end
 
   local function regular_eoc_tests()
@@ -352,10 +371,20 @@ describe('Expressions lexer', function()
     regular_scope_tests()
     regular_is_tests()
 
-    simple_test({'2.0'}, 'Number', 3, {data={is_float=true}, str='2.0'})
-    simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true}, str='2.0e5'})
-    simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true}, str='2.0e+5'})
-    simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true}, str='2.0e-5'})
+    simple_test({'2.2'}, 'Number', 3, {data={is_float=true, base=10, val=2.2}, str='2.2'})
+    simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true, base=10, val=2e5}, str='2.0e5'})
+    simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true, base=10, val=2e5}, str='2.0e+5'})
+    simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true, base=10, val=2e-5}, str='2.0e-5'})
+    simple_test({'2.500000e-5'}, 'Number', 11, {data={is_float=true, base=10, val=2.5e-5}, str='2.500000e-5'})
+    simple_test({'2.5555e2'}, 'Number', 8, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e2'})
+    simple_test({'2.5555e+2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e+2'})
+    simple_test({'2.5555e-2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e-2}, str='2.5555e-2'})
+    simple_test({{data='2.5e-5', size=3}},
+                'Number', 3, {data={is_float=true, base=10, val=2.5}, str='2.5'})
+    simple_test({{data='2.5e5', size=4}},
+                'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
+    simple_test({{data='2.5e-50', size=6}},
+                'Number', 6, {data={is_float=true, base=10, val=2.5e-5}, str='2.5e-5'})
   end)
   itp('treats `is` as an identifier', function()
     flags = tonumber(lib.kELFlagIsNotCmp)