vim-patch:8.0.0219

Problem:    Ubsan reports errors for integer overflow.
Solution:   Define macros for minimum and maximum values.  Select an
            expression based on the value. (Mike Williams)

https://github.com/vim/vim/commit/7a40ea2138102545848ea86a361f1b8dec7552b5

3 files changed, 40 insertions, 13 deletions

diff --git a/src/nvim/charset.c b/src/nvim/charset.c
index bf4dc25efd..48db1030a6 100644
--- a/src/nvim/charset.c
+++ b/src/nvim/charset.c
@@ -1683,7 +1683,12 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len,
       n += 2;  // skip over "0b"
     }
     while ('0' <= *ptr && *ptr <= '1') {
-      un = 2 * un + (uvarnumber_T)(*ptr - '0');
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 2) {
+        un = 2 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
       ptr++;
       if (n++ == maxlen) {
         break;
@@ -1692,7 +1697,12 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len,
   } else if ((pre == '0') || what == STR2NR_OCT + STR2NR_FORCE) {
     // octal
     while ('0' <= *ptr && *ptr <= '7') {
-      un = 8 * un + (uvarnumber_T)(*ptr - '0');
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 8) {
+        un = 8 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
       ptr++;
       if (n++ == maxlen) {
         break;
@@ -1705,7 +1715,12 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len,
       n += 2;  // skip over "0x"
     }
     while (ascii_isxdigit(*ptr)) {
-      un = 16 * un + (uvarnumber_T)hex2nr(*ptr);
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 16) {
+        un = 16 * un + (uvarnumber_T)hex2nr(*ptr);
+      } else {
+        un = UVARNUMBER_MAX;
+      }
       ptr++;
       if (n++ == maxlen) {
         break;
@@ -1714,7 +1729,12 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len,
   } else {
     // decimal
     while (ascii_isdigit(*ptr)) {
-      un = 10 * un + (unsigned long)(*ptr - '0');
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 10) {
+        un = 10 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
       ptr++;
       if (n++ == maxlen) {
         break;
@@ -1731,10 +1751,17 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len,
   }
 
   if (nptr != NULL) {
-    if (negative) {
-      // account for leading '-' for decimal numbers
-      *nptr = -(varnumber_T)un;
+    if (negative) {  // account for leading '-' for decimal numbers
+      // avoid ubsan error for overflow
+      if (un > VARNUMBER_MAX) {
+        *nptr = VARNUMBER_MIN;
+      } else {
+        *nptr = -(varnumber_T)un;
+      }
     } else {
+      if (un > VARNUMBER_MAX) {
+        un = VARNUMBER_MAX;
+      }
       *nptr = (varnumber_T)un;
     }
   }
diff --git a/src/nvim/eval.c b/src/nvim/eval.c
index 19e23190b1..785643b5e0 100644
--- a/src/nvim/eval.c
+++ b/src/nvim/eval.c
@@ -4032,11 +4032,11 @@ eval6 (
         else if (op == '/') {
           if (n2 == 0) {                /* give an error message? */
             if (n1 == 0)
-              n1 = -0x7fffffffffffffff - 1; /* similar to NaN */
+              n1 = VARNUMBER_MIN; /* similar to NaN */
             else if (n1 < 0)
-              n1 = -0x7fffffffffffffff;
+              n1 = -VARNUMBER_MAX;
             else
-              n1 = 0x7fffffffffffffff;
+              n1 = VARNUMBER_MAX;
           } else
             n1 = n1 / n2;
         } else {
@@ -8539,8 +8539,8 @@ static void f_float2nr(typval_T *argvars, typval_T *rettv, FunPtr fptr)
   float_T f;
 
   if (tv_get_float_chk(argvars, &f)) {
-    if (f < VARNUMBER_MIN) {
-      rettv->vval.v_number = VARNUMBER_MIN;
+    if (f < -VARNUMBER_MAX) {
+      rettv->vval.v_number = -VARNUMBER_MAX;
     } else if (f > VARNUMBER_MAX) {
       rettv->vval.v_number = VARNUMBER_MAX;
     } else {
diff --git a/src/nvim/testdir/test_viml.vim b/src/nvim/testdir/test_viml.vim
index 55d0dc21a3..9c1bc802a0 100644
--- a/src/nvim/testdir/test_viml.vim
+++ b/src/nvim/testdir/test_viml.vim
@@ -1078,7 +1078,7 @@ func Test_num64()
 
     call assert_equal( 9223372036854775807,  1 / 0)
     call assert_equal(-9223372036854775807, -1 / 0)
-    call assert_equal(-9223372036854775808,  0 / 0)
+    call assert_equal(-9223372036854775807 - 1,  0 / 0)
 
     call assert_equal( 0x7FFFffffFFFFffff, float2nr( 1.0e150))
     call assert_equal(-0x7FFFffffFFFFffff, float2nr(-1.0e150))