vim-patch:8.0.0219

Problem:    Ubsan reports errors for integer overflow.
Solution:   Define macros for minimum and maximum values.  Select an
            expression based on the value. (Mike Williams)

https://github.com/vim/vim/commit/7a40ea2138102545848ea86a361f1b8dec7552b5

1 files changed, 34 insertions, 7 deletions

diff --git a/src/nvim/charset.c b/src/nvim/charset.c
index bf4dc25efd..48db1030a6 100644
--- a/src/nvim/charset.c
+++ b/src/nvim/charset.c
@@ -1683,7 +1683,12 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len,
       n += 2;  // skip over "0b"
     }
     while ('0' <= *ptr && *ptr <= '1') {
-      un = 2 * un + (uvarnumber_T)(*ptr - '0');
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 2) {
+        un = 2 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
       ptr++;
       if (n++ == maxlen) {
         break;
@@ -1692,7 +1697,12 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len,
   } else if ((pre == '0') || what == STR2NR_OCT + STR2NR_FORCE) {
     // octal
     while ('0' <= *ptr && *ptr <= '7') {
-      un = 8 * un + (uvarnumber_T)(*ptr - '0');
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 8) {
+        un = 8 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
       ptr++;
       if (n++ == maxlen) {
         break;
@@ -1705,7 +1715,12 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len,
       n += 2;  // skip over "0x"
     }
     while (ascii_isxdigit(*ptr)) {
-      un = 16 * un + (uvarnumber_T)hex2nr(*ptr);
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 16) {
+        un = 16 * un + (uvarnumber_T)hex2nr(*ptr);
+      } else {
+        un = UVARNUMBER_MAX;
+      }
       ptr++;
       if (n++ == maxlen) {
         break;
@@ -1714,7 +1729,12 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len,
   } else {
     // decimal
     while (ascii_isdigit(*ptr)) {
-      un = 10 * un + (unsigned long)(*ptr - '0');
+      // avoid ubsan error for overflow
+      if (un < UVARNUMBER_MAX / 10) {
+        un = 10 * un + (uvarnumber_T)(*ptr - '0');
+      } else {
+        un = UVARNUMBER_MAX;
+      }
       ptr++;
       if (n++ == maxlen) {
         break;
@@ -1731,10 +1751,17 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len,
   }
 
   if (nptr != NULL) {
-    if (negative) {
-      // account for leading '-' for decimal numbers
-      *nptr = -(varnumber_T)un;
+    if (negative) {  // account for leading '-' for decimal numbers
+      // avoid ubsan error for overflow
+      if (un > VARNUMBER_MAX) {
+        *nptr = VARNUMBER_MIN;
+      } else {
+        *nptr = -(varnumber_T)un;
+      }
     } else {
+      if (un > VARNUMBER_MAX) {
+        un = VARNUMBER_MAX;
+      }
       *nptr = (varnumber_T)un;
     }
   }