modeline: Handle version number overflow. #5450

Closes #5449

A file containing the string "vim" followed by a very large number in a modeline
location will trigger an overflow in getdigits() which is called by
chk_modeline() when trying to parse the version number.

Add getdigits_safe(), which does not assert overflows, but reports them to the
caller.

2 files changed, 35 insertions, 14 deletions

diff --git a/src/nvim/buffer.c b/src/nvim/buffer.c
index 5fb011885e..a573c20648 100644
--- a/src/nvim/buffer.c
+++ b/src/nvim/buffer.c
@@ -4509,7 +4509,7 @@ chk_modeline (
   char_u      *e;
   char_u      *linecopy;                /* local copy of any modeline found */
   int prev;
-  int vers;
+  intmax_t vers;
   int end;
   int retval = OK;
   char_u      *save_sourcing_name;
@@ -4528,7 +4528,10 @@ chk_modeline (
           e = s + 4;
         else
           e = s + 3;
-        vers = getdigits_int(&e);
+        if (getdigits_safe(&e, &vers) != OK) {
+          continue;
+        }
+
         if (*e == ':'
             && (s[0] != 'V'
                 || STRNCMP(skipwhite(e + 1), "set", 3) == 0)
@@ -4536,8 +4539,9 @@ chk_modeline (
                 || (VIM_VERSION_100 >= vers && isdigit(s[3]))
                 || (VIM_VERSION_100 < vers && s[3] == '<')
                 || (VIM_VERSION_100 > vers && s[3] == '>')
-                || (VIM_VERSION_100 == vers && s[3] == '=')))
+                || (VIM_VERSION_100 == vers && s[3] == '='))) {
           break;
+        }
       }
     }
     prev = *s;
diff --git a/src/nvim/charset.c b/src/nvim/charset.c
index 78f5d96fc7..61c5b10808 100644
--- a/src/nvim/charset.c
+++ b/src/nvim/charset.c
@@ -1732,6 +1732,26 @@ char_u* skiptowhite_esc(char_u *p) {
   return p;
 }
 
+/// Get a number from a string and skip over it, signalling overflows
+///
+/// @param[out]  pp  A pointer to a pointer to char_u.
+///                  It will be advanced past the read number.
+/// @param[out]  nr  Number read from the string.
+///
+/// @return OK on success, FAIL on error/overflow
+int getdigits_safe(char_u **pp, intmax_t *nr)
+{
+  errno = 0;
+  *nr = strtoimax((char *)(*pp), (char **)pp, 10);
+
+  if ((*nr == INTMAX_MIN || *nr == INTMAX_MAX)
+      && errno == ERANGE) {
+    return FAIL;
+  }
+
+  return OK;
+}
+
 /// Get a number from a string and skip over it.
 ///
 /// @param[out]  pp  A pointer to a pointer to char_u.
@@ -1740,17 +1760,16 @@ char_u* skiptowhite_esc(char_u *p) {
 /// @return Number read from the string.
 intmax_t getdigits(char_u **pp)
 {
-  errno = 0;
-  intmax_t number = strtoimax((char *)*pp, (char **)pp, 10);
-  if (number == INTMAX_MAX || number == INTMAX_MIN) {
-    assert(errno != ERANGE);
-  }
+  intmax_t number;
+  int ret = getdigits_safe(pp, &number);
+
+  (void)ret;  // Avoid "unused variable" warning in Release build
+  assert(ret == OK);
+
   return number;
 }
 
-/// Get an int number from a string.
-///
-/// A getdigits wrapper restricted to int values.
+/// Get an int number from a string. Like getdigits(), but restricted to `int`.
 int getdigits_int(char_u **pp)
 {
   intmax_t number = getdigits(pp);
@@ -1760,9 +1779,7 @@ int getdigits_int(char_u **pp)
   return (int)number;
 }
 
-/// Get a long number from a string.
-///
-/// A getdigits wrapper restricted to long values.
+/// Get a long number from a string. Like getdigits(), but restricted to `long`.
 long getdigits_long(char_u **pp)
 {
   intmax_t number = getdigits(pp);