(verbose_)?try_malloc() to use on buf_write()

There will be more use cases for try_malloc(): see #556.

 - Reimplemented xmalloc() using try_malloc().
 - verbose_try_malloc() is just like try_malloc() but shows an out-of-memory
	 error message before returning NULL.
 - Let the compiler generate size>>1 assembly for signed types. We're not using
	 old compilers here.
 - Add proper function attributes to the new functions in memory.h

3 files changed, 78 insertions, 31 deletions

diff --git a/src/fileio.c b/src/fileio.c
index bef1ead599..5c0233b79d 100644
--- a/src/fileio.c
+++ b/src/fileio.c
@@ -190,7 +190,7 @@ void filemess(buf_T *buf, char_u *name, char_u *s, int attr)
 /*
  * Read lines from file "fname" into the buffer after line "from".
  *
- * 1. We allocate blocks with lalloc, as big as possible.
+ * 1. We allocate blocks with try_malloc, as big as possible.
  * 2. Each block is filled with characters from the file with a single read().
  * 3. The lines are inserted in the buffer with ml_append().
  *
@@ -987,10 +987,15 @@ retry:
       if (!skip_read) {
         size = 0x10000L;                            /* use buffer >= 64K */
 
-        for (; size >= 10; size = (long)((long_u)size >> 1)) {
-          if ((new_buffer = lalloc((long_u)(size + linerest + 1),
-                   FALSE)) != NULL)
-              break;
+        for (; size >= 10; size /= 2) {
+          new_buffer = verbose_try_malloc((size_t)size + (size_t)linerest + 1);
+          if (new_buffer) {
+            break;
+          }
+        }
+        if (new_buffer == NULL) {
+          error = TRUE;
+          break;
         }
         if (linerest)           /* copy characters from the previous buffer */
           memmove(new_buffer, ptr - linerest, (size_t)linerest);
@@ -2763,10 +2768,7 @@ buf_write (
         (char_u *)"", 0);               /* show that we are busy */
   msg_scroll = FALSE;               /* always overwrite the file message now */
 
-  buffer = alloc(BUFSIZE);
-  // TODO: decide how to handle this now that alloc never returns NULL. The fact
-  // that the OOM handling code calls this should be considered.
-  //
+  buffer = verbose_try_malloc(BUFSIZE);
   // can't allocate big buffer, use small one (to be able to write when out of
   // memory)
   if (buffer == NULL) {
@@ -3006,7 +3008,12 @@ buf_write (
       int did_set_shortname;
 #endif
 
-      copybuf = alloc(BUFSIZE + 1);
+      copybuf = verbose_try_malloc(BUFSIZE + 1);
+      if (copybuf == NULL) {
+        // out of memory
+        some_error = TRUE;
+        goto nobackup;
+      }
 
       /*
        * Try to make the backup in each directory in the 'bdir' option.
@@ -3372,8 +3379,10 @@ nobackup:
         write_info.bw_conv_buflen = bufsize * 2;
       else       /* FIO_UCS4 */
         write_info.bw_conv_buflen = bufsize * 4;
-      write_info.bw_conv_buf
-        = lalloc((long_u)write_info.bw_conv_buflen, TRUE);
+      write_info.bw_conv_buf = verbose_try_malloc(write_info.bw_conv_buflen);
+      if (!write_info.bw_conv_buf) {
+        end = 0;
+      }
     }
   }
 
@@ -3390,8 +3399,10 @@ nobackup:
     if (write_info.bw_iconv_fd != (iconv_t)-1) {
       /* We're going to use iconv(), allocate a buffer to convert in. */
       write_info.bw_conv_buflen = bufsize * ICONV_MULT;
-      write_info.bw_conv_buf
-        = lalloc((long_u)write_info.bw_conv_buflen, TRUE);
+      write_info.bw_conv_buf = verbose_try_malloc(write_info.bw_conv_buflen);
+      if (!write_info.bw_conv_buf) {
+        end = 0;
+      }
       write_info.bw_first = TRUE;
     } else
 #  endif
@@ -5048,7 +5059,9 @@ int vim_rename(char_u *from, char_u *to)
     return -1;
   }
 
-  buffer = (char *)alloc(BUFSIZE);
+  // Avoid xmalloc() here as vim_rename() is called by buf_write() when neovim
+  // is `preserve_exit()`ing.
+  buffer = try_malloc(BUFSIZE);
   if (buffer == NULL) {
     close(fd_out);
     close(fd_in);
@@ -5616,14 +5629,14 @@ void vim_deltempdir(void)
  */
 static void vim_settempdir(char_u *tempdir)
 {
-  char_u      *buf;
-
-  buf = alloc((unsigned)MAXPATHL + 2);
-  if (vim_FullName(tempdir, buf, MAXPATHL, FALSE) == FAIL)
-    STRCPY(buf, tempdir);
-  add_pathsep(buf);
-  vim_tempdir = vim_strsave(buf);
-  vim_free(buf);
+  char_u *buf = verbose_try_malloc((size_t)MAXPATHL + 2);
+  if (buf) {
+    if (vim_FullName(tempdir, buf, MAXPATHL, FALSE) == FAIL)
+      STRCPY(buf, tempdir);
+    add_pathsep(buf);
+    vim_tempdir = vim_strsave(buf);
+    vim_free(buf);
+  }
 }
 #endif
 
diff --git a/src/memory.c b/src/memory.c
index 2ec1b179ea..4411c1526f 100644
--- a/src/memory.c
+++ b/src/memory.c
@@ -81,24 +81,40 @@ static void try_to_free_memory()
   trying_to_free = false;
 }
 
-void *xmalloc(size_t size)
+void *try_malloc(size_t size)
 {
   void *ret = malloc(size);
 
-  if (!ret && !size)
+  if (!ret && !size) {
     ret = malloc(1);
-
+  }
   if (!ret) {
     try_to_free_memory();
     ret = malloc(size);
-    if (!ret && !size)
+    if (!ret && !size) {
       ret = malloc(1);
-    if (!ret) {
-      OUT_STR("Vim: Error: Out of memory.\n");
-      preserve_exit();
     }
   }
+  return ret;
+}
+
+void *verbose_try_malloc(size_t size)
+{
+  void *ret = try_malloc(size);
+  if (!ret) {
+    do_outofmem_msg((long_u)size);
+  }
+  return ret;
+}
 
+void *xmalloc(size_t size)
+{
+  void *ret = try_malloc(size);
+
+  if (!ret) {
+    OUT_STR("Vim: Error: Out of memory.\n");
+    preserve_exit();
+  }
   return ret;
 }
 
diff --git a/src/memory.h b/src/memory.h
index 05ac9359e5..bc20c0420e 100644
--- a/src/memory.h
+++ b/src/memory.h
@@ -10,6 +10,24 @@ char_u *alloc_clear(unsigned size) FUNC_ATTR_MALLOC FUNC_ATTR_ALLOC_SIZE(1);
 
 /// malloc() wrapper
 ///
+/// try_malloc() is a malloc() wrapper that tries to free some memory before
+/// trying again.
+///
+/// @see {try_to_free_memory}
+/// @param size
+/// @return pointer to allocated space. NULL if out of memory
+void *try_malloc(size_t size) FUNC_ATTR_MALLOC FUNC_ATTR_ALLOC_SIZE(1);
+
+/// try_malloc() wrapper that shows an out-of-memory error message to the user
+/// before returning NULL
+///
+/// @see {try_malloc}
+/// @param size
+/// @return pointer to allocated space. NULL if out of memory
+void *verbose_try_malloc(size_t size) FUNC_ATTR_MALLOC FUNC_ATTR_ALLOC_SIZE(1);
+
+/// malloc() wrapper that never returns NULL
+///
 /// xmalloc() succeeds or gracefully aborts when out of memory.
 /// Before aborting try to free some memory and call malloc again.
 ///
@@ -51,7 +69,7 @@ void *xmallocz(size_t size) FUNC_ATTR_MALLOC FUNC_ATTR_NONNULL_RET;
 /// @see {xmalloc}
 /// @param data Pointer to the data that will be copied
 /// @param len number of bytes that will be copied
-void *xmemdupz(const void *data, size_t len);
+void *xmemdupz(const void *data, size_t len) FUNC_ATTR_NONNULL_RET;
 
 /// strdup() wrapper
 ///