Merge branch 'master' into s-dash-stdin

16 files changed, 457 insertions, 146 deletions

diff --git a/src/nvim/os/env.c b/src/nvim/os/env.c
index de0cd10d9c..3fcb9415c7 100644
--- a/src/nvim/os/env.c
+++ b/src/nvim/os/env.c
@@ -886,8 +886,8 @@ bool os_setenv_append_path(const char *fname)
 // No prescribed maximum on unix.
 # define MAX_ENVPATHLEN INT_MAX
 #endif
-  if (!path_is_absolute_path((char_u *)fname)) {
-    EMSG2(_(e_intern2), "os_setenv_append_path()");
+  if (!path_is_absolute((char_u *)fname)) {
+    internal_error("os_setenv_append_path()");
     return false;
   }
   const char *tail = (char *)path_tail_with_sep((char_u *)fname);
diff --git a/src/nvim/os/fileio.c b/src/nvim/os/fileio.c
index b040904d0b..0ef9307a5a 100644
--- a/src/nvim/os/fileio.c
+++ b/src/nvim/os/fileio.c
@@ -4,7 +4,7 @@
 /// @file fileio.c
 ///
 /// Buffered reading/writing to a file. Unlike fileio.c this is not dealing with
-/// Neovim stuctures for buffer, with autocommands, etc: just fopen/fread/fwrite
+/// Nvim stuctures for buffer, with autocommands, etc: just fopen/fread/fwrite
 /// replacement.
 
 #include <assert.h>
@@ -39,11 +39,11 @@
 /// @param[in]  fname  File name to open.
 /// @param[in]  flags  Flags, @see FileOpenFlags. Currently reading from and
 ///                    writing to the file at once is not supported, so either
-///                    FILE_WRITE_ONLY or FILE_READ_ONLY is required.
+///                    kFileWriteOnly or kFileReadOnly is required.
 /// @param[in]  mode  Permissions for the newly created file (ignored if flags
-///                   does not have FILE_CREATE\*).
+///                   does not have kFileCreate\*).
 ///
-/// @return Error code (@see os_strerror()) or 0.
+/// @return Error code, or 0 on success. @see os_strerror()
 int file_open(FileDescriptor *const ret_fp, const char *const fname,
               const int flags, const int mode)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
@@ -124,12 +124,11 @@ int file_open_fd(FileDescriptor *const ret_fp, const int fd,
 
 /// Like file_open(), but allocate and return ret_fp
 ///
-/// @param[out]  error  Error code, @see os_strerror(). Is set to zero on
-///                     success.
+/// @param[out]  error  Error code, or 0 on success. @see os_strerror()
 /// @param[in]  fname  File name to open.
 /// @param[in]  flags  Flags, @see FileOpenFlags.
 /// @param[in]  mode  Permissions for the newly created file (ignored if flags
-///                   does not have FILE_CREATE\*).
+///                   does not have kFileCreate\*).
 ///
 /// @return [allocated] Opened file or NULL in case of error.
 FileDescriptor *file_open_new(int *const error, const char *const fname,
@@ -146,8 +145,7 @@ FileDescriptor *file_open_new(int *const error, const char *const fname,
 
 /// Like file_open_fd(), but allocate and return ret_fp
 ///
-/// @param[out]  error  Error code, @see os_strerror(). Is set to zero on
-///                     success.
+/// @param[out]  error  Error code, or 0 on success. @see os_strerror()
 /// @param[in]  fd  File descriptor to wrap.
 /// @param[in]  flags  Flags, @see FileOpenFlags.
 /// @param[in]  mode  Permissions for the newly created file (ignored if flags
diff --git a/src/nvim/os/fs.c b/src/nvim/os/fs.c
index 87bab93420..0414794d01 100644
--- a/src/nvim/os/fs.c
+++ b/src/nvim/os/fs.c
@@ -222,7 +222,7 @@ int os_exepath(char *buffer, size_t *size)
 bool os_can_exe(const char_u *name, char_u **abspath, bool use_path)
   FUNC_ATTR_NONNULL_ARG(1)
 {
-  bool no_path = !use_path || path_is_absolute_path(name);
+  bool no_path = !use_path || path_is_absolute(name);
 #ifndef WIN32
   // If the filename is "qualified" (relative or absolute) do not check $PATH.
   no_path |= (name[0] == '.'
@@ -244,7 +244,7 @@ bool os_can_exe(const char_u *name, char_u **abspath, bool use_path)
 #endif
     if (ok) {
       if (abspath != NULL) {
-        *abspath = save_absolute_path(name);
+        *abspath = save_abs_path(name);
       }
       return true;
     }
@@ -357,7 +357,7 @@ static bool is_executable_in_path(const char_u *name, char_u **abspath)
 #endif
     if (ok) {
       if (abspath != NULL) {  // Caller asked for a copy of the path.
-        *abspath = save_absolute_path((char_u *)buf);
+        *abspath = save_abs_path((char_u *)buf);
       }
 
       rv = true;
@@ -485,7 +485,7 @@ ptrdiff_t os_read(const int fd, bool *const ret_eof, char *const ret_buf,
   while (read_bytes != size) {
     assert(size >= read_bytes);
     const ptrdiff_t cur_read_bytes = read(fd, ret_buf + read_bytes,
-                                          size - read_bytes);
+                                          IO_COUNT(size - read_bytes));
     if (cur_read_bytes > 0) {
       read_bytes += (size_t)cur_read_bytes;
     }
@@ -598,7 +598,7 @@ ptrdiff_t os_write(const int fd, const char *const buf, const size_t size,
   while (written_bytes != size) {
     assert(size >= written_bytes);
     const ptrdiff_t cur_written_bytes = write(fd, buf + written_bytes,
-                                              size - written_bytes);
+                                              IO_COUNT(size - written_bytes));
     if (cur_written_bytes > 0) {
       written_bytes += (size_t)cur_written_bytes;
     }
@@ -1026,7 +1026,7 @@ bool os_fileid_equal_fileinfo(const FileID *file_id,
 /// to and return that name in allocated memory.
 /// Otherwise NULL is returned.
 char *os_resolve_shortcut(const char *fname)
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_MALLOC
+  FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_MALLOC
 {
   HRESULT hr;
   IPersistFile *ppf = NULL;
diff --git a/src/nvim/os/input.c b/src/nvim/os/input.c
index 7d6f2abd7f..405500767d 100644
--- a/src/nvim/os/input.c
+++ b/src/nvim/os/input.c
@@ -188,7 +188,7 @@ size_t input_enqueue(String keys)
     uint8_t buf[6] = { 0 };
     unsigned int new_size
         = trans_special((const uint8_t **)&ptr, (size_t)(end - ptr), buf, true,
-                        true);
+                        false);
 
     if (new_size) {
       new_size = handle_mouse_event(&ptr, buf, new_size);
diff --git a/src/nvim/os/lang.c b/src/nvim/os/lang.c
new file mode 100644
index 0000000000..47c278ee97
--- /dev/null
+++ b/src/nvim/os/lang.c
@@ -0,0 +1,43 @@
+// This is an open source non-commercial project. Dear PVS-Studio, please check
+// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
+
+#ifdef __APPLE__
+# define Boolean CFBoolean  // Avoid conflict with API's Boolean
+# include <CoreFoundation/CFLocale.h>
+# include <CoreFoundation/CFString.h>
+# undef Boolean
+#endif
+
+#ifdef HAVE_LOCALE_H
+# include <locale.h>
+#endif
+#include "nvim/os/os.h"
+
+void lang_init(void)
+{
+#ifdef __APPLE__
+  if (os_getenv("LANG") == NULL) {
+    CFLocaleRef cf_locale = CFLocaleCopyCurrent();
+    CFTypeRef cf_lang_region = CFLocaleGetValue(cf_locale,
+                                                kCFLocaleIdentifier);
+    CFRetain(cf_lang_region);
+    CFRelease(cf_locale);
+
+    const char *lang_region = CFStringGetCStringPtr(cf_lang_region,
+                                                    kCFStringEncodingUTF8);
+    if (lang_region) {
+      os_setenv("LANG", lang_region, true);
+    } else {
+      char buf[20] = { 0 };
+      if (CFStringGetCString(cf_lang_region, buf, 20,
+                             kCFStringEncodingUTF8)) {
+        os_setenv("LANG", lang_region, true);
+      }
+    }
+    CFRelease(cf_lang_region);
+# ifdef HAVE_LOCALE_H
+    setlocale(LC_ALL, "");
+# endif
+  }
+#endif
+}
diff --git a/src/nvim/os/lang.h b/src/nvim/os/lang.h
new file mode 100644
index 0000000000..f60e064f57
--- /dev/null
+++ b/src/nvim/os/lang.h
@@ -0,0 +1,7 @@
+#ifndef NVIM_OS_LANG_H
+#define NVIM_OS_LANG_H
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "os/lang.h.generated.h"
+#endif
+#endif  // NVIM_OS_LANG_H
diff --git a/src/nvim/os/os_defs.h b/src/nvim/os/os_defs.h
index 87f8d214bd..f81785675e 100644
--- a/src/nvim/os/os_defs.h
+++ b/src/nvim/os/os_defs.h
@@ -4,7 +4,6 @@
 #include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <sys/param.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 
@@ -14,13 +13,6 @@
 # include "nvim/os/unix_defs.h"
 #endif
 
-/// File descriptor number used for standard IO streams
-enum {
-  OS_STDIN_FILENO = STDIN_FILENO,
-  OS_STDOUT_FILENO = STDOUT_FILENO,
-  OS_STDERR_FILENO = STDERR_FILENO,
-};
-
 #define BASENAMELEN (NAME_MAX - 5)
 
 // Use the system path length if it makes sense.
diff --git a/src/nvim/os/process.c b/src/nvim/os/process.c
new file mode 100644
index 0000000000..a67e7487eb
--- /dev/null
+++ b/src/nvim/os/process.c
@@ -0,0 +1,267 @@
+// This is an open source non-commercial project. Dear PVS-Studio, please check
+// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
+
+/// OS process functions
+///
+/// psutil is a good reference for cross-platform syscall voodoo:
+/// https://github.com/giampaolo/psutil/tree/master/psutil/arch
+
+#include <uv.h>  // for HANDLE (win32)
+
+#ifdef WIN32
+# include <tlhelp32.h>  // for CreateToolhelp32Snapshot
+#endif
+
+#if defined(__FreeBSD__)  // XXX: OpenBSD ?
+# include <string.h>
+# include <sys/types.h>
+# include <sys/user.h>
+#endif
+
+#if defined(__NetBSD__) || defined(__OpenBSD__)
+# include <sys/param.h>
+#endif
+
+#if defined(__APPLE__) || defined(BSD)
+# include <sys/sysctl.h>
+# include <pwd.h>
+#endif
+
+#include "nvim/globals.h"
+#include "nvim/log.h"
+#include "nvim/os/process.h"
+#include "nvim/os/os.h"
+#include "nvim/os/os_defs.h"
+#include "nvim/api/private/helpers.h"
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "os/process.c.generated.h"
+#endif
+
+#ifdef WIN32
+static bool os_proc_tree_kill_rec(HANDLE process, int sig)
+{
+  if (process == NULL) {
+    return false;
+  }
+  PROCESSENTRY32 pe;
+  DWORD pid = GetProcessId(process);
+
+  if (pid != 0) {
+    HANDLE h = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
+    if (h != INVALID_HANDLE_VALUE) {
+      pe.dwSize = sizeof(PROCESSENTRY32);
+      if (!Process32First(h, &pe)) {
+        goto theend;
+      }
+      do {
+        if (pe.th32ParentProcessID == pid) {
+          HANDLE ph = OpenProcess(PROCESS_ALL_ACCESS, false, pe.th32ProcessID);
+          if (ph != NULL) {
+            os_proc_tree_kill_rec(ph, sig);
+            CloseHandle(ph);
+          }
+        }
+      } while (Process32Next(h, &pe));
+      CloseHandle(h);
+    }
+  }
+
+theend:
+  return (bool)TerminateProcess(process, (unsigned int)sig);
+}
+/// Kills process `pid` and its descendants recursively.
+bool os_proc_tree_kill(int pid, int sig)
+{
+  assert(sig >= 0);
+  assert(sig == SIGTERM || sig == SIGKILL);
+  if (pid > 0) {
+    ILOG("terminating process tree: %d", pid);
+    HANDLE h = OpenProcess(PROCESS_ALL_ACCESS, false, (DWORD)pid);
+    return os_proc_tree_kill_rec(h, sig);
+  } else {
+    ELOG("invalid pid: %d", pid);
+  }
+  return false;
+}
+#else
+/// Kills process group where `pid` is the process group leader.
+bool os_proc_tree_kill(int pid, int sig)
+{
+  assert(sig == SIGTERM || sig == SIGKILL);
+  int pgid = getpgid(pid);
+  if (pgid > 0) {  // Ignore error. Never kill self (pid=0).
+    if (pgid == pid) {
+      ILOG("sending %s to process group: -%d",
+           sig == SIGTERM ? "SIGTERM" : "SIGKILL", pgid);
+      int rv = uv_kill(-pgid, sig);
+      return rv == 0;
+    } else {
+      // Should never happen, because process_spawn() did setsid() in the child.
+      ELOG("pgid %d != pid %d", pgid, pid);
+    }
+  } else {
+    ELOG("getpgid(%d) returned %d", pid, pgid);
+  }
+  return false;
+}
+#endif
+
+/// Gets the process ids of the immediate children of process `ppid`.
+///
+/// @param ppid Process to inspect.
+/// @param[out,allocated] proc_list Child process ids.
+/// @param[out] proc_count Number of child processes.
+/// @return 0 on success, 1 if process not found, 2 on other error.
+int os_proc_children(int ppid, int **proc_list, size_t *proc_count)
+{
+  if (ppid < 0) {
+    return 2;
+  }
+
+  int *temp = NULL;
+  *proc_list = NULL;
+  *proc_count = 0;
+
+#ifdef WIN32
+  PROCESSENTRY32 pe;
+
+  // Snapshot of all processes.
+  HANDLE h = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
+  if (h == INVALID_HANDLE_VALUE) {
+    return 2;
+  }
+
+  pe.dwSize = sizeof(PROCESSENTRY32);
+  // Get root process.
+  if (!Process32First(h, &pe)) {
+    CloseHandle(h);
+    return 2;
+  }
+  // Collect processes whose parent matches `ppid`.
+  do {
+    if (pe.th32ParentProcessID == (DWORD)ppid) {
+      temp = xrealloc(temp, (*proc_count + 1) * sizeof(*temp));
+      temp[*proc_count] = (int)pe.th32ProcessID;
+      (*proc_count)++;
+    }
+  } while (Process32Next(h, &pe));
+  CloseHandle(h);
+
+#elif defined(__APPLE__) || defined(BSD)
+# if defined(__APPLE__)
+#  define KP_PID(o) o.kp_proc.p_pid
+#  define KP_PPID(o) o.kp_eproc.e_ppid
+# elif defined(__FreeBSD__)
+#  define KP_PID(o) o.ki_pid
+#  define KP_PPID(o) o.ki_ppid
+# else
+#  define KP_PID(o) o.p_pid
+#  define KP_PPID(o) o.p_ppid
+# endif
+# ifdef __NetBSD__
+  static int name[] = {
+    CTL_KERN, KERN_PROC2, KERN_PROC_ALL, 0, (int)(sizeof(struct kinfo_proc2)), 0
+  };
+# else
+  static int name[] = { CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0 };
+# endif
+
+  // Get total process count.
+  size_t len = 0;
+  int rv = sysctl(name, ARRAY_SIZE(name) - 1, NULL, &len, NULL, 0);
+  if (rv) {
+    return 2;
+  }
+
+  // Get ALL processes.
+# ifdef __NetBSD__
+  struct kinfo_proc2 *p_list = xmalloc(len);
+# else
+  struct kinfo_proc *p_list = xmalloc(len);
+# endif
+  rv = sysctl(name, ARRAY_SIZE(name) - 1, p_list, &len, NULL, 0);
+  if (rv) {
+    xfree(p_list);
+    return 2;
+  }
+
+  // Collect processes whose parent matches `ppid`.
+  bool exists = false;
+  size_t p_count = len / sizeof(*p_list);
+  for (size_t i = 0; i < p_count; i++) {
+    exists = exists || KP_PID(p_list[i]) == ppid;
+    if (KP_PPID(p_list[i]) == ppid) {
+      temp = xrealloc(temp, (*proc_count + 1) * sizeof(*temp));
+      temp[*proc_count] = KP_PID(p_list[i]);
+      (*proc_count)++;
+    }
+  }
+  xfree(p_list);
+  if (!exists) {
+    return 1;  // Process not found.
+  }
+
+#elif defined(__linux__)
+  char proc_p[256] = { 0 };
+  // Collect processes whose parent matches `ppid`.
+  // Rationale: children are defined in thread with same ID of process.
+  snprintf(proc_p, sizeof(proc_p), "/proc/%d/task/%d/children", ppid, ppid);
+  FILE *fp = fopen(proc_p, "r");
+  if (fp == NULL) {
+    return 2;  // Process not found, or /proc/…/children not supported.
+  }
+  int match_pid;
+  while (fscanf(fp, "%d", &match_pid) > 0) {
+    temp = xrealloc(temp, (*proc_count + 1) * sizeof(*temp));
+    temp[*proc_count] = match_pid;
+    (*proc_count)++;
+  }
+  fclose(fp);
+#endif
+
+  *proc_list = temp;
+  return 0;
+}
+
+#ifdef WIN32
+/// Gets various properties of the process identified by `pid`.
+///
+/// @param pid Process to inspect.
+/// @return Map of process properties, empty on error.
+Dictionary os_proc_info(int pid)
+{
+  Dictionary pinfo = ARRAY_DICT_INIT;
+  PROCESSENTRY32 pe;
+
+  // Snapshot of all processes.  This is used instead of:
+  //    OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, …)
+  // to avoid ERROR_PARTIAL_COPY.  https://stackoverflow.com/a/29942376
+  HANDLE h = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
+  if (h == INVALID_HANDLE_VALUE) {
+    return pinfo;  // Return empty.
+  }
+
+  pe.dwSize = sizeof(PROCESSENTRY32);
+  // Get root process.
+  if (!Process32First(h, &pe)) {
+    CloseHandle(h);
+    return pinfo;  // Return empty.
+  }
+  // Find the process.
+  do {
+    if (pe.th32ProcessID == (DWORD)pid) {
+      break;
+    }
+  } while (Process32Next(h, &pe));
+  CloseHandle(h);
+
+  if (pe.th32ProcessID == (DWORD)pid) {
+    PUT(pinfo, "pid", INTEGER_OBJ(pid));
+    PUT(pinfo, "ppid", INTEGER_OBJ((int)pe.th32ParentProcessID));
+    PUT(pinfo, "name", STRING_OBJ(cstr_to_string(pe.szExeFile)));
+  }
+
+  return pinfo;
+}
+#endif
diff --git a/src/nvim/os/process.h b/src/nvim/os/process.h
new file mode 100644
index 0000000000..1722d56bd3
--- /dev/null
+++ b/src/nvim/os/process.h
@@ -0,0 +1,11 @@
+#ifndef NVIM_OS_PROCESS_H
+#define NVIM_OS_PROCESS_H
+
+#include <stddef.h>
+#include "nvim/api/private/defs.h"
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "os/process.h.generated.h"
+#endif
+
+#endif  // NVIM_OS_PROCESS_H
diff --git a/src/nvim/os/pty_process_unix.c b/src/nvim/os/pty_process_unix.c
index 53301e4b53..dfe2cfbb8d 100644
--- a/src/nvim/os/pty_process_unix.c
+++ b/src/nvim/os/pty_process_unix.c
@@ -72,8 +72,7 @@ int pty_process_spawn(PtyProcess *ptyproc)
     ELOG("forkpty failed: %s", strerror(errno));
     return status;
   } else if (pid == 0) {
-    init_child(ptyproc);
-    abort();
+    init_child(ptyproc);  // never returns
   }
 
   // make sure the master file descriptor is non blocking
@@ -146,8 +145,12 @@ void pty_process_teardown(Loop *loop)
   uv_signal_stop(&loop->children_watcher);
 }
 
-static void init_child(PtyProcess *ptyproc) FUNC_ATTR_NONNULL_ALL
+static void init_child(PtyProcess *ptyproc)
+  FUNC_ATTR_NONNULL_ALL
 {
+  // New session/process-group. #6530
+  setsid();
+
   unsetenv("COLUMNS");
   unsetenv("LINES");
   unsetenv("TERMCAP");
@@ -163,14 +166,15 @@ static void init_child(PtyProcess *ptyproc) FUNC_ATTR_NONNULL_ALL
 
   Process *proc = (Process *)ptyproc;
   if (proc->cwd && os_chdir(proc->cwd) != 0) {
-    fprintf(stderr, "chdir failed: %s\n", strerror(errno));
+    ELOG("chdir failed: %s", strerror(errno));
     return;
   }
 
   char *prog = ptyproc->process.argv[0];
   setenv("TERM", ptyproc->term_name ? ptyproc->term_name : "ansi", 1);
   execvp(prog, ptyproc->process.argv);
-  fprintf(stderr, "execvp failed: %s: %s\n", strerror(errno), prog);
+  ELOG("execvp failed: %s: %s", strerror(errno), prog);
+  _exit(122);  // 122 is EXEC_FAILED in the Vim source.
 }
 
 static void init_termios(struct termios *termios) FUNC_ATTR_NONNULL_ALL
diff --git a/src/nvim/os/pty_process_win.c b/src/nvim/os/pty_process_win.c
index 3c4839a076..a6774a6275 100644
--- a/src/nvim/os/pty_process_win.c
+++ b/src/nvim/os/pty_process_win.c
@@ -1,3 +1,6 @@
+// This is an open source non-commercial project. Dear PVS-Studio, please check
+// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
+
 #include <assert.h>
 #include <stdbool.h>
 #include <stdlib.h>
@@ -131,7 +134,7 @@ int pty_process_spawn(PtyProcess *ptyproc)
     }
     goto cleanup;
   }
-  proc->pid = GetProcessId(process_handle);
+  proc->pid = (int)GetProcessId(process_handle);
 
   if (!RegisterWaitForSingleObject(
       &ptyproc->finish_wait,
@@ -339,20 +342,20 @@ static void quote_cmd_arg(char *dest, size_t dest_remaining, const char *src)
   }
 
   // Expected input/output:
-  //   input : hello"world
-  //   output: "hello\"world"
-  //   input : hello""world
-  //   output: "hello\"\"world"
-  //   input : hello\world
-  //   output: hello\world
-  //   input : hello\\world
-  //   output: hello\\world
-  //   input : hello\"world
-  //   output: "hello\\\"world"
-  //   input : hello\\"world
-  //   output: "hello\\\\\"world"
-  //   input : hello world\
-  //   output: "hello world\\"
+  //   input : 'hello"world'
+  //   output: '"hello\"world"'
+  //   input : 'hello""world'
+  //   output: '"hello\"\"world"'
+  //   input : 'hello\world'
+  //   output: 'hello\world'
+  //   input : 'hello\\world'
+  //   output: 'hello\\world'
+  //   input : 'hello\"world'
+  //   output: '"hello\\\"world"'
+  //   input : 'hello\\"world'
+  //   output: '"hello\\\\\"world"'
+  //   input : 'hello world\'
+  //   output: '"hello world\\"'
 
   assert(dest_remaining--);
   *(dest++) = NUL;
diff --git a/src/nvim/os/shell.c b/src/nvim/os/shell.c
index e32c6e05d2..04f59d7522 100644
--- a/src/nvim/os/shell.c
+++ b/src/nvim/os/shell.c
@@ -123,6 +123,9 @@ int os_call_shell(char_u *cmd, ShellOpts opts, char_u *extra_args)
     if (opts & kShellOptRead) {
       output_ptr = &output;
       forward_output = false;
+    } else if (opts & kShellOptDoOut) {
+      // Caller has already redirected output
+      forward_output = false;
     }
   }
 
@@ -133,7 +136,7 @@ int os_call_shell(char_u *cmd, ShellOpts opts, char_u *extra_args)
   xfree(input.data);
 
   if (output) {
-    (void)write_output(output, nread, true, true);
+    (void)write_output(output, nread, true);
     xfree(output);
   }
 
@@ -189,6 +192,7 @@ static int do_os_system(char **argv,
 {
   out_data_decide_throttle(0);  // Initialize throttle decider.
   out_data_ring(NULL, 0);       // Initialize output ring-buffer.
+  bool has_input = (input != NULL && input[0] != '\0');
 
   // the output buffer
   DynamicBuffer buf = DYNAMIC_BUFFER_INIT;
@@ -212,7 +216,7 @@ static int do_os_system(char **argv,
   MultiQueue *events = multiqueue_new_child(main_loop.events);
   proc->events = events;
   proc->argv = argv;
-  int status = process_spawn(proc, input != NULL, true, true);
+  int status = process_spawn(proc, has_input, true, true);
   if (status) {
     loop_poll_events(&main_loop, 0);
     // Failed, probably 'shell' is not executable.
@@ -231,7 +235,7 @@ static int do_os_system(char **argv,
   // deal with stream events as fast a possible.  It prevents closing the
   // streams while there's still data in the OS buffer (due to the process
   // exiting before all data is read).
-  if (input != NULL) {
+  if (has_input) {
     wstream_init(&proc->in, 0);
   }
   rstream_init(&proc->out, 0);
@@ -240,8 +244,8 @@ static int do_os_system(char **argv,
   rstream_start(&proc->err, data_cb, &buf);
 
   // write the input, if any
-  if (input) {
-    WBuffer *input_buffer = wstream_new_buffer((char *) input, len, 1, NULL);
+  if (has_input) {
+    WBuffer *input_buffer = wstream_new_buffer((char *)input, len, 1, NULL);
 
     if (!wstream_write(&proc->in, input_buffer)) {
       // couldn't write, stop the process and tell the user about it
@@ -256,11 +260,25 @@ static int do_os_system(char **argv,
   // busy state.
   ui_busy_start();
   ui_flush();
+  if (forward_output) {
+    msg_sb_eol();
+    msg_start();
+    msg_no_more = true;
+    lines_left = -1;
+  }
   int exitcode = process_wait(proc, -1, NULL);
   if (!got_int && out_data_decide_throttle(0)) {
     // Last chunk of output was skipped; display it now.
     out_data_ring(NULL, SIZE_MAX);
   }
+  if (forward_output) {
+    // caller should decide if wait_return is invoked
+    no_wait_return++;
+    msg_end();
+    no_wait_return--;
+    msg_no_more = false;
+  }
+
   ui_busy_stop();
 
   // prepare the out parameters if requested
@@ -370,10 +388,10 @@ static bool out_data_decide_throttle(size_t size)
   pulse_msg[1] = (tick == 0 || 1 == tick) ? ' ' : '.';
   pulse_msg[2] = (tick == 0 || 1 == tick || 2 == tick) ? ' ' : '.';
   if (visit == 1) {
-    screen_del_lines(0, 0, 1, (int)Rows, NULL);
+    msg_putchar('\n');
   }
-  int lastrow = (int)Rows - 1;
-  screen_puts_len((char_u *)pulse_msg, ARRAY_SIZE(pulse_msg), lastrow, 0, 0);
+  msg_putchar('\r');  // put cursor at start of line
+  msg_puts(pulse_msg);
   ui_flush();
   return true;
 }
@@ -404,7 +422,7 @@ static void out_data_ring(char *output, size_t size)
   }
 
   if (output == NULL && size == SIZE_MAX) {   // Print mode
-    out_data_append_to_screen(last_skipped, last_skipped_len, true);
+    out_data_append_to_screen(last_skipped, &last_skipped_len, true);
     return;
   }
 
@@ -432,60 +450,40 @@ static void out_data_ring(char *output, size_t size)
 /// @param output       Data to append to screen lines.
 /// @param remaining    Size of data.
 /// @param new_line     If true, next data output will be on a new line.
-static void out_data_append_to_screen(char *output, size_t remaining,
-                                      bool new_line)
+static void out_data_append_to_screen(char *output, size_t *count,
+                                      bool eof)
 {
-  static colnr_T last_col = 0;  // Column of last row to append to.
-
-  size_t off = 0;
-  int last_row = (int)Rows - 1;
-
-  while (output != NULL && off < remaining) {
-    // Found end of line?
-    if (output[off] == NL) {
-      // Can we start a new line or do we need to continue the last one?
-      if (last_col == 0) {
-        screen_del_lines(0, 0, 1, (int)Rows, NULL);
+  char *p = output, *end = output + *count;
+  while (p < end) {
+    if (*p == '\n' || *p == '\r' || *p == TAB || *p == BELL) {
+      msg_putchar_attr((uint8_t)(*p), 0);
+      p++;
+    } else {
+      // Note: this is not 100% precise:
+      // 1. we don't check if received continuation bytes are already invalid
+      //    and we thus do some buffering that could be avoided
+      // 2. we don't compose chars over buffer boundaries, even if we see an
+      //    incomplete UTF-8 sequence that could be composing with the last
+      //    complete sequence.
+      // This will be corrected when we switch to vterm based implementation
+      int i = *p ? mb_ptr2len_len((char_u *)p, (int)(end-p)) : 1;
+      if (!eof && i == 1 && utf8len_tab_zero[*(uint8_t *)p] > (end-p)) {
+        *count = (size_t)(p - output);
+        goto end;
       }
-      screen_puts_len((char_u *)output, (int)off, last_row, last_col, 0);
-      last_col = 0;
 
-      size_t skip = off + 1;
-      output += skip;
-      remaining -= skip;
-      off = 0;
-      continue;
-    }
-
-    // TODO(bfredl): using msg_puts would be better until
-    // terminal emulation is implemented.
-    if (output[off] < 0x20) {
-      output[off] = ' ';
-    }
-
-    off++;
-  }
-
-  if (output != NULL && remaining) {
-    if (last_col == 0) {
-      screen_del_lines(0, 0, 1, (int)Rows, NULL);
+      (void)msg_outtrans_len_attr((char_u *)p, i, 0);
+      p += i;
     }
-    screen_puts_len((char_u *)output, (int)remaining, last_row, last_col, 0);
-    last_col += (colnr_T)remaining;
-  }
-
-  if (new_line) {
-    last_col = 0;
   }
 
+end:
   ui_flush();
 }
 
 static void out_data_cb(Stream *stream, RBuffer *buf, size_t count, void *data,
     bool eof)
 {
-  // We always output the whole buffer, so the buffer can never
-  // wrap around.
   size_t cnt;
   char *ptr = rbuffer_read_ptr(buf, &cnt);
 
@@ -494,12 +492,16 @@ static void out_data_cb(Stream *stream, RBuffer *buf, size_t count, void *data,
     // Save the skipped output. If it is the final chunk, we display it later.
     out_data_ring(ptr, cnt);
   } else {
-    out_data_append_to_screen(ptr, cnt, eof);
+    out_data_append_to_screen(ptr, &cnt, eof);
   }
 
   if (cnt) {
     rbuffer_consumed(buf, cnt);
   }
+
+  // Move remaining data to start of buffer, so the buffer can never
+  // wrap around.
+  rbuffer_reset(buf);
 }
 
 /// Parses a command string into a sequence of words, taking quotes into
@@ -588,14 +590,10 @@ static void read_input(DynamicBuffer *buf)
 
     if (len == l) {
       // Finished a line, add a NL, unless this line should not have one.
-      // FIXME need to make this more readable
       if (lnum != curbuf->b_op_end.lnum
-          || (!curbuf->b_p_bin
-            && curbuf->b_p_fixeol)
+          || (!curbuf->b_p_bin && curbuf->b_p_fixeol)
           || (lnum != curbuf->b_no_eol_lnum
-            && (lnum !=
-              curbuf->b_ml.ml_line_count
-              || curbuf->b_p_eol))) {
+              && (lnum != curbuf->b_ml.ml_line_count || curbuf->b_p_eol))) {
         dynamic_buffer_ensure(buf, buf->len + 1);
         buf->data[buf->len++] = NL;
       }
@@ -611,28 +609,20 @@ static void read_input(DynamicBuffer *buf)
   }
 }
 
-static size_t write_output(char *output, size_t remaining, bool to_buffer,
-                           bool eof)
+static size_t write_output(char *output, size_t remaining, bool eof)
 {
   if (!output) {
     return 0;
   }
-  char replacement_NUL = to_buffer ? NL : 1;
 
   char *start = output;
   size_t off = 0;
-  int lastrow = (int)Rows - 1;
   while (off < remaining) {
     if (output[off] == NL) {
       // Insert the line
-      if (to_buffer) {
-        output[off] = NUL;
-        ml_append(curwin->w_cursor.lnum++, (char_u *)output, (int)off + 1,
-                  false);
-      } else {
-        screen_del_lines(0, 0, 1, (int)Rows, NULL);
-        screen_puts_len((char_u *)output, (int)off, lastrow, 0, 0);
-      }
+      output[off] = NUL;
+      ml_append(curwin->w_cursor.lnum++, (char_u *)output, (int)off + 1,
+                false);
       size_t skip = off + 1;
       output += skip;
       remaining -= skip;
@@ -642,24 +632,19 @@ static size_t write_output(char *output, size_t remaining, bool to_buffer,
 
     if (output[off] == NUL) {
       // Translate NUL to NL
-      output[off] = replacement_NUL;
+      output[off] = NL;
     }
     off++;
   }
 
   if (eof) {
     if (remaining) {
-      if (to_buffer) {
-        // append unfinished line
-        ml_append(curwin->w_cursor.lnum++, (char_u *)output, 0, false);
-        // remember that the NL was missing
-        curbuf->b_no_eol_lnum = curwin->w_cursor.lnum;
-      } else {
-        screen_del_lines(0, 0, 1, (int)Rows, NULL);
-        screen_puts_len((char_u *)output, (int)remaining, lastrow, 0, 0);
-      }
+      // append unfinished line
+      ml_append(curwin->w_cursor.lnum++, (char_u *)output, 0, false);
+      // remember that the NL was missing
+      curbuf->b_no_eol_lnum = curwin->w_cursor.lnum;
       output += remaining;
-    } else if (to_buffer) {
+    } else {
       curbuf->b_no_eol_lnum = 0;
     }
   }
diff --git a/src/nvim/os/signal.c b/src/nvim/os/signal.c
index fd6d3b32e4..732be072e1 100644
--- a/src/nvim/os/signal.c
+++ b/src/nvim/os/signal.c
@@ -10,6 +10,7 @@
 #endif
 
 #include "nvim/ascii.h"
+#include "nvim/log.h"
 #include "nvim/vim.h"
 #include "nvim/globals.h"
 #include "nvim/memline.h"
@@ -162,7 +163,7 @@ static void on_signal(SignalWatcher *handle, int signum, void *data)
       }
       break;
     default:
-      fprintf(stderr, "Invalid signal %d", signum);
+      ELOG("invalid signal: %d", signum);
       break;
   }
 }
diff --git a/src/nvim/os/time.c b/src/nvim/os/time.c
index c471352c02..290d421acc 100644
--- a/src/nvim/os/time.c
+++ b/src/nvim/os/time.c
@@ -38,33 +38,33 @@ uint64_t os_hrtime(void)
   return uv_hrtime();
 }
 
-/// Sleeps for a certain amount of milliseconds.
+/// Sleeps for `ms` milliseconds.
 ///
-/// @param milliseconds Number of milliseconds to sleep
+/// @param ms          Number of milliseconds to sleep
 /// @param ignoreinput If true, only SIGINT (CTRL-C) can interrupt.
-void os_delay(uint64_t milliseconds, bool ignoreinput)
+void os_delay(uint64_t ms, bool ignoreinput)
 {
   if (ignoreinput) {
-    if (milliseconds > INT_MAX) {
-      milliseconds = INT_MAX;
+    if (ms > INT_MAX) {
+      ms = INT_MAX;
     }
-    LOOP_PROCESS_EVENTS_UNTIL(&main_loop, NULL, (int)milliseconds, got_int);
+    LOOP_PROCESS_EVENTS_UNTIL(&main_loop, NULL, (int)ms, got_int);
   } else {
-    os_microdelay(milliseconds * 1000u, ignoreinput);
+    os_microdelay(ms * 1000u, ignoreinput);
   }
 }
 
-/// Sleeps for a certain amount of microseconds.
+/// Sleeps for `us` microseconds.
 ///
-/// @param ms          Number of microseconds to sleep.
+/// @param us          Number of microseconds to sleep.
 /// @param ignoreinput If true, ignore all input (including SIGINT/CTRL-C).
 ///                    If false, waiting is aborted on any input.
-void os_microdelay(uint64_t ms, bool ignoreinput)
+void os_microdelay(uint64_t us, bool ignoreinput)
 {
   uint64_t elapsed = 0u;
   uint64_t base = uv_hrtime();
   // Convert microseconds to nanoseconds, or UINT64_MAX on overflow.
-  const uint64_t ns = (ms < UINT64_MAX / 1000u) ? ms * 1000u : UINT64_MAX;
+  const uint64_t ns = (us < UINT64_MAX / 1000u) ? us * 1000u : UINT64_MAX;
 
   uv_mutex_lock(&delay_mutex);
 
diff --git a/src/nvim/os/unix_defs.h b/src/nvim/os/unix_defs.h
index 5c9daca476..60a2dfa882 100644
--- a/src/nvim/os/unix_defs.h
+++ b/src/nvim/os/unix_defs.h
@@ -1,9 +1,8 @@
 #ifndef NVIM_OS_UNIX_DEFS_H
 #define NVIM_OS_UNIX_DEFS_H
 
-// Windows doesn't have unistd.h, so we include it here to avoid numerous
-// instances of `#ifdef WIN32'.
 #include <unistd.h>
+#include <sys/param.h>
 
 // POSIX.1-2008 says that NAME_MAX should be in here
 #include <limits.h>
diff --git a/src/nvim/os/win_defs.h b/src/nvim/os/win_defs.h
index 8fd2e51f8b..db93f016bf 100644
--- a/src/nvim/os/win_defs.h
+++ b/src/nvim/os/win_defs.h
@@ -32,11 +32,8 @@
 
 // Windows defines a RGB macro that produces 0x00bbggrr color values for use
 // with GDI. Our macro is different, and we don't use GDI.
-#if defined(RGB)
-# undef RGB
-  // Duplicated from macros.h to avoid include-order sensitivity.
-# define RGB(r, g, b) ((r << 16) | (g << 8) | b)
-#endif
+// Duplicated from macros.h to avoid include-order sensitivity.
+#define RGB_(r, g, b) ((r << 16) | (g << 8) | b)
 
 #ifdef _MSC_VER
 # ifndef inline
@@ -45,6 +42,9 @@
 # ifndef restrict
 #  define restrict __restrict
 # endif
+# ifndef STDIN_FILENO
+#  define STDIN_FILENO _fileno(stdin)
+# endif
 # ifndef STDOUT_FILENO
 #  define STDOUT_FILENO _fileno(stdout)
 # endif
@@ -60,6 +60,7 @@
 
 #ifdef _MSC_VER
 typedef SSIZE_T ssize_t;
+typedef int mode_t;
 #endif
 
 #ifndef SSIZE_MAX