refactor: Extract os_tty_guess_term()

- Also remove feature-detection of uv_set_vterm_state(): instead, on
  Windows we always require libuv to have that function.

4 files changed, 72 insertions, 73 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6b03945317..5a8aef2be2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -364,24 +364,6 @@ include_directories("${PROJECT_SOURCE_DIR}/src")
 find_package(LibUV REQUIRED)
 include_directories(SYSTEM ${LIBUV_INCLUDE_DIRS})
 
-if(WIN32)
-  list(APPEND CMAKE_REQUIRED_INCLUDES "${LIBUV_INCLUDE_DIRS}")
-  list(APPEND CMAKE_REQUIRED_LIBRARIES "${LIBUV_LIBRARIES}")
-  check_c_source_compiles("
-  #include <uv.h>
-
-  int
-  main(void)
-  {
-    uv_set_vterm_state(UV_UNSUPPORTED);
-    return 0;
-  }
-  " UV_HAS_SET_VTERM_STATE)
-  if(UV_HAS_SET_VTERM_STATE)
-    add_definitions(-DNVIM_UV_HAS_SET_VTERM_STATE)
-  endif()
-endif()
-
 find_package(Msgpack 1.0.0 REQUIRED)
 include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
 
diff --git a/src/nvim/os/tty.c b/src/nvim/os/tty.c
new file mode 100644
index 0000000000..bd5b9b4506
--- /dev/null
+++ b/src/nvim/os/tty.c
@@ -0,0 +1,60 @@
+// This is an open source non-commercial project. Dear PVS-Studio, please check
+// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
+
+//
+// Terminal/console utils
+//
+
+#include "nvim/os/os.h"
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "os/tty.c.generated.h"
+#endif
+
+#ifdef WIN32
+# if !defined(ENABLE_VIRTUAL_TERMINAL_PROCESSING)
+#  define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
+# endif
+/// Guesses the terminal-type.  Calls SetConsoleMode() and uv_set_vterm_state()
+/// if appropriate.
+///
+/// @param[in,out] term Name of the guessed terminal, statically-allocated
+/// @param out_fd stdout file descriptor
+void os_tty_guess_term(const char **term, int out_fd)
+{
+  bool winpty = (os_getenv("NVIM") != NULL);
+
+  if (winpty) {
+    // Force TERM=win32con when running in winpty.
+    *term = "win32con";
+    uv_set_vterm_state(UV_UNSUPPORTED);
+    return;
+  }
+
+  bool conemu_ansi = strequal(os_getenv("ConEmuANSI"), "ON");
+  bool vtp = false;
+
+  HANDLE handle = (HANDLE)_get_osfhandle(out_fd);
+  DWORD dwMode;
+  if (handle != INVALID_HANDLE_VALUE && GetConsoleMode(handle, &dwMode)) {
+    dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+    if (SetConsoleMode(handle, dwMode)) {
+      vtp = true;
+    }
+  }
+
+  if (*term == NULL) {
+    if (vtp) {
+      *term = "vtpcon";
+    } else if (conemu_ansi) {
+      *term = "conemu";
+    } else {
+      *term = "win32con";
+    }
+  }
+
+  if (conemu_ansi) {
+    uv_set_vterm_state(UV_SUPPORTED);
+  }
+}
+#endif
diff --git a/src/nvim/os/tty.h b/src/nvim/os/tty.h
new file mode 100644
index 0000000000..d771e63768
--- /dev/null
+++ b/src/nvim/os/tty.h
@@ -0,0 +1,7 @@
+#ifndef NVIM_OS_TTY_H
+#define NVIM_OS_TTY_H
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "os/tty.h.generated.h"
+#endif
+#endif  // NVIM_OS_TTY_H
diff --git a/src/nvim/tui/tui.c b/src/nvim/tui/tui.c
index c73de1049c..ce05ee6846 100644
--- a/src/nvim/tui/tui.c
+++ b/src/nvim/tui/tui.c
@@ -31,6 +31,7 @@
 #include "nvim/event/signal.h"
 #include "nvim/os/input.h"
 #include "nvim/os/os.h"
+#include "nvim/os/tty.h"
 #include "nvim/strings.h"
 #include "nvim/syntax.h"
 #include "nvim/ui_bridge.h"
@@ -63,10 +64,6 @@
 #define UNIBI_SET_NUM_VAR(var, num) (var).i = (num);
 #endif
 
-#if defined(WIN32) && !defined(ENABLE_VIRTUAL_TERMINAL_PROCESSING)
-# define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
-#endif
-
 typedef struct {
   int top, bot, left, right;
 } Rect;
@@ -216,60 +213,13 @@ static void terminfo_start(UI *ui)
   data->out_fd = 1;
   data->out_isatty = os_isatty(data->out_fd);
 
-  // Set up unibilium/terminfo.
   const char *term = os_getenv("TERM");
-  bool conemu_ansi = false;
 #ifdef WIN32
-  bool winpty = false;
-  bool vtp = false;
-  const char *env = os_getenv("VIM_TERMINAL");
-  if (env) {
-    winpty = true;
-  }
-  // If we change to set environment variable in terminal of nvim,
-  // add condition here
-
-  if (!winpty) {
-# ifdef NVIM_UV_HAS_SET_VTERM_STATE
-    env = os_getenv("ConEmuANSI");
-    if (env && !STRCMP(env, "ON")) {
-      conemu_ansi = true;
-    }
-# endif
-
-    HANDLE handle = (HANDLE)_get_osfhandle(data->out_fd);
-    DWORD dwMode;
-    if (handle != INVALID_HANDLE_VALUE && GetConsoleMode(handle, &dwMode)) {
-      dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
-      if (SetConsoleMode(handle, dwMode)) {
-        vtp = true;
-      }
-    }
-  } else {
-    // If it is running under winpty ignore the TERM environment variable and
-    // force it to be win32con.
-    term = "win32con";
-  }
-
-  if (term == NULL) {
-    if (vtp) {
-      term = "vtpcon";
-    } else if (conemu_ansi) {
-      term = "conemu";
-    } else {
-      term = "win32con";
-    }
-  }
-
+  os_tty_guess_term(&term, data->out_fd);
   os_setenv("TERM", term, 1);
-# ifdef NVIM_UV_HAS_SET_VTERM_STATE
-  if (conemu_ansi) {
-    uv_set_vterm_state(UV_SUPPORTED);
-  } else if (winpty) {
-    uv_set_vterm_state(UV_UNSUPPORTED);
-  }
-# endif
 #endif
+
+  // Set up unibilium/terminfo.
   data->ut = unibi_from_env();
   char *termname = NULL;
   if (!term || !data->ut) {
@@ -310,7 +260,7 @@ static void terminfo_start(UI *ui)
     && !!unibi_get_str(data->ut, unibi_parm_insert_line);
   data->can_erase_chars = !!unibi_get_str(data->ut, unibi_erase_chars);
   data->immediate_wrap_after_last_column =
-    conemu_ansi
+    terminfo_is_term_family(term, "conemu")
     || terminfo_is_term_family(term, "cygwin")
     || terminfo_is_term_family(term, "win32con")
     || terminfo_is_term_family(term, "interix");