Introduce nvim namespace: Move files.

Move files from src/ to src/nvim/.
- src/nvim/ becomes the new root dir for nvim executable sources.
- src/libnvim/ is planned to become root dir of the neovim library.

1 files changed, 4041 insertions, 0 deletions

diff --git a/src/nvim/mbyte.c b/src/nvim/mbyte.c
new file mode 100644
index 0000000000..09cb1ce35b
--- /dev/null
+++ b/src/nvim/mbyte.c
@@ -0,0 +1,4041 @@
+/*
+ * VIM - Vi IMproved	by Bram Moolenaar
+ * Multibyte extensions partly by Sung-Hoon Baek
+ *
+ * Do ":help uganda"  in Vim to read copying and usage conditions.
+ * Do ":help credits" in Vim to see a list of people who contributed.
+ * See README.txt for an overview of the Vim source code.
+ */
+/*
+ * mbyte.c: Code specifically for handling multi-byte characters.
+ *
+ * The encoding used in the core is set with 'encoding'.  When 'encoding' is
+ * changed, the following four variables are set (for speed).
+ * Currently these types of character encodings are supported:
+ *
+ * "enc_dbcs"	    When non-zero it tells the type of double byte character
+ *		    encoding (Chinese, Korean, Japanese, etc.).
+ *		    The cell width on the display is equal to the number of
+ *		    bytes.  (exception: DBCS_JPNU with first byte 0x8e)
+ *		    Recognizing the first or second byte is difficult, it
+ *		    requires checking a byte sequence from the start.
+ * "enc_utf8"	    When TRUE use Unicode characters in UTF-8 encoding.
+ *		    The cell width on the display needs to be determined from
+ *		    the character value.
+ *		    Recognizing bytes is easy: 0xxx.xxxx is a single-byte
+ *		    char, 10xx.xxxx is a trailing byte, 11xx.xxxx is a leading
+ *		    byte of a multi-byte character.
+ *		    To make things complicated, up to six composing characters
+ *		    are allowed.  These are drawn on top of the first char.
+ *		    For most editing the sequence of bytes with composing
+ *		    characters included is considered to be one character.
+ * "enc_unicode"    When 2 use 16-bit Unicode characters (or UTF-16).
+ *		    When 4 use 32-but Unicode characters.
+ *		    Internally characters are stored in UTF-8 encoding to
+ *		    avoid NUL bytes.  Conversion happens when doing I/O.
+ *		    "enc_utf8" will also be TRUE.
+ *
+ * "has_mbyte" is set when "enc_dbcs" or "enc_utf8" is non-zero.
+ *
+ * If none of these is TRUE, 8-bit bytes are used for a character.  The
+ * encoding isn't currently specified (TODO).
+ *
+ * 'encoding' specifies the encoding used in the core.  This is in registers,
+ * text manipulation, buffers, etc.  Conversion has to be done when characters
+ * in another encoding are received or send:
+ *
+ *		       clipboard
+ *			   ^
+ *			   | (2)
+ *			   V
+ *		   +---------------+
+ *	      (1)  |		   | (3)
+ *  keyboard ----->|	 core	   |-----> display
+ *		   |		   |
+ *		   +---------------+
+ *			   ^
+ *			   | (4)
+ *			   V
+ *			 file
+ *
+ * (1) Typed characters arrive in the current locale.  Conversion is to be
+ *     done when 'encoding' is different from 'termencoding'.
+ * (2) Text will be made available with the encoding specified with
+ *     'encoding'.  If this is not sufficient, system-specific conversion
+ *     might be required.
+ * (3) For the GUI the correct font must be selected, no conversion done.
+ *     Otherwise, conversion is to be done when 'encoding' differs from
+ *     'termencoding'.  (Different in the GTK+ 2 port -- 'termencoding'
+ *     is always used for both input and output and must always be set to
+ *     "utf-8".  gui_mch_init() does this automatically.)
+ * (4) The encoding of the file is specified with 'fileencoding'.  Conversion
+ *     is to be done when it's different from 'encoding'.
+ *
+ * The viminfo file is a special case: Only text is converted, not file names.
+ * Vim scripts may contain an ":encoding" command.  This has an effect for
+ * some commands, like ":menutrans"
+ */
+
+#include <string.h>
+# include <wchar.h>
+
+#include "vim.h"
+#include "mbyte.h"
+#include "charset.h"
+#include "fileio.h"
+#include "memline.h"
+#include "message.h"
+#include "misc1.h"
+#include "misc2.h"
+#include "memory.h"
+#include "option.h"
+#include "screen.h"
+#include "spell.h"
+#include "ui.h"
+#include "os/os.h"
+
+# define WINBYTE BYTE
+
+static int enc_canon_search(char_u *name);
+static int dbcs_char2len(int c);
+static int dbcs_char2bytes(int c, char_u *buf);
+static int dbcs_ptr2len(char_u *p);
+static int dbcs_ptr2len_len(char_u *p, int size);
+static int utf_ptr2cells_len(char_u *p, int size);
+static int dbcs_char2cells(int c);
+static int dbcs_ptr2cells_len(char_u *p, int size);
+static int dbcs_ptr2char(char_u *p);
+static int utf_safe_read_char_adv(char_u **s, size_t *n);
+
+/*
+ * Lookup table to quickly get the length in bytes of a UTF-8 character from
+ * the first byte of a UTF-8 string.
+ * Bytes which are illegal when used as the first byte have a 1.
+ * The NUL byte has length 1.
+ */
+static char utf8len_tab[256] =
+{
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+  3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1,
+};
+
+/*
+ * Like utf8len_tab above, but using a zero for illegal lead bytes.
+ */
+static char utf8len_tab_zero[256] =
+{
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+  3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0,
+};
+
+/*
+ * Canonical encoding names and their properties.
+ * "iso-8859-n" is handled by enc_canonize() directly.
+ */
+static struct
+{   char *name;         int prop;               int codepage; }
+enc_canon_table[] =
+{
+#define IDX_LATIN_1     0
+  {"latin1",          ENC_8BIT + ENC_LATIN1,  1252},
+#define IDX_ISO_2       1
+  {"iso-8859-2",      ENC_8BIT,               0},
+#define IDX_ISO_3       2
+  {"iso-8859-3",      ENC_8BIT,               0},
+#define IDX_ISO_4       3
+  {"iso-8859-4",      ENC_8BIT,               0},
+#define IDX_ISO_5       4
+  {"iso-8859-5",      ENC_8BIT,               0},
+#define IDX_ISO_6       5
+  {"iso-8859-6",      ENC_8BIT,               0},
+#define IDX_ISO_7       6
+  {"iso-8859-7",      ENC_8BIT,               0},
+#define IDX_ISO_8       7
+  {"iso-8859-8",      ENC_8BIT,               0},
+#define IDX_ISO_9       8
+  {"iso-8859-9",      ENC_8BIT,               0},
+#define IDX_ISO_10      9
+  {"iso-8859-10",     ENC_8BIT,               0},
+#define IDX_ISO_11      10
+  {"iso-8859-11",     ENC_8BIT,               0},
+#define IDX_ISO_13      11
+  {"iso-8859-13",     ENC_8BIT,               0},
+#define IDX_ISO_14      12
+  {"iso-8859-14",     ENC_8BIT,               0},
+#define IDX_ISO_15      13
+  {"iso-8859-15",     ENC_8BIT + ENC_LATIN9,  0},
+#define IDX_KOI8_R      14
+  {"koi8-r",          ENC_8BIT,               0},
+#define IDX_KOI8_U      15
+  {"koi8-u",          ENC_8BIT,               0},
+#define IDX_UTF8        16
+  {"utf-8",           ENC_UNICODE,            0},
+#define IDX_UCS2        17
+  {"ucs-2",           ENC_UNICODE + ENC_ENDIAN_B + ENC_2BYTE, 0},
+#define IDX_UCS2LE      18
+  {"ucs-2le",         ENC_UNICODE + ENC_ENDIAN_L + ENC_2BYTE, 0},
+#define IDX_UTF16       19
+  {"utf-16",          ENC_UNICODE + ENC_ENDIAN_B + ENC_2WORD, 0},
+#define IDX_UTF16LE     20
+  {"utf-16le",        ENC_UNICODE + ENC_ENDIAN_L + ENC_2WORD, 0},
+#define IDX_UCS4        21
+  {"ucs-4",           ENC_UNICODE + ENC_ENDIAN_B + ENC_4BYTE, 0},
+#define IDX_UCS4LE      22
+  {"ucs-4le",         ENC_UNICODE + ENC_ENDIAN_L + ENC_4BYTE, 0},
+
+  /* For debugging DBCS encoding on Unix. */
+#define IDX_DEBUG       23
+  {"debug",           ENC_DBCS,               DBCS_DEBUG},
+#define IDX_EUC_JP      24
+  {"euc-jp",          ENC_DBCS,               DBCS_JPNU},
+#define IDX_SJIS        25
+  {"sjis",            ENC_DBCS,               DBCS_JPN},
+#define IDX_EUC_KR      26
+  {"euc-kr",          ENC_DBCS,               DBCS_KORU},
+#define IDX_EUC_CN      27
+  {"euc-cn",          ENC_DBCS,               DBCS_CHSU},
+#define IDX_EUC_TW      28
+  {"euc-tw",          ENC_DBCS,               DBCS_CHTU},
+#define IDX_BIG5        29
+  {"big5",            ENC_DBCS,               DBCS_CHT},
+
+  /* MS-DOS and MS-Windows codepages are included here, so that they can be
+   * used on Unix too.  Most of them are similar to ISO-8859 encodings, but
+   * not exactly the same. */
+#define IDX_CP437       30
+  {"cp437",           ENC_8BIT,               437},   /* like iso-8859-1 */
+#define IDX_CP737       31
+  {"cp737",           ENC_8BIT,               737},   /* like iso-8859-7 */
+#define IDX_CP775       32
+  {"cp775",           ENC_8BIT,               775},   /* Baltic */
+#define IDX_CP850       33
+  {"cp850",           ENC_8BIT,               850},   /* like iso-8859-4 */
+#define IDX_CP852       34
+  {"cp852",           ENC_8BIT,               852},   /* like iso-8859-1 */
+#define IDX_CP855       35
+  {"cp855",           ENC_8BIT,               855},   /* like iso-8859-2 */
+#define IDX_CP857       36
+  {"cp857",           ENC_8BIT,               857},   /* like iso-8859-5 */
+#define IDX_CP860       37
+  {"cp860",           ENC_8BIT,               860},   /* like iso-8859-9 */
+#define IDX_CP861       38
+  {"cp861",           ENC_8BIT,               861},   /* like iso-8859-1 */
+#define IDX_CP862       39
+  {"cp862",           ENC_8BIT,               862},   /* like iso-8859-1 */
+#define IDX_CP863       40
+  {"cp863",           ENC_8BIT,               863},   /* like iso-8859-8 */
+#define IDX_CP865       41
+  {"cp865",           ENC_8BIT,               865},   /* like iso-8859-1 */
+#define IDX_CP866       42
+  {"cp866",           ENC_8BIT,               866},   /* like iso-8859-5 */
+#define IDX_CP869       43
+  {"cp869",           ENC_8BIT,               869},   /* like iso-8859-7 */
+#define IDX_CP874       44
+  {"cp874",           ENC_8BIT,               874},   /* Thai */
+#define IDX_CP932       45
+  {"cp932",           ENC_DBCS,               DBCS_JPN},
+#define IDX_CP936       46
+  {"cp936",           ENC_DBCS,               DBCS_CHS},
+#define IDX_CP949       47
+  {"cp949",           ENC_DBCS,               DBCS_KOR},
+#define IDX_CP950       48
+  {"cp950",           ENC_DBCS,               DBCS_CHT},
+#define IDX_CP1250      49
+  {"cp1250",          ENC_8BIT,               1250},   /* Czech, Polish, etc. */
+#define IDX_CP1251      50
+  {"cp1251",          ENC_8BIT,               1251},   /* Cyrillic */
+  /* cp1252 is considered to be equal to latin1 */
+#define IDX_CP1253      51
+  {"cp1253",          ENC_8BIT,               1253},   /* Greek */
+#define IDX_CP1254      52
+  {"cp1254",          ENC_8BIT,               1254},   /* Turkish */
+#define IDX_CP1255      53
+  {"cp1255",          ENC_8BIT,               1255},   /* Hebrew */
+#define IDX_CP1256      54
+  {"cp1256",          ENC_8BIT,               1256},   /* Arabic */
+#define IDX_CP1257      55
+  {"cp1257",          ENC_8BIT,               1257},   /* Baltic */
+#define IDX_CP1258      56
+  {"cp1258",          ENC_8BIT,               1258},   /* Vietnamese */
+
+#define IDX_MACROMAN    57
+  {"macroman",        ENC_8BIT + ENC_MACROMAN, 0},      /* Mac OS */
+#define IDX_DECMCS      58
+  {"dec-mcs",         ENC_8BIT,               0},       /* DEC MCS */
+#define IDX_HPROMAN8    59
+  {"hp-roman8",       ENC_8BIT,               0},       /* HP Roman8 */
+#define IDX_COUNT       60
+};
+
+/*
+ * Aliases for encoding names.
+ */
+static struct
+{   char *name;         int canon; }
+enc_alias_table[] =
+{
+  {"ansi",            IDX_LATIN_1},
+  {"iso-8859-1",      IDX_LATIN_1},
+  {"latin2",          IDX_ISO_2},
+  {"latin3",          IDX_ISO_3},
+  {"latin4",          IDX_ISO_4},
+  {"cyrillic",        IDX_ISO_5},
+  {"arabic",          IDX_ISO_6},
+  {"greek",           IDX_ISO_7},
+  {"hebrew",          IDX_ISO_8},
+  {"latin5",          IDX_ISO_9},
+  {"turkish",         IDX_ISO_9},   /* ? */
+  {"latin6",          IDX_ISO_10},
+  {"nordic",          IDX_ISO_10},   /* ? */
+  {"thai",            IDX_ISO_11},   /* ? */
+  {"latin7",          IDX_ISO_13},
+  {"latin8",          IDX_ISO_14},
+  {"latin9",          IDX_ISO_15},
+  {"utf8",            IDX_UTF8},
+  {"unicode",         IDX_UCS2},
+  {"ucs2",            IDX_UCS2},
+  {"ucs2be",          IDX_UCS2},
+  {"ucs-2be",         IDX_UCS2},
+  {"ucs2le",          IDX_UCS2LE},
+  {"utf16",           IDX_UTF16},
+  {"utf16be",         IDX_UTF16},
+  {"utf-16be",        IDX_UTF16},
+  {"utf16le",         IDX_UTF16LE},
+  {"ucs4",            IDX_UCS4},
+  {"ucs4be",          IDX_UCS4},
+  {"ucs-4be",         IDX_UCS4},
+  {"ucs4le",          IDX_UCS4LE},
+  {"utf32",           IDX_UCS4},
+  {"utf-32",          IDX_UCS4},
+  {"utf32be",         IDX_UCS4},
+  {"utf-32be",        IDX_UCS4},
+  {"utf32le",         IDX_UCS4LE},
+  {"utf-32le",        IDX_UCS4LE},
+  {"932",             IDX_CP932},
+  {"949",             IDX_CP949},
+  {"936",             IDX_CP936},
+  {"gbk",             IDX_CP936},
+  {"950",             IDX_CP950},
+  {"eucjp",           IDX_EUC_JP},
+  {"unix-jis",        IDX_EUC_JP},
+  {"ujis",            IDX_EUC_JP},
+  {"shift-jis",       IDX_SJIS},
+  {"euckr",           IDX_EUC_KR},
+  {"5601",            IDX_EUC_KR},      /* Sun: KS C 5601 */
+  {"euccn",           IDX_EUC_CN},
+  {"gb2312",          IDX_EUC_CN},
+  {"euctw",           IDX_EUC_TW},
+  {"japan",           IDX_EUC_JP},
+  {"korea",           IDX_EUC_KR},
+  {"prc",             IDX_EUC_CN},
+  {"chinese",         IDX_EUC_CN},
+  {"taiwan",          IDX_EUC_TW},
+  {"cp950",           IDX_BIG5},
+  {"950",             IDX_BIG5},
+  {"mac",             IDX_MACROMAN},
+  {"mac-roman",       IDX_MACROMAN},
+  {NULL,              0}
+};
+
+#ifndef CP_UTF8
+# define CP_UTF8 65001  /* magic number from winnls.h */
+#endif
+
+/*
+ * Find encoding "name" in the list of canonical encoding names.
+ * Returns -1 if not found.
+ */
+static int enc_canon_search(char_u *name)
+{
+  int i;
+
+  for (i = 0; i < IDX_COUNT; ++i)
+    if (STRCMP(name, enc_canon_table[i].name) == 0)
+      return i;
+  return -1;
+}
+
+
+
+/*
+ * Find canonical encoding "name" in the list and return its properties.
+ * Returns 0 if not found.
+ */
+int enc_canon_props(char_u *name)
+{
+  int i;
+
+  i = enc_canon_search(name);
+  if (i >= 0)
+    return enc_canon_table[i].prop;
+  if (STRNCMP(name, "2byte-", 6) == 0)
+    return ENC_DBCS;
+  if (STRNCMP(name, "8bit-", 5) == 0 || STRNCMP(name, "iso-8859-", 9) == 0)
+    return ENC_8BIT;
+  return 0;
+}
+
+/*
+ * Set up for using multi-byte characters.
+ * Called in three cases:
+ * - by main() to initialize (p_enc == NULL)
+ * - by set_init_1() after 'encoding' was set to its default.
+ * - by do_set() when 'encoding' has been set.
+ * p_enc must have been passed through enc_canonize() already.
+ * Sets the "enc_unicode", "enc_utf8", "enc_dbcs" and "has_mbyte" flags.
+ * Fills mb_bytelen_tab[] and returns NULL when there are no problems.
+ * When there is something wrong: Returns an error message and doesn't change
+ * anything.
+ */
+char_u * mb_init()
+{
+  int i;
+  int idx;
+  int n;
+  int enc_dbcs_new = 0;
+#if defined(USE_ICONV) && !defined(WIN3264) && !defined(WIN32UNIX) \
+  && !defined(MACOS)
+# define LEN_FROM_CONV
+  vimconv_T vimconv;
+  char_u      *p;
+#endif
+
+  if (p_enc == NULL) {
+    /* Just starting up: set the whole table to one's. */
+    for (i = 0; i < 256; ++i)
+      mb_bytelen_tab[i] = 1;
+    input_conv.vc_type = CONV_NONE;
+    input_conv.vc_factor = 1;
+    output_conv.vc_type = CONV_NONE;
+    return NULL;
+  } else if (STRNCMP(p_enc, "8bit-", 5) == 0
+      || STRNCMP(p_enc, "iso-8859-", 9) == 0) {
+    /* Accept any "8bit-" or "iso-8859-" name. */
+    enc_unicode = 0;
+    enc_utf8 = FALSE;
+  } else if (STRNCMP(p_enc, "2byte-", 6) == 0) {
+    /* Unix: accept any "2byte-" name, assume current locale. */
+    enc_dbcs_new = DBCS_2BYTE;
+  } else if ((idx = enc_canon_search(p_enc)) >= 0) {
+    i = enc_canon_table[idx].prop;
+    if (i & ENC_UNICODE) {
+      /* Unicode */
+      enc_utf8 = TRUE;
+      if (i & (ENC_2BYTE | ENC_2WORD))
+        enc_unicode = 2;
+      else if (i & ENC_4BYTE)
+        enc_unicode = 4;
+      else
+        enc_unicode = 0;
+    } else if (i & ENC_DBCS) {
+      /* 2byte, handle below */
+      enc_dbcs_new = enc_canon_table[idx].codepage;
+    } else {
+      /* Must be 8-bit. */
+      enc_unicode = 0;
+      enc_utf8 = FALSE;
+    }
+  } else    /* Don't know what encoding this is, reject it. */
+    return e_invarg;
+
+  if (enc_dbcs_new != 0) {
+    enc_unicode = 0;
+    enc_utf8 = FALSE;
+  }
+  enc_dbcs = enc_dbcs_new;
+  has_mbyte = (enc_dbcs != 0 || enc_utf8);
+
+
+  /* Detect an encoding that uses latin1 characters. */
+  enc_latin1like = (enc_utf8 || STRCMP(p_enc, "latin1") == 0
+      || STRCMP(p_enc, "iso-8859-15") == 0);
+
+  /*
+   * Set the function pointers.
+   */
+  if (enc_utf8) {
+    mb_ptr2len = utfc_ptr2len;
+    mb_ptr2len_len = utfc_ptr2len_len;
+    mb_char2len = utf_char2len;
+    mb_char2bytes = utf_char2bytes;
+    mb_ptr2cells = utf_ptr2cells;
+    mb_ptr2cells_len = utf_ptr2cells_len;
+    mb_char2cells = utf_char2cells;
+    mb_off2cells = utf_off2cells;
+    mb_ptr2char = utf_ptr2char;
+    mb_head_off = utf_head_off;
+  } else if (enc_dbcs != 0) {
+    mb_ptr2len = dbcs_ptr2len;
+    mb_ptr2len_len = dbcs_ptr2len_len;
+    mb_char2len = dbcs_char2len;
+    mb_char2bytes = dbcs_char2bytes;
+    mb_ptr2cells = dbcs_ptr2cells;
+    mb_ptr2cells_len = dbcs_ptr2cells_len;
+    mb_char2cells = dbcs_char2cells;
+    mb_off2cells = dbcs_off2cells;
+    mb_ptr2char = dbcs_ptr2char;
+    mb_head_off = dbcs_head_off;
+  } else {
+    mb_ptr2len = latin_ptr2len;
+    mb_ptr2len_len = latin_ptr2len_len;
+    mb_char2len = latin_char2len;
+    mb_char2bytes = latin_char2bytes;
+    mb_ptr2cells = latin_ptr2cells;
+    mb_ptr2cells_len = latin_ptr2cells_len;
+    mb_char2cells = latin_char2cells;
+    mb_off2cells = latin_off2cells;
+    mb_ptr2char = latin_ptr2char;
+    mb_head_off = latin_head_off;
+  }
+
+  /*
+   * Fill the mb_bytelen_tab[] for MB_BYTE2LEN().
+   */
+#ifdef LEN_FROM_CONV
+  /* When 'encoding' is different from the current locale mblen() won't
+   * work.  Use conversion to "utf-8" instead. */
+  vimconv.vc_type = CONV_NONE;
+  if (enc_dbcs) {
+    p = enc_locale();
+    if (p == NULL || STRCMP(p, p_enc) != 0) {
+      convert_setup(&vimconv, p_enc, (char_u *)"utf-8");
+      vimconv.vc_fail = TRUE;
+    }
+    free(p);
+  }
+#endif
+
+  for (i = 0; i < 256; ++i) {
+    /* Our own function to reliably check the length of UTF-8 characters,
+     * independent of mblen(). */
+    if (enc_utf8)
+      n = utf8len_tab[i];
+    else if (enc_dbcs == 0)
+      n = 1;
+    else {
+      char buf[MB_MAXBYTES + 1];
+      if (i == NUL)             /* just in case mblen() can't handle "" */
+        n = 1;
+      else {
+        buf[0] = i;
+        buf[1] = 0;
+#ifdef LEN_FROM_CONV
+        if (vimconv.vc_type != CONV_NONE) {
+          /*
+           * string_convert() should fail when converting the first
+           * byte of a double-byte character.
+           */
+          p = string_convert(&vimconv, (char_u *)buf, NULL);
+          if (p != NULL) {
+            free(p);
+            n = 1;
+          } else
+            n = 2;
+        } else
+#endif
+        {
+          /*
+           * mblen() should return -1 for invalid (means the leading
+           * multibyte) character.  However there are some platforms
+           * where mblen() returns 0 for invalid character.
+           * Therefore, following condition includes 0.
+           */
+          ignored = mblen(NULL, 0);             /* First reset the state. */
+          if (mblen(buf, (size_t)1) <= 0)
+            n = 2;
+          else
+            n = 1;
+        }
+      }
+    }
+    mb_bytelen_tab[i] = n;
+  }
+
+#ifdef LEN_FROM_CONV
+  convert_setup(&vimconv, NULL, NULL);
+#endif
+
+  /* The cell width depends on the type of multi-byte characters. */
+  (void)init_chartab();
+
+  /* When enc_utf8 is set or reset, (de)allocate ScreenLinesUC[] */
+  screenalloc(FALSE);
+
+  /* When using Unicode, set default for 'fileencodings'. */
+  if (enc_utf8 && !option_was_set((char_u *)"fencs"))
+    set_string_option_direct((char_u *)"fencs", -1,
+        (char_u *)"ucs-bom,utf-8,default,latin1", OPT_FREE, 0);
+
+#ifdef HAVE_WORKING_LIBINTL
+  /* GNU gettext 0.10.37 supports this feature: set the codeset used for
+   * translated messages independently from the current locale. */
+  (void)bind_textdomain_codeset(VIMPACKAGE,
+      enc_utf8 ? "utf-8" : (char *)p_enc);
+#endif
+
+
+  /* Fire an autocommand to let people do custom font setup. This must be
+   * after Vim has been setup for the new encoding. */
+  apply_autocmds(EVENT_ENCODINGCHANGED, NULL, (char_u *)"", FALSE, curbuf);
+
+  /* Need to reload spell dictionaries */
+  spell_reload();
+
+  return NULL;
+}
+
+/*
+ * Return the size of the BOM for the current buffer:
+ * 0 - no BOM
+ * 2 - UCS-2 or UTF-16 BOM
+ * 4 - UCS-4 BOM
+ * 3 - UTF-8 BOM
+ */
+int bomb_size()
+{
+  int n = 0;
+
+  if (curbuf->b_p_bomb && !curbuf->b_p_bin) {
+    if (*curbuf->b_p_fenc == NUL) {
+      if (enc_utf8) {
+        if (enc_unicode != 0)
+          n = enc_unicode;
+        else
+          n = 3;
+      }
+    } else if (STRCMP(curbuf->b_p_fenc, "utf-8") == 0)
+      n = 3;
+    else if (STRNCMP(curbuf->b_p_fenc, "ucs-2", 5) == 0
+        || STRNCMP(curbuf->b_p_fenc, "utf-16", 6) == 0)
+      n = 2;
+    else if (STRNCMP(curbuf->b_p_fenc, "ucs-4", 5) == 0)
+      n = 4;
+  }
+  return n;
+}
+
+/*
+ * Remove all BOM from "s" by moving remaining text.
+ */
+void remove_bom(char_u *s)
+{
+  if (enc_utf8) {
+    char_u *p = s;
+
+    while ((p = vim_strbyte(p, 0xef)) != NULL) {
+      if (p[1] == 0xbb && p[2] == 0xbf)
+        STRMOVE(p, p + 3);
+      else
+        ++p;
+    }
+  }
+}
+
+/*
+ * Get class of pointer:
+ * 0 for blank or NUL
+ * 1 for punctuation
+ * 2 for an (ASCII) word character
+ * >2 for other word characters
+ */
+int mb_get_class(char_u *p)
+{
+  return mb_get_class_buf(p, curbuf);
+}
+
+int mb_get_class_buf(char_u *p, buf_T *buf)
+{
+  if (MB_BYTE2LEN(p[0]) == 1) {
+    if (p[0] == NUL || vim_iswhite(p[0]))
+      return 0;
+    if (vim_iswordc_buf(p[0], buf))
+      return 2;
+    return 1;
+  }
+  if (enc_dbcs != 0 && p[0] != NUL && p[1] != NUL)
+    return dbcs_class(p[0], p[1]);
+  if (enc_utf8)
+    return utf_class(utf_ptr2char(p));
+  return 0;
+}
+
+/*
+ * Get class of a double-byte character.  This always returns 3 or bigger.
+ * TODO: Should return 1 for punctuation.
+ */
+int dbcs_class(unsigned lead, unsigned trail)
+{
+  switch (enc_dbcs) {
+    /* please add classify routine for your language in here */
+
+    case DBCS_JPNU:       /* ? */
+    case DBCS_JPN:
+      {
+        /* JIS code classification */
+        unsigned char lb = lead;
+        unsigned char tb = trail;
+
+        /* convert process code to JIS */
+        /*
+         * XXX: Code page identification can not use with all
+         *	    system! So, some other encoding information
+         *	    will be needed.
+         *	    In japanese: SJIS,EUC,UNICODE,(JIS)
+         *	    Note that JIS-code system don't use as
+         *	    process code in most system because it uses
+         *	    escape sequences(JIS is context depend encoding).
+         */
+        /* assume process code is JAPANESE-EUC */
+        lb &= 0x7f;
+        tb &= 0x7f;
+        /* exceptions */
+        switch (lb << 8 | tb) {
+          case 0x2121:                 /* ZENKAKU space */
+            return 0;
+          case 0x2122:                 /* TOU-TEN (Japanese comma) */
+          case 0x2123:                 /* KU-TEN (Japanese period) */
+          case 0x2124:                 /* ZENKAKU comma */
+          case 0x2125:                 /* ZENKAKU period */
+            return 1;
+          case 0x213c:                 /* prolongedsound handled as KATAKANA */
+            return 13;
+        }
+        /* sieved by KU code */
+        switch (lb) {
+          case 0x21:
+          case 0x22:
+            /* special symbols */
+            return 10;
+          case 0x23:
+            /* alpha-numeric */
+            return 11;
+          case 0x24:
+            /* hiragana */
+            return 12;
+          case 0x25:
+            /* katakana */
+            return 13;
+          case 0x26:
+            /* greek */
+            return 14;
+          case 0x27:
+            /* russian */
+            return 15;
+          case 0x28:
+            /* lines */
+            return 16;
+          default:
+            /* kanji */
+            return 17;
+        }
+      }
+
+    case DBCS_KORU:       /* ? */
+    case DBCS_KOR:
+      {
+        /* KS code classification */
+        unsigned char c1 = lead;
+        unsigned char c2 = trail;
+
+        /*
+         * 20 : Hangul
+         * 21 : Hanja
+         * 22 : Symbols
+         * 23 : Alpha-numeric/Roman Letter (Full width)
+         * 24 : Hangul Letter(Alphabet)
+         * 25 : Roman Numeral/Greek Letter
+         * 26 : Box Drawings
+         * 27 : Unit Symbols
+         * 28 : Circled/Parenthesized Letter
+         * 29 : Hiragana/Katakana
+         * 30 : Cyrillic Letter
+         */
+
+        if (c1 >= 0xB0 && c1 <= 0xC8)
+          /* Hangul */
+          return 20;
+
+        else if (c1 >= 0xCA && c1 <= 0xFD)
+          /* Hanja */
+          return 21;
+        else switch (c1) {
+          case 0xA1:
+          case 0xA2:
+            /* Symbols */
+            return 22;
+          case 0xA3:
+            /* Alpha-numeric */
+            return 23;
+          case 0xA4:
+            /* Hangul Letter(Alphabet) */
+            return 24;
+          case 0xA5:
+            /* Roman Numeral/Greek Letter */
+            return 25;
+          case 0xA6:
+            /* Box Drawings */
+            return 26;
+          case 0xA7:
+            /* Unit Symbols */
+            return 27;
+          case 0xA8:
+          case 0xA9:
+            if (c2 <= 0xAF)
+              return 25;                    /* Roman Letter */
+            else if (c2 >= 0xF6)
+              return 22;                    /* Symbols */
+            else
+              /* Circled/Parenthesized Letter */
+              return 28;
+          case 0xAA:
+          case 0xAB:
+            /* Hiragana/Katakana */
+            return 29;
+          case 0xAC:
+            /* Cyrillic Letter */
+            return 30;
+        }
+      }
+    default:
+      break;
+  }
+  return 3;
+}
+
+/*
+ * mb_char2len() function pointer.
+ * Return length in bytes of character "c".
+ * Returns 1 for a single-byte character.
+ */
+int latin_char2len(int c)
+{
+  return 1;
+}
+
+static int dbcs_char2len(int c)
+{
+  if (c >= 0x100)
+    return 2;
+  return 1;
+}
+
+/*
+ * mb_char2bytes() function pointer.
+ * Convert a character to its bytes.
+ * Returns the length in bytes.
+ */
+int latin_char2bytes(int c, char_u *buf)
+{
+  buf[0] = c;
+  return 1;
+}
+
+static int dbcs_char2bytes(int c, char_u *buf)
+{
+  if (c >= 0x100) {
+    buf[0] = (unsigned)c >> 8;
+    buf[1] = c;
+    /* Never use a NUL byte, it causes lots of trouble.  It's an invalid
+     * character anyway. */
+    if (buf[1] == NUL)
+      buf[1] = '\n';
+    return 2;
+  }
+  buf[0] = c;
+  return 1;
+}
+
+/*
+ * mb_ptr2len() function pointer.
+ * Get byte length of character at "*p" but stop at a NUL.
+ * For UTF-8 this includes following composing characters.
+ * Returns 0 when *p is NUL.
+ */
+int latin_ptr2len(char_u *p)
+{
+  return MB_BYTE2LEN(*p);
+}
+
+static int dbcs_ptr2len(char_u *p)
+{
+  int len;
+
+  /* Check if second byte is not missing. */
+  len = MB_BYTE2LEN(*p);
+  if (len == 2 && p[1] == NUL)
+    len = 1;
+  return len;
+}
+
+/*
+ * mb_ptr2len_len() function pointer.
+ * Like mb_ptr2len(), but limit to read "size" bytes.
+ * Returns 0 for an empty string.
+ * Returns 1 for an illegal char or an incomplete byte sequence.
+ */
+int latin_ptr2len_len(char_u *p, int size)
+{
+  if (size < 1 || *p == NUL)
+    return 0;
+  return 1;
+}
+
+static int dbcs_ptr2len_len(char_u *p, int size)
+{
+  int len;
+
+  if (size < 1 || *p == NUL)
+    return 0;
+  if (size == 1)
+    return 1;
+  /* Check that second byte is not missing. */
+  len = MB_BYTE2LEN(*p);
+  if (len == 2 && p[1] == NUL)
+    len = 1;
+  return len;
+}
+
+struct interval {
+  long first;
+  long last;
+};
+static int intable(struct interval *table, size_t size, int c);
+
+/*
+ * Return TRUE if "c" is in "table[size / sizeof(struct interval)]".
+ */
+static int intable(struct interval *table, size_t size, int c)
+{
+  int mid, bot, top;
+
+  /* first quick check for Latin1 etc. characters */
+  if (c < table[0].first)
+    return FALSE;
+
+  /* binary search in table */
+  bot = 0;
+  top = (int)(size / sizeof(struct interval) - 1);
+  while (top >= bot) {
+    mid = (bot + top) / 2;
+    if (table[mid].last < c)
+      bot = mid + 1;
+    else if (table[mid].first > c)
+      top = mid - 1;
+    else
+      return TRUE;
+  }
+  return FALSE;
+}
+
+/*
+ * For UTF-8 character "c" return 2 for a double-width character, 1 for others.
+ * Returns 4 or 6 for an unprintable character.
+ * Is only correct for characters >= 0x80.
+ * When p_ambw is "double", return 2 for a character with East Asian Width
+ * class 'A'(mbiguous).
+ */
+int utf_char2cells(int c)
+{
+  /* Sorted list of non-overlapping intervals of East Asian double width
+   * characters, generated with ../runtime/tools/unicode.vim. */
+  static struct interval doublewidth[] =
+  {
+    {0x1100, 0x115f},
+    {0x11a3, 0x11a7},
+    {0x11fa, 0x11ff},
+    {0x2329, 0x232a},
+    {0x2e80, 0x2e99},
+    {0x2e9b, 0x2ef3},
+    {0x2f00, 0x2fd5},
+    {0x2ff0, 0x2ffb},
+    {0x3000, 0x3029},
+    {0x3030, 0x303e},
+    {0x3041, 0x3096},
+    {0x309b, 0x30ff},
+    {0x3105, 0x312d},
+    {0x3131, 0x318e},
+    {0x3190, 0x31b7},
+    {0x31c0, 0x31e3},
+    {0x31f0, 0x321e},
+    {0x3220, 0x3247},
+    {0x3250, 0x32fe},
+    {0x3300, 0x4dbf},
+    {0x4e00, 0xa48c},
+    {0xa490, 0xa4c6},
+    {0xa960, 0xa97c},
+    {0xac00, 0xd7a3},
+    {0xd7b0, 0xd7c6},
+    {0xd7cb, 0xd7fb},
+    {0xf900, 0xfaff},
+    {0xfe10, 0xfe19},
+    {0xfe30, 0xfe52},
+    {0xfe54, 0xfe66},
+    {0xfe68, 0xfe6b},
+    {0xff01, 0xff60},
+    {0xffe0, 0xffe6},
+    {0x1f200, 0x1f200},
+    {0x1f210, 0x1f231},
+    {0x1f240, 0x1f248},
+    {0x20000, 0x2fffd},
+    {0x30000, 0x3fffd}
+  };
+  /* Sorted list of non-overlapping intervals of East Asian Ambiguous
+   * characters, generated with ../runtime/tools/unicode.vim. */
+  static struct interval ambiguous[] =
+  {
+    {0x00a1, 0x00a1},
+    {0x00a4, 0x00a4},
+    {0x00a7, 0x00a8},
+    {0x00aa, 0x00aa},
+    {0x00ad, 0x00ae},
+    {0x00b0, 0x00b4},
+    {0x00b6, 0x00ba},
+    {0x00bc, 0x00bf},
+    {0x00c6, 0x00c6},
+    {0x00d0, 0x00d0},
+    {0x00d7, 0x00d8},
+    {0x00de, 0x00e1},
+    {0x00e6, 0x00e6},
+    {0x00e8, 0x00ea},
+    {0x00ec, 0x00ed},
+    {0x00f0, 0x00f0},
+    {0x00f2, 0x00f3},
+    {0x00f7, 0x00fa},
+    {0x00fc, 0x00fc},
+    {0x00fe, 0x00fe},
+    {0x0101, 0x0101},
+    {0x0111, 0x0111},
+    {0x0113, 0x0113},
+    {0x011b, 0x011b},
+    {0x0126, 0x0127},
+    {0x012b, 0x012b},
+    {0x0131, 0x0133},
+    {0x0138, 0x0138},
+    {0x013f, 0x0142},
+    {0x0144, 0x0144},
+    {0x0148, 0x014b},
+    {0x014d, 0x014d},
+    {0x0152, 0x0153},
+    {0x0166, 0x0167},
+    {0x016b, 0x016b},
+    {0x01ce, 0x01ce},
+    {0x01d0, 0x01d0},
+    {0x01d2, 0x01d2},
+    {0x01d4, 0x01d4},
+    {0x01d6, 0x01d6},
+    {0x01d8, 0x01d8},
+    {0x01da, 0x01da},
+    {0x01dc, 0x01dc},
+    {0x0251, 0x0251},
+    {0x0261, 0x0261},
+    {0x02c4, 0x02c4},
+    {0x02c7, 0x02c7},
+    {0x02c9, 0x02cb},
+    {0x02cd, 0x02cd},
+    {0x02d0, 0x02d0},
+    {0x02d8, 0x02db},
+    {0x02dd, 0x02dd},
+    {0x02df, 0x02df},
+    {0x0391, 0x03a1},
+    {0x03a3, 0x03a9},
+    {0x03b1, 0x03c1},
+    {0x03c3, 0x03c9},
+    {0x0401, 0x0401},
+    {0x0410, 0x044f},
+    {0x0451, 0x0451},
+    {0x2010, 0x2010},
+    {0x2013, 0x2016},
+    {0x2018, 0x2019},
+    {0x201c, 0x201d},
+    {0x2020, 0x2022},
+    {0x2024, 0x2027},
+    {0x2030, 0x2030},
+    {0x2032, 0x2033},
+    {0x2035, 0x2035},
+    {0x203b, 0x203b},
+    {0x203e, 0x203e},
+    {0x2074, 0x2074},
+    {0x207f, 0x207f},
+    {0x2081, 0x2084},
+    {0x20ac, 0x20ac},
+    {0x2103, 0x2103},
+    {0x2105, 0x2105},
+    {0x2109, 0x2109},
+    {0x2113, 0x2113},
+    {0x2116, 0x2116},
+    {0x2121, 0x2122},
+    {0x2126, 0x2126},
+    {0x212b, 0x212b},
+    {0x2153, 0x2154},
+    {0x215b, 0x