diff options
| -rw-r--r-- | src/nvim/charset.c | 356 |
1 files changed, 102 insertions, 254 deletions
diff --git a/src/nvim/charset.c b/src/nvim/charset.c index c501b7e83f..4d150c3230 100644 --- a/src/nvim/charset.c +++ b/src/nvim/charset.c @@ -59,12 +59,8 @@ static char_u g_chartab[256]; /// Depends on the option settings 'iskeyword', 'isident', 'isfname', /// 'isprint' and 'encoding'. /// -/// The index in g_chartab[] depends on 'encoding': -/// - For non-multi-byte index with the byte (same as the character). -/// - For DBCS index with the first byte. -/// - For UTF-8 index with the character (when first byte is up to 0x80 it is -/// the same as the character, if the first byte is 0x80 and above it depends -/// on further bytes). +/// The index in g_chartab[] is the character when first byte is up to 0x80, +/// if the first byte is 0x80 and above it depends on further bytes. /// /// The contents of g_chartab[]: /// - The lower two bits, masked by CT_CELL_MASK, give the number of display @@ -118,15 +114,9 @@ int buf_init_chartab(buf_T *buf, int global) } while (c < 256) { - if (enc_utf8 && (c >= 0xa0)) { + if (c >= 0xa0) { // UTF-8: bytes 0xa0 - 0xff are printable (latin1) g_chartab[c++] = CT_PRINT_CHAR + 1; - } else if ((enc_dbcs == DBCS_JPNU) && (c == 0x8e)) { - // euc-jp characters starting with 0x8e are single width - g_chartab[c++] = CT_PRINT_CHAR + 1; - } else if ((enc_dbcs != 0) && (MB_BYTE2LEN(c) == 2)) { - // other double-byte chars can be printable AND double-width - g_chartab[c++] = CT_PRINT_CHAR + 2; } else { // the rest is unprintable by default g_chartab[c++] = (dy_flags & DY_UHEX) ? 4 : 2; @@ -134,10 +124,8 @@ int buf_init_chartab(buf_T *buf, int global) } // Assume that every multi-byte char is a filename character. - for (c = 1; c < 256; ++c) { - if (((enc_dbcs != 0) && (MB_BYTE2LEN(c) > 1)) - || ((enc_dbcs == DBCS_JPNU) && (c == 0x8e)) - || (enc_utf8 && (c >= 0xa0))) { + for (c = 1; c < 256; c++) { + if (c >= 0xa0) { g_chartab[c] |= CT_FNAME_CHAR; } } @@ -146,15 +134,6 @@ int buf_init_chartab(buf_T *buf, int global) // Init word char flags all to false memset(buf->b_chartab, 0, (size_t)32); - if (enc_dbcs != 0) { - for (c = 0; c < 256; ++c) { - // double-byte characters are probably word characters - if (MB_BYTE2LEN(c) == 2) { - SET_CHARTAB(buf, c); - } - } - } - // In lisp mode the '-' character is included in keywords. if (buf->b_p_lisp) { SET_CHARTAB(buf, '-'); @@ -189,10 +168,8 @@ int buf_init_chartab(buf_T *buf, int global) if (ascii_isdigit(*p)) { c = getdigits_int(&p); - } else if (has_mbyte) { - c = mb_ptr2char_adv(&p); } else { - c = *p++; + c = mb_ptr2char_adv(&p); } c2 = -1; @@ -201,10 +178,8 @@ int buf_init_chartab(buf_T *buf, int global) if (ascii_isdigit(*p)) { c2 = getdigits_int(&p); - } else if (has_mbyte) { - c2 = mb_ptr2char_adv(&p); } else { - c2 = *p++; + c2 = mb_ptr2char_adv(&p); } } @@ -251,8 +226,7 @@ int buf_init_chartab(buf_T *buf, int global) // that we can detect it from the first byte. if (((c < ' ') || (c > '~') - || (p_altkeymap && (F_isalpha(c) || F_isdigit(c)))) - && !(enc_dbcs && (MB_BYTE2LEN(c) == 2))) { + || (p_altkeymap && (F_isalpha(c) || F_isdigit(c))))) { if (tilde) { g_chartab[c] = (uint8_t)((g_chartab[c] & ~CT_CELL_MASK) + ((dy_flags & DY_UHEX) ? 4 : 2)); @@ -313,7 +287,7 @@ void trans_characters(char_u *buf, int bufsize) while (*buf != 0) { // Assume a multi-byte character doesn't need translation. - if (has_mbyte && ((trs_len = (*mb_ptr2len)(buf)) > 1)) { + if ((trs_len = (*mb_ptr2len)(buf)) > 1) { len -= trs_len; } else { trs = transchar_byte(*buf); @@ -347,44 +321,40 @@ char_u *transstr(char_u *s) FUNC_ATTR_NONNULL_RET size_t l; char_u hexbuf[11]; - if (has_mbyte) { - // Compute the length of the result, taking account of unprintable - // multi-byte characters. - size_t len = 0; - p = s; + // Compute the length of the result, taking account of unprintable + // multi-byte characters. + size_t len = 0; + p = s; - while (*p != NUL) { - if ((l = (size_t)(*mb_ptr2len)(p)) > 1) { - c = (*mb_ptr2char)(p); - p += l; + while (*p != NUL) { + if ((l = (size_t)(*mb_ptr2len)(p)) > 1) { + c = (*mb_ptr2char)(p); + p += l; - if (vim_isprintc(c)) { - len += l; - } else { - transchar_hex(hexbuf, c); - len += STRLEN(hexbuf); - } + if (vim_isprintc(c)) { + len += l; } else { - l = (size_t)byte2cells(*p++); + transchar_hex(hexbuf, c); + len += STRLEN(hexbuf); + } + } else { + l = (size_t)byte2cells(*p++); - if (l > 0) { - len += l; - } else { - // illegal byte sequence - len += 4; - } + if (l > 0) { + len += l; + } else { + // illegal byte sequence + len += 4; } } - res = xmallocz(len); - } else { - res = xmallocz((size_t)vim_strsize(s)); } + res = xmallocz(len); *res = NUL; p = s; while (*p != NUL) { - if (has_mbyte && ((l = (size_t)(*mb_ptr2len)(p)) > 1)) { + if ((l = (size_t)(*mb_ptr2len)(p)) > 1) { c = (*mb_ptr2char)(p); if (vim_isprintc(c)) { @@ -443,59 +413,49 @@ char_u* str_foldcase(char_u *str, int orglen, char_u *buf, int buflen) // Make each character lower case. i = 0; while (STR_CHAR(i) != NUL) { - if (enc_utf8 || (has_mbyte && (MB_BYTE2LEN(STR_CHAR(i)) > 1))) { - if (enc_utf8) { - int c = utf_ptr2char(STR_PTR(i)); - int olen = utf_ptr2len(STR_PTR(i)); - int lc = utf_tolower(c); - - // Only replace the character when it is not an invalid - // sequence (ASCII character or more than one byte) and - // utf_tolower() doesn't return the original character. - if (((c < 0x80) || (olen > 1)) && (c != lc)) { - int nlen = utf_char2len(lc); - - // If the byte length changes need to shift the following - // characters forward or backward. - if (olen != nlen) { - if (nlen > olen) { - if (buf == NULL) { - ga_grow(&ga, nlen - olen + 1); - } else { - if (len + nlen - olen >= buflen) { - // out of memory, keep old char - lc = c; - nlen = olen; - } - } - } - - if (olen != nlen) { - if (buf == NULL) { - STRMOVE(GA_PTR(i) + nlen, GA_PTR(i) + olen); - ga.ga_len += nlen - olen; - } else { - STRMOVE(buf + i + nlen, buf + i + olen); - len += nlen - olen; - } + int c = utf_ptr2char(STR_PTR(i)); + int olen = utf_ptr2len(STR_PTR(i)); + int lc = utf_tolower(c); + + // Only replace the character when it is not an invalid + // sequence (ASCII character or more than one byte) and + // utf_tolower() doesn't return the original character. + if (((c < 0x80) || (olen > 1)) && (c != lc)) { + int nlen = utf_char2len(lc); + + // If the byte length changes need to shift the following + // characters forward or backward. + if (olen != nlen) { + if (nlen > olen) { + if (buf == NULL) { + ga_grow(&ga, nlen - olen + 1); + } else { + if (len + nlen - olen >= buflen) { + // out of memory, keep old char + lc = c; + nlen = olen; } } - (void)utf_char2bytes(lc, STR_PTR(i)); } - } - // skip to next multi-byte char - i += (*mb_ptr2len)(STR_PTR(i)); - } else { - if (buf == NULL) { - GA_CHAR(i) = (char_u)TOLOWER_LOC(GA_CHAR(i)); - } else { - buf[i] = (char_u)TOLOWER_LOC(buf[i]); + if (olen != nlen) { + if (buf == NULL) { + STRMOVE(GA_PTR(i) + nlen, GA_PTR(i) + olen); + ga.ga_len += nlen - olen; + } else { + STRMOVE(buf + i + nlen, buf + i + olen); + len += nlen - olen; + } + } } - ++i; + (void)utf_char2bytes(lc, STR_PTR(i)); } + + // skip to next multi-byte char + i += (*mb_ptr2len)(STR_PTR(i)); } + if (buf == NULL) { return (char_u *)ga.ga_data; } @@ -545,7 +505,7 @@ char_u* transchar(int c) /// @return pointer to translated character in transchar_buf. char_u* transchar_byte(int c) { - if (enc_utf8 && (c >= 0x80)) { + if (c >= 0x80) { transchar_nonprint(transchar_buf, c); return transchar_buf; } @@ -578,7 +538,7 @@ void transchar_nonprint(char_u *buf, int c) buf[1] = (char_u)(c ^ 0x40); buf[2] = NUL; - } else if (enc_utf8 && (c >= 0x80)) { + } else if (c >= 0x80) { transchar_hex(buf, c); } else if ((c >= ' ' + 0x80) && (c <= '~' + 0x80)) { // 0xa0 - 0xfe @@ -632,15 +592,15 @@ static unsigned nr2hex(unsigned c) /// Caller must make sure 0 <= b <= 255. /// For multi-byte mode "b" must be the first byte of a character. /// A TAB is counted as two cells: "^I". -/// For UTF-8 mode this will return 0 for bytes >= 0x80, because the number of -/// cells depends on further bytes. +/// This will return 0 for bytes >= 0x80, because the number of +/// cells depends on further bytes in UTF-8. /// /// @param b /// /// @reeturn Number of display cells. int byte2cells(int b) { - if (enc_utf8 && (b >= 0x80)) { + if (b >= 0x80) { return 0; } return g_chartab[b] & CT_CELL_MASK; @@ -662,18 +622,7 @@ int char2cells(int c) if (c >= 0x80) { // UTF-8: above 0x80 need to check the value - if (enc_utf8) { - return utf_char2cells(c); - } - - // DBCS: double-byte means double-width, except for euc-jp with first - // byte 0x8e - if ((enc_dbcs != 0) && (c >= 0x100)) { - if ((enc_dbcs == DBCS_JPNU) && (((unsigned)c >> 8) == 0x8e)) { - return 1; - } - return 2; - } + return utf_char2cells(c); } return g_chartab[c & 0xff] & CT_CELL_MASK; } @@ -687,7 +636,7 @@ int char2cells(int c) int ptr2cells(char_u *p) { // For UTF-8 we need to look at more bytes if the first byte is >= 0x80. - if (enc_utf8 && (*p >= 0x80)) { + if (*p >= 0x80) { return utf_ptr2cells(p); } @@ -722,14 +671,10 @@ int vim_strnsize(char_u *s, int len) assert(s != NULL); int size = 0; while (*s != NUL && --len >= 0) { - if (has_mbyte) { - int l = (*mb_ptr2len)(s); - size += ptr2cells(s); - s += l; - len -= l - 1; - } else { - size += byte2cells(*s++); - } + int l = (*mb_ptr2len)(s); + size += ptr2cells(s); + s += l; + len -= l - 1; } return size; } @@ -840,13 +785,7 @@ bool vim_iswordc_buf(int c, buf_T *buf) FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ARG(2) { if (c >= 0x100) { - if (enc_dbcs != 0) { - return dbcs_class((unsigned)c >> 8, (unsigned)(c & 0xff)) >= 2; - } - - if (enc_utf8) { - return utf_class(c) >= 2; - } + return utf_class(c) >= 2; } return c > 0 && c < 0x100 && GET_CHARTAB(buf, c) != 0; } @@ -859,7 +798,7 @@ bool vim_iswordc_buf(int c, buf_T *buf) bool vim_iswordp(char_u *p) FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - if (has_mbyte && (MB_BYTE2LEN(*p) > 1)) { + if (MB_BYTE2LEN(*p) > 1) { return mb_get_class(p) >= 2; } return GET_CHARTAB(curbuf, *p) != 0; @@ -875,7 +814,7 @@ bool vim_iswordp(char_u *p) bool vim_iswordp_buf(char_u *p, buf_T *buf) FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - if (has_mbyte && (MB_BYTE2LEN(*p) > 1)) { + if (MB_BYTE2LEN(*p) > 1) { return mb_get_class(p) >= 2; } return GET_CHARTAB(buf, *p) != 0; @@ -913,7 +852,7 @@ bool vim_isfilec_or_wc(int c) bool vim_isprintc(int c) FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT { - if (enc_utf8 && (c >= 0x100)) { + if (c >= 0x100) { return utf_printable(c); } return c >= 0x100 || (c > 0 && (g_chartab[c] & CT_PRINT_CHAR)); @@ -928,14 +867,10 @@ bool vim_isprintc(int c) bool vim_isprintc_strict(int c) FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT { - if ((enc_dbcs != 0) && (c < 0x100) && (MB_BYTE2LEN(c) > 1)) { - return false; - } - - if (enc_utf8 && (c >= 0x100)) { + if (c >= 0x100) { return utf_printable(c); } - return c >= 0x100 || (c > 0 && (g_chartab[c] & CT_PRINT_CHAR)); + return c > 0 && (g_chartab[c] & CT_PRINT_CHAR); } /// like chartabsize(), but also check for line breaks on the screen @@ -1052,8 +987,7 @@ int win_lbr_chartabsize(win_T *wp, char_u *line, char_u *s, colnr_T col, int *he break; } } - } else if (has_mbyte - && (size == 2) + } else if ((size == 2) && (MB_BYTE2LEN(*s) > 1) && wp->w_p_wrap && in_win_border(wp, col)) { @@ -1251,28 +1185,24 @@ void getvcol(win_T *wp, pos_T *pos, colnr_T *start, colnr_T *cursor, if (c == TAB) { incr = ts - (vcol % ts); } else { - if (has_mbyte) { - // For utf-8, if the byte is >= 0x80, need to look at - // further bytes to find the cell width. - if (enc_utf8 && (c >= 0x80)) { - incr = utf_ptr2cells(ptr); - } else { - incr = g_chartab[c] & CT_CELL_MASK; - } - - // If a double-cell char doesn't fit at the end of a line - // it wraps to the next line, it's like this char is three - // cells wide. - if ((incr == 2) - && wp->w_p_wrap - && (MB_BYTE2LEN(*ptr) > 1) - && in_win_border(wp, vcol)) { - ++incr; - head = 1; - } + // For utf-8, if the byte is >= 0x80, need to look at + // further bytes to find the cell width. + if (c >= 0x80) { + incr = utf_ptr2cells(ptr); } else { incr = g_chartab[c] & CT_CELL_MASK; } + + // If a double-cell char doesn't fit at the end of a line + // it wraps to the next line, it's like this char is three + // cells wide. + if ((incr == 2) + && wp->w_p_wrap + && (MB_BYTE2LEN(*ptr) > 1) + && in_win_border(wp, vcol)) { + incr++; + head = 1; + } } if ((posptr != NULL) && (ptr >= posptr)) { @@ -1557,36 +1487,6 @@ char_u* skiptohex(char_u *q) // islower()/toupper() etc. do not work properly: they crash when used with // invalid values or can't handle latin1 when the locale is C. // Speed is most important here. -#define LATIN1LOWER 'l' -#define LATIN1UPPER 'U' - -static char_u latin1flags[257] = - " " - " UUUUUUUUUUUUUUUUUUUUUUUUUU llllllllllllllllllllllllll " - " " - "UUUUUUUUUUUUUUUUUUUUUUU UUUUUUUllllllllllllllllllllllll llllllll"; -static char_u latin1upper[257] = - " !\"#$%&'()*+,-./0123456789:;<=>" - "?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~" - "\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e" - "\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e" - "\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae" - "\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe" - "\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce" - "\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde" - "\xdf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce" - "\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xf7\xd8\xd9\xda\xdb\xdc\xdd\xde\xff"; -static char_u latin1lower[257] = - " !\"#$%&'()*+,-./0123456789:;<=>" - "?@abcdefghijklmnopqrstuvwxyz[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" - "\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e" - "\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e" - "\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae" - "\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe" - "\xbf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee" - "\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xd7\xf8\xf9\xfa\xfb\xfc\xfd\xfe" - "\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee" - "\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"; /// Check that the character is lower-case /// @@ -1599,20 +1499,7 @@ bool vim_islower(int c) } if (c >= 0x80) { - if (enc_utf8) { - return utf_islower(c); - } - - if (c >= 0x100) { - if (has_mbyte) { - return iswlower((wint_t)c); - } - - // islower() can't handle these chars and may crash - return false; - } - - return (latin1flags[c] & LATIN1LOWER) == LATIN1LOWER; + return utf_islower(c); } return islower(c); } @@ -1628,20 +1515,7 @@ bool vim_isupper(int c) } if (c >= 0x80) { - if (enc_utf8) { return utf_isupper(c); - } - - if (c >= 0x100) { - if (has_mbyte) { - return iswupper((wint_t)c); - } - - // isupper() can't handle these chars and may crash - return false; - } - - return (latin1flags[c] & LATIN1UPPER) == LATIN1UPPER; } return isupper(c); } @@ -1653,20 +1527,7 @@ int vim_toupper(int c) } if (c >= 0x80) { - if (enc_utf8) { - return utf_toupper(c); - } - - if (c >= 0x100) { - if (has_mbyte) { - return (int)towupper((wint_t)c); - } - - // toupper() can't handle these chars and may crash - return c; - } - - return latin1upper[c]; + return utf_toupper(c); } return TOUPPER_LOC(c); } @@ -1678,20 +1539,7 @@ int vim_tolower(int c) } if (c >= 0x80) { - if (enc_utf8) { - return utf_tolower(c); - } - - if (c >= 0x100) { - if (has_mbyte) { - return (int)towlower((wint_t)c); - } - - // tolower() can't handle these chars and may crash - return c; - } - - return latin1lower[c]; + return utf_tolower(c); } return TOLOWER_LOC(c); } |