vim-patch:8.0.0519: character classes not well tested (#8460)

Problem:    Character classes are not well tested. They can differ between
            platforms.
Solution:   Add tests.  In the documentation make clear which classes depend
            on what library function.  Only use :cntrl: and :graph: for ASCII.
            (Kazunobu Kuriyama, Dominique Pelle, closes vim/vim#1560)
            Update the documentation.
https://github.com/vim/vim/commit/0c078fc7db2902d4ccba04506db082ddbef45a8c

3 files changed, 64 insertions, 15 deletions

diff --git a/src/nvim/regexp.c b/src/nvim/regexp.c
index e4de43b49e..ee7d6d8500 100644
--- a/src/nvim/regexp.c
+++ b/src/nvim/regexp.c
@@ -2328,21 +2328,21 @@ collection:
               regc('\t');
               break;
             case CLASS_CNTRL:
-              for (cu = 1; cu <= 255; cu++) {
+              for (cu = 1; cu <= 127; cu++) {
                 if (iscntrl(cu)) {
                   regmbc(cu);
                 }
               }
               break;
             case CLASS_DIGIT:
-              for (cu = 1; cu <= 255; cu++) {
+              for (cu = 1; cu <= 127; cu++) {
                 if (ascii_isdigit(cu)) {
                   regmbc(cu);
                 }
               }
               break;
             case CLASS_GRAPH:
-              for (cu = 1; cu <= 255; cu++) {
+              for (cu = 1; cu <= 127; cu++) {
                 if (isgraph(cu)) {
                   regmbc(cu);
                 }
diff --git a/src/nvim/regexp_nfa.c b/src/nvim/regexp_nfa.c
index 98fae858f6..0b8e979ca2 100644
--- a/src/nvim/regexp_nfa.c
+++ b/src/nvim/regexp_nfa.c
@@ -4358,16 +4358,18 @@ static int check_char_class(int class, int c)
       return OK;
     break;
   case NFA_CLASS_CNTRL:
-    if (c >= 1 && c <= 255 && iscntrl(c))
+    if (c >= 1 && c <= 127 && iscntrl(c)) {
       return OK;
+    }
     break;
   case NFA_CLASS_DIGIT:
     if (ascii_isdigit(c))
       return OK;
     break;
   case NFA_CLASS_GRAPH:
-    if (c >= 1 && c <= 255 && isgraph(c))
+    if (c >= 1 && c <= 127 && isgraph(c)) {
       return OK;
+    }
     break;
   case NFA_CLASS_LOWER:
     if (mb_islower(c) && c != 170 && c != 186) {
diff --git a/src/nvim/testdir/test_regexp_utf8.vim b/src/nvim/testdir/test_regexp_utf8.vim
index a2f4286d4f..ecd686743e 100644
--- a/src/nvim/testdir/test_regexp_utf8.vim
+++ b/src/nvim/testdir/test_regexp_utf8.vim
@@ -35,12 +35,21 @@ func s:classes_test()
   set isprint=@,161-255
   call assert_equal('Motörhead', matchstr('Motörhead', '[[:print:]]\+'))
 
+  let alnumchars = ''
   let alphachars = ''
+  let backspacechar = ''
+  let blankchars = ''
+  let cntrlchars = ''
+  let digitchars = ''
+  let escapechar = ''
+  let graphchars = ''
   let lowerchars = ''
-  let upperchars = ''
-  let alnumchars = ''
   let printchars = ''
   let punctchars = ''
+  let returnchar = ''
+  let spacechars = ''
+  let tabchar = ''
+  let upperchars = ''
   let xdigitchars = ''
   let i = 1
   while i <= 255
@@ -48,21 +57,48 @@ func s:classes_test()
     if c =~ '[[:alpha:]]'
       let alphachars .= c
     endif
-    if c =~ '[[:lower:]]'
-      let lowerchars .= c
-    endif
-    if c =~ '[[:upper:]]'
-      let upperchars .= c
-    endif
     if c =~ '[[:alnum:]]'
       let alnumchars .= c
     endif
+    if c =~ '[[:backspace:]]'
+      let backspacechar .= c
+    endif
+    if c =~ '[[:blank:]]'
+      let blankchars .= c
+    endif
+    if c =~ '[[:cntrl:]]'
+      let cntrlchars .= c
+    endif
+    if c =~ '[[:digit:]]'
+      let digitchars .= c
+    endif
+    if c =~ '[[:escape:]]'
+      let escapechar .= c
+    endif
+    if c =~ '[[:graph:]]'
+      let graphchars .= c
+    endif
+    if c =~ '[[:lower:]]'
+      let lowerchars .= c
+    endif
     if c =~ '[[:print:]]'
       let printchars .= c
     endif
     if c =~ '[[:punct:]]'
       let punctchars .= c
     endif
+    if c =~ '[[:return:]]'
+      let returnchar .= c
+    endif
+    if c =~ '[[:space:]]'
+      let spacechars .= c
+    endif
+    if c =~ '[[:tab:]]'
+      let tabchar .= c
+    endif
+    if c =~ '[[:upper:]]'
+      let upperchars .= c
+    endif
     if c =~ '[[:xdigit:]]'
       let xdigitchars .= c
     endif
@@ -70,11 +106,22 @@ func s:classes_test()
   endwhile
 
   call assert_equal('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz', alphachars)
-  call assert_equal('abcdefghijklmnopqrstuvwxyzµßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ', lowerchars)
-  call assert_equal('ABCDEFGHIJKLMNOPQRSTUVWXYZÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ', upperchars)
   call assert_equal('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz', alnumchars)
+  call assert_equal("\b", backspacechar)
+  call assert_equal("\t ", blankchars)
+  " Commented out: it succeeds on Linux and Windows, but fails on macOs in Travis.
+  " call assert_equal("\x01\x02\x03\x04\x05\x06\x07\b\t\n\x0b\f\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\e\x1c\x1d\x1e\x1f\x7f", cntrlchars)
+  call assert_equal("0123456789", digitchars)
+  call assert_equal("\<Esc>", escapechar)
+  " Commented out: it succeeds on Linux and Windows, but fails on macOs in Travis.
+  " call assert_equal('!"#$%&''()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~', graphchars)
+  call assert_equal('abcdefghijklmnopqrstuvwxyzµßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ', lowerchars)
   call assert_equal(' !"#$%&''()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ', printchars)
   call assert_equal('!"#$%&''()*+,-./:;<=>?@[\]^_`{|}~', punctchars)
+  call assert_equal('ABCDEFGHIJKLMNOPQRSTUVWXYZÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ', upperchars)
+  call assert_equal("\r", returnchar)
+  call assert_equal("\t\n\x0b\f\r ", spacechars)
+  call assert_equal("\t", tabchar)
   call assert_equal('0123456789ABCDEFabcdef', xdigitchars)
 endfunc