vim-patch:8.2.0578: heredoc for interfaces does not support "trim"

Problem:    Heredoc for interfaces does not support "trim".
Solution:   Update the script heredoc support to be same as the :let command.
            (Yegappan Lakshmanan, closes vim/vim#5916)

https://github.com/vim/vim/commit/6c2b7b8055b96463f78abb70f58c4c6d6d4b9d55

13 files changed, 173 insertions, 56 deletions

diff --git a/runtime/doc/if_perl.txt b/runtime/doc/if_perl.txt
index e4ec6dcf45..c3b49dc5bb 100644
--- a/runtime/doc/if_perl.txt
+++ b/runtime/doc/if_perl.txt
@@ -19,7 +19,7 @@ See |provider-perl| for more information.
 			working: >
 				:perl print "Hello"
 
-:[range]perl << [endmarker]
+:[range]perl << [trim] [{endmarker}]
 {script}
 {endmarker}
 			Execute perl script {script}.
diff --git a/runtime/doc/if_pyth.txt b/runtime/doc/if_pyth.txt
index 45accc2e03..415a58ec71 100644
--- a/runtime/doc/if_pyth.txt
+++ b/runtime/doc/if_pyth.txt
@@ -19,17 +19,16 @@ Commands						*python-commands*
 			the `:python` command is working: >vim
 				:python print "Hello"
 
-:[range]py[thon] << [endmarker]
+:[range]py[thon] << [trim] [{endmarker}]
 {script}
 {endmarker}
 			Execute Python script {script}.  Useful for including
 			python code in Vim scripts.  Requires Python, see
 			|script-here|.
 
-The {endmarker} below the {script} must NOT be preceded by any white space.
-
 If [endmarker] is omitted from after the "<<", a dot '.' must be used after
-{script}, like for the |:append| and |:insert| commands.
+{script}, like for the |:append| and |:insert| commands.  Refer to
+|:let-heredoc| for more information.
 
 Example: >vim
 	function! IcecreamInitialize()
@@ -597,12 +596,12 @@ variants explicitly if Python 3 is required.
 
 							*:py3* *:python3*
 :[range]py3 {stmt}
-:[range]py3 << [endmarker]
+:[range]py3 << [trim] [{endmarker}]
 {script}
 {endmarker}
 
 :[range]python3 {stmt}
-:[range]python3 << [endmarker]
+:[range]python3 << [trim] [{endmarker}]
 {script}
 {endmarker}
 	The `:py3` and `:python3` commands work similar to `:python`.  A
diff --git a/runtime/doc/if_ruby.txt b/runtime/doc/if_ruby.txt
index d88f59eb73..a5aaac1a5f 100644
--- a/runtime/doc/if_ruby.txt
+++ b/runtime/doc/if_ruby.txt
@@ -19,15 +19,15 @@ downloading Ruby there.
 :rub[y] {cmd}		Execute Ruby command {cmd}.  A command to try it out: >
 				:ruby print "Hello"
 
-:rub[y] << [endmarker]
+:rub[y] << [trim] [{endmarker}]
 {script}
 {endmarker}
 			Execute Ruby script {script}.
-			The {endmarker} after {script} must NOT be preceded by
-			any white space.
 
 			If [endmarker] is omitted, it defaults to a dot '.'
-			like for the |:append| and |:insert| commands.
+			like for the |:append| and |:insert| commands.  Refer
+			to |:let-heredoc| for more information.
+
 
 			This form of the |:ruby| command is mainly useful for
 			including ruby code in vim scripts.
diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt
index b3be11efdb..465c83e6ab 100644
--- a/runtime/doc/lua.txt
+++ b/runtime/doc/lua.txt
@@ -247,12 +247,12 @@ arguments separated by " " (space) instead of "\t" (tab).
         :lua =jit.version
 <
                                                                 *:lua-heredoc*
-:lua << [endmarker]
+:lua << [trim] [{endmarker}]
 {script}
 {endmarker}
-    Executes Lua script {script} from within Vimscript. {endmarker} must NOT
-    be preceded by whitespace. You can omit [endmarker] after the "<<" and use
-    a dot "." after {script} (similar to |:append|, |:insert|).
+    Executes Lua script {script} from within Vimscript. You can omit
+    [endmarker] after the "<<" and use a dot "." after {script} (similar to
+    |:append|, |:insert|). Refer to |let-heredoc| for more information.
 
     Example: >vim
         function! CurrentLineInfo()
diff --git a/src/nvim/eval/userfunc.c b/src/nvim/eval/userfunc.c
index 0e22cf54cf..1f5a6eaec4 100644
--- a/src/nvim/eval/userfunc.c
+++ b/src/nvim/eval/userfunc.c
@@ -2571,11 +2571,18 @@ void ex_function(exarg_T *eap)
                   && (!ASCII_ISALPHA(p[2]) || p[2] == 's')))) {
         // ":python <<" continues until a dot, like ":append"
         p = skipwhite(arg + 2);
+        if (strncmp(p, "trim", 4) == 0) {
+          // Ignore leading white space.
+          p = skipwhite(p + 4);
+          heredoc_trimmed = xstrnsave(theline, (size_t)(skipwhite(theline) - theline));
+        }
         if (*p == NUL) {
           skip_until = xstrdup(".");
         } else {
-          skip_until = xstrdup(p);
+          skip_until = xstrnsave(p, (size_t)(skiptowhite(p) - p));
         }
+        do_concat = false;
+        is_heredoc = true;
       }
 
       // Check for ":let v =<< [trim] EOF"
diff --git a/src/nvim/eval/vars.c b/src/nvim/eval/vars.c
index 593ba9f6c3..dee3867a5a 100644
--- a/src/nvim/eval/vars.c
+++ b/src/nvim/eval/vars.c
@@ -155,14 +155,19 @@ char *eval_all_expr_in_str(char *str)
 /// marker, then the leading indentation before the lines (matching the
 /// indentation in the 'cmd' line) is stripped.
 ///
+/// When getting lines for an embedded script (e.g. python, lua, perl, ruby,
+/// tcl, mzscheme), "script_get" is set to true. In this case, if the marker is
+/// missing, then '.' is accepted as a marker.
+///
 /// @return  a List with {lines} or NULL on failure.
-static list_T *heredoc_get(exarg_T *eap, char *cmd)
+list_T *heredoc_get(exarg_T *eap, char *cmd, bool script_get)
 {
   char *marker;
   char *p;
   int marker_indent_len = 0;
   int text_indent_len = 0;
   char *text_indent = NULL;
+  char dot[] = ".";
 
   if (eap->getline == NULL) {
     emsg(_("E991: cannot use =<< here"));
@@ -214,8 +219,14 @@ static list_T *heredoc_get(exarg_T *eap, char *cmd)
       return NULL;
     }
   } else {
-    emsg(_("E172: Missing marker"));
-    return NULL;
+    // When getting lines for an embedded script, if the marker is missing,
+    // accept '.' as the marker.
+    if (script_get) {
+      marker = dot;
+    } else {
+      emsg(_("E172: Missing marker"));
+      return NULL;
+    }
   }
 
   char *theline = NULL;
@@ -353,7 +364,7 @@ void ex_let(exarg_T *eap)
 
   if (expr[0] == '=' && expr[1] == '<' && expr[2] == '<') {
     // HERE document
-    list_T *l = heredoc_get(eap, expr + 3);
+    list_T *l = heredoc_get(eap, expr + 3, false);
     if (l != NULL) {
       tv_list_set_ret(&rettv, l);
       if (!eap->skip) {
diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c
index c0c8a5605b..5018c9268b 100644
--- a/src/nvim/ex_getln.c
+++ b/src/nvim/ex_getln.c
@@ -29,6 +29,7 @@
 #include "nvim/edit.h"
 #include "nvim/eval.h"
 #include "nvim/eval/typval.h"
+#include "nvim/eval/vars.h"
 #include "nvim/ex_cmds.h"
 #include "nvim/ex_cmds_defs.h"
 #include "nvim/ex_docmd.h"
@@ -4561,39 +4562,37 @@ bool is_in_cmdwin(void)
 char *script_get(exarg_T *const eap, size_t *const lenp)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_MALLOC
 {
-  const char *const cmd = eap->arg;
+  char *cmd = eap->arg;
 
   if (cmd[0] != '<' || cmd[1] != '<' || eap->getline == NULL) {
     *lenp = strlen(eap->arg);
     return eap->skip ? NULL : xmemdupz(eap->arg, *lenp);
   }
+  cmd += 2;
 
   garray_T ga = { .ga_data = NULL, .ga_len = 0 };
+
+  list_T *const l = heredoc_get(eap, cmd, true);
+  if (l == NULL) {
+    return NULL;
+  }
+
   if (!eap->skip) {
     ga_init(&ga, 1, 0x400);
   }
 
-  const char *const end_pattern = (cmd[2] != NUL ? skipwhite(cmd + 2) : ".");
-  while (true) {
-    char *const theline = eap->getline(eap->cstack->cs_looplevel > 0 ? -1 : NUL, eap->cookie, 0,
-                                       true);
-
-    if (theline == NULL || strcmp(end_pattern, theline) == 0) {
-      xfree(theline);
-      break;
-    }
-
+  TV_LIST_ITER_CONST(l, li, {
     if (!eap->skip) {
-      ga_concat(&ga, theline);
+      ga_concat(&ga, tv_get_string(TV_LIST_ITEM_TV(li)));
       ga_append(&ga, '\n');
     }
-    xfree(theline);
-  }
+  });
   *lenp = (size_t)ga.ga_len;  // Set length without trailing NUL.
   if (!eap->skip) {
     ga_append(&ga, NUL);
   }
 
+  tv_list_free(l);
   return (char *)ga.ga_data;
 }
 
diff --git a/test/old/testdir/test_perl.vim b/test/old/testdir/test_perl.vim
index 558d0a5d6b..5cef74193b 100644
--- a/test/old/testdir/test_perl.vim
+++ b/test/old/testdir/test_perl.vim
@@ -224,11 +224,11 @@ endfunc
 func Test_stdio()
   throw 'skipped: TODO: '
   redir =>l:out
-  perl <<EOF
+  perl << trim EOF
     VIM::Msg("&VIM::Msg");
     print "STDOUT";
     print STDERR "STDERR";
-EOF
+  EOF
   redir END
   call assert_equal(['&VIM::Msg', 'STDOUT', 'STDERR'], split(l:out, "\n"))
 endfunc
@@ -305,7 +305,13 @@ END
   perl <<
 VIM::DoCommand('let s ..= "B"')
 .
-  call assert_equal('AB', s)
+  perl << trim END
+    VIM::DoCommand('let s ..= "C"')
+  END
+  perl << trim
+    VIM::DoCommand('let s ..= "D"')
+  .
+  call assert_equal('ABCD', s)
 endfunc
 
 " vim: shiftwidth=2 sts=2 expandtab
diff --git a/test/old/testdir/test_python2.vim b/test/old/testdir/test_python2.vim
index ab03408fc4..f0259be70d 100644
--- a/test/old/testdir/test_python2.vim
+++ b/test/old/testdir/test_python2.vim
@@ -171,3 +171,22 @@ func Test_Catch_Exception_Message()
     call assert_match('^Vim(.*):.*RuntimeError: TEST$', v:exception )
   endtry
 endfunc
+
+" Test for various heredoc syntax
+func Test_python_heredoc()
+  python << END
+s='A'
+END
+  python <<
+s+='B'
+.
+  python << trim END
+    s+='C'
+  END
+  python << trim
+    s+='D'
+  .
+  call assert_equal('ABCD', pyxeval('s'))
+endfunc
+
+" vim: shiftwidth=2 sts=2 expandtab
diff --git a/test/old/testdir/test_python3.vim b/test/old/testdir/test_python3.vim
index 6c9676603f..60773c5b2a 100644
--- a/test/old/testdir/test_python3.vim
+++ b/test/old/testdir/test_python3.vim
@@ -227,9 +227,11 @@ func Test_python3_opt_reset_local_to_global()
   " Set the global and buffer-local option values and then clear the
   " buffer-local option value.
   for opt in bopts
-    py3 pyopt = vim.bindeval("opt")
-    py3 vim.options[pyopt[0]] = pyopt[1]
-    py3 curbuf.options[pyopt[0]] = pyopt[2]
+    py3 << trim END
+      pyopt = vim.bindeval("opt")
+      vim.options[pyopt[0]] = pyopt[1]
+      curbuf.options[pyopt[0]] = pyopt[2]
+    END
     exe "call assert_equal(opt[2], &" .. opt[0] .. ")"
     exe "call assert_equal(opt[1], &g:" .. opt[0] .. ")"
     exe "call assert_equal(opt[2], &l:" .. opt[0] .. ")"
@@ -247,9 +249,11 @@ func Test_python3_opt_reset_local_to_global()
         \ ['sidescrolloff', 6, 12, -1],
         \ ['statusline', '%<%f', '%<%F', '']]
   for opt in wopts
-    py3 pyopt = vim.bindeval("opt")
-    py3 vim.options[pyopt[0]] = pyopt[1]
-    py3 curwin.options[pyopt[0]] = pyopt[2]
+    py3 << trim
+      pyopt = vim.bindeval("opt")
+      vim.options[pyopt[0]] = pyopt[1]
+      curwin.options[pyopt[0]] = pyopt[2]
+    .
     exe "call assert_equal(opt[2], &" .. opt[0] .. ")"
     exe "call assert_equal(opt[1], &g:" .. opt[0] .. ")"
     exe "call assert_equal(opt[2], &l:" .. opt[0] .. ")"
@@ -263,4 +267,21 @@ func Test_python3_opt_reset_local_to_global()
   close!
 endfunc
 
+" Test for various heredoc syntax
+func Test_python3_heredoc()
+  python3 << END
+s='A'
+END
+  python3 <<
+s+='B'
+.
+  python3 << trim END
+    s+='C'
+  END
+  python3 << trim
+    s+='D'
+  .
+  call assert_equal('ABCD', pyxeval('s'))
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab
diff --git a/test/old/testdir/test_pyx2.vim b/test/old/testdir/test_pyx2.vim
index eee825fa9b..c7583d0234 100644
--- a/test/old/testdir/test_pyx2.vim
+++ b/test/old/testdir/test_pyx2.vim
@@ -15,10 +15,10 @@ endfunc
 
 func Test_pyx()
   redir => var
-  pyx << EOF
-import sys
-print(sys.version)
-EOF
+  pyx << trim EOF
+    import sys
+    print(sys.version)
+  EOF
   redir END
   call assert_match(s:py2pattern, split(var)[0])
 endfunc
@@ -79,3 +79,22 @@ func Test_Catch_Exception_Message()
     call assert_match('^Vim(.*):.*RuntimeError: TEST$', v:exception )
   endtry
 endfunc
+
+" Test for various heredoc syntaxes
+func Test_pyx2_heredoc()
+  pyx << END
+result='A'
+END
+  pyx <<
+result+='B'
+.
+  pyx << trim END
+    result+='C'
+  END
+  pyx << trim
+    result+='D'
+  .
+  call assert_equal('ABCD', pyxeval('result'))
+endfunc
+
+" vim: shiftwidth=2 sts=2 expandtab
diff --git a/test/old/testdir/test_pyx3.vim b/test/old/testdir/test_pyx3.vim
index db39f5134a..a4666fe3d4 100644
--- a/test/old/testdir/test_pyx3.vim
+++ b/test/old/testdir/test_pyx3.vim
@@ -15,10 +15,10 @@ endfunc
 
 func Test_pyx()
   redir => var
-  pyx << EOF
-import sys
-print(sys.version)
-EOF
+  pyx << trim EOF
+    import sys
+    print(sys.version)
+  EOF
   redir END
   call assert_match(s:py3pattern, split(var)[0])
 endfunc
@@ -79,3 +79,22 @@ func Test_Catch_Exception_Message()
     call assert_match('^Vim(.*):.*RuntimeError: TEST$', v:exception )
   endtry
 endfunc
+
+" Test for various heredoc syntaxes
+func Test_pyx3_heredoc()
+  pyx << END
+result='A'
+END
+  pyx <<
+result+='B'
+.
+  pyx << trim END
+    result+='C'
+  END
+  pyx << trim
+    result+='D'
+  .
+  call assert_equal('ABCD', pyxeval('result'))
+endfunc
+
+" vim: shiftwidth=2 sts=2 expandtab
diff --git a/test/old/testdir/test_ruby.vim b/test/old/testdir/test_ruby.vim
index 1fbf3392d9..80d39309ea 100644
--- a/test/old/testdir/test_ruby.vim
+++ b/test/old/testdir/test_ruby.vim
@@ -341,11 +341,11 @@ func Test_ruby_Vim_evaluate_list()
   call setline(line('$'), ['2 line 2'])
   ruby Vim.command("normal /^2\n")
   let l = ["abc", "def"]
-  ruby << EOF
-  curline = $curbuf.line_number
-  l = Vim.evaluate("l");
-  $curbuf.append(curline, l.join("|"))
-EOF
+  ruby << trim EOF
+    curline = $curbuf.line_number
+    l = Vim.evaluate("l");
+    $curbuf.append(curline, l.join("|"))
+  EOF
   normal j
   .rubydo $_ = $_.gsub(/\|/, '/')
   call assert_equal('abc/def', getline('$'))
@@ -414,4 +414,21 @@ func Test_rubyeval_error()
   call assert_fails('call rubyeval("(")')
 endfunc
 
+" Test for various heredoc syntax
+func Test_ruby_heredoc()
+  ruby << END
+Vim.command('let s = "A"')
+END
+  ruby <<
+Vim.command('let s ..= "B"')
+.
+  ruby << trim END
+    Vim.command('let s ..= "C"')
+  END
+  ruby << trim
+    Vim.command('let s ..= "D"')
+  .
+  call assert_equal('ABCD', s)
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab