vim-patch:8.2.4770: cannot easily mix expression and heredoc

Problem:    Cannot easily mix expression and heredoc.
Solution:   Support  in heredoc. (Yegappan Lakshmanan, closes vim/vim#10138)

https://github.com/vim/vim/commit/efbfa867a146fcd93fdec2435597aa4ae7f1325c

Co-authored-by: Yegappan Lakshmanan <yegappan@yahoo.com>

4 files changed, 239 insertions, 23 deletions

diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
index 351690f4df..c8eea03f5f 100644
--- a/runtime/doc/eval.txt
+++ b/runtime/doc/eval.txt
@@ -2540,14 +2540,30 @@ This does NOT work: >
 
 						*:let=<<* *:let-heredoc*
 					*E990* *E991* *E172* *E221* *E1145*
-:let {var-name} =<< [trim] {endmarker}
+:let {var-name} =<< [trim] [eval] {endmarker}
 text...
 text...
 {endmarker}
 			Set internal variable {var-name} to a |List|
 			containing the lines of text bounded by the string
-			{endmarker}. The lines of text is used as a
-			|literal-string|.
+			{endmarker}.
+
+			If "eval" is not specified, then each line of text is
+			used as a |literal-string|.  If "eval" is specified,
+			then any Vim expression in the form ``={expr}`` is
+			evaluated and the result replaces the expression.
+			Example where $HOME is expanded: >
+				let lines =<< trim eval END
+				  some text
+				  See the file `=$HOME`/.vimrc
+				  more text
+				END
+<			There can be multiple Vim expressions in a single line
+			but an expression cannot span multiple lines.  If any
+			expression evaluation fails, then the assignment fails.
+			once the "`=" has been found {expr} and a backtick
+			must follow.  {expr} cannot be empty.
+
 			{endmarker} must not contain white space.
 			{endmarker} cannot start with a lower case character.
 			The last line should end only with the {endmarker}
@@ -2597,6 +2613,13 @@ text...
 					1 2 3 4
 					5 6 7 8
 				DATA
+
+				let code =<< trim eval CODE
+				   let v = `=10 + 20`
+				   let h = "`=$HOME`"
+				   let s = "`=Str1()` abc `=Str2()`"
+				   let n = `=MyFunc(3, 4)`
+				CODE
 <
 								*E121*
 :let {var-name}	..	List the value of variable {var-name}.  Multiple
diff --git a/src/nvim/eval/userfunc.c b/src/nvim/eval/userfunc.c
index 67c73924c8..67b1e53a35 100644
--- a/src/nvim/eval/userfunc.c
+++ b/src/nvim/eval/userfunc.c
@@ -2483,10 +2483,19 @@ void ex_function(exarg_T *eap)
                 && (!ASCII_ISALNUM(p[2])
                     || (p[2] == 't' && !ASCII_ISALNUM(p[3]))))) {
           p = skipwhite(arg + 3);
-          if (strncmp(p, "trim", 4) == 0) {
-            // Ignore leading white space.
-            p = skipwhite(p + 4);
-            heredoc_trimmed = xstrnsave(theline, (size_t)(skipwhite(theline) - theline));
+          while (true) {
+            if (strncmp(p, "trim", 4) == 0) {
+              // Ignore leading white space.
+              p = skipwhite(p + 4);
+              heredoc_trimmed = xstrnsave(theline, (size_t)(skipwhite(theline) - theline));
+              continue;
+            }
+            if (strncmp(p, "eval", 4) == 0) {
+              // Ignore leading white space.
+              p = skipwhite(p + 4);
+              continue;
+            }
+            break;
           }
           skip_until = xstrnsave(p, (size_t)(skiptowhite(p) - p));
           do_concat = false;
diff --git a/src/nvim/eval/vars.c b/src/nvim/eval/vars.c
index b8a8f39437..3905cf82a6 100644
--- a/src/nvim/eval/vars.c
+++ b/src/nvim/eval/vars.c
@@ -53,6 +53,56 @@
 static const char *e_letunexp = N_("E18: Unexpected characters in :let");
 static const char *e_lock_unlock = N_("E940: Cannot lock or unlock variable %s");
 
+/// Evaluate all the Vim expressions (`=expr`) in string "str" and return the
+/// resulting string.  The caller must free the returned string.
+static char *eval_all_expr_in_str(char *str)
+{
+  garray_T ga;
+  ga_init(&ga, 1, 80);
+  char *p = str;
+
+  // Look for `=expr`, evaluate the expression and replace `=expr` with the
+  // result.
+  while (*p != NUL) {
+    char *s = p;
+    while (*p != NUL && (*p != '`' || p[1] != '=')) {
+      p++;
+    }
+    ga_concat_len(&ga, s, (size_t)(p - s));
+    if (*p == NUL) {
+      break;              // no backtick expression found
+    }
+    s = p;
+    p += 2;         // skip `=
+
+    int status = *p == NUL ? OK : skip_expr(&p, NULL);
+    if (status == FAIL || *p != '`') {
+      // invalid expression or missing ending backtick
+      if (status != FAIL) {
+        emsg(_("E1083: Missing backtick"));
+      }
+      xfree(ga.ga_data);
+      return NULL;
+    }
+    s += 2;         // skip `=
+    char save_c = *p;
+    *p = NUL;
+    char *exprval = eval_to_string(s, true);
+    *p = save_c;
+    p++;
+    if (exprval == NULL) {
+      // expression evaluation failed
+      xfree(ga.ga_data);
+      return NULL;
+    }
+    ga_concat(&ga, exprval);
+    xfree(exprval);
+  }
+  ga_append(&ga, NUL);
+
+  return ga.ga_data;
+}
+
 /// Get a list of lines from a HERE document. The here document is a list of
 /// lines surrounded by a marker.
 ///     cmd << {marker}
@@ -65,7 +115,7 @@ static const char *e_lock_unlock = N_("E940: Cannot lock or unlock variable %s")
 /// marker, then the leading indentation before the lines (matching the
 /// indentation in the 'cmd' line) is stripped.
 ///
-/// @return  a List with {lines} or NULL.
+/// @return  a List with {lines} or NULL on failure.
 static list_T *heredoc_get(exarg_T *eap, char *cmd)
 {
   char *marker;
@@ -81,20 +131,33 @@ static list_T *heredoc_get(exarg_T *eap, char *cmd)
 
   // Check for the optional 'trim' word before the marker
   cmd = skipwhite(cmd);
-  if (strncmp(cmd, "trim", 4) == 0
-      && (cmd[4] == NUL || ascii_iswhite(cmd[4]))) {
-    cmd = skipwhite(cmd + 4);
-
-    // Trim the indentation from all the lines in the here document.
-    // The amount of indentation trimmed is the same as the indentation of
-    // the first line after the :let command line.  To find the end marker
-    // the indent of the :let command line is trimmed.
-    p = *eap->cmdlinep;
-    while (ascii_iswhite(*p)) {
-      p++;
-      marker_indent_len++;
+  bool evalstr = false;
+  bool eval_failed = false;
+  while (true) {
+    if (strncmp(cmd, "trim", 4) == 0
+        && (cmd[4] == NUL || ascii_iswhite(cmd[4]))) {
+      cmd = skipwhite(cmd + 4);
+
+      // Trim the indentation from all the lines in the here document.
+      // The amount of indentation trimmed is the same as the indentation
+      // of the first line after the :let command line.  To find the end
+      // marker the indent of the :let command line is trimmed.
+      p = *eap->cmdlinep;
+      while (ascii_iswhite(*p)) {
+        p++;
+        marker_indent_len++;
+      }
+      text_indent_len = -1;
+
+      continue;
+    }
+    if (strncmp(cmd, "eval", 4) == 0
+        && (cmd[4] == NUL || ascii_iswhite(cmd[4]))) {
+      cmd = skipwhite(cmd + 4);
+      evalstr = true;
+      continue;
     }
-    text_indent_len = -1;
+    break;
   }
 
   // The marker is the next word.
@@ -136,6 +199,14 @@ static list_T *heredoc_get(exarg_T *eap, char *cmd)
       xfree(theline);
       break;
     }
+
+    // If expression evaluation failed in the heredoc, then skip till the
+    // end marker.
+    if (eval_failed) {
+      xfree(theline);
+      continue;
+    }
+
     if (text_indent_len == -1 && *theline != NUL) {
       // set the text indent from the first line.
       p = theline;
@@ -155,11 +226,29 @@ static list_T *heredoc_get(exarg_T *eap, char *cmd)
       }
     }
 
-    tv_list_append_string(l, theline + ti, -1);
+    char *str = theline + ti;
+    if (evalstr) {
+      str = eval_all_expr_in_str(str);
+      if (str == NULL) {
+        // expression evaluation failed
+        xfree(theline);
+        eval_failed = true;
+        continue;
+      }
+      xfree(theline);
+      theline = str;
+    }
+
+    tv_list_append_string(l, str, -1);
     xfree(theline);
   }
   xfree(text_indent);
 
+  if (eval_failed) {
+    // expression evaluation in the heredoc failed
+    tv_list_free(l);
+    return NULL;
+  }
   return l;
 }
 
diff --git a/test/old/testdir/test_let.vim b/test/old/testdir/test_let.vim
index 6fbf305624..fe14709e71 100644
--- a/test/old/testdir/test_let.vim
+++ b/test/old/testdir/test_let.vim
@@ -1,5 +1,7 @@
 " Tests for the :let command.
 
+source vim9.vim
+
 func Test_let()
   " Test to not autoload when assigning.  It causes internal error.
   set runtimepath+=./sautest
@@ -385,7 +387,8 @@ END
   call assert_equal(['Text', 'with', 'indent'], text)
 endfunc
 
-" Test for the setting a variable using the heredoc syntax
+" Test for the setting a variable using the heredoc syntax.
+" Keep near the end, this messes up highlighting.
 func Test_let_heredoc()
   let var1 =<< END
 Some sample text
@@ -493,4 +496,96 @@ END
   call assert_equal(['     x', '     \y', '     z'], [a, b, c])
 endfunc
 
+" Test for evaluating Vim expressions in a heredoc using `=expr`
+" Keep near the end, this messes up highlighting.
+func Test_let_heredoc_eval()
+  let str = ''
+  let code =<< trim eval END
+    let a = `=5 + 10`
+    let b = `=min([10, 6])` + `=max([4, 6])`
+    `=str`
+    let c = "abc`=str`d"
+  END
+  call assert_equal(['let a = 15', 'let b = 6 + 6', '', 'let c = "abcd"'], code)
+  let $TESTVAR = "Hello"
+  let code =<< eval trim END
+    let s = "`=$TESTVAR`"
+  END
+  call assert_equal(['let s = "Hello"'], code)
+  let code =<< eval END
+    let s = "`=$TESTVAR`"
+END
+  call assert_equal(['    let s = "Hello"'], code)
+  let a = 10
+  let data =<< eval END
+`=a`
+END
+  call assert_equal(['10'], data)
+  let x = 'X'
+  let code =<< eval trim END
+    let a = `abc`
+    let b = `=x`
+    let c = `
+  END
+  call assert_equal(['let a = `abc`', 'let b = X', 'let c = `'], code)
+  let code = 'xxx'
+  let code =<< eval trim END
+    let n = `=5 +
+    6`
+  END
+  call assert_equal('xxx', code)
+  let code =<< eval trim END
+     let n = `=min([1, 2]` + `=max([3, 4])`
+  END
+  call assert_equal('xxx', code)
+
+  let lines =<< trim LINES
+      let text =<< eval trim END
+        let b = `=
+      END
+  LINES
+  call CheckScriptFailure(lines, 'E1083:')
+
+  let lines =<< trim LINES
+      let text =<< eval trim END
+        let b = `=abc
+      END
+  LINES
+  call CheckScriptFailure(lines, 'E1083:')
+
+  let lines =<< trim LINES
+      let text =<< eval trim END
+        let b = `=`
+      END
+  LINES
+  call CheckScriptFailure(lines, 'E15:')
+
+  " Test for sourcing a script containing a heredoc with invalid expression.
+  " Variable assignment should fail, if expression evaluation fails
+  new
+  let g:Xvar = 'test'
+  let g:b = 10
+  let lines =<< trim END
+    let Xvar =<< eval CODE
+    let a = 1
+    let b = `=5+`
+    let c = 2
+    CODE
+    let g:Count += 1
+  END
+  call setline(1, lines)
+  let g:Count = 0
+  call assert_fails('source', 'E15:')
+  call assert_equal(1, g:Count)
+  call setline(3, 'let b = `=abc`')
+  call assert_fails('source', 'E121:')
+  call assert_equal(2, g:Count)
+  call setline(3, 'let b = `=abc` + `=min([9, 4])` + 2')
+  call assert_fails('source', 'E121:')
+  call assert_equal(3, g:Count)
+  call assert_equal('test', g:Xvar)
+  call assert_equal(10, g:b)
+  bw!
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab