vim-patch:7.4.1727

Problem:    Cannot detect a crash in tests when caused by garbagecollect().
Solution:   Add garbagecollect_for_testing().  Do not free a job if is still
            useful.

https://github.com/vim/vim/commit/ebf7dfa6f121c82f97d2adca3d45fbaba9ad8f7e

8 files changed, 351 insertions, 18 deletions

diff --git a/src/nvim/eval.c b/src/nvim/eval.c
index bbb6565509..8692ddc334 100644
--- a/src/nvim/eval.c
+++ b/src/nvim/eval.c
@@ -389,6 +389,7 @@ static struct vimvar {
   VV(VV__NULL_LIST,     "_null_list",       VAR_LIST, VV_RO),
   VV(VV__NULL_DICT,     "_null_dict",       VAR_DICT, VV_RO),
   VV(VV_VIM_DID_ENTER,  "vim_did_enter",    VAR_NUMBER, VV_RO),
+  VV(VV_TESTING,        "testing",          VAR_NUMBER, 0),
   VV(VV_TYPE_NUMBER,    "t_number",         VAR_NUMBER, VV_RO),
   VV(VV_TYPE_STRING,    "t_string",         VAR_NUMBER, VV_RO),
   VV(VV_TYPE_FUNC,      "t_func",           VAR_NUMBER, VV_RO),
@@ -648,8 +649,8 @@ void eval_clear(void)
     xfree(SCRIPT_SV(i));
   ga_clear(&ga_scripts);
 
-  /* unreferenced lists and dicts */
-  (void)garbage_collect();
+  // unreferenced lists and dicts
+  (void)garbage_collect(false);
 
   /* functions */
   free_all_functions();
@@ -5813,6 +5814,9 @@ int get_copyID(void)
   return current_copyID;
 }
 
+// Used by get_func_tv()
+static garray_T funcargs = GA_EMPTY_INIT_VALUE;
+
 /*
  * Garbage collection for lists and dictionaries.
  *
@@ -5835,16 +5839,19 @@ int get_copyID(void)
 
 /// Do garbage collection for lists and dicts.
 ///
+/// @param testing  true if called from test_garbagecollect_now().
 /// @returns        true if some memory was freed.
-bool garbage_collect(void)
+bool garbage_collect(bool testing)
 {
   bool abort = false;
 #define ABORTING(func) abort = abort || func
 
-  // Only do this once.
-  want_garbage_collect = false;
-  may_garbage_collect = false;
-  garbage_collect_at_exit = false;
+  if (!testing) {
+    // Only do this once.
+    want_garbage_collect = false;
+    may_garbage_collect = false;
+    garbage_collect_at_exit = false;
+  }
 
   // We advance by two because we add one for items referenced through
   // previous_funccal.
@@ -5954,6 +5961,12 @@ bool garbage_collect(void)
     })
   }
 
+  // function call arguments, if v:testing is set.
+  for (int i = 0; i < funcargs.ga_len; i++) {
+	ABORTING(set_ref_in_item)(((typval_T **)funcargs.ga_data)[i],
+							  copyID, NULL, NULL);
+  }
+
   // v: vars
   ABORTING(set_ref_in_ht)(&vimvarht, copyID, NULL);
 
@@ -6008,7 +6021,7 @@ bool garbage_collect(void)
     if (did_free_funccal) {
       // When a funccal was freed some more items might be garbage
       // collected, so run again.
-      (void)garbage_collect();
+      (void)garbage_collect(testing);
     }
   } else if (p_verbose > 0) {
     verb_msg((char_u *)_(
@@ -7085,9 +7098,24 @@ get_func_tv (
     ret = FAIL;
 
   if (ret == OK) {
-    ret = call_func(name, len, rettv, argcount, argvars,
+    int	i = 0;
+
+    if (get_vim_var_nr(VV_TESTING)) {
+      // Prepare for calling garbagecollect_for_testing(), need to know
+      // what variables are used on the call stack.
+      if (funcargs.ga_itemsize == 0) {
+        ga_init(&funcargs, (int)sizeof(typval_T *), 50);
+      }
+      for (i = 0; i < argcount; i++) {
+        ga_grow(&funcargs, 1);
+        ((typval_T **)funcargs.ga_data)[funcargs.ga_len++] = &argvars[i];
+      }
+	}
+    ret = call_func(name, len, rettv, argcount, argvars, NULL,
                     firstline, lastline, doesrange, evaluate,
                     partial, selfdict);
+
+    funcargs.ga_len -= i;
   } else if (!aborting()) {
     if (argcount == MAX_FUNC_ARGS) {
       emsg_funcname(N_("E740: Too many arguments for function %s"), name);
@@ -17306,6 +17334,14 @@ static void f_termopen(typval_T *argvars, typval_T *rettv, FunPtr fptr)
   return;
 }
 
+// "test_garbagecollect_now()" function
+static void f_test_garbagecollect_now(typval_T *argvars, typval_T *rettv, FunPtr fptr)
+{
+  // This is dangerous, any Lists and Dicts used internally may be freed
+  // while still in use.
+  garbage_collect(true);
+}
+
 static bool callback_from_typval(Callback *callback, typval_T *arg)
 {
   if (arg->v_type == VAR_PARTIAL && arg->vval.v_partial != NULL) {
diff --git a/src/nvim/eval.h b/src/nvim/eval.h
index 630e309442..5bd508d161 100644
--- a/src/nvim/eval.h
+++ b/src/nvim/eval.h
@@ -127,6 +127,7 @@ typedef enum {
     VV__NULL_LIST,  // List with NULL value. For test purposes only.
     VV__NULL_DICT,  // Dictionary with NULL value. For test purposes only.
     VV_VIM_DID_ENTER,
+    VV_TESTING,
     VV_TYPE_NUMBER,
     VV_TYPE_STRING,
     VV_TYPE_FUNC,
diff --git a/src/nvim/eval.lua b/src/nvim/eval.lua
index 964b061e95..7fe32b264e 100644
--- a/src/nvim/eval.lua
+++ b/src/nvim/eval.lua
@@ -303,6 +303,7 @@ return {
     tanh={args=1, func="float_op_wrapper", data="&tanh"},
     tempname={},
     termopen={args={1, 2}},
+    test_garbagecollect_now={},
     timer_start={args={2,3}},
     timer_stop={args=1},
     tolower={args=1},
diff --git a/src/nvim/getchar.c b/src/nvim/getchar.c
index fccbd69dbf..fe889eb860 100644
--- a/src/nvim/getchar.c
+++ b/src/nvim/getchar.c
@@ -1327,8 +1327,9 @@ int using_script(void)
 void before_blocking(void)
 {
   updatescript(0);
-  if (may_garbage_collect)
-    garbage_collect();
+  if (may_garbage_collect) {
+    garbage_collect(false);
+  }
 }
 
 /*
@@ -1366,10 +1367,11 @@ int vgetc(void)
   char_u buf[MB_MAXBYTES + 1];
   int i;
 
-  /* Do garbage collection when garbagecollect() was called previously and
-   * we are now at the toplevel. */
-  if (may_garbage_collect && want_garbage_collect)
-    garbage_collect();
+  // Do garbage collection when garbagecollect() was called previously and
+  // we are now at the toplevel.
+  if (may_garbage_collect && want_garbage_collect) {
+    garbage_collect(false);
+  }
 
   /*
    * If a character was put back with vungetc, it was already processed.
diff --git a/src/nvim/main.c b/src/nvim/main.c
index 88c1990786..fab968836c 100644
--- a/src/nvim/main.c
+++ b/src/nvim/main.c
@@ -624,8 +624,9 @@ void getout(int exitval)
   iconv_end();
 #endif
   cs_end();
-  if (garbage_collect_at_exit)
-    garbage_collect();
+  if (garbage_collect_at_exit) {
+    garbage_collect(false);
+  }
 
   mch_exit(exitval);
 }
diff --git a/src/nvim/testdir/runtest.vim b/src/nvim/testdir/runtest.vim
index 316aba968d..5299fec7c2 100644
--- a/src/nvim/testdir/runtest.vim
+++ b/src/nvim/testdir/runtest.vim
@@ -62,6 +62,12 @@ lang mess C
 " Always use forward slashes.
 set shellslash
 
+" Make sure $HOME does not get read or written.
+let $HOME = '/does/not/exist'
+
+" Prepare for calling garbagecollect_for_testing().
+let v:testing = 1
+
 " Align with vim defaults.
 set directory^=.
 set nohidden
diff --git a/src/nvim/testdir/test_lambda.vim b/src/nvim/testdir/test_lambda.vim
new file mode 100644
index 0000000000..d51b6f7c5a
--- /dev/null
+++ b/src/nvim/testdir/test_lambda.vim
@@ -0,0 +1,286 @@
+" Test for lambda and closure
+
+function! Test_lambda_feature()
+  call assert_equal(1, has('lambda'))
+endfunction
+
+function! Test_lambda_with_filter()
+  let s:x = 2
+  call assert_equal([2, 3], filter([1, 2, 3], {i, v -> v >= s:x}))
+endfunction
+
+function! Test_lambda_with_map()
+  let s:x = 1
+  call assert_equal([2, 3, 4], map([1, 2, 3], {i, v -> v + s:x}))
+endfunction
+
+function! Test_lambda_with_sort()
+  call assert_equal([1, 2, 3, 4, 7], sort([3,7,2,1,4], {a, b -> a - b}))
+endfunction
+
+" function! Test_lambda_with_timer()
+"   if !has('timers')
+"     return
+"   endif
+
+"   let s:n = 0
+"   let s:timer_id = 0
+"   function! s:Foo()
+"     "let n = 0
+"     let s:timer_id = timer_start(50, {-> execute("let s:n += 1 | echo s:n", "")}, {"repeat": -1})
+"   endfunction
+
+"   call s:Foo()
+"   sleep 200ms
+"   " do not collect lambda
+"   call garbagecollect()
+"   let m = s:n
+"   sleep 200ms
+"   call timer_stop(s:timer_id)
+"   call assert_true(m > 1)
+"   call assert_true(s:n > m + 1)
+"   call assert_true(s:n < 9)
+" endfunction
+
+function! Test_lambda_with_partial()
+  let l:Cb = function({... -> ['zero', a:1, a:2, a:3]}, ['one', 'two'])
+  call assert_equal(['zero', 'one', 'two', 'three'], l:Cb('three'))
+endfunction
+
+function Test_lambda_fails()
+  call assert_equal(3, {a, b -> a + b}(1, 2))
+  call assert_fails('echo {a, a -> a + a}(1, 2)', 'E15:')
+  call assert_fails('echo {a, b -> a + b)}(1, 2)', 'E15:')
+endfunc
+
+func Test_not_lambda()
+  let x = {'>' : 'foo'}
+  call assert_equal('foo', x['>'])
+endfunc
+
+function! Test_lambda_capture_by_reference()
+  let v = 1
+  let l:F = {x -> x + v}
+  let v = 2
+  call assert_equal(12, l:F(10))
+endfunction
+
+function! Test_lambda_side_effect()
+  function! s:update_and_return(arr)
+    let a:arr[1] = 5
+    return a:arr
+  endfunction
+
+  function! s:foo(arr)
+    return {-> s:update_and_return(a:arr)}
+  endfunction
+
+  let arr = [3,2,1]
+  call assert_equal([3, 5, 1], s:foo(arr)())
+endfunction
+
+function! Test_lambda_refer_local_variable_from_other_scope()
+  function! s:foo(X)
+    return a:X() " refer l:x in s:bar()
+  endfunction
+
+  function! s:bar()
+    let x = 123
+    return s:foo({-> x})
+  endfunction
+
+  call assert_equal(123, s:bar())
+endfunction
+
+function! Test_lambda_do_not_share_local_variable()
+  function! s:define_funcs()
+    let l:One = {-> split(execute("let a = 'abc' | echo a"))[0]}
+    let l:Two = {-> exists("a") ? a : "no"}
+    return [l:One, l:Two]
+  endfunction
+
+  let l:F = s:define_funcs()
+
+  call assert_equal('no', l:F[1]())
+  call assert_equal('abc', l:F[0]())
+  call assert_equal('no', l:F[1]())
+endfunction
+
+function! Test_lambda_closure_counter()
+  function! s:foo()
+    let x = 0
+    return {-> [execute("let x += 1"), x][-1]}
+  endfunction
+
+  let l:F = s:foo()
+  call garbagecollect()
+  call assert_equal(1, l:F())
+  call assert_equal(2, l:F())
+  call assert_equal(3, l:F())
+  call assert_equal(4, l:F())
+endfunction
+
+function! Test_lambda_with_a_var()
+  function! s:foo()
+    let x = 2
+    return {... -> a:000 + [x]}
+  endfunction
+  function! s:bar()
+    return s:foo()(1)
+  endfunction
+
+  call assert_equal([1, 2], s:bar())
+endfunction
+
+function! Test_lambda_call_lambda_from_lambda()
+  function! s:foo(x)
+    let l:F1 = {-> {-> a:x}}
+    return {-> l:F1()}
+  endfunction
+
+  let l:F = s:foo(1)
+  call assert_equal(1, l:F()())
+endfunction
+
+function! Test_lambda_delfunc()
+  function! s:gen()
+    let pl = l:
+    let l:Foo = {-> get(pl, "Foo", get(pl, "Bar", {-> 0}))}
+    let l:Bar = l:Foo
+    delfunction l:Foo
+    return l:Bar
+  endfunction
+
+  let l:F = s:gen()
+  call assert_fails(':call l:F()', 'E933:')
+endfunction
+
+function! Test_lambda_scope()
+  function! s:NewCounter()
+    let c = 0
+    return {-> [execute('let c += 1'), c][-1]}
+  endfunction
+
+  function! s:NewCounter2()
+    return {-> [execute('let c += 100'), c][-1]}
+  endfunction
+
+  let l:C = s:NewCounter()
+  let l:D = s:NewCounter2()
+
+  call assert_equal(1, l:C())
+  call assert_fails(':call l:D()', 'E15:') " E121: then E15:
+  call assert_equal(2, l:C())
+endfunction
+
+function! Test_lambda_share_scope()
+  function! s:New()
+    let c = 0
+    let l:Inc0 = {-> [execute('let c += 1'), c][-1]}
+    let l:Dec0 = {-> [execute('let c -= 1'), c][-1]}
+    return [l:Inc0, l:Dec0]
+  endfunction
+
+  let [l:Inc, l:Dec] = s:New()
+
+  call assert_equal(1, l:Inc())
+  call assert_equal(2, l:Inc())
+  call assert_equal(1, l:Dec())
+endfunction
+
+function! Test_lambda_circular_reference()
+  function! s:Foo()
+    let d = {}
+    let d.f = {-> d}
+    return d.f
+  endfunction
+
+  call s:Foo()
+  call garbagecollect()
+  let i = 0 | while i < 10000 | call s:Foo() | let i+= 1 | endwhile
+  call garbagecollect()
+endfunction
+
+function! Test_lambda_combination()
+  call assert_equal(2, {x -> {x -> x}}(1)(2))
+  call assert_equal(10, {y -> {x -> x(y)(10)}({y -> y})}({z -> z}))
+  call assert_equal(5.0, {x -> {y -> x / y}}(10)(2.0))
+  call assert_equal(6, {x -> {y -> {z -> x + y + z}}}(1)(2)(3))
+
+  call assert_equal(6, {x -> {f -> f(x)}}(3)({x -> x * 2}))
+  call assert_equal(6, {f -> {x -> f(x)}}({x -> x * 2})(3))
+
+  " Z combinator
+  let Z = {f -> {x -> f({y -> x(x)(y)})}({x -> f({y -> x(x)(y)})})}
+  let Fact = {f -> {x -> x == 0 ? 1 : x * f(x - 1)}}
+  call assert_equal(120, Z(Fact)(5))
+endfunction
+
+function! Test_closure_counter()
+  function! s:foo()
+    let x = 0
+    function! s:bar() closure
+      let x += 1
+      return x
+    endfunction
+    return function('s:bar')
+  endfunction
+
+  let l:F = s:foo()
+  call garbagecollect()
+  call assert_equal(1, l:F())
+  call assert_equal(2, l:F())
+  call assert_equal(3, l:F())
+  call assert_equal(4, l:F())
+endfunction
+
+function! Test_closure_unlet()
+  function! s:foo()
+    let x = 1
+    function! s:bar() closure
+      unlet x
+    endfunction
+    call s:bar()
+    return l:
+  endfunction
+
+  call assert_false(has_key(s:foo(), 'x'))
+  call garbagecollect()
+endfunction
+
+function! LambdaFoo()
+  let x = 0
+  function! LambdaBar() closure
+    let x += 1
+    return x
+  endfunction
+  return function('LambdaBar')
+endfunction
+
+func Test_closure_refcount()
+  let g:Count = LambdaFoo()
+  call test_garbagecollect_now()
+  call assert_equal(1, g:Count())
+  let g:Count2 = LambdaFoo()
+  call test_garbagecollect_now()
+  call assert_equal(1, g:Count2())
+  call assert_equal(2, g:Count())
+  call assert_equal(3, g:Count2())
+
+  delfunc LambdaFoo
+  delfunc LambdaBar
+endfunc
+
+func Test_named_function_closure()
+  func! Afoo()
+    let x = 14
+    func! s:Abar() closure
+      return x
+    endfunc
+    call assert_equal(14, s:Abar())
+  endfunc
+  call Afoo()
+  call assert_equal(14, s:Abar())
+  call garbagecollect()
+  call assert_equal(14, s:Abar())
+endfunc
diff --git a/src/nvim/version.c b/src/nvim/version.c
index 5e33597568..111b597eba 100644
--- a/src/nvim/version.c
+++ b/src/nvim/version.c
@@ -714,7 +714,7 @@ static int included_patches[] = {
   1730,
   // 1729 NA
   1728,
-  // 1727 NA
+  1727,
   // 1726 NA
   // 1725 NA
   // 1724 NA