fnamemodify: fix handling of :r after :e #11165

- Test fnamemodify()
- Test handling of `expand("%:e:e:r")`.
- Fix :e:e:r on filenames with insufficiently many extensions

During `fnamemodify()`, ensuring that we don't go before the filename's
tail is insufficient in cases where we've already handled a ":e"
modifier, for example:

```
"path/to/this.file.ext" :e:e:r:r
         ^    ^-------- *fnamep
         +------------- tail
```

This means for a ":r", we'll go before `*fnamep`, and outside the bounds
of the filename. This is both incorrect and causes neovim to exit with
an allocation error.

We exit because we attempt to calculate `s - *fnamep` (line 23948).
Since `s` is before `*fnamep`, we caluclate a negative length, which
ends up being interpreted as an amount to allocate, causing neovim to
exit with ENOMEM (`memory.c:xmalloc`).

We must instead ensure we don't go before `*fnamep` nor `tail`.
The check for `tail` is still relevant, for example:

```
"path/to/this.file.ext" :r:r:r
 ^       ^------------- tail
 +--------------------- *fnamep
```
Here we don't want to go before `tail`.

close #11165

1 files changed, 118 insertions, 1 deletions

diff --git a/test/functional/eval/fnamemodify_spec.lua b/test/functional/eval/fnamemodify_spec.lua
index fe6b50a544..d54a6db417 100644
--- a/test/functional/eval/fnamemodify_spec.lua
+++ b/test/functional/eval/fnamemodify_spec.lua
@@ -3,8 +3,14 @@ local clear = helpers.clear
 local eq = helpers.eq
 local iswin = helpers.iswin
 local fnamemodify = helpers.funcs.fnamemodify
+local getcwd = helpers.funcs.getcwd
 local command = helpers.command
 local write_file = helpers.write_file
+local alter_slashes = helpers.alter_slashes
+
+local function eq_slashconvert(expected, got)
+  eq(alter_slashes(expected), alter_slashes(got))
+end
 
 describe('fnamemodify()', function()
   setup(function()
@@ -17,7 +23,7 @@ describe('fnamemodify()', function()
     os.remove('Xtest-fnamemodify.txt')
   end)
 
-  it('works', function()
+  it('handles the root path', function()
     local root = helpers.pathroot()
     eq(root, fnamemodify([[/]], ':p:h'))
     eq(root, fnamemodify([[/]], ':p'))
@@ -36,4 +42,115 @@ describe('fnamemodify()', function()
   it(':8 works', function()
     eq('Xtest-fnamemodify.txt', fnamemodify([[Xtest-fnamemodify.txt]], ':8'))
   end)
+
+  it('handles examples from ":help filename-modifiers"', function()
+    local filename = "src/version.c"
+    local cwd = getcwd()
+
+    eq_slashconvert(cwd .. '/src/version.c', fnamemodify(filename, ':p'))
+
+    eq_slashconvert('src/version.c', fnamemodify(filename, ':p:.'))
+    eq_slashconvert(cwd .. '/src', fnamemodify(filename, ':p:h'))
+    eq_slashconvert(cwd .. '', fnamemodify(filename, ':p:h:h'))
+    eq('version.c', fnamemodify(filename, ':p:t'))
+    eq_slashconvert(cwd .. '/src/version', fnamemodify(filename, ':p:r'))
+
+    eq_slashconvert(cwd .. '/src/main.c', fnamemodify(filename, ':s?version?main?:p'))
+
+    local converted_cwd = cwd:gsub('/', '\\')
+    eq(converted_cwd .. '\\src\\version.c', fnamemodify(filename, ':p:gs?/?\\\\?'))
+
+    eq('src', fnamemodify(filename, ':h'))
+    eq('version.c', fnamemodify(filename, ':t'))
+    eq_slashconvert('src/version', fnamemodify(filename, ':r'))
+    eq('version', fnamemodify(filename, ':t:r'))
+    eq('c', fnamemodify(filename, ':e'))
+
+    eq_slashconvert('src/main.c', fnamemodify(filename, ':s?version?main?'))
+  end)
+
+  it('handles advanced examples from ":help filename-modifiers"', function()
+    local filename = "src/version.c.gz"
+
+    eq('gz', fnamemodify(filename, ':e'))
+    eq('c.gz', fnamemodify(filename, ':e:e'))
+    eq('c.gz', fnamemodify(filename, ':e:e:e'))
+
+    eq('c', fnamemodify(filename, ':e:e:r'))
+
+    eq_slashconvert('src/version.c', fnamemodify(filename, ':r'))
+    eq('c', fnamemodify(filename, ':r:e'))
+
+    eq_slashconvert('src/version', fnamemodify(filename, ':r:r'))
+    eq_slashconvert('src/version', fnamemodify(filename, ':r:r:r'))
+  end)
+
+  it('handles :h', function()
+    eq('.', fnamemodify('hello.txt', ':h'))
+
+    eq_slashconvert('path/to', fnamemodify('path/to/hello.txt', ':h'))
+  end)
+
+  it('handles :t', function()
+    eq('hello.txt', fnamemodify('hello.txt', ':t'))
+    eq_slashconvert('hello.txt', fnamemodify('path/to/hello.txt', ':t'))
+  end)
+
+  it('handles :r', function()
+    eq('hello', fnamemodify('hello.txt', ':r'))
+    eq_slashconvert('path/to/hello', fnamemodify('path/to/hello.txt', ':r'))
+  end)
+
+  it('handles :e', function()
+    eq('txt', fnamemodify('hello.txt', ':e'))
+    eq_slashconvert('txt', fnamemodify('path/to/hello.txt', ':e'))
+  end)
+
+  it('handles regex replacements', function()
+    eq('content-there-here.txt', fnamemodify('content-here-here.txt', ':s/here/there/'))
+    eq('content-there-there.txt', fnamemodify('content-here-here.txt', ':gs/here/there/'))
+  end)
+
+  it('handles shell escape', function()
+    local expected
+
+    if iswin() then
+      -- we expand with double-quotes on Windows
+      expected = [["hello there! quote ' newline]] .. '\n' .. [["]]
+    else
+      expected = [['hello there! quote '\'' newline]] .. '\n' .. [[']]
+    end
+
+    eq(expected, fnamemodify("hello there! quote ' newline\n", ':S'))
+  end)
+
+  it('can combine :e and :r', function()
+    -- simple, single extension filename
+    eq('c', fnamemodify('a.c', ':e'))
+    eq('c', fnamemodify('a.c', ':e:e'))
+    eq('c', fnamemodify('a.c', ':e:e:r'))
+    eq('c', fnamemodify('a.c', ':e:e:r:r'))
+
+    -- multi extension filename
+    eq('rb', fnamemodify('a.spec.rb', ':e:r'))
+    eq('rb', fnamemodify('a.spec.rb', ':e:r:r'))
+
+    eq('spec', fnamemodify('a.spec.rb', ':e:e:r'))
+    eq('spec', fnamemodify('a.spec.rb', ':e:e:r:r'))
+
+    eq('spec', fnamemodify('a.b.spec.rb', ':e:e:r'))
+    eq('b.spec', fnamemodify('a.b.spec.rb', ':e:e:e:r'))
+    eq('b', fnamemodify('a.b.spec.rb', ':e:e:e:r:r'))
+
+    eq('spec', fnamemodify('a.b.spec.rb', ':r:e'))
+    eq('b', fnamemodify('a.b.spec.rb', ':r:r:e'))
+
+    -- extraneous :e expansions
+    eq('c', fnamemodify('a.b.c.d.e', ':r:r:e'))
+    eq('b.c', fnamemodify('a.b.c.d.e', ':r:r:e:e'))
+
+    -- :e never includes the whole filename, so "a.b":e:e:e --> "b"
+    eq('b.c', fnamemodify('a.b.c.d.e', ':r:r:e:e:e'))
+    eq('b.c', fnamemodify('a.b.c.d.e', ':r:r:e:e:e:e'))
+  end)
 end)