Merge #2929 'Disallow "set encoding" after startup'

27 files changed, 96 insertions, 86 deletions

diff --git a/runtime/doc/mbyte.txt b/runtime/doc/mbyte.txt
index 8189ec5b38..9086e69959 100644
--- a/runtime/doc/mbyte.txt
+++ b/runtime/doc/mbyte.txt
@@ -89,8 +89,8 @@ See |encoding-values| for a list of acceptable values.
 
 The result is that all the text that is used inside Vim will be in this
 encoding.  Not only the text in the buffers, but also in registers, variables,
-etc.  This also means that changing the value of 'encoding' makes the existing
-text invalid!  The text doesn't change, but it will be displayed wrong.
+etc. 'encoding' cannot be changed after startup (at latest when |vimrc|
+is sourced) as changing the value later would make the existing text invalid.
 
 You can edit files in another encoding than what 'encoding' is set to.  Vim
 will convert the file when you read it and convert it back when you write it.
diff --git a/runtime/doc/message.txt b/runtime/doc/message.txt
index 7d674ae4d7..91e1a97659 100644
--- a/runtime/doc/message.txt
+++ b/runtime/doc/message.txt
@@ -739,6 +739,12 @@ Example: >
 You tried to execute a command that is neither an Ex command nor
 a user-defined command.
 
+							*E905*  >
+  Cannot set this option after startup
+
+This option might only be changed in a |vimrc| file (or earlier). This means
+that plugins can rely on the value being constant after initialization.
+
 ==============================================================================
 3. Messages						*messages*
 
diff --git a/runtime/doc/options.txt b/runtime/doc/options.txt
index 1393b11824..73dbca9c7a 100644
--- a/runtime/doc/options.txt
+++ b/runtime/doc/options.txt
@@ -2217,21 +2217,13 @@ A jump table for the options with a short description can be found at |Q_op|.
 	viminfo file, etc.  It sets the kind of characters which Vim can work
 	with.  See |encoding-names| for the possible values.
 
-	NOTE: Changing this option will not change the encoding of the
-	existing text in Vim.  It may cause non-ASCII text to become invalid.
-	It should normally be kept at its default value, or set when Vim
-	starts up.  See |multibyte|.  To reload the menus see |:menutrans|.
-
-	This option cannot be set from a |modeline|.  It would most likely
-	corrupt the text.
-
-	NOTE: For GTK+ 2 it is highly recommended to set 'encoding' to
-	"utf-8".  Although care has been taken to allow different values of
-	'encoding', "utf-8" is the natural choice for the environment and
-	avoids unnecessary conversion overhead.  "utf-8" has not been made
-	the default to prevent different behavior of the GUI and terminal
-	versions, and to avoid changing the encoding of newly created files
-	without your knowledge (in case 'fileencodings' is empty).
+	This option cannot be changed after startup.
+	Otherwise it would cause non-ASCII text inside Vim to become
+	invalid. It should normally be kept at its default value, or be set
+	in vimrc. See |multibyte|.
+
+	The recommended 'encoding' is "utf-8". Remote plugins and GUI:s
+	only support utf-8.
 
 	The character encoding of files can be different from 'encoding'.
 	This is specified with 'fileencoding'.  The conversion is done with
@@ -2243,6 +2235,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 <
 	Normally 'encoding' will be equal to your current locale.  This will
 	be the default if Vim recognizes your environment settings.
+	"utf-8" is used when the locale encoding could not be detected.
 
 	When you set this option, it fires the |EncodingChanged| autocommand
 	event so that you can set up fonts if necessary.
@@ -2254,12 +2247,6 @@ A jump table for the options with a short description can be found at |Q_op|.
 	For example "Latin-1" becomes "latin1", "ISO_88592" becomes
 	"iso-8859-2" and "utf8" becomes "utf-8".
 
-	Note: "latin1" is also used when the encoding could not be detected.
-	This only works when editing files in the same encoding!  When the
-	actual character set is not latin1, make sure 'fileencoding' and
-	'fileencodings' are empty.  When conversion is needed, switch to using
-	utf-8.
-
 	When "unicode", "ucs-2" or "ucs-4" is used, Vim internally uses utf-8.
 	You don't notice this while editing, but it does matter for the
 	|viminfo-file|.  And Vim expects the terminal to use utf-8 too.  Thus
diff --git a/runtime/doc/vim_diff.txt b/runtime/doc/vim_diff.txt
index 8f88906e38..52146cd946 100644
--- a/runtime/doc/vim_diff.txt
+++ b/runtime/doc/vim_diff.txt
@@ -74,6 +74,8 @@ are always available and may be used simultaneously in separate plugins.  The
    'p')) mkdir() will silently exit. In Vim this was an error.
 3. mkdir() error messages now include strerror() text when mkdir fails.
 
+'encoding' cannot be changed after startup.
+
 ==============================================================================
 4. New Features						     *nvim-features-new*
 
diff --git a/src/nvim/globals.h b/src/nvim/globals.h
index e5a993aa5a..183e8fcff2 100644
--- a/src/nvim/globals.h
+++ b/src/nvim/globals.h
@@ -633,6 +633,10 @@ EXTERN int silent_mode INIT(= FALSE);
 /* set to TRUE when "-s" commandline argument
  * used for ex */
 
+// Set to true when sourcing of startup scripts (nvimrc) is done.
+// Used for options that cannot be changed after startup scripts.
+EXTERN bool did_source_startup_scripts INIT(= false);
+
 EXTERN pos_T VIsual;            /* start position of active Visual selection */
 EXTERN int VIsual_active INIT(= FALSE);
 /* whether Visual mode is active */
@@ -1078,6 +1082,8 @@ EXTERN garray_T error_ga
  * Excluded are errors that are only used once and debugging messages.
  */
 EXTERN char_u e_abort[] INIT(= N_("E470: Command aborted"));
+EXTERN char_u e_afterinit[] INIT(= N_(
+        "E905: Cannot set this option after startup"));
 EXTERN char_u e_api_spawn_failed[] INIT(= N_("E903: Could not spawn API job"));
 EXTERN char_u e_argreq[] INIT(= N_("E471: Argument required"));
 EXTERN char_u e_backslash[] INIT(= N_("E10: \\ should be followed by /, ? or &"));
diff --git a/src/nvim/main.c b/src/nvim/main.c
index dd2b813b1c..27f8340ec7 100644
--- a/src/nvim/main.c
+++ b/src/nvim/main.c
@@ -1911,6 +1911,7 @@ static void source_startup_scripts(mparm_T *parmp)
       need_wait_return = TRUE;
     secure = 0;
   }
+  did_source_startup_scripts = true;
   TIME_MSG("sourcing vimrc file(s)");
 }
 
diff --git a/src/nvim/option.c b/src/nvim/option.c
index 6e82c45edf..c47616620c 100644
--- a/src/nvim/option.c
+++ b/src/nvim/option.c
@@ -575,6 +575,12 @@ void set_init_1(void)
     mb_init();
   }
 
+  // Don't change &encoding when resetting to defaults with ":set all&".
+  opt_idx = findoption((char_u *)"encoding");
+  if (opt_idx >= 0) {
+    options[opt_idx].flags |= P_NODEFAULT;
+  }
+
   /* Set the default for 'helplang'. */
   set_helplang_default(get_mess_lang());
 }
@@ -2271,10 +2277,11 @@ did_set_string_option (
   else if (varp == &p_ei) {
     if (check_ei() == FAIL)
       errmsg = e_invarg;
-  }
   /* 'encoding' and 'fileencoding' */
-  else if (varp == &p_enc || gvarp == &p_fenc) {
-    if (gvarp == &p_fenc) {
+  } else if (varp == &p_enc || gvarp == &p_fenc) {
+    if (varp == &p_enc && did_source_startup_scripts) {
+       errmsg = e_afterinit;
+    } else if (gvarp == &p_fenc) {
       if (!MODIFIABLE(curbuf) && opt_flags != OPT_GLOBAL)
         errmsg = e_modifiable;
       else if (vim_strchr(*varp, ',') != NULL)
@@ -2305,10 +2312,6 @@ did_set_string_option (
        * (with another encoding). */
       if (varp == &p_enc && *curbuf->b_p_keymap != NUL)
         (void)keymap_init();
-
-      if (varp == &p_enc) {
-        ui_update_encoding();
-      }
     }
   } else if (varp == &p_penc) {
     /* Canonize printencoding if VIM standard one */
diff --git a/src/nvim/testdir/test10.in b/src/nvim/testdir/test10.in
index 2d0d546606..769d690acb 100644
--- a/src/nvim/testdir/test10.in
+++ b/src/nvim/testdir/test10.in
@@ -4,7 +4,6 @@ STARTTEST
 :so small.vim
 :" Also test a BOM is ignored.
 :so mbyte.vim
-:set encoding=utf-8
 :7/start of errorfile/,/end of errorfile/w! Xerrorfile1
 :7/start of errorfile/,/end of errorfile/-1w! Xerrorfile2
 :/start of testfile/,/end of testfile/w! Xtestfile
diff --git a/src/nvim/testdir/test39.in b/src/nvim/testdir/test39.in
index c4e46fff26..ebbcbd6d0d 100644
--- a/src/nvim/testdir/test39.in
+++ b/src/nvim/testdir/test39.in
@@ -5,8 +5,6 @@ And test "U" in Visual mode, also on German sharp S.
 STARTTEST
 :so small.vim
 :so mbyte.vim
-:" This only works when 'encoding' is "latin1", don't depend on the environment
-:set enc=latin1
 /^abcde
 :" Test shift-right of a block
 jlllljj>wlljlll>
@@ -39,18 +37,18 @@ G$khhhhhkkcmno
 :exe ":norm! l\<C-V>j$hhAab\<Esc>"
 :.,/^$/w >> test.out
 :" Test for Visual block insert when virtualedit=all and utf-8 encoding
-:set ve=all enc=utf-8
+:set ve=all
 :/\t\tline
 :exe ":norm! 07l\<C-V>jjIx\<Esc>"
 :.,/^$/w >> test.out
 :" Test for Visual block append when virtualedit=all
 :exe ":norm! 012l\<C-v>jjAx\<Esc>"
-:set ve= enc=latin1
+:set ve=
 :.,/^$/w >> test.out
-:" gUe must uppercase a whole word, also when � changes to SS
-Gothe youtu�euu endYpk0wgUe
+:" gUe must uppercase a whole word, also when ß changes to SS
+Gothe youtußeuu endYpk0wgUe
 :" gUfx must uppercase until x, inclusive.
-O- you�tu�exu -0fogUfx
+O- youßtußexu -0fogUfx
 :" VU must uppercase a whole line
 YpkVU
 :" same, when it's the last line in the buffer
@@ -89,7 +87,7 @@ cccc
 dddd
 
 yaaa
-���
+¿¿¿
 bbb
 
 A23
diff --git a/src/nvim/testdir/test39.ok b/src/nvim/testdir/test39.ok
index 5c517e2223..198e5b14dc 100644
--- a/src/nvim/testdir/test39.ok
+++ b/src/nvim/testdir/test39.ok
diff --git a/src/nvim/testdir/test69.in b/src/nvim/testdir/test69.in
index 26f41e8a29..f583947dfb 100644
--- a/src/nvim/testdir/test69.in
+++ b/src/nvim/testdir/test69.in
@@ -5,7 +5,6 @@ Also test byteidx() and byteidxcomp()
 
 STARTTEST
 :so mbyte.vim
-:set encoding=utf-8
 ENDTEST
 
 Results of test69:
diff --git a/src/nvim/testdir/test83.in b/src/nvim/testdir/test83.in
index 297d560d2f..d54b1bcddd 100644
--- a/src/nvim/testdir/test83.in
+++ b/src/nvim/testdir/test83.in
@@ -2,7 +2,6 @@ Tests for tag search with !_TAG_FILE_ENCODING.
 
 STARTTEST
 :so mbyte.vim
-:set enc=utf8
 :if !has('iconv') || iconv("\x82\x60", "cp932", "utf-8") != "\uff21"
 : e! test.ok
 : w! test.out
diff --git a/src/nvim/testdir/test_eval.in b/src/nvim/testdir/test_eval.in
index 328ee2e127..b2b982a434 100644
--- a/src/nvim/testdir/test_eval.in
+++ b/src/nvim/testdir/test_eval.in
@@ -4,7 +4,6 @@ Note: system clipboard is saved, changed and restored.
 
 STARTTEST
 :so small.vim
-:set encoding=latin1
 :set noswapfile
 :lang C
 :fun AppendRegContents(reg)
diff --git a/src/nvim/testdir/unix.vim b/src/nvim/testdir/unix.vim
index f766e74c30..aa1f6a92bc 100644
--- a/src/nvim/testdir/unix.vim
+++ b/src/nvim/testdir/unix.vim
@@ -1,3 +1,6 @@
 " Settings for test script execution
 " Always use "sh", don't use the value of "$SHELL".
 set shell=sh
+
+" Don't depend on system locale, always use utf-8
+set encoding=utf-8
diff --git a/src/nvim/tui/input.c b/src/nvim/tui/input.c
index b680e885df..6c362540d0 100644
--- a/src/nvim/tui/input.c
+++ b/src/nvim/tui/input.c
@@ -25,7 +25,9 @@ void term_input_init(TermInput *input, Loop *loop)
   if (!term) {
     term = "";  // termkey_new_abstract assumes non-null (#2745)
   }
-  input->tk = termkey_new_abstract(term, 0);
+  int enc_flag = enc_utf8 ? TERMKEY_FLAG_UTF8 : TERMKEY_FLAG_RAW;
+  input->tk = termkey_new_abstract(term, enc_flag);
+
   int curflags = termkey_get_canonflags(input->tk);
   termkey_set_canonflags(input->tk, curflags | TERMKEY_CANON_DELBS);
   // setup input handle
@@ -57,13 +59,6 @@ void term_input_stop(TermInput *input)
   time_watcher_stop(&input->timer_handle);
 }
 
-void term_input_set_encoding(TermInput *input, char* enc)
-{
-  int enc_flag = strcmp(enc, "utf-8") == 0 ? TERMKEY_FLAG_UTF8
-                                           : TERMKEY_FLAG_RAW;
-  termkey_set_flags(input->tk, enc_flag);
-}
-
 static void input_enqueue_event(void **argv)
 {
   char *buf = argv[0];
diff --git a/src/nvim/tui/tui.c b/src/nvim/tui/tui.c
index b2bb80a092..4c8e88c383 100644
--- a/src/nvim/tui/tui.c
+++ b/src/nvim/tui/tui.c
@@ -99,7 +99,6 @@ UI *tui_start(void)
   ui->suspend = tui_suspend;
   ui->set_title = tui_set_title;
   ui->set_icon = tui_set_icon;
-  ui->set_encoding = tui_set_encoding;
   return ui_bridge_attach(ui, tui_main, tui_scheduler);
 }
 
@@ -625,12 +624,6 @@ static void tui_set_icon(UI *ui, char *icon)
 {
 }
 
-static void tui_set_encoding(UI *ui, char* enc)
-{
-  TUIData *data = ui->data;
-  term_input_set_encoding(&data->input, enc);
-}
-
 static void invalidate(UI *ui, int top, int bot, int left, int right)
 {
   TUIData *data = ui->data;
diff --git a/src/nvim/ui.c b/src/nvim/ui.c
index e1bbcdb193..786f6026de 100644
--- a/src/nvim/ui.c
+++ b/src/nvim/ui.c
@@ -113,11 +113,6 @@ void ui_set_icon(char *icon)
   UI_CALL(flush);
 }
 
-void ui_update_encoding(void)
-{
-  UI_CALL(set_encoding, (char*)p_enc);
-}
-
 // May update the shape of the cursor.
 void ui_cursor_shape(void)
 {
@@ -188,7 +183,6 @@ void ui_attach(UI *ui)
   }
 
   uis[ui_count++] = ui;
-  ui_update_encoding();
   ui_refresh();
 }
 
diff --git a/src/nvim/ui.h b/src/nvim/ui.h
index c87d7f0c55..4c051fcfbf 100644
--- a/src/nvim/ui.h
+++ b/src/nvim/ui.h
@@ -38,7 +38,6 @@ struct ui_t {
   void (*suspend)(UI *ui);
   void (*set_title)(UI *ui, char *title);
   void (*set_icon)(UI *ui, char *icon);
-  void (*set_encoding)(UI *ui, char *enc);
   void (*stop)(UI *ui);
 };
 
diff --git a/src/nvim/ui_bridge.c b/src/nvim/ui_bridge.c
index 6e1a27cc9c..2ec31de5e1 100644
--- a/src/nvim/ui_bridge.c
+++ b/src/nvim/ui_bridge.c
@@ -52,7 +52,6 @@ UI *ui_bridge_attach(UI *ui, ui_main_fn ui_main, event_scheduler scheduler)
   rv->bridge.suspend = ui_bridge_suspend;
   rv->bridge.set_title = ui_bridge_set_title;
   rv->bridge.set_icon = ui_bridge_set_icon;
-  rv->bridge.set_encoding = ui_bridge_set_encoding;
   rv->scheduler = scheduler;
 
   rv->ui_main = ui_main;
@@ -334,14 +333,3 @@ static void ui_bridge_set_icon_event(void **argv)
   ui->set_icon(ui, argv[1]);
   xfree(argv[1]);
 }
-
-static void ui_bridge_set_encoding(UI *b, char* enc)
-{
-  UI_CALL(b, set_encoding, 2, b, xstrdup(enc));
-}
-static void ui_bridge_set_encoding_event(void **argv)
-{
-  UI *ui = UI(argv[0]);
-  ui->set_encoding(ui, argv[1]);
-  xfree(argv[1]);
-}
diff --git a/test/functional/ex_cmds/encoding_spec.lua b/test/functional/ex_cmds/encoding_spec.lua
new file mode 100644
index 0000000000..997776ca25
--- /dev/null
+++ b/test/functional/ex_cmds/encoding_spec.lua
@@ -0,0 +1,40 @@
+local helpers = require('test.functional.helpers')
+local clear, execute, feed = helpers.clear, helpers.execute, helpers.feed
+local eq, neq, eval = helpers.eq, helpers.neq, helpers.eval
+
+describe('&encoding', function()
+
+  before_each(function()
+    clear()
+    -- sanity check: tests should run with encoding=utf-8
+    eq('utf-8', eval('&encoding'))
+    eq(3, eval('strwidth("Bär")'))
+  end)
+
+  it('cannot be changed after setup', function()
+    execute('set encoding=latin1')
+    -- error message expected
+    feed('<cr>')
+    neq(nil, string.find(eval('v:errmsg'), '^E905:'))
+    eq('utf-8', eval('&encoding'))
+    -- check nvim is still in utf-8 mode
+    eq(3, eval('strwidth("Bär")'))
+  end)
+
+  it('is not changed by `set all&`', function()
+    -- we need to set &encoding to something non-default
+    -- use 'latin1' when enc&vi is 'utf-8', 'utf-8' otherwise
+    execute('set fenc=default')
+    local enc_default, enc_other, width = eval('&fenc'), 'utf-8', 3
+    if enc_default == 'utf-8' then
+      enc_other = 'latin1'
+      width = 4 -- utf-8 string 'Bär' will count as 4 latin1 chars
+    end
+
+    clear('set enc=' .. enc_other)
+    execute('set all&')
+    eq(enc_other, eval('&encoding'))
+    eq(width, eval('strwidth("Bär")'))
+  end)
+
+end)
diff --git a/test/functional/helpers.lua b/test/functional/helpers.lua
index f228f6290c..6055cc3c59 100644
--- a/test/functional/helpers.lua
+++ b/test/functional/helpers.lua
@@ -7,7 +7,7 @@ local Session = require('nvim.session')
 
 local nvim_prog = os.getenv('NVIM_PROG') or 'build/bin/nvim'
 local nvim_argv = {nvim_prog, '-u', 'NONE', '-i', 'NONE', '-N',
-                   '--cmd', 'set shortmess+=I background=light noswapfile noautoindent laststatus=1',
+                   '--cmd', 'set shortmess+=I background=light noswapfile noautoindent laststatus=1 encoding=utf-8',
                    '--embed'}
 
 -- Formulate a path to the directory containing nvim.  We use this to
@@ -183,11 +183,16 @@ local function spawn(argv)
   return session
 end
 
-local function clear()
+local function clear(extra_cmd)
   if session then
     session:exit(0)
   end
-  session = spawn(nvim_argv)
+  local args = {unpack(nvim_argv)}
+  if extra_cmd ~= nil then
+    table.insert(args, '--cmd')
+    table.insert(args, extra_cmd)
+  end
+  session = spawn(args)
 end
 
 local function insert(...)
diff --git a/test/functional/legacy/044_099_regexp_multibyte_magic_spec.lua b/test/functional/legacy/044_099_regexp_multibyte_magic_spec.lua
index 58838e9d6e..efe61aa354 100644
--- a/test/functional/legacy/044_099_regexp_multibyte_magic_spec.lua
+++ b/test/functional/legacy/044_099_regexp_multibyte_magic_spec.lua
@@ -33,7 +33,6 @@ local function run_test_with_regexpengine(regexpengine)
     j 0123❤x
     k combinations]])
 
-  execute('set encoding=utf-8')
   execute('set re=' .. regexpengine)
 
   -- Lines 1-8. Exercise regexp search with various magic settings. On each
diff --git a/test/functional/legacy/075_maparg_spec.lua b/test/functional/legacy/075_maparg_spec.lua
index dac8940314..418abb14d4 100644
--- a/test/functional/legacy/075_maparg_spec.lua
+++ b/test/functional/legacy/075_maparg_spec.lua
@@ -10,7 +10,6 @@ describe('maparg()', function()
 
   it('is working', function()
     execute('set cpo-=<')
-    execute('set encoding=utf8')
 
     -- Test maparg() with a string result
     execute('map foo<C-V> is<F4>foo')
diff --git a/test/functional/legacy/082_string_comparison_spec.lua b/test/functional/legacy/082_string_comparison_spec.lua
index bd4e8a4d79..1615828ca0 100644
--- a/test/functional/legacy/082_string_comparison_spec.lua
+++ b/test/functional/legacy/082_string_comparison_spec.lua
@@ -9,7 +9,6 @@ describe('case-insensitive string comparison in UTF-8', function()
   setup(clear)
 
   it('is working', function()
-    execute('set enc=utf8')
     feed('ggdG<cr>')
     source([[
       function! Ch(a, op, b, expected)
diff --git a/test/functional/legacy/095_regexp_multibyte_spec.lua b/test/functional/legacy/095_regexp_multibyte_spec.lua
index a72fb669d2..559222e2ff 100644
--- a/test/functional/legacy/095_regexp_multibyte_spec.lua
+++ b/test/functional/legacy/095_regexp_multibyte_spec.lua
@@ -15,7 +15,7 @@ describe('regex with multi-byte', function()
       Results of test95:]])
 
     source([=[
-      set encoding=utf-8 nomore
+      set nomore
       let tl = []
 
       call add(tl, [2, '[[:alpha:][=a=]]\+', '879 aiaãâaiuvna ', 'aiaãâaiuvna'])
diff --git a/test/functional/legacy/mapping_spec.lua b/test/functional/legacy/mapping_spec.lua
index 0843506827..899f7423d0 100644
--- a/test/functional/legacy/mapping_spec.lua
+++ b/test/functional/legacy/mapping_spec.lua
@@ -12,8 +12,6 @@ describe('mapping', function()
       test starts here:
       ]])
 
-    execute('set encoding=utf-8')
-
     -- Abbreviations with р (0x80) should work.
     execute('inoreab чкпр   vim')
     feed('GAчкпр <esc>')
diff --git a/test/functional/legacy/utf8_spec.lua b/test/functional/legacy/utf8_spec.lua
index d26f436057..ef717042d0 100644
--- a/test/functional/legacy/utf8_spec.lua
+++ b/test/functional/legacy/utf8_spec.lua
@@ -10,7 +10,6 @@ describe('utf8', function()
   it('is working', function()
     insert('start:')
 
-    execute('set encoding=utf-8')
     execute('new')
     execute('call setline(1, ["aaa", "あああ", "bbb"])')