From b522cb1ac3fbdf6e68eed5d0b6e1cbeaf3ac2254 Mon Sep 17 00:00:00 2001 From: bfredl Date: Mon, 6 Nov 2023 14:52:27 +0100 Subject: refactor(grid): make screen rendering more multibyte than ever before Problem: buffer text with composing chars are converted from UTF-8 to an array of up to seven UTF-32 values and then converted back to UTF-8 strings. Solution: Convert buffer text directly to UTF-8 based schar_T values. The limit of the text size is now in schar_T bytes, which is currently 31+1 but easily could be raised as it no longer multiplies the size of the entire screen grid when not used, the full size is only required for temporary scratch buffers. Also does some general cleanup to win_line text handling, which was unnecessarily complicated due to multibyte rendering being an "opt-in" feature long ago. Nowadays, a char is just a char, regardless if it consists of one ASCII byte or multiple bytes. --- test/functional/ui/fold_spec.lua | 38 +++++++++++++++++------------------ test/functional/ui/multibyte_spec.lua | 30 +++++++++++++++++++++++++++ test/functional/ui/output_spec.lua | 4 ++-- 3 files changed, 50 insertions(+), 22 deletions(-) (limited to 'test/functional') diff --git a/test/functional/ui/fold_spec.lua b/test/functional/ui/fold_spec.lua index 9a0182ea29..1addf7088e 100644 --- a/test/functional/ui/fold_spec.lua +++ b/test/functional/ui/fold_spec.lua @@ -1102,8 +1102,6 @@ describe("folded lines", function() end) it("works with multibyte text", function() - -- Currently the only allowed value of 'maxcombine' - eq(6, meths.get_option_value('maxcombine', {})) eq(true, meths.get_option_value('arabicshape', {})) insert([[ å 语 x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢͟ العَرَبِيَّة @@ -1120,7 +1118,7 @@ describe("folded lines", function() [2:---------------------------------------------]| [3:---------------------------------------------]| ## grid 2 - å 语 x̎͂̀̂͛͛ ﺎﻠﻋَﺮَﺒِﻳَّﺓ | + å 语 x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ ﺎﻠﻋَﺮَﺒِﻳَّﺓ | möre tex^t | {1:~ }| {1:~ }| @@ -1132,7 +1130,7 @@ describe("folded lines", function() ]]) else screen:expect([[ - å 语 x̎͂̀̂͛͛ ﺎﻠﻋَﺮَﺒِﻳَّﺓ | + å 语 x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ ﺎﻠﻋَﺮَﺒِﻳَّﺓ | möre tex^t | {1:~ }| {1:~ }| @@ -1156,7 +1154,7 @@ describe("folded lines", function() [2:---------------------------------------------]| [3:---------------------------------------------]| ## grid 2 - {5:^+-- 2 lines: å 语 x̎͂̀̂͛͛ ﺎﻠﻋَﺮَﺒِﻳَّﺓ·················}| + {5:^+-- 2 lines: å 语 x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ ﺎﻠﻋَﺮَﺒِﻳَّﺓ·················}| {1:~ }| {1:~ }| {1:~ }| @@ -1168,7 +1166,7 @@ describe("folded lines", function() ]]) else screen:expect([[ - {5:^+-- 2 lines: å 语 x̎͂̀̂͛͛ ﺎﻠﻋَﺮَﺒِﻳَّﺓ·················}| + {5:^+-- 2 lines: å 语 x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ ﺎﻠﻋَﺮَﺒِﻳَّﺓ·················}| {1:~ }| {1:~ }| {1:~ }| @@ -1192,7 +1190,7 @@ describe("folded lines", function() [2:---------------------------------------------]| [3:---------------------------------------------]| ## grid 2 - {5:^+-- 2 lines: å 语 x̎͂̀̂͛͛ العَرَبِيَّة·················}| + {5:^+-- 2 lines: å 语 x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ العَرَبِيَّة·················}| {1:~ }| {1:~ }| {1:~ }| @@ -1204,7 +1202,7 @@ describe("folded lines", function() ]]) else screen:expect([[ - {5:^+-- 2 lines: å 语 x̎͂̀̂͛͛ العَرَبِيَّة·················}| + {5:^+-- 2 lines: å 语 x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ العَرَبِيَّة·················}| {1:~ }| {1:~ }| {1:~ }| @@ -1228,7 +1226,7 @@ describe("folded lines", function() [2:---------------------------------------------]| [3:---------------------------------------------]| ## grid 2 - {7:+ }{8: 1 }{5:^+-- 2 lines: å 语 x̎͂̀̂͛͛ العَرَبِيَّة···········}| + {7:+ }{8: 1 }{5:^+-- 2 lines: å 语 x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ العَرَبِيَّة···········}| {1:~ }| {1:~ }| {1:~ }| @@ -1240,7 +1238,7 @@ describe("folded lines", function() ]]) else screen:expect([[ - {7:+ }{8: 1 }{5:^+-- 2 lines: å 语 x̎͂̀̂͛͛ العَرَبِيَّة···········}| + {7:+ }{8: 1 }{5:^+-- 2 lines: å 语 x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ العَرَبِيَّة···········}| {1:~ }| {1:~ }| {1:~ }| @@ -1265,7 +1263,7 @@ describe("folded lines", function() [2:---------------------------------------------]| [3:---------------------------------------------]| ## grid 2 - {5:···········ةيَّبِرَعَلا x̎͂̀̂͛͛ 语 å :senil 2 --^+}{8: 1 }{7: +}| + {5:···········ةيَّبِرَعَلا x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ 语 å :senil 2 --^+}{8: 1 }{7: +}| {1: ~}| {1: ~}| {1: ~}| @@ -1277,7 +1275,7 @@ describe("folded lines", function() ]]) else screen:expect([[ - {5:···········ةيَّبِرَعَلا x̎͂̀̂͛͛ 语 å :senil 2 --^+}{8: 1 }{7: +}| + {5:···········ةيَّبِرَعَلا x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ 语 å :senil 2 --^+}{8: 1 }{7: +}| {1: ~}| {1: ~}| {1: ~}| @@ -1301,7 +1299,7 @@ describe("folded lines", function() [2:---------------------------------------------]| [3:---------------------------------------------]| ## grid 2 - {5:·················ةيَّبِرَعَلا x̎͂̀̂͛͛ 语 å :senil 2 --^+}| + {5:·················ةيَّبِرَعَلا x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ 语 å :senil 2 --^+}| {1: ~}| {1: ~}| {1: ~}| @@ -1313,7 +1311,7 @@ describe("folded lines", function() ]]) else screen:expect([[ - {5:·················ةيَّبِرَعَلا x̎͂̀̂͛͛ 语 å :senil 2 --^+}| + {5:·················ةيَّبِرَعَلا x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ 语 å :senil 2 --^+}| {1: ~}| {1: ~}| {1: ~}| @@ -1337,7 +1335,7 @@ describe("folded lines", function() [2:---------------------------------------------]| [3:---------------------------------------------]| ## grid 2 - {5:·················ﺔﻴَّﺑِﺮَﻌَﻟﺍ x̎͂̀̂͛͛ 语 å :senil 2 --^+}| + {5:·················ﺔﻴَّﺑِﺮَﻌَﻟﺍ x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ 语 å :senil 2 --^+}| {1: ~}| {1: ~}| {1: ~}| @@ -1349,7 +1347,7 @@ describe("folded lines", function() ]]) else screen:expect([[ - {5:·················ﺔﻴَّﺑِﺮَﻌَﻟﺍ x̎͂̀̂͛͛ 语 å :senil 2 --^+}| + {5:·················ﺔﻴَّﺑِﺮَﻌَﻟﺍ x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ 语 å :senil 2 --^+}| {1: ~}| {1: ~}| {1: ~}| @@ -1373,7 +1371,7 @@ describe("folded lines", function() [2:---------------------------------------------]| [3:---------------------------------------------]| ## grid 2 - ﺔﻴَّﺑِﺮَﻌَ^ﻟﺍ x̎͂̀̂͛͛ 语 å| + ﺔﻴَّﺑِﺮَﻌَ^ﻟﺍ x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ 语 å| txet eröm| {1: ~}| {1: ~}| @@ -1385,7 +1383,7 @@ describe("folded lines", function() ]]) else screen:expect([[ - ﺔﻴَّﺑِﺮَﻌَ^ﻟﺍ x̎͂̀̂͛͛ 语 å| + ﺔﻴَّﺑِﺮَﻌَ^ﻟﺍ x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ 语 å| txet eröm| {1: ~}| {1: ~}| @@ -1409,7 +1407,7 @@ describe("folded lines", function() [2:---------------------------------------------]| [3:---------------------------------------------]| ## grid 2 - ةيَّبِرَعَ^لا x̎͂̀̂͛͛ 语 å| + ةيَّبِرَعَ^لا x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ 语 å| txet eröm| {1: ~}| {1: ~}| @@ -1421,7 +1419,7 @@ describe("folded lines", function() ]]) else screen:expect([[ - ةيَّبِرَعَ^لا x̎͂̀̂͛͛ 语 å| + ةيَّبِرَعَ^لا x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢ 语 å| txet eröm| {1: ~}| {1: ~}| diff --git a/test/functional/ui/multibyte_spec.lua b/test/functional/ui/multibyte_spec.lua index 077dd1a779..d72bf27d6b 100644 --- a/test/functional/ui/multibyte_spec.lua +++ b/test/functional/ui/multibyte_spec.lua @@ -228,6 +228,36 @@ describe("multibyte rendering", function() ]]} end) + + it('works with arabicshape and multiple composing chars', function() + -- this tests an important edge case: arabicshape might increase the byte size of the base + -- character in a way so that the last composing char no longer fits. use "g8" on the text + -- to observe what is happening (the final E1 80 B7 gets deleted with 'arabicshape') + -- If we would increase the schar_t size, say from 32 to 64 bytes, we need to extend the + -- test text with even more zalgo energy to still touch this edge case. + + meths.buf_set_lines(0,0,-1,true, {"سلام့̀́̂̃̄̅̆̇̈̉̊̋̌"}) + command('set noarabicshape') + + screen:expect{grid=[[ + ^سلام့̀́̂̃̄̅̆̇̈̉̊̋̌ | + {1:~ }| + {1:~ }| + {1:~ }| + {1:~ }| + | + ]]} + + command('set arabicshape') + screen:expect{grid=[[ + ^ﺱﻼﻣ̀́̂̃̄̅̆̇̈̉̊̋̌ | + {1:~ }| + {1:~ }| + {1:~ }| + {1:~ }| + | + ]]} + end) end) describe('multibyte rendering: statusline', function() diff --git a/test/functional/ui/output_spec.lua b/test/functional/ui/output_spec.lua index 0dd1f0325c..7b93b74eac 100644 --- a/test/functional/ui/output_spec.lua +++ b/test/functional/ui/output_spec.lua @@ -225,8 +225,8 @@ describe("shell command :!", function() å | ref: å̲ | 1: å̲ | - 2: å ̲ | - 3: å ̲ | + 2: å ̲ | + 3: å ̲ | | {3:Press ENTER or type command to continue}^ | ]]) -- cgit