From 88c210e844ff5503faafd0fe23a2a4d915b73a70 Mon Sep 17 00:00:00 2001 From: Tatsunori Uchino Date: Mon, 16 Dec 2024 17:26:22 +0900 Subject: [PATCH] Treat U+FF5E as CJK punctuation (#16832) --- changelog_unreleased/markdown/16832.md | 22 +++++++++++++++++++ cspell.json | 1 + src/language-markdown/constants.evaluate.js | 1 + .../__snapshots__/format.test.js.snap | 7 +++++- .../splitCjkText/symbolSpaceNewLine.md | 5 ++++- 5 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 changelog_unreleased/markdown/16832.md diff --git a/changelog_unreleased/markdown/16832.md b/changelog_unreleased/markdown/16832.md new file mode 100644 index 000000000..ce77e3805 --- /dev/null +++ b/changelog_unreleased/markdown/16832.md @@ -0,0 +1,22 @@ +#### U+FF5E as CJK punctuation (#16832 by @tats-u) + +U+FF5E FULLWIDTH TILDE (~) is commonly used as a substitute for U+301C WAVE DASH (〜) in Windows for Japanese. Full width alphabets are less used in Markdown documents comparing to other types of documents (e.g. Microsoft Office documents), and the full width tilde is much less used as this purpose compared to full width alphabets and digits. Therefore, we can assume that the full width tilde in Markdown documents in the wild are a alternative form of the wave dash and a part of CJK punctuation. + + +```markdown + +a 字 a 字 a 字 +60~ +100点 +60〜 +100点 + + +a 字 a 字 a 字 60~ 100点 60〜10点 + + + +a 字 a 字 a 字 60~10点 60〜100点 +``` + +The first symbol between 60 and 100 in the above example is U+FF5E FULLWIDTH TILDE (~) and the second one is U+301C WAVE DASH (〜). diff --git a/cspell.json b/cspell.json index c35b8b1b1..612e78e55 100644 --- a/cspell.json +++ b/cspell.json @@ -98,6 +98,7 @@ "Friedly", "frobble", "ftest", + "fullwidth", "Georgii", "gettin", "githook", diff --git a/src/language-markdown/constants.evaluate.js b/src/language-markdown/constants.evaluate.js index 816ce3c5f..6166202bb 100644 --- a/src/language-markdown/constants.evaluate.js +++ b/src/language-markdown/constants.evaluate.js @@ -84,6 +84,7 @@ const PUNCTUATION_REGEXP = new RegExp( new Charset(...asciiPunctuationCharacters).toRegExp("u").source, ...unicodePunctuationClasses.map( (charset) => `\\p{General_Category=${charset}}`, + "\u{ff5e}", // Used as a substitute for U+301C in Windows ), ].join("|")})`, "u", diff --git a/tests/format/markdown/splitCjkText/__snapshots__/format.test.js.snap b/tests/format/markdown/splitCjkText/__snapshots__/format.test.js.snap index e587af369..f7b3d631a 100644 --- a/tests/format/markdown/splitCjkText/__snapshots__/format.test.js.snap +++ b/tests/format/markdown/splitCjkText/__snapshots__/format.test.js.snap @@ -283,7 +283,6 @@ English [ウ ィキペディア]: https://ja.wikipedia.org/ - C言 語 ・ @@ -293,6 +292,10 @@ Go ・ Rust +U+301C〜 +U+FF5E~ +U+1F221🈡 + =====================================output===================================== 日本語、にほんご。汉语, 中文. 日本語,にほんご.English words!? 漢字!汉字?「セリフ」(括弧) 文字(括弧)文字【括弧】日本語English @@ -309,5 +312,7 @@ words!? 漢字!汉字?「セリフ」(括弧) 文字(括弧)文字【括 C言語・C++・Go・Rust +U+301C〜U+FF5E~U+1F221🈡 + ================================================================================ `; diff --git a/tests/format/markdown/splitCjkText/symbolSpaceNewLine.md b/tests/format/markdown/splitCjkText/symbolSpaceNewLine.md index 013ca9ba4..bceb8cb66 100644 --- a/tests/format/markdown/splitCjkText/symbolSpaceNewLine.md +++ b/tests/format/markdown/splitCjkText/symbolSpaceNewLine.md @@ -58,7 +58,6 @@ English [ウ ィキペディア]: https://ja.wikipedia.org/ - C言 語 ・ @@ -67,3 +66,7 @@ C++ Go ・ Rust + +U+301C〜 +U+FF5E~ +U+1F221🈡