diff --git a/changelog_unreleased/javascript/18594.md b/changelog_unreleased/javascript/18594.md index 268305beb..46b9af142 100644 --- a/changelog_unreleased/javascript/18594.md +++ b/changelog_unreleased/javascript/18594.md @@ -2,12 +2,13 @@ Some tools treat these spaces as meaningful. -```bash -await require('prettier').format('/**\n * With 2 spaces \n */', {parser: 'babel'}); +```js +await prettier.format("/**\n * With 2 spaces \n */", { parser: "babel" }); -# Prettier stable +/* Prettier stable */ // -> '/**\n * With 2 spaces\n */\n' -# Prettier main +/* Prettier main */ // -> '/**\n * With 2 spaces \n */\n' +// ^^ space preserved ``` diff --git a/changelog_unreleased/markdown/18656.md b/changelog_unreleased/markdown/18656.md new file mode 100644 index 000000000..e00634801 --- /dev/null +++ b/changelog_unreleased/markdown/18656.md @@ -0,0 +1,17 @@ +#### Treat U+3000 as CJK punctuation-equivalent and make sure to recognize U+FF5E as CJK punctuation again in Markdown prose wrap (#18656 by @tats-u) + +We improved the text wrapping in Markdown written in Chinese and Japanese by treating some non Unicode punctuation characters as equivalent to CJK punctuation. A part of this was assumed to have been introduced in 3.5.0 ([#16832](https://github.com/prettier/prettier/pull/16832)), but it was not fully implemented and not noticed until now due to insufficient test cases. + +```js +await prettier.format( + "U+3000\u{3000}\nU+301C\u{301C}\nU+FF5E\u{FF5E}\nU+1F221\u{1F221} 测试 Test テスト\n", + { parser: "markdown", proseWrap: "always" }, +); + +/* Prettier stable */ +// -> 'U+3000  U+301C〜U+FF5E~ U+1F221🈡 测试 Test テスト\n' +//  ^ extra space  ^ extra space + +/* Prettier main */ +// -> 'U+3000 U+301C〜U+FF5E~U+1F221🈡 测试 Test テスト\n' +``` diff --git a/src/language-markdown/constants.evaluate.js b/src/language-markdown/constants.evaluate.js index 4b5ab03de..24c970ee3 100644 --- a/src/language-markdown/constants.evaluate.js +++ b/src/language-markdown/constants.evaluate.js @@ -74,10 +74,13 @@ const unicodePunctuationClasses = [ const PUNCTUATION_REGEXP = new RegExp( `(?:${[ - new Charset(...asciiPunctuationCharacters).toRegExp("u").source, + new Charset( + ...asciiPunctuationCharacters, + "\u{3000}", // ASCII space around it is excessive. https://bugzilla.mozilla.org/show_bug.cgi?id=1950321 + "\u{ff5e}", // Used as a substitute for U+301C in Windows. https://bugzilla.mozilla.org/show_bug.cgi?id=1941096 + ).toRegExp("u").source, ...unicodePunctuationClasses.map( (charset) => String.raw`\p{General_Category=${charset}}`, - "\u{ff5e}", // Used as a substitute for U+301C in Windows ), ].join("|")})`, "u", diff --git a/tests/format/markdown/splitCjkText/__snapshots__/format.test.js.snap b/tests/format/markdown/splitCjkText/__snapshots__/format.test.js.snap index 2dff975e3..c36d8fcf5 100644 --- a/tests/format/markdown/splitCjkText/__snapshots__/format.test.js.snap +++ b/tests/format/markdown/splitCjkText/__snapshots__/format.test.js.snap @@ -296,10 +296,6 @@ Go ・ Rust -U+301C〜 -U+FF5E~ -U+1F221🈡 - =====================================output===================================== 日本語、にほんご。汉语, 中文. 日本語,にほんご.English words!? 漢字!汉字?「セリフ」(括弧) 文字(括弧)文字【括弧】日本語English @@ -316,7 +312,5 @@ words!? 漢字!汉字?「セリフ」(括弧) 文字(括弧)文字【括 C言語・C++・Go・Rust -U+301C〜U+FF5E~U+1F221🈡 - ================================================================================ `; diff --git a/tests/format/markdown/splitCjkText/format.test.js b/tests/format/markdown/splitCjkText/format.test.js index 3e9f1215b..cd2dbc533 100644 --- a/tests/format/markdown/splitCjkText/format.test.js +++ b/tests/format/markdown/splitCjkText/format.test.js @@ -1,30 +1,47 @@ -const code = Array.from({ length: 6 }) - // less "文" <= [..., "文...文 ", "文...文", ...] => more "文" - .map((_, i) => ["文".repeat(39 + (i >> 1)), i & 1 ? "" : " "]) - // ["", "", ...] - .reduce( - (previousInputLines, [base, space]) => [ - ...previousInputLines, - `${base}${space}\n`, - ], - [], - ) - .join(""); -// (39 + 40 + 41) * 2 === 40 * 3 * 2 === 40 * 6 -// lineWidth is 80 and "文" is double-width -// This is being saved for after Chrome and Safari bug is fixed. -const output = /* `${"文".repeat(40)}\n`.repeat(6) */ `${"文".repeat(40 * 6)}\n`; +const snippets = [ + { + name: "Should ignore single trailing space after han before New Line before han", + code: Array.from({ length: 6 }) + // less "文" <= [..., "文...文 ", "文...文", ...] => more "文" + .map((_, i) => ["文".repeat(39 + (i >> 1)), i & 1 ? "" : " "]) + // ["", "", ...] + .reduce( + (previousInputLines, [base, space]) => [ + ...previousInputLines, + `${base}${space}\n`, + ], + [], + ) + .join(""), + // (39 + 40 + 41) * 2 === 40 * 3 * 2 === 40 * 6 + // lineWidth is 80 and "文" is double-width + // This is being saved for after Chrome and Safari bug is fixed. + output: /* `${"文".repeat(40)}\n`.repeat(6) */ `${"文".repeat(40 * 6)}\n`, + }, +]; + +{ + // Switches to insert-space (2.x-style) mode + const afterword = "测试 Test テスト Test"; + const punctuationLike = [ + 0x3000, // Fullwidth Space but should be treated like punctuation + 0x301c, // Genuine CJ(K) punctuation + 0xff5e, // Nearly equivalent to U+301C but its Unicode category is "Sm" + 0x1f221, // "end" symbol. Should be placed at end + ]; + const code = `${punctuationLike.map((cp) => `U+${cp.toString(16).toUpperCase()}${String.fromCodePoint(cp)}`).join("\n")} ${afterword}\n`; + snippets.push({ + name: "Should remove newline around CJ(K) punctuation(-like)", + code, + // Note: replaceAll is not supported in Node 14. + output: `${code.replace(/\n/g, "")}\n`, + }); +} runFormatTest( { importMeta: import.meta, - snippets: [ - { - code, - name: "Should ignore single trailing space after han before New Line before han", - output, - }, - ], + snippets, }, ["markdown"], { proseWrap: "always" }, diff --git a/tests/format/markdown/splitCjkText/symbolSpaceNewLine.md b/tests/format/markdown/splitCjkText/symbolSpaceNewLine.md index bceb8cb66..7a1db142a 100644 --- a/tests/format/markdown/splitCjkText/symbolSpaceNewLine.md +++ b/tests/format/markdown/splitCjkText/symbolSpaceNewLine.md @@ -66,7 +66,3 @@ C++ Go ・ Rust - -U+301C〜 -U+FF5E~ -U+1F221🈡