Treat U+3000 and U+FF5E as CJK punctuation-equivalent in Markdown prose wrap (#18656)

Co-authored-by: fisker <lionkay@gmail.com>

changelog_unreleased/javascript/18594.md

@@ -2,12 +2,13 @@
 
 Some tools treat these spaces as meaningful.
 
-```bash
-await require('prettier').format('/**\n * With 2 spaces   \n */', {parser: 'babel'});
+```js
+await prettier.format("/**\n * With 2 spaces   \n */", { parser: "babel" });
 
-# Prettier stable
+/* Prettier stable */
 // -> '/**\n * With 2 spaces\n */\n'
 
-# Prettier main
+/* Prettier main */
 // -> '/**\n * With 2 spaces  \n */\n'
+//                          ^^ space preserved
 ```

changelog_unreleased/markdown/18656.md

@@ -0,0 +1,17 @@
+#### Treat U+3000 as CJK punctuation-equivalent and make sure to recognize U+FF5E as CJK punctuation again in Markdown prose wrap (#18656 by @tats-u)
+
+We improved the text wrapping in Markdown written in Chinese and Japanese by treating some non Unicode punctuation characters as equivalent to CJK punctuation. A part of this was assumed to have been introduced in 3.5.0 ([#16832](https://github.com/prettier/prettier/pull/16832)), but it was not fully implemented and not noticed until now due to insufficient test cases.
+
+```js
+await prettier.format(
+  "U+3000\u{3000}\nU+301C\u{301C}\nU+FF5E\u{FF5E}\nU+1F221\u{1F221} 测试 Test テスト\n",
+  { parser: "markdown", proseWrap: "always" },
+);
+
+/* Prettier stable */
+// -> 'U+3000　 U+301C〜U+FF5E～ U+1F221🈡 测试 Test テスト\n'
+//           　^ extra space  　^ extra space
+
+/* Prettier main */
+// -> 'U+3000　U+301C〜U+FF5E～U+1F221🈡 测试 Test テスト\n'
+```

src/language-markdown/constants.evaluate.js

@@ -74,10 +74,13 @@ const unicodePunctuationClasses = [
 
 const PUNCTUATION_REGEXP = new RegExp(
   `(?:${[
-    new Charset(...asciiPunctuationCharacters).toRegExp("u").source,
+    new Charset(
+      ...asciiPunctuationCharacters,
+      "\u{3000}", // ASCII space around it is excessive. https://bugzilla.mozilla.org/show_bug.cgi?id=1950321
+      "\u{ff5e}", // Used as a substitute for U+301C in Windows. https://bugzilla.mozilla.org/show_bug.cgi?id=1941096
+    ).toRegExp("u").source,
     ...unicodePunctuationClasses.map(
       (charset) => String.raw`\p{General_Category=${charset}}`,
-      "\u{ff5e}", // Used as a substitute for U+301C in Windows
     ),
   ].join("|")})`,
   "u",

tests/format/markdown/splitCjkText/__snapshots__/format.test.js.snap

@@ -296,10 +296,6 @@ Go
 ・
 Rust
 
-U+301C〜
-U+FF5E～
-U+1F221🈡
-
 =====================================output=====================================
 日本語、にほんご。汉语, 中文. 日本語，にほんご．English
 words!? 漢字！汉字？「セリフ」(括弧) 文字（括弧）文字【括弧】日本語English
@@ -316,7 +312,5 @@ words!? 漢字！汉字？「セリフ」(括弧) 文字（括弧）文字【括
 
 C言語・C++・Go・Rust
 
-U+301C〜U+FF5E～U+1F221🈡
-
 ================================================================================
 `;

tests/format/markdown/splitCjkText/format.test.js

@@ -1,30 +1,47 @@
-const code = Array.from({ length: 6 })
-  // less "文" <= [..., "文...文 ", "文...文", ...] => more "文"
-  .map((_, i) => ["文".repeat(39 + (i >> 1)), i & 1 ? "" : " "])
-  // ["<Line #1>", "<L2>", ...]
-  .reduce(
-    (previousInputLines, [base, space]) => [
-      ...previousInputLines,
-      `${base}${space}\n`,
-    ],
-    [],
-  )
-  .join("");
-// (39 + 40 + 41) * 2 === 40 * 3 * 2 === 40 * 6
-// lineWidth is 80 and "文" is double-width
-// This is being saved for after Chrome and Safari bug is fixed.
-const output = /* `${"文".repeat(40)}\n`.repeat(6) */ `${"文".repeat(40 * 6)}\n`;
+const snippets = [
+  {
+    name: "Should ignore single trailing space after han before New Line before han",
+    code: Array.from({ length: 6 })
+      // less "文" <= [..., "文...文 ", "文...文", ...] => more "文"
+      .map((_, i) => ["文".repeat(39 + (i >> 1)), i & 1 ? "" : " "])
+      // ["<Line #1>", "<L2>", ...]
+      .reduce(
+        (previousInputLines, [base, space]) => [
+          ...previousInputLines,
+          `${base}${space}\n`,
+        ],
+        [],
+      )
+      .join(""),
+    // (39 + 40 + 41) * 2 === 40 * 3 * 2 === 40 * 6
+    // lineWidth is 80 and "文" is double-width
+    // This is being saved for after Chrome and Safari bug is fixed.
+    output: /* `${"文".repeat(40)}\n`.repeat(6) */ `${"文".repeat(40 * 6)}\n`,
+  },
+];
+
+{
+  // Switches to insert-space (2.x-style) mode
+  const afterword = "测试 Test テスト Test";
+  const punctuationLike = [
+    0x3000, // Fullwidth Space but should be treated like punctuation
+    0x301c, // Genuine CJ(K) punctuation
+    0xff5e, // Nearly equivalent to U+301C but its Unicode category is "Sm"
+    0x1f221, // "end" symbol. Should be placed at end
+  ];
+  const code = `${punctuationLike.map((cp) => `U+${cp.toString(16).toUpperCase()}${String.fromCodePoint(cp)}`).join("\n")} ${afterword}\n`;
+  snippets.push({
+    name: "Should remove newline around CJ(K) punctuation(-like)",
+    code,
+    // Note: replaceAll is not supported in Node 14.
+    output: `${code.replace(/\n/g, "")}\n`,
+  });
+}
 
 runFormatTest(
   {
     importMeta: import.meta,
-    snippets: [
-      {
-        code,
-        name: "Should ignore single trailing space after han before New Line before han",
-        output,
-      },
-    ],
+    snippets,
   },
   ["markdown"],
   { proseWrap: "always" },

tests/format/markdown/splitCjkText/symbolSpaceNewLine.md

@@ -66,7 +66,3 @@ C++
 Go
 ・
 Rust
-
-U+301C〜
-U+FF5E～
-U+1F221🈡