Preserve non-ASCII whitespace characters in HTML (#8137)

* Add html whitespace tests

* `U+0005` -> `U+2005`

* Update tests

* Fix whitespace

* Add changelog

* Another space

* Restore `non-breaking` check

* Add space between tests

* Extract all space related codes to utils

* Fix indent detect

* Update changelog

* Another space

* Rename

* Rename

* Rename

* Use `htmlTrimFirstLine` and `htmlTrimEnd`

* Assert with output

* `htmlTrimPreserveIndentation`

* Update changelog

* Fix another space

changelog_unreleased/html/pr-8137.md

@@ -0,0 +1,20 @@
+#### Preserve non-ASCII whitespace characters in HTML ([#8137](https://github.com/prettier/prettier/pull/8137) by [@fisker](https://github.com/fisker))
+
+Non-ASCII whitespace characters like `U+00A0` `U+2005` etc. are not considered whitespace in html, they should not be removed.
+
+```js
+// Prettier stable
+[...require("prettier").format("<i> \u2005 </i>", { parser: "html" })]
+  .slice(3, -5)
+  .map((c) => c.charCodeAt(0).toString(16));
+
+// -> [ '20' ]
+// `U+2005` is removed
+
+// Prettier master
+[...require("prettier").format("<i> \u2005 </i>", { parser: "html" })]
+  .slice(3, -5)
+  .map((c) => c.charCodeAt(0).toString(16));
+
+// -> [ '20', '2005', '20' ]
+```

src/language-html/preprocess.js

@@ -1,6 +1,9 @@
 "use strict";
 
 const {
+  htmlTrim,
+  getLeadingAndTrailingHtmlWhitespace,
+  hasHtmlWhitespace,
   canHaveInterpolation,
   getNodeCssStyleDisplay,
   isDanglingSpaceSensitiveNode,
@@ -176,8 +179,7 @@ function mergeSimpleElementIntoText(ast /*, options */) {
     node.attrs.length === 0 &&
     node.children.length === 1 &&
     node.firstChild.type === "text" &&
-    // \xA0: non-breaking whitespace
-    !/[^\S\xA0]/.test(node.children[0].value) &&
+    !hasHtmlWhitespace(node.children[0].value) &&
     !node.firstChild.hasLeadingSpaces &&
     !node.firstChild.hasTrailingSpaces &&
     node.isLeadingSpaceSensitive &&
@@ -313,7 +315,7 @@ function extractWhitespaces(ast /*, options*/) {
       node.children.length === 0 ||
       (node.children.length === 1 &&
         node.children[0].type === "text" &&
-        node.children[0].value.trim().length === 0)
+        htmlTrim(node.children[0].value).length === 0)
     ) {
       return node.clone({
         children: [],
@@ -336,11 +338,13 @@ function extractWhitespaces(ast /*, options*/) {
 
           const localChildren = [];
 
-          const [, leadingSpaces, text, trailingSpaces] = child.value.match(
-            /^(\s*)([\S\s]*?)(\s*)$/
-          );
+          const {
+            leadingWhitespace,
+            text,
+            trailingWhitespace,
+          } = getLeadingAndTrailingHtmlWhitespace(child.value);
 
-          if (leadingSpaces) {
+          if (leadingWhitespace) {
             localChildren.push({ type: TYPE_WHITESPACE });
           }
 
@@ -351,13 +355,13 @@ function extractWhitespaces(ast /*, options*/) {
               type: "text",
               value: text,
               sourceSpan: new ParseSourceSpan(
-                child.sourceSpan.start.moveBy(leadingSpaces.length),
-                child.sourceSpan.end.moveBy(-trailingSpaces.length)
+                child.sourceSpan.start.moveBy(leadingWhitespace.length),
+                child.sourceSpan.end.moveBy(-trailingWhitespace.length)
               ),
             });
           }
 
-          if (trailingSpaces) {
+          if (trailingWhitespace) {
             localChildren.push({ type: TYPE_WHITESPACE });
           }
 

src/language-html/printer-html.js

@@ -20,6 +20,8 @@ const {
   softline,
 } = builders;
 const {
+  htmlTrimPreserveIndentation,
+  splitByHtmlWhitespace,
   countChars,
   countParents,
   dedentString,
@@ -285,7 +287,7 @@ function genericPrint(path, options, print) {
                           node.isWhitespaceSensitive &&
                           node.isIndentationSensitive)) &&
                       new RegExp(
-                        `\\n\\s{${
+                        `\\n[\\t ]{${
                           options.tabWidth *
                           countParents(
                             path,
@@ -941,11 +943,10 @@ function getTextValueParts(node, value = node.value) {
     ? node.parent.isIndentationSensitive
       ? replaceEndOfLineWith(value, literalline)
       : replaceEndOfLineWith(
-          dedentString(value.replace(/^\s*?\n|\n\s*?$/g, "")),
+          dedentString(htmlTrimPreserveIndentation(value)),
           hardline
         )
-    : // https://infra.spec.whatwg.org/#ascii-whitespace
-      join(line, value.split(/[\t\n\f\r ]+/)).parts;
+    : join(line, splitByHtmlWhitespace(value)).parts;
 }
 
 function printEmbeddedAttributeValue(node, originalTextToDoc, options) {

src/language-html/utils.js

@@ -14,6 +14,29 @@ const htmlElementAttributes = require("html-element-attributes");
 const HTML_TAGS = arrayToMap(htmlTagNames);
 const HTML_ELEMENT_ATTRIBUTES = mapObject(htmlElementAttributes, arrayToMap);
 
+// https://infra.spec.whatwg.org/#ascii-whitespace
+const HTML_WHITESPACE = new Set(["\t", "\n", "\f", "\r", " "]);
+const htmlTrimStart = (string) => string.replace(/^[\t\n\f\r ]+/, "");
+const htmlTrimEnd = (string) => string.replace(/[\t\n\f\r ]+$/, "");
+const htmlTrim = (string) => htmlTrimStart(htmlTrimEnd(string));
+const htmlTrimLeadingBlankLines = (string) =>
+  string.replace(/^[\t\f\r ]*?\n/g, "");
+const htmlTrimPreserveIndentation = (string) =>
+  htmlTrimLeadingBlankLines(htmlTrimEnd(string));
+const splitByHtmlWhitespace = (string) => string.split(/[\t\n\f\r ]+/);
+const getLeadingHtmlWhitespace = (string) => string.match(/^[\t\n\f\r ]*/)[0];
+const getLeadingAndTrailingHtmlWhitespace = (string) => {
+  const [, leadingWhitespace, text, trailingWhitespace] = string.match(
+    /^([\t\n\f\r ]*)([\S\s]*?)([\t\n\f\r ]*)$/
+  );
+  return {
+    leadingWhitespace,
+    trailingWhitespace,
+    text,
+  };
+};
+const hasHtmlWhitespace = (string) => /[\t\n\f\r ]/.test(string);
+
 function arrayToMap(array) {
   const map = Object.create(null);
   for (const value of array) {
@@ -544,11 +567,11 @@ function getMinIndentation(text) {
       continue;
     }
 
-    if (/\S/.test(lineText[0])) {
+    if (!HTML_WHITESPACE.has(lineText[0])) {
       return 0;
     }
 
-    const indentation = lineText.match(/^\s*/)[0].length;
+    const indentation = getLeadingHtmlWhitespace(lineText).length;
 
     if (lineText.length === indentation) {
       continue;
@@ -642,6 +665,11 @@ function isVueCustomBlock(node, options) {
 module.exports = {
   HTML_ELEMENT_ATTRIBUTES,
   HTML_TAGS,
+  htmlTrim,
+  htmlTrimPreserveIndentation,
+  splitByHtmlWhitespace,
+  hasHtmlWhitespace,
+  getLeadingAndTrailingHtmlWhitespace,
   canHaveInterpolation,
   countChars,
   countParents,

tests/html_whitespace/__snapshots__/jsfmt.spec.js.snap

@@ -222,6 +222,273 @@ printWidth: 80
 ================================================================================
 `;
 
+exports[`snippet: #18 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<div> </div>
+=====================================output=====================================
+<div> </div>
+
+================================================================================
+`;
+
+exports[`snippet: #19 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<div>          </div>
+=====================================output=====================================
+<div> </div>
+
+================================================================================
+`;
+
+exports[`snippet: #20 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<div>           </div>
+=====================================output=====================================
+<div>  </div>
+
+================================================================================
+`;
+
+exports[`snippet: #21 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<div>                   </div>
+=====================================output=====================================
+<div>   </div>
+
+================================================================================
+`;
+
+exports[`snippet: #22 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<span> </span>
+=====================================output=====================================
+<span> </span>
+
+================================================================================
+`;
+
+exports[`snippet: #23 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<span>          </span>
+=====================================output=====================================
+<span>   </span>
+
+================================================================================
+`;
+
+exports[`snippet: #24 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<span>           </span>
+=====================================output=====================================
+<span>    </span>
+
+================================================================================
+`;
+
+exports[`snippet: #25 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<span>                   </span>
+=====================================output=====================================
+<span>     </span>
+
+================================================================================
+`;
+
+exports[`snippet: #26 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<img/> <img/>
+=====================================output=====================================
+<img /> <img />
+
+================================================================================
+`;
+
+exports[`snippet: #27 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<img/>          <img/>
+=====================================output=====================================
+<img />   <img />
+
+================================================================================
+`;
+
+exports[`snippet: #28 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<img/>           <img/>
+=====================================output=====================================
+<img />    <img />
+
+================================================================================
+`;
+
+exports[`snippet: #29 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<img/>                   <img/>
+=====================================output=====================================
+<img />     <img />
+
+================================================================================
+`;
+
+exports[`snippet: #30 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<i />   |   <i />
+=====================================output=====================================
+<i />   |   <i />
+
+================================================================================
+`;
+
+exports[`snippet: #31 format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<p><span>X</span>   or   <span>Y</span></p><p>X   or   Y</p>
+=====================================output=====================================
+<p><span>X</span>   or   <span>Y</span></p>
+<p>X   or   Y</p>
+
+================================================================================
+`;
+
+exports[`snippet: \`U+2005\` should format like \`U+005F\` not like \`U+0020\` format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<!-- U+2005 -->
+<div>before<span> </span>afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter</div>
+<!-- U+005F -->
+<div>before<span>_</span>afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter</div>
+<!-- U+0020 -->
+<div>before<span> </span>afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter</div>
+=====================================output=====================================
+<!-- U+2005 -->
+<div>
+  before<span> </span>afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter
+</div>
+<!-- U+005F -->
+<div>
+  before<span>_</span>afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter
+</div>
+<!-- U+0020 -->
+<div>
+  before<span
+  > </span
+  >afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter
+</div>
+
+================================================================================
+`;
+
+exports[`snippet: \`U+2005\` should indent like \`U+005F\` not like \`U+0020\` format 1`] = `
+====================================options=====================================
+parsers: ["html"]
+printWidth: 80
+                                                                                | printWidth
+=====================================input======================================
+<!-- U+2005 -->
+<script type="text/unknown" lang="unknown">
+        // comment
+          // comment
+          // comment
+          // comment
+</script>
+<!-- U+005F -->
+<script type="text/unknown" lang="unknown">
+   _    // comment
+          // comment
+          // comment
+          // comment
+</script>
+<!-- U+0020 -->
+<script type="text/unknown" lang="unknown">
+        // comment
+          // comment
+          // comment
+          // comment
+</script>
+=====================================output=====================================
+<!-- U+2005 -->
+<script type="text/unknown" lang="unknown">
+       // comment
+         // comment
+         // comment
+         // comment
+</script>
+<!-- U+005F -->
+<script type="text/unknown" lang="unknown">
+  _    // comment
+         // comment
+         // comment
+         // comment
+</script>
+<!-- U+0020 -->
+<script type="text/unknown" lang="unknown">
+  // comment
+    // comment
+    // comment
+    // comment
+</script>
+
+================================================================================
+`;
+
 exports[`surrounding-linebreak.html format 1`] = `
 ====================================options=====================================
 parsers: ["html"]

tests/html_whitespace/jsfmt.spec.js

@@ -1 +1,130 @@
-run_spec(__dirname, ["html"]);
+const { outdent } = require("outdent");
+
+run_spec(
+  {
+    dirname: __dirname,
+    snippets: [
+      ...[
+        // https://developer.mozilla.org/en-US/docs/Glossary/Whitespace#In_HTML
+        // single
+        "\u0009",
+        "\u000A",
+        "\u000C",
+        "\u000D",
+        "\u0020",
+
+        // many
+        "\u0009\u000A\u000C\u000D\u0020",
+      ].map((textContent) => ({
+        code: `<div>${textContent}</div>`,
+        name: "should be empty",
+        output: "<div></div>\n",
+      })),
+
+      ...[
+        // single
+        "\u0009",
+        "\u000A",
+        "\u000C",
+        "\u000D",
+        "\u0020",
+
+        // many
+        "\u0009\u000A\u000C\u000D\u0020",
+      ].map((textContent) => ({
+        code: `<span>${textContent}</span>`,
+        name: "should keep one space",
+        output: "<span> </span>\n",
+      })),
+
+      ...[
+        // single
+        "\u0009",
+        "\u000A",
+        "\u000C",
+        "\u000D",
+        "\u0020",
+      ].map((textContent) => ({
+        code: `<img/>${textContent}<img/>`,
+        name: "between",
+        output: "<img /> <img />\n",
+      })),
+
+      {
+        code: "<img/>\u0009\u000A\u000C\u000D\u0020<img/>",
+        output: "<img />\n\n<img />\n",
+      },
+
+      // non-space
+      ...[
+        "\u2005",
+        " \u2005        ",
+        "        \u2005\u2005 ",
+        "        \u2005        \u2005 ",
+      ].map((textContent) => `<div>${textContent}</div>`),
+
+      ...[
+        "\u2005",
+        " \u2005        ",
+        "        \u2005\u2005 ",
+        "        \u2005        \u2005 ",
+      ].map((textContent) => `<span>${textContent}</span>`),
+
+      ...[
+        "\u2005",
+        " \u2005        ",
+        "        \u2005\u2005 ",
+        "        \u2005        \u2005 ",
+      ].map((textContent) => `<img/>${textContent}<img/>`),
+
+      // #7103 minimal reproduction
+      "<i /> \u2005 | \u2005 <i />",
+
+      // #7103
+      "<p><span>X</span> \u2005 or \u2005 <span>Y</span></p><p>X \u2005 or \u2005 Y</p>",
+
+      // This test maybe not good, `U+2005` there don't make sense,
+      // but the node has to be `whitespaceSensitive` and `indentationSensitive`,
+      // to make the `whitespace check logic` work.
+      {
+        name: "`U+2005` should indent like `U+005F` not like `U+0020`",
+        code: outdent`
+          <!-- U+2005 -->
+          <script type="text/unknown" lang="unknown">
+             \u2005    // comment
+                    // comment
+                    // comment
+                    // comment
+          </script>
+          <!-- U+005F -->
+          <script type="text/unknown" lang="unknown">
+             \u005F    // comment
+                    // comment
+                    // comment
+                    // comment
+          </script>
+          <!-- U+0020 -->
+          <script type="text/unknown" lang="unknown">
+             \u0020    // comment
+                    // comment
+                    // comment
+                    // comment
+          </script>
+        `,
+      },
+
+      {
+        name: "`U+2005` should format like `U+005F` not like `U+0020`",
+        code: outdent`
+          <!-- U+2005 -->
+          <div>before<span>\u2005</span>afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter</div>
+          <!-- U+005F -->
+          <div>before<span>\u005F</span>afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter</div>
+          <!-- U+0020 -->
+          <div>before<span>\u0020</span>afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter</div>
+        `,
+      },
+    ],
+  },
+  ["html"]
+);