From f83eb9d5efe5bb017679635ecdbf00d06588552e Mon Sep 17 00:00:00 2001 From: fisker Cheung Date: Sat, 25 Apr 2020 21:56:28 +0800 Subject: [PATCH] Preserve non-ASCII whitespace characters in HTML (#8137) * Add html whitespace tests * `U+0005` -> `U+2005` * Update tests * Fix whitespace * Add changelog * Another space * Restore `non-breaking` check * Add space between tests * Extract all space related codes to utils * Fix indent detect * Update changelog * Another space * Rename * Rename * Rename * Use `htmlTrimFirstLine` and `htmlTrimEnd` * Assert with output * `htmlTrimPreserveIndentation` * Update changelog * Fix another space --- changelog_unreleased/html/pr-8137.md | 20 ++ src/language-html/preprocess.js | 24 +- src/language-html/printer-html.js | 9 +- src/language-html/utils.js | 32 ++- .../__snapshots__/jsfmt.spec.js.snap | 267 ++++++++++++++++++ tests/html_whitespace/jsfmt.spec.js | 131 ++++++++- 6 files changed, 466 insertions(+), 17 deletions(-) create mode 100644 changelog_unreleased/html/pr-8137.md diff --git a/changelog_unreleased/html/pr-8137.md b/changelog_unreleased/html/pr-8137.md new file mode 100644 index 000000000..d602bac34 --- /dev/null +++ b/changelog_unreleased/html/pr-8137.md @@ -0,0 +1,20 @@ +#### Preserve non-ASCII whitespace characters in HTML ([#8137](https://github.com/prettier/prettier/pull/8137) by [@fisker](https://github.com/fisker)) + +Non-ASCII whitespace characters like `U+00A0` `U+2005` etc. are not considered whitespace in html, they should not be removed. + +```js +// Prettier stable +[...require("prettier").format(" \u2005 ", { parser: "html" })] + .slice(3, -5) + .map((c) => c.charCodeAt(0).toString(16)); + +// -> [ '20' ] +// `U+2005` is removed + +// Prettier master +[...require("prettier").format(" \u2005 ", { parser: "html" })] + .slice(3, -5) + .map((c) => c.charCodeAt(0).toString(16)); + +// -> [ '20', '2005', '20' ] +``` diff --git a/src/language-html/preprocess.js b/src/language-html/preprocess.js index 81305da69..a318fce29 100644 --- a/src/language-html/preprocess.js +++ b/src/language-html/preprocess.js @@ -1,6 +1,9 @@ "use strict"; const { + htmlTrim, + getLeadingAndTrailingHtmlWhitespace, + hasHtmlWhitespace, canHaveInterpolation, getNodeCssStyleDisplay, isDanglingSpaceSensitiveNode, @@ -176,8 +179,7 @@ function mergeSimpleElementIntoText(ast /*, options */) { node.attrs.length === 0 && node.children.length === 1 && node.firstChild.type === "text" && - // \xA0: non-breaking whitespace - !/[^\S\xA0]/.test(node.children[0].value) && + !hasHtmlWhitespace(node.children[0].value) && !node.firstChild.hasLeadingSpaces && !node.firstChild.hasTrailingSpaces && node.isLeadingSpaceSensitive && @@ -313,7 +315,7 @@ function extractWhitespaces(ast /*, options*/) { node.children.length === 0 || (node.children.length === 1 && node.children[0].type === "text" && - node.children[0].value.trim().length === 0) + htmlTrim(node.children[0].value).length === 0) ) { return node.clone({ children: [], @@ -336,11 +338,13 @@ function extractWhitespaces(ast /*, options*/) { const localChildren = []; - const [, leadingSpaces, text, trailingSpaces] = child.value.match( - /^(\s*)([\S\s]*?)(\s*)$/ - ); + const { + leadingWhitespace, + text, + trailingWhitespace, + } = getLeadingAndTrailingHtmlWhitespace(child.value); - if (leadingSpaces) { + if (leadingWhitespace) { localChildren.push({ type: TYPE_WHITESPACE }); } @@ -351,13 +355,13 @@ function extractWhitespaces(ast /*, options*/) { type: "text", value: text, sourceSpan: new ParseSourceSpan( - child.sourceSpan.start.moveBy(leadingSpaces.length), - child.sourceSpan.end.moveBy(-trailingSpaces.length) + child.sourceSpan.start.moveBy(leadingWhitespace.length), + child.sourceSpan.end.moveBy(-trailingWhitespace.length) ), }); } - if (trailingSpaces) { + if (trailingWhitespace) { localChildren.push({ type: TYPE_WHITESPACE }); } diff --git a/src/language-html/printer-html.js b/src/language-html/printer-html.js index 7479997e6..00000d135 100644 --- a/src/language-html/printer-html.js +++ b/src/language-html/printer-html.js @@ -20,6 +20,8 @@ const { softline, } = builders; const { + htmlTrimPreserveIndentation, + splitByHtmlWhitespace, countChars, countParents, dedentString, @@ -285,7 +287,7 @@ function genericPrint(path, options, print) { node.isWhitespaceSensitive && node.isIndentationSensitive)) && new RegExp( - `\\n\\s{${ + `\\n[\\t ]{${ options.tabWidth * countParents( path, @@ -941,11 +943,10 @@ function getTextValueParts(node, value = node.value) { ? node.parent.isIndentationSensitive ? replaceEndOfLineWith(value, literalline) : replaceEndOfLineWith( - dedentString(value.replace(/^\s*?\n|\n\s*?$/g, "")), + dedentString(htmlTrimPreserveIndentation(value)), hardline ) - : // https://infra.spec.whatwg.org/#ascii-whitespace - join(line, value.split(/[\t\n\f\r ]+/)).parts; + : join(line, splitByHtmlWhitespace(value)).parts; } function printEmbeddedAttributeValue(node, originalTextToDoc, options) { diff --git a/src/language-html/utils.js b/src/language-html/utils.js index 04df58f85..ec6d39cd8 100644 --- a/src/language-html/utils.js +++ b/src/language-html/utils.js @@ -14,6 +14,29 @@ const htmlElementAttributes = require("html-element-attributes"); const HTML_TAGS = arrayToMap(htmlTagNames); const HTML_ELEMENT_ATTRIBUTES = mapObject(htmlElementAttributes, arrayToMap); +// https://infra.spec.whatwg.org/#ascii-whitespace +const HTML_WHITESPACE = new Set(["\t", "\n", "\f", "\r", " "]); +const htmlTrimStart = (string) => string.replace(/^[\t\n\f\r ]+/, ""); +const htmlTrimEnd = (string) => string.replace(/[\t\n\f\r ]+$/, ""); +const htmlTrim = (string) => htmlTrimStart(htmlTrimEnd(string)); +const htmlTrimLeadingBlankLines = (string) => + string.replace(/^[\t\f\r ]*?\n/g, ""); +const htmlTrimPreserveIndentation = (string) => + htmlTrimLeadingBlankLines(htmlTrimEnd(string)); +const splitByHtmlWhitespace = (string) => string.split(/[\t\n\f\r ]+/); +const getLeadingHtmlWhitespace = (string) => string.match(/^[\t\n\f\r ]*/)[0]; +const getLeadingAndTrailingHtmlWhitespace = (string) => { + const [, leadingWhitespace, text, trailingWhitespace] = string.match( + /^([\t\n\f\r ]*)([\S\s]*?)([\t\n\f\r ]*)$/ + ); + return { + leadingWhitespace, + trailingWhitespace, + text, + }; +}; +const hasHtmlWhitespace = (string) => /[\t\n\f\r ]/.test(string); + function arrayToMap(array) { const map = Object.create(null); for (const value of array) { @@ -544,11 +567,11 @@ function getMinIndentation(text) { continue; } - if (/\S/.test(lineText[0])) { + if (!HTML_WHITESPACE.has(lineText[0])) { return 0; } - const indentation = lineText.match(/^\s*/)[0].length; + const indentation = getLeadingHtmlWhitespace(lineText).length; if (lineText.length === indentation) { continue; @@ -642,6 +665,11 @@ function isVueCustomBlock(node, options) { module.exports = { HTML_ELEMENT_ATTRIBUTES, HTML_TAGS, + htmlTrim, + htmlTrimPreserveIndentation, + splitByHtmlWhitespace, + hasHtmlWhitespace, + getLeadingAndTrailingHtmlWhitespace, canHaveInterpolation, countChars, countParents, diff --git a/tests/html_whitespace/__snapshots__/jsfmt.spec.js.snap b/tests/html_whitespace/__snapshots__/jsfmt.spec.js.snap index c245820c6..7176eac75 100644 --- a/tests/html_whitespace/__snapshots__/jsfmt.spec.js.snap +++ b/tests/html_whitespace/__snapshots__/jsfmt.spec.js.snap @@ -222,6 +222,273 @@ printWidth: 80 ================================================================================ `; +exports[`snippet: #18 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +
 
+=====================================output===================================== +
 
+ +================================================================================ +`; + +exports[`snippet: #19 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +
 
+=====================================output===================================== +
 
+ +================================================================================ +`; + +exports[`snippet: #20 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +
  
+=====================================output===================================== +
  
+ +================================================================================ +`; + +exports[`snippet: #21 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +
   
+=====================================output===================================== +
   
+ +================================================================================ +`; + +exports[`snippet: #22 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +  +=====================================output===================================== +  + +================================================================================ +`; + +exports[`snippet: #23 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +   +=====================================output===================================== +   + +================================================================================ +`; + +exports[`snippet: #24 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +    +=====================================output===================================== +    + +================================================================================ +`; + +exports[`snippet: #25 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +     +=====================================output===================================== +     + +================================================================================ +`; + +exports[`snippet: #26 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +  +=====================================output===================================== +  + +================================================================================ +`; + +exports[`snippet: #27 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +   +=====================================output===================================== +   + +================================================================================ +`; + +exports[`snippet: #28 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +    +=====================================output===================================== +    + +================================================================================ +`; + +exports[`snippet: #29 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +     +=====================================output===================================== +     + +================================================================================ +`; + +exports[`snippet: #30 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +   |   +=====================================output===================================== +   |   + +================================================================================ +`; + +exports[`snippet: #31 format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== +

X   or   Y

X   or   Y

+=====================================output===================================== +

X   or   Y

+

X   or   Y

+ +================================================================================ +`; + +exports[`snippet: \`U+2005\` should format like \`U+005F\` not like \`U+0020\` format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== + +
before afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter
+ +
before_afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter
+ +
before afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter
+=====================================output===================================== + +
+ before afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter +
+ +
+ before_afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter +
+ +
+ before afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter +
+ +================================================================================ +`; + +exports[`snippet: \`U+2005\` should indent like \`U+005F\` not like \`U+0020\` format 1`] = ` +====================================options===================================== +parsers: ["html"] +printWidth: 80 + | printWidth +=====================================input====================================== + + + + + + +=====================================output===================================== + + + + + + + +================================================================================ +`; + exports[`surrounding-linebreak.html format 1`] = ` ====================================options===================================== parsers: ["html"] diff --git a/tests/html_whitespace/jsfmt.spec.js b/tests/html_whitespace/jsfmt.spec.js index 53763df9b..2f87f7740 100644 --- a/tests/html_whitespace/jsfmt.spec.js +++ b/tests/html_whitespace/jsfmt.spec.js @@ -1 +1,130 @@ -run_spec(__dirname, ["html"]); +const { outdent } = require("outdent"); + +run_spec( + { + dirname: __dirname, + snippets: [ + ...[ + // https://developer.mozilla.org/en-US/docs/Glossary/Whitespace#In_HTML + // single + "\u0009", + "\u000A", + "\u000C", + "\u000D", + "\u0020", + + // many + "\u0009\u000A\u000C\u000D\u0020", + ].map((textContent) => ({ + code: `
${textContent}
`, + name: "should be empty", + output: "
\n", + })), + + ...[ + // single + "\u0009", + "\u000A", + "\u000C", + "\u000D", + "\u0020", + + // many + "\u0009\u000A\u000C\u000D\u0020", + ].map((textContent) => ({ + code: `${textContent}`, + name: "should keep one space", + output: " \n", + })), + + ...[ + // single + "\u0009", + "\u000A", + "\u000C", + "\u000D", + "\u0020", + ].map((textContent) => ({ + code: `${textContent}`, + name: "between", + output: " \n", + })), + + { + code: "\u0009\u000A\u000C\u000D\u0020", + output: "\n\n\n", + }, + + // non-space + ...[ + "\u2005", + " \u2005 ", + " \u2005\u2005 ", + " \u2005 \u2005 ", + ].map((textContent) => `
${textContent}
`), + + ...[ + "\u2005", + " \u2005 ", + " \u2005\u2005 ", + " \u2005 \u2005 ", + ].map((textContent) => `${textContent}`), + + ...[ + "\u2005", + " \u2005 ", + " \u2005\u2005 ", + " \u2005 \u2005 ", + ].map((textContent) => `${textContent}`), + + // #7103 minimal reproduction + " \u2005 | \u2005 ", + + // #7103 + "

X \u2005 or \u2005 Y

X \u2005 or \u2005 Y

", + + // This test maybe not good, `U+2005` there don't make sense, + // but the node has to be `whitespaceSensitive` and `indentationSensitive`, + // to make the `whitespace check logic` work. + { + name: "`U+2005` should indent like `U+005F` not like `U+0020`", + code: outdent` + + + + + + + `, + }, + + { + name: "`U+2005` should format like `U+005F` not like `U+0020`", + code: outdent` + +
before\u2005afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter
+ +
before\u005Fafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter
+ +
before\u0020afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter
+ `, + }, + ], + }, + ["html"] +);