From 99ecf47e1d9b8020a08dd01f7ecd761922dd932d Mon Sep 17 00:00:00 2001 From: fisker Cheung Date: Tue, 14 Nov 2023 15:09:49 +0800 Subject: [PATCH] Improve `RegExp`s in markdown printer (#15656) --- scripts/build/esbuild-plugins/evaluate.js | 13 +++-- src/language-markdown/constants.evaluate.js | 61 ++++++++++++--------- src/language-markdown/printer-markdown.js | 6 +- src/language-markdown/utils.js | 19 +++---- 4 files changed, 53 insertions(+), 46 deletions(-) diff --git a/scripts/build/esbuild-plugins/evaluate.js b/scripts/build/esbuild-plugins/evaluate.js index c82be4e52..37621415c 100644 --- a/scripts/build/esbuild-plugins/evaluate.js +++ b/scripts/build/esbuild-plugins/evaluate.js @@ -14,12 +14,15 @@ export default function esbuildPluginEvaluate() { const module = await importModule(path); const text = Object.entries(module) .map(([specifier, value]) => { - value = serialize(value, { space: 2 }); + const code = + value instanceof RegExp + ? `/${value.source}/${value.flags}` + : serialize(value, { space: 2 }); if (specifier === "default") { return format === "cjs" - ? `module.exports = ${value};` - : `export default ${value};`; + ? `module.exports = ${code};` + : `export default ${code};`; } if (!isValidIdentifier(specifier)) { @@ -27,8 +30,8 @@ export default function esbuildPluginEvaluate() { } return format === "cjs" - ? `exports.${specifier} = ${value};` - : `export const ${specifier} = ${value};`; + ? `exports.${specifier} = ${code};` + : `export const ${specifier} = ${code};`; }) .join("\n"); return { contents: text }; diff --git a/src/language-markdown/constants.evaluate.js b/src/language-markdown/constants.evaluate.js index 0f118aa66..d05924062 100644 --- a/src/language-markdown/constants.evaluate.js +++ b/src/language-markdown/constants.evaluate.js @@ -1,33 +1,37 @@ -import * as cjkRegex from "cjk-regex"; -import * as regexpUtil from "regexp-util"; +import { all as getCjkCharset } from "cjk-regex"; +import { Charset } from "regexp-util"; import unicodeRegex from "unicode-regex"; -const cjkPattern = `(?:${cjkRegex - .all() - .union( - unicodeRegex({ - Script_Extensions: ["Han", "Katakana", "Hiragana", "Hangul", "Bopomofo"], - General_Category: [ - "Other_Letter", - "Letter_Number", - "Other_Symbol", - "Modifier_Letter", - "Modifier_Symbol", - "Nonspacing_Mark", - ], - }), - ) - .toString()})(?:${unicodeRegex({ +const cjkCharset = new Charset( + getCjkCharset(), + unicodeRegex({ + Script_Extensions: ["Han", "Katakana", "Hiragana", "Hangul", "Bopomofo"], + General_Category: [ + "Other_Letter", + "Letter_Number", + "Other_Symbol", + "Modifier_Letter", + "Modifier_Symbol", + "Nonspacing_Mark", + ], + }), +); +const variationSelectorsCharset = unicodeRegex({ Block: ["Variation_Selectors", "Variation_Selectors_Supplement"], -}).toString()})?`; +}); -const kRegex = unicodeRegex({ Script: ["Hangul"] }) - .union(unicodeRegex({ Script_Extensions: ["Hangul"] })) - .toRegExp(); +const CJK_REGEXP = new RegExp( + `(?:${cjkCharset.toString()})(?:${variationSelectorsCharset.toString()})?`, +); + +const K_REGEXP = new Charset( + unicodeRegex({ Script: ["Hangul"] }), + unicodeRegex({ Script_Extensions: ["Hangul"] }), +).toRegExp(); // http://spec.commonmark.org/0.25/#ascii-punctuation-character const asciiPunctuationCharset = - /* prettier-ignore */ regexpUtil.charset( + /* prettier-ignore */ new Charset( "!", '"', "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", ":", ";", "<", "=", ">", "?", "@", "[", "\\", "]", "^", "_", "`", "{", "|", @@ -35,7 +39,7 @@ const asciiPunctuationCharset = ); // http://spec.commonmark.org/0.25/#punctuation-character -const punctuationCharset = unicodeRegex({ +const unicodePunctuationCharset = unicodeRegex({ // http://unicode.org/Public/5.1.0/ucd/UCD.html#General_Category_Values General_Category: [ /* Pc */ "Connector_Punctuation", @@ -46,8 +50,11 @@ const punctuationCharset = unicodeRegex({ /* Po */ "Other_Punctuation", /* Ps */ "Open_Punctuation", ], -}).union(asciiPunctuationCharset); +}); -const punctuationPattern = punctuationCharset.toString(); +const PUNCTUATION_REGEXP = new Charset( + asciiPunctuationCharset, + unicodePunctuationCharset, +).toRegExp(); -export { cjkPattern, kRegex, punctuationPattern }; +export { CJK_REGEXP, K_REGEXP, PUNCTUATION_REGEXP }; diff --git a/src/language-markdown/printer-markdown.js b/src/language-markdown/printer-markdown.js index 6082f74de..01012fd11 100644 --- a/src/language-markdown/printer-markdown.js +++ b/src/language-markdown/printer-markdown.js @@ -26,11 +26,11 @@ import { insertPragma } from "./pragma.js"; import { locStart, locEnd } from "./loc.js"; import preprocess from "./print-preprocess.js"; import clean from "./clean.js"; +import { PUNCTUATION_REGEXP } from "./constants.evaluate.js"; import { getFencedCodeBlockValue, hasGitDiffFriendlyOrderedList, splitText, - punctuationPattern, INLINE_NODE_TYPES, INLINE_NODE_WRAPPER_TYPES, isAutolink, @@ -88,8 +88,8 @@ function genericPrint(path, options, print) { .replaceAll( new RegExp( [ - `(^|${punctuationPattern})(_+)`, - `(_+)(${punctuationPattern}|$)`, + `(^|${PUNCTUATION_REGEXP.source})(_+)`, + `(_+)(${PUNCTUATION_REGEXP.source}|$)`, ].join("|"), "g", ), diff --git a/src/language-markdown/utils.js b/src/language-markdown/utils.js index a65400da2..a5bfcf27b 100644 --- a/src/language-markdown/utils.js +++ b/src/language-markdown/utils.js @@ -1,9 +1,9 @@ import assert from "node:assert"; import { locStart, locEnd } from "./loc.js"; import { - cjkPattern, - kRegex, - punctuationPattern, + CJK_REGEXP, + K_REGEXP, + PUNCTUATION_REGEXP, } from "./constants.evaluate.js"; const INLINE_NODE_TYPES = new Set([ @@ -34,8 +34,6 @@ const INLINE_NODE_WRAPPER_TYPES = new Set([ "heading", ]); -const punctuationRegex = new RegExp(punctuationPattern); - const KIND_NON_CJK = "non-cjk"; const KIND_CJ_LETTER = "cj-letter"; const KIND_K_LETTER = "k-letter"; @@ -85,7 +83,7 @@ function splitText(text) { continue; } - const innerTokens = token.split(new RegExp(`(${cjkPattern})`)); + const innerTokens = token.split(new RegExp(`(${CJK_REGEXP.source})`)); for (const [innerIndex, innerToken] of innerTokens.entries()) { if ( (innerIndex === 0 || innerIndex === innerTokens.length - 1) && @@ -101,8 +99,8 @@ function splitText(text) { type: "word", value: innerToken, kind: KIND_NON_CJK, - hasLeadingPunctuation: punctuationRegex.test(innerToken[0]), - hasTrailingPunctuation: punctuationRegex.test(innerToken.at(-1)), + hasLeadingPunctuation: PUNCTUATION_REGEXP.test(innerToken[0]), + hasTrailingPunctuation: PUNCTUATION_REGEXP.test(innerToken.at(-1)), }); } continue; @@ -110,7 +108,7 @@ function splitText(text) { // CJK character appendNode( - punctuationRegex.test(innerToken) + PUNCTUATION_REGEXP.test(innerToken) ? { type: "word", value: innerToken, @@ -122,7 +120,7 @@ function splitText(text) { type: "word", value: innerToken, // Korean uses space to divide words, but Chinese & Japanese do not - kind: kRegex.test(innerToken) ? KIND_K_LETTER : KIND_CJ_LETTER, + kind: K_REGEXP.test(innerToken) ? KIND_K_LETTER : KIND_CJ_LETTER, hasLeadingPunctuation: false, hasTrailingPunctuation: false, }, @@ -241,7 +239,6 @@ function isAutolink(node) { export { mapAst, splitText, - punctuationPattern, getFencedCodeBlockValue, getOrderedListItemInfo, hasGitDiffFriendlyOrderedList,