From 05a32d377f1b9566162945d90b2824db331d21d2 Mon Sep 17 00:00:00 2001 From: fisker Cheung Date: Sat, 6 May 2023 16:56:49 +0800 Subject: [PATCH] Switch `string-width` to `eastasianwidth` (#14793) --- cspell.json | 1 + package.json | 3 +- scripts/build/build-javascript-module.js | 6 ++ .../build/transform/eastasianwidth-module.js | 57 +++++++++++++++++++ src/utils/get-string-width.js | 32 ++++++++++- tests/integration/__tests__/util-shared.js | 24 ++++++-- yarn.lock | 42 ++++++-------- 7 files changed, 129 insertions(+), 36 deletions(-) create mode 100644 scripts/build/transform/eastasianwidth-module.js diff --git a/cspell.json b/cspell.json index 5a2abf2f1..35715fc25 100644 --- a/cspell.json +++ b/cspell.json @@ -86,6 +86,7 @@ "dotfiles", "duailibe", "Duperron", + "eastasianwidth", "Eemeli", "ekkhus", "elektronik", diff --git a/package.json b/package.json index 24352b89d..1a4682c6d 100644 --- a/package.json +++ b/package.json @@ -53,8 +53,10 @@ "css-units-list": "2.0.1", "dashify": "2.0.0", "diff": "5.1.0", + "eastasianwidth": "0.2.0", "editorconfig": "0.15.3", "editorconfig-to-prettier": "1.0.0", + "emoji-regex": "10.2.1", "escape-string-regexp": "5.0.0", "espree": "9.5.1", "fast-glob": "3.2.12", @@ -94,7 +96,6 @@ "remark-math": "3.0.1", "remark-parse": "8.0.3", "sdbm": "2.0.0", - "string-width": "5.0.1", "strip-ansi": "7.0.1", "to-fast-properties": "4.0.0", "typescript": "5.0.4", diff --git a/scripts/build/build-javascript-module.js b/scripts/build/build-javascript-module.js index 5e28e3cb8..72fd6a0f5 100644 --- a/scripts/build/build-javascript-module.js +++ b/scripts/build/build-javascript-module.js @@ -16,6 +16,7 @@ import esbuildPluginShimCommonjsObjects from "./esbuild-plugins/shim-commonjs-ob import esbuildPluginPrimitiveDefine from "./esbuild-plugins/primitive-define.js"; import esbuildPluginAddDefaultExport from "./esbuild-plugins/add-default-export.js"; import transform from "./transform/index.js"; +import transformEastAsianWidthModule from "./transform/eastasianwidth-module.js"; import { getPackageFile } from "./utils.js"; const { dirname, readJsonSync, require } = createEsmUtils(import.meta); @@ -92,6 +93,11 @@ function getEsbuildOptions({ file, files, shouldCollectLicenses, cliOptions }) { .replace(/\nfunction _os\(\).*?\n}/s, "") .replace(/\nfunction _detectNewline\(\).*?\n}/s, ""), }, + // Reduce size + { + module: require.resolve("eastasianwidth"), + process: transformEastAsianWidthModule, + }, ]; const define = { diff --git a/scripts/build/transform/eastasianwidth-module.js b/scripts/build/transform/eastasianwidth-module.js new file mode 100644 index 000000000..95f894968 --- /dev/null +++ b/scripts/build/transform/eastasianwidth-module.js @@ -0,0 +1,57 @@ +import { parse } from "@babel/parser"; +import { traverseFast as traverse } from "@babel/types"; +import babelGenerator from "@babel/generator"; + +const generate = babelGenerator.default; + +function transformEastAsianWidthModule(original) { + let eastAsianWidthFunction; + traverse(parse(original), (node) => { + // We only need `eaw.eastAsianWidth` + if ( + node.type === "AssignmentExpression" && + node.left.type === "MemberExpression" && + !node.left.computed && + !node.left.optional && + node.left.object.type === "Identifier" && + node.left.object.name === "eaw" && + node.left.property.type === "Identifier" && + node.left.property.name === "eastAsianWidth" + ) { + eastAsianWidthFunction = node.right; + } + }); + + const functionBody = eastAsianWidthFunction.body.body; + + // We only need `IfStatement`s returns `"F"` or `"W"` + for (let index = functionBody.length - 1; index >= 0; index--) { + const node = functionBody[index]; + + if ( + node.type === "IfStatement" && + node.consequent.type === "BlockStatement" && + node.consequent.body.length === 1 && + node.consequent.body[0].type === "ReturnStatement" && + node.consequent.body[0].argument.type === "StringLiteral" + ) { + const { value } = node.consequent.body[0].argument; + if (value !== "F" && value !== "W") { + functionBody.splice(index, 1); + } + } + } + + const ast = parse("export default { eastAsianWidth() {} };", { + sourceType: "module", + }); + Object.assign(ast.program.body[0].declaration.properties[0], { + params: eastAsianWidthFunction.params, + body: eastAsianWidthFunction.body, + }); + + const { code } = generate(ast); + return code; +} + +export default transformEastAsianWidthModule; diff --git a/src/utils/get-string-width.js b/src/utils/get-string-width.js index 864df308a..cc8740804 100644 --- a/src/utils/get-string-width.js +++ b/src/utils/get-string-width.js @@ -1,7 +1,10 @@ -import stringWidth from "string-width"; +import emojiRegex from "emoji-regex"; +import eastAsianWidth from "eastasianwidth"; const notAsciiRegex = /[^\x20-\x7F]/; +// Similar to https://github.com/sindresorhus/string-width +// We don't strip ansi, always treat ambiguous width characters as having narrow width. /** * @param {string} text * @returns {number} @@ -11,12 +14,35 @@ function getStringWidth(text) { return 0; } - // shortcut to avoid needless string `RegExp`s, replacements, and allocations within `string-width` + // shortcut to avoid needless string `RegExp`s, replacements, and allocations if (!notAsciiRegex.test(text)) { return text.length; } - return stringWidth(text); + text = text.replace(emojiRegex(), " "); + let width = 0; + + // Use `Intl.Segmenter` when we drop support for Node.js v14 + // https://github.com/prettier/prettier/pull/14793#discussion_r1185840038 + // https://github.com/sindresorhus/string-width/pull/47 + for (const character of text) { + const codePoint = character.codePointAt(0); + + // Ignore control characters + if (codePoint <= 0x1f || (codePoint >= 0x7f && codePoint <= 0x9f)) { + continue; + } + + // Ignore combining characters + if (codePoint >= 0x300 && codePoint <= 0x36f) { + continue; + } + + const code = eastAsianWidth.eastAsianWidth(character); + width += code === "F" || code === "W" ? 2 : 1; + } + + return width; } export default getStringWidth; diff --git a/tests/integration/__tests__/util-shared.js b/tests/integration/__tests__/util-shared.js index eaec94a3f..31239ee87 100644 --- a/tests/integration/__tests__/util-shared.js +++ b/tests/integration/__tests__/util-shared.js @@ -51,29 +51,41 @@ test("sharedUtil.getMaxContinuousCount", () => { test("sharedUtil.getStringWidth", () => { const { getStringWidth } = sharedUtil; - // From https://github.com/sindresorhus/string-width/blob/master/test.js + // From https://github.com/sindresorhus/string-width/blob/main/test.js expect(getStringWidth("abcde")).toBe(5); expect(getStringWidth("古池や")).toBe(6); expect(getStringWidth("あいうabc")).toBe(9); + expect(getStringWidth("あいう★")).toBe(7); + expect(getStringWidth("±")).toBe(1); expect(getStringWidth("ノード.js")).toBe(9); expect(getStringWidth("你好")).toBe(4); expect(getStringWidth("안녕하세요")).toBe(10); expect(getStringWidth("A\uD83C\uDE00BC")).toBe(5); - expect(getStringWidth("\u001B[31m\u001B[39m")).toBe(0); - expect( - getStringWidth("\u001B]8;;https://github.com\u0007Click\u001B]8;;\u0007") - ).toBe(5); + // We don't strip ansi + // expect(getStringWidth("\u001B[31m\u001B[39m")).toBe(0); + // expect( + // getStringWidth("\u001B]8;;https://github.com\u0007Click\u001B]8;;\u0007") + // ).toBe(5); expect(getStringWidth("\u{231A}")).toBe(2); expect(getStringWidth("\u{2194}\u{FE0F}")).toBe(2); expect(getStringWidth("\u{1F469}")).toBe(2); expect(getStringWidth("\u{1F469}\u{1F3FF}")).toBe(2); + // Ideally this should be `2`, switch to use `Intl.Segmenter` will fix it + // https://github.com/prettier/prettier/pull/14793#discussion_r1185840038 + expect(getStringWidth("\u{845B}\u{E0100}")).toBe(3); + expect(getStringWidth(String.fromCharCode(0))).toBe(0); expect(getStringWidth(String.fromCharCode(31))).toBe(0); // expect(getStringWidth(String.fromCharCode(127))).toBe(0); // Different with `string-width` expect(getStringWidth(String.fromCharCode(134))).toBe(0); expect(getStringWidth(String.fromCharCode(159))).toBe(0); expect(getStringWidth("\u001B")).toBe(0); - // expect(getStringWidth("x\u0300"), 1); // Different with `string-width` + expect(getStringWidth("x\u0300")).toBe(1); + + expect(getStringWidth("👶")).toBe(2); + expect(getStringWidth("👶🏽")).toBe(2); + expect(getStringWidth("👩‍👩‍👦‍👦")).toBe(2); + expect(getStringWidth("👨‍❤️‍💋‍👨")).toBe(2); }); test("sharedUtil.getAlignmentSize", () => { diff --git a/yarn.lock b/yarn.lock index 54b21b48a..c550bd497 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3106,6 +3106,13 @@ __metadata: languageName: node linkType: hard +"eastasianwidth@npm:0.2.0": + version: 0.2.0 + resolution: "eastasianwidth@npm:0.2.0" + checksum: 7d00d7cd8e49b9afa762a813faac332dee781932d6f2c848dc348939c4253f1d4564341b7af1d041853bc3f32c2ef141b58e0a4d9862c17a7f08f68df1e0f1ed + languageName: node + linkType: hard + "editorconfig-to-prettier@npm:1.0.0": version: 1.0.0 resolution: "editorconfig-to-prettier@npm:1.0.0" @@ -3141,6 +3148,13 @@ __metadata: languageName: node linkType: hard +"emoji-regex@npm:10.2.1": + version: 10.2.1 + resolution: "emoji-regex@npm:10.2.1" + checksum: 1aa2d16881c56531fdfc03d0b36f5c2b6221cc4097499a5665b88b711dc3fb4d5b8804f0ca6f00c56e5dcf89bac75f0487eee85da1da77df3a33accc6ecbe426 + languageName: node + linkType: hard + "emoji-regex@npm:^8.0.0": version: 8.0.0 resolution: "emoji-regex@npm:8.0.0" @@ -3148,13 +3162,6 @@ __metadata: languageName: node linkType: hard -"emoji-regex@npm:^9.2.2": - version: 9.2.2 - resolution: "emoji-regex@npm:9.2.2" - checksum: 8487182da74aabd810ac6d6f1994111dfc0e331b01271ae01ec1eb0ad7b5ecc2bbbbd2f053c05cb55a1ac30449527d819bbfbf0e3de1023db308cbcb47f86601 - languageName: node - linkType: hard - "encoding@npm:^0.1.13": version: 0.1.13 resolution: "encoding@npm:0.1.13" @@ -4692,13 +4699,6 @@ __metadata: languageName: node linkType: hard -"is-fullwidth-code-point@npm:^4.0.0": - version: 4.0.0 - resolution: "is-fullwidth-code-point@npm:4.0.0" - checksum: 8ae89bf5057bdf4f57b346fb6c55e9c3dd2549983d54191d722d5c739397a903012cc41a04ee3403fd872e811243ef91a7c5196da7b5841dc6b6aae31a264a8d - languageName: node - linkType: hard - "is-generator-fn@npm:^2.0.0": version: 2.1.0 resolution: "is-generator-fn@npm:2.1.0" @@ -6744,8 +6744,10 @@ __metadata: css-units-list: 2.0.1 dashify: 2.0.0 diff: 5.1.0 + eastasianwidth: 0.2.0 editorconfig: 0.15.3 editorconfig-to-prettier: 1.0.0 + emoji-regex: 10.2.1 enquirer: 2.3.6 esbuild: 0.17.18 esbuild-visualizer: 0.4.0 @@ -6815,7 +6817,6 @@ __metadata: semver: 7.3.8 serialize-javascript: 6.0.1 snapshot-diff: 0.10.0 - string-width: 5.0.1 strip-ansi: 7.0.1 tempy: 3.0.0 to-fast-properties: 4.0.0 @@ -7673,17 +7674,6 @@ __metadata: languageName: node linkType: hard -"string-width@npm:5.0.1": - version: 5.0.1 - resolution: "string-width@npm:5.0.1" - dependencies: - emoji-regex: ^9.2.2 - is-fullwidth-code-point: ^4.0.0 - strip-ansi: ^7.0.1 - checksum: 2a7cbd6a4180f8215553fc0dfe8fe62d2ba76d141b0a6fca44df7b81f0089613d0b115bd67bb293ea7e8c5f8295525014a3562cce28d0e06caa6f626980e9c7e - languageName: node - linkType: hard - "string-width@npm:^1.0.2 || 2 || 3 || 4, string-width@npm:^4.1.0, string-width@npm:^4.2.0, string-width@npm:^4.2.3": version: 4.2.3 resolution: "string-width@npm:4.2.3"