Split different tests in format test (#18748)

.github/workflows/dev-test.yml

@@ -38,7 +38,6 @@ jobs:
             node: "24"
             ENABLE_CODE_COVERAGE: true
             FULL_TEST: true
-            CHECK_TEST_PARSERS: true
           # Versions not tested on linux, normally only even versions
           # If latest version is an odd number, it can be listed below too
           - os: "ubuntu-latest"
@@ -48,7 +47,6 @@ jobs:
     env:
       ENABLE_CODE_COVERAGE: ${{ matrix.ENABLE_CODE_COVERAGE }}
       FULL_TEST: ${{ matrix.FULL_TEST }}
-      CHECK_TEST_PARSERS: ${{ matrix.CHECK_TEST_PARSERS }}
     name: ${{ matrix.FULL_TEST && '[Full Test] ' || '' }} Node.js ${{ matrix.node }} on ${{ startsWith(matrix.os, 'macos') && 'MacOS' || startsWith(matrix.os, 'windows') && 'Windows' || 'Linux' }}
     runs-on: ${{ matrix.os }}
     steps:

tests/config/format-test/replace-placeholders.js

@@ -18,6 +18,11 @@ const indexProperties = [
     placeholder: RANGE_END_PLACEHOLDER,
   },
 ];
+
+/**
+@param {string} originalText
+@param {any} originalOptions
+*/
 function replacePlaceholders(originalText, originalOptions) {
   const indexes = indexProperties
     .map(({ property, placeholder }) => {

tests/config/format-test/run-prettier.js

@@ -6,25 +6,20 @@ import {
   TEST_RUNTIME,
   TEST_STANDALONE,
 } from "./constants.js";
-import { replacePlaceholders } from "./replace-placeholders.js";
 import { ensurePromise } from "./utilities.js";
 import visualizeEndOfLine from "./visualize-end-of-line.js";
 
-async function parse(source, options) {
+async function parse(input, options) {
   const prettier = await getPrettier();
   const { ast } = await ensurePromise(
-    prettier.__debug.parse(source, await loadPlugins(options), {
+    prettier.__debug.parse(input, await loadPlugins(options), {
       massage: true,
     }),
   );
   return ast;
 }
 
-async function format(originalText, originalOptions) {
+async function format(input, options) {
-  const { text: input, options } = replacePlaceholders(
-    originalText,
-    originalOptions,
-  );
   const inputWithCursor = insertCursor(input, options.cursorOffset);
   const prettier = await getPrettier();
 

tests/config/format-test/run-test.js

@@ -1,10 +1,13 @@
-import consistentEndOfLine from "./consistent-end-of-line.js";
+import { FULL_TEST } from "./constants.js";
-import { BOM, FULL_TEST } from "./constants.js";
-import createSnapshot from "./create-snapshot.js";
 import * as failedTests from "./failed-format-tests.js";
-import { format, parse } from "./run-prettier.js";
+import { replacePlaceholders } from "./replace-placeholders.js";
+import { format } from "./run-prettier.js";
+import * as testAstCompare from "./test-ast-compare.js";
+import * as testBom from "./test-bom.js";
+import * as testEndOfLine from "./test-end-of-line.js";
+import * as testFormat from "./test-format.js";
+import * as testSecondFormat from "./test-second-format.js";
 import { shouldThrowOnFormat } from "./utilities.js";
-import visualizeEndOfLine from "./visualize-end-of-line.js";
 
 /**
 @import {Fixture} from "./get-fixtures.js"
@@ -22,26 +25,78 @@ function testFixture(fixture) {
     stringifiedOptions ? ` - ${stringifiedOptions}` : ""
   }`;
 
-  const testCases = parsers
-    .filter((parser) => !failedTests.shouldDisable(filepath, parser))
-    .map((parser) => getTestCase(fixture, parser));
-
-  const testCaseForSnapshot = testCases.find(
-    (testCase) =>
-      !testCase.expectFail && typeof testCase.expectedOutput !== "string",
-  );
-  const hasMultipleParsers = testCases.length > 1;
-
   describe(title, () => {
-    for (const testCase of testCases) {
+    const testCases = parsers
-      let testTitle = "format";
+      .filter((parser) => !failedTests.shouldDisable(filepath, parser))
-      if (testCaseForSnapshot !== testCase && hasMultipleParsers) {
+      .map((parser) => getTestCase(fixture, parser));
-        testTitle += `[${testCase.parser}]`;
-      }
 
-      test(testTitle, async () => {
+    const testCaseForSnapshot = testCases.find(
-        await runTest(testCase, testCaseForSnapshot);
+      (testCase) =>
-      });
+        !testCase.expectFail && typeof testCase.expectedOutput !== "string",
+    );
+    const hasMultipleParsers = testCases.length > 1;
+
+    for (const functionality of [
+      {
+        name(testCase) {
+          let name = "format";
+          // Avoid parser display in snapshot
+          if (testCaseForSnapshot !== testCase && hasMultipleParsers) {
+            name += `[${testCase.parser}]`;
+          }
+          return name;
+        },
+        test: {
+          run: (testCase, name) =>
+            testFormat.run(testCase, name, testCaseForSnapshot),
+        },
+      },
+      { name: "ast compare", test: testAstCompare, skip: () => !FULL_TEST },
+      // The following cases only need run on main parser
+      {
+        name: "second format",
+        test: testSecondFormat,
+        skip: (testCase) => !FULL_TEST || testCase !== testCaseForSnapshot,
+      },
+      {
+        name: "end of line (CRLF)",
+        test: {
+          ...testEndOfLine,
+          run: (testCase, name) => testEndOfLine.run(testCase, name, "\r\n"),
+        },
+        skip: (testCase) => !FULL_TEST || testCase !== testCaseForSnapshot,
+      },
+      {
+        name: "end of line (CR)",
+        test: {
+          ...testEndOfLine,
+          run: (testCase, name) => testEndOfLine.run(testCase, name, "\r"),
+        },
+        skip: (testCase) => !FULL_TEST || testCase !== testCaseForSnapshot,
+      },
+      {
+        name: "BOM",
+        test: testBom,
+        skip: (testCase) => !FULL_TEST || testCase !== testCaseForSnapshot,
+      },
+    ]) {
+      for (const testCase of testCases) {
+        if (
+          functionality.skip?.(testCase) ||
+          functionality.test.skip?.(testCase)
+        ) {
+          continue;
+        }
+
+        let { name } = functionality;
+        if (typeof name === "function") {
+          name = name(testCase);
+        } else if (hasMultipleParsers) {
+          name += ` [${testCase.parser}]`;
+        }
+
+        functionality.test.run(testCase, name);
+      }
     }
   });
 }
@@ -51,10 +106,19 @@ function testFixture(fixture) {
 @param {string} parser
 */
 function getTestCase(fixture, parser) {
-  const { code, context, filepath } = fixture;
+  const { code: originalText, context, filepath } = fixture;
-  const { options } = context;
+
-  const formatOptions = { filepath, ...options, parser };
+  const { text: code, options: formatOptions } = replacePlaceholders(
-  const expectFail = shouldThrowOnFormat(fixture, options, parser);
+    originalText,
+    {
+      filepath,
+      ...context.options,
+      parser,
+    },
+  );
+
+  const expectFail = shouldThrowOnFormat(fixture, formatOptions);
+
   /** @type {ReturnType<format> | undefined} */
   let promise;
 
@@ -62,149 +126,15 @@ function getTestCase(fixture, parser) {
     context,
     parser,
     filepath,
+    originalText,
     code,
     formatOptions,
     expectFail,
     expectedOutput: fixture.output,
+    isEmpty: code.trim() === "",
     runFormat: () =>
       promise === undefined ? (promise = format(code, formatOptions)) : promise,
   };
 }
 
-/**
-@param {TestCase} testCase
-@param {TestCase} testCaseForSnapshot
-*/
-async function runTest(testCase, testCaseForSnapshot) {
-  if (testCase.expectFail) {
-    await expect(testCase.runFormat).rejects.toThrowErrorMatchingSnapshot();
-    return;
-  }
-
-  const formatResult = await testCase.runFormat();
-  expect(formatResult).toBeDefined();
-
-  // Make sure output has consistent EOL
-  expect(formatResult.eolVisualizedOutput).toBe(
-    visualizeEndOfLine(consistentEndOfLine(formatResult.outputWithCursor)),
-  );
-
-  if (typeof testCase.expectedOutput === "string") {
-    expect(formatResult.eolVisualizedOutput).toBe(testCase.expectedOutput);
-  } else if (testCase === testCaseForSnapshot) {
-    expect(
-      createSnapshot(formatResult, {
-        parsers: testCase.context.explicitParsers,
-        formatOptions: testCase.formatOptions,
-      }),
-    ).toMatchSnapshot();
-  } else {
-    const { eolVisualizedOutput } = await testCaseForSnapshot.runFormat();
-    expect(formatResult.eolVisualizedOutput).toStrictEqual(eolVisualizedOutput);
-  }
-
-  if (!FULL_TEST) {
-    return;
-  }
-
-  const { filepath, code, formatOptions } = testCase;
-  // Some parsers skip parsing empty files
-  if (formatResult.changed && code.trim()) {
-    const { input, output } = formatResult;
-    const [originalAst, formattedAst] = await Promise.all(
-      [input, output].map((code) => parse(code, formatOptions)),
-    );
-    const isAstUnstableTest = failedTests.isAstUnstable(
-      filepath,
-      formatOptions,
-    );
-
-    if (isAstUnstableTest) {
-      expect(formattedAst).not.toStrictEqual(originalAst);
-    } else {
-      expect(formattedAst).toStrictEqual(originalAst);
-    }
-  }
-
-  // The following cases should only run on main parser
-  if (testCase === testCaseForSnapshot) {
-    return;
-  }
-
-  const isUnstableTest = failedTests.isUnstable(filepath, formatOptions);
-  if (
-    (formatResult.changed || isUnstableTest) &&
-    // No range and cursor
-    formatResult.input === code
-  ) {
-    const { eolVisualizedOutput: firstOutput, output } = formatResult;
-    const { eolVisualizedOutput: secondOutput } = await format(
-      output,
-      formatOptions,
-    );
-    if (isUnstableTest) {
-      if (secondOutput === firstOutput) {
-        throw new Error(
-          `Unstable file '${filepath}' is stable now, please remove from the 'unstableTests' list.`,
-        );
-      }
-
-      // To keep eye on failed tests, this assert never supposed to pass,
-      // if it fails, just remove the file from `unstableTests`
-      expect(secondOutput).not.toBe(firstOutput);
-    } else {
-      expect(secondOutput).toBe(firstOutput);
-    }
-  }
-
-  if (!shouldSkipEolTest(code, formatResult.options)) {
-    await Promise.all(
-      ["\r\n", "\r"].map(async (eol) => {
-        const { eolVisualizedOutput: output } = await format(
-          code.replace(/\n/g, eol),
-          formatOptions,
-        );
-        // Only if `endOfLine: "auto"` the result will be different
-        const expected =
-          formatOptions.endOfLine === "auto"
-            ? visualizeEndOfLine(
-                // All `code` use `LF`, so the `eol` of result is always `LF`
-                formatResult.outputWithCursor.replace(/\n/g, eol),
-              )
-            : formatResult.eolVisualizedOutput;
-
-        expect(output).toBe(expected);
-      }),
-    );
-  }
-
-  if (code.charAt(0) !== BOM) {
-    const { eolVisualizedOutput: output } = await format(
-      BOM + code,
-      formatOptions,
-    );
-    const expected = BOM + formatResult.eolVisualizedOutput;
-    expect(output).toBe(expected);
-  }
-}
-
-function shouldSkipEolTest(code, options) {
-  if (code.includes("\r")) {
-    return true;
-  }
-  const { requirePragma, checkIgnorePragma, rangeStart, rangeEnd } = options;
-  if (requirePragma || checkIgnorePragma) {
-    return true;
-  }
-
-  if (
-    typeof rangeStart === "number" &&
-    typeof rangeEnd === "number" &&
-    rangeStart >= rangeEnd
-  ) {
-    return true;
-  }
-  return false;
-}
-
 export { testFixture };

tests/config/format-test/test-ast-compare.js

@@ -0,0 +1,51 @@
+import * as failedTests from "./failed-format-tests.js";
+import { parse } from "./run-prettier.js";
+
+/**
+@import {TestCase} from "./run-test.js"
+*/
+
+/**
+@param {TestCase} testCase
+@param {string} name
+*/
+function testAstCompare(testCase, name) {
+  test(name, async () => {
+    const formatResult = await testCase.runFormat();
+
+    if (!formatResult.changed) {
+      return;
+    }
+
+    const [originalAst, formattedAst] = await Promise.all(
+      [formatResult.input, formatResult.output].map((code) =>
+        parse(code, formatResult.options),
+      ),
+    );
+    const { filepath, formatOptions } = testCase;
+    const isAstUnstableTest = failedTests.isAstUnstable(
+      filepath,
+      formatOptions,
+    );
+
+    if (isAstUnstableTest) {
+      expect(formattedAst).not.toStrictEqual(originalAst);
+    } else {
+      expect(formattedAst).toStrictEqual(originalAst);
+    }
+  });
+}
+
+/**
+@param {TestCase} testCase
+@return {boolean}
+*/
+function shouldSkip(testCase) {
+  return (
+    testCase.expectFail ||
+    // Some parsers skip parsing empty files
+    testCase.isEmpty
+  );
+}
+
+export { testAstCompare as run, shouldSkip as skip };

tests/config/format-test/test-bom.js

@@ -0,0 +1,39 @@
+import { BOM } from "./constants.js";
+import { replacePlaceholders } from "./replace-placeholders.js";
+import { format } from "./run-prettier.js";
+
+/**
+@import {TestCase} from "./run-test.js"
+*/
+
+/**
+@param {TestCase} testCase
+@param {string} name
+*/
+
+function testBom(testCase, name) {
+  test(name, async () => {
+    const { text, options } = replacePlaceholders(
+      BOM + testCase.originalText,
+      testCase.formatOptions,
+    );
+
+    const [formatResult, { eolVisualizedOutput: output }] = await Promise.all([
+      testCase.runFormat(),
+      format(text, options),
+    ]);
+
+    const expected = BOM + formatResult.eolVisualizedOutput;
+    expect(output).toBe(expected);
+  });
+}
+
+/**
+@param {TestCase} testCase
+@return {boolean}
+*/
+function shouldSkip(testCase) {
+  return testCase.expectFail || testCase.code.charAt(0) === BOM;
+}
+
+export { testBom as run, shouldSkip as skip };

tests/config/format-test/test-end-of-line.js

@@ -0,0 +1,65 @@
+import { replacePlaceholders } from "./replace-placeholders.js";
+import { format } from "./run-prettier.js";
+import visualizeEndOfLine from "./visualize-end-of-line.js";
+
+/**
+@import {TestCase} from "./run-test.js"
+*/
+
+/**
+@param {TestCase} testCase
+@param {string} name
+@param {"\r\n" | "\r"} eol
+*/
+function testEndOfLine(testCase, name, eol) {
+  test(name, async () => {
+    const { text, options } = replacePlaceholders(
+      testCase.originalText.replace(/\n/g, eol),
+      testCase.formatOptions,
+    );
+
+    const [formatResult, { eolVisualizedOutput: output }] = await Promise.all([
+      testCase.runFormat(),
+      format(text, options),
+    ]);
+
+    const expected =
+      options.endOfLine === "auto"
+        ? visualizeEndOfLine(
+            // All `code` use `LF`, so the `eol` of result is always `LF`
+            formatResult.outputWithCursor.replace(/\n/g, eol),
+          )
+        : formatResult.eolVisualizedOutput;
+
+    expect(output).toBe(expected);
+  });
+}
+
+/**
+@param {TestCase} testCase
+@return {boolean}
+*/
+function shouldSkip(testCase) {
+  if (testCase.expectFail || testCase.isEmpty || testCase.code.includes("\r")) {
+    return true;
+  }
+
+  // They may prevent code been formatted
+  const { requirePragma, checkIgnorePragma } = testCase.formatOptions;
+  if (requirePragma || checkIgnorePragma) {
+    return true;
+  }
+
+  const { rangeStart, rangeEnd } = testCase.formatOptions;
+  if (
+    typeof rangeStart === "number" &&
+    typeof rangeEnd === "number" &&
+    rangeStart >= rangeEnd
+  ) {
+    return true;
+  }
+
+  return false;
+}
+
+export { testEndOfLine as run, shouldSkip as skip };

tests/config/format-test/test-format.js

@@ -0,0 +1,51 @@
+import consistentEndOfLine from "./consistent-end-of-line.js";
+import createSnapshot from "./create-snapshot.js";
+import visualizeEndOfLine from "./visualize-end-of-line.js";
+
+/**
+@import {TestCase} from "./run-test.js"
+*/
+
+/**
+@param {TestCase} testCase
+@param {string} name
+@param {TestCase} testCaseForSnapshot
+*/
+function testFormat(testCase, name, testCaseForSnapshot) {
+  test(name, async () => {
+    if (testCase.expectFail) {
+      await expect(testCase.runFormat).rejects.toThrowErrorMatchingSnapshot();
+      return;
+    }
+
+    const formatResult = await testCase.runFormat();
+    expect(formatResult).toBeDefined();
+
+    // Make sure output has consistent EOL
+    expect(formatResult.eolVisualizedOutput).toBe(
+      visualizeEndOfLine(consistentEndOfLine(formatResult.outputWithCursor)),
+    );
+
+    if (typeof testCase.expectedOutput === "string") {
+      expect(formatResult.eolVisualizedOutput).toBe(testCase.expectedOutput);
+      return;
+    }
+
+    if (testCase !== testCaseForSnapshot) {
+      const { eolVisualizedOutput } = await testCaseForSnapshot.runFormat();
+      expect(formatResult.eolVisualizedOutput).toStrictEqual(
+        eolVisualizedOutput,
+      );
+      return;
+    }
+
+    expect(
+      createSnapshot(formatResult, {
+        parsers: testCase.context.explicitParsers,
+        formatOptions: testCase.formatOptions,
+      }),
+    ).toMatchSnapshot();
+  });
+}
+
+export { testFormat as run };

tests/config/format-test/test-second-format.js

@@ -0,0 +1,72 @@
+import * as failedTests from "./failed-format-tests.js";
+import { format } from "./run-prettier.js";
+
+/**
+@import {TestCase} from "./run-test.js"
+*/
+
+/**
+@param {TestCase} testCase
+@param {string} name
+*/
+function testSecondFormat(testCase, name) {
+  test(name, async () => {
+    const {
+      changed,
+      eolVisualizedOutput: firstOutput,
+      output,
+    } = await testCase.runFormat();
+
+    if (!changed) {
+      return;
+    }
+
+    const { filepath, formatOptions } = testCase;
+
+    const { eolVisualizedOutput: secondOutput } = await format(
+      output,
+      formatOptions,
+    );
+
+    const isUnstableTest = failedTests.isUnstable(filepath, formatOptions);
+    if (isUnstableTest && secondOutput === firstOutput) {
+      throw new Error(
+        `Unstable file '${filepath}' is stable now, please remove from the 'unstableTests' list.`,
+      );
+    }
+
+    // To keep eye on failed tests, this assert never supposed to pass,
+    // if it fails, just remove the file from `unstableTests`
+    if (isUnstableTest) {
+      expect(secondOutput).not.toBe(firstOutput);
+      return;
+    }
+
+    expect(secondOutput).toBe(firstOutput);
+  });
+}
+
+/**
+@param {TestCase} testCase
+@return {boolean}
+*/
+function shouldSkip(testCase) {
+  if (testCase.expectFail) {
+    return true;
+  }
+
+  const { rangeStart, rangeEnd, cursorOffset } = testCase.formatOptions;
+
+  // No range and cursor
+  if (
+    typeof cursorOffset === "number" ||
+    typeof rangeStart === "number" ||
+    typeof rangeEnd === "number"
+  ) {
+    return true;
+  }
+
+  return false;
+}
+
+export { testSecondFormat as run, shouldSkip as skip };

tests/config/format-test/utilities.js

@@ -24,8 +24,8 @@ const ensurePromise = (value) => {
   return value;
 };
 
-const shouldThrowOnFormat = ({ filename }, options, parser) => {
+const shouldThrowOnFormat = ({ filename }, options) => {
-  const { errors = {} } = options;
+  const { errors = {}, parser } = options;
   if (errors === true) {
     return true;
   }