diff --git a/src/config/prettier-config/loaders.js b/src/config/prettier-config/loaders.js
index 404af1813..7de2670fa 100644
--- a/src/config/prettier-config/loaders.js
+++ b/src/config/prettier-config/loaders.js
@@ -11,7 +11,7 @@ async function readJson(file) {
   try {
     return parseJson(content);
   } catch (/** @type {any} */ error) {
-    error.message = `JSON Error in ${file}:\n${error.message}`;
+    error.message = `JSON Error in "${file}":\n${error.message}`;
     throw error;
   }
 }
@@ -66,7 +66,7 @@ async function loadYaml(file) {
   try {
     return parseYaml(content);
   } catch (/** @type {any} */ error) {
-    error.message = `YAML Error in ${file}:\n${error.message}`;
+    error.message = `YAML Error in "${file}":\n${error.message}`;
     throw error;
   }
 }
@@ -77,7 +77,7 @@ async function loadToml(file) {
   try {
     return parseToml(content);
   } catch (/** @type {any} */ error) {
-    error.message = `TOML Error in ${file}:\n${error.message}`;
+    error.message = `TOML Error in "${file}":\n${error.message}`;
     throw error;
   }
 }
@@ -87,7 +87,7 @@ async function loadJson5(file) {
   try {
     return parseJson5(content);
   } catch (/** @type {any} */ error) {
-    error.message = `JSON5 Error in ${file}:\n${error.message}`;
+    error.message = `JSON5 Error in "${file}":\n${error.message}`;
     throw error;
   }
 }
diff --git a/tests/integration/__tests__/__snapshots__/config-invalid.js.snap b/tests/integration/__tests__/__snapshots__/config-invalid.js.snap
index 7288ad961..525b6e9f5 100644
--- a/tests/integration/__tests__/__snapshots__/config-invalid.js.snap
+++ b/tests/integration/__tests__/__snapshots__/config-invalid.js.snap
@@ -25,15 +25,6 @@ exports[`show warning with unknown option  1`] = `
 }
 `;
 
-exports[`throw error for unsupported extension  1`] = `
-{
-  "stderr": "[error] Invalid configuration:
-[error] No loader specified for extension ".unsupported"",
-  "stdout": "",
-  "write": [],
-}
-`;
-
 exports[`throw error if both --config and --no-config are submitted  1`] = `
 {
   "stderr": "[error] Cannot use --no-config and --config together.",
@@ -46,20 +37,6 @@ exports[`throw error if both --config and --no-config are submitted  2`] = `
 }
 `;
 
-exports[`throw error with invalid config format  1`] = `
-{
-  "stdout": "",
-  "write": [],
-}
-`;
-
-exports[`throw error with invalid config format  2`] = `
-{
-  "stdout": "",
-  "write": [],
-}
-`;
-
 exports[`throw error with invalid config option (int)  1`] = `
 {
   "stderr": "[error] Invalid tabWidth value. Expected an integer, but received 0.5.",
diff --git a/tests/integration/__tests__/config-invalid.js b/tests/integration/__tests__/config-invalid.js
index a09bc9e12..9bd6c4a67 100644
--- a/tests/integration/__tests__/config-invalid.js
+++ b/tests/integration/__tests__/config-invalid.js
@@ -1,34 +1,8 @@
 import fs from "node:fs/promises";
-import outdent from "outdent";
 import jestPathSerializer from "../path-serializer.js";
 
 expect.addSnapshotSerializer(jestPathSerializer);
 
-describe("throw error for unsupported extension", () => {
-  runCli("cli/config/invalid", [
-    "--config",
-    "file/.prettierrc.unsupported",
-  ]).test({
-    status: "non-zero",
-  });
-});
-
-describe("throw error with invalid config format", () => {
-  runCli("cli/config/invalid", ["--config", "file/.prettierrc"]).test({
-    status: "non-zero",
-    stderr: expect.stringMatching(/Cannot find package '--invalid--'/),
-  });
-});
-
-describe("throw error with invalid config format", () => {
-  runCli("cli/config/invalid", ["--config", "type-error/.prettierrc"]).test({
-    status: "non-zero",
-    stderr: expect.stringContaining(
-      "Config is only allowed to be an object, but received number in",
-    ),
-  });
-});
-
 describe("throw error with invalid config target (directory)", () => {
   runCli("cli/config/invalid", [
     "--config",
@@ -122,80 +96,6 @@ describe("show warning with kebab-case option key", () => {
   });
 });
 
-// #8815, please make sure this error contains code frame
-describe("Invalid json file", () => {
-  runCli("cli/config/invalid", [
-    "--config",
-    "broken-json/.prettierrc.json",
-    "--parser",
-    "babel",
-  ]).test({
-    status: 2,
-    stdout: "",
-    write: [],
-    stderr: expect.stringContaining(
-      outdent`
-        > 1 | {a':}
-            |  ^
-          2 |
-      `
-        .split("\n")
-        .map((line) => `[error] ${line}`)
-        .join("\n"),
-    ),
-  });
-});
-
-describe("Invalid toml file", () => {
-  runCli("cli/config/invalid", [
-    "--config",
-    "broken-toml/.prettierrc.toml",
-    "--parser",
-    "babel",
-  ]).test({
-    status: 2,
-    stdout: "",
-    write: [],
-    stderr: expect.stringContaining(
-      outdent`
-        Invalid TOML document: incomplete declaration: value expected
-
-        1:  a=
-              ^
-        2:    b!=
-      `
-        .split("\n")
-        .map((line) => `[error] ${line}`)
-        .join("\n"),
-    ),
-  });
-});
-
-describe("Invalid yaml file", () => {
-  runCli("cli/config/invalid", [
-    "--config",
-    "broken-yaml/.prettierrc.yaml",
-    "--parser",
-    "babel",
-  ]).test({
-    status: 2,
-    stdout: "",
-    write: [],
-    stderr: expect.stringContaining(
-      // Keep the outdent, since error message changes between versions
-      outdent`
-        Map keys must be unique at line 1, column 3:
-
-        a:
-          ^
-      `
-        .split("\n")
-        .map((line) => `[error] ${line}`)
-        .join("\n"),
-    ),
-  });
-});
-
 describe("Invalid config value", () => {
   runCli("cli/config/invalid", [
     "--config",
diff --git a/tests/integration/cli/config/invalid/broken-json/.prettierrc.json b/tests/integration/cli/config/invalid/broken-json/.prettierrc.json
deleted file mode 100644
index a8189a0ca..000000000
--- a/tests/integration/cli/config/invalid/broken-json/.prettierrc.json
+++ /dev/null
@@ -1 +0,0 @@
-{a':}
diff --git a/tests/integration/cli/config/invalid/broken-toml/.prettierrc.toml b/tests/integration/cli/config/invalid/broken-toml/.prettierrc.toml
deleted file mode 100644
index 14f0cd983..000000000
--- a/tests/integration/cli/config/invalid/broken-toml/.prettierrc.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-a=
-  b!=
diff --git a/tests/integration/cli/config/invalid/broken-yaml/.prettierrc.yaml b/tests/integration/cli/config/invalid/broken-yaml/.prettierrc.yaml
deleted file mode 100644
index 341bbd84a..000000000
--- a/tests/integration/cli/config/invalid/broken-yaml/.prettierrc.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-a:
-a:
diff --git a/tests/integration/cli/config/invalid/file/.prettierrc b/tests/integration/cli/config/invalid/file/.prettierrc
deleted file mode 100644
index 34224b1fa..000000000
--- a/tests/integration/cli/config/invalid/file/.prettierrc
+++ /dev/null
@@ -1 +0,0 @@
---invalid--
\ No newline at end of file
diff --git a/tests/integration/cli/config/invalid/file/.prettierrc.unsupported b/tests/integration/cli/config/invalid/file/.prettierrc.unsupported
deleted file mode 100644
index 9e26dfeeb..000000000
--- a/tests/integration/cli/config/invalid/file/.prettierrc.unsupported
+++ /dev/null
@@ -1 +0,0 @@
-{}
\ No newline at end of file
diff --git a/tests/integration/cli/config/invalid/type-error/.prettierrc b/tests/integration/cli/config/invalid/type-error/.prettierrc
deleted file mode 100644
index d00491fd7..000000000
--- a/tests/integration/cli/config/invalid/type-error/.prettierrc
+++ /dev/null
@@ -1 +0,0 @@
-1
diff --git a/tests/unit/invalid-config.js b/tests/unit/invalid-config.js
new file mode 100644
index 000000000..11e4253be
--- /dev/null
+++ b/tests/unit/invalid-config.js
@@ -0,0 +1,104 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import url from "node:url";
+import { temporaryDirectory as getTemporaryDirectory } from "tempy";
+import loadConfig from "../../src/config/prettier-config/load-config.js";
+import createPathSerializer from "../integration/create-path-serializer.js";
+
+const directory = getTemporaryDirectory();
+expect.addSnapshotSerializer(
+  createPathSerializer({
+    replacements: new Map([[url.pathToFileURL(directory + "/"), "<dir>/"]]),
+  }),
+);
+
+async function loadConfigFile({ filename, content }) {
+  const file = path.join(directory, filename);
+  await fs.writeFile(file, content);
+  return await loadConfig(file);
+}
+
+describe("Invalid configs", () => {
+  afterAll(async () => {
+    await fs.rm(directory, { recursive: true, force: true });
+  });
+
+  test("throw error for unsupported extension", async () => {
+    await expect(() =>
+      loadConfigFile({ filename: ".prettierrc.unsupported", content: "{}" }),
+    ).rejects.toThrowErrorMatchingInlineSnapshot(
+      `"No loader specified for extension ".unsupported""`,
+    );
+  });
+
+  test("throw error with invalid config format", async () => {
+    await expect(() =>
+      loadConfigFile({ filename: ".prettierrc", content: "--invalid--" }),
+    ).rejects.toThrowErrorMatchingInlineSnapshot(
+      `"Cannot find package '--invalid--' imported from <dir>/.prettierrc"`,
+    );
+  });
+
+  test("throw error with invalid config format", async () => {
+    await expect(() =>
+      loadConfigFile({ filename: ".prettierrc", content: "1" }),
+    ).rejects.toThrowErrorMatchingInlineSnapshot(
+      `"Config is only allowed to be an object, but received number in "<dir>/.prettierrc""`,
+    );
+  });
+
+  // #8815, please make sure this error contains code frame
+  test("Invalid JSON file", async () => {
+    await expect(() =>
+      loadConfigFile({ filename: ".prettierrc.json", content: "{a':}" }),
+    ).rejects.toThrowErrorMatchingInlineSnapshot(`
+"JSON Error in "<dir>/.prettierrc.json":
+Expected property name or '}' in JSON at position 1 (line 1 column 2)
+
+> 1 | {a':}
+    |  ^
+
+
+> 1 | {a':}
+    |  ^
+
+Cause: Expected property name or '}' in JSON at position 1 (line 1 column 2)"
+`);
+  });
+
+  test("Invalid TOML file", async () => {
+    await expect(() =>
+      loadConfigFile({ filename: ".prettierrc.toml", content: "a=\n  b!=" }),
+    ).rejects.toThrowErrorMatchingInlineSnapshot(`
+"TOML Error in "<dir>/.prettierrc.toml":
+Invalid TOML document: incomplete declaration: value expected
+
+1:  a=
+      ^
+2:    b!=
+"
+`);
+  });
+
+  test("Invalid YAML file", async () => {
+    await expect(() =>
+      loadConfigFile({ filename: ".prettierrc.yaml", content: "a:\na:" }),
+    ).rejects.toThrowErrorMatchingInlineSnapshot(`
+"YAML Error in "<dir>/.prettierrc.yaml":
+Map keys must be unique at line 1, column 3:
+
+a:
+  ^
+"
+`);
+  });
+
+  test("Invalid JSON5 file", async () => {
+    await expect(() =>
+      loadConfigFile({ filename: ".prettierrc.json5", content: "a:" }),
+    ).rejects.toThrowErrorMatchingInlineSnapshot(`
+"JSON5 Error in "<dir>/.prettierrc.json5":
+JSON5: invalid character 'a' at 1:1"
+`);
+  });
+});