From 2c77149e6a3f6d5a74745a23c38e13f2ffebe28d Mon Sep 17 00:00:00 2001
From: Georgii Dolzhykov <thorn.mailbox@gmail.com>
Date: Sat, 6 Aug 2022 06:25:34 +0300
Subject: [PATCH] Add benchmark script (#13224)

Co-authored-by: fisker <lionkay@gmail.com>
---
 .gitignore                   |  2 ++
 .gitpod.yml                  |  2 +-
 CONTRIBUTING.md              | 12 ++++++++++++
 cspell.json                  |  7 ++++---
 package.json                 |  1 +
 scripts/benchmark/.gitignore |  1 +
 scripts/benchmark/bench.mjs  | 28 ++++++++++++++++++++++++++++
 scripts/benchmark/compare.sh | 25 +++++++++++++++++++++++++
 8 files changed, 74 insertions(+), 4 deletions(-)
 create mode 100644 scripts/benchmark/.gitignore
 create mode 100644 scripts/benchmark/bench.mjs
 create mode 100755 scripts/benchmark/compare.sh

diff --git a/.gitignore b/.gitignore
index b3b441d8c..a4b256cde 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,3 +28,5 @@ package-lock.json
 .pnp.*
 .nyc_output
 .devcontainer
+# When installing software on gitpod.io, `core.*` files are generated
+/core.*
diff --git a/.gitpod.yml b/.gitpod.yml
index 69873400d..543c194cb 100644
--- a/.gitpod.yml
+++ b/.gitpod.yml
@@ -2,8 +2,8 @@
 
 tasks:
   - init: |
+      brew install hyperfine
       yarn
-      yarn build
 
 github:
   # https://www.gitpod.io/docs/prebuilds/#configure-prebuilds
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 575786c89..67cb416c2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -95,6 +95,18 @@ In the above commands:
 
 In addition to the options above, you can use [`node --prof` and `node --prof-process`](https://nodejs.org/en/docs/guides/simple-profiling/), as well as `node --trace-opt --trace-deopt`, to get more advanced performance insights.
 
+The script `scripts/benchmark/compare.sh` can be used to compare performance of two or more commits/branches using [hyperfine](https://github.com/sharkdp/hyperfine). Usage (don't forget to install hyperfine):
+
+```sh
+PRETTIER_PERF_FILENAME=my.js ./compare.sh main some-pr-branch
+```
+
+Without `PRETTIER_PERF_FILENAME`, the script uses a default JS file as input for Prettier:
+
+```sh
+./compare.sh main next
+```
+
 ## Regression testing
 
 We have a cool tool for regression testing that runs on GitHub Actions. Have a look: https://github.com/prettier/prettier-regression-testing
diff --git a/cspell.json b/cspell.json
index 8bd613b15..62598f846 100644
--- a/cspell.json
+++ b/cspell.json
@@ -222,8 +222,8 @@
         "neoclide",
         "neoformat",
         "neovim",
-        "nocheck",
         "nnoremap",
+        "nocheck",
         "nonenumerable",
         "Nonspacing",
         "noopener",
@@ -271,9 +271,9 @@
         "Pschera",
         "quasis",
         "Raghuvir",
+        "raquo",
         "Rasmus",
         "Rattray",
-        "raquo",
         "rattrayalex",
         "readline",
         "readlines",
@@ -291,8 +291,8 @@
         "sandhose",
         "Sapegin",
         "sbdchd",
-        "sdbm",
         "scandir",
+        "sdbm",
         "Serializers",
         "setlocal",
         "setq",
@@ -330,6 +330,7 @@
         "testname",
         "tldr",
         "Tomasek",
+        "toplevel",
         "Tradeshift",
         "Transloadit",
         "trippable",
diff --git a/package.json b/package.json
index d0b8c8ccb..2a8261c47 100644
--- a/package.json
+++ b/package.json
@@ -150,6 +150,7 @@
     "perf": "yarn && yarn build && cross-env NODE_ENV=production node ./dist/bin-prettier.js",
     "perf:inspect": "yarn && yarn build && cross-env NODE_ENV=production node --inspect-brk ./dist/bin-prettier.js",
     "perf:benchmark": "yarn perf --debug-benchmark",
+    "perf:compare": "./scripts/benchmark/compare.sh",
     "lint": "run-p lint:*",
     "lint:typecheck": "tsc",
     "lint:eslint": "cross-env EFF_NO_LINK_RULES=true eslint . --format friendly",
diff --git a/scripts/benchmark/.gitignore b/scripts/benchmark/.gitignore
new file mode 100644
index 000000000..355164c12
--- /dev/null
+++ b/scripts/benchmark/.gitignore
@@ -0,0 +1 @@
+*/
diff --git a/scripts/benchmark/bench.mjs b/scripts/benchmark/bench.mjs
new file mode 100644
index 000000000..6b600d6b9
--- /dev/null
+++ b/scripts/benchmark/bench.mjs
@@ -0,0 +1,28 @@
+import { readFileSync } from "node:fs";
+
+const [, , version, method, groupCountString = 100, groupSizeString = 10] =
+  process.argv;
+const groupCount = Number(groupCountString);
+const groupSize = Number(groupSizeString);
+const { format } = await import(`./${version}/dist/index.js`);
+
+const sourceText = readFileSync(
+  process.env.PRETTIER_PERF_FILENAME || "../../src/language-js/utils/index.js",
+  "utf8"
+);
+
+for (let i = 0; i < groupCount; i++) {
+  if (method === "serial") {
+    for (let i = 0; i < groupSize; i++) {
+      await format(sourceText, { parser: "typescript" });
+    }
+  }
+
+  if (method === "parallel") {
+    await Promise.allSettled(
+      Array.from({ length: groupSize }, () =>
+        format(sourceText, { parser: "typescript" })
+      )
+    );
+  }
+}
diff --git a/scripts/benchmark/compare.sh b/scripts/benchmark/compare.sh
new file mode 100755
index 000000000..68416ed5b
--- /dev/null
+++ b/scripts/benchmark/compare.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+commits=("${@}")
+
+root=`git rev-parse --show-toplevel`
+cd "$root/scripts/benchmark"
+
+function cleanup {
+  echo Cleaning up...
+  rm -rf "${commits[@]}"
+  echo Done!
+}
+trap cleanup EXIT
+
+args=()
+
+for commit in "${commits[@]}"; do
+  rm -rf $commit
+  git -C "$root" archive $commit --prefix $commit/ ':!tests*' ':!website' ':!docs' | tar -x
+  (cd $commit; yarn; yarn build)
+  args+=("node ./bench.mjs $commit serial")
+  args+=("node ./bench.mjs $commit parallel")
+done
+
+hyperfine --warmup 3 "${args[@]}"