perfshield 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Vincent Riemer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,126 @@
+# perfshield
+
+perfshield is a build-system-agnostic benchmarking tool for comparing two
+versions of a JavaScript library across JS runtimes. It borrows statistical
+ideas from Tachometer (confidence intervals, adaptive sampling) and focuses on
+runtime JS engines (Node/V8 today).
+
+## What it does
+
+- Builds your benchmark bundle using your own build command.
+- Saves a baseline bundle (`prepare`).
+- Builds the current bundle and compares it to the baseline (`compare`).
+- Reports results in console and/or JSON.
+- Exits with code 1 when a regression is detected (CI excludes 0 in the slower direction).
+
+## Requirements
+
+- Node.js (used for the CLI and the Node engine).
+- A build command that emits a single ESM benchmark bundle.
+
+## Install
+
+```
+pnpm add -D perfshield
+```
+
+## Quick start
+
+1. Create a benchmark module that exports `benchmarks`:
+
+```js
+export const benchmarks = [
+  {
+    name: "sum",
+    iterations: 1,
+    async fn() {
+      let total = 0;
+      for (let i = 0; i < 10_000; i += 1) {
+        total += i;
+      }
+      return total;
+    },
+  },
+];
+```
+
+2. Create a `perfshield.config.json`:
+
+```json
+{
+  "artifactsDir": ".perfshield",
+  "build": {
+    "command": "pnpm run build:bench",
+    "output": "dist/bench.js"
+  },
+  "engines": [
+    {
+      "name": "node",
+      "command": "node"
+    }
+  ],
+  "sampling": {
+    "minSamples": 30,
+    "timeoutMs": 10000,
+    "conditions": {
+      "absolute": [0],
+      "relative": [0]
+    }
+  },
+  "report": {
+    "formats": ["console", "json"]
+  }
+}
+```
+
+3. Run the workflow:
+
+```
+perfshield prepare --config perfshield.config.json
+perfshield compare --config perfshield.config.json
+```
+
+## Benchmark bundle contract
+
+The build output must be a single ESM file that exports:
+
+```js
+export const benchmarks = [
+  {
+    name: "example",
+    fn: async () => {},
+    setup: async () => {},
+    teardown: async () => {},
+    unit: "ms",
+    iterations: 1,
+    metadata: { tag: "optional" },
+  },
+];
+```
+
+Rules:
+
+- `benchmarks` is required.
+- `fn` is required.
+- `setup`/`teardown` run once per sample (per benchmark, per version).
+- If `setup` throws, the sample fails and the run is aborted.
+- `iterations` repeats `fn` inside a single sample to reduce noise.
+
+## Artifacts
+
+- `prepare` stores a baseline bundle in `artifactsDir` (default `.perfshield`).
+- `compare` uses the stored baseline and the newly built bundle.
+
+## Reports
+
+Supported formats:
+
+- `console`: human‑readable summary.
+- `json`: machine‑readable report.
+
+If any benchmark shows a regression (CI excludes 0 in the slower direction),
+the process exits with code 1.
+
+## Examples
+
+See `examples/simple` for a minimal setup you can run locally.

package/examples/simple/README.md

@@ -0,0 +1,19 @@
+# Simple Example
+
+This example shows the simplest possible setup: a build script that copies a
+benchmark module into `dist/bench.js`, plus a minimal configuration.
+
+## Run
+
+```
+cd examples/simple
+node build.mjs
+perfshield prepare --config perfshield.config.json
+perfshield compare --config perfshield.config.json
+```
+
+## Files
+
+- `bench-source.js`: benchmark module that exports `benchmarks`.
+- `build.mjs`: builds `dist/bench.js`.
+- `perfshield.config.json`: configuration for the CLI.

package/examples/simple/bench-source.js

@@ -0,0 +1,13 @@
+export const benchmarks = [
+  {
+    async fn() {
+      let total = 0;
+      for (let i = 0; i < 10_000; i += 1) {
+        total += i;
+      }
+      return total;
+    },
+    iterations: 1,
+    name: "sum",
+  },
+];

package/examples/simple/build.mjs

@@ -0,0 +1,17 @@
+/* eslint-disable unicorn/prefer-top-level-await */
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+
+const outputPath = resolve("dist/bench.js");
+
+const main = async () => {
+  await mkdir(dirname(outputPath), { recursive: true });
+  const source = await readFile("bench-source.js", "utf8");
+  await writeFile(outputPath, source, "utf8");
+};
+
+main().catch((error) => {
+  // eslint-disable-next-line no-console
+  console.error(error);
+  process.exitCode = 1;
+});

package/examples/simple/perfshield.config.json

@@ -0,0 +1,24 @@
+{
+  "artifactsDir": ".perfshield",
+  "build": {
+    "command": "node build.mjs",
+    "output": "dist/bench.js"
+  },
+  "engines": [
+    {
+      "name": "node",
+      "command": "node"
+    }
+  ],
+  "sampling": {
+    "minSamples": 30,
+    "timeoutMs": 10000,
+    "conditions": {
+      "absolute": [0],
+      "relative": [0]
+    }
+  },
+  "report": {
+    "formats": ["console", "json"]
+  }
+}

package/lib/artifacts.js

@@ -0,0 +1,19 @@
+import { access, copyFile, mkdir } from "node:fs/promises";
+import { basename, join, resolve } from "node:path";
+const getArtifactsDir = config => resolve(config.artifactsDir);
+const getBaselineDir = config => join(getArtifactsDir(config), "baseline");
+export const getBaselinePath = config => join(getBaselineDir(config), basename(config.build.output));
+export const saveBaseline = async (config, outputPath) => {
+  const baselineDir = getBaselineDir(config);
+  await mkdir(baselineDir, {
+    recursive: true
+  });
+  const baselinePath = getBaselinePath(config);
+  await copyFile(outputPath, baselinePath);
+  return baselinePath;
+};
+export const ensureBaseline = async config => {
+  const baselinePath = getBaselinePath(config);
+  await access(baselinePath);
+  return baselinePath;
+};

package/lib/build.js

@@ -0,0 +1,27 @@
+import { spawn } from "node:child_process";
+import { access } from "node:fs/promises";
+import { resolve } from "node:path";
+const runCommand = (command, cwd) => new Promise((resolvePromise, reject) => {
+  const child = spawn(command, {
+    cwd,
+    shell: true,
+    stdio: "inherit"
+  });
+  child.on("error", error => {
+    reject(error);
+  });
+  child.on("exit", code => {
+    if (code === 0) {
+      resolvePromise();
+    } else {
+      reject(new Error(`Build command failed with exit code ${code ?? "?"}.`));
+    }
+  });
+});
+export const runBuild = async build => {
+  const cwd = resolve(build.cwd ?? process.cwd());
+  await runCommand(build.command, cwd);
+  const outputPath = resolve(cwd, build.output);
+  await access(outputPath);
+  return outputPath;
+};

package/lib/cli.js

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+import { ensureBaseline, saveBaseline } from "./artifacts.js";
+import { runBuild } from "./build.js";
+import { ConfigError, formatConfigError, loadConfig } from "./config.js";
+import { getRegressions } from "./regression.js";
+import { renderReports } from "./report/index.js";
+import { runEngineComparison } from "./runner.js";
+const usage = () => {
+  console.error("Usage: perfshield <prepare|compare> [--config path]");
+};
+const getFlagValue = (args, flag) => {
+  const index = args.indexOf(flag);
+  if (index === -1) {
+    return null;
+  }
+  return args[index + 1] ?? null;
+};
+const runPrepare = async config => {
+  const outputPath = await runBuild(config.build);
+  const baselinePath = await saveBaseline(config, outputPath);
+  console.log(`Baseline saved to ${baselinePath}`);
+};
+const runCompare = async config => {
+  const outputPath = await runBuild(config.build);
+  const baselinePath = await ensureBaseline(config);
+  const results = [];
+  for (const engine of config.engines) {
+    results.push(await runEngineComparison({
+      baselinePath,
+      config,
+      currentPath: outputPath,
+      engine
+    }));
+  }
+  const outputs = renderReports(results, config.report.formats);
+  for (const output of outputs) {
+    console.log(output);
+  }
+  const regressions = getRegressions(results);
+  if (regressions.length > 0) {
+    console.error(`Regression detected: ${regressions.length} benchmark(s) are slower.`);
+    process.exitCode = 1;
+  }
+};
+const main = async () => {
+  const args = process.argv.slice(2);
+  const command = args[0];
+  if (command !== "prepare" && command !== "compare") {
+    usage();
+    process.exitCode = 1;
+    return;
+  }
+  const configPath = getFlagValue(args, "--config");
+  let config;
+  try {
+    config = await loadConfig(configPath ?? undefined);
+  } catch (error) {
+    if (error instanceof ConfigError) {
+      console.error(formatConfigError(error));
+      process.exitCode = 1;
+      return;
+    }
+    throw error;
+  }
+  if (command === "prepare") {
+    await runPrepare(config);
+    return;
+  }
+  await runCompare(config);
+};
+main().catch(error => {
+  console.error(error);
+  process.exitCode = 1;
+});

package/lib/config.js

@@ -0,0 +1,328 @@
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+export const DEFAULT_CONFIG_PATH = "perfshield.config.json";
+export class ConfigError extends Error {
+  constructor(configPath, issues) {
+    super(`Invalid configuration in ${configPath}`);
+    this.name = "ConfigError";
+    this.configPath = configPath;
+    this.issues = issues;
+  }
+}
+export const formatConfigError = error => {
+  const lines = [`Invalid configuration in ${error.configPath}:`];
+  for (const issue of error.issues) {
+    lines.push(`- ${issue}`);
+  }
+  return lines.join("\n");
+};
+const addIssue = (issues, message) => {
+  issues.push(message);
+};
+const asObject = (value, label, issues) => {
+  if (value == null || typeof value !== "object" || Array.isArray(value)) {
+    addIssue(issues, `${label} must be an object.`);
+    return null;
+  }
+  return {
+    ...value
+  };
+};
+const asArray = (value, label, issues) => {
+  if (!Array.isArray(value)) {
+    addIssue(issues, `${label} must be an array.`);
+    return null;
+  }
+  return value;
+};
+const asString = (value, label, issues, options) => {
+  if (typeof value !== "string") {
+    addIssue(issues, `${label} must be a string.`);
+    return null;
+  }
+  const trimmed = value.trim();
+  const allowEmpty = options?.allowEmpty === true;
+  if (!allowEmpty && trimmed.length === 0) {
+    addIssue(issues, `${label} must be a non-empty string.`);
+    return null;
+  }
+  return value;
+};
+const asNumber = (value, label, issues, options) => {
+  if (typeof value !== "number" || Number.isNaN(value)) {
+    addIssue(issues, `${label} must be a number.`);
+    return null;
+  }
+  if (!Number.isFinite(value)) {
+    addIssue(issues, `${label} must be a finite number.`);
+    return null;
+  }
+  const requireInteger = options?.integer === true;
+  if (requireInteger && !Number.isInteger(value)) {
+    addIssue(issues, `${label} must be an integer.`);
+    return null;
+  }
+  if (options?.min != null && value < options.min) {
+    addIssue(issues, `${label} must be at least ${options.min}.`);
+    return null;
+  }
+  return value;
+};
+const asStringArray = (value, label, issues, options) => {
+  const entries = asArray(value, label, issues);
+  if (!entries) {
+    return null;
+  }
+  if (options?.minLength != null && entries.length < options.minLength) {
+    addIssue(issues, `${label} must have at least ${options.minLength} entry.`);
+  }
+  return entries.map((entry, index) => {
+    const parsed = asString(entry, `${label}[${index}]`, issues);
+    return parsed == null ? "" : parsed;
+  });
+};
+const asNumberArray = (value, label, issues, options) => {
+  const entries = asArray(value, label, issues);
+  if (!entries) {
+    return null;
+  }
+  if (options?.minLength != null && entries.length < options.minLength) {
+    addIssue(issues, `${label} must have at least ${options.minLength} entry.`);
+  }
+  return entries.map((entry, index) => {
+    const parsed = asNumber(entry, `${label}[${index}]`, issues);
+    return parsed == null ? 0 : parsed;
+  });
+};
+const asEnv = (value, label, issues) => {
+  const env = asObject(value, label, issues);
+  if (!env) {
+    return null;
+  }
+  const next = {};
+  for (const [key, entry] of Object.entries(env)) {
+    const envValue = asString(entry, `${label}.${key}`, issues, {
+      allowEmpty: true
+    });
+    if (envValue != null) {
+      next[key] = envValue;
+    }
+  }
+  return next;
+};
+const validateKeys = (value, allowed, label, issues) => {
+  for (const key of Object.keys(value)) {
+    if (!allowed.includes(key)) {
+      addIssue(issues, `Unknown field: ${label}.${key}.`);
+    }
+  }
+};
+const parseBuildConfig = (value, issues) => {
+  const build = asObject(value, "config.build", issues);
+  if (!build) {
+    return null;
+  }
+  validateKeys(build, ["command", "cwd", "output"], "config.build", issues);
+  const command = asString(build.command, "config.build.command", issues);
+  const output = asString(build.output, "config.build.output", issues);
+  const cwdValue = build.cwd == null ? undefined : asString(build.cwd, "config.build.cwd", issues);
+  const cwd = cwdValue == null ? undefined : cwdValue;
+  if (command == null || output == null) {
+    return null;
+  }
+  return {
+    command,
+    cwd,
+    output
+  };
+};
+const parseEngineConfig = (value, index, issues) => {
+  const engine = asObject(value, `config.engines[${index}]`, issues);
+  if (!engine) {
+    return null;
+  }
+  validateKeys(engine, ["args", "command", "env", "name"], `config.engines[${index}]`, issues);
+  const argsValue = engine.args == null ? undefined : asStringArray(engine.args, `config.engines[${index}].args`, issues);
+  const envValue = engine.env == null ? undefined : asEnv(engine.env, `config.engines[${index}].env`, issues);
+  const args = argsValue == null ? undefined : argsValue;
+  const env = envValue == null ? undefined : envValue;
+  const command = asString(engine.command, `config.engines[${index}].command`, issues);
+  const name = asString(engine.name, `config.engines[${index}].name`, issues);
+  if (command == null || name == null) {
+    return null;
+  }
+  return {
+    args,
+    command,
+    env,
+    name
+  };
+};
+const parseSamplingConditions = (value, issues) => {
+  const conditions = asObject(value, "config.sampling.conditions", issues);
+  if (!conditions) {
+    return null;
+  }
+  validateKeys(conditions, ["absolute", "relative"], "config.sampling.conditions", issues);
+  const absolute = asNumberArray(conditions.absolute, "config.sampling.conditions.absolute", issues, {
+    minLength: 1
+  });
+  const relative = asNumberArray(conditions.relative, "config.sampling.conditions.relative", issues, {
+    minLength: 1
+  });
+  if (!absolute || !relative) {
+    return null;
+  }
+  return {
+    absolute,
+    relative
+  };
+};
+const parseSamplingConfig = (value, issues) => {
+  const sampling = asObject(value, "config.sampling", issues);
+  if (!sampling) {
+    return null;
+  }
+  validateKeys(sampling, ["conditions", "minSamples", "timeoutMs"], "config.sampling", issues);
+  const minSamples = asNumber(sampling.minSamples, "config.sampling.minSamples", issues, {
+    integer: true,
+    min: 1
+  });
+  const timeoutMs = asNumber(sampling.timeoutMs, "config.sampling.timeoutMs", issues, {
+    integer: true,
+    min: 1
+  });
+  const conditions = parseSamplingConditions(sampling.conditions, issues);
+  if (minSamples == null || timeoutMs == null || !conditions) {
+    return null;
+  }
+  return {
+    conditions,
+    minSamples,
+    timeoutMs
+  };
+};
+const parseReportConfig = (value, issues) => {
+  const report = asObject(value, "config.report", issues);
+  if (!report) {
+    return null;
+  }
+  validateKeys(report, ["formats"], "config.report", issues);
+  const rawFormats = asStringArray(report.formats, "config.report.formats", issues, {
+    minLength: 1
+  });
+  if (!rawFormats) {
+    return null;
+  }
+  const formats = [];
+  for (const format of rawFormats) {
+    if (format == null) {
+      continue;
+    }
+    if (format !== "console" && format !== "json") {
+      addIssue(issues, `config.report.formats only supports "console" or "json" (received "${format}").`);
+      continue;
+    }
+    formats.push(format);
+  }
+  if (formats.length === 0) {
+    return null;
+  }
+  return {
+    formats
+  };
+};
+const parseConfig = (raw, issues) => {
+  const config = asObject(raw, "config", issues);
+  if (!config) {
+    return null;
+  }
+  validateKeys(config, ["artifactsDir", "build", "engines", "report", "sampling"], "config", issues);
+  const artifactsDir = asString(config.artifactsDir, "config.artifactsDir", issues);
+  const build = parseBuildConfig(config.build, issues);
+  const enginesRaw = asArray(config.engines, "config.engines", issues);
+  const engines = [];
+  if (enginesRaw) {
+    if (enginesRaw.length === 0) {
+      addIssue(issues, "config.engines must contain at least one entry.");
+    }
+    enginesRaw.forEach((entry, index) => {
+      const parsed = parseEngineConfig(entry, index, issues);
+      if (parsed) {
+        engines.push(parsed);
+      }
+    });
+  }
+  const report = parseReportConfig(config.report, issues);
+  const sampling = parseSamplingConfig(config.sampling, issues);
+  if (artifactsDir == null || build == null || report == null || sampling == null || engines.length === 0) {
+    return null;
+  }
+  return {
+    artifactsDir,
+    build,
+    engines,
+    report,
+    sampling
+  };
+};
+const positionToLineColumn = (contents, index) => {
+  let line = 1;
+  let column = 1;
+  for (let i = 0; i < Math.min(index, contents.length); i += 1) {
+    if (contents[i] === "\n") {
+      line += 1;
+      column = 1;
+    } else {
+      column += 1;
+    }
+  }
+  return {
+    column,
+    line
+  };
+};
+const describeJsonError = (contents, message) => {
+  const match = message.match(/position\s+(\d+)/i);
+  if (match) {
+    const index = Number(match[1]);
+    if (Number.isFinite(index)) {
+      const {
+        column,
+        line
+      } = positionToLineColumn(contents, index);
+      return `Invalid JSON: ${message} (line ${line}, column ${column}).`;
+    }
+  }
+  return `Invalid JSON: ${message}.`;
+};
+const coerceConfig = (raw, configPath) => {
+  const issues = [];
+  const config = parseConfig(raw, issues);
+  if (!config || issues.length > 0) {
+    throw new ConfigError(configPath, issues.length > 0 ? issues : ["Configuration is invalid."]);
+  }
+  return config;
+};
+export const loadConfig = async configPath => {
+  const resolvedPath = resolve(configPath ?? DEFAULT_CONFIG_PATH);
+  let contents;
+  try {
+    contents = await readFile(resolvedPath, "utf8");
+  } catch (error) {
+    const readError = error;
+    if (readError.code === "ENOENT") {
+      throw new ConfigError(resolvedPath, [`Config file not found. Expected a file at ${resolvedPath}.`]);
+    }
+    throw new ConfigError(resolvedPath, [`Failed to read config file: ${readError.message ?? "Unknown error"}.`]);
+  }
+  let raw;
+  try {
+    raw = JSON.parse(contents);
+  } catch (error) {
+    const parseError = error;
+    const issue = describeJsonError(contents, parseError.message ?? "Unable to parse JSON.");
+    throw new ConfigError(resolvedPath, [issue]);
+  }
+  return coerceConfig(raw, resolvedPath);
+};