perfshield 0.0.5 → 0.0.7

package/README.md

@@ -11,7 +11,7 @@ runtime JS engines (Node/V8 today).
 - Saves a baseline bundle (`prepare`).
 - Builds the current bundle and compares it to the baseline (`compare`).
 - Reports results in console and/or JSON.
-- Exits with code 1 when a regression is detected (CI excludes 0 in the slower direction).
+- Exits with code 1 when a regression is detected (both relative and absolute CIs exclude 0 in the slower direction).
 
 ## Requirements
 
@@ -77,6 +77,15 @@ perfshield prepare --config perfshield.config.json
 perfshield compare --config perfshield.config.json
 ```
 
+4. (Optional) Calibrate sampling defaults based on the prepared baseline:
+
+```
+perfshield calibrate --config perfshield.config.json
+```
+
+This prints a JSON snippet with recommended `sampling` values you can paste into
+your config.
+
 ## Benchmark bundle contract
 
 The build output must be a single ESM file that exports:
@@ -115,7 +124,7 @@ Supported formats:
 - `console`: human‑readable summary.
 - `json`: machine‑readable report.
 
-If any benchmark shows a regression (CI excludes 0 in the slower direction),
+If any benchmark shows a regression (both relative and absolute CIs exclude 0 in the slower direction),
 the process exits with code 1.
 
 ## Examples

package/lib/calibrate.js

@@ -0,0 +1,145 @@
+import { resolve } from "node:path";
+import jstat from "jstat";
+import { createNodeHarness } from "./engines/node.js";
+import { buildHarnessIfNeeded, getHarnessPath } from "./harness.js";
+import { summaryStats } from "./stats.js";
+const calibrationSampleCount = 20;
+const calibrationMinSampleMs = 2;
+const targetMinSampleMs = 5;
+const minRecommendedSamples = 5;
+const minTimeoutMs = 1000;
+const timeoutSafetyFactor = 1.5;
+const confidenceLevel = 0.95;
+const defaultMaxRelativeMargin = 0.05;
+const computeIterationsForTarget = (iterationsBase, warmupMs, targetMs) => {
+  if (targetMs <= 0) {
+    return iterationsBase;
+  }
+  const perIter = warmupMs / iterationsBase;
+  if (!Number.isFinite(perIter) || perIter <= 0) {
+    return iterationsBase;
+  }
+  return Math.max(iterationsBase, Math.ceil(targetMs / perIter));
+};
+const collectWarmups = async (harness, benchmarks) => {
+  const warmups = [];
+  for (let index = 0; index < benchmarks.length; index += 1) {
+    const descriptor = benchmarks[index];
+    const result = await harness.runSample({
+      index,
+      iterations: descriptor.iterations,
+      version: "baseline"
+    });
+    warmups.push(result.durationMs);
+  }
+  return warmups;
+};
+const collectCalibrationSamples = async (harness, benchmarks, iterationOverrides) => {
+  const samples = benchmarks.map(() => []);
+  for (let iteration = 0; iteration < calibrationSampleCount; iteration += 1) {
+    for (let index = 0; index < benchmarks.length; index += 1) {
+      const result = await harness.runSample({
+        index,
+        iterations: iterationOverrides[index],
+        version: "baseline"
+      });
+      samples[index].push(result.durationMs);
+    }
+  }
+  return samples;
+};
+const computeRequiredSamples = (stats, targetMargin, tValue) => {
+  if (targetMargin <= 0) {
+    return minRecommendedSamples;
+  }
+  const rsd = stats.relativeStandardDeviation;
+  if (!Number.isFinite(rsd) || rsd <= 0) {
+    return minRecommendedSamples;
+  }
+  const required = Math.ceil((tValue * rsd / targetMargin) ** 2);
+  if (!Number.isFinite(required) || required <= 0) {
+    return minRecommendedSamples;
+  }
+  return Math.max(minRecommendedSamples, required);
+};
+const computeMinTimeMs = warmups => {
+  const finiteWarmups = warmups.filter(value => Number.isFinite(value));
+  if (finiteWarmups.length === 0) {
+    return targetMinSampleMs;
+  }
+  const fastest = Math.min(...finiteWarmups);
+  if (fastest < targetMinSampleMs) {
+    return targetMinSampleMs;
+  }
+  return 0;
+};
+const computeTimeoutMs = (warmups, minSamples, minTimeMs, delayMs) => {
+  let perRound = 0;
+  for (const warmup of warmups) {
+    const base = Number.isFinite(warmup) && warmup > 0 ? warmup : minTimeMs;
+    perRound += Math.max(base, minTimeMs);
+  }
+  const sampleCount = Math.max(1, minSamples);
+  const totalSamples = sampleCount * warmups.length;
+  const estimated = perRound * sampleCount + delayMs * Math.max(0, totalSamples - 1);
+  const timeout = Math.ceil(estimated * timeoutSafetyFactor);
+  if (!Number.isFinite(timeout) || timeout <= 0) {
+    return minTimeoutMs;
+  }
+  return Math.max(minTimeoutMs, timeout);
+};
+const calibrateEngine = async (engine, baselinePath, maxRelativeMargin, delayMs, harnessPath) => {
+  const resolvedBaseline = resolve(baselinePath);
+  const harness = await createNodeHarness(engine, harnessPath, resolvedBaseline, resolvedBaseline);
+  try {
+    const benchmarks = await harness.listBenchmarks();
+    const warmups = await collectWarmups(harness, benchmarks);
+    const iterationOverrides = benchmarks.map((benchmark, index) => computeIterationsForTarget(benchmark.iterations ?? 1, warmups[index], calibrationMinSampleMs));
+    const samples = await collectCalibrationSamples(harness, benchmarks, iterationOverrides);
+    const benchmarkStats = benchmarks.map((benchmark, index) => ({
+      benchmark,
+      stats: summaryStats(samples[index]),
+      warmupMs: warmups[index]
+    }));
+    const tValue = jstat.studentt.inv(1 - (1 - confidenceLevel) / 2, Math.max(1, calibrationSampleCount - 1));
+    const minSamples = benchmarkStats.reduce((max, entry) => {
+      const required = computeRequiredSamples(entry.stats, maxRelativeMargin, tValue);
+      return Math.max(max, required);
+    }, minRecommendedSamples);
+    const minTimeMs = computeMinTimeMs(warmups);
+    const timeoutMs = computeTimeoutMs(warmups, minSamples, minTimeMs, delayMs);
+    return {
+      engine,
+      minSamples,
+      minTimeMs,
+      timeoutMs
+    };
+  } finally {
+    await harness.close();
+  }
+};
+export const calibrateSampling = async (config, baselinePath) => {
+  const maxRelativeMargin = config.sampling.maxRelativeMargin ?? defaultMaxRelativeMargin;
+  const delayMs = config.sampling.delayMs ?? 0;
+  const harnessArtifact = await buildHarnessIfNeeded(getHarnessPath());
+  try {
+    const engineResults = [];
+    for (const engine of config.engines) {
+      engineResults.push(await calibrateEngine(engine, baselinePath, maxRelativeMargin, delayMs, harnessArtifact.path));
+    }
+    const minSamples = engineResults.reduce((max, entry) => Math.max(max, entry.minSamples), minRecommendedSamples);
+    const minTimeMs = engineResults.reduce((max, entry) => Math.max(max, entry.minTimeMs), 0);
+    const timeoutMs = engineResults.reduce((max, entry) => Math.max(max, entry.timeoutMs), minTimeoutMs);
+    return {
+      conditions: config.sampling.conditions,
+      maxRelativeMargin,
+      minSamples,
+      minTimeMs,
+      timeoutMs
+    };
+  } finally {
+    if (harnessArtifact.cleanup) {
+      await harnessArtifact.cleanup();
+    }
+  }
+};

package/lib/cli.js

@@ -1,13 +1,14 @@
 #!/usr/bin/env node
 import { WriteStream } from "node:tty";
 import { ensureBaseline, saveBaseline } from "./artifacts.js";
+import { calibrateSampling } from "./calibrate.js";
 import { runBuild } from "./build.js";
 import { ConfigError, formatConfigError, loadConfig } from "./config.js";
 import { getRegressions } from "./regression.js";
 import { renderReports } from "./report/index.js";
 import { runEngineComparison } from "./runner.js";
 const usage = () => {
-  console.error("Usage: perfshield <prepare|compare> [--config path]");
+  console.error("Usage: perfshield <prepare|compare|calibrate> [--config path]");
 };
 const getFlagValue = (args, flag) => {
   const index = args.indexOf(flag);
@@ -99,10 +100,17 @@ const runCompare = async config => {
     process.exitCode = 1;
   }
 };
+const runCalibrate = async config => {
+  const baselinePath = await ensureBaseline(config);
+  const sampling = await calibrateSampling(config, baselinePath);
+  console.log(JSON.stringify({
+    sampling
+  }, null, 2));
+};
 const main = async () => {
   const args = process.argv.slice(2);
   const command = args[0];
-  if (command !== "prepare" && command !== "compare") {
+  if (command !== "prepare" && command !== "compare" && command !== "calibrate") {
     usage();
     process.exitCode = 1;
     return;
@@ -123,6 +131,10 @@ const main = async () => {
     await runPrepare(config);
     return;
   }
+  if (command === "calibrate") {
+    await runCalibrate(config);
+    return;
+  }
   await runCompare(config);
 };
 main().catch(error => {

package/lib/harness.js

@@ -0,0 +1,51 @@
+import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { transformFileAsync } from "@babel/core";
+const harnessTempPrefix = "perfshield-harness-";
+const hasBabelConfig = async () => {
+  const configPath = resolve(process.cwd(), "babel.config.cjs");
+  try {
+    await access(configPath);
+    return true;
+  } catch {
+    return false;
+  }
+};
+export const getHarnessPath = () => {
+  const override = process.env.WEB_BENCHMARKER_HARNESS_PATH;
+  if (override != null) {
+    return override;
+  }
+  return fileURLToPath(new URL("./engines/node-harness.js", import.meta.url).toString());
+};
+export const buildHarnessIfNeeded = async sourcePath => {
+  const contents = await readFile(sourcePath, "utf8");
+  if (!contents.includes("import type") && !contents.includes("@flow")) {
+    return {
+      cleanup: null,
+      path: sourcePath
+    };
+  }
+  const usesConfig = await hasBabelConfig();
+  const result = await transformFileAsync(sourcePath, {
+    configFile: usesConfig ? resolve(process.cwd(), "babel.config.cjs") : false,
+    presets: usesConfig ? [] : ["@babel/preset-flow"]
+  });
+  if (!result || !result.code) {
+    throw new Error("Failed to compile node harness.");
+  }
+  const dir = await mkdtemp(join(tmpdir(), harnessTempPrefix));
+  const harnessPath = join(dir, "node-harness.js");
+  await writeFile(harnessPath, result.code, "utf8");
+  return {
+    cleanup: async () => {
+      await rm(dir, {
+        force: true,
+        recursive: true
+      });
+    },
+    path: harnessPath
+  };
+};

package/lib/regression.js

@@ -3,7 +3,7 @@ export const getRegressions = results => {
   const findings = [];
   for (const result of results) {
     for (const entry of result.benchmarks) {
-      if (isPositiveInterval(entry.difference.relative.ci)) {
+      if (isPositiveInterval(entry.difference.relative.ci) && isPositiveInterval(entry.difference.absolute.ci)) {
         findings.push({
           benchmark: entry.benchmark.name,
           engine: result.engine.name,

package/lib/report/console.js

@@ -10,11 +10,13 @@ const formatRelativeInterval = (interval, decimals) => formatInterval({
   low: interval.low * 100
 }, decimals, "%");
 const formatRelativeValue = (value, decimals) => `${formatNumber(value * 100, decimals)}%`;
-const classifyDifference = interval => {
-  if (interval.low > 0 && interval.high > 0) {
+const isPositiveInterval = interval => interval.low > 0 && interval.high > 0;
+const isNegativeInterval = interval => interval.low < 0 && interval.high < 0;
+const classifyDifference = difference => {
+  if (isPositiveInterval(difference.relative.ci) && isPositiveInterval(difference.absolute.ci)) {
     return "regression";
   }
-  if (interval.low < 0 && interval.high < 0) {
+  if (isNegativeInterval(difference.relative.ci) && isNegativeInterval(difference.absolute.ci)) {
     return "improvement";
   }
   return "no significant change";
@@ -28,7 +30,7 @@ export const renderConsoleReport = results => {
     lines.push(`Engine: ${result.engine.name}`);
     for (const entry of result.benchmarks) {
       const unit = entry.benchmark.unit != null ? ` ${entry.benchmark.unit}` : "";
-      const status = classifyDifference(entry.difference.relative.ci);
+      const status = classifyDifference(entry.difference);
       if (status === "regression") {
         regressions += 1;
       } else if (status === "improvement") {
@@ -36,7 +38,7 @@ export const renderConsoleReport = results => {
       } else {
         neutral += 1;
       }
-      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    Result: ${status} (mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)})`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
+      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    Result: ${status} (mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)})`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`, `    diff abs mean=${formatNumber(entry.difference.absolute.mean, 4)}${unit} ci=${formatInterval(entry.difference.absolute.ci, 4)}`];
       lines.push(...benchmarkLines);
     }
     lines.push(`  Summary: regressions=${regressions} improvements=${improvements} neutral=${neutral}`, "");

package/lib/runner.js

@@ -1,13 +1,9 @@
-import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { transformFileAsync } from "@babel/core";
+import { resolve } from "node:path";
 import { createNodeHarness } from "./engines/node.js";
+import { buildHarnessIfNeeded, getHarnessPath } from "./harness.js";
 import { computeRelativeDifferenceFromSamples, summaryStats } from "./stats.js";
 const versions = ["baseline", "current"];
 const autoSampleBatchSize = 10;
-const harnessTempPrefix = "perfshield-harness-";
 const defaultMinTimeMs = 20;
 const defaultMaxRelativeMargin = 0.05;
 const getVersionOrder = seed => {
@@ -16,51 +12,6 @@ const getVersionOrder = seed => {
   }
   return [versions[1], versions[0]];
 };
-const getHarnessPath = () => {
-  const override = process.env.WEB_BENCHMARKER_HARNESS_PATH;
-  if (override != null) {
-    return override;
-  }
-  return fileURLToPath(new URL("./engines/node-harness.js", import.meta.url).toString());
-};
-const hasBabelConfig = async () => {
-  const configPath = resolve(process.cwd(), "babel.config.cjs");
-  try {
-    await access(configPath);
-    return true;
-  } catch {
-    return false;
-  }
-};
-const buildHarnessIfNeeded = async sourcePath => {
-  const contents = await readFile(sourcePath, "utf8");
-  if (!contents.includes("import type") && !contents.includes("@flow")) {
-    return {
-      cleanup: null,
-      path: sourcePath
-    };
-  }
-  const usesConfig = await hasBabelConfig();
-  const result = await transformFileAsync(sourcePath, {
-    configFile: usesConfig ? resolve(process.cwd(), "babel.config.cjs") : false,
-    presets: usesConfig ? [] : ["@babel/preset-flow"]
-  });
-  if (!result || !result.code) {
-    throw new Error("Failed to compile node harness.");
-  }
-  const dir = await mkdtemp(join(tmpdir(), harnessTempPrefix));
-  const harnessPath = join(dir, "node-harness.js");
-  await writeFile(harnessPath, result.code, "utf8");
-  return {
-    cleanup: async () => {
-      await rm(dir, {
-        force: true,
-        recursive: true
-      });
-    },
-    path: harnessPath
-  };
-};
 const sleep = async delayMs => {
   if (delayMs <= 0) {
     return;

package/lib/stats.js

@@ -34,6 +34,10 @@ export const samplingDistributionOfRelativeDifferenceOfMeans = (a, b) => ({
   mean: (b.mean - a.mean) / a.mean,
   variance: (a.variance * b.mean * b.mean + b.variance * a.mean * a.mean) / (a.mean * a.mean * a.mean * a.mean)
 });
+export const samplingDistributionOfDifferenceOfMeans = (a, b) => ({
+  mean: b.mean - a.mean,
+  variance: a.variance + b.variance
+});
 export const summaryStats = values => {
   if (values.length === 0) {
     throw new Error("Cannot compute stats for an empty sample set.");
@@ -68,9 +72,14 @@ export const computeDifference = (baseline, current) => {
     mean: current.mean,
     variance: current.variance
   }, current.size);
+  const absoluteDist = samplingDistributionOfDifferenceOfMeans(baselineDist, currentDist);
   const relativeDist = samplingDistributionOfRelativeDifferenceOfMeans(baselineDist, currentDist);
   const size = Math.min(baseline.size, current.size);
   return {
+    absolute: {
+      ci: confidenceInterval95(absoluteDist, size),
+      mean: absoluteDist.mean
+    },
     relative: {
       ci: confidenceInterval95(relativeDist, size),
       mean: relativeDist.mean
@@ -99,7 +108,26 @@ const computePairedRelativeStats = (baselineSamples, currentSamples) => {
     mean: diffStats.mean
   };
 };
+const computePairedAbsoluteStats = (baselineSamples, currentSamples) => {
+  const size = Math.min(baselineSamples.length, currentSamples.length);
+  if (size === 0) {
+    throw new Error("Cannot compute differences with empty sample sets.");
+  }
+  const diffs = [];
+  for (let index = 0; index < size; index += 1) {
+    diffs.push(currentSamples[index] - baselineSamples[index]);
+  }
+  const diffStats = summaryStats(diffs);
+  return {
+    ci: confidenceInterval95(samplingDistributionOfTheMean({
+      mean: diffStats.mean,
+      variance: diffStats.variance
+    }, diffStats.size), diffStats.size),
+    mean: diffStats.mean
+  };
+};
 export const computeRelativeDifferenceFromSamples = (baselineSamples, currentSamples) => ({
+  absolute: computePairedAbsoluteStats(baselineSamples, currentSamples),
   relative: computePairedRelativeStats(baselineSamples, currentSamples)
 });
 export const computeDifferences = stats => stats.map(result => ({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "perfshield",
-  "version": "0.0.5",
+  "version": "0.0.7",
   "description": "A tool for doing web benchmarking across multiple JS engines and with statistical signifigance",
   "license": "MIT",
   "type": "module",