perfshield 0.0.4 → 0.0.7

package/README.md

@@ -11,7 +11,7 @@ runtime JS engines (Node/V8 today).
 - Saves a baseline bundle (`prepare`).
 - Builds the current bundle and compares it to the baseline (`compare`).
 - Reports results in console and/or JSON.
-- Exits with code 1 when a regression is detected (CI excludes 0 in the slower direction).
+- Exits with code 1 when a regression is detected (both relative and absolute CIs exclude 0 in the slower direction).
 
 ## Requirements
 
@@ -77,6 +77,15 @@ perfshield prepare --config perfshield.config.json
 perfshield compare --config perfshield.config.json
 ```
 
+4. (Optional) Calibrate sampling defaults based on the prepared baseline:
+
+```
+perfshield calibrate --config perfshield.config.json
+```
+
+This prints a JSON snippet with recommended `sampling` values you can paste into
+your config.
+
 ## Benchmark bundle contract
 
 The build output must be a single ESM file that exports:
@@ -115,7 +124,7 @@ Supported formats:
 - `console`: human‑readable summary.
 - `json`: machine‑readable report.
 
-If any benchmark shows a regression (CI excludes 0 in the slower direction),
+If any benchmark shows a regression (both relative and absolute CIs exclude 0 in the slower direction),
 the process exits with code 1.
 
 ## Examples

package/lib/calibrate.js

@@ -0,0 +1,145 @@
+import { resolve } from "node:path";
+import jstat from "jstat";
+import { createNodeHarness } from "./engines/node.js";
+import { buildHarnessIfNeeded, getHarnessPath } from "./harness.js";
+import { summaryStats } from "./stats.js";
+const calibrationSampleCount = 20;
+const calibrationMinSampleMs = 2;
+const targetMinSampleMs = 5;
+const minRecommendedSamples = 5;
+const minTimeoutMs = 1000;
+const timeoutSafetyFactor = 1.5;
+const confidenceLevel = 0.95;
+const defaultMaxRelativeMargin = 0.05;
+const computeIterationsForTarget = (iterationsBase, warmupMs, targetMs) => {
+  if (targetMs <= 0) {
+    return iterationsBase;
+  }
+  const perIter = warmupMs / iterationsBase;
+  if (!Number.isFinite(perIter) || perIter <= 0) {
+    return iterationsBase;
+  }
+  return Math.max(iterationsBase, Math.ceil(targetMs / perIter));
+};
+const collectWarmups = async (harness, benchmarks) => {
+  const warmups = [];
+  for (let index = 0; index < benchmarks.length; index += 1) {
+    const descriptor = benchmarks[index];
+    const result = await harness.runSample({
+      index,
+      iterations: descriptor.iterations,
+      version: "baseline"
+    });
+    warmups.push(result.durationMs);
+  }
+  return warmups;
+};
+const collectCalibrationSamples = async (harness, benchmarks, iterationOverrides) => {
+  const samples = benchmarks.map(() => []);
+  for (let iteration = 0; iteration < calibrationSampleCount; iteration += 1) {
+    for (let index = 0; index < benchmarks.length; index += 1) {
+      const result = await harness.runSample({
+        index,
+        iterations: iterationOverrides[index],
+        version: "baseline"
+      });
+      samples[index].push(result.durationMs);
+    }
+  }
+  return samples;
+};
+const computeRequiredSamples = (stats, targetMargin, tValue) => {
+  if (targetMargin <= 0) {
+    return minRecommendedSamples;
+  }
+  const rsd = stats.relativeStandardDeviation;
+  if (!Number.isFinite(rsd) || rsd <= 0) {
+    return minRecommendedSamples;
+  }
+  const required = Math.ceil((tValue * rsd / targetMargin) ** 2);
+  if (!Number.isFinite(required) || required <= 0) {
+    return minRecommendedSamples;
+  }
+  return Math.max(minRecommendedSamples, required);
+};
+const computeMinTimeMs = warmups => {
+  const finiteWarmups = warmups.filter(value => Number.isFinite(value));
+  if (finiteWarmups.length === 0) {
+    return targetMinSampleMs;
+  }
+  const fastest = Math.min(...finiteWarmups);
+  if (fastest < targetMinSampleMs) {
+    return targetMinSampleMs;
+  }
+  return 0;
+};
+const computeTimeoutMs = (warmups, minSamples, minTimeMs, delayMs) => {
+  let perRound = 0;
+  for (const warmup of warmups) {
+    const base = Number.isFinite(warmup) && warmup > 0 ? warmup : minTimeMs;
+    perRound += Math.max(base, minTimeMs);
+  }
+  const sampleCount = Math.max(1, minSamples);
+  const totalSamples = sampleCount * warmups.length;
+  const estimated = perRound * sampleCount + delayMs * Math.max(0, totalSamples - 1);
+  const timeout = Math.ceil(estimated * timeoutSafetyFactor);
+  if (!Number.isFinite(timeout) || timeout <= 0) {
+    return minTimeoutMs;
+  }
+  return Math.max(minTimeoutMs, timeout);
+};
+const calibrateEngine = async (engine, baselinePath, maxRelativeMargin, delayMs, harnessPath) => {
+  const resolvedBaseline = resolve(baselinePath);
+  const harness = await createNodeHarness(engine, harnessPath, resolvedBaseline, resolvedBaseline);
+  try {
+    const benchmarks = await harness.listBenchmarks();
+    const warmups = await collectWarmups(harness, benchmarks);
+    const iterationOverrides = benchmarks.map((benchmark, index) => computeIterationsForTarget(benchmark.iterations ?? 1, warmups[index], calibrationMinSampleMs));
+    const samples = await collectCalibrationSamples(harness, benchmarks, iterationOverrides);
+    const benchmarkStats = benchmarks.map((benchmark, index) => ({
+      benchmark,
+      stats: summaryStats(samples[index]),
+      warmupMs: warmups[index]
+    }));
+    const tValue = jstat.studentt.inv(1 - (1 - confidenceLevel) / 2, Math.max(1, calibrationSampleCount - 1));
+    const minSamples = benchmarkStats.reduce((max, entry) => {
+      const required = computeRequiredSamples(entry.stats, maxRelativeMargin, tValue);
+      return Math.max(max, required);
+    }, minRecommendedSamples);
+    const minTimeMs = computeMinTimeMs(warmups);
+    const timeoutMs = computeTimeoutMs(warmups, minSamples, minTimeMs, delayMs);
+    return {
+      engine,
+      minSamples,
+      minTimeMs,
+      timeoutMs
+    };
+  } finally {
+    await harness.close();
+  }
+};
+export const calibrateSampling = async (config, baselinePath) => {
+  const maxRelativeMargin = config.sampling.maxRelativeMargin ?? defaultMaxRelativeMargin;
+  const delayMs = config.sampling.delayMs ?? 0;
+  const harnessArtifact = await buildHarnessIfNeeded(getHarnessPath());
+  try {
+    const engineResults = [];
+    for (const engine of config.engines) {
+      engineResults.push(await calibrateEngine(engine, baselinePath, maxRelativeMargin, delayMs, harnessArtifact.path));
+    }
+    const minSamples = engineResults.reduce((max, entry) => Math.max(max, entry.minSamples), minRecommendedSamples);
+    const minTimeMs = engineResults.reduce((max, entry) => Math.max(max, entry.minTimeMs), 0);
+    const timeoutMs = engineResults.reduce((max, entry) => Math.max(max, entry.timeoutMs), minTimeoutMs);
+    return {
+      conditions: config.sampling.conditions,
+      maxRelativeMargin,
+      minSamples,
+      minTimeMs,
+      timeoutMs
+    };
+  } finally {
+    if (harnessArtifact.cleanup) {
+      await harnessArtifact.cleanup();
+    }
+  }
+};

package/lib/cli.js

@@ -1,12 +1,14 @@
 #!/usr/bin/env node
+import { WriteStream } from "node:tty";
 import { ensureBaseline, saveBaseline } from "./artifacts.js";
+import { calibrateSampling } from "./calibrate.js";
 import { runBuild } from "./build.js";
 import { ConfigError, formatConfigError, loadConfig } from "./config.js";
 import { getRegressions } from "./regression.js";
 import { renderReports } from "./report/index.js";
 import { runEngineComparison } from "./runner.js";
 const usage = () => {
-  console.error("Usage: perfshield <prepare|compare> [--config path]");
+  console.error("Usage: perfshield <prepare|compare|calibrate> [--config path]");
 };
 const getFlagValue = (args, flag) => {
   const index = args.indexOf(flag);
@@ -20,17 +22,73 @@ const runPrepare = async config => {
   const baselinePath = await saveBaseline(config, outputPath);
   console.log(`Baseline saved to ${baselinePath}`);
 };
+const isStderrTTY = () => {
+  return process.stderr instanceof WriteStream;
+};
+const createProgressReporter = engineName => {
+  let lastWrite = 0;
+  let lastLine = "";
+  const isTTY = isStderrTTY();
+  const writeLine = line => {
+    if (isTTY) {
+      const padded = line.padEnd(80, " ");
+      process.stderr.write(`\r${padded}`);
+    } else if (line !== lastLine) {
+      process.stderr.write(`${line}\n`);
+    }
+    lastLine = line;
+  };
+  const report = event => {
+    const now = Date.now();
+    if (isTTY && now - lastWrite < 100) {
+      return;
+    }
+    lastWrite = now;
+    let message = "";
+    if (event.phase === "warmup") {
+      const current = (event.benchmarkIndex ?? 0) + 1;
+      const total = event.benchmarkCount ?? 0;
+      const name = event.benchmarkName == null ? "" : ` ${event.benchmarkName}`;
+      message = `Warmup ${current}/${total}${name} (${engineName})`;
+    } else if (event.phase === "samples") {
+      const completed = event.completed ?? 0;
+      const total = event.total ?? 0;
+      const percent = total > 0 ? Math.round(completed / total * 100) : 0;
+      message = `Sampling ${completed}/${total} (${percent}%) (${engineName})`;
+    } else {
+      const completed = event.completed ?? 0;
+      const elapsed = event.elapsedMs != null ? Math.round(event.elapsedMs / 1000) : 0;
+      message = `Auto-sampling ${completed} samples (${elapsed}s) (${engineName})`;
+    }
+    writeLine(message);
+  };
+  const finish = () => {
+    if (isTTY && lastLine) {
+      process.stderr.write("\n");
+    }
+  };
+  return {
+    finish,
+    report
+  };
+};
 const runCompare = async config => {
   const outputPath = await runBuild(config.build);
   const baselinePath = await ensureBaseline(config);
   const results = [];
+  const showProgress = config.report.formats.includes("console") || isStderrTTY();
   for (const engine of config.engines) {
+    const progress = showProgress ? createProgressReporter(engine.name) : null;
     results.push(await runEngineComparison({
       baselinePath,
       config,
       currentPath: outputPath,
-      engine
+      engine,
+      progress: progress ? progress.report : undefined
     }));
+    if (progress) {
+      progress.finish();
+    }
   }
   const outputs = renderReports(results, config.report.formats);
   for (const output of outputs) {
@@ -42,10 +100,17 @@ const runCompare = async config => {
     process.exitCode = 1;
   }
 };
+const runCalibrate = async config => {
+  const baselinePath = await ensureBaseline(config);
+  const sampling = await calibrateSampling(config, baselinePath);
+  console.log(JSON.stringify({
+    sampling
+  }, null, 2));
+};
 const main = async () => {
   const args = process.argv.slice(2);
   const command = args[0];
-  if (command !== "prepare" && command !== "compare") {
+  if (command !== "prepare" && command !== "compare" && command !== "calibrate") {
     usage();
     process.exitCode = 1;
     return;
@@ -66,6 +131,10 @@ const main = async () => {
     await runPrepare(config);
     return;
   }
+  if (command === "calibrate") {
+    await runCalibrate(config);
+    return;
+  }
   await runCompare(config);
 };
 main().catch(error => {

package/lib/config.js

@@ -163,7 +163,7 @@ const parseSamplingConfig = (value, issues) => {
   if (!sampling) {
     return null;
   }
-  validateKeys(sampling, ["conditions", "minSamples", "timeoutMs"], "config.sampling", issues);
+  validateKeys(sampling, ["conditions", "delayMs", "maxRelativeMargin", "minSamples", "minTimeMs", "timeoutMs"], "config.sampling", issues);
   const minSamples = asNumber(sampling.minSamples, "config.sampling.minSamples", issues, {
     integer: true,
     min: 1
@@ -172,15 +172,32 @@ const parseSamplingConfig = (value, issues) => {
     integer: true,
     min: 1
   });
+  const minTimeMs = sampling.minTimeMs == null ? undefined : asNumber(sampling.minTimeMs, "config.sampling.minTimeMs", issues, {
+    integer: true,
+    min: 0
+  });
+  const delayMs = sampling.delayMs == null ? undefined : asNumber(sampling.delayMs, "config.sampling.delayMs", issues, {
+    integer: true,
+    min: 0
+  });
+  const maxRelativeMargin = sampling.maxRelativeMargin == null ? undefined : asNumber(sampling.maxRelativeMargin, "config.sampling.maxRelativeMargin", issues, {
+    min: 0
+  });
+  if (maxRelativeMargin != null && maxRelativeMargin > 1) {
+    addIssue(issues, "config.sampling.maxRelativeMargin must be at most 1.");
+  }
   const conditions = asNumberArray(sampling.conditions, "config.sampling.conditions", issues, {
     minLength: 1
   });
-  if (minSamples == null || timeoutMs == null || !conditions) {
+  if (minSamples == null || timeoutMs == null || !conditions || minTimeMs === null || delayMs === null || maxRelativeMargin === null) {
     return null;
   }
   return {
     conditions,
+    delayMs,
+    maxRelativeMargin,
     minSamples,
+    minTimeMs,
     timeoutMs
   };
 };

package/lib/harness.js

@@ -0,0 +1,51 @@
+import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { transformFileAsync } from "@babel/core";
+const harnessTempPrefix = "perfshield-harness-";
+const hasBabelConfig = async () => {
+  const configPath = resolve(process.cwd(), "babel.config.cjs");
+  try {
+    await access(configPath);
+    return true;
+  } catch {
+    return false;
+  }
+};
+export const getHarnessPath = () => {
+  const override = process.env.WEB_BENCHMARKER_HARNESS_PATH;
+  if (override != null) {
+    return override;
+  }
+  return fileURLToPath(new URL("./engines/node-harness.js", import.meta.url).toString());
+};
+export const buildHarnessIfNeeded = async sourcePath => {
+  const contents = await readFile(sourcePath, "utf8");
+  if (!contents.includes("import type") && !contents.includes("@flow")) {
+    return {
+      cleanup: null,
+      path: sourcePath
+    };
+  }
+  const usesConfig = await hasBabelConfig();
+  const result = await transformFileAsync(sourcePath, {
+    configFile: usesConfig ? resolve(process.cwd(), "babel.config.cjs") : false,
+    presets: usesConfig ? [] : ["@babel/preset-flow"]
+  });
+  if (!result || !result.code) {
+    throw new Error("Failed to compile node harness.");
+  }
+  const dir = await mkdtemp(join(tmpdir(), harnessTempPrefix));
+  const harnessPath = join(dir, "node-harness.js");
+  await writeFile(harnessPath, result.code, "utf8");
+  return {
+    cleanup: async () => {
+      await rm(dir, {
+        force: true,
+        recursive: true
+      });
+    },
+    path: harnessPath
+  };
+};

package/lib/regression.js

@@ -3,7 +3,7 @@ export const getRegressions = results => {
   const findings = [];
   for (const result of results) {
     for (const entry of result.benchmarks) {
-      if (isPositiveInterval(entry.difference.relative.ci)) {
+      if (isPositiveInterval(entry.difference.relative.ci) && isPositiveInterval(entry.difference.absolute.ci)) {
         findings.push({
           benchmark: entry.benchmark.name,
           engine: result.engine.name,

package/lib/report/console.js

@@ -10,16 +10,38 @@ const formatRelativeInterval = (interval, decimals) => formatInterval({
   low: interval.low * 100
 }, decimals, "%");
 const formatRelativeValue = (value, decimals) => `${formatNumber(value * 100, decimals)}%`;
+const isPositiveInterval = interval => interval.low > 0 && interval.high > 0;
+const isNegativeInterval = interval => interval.low < 0 && interval.high < 0;
+const classifyDifference = difference => {
+  if (isPositiveInterval(difference.relative.ci) && isPositiveInterval(difference.absolute.ci)) {
+    return "regression";
+  }
+  if (isNegativeInterval(difference.relative.ci) && isNegativeInterval(difference.absolute.ci)) {
+    return "improvement";
+  }
+  return "no significant change";
+};
 export const renderConsoleReport = results => {
   const lines = [];
   for (const result of results) {
+    let regressions = 0;
+    let improvements = 0;
+    let neutral = 0;
     lines.push(`Engine: ${result.engine.name}`);
     for (const entry of result.benchmarks) {
       const unit = entry.benchmark.unit != null ? ` ${entry.benchmark.unit}` : "";
-      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
+      const status = classifyDifference(entry.difference);
+      if (status === "regression") {
+        regressions += 1;
+      } else if (status === "improvement") {
+        improvements += 1;
+      } else {
+        neutral += 1;
+      }
+      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    Result: ${status} (mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)})`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`, `    diff abs mean=${formatNumber(entry.difference.absolute.mean, 4)}${unit} ci=${formatInterval(entry.difference.absolute.ci, 4)}`];
       lines.push(...benchmarkLines);
     }
-    lines.push("");
+    lines.push(`  Summary: regressions=${regressions} improvements=${improvements} neutral=${neutral}`, "");
   }
   return lines.join("\n").trimEnd();
 };

package/lib/runner.js

@@ -1,149 +1,204 @@
-import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { transformFileAsync } from "@babel/core";
+import { resolve } from "node:path";
 import { createNodeHarness } from "./engines/node.js";
-import { computeDifference, summaryStats } from "./stats.js";
+import { buildHarnessIfNeeded, getHarnessPath } from "./harness.js";
+import { computeRelativeDifferenceFromSamples, summaryStats } from "./stats.js";
 const versions = ["baseline", "current"];
 const autoSampleBatchSize = 10;
-const harnessTempPrefix = "perfshield-harness-";
+const defaultMinTimeMs = 20;
+const defaultMaxRelativeMargin = 0.05;
 const getVersionOrder = seed => {
   if (seed % 2 === 0) {
     return versions;
   }
   return [versions[1], versions[0]];
 };
-const getHarnessPath = () => {
-  const override = process.env.WEB_BENCHMARKER_HARNESS_PATH;
-  if (override != null) {
-    return override;
+const sleep = async delayMs => {
+  if (delayMs <= 0) {
+    return;
   }
-  return fileURLToPath(new URL("./engines/node-harness.js", import.meta.url).toString());
+  await new Promise(resolve => {
+    setTimeout(resolve, delayMs);
+  });
 };
-const hasBabelConfig = async () => {
-  const configPath = resolve(process.cwd(), "babel.config.cjs");
-  try {
-    await access(configPath);
-    return true;
-  } catch {
-    return false;
+const warmupBenchmarks = async (harness, benchmarks, delayMs, progress) => {
+  const warmups = [];
+  for (let index = 0; index < benchmarks.length; index += 1) {
+    const descriptor = benchmarks[index];
+    const order = getVersionOrder(index);
+    let baselineSample;
+    let currentSample;
+    for (const version of order) {
+      const result = await harness.runSample({
+        index,
+        iterations: descriptor.iterations,
+        version
+      });
+      if (version === "baseline") {
+        baselineSample = result.durationMs;
+      } else {
+        currentSample = result.durationMs;
+      }
+    }
+    if (baselineSample == null || currentSample == null) {
+      throw new Error("Warmup did not collect baseline/current samples.");
+    }
+    warmups.push({
+      baseline: baselineSample,
+      current: currentSample
+    });
+    if (progress) {
+      progress({
+        benchmarkCount: benchmarks.length,
+        benchmarkIndex: index,
+        benchmarkName: descriptor.name,
+        phase: "warmup"
+      });
+    }
+    await sleep(delayMs);
   }
+  return warmups;
 };
-const buildHarnessIfNeeded = async sourcePath => {
-  const contents = await readFile(sourcePath, "utf8");
-  if (!contents.includes("import type") && !contents.includes("@flow")) {
-    return {
-      cleanup: null,
-      path: sourcePath
-    };
+const computeIterationOverrides = (benchmarks, warmups, minTimeMs) => {
+  if (minTimeMs <= 0) {
+    return benchmarks.map(() => undefined);
   }
-  const usesConfig = await hasBabelConfig();
-  const result = await transformFileAsync(sourcePath, {
-    configFile: usesConfig ? resolve(process.cwd(), "babel.config.cjs") : false,
-    presets: usesConfig ? [] : ["@babel/preset-flow"]
+  return benchmarks.map((benchmark, index) => {
+    const iterationsBase = benchmark.iterations ?? 1;
+    const warmup = warmups[index];
+    const baselinePerIter = warmup.baseline / iterationsBase;
+    const currentPerIter = warmup.current / iterationsBase;
+    const maxPerIter = Math.max(baselinePerIter, currentPerIter);
+    if (!Number.isFinite(maxPerIter) || maxPerIter <= 0) {
+      return iterationsBase;
+    }
+    return Math.max(iterationsBase, Math.ceil(minTimeMs / maxPerIter));
   });
-  if (!result || !result.code) {
-    throw new Error("Failed to compile node harness.");
+};
+const updateIterations = (currentIterations, baselineSample, currentSample, minTimeMs, minimumIterations) => {
+  if (minTimeMs <= 0) {
+    return currentIterations;
   }
-  const dir = await mkdtemp(join(tmpdir(), harnessTempPrefix));
-  const harnessPath = join(dir, "node-harness.js");
-  await writeFile(harnessPath, result.code, "utf8");
-  return {
-    cleanup: async () => {
-      await rm(dir, {
-        force: true,
-        recursive: true
-      });
-    },
-    path: harnessPath
-  };
+  const perIter = Math.max(baselineSample, currentSample) / currentIterations;
+  if (!Number.isFinite(perIter) || perIter <= 0) {
+    return currentIterations;
+  }
+  return Math.max(minimumIterations, Math.ceil(minTimeMs / perIter));
 };
-const withFreshHarness = async (engine, harnessPath, baselinePath, currentPath, callback) => {
-  const harness = await createNodeHarness(engine, harnessPath, baselinePath, currentPath);
-  try {
-    return await callback(harness);
-  } finally {
-    await harness.close();
+const relativeMarginFromDifference = difference => {
+  const mean = difference.relative.mean;
+  if (mean === 0) {
+    return Number.POSITIVE_INFINITY;
+  }
+  const margin = Math.abs(difference.relative.ci.high - mean);
+  return Math.abs(margin / mean);
+};
+const buildSamplePayload = (index, version, iterations) => {
+  const payload = {
+    index,
+    version
+  };
+  if (iterations != null) {
+    payload.iterations = iterations;
   }
+  return payload;
 };
-const runIterationInFreshHarness = async (engine, harnessPath, baselinePath, currentPath, index, iterations, order) => await withFreshHarness(engine, harnessPath, baselinePath, currentPath, async harness => {
-  const results = {};
+const runSamplePair = async (harness, index, iterations, order) => {
+  let baselineSample;
+  let currentSample;
   for (const version of order) {
-    const payload = {
-      index,
-      version
-    };
-    if (iterations != null) {
-      payload.iterations = iterations;
+    const result = await harness.runSample(buildSamplePayload(index, version, iterations));
+    if (version === "baseline") {
+      baselineSample = result.durationMs;
+    } else {
+      currentSample = result.durationMs;
     }
-    const result = await harness.runSample(payload);
-    results[version] = result.durationMs;
   }
-  return results;
-});
-const warmupBenchmarks = async (engine, harnessPath, baselinePath, currentPath, benchmarks) => {
-  let roundRobinSeed = 0;
-  for (let index = 0; index < benchmarks.length; index += 1) {
-    const descriptor = benchmarks[index];
-    const order = getVersionOrder(roundRobinSeed);
-    roundRobinSeed += 1;
-    await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
+  if (baselineSample == null || currentSample == null) {
+    throw new Error("Sample pair did not include baseline/current.");
   }
+  return {
+    baseline: baselineSample,
+    current: currentSample
+  };
 };
-const collectSamples = async (engine, harnessPath, baselinePath, currentPath, benchmarks, minSamples) => {
-  const samples = benchmarks.map(() => ({
+const collectSamples = async (harness, benchmarks, minSamples, iterationOverrides, delayMs, minTimeMs, samples, progress) => {
+  const buckets = samples ?? benchmarks.map(() => ({
     baseline: [],
     current: []
   }));
-  let roundRobinSeed = 0;
+  let completed = 0;
+  const total = minSamples * benchmarks.length;
   for (let iteration = 0; iteration < minSamples; iteration += 1) {
+    const order = getVersionOrder(iteration);
     for (let index = 0; index < benchmarks.length; index += 1) {
-      const descriptor = benchmarks[index];
-      const order = getVersionOrder(roundRobinSeed);
-      roundRobinSeed += 1;
-      const result = await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
-      if (result.baseline != null) {
-        samples[index].baseline.push(result.baseline);
-      }
-      if (result.current != null) {
-        samples[index].current.push(result.current);
+      const iterations = iterationOverrides[index];
+      const minimumIterations = benchmarks[index].iterations ?? 1;
+      const result = await runSamplePair(harness, index, iterations, order);
+      buckets[index].baseline.push(result.baseline);
+      buckets[index].current.push(result.current);
+      const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
+      iterationOverrides[index] = nextIterations;
+      completed += 1;
+      if (progress) {
+        progress({
+          completed,
+          phase: "samples",
+          total
+        });
       }
+      await sleep(delayMs);
     }
   }
-  return samples;
+  return buckets;
 };
 const intervalContains = (interval, value) => interval.low <= value && value <= interval.high;
-const autoSampleResolved = (samples, conditions) => samples.every(bucket => {
-  const baselineStats = summaryStats(bucket.baseline);
-  const currentStats = summaryStats(bucket.current);
-  const diff = computeDifference(baselineStats, currentStats);
+const autoSampleResolved = (samples, conditions, maxRelativeMargin) => samples.every(bucket => {
+  const diff = computeRelativeDifferenceFromSamples(bucket.baseline, bucket.current);
+  const onlyZeroConditions = conditions.every(condition => condition === 0);
+  if (onlyZeroConditions) {
+    const maxAbsBound = Math.max(Math.abs(diff.relative.ci.low), Math.abs(diff.relative.ci.high));
+    if (maxAbsBound <= maxRelativeMargin) {
+      return true;
+    }
+  }
   for (const condition of conditions) {
     if (intervalContains(diff.relative.ci, condition)) {
       return false;
     }
   }
+  if (relativeMarginFromDifference(diff) > maxRelativeMargin) {
+    return false;
+  }
   return true;
 });
-const autoSample = async (engine, harnessPath, baselinePath, currentPath, benchmarks, samples, conditions, timeoutMs) => {
+const autoSample = async (harness, benchmarks, samples, conditions, maxRelativeMargin, iterationOverrides, delayMs, minTimeMs, progress, timeoutMs) => {
   const startTime = Date.now();
   let roundRobinSeed = 0;
+  let completed = 0;
   while (Date.now() - startTime < timeoutMs) {
-    if (autoSampleResolved(samples, conditions)) {
+    if (autoSampleResolved(samples, conditions, maxRelativeMargin)) {
       return;
     }
     for (let batch = 0; batch < autoSampleBatchSize; batch += 1) {
+      const order = getVersionOrder(roundRobinSeed);
+      roundRobinSeed += 1;
       for (let index = 0; index < benchmarks.length; index += 1) {
-        const descriptor = benchmarks[index];
-        const order = getVersionOrder(roundRobinSeed);
-        roundRobinSeed += 1;
-        const result = await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
-        if (result.baseline != null) {
-          samples[index].baseline.push(result.baseline);
-        }
-        if (result.current != null) {
-          samples[index].current.push(result.current);
+        const iterations = iterationOverrides[index];
+        const minimumIterations = benchmarks[index].iterations ?? 1;
+        const result = await runSamplePair(harness, index, iterations, order);
+        samples[index].baseline.push(result.baseline);
+        samples[index].current.push(result.current);
+        const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
+        iterationOverrides[index] = nextIterations;
+        completed += 1;
+        if (progress) {
+          progress({
+            completed,
+            elapsedMs: Date.now() - startTime,
+            phase: "autosample"
+          });
         }
+        await sleep(delayMs);
       }
     }
   }
@@ -158,17 +213,33 @@ export const runEngineComparison = async options => {
   const harnessArtifact = await buildHarnessIfNeeded(getHarnessPath());
   const resolvedBaseline = resolve(baselinePath);
   const resolvedCurrent = resolve(currentPath);
+  const harness = await createNodeHarness(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent);
   try {
-    const benchmarks = await withFreshHarness(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, async harness => await harness.listBenchmarks());
-    await warmupBenchmarks(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks);
-    const samples = await collectSamples(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks, config.sampling.minSamples);
-    await autoSample(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks, samples, config.sampling.conditions, config.sampling.timeoutMs);
+    const benchmarks = await harness.listBenchmarks();
+    const minTimeMs = config.sampling.minTimeMs ?? defaultMinTimeMs;
+    const minSamples = Math.max(1, config.sampling.minSamples);
+    const sampleScale = Math.sqrt(minSamples / 20);
+    const benchmarkScale = Math.sqrt(Math.max(1, benchmarks.length));
+    const effectiveMinTimeMs = minTimeMs / Math.max(1, sampleScale * benchmarkScale);
+    const delayMs = config.sampling.delayMs ?? 0;
+    const maxRelativeMargin = config.sampling.maxRelativeMargin ?? defaultMaxRelativeMargin;
+    const warmups = await warmupBenchmarks(harness, benchmarks, delayMs, options.progress);
+    const iterationOverrides = computeIterationOverrides(benchmarks, warmups, effectiveMinTimeMs);
+    const samples = warmups.map(warmup => ({
+      baseline: [warmup.baseline],
+      current: [warmup.current]
+    }));
+    const remainingSamples = Math.max(0, config.sampling.minSamples - 1);
+    if (remainingSamples > 0) {
+      await collectSamples(harness, benchmarks, remainingSamples, iterationOverrides, delayMs, effectiveMinTimeMs, samples, options.progress);
+    }
+    await autoSample(harness, benchmarks, samples, config.sampling.conditions, maxRelativeMargin, iterationOverrides, delayMs, effectiveMinTimeMs, options.progress, config.sampling.timeoutMs);
     const benchmarkResults = benchmarks.map((benchmark, index) => {
       const baselineSamples = samples[index].baseline;
       const currentSamples = samples[index].current;
       const baselineStats = summaryStats(baselineSamples);
       const currentStats = summaryStats(currentSamples);
-      const difference = computeDifference(baselineStats, currentStats);
+      const difference = computeRelativeDifferenceFromSamples(baselineSamples, currentSamples);
       return {
         benchmark,
         difference,
@@ -187,6 +258,7 @@ export const runEngineComparison = async options => {
       engine
     };
   } finally {
+    await harness.close();
     if (harnessArtifact.cleanup) {
       await harnessArtifact.cleanup();
     }

package/lib/stats.js

@@ -1,4 +1,11 @@
 import jstat from "jstat";
+export const relativeMarginOfError = stats => {
+  if (stats.mean === 0) {
+    return Number.POSITIVE_INFINITY;
+  }
+  const margin = Math.abs(stats.meanCI.high - stats.mean);
+  return Math.abs(margin / stats.mean);
+};
 const sumOf = values => values.reduce((total, value) => total + value, 0);
 const squareResiduals = (values, mean) => values.map(value => {
   const diff = value - mean;
@@ -27,6 +34,10 @@ export const samplingDistributionOfRelativeDifferenceOfMeans = (a, b) => ({
   mean: (b.mean - a.mean) / a.mean,
   variance: (a.variance * b.mean * b.mean + b.variance * a.mean * a.mean) / (a.mean * a.mean * a.mean * a.mean)
 });
+export const samplingDistributionOfDifferenceOfMeans = (a, b) => ({
+  mean: b.mean - a.mean,
+  variance: a.variance + b.variance
+});
 export const summaryStats = values => {
   if (values.length === 0) {
     throw new Error("Cannot compute stats for an empty sample set.");
@@ -61,15 +72,64 @@ export const computeDifference = (baseline, current) => {
     mean: current.mean,
     variance: current.variance
   }, current.size);
+  const absoluteDist = samplingDistributionOfDifferenceOfMeans(baselineDist, currentDist);
   const relativeDist = samplingDistributionOfRelativeDifferenceOfMeans(baselineDist, currentDist);
   const size = Math.min(baseline.size, current.size);
   return {
+    absolute: {
+      ci: confidenceInterval95(absoluteDist, size),
+      mean: absoluteDist.mean
+    },
     relative: {
       ci: confidenceInterval95(relativeDist, size),
       mean: relativeDist.mean
     }
   };
 };
+const computePairedRelativeStats = (baselineSamples, currentSamples) => {
+  const size = Math.min(baselineSamples.length, currentSamples.length);
+  if (size === 0) {
+    throw new Error("Cannot compute differences with empty sample sets.");
+  }
+  const diffs = [];
+  for (let index = 0; index < size; index += 1) {
+    const baseline = baselineSamples[index];
+    if (baseline === 0) {
+      throw new Error("Cannot compute relative difference with baseline sample 0.");
+    }
+    diffs.push((currentSamples[index] - baseline) / baseline);
+  }
+  const diffStats = summaryStats(diffs);
+  return {
+    ci: confidenceInterval95(samplingDistributionOfTheMean({
+      mean: diffStats.mean,
+      variance: diffStats.variance
+    }, diffStats.size), diffStats.size),
+    mean: diffStats.mean
+  };
+};
+const computePairedAbsoluteStats = (baselineSamples, currentSamples) => {
+  const size = Math.min(baselineSamples.length, currentSamples.length);
+  if (size === 0) {
+    throw new Error("Cannot compute differences with empty sample sets.");
+  }
+  const diffs = [];
+  for (let index = 0; index < size; index += 1) {
+    diffs.push(currentSamples[index] - baselineSamples[index]);
+  }
+  const diffStats = summaryStats(diffs);
+  return {
+    ci: confidenceInterval95(samplingDistributionOfTheMean({
+      mean: diffStats.mean,
+      variance: diffStats.variance
+    }, diffStats.size), diffStats.size),
+    mean: diffStats.mean
+  };
+};
+export const computeRelativeDifferenceFromSamples = (baselineSamples, currentSamples) => ({
+  absolute: computePairedAbsoluteStats(baselineSamples, currentSamples),
+  relative: computePairedRelativeStats(baselineSamples, currentSamples)
+});
 export const computeDifferences = stats => stats.map(result => ({
   ...result,
   differences: stats.map(other => other === result ? null : computeDifference(other.stats, result.stats))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "perfshield",
-  "version": "0.0.4",
+  "version": "0.0.7",
   "description": "A tool for doing web benchmarking across multiple JS engines and with statistical signifigance",
   "license": "MIT",
   "type": "module",