perfshield 0.0.4 → 0.0.5

package/lib/cli.js

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+import { WriteStream } from "node:tty";
 import { ensureBaseline, saveBaseline } from "./artifacts.js";
 import { runBuild } from "./build.js";
 import { ConfigError, formatConfigError, loadConfig } from "./config.js";
@@ -20,17 +21,73 @@ const runPrepare = async config => {
   const baselinePath = await saveBaseline(config, outputPath);
   console.log(`Baseline saved to ${baselinePath}`);
 };
+const isStderrTTY = () => {
+  return process.stderr instanceof WriteStream;
+};
+const createProgressReporter = engineName => {
+  let lastWrite = 0;
+  let lastLine = "";
+  const isTTY = isStderrTTY();
+  const writeLine = line => {
+    if (isTTY) {
+      const padded = line.padEnd(80, " ");
+      process.stderr.write(`\r${padded}`);
+    } else if (line !== lastLine) {
+      process.stderr.write(`${line}\n`);
+    }
+    lastLine = line;
+  };
+  const report = event => {
+    const now = Date.now();
+    if (isTTY && now - lastWrite < 100) {
+      return;
+    }
+    lastWrite = now;
+    let message = "";
+    if (event.phase === "warmup") {
+      const current = (event.benchmarkIndex ?? 0) + 1;
+      const total = event.benchmarkCount ?? 0;
+      const name = event.benchmarkName == null ? "" : ` ${event.benchmarkName}`;
+      message = `Warmup ${current}/${total}${name} (${engineName})`;
+    } else if (event.phase === "samples") {
+      const completed = event.completed ?? 0;
+      const total = event.total ?? 0;
+      const percent = total > 0 ? Math.round(completed / total * 100) : 0;
+      message = `Sampling ${completed}/${total} (${percent}%) (${engineName})`;
+    } else {
+      const completed = event.completed ?? 0;
+      const elapsed = event.elapsedMs != null ? Math.round(event.elapsedMs / 1000) : 0;
+      message = `Auto-sampling ${completed} samples (${elapsed}s) (${engineName})`;
+    }
+    writeLine(message);
+  };
+  const finish = () => {
+    if (isTTY && lastLine) {
+      process.stderr.write("\n");
+    }
+  };
+  return {
+    finish,
+    report
+  };
+};
 const runCompare = async config => {
   const outputPath = await runBuild(config.build);
   const baselinePath = await ensureBaseline(config);
   const results = [];
+  const showProgress = config.report.formats.includes("console") || isStderrTTY();
   for (const engine of config.engines) {
+    const progress = showProgress ? createProgressReporter(engine.name) : null;
     results.push(await runEngineComparison({
       baselinePath,
       config,
       currentPath: outputPath,
-      engine
+      engine,
+      progress: progress ? progress.report : undefined
     }));
+    if (progress) {
+      progress.finish();
+    }
   }
   const outputs = renderReports(results, config.report.formats);
   for (const output of outputs) {

package/lib/config.js

@@ -163,7 +163,7 @@ const parseSamplingConfig = (value, issues) => {
   if (!sampling) {
     return null;
   }
-  validateKeys(sampling, ["conditions", "minSamples", "timeoutMs"], "config.sampling", issues);
+  validateKeys(sampling, ["conditions", "delayMs", "maxRelativeMargin", "minSamples", "minTimeMs", "timeoutMs"], "config.sampling", issues);
   const minSamples = asNumber(sampling.minSamples, "config.sampling.minSamples", issues, {
     integer: true,
     min: 1
@@ -172,15 +172,32 @@ const parseSamplingConfig = (value, issues) => {
     integer: true,
     min: 1
   });
+  const minTimeMs = sampling.minTimeMs == null ? undefined : asNumber(sampling.minTimeMs, "config.sampling.minTimeMs", issues, {
+    integer: true,
+    min: 0
+  });
+  const delayMs = sampling.delayMs == null ? undefined : asNumber(sampling.delayMs, "config.sampling.delayMs", issues, {
+    integer: true,
+    min: 0
+  });
+  const maxRelativeMargin = sampling.maxRelativeMargin == null ? undefined : asNumber(sampling.maxRelativeMargin, "config.sampling.maxRelativeMargin", issues, {
+    min: 0
+  });
+  if (maxRelativeMargin != null && maxRelativeMargin > 1) {
+    addIssue(issues, "config.sampling.maxRelativeMargin must be at most 1.");
+  }
   const conditions = asNumberArray(sampling.conditions, "config.sampling.conditions", issues, {
     minLength: 1
   });
-  if (minSamples == null || timeoutMs == null || !conditions) {
+  if (minSamples == null || timeoutMs == null || !conditions || minTimeMs === null || delayMs === null || maxRelativeMargin === null) {
     return null;
   }
   return {
     conditions,
+    delayMs,
+    maxRelativeMargin,
     minSamples,
+    minTimeMs,
     timeoutMs
   };
 };

package/lib/report/console.js

@@ -10,16 +10,36 @@ const formatRelativeInterval = (interval, decimals) => formatInterval({
   low: interval.low * 100
 }, decimals, "%");
 const formatRelativeValue = (value, decimals) => `${formatNumber(value * 100, decimals)}%`;
+const classifyDifference = interval => {
+  if (interval.low > 0 && interval.high > 0) {
+    return "regression";
+  }
+  if (interval.low < 0 && interval.high < 0) {
+    return "improvement";
+  }
+  return "no significant change";
+};
 export const renderConsoleReport = results => {
   const lines = [];
   for (const result of results) {
+    let regressions = 0;
+    let improvements = 0;
+    let neutral = 0;
     lines.push(`Engine: ${result.engine.name}`);
     for (const entry of result.benchmarks) {
       const unit = entry.benchmark.unit != null ? ` ${entry.benchmark.unit}` : "";
-      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
+      const status = classifyDifference(entry.difference.relative.ci);
+      if (status === "regression") {
+        regressions += 1;
+      } else if (status === "improvement") {
+        improvements += 1;
+      } else {
+        neutral += 1;
+      }
+      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    Result: ${status} (mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)})`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
       lines.push(...benchmarkLines);
     }
-    lines.push("");
+    lines.push(`  Summary: regressions=${regressions} improvements=${improvements} neutral=${neutral}`, "");
   }
   return lines.join("\n").trimEnd();
 };

package/lib/runner.js

@@ -4,10 +4,12 @@ import { join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { transformFileAsync } from "@babel/core";
 import { createNodeHarness } from "./engines/node.js";
-import { computeDifference, summaryStats } from "./stats.js";
+import { computeRelativeDifferenceFromSamples, summaryStats } from "./stats.js";
 const versions = ["baseline", "current"];
 const autoSampleBatchSize = 10;
 const harnessTempPrefix = "perfshield-harness-";
+const defaultMinTimeMs = 20;
+const defaultMaxRelativeMargin = 0.05;
 const getVersionOrder = seed => {
   if (seed % 2 === 0) {
     return versions;
@@ -59,91 +61,193 @@ const buildHarnessIfNeeded = async sourcePath => {
     path: harnessPath
   };
 };
-const withFreshHarness = async (engine, harnessPath, baselinePath, currentPath, callback) => {
-  const harness = await createNodeHarness(engine, harnessPath, baselinePath, currentPath);
-  try {
-    return await callback(harness);
-  } finally {
-    await harness.close();
+const sleep = async delayMs => {
+  if (delayMs <= 0) {
+    return;
+  }
+  await new Promise(resolve => {
+    setTimeout(resolve, delayMs);
+  });
+};
+const warmupBenchmarks = async (harness, benchmarks, delayMs, progress) => {
+  const warmups = [];
+  for (let index = 0; index < benchmarks.length; index += 1) {
+    const descriptor = benchmarks[index];
+    const order = getVersionOrder(index);
+    let baselineSample;
+    let currentSample;
+    for (const version of order) {
+      const result = await harness.runSample({
+        index,
+        iterations: descriptor.iterations,
+        version
+      });
+      if (version === "baseline") {
+        baselineSample = result.durationMs;
+      } else {
+        currentSample = result.durationMs;
+      }
+    }
+    if (baselineSample == null || currentSample == null) {
+      throw new Error("Warmup did not collect baseline/current samples.");
+    }
+    warmups.push({
+      baseline: baselineSample,
+      current: currentSample
+    });
+    if (progress) {
+      progress({
+        benchmarkCount: benchmarks.length,
+        benchmarkIndex: index,
+        benchmarkName: descriptor.name,
+        phase: "warmup"
+      });
+    }
+    await sleep(delayMs);
   }
+  return warmups;
 };
-const runIterationInFreshHarness = async (engine, harnessPath, baselinePath, currentPath, index, iterations, order) => await withFreshHarness(engine, harnessPath, baselinePath, currentPath, async harness => {
-  const results = {};
+const computeIterationOverrides = (benchmarks, warmups, minTimeMs) => {
+  if (minTimeMs <= 0) {
+    return benchmarks.map(() => undefined);
+  }
+  return benchmarks.map((benchmark, index) => {
+    const iterationsBase = benchmark.iterations ?? 1;
+    const warmup = warmups[index];
+    const baselinePerIter = warmup.baseline / iterationsBase;
+    const currentPerIter = warmup.current / iterationsBase;
+    const maxPerIter = Math.max(baselinePerIter, currentPerIter);
+    if (!Number.isFinite(maxPerIter) || maxPerIter <= 0) {
+      return iterationsBase;
+    }
+    return Math.max(iterationsBase, Math.ceil(minTimeMs / maxPerIter));
+  });
+};
+const updateIterations = (currentIterations, baselineSample, currentSample, minTimeMs, minimumIterations) => {
+  if (minTimeMs <= 0) {
+    return currentIterations;
+  }
+  const perIter = Math.max(baselineSample, currentSample) / currentIterations;
+  if (!Number.isFinite(perIter) || perIter <= 0) {
+    return currentIterations;
+  }
+  return Math.max(minimumIterations, Math.ceil(minTimeMs / perIter));
+};
+const relativeMarginFromDifference = difference => {
+  const mean = difference.relative.mean;
+  if (mean === 0) {
+    return Number.POSITIVE_INFINITY;
+  }
+  const margin = Math.abs(difference.relative.ci.high - mean);
+  return Math.abs(margin / mean);
+};
+const buildSamplePayload = (index, version, iterations) => {
+  const payload = {
+    index,
+    version
+  };
+  if (iterations != null) {
+    payload.iterations = iterations;
+  }
+  return payload;
+};
+const runSamplePair = async (harness, index, iterations, order) => {
+  let baselineSample;
+  let currentSample;
   for (const version of order) {
-    const payload = {
-      index,
-      version
-    };
-    if (iterations != null) {
-      payload.iterations = iterations;
+    const result = await harness.runSample(buildSamplePayload(index, version, iterations));
+    if (version === "baseline") {
+      baselineSample = result.durationMs;
+    } else {
+      currentSample = result.durationMs;
     }
-    const result = await harness.runSample(payload);
-    results[version] = result.durationMs;
   }
-  return results;
-});
-const warmupBenchmarks = async (engine, harnessPath, baselinePath, currentPath, benchmarks) => {
-  let roundRobinSeed = 0;
-  for (let index = 0; index < benchmarks.length; index += 1) {
-    const descriptor = benchmarks[index];
-    const order = getVersionOrder(roundRobinSeed);
-    roundRobinSeed += 1;
-    await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
+  if (baselineSample == null || currentSample == null) {
+    throw new Error("Sample pair did not include baseline/current.");
   }
+  return {
+    baseline: baselineSample,
+    current: currentSample
+  };
 };
-const collectSamples = async (engine, harnessPath, baselinePath, currentPath, benchmarks, minSamples) => {
-  const samples = benchmarks.map(() => ({
+const collectSamples = async (harness, benchmarks, minSamples, iterationOverrides, delayMs, minTimeMs, samples, progress) => {
+  const buckets = samples ?? benchmarks.map(() => ({
     baseline: [],
     current: []
   }));
-  let roundRobinSeed = 0;
+  let completed = 0;
+  const total = minSamples * benchmarks.length;
   for (let iteration = 0; iteration < minSamples; iteration += 1) {
+    const order = getVersionOrder(iteration);
     for (let index = 0; index < benchmarks.length; index += 1) {
-      const descriptor = benchmarks[index];
-      const order = getVersionOrder(roundRobinSeed);
-      roundRobinSeed += 1;
-      const result = await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
-      if (result.baseline != null) {
-        samples[index].baseline.push(result.baseline);
-      }
-      if (result.current != null) {
-        samples[index].current.push(result.current);
+      const iterations = iterationOverrides[index];
+      const minimumIterations = benchmarks[index].iterations ?? 1;
+      const result = await runSamplePair(harness, index, iterations, order);
+      buckets[index].baseline.push(result.baseline);
+      buckets[index].current.push(result.current);
+      const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
+      iterationOverrides[index] = nextIterations;
+      completed += 1;
+      if (progress) {
+        progress({
+          completed,
+          phase: "samples",
+          total
+        });
       }
+      await sleep(delayMs);
     }
   }
-  return samples;
+  return buckets;
 };
 const intervalContains = (interval, value) => interval.low <= value && value <= interval.high;
-const autoSampleResolved = (samples, conditions) => samples.every(bucket => {
-  const baselineStats = summaryStats(bucket.baseline);
-  const currentStats = summaryStats(bucket.current);
-  const diff = computeDifference(baselineStats, currentStats);
+const autoSampleResolved = (samples, conditions, maxRelativeMargin) => samples.every(bucket => {
+  const diff = computeRelativeDifferenceFromSamples(bucket.baseline, bucket.current);
+  const onlyZeroConditions = conditions.every(condition => condition === 0);
+  if (onlyZeroConditions) {
+    const maxAbsBound = Math.max(Math.abs(diff.relative.ci.low), Math.abs(diff.relative.ci.high));
+    if (maxAbsBound <= maxRelativeMargin) {
+      return true;
+    }
+  }
   for (const condition of conditions) {
     if (intervalContains(diff.relative.ci, condition)) {
       return false;
     }
   }
+  if (relativeMarginFromDifference(diff) > maxRelativeMargin) {
+    return false;
+  }
   return true;
 });
-const autoSample = async (engine, harnessPath, baselinePath, currentPath, benchmarks, samples, conditions, timeoutMs) => {
+const autoSample = async (harness, benchmarks, samples, conditions, maxRelativeMargin, iterationOverrides, delayMs, minTimeMs, progress, timeoutMs) => {
   const startTime = Date.now();
   let roundRobinSeed = 0;
+  let completed = 0;
   while (Date.now() - startTime < timeoutMs) {
-    if (autoSampleResolved(samples, conditions)) {
+    if (autoSampleResolved(samples, conditions, maxRelativeMargin)) {
       return;
     }
     for (let batch = 0; batch < autoSampleBatchSize; batch += 1) {
+      const order = getVersionOrder(roundRobinSeed);
+      roundRobinSeed += 1;
       for (let index = 0; index < benchmarks.length; index += 1) {
-        const descriptor = benchmarks[index];
-        const order = getVersionOrder(roundRobinSeed);
-        roundRobinSeed += 1;
-        const result = await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
-        if (result.baseline != null) {
-          samples[index].baseline.push(result.baseline);
-        }
-        if (result.current != null) {
-          samples[index].current.push(result.current);
+        const iterations = iterationOverrides[index];
+        const minimumIterations = benchmarks[index].iterations ?? 1;
+        const result = await runSamplePair(harness, index, iterations, order);
+        samples[index].baseline.push(result.baseline);
+        samples[index].current.push(result.current);
+        const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
+        iterationOverrides[index] = nextIterations;
+        completed += 1;
+        if (progress) {
+          progress({
+            completed,
+            elapsedMs: Date.now() - startTime,
+            phase: "autosample"
+          });
         }
+        await sleep(delayMs);
       }
     }
   }
@@ -158,17 +262,33 @@ export const runEngineComparison = async options => {
   const harnessArtifact = await buildHarnessIfNeeded(getHarnessPath());
   const resolvedBaseline = resolve(baselinePath);
   const resolvedCurrent = resolve(currentPath);
+  const harness = await createNodeHarness(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent);
   try {
-    const benchmarks = await withFreshHarness(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, async harness => await harness.listBenchmarks());
-    await warmupBenchmarks(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks);
-    const samples = await collectSamples(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks, config.sampling.minSamples);
-    await autoSample(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks, samples, config.sampling.conditions, config.sampling.timeoutMs);
+    const benchmarks = await harness.listBenchmarks();
+    const minTimeMs = config.sampling.minTimeMs ?? defaultMinTimeMs;
+    const minSamples = Math.max(1, config.sampling.minSamples);
+    const sampleScale = Math.sqrt(minSamples / 20);
+    const benchmarkScale = Math.sqrt(Math.max(1, benchmarks.length));
+    const effectiveMinTimeMs = minTimeMs / Math.max(1, sampleScale * benchmarkScale);
+    const delayMs = config.sampling.delayMs ?? 0;
+    const maxRelativeMargin = config.sampling.maxRelativeMargin ?? defaultMaxRelativeMargin;
+    const warmups = await warmupBenchmarks(harness, benchmarks, delayMs, options.progress);
+    const iterationOverrides = computeIterationOverrides(benchmarks, warmups, effectiveMinTimeMs);
+    const samples = warmups.map(warmup => ({
+      baseline: [warmup.baseline],
+      current: [warmup.current]
+    }));
+    const remainingSamples = Math.max(0, config.sampling.minSamples - 1);
+    if (remainingSamples > 0) {
+      await collectSamples(harness, benchmarks, remainingSamples, iterationOverrides, delayMs, effectiveMinTimeMs, samples, options.progress);
+    }
+    await autoSample(harness, benchmarks, samples, config.sampling.conditions, maxRelativeMargin, iterationOverrides, delayMs, effectiveMinTimeMs, options.progress, config.sampling.timeoutMs);
     const benchmarkResults = benchmarks.map((benchmark, index) => {
       const baselineSamples = samples[index].baseline;
       const currentSamples = samples[index].current;
       const baselineStats = summaryStats(baselineSamples);
       const currentStats = summaryStats(currentSamples);
-      const difference = computeDifference(baselineStats, currentStats);
+      const difference = computeRelativeDifferenceFromSamples(baselineSamples, currentSamples);
       return {
         benchmark,
         difference,
@@ -187,6 +307,7 @@ export const runEngineComparison = async options => {
       engine
     };
   } finally {
+    await harness.close();
     if (harnessArtifact.cleanup) {
       await harnessArtifact.cleanup();
     }

package/lib/stats.js

@@ -1,4 +1,11 @@
 import jstat from "jstat";
+export const relativeMarginOfError = stats => {
+  if (stats.mean === 0) {
+    return Number.POSITIVE_INFINITY;
+  }
+  const margin = Math.abs(stats.meanCI.high - stats.mean);
+  return Math.abs(margin / stats.mean);
+};
 const sumOf = values => values.reduce((total, value) => total + value, 0);
 const squareResiduals = (values, mean) => values.map(value => {
   const diff = value - mean;
@@ -70,6 +77,31 @@ export const computeDifference = (baseline, current) => {
     }
   };
 };
+const computePairedRelativeStats = (baselineSamples, currentSamples) => {
+  const size = Math.min(baselineSamples.length, currentSamples.length);
+  if (size === 0) {
+    throw new Error("Cannot compute differences with empty sample sets.");
+  }
+  const diffs = [];
+  for (let index = 0; index < size; index += 1) {
+    const baseline = baselineSamples[index];
+    if (baseline === 0) {
+      throw new Error("Cannot compute relative difference with baseline sample 0.");
+    }
+    diffs.push((currentSamples[index] - baseline) / baseline);
+  }
+  const diffStats = summaryStats(diffs);
+  return {
+    ci: confidenceInterval95(samplingDistributionOfTheMean({
+      mean: diffStats.mean,
+      variance: diffStats.variance
+    }, diffStats.size), diffStats.size),
+    mean: diffStats.mean
+  };
+};
+export const computeRelativeDifferenceFromSamples = (baselineSamples, currentSamples) => ({
+  relative: computePairedRelativeStats(baselineSamples, currentSamples)
+});
 export const computeDifferences = stats => stats.map(result => ({
   ...result,
   differences: stats.map(other => other === result ? null : computeDifference(other.stats, result.stats))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "perfshield",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "description": "A tool for doing web benchmarking across multiple JS engines and with statistical signifigance",
   "license": "MIT",
   "type": "module",