perfshield 0.0.8 → 0.0.10

package/lib/runner.js

@@ -37,43 +37,37 @@ const sleep = async delayMs => {
     setTimeout(resolve, delayMs);
   });
 };
-const warmupBenchmarks = async (harness, benchmarks, delayMs, progress) => {
-  const warmups = [];
-  for (let index = 0; index < benchmarks.length; index += 1) {
-    const descriptor = benchmarks[index];
-    const order = getVersionOrder(index);
-    let baselineSample;
-    let currentSample;
-    for (const version of order) {
-      const result = await harness.runSample({
-        index,
-        iterations: descriptor.iterations,
-        version
-      });
-      if (version === "baseline") {
-        baselineSample = result.durationMs;
-      } else {
-        currentSample = result.durationMs;
-      }
-    }
-    if (baselineSample == null || currentSample == null) {
-      throw new Error("Warmup did not collect baseline/current samples.");
-    }
-    warmups.push({
-      baseline: baselineSample,
-      current: currentSample
+const warmupBenchmark = async (harness, benchmark, index, delayMs, progress) => {
+  const order = getVersionOrder(index);
+  let baselineSample;
+  let currentSample;
+  for (const version of order) {
+    const result = await harness.runSample({
+      index,
+      iterations: benchmark.iterations,
+      version
     });
-    if (progress) {
-      progress({
-        benchmarkCount: benchmarks.length,
-        benchmarkIndex: index,
-        benchmarkName: descriptor.name,
-        phase: "warmup"
-      });
+    if (version === "baseline") {
+      baselineSample = result.durationMs;
+    } else {
+      currentSample = result.durationMs;
     }
-    await sleep(delayMs);
   }
-  return warmups;
+  if (baselineSample == null || currentSample == null) {
+    throw new Error("Warmup did not collect baseline/current samples.");
+  }
+  if (progress) {
+    progress({
+      benchmarkIndex: index,
+      benchmarkName: benchmark.name,
+      phase: "warmup"
+    });
+  }
+  await sleep(delayMs);
+  return {
+    baseline: baselineSample,
+    current: currentSample
+  };
 };
 const computeIterationOverrides = (benchmarks, warmups, minTimeMs) => {
   if (minTimeMs <= 0) {
@@ -138,36 +132,26 @@ const runSamplePair = async (harness, index, iterations, order) => {
     current: currentSample
   };
 };
-const collectSamples = async (harness, benchmarks, minSamples, iterationOverrides, delayMs, minTimeMs, samples, progress) => {
-  const buckets = samples ?? benchmarks.map(() => ({
-    baseline: [],
-    current: []
-  }));
-  let completed = 0;
-  const total = minSamples * benchmarks.length;
+const collectSamplesForBenchmark = async (harness, benchmark, index, minSamples, iterationOverrides, delayMs, minTimeMs, bucket, progress, progressState) => {
   for (let iteration = 0; iteration < minSamples; iteration += 1) {
     const order = getVersionOrder(iteration);
-    const indexOrder = buildIndexOrder(benchmarks.length, iteration);
-    for (const index of indexOrder) {
-      const iterations = iterationOverrides[index];
-      const minimumIterations = benchmarks[index].iterations ?? 1;
-      const result = await runSamplePair(harness, index, iterations, order);
-      buckets[index].baseline.push(result.baseline);
-      buckets[index].current.push(result.current);
-      const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
-      iterationOverrides[index] = nextIterations;
-      completed += 1;
-      if (progress) {
-        progress({
-          completed,
-          phase: "samples",
-          total
-        });
-      }
-      await sleep(delayMs);
+    const iterations = iterationOverrides[index];
+    const minimumIterations = benchmark.iterations ?? 1;
+    const result = await runSamplePair(harness, index, iterations, order);
+    bucket.baseline.push(result.baseline);
+    bucket.current.push(result.current);
+    const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
+    iterationOverrides[index] = nextIterations;
+    if (progress && progressState) {
+      progressState.completed += 1;
+      progress({
+        completed: progressState.completed,
+        phase: "samples",
+        total: progressState.total
+      });
     }
+    await sleep(delayMs);
   }
-  return buckets;
 };
 const intervalContains = (interval, value) => interval.low <= value && value <= interval.high;
 const autoSampleResolved = (samples, conditions, maxRelativeMargin) => samples.every(bucket => {
@@ -189,36 +173,33 @@ const autoSampleResolved = (samples, conditions, maxRelativeMargin) => samples.e
   }
   return true;
 });
-const autoSample = async (harness, benchmarks, samples, conditions, maxRelativeMargin, iterationOverrides, delayMs, minTimeMs, progress, timeoutMs) => {
+const autoSampleForBenchmark = async (harness, benchmark, index, bucket, conditions, maxRelativeMargin, iterationOverrides, delayMs, minTimeMs, progress, timeoutMs) => {
   const startTime = Date.now();
   let roundRobinSeed = 0;
   let completed = 0;
   while (Date.now() - startTime < timeoutMs) {
-    if (autoSampleResolved(samples, conditions, maxRelativeMargin)) {
+    if (autoSampleResolved([bucket], conditions, maxRelativeMargin)) {
       return;
     }
     for (let batch = 0; batch < autoSampleBatchSize; batch += 1) {
       const order = getVersionOrder(roundRobinSeed);
-      const indexOrder = buildIndexOrder(benchmarks.length, roundRobinSeed);
       roundRobinSeed += 1;
-      for (const index of indexOrder) {
-        const iterations = iterationOverrides[index];
-        const minimumIterations = benchmarks[index].iterations ?? 1;
-        const result = await runSamplePair(harness, index, iterations, order);
-        samples[index].baseline.push(result.baseline);
-        samples[index].current.push(result.current);
-        const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
-        iterationOverrides[index] = nextIterations;
-        completed += 1;
-        if (progress) {
-          progress({
-            completed,
-            elapsedMs: Date.now() - startTime,
-            phase: "autosample"
-          });
-        }
-        await sleep(delayMs);
+      const iterations = iterationOverrides[index];
+      const minimumIterations = benchmark.iterations ?? 1;
+      const result = await runSamplePair(harness, index, iterations, order);
+      bucket.baseline.push(result.baseline);
+      bucket.current.push(result.current);
+      const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
+      iterationOverrides[index] = nextIterations;
+      completed += 1;
+      if (progress) {
+        progress({
+          completed,
+          elapsedMs: Date.now() - startTime,
+          phase: "autosample"
+        });
       }
+      await sleep(delayMs);
     }
   }
 };
@@ -242,24 +223,41 @@ export const runEngineComparison = async options => {
     const effectiveMinTimeMs = minTimeMs / Math.max(1, sampleScale * benchmarkScale);
     const delayMs = config.sampling.delayMs ?? 0;
     const maxRelativeMargin = config.sampling.maxRelativeMargin ?? defaultMaxRelativeMargin;
-    const warmups = await warmupBenchmarks(harness, benchmarks, delayMs, options.progress);
-    const iterationOverrides = computeIterationOverrides(benchmarks, warmups, effectiveMinTimeMs);
-    const samples = warmups.map(warmup => ({
-      baseline: [warmup.baseline],
-      current: [warmup.current]
-    }));
     const remainingSamples = Math.max(0, config.sampling.minSamples - 1);
-    if (remainingSamples > 0) {
-      await collectSamples(harness, benchmarks, remainingSamples, iterationOverrides, delayMs, effectiveMinTimeMs, samples, options.progress);
-    }
-    await autoSample(harness, benchmarks, samples, config.sampling.conditions, maxRelativeMargin, iterationOverrides, delayMs, effectiveMinTimeMs, options.progress, config.sampling.timeoutMs);
-    const benchmarkResults = benchmarks.map((benchmark, index) => {
-      const baselineSamples = samples[index].baseline;
-      const currentSamples = samples[index].current;
+    const progressState = {
+      completed: 0,
+      total: remainingSamples * benchmarks.length
+    };
+    const benchmarkResults = new Array(benchmarks.length);
+    const iterationOverrides = benchmarks.map(() => undefined);
+    const benchmarkOrder = buildIndexOrder(benchmarks.length, 0);
+    const autoSampleDeadline = Date.now() + config.sampling.timeoutMs;
+    for (const index of benchmarkOrder) {
+      const benchmark = benchmarks[index];
+      const progress = options.progress;
+      const warmupSample = await warmupBenchmark(harness, benchmark, index, delayMs, progress ? event => progress({
+        ...event,
+        benchmarkCount: benchmarks.length
+      }) : undefined);
+      const iterationOverride = computeIterationOverrides([benchmark], [warmupSample], effectiveMinTimeMs)[0];
+      iterationOverrides[index] = iterationOverride;
+      const bucket = {
+        baseline: [warmupSample.baseline],
+        current: [warmupSample.current]
+      };
+      if (remainingSamples > 0) {
+        await collectSamplesForBenchmark(harness, benchmark, index, remainingSamples, iterationOverrides, delayMs, effectiveMinTimeMs, bucket, progress, progressState);
+      }
+      const remainingTimeoutMs = Math.max(0, autoSampleDeadline - Date.now());
+      if (remainingTimeoutMs > 0) {
+        await autoSampleForBenchmark(harness, benchmark, index, bucket, config.sampling.conditions, maxRelativeMargin, iterationOverrides, delayMs, effectiveMinTimeMs, progress, remainingTimeoutMs);
+      }
+      const baselineSamples = bucket.baseline;
+      const currentSamples = bucket.current;
       const baselineStats = summaryStats(baselineSamples);
       const currentStats = summaryStats(currentSamples);
       const difference = computeRelativeDifferenceFromSamples(baselineSamples, currentSamples);
-      return {
+      benchmarkResults[index] = {
         benchmark,
         difference,
         samples: {
@@ -271,7 +269,7 @@ export const runEngineComparison = async options => {
           current: currentStats
         }
       };
-    });
+    }
     return {
       benchmarks: benchmarkResults,
       engine

package/lib/stats.js

@@ -7,6 +7,102 @@ export const relativeMarginOfError = stats => {
   return Math.abs(margin / stats.mean);
 };
 const sumOf = values => values.reduce((total, value) => total + value, 0);
+const sortNumbers = values => [...values].sort((a, b) => a - b);
+const medianOfSorted = values => {
+  if (values.length === 0) {
+    throw new Error("Cannot compute median of an empty sample set.");
+  }
+  const mid = Math.floor(values.length / 2);
+  if (values.length % 2 === 1) {
+    return values[mid];
+  }
+  return (values[mid - 1] + values[mid]) / 2;
+};
+const walshAverages = values => {
+  const averages = [];
+  for (let i = 0; i < values.length; i += 1) {
+    const base = values[i];
+    for (let j = i; j < values.length; j += 1) {
+      averages.push((base + values[j]) / 2);
+    }
+  }
+  return averages;
+};
+const normalQuantile = prob => {
+  if (prob <= 0 || prob >= 1) {
+    throw new Error("Probability must be between 0 and 1.");
+  }
+  const a1 = Number.parseFloat("-39.69683028665376");
+  const a2 = Number.parseFloat("220.9460984245205");
+  const a3 = Number.parseFloat("-275.9285104469687");
+  const a4 = Number.parseFloat("138.357751867269");
+  const a5 = Number.parseFloat("-30.66479806614716");
+  const a6 = Number.parseFloat("2.506628277459239");
+  const b1 = Number.parseFloat("-54.47609879822406");
+  const b2 = Number.parseFloat("161.5858368580409");
+  const b3 = Number.parseFloat("-155.6989798598866");
+  const b4 = Number.parseFloat("66.80131188771972");
+  const b5 = Number.parseFloat("-13.28068155288572");
+  const c1 = Number.parseFloat("-0.007784894002430293");
+  const c2 = Number.parseFloat("-0.3223964580411365");
+  const c3 = Number.parseFloat("-2.400758277161838");
+  const c4 = Number.parseFloat("-2.549732539343734");
+  const c5 = Number.parseFloat("4.374664141464968");
+  const c6 = Number.parseFloat("2.938163982698783");
+  const d1 = Number.parseFloat("0.007784695709041462");
+  const d2 = Number.parseFloat("0.3224671290700398");
+  const d3 = Number.parseFloat("2.445134137142996");
+  const d4 = Number.parseFloat("3.754408661907416");
+  const plow = Number.parseFloat("0.02425");
+  const phigh = 1 - plow;
+  if (prob < plow) {
+    const q = Math.sqrt(-2 * Math.log(prob));
+    return (((((c1 * q + c2) * q + c3) * q + c4) * q + c5) * q + c6) / ((((d1 * q + d2) * q + d3) * q + d4) * q + 1);
+  }
+  if (prob > phigh) {
+    const q = Math.sqrt(-2 * Math.log(1 - prob));
+    return -(((((c1 * q + c2) * q + c3) * q + c4) * q + c5) * q + c6) / ((((d1 * q + d2) * q + d3) * q + d4) * q + 1);
+  }
+  const q = prob - 0.5;
+  const r = q * q;
+  return (((((a1 * r + a2) * r + a3) * r + a4) * r + a5) * r + a6) * q / (((((b1 * r + b2) * r + b3) * r + b4) * r + b5) * r + 1);
+};
+const hodgesLehmannConfidenceInterval = (sortedWalsh, size) => {
+  if (sortedWalsh.length === 0) {
+    throw new Error("Cannot compute confidence interval for empty samples.");
+  }
+  if (size <= 1) {
+    const estimate = medianOfSorted(sortedWalsh);
+    return {
+      high: estimate,
+      low: estimate
+    };
+  }
+  const walshCount = sortedWalsh.length;
+  const alpha = 0.05;
+  const meanRank = size * (size + 1) / 4;
+  const varianceRank = size * (size + 1) * (2 * size + 1) / 24;
+  const stdDevRank = Math.sqrt(varianceRank);
+  const z = normalQuantile(1 - alpha / 2);
+  const critical = Math.floor(meanRank - z * stdDevRank);
+  const lowIndex = Math.max(0, Math.min(walshCount - 1, critical));
+  const highIndex = Math.max(lowIndex, Math.min(walshCount - 1, walshCount - critical - 1));
+  return {
+    high: sortedWalsh[highIndex],
+    low: sortedWalsh[lowIndex]
+  };
+};
+const hodgesLehmannStats = values => {
+  if (values.length === 0) {
+    throw new Error("Cannot compute stats for an empty sample set.");
+  }
+  const walsh = walshAverages(values);
+  const sortedWalsh = sortNumbers(walsh);
+  return {
+    ci: hodgesLehmannConfidenceInterval(sortedWalsh, values.length),
+    mean: medianOfSorted(sortedWalsh)
+  };
+};
 const squareResiduals = (values, mean) => values.map(value => {
   const diff = value - mean;
   return diff * diff;
@@ -99,14 +195,7 @@ const computePairedRelativeStats = (baselineSamples, currentSamples) => {
     }
     diffs.push((currentSamples[index] - baseline) / baseline);
   }
-  const diffStats = summaryStats(diffs);
-  return {
-    ci: confidenceInterval95(samplingDistributionOfTheMean({
-      mean: diffStats.mean,
-      variance: diffStats.variance
-    }, diffStats.size), diffStats.size),
-    mean: diffStats.mean
-  };
+  return hodgesLehmannStats(diffs);
 };
 const computePairedAbsoluteStats = (baselineSamples, currentSamples) => {
   const size = Math.min(baselineSamples.length, currentSamples.length);
@@ -117,14 +206,7 @@ const computePairedAbsoluteStats = (baselineSamples, currentSamples) => {
   for (let index = 0; index < size; index += 1) {
     diffs.push(currentSamples[index] - baselineSamples[index]);
   }
-  const diffStats = summaryStats(diffs);
-  return {
-    ci: confidenceInterval95(samplingDistributionOfTheMean({
-      mean: diffStats.mean,
-      variance: diffStats.variance
-    }, diffStats.size), diffStats.size),
-    mean: diffStats.mean
-  };
+  return hodgesLehmannStats(diffs);
 };
 export const computeRelativeDifferenceFromSamples = (baselineSamples, currentSamples) => ({
   absolute: computePairedAbsoluteStats(baselineSamples, currentSamples),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "perfshield",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "description": "A tool for doing web benchmarking across multiple JS engines and with statistical signifigance",
   "license": "MIT",
   "type": "module",