perfshield 0.0.7 → 0.0.9

package/lib/runner.js

@@ -12,6 +12,23 @@ const getVersionOrder = seed => {
   }
   return [versions[1], versions[0]];
 };
+const buildIndexOrder = (count, seed) => {
+  const order = Array.from({
+    length: count
+  }, (_, index) => index);
+  if (count <= 1) {
+    return order;
+  }
+  let state = seed >>> 0;
+  for (let i = count - 1; i > 0; i -= 1) {
+    state = state * 1_664_525 + 1_013_904_223 >>> 0;
+    const j = state % (i + 1);
+    const swap = order[i];
+    order[i] = order[j];
+    order[j] = swap;
+  }
+  return order;
+};
 const sleep = async delayMs => {
   if (delayMs <= 0) {
     return;
@@ -20,43 +37,37 @@ const sleep = async delayMs => {
     setTimeout(resolve, delayMs);
   });
 };
-const warmupBenchmarks = async (harness, benchmarks, delayMs, progress) => {
-  const warmups = [];
-  for (let index = 0; index < benchmarks.length; index += 1) {
-    const descriptor = benchmarks[index];
-    const order = getVersionOrder(index);
-    let baselineSample;
-    let currentSample;
-    for (const version of order) {
-      const result = await harness.runSample({
-        index,
-        iterations: descriptor.iterations,
-        version
-      });
-      if (version === "baseline") {
-        baselineSample = result.durationMs;
-      } else {
-        currentSample = result.durationMs;
-      }
-    }
-    if (baselineSample == null || currentSample == null) {
-      throw new Error("Warmup did not collect baseline/current samples.");
-    }
-    warmups.push({
-      baseline: baselineSample,
-      current: currentSample
+const warmupBenchmark = async (harness, benchmark, index, delayMs, progress) => {
+  const order = getVersionOrder(index);
+  let baselineSample;
+  let currentSample;
+  for (const version of order) {
+    const result = await harness.runSample({
+      index,
+      iterations: benchmark.iterations,
+      version
     });
-    if (progress) {
-      progress({
-        benchmarkCount: benchmarks.length,
-        benchmarkIndex: index,
-        benchmarkName: descriptor.name,
-        phase: "warmup"
-      });
+    if (version === "baseline") {
+      baselineSample = result.durationMs;
+    } else {
+      currentSample = result.durationMs;
     }
-    await sleep(delayMs);
   }
-  return warmups;
+  if (baselineSample == null || currentSample == null) {
+    throw new Error("Warmup did not collect baseline/current samples.");
+  }
+  if (progress) {
+    progress({
+      benchmarkIndex: index,
+      benchmarkName: benchmark.name,
+      phase: "warmup"
+    });
+  }
+  await sleep(delayMs);
+  return {
+    baseline: baselineSample,
+    current: currentSample
+  };
 };
 const computeIterationOverrides = (benchmarks, warmups, minTimeMs) => {
   if (minTimeMs <= 0) {
@@ -121,35 +132,26 @@ const runSamplePair = async (harness, index, iterations, order) => {
     current: currentSample
   };
 };
-const collectSamples = async (harness, benchmarks, minSamples, iterationOverrides, delayMs, minTimeMs, samples, progress) => {
-  const buckets = samples ?? benchmarks.map(() => ({
-    baseline: [],
-    current: []
-  }));
-  let completed = 0;
-  const total = minSamples * benchmarks.length;
+const collectSamplesForBenchmark = async (harness, benchmark, index, minSamples, iterationOverrides, delayMs, minTimeMs, bucket, progress, progressState) => {
   for (let iteration = 0; iteration < minSamples; iteration += 1) {
     const order = getVersionOrder(iteration);
-    for (let index = 0; index < benchmarks.length; index += 1) {
-      const iterations = iterationOverrides[index];
-      const minimumIterations = benchmarks[index].iterations ?? 1;
-      const result = await runSamplePair(harness, index, iterations, order);
-      buckets[index].baseline.push(result.baseline);
-      buckets[index].current.push(result.current);
-      const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
-      iterationOverrides[index] = nextIterations;
-      completed += 1;
-      if (progress) {
-        progress({
-          completed,
-          phase: "samples",
-          total
-        });
-      }
-      await sleep(delayMs);
+    const iterations = iterationOverrides[index];
+    const minimumIterations = benchmark.iterations ?? 1;
+    const result = await runSamplePair(harness, index, iterations, order);
+    bucket.baseline.push(result.baseline);
+    bucket.current.push(result.current);
+    const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
+    iterationOverrides[index] = nextIterations;
+    if (progress && progressState) {
+      progressState.completed += 1;
+      progress({
+        completed: progressState.completed,
+        phase: "samples",
+        total: progressState.total
+      });
     }
+    await sleep(delayMs);
   }
-  return buckets;
 };
 const intervalContains = (interval, value) => interval.low <= value && value <= interval.high;
 const autoSampleResolved = (samples, conditions, maxRelativeMargin) => samples.every(bucket => {
@@ -171,35 +173,33 @@ const autoSampleResolved = (samples, conditions, maxRelativeMargin) => samples.e
   }
   return true;
 });
-const autoSample = async (harness, benchmarks, samples, conditions, maxRelativeMargin, iterationOverrides, delayMs, minTimeMs, progress, timeoutMs) => {
+const autoSampleForBenchmark = async (harness, benchmark, index, bucket, conditions, maxRelativeMargin, iterationOverrides, delayMs, minTimeMs, progress, timeoutMs) => {
   const startTime = Date.now();
   let roundRobinSeed = 0;
   let completed = 0;
   while (Date.now() - startTime < timeoutMs) {
-    if (autoSampleResolved(samples, conditions, maxRelativeMargin)) {
+    if (autoSampleResolved([bucket], conditions, maxRelativeMargin)) {
       return;
     }
     for (let batch = 0; batch < autoSampleBatchSize; batch += 1) {
       const order = getVersionOrder(roundRobinSeed);
       roundRobinSeed += 1;
-      for (let index = 0; index < benchmarks.length; index += 1) {
-        const iterations = iterationOverrides[index];
-        const minimumIterations = benchmarks[index].iterations ?? 1;
-        const result = await runSamplePair(harness, index, iterations, order);
-        samples[index].baseline.push(result.baseline);
-        samples[index].current.push(result.current);
-        const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
-        iterationOverrides[index] = nextIterations;
-        completed += 1;
-        if (progress) {
-          progress({
-            completed,
-            elapsedMs: Date.now() - startTime,
-            phase: "autosample"
-          });
-        }
-        await sleep(delayMs);
+      const iterations = iterationOverrides[index];
+      const minimumIterations = benchmark.iterations ?? 1;
+      const result = await runSamplePair(harness, index, iterations, order);
+      bucket.baseline.push(result.baseline);
+      bucket.current.push(result.current);
+      const nextIterations = updateIterations(iterations ?? minimumIterations, result.baseline, result.current, minTimeMs, minimumIterations);
+      iterationOverrides[index] = nextIterations;
+      completed += 1;
+      if (progress) {
+        progress({
+          completed,
+          elapsedMs: Date.now() - startTime,
+          phase: "autosample"
+        });
       }
+      await sleep(delayMs);
     }
   }
 };
@@ -223,24 +223,41 @@ export const runEngineComparison = async options => {
     const effectiveMinTimeMs = minTimeMs / Math.max(1, sampleScale * benchmarkScale);
     const delayMs = config.sampling.delayMs ?? 0;
     const maxRelativeMargin = config.sampling.maxRelativeMargin ?? defaultMaxRelativeMargin;
-    const warmups = await warmupBenchmarks(harness, benchmarks, delayMs, options.progress);
-    const iterationOverrides = computeIterationOverrides(benchmarks, warmups, effectiveMinTimeMs);
-    const samples = warmups.map(warmup => ({
-      baseline: [warmup.baseline],
-      current: [warmup.current]
-    }));
     const remainingSamples = Math.max(0, config.sampling.minSamples - 1);
-    if (remainingSamples > 0) {
-      await collectSamples(harness, benchmarks, remainingSamples, iterationOverrides, delayMs, effectiveMinTimeMs, samples, options.progress);
-    }
-    await autoSample(harness, benchmarks, samples, config.sampling.conditions, maxRelativeMargin, iterationOverrides, delayMs, effectiveMinTimeMs, options.progress, config.sampling.timeoutMs);
-    const benchmarkResults = benchmarks.map((benchmark, index) => {
-      const baselineSamples = samples[index].baseline;
-      const currentSamples = samples[index].current;
+    const progressState = {
+      completed: 0,
+      total: remainingSamples * benchmarks.length
+    };
+    const benchmarkResults = new Array(benchmarks.length);
+    const iterationOverrides = benchmarks.map(() => undefined);
+    const benchmarkOrder = buildIndexOrder(benchmarks.length, 0);
+    const autoSampleDeadline = Date.now() + config.sampling.timeoutMs;
+    for (const index of benchmarkOrder) {
+      const benchmark = benchmarks[index];
+      const progress = options.progress;
+      const warmupSample = await warmupBenchmark(harness, benchmark, index, delayMs, progress ? event => progress({
+        ...event,
+        benchmarkCount: benchmarks.length
+      }) : undefined);
+      const iterationOverride = computeIterationOverrides([benchmark], [warmupSample], effectiveMinTimeMs)[0];
+      iterationOverrides[index] = iterationOverride;
+      const bucket = {
+        baseline: [warmupSample.baseline],
+        current: [warmupSample.current]
+      };
+      if (remainingSamples > 0) {
+        await collectSamplesForBenchmark(harness, benchmark, index, remainingSamples, iterationOverrides, delayMs, effectiveMinTimeMs, bucket, progress, progressState);
+      }
+      const remainingTimeoutMs = Math.max(0, autoSampleDeadline - Date.now());
+      if (remainingTimeoutMs > 0) {
+        await autoSampleForBenchmark(harness, benchmark, index, bucket, config.sampling.conditions, maxRelativeMargin, iterationOverrides, delayMs, effectiveMinTimeMs, progress, remainingTimeoutMs);
+      }
+      const baselineSamples = bucket.baseline;
+      const currentSamples = bucket.current;
       const baselineStats = summaryStats(baselineSamples);
       const currentStats = summaryStats(currentSamples);
       const difference = computeRelativeDifferenceFromSamples(baselineSamples, currentSamples);
-      return {
+      benchmarkResults[index] = {
         benchmark,
         difference,
         samples: {
@@ -252,7 +269,7 @@ export const runEngineComparison = async options => {
           current: currentStats
         }
       };
-    });
+    }
     return {
       benchmarks: benchmarkResults,
       engine

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "perfshield",
-  "version": "0.0.7",
+  "version": "0.0.9",
   "description": "A tool for doing web benchmarking across multiple JS engines and with statistical signifigance",
   "license": "MIT",
   "type": "module",