perfshield 0.0.3 → 0.0.4

package/lib/runner.js

@@ -59,19 +59,39 @@ const buildHarnessIfNeeded = async sourcePath => {
     path: harnessPath
   };
 };
-const warmupBenchmarks = async (harness, benchmarks) => {
+const withFreshHarness = async (engine, harnessPath, baselinePath, currentPath, callback) => {
+  const harness = await createNodeHarness(engine, harnessPath, baselinePath, currentPath);
+  try {
+    return await callback(harness);
+  } finally {
+    await harness.close();
+  }
+};
+const runIterationInFreshHarness = async (engine, harnessPath, baselinePath, currentPath, index, iterations, order) => await withFreshHarness(engine, harnessPath, baselinePath, currentPath, async harness => {
+  const results = {};
+  for (const version of order) {
+    const payload = {
+      index,
+      version
+    };
+    if (iterations != null) {
+      payload.iterations = iterations;
+    }
+    const result = await harness.runSample(payload);
+    results[version] = result.durationMs;
+  }
+  return results;
+});
+const warmupBenchmarks = async (engine, harnessPath, baselinePath, currentPath, benchmarks) => {
+  let roundRobinSeed = 0;
   for (let index = 0; index < benchmarks.length; index += 1) {
     const descriptor = benchmarks[index];
-    for (const version of versions) {
-      await harness.runSample({
-        index,
-        iterations: descriptor.iterations,
-        version
-      });
-    }
+    const order = getVersionOrder(roundRobinSeed);
+    roundRobinSeed += 1;
+    await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
   }
 };
-const collectSamples = async (harness, benchmarks, minSamples) => {
+const collectSamples = async (engine, harnessPath, baselinePath, currentPath, benchmarks, minSamples) => {
   const samples = benchmarks.map(() => ({
     baseline: [],
     current: []
@@ -82,17 +102,12 @@ const collectSamples = async (harness, benchmarks, minSamples) => {
       const descriptor = benchmarks[index];
       const order = getVersionOrder(roundRobinSeed);
       roundRobinSeed += 1;
-      for (const version of order) {
-        const result = await harness.runSample({
-          index,
-          iterations: descriptor.iterations,
-          version
-        });
-        if (version === "baseline") {
-          samples[index].baseline.push(result.durationMs);
-        } else {
-          samples[index].current.push(result.durationMs);
-        }
+      const result = await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
+      if (result.baseline != null) {
+        samples[index].baseline.push(result.baseline);
+      }
+      if (result.current != null) {
+        samples[index].current.push(result.current);
       }
     }
   }
@@ -110,7 +125,7 @@ const autoSampleResolved = (samples, conditions) => samples.every(bucket => {
   }
   return true;
 });
-const autoSample = async (harness, benchmarks, samples, conditions, timeoutMs) => {
+const autoSample = async (engine, harnessPath, baselinePath, currentPath, benchmarks, samples, conditions, timeoutMs) => {
   const startTime = Date.now();
   let roundRobinSeed = 0;
   while (Date.now() - startTime < timeoutMs) {
@@ -122,17 +137,12 @@ const autoSample = async (harness, benchmarks, samples, conditions, timeoutMs) =
         const descriptor = benchmarks[index];
         const order = getVersionOrder(roundRobinSeed);
         roundRobinSeed += 1;
-        for (const version of order) {
-          const result = await harness.runSample({
-            index,
-            iterations: descriptor.iterations,
-            version
-          });
-          if (version === "baseline") {
-            samples[index].baseline.push(result.durationMs);
-          } else {
-            samples[index].current.push(result.durationMs);
-          }
+        const result = await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
+        if (result.baseline != null) {
+          samples[index].baseline.push(result.baseline);
+        }
+        if (result.current != null) {
+          samples[index].current.push(result.current);
         }
       }
     }
@@ -146,12 +156,13 @@ export const runEngineComparison = async options => {
     engine
   } = options;
   const harnessArtifact = await buildHarnessIfNeeded(getHarnessPath());
-  const harness = await createNodeHarness(engine, harnessArtifact.path, resolve(baselinePath), resolve(currentPath));
+  const resolvedBaseline = resolve(baselinePath);
+  const resolvedCurrent = resolve(currentPath);
   try {
-    const benchmarks = await harness.listBenchmarks();
-    await warmupBenchmarks(harness, benchmarks);
-    const samples = await collectSamples(harness, benchmarks, config.sampling.minSamples);
-    await autoSample(harness, benchmarks, samples, config.sampling.conditions, config.sampling.timeoutMs);
+    const benchmarks = await withFreshHarness(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, async harness => await harness.listBenchmarks());
+    await warmupBenchmarks(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks);
+    const samples = await collectSamples(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks, config.sampling.minSamples);
+    await autoSample(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks, samples, config.sampling.conditions, config.sampling.timeoutMs);
     const benchmarkResults = benchmarks.map((benchmark, index) => {
       const baselineSamples = samples[index].baseline;
       const currentSamples = samples[index].current;
@@ -176,7 +187,6 @@ export const runEngineComparison = async options => {
       engine
     };
   } finally {
-    await harness.close();
     if (harnessArtifact.cleanup) {
       await harnessArtifact.cleanup();
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "perfshield",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "A tool for doing web benchmarking across multiple JS engines and with statistical signifigance",
   "license": "MIT",
   "type": "module",