perfshield 0.0.2 → 0.0.4

package/README.md

@@ -62,10 +62,7 @@ export const benchmarks = [
   "sampling": {
     "minSamples": 30,
     "timeoutMs": 10000,
-    "conditions": {
-      "absolute": [0],
-      "relative": [0]
-    }
+    "conditions": [0]
   },
   "report": {
     "formats": ["console", "json"]

package/examples/simple/perfshield.config.json

@@ -13,10 +13,7 @@
   "sampling": {
     "minSamples": 30,
     "timeoutMs": 10000,
-    "conditions": {
-      "absolute": [0],
-      "relative": [0]
-    }
+    "conditions": [0]
   },
   "report": {
     "formats": ["console", "json"]

package/lib/config.js

@@ -158,26 +158,6 @@ const parseEngineConfig = (value, index, issues) => {
     name
   };
 };
-const parseSamplingConditions = (value, issues) => {
-  const conditions = asObject(value, "config.sampling.conditions", issues);
-  if (!conditions) {
-    return null;
-  }
-  validateKeys(conditions, ["absolute", "relative"], "config.sampling.conditions", issues);
-  const absolute = asNumberArray(conditions.absolute, "config.sampling.conditions.absolute", issues, {
-    minLength: 1
-  });
-  const relative = asNumberArray(conditions.relative, "config.sampling.conditions.relative", issues, {
-    minLength: 1
-  });
-  if (!absolute || !relative) {
-    return null;
-  }
-  return {
-    absolute,
-    relative
-  };
-};
 const parseSamplingConfig = (value, issues) => {
   const sampling = asObject(value, "config.sampling", issues);
   if (!sampling) {
@@ -192,7 +172,9 @@ const parseSamplingConfig = (value, issues) => {
     integer: true,
     min: 1
   });
-  const conditions = parseSamplingConditions(sampling.conditions, issues);
+  const conditions = asNumberArray(sampling.conditions, "config.sampling.conditions", issues, {
+    minLength: 1
+  });
   if (minSamples == null || timeoutMs == null || !conditions) {
     return null;
   }

package/lib/regression.js

@@ -3,9 +3,8 @@ export const getRegressions = results => {
   const findings = [];
   for (const result of results) {
     for (const entry of result.benchmarks) {
-      if (isPositiveInterval(entry.difference.absolute.ci) || isPositiveInterval(entry.difference.relative.ci)) {
+      if (isPositiveInterval(entry.difference.relative.ci)) {
         findings.push({
-          absolute: entry.difference.absolute.ci,
           benchmark: entry.benchmark.name,
           engine: result.engine.name,
           relative: entry.difference.relative.ci

package/lib/report/console.js

@@ -16,7 +16,7 @@ export const renderConsoleReport = results => {
     lines.push(`Engine: ${result.engine.name}`);
     for (const entry of result.benchmarks) {
       const unit = entry.benchmark.unit != null ? ` ${entry.benchmark.unit}` : "";
-      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff abs mean=${formatNumber(entry.difference.absolute.mean, 4)}${unit} ci=${formatInterval(entry.difference.absolute.ci, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
+      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
       lines.push(...benchmarkLines);
     }
     lines.push("");

package/lib/runner.js

@@ -59,19 +59,39 @@ const buildHarnessIfNeeded = async sourcePath => {
     path: harnessPath
   };
 };
-const warmupBenchmarks = async (harness, benchmarks) => {
+const withFreshHarness = async (engine, harnessPath, baselinePath, currentPath, callback) => {
+  const harness = await createNodeHarness(engine, harnessPath, baselinePath, currentPath);
+  try {
+    return await callback(harness);
+  } finally {
+    await harness.close();
+  }
+};
+const runIterationInFreshHarness = async (engine, harnessPath, baselinePath, currentPath, index, iterations, order) => await withFreshHarness(engine, harnessPath, baselinePath, currentPath, async harness => {
+  const results = {};
+  for (const version of order) {
+    const payload = {
+      index,
+      version
+    };
+    if (iterations != null) {
+      payload.iterations = iterations;
+    }
+    const result = await harness.runSample(payload);
+    results[version] = result.durationMs;
+  }
+  return results;
+});
+const warmupBenchmarks = async (engine, harnessPath, baselinePath, currentPath, benchmarks) => {
+  let roundRobinSeed = 0;
   for (let index = 0; index < benchmarks.length; index += 1) {
     const descriptor = benchmarks[index];
-    for (const version of versions) {
-      await harness.runSample({
-        index,
-        iterations: descriptor.iterations,
-        version
-      });
-    }
+    const order = getVersionOrder(roundRobinSeed);
+    roundRobinSeed += 1;
+    await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
   }
 };
-const collectSamples = async (harness, benchmarks, minSamples) => {
+const collectSamples = async (engine, harnessPath, baselinePath, currentPath, benchmarks, minSamples) => {
   const samples = benchmarks.map(() => ({
     baseline: [],
     current: []
@@ -82,17 +102,12 @@ const collectSamples = async (harness, benchmarks, minSamples) => {
       const descriptor = benchmarks[index];
       const order = getVersionOrder(roundRobinSeed);
       roundRobinSeed += 1;
-      for (const version of order) {
-        const result = await harness.runSample({
-          index,
-          iterations: descriptor.iterations,
-          version
-        });
-        if (version === "baseline") {
-          samples[index].baseline.push(result.durationMs);
-        } else {
-          samples[index].current.push(result.durationMs);
-        }
+      const result = await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
+      if (result.baseline != null) {
+        samples[index].baseline.push(result.baseline);
+      }
+      if (result.current != null) {
+        samples[index].current.push(result.current);
       }
     }
   }
@@ -103,19 +118,14 @@ const autoSampleResolved = (samples, conditions) => samples.every(bucket => {
   const baselineStats = summaryStats(bucket.baseline);
   const currentStats = summaryStats(bucket.current);
   const diff = computeDifference(baselineStats, currentStats);
-  for (const condition of conditions.absolute) {
-    if (intervalContains(diff.absolute.ci, condition)) {
-      return false;
-    }
-  }
-  for (const condition of conditions.relative) {
+  for (const condition of conditions) {
     if (intervalContains(diff.relative.ci, condition)) {
       return false;
     }
   }
   return true;
 });
-const autoSample = async (harness, benchmarks, samples, conditions, timeoutMs) => {
+const autoSample = async (engine, harnessPath, baselinePath, currentPath, benchmarks, samples, conditions, timeoutMs) => {
   const startTime = Date.now();
   let roundRobinSeed = 0;
   while (Date.now() - startTime < timeoutMs) {
@@ -127,17 +137,12 @@ const autoSample = async (harness, benchmarks, samples, conditions, timeoutMs) =
         const descriptor = benchmarks[index];
         const order = getVersionOrder(roundRobinSeed);
         roundRobinSeed += 1;
-        for (const version of order) {
-          const result = await harness.runSample({
-            index,
-            iterations: descriptor.iterations,
-            version
-          });
-          if (version === "baseline") {
-            samples[index].baseline.push(result.durationMs);
-          } else {
-            samples[index].current.push(result.durationMs);
-          }
+        const result = await runIterationInFreshHarness(engine, harnessPath, baselinePath, currentPath, index, descriptor.iterations, order);
+        if (result.baseline != null) {
+          samples[index].baseline.push(result.baseline);
+        }
+        if (result.current != null) {
+          samples[index].current.push(result.current);
         }
       }
     }
@@ -151,12 +156,13 @@ export const runEngineComparison = async options => {
     engine
   } = options;
   const harnessArtifact = await buildHarnessIfNeeded(getHarnessPath());
-  const harness = await createNodeHarness(engine, harnessArtifact.path, resolve(baselinePath), resolve(currentPath));
+  const resolvedBaseline = resolve(baselinePath);
+  const resolvedCurrent = resolve(currentPath);
   try {
-    const benchmarks = await harness.listBenchmarks();
-    await warmupBenchmarks(harness, benchmarks);
-    const samples = await collectSamples(harness, benchmarks, config.sampling.minSamples);
-    await autoSample(harness, benchmarks, samples, config.sampling.conditions, config.sampling.timeoutMs);
+    const benchmarks = await withFreshHarness(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, async harness => await harness.listBenchmarks());
+    await warmupBenchmarks(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks);
+    const samples = await collectSamples(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks, config.sampling.minSamples);
+    await autoSample(engine, harnessArtifact.path, resolvedBaseline, resolvedCurrent, benchmarks, samples, config.sampling.conditions, config.sampling.timeoutMs);
     const benchmarkResults = benchmarks.map((benchmark, index) => {
       const baselineSamples = samples[index].baseline;
       const currentSamples = samples[index].current;
@@ -181,7 +187,6 @@ export const runEngineComparison = async options => {
       engine
     };
   } finally {
-    await harness.close();
     if (harnessArtifact.cleanup) {
       await harnessArtifact.cleanup();
     }

package/lib/stats.js

@@ -23,10 +23,6 @@ export const samplingDistributionOfTheMean = (distribution, sampleSize) => ({
   mean: distribution.mean,
   variance: distribution.variance / sampleSize
 });
-export const samplingDistributionOfAbsoluteDifferenceOfMeans = (a, b) => ({
-  mean: b.mean - a.mean,
-  variance: a.variance + b.variance
-});
 export const samplingDistributionOfRelativeDifferenceOfMeans = (a, b) => ({
   mean: (b.mean - a.mean) / a.mean,
   variance: (a.variance * b.mean * b.mean + b.variance * a.mean * a.mean) / (a.mean * a.mean * a.mean * a.mean)
@@ -65,14 +61,9 @@ export const computeDifference = (baseline, current) => {
     mean: current.mean,
     variance: current.variance
   }, current.size);
-  const absoluteDist = samplingDistributionOfAbsoluteDifferenceOfMeans(baselineDist, currentDist);
   const relativeDist = samplingDistributionOfRelativeDifferenceOfMeans(baselineDist, currentDist);
   const size = Math.min(baseline.size, current.size);
   return {
-    absolute: {
-      ci: confidenceInterval95(absoluteDist, size),
-      mean: absoluteDist.mean
-    },
     relative: {
       ci: confidenceInterval95(relativeDist, size),
       mean: relativeDist.mean
@@ -92,12 +83,7 @@ export const autoSampleConditionsResolved = (resultStats, conditions) => {
       if (diff == null) {
         continue;
       }
-      for (const condition of conditions.absolute) {
-        if (intervalContains(diff.absolute.ci, condition)) {
-          return false;
-        }
-      }
-      for (const condition of conditions.relative) {
+      for (const condition of conditions) {
         if (intervalContains(diff.relative.ci, condition)) {
           return false;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "perfshield",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "A tool for doing web benchmarking across multiple JS engines and with statistical signifigance",
   "license": "MIT",
   "type": "module",