perfshield 0.0.1 → 0.0.3

package/README.md

@@ -62,10 +62,7 @@ export const benchmarks = [
   "sampling": {
     "minSamples": 30,
     "timeoutMs": 10000,
-    "conditions": {
-      "absolute": [0],
-      "relative": [0]
-    }
+    "conditions": [0]
   },
   "report": {
     "formats": ["console", "json"]

package/examples/simple/perfshield.config.json

@@ -13,10 +13,7 @@
   "sampling": {
     "minSamples": 30,
     "timeoutMs": 10000,
-    "conditions": {
-      "absolute": [0],
-      "relative": [0]
-    }
+    "conditions": [0]
   },
   "report": {
     "formats": ["console", "json"]

package/lib/config.js

@@ -158,26 +158,6 @@ const parseEngineConfig = (value, index, issues) => {
     name
   };
 };
-const parseSamplingConditions = (value, issues) => {
-  const conditions = asObject(value, "config.sampling.conditions", issues);
-  if (!conditions) {
-    return null;
-  }
-  validateKeys(conditions, ["absolute", "relative"], "config.sampling.conditions", issues);
-  const absolute = asNumberArray(conditions.absolute, "config.sampling.conditions.absolute", issues, {
-    minLength: 1
-  });
-  const relative = asNumberArray(conditions.relative, "config.sampling.conditions.relative", issues, {
-    minLength: 1
-  });
-  if (!absolute || !relative) {
-    return null;
-  }
-  return {
-    absolute,
-    relative
-  };
-};
 const parseSamplingConfig = (value, issues) => {
   const sampling = asObject(value, "config.sampling", issues);
   if (!sampling) {
@@ -192,7 +172,9 @@ const parseSamplingConfig = (value, issues) => {
     integer: true,
     min: 1
   });
-  const conditions = parseSamplingConditions(sampling.conditions, issues);
+  const conditions = asNumberArray(sampling.conditions, "config.sampling.conditions", issues, {
+    minLength: 1
+  });
   if (minSamples == null || timeoutMs == null || !conditions) {
     return null;
   }

package/lib/regression.js

@@ -3,9 +3,8 @@ export const getRegressions = results => {
   const findings = [];
   for (const result of results) {
     for (const entry of result.benchmarks) {
-      if (isPositiveInterval(entry.difference.absolute.ci) || isPositiveInterval(entry.difference.relative.ci)) {
+      if (isPositiveInterval(entry.difference.relative.ci)) {
         findings.push({
-          absolute: entry.difference.absolute.ci,
           benchmark: entry.benchmark.name,
           engine: result.engine.name,
           relative: entry.difference.relative.ci

package/lib/report/console.js

@@ -16,7 +16,7 @@ export const renderConsoleReport = results => {
     lines.push(`Engine: ${result.engine.name}`);
     for (const entry of result.benchmarks) {
       const unit = entry.benchmark.unit != null ? ` ${entry.benchmark.unit}` : "";
-      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff abs mean=${formatNumber(entry.difference.absolute.mean, 4)}${unit} ci=${formatInterval(entry.difference.absolute.ci, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
+      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
       lines.push(...benchmarkLines);
     }
     lines.push("");

package/lib/runner.js

@@ -8,6 +8,12 @@ import { computeDifference, summaryStats } from "./stats.js";
 const versions = ["baseline", "current"];
 const autoSampleBatchSize = 10;
 const harnessTempPrefix = "perfshield-harness-";
+const getVersionOrder = seed => {
+  if (seed % 2 === 0) {
+    return versions;
+  }
+  return [versions[1], versions[0]];
+};
 const getHarnessPath = () => {
   const override = process.env.WEB_BENCHMARKER_HARNESS_PATH;
   if (override != null) {
@@ -70,10 +76,13 @@ const collectSamples = async (harness, benchmarks, minSamples) => {
     baseline: [],
     current: []
   }));
+  let roundRobinSeed = 0;
   for (let iteration = 0; iteration < minSamples; iteration += 1) {
     for (let index = 0; index < benchmarks.length; index += 1) {
       const descriptor = benchmarks[index];
-      for (const version of versions) {
+      const order = getVersionOrder(roundRobinSeed);
+      roundRobinSeed += 1;
+      for (const version of order) {
         const result = await harness.runSample({
           index,
           iterations: descriptor.iterations,
@@ -94,12 +103,7 @@ const autoSampleResolved = (samples, conditions) => samples.every(bucket => {
   const baselineStats = summaryStats(bucket.baseline);
   const currentStats = summaryStats(bucket.current);
   const diff = computeDifference(baselineStats, currentStats);
-  for (const condition of conditions.absolute) {
-    if (intervalContains(diff.absolute.ci, condition)) {
-      return false;
-    }
-  }
-  for (const condition of conditions.relative) {
+  for (const condition of conditions) {
     if (intervalContains(diff.relative.ci, condition)) {
       return false;
     }
@@ -108,6 +112,7 @@ const autoSampleResolved = (samples, conditions) => samples.every(bucket => {
 });
 const autoSample = async (harness, benchmarks, samples, conditions, timeoutMs) => {
   const startTime = Date.now();
+  let roundRobinSeed = 0;
   while (Date.now() - startTime < timeoutMs) {
     if (autoSampleResolved(samples, conditions)) {
       return;
@@ -115,7 +120,9 @@ const autoSample = async (harness, benchmarks, samples, conditions, timeoutMs) =
     for (let batch = 0; batch < autoSampleBatchSize; batch += 1) {
       for (let index = 0; index < benchmarks.length; index += 1) {
         const descriptor = benchmarks[index];
-        for (const version of versions) {
+        const order = getVersionOrder(roundRobinSeed);
+        roundRobinSeed += 1;
+        for (const version of order) {
           const result = await harness.runSample({
             index,
             iterations: descriptor.iterations,

package/lib/stats.js

@@ -23,10 +23,6 @@ export const samplingDistributionOfTheMean = (distribution, sampleSize) => ({
   mean: distribution.mean,
   variance: distribution.variance / sampleSize
 });
-export const samplingDistributionOfAbsoluteDifferenceOfMeans = (a, b) => ({
-  mean: b.mean - a.mean,
-  variance: a.variance + b.variance
-});
 export const samplingDistributionOfRelativeDifferenceOfMeans = (a, b) => ({
   mean: (b.mean - a.mean) / a.mean,
   variance: (a.variance * b.mean * b.mean + b.variance * a.mean * a.mean) / (a.mean * a.mean * a.mean * a.mean)
@@ -65,14 +61,9 @@ export const computeDifference = (baseline, current) => {
     mean: current.mean,
     variance: current.variance
   }, current.size);
-  const absoluteDist = samplingDistributionOfAbsoluteDifferenceOfMeans(baselineDist, currentDist);
   const relativeDist = samplingDistributionOfRelativeDifferenceOfMeans(baselineDist, currentDist);
   const size = Math.min(baseline.size, current.size);
   return {
-    absolute: {
-      ci: confidenceInterval95(absoluteDist, size),
-      mean: absoluteDist.mean
-    },
     relative: {
       ci: confidenceInterval95(relativeDist, size),
       mean: relativeDist.mean
@@ -92,12 +83,7 @@ export const autoSampleConditionsResolved = (resultStats, conditions) => {
       if (diff == null) {
         continue;
       }
-      for (const condition of conditions.absolute) {
-        if (intervalContains(diff.absolute.ci, condition)) {
-          return false;
-        }
-      }
-      for (const condition of conditions.relative) {
+      for (const condition of conditions) {
         if (intervalContains(diff.relative.ci, condition)) {
           return false;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "perfshield",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "description": "A tool for doing web benchmarking across multiple JS engines and with statistical signifigance",
   "license": "MIT",
   "type": "module",