perfshield 0.0.2 → 0.0.3

package/README.md

@@ -62,10 +62,7 @@ export const benchmarks = [
   "sampling": {
     "minSamples": 30,
     "timeoutMs": 10000,
-    "conditions": {
-      "absolute": [0],
-      "relative": [0]
-    }
+    "conditions": [0]
   },
   "report": {
     "formats": ["console", "json"]

package/examples/simple/perfshield.config.json

@@ -13,10 +13,7 @@
   "sampling": {
     "minSamples": 30,
     "timeoutMs": 10000,
-    "conditions": {
-      "absolute": [0],
-      "relative": [0]
-    }
+    "conditions": [0]
   },
   "report": {
     "formats": ["console", "json"]

package/lib/config.js

@@ -158,26 +158,6 @@ const parseEngineConfig = (value, index, issues) => {
     name
   };
 };
-const parseSamplingConditions = (value, issues) => {
-  const conditions = asObject(value, "config.sampling.conditions", issues);
-  if (!conditions) {
-    return null;
-  }
-  validateKeys(conditions, ["absolute", "relative"], "config.sampling.conditions", issues);
-  const absolute = asNumberArray(conditions.absolute, "config.sampling.conditions.absolute", issues, {
-    minLength: 1
-  });
-  const relative = asNumberArray(conditions.relative, "config.sampling.conditions.relative", issues, {
-    minLength: 1
-  });
-  if (!absolute || !relative) {
-    return null;
-  }
-  return {
-    absolute,
-    relative
-  };
-};
 const parseSamplingConfig = (value, issues) => {
   const sampling = asObject(value, "config.sampling", issues);
   if (!sampling) {
@@ -192,7 +172,9 @@ const parseSamplingConfig = (value, issues) => {
     integer: true,
     min: 1
   });
-  const conditions = parseSamplingConditions(sampling.conditions, issues);
+  const conditions = asNumberArray(sampling.conditions, "config.sampling.conditions", issues, {
+    minLength: 1
+  });
   if (minSamples == null || timeoutMs == null || !conditions) {
     return null;
   }

package/lib/regression.js

@@ -3,9 +3,8 @@ export const getRegressions = results => {
   const findings = [];
   for (const result of results) {
     for (const entry of result.benchmarks) {
-      if (isPositiveInterval(entry.difference.absolute.ci) || isPositiveInterval(entry.difference.relative.ci)) {
+      if (isPositiveInterval(entry.difference.relative.ci)) {
         findings.push({
-          absolute: entry.difference.absolute.ci,
           benchmark: entry.benchmark.name,
           engine: result.engine.name,
           relative: entry.difference.relative.ci

package/lib/report/console.js

@@ -16,7 +16,7 @@ export const renderConsoleReport = results => {
     lines.push(`Engine: ${result.engine.name}`);
     for (const entry of result.benchmarks) {
       const unit = entry.benchmark.unit != null ? ` ${entry.benchmark.unit}` : "";
-      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff abs mean=${formatNumber(entry.difference.absolute.mean, 4)}${unit} ci=${formatInterval(entry.difference.absolute.ci, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
+      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
       lines.push(...benchmarkLines);
     }
     lines.push("");

package/lib/runner.js

@@ -103,12 +103,7 @@ const autoSampleResolved = (samples, conditions) => samples.every(bucket => {
   const baselineStats = summaryStats(bucket.baseline);
   const currentStats = summaryStats(bucket.current);
   const diff = computeDifference(baselineStats, currentStats);
-  for (const condition of conditions.absolute) {
-    if (intervalContains(diff.absolute.ci, condition)) {
-      return false;
-    }
-  }
-  for (const condition of conditions.relative) {
+  for (const condition of conditions) {
     if (intervalContains(diff.relative.ci, condition)) {
       return false;
     }

package/lib/stats.js

@@ -23,10 +23,6 @@ export const samplingDistributionOfTheMean = (distribution, sampleSize) => ({
   mean: distribution.mean,
   variance: distribution.variance / sampleSize
 });
-export const samplingDistributionOfAbsoluteDifferenceOfMeans = (a, b) => ({
-  mean: b.mean - a.mean,
-  variance: a.variance + b.variance
-});
 export const samplingDistributionOfRelativeDifferenceOfMeans = (a, b) => ({
   mean: (b.mean - a.mean) / a.mean,
   variance: (a.variance * b.mean * b.mean + b.variance * a.mean * a.mean) / (a.mean * a.mean * a.mean * a.mean)
@@ -65,14 +61,9 @@ export const computeDifference = (baseline, current) => {
     mean: current.mean,
     variance: current.variance
   }, current.size);
-  const absoluteDist = samplingDistributionOfAbsoluteDifferenceOfMeans(baselineDist, currentDist);
   const relativeDist = samplingDistributionOfRelativeDifferenceOfMeans(baselineDist, currentDist);
   const size = Math.min(baseline.size, current.size);
   return {
-    absolute: {
-      ci: confidenceInterval95(absoluteDist, size),
-      mean: absoluteDist.mean
-    },
     relative: {
       ci: confidenceInterval95(relativeDist, size),
       mean: relativeDist.mean
@@ -92,12 +83,7 @@ export const autoSampleConditionsResolved = (resultStats, conditions) => {
       if (diff == null) {
         continue;
       }
-      for (const condition of conditions.absolute) {
-        if (intervalContains(diff.absolute.ci, condition)) {
-          return false;
-        }
-      }
-      for (const condition of conditions.relative) {
+      for (const condition of conditions) {
         if (intervalContains(diff.relative.ci, condition)) {
           return false;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "perfshield",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "A tool for doing web benchmarking across multiple JS engines and with statistical signifigance",
   "license": "MIT",
   "type": "module",