perfshield 0.0.1

package/lib/engines/node-harness.js

@@ -0,0 +1,217 @@
+import { createInterface } from "node:readline";
+import { pathToFileURL } from "node:url";
+const toBenchmarkFn = (value, label) => {
+  if (typeof value !== "function") {
+    throw new Error(`${label} must be a function.`);
+  }
+  return (...args) => {
+    return value(...args);
+  };
+};
+const ensureBenchmarkSpec = (spec, label) => {
+  if (spec == null || typeof spec !== "object" || Array.isArray(spec)) {
+    throw new Error(`${label} must be an object.`);
+  }
+  const candidate = spec;
+  const name = candidate.name;
+  if (typeof name !== "string" || name.length === 0) {
+    throw new Error(`${label}.name must be a non-empty string.`);
+  }
+  const fn = toBenchmarkFn(candidate.fn, `${label}.fn`);
+  const setup = candidate.setup == null ? undefined : toBenchmarkFn(candidate.setup, `${label}.setup`);
+  const teardown = candidate.teardown == null ? undefined : toBenchmarkFn(candidate.teardown, `${label}.teardown`);
+  let unit;
+  if (candidate.unit != null) {
+    if (typeof candidate.unit !== "string") {
+      throw new Error(`${label}.unit must be a string.`);
+    }
+    unit = candidate.unit;
+  }
+  let iterations;
+  if (candidate.iterations != null) {
+    if (typeof candidate.iterations !== "number") {
+      throw new Error(`${label}.iterations must be a number.`);
+    }
+    iterations = candidate.iterations;
+  }
+  let metadata;
+  if (candidate.metadata != null) {
+    if (typeof candidate.metadata !== "object" || Array.isArray(candidate.metadata)) {
+      throw new Error(`${label}.metadata must be an object.`);
+    }
+    metadata = candidate.metadata;
+  }
+  return {
+    fn,
+    iterations,
+    metadata,
+    name,
+    setup,
+    teardown,
+    unit
+  };
+};
+const loadBenchmarkModule = async modulePath => {
+  const url = pathToFileURL(modulePath).href;
+  const loaded = await import(url);
+  const benchmarks = loaded.benchmarks;
+  if (!Array.isArray(benchmarks)) {
+    throw new Error("Benchmark module must export a benchmarks array.");
+  }
+  return benchmarks;
+};
+const normalizeBenchmarks = (benchmarks, label) => benchmarks.map((spec, index) => ensureBenchmarkSpec(spec, `${label}.benchmarks[${index}]`));
+const describeBenchmarks = benchmarks => benchmarks.map(benchmark => ({
+  iterations: benchmark.iterations,
+  metadata: benchmark.metadata,
+  name: benchmark.name,
+  unit: benchmark.unit
+}));
+const validateBenchmarkPairs = (baseline, current) => {
+  if (baseline.length !== current.length) {
+    throw new Error(`Benchmark count mismatch: baseline=${baseline.length}, current=${current.length}`);
+  }
+  baseline.forEach((benchmark, index) => {
+    if (benchmark.name !== current[index].name) {
+      throw new Error(`Benchmark name mismatch at index ${index}: ` + `${benchmark.name} vs ${current[index].name}`);
+    }
+  });
+};
+const runBenchmarkSample = async (benchmark, iterationsOverride) => {
+  const iterations = Math.max(1, iterationsOverride ?? benchmark.iterations ?? 1);
+  let setupCompleted = false;
+  if (benchmark.setup) {
+    await benchmark.setup();
+    setupCompleted = true;
+  }
+  const start = process.hrtime.bigint();
+  try {
+    for (let i = 0; i < iterations; i += 1) {
+      await benchmark.fn();
+    }
+  } finally {
+    if (benchmark.teardown && setupCompleted) {
+      await benchmark.teardown();
+    }
+  }
+  const end = process.hrtime.bigint();
+  const durationMs = Number(end - start) / 1e6;
+  return durationMs;
+};
+const state = {
+  current: null
+};
+const handleInit = async message => {
+  const baselinePath = message.baselinePath;
+  const currentPath = message.currentPath;
+  if (baselinePath == null || currentPath == null) {
+    throw new Error("init requires baselinePath and currentPath.");
+  }
+  const baselineBenchmarksRaw = await loadBenchmarkModule(baselinePath);
+  const currentBenchmarksRaw = await loadBenchmarkModule(currentPath);
+  const baselineBenchmarks = normalizeBenchmarks(baselineBenchmarksRaw, "baseline");
+  const currentBenchmarks = normalizeBenchmarks(currentBenchmarksRaw, "current");
+  validateBenchmarkPairs(baselineBenchmarks, currentBenchmarks);
+  state.current = {
+    baseline: baselineBenchmarks,
+    current: currentBenchmarks
+  };
+  return {
+    ok: true
+  };
+};
+const handleList = () => {
+  if (!state.current) {
+    throw new Error("Harness not initialized.");
+  }
+  return {
+    benchmarks: describeBenchmarks(state.current.baseline)
+  };
+};
+const handleRun = async message => {
+  const currentState = state.current;
+  if (!currentState) {
+    throw new Error("Harness not initialized.");
+  }
+  const version = message.version;
+  const index = message.index;
+  if (version !== "baseline" && version !== "current") {
+    throw new Error("run requires version = baseline|current.");
+  }
+  if (typeof index !== "number" || Number.isNaN(index)) {
+    throw new Error("run requires a numeric benchmark index.");
+  }
+  const list = currentState[version];
+  const benchmark = list[index];
+  if (!benchmark) {
+    throw new Error(`Unknown benchmark index ${index}.`);
+  }
+  const durationMs = await runBenchmarkSample(benchmark, message.iterations);
+  return {
+    durationMs
+  };
+};
+const handleMessage = async message => {
+  switch (message.type) {
+    case "init":
+      return await handleInit(message);
+    case "list":
+      return handleList();
+    case "run":
+      return await handleRun(message);
+    case "close":
+      return {
+        ok: true
+      };
+    default:
+      throw new Error(`Unknown message type: ${String(message.type)}`);
+  }
+};
+const sendResponse = payload => {
+  const serialized = JSON.stringify(payload);
+  if (serialized == null) {
+    return;
+  }
+  process.stdout.write(`${serialized}\n`);
+};
+const rl = createInterface({
+  crlfDelay: Infinity,
+  input: process.stdin
+});
+rl.on("line", line => {
+  if (!line.trim()) {
+    return;
+  }
+  let message;
+  try {
+    message = JSON.parse(line);
+  } catch {
+    sendResponse({
+      error: {
+        message: "Invalid JSON payload."
+      },
+      id: null,
+      ok: false
+    });
+    return;
+  }
+  Promise.resolve(handleMessage(message)).then(result => {
+    sendResponse({
+      id: message.id,
+      ok: true,
+      ...result
+    });
+    if (message.type === "close") {
+      rl.close();
+      process.exit(0);
+    }
+  }).catch(error => {
+    sendResponse({
+      error: {
+        message: error instanceof Error ? error.message : String(error)
+      },
+      id: message.id,
+      ok: false
+    });
+  });
+});

package/lib/engines/node.js

@@ -0,0 +1,214 @@
+import { spawn } from "node:child_process";
+import { createInterface } from "node:readline";
+const coerceBenchmarkDescriptor = (value, index) => {
+  if (value == null || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`Benchmark descriptor ${index} must be an object.`);
+  }
+  const descriptor = value;
+  const name = descriptor.name;
+  if (typeof name !== "string" || name.length === 0) {
+    throw new Error(`Benchmark descriptor ${index}.name is required.`);
+  }
+  let unit;
+  if (descriptor.unit != null) {
+    if (typeof descriptor.unit !== "string") {
+      throw new Error(`Benchmark descriptor ${index}.unit must be a string.`);
+    }
+    unit = descriptor.unit;
+  }
+  let iterations;
+  if (descriptor.iterations != null) {
+    if (typeof descriptor.iterations !== "number") {
+      throw new Error(`Benchmark descriptor ${index}.iterations must be a number.`);
+    }
+    iterations = descriptor.iterations;
+  }
+  let metadata;
+  if (descriptor.metadata != null) {
+    if (typeof descriptor.metadata !== "object" || Array.isArray(descriptor.metadata)) {
+      throw new Error(`Benchmark descriptor ${index}.metadata must be an object.`);
+    }
+    metadata = descriptor.metadata;
+  }
+  return {
+    iterations,
+    metadata,
+    name,
+    unit
+  };
+};
+export const runNodeScript = async (engine, scriptPath, args = []) => {
+  const command = engine.command || "node";
+  const commandArgs = [...(engine.args ?? []), scriptPath, ...args];
+  return await new Promise((resolvePromise, reject) => {
+    const child = spawn(command, commandArgs, {
+      env: {
+        ...process.env,
+        ...(engine.env ?? {})
+      },
+      stdio: ["ignore", "pipe", "pipe"]
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", chunk => {
+      stdout += String(chunk);
+    });
+    child.stderr.on("data", chunk => {
+      stderr += String(chunk);
+    });
+    child.on("error", error => {
+      reject(error);
+    });
+    child.on("close", code => {
+      resolvePromise({
+        exitCode: code ?? 0,
+        stderr,
+        stdout
+      });
+    });
+  });
+};
+export const createNodeHarness = async (engine, harnessPath, baselinePath, currentPath) => {
+  const command = engine.command || "node";
+  const args = [...(engine.args ?? []), harnessPath];
+  const child = spawn(command, args, {
+    env: {
+      ...process.env,
+      ...(engine.env ?? {})
+    },
+    stdio: ["pipe", "pipe", "pipe"]
+  });
+  let closing = false;
+  let nextId = 1;
+  const pending = new Map();
+  let stderrBuffer = "";
+  const rejectAll = error => {
+    for (const request of pending.values()) {
+      request.reject(error);
+    }
+    pending.clear();
+  };
+  child.stderr.on("data", chunk => {
+    stderrBuffer += String(chunk);
+  });
+  child.on("exit", code => {
+    if (!closing) {
+      const message = stderrBuffer.trim();
+      const error = new Error(`Harness exited unexpectedly (code ${code ?? "unknown"}).` + (message ? ` ${message}` : ""));
+      rejectAll(error);
+    }
+  });
+  const rl = createInterface({
+    crlfDelay: Infinity,
+    input: child.stdout
+  });
+  rl.on("line", line => {
+    if (!line.trim()) {
+      return;
+    }
+    let payload;
+    try {
+      payload = JSON.parse(line);
+    } catch {
+      return;
+    }
+    if (payload == null || typeof payload.id !== "number") {
+      return;
+    }
+    const request = pending.get(payload.id);
+    if (!request) {
+      return;
+    }
+    pending.delete(payload.id);
+    if (payload.ok === false) {
+      const message = payload.error && typeof payload.error.message === "string" ? payload.error.message : "Harness error.";
+      request.reject(new Error(message));
+      return;
+    }
+    request.resolve(payload);
+  });
+  const send = async (type, body) => new Promise((resolve, reject) => {
+    const id = nextId;
+    nextId += 1;
+    pending.set(id, {
+      reject,
+      resolve
+    });
+    const payloadObject = {
+      id,
+      type
+    };
+    if (body != null) {
+      if (typeof body !== "object" || Array.isArray(body)) {
+        pending.delete(id);
+        reject(new Error("Harness payload must be an object."));
+        return;
+      }
+      for (const [key, value] of Object.entries(body)) {
+        payloadObject[key] = value;
+      }
+    }
+    const payload = JSON.stringify(payloadObject);
+    try {
+      if (!child.stdin.write(`${payload}\n`)) {
+        child.stdin.once("drain", () => {});
+      }
+    } catch (error) {
+      pending.delete(id);
+      reject(error instanceof Error ? error : new Error(String(error)));
+    }
+  });
+  try {
+    await send("init", {
+      baselinePath,
+      currentPath
+    });
+  } catch (error) {
+    closing = true;
+    child.kill();
+    throw error;
+  }
+  const listBenchmarks = async () => {
+    const response = await send("list");
+    if (response == null || typeof response !== "object") {
+      throw new Error("Invalid harness list response.");
+    }
+    const responseObject = response;
+    const benchmarks = responseObject.benchmarks;
+    if (!Array.isArray(benchmarks)) {
+      throw new Error("Harness list response missing benchmarks.");
+    }
+    return benchmarks.map(coerceBenchmarkDescriptor);
+  };
+  const runSample = async options => {
+    const payload = {
+      index: options.index,
+      iterations: options.iterations,
+      version: options.version
+    };
+    const response = await send("run", payload);
+    if (response == null || typeof response !== "object") {
+      throw new Error("Invalid harness run response.");
+    }
+    const durationMs = response.durationMs;
+    if (typeof durationMs !== "number") {
+      throw new Error("Harness run response missing durationMs.");
+    }
+    return {
+      durationMs
+    };
+  };
+  const close = async () => {
+    closing = true;
+    try {
+      await send("close");
+    } finally {
+      child.stdin.end();
+    }
+  };
+  return {
+    close,
+    listBenchmarks,
+    runSample
+  };
+};

package/lib/regression.js

@@ -0,0 +1,17 @@
+const isPositiveInterval = interval => interval.low > 0 && interval.high > 0;
+export const getRegressions = results => {
+  const findings = [];
+  for (const result of results) {
+    for (const entry of result.benchmarks) {
+      if (isPositiveInterval(entry.difference.absolute.ci) || isPositiveInterval(entry.difference.relative.ci)) {
+        findings.push({
+          absolute: entry.difference.absolute.ci,
+          benchmark: entry.benchmark.name,
+          engine: result.engine.name,
+          relative: entry.difference.relative.ci
+        });
+      }
+    }
+  }
+  return findings;
+};

package/lib/report/console.js

@@ -0,0 +1,25 @@
+const formatNumber = (value, decimals) => {
+  if (!Number.isFinite(value)) {
+    return String(value);
+  }
+  return value.toFixed(decimals);
+};
+const formatInterval = (interval, decimals, suffix = "") => `[${formatNumber(interval.low, decimals)}, ${formatNumber(interval.high, decimals)}]${suffix}`;
+const formatRelativeInterval = (interval, decimals) => formatInterval({
+  high: interval.high * 100,
+  low: interval.low * 100
+}, decimals, "%");
+const formatRelativeValue = (value, decimals) => `${formatNumber(value * 100, decimals)}%`;
+export const renderConsoleReport = results => {
+  const lines = [];
+  for (const result of results) {
+    lines.push(`Engine: ${result.engine.name}`);
+    for (const entry of result.benchmarks) {
+      const unit = entry.benchmark.unit != null ? ` ${entry.benchmark.unit}` : "";
+      const benchmarkLines = [`  Benchmark: ${entry.benchmark.name}`, `    baseline mean=${formatNumber(entry.stats.baseline.mean, 4)}${unit} ci=${formatInterval(entry.stats.baseline.meanCI, 4)} sd=${formatNumber(entry.stats.baseline.standardDeviation, 4)}`, `    current  mean=${formatNumber(entry.stats.current.mean, 4)}${unit} ci=${formatInterval(entry.stats.current.meanCI, 4)} sd=${formatNumber(entry.stats.current.standardDeviation, 4)}`, `    diff abs mean=${formatNumber(entry.difference.absolute.mean, 4)}${unit} ci=${formatInterval(entry.difference.absolute.ci, 4)}`, `    diff rel mean=${formatRelativeValue(entry.difference.relative.mean, 2)} ci=${formatRelativeInterval(entry.difference.relative.ci, 2)}`];
+      lines.push(...benchmarkLines);
+    }
+    lines.push("");
+  }
+  return lines.join("\n").trimEnd();
+};

package/lib/report/index.js

@@ -0,0 +1,13 @@
+import { renderConsoleReport } from "./console.js";
+import { renderJsonReport } from "./json.js";
+export const renderReports = (results, formats) => {
+  const outputs = [];
+  for (const format of formats) {
+    if (format === "console") {
+      outputs.push(renderConsoleReport(results));
+    } else if (format === "json") {
+      outputs.push(renderJsonReport(results));
+    }
+  }
+  return outputs;
+};

package/lib/report/json.js

@@ -0,0 +1,12 @@
+export const buildJsonReport = results => ({
+  engines: results.map(result => ({
+    benchmarks: result.benchmarks.map(entry => ({
+      benchmark: entry.benchmark,
+      difference: entry.difference,
+      samples: entry.samples,
+      stats: entry.stats
+    })),
+    engine: result.engine
+  }))
+});
+export const renderJsonReport = results => JSON.stringify(buildJsonReport(results), null, 2);

package/lib/runner.js

@@ -0,0 +1,177 @@
+import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { transformFileAsync } from "@babel/core";
+import { createNodeHarness } from "./engines/node.js";
+import { computeDifference, summaryStats } from "./stats.js";
+const versions = ["baseline", "current"];
+const autoSampleBatchSize = 10;
+const harnessTempPrefix = "perfshield-harness-";
+const getHarnessPath = () => {
+  const override = process.env.WEB_BENCHMARKER_HARNESS_PATH;
+  if (override != null) {
+    return override;
+  }
+  return fileURLToPath(new URL("./engines/node-harness.js", import.meta.url).toString());
+};
+const hasBabelConfig = async () => {
+  const configPath = resolve(process.cwd(), "babel.config.cjs");
+  try {
+    await access(configPath);
+    return true;
+  } catch {
+    return false;
+  }
+};
+const buildHarnessIfNeeded = async sourcePath => {
+  const contents = await readFile(sourcePath, "utf8");
+  if (!contents.includes("import type") && !contents.includes("@flow")) {
+    return {
+      cleanup: null,
+      path: sourcePath
+    };
+  }
+  const usesConfig = await hasBabelConfig();
+  const result = await transformFileAsync(sourcePath, {
+    configFile: usesConfig ? resolve(process.cwd(), "babel.config.cjs") : false,
+    presets: usesConfig ? [] : ["@babel/preset-flow"]
+  });
+  if (!result || !result.code) {
+    throw new Error("Failed to compile node harness.");
+  }
+  const dir = await mkdtemp(join(tmpdir(), harnessTempPrefix));
+  const harnessPath = join(dir, "node-harness.js");
+  await writeFile(harnessPath, result.code, "utf8");
+  return {
+    cleanup: async () => {
+      await rm(dir, {
+        force: true,
+        recursive: true
+      });
+    },
+    path: harnessPath
+  };
+};
+const warmupBenchmarks = async (harness, benchmarks) => {
+  for (let index = 0; index < benchmarks.length; index += 1) {
+    const descriptor = benchmarks[index];
+    for (const version of versions) {
+      await harness.runSample({
+        index,
+        iterations: descriptor.iterations,
+        version
+      });
+    }
+  }
+};
+const collectSamples = async (harness, benchmarks, minSamples) => {
+  const samples = benchmarks.map(() => ({
+    baseline: [],
+    current: []
+  }));
+  for (let iteration = 0; iteration < minSamples; iteration += 1) {
+    for (let index = 0; index < benchmarks.length; index += 1) {
+      const descriptor = benchmarks[index];
+      for (const version of versions) {
+        const result = await harness.runSample({
+          index,
+          iterations: descriptor.iterations,
+          version
+        });
+        if (version === "baseline") {
+          samples[index].baseline.push(result.durationMs);
+        } else {
+          samples[index].current.push(result.durationMs);
+        }
+      }
+    }
+  }
+  return samples;
+};
+const intervalContains = (interval, value) => interval.low <= value && value <= interval.high;
+const autoSampleResolved = (samples, conditions) => samples.every(bucket => {
+  const baselineStats = summaryStats(bucket.baseline);
+  const currentStats = summaryStats(bucket.current);
+  const diff = computeDifference(baselineStats, currentStats);
+  for (const condition of conditions.absolute) {
+    if (intervalContains(diff.absolute.ci, condition)) {
+      return false;
+    }
+  }
+  for (const condition of conditions.relative) {
+    if (intervalContains(diff.relative.ci, condition)) {
+      return false;
+    }
+  }
+  return true;
+});
+const autoSample = async (harness, benchmarks, samples, conditions, timeoutMs) => {
+  const startTime = Date.now();
+  while (Date.now() - startTime < timeoutMs) {
+    if (autoSampleResolved(samples, conditions)) {
+      return;
+    }
+    for (let batch = 0; batch < autoSampleBatchSize; batch += 1) {
+      for (let index = 0; index < benchmarks.length; index += 1) {
+        const descriptor = benchmarks[index];
+        for (const version of versions) {
+          const result = await harness.runSample({
+            index,
+            iterations: descriptor.iterations,
+            version
+          });
+          if (version === "baseline") {
+            samples[index].baseline.push(result.durationMs);
+          } else {
+            samples[index].current.push(result.durationMs);
+          }
+        }
+      }
+    }
+  }
+};
+export const runEngineComparison = async options => {
+  const {
+    baselinePath,
+    config,
+    currentPath,
+    engine
+  } = options;
+  const harnessArtifact = await buildHarnessIfNeeded(getHarnessPath());
+  const harness = await createNodeHarness(engine, harnessArtifact.path, resolve(baselinePath), resolve(currentPath));
+  try {
+    const benchmarks = await harness.listBenchmarks();
+    await warmupBenchmarks(harness, benchmarks);
+    const samples = await collectSamples(harness, benchmarks, config.sampling.minSamples);
+    await autoSample(harness, benchmarks, samples, config.sampling.conditions, config.sampling.timeoutMs);
+    const benchmarkResults = benchmarks.map((benchmark, index) => {
+      const baselineSamples = samples[index].baseline;
+      const currentSamples = samples[index].current;
+      const baselineStats = summaryStats(baselineSamples);
+      const currentStats = summaryStats(currentSamples);
+      const difference = computeDifference(baselineStats, currentStats);
+      return {
+        benchmark,
+        difference,
+        samples: {
+          baseline: baselineSamples,
+          current: currentSamples
+        },
+        stats: {
+          baseline: baselineStats,
+          current: currentStats
+        }
+      };
+    });
+    return {
+      benchmarks: benchmarkResults,
+      engine
+    };
+  } finally {
+    await harness.close();
+    if (harnessArtifact.cleanup) {
+      await harnessArtifact.cleanup();
+    }
+  }
+};