tarsk 0.5.41 → 0.5.43

package/dist/bundled-skills/skill-creator/scripts/aggregate_benchmark.js

@@ -0,0 +1,377 @@
+#!/usr/bin/env node
+/**
+ * Aggregate individual run results into benchmark summary statistics.
+ *
+ * Usage:
+ *   node aggregate_benchmark.js <benchmark_dir>
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+function calculateStats(values) {
+  if (!values.length) {
+    return { mean: 0.0, stddev: 0.0, min: 0.0, max: 0.0 };
+  }
+
+  const n = values.length;
+  const mean = values.reduce((sum, x) => sum + x, 0) / n;
+  let stddev = 0.0;
+  if (n > 1) {
+    const variance = values.reduce((sum, x) => sum + (x - mean) ** 2, 0) / (n - 1);
+    stddev = Math.sqrt(variance);
+  }
+
+  return {
+    mean: Math.round(mean * 10000) / 10000,
+    stddev: Math.round(stddev * 10000) / 10000,
+    min: Math.round(Math.min(...values) * 10000) / 10000,
+    max: Math.round(Math.max(...values) * 10000) / 10000,
+  };
+}
+
+function globEvalDirs(dir) {
+  if (!fs.existsSync(dir)) return [];
+  return fs
+    .readdirSync(dir, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory() && entry.name.startsWith("eval-"))
+    .map((entry) => path.join(dir, entry.name));
+}
+
+function loadRunResults(benchmarkDir) {
+  const runsDir = path.join(benchmarkDir, "runs");
+  let searchDir;
+  if (fs.existsSync(runsDir) && fs.statSync(runsDir).isDirectory()) {
+    searchDir = runsDir;
+  } else if (globEvalDirs(benchmarkDir).length > 0) {
+    searchDir = benchmarkDir;
+  } else {
+    console.log(
+      `No eval directories found in ${benchmarkDir} or ${path.join(benchmarkDir, "runs")}`,
+    );
+    return {};
+  }
+
+  const results = {};
+
+  const evalDirs = globEvalDirs(searchDir).sort();
+  for (let evalIdx = 0; evalIdx < evalDirs.length; evalIdx++) {
+    const evalDir = evalDirs[evalIdx];
+    const metadataPath = path.join(evalDir, "eval_metadata.json");
+    let evalId = evalIdx;
+    if (fs.existsSync(metadataPath)) {
+      try {
+        const metadata = JSON.parse(fs.readFileSync(metadataPath, "utf-8"));
+        evalId = metadata.eval_id ?? evalIdx;
+      } catch {
+        evalId = evalIdx;
+      }
+    } else {
+      const parts = path.basename(evalDir).split("-");
+      const parsed = parseInt(parts[1], 10);
+      evalId = Number.isNaN(parsed) ? evalIdx : parsed;
+    }
+
+    for (const configName of fs.readdirSync(evalDir).sort()) {
+      const configDir = path.join(evalDir, configName);
+      if (!fs.statSync(configDir).isDirectory()) continue;
+
+      const runDirs = fs
+        .readdirSync(configDir)
+        .filter((name) => name.startsWith("run-"))
+        .map((name) => path.join(configDir, name));
+      if (runDirs.length === 0) continue;
+
+      if (!results[configName]) {
+        results[configName] = [];
+      }
+
+      for (const runDir of runDirs.sort()) {
+        const runNumber = parseInt(path.basename(runDir).split("-")[1], 10);
+        const gradingFile = path.join(runDir, "grading.json");
+        if (!fs.existsSync(gradingFile)) {
+          console.log(`Warning: grading.json not found in ${runDir}`);
+          continue;
+        }
+
+        let grading;
+        try {
+          grading = JSON.parse(fs.readFileSync(gradingFile, "utf-8"));
+        } catch (error) {
+          console.log(`Warning: Invalid JSON in ${gradingFile}: ${error.message}`);
+          continue;
+        }
+
+        const result = {
+          eval_id: evalId,
+          run_number: runNumber,
+          pass_rate: grading.summary?.pass_rate ?? 0.0,
+          passed: grading.summary?.passed ?? 0,
+          failed: grading.summary?.failed ?? 0,
+          total: grading.summary?.total ?? 0,
+        };
+
+        const timing = grading.timing ?? {};
+        result.time_seconds = timing.total_duration_seconds ?? 0.0;
+        const timingFile = path.join(runDir, "timing.json");
+        if (result.time_seconds === 0.0 && fs.existsSync(timingFile)) {
+          try {
+            const timingData = JSON.parse(fs.readFileSync(timingFile, "utf-8"));
+            result.time_seconds = timingData.total_duration_seconds ?? 0.0;
+            result.tokens = timingData.total_tokens ?? 0;
+          } catch {
+            // ignore invalid timing.json
+          }
+        }
+
+        const metrics = grading.execution_metrics ?? {};
+        result.tool_calls = metrics.total_tool_calls ?? 0;
+        if (!result.tokens) {
+          result.tokens = metrics.output_chars ?? 0;
+        }
+        result.errors = metrics.errors_encountered ?? 0;
+
+        const rawExpectations = grading.expectations ?? [];
+        for (const exp of rawExpectations) {
+          if (!("text" in exp) || !("passed" in exp)) {
+            console.log(
+              `Warning: expectation in ${gradingFile} missing required fields (text, passed, evidence): ${JSON.stringify(exp)}`,
+            );
+          }
+        }
+        result.expectations = rawExpectations;
+
+        const notesSummary = grading.user_notes_summary ?? {};
+        result.notes = [
+          ...(notesSummary.uncertainties ?? []),
+          ...(notesSummary.needs_review ?? []),
+          ...(notesSummary.workarounds ?? []),
+        ];
+
+        results[configName].push(result);
+      }
+    }
+  }
+
+  return results;
+}
+
+function aggregateResults(results) {
+  const runSummary = {};
+  const configs = Object.keys(results);
+
+  for (const config of configs) {
+    const runs = results[config] ?? [];
+    if (!runs.length) {
+      runSummary[config] = {
+        pass_rate: { mean: 0.0, stddev: 0.0, min: 0.0, max: 0.0 },
+        time_seconds: { mean: 0.0, stddev: 0.0, min: 0.0, max: 0.0 },
+        tokens: { mean: 0, stddev: 0, min: 0, max: 0 },
+      };
+      continue;
+    }
+
+    runSummary[config] = {
+      pass_rate: calculateStats(runs.map((r) => r.pass_rate)),
+      time_seconds: calculateStats(runs.map((r) => r.time_seconds)),
+      tokens: calculateStats(runs.map((r) => r.tokens ?? 0)),
+    };
+  }
+
+  let primary = {};
+  let baseline = {};
+  if (configs.length >= 2) {
+    primary = runSummary[configs[0]] ?? {};
+    baseline = runSummary[configs[1]] ?? {};
+  } else if (configs.length === 1) {
+    primary = runSummary[configs[0]] ?? {};
+  }
+
+  const deltaPassRate = (primary.pass_rate?.mean ?? 0) - (baseline.pass_rate?.mean ?? 0);
+  const deltaTime = (primary.time_seconds?.mean ?? 0) - (baseline.time_seconds?.mean ?? 0);
+  const deltaTokens = (primary.tokens?.mean ?? 0) - (baseline.tokens?.mean ?? 0);
+
+  runSummary.delta = {
+    pass_rate: `${deltaPassRate >= 0 ? "+" : ""}${deltaPassRate.toFixed(2)}`,
+    time_seconds: `${deltaTime >= 0 ? "+" : ""}${deltaTime.toFixed(1)}`,
+    tokens: `${deltaTokens >= 0 ? "+" : ""}${Math.round(deltaTokens)}`,
+  };
+
+  return runSummary;
+}
+
+function generateBenchmark(benchmarkDir, skillName = "", skillPath = "") {
+  const results = loadRunResults(benchmarkDir);
+  const runSummary = aggregateResults(results);
+
+  const runs = [];
+  for (const config of Object.keys(results)) {
+    for (const result of results[config]) {
+      runs.push({
+        eval_id: result.eval_id,
+        configuration: config,
+        run_number: result.run_number,
+        result: {
+          pass_rate: result.pass_rate,
+          passed: result.passed,
+          failed: result.failed,
+          total: result.total,
+          time_seconds: result.time_seconds,
+          tokens: result.tokens ?? 0,
+          tool_calls: result.tool_calls ?? 0,
+          errors: result.errors ?? 0,
+        },
+        expectations: result.expectations,
+        notes: result.notes,
+      });
+    }
+  }
+
+  const evalIds = [
+    ...new Set(
+      Object.values(results)
+        .flat()
+        .map((r) => r.eval_id),
+    ),
+  ].sort((a, b) => a - b);
+
+  return {
+    metadata: {
+      skill_name: skillName || "<skill-name>",
+      skill_path: skillPath || "<path/to/skill>",
+      executor_model: "<model-name>",
+      analyzer_model: "<model-name>",
+      timestamp: new Date().toISOString().replace(/\.\d{3}Z$/, "Z"),
+      evals_run: evalIds,
+      runs_per_configuration: 3,
+    },
+    runs,
+    run_summary: runSummary,
+    notes: [],
+  };
+}
+
+function generateMarkdown(benchmark) {
+  const metadata = benchmark.metadata;
+  const runSummary = benchmark.run_summary;
+  const configs = Object.keys(runSummary).filter((k) => k !== "delta");
+  const configA = configs[0] ?? "config_a";
+  const configB = configs[1] ?? "config_b";
+  const labelA = configA.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+  const labelB = configB.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+
+  const lines = [
+    `# Skill Benchmark: ${metadata.skill_name}`,
+    "",
+    `**Model**: ${metadata.executor_model}`,
+    `**Date**: ${metadata.timestamp}`,
+    `**Evals**: ${metadata.evals_run.join(", ")} (${metadata.runs_per_configuration} runs each per configuration)`,
+    "",
+    "## Summary",
+    "",
+    `| Metric | ${labelA} | ${labelB} | Delta |`,
+    "|--------|------------|---------------|-------|",
+  ];
+
+  const aSummary = runSummary[configA] ?? {};
+  const bSummary = runSummary[configB] ?? {};
+  const delta = runSummary.delta ?? {};
+
+  const aPr = aSummary.pass_rate ?? {};
+  const bPr = bSummary.pass_rate ?? {};
+  lines.push(
+    `| Pass Rate | ${(aPr.mean ?? 0) * 100}% ± ${(aPr.stddev ?? 0) * 100}% | ${(bPr.mean ?? 0) * 100}% ± ${(bPr.stddev ?? 0) * 100}% | ${delta.pass_rate ?? "—"} |`,
+  );
+
+  const aTime = aSummary.time_seconds ?? {};
+  const bTime = bSummary.time_seconds ?? {};
+  lines.push(
+    `| Time | ${(aTime.mean ?? 0).toFixed(1)}s ± ${(aTime.stddev ?? 0).toFixed(1)}s | ${(bTime.mean ?? 0).toFixed(1)}s ± ${(bTime.stddev ?? 0).toFixed(1)}s | ${delta.time_seconds ?? "—"}s |`,
+  );
+
+  const aTokens = aSummary.tokens ?? {};
+  const bTokens = bSummary.tokens ?? {};
+  lines.push(
+    `| Tokens | ${Math.round(aTokens.mean ?? 0)} ± ${Math.round(aTokens.stddev ?? 0)} | ${Math.round(bTokens.mean ?? 0)} ± ${Math.round(bTokens.stddev ?? 0)} | ${delta.tokens ?? "—"} |`,
+  );
+
+  if (benchmark.notes?.length) {
+    lines.push("", "## Notes", "");
+    for (const note of benchmark.notes) {
+      lines.push(`- ${note}`);
+    }
+  }
+
+  return lines.join("\n");
+}
+
+function parseArgs(argv) {
+  const args = { benchmarkDir: null, skillName: "", skillPath: "", output: null };
+  const positional = [];
+  for (let i = 2; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--skill-name") {
+      args.skillName = argv[++i] ?? "";
+    } else if (arg === "--skill-path") {
+      args.skillPath = argv[++i] ?? "";
+    } else if (arg === "--output" || arg === "-o") {
+      args.output = argv[++i] ?? null;
+    } else if (!arg.startsWith("-")) {
+      positional.push(arg);
+    }
+  }
+  args.benchmarkDir = positional[0] ?? null;
+  return args;
+}
+
+function main() {
+  const args = parseArgs(process.argv);
+  if (!args.benchmarkDir) {
+    console.error(
+      "Usage: node aggregate_benchmark.js <benchmark_dir> [--skill-name NAME] [--skill-path PATH] [-o OUTPUT]",
+    );
+    process.exit(1);
+  }
+
+  const benchmarkDir = path.resolve(args.benchmarkDir);
+  if (!fs.existsSync(benchmarkDir)) {
+    console.error(`Directory not found: ${benchmarkDir}`);
+    process.exit(1);
+  }
+
+  const benchmark = generateBenchmark(benchmarkDir, args.skillName, args.skillPath);
+  const outputJson = args.output
+    ? path.resolve(args.output)
+    : path.join(benchmarkDir, "benchmark.json");
+  const outputMd = outputJson.replace(/\.json$/i, ".md");
+
+  fs.writeFileSync(outputJson, JSON.stringify(benchmark, null, 2));
+  console.log(`Generated: ${outputJson}`);
+
+  const markdown = generateMarkdown(benchmark);
+  fs.writeFileSync(outputMd, markdown);
+  console.log(`Generated: ${outputMd}`);
+
+  const runSummary = benchmark.run_summary;
+  const configs = Object.keys(runSummary).filter((k) => k !== "delta");
+  const delta = runSummary.delta ?? {};
+  console.log("\nSummary:");
+  for (const config of configs) {
+    const pr = runSummary[config].pass_rate.mean;
+    const label = config.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+    console.log(`  ${label}: ${(pr * 100).toFixed(1)}% pass rate`);
+  }
+  console.log(`  Delta:         ${delta.pass_rate ?? "—"}`);
+}
+
+if (require.main === module) {
+  main();
+}
+
+module.exports = {
+  calculateStats,
+  loadRunResults,
+  aggregateResults,
+  generateBenchmark,
+  generateMarkdown,
+};