tarsk 0.5.64 → 0.5.66

package/dist/bundled-skills/skill-creator/eval-viewer/generate_review.js

@@ -95,7 +95,7 @@ function findRunsRecursive(root, current, runs) {
   }
 }
 
-function buildRun(root, runDir) {
+function readEvalMetadata(runDir) {
   let prompt = "";
   let evalId = null;
 
@@ -114,57 +114,69 @@ function buildRun(root, runDir) {
     if (prompt) break;
   }
 
-  if (!prompt) {
-    for (const candidate of [
-      path.join(runDir, "transcript.md"),
-      path.join(runDir, "outputs", "transcript.md"),
-    ]) {
-      if (!fs.existsSync(candidate)) continue;
-      try {
-        const text = fs.readFileSync(candidate, "utf-8");
-        const match = text.match(/## Eval Prompt\n\n([\s\S]*?)(?=\n##|$)/);
-        if (match) prompt = match[1].trim();
-      } catch {
-        // ignore
-      }
-      if (prompt) break;
+  return { prompt, evalId };
+}
+
+function extractPromptFromTranscript(runDir) {
+  for (const candidate of [
+    path.join(runDir, "transcript.md"),
+    path.join(runDir, "outputs", "transcript.md"),
+  ]) {
+    if (!fs.existsSync(candidate)) continue;
+    try {
+      const text = fs.readFileSync(candidate, "utf-8");
+      const match = text.match(/## Eval Prompt\n\n([\s\S]*?)(?=\n##|$)/);
+      if (match) return match[1].trim();
+    } catch {
+      // ignore
     }
   }
+  return "";
+}
 
-  if (!prompt) prompt = "(No prompt found)";
-
-  const runId = path.relative(root, runDir).split(path.sep).join("-");
+function collectOutputFiles(runDir) {
   const outputsDir = path.join(runDir, "outputs");
   const outputFiles = [];
-  if (fs.existsSync(outputsDir) && fs.statSync(outputsDir).isDirectory()) {
-    for (const name of fs.readdirSync(outputsDir).sort()) {
-      const filePath = path.join(outputsDir, name);
-      if (fs.statSync(filePath).isFile() && !METADATA_FILES.has(name)) {
-        outputFiles.push(embedFile(filePath));
-      }
+  if (!fs.existsSync(outputsDir) || !fs.statSync(outputsDir).isDirectory()) {
+    return outputFiles;
+  }
+  for (const name of fs.readdirSync(outputsDir).sort()) {
+    const filePath = path.join(outputsDir, name);
+    if (fs.statSync(filePath).isFile() && !METADATA_FILES.has(name)) {
+      outputFiles.push(embedFile(filePath));
     }
   }
+  return outputFiles;
+}
 
-  let grading = null;
+function readGrading(runDir) {
   for (const candidate of [
     path.join(runDir, "grading.json"),
     path.join(runDir, "..", "grading.json"),
   ]) {
     if (!fs.existsSync(candidate)) continue;
     try {
-      grading = JSON.parse(fs.readFileSync(candidate, "utf-8"));
+      const grading = JSON.parse(fs.readFileSync(candidate, "utf-8"));
+      if (grading) return grading;
     } catch {
       // ignore
     }
-    if (grading) break;
   }
+  return null;
+}
+
+function buildRun(root, runDir) {
+  const { prompt: metadataPrompt, evalId } = readEvalMetadata(runDir);
+  const prompt = metadataPrompt || extractPromptFromTranscript(runDir) || "(No prompt found)";
+
+  const runId = path.relative(root, runDir).split(path.sep).join("-");
 
   return {
     id: runId,
     prompt,
     eval_id: evalId,
-    outputs: outputFiles,
-    grading,
+    outputs: collectOutputFiles(runDir),
+    grading: readGrading(runDir),
   };
 }
 
@@ -531,14 +543,3 @@ function main() {
 if (require.main === module) {
   main();
 }
-
-module.exports = {
-  findRuns,
-  buildRun,
-  embedFile,
-  loadPreviousIteration,
-  generateHtml,
-  createReviewServer,
-  killPort,
-  openUrl,
-};

package/dist/bundled-skills/skill-creator/scripts/aggregate_benchmark.js

@@ -38,117 +38,131 @@ function globEvalDirs(dir) {
     .map((entry) => path.join(dir, entry.name));
 }
 
-function loadRunResults(benchmarkDir) {
+function resolveSearchDir(benchmarkDir) {
   const runsDir = path.join(benchmarkDir, "runs");
-  let searchDir;
   if (fs.existsSync(runsDir) && fs.statSync(runsDir).isDirectory()) {
-    searchDir = runsDir;
-  } else if (globEvalDirs(benchmarkDir).length > 0) {
-    searchDir = benchmarkDir;
-  } else {
-    console.log(
-      `No eval directories found in ${benchmarkDir} or ${path.join(benchmarkDir, "runs")}`,
-    );
-    return {};
+    return runsDir;
+  }
+  if (globEvalDirs(benchmarkDir).length > 0) {
+    return benchmarkDir;
   }
+  console.log(`No eval directories found in ${benchmarkDir} or ${path.join(benchmarkDir, "runs")}`);
+  return null;
+}
 
-  const results = {};
+function resolveEvalId(evalDir, evalIdx) {
+  const metadataPath = path.join(evalDir, "eval_metadata.json");
+  if (fs.existsSync(metadataPath)) {
+    try {
+      const metadata = JSON.parse(fs.readFileSync(metadataPath, "utf-8"));
+      return metadata.eval_id ?? evalIdx;
+    } catch {
+      return evalIdx;
+    }
+  }
+  const parts = path.basename(evalDir).split("-");
+  const parsed = parseInt(parts[1], 10);
+  return Number.isNaN(parsed) ? evalIdx : parsed;
+}
+
+function listRunDirs(configDir) {
+  return fs
+    .readdirSync(configDir)
+    .filter((name) => name.startsWith("run-"))
+    .map((name) => path.join(configDir, name))
+    .sort();
+}
+
+function loadGrading(runDir) {
+  const gradingFile = path.join(runDir, "grading.json");
+  if (!fs.existsSync(gradingFile)) {
+    console.log(`Warning: grading.json not found in ${runDir}`);
+    return null;
+  }
+  try {
+    return JSON.parse(fs.readFileSync(gradingFile, "utf-8"));
+  } catch (error) {
+    console.log(`Warning: Invalid JSON in ${gradingFile}: ${error.message}`);
+    return null;
+  }
+}
+
+function buildResult(evalId, runNumber, grading, runDir) {
+  const result = {
+    eval_id: evalId,
+    run_number: runNumber,
+    pass_rate: grading.summary?.pass_rate ?? 0.0,
+    passed: grading.summary?.passed ?? 0,
+    failed: grading.summary?.failed ?? 0,
+    total: grading.summary?.total ?? 0,
+  };
 
+  const timing = grading.timing ?? {};
+  result.time_seconds = timing.total_duration_seconds ?? 0.0;
+  const timingFile = path.join(runDir, "timing.json");
+  if (result.time_seconds === 0.0 && fs.existsSync(timingFile)) {
+    try {
+      const timingData = JSON.parse(fs.readFileSync(timingFile, "utf-8"));
+      result.time_seconds = timingData.total_duration_seconds ?? 0.0;
+      result.tokens = timingData.total_tokens ?? 0;
+    } catch {
+      // ignore invalid timing.json
+    }
+  }
+
+  const metrics = grading.execution_metrics ?? {};
+  result.tool_calls = metrics.total_tool_calls ?? 0;
+  if (!result.tokens) {
+    result.tokens = metrics.output_chars ?? 0;
+  }
+  result.errors = metrics.errors_encountered ?? 0;
+
+  const rawExpectations = grading.expectations ?? [];
+  for (const exp of rawExpectations) {
+    if (!("text" in exp) || !("passed" in exp)) {
+      console.log(
+        `Warning: expectation in ${path.join(runDir, "grading.json")} missing required fields (text, passed, evidence): ${JSON.stringify(exp)}`,
+      );
+    }
+  }
+  result.expectations = rawExpectations;
+
+  const notesSummary = grading.user_notes_summary ?? {};
+  result.notes = [
+    ...(notesSummary.uncertainties ?? []),
+    ...(notesSummary.needs_review ?? []),
+    ...(notesSummary.workarounds ?? []),
+  ];
+
+  return result;
+}
+
+function loadRunResults(benchmarkDir) {
+  const searchDir = resolveSearchDir(benchmarkDir);
+  if (!searchDir) return {};
+
+  const results = {};
   const evalDirs = globEvalDirs(searchDir).sort();
   for (let evalIdx = 0; evalIdx < evalDirs.length; evalIdx++) {
     const evalDir = evalDirs[evalIdx];
-    const metadataPath = path.join(evalDir, "eval_metadata.json");
-    let evalId = evalIdx;
-    if (fs.existsSync(metadataPath)) {
-      try {
-        const metadata = JSON.parse(fs.readFileSync(metadataPath, "utf-8"));
-        evalId = metadata.eval_id ?? evalIdx;
-      } catch {
-        evalId = evalIdx;
-      }
-    } else {
-      const parts = path.basename(evalDir).split("-");
-      const parsed = parseInt(parts[1], 10);
-      evalId = Number.isNaN(parsed) ? evalIdx : parsed;
-    }
+    const evalId = resolveEvalId(evalDir, evalIdx);
 
     for (const configName of fs.readdirSync(evalDir).sort()) {
       const configDir = path.join(evalDir, configName);
       if (!fs.statSync(configDir).isDirectory()) continue;
 
-      const runDirs = fs
-        .readdirSync(configDir)
-        .filter((name) => name.startsWith("run-"))
-        .map((name) => path.join(configDir, name));
+      const runDirs = listRunDirs(configDir);
       if (runDirs.length === 0) continue;
 
       if (!results[configName]) {
         results[configName] = [];
       }
 
-      for (const runDir of runDirs.sort()) {
+      for (const runDir of runDirs) {
         const runNumber = parseInt(path.basename(runDir).split("-")[1], 10);
-        const gradingFile = path.join(runDir, "grading.json");
-        if (!fs.existsSync(gradingFile)) {
-          console.log(`Warning: grading.json not found in ${runDir}`);
-          continue;
-        }
-
-        let grading;
-        try {
-          grading = JSON.parse(fs.readFileSync(gradingFile, "utf-8"));
-        } catch (error) {
-          console.log(`Warning: Invalid JSON in ${gradingFile}: ${error.message}`);
-          continue;
-        }
-
-        const result = {
-          eval_id: evalId,
-          run_number: runNumber,
-          pass_rate: grading.summary?.pass_rate ?? 0.0,
-          passed: grading.summary?.passed ?? 0,
-          failed: grading.summary?.failed ?? 0,
-          total: grading.summary?.total ?? 0,
-        };
-
-        const timing = grading.timing ?? {};
-        result.time_seconds = timing.total_duration_seconds ?? 0.0;
-        const timingFile = path.join(runDir, "timing.json");
-        if (result.time_seconds === 0.0 && fs.existsSync(timingFile)) {
-          try {
-            const timingData = JSON.parse(fs.readFileSync(timingFile, "utf-8"));
-            result.time_seconds = timingData.total_duration_seconds ?? 0.0;
-            result.tokens = timingData.total_tokens ?? 0;
-          } catch {
-            // ignore invalid timing.json
-          }
-        }
-
-        const metrics = grading.execution_metrics ?? {};
-        result.tool_calls = metrics.total_tool_calls ?? 0;
-        if (!result.tokens) {
-          result.tokens = metrics.output_chars ?? 0;
-        }
-        result.errors = metrics.errors_encountered ?? 0;
-
-        const rawExpectations = grading.expectations ?? [];
-        for (const exp of rawExpectations) {
-          if (!("text" in exp) || !("passed" in exp)) {
-            console.log(
-              `Warning: expectation in ${gradingFile} missing required fields (text, passed, evidence): ${JSON.stringify(exp)}`,
-            );
-          }
-        }
-        result.expectations = rawExpectations;
-
-        const notesSummary = grading.user_notes_summary ?? {};
-        result.notes = [
-          ...(notesSummary.uncertainties ?? []),
-          ...(notesSummary.needs_review ?? []),
-          ...(notesSummary.workarounds ?? []),
-        ];
-
-        results[configName].push(result);
+        const grading = loadGrading(runDir);
+        if (!grading) continue;
+        results[configName].push(buildResult(evalId, runNumber, grading, runDir));
       }
     }
   }
@@ -368,10 +382,4 @@ if (require.main === module) {
   main();
 }
 
-module.exports = {
-  calculateStats,
-  loadRunResults,
-  aggregateResults,
-  generateBenchmark,
-  generateMarkdown,
-};
+module.exports = { calculateStats, buildResult, aggregateResults, generateMarkdown };

package/dist/bundled-skills/skill-creator/scripts/generate_report.js

@@ -37,26 +37,8 @@ function scoreClass(correct, total) {
   return "score-bad";
 }
 
-function generateHtml(data, autoRefresh = false, skillName = "") {
-  const history = data.history ?? [];
-  const titlePrefix = skillName ? escapeHtml(`${skillName} \u2014 `) : "";
-
-  const trainQueries = [];
-  const testQueries = [];
-  if (history.length) {
-    for (const r of history[0].train_results ?? history[0].results ?? []) {
-      trainQueries.push({ query: r.query, should_trigger: r.should_trigger ?? true });
-    }
-    if (history[0].test_results) {
-      for (const r of history[0].test_results) {
-        testQueries.push({ query: r.query, should_trigger: r.should_trigger ?? true });
-      }
-    }
-  }
-
-  const refreshTag = autoRefresh ? '    <meta http-equiv="refresh" content="5">\n' : "";
-  const htmlParts = [
-    `<!DOCTYPE html>
+function buildHead(titlePrefix, refreshTag) {
+  return `<!DOCTYPE html>
 <html>
 <head>
     <meta charset="utf-8">
@@ -175,20 +157,23 @@ ${refreshTag}    <title>${titlePrefix}Skill Description Optimization</title>
     <div class="explainer">
         <strong>Optimizing your skill's description.</strong> This page updates automatically as Tarsk tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Tarsk will apply the best-performing description to your skill.
     </div>
-`,
-  ];
+`;
+}
 
+function buildSummary(data) {
   const bestTestScore = data.best_test_score;
-  htmlParts.push(`
+  return `
     <div class="summary">
         <p><strong>Original:</strong> ${escapeHtml(data.original_description ?? "N/A")}</p>
         <p class="best"><strong>Best:</strong> ${escapeHtml(data.best_description ?? "N/A")}</p>
         <p><strong>Best Score:</strong> ${data.best_score ?? "N/A"} ${bestTestScore ? "(test)" : "(train)"}</p>
         <p><strong>Iterations:</strong> ${data.iterations_run ?? 0} | <strong>Train:</strong> ${data.train_size ?? "?"} | <strong>Test:</strong> ${data.test_size ?? "?"}</p>
     </div>
-`);
+`;
+}
 
-  htmlParts.push(`
+function buildLegend() {
+  return `
     <div class="legend">
         <span style="font-weight:600">Query columns:</span>
         <span class="legend-item"><span class="legend-swatch swatch-positive"></span> Should trigger</span>
@@ -196,9 +181,12 @@ ${refreshTag}    <title>${titlePrefix}Skill Description Optimization</title>
         <span class="legend-item"><span class="legend-swatch swatch-train"></span> Train</span>
         <span class="legend-item"><span class="legend-swatch swatch-test"></span> Test</span>
     </div>
-`);
+`;
+}
 
-  htmlParts.push(`
+function buildTableHeader(trainQueries, testQueries) {
+  const parts = [
+    `
     <div class="table-container">
     <table>
         <thead>
@@ -207,37 +195,53 @@ ${refreshTag}    <title>${titlePrefix}Skill Description Optimization</title>
                 <th>Train</th>
                 <th>Test</th>
                 <th class="query-col">Description</th>
-`);
-
+`,
+  ];
   for (const qinfo of trainQueries) {
     const polarity = qinfo.should_trigger ? "positive-col" : "negative-col";
-    htmlParts.push(`                <th class="${polarity}">${escapeHtml(qinfo.query)}</th>\n`);
+    parts.push(`                <th class="${polarity}">${escapeHtml(qinfo.query)}</th>\n`);
   }
   for (const qinfo of testQueries) {
     const polarity = qinfo.should_trigger ? "positive-col" : "negative-col";
-    htmlParts.push(
+    parts.push(
       `                <th class="test-col ${polarity}">${escapeHtml(qinfo.query)}</th>\n`,
     );
   }
-
-  htmlParts.push(`            </tr>
+  parts.push(`            </tr>
         </thead>
         <tbody>
 `);
+  return parts.join("");
+}
 
-  let bestIter;
+function buildResultCell(qinfo, byQuery, isTest) {
+  const r = byQuery[qinfo.query] ?? {};
+  const didPass = r.pass ?? false;
+  const triggers = r.triggers ?? 0;
+  const runs = r.runs ?? 0;
+  const icon = didPass ? "✓" : "✗";
+  const cssClass = didPass ? "pass" : "fail";
+  const testClass = isTest ? "test-result " : "";
+  return `                <td class="result ${testClass}${cssClass}">${icon}<span class="rate">${triggers}/${runs}</span></td>\n`;
+}
+
+function findBestIteration(history, testQueries) {
   if (testQueries.length) {
-    bestIter = history.reduce(
+    return history.reduce(
       (best, h) => ((h.test_passed ?? 0) > (best.test_passed ?? 0) ? h : best),
       history[0],
     ).iteration;
-  } else {
-    bestIter = history.reduce(
-      (best, h) =>
-        (h.train_passed ?? h.passed ?? 0) > (best.train_passed ?? best.passed ?? 0) ? h : best,
-      history[0],
-    ).iteration;
   }
+  return history.reduce(
+    (best, h) =>
+      (h.train_passed ?? h.passed ?? 0) > (best.train_passed ?? best.passed ?? 0) ? h : best,
+    history[0],
+  ).iteration;
+}
+
+function buildTableRows(history, trainQueries, testQueries) {
+  const parts = [];
+  const bestIter = findBestIteration(history, testQueries);
 
   for (const h of history) {
     const iteration = h.iteration ?? "?";
@@ -248,55 +252,62 @@ ${refreshTag}    <title>${titlePrefix}Skill Description Optimization</title>
     const testByQuery = testResults.length
       ? Object.fromEntries(testResults.map((r) => [r.query, r]))
       : {};
-
     const [trainCorrect, trainRuns] = aggregateRuns(trainResults);
     const [testCorrect, testRuns] = aggregateRuns(testResults);
     const trainClass = scoreClass(trainCorrect, trainRuns);
     const testClass = scoreClass(testCorrect, testRuns);
     const rowClass = iteration === bestIter ? "best-row" : "";
 
-    htmlParts.push(`            <tr class="${rowClass}">
+    parts.push(`            <tr class="${rowClass}">
                 <td>${iteration}</td>
                 <td><span class="score ${trainClass}">${trainCorrect}/${trainRuns}</span></td>
                 <td><span class="score ${testClass}">${testCorrect}/${testRuns}</span></td>
                 <td class="description">${escapeHtml(description)}</td>
 `);
-
     for (const qinfo of trainQueries) {
-      const r = trainByQuery[qinfo.query] ?? {};
-      const didPass = r.pass ?? false;
-      const triggers = r.triggers ?? 0;
-      const runs = r.runs ?? 0;
-      const icon = didPass ? "✓" : "✗";
-      const cssClass = didPass ? "pass" : "fail";
-      htmlParts.push(
-        `                <td class="result ${cssClass}">${icon}<span class="rate">${triggers}/${runs}</span></td>\n`,
-      );
+      parts.push(buildResultCell(qinfo, trainByQuery, false));
     }
-
     for (const qinfo of testQueries) {
-      const r = testByQuery[qinfo.query] ?? {};
-      const didPass = r.pass ?? false;
-      const triggers = r.triggers ?? 0;
-      const runs = r.runs ?? 0;
-      const icon = didPass ? "✓" : "✗";
-      const cssClass = didPass ? "pass" : "fail";
-      htmlParts.push(
-        `                <td class="result test-result ${cssClass}">${icon}<span class="rate">${triggers}/${runs}</span></td>\n`,
-      );
+      parts.push(buildResultCell(qinfo, testByQuery, true));
     }
-
-    htmlParts.push("            </tr>\n");
+    parts.push("            </tr>\n");
   }
 
-  htmlParts.push(`        </tbody>
+  parts.push(`        </tbody>
     </table>
     </div>
 </body>
 </html>
 `);
+  return parts.join("");
+}
+
+function generateHtml(data, autoRefresh = false, skillName = "") {
+  const history = data.history ?? [];
+  const titlePrefix = skillName ? escapeHtml(`${skillName} \u2014 `) : "";
+
+  const trainQueries = [];
+  const testQueries = [];
+  if (history.length) {
+    for (const r of history[0].train_results ?? history[0].results ?? []) {
+      trainQueries.push({ query: r.query, should_trigger: r.should_trigger ?? true });
+    }
+    if (history[0].test_results) {
+      for (const r of history[0].test_results) {
+        testQueries.push({ query: r.query, should_trigger: r.should_trigger ?? true });
+      }
+    }
+  }
+
+  const refreshTag = autoRefresh ? '    <meta http-equiv="refresh" content="5">\n' : "";
 
-  return htmlParts.join("");
+  return (
+    buildHead(titlePrefix, refreshTag) +
+    buildSummary(data) +
+    buildLegend() +
+    buildTableHeader(trainQueries, testQueries) +
+    buildTableRows(history, trainQueries, testQueries)
+  );
 }
 
 function parseArgs(argv) {