tarsk 0.5.65 → 0.5.67

package/dist/bundled-skills/skill-creator/scripts/run_loop.js

@@ -70,25 +70,7 @@ function printEvalStats(label, results, elapsed, stderr = process.stderr) {
   }
 }
 
-async function runLoop({
-  evalSet,
-  skillPath,
-  descriptionOverride = null,
-  numWorkers,
-  timeout,
-  maxIterations,
-  runsPerQuery,
-  triggerThreshold,
-  holdout,
-  model,
-  verbose = false,
-  liveReportPath = null,
-  logDir = null,
-}) {
-  const projectRoot = findProjectRoot();
-  const { name, description: originalDescription, content } = parseSkillMd(skillPath);
-  let currentDescription = descriptionOverride ?? originalDescription;
-
+function splitSets(evalSet, holdout, verbose) {
   let trainSet;
   let testSet;
   if (holdout > 0) {
@@ -102,22 +84,106 @@ async function runLoop({
     trainSet = evalSet;
     testSet = [];
   }
+  return { trainSet, testSet };
+}
 
-  const history = [];
-  let exitReason = "unknown";
+function summarizeResults(resultList) {
+  const passed = resultList.filter((r) => r.pass).length;
+  const total = resultList.length;
+  return { passed, failed: total - passed, total };
+}
 
-  for (let iteration = 1; iteration <= maxIterations; iteration++) {
-    if (verbose) {
-      process.stderr.write(`\n${"=".repeat(60)}\n`);
-      process.stderr.write(`Iteration ${iteration}/${maxIterations}\n`);
-      process.stderr.write(`Description: ${currentDescription}\n`);
-      process.stderr.write(`${"=".repeat(60)}\n`);
-    }
+function selectBest(history, testSet) {
+  if (testSet.length) {
+    return history.reduce(
+      (a, b) => ((a.test_passed ?? 0) > (b.test_passed ?? 0) ? a : b),
+      history[0],
+    );
+  }
+  return history.reduce(
+    (a, b) => ((a.train_passed ?? 0) > (b.train_passed ?? 0) ? a : b),
+    history[0],
+  );
+}
 
-    const allQueries = trainSet.concat(testSet);
-    const t0 = Date.now();
-    const allResults = await runEval(
-      allQueries,
+async function runIterationEval({
+  trainSet,
+  testSet,
+  name,
+  currentDescription,
+  numWorkers,
+  timeout,
+  projectRoot,
+  runsPerQuery,
+  triggerThreshold,
+  model,
+}) {
+  const allQueries = trainSet.concat(testSet);
+  const t0 = Date.now();
+  const allResults = await runEval(
+    allQueries,
+    name,
+    currentDescription,
+    numWorkers,
+    timeout,
+    projectRoot,
+    runsPerQuery,
+    triggerThreshold,
+    model,
+  );
+  const evalElapsed = (Date.now() - t0) / 1000;
+
+  const trainQueriesSet = new Set(trainSet.map((q) => q.query));
+  const trainResultList = allResults.results.filter((r) => trainQueriesSet.has(r.query));
+  const testResultList = allResults.results.filter((r) => !trainQueriesSet.has(r.query));
+
+  const trainSummary = summarizeResults(trainResultList);
+  const trainResults = { results: trainResultList, summary: trainSummary };
+
+  let testResults = null;
+  let testSummary = null;
+  if (testSet.length) {
+    testSummary = summarizeResults(testResultList);
+    testResults = { results: testResultList, summary: testSummary };
+  }
+
+  return { evalElapsed, trainResults, testResults, trainSummary, testSummary };
+}
+
+async function runIteration(ctx) {
+  const {
+    iteration,
+    maxIterations,
+    trainSet,
+    testSet,
+    name,
+    currentDescription,
+    numWorkers,
+    timeout,
+    projectRoot,
+    runsPerQuery,
+    triggerThreshold,
+    model,
+    verbose,
+    history,
+    liveReportPath,
+    originalDescription,
+    holdout,
+    content,
+    logDir,
+  } = ctx;
+
+  if (verbose) {
+    process.stderr.write(`\n${"=".repeat(60)}\n`);
+    process.stderr.write(`Iteration ${iteration}/${maxIterations}\n`);
+    process.stderr.write(`Description: ${currentDescription}\n`);
+    process.stderr.write(`${"=".repeat(60)}\n`);
+  }
+
+  const { evalElapsed, trainResults, testResults, trainSummary, testSummary } =
+    await runIterationEval({
+      trainSet,
+      testSet,
       name,
       currentDescription,
       numWorkers,
@@ -126,132 +192,147 @@ async function runLoop({
       runsPerQuery,
       triggerThreshold,
       model,
-    );
-    const evalElapsed = (Date.now() - t0) / 1000;
-
-    const trainQueriesSet = new Set(trainSet.map((q) => q.query));
-    const trainResultList = allResults.results.filter((r) => trainQueriesSet.has(r.query));
-    const testResultList = allResults.results.filter((r) => !trainQueriesSet.has(r.query));
-
-    const trainPassed = trainResultList.filter((r) => r.pass).length;
-    const trainTotal = trainResultList.length;
-    const trainSummary = {
-      passed: trainPassed,
-      failed: trainTotal - trainPassed,
-      total: trainTotal,
-    };
-    const trainResults = { results: trainResultList, summary: trainSummary };
-
-    let testResults = null;
-    let testSummary = null;
-    if (testSet.length) {
-      const testPassed = testResultList.filter((r) => r.pass).length;
-      const testTotal = testResultList.length;
-      testSummary = {
-        passed: testPassed,
-        failed: testTotal - testPassed,
-        total: testTotal,
-      };
-      testResults = { results: testResultList, summary: testSummary };
-    }
-
-    history.push({
-      iteration,
-      description: currentDescription,
-      train_passed: trainSummary.passed,
-      train_failed: trainSummary.failed,
-      train_total: trainSummary.total,
-      train_results: trainResults.results,
-      test_passed: testSummary?.passed ?? null,
-      test_failed: testSummary?.failed ?? null,
-      test_total: testSummary?.total ?? null,
-      test_results: testResults?.results ?? null,
-      passed: trainSummary.passed,
-      failed: trainSummary.failed,
-      total: trainSummary.total,
-      results: trainResults.results,
     });
 
-    if (liveReportPath) {
-      const partialOutput = {
-        original_description: originalDescription,
-        best_description: currentDescription,
-        best_score: "in progress",
-        iterations_run: history.length,
-        holdout,
-        train_size: trainSet.length,
-        test_size: testSet.length,
-        history,
-      };
-      fs.writeFileSync(liveReportPath, generateHtml(partialOutput, true, name));
-    }
+  history.push({
+    iteration,
+    description: currentDescription,
+    train_passed: trainSummary.passed,
+    train_failed: trainSummary.failed,
+    train_total: trainSummary.total,
+    train_results: trainResults.results,
+    test_passed: testSummary?.passed ?? null,
+    test_failed: testSummary?.failed ?? null,
+    test_total: testSummary?.total ?? null,
+    test_results: testResults?.results ?? null,
+    passed: trainSummary.passed,
+    failed: trainSummary.failed,
+    total: trainSummary.total,
+    results: trainResults.results,
+  });
 
-    if (verbose) {
-      printEvalStats("Train", trainResults.results, evalElapsed);
-      if (testSummary) {
-        printEvalStats("Test ", testResults.results, 0);
-      }
-    }
+  if (liveReportPath) {
+    const partialOutput = {
+      original_description: originalDescription,
+      best_description: currentDescription,
+      best_score: "in progress",
+      iterations_run: history.length,
+      holdout,
+      train_size: trainSet.length,
+      test_size: testSet.length,
+      history,
+    };
+    fs.writeFileSync(liveReportPath, generateHtml(partialOutput, true, name));
+  }
 
-    if (trainSummary.failed === 0) {
-      exitReason = `all_passed (iteration ${iteration})`;
-      if (verbose) {
-        process.stderr.write(`\nAll train queries passed on iteration ${iteration}!\n`);
-      }
-      break;
+  if (verbose) {
+    printEvalStats("Train", trainResults.results, evalElapsed);
+    if (testSummary) {
+      printEvalStats("Test ", testResults.results, 0);
     }
+  }
 
-    if (iteration === maxIterations) {
-      exitReason = `max_iterations (${maxIterations})`;
-      if (verbose) {
-        process.stderr.write(`\nMax iterations reached (${maxIterations}).\n`);
-      }
-      break;
+  if (trainSummary.failed === 0) {
+    if (verbose) {
+      process.stderr.write(`\nAll train queries passed on iteration ${iteration}!\n`);
     }
+    return `all_passed (iteration ${iteration})`;
+  }
 
+  if (iteration === maxIterations) {
     if (verbose) {
-      process.stderr.write("\nImproving description...\n");
+      process.stderr.write(`\nMax iterations reached (${maxIterations}).\n`);
     }
+    return `max_iterations (${maxIterations})`;
+  }
 
-    const improveStart = Date.now();
-    const blindedHistory = history.map((h) =>
-      Object.fromEntries(Object.entries(h).filter(([k]) => !k.startsWith("test_"))),
-    );
-    const newDescription = improveDescription(
+  if (verbose) {
+    process.stderr.write("\nImproving description...\n");
+  }
+
+  const improveStart = Date.now();
+  const blindedHistory = history.map((h) =>
+    Object.fromEntries(Object.entries(h).filter(([k]) => !k.startsWith("test_"))),
+  );
+  const newDescription = improveDescription(
+    name,
+    content,
+    currentDescription,
+    trainResults,
+    blindedHistory,
+    model,
+    null,
+    logDir,
+    iteration,
+  );
+  const improveElapsed = (Date.now() - improveStart) / 1000;
+
+  if (verbose) {
+    process.stderr.write(`Proposed (${improveElapsed.toFixed(1)}s): ${newDescription}\n`);
+  }
+
+  ctx.currentDescription = newDescription;
+  return null;
+}
+
+async function runLoop({
+  evalSet,
+  skillPath,
+  descriptionOverride = null,
+  numWorkers,
+  timeout,
+  maxIterations,
+  runsPerQuery,
+  triggerThreshold,
+  holdout,
+  model,
+  verbose = false,
+  liveReportPath = null,
+  logDir = null,
+}) {
+  const projectRoot = findProjectRoot();
+  const { name, description: originalDescription, content } = parseSkillMd(skillPath);
+  let currentDescription = descriptionOverride ?? originalDescription;
+
+  const { trainSet, testSet } = splitSets(evalSet, holdout, verbose);
+
+  const history = [];
+  let exitReason = "unknown";
+
+  for (let iteration = 1; iteration <= maxIterations; iteration++) {
+    const ctx = {
+      iteration,
+      maxIterations,
+      trainSet,
+      testSet,
       name,
-      content,
       currentDescription,
-      trainResults,
-      blindedHistory,
+      numWorkers,
+      timeout,
+      projectRoot,
+      runsPerQuery,
+      triggerThreshold,
       model,
-      null,
+      verbose,
+      history,
+      liveReportPath,
+      originalDescription,
+      holdout,
+      content,
       logDir,
-      iteration,
-    );
-    const improveElapsed = (Date.now() - improveStart) / 1000;
-
-    if (verbose) {
-      process.stderr.write(`Proposed (${improveElapsed.toFixed(1)}s): ${newDescription}\n`);
+    };
+    const reason = await runIteration(ctx);
+    currentDescription = ctx.currentDescription;
+    if (reason) {
+      exitReason = reason;
+      break;
     }
-
-    currentDescription = newDescription;
   }
 
-  let best;
-  let bestScore;
-  if (testSet.length) {
-    best = history.reduce(
-      (a, b) => ((a.test_passed ?? 0) > (b.test_passed ?? 0) ? a : b),
-      history[0],
-    );
-    bestScore = `${best.test_passed}/${best.test_total}`;
-  } else {
-    best = history.reduce(
-      (a, b) => ((a.train_passed ?? 0) > (b.train_passed ?? 0) ? a : b),
-      history[0],
-    );
-    bestScore = `${best.train_passed}/${best.train_total}`;
-  }
+  const best = selectBest(history, testSet);
+  const bestScore = testSet.length
+    ? `${best.test_passed}/${best.test_total}`
+    : `${best.train_passed}/${best.train_total}`;
 
   if (verbose) {
     process.stderr.write(`\nExit reason: ${exitReason}\n`);
@@ -407,5 +488,3 @@ if (require.main === module) {
     process.exit(1);
   });
 }
-
-module.exports = { splitEvalSet, runLoop, openInBrowser };