tarsk 0.5.78 → 0.5.79

package/dist/bundled-skills/skill-creator/scripts/run_loop.js

@@ -150,128 +150,141 @@ async function runIterationEval({
   return { evalElapsed, trainResults, testResults, trainSummary, testSummary };
 }
 
-async function runIteration(ctx) {
-  const {
-    iteration,
-    maxIterations,
-    trainSet,
-    testSet,
-    name,
-    currentDescription,
-    numWorkers,
-    timeout,
-    projectRoot,
-    runsPerQuery,
-    triggerThreshold,
-    model,
-    verbose,
-    history,
-    liveReportPath,
-    originalDescription,
-    holdout,
-    content,
-    logDir,
-  } = ctx;
+function printIterationHeader(ctx) {
+  if (!ctx.verbose) return;
 
-  if (verbose) {
-    process.stderr.write(`\n${"=".repeat(60)}\n`);
-    process.stderr.write(`Iteration ${iteration}/${maxIterations}\n`);
-    process.stderr.write(`Description: ${currentDescription}\n`);
-    process.stderr.write(`${"=".repeat(60)}\n`);
-  }
+  process.stderr.write(`\n${"=".repeat(60)}\n`);
+  process.stderr.write(`Iteration ${ctx.iteration}/${ctx.maxIterations}\n`);
+  process.stderr.write(`Description: ${ctx.currentDescription}\n`);
+  process.stderr.write(`${"=".repeat(60)}\n`);
+}
 
-  const { evalElapsed, trainResults, testResults, trainSummary, testSummary } =
-    await runIterationEval({
-      trainSet,
-      testSet,
-      name,
-      currentDescription,
-      numWorkers,
-      timeout,
-      projectRoot,
-      runsPerQuery,
-      triggerThreshold,
-      model,
-    });
+function createTestHistoryFields(testSummary, testResults) {
+  if (!testSummary) {
+    return {
+      test_passed: null,
+      test_failed: null,
+      test_total: null,
+      test_results: null,
+    };
+  }
+  return {
+    test_passed: testSummary.passed,
+    test_failed: testSummary.failed,
+    test_total: testSummary.total,
+    test_results: testResults.results,
+  };
+}
 
-  history.push({
-    iteration,
-    description: currentDescription,
+function createHistoryEntry(ctx, evaluation) {
+  const { trainResults, testResults, trainSummary, testSummary } = evaluation;
+  return {
+    iteration: ctx.iteration,
+    description: ctx.currentDescription,
     train_passed: trainSummary.passed,
     train_failed: trainSummary.failed,
     train_total: trainSummary.total,
     train_results: trainResults.results,
-    test_passed: testSummary?.passed ?? null,
-    test_failed: testSummary?.failed ?? null,
-    test_total: testSummary?.total ?? null,
-    test_results: testResults?.results ?? null,
+    ...createTestHistoryFields(testSummary, testResults),
     passed: trainSummary.passed,
     failed: trainSummary.failed,
     total: trainSummary.total,
     results: trainResults.results,
-  });
+  };
+}
 
-  if (liveReportPath) {
-    const partialOutput = {
-      original_description: originalDescription,
-      best_description: currentDescription,
-      best_score: "in progress",
-      iterations_run: history.length,
-      holdout,
-      train_size: trainSet.length,
-      test_size: testSet.length,
-      history,
-    };
-    fs.writeFileSync(liveReportPath, generateHtml(partialOutput, true, name));
-  }
+function writeLiveIterationReport(ctx) {
+  if (!ctx.liveReportPath) return;
+
+  const partialOutput = {
+    original_description: ctx.originalDescription,
+    best_description: ctx.currentDescription,
+    best_score: "in progress",
+    iterations_run: ctx.history.length,
+    holdout: ctx.holdout,
+    train_size: ctx.trainSet.length,
+    test_size: ctx.testSet.length,
+    history: ctx.history,
+  };
+  fs.writeFileSync(ctx.liveReportPath, generateHtml(partialOutput, true, ctx.name));
+}
 
-  if (verbose) {
-    printEvalStats("Train", trainResults.results, evalElapsed);
-    if (testSummary) {
-      printEvalStats("Test ", testResults.results, 0);
-    }
-  }
+function printIterationStats(ctx, evaluation) {
+  if (!ctx.verbose) return;
 
-  if (trainSummary.failed === 0) {
-    if (verbose) {
-      process.stderr.write(`\nAll train queries passed on iteration ${iteration}!\n`);
-    }
-    return `all_passed (iteration ${iteration})`;
+  printEvalStats("Train", evaluation.trainResults.results, evaluation.evalElapsed);
+  if (evaluation.testSummary) {
+    printEvalStats("Test ", evaluation.testResults.results, 0);
   }
+}
 
-  if (iteration === maxIterations) {
-    if (verbose) {
-      process.stderr.write(`\nMax iterations reached (${maxIterations}).\n`);
-    }
-    return `max_iterations (${maxIterations})`;
+function getIterationExitReason(trainSummary, iteration, maxIterations) {
+  if (trainSummary.failed === 0) return `all_passed (iteration ${iteration})`;
+  if (iteration === maxIterations) return `max_iterations (${maxIterations})`;
+  return null;
+}
+
+function printIterationExit(ctx, exitReason) {
+  if (!ctx.verbose || !exitReason) return;
+
+  if (exitReason.startsWith("all_passed")) {
+    process.stderr.write(`\nAll train queries passed on iteration ${ctx.iteration}!\n`);
+    return;
   }
+  process.stderr.write(`\nMax iterations reached (${ctx.maxIterations}).\n`);
+}
 
-  if (verbose) {
+function createBlindedHistory(history) {
+  return history.map(function blindEntry(entry) {
+    return Object.fromEntries(
+      Object.entries(entry).filter(function excludeTestResult([key]) {
+        return !key.startsWith("test_");
+      }),
+    );
+  });
+}
+
+function improveIterationDescription(ctx, trainResults) {
+  if (ctx.verbose) {
     process.stderr.write("\nImproving description...\n");
   }
 
   const improveStart = Date.now();
-  const blindedHistory = history.map((h) =>
-    Object.fromEntries(Object.entries(h).filter(([k]) => !k.startsWith("test_"))),
-  );
   const newDescription = improveDescription(
-    name,
-    content,
-    currentDescription,
+    ctx.name,
+    ctx.content,
+    ctx.currentDescription,
     trainResults,
-    blindedHistory,
-    model,
+    createBlindedHistory(ctx.history),
+    ctx.model,
     null,
-    logDir,
-    iteration,
+    ctx.logDir,
+    ctx.iteration,
   );
   const improveElapsed = (Date.now() - improveStart) / 1000;
 
-  if (verbose) {
+  if (ctx.verbose) {
     process.stderr.write(`Proposed (${improveElapsed.toFixed(1)}s): ${newDescription}\n`);
   }
+  return newDescription;
+}
 
-  ctx.currentDescription = newDescription;
+async function runIteration(ctx) {
+  printIterationHeader(ctx);
+  const evaluation = await runIterationEval(ctx);
+  ctx.history.push(createHistoryEntry(ctx, evaluation));
+  writeLiveIterationReport(ctx);
+  printIterationStats(ctx, evaluation);
+
+  const exitReason = getIterationExitReason(
+    evaluation.trainSummary,
+    ctx.iteration,
+    ctx.maxIterations,
+  );
+  printIterationExit(ctx, exitReason);
+  if (exitReason) return exitReason;
+
+  ctx.currentDescription = improveIterationDescription(ctx, evaluation.trainResults);
   return null;
 }
 
@@ -355,11 +368,16 @@ async function runLoop({
   };
 }
 
-function formatTimestamp(date = new Date()) {
-  const pad = (n) => String(n).padStart(2, "0");
+function padTimestampValue(value) {
+  return String(value).padStart(2, "0");
+}
+
+function formatTimestamp(date = new Date(), dateSeparator = "") {
   return (
-    `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}_` +
-    `${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`
+    `${date.getFullYear()}${dateSeparator}${padTimestampValue(date.getMonth() + 1)}` +
+    `${dateSeparator}${padTimestampValue(date.getDate())}_` +
+    `${padTimestampValue(date.getHours())}${padTimestampValue(date.getMinutes())}` +
+    `${padTimestampValue(date.getSeconds())}`
   );
 }
 
@@ -398,57 +416,86 @@ function parseArgs(argv) {
   return args;
 }
 
-async function main() {
-  const args = parseArgs(process.argv);
-  if (!args.evalSet || !args.skillPath || !args.model) {
-    console.error(
-      "Usage: node run_loop.js --eval-set PATH --skill-path PATH --model MODEL [--description TEXT] [--num-workers N] [--timeout SEC] [--max-iterations N] [--runs-per-query N] [--trigger-threshold R] [--holdout F] [--verbose] [--report auto|none|PATH] [--results-dir DIR]",
-    );
-    process.exit(1);
-  }
+function getCurrentDate() {
+  return new Date();
+}
 
-  const evalSet = JSON.parse(fs.readFileSync(args.evalSet, "utf-8"));
-  const skillPath = path.resolve(args.skillPath);
-  if (!fs.existsSync(path.join(skillPath, "SKILL.md"))) {
-    console.error(`Error: No SKILL.md found at ${skillPath}`);
-    process.exit(1);
-  }
+function createMainDependencies(overrides = {}) {
+  return {
+    argv: process.argv,
+    readFile: fs.readFileSync,
+    fileExists: fs.existsSync,
+    writeFile: fs.writeFileSync,
+    makeDirectory: fs.mkdirSync,
+    resolvePath: path.resolve,
+    joinPath: path.join,
+    basename: path.basename,
+    tempDirectory: os.tmpdir,
+    currentDate: getCurrentDate,
+    openReport: openInBrowser,
+    parseSkill: parseSkillMd,
+    runOptimization: runLoop,
+    renderReport: generateHtml,
+    log: console.log,
+    logError: console.error,
+    writeError: process.stderr.write.bind(process.stderr),
+    exit: process.exit.bind(process),
+    ...overrides,
+  };
+}
 
-  const { name } = parseSkillMd(skillPath);
+function validateMainArgs(args, dependencies) {
+  if (args.evalSet && args.skillPath && args.model) return true;
+  dependencies.logError(
+    "Usage: node run_loop.js --eval-set PATH --skill-path PATH --model MODEL [--description TEXT] [--num-workers N] [--timeout SEC] [--max-iterations N] [--runs-per-query N] [--trigger-threshold R] [--holdout F] [--verbose] [--report auto|none|PATH] [--results-dir DIR]",
+  );
+  dependencies.exit(1);
+  return false;
+}
 
-  let liveReportPath = null;
-  if (args.report !== "none") {
-    if (args.report === "auto") {
-      liveReportPath = path.join(
-        os.tmpdir(),
-        `skill_description_report_${path.basename(skillPath)}_${formatTimestamp()}.html`,
-      );
-    } else {
-      liveReportPath = path.resolve(args.report);
-    }
-    fs.writeFileSync(
-      liveReportPath,
-      "<html><body><h1>Starting optimization loop...</h1><meta http-equiv='refresh' content='5'></body></html>",
-    );
-    openInBrowser(liveReportPath);
+function resolveMainInputs(args, dependencies) {
+  const evalSet = JSON.parse(dependencies.readFile(args.evalSet, "utf-8"));
+  const skillPath = dependencies.resolvePath(args.skillPath);
+  if (!dependencies.fileExists(dependencies.joinPath(skillPath, "SKILL.md"))) {
+    dependencies.logError(`Error: No SKILL.md found at ${skillPath}`);
+    dependencies.exit(1);
+    return null;
   }
+  const { name } = dependencies.parseSkill(skillPath);
+  return { evalSet, skillPath, name };
+}
 
-  let resultsDir = null;
-  if (args.resultsDir) {
-    const now = new Date();
-    const pad = (n) => String(n).padStart(2, "0");
-    const timestamp =
-      `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}_` +
-      `${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
-    resultsDir = path.join(path.resolve(args.resultsDir), timestamp);
-    fs.mkdirSync(resultsDir, { recursive: true });
-  }
+function configureReport(args, skillPath, dependencies) {
+  if (args.report === "none") return null;
+  const liveReportPath =
+    args.report === "auto"
+      ? dependencies.joinPath(
+          dependencies.tempDirectory(),
+          `skill_description_report_${dependencies.basename(skillPath)}_${formatTimestamp(dependencies.currentDate())}.html`,
+        )
+      : dependencies.resolvePath(args.report);
+  dependencies.writeFile(
+    liveReportPath,
+    "<html><body><h1>Starting optimization loop...</h1><meta http-equiv='refresh' content='5'></body></html>",
+  );
+  dependencies.openReport(liveReportPath);
+  return liveReportPath;
+}
 
-  const logDir = resultsDir ? path.join(resultsDir, "logs") : null;
+function configureResults(args, dependencies) {
+  if (!args.resultsDir) return null;
+  const resultsDir = dependencies.joinPath(
+    dependencies.resolvePath(args.resultsDir),
+    formatTimestamp(dependencies.currentDate(), "-"),
+  );
+  dependencies.makeDirectory(resultsDir, { recursive: true });
+  return resultsDir;
+}
 
-  const output = await runLoop({
-    evalSet,
-    skillPath,
+function createRunLoopOptions(args, inputs, liveReportPath, resultsDir, dependencies) {
+  return {
+    evalSet: inputs.evalSet,
+    skillPath: inputs.skillPath,
     descriptionOverride: args.description,
     numWorkers: args.numWorkers,
     timeout: args.timeout,
@@ -459,29 +506,56 @@ async function main() {
     model: args.model,
     verbose: args.verbose,
     liveReportPath,
-    logDir,
-  });
+    logDir: resultsDir ? dependencies.joinPath(resultsDir, "logs") : null,
+  };
+}
 
+function writeMainOutput(output, inputs, liveReportPath, resultsDir, dependencies) {
   const jsonOutput = JSON.stringify(output, null, 2);
-  console.log(jsonOutput);
+  dependencies.log(jsonOutput);
   if (resultsDir) {
-    fs.writeFileSync(path.join(resultsDir, "results.json"), jsonOutput);
+    dependencies.writeFile(dependencies.joinPath(resultsDir, "results.json"), jsonOutput);
   }
-
   if (liveReportPath) {
-    fs.writeFileSync(liveReportPath, generateHtml(output, false, name));
-    process.stderr.write(`\nReport: ${liveReportPath}\n`);
+    dependencies.writeFile(liveReportPath, dependencies.renderReport(output, false, inputs.name));
+    dependencies.writeError(`\nReport: ${liveReportPath}\n`);
   }
-
   if (resultsDir && liveReportPath) {
-    fs.writeFileSync(path.join(resultsDir, "report.html"), generateHtml(output, false, name));
+    dependencies.writeFile(
+      dependencies.joinPath(resultsDir, "report.html"),
+      dependencies.renderReport(output, false, inputs.name),
+    );
   }
-
   if (resultsDir) {
-    process.stderr.write(`Results saved to: ${resultsDir}\n`);
+    dependencies.writeError(`Results saved to: ${resultsDir}\n`);
   }
 }
 
+async function main(overrides = {}) {
+  const dependencies = createMainDependencies(overrides);
+  const args = parseArgs(dependencies.argv);
+  if (!validateMainArgs(args, dependencies)) return null;
+
+  const inputs = resolveMainInputs(args, dependencies);
+  if (!inputs) return null;
+
+  const liveReportPath = configureReport(args, inputs.skillPath, dependencies);
+  const resultsDir = configureResults(args, dependencies);
+  const output = await dependencies.runOptimization(
+    createRunLoopOptions(args, inputs, liveReportPath, resultsDir, dependencies),
+  );
+  writeMainOutput(output, inputs, liveReportPath, resultsDir, dependencies);
+  return output;
+}
+
+module.exports = {
+  createBlindedHistory,
+  createHistoryEntry,
+  getIterationExitReason,
+  main,
+  parseArgs,
+};
+
 if (require.main === module) {
   main().catch((error) => {
     console.error(error.message);

package/dist/bundled-skills/skill-creator/scripts/utils.js

@@ -3,58 +3,88 @@
 const fs = require("fs");
 const path = require("path");
 
-function parseSkillMd(skillPath) {
-  const content = fs.readFileSync(path.join(skillPath, "SKILL.md"), "utf-8");
-  const lines = content.split("\n");
+const MULTILINE_DESCRIPTION_MARKERS = new Set([">", "|", ">-", "|-"]);
 
+function findFrontmatterEnd(lines) {
   if (lines[0]?.trim() !== "---") {
     throw new Error("SKILL.md missing frontmatter (no opening ---)");
   }
 
-  let endIdx = null;
-  for (let i = 1; i < lines.length; i++) {
-    if (lines[i]?.trim() === "---") {
-      endIdx = i;
-      break;
-    }
+  const endIndex = lines.findIndex(function (line, index) {
+    return index > 0 && line?.trim() === "---";
+  });
+  if (endIndex === -1) {
+    throw new Error("SKILL.md missing frontmatter (no closing ---)");
   }
 
-  if (endIdx === null) {
-    throw new Error("SKILL.md missing frontmatter (no closing ---)");
+  return endIndex;
+}
+
+function normalizeQuotedScalar(value) {
+  return value.trim().replace(/^["']|["']$/g, "");
+}
+
+function isIndented(line) {
+  return line.startsWith("  ") || line.startsWith("\t");
+}
+
+function consumeMultilineDescription(lines, startIndex) {
+  const continuationLines = [];
+  let nextIndex = startIndex;
+
+  while (nextIndex < lines.length && isIndented(lines[nextIndex])) {
+    continuationLines.push(lines[nextIndex].trim());
+    nextIndex += 1;
   }
 
-  let name = "";
-  let description = "";
-  const frontmatterLines = lines.slice(1, endIdx);
-  let i = 0;
-  while (i < frontmatterLines.length) {
-    const line = frontmatterLines[i];
-    if (line.startsWith("name:")) {
-      name = line
-        .slice("name:".length)
-        .trim()
-        .replace(/^["']|["']$/g, "");
-    } else if (line.startsWith("description:")) {
-      let value = line.slice("description:".length).trim();
-      if (value === ">" || value === "|" || value === ">-" || value === "|-") {
-        const continuationLines = [];
-        i += 1;
-        while (
-          i < frontmatterLines.length &&
-          (frontmatterLines[i].startsWith("  ") || frontmatterLines[i].startsWith("\t"))
-        ) {
-          continuationLines.push(frontmatterLines[i].trim());
-          i += 1;
-        }
-        description = continuationLines.join(" ");
-        continue;
-      }
-      description = value.replace(/^["']|["']$/g, "");
-    }
-    i += 1;
+  return {
+    description: continuationLines.join(" "),
+    nextIndex,
+  };
+}
+
+function consumeDescription(lines, index, metadata) {
+  const value = lines[index].slice("description:".length).trim();
+  if (!MULTILINE_DESCRIPTION_MARKERS.has(value)) {
+    metadata.description = normalizeQuotedScalar(value);
+    return index + 1;
   }
 
-  return { name, description, content };
+  const multiline = consumeMultilineDescription(lines, index + 1);
+  metadata.description = multiline.description;
+  return multiline.nextIndex;
+}
+
+function consumeFrontmatterLine(lines, index, metadata) {
+  const line = lines[index];
+  if (line.startsWith("name:")) {
+    metadata.name = normalizeQuotedScalar(line.slice("name:".length));
+    return index + 1;
+  }
+  if (line.startsWith("description:")) {
+    return consumeDescription(lines, index, metadata);
+  }
+  return index + 1;
+}
+
+function parseFrontmatter(lines) {
+  const metadata = { name: "", description: "" };
+  let index = 0;
+
+  while (index < lines.length) {
+    index = consumeFrontmatterLine(lines, index, metadata);
+  }
+
+  return metadata;
+}
+
+function parseSkillMd(skillPath) {
+  const content = fs.readFileSync(path.join(skillPath, "SKILL.md"), "utf-8");
+  const lines = content.split("\n");
+  const endIndex = findFrontmatterEnd(lines);
+  const metadata = parseFrontmatter(lines.slice(1, endIndex));
+
+  return { ...metadata, content };
 }
 
 module.exports = { parseSkillMd };