tarsk 0.5.78 → 0.5.79

package/dist/bundled-skills/skill-creator/scripts/__tests__/run_eval.test.js

@@ -0,0 +1,125 @@
+const assert = require("node:assert/strict");
+const test = require("node:test");
+
+const { handleStreamEventType } = require("../run_eval.js");
+
+function createState(cleanName = "example-skill") {
+  const finishValues = [];
+  return {
+    accumulatedJson: "stale input",
+    cleanName,
+    finish(value) {
+      finishValues.push(value);
+    },
+    finishValues,
+    pendingToolName: null,
+  };
+}
+
+function send(state, type, event = {}) {
+  return handleStreamEventType(event, type, state);
+}
+
+test("finishes when a supported tool input contains the skill name across deltas", function () {
+  const state = createState();
+
+  assert.equal(
+    send(state, "content_block_start", {
+      content_block: { name: "Skill", type: "tool_use" },
+    }),
+    false,
+  );
+  assert.equal(state.pendingToolName, "Skill");
+  assert.equal(state.accumulatedJson, "");
+
+  assert.equal(
+    send(state, "content_block_delta", {
+      delta: { partial_json: '{"skill":"example-', type: "input_json_delta" },
+    }),
+    false,
+  );
+  assert.equal(
+    send(state, "content_block_delta", {
+      delta: { partial_json: 'skill"}', type: "input_json_delta" },
+    }),
+    true,
+  );
+  assert.deepEqual(state.finishValues, [true]);
+});
+
+test("accepts Read and finishes false when its completed input omits the skill name", function () {
+  const state = createState();
+
+  send(state, "content_block_start", {
+    content_block: { name: "Read", type: "tool_use" },
+  });
+  send(state, "content_block_delta", {
+    delta: { partial_json: '{"file_path":"other.md"}', type: "input_json_delta" },
+  });
+
+  assert.equal(send(state, "content_block_stop"), true);
+  assert.deepEqual(state.finishValues, [false]);
+});
+
+test("rejects unsupported tools when their content block starts", function () {
+  const state = createState();
+
+  assert.equal(
+    send(state, "content_block_start", {
+      content_block: { name: "Bash", type: "tool_use" },
+    }),
+    true,
+  );
+  assert.deepEqual(state.finishValues, [false]);
+});
+
+test("only accumulates input JSON while a supported tool is pending", function () {
+  const state = createState();
+
+  assert.equal(
+    send(state, "content_block_delta", {
+      delta: { partial_json: "example-skill", type: "input_json_delta" },
+    }),
+    false,
+  );
+  assert.equal(state.accumulatedJson, "stale input");
+
+  send(state, "content_block_start", {
+    content_block: { name: "Skill", type: "tool_use" },
+  });
+  assert.equal(
+    send(state, "content_block_delta", {
+      delta: { partial_json: "example-skill", type: "text_delta" },
+    }),
+    false,
+  );
+  assert.equal(state.accumulatedJson, "");
+  assert.deepEqual(state.finishValues, []);
+});
+
+test("message stop finishes based on pending tool state", function () {
+  const stateWithoutTool = createState();
+  assert.equal(send(stateWithoutTool, "message_stop"), true);
+  assert.deepEqual(stateWithoutTool.finishValues, [false]);
+
+  const stateWithTool = createState();
+  stateWithTool.pendingToolName = "Skill";
+  stateWithTool.accumulatedJson = '{"skill":"example-skill"}';
+  assert.equal(send(stateWithTool, "message_stop"), true);
+  assert.deepEqual(stateWithTool.finishValues, [true]);
+});
+
+test("ignores unrelated blocks, deltas, stops, and event types", function () {
+  const state = createState();
+
+  assert.equal(send(state, "content_block_start", { content_block: { type: "text" } }), false);
+  assert.equal(
+    send(state, "content_block_delta", {
+      delta: { partial_json: "example-skill", type: "input_json_delta" },
+    }),
+    false,
+  );
+  assert.equal(send(state, "content_block_stop"), false);
+  assert.equal(send(state, "ping"), false);
+  assert.deepEqual(state.finishValues, []);
+});

package/dist/bundled-skills/skill-creator/scripts/__tests__/utils.test.js

@@ -0,0 +1,149 @@
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+const test = require("node:test");
+
+const { parseSkillMd } = require("../utils.js");
+
+function withSkill(content, callback) {
+  const skillPath = fs.mkdtempSync(path.join(os.tmpdir(), "tarsk-skill-utils-"));
+  try {
+    fs.writeFileSync(path.join(skillPath, "SKILL.md"), content);
+    return callback(skillPath);
+  } finally {
+    fs.rmSync(skillPath, { recursive: true, force: true });
+  }
+}
+
+function parse(content) {
+  return withSkill(content, parseSkillMd);
+}
+
+const scalarCases = [
+  {
+    name: "plain scalars",
+    content: "---\nname: example-skill\ndescription: A useful skill\n---\nBody\n",
+    expectedName: "example-skill",
+    expectedDescription: "A useful skill",
+  },
+  {
+    name: "double-quoted scalars",
+    content: '---\nname: "example-skill"\ndescription: "A useful skill"\n---\n',
+    expectedName: "example-skill",
+    expectedDescription: "A useful skill",
+  },
+  {
+    name: "single-quoted scalars",
+    content: "---\nname: 'example-skill'\ndescription: 'A useful skill'\n---\n",
+    expectedName: "example-skill",
+    expectedDescription: "A useful skill",
+  },
+];
+
+for (const testCase of scalarCases) {
+  test(`parses ${testCase.name}`, function () {
+    const result = parse(testCase.content);
+
+    assert.equal(result.name, testCase.expectedName);
+    assert.equal(result.description, testCase.expectedDescription);
+    assert.equal(result.content, testCase.content);
+  });
+}
+
+const multilineMarkers = [">", "|", ">-", "|-"];
+
+for (const marker of multilineMarkers) {
+  test(`parses a ${marker} multiline description with supported indentation`, function () {
+    const result = parse(
+      `---\nname: multiline\ndescription: ${marker}\n  first line\n\tsecond line\nignored: value\n---\n`,
+    );
+
+    assert.equal(result.description, "first line second line");
+  });
+}
+
+test("stops a multiline description at a non-indented line", function () {
+  const result = parse("---\ndescription: >\n  first line\nname: after-description\n---\n");
+
+  assert.equal(result.name, "after-description");
+  assert.equal(result.description, "first line");
+});
+
+test("does not consume a line indented by only one space", function () {
+  const result = parse("---\ndescription: >\n unsupported continuation\n---\n");
+
+  assert.equal(result.description, "");
+});
+
+test("treats a quoted multiline marker as a scalar", function () {
+  const result = parse('---\ndescription: ">"\n  ignored continuation\n---\n');
+
+  assert.equal(result.description, ">");
+});
+
+const duplicateCases = [
+  {
+    name: "duplicate names",
+    content: "---\nname: first\nname: second\n---\n",
+    expectedName: "second",
+    expectedDescription: "",
+  },
+  {
+    name: "duplicate scalar descriptions",
+    content: "---\ndescription: first\ndescription: second\n---\n",
+    expectedName: "",
+    expectedDescription: "second",
+  },
+  {
+    name: "a scalar after a multiline description",
+    content: "---\ndescription: >\n  first\ndescription: second\n---\n",
+    expectedName: "",
+    expectedDescription: "second",
+  },
+  {
+    name: "a multiline description after a scalar",
+    content: "---\ndescription: first\ndescription: |\n  second\n---\n",
+    expectedName: "",
+    expectedDescription: "second",
+  },
+];
+
+for (const testCase of duplicateCases) {
+  test(`uses the last value for ${testCase.name}`, function () {
+    const result = parse(testCase.content);
+
+    assert.equal(result.name, testCase.expectedName);
+    assert.equal(result.description, testCase.expectedDescription);
+  });
+}
+
+const malformedCases = [
+  {
+    name: "missing opening delimiter",
+    content: "name: example\n---\n",
+    message: "SKILL.md missing frontmatter (no opening ---)",
+  },
+  {
+    name: "missing closing delimiter",
+    content: "---\nname: example\n",
+    message: "SKILL.md missing frontmatter (no closing ---)",
+  },
+];
+
+for (const testCase of malformedCases) {
+  test(`rejects frontmatter with a ${testCase.name}`, function () {
+    assert.throws(
+      function () {
+        parse(testCase.content);
+      },
+      { message: testCase.message },
+    );
+  });
+}
+
+test("returns empty metadata when frontmatter has no supported fields", function () {
+  const content = "---\ncompatibility: everywhere\n---\nBody\n";
+
+  assert.deepEqual(parse(content), { name: "", description: "", content });
+});

package/dist/bundled-skills/skill-creator/scripts/aggregate_benchmark.js

@@ -87,6 +87,7 @@ function loadGrading(runDir) {
   }
 }
 
+// fallow-ignore-next-line complexity
 function buildResult(evalId, runNumber, grading, runDir) {
   const result = {
     eval_id: evalId,
@@ -265,6 +266,7 @@ function generateBenchmark(benchmarkDir, skillName = "", skillPath = "") {
   };
 }
 
+// fallow-ignore-next-line complexity
 function generateMarkdown(benchmark) {
   const metadata = benchmark.metadata;
   const runSummary = benchmark.run_summary;

package/dist/bundled-skills/skill-creator/scripts/improve_description.js

@@ -71,32 +71,52 @@ function buildFailuresText(evalResults) {
   return text;
 }
 
+function isDefined(value) {
+  return value != null;
+}
+
+function firstDefined(...values) {
+  return values.find(isDefined);
+}
+
+function formatHistoryScore(attempt) {
+  const trainPassed = firstDefined(attempt.train_passed, attempt.passed, 0);
+  const trainTotal = firstDefined(attempt.train_total, attempt.total, 0);
+  const trainScore = `${trainPassed}/${trainTotal}`;
+  if (attempt.test_passed == null) {
+    return `train=${trainScore}`;
+  }
+
+  const testTotal = firstDefined(attempt.test_total, "?");
+  return `train=${trainScore}, test=${attempt.test_passed}/${testTotal}`;
+}
+
+function formatHistoryResult(result) {
+  const status = result.pass ? "PASS" : "FAIL";
+  return `  [${status}] "${result.query.slice(0, 80)}" (triggered ${result.triggers}/${result.runs})\n`;
+}
+
+function renderHistoryAttempt(attempt) {
+  let text = `<attempt ${formatHistoryScore(attempt)}>\n`;
+  text += `Description: "${attempt.description}"\n`;
+  if (attempt.results) {
+    text += "Train results:\n";
+    text += attempt.results.map(formatHistoryResult).join("");
+  }
+  if (attempt.note) {
+    text += `Note: ${attempt.note}\n`;
+  }
+  return `${text}</attempt>\n\n`;
+}
+
 function buildHistoryText(history) {
   if (!history.length) {
     return "";
   }
 
-  let text = "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n";
-  for (const h of history) {
-    const trainS = `${h.train_passed ?? h.passed ?? 0}/${h.train_total ?? h.total ?? 0}`;
-    const testS = h.test_passed != null ? `${h.test_passed}/${h.test_total ?? "?"}` : null;
-    const scoreStr = `train=${trainS}` + (testS ? `, test=${testS}` : "");
-    text += `<attempt ${scoreStr}>\n`;
-    text += `Description: "${h.description}"\n`;
-    if (h.results) {
-      text += "Train results:\n";
-      for (const r of h.results) {
-        const status = r.pass ? "PASS" : "FAIL";
-        text += `  [${status}] "${r.query.slice(0, 80)}" (triggered ${r.triggers}/${r.runs})\n`;
-      }
-    }
-    if (h.note) {
-      text += `Note: ${h.note}\n`;
-    }
-    text += "</attempt>\n\n";
-  }
-
-  return text;
+  const header =
+    "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n";
+  return header + history.map(renderHistoryAttempt).join("");
 }
 
 function buildImprovePrompt(
@@ -307,4 +327,4 @@ if (require.main === module) {
   main();
 }
 
-module.exports = { improveDescription, callTarsk };
+module.exports = { improveDescription, buildHistoryText };

package/dist/bundled-skills/skill-creator/scripts/package_skill.js

@@ -4,7 +4,7 @@
 const fs = require("fs");
 const path = require("path");
 const os = require("os");
-const { execSync } = require("child_process");
+const { execFileSync } = require("child_process");
 const { validateSkill } = require("./quick_validate.js");
 
 const EXCLUDE_DIRS = new Set(["__pycache__", "node_modules"]);
@@ -47,75 +47,68 @@ function walkFiles(dir) {
   return files;
 }
 
-function packageSkill(skillPath, outputDir = null) {
-  skillPath = path.resolve(skillPath);
-
+function validateSkillDirectory(skillPath) {
   if (!fs.existsSync(skillPath)) {
     console.log(`❌ Error: Skill folder not found: ${skillPath}`);
-    return null;
+    return false;
   }
-
   if (!fs.statSync(skillPath).isDirectory()) {
     console.log(`❌ Error: Path is not a directory: ${skillPath}`);
-    return null;
+    return false;
   }
-
-  const skillMd = path.join(skillPath, "SKILL.md");
-  if (!fs.existsSync(skillMd)) {
+  if (!fs.existsSync(path.join(skillPath, "SKILL.md"))) {
     console.log(`❌ Error: SKILL.md not found in ${skillPath}`);
-    return null;
-  }
-
-  console.log("🔍 Validating skill...");
-  const [valid, message] = validateSkill(skillPath);
-  if (!valid) {
-    console.log(`❌ Validation failed: ${message}`);
-    console.log("   Please fix the validation errors before packaging.");
-    return null;
+    return false;
   }
-  console.log(`✅ ${message}\n`);
+  return true;
+}
 
-  const skillName = path.basename(skillPath);
+function prepareOutputPath(skillPath, outputDir) {
   const outputPath = outputDir ? path.resolve(outputDir) : process.cwd();
   if (outputDir) {
     fs.mkdirSync(outputPath, { recursive: true });
   }
-  const skillFilename = path.join(outputPath, `${skillName}.skill`);
+  return path.join(outputPath, `${path.basename(skillPath)}.skill`);
+}
 
+function collectIncludedFiles(skillPath) {
   const parentDir = path.dirname(skillPath);
   const includedFiles = [];
   for (const filePath of walkFiles(skillPath)) {
     const arcname = path.relative(parentDir, filePath);
     if (shouldExclude(arcname)) {
       console.log(`  Skipped: ${arcname}`);
-      continue;
+    } else {
+      includedFiles.push({ filePath, arcname });
     }
-    includedFiles.push({ filePath, arcname });
   }
+  return includedFiles;
+}
 
-  if (includedFiles.length === 0) {
-    console.log("❌ Error: No files to package");
-    return null;
+function stageFiles(includedFiles, stagingDir) {
+  for (const { filePath, arcname } of includedFiles) {
+    const destination = path.join(stagingDir, arcname);
+    fs.mkdirSync(path.dirname(destination), { recursive: true });
+    fs.copyFileSync(filePath, destination);
+    console.log(`  Added: ${arcname}`);
   }
+}
 
+function executeArchive(stagingDir, skillFilename, skillName) {
+  execFileSync("zip", ["-r", "-q", skillFilename, skillName], {
+    cwd: stagingDir,
+    stdio: "inherit",
+  });
+}
+
+function createArchive(skillPath, skillFilename, archiveExecutor) {
   const stagingDir = fs.mkdtempSync(path.join(os.tmpdir(), "skill-pack-"));
   try {
-    for (const { filePath, arcname } of includedFiles) {
-      const dest = path.join(stagingDir, arcname);
-      fs.mkdirSync(path.dirname(dest), { recursive: true });
-      fs.copyFileSync(filePath, dest);
-      console.log(`  Added: ${arcname}`);
-    }
-
+    stageFiles(collectIncludedFiles(skillPath), stagingDir);
     if (fs.existsSync(skillFilename)) {
       fs.unlinkSync(skillFilename);
     }
-
-    execSync(`zip -r -q "${skillFilename}" "${skillName}"`, {
-      cwd: stagingDir,
-      stdio: "inherit",
-    });
-
+    archiveExecutor(stagingDir, skillFilename, path.basename(skillPath));
     console.log(`\n✅ Successfully packaged skill to: ${skillFilename}`);
     return skillFilename;
   } catch (error) {
@@ -126,6 +119,41 @@ function packageSkill(skillPath, outputDir = null) {
   }
 }
 
+function resolveDependencies(dependencies) {
+  const providedDependencies = dependencies || {};
+  return {
+    archiveExecutor: providedDependencies.executeArchive ?? executeArchive,
+    skillValidator: providedDependencies.validateSkill ?? validateSkill,
+  };
+}
+
+function validateBeforePackaging(skillPath, skillValidator) {
+  console.log("🔍 Validating skill...");
+  const [valid, message] = skillValidator(skillPath);
+  if (!valid) {
+    console.log(`❌ Validation failed: ${message}`);
+    console.log("   Please fix the validation errors before packaging.");
+    return false;
+  }
+  console.log(`✅ ${message}\n`);
+  return true;
+}
+
+function packageSkill(skillPath, outputDir, dependencies) {
+  const resolvedSkillPath = path.resolve(skillPath);
+  if (!validateSkillDirectory(resolvedSkillPath)) {
+    return null;
+  }
+
+  const { archiveExecutor, skillValidator } = resolveDependencies(dependencies);
+  if (!validateBeforePackaging(resolvedSkillPath, skillValidator)) {
+    return null;
+  }
+
+  const skillFilename = prepareOutputPath(resolvedSkillPath, outputDir);
+  return createArchive(resolvedSkillPath, skillFilename, archiveExecutor);
+}
+
 if (require.main === module) {
   if (process.argv.length < 3) {
     console.log("Usage: node package_skill.js <path/to/skill-folder> [output-directory]");

package/dist/bundled-skills/skill-creator/scripts/run_eval.js

@@ -29,41 +29,69 @@ function buildEnv() {
   return env;
 }
 
-function handleStreamEventType(se, seType, state) {
-  if (seType === "content_block_start") {
-    const cb = se.content_block ?? {};
-    if (cb.type === "tool_use") {
-      const toolName = cb.name ?? "";
-      if (toolName === "Skill" || toolName === "Read") {
-        state.pendingToolName = toolName;
-        state.accumulatedJson = "";
-      } else {
-        state.finish(false);
-        return true;
-      }
-    }
-  } else if (seType === "content_block_delta" && state.pendingToolName) {
-    const delta = se.delta ?? {};
-    if (delta.type === "input_json_delta") {
-      state.accumulatedJson += delta.partial_json ?? "";
-      if (state.accumulatedJson.includes(state.cleanName)) {
-        state.finish(true);
-        return true;
-      }
-    }
-  } else if (seType === "content_block_stop" || seType === "message_stop") {
-    if (state.pendingToolName) {
-      state.finish(state.accumulatedJson.includes(state.cleanName));
-      return true;
-    }
-    if (seType === "message_stop") {
-      state.finish(false);
-      return true;
-    }
+function isSupportedTool(toolName) {
+  return toolName === "Skill" || toolName === "Read";
+}
+
+function handleContentBlockStart(se, state) {
+  const contentBlock = se.content_block ?? {};
+  if (contentBlock.type !== "tool_use") return false;
+
+  const toolName = contentBlock.name ?? "";
+  if (!isSupportedTool(toolName)) {
+    state.finish(false);
+    return true;
   }
+
+  state.pendingToolName = toolName;
+  state.accumulatedJson = "";
   return false;
 }
 
+function handleContentBlockDelta(se, state) {
+  if (!state.pendingToolName) return false;
+
+  const delta = se.delta ?? {};
+  if (delta.type !== "input_json_delta") return false;
+
+  state.accumulatedJson += delta.partial_json ?? "";
+  if (!state.accumulatedJson.includes(state.cleanName)) return false;
+
+  state.finish(true);
+  return true;
+}
+
+function finishFromAccumulatedInput(state) {
+  state.finish(state.accumulatedJson.includes(state.cleanName));
+  return true;
+}
+
+function handleContentBlockStop(state) {
+  if (!state.pendingToolName) return false;
+  return finishFromAccumulatedInput(state);
+}
+
+function handleMessageStop(state) {
+  if (state.pendingToolName) return finishFromAccumulatedInput(state);
+  state.finish(false);
+  return true;
+}
+
+function handleStreamEventType(se, seType, state) {
+  switch (seType) {
+    case "content_block_start":
+      return handleContentBlockStart(se, state);
+    case "content_block_delta":
+      return handleContentBlockDelta(se, state);
+    case "content_block_stop":
+      return handleContentBlockStop(state);
+    case "message_stop":
+      return handleMessageStop(state);
+    default:
+      return false;
+  }
+}
+
 function handleAssistantEvent(message, state) {
   for (const contentItem of message.content ?? []) {
     if (contentItem.type !== "tool_use") continue;
@@ -364,4 +392,4 @@ if (require.main === module) {
   });
 }
 
-module.exports = { findProjectRoot, runEval };
+module.exports = { findProjectRoot, handleStreamEventType, runEval };