tarsk 0.5.77 → 0.5.79

package/dist/bundled-skills/skill-creator/eval-viewer/generate_review.js

@@ -247,37 +247,47 @@ function embedFile(filePath) {
   }
 }
 
-function loadPreviousIteration(workspace) {
-  const result = {};
+function readFeedbackMap(workspace) {
   const feedbackMap = {};
   const feedbackPath = path.join(workspace, "feedback.json");
-  if (fs.existsSync(feedbackPath)) {
-    try {
-      const data = JSON.parse(fs.readFileSync(feedbackPath, "utf-8"));
-      for (const review of data.reviews ?? []) {
-        if ((review.feedback ?? "").trim()) {
-          feedbackMap[review.run_id] = review.feedback;
-        }
+  if (!fs.existsSync(feedbackPath)) return feedbackMap;
+
+  try {
+    const data = JSON.parse(fs.readFileSync(feedbackPath, "utf-8"));
+    for (const review of data.reviews ?? []) {
+      if ((review.feedback ?? "").trim()) {
+        feedbackMap[review.run_id] = review.feedback;
       }
-    } catch {
-      // ignore
     }
+  } catch {
+    // Ignore missing or malformed previous feedback.
   }
+  return feedbackMap;
+}
 
-  const prevRuns = findRuns(workspace);
-  for (const run of prevRuns) {
+function mapRunsToPreviousIteration(runs, feedbackMap) {
+  const result = {};
+  for (const run of runs) {
     result[run.id] = {
       feedback: feedbackMap[run.id] ?? "",
       outputs: run.outputs ?? [],
     };
   }
+  return result;
+}
 
+function addOrphanedFeedback(result, feedbackMap) {
   for (const [runId, feedback] of Object.entries(feedbackMap)) {
     if (!result[runId]) {
       result[runId] = { feedback, outputs: [] };
     }
   }
+}
 
+function loadPreviousIteration(workspace) {
+  const feedbackMap = readFeedbackMap(workspace);
+  const result = mapRunsToPreviousIteration(findRuns(workspace), feedbackMap);
+  addOrphanedFeedback(result, feedbackMap);
   return result;
 }
 
@@ -306,32 +316,80 @@ function generateHtml(runs, skillName, previous = null, benchmark = null) {
   return template.replace("/*__EMBEDDED_DATA__*/", `const EMBEDDED_DATA = ${dataJson};`);
 }
 
-function killPort(port) {
-  try {
-    const result = spawnSync("lsof", ["-ti", `:${port}`], {
-      encoding: "utf-8",
-      timeout: 5000,
-    });
-    for (const pidStr of (result.stdout ?? "").trim().split("\n")) {
-      if (!pidStr.trim()) continue;
-      try {
-        process.kill(parseInt(pidStr.trim(), 10), "SIGTERM");
-      } catch {
-        // ignore
-      }
+function parseProcessIds(output) {
+  const processIds = new Set();
+  for (const line of (output ?? "").split("\n")) {
+    const candidate = line.trim();
+    if (!/^\d+$/.test(candidate)) continue;
+
+    const processId = Number.parseInt(candidate, 10);
+    if (Number.isSafeInteger(processId) && processId > 0) {
+      processIds.add(processId);
     }
-    if ((result.stdout ?? "").trim()) {
-      const sleep = spawnSync("sleep", ["0.5"], { stdio: "ignore" });
-      if (sleep.error) {
-        const deadline = Date.now() + 500;
-        while (Date.now() < deadline) {
-          // brief pause after killing port holder
-        }
-      }
+  }
+  return [...processIds];
+}
+
+function discoverPortProcesses(port, runCommand) {
+  const result = runCommand("lsof", ["-ti", `:${port}`], {
+    encoding: "utf-8",
+    timeout: 5000,
+  });
+  if (result.error) throw result.error;
+  return parseProcessIds(result.stdout);
+}
+
+function terminateProcesses(processIds, terminateProcess) {
+  for (const processId of processIds) {
+    try {
+      terminateProcess(processId, "SIGTERM");
+    } catch {
+      // Ignore processes which have already exited or cannot be terminated.
     }
+  }
+}
+
+function busyWait(duration, getTime) {
+  const deadline = getTime() + duration;
+  while (getTime() < deadline) {
+    // Preserve the fallback when the sleep utility is unavailable.
+  }
+}
+
+function pauseAfterTermination(runCommand, getTime) {
+  try {
+    const sleep = runCommand("sleep", ["0.5"], { stdio: "ignore" });
+    if (!sleep.error) return;
+  } catch {
+    // Fall back to a brief busy wait below.
+  }
+  busyWait(500, getTime);
+}
+
+function killProcess(processId, signal) {
+  process.kill(processId, signal);
+}
+
+function writeStandardError(message) {
+  process.stderr.write(message);
+}
+
+function killPort(
+  port,
+  {
+    runCommand = spawnSync,
+    terminateProcess = killProcess,
+    getTime = Date.now,
+    writeError = writeStandardError,
+  } = {},
+) {
+  try {
+    const processIds = discoverPortProcesses(port, runCommand);
+    terminateProcesses(processIds, terminateProcess);
+    if (processIds.length > 0) pauseAfterTermination(runCommand, getTime);
   } catch (error) {
     if (error.message?.includes("ENOENT")) {
-      process.stderr.write("Note: lsof not found, cannot check if port is in use\n");
+      writeError("Note: lsof not found, cannot check if port is in use\n");
     }
   }
 }
@@ -346,70 +404,98 @@ function openUrl(url) {
   }
 }
 
-function createReviewServer({ workspace, skillName, feedbackPath, previous, benchmarkPath }) {
-  return http.createServer((req, res) => {
-    const url = new URL(req.url ?? "/", "http://127.0.0.1");
-
-    if (req.method === "GET" && (url.pathname === "/" || url.pathname === "/index.html")) {
-      const runs = findRuns(workspace);
-      let benchmark = null;
-      if (benchmarkPath && fs.existsSync(benchmarkPath)) {
-        try {
-          benchmark = JSON.parse(fs.readFileSync(benchmarkPath, "utf-8"));
-        } catch {
-          // ignore
-        }
-      }
-      const html = generateHtml(runs, skillName, previous, benchmark);
-      const content = Buffer.from(html, "utf-8");
-      res.writeHead(200, {
-        "Content-Type": "text/html; charset=utf-8",
-        "Content-Length": content.length,
-      });
-      res.end(content);
-      return;
-    }
+function loadBenchmark(benchmarkPath) {
+  if (!benchmarkPath || !fs.existsSync(benchmarkPath)) return null;
 
-    if (req.method === "GET" && url.pathname === "/api/feedback") {
-      const data = fs.existsSync(feedbackPath) ? fs.readFileSync(feedbackPath) : Buffer.from("{}");
-      res.writeHead(200, {
-        "Content-Type": "application/json",
-        "Content-Length": data.length,
-      });
-      res.end(data);
-      return;
-    }
+  try {
+    return JSON.parse(fs.readFileSync(benchmarkPath, "utf-8"));
+  } catch {
+    return null;
+  }
+}
 
-    if (req.method === "POST" && url.pathname === "/api/feedback") {
-      const chunks = [];
-      req.on("data", (chunk) => chunks.push(chunk));
-      req.on("end", () => {
-        let resp;
-        let status = 200;
-        try {
-          const body = Buffer.concat(chunks).toString("utf-8");
-          const data = JSON.parse(body);
-          if (!data || typeof data !== "object" || !("reviews" in data)) {
-            throw new Error("Expected JSON object with 'reviews' key");
-          }
-          fs.writeFileSync(feedbackPath, `${JSON.stringify(data, null, 2)}\n`);
-          resp = Buffer.from('{"ok":true}');
-        } catch (error) {
-          resp = Buffer.from(JSON.stringify({ error: error.message }));
-          status = 500;
-        }
-        res.writeHead(status, {
-          "Content-Type": "application/json",
-          "Content-Length": resp.length,
-        });
-        res.end(resp);
-      });
-      return;
+function sendResponse(res, status, contentType, content) {
+  res.writeHead(status, {
+    "Content-Type": contentType,
+    "Content-Length": content.length,
+  });
+  res.end(content);
+}
+
+function renderReviewPage(res, { workspace, skillName, previous, benchmarkPath }) {
+  const runs = findRuns(workspace);
+  const benchmark = loadBenchmark(benchmarkPath);
+  const html = generateHtml(runs, skillName, previous, benchmark);
+  sendResponse(res, 200, "text/html; charset=utf-8", Buffer.from(html, "utf-8"));
+}
+
+function sendFeedback(res, feedbackPath) {
+  const data = fs.existsSync(feedbackPath) ? fs.readFileSync(feedbackPath) : Buffer.from("{}");
+  sendResponse(res, 200, "application/json", data);
+}
+
+function persistFeedback(res, feedbackPath, chunks) {
+  let content;
+  let status = 200;
+  try {
+    const body = Buffer.concat(chunks).toString("utf-8");
+    const data = JSON.parse(body);
+    if (!data || typeof data !== "object" || !("reviews" in data)) {
+      throw new Error("Expected JSON object with 'reviews' key");
     }
+    fs.writeFileSync(feedbackPath, `${JSON.stringify(data, null, 2)}\n`);
+    content = Buffer.from('{"ok":true}');
+  } catch (error) {
+    content = Buffer.from(JSON.stringify({ error: error.message }));
+    status = 500;
+  }
+  sendResponse(res, status, "application/json", content);
+}
 
-    res.writeHead(404);
-    res.end();
-  });
+function receiveFeedback(req, res, feedbackPath) {
+  const chunks = [];
+
+  function collectChunk(chunk) {
+    chunks.push(chunk);
+  }
+
+  function finishFeedback() {
+    persistFeedback(res, feedbackPath, chunks);
+  }
+
+  req.on("data", collectChunk);
+  req.on("end", finishFeedback);
+}
+
+function routeReviewRequest(req, res, options) {
+  const url = new URL(req.url ?? "/", "http://127.0.0.1");
+  const isFeedbackRequest = url.pathname === "/api/feedback";
+
+  if (req.method === "GET" && (url.pathname === "/" || url.pathname === "/index.html")) {
+    renderReviewPage(res, options);
+    return;
+  }
+  if (req.method === "GET" && isFeedbackRequest) {
+    sendFeedback(res, options.feedbackPath);
+    return;
+  }
+  if (req.method === "POST" && isFeedbackRequest) {
+    receiveFeedback(req, res, options.feedbackPath);
+    return;
+  }
+
+  res.writeHead(404);
+  res.end();
+}
+
+function createReviewServer({ workspace, skillName, feedbackPath, previous, benchmarkPath }) {
+  const options = { workspace, skillName, feedbackPath, previous, benchmarkPath };
+
+  function handleReviewRequest(req, res) {
+    routeReviewRequest(req, res, options);
+  }
+
+  return http.createServer(handleReviewRequest);
 }
 
 function parseArgs(argv) {
@@ -540,6 +626,8 @@ function main() {
     });
 }
 
+module.exports = { createReviewServer, killPort, loadPreviousIteration };
+
 if (require.main === module) {
   main();
 }

package/dist/bundled-skills/skill-creator/scripts/__tests__/improve_description.test.js

@@ -0,0 +1,77 @@
+const assert = require("node:assert/strict");
+const test = require("node:test");
+
+const { buildHistoryText } = require("../improve_description.js");
+
+const header =
+  "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n";
+const longQuery = "q".repeat(81);
+
+const cases = [
+  {
+    name: "returns an empty string for empty history",
+    history: [],
+    expected: "",
+  },
+  {
+    name: "renders legacy score fields",
+    history: [{ description: "Legacy description", passed: 2, total: 3 }],
+    expected: header + '<attempt train=2/3>\nDescription: "Legacy description"\n</attempt>\n\n',
+  },
+  {
+    name: "renders current train and test score fields",
+    history: [
+      {
+        description: "Current description",
+        train_passed: 4,
+        train_total: 5,
+        test_passed: 0,
+        test_total: 2,
+      },
+    ],
+    expected:
+      header + '<attempt train=4/5, test=0/2>\nDescription: "Current description"\n</attempt>\n\n',
+  },
+  {
+    name: "renders pass and fail results with truncated queries",
+    history: [
+      {
+        description: "Results description",
+        train_passed: 1,
+        train_total: 2,
+        results: [
+          { pass: true, query: "short query", triggers: 2, runs: 2 },
+          { pass: false, query: longQuery, triggers: 0, runs: 3 },
+        ],
+      },
+    ],
+    expected:
+      header +
+      '<attempt train=1/2>\nDescription: "Results description"\nTrain results:\n' +
+      '  [PASS] "short query" (triggered 2/2)\n' +
+      `  [FAIL] "${"q".repeat(80)}" (triggered 0/3)\n` +
+      "</attempt>\n\n",
+  },
+  {
+    name: "renders notes and unknown test totals",
+    history: [
+      {
+        description: "Noted description",
+        train_passed: 3,
+        train_total: 3,
+        test_passed: 1,
+        note: "Try narrower wording.",
+      },
+    ],
+    expected:
+      header +
+      '<attempt train=3/3, test=1/?>\nDescription: "Noted description"\n' +
+      "Note: Try narrower wording.\n</attempt>\n\n",
+  },
+];
+
+for (const testCase of cases) {
+  test(testCase.name, () => {
+    assert.equal(buildHistoryText(testCase.history), testCase.expected);
+  });
+}

package/dist/bundled-skills/skill-creator/scripts/__tests__/package_skill.test.js

@@ -0,0 +1,203 @@
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+const test = require("node:test");
+
+const { packageSkill } = require("../package_skill.js");
+
+function withTemporaryDirectory(callback) {
+  const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "tarsk-package-skill-"));
+  try {
+    return callback(temporaryDirectory);
+  } finally {
+    fs.rmSync(temporaryDirectory, { recursive: true, force: true });
+  }
+}
+
+function createSkill(rootDirectory, name = "example-skill") {
+  const skillPath = path.join(rootDirectory, name);
+  fs.mkdirSync(skillPath, { recursive: true });
+  fs.writeFileSync(
+    path.join(skillPath, "SKILL.md"),
+    "---\nname: example-skill\ndescription: Test\n---\n",
+  );
+  return skillPath;
+}
+
+function successfulValidation() {
+  return [true, "Skill is valid!"];
+}
+
+function unexpectedArchiveExecution() {
+  assert.fail("archive execution should not be reached");
+}
+
+function dependencies(overrides = {}) {
+  return {
+    validateSkill: successfulValidation,
+    executeArchive: unexpectedArchiveExecution,
+    ...overrides,
+  };
+}
+
+test("rejects a missing skill path before validation", function () {
+  withTemporaryDirectory(function (temporaryDirectory) {
+    const missingPath = path.join(temporaryDirectory, "missing");
+    assert.equal(packageSkill(missingPath, null, dependencies()), null);
+  });
+});
+
+test("rejects a skill path that is not a directory", function () {
+  withTemporaryDirectory(function (temporaryDirectory) {
+    const filePath = path.join(temporaryDirectory, "skill.txt");
+    fs.writeFileSync(filePath, "not a directory");
+    assert.equal(packageSkill(filePath, null, dependencies()), null);
+  });
+});
+
+test("rejects a directory without SKILL.md", function () {
+  withTemporaryDirectory(function (temporaryDirectory) {
+    assert.equal(packageSkill(temporaryDirectory, null, dependencies()), null);
+  });
+});
+
+test("returns null when skill validation fails", function () {
+  withTemporaryDirectory(function (temporaryDirectory) {
+    const skillPath = createSkill(temporaryDirectory);
+    const result = packageSkill(
+      skillPath,
+      null,
+      dependencies({
+        validateSkill: function validateInvalidSkill() {
+          return [false, "Invalid frontmatter"];
+        },
+      }),
+    );
+    assert.equal(result, null);
+  });
+});
+
+test("creates an archive in the current directory by default", function () {
+  withTemporaryDirectory(function (temporaryDirectory) {
+    const skillPath = createSkill(temporaryDirectory);
+    const archivePath = path.join(temporaryDirectory, "example-skill.skill");
+    const previousWorkingDirectory = process.cwd();
+    fs.writeFileSync(archivePath, "old archive");
+
+    try {
+      process.chdir(temporaryDirectory);
+      const resolvedArchivePath = path.join(process.cwd(), "example-skill.skill");
+      const result = packageSkill(
+        skillPath,
+        null,
+        dependencies({
+          executeArchive: function createTestArchive(stagingDir, filename, skillName) {
+            assert.equal(filename, resolvedArchivePath);
+            assert.equal(skillName, "example-skill");
+            assert.equal(fs.existsSync(filename), false);
+            assert.equal(fs.existsSync(path.join(stagingDir, skillName, "SKILL.md")), true);
+            fs.writeFileSync(filename, "new archive");
+          },
+        }),
+      );
+
+      assert.equal(result, resolvedArchivePath);
+      assert.equal(fs.readFileSync(archivePath, "utf8"), "new archive");
+    } finally {
+      process.chdir(previousWorkingDirectory);
+    }
+  });
+});
+
+test("creates a missing custom output directory", function () {
+  withTemporaryDirectory(function (temporaryDirectory) {
+    const skillPath = createSkill(temporaryDirectory);
+    const outputDirectory = path.join(temporaryDirectory, "nested", "output");
+    const expectedArchive = path.join(outputDirectory, "example-skill.skill");
+
+    const result = packageSkill(
+      skillPath,
+      outputDirectory,
+      dependencies({
+        executeArchive: function createTestArchive(_stagingDir, filename) {
+          fs.writeFileSync(filename, "archive");
+        },
+      }),
+    );
+
+    assert.equal(result, expectedArchive);
+    assert.equal(fs.existsSync(expectedArchive), true);
+  });
+});
+
+test("excludes generated, dependency, metadata, and root eval files", function () {
+  withTemporaryDirectory(function (temporaryDirectory) {
+    const skillPath = createSkill(temporaryDirectory);
+    const files = {
+      "keep.txt": "keep",
+      "compiled.pyc": "exclude",
+      ".DS_Store": "exclude",
+      "node_modules/package/index.js": "exclude",
+      "__pycache__/module.pyc": "exclude",
+      "evals/result.json": "exclude",
+      "nested/evals/keep.json": "keep",
+    };
+    for (const [relativePath, content] of Object.entries(files)) {
+      const filename = path.join(skillPath, relativePath);
+      fs.mkdirSync(path.dirname(filename), { recursive: true });
+      fs.writeFileSync(filename, content);
+    }
+
+    let stagedFiles = [];
+    const result = packageSkill(
+      skillPath,
+      temporaryDirectory,
+      dependencies({
+        executeArchive: function inspectStagedFiles(stagingDir, filename) {
+          stagedFiles = fs
+            .readdirSync(path.join(stagingDir, "example-skill"), {
+              recursive: true,
+              withFileTypes: true,
+            })
+            .filter(function selectFiles(entry) {
+              return entry.isFile();
+            })
+            .map(function relativeFilename(entry) {
+              return path.relative(stagingDir, path.join(entry.parentPath, entry.name));
+            })
+            .sort();
+          fs.writeFileSync(filename, "archive");
+        },
+      }),
+    );
+
+    assert.equal(result, path.join(temporaryDirectory, "example-skill.skill"));
+    assert.deepEqual(stagedFiles, [
+      "example-skill/SKILL.md",
+      "example-skill/keep.txt",
+      "example-skill/nested/evals/keep.json",
+    ]);
+  });
+});
+
+test("returns null and removes staging files when archive execution fails", function () {
+  withTemporaryDirectory(function (temporaryDirectory) {
+    const skillPath = createSkill(temporaryDirectory);
+    let stagingDirectory;
+
+    const result = packageSkill(
+      skillPath,
+      temporaryDirectory,
+      dependencies({
+        executeArchive: function failArchive(stagingDir) {
+          stagingDirectory = stagingDir;
+          throw new Error("zip failed");
+        },
+      }),
+    );
+
+    assert.equal(result, null);
+    assert.equal(fs.existsSync(stagingDirectory), false);
+  });
+});