tarsk 0.5.42 → 0.5.44

package/dist/bundled-skills/skill-creator/scripts/package_skill.js

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+/** Skill Packager - Creates a distributable .skill file of a skill folder. */
+
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+const { execSync } = require("child_process");
+const { validateSkill } = require("./quick_validate.js");
+
+const EXCLUDE_DIRS = new Set(["__pycache__", "node_modules"]);
+const EXCLUDE_GLOBS = ["*.pyc"];
+const EXCLUDE_FILES = new Set([".DS_Store"]);
+const ROOT_EXCLUDE_DIRS = new Set(["evals"]);
+
+function matchesGlob(name, pattern) {
+  const regex = new RegExp(
+    "^" + pattern.replace(/\./g, "\\.").replace(/\*/g, ".*").replace(/\?/g, ".") + "$",
+  );
+  return regex.test(name);
+}
+
+function shouldExclude(relPath) {
+  const parts = relPath.split(path.sep);
+  if (parts.some((part) => EXCLUDE_DIRS.has(part))) {
+    return true;
+  }
+  if (parts.length > 1 && ROOT_EXCLUDE_DIRS.has(parts[1])) {
+    return true;
+  }
+  const name = parts[parts.length - 1];
+  if (EXCLUDE_FILES.has(name)) {
+    return true;
+  }
+  return EXCLUDE_GLOBS.some((pattern) => matchesGlob(name, pattern));
+}
+
+function walkFiles(dir) {
+  const files = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...walkFiles(fullPath));
+    } else if (entry.isFile()) {
+      files.push(fullPath);
+    }
+  }
+  return files;
+}
+
+function packageSkill(skillPath, outputDir = null) {
+  skillPath = path.resolve(skillPath);
+
+  if (!fs.existsSync(skillPath)) {
+    console.log(`❌ Error: Skill folder not found: ${skillPath}`);
+    return null;
+  }
+
+  if (!fs.statSync(skillPath).isDirectory()) {
+    console.log(`❌ Error: Path is not a directory: ${skillPath}`);
+    return null;
+  }
+
+  const skillMd = path.join(skillPath, "SKILL.md");
+  if (!fs.existsSync(skillMd)) {
+    console.log(`❌ Error: SKILL.md not found in ${skillPath}`);
+    return null;
+  }
+
+  console.log("🔍 Validating skill...");
+  const [valid, message] = validateSkill(skillPath);
+  if (!valid) {
+    console.log(`❌ Validation failed: ${message}`);
+    console.log("   Please fix the validation errors before packaging.");
+    return null;
+  }
+  console.log(`✅ ${message}\n`);
+
+  const skillName = path.basename(skillPath);
+  const outputPath = outputDir ? path.resolve(outputDir) : process.cwd();
+  if (outputDir) {
+    fs.mkdirSync(outputPath, { recursive: true });
+  }
+  const skillFilename = path.join(outputPath, `${skillName}.skill`);
+
+  const parentDir = path.dirname(skillPath);
+  const includedFiles = [];
+  for (const filePath of walkFiles(skillPath)) {
+    const arcname = path.relative(parentDir, filePath);
+    if (shouldExclude(arcname)) {
+      console.log(`  Skipped: ${arcname}`);
+      continue;
+    }
+    includedFiles.push({ filePath, arcname });
+  }
+
+  if (includedFiles.length === 0) {
+    console.log("❌ Error: No files to package");
+    return null;
+  }
+
+  const stagingDir = fs.mkdtempSync(path.join(os.tmpdir(), "skill-pack-"));
+  try {
+    for (const { filePath, arcname } of includedFiles) {
+      const dest = path.join(stagingDir, arcname);
+      fs.mkdirSync(path.dirname(dest), { recursive: true });
+      fs.copyFileSync(filePath, dest);
+      console.log(`  Added: ${arcname}`);
+    }
+
+    if (fs.existsSync(skillFilename)) {
+      fs.unlinkSync(skillFilename);
+    }
+
+    execSync(`zip -r -q "${skillFilename}" "${skillName}"`, {
+      cwd: stagingDir,
+      stdio: "inherit",
+    });
+
+    console.log(`\n✅ Successfully packaged skill to: ${skillFilename}`);
+    return skillFilename;
+  } catch (error) {
+    console.log(`❌ Error creating .skill file: ${error.message}`);
+    return null;
+  } finally {
+    fs.rmSync(stagingDir, { recursive: true, force: true });
+  }
+}
+
+if (require.main === module) {
+  if (process.argv.length < 3) {
+    console.log("Usage: node package_skill.js <path/to/skill-folder> [output-directory]");
+    process.exit(1);
+  }
+
+  const skillPath = process.argv[2];
+  const outputDir = process.argv[3] ?? null;
+
+  console.log(`📦 Packaging skill: ${skillPath}`);
+  if (outputDir) {
+    console.log(`   Output directory: ${outputDir}`);
+  }
+  console.log();
+
+  process.exit(packageSkill(skillPath, outputDir) ? 0 : 1);
+}
+
+module.exports = { packageSkill };

package/dist/bundled-skills/skill-creator/scripts/quick_validate.js

@@ -0,0 +1,132 @@
+#!/usr/bin/env node
+/** Quick validation script for skills. */
+
+const fs = require("fs");
+const path = require("path");
+
+const ALLOWED_PROPERTIES = new Set([
+  "name",
+  "description",
+  "license",
+  "allowed-tools",
+  "metadata",
+  "compatibility",
+]);
+
+function parseSimpleFrontmatter(text) {
+  const frontmatter = {};
+  for (const line of text.split("\n")) {
+    const colonIndex = line.indexOf(":");
+    if (colonIndex === -1) continue;
+    const key = line.slice(0, colonIndex).trim();
+    let value = line.slice(colonIndex + 1).trim();
+    if (
+      (value.startsWith('"') && value.endsWith('"')) ||
+      (value.startsWith("'") && value.endsWith("'"))
+    ) {
+      value = value.slice(1, -1);
+    }
+    if (value === ">" || value === "|") {
+      continue;
+    }
+    frontmatter[key] = value;
+  }
+  return frontmatter;
+}
+
+function validateSkill(skillPath) {
+  skillPath = path.resolve(skillPath);
+  const skillMd = path.join(skillPath, "SKILL.md");
+
+  if (!fs.existsSync(skillMd)) {
+    return [false, "SKILL.md not found"];
+  }
+
+  const content = fs.readFileSync(skillMd, "utf-8");
+  if (!content.startsWith("---")) {
+    return [false, "No YAML frontmatter found"];
+  }
+
+  const match = content.match(/^---\n([\s\S]*?)\n---/);
+  if (!match) {
+    return [false, "Invalid frontmatter format"];
+  }
+
+  const frontmatter = parseSimpleFrontmatter(match[1]);
+  if (!frontmatter || typeof frontmatter !== "object") {
+    return [false, "Frontmatter must be a YAML dictionary"];
+  }
+
+  const unexpectedKeys = Object.keys(frontmatter).filter((key) => !ALLOWED_PROPERTIES.has(key));
+  if (unexpectedKeys.length > 0) {
+    return [
+      false,
+      `Unexpected key(s) in SKILL.md frontmatter: ${unexpectedKeys.sort().join(", ")}. ` +
+        `Allowed properties are: ${[...ALLOWED_PROPERTIES].sort().join(", ")}`,
+    ];
+  }
+
+  if (!("name" in frontmatter)) {
+    return [false, "Missing 'name' in frontmatter"];
+  }
+  if (!("description" in frontmatter)) {
+    return [false, "Missing 'description' in frontmatter"];
+  }
+
+  const name = String(frontmatter.name ?? "").trim();
+  if (name) {
+    if (!/^[a-z0-9-]+$/.test(name)) {
+      return [
+        false,
+        `Name '${name}' should be kebab-case (lowercase letters, digits, and hyphens only)`,
+      ];
+    }
+    if (name.startsWith("-") || name.endsWith("-") || name.includes("--")) {
+      return [false, `Name '${name}' cannot start/end with hyphen or contain consecutive hyphens`];
+    }
+    if (name.length > 64) {
+      return [false, `Name is too long (${name.length} characters). Maximum is 64 characters.`];
+    }
+  }
+
+  const description = String(frontmatter.description ?? "").trim();
+  if (description) {
+    if (description.includes("<") || description.includes(">")) {
+      return [false, "Description cannot contain angle brackets (< or >)"];
+    }
+    if (description.length > 1024) {
+      return [
+        false,
+        `Description is too long (${description.length} characters). Maximum is 1024 characters.`,
+      ];
+    }
+  }
+
+  const compatibility = frontmatter.compatibility;
+  if (compatibility) {
+    if (typeof compatibility !== "string") {
+      return [false, `Compatibility must be a string, got ${typeof compatibility}`];
+    }
+    if (compatibility.length > 500) {
+      return [
+        false,
+        `Compatibility is too long (${compatibility.length} characters). Maximum is 500 characters.`,
+      ];
+    }
+  }
+
+  return [true, "Skill is valid!"];
+}
+
+if (require.main === module) {
+  if (process.argv.length !== 3) {
+    console.log("Usage: node quick_validate.js <skill_directory>");
+    process.exit(1);
+  }
+
+  const [valid, message] = validateSkill(process.argv[2]);
+  console.log(message);
+  process.exit(valid ? 0 : 1);
+}
+
+module.exports = { validateSkill };

package/dist/bundled-skills/skill-creator/scripts/run_eval.js

@@ -0,0 +1,345 @@
+#!/usr/bin/env node
+/** Run trigger evaluation for a skill description. */
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+const { spawn } = require("child_process");
+const { parseSkillMd } = require("./utils.js");
+
+function findProjectRoot() {
+  let current = process.cwd();
+  while (true) {
+    if (
+      fs.existsSync(path.join(current, ".Tarsk")) &&
+      fs.statSync(path.join(current, ".Tarsk")).isDirectory()
+    ) {
+      return current;
+    }
+    const parent = path.dirname(current);
+    if (parent === current) break;
+    current = parent;
+  }
+  return process.cwd();
+}
+
+function buildEnv() {
+  const env = { ...process.env };
+  delete env.TarskCODE;
+  return env;
+}
+
+function runSingleQuery(query, skillName, skillDescription, timeoutSec, projectRoot, model = null) {
+  return new Promise((resolve) => {
+    const uniqueId = crypto.randomBytes(4).toString("hex").slice(0, 8);
+    const cleanName = `${skillName}-skill-${uniqueId}`;
+    const projectCommandsDir = path.join(projectRoot, ".Tarsk", "commands");
+    const commandFile = path.join(projectCommandsDir, `${cleanName}.md`);
+
+    function cleanup() {
+      if (fs.existsSync(commandFile)) {
+        fs.unlinkSync(commandFile);
+      }
+    }
+
+    try {
+      fs.mkdirSync(projectCommandsDir, { recursive: true });
+      const indentedDesc = skillDescription.split("\n").join("\n  ");
+      const commandContent =
+        "---\n" +
+        "description: |\n" +
+        `  ${indentedDesc}\n` +
+        "---\n\n" +
+        `# ${skillName}\n\n` +
+        `This skill handles: ${skillDescription}\n`;
+      fs.writeFileSync(commandFile, commandContent);
+
+      const cmd = [
+        "tarsk",
+        "-p",
+        query,
+        "--output-format",
+        "stream-json",
+        "--verbose",
+        "--include-partial-messages",
+      ];
+      if (model) {
+        cmd.push("--model", model);
+      }
+
+      const proc = spawn(cmd[0], cmd.slice(1), {
+        cwd: projectRoot,
+        env: buildEnv(),
+        stdio: ["ignore", "pipe", "ignore"],
+      });
+
+      let buffer = "";
+      let triggered = false;
+      let pendingToolName = null;
+      let accumulatedJson = "";
+      let finished = false;
+
+      function finish(value) {
+        if (finished) return;
+        finished = true;
+        clearTimeout(timeoutId);
+        if (proc.exitCode === null) {
+          proc.kill();
+        }
+        cleanup();
+        resolve(value);
+      }
+
+      const timeoutId = setTimeout(() => finish(triggered), timeoutSec * 1000);
+
+      proc.stdout.on("data", (chunk) => {
+        buffer += chunk.toString("utf-8");
+        while (true) {
+          const newlineIdx = buffer.indexOf("\n");
+          if (newlineIdx === -1) break;
+          const line = buffer.slice(0, newlineIdx).trim();
+          buffer = buffer.slice(newlineIdx + 1);
+          if (!line) continue;
+
+          let event;
+          try {
+            event = JSON.parse(line);
+          } catch {
+            continue;
+          }
+
+          if (event.type === "stream_event") {
+            const se = event.event ?? {};
+            const seType = se.type ?? "";
+
+            if (seType === "content_block_start") {
+              const cb = se.content_block ?? {};
+              if (cb.type === "tool_use") {
+                const toolName = cb.name ?? "";
+                if (toolName === "Skill" || toolName === "Read") {
+                  pendingToolName = toolName;
+                  accumulatedJson = "";
+                } else {
+                  finish(false);
+                  return;
+                }
+              }
+            } else if (seType === "content_block_delta" && pendingToolName) {
+              const delta = se.delta ?? {};
+              if (delta.type === "input_json_delta") {
+                accumulatedJson += delta.partial_json ?? "";
+                if (accumulatedJson.includes(cleanName)) {
+                  finish(true);
+                  return;
+                }
+              }
+            } else if (seType === "content_block_stop" || seType === "message_stop") {
+              if (pendingToolName) {
+                finish(accumulatedJson.includes(cleanName));
+                return;
+              }
+              if (seType === "message_stop") {
+                finish(false);
+                return;
+              }
+            }
+          } else if (event.type === "assistant") {
+            const message = event.message ?? {};
+            for (const contentItem of message.content ?? []) {
+              if (contentItem.type !== "tool_use") continue;
+              const toolName = contentItem.name ?? "";
+              const toolInput = contentItem.input ?? {};
+              if (toolName === "Skill" && (toolInput.skill ?? "").includes(cleanName)) {
+                triggered = true;
+              } else if (toolName === "Read" && (toolInput.file_path ?? "").includes(cleanName)) {
+                triggered = true;
+              }
+              finish(triggered);
+              return;
+            }
+          } else if (event.type === "result") {
+            finish(triggered);
+            return;
+          }
+        }
+      });
+
+      proc.on("close", () => finish(triggered));
+      proc.on("error", () => finish(false));
+    } catch {
+      cleanup();
+      resolve(false);
+    }
+  });
+}
+
+async function runEval(
+  evalSet,
+  skillName,
+  description,
+  numWorkers,
+  timeout,
+  projectRoot,
+  runsPerQuery = 1,
+  triggerThreshold = 0.5,
+  model = null,
+) {
+  const tasks = [];
+  for (const item of evalSet) {
+    for (let runIdx = 0; runIdx < runsPerQuery; runIdx++) {
+      tasks.push({ item, runIdx });
+    }
+  }
+
+  const queryTriggers = {};
+  const queryItems = {};
+  let nextIndex = 0;
+
+  async function worker() {
+    while (nextIndex < tasks.length) {
+      const taskIndex = nextIndex++;
+      const { item } = tasks[taskIndex];
+      const query = item.query;
+      queryItems[query] = item;
+      if (!queryTriggers[query]) {
+        queryTriggers[query] = [];
+      }
+      try {
+        const triggered = await runSingleQuery(
+          query,
+          skillName,
+          description,
+          timeout,
+          projectRoot,
+          model,
+        );
+        queryTriggers[query].push(triggered);
+      } catch (error) {
+        console.error(`Warning: query failed: ${error.message}`);
+        queryTriggers[query].push(false);
+      }
+    }
+  }
+
+  const workers = Math.max(1, numWorkers);
+  await Promise.all(Array.from({ length: workers }, () => worker()));
+
+  const results = [];
+  for (const [query, triggers] of Object.entries(queryTriggers)) {
+    const item = queryItems[query];
+    const triggerRate = triggers.reduce((sum, t) => sum + (t ? 1 : 0), 0) / triggers.length;
+    const shouldTrigger = item.should_trigger;
+    const didPass = shouldTrigger
+      ? triggerRate >= triggerThreshold
+      : triggerRate < triggerThreshold;
+    results.push({
+      query,
+      should_trigger: shouldTrigger,
+      trigger_rate: triggerRate,
+      triggers: triggers.filter(Boolean).length,
+      runs: triggers.length,
+      pass: didPass,
+    });
+  }
+
+  const passed = results.filter((r) => r.pass).length;
+  const total = results.length;
+
+  return {
+    skill_name: skillName,
+    description,
+    results,
+    summary: {
+      total,
+      passed,
+      failed: total - passed,
+    },
+  };
+}
+
+function parseArgs(argv) {
+  const args = {
+    evalSet: null,
+    skillPath: null,
+    description: null,
+    numWorkers: 10,
+    timeout: 30,
+    runsPerQuery: 3,
+    triggerThreshold: 0.5,
+    model: null,
+    verbose: false,
+  };
+  for (let i = 2; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--eval-set") args.evalSet = argv[++i];
+    else if (arg === "--skill-path") args.skillPath = argv[++i];
+    else if (arg === "--description") args.description = argv[++i];
+    else if (arg === "--num-workers") args.numWorkers = parseInt(argv[++i], 10);
+    else if (arg === "--timeout") args.timeout = parseInt(argv[++i], 10);
+    else if (arg === "--runs-per-query") args.runsPerQuery = parseInt(argv[++i], 10);
+    else if (arg === "--trigger-threshold") args.triggerThreshold = parseFloat(argv[++i]);
+    else if (arg === "--model") args.model = argv[++i];
+    else if (arg === "--verbose") args.verbose = true;
+  }
+  return args;
+}
+
+async function main() {
+  const args = parseArgs(process.argv);
+  if (!args.evalSet || !args.skillPath) {
+    console.error(
+      "Usage: node run_eval.js --eval-set PATH --skill-path PATH [--description TEXT] [--num-workers N] [--timeout SEC] [--runs-per-query N] [--trigger-threshold R] [--model MODEL] [--verbose]",
+    );
+    process.exit(1);
+  }
+
+  const evalSet = JSON.parse(fs.readFileSync(args.evalSet, "utf-8"));
+  const skillPath = path.resolve(args.skillPath);
+  if (!fs.existsSync(path.join(skillPath, "SKILL.md"))) {
+    console.error(`Error: No SKILL.md found at ${skillPath}`);
+    process.exit(1);
+  }
+
+  const { name, description: originalDescription } = parseSkillMd(skillPath);
+  const description = args.description ?? originalDescription;
+  const projectRoot = findProjectRoot();
+
+  if (args.verbose) {
+    console.error(`Evaluating: ${description}`);
+  }
+
+  const output = await runEval(
+    evalSet,
+    name,
+    description,
+    args.numWorkers,
+    args.timeout,
+    projectRoot,
+    args.runsPerQuery,
+    args.triggerThreshold,
+    args.model,
+  );
+
+  if (args.verbose) {
+    const summary = output.summary;
+    console.error(`Results: ${summary.passed}/${summary.total} passed`);
+    for (const r of output.results) {
+      const status = r.pass ? "PASS" : "FAIL";
+      const rateStr = `${r.triggers}/${r.runs}`;
+      console.error(
+        `  [${status}] rate=${rateStr} expected=${r.should_trigger}: ${r.query.slice(0, 70)}`,
+      );
+    }
+  }
+
+  console.log(JSON.stringify(output, null, 2));
+}
+
+if (require.main === module) {
+  main().catch((error) => {
+    console.error(error.message);
+    process.exit(1);
+  });
+}
+
+module.exports = { findProjectRoot, runEval, runSingleQuery };