tarsk 0.5.41 → 0.5.43

package/dist/bundled-skills/skill-creator/eval-viewer/generate_review.js

@@ -0,0 +1,544 @@
+#!/usr/bin/env node
+/** Generate and serve a review page for eval results. */
+
+const fs = require("fs");
+const path = require("path");
+const http = require("http");
+const { spawnSync } = require("child_process");
+
+const METADATA_FILES = new Set(["transcript.md", "user_notes.md", "metrics.json"]);
+const TEXT_EXTENSIONS = new Set([
+  ".txt",
+  ".md",
+  ".json",
+  ".csv",
+  ".py",
+  ".js",
+  ".ts",
+  ".tsx",
+  ".jsx",
+  ".yaml",
+  ".yml",
+  ".xml",
+  ".html",
+  ".css",
+  ".sh",
+  ".rb",
+  ".go",
+  ".rs",
+  ".java",
+  ".c",
+  ".cpp",
+  ".h",
+  ".hpp",
+  ".sql",
+  ".r",
+  ".toml",
+]);
+const IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"]);
+const MIME_OVERRIDES = {
+  ".svg": "image/svg+xml",
+  ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+};
+
+function getMimeType(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  if (MIME_OVERRIDES[ext]) return MIME_OVERRIDES[ext];
+  const mimeTypes = {
+    ".txt": "text/plain",
+    ".md": "text/markdown",
+    ".json": "application/json",
+    ".html": "text/html",
+    ".css": "text/css",
+    ".js": "text/javascript",
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".pdf": "application/pdf",
+  };
+  return mimeTypes[ext] ?? "application/octet-stream";
+}
+
+function findRuns(workspace) {
+  const runs = [];
+  findRunsRecursive(workspace, workspace, runs);
+  runs.sort((a, b) => {
+    const aEval = a.eval_id ?? Number.POSITIVE_INFINITY;
+    const bEval = b.eval_id ?? Number.POSITIVE_INFINITY;
+    if (aEval !== bEval) return aEval - bEval;
+    return a.id.localeCompare(b.id);
+  });
+  return runs;
+}
+
+function findRunsRecursive(root, current, runs) {
+  if (!fs.existsSync(current) || !fs.statSync(current).isDirectory()) return;
+
+  const outputsDir = path.join(current, "outputs");
+  if (fs.existsSync(outputsDir) && fs.statSync(outputsDir).isDirectory()) {
+    const run = buildRun(root, current);
+    if (run) runs.push(run);
+    return;
+  }
+
+  const skip = new Set(["node_modules", ".git", "__pycache__", "skill", "inputs"]);
+  for (const childName of fs.readdirSync(current).sort()) {
+    if (skip.has(childName)) continue;
+    const child = path.join(current, childName);
+    if (fs.statSync(child).isDirectory()) {
+      findRunsRecursive(root, child, runs);
+    }
+  }
+}
+
+function buildRun(root, runDir) {
+  let prompt = "";
+  let evalId = null;
+
+  for (const candidate of [
+    path.join(runDir, "eval_metadata.json"),
+    path.join(runDir, "..", "eval_metadata.json"),
+  ]) {
+    if (!fs.existsSync(candidate)) continue;
+    try {
+      const metadata = JSON.parse(fs.readFileSync(candidate, "utf-8"));
+      prompt = metadata.prompt ?? "";
+      evalId = metadata.eval_id ?? null;
+    } catch {
+      // ignore
+    }
+    if (prompt) break;
+  }
+
+  if (!prompt) {
+    for (const candidate of [
+      path.join(runDir, "transcript.md"),
+      path.join(runDir, "outputs", "transcript.md"),
+    ]) {
+      if (!fs.existsSync(candidate)) continue;
+      try {
+        const text = fs.readFileSync(candidate, "utf-8");
+        const match = text.match(/## Eval Prompt\n\n([\s\S]*?)(?=\n##|$)/);
+        if (match) prompt = match[1].trim();
+      } catch {
+        // ignore
+      }
+      if (prompt) break;
+    }
+  }
+
+  if (!prompt) prompt = "(No prompt found)";
+
+  const runId = path.relative(root, runDir).split(path.sep).join("-");
+  const outputsDir = path.join(runDir, "outputs");
+  const outputFiles = [];
+  if (fs.existsSync(outputsDir) && fs.statSync(outputsDir).isDirectory()) {
+    for (const name of fs.readdirSync(outputsDir).sort()) {
+      const filePath = path.join(outputsDir, name);
+      if (fs.statSync(filePath).isFile() && !METADATA_FILES.has(name)) {
+        outputFiles.push(embedFile(filePath));
+      }
+    }
+  }
+
+  let grading = null;
+  for (const candidate of [
+    path.join(runDir, "grading.json"),
+    path.join(runDir, "..", "grading.json"),
+  ]) {
+    if (!fs.existsSync(candidate)) continue;
+    try {
+      grading = JSON.parse(fs.readFileSync(candidate, "utf-8"));
+    } catch {
+      // ignore
+    }
+    if (grading) break;
+  }
+
+  return {
+    id: runId,
+    prompt,
+    eval_id: evalId,
+    outputs: outputFiles,
+    grading,
+  };
+}
+
+function embedFile(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  const mime = getMimeType(filePath);
+
+  if (TEXT_EXTENSIONS.has(ext)) {
+    let content;
+    try {
+      content = fs.readFileSync(filePath, "utf-8");
+    } catch {
+      content = "(Error reading file)";
+    }
+    return { name: path.basename(filePath), type: "text", content };
+  }
+
+  if (IMAGE_EXTENSIONS.has(ext)) {
+    try {
+      const raw = fs.readFileSync(filePath);
+      const b64 = raw.toString("base64");
+      return {
+        name: path.basename(filePath),
+        type: "image",
+        mime,
+        data_uri: `data:${mime};base64,${b64}`,
+      };
+    } catch {
+      return { name: path.basename(filePath), type: "error", content: "(Error reading file)" };
+    }
+  }
+
+  if (ext === ".pdf") {
+    try {
+      const raw = fs.readFileSync(filePath);
+      const b64 = raw.toString("base64");
+      return {
+        name: path.basename(filePath),
+        type: "pdf",
+        data_uri: `data:${mime};base64,${b64}`,
+      };
+    } catch {
+      return { name: path.basename(filePath), type: "error", content: "(Error reading file)" };
+    }
+  }
+
+  if (ext === ".xlsx") {
+    try {
+      const raw = fs.readFileSync(filePath);
+      const b64 = raw.toString("base64");
+      return { name: path.basename(filePath), type: "xlsx", data_b64: b64 };
+    } catch {
+      return { name: path.basename(filePath), type: "error", content: "(Error reading file)" };
+    }
+  }
+
+  try {
+    const raw = fs.readFileSync(filePath);
+    const b64 = raw.toString("base64");
+    return {
+      name: path.basename(filePath),
+      type: "binary",
+      mime,
+      data_uri: `data:${mime};base64,${b64}`,
+    };
+  } catch {
+    return { name: path.basename(filePath), type: "error", content: "(Error reading file)" };
+  }
+}
+
+function loadPreviousIteration(workspace) {
+  const result = {};
+  const feedbackMap = {};
+  const feedbackPath = path.join(workspace, "feedback.json");
+  if (fs.existsSync(feedbackPath)) {
+    try {
+      const data = JSON.parse(fs.readFileSync(feedbackPath, "utf-8"));
+      for (const review of data.reviews ?? []) {
+        if ((review.feedback ?? "").trim()) {
+          feedbackMap[review.run_id] = review.feedback;
+        }
+      }
+    } catch {
+      // ignore
+    }
+  }
+
+  const prevRuns = findRuns(workspace);
+  for (const run of prevRuns) {
+    result[run.id] = {
+      feedback: feedbackMap[run.id] ?? "",
+      outputs: run.outputs ?? [],
+    };
+  }
+
+  for (const [runId, feedback] of Object.entries(feedbackMap)) {
+    if (!result[runId]) {
+      result[runId] = { feedback, outputs: [] };
+    }
+  }
+
+  return result;
+}
+
+function generateHtml(runs, skillName, previous = null, benchmark = null) {
+  const templatePath = path.join(__dirname, "viewer.html");
+  const template = fs.readFileSync(templatePath, "utf-8");
+
+  const previousFeedback = {};
+  const previousOutputs = {};
+  if (previous) {
+    for (const [runId, data] of Object.entries(previous)) {
+      if (data.feedback) previousFeedback[runId] = data.feedback;
+      if (data.outputs?.length) previousOutputs[runId] = data.outputs;
+    }
+  }
+
+  const embedded = {
+    skill_name: skillName,
+    runs,
+    previous_feedback: previousFeedback,
+    previous_outputs: previousOutputs,
+  };
+  if (benchmark) embedded.benchmark = benchmark;
+
+  const dataJson = JSON.stringify(embedded);
+  return template.replace("/*__EMBEDDED_DATA__*/", `const EMBEDDED_DATA = ${dataJson};`);
+}
+
+function killPort(port) {
+  try {
+    const result = spawnSync("lsof", ["-ti", `:${port}`], {
+      encoding: "utf-8",
+      timeout: 5000,
+    });
+    for (const pidStr of (result.stdout ?? "").trim().split("\n")) {
+      if (!pidStr.trim()) continue;
+      try {
+        process.kill(parseInt(pidStr.trim(), 10), "SIGTERM");
+      } catch {
+        // ignore
+      }
+    }
+    if ((result.stdout ?? "").trim()) {
+      const sleep = spawnSync("sleep", ["0.5"], { stdio: "ignore" });
+      if (sleep.error) {
+        const deadline = Date.now() + 500;
+        while (Date.now() < deadline) {
+          // brief pause after killing port holder
+        }
+      }
+    }
+  } catch (error) {
+    if (error.message?.includes("ENOENT")) {
+      process.stderr.write("Note: lsof not found, cannot check if port is in use\n");
+    }
+  }
+}
+
+function openUrl(url) {
+  if (process.platform === "darwin") {
+    spawnSync("open", [url], { stdio: "ignore" });
+  } else if (process.platform === "win32") {
+    spawnSync("cmd", ["/c", "start", "", url], { stdio: "ignore", shell: true });
+  } else {
+    spawnSync("xdg-open", [url], { stdio: "ignore" });
+  }
+}
+
+function createReviewServer({ workspace, skillName, feedbackPath, previous, benchmarkPath }) {
+  return http.createServer((req, res) => {
+    const url = new URL(req.url ?? "/", "http://127.0.0.1");
+
+    if (req.method === "GET" && (url.pathname === "/" || url.pathname === "/index.html")) {
+      const runs = findRuns(workspace);
+      let benchmark = null;
+      if (benchmarkPath && fs.existsSync(benchmarkPath)) {
+        try {
+          benchmark = JSON.parse(fs.readFileSync(benchmarkPath, "utf-8"));
+        } catch {
+          // ignore
+        }
+      }
+      const html = generateHtml(runs, skillName, previous, benchmark);
+      const content = Buffer.from(html, "utf-8");
+      res.writeHead(200, {
+        "Content-Type": "text/html; charset=utf-8",
+        "Content-Length": content.length,
+      });
+      res.end(content);
+      return;
+    }
+
+    if (req.method === "GET" && url.pathname === "/api/feedback") {
+      const data = fs.existsSync(feedbackPath) ? fs.readFileSync(feedbackPath) : Buffer.from("{}");
+      res.writeHead(200, {
+        "Content-Type": "application/json",
+        "Content-Length": data.length,
+      });
+      res.end(data);
+      return;
+    }
+
+    if (req.method === "POST" && url.pathname === "/api/feedback") {
+      const chunks = [];
+      req.on("data", (chunk) => chunks.push(chunk));
+      req.on("end", () => {
+        let resp;
+        let status = 200;
+        try {
+          const body = Buffer.concat(chunks).toString("utf-8");
+          const data = JSON.parse(body);
+          if (!data || typeof data !== "object" || !("reviews" in data)) {
+            throw new Error("Expected JSON object with 'reviews' key");
+          }
+          fs.writeFileSync(feedbackPath, `${JSON.stringify(data, null, 2)}\n`);
+          resp = Buffer.from('{"ok":true}');
+        } catch (error) {
+          resp = Buffer.from(JSON.stringify({ error: error.message }));
+          status = 500;
+        }
+        res.writeHead(status, {
+          "Content-Type": "application/json",
+          "Content-Length": resp.length,
+        });
+        res.end(resp);
+      });
+      return;
+    }
+
+    res.writeHead(404);
+    res.end();
+  });
+}
+
+function parseArgs(argv) {
+  const args = {
+    workspace: null,
+    port: 3117,
+    skillName: null,
+    previousWorkspace: null,
+    benchmark: null,
+    static: null,
+  };
+  const positional = [];
+  for (let i = 2; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--port" || arg === "-p") args.port = parseInt(argv[++i], 10);
+    else if (arg === "--skill-name" || arg === "-n") args.skillName = argv[++i];
+    else if (arg === "--previous-workspace") args.previousWorkspace = argv[++i];
+    else if (arg === "--benchmark") args.benchmark = argv[++i];
+    else if (arg === "--static" || arg === "-s") args.static = argv[++i];
+    else if (!arg.startsWith("-")) positional.push(arg);
+  }
+  args.workspace = positional[0] ?? null;
+  return args;
+}
+
+function main() {
+  const args = parseArgs(process.argv);
+  if (!args.workspace) {
+    console.error(
+      "Usage: node generate_review.js <workspace-path> [--port PORT] [--skill-name NAME] [--previous-workspace PATH] [--benchmark PATH] [--static OUTPUT.html]",
+    );
+    process.exit(1);
+  }
+
+  const workspace = path.resolve(args.workspace);
+  if (!fs.existsSync(workspace) || !fs.statSync(workspace).isDirectory()) {
+    console.error(`Error: ${workspace} is not a directory`);
+    process.exit(1);
+  }
+
+  const runs = findRuns(workspace);
+  if (!runs.length) {
+    console.error(`No runs found in ${workspace}`);
+    process.exit(1);
+  }
+
+  const skillName = args.skillName ?? path.basename(workspace).replace("-workspace", "");
+  const feedbackPath = path.join(workspace, "feedback.json");
+
+  let previous = {};
+  if (args.previousWorkspace) {
+    previous = loadPreviousIteration(path.resolve(args.previousWorkspace));
+  }
+
+  const benchmarkPath = args.benchmark ? path.resolve(args.benchmark) : null;
+  let benchmark = null;
+  if (benchmarkPath && fs.existsSync(benchmarkPath)) {
+    try {
+      benchmark = JSON.parse(fs.readFileSync(benchmarkPath, "utf-8"));
+    } catch {
+      // ignore
+    }
+  }
+
+  if (args.static) {
+    const staticPath = path.resolve(args.static);
+    fs.mkdirSync(path.dirname(staticPath), { recursive: true });
+    fs.writeFileSync(staticPath, generateHtml(runs, skillName, previous, benchmark));
+    console.log(`\n  Static viewer written to: ${staticPath}\n`);
+    process.exit(0);
+  }
+
+  killPort(args.port);
+  const server = createReviewServer({
+    workspace,
+    skillName,
+    feedbackPath,
+    previous,
+    benchmarkPath,
+  });
+
+  let port = args.port;
+  server.on("error", () => {
+    // handled below via listen retry
+  });
+
+  function listenOnPort(targetPort) {
+    return new Promise((resolve, reject) => {
+      server.once("error", reject);
+      server.listen(targetPort, "127.0.0.1", () => {
+        server.removeListener("error", reject);
+        resolve(server.address().port);
+      });
+    });
+  }
+
+  listenOnPort(port)
+    .catch(() => listenOnPort(0))
+    .then((actualPort) => {
+      port = actualPort;
+      const url = `http://localhost:${port}`;
+      console.log("\n  Eval Viewer");
+      console.log("  ─────────────────────────────────");
+      console.log(`  URL:       ${url}`);
+      console.log(`  Workspace: ${workspace}`);
+      console.log(`  Feedback:  ${feedbackPath}`);
+      if (args.previousWorkspace) {
+        console.log(
+          `  Previous:  ${args.previousWorkspace} (${Object.keys(previous).length} runs)`,
+        );
+      }
+      if (benchmarkPath) {
+        console.log(`  Benchmark: ${benchmarkPath}`);
+      }
+      console.log("\n  Press Ctrl+C to stop.\n");
+
+      openUrl(url);
+
+      process.on("SIGINT", () => {
+        console.log("\nStopped.");
+        server.close();
+        process.exit(0);
+      });
+    })
+    .catch((error) => {
+      console.error(error.message);
+      process.exit(1);
+    });
+}
+
+if (require.main === module) {
+  main();
+}
+
+module.exports = {
+  findRuns,
+  buildRun,
+  embedFile,
+  loadPreviousIteration,
+  generateHtml,
+  createReviewServer,
+  killPort,
+  openUrl,
+};