selftune 0.1.4 → 0.2.1

package/cli/selftune/dashboard.ts

@@ -1,124 +1,12 @@
 /**
- * selftune dashboard — Exports JSONL data into a standalone HTML viewer.
+ * selftune dashboard — Start the local React SPA dashboard server.
  *
  * Usage:
- *   selftune dashboard              — Open dashboard in default browser
- *   selftune dashboard --export     — Export data-embedded HTML to stdout
- *   selftune dashboard --out FILE   — Write data-embedded HTML to FILE
+ *   selftune dashboard              — Start server on port 3141 and open browser
+ *   selftune dashboard --port 8080  — Start on custom port
+ *   selftune dashboard --serve      — Deprecated alias for the default behavior
  */
 
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { dirname, join, resolve } from "node:path";
-import { EVOLUTION_AUDIT_LOG, QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "./constants.js";
-import { getLastDeployedProposal, readAuditTrail } from "./evolution/audit.js";
-import { computeMonitoringSnapshot } from "./monitoring/watch.js";
-import type {
-  EvolutionAuditEntry,
-  QueryLogRecord,
-  SessionTelemetryRecord,
-  SkillUsageRecord,
-} from "./types.js";
-import { readJsonl } from "./utils/jsonl.js";
-
-function findViewerHTML(): string {
-  // Try relative to this module first (works for both dev and installed)
-  const candidates = [
-    join(dirname(import.meta.dir), "..", "dashboard", "index.html"),
-    join(dirname(import.meta.dir), "dashboard", "index.html"),
-    resolve("dashboard", "index.html"),
-  ];
-  for (const c of candidates) {
-    if (existsSync(c)) return c;
-  }
-  throw new Error("Could not find dashboard/index.html. Ensure it exists in the selftune repo.");
-}
-
-function buildEmbeddedHTML(): string {
-  const template = readFileSync(findViewerHTML(), "utf-8");
-
-  const telemetry = readJsonl<SessionTelemetryRecord>(TELEMETRY_LOG);
-  const skills = readJsonl<SkillUsageRecord>(SKILL_LOG);
-  const queries = readJsonl<QueryLogRecord>(QUERY_LOG);
-  const evolution = readJsonl<EvolutionAuditEntry>(EVOLUTION_AUDIT_LOG);
-
-  const totalRecords = telemetry.length + skills.length + queries.length + evolution.length;
-
-  if (totalRecords === 0) {
-    console.error("No log data found. Run some sessions first.");
-    console.error(`  Checked: ${TELEMETRY_LOG}`);
-    console.error(`           ${SKILL_LOG}`);
-    console.error(`           ${QUERY_LOG}`);
-    console.error(`           ${EVOLUTION_AUDIT_LOG}`);
-    process.exit(1);
-  }
-
-  // Compute per-skill monitoring snapshots
-  const skillNames = [...new Set(skills.map((r) => r.skill_name))];
-  const snapshots: Record<string, ReturnType<typeof computeMonitoringSnapshot>> = {};
-  for (const name of skillNames) {
-    const lastDeployed = getLastDeployedProposal(name);
-    const baselinePassRate = lastDeployed?.eval_snapshot?.pass_rate ?? 0.5;
-    snapshots[name] = computeMonitoringSnapshot(
-      name,
-      telemetry,
-      skills,
-      queries,
-      telemetry.length,
-      baselinePassRate,
-    );
-  }
-
-  // Compute unmatched queries
-  const triggeredQueries = new Set(
-    skills.filter((r) => r.triggered).map((r) => r.query.toLowerCase().trim()),
-  );
-  const unmatched = queries
-    .filter((q) => !triggeredQueries.has(q.query.toLowerCase().trim()))
-    .map((q) => ({
-      timestamp: q.timestamp,
-      session_id: q.session_id,
-      query: q.query,
-    }));
-
-  // Compute pending proposals
-  const auditTrail = readAuditTrail();
-  const proposalStatus: Record<string, string[]> = {};
-  for (const e of auditTrail) {
-    if (!proposalStatus[e.proposal_id]) proposalStatus[e.proposal_id] = [];
-    proposalStatus[e.proposal_id].push(e.action);
-  }
-  // Deduplicate by proposal_id: one entry per pending proposal
-  const terminalActions = new Set(["deployed", "rejected", "rolled_back"]);
-  const seenProposals = new Set<string>();
-  const pendingProposals = auditTrail.filter((e) => {
-    if (e.action !== "created" && e.action !== "validated") return false;
-    if (seenProposals.has(e.proposal_id)) return false;
-    const actions = proposalStatus[e.proposal_id] || [];
-    const isPending = !actions.some((a: string) => terminalActions.has(a));
-    if (isPending) seenProposals.add(e.proposal_id);
-    return isPending;
-  });
-
-  const data = {
-    telemetry,
-    skills,
-    queries,
-    evolution,
-    computed: {
-      snapshots,
-      unmatched,
-      pendingProposals,
-    },
-  };
-
-  // Inject embedded data right before </body>
-  // Escape </script> sequences to prevent XSS via embedded JSON
-  const safeJson = JSON.stringify(data).replace(/<\/script>/gi, "<\\/script>");
-  const dataScript = `<script id="embedded-data" type="application/json">${safeJson}</script>`;
-  return template.replace("</body>", `${dataScript}\n</body>`);
-}
-
 export async function cliMain(): Promise<void> {
   const args = process.argv.slice(2);
 
@@ -126,51 +14,50 @@ export async function cliMain(): Promise<void> {
     console.log(`selftune dashboard — Visual data dashboard
 
 Usage:
-  selftune dashboard              Open dashboard in default browser
-  selftune dashboard --export     Export data-embedded HTML to stdout
-  selftune dashboard --out FILE   Write data-embedded HTML to FILE`);
+  selftune dashboard                      Start dashboard server (port 3141)
+  selftune dashboard --port 8080          Start on custom port
+  selftune dashboard --serve              Deprecated alias for default behavior
+  selftune dashboard --no-open            Start server without opening browser`);
     process.exit(0);
   }
 
-  if (args.includes("--export")) {
-    process.stdout.write(buildEmbeddedHTML());
-    return;
+  if (args.includes("--export") || args.includes("--out")) {
+    console.error("Legacy dashboard export was removed.");
+    console.error(
+      "Use `selftune dashboard` to run the SPA locally, then share a route or screenshot instead.",
+    );
+    process.exit(1);
   }
 
-  const outIdx = args.indexOf("--out");
-  if (outIdx !== -1) {
-    const outPath = args[outIdx + 1];
-    if (!outPath) {
-      console.error("--out requires a file path argument");
+  const portIdx = args.indexOf("--port");
+  let port: number | undefined;
+  if (portIdx !== -1) {
+    const parsed = Number.parseInt(args[portIdx + 1], 10);
+    if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) {
+      console.error(`Invalid port "${args[portIdx + 1]}": must be an integer between 1 and 65535.`);
       process.exit(1);
     }
-    const html = buildEmbeddedHTML();
-    writeFileSync(outPath, html, "utf-8");
-    console.log(`Dashboard written to ${outPath}`);
-    return;
-  }
-
-  // Default: write to temp file and open in browser
-  const tmpDir = join(homedir(), ".selftune");
-  if (!existsSync(tmpDir)) {
-    mkdirSync(tmpDir, { recursive: true });
-  }
-  const tmpPath = join(tmpDir, "dashboard.html");
-  const html = buildEmbeddedHTML();
-  writeFileSync(tmpPath, html, "utf-8");
-
-  console.log(`Dashboard saved to ${tmpPath}`);
-  console.log("Opening in browser...");
-
-  try {
-    const platform = process.platform;
-    const cmd = platform === "darwin" ? "open" : platform === "linux" ? "xdg-open" : null;
-    if (!cmd) throw new Error("Unsupported platform");
-    const proc = Bun.spawn([cmd, tmpPath], { stdio: ["ignore", "ignore", "ignore"] });
-    await proc.exited;
-    if (proc.exitCode !== 0) throw new Error(`Failed to launch ${cmd}`);
-  } catch {
-    console.log(`Open manually: file://${tmpPath}`);
-  }
-  process.exit(0);
+    port = parsed;
+  }
+
+  if (args.includes("--serve")) {
+    console.warn("`selftune dashboard --serve` is deprecated; use `selftune dashboard` instead.");
+  }
+
+  const openBrowser = !args.includes("--no-open");
+  const { startDashboardServer } = await import("./dashboard-server.js");
+  const { stop } = await startDashboardServer({ port, openBrowser });
+  await new Promise<void>((resolve) => {
+    let closed = false;
+    const keepAlive = setInterval(() => {}, 1 << 30);
+    const shutdown = () => {
+      if (closed) return;
+      closed = true;
+      clearInterval(keepAlive);
+      stop();
+      resolve();
+    };
+    process.on("SIGINT", shutdown);
+    process.on("SIGTERM", shutdown);
+  });
 }

package/cli/selftune/eval/baseline.ts

@@ -0,0 +1,248 @@
+/**
+ * baseline.ts
+ *
+ * Measures the value a skill adds over a no-skill baseline.
+ *
+ * Runs trigger checks against an EMPTY string description (no-skill baseline)
+ * and against the current description (with-skill), then computes lift.
+ * A skill "adds value" when lift >= 0.05 (5 percentage points).
+ */
+
+import { parseArgs } from "node:util";
+
+import type { BaselineResult, EvalEntry } from "../types.js";
+import { callLlm } from "../utils/llm-call.js";
+import { buildTriggerCheckPrompt, parseTriggerResponse } from "../utils/trigger-check.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface BaselineOptions {
+  evalSet: EvalEntry[];
+  skillDescription: string;
+  skillName: string;
+  agent: string;
+  modelFlag?: string;
+}
+
+export interface BaselineMeasurement {
+  skill_name: string;
+  baseline_pass_rate: number;
+  with_skill_pass_rate: number;
+  lift: number;
+  adds_value: boolean;
+  per_entry: BaselineResult[];
+  measured_at: string;
+}
+
+/**
+ * Injectable dependencies for measureBaseline(). When omitted, the real
+ * module imports are used. Pass overrides in tests to avoid real LLM calls.
+ */
+export interface BaselineDeps {
+  callLlm?: typeof callLlm;
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const LIFT_THRESHOLD = 0.05;
+const SYSTEM_PROMPT = "You are an evaluation assistant. Answer only YES or NO.";
+
+// ---------------------------------------------------------------------------
+// Core measurement
+// ---------------------------------------------------------------------------
+
+/** Measure baseline vs. with-skill trigger accuracy across an eval set. */
+export async function measureBaseline(
+  options: BaselineOptions,
+  _deps: BaselineDeps = {},
+): Promise<BaselineMeasurement> {
+  const { evalSet, skillDescription, skillName, agent, modelFlag } = options;
+  const _callLlm = _deps.callLlm ?? callLlm;
+
+  if (evalSet.length === 0) {
+    return {
+      skill_name: skillName,
+      baseline_pass_rate: 0,
+      with_skill_pass_rate: 0,
+      lift: 0,
+      adds_value: false,
+      per_entry: [],
+      measured_at: new Date().toISOString(),
+    };
+  }
+
+  const perEntry: BaselineResult[] = [];
+  let baselinePassed = 0;
+  let withSkillPassed = 0;
+
+  for (const entry of evalSet) {
+    // --- Baseline check (empty description) ---
+    const baselinePrompt = buildTriggerCheckPrompt("", entry.query);
+    const baselineRaw = await _callLlm(SYSTEM_PROMPT, baselinePrompt, agent, modelFlag);
+    const baselineTriggered = parseTriggerResponse(baselineRaw);
+    const baselinePass =
+      (entry.should_trigger && baselineTriggered) || (!entry.should_trigger && !baselineTriggered);
+
+    if (baselinePass) baselinePassed++;
+
+    perEntry.push({
+      skill_name: skillName,
+      query: entry.query,
+      with_skill: false,
+      triggered: baselineTriggered,
+      pass: baselinePass,
+      measured_at: new Date().toISOString(),
+    });
+
+    // --- With-skill check (actual description) ---
+    const withSkillPrompt = buildTriggerCheckPrompt(skillDescription, entry.query);
+    const withSkillRaw = await _callLlm(SYSTEM_PROMPT, withSkillPrompt, agent, modelFlag);
+    const withSkillTriggered = parseTriggerResponse(withSkillRaw);
+    const withSkillPass =
+      (entry.should_trigger && withSkillTriggered) ||
+      (!entry.should_trigger && !withSkillTriggered);
+
+    if (withSkillPass) withSkillPassed++;
+
+    perEntry.push({
+      skill_name: skillName,
+      query: entry.query,
+      with_skill: true,
+      triggered: withSkillTriggered,
+      pass: withSkillPass,
+      measured_at: new Date().toISOString(),
+    });
+  }
+
+  const total = evalSet.length;
+  const baselinePassRate = baselinePassed / total;
+  const withSkillPassRate = withSkillPassed / total;
+  const lift = withSkillPassRate - baselinePassRate;
+
+  return {
+    skill_name: skillName,
+    baseline_pass_rate: baselinePassRate,
+    with_skill_pass_rate: withSkillPassRate,
+    lift,
+    adds_value: lift >= LIFT_THRESHOLD,
+    per_entry: perEntry,
+    measured_at: new Date().toISOString(),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry point
+// ---------------------------------------------------------------------------
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      skill: { type: "string" },
+      "skill-path": { type: "string" },
+      "eval-set": { type: "string" },
+      agent: { type: "string" },
+      help: { type: "boolean", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    console.log(`selftune grade baseline — Measure skill value vs. no-skill baseline
+
+Usage:
+  selftune grade baseline --skill <name> --skill-path <path> [options]
+
+Options:
+  --skill         Skill name (required)
+  --skill-path    Path to SKILL.md (required)
+  --eval-set      Path to eval set JSON (optional, builds from logs if omitted)
+  --agent         Agent CLI to use (claude, codex, opencode)
+  --help          Show this help message`);
+    process.exit(0);
+  }
+
+  if (!values.skill || !values["skill-path"]) {
+    console.error("[ERROR] --skill and --skill-path are required");
+    process.exit(1);
+  }
+
+  const { existsSync, readFileSync } = await import("node:fs");
+
+  // Read skill description
+  const skillPath = values["skill-path"];
+  if (!existsSync(skillPath)) {
+    console.error(`[ERROR] SKILL.md not found at ${skillPath}`);
+    process.exit(1);
+  }
+  const skillDescription = readFileSync(skillPath, "utf-8");
+
+  // Load eval set
+  let evalSet: EvalEntry[];
+  if (values["eval-set"] && existsSync(values["eval-set"])) {
+    const raw = readFileSync(values["eval-set"], "utf-8");
+    evalSet = JSON.parse(raw) as EvalEntry[];
+  } else {
+    // Build from logs
+    const { QUERY_LOG } = await import("../constants.js");
+    const { readJsonl } = await import("../utils/jsonl.js");
+    const { readEffectiveSkillUsageRecords } = await import("../utils/skill-log.js");
+    const { buildEvalSet } = await import("./hooks-to-evals.js");
+    const skillRecords = readEffectiveSkillUsageRecords();
+    const queryRecords = readJsonl(QUERY_LOG);
+    evalSet = buildEvalSet(skillRecords, queryRecords, values.skill);
+  }
+
+  // Detect agent
+  const { detectAgent } = await import("../utils/llm-call.js");
+  const requestedAgent = values.agent;
+  if (requestedAgent && !Bun.which(requestedAgent)) {
+    console.error(
+      JSON.stringify({
+        level: "error",
+        code: "agent_not_in_path",
+        message: `Agent CLI '${requestedAgent}' not found in PATH.`,
+        action: "Install it or omit --agent to use auto-detection.",
+      }),
+    );
+    process.exit(1);
+  }
+  const agent = requestedAgent ?? detectAgent();
+  if (!agent) {
+    console.error(
+      JSON.stringify({
+        level: "error",
+        code: "agent_not_found",
+        message: "No agent CLI (claude/codex/opencode) found in PATH.",
+        action: "Install Claude Code, Codex, or OpenCode.",
+      }),
+    );
+    process.exit(1);
+  }
+
+  const result = await measureBaseline({
+    evalSet,
+    skillDescription,
+    skillName: values.skill,
+    agent,
+  });
+
+  console.log(JSON.stringify(result, null, 2));
+  process.exit(result.adds_value ? 0 : 1);
+}
+
+if (import.meta.main) {
+  cliMain().catch((err) => {
+    console.error(
+      JSON.stringify({
+        level: "fatal",
+        message: err instanceof Error ? err.message : String(err),
+        stack: err instanceof Error ? err.stack : undefined,
+      }),
+    );
+    process.exit(1);
+  });
+}