selftune 0.2.0 → 0.2.2

package/cli/selftune/dashboard.ts

@@ -1,126 +1,12 @@
 /**
- * selftune dashboard — Exports JSONL data into a standalone HTML viewer.
+ * selftune dashboard — Start the local React SPA dashboard server.
  *
  * Usage:
- *   selftune dashboard              — Open dashboard in default browser
- *   selftune dashboard --export     — Export data-embedded HTML to stdout
- *   selftune dashboard --out FILE   — Write data-embedded HTML to FILE
- *   selftune dashboard --serve      — Start live dashboard server (default port 3141)
- *   selftune dashboard --serve --port 8080 — Start on custom port
+ *   selftune dashboard              — Start server on port 3141 and open browser
+ *   selftune dashboard --port 8080  — Start on custom port
+ *   selftune dashboard --serve      — Deprecated alias for the default behavior
  */
 
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { dirname, join, resolve } from "node:path";
-import { EVOLUTION_AUDIT_LOG, QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "./constants.js";
-import { getLastDeployedProposal, readAuditTrail } from "./evolution/audit.js";
-import { computeMonitoringSnapshot } from "./monitoring/watch.js";
-import type {
-  EvolutionAuditEntry,
-  QueryLogRecord,
-  SessionTelemetryRecord,
-  SkillUsageRecord,
-} from "./types.js";
-import { readJsonl } from "./utils/jsonl.js";
-
-function findViewerHTML(): string {
-  // Try relative to this module first (works for both dev and installed)
-  const candidates = [
-    join(dirname(import.meta.dir), "..", "dashboard", "index.html"),
-    join(dirname(import.meta.dir), "dashboard", "index.html"),
-    resolve("dashboard", "index.html"),
-  ];
-  for (const c of candidates) {
-    if (existsSync(c)) return c;
-  }
-  throw new Error("Could not find dashboard/index.html. Ensure it exists in the selftune repo.");
-}
-
-function buildEmbeddedHTML(): string {
-  const template = readFileSync(findViewerHTML(), "utf-8");
-
-  const telemetry = readJsonl<SessionTelemetryRecord>(TELEMETRY_LOG);
-  const skills = readJsonl<SkillUsageRecord>(SKILL_LOG);
-  const queries = readJsonl<QueryLogRecord>(QUERY_LOG);
-  const evolution = readJsonl<EvolutionAuditEntry>(EVOLUTION_AUDIT_LOG);
-
-  const totalRecords = telemetry.length + skills.length + queries.length + evolution.length;
-
-  if (totalRecords === 0) {
-    console.error("No log data found. Run some sessions first.");
-    console.error(`  Checked: ${TELEMETRY_LOG}`);
-    console.error(`           ${SKILL_LOG}`);
-    console.error(`           ${QUERY_LOG}`);
-    console.error(`           ${EVOLUTION_AUDIT_LOG}`);
-    process.exit(1);
-  }
-
-  // Compute per-skill monitoring snapshots
-  const skillNames = [...new Set(skills.map((r) => r.skill_name))];
-  const snapshots: Record<string, ReturnType<typeof computeMonitoringSnapshot>> = {};
-  for (const name of skillNames) {
-    const lastDeployed = getLastDeployedProposal(name);
-    const baselinePassRate = lastDeployed?.eval_snapshot?.pass_rate ?? 0.5;
-    snapshots[name] = computeMonitoringSnapshot(
-      name,
-      telemetry,
-      skills,
-      queries,
-      telemetry.length,
-      baselinePassRate,
-    );
-  }
-
-  // Compute unmatched queries
-  const triggeredQueries = new Set(
-    skills.filter((r) => r.triggered).map((r) => r.query.toLowerCase().trim()),
-  );
-  const unmatched = queries
-    .filter((q) => !triggeredQueries.has(q.query.toLowerCase().trim()))
-    .map((q) => ({
-      timestamp: q.timestamp,
-      session_id: q.session_id,
-      query: q.query,
-    }));
-
-  // Compute pending proposals
-  const auditTrail = readAuditTrail();
-  const proposalStatus: Record<string, string[]> = {};
-  for (const e of auditTrail) {
-    if (!proposalStatus[e.proposal_id]) proposalStatus[e.proposal_id] = [];
-    proposalStatus[e.proposal_id].push(e.action);
-  }
-  // Deduplicate by proposal_id: one entry per pending proposal
-  const terminalActions = new Set(["deployed", "rejected", "rolled_back"]);
-  const seenProposals = new Set<string>();
-  const pendingProposals = auditTrail.filter((e) => {
-    if (e.action !== "created" && e.action !== "validated") return false;
-    if (seenProposals.has(e.proposal_id)) return false;
-    const actions = proposalStatus[e.proposal_id] || [];
-    const isPending = !actions.some((a: string) => terminalActions.has(a));
-    if (isPending) seenProposals.add(e.proposal_id);
-    return isPending;
-  });
-
-  const data = {
-    telemetry,
-    skills,
-    queries,
-    evolution,
-    computed: {
-      snapshots,
-      unmatched,
-      pendingProposals,
-    },
-  };
-
-  // Inject embedded data right before </body>
-  // Escape </script> sequences to prevent XSS via embedded JSON
-  const safeJson = JSON.stringify(data).replace(/<\/script>/gi, "<\\/script>");
-  const dataScript = `<script id="embedded-data" type="application/json">${safeJson}</script>`;
-  return template.replace("</body>", `${dataScript}\n</body>`);
-}
-
 export async function cliMain(): Promise<void> {
   const args = process.argv.slice(2);
 
@@ -128,71 +14,50 @@ export async function cliMain(): Promise<void> {
     console.log(`selftune dashboard — Visual data dashboard
 
 Usage:
-  selftune dashboard                        Open dashboard in default browser
-  selftune dashboard --export               Export data-embedded HTML to stdout
-  selftune dashboard --out FILE             Write data-embedded HTML to FILE
-  selftune dashboard --serve                Start live dashboard server (port 3141)
-  selftune dashboard --serve --port 8080    Start on custom port`);
+  selftune dashboard                      Start dashboard server (port 3141)
+  selftune dashboard --port 8080          Start on custom port
+  selftune dashboard --serve              Deprecated alias for default behavior
+  selftune dashboard --no-open            Start server without opening browser`);
     process.exit(0);
   }
 
-  if (args.includes("--serve")) {
-    const portIdx = args.indexOf("--port");
-    let port: number | undefined;
-    if (portIdx !== -1) {
-      const parsed = Number.parseInt(args[portIdx + 1], 10);
-      if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) {
-        console.error(
-          `Invalid port "${args[portIdx + 1]}": must be an integer between 1 and 65535.`,
-        );
-        process.exit(1);
-      }
-      port = parsed;
-    }
-    const { startDashboardServer } = await import("./dashboard-server.js");
-    await startDashboardServer({ port, openBrowser: true });
-    return;
-  }
-
-  if (args.includes("--export")) {
-    process.stdout.write(buildEmbeddedHTML());
-    return;
+  if (args.includes("--export") || args.includes("--out")) {
+    console.error("Legacy dashboard export was removed.");
+    console.error(
+      "Use `selftune dashboard` to run the SPA locally, then share a route or screenshot instead.",
+    );
+    process.exit(1);
   }
 
-  const outIdx = args.indexOf("--out");
-  if (outIdx !== -1) {
-    const outPath = args[outIdx + 1];
-    if (!outPath) {
-      console.error("--out requires a file path argument");
+  const portIdx = args.indexOf("--port");
+  let port: number | undefined;
+  if (portIdx !== -1) {
+    const parsed = Number.parseInt(args[portIdx + 1], 10);
+    if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) {
+      console.error(`Invalid port "${args[portIdx + 1]}": must be an integer between 1 and 65535.`);
       process.exit(1);
     }
-    const html = buildEmbeddedHTML();
-    writeFileSync(outPath, html, "utf-8");
-    console.log(`Dashboard written to ${outPath}`);
-    return;
-  }
-
-  // Default: write to temp file and open in browser
-  const tmpDir = join(homedir(), ".selftune");
-  if (!existsSync(tmpDir)) {
-    mkdirSync(tmpDir, { recursive: true });
+    port = parsed;
   }
-  const tmpPath = join(tmpDir, "dashboard.html");
-  const html = buildEmbeddedHTML();
-  writeFileSync(tmpPath, html, "utf-8");
 
-  console.log(`Dashboard saved to ${tmpPath}`);
-  console.log("Opening in browser...");
-
-  try {
-    const platform = process.platform;
-    const cmd = platform === "darwin" ? "open" : platform === "linux" ? "xdg-open" : null;
-    if (!cmd) throw new Error("Unsupported platform");
-    const proc = Bun.spawn([cmd, tmpPath], { stdio: ["ignore", "ignore", "ignore"] });
-    await proc.exited;
-    if (proc.exitCode !== 0) throw new Error(`Failed to launch ${cmd}`);
-  } catch {
-    console.log(`Open manually: file://${tmpPath}`);
-  }
-  process.exit(0);
+  if (args.includes("--serve")) {
+    console.warn("`selftune dashboard --serve` is deprecated; use `selftune dashboard` instead.");
+  }
+
+  const openBrowser = !args.includes("--no-open");
+  const { startDashboardServer } = await import("./dashboard-server.js");
+  const { stop } = await startDashboardServer({ port, openBrowser });
+  await new Promise<void>((resolve) => {
+    let closed = false;
+    const keepAlive = setInterval(() => {}, 1 << 30);
+    const shutdown = () => {
+      if (closed) return;
+      closed = true;
+      clearInterval(keepAlive);
+      stop();
+      resolve();
+    };
+    process.on("SIGINT", shutdown);
+    process.on("SIGTERM", shutdown);
+  });
 }

package/cli/selftune/eval/baseline.ts

@@ -151,10 +151,10 @@ export async function cliMain(): Promise<void> {
   });
 
   if (values.help) {
-    console.log(`selftune baseline — Measure skill value vs. no-skill baseline
+    console.log(`selftune grade baseline — Measure skill value vs. no-skill baseline
 
 Usage:
-  selftune baseline --skill <name> --skill-path <path> [options]
+  selftune grade baseline --skill <name> --skill-path <path> [options]
 
 Options:
   --skill         Skill name (required)
@@ -187,10 +187,11 @@ Options:
     evalSet = JSON.parse(raw) as EvalEntry[];
   } else {
     // Build from logs
-    const { QUERY_LOG, SKILL_LOG } = await import("../constants.js");
+    const { QUERY_LOG } = await import("../constants.js");
     const { readJsonl } = await import("../utils/jsonl.js");
+    const { readEffectiveSkillUsageRecords } = await import("../utils/skill-log.js");
     const { buildEvalSet } = await import("./hooks-to-evals.js");
-    const skillRecords = readJsonl(SKILL_LOG);
+    const skillRecords = readEffectiveSkillUsageRecords();
     const queryRecords = readJsonl(QUERY_LOG);
     evalSet = buildEvalSet(skillRecords, queryRecords, values.skill);
   }

package/cli/selftune/eval/composability-v2.ts

@@ -0,0 +1,273 @@
+/**
+ * composability-v2.ts
+ *
+ * Extended composability analysis with synergy detection and sequence extraction.
+ * Builds on v1 patterns but adds:
+ *   - Synergy scores (positive = better together, negative = conflict)
+ *   - Ordered skill sequence detection from timestamps
+ *   - Workflow candidate flagging
+ *
+ * Pure function -- no I/O. CLI wrapper handles reading JSONL.
+ */
+
+import type {
+  ComposabilityReportV2,
+  CoOccurrencePairV2,
+  SessionTelemetryRecord,
+  SkillSequence,
+  SkillUsageRecord,
+} from "../types.js";
+import { clamp } from "../utils/math.js";
+
+/**
+ * Extended composability analysis with synergy detection and sequence extraction.
+ *
+ * @param skillName  - The skill to analyze
+ * @param telemetry  - Session telemetry records
+ * @param usage      - Skill usage records (for timestamp-based ordering)
+ * @param options    - Analysis options
+ * @returns ComposabilityReportV2 with synergy pairs, sequences, and workflow candidates
+ */
+export function analyzeComposabilityV2(
+  skillName: string,
+  telemetry: SessionTelemetryRecord[],
+  usage: SkillUsageRecord[],
+  options?: {
+    window?: number;
+    minOccurrences?: number;
+  },
+): ComposabilityReportV2 {
+  const minOccurrences = options?.minOccurrences ?? 3;
+
+  // Apply window: sort by timestamp descending, take last N
+  let sessions = telemetry.filter((r) => r && Array.isArray(r.skills_triggered));
+
+  if (options?.window && options.window > 0) {
+    sessions = sessions
+      .sort((a, b) => (b.timestamp ?? "").localeCompare(a.timestamp ?? ""))
+      .slice(0, options.window);
+  }
+
+  // Build a set of session IDs in scope (after windowing)
+  const sessionIdSet = new Set(sessions.map((s) => s.session_id));
+
+  // Sessions where the target skill was triggered
+  const skillSessions = sessions.filter((r) => r.skills_triggered.includes(skillName));
+
+  // Sessions where the target skill was triggered ALONE (no other skills)
+  const aloneSessions = skillSessions.filter((r) => r.skills_triggered.length === 1);
+
+  // Average errors when skill is used alone
+  const errorsAlone =
+    aloneSessions.length > 0
+      ? aloneSessions.reduce((sum, r) => sum + (r.errors_encountered ?? 0), 0) /
+        aloneSessions.length
+      : 0;
+
+  // Find all co-occurring skills
+  const coSkills = new Set<string>();
+  for (const r of skillSessions) {
+    for (const s of r.skills_triggered) {
+      if (s !== skillName) coSkills.add(s);
+    }
+  }
+
+  // -----------------------------------------------------------------------
+  // Synergy computation for each co-occurring skill
+  // -----------------------------------------------------------------------
+
+  const pairs: CoOccurrencePairV2[] = [];
+  for (const coSkill of coSkills) {
+    // Sessions where BOTH skills are triggered together
+    const togetherSessions = skillSessions.filter((r) => r.skills_triggered.includes(coSkill));
+    const coOccurrenceCount = togetherSessions.length;
+
+    // Average errors when both skills are used together
+    const avgErrorsTogether =
+      togetherSessions.length > 0
+        ? togetherSessions.reduce((sum, r) => sum + (r.errors_encountered ?? 0), 0) /
+          togetherSessions.length
+        : 0;
+
+    // Baseline: consider BOTH skills' solo error rates, take the max
+    const coSkillAloneSessions = sessions.filter(
+      (r) => r.skills_triggered.length === 1 && r.skills_triggered.includes(coSkill),
+    );
+    const errorsCoSkillAlone =
+      coSkillAloneSessions.length > 0
+        ? coSkillAloneSessions.reduce((sum, r) => sum + (r.errors_encountered ?? 0), 0) /
+          coSkillAloneSessions.length
+        : errorsAlone;
+    const avgErrorsAlone = Math.max(errorsAlone, errorsCoSkillAlone);
+
+    // synergy_score = clamp((avg_errors_alone - avg_errors_together) / (avg_errors_alone + 1), -1, 1)
+    const synergyScore = clamp((avgErrorsAlone - avgErrorsTogether) / (avgErrorsAlone + 1), -1, 1);
+
+    const conflictDetected = synergyScore < -0.3;
+    const workflowCandidate = synergyScore > 0.3 && coOccurrenceCount >= minOccurrences;
+
+    const pair: CoOccurrencePairV2 = {
+      skill_a: skillName,
+      skill_b: coSkill,
+      co_occurrence_count: coOccurrenceCount,
+      conflict_detected: conflictDetected,
+      synergy_score: synergyScore,
+      avg_errors_together: avgErrorsTogether,
+      avg_errors_alone: avgErrorsAlone,
+      workflow_candidate: workflowCandidate,
+    };
+
+    if (conflictDetected) {
+      pair.conflict_reason = `synergy_score=${synergyScore.toFixed(3)} (avg errors together=${avgErrorsTogether.toFixed(1)} vs alone=${avgErrorsAlone.toFixed(1)})`;
+    }
+
+    pairs.push(pair);
+  }
+
+  // Sort by co-occurrence count descending
+  pairs.sort((a, b) => b.co_occurrence_count - a.co_occurrence_count);
+
+  // -----------------------------------------------------------------------
+  // Sequence extraction from usage records
+  // -----------------------------------------------------------------------
+
+  // Filter usage records for sessions in scope that contain the target skill
+  const usageInScope = usage.filter((u) => sessionIdSet.has(u.session_id));
+
+  // Group by session_id
+  const usageBySession = new Map<string, SkillUsageRecord[]>();
+  for (const u of usageInScope) {
+    const group = usageBySession.get(u.session_id);
+    if (group) {
+      group.push(u);
+    } else {
+      usageBySession.set(u.session_id, [u]);
+    }
+  }
+
+  // Build ordered sequences per session (only sessions containing target skill)
+  const sessionSequences: Array<{ skills: string[]; sessionId: string; firstQuery: string }> = [];
+
+  for (const [sessionId, records] of usageBySession) {
+    // Only sessions containing the target skill
+    if (!records.some((r) => r.skill_name === skillName)) continue;
+
+    // Sort by timestamp ascending
+    const sorted = [...records].sort((a, b) =>
+      (a.timestamp ?? "").localeCompare(b.timestamp ?? ""),
+    );
+
+    // Extract skill names, deduplicate consecutive same-skill entries
+    const skills: string[] = [];
+    for (const r of sorted) {
+      if (skills.length === 0 || skills[skills.length - 1] !== r.skill_name) {
+        skills.push(r.skill_name);
+      }
+    }
+
+    // Only record sequences with 2+ skills
+    if (skills.length >= 2) {
+      sessionSequences.push({
+        skills,
+        sessionId,
+        firstQuery: sorted[0]?.query ?? "",
+      });
+    }
+  }
+
+  // Count frequency of each unique sequence (by JSON key)
+  const sequenceCounts = new Map<
+    string,
+    { count: number; queryCounts: Map<string, number>; skills: string[] }
+  >();
+  for (const seq of sessionSequences) {
+    const key = JSON.stringify(seq.skills);
+    const existing = sequenceCounts.get(key);
+    if (existing) {
+      existing.count++;
+      existing.queryCounts.set(seq.firstQuery, (existing.queryCounts.get(seq.firstQuery) ?? 0) + 1);
+    } else {
+      sequenceCounts.set(key, {
+        count: 1,
+        queryCounts: new Map([[seq.firstQuery, 1]]),
+        skills: seq.skills,
+      });
+    }
+  }
+
+  // Also count all orderings of each skill set (for consistency computation)
+  // Key: sorted skill set -> total count of all orderings
+  const skillSetCounts = new Map<string, number>();
+  for (const seq of sessionSequences) {
+    const setKey = JSON.stringify([...seq.skills].sort());
+    skillSetCounts.set(setKey, (skillSetCounts.get(setKey) ?? 0) + 1);
+  }
+
+  // Build telemetry lookup by session_id for synergy scoring
+  const telemetryBySession = new Map<string, SessionTelemetryRecord>();
+  for (const s of sessions) {
+    telemetryBySession.set(s.session_id, s);
+  }
+
+  // Build sequences, filtered by minOccurrences
+  const sequences: SkillSequence[] = [];
+  for (const [key, data] of sequenceCounts) {
+    if (data.count < minOccurrences) continue;
+
+    // Compute synergy_score for this sequence's sessions
+    const matchingSessions = sessionSequences
+      .filter((s) => JSON.stringify(s.skills) === key)
+      .map((s) => telemetryBySession.get(s.sessionId))
+      .filter((s): s is SessionTelemetryRecord => s !== undefined);
+
+    const seqErrorsTogether =
+      matchingSessions.length > 0
+        ? matchingSessions.reduce((sum, r) => sum + (r.errors_encountered ?? 0), 0) /
+          matchingSessions.length
+        : 0;
+
+    const seqSynergyScore = clamp((errorsAlone - seqErrorsTogether) / (errorsAlone + 1), -1, 1);
+
+    // Consistency: count of this exact order / count of all orderings of same skill set
+    const setKey = JSON.stringify([...data.skills].sort());
+    const totalOrderings = skillSetCounts.get(setKey) ?? data.count;
+    const sequenceConsistency = totalOrderings > 0 ? data.count / totalOrderings : 1;
+
+    let representativeQuery = "";
+    let highestFrequency = -1;
+    for (const [query, frequency] of data.queryCounts) {
+      if (frequency > highestFrequency) {
+        representativeQuery = query;
+        highestFrequency = frequency;
+      }
+    }
+
+    sequences.push({
+      skills: data.skills,
+      occurrence_count: data.count,
+      synergy_score: seqSynergyScore,
+      representative_query: representativeQuery,
+      sequence_consistency: sequenceConsistency,
+    });
+  }
+
+  // Sort sequences by occurrence_count descending
+  sequences.sort((a, b) => b.occurrence_count - a.occurrence_count);
+
+  // -----------------------------------------------------------------------
+  // Assemble report
+  // -----------------------------------------------------------------------
+
+  const workflowCandidates = pairs.filter((p) => p.workflow_candidate);
+  const synergyCount = pairs.filter((p) => p.synergy_score > 0.3).length;
+
+  return {
+    pairs,
+    sequences,
+    workflow_candidates: workflowCandidates,
+    synergy_count: synergyCount,
+    total_sessions_analyzed: skillSessions.length,
+    conflict_count: pairs.filter((p) => p.conflict_detected).length,
+    generated_at: new Date().toISOString(),
+  };
+}