selftune 0.1.2 → 0.2.0

package/cli/selftune/memory/writer.ts

@@ -0,0 +1,447 @@
+/**
+ * Memory writer — pure functions for reading/writing evolution memory files.
+ *
+ * Memory files live at ~/.selftune/memory/ and provide human-readable session
+ * context that survives context resets. Three files:
+ *   - context.md  — active evolutions, known issues
+ *   - plan.md     — current priorities, strategy
+ *   - decisions.md — append-only decision log
+ *
+ * All functions accept an optional memoryDir parameter for testability.
+ * Default: MEMORY_DIR from constants.
+ */
+
+import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { MEMORY_DIR } from "../constants.js";
+import type { EvolveResult } from "../evolution/evolve.js";
+import type { RollbackResult } from "../evolution/rollback.js";
+import type {
+  DecisionRecord,
+  EvolutionProposal,
+  MemoryContext,
+  MemoryPlan,
+  MonitoringSnapshot,
+} from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Directory management
+// ---------------------------------------------------------------------------
+
+export function ensureMemoryDir(memoryDir: string = MEMORY_DIR): void {
+  if (!existsSync(memoryDir)) {
+    mkdirSync(memoryDir, { recursive: true });
+  }
+}
+
+// ---------------------------------------------------------------------------
+// context.md
+// ---------------------------------------------------------------------------
+
+function formatContext(data: MemoryContext): string {
+  const lines: string[] = ["# Selftune Context", ""];
+
+  lines.push("## Active Evolutions");
+  if (data.activeEvolutions.length === 0) {
+    lines.push("- (none)");
+  } else {
+    for (const evo of data.activeEvolutions) {
+      lines.push(`- ${evo.skillName}: ${evo.status} — ${evo.description}`);
+    }
+  }
+  lines.push("");
+
+  lines.push("## Known Issues");
+  if (data.knownIssues.length === 0) {
+    lines.push("- (none)");
+  } else {
+    for (const issue of data.knownIssues) {
+      lines.push(`- ${issue}`);
+    }
+  }
+  lines.push("");
+
+  lines.push("## Last Updated");
+  lines.push(data.lastUpdated);
+  lines.push("");
+
+  return lines.join("\n");
+}
+
+function parseContext(content: string): MemoryContext {
+  const result: MemoryContext = {
+    activeEvolutions: [],
+    knownIssues: [],
+    lastUpdated: "",
+  };
+
+  const lines = content.split("\n");
+  let section = "";
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+
+    if (trimmed === "## Active Evolutions") {
+      section = "evolutions";
+      continue;
+    }
+    if (trimmed === "## Known Issues") {
+      section = "issues";
+      continue;
+    }
+    if (trimmed === "## Last Updated") {
+      section = "updated";
+      continue;
+    }
+    if (trimmed.startsWith("# ")) {
+      section = "";
+      continue;
+    }
+
+    if (section === "evolutions" && trimmed.startsWith("- ") && trimmed !== "- (none)") {
+      // Format: "- skillName: status — description"
+      const body = trimmed.slice(2);
+      const colonIdx = body.indexOf(":");
+      if (colonIdx === -1) continue;
+      const skillName = body.slice(0, colonIdx).trim();
+      const rest = body.slice(colonIdx + 1).trim();
+      const dashIdx = rest.indexOf("—");
+      if (dashIdx === -1) {
+        result.activeEvolutions.push({ skillName, status: rest.trim(), description: "" });
+      } else {
+        const status = rest.slice(0, dashIdx).trim();
+        const description = rest.slice(dashIdx + 1).trim();
+        result.activeEvolutions.push({ skillName, status, description });
+      }
+    }
+
+    if (section === "issues" && trimmed.startsWith("- ") && trimmed !== "- (none)") {
+      result.knownIssues.push(trimmed.slice(2));
+    }
+
+    if (section === "updated" && trimmed.length > 0) {
+      result.lastUpdated = trimmed;
+      section = "";
+    }
+  }
+
+  return result;
+}
+
+export function writeContext(data: MemoryContext, memoryDir: string = MEMORY_DIR): void {
+  ensureMemoryDir(memoryDir);
+  const filePath = join(memoryDir, "context.md");
+  writeFileSync(filePath, formatContext(data), "utf-8");
+}
+
+export function readContext(memoryDir: string = MEMORY_DIR): MemoryContext {
+  const filePath = join(memoryDir, "context.md");
+  if (!existsSync(filePath)) {
+    return { activeEvolutions: [], knownIssues: [], lastUpdated: "" };
+  }
+  const content = readFileSync(filePath, "utf-8");
+  return parseContext(content);
+}
+
+// ---------------------------------------------------------------------------
+// plan.md
+// ---------------------------------------------------------------------------
+
+function formatPlan(data: MemoryPlan): string {
+  const lines: string[] = ["# Evolution Plan", ""];
+
+  lines.push("## Current Priorities");
+  if (data.currentPriorities.length === 0) {
+    lines.push("1. (none)");
+  } else {
+    for (let i = 0; i < data.currentPriorities.length; i++) {
+      lines.push(`${i + 1}. ${data.currentPriorities[i]}`);
+    }
+  }
+  lines.push("");
+
+  lines.push("## Strategy");
+  lines.push(data.strategy || "(no strategy defined)");
+  lines.push("");
+
+  lines.push("## Last Updated");
+  lines.push(data.lastUpdated);
+  lines.push("");
+
+  return lines.join("\n");
+}
+
+function parsePlan(content: string): MemoryPlan {
+  const result: MemoryPlan = {
+    currentPriorities: [],
+    strategy: "",
+    lastUpdated: "",
+  };
+
+  const lines = content.split("\n");
+  let section = "";
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+
+    if (trimmed === "## Current Priorities") {
+      section = "priorities";
+      continue;
+    }
+    if (trimmed === "## Strategy") {
+      section = "strategy";
+      continue;
+    }
+    if (trimmed === "## Last Updated") {
+      section = "updated";
+      continue;
+    }
+    if (trimmed.startsWith("# ")) {
+      section = "";
+      continue;
+    }
+
+    if (section === "priorities") {
+      // Format: "1. priority text"
+      const match = trimmed.match(/^\d+\.\s+(.+)$/);
+      if (match && match[1] !== "(none)") {
+        result.currentPriorities.push(match[1]);
+      }
+    }
+
+    // Intentionally captures only the first non-empty line as the strategy for simplicity
+    if (section === "strategy" && trimmed.length > 0 && trimmed !== "(no strategy defined)") {
+      result.strategy = trimmed;
+    }
+
+    if (section === "updated" && trimmed.length > 0) {
+      result.lastUpdated = trimmed;
+      section = "";
+    }
+  }
+
+  return result;
+}
+
+export function writePlan(data: MemoryPlan, memoryDir: string = MEMORY_DIR): void {
+  ensureMemoryDir(memoryDir);
+  const filePath = join(memoryDir, "plan.md");
+  writeFileSync(filePath, formatPlan(data), "utf-8");
+}
+
+export function readPlan(memoryDir: string = MEMORY_DIR): MemoryPlan {
+  const filePath = join(memoryDir, "plan.md");
+  if (!existsSync(filePath)) {
+    return { currentPriorities: [], strategy: "", lastUpdated: "" };
+  }
+  const content = readFileSync(filePath, "utf-8");
+  return parsePlan(content);
+}
+
+// ---------------------------------------------------------------------------
+// decisions.md (append-only)
+// ---------------------------------------------------------------------------
+
+function formatDecisionEntry(record: DecisionRecord): string {
+  const lines: string[] = [
+    `## ${record.timestamp} — ${record.actionType}`,
+    `- **Skill:** ${record.skillName}`,
+    `- **Action:** ${record.action}`,
+    `- **Rationale:** ${record.rationale}`,
+    `- **Result:** ${record.result}`,
+    "",
+    "---",
+    "",
+  ];
+  return lines.join("\n");
+}
+
+function parseDecisions(content: string): DecisionRecord[] {
+  const records: DecisionRecord[] = [];
+  // Split on --- separators
+  const blocks = content.split(/^---$/m);
+
+  for (const block of blocks) {
+    const lines = block
+      .split("\n")
+      .map((l) => l.trim())
+      .filter((l) => l.length > 0);
+
+    let timestamp = "";
+    let actionType = "";
+    let skillName = "";
+    let action: DecisionRecord["action"] = "watched";
+    let rationale = "";
+    let result = "";
+
+    for (const line of lines) {
+      // Header: "## 2026-03-01T00:00:00Z — evolve"
+      const headerMatch = line.match(/^## (.+?) — (.+)$/);
+      if (headerMatch) {
+        timestamp = headerMatch[1];
+        actionType = headerMatch[2];
+        continue;
+      }
+
+      if (line.startsWith("- **Skill:**")) {
+        skillName = line.replace("- **Skill:**", "").trim();
+      } else if (line.startsWith("- **Action:**")) {
+        const raw = line.replace("- **Action:**", "").trim();
+        if (raw === "evolved" || raw === "rolled-back" || raw === "watched") {
+          action = raw;
+        }
+      } else if (line.startsWith("- **Rationale:**")) {
+        rationale = line.replace("- **Rationale:**", "").trim();
+      } else if (line.startsWith("- **Result:**")) {
+        result = line.replace("- **Result:**", "").trim();
+      }
+    }
+
+    if (timestamp && skillName) {
+      records.push({ timestamp, actionType, skillName, action, rationale, result });
+    }
+  }
+
+  return records;
+}
+
+export function appendDecision(record: DecisionRecord, memoryDir: string = MEMORY_DIR): void {
+  ensureMemoryDir(memoryDir);
+  const filePath = join(memoryDir, "decisions.md");
+
+  if (!existsSync(filePath)) {
+    writeFileSync(filePath, "# Decision Log\n\n", "utf-8");
+  }
+
+  const entry = formatDecisionEntry(record);
+  appendFileSync(filePath, entry, "utf-8");
+}
+
+export function readDecisions(memoryDir: string = MEMORY_DIR): DecisionRecord[] {
+  const filePath = join(memoryDir, "decisions.md");
+  if (!existsSync(filePath)) {
+    return [];
+  }
+  const content = readFileSync(filePath, "utf-8");
+  return parseDecisions(content);
+}
+
+// ---------------------------------------------------------------------------
+// High-level helpers for integration
+// ---------------------------------------------------------------------------
+
+export function updateContextAfterEvolve(
+  skillName: string,
+  proposal: EvolutionProposal,
+  result: EvolveResult,
+  memoryDir: string = MEMORY_DIR,
+): void {
+  const now = new Date().toISOString();
+  const context = readContext(memoryDir);
+
+  const status = result.deployed ? "deployed" : "failed";
+  const description = proposal.rationale || result.reason;
+
+  // Update or add the evolution entry
+  const idx = context.activeEvolutions.findIndex((e) => e.skillName === skillName);
+  if (idx >= 0) {
+    context.activeEvolutions[idx] = { skillName, status, description };
+  } else {
+    context.activeEvolutions.push({ skillName, status, description });
+  }
+
+  context.lastUpdated = now;
+  writeContext(context, memoryDir);
+
+  // Append decision
+  appendDecision(
+    {
+      timestamp: now,
+      actionType: "evolve",
+      skillName,
+      action: "evolved",
+      rationale: proposal.rationale || "Evolution triggered",
+      result: result.reason,
+    },
+    memoryDir,
+  );
+}
+
+export function updateContextAfterRollback(
+  skillName: string,
+  result: RollbackResult,
+  memoryDir: string = MEMORY_DIR,
+): void {
+  const now = new Date().toISOString();
+  const context = readContext(memoryDir);
+
+  const status = result.rolledBack ? "rolled-back" : "rollback-failed";
+  const description = result.reason;
+
+  const idx = context.activeEvolutions.findIndex((e) => e.skillName === skillName);
+  if (idx >= 0) {
+    context.activeEvolutions[idx] = { skillName, status, description };
+  } else {
+    context.activeEvolutions.push({ skillName, status, description });
+  }
+
+  context.lastUpdated = now;
+  writeContext(context, memoryDir);
+
+  appendDecision(
+    {
+      timestamp: now,
+      actionType: "rollback",
+      skillName,
+      action: "rolled-back",
+      rationale: result.reason,
+      result: result.rolledBack ? "Successfully rolled back" : "Rollback failed",
+    },
+    memoryDir,
+  );
+}
+
+export function updateContextAfterWatch(
+  skillName: string,
+  snapshot: MonitoringSnapshot,
+  memoryDir: string = MEMORY_DIR,
+): void {
+  const now = new Date().toISOString();
+  const context = readContext(memoryDir);
+
+  const status = snapshot.regression_detected ? "regression" : "healthy";
+  const description = `pass_rate=${snapshot.pass_rate.toFixed(2)}, baseline=${snapshot.baseline_pass_rate.toFixed(2)}`;
+
+  const idx = context.activeEvolutions.findIndex((e) => e.skillName === skillName);
+  if (idx >= 0) {
+    context.activeEvolutions[idx] = { skillName, status, description };
+  } else {
+    context.activeEvolutions.push({ skillName, status, description });
+  }
+
+  // Add known issue if regression detected
+  if (snapshot.regression_detected) {
+    const issue = `Regression detected for ${skillName}: pass_rate=${snapshot.pass_rate.toFixed(2)} below baseline=${snapshot.baseline_pass_rate.toFixed(2)}`;
+    if (!context.knownIssues.some((i) => i.includes(skillName) && i.includes("Regression"))) {
+      context.knownIssues.push(issue);
+    }
+  }
+
+  context.lastUpdated = now;
+  writeContext(context, memoryDir);
+
+  appendDecision(
+    {
+      timestamp: now,
+      actionType: "watch",
+      skillName,
+      action: "watched",
+      rationale: `Monitoring check: pass_rate=${snapshot.pass_rate.toFixed(2)}, regression=${snapshot.regression_detected}`,
+      result: snapshot.regression_detected
+        ? `Regression detected (pass_rate=${snapshot.pass_rate.toFixed(2)})`
+        : `Healthy (pass_rate=${snapshot.pass_rate.toFixed(2)})`,
+    },
+    memoryDir,
+  );
+}

package/cli/selftune/monitoring/watch.ts

@@ -9,7 +9,9 @@
 import { parseArgs } from "node:util";
 
 import { QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "../constants.js";
+import { classifyInvocation } from "../eval/hooks-to-evals.js";
 import { getLastDeployedProposal } from "../evolution/audit.js";
+import { updateContextAfterWatch } from "../memory/writer.js";
 import type {
   InvocationType,
   MonitoringSnapshot,
@@ -116,13 +118,20 @@ export function computeMonitoringSnapshot(
   const falseNegatives = filteredSkillRecords.filter((r) => !r.triggered).length;
   const falseNegativeRate = totalSkillChecks === 0 ? 0 : falseNegatives / totalSkillChecks;
 
-  // 6. by_invocation_type: MVP classifies everything as "implicit"
+  // 6. by_invocation_type: classify each skill record using classifyInvocation
   const byInvocationType: Record<InvocationType, { passed: number; total: number }> = {
     explicit: { passed: 0, total: 0 },
-    implicit: { passed: triggeredCount, total: totalSkillChecks },
+    implicit: { passed: 0, total: 0 },
     contextual: { passed: 0, total: 0 },
     negative: { passed: 0, total: 0 },
   };
+  for (const record of filteredSkillRecords) {
+    const invType = classifyInvocation(record.query, skillName);
+    byInvocationType[invType].total++;
+    if (record.triggered) {
+      byInvocationType[invType].passed++;
+    }
+  }
 
   // 7. Regression detection: pass_rate < baseline - threshold
   // Use rounding to avoid floating-point boundary issues (e.g. 0.8 - 0.1 = 0.7000000000000001)
@@ -212,6 +221,20 @@ export async function watch(options: WatchOptions): Promise<WatchResult> {
     recommendation = `Skill "${skillName}" is stable. Pass rate ${snapshot.pass_rate.toFixed(2)} is within acceptable range of baseline ${baselinePassRate.toFixed(2)}.`;
   }
 
+  // Update evolution memory (fail-open)
+  try {
+    updateContextAfterWatch(skillName, snapshot);
+  } catch (err) {
+    // Fail-open: memory writes should never fail the main operation
+    console.error(
+      JSON.stringify({
+        level: "debug",
+        code: "memory_write_failed",
+        message: `Failed to update memory after watch for "${skillName}": ${err instanceof Error ? err.message : String(err)}`,
+      }),
+    );
+  }
+
   return {
     snapshot,
     alert,

package/cli/selftune/status.ts

@@ -8,7 +8,6 @@
  */
 
 import { EVOLUTION_AUDIT_LOG, QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "./constants.js";
-import { getLastDeployedProposal } from "./evolution/audit.js";
 import { computeMonitoringSnapshot } from "./monitoring/watch.js";
 import { doctor } from "./observability.js";
 import type {
@@ -30,7 +29,7 @@ export interface SkillStatus {
   passRate: number | null;
   trend: "up" | "down" | "stable" | "unknown";
   missedQueries: number;
-  status: "HEALTHY" | "REGRESSED" | "NO DATA";
+  status: "HEALTHY" | "WARNING" | "CRITICAL" | "UNKNOWN";
   snapshot: MonitoringSnapshot | null;
 }
 
@@ -103,12 +102,14 @@ export function computeStatus(
     // Count missed queries for this skill (queries where skill was checked but not triggered)
     const missedQueries = skillSpecificRecords.filter((r) => !r.triggered).length;
 
-    // Determine status
-    let status: "HEALTHY" | "REGRESSED" | "NO DATA";
+    // Determine status (4-state)
+    let status: "HEALTHY" | "WARNING" | "CRITICAL" | "UNKNOWN";
     if (!hasData || passRate === null) {
-      status = "NO DATA";
-    } else if (snapshot.regression_detected) {
-      status = "REGRESSED";
+      status = "UNKNOWN";
+    } else if (snapshot.regression_detected || passRate < 0.4) {
+      status = "CRITICAL";
+    } else if (passRate < 0.7) {
+      status = "WARNING";
     } else {
       status = "HEALTHY";
     }
@@ -116,8 +117,8 @@ export function computeStatus(
     return { name: skillName, passRate, trend, missedQueries, status, snapshot };
   });
 
-  // Sort: REGRESSED first, then HEALTHY, then NO DATA
-  const statusOrder = { REGRESSED: 0, HEALTHY: 1, "NO DATA": 2 };
+  // Sort: CRITICAL first, then WARNING, then HEALTHY, then UNKNOWN
+  const statusOrder: Record<string, number> = { CRITICAL: 0, WARNING: 1, HEALTHY: 2, UNKNOWN: 3 };
   skills.sort((a, b) => statusOrder[a.status] - statusOrder[b.status]);
 
   // Unmatched queries: queries whose text appears in zero triggered skill_usage_log entries
@@ -227,12 +228,12 @@ export function formatStatus(result: StatusResult): string {
   // Skills table
   const skillCount = result.skills.length;
   lines.push(
-    `Skills (${skillCount})${" ".repeat(36 - `Skills (${skillCount})`.length)}Last 7 days`,
+    `Skills (${skillCount})${" ".repeat(36 - `Skills (${skillCount})`.length)}Recent data`,
   );
   lines.push("  Name            Pass Rate  Trend  Missed  Status");
 
   for (const skill of result.skills) {
-    const name = skill.name.padEnd(16);
+    const name = skill.name.slice(0, 16).padEnd(16);
     const passRate =
       skill.passRate !== null
         ? `${Math.round(skill.passRate * 100)}%`.padEnd(11)
@@ -240,11 +241,13 @@ export function formatStatus(result: StatusResult): string {
     const trend = TREND_SYMBOLS[skill.trend].padEnd(7);
     const missed = String(skill.missedQueries).padEnd(8);
     const statusText =
-      skill.status === "REGRESSED"
+      skill.status === "CRITICAL"
         ? red(skill.status)
-        : skill.status === "HEALTHY"
-          ? green(skill.status)
-          : amber(skill.status);
+        : skill.status === "WARNING"
+          ? amber(skill.status)
+          : skill.status === "HEALTHY"
+            ? green(skill.status)
+            : amber(skill.status);
     lines.push(`  ${name}${passRate}${trend}${missed}${statusText}`);
   }