selftune 0.2.22 → 0.2.24

package/cli/selftune/evolution/apply-proposal.ts

@@ -0,0 +1,295 @@
+/**
+ * apply-proposal.ts
+ *
+ * Fetches an approved contributor proposal from the cloud API, applies the
+ * proposed update to the local SKILL.md, and marks the proposal as applied.
+ *
+ * Usage:
+ *   selftune evolve apply-proposal --id <proposal-id> --skill-path <path>
+ */
+
+import { copyFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
+import { parseArgs } from "node:util";
+
+import { readAlphaIdentity } from "../alpha-identity.js";
+import { SELFTUNE_CONFIG_PATH } from "../constants.js";
+import { CLIError, handleCLIError } from "../utils/cli-error.js";
+import { replaceDescription } from "../utils/frontmatter.js";
+import { getSelftuneVersion } from "../utils/selftune-meta.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+interface ProposalRecord {
+  id: string;
+  skill_id: string;
+  skill_name: string;
+  proposal_type: string;
+  current_value: string;
+  proposed_value: string;
+  reason: string | null;
+  pass_rate_before: number | null;
+  projected_pass_rate: number | null;
+  status: "pending" | "approved" | "rejected" | "applied";
+  proposed_by: string;
+  reviewed_by: string | null;
+  reviewed_at: string | null;
+  applied_at: string | null;
+  created_at: string;
+}
+
+// ---------------------------------------------------------------------------
+// Cloud API helpers (follows registry/client.ts pattern)
+// ---------------------------------------------------------------------------
+
+function getCloudConfig(): { apiUrl: string; apiKey: string } | null {
+  try {
+    const identity = readAlphaIdentity(SELFTUNE_CONFIG_PATH);
+    if (!identity?.api_key) return null;
+    const apiUrl = identity.cloud_api_url || "https://api.selftune.dev";
+    return { apiUrl, apiKey: identity.api_key };
+  } catch {
+    return null;
+  }
+}
+
+async function fetchProposal(
+  proposalId: string,
+  config: { apiUrl: string; apiKey: string },
+): Promise<ProposalRecord> {
+  const url = `${config.apiUrl}/api/v1/proposals/${encodeURIComponent(proposalId)}`;
+  const response = await fetch(url, {
+    method: "GET",
+    headers: {
+      Authorization: `Bearer ${config.apiKey}`,
+      "User-Agent": `selftune/${getSelftuneVersion()}`,
+      Accept: "application/json",
+    },
+    signal: AbortSignal.timeout(15_000),
+  });
+
+  if (!response.ok) {
+    const text = await response.text().catch(() => "unknown error");
+    if (response.status === 404) {
+      throw new CLIError(
+        `Proposal ${proposalId} not found.`,
+        "NOT_FOUND",
+        "Check the proposal ID and try again.",
+      );
+    }
+    throw new CLIError(
+      `Failed to fetch proposal: HTTP ${response.status}: ${text.slice(0, 200)}`,
+      "API_ERROR",
+      "Check your credentials and network connection.",
+    );
+  }
+
+  const body = (await response.json()) as { proposal: ProposalRecord };
+  return body.proposal;
+}
+
+async function markProposalApplied(
+  proposalId: string,
+  config: { apiUrl: string; apiKey: string },
+): Promise<boolean> {
+  const url = `${config.apiUrl}/api/v1/proposals/${encodeURIComponent(proposalId)}`;
+
+  try {
+    const response = await fetch(url, {
+      method: "PATCH",
+      headers: {
+        Authorization: `Bearer ${config.apiKey}`,
+        "User-Agent": `selftune/${getSelftuneVersion()}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({ status: "applied" }),
+      signal: AbortSignal.timeout(15_000),
+    });
+
+    if (!response.ok) {
+      const text = await response.text().catch(() => "unknown error");
+      console.error(
+        `Warning: Failed to mark proposal as applied: HTTP ${response.status}: ${text.slice(0, 200)}`,
+      );
+      return false;
+    }
+
+    return true;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(`Warning: Failed to mark proposal as applied: ${message}`);
+    return false;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Apply logic
+// ---------------------------------------------------------------------------
+
+function applyProposalToSkill(skillPath: string, proposal: ProposalRecord): { backupPath: string } {
+  if (!existsSync(skillPath)) {
+    throw new CLIError(
+      `Skill file not found: ${skillPath}`,
+      "FILE_NOT_FOUND",
+      "Verify the --skill-path argument points to your SKILL.md.",
+    );
+  }
+
+  const content = readFileSync(skillPath, "utf-8");
+
+  // Back up before modifying
+  const backupPath = `${skillPath}.bak`;
+  copyFileSync(skillPath, backupPath);
+
+  let updated: string;
+  if (proposal.proposal_type === "description") {
+    updated = replaceDescription(content, proposal.proposed_value);
+  } else if (proposal.proposal_type === "body") {
+    const lines = content.split("\n");
+    let endIdx = -1;
+    if (lines[0]?.trim() === "---") {
+      for (let i = 1; i < lines.length; i++) {
+        if (lines[i].trim() === "---") {
+          endIdx = i;
+          break;
+        }
+      }
+    }
+    if (endIdx >= 0) {
+      updated = lines.slice(0, endIdx + 1).join("\n") + "\n\n" + proposal.proposed_value;
+    } else {
+      // No frontmatter -- replace entire content
+      updated = proposal.proposed_value;
+    }
+  } else {
+    throw new CLIError(
+      `Unsupported proposal type: ${proposal.proposal_type}`,
+      "UNSUPPORTED_TYPE",
+      "Only 'description' and 'body' proposal types can be applied.",
+    );
+  }
+
+  writeFileSync(skillPath, updated, "utf-8");
+  return { backupPath };
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry point
+// ---------------------------------------------------------------------------
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      id: { type: "string" },
+      "skill-path": { type: "string" },
+      "dry-run": { type: "boolean", default: false },
+      help: { type: "boolean", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    console.log(`selftune evolve apply-proposal -- Apply an approved contributor proposal
+
+Usage:
+  selftune evolve apply-proposal --id <proposal-id> --skill-path <path> [options]
+
+Options:
+  --id            Proposal UUID (required)
+  --skill-path    Path to the target SKILL.md (required)
+  --dry-run       Preview the proposal without applying
+  --help          Show this help message
+
+The proposal must be proposed by "contributor_aggregate" and have status
+"approved". The command fetches the proposal from the cloud API, applies
+the proposed change to the local SKILL.md, and marks the proposal as applied.`);
+    process.exit(0);
+  }
+
+  if (!values.id) {
+    throw new CLIError(
+      "--id is required",
+      "MISSING_FLAG",
+      "selftune evolve apply-proposal --id <proposal-id> --skill-path <path>",
+    );
+  }
+  if (!values["skill-path"]) {
+    throw new CLIError(
+      "--skill-path is required",
+      "MISSING_FLAG",
+      "selftune evolve apply-proposal --id <proposal-id> --skill-path <path>",
+    );
+  }
+
+  const proposalId = values.id;
+  const skillPath = values["skill-path"];
+  const dryRun = values["dry-run"] ?? false;
+
+  try {
+    // Resolve cloud config once for both fetch and mark calls
+    const config = getCloudConfig();
+    if (!config) {
+      throw new CLIError(
+        "Not authenticated. Run 'selftune init' to set up cloud credentials.",
+        "AUTH_MISSING",
+        "selftune init",
+      );
+    }
+
+    // 1. Fetch the proposal from the cloud API
+    console.log(`Fetching proposal ${proposalId}...`);
+    const proposal = await fetchProposal(proposalId, config);
+
+    // 2. Validate the proposal
+    if (proposal.proposed_by !== "contributor_aggregate") {
+      throw new CLIError(
+        `Proposal was proposed by "${proposal.proposed_by}", not "contributor_aggregate".`,
+        "INVALID_PROPOSAL",
+        "Only contributor aggregate proposals can be applied via this command.",
+      );
+    }
+
+    if (proposal.status !== "approved") {
+      throw new CLIError(
+        `Proposal status is "${proposal.status}", expected "approved".`,
+        "INVALID_STATUS",
+        "Approve the proposal in the dashboard first, then apply it.",
+      );
+    }
+
+    // 3. Print proposal summary
+    console.log(`\nProposal: ${proposal.id}`);
+    console.log(`  Skill:         ${proposal.skill_name}`);
+    console.log(`  Type:          ${proposal.proposal_type}`);
+    console.log(`  Proposed by:   ${proposal.proposed_by}`);
+    console.log(`  Reason:        ${proposal.reason ?? "(none)"}`);
+    if (proposal.pass_rate_before != null) {
+      console.log(
+        `  Pass rate:     ${(proposal.pass_rate_before * 100).toFixed(1)}% -> ${proposal.projected_pass_rate != null ? (proposal.projected_pass_rate * 100).toFixed(1) + "%" : "?"}`,
+      );
+    }
+    console.log(`\n--- Current Value ---`);
+    console.log(proposal.current_value.slice(0, 500));
+    console.log(`\n--- Proposed Value ---`);
+    console.log(proposal.proposed_value.slice(0, 500));
+
+    if (dryRun) {
+      console.log("\n[dry-run] No changes written.");
+      return;
+    }
+
+    // 4. Apply the proposal to the local SKILL.md
+    const { backupPath } = applyProposalToSkill(skillPath, proposal);
+    console.log(`\nApplied proposal to ${skillPath}`);
+    console.log(`Backup saved to ${backupPath}`);
+
+    // 5. Mark the proposal as applied in the cloud
+    const markedApplied = await markProposalApplied(proposalId, config);
+    if (markedApplied) {
+      console.log(`Proposal ${proposalId} marked as applied.`);
+    }
+  } catch (err) {
+    handleCLIError(err);
+  }
+}

package/cli/selftune/evolution/engines/judge-engine.ts

@@ -0,0 +1,96 @@
+/**
+ * judge-engine.ts
+ *
+ * LLM judge validation engine: runs trigger accuracy checks using
+ * an LLM as a YES/NO judge for each eval entry.
+ *
+ * Extracted from validate-routing.ts and validate-body.ts to isolate
+ * LLM-judge-specific concerns from replay-specific concerns.
+ */
+
+import type { EvalEntry, ValidationMode } from "../../types.js";
+import { callLlm } from "../../utils/llm-call.js";
+import { buildTriggerCheckPrompt, parseTriggerResponse } from "../../utils/trigger-check.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface JudgeValidationResult {
+  before_pass_rate: number;
+  after_pass_rate: number;
+  improved: boolean;
+  regressions: string[];
+  validation_mode: ValidationMode;
+  validation_agent: string;
+}
+
+// ---------------------------------------------------------------------------
+// Judge validation engine
+// ---------------------------------------------------------------------------
+
+/**
+ * Run LLM-judge-based trigger accuracy checks on an eval set.
+ * For each entry, asks the LLM whether the content would trigger
+ * the skill for the given query, comparing original vs proposed.
+ */
+export async function runJudgeValidation(
+  originalContent: string,
+  proposedContent: string,
+  evalSet: EvalEntry[],
+  agent: string,
+  modelFlag?: string,
+): Promise<JudgeValidationResult> {
+  if (evalSet.length === 0) {
+    return {
+      before_pass_rate: 0,
+      after_pass_rate: 0,
+      improved: false,
+      regressions: [],
+      validation_mode: "llm_judge",
+      validation_agent: agent,
+    };
+  }
+
+  const systemPrompt = "You are an evaluation assistant. Answer only YES or NO.";
+  let beforePassed = 0;
+  let afterPassed = 0;
+  const regressions: string[] = [];
+
+  for (const entry of evalSet) {
+    // Check with original content
+    const beforePrompt = buildTriggerCheckPrompt(originalContent, entry.query);
+    const beforeRaw = await callLlm(systemPrompt, beforePrompt, agent, modelFlag);
+    const beforeTriggered = parseTriggerResponse(beforeRaw);
+    const beforePass =
+      (entry.should_trigger && beforeTriggered) || (!entry.should_trigger && !beforeTriggered);
+
+    // Check with proposed content
+    const afterPrompt = buildTriggerCheckPrompt(proposedContent, entry.query);
+    const afterRaw = await callLlm(systemPrompt, afterPrompt, agent, modelFlag);
+    const afterTriggered = parseTriggerResponse(afterRaw);
+    const afterPass =
+      (entry.should_trigger && afterTriggered) || (!entry.should_trigger && !afterTriggered);
+
+    if (beforePass) beforePassed++;
+    if (afterPass) afterPassed++;
+
+    // Track regressions
+    if (beforePass && !afterPass) {
+      regressions.push(entry.query);
+    }
+  }
+
+  const total = evalSet.length;
+  const beforePassRate = beforePassed / total;
+  const afterPassRate = afterPassed / total;
+
+  return {
+    before_pass_rate: beforePassRate,
+    after_pass_rate: afterPassRate,
+    improved: afterPassRate > beforePassRate,
+    regressions,
+    validation_mode: "llm_judge",
+    validation_agent: agent,
+  };
+}

package/cli/selftune/evolution/engines/replay-engine.ts

@@ -0,0 +1,180 @@
+/**
+ * replay-engine.ts
+ *
+ * Cohesive module for all replay-based validation logic:
+ *   - Host/runtime replay (PRIMARY path — real agent routing decisions)
+ *   - Custom replay runner support
+ *
+ * Host/runtime replay is preferred because it captures actual agent routing
+ * behavior. If the runtime path is unavailable or fails, callers must fall
+ * back explicitly to another validation mode instead of treating simulated
+ * fixture matching as equivalent replay evidence.
+ *
+ * Extracted from validate-routing.ts and validate-body.ts to isolate
+ * replay-specific concerns from judge-specific concerns.
+ */
+
+import type {
+  EvalEntry,
+  RoutingReplayEntryResult,
+  RoutingReplayFixture,
+  ValidationMode,
+} from "../../types.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ReplayRunnerInput {
+  routing: string;
+  evalSet: EvalEntry[];
+  agent: string;
+  fixture: RoutingReplayFixture;
+}
+
+export type ReplayRunner = (input: ReplayRunnerInput) => Promise<RoutingReplayEntryResult[]>;
+
+export interface ReplayValidationOptions {
+  replayFixture?: RoutingReplayFixture;
+  /** Host/runtime replay runner — PRIMARY validation path when provided. */
+  replayRunner?: ReplayRunner;
+}
+
+export interface ReplayValidationResult {
+  before_pass_rate: number;
+  after_pass_rate: number;
+  improved: boolean;
+  validation_mode: ValidationMode;
+  validation_agent: string;
+  validation_fixture_id?: string;
+  per_entry_results?: RoutingReplayEntryResult[];
+  /** Before-phase per-entry results for structured persistence. */
+  before_entry_results?: RoutingReplayEntryResult[];
+}
+
+export interface ReplayValidationAttempt {
+  result: ReplayValidationResult | null;
+  fallbackReason?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+function computeReplayResult(
+  beforeResults: RoutingReplayEntryResult[],
+  afterResults: RoutingReplayEntryResult[],
+  total: number,
+  mode: ValidationMode,
+  agent: string,
+  fixtureId: string,
+): ReplayValidationResult {
+  const beforePassed = beforeResults.filter((result) => result.passed).length;
+  const afterPassed = afterResults.filter((result) => result.passed).length;
+  const beforePassRate = beforePassed / total;
+  const afterPassRate = afterPassed / total;
+  const netChange = afterPassRate - beforePassRate;
+  const beforePassedByQuery = new Map<string, boolean>();
+  let regressionCount = 0;
+  let newPassCount = 0;
+
+  for (const result of beforeResults) {
+    beforePassedByQuery.set(result.query, result.passed);
+  }
+
+  for (const result of afterResults) {
+    const beforePass = beforePassedByQuery.get(result.query) ?? false;
+    const afterPass = result.passed;
+    if (beforePass && !afterPass) regressionCount++;
+    if (!beforePass && afterPass) newPassCount++;
+  }
+
+  return {
+    before_pass_rate: beforePassRate,
+    after_pass_rate: afterPassRate,
+    improved:
+      afterPassRate > beforePassRate &&
+      regressionCount < total * 0.05 &&
+      (netChange >= 0.1 || newPassCount >= 2),
+    validation_mode: mode,
+    validation_agent: agent,
+    validation_fixture_id: fixtureId,
+    per_entry_results: afterResults,
+    before_entry_results: beforeResults,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Replay validation engine
+// ---------------------------------------------------------------------------
+
+/**
+ * Attempt replay-backed validation using a real host/runtime runner.
+ *
+ * Returns a null result with a fallback reason when runtime replay is
+ * unavailable or fails. Callers decide whether to fall back to a judge-based
+ * validator (`auto`) or surface an explicit unavailable error (`replay`).
+ */
+export async function runReplayValidation(
+  originalContent: string,
+  proposedContent: string,
+  evalSet: EvalEntry[],
+  agent: string,
+  options: ReplayValidationOptions = {},
+): Promise<ReplayValidationAttempt> {
+  if (evalSet.length === 0) {
+    return { result: null };
+  }
+
+  if (!options.replayFixture) {
+    return {
+      result: null,
+      fallbackReason: "no replay fixture is available for runtime validation",
+    };
+  }
+
+  if (!options.replayRunner) {
+    return {
+      result: null,
+      fallbackReason: "no real host/runtime replay runner is configured",
+    };
+  }
+
+  const fixture = options.replayFixture;
+  const total = evalSet.length;
+
+  try {
+    const beforeResults = await options.replayRunner({
+      routing: originalContent,
+      evalSet,
+      agent,
+      fixture,
+    });
+    const afterResults = await options.replayRunner({
+      routing: proposedContent,
+      evalSet,
+      agent,
+      fixture,
+    });
+
+    return {
+      result: computeReplayResult(
+        beforeResults,
+        afterResults,
+        total,
+        "host_replay",
+        agent,
+        fixture.fixture_id,
+      ),
+    };
+  } catch (error) {
+    const message =
+      error instanceof Error && error.message.trim()
+        ? error.message.trim()
+        : "runtime replay failed before producing a routing decision";
+    return {
+      result: null,
+      fallbackReason: `real host/runtime replay failed: ${message}`,
+    };
+  }
+}

package/cli/selftune/evolution/evidence.ts

@@ -12,11 +12,7 @@ import { queryEvolutionEvidence } from "../localdb/queries.js";
 import type { EvolutionEvidenceEntry } from "../types.js";
 
 /** Append a structured evidence artifact to the evolution evidence log (SQLite). */
-export function appendEvidenceEntry(
-  entry: EvolutionEvidenceEntry,
-  /** @deprecated Unused; retained for API compatibility during migration */
-  _logPath?: string,
-): void {
+export function appendEvidenceEntry(entry: EvolutionEvidenceEntry): void {
   writeEvolutionEvidenceToDb(entry);
 }
 
@@ -25,7 +21,7 @@ export function appendEvidenceEntry(
  *
  * @param skillName - Optional skill name to filter by
  */
-export function readEvidenceTrail(skillName?: string, _logPath?: string): EvolutionEvidenceEntry[] {
+export function readEvidenceTrail(skillName?: string): EvolutionEvidenceEntry[] {
   const db = getDb();
   return queryEvolutionEvidence(db, skillName) as EvolutionEvidenceEntry[];
 }