selftune 0.2.22 → 0.2.24

package/cli/selftune/orchestrate/prepare.ts

@@ -0,0 +1,334 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+
+import {
+  buildDefaultGradingOutputPath,
+  deriveExpectationsFromSkill,
+  gradeSession,
+  resolveLatestSessionForSkill,
+} from "../grading/grade-session.js";
+import { writeGradingResultToDb } from "../localdb/direct-write.js";
+import { createDefaultSyncOptions } from "../sync.js";
+import type {
+  ImprovementSignalRecord,
+  QueryLogRecord,
+  SessionTelemetryRecord,
+  SkillUsageRecord,
+} from "../types.js";
+import { readExcerpt } from "../utils/transcript.js";
+import type { OrchestrateOptions, SkillAction } from "../orchestrate.js";
+import { selectCandidates } from "./plan.js";
+import { groupSignalsBySkill, readPendingSignals } from "./signals.js";
+import type { ResolvedOrchestrateRuntime } from "./runtime.js";
+
+export interface PreparedOrchestrateRun {
+  syncResult: ReturnType<ResolvedOrchestrateRuntime["syncSources"]>;
+  statusResult: ReturnType<ResolvedOrchestrateRuntime["computeStatus"]>;
+  telemetry: SessionTelemetryRecord[];
+  skillRecords: SkillUsageRecord[];
+  pendingSignals: ImprovementSignalRecord[];
+  candidates: SkillAction[];
+  evolveCandidates: SkillAction[];
+  agent: string | null;
+  autoGradedCount: number;
+}
+
+/**
+ * Detects significant overlap between the positive eval sets of evolution
+ * candidates. When two skills share >30% of their positive queries, it
+ * suggests a routing boundary problem. Console-only — no persistence.
+ */
+export async function detectCrossSkillOverlap(
+  candidates: Array<{ skill: string }>,
+  skillRecords: SkillUsageRecord[],
+  queryRecords: QueryLogRecord[],
+): Promise<
+  Array<{ skill_a: string; skill_b: string; overlap_pct: number; shared_queries: string[] }>
+> {
+  if (candidates.length < 2) return [];
+
+  const { buildEvalSet } = await import("../eval/hooks-to-evals.js");
+
+  const evalSets = new Map<string, Set<string>>();
+
+  for (const candidate of candidates) {
+    const evalSet = buildEvalSet(skillRecords, queryRecords, candidate.skill);
+    const positives = new Set(
+      evalSet
+        .filter((entry: { should_trigger: boolean }) => entry.should_trigger)
+        .map((entry: { query: string }) => entry.query.toLowerCase()),
+    );
+    evalSets.set(candidate.skill, positives);
+  }
+
+  const overlaps: Array<{
+    skill_a: string;
+    skill_b: string;
+    overlap_pct: number;
+    shared_queries: string[];
+  }> = [];
+  const skillNames = [...evalSets.keys()];
+
+  for (let i = 0; i < skillNames.length; i++) {
+    for (let j = i + 1; j < skillNames.length; j++) {
+      const setA = evalSets.get(skillNames[i]);
+      const setB = evalSets.get(skillNames[j]);
+      if (!setA || !setB || setA.size === 0 || setB.size === 0) continue;
+
+      const shared: string[] = [];
+      for (const query of setA) {
+        if (setB.has(query)) shared.push(query);
+      }
+
+      const overlapPct = shared.length / Math.min(setA.size, setB.size);
+      if (overlapPct > 0.3) {
+        overlaps.push({
+          skill_a: skillNames[i],
+          skill_b: skillNames[j],
+          overlap_pct: overlapPct,
+          shared_queries: shared.slice(0, 10),
+        });
+      }
+    }
+  }
+
+  return overlaps;
+}
+
+/**
+ * Auto-grade the top ungraded skills that have some session data.
+ * Fail-open: individual grading errors are logged but never propagated.
+ */
+export async function autoGradeTopUngraded(
+  skills: Array<ReturnType<ResolvedOrchestrateRuntime["computeStatus"]>["skills"][number]>,
+  maxAutoGrade: number,
+  agent: string,
+  deps: {
+    readTelemetry: () => SessionTelemetryRecord[];
+    readSkillRecords: () => SkillUsageRecord[];
+  },
+): Promise<number> {
+  const ungradedWithData = skills
+    .filter((skill) => skill.status === "UNGRADED" && (skill.snapshot?.skill_checks ?? 0) > 0)
+    .sort((a, b) => (b.snapshot?.skill_checks ?? 0) - (a.snapshot?.skill_checks ?? 0))
+    .slice(0, maxAutoGrade);
+
+  if (ungradedWithData.length === 0) return 0;
+
+  // Cache data reads outside the loop — SQLite tables don't change during grading
+  // iterations, so re-reading per-skill is wasteful.
+  const cachedTelemetry = deps.readTelemetry();
+  const cachedSkillUsage = deps.readSkillRecords();
+
+  let graded = 0;
+
+  for (const skill of ungradedWithData) {
+    try {
+      const resolved = resolveLatestSessionForSkill(cachedTelemetry, cachedSkillUsage, skill.name);
+      if (!resolved) {
+        console.error(`  [auto-grade] ${skill.name}: no session found, skipping`);
+        continue;
+      }
+
+      const derived = deriveExpectationsFromSkill(skill.name);
+      let transcriptExcerpt = "(no transcript)";
+      if (resolved.transcriptPath) {
+        try {
+          transcriptExcerpt = readExcerpt(resolved.transcriptPath);
+        } catch {
+          transcriptExcerpt = "(no transcript)";
+        }
+      }
+
+      console.error(`  [auto-grade] Grading "${skill.name}" (session ${resolved.sessionId})...`);
+
+      const result = await gradeSession({
+        expectations: derived.expectations,
+        telemetry: resolved.telemetry,
+        sessionId: resolved.sessionId,
+        skillName: skill.name,
+        transcriptExcerpt,
+        transcriptPath: resolved.transcriptPath,
+        agent,
+      });
+
+      let persisted = false;
+      try {
+        persisted = writeGradingResultToDb(result);
+      } catch {
+        persisted = false;
+      }
+
+      if (!persisted) {
+        console.error(`  [auto-grade] ${skill.name}: graded but failed to persist result`);
+        continue;
+      }
+
+      try {
+        const basePath = buildDefaultGradingOutputPath(resolved.sessionId);
+        const safeName = skill.name.replace(/[^a-zA-Z0-9_-]/g, "_");
+        const outputPath = basePath.replace(/\.json$/, `_${safeName}.json`);
+        mkdirSync(dirname(outputPath), { recursive: true });
+        writeFileSync(outputPath, JSON.stringify(result, null, 2), "utf-8");
+      } catch {
+        // DB is authoritative; file output is supplementary only.
+      }
+
+      const passRate = result.summary.pass_rate;
+      console.error(
+        `  [auto-grade] ${skill.name}: ${result.summary.passed}/${result.summary.total} passed (${Math.round(passRate * 100)}%)`,
+      );
+      graded++;
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.error(
+        `  [auto-grade] ${skill.name}: error — ${msg}. Retry with: selftune grade ${skill.name}`,
+      );
+    }
+  }
+
+  return graded;
+}
+
+export async function prepareOrchestrateRun(
+  options: OrchestrateOptions,
+  runtime: ResolvedOrchestrateRuntime,
+): Promise<PreparedOrchestrateRun> {
+  console.error("[orchestrate] Syncing source-truth telemetry...");
+  const syncResult = runtime.syncSources(createDefaultSyncOptions({ force: options.syncForce }));
+  const sourceSynced = Object.values(syncResult.sources).reduce(
+    (sum, source) => sum + source.synced,
+    0,
+  );
+  console.error(
+    `[orchestrate] Sync complete: ${sourceSynced} sessions synced, ${syncResult.repair.repaired_records} repaired`,
+  );
+
+  console.error("[orchestrate] Computing skill status...");
+  const telemetry = runtime.readTelemetry();
+  const skillRecords = runtime.readSkillRecords();
+  const queryRecords = runtime.readQueryRecords();
+  const auditEntries = runtime.readAuditEntries();
+  const doctorResult = await runtime.doctor();
+
+  let statusResult = runtime.computeStatus(
+    telemetry,
+    skillRecords,
+    queryRecords,
+    auditEntries,
+    doctorResult,
+  );
+  console.error(
+    `[orchestrate] Status: ${statusResult.skills.length} skills, system=${statusResult.system.healthy ? "healthy" : "unhealthy"}`,
+  );
+
+  let autoGradedCount = 0;
+  const scopedSkills = options.skillFilter
+    ? statusResult.skills.filter((skill) => skill.name === options.skillFilter)
+    : statusResult.skills;
+  const ungradedWithData = scopedSkills.filter(
+    (skill) => skill.status === "UNGRADED" && (skill.snapshot?.skill_checks ?? 0) > 0,
+  );
+
+  if (!options.dryRun && options.maxAutoGrade > 0 && ungradedWithData.length > 0) {
+    const gradeAgent = runtime.detectAgent();
+    if (gradeAgent) {
+      console.error(
+        `[orchestrate] Auto-grading ${Math.min(ungradedWithData.length, options.maxAutoGrade)} ungraded skill(s)...`,
+      );
+      autoGradedCount = await autoGradeTopUngraded(scopedSkills, options.maxAutoGrade, gradeAgent, {
+        readTelemetry: runtime.readTelemetry,
+        readSkillRecords: runtime.readSkillRecords,
+      });
+
+      if (autoGradedCount > 0) {
+        console.error(
+          `[orchestrate] Recomputing status after grading ${autoGradedCount} skill(s)...`,
+        );
+        try {
+          // Re-read telemetry and skill records (grading writes new rows), but
+          // reuse queryRecords and auditEntries — auto-grading doesn't touch those tables.
+          statusResult = runtime.computeStatus(
+            runtime.readTelemetry(),
+            runtime.readSkillRecords(),
+            queryRecords,
+            auditEntries,
+            doctorResult,
+          );
+        } catch (recomputeErr) {
+          console.error(
+            `[orchestrate] Warning: failed to recompute status after grading — using pre-grade status. ${recomputeErr instanceof Error ? recomputeErr.message : String(recomputeErr)}`,
+          );
+        }
+      }
+    } else {
+      console.error(
+        "[orchestrate] No agent CLI found — skipping auto-grade. To disable, rerun with: selftune orchestrate --max-auto-grade 0",
+      );
+    }
+  }
+
+  const pendingSignals = readPendingSignals(runtime.readSignals);
+  const signaledSkills = groupSignalsBySkill(pendingSignals);
+  if (signaledSkills.size > 0) {
+    console.error(
+      `[orchestrate] Improvement signals: ${pendingSignals.length} pending for ${signaledSkills.size} skill(s)`,
+    );
+  }
+
+  const candidates = selectCandidates(statusResult.skills, {
+    skillFilter: options.skillFilter,
+    maxSkills: options.maxSkills,
+    auditEntries,
+    signaledSkills,
+  });
+
+  const evolveCandidates = candidates.filter((candidate) => candidate.action === "evolve");
+  const skipCount = candidates.filter((candidate) => candidate.action === "skip").length;
+  console.error(
+    `[orchestrate] Candidates: ${evolveCandidates.length} to evolve, ${skipCount} skipped`,
+  );
+  for (const candidate of candidates) {
+    console.error(
+      `  ${candidate.action === "skip" ? "⊘" : "→"} ${candidate.skill}: ${candidate.reason}`,
+    );
+  }
+
+  if (evolveCandidates.length >= 2) {
+    try {
+      const overlap = await detectCrossSkillOverlap(evolveCandidates, skillRecords, queryRecords);
+      if (overlap.length > 0) {
+        console.error("\n[orchestrate] Cross-skill eval overlap detected:");
+        for (const entry of overlap) {
+          console.error(
+            `  ⚠ ${entry.skill_a} ↔ ${entry.skill_b}: ${(entry.overlap_pct * 100).toFixed(0)}% shared queries (${entry.shared_queries.length} queries)`,
+          );
+        }
+        console.error("");
+      }
+    } catch {
+      // Overlap detection is informative only.
+    }
+  }
+
+  const agent = runtime.detectAgent();
+  if (!agent && evolveCandidates.length > 0) {
+    console.error("[orchestrate] WARNING: No agent CLI found in PATH. Evolve will be skipped.");
+    for (const candidate of evolveCandidates) {
+      candidate.action = "skip";
+      candidate.reason = "no agent CLI available";
+    }
+  }
+
+  return {
+    syncResult,
+    statusResult,
+    telemetry,
+    skillRecords,
+    pendingSignals,
+    candidates,
+    evolveCandidates,
+    agent,
+    autoGradedCount,
+  };
+}

package/cli/selftune/orchestrate/report.ts

@@ -0,0 +1,182 @@
+import type { OrchestrateResult, SkillAction } from "../orchestrate.js";
+import type { WorkflowSkillProposal } from "../workflows/proposals.js";
+
+function formatSyncPhase(syncResult: OrchestrateResult["syncResult"]): string[] {
+  const lines: string[] = ["Phase 1: Sync"];
+  const sources: [string, keyof OrchestrateResult["syncResult"]["sources"]][] = [
+    ["Claude", "claude"],
+    ["Codex", "codex"],
+    ["OpenCode", "opencode"],
+    ["OpenClaw", "openclaw"],
+  ];
+
+  for (const [label, key] of sources) {
+    const source = syncResult.sources[key];
+    if (!source.available) {
+      lines.push(`  ${label.padEnd(12)}not available`);
+    } else if (source.synced > 0) {
+      lines.push(`  ${label.padEnd(12)}scanned ${source.scanned}, synced ${source.synced}`);
+    } else {
+      lines.push(`  ${label.padEnd(12)}scanned ${source.scanned}, up to date`);
+    }
+  }
+
+  if (syncResult.repair.ran && syncResult.repair.repaired_records > 0) {
+    lines.push(
+      `  Repair      ${syncResult.repair.repaired_records} records across ${syncResult.repair.repaired_sessions} sessions`,
+    );
+  }
+
+  return lines;
+}
+
+function formatStatusPhase(statusResult: OrchestrateResult["statusResult"]): string[] {
+  const lines: string[] = ["Phase 2: Status"];
+  const byStatus: Record<string, number> = {};
+  for (const skill of statusResult.skills) {
+    byStatus[skill.status] = (byStatus[skill.status] ?? 0) + 1;
+  }
+  const healthLabel = statusResult.system.healthy ? "healthy" : "UNHEALTHY";
+  lines.push(`  ${statusResult.skills.length} skills found, system ${healthLabel}`);
+
+  const parts: string[] = [];
+  for (const status of ["CRITICAL", "WARNING", "HEALTHY", "UNGRADED", "UNKNOWN"]) {
+    if (byStatus[status]) parts.push(`${byStatus[status]} ${status}`);
+  }
+  if (parts.length > 0) lines.push(`  ${parts.join(", ")}`);
+
+  return lines;
+}
+
+function formatDecisionPhase(candidates: SkillAction[]): string[] {
+  const lines: string[] = ["Phase 3: Skill Decisions"];
+  if (candidates.length === 0) {
+    lines.push("  (no skills to evaluate)");
+    return lines;
+  }
+
+  for (const candidate of candidates) {
+    const icon = candidate.action === "skip" ? "⊘" : candidate.action === "watch" ? "○" : "→";
+    const actionLabel = candidate.action.toUpperCase().padEnd(7);
+    lines.push(`  ${icon} ${candidate.skill.padEnd(20)} ${actionLabel} ${candidate.reason}`);
+  }
+
+  return lines;
+}
+
+function formatEvolutionPhase(candidates: SkillAction[]): string[] {
+  const evolved = candidates.filter(
+    (candidate) => candidate.action === "evolve" && candidate.evolveResult !== undefined,
+  );
+  if (evolved.length === 0) return [];
+
+  const lines: string[] = ["Phase 4: Evolution Results"];
+  for (const candidate of evolved) {
+    const evolveResult = candidate.evolveResult as NonNullable<typeof candidate.evolveResult>;
+    const status = evolveResult.deployed ? "deployed" : "not deployed";
+    const detail = evolveResult.reason;
+    const validation = evolveResult.validation
+      ? ` (${(evolveResult.validation.before_pass_rate * 100).toFixed(0)}% → ${(evolveResult.validation.after_pass_rate * 100).toFixed(0)}%)`
+      : "";
+    lines.push(`  ${candidate.skill.padEnd(20)} ${status}${validation}`);
+    lines.push(`  ${"".padEnd(20)} ${detail}`);
+  }
+
+  return lines;
+}
+
+function formatWatchPhase(candidates: SkillAction[]): string[] {
+  const watched = candidates.filter((candidate) => candidate.action === "watch");
+  if (watched.length === 0) return [];
+
+  const lines: string[] = ["Phase 5: Watch"];
+  for (const candidate of watched) {
+    const snapshot = candidate.watchResult?.snapshot;
+    const metrics = snapshot
+      ? ` (pass_rate=${snapshot.pass_rate.toFixed(2)}, baseline=${snapshot.baseline_pass_rate.toFixed(2)})`
+      : "";
+    const alertTag = candidate.watchResult?.alert ? " [ALERT]" : "";
+    const rollbackTag = candidate.watchResult?.rolledBack ? " [ROLLED BACK]" : "";
+    lines.push(
+      `  ${candidate.skill.padEnd(20)} ${candidate.reason}${alertTag}${rollbackTag}${metrics}`,
+    );
+  }
+
+  return lines;
+}
+
+function formatWorkflowProposalPhase(proposals: WorkflowSkillProposal[]): string[] {
+  if (proposals.length === 0) return [];
+
+  const lines: string[] = ["Phase 6: Workflow Skill Proposals"];
+  for (const proposal of proposals) {
+    lines.push(
+      `  + ${proposal.source_skill_name.padEnd(20)} NEW_SKILL ${proposal.draft.skill_name} (${proposal.workflow.skills.join(" -> ")})`,
+    );
+    lines.push(`  ${"".padEnd(20)} ${proposal.summary}`);
+  }
+  return lines;
+}
+
+export function formatOrchestrateReport(result: OrchestrateResult): string {
+  const separator = "═".repeat(48);
+  const lines: string[] = [];
+
+  lines.push(separator);
+  lines.push("selftune orchestrate — decision report");
+  lines.push(separator);
+  lines.push("");
+
+  if (result.summary.dryRun) {
+    lines.push("Mode: DRY RUN (no mutations applied)");
+  } else if (result.summary.approvalMode === "review") {
+    lines.push("Mode: REVIEW (proposals validated but not deployed)");
+  } else {
+    lines.push("Mode: AUTONOMOUS (validated changes deployed automatically)");
+  }
+  lines.push("");
+
+  lines.push(...formatSyncPhase(result.syncResult));
+  lines.push("");
+  lines.push(...formatStatusPhase(result.statusResult));
+  lines.push("");
+  lines.push(...formatDecisionPhase(result.candidates));
+  lines.push("");
+
+  const evolutionLines = formatEvolutionPhase(result.candidates);
+  if (evolutionLines.length > 0) {
+    lines.push(...evolutionLines);
+    lines.push("");
+  }
+
+  const watchLines = formatWatchPhase(result.candidates);
+  if (watchLines.length > 0) {
+    lines.push(...watchLines);
+    lines.push("");
+  }
+
+  const workflowProposalLines = formatWorkflowProposalPhase(result.workflowProposals);
+  if (workflowProposalLines.length > 0) {
+    lines.push(...workflowProposalLines);
+    lines.push("");
+  }
+
+  lines.push("Summary");
+  lines.push(`  Auto-graded:  ${result.summary.autoGraded}`);
+  lines.push(`  Evaluated:    ${result.summary.evaluated} skills`);
+  lines.push(`  Deployed:     ${result.summary.deployed}`);
+  lines.push(`  Proposed:     ${result.workflowProposals.length} workflow skills`);
+  lines.push(`  Watched:      ${result.summary.watched}`);
+  lines.push(`  Skipped:      ${result.summary.skipped}`);
+  lines.push(`  Elapsed:      ${(result.summary.elapsedMs / 1000).toFixed(1)}s`);
+
+  if (result.summary.dryRun && result.summary.evaluated > 0) {
+    lines.push("");
+    lines.push("  Rerun without --dry-run to allow validated deployments.");
+  } else if (result.summary.approvalMode === "review" && result.summary.evaluated > 0) {
+    lines.push("");
+    lines.push("  Rerun without --review-required to allow validated deployments.");
+  }
+
+  return lines.join("\n");
+}

package/cli/selftune/orchestrate/runtime.ts

@@ -0,0 +1,120 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+import { readAlphaIdentity } from "../alpha-identity.js";
+import { SELFTUNE_CONFIG_PATH } from "../constants.js";
+import { readGradingResultsForSkill } from "../grading/results.js";
+import { getDb } from "../localdb/db.js";
+import {
+  queryEvolutionAudit,
+  queryQueryLog,
+  querySessionTelemetry,
+  querySkillUsageRecords,
+} from "../localdb/queries.js";
+import { doctor } from "../observability.js";
+import { computeStatus } from "../status.js";
+import { syncSources } from "../sync.js";
+import type {
+  AlphaIdentity,
+  EvolutionAuditEntry,
+  ImprovementSignalRecord,
+  QueryLogRecord,
+  SessionTelemetryRecord,
+  SkillUsageRecord,
+} from "../types.js";
+import { detectLlmAgent, type LlmBackedAgent } from "../utils/llm-call.js";
+import {
+  findInstalledSkillPath,
+  findRepositoryClaudeSkillDirs,
+  findRepositorySkillDirs,
+} from "../utils/skill-discovery.js";
+import {
+  discoverWorkflowSkillProposals,
+  persistWorkflowSkillProposal,
+} from "../workflows/proposals.js";
+import type { OrchestrateDeps } from "../orchestrate.js";
+import { buildReplayValidationOptions } from "./execute.js";
+
+export interface ResolvedOrchestrateRuntime {
+  syncSources: typeof syncSources;
+  computeStatus: typeof computeStatus;
+  evolve: typeof import("../evolution/evolve.js").evolve;
+  watch: typeof import("../monitoring/watch.js").watch;
+  detectAgent: () => LlmBackedAgent | null;
+  doctor: typeof doctor;
+  readTelemetry: () => SessionTelemetryRecord[];
+  readSkillRecords: () => SkillUsageRecord[];
+  readQueryRecords: () => QueryLogRecord[];
+  readAuditEntries: () => EvolutionAuditEntry[];
+  resolveSkillPath: (skillName: string) => string | undefined;
+  readGradingResults: (skillName: string) => ReturnType<typeof readGradingResultsForSkill>;
+  readSignals?: () => ImprovementSignalRecord[];
+  readAlphaIdentity: () => AlphaIdentity | null;
+  discoverWorkflowSkillProposals: typeof discoverWorkflowSkillProposals;
+  persistWorkflowSkillProposal: typeof persistWorkflowSkillProposal;
+  buildReplayOptions: typeof buildReplayValidationOptions;
+}
+
+export function getSkillSearchDirs(): string[] {
+  const home = homedir();
+  const cwd = process.cwd();
+  return [
+    join(home, ".claude", "skills"),
+    join(home, ".agents", "skills"),
+    join(home, ".codex", "skills"),
+    ...findRepositorySkillDirs(cwd),
+    ...findRepositoryClaudeSkillDirs(cwd),
+  ];
+}
+
+export function defaultResolveSkillPath(skillName: string): string | undefined {
+  return findInstalledSkillPath(skillName, getSkillSearchDirs());
+}
+
+export async function resolveOrchestrateRuntime(
+  deps: OrchestrateDeps = {},
+): Promise<ResolvedOrchestrateRuntime> {
+  const evolve = deps.evolve ?? (await import("../evolution/evolve.js")).evolve;
+  const watch = deps.watch ?? (await import("../monitoring/watch.js")).watch;
+
+  return {
+    syncSources: deps.syncSources ?? syncSources,
+    computeStatus: deps.computeStatus ?? computeStatus,
+    evolve,
+    watch,
+    detectAgent: deps.detectAgent ?? detectLlmAgent,
+    doctor: deps.doctor ?? doctor,
+    readTelemetry:
+      deps.readTelemetry ??
+      (() => {
+        const db = getDb();
+        return querySessionTelemetry(db) as SessionTelemetryRecord[];
+      }),
+    readSkillRecords:
+      deps.readSkillRecords ??
+      (() => {
+        const db = getDb();
+        return querySkillUsageRecords(db) as SkillUsageRecord[];
+      }),
+    readQueryRecords:
+      deps.readQueryRecords ??
+      (() => {
+        const db = getDb();
+        return queryQueryLog(db) as QueryLogRecord[];
+      }),
+    readAuditEntries:
+      deps.readAuditEntries ??
+      (() => {
+        const db = getDb();
+        return queryEvolutionAudit(db) as EvolutionAuditEntry[];
+      }),
+    resolveSkillPath: deps.resolveSkillPath ?? defaultResolveSkillPath,
+    readGradingResults: deps.readGradingResults ?? readGradingResultsForSkill,
+    readSignals: deps.readSignals,
+    readAlphaIdentity: deps.readAlphaIdentity ?? (() => readAlphaIdentity(SELFTUNE_CONFIG_PATH)),
+    discoverWorkflowSkillProposals:
+      deps.discoverWorkflowSkillProposals ?? discoverWorkflowSkillProposals,
+    persistWorkflowSkillProposal: deps.persistWorkflowSkillProposal ?? persistWorkflowSkillProposal,
+    buildReplayOptions: deps.buildReplayOptions ?? buildReplayValidationOptions,
+  };
+}

package/cli/selftune/orchestrate/signals.ts

@@ -0,0 +1,48 @@
+import { updateSignalConsumed } from "../localdb/direct-write.js";
+import { getDb } from "../localdb/db.js";
+import { queryImprovementSignals } from "../localdb/queries.js";
+import type { ImprovementSignalRecord } from "../types.js";
+
+export function readPendingSignals(
+  reader?: () => ImprovementSignalRecord[],
+): ImprovementSignalRecord[] {
+  const read =
+    reader ??
+    (() => {
+      const db = getDb();
+      return queryImprovementSignals(db, false) as ImprovementSignalRecord[];
+    });
+
+  try {
+    return read().filter((signal) => !signal.consumed);
+  } catch {
+    return [];
+  }
+}
+
+export function groupSignalsBySkill(signals: ImprovementSignalRecord[]): Map<string, number> {
+  const map = new Map<string, number>();
+  for (const signal of signals) {
+    if (signal.mentioned_skill) {
+      const key = signal.mentioned_skill.toLowerCase();
+      map.set(key, (map.get(key) ?? 0) + 1);
+    }
+  }
+  return map;
+}
+
+export function markSignalsConsumed(signals: ImprovementSignalRecord[], runId: string): void {
+  try {
+    if (signals.length === 0) return;
+    for (const signal of signals) {
+      const ok = updateSignalConsumed(signal.session_id, signal.query, signal.signal_type, runId);
+      if (!ok) {
+        console.error(
+          `[orchestrate] failed to mark signal consumed: session_id=${signal.session_id}, signal_type=${signal.signal_type}`,
+        );
+      }
+    }
+  } catch {
+    // Silent on errors.
+  }
+}