selftune 0.2.18 → 0.2.20

package/cli/selftune/status.ts

@@ -20,13 +20,16 @@ import { getDb } from "./localdb/db.js";
 import {
   getLastUploadError,
   getLastUploadSuccess,
+  getSkillTrustSummaries,
   queryEvolutionAudit,
   queryQueryLog,
   querySessionTelemetry,
   querySkillUsageRecords,
+  type SkillTrustSummary,
 } from "./localdb/queries.js";
 import { computeMonitoringSnapshot, MIN_MONITORING_SKILL_CHECKS } from "./monitoring/watch.js";
 import { doctor } from "./observability.js";
+import { deriveTrustBucket, deriveTrustBucketReason } from "./trust-model.js";
 import type {
   AgentCommandGuidance,
   AlphaLinkState,
@@ -273,7 +276,44 @@ const TREND_SYMBOLS: Record<string, string> = {
   unknown: "?",
 };
 
-export function formatStatus(result: StatusResult): string {
+function formatTrustHighlights(trustSummaries: SkillTrustSummary[] | undefined): string[] {
+  if (!trustSummaries || trustSummaries.length === 0) return [];
+
+  const recentSort = (a: SkillTrustSummary, b: SkillTrustSummary) =>
+    (b.last_seen ?? "").localeCompare(a.last_seen ?? "");
+  const attention = [...trustSummaries]
+    .filter((summary) => deriveTrustBucket(summary) === "at_risk")
+    .sort(recentSort)
+    .slice(0, 3);
+  const improving = [...trustSummaries]
+    .filter((summary) => deriveTrustBucket(summary) === "improving")
+    .sort(recentSort)
+    .slice(0, 3);
+
+  if (attention.length === 0 && improving.length === 0) return [];
+
+  const lines = ["Highlights"];
+  if (attention.length > 0) {
+    lines.push(
+      `  Attention: ${attention
+        .map((summary) => `${summary.skill_name} (${deriveTrustBucketReason("at_risk", summary)})`)
+        .join("; ")}`,
+    );
+  }
+  if (improving.length > 0) {
+    lines.push(
+      `  Improving: ${improving
+        .map(
+          (summary) => `${summary.skill_name} (${deriveTrustBucketReason("improving", summary)})`,
+        )
+        .join("; ")}`,
+    );
+  }
+
+  return lines;
+}
+
+export function formatStatus(result: StatusResult, trustSummaries?: SkillTrustSummary[]): string {
   const noColor = !!process.env.NO_COLOR;
 
   const green = noColor ? (s: string) => s : (s: string) => colorize(s, "#788c5d");
@@ -284,6 +324,14 @@ export function formatStatus(result: StatusResult): string {
   lines.push("selftune status");
   lines.push("\u2550".repeat(15));
   lines.push("");
+  lines.push(formatStatusSummary(result, trustSummaries));
+  lines.push("");
+
+  const highlightLines = formatTrustHighlights(trustSummaries);
+  if (highlightLines.length > 0) {
+    lines.push(...highlightLines);
+    lines.push("");
+  }
 
   // Skills table
   const skillCount = result.skills.length;
@@ -351,6 +399,36 @@ export function formatStatus(result: StatusResult): string {
   return lines.join("\n");
 }
 
+export function formatStatusSummary(
+  result: StatusResult,
+  trustSummaries?: SkillTrustSummary[],
+): string {
+  const watched = trustSummaries?.length ?? result.skills.length;
+  const improving =
+    trustSummaries?.filter((summary) => deriveTrustBucket(summary) === "improving").length ??
+    result.skills.filter((skill) => skill.trend === "up").length;
+  const needsAttention =
+    trustSummaries?.filter((summary) => deriveTrustBucket(summary) === "at_risk").length ??
+    result.skills.filter((skill) => skill.status === "WARNING" || skill.status === "CRITICAL")
+      .length;
+
+  const watchedText = `${watched} ${watched === 1 ? "skill" : "skills"} watched`;
+  const improvingText =
+    improving > 0
+      ? `${improving} improving`
+      : result.lastSession
+        ? "no recent lift"
+        : "no recent data";
+  const attentionText =
+    needsAttention > 0
+      ? `${needsAttention} needing attention`
+      : watched > 0
+        ? "nothing urgent"
+        : "nothing tracked yet";
+
+  return `${watchedText} | ${improvingText} | ${attentionText}`;
+}
+
 // ---------------------------------------------------------------------------
 // Terminal color helper using ANSI escapes
 // ---------------------------------------------------------------------------
@@ -506,7 +584,8 @@ export async function cliMain(): Promise<void> {
     const doctorResult = await doctor();
 
     const result = computeStatus(telemetry, skillRecords, queryRecords, auditEntries, doctorResult);
-    const output = formatStatus(result);
+    const trustSummaries = getSkillTrustSummaries(db);
+    const output = formatStatus(result, trustSummaries);
     console.log(output);
 
     // Alpha upload status section

package/cli/selftune/sync.ts

@@ -31,6 +31,7 @@ import {
   SKILL_LOG,
   TELEMETRY_LOG,
 } from "./constants.js";
+import { stageCreatorContributionSignals } from "./contribution-staging.js";
 import {
   findTranscriptFiles,
   parseSession,
@@ -58,6 +59,7 @@ import {
 import { getDb } from "./localdb/db.js";
 import { querySkillUsageRecords } from "./localdb/queries.js";
 import {
+  persistRepairedSkillUsageToDb,
   rebuildSkillUsageFromCodexRollouts,
   rebuildSkillUsageFromTranscripts,
 } from "./repair/skill-usage.js";
@@ -96,6 +98,12 @@ export interface SyncResult {
     repaired_records: number;
     codex_repaired_records: number;
   };
+  creator_contributions: {
+    ran: boolean;
+    eligible_skills: number;
+    built_signals: number;
+    staged_signals: number;
+  };
   timings: SyncPhaseTiming[];
   total_elapsed_ms: number;
 }
@@ -130,6 +138,14 @@ export interface SyncDeps {
     repairedRecords: number;
     codexRepairedRecords: number;
   };
+  stageCreatorContributions?: (
+    db: ReturnType<typeof getDb>,
+    options: { dryRun: boolean },
+  ) => {
+    eligible_skills: number;
+    built_signals: number;
+    staged_signals: number;
+  };
 }
 
 export function createDefaultSyncOptions(overrides: Partial<SyncOptions> = {}): SyncOptions {
@@ -344,6 +360,7 @@ function rebuildSkillUsageOverlay(
   options: SyncOptions,
   onProgress?: SyncProgressCallback,
   cache?: FileListCache,
+  db: ReturnType<typeof getDb> = getDb(),
 ): {
   repairedSessions: number;
   repairedRecords: number;
@@ -363,7 +380,6 @@ function rebuildSkillUsageOverlay(
   let rawSkillRecords: SkillUsageRecord[];
   if (options.skillLogPath === SKILL_LOG) {
     try {
-      const db = getDb();
       rawSkillRecords = querySkillUsageRecords(db) as SkillUsageRecord[];
     } catch {
       rawSkillRecords = readJsonl<SkillUsageRecord>(options.skillLogPath);
@@ -389,6 +405,7 @@ function rebuildSkillUsageOverlay(
   repairedRecords.push(...codexRecords);
 
   if (!options.dryRun) {
+    persistRepairedSkillUsageToDb(db, repairedRecords);
     writeRepairedSkillUsageRecords(
       repairedRecords,
       repairedSessionIds,
@@ -429,6 +446,8 @@ export function syncSources(
   const runOpenCode = deps.syncOpenCode;
   const runOpenClaw = deps.syncOpenClaw;
   const runRepair = deps.rebuildSkillUsage;
+  const runCreatorContributions = deps.stageCreatorContributions;
+  const db = getDb();
 
   const disabledStep: SyncStepResult = { available: false, scanned: 0, synced: 0, skipped: 0 };
 
@@ -470,11 +489,27 @@ export function syncSources(
     ? timePhase(
         "repair",
         () =>
-          runRepair ? runRepair(options) : rebuildSkillUsageOverlay(options, onProgress, cache),
+          runRepair ? runRepair(options) : rebuildSkillUsageOverlay(options, onProgress, cache, db),
         timings,
       )
     : { repairedSessions: 0, repairedRecords: 0, codexRepairedRecords: 0 };
 
+  const creatorContributions = timePhase(
+    "creator_contributions",
+    () => {
+      const staged = runCreatorContributions
+        ? runCreatorContributions(db, { dryRun: options.dryRun })
+        : stageCreatorContributionSignals(db, { dryRun: options.dryRun });
+      return {
+        ran: true,
+        eligible_skills: staged.eligible_skills,
+        built_signals: staged.built_signals,
+        staged_signals: staged.staged_signals,
+      };
+    },
+    timings,
+  );
+
   const totalElapsed = Math.round(performance.now() - totalStart);
 
   return {
@@ -487,6 +522,7 @@ export function syncSources(
       repaired_records: repair.repairedRecords,
       codex_repaired_records: repair.codexRepairedRecords,
     },
+    creator_contributions: creatorContributions,
     timings,
     total_elapsed_ms: totalElapsed,
   };
@@ -636,6 +672,24 @@ Options:
       );
     }
 
+    if (
+      result.creator_contributions.eligible_skills > 0 ||
+      result.creator_contributions.built_signals > 0
+    ) {
+      const contributionTiming = timingMap.get("creator_contributions");
+      const contributionTime = contributionTiming
+        ? ` (${formatMs(contributionTiming.elapsed_ms)})`
+        : "";
+      process.stderr.write(
+        `Creator contributions: ${result.creator_contributions.built_signals} signals from ` +
+          `${result.creator_contributions.eligible_skills} skills` +
+          (result.dry_run
+            ? " ready to stage"
+            : ` staged=${result.creator_contributions.staged_signals}`) +
+          `${contributionTime}\n`,
+      );
+    }
+
     process.stderr.write(`\nDone in ${formatMs(result.total_elapsed_ms)}\n`);
   }
 

package/cli/selftune/trust-model.ts

@@ -0,0 +1,66 @@
+import type { TrustBucket, TrustState, TrustWatchlistEntry } from "./dashboard-contract.js";
+import type { SkillTrustSummary } from "./localdb/queries.js";
+
+const AT_RISK_MISS_RATE_THRESHOLD = 0.15;
+const UNCERTAIN_MIN_CHECKS = 10;
+
+function formatPercent(value: number): string {
+  return `${(value * 100).toFixed(1).replace(/\.0$/, "")}%`;
+}
+
+export function deriveTrustState(summary: SkillTrustSummary): TrustState {
+  if (summary.latest_action === "rolled_back") return "rolled_back";
+  if (summary.latest_action === "deployed") return "deployed";
+  if (summary.latest_action === "validated") return "validated";
+  if (summary.latest_action === "watch") return "watch";
+  if (summary.total_checks < 5) return "low_sample";
+  return "observed";
+}
+
+export function deriveTrustBucket(summary: SkillTrustSummary): TrustBucket {
+  if (summary.latest_action === "rolled_back" || summary.miss_rate > AT_RISK_MISS_RATE_THRESHOLD) {
+    return "at_risk";
+  }
+  if (
+    summary.latest_action === "validated" ||
+    summary.latest_action === "created" ||
+    summary.latest_action === "proposed"
+  ) {
+    return "improving";
+  }
+  if (summary.total_checks < UNCERTAIN_MIN_CHECKS || summary.latest_action === "watch") {
+    return "uncertain";
+  }
+  return "stable";
+}
+
+export function deriveTrustBucketReason(bucket: TrustBucket, summary: SkillTrustSummary): string {
+  switch (bucket) {
+    case "at_risk":
+      if (summary.latest_action === "rolled_back") return "Recently rolled back";
+      return `High miss rate (${formatPercent(summary.miss_rate)})`;
+    case "improving":
+      if (summary.latest_action === "validated") return "Proposal validated, pending deploy";
+      return "Has pending evolution proposal";
+    case "uncertain":
+      if (summary.total_checks < 10) return `Low sample size (${summary.total_checks} checks)`;
+      return "Under active observation";
+    case "stable":
+      return "Routing healthy, no issues detected";
+  }
+}
+
+export function buildTrustWatchlist(summaries: SkillTrustSummary[]): TrustWatchlistEntry[] {
+  return summaries.map((summary) => {
+    const bucket = deriveTrustBucket(summary);
+    return {
+      skill_name: summary.skill_name,
+      bucket,
+      trust_state: deriveTrustState(summary),
+      reason: deriveTrustBucketReason(bucket, summary),
+      pass_rate: summary.pass_rate,
+      checks: summary.total_checks,
+      last_seen: summary.last_seen,
+    };
+  });
+}

package/cli/selftune/types.ts

@@ -400,6 +400,10 @@ export interface EvolutionAuditEntry {
   details: string;
   eval_snapshot?: EvalPassRate;
   iterations_used?: number;
+  validation_mode?: ValidationMode;
+  validation_agent?: string;
+  validation_fixture_id?: string;
+  validation_evidence_ref?: string;
 }
 
 export interface EvolutionEvidenceValidation {
@@ -413,6 +417,10 @@ export interface EvolutionEvidenceValidation {
   gates_passed?: number;
   gates_total?: number;
   gate_results?: Array<{ gate: ValidationGate; passed: boolean; reason: string }>;
+  validation_mode?: ValidationMode;
+  validation_agent?: string;
+  validation_fixture_id?: string;
+  validation_evidence_ref?: string;
 }
 
 export interface EvolutionEvidenceEntry {
@@ -697,6 +705,25 @@ export interface BodyEvolutionProposal {
 /** Closed union of gate names used in the validation pipeline. */
 export type ValidationGate = "structural" | "trigger_accuracy" | "quality";
 
+export type ValidationMode = "structural_guard" | "host_replay" | "llm_judge";
+
+export interface RoutingReplayFixture {
+  fixture_id: string;
+  platform: "claude_code" | "codex";
+  target_skill_name: string;
+  target_skill_path: string;
+  competing_skill_paths: string[];
+  workspace_root?: string;
+}
+
+export interface RoutingReplayEntryResult {
+  query: string;
+  should_trigger: boolean;
+  triggered: boolean;
+  passed: boolean;
+  evidence?: string;
+}
+
 /** Result of validating a body evolution proposal. */
 export interface BodyValidationResult {
   proposal_id: string;
@@ -705,6 +732,12 @@ export interface BodyValidationResult {
   gate_results: Array<{ gate: ValidationGate; passed: boolean; reason: string }>;
   improved: boolean;
   regressions: string[];
+  validation_mode?: ValidationMode;
+  validation_agent?: string;
+  validation_fixture_id?: string;
+  before_pass_rate?: number;
+  after_pass_rate?: number;
+  per_entry_results?: RoutingReplayEntryResult[];
 }
 
 /** Configuration for which LLM model a role should use. */
@@ -854,6 +887,76 @@ export interface ComposabilityReportV2 extends ComposabilityReport {
   synergy_count: number;
 }
 
+// ---------------------------------------------------------------------------
+// Skill family overlap / consolidation types
+// ---------------------------------------------------------------------------
+
+export interface SkillFamilyOverlapMember {
+  skill_name: string;
+  skill_path?: string;
+  positive_query_count: number;
+}
+
+export interface SkillFamilyOverlapPair {
+  skill_a: string;
+  skill_b: string;
+  overlap_pct: number;
+  shared_query_count: number;
+  shared_queries: string[];
+  consolidation_pressure: "low" | "medium" | "high";
+}
+
+export interface SkillFamilyColdStartPair {
+  skill_a: string;
+  skill_b: string;
+  description_similarity: number;
+  when_to_use_similarity: number;
+  shared_command_surfaces: string[];
+  shared_terms: string[];
+  synthetic_confusion_queries: string[];
+  suspicion_level: "low" | "medium" | "high";
+}
+
+export interface SkillFamilyColdStartSuspicion {
+  candidate: boolean;
+  analyzed_pairs: number;
+  suspicious_pair_count: number;
+  average_static_similarity: number;
+  pairs: SkillFamilyColdStartPair[];
+  rationale: string[];
+}
+
+export interface SkillFamilyRefactorWorkflow {
+  workflow_name: string;
+  source_skill: string;
+  suggested_path: string;
+}
+
+export interface SkillFamilyRefactorProposal {
+  parent_skill_name: string;
+  family_prefix?: string;
+  internal_workflows: SkillFamilyRefactorWorkflow[];
+  compatibility_aliases: Array<{ skill_name: string; target_workflow: string }>;
+  migration_notes: string[];
+}
+
+export interface SkillFamilyOverlapReport {
+  family_prefix?: string;
+  analyzed_skills: string[];
+  members: SkillFamilyOverlapMember[];
+  pairs: SkillFamilyOverlapPair[];
+  cold_start_suspicion?: SkillFamilyColdStartSuspicion;
+  total_pairs_analyzed: number;
+  overlap_count: number;
+  overlap_density: number;
+  average_overlap_pct: number;
+  consolidation_candidate: boolean;
+  recommendation: string;
+  rationale: string[];
+  refactor_proposal?: SkillFamilyRefactorProposal;
+  generated_at: string;
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Support types
 // ---------------------------------------------------------------------------

package/cli/selftune/utils/skill-detection.ts

@@ -0,0 +1,43 @@
+export function normalizeSkillName(value: string): string {
+  return value.trim().toLowerCase();
+}
+
+export function getInternalPromptTargetSkill(
+  text: string,
+  knownSkillNames: Iterable<string>,
+): string | null {
+  if (!text) return null;
+  const isInternalSkillPrompt =
+    text.includes("You are a skill description optimizer") ||
+    text.includes("You are an evaluation assistant") ||
+    text.includes("Given this skill description");
+  if (!isInternalSkillPrompt) return null;
+
+  const candidates = [
+    /Skill Name:\s*([^\n]+)/i,
+    /Propose an improved description for the "([^"]+)" skill/i,
+    /would each query trigger the "([^"]+)" skill/i,
+  ];
+  for (const pattern of candidates) {
+    const match = text.match(pattern);
+    const rawSkillName = match?.[1]?.trim();
+    if (!rawSkillName) continue;
+    const normalizedTarget = normalizeSkillName(rawSkillName);
+    for (const skillName of knownSkillNames) {
+      if (normalizeSkillName(skillName) === normalizedTarget) {
+        return skillName;
+      }
+    }
+    return rawSkillName;
+  }
+  return null;
+}
+
+export function isWrappedNonUserPart(text: string): boolean {
+  const trimmed = text.trimStart();
+  return (
+    trimmed.startsWith("# AGENTS.md instructions for ") ||
+    trimmed.startsWith("<environment_context>") ||
+    trimmed.startsWith("<INSTRUCTIONS>")
+  );
+}

package/cli/selftune/utils/text-similarity.ts

@@ -0,0 +1,73 @@
+const BASE_TEXT_SIMILARITY_STOPWORDS = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "for",
+  "from",
+  "how",
+  "in",
+  "into",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "that",
+  "the",
+  "this",
+  "to",
+  "use",
+  "user",
+  "when",
+  "with",
+]);
+
+export function buildStopwordSet(additionalStopwords: string[] = []): Set<string> {
+  return new Set([...BASE_TEXT_SIMILARITY_STOPWORDS, ...additionalStopwords]);
+}
+
+export function tokenizeText(
+  text: string,
+  stopwords = BASE_TEXT_SIMILARITY_STOPWORDS,
+): Set<string> {
+  return new Set(
+    text
+      .toLowerCase()
+      .split(/[^a-z0-9]+/i)
+      .map((token) => token.trim())
+      .filter((token) => token.length >= 3 && !stopwords.has(token)),
+  );
+}
+
+export function jaccardSimilarity(left: Set<string>, right: Set<string>): number {
+  if (left.size === 0 || right.size === 0) return 0;
+  let shared = 0;
+  for (const token of left) {
+    if (right.has(token)) shared += 1;
+  }
+  const union = left.size + right.size - shared;
+  return union > 0 ? shared / union : 0;
+}
+
+export function extractWhenToUseLines(body: string): string[] {
+  const lines = body.split("\n");
+  const start = lines.findIndex((line) => /^##+\s+when to use\s*$/i.test(line.trim()));
+  if (start === -1) return [];
+
+  const extracted: string[] = [];
+  for (let i = start + 1; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    if (/^##+\s+/.test(line)) break;
+    if (/^[-*]\s+/.test(line)) {
+      extracted.push(line.replace(/^[-*]\s+/, "").trim());
+      continue;
+    }
+    extracted.push(line);
+  }
+  return extracted;
+}

package/cli/selftune/watchlist.ts

@@ -0,0 +1,65 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+
+import { SELFTUNE_CONFIG_DIR, WATCHED_SKILLS_PATH } from "./constants.js";
+
+const CURRENT_WATCHLIST_VERSION = 1;
+
+interface WatchlistPayload {
+  version: typeof CURRENT_WATCHLIST_VERSION;
+  skills: string[];
+}
+
+function normalizeSkills(skills: string[]): string[] {
+  const seen = new Set<string>();
+  const normalized: string[] = [];
+  for (const skill of skills) {
+    const trimmed = skill.trim();
+    if (!trimmed || seen.has(trimmed)) continue;
+    seen.add(trimmed);
+    normalized.push(trimmed);
+  }
+  return normalized;
+}
+
+export function loadWatchedSkills(): string[] {
+  try {
+    if (!existsSync(WATCHED_SKILLS_PATH)) return [];
+    const parsed = JSON.parse(
+      readFileSync(WATCHED_SKILLS_PATH, "utf-8"),
+    ) as Partial<WatchlistPayload>;
+    return parsed.version === CURRENT_WATCHLIST_VERSION && Array.isArray(parsed.skills)
+      ? normalizeSkills(parsed.skills.filter((skill): skill is string => typeof skill === "string"))
+      : [];
+  } catch {
+    return [];
+  }
+}
+
+export function saveWatchedSkills(skills: string[]): string[] {
+  const normalized = normalizeSkills(skills);
+  mkdirSync(SELFTUNE_CONFIG_DIR, { recursive: true });
+  const tempPath = `${WATCHED_SKILLS_PATH}.${process.pid}.${Date.now()}.tmp`;
+  try {
+    writeFileSync(
+      tempPath,
+      JSON.stringify({ version: CURRENT_WATCHLIST_VERSION, skills: normalized }, null, 2),
+      "utf-8",
+    );
+    renameSync(tempPath, WATCHED_SKILLS_PATH);
+  } catch (error) {
+    try {
+      if (existsSync(tempPath)) unlinkSync(tempPath);
+    } catch {
+      // Best-effort cleanup for interrupted temp writes.
+    }
+    throw error;
+  }
+  return normalized;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "selftune",
-  "version": "0.2.18",
+  "version": "0.2.20",
   "description": "Self-improving skills CLI for AI agents",
   "keywords": [
     "agent",