selftune 0.2.19 → 0.2.20

package/cli/selftune/localdb/queries.ts

@@ -1130,6 +1130,10 @@ export function queryEvolutionAudit(
   action: string;
   details: string;
   eval_snapshot?: Record<string, unknown>;
+  validation_mode?: string;
+  validation_agent?: string;
+  validation_fixture_id?: string;
+  validation_evidence_ref?: string;
 }> {
   const sql = skillName
     ? `SELECT * FROM evolution_audit
@@ -1143,12 +1147,18 @@ export function queryEvolutionAudit(
   return rows.map((r) => ({
     timestamp: r.timestamp as string,
     proposal_id: r.proposal_id as string,
-    skill_name: r.skill_name as string | undefined,
+    skill_name: typeof r.skill_name === "string" ? r.skill_name : undefined,
     action: r.action as string,
     details: r.details as string,
     eval_snapshot: r.eval_snapshot_json
       ? (safeParseJson(r.eval_snapshot_json as string) as Record<string, unknown>)
       : undefined,
+    validation_mode: typeof r.validation_mode === "string" ? r.validation_mode : undefined,
+    validation_agent: typeof r.validation_agent === "string" ? r.validation_agent : undefined,
+    validation_fixture_id:
+      typeof r.validation_fixture_id === "string" ? r.validation_fixture_id : undefined,
+    validation_evidence_ref:
+      typeof r.validation_evidence_ref === "string" ? r.validation_evidence_ref : undefined,
   }));
 }
 

package/cli/selftune/localdb/schema.ts

@@ -121,7 +121,12 @@ CREATE TABLE IF NOT EXISTS evolution_audit (
   skill_name      TEXT,
   action          TEXT NOT NULL,
   details         TEXT,
-  eval_snapshot_json TEXT
+  eval_snapshot_json TEXT,
+  iterations_used  INTEGER,
+  validation_mode  TEXT,
+  validation_agent TEXT,
+  validation_fixture_id TEXT,
+  validation_evidence_ref TEXT
 )`;
 
 // -- Local telemetry tables (from JSONL logs) ---------------------------------
@@ -369,6 +374,10 @@ export const MIGRATIONS = [
   `ALTER TABLE skill_invocations ADD COLUMN source TEXT`,
   // Track how many iteration loops each evolution run used
   `ALTER TABLE evolution_audit ADD COLUMN iterations_used INTEGER`,
+  `ALTER TABLE evolution_audit ADD COLUMN validation_mode TEXT`,
+  `ALTER TABLE evolution_audit ADD COLUMN validation_agent TEXT`,
+  `ALTER TABLE evolution_audit ADD COLUMN validation_fixture_id TEXT`,
+  `ALTER TABLE evolution_audit ADD COLUMN validation_evidence_ref TEXT`,
   // Canonical contract fields for upload staging (sessions already has schema_version, platform, normalized_at)
   `ALTER TABLE sessions ADD COLUMN normalizer_version TEXT`,
   `ALTER TABLE sessions ADD COLUMN capture_mode TEXT`,

package/cli/selftune/routes/skill-report.ts

@@ -28,7 +28,8 @@ export function handleSkillReport(
   // 1. Evolution audit with eval_snapshot
   const evolution = db
     .query(
-      `SELECT timestamp, proposal_id, skill_name, action, details, eval_snapshot_json
+      `SELECT timestamp, proposal_id, skill_name, action, details, eval_snapshot_json,
+              validation_mode, validation_agent, validation_fixture_id, validation_evidence_ref
        FROM evolution_audit
        WHERE skill_name = ? OR (skill_name IS NULL AND proposal_id LIKE 'evo-' || ? || '-%')
        ORDER BY timestamp DESC
@@ -41,6 +42,10 @@ export function handleSkillReport(
     action: string;
     details: string;
     eval_snapshot_json: string | null;
+    validation_mode: string | null;
+    validation_agent: string | null;
+    validation_fixture_id: string | null;
+    validation_evidence_ref: string | null;
   }>;
   const evolutionWithSnapshot = evolution.map((e) => ({
     ...e,

package/cli/selftune/types.ts

@@ -400,6 +400,10 @@ export interface EvolutionAuditEntry {
   details: string;
   eval_snapshot?: EvalPassRate;
   iterations_used?: number;
+  validation_mode?: ValidationMode;
+  validation_agent?: string;
+  validation_fixture_id?: string;
+  validation_evidence_ref?: string;
 }
 
 export interface EvolutionEvidenceValidation {
@@ -413,6 +417,10 @@ export interface EvolutionEvidenceValidation {
   gates_passed?: number;
   gates_total?: number;
   gate_results?: Array<{ gate: ValidationGate; passed: boolean; reason: string }>;
+  validation_mode?: ValidationMode;
+  validation_agent?: string;
+  validation_fixture_id?: string;
+  validation_evidence_ref?: string;
 }
 
 export interface EvolutionEvidenceEntry {
@@ -697,6 +705,25 @@ export interface BodyEvolutionProposal {
 /** Closed union of gate names used in the validation pipeline. */
 export type ValidationGate = "structural" | "trigger_accuracy" | "quality";
 
+export type ValidationMode = "structural_guard" | "host_replay" | "llm_judge";
+
+export interface RoutingReplayFixture {
+  fixture_id: string;
+  platform: "claude_code" | "codex";
+  target_skill_name: string;
+  target_skill_path: string;
+  competing_skill_paths: string[];
+  workspace_root?: string;
+}
+
+export interface RoutingReplayEntryResult {
+  query: string;
+  should_trigger: boolean;
+  triggered: boolean;
+  passed: boolean;
+  evidence?: string;
+}
+
 /** Result of validating a body evolution proposal. */
 export interface BodyValidationResult {
   proposal_id: string;
@@ -705,6 +732,12 @@ export interface BodyValidationResult {
   gate_results: Array<{ gate: ValidationGate; passed: boolean; reason: string }>;
   improved: boolean;
   regressions: string[];
+  validation_mode?: ValidationMode;
+  validation_agent?: string;
+  validation_fixture_id?: string;
+  before_pass_rate?: number;
+  after_pass_rate?: number;
+  per_entry_results?: RoutingReplayEntryResult[];
 }
 
 /** Configuration for which LLM model a role should use. */
@@ -873,6 +906,26 @@ export interface SkillFamilyOverlapPair {
   consolidation_pressure: "low" | "medium" | "high";
 }
 
+export interface SkillFamilyColdStartPair {
+  skill_a: string;
+  skill_b: string;
+  description_similarity: number;
+  when_to_use_similarity: number;
+  shared_command_surfaces: string[];
+  shared_terms: string[];
+  synthetic_confusion_queries: string[];
+  suspicion_level: "low" | "medium" | "high";
+}
+
+export interface SkillFamilyColdStartSuspicion {
+  candidate: boolean;
+  analyzed_pairs: number;
+  suspicious_pair_count: number;
+  average_static_similarity: number;
+  pairs: SkillFamilyColdStartPair[];
+  rationale: string[];
+}
+
 export interface SkillFamilyRefactorWorkflow {
   workflow_name: string;
   source_skill: string;
@@ -892,6 +945,7 @@ export interface SkillFamilyOverlapReport {
   analyzed_skills: string[];
   members: SkillFamilyOverlapMember[];
   pairs: SkillFamilyOverlapPair[];
+  cold_start_suspicion?: SkillFamilyColdStartSuspicion;
   total_pairs_analyzed: number;
   overlap_count: number;
   overlap_density: number;

package/cli/selftune/utils/text-similarity.ts

@@ -0,0 +1,73 @@
+const BASE_TEXT_SIMILARITY_STOPWORDS = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "for",
+  "from",
+  "how",
+  "in",
+  "into",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "that",
+  "the",
+  "this",
+  "to",
+  "use",
+  "user",
+  "when",
+  "with",
+]);
+
+export function buildStopwordSet(additionalStopwords: string[] = []): Set<string> {
+  return new Set([...BASE_TEXT_SIMILARITY_STOPWORDS, ...additionalStopwords]);
+}
+
+export function tokenizeText(
+  text: string,
+  stopwords = BASE_TEXT_SIMILARITY_STOPWORDS,
+): Set<string> {
+  return new Set(
+    text
+      .toLowerCase()
+      .split(/[^a-z0-9]+/i)
+      .map((token) => token.trim())
+      .filter((token) => token.length >= 3 && !stopwords.has(token)),
+  );
+}
+
+export function jaccardSimilarity(left: Set<string>, right: Set<string>): number {
+  if (left.size === 0 || right.size === 0) return 0;
+  let shared = 0;
+  for (const token of left) {
+    if (right.has(token)) shared += 1;
+  }
+  const union = left.size + right.size - shared;
+  return union > 0 ? shared / union : 0;
+}
+
+export function extractWhenToUseLines(body: string): string[] {
+  const lines = body.split("\n");
+  const start = lines.findIndex((line) => /^##+\s+when to use\s*$/i.test(line.trim()));
+  if (start === -1) return [];
+
+  const extracted: string[] = [];
+  for (let i = start + 1; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    if (/^##+\s+/.test(line)) break;
+    if (/^[-*]\s+/.test(line)) {
+      extracted.push(line.replace(/^[-*]\s+/, "").trim());
+      continue;
+    }
+    extracted.push(line);
+  }
+  return extracted;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "selftune",
-  "version": "0.2.19",
+  "version": "0.2.20",
   "description": "Self-improving skills CLI for AI agents",
   "keywords": [
     "agent",

package/packages/ui/src/components/EvidenceViewer.tsx

@@ -12,6 +12,7 @@ import {
   ListChecksIcon,
 } from "lucide-react";
 import { useMemo, useState } from "react";
+import type { ReactNode } from "react";
 import Markdown from "react-markdown";
 
 import { formatRate, timeAgo } from "../lib/format";
@@ -34,6 +35,37 @@ interface Props {
   showContextBanner?: boolean;
 }
 
+function getValidationModeMeta(mode?: string | null): {
+  label: string;
+  variant: "default" | "secondary" | "destructive" | "outline";
+  description: string;
+} | null {
+  switch (mode) {
+    case "host_replay":
+      return {
+        label: "Replay-backed validation",
+        variant: "default",
+        description:
+          "Validated against a controlled replay fixture instead of a free-form judge prompt.",
+      };
+    case "llm_judge":
+      return {
+        label: "Model judgment",
+        variant: "secondary",
+        description: "Validated by an LLM trigger check rather than a replay fixture.",
+      };
+    case "structural_guard":
+      return {
+        label: "Structural guard",
+        variant: "outline",
+        description:
+          "Only deterministic structural checks ran; no replay or judge validation was needed.",
+      };
+    default:
+      return null;
+  }
+}
+
 function sentenceCase(value: string): string {
   return value.replace(/_/g, " ");
 }
@@ -42,7 +74,7 @@ function getOutcomePresentation(action?: string | null): {
   title: string;
   summary: string;
   tone: string;
-  icon: React.ReactNode;
+  icon: ReactNode;
   liveSkillNote: string;
 } {
   switch (action) {
@@ -180,7 +212,7 @@ function SkillContentBlock({
 }
 
 /** Smart formatting for a single validation value */
-function formatValidationValue(key: string, val: unknown): React.ReactNode {
+function formatValidationValue(key: string, val: unknown): ReactNode {
   // Booleans
   if (typeof val === "boolean") {
     return val ? (
@@ -279,12 +311,18 @@ function ValidationResults({ validation }: { validation: Record<string, unknown>
     regressions,
     new_passes,
     per_entry_results,
+    validation_mode,
+    validation_agent,
+    validation_fixture_id,
+    validation_evidence_ref,
     ...rest
   } = validation;
 
   const regressionsArr = Array.isArray(regressions) ? regressions : [];
   const newPassesArr = Array.isArray(new_passes) ? new_passes : [];
   const perEntryArr = Array.isArray(per_entry_results) ? per_entry_results : [];
+  const validationMeta =
+    typeof validation_mode === "string" ? getValidationModeMeta(validation_mode) : null;
 
   return (
     <div className="rounded-md border bg-muted/30 p-3 space-y-3">
@@ -295,6 +333,34 @@ function ValidationResults({ validation }: { validation: Record<string, unknown>
         </span>
       </p>
 
+      {validationMeta && (
+        <div className="rounded-md border bg-card px-3 py-2">
+          <div className="flex flex-wrap items-center gap-2">
+            <Badge variant={validationMeta.variant} className="text-[10px]">
+              {validationMeta.label}
+            </Badge>
+            {typeof validation_agent === "string" && validation_agent.trim() && (
+              <Badge variant="outline" className="text-[10px]">
+                agent: {validation_agent}
+              </Badge>
+            )}
+            {typeof validation_fixture_id === "string" && validation_fixture_id.trim() && (
+              <Badge variant="outline" className="text-[10px]">
+                fixture: {validation_fixture_id}
+              </Badge>
+            )}
+          </div>
+          <p className="mt-1 text-[11px] leading-relaxed text-muted-foreground">
+            {validationMeta.description}
+          </p>
+          {typeof validation_evidence_ref === "string" && validation_evidence_ref.trim() && (
+            <p className="mt-1 text-[10px] font-mono text-muted-foreground/70">
+              {validation_evidence_ref}
+            </p>
+          )}
+        </div>
+      )}
+
       {/* Summary bar */}
       <div className="flex items-center gap-3 flex-wrap">
         {improved !== undefined && (
@@ -730,6 +796,7 @@ export function EvidenceViewer({
   const latestStep = steps[steps.length - 1] ?? null;
   const lifecycleLabel = steps.map((step) => step.action.replace("_", " ")).join(" -> ");
   const outcome = getOutcomePresentation(latestStep?.action);
+  const validationMeta = getValidationModeMeta(latestStep?.validation_mode);
   const latestProposalConfidence = useMemo(() => {
     for (let i = proposalEntries.length - 1; i >= 0; i--) {
       if (proposalEntries[i].confidence !== null) {
@@ -824,6 +891,16 @@ export function EvidenceViewer({
             <Badge variant="outline" className="text-[10px]">
               {entries.length} evidence {entries.length === 1 ? "row" : "rows"}
             </Badge>
+            {validationMeta && (
+              <Badge variant={validationMeta.variant} className="text-[10px]">
+                {validationMeta.label}
+              </Badge>
+            )}
+            {latestStep?.validation_fixture_id && (
+              <Badge variant="outline" className="text-[10px]">
+                fixture: {latestStep.validation_fixture_id}
+              </Badge>
+            )}
             {latestProposalConfidence != null && (
               <Badge variant="secondary" className="text-[10px]">
                 {Math.round(latestProposalConfidence * 100)}% confidence
@@ -831,6 +908,12 @@ export function EvidenceViewer({
             )}
           </div>
 
+          {validationMeta && (
+            <p className="text-[11px] leading-relaxed text-muted-foreground">
+              {validationMeta.description}
+            </p>
+          )}
+
           <div className="flex flex-wrap items-center gap-2 text-[11px] text-muted-foreground">
             <span className="font-headline uppercase tracking-[0.16em] text-muted-foreground/80">
               Lifecycle

package/packages/ui/src/components/EvolutionTimeline.tsx

@@ -10,13 +10,14 @@ import {
   ChevronRightIcon,
 } from "lucide-react";
 import { useState } from "react";
+import type { ReactNode } from "react";
 
 import { timeAgo } from "../lib/format";
 import { cn } from "../lib/utils";
 import { Badge } from "../primitives/badge";
 import type { EvalSnapshot, EvolutionEntry } from "../types";
 
-const ACTION_ICON: Record<string, React.ReactNode> = {
+const ACTION_ICON: Record<string, ReactNode> = {
   created: <CircleDotIcon className="size-3.5" />,
   validated: <ShieldCheckIcon className="size-3.5" />,
   deployed: <RocketIcon className="size-3.5" />,
@@ -70,6 +71,21 @@ interface Props {
   onSelect: (proposalId: string) => void;
 }
 
+function validationModeBadge(
+  mode?: string | null,
+): { label: string; variant: "default" | "secondary" | "outline" } | null {
+  switch (mode) {
+    case "host_replay":
+      return { label: "replay", variant: "default" };
+    case "llm_judge":
+      return { label: "judge", variant: "secondary" };
+    case "structural_guard":
+      return { label: "structural", variant: "outline" };
+    default:
+      return null;
+  }
+}
+
 /** Group evolution entries by proposal_id, ordered newest-first. */
 function groupByProposal(entries: EvolutionEntry[]) {
   const map = new Map<string, EvolutionEntry[]>();
@@ -201,6 +217,7 @@ export function EvolutionTimeline({ entries, selectedProposalId, onSelect }: Pro
           const lineColor = ACTION_LINE[terminal] ?? "bg-border";
           const isLast = groupIdx === groups.length - 1;
           const snapshot = findEvalSnapshot(steps);
+          const validationBadge = validationModeBadge(lastStep.validation_mode);
 
           return (
             <div key={proposalId} className="relative flex gap-3">
@@ -245,6 +262,11 @@ export function EvolutionTimeline({ entries, selectedProposalId, onSelect }: Pro
                   <span className="text-[10px] text-muted-foreground">
                     {timeAgo(lastStep.timestamp)}
                   </span>
+                  {validationBadge && (
+                    <Badge variant={validationBadge.variant} className="text-[9px] uppercase">
+                      {validationBadge.label}
+                    </Badge>
+                  )}
                 </div>
                 {/* Pass rate delta from eval snapshot */}
                 {snapshot && (

package/packages/ui/src/types.ts

@@ -31,6 +31,10 @@ export interface EvolutionEntry {
   action: string;
   details: string;
   eval_snapshot?: EvalSnapshot | null;
+  validation_mode?: "structural_guard" | "host_replay" | "llm_judge" | null;
+  validation_agent?: string | null;
+  validation_fixture_id?: string | null;
+  validation_evidence_ref?: string | null;
 }
 
 export interface UnmatchedQuery {

package/skill/Workflows/Composability.md

@@ -104,6 +104,19 @@ This is for packaging questions like:
 - "Are my sibling skills competing for the same user intent?"
 - "Should I stop evolving these independently and redesign the family?"
 
+When trusted telemetry is sparse, the same command also emits a
+`cold_start_suspicion` block. That is a weaker, earlier signal based on the
+installed skill surfaces:
+
+1. Frontmatter / top-level description similarity
+2. Overlap in `## When to Use` language
+3. Shared command surface (for example, siblings that both wrap `mentor search`)
+4. Synthetic sibling-confusion probes derived from those overlapping surfaces
+
+Treat `cold_start_suspicion.candidate` as architecture suspicion, not proof.
+It is meant to tell you "this family may want a parent skill" before enough
+real usage exists to confirm it through trusted positive-query overlap.
+
 ## Steps
 
 ### 1. Run Analysis
@@ -140,6 +153,7 @@ Interpretation:
 
 - `consolidation_candidate: false` means keep improving the sibling descriptions/workflows separately
 - `consolidation_candidate: true` means the problem is likely packaging, not just wording
+- `cold_start_suspicion.candidate: true` means installed skill surfaces already look suspicious even though trusted telemetry is still sparse
 - `refactor_proposal` is a draft for human review only; do not auto-deploy a family rewrite
 
 ## Subagent Escalation
@@ -173,4 +187,4 @@ resolution plan with trigger ownership recommendations.
 
 **"Should I consolidate this sibling skill family?"**
 
-> Run `selftune eval family-overlap` and look for `consolidation_candidate` plus the `refactor_proposal`.
+> Run `selftune eval family-overlap` and look for `consolidation_candidate` when you have live evidence, or `cold_start_suspicion` when you only have installed skill surfaces plus cold-start evals.

package/skill/Workflows/Evolve.md

@@ -76,6 +76,29 @@ The evolution process writes multiple audit entries:
 | `validated` | Proposal tested against eval set | `eval_snapshot` with before/after pass rates      |
 | `deployed`  | Updated SKILL.md written to disk | `eval_snapshot` with final rates                  |
 
+Routing/body validation may also carry provenance fields such as:
+
+- `validation_mode` — `llm_judge`, `host_replay`, or `structural_guard`
+- `validation_agent` — which host/agent performed the validation
+- `validation_fixture_id` — fixture identifier when replay-backed validation is used
+- `before_pass_rate` / `after_pass_rate` — only present when trigger validation actually ran; structural-guard exits do not emit synthetic pass rates
+
+Most evolve runs today still validate through `llm_judge`. Routing evolution now
+auto-builds a replay fixture from the target skill plus installed sibling
+skills in the same registry, so replay-backed validation is preferred whenever
+that local fixture can be constructed because it captures host-style routing
+behavior instead of model judgment.
+
+The current replay path is fixture-backed: it evaluates the target routing table
+against the installed target/competing skill surfaces in a controlled replay
+fixture and records per-entry evidence. That is still a stronger signal than a
+free-form judge prompt, but you should describe it as replay-backed validation,
+not as live operator telemetry.
+
+Replay parsing is intentionally conservative: unreadable skill files degrade to
+empty surfaces instead of throwing, and malformed routing rows with empty
+trigger cells are ignored rather than treated as valid triggers.
+
 ## Parsing Instructions
 
 ### Track Evolution Progress