selftune 0.2.23 → 0.2.25

package/cli/selftune/localdb/queries/staging.ts

@@ -0,0 +1,270 @@
+import type { Database } from "bun:sqlite";
+
+import { safeParseJson } from "./json.js";
+
+export function getCreatorContributionStagingCounts(db: Database): Array<{
+  skill_name: string;
+  pending_count: number;
+}> {
+  return db
+    .query(
+      `SELECT skill_name, COUNT(*) AS pending_count
+       FROM creator_contribution_staging
+       WHERE status = 'pending'
+       GROUP BY skill_name
+       ORDER BY skill_name`,
+    )
+    .all() as Array<{
+    skill_name: string;
+    pending_count: number;
+  }>;
+}
+
+export interface CreatorContributionRelayStats {
+  pending: number;
+  sending: number;
+  sent: number;
+  failed: number;
+}
+
+export interface CreatorContributionStagingRow {
+  id: number;
+  dedupe_key: string;
+  skill_name: string;
+  creator_id: string;
+  payload_json: string;
+  status: string;
+  staged_at: string;
+  updated_at: string;
+  last_error: string | null;
+}
+
+export function getCreatorContributionRelayStats(db: Database): CreatorContributionRelayStats {
+  const row = db
+    .query(
+      `SELECT
+         COALESCE(SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END), 0) AS pending,
+         COALESCE(SUM(CASE WHEN status = 'sending' THEN 1 ELSE 0 END), 0) AS sending,
+         COALESCE(SUM(CASE WHEN status = 'sent' THEN 1 ELSE 0 END), 0) AS sent,
+         COALESCE(SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END), 0) AS failed
+       FROM creator_contribution_staging`,
+    )
+    .get() as CreatorContributionRelayStats | null;
+  return row ?? { pending: 0, sending: 0, sent: 0, failed: 0 };
+}
+
+export function getPendingCreatorContributionRows(
+  db: Database,
+  limit = 50,
+): CreatorContributionStagingRow[] {
+  return db
+    .query(
+      `SELECT id, dedupe_key, skill_name, creator_id, payload_json, status, staged_at, updated_at, last_error
+       FROM creator_contribution_staging
+       WHERE status = 'pending'
+       ORDER BY id ASC
+       LIMIT ?`,
+    )
+    .all(limit) as CreatorContributionStagingRow[];
+}
+
+export function queryCanonicalRecordsForStaging(db: Database): Record<string, unknown>[] {
+  const records: Record<string, unknown>[] = [];
+
+  const sessions = db
+    .query(
+      `SELECT session_id, started_at, ended_at, platform, model, completion_status,
+              source_session_kind, agent_cli, workspace_path, repo_remote, branch,
+              schema_version, normalized_at, normalizer_version, capture_mode, raw_source_ref
+       FROM sessions ORDER BY normalized_at`,
+    )
+    .all() as Array<Record<string, unknown>>;
+  const sessionById = new Map(sessions.map((session) => [session.session_id as string, session]));
+
+  for (const session of sessions) {
+    records.push({
+      record_kind: "session",
+      schema_version: session.schema_version ?? undefined,
+      normalizer_version: session.normalizer_version ?? undefined,
+      normalized_at: session.normalized_at ?? undefined,
+      platform: session.platform ?? undefined,
+      capture_mode: session.capture_mode ?? undefined,
+      raw_source_ref: safeParseJson(session.raw_source_ref as string | null) ?? undefined,
+      source_session_kind: session.source_session_kind ?? undefined,
+      session_id: session.session_id,
+      started_at: session.started_at ?? undefined,
+      ended_at: session.ended_at ?? undefined,
+      model: session.model ?? undefined,
+      completion_status: session.completion_status ?? undefined,
+      agent_cli: session.agent_cli ?? undefined,
+      workspace_path: session.workspace_path ?? undefined,
+      repo_remote: session.repo_remote ?? undefined,
+      branch: session.branch ?? undefined,
+    });
+  }
+
+  const prompts = db
+    .query(
+      `SELECT prompt_id, session_id, occurred_at, prompt_kind, is_actionable, prompt_index, prompt_text,
+              schema_version, platform, normalized_at, normalizer_version, capture_mode, raw_source_ref
+       FROM prompts ORDER BY occurred_at`,
+    )
+    .all() as Array<Record<string, unknown>>;
+  for (const prompt of prompts) {
+    const sessionEnvelope = sessionById.get(prompt.session_id as string);
+    records.push({
+      record_kind: "prompt",
+      schema_version: prompt.schema_version ?? sessionEnvelope?.schema_version ?? undefined,
+      normalizer_version:
+        prompt.normalizer_version ?? sessionEnvelope?.normalizer_version ?? undefined,
+      normalized_at: prompt.normalized_at ?? sessionEnvelope?.normalized_at ?? undefined,
+      platform: prompt.platform ?? sessionEnvelope?.platform ?? undefined,
+      capture_mode: prompt.capture_mode ?? sessionEnvelope?.capture_mode ?? undefined,
+      raw_source_ref:
+        safeParseJson(prompt.raw_source_ref as string | null) ??
+        safeParseJson(sessionEnvelope?.raw_source_ref as string | null) ??
+        undefined,
+      source_session_kind: sessionEnvelope?.source_session_kind ?? undefined,
+      session_id: prompt.session_id,
+      prompt_id: prompt.prompt_id,
+      occurred_at: prompt.occurred_at,
+      prompt_text: prompt.prompt_text,
+      prompt_kind: prompt.prompt_kind,
+      is_actionable: (prompt.is_actionable as number) === 1,
+      prompt_index: prompt.prompt_index ?? undefined,
+    });
+  }
+
+  const invocations = db
+    .query(
+      `SELECT skill_invocation_id, session_id, occurred_at, skill_name, skill_path, invocation_mode,
+              triggered, confidence, tool_name, matched_prompt_id, agent_type,
+              schema_version, platform, normalized_at, normalizer_version, capture_mode, raw_source_ref
+       FROM skill_invocations ORDER BY occurred_at`,
+    )
+    .all() as Array<Record<string, unknown>>;
+  for (const invocation of invocations) {
+    const sessionEnvelope = sessionById.get(invocation.session_id as string);
+    records.push({
+      record_kind: "skill_invocation",
+      schema_version: invocation.schema_version ?? sessionEnvelope?.schema_version ?? undefined,
+      normalizer_version:
+        invocation.normalizer_version ?? sessionEnvelope?.normalizer_version ?? undefined,
+      normalized_at: invocation.normalized_at ?? sessionEnvelope?.normalized_at ?? undefined,
+      platform: invocation.platform ?? sessionEnvelope?.platform ?? undefined,
+      capture_mode: invocation.capture_mode ?? sessionEnvelope?.capture_mode ?? undefined,
+      raw_source_ref:
+        safeParseJson(invocation.raw_source_ref as string | null) ??
+        safeParseJson(sessionEnvelope?.raw_source_ref as string | null) ??
+        undefined,
+      source_session_kind: sessionEnvelope?.source_session_kind ?? undefined,
+      session_id: invocation.session_id,
+      skill_invocation_id: invocation.skill_invocation_id,
+      occurred_at: invocation.occurred_at,
+      skill_name: invocation.skill_name,
+      skill_path: invocation.skill_path ?? undefined,
+      invocation_mode: invocation.invocation_mode,
+      triggered: (invocation.triggered as number) === 1,
+      confidence: invocation.confidence,
+      tool_name: invocation.tool_name ?? undefined,
+      matched_prompt_id: invocation.matched_prompt_id ?? undefined,
+      agent_type: invocation.agent_type ?? undefined,
+    });
+  }
+
+  const facts = db
+    .query(
+      `SELECT id AS execution_fact_id, session_id, occurred_at, prompt_id, tool_calls_json, total_tool_calls,
+              assistant_turns, errors_encountered, input_tokens, output_tokens,
+              duration_ms, completion_status,
+              schema_version, platform, normalized_at, normalizer_version, capture_mode, raw_source_ref
+       FROM execution_facts ORDER BY occurred_at`,
+    )
+    .all() as Array<Record<string, unknown>>;
+  for (const fact of facts) {
+    const sessionEnvelope = sessionById.get(fact.session_id as string);
+    records.push({
+      record_kind: "execution_fact",
+      schema_version: fact.schema_version ?? sessionEnvelope?.schema_version ?? undefined,
+      normalizer_version:
+        fact.normalizer_version ?? sessionEnvelope?.normalizer_version ?? undefined,
+      normalized_at: fact.normalized_at ?? sessionEnvelope?.normalized_at ?? undefined,
+      platform: fact.platform ?? sessionEnvelope?.platform ?? undefined,
+      capture_mode: fact.capture_mode ?? sessionEnvelope?.capture_mode ?? undefined,
+      raw_source_ref:
+        safeParseJson(fact.raw_source_ref as string | null) ??
+        safeParseJson(sessionEnvelope?.raw_source_ref as string | null) ??
+        undefined,
+      source_session_kind: sessionEnvelope?.source_session_kind ?? undefined,
+      session_id: fact.session_id,
+      execution_fact_id: String(fact.execution_fact_id),
+      occurred_at: fact.occurred_at,
+      prompt_id: fact.prompt_id ?? undefined,
+      tool_calls_json: safeParseJson(fact.tool_calls_json as string | null) ?? {},
+      total_tool_calls: fact.total_tool_calls,
+      assistant_turns: fact.assistant_turns,
+      errors_encountered: fact.errors_encountered,
+      input_tokens: fact.input_tokens ?? undefined,
+      output_tokens: fact.output_tokens ?? undefined,
+      duration_ms: fact.duration_ms ?? undefined,
+      completion_status: fact.completion_status ?? undefined,
+    });
+  }
+
+  return records;
+}
+
+export function getLastUploadError(
+  db: Database,
+): { last_error: string | null; updated_at: string } | null {
+  try {
+    const row = db
+      .query(
+        `SELECT last_error, updated_at
+         FROM upload_queue
+         WHERE status = 'failed'
+         ORDER BY updated_at DESC
+         LIMIT 1`,
+      )
+      .get() as { last_error: string | null; updated_at: string } | null;
+    return row ?? null;
+  } catch {
+    return null;
+  }
+}
+
+export function getLastUploadSuccess(db: Database): { updated_at: string } | null {
+  try {
+    const row = db
+      .query(
+        `SELECT updated_at
+         FROM upload_queue
+         WHERE status = 'sent'
+         ORDER BY updated_at DESC
+         LIMIT 1`,
+      )
+      .get() as { updated_at: string } | null;
+    return row ?? null;
+  } catch {
+    return null;
+  }
+}
+
+export function getOldestPendingAge(db: Database): number | null {
+  try {
+    const row = db
+      .query(
+        `SELECT created_at
+         FROM upload_queue
+         WHERE status = 'pending'
+         ORDER BY created_at ASC
+         LIMIT 1`,
+      )
+      .get() as { created_at: string } | null;
+    if (!row) return null;
+    const ageMs = Date.now() - new Date(row.created_at).getTime();
+    return Math.floor(ageMs / 1000);
+  } catch {
+    return null;
+  }
+}

package/cli/selftune/localdb/queries/trust.ts

@@ -0,0 +1,392 @@
+import type { Database } from "bun:sqlite";
+
+import type { AttentionItem, AutonomousDecision, DecisionKind } from "../../dashboard-contract.js";
+import { safeParseJson } from "./json.js";
+import { getPendingProposals } from "./evolution.js";
+
+export interface SkillTrustSummary {
+  skill_name: string;
+  total_checks: number;
+  triggered_count: number;
+  miss_rate: number;
+  system_like_count: number;
+  system_like_rate: number;
+  prompt_link_rate: number;
+  latest_action: string | null;
+  pass_rate: number;
+  last_seen: string | null;
+}
+
+export interface TrustedSkillObservationRow {
+  skill_name: string;
+  session_id: string;
+  occurred_at: string | null;
+  triggered: number;
+  matched_prompt_id: string | null;
+  confidence: number | null;
+  invocation_mode: string | null;
+  query_text: string;
+}
+
+export function queryTrustedSkillObservationRows(db: Database): TrustedSkillObservationRow[] {
+  const SYSTEM_LIKE_PREFIXES = ["<system_instruction>", "<system-instruction>", "<command-name>"];
+  const INTERNAL_EVAL_MARKERS = [
+    "you are an evaluation assistant",
+    "you are a skill description optimizer",
+    "would each query trigger this skill",
+    "propose an improved description",
+    "failure patterns:",
+    "output only valid json",
+  ];
+  const isSystemLike = (text: string | null | undefined): boolean => {
+    if (!text) return false;
+    const trimmed = text.trimStart();
+    return SYSTEM_LIKE_PREFIXES.some((prefix) => trimmed.startsWith(prefix));
+  };
+  const isInternalSelftunePrompt = (
+    text: string | null | undefined,
+    promptKind: string | null | undefined,
+  ): boolean => {
+    if (!text) return false;
+    const lowered = text.toLowerCase();
+    return (
+      promptKind === "meta" && INTERNAL_EVAL_MARKERS.some((marker) => lowered.includes(marker))
+    );
+  };
+  const isPollutingPrompt = (
+    text: string | null | undefined,
+    promptKind: string | null | undefined,
+  ): boolean => isSystemLike(text) || isInternalSelftunePrompt(text, promptKind);
+  const classifyObservationKind = (
+    skillInvocationId: string,
+    captureMode: string | null,
+    triggered: number,
+    rawSourceRefJson: string | null,
+  ): "canonical" | "repaired_trigger" | "repaired_contextual_miss" | "legacy_materialized" => {
+    if (skillInvocationId.includes(":su:")) return "legacy_materialized";
+    if (captureMode === "repair") {
+      const rawSourceRef = safeParseJson(rawSourceRefJson) as {
+        metadata?: { miss_type?: string };
+      } | null;
+      if (triggered === 0 && rawSourceRef?.metadata?.miss_type === "contextual_read") {
+        return "repaired_contextual_miss";
+      }
+      return "repaired_trigger";
+    }
+    return "canonical";
+  };
+  const normalizeQueryForGrouping = (query: string) =>
+    query.replace(/\s+/g, " ").trim().toLowerCase();
+
+  const rows = db
+    .query(
+      `SELECT
+         si.skill_name,
+         si.session_id,
+         si.occurred_at,
+         si.triggered,
+         si.matched_prompt_id,
+         si.confidence,
+         si.invocation_mode,
+         si.skill_invocation_id,
+         si.capture_mode,
+         si.raw_source_ref,
+         si.query,
+         p.prompt_text,
+         p.prompt_kind
+       FROM skill_invocations si
+       LEFT JOIN prompts p ON si.matched_prompt_id = p.prompt_id`,
+    )
+    .all() as Array<{
+    skill_name: string;
+    session_id: string;
+    occurred_at: string | null;
+    triggered: number;
+    matched_prompt_id: string | null;
+    confidence: number | null;
+    invocation_mode: string | null;
+    skill_invocation_id: string;
+    capture_mode: string | null;
+    raw_source_ref: string | null;
+    query: string | null;
+    prompt_text: string | null;
+    prompt_kind: string | null;
+  }>;
+
+  const bySkill = new Map<
+    string,
+    Array<{
+      skill_name: string;
+      session_id: string;
+      occurred_at: string | null;
+      triggered: number;
+      matched_prompt_id: string | null;
+      confidence: number | null;
+      invocation_mode: string | null;
+      queryText: string;
+      observation_kind:
+        | "canonical"
+        | "repaired_trigger"
+        | "repaired_contextual_miss"
+        | "legacy_materialized";
+      groupKey: string;
+    }>
+  >();
+  const trustedRows: TrustedSkillObservationRow[] = [];
+
+  for (const row of rows) {
+    const queryText = row.query || row.prompt_text || "";
+    const pollutionText = row.prompt_text || row.query || "";
+    const observationKind = classifyObservationKind(
+      row.skill_invocation_id,
+      row.capture_mode,
+      row.triggered,
+      row.raw_source_ref,
+    );
+    if (isPollutingPrompt(pollutionText, row.prompt_kind)) continue;
+    if (observationKind === "legacy_materialized") continue;
+
+    const normalizedQuery = normalizeQueryForGrouping(queryText);
+    const groupKey =
+      normalizedQuery.length > 0
+        ? `${row.session_id}::${normalizedQuery}`
+        : `${row.skill_invocation_id}`;
+    const observation = {
+      skill_name: row.skill_name,
+      session_id: row.session_id,
+      occurred_at: row.occurred_at,
+      triggered: row.triggered,
+      matched_prompt_id: row.matched_prompt_id,
+      confidence: row.confidence,
+      invocation_mode: row.invocation_mode,
+      queryText,
+      observation_kind: observationKind,
+      groupKey,
+    };
+    const existing = bySkill.get(row.skill_name);
+    if (existing) existing.push(observation);
+    else bySkill.set(row.skill_name, [observation]);
+  }
+
+  for (const skillRows of bySkill.values()) {
+    const grouped = new Map<string, typeof skillRows>();
+    for (const row of skillRows) {
+      const existing = grouped.get(row.groupKey);
+      if (existing) existing.push(row);
+      else grouped.set(row.groupKey, [row]);
+    }
+
+    const deduped = [...grouped.values()].map((group) => {
+      const sorted = [...group].sort((a, b) => {
+        const aScore =
+          (a.triggered === 1 ? 100 : 0) +
+          (a.observation_kind === "canonical" ? 20 : 0) +
+          (a.observation_kind === "repaired_trigger" ? 15 : 0);
+        const bScore =
+          (b.triggered === 1 ? 100 : 0) +
+          (b.observation_kind === "canonical" ? 20 : 0) +
+          (b.observation_kind === "repaired_trigger" ? 15 : 0);
+        if (aScore !== bScore) return bScore - aScore;
+        return (b.occurred_at ?? "").localeCompare(a.occurred_at ?? "");
+      });
+      return sorted[0]!;
+    });
+
+    trustedRows.push(
+      ...deduped.map((row) => ({
+        skill_name: row.skill_name,
+        session_id: row.session_id,
+        occurred_at: row.occurred_at,
+        triggered: row.triggered,
+        matched_prompt_id: row.matched_prompt_id,
+        confidence: row.confidence,
+        invocation_mode: row.invocation_mode,
+        query_text: row.queryText,
+      })),
+    );
+  }
+
+  return trustedRows;
+}
+
+export function getSkillTrustSummaries(db: Database): SkillTrustSummary[] {
+  const rows = queryTrustedSkillObservationRows(db);
+  const auditRows = db
+    .query(
+      `SELECT skill_name, action, timestamp
+       FROM evolution_audit
+       WHERE skill_name IS NOT NULL
+       ORDER BY timestamp DESC`,
+    )
+    .all() as Array<{
+    skill_name: string | null;
+    action: string;
+    timestamp: string;
+  }>;
+
+  const latestActions = new Map<string, string>();
+  for (const row of auditRows) {
+    if (row.skill_name && !latestActions.has(row.skill_name)) {
+      latestActions.set(row.skill_name, row.action);
+    }
+  }
+
+  const rowsBySkill = new Map<string, typeof rows>();
+  for (const row of rows) {
+    const existing = rowsBySkill.get(row.skill_name);
+    if (existing) existing.push(row);
+    else rowsBySkill.set(row.skill_name, [row]);
+  }
+
+  const summaries: SkillTrustSummary[] = [];
+  for (const [skillName, skillRows] of rowsBySkill.entries()) {
+    const total = skillRows.length;
+    const triggered = skillRows.filter((row) => row.triggered === 1).length;
+    const promptLinked = skillRows.filter((row) => row.matched_prompt_id != null).length;
+    const lastSeen =
+      skillRows
+        .map((row) => row.occurred_at)
+        .filter((value): value is string => value != null)
+        .sort((a, b) => b.localeCompare(a))[0] ?? null;
+
+    summaries.push({
+      skill_name: skillName,
+      total_checks: total,
+      triggered_count: triggered,
+      miss_rate: total > 0 ? (total - triggered) / total : 0,
+      system_like_count: 0,
+      system_like_rate: 0,
+      prompt_link_rate: total > 0 ? promptLinked / total : 0,
+      latest_action: latestActions.get(skillName) ?? null,
+      pass_rate: total > 0 ? triggered / total : 0,
+      last_seen: lastSeen,
+    });
+  }
+
+  return summaries;
+}
+
+export function getAttentionQueue(db: Database): AttentionItem[] {
+  const summaries = getSkillTrustSummaries(db);
+  const pending = getPendingProposals(db);
+  const pendingSkills = new Set(pending.map((proposal) => proposal.skill_name).filter(Boolean));
+
+  const items: AttentionItem[] = [];
+
+  for (const summary of summaries) {
+    if (summary.latest_action === "rolled_back") {
+      items.push({
+        skill_name: summary.skill_name,
+        category: "needs_review",
+        severity: "critical",
+        reason: "Rolled back after deployment",
+        recommended_action: "Review rollback evidence and decide whether to re-evolve",
+        timestamp: summary.last_seen ?? "",
+      });
+      continue;
+    }
+
+    if (pendingSkills.has(summary.skill_name)) {
+      items.push({
+        skill_name: summary.skill_name,
+        category: "needs_review",
+        severity: "info",
+        reason: "Proposal awaiting review",
+        recommended_action: "Review and approve or reject the pending proposal",
+        timestamp: summary.last_seen ?? "",
+      });
+      continue;
+    }
+
+    if (summary.total_checks < 5) continue;
+
+    if (summary.miss_rate > 0.1) {
+      items.push({
+        skill_name: summary.skill_name,
+        category: "regression",
+        severity: "warning",
+        reason: `High miss rate (${Math.round(summary.miss_rate * 100)}%)`,
+        recommended_action: "Review missed invocations and consider evolving the skill description",
+        timestamp: summary.last_seen ?? "",
+      });
+      continue;
+    }
+
+    if (summary.system_like_rate > 0.1) {
+      items.push({
+        skill_name: summary.skill_name,
+        category: "polluted",
+        severity: "warning",
+        reason: `Possible telemetry pollution (${Math.round(summary.system_like_rate * 100)}% system-like)`,
+        recommended_action: "Inspect prompts for system-injected noise",
+        timestamp: summary.last_seen ?? "",
+      });
+    }
+  }
+
+  return items;
+}
+
+export function getRecentDecisions(db: Database, limit = 20): AutonomousDecision[] {
+  const rows = db
+    .query(
+      `SELECT timestamp, proposal_id, skill_name, action, details, eval_snapshot_json
+       FROM evolution_audit
+       WHERE timestamp >= datetime('now', '-7 days')
+       ORDER BY timestamp DESC
+       LIMIT ?`,
+    )
+    .all(limit) as Array<{
+    timestamp: string;
+    proposal_id: string;
+    skill_name: string | null;
+    action: string;
+    details: string;
+    eval_snapshot_json: string | null;
+  }>;
+
+  return rows
+    .filter((row) => row.skill_name != null)
+    .flatMap((row) => {
+      const evalSnapshot = safeParseJson(row.eval_snapshot_json) as {
+        regressions?: unknown[];
+      } | null;
+
+      let kind: DecisionKind | null;
+      switch (row.action) {
+        case "proposed":
+        case "created":
+          kind = "proposal_created";
+          break;
+        case "rejected":
+          kind = "proposal_rejected";
+          break;
+        case "validated":
+          kind =
+            evalSnapshot?.regressions && evalSnapshot.regressions.length > 0
+              ? "validation_failed"
+              : "proposal_created";
+          break;
+        case "deployed":
+          kind = "proposal_deployed";
+          break;
+        case "rolled_back":
+          kind = "rollback_triggered";
+          break;
+        default:
+          kind = null;
+      }
+
+      if (!kind) return [];
+
+      return [
+        {
+          timestamp: row.timestamp,
+          kind,
+          skill_name: row.skill_name!,
+          proposal_id: row.proposal_id,
+          summary: row.details ?? "",
+        },
+      ];
+    });
+}