selftune 0.2.16 → 0.2.19

package/cli/selftune/localdb/queries.ts

@@ -8,12 +8,25 @@
 import type { Database } from "bun:sqlite";
 
 import type {
+  AnalyticsResponse,
+  AttentionItem,
+  AutonomousDecision,
+  CommitRecord,
+  CommitSummary,
+  DecisionKind,
+  ExecutionMetrics,
   OrchestrateRunReport,
+  OverviewPaginatedPayload,
   OverviewPayload,
+  PaginatedResult,
+  PaginationCursor,
   PendingProposal,
   RecentActivityItem,
+  SkillReportPaginatedPayload,
   SkillReportPayload,
   SkillSummary,
+  SkillUsageRecord,
+  TelemetryRecord,
 } from "../dashboard-contract.js";
 
 /**
@@ -243,39 +256,417 @@ export function getSkillReportPayload(db: Database, skillName: string): SkillRep
   };
 }
 
+// -- Cursor-based paginated queries -------------------------------------------
+
+export interface OverviewPaginationOptions {
+  telemetry_cursor?: PaginationCursor | null;
+  telemetry_limit?: number;
+  skills_cursor?: PaginationCursor | null;
+  skills_limit?: number;
+}
+
+export interface SkillReportPaginationOptions {
+  invocations_cursor?: PaginationCursor | null;
+  invocations_limit?: number;
+}
+
 /**
- * Get a summary list of all skills with aggregated stats.
+ * Build a paginated overview payload from SQLite.
+ *
+ * Uses (timestamp, session_id) composite cursors for stable backward pagination.
+ * When no cursor is provided, returns the first page starting from most recent.
  */
-export function getSkillsList(db: Database): SkillSummary[] {
-  const rows = db
+export function getOverviewPayloadPaginated(
+  db: Database,
+  opts: OverviewPaginationOptions = {},
+): OverviewPaginatedPayload {
+  const telemetryLimit = opts.telemetry_limit ?? 1000;
+  const skillsLimit = opts.skills_limit ?? 2000;
+
+  // Paginated telemetry
+  const telemetry_page = paginateTelemetry(db, telemetryLimit, opts.telemetry_cursor ?? null);
+
+  // Paginated skill invocations
+  const skills_page = paginateSkillInvocations(db, skillsLimit, opts.skills_cursor ?? null);
+
+  // Non-paginated parts reuse existing logic
+  const evolution = db
+    .query(
+      `SELECT timestamp, proposal_id, skill_name, action, details
+       FROM evolution_audit
+       ORDER BY timestamp DESC
+       LIMIT 500`,
+    )
+    .all() as Array<{
+    timestamp: string;
+    proposal_id: string;
+    skill_name: string | null;
+    action: string;
+    details: string;
+  }>;
+
+  const counts = db
     .query(
       `SELECT
-         si.skill_name,
-         COALESCE(
-           (SELECT s2.skill_scope FROM skill_invocations s2
-            WHERE s2.skill_name = si.skill_name AND s2.skill_scope IS NOT NULL
-            ORDER BY s2.occurred_at DESC LIMIT 1),
-           (SELECT su.skill_scope FROM skill_usage su
-            WHERE su.skill_name = si.skill_name AND su.skill_scope IS NOT NULL
-            ORDER BY su.timestamp DESC LIMIT 1)
-         ) as skill_scope,
-         COUNT(*) as total_checks,
-         SUM(CASE WHEN si.triggered = 1 THEN 1 ELSE 0 END) as triggered_count,
-         COUNT(DISTINCT si.session_id) as unique_sessions,
-         MAX(si.occurred_at) as last_seen
+         (SELECT COUNT(*) FROM session_telemetry) as telemetry,
+         (SELECT COUNT(*) FROM skill_invocations) as skills,
+         (SELECT COUNT(*) FROM evolution_audit) as evolution,
+         (SELECT COUNT(*) FROM evolution_evidence) as evidence,
+         (SELECT COUNT(*) FROM sessions) as sessions,
+         (SELECT COUNT(*) FROM prompts) as prompts`,
+    )
+    .get() as {
+    telemetry: number;
+    skills: number;
+    evolution: number;
+    evidence: number;
+    sessions: number;
+    prompts: number;
+  };
+
+  const unmatchedRows = db
+    .query(
+      `SELECT si.occurred_at AS timestamp, si.session_id, si.query
        FROM skill_invocations si
-       GROUP BY si.skill_name
-       ORDER BY total_checks DESC`,
+       WHERE si.triggered = 0
+         AND NOT EXISTS (
+           SELECT 1 FROM skill_invocations si2
+           WHERE si2.query = si.query AND si2.triggered = 1
+         )
+       ORDER BY si.occurred_at DESC
+       LIMIT 500`,
     )
-    .all() as Array<{
+    .all() as Array<{ timestamp: string; session_id: string; query: string }>;
+
+  const pending_proposals = getPendingProposals(db);
+  const active_sessions = getActiveSessionCount(db);
+  const recent_activity = getRecentActivity(db);
+
+  return {
+    telemetry_page,
+    skills_page,
+    evolution,
+    counts,
+    unmatched_queries: unmatchedRows,
+    pending_proposals,
+    active_sessions,
+    recent_activity,
+  };
+}
+
+/**
+ * Build a paginated skill report payload for a specific skill.
+ *
+ * Uses (occurred_at, skill_invocation_id) composite cursor for the recent
+ * invocations sub-query. Non-paginated fields (usage stats, evidence, sessions)
+ * are returned in full.
+ */
+export function getSkillReportPayloadPaginated(
+  db: Database,
+  skillName: string,
+  opts: SkillReportPaginationOptions = {},
+): SkillReportPaginatedPayload {
+  const invocationsLimit = opts.invocations_limit ?? 100;
+
+  // Usage stats (unchanged)
+  const usageRow = db
+    .query(
+      `SELECT
+         COUNT(*) as total_checks,
+         SUM(CASE WHEN triggered = 1 THEN 1 ELSE 0 END) as triggered_count
+       FROM skill_invocations
+       WHERE skill_name = ?`,
+    )
+    .get(skillName) as { total_checks: number; triggered_count: number };
+
+  const total = usageRow.total_checks;
+  const triggered = usageRow.triggered_count;
+  const passRate = total > 0 ? triggered / total : 0;
+
+  // Paginated invocations
+  const invocations_page = paginateSkillReportInvocations(
+    db,
+    skillName,
+    invocationsLimit,
+    opts.invocations_cursor ?? null,
+  );
+
+  // Evidence (unchanged)
+  const evidenceRows = db
+    .query(
+      `SELECT proposal_id, target, stage, timestamp, rationale, confidence,
+              original_text, proposed_text, validation_json, details, eval_set_json
+       FROM evolution_evidence
+       WHERE skill_name = ?
+       ORDER BY timestamp DESC
+       LIMIT 200`,
+    )
+    .all(skillName) as Array<{
+    proposal_id: string;
+    target: string;
+    stage: string;
+    timestamp: string;
+    rationale: string | null;
+    confidence: number | null;
+    original_text: string | null;
+    proposed_text: string | null;
+    validation_json: string | null;
+    details: string | null;
+    eval_set_json: string | null;
+  }>;
+
+  const evidence = evidenceRows.map((row) => ({
+    proposal_id: row.proposal_id,
+    target: row.target,
+    stage: row.stage,
+    timestamp: row.timestamp,
+    rationale: row.rationale,
+    confidence: row.confidence,
+    original_text: row.original_text,
+    proposed_text: row.proposed_text,
+    validation: safeParseJson(row.validation_json),
+    details: row.details,
+    eval_set: safeParseJsonArray<Record<string, unknown>>(row.eval_set_json),
+  }));
+
+  const sessionsRow = db
+    .query(`SELECT COUNT(DISTINCT session_id) as c FROM skill_invocations WHERE skill_name = ?`)
+    .get(skillName) as { c: number };
+
+  return {
+    skill_name: skillName,
+    usage: {
+      total_checks: total,
+      triggered_count: triggered,
+      pass_rate: passRate,
+    },
+    invocations_page,
+    evidence,
+    sessions_with_skill: sessionsRow.c,
+  };
+}
+
+// -- Internal pagination helpers ------------------------------------------------
+
+function paginateTelemetry(
+  db: Database,
+  limit: number,
+  cursor: PaginationCursor | null,
+): PaginatedResult<TelemetryRecord> {
+  // Fetch one extra to detect has_more
+  const fetchLimit = limit + 1;
+
+  let rows: Array<{
+    timestamp: string;
+    session_id: string;
+    skills_triggered_json: string | null;
+    errors_encountered: number;
+    total_tool_calls: number;
+  }>;
+
+  if (cursor) {
+    rows = db
+      .query(
+        `SELECT timestamp, session_id, skills_triggered_json, errors_encountered, total_tool_calls
+         FROM session_telemetry
+         WHERE (timestamp < ? OR (timestamp = ? AND session_id < ?))
+         ORDER BY timestamp DESC, session_id DESC
+         LIMIT ?`,
+      )
+      .all(cursor.timestamp, cursor.timestamp, String(cursor.id), fetchLimit) as typeof rows;
+  } else {
+    rows = db
+      .query(
+        `SELECT timestamp, session_id, skills_triggered_json, errors_encountered, total_tool_calls
+         FROM session_telemetry
+         ORDER BY timestamp DESC, session_id DESC
+         LIMIT ?`,
+      )
+      .all(fetchLimit) as typeof rows;
+  }
+
+  const hasMore = rows.length > limit;
+  const pageRows = hasMore ? rows.slice(0, limit) : rows;
+
+  const items: TelemetryRecord[] = pageRows.map((row) => ({
+    timestamp: row.timestamp,
+    session_id: row.session_id,
+    skills_triggered: safeParseJsonArray<string>(row.skills_triggered_json),
+    errors_encountered: row.errors_encountered,
+    total_tool_calls: row.total_tool_calls,
+  }));
+
+  const lastItem = pageRows[pageRows.length - 1];
+  const next_cursor: PaginationCursor | null =
+    hasMore && lastItem ? { timestamp: lastItem.timestamp, id: lastItem.session_id } : null;
+
+  return { items, next_cursor, has_more: hasMore };
+}
+
+function paginateSkillInvocations(
+  db: Database,
+  limit: number,
+  cursor: PaginationCursor | null,
+): PaginatedResult<SkillUsageRecord> {
+  const fetchLimit = limit + 1;
+
+  let rows: Array<{
+    occurred_at: string;
+    session_id: string;
     skill_name: string;
-    skill_scope: string | null;
-    total_checks: number;
-    triggered_count: number;
-    unique_sessions: number;
-    last_seen: string | null;
+    skill_path: string;
+    query: string;
+    triggered: number;
+    source: string | null;
+    skill_invocation_id: string;
   }>;
 
+  if (cursor) {
+    rows = db
+      .query(
+        `SELECT occurred_at, session_id, skill_name, skill_path, query, triggered, source, skill_invocation_id
+         FROM skill_invocations
+         WHERE (occurred_at < ? OR (occurred_at = ? AND skill_invocation_id < ?))
+         ORDER BY occurred_at DESC, skill_invocation_id DESC
+         LIMIT ?`,
+      )
+      .all(cursor.timestamp, cursor.timestamp, String(cursor.id), fetchLimit) as typeof rows;
+  } else {
+    rows = db
+      .query(
+        `SELECT occurred_at, session_id, skill_name, skill_path, query, triggered, source, skill_invocation_id
+         FROM skill_invocations
+         ORDER BY occurred_at DESC, skill_invocation_id DESC
+         LIMIT ?`,
+      )
+      .all(fetchLimit) as typeof rows;
+  }
+
+  const hasMore = rows.length > limit;
+  const pageRows = hasMore ? rows.slice(0, limit) : rows;
+
+  const items: SkillUsageRecord[] = pageRows.map((row) => ({
+    timestamp: row.occurred_at,
+    session_id: row.session_id,
+    skill_name: row.skill_name,
+    skill_path: row.skill_path,
+    query: row.query,
+    triggered: row.triggered === 1,
+    source: row.source,
+  }));
+
+  const lastRow = pageRows[pageRows.length - 1];
+  const next_cursor: PaginationCursor | null =
+    hasMore && lastRow ? { timestamp: lastRow.occurred_at, id: lastRow.skill_invocation_id } : null;
+
+  return { items, next_cursor, has_more: hasMore };
+}
+
+function paginateSkillReportInvocations(
+  db: Database,
+  skillName: string,
+  limit: number,
+  cursor: PaginationCursor | null,
+): PaginatedResult<{
+  timestamp: string;
+  session_id: string;
+  query: string;
+  triggered: boolean;
+  source: string | null;
+}> {
+  const fetchLimit = limit + 1;
+
+  let rows: Array<{
+    occurred_at: string;
+    session_id: string;
+    query: string;
+    triggered: number;
+    source: string | null;
+    skill_invocation_id: string;
+  }>;
+
+  if (cursor) {
+    rows = db
+      .query(
+        `SELECT si.occurred_at, si.session_id, COALESCE(si.query, p.prompt_text) as query,
+                si.triggered, si.source, si.skill_invocation_id
+         FROM skill_invocations si
+         LEFT JOIN prompts p ON si.matched_prompt_id = p.prompt_id
+         WHERE si.skill_name = ?
+           AND (si.occurred_at < ? OR (si.occurred_at = ? AND si.skill_invocation_id < ?))
+         ORDER BY si.occurred_at DESC, si.skill_invocation_id DESC
+         LIMIT ?`,
+      )
+      .all(
+        skillName,
+        cursor.timestamp,
+        cursor.timestamp,
+        String(cursor.id),
+        fetchLimit,
+      ) as typeof rows;
+  } else {
+    rows = db
+      .query(
+        `SELECT si.occurred_at, si.session_id, COALESCE(si.query, p.prompt_text) as query,
+                si.triggered, si.source, si.skill_invocation_id
+         FROM skill_invocations si
+         LEFT JOIN prompts p ON si.matched_prompt_id = p.prompt_id
+         WHERE si.skill_name = ?
+         ORDER BY si.occurred_at DESC, si.skill_invocation_id DESC
+         LIMIT ?`,
+      )
+      .all(skillName, fetchLimit) as typeof rows;
+  }
+
+  const hasMore = rows.length > limit;
+  const pageRows = hasMore ? rows.slice(0, limit) : rows;
+
+  const items = pageRows.map((row) => ({
+    timestamp: row.occurred_at,
+    session_id: row.session_id,
+    query: row.query ?? "",
+    triggered: row.triggered === 1,
+    source: row.source,
+  }));
+
+  const lastRow = pageRows[pageRows.length - 1];
+  const next_cursor: PaginationCursor | null =
+    hasMore && lastRow ? { timestamp: lastRow.occurred_at, id: lastRow.skill_invocation_id } : null;
+
+  return { items, next_cursor, has_more: hasMore };
+}
+
+/**
+ * Get a summary list of all skills with aggregated stats.
+ */
+export function getSkillsList(db: Database): SkillSummary[] {
+  const trustedRows = queryTrustedSkillObservationRows(db);
+  const bySkill = new Map<
+    string,
+    Array<{
+      skill_name: string;
+      session_id: string;
+      occurred_at: string | null;
+      triggered: number;
+      matched_prompt_id: string | null;
+      confidence: number | null;
+    }>
+  >();
+
+  for (const row of trustedRows) {
+    const arr = bySkill.get(row.skill_name);
+    const base = {
+      skill_name: row.skill_name,
+      session_id: row.session_id,
+      occurred_at: row.occurred_at,
+      triggered: row.triggered,
+      matched_prompt_id: row.matched_prompt_id,
+      confidence: row.confidence,
+    };
+    if (arr) arr.push(base);
+    else bySkill.set(row.skill_name, [base]);
+  }
+
   // Get set of skill names with evidence
   const evidenceSkills = new Set(
     (
@@ -285,16 +676,214 @@ export function getSkillsList(db: Database): SkillSummary[] {
     ).map((r) => r.skill_name),
   );
 
-  return rows.map((row) => ({
+  const skillScopeRows = db
+    .query(
+      `SELECT
+         si.skill_name,
+         COALESCE(
+           (SELECT s2.skill_scope FROM skill_invocations s2
+            WHERE s2.skill_name = si.skill_name AND s2.skill_scope IS NOT NULL
+            ORDER BY s2.occurred_at DESC LIMIT 1),
+           (SELECT su.skill_scope FROM skill_usage su
+            WHERE su.skill_name = si.skill_name AND su.skill_scope IS NOT NULL
+            ORDER BY su.timestamp DESC LIMIT 1)
+         ) as skill_scope
+       FROM skill_invocations si
+       GROUP BY si.skill_name`,
+    )
+    .all() as Array<{ skill_name: string; skill_scope: string | null }>;
+  const scopeBySkill = new Map(skillScopeRows.map((row) => [row.skill_name, row.skill_scope]));
+
+  return [...bySkill.entries()]
+    .map(([skillName, rows]) => {
+      const totalChecks = rows.length;
+      const triggeredCount = rows.filter((row) => row.triggered === 1).length;
+      const uniqueSessions = new Set(rows.map((row) => row.session_id)).size;
+      const lastSeen =
+        rows
+          .map((row) => row.occurred_at)
+          .filter((value): value is string => value != null)
+          .sort((a, b) => b.localeCompare(a))[0] ?? null;
+      const withConfidence = rows.filter((row) => row.confidence != null);
+      const routingConfidence =
+        withConfidence.length > 0
+          ? withConfidence.reduce((sum, row) => sum + (row.confidence ?? 0), 0) /
+            withConfidence.length
+          : null;
+
+      return {
+        skill_name: skillName,
+        skill_scope: scopeBySkill.get(skillName) ?? null,
+        total_checks: totalChecks,
+        triggered_count: triggeredCount,
+        pass_rate: totalChecks > 0 ? triggeredCount / totalChecks : 0,
+        unique_sessions: uniqueSessions,
+        last_seen: lastSeen,
+        has_evidence: evidenceSkills.has(skillName),
+        routing_confidence: routingConfidence,
+        confidence_coverage: totalChecks > 0 ? withConfidence.length / totalChecks : 0,
+      };
+    })
+    .sort((a, b) => b.total_checks - a.total_checks);
+}
+
+/**
+ * Build the performance analytics payload from SQLite.
+ * Powers the GET /api/v2/analytics endpoint.
+ */
+export function getAnalyticsPayload(db: Database): AnalyticsResponse {
+  const trustedRows = queryTrustedSkillObservationRows(db);
+  const today = new Date();
+  const dateKey = (value: string | null): string | null => {
+    if (!value) return null;
+    const parsed = new Date(value);
+    return Number.isNaN(parsed.getTime()) ? null : parsed.toISOString().slice(0, 10);
+  };
+  const cutoffDate = (days: number): string => {
+    const cutoff = new Date(today);
+    cutoff.setUTCDate(cutoff.getUTCDate() - days);
+    return cutoff.toISOString().slice(0, 10);
+  };
+
+  // 1. Pass rate trend — last 90 days, bucketed by day
+  const passRateTrendByDate = new Map<string, { triggered: number; total: number }>();
+  for (const row of trustedRows) {
+    const occurredDate = dateKey(row.occurred_at);
+    if (!occurredDate || occurredDate < cutoffDate(90)) continue;
+    const counts = passRateTrendByDate.get(occurredDate) ?? { triggered: 0, total: 0 };
+    counts.total += 1;
+    if (row.triggered === 1) counts.triggered += 1;
+    passRateTrendByDate.set(occurredDate, counts);
+  }
+  const passRateTrendRows = [...passRateTrendByDate.entries()]
+    .map(([date, counts]) => ({
+      date,
+      pass_rate: counts.total > 0 ? counts.triggered / counts.total : 0,
+      total_checks: counts.total,
+    }))
+    .sort((a, b) => a.date.localeCompare(b.date));
+
+  const pass_rate_trend = passRateTrendRows.map((row) => ({
+    date: row.date,
+    pass_rate: row.pass_rate,
+    total_checks: row.total_checks,
+  }));
+
+  // 2. Skill rankings — all skills with at least 1 check, ordered by pass rate
+  const skillRankingMap = new Map<string, { triggered_count: number; total_checks: number }>();
+  for (const row of trustedRows) {
+    const counts = skillRankingMap.get(row.skill_name) ?? { triggered_count: 0, total_checks: 0 };
+    counts.total_checks += 1;
+    if (row.triggered === 1) counts.triggered_count += 1;
+    skillRankingMap.set(row.skill_name, counts);
+  }
+  const skillRankingRows = [...skillRankingMap.entries()]
+    .map(([skill_name, counts]) => ({
+      skill_name,
+      pass_rate: counts.total_checks > 0 ? counts.triggered_count / counts.total_checks : 0,
+      total_checks: counts.total_checks,
+      triggered_count: counts.triggered_count,
+    }))
+    .sort(
+      (a, b) =>
+        b.pass_rate - a.pass_rate ||
+        b.total_checks - a.total_checks ||
+        a.skill_name.localeCompare(b.skill_name),
+    );
+
+  const skill_rankings = skillRankingRows.map((row) => ({
     skill_name: row.skill_name,
-    skill_scope: row.skill_scope,
+    pass_rate: row.pass_rate,
     total_checks: row.total_checks,
     triggered_count: row.triggered_count,
-    pass_rate: row.total_checks > 0 ? row.triggered_count / row.total_checks : 0,
-    unique_sessions: row.unique_sessions,
-    last_seen: row.last_seen,
-    has_evidence: evidenceSkills.has(row.skill_name),
   }));
+
+  // 3. Daily activity — last 84 days (12 weeks) for heatmap
+  const dailyActivityByDate = new Map<string, number>();
+  for (const row of trustedRows) {
+    const occurredDate = dateKey(row.occurred_at);
+    if (!occurredDate || occurredDate < cutoffDate(84)) continue;
+    dailyActivityByDate.set(occurredDate, (dailyActivityByDate.get(occurredDate) ?? 0) + 1);
+  }
+  const dailyActivityRows = [...dailyActivityByDate.entries()]
+    .map(([date, checks]) => ({ date, checks }))
+    .sort((a, b) => a.date.localeCompare(b.date));
+
+  const daily_activity = dailyActivityRows.map((row) => ({
+    date: row.date,
+    checks: row.checks,
+  }));
+
+  // 4. Evolution impact — before/after pass rates for deployed evolutions
+  const deployedRows = db
+    .query(
+      `SELECT ea.skill_name, ea.proposal_id, ea.timestamp as deployed_at
+       FROM evolution_audit ea
+       WHERE ea.action = 'deployed' AND ea.skill_name IS NOT NULL
+       ORDER BY ea.timestamp DESC`,
+    )
+    .all() as Array<{ skill_name: string; proposal_id: string; deployed_at: string }>;
+
+  const evolution_impact: AnalyticsResponse["evolution_impact"] = [];
+  for (const deploy of deployedRows) {
+    const beforeRows = trustedRows.filter(
+      (row) => row.skill_name === deploy.skill_name && (row.occurred_at ?? "") < deploy.deployed_at,
+    );
+    const afterRows = trustedRows.filter(
+      (row) =>
+        row.skill_name === deploy.skill_name && (row.occurred_at ?? "") >= deploy.deployed_at,
+    );
+
+    evolution_impact.push({
+      skill_name: deploy.skill_name,
+      proposal_id: deploy.proposal_id,
+      deployed_at: deploy.deployed_at,
+      pass_rate_before:
+        beforeRows.length > 0
+          ? beforeRows.filter((row) => row.triggered === 1).length / beforeRows.length
+          : 0,
+      pass_rate_after:
+        afterRows.length > 0
+          ? afterRows.filter((row) => row.triggered === 1).length / afterRows.length
+          : 0,
+    });
+  }
+
+  // 5. Summary aggregates
+  const totalEvolutionsRow = db
+    .query(`SELECT COUNT(*) as c FROM evolution_audit WHERE action = 'deployed'`)
+    .get() as { c: number } | null;
+
+  const checks30dRows = trustedRows.filter((row) => {
+    const occurredDate = dateKey(row.occurred_at);
+    return occurredDate != null && occurredDate >= cutoffDate(30);
+  });
+  const activeSkills30d = new Set(checks30dRows.map((row) => row.skill_name));
+
+  // Average improvement across all deployed evolutions
+  let avgImprovement = 0;
+  if (evolution_impact.length > 0) {
+    const totalImprovement = evolution_impact.reduce(
+      (sum, e) => sum + (e.pass_rate_after - e.pass_rate_before),
+      0,
+    );
+    avgImprovement = totalImprovement / evolution_impact.length;
+  }
+
+  const summary: AnalyticsResponse["summary"] = {
+    total_evolutions: totalEvolutionsRow?.c ?? 0,
+    avg_improvement: avgImprovement,
+    total_checks_30d: checks30dRows.length,
+    active_skills: activeSkills30d.size,
+  };
+
+  return {
+    pass_rate_trend,
+    skill_rankings,
+    daily_activity,
+    evolution_impact,
+    summary,
+  };
 }
 
 /**
@@ -390,13 +979,12 @@ export function getActiveSessionCount(db: Database): number {
  * session is still in progress (no session_telemetry row yet).
  */
 export function getRecentActivity(db: Database, limit = 20): RecentActivityItem[] {
+  // Step 1: Fetch recent invocations without JOIN (avoids materializing full JOIN before LIMIT)
   const rows = db
     .query(
-      `SELECT si.occurred_at, si.session_id, si.skill_name, si.query, si.triggered,
-              CASE WHEN st.session_id IS NULL THEN 1 ELSE 0 END as is_live
-       FROM skill_invocations si
-       LEFT JOIN session_telemetry st ON si.session_id = st.session_id
-       ORDER BY si.occurred_at DESC
+      `SELECT occurred_at, session_id, skill_name, query, triggered
+       FROM skill_invocations
+       ORDER BY occurred_at DESC
        LIMIT ?`,
     )
     .all(limit) as Array<{
@@ -405,16 +993,27 @@ export function getRecentActivity(db: Database, limit = 20): RecentActivityItem[
     skill_name: string;
     query: string;
     triggered: number;
-    is_live: number;
   }>;
 
+  if (rows.length === 0) return [];
+
+  // Step 2: Batch lookup which sessions have completed (have a telemetry row)
+  const uniqueSessionIds = [...new Set(rows.map((r) => r.session_id))];
+  const placeholders = uniqueSessionIds.map(() => "?").join(",");
+  const completedRows = db
+    .query(
+      `SELECT DISTINCT session_id FROM session_telemetry WHERE session_id IN (${placeholders})`,
+    )
+    .all(...uniqueSessionIds) as Array<{ session_id: string }>;
+  const completedSessions = new Set(completedRows.map((r) => r.session_id));
+
   return rows.map((row) => ({
     timestamp: row.occurred_at,
     session_id: row.session_id,
     skill_name: row.skill_name,
     query: row.query ?? "",
     triggered: row.triggered === 1,
-    is_live: row.is_live === 1,
+    is_live: !completedSessions.has(row.session_id),
   }));
 }
 
@@ -688,6 +1287,72 @@ export function queryGradingResults(db: Database): Array<{
   }>;
 }
 
+export function getCreatorContributionStagingCounts(db: Database): Array<{
+  skill_name: string;
+  pending_count: number;
+}> {
+  return db
+    .query(
+      `SELECT skill_name, COUNT(*) AS pending_count
+       FROM creator_contribution_staging
+       WHERE status = 'pending'
+       GROUP BY skill_name
+       ORDER BY skill_name`,
+    )
+    .all() as Array<{
+    skill_name: string;
+    pending_count: number;
+  }>;
+}
+
+export interface CreatorContributionRelayStats {
+  pending: number;
+  sending: number;
+  sent: number;
+  failed: number;
+}
+
+export interface CreatorContributionStagingRow {
+  id: number;
+  dedupe_key: string;
+  skill_name: string;
+  creator_id: string;
+  payload_json: string;
+  status: string;
+  staged_at: string;
+  updated_at: string;
+  last_error: string | null;
+}
+
+export function getCreatorContributionRelayStats(db: Database): CreatorContributionRelayStats {
+  const row = db
+    .query(
+      `SELECT
+         COALESCE(SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END), 0) AS pending,
+         COALESCE(SUM(CASE WHEN status = 'sending' THEN 1 ELSE 0 END), 0) AS sending,
+         COALESCE(SUM(CASE WHEN status = 'sent' THEN 1 ELSE 0 END), 0) AS sent,
+         COALESCE(SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END), 0) AS failed
+       FROM creator_contribution_staging`,
+    )
+    .get() as CreatorContributionRelayStats | null;
+  return row ?? { pending: 0, sending: 0, sent: 0, failed: 0 };
+}
+
+export function getPendingCreatorContributionRows(
+  db: Database,
+  limit = 50,
+): CreatorContributionStagingRow[] {
+  return db
+    .query(
+      `SELECT id, dedupe_key, skill_name, creator_id, payload_json, status, staged_at, updated_at, last_error
+       FROM creator_contribution_staging
+       WHERE status = 'pending'
+       ORDER BY id ASC
+       LIMIT ?`,
+    )
+    .all(limit) as CreatorContributionStagingRow[];
+}
+
 // -- Canonical record staging query -------------------------------------------
 
 /**
@@ -914,6 +1579,557 @@ export function getOldestPendingAge(db: Database): number | null {
   }
 }
 
+// -- Execution metrics & commit tracking queries ------------------------------
+
+/**
+ * Aggregate execution_facts enrichment columns for a set of session IDs.
+ *
+ * Returns file change stats, cost totals, token breakdowns, artifact counts,
+ * and session_type distribution across the provided sessions.
+ */
+export function getExecutionMetrics(db: Database, sessionIds: string[]): ExecutionMetrics {
+  const empty: ExecutionMetrics = {
+    avg_files_changed: 0,
+    total_lines_added: 0,
+    total_lines_removed: 0,
+    total_cost_usd: 0,
+    avg_cost_usd: 0,
+    cached_input_tokens_total: 0,
+    reasoning_output_tokens_total: 0,
+    artifact_count: 0,
+    session_type_distribution: {},
+  };
+  if (sessionIds.length === 0) return empty;
+
+  const placeholders = sessionIds.map(() => "?").join(",");
+  const row = db
+    .query(
+      `SELECT
+         COALESCE(AVG(files_changed), 0) AS avg_files_changed,
+         COALESCE(SUM(lines_added), 0) AS total_lines_added,
+         COALESCE(SUM(lines_removed), 0) AS total_lines_removed,
+         COALESCE(SUM(cost_usd), 0) AS total_cost_usd,
+         COALESCE(AVG(cost_usd), 0) AS avg_cost_usd,
+         COALESCE(SUM(cached_input_tokens), 0) AS cached_input_tokens_total,
+         COALESCE(SUM(reasoning_output_tokens), 0) AS reasoning_output_tokens_total,
+         COALESCE(SUM(artifact_count), 0) AS artifact_count
+       FROM execution_facts
+       WHERE session_id IN (${placeholders})`,
+    )
+    .get(...sessionIds) as {
+    avg_files_changed: number;
+    total_lines_added: number;
+    total_lines_removed: number;
+    total_cost_usd: number;
+    avg_cost_usd: number;
+    cached_input_tokens_total: number;
+    reasoning_output_tokens_total: number;
+    artifact_count: number;
+  } | null;
+
+  // Session type distribution
+  const typeRows = db
+    .query(
+      `SELECT session_type, COUNT(*) AS count
+       FROM execution_facts
+       WHERE session_id IN (${placeholders}) AND session_type IS NOT NULL
+       GROUP BY session_type`,
+    )
+    .all(...sessionIds) as Array<{ session_type: string; count: number }>;
+
+  const session_type_distribution: Record<string, number> = {};
+  for (const tr of typeRows) {
+    session_type_distribution[tr.session_type] = tr.count;
+  }
+
+  return {
+    avg_files_changed: row?.avg_files_changed ?? 0,
+    total_lines_added: row?.total_lines_added ?? 0,
+    total_lines_removed: row?.total_lines_removed ?? 0,
+    total_cost_usd: row?.total_cost_usd ?? 0,
+    avg_cost_usd: row?.avg_cost_usd ?? 0,
+    cached_input_tokens_total: row?.cached_input_tokens_total ?? 0,
+    reasoning_output_tokens_total: row?.reasoning_output_tokens_total ?? 0,
+    artifact_count: row?.artifact_count ?? 0,
+    session_type_distribution,
+  };
+}
+
+/**
+ * Get all commits tracked for a given session.
+ */
+export function getSessionCommits(db: Database, sessionId: string): CommitRecord[] {
+  return db
+    .query(
+      `SELECT commit_sha, commit_title, branch, repo_remote, timestamp
+       FROM commit_tracking
+       WHERE session_id = ?
+       ORDER BY timestamp DESC`,
+    )
+    .all(sessionId) as CommitRecord[];
+}
+
+/**
+ * Aggregate commit stats for a skill by joining commit_tracking to skill_invocations
+ * via shared session_id.
+ */
+export function getSkillCommitSummary(db: Database, skillName: string): CommitSummary {
+  const empty: CommitSummary = {
+    total_commits: 0,
+    unique_branches: 0,
+    recent_commits: [],
+  };
+
+  const statsRow = db
+    .query(
+      `WITH skill_sessions AS (
+         SELECT DISTINCT session_id FROM skill_invocations WHERE skill_name = ?
+       )
+       SELECT
+         COUNT(*) AS total_commits,
+         COUNT(DISTINCT ct.branch) AS unique_branches
+       FROM commit_tracking ct
+       WHERE ct.session_id IN (SELECT session_id FROM skill_sessions)`,
+    )
+    .get(skillName) as { total_commits: number; unique_branches: number } | null;
+
+  if (!statsRow || statsRow.total_commits === 0) return empty;
+
+  const recentRows = db
+    .query(
+      `WITH skill_sessions AS (
+         SELECT DISTINCT session_id FROM skill_invocations WHERE skill_name = ?
+       )
+       SELECT ct.commit_sha, ct.commit_title, ct.branch, ct.timestamp
+       FROM commit_tracking ct
+       WHERE ct.session_id IN (SELECT session_id FROM skill_sessions)
+       ORDER BY ct.timestamp DESC
+       LIMIT 20`,
+    )
+    .all(skillName) as Array<{
+    commit_sha: string;
+    commit_title: string | null;
+    branch: string | null;
+    timestamp: string;
+  }>;
+
+  return {
+    total_commits: statsRow.total_commits,
+    unique_branches: statsRow.unique_branches,
+    recent_commits: recentRows.map((r) => ({
+      sha: r.commit_sha,
+      title: r.commit_title ?? "",
+      branch: r.branch ?? "",
+      timestamp: r.timestamp,
+    })),
+  };
+}
+
+// -- Helpers ------------------------------------------------------------------
+
+// -- Autonomy-first dashboard queries -----------------------------------------
+
+export interface SkillTrustSummary {
+  skill_name: string;
+  total_checks: number;
+  triggered_count: number;
+  miss_rate: number;
+  system_like_count: number;
+  system_like_rate: number;
+  prompt_link_rate: number;
+  latest_action: string | null;
+  pass_rate: number;
+  last_seen: string | null;
+}
+
+export interface TrustedSkillObservationRow {
+  skill_name: string;
+  session_id: string;
+  occurred_at: string | null;
+  triggered: number;
+  matched_prompt_id: string | null;
+  confidence: number | null;
+  invocation_mode: string | null;
+  query_text: string;
+}
+
+export function queryTrustedSkillObservationRows(db: Database): TrustedSkillObservationRow[] {
+  const SYSTEM_LIKE_PREFIXES = ["<system_instruction>", "<system-instruction>", "<command-name>"];
+  const INTERNAL_EVAL_MARKERS = [
+    "you are an evaluation assistant",
+    "you are a skill description optimizer",
+    "would each query trigger this skill",
+    "propose an improved description",
+    "failure patterns:",
+    "output only valid json",
+  ];
+  const isSystemLike = (text: string | null | undefined): boolean => {
+    if (!text) return false;
+    const trimmed = text.trimStart();
+    return SYSTEM_LIKE_PREFIXES.some((p) => trimmed.startsWith(p));
+  };
+  const isInternalSelftunePrompt = (
+    text: string | null | undefined,
+    promptKind: string | null | undefined,
+  ): boolean => {
+    if (!text) return false;
+    const lowered = text.toLowerCase();
+    return (
+      promptKind === "meta" && INTERNAL_EVAL_MARKERS.some((marker) => lowered.includes(marker))
+    );
+  };
+  const isPollutingPrompt = (
+    text: string | null | undefined,
+    promptKind: string | null | undefined,
+  ): boolean => isSystemLike(text) || isInternalSelftunePrompt(text, promptKind);
+  const classifyObservationKind = (
+    skillInvocationId: string,
+    captureMode: string | null,
+    triggered: number,
+    rawSourceRefJson: string | null,
+  ): "canonical" | "repaired_trigger" | "repaired_contextual_miss" | "legacy_materialized" => {
+    if (skillInvocationId.includes(":su:")) return "legacy_materialized";
+    if (captureMode === "repair") {
+      const rawSourceRef = safeParseJson(rawSourceRefJson) as {
+        metadata?: { miss_type?: string };
+      } | null;
+      if (triggered === 0 && rawSourceRef?.metadata?.miss_type === "contextual_read") {
+        return "repaired_contextual_miss";
+      }
+      return "repaired_trigger";
+    }
+    return "canonical";
+  };
+  const normalizeQueryForGrouping = (query: string) =>
+    query.replace(/\s+/g, " ").trim().toLowerCase();
+
+  const rows = db
+    .query(
+      `SELECT
+         si.skill_name,
+         si.session_id,
+         si.occurred_at,
+         si.triggered,
+         si.matched_prompt_id,
+         si.confidence,
+         si.invocation_mode,
+         si.skill_invocation_id,
+         si.capture_mode,
+         si.raw_source_ref,
+         si.query,
+         p.prompt_text,
+         p.prompt_kind
+       FROM skill_invocations si
+       LEFT JOIN prompts p ON si.matched_prompt_id = p.prompt_id`,
+    )
+    .all() as Array<{
+    skill_name: string;
+    session_id: string;
+    occurred_at: string | null;
+    triggered: number;
+    matched_prompt_id: string | null;
+    confidence: number | null;
+    invocation_mode: string | null;
+    skill_invocation_id: string;
+    capture_mode: string | null;
+    raw_source_ref: string | null;
+    query: string | null;
+    prompt_text: string | null;
+    prompt_kind: string | null;
+  }>;
+
+  const bySkill = new Map<
+    string,
+    Array<{
+      skill_name: string;
+      session_id: string;
+      occurred_at: string | null;
+      triggered: number;
+      matched_prompt_id: string | null;
+      confidence: number | null;
+      invocation_mode: string | null;
+      queryText: string;
+      isPolluting: boolean;
+      observation_kind:
+        | "canonical"
+        | "repaired_trigger"
+        | "repaired_contextual_miss"
+        | "legacy_materialized";
+      groupKey: string;
+    }>
+  >();
+  const trustedRows: Array<{
+    skill_name: string;
+    session_id: string;
+    occurred_at: string | null;
+    triggered: number;
+    matched_prompt_id: string | null;
+    confidence: number | null;
+    invocation_mode: string | null;
+    query_text: string;
+  }> = [];
+
+  for (const row of rows) {
+    const queryText = row.query || row.prompt_text || "";
+    const pollutionText = row.prompt_text || row.query || "";
+    const observation_kind = classifyObservationKind(
+      row.skill_invocation_id,
+      row.capture_mode,
+      row.triggered,
+      row.raw_source_ref,
+    );
+    if (isPollutingPrompt(pollutionText, row.prompt_kind)) continue;
+    if (observation_kind === "legacy_materialized") continue;
+
+    const normalizedQuery = normalizeQueryForGrouping(queryText);
+    const groupKey =
+      normalizedQuery.length > 0
+        ? `${row.session_id}::${normalizedQuery}`
+        : `${row.skill_invocation_id}`;
+    const arr = bySkill.get(row.skill_name);
+    const enriched = {
+      skill_name: row.skill_name,
+      session_id: row.session_id,
+      occurred_at: row.occurred_at,
+      triggered: row.triggered,
+      matched_prompt_id: row.matched_prompt_id,
+      confidence: row.confidence,
+      invocation_mode: row.invocation_mode,
+      queryText,
+      isPolluting: false,
+      observation_kind,
+      groupKey,
+    };
+    if (arr) arr.push(enriched);
+    else bySkill.set(row.skill_name, [enriched]);
+  }
+
+  for (const [, skillRows] of bySkill.entries()) {
+    const grouped = new Map<string, typeof skillRows>();
+    for (const row of skillRows) {
+      const arr = grouped.get(row.groupKey);
+      if (arr) arr.push(row);
+      else grouped.set(row.groupKey, [row]);
+    }
+
+    const deduped = [...grouped.values()].map((group) => {
+      const sorted = [...group].sort((a, b) => {
+        const aScore =
+          (a.triggered === 1 ? 100 : 0) +
+          (a.observation_kind === "canonical" ? 20 : 0) +
+          (a.observation_kind === "repaired_trigger" ? 15 : 0);
+        const bScore =
+          (b.triggered === 1 ? 100 : 0) +
+          (b.observation_kind === "canonical" ? 20 : 0) +
+          (b.observation_kind === "repaired_trigger" ? 15 : 0);
+        if (aScore !== bScore) return bScore - aScore;
+        return (b.occurred_at ?? "").localeCompare(a.occurred_at ?? "");
+      });
+      return sorted[0]!;
+    });
+
+    trustedRows.push(
+      ...deduped.map((row) => ({
+        skill_name: row.skill_name,
+        session_id: row.session_id,
+        occurred_at: row.occurred_at,
+        triggered: row.triggered,
+        matched_prompt_id: row.matched_prompt_id,
+        confidence: row.confidence,
+        invocation_mode: row.invocation_mode,
+        query_text: row.queryText,
+      })),
+    );
+  }
+
+  return trustedRows;
+}
+
+export function getSkillTrustSummaries(db: Database): SkillTrustSummary[] {
+  const rows = queryTrustedSkillObservationRows(db);
+
+  // Build latest_action map from evolution_audit
+  const auditRows = db
+    .query(
+      `SELECT skill_name, action, timestamp
+       FROM evolution_audit
+       WHERE skill_name IS NOT NULL
+       ORDER BY timestamp DESC`,
+    )
+    .all() as Array<{
+    skill_name: string | null;
+    action: string;
+    timestamp: string;
+  }>;
+
+  const latestActions = new Map<string, string>();
+  for (const row of auditRows) {
+    if (row.skill_name && !latestActions.has(row.skill_name)) {
+      latestActions.set(row.skill_name, row.action);
+    }
+  }
+
+  const rowsBySkill = new Map<string, typeof rows>();
+  for (const row of rows) {
+    const arr = rowsBySkill.get(row.skill_name);
+    if (arr) arr.push(row);
+    else rowsBySkill.set(row.skill_name, [row]);
+  }
+
+  const summaries: SkillTrustSummary[] = [];
+  for (const [skillName, skillRows] of rowsBySkill.entries()) {
+    const total = skillRows.length;
+    const triggered = skillRows.filter((row) => row.triggered === 1).length;
+    const promptLinked = skillRows.filter((row) => row.matched_prompt_id != null).length;
+    const lastSeen =
+      skillRows
+        .map((row) => row.occurred_at)
+        .filter((value): value is string => value != null)
+        .sort((a, b) => b.localeCompare(a))[0] ?? null;
+
+    summaries.push({
+      skill_name: skillName,
+      total_checks: total,
+      triggered_count: triggered,
+      miss_rate: total > 0 ? (total - triggered) / total : 0,
+      system_like_count: 0,
+      system_like_rate: 0,
+      prompt_link_rate: total > 0 ? promptLinked / total : 0,
+      latest_action: latestActions.get(skillName) ?? null,
+      pass_rate: total > 0 ? triggered / total : 0,
+      last_seen: lastSeen,
+    });
+  }
+
+  return summaries;
+}
+
+export function getAttentionQueue(db: Database): AttentionItem[] {
+  const summaries = getSkillTrustSummaries(db);
+  const pending = getPendingProposals(db);
+  const pendingSkills = new Set(pending.map((p) => p.skill_name).filter(Boolean));
+
+  const items: AttentionItem[] = [];
+
+  for (const s of summaries) {
+    if (s.latest_action === "rolled_back") {
+      items.push({
+        skill_name: s.skill_name,
+        category: "needs_review",
+        severity: "critical",
+        reason: "Rolled back after deployment",
+        recommended_action: "Review rollback evidence and decide whether to re-evolve",
+        timestamp: s.last_seen ?? "",
+      });
+      continue;
+    }
+
+    if (pendingSkills.has(s.skill_name)) {
+      items.push({
+        skill_name: s.skill_name,
+        category: "needs_review",
+        severity: "info",
+        reason: "Proposal awaiting review",
+        recommended_action: "Review and approve or reject the pending proposal",
+        timestamp: s.last_seen ?? "",
+      });
+      continue;
+    }
+
+    if (s.total_checks < 5) continue;
+
+    if (s.miss_rate > 0.1) {
+      items.push({
+        skill_name: s.skill_name,
+        category: "regression",
+        severity: "warning",
+        reason: `High miss rate (${Math.round(s.miss_rate * 100)}%)`,
+        recommended_action: "Review missed invocations and consider evolving the skill description",
+        timestamp: s.last_seen ?? "",
+      });
+      continue;
+    }
+
+    if (s.system_like_rate > 0.1) {
+      items.push({
+        skill_name: s.skill_name,
+        category: "polluted",
+        severity: "warning",
+        reason: `Possible telemetry pollution (${Math.round(s.system_like_rate * 100)}% system-like)`,
+        recommended_action: "Inspect prompts for system-injected noise",
+        timestamp: s.last_seen ?? "",
+      });
+      continue;
+    }
+  }
+
+  return items;
+}
+
+export function getRecentDecisions(db: Database, limit = 20): AutonomousDecision[] {
+  const rows = db
+    .query(
+      `SELECT timestamp, proposal_id, skill_name, action, details, eval_snapshot_json
+       FROM evolution_audit
+       WHERE timestamp >= datetime('now', '-7 days')
+       ORDER BY timestamp DESC
+       LIMIT ?`,
+    )
+    .all(limit) as Array<{
+    timestamp: string;
+    proposal_id: string;
+    skill_name: string | null;
+    action: string;
+    details: string;
+    eval_snapshot_json: string | null;
+  }>;
+
+  return rows
+    .filter((row) => row.skill_name != null)
+    .flatMap((row) => {
+      const evalSnapshot = safeParseJson(row.eval_snapshot_json) as {
+        regressions?: unknown[];
+      } | null;
+
+      let kind: DecisionKind | null;
+      switch (row.action) {
+        case "proposed":
+        case "created":
+          kind = "proposal_created";
+          break;
+        case "rejected":
+          kind = "proposal_rejected";
+          break;
+        case "validated":
+          kind =
+            evalSnapshot?.regressions && evalSnapshot.regressions.length > 0
+              ? "validation_failed"
+              : "proposal_created"; // validated without regressions is still a creation step
+          break;
+        case "deployed":
+          kind = "proposal_deployed";
+          break;
+        case "rolled_back":
+          kind = "rollback_triggered";
+          break;
+        default:
+          kind = null;
+      }
+
+      if (!kind) return [];
+
+      return [
+        {
+          timestamp: row.timestamp,
+          kind,
+          skill_name: row.skill_name!,
+          proposal_id: row.proposal_id,
+          summary: row.details ?? "",
+        },
+      ];
+    });
+}
+
 // -- Helpers ------------------------------------------------------------------
 
 export function safeParseJsonArray<T = string>(json: string | null): T[] {