tanuki-telemetry 1.1.0

package/src/db.ts

@@ -0,0 +1,1009 @@
+import Database from "better-sqlite3";
+import fs from "fs";
+import path from "path";
+import sharp from "sharp";
+import type {
+  Session,
+  Event,
+  Iteration,
+  Screenshot,
+  Artifact,
+  FinalResult,
+  Walkthrough,
+  WalkthroughAction,
+  WalkthroughScreenshot,
+} from "./types.js";
+
+const DB_PATH = process.env.DB_PATH || "/data/telemetry.db";
+
+let db: Database.Database;
+
+export function getDb(): Database.Database {
+  if (!db) {
+    db = new Database(DB_PATH);
+    db.pragma("journal_mode = WAL");
+    db.pragma("foreign_keys = ON");
+    initTables();
+  }
+  return db;
+}
+
+function initTables(): void {
+  const d = getDb();
+
+  d.exec(`
+    CREATE TABLE IF NOT EXISTS sessions (
+      id TEXT PRIMARY KEY,
+      worktree_name TEXT NOT NULL,
+      ticket_id TEXT,
+      ticket_title TEXT,
+      branch_name TEXT,
+      mode TEXT CHECK(mode IN ('local', 'remote')) DEFAULT 'local',
+      status TEXT CHECK(status IN ('in_progress', 'completed', 'failed', 'interrupted')) DEFAULT 'in_progress',
+      total_input_tokens INTEGER DEFAULT 0,
+      total_output_tokens INTEGER DEFAULT 0,
+      total_iterations INTEGER DEFAULT 0,
+      max_iterations INTEGER DEFAULT 10,
+      started_at TEXT NOT NULL DEFAULT (datetime('now')),
+      ended_at TEXT,
+      duration_seconds INTEGER,
+      final_result TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS events (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT NOT NULL REFERENCES sessions(id),
+      phase TEXT NOT NULL,
+      event_type TEXT NOT NULL,
+      message TEXT NOT NULL,
+      metadata TEXT,
+      tokens_used INTEGER DEFAULT 0,
+      timestamp TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS iterations (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT NOT NULL REFERENCES sessions(id),
+      iteration_number INTEGER NOT NULL,
+      trigger TEXT NOT NULL,
+      error_summary TEXT NOT NULL,
+      fix_description TEXT,
+      files_changed TEXT,
+      result TEXT CHECK(result IN ('pass', 'fail', 'partial')) NOT NULL,
+      duration_seconds INTEGER,
+      timestamp TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS screenshots (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT NOT NULL REFERENCES sessions(id),
+      iteration_number INTEGER,
+      phase TEXT NOT NULL,
+      description TEXT NOT NULL,
+      file_path TEXT NOT NULL,
+      timestamp TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS plan_steps (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT NOT NULL REFERENCES sessions(id),
+      step_number INTEGER NOT NULL,
+      title TEXT NOT NULL,
+      description TEXT NOT NULL DEFAULT '',
+      status TEXT CHECK(status IN ('pending', 'in_progress', 'completed', 'skipped', 'failed')) DEFAULT 'pending',
+      parent_step INTEGER,
+      file_targets TEXT,
+      outcome TEXT,
+      started_at TEXT,
+      completed_at TEXT,
+      duration_seconds INTEGER,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS insights (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT NOT NULL REFERENCES sessions(id),
+      insight_type TEXT CHECK(insight_type IN ('mistake', 'success_pattern', 'codebase_gotcha', 'optimization', 'rule_learned')) NOT NULL,
+      category TEXT NOT NULL,
+      title TEXT NOT NULL,
+      description TEXT NOT NULL,
+      evidence TEXT,
+      confidence REAL DEFAULT 0.5,
+      times_validated INTEGER DEFAULT 0,
+      file_patterns TEXT,
+      error_patterns TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_events_session ON events(session_id);
+    CREATE INDEX IF NOT EXISTS idx_iterations_session ON iterations(session_id);
+    CREATE INDEX IF NOT EXISTS idx_screenshots_session ON screenshots(session_id);
+    CREATE INDEX IF NOT EXISTS idx_plan_steps_session ON plan_steps(session_id);
+    CREATE INDEX IF NOT EXISTS idx_insights_session ON insights(session_id);
+    CREATE INDEX IF NOT EXISTS idx_insights_type ON insights(insight_type);
+    CREATE INDEX IF NOT EXISTS idx_insights_category ON insights(category);
+  `);
+
+  // Migration: add parent_session_id column to sessions
+  try {
+    d.exec(`ALTER TABLE sessions ADD COLUMN parent_session_id TEXT REFERENCES sessions(id)`);
+  } catch {
+    // Column already exists — ignore duplicate column error
+  }
+  d.exec(`CREATE INDEX IF NOT EXISTS idx_sessions_parent ON sessions(parent_session_id)`);
+
+  // Migration: add stored_path column to screenshots
+  try {
+    d.exec(`ALTER TABLE screenshots ADD COLUMN stored_path TEXT`);
+  } catch {
+    // Column already exists
+  }
+
+  // Migration: add thumb_path column to screenshots
+  try {
+    d.exec(`ALTER TABLE screenshots ADD COLUMN thumb_path TEXT`);
+  } catch {
+    // Column already exists
+  }
+
+  // Migration: add event_id column to screenshots
+  try {
+    d.exec(`ALTER TABLE screenshots ADD COLUMN event_id INTEGER REFERENCES events(id)`);
+  } catch {
+    // Column already exists
+  }
+
+  // Migration: add user_email column to sessions
+  try {
+    d.exec(`ALTER TABLE sessions ADD COLUMN user_email TEXT`);
+  } catch {
+    // Column already exists
+  }
+  d.exec(`CREATE INDEX IF NOT EXISTS idx_sessions_user_email ON sessions(user_email)`);
+
+  // Coordinator tables
+  d.exec(`
+    CREATE TABLE IF NOT EXISTS coordinator_state (
+      session_id TEXT PRIMARY KEY,
+      state TEXT NOT NULL,
+      started_at TEXT NOT NULL DEFAULT (datetime('now')),
+      updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS coordinator_history (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT NOT NULL,
+      snapshot TEXT NOT NULL,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_coordinator_history_session ON coordinator_history(session_id);
+  `);
+
+  // Migration: add artifacts table
+  d.exec(`
+    CREATE TABLE IF NOT EXISTS artifacts (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT NOT NULL REFERENCES sessions(id),
+      artifact_type TEXT NOT NULL,
+      description TEXT NOT NULL,
+      file_path TEXT NOT NULL,
+      stored_path TEXT,
+      mime_type TEXT,
+      file_size INTEGER,
+      metadata TEXT,
+      event_id INTEGER REFERENCES events(id),
+      timestamp TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_artifacts_session ON artifacts(session_id);
+  `);
+
+  // Walkthrough tables
+  d.exec(`
+    CREATE TABLE IF NOT EXISTS walkthroughs (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      url TEXT NOT NULL,
+      app_name TEXT,
+      scenario TEXT,
+      status TEXT CHECK(status IN ('in_progress', 'pass', 'fail', 'partial')) DEFAULT 'in_progress',
+      summary TEXT,
+      total_actions INTEGER DEFAULT 0,
+      passed INTEGER DEFAULT 0,
+      failed INTEGER DEFAULT 0,
+      started_at TEXT NOT NULL DEFAULT (datetime('now')),
+      ended_at TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS walkthrough_actions (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      walkthrough_id INTEGER NOT NULL REFERENCES walkthroughs(id),
+      action_type TEXT CHECK(action_type IN ('navigate', 'click', 'type', 'assert', 'wait', 'screenshot')) NOT NULL,
+      target TEXT NOT NULL,
+      value TEXT,
+      status TEXT CHECK(status IN ('pass', 'fail')) NOT NULL,
+      message TEXT,
+      timestamp TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS walkthrough_screenshots (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      walkthrough_id INTEGER NOT NULL REFERENCES walkthroughs(id),
+      action_id INTEGER REFERENCES walkthrough_actions(id),
+      name TEXT NOT NULL,
+      annotation TEXT,
+      stored_path TEXT NOT NULL,
+      timestamp TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_walkthrough_actions_wt ON walkthrough_actions(walkthrough_id);
+    CREATE INDEX IF NOT EXISTS idx_walkthrough_screenshots_wt ON walkthrough_screenshots(walkthrough_id);
+  `);
+
+  // Ensure storage directories exist
+  const screenshotsDir = "/data/screenshots";
+  if (!fs.existsSync(screenshotsDir)) {
+    fs.mkdirSync(screenshotsDir, { recursive: true });
+  }
+  const artifactsDir = "/data/artifacts";
+  if (!fs.existsSync(artifactsDir)) {
+    fs.mkdirSync(artifactsDir, { recursive: true });
+  }
+  const walkthroughScreenshotsDir = "/data/walkthrough-screenshots";
+  if (!fs.existsSync(walkthroughScreenshotsDir)) {
+    fs.mkdirSync(walkthroughScreenshotsDir, { recursive: true });
+  }
+}
+
+export function createSession(
+  id: string,
+  worktree_name: string,
+  ticket_id?: string,
+  ticket_title?: string,
+  branch_name?: string,
+  mode?: string,
+  max_iterations?: number,
+  parent_session_id?: string,
+  user_email?: string
+): void {
+  const d = getDb();
+  const stmt = d.prepare(`
+    INSERT INTO sessions (id, worktree_name, ticket_id, ticket_title, branch_name, mode, max_iterations, parent_session_id, user_email)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+  stmt.run(
+    id,
+    worktree_name,
+    ticket_id ?? null,
+    ticket_title ?? null,
+    branch_name ?? null,
+    mode ?? "local",
+    max_iterations ?? 10,
+    parent_session_id ?? null,
+    user_email ?? null
+  );
+}
+
+export function insertEvent(
+  session_id: string,
+  phase: string,
+  event_type: string,
+  message: string,
+  metadata?: object,
+  tokens_used?: number,
+  input_tokens?: number,
+  output_tokens?: number
+): number {
+  const d = getDb();
+  const stmt = d.prepare(`
+    INSERT INTO events (session_id, phase, event_type, message, metadata, tokens_used)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+  const result = stmt.run(
+    session_id,
+    phase,
+    event_type,
+    message,
+    metadata ? JSON.stringify(metadata) : null,
+    tokens_used ?? 0
+  );
+
+  // Accumulate token counts on the session
+  const inTok = input_tokens ?? 0;
+  const outTok = output_tokens ?? 0;
+  if (inTok > 0 || outTok > 0) {
+    d.prepare(`
+      UPDATE sessions
+      SET total_input_tokens = total_input_tokens + ?,
+          total_output_tokens = total_output_tokens + ?
+      WHERE id = ?
+    `).run(inTok, outTok, session_id);
+  }
+
+  return Number(result.lastInsertRowid);
+}
+
+export function insertIteration(
+  session_id: string,
+  iteration_number: number,
+  trigger: string,
+  error_summary: string,
+  fix_description?: string,
+  files_changed?: string[],
+  result?: string,
+  duration_seconds?: number
+): number {
+  const d = getDb();
+
+  const stmt = d.prepare(`
+    INSERT INTO iterations (session_id, iteration_number, trigger, error_summary, fix_description, files_changed, result, duration_seconds)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+  const res = stmt.run(
+    session_id,
+    iteration_number,
+    trigger,
+    error_summary,
+    fix_description ?? null,
+    files_changed ? JSON.stringify(files_changed) : null,
+    result ?? "fail",
+    duration_seconds ?? null
+  );
+
+  // Update total_iterations on the session
+  d.prepare(`
+    UPDATE sessions SET total_iterations = (
+      SELECT COUNT(*) FROM iterations WHERE session_id = ?
+    ) WHERE id = ?
+  `).run(session_id, session_id);
+
+  return Number(res.lastInsertRowid);
+}
+
+export function insertScreenshot(
+  session_id: string,
+  iteration_number: number | undefined,
+  phase: string,
+  description: string,
+  file_path: string,
+  event_id?: number
+): number {
+  const d = getDb();
+
+  // Insert first to get the ID
+  const stmt = d.prepare(`
+    INSERT INTO screenshots (session_id, iteration_number, phase, description, file_path, event_id)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+  const result = stmt.run(
+    session_id,
+    iteration_number ?? null,
+    phase,
+    description,
+    file_path,
+    event_id ?? null
+  );
+  const screenshotId = Number(result.lastInsertRowid);
+
+  // Try to copy the file into /data/screenshots/ for self-contained serving
+  const ext = path.extname(file_path) || ".png";
+  const storedName = `${session_id}_${screenshotId}${ext}`;
+  const thumbName = `${session_id}_${screenshotId}_thumb${ext}`;
+  const storedPath = path.join("/data", "screenshots", storedName);
+  const thumbPath = path.join("/data", "screenshots", thumbName);
+
+  try {
+    // The file_path might be a host path — try multiple locations
+    const candidates = [
+      file_path,
+      // Map host path into /data mount: /Users/.../outputs/foo → /data/foo
+      file_path.replace(/^.*?outputs\//, "/data/"),
+    ];
+
+    for (const candidate of candidates) {
+      if (fs.existsSync(candidate)) {
+        // Copy full-size
+        fs.copyFileSync(candidate, storedPath);
+
+        // Generate thumbnail async (don't block the response)
+        sharp(candidate)
+          .resize(300)
+          .toFile(thumbPath)
+          .then(() => {
+            d.prepare("UPDATE screenshots SET stored_path = ?, thumb_path = ? WHERE id = ?").run(storedPath, thumbPath, screenshotId);
+          })
+          .catch(() => {
+            // Thumbnail generation failed — just store the full path
+            d.prepare("UPDATE screenshots SET stored_path = ? WHERE id = ?").run(storedPath, screenshotId);
+          });
+
+        break;
+      }
+    }
+  } catch {
+    // File copy failed — stored_path stays null, original file_path still works via volume mount
+  }
+
+  return screenshotId;
+}
+
+const MIME_TYPES: Record<string, string> = {
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".gif": "image/gif",
+  ".svg": "image/svg+xml",
+  ".webp": "image/webp",
+  ".pdf": "application/pdf",
+  ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  ".json": "application/json",
+  ".csv": "text/csv",
+  ".txt": "text/plain",
+  ".md": "text/markdown",
+  ".html": "text/html",
+  ".zip": "application/zip",
+};
+
+export function insertArtifact(
+  session_id: string,
+  file_path: string,
+  artifact_type: string,
+  description: string,
+  metadata?: object,
+  event_id?: number
+): number {
+  const d = getDb();
+
+  const ext = path.extname(file_path).toLowerCase();
+  const mime_type = MIME_TYPES[ext] || "application/octet-stream";
+
+  const stmt = d.prepare(`
+    INSERT INTO artifacts (session_id, artifact_type, description, file_path, mime_type, metadata, event_id)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+  `);
+  const result = stmt.run(
+    session_id,
+    artifact_type,
+    description,
+    file_path,
+    mime_type,
+    metadata ? JSON.stringify(metadata) : null,
+    event_id ?? null
+  );
+  const artifactId = Number(result.lastInsertRowid);
+
+  // Try to copy the file into /data/artifacts/ for self-contained serving
+  const storedName = `${session_id}_${artifactId}${ext}`;
+  const storedPath = path.join("/data", "artifacts", storedName);
+
+  try {
+    const candidates = [
+      file_path,
+      file_path.replace(/^.*?outputs\//, "/data/"),
+    ];
+
+    for (const candidate of candidates) {
+      if (fs.existsSync(candidate)) {
+        fs.copyFileSync(candidate, storedPath);
+        const stat = fs.statSync(storedPath);
+        d.prepare("UPDATE artifacts SET stored_path = ?, file_size = ? WHERE id = ?").run(
+          storedPath,
+          stat.size,
+          artifactId
+        );
+        break;
+      }
+    }
+  } catch {
+    // File copy failed — stored_path stays null
+  }
+
+  return artifactId;
+}
+
+export function endSession(
+  session_id: string,
+  status: string,
+  total_input_tokens?: number,
+  total_output_tokens?: number,
+  final_result?: FinalResult
+): { duration_seconds: number; total_iterations: number } {
+  const d = getDb();
+
+  const session = d
+    .prepare(`SELECT started_at, total_iterations, total_input_tokens, total_output_tokens FROM sessions WHERE id = ?`)
+    .get(session_id) as { started_at: string; total_iterations: number; total_input_tokens: number; total_output_tokens: number } | undefined;
+
+  if (!session) {
+    throw new Error(`Session ${session_id} not found`);
+  }
+
+  const startedAt = new Date(session.started_at + "Z");
+  const now = new Date();
+  const duration_seconds = Math.round((now.getTime() - startedAt.getTime()) / 1000);
+
+  // Use caller-provided totals only if they exceed the already-accumulated values.
+  // This prevents overwriting accumulated event-level tokens with 0.
+  let finalInput = Math.max(total_input_tokens ?? 0, session.total_input_tokens);
+  let finalOutput = Math.max(total_output_tokens ?? 0, session.total_output_tokens);
+
+  // Estimation fallback: if still 0/0, estimate from event count.
+  // Claude Code agents can't access their own token counts, so this provides
+  // a rough estimate rather than showing misleading zeroes.
+  if (finalInput === 0 && finalOutput === 0) {
+    const eventCount = (d.prepare(
+      "SELECT COUNT(*) as count FROM events WHERE session_id = ?"
+    ).get(session_id) as { count: number }).count;
+
+    if (eventCount > 0) {
+      // Each MCP tool call round-trip is ~3k input + ~1.5k output tokens on average
+      finalInput = eventCount * 3000;
+      finalOutput = eventCount * 1500;
+    }
+  }
+
+  d.prepare(`
+    UPDATE sessions
+    SET status = ?,
+        total_input_tokens = ?,
+        total_output_tokens = ?,
+        final_result = ?,
+        ended_at = datetime('now'),
+        duration_seconds = ?
+    WHERE id = ?
+  `).run(
+    status,
+    finalInput,
+    finalOutput,
+    final_result ? JSON.stringify(final_result) : null,
+    duration_seconds,
+    session_id
+  );
+
+  return { duration_seconds, total_iterations: session.total_iterations };
+}
+
+export function createPlan(
+  session_id: string,
+  steps: Array<{
+    step_number: number;
+    title: string;
+    description?: string;
+    parent_step?: number;
+    file_targets?: string[];
+  }>
+): number[] {
+  const d = getDb();
+  const stmt = d.prepare(`
+    INSERT INTO plan_steps (session_id, step_number, title, description, parent_step, file_targets)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+
+  const ids: number[] = [];
+  const insertAll = d.transaction(() => {
+    for (const step of steps) {
+      const result = stmt.run(
+        session_id,
+        step.step_number,
+        step.title,
+        step.description ?? "",
+        step.parent_step ?? null,
+        step.file_targets ? JSON.stringify(step.file_targets) : null
+      );
+      ids.push(Number(result.lastInsertRowid));
+    }
+  });
+  insertAll();
+  return ids;
+}
+
+export function updatePlanStep(
+  step_id: number,
+  status: string,
+  outcome?: string
+): void {
+  const d = getDb();
+
+  if (status === "in_progress") {
+    d.prepare(`
+      UPDATE plan_steps SET status = ?, started_at = datetime('now') WHERE id = ?
+    `).run(status, step_id);
+  } else if (status === "completed" || status === "failed" || status === "skipped") {
+    // Calculate duration from started_at
+    const step = d.prepare("SELECT started_at FROM plan_steps WHERE id = ?").get(step_id) as { started_at: string | null } | undefined;
+    let duration: number | null = null;
+    if (step?.started_at) {
+      duration = Math.round((Date.now() - new Date(step.started_at + "Z").getTime()) / 1000);
+    }
+    d.prepare(`
+      UPDATE plan_steps SET status = ?, outcome = ?, completed_at = datetime('now'), duration_seconds = ? WHERE id = ?
+    `).run(status, outcome ?? null, duration, step_id);
+  } else {
+    d.prepare(`UPDATE plan_steps SET status = ? WHERE id = ?`).run(status, step_id);
+  }
+}
+
+export function getPlanSteps(session_id: string): import("./types.js").PlanStep[] {
+  const d = getDb();
+  return d.prepare(
+    "SELECT * FROM plan_steps WHERE session_id = ? ORDER BY step_number ASC"
+  ).all(session_id) as import("./types.js").PlanStep[];
+}
+
+export function insertInsight(
+  session_id: string,
+  insight_type: string,
+  category: string,
+  title: string,
+  description: string,
+  evidence?: string,
+  confidence?: number,
+  file_patterns?: string[],
+  error_patterns?: string[]
+): number {
+  const d = getDb();
+  const stmt = d.prepare(`
+    INSERT INTO insights (session_id, insight_type, category, title, description, evidence, confidence, file_patterns, error_patterns)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+  const result = stmt.run(
+    session_id,
+    insight_type,
+    category,
+    title,
+    description,
+    evidence ?? null,
+    confidence ?? 0.5,
+    file_patterns ? JSON.stringify(file_patterns) : null,
+    error_patterns ? JSON.stringify(error_patterns) : null
+  );
+  return Number(result.lastInsertRowid);
+}
+
+export function validateInsight(insight_id: number): void {
+  const d = getDb();
+  d.prepare(`
+    UPDATE insights SET times_validated = times_validated + 1, confidence = MIN(1.0, confidence + 0.1)
+    WHERE id = ?
+  `).run(insight_id);
+}
+
+export function getInsightsForContext(
+  category?: string,
+  file_pattern?: string,
+  limit: number = 20
+): import("./types.js").Insight[] {
+  const d = getDb();
+
+  if (category && file_pattern) {
+    return d.prepare(`
+      SELECT * FROM insights
+      WHERE category = ? AND (file_patterns LIKE ? OR file_patterns IS NULL)
+      ORDER BY confidence DESC, times_validated DESC
+      LIMIT ?
+    `).all(category, `%${file_pattern}%`, limit) as import("./types.js").Insight[];
+  }
+
+  if (category) {
+    return d.prepare(`
+      SELECT * FROM insights
+      WHERE category = ?
+      ORDER BY confidence DESC, times_validated DESC
+      LIMIT ?
+    `).all(category, limit) as import("./types.js").Insight[];
+  }
+
+  return d.prepare(`
+    SELECT * FROM insights
+    ORDER BY confidence DESC, times_validated DESC
+    LIMIT ?
+  `).all(limit) as import("./types.js").Insight[];
+}
+
+export function getSessionSummary(session_id?: string): {
+  session: Session;
+  events: Event[];
+  iterations: Iteration[];
+  screenshots: Screenshot[];
+  artifacts: Artifact[];
+  insights: import("./types.js").Insight[];
+  plan_steps: import("./types.js").PlanStep[];
+  child_sessions: Session[];
+} | null {
+  const d = getDb();
+
+  let session: Session | undefined;
+  if (session_id) {
+    session = d
+      .prepare(`SELECT * FROM sessions WHERE id = ?`)
+      .get(session_id) as Session | undefined;
+  } else {
+    session = d
+      .prepare(`SELECT * FROM sessions ORDER BY created_at DESC LIMIT 1`)
+      .get() as Session | undefined;
+  }
+
+  if (!session) {
+    return null;
+  }
+
+  const events = d
+    .prepare(`SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`)
+    .all(session.id) as Event[];
+
+  const iterations = d
+    .prepare(
+      `SELECT * FROM iterations WHERE session_id = ? ORDER BY iteration_number ASC`
+    )
+    .all(session.id) as Iteration[];
+
+  const screenshots = d
+    .prepare(`SELECT * FROM screenshots WHERE session_id = ? ORDER BY timestamp ASC`)
+    .all(session.id) as Screenshot[];
+
+  const artifacts = d
+    .prepare(`SELECT * FROM artifacts WHERE session_id = ? ORDER BY timestamp ASC`)
+    .all(session.id) as Artifact[];
+
+  const insights = d
+    .prepare(`SELECT * FROM insights WHERE session_id = ? ORDER BY confidence DESC`)
+    .all(session.id) as import("./types.js").Insight[];
+
+  const plan_steps = d
+    .prepare(`SELECT * FROM plan_steps WHERE session_id = ? ORDER BY step_number ASC`)
+    .all(session.id) as import("./types.js").PlanStep[];
+
+  const child_sessions = d
+    .prepare(`SELECT * FROM sessions WHERE parent_session_id = ? ORDER BY created_at ASC`)
+    .all(session.id) as Session[];
+
+  return { session, events, iterations, screenshots, artifacts, insights, plan_steps, child_sessions };
+}
+
+export interface ComparisonFinding {
+  severity: string;
+  title: string;
+  details: string;
+}
+
+export interface ComparisonResult {
+  slide: number;
+  template: string;
+  diff_pct: number;
+  findings: ComparisonFinding[];
+  message: string;
+  screenshot_id: number | null;
+  screenshot_url: string | null;
+  timestamp: string;
+}
+
+export function getComparisonResults(
+  session_id: string,
+  min_severity?: string
+): { results: ComparisonResult[]; summary: Record<string, unknown> | null } {
+  const d = getDb();
+
+  // Get per-slide comparison events (phase=verification, event_type=action with findings in metadata)
+  const events = d.prepare(`
+    SELECT e.id, e.message, e.metadata, e.timestamp
+    FROM events e
+    WHERE e.session_id = ?
+      AND e.phase = 'verification'
+      AND e.event_type = 'action'
+      AND e.metadata LIKE '%"findings"%'
+    ORDER BY e.timestamp ASC
+  `).all(session_id) as Array<{ id: number; message: string; metadata: string | null; timestamp: string }>;
+
+  const severityRank: Record<string, number> = { CRITICAL: 3, NOTABLE: 2, MINOR: 1, GOOD: 0 };
+  const minRank = min_severity ? (severityRank[min_severity.toUpperCase()] ?? 0) : 0;
+
+  const results: ComparisonResult[] = [];
+
+  for (const ev of events) {
+    if (!ev.metadata) continue;
+    let meta: Record<string, unknown>;
+    try {
+      meta = JSON.parse(ev.metadata);
+    } catch {
+      continue;
+    }
+
+    const findings = (meta.findings as ComparisonFinding[] | undefined) ?? [];
+
+    // Filter by minimum severity if requested
+    const filtered = min_severity
+      ? findings.filter((f) => (severityRank[f.severity?.toUpperCase()] ?? 0) >= minRank)
+      : findings;
+
+    if (min_severity && filtered.length === 0) continue;
+
+    // Find linked screenshot
+    const screenshot = d.prepare(
+      "SELECT id FROM screenshots WHERE session_id = ? AND event_id = ? LIMIT 1"
+    ).get(session_id, ev.id) as { id: number } | undefined;
+
+    // If no event_id link, try matching by timestamp proximity
+    let screenshotId = screenshot?.id ?? null;
+    if (!screenshotId) {
+      const nearby = d.prepare(
+        "SELECT id FROM screenshots WHERE session_id = ? AND phase = 'verification' AND abs(julianday(timestamp) - julianday(?)) < 0.001 LIMIT 1"
+      ).get(session_id, ev.timestamp) as { id: number } | undefined;
+      screenshotId = nearby?.id ?? null;
+    }
+
+    results.push({
+      slide: (meta.slide as number) ?? 0,
+      template: (meta.template as string) ?? "unknown",
+      diff_pct: (meta.diff_pct as number) ?? 0,
+      findings: filtered,
+      message: ev.message,
+      screenshot_id: screenshotId,
+      screenshot_url: screenshotId ? `/api/screenshots/by-id/${screenshotId}` : null,
+      timestamp: ev.timestamp,
+    });
+  }
+
+  // Get the summary event if it exists
+  const summaryEvent = d.prepare(`
+    SELECT metadata FROM events
+    WHERE session_id = ?
+      AND phase = 'deliverables'
+      AND event_type = 'info'
+      AND metadata LIKE '%"total"%'
+    ORDER BY timestamp DESC LIMIT 1
+  `).get(session_id) as { metadata: string | null } | undefined;
+
+  let summary: Record<string, unknown> | null = null;
+  if (summaryEvent?.metadata) {
+    try {
+      summary = JSON.parse(summaryEvent.metadata);
+    } catch { /* ignore */ }
+  }
+
+  return { results, summary };
+}
+
+export function listSessions(
+  limit: number = 10,
+  status?: string,
+  parent_session_id?: string | null,
+  user_email?: string
+): Session[] {
+  const d = getDb();
+  const conditions: string[] = [];
+  const params: unknown[] = [];
+
+  if (parent_session_id) {
+    conditions.push("parent_session_id = ?");
+    params.push(parent_session_id);
+  } else {
+    // Default: only show top-level sessions (no parent)
+    conditions.push("parent_session_id IS NULL");
+  }
+
+  if (status) {
+    conditions.push("status = ?");
+    params.push(status);
+  }
+
+  if (user_email) {
+    conditions.push("user_email = ?");
+    params.push(user_email);
+  }
+
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  params.push(limit);
+
+  return d
+    .prepare(`SELECT * FROM sessions ${where} ORDER BY created_at DESC LIMIT ?`)
+    .all(...params) as Session[];
+}
+
+// --- Walkthrough functions ---
+
+export function createWalkthrough(
+  url: string,
+  app_name?: string,
+  scenario?: string
+): number {
+  const d = getDb();
+  const result = d.prepare(`
+    INSERT INTO walkthroughs (url, app_name, scenario)
+    VALUES (?, ?, ?)
+  `).run(url, app_name ?? null, scenario ?? null);
+  return Number(result.lastInsertRowid);
+}
+
+export function insertWalkthroughAction(
+  walkthrough_id: number,
+  action_type: string,
+  target: string,
+  value?: string,
+  status?: string,
+  message?: string
+): number {
+  const d = getDb();
+  const result = d.prepare(`
+    INSERT INTO walkthrough_actions (walkthrough_id, action_type, target, value, status, message)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `).run(walkthrough_id, action_type, target, value ?? null, status ?? "pass", message ?? null);
+  return Number(result.lastInsertRowid);
+}
+
+export function insertWalkthroughScreenshot(
+  walkthrough_id: number,
+  name: string,
+  image_data?: string,
+  annotation?: string,
+  action_id?: number,
+  file_path_input?: string
+): number {
+  const d = getDb();
+
+  const safeName = name.replace(/[^a-zA-Z0-9_-]/g, "_");
+  const filename = `wt_${walkthrough_id}_${Date.now()}_${safeName}.png`;
+  const storedPath = path.join("/data", "walkthrough-screenshots", filename);
+
+  if (file_path_input && fs.existsSync(file_path_input)) {
+    // Copy from file path
+    fs.copyFileSync(file_path_input, storedPath);
+  } else if (image_data) {
+    // Decode base64 and save to disk
+    const buffer = Buffer.from(image_data, "base64");
+    fs.writeFileSync(storedPath, buffer);
+  } else {
+    throw new Error("Either image_data or file_path must be provided");
+  }
+
+  const result = d.prepare(`
+    INSERT INTO walkthrough_screenshots (walkthrough_id, action_id, name, annotation, stored_path)
+    VALUES (?, ?, ?, ?, ?)
+  `).run(walkthrough_id, action_id ?? null, name, annotation ?? null, storedPath);
+  return Number(result.lastInsertRowid);
+}
+
+export function endWalkthrough(
+  walkthrough_id: number,
+  status: string,
+  summary: string,
+  total_actions?: number,
+  passed?: number,
+  failed?: number
+): void {
+  const d = getDb();
+  d.prepare(`
+    UPDATE walkthroughs
+    SET status = ?, summary = ?, total_actions = ?, passed = ?, failed = ?, ended_at = datetime('now')
+    WHERE id = ?
+  `).run(status, summary, total_actions ?? 0, passed ?? 0, failed ?? 0, walkthrough_id);
+}
+
+export function listWalkthroughs(limit: number = 50): Walkthrough[] {
+  const d = getDb();
+  return d.prepare(
+    "SELECT * FROM walkthroughs ORDER BY started_at DESC LIMIT ?"
+  ).all(limit) as Walkthrough[];
+}
+
+export function getWalkthroughDetail(id: number): {
+  walkthrough: Walkthrough;
+  actions: WalkthroughAction[];
+  screenshots: WalkthroughScreenshot[];
+} | null {
+  const d = getDb();
+  const walkthrough = d.prepare("SELECT * FROM walkthroughs WHERE id = ?").get(id) as Walkthrough | undefined;
+  if (!walkthrough) return null;
+
+  const actions = d.prepare(
+    "SELECT * FROM walkthrough_actions WHERE walkthrough_id = ? ORDER BY timestamp ASC"
+  ).all(id) as WalkthroughAction[];
+
+  const screenshots = d.prepare(
+    "SELECT * FROM walkthrough_screenshots WHERE walkthrough_id = ? ORDER BY timestamp ASC"
+  ).all(id) as WalkthroughScreenshot[];
+
+  return { walkthrough, actions, screenshots };
+}