selftune 0.2.6 → 0.2.8

package/cli/selftune/localdb/materialize.ts

@@ -7,13 +7,20 @@
  *  - Incremental: only inserts records newer than last materialization
  */
 
+// NOTE: With dual-write active (Phase 1+), hooks insert directly into SQLite.
+// The materializer is only needed for:
+//   1. Initial startup (to catch pre-existing JSONL data from before dual-write)
+//   2. Manual recovery after exporting JSONL and recreating the DB file
+//   3. Backfill from batch ingestors that don't yet dual-write
+
 import type { Database } from "bun:sqlite";
-import type {
-  CanonicalExecutionFactRecord,
-  CanonicalPromptRecord,
-  CanonicalRecord,
-  CanonicalSessionRecord,
-  CanonicalSkillInvocationRecord,
+import {
+  type CanonicalExecutionFactRecord,
+  type CanonicalPromptRecord,
+  type CanonicalRecord,
+  type CanonicalSessionRecord,
+  type CanonicalSkillInvocationRecord,
+  isCanonicalRecord,
 } from "@selftune/telemetry-contract";
 import {
   CANONICAL_LOG,
@@ -23,7 +30,6 @@ import {
   TELEMETRY_LOG,
 } from "../constants.js";
 import type { OrchestrateRunReport } from "../dashboard-contract.js";
-import { readEvidenceTrail } from "../evolution/evidence.js";
 import type {
   EvolutionAuditEntry,
   EvolutionEvidenceEntry,
@@ -31,19 +37,134 @@ import type {
   SkillUsageRecord,
 } from "../types.js";
 import { readCanonicalRecords } from "../utils/canonical-log.js";
-import { readJsonl } from "../utils/jsonl.js";
+import { readJsonl, readJsonlFrom } from "../utils/jsonl.js";
 import { readEffectiveSkillUsageRecords } from "../utils/skill-log.js";
 import { getMeta, setMeta } from "./db.js";
 
+/** Tables that contain SQLite-only data (written by hooks, not just materialized from JSONL). */
+const _PROTECTED_TABLES = [
+  { table: "evolution_audit", tsColumn: "timestamp", jsonlLog: EVOLUTION_AUDIT_LOG },
+  { table: "evolution_evidence", tsColumn: "timestamp", jsonlLog: EVOLUTION_EVIDENCE_LOG },
+  { table: "orchestrate_runs", tsColumn: "timestamp", jsonlLog: ORCHESTRATE_RUN_LOG },
+] as const;
+
+/**
+ * Preflight check before full rebuild: detect tables where SQLite has rows
+ * newer than the corresponding JSONL file. If found and `force` is not set,
+ * throw an error so the user can export first.
+ */
+function preflightRebuildGuard(db: Database, options?: MaterializeOptions): void {
+  if (options?.force) return;
+
+  const protectedTables = [
+    {
+      table: "evolution_audit",
+      tsColumn: "timestamp",
+      jsonlLog: options?.evolutionAuditPath ?? EVOLUTION_AUDIT_LOG,
+    },
+    {
+      table: "evolution_evidence",
+      tsColumn: "timestamp",
+      jsonlLog: options?.evolutionEvidencePath ?? EVOLUTION_EVIDENCE_LOG,
+    },
+    {
+      table: "orchestrate_runs",
+      tsColumn: "timestamp",
+      jsonlLog: options?.orchestrateRunLogPath ?? ORCHESTRATE_RUN_LOG,
+    },
+  ];
+
+  const warnings: string[] = [];
+  for (const { table, tsColumn, jsonlLog } of protectedTables) {
+    // Get newest timestamp in SQLite
+    let sqliteMax: string | null = null;
+    try {
+      const row = db.query(`SELECT MAX(${tsColumn}) AS max_ts FROM ${table}`).get() as {
+        max_ts: string | null;
+      } | null;
+      sqliteMax = row?.max_ts ?? null;
+    } catch {
+      continue; // table doesn't exist yet — safe to rebuild
+    }
+
+    if (!sqliteMax) continue; // no rows in SQLite — safe
+
+    // Get newest timestamp from JSONL
+    let jsonlMax: string | null = null;
+    let jsonlBoundaryCount = 0;
+    try {
+      const records = readJsonl<{ timestamp: string }>(jsonlLog);
+      if (records.length > 0) {
+        jsonlMax = records.reduce(
+          (max, r) => (r.timestamp > max ? r.timestamp : max),
+          records[0].timestamp,
+        );
+        jsonlBoundaryCount = records.filter((record) => record.timestamp === jsonlMax).length;
+      }
+    } catch {
+      // JSONL file doesn't exist or is empty — SQLite has data JSONL doesn't
+      jsonlMax = null;
+    }
+
+    let newerCount = 0;
+    let sqliteBoundaryCount = 0;
+    try {
+      if (!jsonlMax) {
+        const row = db.query(`SELECT COUNT(*) AS newer_count FROM ${table}`).get() as {
+          newer_count: number;
+        } | null;
+        newerCount = row?.newer_count ?? 0;
+      } else if (sqliteMax > jsonlMax) {
+        const row = db
+          .query(`SELECT COUNT(*) AS newer_count FROM ${table} WHERE ${tsColumn} > ?`)
+          .get(jsonlMax) as {
+          newer_count: number;
+        } | null;
+        newerCount = row?.newer_count ?? 0;
+      }
+      if (jsonlMax) {
+        const boundaryRow = db
+          .query(`SELECT COUNT(*) AS boundary_count FROM ${table} WHERE ${tsColumn} = ?`)
+          .get(jsonlMax) as {
+          boundary_count: number;
+        } | null;
+        sqliteBoundaryCount = boundaryRow?.boundary_count ?? 0;
+      }
+    } catch {
+      newerCount = 0;
+      sqliteBoundaryCount = 0;
+    }
+
+    if (
+      !jsonlMax ||
+      newerCount > 0 ||
+      (sqliteMax === jsonlMax && sqliteBoundaryCount !== jsonlBoundaryCount)
+    ) {
+      warnings.push(
+        `  - ${table}: ${newerCount} SQLite-only row(s), SQLite max=${sqliteMax}, JSONL max=${jsonlMax ?? "(empty)"}, boundary_count(SQLite=${sqliteBoundaryCount}, JSONL=${jsonlBoundaryCount})`,
+      );
+    }
+  }
+
+  if (warnings.length > 0) {
+    throw new Error(
+      `Rebuild blocked: the following tables have SQLite-only rows that would be lost:\n${warnings.join("\n")}\n\nRun \`selftune export\` first to preserve this data, then retry with --force.`,
+    );
+  }
+}
+
 /** Meta key tracking last materialization timestamp. */
 const META_LAST_MATERIALIZED = "last_materialized_at";
+/** Meta key prefix for per-file byte offsets (append-only incremental reads). */
+const META_OFFSET_PREFIX = "file_offset:";
 
 /**
  * Full rebuild: drop all data tables, then re-insert everything.
  */
 export function materializeFull(db: Database, options?: MaterializeOptions): MaterializeResult {
+  preflightRebuildGuard(db, options);
+
   const tables = [
-    "skill_usage",
     "session_telemetry",
     "evolution_audit",
     "evolution_evidence",
@@ -56,6 +177,8 @@ export function materializeFull(db: Database, options?: MaterializeOptions): Mat
   for (const table of tables) {
     db.run(`DELETE FROM ${table}`);
   }
+  // Clear byte offsets so full rebuild reads from start of each file
+  db.run("DELETE FROM _meta WHERE key LIKE ?", [`${META_OFFSET_PREFIX}%`]);
 
   return materializeIncremental(db, { ...options, since: null });
 }
@@ -67,6 +190,8 @@ export interface MaterializeOptions {
   evolutionEvidencePath?: string;
   orchestrateRunLogPath?: string;
   since?: string | null;
+  /** Skip the preflight rebuild guard (use after `selftune export`). */
+  force?: boolean;
 }
 
 export interface MaterializeResult {
@@ -105,11 +230,30 @@ export function materializeIncremental(
     orchestrateRuns: 0,
   };
 
-  // -- Read all data BEFORE opening the transaction ---------------------------
-  // This keeps file I/O out of the write lock for better concurrency.
+  // -- Read only NEW data using byte offsets -----------------------------------
+  // Append-only JSONL files: track byte offset per file in _meta so we only
+  // read bytes appended since the last materialization. Falls back to full
+  // read when since is null (first run / full rebuild).
 
-  const canonical = readCanonicalRecords(options?.canonicalLogPath ?? CANONICAL_LOG);
-  const filteredCanonical = since ? canonical.filter((r) => r.normalized_at > since) : canonical;
+  function getOffset(filePath: string): number {
+    if (!since) return 0; // full rebuild — read everything
+    const raw = getMeta(db, `${META_OFFSET_PREFIX}${filePath}`);
+    return raw ? Number.parseInt(raw, 10) : 0;
+  }
+  const newOffsets: Array<[string, number]> = [];
+
+  const canonicalPath = options?.canonicalLogPath ?? CANONICAL_LOG;
+  let filteredCanonical: CanonicalRecord[];
+  if (!since) {
+    filteredCanonical = readCanonicalRecords(canonicalPath);
+  } else {
+    const { records, newOffset } = readJsonlFrom<CanonicalRecord>(
+      canonicalPath,
+      getOffset(canonicalPath),
+    );
+    filteredCanonical = records.filter(isCanonicalRecord);
+    newOffsets.push([canonicalPath, newOffset]);
+  }
 
   // Pre-partition canonical records by kind (single pass instead of 4x full scan)
   const byKind = new Map<string, CanonicalRecord[]>();
@@ -119,27 +263,63 @@ export function materializeIncremental(
     else byKind.set(r.record_kind, [r]);
   }
 
-  const telemetry = readJsonl<SessionTelemetryRecord>(options?.telemetryLogPath ?? TELEMETRY_LOG);
-  const filteredTelemetry = since ? telemetry.filter((r) => r.timestamp > since) : telemetry;
+  const telemetryPath = options?.telemetryLogPath ?? TELEMETRY_LOG;
+  let filteredTelemetry: SessionTelemetryRecord[];
+  if (!since) {
+    filteredTelemetry = readJsonl<SessionTelemetryRecord>(telemetryPath);
+  } else {
+    const { records, newOffset } = readJsonlFrom<SessionTelemetryRecord>(
+      telemetryPath,
+      getOffset(telemetryPath),
+    );
+    filteredTelemetry = records;
+    newOffsets.push([telemetryPath, newOffset]);
+  }
 
+  // Skill usage uses a merge of raw + repaired logs — always full read
+  // since readEffectiveSkillUsageRecords handles dedup internally.
+  // However, when doing incremental, filter by timestamp.
   const skills = readEffectiveSkillUsageRecords();
   const filteredSkills = since ? skills.filter((r) => r.timestamp > since) : skills;
 
-  const audit = readJsonl<EvolutionAuditEntry>(options?.evolutionAuditPath ?? EVOLUTION_AUDIT_LOG);
-  const filteredAudit = since ? audit.filter((r) => r.timestamp > since) : audit;
+  const auditPath = options?.evolutionAuditPath ?? EVOLUTION_AUDIT_LOG;
+  let filteredAudit: EvolutionAuditEntry[];
+  if (!since) {
+    filteredAudit = readJsonl<EvolutionAuditEntry>(auditPath);
+  } else {
+    const { records, newOffset } = readJsonlFrom<EvolutionAuditEntry>(
+      auditPath,
+      getOffset(auditPath),
+    );
+    filteredAudit = records;
+    newOffsets.push([auditPath, newOffset]);
+  }
 
-  const evidence = readEvidenceTrail(
-    undefined,
-    options?.evolutionEvidencePath ?? EVOLUTION_EVIDENCE_LOG,
-  );
-  const filteredEvidence = since ? evidence.filter((r) => r.timestamp > since) : evidence;
+  const evidencePath = options?.evolutionEvidencePath ?? EVOLUTION_EVIDENCE_LOG;
+  let filteredEvidence: EvolutionEvidenceEntry[];
+  if (!since) {
+    filteredEvidence = readJsonl<EvolutionEvidenceEntry>(evidencePath);
+  } else {
+    const { records, newOffset } = readJsonlFrom<EvolutionEvidenceEntry>(
+      evidencePath,
+      getOffset(evidencePath),
+    );
+    filteredEvidence = records;
+    newOffsets.push([evidencePath, newOffset]);
+  }
 
-  const orchestrateRuns = readJsonl<OrchestrateRunReport>(
-    options?.orchestrateRunLogPath ?? ORCHESTRATE_RUN_LOG,
-  );
-  const filteredOrchestrateRuns = since
-    ? orchestrateRuns.filter((r) => r.timestamp > since)
-    : orchestrateRuns;
+  const orchestratePath = options?.orchestrateRunLogPath ?? ORCHESTRATE_RUN_LOG;
+  let filteredOrchestrateRuns: OrchestrateRunReport[];
+  if (!since) {
+    filteredOrchestrateRuns = readJsonl<OrchestrateRunReport>(orchestratePath);
+  } else {
+    const { records, newOffset } = readJsonlFrom<OrchestrateRunReport>(
+      orchestratePath,
+      getOffset(orchestratePath),
+    );
+    filteredOrchestrateRuns = records;
+    newOffsets.push([orchestratePath, newOffset]);
+  }
 
   // -- Insert everything inside a single transaction --------------------------
   db.run("BEGIN TRANSACTION");
@@ -154,6 +334,10 @@ export function materializeIncremental(
     result.evolutionEvidence = insertEvolutionEvidence(db, filteredEvidence);
     result.orchestrateRuns = insertOrchestrateRuns(db, filteredOrchestrateRuns);
 
+    // Persist byte offsets so next incremental run skips already-read data
+    for (const [filePath, offset] of newOffsets) {
+      setMeta(db, `${META_OFFSET_PREFIX}${filePath}`, String(offset));
+    }
     setMeta(db, META_LAST_MATERIALIZED, now);
     db.run("COMMIT");
   } catch (err) {
@@ -167,12 +351,24 @@ export function materializeIncremental(
 // -- Insert helpers -----------------------------------------------------------
 
 function insertSessions(db: Database, records: CanonicalRecord[]): number {
+  // Use upsert to merge non-null fields from duplicate session records.
+  // Multiple canonical records may exist for the same session (e.g., Stop hook
+  // writes one without model, replay ingestor writes another with model).
   const stmt = db.prepare(`
-    INSERT OR IGNORE INTO sessions
+    INSERT INTO sessions
       (session_id, started_at, ended_at, platform, model, completion_status,
        source_session_kind, agent_cli, workspace_path, repo_remote, branch,
        schema_version, normalized_at)
     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(session_id) DO UPDATE SET
+      started_at = COALESCE(sessions.started_at, excluded.started_at),
+      ended_at = COALESCE(sessions.ended_at, excluded.ended_at),
+      model = COALESCE(sessions.model, excluded.model),
+      completion_status = COALESCE(sessions.completion_status, excluded.completion_status),
+      agent_cli = COALESCE(sessions.agent_cli, excluded.agent_cli),
+      repo_remote = COALESCE(sessions.repo_remote, excluded.repo_remote),
+      branch = COALESCE(sessions.branch, excluded.branch),
+      workspace_path = COALESCE(sessions.workspace_path, excluded.workspace_path)
   `);
 
   let count = 0;
@@ -223,16 +419,31 @@ function insertPrompts(db: Database, records: CanonicalRecord[]): number {
 }
 
 function insertSkillInvocations(db: Database, records: CanonicalRecord[]): number {
+  // Ensure session stubs exist for FK satisfaction — hooks may write
+  // skill_invocation records before a full session record is available.
+  const sessionStub = db.prepare(`
+    INSERT OR IGNORE INTO sessions
+      (session_id, platform, schema_version, normalized_at)
+    VALUES (?, ?, ?, ?)
+  `);
+
   const stmt = db.prepare(`
     INSERT OR IGNORE INTO skill_invocations
       (skill_invocation_id, session_id, occurred_at, skill_name, invocation_mode,
-       triggered, confidence, tool_name, matched_prompt_id)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+       triggered, confidence, tool_name, matched_prompt_id, agent_type,
+       query, skill_path, skill_scope, source)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `);
 
   let count = 0;
   for (const r of records) {
     const si = r as CanonicalSkillInvocationRecord;
+    sessionStub.run(
+      si.session_id,
+      si.platform ?? "unknown",
+      si.schema_version ?? "1.0.0",
+      si.normalized_at ?? new Date().toISOString(),
+    );
     stmt.run(
       si.skill_invocation_id,
       si.session_id,
@@ -243,6 +454,11 @@ function insertSkillInvocations(db: Database, records: CanonicalRecord[]): numbe
       si.confidence,
       si.tool_name ?? null,
       si.matched_prompt_id ?? null,
+      si.agent_type ?? null,
+      ((si as Record<string, unknown>).query as string) ?? null,
+      ((si as Record<string, unknown>).skill_path as string) ?? null,
+      ((si as Record<string, unknown>).skill_scope as string) ?? null,
+      ((si as Record<string, unknown>).source as string) ?? null,
     );
     count++;
   }
@@ -281,12 +497,29 @@ function insertExecutionFacts(db: Database, records: CanonicalRecord[]): number
 
 function insertSessionTelemetry(db: Database, records: SessionTelemetryRecord[]): number {
   const stmt = db.prepare(`
-    INSERT OR IGNORE INTO session_telemetry
+    INSERT INTO session_telemetry
       (session_id, timestamp, cwd, transcript_path, tool_calls_json,
        total_tool_calls, bash_commands_json, skills_triggered_json,
        skills_invoked_json, assistant_turns, errors_encountered,
        transcript_chars, last_user_query, source, input_tokens, output_tokens)
     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(session_id) DO UPDATE SET
+      timestamp = excluded.timestamp,
+      cwd = COALESCE(excluded.cwd, session_telemetry.cwd),
+      transcript_path = COALESCE(excluded.transcript_path, session_telemetry.transcript_path),
+      source = COALESCE(excluded.source, session_telemetry.source),
+      tool_calls_json = excluded.tool_calls_json,
+      total_tool_calls = excluded.total_tool_calls,
+      bash_commands_json = excluded.bash_commands_json,
+      skills_triggered_json = COALESCE(excluded.skills_triggered_json, session_telemetry.skills_triggered_json),
+      skills_invoked_json = COALESCE(excluded.skills_invoked_json, session_telemetry.skills_invoked_json),
+      assistant_turns = excluded.assistant_turns,
+      errors_encountered = excluded.errors_encountered,
+      transcript_chars = excluded.transcript_chars,
+      last_user_query = excluded.last_user_query,
+      input_tokens = COALESCE(excluded.input_tokens, session_telemetry.input_tokens),
+      output_tokens = COALESCE(excluded.output_tokens, session_telemetry.output_tokens)
+    WHERE session_telemetry.timestamp IS NULL OR excluded.timestamp >= session_telemetry.timestamp
   `);
 
   let count = 0;
@@ -315,24 +548,44 @@ function insertSessionTelemetry(db: Database, records: SessionTelemetryRecord[])
 }
 
 function insertSkillUsage(db: Database, records: SkillUsageRecord[]): number {
-  // Uses INSERT OR IGNORE with a UNIQUE index on the dedup composite key
-  // (idx_skill_usage_dedup defined in schema.ts).
+  // Skill usage records now go into the unified skill_invocations table.
+  // Uses INSERT OR IGNORE with the dedup index on skill_invocations.
+  const sessionStub = db.prepare(`
+    INSERT OR IGNORE INTO sessions
+      (session_id, platform, schema_version, normalized_at)
+    VALUES (?, ?, ?, ?)
+  `);
+
   const stmt = db.prepare(`
-    INSERT OR IGNORE INTO skill_usage
-      (timestamp, session_id, skill_name, skill_path, skill_scope, query, triggered, source)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    INSERT OR IGNORE INTO skill_invocations
+      (skill_invocation_id, session_id, occurred_at, skill_name, invocation_mode,
+       triggered, confidence, tool_name, matched_prompt_id, agent_type,
+       query, skill_path, skill_scope, source)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `);
 
   let count = 0;
   for (const r of records) {
+    // Ensure session stub exists for FK satisfaction
+    sessionStub.run(r.session_id, "unknown", "1.0.0", new Date().toISOString());
+
+    // Derive a unique skill_invocation_id for skill_usage records
+    const invocationId = `${r.session_id}:su:${r.timestamp}:${r.skill_name}`;
+
     stmt.run(
-      r.timestamp,
+      invocationId,
       r.session_id,
+      r.timestamp, // timestamp → occurred_at
       r.skill_name,
+      null, // invocation_mode — not available from skill_usage
+      r.triggered ? 1 : 0,
+      null, // confidence — not available from skill_usage
+      null, // tool_name — not available from skill_usage
+      null, // matched_prompt_id — not available from skill_usage
+      null, // agent_type — not available from skill_usage
+      r.query,
       r.skill_path,
       r.skill_scope ?? null,
-      r.query,
-      r.triggered ? 1 : 0,
       r.source ?? null,
     );
     count++;
@@ -345,8 +598,8 @@ function insertEvolutionAudit(db: Database, records: EvolutionAuditEntry[]): num
   // (idx_evo_audit_dedup defined in schema.ts).
   const stmt = db.prepare(`
     INSERT OR IGNORE INTO evolution_audit
-      (timestamp, proposal_id, skill_name, action, details, eval_snapshot_json)
-    VALUES (?, ?, ?, ?, ?, ?)
+      (timestamp, proposal_id, skill_name, action, details, eval_snapshot_json, iterations_used)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
   `);
 
   let count = 0;
@@ -358,6 +611,7 @@ function insertEvolutionAudit(db: Database, records: EvolutionAuditEntry[]): num
       r.action,
       r.details,
       r.eval_snapshot ? JSON.stringify(r.eval_snapshot) : null,
+      r.iterations_used ?? null,
     );
     count++;
   }