trajectories 0.0.0 → 0.1.2

package/legacy/src/ingest.ts

@@ -0,0 +1,791 @@
+import { createHash } from "node:crypto";
+import { closeSync, fstatSync, openSync, readSync, statSync, type BigIntStats } from "node:fs";
+import { TombstonedSessionError, type IngestCommitOutcome, type TrajectoryDb } from "./db.js";
+import { aggregateIngestDiagnostics } from "./diagnostics.js";
+import { nativeId } from "./identity.js";
+import { canonicalRecordsWithDiagnostics, normalizeDiscoveredSession } from "./normalizer.js";
+import type {
+  CanonicalDbRecord,
+  DiscoveredSession,
+  IngestCursor,
+  IngestDiagnosticAggregate,
+  SessionRecord,
+} from "./types.js";
+import { nativeIdentity, nativeSourceFingerprint } from "./native.js";
+import { NativeInputError } from "./native/common.js";
+
+const INCREMENTAL_JSONL_SOURCES = new Set([
+  "claude-code", "codex", "cursor", "pi", "omp", "droid",
+]);
+export const DEFAULT_INGEST_CHUNK_BYTES = 1024 * 1024;
+export const DEFAULT_INGEST_RECORD_BYTES = 64 * 1024 * 1024;
+// Bump when normalization semantics change so cached deterministic failures are retried.
+// Version 5 rebuilds old truncated records using the full-history normalizer.
+export const INGEST_VERSION = 5;
+export const APPEND_CHECKPOINT_VERSION = 1;
+export const APPEND_TAIL_BYTES = 128 * 1024;
+const HASH_BUFFER_BYTES = 64 * 1024;
+
+/**
+ * `tombstoned` is not a failure and not a skip: it is the verdict a
+ * user-initiated deletion produces on the ingest side, and it is its own kind so
+ * a pass reports "the user deleted this" rather than folding it into a counter
+ * that means "the source had nothing for us".
+ */
+export type IngestKind = "imported" | "appended" | "deduped" | "skipped" | "unchanged" | "deferred" | "tombstoned";
+
+export interface IngestWork {
+  source_bytes_read: number;
+  source_bytes_hashed: number;
+  historical_records_hashed: number;
+}
+
+export interface IngestResult {
+  kind: IngestKind;
+  sessionId?: string;
+  warnings?: string[];
+  work?: IngestWork;
+}
+
+export interface IngestOptions {
+  embed?: boolean;
+  /** Bounds one normalization call; larger individual lines require an explicit larger budget. */
+  maxChunkBytes?: number;
+  /** Hard bound for one JSONL record or one assembled native session. */
+  maxRecordBytes?: number;
+  /** Maximum source bytes this call may read before leaving its checkpoint unchanged. */
+  maxSourceBytes?: number;
+  /** Application-enforced source read rate. Omit for interactive foreground use. */
+  readBytesPerSecond?: number;
+  /** Cooperative deadline; the sync parent also enforces a kill deadline around the worker. */
+  maxWallMs?: number;
+}
+
+export interface SourceFingerprint {
+  mtime: string;
+  size: number;
+  file_identity?: string | null;
+}
+
+interface FileIdentity {
+  version: number;
+  device: string;
+  inode: string;
+  birthtime: string;
+  modified: string;
+  changed: string;
+  waitingForNewline: boolean;
+  copilotProjection?: 1;
+  failureLimits?: { chunk: number; record: number };
+  claude?: ClaudeClassification;
+}
+
+interface ClaudeClassification {
+  mode: "parent" | "standalone";
+  groupId: string;
+}
+
+export class IngestBudgetExceeded extends Error {
+  constructor(
+    readonly budget: "source_bytes" | "wall_time",
+    readonly work: IngestWork,
+  ) {
+    super(`Ingestion ${budget} budget exhausted`);
+    this.name = "IngestBudgetExceeded";
+  }
+}
+
+class IngestWorkMeter {
+  sourceBytesRead = 0;
+  sourceBytesHashed = 0;
+  historicalRecordsHashed = 0;
+  private readonly startedAt = performance.now();
+  private readonly maxSourceBytes?: number;
+  private readonly readBytesPerSecond?: number;
+  private readonly deadline?: number;
+
+  constructor(options: IngestOptions) {
+    this.maxSourceBytes = positiveBudget(options.maxSourceBytes, "maxSourceBytes");
+    this.readBytesPerSecond = positiveBudget(options.readBytesPerSecond, "readBytesPerSecond");
+    const maxWallMs = positiveBudget(options.maxWallMs, "maxWallMs");
+    this.deadline = maxWallMs === undefined ? undefined : this.startedAt + maxWallMs;
+  }
+
+  checkpoint(): void {
+    if (this.deadline !== undefined && performance.now() >= this.deadline) {
+      throw new IngestBudgetExceeded("wall_time", this.snapshot());
+    }
+  }
+
+  /**
+   * Fail closed before a whole-artifact allocation that cannot finish under
+   * the remaining source-byte budget. Does not charge or rate-limit; later
+   * reads still go through prepareRead.
+   */
+  rejectUnbudgetedAllocation(bytes: number): void {
+    this.checkpoint();
+    if (this.exceedsSourceBudget(bytes)) {
+      throw new IngestBudgetExceeded("source_bytes", this.snapshot());
+    }
+  }
+
+  prepareRead(bytes: number): void {
+    this.checkpoint();
+    if (this.exceedsSourceBudget(bytes)) {
+      throw new IngestBudgetExceeded("source_bytes", this.snapshot());
+    }
+    if (this.readBytesPerSecond !== undefined) {
+      const due = this.startedAt + ((this.sourceBytesRead + bytes) / this.readBytesPerSecond) * 1000;
+      if (this.deadline !== undefined && due >= this.deadline) {
+        throw new IngestBudgetExceeded("wall_time", this.snapshot());
+      }
+      const delay = due - performance.now();
+      if (delay > 0) Bun.sleepSync(delay);
+      this.checkpoint();
+    }
+  }
+
+  read(bytes: number): void {
+    this.sourceBytesRead += bytes;
+  }
+
+  hashed(bytes: number): void {
+    this.sourceBytesHashed += bytes;
+    this.checkpoint();
+  }
+
+  snapshot(): IngestWork {
+    return {
+      source_bytes_read: this.sourceBytesRead,
+      source_bytes_hashed: this.sourceBytesHashed,
+      historical_records_hashed: this.historicalRecordsHashed,
+    };
+  }
+
+  private exceedsSourceBudget(bytes: number): boolean {
+    return this.maxSourceBytes !== undefined && this.sourceBytesRead + bytes > this.maxSourceBytes;
+  }
+}
+
+function positiveBudget(value: number | undefined, name: string): number | undefined {
+  if (value === undefined) return undefined;
+  if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive safe integer`);
+  return value;
+}
+
+export function isIncrementalJsonl(item: DiscoveredSession): boolean {
+  return !item.native && INCREMENTAL_JSONL_SOURCES.has(item.source) && item.path.endsWith(".jsonl");
+}
+
+function fingerprintFromStat(stat: BigIntStats, source?: string): SourceFingerprint {
+  const identity: FileIdentity = {
+    version: INGEST_VERSION,
+    device: String(stat.dev), inode: String(stat.ino), birthtime: String(stat.birthtimeNs),
+    modified: String(stat.mtimeNs), changed: String(stat.ctimeNs), waitingForNewline: false,
+    // Rebuild this source's old lossy projection without invalidating other readers.
+    ...(source === "copilot-cli" ? { copilotProjection: 1 as const } : {}),
+  };
+  return { mtime: stat.mtime.toISOString(), size: Number(stat.size), file_identity: JSON.stringify(identity) };
+}
+
+function fileIdentity(value?: string | null): FileIdentity | null {
+  if (!value) return null;
+  try {
+    const parsed = JSON.parse(value);
+    if (parsed?.version !== INGEST_VERSION || !["device", "inode", "birthtime", "modified", "changed"]
+      .every(key => typeof parsed[key] === "string") || typeof parsed.waitingForNewline !== "boolean") return null;
+    if (parsed.claude && (!["parent", "standalone"].includes(parsed.claude.mode) ||
+      typeof parsed.claude.groupId !== "string" || !parsed.claude.groupId)) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+function sameGeneration(a: FileIdentity | null, b: FileIdentity | null): boolean {
+  return !!a && !!b && a.device === b.device && a.inode === b.inode && a.birthtime === b.birthtime;
+}
+
+function sameSnapshot(a: SourceFingerprint, b: SourceFingerprint): boolean {
+  if (a.mtime !== b.mtime || a.size !== b.size) return false;
+  const native = nativeIdentity(a.file_identity);
+  if (native) return native.snapshot === nativeIdentity(b.file_identity)?.snapshot;
+  if (!a.file_identity && !b.file_identity) return true;
+  const left = fileIdentity(a.file_identity);
+  const right = fileIdentity(b.file_identity);
+  return sameGeneration(left, right) && left!.modified === right!.modified && left!.changed === right!.changed &&
+    left!.copilotProjection === right!.copilotProjection;
+}
+
+export function sourceFingerprint(item: DiscoveredSession): SourceFingerprint | null {
+  if (item.native) return nativeSourceFingerprint(item);
+  if (item.source === "hermes") return { mtime: item.updatedAt ?? "", size: item.sizeBytes ?? -1 };
+  try {
+    const stat = statSync(item.path, { bigint: true });
+    return stat.isFile() ? fingerprintFromStat(stat, item.source) : null;
+  } catch {
+    if (!isIncrementalJsonl(item) && item.updatedAt != null && item.sizeBytes != null) {
+      return { mtime: item.updatedAt, size: item.sizeBytes };
+    }
+    return null;
+  }
+}
+
+function isUnchanged(item: DiscoveredSession, cursor: IngestCursor | null, fingerprint: SourceFingerprint, options: IngestOptions): boolean {
+  if (!cursor || !sameSnapshot(cursor, fingerprint)) return false;
+  if (cursor.status === "pending") return false;
+  if (cursor.status === "failed") {
+    if (item.native) return nativeIdentity(cursor.file_identity)?.failureMaxBytes === (options.maxRecordBytes ?? DEFAULT_INGEST_RECORD_BYTES);
+    const limits = fileIdentity(cursor.file_identity)?.failureLimits;
+    return !!limits && limits.chunk === (options.maxChunkBytes ?? DEFAULT_INGEST_CHUNK_BYTES) &&
+      limits.record === (options.maxRecordBytes ?? DEFAULT_INGEST_RECORD_BYTES);
+  }
+  if (!isIncrementalJsonl(item)) return true;
+  if (!cursor.prefix_hash || !fileIdentity(cursor.file_identity)) return false;
+  return cursor.bytes_ingested === fingerprint.size ||
+    (cursor.bytes_ingested < fingerprint.size && fileIdentity(cursor.file_identity)!.waitingForNewline);
+}
+
+export function sessionNeedsIngest(db: TrajectoryDb, item: DiscoveredSession, options: IngestOptions = {}): boolean {
+  // A deleted session is never a candidate: the listing's own axis refuses it
+  // here, before the source is stat-ed, so no pass queues work for it and the
+  // sync planner's `removeQueuedIngest` drops one that was already queued.
+  if (db.tombstoneForListing(item)) return false;
+  const fingerprint = sourceFingerprint(item);
+  return !!fingerprint && !isUnchanged(item, db.getIngestCursor(item.source, item.id, item.path), fingerprint, options);
+}
+
+const TRANSIENT_NATIVE_CODES = new Set(["EBUSY", "EAGAIN", "ETIMEDOUT", "EINTR"]);
+
+function publicErrorLabel(error: unknown): string {
+  const name = error instanceof NativeInputError ? "NativeInputError"
+    : error instanceof Error && error.name ? error.name : "Error";
+  if (!error || typeof error !== "object" || !("code" in error)) return name;
+  const code = error.code;
+  if (typeof code === "number" && Number.isFinite(code)) return `${name}/${code}`;
+  if (typeof code !== "string" || code.length === 0 || code.length > 64) return name;
+  if (code.includes("/") || code.includes("\\") || code.includes("\0")) return name;
+  return `${name}/${code}`;
+}
+
+function isTransientNativeError(error: unknown): boolean {
+  if (!error || typeof error !== "object" || !("code" in error) || typeof error.code !== "string") return false;
+  return TRANSIENT_NATIVE_CODES.has(error.code) ||
+    error.code.startsWith("SQLITE_BUSY") || error.code.startsWith("SQLITE_LOCKED");
+}
+
+/** NativeInputError and lock/busy classes skip; unexpected failures rethrow. */
+export function nativeIngestCatchResult(error: unknown, sessionId: string): IngestResult {
+  const label = publicErrorLabel(error);
+  if (error instanceof NativeInputError) {
+    return warning({ kind: "skipped", sessionId },
+      `Native assembly or normalization failed (${label}); retry after source or byte-limit changes.`);
+  }
+  if (isTransientNativeError(error)) {
+    return warning({ kind: "skipped", sessionId },
+      `Native ingestion deferred (${label}); source remains eligible for retry.`);
+  }
+  throw error;
+}
+
+/**
+ * The pass's verdict for a commit the store refused because the session was
+ * deleted: `tombstoned`, the same answer `sessionNeedsIngest` gives a listing
+ * before it is read, arriving later for the shapes that axis cannot name — a
+ * source that groups several files under one session id reads a member listing
+ * under a different id first, and a queued item can outlive the deletion that
+ * now covers it. Every other error is not this function's to interpret.
+ */
+function tombstonedIngestResult(error: unknown): IngestResult {
+  if (!(error instanceof TombstonedSessionError)) throw error;
+  return { kind: "tombstoned", sessionId: error.sessionId };
+}
+
+export async function ingestDiscoveredSession(
+  db: TrajectoryDb, item: DiscoveredSession, options: IngestOptions = {},
+): Promise<IngestResult> {
+  const work = new IngestWorkMeter(options);
+  // Admission, before the source is opened or stat-ed: the double-track's
+  // listing axis, which the sync planner already consults and which every other
+  // caller (`build`, `watch`, the worker child) reaches through this function.
+  // The store's own write boundary refuses the same session again by its
+  // committed id, which is what covers a source whose listings name a session
+  // differently than its group does.
+  const tombstone = db.tombstoneForListing(item);
+  if (tombstone) return { kind: "tombstoned", sessionId: tombstone.session_id, work: work.snapshot() };
+  const fingerprint = sourceFingerprint(item);
+  if (!fingerprint) return { kind: "skipped", sessionId: item.id, work: work.snapshot() };
+  const cursor = db.getIngestCursor(item.source, item.id, item.path);
+  if (isUnchanged(item, cursor, fingerprint, options)) {
+    return { kind: "unchanged", sessionId: cursor!.session_id ?? item.id, work: work.snapshot() };
+  }
+  if (isIncrementalJsonl(item)) {
+    try {
+      const result = await ingestJsonl(db, item, cursor, options, work);
+      return { ...result, work: work.snapshot() };
+    } catch (error) {
+      return { ...tombstonedIngestResult(error), work: work.snapshot() };
+    }
+  }
+
+  let deterministicFailure = true;
+  let norm: Awaited<ReturnType<typeof normalizeDiscoveredSession>>;
+  if (!item.native && item.source !== "hermes" && item.source !== "deepagents" && fingerprint.size >= 0) {
+    let fd: number;
+    try { fd = openSync(item.path, "r"); }
+    catch { return { ...warning({ kind: "skipped", sessionId: item.id }, "Source could not be opened; checkpoint unchanged."), work: work.snapshot() }; }
+    try {
+      const stat = fstatSync(fd, { bigint: true });
+      if (!stat.isFile() || !sameSnapshot(fingerprint, fingerprintFromStat(stat, item.source))) {
+        return { ...changedDuringRead(item), work: work.snapshot() };
+      }
+      if (item.source === "copilot-cli" && Number(stat.size) > (options.maxRecordBytes ?? DEFAULT_INGEST_RECORD_BYTES)) {
+        return { ...warning({ kind: "skipped", sessionId: item.id },
+          `Copilot complete journal exceeds maxRecordBytes=${options.maxRecordBytes ?? DEFAULT_INGEST_RECORD_BYTES}; checkpoint unchanged. Increase the bound explicitly.`), work: work.snapshot() };
+      }
+      // Non-JSONL adapters need the whole artifact. Reject before alloc so a
+      // large file cannot OOM the worker ahead of the remaining source-byte budget.
+      work.rejectUnbudgetedAllocation(fingerprint.size);
+      const transcript = Buffer.alloc(fingerprint.size);
+      readExactly(fd, transcript, 0, work);
+      norm = await normalizeDiscoveredSession(item, { transcript: transcript.toString("utf8") });
+    } finally {
+      closeSync(fd);
+    }
+  } else {
+    // SQLite/checkpoint adapters own their reads. Conservatively admit and
+    // charge the discovered artifact size; sync still runs them in a worker.
+    const admitted = Math.max(fingerprint.size, 0);
+    work.prepareRead(admitted);
+    work.read(admitted);
+    norm = await normalizeDiscoveredSession(item, {
+      maxSourceBytes: options.maxRecordBytes,
+      onReadError: error => { deterministicFailure = error instanceof NativeInputError; },
+    });
+  }
+  work.checkpoint();
+  const current = sourceFingerprint(item);
+  if (!current || !sameSnapshot(fingerprint, current) || !cursorStillCurrent(db, item, cursor)) {
+    return { ...changedDuringRead(item), work: work.snapshot() };
+  }
+  if (!norm) {
+    const identity = nativeIdentity(fingerprint.file_identity);
+    const failed = identity && deterministicFailure ? { ...fingerprint, file_identity: JSON.stringify({ ...identity, failureMaxBytes: options.maxRecordBytes ?? DEFAULT_INGEST_RECORD_BYTES }) } : fingerprint;
+    db.upsertIngestCursor(makeCursor(item, failed, {
+      session_id: cursor?.session_id ?? null, bytes_ingested: cursor?.bytes_ingested ?? 0,
+      record_count: cursor?.record_count ?? 0, status: "failed",
+    }));
+    return { ...warning({ kind: "skipped", sessionId: item.id }, identity && deterministicFailure
+      ? "Native assembly or normalization failed; retry after source or byte-limit changes."
+      : "Normalization failed; source remains eligible for retry."), work: work.snapshot() };
+  }
+  const next = makeCursor(item, fingerprint, {
+    session_id: norm.session.id, bytes_ingested: Math.max(fingerprint.size, 0),
+    record_count: norm.records.length, status: "ok",
+  });
+  let saved: IngestCommitOutcome;
+  try {
+    saved = db.commitIngest(norm.session, norm.records, next, false, true, {
+      diagnostics: aggregateIngestDiagnostics(norm.diagnostics),
+    });
+  } catch (error) {
+    return { ...tombstonedIngestResult(error), work: work.snapshot() };
+  }
+  work.historicalRecordsHashed += saved.historicalRecordsHashed;
+  return { ...await finishEmbedding(db, norm.session, norm.records, {
+    kind: saved.deduped ? "deduped" : "imported", sessionId: norm.session.id,
+  }, options.embed), work: work.snapshot() };
+}
+
+async function ingestJsonl(
+  db: TrajectoryDb, item: DiscoveredSession, cursor: IngestCursor | null, options: IngestOptions,
+  work: IngestWorkMeter,
+): Promise<IngestResult> {
+  const budget = options.maxChunkBytes ?? DEFAULT_INGEST_CHUNK_BYTES;
+  const recordLimit = options.maxRecordBytes ?? DEFAULT_INGEST_RECORD_BYTES;
+  if (!Number.isSafeInteger(budget) || budget <= 0) throw new Error("maxChunkBytes must be a positive safe integer");
+  if (!Number.isSafeInteger(recordLimit) || recordLimit < budget) throw new Error("maxRecordBytes must be a safe integer at least maxChunkBytes");
+  let fd: number;
+  try { fd = openSync(item.path, "r"); }
+  catch { return warning({ kind: "skipped", sessionId: item.id }, "Source could not be opened; checkpoint unchanged."); }
+  try {
+    const stat = fstatSync(fd, { bigint: true });
+    if (!stat.isFile()) return { kind: "skipped", sessionId: item.id };
+    const snapshot = fingerprintFromStat(stat);
+    let session = cursor ? db.getIngestSession(cursor) : null;
+    let offset = 0;
+    let prefixHash = initialSourceCheckpoint();
+    let lines = 0;
+    let previousTail: Buffer<ArrayBufferLike> = Buffer.alloc(0);
+    if (session && cursor && cursor.bytes_ingested > 0) {
+      const sameFile = cursor.bytes_ingested <= snapshot.size &&
+        sameGeneration(fileIdentity(cursor.file_identity), fileIdentity(snapshot.file_identity));
+      if (!sameFile || !hasAppendCheckpoint(cursor)) {
+        return deferForRebuild(db, item, cursor, snapshot,
+          "Existing history needs a bounded cold rebuild before another append.");
+      }
+      const candidate = verifiedAppendCheckpoint(fd, cursor, work);
+      if (!candidate) {
+        return deferForRebuild(db, item, cursor, snapshot,
+          "Source continuity anchor changed; live history was preserved for a bounded rebuild.");
+      }
+      offset = cursor.bytes_ingested;
+      prefixHash = cursor.prefix_hash!;
+      lines = cursor.line_count!;
+      previousTail = candidate.tail;
+    }
+    let claude: ClaudeClassification | undefined;
+    if (item.source === "claude-code") {
+      const previous = fileIdentity(cursor?.file_identity)?.claude;
+      const classification = previous && cursor && offset > 0
+        ? previous : classifyClaude(fd, snapshot.size, budget, recordLimit, work);
+      if (typeof classification === "string") {
+        if (!snapshotStillCurrent(fd, item, snapshot) || !cursorStillCurrent(db, item, cursor)) return changedDuringRead(item);
+        const identity = fileIdentity(snapshot.file_identity)!;
+        identity.failureLimits = { chunk: budget, record: recordLimit };
+        db.upsertIngestCursor(makeCursor(item, { ...snapshot, file_identity: JSON.stringify(identity) }, {
+          session_id: session?.id ?? cursor?.session_id ?? null, bytes_ingested: offset,
+          record_count: offset ? session?.record_count ?? 0 : 0, status: "failed", prefix_hash: prefixHash,
+          checkpoint_version: APPEND_CHECKPOINT_VERSION, line_count: lines,
+          ...tailFields(offset, previousTail),
+        }));
+        return warning({ kind: "skipped", sessionId: item.id }, classification);
+      }
+      claude = classification;
+      if (!previous || previous.mode !== claude.mode || previous.groupId !== claude.groupId) {
+        if (offset > 0) {
+          return deferForRebuild(db, item, cursor!, snapshot,
+            "Claude source classification changed; live history was preserved for a bounded rebuild.");
+        }
+        offset = 0;
+        prefixHash = initialSourceCheckpoint();
+        lines = 0;
+        previousTail = Buffer.alloc(0);
+      }
+    }
+    const append = offset > 0 && !!session;
+    if (!append) session = null;
+
+    const buffer = readChunk(fd, offset, snapshot.size, budget, recordLimit, work);
+    const consumed = buffer.lastIndexOf(0x0a) + 1;
+    const reachedEof = offset + buffer.length === snapshot.size;
+    const waitingForNewline = reachedEof && consumed < buffer.length;
+    const nextIdentity = fileIdentity(snapshot.file_identity)!;
+    nextIdentity.waitingForNewline = waitingForNewline;
+    if (claude) nextIdentity.claude = claude;
+    const checkpointSnapshot = { ...snapshot, file_identity: JSON.stringify(nextIdentity) };
+    const next = makeCursor(item, checkpointSnapshot, {
+      // Cold passes null `session` before building the checkpoint; keep claiming
+      // the prior committed owner so failure/skip writes cannot unclaim a live
+      // session and strand a later genuine replacement.
+      session_id: session?.id ?? cursor?.session_id ?? null, bytes_ingested: offset, record_count: session?.record_count ?? 0,
+      status: "ok", prefix_hash: prefixHash, checkpoint_version: APPEND_CHECKPOINT_VERSION,
+      line_count: lines, ...tailFields(offset, previousTail),
+    });
+
+    if (consumed === 0) {
+      if (!snapshotStillCurrent(fd, item, snapshot) || !cursorStillCurrent(db, item, cursor)) return changedDuringRead(item);
+      if (!reachedEof) {
+        next.status = "failed";
+        nextIdentity.failureLimits = { chunk: budget, record: recordLimit };
+        next.file_identity = JSON.stringify(nextIdentity);
+        db.upsertIngestCursor(next);
+        return warning({ kind: "skipped", sessionId: session?.id ?? item.id },
+          `JSONL line at byte ${offset} exceeds maxRecordBytes=${recordLimit}; no bytes skipped. Increase maxRecordBytes explicitly or use a bounded staged importer.`);
+      }
+      db.upsertIngestCursor(next);
+      return { kind: session ? "unchanged" : "skipped", sessionId: session?.id ?? item.id };
+    }
+
+    const complete = buffer.subarray(0, consumed);
+    const supplied = claude?.mode === "parent" ? maskClaudeSidechains(complete) : complete;
+    let records: CanonicalDbRecord[] | null;
+    let diagnostics: IngestDiagnosticAggregate[] | undefined;
+    if (append) {
+      // A continuation chunk is normalized on its own, so its diagnostics cover
+      // exactly the records it contributes to the staged replacement.
+      const chunked = canonicalRecordsWithDiagnostics(
+        item.source as Parameters<typeof canonicalRecordsWithDiagnostics>[0], supplied.toString("utf8"),
+        nativeId(item.source, session!.id), offset, lines,
+      );
+      records = chunked?.records ?? null;
+      if (chunked) diagnostics = aggregateIngestDiagnostics(chunked.diagnostics);
+    } else {
+      const norm = await normalizeDiscoveredSession(claude ? { ...item, id: claude.groupId } : item, {
+        transcript: supplied.toString("utf8"), ...(claude ? { groupId: claude.groupId } : {}),
+      });
+      session = norm?.session ?? null;
+      records = norm?.records ?? null;
+      if (norm) diagnostics = aggregateIngestDiagnostics(norm.diagnostics);
+    }
+    work.checkpoint();
+    if (!snapshotStillCurrent(fd, item, snapshot) || !cursorStillCurrent(db, item, cursor)) return changedDuringRead(item);
+    if (!session || !records) {
+      next.status = "failed";
+      nextIdentity.failureLimits = { chunk: budget, record: recordLimit };
+      next.file_identity = JSON.stringify(nextIdentity);
+      db.upsertIngestCursor(next);
+      return warning({ kind: "skipped", sessionId: item.id }, `Normalization failed at byte ${offset}; checkpoint did not advance.`);
+    }
+
+    prefixHash = extendSourceCheckpoint(prefixHash, complete, work);
+    next.bytes_ingested = offset + consumed;
+    next.prefix_hash = prefixHash;
+    next.line_count = lines + countNewlines(complete);
+    const nextTail = appendTail(previousTail, complete);
+    Object.assign(next, tailFields(next.bytes_ingested, nextTail));
+    next.session_id = session.id;
+    const saved = db.commitIngest(session, records, next, append, reachedEof, { diagnostics });
+    work.historicalRecordsHashed += saved.historicalRecordsHashed;
+    if (saved.deferred) {
+      return warning({ kind: "deferred", sessionId: session.id },
+        `Append requires cold rebuild (${saved.deferReason ?? "non_append_change"}); live history was preserved.`);
+    }
+    if (saved.staged) return { kind: "unchanged", sessionId: session.id };
+    return finishEmbedding(db, session, records, {
+      kind: saved.inserted || saved.updated ? "imported" :
+        (append ? (saved.appended > 0 ? "appended" : "unchanged") : "deduped"),
+      sessionId: session.id,
+    }, options.embed);
+  } finally {
+    closeSync(fd);
+  }
+}
+
+function* jsonlObjects(buffer: Buffer): Generator<{ value: Record<string, unknown>; start: number; end: number }> {
+  let start = 0;
+  for (let end = buffer.indexOf(0x0a); end >= 0; end = buffer.indexOf(0x0a, start)) {
+    try {
+      const value = JSON.parse(buffer.subarray(start, end).toString("utf8"));
+      if (value && typeof value === "object" && !Array.isArray(value)) yield { value, start, end };
+    } catch { /* Trajectory reports malformed complete lines as recoverable diagnostics. */ }
+    start = end + 1;
+  }
+}
+
+// Resolve only format flags and native grouping; Trajectory still owns all event normalization.
+function classifyClaude(
+  fd: number,
+  size: number,
+  budget: number,
+  recordLimit: number,
+  work: IngestWorkMeter,
+): ClaudeClassification | string {
+  let agentId: string | undefined;
+  let sessionId: string | undefined;
+  let ambiguousAgent = false;
+  let ambiguousSession = false;
+  let ordinary = false;
+  for (let offset = 0; offset < size;) {
+    const buffer = readChunk(fd, offset, size, budget, recordLimit, work);
+    const consumed = buffer.lastIndexOf(0x0a) + 1;
+    if (!consumed) {
+      if (offset + buffer.length < size) return `Claude classification requires a line exceeding maxRecordBytes=${recordLimit} at byte ${offset}; no bytes skipped.`;
+      break;
+    }
+    for (const { value } of jsonlObjects(buffer.subarray(0, consumed))) {
+      const conversational = (value.type === "user" || value.type === "assistant") &&
+        value.message && typeof value.message === "object" && !Array.isArray(value.message);
+      if (!conversational) continue;
+      if (value.isSidechain !== true) {
+        ordinary = true;
+        if (typeof value.sessionId === "string" && value.sessionId) return { mode: "parent", groupId: value.sessionId };
+        continue;
+      }
+      if (typeof value.agentId === "string" && value.agentId) {
+        ambiguousAgent ||= agentId !== undefined && agentId !== value.agentId;
+        agentId ??= value.agentId;
+      }
+      if (typeof value.sessionId === "string" && value.sessionId) {
+        ambiguousSession ||= sessionId !== undefined && sessionId !== value.sessionId;
+        sessionId ??= value.sessionId;
+      }
+    }
+    offset += consumed;
+  }
+  const groupId = !ambiguousAgent && (agentId || (!ambiguousSession && sessionId));
+  if (!ordinary && groupId) return { mode: "standalone", groupId };
+  return "Claude source classification has no unambiguous native group yet; no bytes skipped.";
+}
+
+function maskClaudeSidechains(original: Buffer): Buffer {
+  let masked: Buffer | undefined;
+  for (const { value, start, end } of jsonlObjects(original)) {
+    if (value.isSidechain !== true) continue;
+    masked ??= Buffer.from(original);
+    masked.fill(0x20, start, original[end - 1] === 0x0d ? end - 1 : end);
+  }
+  return masked ?? original;
+}
+
+function readChunk(
+  fd: number,
+  offset: number,
+  size: number,
+  budget: number,
+  recordLimit: number,
+  work: IngestWorkMeter,
+): Buffer {
+  const first = Buffer.alloc(Math.min(budget, size - offset));
+  readExactly(fd, first, offset, work);
+  if (first.includes(0x0a) || offset + first.length === size) return first;
+  const pieces = [first];
+  let length = first.length;
+  while (length < recordLimit && offset + length < size) {
+    const piece = Buffer.alloc(Math.min(HASH_BUFFER_BYTES, recordLimit - length, size - offset - length));
+    readExactly(fd, piece, offset + length, work);
+    const newline = piece.indexOf(0x0a);
+    const used = newline < 0 ? piece : piece.subarray(0, newline + 1);
+    pieces.push(used);
+    length += used.length;
+    if (newline >= 0) break;
+  }
+  return Buffer.concat(pieces, length);
+}
+
+function countNewlines(buffer: Buffer): number {
+  let lines = 0;
+  for (let at = buffer.indexOf(0x0a); at >= 0; at = buffer.indexOf(0x0a, at + 1)) lines++;
+  return lines;
+}
+
+function readExactly(fd: number, buffer: Buffer, offset: number, work?: IngestWorkMeter): void {
+  let read = 0;
+  while (read < buffer.length) {
+    const requested = Math.min(buffer.length - read, work ? HASH_BUFFER_BYTES : buffer.length - read);
+    work?.prepareRead(requested);
+    const count = readSync(fd, buffer, read, requested, offset + read);
+    if (count === 0) throw new Error("Source shortened during snapshot read; checkpoint unchanged");
+    read += count;
+    work?.read(count);
+  }
+}
+
+function sha256(parts: Array<string | Buffer>): string {
+  const hash = createHash("sha256");
+  for (const part of parts) hash.update(part);
+  return hash.digest("hex");
+}
+
+function initialSourceCheckpoint(): string {
+  return sha256([`ledger-source-checkpoint:v${APPEND_CHECKPOINT_VERSION}\n`]);
+}
+
+function extendSourceCheckpoint(previous: string, complete: Buffer, work: IngestWorkMeter): string {
+  if (!/^[a-f0-9]{64}$/.test(previous)) throw new Error("Invalid prior source checkpoint");
+  let state = previous;
+  let start = 0;
+  for (let end = complete.indexOf(0x0a); end >= 0; end = complete.indexOf(0x0a, start)) {
+    const line = complete.subarray(start, end + 1);
+    state = sha256([
+      `ledger-source-line:v${APPEND_CHECKPOINT_VERSION}\n`,
+      Buffer.from(state, "hex"),
+      "\n",
+      line,
+    ]);
+    work.hashed(line.length);
+    start = end + 1;
+  }
+  if (start !== complete.length) throw new Error("Source checkpoint requires complete JSONL lines");
+  return state;
+}
+
+function hasAppendCheckpoint(cursor: IngestCursor): boolean {
+  const expectedTailLength = Math.min(APPEND_TAIL_BYTES, cursor.bytes_ingested);
+  return cursor.checkpoint_version === APPEND_CHECKPOINT_VERSION &&
+    Number.isSafeInteger(cursor.line_count) && cursor.line_count! >= 0 &&
+    typeof cursor.prefix_hash === "string" && /^[a-f0-9]{64}$/.test(cursor.prefix_hash) &&
+    cursor.tail_offset === cursor.bytes_ingested - expectedTailLength &&
+    typeof cursor.tail_hash === "string" && /^[a-f0-9]{64}$/.test(cursor.tail_hash);
+}
+
+function verifiedAppendCheckpoint(
+  fd: number,
+  cursor: IngestCursor,
+  work: IngestWorkMeter,
+): { tail: Buffer } | null {
+  if (!hasAppendCheckpoint(cursor)) return null;
+  const length = cursor.bytes_ingested - cursor.tail_offset!;
+  const tail = Buffer.alloc(length);
+  try { readExactly(fd, tail, cursor.tail_offset!, work); }
+  catch { return null; }
+  return sha256([tail]) === cursor.tail_hash ? { tail } : null;
+}
+
+function appendTail(previous: Buffer, complete: Buffer): Buffer {
+  if (complete.length >= APPEND_TAIL_BYTES) {
+    return Buffer.from(complete.subarray(complete.length - APPEND_TAIL_BYTES));
+  }
+  const keep = Math.min(previous.length, APPEND_TAIL_BYTES - complete.length);
+  return Buffer.concat([previous.subarray(previous.length - keep), complete], keep + complete.length);
+}
+
+function tailFields(offset: number, tail: Buffer): Pick<IngestCursor, "tail_offset" | "tail_hash"> {
+  if (offset === 0) return { tail_offset: null, tail_hash: null };
+  if (tail.length !== Math.min(APPEND_TAIL_BYTES, offset)) throw new Error("Invalid append continuity tail");
+  return { tail_offset: offset - tail.length, tail_hash: sha256([tail]) };
+}
+
+function deferForRebuild(
+  db: TrajectoryDb,
+  item: DiscoveredSession,
+  cursor: IngestCursor,
+  snapshot: SourceFingerprint,
+  message: string,
+): IngestResult {
+  db.deferIngestRebuild(makeCursor(item, snapshot, {
+    session_id: cursor.session_id,
+    bytes_ingested: 0,
+    record_count: 0,
+    status: "pending",
+    prefix_hash: null,
+    checkpoint_version: APPEND_CHECKPOINT_VERSION,
+    line_count: 0,
+    tail_offset: null,
+    tail_hash: null,
+  }));
+  return warning({ kind: "deferred", sessionId: cursor.session_id ?? item.id }, message);
+}
+
+function snapshotStillCurrent(fd: number, item: DiscoveredSession, snapshot: SourceFingerprint): boolean {
+  try {
+    const pathStat = statSync(item.path, { bigint: true });
+    return pathStat.isFile() && sameSnapshot(snapshot, fingerprintFromStat(fstatSync(fd, { bigint: true }))) &&
+      sameSnapshot(snapshot, fingerprintFromStat(pathStat));
+  } catch { return false; }
+}
+
+function cursorStillCurrent(db: TrajectoryDb, item: DiscoveredSession, expected: IngestCursor | null): boolean {
+  return JSON.stringify(db.getIngestCursor(item.source, item.id, item.path)) === JSON.stringify(expected);
+}
+
+function warning(result: IngestResult, message: string): IngestResult {
+  console.warn(`[ledger ingest] ${message}`);
+  return { ...result, warnings: [...(result.warnings ?? []), message] };
+}
+
+function changedDuringRead(item: DiscoveredSession): IngestResult {
+  return warning({ kind: "skipped", sessionId: item.id }, "Source or cursor changed during ingestion; snapshot discarded for retry.");
+}
+
+async function finishEmbedding(
+  db: TrajectoryDb, session: SessionRecord, records: CanonicalDbRecord[], result: IngestResult, embed?: boolean,
+): Promise<IngestResult> {
+  if (embed && records.length > 0) {
+    try { await db.embedSessionRecords(session.id, records); }
+    catch { return warning(result, "Embedding failed after the lexical checkpoint committed; use embedding backfill to retry."); }
+  }
+  return result;
+}
+
+function makeCursor(
+  item: DiscoveredSession, fingerprint: SourceFingerprint,
+  fields: Pick<IngestCursor, "session_id" | "bytes_ingested" | "record_count" | "status"> &
+    Partial<Pick<IngestCursor, "prefix_hash" | "checkpoint_version" | "line_count" | "tail_offset" | "tail_hash">>,
+): IngestCursor {
+  return {
+    source: item.source, listing_id: item.id, source_path: item.path,
+    mtime: fingerprint.mtime, size: fingerprint.size, file_identity: fingerprint.file_identity ?? null,
+    updated_at: new Date().toISOString(), ...fields,
+  };
+}