trajectories 0.0.0 → 0.1.1

package/legacy/src/normalizer.ts

@@ -0,0 +1,457 @@
+import { readFileSync } from "node:fs";
+import { readSnapshot as readPortableSnapshot } from "../../dist/index.js";
+import {
+  NormalizationError,
+  normalizeToCanonical as upstreamNormalizeToCanonical,
+  normalizeTranscript as upstreamNormalizeTranscript,
+} from "../../dist/normalization-runtime.js";
+import { ledgerId } from "./identity.js";
+import {
+  type AnyTrajectorySource,
+  type CanonicalRecord,
+  type Diagnostic,
+  type NormalizedRecord,
+  type TrajectorySource,
+  normalizeCheckpointToCanonical as upstreamNormalizeCheckpointToCanonical,
+} from "@letta-ai/trajectory";
+import { extractSessionParentage, type ParentageDiagnostic } from "./parentage.js";
+import { readNativeSession } from "./native.js";
+import { GROK_PROJECTION_SOURCE } from "./native/grok.js";
+import { KIMI_PROJECTION_SOURCE } from "./native/kimi.js";
+import type { NativeTranscript } from "./native/common.js";
+import { withReadOnlyDb } from "./native/sqlite-ro.js";
+import {
+  hashSessionRecords,
+  SESSION_HASH_VERSION,
+} from "./session-hash.js";
+import type {
+  CanonicalDbRecord,
+  DiscoveredSession,
+  NormalizedSessionResult,
+  SessionRecord,
+} from "./types.js";
+
+// Portable histories must never inherit the search normalizer's truncating defaults.
+const fullHistory = {
+  bounds: { toolArguments: { maxCharacters: null }, toolResults: { maxCharacters: null } },
+  filters: { systemMessages: "include" as const, toolResults: "include" as const },
+};
+const normalizeToCanonical = (input: Parameters<typeof upstreamNormalizeToCanonical>[0]) =>
+  upstreamNormalizeToCanonical({ ...input, ...fullHistory });
+const normalizeTranscript = (input: Parameters<typeof upstreamNormalizeTranscript>[0]) =>
+  upstreamNormalizeTranscript({ ...input, ...fullHistory });
+const normalizeCheckpointToCanonical = (input: Parameters<typeof upstreamNormalizeCheckpointToCanonical>[0]) =>
+  upstreamNormalizeCheckpointToCanonical({ ...input, ...fullHistory });
+
+/**
+ * The sources the pinned library cannot read, each with the lane its native
+ * reader projects into. A source's own reader emits the projection; the
+ * normalizer restores the records' provenance, and treats the listing's own id
+ * as authoritative so a projected stream that carries no group cannot take the
+ * library's `default` sentinel as its session id.
+ */
+const PROJECTION_LANES: Record<string, TrajectorySource> = {
+  grok: GROK_PROJECTION_SOURCE,
+  kimi: KIMI_PROJECTION_SOURCE,
+};
+
+export async function normalizeDiscoveredSession(
+  item: DiscoveredSession,
+  options: { transcript?: string; groupId?: string; maxSourceBytes?: number; onReadError?: (error: unknown) => void; onParentageDiagnostic?: (diagnostic: ParentageDiagnostic) => void } = {},
+): Promise<NormalizedSessionResult | null> {
+  const source = item.source as AnyTrajectorySource;
+
+  let canonicalRecords: CanonicalRecord[] = [];
+  let rawNormalizedRecords: NormalizedRecord[] = [];
+  let diagnostics: Diagnostic[] = [];
+  let fileTranscript: string | null = null;
+  let nativeTranscript: NativeTranscript | undefined;
+
+  try {
+    if (source === "hermes") {
+      // Same gradient as the native adapters: a Hermes store held by its own
+      // writer is read from a private copy instead of failing the session.
+      const payload = withReadOnlyDb(item.path, (db) => JSON.stringify({
+        session: db
+          .query("SELECT * FROM sessions WHERE id = ? LIMIT 1")
+          .get(item.id) ?? undefined,
+        messages: db
+          .query("SELECT * FROM messages WHERE session_id = ? ORDER BY id ASC")
+          .all(item.id),
+      }));
+
+      const canonicalResult = normalizeWithFallback("hermes", payload, item.id);
+      if (!canonicalResult) return null;
+
+      canonicalRecords = canonicalResult.records;
+      diagnostics = canonicalResult.diagnostics;
+
+      try {
+        const normResult = normalizeTranscript({
+          source: "hermes",
+          transcript: payload,
+        });
+        rawNormalizedRecords = normResult.records;
+      } catch {
+        // ignore
+      }
+    } else if (source === "deepagents") {
+      const canonicalResult = await normalizeCheckpointToCanonical({
+        source: "deepagents",
+        checkpoint: {
+          path: item.path,
+          threadId: item.id,
+        },
+      });
+      canonicalRecords = canonicalResult.records;
+      diagnostics = canonicalResult.diagnostics;
+    } else if (item.source === "cline" || item.source === "copilot-cli") {
+      const transcript = options.transcript ?? readFileSync(item.path, "utf8");
+      fileTranscript = transcript;
+      const canonicalResult = (await readPortableSnapshot({
+        harness: item.source === "cline" ? "cline" : "github-copilot",
+        transcript,
+        nativeSessionId: item.id,
+      })).canonical;
+      const nativeId = canonicalResult.records[0]?.source_group_id;
+      if (nativeId !== item.id) throw new Error(`${item.source} packet and native directory identities disagree.`);
+      canonicalRecords = canonicalResult.records;
+      diagnostics = canonicalResult.diagnostics;
+      const seen = new Set<string>();
+      for (const record of canonicalRecords) {
+        if (item.source !== "copilot-cli" && seen.has(record.source_order_id)) continue;
+        seen.add(record.source_order_id);
+        rawNormalizedRecords.push(JSON.parse(record.record_json) as NormalizedRecord);
+      }
+    } else if (item.source === "fx") {
+      fileTranscript = options.transcript ?? readFileSync(item.path, "utf8");
+      // Schema 4 keeps authoritative history and result bodies beside session.json.
+      // Scanner-discovered FX sessions use the path so the SDK can capture them.
+      const canonicalResult = (await readPortableSnapshot({
+        harness: "fx",
+        ...(item.native?.kind === "fx-directory" ? { path: item.path } : { transcript: fileTranscript }),
+        nativeSessionId: item.id,
+      })).canonical;
+      const nativeId = canonicalResult.records[0]?.source_group_id;
+      if (nativeId !== item.id) throw new Error("fx packet and native directory identities disagree.");
+      canonicalRecords = canonicalResult.records;
+      diagnostics = canonicalResult.diagnostics;
+      const seen = new Set<string>();
+      for (const record of canonicalRecords) {
+        if (seen.has(record.source_order_id)) continue;
+        seen.add(record.source_order_id);
+        rawNormalizedRecords.push(JSON.parse(record.record_json) as NormalizedRecord);
+      }
+    } else {
+      // File-backed source
+      if (item.native && options.transcript === undefined) {
+        nativeTranscript = readNativeSession(item, { maxBytes: options.maxSourceBytes });
+      }
+      const transcript = options.transcript ?? nativeTranscript?.transcript ?? readFileSync(item.path, "utf8");
+      fileTranscript = transcript;
+      // The pinned library has no adapter for this source, so the native reader
+      // projects it into one it does know; what it read is still this source, so
+      // the records keep this source's provenance. That projection also makes the
+      // listing's own id authoritative: the reader stamps it on every group
+      // carrier it can emit, and a projected stream that carries none (a torn or
+      // missing sidecar on an agent-only session) would otherwise take the
+      // library's `default` sentinel as its group — every such session sharing one
+      // ledger id, each replacing the previous one's records.
+      const lane = PROJECTION_LANES[item.source];
+      const projected = lane !== undefined;
+      const normalizerSource = projected ? lane : source;
+      const canonicalResult = normalizeWithFallback(
+        normalizerSource, transcript, options.groupId ?? item.id, 0,
+        options.groupId !== undefined || projected,
+      );
+      if (!canonicalResult) return null;
+
+      canonicalRecords = projected
+        ? canonicalResult.records.map((record) => ({ ...record, source_type: item.source }))
+        : canonicalResult.records;
+      diagnostics = [...canonicalResult.diagnostics, ...(nativeTranscript?.diagnostics ?? [])];
+
+      try {
+        const normResult = normalizeTranscript({
+          source: normalizerSource,
+          transcript,
+        });
+        rawNormalizedRecords = normResult.records;
+      } catch {
+        // ignore
+      }
+    }
+  } catch (error) {
+    options.onReadError?.(error);
+    return null;
+  }
+
+  if (canonicalRecords.length === 0) {
+    return null;
+  }
+
+  const nativeSessionId = canonicalRecords[0].source_group_id || item.id;
+  // Upstream group/record identities are scoped to one source. Ledger stores
+  // many independent harnesses in one database, so make that scope explicit in
+  // primary keys rather than relying on native ids to be globally unique.
+  const ledgerSessionId = ledgerId(item.source, nativeSessionId);
+
+  // Extract session metadata
+  let cwd: string | null = null;
+  let model: string | null = null;
+  let startedAt: string | null = null;
+  let endedAt: string | null = null;
+  let firstUserText: string | null = null;
+
+  for (const r of canonicalRecords) {
+    const ts = r.source_timestamp ?? r.record_timestamp;
+    if (ts) {
+      if (!startedAt || ts < startedAt) startedAt = ts;
+      if (!endedAt || ts > endedAt) endedAt = ts;
+    }
+
+    if (r.record_type === "meta") {
+      try {
+        const parsed = JSON.parse(r.record_json);
+        if (parsed.cwd && typeof parsed.cwd === "string") cwd = parsed.cwd;
+        if (parsed.model && typeof parsed.model === "string") model = parsed.model;
+      } catch {
+        // ignore
+      }
+    } else if (r.record_type === "user" && !firstUserText && r.content) {
+      const t = r.content.trim();
+      // Skip harness-injected boilerplate when deriving a session title —
+      // these wrappers (plugin catalogs, env contexts) would otherwise
+      // become the display title of every affected session.
+      const isBoilerplate =
+        t.startsWith("<recommended_plugins>") ||
+        t.startsWith("<environment_context>") ||
+        t.startsWith("<user_info>") ||
+        t.startsWith("<system_warning>");
+      if (!isBoilerplate) {
+        firstUserText = t.slice(0, 120);
+      }
+    }
+  }
+
+  if (!startedAt && item.updatedAt) {
+    startedAt = item.updatedAt;
+    endedAt = item.updatedAt;
+  }
+  cwd = nativeTranscript?.cwd ?? cwd;
+  model = nativeTranscript?.model ?? model;
+  startedAt = nativeTranscript?.startedAt ?? startedAt;
+  endedAt = nativeTranscript?.endedAt ?? endedAt;
+
+  // Accumulate records into an appendable deterministic identity without
+  // rereading historical database rows.
+  const hashRecords = canonicalRecords.map((record) => ({
+    id: ledgerId(item.source, record.record_id),
+    record_type: record.record_type,
+    source_order_id: record.source_order_id,
+    component_index: record.component_index,
+    content_hash: record.content_hash,
+  }));
+  const contentHash = hashSessionRecords(item.source, ledgerSessionId, hashRecords);
+
+  const title = nativeTranscript?.title ?? item.title ?? firstUserText ?? `Session ${nativeSessionId.slice(0, 16)}`;
+  const now = new Date().toISOString();
+  const parentage = extractSessionParentage(item.source, item.path, {
+    sessionId: nativeSessionId,
+    transcript: fileTranscript ?? undefined,
+    // A source whose parentage lives in another session's directory records the
+    // edge at scan time, where the store is walked once; the extractor consumes
+    // it instead of re-walking per session.
+    registryParentSessionId: item.native?.kind === "grok-updates" ? item.native.parentSessionId : undefined,
+    registryParentToolUseId: item.native?.kind === "kimi-wire" ? item.native.parentToolUseId : undefined,
+    onDiagnostic: options.onParentageDiagnostic ?? ((diagnostic) => console.warn(JSON.stringify({ type: "parentage_warning", ...diagnostic }))),
+  });
+  if (parentage.parent_session_id) parentage.parent_session_id = ledgerId(item.source, parentage.parent_session_id);
+  if (parentage.parent_session_id === ledgerSessionId) parentage.parent_session_id = null;
+
+  const session: SessionRecord = {
+    id: ledgerSessionId,
+    source: item.source,
+    source_path: item.path,
+    content_hash: contentHash,
+    content_hash_version: SESSION_HASH_VERSION,
+    title,
+    cwd,
+    model,
+    started_at: startedAt,
+    ended_at: endedAt,
+    record_count: canonicalRecords.length,
+    created_at: now,
+    updated_at: now,
+    ...parentage,
+  };
+
+  const records: CanonicalDbRecord[] = canonicalRecords.map((r) => ({
+    id: ledgerId(item.source, r.record_id),
+    session_id: ledgerSessionId,
+    source_type: r.source_type,
+    record_type: r.record_type,
+    stable_source_record_id: r.stable_source_record_id ?? null,
+    source_identity_kind: r.source_identity_kind ?? null,
+    source_order_id: r.source_order_id,
+    component_index: r.component_index,
+    record_timestamp: r.record_timestamp ?? null,
+    source_timestamp: r.source_timestamp ?? null,
+    content: r.content ?? null,
+    tool_call_id: r.tool_call_id ?? null,
+    tool_name: r.tool_name ?? null,
+    tool_arguments_json: r.tool_arguments_json ?? null,
+    tool_result_json: r.tool_result_json ?? null,
+    tool_result_ok:
+      r.tool_result_ok === true ? 1 : r.tool_result_ok === false ? 0 : null,
+    record_json: r.record_json,
+    content_hash: r.content_hash,
+    record_hash: r.record_hash,
+  }));
+
+  return {
+    session,
+    records,
+    rawNormalizedRecords,
+    diagnostics,
+  };
+}
+
+export function canonicalRecordsFromTranscript(
+  source: TrajectorySource,
+  transcript: string,
+  groupId: string,
+  baseByteOffset = 0,
+  baseLineOffset = 0,
+): CanonicalDbRecord[] | null {
+  return canonicalRecordsWithDiagnostics(source, transcript, groupId, baseByteOffset, baseLineOffset)?.records ?? null;
+}
+
+/**
+ * The same projection as {@link canonicalRecordsFromTranscript}, keeping the
+ * normalization diagnostics of the chunk that produced the records so a chunked
+ * caller can accumulate coverage without re-normalizing the whole source.
+ */
+export function canonicalRecordsWithDiagnostics(
+  source: TrajectorySource,
+  transcript: string,
+  groupId: string,
+  baseByteOffset = 0,
+  baseLineOffset = 0,
+): { records: CanonicalDbRecord[]; diagnostics: Diagnostic[] } | null {
+  // Native hook/user pairing needs the complete journal. Copilot ingestion uses
+  // the bounded whole-file SDK path; the old fragment parser loses direct prompts.
+  if (source === "copilot-cli") return null;
+  if (!Number.isSafeInteger(baseLineOffset) || baseLineOffset < 0) throw new Error("baseLineOffset must be a nonnegative safe integer");
+  const canonicalResult = normalizeWithFallback(
+    source,
+    transcript,
+    groupId,
+    baseByteOffset,
+  );
+  if (!canonicalResult) return null;
+  const sessionId = ledgerId(source, canonicalResult.records[0]?.source_group_id || groupId);
+  const records: CanonicalDbRecord[] = canonicalResult.records.map((r) => ({
+    id: ledgerId(source, r.record_id),
+    session_id: sessionId,
+    source_type: r.source_type,
+    record_type: r.record_type,
+    stable_source_record_id: r.stable_source_record_id ?? null,
+    source_identity_kind: r.source_identity_kind ?? null,
+    source_order_id: rebaseLineOrder(source, r, baseLineOffset),
+    component_index: r.component_index,
+    record_timestamp: r.record_timestamp ?? null,
+    source_timestamp: r.source_timestamp ?? null,
+    content: r.content ?? null,
+    tool_call_id: r.tool_call_id ?? null,
+    tool_name: r.tool_name ?? null,
+    tool_arguments_json: r.tool_arguments_json ?? null,
+    tool_result_json: r.tool_result_json ?? null,
+    tool_result_ok:
+      r.tool_result_ok === true ? 1 : r.tool_result_ok === false ? 0 : null,
+    record_json: r.record_json,
+    content_hash: r.content_hash,
+    record_hash: r.record_hash,
+  }));
+  return { records, diagnostics: canonicalResult.diagnostics };
+}
+
+function rebaseLineOrder(source: string, record: CanonicalRecord, baseLineOffset: number): string {
+  if (!baseLineOffset || record.record_type === "meta" || (source !== "cursor" && source !== "copilot-cli")) {
+    return record.source_order_id;
+  }
+  // Trajectory 0.3 uses chunk-local line sequences for these two adapters.
+  // Project that sequence into the verified source file; native identities stay unchanged.
+  const [rank, timestamp, sequence, ...identity] = record.source_order_id.split("|");
+  if (!sequence || !/^\d+$/.test(sequence) || identity.length === 0) throw new Error("Unsupported Trajectory source order format");
+  const absolute = (BigInt(sequence) + BigInt(baseLineOffset)).toString().padStart(sequence.length, "0");
+  return [rank, timestamp, absolute, ...identity].join("|");
+}
+
+function normalizeWithFallback(
+  source: TrajectorySource,
+  transcript: string,
+  fallbackGroupId: string,
+  baseByteOffset = 0,
+  groupIsAuthoritative = false,
+) {
+  const sourceContext = {
+    groupId: fallbackGroupId,
+    ...(baseByteOffset > 0 ? { baseByteOffset, partial: true as const } : {}),
+  };
+
+  // Step 1: try plain normalizeToCanonical
+  try {
+    return normalizeToCanonical({
+      source,
+      transcript,
+      ...(baseByteOffset > 0 || groupIsAuthoritative ? { sourceContext } : {}),
+    });
+  } catch (e1) {
+    if (e1 instanceof NormalizationError) {
+      if (e1.code === "source_group_required") {
+        // Retry with sourceContext.groupId
+        try {
+          return normalizeToCanonical({
+            source,
+            transcript,
+            sourceContext,
+          });
+        } catch (e2) {
+          if (e2 instanceof NormalizationError && (e2.code === "missing_assistant_records" || e2.code === "missing_user_records")) {
+            try {
+              return normalizeToCanonical({
+                source,
+                transcript,
+                sourceContext: { ...sourceContext, partial: true },
+              });
+            } catch {
+              return null;
+            }
+          }
+          return null;
+        }
+      }
+
+      if (e1.code === "missing_assistant_records" || e1.code === "missing_user_records") {
+        try {
+          return normalizeToCanonical({
+            source,
+            transcript,
+            sourceContext: baseByteOffset > 0 || groupIsAuthoritative ? { ...sourceContext, partial: true } : { partial: true },
+          });
+        } catch (e2) {
+          if (e2 instanceof NormalizationError && e2.code === "source_group_required") {
+            try {
+              return normalizeToCanonical({ source, transcript, sourceContext: { ...sourceContext, partial: true } });
+            } catch {
+              return null;
+            }
+          }
+          return null;
+        }
+      }
+    }
+    return null;
+  }
+}