trajectories 0.0.0 → 0.1.1

package/legacy/scripts/import-claude-history.ts

@@ -0,0 +1,130 @@
+import { normalizeToCanonical } from "@letta-ai/trajectory";
+import { createHash } from "node:crypto";
+import { createReadStream, mkdirSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { TrajectoryDb } from "../src/db.js";
+import { ledgerId } from "../src/identity.js";
+import type { CanonicalDbRecord, SessionRecord } from "../src/types.js";
+import { acquireWriterLease, isMemoryStorePath } from "../src/writer-lease.js";
+
+type SliceOptions = { history: string; session: string; from: string; to: string };
+const sha256 = (value: string | Buffer) => createHash("sha256").update(value).digest("hex");
+
+async function* sourceLines(path: string) {
+  let pending = Buffer.alloc(0), offset = 0, line = 0;
+  for await (const chunk of createReadStream(path)) {
+    pending = Buffer.concat([pending, chunk as Buffer]);
+    let end: number;
+    while ((end = pending.indexOf(10)) !== -1) {
+      const bytes = pending.subarray(0, end + 1);
+      yield { bytes, offset, line: ++line };
+      offset += bytes.length;
+      pending = pending.subarray(end + 1);
+    }
+  }
+  if (pending.length) yield { bytes: pending, offset, line: ++line };
+}
+
+export async function prepareHistorySlice(options: SliceOptions) {
+  const from = Date.parse(options.from), to = Date.parse(options.to);
+  const zoned = /T.*(?:Z|[+-]\d{2}:\d{2})$/;
+  if (!zoned.test(options.from) || !zoned.test(options.to) || !Number.isFinite(from) || !Number.isFinite(to) || from >= to) {
+    throw new Error("from/to must be increasing ISO timestamps with explicit timezones");
+  }
+  if (!options.session) throw new Error("An explicit session ID is required");
+  const history = resolve(options.history);
+  const scope = { history, session: options.session, from: new Date(from).toISOString(), to: new Date(to).toISOString() };
+  const sliceId = `claude-history:${options.session}:${sha256(JSON.stringify(scope))}`;
+  const sessionId = ledgerId("claude-code", sliceId);
+  const records: CanonicalDbRecord[] = [];
+  const evidence: Array<{ record_id: string; line: number; byte_offset: number; byte_length: number; raw_sha256: string }> = [];
+  const rows: Buffer[] = [];
+  let scanned = 0, cwd: string | null = null;
+  for await (const { bytes, offset, line } of sourceLines(history)) {
+    scanned++;
+    let row;
+    try { row = JSON.parse(bytes.toString("utf8")); }
+    catch { throw new Error(`Invalid JSON in history at line ${line}`); }
+    if (row.sessionId !== options.session || typeof row.timestamp !== "number" || row.timestamp < from || row.timestamp >= to) continue;
+    if (typeof row.display !== "string" || !row.display.trim()) continue;
+    if (typeof row.project === "string") cwd ??= row.project;
+
+    // This is a partial prompt projection, not a recovered assistant transcript.
+    // Per-row normalization retains locations in the original file, not a reserialized envelope.
+    const result = normalizeToCanonical({
+      source: "claude-code",
+      transcript: JSON.stringify({ type: "user", timestamp: row.timestamp, message: { content: row.display } }),
+      sourceContext: { groupId: sliceId, baseByteOffset: offset, partial: true },
+    });
+    const users = result.records.filter((r) => r.record_type === "user");
+    if (users.length !== 1 || users[0].content !== row.display || users[0].source_timestamp !== new Date(row.timestamp).toISOString()) {
+      throw new Error(`History mapping changed text or timestamp at line ${line}`);
+    }
+    const r = users[0];
+    records.push({
+      id: ledgerId("claude-code", r.record_id), session_id: sessionId, source_type: r.source_type,
+      record_type: r.record_type, stable_source_record_id: r.stable_source_record_id ?? null,
+      source_identity_kind: r.source_identity_kind ?? null,
+      source_order_id: r.source_order_id, component_index: r.component_index,
+      record_timestamp: r.record_timestamp ?? null, source_timestamp: r.source_timestamp ?? null,
+      content: r.content ?? null, tool_call_id: null, tool_name: null,
+      tool_arguments_json: null, tool_result_json: null, tool_result_ok: null,
+      record_json: r.record_json, content_hash: r.content_hash, record_hash: r.record_hash,
+    });
+    rows.push(Buffer.from(bytes));
+    evidence.push({ record_id: ledgerId("claude-code", r.record_id), line, byte_offset: offset, byte_length: bytes.length, raw_sha256: sha256(bytes) });
+  }
+  if (!records.length) throw new Error("No prompts in the selected session/time window; nothing imported");
+  const raw = Buffer.concat(rows);
+  const timestamps = records.map((r) => r.source_timestamp!).sort();
+  const now = new Date().toISOString();
+  const session: SessionRecord = {
+    id: sessionId, source: "claude-code", source_path: history,
+    content_hash: sha256(JSON.stringify(records.map((r) => [r.id, r.record_hash]))),
+    title: `Claude prompt history (partial): ${scope.from} to ${scope.to}`,
+    cwd, model: null, started_at: timestamps[0], ended_at: timestamps.at(-1)!,
+    record_count: records.length, created_at: now, updated_at: now,
+  };
+  return {
+    session, records, raw,
+    provenance: { mapping: "claude-history-prompt-slice-v1", partial: true, scope, scanned_lines: scanned,
+      selected_prompts: records.length, fixture_sha256: sha256(raw), records: evidence },
+  };
+}
+
+export function saveHistorySlice(slice: Awaited<ReturnType<typeof prepareHistorySlice>>, dbPath: string, evidenceDir: string) {
+  const lease = isMemoryStorePath(dbPath) ? null : acquireWriterLease(dbPath, { holder: "import-claude-history" });
+  try {
+    const directory = resolve(evidenceDir);
+    mkdirSync(directory, { recursive: true, mode: 0o700 });
+    writeFileSync(resolve(directory, "selected-history.jsonl"), slice.raw, { mode: 0o600 });
+    writeFileSync(resolve(directory, "provenance.json"), JSON.stringify(slice.provenance, null, 2) + "\n", { mode: 0o600 });
+    const db = new TrajectoryDb(dbPath);
+    try {
+      return { ...db.saveSession(slice.session, slice.records), session_id: slice.session.id,
+        prompts: slice.records.length, fixture_sha256: slice.provenance.fixture_sha256, evidence: directory };
+    } finally { db.close(); }
+  } finally { lease?.close(); }
+}
+
+export async function runHistoryImport(argv: string[]): Promise<void> {
+  const { values } = parseArgs({ args: argv, options: {
+    history: { type: "string" }, session: { type: "string" }, from: { type: "string" },
+    to: { type: "string" }, db: { type: "string" }, evidence: { type: "string" },
+    robot: { type: "boolean" }, json: { type: "boolean" },
+  } });
+  if (!values.history || !values.session || !values.from || !values.to || !values.db || !values.evidence) {
+    throw new Error("Required: --history --session --from --to --db --evidence (local private directory)");
+  }
+  const started = performance.now(), cpu = process.cpuUsage(), baseline = process.memoryUsage().rss;
+  const slice = await prepareHistorySlice({ history: values.history, session: values.session, from: values.from, to: values.to });
+  const result = saveHistorySlice(slice, values.db, values.evidence);
+  const used = process.cpuUsage(cpu);
+  console.log(JSON.stringify({ ...result, wall_seconds: (performance.now() - started) / 1000,
+    cpu_seconds: (used.user + used.system) / 1e6, rss_baseline_bytes: baseline,
+    process_max_rss_bytes: process.resourceUsage().maxRSS * 1024,
+    rss_before_exit_bytes: process.memoryUsage().rss, embeddings: "not generated; lexical index only" }));
+}
+
+if (import.meta.main) await runHistoryImport(process.argv.slice(2));

package/legacy/scripts/native-history-smoke.ts

@@ -0,0 +1,52 @@
+import { chmodSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { TrajectoryDb } from "../src/db.js";
+import { ingestDiscoveredSession, sessionNeedsIngest } from "../src/ingest.js";
+import { scanSessions } from "../src/scanner.js";
+import type { ScanOutcome } from "../src/types.js";
+
+process.umask(0o077);
+const [source, path] = process.argv.slice(2);
+if (!["cursor", "opencode"].includes(source) || !path || !/\.(?:db|vscdb|sqlite|sqlite3)$/i.test(path)) {
+  throw new Error("Usage: bun scripts/native-history-smoke.ts <cursor|opencode> <native-database-path>");
+}
+const root = mkdtempSync(join(tmpdir(), "trajectories-native-smoke-"));
+chmodSync(root, 0o700);
+const db = new TrajectoryDb(join(root, "trajectories.sqlite"));
+const report = {
+  source, candidateLimit: 3, byteLimit: 8 * 1024 * 1024, discovered: 0,
+  diagnostics: {} as Record<string, number>, outcomes: [] as ScanOutcome[],
+  attempted: 0, stored: 0,
+  records: 0, recordKinds: {} as Record<string, number>,
+  duplicateFree: false, unchanged: false, needsIngest: true,
+};
+try {
+  const { sessions: items, outcomes } = await scanSessions({ sources: [source], roots: { [source]: path }, limit: 3,
+    onDiagnostic: diagnostic => { report.diagnostics[diagnostic.code] = (report.diagnostics[diagnostic.code] ?? 0) + 1; },
+  });
+  report.outcomes = outcomes;
+  report.discovered = items.length;
+  for (const item of items) {
+    report.attempted++;
+    const result = await ingestDiscoveredSession(db, item, { embed: false, maxRecordBytes: report.byteLimit });
+    if (result.kind !== "imported" || !result.sessionId) continue;
+    const records = db.getSessionRecords(result.sessionId);
+    report.stored = db.getSessionCount();
+    if (!records.some(record => record.record_type !== "meta")) continue;
+    report.records = records.length;
+    for (const record of records) report.recordKinds[record.record_type] = (report.recordKinds[record.record_type] ?? 0) + 1;
+    report.duplicateFree = new Set(records.map(record => record.id)).size === records.length;
+    report.unchanged = (await ingestDiscoveredSession(db, item, { embed: false, maxRecordBytes: report.byteLimit })).kind === "unchanged";
+    report.needsIngest = sessionNeedsIngest(db, item, { maxRecordBytes: report.byteLimit });
+    break;
+  }
+  console.log(JSON.stringify(report));
+  if (!report.records || !report.duplicateFree || !report.unchanged || report.needsIngest) process.exitCode = 1;
+} catch {
+  console.log(JSON.stringify({ ...report, error: "native smoke failed; private error details suppressed" }));
+  process.exitCode = 1;
+} finally {
+  db.close();
+  rmSync(root, { recursive: true, force: true });
+}