trajectories 0.0.0 → 0.1.2

package/legacy/src/native/grok.ts

@@ -0,0 +1,540 @@
+import { existsSync, lstatSync, readdirSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join, resolve } from "node:path";
+import type { Diagnostic } from "@letta-ai/trajectory";
+import type { DiscoveredSession } from "../types.js";
+import {
+  MAX_NATIVE_RECORDS, NativeInputError, nativeByteLimit, object, readBoundedText,
+  reportNativeError, safeNativeId, type NativeDiscoveryOptions, type NativeReadOptions, type NativeTranscript,
+} from "./common.js";
+
+/**
+ * Grok Build keeps one directory per session under
+ * `~/.grok/sessions/<url-encoded-cwd>/<session-id>/`. `updates.jsonl` is the
+ * transcript source of truth: an append-only ACP notification stream
+ * (`{timestamp, method, params:{sessionId, update}}`) whose prose arrives as
+ * role-tagged chunks and whose tool results arrive as `tool_call_update` rows
+ * keyed by `toolCallId`. `summary.json` is a metadata sidecar (cwd, title,
+ * timestamps, model, branch); `chat_history.jsonl` is a compacted snapshot and
+ * `session_search.sqlite` is Grok's own index — neither is a session source.
+ * Format facts: docs/research/2026-09-16-c9-grok.md.
+ */
+const GROK_SESSION_FILE = "updates.jsonl";
+
+/**
+ * The pinned library has no Grok adapter, so this reader projects each replayed
+ * update into the Copilot CLI event contract — the closest documented
+ * event-per-line contract to an ACP update stream, and one that keeps what a
+ * canonical record needs: a native per-update id, a per-event timestamp, tool
+ * calls linked to their results by call id, and a terminal outcome flag.
+ * `source_type` is restored to this source in the normalizer, so the projection
+ * stays a detail of a source the library cannot read.
+ */
+export const GROK_PROJECTION_SOURCE = "copilot-cli";
+
+const SUMMARY_FILE = "summary.json";
+const REGISTRY_DIRECTORY = "subagents";
+const REGISTRY_FILE = "meta.json";
+// <root>/<cwd>/<session>/updates.jsonl is the documented layout; the spare
+// levels hold a listing rooted deeper, without turning discovery into an
+// unbounded walk. A directory that holds a session is not descended into: its
+// other subdirectories hold a session's own logs, not sessions.
+const MAX_DEPTH = 4;
+const MAX_SIDECAR_BYTES = 4 * 1024 * 1024;
+
+/** Lifecycle updates Grok writes beside the transcript: state, not conversation. */
+const LIFECYCLE_KINDS = new Set([
+  "hook_execution",
+  "turn_completed",
+  "session_recap",
+  "retry_state",
+  "task_backgrounded",
+  "task_completed",
+  "auto_compact_started",
+  "auto_compact_completed",
+  "compaction_checkpoint",
+  "plan",
+  "current_mode_update",
+]);
+
+export function grokRoots(): string[] {
+  return [join(homedir(), ".grok", "sessions")];
+}
+
+interface Summary {
+  cwd?: string;
+  branch?: string;
+  model?: string;
+  title?: string;
+  startedAt?: string;
+  updatedAt?: string;
+}
+
+function isoTime(value: unknown): string | undefined {
+  if (typeof value !== "string" || !value.trim()) return undefined;
+  const time = new Date(value);
+  return Number.isFinite(time.getTime()) ? time.toISOString() : undefined;
+}
+
+/**
+ * The sidecar is metadata, not transcript: a missing, oversized, or malformed
+ * summary costs title, cwd and model, and never the session, so every failure
+ * here degrades instead of throwing.
+ */
+function readSummary(path: string): { summary: Summary; unreadable: boolean } {
+  if (!existsSync(path)) return { summary: {}, unreadable: false };
+  try {
+    const value: unknown = JSON.parse(readBoundedText(path, MAX_SIDECAR_BYTES));
+    if (!object(value)) return { summary: {}, unreadable: true };
+    const info = object(value.info) ? value.info : {};
+    const title = [value.generated_title, value.session_summary]
+      .find((candidate): candidate is string => typeof candidate === "string" && candidate.trim().length > 0);
+    return {
+      summary: {
+        cwd: typeof info.cwd === "string" && info.cwd.trim() ? info.cwd : undefined,
+        branch: typeof value.head_branch === "string" && value.head_branch.trim() ? value.head_branch : undefined,
+        model: typeof value.current_model_id === "string" && value.current_model_id.trim() ? value.current_model_id : undefined,
+        title,
+        startedAt: isoTime(value.created_at),
+        updatedAt: isoTime(value.updated_at) ?? isoTime(value.last_active_at),
+      },
+      unreadable: false,
+    };
+  } catch {
+    return { summary: {}, unreadable: true };
+  }
+}
+
+export function discoverGrokSessions(roots: string[], options: NativeDiscoveryOptions): DiscoveredSession[] {
+  if (!Number.isSafeInteger(options.limit) || options.limit < 1) throw new RangeError("Grok discovery limit must be positive.");
+  const items: DiscoveredSession[] = [];
+  const edges = new Map<string, string>();
+  const seen = new Set<string>();
+
+  function walk(path: string, depth: number): void {
+    if (seen.has(path)) return;
+    seen.add(path);
+    try {
+      const stat = lstatSync(path);
+      if (stat.isSymbolicLink() || stat.isFile()) return;
+      if (depth > MAX_DEPTH) {
+        options.onDiagnostic?.({ code: "incomplete_evidence", sourcePath: path, message: "Grok discovery reached its directory depth limit." });
+        return;
+      }
+      const entries = readdirSync(path, { withFileTypes: true });
+      const names = new Set(entries.map((entry) => entry.name));
+      if (names.has(GROK_SESSION_FILE)) {
+        const id = basename(path);
+        if (!safeNativeId(id)) {
+          options.onDiagnostic?.({ code: "invalid_metadata", sourcePath: path, message: "Grok session has an invalid native session ID." });
+          return;
+        }
+        const updates = statSync(join(path, GROK_SESSION_FILE));
+        const { summary } = readSummary(join(path, SUMMARY_FILE));
+        items.push({
+          id,
+          source: "grok",
+          path: join(path, GROK_SESSION_FILE),
+          updatedAt: summary.updatedAt ?? updates.mtime.toISOString(),
+          sizeBytes: updates.size + (names.has(SUMMARY_FILE) ? statSync(join(path, SUMMARY_FILE)).size : 0),
+          ...(summary.title ? { title: summary.title } : {}),
+        });
+        if (names.has(REGISTRY_DIRECTORY)) collectSubagentEdges(path, id, options, edges);
+        return;
+      }
+      for (const entry of entries) {
+        if (entry.isDirectory()) walk(join(path, entry.name), depth + 1);
+      }
+    } catch (error) {
+      if (object(error) && error.code === "ENOENT" && depth === 0) return;
+      reportNativeError(options, path);
+    }
+  }
+
+  for (const root of roots) walk(resolve(root), 0);
+
+  const ordered = items
+    .sort((a, b) => (b.updatedAt ?? "").localeCompare(a.updatedAt ?? "") || a.path.localeCompare(b.path));
+  const nativeIds = new Set<string>();
+  const deduplicated = ordered.filter((item) => {
+    if (nativeIds.has(item.id)) return false;
+    nativeIds.add(item.id);
+    return true;
+  }).map((item) => {
+    const parent = edges.get(item.id);
+    return { ...item, native: { kind: "grok-updates" as const, ...(parent ? { parentSessionId: parent } : {}) } };
+  });
+  return options.select ? options.select(deduplicated) : deduplicated.slice(0, options.limit);
+}
+
+/**
+ * Parentage evidence read from the parent's side: every subagent Grok spawns is
+ * registered at `<parent>/subagents/<child>/meta.json`. The registration names
+ * the child by either of two fields and the parent by `parent_session_id`,
+ * falling back to the enclosing session directory; a session is never its own
+ * parent, and the first registration wins — the rules Wake's
+ * `load_subagent_parents` applies (`crates/wake-core/src/adapters/grok_group.rs`).
+ *
+ * The edge travels with the child's listing, so ingest never re-walks the store
+ * and a registry added between passes is visible to the next scan.
+ */
+function collectSubagentEdges(
+  directory: string,
+  parent: string,
+  options: NativeDiscoveryOptions,
+  edges: Map<string, string>,
+): void {
+  const registry = join(directory, REGISTRY_DIRECTORY);
+  let entries: string[];
+  try {
+    entries = readdirSync(registry);
+  } catch {
+    reportNativeError(options, registry);
+    return;
+  }
+  for (const entry of entries) {
+    const metaPath = join(registry, entry, REGISTRY_FILE);
+    let meta: Record<string, unknown>;
+    try {
+      const parsed: unknown = JSON.parse(readBoundedText(metaPath, MAX_SIDECAR_BYTES));
+      if (!object(parsed)) throw new NativeInputError("Grok subagent registration must be a JSON object.");
+      meta = parsed;
+    } catch (error) {
+      if (object(error) && error.code === "ENOENT") continue;
+      options.onDiagnostic?.({ code: "invalid_metadata", sourcePath: metaPath, message: "Grok subagent registration could not be read." });
+      continue;
+    }
+    const parentId = typeof meta.parent_session_id === "string" && meta.parent_session_id.trim() ? meta.parent_session_id : parent;
+    for (const field of ["child_session_id", "subagent_id"]) {
+      const child = meta[field];
+      if (typeof child !== "string" || !safeNativeId(child) || child === parentId) continue;
+      if (!edges.has(child)) edges.set(child, parentId);
+    }
+  }
+}
+
+type RunRole = "user" | "thought" | "message";
+type Row = Record<string, unknown>;
+
+interface Run {
+  role: RunRole;
+  text: string;
+  /** The run's identity: the id of its first fragment. */
+  id: string;
+  timestamp: string | null;
+  model: string | undefined;
+}
+
+interface Skips {
+  lifecycle: Map<string, number>;
+  progress: number;
+  unknown: Map<string, number>;
+  content: number;
+  invalidLines: number;
+  malformedLines: number;
+  /** Terminal rows for a call whose result was already projected. */
+  repeatedTerminals: number;
+}
+
+function tally(values: Map<string, number>): number {
+  let total = 0;
+  for (const count of values.values()) total += count;
+  return total;
+}
+
+function named(values: Map<string, number>, limit = 4): string {
+  return [...values.keys()].sort().slice(0, limit).join(", ");
+}
+
+/**
+ * Replay the ACP stream.
+ *
+ * A run of same-role chunks becomes one record: chunk text is concatenated
+ * without a separator, because Grok splits tokens (including the spaces between
+ * them) across rows. A run flushes on a role change or when tool activity
+ * interrupts it, so one record's text never spans two speakers or straddles a
+ * tool call. A `tool_call_update` is a result only in its terminal state:
+ * in_progress and status-less progress rows are superseded by the terminal row
+ * that follows, and are counted rather than projected.
+ *
+ * Grok appends while it runs, so a torn last line or a line from a newer
+ * version is counted, never fatal: the transcript is what the prefix holds.
+ */
+function replayUpdates(text: string, sessionId: string, summary: Summary, limit: number): {
+  rows: Row[]; skips: Skips; model: string | undefined;
+} {
+  const rows: Row[] = [];
+  const skips: Skips = {
+    lifecycle: new Map(), progress: 0, unknown: new Map(), content: 0, invalidLines: 0, malformedLines: 0,
+    repeatedTerminals: 0,
+  };
+  // One result per call id: a repeated terminal row (a retry after a failure, a
+  // re-emitted update) must not project a second result for the same call.
+  const terminals = new Set<string>();
+  let run: Run | null = null;
+  let model = summary.model;
+  let written = 0;
+
+  function emit(row: Row): void {
+    written += Buffer.byteLength(JSON.stringify(row)) + 1;
+    if (written > limit) throw new NativeInputError("Grok transcript exceeds the native byte limit.");
+    rows.push(row);
+  }
+
+  function flush(): void {
+    const pending = run;
+    run = null;
+    if (!pending || !pending.text.length) return;
+    if (pending.role === "user") {
+      emit({
+        type: "hook.start",
+        id: pending.id,
+        ...(pending.timestamp ? { timestamp: pending.timestamp } : {}),
+        data: {
+          hookType: "userPromptSubmitted",
+          input: { prompt: pending.text, sessionId, ...(summary.cwd ? { cwd: summary.cwd } : {}) },
+        },
+      });
+      return;
+    }
+    const eventModel = pending.model ?? model;
+    emit({
+      type: "assistant.message",
+      id: pending.id,
+      ...(pending.timestamp ? { timestamp: pending.timestamp } : {}),
+      data: {
+        ...(pending.role === "thought" ? { reasoningText: pending.text } : { content: pending.text }),
+        ...(eventModel ? { model: eventModel } : {}),
+      },
+    });
+  }
+
+  const lines = text.split("\n");
+  for (let index = 0; index < lines.length; index += 1) {
+    const line = lines[index];
+    if (!line || !line.trim()) continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      skips.invalidLines += 1;
+      continue;
+    }
+    const row = object(parsed) ? parsed : null;
+    const params = row && object(row.params) ? row.params : null;
+    const update = params && object(params.update) ? params.update : null;
+    if (!row || !params || !update) {
+      skips.malformedLines += 1;
+      continue;
+    }
+    const meta = object(params._meta) ? params._meta : {};
+    const agentTimestampMs = typeof meta.agentTimestampMs === "number" && Number.isFinite(meta.agentTimestampMs)
+      ? meta.agentTimestampMs
+      : typeof update._meta === "object" && object(update._meta) && typeof update._meta.agentTimestampMs === "number"
+        && Number.isFinite(update._meta.agentTimestampMs)
+        ? update._meta.agentTimestampMs
+        : null;
+    const timestamp = agentTimestampMs !== null
+      ? new Date(agentTimestampMs).toISOString()
+      : typeof row.timestamp === "number" && Number.isFinite(row.timestamp) && row.timestamp > 0
+        ? new Date(row.timestamp * 1000).toISOString()
+        : null;
+    const chunkModel = object(update._meta) && typeof update._meta.modelId === "string" ? update._meta.modelId : undefined;
+    if (chunkModel) model = chunkModel;
+    // A row without a usable update id keeps a stable identity from its own
+    // line: the file is append-only, so a line number never moves.
+    const id = typeof meta.eventId === "string" && safeNativeId(meta.eventId) ? meta.eventId : `grok-line-${index + 1}`;
+    const kind = update.sessionUpdate;
+    if (kind === "user_message_chunk" || kind === "agent_thought_chunk" || kind === "agent_message_chunk") {
+      const content = update.content;
+      const chunk = object(content) && content.type === "text" && typeof content.text === "string" ? content.text : null;
+      if (chunk === null) {
+        skips.content += 1;
+        continue;
+      }
+      const role: RunRole = kind === "user_message_chunk" ? "user" : kind === "agent_thought_chunk" ? "thought" : "message";
+      if (run && run.role !== role) flush();
+      run ??= { role, text: "", id, timestamp, model: chunkModel };
+      run.text += chunk;
+      continue;
+    }
+    if (kind === "tool_call") {
+      const callId = update.toolCallId;
+      if (!safeNativeId(callId)) {
+        skips.unknown.set("tool_call_without_id", (skips.unknown.get("tool_call_without_id") ?? 0) + 1);
+        continue;
+      }
+      flush();
+      const tool = object(update._meta) && object(update._meta["x.ai/tool"]) ? update._meta["x.ai/tool"] : null;
+      const name = typeof update.title === "string" && update.title.trim()
+        ? update.title
+        : tool && typeof tool.name === "string" && tool.name.trim() ? tool.name : undefined;
+      const eventModel = chunkModel ?? model;
+      emit({
+        type: "assistant.message",
+        id,
+        ...(timestamp ? { timestamp } : {}),
+        data: {
+          toolRequests: [{ toolCallId: callId, ...(name ? { name } : {}), arguments: update.rawInput ?? {} }],
+          ...(eventModel ? { model: eventModel } : {}),
+        },
+      });
+      continue;
+    }
+    if (kind === "tool_call_update") {
+      const callId = update.toolCallId;
+      if (!safeNativeId(callId)) {
+        skips.unknown.set("tool_call_update_without_id", (skips.unknown.get("tool_call_update_without_id") ?? 0) + 1);
+        continue;
+      }
+      const status = update.status;
+      if (status !== "completed" && status !== "failed") {
+        skips.progress += 1;
+        continue;
+      }
+      if (terminals.has(callId)) {
+        skips.repeatedTerminals += 1;
+        continue;
+      }
+      terminals.add(callId);
+      flush();
+      // ACP tool output is content blocks; a structured rawOutput is the only
+      // text a result can carry when no content block was written.
+      const blocks = update.content;
+      const parts = Array.isArray(blocks)
+        ? blocks
+          .map((block) => (object(block) && object(block.content) && block.content.type === "text" && typeof block.content.text === "string"
+            ? block.content.text
+            : null))
+          .filter((part): part is string => part !== null && part.length > 0)
+        : [];
+      const raw = update.rawOutput;
+      const output = parts.length
+        ? parts.join("\n")
+        : raw === undefined || raw === null ? "" : typeof raw === "string" ? raw : JSON.stringify(raw);
+      emit({
+        type: "tool.execution_complete",
+        id,
+        ...(timestamp ? { timestamp } : {}),
+        data: {
+          toolCallId: callId,
+          success: status !== "failed",
+          ...(status === "failed" ? { error: { message: output } } : { result: { content: output } }),
+        },
+      });
+      continue;
+    }
+    if (typeof kind === "string" && LIFECYCLE_KINDS.has(kind)) {
+      skips.lifecycle.set(kind, (skips.lifecycle.get(kind) ?? 0) + 1);
+      continue;
+    }
+    const label = typeof kind === "string" && kind ? kind : "missing_kind";
+    skips.unknown.set(label, (skips.unknown.get(label) ?? 0) + 1);
+  }
+  flush();
+  return { rows, skips, model };
+}
+
+/** Every skip is typed and counted, so a dropped update never vanishes silently. */
+function skipDiagnostics(skips: Skips): Diagnostic[] {
+  const diagnostics: Diagnostic[] = [];
+  if (skips.lifecycle.size) {
+    diagnostics.push({
+      code: "noise_record_dropped",
+      message: `Skipped ${tally(skips.lifecycle)} Grok lifecycle update(s) (${named(skips.lifecycle)}).`,
+      count: tally(skips.lifecycle),
+    });
+  }
+  if (skips.progress) {
+    diagnostics.push({
+      code: "noise_record_dropped",
+      message: `Skipped ${skips.progress} non-terminal Grok tool progress update(s); the terminal update carries the result.`,
+      count: skips.progress,
+    });
+  }
+  if (skips.repeatedTerminals) {
+    diagnostics.push({
+      code: "noise_record_dropped",
+      message: `Skipped ${skips.repeatedTerminals} repeated Grok tool terminal update(s); the call's first terminal update is its result.`,
+      count: skips.repeatedTerminals,
+    });
+  }
+  if (skips.unknown.size) {
+    diagnostics.push({
+      code: "noise_record_dropped",
+      message: `Skipped ${tally(skips.unknown)} Grok update(s) with no supported transcript shape (${named(skips.unknown)}).`,
+      count: tally(skips.unknown),
+    });
+  }
+  if (skips.content) {
+    diagnostics.push({
+      code: "noise_record_dropped",
+      message: `Skipped ${skips.content} Grok message update(s) whose content is not a text block.`,
+      count: skips.content,
+    });
+  }
+  if (skips.invalidLines) {
+    diagnostics.push({
+      code: "invalid_json_line",
+      message: `Skipped ${skips.invalidLines} invalid JSON line(s) in the Grok update stream.`,
+      count: skips.invalidLines,
+    });
+  }
+  if (skips.malformedLines) {
+    diagnostics.push({
+      code: "invalid_json_line",
+      message: `Skipped ${skips.malformedLines} Grok line(s) with no session update object.`,
+      count: skips.malformedLines,
+    });
+  }
+  return diagnostics;
+}
+
+export function readGrokSession(item: DiscoveredSession, options: NativeReadOptions = {}): NativeTranscript {
+  if (!safeNativeId(item.id)) throw new NativeInputError("Unsupported Grok native session ID.");
+  if (item.native && item.native.kind !== "grok-updates") throw new NativeInputError("Unsupported Grok native session.");
+  const limit = nativeByteLimit(options);
+  const text = readBoundedText(item.path, limit);
+  const { summary, unreadable } = readSummary(join(dirname(item.path), SUMMARY_FILE));
+  const replay = replayUpdates(text, item.id, summary, limit);
+  if (replay.rows.length > MAX_NATIVE_RECORDS) throw new NativeInputError("Grok session exceeds the native record limit.");
+
+  const rows: Row[] = summary.startedAt
+    ? [{
+      type: "session.start",
+      timestamp: summary.startedAt,
+      data: {
+        sessionId: item.id,
+        startTime: summary.startedAt,
+        context: { ...(summary.cwd ? { cwd: summary.cwd } : {}), ...(summary.branch ? { branch: summary.branch } : {}) },
+      },
+    }, ...replay.rows]
+    : replay.rows;
+
+  const diagnostics = skipDiagnostics(replay.skips);
+  if (unreadable) {
+    diagnostics.push({
+      code: "invalid_json_line",
+      message: "Ignored the Grok session summary sidecar: it is not a readable JSON object, so title, cwd and model stay unset.",
+    });
+  }
+
+  const model = replay.model ?? summary.model;
+  return {
+    transcript: rows.length ? `${rows.map((row) => JSON.stringify(row)).join("\n")}\n` : "",
+    ...(summary.title ? { title: summary.title } : {}),
+    ...(summary.cwd ? { cwd: summary.cwd } : {}),
+    ...(model ? { model } : {}),
+    // The sidecar's own envelope bounds the session: it starts before the first
+    // update and ends at Grok's last recorded activity, which a hook-only
+    // session (no records at all) would otherwise have no span for.
+    ...(summary.startedAt ? { startedAt: summary.startedAt } : {}),
+    ...(summary.updatedAt ? { endedAt: summary.updatedAt } : {}),
+    ...(diagnostics.length ? { diagnostics } : {}),
+  };
+}
+
+/** The files whose change invalidates a Grok session's snapshot. */
+export function grokSessionFiles(item: DiscoveredSession): string[] {
+  return [item.path, join(dirname(item.path), SUMMARY_FILE)];
+}