pim-agent 0.0.1 → 0.10.0

package/packages/server/src/SessionCatalogue.ts

@@ -0,0 +1,456 @@
+import { EventLog } from "#core/session/EventLog";
+import type { SessionDigest } from "#core/session/EventLog";
+import type { ReadCursors } from "#core/session/ReadCursors";
+import { SearchIndex } from "#core/session/SearchIndex";
+import type { SearchHit } from "#core/session/SearchIndex";
+import type {
+  Pinning,
+  SessionEntry,
+  SessionMeta,
+} from "#core/session/SessionMeta";
+import type {
+  SessionRegistry,
+  SessionSummary,
+} from "#core/session/SessionRegistry";
+import { FileWatch } from "#core/shared/FileWatch";
+import { Pool } from "#core/shared/Pool";
+import type { Command } from "#protocol/Command";
+import type {
+  ProjectView,
+  SearchHitView,
+  ServerEvent,
+  SessionListing,
+  SessionSearch,
+  SessionStatus,
+  SessionSummaryView,
+} from "#protocol/ServerEvent";
+
+export type SessionCatalogueDeps = {
+  readonly registry: SessionRegistry;
+  /** Which sessions have been read, shared by every client. */
+  readonly cursors: ReadCursors;
+  /** Archived, held-unread and pinned, shared by every client. */
+  readonly meta: SessionMeta;
+  readonly liveStatus: (sessionId: string) => SessionStatus | undefined;
+  readonly liveSessionIds: () => Iterable<string>;
+  readonly isBeingRead: (sessionId: string) => boolean;
+  readonly announce: (event: ServerEvent) => void;
+};
+
+const DEFAULT_SESSION_LIMIT = 50;
+
+/** A digest is a whole-file read; a page of them at once is a page of files in memory at once. */
+const DIGEST_READS = 16;
+
+/** Long enough that a turn's entries are one announcement, short enough that a new session appears while the user is still looking. */
+const ANNOUNCE_MS = 500;
+
+type CachedDigest = SessionDigest & { readonly modifiedAt: number };
+
+type Overrides = ReadonlyMap<string, SessionEntry>;
+
+type PageScope = {
+  readonly overrides: Overrides;
+  /** True while listing the archived sessions rather than the live ones. */
+  readonly archived: boolean;
+  readonly limit: number;
+  readonly perProject: number;
+};
+
+export class SessionCatalogue {
+  private readonly deps: SessionCatalogueDeps;
+  private readonly digests = new Map<string, CachedDigest>();
+  private readonly activity = new Map<string, SessionStatus>();
+  private readonly settled = new Map<string, number>();
+  private readonly watches = new Map<string, () => void>();
+  private index: SearchIndex;
+  private watching = false;
+  private announcing: ReturnType<typeof setTimeout> | undefined;
+
+  public constructor(deps: SessionCatalogueDeps) {
+    this.deps = deps;
+    this.index = this.newIndex();
+  }
+
+  /**
+   * Follow the sessions tree while anyone is connected. A session the terminal
+   * or a vanilla `pi` starts is a file this process never hears about
+   * otherwise, and a list nobody asked to re-fetch is a list that never learns
+   * about it.
+   */
+  public watch(active: boolean): void {
+    if (active === this.watching) {
+      return;
+    }
+    this.watching = active;
+    if (!active) {
+      for (const stop of this.watches.values()) {
+        stop();
+      }
+      this.watches.clear();
+      clearTimeout(this.announcing);
+      this.announcing = undefined;
+      return;
+    }
+    this.follow();
+  }
+
+  public async list(
+    command: Command & { readonly type: "list_sessions" }
+  ): Promise<SessionListing> {
+    const [summaries, overrides, pinning] = await Promise.all([
+      this.deps.registry.list(command.cwd),
+      this.deps.meta.sessions(),
+      this.deps.meta.pinning(),
+    ]);
+    // Prune only on an unfiltered listing: a cwd-filtered one would forget every other directory.
+    if (command.cwd === undefined) {
+      const alive = new Set([
+        ...summaries.map((summary) => summary.sessionId),
+        ...this.deps.liveSessionIds(),
+      ]);
+      await Promise.all([
+        this.deps.cursors.prune(alive),
+        this.deps.meta.prune(alive),
+      ]);
+    }
+    const scope = command.archived === true;
+    // Before the cut: an archived row that ate the page budget would push a live one off the end.
+    const inScope = summaries.filter(
+      (summary) =>
+        (overrides.get(summary.sessionId)?.archived === true) === scope
+    );
+    const limit = command.limit ?? DEFAULT_SESSION_LIMIT;
+    return {
+      sessions: await this.page(inScope, {
+        overrides,
+        archived: scope,
+        limit,
+        // Absent, a project may fill the page; it is the page that bounds it either way.
+        perProject: command.perProject ?? limit,
+      }),
+      // Counted before the cut, and over the whole scope: a collapsed group says how many it holds, and a row the cut dropped is one a client can still ask for.
+      projects: projectsOf(inScope, pinning),
+    };
+  }
+
+  /**
+   * The index cannot see the sidecar, so the archived scope is a predicate it
+   * calls before its own limit, and the badge is a `Map.get` over the one read
+   * of `sessions.json` this query makes.
+   */
+  public async search(
+    command: Command & { readonly type: "search_sessions" }
+  ): Promise<SessionSearch> {
+    const overrides = await this.deps.meta.sessions();
+    const archivedOf = (sessionId: string): boolean =>
+      overrides.get(sessionId)?.archived === true;
+    const { hits, dropped, scanned } = await this.index.search(command.query, {
+      ...(command.limit === undefined ? {} : { limit: command.limit }),
+      ...(command.cwd === undefined ? {} : { cwd: command.cwd }),
+      ...(command.archived === undefined
+        ? {}
+        : {
+            accept: (sessionId: string) =>
+              archivedOf(sessionId) === command.archived,
+          }),
+    });
+    return {
+      hits: hits.map((hit) => hitOf(hit, archivedOf(hit.sessionId))),
+      dropped,
+      scanned,
+    };
+  }
+
+  /** Announces a status edge to every client; the announcement must precede the read mark. */
+  public onStatus(sessionId: string, status: SessionStatus): void {
+    if (this.activity.get(sessionId) === status) {
+      return;
+    }
+    this.activity.set(sessionId, status);
+    this.deps.announce({ type: "session_activity", sessionId, status });
+    if (status === "idle" && this.deps.isBeingRead(sessionId)) {
+      void this.markRead(sessionId);
+    }
+  }
+
+  public track(sessionId: string, status: SessionStatus): void {
+    this.activity.set(sessionId, status);
+  }
+
+  /** Moves a session's read cursor to now, drops the mark a reader left on it by hand, and says so to every client. */
+  public async markRead(sessionId: string): Promise<void> {
+    await this.deps.cursors.mark(sessionId);
+    // The only place the sticky flag is cleared; a listing must never clear it.
+    if ((await this.deps.meta.of(sessionId)).unread === true) {
+      await this.deps.meta.setUnread(sessionId, false);
+    }
+    this.deps.announce({ type: "session_read", sessionId });
+  }
+
+  public async flush(): Promise<void> {
+    await Promise.all([this.deps.cursors.flush(), this.deps.meta.flush()]);
+  }
+
+  public clear(): void {
+    this.digests.clear();
+    this.activity.clear();
+    this.settled.clear();
+    this.index = this.newIndex();
+  }
+
+  /** One index per catalogue, and so one per session root; the registry's listing is the tree it follows. */
+  private newIndex(): SearchIndex {
+    return new SearchIndex({ list: () => this.deps.registry.list() });
+  }
+
+  /**
+   * Selects by modified time under a per-project budget and digests only what
+   * it selects: ordering by settle time would read every session on disk. A
+   * session with nothing to call itself is not a row, and the budget it spent
+   * goes back to its project, so a directory of abandoned files still shows
+   * the sessions underneath them.
+   */
+  private async page(
+    candidates: readonly SessionSummary[],
+    scope: PageScope
+  ): Promise<readonly SessionSummaryView[]> {
+    const rows: SessionSummaryView[] = [];
+    const taken = new Map<string, number>();
+    const consumed = new Set<SessionSummary>();
+    for (;;) {
+      const batch = choose(candidates, consumed, taken, {
+        room: scope.limit - rows.length,
+        perProject: scope.perProject,
+      });
+      if (batch.length === 0) {
+        break;
+      }
+      const built = await Pool.mapPooled(batch, DIGEST_READS, (summary) =>
+        this.row(summary, scope)
+      );
+      for (const [index, row] of built.entries()) {
+        if (row === undefined) {
+          const { cwd } = batch[index]!;
+          taken.set(cwd, (taken.get(cwd) ?? 1) - 1);
+          continue;
+        }
+        rows.push(row);
+      }
+    }
+    return rows.sort((a, b) => b.settledAt - a.settledAt);
+  }
+
+  /** Nothing for a session that never asked anything and was never named: the row it would draw is a truncated UUID. */
+  private async row(
+    { sessionId, cwd, path, createdAt, modifiedAt }: SessionSummary,
+    { overrides, archived }: PageScope
+  ): Promise<SessionSummaryView | undefined> {
+    const status = this.deps.liveStatus(sessionId);
+    const { title, named, settledAt } = await this.digestOf(
+      sessionId,
+      path,
+      modifiedAt,
+      status
+    );
+    if (title === undefined) {
+      return undefined;
+    }
+    const answeredAt = this.answerTime(sessionId, status, settledAt);
+    const unread =
+      overrides.get(sessionId)?.unread === true ||
+      (await this.deps.cursors.isUnread(sessionId, answeredAt));
+    return {
+      sessionId,
+      cwd,
+      createdAt,
+      settledAt: answeredAt ?? settledAt ?? createdAt,
+      title,
+      ...(named === true ? { named } : {}),
+      ...(archived ? { archived: true as const } : {}),
+      ...(status === undefined || status === "idle" ? {} : { status }),
+      ...(unread ? { unread: true } : {}),
+    };
+  }
+
+  // One watch per directory, never a recursive one: `fs.watch` recursion is unsupported on some platforms and silent on others.
+  private follow(): void {
+    const root = this.deps.registry.sessionsRoot;
+    const wanted = new Set([root, ...FileWatch.subdirectories(root)]);
+    for (const [path, stop] of this.watches) {
+      if (!wanted.has(path)) {
+        stop();
+        this.watches.delete(path);
+      }
+    }
+    for (const path of wanted) {
+      if (!this.watches.has(path)) {
+        this.watches.set(
+          path,
+          FileWatch.directory(path, () => {
+            this.onTreeChange();
+          })
+        );
+      }
+    }
+  }
+
+  // Debounced: a turn writes a dozen entries, and re-listing is a read of every session header.
+  private onTreeChange(): void {
+    if (this.announcing !== undefined || !this.watching) {
+      return;
+    }
+    this.announcing = setTimeout(() => {
+      this.announcing = undefined;
+      this.follow();
+      this.deps.announce({ type: "sessions_changed" });
+    }, ANNOUNCE_MS);
+    this.announcing.unref?.();
+  }
+
+  // A running session must be answered for by its last idle reading: mid-turn the file's time is the line just written.
+  private answerTime(
+    sessionId: string,
+    status: SessionStatus | undefined,
+    fromFile: number | undefined
+  ): number | undefined {
+    if (status === undefined) {
+      return fromFile;
+    }
+    if (status !== "idle") {
+      return this.settled.get(sessionId);
+    }
+    if (fromFile !== undefined) {
+      this.settled.set(sessionId, fromFile);
+    }
+    return fromFile;
+  }
+
+  private async digestOf(
+    sessionId: string,
+    path: string,
+    modifiedAt: number,
+    status: SessionStatus | undefined
+  ): Promise<SessionDigest> {
+    const name = this.liveName(sessionId);
+    const digest =
+      this.reusable(path, modifiedAt, status, name) ??
+      (await this.readDigest(path, modifiedAt));
+    return renamed(digest, name);
+  }
+
+  /**
+   * Mid-turn the file is appended to between listings, so its digest never hits
+   * the cache and the whole-file read lands on the busiest session there is.
+   * Nothing it answers can have moved: the opening message was written long
+   * ago, the settle time is deliberately not read from the file while a turn
+   * runs, and the name is the live session's own to say.
+   */
+  private reusable(
+    path: string,
+    modifiedAt: number,
+    status: SessionStatus | undefined,
+    name: string | null | undefined
+  ): CachedDigest | undefined {
+    const cached = this.digests.get(path);
+    if (cached === undefined) {
+      return undefined;
+    }
+    // Except a name the live session has since cleared: the opening message a
+    // nameless row falls back to is in the file and nowhere else.
+    if (name === null && cached.named === true) {
+      return undefined;
+    }
+    const working = status !== undefined && status !== "idle";
+    return cached.modifiedAt === modifiedAt || working ? cached : undefined;
+  }
+
+  private async readDigest(
+    path: string,
+    modifiedAt: number
+  ): Promise<SessionDigest> {
+    const digest = await new EventLog(path).digest();
+    this.digests.set(path, { ...digest, modifiedAt });
+    return digest;
+  }
+
+  /** What a live session calls itself, without reading its file; `null` is live and unnamed. */
+  private liveName(sessionId: string): string | null | undefined {
+    const agent = this.deps.registry.peek(sessionId)?.agentSession;
+    return agent === undefined
+      ? undefined
+      : (agent.sessionManager.getSessionName() ?? null);
+  }
+}
+
+/** The row the wire carries: the index's hit, plus the one thing about it only the sidecar knows. */
+function hitOf(hit: SearchHit, archived: boolean): SearchHitView {
+  return {
+    sessionId: hit.sessionId,
+    cwd: hit.cwd,
+    ...(hit.title === undefined ? {} : { title: hit.title }),
+    titleRanges: hit.titleRanges,
+    ...(hit.opening === undefined ? {} : { opening: hit.opening }),
+    settledAt: hit.settledAt,
+    ...(archived ? { archived: true as const } : {}),
+    snippets: hit.snippets,
+    total: hit.total,
+  };
+}
+
+/** A live session's own name wins over the file's: a rename lands on the row before the digest behind it expires. */
+function renamed(
+  digest: SessionDigest,
+  name: string | null | undefined
+): SessionDigest {
+  return typeof name === "string"
+    ? { ...digest, title: name, named: true }
+    : digest;
+}
+
+/** The next sessions to digest, newest first, skipping the projects that have spent their budget and the candidates an earlier round already took. */
+function choose(
+  candidates: readonly SessionSummary[],
+  consumed: Set<SessionSummary>,
+  taken: Map<string, number>,
+  budget: { readonly room: number; readonly perProject: number }
+): readonly SessionSummary[] {
+  const batch: SessionSummary[] = [];
+  for (const summary of candidates) {
+    if (batch.length >= budget.room) {
+      break;
+    }
+    const used = taken.get(summary.cwd) ?? 0;
+    if (consumed.has(summary) || used >= budget.perProject) {
+      continue;
+    }
+    taken.set(summary.cwd, used + 1);
+    consumed.add(summary);
+    batch.push(summary);
+  }
+  return batch;
+}
+
+/** Every working directory the scope holds sessions in, newest first; the count is the header scan's, so it owes nothing to the page. */
+function projectsOf(
+  summaries: readonly SessionSummary[],
+  { projects, order }: Pinning
+): readonly ProjectView[] {
+  const counts = new Map<string, number>();
+  for (const { cwd } of summaries) {
+    counts.set(cwd, (counts.get(cwd) ?? 0) + 1);
+  }
+  const ranks = new Map(order.map((cwd, rank) => [cwd, rank]));
+  return [...counts].map(([cwd, count]) => {
+    const entry = projects.get(cwd);
+    return {
+      cwd,
+      count,
+      ...(entry?.pinned === true
+        ? { pinned: true as const, pinRank: ranks.get(cwd) ?? 0 }
+        : {}),
+      ...(entry?.expanded === true ? { expanded: true as const } : {}),
+      ...(entry?.label === undefined ? {} : { label: entry.label }),
+    };
+  });
+}

package/packages/server/src/SessionProjection.ts

@@ -0,0 +1,194 @@
+import type {
+  AgentMessage,
+  AgentToolResult,
+} from "@earendil-works/pi-agent-core";
+import type { FileEntry } from "@earendil-works/pi-coding-agent";
+
+import { Attachments } from "#core/attachments/Attachments";
+import { EventLog, type LoggedEntry } from "#core/session/EventLog";
+import { MessageText } from "#core/session/MessageText";
+import { Tools } from "#core/shared/Tools";
+import type {
+  AttachmentView,
+  DurableEvent,
+  ToolCallView,
+} from "#protocol/ServerEvent";
+import { attachmentUrl } from "./AttachmentEndpoint";
+
+type PendingCall = {
+  readonly name: string;
+  readonly args: unknown;
+};
+
+/** Projects pi's session file into durable events: one line in, at most one `DurableEvent` out, tagged with that line's ordinal. */
+export class SessionProjection {
+  private readonly log: EventLog;
+  private readonly cwd: () => string;
+  private readonly events: DurableEvent[] = [];
+  private readonly calls = new Map<string, PendingCall>();
+  private draining: Promise<void> = Promise.resolve();
+  private highest = 0;
+
+  public constructor(path: string, cwd: () => string) {
+    this.log = new EventLog(path);
+    this.cwd = cwd;
+  }
+
+  /** Highest line ordinal projected, whether or not it emitted. */
+  public get head(): number {
+    return this.highest;
+  }
+
+  public since(fromSeq: number): readonly DurableEvent[] {
+    let low = 0;
+    let high = this.events.length;
+    while (low < high) {
+      const mid = (low + high) >>> 1;
+      if (this.events[mid]!.seq > fromSeq) {
+        high = mid;
+      } else {
+        low = mid + 1;
+      }
+    }
+    return this.events.slice(low);
+  }
+
+  /** Read whatever pi has appended since the last drain; serialized, as overlapping reads double-project lines. */
+  public async drain(): Promise<readonly DurableEvent[]> {
+    const before = this.events.length;
+    const done = this.draining.then(async () => {
+      for (const logged of await this.log.read(this.highest)) {
+        this.highest = logged.seq;
+        const event = this.project(logged);
+        if (event) {
+          this.events.push(event);
+        }
+      }
+    });
+    this.draining = done.catch(() => {});
+    await done;
+    return this.events.slice(before);
+  }
+
+  private project(logged: LoggedEntry): DurableEvent | undefined {
+    const { seq, entry } = logged;
+    if (entry.type === "compaction") {
+      return {
+        seq,
+        type: "notice",
+        severity: "info",
+        text: `Context compacted (${entry.tokensBefore} tokens before).`,
+      };
+    }
+    if (!isMessageEntry(entry)) {
+      return undefined;
+    }
+    const message = entry.message;
+    const timestamp = Date.parse(entry.timestamp);
+    switch (message.role) {
+      case "user": {
+        const said = Attachments.parse(MessageText.textOf(message.content));
+        const attachments: readonly AttachmentView[] = said.files.map(
+          (file) => ({
+            name: Attachments.nameOf(file.path),
+            url: attachmentUrl(file.path),
+            isImage: file.isImage,
+          })
+        );
+        return {
+          seq,
+          type: "message",
+          messageId: entry.id,
+          role: "user",
+          text: said.text,
+          timestamp,
+          ...(attachments.length === 0 ? {} : { attachments }),
+        };
+      }
+      case "assistant": {
+        const toolCalls: ToolCallView[] = [];
+        for (const part of message.content) {
+          if (part.type !== "toolCall") {
+            continue;
+          }
+          this.calls.set(part.id, { name: part.name, args: part.arguments });
+          toolCalls.push({
+            callId: part.id,
+            name: part.name,
+            view: Tools.viewOf({
+              name: part.name,
+              args: part.arguments,
+              isPartial: true,
+              cwd: this.cwd(),
+            }),
+          });
+        }
+        const thinking = MessageText.textOf(message.content, "thinking");
+        const error =
+          message.stopReason === "error"
+            ? (message.errorMessage ?? "The model call failed.")
+            : undefined;
+        return {
+          seq,
+          type: "message",
+          messageId: entry.id,
+          role: "assistant",
+          text: MessageText.textOf(message.content),
+          timestamp,
+          ...(thinking ? { thinking } : {}),
+          ...(toolCalls.length > 0 ? { toolCalls } : {}),
+          ...(error === undefined ? {} : { error }),
+        };
+      }
+      case "toolResult": {
+        const call = this.calls.get(message.toolCallId);
+        return {
+          seq,
+          type: "tool_result",
+          callId: message.toolCallId,
+          name: message.toolName,
+          isError: message.isError,
+          view: Tools.viewOf({
+            name: message.toolName,
+            args: call?.args ?? {},
+            isError: message.isError,
+            result: {
+              content: withoutImages(message.content),
+              details: message.details,
+            } as AgentToolResult<unknown>,
+            isPartial: false,
+            cwd: this.cwd(),
+          }),
+        };
+      }
+      default:
+        return undefined;
+    }
+  }
+}
+
+type MessageEntry = {
+  readonly id: string;
+  readonly timestamp: string;
+  readonly message: AgentMessage;
+};
+
+type ToolResultContent = Extract<
+  AgentMessage,
+  { role: "toolResult" }
+>["content"];
+
+/** No base64 crosses the websocket: a view reaches its picture through `details.sha256` instead. */
+function withoutImages(content: ToolResultContent): ToolResultContent {
+  return content.some((part) => part.type === "image")
+    ? content.map((part) =>
+        part.type === "image"
+          ? { type: "text" as const, text: "[image]" }
+          : part
+      )
+    : content;
+}
+
+function isMessageEntry(entry: FileEntry): entry is FileEntry & MessageEntry {
+  return entry.type === "message";
+}