portable-agent-layer 0.98.0 → 0.99.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "portable-agent-layer",
-  "version": "0.98.0",
+  "version": "0.99.0",
   "description": "PAL — Portable Agent Layer: persistent personal context for AI coding assistants",
   "type": "module",
   "bin": {
@@ -80,7 +80,7 @@
     "@commitlint/cli": "21.0.1",
     "@commitlint/config-conventional": "21.0.1",
     "@hughescr/stryker-bun-runner": "1.3.8",
-    "@konvert7/klint": "0.34.0",
+    "@konvert7/klint": "0.35.1",
     "@opencode-ai/plugin": "latest",
     "@secretlint/secretlint-rule-preset-recommend": "13.0.2",
     "@semantic-release/changelog": "^6.0.3",

package/src/hooks/lib/commitment-events.ts

@@ -1,11 +1,11 @@
 /**
- * A commitment is an ISC: completing it keeps it, retiring it drops it, and a
- * snooze puts it off. Each change of state is one event, whichever surface made it.
+ * A commitment is an ISC: adding it opens it, completing it keeps it, retiring
+ * it drops it, and a snooze puts it off. Each change of state is one event, whichever surface made it.
  */
 
 import { appendEvent, logQuietly } from "./events";
 
-export type CommitmentOutcome = "kept" | "dropped" | "snoozed" | "reopened";
+export type CommitmentOutcome = "opened" | "kept" | "dropped" | "snoozed" | "reopened";
 
 export function logCommitment(
   iscProject: string,

package/src/hooks/lib/entity-events.ts

@@ -3,14 +3,27 @@
  * store. The name stays in the content store; the event carries only its kind.
  */
 
-import { appendEvent, logQuietly } from "./events";
+import { type AppendOptions, appendEvent, logQuietly } from "./events";
+
+export interface SeenEntity {
+  domain: string;
+  slug: string;
+  entityType: string;
+  name: string;
+}
+
+export const entityEventId = (entity: Pick<SeenEntity, "domain" | "slug">) =>
+  `entity:${entity.domain}:${entity.slug}`;
 
 export function logEntityFirstSeen(
-  domain: string,
-  entityType: string,
-  name: string
+  entity: SeenEntity,
+  options: AppendOptions = {}
 ): void {
   logQuietly("entity_first_seen", () =>
-    appendEvent("entity_first_seen", { domain, entityType }, { content: { name } })
+    appendEvent(
+      "entity_first_seen",
+      { domain: entity.domain, entityType: entity.entityType },
+      { id: entityEventId(entity), content: { name: entity.name }, ...options }
+    )
   );
 }

package/src/hooks/lib/events.ts

@@ -116,6 +116,9 @@ export const EVENT_FIELDS = {
     "aiCoAuthored",
   ],
   rating: ["rating", "source"],
+  project: ["change", "status"],
+  decision: ["words"],
+  promise: ["change", "dueInSec", "lateSec", "snoozes", "nags"],
   action: [
     "tool",
     "outcome",
@@ -178,6 +181,7 @@ export interface AppendOptions {
   session?: string | null;
   runtime?: AgentType | "git" | "unknown";
   attribution?: Partial<RecordAttribution>;
+  project?: string | null;
   channel?: string | null;
   cwd?: string;
   content?: Record<string, unknown>;
@@ -241,11 +245,16 @@ function envelope(type: EventType, options: AppendOptions): EventEnvelope {
     ...options.attribution,
     channel:
       options.channel !== undefined ? options.channel : process.env.PAL_CHANNEL || null,
-    project: cwd ? projectOf(cwd) : null,
+    project: projectFor(options, cwd),
     cwd,
   };
 }
 
+function projectFor(options: AppendOptions, cwd: string): string | null {
+  if (options.project !== undefined) return options.project;
+  return cwd ? projectOf(cwd) : null;
+}
+
 /** Appends one event and returns its id, or null when the log is off or the write failed. */
 export function appendEvent(
   type: EventType,

package/src/tools/agent/project.ts

@@ -28,6 +28,7 @@ import {
 } from "../../hooks/lib/projects";
 import { isServesKind, SERVES_KINDS, setServes } from "../../hooks/lib/serves";
 import { group, type Leaf, leaf, runCommand, UsageError } from "../lib/command";
+import { decisionRow, emitHistoryRow, registrationRow } from "../lib/project-history";
 import {
   archiveLine,
   completeIsc,
@@ -169,6 +170,7 @@ function cmdCreate(given: string | undefined, values: CreateValues): undefined {
     ...(values["serves-note"] ? { serves_note: values["serves-note"] } : {}),
   };
   writeProject(project);
+  emitHistoryRow(registrationRow(project));
   ok({ created: true, project });
 }
 
@@ -258,11 +260,13 @@ function addDecision(name: string, decision: string, rationale: string): undefin
   const what = nonBlank(decision, "<decision>");
   const why = nonBlank(rationale, "<rationale...>");
   const p = requireProject(name);
-  const date = new Date().toISOString().slice(0, 10);
+  const ts = now();
+  const date = ts.slice(0, 10);
   const line = `- ${date}: ${what} (${why})`;
   p.decisions = p.decisions ? `${p.decisions}\n${line}` : line;
-  p.updated = now();
+  p.updated = ts;
   writeProject(p);
+  emitHistoryRow({ ...decisionRow(name, date, `${what} (${why})`), ts });
   ok({ updated: true, name });
 }
 
@@ -434,6 +438,7 @@ function cmdAddIsc(name: string, text: string): undefined {
   p.criteria = current ? `${current.trimEnd()}\n${newLine}` : newLine;
   p.updated = now();
   writeProject(p);
+  logCommitment(name, id, "opened", "cli");
   ok({
     added: true,
     id,

package/src/tools/knowledge/lib.ts

@@ -31,7 +31,7 @@ import {
   writeFileSync,
 } from "node:fs";
 import { resolve } from "node:path";
-import { logEntityFirstSeen } from "../../hooks/lib/entity-events";
+import { logEntityFirstSeen, type SeenEntity } from "../../hooks/lib/entity-events";
 import { ensureDir, paths } from "../../hooks/lib/paths";
 
 // --- Constants --------------------------------------------------------------
@@ -411,6 +411,15 @@ export function exists(domain: Domain, slug: string, rootDir?: string): boolean
   return existsSync(entityPath(domain, slug, rootDir));
 }
 
+export function seenEntity(entity: Entity): SeenEntity {
+  return {
+    domain: entity.domain,
+    slug: entity.slug,
+    entityType: entity.frontmatter.type,
+    name: entity.frontmatter.title,
+  };
+}
+
 export function save(entity: Entity, rootDir?: string): void {
   const p = entityPath(entity.domain, entity.slug, rootDir);
   const firstSeenInTheStore = !rootDir && !existsSync(p);
@@ -418,7 +427,7 @@ export function save(entity: Entity, rootDir?: string): void {
   writeFileSync(tmp, serialize(entity), "utf-8");
   renameSync(tmp, p);
   if (firstSeenInTheStore) {
-    logEntityFirstSeen(entity.domain, entity.frontmatter.type, entity.frontmatter.title);
+    logEntityFirstSeen(seenEntity(entity));
   }
 }
 

package/src/tools/lib/backfill/entities.ts

@@ -0,0 +1,31 @@
+/**
+ * Every entity in the knowledge store, first seen when it was created. The
+ * live event shares its id, so an entity already logged is skipped.
+ */
+
+import { entityEventId, logEntityFirstSeen } from "../../../hooks/lib/entity-events";
+import { type Entity, list, seenEntity } from "../../knowledge/lib";
+import { emitUnseen } from "./rows";
+
+interface EntityRow {
+  entity: Entity;
+  ts: string;
+}
+
+function emitEntity(row: EntityRow): void {
+  logEntityFirstSeen(seenEntity(row.entity), {
+    ts: row.ts,
+    session: null,
+    channel: null,
+    cwd: "",
+    attribution: { machine: "unknown", actor: "unknown", runtime: "unknown" },
+  });
+}
+
+/** Appends the entities not yet logged and returns how many. */
+export function backfillEntities(known: Set<string>, since?: string): number {
+  const rows = list()
+    .filter((entity) => entity.frontmatter.created)
+    .map((entity) => ({ entity, ts: entity.frontmatter.created }));
+  return emitUnseen(rows, (row) => entityEventId(row.entity), emitEntity, known, since);
+}

package/src/tools/lib/backfill/index.ts

@@ -6,8 +6,11 @@
 import { loggedEventIds } from "../../../hooks/lib/events";
 import { readAllProjects } from "../../../hooks/lib/projects";
 import { eventLogEnabled } from "../../../hooks/lib/settings";
+import { backfillEntities } from "./entities";
 import { backfillGit } from "./git";
 import { backfillLedger } from "./ledger";
+import { backfillProjects } from "./projects";
+import { backfillPromises } from "./promises";
 import { backfillRatings } from "./ratings";
 import { backfillSessions } from "./sessions";
 import { backfillTurns } from "./turns";
@@ -28,5 +31,8 @@ export function backfillEvents(since?: string): Record<string, number> {
     turns: backfillTurns(known, since),
     ratings: backfillRatings(known, since),
     ledger: backfillLedger(known, since),
+    projects: backfillProjects(known, since),
+    promises: backfillPromises(known, since),
+    entities: backfillEntities(known, since),
   };
 }

package/src/tools/lib/backfill/projects.ts

@@ -0,0 +1,40 @@
+/**
+ * What each project's ISA still records about its own history. An ISC's
+ * opening was never dated, so it cannot be backfilled; a close already
+ * streamed live as a commitment is not logged twice.
+ */
+
+import { eventsSince } from "../../../hooks/lib/events";
+import { readAllProjects } from "../../../hooks/lib/projects";
+import {
+  closedIscRows,
+  decisionRows,
+  emitHistoryRow,
+  type HistoryRow,
+  registrationRow,
+} from "../project-history";
+import { emitUnseen, recordedAttribution } from "./rows";
+
+const commitmentKey = (fields: Record<string, unknown>) =>
+  `${fields.iscProject}:${fields.isc}:${fields.outcome}`;
+
+function liveCommitments(): Set<string> {
+  return new Set(eventsSince("commitment", new Date(0)).map(commitmentKey));
+}
+
+function emitHistory(row: HistoryRow): void {
+  emitHistoryRow(row, recordedAttribution(row));
+}
+
+/** Appends the project history not yet logged and returns how many events. */
+export function backfillProjects(known: Set<string>, since?: string): number {
+  const live = liveCommitments();
+  const rows = readAllProjects()
+    .flatMap((project) => [
+      registrationRow(project),
+      ...closedIscRows(project),
+      ...decisionRows(project),
+    ])
+    .filter((row) => row.type !== "commitment" || !live.has(commitmentKey(row.fields)));
+  return emitUnseen(rows, (row) => row.id, emitHistory, known, since);
+}

package/src/tools/lib/backfill/promises.ts

@@ -0,0 +1,91 @@
+/**
+ * The promises the Discord bot nags about, from its store: each one made, and
+ * each one closed as done or dropped. The title is text, so it goes to the
+ * content store.
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { appendEvent } from "../../../hooks/lib/events";
+import { paths } from "../../../hooks/lib/paths";
+import { emitUnseen } from "./rows";
+
+interface StoredPromise {
+  id: number;
+  title: string;
+  due: string;
+  status: "open" | "done" | "dropped";
+  createdAt: string;
+  closedAt?: string;
+  project?: string;
+  snoozes: number;
+  nagCount: number;
+}
+
+interface PromiseRow {
+  id: string;
+  ts: string;
+  project: string | null;
+  fields: Record<string, unknown>;
+  title: string;
+}
+
+const secondsBetween = (from: string, to: string) =>
+  Math.round((Date.parse(to) - Date.parse(from)) / 1000);
+
+function readPromises(): StoredPromise[] {
+  const file = resolve(paths.memory(), "commitments.json");
+  if (!existsSync(file)) return [];
+  try {
+    return JSON.parse(readFileSync(file, "utf-8")).items ?? [];
+  } catch {
+    return [];
+  }
+}
+
+function made(promise: StoredPromise): PromiseRow {
+  return {
+    id: `promise:${promise.id}:${promise.createdAt}:made`,
+    ts: promise.createdAt,
+    project: promise.project ?? null,
+    fields: { change: "made", dueInSec: secondsBetween(promise.createdAt, promise.due) },
+    title: promise.title,
+  };
+}
+
+function closed(promise: StoredPromise): PromiseRow[] {
+  if (promise.status === "open" || !promise.closedAt) return [];
+  return [
+    {
+      id: `promise:${promise.id}:${promise.createdAt}:${promise.status}`,
+      ts: promise.closedAt,
+      project: promise.project ?? null,
+      fields: {
+        change: promise.status,
+        lateSec: secondsBetween(promise.due, promise.closedAt),
+        snoozes: promise.snoozes,
+        nags: promise.nagCount,
+      },
+      title: promise.title,
+    },
+  ];
+}
+
+function emitPromise(row: PromiseRow): void {
+  appendEvent("promise", row.fields, {
+    id: row.id,
+    ts: row.ts,
+    session: null,
+    channel: null,
+    cwd: "",
+    project: row.project,
+    attribution: { machine: "unknown", actor: "unknown", runtime: "unknown" },
+    content: { title: row.title },
+  });
+}
+
+/** Appends the promises not yet logged and returns how many events. */
+export function backfillPromises(known: Set<string>, since?: string): number {
+  const rows = readPromises().flatMap((promise) => [made(promise), ...closed(promise)]);
+  return emitUnseen(rows, (row) => row.id, emitPromise, known, since);
+}

package/src/tools/lib/collectors/claude.ts

@@ -7,8 +7,8 @@
  * cursor stops where it starts.
  */
 
-import { readFileSync, statSync } from "node:fs";
-import { relative, sep } from "node:path";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { relative, resolve, sep } from "node:path";
 import { appendEvent } from "../../../hooks/lib/events";
 import { cacheWritesOf, type TranscriptUsage } from "../transcript-usage";
 import { claudeTranscripts } from "../usage-buckets";
@@ -60,8 +60,7 @@ function parsed(text: string): ClaudeLine | null {
   }
 }
 
-function completeLinesFrom(filepath: string, offset: number) {
-  const bytes = readFileSync(filepath);
+function completeLinesFrom(bytes: Buffer, offset: number) {
   const end = bytes.lastIndexOf(10) + 1;
   if (end <= offset) return { lines: [], end: offset };
   const lines: { text: string; start: number }[] = [];
@@ -76,12 +75,8 @@ function completeLinesFrom(filepath: string, offset: number) {
   return { lines, end };
 }
 
-function settledMessages(
-  filepath: string,
-  offset: number,
-  settled: boolean
-): SettledRead {
-  const { lines, end } = completeLinesFrom(filepath, offset);
+function settledMessages(bytes: Buffer, offset: number, settled: boolean): SettledRead {
+  const { lines, end } = completeLinesFrom(bytes, offset);
   const groups = new Map<string, { start: number; line: ModelCallLine }>();
   let newest: string | undefined;
   for (const { text, start } of lines) {
@@ -136,21 +131,58 @@ function isSettled(filepath: string, now: Date): boolean {
   return now.getTime() - statSync(filepath).mtimeMs > SETTLE_MS;
 }
 
+interface Transcript {
+  key: string;
+  bytes: () => Buffer;
+  settled: boolean;
+}
+
+function liveTranscripts(projectsDir: string, now: Date): Transcript[] {
+  return claudeTranscripts(projectsDir).map((filepath) => ({
+    key: relative(projectsDir, filepath).split(sep).join("/"),
+    bytes: () => readFileSync(filepath),
+    settled: isSettled(filepath, now),
+  }));
+}
+
+function copiesOfDeletedTranscripts(
+  copiesDir: string,
+  projectsDir: string
+): Transcript[] {
+  if (!existsSync(copiesDir)) return [];
+  return readdirSync(copiesDir, { recursive: true, encoding: "utf-8" })
+    .map((file) => file.split(sep).join("/"))
+    .filter((file) => file.endsWith(".jsonl.gz"))
+    .map((file) => file.slice(0, -".gz".length))
+    .filter((key) => !existsSync(resolve(projectsDir, key)))
+    .map((key) => ({
+      key,
+      bytes: () =>
+        Buffer.from(Bun.gunzipSync(readFileSync(resolve(copiesDir, `${key}.gz`)))),
+      settled: true,
+    }));
+}
+
+/** Reads each transcript from its cursor; one Claude Code already deleted is read from PAL's copy. */
 export function collectClaude(
   projectsDir: string,
   now: Date = new Date(),
-  billingAt: (at: string) => Billing = () => UNKNOWN
+  billingAt: (at: string) => Billing = () => UNKNOWN,
+  copiesDir?: string
 ): number {
   const cursor = readCursor("claude");
   let emitted = 0;
-  for (const filepath of claudeTranscripts(projectsDir)) {
-    const key = relative(projectsDir, filepath).split(sep).join("/");
-    const offset = Number(cursor[key] ?? 0);
-    const read = settledMessages(filepath, offset, isSettled(filepath, now));
+  const transcripts = [
+    ...liveTranscripts(projectsDir, now),
+    ...(copiesDir ? copiesOfDeletedTranscripts(copiesDir, projectsDir) : []),
+  ];
+  for (const transcript of transcripts) {
+    const offset = Number(cursor[transcript.key] ?? 0);
+    const read = settledMessages(transcript.bytes(), offset, transcript.settled);
     if (read.resumeAt === offset) continue;
     for (const line of read.messages) emitModelCall(line, billingAt(line.timestamp));
     emitted += read.messages.length;
-    cursor[key] = read.resumeAt;
+    cursor[transcript.key] = read.resumeAt;
     writeCursor("claude", cursor);
   }
   return emitted;

package/src/tools/lib/collectors/index.ts

@@ -25,7 +25,8 @@ const COLLECTORS: Record<string, () => number> = {
     collectClaude(
       resolve(platform.claudeDir(), "projects"),
       new Date(),
-      claudeBilling(claudeConfigFile())
+      claudeBilling(claudeConfigFile()),
+      resolve(paths.content(), "claude")
     ),
   opencode: () => collectOpencode(opencodeDatabase()),
   codex: () => collectCodex(platform.codexDir()),

package/src/tools/lib/project-history.ts

@@ -0,0 +1,100 @@
+/**
+ * A project's history as events: its registration, its dated decisions, and
+ * each ISC it closed or retired as the commitment kept or dropped. The live
+ * commands and the backfill build the same rows, so both give one id.
+ */
+
+import type { RecordAttribution } from "../../hooks/lib/actor";
+import { appendEvent } from "../../hooks/lib/events";
+import type { ProjectProgress } from "../../hooks/lib/projects";
+import { parseIscs } from "./project-isc";
+
+export interface HistoryRow {
+  id: string;
+  ts: string;
+  type: "project" | "commitment" | "decision";
+  project: string;
+  fields: Record<string, unknown>;
+  content?: Record<string, unknown>;
+}
+
+const ARCHIVE_HEADING = /^### (Archived|Retired) (\d{4}-\d{2}-\d{2})$/;
+const DECISION_LINE = /^- (\d{4}-\d{2}-\d{2}): (.+)$/;
+
+const dayStart = (day: string) => `${day}T00:00:00.000Z`;
+
+function shortHash(text: string): string {
+  return new Bun.CryptoHasher("sha256").update(text).digest("hex").slice(0, 12);
+}
+
+export function registrationRow(
+  project: Pick<ProjectProgress, "name" | "created" | "status">
+): HistoryRow {
+  return {
+    id: `project:${project.name}:registered`,
+    ts: project.created,
+    type: "project",
+    project: project.name,
+    fields: { change: "registered", status: project.status },
+  };
+}
+
+export function decisionRow(project: string, day: string, text: string): HistoryRow {
+  return {
+    id: `decision:${project}:${day}:${shortHash(text)}`,
+    ts: dayStart(day),
+    type: "decision",
+    project,
+    fields: { words: text.split(/\s+/).length },
+    content: { text },
+  };
+}
+
+export function decisionRows(project: ProjectProgress): HistoryRow[] {
+  return (project.decisions ?? "").split("\n").flatMap((line) => {
+    const decision = DECISION_LINE.exec(line);
+    return decision ? [decisionRow(project.name, decision[1], decision[2])] : [];
+  });
+}
+
+export function closedIscRows(project: ProjectProgress): HistoryRow[] {
+  const rows: HistoryRow[] = [];
+  let heading: RegExpExecArray | null = null;
+  for (const line of (project.changelog ?? "").split("\n")) {
+    heading = ARCHIVE_HEADING.exec(line) ?? heading;
+    const [isc] = parseIscs(line);
+    if (!heading || !isc) continue;
+    const outcome = heading[1] === "Retired" ? "dropped" : "kept";
+    rows.push({
+      id: `commitment:${project.name}:${isc.id}:${outcome}:${heading[2]}`,
+      ts: dayStart(heading[2]),
+      type: "commitment",
+      project: project.name,
+      fields: {
+        iscProject: project.name,
+        isc: isc.id,
+        outcome,
+        surface: null,
+        snoozeDays: null,
+      },
+      content: { text: isc.text },
+    });
+  }
+  return rows;
+}
+
+export function emitHistoryRow(
+  row: HistoryRow,
+  attribution: Partial<RecordAttribution> = {}
+): string | null {
+  return appendEvent(row.type, row.fields, {
+    id: row.id,
+    ts: row.ts,
+    session: null,
+    channel: null,
+    cwd: "",
+    project: row.project,
+    attribution,
+    ...(row.content && { content: row.content }),
+  });
+}