portable-agent-layer 0.93.0 → 0.95.0

package/src/hooks/lib/events.ts

@@ -0,0 +1,252 @@
+/**
+ * The local event log: one append-only metadata stream every pattern, report
+ * and dashboard is derived from. Events hold counts, durations, ids and enums;
+ * any text goes to the content store, which never leaves the machine.
+ */
+
+import { appendFileSync, existsSync, readdirSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { currentAttribution } from "./actor";
+import { logError } from "./log";
+import { ensureDir, paths } from "./paths";
+import { readAllProjects, resolveProjectFromCwd } from "./projects";
+import { eventLogEnabled } from "./settings";
+
+export const EVENT_SCHEMA_VERSION = 1;
+const MAX_STRING = 64;
+const MAX_ARRAY = 50;
+
+export const EVENT_FIELDS = {
+  turn: [
+    "hour",
+    "weekday",
+    "words",
+    "gapSec",
+    "afterBreak",
+    "skimmed",
+    "interrupted",
+    "repeated",
+    "corrected",
+    "reply",
+    "reaction",
+    "reactionRules",
+    "mood",
+    "hinted",
+    "complied",
+    "language",
+    "pastedChars",
+    "images",
+    "command",
+    "backAfterSec",
+  ],
+  reply: ["words", "listItems", "headings", "asked", "mode", "latencySec"],
+  session_start: [
+    "source",
+    "model",
+    "effort",
+    "entrypoint",
+    "branch",
+    "palVersion",
+    "tz",
+    "utcOffsetMin",
+  ],
+  session_end: ["reason", "durationSec", "turns"],
+  compaction: ["trigger", "withInstructions", "contextTokens"],
+  hook_timing: ["hook", "totalMs", "handlers"],
+  inference_call: [
+    "caller",
+    "route",
+    "model",
+    "tier",
+    "web",
+    "latencyMs",
+    "failure",
+    "inputTokens",
+    "outputTokens",
+  ],
+  pal_update: ["outcome", "from", "to", "durationSec", "skipReason"],
+  commitment: ["iscProject", "isc", "outcome", "surface", "snoozeDays"],
+  entity_first_seen: ["domain", "entityType"],
+  discord_nag: ["stage", "kind", "responseSec"],
+  permission_prompt: ["tool"],
+  notification: ["kind"],
+  tool_call: [
+    "tool",
+    "outcome",
+    "promptWaitSec",
+    "durationMs",
+    "inputChars",
+    "outputChars",
+    "fileExt",
+    "program",
+    "skill",
+    "subagent",
+    "subagentTokens",
+    "subagentTools",
+    "mcpServer",
+    "host",
+  ],
+} as const satisfies Record<string, readonly string[]>;
+
+export type EventType = keyof typeof EVENT_FIELDS;
+type EventValue =
+  | string
+  | number
+  | boolean
+  | null
+  | EventValue[]
+  | { [k: string]: EventValue };
+
+interface EventEnvelope {
+  v: number;
+  id: string;
+  ts: string;
+  type: EventType;
+  session: string | null;
+  runtime: string;
+  machine: string;
+  actor: string;
+  authority: string;
+  channel: string | null;
+  project: string | null;
+  cwd: string;
+}
+
+export type LoggedEvent = EventEnvelope & Record<string, unknown>;
+
+export interface AppendOptions {
+  id?: string;
+  ts?: string;
+  session?: string | null;
+  channel?: string | null;
+  cwd?: string;
+  content?: Record<string, unknown>;
+}
+
+function isEventValue(value: unknown, depth = 0): value is EventValue {
+  if (value === null || typeof value === "boolean") return true;
+  if (typeof value === "number") return Number.isFinite(value);
+  if (typeof value === "string") return value.length <= MAX_STRING;
+  if (depth > 1) return false;
+  if (Array.isArray(value))
+    return value.length <= MAX_ARRAY && value.every((v) => isEventValue(v, depth + 1));
+  if (typeof value === "object")
+    return Object.values(value).every((v) => isEventValue(v, depth + 1));
+  return false;
+}
+
+export function admittedFields(
+  type: EventType,
+  fields: Record<string, unknown>
+): Record<string, EventValue> {
+  const allowed: readonly string[] = EVENT_FIELDS[type];
+  const kept: Record<string, EventValue> = {};
+  for (const [name, value] of Object.entries(fields)) {
+    if (allowed.includes(name) && isEventValue(value)) kept[name] = value;
+  }
+  return kept;
+}
+
+const projectByCwd = new Map<string, string | null>();
+
+function projectOf(cwd: string): string | null {
+  if (!projectByCwd.has(cwd)) {
+    projectByCwd.set(cwd, resolveProjectFromCwd(cwd, readAllProjects())?.name ?? null);
+  }
+  return projectByCwd.get(cwd) ?? null;
+}
+
+function monthOf(ts: string): string {
+  return ts.slice(0, 7);
+}
+
+function eventsDir(): string {
+  return ensureDir(resolve(paths.memory(), "events"));
+}
+
+function contentDir(): string {
+  return ensureDir(resolve(paths.content(), "events"));
+}
+
+function envelope(type: EventType, options: AppendOptions): EventEnvelope {
+  const cwd = options.cwd ?? process.cwd();
+  return {
+    v: EVENT_SCHEMA_VERSION,
+    id: options.id ?? crypto.randomUUID(),
+    ts: options.ts ?? new Date().toISOString(),
+    type,
+    session: options.session ?? null,
+    ...currentAttribution(),
+    channel:
+      options.channel !== undefined ? options.channel : process.env.PAL_CHANNEL || null,
+    project: projectOf(cwd),
+    cwd,
+  };
+}
+
+/** Appends one event and returns its id, or null when the log is off or the write failed. */
+export function appendEvent(
+  type: EventType,
+  fields: Record<string, unknown>,
+  options: AppendOptions = {}
+): string | null {
+  if (!eventLogEnabled()) return null;
+  try {
+    const head = envelope(type, options);
+    const line = { ...head, ...admittedFields(type, fields) };
+    appendFileSync(
+      resolve(eventsDir(), `${monthOf(head.ts)}.jsonl`),
+      `${JSON.stringify(line)}\n`
+    );
+    if (options.content) {
+      const stored = { id: head.id, ts: head.ts, type, ...options.content };
+      appendFileSync(
+        resolve(contentDir(), `${monthOf(head.ts)}.jsonl`),
+        `${JSON.stringify(stored)}\n`
+      );
+    }
+    return head.id;
+  } catch (err) {
+    logError("events:append", err);
+    return null;
+  }
+}
+
+/** Runs a logging step so that nothing it does can fail the hook around it. */
+export function logQuietly(label: string, write: () => void): void {
+  try {
+    write();
+  } catch (err) {
+    logError(`events:${label}`, err);
+  }
+}
+
+function parseLines(file: string): LoggedEvent[] {
+  if (!existsSync(file)) return [];
+  return readFileSync(file, "utf-8")
+    .split("\n")
+    .filter(Boolean)
+    .flatMap((line) => {
+      try {
+        return [JSON.parse(line) as LoggedEvent];
+      } catch {
+        return [];
+      }
+    });
+}
+
+export function eventsInMonth(type: EventType, month: Date): LoggedEvent[] {
+  const file = resolve(eventsDir(), `${monthOf(month.toISOString())}.jsonl`);
+  return parseLines(file).filter((e) => e.type === type);
+}
+
+export function eventsSince(type: EventType, since: Date): LoggedEvent[] {
+  const dir = eventsDir();
+  const sinceTs = since.toISOString();
+  const fromMonth = monthOf(sinceTs);
+  return readdirSync(dir)
+    .filter((f) => f.endsWith(".jsonl") && f.slice(0, 7) >= fromMonth)
+    .sort()
+    .flatMap((f) => parseLines(resolve(dir, f)))
+    .filter((e) => e.type === type && e.ts >= sinceTs);
+}

package/src/hooks/lib/export.ts

@@ -20,7 +20,7 @@ import { palHome } from "./paths";
 const EXPORT_DIRS = ["telos", "memory", "skills", "agents"];
 
 /** Subdirectories/files to skip during export. */
-const SKIP_PATTERNS = ["memory/downloads"];
+const SKIP_PATTERNS = ["memory/downloads", "memory/content"];
 
 function shouldSkip(relPath: string): boolean {
   const normalized = relPath.replaceAll("\\", "/");

package/src/hooks/lib/hook-timing.ts

@@ -0,0 +1,33 @@
+/**
+ * How long a hook and each of its handlers took, logged as one event per run.
+ */
+
+import { appendEvent, logQuietly } from "./events";
+
+function elapsedMs(since: number): number {
+  return Math.round((performance.now() - since) * 10) / 10;
+}
+
+export function startHookTiming(hook: string) {
+  const started = performance.now();
+  const handlers: Record<string, number> = {};
+  return {
+    async time<T>(name: string, work: Promise<T> | (() => Promise<T> | T)): Promise<T> {
+      const begun = performance.now();
+      try {
+        return await (typeof work === "function" ? work() : work);
+      } finally {
+        handlers[name] = elapsedMs(begun);
+      }
+    },
+    log(session: string | null | undefined): void {
+      logQuietly("hook_timing", () =>
+        appendEvent(
+          "hook_timing",
+          { hook, totalMs: elapsedMs(started), handlers },
+          { session: session ?? null }
+        )
+      );
+    },
+  };
+}

package/src/hooks/lib/inference-events.ts

@@ -0,0 +1,56 @@
+/**
+ * One event per inference PAL makes for itself: who asked, which route and
+ * model answered, how long it took and how it ended.
+ */
+
+import { appendEvent, logQuietly } from "./events";
+
+interface InferenceCall {
+  caller?: string;
+  sessionId?: string;
+  tier?: string;
+  web?: boolean;
+}
+
+interface InferenceOutcome {
+  success: boolean;
+  output?: string;
+  error?: string;
+  usage?: { inputTokens: number; outputTokens: number };
+}
+
+export const SCHEMA_MISMATCH = "the reply does not match the schema";
+const NO_ROUTE = ["none", "disabled"];
+
+function failureOf(outcome: InferenceOutcome, route: string): string | null {
+  if (outcome.success) return null;
+  if (NO_ROUTE.includes(route)) return "unavailable";
+  if (outcome.error?.startsWith(SCHEMA_MISMATCH)) return "schema";
+  if (outcome.output) return "unparsable";
+  return outcome.error ? "error" : "no-reply";
+}
+
+export function logInferenceCall(
+  call: InferenceCall,
+  planned: { route: string; model: string | null },
+  outcome: InferenceOutcome,
+  latencyMs: number
+): void {
+  logQuietly("inference_call", () =>
+    appendEvent(
+      "inference_call",
+      {
+        caller: call.caller ?? "anonymous",
+        route: planned.route,
+        model: planned.model,
+        tier: call.tier ?? "small",
+        web: Boolean(call.web),
+        latencyMs: Math.round(latencyMs),
+        failure: failureOf(outcome, planned.route),
+        inputTokens: outcome.usage?.inputTokens ?? null,
+        outputTokens: outcome.usage?.outputTokens ?? null,
+      },
+      { session: call.sessionId ?? null }
+    )
+  );
+}

package/src/hooks/lib/inference.ts

@@ -40,6 +40,7 @@ import {
   streamJsonPrompt,
   writeInferenceAgent,
 } from "./antigravity-inference";
+import { logInferenceCall, SCHEMA_MISMATCH } from "./inference-events";
 import { logDebug, logError } from "./log";
 import {
   type FixedModelRoute,
@@ -134,6 +135,13 @@ function previewRoute(): RoutePreview {
 }
 
 /** True if any inference path is currently usable (subscription CLI OR API key). */
+/** Routes whose CLI can be limited to web search and fetch alone. */
+function canSearchWeb(): boolean {
+  if (isClaude() && hasClaudeBinary()) return true;
+  if (isCodex() && hasCodexBinary()) return true;
+  return isCopilot() && hasCopilotBinary();
+}
+
 export function canInfer(): boolean {
   if (isClaude() && hasClaudeBinary()) return true;
   if (isCodex() && hasCodexBinary()) return true;
@@ -158,6 +166,8 @@ interface InferenceOptions {
   caller?: string;
   /** Session ID the call is associated with — appears in debug logs as sessionId=X */
   sessionId?: string;
+  /** Let the model search and read the web, and nothing else; a route that can't fails */
+  web?: boolean;
 }
 
 interface InferenceResult {
@@ -179,7 +189,22 @@ function schemaProblem(
     .join("; ");
 }
 
+function plannedRoute(opts: InferenceOptions): { route: string; model: string | null } {
+  const { route } = previewRoute();
+  if (route === "opencode-spawn")
+    return { route, model: opencodeTierModel(opts.tier ?? "small") };
+  if (isFixedModelRoute(route)) return { route, model: modelFor(route, opts) };
+  return { route, model: null };
+}
+
 export async function inference(opts: InferenceOptions): Promise<InferenceResult> {
+  const started = performance.now();
+  const result = await checkedInference(opts);
+  logInferenceCall(opts, plannedRoute(opts), result, performance.now() - started);
+  return result;
+}
+
+async function checkedInference(opts: InferenceOptions): Promise<InferenceResult> {
   const result = await routeInference(opts);
   if (!opts.jsonSchema || !result.success || !result.output) return result;
   const problem = schemaProblem(result.output, opts.jsonSchema);
@@ -191,7 +216,7 @@ export async function inference(opts: InferenceOptions): Promise<InferenceResult
   return {
     ...result,
     success: false,
-    error: `the reply does not match the schema: ${problem}`,
+    error: `${SCHEMA_MISMATCH}: ${problem}`,
   };
 }
 
@@ -210,6 +235,10 @@ async function routeInference(opts: InferenceOptions): Promise<InferenceResult>
   const caller = opts.caller ?? "anonymous";
   const session = opts.sessionId ?? "-";
   const tag = `caller=${caller} sessionId=${session}`;
+  if (opts.web && !canSearchWeb()) {
+    logDebug("inference", `${tag} refuse: no web route agent=${agent}`);
+    return { success: false, error: `web research is not available on ${agent} yet` };
+  }
   if (isClaude()) {
     const bin = getClaudeBinary();
     if (bin) {
@@ -342,6 +371,15 @@ function hasAntigravityBinary(): boolean {
  * line. System, user and any JSON-schema instruction all travel together on
  * stdin instead, the same way every other agent receives them.
  */
+const CLAUDE_WEB_TOOLS = "WebSearch,WebFetch";
+
+/** --tools alone makes the web tools visible; --allowed-tools spares them the prompt. */
+function claudeTools(opts: InferenceOptions): string[] {
+  return opts.web
+    ? ["--tools", CLAUDE_WEB_TOOLS, "--allowed-tools", CLAUDE_WEB_TOOLS]
+    : ["--tools", ""];
+}
+
 export function buildClaudeArgs(
   opts: InferenceOptions,
   systemPromptFile?: string
@@ -350,8 +388,7 @@ export function buildClaudeArgs(
     "--print",
     "--model",
     modelFor("claude-spawn", opts),
-    "--tools",
-    "",
+    ...claudeTools(opts),
     "--output-format",
     "text",
     "--setting-sources",
@@ -447,6 +484,7 @@ export function buildCodexArgs(opts: InferenceOptions): string[] {
     "--sandbox",
     "read-only",
     "--ephemeral",
+    ...(opts.web ? ["-c", "web_search=live"] : []),
   ];
 }
 
@@ -584,6 +622,8 @@ async function inferenceViaAntigravitySpawn(
  *                                naming an unknown tool hides every tool
  *   --deny-tool=…             → backstop: deny rules win over any allow rule
  *   --silent                  → keeps the tool-filter notices out of stdout
+ *   web: only web_fetch, pre-approved for every URL; Copilot has no search tool,
+ *        so it fetches search pages itself
  *
  * Copilot has no --system-prompt flag, so system + user + JSON-schema are
  * concatenated into one prompt delivered on stdin. `-p/--prompt` is deliberately
@@ -591,20 +631,25 @@ async function inferenceViaAntigravitySpawn(
  * survive cmd.exe when Bun.spawn resolves copilot to its Windows .cmd shim.
  * Piping stdin keeps copilot non-interactive, so dropping -p costs nothing.
  */
-export function buildCopilotArgs(_opts: InferenceOptions): string[] {
+export function buildCopilotArgs(opts: InferenceOptions): string[] {
   return [
     "--no-custom-instructions",
     "--disable-builtin-mcps",
     "--no-auto-update",
     "--no-color",
     "--silent",
-    "--available-tools=none",
+    ...copilotTools(opts),
     "--deny-tool=shell",
     "--deny-tool=write",
-    "--deny-tool=url",
   ];
 }
 
+function copilotTools(opts: InferenceOptions): string[] {
+  return opts.web
+    ? ["--available-tools=web_fetch", "--allow-tool=web_fetch", "--allow-all-urls"]
+    : ["--available-tools=none", "--deny-tool=url"];
+}
+
 /**
  * Extract the agent's text reply from opencode --format json NDJSON output.
  * Concatenates all `type:"text"` event payloads in order. Returns empty

package/src/hooks/lib/interaction.ts

@@ -1,18 +1,13 @@
 /**
- * Measures every user turn in code and logs the features, never the text.
+ * Measures every user turn in code and logs the features as turn events; the
+ * prompt text goes only to the content store, which never leaves the machine.
  * Per-session tracking lives in state and is discarded with the session.
  */
 
-import {
-  appendFileSync,
-  existsSync,
-  readdirSync,
-  readFileSync,
-  writeFileSync,
-} from "node:fs";
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { resolve } from "node:path";
-import { currentAttribution } from "./actor";
 import { hearsPromptContext } from "./agent";
+import { appendEvent, eventsInMonth, eventsSince, logQuietly } from "./events";
 import { asksForLess, followedHint, ignoredHintReminder } from "./interaction-hint-check";
 import {
   type Baseline,
@@ -34,6 +29,8 @@ import { keepSample, replyEnd } from "./interaction-samples";
 import { ensureDir, paths } from "./paths";
 import { isSystemText, stripInjectedTags } from "./prompt-text";
 import { isEnabled } from "./settings";
+import { responseModeOf, turnExtras } from "./turn-extras";
+import { backAfterNotificationSec } from "./waiting-on-user";
 import { localClock } from "./wall-clock";
 
 interface ReplyShape {
@@ -135,20 +132,51 @@ function measureTurn(
   };
 }
 
-function eventsDir(): string {
+function legacyTurnsDir(): string {
   return ensureDir(resolve(paths.signals(), "interaction"));
 }
 
-function monthFile(date: Date): string {
-  return resolve(eventsDir(), `${date.toISOString().slice(0, 7)}.jsonl`);
+function logTurn(event: TurnEvent, prompt: string, text: string): void {
+  const { ts, session, channel, ...fields } = event;
+  logQuietly("turn", () =>
+    appendEvent(
+      "turn",
+      {
+        ...fields,
+        ...turnExtras(prompt, text),
+        backAfterSec: backAfterNotificationSec(session, new Date(ts)),
+      },
+      { ts, session, channel, content: { prompt: text } }
+    )
+  );
 }
 
-function appendEvent(event: TurnEvent): void {
-  const line = { ...currentAttribution(), type: "turn", ...event };
-  appendFileSync(monthFile(new Date(event.ts)), `${JSON.stringify(line)}\n`);
+function secondsBetween(from: string | undefined, to: Date): number | null {
+  if (!from) return null;
+  return Math.round((to.getTime() - new Date(from).getTime()) / 100) / 10;
 }
 
-function readEvents(file: string): TurnEvent[] {
+function logReply(
+  session: string,
+  reply: string,
+  shape: ReplyShape,
+  track: SessionTrack
+) {
+  const { at, ...features } = shape;
+  logQuietly("reply", () =>
+    appendEvent(
+      "reply",
+      {
+        ...features,
+        mode: responseModeOf(reply),
+        latencySec: secondsBetween(track.lastPromptAt, new Date(at)),
+      },
+      { ts: at, session, content: { reply } }
+    )
+  );
+}
+
+function readLegacyTurns(file: string): TurnEvent[] {
   if (!existsSync(file)) return [];
   return readFileSync(file, "utf-8")
     .split("\n")
@@ -162,20 +190,33 @@ function readEvents(file: string): TurnEvent[] {
     });
 }
 
-export function turnsSince(since: Date): TurnEvent[] {
+function legacyTurnsSince(since: Date): TurnEvent[] {
   const fromMonth = since.toISOString().slice(0, 7);
   const sinceTs = since.toISOString();
-  return readdirSync(eventsDir())
+  return readdirSync(legacyTurnsDir())
     .filter((f) => f.endsWith(".jsonl") && f.slice(0, 7) >= fromMonth)
     .sort()
-    .flatMap((f) => readEvents(resolve(eventsDir(), f)))
+    .flatMap((f) => readLegacyTurns(resolve(legacyTurnsDir(), f)))
     .filter((e) => e.ts >= sinceTs);
 }
 
+export function turnsSince(since: Date): TurnEvent[] {
+  const streamed = eventsSince("turn", since) as unknown as TurnEvent[];
+  return [...legacyTurnsSince(since), ...streamed].sort((a, b) =>
+    a.ts.localeCompare(b.ts)
+  );
+}
+
+function turnsInMonth(month: Date): TurnEvent[] {
+  const file = `${month.toISOString().slice(0, 7)}.jsonl`;
+  const streamed = eventsInMonth("turn", month) as unknown as TurnEvent[];
+  return [...readLegacyTurns(resolve(legacyTurnsDir(), file)), ...streamed];
+}
+
 /** This month and last, outside the session being judged. */
 function historyOutside(session: string, now: Date): TurnEvent[] {
   const lastMonth = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 1, 1));
-  return [...readEvents(monthFile(lastMonth)), ...readEvents(monthFile(now))].filter(
+  return [...turnsInMonth(lastMonth), ...turnsInMonth(now)].filter(
     (e) => e.session !== session
   );
 }
@@ -234,10 +275,12 @@ export function recordReply(
 ): void {
   if (!session || !reply.trim() || !isEnabled("interactionAwareness")) return;
   const track = trackOf(session);
+  const shape = replyShape(reply, now);
+  logReply(session, reply, shape, track);
   writeTrack(session, {
     ...track,
     updated: now.toISOString(),
-    reply: replyShape(reply, now),
+    reply: shape,
     replyEnd: replyEnd(reply),
   });
 }
@@ -295,12 +338,16 @@ export function observeTurn(
     moodReminder(mood, track.mood ?? ""),
     reminderForIgnoredHint(complied, mood.key, replyWords, usualReply, history)
   );
-  appendEvent({
-    ...event,
-    mood: mood.key,
-    hinted: reminder !== null && hearsPromptContext(),
-    ...(complied !== null && { complied }),
-  });
+  logTurn(
+    {
+      ...event,
+      mood: mood.key,
+      hinted: reminder !== null && hearsPromptContext(),
+      ...(complied !== null && { complied }),
+    },
+    prompt,
+    text
+  );
   sampleReaction(event, text, track);
   writeTrack(session, {
     ...track,