portable-agent-layer 0.93.0 → 0.95.0

package/src/hooks/SessionClose.ts

@@ -13,14 +13,19 @@
 import { autoUpdateOnClose } from "./lib/auto-update";
 import { duplicatesCursorHooks } from "./lib/cursor-shadow";
 import { logError } from "./lib/log";
+import { logSessionEnd } from "./lib/session-events";
 import { isPalSpawnedInference } from "./lib/spawn-guard";
 import { readStdinJSON } from "./lib/stdin";
+import { spawnTranscriptPreserve } from "./lib/transcript-preserve";
 
 if (isPalSpawnedInference()) process.exit(0);
 if (duplicatesCursorHooks()) process.exit(0);
 
+spawnTranscriptPreserve();
+
 try {
   const input = await readStdinJSON<{ reason?: string }>();
+  logSessionEnd(input);
   autoUpdateOnClose(input?.reason);
 } catch (err) {
   logError("SessionClose", err);

package/src/hooks/UserPromptOrchestrator.ts

@@ -9,6 +9,7 @@
 
 import { injectPromptContext } from "./handlers/inject-retrieval";
 import { duplicatesCursorHooks } from "./lib/cursor-shadow";
+import { startHookTiming } from "./lib/hook-timing";
 import { type HookTurnPayload, hookSessionId } from "./lib/hook-turn";
 import { logDebug, logPromptSnapshot } from "./lib/log";
 import { capturePrompt } from "./lib/prompt-capture";
@@ -28,9 +29,15 @@ const input = await readStdinJSON<PromptSubmitInput>();
 logDebug("UserPromptOrchestrator", `Input: ${JSON.stringify(input).slice(0, 200)}`);
 if (!input?.prompt) process.exit(0);
 
+const timing = startHookTiming("prompt");
 const sentAt = new Date();
 const sessionId = hookSessionId(input);
-const injected = await injectPromptContext(input.prompt, sessionId);
+const injected = await timing.time("inject-context", () =>
+  injectPromptContext(input.prompt, sessionId)
+);
 logPromptSnapshot(input.prompt, injected);
 
-await capturePrompt(input.prompt, sessionId, sentAt, "UserPromptOrchestrator");
+await timing.time("capture", () =>
+  capturePrompt(input.prompt, sessionId, sentAt, "UserPromptOrchestrator")
+);
+timing.log(sessionId);

package/src/hooks/handlers/entity-extraction.ts

@@ -1,7 +1,8 @@
 /**
  * Stop handler: learn the people and companies the user mentioned since the last
- * stop, so the prompt-time cards know them next time. Runs detached; one model
- * call at most, and only when the new text names someone.
+ * stop, so the prompt-time cards know them next time, and research the new ones on
+ * the web for review. Runs detached; one extraction call, only when the new text
+ * names someone, and one web research call per new entry.
  */
 
 import { ingestEntities } from "../../tools/knowledge/ingest";
@@ -18,9 +19,10 @@ import {
   unseenUserText,
   worthExtracting,
 } from "../lib/entity-extraction";
+import { newResearchTargets, researchNewEntities } from "../lib/entity-research";
 import { canInfer, inference } from "../lib/inference";
 import { logDebug, logError } from "../lib/log";
-import { identity, isEnabled } from "../lib/settings";
+import { identity, isEnabled, isOptedIn } from "../lib/settings";
 import { logTokenUsage } from "../lib/token-usage";
 import { parseMessages } from "../lib/transcript";
 
@@ -65,6 +67,11 @@ export async function extractSessionEntities(
     "entity-extraction",
     `${written.people.length} people, ${written.companies.length} companies, ${plan.review.length} for review`
   );
+  if (isOptedIn("entityResearch"))
+    await researchNewEntities(newResearchTargets(plan.ingest, written), {
+      principal: id.principal.name,
+      infer: inference,
+    });
 }
 
 if (process.argv[2] === "--run") {

package/src/hooks/lib/attention-events.ts

@@ -0,0 +1,43 @@
+/**
+ * When the agent waits on the user: a permission prompt, or a notification that
+ * it needs them.
+ */
+
+import { normalizeToolUse } from "./agent";
+import { appendEvent, logQuietly } from "./events";
+import { hookSessionId } from "./hook-turn";
+import { markWaiting } from "./waiting-on-user";
+
+function logPermissionPrompt(
+  payload: Record<string, unknown> | null,
+  tool: string | null,
+  now: Date = new Date()
+): void {
+  logQuietly("permission_prompt", () => {
+    const session = hookSessionId(payload) ?? null;
+    appendEvent("permission_prompt", { tool }, { ts: now.toISOString(), session });
+    if (session)
+      markWaiting("permission", session, { ts: now.toISOString(), tool: tool ?? "" });
+  });
+}
+
+function logNotification(payload: Record<string, unknown> | null, now: Date): void {
+  logQuietly("notification", () => {
+    const session = hookSessionId(payload) ?? null;
+    const kind =
+      typeof payload?.notification_type === "string" ? payload.notification_type : null;
+    appendEvent("notification", { kind }, { ts: now.toISOString(), session });
+    if (session) markWaiting("notified", session, { ts: now.toISOString() });
+  });
+}
+
+/** Claude's PermissionRequest and Notification share one hook; it prints nothing. */
+export function logAttention(
+  payload: Record<string, unknown> | null,
+  now: Date = new Date()
+): void {
+  const event = payload?.hook_event_name ?? payload?.hookEventName;
+  if (event === "Notification") logNotification(payload, now);
+  if (event === "PermissionRequest")
+    logPermissionPrompt(payload, normalizeToolUse(payload)?.toolName ?? null, now);
+}

package/src/hooks/lib/auto-update.ts

@@ -15,6 +15,7 @@ import { spawn, spawnSync } from "node:child_process";
 import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { resolve } from "node:path";
 import { cachedStatus, getInstalledVersion, isRepoMode } from "../handlers/update-check";
+import { appendEvent, logQuietly } from "./events";
 import { logDebug, logError } from "./log";
 import { assets, palPkg, paths } from "./paths";
 import { raw as rawSettings } from "./settings";
@@ -118,8 +119,23 @@ function hasUncommittedChanges(): boolean {
   return (status.stdout ?? "").trim().length > 0;
 }
 
+function outcomeOf(entry: AutoUpdateLedger): string {
+  if (!entry.ok) return "failed";
+  return entry.from === entry.to ? "unchanged" : "updated";
+}
+
+function secondsBetween(from?: string, to?: string): number | null {
+  if (!from || !to) return null;
+  return Math.round((Date.parse(to) - Date.parse(from)) / 1000);
+}
+
+function logUpdateEvent(fields: Record<string, unknown>): void {
+  logQuietly("pal_update", () => appendEvent("pal_update", fields));
+}
+
 function recordSkip(reason: string): void {
   writeLedger({ ...readLedger(), skippedAt: new Date().toISOString(), skipped: reason });
+  logUpdateEvent({ outcome: "skipped", skipReason: reason });
   logDebug("auto-update", `skipped: ${reason}`);
 }
 
@@ -278,7 +294,7 @@ function update(): AutoUpdateLedger {
 
   const ok = run.status === 0;
   logDebug("auto-update", `update exited ${run.status}`);
-  return writeLedger({
+  const entry = writeLedger({
     ...readLedger(),
     finishedAt: new Date().toISOString(),
     ok,
@@ -286,6 +302,13 @@ function update(): AutoUpdateLedger {
     to: getInstalledVersion(),
     error: ok ? undefined : (run.error?.message ?? `exit ${run.status}`),
   });
+  logUpdateEvent({
+    outcome: outcomeOf(entry),
+    from: entry.from ?? null,
+    to: entry.to ?? null,
+    durationSec: secondsBetween(entry.attemptedAt, entry.finishedAt),
+  });
+  return entry;
 }
 
 /** Hands the update to its own process, which outlives the session that closed. */

package/src/hooks/lib/commitment-events.ts

@@ -0,0 +1,20 @@
+/**
+ * A commitment is an ISC: completing it keeps it, retiring it drops it, and a
+ * snooze puts it off. Each change of state is one event, whichever surface made it.
+ */
+
+import { appendEvent, logQuietly } from "./events";
+
+export type CommitmentOutcome = "kept" | "dropped" | "snoozed" | "reopened";
+
+export function logCommitment(
+  iscProject: string,
+  isc: number,
+  outcome: CommitmentOutcome,
+  surface: "cli" | "control-room",
+  snoozeDays: number | null = null
+): void {
+  logQuietly("commitment", () =>
+    appendEvent("commitment", { iscProject, isc, outcome, surface, snoozeDays })
+  );
+}

package/src/hooks/lib/discord-events.ts

@@ -0,0 +1,25 @@
+/**
+ * A nag the Discord bot sent, and how long the user took to answer it. The bot
+ * runs outside PAL, so it reports both through `pal cli events discord-nag`.
+ */
+
+import { appendEvent } from "./events";
+
+export const NAG_STAGES = ["sent", "answered"] as const;
+export type NagStage = (typeof NAG_STAGES)[number];
+
+export function isNagStage(value: unknown): value is NagStage {
+  return typeof value === "string" && (NAG_STAGES as readonly string[]).includes(value);
+}
+
+export function logDiscordNag(
+  stage: NagStage,
+  kind: string | null,
+  responseSec: number | null
+): string | null {
+  return appendEvent("discord_nag", {
+    stage,
+    kind,
+    responseSec: stage === "answered" ? responseSec : null,
+  });
+}

package/src/hooks/lib/entity-events.ts

@@ -0,0 +1,16 @@
+/**
+ * The first time a person, company, idea or research topic enters the knowledge
+ * store. The name stays in the content store; the event carries only its kind.
+ */
+
+import { appendEvent, logQuietly } from "./events";
+
+export function logEntityFirstSeen(
+  domain: string,
+  entityType: string,
+  name: string
+): void {
+  logQuietly("entity_first_seen", () =>
+    appendEvent("entity_first_seen", { domain, entityType }, { content: { name } })
+  );
+}

package/src/hooks/lib/entity-extraction.ts

@@ -35,13 +35,35 @@ export interface ExtractedEntity {
   fact: string;
 }
 
+export interface ResearchProfile {
+  match: "sure" | "unsure" | "none";
+  summary: string;
+  role: string;
+  organization: string;
+  website: string;
+  linkedin: string;
+  socials: { platform: string; url: string }[];
+  registry: {
+    name: string;
+    number: string;
+    taxNumber: string;
+    seat: string;
+    status: string;
+    managers: string[];
+    url: string;
+  };
+  news: { date: string; title: string; url: string }[];
+  sources: string[];
+}
+
 export interface ReviewItem {
   ts: string;
   source: string;
   id: string;
-  reason: "ambiguous" | "unknown-existing" | "first-name-only";
+  reason: "ambiguous" | "unknown-existing" | "first-name-only" | "web-profile";
   entity: ExtractedEntity;
   candidates: string[];
+  profile?: ResearchProfile;
 }
 
 export interface ExtractionPlan {
@@ -49,7 +71,7 @@ export interface ExtractionPlan {
   review: ReviewItem[];
 }
 
-function closedObject<P extends Record<string, unknown>>(properties: P) {
+export function closedObject<P extends Record<string, unknown>>(properties: P) {
   return {
     type: "object" as const,
     additionalProperties: false,

package/src/hooks/lib/entity-research.ts

@@ -0,0 +1,305 @@
+/**
+ * Researching a person or company on the web: website, LinkedIn, social accounts,
+ * the company registry entry and recent news, each with its source. A profile only
+ * reaches the store through the review queue.
+ */
+
+import type { IngestInput, IngestResult } from "../../tools/knowledge/ingest";
+import {
+  closedObject,
+  type ExtractedEntity,
+  queueForReview,
+  type ResearchProfile,
+  type ReviewItem,
+} from "./entity-extraction";
+import type { inference } from "./inference";
+import { logDebug, logError } from "./log";
+import { logTokenUsage } from "./token-usage";
+
+export interface ResearchTarget {
+  kind: ExtractedEntity["kind"];
+  name: string;
+  slug: string;
+  organization: string;
+  fact: string;
+}
+
+export interface ResearchDeps {
+  principal: string;
+  infer: typeof inference;
+  today?: string;
+}
+
+const MAX_TARGETS = 3;
+const RESEARCH_TIMEOUT_MS = 480_000;
+
+function created(written: IngestResult["people"]): Set<string> {
+  return new Set(written.filter((w) => w.created).map((w) => w.slug));
+}
+
+function target(
+  kind: ResearchTarget["kind"],
+  name: string,
+  slug: string,
+  organization: string | null | undefined,
+  fact: string | null | undefined
+): ResearchTarget {
+  return { kind, name, slug, organization: organization ?? "", fact: fact ?? "" };
+}
+
+/** New companies, and new people with an organization to tell namesakes apart by. */
+export function newResearchTargets(
+  input: IngestInput,
+  written: IngestResult
+): ResearchTarget[] {
+  const newCompanies = created(written.companies);
+  const newPeople = created(written.people);
+  const companies = (input.companies ?? []).flatMap((c, i) => {
+    const slug = written.companies[i]?.slug ?? "";
+    return newCompanies.has(slug) ? [target("company", c.name, slug, "", c.context)] : [];
+  });
+  const people = (input.people ?? []).flatMap((p, i) => {
+    const slug = written.people[i]?.slug ?? "";
+    return newPeople.has(slug) && p.company
+      ? [target("person", p.name, slug, p.company, p.context)]
+      : [];
+  });
+  return [...companies, ...people].slice(0, MAX_TARGETS);
+}
+
+export function researchSchema() {
+  const text = { type: "string" as const };
+  const list = (items: Record<string, unknown>) => ({ type: "array" as const, items });
+  return closedObject({
+    match: { type: "string" as const, enum: ["sure", "unsure", "none"] },
+    summary: text,
+    role: text,
+    organization: text,
+    website: text,
+    linkedin: text,
+    socials: list(closedObject({ platform: text, url: text })),
+    registry: closedObject({
+      name: text,
+      number: text,
+      taxNumber: text,
+      seat: text,
+      status: text,
+      managers: list(text),
+      url: text,
+    }),
+    news: list(closedObject({ date: text, title: text, url: text })),
+    sources: list(text),
+  });
+}
+
+const CHECKLIST = {
+  company: [
+    "- website: the official website.",
+    "- linkedin: the LinkedIn company page.",
+    "- socials: other official accounts (X, Facebook, Instagram, YouTube, GitHub).",
+    "- registry: its entry in the official company register of its country: registered name, registration number, tax number, registered seat, status (active, in liquidation…), managing directors, and the URL you read it on. For a Hungarian company (Kft., Zrt., Bt., Nyrt., a .hu site) that is the cégjegyzék: e-cegjegyzek.hu, or public mirrors such as ceginformacio.hu, nemzeticegtar.hu or opten.hu.",
+    "- news: up to five news items or public statements from the last 12 months, newest first.",
+    "- summary: two or three sentences on what the company does, its size and where it is based.",
+  ],
+  person: [
+    "- role and organization: their current position and employer.",
+    "- website: their own site, if they have one.",
+    "- linkedin: their LinkedIn profile.",
+    "- socials: their accounts on X, GitHub, Instagram, YouTube or similar.",
+    "- news: up to five recent news items, interviews, talks or public statements from the last 12 months, newest first.",
+    "- summary: two or three sentences on who they are and what they work on.",
+  ],
+};
+
+export function researchSystem(principal: string, kind: ResearchTarget["kind"]): string {
+  return [
+    `You research a ${kind} from ${principal}'s contact book on the web. Search the web and open pages; do not answer from memory.`,
+    "Find:",
+    ...CHECKLIST[kind],
+    "Rules:",
+    `- match: "sure" when what you found is clearly this ${kind} (the same name AND it fits what ${principal} said about them), "unsure" when you found namesakes and cannot tell which, "none" when you found nothing. Only "sure" profiles are kept.`,
+    "- Only state what a page you opened or a search result shows. Never guess a URL, number or date.",
+    "- Every URL must be one you saw. List every page you used in sources.",
+    "- Leave a field empty, or a list empty, when you did not find it.",
+    "- Dates as YYYY-MM-DD, or YYYY-MM when the day is not given.",
+  ].join("\n");
+}
+
+function researchUser(target: ResearchTarget, today: string): string {
+  return [
+    `Today is ${today}.`,
+    `${target.kind}: ${target.name}`,
+    target.organization && `organization: ${target.organization}`,
+    target.fact &&
+      `what ${target.kind === "person" ? "was said about them" : "was said about it"}: ${target.fact}`,
+  ]
+    .filter(Boolean)
+    .join("\n");
+}
+
+function parseProfile(output: string | undefined): ResearchProfile | null {
+  try {
+    return output ? (JSON.parse(output) as ResearchProfile) : null;
+  } catch {
+    return null;
+  }
+}
+
+function isWebUrl(url: string): boolean {
+  return /^https?:\/\/[^\s]+$/.test(url);
+}
+
+/** Drops any link that is not a plain http(s) URL, so the store never holds a guess dressed as one. */
+function withWebUrlsOnly(p: ResearchProfile): ResearchProfile {
+  const url = (u: string) => (isWebUrl(u) ? u : "");
+  return {
+    ...p,
+    website: url(p.website),
+    linkedin: url(p.linkedin),
+    socials: p.socials.filter((s) => isWebUrl(s.url)),
+    registry: { ...p.registry, url: url(p.registry.url) },
+    news: p.news.filter((n) => isWebUrl(n.url)),
+    sources: p.sources.filter(isWebUrl),
+  };
+}
+
+export type ResearchOutcome =
+  | { found: true; profile: ResearchProfile }
+  | { found: false; reason: string };
+
+export async function researchEntity(
+  target: ResearchTarget,
+  deps: ResearchDeps
+): Promise<ResearchOutcome> {
+  const result = await deps.infer({
+    system: researchSystem(deps.principal, target.kind),
+    user: researchUser(target, deps.today ?? new Date().toISOString().slice(0, 10)),
+    tier: "medium",
+    timeout: RESEARCH_TIMEOUT_MS,
+    jsonSchema: researchSchema(),
+    caller: "entity-research",
+    web: true,
+  });
+  if (result.usage) logTokenUsage("entity-research", result.usage);
+  const profile = result.success ? parseProfile(result.output) : null;
+  if (!profile) return { found: false, reason: result.error ?? "no usable reply" };
+  if (profile.match !== "sure")
+    return { found: false, reason: `match: ${profile.match}. ${profile.summary}`.trim() };
+  const checked = withWebUrlsOnly(profile);
+  if (checked.sources.length === 0) return { found: false, reason: "no sources given" };
+  return { found: true, profile: checked };
+}
+
+export function researchItem(
+  target: ResearchTarget,
+  profile: ResearchProfile
+): ReviewItem {
+  return {
+    id: crypto.randomUUID().slice(0, 8),
+    ts: new Date().toISOString(),
+    source: profile.sources[0],
+    reason: "web-profile",
+    entity: {
+      kind: target.kind,
+      name: target.name,
+      existing: target.slug,
+      aliases: [],
+      role: profile.role,
+      organization: profile.organization || target.organization,
+      relation: "",
+      fact: profile.summary,
+    },
+    candidates: target.slug ? [target.slug] : [],
+    profile,
+  };
+}
+
+async function researchOrLog(
+  target: ResearchTarget,
+  deps: ResearchDeps
+): Promise<ReviewItem[]> {
+  try {
+    const outcome = await researchEntity(target, deps);
+    if (outcome.found) return [researchItem(target, outcome.profile)];
+    logDebug("entity-research", `${target.slug}: ${outcome.reason}`);
+    return [];
+  } catch (err) {
+    logError(`entity-research:${target.slug}`, err);
+    return [];
+  }
+}
+
+export async function researchNewEntities(
+  targets: ResearchTarget[],
+  deps: ResearchDeps
+): Promise<void> {
+  const found = (await Promise.all(targets.map((t) => researchOrLog(t, deps)))).flat();
+  queueForReview(found);
+  logDebug("entity-research", `${found.length} of ${targets.length} found for review`);
+}
+
+function registryLines(r: ResearchProfile["registry"]): string[] {
+  const fields: Array<[string, string]> = [
+    ["Registered name", r.name],
+    ["Registration number", r.number],
+    ["Tax number", r.taxNumber],
+    ["Seat", r.seat],
+    ["Status", r.status],
+    ["Managing directors", r.managers.join(", ")],
+    ["Registry entry", r.url],
+  ];
+  return fields.filter(([, v]) => v).map(([k, v]) => `- ${k}: ${v}`);
+}
+
+function section(title: string, lines: string[]): string[] {
+  return lines.length > 0 ? ["", `**${title}**`, ...lines] : [];
+}
+
+/** The profile as markdown: what the review list shows and what an accepted entry keeps. */
+export function renderProfile(p: ResearchProfile): string {
+  const links = [
+    p.website && `- Website: ${p.website}`,
+    p.linkedin && `- LinkedIn: ${p.linkedin}`,
+    ...p.socials.map((s) => `- ${s.platform}: ${s.url}`),
+  ].filter((l): l is string => Boolean(l));
+  return [
+    p.summary,
+    ...section("Links", links),
+    ...section("Registry", registryLines(p.registry)),
+    ...section(
+      "News",
+      p.news.map((n) => `- ${n.date} ${n.title} (${n.url})`)
+    ),
+    ...section(
+      "Sources",
+      p.sources.map((s) => `- ${s}`)
+    ),
+  ].join("\n");
+}
+
+function socialKey(platform: string): string {
+  return platform.toLowerCase().replace(/[^a-z0-9]+/g, "") || "link";
+}
+
+function hostname(url: string): string | null {
+  return url ? new URL(url).hostname.replace(/^www\./, "") : null;
+}
+
+/** The accepted profile, written to the entry it was researched for. */
+export function profileIngest(item: ReviewItem, name: string): IngestInput {
+  const p = item.profile as ResearchProfile;
+  const slug = item.entity.existing || null;
+  const context = renderProfile(p);
+  if (item.entity.kind === "company")
+    return { companies: [{ name, slug, domain: hostname(p.website), context }] };
+  const social = Object.fromEntries(
+    [
+      ["website", p.website],
+      ["linkedin", p.linkedin],
+      ...p.socials.map((s) => [socialKey(s.platform), s.url]),
+    ].filter(([, url]) => url)
+  );
+  return {
+    people: [{ name, slug, title: p.role || null, social, context }],
+  };
+}

package/src/hooks/lib/entity-review.ts

@@ -4,10 +4,11 @@
  */
 
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import { ingestEntities } from "../../tools/knowledge/ingest";
+import { type IngestInput, ingestEntities } from "../../tools/knowledge/ingest";
 import { loadKnownEntities, loadNameIndex } from "./entity-cards";
 import { ingestInput, type ReviewItem, reviewQueueFile } from "./entity-extraction";
 import { indexedEntities, type NamedEntity, type NameIndex } from "./entity-names";
+import { profileIngest } from "./entity-research";
 
 export interface AcceptChoice {
   as?: string;
@@ -68,6 +69,22 @@ export function loadEntityReviewNudge(): string {
   ].join("\n");
 }
 
+function acceptedInput(
+  item: ReviewItem,
+  choice: AcceptChoice,
+  index: NameIndex
+): IngestInput {
+  if (!item.profile) {
+    const known = chosenEntity(item, choice.as, index);
+    return ingestInput(item.entity, index, known, choice.name);
+  }
+  const target = chosenEntity(item, choice.as ?? item.entity.existing, index);
+  return profileIngest(
+    { ...item, entity: { ...item.entity, existing: target?.slug ?? "" } },
+    target?.title ?? choice.name ?? item.entity.name
+  );
+}
+
 export function rejectReview(id: string): ReviewItem | null {
   return takeFromQueue(id);
 }
@@ -77,11 +94,7 @@ export function acceptReview(id: string, choice: AcceptChoice = {}): Accepted |
   const queued = pendingReviews().find((i) => i.id === id);
   if (!queued) return null;
   const index = loadNameIndex(loadKnownEntities());
-  const known = chosenEntity(queued, choice.as, index);
-  const result = ingestEntities(
-    ingestInput(queued.entity, index, known, choice.name),
-    queued.source
-  );
+  const result = ingestEntities(acceptedInput(queued, choice, index), queued.source);
   takeFromQueue(id);
   const written = result.people[0] ?? result.companies[0];
   return { item: queued, slug: written.slug, created: written.created };