portable-agent-layer 0.32.0 → 0.34.0

package/src/hooks/handlers/inject-retrieval.ts

@@ -0,0 +1,50 @@
+/**
+ * UserPromptSubmit handler: inject the top-N matching prior lessons into the prompt.
+ *
+ * Called from UserPromptOrchestrator. Reads the retrieval index, ranks the prompt
+ * against the corpus, prints a `<system-reminder>` block to stdout (Claude Code
+ * prepends UserPromptSubmit hook stdout to the prompt). Fail-closed: any error or
+ * timeout produces empty output, never blocks the prompt.
+ */
+
+import { logDebug, logError } from "../lib/log";
+import { runRetrieval } from "../lib/retrieval";
+import { ensureIndex } from "../lib/retrieval-index";
+import { isEnabled } from "../lib/settings";
+
+const TIMEOUT_MS = 250;
+
+function withTimeout<T>(work: () => T, ms: number): Promise<T | null> {
+  return new Promise((resolve) => {
+    const timer = setTimeout(() => resolve(null), ms);
+    try {
+      const result = work();
+      clearTimeout(timer);
+      resolve(result);
+    } catch (err) {
+      clearTimeout(timer);
+      logError("inject-retrieval", err);
+      resolve(null);
+    }
+  });
+}
+
+export async function injectRetrieval(prompt: string): Promise<void> {
+  if (!prompt?.trim()) return;
+  if (!isEnabled("learningInjection")) return;
+
+  const result = await withTimeout(() => {
+    const index = ensureIndex();
+    if (index.corpusSize === 0) return null;
+    return runRetrieval(prompt, index, process.cwd());
+  }, TIMEOUT_MS);
+
+  if (!result?.reminder) return;
+
+  logDebug(
+    "inject-retrieval",
+    `injected ${result.matches.length} matches; top score=${result.matches[0]?.confidence.toFixed(3)}`
+  );
+
+  process.stdout.write(`${result.reminder}\n`);
+}

package/src/hooks/handlers/project-touch.ts

@@ -0,0 +1,39 @@
+/**
+ * Stop handler: when cwd resolves to an active registered project, bump its
+ * `updated` timestamp and optionally capture a handoff from the last assistant
+ * message. Otherwise no-op (parent-dir browse mode, unregistered cwd,
+ * paused/complete/archived projects all fall through cleanly).
+ *
+ * The plan calls for the JSONs to never go stale silently — this is the
+ * mechanism. CLI invocation is for explicit edits; this handler is for the
+ * "you were just working in this project" auto-bump.
+ */
+
+import { logDebug, logError } from "../lib/log";
+import { readAllProjects, resolveProjectFromCwd, writeProject } from "../lib/projects";
+import { extractHandoff } from "../lib/work-tracking";
+
+const HANDOFF_CAP = 300;
+
+export async function projectTouch(lastAssistantMessage?: string): Promise<void> {
+  try {
+    const projects = readAllProjects();
+    if (projects.length === 0) return;
+
+    const resolved = resolveProjectFromCwd(process.cwd(), projects);
+    if (!resolved) return;
+    if (resolved.status !== "active") return;
+
+    resolved.updated = new Date().toISOString();
+
+    if (lastAssistantMessage?.trim()) {
+      const handoff = extractHandoff(lastAssistantMessage).slice(0, HANDOFF_CAP);
+      if (handoff) resolved.handoff = handoff;
+    }
+
+    writeProject(resolved);
+    logDebug("project-touch", `bumped ${resolved.name} (${resolved.path})`);
+  } catch (err) {
+    logError("project-touch", err);
+  }
+}

package/src/hooks/lib/context.ts

@@ -10,6 +10,7 @@ import { parse } from "./frontmatter";
 import { readFailures, readLearnings } from "./learning-store";
 import { loadOpinionContext } from "./opinions";
 import { paths } from "./paths";
+import { loadActiveProjectsContext } from "./projects";
 import { loadRecentNotes } from "./relationship";
 import { readSessionNames } from "./session-names";
 import * as settings from "./settings";
@@ -301,7 +302,7 @@ export function loadSignalTrends(): string {
 export function loadProjectHistoryContext(): string {
   try {
     const cwd = process.cwd();
-    const entries = readProjectHistory(cwd, 15);
+    const entries = readProjectHistory(cwd, 3);
     if (entries.length === 0) return "";
 
     const lines: string[] = ["## This Project — Session History"];
@@ -351,13 +352,9 @@ export function loadSessionIntelligence(): string {
         }
         lines.push("→ These are directly relevant to your current work.");
       }
-      if (other.length > 0) {
-        lines.push("");
-        lines.push(`**Open threads — other projects (${other.length}):**`);
-        for (const t of other) {
-          lines.push(`- ${t.title} (opened ${t.opened})`);
-        }
-      }
+      // Cross-project threads intentionally omitted — they were noise in 90%+ of sessions.
+      // To surface them on demand, add a `threads` slash-command or a flag in pal-settings.
+      void other;
     }
 
     // Rating Trend
@@ -455,6 +452,9 @@ export function buildSystemReminder(): string {
   const projectHistory = settings.isEnabled("projectHistory")
     ? loadProjectHistoryContext()
     : "";
+  const activeProjects = settings.isEnabled("projects")
+    ? loadActiveProjectsContext()
+    : "";
   const trends = settings.isEnabled("signalTrends") ? loadSignalTrends() : "";
   const failures = settings.isEnabled("failurePatterns") ? loadFailurePatterns() : "";
   const synthesis = settings.isEnabled("synthesis") ? loadSynthesisRecommendations() : "";
@@ -472,6 +472,7 @@ export function buildSystemReminder(): string {
   if (opinions) parts.push(opinions);
   if (intelligence) parts.push(intelligence);
   if (relationship) parts.push(relationship);
+  if (activeProjects) parts.push(activeProjects);
   if (projectHistory) parts.push(projectHistory);
   if (digest) parts.push(digest);
   if (synthesis) parts.push(synthesis);

package/src/hooks/lib/paths.ts

@@ -49,6 +49,8 @@ export const paths = {
   relationship: () => ensureDir(home("memory", "relationship")),
   entities: () => ensureDir(home("memory", "entities")),
   failures: () => ensureDir(home("memory", "learning", "failures")),
+  retrievalIndex: () => home("memory", "learning", ".retrieval-index.json"),
+  progress: () => ensureDir(home("memory", "state", "progress")),
   projectHistory: () => ensureDir(home("memory", "projects")),
   sessionLearning: () => ensureDir(home("memory", "learning", "session")),
   synthesis: () => ensureDir(home("memory", "learning", "synthesis")),

package/src/hooks/lib/projects.ts

@@ -0,0 +1,270 @@
+/**
+ * Projects — registry of user-curated projects with auto-managed state.
+ *
+ * Each project lives in `~/.pal/memory/state/progress/{slug}.json`. The CLI
+ * (`src/tools/agent/project.ts`) is the user/AI-facing writer; the Stop hook
+ * auto-touches `updated` when cwd resolves into a registered project.
+ *
+ * Replaces the hand-edited `~/.pal/telos/PROJECTS.md` — see plan
+ * `~/.claude/plans/clever-frolicking-harp.md` for context.
+ */
+
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { basename, dirname, parse as parsePath, resolve, sep } from "node:path";
+import { paths } from "./paths";
+
+export type ProjectStatus = "active" | "paused" | "complete" | "archived";
+
+export interface Decision {
+  ts: string;
+  decision: string;
+  rationale: string;
+}
+
+export interface ProjectProgress {
+  name: string;
+  path: string;
+  status: ProjectStatus;
+  created: string;
+  updated: string;
+  facts?: string[];
+  objectives?: string[];
+  next_steps?: string[];
+  blockers?: string[];
+  handoff?: string;
+  decisions?: Decision[];
+}
+
+export const PROJECT_STALE_DAYS_DEFAULT = 14;
+
+const PROJECT_MARKERS = [
+  ".git",
+  "package.json",
+  "pyproject.toml",
+  "Cargo.toml",
+  "go.mod",
+  "deno.json",
+  "Gemfile",
+];
+
+function progressDir(): string {
+  const dir = paths.progress();
+  mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+function progressFile(slug: string): string {
+  return resolve(progressDir(), `${slug}.json`);
+}
+
+/**
+ * Compute the default project slug from a cwd.
+ *
+ * Returns the FULL last path segment, lowercased, with non-[a-z0-9_-] chars
+ * collapsed to a single hyphen. Critically: never split on `-` or any
+ * separator within the basename. `/repos/portable-agent-layer` →
+ * `portable-agent-layer`, NOT `layer`.
+ */
+export function defaultSlug(cwd: string): string {
+  const base = basename(resolve(cwd));
+  const cleaned = base
+    .toLowerCase()
+    .replace(/[^a-z0-9_-]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  return cleaned || "unnamed";
+}
+
+/**
+ * Heuristic: does this directory look like a project root (has a project marker)?
+ * Used by the SessionStart loader to hint the AI when an unregistered cwd is
+ * project-shaped, so registration can be suggested in conversation.
+ */
+export function looksLikeProjectRoot(cwd: string): boolean {
+  const cwdAbs = resolve(cwd);
+  return PROJECT_MARKERS.some((marker) => existsSync(resolve(cwdAbs, marker)));
+}
+
+export function readAllProjects(): ProjectProgress[] {
+  const dir = progressDir();
+  if (!existsSync(dir)) return [];
+  const out: ProjectProgress[] = [];
+  for (const file of readdirSync(dir).filter((f) => f.endsWith(".json"))) {
+    try {
+      const parsed = JSON.parse(
+        readFileSync(resolve(dir, file), "utf-8")
+      ) as ProjectProgress;
+      if (parsed?.name && parsed?.path && parsed?.status) out.push(parsed);
+    } catch {
+      /* skip malformed */
+    }
+  }
+  return out;
+}
+
+export function readProject(name: string): ProjectProgress | null {
+  const file = progressFile(name);
+  if (!existsSync(file)) return null;
+  try {
+    return JSON.parse(readFileSync(file, "utf-8")) as ProjectProgress;
+  } catch {
+    return null;
+  }
+}
+
+export function writeProject(p: ProjectProgress): void {
+  writeFileSync(progressFile(p.name), `${JSON.stringify(p, null, 2)}\n`, "utf-8");
+}
+
+export function deleteProject(name: string): boolean {
+  const file = progressFile(name);
+  if (!existsSync(file)) return false;
+  unlinkSync(file);
+  return true;
+}
+
+/**
+ * Resolve `cwd` to the registered project that contains it, if any.
+ *
+ * Rules:
+ *  - Exact-match registered path → that project.
+ *  - Descendant of exactly one registered path → that project.
+ *  - Descendant of multiple (nested) → longest registered path wins.
+ *  - Ancestor of registered project (parent-dir browse mode) → null.
+ *  - Unrelated cwd → null.
+ *
+ * The parent-dir case is the load-bearing one: opening `/repos/` (an ancestor
+ * of multiple registered repos) MUST return null so the Stop-hook auto-write
+ * doesn't ambiguously bump one of N children.
+ */
+export function resolveProjectFromCwd(
+  cwd: string,
+  projects: ProjectProgress[]
+): ProjectProgress | null {
+  const cwdAbs = resolve(cwd);
+  const matches = projects.filter((p) => {
+    const projAbs = resolve(p.path);
+    return cwdAbs === projAbs || cwdAbs.startsWith(projAbs + sep);
+  });
+  if (matches.length === 0) return null;
+  matches.sort((a, b) => b.path.length - a.path.length);
+  return matches[0];
+}
+
+export function isStale(
+  p: ProjectProgress,
+  thresholdDays = PROJECT_STALE_DAYS_DEFAULT
+): boolean {
+  if (!p.updated) return false;
+  const age = Date.now() - new Date(p.updated).getTime();
+  if (!Number.isFinite(age) || age < 0) return false;
+  return age > thresholdDays * 86_400_000;
+}
+
+/**
+ * Walk up from `cwd` looking for the nearest dir with a project marker.
+ * Returns absolute path or null. Bounded at 12 levels to avoid runaway walks.
+ */
+export function findProjectRoot(cwd: string): string | null {
+  let dir = resolve(cwd);
+  const fsRoot = parsePath(dir).root;
+  for (let i = 0; i < 12; i++) {
+    if (looksLikeProjectRoot(dir)) return dir;
+    if (dir === fsRoot) return null;
+    dir = dirname(dir);
+  }
+  return null;
+}
+
+function formatAgo(ts: string): string {
+  if (!ts) return "?";
+  const age = Date.now() - new Date(ts).getTime();
+  if (!Number.isFinite(age)) return "?";
+  if (age < 60_000) return "just now";
+  if (age < 3_600_000) return `${Math.floor(age / 60_000)}m ago`;
+  if (age < 86_400_000) return "today";
+  const days = Math.floor(age / 86_400_000);
+  if (days < 30) return `${days}d ago`;
+  return `${Math.floor(days / 30)}mo ago`;
+}
+
+const MAX_INLINE_BULLETS = 3;
+
+/**
+ * Format the SessionStart "Active Projects" section.
+ *
+ * Output is empty when there's nothing to say (no active/paused projects AND
+ * no project-shaped unregistered cwd). When non-empty, includes:
+ *  - A `## Active Projects` block listing every active/paused project
+ *  - Full Objectives / Next / Blockers detail ONLY for the cwd-resolved project
+ *    (`→ here`); every other project renders as a one-liner with next/blocker
+ *    counts, to keep the section bounded as the project corpus grows
+ *  - A `⚠ stale` flag for projects updated > threshold days ago
+ *  - A trailing AI-visible hint when cwd resolves to no project but
+ *    `findProjectRoot(cwd)` reveals a project-shaped ancestor
+ */
+export function loadActiveProjectsContext(cwd: string = process.cwd()): string {
+  const all = readAllProjects();
+  const visible = all.filter((p) => p.status === "active" || p.status === "paused");
+  const resolved = resolveProjectFromCwd(cwd, visible);
+  const projectRoot = findProjectRoot(cwd);
+  const alreadyRegistered =
+    projectRoot !== null && all.some((p) => resolve(p.path) === projectRoot);
+  const showHint = resolved === null && projectRoot !== null && !alreadyRegistered;
+
+  if (visible.length === 0 && !showHint) return "";
+
+  const lines: string[] = [];
+
+  if (visible.length > 0) {
+    lines.push("## Active Projects");
+    lines.push("");
+    const sorted = [...visible].sort((a, b) => b.updated.localeCompare(a.updated));
+    for (const p of sorted) {
+      const ago = formatAgo(p.updated);
+      const stale = isStale(p) ? " ⚠ stale" : "";
+      const isResolved = resolved !== null && p.name === resolved.name;
+      const here = isResolved ? " → here" : "";
+      const statusPrefix = p.status === "paused" ? "paused, " : "";
+
+      if (isResolved) {
+        lines.push(`- **${p.name}** (${statusPrefix}${ago})${stale}${here}`);
+        if (p.facts?.length) {
+          lines.push(`  Facts: ${p.facts.slice(0, MAX_INLINE_BULLETS).join("; ")}`);
+        }
+        if (p.objectives?.length) {
+          lines.push(
+            `  Objectives: ${p.objectives.slice(0, MAX_INLINE_BULLETS).join("; ")}`
+          );
+        }
+        if (p.next_steps?.length) {
+          lines.push(`  Next: ${p.next_steps.slice(0, MAX_INLINE_BULLETS).join("; ")}`);
+        }
+        if (p.blockers?.length) {
+          lines.push(`  Blockers: ${p.blockers.slice(0, MAX_INLINE_BULLETS).join("; ")}`);
+        }
+      } else {
+        const counts: string[] = [];
+        if (p.next_steps?.length) counts.push(`${p.next_steps.length} next`);
+        if (p.blockers?.length) counts.push(`${p.blockers.length} blockers`);
+        const countsSuffix = counts.length > 0 ? ` — ${counts.join(", ")}` : "";
+        lines.push(`- **${p.name}** (${statusPrefix}${ago})${countsSuffix}${stale}`);
+      }
+    }
+  }
+
+  if (showHint) {
+    if (visible.length > 0) lines.push("");
+    lines.push(
+      `💡 \`${projectRoot}\` looks like a project but isn't registered. If substantive work starts here, suggest registering it via \`bun ~/.pal/tools/project.ts create\`.`
+    );
+  }
+
+  return lines.join("\n");
+}

package/src/hooks/lib/retrieval-index.ts

@@ -0,0 +1,223 @@
+/**
+ * Retrieval index — single JSON file with per-doc term-frequency vectors and a global
+ * document-frequency table over the failures + wisdom-frames corpus.
+ *
+ * Built once on first read, rebuilt in the background when source dirs change.
+ * Read by the UserPromptSubmit retrieval handler; written by ensureIndex (sync bootstrap)
+ * or by a detached `bun src/hooks/lib/retrieval-index.ts --rebuild` invocation.
+ */
+
+import { spawn } from "node:child_process";
+import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { SIMILARITY_THRESHOLD } from "./graduation";
+import { readFailures } from "./learning-store";
+import { logDebug, logError } from "./log";
+import { palPkg, paths } from "./paths";
+import { similarity, tokenize } from "./text-similarity";
+import { readFramesForRetrieval } from "./wisdom";
+
+export const INDEX_VERSION = 1;
+
+export interface IndexedDoc {
+  id: string;
+  source: "failure" | "wisdom";
+  path: string;
+  rating: number;
+  ts: string;
+  tf: Record<string, number>;
+  len: number;
+  displayPrinciple: string;
+  displayContext: string;
+}
+
+export interface RetrievalIndex {
+  version: number;
+  builtAt: string;
+  corpusSize: number;
+  df: Record<string, number>;
+  docs: IndexedDoc[];
+}
+
+const CONVERSATION_BLOCK_RE = /## Conversation Summary[\s\S]*?(?=\n## |\n# |$)/g;
+
+/** Read first 800 chars of body excluding the Conversation Summary block. */
+function failureBodyExcerpt(content: string): string {
+  const stripped = content.replace(CONVERSATION_BLOCK_RE, "");
+  const bodyStart = stripped.indexOf("\n---\n");
+  const body = bodyStart >= 0 ? stripped.slice(bodyStart + 5) : stripped;
+  return body.slice(0, 800);
+}
+
+function buildTermFreq(tokens: string[]): { tf: Record<string, number>; len: number } {
+  const tf: Record<string, number> = {};
+  for (const t of tokens) tf[t] = (tf[t] ?? 0) + 1;
+  return { tf, len: tokens.length };
+}
+
+/** Concatenate weighted fields, tokenize, count tf. Weights produce N-fold token repetition. */
+function buildDocTokens(parts: { text: string; weight: number }[]): string[] {
+  const all: string[] = [];
+  for (const { text, weight } of parts) {
+    if (!text) continue;
+    const toks = tokenize(text);
+    for (let i = 0; i < weight; i++) all.push(...toks);
+  }
+  return all;
+}
+
+export function buildIndex(): RetrievalIndex {
+  const docs: IndexedDoc[] = [];
+  const df: Record<string, number> = {};
+
+  const frames = readFramesForRetrieval();
+  const framePrinciples = frames.map((fr) => fr.principle).filter(Boolean);
+
+  // Failures — skip captures whose principle has already graduated into a frame.
+  // Reuses graduation.ts's threshold so the same Dice-similarity rule that promotes
+  // patterns to wisdom is the one that hides their source captures from retrieval.
+  const allFailures = readFailures(paths.failures());
+  const failures = allFailures.filter((f) => {
+    if (!f.principle) return true;
+    return !framePrinciples.some(
+      (fp) => similarity(f.principle, fp) >= SIMILARITY_THRESHOLD
+    );
+  });
+  const skipped = allFailures.length - failures.length;
+  if (skipped > 0)
+    logDebug(
+      "retrieval-index",
+      `dedup: skipped ${skipped} captures already represented by graduated frames`
+    );
+
+  for (const f of failures) {
+    let raw = "";
+    try {
+      raw = readFileSync(f.path, "utf-8");
+    } catch {
+      continue;
+    }
+    const body = failureBodyExcerpt(raw);
+    const tokens = buildDocTokens([
+      { text: f.principle, weight: 3 },
+      { text: f.context, weight: 2 },
+      { text: body, weight: 1 },
+    ]);
+    if (tokens.length === 0) continue;
+    const { tf, len } = buildTermFreq(tokens);
+    for (const term of Object.keys(tf)) df[term] = (df[term] ?? 0) + 1;
+    docs.push({
+      id: f.slug,
+      source: "failure",
+      path: f.path,
+      rating: f.rating,
+      ts: f.ts,
+      tf,
+      len,
+      displayPrinciple: f.principle,
+      displayContext: f.context,
+    });
+  }
+
+  // Wisdom frames (each principle = pseudo-doc)
+  for (const fr of frames) {
+    const tokens = buildDocTokens([
+      { text: fr.principle, weight: 3 },
+      { text: fr.domain, weight: 2 },
+      { text: fr.body, weight: 1 },
+    ]);
+    if (tokens.length === 0) continue;
+    const { tf, len } = buildTermFreq(tokens);
+    for (const term of Object.keys(tf)) df[term] = (df[term] ?? 0) + 1;
+    docs.push({
+      id: `${fr.domain}:${fr.principle.slice(0, 40)}`,
+      source: "wisdom",
+      path: resolve(paths.wisdom(), `${fr.domain}.md`),
+      rating: fr.confidence,
+      ts: "",
+      tf,
+      len,
+      displayPrinciple: fr.principle,
+      displayContext: fr.domain,
+    });
+  }
+
+  return {
+    version: INDEX_VERSION,
+    builtAt: new Date().toISOString(),
+    corpusSize: docs.length,
+    df,
+    docs,
+  };
+}
+
+export function writeIndex(index: RetrievalIndex): void {
+  writeFileSync(paths.retrievalIndex(), JSON.stringify(index));
+}
+
+export function readIndex(): RetrievalIndex | null {
+  const p = paths.retrievalIndex();
+  if (!existsSync(p)) return null;
+  try {
+    const parsed = JSON.parse(readFileSync(p, "utf-8")) as RetrievalIndex;
+    if (parsed?.version !== INDEX_VERSION) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+/** True if any source directory was modified after the index was built. */
+export function isStale(index: RetrievalIndex): boolean {
+  try {
+    const builtMs = new Date(index.builtAt).getTime();
+    for (const dir of [paths.failures(), paths.wisdom()]) {
+      if (!existsSync(dir)) continue;
+      if (statSync(dir).mtimeMs > builtMs) return true;
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}
+
+/** Detached background rebuild — fire and forget, never throws. */
+export function spawnBackgroundRebuild(): void {
+  try {
+    const script = resolve(palPkg(), "src", "hooks", "lib", "retrieval-index.ts");
+    const child = spawn("bun", ["run", script, "--rebuild"], {
+      detached: true,
+      stdio: "ignore",
+      env: process.env,
+    });
+    child.unref();
+  } catch (err) {
+    logError("retrieval-index:spawn", err);
+  }
+}
+
+/** Return the freshest usable index. Builds synchronously on first run; otherwise
+ *  returns the cached index and triggers a background rebuild if the corpus moved. */
+export function ensureIndex(): RetrievalIndex {
+  const existing = readIndex();
+  if (!existing) {
+    logDebug("retrieval-index", "no index — building synchronously");
+    const fresh = buildIndex();
+    writeIndex(fresh);
+    return fresh;
+  }
+  if (isStale(existing)) {
+    logDebug("retrieval-index", "stale index — using cached, rebuilding in background");
+    spawnBackgroundRebuild();
+  }
+  return existing;
+}
+
+// CLI entry — `bun run src/hooks/lib/retrieval-index.ts --rebuild`
+if (import.meta.main) {
+  const fresh = buildIndex();
+  writeIndex(fresh);
+  console.log(
+    `built retrieval index — ${fresh.corpusSize} docs, ${Object.keys(fresh.df).length} terms`
+  );
+}