portable-agent-layer 0.32.0 → 0.33.0

package/src/hooks/lib/retrieval.ts

@@ -0,0 +1,170 @@
+/**
+ * Retrieval ranker — score the indexed corpus against a fresh prompt and produce a
+ * ≤500-char `<system-reminder>` block of the top-N matching principles.
+ *
+ * Algorithm: IDF-weighted term overlap with capped tf, sqrt length-norm, age decay,
+ * cwd-fingerprint scope boost, confidence threshold.
+ */
+
+import { basename } from "node:path";
+import type { IndexedDoc, RetrievalIndex } from "./retrieval-index";
+import { extractKeywords } from "./text-similarity";
+
+const TF_CAP = 3;
+const TIME_HALF_LIFE_DAYS = 90;
+const SCOPE_BOOST = 1.5;
+const CONFIDENCE_THRESHOLD = 0.18;
+const MAX_MATCHES = 2;
+const MAX_REMINDER_BYTES = 500;
+const PRINCIPLE_TRUNC = 200;
+
+export interface ScoredDoc {
+  doc: IndexedDoc;
+  score: number;
+  confidence: number;
+  scopeMatch: boolean;
+}
+
+function idf(term: string, df: Record<string, number>, N: number): number {
+  return Math.log((N + 1) / ((df[term] ?? 0) + 1)) + 1;
+}
+
+function ageDecay(ts: string): number {
+  if (!ts) return 1;
+  const age = Date.now() - new Date(ts).getTime();
+  if (!Number.isFinite(age) || age <= 0) return 1;
+  const days = age / 86_400_000;
+  return Math.exp(-days / TIME_HALF_LIFE_DAYS);
+}
+
+/** Raw IDF-weighted overlap score, capped tf, sqrt-length-normalized. */
+export function scoreDoc(
+  queryTerms: Set<string>,
+  doc: IndexedDoc,
+  df: Record<string, number>,
+  N: number
+): number {
+  if (doc.len === 0) return 0;
+  let s = 0;
+  for (const t of queryTerms) {
+    const tf = doc.tf[t];
+    if (!tf) continue;
+    s += idf(t, df, N) * Math.min(tf, TF_CAP);
+  }
+  if (s === 0) return 0;
+  return s / Math.sqrt(doc.len);
+}
+
+/** Upper bound: query treated as a doc with tf=1 per unique term. */
+export function selfScore(
+  queryTerms: Set<string>,
+  df: Record<string, number>,
+  N: number
+): number {
+  if (queryTerms.size === 0) return 0;
+  let s = 0;
+  for (const t of queryTerms) s += idf(t, df, N);
+  return s / Math.sqrt(queryTerms.size);
+}
+
+/** Does any token of the doc's tf map match the cwd basename (lowercased)? */
+function scopeMatches(doc: IndexedDoc, scopeKey: string): boolean {
+  if (!scopeKey) return false;
+  return doc.tf[scopeKey] !== undefined;
+}
+
+/** Rank the index against `query`. Returns docs above the confidence threshold,
+ *  scope-boosted and age-decayed, sorted high-to-low, capped at MAX_MATCHES. */
+export function rank(query: string, index: RetrievalIndex, cwd: string): ScoredDoc[] {
+  const queryTerms = extractKeywords(query);
+  if (queryTerms.size === 0) return [];
+
+  const N = Math.max(index.corpusSize, 1);
+  const self = selfScore(queryTerms, index.df, N);
+  if (self === 0) return [];
+
+  const scopeKey = basename(cwd)
+    .toLowerCase()
+    .replace(/[^a-z0-9-]/g, "");
+  const scopeTokens = scopeKey ? extractKeywords(scopeKey) : new Set<string>();
+
+  const scored: ScoredDoc[] = [];
+  for (const doc of index.docs) {
+    const raw = scoreDoc(queryTerms, doc, index.df, N);
+    if (raw === 0) continue;
+    const scopeMatch = [...scopeTokens].some((t) => scopeMatches(doc, t));
+    const boosted = raw * (scopeMatch ? SCOPE_BOOST : 1) * ageDecay(doc.ts);
+    const confidence = boosted / self;
+    if (confidence < CONFIDENCE_THRESHOLD) continue;
+    scored.push({ doc, score: boosted, confidence, scopeMatch });
+  }
+
+  scored.sort((a, b) => b.score - a.score);
+  return scored.slice(0, MAX_MATCHES);
+}
+
+function truncate(text: string, max: number): string {
+  if (text.length <= max) return text;
+  return `${text.slice(0, max - 1).trimEnd()}…`;
+}
+
+function formatAgo(ts: string): string {
+  if (!ts) return "";
+  const age = Date.now() - new Date(ts).getTime();
+  if (!Number.isFinite(age) || age <= 0) return "";
+  const days = Math.floor(age / 86_400_000);
+  if (days < 1) return "today";
+  if (days < 30) return `${days}d ago`;
+  const months = Math.floor(days / 30);
+  return `${months}mo ago`;
+}
+
+function formatLine(s: ScoredDoc): string {
+  const tag =
+    s.doc.source === "wisdom"
+      ? `[${s.doc.displayContext}]`
+      : s.scopeMatch
+        ? "[project]"
+        : "[global]";
+  const text = s.doc.displayPrinciple || s.doc.displayContext || "";
+  const principle = truncate(text, PRINCIPLE_TRUNC);
+  if (s.doc.source === "wisdom") {
+    return `- ${tag} ${principle} (CRYSTAL ${s.doc.rating}%)`;
+  }
+  const ago = formatAgo(s.doc.ts);
+  const meta = ago ? `rating ${s.doc.rating}/10, ${ago}` : `rating ${s.doc.rating}/10`;
+  return `- ${tag} ${principle} (${meta})`;
+}
+
+/** Build the `<system-reminder>` block. Drops lowest-ranked lines until ≤500 bytes. */
+export function formatReminder(matches: ScoredDoc[]): string {
+  if (matches.length === 0) return "";
+  let lines = matches.map(formatLine);
+
+  let block = render(lines);
+  while (block.length > MAX_REMINDER_BYTES && lines.length > 1) {
+    lines = lines.slice(0, -1);
+    block = render(lines);
+  }
+  if (block.length > MAX_REMINDER_BYTES) return "";
+  return block;
+}
+
+function render(lines: string[]): string {
+  return [
+    "<system-reminder>",
+    "**Relevant prior lessons** (matched on your prompt):",
+    ...lines,
+    "</system-reminder>",
+  ].join("\n");
+}
+
+/** End-to-end retrieval: takes a query + index, returns the formatted reminder string. */
+export function runRetrieval(
+  query: string,
+  index: RetrievalIndex,
+  cwd: string
+): { reminder: string; matches: ScoredDoc[] } {
+  const matches = rank(query, index, cwd);
+  return { reminder: formatReminder(matches), matches };
+}

package/src/hooks/lib/security.ts

@@ -36,6 +36,7 @@ export const HOOK_MANAGED_FILES = [
   "pal-settings.json",
   "skill-index.json",
   "algorithm-reflections.jsonl",
+  ".retrieval-index.json",
 ];
 
 /** Hook-managed directories — AI must not write to or delete from these */
@@ -47,6 +48,7 @@ export const HOOK_MANAGED_DIRS = [
   "memory/relationship",
   "memory/wisdom/state",
   "memory/projects",
+  "memory/state/progress",
 ];
 
 /** Escape a string for use in a RegExp */

package/src/hooks/lib/stop.ts

@@ -5,9 +5,11 @@
 
 import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
 import { resolve } from "node:path";
+import { autoGraduate } from "../handlers/auto-graduate";
 import { autoBackup } from "../handlers/backup";
 import { notifyDesktop } from "../handlers/desktop-notify";
 import { captureFailure } from "../handlers/failure";
+import { projectTouch } from "../handlers/project-touch";
 import { checkReflectTrigger } from "../handlers/reflect-trigger";
 import { checkSelfModelTrigger } from "../handlers/self-model-trigger";
 import { captureSessionIntelligence } from "../handlers/session-intelligence";
@@ -38,7 +40,9 @@ export async function runStopHandlers(
   // Cache last assistant response (session-scoped)
   cacheLastResponse(messages, options.lastAssistantMessage, options.sessionId);
 
-  // Run all handlers concurrently (manual wisdom extraction only - no automatic extraction)
+  // Run all handlers concurrently. Auto-graduate is idempotent (24h TTL +
+  // state-dedup + content-dedup) so it's safe to fire on every Stop.
+  // project-touch only fires when cwd resolves to an active registered project.
   const results = await Promise.allSettled([
     captureWorkSession(transcript, options.sessionId),
     resetTab(),
@@ -49,6 +53,8 @@ export async function runStopHandlers(
     checkReflectTrigger(),
     checkSelfModelTrigger(),
     runSynthesis(),
+    autoGraduate(),
+    projectTouch(options.lastAssistantMessage),
     notifyDesktop(options.sessionId),
   ]);
 
@@ -62,6 +68,8 @@ export async function runStopHandlers(
     "reflect-trigger",
     "self-model-trigger",
     "synthesis",
+    "auto-graduate",
+    "project-touch",
     "desktop-notify",
   ];
   for (let i = 0; i < results.length; i++) {

package/src/hooks/lib/text-similarity.ts

@@ -126,16 +126,20 @@ function stem(word: string): string {
   return stemOnce(stemOnce(word));
 }
 
+/** Lowercase → strip punctuation → split → drop stop-words and shorts → stem → drop shorts.
+ *  Returns the token stream WITH duplicates (needed for term-frequency counting). */
+export function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, " ")
+    .split(/\s+/)
+    .filter((w) => w.length > 2 && !STOP_WORDS.has(w))
+    .map((w) => stem(w))
+    .filter((w) => w.length > 2);
+}
+
 export function extractKeywords(text: string): Set<string> {
-  return new Set(
-    text
-      .toLowerCase()
-      .replace(/[^a-z0-9\s-]/g, " ")
-      .split(/\s+/)
-      .filter((w) => w.length > 2 && !STOP_WORDS.has(w))
-      .map((w) => stem(w))
-      .filter((w) => w.length > 2)
-  );
+  return new Set(tokenize(text));
 }
 
 /** Dice coefficient on keyword sets. Returns 0-1. */

package/src/hooks/lib/wisdom.ts

@@ -8,9 +8,163 @@
  * not auto-extracted from transcripts.
  */
 
-import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { resolve } from "node:path";
 import { paths } from "./paths";
+import { similarity } from "./text-similarity";
+
+/** Dice-similarity floor for "principle already represented" — matches graduation.ts. */
+const PRINCIPLE_DEDUP_THRESHOLD = 0.3;
+const PRINCIPLES_PLACEHOLDER =
+  "*No crystallized principles yet. Observations accumulating.*";
+
+function today(): string {
+  return new Date().toISOString().slice(0, 10);
+}
+
+function scaffoldFrame(domain: string): string {
+  const t = today();
+  return `# Frame: ${domain.charAt(0).toUpperCase() + domain.slice(1)}
+
+## Meta
+- **Domain:** ${domain}
+- **Observation Count:** 0
+- **Last Updated:** ${t}
+
+---
+
+## Core Principles
+
+${PRINCIPLES_PLACEHOLDER}
+
+---
+
+## Contextual Rules
+
+*None yet.*
+
+---
+
+## Anti-Patterns
+
+*None yet.*
+
+---
+
+## Evolution Log
+- ${t}: Frame scaffolded by auto-graduate
+`;
+}
+
+/** Existing CRYSTAL principle texts in the frame content (both v4 heading + legacy bullet). */
+function existingCrystalPrinciples(content: string): string[] {
+  const out: string[] = [];
+  for (const m of content.matchAll(/^### (.+?) \[CRYSTAL:\s*\d+%\]/gm)) {
+    if (m[1]) out.push(m[1].trim());
+  }
+  for (const line of content.split("\n")) {
+    const m = line.match(/^\s*-\s*(.+?)\s*\[CRYSTAL:\s*\d+%\]\s*$/);
+    if (m?.[1]) out.push(m[1].trim());
+  }
+  return out;
+}
+
+export interface PromoteCrystalResult {
+  domain: string;
+  principle: string;
+  confidence: number;
+  framePath: string;
+  /** "duplicate" → a Dice-similar CRYSTAL line already existed; nothing written. */
+  skipped: "duplicate" | null;
+}
+
+/**
+ * Idempotently promote a graduated principle to a wisdom-frame CRYSTAL line.
+ *
+ * Content-dedup: if any existing CRYSTAL line in the frame is Dice-similar
+ * (≥0.3) to `principle`, the call is a no-op. Combined with the auto-graduate
+ * handler's TTL guard and state-dedup, this means N rapid calls with the same
+ * input produce ≤1 file write — the property the past failed attempt missed.
+ */
+export function promoteCrystal(
+  domain: string,
+  principle: string,
+  confidence: number
+): PromoteCrystalResult {
+  const framesDir = paths.wisdom();
+  mkdirSync(framesDir, { recursive: true });
+  const framePath = resolve(framesDir, `${domain}.md`);
+
+  if (!existsSync(framePath)) {
+    writeFileSync(framePath, scaffoldFrame(domain));
+  }
+
+  let content = readFileSync(framePath, "utf-8");
+
+  for (const existing of existingCrystalPrinciples(content)) {
+    if (similarity(principle, existing) >= PRINCIPLE_DEDUP_THRESHOLD) {
+      return { domain, principle, confidence, framePath, skipped: "duplicate" };
+    }
+  }
+
+  const newLine = `### ${principle} [CRYSTAL: ${confidence}%]`;
+
+  if (content.includes(PRINCIPLES_PLACEHOLDER)) {
+    content = content.replace(PRINCIPLES_PLACEHOLDER, newLine);
+  } else {
+    content = content.replace(/(## Core Principles\n+)/, `$1${newLine}\n\n`);
+  }
+
+  content = content.replace(/(\*\*Last Updated:\*\*\s*)\S+/, `$1${today()}`);
+  const evolutionEntry = `- ${today()}: Auto-promoted to CRYSTAL ${confidence}% — ${principle}`;
+  content = content.replace(/(## Evolution Log\n)/, `$1${evolutionEntry}\n`);
+
+  writeFileSync(framePath, content);
+  return { domain, principle, confidence, framePath, skipped: null };
+}
+
+export interface FrameDoc {
+  domain: string;
+  principle: string;
+  body: string;
+  confidence: number;
+}
+
+/** Extract every CRYSTAL principle as a structured doc (domain + principle + surrounding body + confidence).
+ *  Body is the first ~600 chars of the frame for ranking context.
+ *  Used by the retrieval indexer; readFramePrinciples remains for SessionStart formatting. */
+export function readFramesForRetrieval(): FrameDoc[] {
+  const framesDir = paths.wisdom();
+  const docs: FrameDoc[] = [];
+
+  if (!existsSync(framesDir)) return docs;
+
+  for (const file of readdirSync(framesDir).filter((f) => f.endsWith(".md"))) {
+    const domain = file.replace(".md", "");
+    const content = readFileSync(resolve(framesDir, file), "utf-8");
+    const body = content.slice(0, 600);
+
+    for (const match of content.matchAll(/^### (.+?) \[CRYSTAL:\s*(\d+)%\]/gm)) {
+      const name = match[1]?.trim();
+      const pct = parseInt(match[2] ?? "", 10);
+      if (name && Number.isFinite(pct) && pct >= 85) {
+        docs.push({ domain, principle: name, body, confidence: pct });
+      }
+    }
+
+    for (const line of content.split("\n")) {
+      const m = line.match(/^\s*-\s*(.+?)\s*\[CRYSTAL:\s*(\d+)%\]\s*$/);
+      if (!m) continue;
+      const name = m[1]?.trim();
+      const pct = parseInt(m[2] ?? "", 10);
+      if (name && Number.isFinite(pct) && pct >= 85) {
+        docs.push({ domain, principle: name, body, confidence: pct });
+      }
+    }
+  }
+
+  return docs;
+}
 
 /** Extract CRYSTAL principles (≥85% confidence) from all frame files */
 export function readFramePrinciples(): string[] {