pi-fold 1.0.2 → 1.0.4

package/extensions/lib/tool-surface.ts

@@ -29,16 +29,28 @@ export function buildActiveContextTool(input: {
   handler: ToolHandler;
 }) {
   const peekGuidance = input.allowedActions.includes("peek")
-    ? ` Peek is the point read: it returns one fold's index view -- every nested fold's brief in full, plus the head and tail of the raw source with the omitted middle stated -- at any depth, ancestors still collapsed, changing nothing. It returns at most ${input.defaultPeekBytes} bytes unless you widen it with bytes, and it stays in the window like any other tool result. Expand is the durable in-place restoration.`
+    ? ` Peek is the point read: it returns one fold's index view -- every nested fold's brief in full, plus the head and tail of that fold's own span with the omitted middle stated -- for any fold id, ancestors still collapsed, changing nothing. It serves ONE level: raw entries come back exactly, folds nested inside stay placeheld, and peeking a child id you see there returns that child's own bytes, so anything folded is one hop away at any depth. It returns at most ${input.defaultPeekBytes} bytes unless you widen it with bytes, and it stays in the window like any other tool result. Expand is the durable in-place restoration.`
     : "";
   const correctionGuidance = input.allowedActions.includes("rebrief")
     ? " Curation is correctable: rebrief replaces a fold's brief, and reboundary with ids re-cuts a span into exactly one fold, which merges adjacent folds when the span covers several and splits one when the span sits inside it. Reboundary with a single id returns that fold's span to raw."
     : "";
+  // The pin. Protection predates this sentence; the sentence exists because a verb
+  // nobody can discover is a verb nobody uses.
+  const pinGuidance = input.allowedActions.includes("protect")
+    ? " Protect is the pin: protect with ids holds folds or entries against automatic folding, consolidation and refolding until unprotect releases them, so an expanded span you still need stays expanded. Protect reveals nothing on its own: a folded span stays folded, just held. Pinning moves no context bytes, and fold receipts report how much mass you have pinned."
+    : "";
   // Marking is yours, folding is the runtime's. The epoch guidance says how marks
   // behave and stops there: there is no agent-callable commit verb to describe.
   const epochGuidance = input.allowedActions.includes("unmark")
     ? " Fold scheduling is epoch mode: fold records pending marks and moves no context bytes, and the runtime applies every pending mark in one rewrite when the fold event fires. Mark SEVERAL spans in one call: marks carries several {ids, brief} pairs, and one call answers with your whole picture, every span now held plus the unmarked remainder and what share of the non-fresh window it is. Between fold events nothing else in your context changes, so that result is where the picture lives. Mark freely as you work; when to fold is the runtime's to decide, and unmark withdraws a mark you no longer want."
     : "";
+  // The recovery norm. Every placeholder already states the exact expand call, and a
+  // full run answered questions about folded material from memory anyway, inventing
+  // plausible values instead of reopening the fold. The syntax was discoverable; the
+  // norm was not, so the norm is stated where the syntax lives.
+  const recoveryGuidance = input.allowedActions.includes("expand")
+    ? " A fold brief is an index entry, not the source: when a question or task depends on material that is folded, peek or expand that fold and answer from the exact bytes, never from memory of them. Recalling folded detail without reopening it is how specifics get misremembered."
+    : "";
   // The standing facts about how this runtime behaves. They belong HERE, in the tool
   // surface, because the tool surface sits in the stable prefix: it is cached from the
   // first request and costs nothing per request thereafter, forever. The same sentences
@@ -49,8 +61,8 @@ export function buildActiveContextTool(input: {
     name: input.name,
     label: input.label,
     description: input.fullSurface
-      ? `Page, peek, fold, expand, refold, protect, rebrief, or re-cut exact Pi active-context evidence.${peekGuidance}${correctionGuidance}${epochGuidance}${standingGuidance} Mutations persist immediately and affect the next model call inside the same continuing turn; no turn boundary is required. Supplied fold briefs have a hard 1200-character maximum.`
-      : `Use only the configured active-context actions: ${input.allowedActions.join(", ")}.${peekGuidance}${correctionGuidance}${epochGuidance}${standingGuidance} Call fold only by copying the exact eligibleChapter.action returned by status; if status has no eligibleChapter, continue the task without folding. Supplied fold briefs have a hard 1200-character maximum.`,
+      ? `Page, peek, fold, expand, refold, protect, rebrief, or re-cut exact Pi active-context evidence.${peekGuidance}${correctionGuidance}${pinGuidance}${epochGuidance}${recoveryGuidance}${standingGuidance} Mutations persist immediately and affect the next model call inside the same continuing turn; no turn boundary is required. Supplied fold briefs have a hard 1200-character maximum.`
+      : `Use only the configured active-context actions: ${input.allowedActions.join(", ")}.${peekGuidance}${correctionGuidance}${pinGuidance}${epochGuidance}${recoveryGuidance}${standingGuidance} Call fold only by copying the exact eligibleChapter.action returned by status; if status has no eligibleChapter, continue the task without folding. Supplied fold briefs have a hard 1200-character maximum.`,
     parameters: {
       type: "object",
       additionalProperties: false,
@@ -70,7 +82,14 @@ export function buildActiveContextTool(input: {
             required: ["ids"],
             properties: {
               ids: { type: "array", minItems: 1, maxItems: 64, items: { type: "string", minLength: 1 } },
-              brief: { type: "string", minLength: 1, maxLength: input.maxBriefChars },
+              brief: {
+                type: "string",
+                minLength: 1,
+                maxLength: input.maxBriefChars,
+                description: "Write one for every span you mark. You know why the span mattered and " +
+                  "what you will want back from it; a generator reading the span alone does not. " +
+                  "Leaving it out hands the brief to the summarizer rather than skipping it.",
+              },
             },
           },
         },
@@ -108,7 +127,8 @@ export function buildActiveContextCommands(input: {
     },
     {
       name: input.foldName,
-      description: "Losslessly fold a stale context span; works without a main-model request",
+      description: "Losslessly fold a stale context span, or \"commit\" to apply every eligible mark now; " +
+        "works without a main-model request",
       handler: input.foldHandler,
     },
   ];

package/extensions/lib/transcript.ts

@@ -16,16 +16,19 @@ import {
 } from "./canonical.ts";
 import {
   ACTIVE_CONTEXT_POLICY,
-  BYTES_PER_TOKEN_FLOOR,
+  AUTO_FOLD_BLACKLIST_DEFAULT,
   DEFAULT_ACTIVE_CONTEXT_BRAND_NOUN,
   DEFAULT_ACTIVE_CONTEXT_ENTRY_TYPE_PREFIX,
   DEFAULT_ACTIVE_CONTEXT_TOOL_NAME,
   DEFAULT_CONTEXT_WINDOW,
-  READ_ONLY_CONTEXT_ACTIONS_DEFAULT,
-  READ_ONLY_TOOLS_DEFAULT,
+  DEFAULT_THRESHOLDS,
+  PEEK_READ_ONLY_CONTEXT_ACTIONS,
+  servingBudgetTokens,
+  zoneBytes,
 } from "./policy.ts";
 import type {
   ActiveContextSnapshot,
+  ActiveContextThresholds,
   BranchObject,
   CompleteTurn,
   MappedMessage,
@@ -72,23 +75,33 @@ export function leadingCompactionContinuation(messages: unknown[]): CompleteTurn
 }
 
 /**
- * The exact argument surface each read-only active-context action may carry. A call
- * outside this surface is not classified read-only, so its batch never folds.
+ * The exact argument surface each read-only active-context action may carry. A context
+ * call outside this surface is not classified read-only, so its batch never folds
+ * unmarked.
  */
 export const READ_ONLY_CONTEXT_ACTION_ARGUMENTS: Readonly<Record<string, readonly string[]>> = Object.freeze({
   status: Object.freeze(["action", "detail", "offset", "limit"]),
   peek: Object.freeze(["action", "id"]),
 });
 
-export function isReadOnlyContextTool(
+/**
+ * May the ladder fold this completed call's batch without an agent mark?
+ *
+ * Yes for every tool, unless the deployment blacklisted it. The one carve-out is the
+ * active-context tool itself: a mutating context call folding its own batch would be a
+ * rewrite caused by a tool call, so only its read-only actions inside the exact argument
+ * surface above are foldable.
+ */
+export function isAutoFoldableToolCall(
   name: string,
   args?: unknown,
   toolName = DEFAULT_ACTIVE_CONTEXT_TOOL_NAME,
-  readOnlyTools: ReadonlySet<string> = READ_ONLY_TOOLS_DEFAULT,
-  readOnlyContextActions: ReadonlySet<string> = READ_ONLY_CONTEXT_ACTIONS_DEFAULT,
+  blacklistAutoFoldTools: ReadonlySet<string> = AUTO_FOLD_BLACKLIST_DEFAULT,
+  readOnlyContextActions: ReadonlySet<string> = PEEK_READ_ONLY_CONTEXT_ACTIONS,
 ): boolean {
-  if (readOnlyTools.has(name)) return true;
-  if (name !== toolName || !isPlainRecord(args)) return false;
+  if (blacklistAutoFoldTools.has(name)) return false;
+  if (name !== toolName) return true;
+  if (!isPlainRecord(args)) return false;
   const action = ownValue(args, "action");
   if (typeof action !== "string" || !readOnlyContextActions.has(action)) return false;
   const permitted = ownValue(READ_ONLY_CONTEXT_ACTION_ARGUMENTS, action);
@@ -125,8 +138,8 @@ export function scanTurnToolBatches(
   turn: CompleteTurn,
   allowIncomplete = false,
   toolName = DEFAULT_ACTIVE_CONTEXT_TOOL_NAME,
-  readOnlyTools: ReadonlySet<string> = READ_ONLY_TOOLS_DEFAULT,
-  readOnlyContextActions: ReadonlySet<string> = READ_ONLY_CONTEXT_ACTIONS_DEFAULT,
+  blacklistAutoFoldTools: ReadonlySet<string> = AUTO_FOLD_BLACKLIST_DEFAULT,
+  readOnlyContextActions: ReadonlySet<string> = PEEK_READ_ONLY_CONTEXT_ACTIONS,
 ): ScannedToolBatches | null {
   if (!Number.isSafeInteger(turn.start) || !Number.isSafeInteger(turn.end) ||
       turn.start < 0 || turn.end > messages.length || turn.start >= turn.end) return null;
@@ -163,8 +176,8 @@ export function scanTurnToolBatches(
       const id = ownValue(part, "id");
       const name = ownValue(part, "name");
       if (typeof id !== "string" || !id || typeof name !== "string" || !name ||
-          calls.has(id) || !isReadOnlyContextTool(
-            name, ownValue(part, "arguments"), toolName, readOnlyTools, readOnlyContextActions,
+          calls.has(id) || !isAutoFoldableToolCall(
+            name, ownValue(part, "arguments"), toolName, blacklistAutoFoldTools, readOnlyContextActions,
           )) safe = false;
       else calls.set(id, { id, name, assistantIndex: index });
     }
@@ -207,16 +220,16 @@ export function scanTurnToolBatches(
   return { calls, unsafeIndices };
 }
 
-/** Validate a complete turn as wholly read-only. Granular callers may still use its safe batches. */
+/** Validate a complete turn as wholly auto-foldable. Granular callers may still use its safe batches. */
 export function validateTurnToolBatch(
   messages: unknown[],
   turn: CompleteTurn,
   toolName = DEFAULT_ACTIVE_CONTEXT_TOOL_NAME,
-  readOnlyTools: ReadonlySet<string> = READ_ONLY_TOOLS_DEFAULT,
-  readOnlyContextActions: ReadonlySet<string> = READ_ONLY_CONTEXT_ACTIONS_DEFAULT,
+  blacklistAutoFoldTools: ReadonlySet<string> = AUTO_FOLD_BLACKLIST_DEFAULT,
+  readOnlyContextActions: ReadonlySet<string> = PEEK_READ_ONLY_CONTEXT_ACTIONS,
 ): ValidatedToolBatch | null {
   const scanned = scanTurnToolBatches(
-    messages, turn, false, toolName, readOnlyTools, readOnlyContextActions,
+    messages, turn, false, toolName, blacklistAutoFoldTools, readOnlyContextActions,
   );
   return scanned && scanned.unsafeIndices.size === 0 ? { calls: scanned.calls } : null;
 }
@@ -340,7 +353,18 @@ export function unsafeChapterIndices(messages: unknown[]): Set<number> {
   return unsafe;
 }
 
-export function freshBoundary(messages: unknown[], turns: CompleteTurn[], policy: typeof ACTIVE_CONTEXT_POLICY): number {
+/**
+ * The newest end of the window, where nothing folds.
+ *
+ * Two things decide it, and only two. The STRUCTURAL invariant comes first: an
+ * unfinished turn is never foldable at any share, because the excursion in progress is
+ * still using the evidence it gathered. Then the freshTail share, walked back turn by
+ * turn until the suffix carries its bytes. The old hybrid rule -- three complete turns
+ * OR 24,000 bytes, capped at a quarter of a small window -- is gone: it was three
+ * numbers in two units answering one question, and at small windows the turn leg and
+ * the byte leg disagreed about which was the protection.
+ */
+export function freshBoundary(messages: unknown[], turns: CompleteTurn[], freshBytes: number): number {
   if (!messages.length) return 0;
   let unfinishedUser = messages.length;
   for (let index = messages.length - 1; index >= 0; index -= 1) {
@@ -353,19 +377,16 @@ export function freshBoundary(messages: unknown[], turns: CompleteTurn[], policy
     break;
   }
   const leading = leadingCompactionContinuation(messages);
-  let boundary = turns.length >= policy.freshTurns
-    ? turns[turns.length - policy.freshTurns].start
-    : leading?.end ?? 0;
-  boundary = Math.min(boundary, unfinishedUser);
+  let boundary = Math.min(messages.length, unfinishedUser);
   while (boundary > 0) {
     const suffix: unknown[] = [];
     for (let index = boundary; index < messages.length; index += 1) suffix.push(messages[index]);
-    if (bytes(suffix) >= policy.freshBytes) break;
+    if (bytes(suffix) >= freshBytes) break;
     let previous: CompleteTurn | null = null;
     for (let turnIndex = 0; turnIndex < turns.length; turnIndex += 1) {
       if (turns[turnIndex].start < boundary) previous = turns[turnIndex];
     }
-    if (!previous) { boundary = 0; break; }
+    if (!previous) { boundary = leading?.end ?? 0; break; }
     boundary = previous.start;
   }
   return boundary;
@@ -373,18 +394,13 @@ export function freshBoundary(messages: unknown[], turns: CompleteTurn[], policy
 
 export function toolFreshIndices(
   messages: unknown[],
-  turns: CompleteTurn[],
-  policy: typeof ACTIVE_CONTEXT_POLICY,
+  _turns: CompleteTurn[],
+  freshBytes: number,
 ): Set<number> {
   const protectedIndices = new Set<number>();
-  const firstFreshTurn = Math.max(0, turns.length - policy.freshTurns);
-  for (let turnIndex = firstFreshTurn; turnIndex < turns.length; turnIndex += 1) {
-    const turn = turns[turnIndex];
-    for (let index = turn.start; index < turn.end; index += 1) protectedIndices.add(index);
-  }
   let tailBytes = 0;
   let boundary = messages.length;
-  while (boundary > 0 && tailBytes < policy.freshBytes) {
+  while (boundary > 0 && tailBytes < freshBytes) {
     boundary -= 1;
     tailBytes += bytes(messages[boundary]);
   }
@@ -392,6 +408,57 @@ export function toolFreshIndices(
   return protectedIndices;
 }
 
+type EntryEvidence = { message: unknown; ref: BranchObject["ref"] };
+
+/**
+ * Evidence for ONE session entry, derived once and kept.
+ *
+ * A session is append-only, and `buildContextEntries` hands back the session manager's own
+ * stored entry objects rather than copies, so an entry mapped once cannot change shape
+ * underneath us. Keying on the entry OBJECT rather than its id means a replaced entry
+ * misses and is recomputed instead of served stale: there is no invalidation rule here to
+ * get wrong, and a branch switch is simply a different path over the same stable objects.
+ *
+ * What this removes is not loop overhead. It is a `stableStringify` and a `sha256` per
+ * message over the WHOLE history on every call, while the window that history projects
+ * into stays the same size. That is superlinear in total session bytes, and it spent 135
+ * of the 258 minutes of the 2026-08-11 rep, more than the run spent waiting on the model.
+ *
+ * `branchIndex` is deliberately NOT cached. It is a position in the branch being mapped
+ * now, and a different leaf yields a different path across the same entries.
+ */
+const ENTRY_EVIDENCE = new WeakMap<object, {
+  sessionId: string;
+  projectEntry: (entry: Record<string, unknown>) => unknown[];
+  items: EntryEvidence[];
+}>();
+
+function entryEvidence(
+  entry: Record<string, unknown>,
+  entryId: string,
+  sessionId: string,
+  projectEntry: (entry: Record<string, unknown>) => unknown[],
+): EntryEvidence[] {
+  const cached = ENTRY_EVIDENCE.get(entry);
+  if (cached && cached.sessionId === sessionId && cached.projectEntry === projectEntry) {
+    return cached.items;
+  }
+  const items: EntryEvidence[] = [];
+  const projected = denseOwnArrayValues(projectEntry(entry));
+  if (projected) {
+    for (let messageIndex = 0; messageIndex < projected.length; messageIndex += 1) {
+      const message = projected[messageIndex];
+      try {
+        items.push({ message, ref: evidenceRef(sessionId, entryId, message) });
+      } catch {
+        // Unknown persisted shapes stay unavailable rather than weakening exact refs.
+      }
+    }
+  }
+  ENTRY_EVIDENCE.set(entry, { sessionId, projectEntry, items });
+  return items;
+}
+
 export function mapActiveContext(input: {
   sessionId: string;
   eventMessages: unknown[];
@@ -401,9 +468,12 @@ export function mapActiveContext(input: {
   toolName?: string;
   brandNoun?: string;
   entryTypePrefix?: string;
-  readOnlyTools?: ReadonlySet<string>;
+  blacklistAutoFoldTools?: ReadonlySet<string>;
   readOnlyContextActions?: ReadonlySet<string>;
   contextWindow?: number;
+  /** True when `contextWindow` is a declared serving budget rather than a raw window. */
+  netBudget?: boolean;
+  thresholds?: ActiveContextThresholds;
 }): ActiveContextSnapshot {
   const policy = Object.freeze({ ...ACTIVE_CONTEXT_POLICY, ...(input.policy ?? {}) }) as typeof ACTIVE_CONTEXT_POLICY;
   const projectEntry = input.projectEntry ?? sessionEntryMessages;
@@ -412,19 +482,8 @@ export function mapActiveContext(input: {
     const entry = input.contextEntries[entryIndex];
     const entryId = typeof entry?.id === "string" ? entry.id : "";
     if (!entryId) continue;
-    const projected = denseOwnArrayValues(projectEntry(entry));
-    if (!projected) continue;
-    for (let messageIndex = 0; messageIndex < projected.length; messageIndex += 1) {
-      const message = projected[messageIndex];
-      try {
-        branchObjects.push({
-          branchIndex: branchObjects.length,
-          message,
-          ref: evidenceRef(input.sessionId, entryId, message),
-        });
-      } catch {
-        // Unknown persisted shapes stay unavailable rather than weakening exact refs.
-      }
+    for (const item of entryEvidence(entry, entryId, input.sessionId, projectEntry)) {
+      branchObjects.push({ branchIndex: branchObjects.length, message: item.message, ref: item.ref });
     }
   }
 
@@ -460,19 +519,41 @@ export function mapActiveContext(input: {
     Number.isFinite(input.contextWindow) && input.contextWindow > 0
     ? input.contextWindow
     : null;
-  // A fixed byte floor can dominate a small or shrunken window. Cap the protected
-  // tail at freshWindowShare of the window via a conservative bytes-per-token
-  // floor: this binds only on genuinely small windows and never widens the
-  // protected tail.
-  const tailPolicy = reportedContextWindow === null ? policy : Object.freeze({
-    ...policy,
-    freshBytes: Math.min(policy.freshBytes, Math.floor(
-      reportedContextWindow * policy.freshWindowShare * BYTES_PER_TOKEN_FLOOR)),
-  });
+  // One denominator: both zone widths are shares of the serving budget the declared
+  // window yields, so a smaller window narrows the tails in proportion instead of
+  // meeting a fixed byte floor that only a second constant could rescue.
+  // A deployment that declared `providerInputBudget` passes its window in ALREADY NET of
+  // the reservation it holds back, so the zone widths divide by it as given. Only the
+  // descriptor fallback estimates a reservation out of the raw window.
+  const thresholds = input.thresholds ?? DEFAULT_THRESHOLDS;
+  const budgetTokens = input.netBudget === true
+    ? (reportedContextWindow ?? DEFAULT_CONTEXT_WINDOW)
+    : servingBudgetTokens(reportedContextWindow ?? DEFAULT_CONTEXT_WINDOW);
+  const freshBytes = zoneBytes(thresholds.freshTail, budgetTokens);
   const turns = completeTurns(input.eventMessages);
-  const boundary = freshBoundary(input.eventMessages, turns, tailPolicy);
+  const boundary = freshBoundary(input.eventMessages, turns, freshBytes);
+  // THERE IS NO POSITIONAL REACH HERE ANY MORE.
+  //
+  // This function used to compute a second boundary, a byte prefix of the window called
+  // the stale zone, and the selectors would take nothing above it. Two reps died of it.
+  // Rep 2 clamped it to the last CLOSED turn, so a marathon session that never closed
+  // one had a zero-width zone: the last call announced 19 unmarked spans worth 274,173
+  // tokens while the selector could propose nothing, no commit ever fired, and the
+  // window was rejected twice at 375,830 estimated tokens. Rep 3 unclamped it and walked
+  // raw bytes, so a folded prefix was charged for evidence it had already replaced: 17
+  // folded results cost 1,002,801 raw bytes of a 1,072,081-byte allowance, the reach
+  // froze at index 36 of 133, and the run died against the fence at 0.972 occupancy with
+  // 25 takeable batches sitting above the line.
+  //
+  // Shane's ruling (2026-08-10): stale is a CLASS, not a position. A span is
+  // automatically foldable because of what it IS -- a completed tool batch the deployment
+  // did not blacklist, or older material a chapter or consolidation can compose -- and
+  // never because of where it sits. What this function still produces is the protection
+  // that is genuinely positional: the fresh byte tail, set-shaped, in `protectedIndices`
+  // and `toolProtectedIndices`. The rest of the law is membership plus the agent's pins
+  // plus the guard-with-waiver adjudicated once at commit.
   const protectedIndices = unsafeChapterIndices(input.eventMessages);
-  const toolProtectedIndices = toolFreshIndices(input.eventMessages, turns, tailPolicy);
+  const toolProtectedIndices = toolFreshIndices(input.eventMessages, turns, freshBytes);
   // Chapter protection is set-shaped: preserve the newest complete turns and
   // raw byte tail, while allowing an older closed prefix of the current turn.
   for (const index of toolProtectedIndices) protectedIndices.add(index);
@@ -482,24 +563,27 @@ export function mapActiveContext(input: {
     protectedIndices.add(item.index);
     toolProtectedIndices.add(item.index);
   }
-  return {
+  const snapshot: ActiveContextSnapshot = {
     sessionId: input.sessionId,
     messages: clone(input.eventMessages),
     mapped,
     branchObjects,
     completeTurns: turns,
     freshBoundary: boundary,
+    thresholds,
+    budgetTokens,
     protectedIndices,
     toolProtectedIndices,
-    policy: tailPolicy,
+    policy,
     toolName: input.toolName ?? DEFAULT_ACTIVE_CONTEXT_TOOL_NAME,
     brandNoun: input.brandNoun ?? DEFAULT_ACTIVE_CONTEXT_BRAND_NOUN,
     entryTypePrefix: input.entryTypePrefix ?? DEFAULT_ACTIVE_CONTEXT_ENTRY_TYPE_PREFIX,
-    readOnlyTools: input.readOnlyTools ?? READ_ONLY_TOOLS_DEFAULT,
-    readOnlyContextActions: input.readOnlyContextActions ?? READ_ONLY_CONTEXT_ACTIONS_DEFAULT,
+    blacklistAutoFoldTools: input.blacklistAutoFoldTools ?? AUTO_FOLD_BLACKLIST_DEFAULT,
+    readOnlyContextActions: input.readOnlyContextActions ?? PEEK_READ_ONLY_CONTEXT_ACTIONS,
     contextWindow: reportedContextWindow ?? DEFAULT_CONTEXT_WINDOW,
     windowSource: reportedContextWindow === null ? "fallback" : "reported",
   };
+  return snapshot;
 }
 
 /**