pi-fold 3.0.0 → 4.0.0

package/extensions/lib/folding.ts

@@ -10,6 +10,7 @@ import {
   stableStringify,
 } from "../json.ts";
 import {
+  contentText,
   boundedUtf8,
   bytes,
   clone,
@@ -71,7 +72,9 @@ import {
   chapterUnits,
   unpinnedStaleFolds,
   deterministicChapterCandidateBrief,
+  toolClipHead,
   deterministicConsolidationBrief,
+  oneLine,
   NO_FOLD_KINDS,
   partsForRange,
   resultCall,
@@ -79,6 +82,7 @@ import {
   selectAutomaticRefold,
   selectAutomaticToolBatch,
 } from "./selection.ts";
+import { staleSpanMatureEnd } from "./transcript.ts";
 
 export function selectAutomaticChapter(
   snapshot: ActiveContextSnapshot,
@@ -87,19 +91,44 @@ export function selectAutomaticChapter(
 ): FoldCandidate | null {
   const units = chapterUnits(snapshot);
   const allowedChildren = NO_FOLD_KINDS;
+  // The maturity floor: a stale span may end only where the batch lane has already had
+  // its chance, so a chapter can never claim a tool batch one turn before it qualifies.
+  const matureEnd = staleSpanMatureEnd(snapshot.messages);
   for (let unitIndex = 0; unitIndex < units.length; unitIndex += 1) {
     const first = units[unitIndex];
+    if (first.end > matureEnd) break;
     let best: FoldCandidate | null = null;
     for (let endIndex = unitIndex; endIndex < units.length; endIndex += 1) {
       const unit = units[endIndex];
+      if (unit.end > matureEnd) break;
       if (endIndex > unitIndex && unit.start !== units[endIndex - 1].end) break;
       const coherentSegment = endIndex > unitIndex || first.end - first.start > 1;
       if (!coherentSegment) continue;
+      // A DISQUALIFIED RANGE DISQUALIFIES EVERY LARGER ONE, so the walk stops here rather
+      // than rediscovering the same obstruction at every remaining endIndex (issue #2,
+      // 2026-08-28). The inner loop only ever EXTENDS the range to the right, and all
+      // three conditions below are monotone under that extension: an unmapped index or an
+      // overlapping root stays inside a larger range, a claimed ref stays claimed because
+      // `claimed` is fixed for the whole walk, and a protected ref stays protected. Only
+      // `size > MAX_FOLD_SPAN_CHARS` bounded this loop, and these three `continue`s skipped
+      // the sizing that would have hit it, so an obstruction near the start of the window
+      // let the walk run to the end of `units` for every start: O(units^2) iterations doing
+      // O(span) work each. Instrumented on a 549KB window, ONE context event ran 457,986
+      // inner iterations and allocated 98,811,691 FoldPart objects, of which 456,724
+      // iterations (99.7 percent) died on the `claimed` check and only 72 ever reached the
+      // sizing. That is 42 seconds of CPU at 824KB, ungated by occupancy, on every event.
+      // With the walk stopped at the first obstruction the same event costs 377ms.
+      //
+      // This is 00dfeb3's defect one layer out. That commit memoized `orderedRoots` to kill
+      // the O(roots x folds) rebuild INSIDE each iteration and its own comment named "a walk
+      // that is quadratic in units" as the standing cost; this is that walk. Nothing about
+      // WHICH candidate is selected changes: every iteration this skips would have been
+      // skipped by the same condition.
       const parts = partsForRange(snapshot, state, first.start, unit.end - 1, allowedChildren);
-      if (!parts || parts.some((part) => part.kind === "fold" && state.expanded.includes(part.foldId))) continue;
+      if (!parts || parts.some((part) => part.kind === "fold" && state.expanded.includes(part.foldId))) break;
       const refs = candidateSourceRefs(parts, state);
-      if (claimed.size && refs.some((ref) => claimed.has(objectRefKey(ref)))) continue;
-      if (refsProtected(refs, state, snapshot)) continue;
+      if (claimed.size && refs.some((ref) => claimed.has(objectRefKey(ref)))) break;
+      if (refsProtected(refs, state, snapshot)) break;
       const size = bytes(encodedFoldSource(snapshot, state, parts, "chapter"));
       const biteSized = size <= MAX_FOLD_SPAN_CHARS || endIndex === unitIndex;
       // ACCUMULATE UNTIL OVER, never cut to hit the number. The walk grows the span one
@@ -414,13 +443,43 @@ export async function prepareFold(input: {
   let brief: string;
   let provenance: BriefProvenance;
   if (input.brief !== undefined) {
-    const complaint = briefContractComplaint(
-      input.brief.trim(), snapshot.policy.maxBriefChars, snapshot.toolName);
-    if (complaint !== null) throw new Error(`Supplied brief rejected. ${complaint}`);
-    brief = input.brief.trim();
-    provenance = typeof input.briefProvenance === "object"
-      ? clone(input.briefProvenance)
-      : { kind: input.briefProvenance ?? "supplied" };
+    const declaredKind = typeof input.briefProvenance === "object"
+      ? input.briefProvenance.kind
+      : input.briefProvenance ?? "supplied";
+    if (declaredKind === "supplied") {
+      // A SUPPLIED BRIEF AUGMENTS THE HEAD, NEVER REPLACES IT (rep 7 of
+      // sol-20260823-live: replacement deleted the fact-carrying identification and 74
+      // of 95 peeks recovered values the run's own briefs dropped). The agent's clause
+      // is judged against its own reserve so the refusal names the budget the agent
+      // actually has, and the deterministic head keeps the remainder of the one policy
+      // cap. Only the agent's OWN words take this path: a deterministic brief arriving
+      // through the same door IS the head, and a re-prepared augmented brief is already
+      // composed, so each passes through under the whole-brief cap instead.
+      const complaint = briefContractComplaint(
+        input.brief.trim(), snapshot.policy.agentBriefReserve, snapshot.toolName);
+      if (complaint !== null) throw new Error(`Supplied brief rejected. ${complaint}`);
+      const clause = input.brief.trim();
+      const head = candidate.kind === "consolidation"
+        ? deterministicConsolidationBrief(candidate, state, snapshot.toolName, snapshot)
+        : deterministicChapterCandidateBrief(snapshot, candidate);
+      if (head) {
+        const seam = " · Agent: ";
+        const headRoom = Math.max(0, snapshot.policy.maxBriefChars - seam.length - clause.length);
+        brief = `${oneLine(head, headRoom)}${seam}${clause}`;
+        provenance = { kind: "augmented" };
+      } else {
+        brief = clause;
+        provenance = { kind: "supplied" };
+      }
+    } else {
+      const complaint = briefContractComplaint(
+        input.brief.trim(), snapshot.policy.maxBriefChars, snapshot.toolName);
+      if (complaint !== null) throw new Error(`Supplied brief rejected. ${complaint}`);
+      brief = input.brief.trim();
+      provenance = typeof input.briefProvenance === "object"
+        ? clone(input.briefProvenance)
+        : { kind: declaredKind } as BriefProvenance;
+    }
   } else {
     brief = candidate.kind === "consolidation"
       ? deterministicConsolidationBrief(candidate, state, snapshot.toolName, snapshot)
@@ -806,6 +865,13 @@ export function pinnedPeekMass(
   state: ActiveContextState,
 ): { bytes: number; results: number } {
   const explicitProtected = explicitProtectedKeys(state);
+  // ITS TWO SIBLINGS ALREADY RETURN HERE (2026-08-28). `protectedStaleMass` and
+  // `explicitProtectedMass` both open with this guard; this one did the whole quadratic walk
+  // and then returned zero, because every increment in its final loop is gated on
+  // `explicitProtected.has(...)`. So the guard is exact, not an approximation. It is not
+  // hypothetical either: `state.protected` is empty in all 10,521 persisted state records
+  // across 36 sealed runs, the corpus carrying no `context.pin` event at all.
+  if (!explicitProtected.size) return { bytes: 0, results: 0 };
   const folded = new Set<string>();
   for (const fold of state.folds) {
     for (const ref of flattenFoldRefs(fold, state)) folded.add(objectRefKey(ref));
@@ -859,10 +925,55 @@ export function projectActiveContext(
       index += 1;
     }
   }
+  applyToolClips(output, state, snapshot);
   assertProjectionPreservesToolLinkage(snapshot.messages, output);
   return output;
 }
 
+// THE TOOL-CALL DIET'S VIEW (2026-08-24). A clipped result renders in place as its
+// identified head with the cut stated and the recovery named, matched BY CALL ID and
+// identity-preserving on role, call id, tool name, error flag and timestamp, exactly
+// the ephemeral-peek withdrawal's shape one mechanism over. Clips are only ever ADDED
+// inside a commit transaction, and a commit already rewrites the projection, so a
+// non-commit pass with unchanged clips renders byte-identically and the frozen prefix
+// never moves. Every reader shares this pass because every reader shares
+// projectActiveContext: the live projection, the persistence readers and the
+// attribution lens all see the same clipped view, and the full bytes stay in the
+// transcript behind the entry id the marker names.
+function applyToolClips(
+  output: unknown[],
+  state: ActiveContextState,
+  snapshot: ActiveContextSnapshot,
+): void {
+  if (!state.clips?.length) return;
+  const byCallId = new Map(state.clips.map((clip) => [clip.callId, clip]));
+  for (let index = 0; index < output.length; index += 1) {
+    const message = output[index] as Record<string, unknown> | null;
+    if (!message || (message as { role?: unknown }).role !== "toolResult") continue;
+    const callId = (message as { toolCallId?: unknown }).toolCallId;
+    if (typeof callId !== "string") continue;
+    const clip = byCallId.get(callId);
+    if (!clip) continue;
+    const details = (message as { details?: unknown }).details;
+    if (details && typeof (details as { projection?: unknown }).projection === "string") continue;
+    const full = contentText(message);
+    if (full.length <= snapshot.policy.toolClipFloorChars) continue;
+    const head = toolClipHead(full, snapshot.policy.toolClipHeadChars);
+    const hidden = full.length - head.length;
+    if (hidden <= 0) continue;
+    output[index] = {
+      role: "toolResult",
+      toolCallId: callId,
+      toolName: (message as { toolName?: unknown }).toolName,
+      content: [{ type: "text", text: `${head}\n... [${hidden} more chars clipped; ` +
+        `{"action":"peek","id":"${clip.entryId}"} returns the full result]` }],
+      isError: (message as { isError?: unknown }).isError ?? false,
+      details: { projection: "tool-clip" },
+      timestamp: (message as { timestamp?: unknown }).timestamp,
+    };
+  }
+}
+
 export function foldStatusRow(fold: ActiveFold, state: ActiveContextState, snapshot: ActiveContextSnapshot): Record<string, unknown> {
   const navigation = foldNavigation(fold, state, snapshot);
   const interval = foldInterval(fold, state, snapshot);
@@ -1299,11 +1410,26 @@ export function peekFoldSource(input: {
   const source = stableStringify(messages);
   const sourceBytes = bytes(source);
   const offset = Math.min(Math.max(0, input.offset ?? 0), sourceBytes);
-  const budget = input.maximumBytes ?? PEEK_DEFAULT_MAX_BYTES;
+  // The widest read anyone may ask for is the policy's own source cap, the same ceiling
+  // the `wider` hint has always offered; an unclamped `bytes` was the one door left to a
+  // single result larger than any fold the runtime would cut.
+  const budget = Math.min(
+    input.maximumBytes ?? PEEK_DEFAULT_MAX_BYTES,
+    ACTIVE_CONTEXT_POLICY.maxSourceChars,
+  );
+  // ONE BUDGET GOVERNS THE WHOLE RESULT (sol-20260826-full2, pifold rep 1). The byte
+  // bound used to govern `source` alone while the descendant index rode free, so peeking
+  // the root of a deep consolidation tree returned 320 nested briefs, 423KB against a
+  // stated 16KB bound, durably into a window already near its budget; that one result is
+  // what the projection fence then aborted thirty-five requests over. The index seats
+  // first, into at most half the budget, because a parent's index is the point read; the
+  // source view takes everything the seated rows leave.
+  const catalog = boundedDescendantIndex(fold, input.state, Math.floor(budget / 2));
+  const sourceBudget = budget - catalog.seatedBytes;
   const window = offset > 0 ? utf8Slice(source, offset) : source;
   const view = offset > 0
-    ? { text: boundedUtf8(window, budget), omittedBytes: 0, contiguous: true }
-    : boundedHeadTail(window, budget);
+    ? { text: boundedUtf8(window, sourceBudget), omittedBytes: 0, contiguous: true }
+    : boundedHeadTail(window, sourceBudget);
   const returned = view.text;
   const returnedBytes = bytes(returned) - bytes(headTailMarker(view.omittedBytes));
   const nextOffset = view.contiguous && offset + returnedBytes < sourceBytes
@@ -1311,7 +1437,7 @@ export function peekFoldSource(input: {
     : null;
   const truncated = view.omittedBytes > 0 || nextOffset !== null;
   const children = childFoldIds(fold);
-  const index = descendantIndexRows(fold, input.state);
+  const index = catalog.rows;
   return {
     version: 1,
     action: "peek",
@@ -1330,25 +1456,31 @@ export function peekFoldSource(input: {
     truncated,
     view: offset > 0 ? "slice" : (view.omittedBytes > 0 ? "index" : "complete"),
     ...(index.length ? { index } : {}),
+    ...(catalog.omitted ? { indexOmitted: catalog.omitted } : {}),
     children,
     narrower: {
       ...(nextOffset === null ? {} : { slice: { action: "peek", id: fold.id, offset: nextOffset, bytes: returnedBytes } }),
       ...(children.length ? { child: { action: "peek", id: children[0] } } : {}),
     },
-    ...(truncated
+    ...(truncated || catalog.omitted
       ? { wider: { action: "peek", id: fold.id, bytes: Math.min(sourceBytes, ACTIVE_CONTEXT_POLICY.maxSourceChars) } }
       : {}),
     lifetime: "these bytes stay in the window exactly as returned, like any other tool result, and " +
       `nothing rewrites this result in place. It is a COPY of fold ${fold.id}'s stored source, so the ` +
       `next commit reclaims it behind a placeholder naming ${fold.id}; peek ${fold.id} again for the ` +
       "same verbatim bytes, or pin this result to keep the copy raw.",
-    note: truncated
+    note: (truncated
       ? `Bounded read: ${returnedBytes} of ${sourceBytes} exact source bytes, ${view.omittedBytes} omitted ` +
         `from the middle. Widen with bytes, page with offset, or expand ${fold.id} to restore it in place.`
       : children.length
         ? `Complete stored span, one level: raw entries exactly, and ${children.length} child fold(s) still ` +
           "placeheld. Peek a child id to read its own span; the fold stayed collapsed and no projection changed."
-        : "Complete exact source; the fold stayed collapsed and no projection changed.",
+        : "Complete exact source; the fold stayed collapsed and no projection changed.") +
+      (catalog.omitted
+        ? ` The nested index seats the shallowest ${index.length} of ${index.length + catalog.omitted} ` +
+          "descendant briefs inside the byte budget; the deeper rows are one hop away by peeking a child id, " +
+          "or widen with bytes."
+        : ""),
     source: returned,
     ...(truncated
       ? {
@@ -1388,6 +1520,38 @@ export function boundedHeadTail(
   };
 }
 
+/**
+ * The descendant index, seated inside a byte budget: shallowest rows first, whole rows
+ * only, the omitted counted rather than dropped. Depth order is the navigational order,
+ * because a row's own children are reachable by peeking it, so the rows worth seating
+ * first are the ones every deeper row is reachable THROUGH. Walk order breaks depth
+ * ties, which keeps siblings in span order. The seating stops at the first row that
+ * does not fit, so what is omitted is always a contiguous deep tail of the depth order
+ * and the note can state it as one count.
+ */
+export function boundedDescendantIndex(
+  fold: ActiveFold,
+  state: ActiveContextState,
+  budget: number,
+): { rows: Array<Record<string, unknown>>; omitted: number; seatedBytes: number } {
+  const walked = descendantIndexRows(fold, state)
+    .map((row, walk) => ({ row, walk }))
+    .sort((left, right) =>
+      Number(left.row.depth) - Number(right.row.depth) || left.walk - right.walk);
+  const rows: Array<Record<string, unknown>> = [];
+  // Counted as the array actually serializes, brackets and joining commas included, so
+  // "seated bytes" is the bytes the result carries and not an under-read of them.
+  let seatedBytes = 2;
+  for (const { row } of walked) {
+    const cost = bytes(stableStringify(row)) + (rows.length ? 1 : 0);
+    if (seatedBytes + cost > budget) break;
+    seatedBytes += cost;
+    rows.push(row);
+  }
+  if (!rows.length) seatedBytes = 0;
+  return { rows, omitted: walked.length - rows.length, seatedBytes };
+}
+
 export function descendantIndexRows(
   fold: ActiveFold,
   state: ActiveContextState,

package/extensions/lib/instrumentation.ts

@@ -87,17 +87,11 @@ export type ContextEventKind =
   | "context.pin"
   | "context.capacity"
   | "context.receipt"
-  | "context.lastcall"
-  | "context.response"
-  | "context.suggestion"
-  | "context.outcome"
-  | "context.notice"
   | "context.frontier"
   | "context.commit"
   | "context.fold"
   | "context.brief"
   | "context.absorb"
-  | "context.split"
   | "context.recovery"
   | "context.suspend"
   | "context.rollback"

package/extensions/lib/measurement.ts

@@ -15,7 +15,7 @@ import {
 } from "./canonical.ts";
 import {
   childFoldIds,
-  deriveFoldParents,
+  assignFoldParents,
   flattenFoldRefs,
   foldMap,
   parseActiveContextState,
@@ -270,11 +270,42 @@ export function foldInterval(fold: ActiveFold, state: ActiveContextState, snapsh
   return indices ? { start: indices[0], end: indices.at(-1)! } : null;
 }
 
-export function orderedRoots(state: ActiveContextState, snapshot: ActiveContextSnapshot): Array<{ fold: ActiveFold; start: number; end: number }> {
-  return rootFolds(state).flatMap((fold) => {
+export type OrderedRoot = { fold: ActiveFold; start: number; end: number };
+
+// ROOT INTERVALS ARE DERIVED ONCE PER (STATE, SNAPSHOT) PAIR (2026-08-24). This derivation
+// is O(roots x folds) on its own, because `foldInterval` calls `flattenFoldRefs`, which
+// rebuilds `foldMap` for every root it walks. `partsForRange` opens with it, and
+// `selectAutomaticChapter` calls `partsForRange` from a walk that is quadratic in units,
+// so the cost was O(units^2 x roots x folds) for an answer that cannot change inside a
+// pass. Replaying sealed sol-20260823-live rep 3 at 125 folds, ONE frontier walk made
+// 12,333 calls and spent 22.5 seconds, and 12,328 of them returned null without reading a
+// root, because the automatic chapter passes an EMPTY allowed-child set and any overlap
+// refuses. At 140 folds the walk was 98.8 percent of the runtime's whole per-request cost;
+// the per-turn gap between a response and the next projection ran 3s at zero folds to 164s
+// past 150, and five per-turn call sites each pay a walk.
+//
+// The memo is gate 113's rule and gate 121's rule at a third address: kept against the
+// state and snapshot OBJECTS rather than their contents, because both are REPLACED on
+// every change rather than mutated, so a replaced one misses and derives again instead of
+// serving something stale. It hands every caller the SAME array, which is safe only while
+// no reading mutates it; gate 149 drives every call site against a frozen one so a reading
+// that starts mutating throws rather than quietly poisoning the memo.
+export const ORDERED_ROOTS = new WeakMap<ActiveContextState, WeakMap<ActiveContextSnapshot, OrderedRoot[]>>();
+
+export function orderedRoots(state: ActiveContextState, snapshot: ActiveContextSnapshot): OrderedRoot[] {
+  let bySnapshot = ORDERED_ROOTS.get(state);
+  if (!bySnapshot) {
+    bySnapshot = new WeakMap<ActiveContextSnapshot, OrderedRoot[]>();
+    ORDERED_ROOTS.set(state, bySnapshot);
+  }
+  const memoized = bySnapshot.get(snapshot);
+  if (memoized) return memoized;
+  const roots = rootFolds(state).flatMap((fold) => {
     const interval = foldInterval(fold, state, snapshot);
     return interval ? [{ fold, ...interval }] : [];
   }).sort((left, right) => left.start - right.start);
+  bySnapshot.set(snapshot, roots);
+  return roots;
 }
 
 export function visibleCollapsedRoots(state: ActiveContextState, snapshot: ActiveContextSnapshot) {
@@ -317,7 +348,9 @@ export function persistenceProjection(state: ActiveContextState, snapshot: Activ
   if (projected.prepared && projected.prepared.sourceRefs.some((ref) => !mapped.has(objectRefKey(ref)))) {
     delete projected.prepared;
   }
-  projected.folds = deriveFoldParents(projected.folds);
+  // The parse on the next line validates this forest, so `assignFoldParents` is the whole
+  // job here; `deriveFoldParents` would walk it twice.
+  projected.folds = assignFoldParents(projected.folds);
   return parseActiveContextState(projected, state.sessionId);
 }
 
@@ -537,3 +570,50 @@ export function parseNativeCompactionCompletion(value: unknown, expectedSessionI
   }
   return { ...(clone(value) as unknown as NativeCompactionCompletionReceipt), decision };
 }
+
+/**
+ * AN IMAGE IS NOT TEXT, AND MEASURING IT AS TEXT KILLS SESSIONS (2026-08-28).
+ *
+ * The projection is sized by serializing it and counting UTF-8 bytes, which is right for
+ * prose and catastrophically wrong for an image: a content block carries its pixels as
+ * base64, so a screenshot lands as several hundred thousand characters that the estimator
+ * then divides by a chars-per-token ratio learned from PROSE. A live session read three
+ * PNGs of 364,801, 631,045 and 827,513 bytes; the projection went 636,687 chars to
+ * 1,268,275 in one step and the estimate went 110,687 tokens to 220,883, while the
+ * provider's own count for that same request came back 111,837. Every token of the
+ * 109,046 difference was base64 counted as if it were sentences.
+ *
+ * The calibration cannot learn its way out, and that is the part worth understanding.
+ * `projectionCharsPerToken` takes the MINIMUM ratio over its window so the estimate never
+ * under-reads. An image pushes the true ratio UP, so the minimum discards exactly the
+ * sample that would have corrected it and keeps dividing base64 by the prose ratio. The
+ * conservative rule that protects against under-reading is what locks in the over-read.
+ *
+ * So image payload is removed from the character count and replaced by a flat per-image
+ * token cost. The count cannot be exact across providers, and it does not need to be: it
+ * needs to be the right order of magnitude instead of wrong by seventy times, and the
+ * provider anchor corrects the remainder the moment a real measurement lands.
+ */
+export function imageMass(projected: unknown[]): { base64Chars: number; images: number } {
+  let base64Chars = 0;
+  let images = 0;
+  for (const message of projected) {
+    const content = ownValue(message, "content");
+    const blocks = denseOwnArrayValues(content);
+    if (!blocks) continue;
+    for (const block of blocks) {
+      if (ownValue(block, "type") !== "image") continue;
+      images += 1;
+      // Two shapes reach here: the flat `data` a Pi tool result carries, and the nested
+      // `source.data` of a provider-native block. Both are counted; neither is required.
+      const direct = ownValue(block, "data");
+      const nested = ownValue(ownValue(block, "source"), "data");
+      const payload = typeof direct === "string" ? direct
+        : typeof nested === "string" ? nested : "";
+      // The serialization counts the JSON-escaped string, and base64 has no character
+      // JSON escapes, so its byte length in the serialized form is its own length.
+      base64Chars += bytes(payload);
+    }
+  }
+  return { base64Chars, images };
+}