pi-fold 1.0.4 → 3.0.0

package/extensions/lib/instrumentation.ts

@@ -1,24 +1,5 @@
 import { evidenceSha256, sha256Value } from "../json.ts";
 import { ownValue } from "./canonical.ts";
-import {
-  MAX_CONTEXT_ATTEMPT_RECORDS,
-  MAX_PROJECTION_HASH_RECORDS,
-} from "./policy.ts";
-
-/**
- * Projection instrumentation.
- *
- * The iteration-1 ledger reported 19 "observed mutations", and reconstruction showed
- * only 7 of them were projection rewrites this runtime performed. The other 12 were
- * provider-side cache misses on projections that had only GROWN: message count up by
- * one, no message removed, system prompt and tool digests constant, and the request
- * still re-paid ~200k fresh tokens. One dial counting both makes a scheduling lever
- * look responsible for a cost no scheduling lever can remove.
- *
- * So the two are counted separately, and the append case is made PROVABLE from
- * artifacts: per-message digests of consecutive projections show exactly where, or
- * whether, the prefix diverged.
- */
 
 export type ProjectionChange = "identical" | "append" | "rewrite";
 
@@ -27,7 +8,6 @@ export interface ProjectionComparison {
   previousCount: number;
   nextCount: number;
   appendedCount: number;
-  /** First index whose digest differs; null when the prefix is intact. */
   firstDivergentIndex: number | null;
 }
 
@@ -38,11 +18,6 @@ export function messageDigests(messages: readonly unknown[]): string[] {
   });
 }
 
-/**
- * Compare two consecutive projections by per-message digest. A pure append leaves the
- * whole previous prefix intact, which is precisely the condition under which a
- * provider cache SHOULD hit; anything else is a rewrite this runtime caused.
- */
 export function compareProjections(
   previous: readonly string[] | null,
   next: readonly string[],
@@ -84,41 +59,15 @@ export interface ProjectionRecord extends ProjectionComparison {
   prefixSha256: string;
 }
 
-/** One provider response read against the projection that produced it. */
 export interface CacheObservation {
   ordinal: number;
   change: ProjectionChange;
   inputTokens: number;
   cacheReadTokens: number;
-  /** A fresh re-read of a prefix that did not move: nothing here is ours to fix. */
+  cacheWriteTokens: number;
   providerSideMiss: boolean;
 }
 
-/**
- * THE context-management event stream.
- *
- * One stream, one envelope, one naming convention. Every context event this runtime
- * produces -- tool attempts and their refusals, span corrections, gate lifecycle,
- * receipts, commits, absorptions, splits, recoveries, projections and provider usage --
- * lands here as a flat JSONL-style record, and every record is also appended as a
- * durable session entry so an analyst can reconstruct the whole timeline from session
- * artifacts alone.
- *
- * The envelope is fixed and self-describing:
- *   v            schema version of the envelope
- *   seq          monotonic per session, from 1, so records order without a clock
- *   kind         "context.<noun>", the only field a reader must know to dispatch
- *   session_id   the session these events belong to
- *   ordinal      transcript position, the deterministic clock this runtime reasons in
- *   at           wall-clock milliseconds, the join key against provider-side telemetry
- *   revision     durable state revision at emit time
- *
- * Everything after the envelope is a FLAT snake_case payload. Records are not shaped
- * around any metric we compute today: a reader that wants a number we never thought of
- * should be able to get it with a query rather than a code change. Nothing nests, and
- * anything one-to-many (a correction, an absorbed wedge, a created fold) is its own
- * record referencing the record it belongs to by `seq`.
- */
 export const CONTEXT_EVENT_SCHEMA_VERSION = 1;
 
 export interface ContextEvent {
@@ -133,63 +82,29 @@ export interface ContextEvent {
 }
 
 export type ContextEventKind =
-  /** One context-management tool call, accepted or refused. */
   | "context.attempt"
-  /** One auto-snap applied to a requested span, referencing its attempt. */
   | "context.correction"
-  /** One protect or unprotect: durable pin bookkeeping; no window bytes move. */
-  | "context.protect"
-  /** The resolved serving budget, stated once at startup so a live run can prove it. */
+  | "context.pin"
   | "context.capacity"
-  /** One receipt delivered into the window. */
   | "context.receipt"
-  /** One post-commit rider composed and persisted: at most one per fold epoch. */
-  | "context.rider"
-  /** One pre-commit last-call exposure: the band-top prompt, with its telemetry. */
   | "context.lastcall"
-  /** One gated round's outcome, attributed to the exposure it answers. */
   | "context.response"
-  /** One surfacing suggestion issued: both channel scores, the divergence, the carrier. */
   | "context.suggestion"
-  /** One surfacing suggestion graded: the acted/used/ignored join and its transition. */
   | "context.outcome"
-  /** One append-once occupancy threshold notice delivered into the window. */
   | "context.notice"
-  /** One commit epoch: the single mutation an epoch pays for. */
+  | "context.frontier"
   | "context.commit"
-  /** One fold created, by the ladder or by the agent. */
   | "context.fold"
-  /**
-   * One generator call for a brief: what it read, what it wrote, and what it cost.
-   *
-   * The generator runs outside the session, so its spend appears in no provider ledger
-   * and its latency appears in no turn. Without this record a deployment cannot say what
-   * briefing costs it, which is the question the async lane exists to answer.
-   */
   | "context.brief"
-  /** One boundary sliver absorbed into its later neighbour. */
   | "context.absorb"
-  /** One oversized span split into sequential bite-sized folds. */
   | "context.split"
-  /** One overflow rollback and the recovery that followed it. */
   | "context.recovery"
-  /** One tree rollback: what left the branch, and whether the request replayed. */
+  | "context.suspend"
   | "context.rollback"
-  /** One projection built, classified against the one before it. */
   | "context.projection"
-  /** One provider response, which is where this stream joins provider usage. */
   | "context.usage"
-  /** One handoff, compared byte-for-byte against the previous transmitted projection. */
   | "context.prefix";
 
-/**
- * Where the prefix first differs from the previous TRANSMITTED projection, in chars.
- *
- * A positional cache is keyed on a byte prefix, so the message-level classification is
- * not enough to attribute a miss: the analyst needs the position, and only the runtime
- * knows why a byte at that position moved. This is the cheap half -- both projections
- * are in hand at handoff -- and it is emission only. No analysis of it belongs here.
- */
 export function prefixDivergence(
   previous: string | null,
   next: string,
@@ -208,18 +123,14 @@ export function prefixDivergence(
 
 export interface InstrumentationLedger {
   projections: number;
-  /** Projections this runtime actually rewrote. The scheduling dial. */
   rewrites: number;
   appends: number;
   identical: number;
-  /** Responses that re-read an unchanged prefix from scratch. The provider's dial. */
   providerSideMisses: number;
   observedMisses: number;
   records: ProjectionRecord[];
   observations: CacheObservation[];
-  /** The stream itself, oldest first. Bounded in memory; durable in full. */
   events: ContextEvent[];
-  /** The monotonic sequence every record carries. */
   sequence: number;
   countsByKind: Record<string, number>;
 }
@@ -240,10 +151,6 @@ export function emptyLedger(): InstrumentationLedger {
   };
 }
 
-/**
- * Stamp the envelope and append. The in-memory ring is the status view and is bounded;
- * the durable session entry is the audit copy and is not.
- */
 export function recordContextEvent(
   ledger: InstrumentationLedger,
   kind: ContextEventKind,
@@ -263,19 +170,9 @@ export function recordContextEvent(
   };
   ledger.countsByKind[kind] = (ledger.countsByKind[kind] ?? 0) + 1;
   ledger.events.push(record);
-  if (ledger.events.length > MAX_CONTEXT_ATTEMPT_RECORDS) {
-    ledger.events = ledger.events.slice(ledger.events.length - MAX_CONTEXT_ATTEMPT_RECORDS);
-  }
   return record;
 }
 
-function bounded<T>(values: T[], next: T): T[] {
-  values.push(next);
-  return values.length > MAX_PROJECTION_HASH_RECORDS
-    ? values.slice(values.length - MAX_PROJECTION_HASH_RECORDS)
-    : values;
-}
-
 export function recordProjection(
   ledger: InstrumentationLedger,
   comparison: ProjectionComparison,
@@ -285,29 +182,13 @@ export function recordProjection(
   if (comparison.change === "rewrite") ledger.rewrites += 1;
   else if (comparison.change === "append") ledger.appends += 1;
   else ledger.identical += 1;
-  ledger.records = bounded(ledger.records, {
+  ledger.records.push({
     ...comparison,
     ordinal: ledger.projections,
-    // The prefix digest, not the whole projection: it is what a positional cache keys
-    // on, so two equal values are the claim that a cache hit was available.
     prefixSha256: sha256Value(digests.slice(0, comparison.previousCount || digests.length)),
   });
 }
 
-/**
- * Read one provider response against the projection that produced it. A miss on a
- * projection that only grew is provider-side by construction: no byte we control
- * moved, so no scheduling change could have prevented the re-read.
- *
- * A rewrite used to be charged to us unconditionally, and that hid the whole rep 21
- * finding: a rewrite that preserved 99.86% of the prompt still read back as ours, so
- * our own metrics reported a negligible divergence and a total cache loss and never
- * put the two side by side. Attribution now asks how much of the previous prompt the
- * rewrite actually kept. If we preserved nearly all of it and the provider still
- * served nothing, our divergence cannot account for the loss and the miss is theirs.
- * That is a claim about ATTRIBUTION, not about cost: a preserved-share miss is still
- * a full rebuild, and it is still worth not causing.
- */
 export const PROVIDER_SIDE_MISS_PRESERVED_SHARE = 0.99;
 
 export function observeCacheUsage(
@@ -317,9 +198,14 @@ export function observeCacheUsage(
   const usage = input.usage;
   const inputTokens = ownValue(usage, "input");
   const cacheReadTokens = ownValue(usage, "cacheRead");
+  const rawCacheWriteTokens = ownValue(usage, "cacheWrite");
   if (typeof inputTokens !== "number" || !Number.isFinite(inputTokens) || inputTokens <= 0 ||
       typeof cacheReadTokens !== "number" || !Number.isFinite(cacheReadTokens) ||
       cacheReadTokens < 0) return null;
+  const cacheWriteTokens = typeof rawCacheWriteTokens === "number" &&
+      Number.isFinite(rawCacheWriteTokens) && rawCacheWriteTokens >= 0
+    ? rawCacheWriteTokens
+    : 0;
   const missed = cacheReadTokens === 0;
   const preserved = typeof input.preservedShare === "number" && Number.isFinite(input.preservedShare)
     ? input.preservedShare
@@ -329,16 +215,16 @@ export function observeCacheUsage(
     change: input.change,
     inputTokens,
     cacheReadTokens,
-    providerSideMiss: missed && (input.change !== "rewrite" ||
+    cacheWriteTokens,
+    providerSideMiss: missed && cacheWriteTokens === 0 && (input.change !== "rewrite" ||
       (preserved !== null && preserved >= PROVIDER_SIDE_MISS_PRESERVED_SHARE)),
   };
   if (missed) ledger.observedMisses += 1;
   if (observation.providerSideMiss) ledger.providerSideMisses += 1;
-  ledger.observations = bounded(ledger.observations, observation);
+  ledger.observations.push(observation);
   return observation;
 }
 
-/** The compact ledger the evidence envelope carries; the record rings stay for audit. */
 export function ledgerSummary(ledger: InstrumentationLedger): Record<string, unknown> {
   return {
     projections: ledger.projections,

package/extensions/lib/measurement.ts

@@ -89,32 +89,12 @@ export function contextUsageRatio(value: unknown): number | null {
   return tokens / contextWindow;
 }
 
-/**
- * A measured WINDOW ratio, restated as occupancy of the serving budget.
- *
- * The provider reports tokens against its total window and the thermostat is declared
- * against the serving budget, so exactly one conversion stands between them and it is
- * written here rather than inferred at each reader. Before this build the trigger
- * divided by the budget while the freeing target and the backstop divided by the
- * window, which at a 383,616-token budget behind a 400,000-token window is a 4.5-point
- * disagreement, and 10 points at a 100,000-token fixture.
- */
 export function budgetOccupancy(snapshot: ActiveContextSnapshot, windowRatio: number | null): number | null {
   if (typeof windowRatio !== "number" || !Number.isFinite(windowRatio)) return null;
   if (!(snapshot.budgetTokens > 0) || !(snapshot.contextWindow > 0)) return null;
   return windowRatio * snapshot.contextWindow / snapshot.budgetTokens;
 }
 
-/**
- * The fence, as a ratio of the window the measured ratio is denominated against.
- *
- * A measured ratio is tokens over `contextWindow`, so "the request reached its serving
- * budget" is exactly `ratio >= budgetTokens / contextWindow`. When the snapshot carries
- * a budget the runtime uses it verbatim: a deployment that DECLARED its input budget has
- * already netted out its output reservation, and re-subtracting one here would fence it
- * roughly four points early. Only the ctx fallback, where nothing but a raw provider
- * window is known, still estimates the reservation.
- */
 export function hardFenceRatio(value?: unknown, ctx?: any): number {
   const direct = ownValue(value, "contextWindow");
   const reported = ownValue(ctx?.getContextUsage?.(), "contextWindow");
@@ -134,45 +114,16 @@ export function hardFenceRatio(value?: unknown, ctx?: any): number {
   return (contextWindow - reserve) / contextWindow;
 }
 
-/**
- * What the runtime is actually allowed to send, in tokens.
- *
- * The per-request max-input DESCRIPTOR a provider advertises assumes a full output
- * reservation, so it understates the real ceiling by whatever the deployment did not
- * reserve. Measured 2026-08-06 on openai-codex gpt-5.x: a run aborted at ~297k
- * projected tokens against a 272k descriptor, in the same session where the provider
- * had just accepted 339,689 tokens. Serving admits roughly the TOTAL window minus the
- * output reservation actually in force, which at a 16,384 reservation is ~384k.
- *
- * Every number here is introspectable in status on purpose: a fence nobody can audit
- * is how a 25 percent error survived four instrumented runs.
- */
 export interface CapacityAccounting {
-  /** "truthful" once the deployment declares its own serving budget. */
   mode: "descriptor" | "truthful";
-  /** The denominator every pressure ratio and fence is computed against. */
   window: number;
   outputReservation: number;
-  /** The token ceiling: window minus the output reservation. */
   budgetTokens: number;
   usedTokens: number | null;
   headroomTokens: number | null;
-  /** What the provider descriptor claimed, kept so the gap stays visible. */
   descriptorWindow: number | null;
 }
 
-/**
- * ONE denominator, and no guess in front of it.
- *
- * `truthful` means the deployment stated the budget it may actually fill, already net of
- * whatever output reservation it holds back, so the value passes through untouched and
- * the reservation this runtime reports is zero because it withheld nothing. Only the
- * descriptor fallback still subtracts, because a per-request max-input descriptor is a
- * total-window claim and the reservation inside it has to be estimated. That estimate was
- * the defect the reshape removed: the runtime subtracted a fixed 16,384 tokens capped at
- * a tenth of the window while the documentation claimed it subtracted the reservation
- * actually in force.
- */
 export function capacityAccounting(input: {
   window: number;
   truthful: boolean;
@@ -352,8 +303,6 @@ export function persistenceProjection(state: ActiveContextState, snapshot: Activ
     leases: Object.fromEntries(Object.entries(state.leases)
       .filter(([id]) => retained.has(id))),
   };
-  // A pending mark is a promise about evidence that is still in the branch. Once its
-  // source or its named fold leaves, the mark is unfulfillable and must not survive.
   const survivingMarks = (state.pendingMarks ?? []).filter((mark) => mark.mark === "refold"
     ? retained.has(mark.id)
     : mark.parts.every((part) => part.kind === "raw"
@@ -362,8 +311,6 @@ export function persistenceProjection(state: ActiveContextState, snapshot: Activ
   if (survivingMarks.length) projected.pendingMarks = clone(survivingMarks);
   else delete projected.pendingMarks;
   delete projected.pinnedPeeks;
-  // A brief correction names a fold; once that fold is gone the correction has
-  // nothing to present, exactly like a mark whose source left the branch.
   const survivingBriefs = Object.entries(state.briefs ?? {}).filter(([id]) => retained.has(id));
   if (survivingBriefs.length) projected.briefs = Object.fromEntries(clone(survivingBriefs));
   else delete projected.briefs;
@@ -378,6 +325,12 @@ export function explicitProtectedKeys(state: ActiveContextState): Set<string> {
   return new Set(state.protected.map(objectRefKey));
 }
 
+/**
+ * Held against folding: pinned by name, or unfoldable for validity (an unfinished turn, an
+ * entry with no evidence ref). There used to be a second reading of this for tool spans,
+ * differing only in which protected set it consulted; the two sets collapsed into one when
+ * fresh-tail was deleted, and the two functions were byte-identical after that.
+ */
 export function refsProtected(
   refs: EvidenceRef[],
   state: ActiveContextState,
@@ -390,23 +343,6 @@ export function refsProtected(
   });
 }
 
-export function toolRefsProtected(
-  refs: EvidenceRef[],
-  state: ActiveContextState,
-  snapshot: ActiveContextSnapshot,
-): boolean {
-  const explicit = explicitProtectedKeys(state);
-  return refs.some((ref) => {
-    const item = exactMapped(snapshot, ref);
-    return !item || explicit.has(objectRefKey(ref)) || snapshot.toolProtectedIndices.has(item.index);
-  });
-}
-
-/**
- * Mass the agent's pins hold out of the ladder's reach: explicitly protected evidence
- * outside the fresh tail. A pin's cost must be a number the agent sees on receipts,
- * never a silent commit shortfall.
- */
 export function protectedStaleMass(
   snapshot: ActiveContextSnapshot,
   state: ActiveContextState,
@@ -424,12 +360,6 @@ export function protectedStaleMass(
   return { bytes: total, refs };
 }
 
-/**
- * Every byte protect currently holds raw, fresh or stale: the mass the pinned-share
- * cap is measured against. protectedStaleMass narrows to what a commit could not
- * reclaim TODAY; the cap has to see the whole promise, because a pin inside the
- * fresh tail outlives the tail.
- */
 export function explicitProtectedMass(
   snapshot: ActiveContextSnapshot,
   state: ActiveContextState,
@@ -452,16 +382,9 @@ export interface AdmissionVerdict {
   requestedTokens: number;
   headroomTokens: number | null;
   budgetTokens: number;
-  /** Bytes of the target that WOULD fit; 0 when even the minimum slice will not. */
   affordableBytes: number;
 }
 
-/**
- * Preflight one read against the exact stored size of its target and the truthful
- * headroom. A read that has not been measured yet is ADMITTED and says so: refusing
- * on absent data would stall a session for a number we never had, and the absence is
- * reported rather than papered over.
- */
 export function admissionVerdict(input: {
   requestedBytes: number;
   capacity: CapacityAccounting;