pi-fold 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -7
- package/extensions/active-context.ts +2252 -182
- package/extensions/lib/advisory.ts +171 -14
- package/extensions/lib/canonical.ts +20 -0
- package/extensions/lib/curation.ts +437 -0
- package/extensions/lib/folding.ts +468 -22
- package/extensions/lib/instrumentation.ts +333 -0
- package/extensions/lib/measurement.ts +111 -0
- package/extensions/lib/persistence.ts +158 -2
- package/extensions/lib/policy.ts +380 -4
- package/extensions/lib/scheduling.ts +923 -0
- package/extensions/lib/selection.ts +437 -12
- package/extensions/lib/surfacing.ts +349 -0
- package/extensions/lib/tool-surface.ts +52 -3
- package/extensions/lib/transcript.ts +58 -4
- package/package.json +2 -2
|
@@ -3,18 +3,21 @@ import {
|
|
|
3
3
|
denseOwnArrayValues,
|
|
4
4
|
evidenceRef,
|
|
5
5
|
evidenceSha256,
|
|
6
|
+
isPlainRecord,
|
|
6
7
|
objectRefKey,
|
|
7
8
|
sha256Text,
|
|
8
9
|
sha256Value,
|
|
9
10
|
stableStringify,
|
|
10
11
|
} from "../json.ts";
|
|
11
12
|
import {
|
|
13
|
+
boundedUtf8,
|
|
12
14
|
bytes,
|
|
13
15
|
clone,
|
|
14
16
|
messageRole,
|
|
15
17
|
ownValue,
|
|
16
18
|
sessionEntryMessages,
|
|
17
19
|
usefulBrief,
|
|
20
|
+
utf8Slice,
|
|
18
21
|
} from "./canonical.ts";
|
|
19
22
|
import {
|
|
20
23
|
branchSha256,
|
|
@@ -33,6 +36,7 @@ import {
|
|
|
33
36
|
childFoldIds,
|
|
34
37
|
clearPrepared,
|
|
35
38
|
flattenFoldRefs,
|
|
39
|
+
foldBrief,
|
|
36
40
|
foldIdFor,
|
|
37
41
|
foldMap,
|
|
38
42
|
normalizedPart,
|
|
@@ -41,11 +45,17 @@ import {
|
|
|
41
45
|
validateFoldForest,
|
|
42
46
|
} from "./persistence.ts";
|
|
43
47
|
import {
|
|
48
|
+
ACTIVE_CONTEXT_POLICY,
|
|
44
49
|
activeContextBrand,
|
|
45
50
|
activeContextSource,
|
|
46
51
|
CONSOLIDATION_WIDTH_THRESHOLD,
|
|
52
|
+
CONTEXT_STATUS_RESPONSE_BYTES,
|
|
47
53
|
EXPAND_LEASE_GENERATIONS,
|
|
48
54
|
MAX_EXPAND_LEASES,
|
|
55
|
+
MAX_FOLD_SPAN_CHARS,
|
|
56
|
+
PEEK_DEFAULT_MAX_BYTES,
|
|
57
|
+
PEEK_HEAD_SHARE,
|
|
58
|
+
STATUS_DIET_SUGGESTIONS,
|
|
49
59
|
} from "./policy.ts";
|
|
50
60
|
import type {
|
|
51
61
|
ActiveContextSnapshot,
|
|
@@ -74,6 +84,7 @@ export function selectAutomaticChapter(
|
|
|
74
84
|
snapshot: ActiveContextSnapshot,
|
|
75
85
|
state: ActiveContextState,
|
|
76
86
|
maximumSourceRefs: number = snapshot.policy.maxFoldSourceRefs,
|
|
87
|
+
claimed: ReadonlySet<string> = new Set<string>(),
|
|
77
88
|
): FoldCandidate | null {
|
|
78
89
|
const units = chapterUnits(snapshot);
|
|
79
90
|
const allowedChildren = new Set<FoldKind>(["tool-result"]);
|
|
@@ -92,10 +103,21 @@ export function selectAutomaticChapter(
|
|
|
92
103
|
if (!parts || parts.some((part) => part.kind === "fold" && state.expanded.includes(part.foldId))) continue;
|
|
93
104
|
const refs = candidateSourceRefs(parts, state);
|
|
94
105
|
if (refs.length > maximumSourceRefs) break;
|
|
106
|
+
if (claimed.size && refs.some((ref) => claimed.has(objectRefKey(ref)))) continue;
|
|
95
107
|
if (refsProtected(refs, state, snapshot)) continue;
|
|
96
108
|
const size = bytes(encodedFoldSource(snapshot, state, parts, "chapter"));
|
|
97
|
-
|
|
98
|
-
|
|
109
|
+
// The chapter cap bounds what the rung COMPOSES, never what it may reclaim: a
|
|
110
|
+
// single closed unit that alone exceeds it is still accepted (rep 19: six status
|
|
111
|
+
// results, each above the cap, were exactly the mass nothing could fold).
|
|
112
|
+
if (size > snapshot.policy.maxChapterChars && endIndex > unitIndex) break;
|
|
113
|
+
// Bite-sized by construction. A multi-unit chapter never grows past the cap, so
|
|
114
|
+
// the ladder cannot build the 60kB fold whose tail hid the answer (rep 6); a
|
|
115
|
+
// SINGLE unit that alone exceeds it is still accepted, because there is no
|
|
116
|
+
// interior boundary to stop at and refusing would leave the biggest span in the
|
|
117
|
+
// window as the one thing nothing may reclaim.
|
|
118
|
+
const biteSized = size <= MAX_FOLD_SPAN_CHARS || endIndex === unitIndex;
|
|
119
|
+
if (size >= snapshot.policy.minChapterChars && biteSized) best = { kind: "chapter", parts, sourceRefs: refs };
|
|
120
|
+
if (size > MAX_FOLD_SPAN_CHARS) break;
|
|
99
121
|
}
|
|
100
122
|
if (best) return best;
|
|
101
123
|
}
|
|
@@ -115,6 +137,14 @@ export interface AutomaticRungSelectionOptions {
|
|
|
115
137
|
toolOnly?: boolean;
|
|
116
138
|
summarizerAvailable?: boolean;
|
|
117
139
|
failedPreparationIds?: ReadonlySet<string>;
|
|
140
|
+
/**
|
|
141
|
+
* Evidence and folds already spoken for by something that is not yet a fold, i.e.
|
|
142
|
+
* a pending epoch mark. Excluding them makes each eligible turn select NEW stale
|
|
143
|
+
* content instead of re-selecting the batch a mark already covers, which is the
|
|
144
|
+
* only way marks accumulate below the commit threshold.
|
|
145
|
+
*/
|
|
146
|
+
claimed?: ReadonlySet<string>;
|
|
147
|
+
claimedFoldIds?: ReadonlySet<string>;
|
|
118
148
|
}
|
|
119
149
|
|
|
120
150
|
export function automaticPreparationId(candidate: FoldCandidate, state: ActiveContextState): string {
|
|
@@ -132,19 +162,21 @@ export function selectAutomaticRung(
|
|
|
132
162
|
ratio: number,
|
|
133
163
|
options: AutomaticRungSelectionOptions = {},
|
|
134
164
|
): AutomaticRungSelection | null {
|
|
165
|
+
const claimed = options.claimed ?? new Set<string>();
|
|
166
|
+
const claimedFoldIds = options.claimedFoldIds ?? new Set<string>();
|
|
135
167
|
if (!options.toolOnly && state.prepared) {
|
|
136
168
|
if (!Number.isFinite(ratio) || ratio < hardFenceRatio(snapshot)) return null;
|
|
137
|
-
const candidate = selectAutomaticChapter(snapshot, state);
|
|
169
|
+
const candidate = selectAutomaticChapter(snapshot, state, snapshot.policy.maxFoldSourceRefs, claimed);
|
|
138
170
|
return candidate ? { kind: "prepared-chapter", candidate } : null;
|
|
139
171
|
}
|
|
140
|
-
const tool = selectAutomaticToolForRung(snapshot, state, ratio, options.waiveToolCadence);
|
|
172
|
+
const tool = selectAutomaticToolForRung(snapshot, state, ratio, options.waiveToolCadence, claimed);
|
|
141
173
|
if (tool) return { kind: "tool", candidate: tool };
|
|
142
174
|
if (options.toolOnly || !Number.isFinite(ratio)) return null;
|
|
143
|
-
const refold = selectAutomaticRefold(snapshot, state, ratio);
|
|
175
|
+
const refold = selectAutomaticRefold(snapshot, state, ratio, claimedFoldIds);
|
|
144
176
|
if (refold) return { kind: "refold", foldId: refold };
|
|
145
|
-
const consolidation = selectAutomaticConsolidation(snapshot, state, ratio);
|
|
177
|
+
const consolidation = selectAutomaticConsolidation(snapshot, state, ratio, claimed);
|
|
146
178
|
if (consolidation) return { kind: "consolidation", candidate: consolidation };
|
|
147
|
-
const chapter = selectAutomaticChapter(snapshot, state);
|
|
179
|
+
const chapter = selectAutomaticChapter(snapshot, state, snapshot.policy.maxFoldSourceRefs, claimed);
|
|
148
180
|
if (!chapter) return null;
|
|
149
181
|
const preparationFailed = options.failedPreparationIds?.has(
|
|
150
182
|
automaticPreparationId(chapter, state),
|
|
@@ -313,7 +345,12 @@ export async function prepareFold(input: {
|
|
|
313
345
|
state: ActiveContextState;
|
|
314
346
|
generation: number;
|
|
315
347
|
brief?: string;
|
|
316
|
-
|
|
348
|
+
/**
|
|
349
|
+
* Provenance for a supplied brief. A full record round-trips a brief whose
|
|
350
|
+
* generator already ran (a pending mark carrying a model brief); the string
|
|
351
|
+
* forms stay the shorthand for the two model-free kinds.
|
|
352
|
+
*/
|
|
353
|
+
briefProvenance?: "supplied" | "deterministic" | BriefProvenance;
|
|
317
354
|
summarize?: (request: Record<string, unknown>, ctx?: unknown) => Promise<Record<string, unknown>>;
|
|
318
355
|
onSummarizerFailure?: (error: unknown) => void;
|
|
319
356
|
ctx?: unknown;
|
|
@@ -348,7 +385,9 @@ export async function prepareFold(input: {
|
|
|
348
385
|
throw new Error(`Supplied brief must be non-structural and at most ${snapshot.policy.maxBriefChars} characters`);
|
|
349
386
|
}
|
|
350
387
|
brief = input.brief.trim();
|
|
351
|
-
provenance =
|
|
388
|
+
provenance = typeof input.briefProvenance === "object"
|
|
389
|
+
? clone(input.briefProvenance)
|
|
390
|
+
: { kind: input.briefProvenance ?? "supplied" };
|
|
352
391
|
} else {
|
|
353
392
|
let modelBrief: { brief: string; provenance: BriefProvenance } | null = null;
|
|
354
393
|
if (input.summarize) {
|
|
@@ -699,7 +738,7 @@ export function foldPlaceholder(fold: ActiveFold, state: ActiveContextState, sna
|
|
|
699
738
|
const parent = fold.parentId ?? "root";
|
|
700
739
|
return [
|
|
701
740
|
`[${activeContextBrand(snapshot.brandNoun)} fold ${fold.id}]`,
|
|
702
|
-
fold
|
|
741
|
+
foldBrief(fold, state),
|
|
703
742
|
`Topology: kind=${fold.kind}; parent=${parent}; children=${childFoldIds(fold).length}; ` +
|
|
704
743
|
`previous=${navigation.previous ?? "none"}; next=${navigation.next ?? "none"}.`,
|
|
705
744
|
`Expand exactly: ${snapshot.toolName} {"action":"expand","id":"${fold.id}"}`,
|
|
@@ -789,6 +828,50 @@ export function assertProjectionPreservesToolLinkage(source: unknown[], projecte
|
|
|
789
828
|
}
|
|
790
829
|
}
|
|
791
830
|
|
|
831
|
+
/**
|
|
832
|
+
* Peek mass the ladder cannot take: results whose evidence the agent protected
|
|
833
|
+
* outright. This is the number that made rep 7's starvation invisible. The mass sat
|
|
834
|
+
* raw in the window, the ROI trigger's eligibility arithmetic could never reach its
|
|
835
|
+
* threshold over it, and no accounting field said so while the window grew to 235k
|
|
836
|
+
* tokens.
|
|
837
|
+
*/
|
|
838
|
+
export function pinnedPeekMass(
|
|
839
|
+
snapshot: ActiveContextSnapshot,
|
|
840
|
+
state: ActiveContextState,
|
|
841
|
+
): { bytes: number; results: number } {
|
|
842
|
+
const explicitProtected = explicitProtectedKeys(state);
|
|
843
|
+
const folded = new Set<string>();
|
|
844
|
+
for (const fold of state.folds) {
|
|
845
|
+
for (const ref of flattenFoldRefs(fold, state)) folded.add(objectRefKey(ref));
|
|
846
|
+
}
|
|
847
|
+
const peekCalls = new Map<string, string>();
|
|
848
|
+
for (const message of snapshot.messages) {
|
|
849
|
+
if (messageRole(message) !== "assistant") continue;
|
|
850
|
+
for (const part of denseOwnArrayValues(ownValue(message, "content")) ?? []) {
|
|
851
|
+
if (ownValue(part, "type") !== "toolCall" || ownValue(part, "name") !== snapshot.toolName) continue;
|
|
852
|
+
const callId = ownValue(part, "id");
|
|
853
|
+
const args = ownValue(part, "arguments");
|
|
854
|
+
const foldId = ownValue(args, "id");
|
|
855
|
+
if (typeof callId !== "string" || !callId || ownValue(args, "action") !== "peek" ||
|
|
856
|
+
typeof foldId !== "string" || !foldId) continue;
|
|
857
|
+
peekCalls.set(callId, foldId);
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
let total = 0;
|
|
861
|
+
let results = 0;
|
|
862
|
+
for (const item of snapshot.mapped) {
|
|
863
|
+
if (messageRole(item.message) !== "toolResult" || !item.ref) continue;
|
|
864
|
+
if (folded.has(objectRefKey(item.ref))) continue;
|
|
865
|
+
const toolCallId = ownValue(item.message, "toolCallId");
|
|
866
|
+
const foldId = typeof toolCallId === "string" ? peekCalls.get(toolCallId) : undefined;
|
|
867
|
+
if (!foldId) continue;
|
|
868
|
+
if (!explicitProtected.has(objectRefKey(item.ref))) continue;
|
|
869
|
+
total += bytes(item.message);
|
|
870
|
+
results += 1;
|
|
871
|
+
}
|
|
872
|
+
return { bytes: total, results };
|
|
873
|
+
}
|
|
874
|
+
|
|
792
875
|
export function projectActiveContext(
|
|
793
876
|
snapshot: ActiveContextSnapshot,
|
|
794
877
|
state: ActiveContextState,
|
|
@@ -810,6 +893,13 @@ export function projectActiveContext(
|
|
|
810
893
|
index += 1;
|
|
811
894
|
}
|
|
812
895
|
}
|
|
896
|
+
// A peek result is APPEND-ONLY. It used to be rewritten in place once a model call
|
|
897
|
+
// had seen it, on the theory that the edit was tail-local; rep 22 disproved that.
|
|
898
|
+
// The result is only tail-local when it ARRIVES, and reclamation deliberately waits
|
|
899
|
+
// until a later assistant message exists, by which point the window has grown over
|
|
900
|
+
// it and the edit lands mid-prefix. Two peeks cost 100k fresh tokens that way. The
|
|
901
|
+
// duplicate bytes are still reclaimed, by the tool-fold rung at the next commit,
|
|
902
|
+
// which is the one moment a rewrite is already being paid for.
|
|
813
903
|
assertProjectionPreservesToolLinkage(snapshot.messages, output);
|
|
814
904
|
return output;
|
|
815
905
|
}
|
|
@@ -838,6 +928,11 @@ export function foldStatusRow(fold: ActiveFold, state: ActiveContextState, snaps
|
|
|
838
928
|
sourceCount: allSourceIds.length,
|
|
839
929
|
sourceIdsTruncated: sourceIds.length < allSourceIds.length,
|
|
840
930
|
sourceSha256: fold.sourceSha256,
|
|
931
|
+
// A paged row without the brief is a chronological id and nothing else: the
|
|
932
|
+
// agent that paged the tree to find WHICH fold holds something got the one
|
|
933
|
+
// field that answers it withheld, while the tree detail carried it all along.
|
|
934
|
+
brief: foldBrief(fold, state),
|
|
935
|
+
sourceChars: fold.sourceChars,
|
|
841
936
|
actions: {
|
|
842
937
|
primary: projection === "folded"
|
|
843
938
|
? { action: "expand", id: fold.id }
|
|
@@ -849,14 +944,8 @@ export function foldStatusRow(fold: ActiveFold, state: ActiveContextState, snaps
|
|
|
849
944
|
};
|
|
850
945
|
}
|
|
851
946
|
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
state: ActiveContextState,
|
|
855
|
-
offset = 0,
|
|
856
|
-
limit = 40,
|
|
857
|
-
maximumChapterSourceRefs = Number.MAX_SAFE_INTEGER,
|
|
858
|
-
): Record<string, unknown> {
|
|
859
|
-
const roots = orderedRoots(state, snapshot).map((item) => item.fold.id);
|
|
947
|
+
/** Every fold, nested ones included, in transcript order with each parent before its children. */
|
|
948
|
+
export function orderedFoldTree(state: ActiveContextState, snapshot: ActiveContextSnapshot): ActiveFold[] {
|
|
860
949
|
const byId = foldMap(state);
|
|
861
950
|
const ordered: ActiveFold[] = [];
|
|
862
951
|
const seen = new Set<string>();
|
|
@@ -867,7 +956,93 @@ export function activeContextStatus(
|
|
|
867
956
|
ordered.push(fold);
|
|
868
957
|
for (const child of childFoldIds(fold)) visit(child);
|
|
869
958
|
};
|
|
870
|
-
for (const root of
|
|
959
|
+
for (const root of orderedRoots(state, snapshot)) visit(root.fold.id);
|
|
960
|
+
return ordered;
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
export function foldDepth(state: ActiveContextState, fold: ActiveFold): number {
|
|
964
|
+
const byId = foldMap(state);
|
|
965
|
+
let depth = 0;
|
|
966
|
+
for (let parentId = fold.parentId; parentId; parentId = byId.get(parentId)?.parentId ?? null) depth += 1;
|
|
967
|
+
return depth;
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
export function foldTreeDetail(
|
|
971
|
+
snapshot: ActiveContextSnapshot,
|
|
972
|
+
state: ActiveContextState,
|
|
973
|
+
): Array<Record<string, unknown>> {
|
|
974
|
+
return orderedFoldTree(state, snapshot).map((fold) => ({
|
|
975
|
+
id: fold.id,
|
|
976
|
+
kind: fold.kind,
|
|
977
|
+
depth: foldDepth(state, fold),
|
|
978
|
+
parentId: fold.parentId,
|
|
979
|
+
brief: foldBrief(fold, state),
|
|
980
|
+
sourceCount: flattenFoldRefs(fold, state).length,
|
|
981
|
+
state: state.expanded.includes(fold.id) ? "expanded" : "folded",
|
|
982
|
+
peekable: true,
|
|
983
|
+
}));
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
/**
|
|
987
|
+
* One compact row per collapsed fold: what is behind it, where its span starts and
|
|
988
|
+
* ends, and the brief that carries whatever key material the fold captured. This is
|
|
989
|
+
* the answer to "which fold has X" that agents were paying 56-83k chars per peek to
|
|
990
|
+
* reconstruct, and it costs a line.
|
|
991
|
+
*/
|
|
992
|
+
export function foldIndexRow(
|
|
993
|
+
fold: ActiveFold,
|
|
994
|
+
state: ActiveContextState,
|
|
995
|
+
snapshot: ActiveContextSnapshot,
|
|
996
|
+
): Record<string, unknown> {
|
|
997
|
+
const refs = flattenFoldRefs(fold, state);
|
|
998
|
+
const interval = foldInterval(fold, state, snapshot);
|
|
999
|
+
const replacement = renderFold(fold, state, snapshot);
|
|
1000
|
+
const source = renderFold(fold, { ...state, expanded: [...state.expanded, fold.id] }, snapshot);
|
|
1001
|
+
const reclaimableBytes = replacement && source ? Math.max(0, bytes(source) - bytes(replacement)) : 0;
|
|
1002
|
+
return {
|
|
1003
|
+
id: fold.id,
|
|
1004
|
+
kind: fold.kind,
|
|
1005
|
+
depth: foldDepth(state, fold),
|
|
1006
|
+
state: state.expanded.includes(fold.id) ? "expanded" : "folded",
|
|
1007
|
+
startId: refs[0]?.entryId ?? null,
|
|
1008
|
+
endId: refs.at(-1)?.entryId ?? null,
|
|
1009
|
+
startPosition: interval?.start ?? null,
|
|
1010
|
+
endPosition: interval?.end ?? null,
|
|
1011
|
+
sourceCount: refs.length,
|
|
1012
|
+
sourceBytes: fold.sourceChars,
|
|
1013
|
+
reclaimableBytes,
|
|
1014
|
+
brief: foldBrief(fold, state),
|
|
1015
|
+
peek: { action: "peek", id: fold.id },
|
|
1016
|
+
expand: { action: "expand", id: fold.id },
|
|
1017
|
+
};
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
/** Every mapped source id to the fold that now holds it, so a lookup never needs a read. */
|
|
1021
|
+
export function foldSourceMap(
|
|
1022
|
+
state: ActiveContextState,
|
|
1023
|
+
snapshot: ActiveContextSnapshot,
|
|
1024
|
+
limit = 512,
|
|
1025
|
+
): { entries: Array<[string, string]>; total: number; truncated: boolean } {
|
|
1026
|
+
const entries: Array<[string, string]> = [];
|
|
1027
|
+
for (const fold of orderedFoldTree(state, snapshot)) {
|
|
1028
|
+
for (const ref of flattenFoldRefs(fold, state)) entries.push([ref.entryId, fold.id]);
|
|
1029
|
+
}
|
|
1030
|
+
// Deepest wins: the innermost fold is the one that actually holds the source.
|
|
1031
|
+
const byEntry = new Map(entries);
|
|
1032
|
+
const all = [...byEntry.entries()];
|
|
1033
|
+
return { entries: all.slice(0, limit), total: all.length, truncated: all.length > limit };
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
export function activeContextStatus(
|
|
1037
|
+
snapshot: ActiveContextSnapshot,
|
|
1038
|
+
state: ActiveContextState,
|
|
1039
|
+
offset = 0,
|
|
1040
|
+
limit = 40,
|
|
1041
|
+
maximumChapterSourceRefs = Number.MAX_SAFE_INTEGER,
|
|
1042
|
+
statusOptions: { diet?: boolean; suggestions?: number } = {},
|
|
1043
|
+
): Record<string, unknown> {
|
|
1044
|
+
const roots = orderedRoots(state, snapshot).map((item) => item.fold.id);
|
|
1045
|
+
const ordered = orderedFoldTree(state, snapshot);
|
|
871
1046
|
const selected = ordered.slice(offset, offset + limit);
|
|
872
1047
|
const protectedKeys = explicitProtectedKeys(state);
|
|
873
1048
|
const objects = snapshot.mapped.flatMap((item) => item.ref ? [{
|
|
@@ -887,18 +1062,49 @@ export function activeContextStatus(
|
|
|
887
1062
|
const eligibleEndpoints = eligibleSourceIds.length
|
|
888
1063
|
? [...new Set([eligibleSourceIds[0], eligibleSourceIds.at(-1)!])]
|
|
889
1064
|
: [];
|
|
1065
|
+
// The index that used to ride every status call grew from 12.6k to 86.5k chars in
|
|
1066
|
+
// one measured session, injected on every request that asked for state. On the diet
|
|
1067
|
+
// the default payload carries counts, the source map and the folds worth acting on;
|
|
1068
|
+
// the full tree and the object list move behind an explicit paged query.
|
|
1069
|
+
const index = statusOptions.diet
|
|
1070
|
+
? visibleCollapsedFolds(state, snapshot)
|
|
1071
|
+
.map((fold) => foldIndexRow(fold, state, snapshot))
|
|
1072
|
+
.sort((left, right) =>
|
|
1073
|
+
Number(right.reclaimableBytes) - Number(left.reclaimableBytes) ||
|
|
1074
|
+
String(left.id).localeCompare(String(right.id)))
|
|
1075
|
+
.slice(0, statusOptions.suggestions ?? STATUS_DIET_SUGGESTIONS)
|
|
1076
|
+
: null;
|
|
1077
|
+
const sourceMap = statusOptions.diet ? foldSourceMap(state, snapshot) : null;
|
|
890
1078
|
return {
|
|
891
1079
|
version: 1,
|
|
892
1080
|
service: "active-context-folding",
|
|
893
1081
|
roots,
|
|
894
|
-
|
|
1082
|
+
...(statusOptions.diet
|
|
1083
|
+
? {
|
|
1084
|
+
index: "diet",
|
|
1085
|
+
topFolds: index,
|
|
1086
|
+
sourceMap: sourceMap!.entries,
|
|
1087
|
+
sourceMapTotal: sourceMap!.total,
|
|
1088
|
+
sourceMapTruncated: sourceMap!.truncated,
|
|
1089
|
+
paging: {
|
|
1090
|
+
note: "Results are delivered in bounded pages.",
|
|
1091
|
+
folds: { action: "status", detail: "folds", offset: 0, limit: 40 },
|
|
1092
|
+
objects: { action: "status", detail: "objects", offset: 0, limit: 40 },
|
|
1093
|
+
tree: { action: "status", detail: "tree", offset: 0 },
|
|
1094
|
+
},
|
|
1095
|
+
}
|
|
1096
|
+
: {
|
|
1097
|
+
folds: selected.map((fold) => foldStatusRow(fold, state, snapshot)),
|
|
1098
|
+
objects: selectedObjects,
|
|
1099
|
+
nextObjectOffset: offset + selectedObjects.length < objects.length
|
|
1100
|
+
? offset + selectedObjects.length
|
|
1101
|
+
: null,
|
|
1102
|
+
}),
|
|
895
1103
|
offset,
|
|
896
1104
|
nextOffset: offset + selected.length < ordered.length ? offset + selected.length : null,
|
|
897
1105
|
totalFolds: ordered.length,
|
|
898
1106
|
protectedSourceIds: state.protected.flatMap((ref) => exactMapped(snapshot, ref) ? [ref.entryId] : []),
|
|
899
|
-
objects: selectedObjects,
|
|
900
1107
|
totalObjects: objects.length,
|
|
901
|
-
nextObjectOffset: offset + selectedObjects.length < objects.length ? offset + selectedObjects.length : null,
|
|
902
1108
|
eligibleChapter: eligibleChapter ? {
|
|
903
1109
|
kind: "chapter",
|
|
904
1110
|
sourceCount: eligibleSourceIds.length,
|
|
@@ -918,6 +1124,91 @@ export function activeContextStatus(
|
|
|
918
1124
|
};
|
|
919
1125
|
}
|
|
920
1126
|
|
|
1127
|
+
/** Room reserved for the one-line continuation marker a truncated page carries. */
|
|
1128
|
+
const STATUS_CONTINUATION_RESERVE_BYTES = 400;
|
|
1129
|
+
|
|
1130
|
+
/**
|
|
1131
|
+
* Enforce the hard status page cap on one fully assembled status payload.
|
|
1132
|
+
*
|
|
1133
|
+
* Measured 2026-08-07 (rep 19): one status answer serialized to 526KB, 6.2x the
|
|
1134
|
+
* remaining headroom, and aborted the request. The paging affordance existed;
|
|
1135
|
+
* nothing forced it. This function is the force: every listing in the payload is
|
|
1136
|
+
* truncated at a unit boundary (whole rows, never mid-unit) until the serialized
|
|
1137
|
+
* page fits, and a one-line continuation marker names the next offset. Trim order
|
|
1138
|
+
* spends the diagnostics first and the listing the caller asked for last, and the
|
|
1139
|
+
* requested listing always keeps at least one unit so paging always progresses.
|
|
1140
|
+
*/
|
|
1141
|
+
export function boundStatusPayload(
|
|
1142
|
+
payload: Record<string, unknown>,
|
|
1143
|
+
detail: string | null,
|
|
1144
|
+
offset: number,
|
|
1145
|
+
cap: number = CONTEXT_STATUS_RESPONSE_BYTES,
|
|
1146
|
+
): Record<string, unknown> {
|
|
1147
|
+
const size = (): number => Buffer.byteLength(JSON.stringify(payload, null, 2), "utf8");
|
|
1148
|
+
if (size() <= cap) return payload;
|
|
1149
|
+
const target = cap - STATUS_CONTINUATION_RESERVE_BYTES;
|
|
1150
|
+
const fits = (): boolean => size() <= target;
|
|
1151
|
+
const omitted: string[] = [];
|
|
1152
|
+
// keepEnd keeps the NEWEST units of a time-ordered ledger; a paged listing keeps
|
|
1153
|
+
// its head so the continuation offset is the next unread unit. Binary search for
|
|
1154
|
+
// the largest keep that fits; a stage that cannot fit alone trims to its floor.
|
|
1155
|
+
const shrink = (owner: unknown, key: string, label: string, keepEnd = false, minKeep = 0): number => {
|
|
1156
|
+
if (!isPlainRecord(owner) || fits()) return 0;
|
|
1157
|
+
const original = ownValue(owner, key);
|
|
1158
|
+
if (!Array.isArray(original) || original.length <= minKeep) return 0;
|
|
1159
|
+
const container = owner as Record<string, unknown>;
|
|
1160
|
+
const apply = (keep: number): void => {
|
|
1161
|
+
container[key] = keepEnd ? original.slice(original.length - keep) : original.slice(0, keep);
|
|
1162
|
+
};
|
|
1163
|
+
let low = minKeep;
|
|
1164
|
+
let high = original.length - 1;
|
|
1165
|
+
let keep = minKeep;
|
|
1166
|
+
while (low <= high) {
|
|
1167
|
+
const mid = (low + high) >> 1;
|
|
1168
|
+
apply(mid);
|
|
1169
|
+
if (fits()) { keep = mid; low = mid + 1; } else { high = mid - 1; }
|
|
1170
|
+
}
|
|
1171
|
+
apply(keep);
|
|
1172
|
+
const dropped = original.length - keep;
|
|
1173
|
+
if (dropped > 0) omitted.push(`${dropped} ${label}`);
|
|
1174
|
+
return dropped;
|
|
1175
|
+
};
|
|
1176
|
+
const automatic = ownValue(payload, "automatic");
|
|
1177
|
+
const instrumentation = isPlainRecord(automatic) ? ownValue(automatic, "instrumentation") : undefined;
|
|
1178
|
+
const surfacing = isPlainRecord(automatic) ? ownValue(automatic, "surfacing") : undefined;
|
|
1179
|
+
shrink(instrumentation, "projectionRecords", "projection record(s)", true);
|
|
1180
|
+
shrink(instrumentation, "cacheObservations", "cache observation(s)", true);
|
|
1181
|
+
shrink(instrumentation, "events", "context event(s)", true);
|
|
1182
|
+
shrink(surfacing, "log", "surfacing log row(s)", true);
|
|
1183
|
+
if (shrink(payload, "sourceMap", "source map row(s)") > 0) payload.sourceMapTruncated = true;
|
|
1184
|
+
shrink(payload, "topFolds", "top fold row(s)");
|
|
1185
|
+
const rowLabel = (key: string): string => key === "folds" ? "fold row(s)" : key === "objects" ? "object row(s)" : "tree row(s)";
|
|
1186
|
+
// The listing the caller asked for goes last and never below one unit. Its
|
|
1187
|
+
// machine-readable next offset moves back to the first unit the page dropped.
|
|
1188
|
+
let continueAt: number | null = null;
|
|
1189
|
+
if (detail === "folds" || detail === "objects") {
|
|
1190
|
+
const rideKey = detail === "folds" ? "objects" : "folds";
|
|
1191
|
+
if (shrink(payload, rideKey, rowLabel(rideKey)) > 0) {
|
|
1192
|
+
const rideKept = (payload[rideKey] as unknown[]).length;
|
|
1193
|
+
if (rideKey === "folds") payload.nextOffset = offset + rideKept;
|
|
1194
|
+
else payload.nextObjectOffset = offset + rideKept;
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
const primaryKey = detail === "objects" ? "objects" : detail === "tree" ? "tree" : detail === "folds" ? "folds" : null;
|
|
1198
|
+
if (primaryKey && shrink(payload, primaryKey, rowLabel(primaryKey), false, 1) > 0) {
|
|
1199
|
+
continueAt = offset + (payload[primaryKey] as unknown[]).length;
|
|
1200
|
+
if (primaryKey === "objects") payload.nextObjectOffset = continueAt;
|
|
1201
|
+
else payload.nextOffset = continueAt;
|
|
1202
|
+
}
|
|
1203
|
+
if (!omitted.length) return payload;
|
|
1204
|
+
const resume = continueAt === null
|
|
1205
|
+
? "the full lists stay reachable through the paged status details"
|
|
1206
|
+
: `continue at ${JSON.stringify({ action: "status", detail, offset: continueAt })}`;
|
|
1207
|
+
payload.continuation =
|
|
1208
|
+
`Status pages are bounded to ${cap} bytes; omitted ${omitted.join(", ")}; ${resume}.`;
|
|
1209
|
+
return payload;
|
|
1210
|
+
}
|
|
1211
|
+
|
|
921
1212
|
export function visibleCollapsedFolds(
|
|
922
1213
|
state: ActiveContextState,
|
|
923
1214
|
snapshot: ActiveContextSnapshot,
|
|
@@ -1010,3 +1301,158 @@ export function recoverFoldMessages(input: {
|
|
|
1010
1301
|
return clone(message);
|
|
1011
1302
|
});
|
|
1012
1303
|
}
|
|
1304
|
+
|
|
1305
|
+
/**
|
|
1306
|
+
* Ephemeral point read of one fold's exact source at any depth. It recovers the same
|
|
1307
|
+
* SHA-256-verified messages expansion restores, bounded for one model call, and changes no
|
|
1308
|
+
* projection: the fold stays collapsed and the durable state is untouched.
|
|
1309
|
+
*/
|
|
1310
|
+
export function peekFoldSource(input: {
|
|
1311
|
+
foldId: string;
|
|
1312
|
+
state: ActiveContextState;
|
|
1313
|
+
entries: Array<Record<string, unknown>>;
|
|
1314
|
+
sessionId: string;
|
|
1315
|
+
/** Explicit widening. Absent, a peek returns the bounded index view. */
|
|
1316
|
+
maximumBytes?: number;
|
|
1317
|
+
/** Byte offset into the exact stored source; the narrowing half of admission control. */
|
|
1318
|
+
offset?: number;
|
|
1319
|
+
toolName?: string;
|
|
1320
|
+
projectEntry?: (entry: Record<string, unknown>) => unknown[];
|
|
1321
|
+
}): Record<string, unknown> {
|
|
1322
|
+
const fold = input.state.folds.find((item) => item.id === input.foldId);
|
|
1323
|
+
if (!fold) throw new Error(`Unknown active-context fold ${input.foldId}`);
|
|
1324
|
+
const messages = recoverFoldMessages({
|
|
1325
|
+
foldId: input.foldId,
|
|
1326
|
+
state: input.state,
|
|
1327
|
+
entries: input.entries,
|
|
1328
|
+
sessionId: input.sessionId,
|
|
1329
|
+
projectEntry: input.projectEntry,
|
|
1330
|
+
});
|
|
1331
|
+
const source = stableStringify(messages);
|
|
1332
|
+
const sourceBytes = bytes(source);
|
|
1333
|
+
const offset = Math.min(Math.max(0, input.offset ?? 0), sourceBytes);
|
|
1334
|
+
const budget = input.maximumBytes ?? PEEK_DEFAULT_MAX_BYTES;
|
|
1335
|
+
const window = offset > 0 ? utf8Slice(source, offset) : source;
|
|
1336
|
+
// Offset zero is the INDEX read and takes head AND tail, because chain keys and
|
|
1337
|
+
// conclusions live at the end of a source and a head-only slice silently omits
|
|
1338
|
+
// exactly that region. An explicit offset is paging and stays contiguous.
|
|
1339
|
+
const view = offset > 0
|
|
1340
|
+
? { text: boundedUtf8(window, budget), omittedBytes: 0, contiguous: true }
|
|
1341
|
+
: boundedHeadTail(window, budget);
|
|
1342
|
+
const returned = view.text;
|
|
1343
|
+
const returnedBytes = bytes(returned) - bytes(headTailMarker(view.omittedBytes));
|
|
1344
|
+
const nextOffset = view.contiguous && offset + returnedBytes < sourceBytes
|
|
1345
|
+
? offset + returnedBytes
|
|
1346
|
+
: null;
|
|
1347
|
+
const truncated = view.omittedBytes > 0 || nextOffset !== null;
|
|
1348
|
+
const children = childFoldIds(fold);
|
|
1349
|
+
// The index view: every nested fold's brief in FULL. A brief is the navigable unit,
|
|
1350
|
+
// so truncating it is the one economy that costs the read its purpose.
|
|
1351
|
+
const index = descendantIndexRows(fold, input.state);
|
|
1352
|
+
return {
|
|
1353
|
+
version: 1,
|
|
1354
|
+
action: "peek",
|
|
1355
|
+
id: fold.id,
|
|
1356
|
+
kind: fold.kind,
|
|
1357
|
+
parentId: fold.parentId,
|
|
1358
|
+
depth: foldDepth(input.state, fold),
|
|
1359
|
+
brief: foldBrief(fold, input.state),
|
|
1360
|
+
sourceCount: messages.length,
|
|
1361
|
+
sourceSha256: fold.sourceSha256,
|
|
1362
|
+
sourceBytes,
|
|
1363
|
+
offset,
|
|
1364
|
+
returnedBytes,
|
|
1365
|
+
omittedBytes: view.omittedBytes,
|
|
1366
|
+
nextOffset,
|
|
1367
|
+
truncated,
|
|
1368
|
+
view: offset > 0 ? "slice" : (view.omittedBytes > 0 ? "index" : "complete"),
|
|
1369
|
+
...(index.length ? { index } : {}),
|
|
1370
|
+
// Narrower and WIDER reads that are always constructible, so a refusal or a
|
|
1371
|
+
// truncation can name one.
|
|
1372
|
+
children,
|
|
1373
|
+
narrower: {
|
|
1374
|
+
...(nextOffset === null ? {} : { slice: { action: "peek", id: fold.id, offset: nextOffset, bytes: returnedBytes } }),
|
|
1375
|
+
...(children.length ? { child: { action: "peek", id: children[0] } } : {}),
|
|
1376
|
+
},
|
|
1377
|
+
...(truncated
|
|
1378
|
+
? { wider: { action: "peek", id: fold.id, bytes: Math.min(sourceBytes, ACTIVE_CONTEXT_POLICY.maxChapterChars) } }
|
|
1379
|
+
: {}),
|
|
1380
|
+
lifetime: "these bytes stay in the window exactly as returned, like any other tool " +
|
|
1381
|
+
"result, until the ladder folds them at a commit. Nothing rewrites this result in place.",
|
|
1382
|
+
note: truncated
|
|
1383
|
+
? `Bounded read: ${returnedBytes} of ${sourceBytes} exact source bytes, ${view.omittedBytes} omitted ` +
|
|
1384
|
+
`from the middle. Widen with bytes, page with offset, or expand ${fold.id} to restore it in place.`
|
|
1385
|
+
: "Complete exact source; the fold stayed collapsed and no projection changed.",
|
|
1386
|
+
source: returned,
|
|
1387
|
+
// Serialized AFTER the source on purpose: a reader that just consumed a bounded slice
|
|
1388
|
+
// decides its next action at the end of the payload, and a notice buried before ten
|
|
1389
|
+
// thousand source bytes is a notice unread (measured: a run concluded a staged chain
|
|
1390
|
+
// was finished because the chain key lived in the unshown tail of a truncated peek).
|
|
1391
|
+
...(truncated
|
|
1392
|
+
? {
|
|
1393
|
+
truncationReminder: `STOP: ${view.omittedBytes || sourceBytes - returnedBytes} of ${sourceBytes} ` +
|
|
1394
|
+
"bytes were NOT shown" +
|
|
1395
|
+
(view.omittedBytes ? " (the middle; the head and tail are both above)" : " (everything after this slice)") +
|
|
1396
|
+
`. Widen with {"action":"peek","id":"${fold.id}","bytes":${
|
|
1397
|
+
Math.min(sourceBytes, ACTIVE_CONTEXT_POLICY.maxChapterChars)}}` +
|
|
1398
|
+
(nextOffset === null ? "" : ` or read on with {"action":"peek","id":"${fold.id}","offset":${nextOffset}}`) +
|
|
1399
|
+
".",
|
|
1400
|
+
}
|
|
1401
|
+
: {}),
|
|
1402
|
+
};
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
/** The marker a head+tail view puts where the omitted middle was. */
|
|
1406
|
+
export function headTailMarker(omittedBytes: number): string {
|
|
1407
|
+
return omittedBytes > 0 ? `\n... [${omittedBytes} exact source bytes omitted] ...\n` : "";
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
/**
|
|
1411
|
+
* A bounded view that keeps the head AND the tail. A head-only bound drops exactly the
|
|
1412
|
+
* region a staged chain keeps its key in; the omitted size is stated where the reading
|
|
1413
|
+
* ends so "there is more" is never an inference.
|
|
1414
|
+
*/
|
|
1415
|
+
export function boundedHeadTail(
|
|
1416
|
+
source: string,
|
|
1417
|
+
budget: number,
|
|
1418
|
+
): { text: string; omittedBytes: number; contiguous: boolean } {
|
|
1419
|
+
const total = bytes(source);
|
|
1420
|
+
if (total <= budget) return { text: source, omittedBytes: 0, contiguous: true };
|
|
1421
|
+
const headBudget = Math.max(1, Math.floor(budget * PEEK_HEAD_SHARE));
|
|
1422
|
+
const head = boundedUtf8(source, headBudget);
|
|
1423
|
+
const tailBudget = Math.max(1, budget - bytes(head));
|
|
1424
|
+
const tailStart = Math.max(bytes(head), total - tailBudget);
|
|
1425
|
+
const tail = boundedUtf8(utf8Slice(source, tailStart), tailBudget);
|
|
1426
|
+
const omittedBytes = Math.max(0, total - bytes(head) - bytes(tail));
|
|
1427
|
+
if (omittedBytes <= 0) return { text: source, omittedBytes: 0, contiguous: true };
|
|
1428
|
+
return {
|
|
1429
|
+
text: `${head}${headTailMarker(omittedBytes)}${tail}`,
|
|
1430
|
+
omittedBytes,
|
|
1431
|
+
contiguous: false,
|
|
1432
|
+
};
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
/** Every descendant fold, in transcript order, with its brief in full. */
|
|
1436
|
+
export function descendantIndexRows(
|
|
1437
|
+
fold: ActiveFold,
|
|
1438
|
+
state: ActiveContextState,
|
|
1439
|
+
): Array<Record<string, unknown>> {
|
|
1440
|
+
const byId = foldMap(state);
|
|
1441
|
+
const rows: Array<Record<string, unknown>> = [];
|
|
1442
|
+
const visit = (parent: ActiveFold): void => {
|
|
1443
|
+
for (const childId of childFoldIds(parent)) {
|
|
1444
|
+
const child = byId.get(childId);
|
|
1445
|
+
if (!child) continue;
|
|
1446
|
+
rows.push({
|
|
1447
|
+
id: child.id,
|
|
1448
|
+
kind: child.kind,
|
|
1449
|
+
depth: foldDepth(state, child),
|
|
1450
|
+
brief: foldBrief(child, state),
|
|
1451
|
+
peek: { action: "peek", id: child.id },
|
|
1452
|
+
});
|
|
1453
|
+
visit(child);
|
|
1454
|
+
}
|
|
1455
|
+
};
|
|
1456
|
+
visit(fold);
|
|
1457
|
+
return rows;
|
|
1458
|
+
}
|