pi-better-background-tasks 0.2.19 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/package.json +1 -1
- package/src/failures.ts +25 -4
- package/src/logs.ts +51 -7
- package/src/navigator-provider.ts +2 -2
- package/src/output.ts +753 -0
- package/src/process.ts +95 -8
- package/src/registry.ts +84 -7
- package/src/runtime.ts +55 -16
- package/src/shared-callback-batcher.ts +327 -29
- package/src/shared-failure-observations.ts +501 -22
- package/src/shared-log-utils.ts +1025 -2
- package/src/shared-sandbox-core.ts +11 -8
- package/src/tools.ts +105 -135
- package/src/types.ts +16 -0
|
@@ -3,24 +3,47 @@ import { appendFileSync, closeSync, existsSync, openSync, readFileSync, mkdirSyn
|
|
|
3
3
|
import { dirname } from "node:path";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Explicit, append-only incident dispositions (#315).
|
|
8
|
+
* - recovered: the same operation later passed (verified by the adapter).
|
|
9
|
+
* - superseded: a different verification or remediation established the outcome.
|
|
10
|
+
* - expected: the non-success result was intentional.
|
|
11
|
+
* - open: explicitly classified as still needing action; the incident stays actionable.
|
|
12
|
+
*/
|
|
13
|
+
export type IncidentDisposition = "recovered" | "superseded" | "expected" | "open";
|
|
14
|
+
export const INCIDENT_DISPOSITIONS: readonly IncidentDisposition[] = Object.freeze(["recovered", "superseded", "expected", "open"]);
|
|
15
|
+
|
|
6
16
|
export interface FailureEvent {
|
|
7
17
|
/** Stable source event identity: replay must reuse this id. */
|
|
8
18
|
id: string;
|
|
9
19
|
operation: string;
|
|
10
|
-
kind: "failure" | "recovered" | "incomplete" | "delivered";
|
|
20
|
+
kind: "failure" | "recovered" | "incomplete" | "delivered" | "disposition";
|
|
11
21
|
/** Event time when known; never fabricate it from log mtime. */
|
|
12
22
|
at?: number;
|
|
13
23
|
summary?: string;
|
|
14
24
|
category?: string;
|
|
15
25
|
evidence?: string;
|
|
16
26
|
expected?: boolean;
|
|
17
|
-
/** Recovery/delivery must name the incidents it resolves/delivers. */
|
|
27
|
+
/** Recovery/delivery/disposition must name the incidents it resolves/delivers/classifies. */
|
|
18
28
|
incidents?: string[];
|
|
29
|
+
/** Disposition events only. */
|
|
30
|
+
disposition?: IncidentDisposition;
|
|
31
|
+
/** Disposition events only: why the incidents are classified this way. */
|
|
32
|
+
reason?: string;
|
|
33
|
+
}
|
|
34
|
+
export interface DispositionRecord {
|
|
35
|
+
eventId: string;
|
|
36
|
+
disposition: IncidentDisposition;
|
|
37
|
+
incidents: string[];
|
|
38
|
+
reason: string;
|
|
39
|
+
evidence?: string;
|
|
40
|
+
at: number;
|
|
19
41
|
}
|
|
20
42
|
export interface FailureObservation {
|
|
21
43
|
id: string;
|
|
22
44
|
operation: string;
|
|
23
|
-
|
|
45
|
+
/** `resolved` is the historical name for a recovered incident. */
|
|
46
|
+
status: "unresolved" | "expected" | "resolved" | "superseded";
|
|
24
47
|
category: string;
|
|
25
48
|
summary: string;
|
|
26
49
|
evidence?: string;
|
|
@@ -31,6 +54,8 @@ export interface FailureObservation {
|
|
|
31
54
|
at?: number;
|
|
32
55
|
count: number;
|
|
33
56
|
resolvedAt?: number;
|
|
57
|
+
/** Latest explicit disposition applied to this incident. */
|
|
58
|
+
disposition?: DispositionRecord;
|
|
34
59
|
}
|
|
35
60
|
export interface FailureState {
|
|
36
61
|
version: 1;
|
|
@@ -38,6 +63,10 @@ export interface FailureState {
|
|
|
38
63
|
observations: Record<string, FailureObservation>;
|
|
39
64
|
delivered: Record<string, number>;
|
|
40
65
|
resolved?: Record<string, number>;
|
|
66
|
+
/** Closed incidents replaced by a newer incident of the same operation. Never deleted. */
|
|
67
|
+
history?: Record<string, FailureObservation>;
|
|
68
|
+
/** Accepted disposition events in journal order. */
|
|
69
|
+
dispositions?: DispositionRecord[];
|
|
41
70
|
}
|
|
42
71
|
export function emptyFailureState(): FailureState {
|
|
43
72
|
return { version: 1, seen: [], observations: {}, delivered: {} };
|
|
@@ -48,26 +77,102 @@ export function failureIdentity(...parts: unknown[]): string {
|
|
|
48
77
|
function text(value: string | undefined, fallback: string): string {
|
|
49
78
|
return (value || fallback).replace(/[\x00-\x1f\x7f]/g, " ").slice(0, 400);
|
|
50
79
|
}
|
|
80
|
+
|
|
81
|
+
/** Agent tool attempts: the agent handles its own tool errors, so one failure is not yet actionable. */
|
|
82
|
+
export const AGENT_TOOL_CATEGORY = "tool";
|
|
83
|
+
/** A command the confined intent bash refused before running (malformed or reused intent). Agent-owned, never grouped. */
|
|
84
|
+
export const REJECTED_INTENT_CATEGORY = "rejected-intent";
|
|
85
|
+
/** The same unresolved agent tool operation failing this many times is treated as stuck. */
|
|
86
|
+
export const REPEATED_FAILURE_THRESHOLD = 3;
|
|
87
|
+
|
|
88
|
+
/** The one current incident with this id, if it is still in the reduced state. */
|
|
89
|
+
export function findIncident(state: FailureState, id: string): FailureObservation | undefined {
|
|
90
|
+
return Object.values(state.observations).find((item) => item.id === id);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Current actionability, derived only from reduced structured state. Lifecycle is not an input.
|
|
95
|
+
* Agent tool failures need an explicit `open` disposition or a repeated failure of the same
|
|
96
|
+
* operation; other producer failures (exit, model, supervision, watch) are actionable at once.
|
|
97
|
+
*/
|
|
98
|
+
export function requiresAction(x: FailureObservation): boolean {
|
|
99
|
+
if (x.status !== "unresolved" || x.category === "observation-incomplete") return false;
|
|
100
|
+
if (x.category !== AGENT_TOOL_CATEGORY && x.category !== REJECTED_INTENT_CATEGORY) return true;
|
|
101
|
+
return x.disposition?.disposition === "open" || x.count >= REPEATED_FAILURE_THRESHOLD;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export function failureLabel(x: FailureObservation): string {
|
|
105
|
+
if (x.category === "observation-incomplete") return "Observation incomplete";
|
|
106
|
+
if (x.status === "expected") return "Expected failure";
|
|
107
|
+
if (x.status === "resolved") return "Recovered";
|
|
108
|
+
if (x.status === "superseded") return "Superseded";
|
|
109
|
+
return requiresAction(x) ? "Action required" : "Unclassified failure observation";
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Validate a disposition against the reduced state. Returns the rejection reason, or undefined. */
|
|
113
|
+
export function validateDisposition(state: FailureState, event: FailureEvent): string | undefined {
|
|
114
|
+
if (event.kind !== "disposition") return "Not a disposition event";
|
|
115
|
+
if (!event.disposition || !INCIDENT_DISPOSITIONS.includes(event.disposition)) return `Unknown disposition ${JSON.stringify(event.disposition ?? "")}`;
|
|
116
|
+
const ids = event.incidents;
|
|
117
|
+
if (!Array.isArray(ids) || ids.length === 0) return "A disposition must name at least one incident";
|
|
118
|
+
if (ids.some((id) => typeof id !== "string" || !id)) return "Incident ids must be non-empty strings";
|
|
119
|
+
if (new Set(ids).size !== ids.length) return "A disposition names the same incident more than once";
|
|
120
|
+
if (typeof event.reason !== "string" || !event.reason.trim()) return "A disposition requires a reason";
|
|
121
|
+
if ((event.disposition === "recovered" || event.disposition === "superseded") && (typeof event.evidence !== "string" || !event.evidence.trim())) {
|
|
122
|
+
return `A ${event.disposition} disposition requires evidence`;
|
|
123
|
+
}
|
|
124
|
+
for (const id of ids) {
|
|
125
|
+
const current = findIncident(state, id);
|
|
126
|
+
if (!current) {
|
|
127
|
+
return Object.hasOwn(state.history ?? {}, id) || Object.hasOwn(state.resolved ?? {}, id)
|
|
128
|
+
? `Incident ${id} is already disposed` : `Unknown incident ${id}`;
|
|
129
|
+
}
|
|
130
|
+
if (current.category === "observation-incomplete") return `Incident ${id} is an observation gap; it cannot be disposed`;
|
|
131
|
+
if (current.status !== "unresolved") return `Incident ${id} is already disposed (${failureLabel(current).toLowerCase()})`;
|
|
132
|
+
if (event.disposition === "open" && current.disposition?.disposition === "open") return `Incident ${id} is already open`;
|
|
133
|
+
}
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
|
|
51
137
|
/** Pure transition. Lifecycle is deliberately not an input or an output. */
|
|
52
138
|
export function reduceFailure(state: FailureState, event: FailureEvent, observedAt: number): FailureState {
|
|
53
139
|
if (state.seen.includes(event.id)) return state;
|
|
140
|
+
// Invalid dispositions fail closed: nothing changes and the id is not consumed.
|
|
141
|
+
if (event.kind === "disposition" && validateDisposition(state, event)) return state;
|
|
54
142
|
const next: FailureState = { ...state, seen: [...state.seen, event.id],
|
|
55
143
|
observations: { ...state.observations }, delivered: { ...state.delivered } };
|
|
56
144
|
if (event.kind === "delivered") {
|
|
57
145
|
for (const id of event.incidents ?? []) next.delivered = { ...next.delivered, [id]: observedAt };
|
|
58
146
|
return next;
|
|
59
147
|
}
|
|
148
|
+
if (event.kind === "disposition") {
|
|
149
|
+
const at = event.at ?? observedAt;
|
|
150
|
+
const record: DispositionRecord = { eventId: event.id, disposition: event.disposition!, incidents: [...event.incidents!],
|
|
151
|
+
reason: text(event.reason, ""), ...(event.evidence ? { evidence: text(event.evidence, "") } : {}), at };
|
|
152
|
+
for (const id of record.incidents) {
|
|
153
|
+
const [key, current] = Object.entries(next.observations).find(([, item]) => item.id === id)!;
|
|
154
|
+
const status = record.disposition === "recovered" ? "resolved" : record.disposition === "superseded" ? "superseded"
|
|
155
|
+
: record.disposition === "expected" ? "expected" : current.status;
|
|
156
|
+
next.observations[key] = { ...current, status, disposition: record,
|
|
157
|
+
...(status === "resolved" || status === "superseded" ? { resolvedAt: at } : {}) };
|
|
158
|
+
if (status === "resolved" || status === "superseded") next.resolved = { ...next.resolved, [id]: at };
|
|
159
|
+
}
|
|
160
|
+
next.dispositions = [...(state.dispositions ?? []), record];
|
|
161
|
+
return next;
|
|
162
|
+
}
|
|
60
163
|
const key = failureIdentity(event.operation);
|
|
61
164
|
const previous = state.observations[key];
|
|
62
165
|
if (event.kind === "recovered") {
|
|
63
|
-
|
|
166
|
+
// Only an unresolved incident recovers; an expected one keeps its classification.
|
|
167
|
+
if (previous && previous.status === "unresolved" && event.incidents?.includes(previous.id)) {
|
|
64
168
|
next.observations[key] = { ...previous, status: "resolved", resolvedAt: event.at ?? observedAt };
|
|
65
169
|
next.resolved = { ...state.resolved, [previous.id]: event.at ?? observedAt };
|
|
66
170
|
}
|
|
67
171
|
return next;
|
|
68
172
|
}
|
|
69
|
-
const active = previous && previous.status !== "resolved" &&
|
|
173
|
+
const active = previous && previous.status !== "resolved" && previous.status !== "superseded" &&
|
|
70
174
|
!(previous.status === "expected" && !event.expected);
|
|
175
|
+
if (previous && !active) next.history = { ...state.history, [previous.id]: previous };
|
|
71
176
|
next.observations[key] = {
|
|
72
177
|
id: active ? previous.id : event.id, operation: event.operation,
|
|
73
178
|
status: active ? previous.status : event.expected ? "expected" : "unresolved",
|
|
@@ -76,30 +181,386 @@ export function reduceFailure(state: FailureState, event: FailureEvent, observed
|
|
|
76
181
|
firstObservedAt: active ? previous.firstObservedAt : observedAt,
|
|
77
182
|
lastObservedAt: observedAt, lastSequence: next.seen.length, at: event.at,
|
|
78
183
|
count: active ? previous.count + 1 : 1,
|
|
184
|
+
...(active && previous.disposition ? { disposition: previous.disposition } : {}),
|
|
79
185
|
};
|
|
80
186
|
return next;
|
|
81
187
|
}
|
|
188
|
+
function priority(x: FailureObservation): number {
|
|
189
|
+
if (x.category === "observation-incomplete") return 0;
|
|
190
|
+
if (x.status === "expected") return 3;
|
|
191
|
+
return requiresAction(x) ? 1 : 2;
|
|
192
|
+
}
|
|
193
|
+
/** Current (not recovered or superseded) incidents in priority order. History is excluded. */
|
|
82
194
|
export function activeFailures(state: FailureState): FailureObservation[] {
|
|
83
|
-
|
|
84
|
-
return Object.values(state.observations).filter((x) => x.status !== "resolved")
|
|
195
|
+
return Object.values(state.observations).filter((x) => x.status !== "resolved" && x.status !== "superseded")
|
|
85
196
|
.sort((a, b) => priority(a) - priority(b) || (b.lastSequence ?? 0) - (a.lastSequence ?? 0) || b.lastObservedAt - a.lastObservedAt);
|
|
86
197
|
}
|
|
198
|
+
/** Every incident ever reduced, current and closed. Nothing is removed. */
|
|
199
|
+
export function failureHistory(state: FailureState): FailureObservation[] {
|
|
200
|
+
return [...Object.values(state.history ?? {}), ...Object.values(state.observations)];
|
|
201
|
+
}
|
|
202
|
+
export interface FailureCounts {
|
|
203
|
+
actionRequired: number;
|
|
204
|
+
unclassified: number;
|
|
205
|
+
expected: number;
|
|
206
|
+
incomplete: number;
|
|
207
|
+
recovered: number;
|
|
208
|
+
superseded: number;
|
|
209
|
+
}
|
|
210
|
+
/** Separate facts: current actionability, retained classification, and closed history. */
|
|
211
|
+
export function failureCounts(state: FailureState): FailureCounts {
|
|
212
|
+
const counts: FailureCounts = { actionRequired: 0, unclassified: 0, expected: 0, incomplete: 0, recovered: 0, superseded: 0 };
|
|
213
|
+
for (const x of failureHistory(state)) {
|
|
214
|
+
if (x.status === "resolved") counts.recovered += 1;
|
|
215
|
+
else if (x.status === "superseded") counts.superseded += 1;
|
|
216
|
+
else if (x.category === "observation-incomplete") counts.incomplete += 1;
|
|
217
|
+
else if (x.status === "expected") counts.expected += 1;
|
|
218
|
+
else if (requiresAction(x)) counts.actionRequired += 1;
|
|
219
|
+
else counts.unclassified += 1;
|
|
220
|
+
}
|
|
221
|
+
return counts;
|
|
222
|
+
}
|
|
223
|
+
const INCIDENT_CURSOR_PREFIX = "i1.";
|
|
224
|
+
const encoder = new TextEncoder();
|
|
225
|
+
|
|
226
|
+
function failureRow(x: FailureObservation): string {
|
|
227
|
+
const time = x.at === undefined ? `observed ${new Date(x.firstObservedAt).toISOString()}` : new Date(x.at).toISOString();
|
|
228
|
+
const disposition = x.disposition ? ` · ${x.disposition.disposition}: ${text(x.disposition.reason, "")}` : "";
|
|
229
|
+
return `${failureLabel(x)} · ${time} · ${x.summary}${x.count > 1 ? ` (${x.count} occurrences)` : ""}${disposition}${x.evidence ? ` · evidence: ${text(x.evidence, "")}` : ""}`;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Every active incident as a priority row. Consumers page these; they are not a lossy summary. */
|
|
233
|
+
export function formatFailureLines(state: FailureState): string[] {
|
|
234
|
+
return activeFailures(state).map(failureRow);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function closedHistoryLine(state: FailureState): string | undefined {
|
|
238
|
+
const counts = failureCounts(state);
|
|
239
|
+
if (!counts.recovered && !counts.superseded) return undefined;
|
|
240
|
+
const parts = [counts.recovered ? `${counts.recovered} recovered` : "", counts.superseded ? `${counts.superseded} superseded` : ""].filter(Boolean);
|
|
241
|
+
return `Closed incidents retained in history: ${parts.join(" · ")}.`;
|
|
242
|
+
}
|
|
243
|
+
|
|
87
244
|
/** Shared priority text, placed BEFORE assistant progress on every consumer surface. */
|
|
88
245
|
export function formatFailureSummary(state: FailureState): string {
|
|
89
|
-
const
|
|
90
|
-
if (!
|
|
91
|
-
const rows =
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
return `${label} · ${time} · ${x.summary}${x.count > 1 ? ` (${x.count} occurrences)` : ""}${x.evidence ? ` · evidence: ${text(x.evidence, "")}` : ""}`;
|
|
96
|
-
});
|
|
97
|
-
if (failures.length > 5) rows.push(`${failures.length - 5} additional active failure observations retained in the failure journal.`);
|
|
246
|
+
const lines = formatFailureLines(state);
|
|
247
|
+
if (!lines.length) return "";
|
|
248
|
+
const rows = lines.slice(0, 5);
|
|
249
|
+
if (lines.length > 5) rows.push(`${lines.length - 5} additional active failure observations retained in the failure journal.`);
|
|
250
|
+
const history = closedHistoryLine(state);
|
|
251
|
+
if (history) rows.push(history);
|
|
98
252
|
return rows.join("\n");
|
|
99
253
|
}
|
|
100
|
-
|
|
254
|
+
|
|
255
|
+
/** Incident rows for exactly `incidents`, priority order. Unknown ids fail closed. */
|
|
256
|
+
export function pendingAttentionRows(state: FailureState, incidents: readonly string[]): string[] {
|
|
257
|
+
const wanted = new Set(incidents);
|
|
258
|
+
const rows = activeFailures(state).filter((x) => wanted.has(x.id));
|
|
259
|
+
if (rows.length !== wanted.size) throw new Error("Failure incident evidence is unavailable; defer notification delivery");
|
|
260
|
+
return rows.map(failureRow);
|
|
261
|
+
}
|
|
262
|
+
/** Count of other active incidents, which a notification references but never re-lists. */
|
|
263
|
+
export function pendingAttentionNote(state: FailureState, incidents: readonly string[]): string | undefined {
|
|
264
|
+
const wanted = new Set(incidents);
|
|
265
|
+
const others = activeFailures(state).filter((x) => !wanted.has(x.id)).length;
|
|
266
|
+
return others > 0 ? `${others} other active failure observation${others === 1 ? " was" : "s were"} reported earlier or not actionable; not repeated here.` : undefined;
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* Attention text for exactly `incidents` (normally `pending.incidents`). Earlier delivered or
|
|
270
|
+
* non-actionable incidents are counted, never re-listed. Unknown ids fail closed.
|
|
271
|
+
*/
|
|
272
|
+
export function formatPendingAttention(state: FailureState, incidents: readonly string[], options: { maxRows?: number } = {}): string {
|
|
273
|
+
const rows = pendingAttentionRows(state, incidents);
|
|
274
|
+
const max = options.maxRows ?? 5;
|
|
275
|
+
const lines = rows.slice(0, max);
|
|
276
|
+
if (rows.length > max) lines.push(`${rows.length - max} additional pending incidents retained in the failure journal.`);
|
|
277
|
+
const note = pendingAttentionNote(state, incidents);
|
|
278
|
+
if (note) lines.push(note);
|
|
279
|
+
return lines.join("\n");
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
export interface TerminalFailureParts {
|
|
283
|
+
/** Actionable and observation-incomplete incidents among `incidents`, one row each. */
|
|
284
|
+
rows: string[];
|
|
285
|
+
/** Counts and separate facts: earlier-reported, unclassified, expected, closed history, correctness. */
|
|
286
|
+
notes: string[];
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* Terminal/completion facts: actionable and incomplete incidents from `incidents` as rows,
|
|
290
|
+
* unclassified agent tool failures as a count. Lifecycle is reported by the caller, separately.
|
|
291
|
+
*/
|
|
292
|
+
export function terminalFailureParts(state: FailureState, incidents: readonly string[] = []): TerminalFailureParts {
|
|
293
|
+
const wanted = new Set(incidents);
|
|
294
|
+
const reportable = (x: FailureObservation) => requiresAction(x) || x.category === "observation-incomplete";
|
|
295
|
+
const rows = activeFailures(state).filter((x) => wanted.has(x.id) && reportable(x)).map(failureRow);
|
|
296
|
+
const notes: string[] = [];
|
|
297
|
+
const earlier = activeFailures(state).filter((x) => !wanted.has(x.id) && reportable(x)).length;
|
|
298
|
+
if (earlier) notes.push(`${earlier} actionable incident${earlier === 1 ? " was" : "s were"} reported earlier; not repeated here.`);
|
|
299
|
+
const counts = failureCounts(state);
|
|
300
|
+
if (counts.unclassified) notes.push(`${counts.unclassified} earlier tool failure${counts.unclassified === 1 ? "" : "s"} remain${counts.unclassified === 1 ? "s" : ""} unclassified.`);
|
|
301
|
+
if (counts.expected) notes.push(`${counts.expected} expected failure${counts.expected === 1 ? "" : "s"} recorded.`);
|
|
302
|
+
const history = closedHistoryLine(state);
|
|
303
|
+
if (history) notes.push(history);
|
|
304
|
+
if (counts.unclassified || counts.actionRequired) notes.push("Work correctness was not inferred from lifecycle alone.");
|
|
305
|
+
return { rows, notes };
|
|
306
|
+
}
|
|
307
|
+
export function formatTerminalFailureFacts(state: FailureState, incidents: readonly string[] = []): string {
|
|
308
|
+
const { rows, notes } = terminalFailureParts(state, incidents);
|
|
309
|
+
const lines = rows.slice(0, 5);
|
|
310
|
+
if (rows.length > 5) lines.push(`${rows.length - 5} additional actionable incidents retained in the failure journal.`);
|
|
311
|
+
return [...lines, ...notes].join("\n");
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
export interface FailureIncidentPage {
|
|
315
|
+
text: string;
|
|
316
|
+
total: number;
|
|
317
|
+
/** Rows completed on this page (a row split across pages counts where it ends). */
|
|
318
|
+
represented: number;
|
|
319
|
+
/** Rows not fully shown through this page, including a partially shown row. */
|
|
320
|
+
omitted: number;
|
|
321
|
+
cursor: string;
|
|
322
|
+
nextCursor: string;
|
|
323
|
+
hasMore: boolean;
|
|
324
|
+
reset?: "stale-cursor" | "source-replaced";
|
|
325
|
+
/** The first line continues a row begun on an earlier page. */
|
|
326
|
+
startsPartial: boolean;
|
|
327
|
+
/** The last line is the start of a row that continues on the next page. */
|
|
328
|
+
endsPartial: boolean;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Signature of every observation's reportable state. A change here is a
|
|
333
|
+
* failure-only change even when log bytes are unchanged; receipts and replay
|
|
334
|
+
* bookkeeping do not change it.
|
|
335
|
+
*/
|
|
336
|
+
export function failureRevision(state: FailureState): string {
|
|
337
|
+
return failureIdentity(Object.values(state.observations)
|
|
338
|
+
.map((item) => [item.id, item.status, item.category, item.count, item.lastSequence ?? 0, item.summary, item.evidence ?? ""])
|
|
339
|
+
.sort((a, b) => String(a[0]).localeCompare(String(b[0]))));
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function incidentRevision(state: FailureState): string {
|
|
343
|
+
return failureIdentity(activeFailures(state).map((item) => [item.id, item.status, item.count, item.summary, item.evidence ?? ""])).slice(0, 16);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/** Cursors carry a digest of their resource/scope, not the scope text. */
|
|
347
|
+
function resourceTag(resource: string | undefined): string | undefined {
|
|
348
|
+
return resource === undefined ? undefined : createHash("sha256").update(resource).digest("base64url").slice(0, 16);
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
interface IncidentCursor { o: number; b: number; v: string; n: number; r?: string }
|
|
352
|
+
|
|
353
|
+
function encodeIncidentCursor(cursor: IncidentCursor): string {
|
|
354
|
+
return INCIDENT_CURSOR_PREFIX + Buffer.from(JSON.stringify({ k: "i", o: cursor.o, ...(cursor.b ? { b: cursor.b } : {}),
|
|
355
|
+
v: cursor.v, n: cursor.n, ...(cursor.r !== undefined ? { r: cursor.r } : {}) }), "utf8").toString("base64url");
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
function decodeIncidentCursor(cursor: string | undefined): IncidentCursor | undefined {
|
|
359
|
+
if (!cursor || !cursor.startsWith(INCIDENT_CURSOR_PREFIX)) return undefined;
|
|
360
|
+
try {
|
|
361
|
+
const parsed = JSON.parse(Buffer.from(cursor.slice(INCIDENT_CURSOR_PREFIX.length), "base64url").toString("utf8")) as { k?: string; o?: number; b?: number; v?: string; n?: number; r?: string };
|
|
362
|
+
if (parsed?.k === "i" && typeof parsed.v === "string") {
|
|
363
|
+
return { o: Math.max(0, Math.floor(parsed.o ?? 0)), b: Math.max(0, Math.floor(parsed.b ?? 0)), v: parsed.v,
|
|
364
|
+
n: Math.max(0, Math.floor(parsed.n ?? 0)), ...(typeof parsed.r === "string" ? { r: parsed.r } : {}) };
|
|
365
|
+
}
|
|
366
|
+
} catch { /* stale */ }
|
|
367
|
+
return undefined;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
export function isIncidentCursor(cursor: string | undefined): boolean {
|
|
371
|
+
return Boolean(cursor?.startsWith(INCIDENT_CURSOR_PREFIX));
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
function utf8Length(value: string): number {
|
|
375
|
+
return encoder.encode(value).byteLength;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/** Largest prefix of `bytes` within `room` that does not split a code point. */
|
|
379
|
+
function utf8Prefix(bytes: Uint8Array, room: number): number {
|
|
380
|
+
if (room <= 0) return 0;
|
|
381
|
+
if (room >= bytes.length) return bytes.length;
|
|
382
|
+
let end = room;
|
|
383
|
+
while (end > 0 && (bytes[end]! & 0xc0) === 0x80) end -= 1;
|
|
384
|
+
return end;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
export interface IncidentPageRequest {
|
|
388
|
+
cursor?: string;
|
|
389
|
+
maxBytes?: number;
|
|
390
|
+
/** Resource/scope bound into cursors; a cursor minted for another scope resets. */
|
|
391
|
+
resource?: string;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* Caller-owned incident pages. Whole rows are preferred; a row larger than the
|
|
396
|
+
* page is split at a code-point boundary and resumes at that byte, so pages
|
|
397
|
+
* reconstruct formatFailureLines() exactly (join with "\n" except after a page
|
|
398
|
+
* that `endsPartial`). A page never exceeds `maxBytes`.
|
|
399
|
+
*/
|
|
400
|
+
export function pageFailureIncidents(state: FailureState, request: IncidentPageRequest = {}): FailureIncidentPage {
|
|
401
|
+
const lines = formatFailureLines(state);
|
|
402
|
+
const revision = incidentRevision(state);
|
|
403
|
+
const total = lines.length;
|
|
404
|
+
const resource = resourceTag(request.resource);
|
|
405
|
+
let offset = 0;
|
|
406
|
+
let byte = 0;
|
|
407
|
+
let reset: FailureIncidentPage["reset"];
|
|
408
|
+
if (request.cursor) {
|
|
409
|
+
const parsed = decodeIncidentCursor(request.cursor);
|
|
410
|
+
if (!parsed || parsed.r !== resource) reset = "stale-cursor";
|
|
411
|
+
else if (parsed.v !== revision) reset = "source-replaced";
|
|
412
|
+
else { offset = Math.min(total, parsed.o); byte = offset < total ? parsed.b : 0; }
|
|
413
|
+
}
|
|
414
|
+
const maxBytes = Number.isFinite(request.maxBytes) && (request.maxBytes ?? -1) >= 0
|
|
415
|
+
? Math.floor(request.maxBytes as number)
|
|
416
|
+
: 2 * 1024;
|
|
417
|
+
const mint = (o: number, b: number) => encodeIncidentCursor({ o, b, v: revision, n: total, ...(resource !== undefined ? { r: resource } : {}) });
|
|
418
|
+
const parts: string[] = [];
|
|
419
|
+
let used = 0;
|
|
420
|
+
let nextRow = offset;
|
|
421
|
+
let nextByte = byte;
|
|
422
|
+
let endsPartial = false;
|
|
423
|
+
for (let i = offset; i < total; i += 1) {
|
|
424
|
+
const encoded = encoder.encode(lines[i]!);
|
|
425
|
+
const from = i === offset ? Math.min(byte, encoded.length) : 0;
|
|
426
|
+
const rest = encoded.subarray(from);
|
|
427
|
+
const sep = parts.length ? 1 : 0;
|
|
428
|
+
if (used + sep + rest.length <= maxBytes) {
|
|
429
|
+
parts.push(Buffer.from(rest).toString("utf8"));
|
|
430
|
+
used += sep + rest.length;
|
|
431
|
+
nextRow = i + 1;
|
|
432
|
+
nextByte = 0;
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
if (parts.length === 0) {
|
|
436
|
+
const cut = utf8Prefix(rest, maxBytes);
|
|
437
|
+
if (cut > 0) {
|
|
438
|
+
parts.push(Buffer.from(rest.subarray(0, cut)).toString("utf8"));
|
|
439
|
+
used += cut;
|
|
440
|
+
nextRow = i;
|
|
441
|
+
nextByte = from + cut;
|
|
442
|
+
endsPartial = true;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
break;
|
|
446
|
+
}
|
|
447
|
+
const startsPartial = byte > 0 && offset < total;
|
|
448
|
+
const represented = Math.max(0, nextRow - offset);
|
|
449
|
+
return {
|
|
450
|
+
text: parts.join("\n"),
|
|
451
|
+
total,
|
|
452
|
+
represented,
|
|
453
|
+
omitted: Math.max(0, total - nextRow),
|
|
454
|
+
cursor: mint(offset, byte),
|
|
455
|
+
nextCursor: mint(nextRow, nextByte),
|
|
456
|
+
hasMore: nextRow < total,
|
|
457
|
+
...(reset ? { reset } : {}),
|
|
458
|
+
startsPartial,
|
|
459
|
+
endsPartial,
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
export function incidentCursorAt(state: FailureState, offset: number, resource?: string): string {
|
|
464
|
+
const lines = formatFailureLines(state);
|
|
465
|
+
const tag = resourceTag(resource);
|
|
466
|
+
return encodeIncidentCursor({ o: Math.max(0, Math.floor(offset)), b: 0, v: incidentRevision(state), n: lines.length,
|
|
467
|
+
...(tag !== undefined ? { r: tag } : {}) });
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
const MIN_PARTIAL_ROW_BYTES = 96;
|
|
471
|
+
|
|
472
|
+
export interface IncidentSummary {
|
|
473
|
+
text: string;
|
|
474
|
+
total: number;
|
|
475
|
+
/** Rows fully shown. */
|
|
476
|
+
represented: number;
|
|
477
|
+
/** Rows not fully shown; retrievable from `nextCursor`. */
|
|
478
|
+
omitted: number;
|
|
479
|
+
nextCursor?: string;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* The shared priority failure section for a byte budget. When every active
|
|
484
|
+
* incident fits, the rows are returned unchanged. Otherwise a count line leads:
|
|
485
|
+
* total, fully shown, omitted, and the incident cursor that resumes at the
|
|
486
|
+
* first byte not shown. Counts survive any budget that fits the count line.
|
|
487
|
+
*/
|
|
488
|
+
export function formatIncidentSummary(state: FailureState, options: { maxBytes: number; resource?: string; retrieval?: string }): IncidentSummary {
|
|
489
|
+
const maxBytes = Math.max(0, Math.floor(options.maxBytes));
|
|
490
|
+
const whole = pageFailureIncidents(state, { maxBytes, resource: options.resource });
|
|
491
|
+
if (whole.total === 0) return { text: "", total: 0, represented: 0, omitted: 0 };
|
|
492
|
+
if (!whole.hasMore) return { text: whole.text, total: whole.total, represented: whole.represented, omitted: 0 };
|
|
493
|
+
const header = (page: FailureIncidentPage): string =>
|
|
494
|
+
`${page.total} active failure observation${page.total === 1 ? "" : "s"} · ${page.represented} shown · ${page.omitted} omitted` +
|
|
495
|
+
` · incidentCursor=${page.nextCursor}${options.retrieval ? ` (${options.retrieval})` : ""}`;
|
|
496
|
+
let rowBudget = maxBytes - utf8Length(header(whole)) - 1;
|
|
497
|
+
for (let attempt = 0; attempt < 6; attempt += 1) {
|
|
498
|
+
let page = pageFailureIncidents(state, { maxBytes: Math.max(0, rowBudget), resource: options.resource });
|
|
499
|
+
// A few bytes of a clipped row are noise; show the count line alone and
|
|
500
|
+
// let the cursor start at that row.
|
|
501
|
+
if (page.represented === 0 && page.endsPartial && utf8Length(page.text) < MIN_PARTIAL_ROW_BYTES) {
|
|
502
|
+
page = pageFailureIncidents(state, { maxBytes: 0, resource: options.resource });
|
|
503
|
+
}
|
|
504
|
+
const line = header(page);
|
|
505
|
+
const text = page.text ? `${line}\n${page.text}` : line;
|
|
506
|
+
const overflow = utf8Length(text) - maxBytes;
|
|
507
|
+
if (overflow <= 0) {
|
|
508
|
+
return { text, total: page.total, represented: page.represented, omitted: page.omitted, nextCursor: page.nextCursor };
|
|
509
|
+
}
|
|
510
|
+
if (rowBudget <= 0) break;
|
|
511
|
+
rowBudget -= overflow;
|
|
512
|
+
}
|
|
513
|
+
// Not even the count line with its cursor fits. Never emit a clipped cursor:
|
|
514
|
+
// keep the exact counts and drop the cursor token whole, saying so.
|
|
515
|
+
const empty = pageFailureIncidents(state, { maxBytes: 0, resource: options.resource });
|
|
516
|
+
const withCursor = header(empty);
|
|
517
|
+
if (utf8Length(withCursor) <= maxBytes) {
|
|
518
|
+
return { text: withCursor, total: empty.total, represented: 0, omitted: empty.total, nextCursor: empty.nextCursor };
|
|
519
|
+
}
|
|
520
|
+
const plural = empty.total === 1 ? "" : "s";
|
|
521
|
+
for (const candidate of [
|
|
522
|
+
`${empty.total} active failure observation${plural} · 0 shown · ${empty.total} omitted · incident cursor not shown (page too small; retry with a larger maxBytes${options.retrieval ? ` or ${options.retrieval}` : ""})`,
|
|
523
|
+
`${empty.total} active failure observation${plural} · 0 shown · ${empty.total} omitted · incident cursor not shown (page too small)`,
|
|
524
|
+
`${empty.total} active failure observation${plural} (page too small for details)`,
|
|
525
|
+
]) {
|
|
526
|
+
if (utf8Length(candidate) <= maxBytes) return { text: candidate, total: empty.total, represented: 0, omitted: empty.total };
|
|
527
|
+
}
|
|
528
|
+
return { text: "", total: empty.total, represented: 0, omitted: empty.total };
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* Terminal failure section for a byte budget (#315): lifecycle-independent facts. Actionable and
|
|
532
|
+
* observation-incomplete incidents (which lead the priority order) are shown as whole rows;
|
|
533
|
+
* unclassified, expected, and closed incidents are counts. When any active row is not shown, a
|
|
534
|
+
* count line gives the exact total, shown, omitted, and an incident cursor at the first unshown row.
|
|
535
|
+
*/
|
|
536
|
+
export function formatTerminalIncidentSummary(state: FailureState, options: { maxBytes: number; resource?: string; retrieval?: string }): IncidentSummary {
|
|
537
|
+
const active = activeFailures(state);
|
|
538
|
+
const total = active.length;
|
|
539
|
+
const { notes } = terminalFailureParts(state, active.map((x) => x.id));
|
|
540
|
+
if (total === 0 && notes.length === 0) return { text: "", total: 0, represented: 0, omitted: 0 };
|
|
541
|
+
const reportable = active.filter((x) => requiresAction(x) || x.category === "observation-incomplete").length;
|
|
542
|
+
const lines = formatFailureLines(state).slice(0, reportable);
|
|
543
|
+
const maxBytes = Math.max(0, Math.floor(options.maxBytes));
|
|
544
|
+
const header = (shown: number): string | undefined => shown >= total ? undefined :
|
|
545
|
+
`${total} active failure observation${total === 1 ? "" : "s"} · ${shown} shown · ${total - shown} omitted` +
|
|
546
|
+
` · incidentCursor=${incidentCursorAt(state, shown, options.resource)}${options.retrieval ? ` (${options.retrieval})` : ""}`;
|
|
547
|
+
const render = (shown: number) => [header(shown), ...lines.slice(0, shown), ...notes].filter((x): x is string => Boolean(x)).join("\n");
|
|
548
|
+
let shown = lines.length;
|
|
549
|
+
while (shown > 0 && utf8Length(render(shown)) > maxBytes) shown -= 1;
|
|
550
|
+
const omitted = total - shown;
|
|
551
|
+
return { text: render(shown), total, represented: shown, omitted, ...(omitted > 0 ? { nextCursor: incidentCursorAt(state, shown, options.resource) } : {}) };
|
|
552
|
+
}
|
|
553
|
+
/**
|
|
554
|
+
* Incidents due for a notification. Terminal: every current unresolved incident not yet delivered
|
|
555
|
+
* (reported once, then receipted). Running: observation gaps immediately (unless the consumer
|
|
556
|
+
* defers them to its terminal/health callback), actionable incidents after the grace period;
|
|
557
|
+
* a single agent tool failure is left to the agent that owns it.
|
|
558
|
+
*/
|
|
559
|
+
export function pendingFailureAttention(state: FailureState, now: number,
|
|
560
|
+
options: { terminal?: boolean; graceMs?: number; deferObservationGaps?: boolean } = {}): { key: string; incidents: string[]; summary: string } | undefined {
|
|
101
561
|
const due = activeFailures(state).filter((x) => x.status === "unresolved" && !Object.hasOwn(state.delivered, x.id) &&
|
|
102
|
-
(options.terminal || x.category === "observation-incomplete"
|
|
562
|
+
(options.terminal || (x.category === "observation-incomplete" ? !options.deferObservationGaps
|
|
563
|
+
: requiresAction(x) && now - x.firstObservedAt >= (options.graceMs ?? 60_000))));
|
|
103
564
|
if (!due.length) return undefined;
|
|
104
565
|
const incidents = due.map((x) => x.id).sort();
|
|
105
566
|
return { key: failureIdentity(incidents), incidents,
|
|
@@ -153,11 +614,12 @@ function readStoredState(path: string): FailureState {
|
|
|
153
614
|
const e = row.event;
|
|
154
615
|
if (!e || typeof e.id !== "string" || !e.id || typeof e.operation !== "string" || !e.operation ||
|
|
155
616
|
(e.expected !== undefined && typeof e.expected !== "boolean") ||
|
|
156
|
-
!["failure", "incomplete", "recovered", "delivered"].includes(e.kind) ||
|
|
617
|
+
!["failure", "incomplete", "recovered", "delivered", "disposition"].includes(e.kind) ||
|
|
618
|
+
(e.disposition !== undefined && !INCIDENT_DISPOSITIONS.includes(e.disposition)) ||
|
|
157
619
|
!Number.isFinite(row.observedAt) || Math.abs(row.observedAt) > 8.64e15 ||
|
|
158
620
|
(e.at !== undefined && (!Number.isFinite(e.at) || Math.abs(e.at) > 8.64e15)) ||
|
|
159
621
|
(e.incidents !== undefined && (!Array.isArray(e.incidents) || !e.incidents.every((id: unknown) => typeof id === "string"))) ||
|
|
160
|
-
[e.summary, e.category, e.evidence].some((v) => v !== undefined && typeof v !== "string")) throw new Error("invalid record");
|
|
622
|
+
[e.summary, e.category, e.evidence, e.reason].some((v) => v !== undefined && typeof v !== "string")) throw new Error("invalid record");
|
|
161
623
|
state = reduceFailure(state, e, row.observedAt);
|
|
162
624
|
} catch { state = storageProblem(state, "Failure journal contains unreadable records; observations may be incomplete"); }
|
|
163
625
|
}
|
|
@@ -175,10 +637,12 @@ export function observeFailures(path: string, events: readonly FailureEvent[], n
|
|
|
175
637
|
if (state.seen.includes(raw.id)) continue;
|
|
176
638
|
if (raw.kind === "recovered") {
|
|
177
639
|
const prior = state.observations[failureIdentity(raw.operation)];
|
|
178
|
-
if (!prior || prior.status
|
|
640
|
+
if (!prior || prior.status !== "unresolved" || !raw.incidents?.includes(prior.id)) continue;
|
|
179
641
|
}
|
|
642
|
+
// Rejected dispositions are never journaled or marked seen.
|
|
643
|
+
if (raw.kind === "disposition" && validateDisposition(state, raw)) continue;
|
|
180
644
|
const event = { ...raw, ...(raw.summary ? { summary: text(raw.summary, "") } : {}),
|
|
181
|
-
...(raw.evidence ? { evidence: text(raw.evidence, "") } : {}) };
|
|
645
|
+
...(raw.evidence ? { evidence: text(raw.evidence, "") } : {}), ...(raw.reason ? { reason: text(raw.reason, "") } : {}) };
|
|
182
646
|
try {
|
|
183
647
|
if (pendingWrites.has(path)) throw new Error("Earlier evidence is awaiting persistence");
|
|
184
648
|
appendRecord(path, { event, observedAt: now });
|
|
@@ -203,3 +667,18 @@ export function failureAttentionHandled(state: FailureState, incidents: readonly
|
|
|
203
667
|
export function markFailureAttentionDelivered(path: string, pending: { key: string; incidents: string[] }, at = Date.now()): FailureState {
|
|
204
668
|
return observeFailures(path, [{ id: `delivered:${pending.key}`, operation: "attention-delivery", kind: "delivered", incidents: pending.incidents }], at);
|
|
205
669
|
}
|
|
670
|
+
export interface DispositionResult { accepted: boolean; error?: string; state: FailureState }
|
|
671
|
+
/**
|
|
672
|
+
* Append one explicit disposition. Validation happens before the journal write; a rejected
|
|
673
|
+
* request (unknown, already disposed, missing reason or evidence) writes nothing. Replaying an
|
|
674
|
+
* already accepted event id is an idempotent success.
|
|
675
|
+
*/
|
|
676
|
+
export function disposeIncidents(path: string, event: FailureEvent, now = Date.now()): DispositionResult {
|
|
677
|
+
const state = readFailureState(path);
|
|
678
|
+
if (event.kind !== "disposition") return { accepted: false, error: "Not a disposition event", state };
|
|
679
|
+
if (state.seen.includes(event.id)) return { accepted: true, state };
|
|
680
|
+
const error = validateDisposition(state, event);
|
|
681
|
+
if (error) return { accepted: false, error, state };
|
|
682
|
+
const next = observeFailures(path, [event], now);
|
|
683
|
+
return next.seen.includes(event.id) ? { accepted: true, state: next } : { accepted: false, error: "Disposition was not recorded", state: next };
|
|
684
|
+
}
|