pi-better-background-tasks 0.2.20 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -2
- package/package.json +1 -1
- package/src/failures.ts +101 -6
- package/src/index.ts +5 -0
- package/src/logs.ts +51 -7
- package/src/navigator-provider.ts +4 -3
- package/src/output.ts +783 -0
- package/src/process.ts +95 -8
- package/src/registry.ts +84 -7
- package/src/runtime.ts +228 -33
- package/src/sandbox.ts +6 -4
- package/src/shared-callback-batcher.ts +358 -29
- package/src/shared-failure-observations.ts +833 -23
- package/src/shared-log-utils.ts +1139 -2
- package/src/shared-navigator.ts +12 -10
- package/src/shared-sandbox-core.ts +660 -17
- package/src/tools.ts +201 -156
- package/src/types.ts +23 -0
|
@@ -1,26 +1,49 @@
|
|
|
1
1
|
// Generated from packages/failure-observations/index.ts. Do not edit directly.
|
|
2
|
-
import { appendFileSync, closeSync, existsSync, openSync, readFileSync, mkdirSync } from "node:fs";
|
|
2
|
+
import { appendFileSync, closeSync, existsSync, openSync, readFileSync, mkdirSync, statSync } from "node:fs";
|
|
3
3
|
import { dirname } from "node:path";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Explicit, append-only incident dispositions (#315).
|
|
8
|
+
* - recovered: the same operation later passed (verified by the adapter).
|
|
9
|
+
* - superseded: a different verification or remediation established the outcome.
|
|
10
|
+
* - expected: the non-success result was intentional.
|
|
11
|
+
* - open: explicitly classified as still needing action; the incident stays actionable.
|
|
12
|
+
*/
|
|
13
|
+
export type IncidentDisposition = "recovered" | "superseded" | "expected" | "open";
|
|
14
|
+
export const INCIDENT_DISPOSITIONS: readonly IncidentDisposition[] = Object.freeze(["recovered", "superseded", "expected", "open"]);
|
|
15
|
+
|
|
6
16
|
export interface FailureEvent {
|
|
7
17
|
/** Stable source event identity: replay must reuse this id. */
|
|
8
18
|
id: string;
|
|
9
19
|
operation: string;
|
|
10
|
-
kind: "failure" | "recovered" | "incomplete" | "delivered";
|
|
20
|
+
kind: "failure" | "recovered" | "incomplete" | "delivered" | "disposition";
|
|
11
21
|
/** Event time when known; never fabricate it from log mtime. */
|
|
12
22
|
at?: number;
|
|
13
23
|
summary?: string;
|
|
14
24
|
category?: string;
|
|
15
25
|
evidence?: string;
|
|
16
26
|
expected?: boolean;
|
|
17
|
-
/** Recovery/delivery must name the incidents it resolves/delivers. */
|
|
27
|
+
/** Recovery/delivery/disposition must name the incidents it resolves/delivers/classifies. */
|
|
18
28
|
incidents?: string[];
|
|
29
|
+
/** Disposition events only. */
|
|
30
|
+
disposition?: IncidentDisposition;
|
|
31
|
+
/** Disposition events only: why the incidents are classified this way. */
|
|
32
|
+
reason?: string;
|
|
33
|
+
}
|
|
34
|
+
export interface DispositionRecord {
|
|
35
|
+
eventId: string;
|
|
36
|
+
disposition: IncidentDisposition;
|
|
37
|
+
incidents: string[];
|
|
38
|
+
reason: string;
|
|
39
|
+
evidence?: string;
|
|
40
|
+
at: number;
|
|
19
41
|
}
|
|
20
42
|
export interface FailureObservation {
|
|
21
43
|
id: string;
|
|
22
44
|
operation: string;
|
|
23
|
-
|
|
45
|
+
/** `resolved` is the historical name for a recovered incident. */
|
|
46
|
+
status: "unresolved" | "expected" | "resolved" | "superseded";
|
|
24
47
|
category: string;
|
|
25
48
|
summary: string;
|
|
26
49
|
evidence?: string;
|
|
@@ -31,6 +54,8 @@ export interface FailureObservation {
|
|
|
31
54
|
at?: number;
|
|
32
55
|
count: number;
|
|
33
56
|
resolvedAt?: number;
|
|
57
|
+
/** Latest explicit disposition applied to this incident. */
|
|
58
|
+
disposition?: DispositionRecord;
|
|
34
59
|
}
|
|
35
60
|
export interface FailureState {
|
|
36
61
|
version: 1;
|
|
@@ -38,6 +63,10 @@ export interface FailureState {
|
|
|
38
63
|
observations: Record<string, FailureObservation>;
|
|
39
64
|
delivered: Record<string, number>;
|
|
40
65
|
resolved?: Record<string, number>;
|
|
66
|
+
/** Closed incidents replaced by a newer incident of the same operation. Never deleted. */
|
|
67
|
+
history?: Record<string, FailureObservation>;
|
|
68
|
+
/** Accepted disposition events in journal order. */
|
|
69
|
+
dispositions?: DispositionRecord[];
|
|
41
70
|
}
|
|
42
71
|
export function emptyFailureState(): FailureState {
|
|
43
72
|
return { version: 1, seen: [], observations: {}, delivered: {} };
|
|
@@ -48,26 +77,181 @@ export function failureIdentity(...parts: unknown[]): string {
|
|
|
48
77
|
function text(value: string | undefined, fallback: string): string {
|
|
49
78
|
return (value || fallback).replace(/[\x00-\x1f\x7f]/g, " ").slice(0, 400);
|
|
50
79
|
}
|
|
80
|
+
|
|
81
|
+
/** Agent tool attempts: the agent handles its own tool errors, so one failure is not yet actionable. */
|
|
82
|
+
export const AGENT_TOOL_CATEGORY = "tool";
|
|
83
|
+
/** A command the confined intent bash refused before running (malformed or reused intent). Agent-owned, never grouped. */
|
|
84
|
+
export const REJECTED_INTENT_CATEGORY = "rejected-intent";
|
|
85
|
+
/** The same unresolved agent tool operation failing this many times is treated as stuck. */
|
|
86
|
+
export const REPEATED_FAILURE_THRESHOLD = 3;
|
|
87
|
+
|
|
88
|
+
/** Caller-chosen intent identifiers: short, printable, no whitespace. */
|
|
89
|
+
export const INTENT_ID_PATTERN = "^[A-Za-z0-9][A-Za-z0-9._:/-]{0,63}$";
|
|
90
|
+
const INTENT_ID = new RegExp(INTENT_ID_PATTERN);
|
|
91
|
+
export const MAX_EXPECTED_EXIT_CODES = 16;
|
|
92
|
+
/**
|
|
93
|
+
* Structured command intent (#315), declared by the caller before a command runs. Shared by the
|
|
94
|
+
* subagent task runtime's bash and background tasks (#325) so both validate identically.
|
|
95
|
+
*/
|
|
96
|
+
export interface CommandIntent {
|
|
97
|
+
/** Stable across caller-declared modified retries of one logical operation. */
|
|
98
|
+
operationId?: string;
|
|
99
|
+
/** Names one concrete execution; evidence identity, never an operation match. */
|
|
100
|
+
attemptId?: string;
|
|
101
|
+
/** Non-zero exit codes declared intentional before execution. */
|
|
102
|
+
expectedExitCodes?: number[];
|
|
103
|
+
}
|
|
104
|
+
export type CommandIntentField = keyof CommandIntent;
|
|
105
|
+
const INTENT_FIELDS: readonly CommandIntentField[] = ["operationId", "attemptId", "expectedExitCodes"];
|
|
106
|
+
/**
|
|
107
|
+
* The one normalization of intent-bearing arguments: an explicit `null` intent field means
|
|
108
|
+
* "not declared", exactly like an omitted one. Models routinely send optional fields as null, and
|
|
109
|
+
* Pi's argument validation (0.87+) drops optional nulls before a tool runs, while the process log
|
|
110
|
+
* and session record keep the raw arguments. Every reader of intent, the executing tool and the
|
|
111
|
+
* replay alike, goes through this so both sides see the same declaration from the same input.
|
|
112
|
+
*/
|
|
113
|
+
export function withoutAbsentIntent<T>(args: T): T {
|
|
114
|
+
if (!args || typeof args !== "object" || Array.isArray(args)) return args;
|
|
115
|
+
const input = args as Record<string, unknown>;
|
|
116
|
+
if (!INTENT_FIELDS.some((key) => key in input && input[key] == null)) return args;
|
|
117
|
+
const out = { ...input };
|
|
118
|
+
for (const key of INTENT_FIELDS) if (key in out && out[key] == null) delete out[key];
|
|
119
|
+
return out as T;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Validate structured intent fields. Absent fields (omitted, undefined, or null) are fine; malformed
|
|
123
|
+
* ones are an error, and the caller must not run the command. `names` renames fields in the error
|
|
124
|
+
* (e.g. snake_case parameters).
|
|
125
|
+
*/
|
|
126
|
+
export function readCommandIntent(args: unknown, names: Partial<Record<CommandIntentField, string>> = {}): { intent: CommandIntent; error?: string } {
|
|
127
|
+
const input = withoutAbsentIntent((args && typeof args === "object" ? args : {}) as Record<string, unknown>);
|
|
128
|
+
const intent: CommandIntent = {};
|
|
129
|
+
for (const key of ["operationId", "attemptId"] as const) {
|
|
130
|
+
const value = input[key];
|
|
131
|
+
if (value === undefined) continue;
|
|
132
|
+
if (typeof value !== "string" || !INTENT_ID.test(value)) {
|
|
133
|
+
return { intent: {}, error: `${names[key] ?? key} must match ${INTENT_ID_PATTERN}` };
|
|
134
|
+
}
|
|
135
|
+
intent[key] = value;
|
|
136
|
+
}
|
|
137
|
+
const codes = input.expectedExitCodes;
|
|
138
|
+
if (codes !== undefined) {
|
|
139
|
+
if (!Array.isArray(codes) || codes.length === 0 || codes.length > MAX_EXPECTED_EXIT_CODES ||
|
|
140
|
+
!codes.every((code) => Number.isInteger(code) && code >= 1 && code <= 255) || new Set(codes).size !== codes.length) {
|
|
141
|
+
return { intent: {}, error: `${names.expectedExitCodes ?? "expectedExitCodes"} must be 1-${MAX_EXPECTED_EXIT_CODES} distinct integers from 1 to 255` };
|
|
142
|
+
}
|
|
143
|
+
intent.expectedExitCodes = [...codes] as number[];
|
|
144
|
+
}
|
|
145
|
+
return { intent };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** The one current incident with this id, if it is still in the reduced state. */
|
|
149
|
+
export function findIncident(state: FailureState, id: string): FailureObservation | undefined {
|
|
150
|
+
return Object.values(state.observations).find((item) => item.id === id);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Current actionability, derived only from reduced structured state. Lifecycle is not an input.
|
|
155
|
+
* Agent tool failures need an explicit `open` disposition or a repeated failure of the same
|
|
156
|
+
* operation; other producer failures (exit, model, supervision, watch) are actionable at once.
|
|
157
|
+
*/
|
|
158
|
+
export function requiresAction(x: FailureObservation): boolean {
|
|
159
|
+
if (x.status !== "unresolved" || x.category === "observation-incomplete") return false;
|
|
160
|
+
if (x.category !== AGENT_TOOL_CATEGORY && x.category !== REJECTED_INTENT_CATEGORY) return true;
|
|
161
|
+
return x.disposition?.disposition === "open" || x.count >= REPEATED_FAILURE_THRESHOLD;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export function failureLabel(x: FailureObservation): string {
|
|
165
|
+
if (x.category === "observation-incomplete") return "Observation incomplete";
|
|
166
|
+
if (x.status === "expected") return "Expected failure";
|
|
167
|
+
if (x.status === "resolved") return "Recovered";
|
|
168
|
+
if (x.status === "superseded") return "Superseded";
|
|
169
|
+
return requiresAction(x) ? "Action required" : "Unclassified failure observation";
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Whether an incident needs someone's action now: an unresolved `Action required` incident
|
|
174
|
+
* (non-tool failure, repeated tool failure, or disposed `open`) or an observation gap. Unclassified
|
|
175
|
+
* and expected failures are history: counted on every surface, listed only on explicit request.
|
|
176
|
+
*/
|
|
177
|
+
export function needsAction(x: FailureObservation): boolean {
|
|
178
|
+
return x.status === "unresolved" && (x.category === "observation-incomplete" || requiresAction(x));
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Which incidents a surface lists. `actionable` (the default everywhere) is what needs action;
|
|
183
|
+
* `all` is the explicit history view: every incident ever reduced, actionable first, then unclassified,
|
|
184
|
+
* expected, and closed (recovered/superseded) incidents.
|
|
185
|
+
*/
|
|
186
|
+
export type IncidentScope = "actionable" | "all";
|
|
187
|
+
/** `compact` rows (the default) unwrap tool-result JSON, cap the excerpt, and shorten evidence paths; `full` keeps them. */
|
|
188
|
+
export type IncidentDetail = "compact" | "full";
|
|
189
|
+
export interface IncidentRowOptions { scope?: IncidentScope; detail?: IncidentDetail }
|
|
190
|
+
|
|
191
|
+
/** Validate a disposition against the reduced state. Returns the rejection reason, or undefined. */
|
|
192
|
+
export function validateDisposition(state: FailureState, event: FailureEvent): string | undefined {
|
|
193
|
+
if (event.kind !== "disposition") return "Not a disposition event";
|
|
194
|
+
if (!event.disposition || !INCIDENT_DISPOSITIONS.includes(event.disposition)) return `Unknown disposition ${JSON.stringify(event.disposition ?? "")}`;
|
|
195
|
+
const ids = event.incidents;
|
|
196
|
+
if (!Array.isArray(ids) || ids.length === 0) return "A disposition must name at least one incident";
|
|
197
|
+
if (ids.some((id) => typeof id !== "string" || !id)) return "Incident ids must be non-empty strings";
|
|
198
|
+
if (new Set(ids).size !== ids.length) return "A disposition names the same incident more than once";
|
|
199
|
+
if (typeof event.reason !== "string" || !event.reason.trim()) return "A disposition requires a reason";
|
|
200
|
+
if ((event.disposition === "recovered" || event.disposition === "superseded") && (typeof event.evidence !== "string" || !event.evidence.trim())) {
|
|
201
|
+
return `A ${event.disposition} disposition requires evidence`;
|
|
202
|
+
}
|
|
203
|
+
for (const id of ids) {
|
|
204
|
+
const current = findIncident(state, id);
|
|
205
|
+
if (!current) {
|
|
206
|
+
return Object.hasOwn(state.history ?? {}, id) || Object.hasOwn(state.resolved ?? {}, id)
|
|
207
|
+
? `Incident ${id} is already disposed` : `Unknown incident ${id}`;
|
|
208
|
+
}
|
|
209
|
+
if (current.category === "observation-incomplete") return `Incident ${id} is an observation gap; it cannot be disposed`;
|
|
210
|
+
if (current.status !== "unresolved") return `Incident ${id} is already disposed (${failureLabel(current).toLowerCase()})`;
|
|
211
|
+
if (event.disposition === "open" && current.disposition?.disposition === "open") return `Incident ${id} is already open`;
|
|
212
|
+
}
|
|
213
|
+
return undefined;
|
|
214
|
+
}
|
|
215
|
+
|
|
51
216
|
/** Pure transition. Lifecycle is deliberately not an input or an output. */
|
|
52
217
|
export function reduceFailure(state: FailureState, event: FailureEvent, observedAt: number): FailureState {
|
|
53
218
|
if (state.seen.includes(event.id)) return state;
|
|
219
|
+
// Invalid dispositions fail closed: nothing changes and the id is not consumed.
|
|
220
|
+
if (event.kind === "disposition" && validateDisposition(state, event)) return state;
|
|
54
221
|
const next: FailureState = { ...state, seen: [...state.seen, event.id],
|
|
55
222
|
observations: { ...state.observations }, delivered: { ...state.delivered } };
|
|
56
223
|
if (event.kind === "delivered") {
|
|
57
224
|
for (const id of event.incidents ?? []) next.delivered = { ...next.delivered, [id]: observedAt };
|
|
58
225
|
return next;
|
|
59
226
|
}
|
|
227
|
+
if (event.kind === "disposition") {
|
|
228
|
+
const at = event.at ?? observedAt;
|
|
229
|
+
const record: DispositionRecord = { eventId: event.id, disposition: event.disposition!, incidents: [...event.incidents!],
|
|
230
|
+
reason: text(event.reason, ""), ...(event.evidence ? { evidence: text(event.evidence, "") } : {}), at };
|
|
231
|
+
for (const id of record.incidents) {
|
|
232
|
+
const [key, current] = Object.entries(next.observations).find(([, item]) => item.id === id)!;
|
|
233
|
+
const status = record.disposition === "recovered" ? "resolved" : record.disposition === "superseded" ? "superseded"
|
|
234
|
+
: record.disposition === "expected" ? "expected" : current.status;
|
|
235
|
+
next.observations[key] = { ...current, status, disposition: record,
|
|
236
|
+
...(status === "resolved" || status === "superseded" ? { resolvedAt: at } : {}) };
|
|
237
|
+
if (status === "resolved" || status === "superseded") next.resolved = { ...next.resolved, [id]: at };
|
|
238
|
+
}
|
|
239
|
+
next.dispositions = [...(state.dispositions ?? []), record];
|
|
240
|
+
return next;
|
|
241
|
+
}
|
|
60
242
|
const key = failureIdentity(event.operation);
|
|
61
243
|
const previous = state.observations[key];
|
|
62
244
|
if (event.kind === "recovered") {
|
|
63
|
-
|
|
245
|
+
// Only an unresolved incident recovers; an expected one keeps its classification.
|
|
246
|
+
if (previous && previous.status === "unresolved" && event.incidents?.includes(previous.id)) {
|
|
64
247
|
next.observations[key] = { ...previous, status: "resolved", resolvedAt: event.at ?? observedAt };
|
|
65
248
|
next.resolved = { ...state.resolved, [previous.id]: event.at ?? observedAt };
|
|
66
249
|
}
|
|
67
250
|
return next;
|
|
68
251
|
}
|
|
69
|
-
const active = previous && previous.status !== "resolved" &&
|
|
252
|
+
const active = previous && previous.status !== "resolved" && previous.status !== "superseded" &&
|
|
70
253
|
!(previous.status === "expected" && !event.expected);
|
|
254
|
+
if (previous && !active) next.history = { ...state.history, [previous.id]: previous };
|
|
71
255
|
next.observations[key] = {
|
|
72
256
|
id: active ? previous.id : event.id, operation: event.operation,
|
|
73
257
|
status: active ? previous.status : event.expected ? "expected" : "unresolved",
|
|
@@ -76,30 +260,638 @@ export function reduceFailure(state: FailureState, event: FailureEvent, observed
|
|
|
76
260
|
firstObservedAt: active ? previous.firstObservedAt : observedAt,
|
|
77
261
|
lastObservedAt: observedAt, lastSequence: next.seen.length, at: event.at,
|
|
78
262
|
count: active ? previous.count + 1 : 1,
|
|
263
|
+
...(active && previous.disposition ? { disposition: previous.disposition } : {}),
|
|
79
264
|
};
|
|
80
265
|
return next;
|
|
81
266
|
}
|
|
267
|
+
function priority(x: FailureObservation): number {
|
|
268
|
+
if (x.category === "observation-incomplete") return 0;
|
|
269
|
+
if (x.status === "expected") return 3;
|
|
270
|
+
return requiresAction(x) ? 1 : 2;
|
|
271
|
+
}
|
|
272
|
+
/** Current (not recovered or superseded) incidents in priority order. History is excluded. */
|
|
82
273
|
export function activeFailures(state: FailureState): FailureObservation[] {
|
|
83
|
-
|
|
84
|
-
return Object.values(state.observations).filter((x) => x.status !== "resolved")
|
|
274
|
+
return Object.values(state.observations).filter((x) => x.status !== "resolved" && x.status !== "superseded")
|
|
85
275
|
.sort((a, b) => priority(a) - priority(b) || (b.lastSequence ?? 0) - (a.lastSequence ?? 0) || b.lastObservedAt - a.lastObservedAt);
|
|
86
276
|
}
|
|
277
|
+
/** Incidents that need action now, priority order. */
|
|
278
|
+
export function actionableFailures(state: FailureState): FailureObservation[] {
|
|
279
|
+
return activeFailures(state).filter(needsAction);
|
|
280
|
+
}
|
|
281
|
+
function closedOrder(a: FailureObservation, b: FailureObservation): number {
|
|
282
|
+
return (b.lastSequence ?? 0) - (a.lastSequence ?? 0) || b.lastObservedAt - a.lastObservedAt;
|
|
283
|
+
}
|
|
284
|
+
/** The incidents a scope lists, in the order rows render. */
|
|
285
|
+
export function scopedFailures(state: FailureState, scope: IncidentScope = "actionable"): FailureObservation[] {
|
|
286
|
+
if (scope !== "all") return actionableFailures(state);
|
|
287
|
+
const active = activeFailures(state);
|
|
288
|
+
const closed = failureHistory(state).filter((x) => x.status === "resolved" || x.status === "superseded").sort(closedOrder);
|
|
289
|
+
return [...active, ...closed];
|
|
290
|
+
}
|
|
291
|
+
/** Every incident ever reduced, current and closed. Nothing is removed. */
|
|
292
|
+
export function failureHistory(state: FailureState): FailureObservation[] {
|
|
293
|
+
return [...Object.values(state.history ?? {}), ...Object.values(state.observations)];
|
|
294
|
+
}
|
|
295
|
+
export interface FailureCounts {
|
|
296
|
+
actionRequired: number;
|
|
297
|
+
unclassified: number;
|
|
298
|
+
expected: number;
|
|
299
|
+
incomplete: number;
|
|
300
|
+
recovered: number;
|
|
301
|
+
superseded: number;
|
|
302
|
+
}
|
|
303
|
+
/** Separate facts: current actionability, retained classification, and closed history. */
|
|
304
|
+
export function failureCounts(state: FailureState): FailureCounts {
|
|
305
|
+
const counts: FailureCounts = { actionRequired: 0, unclassified: 0, expected: 0, incomplete: 0, recovered: 0, superseded: 0 };
|
|
306
|
+
for (const x of failureHistory(state)) {
|
|
307
|
+
if (x.status === "resolved") counts.recovered += 1;
|
|
308
|
+
else if (x.status === "superseded") counts.superseded += 1;
|
|
309
|
+
else if (x.category === "observation-incomplete") counts.incomplete += 1;
|
|
310
|
+
else if (x.status === "expected") counts.expected += 1;
|
|
311
|
+
else if (requiresAction(x)) counts.actionRequired += 1;
|
|
312
|
+
else counts.unclassified += 1;
|
|
313
|
+
}
|
|
314
|
+
return counts;
|
|
315
|
+
}
|
|
316
|
+
function plural(count: number, one: string, many = `${one}s`): string {
|
|
317
|
+
return `${count} ${count === 1 ? one : many}`;
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Counts of incidents that do not need action, as one short fragment, or undefined when there are
|
|
321
|
+
* none: e.g. `8 unclassified tool errors · 2 expected · 1 recovered`.
|
|
322
|
+
*/
|
|
323
|
+
export function historyCountsText(state: FailureState): string | undefined {
|
|
324
|
+
const counts = failureCounts(state);
|
|
325
|
+
const parts = [
|
|
326
|
+
counts.unclassified ? plural(counts.unclassified, "unclassified tool error") : "",
|
|
327
|
+
counts.expected ? `${counts.expected} expected` : "",
|
|
328
|
+
counts.recovered ? `${counts.recovered} recovered` : "",
|
|
329
|
+
counts.superseded ? `${counts.superseded} superseded` : "",
|
|
330
|
+
].filter(Boolean);
|
|
331
|
+
return parts.length ? parts.join(" · ") : undefined;
|
|
332
|
+
}
|
|
333
|
+
/** One line for a run where nothing needs action but history exists. No cursor: history is explicit. */
|
|
334
|
+
export function quietFailureLine(state: FailureState): string | undefined {
|
|
335
|
+
if (actionableFailures(state).length) return undefined;
|
|
336
|
+
const counts = failureCounts(state);
|
|
337
|
+
// Closed incidents alone are not worth a line on every surface; they ride along with current ones.
|
|
338
|
+
if (!counts.unclassified && !counts.expected) return undefined;
|
|
339
|
+
const history = historyCountsText(state);
|
|
340
|
+
return history ? `No failures need action · ${history} (history)` : undefined;
|
|
341
|
+
}
|
|
342
|
+
/** Trailing history count after actionable rows, or undefined when there is none. */
|
|
343
|
+
export function historyTailLine(state: FailureState): string | undefined {
|
|
344
|
+
const history = historyCountsText(state);
|
|
345
|
+
return history ? `Also in history: ${history}` : undefined;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
const INCIDENT_CURSOR_PREFIX = "i1.";
|
|
349
|
+
const encoder = new TextEncoder();
|
|
350
|
+
|
|
351
|
+
/** Longest summary or disposition-reason excerpt a compact row shows, in UTF-8 bytes. */
|
|
352
|
+
export const COMPACT_EXCERPT_BYTES = 120;
|
|
353
|
+
const TOOL_RESULT_WRAPPER = /\{"content":\[\{"type":"text","text":"/;
|
|
354
|
+
const JSON_ESCAPES: Record<string, string> = { '"': '"', "\\": "\\", "/": "/", b: " ", f: " ", n: " ", r: " ", t: " " };
|
|
355
|
+
|
|
356
|
+
/** Remove unpaired UTF-16 surrogates, which would otherwise encode as U+FFFD. */
|
|
357
|
+
function dropLoneSurrogates(value: string): string {
|
|
358
|
+
return value.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g, "");
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* A tool error recorded as its raw result JSON (`{"content":[{"type":"text","text":"…"}]}`, often
|
|
363
|
+
* already cut mid-string) rendered as the text it carries. Decoding is tolerant of the cut; text
|
|
364
|
+
* before the wrapper (e.g. `read failed: `) is kept.
|
|
365
|
+
*/
|
|
366
|
+
export function unwrapToolResultText(value: string): string {
|
|
367
|
+
const match = TOOL_RESULT_WRAPPER.exec(value);
|
|
368
|
+
if (!match) return value;
|
|
369
|
+
let out = "";
|
|
370
|
+
let i = match.index + match[0].length;
|
|
371
|
+
while (i < value.length) {
|
|
372
|
+
const ch = value[i]!;
|
|
373
|
+
if (ch === '"') break;
|
|
374
|
+
if (ch !== "\\") { out += ch; i += 1; continue; }
|
|
375
|
+
const next = value[i + 1];
|
|
376
|
+
if (next === "u") {
|
|
377
|
+
const hex = value.slice(i + 2, i + 6);
|
|
378
|
+
if (!/^[0-9a-fA-F]{4}$/.test(hex)) break;
|
|
379
|
+
out += String.fromCharCode(parseInt(hex, 16));
|
|
380
|
+
i += 6;
|
|
381
|
+
continue;
|
|
382
|
+
}
|
|
383
|
+
if (next === undefined || !(next in JSON_ESCAPES)) break;
|
|
384
|
+
out += JSON_ESCAPES[next];
|
|
385
|
+
i += 2;
|
|
386
|
+
}
|
|
387
|
+
// A wrapper cut inside a surrogate pair, or an escape of one half, leaves a lone surrogate
|
|
388
|
+
// (rendered as U+FFFD); decoded control characters (e.g. an escaped NUL) are not shown either.
|
|
389
|
+
const decoded = dropLoneSurrogates(out).replace(/[\x00-\x1f\x7f]/g, " ");
|
|
390
|
+
return `${value.slice(0, match.index)}${decoded}`.replace(/\s+/g, " ").trim();
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** At most `max` UTF-8 bytes of `value`, whole code points, ending in `…` when cut. */
|
|
394
|
+
export function capUtf8(value: string, max: number): string {
|
|
395
|
+
const bytes = encoder.encode(value);
|
|
396
|
+
if (bytes.length <= max) return value;
|
|
397
|
+
const ellipsis = "…";
|
|
398
|
+
const cut = utf8Prefix(bytes, Math.max(0, max - utf8Length(ellipsis)));
|
|
399
|
+
return `${Buffer.from(bytes.subarray(0, cut)).toString("utf8").trimEnd()}${ellipsis}`;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/**
|
|
403
|
+
* Evidence reference as a compact row shows it: an absolute path loses its directories
|
|
404
|
+
* (`/private/var/…/runs/<id>/output.log#byte=N` → `output.log#byte=N`); the run or task id already
|
|
405
|
+
* names the log. Full rows and the journal keep the whole reference.
|
|
406
|
+
*/
|
|
407
|
+
export function shortEvidence(evidence: string): string {
|
|
408
|
+
const match = /^(?:[A-Za-z]:)?[\\/](?:[^#]*[\\/])?([^\\/#]+)(#.*)?$/.exec(evidence);
|
|
409
|
+
return match ? `${match[1]}${match[2] ?? ""}` : evidence;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function failureRow(x: FailureObservation, detail: IncidentDetail = "compact"): string {
|
|
413
|
+
const time = x.at === undefined ? `observed ${new Date(x.firstObservedAt).toISOString()}` : new Date(x.at).toISOString();
|
|
414
|
+
const compact = detail !== "full";
|
|
415
|
+
// Full rows (raw evidence) show the journal summary verbatim; compact rows unwrap and cap it.
|
|
416
|
+
const summary = compact ? capUtf8(dropLoneSurrogates(unwrapToolResultText(x.summary)).replace(/\s+/g, " ").trim(), COMPACT_EXCERPT_BYTES) : x.summary;
|
|
417
|
+
const reason = x.disposition ? text(x.disposition.reason, "") : "";
|
|
418
|
+
const disposition = x.disposition ? ` · ${x.disposition.disposition}: ${compact ? capUtf8(reason, COMPACT_EXCERPT_BYTES) : reason}` : "";
|
|
419
|
+
const evidence = x.evidence ? text(x.evidence, "") : "";
|
|
420
|
+
return `${failureLabel(x)} · ${time} · ${summary}${x.count > 1 ? ` (${x.count} occurrences)` : ""}${disposition}${evidence ? ` · evidence: ${compact ? shortEvidence(evidence) : evidence}` : ""}`;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Incident rows in priority order. By default, only incidents that need action, as compact rows;
|
|
425
|
+
* `scope: "all"` lists history too and `detail: "full"` keeps whole excerpts and evidence paths.
|
|
426
|
+
* Consumers page these; they are not a lossy summary.
|
|
427
|
+
*/
|
|
428
|
+
export function formatFailureLines(state: FailureState, options: IncidentRowOptions = {}): string[] {
|
|
429
|
+
return scopedFailures(state, options.scope).map((x) => failureRow(x, options.detail));
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function closedHistoryLine(state: FailureState): string | undefined {
|
|
433
|
+
const counts = failureCounts(state);
|
|
434
|
+
if (!counts.recovered && !counts.superseded) return undefined;
|
|
435
|
+
const parts = [counts.recovered ? `${counts.recovered} recovered` : "", counts.superseded ? `${counts.superseded} superseded` : ""].filter(Boolean);
|
|
436
|
+
return `Closed incidents retained in history: ${parts.join(" · ")}.`;
|
|
437
|
+
}
|
|
438
|
+
|
|
87
439
|
/** Shared priority text, placed BEFORE assistant progress on every consumer surface. */
|
|
88
440
|
export function formatFailureSummary(state: FailureState): string {
|
|
89
|
-
const
|
|
90
|
-
if (!
|
|
91
|
-
const rows =
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
return `${label} · ${time} · ${x.summary}${x.count > 1 ? ` (${x.count} occurrences)` : ""}${x.evidence ? ` · evidence: ${text(x.evidence, "")}` : ""}`;
|
|
96
|
-
});
|
|
97
|
-
if (failures.length > 5) rows.push(`${failures.length - 5} additional active failure observations retained in the failure journal.`);
|
|
441
|
+
const lines = formatFailureLines(state);
|
|
442
|
+
if (!lines.length) return quietFailureLine(state) ?? "";
|
|
443
|
+
const rows = lines.slice(0, 5);
|
|
444
|
+
if (lines.length > 5) rows.push(`${lines.length - 5} additional active failure observations retained in the failure journal.`);
|
|
445
|
+
const history = historyTailLine(state);
|
|
446
|
+
if (history) rows.push(history);
|
|
98
447
|
return rows.join("\n");
|
|
99
448
|
}
|
|
100
|
-
|
|
449
|
+
|
|
450
|
+
/** Incident rows for exactly `incidents`, priority order. Unknown ids fail closed. */
|
|
451
|
+
export function pendingAttentionRows(state: FailureState, incidents: readonly string[]): string[] {
|
|
452
|
+
const wanted = new Set(incidents);
|
|
453
|
+
const rows = activeFailures(state).filter((x) => wanted.has(x.id));
|
|
454
|
+
if (rows.length !== wanted.size) throw new Error("Failure incident evidence is unavailable; defer notification delivery");
|
|
455
|
+
return rows.map((x) => failureRow(x));
|
|
456
|
+
}
|
|
457
|
+
/** Count of other active incidents, which a notification references but never re-lists. */
|
|
458
|
+
export function pendingAttentionNote(state: FailureState, incidents: readonly string[]): string | undefined {
|
|
459
|
+
const wanted = new Set(incidents);
|
|
460
|
+
const others = activeFailures(state).filter((x) => !wanted.has(x.id)).length;
|
|
461
|
+
return others > 0 ? `${others} other active failure observation${others === 1 ? " was" : "s were"} reported earlier or not actionable; not repeated here.` : undefined;
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* Attention text for exactly `incidents` (normally `pending.incidents`). Earlier delivered or
|
|
465
|
+
* non-actionable incidents are counted, never re-listed. Unknown ids fail closed.
|
|
466
|
+
*/
|
|
467
|
+
export function formatPendingAttention(state: FailureState, incidents: readonly string[], options: { maxRows?: number } = {}): string {
|
|
468
|
+
const rows = pendingAttentionRows(state, incidents);
|
|
469
|
+
const max = options.maxRows ?? 5;
|
|
470
|
+
const lines = rows.slice(0, max);
|
|
471
|
+
if (rows.length > max) lines.push(`${rows.length - max} additional pending incidents retained in the failure journal.`);
|
|
472
|
+
const note = pendingAttentionNote(state, incidents);
|
|
473
|
+
if (note) lines.push(note);
|
|
474
|
+
return lines.join("\n");
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
/** The terminal fact that lifecycle success is not evidence of correct work. Kept whenever it fits and never cut mid-sentence (#325). */
|
|
478
|
+
export const CORRECTNESS_NOTE: string = "Work correctness was not inferred from lifecycle alone.";
|
|
479
|
+
export interface TerminalFailureParts {
|
|
480
|
+
/** Actionable and observation-incomplete incidents among `incidents`, one row each. */
|
|
481
|
+
rows: string[];
|
|
482
|
+
/** Counts and separate facts: earlier-reported, unclassified, expected, closed history, correctness. */
|
|
483
|
+
notes: string[];
|
|
484
|
+
}
|
|
485
|
+
/**
|
|
486
|
+
* Terminal/completion facts: actionable and incomplete incidents from `incidents` as rows,
|
|
487
|
+
* unclassified agent tool failures as a count. Lifecycle is reported by the caller, separately.
|
|
488
|
+
*/
|
|
489
|
+
export function terminalFailureParts(state: FailureState, incidents: readonly string[] = []): TerminalFailureParts {
|
|
490
|
+
const wanted = new Set(incidents);
|
|
491
|
+
const reportable = (x: FailureObservation) => requiresAction(x) || x.category === "observation-incomplete";
|
|
492
|
+
const rows = activeFailures(state).filter((x) => wanted.has(x.id) && reportable(x)).map((x) => failureRow(x));
|
|
493
|
+
const notes: string[] = [];
|
|
494
|
+
const earlier = activeFailures(state).filter((x) => !wanted.has(x.id) && reportable(x)).length;
|
|
495
|
+
if (earlier) notes.push(`${earlier} actionable incident${earlier === 1 ? " was" : "s were"} reported earlier; not repeated here.`);
|
|
496
|
+
const counts = failureCounts(state);
|
|
497
|
+
if (counts.unclassified) notes.push(`${counts.unclassified} earlier tool failure${counts.unclassified === 1 ? "" : "s"} remain${counts.unclassified === 1 ? "s" : ""} unclassified.`);
|
|
498
|
+
if (counts.expected) notes.push(`${counts.expected} expected failure${counts.expected === 1 ? "" : "s"} recorded.`);
|
|
499
|
+
const history = closedHistoryLine(state);
|
|
500
|
+
if (history) notes.push(history);
|
|
501
|
+
if (counts.unclassified || counts.actionRequired) notes.push(CORRECTNESS_NOTE);
|
|
502
|
+
return { rows, notes };
|
|
503
|
+
}
|
|
504
|
+
export function formatTerminalFailureFacts(state: FailureState, incidents: readonly string[] = []): string {
|
|
505
|
+
const { rows, notes } = terminalFailureParts(state, incidents);
|
|
506
|
+
const lines = rows.slice(0, 5);
|
|
507
|
+
if (rows.length > 5) lines.push(`${rows.length - 5} additional actionable incidents retained in the failure journal.`);
|
|
508
|
+
return [...lines, ...notes].join("\n");
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
export interface FailureIncidentPage {
|
|
512
|
+
text: string;
|
|
513
|
+
total: number;
|
|
514
|
+
/** Rows completed on this page (a row split across pages counts where it ends). */
|
|
515
|
+
represented: number;
|
|
516
|
+
/** Rows not fully shown through this page, including a partially shown row. */
|
|
517
|
+
omitted: number;
|
|
518
|
+
cursor: string;
|
|
519
|
+
nextCursor: string;
|
|
520
|
+
hasMore: boolean;
|
|
521
|
+
reset?: "stale-cursor" | "source-replaced";
|
|
522
|
+
/** The first line continues a row begun on an earlier page. */
|
|
523
|
+
startsPartial: boolean;
|
|
524
|
+
/** The last line is the start of a row that continues on the next page. */
|
|
525
|
+
endsPartial: boolean;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* Signature of every observation's reportable state. A change here is a
|
|
530
|
+
* failure-only change even when log bytes are unchanged; receipts and replay
|
|
531
|
+
* bookkeeping do not change it.
|
|
532
|
+
*/
|
|
533
|
+
export function failureRevision(state: FailureState): string {
|
|
534
|
+
return failureIdentity(Object.values(state.observations)
|
|
535
|
+
.map((item) => [item.id, item.status, item.category, item.count, item.lastSequence ?? 0, item.summary, item.evidence ?? ""])
|
|
536
|
+
.sort((a, b) => String(a[0]).localeCompare(String(b[0]))));
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** Digest of the rows a cursor pages. Scope and detail are part of it: the same incidents render different bytes. */
|
|
540
|
+
function incidentRevision(state: FailureState, scope: IncidentScope = "actionable", detail: IncidentDetail = "compact"): string {
|
|
541
|
+
return failureIdentity(scope, detail, scopedFailures(state, scope).map((item) => [item.id, item.status, item.count, item.summary, item.evidence ?? ""])).slice(0, 16);
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/** Cursors carry a digest of their resource/scope, not the scope text. */
|
|
545
|
+
function resourceTag(resource: string | undefined): string | undefined {
|
|
546
|
+
return resource === undefined ? undefined : createHash("sha256").update(resource).digest("base64url").slice(0, 16);
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* Incident cursor format. `x` is the format version: cursors minted before scope and row detail
|
|
551
|
+
* existed (no `x`) carry byte offsets into full-detail rows of every active incident, so they reset
|
|
552
|
+
* as stale instead of resuming inside a compact row. `h`: the explicit history view; `f`: full rows.
|
|
553
|
+
*/
|
|
554
|
+
const INCIDENT_CURSOR_FORMAT = 2;
|
|
555
|
+
interface IncidentCursor { o: number; b: number; v: string; n: number; r?: string; h?: 1; f?: 1 }
|
|
556
|
+
|
|
557
|
+
function encodeIncidentCursor(cursor: IncidentCursor): string {
|
|
558
|
+
return INCIDENT_CURSOR_PREFIX + Buffer.from(JSON.stringify({ k: "i", x: INCIDENT_CURSOR_FORMAT, o: cursor.o, ...(cursor.b ? { b: cursor.b } : {}),
|
|
559
|
+
v: cursor.v, n: cursor.n, ...(cursor.r !== undefined ? { r: cursor.r } : {}), ...(cursor.h ? { h: 1 } : {}),
|
|
560
|
+
...(cursor.f ? { f: 1 } : {}) }), "utf8").toString("base64url");
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
function decodeIncidentCursor(cursor: string | undefined): IncidentCursor | undefined {
|
|
564
|
+
if (!cursor || !cursor.startsWith(INCIDENT_CURSOR_PREFIX)) return undefined;
|
|
565
|
+
try {
|
|
566
|
+
const parsed = JSON.parse(Buffer.from(cursor.slice(INCIDENT_CURSOR_PREFIX.length), "base64url").toString("utf8")) as { k?: string; x?: number; o?: number; b?: number; v?: string; n?: number; r?: string; h?: number; f?: number };
|
|
567
|
+
if (parsed?.k === "i" && parsed.x === INCIDENT_CURSOR_FORMAT && typeof parsed.v === "string") {
|
|
568
|
+
return { o: Math.max(0, Math.floor(parsed.o ?? 0)), b: Math.max(0, Math.floor(parsed.b ?? 0)), v: parsed.v,
|
|
569
|
+
n: Math.max(0, Math.floor(parsed.n ?? 0)), ...(typeof parsed.r === "string" ? { r: parsed.r } : {}),
|
|
570
|
+
...(parsed.h === 1 ? { h: 1 as const } : {}), ...(parsed.f === 1 ? { f: 1 as const } : {}) };
|
|
571
|
+
}
|
|
572
|
+
} catch { /* stale */ }
|
|
573
|
+
return undefined;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
export function isIncidentCursor(cursor: string | undefined): boolean {
|
|
577
|
+
return Boolean(cursor?.startsWith(INCIDENT_CURSOR_PREFIX));
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
/** The scope an incident cursor pages, or undefined for anything that is not an incident cursor. */
|
|
581
|
+
export function incidentCursorScope(cursor: string | undefined): IncidentScope | undefined {
|
|
582
|
+
const parsed = decodeIncidentCursor(cursor);
|
|
583
|
+
return parsed ? (parsed.h ? "all" : "actionable") : undefined;
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
function utf8Length(value: string): number {
|
|
587
|
+
return encoder.encode(value).byteLength;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/** Largest prefix of `bytes` within `room` that does not split a code point. */
|
|
591
|
+
function utf8Prefix(bytes: Uint8Array, room: number): number {
|
|
592
|
+
if (room <= 0) return 0;
|
|
593
|
+
if (room >= bytes.length) return bytes.length;
|
|
594
|
+
let end = room;
|
|
595
|
+
while (end > 0 && (bytes[end]! & 0xc0) === 0x80) end -= 1;
|
|
596
|
+
return end;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
export interface IncidentPageRequest extends IncidentRowOptions {
|
|
600
|
+
cursor?: string;
|
|
601
|
+
maxBytes?: number;
|
|
602
|
+
/** Resource/scope bound into cursors; a cursor minted for another scope resets. */
|
|
603
|
+
resource?: string;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
/**
|
|
607
|
+
* Caller-owned incident pages. Whole rows are preferred; a row larger than the
|
|
608
|
+
* page is split at a code-point boundary and resumes at that byte, so pages
|
|
609
|
+
* reconstruct formatFailureLines() exactly (join with "\n" except after a page
|
|
610
|
+
* that `endsPartial`). A page never exceeds `maxBytes`.
|
|
611
|
+
*/
|
|
612
|
+
export function pageFailureIncidents(state: FailureState, request: IncidentPageRequest = {}): FailureIncidentPage {
|
|
613
|
+
// A cursor carries its view: following a history cursor stays in history without the flag.
|
|
614
|
+
const parsed = request.cursor ? decodeIncidentCursor(request.cursor) : undefined;
|
|
615
|
+
const scope: IncidentScope = request.scope ?? (parsed?.h ? "all" : "actionable");
|
|
616
|
+
const detail: IncidentDetail = request.detail ?? (parsed?.f ? "full" : "compact");
|
|
617
|
+
const lines = formatFailureLines(state, { scope, detail });
|
|
618
|
+
const revision = incidentRevision(state, scope, detail);
|
|
619
|
+
const total = lines.length;
|
|
620
|
+
const resource = resourceTag(request.resource);
|
|
621
|
+
let offset = 0;
|
|
622
|
+
let byte = 0;
|
|
623
|
+
let reset: FailureIncidentPage["reset"];
|
|
624
|
+
if (request.cursor) {
|
|
625
|
+
if (!parsed || parsed.r !== resource || Boolean(parsed.h) !== (scope === "all") || Boolean(parsed.f) !== (detail === "full")) reset = "stale-cursor";
|
|
626
|
+
else if (parsed.v !== revision) reset = "source-replaced";
|
|
627
|
+
else { offset = Math.min(total, parsed.o); byte = offset < total ? parsed.b : 0; }
|
|
628
|
+
}
|
|
629
|
+
const maxBytes = Number.isFinite(request.maxBytes) && (request.maxBytes ?? -1) >= 0
|
|
630
|
+
? Math.floor(request.maxBytes as number)
|
|
631
|
+
: 2 * 1024;
|
|
632
|
+
const mint = (o: number, b: number) => encodeIncidentCursor({ o, b, v: revision, n: total, ...(resource !== undefined ? { r: resource } : {}),
|
|
633
|
+
...(scope === "all" ? { h: 1 as const } : {}), ...(detail === "full" ? { f: 1 as const } : {}) });
|
|
634
|
+
const parts: string[] = [];
|
|
635
|
+
let used = 0;
|
|
636
|
+
let nextRow = offset;
|
|
637
|
+
let nextByte = byte;
|
|
638
|
+
let endsPartial = false;
|
|
639
|
+
for (let i = offset; i < total; i += 1) {
|
|
640
|
+
const encoded = encoder.encode(lines[i]!);
|
|
641
|
+
const from = i === offset ? Math.min(byte, encoded.length) : 0;
|
|
642
|
+
const rest = encoded.subarray(from);
|
|
643
|
+
const sep = parts.length ? 1 : 0;
|
|
644
|
+
if (used + sep + rest.length <= maxBytes) {
|
|
645
|
+
parts.push(Buffer.from(rest).toString("utf8"));
|
|
646
|
+
used += sep + rest.length;
|
|
647
|
+
nextRow = i + 1;
|
|
648
|
+
nextByte = 0;
|
|
649
|
+
continue;
|
|
650
|
+
}
|
|
651
|
+
if (parts.length === 0) {
|
|
652
|
+
const cut = utf8Prefix(rest, maxBytes);
|
|
653
|
+
if (cut > 0) {
|
|
654
|
+
parts.push(Buffer.from(rest.subarray(0, cut)).toString("utf8"));
|
|
655
|
+
used += cut;
|
|
656
|
+
nextRow = i;
|
|
657
|
+
nextByte = from + cut;
|
|
658
|
+
endsPartial = true;
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
break;
|
|
662
|
+
}
|
|
663
|
+
const startsPartial = byte > 0 && offset < total;
|
|
664
|
+
const represented = Math.max(0, nextRow - offset);
|
|
665
|
+
return {
|
|
666
|
+
text: parts.join("\n"),
|
|
667
|
+
total,
|
|
668
|
+
represented,
|
|
669
|
+
omitted: Math.max(0, total - nextRow),
|
|
670
|
+
cursor: mint(offset, byte),
|
|
671
|
+
nextCursor: mint(nextRow, nextByte),
|
|
672
|
+
hasMore: nextRow < total,
|
|
673
|
+
...(reset ? { reset } : {}),
|
|
674
|
+
startsPartial,
|
|
675
|
+
endsPartial,
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/** Cursor resource for one task/run's incidents in one session scope. */
|
|
680
|
+
export function incidentResource(scopeKey: string, id: string): string {
|
|
681
|
+
return `incidents:${scopeKey}:${id}`;
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/** Heading for an explicit incident page. */
|
|
685
|
+
export function incidentPageHeading(total: number, scope: IncidentScope = "actionable"): string {
|
|
686
|
+
return scope === "all"
|
|
687
|
+
? `History page of ${total} failure observation${total === 1 ? "" : "s"} (all, including those that need no action).`
|
|
688
|
+
: `Incident page of ${total} active failure observation${total === 1 ? "" : "s"}.`;
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* One explicit incident page shaped as an envelope verbatim page (log-utils
|
|
693
|
+
* `VerbatimPage`): the shared body of every consumer's `incidentCursor`
|
|
694
|
+
* response. Omitted counts are rows, never bytes.
|
|
695
|
+
*/
|
|
696
|
+
export function incidentVerbatimPage(state: FailureState, request: IncidentPageRequest = {}): {
|
|
697
|
+
text: string;
|
|
698
|
+
hasMore: boolean;
|
|
699
|
+
cursor: string;
|
|
700
|
+
nextCursor: string;
|
|
701
|
+
omittedBytes: number;
|
|
702
|
+
omittedRows: number;
|
|
703
|
+
reset?: "stale-cursor" | "source-replaced";
|
|
704
|
+
} {
|
|
705
|
+
const page = pageFailureIncidents(state, request);
|
|
706
|
+
const scope = request.scope ?? incidentCursorScope(request.cursor) ?? "actionable";
|
|
707
|
+
const empty = scope === "all" ? "No failure observations recorded."
|
|
708
|
+
: [quietFailureLine(state) ? "No failures need action." : "No active failure observations.", historyCountsText(state) ? "Pass history:true to list history." : ""].filter(Boolean).join(" ");
|
|
709
|
+
return {
|
|
710
|
+
text: page.text || (page.total === 0 ? empty : ""),
|
|
711
|
+
hasMore: page.hasMore,
|
|
712
|
+
cursor: page.cursor,
|
|
713
|
+
nextCursor: page.nextCursor,
|
|
714
|
+
omittedBytes: 0,
|
|
715
|
+
omittedRows: page.omitted,
|
|
716
|
+
...(page.reset ? { reset: page.reset } : {}),
|
|
717
|
+
};
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
/**
|
|
721
|
+
* Cheap change fingerprint of a failure journal: the file's identity and size
|
|
722
|
+
* (or its read error), whether the durable existence marker is present, and
|
|
723
|
+
* how many records are still awaiting persistence in this process. Every
|
|
724
|
+
* journal append, truncation, deletion, or pending in-memory record changes
|
|
725
|
+
* it, so a cache keyed by it never serves stale incident counts.
|
|
726
|
+
*/
|
|
727
|
+
export function failureJournalFingerprint(path: string): string {
|
|
728
|
+
let file: string;
|
|
729
|
+
try {
|
|
730
|
+
const stats = statSync(path, { bigint: true });
|
|
731
|
+
file = `${stats.dev}:${stats.ino}:${stats.size}:${stats.mtimeNs}:${stats.ctimeNs}`;
|
|
732
|
+
} catch (error) {
|
|
733
|
+
file = `unreadable:${(error as NodeJS.ErrnoException).code ?? "error"}`;
|
|
734
|
+
}
|
|
735
|
+
return `${file}|${existsSync(`${path}.observed`) ? 1 : 0}|${pendingWrites.get(path)?.length ?? 0}`;
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
export function incidentCursorAt(state: FailureState, offset: number, resource?: string, options: IncidentRowOptions = {}): string {
|
|
739
|
+
const scope = options.scope ?? "actionable";
|
|
740
|
+
const detail = options.detail ?? "compact";
|
|
741
|
+
const lines = formatFailureLines(state, { scope, detail });
|
|
742
|
+
const tag = resourceTag(resource);
|
|
743
|
+
return encodeIncidentCursor({ o: Math.max(0, Math.floor(offset)), b: 0, v: incidentRevision(state, scope, detail), n: lines.length,
|
|
744
|
+
...(tag !== undefined ? { r: tag } : {}), ...(scope === "all" ? { h: 1 as const } : {}), ...(options.detail === "full" ? { f: 1 as const } : {}) });
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
const MIN_PARTIAL_ROW_BYTES = 96;
|
|
748
|
+
|
|
749
|
+
export interface IncidentSummary {
|
|
750
|
+
text: string;
|
|
751
|
+
total: number;
|
|
752
|
+
/** Rows fully shown. */
|
|
753
|
+
represented: number;
|
|
754
|
+
/** Rows not fully shown; retrievable from `nextCursor`. */
|
|
755
|
+
omitted: number;
|
|
756
|
+
nextCursor?: string;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
/**
|
|
760
|
+
* The shared priority failure section for a byte budget. When every active
|
|
761
|
+
* incident fits, the rows are returned unchanged. Otherwise a count line leads:
|
|
762
|
+
* total, fully shown, omitted, and the incident cursor that resumes at the
|
|
763
|
+
* first byte not shown. Counts survive any budget that fits the count line.
|
|
764
|
+
*/
|
|
765
|
+
export interface IncidentSummaryOptions {
|
|
766
|
+
maxBytes: number;
|
|
767
|
+
resource?: string;
|
|
768
|
+
retrieval?: string;
|
|
769
|
+
/** Row detail; cursors minted here page the same detail. Default compact. */
|
|
770
|
+
detail?: IncidentDetail;
|
|
771
|
+
}
|
|
772
|
+
export function formatIncidentSummary(state: FailureState, options: IncidentSummaryOptions): IncidentSummary {
|
|
773
|
+
const maxBytes = Math.max(0, Math.floor(options.maxBytes));
|
|
774
|
+
const summary = actionableIncidentSummary(state, options);
|
|
775
|
+
// Nothing needs action: one short line, no cursor. History is listed only on request.
|
|
776
|
+
if (summary.total === 0) {
|
|
777
|
+
const quiet = quietFailureLine(state);
|
|
778
|
+
return { text: quiet && utf8Length(quiet) <= maxBytes ? quiet : "", total: 0, represented: 0, omitted: 0 };
|
|
779
|
+
}
|
|
780
|
+
const tail = historyTailLine(state);
|
|
781
|
+
if (tail && summary.text && utf8Length(summary.text) + 1 + utf8Length(tail) <= maxBytes) return { ...summary, text: `${summary.text}\n${tail}` };
|
|
782
|
+
return summary;
|
|
783
|
+
}
|
|
784
|
+
function actionableIncidentSummary(state: FailureState, options: IncidentSummaryOptions): IncidentSummary {
|
|
785
|
+
const maxBytes = Math.max(0, Math.floor(options.maxBytes));
|
|
786
|
+
const whole = pageFailureIncidents(state, { maxBytes, resource: options.resource, detail: options.detail });
|
|
787
|
+
if (whole.total === 0) return { text: "", total: 0, represented: 0, omitted: 0 };
|
|
788
|
+
if (!whole.hasMore) return { text: whole.text, total: whole.total, represented: whole.represented, omitted: 0 };
|
|
789
|
+
const header = (page: FailureIncidentPage): string =>
|
|
790
|
+
`${page.total} active failure observation${page.total === 1 ? "" : "s"} · ${page.represented} shown · ${page.omitted} omitted` +
|
|
791
|
+
` · incidentCursor=${page.nextCursor}${options.retrieval ? ` (${options.retrieval})` : ""}`;
|
|
792
|
+
let rowBudget = maxBytes - utf8Length(header(whole)) - 1;
|
|
793
|
+
for (let attempt = 0; attempt < 6; attempt += 1) {
|
|
794
|
+
let page = pageFailureIncidents(state, { maxBytes: Math.max(0, rowBudget), resource: options.resource, detail: options.detail });
|
|
795
|
+
// A few bytes of a clipped row are noise; show the count line alone and
|
|
796
|
+
// let the cursor start at that row.
|
|
797
|
+
if (page.represented === 0 && page.endsPartial && utf8Length(page.text) < MIN_PARTIAL_ROW_BYTES) {
|
|
798
|
+
page = pageFailureIncidents(state, { maxBytes: 0, resource: options.resource, detail: options.detail });
|
|
799
|
+
}
|
|
800
|
+
const line = header(page);
|
|
801
|
+
const text = page.text ? `${line}\n${page.text}` : line;
|
|
802
|
+
const overflow = utf8Length(text) - maxBytes;
|
|
803
|
+
if (overflow <= 0) {
|
|
804
|
+
return { text, total: page.total, represented: page.represented, omitted: page.omitted, nextCursor: page.nextCursor };
|
|
805
|
+
}
|
|
806
|
+
if (rowBudget <= 0) break;
|
|
807
|
+
rowBudget -= overflow;
|
|
808
|
+
}
|
|
809
|
+
// Not even the count line with its cursor fits. Never emit a clipped cursor:
|
|
810
|
+
// keep the exact counts and drop the cursor token whole, saying so.
|
|
811
|
+
const empty = pageFailureIncidents(state, { maxBytes: 0, resource: options.resource, detail: options.detail });
|
|
812
|
+
const withCursor = header(empty);
|
|
813
|
+
if (utf8Length(withCursor) <= maxBytes) {
|
|
814
|
+
return { text: withCursor, total: empty.total, represented: 0, omitted: empty.total, nextCursor: empty.nextCursor };
|
|
815
|
+
}
|
|
816
|
+
const plural = empty.total === 1 ? "" : "s";
|
|
817
|
+
for (const candidate of [
|
|
818
|
+
`${empty.total} active failure observation${plural} · 0 shown · ${empty.total} omitted · incident cursor not shown (page too small; retry with a larger maxBytes${options.retrieval ? ` or ${options.retrieval}` : ""})`,
|
|
819
|
+
`${empty.total} active failure observation${plural} · 0 shown · ${empty.total} omitted · incident cursor not shown (page too small)`,
|
|
820
|
+
`${empty.total} active failure observation${plural} (page too small for details)`,
|
|
821
|
+
]) {
|
|
822
|
+
if (utf8Length(candidate) <= maxBytes) return { text: candidate, total: empty.total, represented: 0, omitted: empty.total };
|
|
823
|
+
}
|
|
824
|
+
return { text: "", total: empty.total, represented: 0, omitted: empty.total };
|
|
825
|
+
}
|
|
826
|
+
/**
|
|
827
|
+
* Terminal failure section for a byte budget (#315): lifecycle-independent facts. Actionable and
|
|
828
|
+
* observation-incomplete incidents (which lead the priority order) are shown as whole rows;
|
|
829
|
+
* unclassified, expected, and closed incidents are counts. When any active row is not shown, a
|
|
830
|
+
* count line gives the exact total, shown, omitted, and an incident cursor at the first unshown row.
|
|
831
|
+
*
|
|
832
|
+
* The result never exceeds `maxBytes` and is made of whole lines (#325). Under a tight budget it
|
|
833
|
+
* drops, in order: incident rows; the lower-priority notes (closed history, expected, earlier
|
|
834
|
+
* reported); the cursor's retrieval hint; the unclassified count; the cursor; the count line.
|
|
835
|
+
* The correctness note goes last, and only when it alone does not fit: it is never cut mid-sentence.
|
|
836
|
+
*/
|
|
837
|
+
export function formatTerminalIncidentSummary(state: FailureState, options: IncidentSummaryOptions): IncidentSummary {
|
|
838
|
+
const active = activeFailures(state);
|
|
839
|
+
// Counts, rows, and the cursor cover only incidents that need action; the rest is history.
|
|
840
|
+
const lines = formatFailureLines(state, { detail: options.detail });
|
|
841
|
+
const total = lines.length;
|
|
842
|
+
const parts = terminalFailureParts(state, active.map((x) => x.id));
|
|
843
|
+
const quiet = quietFailureLine(state);
|
|
844
|
+
// Nothing needs action: one short history line (no cursor) and, when tool failures stay
|
|
845
|
+
// unclassified, the correctness note.
|
|
846
|
+
const notes = total === 0 && quiet ? [quiet, ...parts.notes.filter((x) => x === CORRECTNESS_NOTE)] : parts.notes;
|
|
847
|
+
if (total === 0 && notes.length === 0) return { text: "", total: 0, represented: 0, omitted: 0 };
|
|
848
|
+
const maxBytes = Math.max(0, Math.floor(options.maxBytes));
|
|
849
|
+
const fits = (parts: readonly (string | undefined)[]) => utf8Length(parts.filter(Boolean).join("\n")) <= maxBytes;
|
|
850
|
+
// "full": cursor and retrieval hint; "bare": cursor only; "none": exact counts, no cursor.
|
|
851
|
+
type Header = "full" | "bare" | "none";
|
|
852
|
+
const header = (shown: number, mode: Header): string | undefined => shown >= total ? undefined :
|
|
853
|
+
`${total} active failure observation${total === 1 ? "" : "s"} · ${shown} shown · ${total - shown} omitted` +
|
|
854
|
+
(mode === "none" ? " · incident cursor not shown (page too small)"
|
|
855
|
+
: ` · incidentCursor=${incidentCursorAt(state, shown, options.resource, { detail: options.detail })}${mode === "full" && options.retrieval ? ` (${options.retrieval})` : ""}`);
|
|
856
|
+
const done = (parts: readonly (string | undefined)[], shown: number, mode: Header): IncidentSummary => {
|
|
857
|
+
const omitted = total - shown;
|
|
858
|
+
return { text: parts.filter((x): x is string => Boolean(x)).join("\n"), total, represented: shown, omitted,
|
|
859
|
+
...(omitted > 0 && mode !== "none" ? { nextCursor: incidentCursorAt(state, shown, options.resource, { detail: options.detail }) } : {}) };
|
|
860
|
+
};
|
|
861
|
+
// 1. Drop incident rows, lowest priority first, keeping every note and the count line with its cursor.
|
|
862
|
+
for (let shown = lines.length; shown >= 0; shown -= 1) {
|
|
863
|
+
const parts = [header(shown, "full"), ...lines.slice(0, shown), ...notes];
|
|
864
|
+
if (fits(parts)) return done(parts, shown, "full");
|
|
865
|
+
}
|
|
866
|
+
// 2. No rows. Then, in order: the lower-priority notes (history, expected, earlier reported), the
|
|
867
|
+
// cursor's retrieval hint, the unclassified count, the cursor, and the count line. The
|
|
868
|
+
// correctness note goes last, and only when it alone does not fit.
|
|
869
|
+
const correctness: string[] = notes.filter((x) => x === CORRECTNESS_NOTE);
|
|
870
|
+
const unclassified = notes.filter((x) => x !== CORRECTNESS_NOTE && /remains? unclassified\.$/.test(x));
|
|
871
|
+
const low = notes.filter((x) => !correctness.includes(x) && !unclassified.includes(x));
|
|
872
|
+
const ladder: Array<{ mode: Header | undefined; notes: string[] }> = [];
|
|
873
|
+
for (let keep = low.length - 1; keep >= 0; keep -= 1) ladder.push({ mode: "full", notes: [...low.slice(0, keep), ...unclassified, ...correctness] });
|
|
874
|
+
ladder.push({ mode: "bare", notes: [...unclassified, ...correctness] });
|
|
875
|
+
ladder.push({ mode: "bare", notes: correctness });
|
|
876
|
+
ladder.push({ mode: "none", notes: correctness });
|
|
877
|
+
ladder.push({ mode: undefined, notes: correctness });
|
|
878
|
+
for (const step of ladder) {
|
|
879
|
+
const parts = [step.mode ? header(0, step.mode) : undefined, ...step.notes];
|
|
880
|
+
if (parts.some(Boolean) && fits(parts)) return done(parts, 0, step.mode ?? "none");
|
|
881
|
+
}
|
|
882
|
+
return done([], 0, "none");
|
|
883
|
+
}
|
|
884
|
+
/**
|
|
885
|
+
* Incidents due for a notification. Terminal: every current unresolved incident not yet delivered
|
|
886
|
+
* (reported once, then receipted). Running: observation gaps immediately (unless the consumer
|
|
887
|
+
* defers them to its terminal/health callback), actionable incidents after the grace period;
|
|
888
|
+
* a single agent tool failure is left to the agent that owns it.
|
|
889
|
+
*/
|
|
890
|
+
export function pendingFailureAttention(state: FailureState, now: number,
|
|
891
|
+
options: { terminal?: boolean; graceMs?: number; deferObservationGaps?: boolean } = {}): { key: string; incidents: string[]; summary: string } | undefined {
|
|
101
892
|
const due = activeFailures(state).filter((x) => x.status === "unresolved" && !Object.hasOwn(state.delivered, x.id) &&
|
|
102
|
-
(options.terminal || x.category === "observation-incomplete"
|
|
893
|
+
(options.terminal || (x.category === "observation-incomplete" ? !options.deferObservationGaps
|
|
894
|
+
: requiresAction(x) && now - x.firstObservedAt >= (options.graceMs ?? 60_000))));
|
|
103
895
|
if (!due.length) return undefined;
|
|
104
896
|
const incidents = due.map((x) => x.id).sort();
|
|
105
897
|
return { key: failureIdentity(incidents), incidents,
|
|
@@ -153,11 +945,12 @@ function readStoredState(path: string): FailureState {
|
|
|
153
945
|
const e = row.event;
|
|
154
946
|
if (!e || typeof e.id !== "string" || !e.id || typeof e.operation !== "string" || !e.operation ||
|
|
155
947
|
(e.expected !== undefined && typeof e.expected !== "boolean") ||
|
|
156
|
-
!["failure", "incomplete", "recovered", "delivered"].includes(e.kind) ||
|
|
948
|
+
!["failure", "incomplete", "recovered", "delivered", "disposition"].includes(e.kind) ||
|
|
949
|
+
(e.disposition !== undefined && !INCIDENT_DISPOSITIONS.includes(e.disposition)) ||
|
|
157
950
|
!Number.isFinite(row.observedAt) || Math.abs(row.observedAt) > 8.64e15 ||
|
|
158
951
|
(e.at !== undefined && (!Number.isFinite(e.at) || Math.abs(e.at) > 8.64e15)) ||
|
|
159
952
|
(e.incidents !== undefined && (!Array.isArray(e.incidents) || !e.incidents.every((id: unknown) => typeof id === "string"))) ||
|
|
160
|
-
[e.summary, e.category, e.evidence].some((v) => v !== undefined && typeof v !== "string")) throw new Error("invalid record");
|
|
953
|
+
[e.summary, e.category, e.evidence, e.reason].some((v) => v !== undefined && typeof v !== "string")) throw new Error("invalid record");
|
|
161
954
|
state = reduceFailure(state, e, row.observedAt);
|
|
162
955
|
} catch { state = storageProblem(state, "Failure journal contains unreadable records; observations may be incomplete"); }
|
|
163
956
|
}
|
|
@@ -175,10 +968,12 @@ export function observeFailures(path: string, events: readonly FailureEvent[], n
|
|
|
175
968
|
if (state.seen.includes(raw.id)) continue;
|
|
176
969
|
if (raw.kind === "recovered") {
|
|
177
970
|
const prior = state.observations[failureIdentity(raw.operation)];
|
|
178
|
-
if (!prior || prior.status
|
|
971
|
+
if (!prior || prior.status !== "unresolved" || !raw.incidents?.includes(prior.id)) continue;
|
|
179
972
|
}
|
|
973
|
+
// Rejected dispositions are never journaled or marked seen.
|
|
974
|
+
if (raw.kind === "disposition" && validateDisposition(state, raw)) continue;
|
|
180
975
|
const event = { ...raw, ...(raw.summary ? { summary: text(raw.summary, "") } : {}),
|
|
181
|
-
...(raw.evidence ? { evidence: text(raw.evidence, "") } : {}) };
|
|
976
|
+
...(raw.evidence ? { evidence: text(raw.evidence, "") } : {}), ...(raw.reason ? { reason: text(raw.reason, "") } : {}) };
|
|
182
977
|
try {
|
|
183
978
|
if (pendingWrites.has(path)) throw new Error("Earlier evidence is awaiting persistence");
|
|
184
979
|
appendRecord(path, { event, observedAt: now });
|
|
@@ -203,3 +998,18 @@ export function failureAttentionHandled(state: FailureState, incidents: readonly
|
|
|
203
998
|
export function markFailureAttentionDelivered(path: string, pending: { key: string; incidents: string[] }, at = Date.now()): FailureState {
|
|
204
999
|
return observeFailures(path, [{ id: `delivered:${pending.key}`, operation: "attention-delivery", kind: "delivered", incidents: pending.incidents }], at);
|
|
205
1000
|
}
|
|
1001
|
+
export interface DispositionResult { accepted: boolean; error?: string; state: FailureState }
|
|
1002
|
+
/**
|
|
1003
|
+
* Append one explicit disposition. Validation happens before the journal write; a rejected
|
|
1004
|
+
* request (unknown, already disposed, missing reason or evidence) writes nothing. Replaying an
|
|
1005
|
+
* already accepted event id is an idempotent success.
|
|
1006
|
+
*/
|
|
1007
|
+
export function disposeIncidents(path: string, event: FailureEvent, now = Date.now()): DispositionResult {
|
|
1008
|
+
const state = readFailureState(path);
|
|
1009
|
+
if (event.kind !== "disposition") return { accepted: false, error: "Not a disposition event", state };
|
|
1010
|
+
if (state.seen.includes(event.id)) return { accepted: true, state };
|
|
1011
|
+
const error = validateDisposition(state, event);
|
|
1012
|
+
if (error) return { accepted: false, error, state };
|
|
1013
|
+
const next = observeFailures(path, [event], now);
|
|
1014
|
+
return next.seen.includes(event.id) ? { accepted: true, state: next } : { accepted: false, error: "Disposition was not recorded", state: next };
|
|
1015
|
+
}
|