pi-background-tasks 2.1.2 → 2.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/TESTING.md +1 -1
- package/TEST_PLAN.md +4 -4
- package/docs/INDEX.md +2 -2
- package/docs/manifest.json +4 -4
- package/docs/operations/configuration.md +4 -1
- package/docs/operations/troubleshooting.md +3 -2
- package/docs/reference/runtime-contracts.md +53 -52
- package/docs/subsystems/docs-freshness-gate.md +2 -2
- package/docs/subsystems/fusion.md +9 -4
- package/docs/tools/bg_delegate.md +1 -1
- package/docs/tools/bg_result.md +8 -1
- package/package.json +1 -1
- package/src/core/fusion/artifacts.ts +265 -3
- package/src/core/fusion/child-protocol.ts +3 -12
- package/src/core/fusion/orchestrator.ts +47 -57
- package/src/core/fusion/pi-child.ts +9 -78
- package/src/core/fusion/result-package.ts +550 -3
- package/src/core/fusion/types.ts +89 -1
- package/src/delegate-extension.ts +66 -5
- package/src/fusion-child-extension.ts +12 -126
|
@@ -7,12 +7,18 @@ import { replaceFileDurable } from '../durable-fs.js';
|
|
|
7
7
|
import {
|
|
8
8
|
EMPTY_FUSION_USAGE,
|
|
9
9
|
FUSION_COMMITTED_RESULT_SCHEMA_VERSION,
|
|
10
|
+
FUSION_FAILURE_SUMMARY_SCHEMA_VERSION,
|
|
10
11
|
FUSION_MANIFEST_SCHEMA_VERSION,
|
|
11
12
|
FUSION_VALIDATE_CANDIDATE_CONTRACT_EVENT_SCHEMA_VERSION,
|
|
12
13
|
FusionError,
|
|
13
14
|
cloneFusionUsage,
|
|
14
15
|
type FusionArtifactManifest,
|
|
15
16
|
type FusionArtifactRef,
|
|
17
|
+
type FusionFailureArtifactClassification,
|
|
18
|
+
type FusionFailureAttemptMetadata,
|
|
19
|
+
type FusionFailureEvidenceArtifact,
|
|
20
|
+
type FusionFailureSummaryV1,
|
|
21
|
+
type FusionRunProgress,
|
|
16
22
|
type FusionAttemptArtifactRecord,
|
|
17
23
|
type FusionBudgetPlanV1,
|
|
18
24
|
type FusionCalibrationViolation,
|
|
@@ -318,6 +324,210 @@ function artifactRefSha256Hex(value: string): string {
|
|
|
318
324
|
return hex;
|
|
319
325
|
}
|
|
320
326
|
|
|
327
|
+
/** A manifest artifact reference is always one safe basename below its run directory. */
|
|
328
|
+
export function assertFusionArtifactBasename(name: string): string {
|
|
329
|
+
if (
|
|
330
|
+
name.length === 0 ||
|
|
331
|
+
name !== basename(name) ||
|
|
332
|
+
name.includes('/') ||
|
|
333
|
+
name.includes('\\') ||
|
|
334
|
+
name === '.' ||
|
|
335
|
+
name === '..' ||
|
|
336
|
+
Buffer.byteLength(name, 'utf8') > 255
|
|
337
|
+
) {
|
|
338
|
+
throw errorForArtifact(`invalid fusion artifact name: ${name}`);
|
|
339
|
+
}
|
|
340
|
+
return name;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export const FUSION_FAILURE_SUMMARY_INLINE_MESSAGE_BYTES = 1024;
|
|
344
|
+
export const FUSION_FAILURE_SUMMARY_ATTEMPT_CAP = 12;
|
|
345
|
+
export const FUSION_FAILURE_SUMMARY_EVIDENCE_CAP = 24;
|
|
346
|
+
export const FUSION_FAILURE_SUMMARY_MAX_BYTES = 32 * 1024;
|
|
347
|
+
|
|
348
|
+
function compareArtifactText(left: string, right: string): number {
|
|
349
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
interface FusionProgressManifest {
|
|
353
|
+
state: FusionState;
|
|
354
|
+
artifacts: Readonly<Record<string, FusionArtifactRef>>;
|
|
355
|
+
attempts: readonly {
|
|
356
|
+
stage: FusionStage;
|
|
357
|
+
slot?: 1 | 2 | 3 | undefined;
|
|
358
|
+
status: 'completed' | 'failed' | 'cancelled';
|
|
359
|
+
child_created: boolean;
|
|
360
|
+
}[];
|
|
361
|
+
usage: FusionUsage;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
function failureStageProgress(
|
|
365
|
+
manifest: FusionProgressManifest,
|
|
366
|
+
stage: FusionStage,
|
|
367
|
+
): FusionRunProgress['candidates'] {
|
|
368
|
+
const attempts = manifest.attempts.filter((attempt) => attempt.stage === stage);
|
|
369
|
+
const created = attempts.filter((attempt) => attempt.child_created).length;
|
|
370
|
+
const completed = attempts.filter(
|
|
371
|
+
(attempt) => attempt.child_created && attempt.status === 'completed',
|
|
372
|
+
).length;
|
|
373
|
+
const failed = attempts.filter(
|
|
374
|
+
(attempt) => attempt.child_created && attempt.status === 'failed',
|
|
375
|
+
).length;
|
|
376
|
+
const cancelled = attempts.filter(
|
|
377
|
+
(attempt) => attempt.child_created && attempt.status === 'cancelled',
|
|
378
|
+
).length;
|
|
379
|
+
const completedByState =
|
|
380
|
+
stage === 'candidate'
|
|
381
|
+
? completed >= 3
|
|
382
|
+
: stage === 'evaluation'
|
|
383
|
+
? manifest.artifacts['evaluation.json'] !== undefined ||
|
|
384
|
+
manifest.state === 'evaluation_complete' ||
|
|
385
|
+
manifest.state === 'merging' ||
|
|
386
|
+
manifest.state === 'completed'
|
|
387
|
+
: manifest.artifacts['merged.md'] !== undefined || manifest.state === 'completed';
|
|
388
|
+
const progress: FusionRunProgress['candidates'] = {
|
|
389
|
+
status: completedByState ? 'completed' : created === 0 ? 'not_started' : 'incomplete',
|
|
390
|
+
attempts_recorded: attempts.length,
|
|
391
|
+
children_created: created,
|
|
392
|
+
children_completed: completed,
|
|
393
|
+
children_failed: failed,
|
|
394
|
+
children_cancelled: cancelled,
|
|
395
|
+
};
|
|
396
|
+
if (stage === 'candidate') {
|
|
397
|
+
const createdSlots = new Set(
|
|
398
|
+
attempts.flatMap((attempt) =>
|
|
399
|
+
attempt.child_created && attempt.slot !== undefined ? [attempt.slot] : [],
|
|
400
|
+
),
|
|
401
|
+
);
|
|
402
|
+
progress.not_started_slots = 3 - createdSlots.size;
|
|
403
|
+
}
|
|
404
|
+
return progress;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/** Derive terminal progress solely from the durable manifest. */
|
|
408
|
+
export function buildFusionRunProgress(manifest: FusionProgressManifest): FusionRunProgress {
|
|
409
|
+
return {
|
|
410
|
+
manifest_state: manifest.state,
|
|
411
|
+
candidates: failureStageProgress(manifest, 'candidate'),
|
|
412
|
+
evaluation: failureStageProgress(manifest, 'evaluation'),
|
|
413
|
+
merge: failureStageProgress(manifest, 'merge'),
|
|
414
|
+
usage_so_far: cloneFusionUsage(manifest.usage),
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function terminalMessageMetadata(message: string): FusionFailureSummaryV1['failure']['message'] {
|
|
419
|
+
const bytes = Buffer.from(message, 'utf8');
|
|
420
|
+
return {
|
|
421
|
+
byte_length: bytes.length,
|
|
422
|
+
sha256: sha256Buffer(bytes),
|
|
423
|
+
...(bytes.length <= FUSION_FAILURE_SUMMARY_INLINE_MESSAGE_BYTES
|
|
424
|
+
? { inline_message: message }
|
|
425
|
+
: { omission_reason: 'exceeds_inline_message_bytes_cap' as const }),
|
|
426
|
+
};
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
function failureArtifactClassification(
|
|
430
|
+
name: string,
|
|
431
|
+
ref: FusionArtifactRef,
|
|
432
|
+
manifest: FusionArtifactManifest,
|
|
433
|
+
): FusionFailureArtifactClassification {
|
|
434
|
+
for (const attempt of manifest.attempts) {
|
|
435
|
+
if (attempt.response_path === name) {
|
|
436
|
+
return ref.byte_length === 0 && attempt.status !== 'completed'
|
|
437
|
+
? 'empty_rejected_output'
|
|
438
|
+
: 'complete_stage_output';
|
|
439
|
+
}
|
|
440
|
+
if (attempt.partial_response_path === name) return 'partial_stage_output';
|
|
441
|
+
if (attempt.output_recovery?.original_response_path === name) return 'oversized_original';
|
|
442
|
+
}
|
|
443
|
+
return 'evidence_only';
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function failureAttemptMetadata(manifest: FusionArtifactManifest): readonly FusionFailureAttemptMetadata[] {
|
|
447
|
+
return manifest.attempts
|
|
448
|
+
.map((attempt) => ({
|
|
449
|
+
stage: attempt.stage,
|
|
450
|
+
...(attempt.slot === undefined ? {} : { slot: attempt.slot }),
|
|
451
|
+
attempt: attempt.attempt,
|
|
452
|
+
status: attempt.status,
|
|
453
|
+
child_created: attempt.child_created,
|
|
454
|
+
}))
|
|
455
|
+
.sort((left, right) =>
|
|
456
|
+
compareArtifactText(left.stage, right.stage) ||
|
|
457
|
+
(left.slot ?? 0) - (right.slot ?? 0) ||
|
|
458
|
+
left.attempt - right.attempt,
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
export function buildFusionFailureSummary(input: {
|
|
463
|
+
manifest: FusionArtifactManifest;
|
|
464
|
+
terminalError: FusionError;
|
|
465
|
+
progress: FusionRunProgress;
|
|
466
|
+
terminalState: Exclude<FusionTerminalState, 'completed'>;
|
|
467
|
+
createdAt: string;
|
|
468
|
+
}): FusionFailureSummaryV1 {
|
|
469
|
+
if (
|
|
470
|
+
(input.terminalState !== 'failed' && input.terminalState !== 'cancelled') ||
|
|
471
|
+
input.manifest.state !== input.terminalState
|
|
472
|
+
) {
|
|
473
|
+
throw errorForArtifact('failure summary requires a matching failed/cancelled terminal manifest');
|
|
474
|
+
}
|
|
475
|
+
if (input.manifest.error !== input.terminalError.message) {
|
|
476
|
+
throw errorForArtifact('failure summary terminal error does not match the durable manifest');
|
|
477
|
+
}
|
|
478
|
+
if (canonicalJson(input.progress) !== canonicalJson(buildFusionRunProgress(input.manifest))) {
|
|
479
|
+
throw errorForArtifact('failure summary progress does not match the durable terminal manifest');
|
|
480
|
+
}
|
|
481
|
+
if (input.manifest.artifacts['failure-summary.json'] !== undefined) {
|
|
482
|
+
throw errorForArtifact('failure summary already exists in the terminal manifest');
|
|
483
|
+
}
|
|
484
|
+
const attempts = failureAttemptMetadata(input.manifest);
|
|
485
|
+
const evidence: FusionFailureEvidenceArtifact[] = Object.entries(input.manifest.artifacts)
|
|
486
|
+
.map(([name, ref]) => ({
|
|
487
|
+
name: assertFusionArtifactBasename(name),
|
|
488
|
+
classification: failureArtifactClassification(name, ref, input.manifest),
|
|
489
|
+
ref: { ...ref },
|
|
490
|
+
}))
|
|
491
|
+
.sort((left, right) => compareArtifactText(left.name, right.name));
|
|
492
|
+
return {
|
|
493
|
+
schema_version: FUSION_FAILURE_SUMMARY_SCHEMA_VERSION,
|
|
494
|
+
run_id: input.manifest.run_id,
|
|
495
|
+
workflow: input.manifest.workflow,
|
|
496
|
+
source: input.manifest.source,
|
|
497
|
+
terminal_state: input.terminalState,
|
|
498
|
+
created_at: input.createdAt,
|
|
499
|
+
answer: { present: false, reason: 'run_did_not_commit' },
|
|
500
|
+
failure: {
|
|
501
|
+
code: input.terminalError.code,
|
|
502
|
+
...(input.terminalError.stage === undefined ? {} : { stage: input.terminalError.stage }),
|
|
503
|
+
...(input.terminalError.slot === undefined ? {} : { slot: input.terminalError.slot }),
|
|
504
|
+
...(input.terminalError.attempt === undefined
|
|
505
|
+
? {}
|
|
506
|
+
: { attempt: input.terminalError.attempt }),
|
|
507
|
+
child_created: input.terminalError.childCreated,
|
|
508
|
+
message: terminalMessageMetadata(input.terminalError.message),
|
|
509
|
+
},
|
|
510
|
+
progress: input.progress,
|
|
511
|
+
usage_so_far: cloneFusionUsage(input.manifest.usage),
|
|
512
|
+
attempts: {
|
|
513
|
+
listed: attempts.filter((_attempt, index) => index < FUSION_FAILURE_SUMMARY_ATTEMPT_CAP),
|
|
514
|
+
omitted_count:
|
|
515
|
+
attempts.length - Math.min(attempts.length, FUSION_FAILURE_SUMMARY_ATTEMPT_CAP),
|
|
516
|
+
},
|
|
517
|
+
evidence_artifacts: {
|
|
518
|
+
listed: evidence.filter((_artifact, index) => index < FUSION_FAILURE_SUMMARY_EVIDENCE_CAP),
|
|
519
|
+
omitted_count:
|
|
520
|
+
evidence.length - Math.min(evidence.length, FUSION_FAILURE_SUMMARY_EVIDENCE_CAP),
|
|
521
|
+
},
|
|
522
|
+
remediation_ids: [
|
|
523
|
+
'inspect_manifest_bound_evidence',
|
|
524
|
+
'inspect_terminal_error',
|
|
525
|
+
'split_or_reduce_work',
|
|
526
|
+
'retry_same_route_after_operator_review',
|
|
527
|
+
],
|
|
528
|
+
};
|
|
529
|
+
}
|
|
530
|
+
|
|
321
531
|
export class FusionArtifactStore {
|
|
322
532
|
private readonly runDirAbs: string;
|
|
323
533
|
private readonly runDirDisplay: string;
|
|
@@ -534,6 +744,59 @@ export class FusionArtifactStore {
|
|
|
534
744
|
});
|
|
535
745
|
}
|
|
536
746
|
|
|
747
|
+
/**
|
|
748
|
+
* Writes the terminal evidence summary exactly once after writeError has made
|
|
749
|
+
* the manifest terminal. The summary deliberately contains refs only, never
|
|
750
|
+
* stage-output bodies.
|
|
751
|
+
*/
|
|
752
|
+
async writeFailureSummary(summary: FusionFailureSummaryV1): Promise<FusionArtifactRef> {
|
|
753
|
+
if (
|
|
754
|
+
this.manifest.state !== 'failed' &&
|
|
755
|
+
this.manifest.state !== 'cancelled'
|
|
756
|
+
) {
|
|
757
|
+
throw errorForArtifact('failure summary requires a failed/cancelled terminal manifest');
|
|
758
|
+
}
|
|
759
|
+
if (summary.terminal_state !== this.manifest.state) {
|
|
760
|
+
throw errorForArtifact('failure summary terminal state does not match the manifest');
|
|
761
|
+
}
|
|
762
|
+
if (
|
|
763
|
+
summary.run_id !== this.manifest.run_id ||
|
|
764
|
+
summary.workflow !== this.manifest.workflow ||
|
|
765
|
+
summary.source !== this.manifest.source
|
|
766
|
+
) {
|
|
767
|
+
throw errorForArtifact('failure summary identity does not match the terminal manifest');
|
|
768
|
+
}
|
|
769
|
+
if (
|
|
770
|
+
summary.answer?.present !== false ||
|
|
771
|
+
summary.answer.reason !== 'run_did_not_commit'
|
|
772
|
+
) {
|
|
773
|
+
throw errorForArtifact('failure summary must assert that no answer was committed');
|
|
774
|
+
}
|
|
775
|
+
if (this.manifest.error === undefined || this.manifest.artifacts['error.json'] === undefined) {
|
|
776
|
+
throw errorForArtifact('failure summary requires durable terminal error evidence');
|
|
777
|
+
}
|
|
778
|
+
if (
|
|
779
|
+
canonicalJson(summary.failure.message) !==
|
|
780
|
+
canonicalJson(terminalMessageMetadata(this.manifest.error))
|
|
781
|
+
) {
|
|
782
|
+
throw errorForArtifact('failure summary terminal error metadata does not match the manifest');
|
|
783
|
+
}
|
|
784
|
+
if (canonicalJson(summary.progress) !== canonicalJson(buildFusionRunProgress(this.snapshot()))) {
|
|
785
|
+
throw errorForArtifact('failure summary progress does not match the terminal manifest');
|
|
786
|
+
}
|
|
787
|
+
if (canonicalJson(summary.usage_so_far) !== canonicalJson(this.manifest.usage)) {
|
|
788
|
+
throw errorForArtifact('failure summary usage does not match the terminal manifest');
|
|
789
|
+
}
|
|
790
|
+
if (this.manifest.artifacts['failure-summary.json'] !== undefined) {
|
|
791
|
+
throw errorForArtifact('failure summary is already bound in the manifest');
|
|
792
|
+
}
|
|
793
|
+
const bytes = Buffer.from(`${canonicalJson(summary)}\n`, 'utf8');
|
|
794
|
+
if (bytes.length > FUSION_FAILURE_SUMMARY_MAX_BYTES) {
|
|
795
|
+
throw errorForArtifact('failure summary exceeds its bounded diagnostics artifact limit');
|
|
796
|
+
}
|
|
797
|
+
return this.writeArtifact('failure-summary.json', bytes);
|
|
798
|
+
}
|
|
799
|
+
|
|
537
800
|
async recordChildAttempt(input: RecordFusionChildAttemptInput): Promise<void> {
|
|
538
801
|
const prefix = attemptPrefix(input.result.stage, input.result.slot, input.result.attempt);
|
|
539
802
|
await this.writeArtifact(`${prefix}.system-prompt.txt`, input.systemPrompt);
|
|
@@ -664,6 +927,7 @@ export class FusionArtifactStore {
|
|
|
664
927
|
});
|
|
665
928
|
}
|
|
666
929
|
|
|
930
|
+
/** Writes a durable artifact then binds its exact bytes in the manifest. */
|
|
667
931
|
private async writeArtifact(name: string, data: Buffer | string): Promise<FusionArtifactRef> {
|
|
668
932
|
const absPath = this.artifactPath(name);
|
|
669
933
|
const ref = await writePrivateFile(absPath, data);
|
|
@@ -674,9 +938,7 @@ export class FusionArtifactStore {
|
|
|
674
938
|
}
|
|
675
939
|
|
|
676
940
|
private artifactPath(name: string): string {
|
|
677
|
-
|
|
678
|
-
throw errorForArtifact(`invalid fusion artifact name: ${name}`);
|
|
679
|
-
}
|
|
941
|
+
assertFusionArtifactBasename(name);
|
|
680
942
|
const absPath = join(this.runDirAbs, name);
|
|
681
943
|
if (!pathInside(this.runDirAbs, absPath)) {
|
|
682
944
|
throw errorForArtifact(`fusion artifact path escapes run directory: ${name}`);
|
|
@@ -21,25 +21,21 @@ export const FUSION_TOOL_CALL_SEAL_SCHEMA_VERSION =
|
|
|
21
21
|
'pi-background-tasks.fusion-tool-call-seal.v1' as const;
|
|
22
22
|
export const FUSION_TOOL_CALL_SEAL_SUFFIX = '.seal.json';
|
|
23
23
|
export const FUSION_RUNTIME_GUARD_SCHEMA_VERSION =
|
|
24
|
-
'pi-background-tasks.fusion-runtime-guard.
|
|
24
|
+
'pi-background-tasks.fusion-runtime-guard.v2' as const;
|
|
25
25
|
export const FUSION_RUNTIME_GUARD_PREFIX = '\u001ePI_FUSION_RUNTIME_GUARD ';
|
|
26
26
|
export const FUSION_CHILD_MAX_PROVIDER_REQUESTS = 128;
|
|
27
27
|
export const FUSION_CHILD_MAX_TOOL_CALLS = 192;
|
|
28
|
-
export const FUSION_CHILD_MIN_OUTPUT_RESERVE_TOKENS = 32_768;
|
|
29
|
-
export const FUSION_CHILD_SAFETY_RESERVE_TOKENS = 4_096;
|
|
30
28
|
|
|
31
29
|
/**
|
|
32
30
|
* Aggregate ceiling on tool-result bytes a single candidate child may accumulate.
|
|
33
31
|
*
|
|
34
|
-
* The byte ceiling complements the
|
|
35
|
-
*
|
|
36
|
-
* compaction keeps each individual provider request within the route context window.
|
|
32
|
+
* The byte ceiling complements the tool/request count limits and pre-spawn stage
|
|
33
|
+
* budgets. It remains an independent bound on total tool material across the child run.
|
|
37
34
|
*/
|
|
38
35
|
export const FUSION_CHILD_MAX_TOTAL_TOOL_RESULT_BYTES = 8 * 1024 * 1024;
|
|
39
36
|
|
|
40
37
|
export type FusionRuntimeGuardCode =
|
|
41
38
|
| 'provider_request_limit'
|
|
42
|
-
| 'provider_request_budget'
|
|
43
39
|
| 'provider_payload_invalid'
|
|
44
40
|
| 'claude_cache_policy'
|
|
45
41
|
| 'tool_call_limit';
|
|
@@ -53,11 +49,6 @@ export interface FusionRuntimeGuardRecord {
|
|
|
53
49
|
tool_call_count: number;
|
|
54
50
|
payload_bytes: number;
|
|
55
51
|
payload_sha256: string;
|
|
56
|
-
estimated_input_tokens: number;
|
|
57
|
-
context_window_tokens: number;
|
|
58
|
-
reserved_output_tokens: number;
|
|
59
|
-
safety_reserve_tokens: number;
|
|
60
|
-
allowed_input_tokens: number;
|
|
61
52
|
message: string;
|
|
62
53
|
}
|
|
63
54
|
|
|
@@ -5,6 +5,8 @@ import { FUSION_BUDGET_POLICY, FusionBudget } from './budget.js';
|
|
|
5
5
|
import { assertChildOutputWithinContract } from './output-contract.js';
|
|
6
6
|
import {
|
|
7
7
|
FusionArtifactStore,
|
|
8
|
+
buildFusionFailureSummary,
|
|
9
|
+
buildFusionRunProgress as deriveFusionRunProgress,
|
|
8
10
|
type CreateFusionArtifactStoreOptions,
|
|
9
11
|
type RecordFusionFailedAttemptInput,
|
|
10
12
|
} from './artifacts.js';
|
|
@@ -40,9 +42,7 @@ import {
|
|
|
40
42
|
FUSION_VALIDATE_CANDIDATE_SCHEMA_VERSION,
|
|
41
43
|
FusionError,
|
|
42
44
|
addFusionUsage,
|
|
43
|
-
cloneFusionUsage,
|
|
44
45
|
createEmptyFusionUsage,
|
|
45
|
-
type FusionArtifactManifest,
|
|
46
46
|
type FusionCalibrationViolation,
|
|
47
47
|
type FusionCapability,
|
|
48
48
|
type FusionCanonicalInputV3,
|
|
@@ -176,58 +176,7 @@ function asFusionError(error: unknown, artifactDir: string, messageOverride?: st
|
|
|
176
176
|
});
|
|
177
177
|
}
|
|
178
178
|
|
|
179
|
-
|
|
180
|
-
manifest: FusionArtifactManifest,
|
|
181
|
-
stage: FusionStage,
|
|
182
|
-
): FusionRunProgress['candidates'] {
|
|
183
|
-
const attempts = manifest.attempts.filter((attempt) => attempt.stage === stage);
|
|
184
|
-
const created = attempts.filter((attempt) => attempt.child_created).length;
|
|
185
|
-
const completed = attempts.filter(
|
|
186
|
-
(attempt) => attempt.child_created && attempt.status === 'completed',
|
|
187
|
-
).length;
|
|
188
|
-
const failed = attempts.filter(
|
|
189
|
-
(attempt) => attempt.child_created && attempt.status === 'failed',
|
|
190
|
-
).length;
|
|
191
|
-
const cancelled = attempts.filter(
|
|
192
|
-
(attempt) => attempt.child_created && attempt.status === 'cancelled',
|
|
193
|
-
).length;
|
|
194
|
-
const completedByState =
|
|
195
|
-
stage === 'candidate'
|
|
196
|
-
? completed >= 3
|
|
197
|
-
: stage === 'evaluation'
|
|
198
|
-
? manifest.artifacts['evaluation.json'] !== undefined ||
|
|
199
|
-
manifest.state === 'evaluation_complete' ||
|
|
200
|
-
manifest.state === 'merging' ||
|
|
201
|
-
manifest.state === 'completed'
|
|
202
|
-
: manifest.artifacts['merged.md'] !== undefined || manifest.state === 'completed';
|
|
203
|
-
const progress: FusionRunProgress['candidates'] = {
|
|
204
|
-
status: completedByState ? 'completed' : created === 0 ? 'not_started' : 'incomplete',
|
|
205
|
-
attempts_recorded: attempts.length,
|
|
206
|
-
children_created: created,
|
|
207
|
-
children_completed: completed,
|
|
208
|
-
children_failed: failed,
|
|
209
|
-
children_cancelled: cancelled,
|
|
210
|
-
};
|
|
211
|
-
if (stage === 'candidate') {
|
|
212
|
-
const createdSlots = new Set(
|
|
213
|
-
attempts.flatMap((attempt) =>
|
|
214
|
-
attempt.child_created && attempt.slot !== undefined ? [attempt.slot] : [],
|
|
215
|
-
),
|
|
216
|
-
);
|
|
217
|
-
progress.not_started_slots = 3 - createdSlots.size;
|
|
218
|
-
}
|
|
219
|
-
return progress;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
export function buildFusionRunProgress(manifest: FusionArtifactManifest): FusionRunProgress {
|
|
223
|
-
return {
|
|
224
|
-
manifest_state: manifest.state,
|
|
225
|
-
candidates: fusionStageProgress(manifest, 'candidate'),
|
|
226
|
-
evaluation: fusionStageProgress(manifest, 'evaluation'),
|
|
227
|
-
merge: fusionStageProgress(manifest, 'merge'),
|
|
228
|
-
usage_so_far: cloneFusionUsage(manifest.usage),
|
|
229
|
-
};
|
|
230
|
-
}
|
|
179
|
+
export { buildFusionRunProgress } from './artifacts.js';
|
|
231
180
|
|
|
232
181
|
function formatFusionRunStage(name: string, stage: FusionRunProgress['candidates']): string {
|
|
233
182
|
const notStarted =
|
|
@@ -237,7 +186,31 @@ function formatFusionRunStage(name: string, stage: FusionRunProgress['candidates
|
|
|
237
186
|
return `${name}=${stage.status} (${String(stage.children_created)} created, ${String(stage.children_completed)} completed, ${String(stage.children_failed)} failed, ${String(stage.children_cancelled)} cancelled${notStarted})`;
|
|
238
187
|
}
|
|
239
188
|
|
|
240
|
-
export function
|
|
189
|
+
export function summaryUnavailableNote(error: unknown): string {
|
|
190
|
+
const detail = errorText(error);
|
|
191
|
+
const detailBytes = Buffer.from(detail, 'utf8');
|
|
192
|
+
if (detailBytes.length > 1024) {
|
|
193
|
+
return 'failure-summary.json unavailable after terminal publication; write failure detail omitted because it exceeds the 1024-byte diagnostic cap.';
|
|
194
|
+
}
|
|
195
|
+
return `failure-summary.json unavailable after terminal publication: ${detail}`;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function withSummaryUnavailableNote(error: FusionError, summaryError: unknown): FusionError {
|
|
199
|
+
const details: FusionErrorDetails = {
|
|
200
|
+
code: error.code,
|
|
201
|
+
transient: error.transient,
|
|
202
|
+
childCreated: error.childCreated,
|
|
203
|
+
};
|
|
204
|
+
if (error.artifactDir !== undefined) details.artifactDir = error.artifactDir;
|
|
205
|
+
if (error.stage !== undefined) details.stage = error.stage;
|
|
206
|
+
if (error.slot !== undefined) details.slot = error.slot;
|
|
207
|
+
if (error.attempt !== undefined) details.attempt = error.attempt;
|
|
208
|
+
if (error.budget !== undefined) details.budget = error.budget;
|
|
209
|
+
if (error.runProgress !== undefined) details.runProgress = error.runProgress;
|
|
210
|
+
return new FusionError(`${error.message}\n${summaryUnavailableNote(summaryError)}`, details);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function formatFusionRunProgress(progress: FusionRunProgress): string {
|
|
241
214
|
const usage = progress.usage_so_far;
|
|
242
215
|
const optionalUsage = [
|
|
243
216
|
usage.cacheWrite1h === undefined ? undefined : `cacheWrite1h=${String(usage.cacheWrite1h)}`,
|
|
@@ -962,9 +935,26 @@ export class FusionOrchestrator {
|
|
|
962
935
|
terminalError = withRunProgress(
|
|
963
936
|
error,
|
|
964
937
|
store.artifactDir,
|
|
965
|
-
|
|
938
|
+
deriveFusionRunProgress(store.snapshot()),
|
|
966
939
|
);
|
|
967
|
-
|
|
940
|
+
const terminalState = cancelled ? 'cancelled' : 'failed';
|
|
941
|
+
await store.writeError(terminalState, terminalError.message);
|
|
942
|
+
// The terminal manifest/error are authoritative. Summary persistence is
|
|
943
|
+
// subordinate and intentionally attempted once from that fresh snapshot.
|
|
944
|
+
try {
|
|
945
|
+
const terminalManifest = store.snapshot();
|
|
946
|
+
await store.writeFailureSummary(
|
|
947
|
+
buildFusionFailureSummary({
|
|
948
|
+
manifest: terminalManifest,
|
|
949
|
+
terminalError,
|
|
950
|
+
progress: deriveFusionRunProgress(terminalManifest),
|
|
951
|
+
terminalState,
|
|
952
|
+
createdAt: terminalManifest.updated_at,
|
|
953
|
+
}),
|
|
954
|
+
);
|
|
955
|
+
} catch (summaryError) {
|
|
956
|
+
terminalError = withSummaryUnavailableNote(terminalError, summaryError);
|
|
957
|
+
}
|
|
968
958
|
} catch (artifactError) {
|
|
969
959
|
throw withTerminalArtifactFailure(error, store.artifactDir, artifactError);
|
|
970
960
|
}
|
|
@@ -9,10 +9,8 @@ import {
|
|
|
9
9
|
FUSION_CANDIDATE_OUTPUT_RECOVERY_PATH_ENV,
|
|
10
10
|
FUSION_CHILD_MAX_PROVIDER_REQUESTS,
|
|
11
11
|
FUSION_CHILD_MAX_TOOL_CALLS,
|
|
12
|
-
FUSION_CHILD_MIN_OUTPUT_RESERVE_TOKENS,
|
|
13
12
|
FUSION_CHILD_RESULT_PREFIX,
|
|
14
13
|
FUSION_CHILD_RESULT_SCHEMA_VERSION,
|
|
15
|
-
FUSION_CHILD_SAFETY_RESERVE_TOKENS,
|
|
16
14
|
FUSION_CHILD_SETTLEMENT_PREFIX,
|
|
17
15
|
FUSION_CHILD_SETTLEMENT_SCHEMA_VERSION,
|
|
18
16
|
FUSION_RESEARCH_ENABLED_ENV,
|
|
@@ -854,7 +852,6 @@ function parseChildResultMetadata(value: unknown): FusionChildResultMetadata {
|
|
|
854
852
|
|
|
855
853
|
const FUSION_RUNTIME_GUARD_CODES = new Set<FusionRuntimeGuardCode>([
|
|
856
854
|
'provider_request_limit',
|
|
857
|
-
'provider_request_budget',
|
|
858
855
|
'provider_payload_invalid',
|
|
859
856
|
'claude_cache_policy',
|
|
860
857
|
'tool_call_limit',
|
|
@@ -885,11 +882,6 @@ export function parseFusionRuntimeGuard(stderr: Buffer): FusionRuntimeGuardRecor
|
|
|
885
882
|
'tool_call_count',
|
|
886
883
|
'payload_bytes',
|
|
887
884
|
'payload_sha256',
|
|
888
|
-
'estimated_input_tokens',
|
|
889
|
-
'context_window_tokens',
|
|
890
|
-
'reserved_output_tokens',
|
|
891
|
-
'safety_reserve_tokens',
|
|
892
|
-
'allowed_input_tokens',
|
|
893
885
|
'message',
|
|
894
886
|
],
|
|
895
887
|
'fusion runtime guard',
|
|
@@ -914,57 +906,13 @@ export function parseFusionRuntimeGuard(stderr: Buffer): FusionRuntimeGuardRecor
|
|
|
914
906
|
tool_call_count: requireUsageInteger(record, 'tool_call_count', 'fusion runtime guard'),
|
|
915
907
|
payload_bytes: requireUsageInteger(record, 'payload_bytes', 'fusion runtime guard'),
|
|
916
908
|
payload_sha256: requireSha256(record, 'payload_sha256', 'fusion runtime guard'),
|
|
917
|
-
estimated_input_tokens: requireUsageInteger(
|
|
918
|
-
record,
|
|
919
|
-
'estimated_input_tokens',
|
|
920
|
-
'fusion runtime guard',
|
|
921
|
-
),
|
|
922
|
-
context_window_tokens: requireUsageInteger(
|
|
923
|
-
record,
|
|
924
|
-
'context_window_tokens',
|
|
925
|
-
'fusion runtime guard',
|
|
926
|
-
),
|
|
927
|
-
reserved_output_tokens: requirePositiveSafeInteger(
|
|
928
|
-
record,
|
|
929
|
-
'reserved_output_tokens',
|
|
930
|
-
'fusion runtime guard',
|
|
931
|
-
),
|
|
932
|
-
safety_reserve_tokens: requirePositiveSafeInteger(
|
|
933
|
-
record,
|
|
934
|
-
'safety_reserve_tokens',
|
|
935
|
-
'fusion runtime guard',
|
|
936
|
-
),
|
|
937
|
-
allowed_input_tokens: requireUsageInteger(
|
|
938
|
-
record,
|
|
939
|
-
'allowed_input_tokens',
|
|
940
|
-
'fusion runtime guard',
|
|
941
|
-
),
|
|
942
909
|
message: requireNonBlankString(record, 'message', 'fusion runtime guard'),
|
|
943
910
|
};
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
911
|
+
const emptyPayloadHash = createHash('sha256').update(Buffer.alloc(0)).digest('hex');
|
|
912
|
+
if (frame.code === 'claude_cache_policy' || frame.code === 'provider_payload_invalid') {
|
|
913
|
+
if (frame.payload_bytes !== 0 || frame.payload_sha256 !== emptyPayloadHash) {
|
|
914
|
+
throw new Error('fusion runtime guard invalid-payload evidence mismatch');
|
|
947
915
|
}
|
|
948
|
-
const expectedAllowed =
|
|
949
|
-
frame.context_window_tokens - frame.reserved_output_tokens - frame.safety_reserve_tokens;
|
|
950
|
-
if (expectedAllowed < 0 || frame.allowed_input_tokens !== expectedAllowed) {
|
|
951
|
-
throw new Error('fusion runtime guard allowed-input arithmetic mismatch');
|
|
952
|
-
}
|
|
953
|
-
}
|
|
954
|
-
if (frame.code === 'claude_cache_policy') {
|
|
955
|
-
if (
|
|
956
|
-
frame.payload_bytes !== 0 ||
|
|
957
|
-
frame.payload_sha256 !== createHash('sha256').update(Buffer.alloc(0)).digest('hex') ||
|
|
958
|
-
frame.estimated_input_tokens !== 0
|
|
959
|
-
) {
|
|
960
|
-
throw new Error('fusion runtime guard Claude cache policy payload evidence mismatch');
|
|
961
|
-
}
|
|
962
|
-
}
|
|
963
|
-
if (
|
|
964
|
-
frame.code === 'provider_request_budget' &&
|
|
965
|
-
frame.estimated_input_tokens <= frame.allowed_input_tokens
|
|
966
|
-
) {
|
|
967
|
-
throw new Error('fusion runtime guard provider budget code has no token overage');
|
|
968
916
|
}
|
|
969
917
|
if (
|
|
970
918
|
frame.code === 'provider_request_limit' &&
|
|
@@ -976,11 +924,7 @@ export function parseFusionRuntimeGuard(stderr: Buffer): FusionRuntimeGuardRecor
|
|
|
976
924
|
if (frame.tool_call_count <= FUSION_CHILD_MAX_TOOL_CALLS) {
|
|
977
925
|
throw new Error('fusion runtime guard tool call limit was not exceeded');
|
|
978
926
|
}
|
|
979
|
-
if (
|
|
980
|
-
frame.payload_bytes !== 0 ||
|
|
981
|
-
frame.payload_sha256 !== createHash('sha256').update(Buffer.alloc(0)).digest('hex') ||
|
|
982
|
-
frame.estimated_input_tokens !== 0
|
|
983
|
-
) {
|
|
927
|
+
if (frame.payload_bytes !== 0 || frame.payload_sha256 !== emptyPayloadHash) {
|
|
984
928
|
throw new Error('fusion runtime guard tool call limit payload evidence mismatch');
|
|
985
929
|
}
|
|
986
930
|
}
|
|
@@ -1178,22 +1122,7 @@ export function assertFusionRuntimeGuardMatchesModel(
|
|
|
1178
1122
|
);
|
|
1179
1123
|
}
|
|
1180
1124
|
if (routeUnknown && guard.code !== 'provider_payload_invalid') {
|
|
1181
|
-
throw new Error('fusion runtime guard omitted the route for a
|
|
1182
|
-
}
|
|
1183
|
-
if (guard.code === 'provider_payload_invalid') return;
|
|
1184
|
-
const expectedReservedOutput = Math.max(
|
|
1185
|
-
FUSION_CHILD_MIN_OUTPUT_RESERVE_TOKENS,
|
|
1186
|
-
model.maxOutputTokens,
|
|
1187
|
-
);
|
|
1188
|
-
const expectedAllowedInput =
|
|
1189
|
-
model.contextWindow - expectedReservedOutput - FUSION_CHILD_SAFETY_RESERVE_TOKENS;
|
|
1190
|
-
if (
|
|
1191
|
-
guard.context_window_tokens !== model.contextWindow ||
|
|
1192
|
-
guard.reserved_output_tokens !== expectedReservedOutput ||
|
|
1193
|
-
guard.safety_reserve_tokens !== FUSION_CHILD_SAFETY_RESERVE_TOKENS ||
|
|
1194
|
-
guard.allowed_input_tokens !== expectedAllowedInput
|
|
1195
|
-
) {
|
|
1196
|
-
throw new Error('fusion runtime guard capacity evidence does not match the resolved route');
|
|
1125
|
+
throw new Error('fusion runtime guard omitted the route for a route-bound refusal');
|
|
1197
1126
|
}
|
|
1198
1127
|
}
|
|
1199
1128
|
|
|
@@ -2417,7 +2346,9 @@ export async function runPiChild(options: RunPiChildOptions): Promise<FusionChil
|
|
|
2417
2346
|
? 'child_exit_failed'
|
|
2418
2347
|
: runtimeGuard.code === 'claude_cache_policy'
|
|
2419
2348
|
? 'child_cache_policy_invalid'
|
|
2420
|
-
: '
|
|
2349
|
+
: runtimeGuard.code === 'provider_payload_invalid'
|
|
2350
|
+
? 'child_runtime_payload_invalid'
|
|
2351
|
+
: 'child_runtime_limit_exceeded',
|
|
2421
2352
|
options,
|
|
2422
2353
|
),
|
|
2423
2354
|
state.cleanupErrors,
|