pi-harness-runtime 1.1.31 → 1.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/harness/langchain/checkpointer.test.ts +269 -0
- package/harness/langchain/checkpointer.ts +343 -0
- package/harness/langchain/daemon.test.ts +45 -0
- package/harness/langchain/daemon.ts +51 -10
- package/harness/langchain/graph.ts +11 -4
- package/harness/langchain/run.ts +3 -1
- package/harness/langchain/surge.test.ts +214 -0
- package/harness/langchain/surge.ts +217 -0
- package/package.json +8 -2
- package/packages/a2a-adapter/package.json +1 -1
- package/packages/architecture-generator/package.json +1 -1
- package/packages/auth/package.json +1 -1
- package/packages/autonomous-refactor/package.json +1 -1
- package/packages/autonomous-runtime/package.json +1 -1
- package/packages/cache-strategy/package.json +1 -1
- package/packages/capability-registry/package.json +1 -1
- package/packages/checkpoint/package.json +1 -1
- package/packages/cli-plugin-sdk/package.json +1 -1
- package/packages/clipboard/package.json +1 -1
- package/packages/clipboard-plugin/package.json +1 -1
- package/packages/code-generation/package.json +1 -1
- package/packages/code-review/package.json +1 -1
- package/packages/codex-adapter/package.json +1 -1
- package/packages/config-capture/package.json +1 -1
- package/packages/context-compiler/package.json +1 -1
- package/packages/context-discovery/package.json +1 -1
- package/packages/context-manager/package.json +1 -1
- package/packages/cookie-sanitizer/package.json +1 -1
- package/packages/cost-optimizer/package.json +1 -1
- package/packages/dependency-analyzer/package.json +1 -1
- package/packages/django-plugin/package.json +1 -1
- package/packages/doc-generator/package.json +1 -1
- package/packages/evaluation-engine/package.json +1 -1
- package/packages/evaluation-runner/package.json +1 -1
- package/packages/event-bus/package.json +1 -1
- package/packages/event-store/package.json +1 -1
- package/packages/experience-replay/package.json +1 -1
- package/packages/feedback-collector/package.json +1 -1
- package/packages/file-copy-helper/package.json +1 -1
- package/packages/framework-detector/package.json +1 -1
- package/packages/framework-plugin-sdk/package.json +1 -1
- package/packages/frappe-plugin/package.json +1 -1
- package/packages/generic-web-plugin/package.json +1 -1
- package/packages/health-monitor/package.json +1 -1
- package/packages/intent-analyzer/package.json +1 -1
- package/packages/knowledge-graph/package.json +1 -1
- package/packages/knowledge-retrieval/package.json +1 -1
- package/packages/laravel-plugin/package.json +1 -1
- package/packages/learning-engine/package.json +1 -1
- package/packages/mcp-adapter/package.json +1 -1
- package/packages/memory-engine/package.json +1 -1
- package/packages/milestone-manager/package.json +1 -1
- package/packages/model-registry/package.json +1 -1
- package/packages/nextjs-plugin/package.json +1 -1
- package/packages/notification/package.json +1 -1
- package/packages/observability/package.json +1 -1
- package/packages/okf-indexer/package.json +1 -1
- package/packages/performance-optimizer/package.json +1 -1
- package/packages/privilege-broker/package.json +1 -1
- package/packages/project-analyzer/package.json +1 -1
- package/packages/project-bootstrap/package.json +1 -1
- package/packages/projection-engine/package.json +1 -1
- package/packages/prompt-compiler/package.json +1 -1
- package/packages/prompt-versioning/package.json +1 -1
- package/packages/provider-adapter-sdk/package.json +1 -1
- package/packages/provider-router/package.json +1 -1
- package/packages/provider-selector/package.json +1 -1
- package/packages/providers/package.json +1 -1
- package/packages/quota-manager/package.json +1 -1
- package/packages/rate-limiter/package.json +1 -1
- package/packages/react-vite-plugin/package.json +1 -1
- package/packages/release-manager/package.json +1 -1
- package/packages/requirement-compiler/package.json +1 -1
- package/packages/runtime/package.json +1 -1
- package/packages/scheduler/package.json +1 -1
- package/packages/scheduler-adapter/package.json +1 -1
- package/packages/session/package.json +1 -1
- package/packages/session-api/package.json +1 -1
- package/packages/session-export/package.json +1 -1
- package/packages/shared-context/package.json +1 -1
- package/packages/skill-mcp-client/package.json +1 -1
- package/packages/skill-registry/package.json +1 -1
- package/packages/sprint-planner/package.json +1 -1
- package/packages/subscription-engine/package.json +1 -1
- package/packages/task-compiler/package.json +1 -1
- package/packages/tencentdb-memory/package.json +1 -1
- package/packages/tencentdb-sync/package.json +1 -1
- package/packages/test-data-generator/package.json +1 -1
- package/packages/test-generator/package.json +1 -1
- package/packages/todo-bd-sync/package.json +1 -1
- package/packages/token-estimation/package.json +1 -1
- package/packages/token-optimizer/package.json +1 -1
- package/packages/tui/package.json +1 -1
- package/packages/types/package.json +1 -1
- package/packages/workflow-events/package.json +1 -1
- package/packages/workspace-scanner/package.json +1 -1
- package/packages/worktree/package.json +1 -1
- package/packages/write-review/package.json +1 -1
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
* M1 — Daemon skeleton + inbox/bus watchers
|
|
21
21
|
* M2 — Runner integration: LeaseManager → buildWriteReviewLoop → publish
|
|
22
22
|
* M3 — Gates & notifications
|
|
23
|
-
* M4 — Robustness: retry
|
|
23
|
+
* M4 — Robustness: surge/529 auto-resume + retry backoff, stale-lease, bd-tasks
|
|
24
|
+
* watcher (worker heartbeats: still TODO)
|
|
24
25
|
*
|
|
25
26
|
* Wiki: wiki/auto-trigger-multi-agent.md
|
|
26
27
|
*/
|
|
@@ -52,6 +53,9 @@ import {
|
|
|
52
53
|
} from "../../packages/event-bus/src/herdr-bus.js";
|
|
53
54
|
import { NotificationCenter } from "../../packages/notification/dist/notification-center.js";
|
|
54
55
|
|
|
56
|
+
import { invokeWithSurgeRetry, type SurgePolicy } from "./surge.js";
|
|
57
|
+
import { createLoopCheckpointer } from "./checkpointer.js";
|
|
58
|
+
|
|
55
59
|
import {
|
|
56
60
|
buildDryRunDeps,
|
|
57
61
|
buildRealLoopDeps,
|
|
@@ -84,6 +88,12 @@ export interface DaemonConfig {
|
|
|
84
88
|
dryRun?: boolean;
|
|
85
89
|
/** Abort loop if cumulative cost exceeds this. Default: no cap */
|
|
86
90
|
costCapUsd?: number;
|
|
91
|
+
/** Surge (529/overloaded) retry policy. Default: 3→6→12 min, max 5 pauses */
|
|
92
|
+
surgePolicy?: Partial<SurgePolicy>;
|
|
93
|
+
/** Use persistent file-based checkpointer for crash-resume. Default: false (MemorySaver). */
|
|
94
|
+
checkpointer?: boolean | string;
|
|
95
|
+
/** @internal Test injection — overrides dryRun/real deps entirely */
|
|
96
|
+
deps?: LoopDeps;
|
|
87
97
|
/** Which watchers to enable. Default: ["inbox", "bus"] */
|
|
88
98
|
sources?: TriggerSource[];
|
|
89
99
|
/** Herdr workspace override. Default: from herdr-bus */
|
|
@@ -530,6 +540,7 @@ export class LoopDaemon {
|
|
|
530
540
|
sources: config.sources ?? ["inbox", "bus"],
|
|
531
541
|
workspace: config.workspace ?? getHerdrWorkspace(),
|
|
532
542
|
notificationConfig: config.notificationConfig ?? undefined,
|
|
543
|
+
checkpointer: config.checkpointer,
|
|
533
544
|
} as Required<DaemonConfig>;
|
|
534
545
|
|
|
535
546
|
this.agentId = this.config.agentId;
|
|
@@ -692,9 +703,13 @@ export class LoopDaemon {
|
|
|
692
703
|
}
|
|
693
704
|
|
|
694
705
|
// ── Build loop deps ───────────────────────────────────────────────
|
|
695
|
-
const deps: LoopDeps =
|
|
696
|
-
|
|
697
|
-
|
|
706
|
+
const deps: LoopDeps =
|
|
707
|
+
this.config.deps ??
|
|
708
|
+
(this.config.dryRun
|
|
709
|
+
? buildDryRunDeps({ maxIterations: this.config.maxIterations })
|
|
710
|
+
: await buildRealLoopDeps({
|
|
711
|
+
maxIterations: this.config.maxIterations,
|
|
712
|
+
}));
|
|
698
713
|
|
|
699
714
|
// Inject transition publishing into onStep
|
|
700
715
|
const originalOnStep = deps.onStep;
|
|
@@ -703,15 +718,41 @@ export class LoopDaemon {
|
|
|
703
718
|
this._publishTransition(loopId, step, state);
|
|
704
719
|
};
|
|
705
720
|
|
|
706
|
-
// ── Run the graph
|
|
721
|
+
// ── Run the graph (surge-aware: 529/overloaded pauses auto-resume) ──
|
|
707
722
|
log(
|
|
708
723
|
`Starting write-review loop (maxIterations=${this.config.maxIterations})`,
|
|
709
724
|
);
|
|
710
|
-
const
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
725
|
+
const checkpointer =
|
|
726
|
+
this.config.checkpointer === false
|
|
727
|
+
? false
|
|
728
|
+
: this.config.checkpointer === true || typeof this.config.checkpointer === "string"
|
|
729
|
+
? createLoopCheckpointer(
|
|
730
|
+
this.config.checkpointer === true
|
|
731
|
+
? this.bus.getWorkspace()
|
|
732
|
+
: this.config.checkpointer as string,
|
|
733
|
+
)
|
|
734
|
+
: undefined; // default MemorySaver
|
|
735
|
+
const loop: WriteReviewLoop = buildWriteReviewLoop(deps, { checkpointer });
|
|
736
|
+
|
|
737
|
+
const finalState = await invokeWithSurgeRetry(
|
|
738
|
+
() =>
|
|
739
|
+
loop.invoke(
|
|
740
|
+
{ request: task.request },
|
|
741
|
+
{ configurable: { thread_id: loopId } },
|
|
742
|
+
),
|
|
743
|
+
{
|
|
744
|
+
policy: this.config.surgePolicy,
|
|
745
|
+
onSurge: ({ attempt, delayMs }) =>
|
|
746
|
+
log(
|
|
747
|
+
`Provider surge (529) — resume in ${Math.round(delayMs / 1000)}s (attempt ${attempt})`,
|
|
748
|
+
),
|
|
749
|
+
onExhausted: () => {
|
|
750
|
+
this._notifyHumanReviewNeeded(
|
|
751
|
+
task,
|
|
752
|
+
"Provider surged past max surge attempts — task failed",
|
|
753
|
+
);
|
|
754
|
+
},
|
|
755
|
+
},
|
|
715
756
|
);
|
|
716
757
|
|
|
717
758
|
const verdict = finalState.review?.verdict ?? "blocked";
|
|
@@ -133,7 +133,7 @@ function routeAfterReview(
|
|
|
133
133
|
|
|
134
134
|
export function buildWriteReviewLoop(
|
|
135
135
|
deps: LoopDeps,
|
|
136
|
-
opts: { checkpointer?: boolean } = {},
|
|
136
|
+
opts: { checkpointer?: boolean | unknown } = {},
|
|
137
137
|
) {
|
|
138
138
|
// Node names must not collide with state channel names (LangGraph rule),
|
|
139
139
|
// hence the "*Step" suffixes.
|
|
@@ -153,9 +153,16 @@ export function buildWriteReviewLoop(
|
|
|
153
153
|
)
|
|
154
154
|
.addEdge("finishStep", END);
|
|
155
155
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
156
|
+
// Resolve checkpointer: false=disabled, object=use it, true/undefined=default MemorySaver
|
|
157
|
+
const cp = opts.checkpointer;
|
|
158
|
+
const checkpointerToUse =
|
|
159
|
+
cp === false
|
|
160
|
+
? undefined
|
|
161
|
+
: cp != null && cp !== true
|
|
162
|
+
? // eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
163
|
+
(cp as any)
|
|
164
|
+
: new MemorySaver();
|
|
165
|
+
return builder.compile({ checkpointer: checkpointerToUse });
|
|
159
166
|
}
|
|
160
167
|
|
|
161
168
|
/** Inferred compiled-graph type (do not hand-roll langgraph generics). */
|
package/harness/langchain/run.ts
CHANGED
|
@@ -90,7 +90,9 @@ function parseArgs(argv: string[]): CliArgs {
|
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
if (!args.request && !args.daemon) {
|
|
93
|
-
throw new Error(
|
|
93
|
+
throw new Error(
|
|
94
|
+
'A request is required: --request="..." or a bare string (or use --daemon to start the watcher)',
|
|
95
|
+
);
|
|
94
96
|
}
|
|
95
97
|
return args;
|
|
96
98
|
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Surge auto-resume tests — S2/S4 acceptance (wiki/peak-hour-surge-auto-resume.md)
|
|
3
|
+
*
|
|
4
|
+
* All deterministic via injected sleep/rand — no real timers, no API keys.
|
|
5
|
+
*
|
|
6
|
+
* Run:
|
|
7
|
+
* bun test harness/langchain/surge.test.ts
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { describe, it } from "node:test";
|
|
11
|
+
import assert from "node:assert";
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
classifySurge,
|
|
15
|
+
computeSurgeDelayMs,
|
|
16
|
+
invokeWithSurgeRetry,
|
|
17
|
+
SurgeScheduler,
|
|
18
|
+
type SurgePolicy,
|
|
19
|
+
} from "./surge.js";
|
|
20
|
+
import type { LoopDeps } from "./graph.js";
|
|
21
|
+
|
|
22
|
+
// ─── Fixtures ────────────────────────────────────────────────────────────────
|
|
23
|
+
|
|
24
|
+
/** The exact incident message from the amos-saas E2E failure (2026-08-31). */
|
|
25
|
+
const INCIDENT =
|
|
26
|
+
'529 {"type":"error","error":{"type":"overloaded_error","message":"The system is currently experiencing a peak-hour surge, and the server is temporarily busy. It usually recovers within 1–5 minutes. Please try again shortly (2064)"},"request_id":"06e405e2e5965672576925264941a057"}';
|
|
27
|
+
|
|
28
|
+
const SIGNAL_3MIN = {
|
|
29
|
+
retryAfterMs: 180_000,
|
|
30
|
+
explicit: true,
|
|
31
|
+
sourceText: INCIDENT,
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
const FAST_POLICY: Partial<SurgePolicy> = {
|
|
35
|
+
baseDelayMs: 100,
|
|
36
|
+
multiplier: 2,
|
|
37
|
+
minDelayMs: 10,
|
|
38
|
+
maxDelayMs: 1_000,
|
|
39
|
+
jitterRatio: 0,
|
|
40
|
+
maxAttempts: 3,
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
// ─── T1 Classification ───────────────────────────────────────────────────────
|
|
44
|
+
|
|
45
|
+
describe("T1 — classifySurge", () => {
|
|
46
|
+
it("classifies the incident message (midpoint of 1–5 min = 3 min)", () => {
|
|
47
|
+
const sig = classifySurge(new Error(INCIDENT));
|
|
48
|
+
assert.ok(sig, "should classify");
|
|
49
|
+
assert.equal(sig?.retryAfterMs, 180_000);
|
|
50
|
+
assert.equal(sig?.explicit, true);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it("classifies plain 529 status", () => {
|
|
54
|
+
const sig = classifySurge(new Error("HTTP 529"));
|
|
55
|
+
assert.ok(sig);
|
|
56
|
+
assert.equal(sig?.explicit, false); // no stated delay → 2 min default
|
|
57
|
+
assert.equal(sig?.retryAfterMs, 120_000);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("parses 'retry after 30 seconds'", () => {
|
|
61
|
+
const sig = classifySurge(
|
|
62
|
+
new Error("overloaded — please retry after 30 seconds"),
|
|
63
|
+
);
|
|
64
|
+
assert.ok(sig);
|
|
65
|
+
assert.equal(sig?.retryAfterMs, 30_000);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it("parses retry-after milliseconds", () => {
|
|
69
|
+
const sig = classifySurge(new Error("529; retry-after: 45000ms"));
|
|
70
|
+
assert.ok(sig);
|
|
71
|
+
assert.equal(sig?.retryAfterMs, 45_000);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it("rejects non-surge errors", () => {
|
|
75
|
+
assert.equal(classifySurge(new Error("429 too many requests")), null);
|
|
76
|
+
assert.equal(classifySurge(new Error("ECONNREFUSED")), null);
|
|
77
|
+
assert.equal(classifySurge("file not found"), null);
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
// ─── T2/T3 Delay computation & escalation ────────────────────────────────────
|
|
82
|
+
|
|
83
|
+
describe("T2 — computeSurgeDelayMs escalation", () => {
|
|
84
|
+
it("escalates 3 → 6 → 12 min and caps at max", () => {
|
|
85
|
+
const p = { jitterRatio: 0 } as const; // deterministic
|
|
86
|
+
assert.equal(computeSurgeDelayMs(SIGNAL_3MIN, 1, p), 180_000);
|
|
87
|
+
assert.equal(computeSurgeDelayMs(SIGNAL_3MIN, 2, p), 360_000);
|
|
88
|
+
assert.equal(computeSurgeDelayMs(SIGNAL_3MIN, 3, p), 720_000);
|
|
89
|
+
assert.equal(computeSurgeDelayMs(SIGNAL_3MIN, 9, p), 900_000); // 15 min cap
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it("floors at minDelayMs (never hammer)", () => {
|
|
93
|
+
const d = computeSurgeDelayMs({ ...SIGNAL_3MIN, retryAfterMs: 1_000 }, 1, {
|
|
94
|
+
minDelayMs: 30_000,
|
|
95
|
+
jitterRatio: 0,
|
|
96
|
+
});
|
|
97
|
+
assert.equal(d, 30_000);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("jitter stays inside ±ratio and within bounds", () => {
|
|
101
|
+
const lo = computeSurgeDelayMs(SIGNAL_3MIN, 1, { jitterRatio: 0.2 }, 0);
|
|
102
|
+
const hi = computeSurgeDelayMs(SIGNAL_3MIN, 1, { jitterRatio: 0.2 }, 1);
|
|
103
|
+
assert.ok(lo >= 180_000 * 0.8 && lo < 180_000);
|
|
104
|
+
assert.ok(hi > 180_000 && hi <= 180_000 * 1.2);
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
describe("T3 — SurgeScheduler attempt cap", () => {
|
|
109
|
+
it("counts 1..maxAttempts then null", () => {
|
|
110
|
+
const s = new SurgeScheduler({ maxAttempts: 3 });
|
|
111
|
+
assert.equal(s.nextAttempt(), 1);
|
|
112
|
+
assert.equal(s.nextAttempt(), 2);
|
|
113
|
+
assert.equal(s.nextAttempt(), 3);
|
|
114
|
+
assert.equal(s.nextAttempt(), null);
|
|
115
|
+
assert.equal(s.attempts, 3);
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
// ─── T-surge-retry Wrapper behavior ─────────────────────────────────────────
|
|
120
|
+
|
|
121
|
+
describe("invokeWithSurgeRetry", () => {
|
|
122
|
+
it("retries through a 529 and succeeds (the incident scenario)", async () => {
|
|
123
|
+
const sleeps: number[] = [];
|
|
124
|
+
let calls = 0;
|
|
125
|
+
const result = await invokeWithSurgeRetry(
|
|
126
|
+
async () => {
|
|
127
|
+
calls += 1;
|
|
128
|
+
if (calls < 3) throw new Error(INCIDENT);
|
|
129
|
+
return "approved";
|
|
130
|
+
},
|
|
131
|
+
{ policy: FAST_POLICY, sleep: async (ms) => void sleeps.push(ms) },
|
|
132
|
+
);
|
|
133
|
+
assert.equal(result, "approved");
|
|
134
|
+
assert.equal(calls, 3);
|
|
135
|
+
assert.deepEqual(sleeps, [100, 200]); // escalated, no jitter
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it("rethrows non-surge errors immediately (no retry)", async () => {
|
|
139
|
+
const sleeps: number[] = [];
|
|
140
|
+
let calls = 0;
|
|
141
|
+
await assert.rejects(
|
|
142
|
+
invokeWithSurgeRetry(
|
|
143
|
+
async () => {
|
|
144
|
+
calls += 1;
|
|
145
|
+
throw new Error("ECONNREFUSED");
|
|
146
|
+
},
|
|
147
|
+
{ policy: FAST_POLICY, sleep: async (ms) => void sleeps.push(ms) },
|
|
148
|
+
),
|
|
149
|
+
/ECONNREFUSED/,
|
|
150
|
+
);
|
|
151
|
+
assert.equal(calls, 1);
|
|
152
|
+
assert.equal(sleeps.length, 0);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it("gives up after maxAttempts, fires onExhausted, rethrows last error", async () => {
|
|
156
|
+
const pauses: number[] = [];
|
|
157
|
+
let exhausted = false;
|
|
158
|
+
let calls = 0;
|
|
159
|
+
await assert.rejects(
|
|
160
|
+
invokeWithSurgeRetry(
|
|
161
|
+
async () => {
|
|
162
|
+
calls += 1;
|
|
163
|
+
throw new Error(INCIDENT);
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
policy: FAST_POLICY,
|
|
167
|
+
sleep: async (ms) => void pauses.push(ms),
|
|
168
|
+
onSurge: ({ attempt }) => assert.ok(attempt >= 1 && attempt <= 3),
|
|
169
|
+
onExhausted: () => {
|
|
170
|
+
exhausted = true;
|
|
171
|
+
},
|
|
172
|
+
},
|
|
173
|
+
),
|
|
174
|
+
/overloaded_error/,
|
|
175
|
+
);
|
|
176
|
+
assert.equal(calls, 4); // initial + 3 retries
|
|
177
|
+
assert.deepEqual(pauses, [100, 200, 400]);
|
|
178
|
+
assert.equal(exhausted, true);
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
// ─── T8 Daemon-level: graph deps that surge once, then succeed ───────────────
|
|
183
|
+
|
|
184
|
+
describe("T8 — loop deps surviving a mid-graph 529", () => {
|
|
185
|
+
it("writeStep throws the incident once; retry wrapper completes the loop", async () => {
|
|
186
|
+
const { buildWriteReviewLoop, buildDryRunDeps } = await import("./graph.js");
|
|
187
|
+
|
|
188
|
+
const base = buildDryRunDeps({ maxIterations: 1 });
|
|
189
|
+
let writeCalls = 0;
|
|
190
|
+
const deps: LoopDeps = {
|
|
191
|
+
...base,
|
|
192
|
+
write: async (plan, review) => {
|
|
193
|
+
writeCalls += 1;
|
|
194
|
+
if (writeCalls === 1) throw new Error(INCIDENT);
|
|
195
|
+
return base.write(plan, review);
|
|
196
|
+
},
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
const loop = buildWriteReviewLoop(deps);
|
|
200
|
+
const sleeps: number[] = [];
|
|
201
|
+
const state = await invokeWithSurgeRetry(
|
|
202
|
+
() =>
|
|
203
|
+
loop.invoke(
|
|
204
|
+
{ request: "surge survival test" },
|
|
205
|
+
{ configurable: { thread_id: "t8-surge" } },
|
|
206
|
+
),
|
|
207
|
+
{ policy: FAST_POLICY, sleep: async (ms) => void sleeps.push(ms) },
|
|
208
|
+
);
|
|
209
|
+
|
|
210
|
+
assert.equal(writeCalls, 2); // first surged, retry succeeded
|
|
211
|
+
assert.ok(state.review, "loop produced a review");
|
|
212
|
+
assert.deepEqual(sleeps, [100]);
|
|
213
|
+
});
|
|
214
|
+
});
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Surge auto-resume — 529 / overloaded_error detection, escalation backoff,
|
|
3
|
+
* and a retry wrapper for the daemon's graph invocations.
|
|
4
|
+
*
|
|
5
|
+
* Implements S2 + S4 of wiki/peak-hour-surge-auto-resume.md:
|
|
6
|
+
* seconds-scale blits stay in the provider client; minutes-scale surges are
|
|
7
|
+
* handled HERE with scheduled pauses (3 → 6 → 12 min, jittered, capped).
|
|
8
|
+
*
|
|
9
|
+
* Verdict vocabulary and patterns intentionally match:
|
|
10
|
+
* - harness/loop-runtime.ts 529 path (commit 9ae0ffd)
|
|
11
|
+
* - harness/e2e/glm-quota-scraper.ts parseMinimaxOverloadResetTime()
|
|
12
|
+
* - packages/providers/src/adapters.ts isOverloaded
|
|
13
|
+
*
|
|
14
|
+
* Wiki: wiki/peak-hour-surge-auto-resume.md
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
// ─── Classification ─────────────────────────────────────────────────────────
|
|
18
|
+
|
|
19
|
+
export interface SurgeSignal {
|
|
20
|
+
/** Parsed recovery delay in ms (midpoint of the stated range, or fallback) */
|
|
21
|
+
retryAfterMs: number;
|
|
22
|
+
/** True when the delay came from an explicit provider statement */
|
|
23
|
+
explicit: boolean;
|
|
24
|
+
/** Raw error text for logs */
|
|
25
|
+
sourceText: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const OVERLOADED_PATTERNS: RegExp[] = [
|
|
29
|
+
/529/,
|
|
30
|
+
/overloaded_error/i,
|
|
31
|
+
/peak.?hour.*surge/i,
|
|
32
|
+
/(?:overload|surge|busy)/i,
|
|
33
|
+
];
|
|
34
|
+
|
|
35
|
+
const RECOVERY_RANGE = /recovers? within (\d+)\s*[\u2013-]\s*(\d+)\s*minutes/i;
|
|
36
|
+
const SIMPLE_DELAY = /retry (?:after|in) (\d+)\s*(seconds?|secs?|s)\b/i;
|
|
37
|
+
const RETRY_AFTER_MS = /retry.?after[^0-9]{0,12}(\d{4,})\s*ms/i;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Classify an unknown error as a transient provider surge.
|
|
41
|
+
* Returns null for anything that is not a 529/overload-class failure.
|
|
42
|
+
*/
|
|
43
|
+
export function classifySurge(err: unknown): SurgeSignal | null {
|
|
44
|
+
const sourceText =
|
|
45
|
+
err instanceof Error
|
|
46
|
+
? err.message
|
|
47
|
+
: typeof err === "string"
|
|
48
|
+
? err
|
|
49
|
+
: String(err);
|
|
50
|
+
|
|
51
|
+
if (!OVERLOADED_PATTERNS.some((p) => p.test(sourceText))) {
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// "recovers within 1–5 minutes" → midpoint (3 min)
|
|
56
|
+
const range = sourceText.match(RECOVERY_RANGE);
|
|
57
|
+
if (range) {
|
|
58
|
+
const min = Number.parseInt(range[1] as string, 10);
|
|
59
|
+
const max = Number.parseInt(range[2] as string, 10);
|
|
60
|
+
if (Number.isFinite(min) && Number.isFinite(max) && max >= min) {
|
|
61
|
+
return {
|
|
62
|
+
retryAfterMs: Math.round(((min + max) / 2) * 60_000),
|
|
63
|
+
explicit: true,
|
|
64
|
+
sourceText,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// "retry after 30 seconds"
|
|
70
|
+
const simple = sourceText.match(SIMPLE_DELAY);
|
|
71
|
+
if (simple) {
|
|
72
|
+
const secs = Number.parseInt(simple[1] as string, 10);
|
|
73
|
+
if (Number.isFinite(secs) && secs > 0) {
|
|
74
|
+
return {
|
|
75
|
+
retryAfterMs: secs * 1000,
|
|
76
|
+
explicit: true,
|
|
77
|
+
sourceText,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// "retry-after 120000ms" (header-style)
|
|
83
|
+
const ms = sourceText.match(RETRY_AFTER_MS);
|
|
84
|
+
if (ms) {
|
|
85
|
+
const val = Number.parseInt(ms[1] as string, 10);
|
|
86
|
+
if (Number.isFinite(val) && val > 0) {
|
|
87
|
+
return { retryAfterMs: val, explicit: true, sourceText };
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Overload-class error without a stated delay → conservative default
|
|
92
|
+
return { retryAfterMs: 120_000, explicit: false, sourceText };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// ─── Policy ─────────────────────────────────────────────────────────────────
|
|
96
|
+
|
|
97
|
+
export interface SurgePolicy {
|
|
98
|
+
/** Override the signal's parsed delay (ms). Default: use the signal. */
|
|
99
|
+
baseDelayMs?: number;
|
|
100
|
+
/** Delay multiplier per attempt. Default: 2 (3 → 6 → 12 min) */
|
|
101
|
+
multiplier: number;
|
|
102
|
+
/** Floor for any wait. Default: 30_000 (never hammer a surging provider) */
|
|
103
|
+
minDelayMs: number;
|
|
104
|
+
/** Ceiling for any wait. Default: 15 min */
|
|
105
|
+
maxDelayMs: number;
|
|
106
|
+
/** ± ratio applied as jitter. Default: 0.2 */
|
|
107
|
+
jitterRatio: number;
|
|
108
|
+
/** Max surge pauses before giving up. Default: 5 */
|
|
109
|
+
maxAttempts: number;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export const DEFAULT_SURGE_POLICY: SurgePolicy = {
|
|
113
|
+
multiplier: 2,
|
|
114
|
+
minDelayMs: 30_000,
|
|
115
|
+
maxDelayMs: 15 * 60_000,
|
|
116
|
+
jitterRatio: 0.2,
|
|
117
|
+
maxAttempts: 5,
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Deterministic delay computation (unit-testable, no timers).
|
|
122
|
+
* `rand` in [0,1] controls jitter; omit it for zero jitter.
|
|
123
|
+
*/
|
|
124
|
+
export function computeSurgeDelayMs(
|
|
125
|
+
signal: SurgeSignal,
|
|
126
|
+
attempt: number,
|
|
127
|
+
policy: Partial<SurgePolicy> = {},
|
|
128
|
+
rand?: number,
|
|
129
|
+
): number {
|
|
130
|
+
const p = { ...DEFAULT_SURGE_POLICY, ...policy };
|
|
131
|
+
const base = p.baseDelayMs ?? signal.retryAfterMs;
|
|
132
|
+
const escalated = base * p.multiplier ** Math.max(0, attempt - 1);
|
|
133
|
+
const clamped = Math.min(Math.max(escalated, p.minDelayMs), p.maxDelayMs);
|
|
134
|
+
if (rand === undefined || p.jitterRatio === 0) return clamped;
|
|
135
|
+
const jitter = clamped * p.jitterRatio * (2 * rand - 1);
|
|
136
|
+
return Math.min(Math.max(clamped + jitter, p.minDelayMs), p.maxDelayMs);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// ─── Scheduler ──────────────────────────────────────────────────────────────
|
|
140
|
+
|
|
141
|
+
/** Attempt counter with escalation; persists across a task's lifetime. */
|
|
142
|
+
export class SurgeScheduler {
|
|
143
|
+
private attempt = 0;
|
|
144
|
+
private readonly policy: SurgePolicy;
|
|
145
|
+
|
|
146
|
+
constructor(policy: Partial<SurgePolicy> = {}) {
|
|
147
|
+
this.policy = { ...DEFAULT_SURGE_POLICY, ...policy };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Next attempt number, or null when maxAttempts is exhausted. */
|
|
151
|
+
nextAttempt(): number | null {
|
|
152
|
+
if (this.attempt >= this.policy.maxAttempts) return null;
|
|
153
|
+
this.attempt += 1;
|
|
154
|
+
return this.attempt;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
get attempts(): number {
|
|
158
|
+
return this.attempt;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
delayFor(signal: SurgeSignal, rand?: number): number {
|
|
162
|
+
return computeSurgeDelayMs(signal, this.attempt, this.policy, rand);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// ─── Retry wrapper ──────────────────────────────────────────────────────────
|
|
167
|
+
|
|
168
|
+
export interface SurgeRetryOptions {
|
|
169
|
+
policy?: Partial<SurgePolicy>;
|
|
170
|
+
/** Called before each surge pause (logging / notifications) */
|
|
171
|
+
onSurge?: (info: {
|
|
172
|
+
attempt: number;
|
|
173
|
+
delayMs: number;
|
|
174
|
+
signal: SurgeSignal;
|
|
175
|
+
}) => void;
|
|
176
|
+
/** Called once when attempts are exhausted (before rethrow) */
|
|
177
|
+
onExhausted?: (signal: SurgeSignal) => void;
|
|
178
|
+
/** Injectable sleep for tests. Default: real setTimeout */
|
|
179
|
+
sleep?: (ms: number) => Promise<void>;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const realSleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Run `fn`, retrying only on surge-classified failures with escalated,
|
|
186
|
+
* jittered pauses. Non-surge errors rethrow immediately. On exhaustion the
|
|
187
|
+
* last error is rethrown after `onExhausted`.
|
|
188
|
+
*/
|
|
189
|
+
export async function invokeWithSurgeRetry<T>(
|
|
190
|
+
fn: () => Promise<T>,
|
|
191
|
+
opts: SurgeRetryOptions = {},
|
|
192
|
+
): Promise<T> {
|
|
193
|
+
const scheduler = new SurgeScheduler(opts.policy);
|
|
194
|
+
const sleep = opts.sleep ?? realSleep;
|
|
195
|
+
|
|
196
|
+
for (;;) {
|
|
197
|
+
try {
|
|
198
|
+
return await fn();
|
|
199
|
+
} catch (err) {
|
|
200
|
+
const signal = classifySurge(err);
|
|
201
|
+
if (!signal) throw err; // not a surge — fail fast
|
|
202
|
+
|
|
203
|
+
const attempt = scheduler.nextAttempt();
|
|
204
|
+
if (attempt === null) {
|
|
205
|
+
opts.onExhausted?.(signal);
|
|
206
|
+
throw err;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const delayMs = scheduler.delayFor(
|
|
210
|
+
signal,
|
|
211
|
+
Math.random(), // jitter draw
|
|
212
|
+
);
|
|
213
|
+
opts.onSurge?.({ attempt, delayMs, signal });
|
|
214
|
+
await sleep(delayMs);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-harness-runtime",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.32",
|
|
4
4
|
"description": "[BETA] Codex-style /usage status + autonomous coding harness for pi. Not production ready — expect breaking changes.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"scripts": {
|
|
@@ -89,7 +89,13 @@
|
|
|
89
89
|
"packages/*"
|
|
90
90
|
],
|
|
91
91
|
"dependencies": {
|
|
92
|
+
"@langchain/core": "^1.2.9",
|
|
93
|
+
"@langchain/langgraph": "^1.4.13",
|
|
94
|
+
"@langchain/langgraph-checkpoint": "^1.1.5",
|
|
95
|
+
"@langchain/openai": "^1.5.10",
|
|
92
96
|
"chokidar": "^5.0.0",
|
|
93
|
-
"deepagents": "^1.10.5"
|
|
97
|
+
"deepagents": "^1.10.5",
|
|
98
|
+
"langchain": "^1.5.10",
|
|
99
|
+
"zod": "^4.5.4"
|
|
94
100
|
}
|
|
95
101
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pi-harness/capability-registry",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.32",
|
|
4
4
|
"description": "Capability Registry for pi-harness-runtime — tracks model capabilities and enables capability-based routing",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|