pi-harness-runtime 1.1.30 → 1.1.32

package/harness/langchain/surge.test.ts

@@ -0,0 +1,214 @@
+/**
+ * Surge auto-resume tests — S2/S4 acceptance (wiki/peak-hour-surge-auto-resume.md)
+ *
+ * All deterministic via injected sleep/rand — no real timers, no API keys.
+ *
+ * Run:
+ *   bun test harness/langchain/surge.test.ts
+ */
+
+import { describe, it } from "node:test";
+import assert from "node:assert";
+
+import {
+	classifySurge,
+	computeSurgeDelayMs,
+	invokeWithSurgeRetry,
+	SurgeScheduler,
+	type SurgePolicy,
+} from "./surge.js";
+import type { LoopDeps } from "./graph.js";
+
+// ─── Fixtures ────────────────────────────────────────────────────────────────
+
+/** The exact incident message from the amos-saas E2E failure (2026-08-31). */
+const INCIDENT =
+	'529 {"type":"error","error":{"type":"overloaded_error","message":"The system is currently experiencing a peak-hour surge, and the server is temporarily busy. It usually recovers within 1–5 minutes. Please try again shortly (2064)"},"request_id":"06e405e2e5965672576925264941a057"}';
+
+const SIGNAL_3MIN = {
+	retryAfterMs: 180_000,
+	explicit: true,
+	sourceText: INCIDENT,
+};
+
+const FAST_POLICY: Partial<SurgePolicy> = {
+	baseDelayMs: 100,
+	multiplier: 2,
+	minDelayMs: 10,
+	maxDelayMs: 1_000,
+	jitterRatio: 0,
+	maxAttempts: 3,
+};
+
+// ─── T1 Classification ───────────────────────────────────────────────────────
+
+describe("T1 — classifySurge", () => {
+	it("classifies the incident message (midpoint of 1–5 min = 3 min)", () => {
+		const sig = classifySurge(new Error(INCIDENT));
+		assert.ok(sig, "should classify");
+		assert.equal(sig?.retryAfterMs, 180_000);
+		assert.equal(sig?.explicit, true);
+	});
+
+	it("classifies plain 529 status", () => {
+		const sig = classifySurge(new Error("HTTP 529"));
+		assert.ok(sig);
+		assert.equal(sig?.explicit, false); // no stated delay → 2 min default
+		assert.equal(sig?.retryAfterMs, 120_000);
+	});
+
+	it("parses 'retry after 30 seconds'", () => {
+		const sig = classifySurge(
+			new Error("overloaded — please retry after 30 seconds"),
+		);
+		assert.ok(sig);
+		assert.equal(sig?.retryAfterMs, 30_000);
+	});
+
+	it("parses retry-after milliseconds", () => {
+		const sig = classifySurge(new Error("529; retry-after: 45000ms"));
+		assert.ok(sig);
+		assert.equal(sig?.retryAfterMs, 45_000);
+	});
+
+	it("rejects non-surge errors", () => {
+		assert.equal(classifySurge(new Error("429 too many requests")), null);
+		assert.equal(classifySurge(new Error("ECONNREFUSED")), null);
+		assert.equal(classifySurge("file not found"), null);
+	});
+});
+
+// ─── T2/T3 Delay computation & escalation ────────────────────────────────────
+
+describe("T2 — computeSurgeDelayMs escalation", () => {
+	it("escalates 3 → 6 → 12 min and caps at max", () => {
+		const p = { jitterRatio: 0 } as const; // deterministic
+		assert.equal(computeSurgeDelayMs(SIGNAL_3MIN, 1, p), 180_000);
+		assert.equal(computeSurgeDelayMs(SIGNAL_3MIN, 2, p), 360_000);
+		assert.equal(computeSurgeDelayMs(SIGNAL_3MIN, 3, p), 720_000);
+		assert.equal(computeSurgeDelayMs(SIGNAL_3MIN, 9, p), 900_000); // 15 min cap
+	});
+
+	it("floors at minDelayMs (never hammer)", () => {
+		const d = computeSurgeDelayMs({ ...SIGNAL_3MIN, retryAfterMs: 1_000 }, 1, {
+			minDelayMs: 30_000,
+			jitterRatio: 0,
+		});
+		assert.equal(d, 30_000);
+	});
+
+	it("jitter stays inside ±ratio and within bounds", () => {
+		const lo = computeSurgeDelayMs(SIGNAL_3MIN, 1, { jitterRatio: 0.2 }, 0);
+		const hi = computeSurgeDelayMs(SIGNAL_3MIN, 1, { jitterRatio: 0.2 }, 1);
+		assert.ok(lo >= 180_000 * 0.8 && lo < 180_000);
+		assert.ok(hi > 180_000 && hi <= 180_000 * 1.2);
+	});
+});
+
+describe("T3 — SurgeScheduler attempt cap", () => {
+	it("counts 1..maxAttempts then null", () => {
+		const s = new SurgeScheduler({ maxAttempts: 3 });
+		assert.equal(s.nextAttempt(), 1);
+		assert.equal(s.nextAttempt(), 2);
+		assert.equal(s.nextAttempt(), 3);
+		assert.equal(s.nextAttempt(), null);
+		assert.equal(s.attempts, 3);
+	});
+});
+
+// ─── T-surge-retry Wrapper behavior ─────────────────────────────────────────
+
+describe("invokeWithSurgeRetry", () => {
+	it("retries through a 529 and succeeds (the incident scenario)", async () => {
+		const sleeps: number[] = [];
+		let calls = 0;
+		const result = await invokeWithSurgeRetry(
+			async () => {
+				calls += 1;
+				if (calls < 3) throw new Error(INCIDENT);
+				return "approved";
+			},
+			{ policy: FAST_POLICY, sleep: async (ms) => void sleeps.push(ms) },
+		);
+		assert.equal(result, "approved");
+		assert.equal(calls, 3);
+		assert.deepEqual(sleeps, [100, 200]); // escalated, no jitter
+	});
+
+	it("rethrows non-surge errors immediately (no retry)", async () => {
+		const sleeps: number[] = [];
+		let calls = 0;
+		await assert.rejects(
+			invokeWithSurgeRetry(
+				async () => {
+					calls += 1;
+					throw new Error("ECONNREFUSED");
+				},
+				{ policy: FAST_POLICY, sleep: async (ms) => void sleeps.push(ms) },
+			),
+			/ECONNREFUSED/,
+		);
+		assert.equal(calls, 1);
+		assert.equal(sleeps.length, 0);
+	});
+
+	it("gives up after maxAttempts, fires onExhausted, rethrows last error", async () => {
+		const pauses: number[] = [];
+		let exhausted = false;
+		let calls = 0;
+		await assert.rejects(
+			invokeWithSurgeRetry(
+				async () => {
+					calls += 1;
+					throw new Error(INCIDENT);
+				},
+				{
+					policy: FAST_POLICY,
+					sleep: async (ms) => void pauses.push(ms),
+					onSurge: ({ attempt }) => assert.ok(attempt >= 1 && attempt <= 3),
+					onExhausted: () => {
+						exhausted = true;
+					},
+				},
+			),
+			/overloaded_error/,
+		);
+		assert.equal(calls, 4); // initial + 3 retries
+		assert.deepEqual(pauses, [100, 200, 400]);
+		assert.equal(exhausted, true);
+	});
+});
+
+// ─── T8 Daemon-level: graph deps that surge once, then succeed ───────────────
+
+describe("T8 — loop deps surviving a mid-graph 529", () => {
+	it("writeStep throws the incident once; retry wrapper completes the loop", async () => {
+		const { buildWriteReviewLoop, buildDryRunDeps } = await import("./graph.js");
+
+		const base = buildDryRunDeps({ maxIterations: 1 });
+		let writeCalls = 0;
+		const deps: LoopDeps = {
+			...base,
+			write: async (plan, review) => {
+				writeCalls += 1;
+				if (writeCalls === 1) throw new Error(INCIDENT);
+				return base.write(plan, review);
+			},
+		};
+
+		const loop = buildWriteReviewLoop(deps);
+		const sleeps: number[] = [];
+		const state = await invokeWithSurgeRetry(
+			() =>
+				loop.invoke(
+					{ request: "surge survival test" },
+					{ configurable: { thread_id: "t8-surge" } },
+				),
+			{ policy: FAST_POLICY, sleep: async (ms) => void sleeps.push(ms) },
+		);
+
+		assert.equal(writeCalls, 2); // first surged, retry succeeded
+		assert.ok(state.review, "loop produced a review");
+		assert.deepEqual(sleeps, [100]);
+	});
+});

package/harness/langchain/surge.ts

@@ -0,0 +1,217 @@
+/**
+ * Surge auto-resume — 529 / overloaded_error detection, escalation backoff,
+ * and a retry wrapper for the daemon's graph invocations.
+ *
+ * Implements S2 + S4 of wiki/peak-hour-surge-auto-resume.md:
+ * seconds-scale blits stay in the provider client; minutes-scale surges are
+ * handled HERE with scheduled pauses (3 → 6 → 12 min, jittered, capped).
+ *
+ * Verdict vocabulary and patterns intentionally match:
+ *   - harness/loop-runtime.ts 529 path (commit 9ae0ffd)
+ *   - harness/e2e/glm-quota-scraper.ts parseMinimaxOverloadResetTime()
+ *   - packages/providers/src/adapters.ts isOverloaded
+ *
+ * Wiki: wiki/peak-hour-surge-auto-resume.md
+ */
+
+// ─── Classification ─────────────────────────────────────────────────────────
+
+export interface SurgeSignal {
+	/** Parsed recovery delay in ms (midpoint of the stated range, or fallback) */
+	retryAfterMs: number;
+	/** True when the delay came from an explicit provider statement */
+	explicit: boolean;
+	/** Raw error text for logs */
+	sourceText: string;
+}
+
+const OVERLOADED_PATTERNS: RegExp[] = [
+	/529/,
+	/overloaded_error/i,
+	/peak.?hour.*surge/i,
+	/(?:overload|surge|busy)/i,
+];
+
+const RECOVERY_RANGE = /recovers? within (\d+)\s*[\u2013-]\s*(\d+)\s*minutes/i;
+const SIMPLE_DELAY = /retry (?:after|in) (\d+)\s*(seconds?|secs?|s)\b/i;
+const RETRY_AFTER_MS = /retry.?after[^0-9]{0,12}(\d{4,})\s*ms/i;
+
+/**
+ * Classify an unknown error as a transient provider surge.
+ * Returns null for anything that is not a 529/overload-class failure.
+ */
+export function classifySurge(err: unknown): SurgeSignal | null {
+	const sourceText =
+		err instanceof Error
+			? err.message
+			: typeof err === "string"
+				? err
+				: String(err);
+
+	if (!OVERLOADED_PATTERNS.some((p) => p.test(sourceText))) {
+		return null;
+	}
+
+	// "recovers within 1–5 minutes" → midpoint (3 min)
+	const range = sourceText.match(RECOVERY_RANGE);
+	if (range) {
+		const min = Number.parseInt(range[1] as string, 10);
+		const max = Number.parseInt(range[2] as string, 10);
+		if (Number.isFinite(min) && Number.isFinite(max) && max >= min) {
+			return {
+				retryAfterMs: Math.round(((min + max) / 2) * 60_000),
+				explicit: true,
+				sourceText,
+			};
+		}
+	}
+
+	// "retry after 30 seconds"
+	const simple = sourceText.match(SIMPLE_DELAY);
+	if (simple) {
+		const secs = Number.parseInt(simple[1] as string, 10);
+		if (Number.isFinite(secs) && secs > 0) {
+			return {
+				retryAfterMs: secs * 1000,
+				explicit: true,
+				sourceText,
+			};
+		}
+	}
+
+	// "retry-after 120000ms" (header-style)
+	const ms = sourceText.match(RETRY_AFTER_MS);
+	if (ms) {
+		const val = Number.parseInt(ms[1] as string, 10);
+		if (Number.isFinite(val) && val > 0) {
+			return { retryAfterMs: val, explicit: true, sourceText };
+		}
+	}
+
+	// Overload-class error without a stated delay → conservative default
+	return { retryAfterMs: 120_000, explicit: false, sourceText };
+}
+
+// ─── Policy ─────────────────────────────────────────────────────────────────
+
+export interface SurgePolicy {
+	/** Override the signal's parsed delay (ms). Default: use the signal. */
+	baseDelayMs?: number;
+	/** Delay multiplier per attempt. Default: 2 (3 → 6 → 12 min) */
+	multiplier: number;
+	/** Floor for any wait. Default: 30_000 (never hammer a surging provider) */
+	minDelayMs: number;
+	/** Ceiling for any wait. Default: 15 min */
+	maxDelayMs: number;
+	/** ± ratio applied as jitter. Default: 0.2 */
+	jitterRatio: number;
+	/** Max surge pauses before giving up. Default: 5 */
+	maxAttempts: number;
+}
+
+export const DEFAULT_SURGE_POLICY: SurgePolicy = {
+	multiplier: 2,
+	minDelayMs: 30_000,
+	maxDelayMs: 15 * 60_000,
+	jitterRatio: 0.2,
+	maxAttempts: 5,
+};
+
+/**
+ * Deterministic delay computation (unit-testable, no timers).
+ * `rand` in [0,1] controls jitter; omit it for zero jitter.
+ */
+export function computeSurgeDelayMs(
+	signal: SurgeSignal,
+	attempt: number,
+	policy: Partial<SurgePolicy> = {},
+	rand?: number,
+): number {
+	const p = { ...DEFAULT_SURGE_POLICY, ...policy };
+	const base = p.baseDelayMs ?? signal.retryAfterMs;
+	const escalated = base * p.multiplier ** Math.max(0, attempt - 1);
+	const clamped = Math.min(Math.max(escalated, p.minDelayMs), p.maxDelayMs);
+	if (rand === undefined || p.jitterRatio === 0) return clamped;
+	const jitter = clamped * p.jitterRatio * (2 * rand - 1);
+	return Math.min(Math.max(clamped + jitter, p.minDelayMs), p.maxDelayMs);
+}
+
+// ─── Scheduler ──────────────────────────────────────────────────────────────
+
+/** Attempt counter with escalation; persists across a task's lifetime. */
+export class SurgeScheduler {
+	private attempt = 0;
+	private readonly policy: SurgePolicy;
+
+	constructor(policy: Partial<SurgePolicy> = {}) {
+		this.policy = { ...DEFAULT_SURGE_POLICY, ...policy };
+	}
+
+	/** Next attempt number, or null when maxAttempts is exhausted. */
+	nextAttempt(): number | null {
+		if (this.attempt >= this.policy.maxAttempts) return null;
+		this.attempt += 1;
+		return this.attempt;
+	}
+
+	get attempts(): number {
+		return this.attempt;
+	}
+
+	delayFor(signal: SurgeSignal, rand?: number): number {
+		return computeSurgeDelayMs(signal, this.attempt, this.policy, rand);
+	}
+}
+
+// ─── Retry wrapper ──────────────────────────────────────────────────────────
+
+export interface SurgeRetryOptions {
+	policy?: Partial<SurgePolicy>;
+	/** Called before each surge pause (logging / notifications) */
+	onSurge?: (info: {
+		attempt: number;
+		delayMs: number;
+		signal: SurgeSignal;
+	}) => void;
+	/** Called once when attempts are exhausted (before rethrow) */
+	onExhausted?: (signal: SurgeSignal) => void;
+	/** Injectable sleep for tests. Default: real setTimeout */
+	sleep?: (ms: number) => Promise<void>;
+}
+
+const realSleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+/**
+ * Run `fn`, retrying only on surge-classified failures with escalated,
+ * jittered pauses. Non-surge errors rethrow immediately. On exhaustion the
+ * last error is rethrown after `onExhausted`.
+ */
+export async function invokeWithSurgeRetry<T>(
+	fn: () => Promise<T>,
+	opts: SurgeRetryOptions = {},
+): Promise<T> {
+	const scheduler = new SurgeScheduler(opts.policy);
+	const sleep = opts.sleep ?? realSleep;
+
+	for (;;) {
+		try {
+			return await fn();
+		} catch (err) {
+			const signal = classifySurge(err);
+			if (!signal) throw err; // not a surge — fail fast
+
+			const attempt = scheduler.nextAttempt();
+			if (attempt === null) {
+				opts.onExhausted?.(signal);
+				throw err;
+			}
+
+			const delayMs = scheduler.delayFor(
+				signal,
+				Math.random(), // jitter draw
+			);
+			opts.onSurge?.({ attempt, delayMs, signal });
+			await sleep(delayMs);
+		}
+	}
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-harness-runtime",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "[BETA] Codex-style /usage status + autonomous coding harness for pi. Not production ready — expect breaking changes.",
   "type": "module",
   "scripts": {
@@ -89,7 +89,13 @@
     "packages/*"
   ],
   "dependencies": {
+    "@langchain/core": "^1.2.9",
+    "@langchain/langgraph": "^1.4.13",
+    "@langchain/langgraph-checkpoint": "^1.1.5",
+    "@langchain/openai": "^1.5.10",
     "chokidar": "^5.0.0",
-    "deepagents": "^1.10.5"
+    "deepagents": "^1.10.5",
+    "langchain": "^1.5.10",
+    "zod": "^4.5.4"
   }
 }

package/packages/a2a-adapter/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/a2a-adapter",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "type": "module",
   "description": "Google Agent-to-Agent (A2A) protocol adapter for pi-harness",
   "main": "dist/index.js",

package/packages/architecture-generator/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/architecture-generator",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Generate ADR and Mermaid diagrams from project analysis",
   "type": "module",
   "main": "./dist/index.js",

package/packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/auth",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "pi-harness-runtime — auth",
   "type": "module",
   "main": "./dist/index.js",

package/packages/autonomous-refactor/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/autonomous-refactor",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "type": "module",
   "description": "Autonomous refactoring engine",
   "main": "dist/index.js",

package/packages/autonomous-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/autonomous-runtime",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Autonomous Operations Runtime — task inbox, lease management, and worker lifecycle (RFC-0101)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/cache-strategy/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/cache-strategy",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "type": "module",
   "description": "Smart caching with TTL and invalidation strategies (RFC-0022)",
   "main": "dist/index.js",

package/packages/capability-registry/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/capability-registry",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Capability Registry for pi-harness-runtime — tracks model capabilities and enables capability-based routing",
   "type": "module",
   "main": "./dist/index.js",

package/packages/checkpoint/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/checkpoint",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Enhanced checkpoint system with incremental snapshots, diff-based storage, and recovery strategies",
   "type": "module",
   "main": "./dist/index.js",

package/packages/cli-plugin-sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/cli-plugin-sdk",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "type": "module",
   "description": "CLI Plugin SDK for pi-harness — install, list, remove, update, invoke plugins",
   "main": "dist/index.js",

package/packages/clipboard/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/clipboard",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/clipboard-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/clipboard-plugin",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/code-generation/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/code-generation",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Code generation pipeline for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/code-review/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/code-review",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Automated code review engine for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/codex-adapter/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/codex-adapter",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "type": "module",
   "description": "OpenAI Codex provider adapter for pi-harness",
   "main": "dist/index.js",

package/packages/config-capture/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moocoding/config-capture",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Auto-capture API-fetched configuration values to AGENTS.md",
   "type": "module",
   "main": "src/index.ts",

package/packages/context-compiler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/context-compiler",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Bounded context selection engine for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/context-discovery/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/context-discovery",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/context-manager/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/context-manager",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Context Window Manager — prevents model context overflow via compaction and resume prompts (RFC-0010)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/cookie-sanitizer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/cookie-sanitizer",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "pi-harness-runtime — cookie sanitizer (provider-quota cookie file normaliser)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/cost-optimizer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/cost-optimizer",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Cost Optimizer for pi-harness-runtime — budget tracking, forecasting, and model selection optimization",
   "type": "module",
   "main": "./dist/index.js",

package/packages/dependency-analyzer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/dependency-analyzer",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/django-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/django-plugin",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Django framework analysis — models, management commands, settings",
   "type": "module",
   "main": "./dist/index.js",

package/packages/doc-generator/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/doc-generator",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Auto-Doc Generator — generate documentation from source code (RFC-0014)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/evaluation-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/evaluation-engine",
-  "version": "1.1.30",
+  "version": "1.1.32",
   "description": "Evaluation engine for pi-harness runtime (RFC-0057)",
   "type": "module",
   "main": "./dist/index.js",