pi-harness-runtime 1.1.49 → 1.1.71-3

package/harness/langchain/surge.test.ts

@@ -212,3 +212,182 @@ describe("T8 — loop deps surviving a mid-graph 529", () => {
 		assert.deepEqual(sleeps, [100]);
 	});
 });
+
+// ─── GLM Quota Tests ─────────────────────────────────────────────────────────
+
+import { classifyGLMQuota, invokeWithGLMRetry } from "./surge.js";
+
+describe("classifyGLMQuota", () => {
+	// The dates in the error messages are in the past (2026-09-04).
+	// classifyGLMQuota shifts them to the NEXT occurrence (+1 day).
+	// The tests verify: shift happened + time preserved verbatim.
+	it("parses GLM 1308 error with reset timestamp", () => {
+		const err = new Error(
+			'429: {"code":"1308","message":"Usage limit reached for 5 hour. Your limit will reset at 2026-09-04 20:29:24"}',
+		);
+		const sig = classifyGLMQuota(err);
+		assert.ok(sig, "should be classified as GLM quota");
+		// 2026-09-04 is past → shifted to 2026-09-05; epoch must be in the future
+		assert.ok(sig!.resetAtEpoch > Date.now(), "resetAtEpoch is in the future");
+		assert.ok(
+			sig!.resetAt.includes("2026-09-05"),
+			"date shifted to next occurrence",
+		);
+		assert.ok(sig!.resetAt.includes("20:29:24"), "time preserved verbatim");
+	});
+
+	it("parses 1308 from plain string error", () => {
+		const err =
+			'{"code":"1308","message":"Usage limit reached. Your limit will reset at 2026-09-04 20:29:24"}';
+		const sig = classifyGLMQuota(err);
+		assert.ok(sig, "should be classified as GLM quota");
+		// space separator → normalised to T; past date → shifted to next occurrence
+		assert.ok(
+			sig!.resetAt.includes("2026-09-05"),
+			"space normalised and date shifted",
+		);
+	});
+
+	it("parses reset time with T separator", () => {
+		const err = new Error(
+			'{"code":"1308","message":"Usage limit reached. Your limit will reset at 2026-09-04T20:29:24"}',
+		);
+		const sig = classifyGLMQuota(err);
+		assert.ok(sig, "should be classified as GLM quota");
+		// past date → next occurrence (2026-09-05); T separator preserved verbatim
+		assert.ok(
+			sig!.resetAt.includes("2026-09-05"),
+			"date shifted to next occurrence",
+		);
+		assert.ok(sig!.resetAt.includes("T"), "T separator preserved verbatim");
+		assert.ok(sig!.resetAt.includes("20:29:24"), "time preserved verbatim");
+	});
+
+	it("returns null for non-GLM errors", () => {
+		const err = new Error("529: overloaded error");
+		assert.equal(classifyGLMQuota(err), null);
+	});
+
+	it("returns null for plain errors without 1308", () => {
+		const err = new Error("Something went wrong");
+		assert.equal(classifyGLMQuota(err), null);
+	});
+
+	it("1308 without reset time uses 5-min fallback", () => {
+		const err = new Error('{"code":"1308","message":"Usage limit reached"}');
+		const sig = classifyGLMQuota(err);
+		assert.ok(sig, "should be classified");
+		const diffMs = sig!.resetAtEpoch - Date.now();
+		// Should be approximately 5 minutes
+		assert.ok(diffMs >= 4 * 60_000 && diffMs <= 6 * 60_000, "5-min fallback");
+	});
+});
+
+describe("invokeWithGLMRetry", () => {
+	it("succeeds immediately when fn succeeds", async () => {
+		const result = await invokeWithGLMRetry(() => Promise.resolve("ok"));
+		assert.equal(result, "ok");
+	});
+
+	it("throws non-GLM errors immediately", async () => {
+		await assert.rejects(
+			() =>
+				invokeWithGLMRetry(() => {
+					throw new Error("not a GLM error");
+				}),
+			/not a GLM error/,
+		);
+	});
+
+	it("waits until reset time before retry", async () => {
+		const sleeps: number[] = [];
+		const sleep = async (ms: number) => void sleeps.push(ms);
+		// Use a reset time 100ms in the future
+		const resetIn = 100;
+		const resetAt = new Date(Date.now() + resetIn).toISOString();
+
+		let attempts = 0;
+		const result = await invokeWithGLMRetry(
+			async () => {
+				attempts++;
+				if (attempts === 1) {
+					throw new Error(
+						`{"code":"1308","message":"Usage limit. Reset at ${resetAt}"}`,
+					);
+				}
+				return "success after retry";
+			},
+			{ sleep, tickMs: 10 },
+		);
+
+		assert.equal(result, "success after retry");
+		assert.equal(attempts, 2, "retried after wait");
+		assert.ok(sleeps.length >= 1, "slept at least once during countdown");
+		assert.ok(
+			sleeps.every((ms) => ms === 10),
+			"tick interval used",
+		);
+	});
+
+	it("fires onGLMQuota callback during countdown", async () => {
+		const ticks: string[] = [];
+		const sleep = async (ms: number) => void ticks.push(`sleep:${ms}`);
+		const resetAt = new Date(Date.now() + 50).toISOString();
+
+		let attempts = 0;
+		await invokeWithGLMRetry(
+			async () => {
+				attempts++;
+				if (attempts === 1) {
+					throw new Error(
+						`{"code":"1308","message":"Usage limit. Reset at ${resetAt}"}`,
+					);
+				}
+				return "done";
+			},
+			{
+				sleep,
+				tickMs: 10,
+				onGLMQuota: (sig) => ticks.push(`quota:${sig.resetAtEpoch}`),
+			},
+		);
+
+		assert.ok(
+			ticks.some((t) => t.startsWith("quota:")),
+			"onGLMQuota was called at least once",
+		);
+	});
+
+	it("fires onGLMRetry callback after countdown expires", async () => {
+		const events: string[] = [];
+		const sleep = async (_ms: number) => void null;
+		const resetAt = new Date(Date.now() + 20).toISOString();
+
+		let attempts = 0;
+		await invokeWithGLMRetry(
+			async () => {
+				attempts++;
+				if (attempts === 1) {
+					throw new Error(
+						`{"code":"1308","message":"Usage limit. Reset at ${resetAt}"}`,
+					);
+				}
+				return "done";
+			},
+			{
+				sleep,
+				tickMs: 10,
+				onGLMQuota: () => events.push("quota"),
+				onGLMRetry: () => events.push("retry"),
+			},
+		);
+
+		assert.ok(events.includes("quota"), "quota fired");
+		assert.ok(events.includes("retry"), "retry fired");
+		// retry fires AFTER quota
+		assert.ok(
+			events.indexOf("retry") > events.indexOf("quota"),
+			"retry after quota",
+		);
+	});
+});

package/harness/langchain/surge.ts

@@ -1,15 +1,10 @@
 /**
- * Surge auto-resume — 529 / overloaded_error detection, escalation backoff,
- * and a retry wrapper for the daemon's graph invocations.
+ * Surge auto-resume — 529 / overloaded_error (MiniMax) + 1308 (GLM quota) detection,
+ * escalation backoff, and retry wrappers for the daemon's graph invocations.
  *
- * Implements S2 + S4 of wiki/peak-hour-surge-auto-resume.md:
- * seconds-scale blits stay in the provider client; minutes-scale surges are
- * handled HERE with scheduled pauses (3 → 6 → 12 min, jittered, capped).
- *
- * Verdict vocabulary and patterns intentionally match:
- *   - harness/loop-runtime.ts 529 path (commit 9ae0ffd)
- *   - harness/e2e/glm-quota-scraper.ts parseMinimaxOverloadResetTime()
- *   - packages/providers/src/adapters.ts isOverloaded
+ * Two wrappers:
+ *   invokeWithSurgeRetry     — MiniMax 529/overload; exponential backoff
+ *   invokeWithGLMRetry       — GLM 1308 quota; waits until reset time
  *
  * Wiki: wiki/peak-hour-surge-auto-resume.md
  */
@@ -37,7 +32,7 @@ const SIMPLE_DELAY = /retry (?:after|in) (\d+)\s*(seconds?|secs?|s)\b/i;
 const RETRY_AFTER_MS = /retry.?after[^0-9]{0,12}(\d{4,})\s*ms/i;
 
 /**
- * Classify an unknown error as a transient provider surge.
+ * Classify an unknown error as a transient provider surge (MiniMax).
  * Returns null for anything that is not a 529/overload-class failure.
  */
 export function classifySurge(err: unknown): SurgeSignal | null {
@@ -92,6 +87,87 @@ export function classifySurge(err: unknown): SurgeSignal | null {
 	return { retryAfterMs: 120_000, explicit: false, sourceText };
 }
 
+// ─── GLM Quota Classification ─────────────────────────────────────────────────
+
+/** Matches the reset-at timestamp from a GLM 1308 error. */
+const GLM_RESET_AT = /reset at (\d{4}-\d{2}-\d{2}[T ]?\d{2}:\d{2}:\d{2})/i;
+
+/**
+ * Add 1 calendar day to an ISO date string "YYYY-MM-DDTHH:MM:SS".
+ * Used to shift the reset date when it has already passed.
+ */
+function _shiftDateByOneDay(isoDateTime: string): string {
+	const d = new Date(isoDateTime + "Z");
+	d.setUTCDate(d.getUTCDate() + 1);
+	// Re-format: YYYY-MM-DDTHH:MM:SS  (always 19 chars, seconds included)
+	const yyyy = d.getUTCFullYear();
+	const mm = String(d.getUTCMonth() + 1).padStart(2, "0");
+	const dd = String(d.getUTCDate()).padStart(2, "0");
+	// Preserve the time part from the original string
+	const timePart = isoDateTime.slice(11); // "HH:MM:SS"
+	return `${yyyy}-${mm}-${dd}T${timePart}`;
+}
+
+/**
+ * Classify a GLM 1308 quota exhaustion error.
+ * Returns the reset-at ISO timestamp, or null if not a GLM quota error.
+ */
+export function classifyGLMQuota(err: unknown): GLMQuotaSignal | null {
+	const sourceText =
+		err instanceof Error
+			? err.message
+			: typeof err === "string"
+				? err
+				: String(err);
+
+	// Detect GLM 1308 — quota exhaustion
+	const has1308 = /"code"\s*:\s*"1308"|code.*1308/i.test(sourceText);
+	const hasReset = GLM_RESET_AT.test(sourceText);
+	if (!has1308 && !hasReset) return null;
+
+	// Extract reset timestamp
+	const match = sourceText.match(GLM_RESET_AT);
+	if (!match) {
+		// 1308 without a reset time — use a conservative default (5 min)
+		return {
+			resetAt: new Date(Date.now() + 5 * 60_000).toISOString(),
+			resetAtEpoch: Date.now() + 5 * 60_000,
+			sourceText,
+		};
+	}
+
+	// Normalise: space → T, then ensure HH:MM:SS (append :00 if only HH:MM)
+	const normalised = match[1].replace(" ", "T");
+	// "2026-09-04T20:29:24" (19 chars, with seconds) → use as-is
+	// "2026-09-04T20:29"    (16 chars, no seconds)   → append :00
+	const resetAt = normalised.length === 16 ? `${normalised}:00` : normalised;
+	// Parse as UTC so the date string is preserved exactly (no TZ shift).
+	const resetAtEpoch = new Date(resetAt + "Z").getTime();
+	const now = Date.now();
+
+	// If the parsed time is in the past, the date is already "next occurrence".
+	// Preserve the original resetAt string so callers can include it verbatim.
+	const adjustedResetAt =
+		resetAtEpoch <= now ? _shiftDateByOneDay(resetAt) : resetAt;
+	const adjustedEpoch =
+		resetAtEpoch <= now ? resetAtEpoch + 24 * 60 * 60 * 1000 : resetAtEpoch;
+
+	return {
+		resetAt: adjustedResetAt,
+		resetAtEpoch: adjustedEpoch,
+		sourceText,
+	};
+}
+
+export interface GLMQuotaSignal {
+	/** ISO timestamp when the quota resets */
+	resetAt: string;
+	/** Epoch ms when the quota resets */
+	resetAtEpoch: number;
+	/** Raw error text for logs */
+	sourceText: string;
+}
+
 // ─── Policy ─────────────────────────────────────────────────────────────────
 
 export interface SurgePolicy {
@@ -163,18 +239,29 @@ export class SurgeScheduler {
 	}
 }
 
-// ─── Retry wrapper ──────────────────────────────────────────────────────────
+// ─── Retry wrappers ─────────────────────────────────────────────────────────
 
 export interface SurgeRetryOptions {
 	policy?: Partial<SurgePolicy>;
+	/**
+	 * Pre-existing scheduler to use. When provided, the attempt counter persists
+	 * across calls — enabling true exponential escalation across multiple surges
+	 * within the same task (e.g. surge on 2nd loop.invoke, surge on 3rd loop.invoke).
+	 * If omitted, a fresh scheduler is created (attempt counter resets per call).
+	 */
+	scheduler?: SurgeScheduler;
 	/** Called before each surge pause (logging / notifications) */
 	onSurge?: (info: {
 		attempt: number;
 		delayMs: number;
 		signal: SurgeSignal;
 	}) => void;
-	/** Called once when attempts are exhausted (before rethrow) */
-	onExhausted?: (signal: SurgeSignal) => void;
+	/** Called once when attempts are exhausted (no more retries in this scheduler).
+	 * Can be async to allow sleeping. Return true to retry, false/undefined to throw.
+	 */
+	onExhausted?: (
+		signal: SurgeSignal,
+	) => boolean | void | Promise<boolean | void>;
 	/** Injectable sleep for tests. Default: real setTimeout */
 	sleep?: (ms: number) => Promise<void>;
 }
@@ -182,15 +269,16 @@ export interface SurgeRetryOptions {
 const realSleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
 
 /**
- * Run `fn`, retrying only on surge-classified failures with escalated,
- * jittered pauses. Non-surge errors rethrow immediately. On exhaustion the
- * last error is rethrown after `onExhausted`.
+ * Run `fn`, retrying only on surge-classified failures (MiniMax 529/overload)
+ * with exponential backoff. Non-surge errors rethrow immediately.
  */
 export async function invokeWithSurgeRetry<T>(
 	fn: () => Promise<T>,
 	opts: SurgeRetryOptions = {},
 ): Promise<T> {
-	const scheduler = new SurgeScheduler(opts.policy);
+	// Reuse provided scheduler to persist attempt counter across calls.
+	// A fresh scheduler is created only when none is provided.
+	const scheduler = opts.scheduler ?? new SurgeScheduler(opts.policy);
 	const sleep = opts.sleep ?? realSleep;
 
 	for (;;) {
@@ -202,7 +290,12 @@ export async function invokeWithSurgeRetry<T>(
 
 			const attempt = scheduler.nextAttempt();
 			if (attempt === null) {
-				opts.onExhausted?.(signal);
+				// Scheduler exhausted — call onExhausted. If it returns true, do NOT throw
+				// (caller handled it, e.g. slept and will retry). If false/undefined, throw.
+				const handled = await opts.onExhausted?.(signal);
+				if (handled === true) {
+					continue; // retry
+				}
 				throw err;
 			}
 
@@ -215,3 +308,79 @@ export async function invokeWithSurgeRetry<T>(
 		}
 	}
 }
+
+// ─── GLM Quota Retry ────────────────────────────────────────────────────────
+
+export interface GLMRetryOptions {
+	/** Called when GLM quota exhaustion is detected (for status-line countdown) */
+	onGLMQuota?: (signal: GLMQuotaSignal) => void;
+	/** Called after the countdown expires and the loop retries */
+	onGLMRetry?: (signal: GLMQuotaSignal) => void;
+	/** Called when the GLM quota error is non-retryable (e.g., no reset time) */
+	onGLMExhausted?: (signal: GLMQuotaSignal) => void;
+	/** Injectable sleep for tests. Default: real setTimeout */
+	sleep?: (ms: number) => Promise<void>;
+	/** How often to tick the countdown callback (default: 30s) */
+	tickMs?: number;
+}
+
+/**
+ * Run `fn`, retrying on GLM 1308 quota errors.
+ * After the first 1308, waits until the reset time before retrying.
+ * Subsequent 1308s within the same wait window are silently ignored.
+ * Non-GLM-quota errors rethrow immediately.
+ */
+export async function invokeWithGLMRetry<T>(
+	fn: () => Promise<T>,
+	opts: GLMRetryOptions = {},
+): Promise<T> {
+	const sleep = opts.sleep ?? realSleep;
+	const tickMs = opts.tickMs ?? 30_000;
+
+	let waitingForReset = false;
+
+	while (true) {
+		try {
+			return await fn();
+		} catch (err) {
+			const signal = classifyGLMQuota(err);
+
+			if (!signal) throw err; // not a GLM quota error — fail fast
+
+			// If we are already waiting for a reset and get another 1308,
+			// ignore it — the existing wait is already in progress.
+			if (waitingForReset) {
+				// Just log and continue waiting (the sleep loop handles the timeout)
+				await sleep(tickMs);
+				continue;
+			}
+
+			const delayMs = Math.max(0, signal.resetAtEpoch - Date.now());
+
+			opts.onGLMQuota?.(signal);
+			waitingForReset = true;
+
+			if (delayMs <= 0) {
+				// Reset time already passed — retry immediately
+				waitingForReset = false;
+				opts.onGLMRetry?.(signal);
+				continue;
+			}
+
+			// Wait in chunks so onGLMQuota fires periodically (drives countdown display)
+			const startMs = Date.now();
+			while (Date.now() - startMs < delayMs) {
+				await sleep(tickMs);
+				// Re-fire the callback to refresh the countdown display
+				opts.onGLMQuota?.({
+					...signal,
+					resetAtEpoch: signal.resetAtEpoch, // unchanged
+				});
+			}
+
+			// Countdown expired — retry
+			waitingForReset = false;
+			opts.onGLMRetry?.(signal);
+		}
+	}
+}

package/harness/langchain/widget.d.ts

@@ -0,0 +1,105 @@
+/**
+ * LoopWidget — harness progress display for the pi host status line.
+ *
+ * Architecture mirrors pi-lens's `clients/widget-state.ts` + `renderWidget()`:
+ * - State store: per-file step counts, iteration, phase
+ * - Render function: builds the `!NW ●ME` style summary line
+ * - Pi host integration: wired via ExtensionUIContext.setWidget() when available
+ *
+ * The widget is designed as a dependency injected into LoopDaemon/LoopRuntime,
+ * so it works in two modes:
+ * 1. Standalone (console): logs human-readable progress to stderr
+ * 2. Pi host: calls ui.setWidget("harness-loop", renderFn) for TUI display
+ *
+ * Wiki: wiki/multi-agent-langchain.md
+ */
+import type { TUI } from "@earendil-works/pi-tui";
+import type { Theme } from "@earendil-works/pi-coding-agent";
+/** Valid theme color keys passed to `theme.fg()`.*/
+type ColorKey = "dim" | "error" | "warning" | "success" | "accent";
+export type LoopPhase = "idle" | "planning" | "writing" | "reviewing" | "fixing" | "approving" | "blocked" | "surge-pause" | "complete" | "error";
+export interface LoopWidgetOptions {
+    /** Called with the rendered lines each time the widget updates. */
+    onRender?: (lines: string[]) => void;
+}
+export declare class LoopWidget {
+    private files;
+    private phase;
+    private iteration;
+    private maxIterations;
+    private stepLabel;
+    private errors;
+    private warnings;
+    private readonly onRender?;
+    constructor(options?: LoopWidgetOptions);
+    /** Called when the loop starts a new iteration. */
+    startIteration(iter: number, max: number): void;
+    /** Called when the loop transitions to a new phase. */
+    setPhase(phase: LoopPhase, label?: string): void;
+    /** Called when the coder writes to a file. */
+    recordWrite(filePath: string): void;
+    /** Called when the reviewer reports blockers for a file. */
+    recordBlockers(filePath: string, count: number): void;
+    /** Called when the reviewer passes a file (no blockers). */
+    recordReviewPass(filePath: string): void;
+    /** Called on surge-pause / quota-wait events. */
+    setSurgePause(resetAt?: Date): void;
+    /** Called on unrecoverable error. */
+    setError(message: string): void;
+    /** Called when the loop completes with a verdict. */
+    setComplete(verdict: "approved" | "rejected" | "max_iterations"): void;
+    /** Reset to idle state. */
+    reset(): void;
+    /**
+     * Build widget lines for the pi host's `setWidget()` renderer.
+     *
+     * Returns 0-3 lines:
+     * Line 1: `harness  [iter 2/5]  ✍ coder  !2W`   — header + summary
+     * Line 2: `  surge pause — resets 19:17`           — surge detail (when applicable)
+     * Line 3: `  ●surge.ts  ✍2  🔍3  !1`             — file rows (up to 5)
+     */
+    renderWidget(width: number, theme: {
+        fg: (color: ColorKey, s: string) => string;
+    }): string[];
+    /** Console fallback (standalone mode). */
+    toConsoleString(): string;
+    /**
+     * Returns a pi-tui Component factory for use with `ui.setWidget("harness-loop", factory)`.
+     * The caller (pi host integration) owns the `setWidget` call — this factory is just
+     * the render function.
+     *
+     * Usage in pi host:
+     * ```
+     * setWidget("harness-loop", makeWidgetRenderer(widget, theme), { placement: "belowEditor" });
+     * ```
+     */
+    /**
+     * Returns a factory suitable for `ui.setWidget("harness-loop", factory)` in the
+     * pi host. The factory's `render(width)` is called by the pi-tui render loop
+     * on every frame; `invalidate` is called by the harness to trigger a re-render.
+     */
+    makeRenderer(): {
+        factory: (tui: TUI, theme: Theme) => {
+            render: (width: number) => string[];
+            invalidate: () => void;
+            dispose: () => void;
+        };
+        invalidate: () => void;
+    };
+    private getOrCreate;
+    private fileRecords;
+    private fileBasename;
+    private classifyFileTier;
+    private phaseLabel;
+    private phaseIcon;
+    private formatResetTime;
+    /**
+     * Truncate `line` to `width` visible columns.
+     * Uses `visibleWidth` (not raw char count) so ANSI codes don't cause
+     * false wrapping — matching how the terminal measures columns.
+     */
+    private fitLine;
+    private render;
+}
+export declare function getLoopWidget(): LoopWidget;
+//# sourceMappingURL=widget.d.ts.map

package/harness/langchain/widget.test.d.ts

@@ -0,0 +1,8 @@
+/**
+ * LoopWidget tests — T14
+ *
+ * Run:
+ *   bun test harness/langchain/widget.test.ts
+ */
+export {};
+//# sourceMappingURL=widget.test.d.ts.map