pi-freeflow 1.40.0 → 1.40.1

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.40.1
+
+### Patch Changes
+
+- Long streams are no longer cut at the five-minute ceiling mid-turn; a stalled relay is cut and re-fired instead.
+  
+  Relay attempts carried a single deadline for the whole request. When a heavy reasoning turn ran past five minutes — big-pickle and long deepseek sessions routinely do — the deadline fired on live bytes, the proxy saw the dead stream as an upstream truncation, and silently re-fired the turn on another relay: on the screen the answer suddenly stopped and then continued as a fresh full generation, with the earlier partial content above it. The deadline now bounds only the header wait and is released the moment headers arrive, so a streaming body lives as long as bytes keep flowing — exactly like the other proxy paths that already cleared their timer on headers.
+  
+  A body that stops flowing entirely is cut by a per-turn idle watchdog (120s of byte silence on a live socket is a wedged tunnel, not a thinking pause) and follows the existing one-refire truncation path, so a hung relay leg still hands the turn to a sibling instead of stalling until the host gives up.
+
 ## 1.40.0
 
 ### Minor Changes

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pi-freeflow",
   "type": "module",
-  "version": "1.40.0",
+  "version": "1.40.1",
   "description": "Thin provider for OMP/Pi — model list + dumb relay proxy + log; host pi-ai owns thinking/normalization",
   "main": "extensions/index.ts",
   "types": "src/index.ts",

package/src/relay.ts

@@ -31,8 +31,8 @@ const ROLL_NOTIFY_MS = 5 * 60 * 1_000;
  * per attempt. Relay leg only - the direct-to-upstream
  * fallback keeps the built-in global dispatcher. headersTimeout
  * reaps wedged tunnels, bodyTimeout 0 disables per-chunk kill
- * (the 300s total backstop still bounds a turn), connections 50
- * caps burst spray.
+ * (bodies stream unbounded — a wedged leg is cut by stream-pipe's
+ * body-stall watchdog instead), connections 50 caps burst spray.
  *
  * Compat: an npm undici Agent is rejected as a `dispatcher`
  * by an older-major global fetch bundled in Node
@@ -75,6 +75,18 @@ export function _setHedgeGraceForTest(ms: number): () => void {
 }
 /** Test-only: reset roll-notify throttle */
 export function _resetRollNotifyForTest(): void { lastRollNotify = 0; }
+/********************************************************
+ * Relay attempt header-wait deadline (config default).
+ * Bounded only to the header wait; the streaming body must never be
+ * killed by a fixed wall-clock ceiling (see buildAttempt).
+ ********************************************************/
+let attemptHeaderTimeoutMs = UPSTREAM_HEADER_TIMEOUT_MS;
+/** Test-only: override the relay attempt header deadline; returns a restore function. */
+export function _setAttemptHeaderTimeoutForTest(ms: number): () => void {
+	const prev = attemptHeaderTimeoutMs;
+	attemptHeaderTimeoutMs = ms;
+	return () => { attemptHeaderTimeoutMs = prev; };
+}
 
 /**
  * Determine if an HTTP status code indicates a temporary relay or upstream error
@@ -263,12 +275,28 @@ export async function relayFetch(
 				// window, combined with the caller's signal so either can fire.
 				// A hung relay then trips only its own timeout (roll) instead of
 				// vetoing the pool; a genuine client cancel (caller signal
-				// aborted) still propagates to both hedged attempts.
-				const attemptTimeout = AbortSignal.timeout(UPSTREAM_HEADER_TIMEOUT_MS);
+				// aborted) still propagates to both hedged attempts. The timer
+				// is disarmed the moment the fetch settles: the deadline bounds
+				// the header WAIT only, never the streaming body (an
+				// AbortSignal.timeout would keep hanging over the body and kill
+				// a healthy turn past five minutes mid-generation).
+				const headerDeadlineCtl = new AbortController();
+				let headerTimer: NodeJS.Timeout | undefined;
+				let headerDeadlineDisarmed = false;
+				const disarmHeaderTimeout = (): void => {
+					if (headerDeadlineDisarmed) return;
+					headerDeadlineDisarmed = true;
+					clearTimeout(headerTimer);
+					headerTimer = undefined;
+				};
+				headerTimer = setTimeout(() => {
+					disarmHeaderTimeout();
+					headerDeadlineCtl.abort();
+				}, attemptHeaderTimeoutMs);
 				const attemptSignal = opts.signal
-					? AbortSignal.any([opts.signal, attemptTimeout, hedgeCtl.signal])
-					: AbortSignal.any([attemptTimeout, hedgeCtl.signal]);
-				return { headers: attemptHeaders, signal: attemptSignal };
+					? AbortSignal.any([opts.signal, headerDeadlineCtl.signal, hedgeCtl.signal])
+					: AbortSignal.any([headerDeadlineCtl.signal, hedgeCtl.signal]);
+				return { headers: attemptHeaders, signal: attemptSignal, disarmHeaderTimeout };
 			};
 
 			// Hedged failover: an attempt with no response headers within
@@ -289,14 +317,26 @@ export async function relayFetch(
 			if (!canHedge) {
 				const soloCtl = new AbortController();
 				const solo = buildAttempt(targetUrl, soloCtl);
-				res = await fetch(targetUrl, { ...opts, headers: solo.headers, signal: solo.signal, ...relayLegInit() } as unknown as RequestInit);
+				try {
+					res = await fetch(targetUrl, { ...opts, headers: solo.headers, signal: solo.signal, ...relayLegInit() } as unknown as RequestInit);
+				} finally {
+					solo.disarmHeaderTimeout();
+				}
 			} else {
 				const primaryUrl = targetUrl;
 				const nextUrl = candidates[i + 1];
 				const nextCheck = validateRelayUrl(nextUrl);
 				const hedgeCtlA = new AbortController();
 				const first = buildAttempt(primaryUrl, hedgeCtlA);
-				const fetchA = fetch(primaryUrl, { ...opts, headers: first.headers, signal: first.signal, ...relayLegInit() } as unknown as RequestInit);
+				// Disarm a settled leg's header deadline: once the fetch resolves
+				// with headers (or rejects) the attempt's abort window is closed,
+				// so a streaming body is never killed past the fixed ceiling.
+				// Wrapping here covers both the staged promise reader and the
+				// hedge race below.
+				const fetchA = fetch(primaryUrl, { ...opts, headers: first.headers, signal: first.signal, ...relayLegInit() } as unknown as RequestInit).then(
+					(r: Response) => { first.disarmHeaderTimeout(); return r; },
+					(e: unknown) => { first.disarmHeaderTimeout(); throw e; },
+				);
 				const stalled = await new Promise<boolean>((resolve) => {
 					const graceTimer = setTimeout(() => resolve(true), HEDGE_GRACE_MS);
 					fetchA.then(
@@ -311,7 +351,10 @@ export async function relayFetch(
 					const hedgeCtlB = new AbortController();
 					const second = buildAttempt(nextUrl, hedgeCtlB);
 					const hedgeStart = Date.now();
-					const fetchB = fetch(nextUrl, { ...opts, headers: second.headers, signal: second.signal, ...relayLegInit() } as unknown as RequestInit);
+					const fetchB = fetch(nextUrl, { ...opts, headers: second.headers, signal: second.signal, ...relayLegInit() } as unknown as RequestInit).then(
+						(r: Response) => { second.disarmHeaderTimeout(); return r; },
+						(e: unknown) => { second.disarmHeaderTimeout(); throw e; },
+					);
 					// Swallow the loser path so the deliberate abort never
 					// surfaces as an unhandled rejection or a direct fallback.
 					fetchA.catch(() => {});
@@ -671,7 +714,7 @@ export async function relayFetch(
 			rateLimitOnly = false;
 			const errMsg =
 				(err as Error)?.name === "AbortError"
-					? `relay header timeout (${Math.round(UPSTREAM_HEADER_TIMEOUT_MS / 1000)}s exceeded)`
+					? `relay header timeout (${Math.round(attemptHeaderTimeoutMs / 1000)}s exceeded)`
 					: (err as Error)?.message || String(err);
 			markRelayFailure(targetUrl, 0, errMsg);
 			log(

package/src/stream-pipe.ts

@@ -27,6 +27,17 @@ import { repairToolArguments } from "./tool-args.ts";
  */
 export const SUBSTANTIAL_MIN_CHUNKS = 50;
 export const SUBSTANTIAL_MIN_BYTES = 100 * 1024;
+// Body-stall cut for streamed upstream bodies (see pipeUpstreamStream).
+// Generous: every healthy stream observed on this proxy re-arms it within a
+// few hundred ms; a 120s byte-silence on an open socket is a wedged relay
+// leg, not a building model.
+let bodyStallCutMs = 120_000;
+/** Test-only: override the body-stall cut; returns a restore function. */
+export function _setBodyStallCutForTest(ms: number): () => void {
+ const prev = bodyStallCutMs;
+ bodyStallCutMs = ms;
+ return () => { bodyStallCutMs = prev; };
+}
 // Bounded lookahead for a streaming response whose upstream omitted the
 // content-type: the pipe holds at most this many bytes while it waits for SSE
 // evidence, then classifies the body itself (raw passthrough on overflow —
@@ -993,6 +1004,46 @@ export function pipeUpstreamStream(
  // One-shot resume guard: the refire hook below fires at most once per turn.
  let refireUsed = false;
 
+ // Body-stall watchdog: after response headers a streaming body has no fixed
+ // total deadline (See relay.ts: the per-attempt timer is headers-only).
+ // The only failure left is a wedged leg that keeps the socket open
+ // and sends nothing — protected by a per-stream idle cut well above every
+ // observed healthy inter-chunk gap (thinking-heavy streams emit sub-second,
+ // even the slowest logged TTFB is tens of seconds). On expiry the CURRENT
+ // stream is destroyed with a relay-fault error so the truncation path
+ // (health mark + one mid-stream refire) owns the turn. Every forwarded byte
+ // re-arms it; end/close/error disarm it; a rescue re-arms the replacement.
+ let stallCutTimer: NodeJS.Timeout | undefined;
+ let stallCutStream: Readable | null = null;
+ const fireStallCut = (target: Readable): void => {
+  stallCutTimer = undefined;
+  stallCutStream = null;
+  if (target.destroyed || clientAborted || target !== nodeStream) return;
+  const stallErr = new Error(
+   `upstream body idle (no data for ${bodyStallCutMs}ms)`,
+  ) as Error & { code?: string };
+  stallErr.code = "FF_BODY_STALL";
+  stallErr.name = "TimeoutError";
+  target.destroy(stallErr);
+ };
+ const scheduleStallCut = (s: Readable): void => {
+  clearTimeout(stallCutTimer);
+  stallCutStream = s;
+  const timer = setTimeout(() => fireStallCut(s), bodyStallCutMs);
+  timer.unref?.();
+  stallCutTimer = timer;
+ };
+ const breatheStallCut = (): void => {
+  if (stallCutTimer !== undefined && stallCutStream === nodeStream) {
+   scheduleStallCut(nodeStream);
+  }
+ };
+ const disarmStallCut = (): void => {
+  clearTimeout(stallCutTimer);
+  stallCutTimer = undefined;
+  stallCutStream = null;
+ };
+
  const sniffThinking = (chunk: Buffer | string): boolean => {
   const s =
    typeof chunk === "string"
@@ -1287,6 +1338,7 @@ export function pipeUpstreamStream(
   sseDecoder = new StringDecoder("utf8");
   terminalScanCarry = Buffer.alloc(0);
   watch(next);
+  scheduleStallCut(next);
   log("debug", "mid-stream refire: continuing turn on replacement stream", { totalChunks, totalBytes }, rid);
   return true;
  };
@@ -1341,6 +1393,7 @@ export function pipeUpstreamStream(
  };
  const onData = (chunk: Buffer | string): void => {
   try {
+   breatheStallCut();
    // While a replacement stream is being fetched the cut stream is stale:
    // its late frames must not interleave with the rescued turn.
    if (refirePending) return;
@@ -1386,6 +1439,7 @@ export function pipeUpstreamStream(
  };
 
  const onError = async (e: unknown): Promise<void> => {
+  disarmStallCut();
   // A rescue is already in flight (its outcome decides the turn): a late
   // error from the cut stream must not inject a terminal mid-rescue.
   if (refirePending) return;
@@ -1437,6 +1491,7 @@ export function pipeUpstreamStream(
  };
 
  const onEnd = async (): Promise<void> => {
+  disarmStallCut();
   // A rescue is in flight (its outcome owns the turn): the cut stream's FIN
   // must neither inject a terminal nor end res mid-rescue.
   if (refirePending) return;
@@ -1542,6 +1597,7 @@ export function pipeUpstreamStream(
  };
 
  const onClose = async (): Promise<void> => {
+  disarmStallCut();
   // A stalled `close` trailing an `end` that is mid-rescue must neither
   // inject a terminal nor end res — the replacement stream owns the turn now.
   if (refirePending) return;
@@ -1603,6 +1659,7 @@ export function pipeUpstreamStream(
   s.on("close", () => { if (s === nodeStream) void onClose(); });
  };
  watch(nodeStream);
+ scheduleStallCut(nodeStream);
 
  req.on("aborted", () => {
   clientAborted = true;