pi-freeflow 1.32.0 → 1.32.2

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 1.32.2
+
+### Patch Changes
+
+- Prevent JSON parse errors on interrupted responses: network packet boundaries split mid-delta no longer produce unterminated string syntax errors when a turn is retried or terminated.
+
+## 1.32.1
+
+### Patch Changes
+
+- Recover interrupted answers automatically: when a response stream cuts off mid-answer, the pending turn is retried once on the next healthy relay instead of stopping partway
+- Retry once automatically when the model briefly fails a request: a transient model-side error is now retried on the next relay instead of surfacing immediately, and health reporting distinguishes model failures from relay problems
+- 9b523e9: Keep relay connections warm across turns for faster repeat requests
+
 ## 1.32.0
 
 ### Minor Changes

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pi-freeflow",
   "type": "module",
-  "version": "1.32.0",
+  "version": "1.32.2",
   "description": "Thin provider for OMP/Pi — model list + dumb relay proxy + log; host pi-ai owns thinking/normalization",
   "main": "extensions/index.ts",
   "types": "src/index.ts",
@@ -57,5 +57,7 @@
     "typescript": "^5.8.2",
     "vitepress": "^1.6.4"
   },
-  "dependencies": {}
+  "dependencies": {
+    "undici": "7.30.0"
+  }
 }

package/src/health.ts

@@ -1,151 +1,158 @@
-/**
- * Health endpoint for pi-freeflow proxy
- * Loopback-only GET /_health (and alias /health) returning relay health snapshot.
- */
-
-import type * as http from "node:http";
-import { ALL_MODELS } from "./models.ts";
-import { getActiveRelayState, getRelayHealth, isRelayHealthy } from "./relay-state.ts";
-import { getLastActivityAt, getLeaseCount, getLeaseSnapshot } from "./lease.ts";
-import { PKG_VERSION, PORT } from "./config.ts";
-import { getLastForwardedByteAt, getSseStats } from "./stream-pipe.ts";
-
-export interface HealthRelayInfo {
-	url: string;
-	label?: string;
-	healthy: boolean;
-	cooldownUntil: number;
-	consecutiveFailures: number;
-}
-
-export interface HealthData {
-	port: number;
-	active: string;
-	mode: string;
-	enabled: boolean;
-	relays: HealthRelayInfo[];
-	catalog: number;
-	version: string;
-	/** In-flight proxied requests right now (stale-daemon replacement guard). */
-	activeRequests: number;
-	/** Clients holding a live lease (detached-daemon GC). */
-	clients: number;
-	/** clientId -> lastSeenAt for every live lease. */
-	leases: Record<string, number>;
-	/** Last time any request was proxied (request-touch for legacy clients). */
-	lastActivityAt: number;
-	/** Failed-SSE rolling window: failures in the last 20 streams. */
-	sseFailed: number;
-	/** Streams recorded in the current rolling window (max 20). */
-	sseTotal: number;
-	/** Failure rate over the window (0 when empty). */
-	sseRate: number;
-	/** True when the window holds enough samples and the rate exceeds 50%. */
-	sseDegraded: boolean;
-	/** Last time any stream byte was forwarded (busy-bypass quiet check), 0 when no stream yet. */
-	lastBytesAt: number;
-}
-
-/**
- * Wire subset of HealthData fetched from a running daemon over loopback.
- * Single source of truth for the recovery-decision shape (client.ts
- * HealthForRecovery) and the fetch shape (proxy.ts getDaemonHealth): each
- * field's type follows HealthData, so a rename there breaks here at compile
- * time instead of silently desyncing a hand-duplicated copy.
- * Fields stay optional — older daemons predate them — and version is nullable
- * for unversioned responses. The key list must match what getDaemonHealth parses.
- */
-export type DaemonHealthSnapshot = {
-	[K in "activeRequests" | "sseRate" | "sseDegraded" | "lastBytesAt"]?: HealthData[K] | undefined;
-} & {
-	version: string | null;
-} | null;
-
-/**
- * Collect current health snapshot.
- * @param portOverride - actual listening port (defaults to config PORT)
- * @param activeRequests - in-flight proxied requests (defaults to 0 for callers that do not track)
- */
-export function getHealthData(portOverride?: number, activeRequests = 0): HealthData {
-	const state = getActiveRelayState();
-	const relays: HealthRelayInfo[] = (state.relays || []).map((r) => {
-		const h = getRelayHealth(r.url);
-		const healthy = isRelayHealthy(r.url);
-		return {
-			url: r.url,
-			label: r.label,
-			healthy,
-			cooldownUntil: h?.cooldownUntil ?? 0,
-			consecutiveFailures: h?.consecutiveFailures ?? 0,
-		};
-	});
-	const sse = getSseStats();
-	return {
-		port: portOverride ?? PORT,
-		active: state.url || "",
-		mode: (state.mode as string) ?? "auto",
-		enabled: Boolean(state.enabled),
-		relays,
-		catalog: ALL_MODELS.length,
-		version: PKG_VERSION,
-		activeRequests,
-		clients: getLeaseCount(),
-		leases: getLeaseSnapshot(),
-		lastActivityAt: getLastActivityAt(),
-		sseFailed: sse.failures,
-		sseTotal: sse.total,
-		sseRate: sse.rate,
-		sseDegraded: sse.degraded,
-		lastBytesAt: getLastForwardedByteAt(),
-	};
-}
-
-/**
- * Check whether an IP address is a loopback address (127.0.0.1, ::1, localhost).
- */
-export function isLoopbackIP(ip: string): boolean {
-	if (!ip) return false;
-	const withoutZone = ip.split("%")[0];
-	const clean = withoutZone.startsWith("::ffff:") ? withoutZone.slice(7) : withoutZone;
-	return clean === "127.0.0.1" || clean === "::1" || clean === "localhost";
-}
-
-/**
- * Handle loopback health requests.
- * Returns true if request was a health endpoint (handled, response already sent).
- * Returns false if not a health path (caller should continue).
- */
-export function handleHealthRequest(
-	req: http.IncomingMessage,
-	res: http.ServerResponse,
-	portOverride?: number,
-	activeRequests = 0,
-): boolean {
-	if (req.method !== "GET") return false;
-
-	let reqPathname: string | null = null;
-	try {
-		reqPathname = new URL(req.url ?? "/", `http://127.0.0.1`).pathname;
-	} catch {
-		return false;
-	}
-	if (reqPathname === null) return false;
-	if (reqPathname !== "/_health" && reqPathname !== "/health") return false;
-
-	const clientIP = req.socket.remoteAddress ?? "";
-	if (!isLoopbackIP(clientIP)) {
-		res.writeHead(403, { "content-type": "application/json" });
-		res.end(JSON.stringify({ error: "loopback only" }));
-		return true;
-	}
-
-	const data = getHealthData(portOverride, activeRequests);
-	const body = JSON.stringify(data, null, 2);
-	res.writeHead(200, {
-		"content-type": "application/json",
-		"content-length": Buffer.byteLength(body),
-		"cache-control": "no-store",
-	});
-	res.end(body);
-	return true;
+/**
+ * Health endpoint for pi-freeflow proxy
+ * Loopback-only GET /_health (and alias /health) returning relay health snapshot.
+ */
+
+import type * as http from "node:http";
+import { ALL_MODELS } from "./models.ts";
+import { getActiveRelayState, getRelayHealth, isRelayHealthy } from "./relay-state.ts";
+import { getLastActivityAt, getLeaseCount, getLeaseSnapshot } from "./lease.ts";
+import { PKG_VERSION, PORT } from "./config.ts";
+import { getLastForwardedByteAt, getMidStreamFaultStats, getSseStats } from "./stream-pipe.ts";
+
+export interface HealthRelayInfo {
+	url: string;
+	label?: string;
+	healthy: boolean;
+	cooldownUntil: number;
+	consecutiveFailures: number;
+}
+
+export interface HealthData {
+	port: number;
+	active: string;
+	mode: string;
+	enabled: boolean;
+	relays: HealthRelayInfo[];
+	catalog: number;
+	version: string;
+	/** In-flight proxied requests right now (stale-daemon replacement guard). */
+	activeRequests: number;
+	/** Clients holding a live lease (detached-daemon GC). */
+	clients: number;
+	/** clientId -> lastSeenAt for every live lease. */
+	leases: Record<string, number>;
+	/** Last time any request was proxied (request-touch for legacy clients). */
+	lastActivityAt: number;
+	/** Failed-SSE rolling window: failures in the last 20 streams. */
+	sseFailed: number;
+	/** Streams recorded in the current rolling window (max 20). */
+	sseTotal: number;
+	/** Failure rate over the window (0 when empty). */
+	sseRate: number;
+	/** True when the window holds enough samples and the rate exceeds 50%. */
+	sseDegraded: boolean;
+	/** Last time any stream byte was forwarded (busy-bypass quiet check), 0 when no stream yet. */
+	lastBytesAt: number;
+	/** Mid-stream turns the upstream model itself failed (model fault, never a relay fault). */
+	midStreamModelFaults: number;
+	/** Mid-stream turns cut by the relay/transport (relay fault, rotated away from). */
+	midStreamRelayTruncations: number;
+}
+
+/**
+ * Wire subset of HealthData fetched from a running daemon over loopback.
+ * Single source of truth for the recovery-decision shape (client.ts
+ * HealthForRecovery) and the fetch shape (proxy.ts getDaemonHealth): each
+ * field's type follows HealthData, so a rename there breaks here at compile
+ * time instead of silently desyncing a hand-duplicated copy.
+ * Fields stay optional — older daemons predate them — and version is nullable
+ * for unversioned responses. The key list must match what getDaemonHealth parses.
+ */
+export type DaemonHealthSnapshot = {
+	[K in "activeRequests" | "sseRate" | "sseDegraded" | "lastBytesAt"]?: HealthData[K] | undefined;
+} & {
+	version: string | null;
+} | null;
+
+/**
+ * Collect current health snapshot.
+ * @param portOverride - actual listening port (defaults to config PORT)
+ * @param activeRequests - in-flight proxied requests (defaults to 0 for callers that do not track)
+ */
+export function getHealthData(portOverride?: number, activeRequests = 0): HealthData {
+	const state = getActiveRelayState();
+	const relays: HealthRelayInfo[] = (state.relays || []).map((r) => {
+		const h = getRelayHealth(r.url);
+		const healthy = isRelayHealthy(r.url);
+		return {
+			url: r.url,
+			label: r.label,
+			healthy,
+			cooldownUntil: h?.cooldownUntil ?? 0,
+			consecutiveFailures: h?.consecutiveFailures ?? 0,
+		};
+	});
+	const sse = getSseStats();
+	const midStreamFaults = getMidStreamFaultStats();
+	return {
+		port: portOverride ?? PORT,
+		active: state.url || "",
+		mode: (state.mode as string) ?? "auto",
+		enabled: Boolean(state.enabled),
+		relays,
+		catalog: ALL_MODELS.length,
+		version: PKG_VERSION,
+		activeRequests,
+		clients: getLeaseCount(),
+		leases: getLeaseSnapshot(),
+		lastActivityAt: getLastActivityAt(),
+		sseFailed: sse.failures,
+		sseTotal: sse.total,
+		sseRate: sse.rate,
+		sseDegraded: sse.degraded,
+		lastBytesAt: getLastForwardedByteAt(),
+		midStreamModelFaults: midStreamFaults.modelFaults,
+		midStreamRelayTruncations: midStreamFaults.relayTruncations,
+	};
+}
+
+/**
+ * Check whether an IP address is a loopback address (127.0.0.1, ::1, localhost).
+ */
+export function isLoopbackIP(ip: string): boolean {
+	if (!ip) return false;
+	const withoutZone = ip.split("%")[0];
+	const clean = withoutZone.startsWith("::ffff:") ? withoutZone.slice(7) : withoutZone;
+	return clean === "127.0.0.1" || clean === "::1" || clean === "localhost";
+}
+
+/**
+ * Handle loopback health requests.
+ * Returns true if request was a health endpoint (handled, response already sent).
+ * Returns false if not a health path (caller should continue).
+ */
+export function handleHealthRequest(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+	portOverride?: number,
+	activeRequests = 0,
+): boolean {
+	if (req.method !== "GET") return false;
+
+	let reqPathname: string | null = null;
+	try {
+		reqPathname = new URL(req.url ?? "/", `http://127.0.0.1`).pathname;
+	} catch {
+		return false;
+	}
+	if (reqPathname === null) return false;
+	if (reqPathname !== "/_health" && reqPathname !== "/health") return false;
+
+	const clientIP = req.socket.remoteAddress ?? "";
+	if (!isLoopbackIP(clientIP)) {
+		res.writeHead(403, { "content-type": "application/json" });
+		res.end(JSON.stringify({ error: "loopback only" }));
+		return true;
+	}
+
+	const data = getHealthData(portOverride, activeRequests);
+	const body = JSON.stringify(data, null, 2);
+	res.writeHead(200, {
+		"content-type": "application/json",
+		"content-length": Buffer.byteLength(body),
+		"cache-control": "no-store",
+	});
+	res.end(body);
+	return true;
 }

package/src/proxy.ts

@@ -53,7 +53,7 @@ import {
 } from "./tool-translation.ts";
 // normalize removed — host pi-ai already normalizes thinking/reasoning before proxy
 import { relayFetch } from "./relay.ts";
-import { getActiveRelayState, orderedRelayCandidates } from "./relay-state.ts";
+import { getActiveRelayState, markRelayFailure, orderedRelayCandidates } from "./relay-state.ts";
 import {
  issuerRelayFor,
  prepareChatFailoverBody,
@@ -1376,6 +1376,17 @@ export function startProxy(
           connection: "keep-alive",
           "x-accel-buffering": "no",
          });
+         // Mid-stream failover (budget: exactly one re-fire): headers are
+         // already sent, so a relay that FINs the SSE body without a terminal
+         // marker cannot roll the usual pre-stream way. pipeUpstreamStream
+         // invokes refire at most once, only for genuine upstream truncation
+         // with the client still attached — client aborts and clean terminals
+         // never re-fire. Public practice matches: gateways retry the request
+         // (LiteLLM router fallbacks); SSE resume is not a thing (the API has
+         // no range semantics). The host abandons partial tool calls on
+         // incomplete turns, so the re-fired turn yields a fresh complete
+         // answer instead of a dead stop.
+         let midStreamRefires = 0;
          pipeUpstreamStream(
           Readable.fromWeb(
            response.body as unknown as WebReadableStream,
@@ -1385,6 +1396,54 @@ export function startProxy(
           reqId,
           servedIssuer ?? "direct",
           streamCloak,
+          {
+           refire: async () => {
+            if (midStreamRefires >= 1) return null;
+            midStreamRefires += 1;
+            const cutter = servedIssuer ?? "direct";
+            log(
+             "warn",
+             `responses stream truncated on ${cutter === "direct" ? "direct upstream" : `relay ${cutter}`} — re-firing once on next relay`,
+             { model: (parsedBody as Record<string, unknown> | null)?.model, path: req.url, relay: cutter },
+             reqId,
+            );
+            let retryRes: Response;
+            // Mark the cutter BEFORE re-sending: candidate ordering partitions
+            // healthy-first, so only a pre-marked (cooling) cutter guarantees
+            // the re-fire rolls to a sibling instead of retrying the relay
+            // that just dropped us. A mark here is deserved either way — the
+            // relay demonstrably cut a live stream. (If the re-fire itself
+            // finds nothing usable, the pipe fallthrough may add its own mark
+            // on socket-error paths; bounded by the 4x escalation cap.)
+            if (cutter !== "direct") markRelayFailure(cutter, 0, "stream truncated mid-stream");
+            try {
+             retryRes = await sendViaRelay(bodyForUpstream);
+             if (responsesRequest) {
+              retryRes = await retryWithoutReasoningEncryption(
+               retryRes,
+               requestBody,
+               sendViaRelay,
+               reqId,
+               conversationKey,
+              );
+             }
+            } catch {
+             return null;
+            }
+            if (!retryRes.ok || !retryRes.body) {
+             try {
+              await retryRes.body?.cancel();
+             } catch { }
+             return null;
+            }
+            relayResponse = retryRes;
+            recordUpstreamSuccess("zen", { sessionKey, canary: isCanary });
+            if (responsesRequest && conversationKey !== null && retryRes.ok && issuerReported) {
+             rememberIssuerRelay(conversationKey, servedIssuer);
+            }
+            return Readable.fromWeb(retryRes.body as unknown as WebReadableStream);
+           },
+          },
          );
         } else {
          if (!response.ok) {
@@ -1572,6 +1631,10 @@ export function startProxy(
       if (ka) outHeaders["keep-alive"] = ka;
 
       if (isStream && upstreamRes.body) {
+       // Force anti-buffer headers on the streaming leg: upstream rarely sends
+       // them, and without them a downstream hop may buffer SSE chunks.
+       outHeaders["cache-control"] = outHeaders["cache-control"] ?? "no-cache, no-transform";
+       outHeaders["x-accel-buffering"] = "no";
        res.writeHead(upstreamRes.status, outHeaders);
        pipeUpstreamStream(
         Readable.fromWeb(upstreamRes.body as unknown as WebReadableStream),