pi-freeflow 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -18
- package/package.json +4 -1
- package/src/commands.ts +13 -1
- package/src/index.ts +2 -2
- package/src/models.ts +12 -19
- package/src/relay-state.ts +33 -4
- package/src/relay.ts +10 -1
- package/src/stream-pipe.ts +61 -13
package/README.md
CHANGED
|
@@ -46,11 +46,11 @@ Optimized for deep reasoning, long-horizon coding & autonomous agentic workflows
|
|
|
46
46
|
| `muse-spark-1.2-contributor-free` | Meta Superintelligence Labs | **1M** (1.048.576) | **131K** (131.072) | `minimal → max` | ✅ |
|
|
47
47
|
| `x-preview-f-free` | Ox Alpha | **1M** (1.048.576) | **131K** (131.072) | `low / high / max` | ✅ |
|
|
48
48
|
| `mimo-v2.5-free` | Xiaomi MiMo | **1M** (1.048.576) | **131K** (131.072) | `low / medium / high` | ✅ |
|
|
49
|
-
| `deepseek-v4-flash-free` | DeepSeek | **1M** (1.000.000) | **384K** (384.000) | `
|
|
50
|
-
| `laguna-s-2.1-free` | Poolside | **1M** (1.048.576) | **131K** (131.072) | `
|
|
51
|
-
| `nemotron-3.5-lightning-free` | NVIDIA | **1M** (1.000.000) | **262K** (262.144) | `
|
|
52
|
-
| `nemotron-3-ultra-free` | NVIDIA | **1M** (1.000.000) | **128K** (128.000) | `
|
|
53
|
-
| `hy3-free` | Tencent Hunyuan | **262K** (262.144) | **262K** (262.144) | `
|
|
49
|
+
| `deepseek-v4-flash-free` | DeepSeek | **1M** (1.000.000) | **384K** (384.000) | `low → max` | ❌ |
|
|
50
|
+
| `laguna-s-2.1-free` | Poolside | **1M** (1.048.576) | **131K** (131.072) | `low → max` | ❌ |
|
|
51
|
+
| `nemotron-3.5-lightning-free` | NVIDIA | **1M** (1.000.000) | **262K** (262.144) | `low → max` | ❌ |
|
|
52
|
+
| `nemotron-3-ultra-free` | NVIDIA | **1M** (1.000.000) | **128K** (128.000) | `low → max` | ❌ |
|
|
53
|
+
| `hy3-free` | Tencent Hunyuan | **262K** (262.144) | **262K** (262.144) | `low → max` | ❌ |
|
|
54
54
|
| `big-pickle` | Big Pickle | **200K** (200.000) | **32K** (32.000) | `high / max` | ❌ |
|
|
55
55
|
|
|
56
56
|
#### KiloCode Gateway (14 Models) — OpenRouter Compatible
|
|
@@ -59,15 +59,15 @@ Keyless access with `Bearer kilo-free`. Clean slash-free and colon-free CLI alia
|
|
|
59
59
|
| Model ID | Creator / Lab | Context | Max Output | Thinking | Vision |
|
|
60
60
|
| :--- | :--- | :--- | :--- | :--- | :--- |
|
|
61
61
|
| `dots-3-note-preview` (`dots-studio/...:free`) | Dots Studio | **512K** (512.000) | **512K** (512.000) | `openrouter` (auto) | ✅ |
|
|
62
|
-
| `step-3.7-flash` (`stepfun/...:free`) | StepFun | **262K** (262.144) | **262K** (262.144) | `
|
|
62
|
+
| `step-3.7-flash` (`stepfun/...:free`) | StepFun | **262K** (262.144) | **262K** (262.144) | `openrouter` (auto) | ✅ |
|
|
63
63
|
| `nemotron-3-nano-omni` (`nvidia/...:free`) | NVIDIA | **256K** (256.000) | **65K** (65.536) | `openrouter` (auto) | ✅ |
|
|
64
64
|
| `nemotron-3-ultra-550b` (`nvidia/...:free`) | NVIDIA | **1M** (1.000.000) | **65K** (65.536) | `openrouter` (auto) | ❌ |
|
|
65
|
-
| `nemotron-3.5-lightning
|
|
65
|
+
| `nvidia/nemotron-3.5-lightning:free` | NVIDIA | **1M** (1.000.000) | **131K** (131.072) | `openrouter` (auto) | ❌ |
|
|
66
66
|
| `nemotron-3-super` (`nvidia/...:free`) | NVIDIA | **262K** (262.144) | **262K** (262.144) | `openrouter` (auto) | ❌ |
|
|
67
|
-
| `hy3` (`tencent/hy3:free`) | Tencent Hunyuan | **262K** (262.144) | **262K** (262.144) | `openrouter` (auto) | ❌ |
|
|
67
|
+
| `hy3:free` (`tencent/hy3:free`) | Tencent Hunyuan | **262K** (262.144) | **262K** (262.144) | `openrouter` (auto) | ❌ |
|
|
68
68
|
| `north-mini-code` (`cohere/...:free`) | Cohere | **256K** (256.000) | **64K** (64.000) | `openrouter` (auto) | ❌ |
|
|
69
|
-
| `laguna-s-2.1` (`poolside/...:free`) | Poolside | **1M** (1.048.576) | **131K** (131.072) | `openrouter` (auto) | ❌ |
|
|
70
|
-
| `laguna-xs-2.1` (`poolside/...:free`) | Poolside | **262K** (262.144) | **32K** (32.768) | `openrouter` (auto) | ❌ |
|
|
69
|
+
| `laguna-s-2.1:free` (`poolside/...:free`) | Poolside | **1M** (1.048.576) | **131K** (131.072) | `openrouter` (auto) | ❌ |
|
|
70
|
+
| `laguna-xs-2.1:free` (`poolside/...:free`) | Poolside | **262K** (262.144) | **32K** (32.768) | `openrouter` (auto) | ❌ |
|
|
71
71
|
| `lfm-2.5` (`liquid/lfm-2.5-2.6b:free`) | Liquid AI | **128K** (128.000) | **32K** (32.768) | `openrouter` (auto) | ❌ |
|
|
72
72
|
| `kilo-auto` (`kilo-auto/free`) | Kilo Gateway Auto | **256K** (256.000) | **10K** (10.000) | ❌ *(non-thinking)* | ❌ |
|
|
73
73
|
| `openrouter` (`openrouter/free`) | OpenRouter Free | **200K** (200.000) | **65K** (65.536) | ❌ *(non-thinking)* | ✅ |
|
|
@@ -84,7 +84,7 @@ You → 127.0.0.1:18080 (dumb proxy, host-normalized) → x-relay-target → N e
|
|
|
84
84
|
|
|
85
85
|
1. **Per-Request Round-Robin**: 10 parallel subagents hit N different egress IPs (your pool size). No thundering herd.
|
|
86
86
|
2. **Adaptive Health & Error Cooldown**: Relays hitting 429, 504, or socket disconnects enter temporary cooldown (30-90s) and automatically move behind healthy candidates. Healthy relays handle traffic with 0ms wasted delay.
|
|
87
|
-
3. **Seamless 429 Roll**: `429 / 502 / 503 / 504` → instant roll to next relay, never 429 to agent.
|
|
87
|
+
3. **Seamless 429 Roll**: `429 / 408 / 502 / 503 / 504 / 520-530` → instant roll to next relay, never 429 to agent.
|
|
88
88
|
4. **Stream Truncation Resilience**: Stateful SSE terminal tracking prevents fatal unhandled stream closed errors when connections drop.
|
|
89
89
|
5. **Direct Fallback Safety Net**: If all relays in the pool are exhausted, transparent direct fetch to upstream.
|
|
90
90
|
6. **Zero Subagent Connect Errors**: 24h `DISK_CACHE_ONLY` model catalog avoids subagents hammering remote catalogs.
|
|
@@ -105,7 +105,7 @@ Manage your relay pool directly from the OMP / Pi terminal:
|
|
|
105
105
|
/freeflow label <index|url> <name># Assign a friendly label to a relay
|
|
106
106
|
/freeflow remove <index|url|label># Remove a relay from the pool
|
|
107
107
|
/freeflow on | off | auto # Toggle relay mode (auto = enabled for freeflow)
|
|
108
|
-
/freeflow deploy # Guided
|
|
108
|
+
/freeflow deploy # Guided deploy of ONE Vercel Edge relay (in-memory token)
|
|
109
109
|
/freeflow logs [lines] # Inspect recent proxy logs
|
|
110
110
|
/freeflow debug on | off # Toggle full HTTP lifecycle debug logging
|
|
111
111
|
```
|
|
@@ -146,18 +146,63 @@ Default ships direct. Add relays via `/freeflow add <url> [label]`.
|
|
|
146
146
|
|
|
147
147
|
**Option A: Cloudflare Workers (100k req/day, no 25s timeout)**
|
|
148
148
|
1. Go to `dash.cloudflare.com` → Workers → Create → Deploy → Edit code
|
|
149
|
-
2. Paste
|
|
149
|
+
2. Paste this Worker relay snippet:
|
|
150
|
+
|
|
151
|
+
```js
|
|
152
|
+
// Only the 2 upstreams pi-freeflow talks to. Anything else = open proxy abuse.
|
|
153
|
+
const ALLOWED_TARGETS = ["https://opencode.ai", "https://api.kilo.ai"];
|
|
154
|
+
|
|
155
|
+
export default {
|
|
156
|
+
async fetch(req) {
|
|
157
|
+
const target = req.headers.get("x-relay-target");
|
|
158
|
+
const relayPath = req.headers.get("x-relay-path") || "/";
|
|
159
|
+
if (!target) return new Response(JSON.stringify({ error: "Missing x-relay-target header" }), { status: 400 });
|
|
160
|
+
const cleanTarget = target.replace(/\/$/, "");
|
|
161
|
+
if (!ALLOWED_TARGETS.includes(cleanTarget)) return new Response(JSON.stringify({ error: "Forbidden target" }), { status: 403 });
|
|
162
|
+
if (!relayPath.startsWith("/")) return new Response(JSON.stringify({ error: "Bad path" }), { status: 400 });
|
|
163
|
+
const headers = new Headers(req.headers);
|
|
164
|
+
headers.delete("x-relay-target"); headers.delete("x-relay-path"); headers.delete("host");
|
|
165
|
+
return fetch(cleanTarget + relayPath, { method: req.method, headers, body: req.method !== "GET" && req.method !== "HEAD" ? req.body : undefined });
|
|
166
|
+
},
|
|
167
|
+
};
|
|
168
|
+
```
|
|
150
169
|
3. Deploy and add it:
|
|
151
170
|
|
|
152
171
|
```bash
|
|
153
172
|
omp → /freeflow add https://your.workers.dev cf-worker-1
|
|
154
173
|
```
|
|
155
174
|
|
|
156
|
-
**Option B:
|
|
175
|
+
**Option B: Vercel Edge Relay (1M req/mo) — Auto Deploy**
|
|
157
176
|
```bash
|
|
158
177
|
/freeflow deploy # prompts token in-memory, auto-adds to pool
|
|
159
178
|
```
|
|
160
179
|
|
|
180
|
+
**Option C: Deno Deploy (100k req/day)**
|
|
181
|
+
1. Go to `dash.deno.com` → New Project → Playground
|
|
182
|
+
2. Paste this relay and click **Deploy**:
|
|
183
|
+
|
|
184
|
+
```ts
|
|
185
|
+
const ALLOWED_TARGETS = ["https://opencode.ai", "https://api.kilo.ai"];
|
|
186
|
+
|
|
187
|
+
Deno.serve(async (req) => {
|
|
188
|
+
const target = req.headers.get("x-relay-target");
|
|
189
|
+
const relayPath = req.headers.get("x-relay-path") || "/";
|
|
190
|
+
if (!target || !ALLOWED_TARGETS.includes(target.replace(/\/$/, ""))) {
|
|
191
|
+
return new Response(JSON.stringify({ error: "Forbidden target" }), { status: 403 });
|
|
192
|
+
}
|
|
193
|
+
const headers = new Headers(req.headers);
|
|
194
|
+
headers.delete("x-relay-target"); headers.delete("x-relay-path"); headers.delete("host");
|
|
195
|
+
const res = await fetch(target.replace(/\/$/, "") + relayPath, { method: req.method, headers, body: req.method !== "GET" && req.method !== "HEAD" ? req.body : undefined });
|
|
196
|
+
return new Response(res.body, { status: res.status, headers: res.headers });
|
|
197
|
+
});
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
3. Add it:
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
/freeflow add https://your-project.deno.dev deno-relay-1
|
|
204
|
+
```
|
|
205
|
+
|
|
161
206
|
**Verify your pool:**
|
|
162
207
|
```bash
|
|
163
208
|
/freeflow status # relay-A 1/N (ON) → candidates:N
|
|
@@ -186,12 +231,12 @@ Log rotation at 5MB. Clean, parseable, real-time HTTP lifecycle tracking.
|
|
|
186
231
|
|
|
187
232
|
| Fat Plugin (Before) | Thin FreeFlow (Now) |
|
|
188
233
|
| :--- | :--- |
|
|
189
|
-
| `src/normalizer.ts` 200 LOC duplicates host | Deleted — host `pi-ai` owns
|
|
234
|
+
| `src/normalizer.ts` 200 LOC duplicates host | Deleted — host `pi-ai` owns 10 `thinkingFormat` |
|
|
190
235
|
| Live fetch 10s in subagent → timeout | Disk-only 24h, proxy-only |
|
|
191
236
|
| 400 loops N× relays | Narrow 429/5xx only, 400 surfaces instantly |
|
|
192
237
|
| `GET /v1/models?foo` leaks paid | Pathname-guarded |
|
|
193
238
|
|
|
194
|
-
**Result:
|
|
239
|
+
**Result: ~3.6k LOC total, 53 tests PASS, typecheck PASS.**
|
|
195
240
|
|
|
196
241
|
---
|
|
197
242
|
|
|
@@ -216,8 +261,6 @@ We use OpenCode Zen & Kilo free tiers. You pay only with your own Cloudflare/Ver
|
|
|
216
261
|
|
|
217
262
|
### Contributing
|
|
218
263
|
|
|
219
|
-
> "I shipped 10 parallel subagents on 1M context, zero keys, zero 429s. FreeFlow did the rolling, Pi did the thinking."
|
|
220
|
-
|
|
221
264
|
```bash
|
|
222
265
|
git clone https://github.com/trefeon/pi-freeflow
|
|
223
266
|
pnpm install
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-freeflow",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.4.
|
|
4
|
+
"version": "1.4.2",
|
|
5
5
|
"description": "Thin provider for OMP/Pi — model list + dumb relay proxy + log; host pi-ai owns thinking/normalization",
|
|
6
6
|
"main": "extensions/index.ts",
|
|
7
7
|
"types": "src/index.ts",
|
|
@@ -23,6 +23,9 @@
|
|
|
23
23
|
"url": "git+https://github.com/trefeon/pi-freeflow.git"
|
|
24
24
|
},
|
|
25
25
|
"homepage": "https://github.com/trefeon/pi-freeflow#readme",
|
|
26
|
+
"engines": {
|
|
27
|
+
"node": ">=22.6.0"
|
|
28
|
+
},
|
|
26
29
|
"omp": {
|
|
27
30
|
"extensions": [
|
|
28
31
|
"./extensions"
|
package/src/commands.ts
CHANGED
|
@@ -410,7 +410,19 @@ export function createCommandSpec(
|
|
|
410
410
|
);
|
|
411
411
|
} else if (arg.startsWith("level")) {
|
|
412
412
|
const lvl = arg.split(/\s+/)[1] as LogLevel | undefined;
|
|
413
|
-
|
|
413
|
+
// Allowlist only levels that actually have emitters. 'audit' exists in
|
|
414
|
+
// LOG_LEVEL_ORDER but nothing ever logs at it, so accepting it here
|
|
415
|
+
// silently suppressed ALL log output until manually reset.
|
|
416
|
+
const EMITTED_LEVELS: readonly LogLevel[] = [
|
|
417
|
+
"debug",
|
|
418
|
+
"info",
|
|
419
|
+
"warn",
|
|
420
|
+
"error",
|
|
421
|
+
];
|
|
422
|
+
if (
|
|
423
|
+
lvl &&
|
|
424
|
+
(EMITTED_LEVELS as readonly string[]).includes(lvl)
|
|
425
|
+
) {
|
|
414
426
|
saveDebugState({ debug: false, level: lvl });
|
|
415
427
|
ctx.ui.notify(
|
|
416
428
|
`Log level set to ${lvl} (persisted to ${DEBUG_STATE_FILE})`,
|
package/src/index.ts
CHANGED
|
@@ -139,11 +139,11 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
139
139
|
|
|
140
140
|
// 3. Instant 0ms Static Catalog Registration
|
|
141
141
|
// Register static models immediately on boot so Pi/OMP picker is populated with zero latency!
|
|
142
|
-
const registeredCatalog: RegisteredModel[] =
|
|
142
|
+
const registeredCatalog: RegisteredModel[] = ALL_MODELS.map((m) => ({
|
|
143
143
|
...m,
|
|
144
144
|
source: KILO_MODEL_IDS.has(m.id) ? "kilo" : "opencode",
|
|
145
145
|
}));
|
|
146
|
-
setAliveCatalog(
|
|
146
|
+
setAliveCatalog(registeredCatalog);
|
|
147
147
|
pi.registerProvider(
|
|
148
148
|
"freeflow",
|
|
149
149
|
buildProviderConfig(registeredCatalog, actualPort),
|
package/src/models.ts
CHANGED
|
@@ -357,36 +357,29 @@ export const KILO_MODEL_IDS = new Set<string>([
|
|
|
357
357
|
]);
|
|
358
358
|
|
|
359
359
|
/**
|
|
360
|
-
* Combined list of all 23 static free models
|
|
360
|
+
* Combined list of all 23 static free models (canonical)
|
|
361
361
|
*/
|
|
362
362
|
export const ALL_MODELS: ModelDef[] = [...OPENCODE_MODELS, ...KILO_MODELS];
|
|
363
363
|
|
|
364
364
|
/**
|
|
365
365
|
* Map of model ID -> ModelDef
|
|
366
366
|
*/
|
|
367
|
-
export const MODEL_MAP = new Map<string, ModelDef>(
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
return [
|
|
372
|
-
aliasId,
|
|
373
|
-
base
|
|
374
|
-
? { ...base, id: aliasId }
|
|
375
|
-
: { id: aliasId, name: aliasId, reasoning: false, contextWindow: 200_000, maxTokens: 32_000, input: ["text"] },
|
|
376
|
-
];
|
|
377
|
-
}),
|
|
378
|
-
]);
|
|
367
|
+
export const MODEL_MAP = new Map<string, ModelDef>(
|
|
368
|
+
ALL_MODELS.map((m): [string, ModelDef] => [m.id, m]),
|
|
369
|
+
);
|
|
370
|
+
|
|
379
371
|
/**
|
|
380
|
-
*
|
|
372
|
+
* Get full list of registered canonical models for Pi/OMP provider registration
|
|
381
373
|
*/
|
|
382
|
-
export function
|
|
383
|
-
return
|
|
374
|
+
export function getAllRegisteredModels(): ModelDef[] {
|
|
375
|
+
return ALL_MODELS;
|
|
384
376
|
}
|
|
377
|
+
|
|
385
378
|
/**
|
|
386
|
-
*
|
|
379
|
+
* Lookup a model definition by ID (supporting alias fallback)
|
|
387
380
|
*/
|
|
388
|
-
export function
|
|
389
|
-
return
|
|
381
|
+
export function getModelDef(id: string): ModelDef | undefined {
|
|
382
|
+
return MODEL_MAP.get(id) || MODEL_MAP.get(resolveCanonicalModelId(id));
|
|
390
383
|
}
|
|
391
384
|
|
|
392
385
|
|
package/src/relay-state.ts
CHANGED
|
@@ -41,6 +41,32 @@ export function loadRelayState(): RelayState {
|
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
/** Block synchronously for ms without burning CPU. */
|
|
45
|
+
function sleepSync(ms: number): void {
|
|
46
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, Date.now() + ms);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Rename with bounded retry: Windows briefly locks the target when another
|
|
51
|
+
* process holds the state file open, surfacing as EPERM/EACCES/EBUSY.
|
|
52
|
+
* Three attempts with a fixed 50ms backoff; any other error fails immediately.
|
|
53
|
+
*/
|
|
54
|
+
function renameWithRetry(from: string, to: string): void {
|
|
55
|
+
const retryable: Record<string, true> = { EPERM: true, EACCES: true, EBUSY: true };
|
|
56
|
+
for (let attempt = 1; ; attempt++) {
|
|
57
|
+
try {
|
|
58
|
+
fs.renameSync(from, to);
|
|
59
|
+
return;
|
|
60
|
+
} catch (e) {
|
|
61
|
+
const code = (e as NodeJS.ErrnoException | null)?.code;
|
|
62
|
+
if (!code || !retryable[code] || attempt >= 3) {
|
|
63
|
+
throw e;
|
|
64
|
+
}
|
|
65
|
+
sleepSync(50);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
44
70
|
/**
|
|
45
71
|
* Atomically save relay state to disk using a temporary file and rename.
|
|
46
72
|
*/
|
|
@@ -52,7 +78,7 @@ export function saveRelayState(s: RelayState): void {
|
|
|
52
78
|
}
|
|
53
79
|
const tmpPath = `${RELAY_STATE_FILE}.${randomUUID()}.tmp`;
|
|
54
80
|
fs.writeFileSync(tmpPath, JSON.stringify(s, null, 2), "utf8");
|
|
55
|
-
|
|
81
|
+
renameWithRetry(tmpPath, RELAY_STATE_FILE);
|
|
56
82
|
} catch (e) {
|
|
57
83
|
logWarn("Could not persist relay state", { error: String(e) });
|
|
58
84
|
}
|
|
@@ -212,10 +238,13 @@ export function markRelayFailure(url: string, status?: number, error?: string):
|
|
|
212
238
|
cooldownMs = 45_000; // 45s for 5xx errors
|
|
213
239
|
}
|
|
214
240
|
|
|
241
|
+
// Escalate with consecutive failures so chronic offenders back off up to
|
|
242
|
+
// 4x their base cooldown instead of re-hammering at a fixed interval.
|
|
243
|
+
const multiplier = Math.min(4, consecutive);
|
|
215
244
|
relayHealthMap.set(clean, {
|
|
216
245
|
consecutiveFailures: consecutive,
|
|
217
246
|
lastFailureTime: now,
|
|
218
|
-
cooldownUntil: now + cooldownMs,
|
|
247
|
+
cooldownUntil: now + cooldownMs * multiplier,
|
|
219
248
|
lastStatus: status,
|
|
220
249
|
lastError: error,
|
|
221
250
|
});
|
|
@@ -370,9 +399,9 @@ export function getOrderedRelayUrls(): string[] {
|
|
|
370
399
|
const cooling = rawOrdered.filter((u) => !isRelayHealthy(u));
|
|
371
400
|
const ordered = [...healthy, ...cooling];
|
|
372
401
|
|
|
373
|
-
return ordered
|
|
402
|
+
return ordered;
|
|
374
403
|
}
|
|
375
|
-
return [
|
|
404
|
+
return [];
|
|
376
405
|
}
|
|
377
406
|
|
|
378
407
|
/**
|
package/src/relay.ts
CHANGED
|
@@ -27,7 +27,6 @@ export function isRetriableStatus(status: number): boolean {
|
|
|
27
27
|
return (
|
|
28
28
|
status === 429 ||
|
|
29
29
|
status === 408 ||
|
|
30
|
-
status === 500 ||
|
|
31
30
|
status === 502 ||
|
|
32
31
|
status === 503 ||
|
|
33
32
|
status === 504 ||
|
|
@@ -56,6 +55,14 @@ export async function relayFetch(
|
|
|
56
55
|
}
|
|
57
56
|
|
|
58
57
|
const candidates = getOrderedRelayUrls();
|
|
58
|
+
|
|
59
|
+
if (candidates.length === 0) {
|
|
60
|
+
// Empty pool: skip straight to upstream instead of logging a misleading
|
|
61
|
+
// "relays bypassed/exhausted" WARN on every request.
|
|
62
|
+
log("debug", `relayFetch: direct (empty relay pool) -> ${url}`, undefined, rid);
|
|
63
|
+
return fetch(url, opts);
|
|
64
|
+
}
|
|
65
|
+
|
|
59
66
|
let lastResponse: Response | null = null;
|
|
60
67
|
let lastError: unknown = null;
|
|
61
68
|
const u = new URL(url);
|
|
@@ -109,6 +116,7 @@ export async function relayFetch(
|
|
|
109
116
|
// Fast fallback directly to upstream instead of cycling through multiple 25s timeouts.
|
|
110
117
|
if (res.status === 504) {
|
|
111
118
|
markRelayFailure(targetUrl, 504, "Gateway Timeout (25s exceeded)");
|
|
119
|
+
res.body?.cancel().catch(() => {});
|
|
112
120
|
log(
|
|
113
121
|
"warn",
|
|
114
122
|
`relay ${targetUrl} hit HTTP 504 Gateway Timeout in ${elapsed}s (prompt evaluation exceeded Vercel 25s limit) — fast fallback to direct upstream`,
|
|
@@ -120,6 +128,7 @@ export async function relayFetch(
|
|
|
120
128
|
|
|
121
129
|
if (isRetriableStatus(res.status)) {
|
|
122
130
|
markRelayFailure(targetUrl, res.status);
|
|
131
|
+
lastResponse?.body?.cancel().catch(() => {});
|
|
123
132
|
lastResponse = res;
|
|
124
133
|
log(
|
|
125
134
|
"warn",
|
package/src/stream-pipe.ts
CHANGED
|
@@ -34,6 +34,12 @@ export function pipeUpstreamStream(
|
|
|
34
34
|
const startAt = Date.now();
|
|
35
35
|
const isResponsesApi = req.url?.includes("/responses") ?? false;
|
|
36
36
|
let hasTerminalEvent = false;
|
|
37
|
+
// Abort-cause tracking: only genuine upstream-side truncation may penalize
|
|
38
|
+
// relay health. Client disconnects and clean upstream ends must not.
|
|
39
|
+
let upstreamEnded = false;
|
|
40
|
+
let clientAborted = false;
|
|
41
|
+
const TERMINAL_HEAD_SCAN_BYTES = 2000;
|
|
42
|
+
const TERMINAL_TAIL_SCAN_BYTES = 2000;
|
|
37
43
|
|
|
38
44
|
const sniffThinking = (chunk: Buffer | string): boolean => {
|
|
39
45
|
const s =
|
|
@@ -59,10 +65,15 @@ export function pipeUpstreamStream(
|
|
|
59
65
|
s.includes("[DONE]")
|
|
60
66
|
);
|
|
61
67
|
};
|
|
62
|
-
|
|
63
|
-
|
|
68
|
+
const ensureTerminalEvent = (
|
|
69
|
+
isError = false,
|
|
70
|
+
errorMsg?: string,
|
|
71
|
+
penalizeRelay = true,
|
|
72
|
+
) => {
|
|
64
73
|
if (hasTerminalEvent || res.writableEnded) return;
|
|
65
|
-
|
|
74
|
+
// Only genuine upstream-side truncation penalizes relay health.
|
|
75
|
+
// Client disconnects and clean upstream ends leave it untouched.
|
|
76
|
+
if (penalizeRelay && relayUrl && relayUrl !== "direct") {
|
|
66
77
|
markRelayFailure(relayUrl, 0, errorMsg || "stream truncated prematurely");
|
|
67
78
|
}
|
|
68
79
|
if (isResponsesApi && totalChunks > 0) {
|
|
@@ -125,11 +136,19 @@ export function pipeUpstreamStream(
|
|
|
125
136
|
);
|
|
126
137
|
}
|
|
127
138
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
139
|
+
let strPreview: string;
|
|
140
|
+
if (typeof chunk === "string") {
|
|
141
|
+
strPreview = chunk;
|
|
142
|
+
} else if (chunk.length <= TERMINAL_HEAD_SCAN_BYTES) {
|
|
143
|
+
strPreview = chunk.toString("utf8");
|
|
144
|
+
} else {
|
|
145
|
+
// Coalesced final chunks can carry "[DONE]" well past the head —
|
|
146
|
+
// scan the tail too so healthy completions are not penalized.
|
|
147
|
+
strPreview = `${chunk.toString("utf8", 0, TERMINAL_HEAD_SCAN_BYTES)}\n${chunk.toString(
|
|
148
|
+
"utf8",
|
|
149
|
+
chunk.length - TERMINAL_TAIL_SCAN_BYTES,
|
|
150
|
+
)}`;
|
|
151
|
+
}
|
|
133
152
|
if (!hasTerminalEvent && checkTerminalEvent(strPreview)) {
|
|
134
153
|
hasTerminalEvent = true;
|
|
135
154
|
}
|
|
@@ -163,9 +182,16 @@ export function pipeUpstreamStream(
|
|
|
163
182
|
});
|
|
164
183
|
|
|
165
184
|
nodeStream.on("end", () => {
|
|
185
|
+
upstreamEnded = true;
|
|
166
186
|
const elapsed = ((Date.now() - startAt) / 1000).toFixed(1);
|
|
167
187
|
if (!hasTerminalEvent && totalChunks > 0) {
|
|
168
|
-
|
|
188
|
+
// Clean upstream end without a detectable marker: keep the host
|
|
189
|
+
// contract (synthetic terminal) but never blame the relay.
|
|
190
|
+
ensureTerminalEvent(
|
|
191
|
+
false,
|
|
192
|
+
"upstream ended without terminal marker",
|
|
193
|
+
false,
|
|
194
|
+
);
|
|
169
195
|
}
|
|
170
196
|
if (thinkingChunks > 0) {
|
|
171
197
|
log(
|
|
@@ -190,26 +216,48 @@ export function pipeUpstreamStream(
|
|
|
190
216
|
nodeStream.on("close", () => {
|
|
191
217
|
try {
|
|
192
218
|
if (!hasTerminalEvent && totalChunks > 0) {
|
|
193
|
-
|
|
219
|
+
if (clientAborted || upstreamEnded) {
|
|
220
|
+
// Client disconnect teardown or post-end cleanup — the relay
|
|
221
|
+
// is not at fault; still give the host a terminal event.
|
|
222
|
+
ensureTerminalEvent(false, "stream interrupted by client", false);
|
|
223
|
+
} else {
|
|
224
|
+
// Upstream socket died mid-stream with no error event:
|
|
225
|
+
// genuine upstream truncation — penalize the relay.
|
|
226
|
+
ensureTerminalEvent(true, "stream closed prematurely");
|
|
227
|
+
}
|
|
194
228
|
}
|
|
195
229
|
if (!res.writableEnded) res.end();
|
|
196
230
|
} catch {}
|
|
197
231
|
});
|
|
198
232
|
|
|
199
233
|
req.on("aborted", () => {
|
|
234
|
+
clientAborted = true;
|
|
200
235
|
log("warn", "client aborted — destroying upstream", { totalChunks }, rid);
|
|
201
236
|
if (!nodeStream.destroyed) nodeStream.destroy();
|
|
202
237
|
});
|
|
203
238
|
|
|
204
239
|
req.on("close", () => {
|
|
205
|
-
if (!nodeStream.destroyed)
|
|
240
|
+
if (!upstreamEnded && !nodeStream.destroyed) {
|
|
241
|
+
clientAborted = true;
|
|
242
|
+
nodeStream.destroy();
|
|
243
|
+
}
|
|
206
244
|
});
|
|
207
245
|
|
|
208
246
|
res.on("close", () => {
|
|
209
|
-
if (
|
|
247
|
+
if (
|
|
248
|
+
!upstreamEnded &&
|
|
249
|
+
!res.writableEnded &&
|
|
250
|
+
!nodeStream.destroyed
|
|
251
|
+
) {
|
|
252
|
+
clientAborted = true;
|
|
253
|
+
nodeStream.destroy();
|
|
254
|
+
}
|
|
210
255
|
});
|
|
211
256
|
|
|
212
257
|
res.on("error", () => {
|
|
213
|
-
if (!nodeStream.destroyed)
|
|
258
|
+
if (!upstreamEnded && !nodeStream.destroyed) {
|
|
259
|
+
clientAborted = true;
|
|
260
|
+
nodeStream.destroy();
|
|
261
|
+
}
|
|
214
262
|
});
|
|
215
263
|
}
|