pi-freeflow 1.44.0 → 1.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +11 -13
- package/package.json +1 -1
- package/src/catalog.ts +38 -14
- package/src/client-recovery.ts +238 -0
- package/src/client-staging.ts +316 -0
- package/src/client.ts +117 -472
- package/src/cline-accounts.ts +26 -1223
- package/src/cline-device-auth.ts +5 -3
- package/src/cline-pool.ts +691 -0
- package/src/cline-roll.ts +642 -0
- package/src/commands-cline.ts +283 -0
- package/src/commands-relay.ts +808 -0
- package/src/commands-startup.ts +264 -0
- package/src/commands.ts +119 -1202
- package/src/config-paths.ts +254 -0
- package/src/config.ts +6 -687
- package/src/daemon.ts +16 -1
- package/src/deploy.ts +3 -1
- package/src/fingerprint-env.ts +334 -0
- package/src/fingerprint-tools.ts +348 -0
- package/src/index.ts +10 -6
- package/src/lane-translation.ts +263 -0
- package/src/models.ts +70 -112
- package/src/opencode-fingerprint.ts +29 -1310
- package/src/opencode-identity.ts +464 -0
- package/src/proxy-lanes.ts +1125 -0
- package/src/proxy-ports.ts +317 -0
- package/src/proxy.ts +74 -1385
- package/src/relay-codec.ts +365 -0
- package/src/relay-state.ts +10 -367
- package/src/relay.ts +78 -31
- package/src/sse-aggregator.ts +659 -0
- package/src/stream-cloak.ts +882 -0
- package/src/stream-pipe.ts +22 -871
- package/src/system-ca-fetch.ts +15 -4
- package/src/token.ts +7 -2
- package/src/tool-args.ts +10 -3
- package/src/tool-translation.ts +1 -236
- package/src/toolcall-audit.ts +20 -7
- package/src/update-checker.ts +17 -1
- package/src/update-core.ts +5 -2
- package/src/updater.ts +8 -3
- package/src/upstream-health.ts +22 -2
- package/src/zen-sidecar.ts +1 -0
- package/src/zen-transport.ts +48 -18
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,34 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.45.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- Synchronize the free model catalog with live upstreams and resolve cross-platform runtime, stream lifecycle, and process management issues across Windows and Linux (Node 22/24/26, Bun, Pi, and OMP).
|
|
8
|
+
|
|
9
|
+
- **Catalog Sync (30 Models Total)**:
|
|
10
|
+
- Add 3 live verified OpenCode Zen models: `crouton-alpha-free` (1M context, 270K output), `iceman-free` (262K context, 131K output), and `inkling-2-small-free` (1M context, 131K output, multimodal).
|
|
11
|
+
- Add `liquid/lfm-3-1.7b:free` on KiloCode Gateway.
|
|
12
|
+
- Retire 6 discontinued KiloCode models confirmed 404 upstream (`poolside/laguna-s-2.1:free`, `poolside/laguna-xs-2.1:free`, `liquid/lfm-2.5-2.6b:free`, `reflection/beam-free`, `recursive/flint-v1.0:free`, `xingchen-agi/xing4.0-29b-a4b:free`) and dead-list them to prevent revival from stale caches.
|
|
13
|
+
- Update model aliases with clean mappings (`crouton`, `iceman`, `inkling-2-small`, `lfm-3`).
|
|
14
|
+
|
|
15
|
+
- **Cross-Platform & Stream Lifecycle Fixes**:
|
|
16
|
+
- Add a `destroy` hook to the Zen sidecar stream and terminate orphaned child processes on stream cancellation, client disconnect, or stall timeouts.
|
|
17
|
+
- Support Windows `.cmd`/`.bat` runtime shims in child process spawns with `shell: true` and argument quoting, avoiding `EINVAL` under Node 22+.
|
|
18
|
+
- Fix `isDaemonMain()` to resolve symlinks and compare paths case-insensitively on Windows so symlinked daemon launches and lowercase drive paths start normally.
|
|
19
|
+
- Ensure client leases detach cleanly on `session_shutdown` and process exit across Node 22/24/26 and Bun.
|
|
20
|
+
- Filter `lsof` with `-sTCP:LISTEN` on Linux so active client sessions sharing port 28180 are never killed.
|
|
21
|
+
- Prioritize live registry versions and disk caches over installed 1.x binaries during User-Agent generation to fix version-drift auto-repair.
|
|
22
|
+
- Prevent tool argument corruption in `stream-cloak.ts` when function names repeat across stream deltas.
|
|
23
|
+
- Fix the hedged relay race so transient errors on one leg do not prematurely cancel healthy in-flight responses.
|
|
24
|
+
- Add single-flight deduplication and lockfile serialization to Cline token refreshes to prevent concurrent subagent requests from invalidating single-use refresh tokens.
|
|
25
|
+
|
|
26
|
+
## 1.44.1
|
|
27
|
+
|
|
28
|
+
### Patch Changes
|
|
29
|
+
|
|
30
|
+
- Split oversized source modules into focused files with no behavior change. The proxy, command registry, stream pipeline, fingerprint, tool translation, Cline pool, and config internals now live in single-responsibility modules behind the same import paths, so updates and reviews land in smaller files while every command, lane, and gate behaves exactly as before.
|
|
31
|
+
|
|
3
32
|
## 1.44.0
|
|
4
33
|
|
|
5
34
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -19,7 +19,7 @@ Thin by design: a model list, a relay proxy, and a log. The host (`pi-ai`) handl
|
|
|
19
19
|
| **32 free models** | 8 from OpenCode Zen, 15 from KiloCode Gateway, 1 from Cline, 8 from OVHcloud AI Endpoints, context windows up to 1M. Full list below. | **$0** |
|
|
20
20
|
| **Relay pool** | Route requests through your own relays: a self-hosted egress pool (many egress IPs on one small host) or Cloudflare Workers / Deno Deploy / Vercel Edge workers. Traffic spreads across the pool, or stays on one sticky relay. A relay that rate-limits, times out, or drops the connection cools down while healthy ones take its traffic. | **$0** beyond your host or platforms' free tiers |
|
|
21
21
|
| **Automatic fallback** | When every relay is cooling down, requests go direct to upstream instead of failing. | **$0** |
|
|
22
|
-
| **Short model names** | Most models have a slash-free, colon-free alias, plus an optional `:effort` suffix for thinking depth. You type `freeflow/<name
|
|
22
|
+
| **Short model names** | Most models have a slash-free, colon-free alias, plus an optional `:effort` suffix for thinking depth. You type `freeflow/<name>` (including `cline-super-vl` for Cline). | **$0** |
|
|
23
23
|
| **Shared local proxy** | One daemon on `127.0.0.1:28180` serves every session on the machine, so parallel subagents reuse it instead of opening their own connections. Loopback-only and token-gated: hosts send the token automatically, external tools read `~/.pi/agent/token`. | **$0** |
|
|
24
24
|
| **Logs you can read** | `~/.pi/agent/pi-freeflow.log`, rotated at 10MB. Tail it with `/freeflow logs`. | **$0** |
|
|
25
25
|
| **Tool-call diagnostics** | `~/.pi/agent/toolcall-audit.jsonl`, rotated at 5MB. One line per streamed turn that declared tools (and per non-streamed Chat Completions turn) — the calls it made, argument size and validity, finish reason, classified failure — plus one line per abandoned upstream attempt. Names, sizes and reasons only. | **$0** |
|
|
@@ -134,13 +134,13 @@ For `api/relay.js`, use the canonical worker source (see below); `vercel.json` s
|
|
|
134
134
|
|
|
135
135
|
**Canonical worker source (all platforms)**
|
|
136
136
|
|
|
137
|
-
The relay worker template is generated per deployment by `/freeflow deploy`, with one shared core adapted for Vercel, Cloudflare, and Deno. Every deploy embeds a random 32-byte secret, and the generated worker answers `401` to any request without a matching `x-relay-auth` header; the client sends that secret to each relay in the pool. Only a hand-pasted legacy worker built without a secret is open to anyone. Either way, a relay forwards only to an allowlist (`https://opencode.ai`, `https://api.kilo.ai`) with private-host and path checks.
|
|
137
|
+
The relay worker template is generated per deployment by `/freeflow deploy`, with one shared core adapted for Vercel, Cloudflare, and Deno. Every deploy embeds a random 32-byte secret, and the generated worker answers `401` to any request without a matching `x-relay-auth` header; the client sends that secret to each relay in the pool. Only a hand-pasted legacy worker built without a secret is open to anyone. Either way, a relay forwards only to an allowlist (`https://opencode.ai`, `https://api.kilo.ai`, `https://oai.endpoints.kepler.ai.cloud.ovh.net`) with private-host and path checks.
|
|
138
138
|
|
|
139
139
|
```js
|
|
140
140
|
// Minimal Cloudflare illustration. Prefer /freeflow deploy: the generated
|
|
141
141
|
// worker is the hardened source for all three platforms.
|
|
142
142
|
// This example omits the private-host guard, path validation, and auth.
|
|
143
|
-
const ALLOWED_TARGETS = ["https://opencode.ai", "https://api.kilo.ai"];
|
|
143
|
+
const ALLOWED_TARGETS = ["https://opencode.ai", "https://api.kilo.ai", "https://oai.endpoints.kepler.ai.cloud.ovh.net"];
|
|
144
144
|
export default {
|
|
145
145
|
async fetch(req) {
|
|
146
146
|
const target = req.headers.get("x-relay-target");
|
|
@@ -215,7 +215,7 @@ The same command set works identically in OMP and Pi:
|
|
|
215
215
|
|
|
216
216
|
---
|
|
217
217
|
|
|
218
|
-
###
|
|
218
|
+
### 30 models, one command
|
|
219
219
|
|
|
220
220
|
```bash
|
|
221
221
|
/model → freeflow → pick
|
|
@@ -225,7 +225,7 @@ Every model specification is reconciled against [models.dev](https://models.dev/
|
|
|
225
225
|
|
|
226
226
|
The host catalog carries text and image inputs only. Upstream video and audio support (MiMo: audio + video; GLM 5.3 Flash: video + PDF; Nemotron Super VL and Nano Omni: video) is therefore not selectable from the picker and is not declared in `input`.
|
|
227
227
|
|
|
228
|
-
#### OpenCode Zen (
|
|
228
|
+
#### OpenCode Zen (11 models), Chat Completions API
|
|
229
229
|
|
|
230
230
|
Good defaults for long coding sessions and agentic work.
|
|
231
231
|
|
|
@@ -239,8 +239,11 @@ Good defaults for long coding sessions and agentic work.
|
|
|
239
239
|
| `beam-free` | Reflection AI | **1M** (1.048.576) | **131K** (131.072) | `low … max` | ❌ |
|
|
240
240
|
| `deepseek-v4.1-flash-free` | DeepSeek | **1M** (1.048.576) | **393K** (393.216) | `low / high / max` | ✅ |
|
|
241
241
|
| `flint-40b-free` | Recursive | **262K** (262.144) | **65K** (65.536) | `low … max` | ❌ |
|
|
242
|
+
| `crouton-alpha-free` | Stealth (Thinking Machines) | **1M** (1.048.576) | **270K** (270.000) | `minimal … max` | ✅ |
|
|
243
|
+
| `iceman-free` | Stealth | **262K** (262.144) | **131K** (131.072) | `low / high / max` | ❌ |
|
|
244
|
+
| `inkling-2-small-free` | Thinking Machines | **1M** (1.048.576) | **131K** (131.072) | `minimal … max` | ✅ |
|
|
242
245
|
|
|
243
|
-
#### KiloCode Gateway (
|
|
246
|
+
#### KiloCode Gateway (10 models), OpenRouter compatible
|
|
244
247
|
|
|
245
248
|
Keyless access. Most rows have a short alias (shown with the full ID in parentheses); a row that shows only the full ID has no short form.
|
|
246
249
|
|
|
@@ -251,16 +254,11 @@ Keyless access. Most rows have a short alias (shown with the full ID in parenthe
|
|
|
251
254
|
| `nemotron-3-ultra-550b` (`nvidia/...:free`) | NVIDIA | **1M** (1.000.000) | **182K** (182.520) | `minimal…xhigh`\* | ❌ |
|
|
252
255
|
| `nemotron-3.5-lightning` (`nvidia/nemotron-3.5-lightning:free`) | NVIDIA | **1M** (1.000.000) | **235K** (235.929) | `minimal…xhigh`\* | ❌ |
|
|
253
256
|
| `north-mini-code` (`cohere/...:free`) | Cohere | **256K** (256.000) | **64K** (64.000) | `minimal…xhigh`\* | ❌ |
|
|
254
|
-
| `
|
|
255
|
-
| `laguna-xs-2.1:free` (`poolside/...:free`) | Poolside | **262K** (262.144) | **32K** (32.768) | `minimal…xhigh`\* | ❌ |
|
|
256
|
-
| `lfm-2.5` (`liquid/lfm-2.5-2.6b:free`) | Liquid AI | **128K** (128.000) | **32K** (32.768) | `minimal…xhigh`\* | ❌ |
|
|
257
|
+
| `lfm-3` (`liquid/lfm-3-1.7b:free`) | Liquid AI | **262K** (262.144) | **8K** (8.192) | `minimal…xhigh`\* | ❌ |
|
|
257
258
|
| `kilo-auto` (`kilo-auto/free`) | Kilo Gateway Auto | **256K** (256.000) | **10K** (10.000) | `minimal…xhigh`\* | ❌ |
|
|
258
259
|
| `openrouter` (`openrouter/free`) | OpenRouter Free | **200K** (200.000) | **65K** (65.536) | `minimal…xhigh`\* | ✅ |
|
|
259
260
|
| `glyph-cluster` (`stealth/glyph-cluster`) | Stealth preview (lab undisclosed) | **256K** (256.000) | **256K** (256.000) | `minimal…xhigh`\* | ❌ |
|
|
260
|
-
| `nvidia/nemotron-3.5-super-vl:free` | NVIDIA | **1M** (1.000.000) | **131K** (131.072) | `minimal…xhigh`\* | ✅ |
|
|
261
|
-
| `beam-kilo` (`reflection/beam-free`) | Reflection AI | **1M** (1.048.576) | **131K** (131.072) | `low…max`\* | ❌ |
|
|
262
|
-
| `flint-kilo` (`recursive/flint-v1.0:free`) | Recursive | **262K** (262.144) | **65K** (65.536) | `minimal…xhigh`\* | ❌ |
|
|
263
|
-
| `xing4.0` (`xingchen-agi/xing4.0-29b-a4b:free`) | XingChen | **262K** (262.144) | **235K** (235.929) | `minimal…xhigh`\* | ❌ |
|
|
261
|
+
| `nemotron-3.5-super-vl-kilo` (`nvidia/nemotron-3.5-super-vl:free`) | NVIDIA | **1M** (1.000.000) | **131K** (131.072) | `minimal…xhigh`\* | ✅ |
|
|
264
262
|
|
|
265
263
|
\* Levels are forwarded as-is through the OpenRouter-style nested `reasoning` parameter; effort mapping is decided by each model. MiMo collapses `minimal→low` and `xhigh→high` upstream, so its selector shows 5 labels but only 3 distinct effort values.
|
|
266
264
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-freeflow",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.45.0",
|
|
5
5
|
"description": "Thin provider for OMP/Pi — model list + dumb relay proxy + log; host pi-ai owns thinking/normalization",
|
|
6
6
|
"main": "extensions/index.ts",
|
|
7
7
|
"types": "src/index.ts",
|
package/src/catalog.ts
CHANGED
|
@@ -195,6 +195,25 @@ export const DEAD_MODEL_IDS = new Set<string>([
|
|
|
195
195
|
// sibling Kilo id nvidia/nemotron-3-ultra-550b-a55b:free is a separate
|
|
196
196
|
// serving route and is unaffected.
|
|
197
197
|
"nemotron-3-ultra-free",
|
|
198
|
+
// poolside/laguna-s-2.1:free and poolside/laguna-xs-2.1:free: discontinued
|
|
199
|
+
// upstream on KiloCode (2026-11-09) — removed from live /api/gateway/models,
|
|
200
|
+
// answers HTTP 404 "The requested model does not exist" across all egresses.
|
|
201
|
+
"poolside/laguna-s-2.1:free",
|
|
202
|
+
"poolside/laguna-xs-2.1:free",
|
|
203
|
+
// liquid/lfm-2.5-2.6b:free: replaced upstream on KiloCode by liquid/lfm-3-1.7b:free
|
|
204
|
+
// (2026-11-09) — answers HTTP 404 "The requested model is currently unavailable".
|
|
205
|
+
"liquid/lfm-2.5-2.6b:free",
|
|
206
|
+
// reflection/beam-free: discontinued upstream on KiloCode (2026-11-09) —
|
|
207
|
+
// answers HTTP 404 "The requested model does not exist". The Zen beam-free
|
|
208
|
+
// twin is a separate serving route and remains active.
|
|
209
|
+
"reflection/beam-free",
|
|
210
|
+
// recursive/flint-v1.0:free: discontinued upstream on KiloCode (2026-11-09) —
|
|
211
|
+
// answers HTTP 404 "The requested model is currently unavailable". The Zen
|
|
212
|
+
// flint-40b-free twin is a separate serving route and remains active.
|
|
213
|
+
"recursive/flint-v1.0:free",
|
|
214
|
+
// xingchen-agi/xing4.0-29b-a4b:free: discontinued upstream on KiloCode (2026-11-09) —
|
|
215
|
+
// answers HTTP 404 "The requested model is currently unavailable".
|
|
216
|
+
"xingchen-agi/xing4.0-29b-a4b:free",
|
|
198
217
|
]);
|
|
199
218
|
/**
|
|
200
219
|
* Free-tier allowlist for anything entering the picker via network or stale disk.
|
|
@@ -336,25 +355,30 @@ export function enrichModelDef(raw: RawModelItem, source: Upstream): RegisteredM
|
|
|
336
355
|
// Declared fields win over wire aliases: a listing that already names
|
|
337
356
|
// contextWindow/maxTokens/reasoning keeps its values; the snake_case
|
|
338
357
|
// upstream aliases and the id heuristics only fill gaps.
|
|
339
|
-
|
|
340
|
-
typeof
|
|
341
|
-
|
|
358
|
+
const validPosNum = (n: unknown): n is number =>
|
|
359
|
+
typeof n === "number" && Number.isFinite(n) && n > 0;
|
|
360
|
+
const declaredContext =
|
|
361
|
+
validPosNum(raw.contextWindow) ? raw.contextWindow
|
|
362
|
+
: validPosNum(raw.context_length) ? raw.context_length : undefined;
|
|
363
|
+
let contextWindow = declaredContext ?? 262_144;
|
|
342
364
|
if (
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
365
|
+
declaredContext === undefined && (
|
|
366
|
+
idLower.includes("1m") ||
|
|
367
|
+
idLower.includes("ultra") ||
|
|
368
|
+
idLower.includes("lightning") ||
|
|
369
|
+
idLower.includes("mimo-v2.5") ||
|
|
370
|
+
idLower.includes("muse-spark")
|
|
371
|
+
)
|
|
348
372
|
) {
|
|
349
373
|
contextWindow = 1_048_576;
|
|
350
374
|
}
|
|
351
375
|
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
:
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
if (idLower.includes("ultra") || idLower.includes("lightning")) {
|
|
376
|
+
const declaredMaxTokens =
|
|
377
|
+
validPosNum(raw.maxTokens) ? raw.maxTokens
|
|
378
|
+
: validPosNum(raw.max_output_tokens) ? raw.max_output_tokens
|
|
379
|
+
: validPosNum(raw.max_tokens) ? raw.max_tokens : undefined;
|
|
380
|
+
let maxTokens = declaredMaxTokens ?? 65_536;
|
|
381
|
+
if (declaredMaxTokens === undefined && (idLower.includes("ultra") || idLower.includes("lightning"))) {
|
|
358
382
|
maxTokens = 131_072;
|
|
359
383
|
}
|
|
360
384
|
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Watchdog recovery, backoff, and circuit-breaker logic for pi-freeflow daemon.
|
|
3
|
+
*
|
|
4
|
+
* Implements exponential backoff, stuck-busy heuristics, and automatic
|
|
5
|
+
* circuit-breaker halting when repeated recovery attempts fail.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import {
|
|
9
|
+
BREAKER_FAILURES,
|
|
10
|
+
BREAKER_HALT_MS,
|
|
11
|
+
BREAKER_WINDOW_MS,
|
|
12
|
+
BUSY_BYPASS_CONTINUOUS_MS,
|
|
13
|
+
BUSY_BYPASS_QUIET_MS,
|
|
14
|
+
PKG_VERSION,
|
|
15
|
+
RECOVERY_BACKOFF_BASE_MS,
|
|
16
|
+
RECOVERY_BACKOFF_CAP_MS,
|
|
17
|
+
} from "./config.ts";
|
|
18
|
+
import type { DaemonHealthSnapshot } from "./health.ts";
|
|
19
|
+
import { logInfo, logWarn } from "./logger.ts";
|
|
20
|
+
import { compareVersions } from "./update-core.ts";
|
|
21
|
+
|
|
22
|
+
/** First tick (Date.now()) the daemon has looked continuously busy; 0 = idle. */
|
|
23
|
+
let busySince = 0;
|
|
24
|
+
/** Consecutive recovery attempts (backoff exponent n). Reset on healthy watchdog tick. */
|
|
25
|
+
let backoffAttempt = 0;
|
|
26
|
+
/** Timestamps of recent recovery failures (breaker window). */
|
|
27
|
+
let recoveryFailures: number[] = [];
|
|
28
|
+
/** Watchdog respawns halted until this time (breaker open). */
|
|
29
|
+
let breakerOpenUntil = 0;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Respawn backoff: min(2s * 2^n, 60s) + jitter. n is the consecutive-failure
|
|
33
|
+
* count; pass jitterMs = 0 in tests for a deterministic base.
|
|
34
|
+
*/
|
|
35
|
+
export function computeRecoveryBackoffMs(attempt: number, jitterMs = Math.random() * 500): number {
|
|
36
|
+
const n = Math.max(0, Math.floor(attempt));
|
|
37
|
+
const base = Math.min(RECOVERY_BACKOFF_BASE_MS * 2 ** n, RECOVERY_BACKOFF_CAP_MS);
|
|
38
|
+
return base + Math.max(0, jitterMs);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Stuck-busy check (pure, testable): a busy daemon may only be replaced after
|
|
43
|
+
* 5min of continuous busy AND zero forwarded stream bytes for 60s+. Anything
|
|
44
|
+
* else must be left untouched mid-request.
|
|
45
|
+
*/
|
|
46
|
+
export function isStuckBusy(busySinceMs: number, lastBytesAt: number, now = Date.now()): boolean {
|
|
47
|
+
if (!busySinceMs) return false;
|
|
48
|
+
if (now - busySinceMs < BUSY_BYPASS_CONTINUOUS_MS) return false;
|
|
49
|
+
if (lastBytesAt && now - lastBytesAt < BUSY_BYPASS_QUIET_MS) return false;
|
|
50
|
+
return true;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Track the busy/idle edge from a health snapshot; returns the updated busySince. */
|
|
54
|
+
export function trackBusyEdge(activeRequests: number, lastBytesAt: number, now = Date.now()): number {
|
|
55
|
+
if (activeRequests > 0) {
|
|
56
|
+
if (!busySince) busySince = now;
|
|
57
|
+
} else {
|
|
58
|
+
busySince = 0;
|
|
59
|
+
}
|
|
60
|
+
return busySince;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function getBusySince(): number {
|
|
64
|
+
return busySince;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function resetBusySince(): void {
|
|
68
|
+
busySince = 0;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** True while the breaker halt is in effect (respawns paused, fallback serves). */
|
|
72
|
+
export function isBreakerHalted(now = Date.now()): boolean {
|
|
73
|
+
return now < breakerOpenUntil;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Record a recovery failure; opens the breaker after 5 straight failures in 5min. */
|
|
77
|
+
export function recordRecoveryFailure(now = Date.now()): boolean {
|
|
78
|
+
recoveryFailures.push(now);
|
|
79
|
+
recoveryFailures = recoveryFailures.filter((t) => now - t <= BREAKER_WINDOW_MS);
|
|
80
|
+
backoffAttempt += 1;
|
|
81
|
+
if (recoveryFailures.length >= BREAKER_FAILURES) {
|
|
82
|
+
breakerOpenUntil = now + BREAKER_HALT_MS;
|
|
83
|
+
logWarn("recovery breaker open — 5 straight failures in 5min, halting respawn for 10min (in-process fallback meanwhile)");
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Record a healthy watchdog tick: clears the straight-failure run and backoff. */
|
|
90
|
+
export function recordRecoverySuccess(): void {
|
|
91
|
+
recoveryFailures = [];
|
|
92
|
+
backoffAttempt = 0;
|
|
93
|
+
breakerOpenUntil = 0;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Test seam: snapshot of the recovery counters. */
|
|
97
|
+
export function _getRecoveryStateForTest(): {
|
|
98
|
+
backoffAttempt: number;
|
|
99
|
+
failures: number[];
|
|
100
|
+
breakerOpenUntil: number;
|
|
101
|
+
busySince: number;
|
|
102
|
+
} {
|
|
103
|
+
return { backoffAttempt, failures: [...recoveryFailures], breakerOpenUntil, busySince };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Test seam: reset recovery counters without touching timers. */
|
|
107
|
+
export function _resetRecoveryForTest(): void {
|
|
108
|
+
backoffAttempt = 0;
|
|
109
|
+
recoveryFailures = [];
|
|
110
|
+
breakerOpenUntil = 0;
|
|
111
|
+
busySince = 0;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Recovery-decision shape; canonical definition lives in health.ts. */
|
|
115
|
+
export type HealthForRecovery = DaemonHealthSnapshot;
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Older-version guard: only a daemon strictly older than this build is worth
|
|
119
|
+
* recovering. shouldReplaceDaemon deliberately leaves a NEWER daemon running,
|
|
120
|
+
* so recovering from one could never succeed (watchdog → recover → refused →
|
|
121
|
+
* reattach → watchdog, until the breaker parks into the in-process fallback).
|
|
122
|
+
* Null/undefined version (pre-version daemon) is not older on its own.
|
|
123
|
+
*/
|
|
124
|
+
export function isOlderDaemonVersion(version: string | null | undefined): boolean {
|
|
125
|
+
return version != null && compareVersions(version, PKG_VERSION) < 0;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Watchdog decision matrix (pure, testable). Recover when the daemon is gone,
|
|
130
|
+
* serves an older version, or reports a degraded failed-SSE window. Missing
|
|
131
|
+
* sseDegraded (pre-window daemon) never triggers on its own.
|
|
132
|
+
*/
|
|
133
|
+
export function shouldRecoverOnHealth(health: HealthForRecovery): boolean {
|
|
134
|
+
if (health === null) return true;
|
|
135
|
+
if (isOlderDaemonVersion(health.version)) return true;
|
|
136
|
+
if (health.sseDegraded === true) return true;
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export interface RecoveryDeps {
|
|
141
|
+
readonly isEnsuring: () => boolean;
|
|
142
|
+
readonly setEnsuring: (val: boolean) => void;
|
|
143
|
+
readonly ensureInProcessFallback: () => Promise<number>;
|
|
144
|
+
readonly stopHeartbeatInternal: () => void;
|
|
145
|
+
readonly resetHeartbeatPort: () => void;
|
|
146
|
+
readonly attachTo: (port: number) => Promise<void>;
|
|
147
|
+
readonly killStaleDaemon: (port: number, ver: string, what: string) => Promise<boolean>;
|
|
148
|
+
readonly ensureDaemon: () => Promise<number>;
|
|
149
|
+
readonly getDaemonHealth?: (port: number) => Promise<DaemonHealthSnapshot>;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export async function triggerRecovery(port: number, why: string, deps: RecoveryDeps): Promise<void> {
|
|
153
|
+
if (deps.isEnsuring()) return;
|
|
154
|
+
deps.setEnsuring(true);
|
|
155
|
+
try {
|
|
156
|
+
const now = Date.now();
|
|
157
|
+
if (isBreakerHalted(now)) {
|
|
158
|
+
logWarn(`recovery breaker halted — serving in-process fallback meanwhile (${why})`);
|
|
159
|
+
void deps.ensureInProcessFallback();
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const opened = recordRecoveryFailure(now);
|
|
163
|
+
if (opened) {
|
|
164
|
+
void deps.ensureInProcessFallback();
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
const delay = computeRecoveryBackoffMs(backoffAttempt - 1);
|
|
168
|
+
logWarn(`watchdog recovery (${why}) — backing off ${Math.round(delay)}ms before respawn`);
|
|
169
|
+
deps.stopHeartbeatInternal();
|
|
170
|
+
deps.resetHeartbeatPort();
|
|
171
|
+
const { promise: backoffDone, resolve: resolveBackoff } = Promise.withResolvers<void>();
|
|
172
|
+
const backoffTimer = setTimeout(resolveBackoff, delay);
|
|
173
|
+
try { backoffTimer.unref(); } catch { }
|
|
174
|
+
await backoffDone;
|
|
175
|
+
try {
|
|
176
|
+
const health = deps.getDaemonHealth ? await deps.getDaemonHealth(port) : null;
|
|
177
|
+
if (health && (health.activeRequests ?? 0) > 0) {
|
|
178
|
+
trackBusyEdge(health.activeRequests ?? 0, health.lastBytesAt ?? 0, Date.now());
|
|
179
|
+
if (!isStuckBusy(busySince, health.lastBytesAt ?? 0, Date.now())) {
|
|
180
|
+
logInfo(`watchdog recovery deferred — daemon busy (${health.activeRequests} active, waiting for 5min-stuck + 60s-quiet)`);
|
|
181
|
+
await deps.attachTo(port);
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
const ver = health?.version ?? PKG_VERSION;
|
|
186
|
+
await deps.killStaleDaemon(port, ver, "proxy daemon");
|
|
187
|
+
} catch (e) {
|
|
188
|
+
logWarn("watchdog pre-respawn probe failed", { error: String(e) });
|
|
189
|
+
}
|
|
190
|
+
// Release before delegating: ensureDaemon owns the flag from here (it
|
|
191
|
+
// early-returns while ensuring is true). The handoff is synchronous up
|
|
192
|
+
// to ensureDaemon's own ensuring=true, so no tick slips between.
|
|
193
|
+
deps.setEnsuring(false);
|
|
194
|
+
await deps.ensureDaemon();
|
|
195
|
+
} finally {
|
|
196
|
+
deps.setEnsuring(false);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* 5s watchdog tick: polls /_health for version + failed-SSE rate and drives
|
|
202
|
+
* stopHeartbeat → busy-ruled kill → ensureDaemon when the daemon is stale or
|
|
203
|
+
* degraded. Healthy ticks reset backoff/breaker and track the busy edge.
|
|
204
|
+
*/
|
|
205
|
+
export async function watchdogCheck(port: number, deps: RecoveryDeps): Promise<void> {
|
|
206
|
+
if (deps.isEnsuring()) return;
|
|
207
|
+
if (isBreakerHalted()) return;
|
|
208
|
+
if (!deps.getDaemonHealth) return;
|
|
209
|
+
let health: DaemonHealthSnapshot;
|
|
210
|
+
try {
|
|
211
|
+
health = await deps.getDaemonHealth(port);
|
|
212
|
+
} catch {
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (health === null) {
|
|
216
|
+
await new Promise<void>((r) => setTimeout(r, 200));
|
|
217
|
+
try {
|
|
218
|
+
health = await deps.getDaemonHealth(port);
|
|
219
|
+
} catch {
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
if (health && (health.activeRequests ?? 0) > 0) {
|
|
224
|
+
trackBusyEdge(health.activeRequests ?? 0, health.lastBytesAt ?? 0);
|
|
225
|
+
} else {
|
|
226
|
+
trackBusyEdge(0, 0);
|
|
227
|
+
}
|
|
228
|
+
if (!shouldRecoverOnHealth(health)) {
|
|
229
|
+
recordRecoverySuccess();
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
if (health && (health.activeRequests ?? 0) > 0 && !isStuckBusy(busySince, health.lastBytesAt ?? 0)) return;
|
|
233
|
+
void triggerRecovery(
|
|
234
|
+
port,
|
|
235
|
+
health === null ? "daemon gone" : isOlderDaemonVersion(health.version) ? `stale v${health.version}` : "failed-SSE degraded",
|
|
236
|
+
deps,
|
|
237
|
+
).catch((e) => logWarn("watchdog recovery failed", { error: String(e) }));
|
|
238
|
+
}
|