pi-freeflow 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -86
- package/package.json +2 -3
- package/src/catalog.ts +21 -12
- package/src/commands.ts +308 -146
- package/src/config.ts +24 -2
- package/src/deploy.ts +128 -185
- package/src/health.ts +2 -7
- package/src/index.ts +22 -4
- package/src/logger.ts +52 -24
- package/src/models.ts +139 -67
- package/src/probe.ts +49 -0
- package/src/proxy.ts +150 -57
- package/src/relay-state.ts +380 -69
- package/src/relay.ts +72 -26
- package/src/stream-pipe.ts +307 -271
- package/src/types.ts +4 -2
- package/src/update-checker.ts +21 -5
package/src/relay-state.ts
CHANGED
|
@@ -7,68 +7,205 @@
|
|
|
7
7
|
import { randomUUID } from "node:crypto";
|
|
8
8
|
import fs from "node:fs";
|
|
9
9
|
import path from "node:path";
|
|
10
|
-
import {
|
|
10
|
+
import { RELAY_STATE_FILE } from "./config.ts";
|
|
11
|
+
import { ALLOW_UNSAFE_RELAY_ENV } from "./config.ts";
|
|
11
12
|
import { logWarn } from "./logger.ts";
|
|
12
13
|
import type { ExtensionUIContext, KnownRelay, RelayMode, RelayState } from "./types.ts";
|
|
14
|
+
|
|
13
15
|
/**
|
|
14
|
-
*
|
|
16
|
+
* Reject relay hostnames the deployed worker templates also refuse: loopback,
|
|
17
|
+
* RFC1918, CGNAT, link-local, ULA, and *.local/*.internal names. Mirrors the
|
|
18
|
+
* isPrivateHostname discipline embedded in the Vercel/Cloudflare/Deno templates.
|
|
15
19
|
*/
|
|
16
|
-
|
|
20
|
+
function isPrivateRelayHostname(hostname: string): boolean {
|
|
21
|
+
let host = hostname.trim().toLowerCase().replace(/^\[|\]$/g, "");
|
|
22
|
+
if (host.length > 1 && host.endsWith(".")) host = host.slice(0, -1);
|
|
23
|
+
if (!host) return true;
|
|
24
|
+
if (
|
|
25
|
+
host === "localhost" ||
|
|
26
|
+
host === "0.0.0.0" ||
|
|
27
|
+
host === "127.0.0.1" ||
|
|
28
|
+
host.endsWith(".localhost") ||
|
|
29
|
+
host.endsWith(".local") ||
|
|
30
|
+
host.endsWith(".internal")
|
|
31
|
+
) {
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
// IPv6 loopback / unspecified / any ::-prefixed form
|
|
35
|
+
if (host.startsWith("::")) return true;
|
|
36
|
+
const v4 = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
37
|
+
if (v4) {
|
|
38
|
+
const a = Number(v4[1]);
|
|
39
|
+
const b = Number(v4[2]);
|
|
40
|
+
if (a === 0 || a === 10 || a === 127) return true;
|
|
41
|
+
if (a === 169 && b === 254) return true;
|
|
42
|
+
if (a === 192 && b === 168) return true;
|
|
43
|
+
if (a === 172 && b >= 16 && b <= 31) return true;
|
|
44
|
+
if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT 100.64/10
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
if (host.includes(":")) {
|
|
48
|
+
if (host.startsWith("fc") || host.startsWith("fd")) return true; // ULA fc00::/7
|
|
49
|
+
if (/^fe[89ab]/.test(host)) return true; // link-local fe80::/10
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Validate a relay URL the way the deployed worker templates validate their
|
|
57
|
+
* x-relay-target: https-only (unless ALLOW_UNSAFE_RELAY_ENV=1), no embedded
|
|
58
|
+
* credentials, public hostname only, default port only.
|
|
59
|
+
*
|
|
60
|
+
* Returns { ok: true, url } or { ok: false, reason } — never throws.
|
|
61
|
+
*/
|
|
62
|
+
export function validateRelayUrl(
|
|
63
|
+
raw: string,
|
|
64
|
+
): { ok: true; url: URL } | { ok: false; reason: string } {
|
|
65
|
+
const unsafe = process.env[ALLOW_UNSAFE_RELAY_ENV] === "1";
|
|
66
|
+
const trimmed = (raw || "").trim();
|
|
67
|
+
if (!trimmed) return { ok: false, reason: "Relay URL cannot be empty" };
|
|
68
|
+
let url: URL;
|
|
17
69
|
try {
|
|
18
|
-
|
|
19
|
-
|
|
70
|
+
url = new URL(trimmed);
|
|
71
|
+
} catch {
|
|
72
|
+
return { ok: false, reason: "invalid URL" };
|
|
73
|
+
}
|
|
74
|
+
if (!unsafe) {
|
|
75
|
+
if (url.protocol !== "https:") {
|
|
76
|
+
return { ok: false, reason: "only https relay URLs are allowed" };
|
|
20
77
|
}
|
|
21
|
-
|
|
78
|
+
} else if (url.protocol !== "https:" && url.protocol !== "http:") {
|
|
79
|
+
return { ok: false, reason: "only http/https relay URLs are allowed" };
|
|
80
|
+
}
|
|
81
|
+
if (url.username || url.password) {
|
|
82
|
+
return { ok: false, reason: "credentials are not allowed in relay URLs" };
|
|
83
|
+
}
|
|
84
|
+
if (!unsafe) {
|
|
85
|
+
if (isPrivateRelayHostname(url.hostname)) {
|
|
86
|
+
return { ok: false, reason: "private/loopback relay hosts are not allowed" };
|
|
87
|
+
}
|
|
88
|
+
if (url.port && url.port !== "443") {
|
|
89
|
+
return { ok: false, reason: "non-default relay ports are not allowed" };
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return { ok: true, url };
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Parse a relay-state JSON blob into a usable state.
|
|
96
|
+
* Returns null when the blob is corrupt or has an empty relay pool —
|
|
97
|
+
* either makes the state unusable for routing.
|
|
98
|
+
*/
|
|
99
|
+
function parseRelayState(raw: string): RelayState | null {
|
|
100
|
+
try {
|
|
101
|
+
const s = JSON.parse(raw);
|
|
22
102
|
const relays: KnownRelay[] = Array.isArray(s?.relays) ? s.relays : [];
|
|
103
|
+
if (relays.length === 0) {
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
23
106
|
const mode: RelayMode =
|
|
24
107
|
s?.mode === "on" || s?.mode === "off" || s?.mode === "auto"
|
|
25
108
|
? s.mode
|
|
26
109
|
: "auto";
|
|
27
|
-
const enabled =
|
|
28
|
-
mode === "on"
|
|
29
|
-
? true
|
|
30
|
-
: mode === "off"
|
|
31
|
-
? false
|
|
32
|
-
: s?.enabled !== false && relays.length > 0;
|
|
33
110
|
return {
|
|
34
111
|
mode,
|
|
35
|
-
enabled
|
|
36
|
-
|
|
112
|
+
enabled:
|
|
113
|
+
mode === "on"
|
|
114
|
+
? true
|
|
115
|
+
: mode === "off"
|
|
116
|
+
? false
|
|
117
|
+
: s?.enabled !== false && relays.length > 0,
|
|
118
|
+
url: typeof s?.url === "string" ? s.url.trim() : (relays[0]?.url || ""),
|
|
37
119
|
relays,
|
|
38
120
|
};
|
|
39
121
|
} catch {
|
|
40
|
-
return
|
|
122
|
+
return null;
|
|
41
123
|
}
|
|
42
124
|
}
|
|
43
125
|
|
|
44
|
-
/**
|
|
45
|
-
|
|
46
|
-
|
|
126
|
+
/**
|
|
127
|
+
* Remove leftover `<state>.<uuid>.tmp` files from crashed or failed saves.
|
|
128
|
+
* Only files older than an hour are removed, so an in-flight save by a
|
|
129
|
+
* sibling process is never disturbed.
|
|
130
|
+
*/
|
|
131
|
+
function cleanupStaleTmpFiles(): void {
|
|
132
|
+
try {
|
|
133
|
+
const dir = path.dirname(RELAY_STATE_FILE);
|
|
134
|
+
const prefix = `${path.basename(RELAY_STATE_FILE)}.`;
|
|
135
|
+
const cutoff = Date.now() - 60 * 60 * 1000;
|
|
136
|
+
for (const entry of fs.readdirSync(dir)) {
|
|
137
|
+
if (!entry.startsWith(prefix) || !entry.endsWith(".tmp")) {
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
const full = path.join(dir, entry);
|
|
141
|
+
try {
|
|
142
|
+
if (fs.statSync(full).mtimeMs < cutoff) {
|
|
143
|
+
fs.rmSync(full, { force: true });
|
|
144
|
+
}
|
|
145
|
+
} catch {}
|
|
146
|
+
}
|
|
147
|
+
} catch {}
|
|
47
148
|
}
|
|
48
149
|
|
|
49
150
|
/**
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
* Three attempts with a fixed 50ms backoff; any other error fails immediately.
|
|
151
|
+
* Recover relay state from the .bak file and heal the main file so the
|
|
152
|
+
* recovery is sticky. Returns null when no usable backup exists.
|
|
53
153
|
*/
|
|
54
|
-
function
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
154
|
+
function recoverFromBackup(description: string): RelayState | null {
|
|
155
|
+
try {
|
|
156
|
+
const bak = parseRelayState(fs.readFileSync(`${RELAY_STATE_FILE}.bak`, "utf8"));
|
|
157
|
+
if (bak) {
|
|
158
|
+
logWarn(`relay state ${description} — recovered from .bak`, { relays: bak.relays.length });
|
|
159
|
+
saveRelayState(bak); // heal the main file so recovery is sticky
|
|
160
|
+
return bak;
|
|
161
|
+
}
|
|
162
|
+
} catch {}
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Load persisted relay state from disk.
|
|
168
|
+
* Falls back to .bak when the main file is corrupt OR empty — an empty save
|
|
169
|
+
* over real relays leaves exactly a valid-but-empty main, and the backup is
|
|
170
|
+
* the only surviving copy (see tmp 50-byte litter wipe). A successful
|
|
171
|
+
* recovery is written back to the main file so it is not re-done on every
|
|
172
|
+
* load.
|
|
173
|
+
*/
|
|
174
|
+
export function loadRelayState(): RelayState {
|
|
175
|
+
try {
|
|
176
|
+
cleanupStaleTmpFiles();
|
|
177
|
+
if (!fs.existsSync(RELAY_STATE_FILE)) {
|
|
178
|
+
// Main file missing (deleted outright, never healed after a wipe):
|
|
179
|
+
// heal from .bak when the backup still holds relays — losing the
|
|
180
|
+
// pool to a missing main is just as bad as a corrupt one.
|
|
181
|
+
const healed = recoverFromBackup("main state file missing");
|
|
182
|
+
if (healed) {
|
|
183
|
+
return healed;
|
|
64
184
|
}
|
|
65
|
-
|
|
185
|
+
return { mode: "auto", enabled: true, url: "", relays: [] };
|
|
66
186
|
}
|
|
187
|
+
const main = parseRelayState(fs.readFileSync(RELAY_STATE_FILE, "utf8"));
|
|
188
|
+
if (main) {
|
|
189
|
+
return main;
|
|
190
|
+
}
|
|
191
|
+
// Try backup before giving up — user data loss is worse than stale data
|
|
192
|
+
const bak = recoverFromBackup("unusable");
|
|
193
|
+
if (bak) {
|
|
194
|
+
return bak;
|
|
195
|
+
}
|
|
196
|
+
logWarn("relay state file unusable and no valid backup — starting fresh", {
|
|
197
|
+
path: RELAY_STATE_FILE,
|
|
198
|
+
});
|
|
199
|
+
return { mode: "auto", enabled: true, url: "", relays: [] };
|
|
200
|
+
} catch {
|
|
201
|
+
return { mode: "auto", enabled: true, url: "", relays: [] };
|
|
67
202
|
}
|
|
68
203
|
}
|
|
69
204
|
|
|
70
205
|
/**
|
|
71
206
|
* Atomically save relay state to disk using a temporary file and rename.
|
|
207
|
+
* Backs up the current non-empty state to .bak before overwriting so a
|
|
208
|
+
* corrupt or empty save can always be recovered by loadRelayState.
|
|
72
209
|
*/
|
|
73
210
|
export function saveRelayState(s: RelayState): void {
|
|
74
211
|
try {
|
|
@@ -76,6 +213,17 @@ export function saveRelayState(s: RelayState): void {
|
|
|
76
213
|
if (!fs.existsSync(dir)) {
|
|
77
214
|
fs.mkdirSync(dir, { recursive: true });
|
|
78
215
|
}
|
|
216
|
+
// Backup current file before overwrite for recovery
|
|
217
|
+
try {
|
|
218
|
+
if (fs.existsSync(RELAY_STATE_FILE)) {
|
|
219
|
+
const cur = fs.readFileSync(RELAY_STATE_FILE, "utf8");
|
|
220
|
+
// Only backup non-empty states
|
|
221
|
+
const parsed = JSON.parse(cur);
|
|
222
|
+
if (Array.isArray(parsed?.relays) && parsed.relays.length > 0) {
|
|
223
|
+
fs.writeFileSync(`${RELAY_STATE_FILE}.bak`, cur, "utf8");
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
} catch {}
|
|
79
227
|
const tmpPath = `${RELAY_STATE_FILE}.${randomUUID()}.tmp`;
|
|
80
228
|
fs.writeFileSync(tmpPath, JSON.stringify(s, null, 2), "utf8");
|
|
81
229
|
renameWithRetry(tmpPath, RELAY_STATE_FILE);
|
|
@@ -84,6 +232,32 @@ export function saveRelayState(s: RelayState): void {
|
|
|
84
232
|
}
|
|
85
233
|
}
|
|
86
234
|
|
|
235
|
+
/** Block synchronously for ms without burning CPU. */
|
|
236
|
+
function sleepSync(ms: number): void {
|
|
237
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, Date.now() + ms);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Rename with bounded retry: Windows briefly locks the target when another
|
|
242
|
+
* process holds the state file open, surfacing as EPERM/EACCES/EBUSY.
|
|
243
|
+
* Three attempts with a fixed 50ms backoff; any other error fails immediately.
|
|
244
|
+
*/
|
|
245
|
+
function renameWithRetry(from: string, to: string): void {
|
|
246
|
+
const retryable: Record<string, true> = { EPERM: true, EACCES: true, EBUSY: true };
|
|
247
|
+
for (let attempt = 1; ; attempt++) {
|
|
248
|
+
try {
|
|
249
|
+
fs.renameSync(from, to);
|
|
250
|
+
return;
|
|
251
|
+
} catch (e) {
|
|
252
|
+
const code = (e as NodeJS.ErrnoException | null)?.code;
|
|
253
|
+
if (!code || !retryable[code] || attempt >= 3) {
|
|
254
|
+
throw e;
|
|
255
|
+
}
|
|
256
|
+
sleepSync(50);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
87
261
|
/**
|
|
88
262
|
* Deduplicate and add or update a relay URL in the known relay list.
|
|
89
263
|
*/
|
|
@@ -96,6 +270,10 @@ export function ensureRelay(
|
|
|
96
270
|
if (!cleanUrl) {
|
|
97
271
|
throw new Error("Relay URL cannot be empty");
|
|
98
272
|
}
|
|
273
|
+
const check = validateRelayUrl(cleanUrl);
|
|
274
|
+
if (!check.ok) {
|
|
275
|
+
throw new Error(`Relay URL rejected: ${check.reason}`);
|
|
276
|
+
}
|
|
99
277
|
const cleanLabel = (label || "").trim() || undefined;
|
|
100
278
|
const existing = s.relays.find((r) => r.url === cleanUrl);
|
|
101
279
|
if (existing) {
|
|
@@ -129,14 +307,15 @@ export function setRelayLabel(
|
|
|
129
307
|
}
|
|
130
308
|
|
|
131
309
|
/**
|
|
132
|
-
* Find a relay in state by 1-based index
|
|
310
|
+
* Find a relay in state by 1-based index (numeric 0 also maps to the first relay),
|
|
311
|
+
* short name / label, or URL.
|
|
133
312
|
*/
|
|
134
313
|
export function findRelay(
|
|
135
314
|
s: RelayState,
|
|
136
315
|
identifier: string | number,
|
|
137
316
|
): KnownRelay | undefined {
|
|
138
317
|
if (typeof identifier === "number") {
|
|
139
|
-
const idx = identifier - 1;
|
|
318
|
+
const idx = identifier === 0 ? 0 : identifier - 1;
|
|
140
319
|
return s.relays[idx];
|
|
141
320
|
}
|
|
142
321
|
const str = String(identifier || "").trim();
|
|
@@ -172,23 +351,12 @@ export function removeRelay(s: RelayState, url: string): void {
|
|
|
172
351
|
}
|
|
173
352
|
|
|
174
353
|
/**
|
|
175
|
-
* Resolve relay state with defaults.
|
|
354
|
+
* Resolve relay state with defaults. The default relay URL is empty in this
|
|
355
|
+
* build (fresh installs start in direct mode), so the former seeding branch
|
|
356
|
+
* was dead code and is removed; loadRelayState already returns usable state.
|
|
176
357
|
*/
|
|
177
358
|
export function resolveRelayState(): RelayState {
|
|
178
|
-
|
|
179
|
-
// Seed the relay list from defaults when state lacks entries.
|
|
180
|
-
if (!s.relays.length) {
|
|
181
|
-
if (DEFAULT_RELAY_URL) {
|
|
182
|
-
ensureRelay(s, DEFAULT_RELAY_URL, "Default");
|
|
183
|
-
}
|
|
184
|
-
if (s.url && s.url !== DEFAULT_RELAY_URL) {
|
|
185
|
-
ensureRelay(s, s.url, "previous");
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
if (!s.url) {
|
|
189
|
-
s.url = DEFAULT_RELAY_URL;
|
|
190
|
-
}
|
|
191
|
-
return s;
|
|
359
|
+
return loadRelayState();
|
|
192
360
|
}
|
|
193
361
|
|
|
194
362
|
let activeRelayState: RelayState = resolveRelayState();
|
|
@@ -201,17 +369,41 @@ export interface RelayHealth {
|
|
|
201
369
|
cooldownUntil: number;
|
|
202
370
|
lastStatus?: number;
|
|
203
371
|
lastError?: string;
|
|
372
|
+
/** most recent successful response latency in ms */
|
|
373
|
+
lastLatencyMs?: number;
|
|
374
|
+
/** total successful responses recorded while keeping the health record */
|
|
375
|
+
successCount?: number;
|
|
376
|
+
/** total failures recorded since the health record was created */
|
|
377
|
+
failureCount?: number;
|
|
378
|
+
/** timestamp of the most recent HTTP 429 (if any) */
|
|
379
|
+
last429At?: number;
|
|
204
380
|
}
|
|
205
381
|
|
|
206
382
|
const relayHealthMap = new Map<string, RelayHealth>();
|
|
207
383
|
let last429Warn = 0;
|
|
384
|
+
// Sliding 60-second window of HTTP 429 events used by markRelayFailure to
|
|
385
|
+
// keep the "burst >5/min" warning honest (see markRelayFailure).
|
|
386
|
+
const RELAY_429_BURST_WINDOW_MS = 60_000;
|
|
387
|
+
const RELAY_429_BURST_THRESHOLD = 5;
|
|
388
|
+
const recent429Timestamps: number[] = [];
|
|
208
389
|
|
|
209
390
|
/**
|
|
210
391
|
* Mark a relay as healthy and active on successful response.
|
|
392
|
+
* With a finite latencyMs the health record is kept (failures reset, latency
|
|
393
|
+
* recorded); without one the record is deleted entirely.
|
|
211
394
|
*/
|
|
212
|
-
export function markRelaySuccess(url: string): void {
|
|
395
|
+
export function markRelaySuccess(url: string, latencyMs?: number): void {
|
|
213
396
|
if (!url) return;
|
|
214
|
-
|
|
397
|
+
const clean = url.trim();
|
|
398
|
+
if (typeof latencyMs === "number" && Number.isFinite(latencyMs)) {
|
|
399
|
+
const prev = relayHealthMap.get(clean);
|
|
400
|
+
const record: RelayHealth = prev
|
|
401
|
+
? { ...prev, consecutiveFailures: 0, lastFailureTime: 0, cooldownUntil: 0, lastLatencyMs: Math.round(latencyMs), successCount: (prev?.successCount ?? 0) + 1 }
|
|
402
|
+
: { consecutiveFailures: 0, lastFailureTime: 0, cooldownUntil: 0, lastLatencyMs: Math.round(latencyMs) };
|
|
403
|
+
relayHealthMap.set(clean, record);
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
relayHealthMap.delete(clean);
|
|
215
407
|
}
|
|
216
408
|
|
|
217
409
|
/**
|
|
@@ -231,7 +423,18 @@ export function markRelayFailure(url: string, status?: number, error?: string):
|
|
|
231
423
|
|
|
232
424
|
if (status === 429) {
|
|
233
425
|
cooldownMs = 90_000;
|
|
234
|
-
|
|
426
|
+
// 60-second sliding window: record the event, then warn only when the
|
|
427
|
+
// observed rate crosses >5/min (transition into a burst episode).
|
|
428
|
+
// One warning per episode beats both spamming (old 10-min cooldown
|
|
429
|
+
// re-warned on every boundary during a long burst) and blindness (a
|
|
430
|
+
// fresh burst inside the cooldown stayed silent).
|
|
431
|
+
const cutoff = now - RELAY_429_BURST_WINDOW_MS;
|
|
432
|
+
while (recent429Timestamps.length > 0 && recent429Timestamps[0] <= cutoff) {
|
|
433
|
+
recent429Timestamps.shift();
|
|
434
|
+
}
|
|
435
|
+
const beforeBurst = recent429Timestamps.length;
|
|
436
|
+
recent429Timestamps.push(now);
|
|
437
|
+
if (beforeBurst <= RELAY_429_BURST_THRESHOLD && recent429Timestamps.length > RELAY_429_BURST_THRESHOLD) {
|
|
235
438
|
logWarn("relay 429 burst >5/min, consider adding egress", { relay: clean });
|
|
236
439
|
last429Warn = now;
|
|
237
440
|
}
|
|
@@ -245,11 +448,14 @@ export function markRelayFailure(url: string, status?: number, error?: string):
|
|
|
245
448
|
// 4x their base cooldown instead of re-hammering at a fixed interval.
|
|
246
449
|
const multiplier = Math.min(4, consecutive);
|
|
247
450
|
relayHealthMap.set(clean, {
|
|
451
|
+
...prev,
|
|
248
452
|
consecutiveFailures: consecutive,
|
|
249
453
|
lastFailureTime: now,
|
|
250
454
|
cooldownUntil: now + cooldownMs * multiplier,
|
|
251
455
|
lastStatus: status,
|
|
252
456
|
lastError: error,
|
|
457
|
+
failureCount: (prev.failureCount ?? 0) + 1,
|
|
458
|
+
last429At: status === 429 ? now : prev.last429At || undefined,
|
|
253
459
|
});
|
|
254
460
|
}
|
|
255
461
|
|
|
@@ -276,10 +482,14 @@ export function getRelayHealth(url: string): RelayHealth | undefined {
|
|
|
276
482
|
*/
|
|
277
483
|
export function resetAllRelayHealth(): void {
|
|
278
484
|
relayHealthMap.clear();
|
|
485
|
+
recent429Timestamps.length = 0;
|
|
279
486
|
last429Warn = 0;
|
|
280
487
|
}
|
|
281
488
|
/** Test-only: reset 429 warn throttle */
|
|
282
|
-
export function _reset429WarnForTest(): void {
|
|
489
|
+
export function _reset429WarnForTest(): void {
|
|
490
|
+
recent429Timestamps.length = 0;
|
|
491
|
+
last429Warn = 0;
|
|
492
|
+
}
|
|
283
493
|
/**
|
|
284
494
|
* Mtime of the on-disk state file at the moment we last read or wrote it.
|
|
285
495
|
* session's master daemon, while never clobbering this process's own
|
|
@@ -320,6 +530,80 @@ export function setActiveRelayState(s: RelayState, persist = true): void {
|
|
|
320
530
|
lastKnownStateMtimeMs = currentDiskStateMtimeMs();
|
|
321
531
|
}
|
|
322
532
|
|
|
533
|
+
// ── CAS state mutation ──────────────────────────────────────────────
|
|
534
|
+
// Command handlers mutate relay state through a compare-and-swap discipline
|
|
535
|
+
// (withRelayState) instead of editing a stale in-memory snapshot: the
|
|
536
|
+
// operation is re-applied to the freshest on-disk state at write time, so
|
|
537
|
+
// concurrent sessions cannot silently clobber each other's edits.
|
|
538
|
+
|
|
539
|
+
const RELAY_STATE_LOCK_FILE = `${RELAY_STATE_FILE}.lock`;
|
|
540
|
+
const LOCK_WAIT_MAX_MS = 2_000;
|
|
541
|
+
const LOCK_RETRY_START_MS = 25;
|
|
542
|
+
const LOCK_RETRY_MAX_MS = 200;
|
|
543
|
+
/** A lock older than this is presumed abandoned (crashed holder) and taken over. */
|
|
544
|
+
const LOCK_STALE_MS = 30_000;
|
|
545
|
+
|
|
546
|
+
function tryAcquireRelayLock(): boolean {
|
|
547
|
+
try {
|
|
548
|
+
fs.writeFileSync(RELAY_STATE_LOCK_FILE, String(process.pid), { flag: "wx" });
|
|
549
|
+
return true;
|
|
550
|
+
} catch (e) {
|
|
551
|
+
const code = (e as NodeJS.ErrnoException | null)?.code;
|
|
552
|
+
if (code !== "EEXIST") {
|
|
553
|
+
// Lock cannot be created for an unrelated reason (permissions etc.) —
|
|
554
|
+
// proceed unlocked; the atomic tmp+rename save already prevents torn files.
|
|
555
|
+
return false;
|
|
556
|
+
}
|
|
557
|
+
// Held or stale: take over locks older than LOCK_STALE_MS so a crashed
|
|
558
|
+
// process cannot wedge the state file forever.
|
|
559
|
+
try {
|
|
560
|
+
const age = Date.now() - fs.statSync(RELAY_STATE_LOCK_FILE).mtimeMs;
|
|
561
|
+
if (age > LOCK_STALE_MS) {
|
|
562
|
+
fs.rmSync(RELAY_STATE_LOCK_FILE, { force: true });
|
|
563
|
+
return tryAcquireRelayLock();
|
|
564
|
+
}
|
|
565
|
+
} catch {
|
|
566
|
+
return tryAcquireRelayLock(); // lock vanished between stat and here
|
|
567
|
+
}
|
|
568
|
+
return false;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* Apply a mutation to relay state with a compare-and-swap discipline.
|
|
574
|
+
* Locks the state file (bounded retry, ~2s), loads the freshest on-disk state,
|
|
575
|
+
* runs the updater against it, persists atomically, and releases the lock.
|
|
576
|
+
* On lock timeout the write proceeds unlocked — the atomic tmp+rename save
|
|
577
|
+
* already prevents torn files; the lock only serializes write intent.
|
|
578
|
+
* Returns the state that was actually persisted (also becomes the in-memory
|
|
579
|
+
* active state).
|
|
580
|
+
*/
|
|
581
|
+
export function withRelayState(updater: (s: RelayState) => RelayState): RelayState {
|
|
582
|
+
const deadline = Date.now() + LOCK_WAIT_MAX_MS;
|
|
583
|
+
let locked = tryAcquireRelayLock();
|
|
584
|
+
let delay = LOCK_RETRY_START_MS;
|
|
585
|
+
while (!locked && Date.now() < deadline) {
|
|
586
|
+
sleepSync(delay);
|
|
587
|
+
locked = tryAcquireRelayLock();
|
|
588
|
+
delay = Math.min(delay * 2, LOCK_RETRY_MAX_MS);
|
|
589
|
+
}
|
|
590
|
+
if (!locked) {
|
|
591
|
+
logWarn("relay state lock held — proceeding without lock after timeout", {
|
|
592
|
+
lock: RELAY_STATE_LOCK_FILE,
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
try {
|
|
596
|
+
const next = updater(loadRelayState());
|
|
597
|
+
saveRelayState(next);
|
|
598
|
+
setActiveRelayState(next, false);
|
|
599
|
+
return next;
|
|
600
|
+
} finally {
|
|
601
|
+
if (locked) {
|
|
602
|
+
try { fs.rmSync(RELAY_STATE_LOCK_FILE, { force: true }); } catch {}
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
|
|
323
607
|
/**
|
|
324
608
|
* Register the Pi extension UI context for status line updates
|
|
325
609
|
*/
|
|
@@ -411,8 +695,48 @@ export function getOrderedRelayUrls(): string[] {
|
|
|
411
695
|
}
|
|
412
696
|
|
|
413
697
|
/**
|
|
414
|
-
*
|
|
698
|
+
* Shared status-widget label: the active relay line shown in the host status
|
|
699
|
+
* bar. Returns null when the widget should be cleared (hidden, direct mode,
|
|
700
|
+
* or an empty pool with no explicit OFF mode).
|
|
415
701
|
*/
|
|
702
|
+
export function formatRelayStatusLabel(state: RelayState, targetUrl?: string): string | null {
|
|
703
|
+
if (!state || state.hideWidget) {
|
|
704
|
+
return null;
|
|
705
|
+
}
|
|
706
|
+
if (!state.enabled || !state.relays || state.relays.length === 0) {
|
|
707
|
+
return state.mode === "off" ? "relay: OFF (direct)" : null;
|
|
708
|
+
}
|
|
709
|
+
const currentUrl = targetUrl || state.url || state.relays[0]?.url || "";
|
|
710
|
+
const label = shortRelayLabel(currentUrl, state.relays);
|
|
711
|
+
const total = state.relays.length || 1;
|
|
712
|
+
const pos = Math.max(1, state.relays.findIndex((r) => r.url === currentUrl) + 1);
|
|
713
|
+
const modeLabel = state.mode === "on" ? "ON" : "AUTO (ON)";
|
|
714
|
+
return `relay: ${modeLabel} | ${label} ${pos}/${total}`;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* Shared relay-picker item text ("[1] [label] → url" with an active marker).
|
|
719
|
+
*/
|
|
720
|
+
export function formatRelayPickerItem(r: KnownRelay, idx: number, activeUrl?: string): string {
|
|
721
|
+
const isActive = typeof activeUrl === "string" && r.url === activeUrl;
|
|
722
|
+
const lbl = r.label ? `[${r.label}] ` : `[${shortRelayLabel(r.url)}] `;
|
|
723
|
+
return `${isActive ? "★ " : ""}[${idx + 1}] ${lbl}→ ${r.url}`;
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* Shared /freeflow flash notification text for the current relay mode/pool state.
|
|
728
|
+
*/
|
|
729
|
+
export function formatRelayFlash(state: RelayState): string {
|
|
730
|
+
const activeLabel = shortRelayLabel(state.url, state.relays);
|
|
731
|
+
const activeIdx = Math.max(
|
|
732
|
+
1,
|
|
733
|
+
state.relays.findIndex((r) => r.url === state.url) + 1,
|
|
734
|
+
);
|
|
735
|
+
const total = state.relays.length || 1;
|
|
736
|
+
const modeStr = (state.mode || "auto").toUpperCase();
|
|
737
|
+
return `Relay mode: ${modeStr} (${state.enabled ? "ON" : "OFF"})${state.enabled ? ` → ${activeLabel} (${activeIdx}/${total})` : " (direct)"} | saved=${state.relays.length} (auto-fallback rolling)`;
|
|
738
|
+
}
|
|
739
|
+
|
|
416
740
|
export function updateRelayStatusUi(targetUrl?: string): void {
|
|
417
741
|
if (!activeStatusUi?.setStatus) {
|
|
418
742
|
return;
|
|
@@ -422,19 +746,6 @@ export function updateRelayStatusUi(targetUrl?: string): void {
|
|
|
422
746
|
activeStatusUi.setStatus("freeflow", undefined);
|
|
423
747
|
return;
|
|
424
748
|
}
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
return;
|
|
428
|
-
}
|
|
429
|
-
const state = getActiveRelayState();
|
|
430
|
-
if (!state.enabled || !state.relays || state.relays.length === 0) {
|
|
431
|
-
activeStatusUi.setStatus("freeflow", undefined);
|
|
432
|
-
return;
|
|
433
|
-
}
|
|
434
|
-
const currentUrl = targetUrl || state.url || state.relays[0]?.url || "";
|
|
435
|
-
const label = shortRelayLabel(currentUrl);
|
|
436
|
-
const total = state.relays.length || 1;
|
|
437
|
-
const pos = Math.max(1, state.relays.findIndex((r) => r.url === currentUrl) + 1);
|
|
438
|
-
const modeLabel = state.mode === "on" ? "ON" : "AUTO (ON)";
|
|
439
|
-
activeStatusUi.setStatus("freeflow", `relay: ${modeLabel} | ${label} ${pos}/${total}`);
|
|
749
|
+
const label = formatRelayStatusLabel(getActiveRelayState(), targetUrl);
|
|
750
|
+
activeStatusUi.setStatus("freeflow", label ?? undefined);
|
|
440
751
|
}
|