veryfront 0.1.1067 → 0.1.1069
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/deploy/command.d.ts +12 -1
- package/esm/cli/commands/deploy/command.d.ts.map +1 -1
- package/esm/cli/commands/deploy/command.js +28 -1
- package/esm/cli/commands/deploy/index.d.ts +2 -2
- package/esm/cli/commands/deploy/index.d.ts.map +1 -1
- package/esm/cli/commands/deploy/index.js +1 -1
- package/esm/cli/shared/config.d.ts.map +1 -1
- package/esm/cli/shared/config.js +3 -10
- package/esm/deno.js +1 -1
- package/esm/src/agent/runtime/index.d.ts.map +1 -1
- package/esm/src/agent/runtime/index.js +64 -38
- package/esm/src/proxy/handler.d.ts +19 -0
- package/esm/src/proxy/handler.d.ts.map +1 -1
- package/esm/src/proxy/handler.js +160 -13
- package/esm/src/proxy/main.js +45 -6
- package/esm/src/proxy/retry.d.ts +1 -0
- package/esm/src/proxy/retry.d.ts.map +1 -1
- package/esm/src/proxy/retry.js +31 -14
- package/esm/src/proxy/routing-invalidation-redis.d.ts +33 -0
- package/esm/src/proxy/routing-invalidation-redis.d.ts.map +1 -0
- package/esm/src/proxy/routing-invalidation-redis.js +394 -0
- package/esm/src/proxy/routing-invalidation.d.ts +32 -0
- package/esm/src/proxy/routing-invalidation.d.ts.map +1 -0
- package/esm/src/proxy/routing-invalidation.js +135 -0
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +5 -5
package/esm/src/proxy/handler.js
CHANGED
|
@@ -26,6 +26,7 @@ export const INTERNAL_PROXY_HEADERS = [
|
|
|
26
26
|
];
|
|
27
27
|
const DEFAULT_PROXY_ROUTING_CACHE_TTL_MS = 60_000;
|
|
28
28
|
const DEFAULT_PROXY_ROUTING_CACHE_MAX_ENTRIES = 1_000;
|
|
29
|
+
const MAX_ROUTING_LOOKUP_INVALIDATION_RETRIES = 2;
|
|
29
30
|
function readNonNegativeIntegerEnv(name, fallback) {
|
|
30
31
|
const raw = getEnv(name);
|
|
31
32
|
if (!raw)
|
|
@@ -46,6 +47,12 @@ class ProxyLookupAuthError extends Error {
|
|
|
46
47
|
this.name = "ProxyLookupAuthError";
|
|
47
48
|
}
|
|
48
49
|
}
|
|
50
|
+
class ProxyRoutingInvalidationRaceError extends Error {
|
|
51
|
+
constructor() {
|
|
52
|
+
super("Project routing changed during request; retry");
|
|
53
|
+
this.name = "ProxyRoutingInvalidationRaceError";
|
|
54
|
+
}
|
|
55
|
+
}
|
|
49
56
|
function isProxyLookupAuthError(error) {
|
|
50
57
|
return error instanceof ProxyLookupAuthError;
|
|
51
58
|
}
|
|
@@ -225,6 +232,11 @@ export function createProxyHandler(options) {
|
|
|
225
232
|
const routingCacheMaxEntries = readNonNegativeIntegerEnv("VERYFRONT_PROXY_ROUTING_CACHE_MAX_ENTRIES", DEFAULT_PROXY_ROUTING_CACHE_MAX_ENTRIES);
|
|
226
233
|
const routingLookupCache = new Map();
|
|
227
234
|
const routingLookupInflight = new Map();
|
|
235
|
+
const projectInvalidationGenerations = new Map();
|
|
236
|
+
const lookupKeyInvalidationGenerations = new Map();
|
|
237
|
+
const activeRoutingLookupGenerations = new Map();
|
|
238
|
+
const maxTrackedInvalidationGenerations = Math.max(routingCacheMaxEntries, DEFAULT_PROXY_ROUTING_CACHE_MAX_ENTRIES);
|
|
239
|
+
let routingLookupGeneration = 0;
|
|
228
240
|
async function resolveProjectLookup(lookupKey, token, timing) {
|
|
229
241
|
return await profileProxyServerTimingPhase(timing ?? { enabled: false, startedAt: 0, phases: new Map() }, "proxy.project_lookup", () => lookupProjectByDomain(lookupKey, config.apiBaseUrl, token, logger));
|
|
230
242
|
}
|
|
@@ -261,6 +273,71 @@ export function createProxyHandler(options) {
|
|
|
261
273
|
expiresAt: Date.now() + routingCacheTtlMs,
|
|
262
274
|
});
|
|
263
275
|
}
|
|
276
|
+
function pruneInvalidationGenerations(generations) {
|
|
277
|
+
const oldestActiveGeneration = activeRoutingLookupGenerations.size > 0
|
|
278
|
+
? Math.min(...activeRoutingLookupGenerations.keys())
|
|
279
|
+
: Number.POSITIVE_INFINITY;
|
|
280
|
+
while (generations.size > maxTrackedInvalidationGenerations) {
|
|
281
|
+
const oldestEntry = generations.entries().next().value;
|
|
282
|
+
if (!oldestEntry || oldestEntry[1] > oldestActiveGeneration)
|
|
283
|
+
break;
|
|
284
|
+
generations.delete(oldestEntry[0]);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
function rememberInvalidationGeneration(generations, key, generation) {
|
|
288
|
+
generations.delete(key);
|
|
289
|
+
generations.set(key, generation);
|
|
290
|
+
pruneInvalidationGenerations(generations);
|
|
291
|
+
}
|
|
292
|
+
function beginRoutingLookup(generation) {
|
|
293
|
+
activeRoutingLookupGenerations.set(generation, (activeRoutingLookupGenerations.get(generation) ?? 0) + 1);
|
|
294
|
+
}
|
|
295
|
+
function endRoutingLookup(generation) {
|
|
296
|
+
const activeCount = activeRoutingLookupGenerations.get(generation) ?? 0;
|
|
297
|
+
if (activeCount <= 1)
|
|
298
|
+
activeRoutingLookupGenerations.delete(generation);
|
|
299
|
+
else
|
|
300
|
+
activeRoutingLookupGenerations.set(generation, activeCount - 1);
|
|
301
|
+
pruneInvalidationGenerations(projectInvalidationGenerations);
|
|
302
|
+
pruneInvalidationGenerations(lookupKeyInvalidationGenerations);
|
|
303
|
+
}
|
|
304
|
+
function wasRoutingLookupInvalidated(cacheKey, result, startedAtGeneration) {
|
|
305
|
+
const keyGeneration = lookupKeyInvalidationGenerations.get(cacheKey) ?? 0;
|
|
306
|
+
if (keyGeneration > startedAtGeneration)
|
|
307
|
+
return true;
|
|
308
|
+
if (!result)
|
|
309
|
+
return false;
|
|
310
|
+
return (projectInvalidationGenerations.get(result.id) ?? 0) > startedAtGeneration;
|
|
311
|
+
}
|
|
312
|
+
function invalidateRoutingLookup(input) {
|
|
313
|
+
const generation = ++routingLookupGeneration;
|
|
314
|
+
rememberInvalidationGeneration(projectInvalidationGenerations, input.projectId, generation);
|
|
315
|
+
const normalizedProjectSlug = input.projectSlug
|
|
316
|
+
? normalizeProjectLookupKey(input.projectSlug)
|
|
317
|
+
: undefined;
|
|
318
|
+
if (normalizedProjectSlug) {
|
|
319
|
+
rememberInvalidationGeneration(lookupKeyInvalidationGenerations, normalizedProjectSlug, generation);
|
|
320
|
+
}
|
|
321
|
+
let evictedEntries = 0;
|
|
322
|
+
for (const [cacheKey, entry] of routingLookupCache) {
|
|
323
|
+
if (entry.value.id !== input.projectId && cacheKey !== normalizedProjectSlug)
|
|
324
|
+
continue;
|
|
325
|
+
routingLookupCache.delete(cacheKey);
|
|
326
|
+
rememberInvalidationGeneration(lookupKeyInvalidationGenerations, cacheKey, generation);
|
|
327
|
+
evictedEntries++;
|
|
328
|
+
}
|
|
329
|
+
logger?.info("Proxy routing metadata invalidated after deployment activation", {
|
|
330
|
+
projectId: input.projectId,
|
|
331
|
+
projectSlug: input.projectSlug,
|
|
332
|
+
deploymentId: input.deploymentId,
|
|
333
|
+
environmentId: input.environmentId,
|
|
334
|
+
environmentName: input.environmentName,
|
|
335
|
+
releaseId: input.releaseId,
|
|
336
|
+
generation,
|
|
337
|
+
evictedEntries,
|
|
338
|
+
});
|
|
339
|
+
return { evictedEntries, generation };
|
|
340
|
+
}
|
|
264
341
|
async function resolveProjectRoutingLookup(lookupKey, token, timing) {
|
|
265
342
|
const cacheKey = normalizeProjectLookupKey(lookupKey);
|
|
266
343
|
return await profileProxyServerTimingPhase(timing ?? { enabled: false, startedAt: 0, phases: new Map() }, "proxy.routing_lookup", async () => {
|
|
@@ -270,27 +347,89 @@ export function createProxyHandler(options) {
|
|
|
270
347
|
return cached;
|
|
271
348
|
}
|
|
272
349
|
const existingLookup = routingLookupInflight.get(cacheKey);
|
|
273
|
-
if (existingLookup) {
|
|
350
|
+
if (existingLookup?.generation === routingLookupGeneration) {
|
|
274
351
|
logger?.debug("Proxy routing metadata lookup joined in-flight request", { lookupKey });
|
|
275
|
-
return await existingLookup;
|
|
352
|
+
return await existingLookup.promise;
|
|
276
353
|
}
|
|
277
|
-
const lookupPromise =
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
354
|
+
const lookupPromise = (async () => {
|
|
355
|
+
for (let attempt = 0; attempt <= MAX_ROUTING_LOOKUP_INVALIDATION_RETRIES; attempt++) {
|
|
356
|
+
const startedAtGeneration = routingLookupGeneration;
|
|
357
|
+
beginRoutingLookup(startedAtGeneration);
|
|
358
|
+
try {
|
|
359
|
+
const result = await lookupProjectRoutingMetadata(lookupKey, config.apiBaseUrl, token, logger);
|
|
360
|
+
if (!wasRoutingLookupInvalidated(cacheKey, result, startedAtGeneration)) {
|
|
361
|
+
if (result)
|
|
362
|
+
setCachedRoutingLookup(cacheKey, result);
|
|
363
|
+
return result;
|
|
364
|
+
}
|
|
365
|
+
logger?.info("Retrying proxy routing metadata lookup after invalidation race", {
|
|
366
|
+
lookupKey,
|
|
367
|
+
attempt: attempt + 1,
|
|
368
|
+
generation: routingLookupGeneration,
|
|
369
|
+
});
|
|
370
|
+
if (attempt === MAX_ROUTING_LOOKUP_INVALIDATION_RETRIES) {
|
|
371
|
+
logger?.warn("Proxy routing metadata changed repeatedly during lookup; failing request closed", {
|
|
372
|
+
lookupKey,
|
|
373
|
+
attempts: attempt + 1,
|
|
374
|
+
});
|
|
375
|
+
throw new ProxyRoutingInvalidationRaceError();
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
finally {
|
|
379
|
+
endRoutingLookup(startedAtGeneration);
|
|
380
|
+
}
|
|
281
381
|
}
|
|
282
|
-
return
|
|
283
|
-
})
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
382
|
+
return null;
|
|
383
|
+
})();
|
|
384
|
+
const inflightEntry = {
|
|
385
|
+
generation: routingLookupGeneration,
|
|
386
|
+
promise: lookupPromise,
|
|
387
|
+
};
|
|
388
|
+
routingLookupInflight.set(cacheKey, inflightEntry);
|
|
389
|
+
try {
|
|
390
|
+
return await lookupPromise;
|
|
391
|
+
}
|
|
392
|
+
finally {
|
|
393
|
+
if (routingLookupInflight.get(cacheKey) === inflightEntry) {
|
|
394
|
+
routingLookupInflight.delete(cacheKey);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
289
397
|
});
|
|
290
398
|
}
|
|
291
399
|
async function resolveProjectAccessLookup(lookupKey, token, includeUsers, timing) {
|
|
292
400
|
return await profileProxyServerTimingPhase(timing ?? { enabled: false, startedAt: 0, phases: new Map() }, "proxy.access_lookup", () => lookupProjectAccessMetadata(lookupKey, config.apiBaseUrl, token, includeUsers, logger));
|
|
293
401
|
}
|
|
402
|
+
async function invalidateAndConfirmRoutingLookup(input) {
|
|
403
|
+
invalidateRoutingLookup(input);
|
|
404
|
+
const scope = getScope(input.environmentName.toLowerCase());
|
|
405
|
+
const resolveWithToken = async (token) => {
|
|
406
|
+
const result = await resolveProjectRoutingLookup(input.projectSlug, token);
|
|
407
|
+
const environment = result?.environments?.find((candidate) => candidate.id === input.environmentId);
|
|
408
|
+
if (result?.id !== input.projectId ||
|
|
409
|
+
environment?.active_release_id !== input.releaseId) {
|
|
410
|
+
throw new Error(`Proxy routing metadata did not converge for project ${input.projectId} environment ${input.environmentId}`);
|
|
411
|
+
}
|
|
412
|
+
};
|
|
413
|
+
let token = await tokenManager.getToken(scope, input.projectSlug);
|
|
414
|
+
try {
|
|
415
|
+
await resolveWithToken(token);
|
|
416
|
+
}
|
|
417
|
+
catch (error) {
|
|
418
|
+
if (!isProxyLookupAuthError(error))
|
|
419
|
+
throw error;
|
|
420
|
+
await tokenManager.invalidateToken(scope, input.projectSlug);
|
|
421
|
+
token = await tokenManager.getToken(scope, input.projectSlug);
|
|
422
|
+
await resolveWithToken(token);
|
|
423
|
+
}
|
|
424
|
+
logger?.info("Proxy routing metadata converged after deployment activation", {
|
|
425
|
+
projectId: input.projectId,
|
|
426
|
+
projectSlug: input.projectSlug,
|
|
427
|
+
deploymentId: input.deploymentId,
|
|
428
|
+
environmentId: input.environmentId,
|
|
429
|
+
environmentName: input.environmentName,
|
|
430
|
+
releaseId: input.releaseId,
|
|
431
|
+
});
|
|
432
|
+
}
|
|
294
433
|
function validateConfig() {
|
|
295
434
|
const missing = [];
|
|
296
435
|
if (!config.apiClientId)
|
|
@@ -410,6 +549,9 @@ export function createProxyHandler(options) {
|
|
|
410
549
|
return await resolveWithCurrentToken();
|
|
411
550
|
}
|
|
412
551
|
catch (error) {
|
|
552
|
+
if (error instanceof ProxyRoutingInvalidationRaceError) {
|
|
553
|
+
return { error: { status: 503, message: error.message } };
|
|
554
|
+
}
|
|
413
555
|
if (!isProxyLookupAuthError(error))
|
|
414
556
|
throw error;
|
|
415
557
|
const projectKey = tokenIdentity.projectSlug ?? tokenIdentity.customDomain;
|
|
@@ -444,6 +586,9 @@ export function createProxyHandler(options) {
|
|
|
444
586
|
return await resolveWithCurrentToken();
|
|
445
587
|
}
|
|
446
588
|
catch (retryError) {
|
|
589
|
+
if (retryError instanceof ProxyRoutingInvalidationRaceError) {
|
|
590
|
+
return { error: { status: 503, message: retryError.message } };
|
|
591
|
+
}
|
|
447
592
|
if (!isProxyLookupAuthError(retryError))
|
|
448
593
|
throw retryError;
|
|
449
594
|
logger?.error("Proxy API token rejected after refresh", retryError, {
|
|
@@ -689,6 +834,8 @@ export function createProxyHandler(options) {
|
|
|
689
834
|
getStats,
|
|
690
835
|
close,
|
|
691
836
|
validateConfig,
|
|
837
|
+
invalidateRoutingLookup,
|
|
838
|
+
invalidateAndConfirmRoutingLookup,
|
|
692
839
|
localProjects,
|
|
693
840
|
};
|
|
694
841
|
}
|
package/esm/src/proxy/main.js
CHANGED
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
* - LOCAL_PROJECTS: JSON map of slug → filesystem path (for dev)
|
|
16
16
|
* - CACHE_TYPE: "memory" (default) or "redis"
|
|
17
17
|
* - REDIS_URL: Redis connection URL (required if CACHE_TYPE=redis)
|
|
18
|
+
* - VERYFRONT_PROXY_EXPECTED_REPLICAS: Minimum proxy replicas required to acknowledge routing changes
|
|
19
|
+
* - VERYFRONT_PROXY_ROUTING_INVALIDATION_SECRET: HMAC secret for Redis routing events and acknowledgements
|
|
18
20
|
* - VERYFRONT_API_INTERNAL_URL: API URL for internal endpoints (falls back to VERYFRONT_PROXY_API_BASE_URL)
|
|
19
21
|
* - VERYFRONT_API_INTERNAL_USER: Basic auth user for internal API
|
|
20
22
|
* - VERYFRONT_API_INTERNAL_PASS: Basic auth pass for internal API
|
|
@@ -42,6 +44,8 @@ import { createUpstreamFailureResponse, createUpstreamTimeoutResponse, UPSTREAM_
|
|
|
42
44
|
import { createProxyServerTiming, markProxyServerTimingPhase, profileProxyServerTimingPhase, withProxyServerTimingHeader, } from "./server-timing.js";
|
|
43
45
|
import { removeStickyCookieFromPublicCacheableResponse } from "./response-headers.js";
|
|
44
46
|
import { closeProxyServerWithin, createProxyDrainingResponse, parseProxyDrainTimeoutMs, ProxyRequestDrainTracker, } from "./request-drain.js";
|
|
47
|
+
import { handleProxyRoutingInvalidationRequest, PROXY_ROUTING_INVALIDATION_PATH, } from "./routing-invalidation.js";
|
|
48
|
+
import { startProxyRoutingInvalidationBus } from "./routing-invalidation-redis.js";
|
|
45
49
|
function getLocalProjects() {
|
|
46
50
|
const raw = getEnv("LOCAL_PROJECTS");
|
|
47
51
|
return raw ? JSON.parse(raw) : {};
|
|
@@ -98,6 +102,17 @@ const DEFAULT_SERVER_RETRY_DELAY_MS = 100;
|
|
|
98
102
|
const VERYFRONT_SERVER_RETRY_COUNT = parseInt(getEnv("VERYFRONT_SERVER_RETRY_COUNT") || String(DEFAULT_SERVER_RETRY_COUNT));
|
|
99
103
|
const VERYFRONT_SERVER_RETRY_DELAY_MS = parseInt(getEnv("VERYFRONT_SERVER_RETRY_DELAY_MS") || String(DEFAULT_SERVER_RETRY_DELAY_MS));
|
|
100
104
|
const SHUTDOWN_DRAIN_TIMEOUT_MS = parseProxyDrainTimeoutMs(getEnv("SHUTDOWN_DRAIN_TIMEOUT_MS"), DEFAULT_SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
105
|
+
const routingInvalidationSecret = getEnv("VERYFRONT_PROXY_ROUTING_INVALIDATION_SECRET") ?? "";
|
|
106
|
+
const routingInvalidationSecretBytes = new TextEncoder().encode(routingInvalidationSecret).byteLength;
|
|
107
|
+
const expectedReplicasRaw = getEnv("VERYFRONT_PROXY_EXPECTED_REPLICAS");
|
|
108
|
+
const expectedReplicas = Number(expectedReplicasRaw);
|
|
109
|
+
const hasValidExpectedReplicas = Number.isInteger(expectedReplicas) && expectedReplicas > 0;
|
|
110
|
+
if (isProduction() && !hasValidExpectedReplicas) {
|
|
111
|
+
throw new Error("VERYFRONT_PROXY_EXPECTED_REPLICAS must be a positive integer in production");
|
|
112
|
+
}
|
|
113
|
+
if (isProduction() && routingInvalidationSecretBytes < 32) {
|
|
114
|
+
throw new Error("VERYFRONT_PROXY_ROUTING_INVALIDATION_SECRET must contain at least 32 bytes in production");
|
|
115
|
+
}
|
|
101
116
|
const proxyRequestDrainTracker = new ProxyRequestDrainTracker();
|
|
102
117
|
let shuttingDown = false;
|
|
103
118
|
const { createAuthProvider } = await importFirstPartyExtensionModule("ext-auth-jwt", "@veryfront/ext-auth-jwt").catch((error) => {
|
|
@@ -106,16 +121,32 @@ const { createAuthProvider } = await importFirstPartyExtensionModule("ext-auth-j
|
|
|
106
121
|
register("AuthProvider", createAuthProvider({}));
|
|
107
122
|
// Initialize cache and proxy handler
|
|
108
123
|
const cache = await createCacheFromEnv();
|
|
124
|
+
const routingInvalidationLogger = {
|
|
125
|
+
debug: (msg, extra) => proxyLogger.debug(msg, extra),
|
|
126
|
+
info: (msg, extra) => proxyLogger.info(msg, extra),
|
|
127
|
+
warn: (msg, extra) => proxyLogger.warn(msg, extra),
|
|
128
|
+
error: (msg, error, extra) => proxyLogger.error(msg, extra ?? {}, error),
|
|
129
|
+
};
|
|
109
130
|
const proxyHandler = createProxyHandler({
|
|
110
131
|
config,
|
|
111
132
|
cache,
|
|
112
|
-
logger:
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
133
|
+
logger: routingInvalidationLogger,
|
|
134
|
+
});
|
|
135
|
+
const routingInvalidationBus = await startProxyRoutingInvalidationBus({
|
|
136
|
+
expectedReplicas: hasValidExpectedReplicas ? expectedReplicas : undefined,
|
|
137
|
+
integritySecret: routingInvalidationSecret,
|
|
138
|
+
logger: routingInvalidationLogger,
|
|
139
|
+
onInvalidate: proxyHandler.invalidateAndConfirmRoutingLookup,
|
|
140
|
+
}).catch((error) => {
|
|
141
|
+
if (isProduction()) {
|
|
142
|
+
throw new Error("Proxy routing invalidation bus failed to start", { cause: error });
|
|
143
|
+
}
|
|
144
|
+
proxyLogger.error("Proxy routing invalidation bus failed; TTL recovery remains active", {}, error instanceof Error ? error : new Error(String(error)));
|
|
145
|
+
return null;
|
|
118
146
|
});
|
|
147
|
+
if (isProduction() && !routingInvalidationBus) {
|
|
148
|
+
throw new Error("Proxy routing invalidation bus requires REDIS_URL and a valid VERYFRONT_PROXY_ROUTING_INVALIDATION_SECRET in production");
|
|
149
|
+
}
|
|
119
150
|
// Validate configuration on startup
|
|
120
151
|
const missingCredentials = proxyHandler.validateConfig();
|
|
121
152
|
if (missingCredentials.length > 0) {
|
|
@@ -499,6 +530,11 @@ async function router(req) {
|
|
|
499
530
|
if (req.headers.get("upgrade")?.toLowerCase() === "websocket") {
|
|
500
531
|
response = await handleWebSocketUpgrade(req, url);
|
|
501
532
|
}
|
|
533
|
+
else if (url.pathname === PROXY_ROUTING_INVALIDATION_PATH) {
|
|
534
|
+
response = await handleProxyRoutingInvalidationRequest(req, {
|
|
535
|
+
publisher: routingInvalidationBus,
|
|
536
|
+
});
|
|
537
|
+
}
|
|
502
538
|
else if (url.pathname === "/_proxy/stats") {
|
|
503
539
|
response = Object.keys(proxyHandler.localProjects).length === 0
|
|
504
540
|
? new Response("Forbidden", { status: 403 })
|
|
@@ -533,6 +569,8 @@ async function shutdown(signal) {
|
|
|
533
569
|
drainTimeoutMs: SHUTDOWN_DRAIN_TIMEOUT_MS,
|
|
534
570
|
});
|
|
535
571
|
try {
|
|
572
|
+
// New requests receive the draining response after shuttingDown is set.
|
|
573
|
+
// Keep this replica subscribed while already-started responses finish.
|
|
536
574
|
const drained = await proxyRequestDrainTracker.waitForDrain(SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
537
575
|
if (!drained) {
|
|
538
576
|
const now = performance.now();
|
|
@@ -545,6 +583,7 @@ async function shutdown(signal) {
|
|
|
545
583
|
})),
|
|
546
584
|
});
|
|
547
585
|
}
|
|
586
|
+
await routingInvalidationBus?.close();
|
|
548
587
|
const closed = await closeProxyServerWithin(() => server.close(), PROXY_SERVER_CLOSE_TIMEOUT_MS);
|
|
549
588
|
if (!closed) {
|
|
550
589
|
proxyLogger.warn("Proxy server close timed out; process exit will close remaining connections", {
|
package/esm/src/proxy/retry.d.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* These occur when renderer pods are being recycled or temporarily unavailable.
|
|
7
7
|
*/
|
|
8
8
|
export declare function isRetryableConnectionError(error: unknown): boolean;
|
|
9
|
+
export declare function isConnectionRefusedError(error: unknown): boolean;
|
|
9
10
|
/**
|
|
10
11
|
* Decide how many times the proxy can safely retry an upstream request.
|
|
11
12
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../../../src/src/proxy/retry.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../../../src/src/proxy/retry.ts"],"names":[],"mappings":"AAAA;;GAEG;AAyCH;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAWlE;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAQhE;AAkCD;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,oBAAoB,EAAE,MAAM,GAC3B,MAAM,CAUR;AAED,0EAA0E;AAC1E,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,OAAO,GACb,OAAO,CAQT;AAED;;;;;GAKG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,OAAO,EAChB,UAAU,EAAE,MAAM,GACjB,KAAK,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAc1C"}
|
package/esm/src/proxy/retry.js
CHANGED
|
@@ -3,26 +3,43 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { DEFAULT_MAX_BODY_SIZE_BYTES } from "../utils/constants/index.js";
|
|
5
5
|
const CONTROL_PLANE_RUN_STREAM_PATH = /^\/api\/control-plane\/runs\/[^/]+\/stream$/;
|
|
6
|
+
const MAX_ERROR_CAUSE_DEPTH = 8;
|
|
7
|
+
const RETRYABLE_CONNECTION_CODES = new Set(["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT"]);
|
|
8
|
+
const CONNECTION_REFUSED_CODES = new Set(["ECONNREFUSED"]);
|
|
9
|
+
function errorChainMatches(error, predicate) {
|
|
10
|
+
if (!(error instanceof Error))
|
|
11
|
+
return false;
|
|
12
|
+
const seen = new Set();
|
|
13
|
+
let current = error;
|
|
14
|
+
for (let depth = 0; depth < MAX_ERROR_CAUSE_DEPTH; depth++) {
|
|
15
|
+
if (typeof current !== "object" || current === null || seen.has(current))
|
|
16
|
+
return false;
|
|
17
|
+
seen.add(current);
|
|
18
|
+
const details = current;
|
|
19
|
+
const message = typeof details.message === "string" ? details.message.toLowerCase() : "";
|
|
20
|
+
const code = typeof details.code === "string" ? details.code.toUpperCase() : "";
|
|
21
|
+
if (predicate(message, code))
|
|
22
|
+
return true;
|
|
23
|
+
current = details.cause;
|
|
24
|
+
}
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
6
27
|
/**
|
|
7
28
|
* Check if a fetch error is a transient connection error worth retrying.
|
|
8
29
|
* These occur when renderer pods are being recycled or temporarily unavailable.
|
|
9
30
|
*/
|
|
10
31
|
export function isRetryableConnectionError(error) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
msg.includes("os error 104") || // ECONNRESET
|
|
18
|
-
msg.includes("os error 111") // ECONNREFUSED
|
|
19
|
-
);
|
|
32
|
+
return errorChainMatches(error, (message, code) => RETRYABLE_CONNECTION_CODES.has(code) ||
|
|
33
|
+
message.includes("connection reset") ||
|
|
34
|
+
message.includes("connection closed") ||
|
|
35
|
+
message.includes("connection refused") ||
|
|
36
|
+
message.includes("os error 104") || // ECONNRESET
|
|
37
|
+
message.includes("os error 111"));
|
|
20
38
|
}
|
|
21
|
-
function isConnectionRefusedError(error) {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
return message.includes("connection refused") || message.includes("os error 111");
|
|
39
|
+
export function isConnectionRefusedError(error) {
|
|
40
|
+
return errorChainMatches(error, (message, code) => CONNECTION_REFUSED_CODES.has(code) ||
|
|
41
|
+
message.includes("connection refused") ||
|
|
42
|
+
message.includes("os error 111"));
|
|
26
43
|
}
|
|
27
44
|
function isIdempotentMethod(method) {
|
|
28
45
|
return method === "GET" || method === "HEAD" || method === "OPTIONS";
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { ProxyRoutingInvalidationEvent, ProxyRoutingInvalidationPublisher } from "./routing-invalidation.js";
|
|
2
|
+
type RedisListener = (message: string, channel: string) => void;
|
|
3
|
+
export interface RoutingInvalidationRedisClient {
|
|
4
|
+
connect(): Promise<void>;
|
|
5
|
+
publish(channel: string, message: string): Promise<number>;
|
|
6
|
+
subscribe(channel: string, listener: RedisListener): Promise<number>;
|
|
7
|
+
unsubscribe(channel: string): Promise<number>;
|
|
8
|
+
close(): Promise<void>;
|
|
9
|
+
destroy(): void;
|
|
10
|
+
on?(event: "error", listener: (error: unknown) => void): unknown;
|
|
11
|
+
}
|
|
12
|
+
interface RoutingInvalidationLogger {
|
|
13
|
+
info(message: string, extra?: Record<string, unknown>): void;
|
|
14
|
+
warn(message: string, extra?: Record<string, unknown>): void;
|
|
15
|
+
error(message: string, error?: Error, extra?: Record<string, unknown>): void;
|
|
16
|
+
}
|
|
17
|
+
export interface ProxyRoutingInvalidationBus extends ProxyRoutingInvalidationPublisher {
|
|
18
|
+
close(): Promise<void>;
|
|
19
|
+
}
|
|
20
|
+
interface StartProxyRoutingInvalidationBusOptions {
|
|
21
|
+
acknowledgementTimeoutMs?: number;
|
|
22
|
+
createClient?: (redisUrl: string) => RoutingInvalidationRedisClient | Promise<RoutingInvalidationRedisClient>;
|
|
23
|
+
expectedReplicas?: number;
|
|
24
|
+
now?: () => number;
|
|
25
|
+
logger?: RoutingInvalidationLogger;
|
|
26
|
+
onInvalidate: (event: ProxyRoutingInvalidationEvent) => unknown | Promise<unknown>;
|
|
27
|
+
integritySecret?: string;
|
|
28
|
+
redisUrl?: string;
|
|
29
|
+
replicaId?: string;
|
|
30
|
+
}
|
|
31
|
+
export declare function startProxyRoutingInvalidationBus(options: StartProxyRoutingInvalidationBusOptions): Promise<ProxyRoutingInvalidationBus | null>;
|
|
32
|
+
export {};
|
|
33
|
+
//# sourceMappingURL=routing-invalidation-redis.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"routing-invalidation-redis.d.ts","sourceRoot":"","sources":["../../../src/src/proxy/routing-invalidation-redis.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,6BAA6B,EAC7B,iCAAiC,EAElC,MAAM,2BAA2B,CAAC;AAgBnC,KAAK,aAAa,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;AAGhE,MAAM,WAAW,8BAA8B;IAC7C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3D,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,OAAO,IAAI,IAAI,CAAC;IAChB,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC;CAClE;AAED,UAAU,yBAAyB;IACjC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC7D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC7D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC9E;AAED,MAAM,WAAW,2BAA4B,SAAQ,iCAAiC;IACpF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,UAAU,uCAAuC;IAC/C,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,CAAC,EAAE,CACb,QAAQ,EAAE,MAAM,KACb,8BAA8B,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAC9E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,yBAAyB,CAAC;IACnC,YAAY,EAAE,CAAC,KAAK,EAAE,6BAA6B,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACnF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAmPD,wBAAsB,gCAAgC,CACpD,OAAO,EAAE,uCAAuC,GAC/C,OAAO,CAAC,2BAA2B,GAAG,IAAI,CAAC,CAgP7C"}
|