svelte-adapter-uws-extensions 0.6.0-next.39 → 0.6.0-next.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/package.json +1 -1
- package/src/capability-cookie.js +25 -17
- package/src/redis/clock-skew.d.ts +32 -0
- package/src/redis/clock-skew.js +89 -2
- package/src/redis/pubsub.js +5 -0
- package/src/redis/ratelimit.d.ts +14 -0
- package/src/redis/ratelimit.js +109 -10
- package/src/redis/sharded-pubsub.js +3 -0
- package/src/testing/mock-platform.js +4 -0
package/README.md
CHANGED
|
@@ -1084,6 +1084,9 @@ export async function message(ws, { data, platform }) {
|
|
|
1084
1084
|
| `interval` | *required* | Refill interval in ms |
|
|
1085
1085
|
| `blockDuration` | `0` | Auto-ban duration in ms (0 = no ban) |
|
|
1086
1086
|
| `keyBy` | `'ip'` | `'ip'`, `'connection'`, or a function |
|
|
1087
|
+
| `localFloorOnStorageFailure` | `false` | Degraded mode for `consume()`: decide on an in-process token bucket while Redis is unreachable (or the breaker is open) instead of rejecting the promise. `true` reuses `points`/`interval`; `{ points?, interval? }` sets a tighter per-instance budget. |
|
|
1088
|
+
|
|
1089
|
+
> **Degraded verdicts.** Without the floor, a Redis outage rejects every `consume()` promise and the caller decides (usually fail-closed). With it, limits keep being enforced per instance with the same semantics (refill, `blockDuration` bans) - the worst case is N instances allowing N times the floor budget until the store recovers, which is why a tighter per-instance budget (`points / instanceCount`) is offered. Floor state never leaks back into Redis; admin ops (`reset` / `ban` / `unban` / `clear`) still reject while degraded. Floor-decided verdicts count in `ratelimit_storage_fallbacks_total` when a metrics registry is configured, and the first degraded verdict logs a one-shot warning.
|
|
1087
1090
|
|
|
1088
1091
|
> **`keyBy: 'ip'` behind a proxy.** In `'ip'` mode (the default) the bucket key is `userData.remoteAddress`, which the adapter resolves from `ADDRESS_HEADER` / `XFF_DEPTH`. If the server sits behind an address-rewriting proxy (a docker userland-proxy, an L4 load balancer, a non-XFF proxy) and `ADDRESS_HEADER` is unset, every client arrives as the same gateway address and the per-IP bucket collapses into one shared global bucket. Set `ADDRESS_HEADER` (and `XFF_DEPTH` for an X-Forwarded-For chain) so the real client IP is resolved, or pass an explicit `keyBy`. The limiter logs a one-shot warning the first time it denies on a loopback/private key while `ADDRESS_HEADER` is unset.
|
|
1089
1092
|
|
|
@@ -2085,6 +2088,8 @@ It reads the exact wall clock through the package's runtime seam - not the coars
|
|
|
2085
2088
|
| `onWarn` | - | Called when `warnMs <= |skew| < tripMs`. |
|
|
2086
2089
|
| `onTrip` | - | Called when `|skew| >= tripMs`. |
|
|
2087
2090
|
| `onError` | - | Called when a sample fails (Redis error). The last good gauge value is retained. |
|
|
2091
|
+
| `fence` | `false` | Opt-in self-fencing: `true` or `{ tripSamples?: 2, releaseSamples?: 3 }`. Turns the trip threshold from an alert into a state (see below). |
|
|
2092
|
+
| `onFence` | - | Called on every fence-state transition with the new boolean state. |
|
|
2088
2093
|
| `breaker` | - | Optional circuit breaker. Sample failures count via `breaker.failure(err)`; successes via `breaker.success()`. |
|
|
2089
2094
|
| `metrics` | - | Optional Prometheus metrics registry. |
|
|
2090
2095
|
|
|
@@ -2093,14 +2098,31 @@ It reads the exact wall clock through the package's runtime seam - not the coars
|
|
|
2093
2098
|
| Method | Description |
|
|
2094
2099
|
|---|---|
|
|
2095
2100
|
| `current()` | The most recent signed skew in milliseconds, or `null` before the first successful sample. |
|
|
2101
|
+
| `fenced()` | Whether the sampler is in the FENCED state (always `false` with the `fence` option off). |
|
|
2096
2102
|
| `sample()` | Run one sample immediately and return the signed median skew (or `null` on failure). Also updates the gauge and fires callbacks. |
|
|
2097
2103
|
| `stop()` | Stop the interval and await any in-flight sample. Idempotent. Never throws. |
|
|
2098
2104
|
|
|
2105
|
+
#### Self-fencing (`fence`)
|
|
2106
|
+
|
|
2107
|
+
Measurement alone leaves a drifted clock free to keep corrupting whatever it stamps. With `fence` set, the sampler holds a FENCED state with hysteresis: it enters after `tripSamples` (default 2) **consecutive** trip-level samples - one bad sample never sheds authority - and releases after `releaseSamples` (default 3) consecutive below-warn samples; a warn-level sample holds the current state, so the warn..trip band is the deliberate hysteresis gap. A failed sample never changes fence state (losing the measurement is not evidence of drift).
|
|
2108
|
+
|
|
2109
|
+
Wire it to the platform once and the framework consumes it:
|
|
2110
|
+
|
|
2111
|
+
```js
|
|
2112
|
+
import { createClockSkewSampler, attachClockFence } from 'svelte-adapter-uws-extensions/redis/clock-skew';
|
|
2113
|
+
|
|
2114
|
+
const skew = createClockSkewSampler(redis, { metrics, fence: true });
|
|
2115
|
+
attachClockFence(platform, skew); // platform.clockFence = { fenced }
|
|
2116
|
+
```
|
|
2117
|
+
|
|
2118
|
+
`bus.wrap` forwards `platform.clockFence`, and clustered `live.smooth` reads it at every authority tick: a fenced owner stands down exactly as if it had crashed - it releases its ownership lease (so a healthy-clocked sibling claims immediately) and stops stamping, riding the already-shipped takeover machinery instead of a new failure mode. Single-instance deployments have no sibling to hand off to; there the fence surfaces only through the gauge and `onFence` - which is also the escalation hook if you want a hard shed (`onFence: (f) => { if (f) process.exit(1); }` under a supervisor that restarts, or flip your load balancer's health check).
|
|
2119
|
+
|
|
2099
2120
|
#### Metrics
|
|
2100
2121
|
|
|
2101
2122
|
| Metric | Description |
|
|
2102
2123
|
|---|---|
|
|
2103
2124
|
| `platform_clock_skew_ms` | Gauge of the signed clock skew of this instance's wall clock relative to the Redis server clock (positive = local ahead). The median of several round-trip-compensated Redis `TIME` reads. |
|
|
2125
|
+
| `platform_clock_fenced` | Gauge (0/1): whether this instance has self-fenced on clock skew (registered only with the `fence` option on). |
|
|
2104
2126
|
|
|
2105
2127
|
> On a Redis Cluster client, `TIME` carries no key and is routed to an arbitrary node, so successive reads may sample different nodes. For a coarse drift gauge that is fine; pin to one node upstream if you need per-node attribution.
|
|
2106
2128
|
|
package/package.json
CHANGED
package/src/capability-cookie.js
CHANGED
|
@@ -262,36 +262,43 @@ export function capabilityCookie(options) {
|
|
|
262
262
|
*
|
|
263
263
|
* @param {string} value
|
|
264
264
|
*/
|
|
265
|
+
// A malformed cookie exits before any real verification; burn one
|
|
266
|
+
// signature + compare over fixed inputs first so "unparseable" is not
|
|
267
|
+
// separable from "wrong signature" by the missing HMAC's timing.
|
|
268
|
+
function burnAndFail() {
|
|
269
|
+
safeEqual(sign(secret, '', 0, ''), 'x');
|
|
270
|
+
return null;
|
|
271
|
+
}
|
|
272
|
+
|
|
265
273
|
function decode(value) {
|
|
266
274
|
if (typeof value !== 'string') return null;
|
|
267
275
|
const parts = value.split('.');
|
|
268
|
-
if (parts.length !== 4) return
|
|
276
|
+
if (parts.length !== 4) return burnAndFail();
|
|
269
277
|
const [sidB64, issuedRaw, salt, presentedSig] = parts;
|
|
270
278
|
|
|
271
279
|
let sessionId;
|
|
272
280
|
try {
|
|
273
281
|
sessionId = unb64u(sidB64);
|
|
274
282
|
} catch {
|
|
275
|
-
return
|
|
283
|
+
return burnAndFail();
|
|
276
284
|
}
|
|
277
285
|
const issuedAt = Number(issuedRaw);
|
|
278
|
-
if (!Number.isInteger(issuedAt) || issuedAt <= 0) return
|
|
279
|
-
|
|
280
|
-
// Expiry is independent of the secret: an expired cookie is invalid even
|
|
281
|
-
// under the current secret.
|
|
282
|
-
if (now() - issuedAt > ttlMs) return null;
|
|
286
|
+
if (!Number.isInteger(issuedAt) || issuedAt <= 0) return burnAndFail();
|
|
283
287
|
|
|
288
|
+
// Signature FIRST: authentication is the constant-work step, so every
|
|
289
|
+
// parseable cookie costs the same HMAC work whether it turns out to be
|
|
290
|
+
// expired or forged - the check order no longer separates the two by
|
|
291
|
+
// timing. Expiry stays independent of the secret: an expired cookie is
|
|
292
|
+
// invalid even when signed with the current key.
|
|
284
293
|
const expectCurrent = sign(secret, sessionId, issuedAt, salt);
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
}
|
|
288
|
-
if (previousSecret) {
|
|
294
|
+
let authentic = safeEqual(expectCurrent, presentedSig);
|
|
295
|
+
if (!authentic && previousSecret) {
|
|
289
296
|
const expectPrev = sign(previousSecret, sessionId, issuedAt, salt);
|
|
290
|
-
|
|
291
|
-
return { sessionId, issuedAt };
|
|
292
|
-
}
|
|
297
|
+
authentic = safeEqual(expectPrev, presentedSig);
|
|
293
298
|
}
|
|
294
|
-
return null;
|
|
299
|
+
if (!authentic) return null;
|
|
300
|
+
if (now() - issuedAt > ttlMs) return null;
|
|
301
|
+
return { sessionId, issuedAt };
|
|
295
302
|
}
|
|
296
303
|
|
|
297
304
|
function serializeCookie(value) {
|
|
@@ -340,8 +347,9 @@ export function capabilityCookie(options) {
|
|
|
340
347
|
// posture, so it counts regardless of `required`. Expired cookies
|
|
341
348
|
// from idle real users land here too (`refresh()` on page responses
|
|
342
349
|
// keeps live users out of this bucket); expiry is deliberately not
|
|
343
|
-
// its own reason -
|
|
344
|
-
//
|
|
350
|
+
// its own reason - `issuedAt` is sender-supplied, so an "expired"
|
|
351
|
+
// bucket would be forgeable, and decode's rejection is uniform
|
|
352
|
+
// (signature-first, same HMAC work) across every failure cause.
|
|
345
353
|
const ok = decode(value) !== null;
|
|
346
354
|
if (!ok) mMisses?.inc({ reason: 'invalid' });
|
|
347
355
|
return ok;
|
|
@@ -40,6 +40,20 @@ export interface ClockSkewSamplerOptions {
|
|
|
40
40
|
* retained.
|
|
41
41
|
*/
|
|
42
42
|
onError?: (err: Error) => void;
|
|
43
|
+
/**
|
|
44
|
+
* Opt-in self-fencing: turn the trip threshold from an alert into a state.
|
|
45
|
+
* The sampler enters the FENCED state after `tripSamples` (default 2)
|
|
46
|
+
* consecutive trip-level samples and releases after `releaseSamples`
|
|
47
|
+
* (default 3) consecutive below-warn samples; warn-level samples hold the
|
|
48
|
+
* current state (deliberate hysteresis). While fenced, `fenced()` reads
|
|
49
|
+
* true and the `platform_clock_fenced` gauge reads 1; consumers that stamp
|
|
50
|
+
* or order by this instance's clock stand down (`attachClockFence` + the
|
|
51
|
+
* clustered `live.smooth` authority gate consume it that way). A failed
|
|
52
|
+
* sample never changes fence state. @default false
|
|
53
|
+
*/
|
|
54
|
+
fence?: boolean | { tripSamples?: number; releaseSamples?: number };
|
|
55
|
+
/** Called on every fence-state transition. */
|
|
56
|
+
onFence?: (fenced: boolean) => void;
|
|
43
57
|
breaker?: CircuitBreaker;
|
|
44
58
|
metrics?: MetricsRegistry;
|
|
45
59
|
}
|
|
@@ -50,6 +64,11 @@ export interface ClockSkewSampler {
|
|
|
50
64
|
* of Redis), or `null` before the first successful sample.
|
|
51
65
|
*/
|
|
52
66
|
current(): number | null;
|
|
67
|
+
/**
|
|
68
|
+
* Whether the sampler is in the FENCED state. Always `false` when the
|
|
69
|
+
* `fence` option is off.
|
|
70
|
+
*/
|
|
71
|
+
fenced(): boolean;
|
|
53
72
|
/**
|
|
54
73
|
* Run one sample immediately and return the signed median skew (or `null` on
|
|
55
74
|
* failure). Also updates the gauge and fires callbacks.
|
|
@@ -61,6 +80,19 @@ export interface ClockSkewSampler {
|
|
|
61
80
|
stop(): Promise<void>;
|
|
62
81
|
}
|
|
63
82
|
|
|
83
|
+
/**
|
|
84
|
+
* Attach a sampler's fence surface to the platform as `platform.clockFence`,
|
|
85
|
+
* so framework layers that stamp or order by this instance's clock (the
|
|
86
|
+
* clustered `live.smooth` authority renewal gate today) stand down while the
|
|
87
|
+
* clock is untrustworthy. `bus.wrap` forwards it, so the wrapped platform a
|
|
88
|
+
* clustered app hands to the framework carries it too. Requires a sampler
|
|
89
|
+
* constructed with the `fence` option. Returns the platform.
|
|
90
|
+
*/
|
|
91
|
+
export function attachClockFence(
|
|
92
|
+
platform: object,
|
|
93
|
+
sampler: ClockSkewSampler
|
|
94
|
+
): object;
|
|
95
|
+
|
|
64
96
|
/**
|
|
65
97
|
* Create a clock-skew sampler that measures this instance's wall clock against
|
|
66
98
|
* the Redis server clock and exposes it as the `platform_clock_skew_ms` gauge.
|
package/src/redis/clock-skew.js
CHANGED
|
@@ -47,6 +47,18 @@ const DEFAULT_TRIP_MS = 500;
|
|
|
47
47
|
* @property {(skewMs: number) => void} [onWarn] - Called when `warnMs <= |skew| < tripMs`.
|
|
48
48
|
* @property {(skewMs: number) => void} [onTrip] - Called when `|skew| >= tripMs`.
|
|
49
49
|
* @property {(err: Error) => void} [onError] - Called when a sample fails (Redis error). The last good gauge value is retained.
|
|
50
|
+
* @property {boolean | { tripSamples?: number, releaseSamples?: number }} [fence=false] -
|
|
51
|
+
* Opt-in self-fencing: turn the trip threshold from an alert into a state.
|
|
52
|
+
* The sampler enters the FENCED state after `tripSamples` (default 2)
|
|
53
|
+
* consecutive trip-level samples - one bad sample must never shed authority
|
|
54
|
+
* - and releases after `releaseSamples` (default 3) consecutive below-warn
|
|
55
|
+
* samples (warn-level samples hold the current state; the hysteresis gap is
|
|
56
|
+
* deliberate). While fenced, `fenced()` reads true; consumers that stamp or
|
|
57
|
+
* order by this instance's clock should stand down and let a healthy
|
|
58
|
+
* instance take over (`attachClockFence` + the smooth authority gate consume
|
|
59
|
+
* it that way). A failed sample never changes fence state - losing the
|
|
60
|
+
* measurement is not evidence of drift.
|
|
61
|
+
* @property {(fenced: boolean) => void} [onFence] - Called on every fence-state transition.
|
|
50
62
|
* @property {import('../shared/breaker.js').CircuitBreaker} [breaker]
|
|
51
63
|
* @property {import('../prometheus/index.js').MetricsRegistry} [metrics]
|
|
52
64
|
*/
|
|
@@ -54,6 +66,7 @@ const DEFAULT_TRIP_MS = 500;
|
|
|
54
66
|
/**
|
|
55
67
|
* @typedef {Object} ClockSkewSampler
|
|
56
68
|
* @property {() => number | null} current - The most recent signed skew in milliseconds, or `null` before the first successful sample.
|
|
69
|
+
* @property {() => boolean} fenced - Whether the sampler is in the FENCED state (always false when the `fence` option is off).
|
|
57
70
|
* @property {() => Promise<number | null>} sample - Run one sample immediately and return the signed median skew (or `null` on failure). Also updates the gauge and fires callbacks.
|
|
58
71
|
* @property {() => Promise<void>} stop - Stop the interval and await any in-flight sample. Idempotent. Never throws.
|
|
59
72
|
*/
|
|
@@ -124,26 +137,60 @@ export function createClockSkewSampler(client, options = {}) {
|
|
|
124
137
|
if (!Number.isFinite(warnMs) || warnMs < 0 || !Number.isFinite(tripMs) || tripMs < 0) {
|
|
125
138
|
throw new Error('clock-skew: warnMs and tripMs must be non-negative numbers (ms)');
|
|
126
139
|
}
|
|
127
|
-
for (const name of ['onSkew', 'onWarn', 'onTrip', 'onError']) {
|
|
140
|
+
for (const name of ['onSkew', 'onWarn', 'onTrip', 'onError', 'onFence']) {
|
|
128
141
|
if (options[name] !== undefined && typeof options[name] !== 'function') {
|
|
129
142
|
throw new Error(`clock-skew: ${name} must be a function`);
|
|
130
143
|
}
|
|
131
144
|
}
|
|
145
|
+
const fenceOpt = options.fence;
|
|
146
|
+
let fenceEnabled = false;
|
|
147
|
+
let fenceTripSamples = 2;
|
|
148
|
+
let fenceReleaseSamples = 3;
|
|
149
|
+
if (fenceOpt !== undefined && fenceOpt !== false) {
|
|
150
|
+
if (fenceOpt !== true && (typeof fenceOpt !== 'object' || fenceOpt === null)) {
|
|
151
|
+
throw new Error('clock-skew: fence must be a boolean or { tripSamples?, releaseSamples? }');
|
|
152
|
+
}
|
|
153
|
+
fenceEnabled = true;
|
|
154
|
+
if (fenceOpt !== true) {
|
|
155
|
+
fenceTripSamples = fenceOpt.tripSamples ?? 2;
|
|
156
|
+
fenceReleaseSamples = fenceOpt.releaseSamples ?? 3;
|
|
157
|
+
if (!Number.isInteger(fenceTripSamples) || fenceTripSamples < 1
|
|
158
|
+
|| !Number.isInteger(fenceReleaseSamples) || fenceReleaseSamples < 1) {
|
|
159
|
+
throw new Error('clock-skew: fence tripSamples and releaseSamples must be positive integers');
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
132
163
|
|
|
133
164
|
const redis = client.redis;
|
|
134
165
|
const breaker = options.breaker;
|
|
135
|
-
const { onSkew, onWarn, onTrip, onError } = options;
|
|
166
|
+
const { onSkew, onWarn, onTrip, onError, onFence } = options;
|
|
136
167
|
|
|
137
168
|
const m = options.metrics;
|
|
138
169
|
const mSkew = m?.gauge(
|
|
139
170
|
'platform_clock_skew_ms',
|
|
140
171
|
'Signed clock skew of this instance wall clock relative to the Redis server clock, in milliseconds (positive = local clock ahead of Redis). The median of several round-trip-compensated Redis TIME reads.'
|
|
141
172
|
);
|
|
173
|
+
const mFenced = fenceEnabled ? m?.gauge(
|
|
174
|
+
'platform_clock_fenced',
|
|
175
|
+
'Whether this instance has self-fenced on clock skew (1 = fenced: stamping/ordering authority stood down until the skew clears)'
|
|
176
|
+
) : undefined;
|
|
177
|
+
mFenced?.set(0);
|
|
142
178
|
|
|
143
179
|
let lastSkew = null;
|
|
144
180
|
let stopped = false;
|
|
145
181
|
let timer = null;
|
|
146
182
|
let inFlight = null;
|
|
183
|
+
let fenced = false;
|
|
184
|
+
let tripStreak = 0;
|
|
185
|
+
let calmStreak = 0;
|
|
186
|
+
|
|
187
|
+
/** @param {boolean} next */
|
|
188
|
+
function setFenced(next) {
|
|
189
|
+
if (fenced === next) return;
|
|
190
|
+
fenced = next;
|
|
191
|
+
mFenced?.set(next ? 1 : 0);
|
|
192
|
+
if (onFence) { try { onFence(next); } catch { /* swallow */ } }
|
|
193
|
+
}
|
|
147
194
|
|
|
148
195
|
/** Run one sample. Resolves to the signed median skew, or null on failure. */
|
|
149
196
|
async function sample() {
|
|
@@ -176,6 +223,24 @@ export function createClockSkewSampler(client, options = {}) {
|
|
|
176
223
|
} else if (abs >= warnMs) {
|
|
177
224
|
if (onWarn) { try { onWarn(skew); } catch { /* swallow */ } }
|
|
178
225
|
}
|
|
226
|
+
if (fenceEnabled) {
|
|
227
|
+
// Enter on consecutive trips, release on consecutive calm
|
|
228
|
+
// (below-warn) samples; a warn-level sample holds the current
|
|
229
|
+
// state, so the warn..trip band is the hysteresis gap.
|
|
230
|
+
if (abs >= tripMs) {
|
|
231
|
+
tripStreak++;
|
|
232
|
+
calmStreak = 0;
|
|
233
|
+
if (tripStreak >= fenceTripSamples) setFenced(true);
|
|
234
|
+
} else {
|
|
235
|
+
tripStreak = 0;
|
|
236
|
+
if (abs < warnMs) {
|
|
237
|
+
calmStreak++;
|
|
238
|
+
if (calmStreak >= fenceReleaseSamples) setFenced(false);
|
|
239
|
+
} else {
|
|
240
|
+
calmStreak = 0;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
179
244
|
return skew;
|
|
180
245
|
} catch (err) {
|
|
181
246
|
breaker?.failure(err);
|
|
@@ -213,7 +278,29 @@ export function createClockSkewSampler(client, options = {}) {
|
|
|
213
278
|
|
|
214
279
|
return {
|
|
215
280
|
current: () => lastSkew,
|
|
281
|
+
fenced: () => fenced,
|
|
216
282
|
sample,
|
|
217
283
|
stop
|
|
218
284
|
};
|
|
219
285
|
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Attach a sampler's fence surface to the platform as the `clockFence`
|
|
289
|
+
* convention, so framework layers that stamp or order by this instance's
|
|
290
|
+
* clock (the smooth authority renewal gate today) can stand down while the
|
|
291
|
+
* clock is untrustworthy. `bus.wrap` forwards it, so the wrapped platform a
|
|
292
|
+
* clustered app hands to the framework carries it too.
|
|
293
|
+
*
|
|
294
|
+
* @param {any} platform
|
|
295
|
+
* @param {ClockSkewSampler} sampler
|
|
296
|
+
*/
|
|
297
|
+
export function attachClockFence(platform, sampler) {
|
|
298
|
+
if (!platform || typeof platform !== 'object') {
|
|
299
|
+
throw new Error('clock-skew: attachClockFence requires a platform object');
|
|
300
|
+
}
|
|
301
|
+
if (!sampler || typeof sampler.fenced !== 'function') {
|
|
302
|
+
throw new Error('clock-skew: attachClockFence requires a sampler with a fenced() surface (set the fence option)');
|
|
303
|
+
}
|
|
304
|
+
platform.clockFence = { fenced: sampler.fenced };
|
|
305
|
+
return platform;
|
|
306
|
+
}
|
package/src/redis/pubsub.js
CHANGED
|
@@ -387,6 +387,11 @@ export function createPubSubBus(client, options = {}) {
|
|
|
387
387
|
get presence() { return platform.presence; },
|
|
388
388
|
get crdt() { return platform.crdt; },
|
|
389
389
|
get smooth() { return platform.smooth; },
|
|
390
|
+
// Clock-fence convention (attachClockFence): layers that stamp
|
|
391
|
+
// or order by this instance's clock read it to stand down while
|
|
392
|
+
// the clock is fenced. Live getter so post-wrap attachment
|
|
393
|
+
// propagates.
|
|
394
|
+
get clockFence() { return platform.clockFence; },
|
|
390
395
|
// The topic-broadcast cluster coordinator (createTopicBroadcast),
|
|
391
396
|
// attached by app init like the other plugins. Live getter so
|
|
392
397
|
// post-wrap assignment propagates; realtime detects it on the wrapped
|
package/src/redis/ratelimit.d.ts
CHANGED
|
@@ -29,6 +29,20 @@ export interface RedisRateLimitOptions {
|
|
|
29
29
|
* single-tenant deploy (byte-identical).
|
|
30
30
|
*/
|
|
31
31
|
tenant?: (ws: any) => string | null | undefined;
|
|
32
|
+
/**
|
|
33
|
+
* Opt-in degraded mode for `consume()`: when Redis is unreachable (or the
|
|
34
|
+
* breaker is open), decide on an in-process token bucket with the same
|
|
35
|
+
* semantics instead of rejecting the promise. `true` reuses the configured
|
|
36
|
+
* points/interval; the object form sets a tighter per-instance budget
|
|
37
|
+
* (e.g. `points / instanceCount` keeps the fleet-wide allowance roughly
|
|
38
|
+
* constant while degraded). Floor state is per process, so N instances
|
|
39
|
+
* allow up to N times the floor budget in the worst case. Admin ops
|
|
40
|
+
* (`reset` / `ban` / `unban` / `clear` / `purgeUser`) still reject while
|
|
41
|
+
* the store is down - only the request-path verdict degrades. When a
|
|
42
|
+
* metrics registry is configured, floor-decided verdicts count in
|
|
43
|
+
* `ratelimit_storage_fallbacks_total`. @default false
|
|
44
|
+
*/
|
|
45
|
+
localFloorOnStorageFailure?: boolean | { points?: number; interval?: number };
|
|
32
46
|
/** Prometheus metrics registry. */
|
|
33
47
|
metrics?: MetricsRegistry;
|
|
34
48
|
/** Circuit breaker instance. */
|
package/src/redis/ratelimit.js
CHANGED
|
@@ -14,8 +14,61 @@
|
|
|
14
14
|
import { scanAndUnlink } from '../shared/redis-scan.js';
|
|
15
15
|
import { withBreaker } from '../shared/breaker.js';
|
|
16
16
|
import { isPrivateOrLoopbackAddress, isAddressHeaderConfigured } from '../shared/client-ip.js';
|
|
17
|
+
import { wallEpoch } from '../shared/runtime.js';
|
|
17
18
|
import { CONSUME_SCRIPT } from './token-bucket-script.js';
|
|
18
19
|
|
|
20
|
+
// Upper bound on in-process floor buckets so a keyed flood during an outage
|
|
21
|
+
// cannot exhaust memory; least-recently-touched entries are evicted first.
|
|
22
|
+
const FLOOR_MAX_ENTRIES = 10000;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* In-process token bucket mirroring CONSUME_SCRIPT's semantics (init at full,
|
|
26
|
+
* ban check, interval refill, consume, ban-on-exhaust) so a verdict decided
|
|
27
|
+
* on the floor matches what Redis would have said for a single instance.
|
|
28
|
+
* State is per process: while degraded, N instances allow up to N times the
|
|
29
|
+
* configured budget in the worst case, which is the deliberate trade against
|
|
30
|
+
* denying everything (fail closed) or counting nothing (fail open).
|
|
31
|
+
*
|
|
32
|
+
* @param {number} maxPoints
|
|
33
|
+
* @param {number} interval
|
|
34
|
+
* @param {number} blockDuration
|
|
35
|
+
* @returns {(key: string, cost: number) => ConsumeResult}
|
|
36
|
+
*/
|
|
37
|
+
function createLocalFloorBucket(maxPoints, interval, blockDuration) {
|
|
38
|
+
/** @type {Map<string, { points: number, resetAt: number, bannedUntil: number }>} */
|
|
39
|
+
const buckets = new Map();
|
|
40
|
+
return function consumeLocal(key, cost) {
|
|
41
|
+
const nowMs = wallEpoch();
|
|
42
|
+
let entry = buckets.get(key);
|
|
43
|
+
if (entry !== undefined) {
|
|
44
|
+
// Reinsert on touch: insertion order then doubles as LRU order.
|
|
45
|
+
buckets.delete(key);
|
|
46
|
+
} else {
|
|
47
|
+
entry = { points: maxPoints, resetAt: nowMs + interval, bannedUntil: 0 };
|
|
48
|
+
}
|
|
49
|
+
buckets.set(key, entry);
|
|
50
|
+
if (buckets.size > FLOOR_MAX_ENTRIES) {
|
|
51
|
+
buckets.delete(buckets.keys().next().value);
|
|
52
|
+
}
|
|
53
|
+
if (entry.bannedUntil > nowMs) {
|
|
54
|
+
return { allowed: false, remaining: 0, resetMs: entry.bannedUntil - nowMs };
|
|
55
|
+
}
|
|
56
|
+
if (entry.resetAt <= nowMs) {
|
|
57
|
+
entry.points = maxPoints;
|
|
58
|
+
entry.resetAt = nowMs + interval;
|
|
59
|
+
}
|
|
60
|
+
if (entry.points >= cost) {
|
|
61
|
+
entry.points -= cost;
|
|
62
|
+
return { allowed: true, remaining: entry.points, resetMs: entry.resetAt - nowMs };
|
|
63
|
+
}
|
|
64
|
+
if (blockDuration > 0) {
|
|
65
|
+
entry.bannedUntil = nowMs + blockDuration;
|
|
66
|
+
return { allowed: false, remaining: 0, resetMs: blockDuration };
|
|
67
|
+
}
|
|
68
|
+
return { allowed: false, remaining: Math.max(0, entry.points), resetMs: entry.resetAt - nowMs };
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
19
72
|
const BAN_SCRIPT = `
|
|
20
73
|
local key = KEYS[1]
|
|
21
74
|
local duration = tonumber(ARGV[1])
|
|
@@ -57,6 +110,16 @@ return 1
|
|
|
57
110
|
* null/undefined for an unscoped connection. Omit for a single-tenant deploy (byte-identical
|
|
58
111
|
* to before). The id should be a delimiter-safe slug - it is joined to the key with a NUL,
|
|
59
112
|
* so it stays unambiguous even when the key is an IPv6 address.
|
|
113
|
+
* @property {boolean | { points?: number, interval?: number }} [localFloorOnStorageFailure=false] -
|
|
114
|
+
* Opt-in degraded mode for `consume()`: when Redis is unreachable (or the breaker is open),
|
|
115
|
+
* decide on an in-process token bucket with the same semantics instead of rejecting the
|
|
116
|
+
* promise. `true` reuses the configured points/interval; the object form sets a tighter
|
|
117
|
+
* per-instance budget (e.g. `points / instanceCount` keeps the fleet-wide allowance
|
|
118
|
+
* roughly constant while degraded). Floor state is per process and per instance, so N
|
|
119
|
+
* instances allow up to N times the floor budget in the worst case; it never leaks back
|
|
120
|
+
* into Redis. Admin ops (`reset` / `ban` / `unban` / `clear` / `purgeUser`) are operator
|
|
121
|
+
* actions and still reject while the store is down - only the request-path verdict
|
|
122
|
+
* degrades. Off (today's reject-to-caller behavior) unless set.
|
|
60
123
|
*/
|
|
61
124
|
|
|
62
125
|
/**
|
|
@@ -105,6 +168,24 @@ export function createRateLimit(client, options) {
|
|
|
105
168
|
throw new Error('redis ratelimit: tenant must be a function (ws) => id | null');
|
|
106
169
|
}
|
|
107
170
|
|
|
171
|
+
const floorOpt = options.localFloorOnStorageFailure;
|
|
172
|
+
let consumeFloor = null;
|
|
173
|
+
if (floorOpt !== undefined && floorOpt !== false) {
|
|
174
|
+
if (floorOpt !== true && (typeof floorOpt !== 'object' || floorOpt === null)) {
|
|
175
|
+
throw new Error('redis ratelimit: localFloorOnStorageFailure must be a boolean or { points?, interval? }');
|
|
176
|
+
}
|
|
177
|
+
const floorPoints = floorOpt === true ? points : (floorOpt.points ?? points);
|
|
178
|
+
const floorInterval = floorOpt === true ? interval : (floorOpt.interval ?? interval);
|
|
179
|
+
if (!Number.isInteger(floorPoints) || floorPoints <= 0) {
|
|
180
|
+
throw new Error('redis ratelimit: localFloorOnStorageFailure.points must be a positive integer');
|
|
181
|
+
}
|
|
182
|
+
if (typeof floorInterval !== 'number' || !Number.isFinite(floorInterval) || floorInterval <= 0) {
|
|
183
|
+
throw new Error('redis ratelimit: localFloorOnStorageFailure.interval must be a positive number');
|
|
184
|
+
}
|
|
185
|
+
consumeFloor = createLocalFloorBucket(floorPoints, floorInterval, blockDuration);
|
|
186
|
+
}
|
|
187
|
+
let warnedStorageFloor = false;
|
|
188
|
+
|
|
108
189
|
const redis = client.redis;
|
|
109
190
|
|
|
110
191
|
// Version prefix for Redis keys. Different script versions use different
|
|
@@ -123,6 +204,9 @@ export function createRateLimit(client, options) {
|
|
|
123
204
|
const mAllowed = m?.counter('ratelimit_allowed_total', 'Requests allowed', tlabels);
|
|
124
205
|
const mDenied = m?.counter('ratelimit_denied_total', 'Requests denied', tlabels);
|
|
125
206
|
const mBans = m?.counter('ratelimit_bans_total', 'Bans applied', tlabels);
|
|
207
|
+
const mFloor = consumeFloor !== null
|
|
208
|
+
? m?.counter('ratelimit_storage_fallbacks_total', 'Rate-limit verdicts decided by the in-process floor while the store was unreachable')
|
|
209
|
+
: undefined;
|
|
126
210
|
|
|
127
211
|
// Per-connection keying uses a WeakMap to avoid leaks
|
|
128
212
|
const wsKeys = new WeakMap();
|
|
@@ -199,25 +283,40 @@ export function createRateLimit(client, options) {
|
|
|
199
283
|
}
|
|
200
284
|
const key = resolveKey(ws);
|
|
201
285
|
const tenantId = tenant ? tenant(ws) : null;
|
|
286
|
+
// Resolved before the try so a tenant-id validation throw surfaces
|
|
287
|
+
// to the caller and can never be mistaken for a storage failure.
|
|
288
|
+
const bk = bucketKey(key, tenantId);
|
|
202
289
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
290
|
+
let verdict;
|
|
291
|
+
try {
|
|
292
|
+
const result = await withBreaker(b, () =>
|
|
293
|
+
redis.eval(CONSUME_SCRIPT, 1, bk, points, interval, cost, blockDuration)
|
|
294
|
+
);
|
|
295
|
+
verdict = { allowed: result[0] === 1, remaining: result[1], resetMs: result[2] };
|
|
296
|
+
} catch (err) {
|
|
297
|
+
if (consumeFloor === null) throw err;
|
|
298
|
+
mFloor?.inc();
|
|
299
|
+
if (!warnedStorageFloor) {
|
|
300
|
+
warnedStorageFloor = true;
|
|
301
|
+
console.warn(
|
|
302
|
+
'redis ratelimit: store unreachable; deciding on the in-process floor. ' +
|
|
303
|
+
'Limits hold per instance while degraded (N instances allow up to N times ' +
|
|
304
|
+
'the floor budget); cross-instance state resumes when the store recovers. ' +
|
|
305
|
+
'This warning fires once.'
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
verdict = consumeFloor(bk, cost);
|
|
309
|
+
}
|
|
206
310
|
|
|
207
|
-
const allowed = result[0] === 1;
|
|
208
311
|
const labels = labelTenants ? { tenant_id: tenantId || '' } : undefined;
|
|
209
|
-
if (allowed) {
|
|
312
|
+
if (verdict.allowed) {
|
|
210
313
|
mAllowed?.inc(labels);
|
|
211
314
|
} else {
|
|
212
315
|
mDenied?.inc(labels);
|
|
213
316
|
maybeWarnProxyCollapse(key);
|
|
214
317
|
}
|
|
215
318
|
|
|
216
|
-
return
|
|
217
|
-
allowed,
|
|
218
|
-
remaining: result[1],
|
|
219
|
-
resetMs: result[2]
|
|
220
|
-
};
|
|
319
|
+
return verdict;
|
|
221
320
|
},
|
|
222
321
|
|
|
223
322
|
async reset(key, tenantId) {
|
|
@@ -919,6 +919,9 @@ export function createShardedBus(client, options = {}) {
|
|
|
919
919
|
get presence() { return platform.presence; },
|
|
920
920
|
get crdt() { return platform.crdt; },
|
|
921
921
|
get smooth() { return platform.smooth; },
|
|
922
|
+
// Clock-fence convention (attachClockFence); live getter so
|
|
923
|
+
// post-wrap attachment propagates.
|
|
924
|
+
get clockFence() { return platform.clockFence; },
|
|
922
925
|
// The topic-broadcast cluster coordinator; live getter so realtime
|
|
923
926
|
// detects it on the wrapped seam to fan live.push({ topic }) cluster-wide.
|
|
924
927
|
get topicBroadcast() { return platform.topicBroadcast; },
|
|
@@ -57,6 +57,10 @@ export function mockPlatform() {
|
|
|
57
57
|
// reads `platform.smooth` (the cross-instance smooth coordinator). Default
|
|
58
58
|
// `undefined`; the wrap-parity test drives the forward-when-set behavior.
|
|
59
59
|
smooth: undefined,
|
|
60
|
+
// Framework-convention slot: layers that stamp or order by this
|
|
61
|
+
// instance's clock read `platform.clockFence` (attachClockFence) to
|
|
62
|
+
// stand down while the clock is fenced. Default undefined = no fence.
|
|
63
|
+
clockFence: undefined,
|
|
60
64
|
// platform.pressure stub. Default snapshot mirrors a healthy worker.
|
|
61
65
|
// Tests drive transitions via _setPressure(snapshot).
|
|
62
66
|
pressure: {
|