redis-graph-cache 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +857 -0
- package/dist/core/compression.d.ts +58 -0
- package/dist/core/compression.js +115 -0
- package/dist/core/hydration-engine.d.ts +68 -0
- package/dist/core/hydration-engine.js +288 -0
- package/dist/core/index.d.ts +13 -0
- package/dist/core/index.js +29 -0
- package/dist/core/lua-scripts.d.ts +148 -0
- package/dist/core/lua-scripts.js +259 -0
- package/dist/core/normalization-engine.d.ts +76 -0
- package/dist/core/normalization-engine.js +286 -0
- package/dist/core/redis-connection-manager.d.ts +255 -0
- package/dist/core/redis-connection-manager.js +745 -0
- package/dist/core/schema-manager.d.ts +133 -0
- package/dist/core/schema-manager.js +432 -0
- package/dist/core/serializer.d.ts +69 -0
- package/dist/core/serializer.js +187 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +32 -0
- package/dist/redis-graph-cache.d.ts +302 -0
- package/dist/redis-graph-cache.js +839 -0
- package/dist/types/config.types.d.ts +138 -0
- package/dist/types/config.types.js +5 -0
- package/dist/types/error.types.d.ts +54 -0
- package/dist/types/error.types.js +89 -0
- package/dist/types/index.d.ts +8 -0
- package/dist/types/index.js +29 -0
- package/dist/types/internal.types.d.ts +67 -0
- package/dist/types/internal.types.js +73 -0
- package/dist/types/operation.types.d.ts +101 -0
- package/dist/types/operation.types.js +5 -0
- package/dist/types/schema.types.d.ts +104 -0
- package/dist/types/schema.types.js +5 -0
- package/package.json +44 -0
|
@@ -0,0 +1,745 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Redis Connection Manager
|
|
4
|
+
*
|
|
5
|
+
* Owns the ioredis client and exposes a small set of typed methods that all
|
|
6
|
+
* Redis traffic flows through. Every call goes through a circuit breaker +
|
|
7
|
+
* retry wrapper, records latency, and updates cache hit/miss counters where
|
|
8
|
+
* applicable. Atomic Lua scripts are registered on construction and exposed
|
|
9
|
+
* as typed helpers (casSet, setIfExists, listAdd, listRemove) so callers
|
|
10
|
+
* never need direct access to the underlying client for normal work.
|
|
11
|
+
*/
|
|
12
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
13
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
14
|
+
};
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.RedisConnectionManager = void 0;
|
|
17
|
+
const ioredis_1 = __importDefault(require("ioredis"));
|
|
18
|
+
const types_1 = require("../types");
|
|
19
|
+
const lua_scripts_1 = require("./lua-scripts");
|
|
20
|
+
class RedisConnectionManager {
|
|
21
|
+
/**
|
|
22
|
+
* Typed view of `this.redis` that includes the Lua-script methods
|
|
23
|
+
* registered via `defineCommand`. Use this whenever you need to
|
|
24
|
+
* call an `rse*` script or a variadic ZRANGEBYSCORE-family command;
|
|
25
|
+
* ordinary commands (`get`, `set`, `del`, `exists`, etc.) are typed
|
|
26
|
+
* on the base instance and should keep using `this.redis`.
|
|
27
|
+
*
|
|
28
|
+
* Calling this advances the round-robin index identically to
|
|
29
|
+
* `this.redis`, so a single call site stays pinned to one client.
|
|
30
|
+
*/
|
|
31
|
+
get scripted() {
|
|
32
|
+
return this.redis;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Round-robin client picker. Hot-path methods read `this.redis` once
|
|
36
|
+
* per call which advances the index, distributing load uniformly.
|
|
37
|
+
* Pipelines obtained via `this.redis.pipeline()` are owned by a
|
|
38
|
+
* single client because the getter is called exactly once.
|
|
39
|
+
*/
|
|
40
|
+
get redis() {
|
|
41
|
+
if (this.clients.length === 0) {
|
|
42
|
+
throw new types_1.RedisConnectionError('Redis pool not initialized');
|
|
43
|
+
}
|
|
44
|
+
if (this.clients.length === 1)
|
|
45
|
+
return this.clients[0];
|
|
46
|
+
const c = this.clients[this.rrIndex];
|
|
47
|
+
this.rrIndex = (this.rrIndex + 1) % this.clients.length;
|
|
48
|
+
return c;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* True when at least one pool client reports connected. Health checks
|
|
52
|
+
* succeed under partial pool failures, which is the right behaviour:
|
|
53
|
+
* one degraded socket shouldn't take the whole engine offline.
|
|
54
|
+
*/
|
|
55
|
+
get isConnected() {
|
|
56
|
+
return this.connectedFlags.some((f) => f);
|
|
57
|
+
}
|
|
58
|
+
constructor(config, circuitBreakerConfig, retryConfig) {
|
|
59
|
+
this.config = config;
|
|
60
|
+
/**
|
|
61
|
+
* The pool of ioredis clients. Default size 1 (a single connection)
|
|
62
|
+
* for backward compatibility. When `redis.poolSize` is configured
|
|
63
|
+
* higher, every hot-path command is round-robined across these
|
|
64
|
+
* clients, giving genuine parallelism on the Redis socket.
|
|
65
|
+
*/
|
|
66
|
+
this.clients = [];
|
|
67
|
+
this.rrIndex = 0;
|
|
68
|
+
/**
|
|
69
|
+
* Per-client connected flags. `isConnected` (below) is the OR across
|
|
70
|
+
* the pool, so health checks succeed when *any* client is up.
|
|
71
|
+
*/
|
|
72
|
+
this.connectedFlags = [];
|
|
73
|
+
this.circuitBreaker = new CircuitBreaker(circuitBreakerConfig);
|
|
74
|
+
this.retryManager = new RetryManager(retryConfig);
|
|
75
|
+
this.healthMetrics = new HealthMetrics();
|
|
76
|
+
this.initializeConnection();
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Initialize the Redis pool. Creates `poolSize` independent clients
|
|
80
|
+
* (defaulting to 1 for parity with the pre-pool implementation). Each
|
|
81
|
+
* client gets:
|
|
82
|
+
* - the same connection options
|
|
83
|
+
* - all atomic Lua scripts pre-registered as named commands
|
|
84
|
+
* - the same lifecycle event handlers
|
|
85
|
+
* Failures in any single client during init bubble up as connection
|
|
86
|
+
* errors so the engine fails closed rather than silently degrading.
|
|
87
|
+
*/
|
|
88
|
+
initializeConnection() {
|
|
89
|
+
try {
|
|
90
|
+
const poolSize = Math.max(1, this.config.poolSize ?? 1);
|
|
91
|
+
this.clients = [];
|
|
92
|
+
this.connectedFlags = [];
|
|
93
|
+
for (let i = 0; i < poolSize; i++) {
|
|
94
|
+
const client = new ioredis_1.default({
|
|
95
|
+
host: this.config.host || 'localhost',
|
|
96
|
+
port: this.config.port || 6379,
|
|
97
|
+
password: this.config.password,
|
|
98
|
+
db: this.config.db || 0,
|
|
99
|
+
family: this.config.family || 4,
|
|
100
|
+
// Distinct connection name per client makes pool members
|
|
101
|
+
// identifiable in Redis CLIENT LIST.
|
|
102
|
+
connectionName: this.config.connectionName
|
|
103
|
+
? `${this.config.connectionName}#${i}`
|
|
104
|
+
: undefined,
|
|
105
|
+
lazyConnect: false,
|
|
106
|
+
maxRetriesPerRequest: this.config.maxRetriesPerRequest,
|
|
107
|
+
enableReadyCheck: this.config.enableReadyCheck !== false,
|
|
108
|
+
connectTimeout: this.config.connectTimeout || 10000,
|
|
109
|
+
commandTimeout: this.config.commandTimeout || 5000,
|
|
110
|
+
retryStrategy: this.config.retryStrategy ||
|
|
111
|
+
((times) => Math.min(times * 50, 2000)),
|
|
112
|
+
keyPrefix: this.config.keyPrefix,
|
|
113
|
+
});
|
|
114
|
+
// Each client must have its own copy of the Lua scripts since
|
|
115
|
+
// ioredis registers commands per-instance.
|
|
116
|
+
for (const def of lua_scripts_1.SCRIPT_DEFINITIONS) {
|
|
117
|
+
client.defineCommand(def.name, {
|
|
118
|
+
numberOfKeys: def.numberOfKeys,
|
|
119
|
+
lua: def.script,
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
this.clients.push(client);
|
|
123
|
+
this.connectedFlags.push(false);
|
|
124
|
+
}
|
|
125
|
+
this.setupEventHandlers();
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
throw new types_1.RedisConnectionError('Failed to initialize Redis connection', error);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Attach lifecycle handlers to every pool client. Per-client connect/
|
|
133
|
+
* close events flip that client's slot in `connectedFlags` so the
|
|
134
|
+
* aggregate `isConnected` getter reflects pool-wide state without
|
|
135
|
+
* extra coordination.
|
|
136
|
+
*/
|
|
137
|
+
setupEventHandlers() {
|
|
138
|
+
this.clients.forEach((client, idx) => {
|
|
139
|
+
client.on('connect', () => {
|
|
140
|
+
this.connectedFlags[idx] = true;
|
|
141
|
+
this.healthMetrics.recordConnection();
|
|
142
|
+
});
|
|
143
|
+
client.on('ready', () => {
|
|
144
|
+
this.connectedFlags[idx] = true;
|
|
145
|
+
this.circuitBreaker.onSuccess();
|
|
146
|
+
});
|
|
147
|
+
client.on('error', (error) => {
|
|
148
|
+
this.healthMetrics.recordError();
|
|
149
|
+
this.circuitBreaker.onFailure();
|
|
150
|
+
console.error(`[RedisSchemaEngine] Redis client #${idx} error:`, error.message);
|
|
151
|
+
});
|
|
152
|
+
client.on('close', () => {
|
|
153
|
+
this.connectedFlags[idx] = false;
|
|
154
|
+
});
|
|
155
|
+
client.on('reconnecting', () => {
|
|
156
|
+
// intentional no-op; reconnection handled by ioredis retryStrategy
|
|
157
|
+
});
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Manually connect every pool client. Used when `lazyConnect: true`
|
|
162
|
+
* was passed via redis options; otherwise clients connect at
|
|
163
|
+
* construction time. Errors from any single client bubble up.
|
|
164
|
+
*/
|
|
165
|
+
async connect() {
|
|
166
|
+
try {
|
|
167
|
+
await Promise.all(this.clients.map(async (client, idx) => {
|
|
168
|
+
await client.connect();
|
|
169
|
+
this.connectedFlags[idx] = true;
|
|
170
|
+
}));
|
|
171
|
+
}
|
|
172
|
+
catch (error) {
|
|
173
|
+
throw new types_1.RedisConnectionError('Failed to connect to Redis', error);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
// =====================================================
|
|
177
|
+
// CORE EXECUTION WRAPPER
|
|
178
|
+
// =====================================================
|
|
179
|
+
/**
|
|
180
|
+
* Execute a Redis operation through circuit breaker + retry. All public
|
|
181
|
+
* helpers below funnel through this. Latency is recorded for both success
|
|
182
|
+
* and failure so health metrics reflect real traffic.
|
|
183
|
+
*/
|
|
184
|
+
async executeOperation(operation, operationName = 'redis_operation') {
|
|
185
|
+
return this.circuitBreaker.execute(async () => {
|
|
186
|
+
return this.retryManager.executeWithRetry(async () => {
|
|
187
|
+
const startTime = Date.now();
|
|
188
|
+
try {
|
|
189
|
+
const result = await operation();
|
|
190
|
+
this.healthMetrics.recordSuccess(Date.now() - startTime);
|
|
191
|
+
return result;
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
this.healthMetrics.recordFailure(Date.now() - startTime);
|
|
195
|
+
throw new types_1.RedisConnectionError(`Redis operation '${operationName}' failed`, error);
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
// =====================================================
|
|
201
|
+
// TYPED COMMAND WRAPPERS (all routed through executeOperation)
|
|
202
|
+
// =====================================================
|
|
203
|
+
/**
|
|
204
|
+
* GET wrapper. Records cache hit/miss for visibility into real traffic.
|
|
205
|
+
*/
|
|
206
|
+
async get(key) {
|
|
207
|
+
return this.executeOperation(async () => {
|
|
208
|
+
const value = await this.redis.get(key);
|
|
209
|
+
if (value === null) {
|
|
210
|
+
this.healthMetrics.recordMiss();
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
this.healthMetrics.recordHit();
|
|
214
|
+
}
|
|
215
|
+
return value;
|
|
216
|
+
}, 'get');
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* MGET wrapper. Records hit/miss per key for accurate hit-rate stats.
|
|
220
|
+
*/
|
|
221
|
+
async mget(keys) {
|
|
222
|
+
if (keys.length === 0)
|
|
223
|
+
return [];
|
|
224
|
+
return this.executeOperation(async () => {
|
|
225
|
+
const values = await this.redis.mget(...keys);
|
|
226
|
+
for (const v of values) {
|
|
227
|
+
if (v === null)
|
|
228
|
+
this.healthMetrics.recordMiss();
|
|
229
|
+
else
|
|
230
|
+
this.healthMetrics.recordHit();
|
|
231
|
+
}
|
|
232
|
+
return values;
|
|
233
|
+
}, 'mget');
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* SET wrapper with optional TTL in seconds.
|
|
237
|
+
*/
|
|
238
|
+
async set(key, value, ttlSeconds) {
|
|
239
|
+
return this.executeOperation(async () => {
|
|
240
|
+
if (ttlSeconds && ttlSeconds > 0) {
|
|
241
|
+
await this.redis.set(key, value, 'EX', ttlSeconds);
|
|
242
|
+
}
|
|
243
|
+
else {
|
|
244
|
+
await this.redis.set(key, value);
|
|
245
|
+
}
|
|
246
|
+
}, 'set');
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* DEL wrapper. Returns the number of keys removed.
|
|
250
|
+
*/
|
|
251
|
+
async del(...keys) {
|
|
252
|
+
if (keys.length === 0)
|
|
253
|
+
return 0;
|
|
254
|
+
return this.executeOperation(async () => {
|
|
255
|
+
return this.redis.del(...keys);
|
|
256
|
+
}, 'del');
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* EXISTS wrapper. Returns the number of keys that exist (0 or 1 for single).
|
|
260
|
+
*/
|
|
261
|
+
async exists(key) {
|
|
262
|
+
return this.executeOperation(async () => {
|
|
263
|
+
return this.redis.exists(key);
|
|
264
|
+
}, 'exists');
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Compare-and-set via Lua. Writes newValue only if current value at key
|
|
268
|
+
* exactly equals expectedValue. Pass empty string to expect absence.
|
|
269
|
+
* Returns true on success, false on conflict.
|
|
270
|
+
*/
|
|
271
|
+
async casSet(key, expectedValue, newValue, ttlSeconds = 0) {
|
|
272
|
+
return this.executeOperation(async () => {
|
|
273
|
+
const result = await this.scripted.rseCasSet(key, expectedValue, newValue, String(ttlSeconds));
|
|
274
|
+
return result === 1;
|
|
275
|
+
}, 'casSet');
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Atomic set that only writes if the key already exists. Used for
|
|
279
|
+
* "update only if cached" semantics. Returns true if applied.
|
|
280
|
+
*/
|
|
281
|
+
async setIfExists(key, value, ttlSeconds = 0) {
|
|
282
|
+
return this.executeOperation(async () => {
|
|
283
|
+
const result = await this.scripted.rseSetIfExists(key, value, String(ttlSeconds));
|
|
284
|
+
return result === 1;
|
|
285
|
+
}, 'setIfExists');
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Atomic add to a JSON-array list at key. Idempotent; returns true if
|
|
289
|
+
* the id was newly inserted, false if already present.
|
|
290
|
+
*/
|
|
291
|
+
async listAdd(listKey, id, ttlSeconds = 0) {
|
|
292
|
+
return this.executeOperation(async () => {
|
|
293
|
+
const numericFlag = typeof id === 'number' ? '1' : '0';
|
|
294
|
+
const result = await this.scripted.rseListAdd(listKey, String(id), String(ttlSeconds), numericFlag);
|
|
295
|
+
return result === 1;
|
|
296
|
+
}, 'listAdd');
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Atomic remove of an id from a JSON-array list. Preserves the existing
|
|
300
|
+
* TTL on the key. Returns true if the id was found and removed.
|
|
301
|
+
*/
|
|
302
|
+
async listRemove(listKey, id) {
|
|
303
|
+
return this.executeOperation(async () => {
|
|
304
|
+
const result = await this.scripted.rseListRemove(listKey, String(id));
|
|
305
|
+
return result === 1;
|
|
306
|
+
}, 'listRemove');
|
|
307
|
+
}
|
|
308
|
+
// =====================================================
|
|
309
|
+
// ZSET (indexed list) wrappers
|
|
310
|
+
// =====================================================
|
|
311
|
+
/**
|
|
312
|
+
* Atomic ZADD with optional max-size trim and optional membership-index
|
|
313
|
+
* update. Returns true when a brand-new member was added; false when an
|
|
314
|
+
* existing member's score was updated (still considered "success").
|
|
315
|
+
*/
|
|
316
|
+
async zListAdd(listKey, membershipKey, score, id, ttlSeconds = 0, maxSize = 0, trackMembership = false) {
|
|
317
|
+
return this.executeOperation(async () => {
|
|
318
|
+
const result = await this.scripted.rseZListAdd(listKey, membershipKey, String(score), String(id), String(ttlSeconds), String(maxSize), trackMembership ? '1' : '0');
|
|
319
|
+
return result === 1;
|
|
320
|
+
}, 'zListAdd');
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Atomic ZREM. Returns true if the id was a member of the set.
|
|
324
|
+
*/
|
|
325
|
+
async zListRemove(listKey, membershipKey, id, trackMembership = false) {
|
|
326
|
+
return this.executeOperation(async () => {
|
|
327
|
+
const result = await this.scripted.rseZListRemove(listKey, membershipKey, String(id), trackMembership ? '1' : '0');
|
|
328
|
+
return result === 1;
|
|
329
|
+
}, 'zListRemove');
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* Paginated read of a ZSET. Range semantics:
|
|
333
|
+
* - `reverse: false` (default) returns members in ascending score
|
|
334
|
+
* order (oldest first when score is timestamp).
|
|
335
|
+
* - `reverse: true` returns members in descending score order
|
|
336
|
+
* (newest first when score is timestamp) — the common feed case.
|
|
337
|
+
* - `offset` and `limit` are applied AFTER the score filter.
|
|
338
|
+
* - `minScore`/`maxScore` filter by score; defaults are -inf/+inf.
|
|
339
|
+
*
|
|
340
|
+
* Returns just the ids by default. Pass `withScores: true` to get the
|
|
341
|
+
* raw `[id, score]` pairs for downstream filtering.
|
|
342
|
+
*/
|
|
343
|
+
async zListRange(listKey, opts = {}) {
|
|
344
|
+
const offset = opts.offset ?? 0;
|
|
345
|
+
const limit = opts.limit ?? 50;
|
|
346
|
+
const reverse = !!opts.reverse;
|
|
347
|
+
const min = Number.isFinite(opts.minScore) ? String(opts.minScore) : '-inf';
|
|
348
|
+
const max = Number.isFinite(opts.maxScore) ? String(opts.maxScore) : '+inf';
|
|
349
|
+
return this.executeOperation(async () => {
|
|
350
|
+
// Hits/misses recorded once per call for cardinality reasons. A
|
|
351
|
+
// ZRANGEBYSCORE is one logical lookup; per-id miss tracking would
|
|
352
|
+
// distort the metric.
|
|
353
|
+
const args = reverse
|
|
354
|
+
? [listKey, max, min, 'LIMIT', offset, limit]
|
|
355
|
+
: [listKey, min, max, 'LIMIT', offset, limit];
|
|
356
|
+
let raw;
|
|
357
|
+
if (opts.withScores) {
|
|
358
|
+
args.splice(3, 0, 'WITHSCORES');
|
|
359
|
+
}
|
|
360
|
+
if (reverse) {
|
|
361
|
+
raw = await this.scripted.zrevrangebyscore(...args);
|
|
362
|
+
}
|
|
363
|
+
else {
|
|
364
|
+
raw = await this.scripted.zrangebyscore(...args);
|
|
365
|
+
}
|
|
366
|
+
if (raw.length === 0) {
|
|
367
|
+
this.healthMetrics.recordMiss();
|
|
368
|
+
}
|
|
369
|
+
else {
|
|
370
|
+
this.healthMetrics.recordHit();
|
|
371
|
+
}
|
|
372
|
+
if (!opts.withScores)
|
|
373
|
+
return raw;
|
|
374
|
+
const pairs = [];
|
|
375
|
+
for (let i = 0; i < raw.length; i += 2) {
|
|
376
|
+
pairs.push({ id: raw[i], score: Number(raw[i + 1]) });
|
|
377
|
+
}
|
|
378
|
+
return pairs;
|
|
379
|
+
}, 'zListRange');
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* Number of members currently in the ZSET. Returns 0 when the key is
|
|
383
|
+
* absent (which Redis ZCARD reports natively).
|
|
384
|
+
*/
|
|
385
|
+
async zListSize(listKey) {
|
|
386
|
+
return this.executeOperation(async () => {
|
|
387
|
+
return (await this.redis.zcard(listKey)) ?? 0;
|
|
388
|
+
}, 'zListSize');
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Cascade-invalidate an entity: ZREMs it from every tracked list and
|
|
392
|
+
* deletes both the entity key and the membership back-index. Returns
|
|
393
|
+
* the number of lists the entity was removed from. Safe to call when
|
|
394
|
+
* the back-index does not exist (returns 0).
|
|
395
|
+
*/
|
|
396
|
+
async cascadeInvalidate(membershipKey, entityKey, id) {
|
|
397
|
+
return this.executeOperation(async () => {
|
|
398
|
+
const result = await this.scripted.rseCascadeInvalidate(membershipKey, entityKey, String(id));
|
|
399
|
+
return Number(result) || 0;
|
|
400
|
+
}, 'cascadeInvalidate');
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* Run an arbitrary pipeline. Internal use by the normalization engine for
|
|
404
|
+
* bulk first-write operations. Errors inside the pipeline are surfaced.
|
|
405
|
+
*/
|
|
406
|
+
async runPipeline(build) {
|
|
407
|
+
return this.executeOperation(async () => {
|
|
408
|
+
const pipeline = this.redis.pipeline();
|
|
409
|
+
build(pipeline);
|
|
410
|
+
const results = await pipeline.exec();
|
|
411
|
+
if (!results) {
|
|
412
|
+
throw new Error('Pipeline execution returned null');
|
|
413
|
+
}
|
|
414
|
+
const errors = results.filter(([err]) => err !== null);
|
|
415
|
+
if (errors.length > 0) {
|
|
416
|
+
throw new Error(`Pipeline errors: ${errors
|
|
417
|
+
.map(([err]) => err?.message ?? 'unknown')
|
|
418
|
+
.join(', ')}`);
|
|
419
|
+
}
|
|
420
|
+
return results.map(([, value]) => value);
|
|
421
|
+
}, 'pipeline');
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* Escape hatch returning one client from the pool. Prefer the typed
|
|
425
|
+
* helpers above so traffic is observed and breaker-protected. This is
|
|
426
|
+
* here for migrations and tests only and should not be used in hot
|
|
427
|
+
* paths.
|
|
428
|
+
*/
|
|
429
|
+
getRawClient() {
|
|
430
|
+
return this.redis;
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* Return all clients in the pool. Internal use for tests verifying
|
|
434
|
+
* pool size and per-client state.
|
|
435
|
+
*/
|
|
436
|
+
getAllRawClients() {
|
|
437
|
+
return this.clients;
|
|
438
|
+
}
|
|
439
|
+
// =====================================================
|
|
440
|
+
// ADMIN / HEALTH
|
|
441
|
+
// =====================================================
|
|
442
|
+
/**
|
|
443
|
+
* FLUSHDB wrapper. Caller is responsible for guarding this; the engine
|
|
444
|
+
* adds an additional confirmation gate at the public API.
|
|
445
|
+
*/
|
|
446
|
+
async flushDb() {
|
|
447
|
+
return this.executeOperation(async () => {
|
|
448
|
+
await this.redis.flushdb();
|
|
449
|
+
}, 'flushdb');
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* Delete every key that starts with the configured `keyPrefix`, using
|
|
453
|
+
* non-blocking `SCAN` + `UNLINK` (async delete). This is the safe
|
|
454
|
+
* equivalent of `FLUSHDB` for deployments that share a Redis DB with
|
|
455
|
+
* other applications or environments: it only touches keys this engine
|
|
456
|
+
* owns and never blocks the Redis server on large keyspaces.
|
|
457
|
+
*
|
|
458
|
+
* Implementation notes:
|
|
459
|
+
* - `SCAN` with `{ match: '*' }` works correctly because ioredis
|
|
460
|
+
* automatically prepends `keyPrefix` to the `MATCH` pattern on
|
|
461
|
+
* the wire, so Redis only returns keys under our namespace.
|
|
462
|
+
* - Redis returns the fully-prefixed keys. We strip the prefix
|
|
463
|
+
* before calling `UNLINK`, because ioredis re-applies the prefix
|
|
464
|
+
* on outbound commands — passing the raw prefixed keys would
|
|
465
|
+
* produce a double prefix and delete nothing.
|
|
466
|
+
* - Pinned to a single pool client (`clients[0]`) so the SCAN cursor
|
|
467
|
+
* is consistent; UNLINK batches can go through any client but
|
|
468
|
+
* using the same one keeps this simple and deterministic.
|
|
469
|
+
* - Returns the count of deleted keys for observability.
|
|
470
|
+
*
|
|
471
|
+
* Throws if `keyPrefix` is empty; caller should use `flushDb()` in
|
|
472
|
+
* that case (FLUSHDB is still the right tool for a no-prefix setup).
|
|
473
|
+
*/
|
|
474
|
+
async clearPrefixedKeys() {
|
|
475
|
+
const prefix = this.config.keyPrefix;
|
|
476
|
+
if (!prefix || prefix.length === 0) {
|
|
477
|
+
throw new types_1.RedisConnectionError('clearPrefixedKeys requires redis.keyPrefix to be set');
|
|
478
|
+
}
|
|
479
|
+
return this.executeOperation(async () => {
|
|
480
|
+
const client = this.clients[0];
|
|
481
|
+
if (!client) {
|
|
482
|
+
throw new types_1.RedisConnectionError('Redis pool not initialized');
|
|
483
|
+
}
|
|
484
|
+
let deleted = 0;
|
|
485
|
+
// SCAN in batches of 500. Larger counts reduce round-trips but
|
|
486
|
+
// increase per-call work on the Redis side; 500 is a safe middle
|
|
487
|
+
// ground that performs well on keyspaces well past the 100k mark.
|
|
488
|
+
const stream = client.scanStream({ match: '*', count: 500 });
|
|
489
|
+
await new Promise((resolve, reject) => {
|
|
490
|
+
stream.on('data', (keys) => {
|
|
491
|
+
if (!keys || keys.length === 0)
|
|
492
|
+
return;
|
|
493
|
+
// Strip the prefix ioredis will re-apply in UNLINK. A key
|
|
494
|
+
// that doesn't start with the prefix should not happen (SCAN
|
|
495
|
+
// already filtered by prefix), but we skip it defensively
|
|
496
|
+
// rather than delete something we don't own.
|
|
497
|
+
const stripped = [];
|
|
498
|
+
for (const k of keys) {
|
|
499
|
+
if (k.startsWith(prefix)) {
|
|
500
|
+
stripped.push(k.slice(prefix.length));
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
if (stripped.length === 0)
|
|
504
|
+
return;
|
|
505
|
+
// Pause the stream while we UNLINK so we don't build up an
|
|
506
|
+
// unbounded backlog of in-flight deletes on huge keyspaces.
|
|
507
|
+
stream.pause();
|
|
508
|
+
client
|
|
509
|
+
.unlink(...stripped)
|
|
510
|
+
.then((n) => {
|
|
511
|
+
deleted += Number(n) || 0;
|
|
512
|
+
stream.resume();
|
|
513
|
+
})
|
|
514
|
+
.catch((err) => {
|
|
515
|
+
stream.destroy(err);
|
|
516
|
+
});
|
|
517
|
+
});
|
|
518
|
+
stream.on('end', () => resolve());
|
|
519
|
+
stream.on('error', (err) => reject(err));
|
|
520
|
+
});
|
|
521
|
+
return deleted;
|
|
522
|
+
}, 'clearPrefixedKeys');
|
|
523
|
+
}
|
|
524
|
+
/**
|
|
525
|
+
* The configured keyPrefix, or empty string if none. Consumers (e.g.
|
|
526
|
+
* `clearAllCache`) branch on this to pick between scoped delete and
|
|
527
|
+
* FLUSHDB.
|
|
528
|
+
*/
|
|
529
|
+
getKeyPrefix() {
|
|
530
|
+
return this.config.keyPrefix ?? '';
|
|
531
|
+
}
|
|
532
|
+
async isHealthy() {
|
|
533
|
+
try {
|
|
534
|
+
await this.executeOperation(async () => {
|
|
535
|
+
await this.redis.ping();
|
|
536
|
+
}, 'ping');
|
|
537
|
+
return true;
|
|
538
|
+
}
|
|
539
|
+
catch {
|
|
540
|
+
return false;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* Read Redis used_memory in bytes via the INFO command. Returns 0 if the
|
|
545
|
+
* value cannot be parsed (e.g. permission-restricted environments).
|
|
546
|
+
*/
|
|
547
|
+
async getRedisUsedMemory() {
|
|
548
|
+
try {
|
|
549
|
+
const info = await this.executeOperation(async () => this.redis.info('memory'), 'info_memory');
|
|
550
|
+
const match = /used_memory:(\d+)/.exec(info);
|
|
551
|
+
return match ? Number(match[1]) : 0;
|
|
552
|
+
}
|
|
553
|
+
catch {
|
|
554
|
+
return 0;
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
getHealthMetrics() {
|
|
558
|
+
return {
|
|
559
|
+
isConnected: this.isConnected,
|
|
560
|
+
circuitBreakerState: this.circuitBreaker.getState(),
|
|
561
|
+
...this.healthMetrics.getMetrics(),
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
/**
|
|
565
|
+
* Cleanly close every pool client. Errors on individual clients are
|
|
566
|
+
* logged but don't stop the others from quitting; the goal here is
|
|
567
|
+
* resource cleanup, not strict success.
|
|
568
|
+
*/
|
|
569
|
+
async disconnect() {
|
|
570
|
+
await Promise.all(this.clients.map(async (client, idx) => {
|
|
571
|
+
try {
|
|
572
|
+
if (this.connectedFlags[idx]) {
|
|
573
|
+
await client.quit();
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
catch (error) {
|
|
577
|
+
console.error(`[RedisSchemaEngine] Disconnect error on client #${idx}:`, error);
|
|
578
|
+
}
|
|
579
|
+
finally {
|
|
580
|
+
this.connectedFlags[idx] = false;
|
|
581
|
+
}
|
|
582
|
+
}));
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* Number of clients in the pool. Useful for tests and observability.
|
|
586
|
+
*/
|
|
587
|
+
getPoolSize() {
|
|
588
|
+
return this.clients.length;
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
exports.RedisConnectionManager = RedisConnectionManager;
|
|
592
|
+
/**
|
|
593
|
+
* Circuit Breaker implementation
|
|
594
|
+
*/
|
|
595
|
+
class CircuitBreaker {
|
|
596
|
+
constructor(config) {
|
|
597
|
+
this.config = config;
|
|
598
|
+
this.state = {
|
|
599
|
+
state: 'CLOSED',
|
|
600
|
+
failures: 0,
|
|
601
|
+
lastFailureTime: 0,
|
|
602
|
+
lastSuccessTime: Date.now(),
|
|
603
|
+
};
|
|
604
|
+
}
|
|
605
|
+
async execute(operation) {
|
|
606
|
+
if (this.state.state === 'OPEN') {
|
|
607
|
+
if (this.shouldAttemptReset()) {
|
|
608
|
+
this.state.state = 'HALF_OPEN';
|
|
609
|
+
}
|
|
610
|
+
else {
|
|
611
|
+
throw new types_1.CircuitBreakerOpenError();
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
try {
|
|
615
|
+
const result = await operation();
|
|
616
|
+
this.onSuccess();
|
|
617
|
+
return result;
|
|
618
|
+
}
|
|
619
|
+
catch (error) {
|
|
620
|
+
this.onFailure();
|
|
621
|
+
throw error;
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
onSuccess() {
|
|
625
|
+
this.state.failures = 0;
|
|
626
|
+
this.state.state = 'CLOSED';
|
|
627
|
+
this.state.lastSuccessTime = Date.now();
|
|
628
|
+
}
|
|
629
|
+
onFailure() {
|
|
630
|
+
this.state.failures++;
|
|
631
|
+
this.state.lastFailureTime = Date.now();
|
|
632
|
+
if (this.state.failures >= this.config.threshold) {
|
|
633
|
+
this.state.state = 'OPEN';
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
shouldAttemptReset() {
|
|
637
|
+
return Date.now() - this.state.lastFailureTime > this.config.resetTimeout;
|
|
638
|
+
}
|
|
639
|
+
getState() {
|
|
640
|
+
return { ...this.state };
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Retry Manager implementation
|
|
645
|
+
*/
|
|
646
|
+
class RetryManager {
|
|
647
|
+
constructor(config) {
|
|
648
|
+
this.config = config;
|
|
649
|
+
}
|
|
650
|
+
async executeWithRetry(operation, options = {}) {
|
|
651
|
+
const { maxAttempts = this.config.maxAttempts, baseDelay = this.config.baseDelay, maxDelay = this.config.maxDelay, backoffFactor = this.config.backoffFactor, } = options;
|
|
652
|
+
let lastError;
|
|
653
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
654
|
+
try {
|
|
655
|
+
return await operation();
|
|
656
|
+
}
|
|
657
|
+
catch (error) {
|
|
658
|
+
lastError = error;
|
|
659
|
+
if (attempt === maxAttempts || !this.isRetryableError(error)) {
|
|
660
|
+
throw error;
|
|
661
|
+
}
|
|
662
|
+
const delay = Math.min(baseDelay * Math.pow(backoffFactor, attempt - 1), maxDelay);
|
|
663
|
+
await this.sleep(delay);
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
throw lastError;
|
|
667
|
+
}
|
|
668
|
+
isRetryableError(error) {
|
|
669
|
+
const retryableMessages = [
|
|
670
|
+
'ECONNRESET',
|
|
671
|
+
'ETIMEDOUT',
|
|
672
|
+
'ENOTFOUND',
|
|
673
|
+
'ECONNREFUSED',
|
|
674
|
+
'Connection is closed',
|
|
675
|
+
];
|
|
676
|
+
return retryableMessages.some((msg) => error.message.includes(msg) || error.name.includes(msg));
|
|
677
|
+
}
|
|
678
|
+
sleep(ms) {
|
|
679
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
/**
|
|
683
|
+
* Health metrics tracker. Counts ops, latency, and cache hits/misses so the
|
|
684
|
+
* engine can expose real (not stubbed) metrics. Lock-free; correct under
|
|
685
|
+
* Node's single-threaded event loop.
|
|
686
|
+
*/
|
|
687
|
+
class HealthMetrics {
|
|
688
|
+
constructor() {
|
|
689
|
+
this.metrics = {
|
|
690
|
+
totalOperations: 0,
|
|
691
|
+
successfulOperations: 0,
|
|
692
|
+
failedOperations: 0,
|
|
693
|
+
totalLatency: 0,
|
|
694
|
+
connections: 0,
|
|
695
|
+
errors: 0,
|
|
696
|
+
lastOperationTime: 0,
|
|
697
|
+
cacheHits: 0,
|
|
698
|
+
cacheMisses: 0,
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
recordSuccess(latency) {
|
|
702
|
+
this.metrics.totalOperations++;
|
|
703
|
+
this.metrics.successfulOperations++;
|
|
704
|
+
this.metrics.totalLatency += latency;
|
|
705
|
+
this.metrics.lastOperationTime = Date.now();
|
|
706
|
+
}
|
|
707
|
+
recordFailure(latency) {
|
|
708
|
+
this.metrics.totalOperations++;
|
|
709
|
+
this.metrics.failedOperations++;
|
|
710
|
+
this.metrics.totalLatency += latency;
|
|
711
|
+
this.metrics.lastOperationTime = Date.now();
|
|
712
|
+
}
|
|
713
|
+
recordConnection() {
|
|
714
|
+
this.metrics.connections++;
|
|
715
|
+
}
|
|
716
|
+
recordError() {
|
|
717
|
+
this.metrics.errors++;
|
|
718
|
+
}
|
|
719
|
+
recordHit() {
|
|
720
|
+
this.metrics.cacheHits++;
|
|
721
|
+
}
|
|
722
|
+
recordMiss() {
|
|
723
|
+
this.metrics.cacheMisses++;
|
|
724
|
+
}
|
|
725
|
+
getMetrics() {
|
|
726
|
+
const totalOps = this.metrics.totalOperations;
|
|
727
|
+
const totalLookups = this.metrics.cacheHits + this.metrics.cacheMisses;
|
|
728
|
+
const avgLatency = totalOps > 0 ? this.metrics.totalLatency / totalOps : 0;
|
|
729
|
+
const successRate = totalOps > 0 ? this.metrics.successfulOperations / totalOps : 0;
|
|
730
|
+
const hitRate = totalLookups > 0 ? this.metrics.cacheHits / totalLookups : 0;
|
|
731
|
+
return {
|
|
732
|
+
totalOperations: this.metrics.totalOperations,
|
|
733
|
+
successfulOperations: this.metrics.successfulOperations,
|
|
734
|
+
failedOperations: this.metrics.failedOperations,
|
|
735
|
+
averageLatency: Math.round(avgLatency),
|
|
736
|
+
successRate: Math.round(successRate * 100) / 100,
|
|
737
|
+
connections: this.metrics.connections,
|
|
738
|
+
errors: this.metrics.errors,
|
|
739
|
+
lastOperationTime: this.metrics.lastOperationTime,
|
|
740
|
+
cacheHits: this.metrics.cacheHits,
|
|
741
|
+
cacheMisses: this.metrics.cacheMisses,
|
|
742
|
+
hitRate: Math.round(hitRate * 1000) / 1000,
|
|
743
|
+
};
|
|
744
|
+
}
|
|
745
|
+
}
|