redis-graph-cache 1.0.0

package/dist/core/normalization-engine.js

@@ -0,0 +1,286 @@
+"use strict";
+/**
+ * Normalization Engine - Converts nested objects into normalized Redis storage
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NormalizationEngine = void 0;
+const compression_1 = require("./compression");
+const serializer_1 = require("./serializer");
+const types_1 = require("../types");
+class NormalizationEngine {
+    constructor(schemaManager, connectionManager, codec = compression_1.NOOP_CODEC, 
+    // Pluggable (de)serializer. Defaults to the engine's lossless
+    // tagged JSON serializer so callers don't have to opt in to get
+    // correct `Date`/`BigInt`/`Map`/`Set` round-tripping. Passed in
+    // by `RedisSchemaEngine` so reads and writes share one instance.
+    serializer = serializer_1.TAGGED_SERIALIZER) {
+        this.schemaManager = schemaManager;
+        this.connectionManager = connectionManager;
+        this.codec = codec;
+        this.serializer = serializer;
+    }
+    /**
+     * Normalize a complex nested entity into separate Redis keys
+     */
+    async normalizeEntity(entityType, data, operationId = this.generateOperationId()) {
+        const schema = this.schemaManager.getEntitySchema(entityType);
+        const normalizedEntities = new Map();
+        // Normalize the main entity and all nested entities
+        await this.normalizeRecursive(entityType, data, normalizedEntities, operationId);
+        const entityId = data[schema.id];
+        const primaryEntityKey = this.schemaManager.generateEntityKey(entityType, entityId);
+        return {
+            normalizedEntities,
+            primaryEntityKey,
+            operationId,
+        };
+    }
+    /**
+     * Write normalized entities to Redis using a per-key compare-and-set so
+     * concurrent writers cannot clobber each other's partial updates. The
+     * caller-supplied entity-type lookup determines TTL per key.
+     *
+     * Flow per key:
+     *   1. GET current value
+     *   2. Merge in Node (preserves "missing fields keep existing" semantics)
+     *   3. CAS write via Lua. If another writer raced us, retry up to
+     *      `maxCasAttempts` times with the freshly read value.
+     *
+     * All keys are processed in parallel (Promise.all) and ioredis pipelines
+     * them automatically across the same connection, so latency stays close
+     * to the original implementation while gaining atomicity.
+     */
+    async writeNormalizedEntities(normalizationResult, resolveTTL = () => 0) {
+        const { normalizedEntities, operationId } = normalizationResult;
+        const keys = Array.from(normalizedEntities.keys());
+        try {
+            await Promise.all(keys.map((key) => this.atomicMergeWrite(key, normalizedEntities.get(key), resolveTTL(key))));
+            return {
+                success: true,
+                keys,
+                operationId,
+                timestamp: Date.now(),
+            };
+        }
+        catch (error) {
+            throw new types_1.RedisConnectionError('Failed to write normalized entities', error);
+        }
+    }
+    /**
+     * Per-key compare-and-set merge with bounded retry. Each retry re-reads
+     * the current value so the merge is always against the freshest view.
+     * Surfaces a RedisConnectionError if the contention budget is exhausted,
+     * which is the right behaviour: callers should retry at the request level
+     * rather than spin forever inside a single write.
+     */
+    async atomicMergeWrite(key, incoming, ttlSeconds) {
+        const maxCasAttempts = 5;
+        for (let attempt = 1; attempt <= maxCasAttempts; attempt++) {
+            // currentRaw is the literal bytes stored in Redis; it may be a
+            // compressed envelope (`\x00z1:...base64...`) or plain JSON. We
+            // pass it back to CAS as `expected` unchanged so byte equality
+            // holds against whatever is actually stored.
+            const currentRaw = await this.connectionManager.get(key);
+            let merged;
+            if (currentRaw) {
+                // Decode for the merge step only; the merge needs structured
+                // data, not the storage envelope. Routing through the shared
+                // serializer ensures previously stored Dates/BigInts/Maps/etc.
+                // come back as their original JS types so smartMerge sees the
+                // same structure the original writer wrote.
+                let existing = null;
+                try {
+                    existing = this.serializer.parse(this.codec.decode(currentRaw));
+                }
+                catch {
+                    existing = null;
+                }
+                merged = existing ? this.smartMerge(existing, incoming) : incoming;
+            }
+            else {
+                merged = incoming;
+            }
+            // Serialize through the configured serializer so non-JSON-native
+            // values (Date, BigInt, Map, Set, RegExp, Buffer, NaN/Infinity)
+            // are tagged for lossless recovery on read. Plain JSON-native
+            // payloads still produce identical bytes to raw JSON.stringify.
+            const newJson = this.serializer.stringify(merged);
+            // Encode for storage. May or may not actually compress depending
+            // on the codec config and payload size; either way the result is
+            // a single string Redis stores verbatim.
+            const newEncoded = this.codec.encode(newJson);
+            const expected = currentRaw ?? '';
+            const ok = await this.connectionManager.casSet(key, expected, newEncoded, ttlSeconds);
+            if (ok)
+                return;
+        }
+        throw new types_1.RedisConnectionError(`CAS contention: failed to write '${key}' after retries`);
+    }
+    /**
+     * Recursively normalize entity and its relationships
+     */
+    async normalizeRecursive(entityType, data, result, operationId, visited = new Set()) {
+        if (!data || typeof data !== 'object') {
+            return;
+        }
+        const schema = this.schemaManager.getEntitySchema(entityType);
+        const entityId = data[schema.id];
+        if (!entityId) {
+            throw new types_1.InvalidOperationError('normalize', `Missing required ID field '${schema.id}' in entity '${entityType}'`, { entityType, data });
+        }
+        const entityKey = this.schemaManager.generateEntityKey(entityType, entityId);
+        // Prevent infinite recursion
+        const visitKey = `${entityType}:${entityId}`;
+        if (visited.has(visitKey)) {
+            return;
+        }
+        visited.add(visitKey);
+        // Create normalized entity
+        const normalizedEntity = {};
+        // Add metadata
+        normalizedEntity[types_1.VERSION_FIELD] = schema.version || '1.0.0';
+        normalizedEntity[types_1.TIMESTAMP_FIELD] = Date.now();
+        normalizedEntity[types_1.METADATA_FIELD] = {
+            entityType,
+            originalId: entityId,
+            relationshipIds: {},
+            lastUpdated: Date.now(),
+            source: operationId,
+        };
+        // Process regular fields
+        if (schema.fields) {
+            for (const [fieldName, fieldDef] of Object.entries(schema.fields)) {
+                if (fieldName in data) {
+                    normalizedEntity[fieldName] = data[fieldName];
+                }
+            }
+        }
+        // Process relationships
+        if (schema.relations) {
+            for (const [relationName, relationDef] of Object.entries(schema.relations)) {
+                if (relationName in data) {
+                    await this.processRelationship(relationName, relationDef, data[relationName], normalizedEntity, result, operationId, visited);
+                }
+            }
+        }
+        // Store normalized entity
+        result.set(entityKey, normalizedEntity);
+    }
+    /**
+     * Process a relationship and normalize related entities
+     */
+    async processRelationship(relationName, relationDef, relationData, parentEntity, result, operationId, visited) {
+        const relatedSchema = this.schemaManager.getEntitySchema(relationDef.type);
+        const relationIdField = (0, types_1.generateRelationIdField)(relationName, relationDef.kind);
+        if (relationDef.kind === 'one') {
+            // One-to-one relationship
+            if (relationData && typeof relationData === 'object') {
+                const relatedId = relationData[relatedSchema.id];
+                if (relatedId) {
+                    parentEntity[relationIdField] = relatedId;
+                    // Store relationship ID in metadata
+                    const metadata = parentEntity[types_1.METADATA_FIELD];
+                    metadata.relationshipIds[relationName] = relatedId;
+                    // Recursively normalize related entity
+                    await this.normalizeRecursive(relationDef.type, relationData, result, operationId, visited);
+                }
+            }
+        }
+        else if (relationDef.kind === 'many') {
+            // One-to-many relationship
+            if (Array.isArray(relationData)) {
+                const relatedIds = [];
+                for (const relatedItem of relationData) {
+                    if (relatedItem && typeof relatedItem === 'object') {
+                        const relatedId = relatedItem[relatedSchema.id];
+                        if (relatedId) {
+                            relatedIds.push(relatedId);
+                            // Recursively normalize related entity
+                            await this.normalizeRecursive(relationDef.type, relatedItem, result, operationId, visited);
+                        }
+                    }
+                }
+                if (relatedIds.length > 0) {
+                    parentEntity[relationIdField] = relatedIds;
+                    // Store relationship IDs in metadata
+                    const metadata = parentEntity[types_1.METADATA_FIELD];
+                    metadata.relationshipIds[relationName] = relatedIds;
+                }
+            }
+        }
+    }
+    /**
+     * Smart merge strategy based on field types
+     */
+    smartMerge(existingEntity, newEntity) {
+        const merged = { ...existingEntity };
+        for (const [key, newValue] of Object.entries(newEntity)) {
+            const existingValue = existingEntity[key];
+            if (key.startsWith(types_1.INTERNAL_FIELD_PREFIX)) {
+                // Always update internal fields
+                merged[key] = newValue;
+            }
+            else if (this.isPrimitiveValue(newValue)) {
+                // Primitive fields: replace with new value
+                merged[key] = newValue;
+            }
+            else if (Array.isArray(newValue)) {
+                // Array fields: replace entire array
+                merged[key] = newValue;
+            }
+            else if (this.isObject(newValue) && this.isObject(existingValue)) {
+                // Object fields: deep merge
+                merged[key] = this.deepMergeObjects(existingValue, newValue);
+            }
+            else {
+                // Default: replace with new value
+                merged[key] = newValue;
+            }
+        }
+        // Update timestamp
+        merged[types_1.TIMESTAMP_FIELD] = Date.now();
+        return merged;
+    }
+    /**
+     * Check if value is primitive (string, number, boolean, null, undefined)
+     */
+    isPrimitiveValue(value) {
+        return (value === null ||
+            value === undefined ||
+            typeof value === 'string' ||
+            typeof value === 'number' ||
+            typeof value === 'boolean');
+    }
+    /**
+     * Check if value is a plain object
+     */
+    isObject(value) {
+        return (value !== null &&
+            typeof value === 'object' &&
+            !Array.isArray(value) &&
+            value.constructor === Object);
+    }
+    /**
+     * Deep merge two objects
+     */
+    deepMergeObjects(existing, incoming) {
+        const result = { ...existing };
+        for (const [key, value] of Object.entries(incoming)) {
+            const existingValue = existing[key];
+            if (this.isObject(value) && this.isObject(existingValue)) {
+                result[key] = this.deepMergeObjects(existingValue, value);
+            }
+            else {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+    /**
+     * Generate unique operation ID
+     */
+    generateOperationId() {
+        return `op_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
+    }
+}
+exports.NormalizationEngine = NormalizationEngine;

package/dist/core/redis-connection-manager.d.ts

@@ -0,0 +1,255 @@
+/**
+ * Redis Connection Manager
+ *
+ * Owns the ioredis client and exposes a small set of typed methods that all
+ * Redis traffic flows through. Every call goes through a circuit breaker +
+ * retry wrapper, records latency, and updates cache hit/miss counters where
+ * applicable. Atomic Lua scripts are registered on construction and exposed
+ * as typed helpers (casSet, setIfExists, listAdd, listRemove) so callers
+ * never need direct access to the underlying client for normal work.
+ */
+import { Redis as RedisInstance, ChainableCommander } from 'ioredis';
+import { RedisConfig, CircuitBreakerConfig, RetryConfig, CircuitBreakerState } from '../types';
+export declare class RedisConnectionManager {
+    private config;
+    /**
+     * The pool of ioredis clients. Default size 1 (a single connection)
+     * for backward compatibility. When `redis.poolSize` is configured
+     * higher, every hot-path command is round-robined across these
+     * clients, giving genuine parallelism on the Redis socket.
+     */
+    private clients;
+    private rrIndex;
+    /**
+     * Per-client connected flags. `isConnected` (below) is the OR across
+     * the pool, so health checks succeed when *any* client is up.
+     */
+    private connectedFlags;
+    private circuitBreaker;
+    private retryManager;
+    private healthMetrics;
+    /**
+     * Typed view of `this.redis` that includes the Lua-script methods
+     * registered via `defineCommand`. Use this whenever you need to
+     * call an `rse*` script or a variadic ZRANGEBYSCORE-family command;
+     * ordinary commands (`get`, `set`, `del`, `exists`, etc.) are typed
+     * on the base instance and should keep using `this.redis`.
+     *
+     * Calling this advances the round-robin index identically to
+     * `this.redis`, so a single call site stays pinned to one client.
+     */
+    private get scripted();
+    /**
+     * Round-robin client picker. Hot-path methods read `this.redis` once
+     * per call which advances the index, distributing load uniformly.
+     * Pipelines obtained via `this.redis.pipeline()` are owned by a
+     * single client because the getter is called exactly once.
+     */
+    private get redis();
+    /**
+     * True when at least one pool client reports connected. Health checks
+     * succeed under partial pool failures, which is the right behaviour:
+     * one degraded socket shouldn't take the whole engine offline.
+     */
+    private get isConnected();
+    constructor(config: RedisConfig, circuitBreakerConfig: CircuitBreakerConfig, retryConfig: RetryConfig);
+    /**
+     * Initialize the Redis pool. Creates `poolSize` independent clients
+     * (defaulting to 1 for parity with the pre-pool implementation). Each
+     * client gets:
+     *   - the same connection options
+     *   - all atomic Lua scripts pre-registered as named commands
+     *   - the same lifecycle event handlers
+     * Failures in any single client during init bubble up as connection
+     * errors so the engine fails closed rather than silently degrading.
+     */
+    private initializeConnection;
+    /**
+     * Attach lifecycle handlers to every pool client. Per-client connect/
+     * close events flip that client's slot in `connectedFlags` so the
+     * aggregate `isConnected` getter reflects pool-wide state without
+     * extra coordination.
+     */
+    private setupEventHandlers;
+    /**
+     * Manually connect every pool client. Used when `lazyConnect: true`
+     * was passed via redis options; otherwise clients connect at
+     * construction time. Errors from any single client bubble up.
+     */
+    connect(): Promise<void>;
+    /**
+     * Execute a Redis operation through circuit breaker + retry. All public
+     * helpers below funnel through this. Latency is recorded for both success
+     * and failure so health metrics reflect real traffic.
+     */
+    executeOperation<T>(operation: () => Promise<T>, operationName?: string): Promise<T>;
+    /**
+     * GET wrapper. Records cache hit/miss for visibility into real traffic.
+     */
+    get(key: string): Promise<string | null>;
+    /**
+     * MGET wrapper. Records hit/miss per key for accurate hit-rate stats.
+     */
+    mget(keys: string[]): Promise<Array<string | null>>;
+    /**
+     * SET wrapper with optional TTL in seconds.
+     */
+    set(key: string, value: string, ttlSeconds?: number): Promise<void>;
+    /**
+     * DEL wrapper. Returns the number of keys removed.
+     */
+    del(...keys: string[]): Promise<number>;
+    /**
+     * EXISTS wrapper. Returns the number of keys that exist (0 or 1 for single).
+     */
+    exists(key: string): Promise<number>;
+    /**
+     * Compare-and-set via Lua. Writes newValue only if current value at key
+     * exactly equals expectedValue. Pass empty string to expect absence.
+     * Returns true on success, false on conflict.
+     */
+    casSet(key: string, expectedValue: string, newValue: string, ttlSeconds?: number): Promise<boolean>;
+    /**
+     * Atomic set that only writes if the key already exists. Used for
+     * "update only if cached" semantics. Returns true if applied.
+     */
+    setIfExists(key: string, value: string, ttlSeconds?: number): Promise<boolean>;
+    /**
+     * Atomic add to a JSON-array list at key. Idempotent; returns true if
+     * the id was newly inserted, false if already present.
+     */
+    listAdd(listKey: string, id: string | number, ttlSeconds?: number): Promise<boolean>;
+    /**
+     * Atomic remove of an id from a JSON-array list. Preserves the existing
+     * TTL on the key. Returns true if the id was found and removed.
+     */
+    listRemove(listKey: string, id: string | number): Promise<boolean>;
+    /**
+     * Atomic ZADD with optional max-size trim and optional membership-index
+     * update. Returns true when a brand-new member was added; false when an
+     * existing member's score was updated (still considered "success").
+     */
+    zListAdd(listKey: string, membershipKey: string, score: number, id: string | number, ttlSeconds?: number, maxSize?: number, trackMembership?: boolean): Promise<boolean>;
+    /**
+     * Atomic ZREM. Returns true if the id was a member of the set.
+     */
+    zListRemove(listKey: string, membershipKey: string, id: string | number, trackMembership?: boolean): Promise<boolean>;
+    /**
+     * Paginated read of a ZSET. Range semantics:
+     *   - `reverse: false` (default) returns members in ascending score
+     *     order (oldest first when score is timestamp).
+     *   - `reverse: true` returns members in descending score order
+     *     (newest first when score is timestamp) — the common feed case.
+     *   - `offset` and `limit` are applied AFTER the score filter.
+     *   - `minScore`/`maxScore` filter by score; defaults are -inf/+inf.
+     *
+     * Returns just the ids by default. Pass `withScores: true` to get the
+     * raw `[id, score]` pairs for downstream filtering.
+     */
+    zListRange(listKey: string, opts?: {
+        offset?: number;
+        limit?: number;
+        reverse?: boolean;
+        minScore?: number;
+        maxScore?: number;
+        withScores?: boolean;
+    }): Promise<string[] | Array<{
+        id: string;
+        score: number;
+    }>>;
+    /**
+     * Number of members currently in the ZSET. Returns 0 when the key is
+     * absent (which Redis ZCARD reports natively).
+     */
+    zListSize(listKey: string): Promise<number>;
+    /**
+     * Cascade-invalidate an entity: ZREMs it from every tracked list and
+     * deletes both the entity key and the membership back-index. Returns
+     * the number of lists the entity was removed from. Safe to call when
+     * the back-index does not exist (returns 0).
+     */
+    cascadeInvalidate(membershipKey: string, entityKey: string, id: string | number): Promise<number>;
+    /**
+     * Run an arbitrary pipeline. Internal use by the normalization engine for
+     * bulk first-write operations. Errors inside the pipeline are surfaced.
+     */
+    runPipeline(build: (pipeline: ChainableCommander) => void): Promise<any[]>;
+    /**
+     * Escape hatch returning one client from the pool. Prefer the typed
+     * helpers above so traffic is observed and breaker-protected. This is
+     * here for migrations and tests only and should not be used in hot
+     * paths.
+     */
+    getRawClient(): RedisInstance;
+    /**
+     * Return all clients in the pool. Internal use for tests verifying
+     * pool size and per-client state.
+     */
+    getAllRawClients(): readonly RedisInstance[];
+    /**
+     * FLUSHDB wrapper. Caller is responsible for guarding this; the engine
+     * adds an additional confirmation gate at the public API.
+     */
+    flushDb(): Promise<void>;
+    /**
+     * Delete every key that starts with the configured `keyPrefix`, using
+     * non-blocking `SCAN` + `UNLINK` (async delete). This is the safe
+     * equivalent of `FLUSHDB` for deployments that share a Redis DB with
+     * other applications or environments: it only touches keys this engine
+     * owns and never blocks the Redis server on large keyspaces.
+     *
+     * Implementation notes:
+     *   - `SCAN` with `{ match: '*' }` works correctly because ioredis
+     *     automatically prepends `keyPrefix` to the `MATCH` pattern on
+     *     the wire, so Redis only returns keys under our namespace.
+     *   - Redis returns the fully-prefixed keys. We strip the prefix
+     *     before calling `UNLINK`, because ioredis re-applies the prefix
+     *     on outbound commands — passing the raw prefixed keys would
+     *     produce a double prefix and delete nothing.
+     *   - Pinned to a single pool client (`clients[0]`) so the SCAN cursor
+     *     is consistent; UNLINK batches can go through any client but
+     *     using the same one keeps this simple and deterministic.
+     *   - Returns the count of deleted keys for observability.
+     *
+     * Throws if `keyPrefix` is empty; caller should use `flushDb()` in
+     * that case (FLUSHDB is still the right tool for a no-prefix setup).
+     */
+    clearPrefixedKeys(): Promise<number>;
+    /**
+     * The configured keyPrefix, or empty string if none. Consumers (e.g.
+     * `clearAllCache`) branch on this to pick between scoped delete and
+     * FLUSHDB.
+     */
+    getKeyPrefix(): string;
+    isHealthy(): Promise<boolean>;
+    /**
+     * Read Redis used_memory in bytes via the INFO command. Returns 0 if the
+     * value cannot be parsed (e.g. permission-restricted environments).
+     */
+    getRedisUsedMemory(): Promise<number>;
+    getHealthMetrics(): {
+        isConnected: boolean;
+        circuitBreakerState: CircuitBreakerState;
+        totalOperations: number;
+        successfulOperations: number;
+        failedOperations: number;
+        averageLatency: number;
+        successRate: number;
+        connections: number;
+        errors: number;
+        lastOperationTime: number;
+        cacheHits: number;
+        cacheMisses: number;
+        hitRate: number;
+    };
+    /**
+     * Cleanly close every pool client. Errors on individual clients are
+     * logged but don't stop the others from quitting; the goal here is
+     * resource cleanup, not strict success.
+     */
+    disconnect(): Promise<void>;
+    /**
+     * Number of clients in the pool. Useful for tests and observability.
+     */
+    getPoolSize(): number;
+}