vidspotai-shared 1.0.72 → 1.0.73

package/lib/services/bullmq.service.d.ts

@@ -10,11 +10,36 @@ declare class BullMQService {
     private queues;
     private workers;
     private redisOptions;
+    private connection?;
     private concurrency;
     private initialized;
     private shuttingDown;
     private constructor();
     static getInstance(options: BullMQServiceOptions): BullMQService;
+    /**
+     * Lazily create and return the single shared ioredis connection for this
+     * process. ALL Queues and Workers must use this instance rather than a plain
+     * options object.
+     *
+     * WHY THIS EXISTS — Redis client exhaustion:
+     * Passing a plain RedisOptions object to `new Queue`/`new Worker` makes
+     * BullMQ open a BRAND-NEW connection for every component, so total
+     * connections scale as (instances × queues). With 7 queues across N warm
+     * Cloud Run instances + the worker fleet, this blew past the Redis
+     * `maxclients` cap → "ERR max number of clients reached".
+     *
+     * Passing a shared IORedis instance instead:
+     *   - Queues reuse this one connection (0 extra per queue).
+     *   - Workers reuse it for non-blocking commands and BullMQ internally
+     *     `.duplicate()`s it for each worker's MANDATORY blocking client
+     *     (one blocking conn per worker is unavoidable by BullMQ design).
+     * Net: a producer (Functions) holds exactly 1 connection regardless of how
+     * many queues it enqueues to; a worker host holds 1 + (1 per worker).
+     *
+     * `maxRetriesPerRequest: null` (in sharedRedisOptions) is REQUIRED by BullMQ
+     * for the connection it blocks on.
+     */
+    private getConnection;
     /** Initialize BullMQ service (like your initRedis) */
     init(): Promise<void>;
     /** Add a job */

package/lib/services/bullmq.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bullmq.service.d.ts","sourceRoot":"","sources":["../../src/services/bullmq.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAiB,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAIvC,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAE1C,UAAU,oBAAoB;IAC5B,YAAY,EAAE,YAAY,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,cAAM,aAAa;IACjB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAgB;IACvC,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAS;IAE7B,OAAO;WAQO,WAAW,CAAC,OAAO,EAAE,oBAAoB;IAOvD,sDAAsD;IACzC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBlC,gBAAgB;IACH,MAAM,CACjB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,OAAO,EACb,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IAUpC,YAAY,CACjB,UAAU,EAAE,MAAM,EAAE,EACpB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC,EACtC,eAAe,CAAC,EAAE,MAAM,CACtB,MAAM,EACN;QACE,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CACF;CA+FJ;AASD,eAAO,MAAM,MAAM,eAGjB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,GAAG,CAAC"}
+{"version":3,"file":"bullmq.service.d.ts","sourceRoot":"","sources":["../../src/services/bullmq.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAiB,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAgB,EAAwB,YAAY,EAAE,MAAM,SAAS,CAAC;AAItE,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAE1C,UAAU,oBAAoB;IAC5B,YAAY,EAAE,YAAY,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,cAAM,aAAa;IACjB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAgB;IACvC,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,YAAY,CAAgB;IAIpC,OAAO,CAAC,UAAU,CAAC,CAAc;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAS;IAE7B,OAAO;WAQO,WAAW,CAAC,OAAO,EAAE,oBAAoB;IAOvD;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,aAAa;IAOrB,sDAAsD;IACzC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBlC,gBAAgB;IACH,MAAM,CACjB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,OAAO,EACb,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IAUpC,YAAY,CACjB,UAAU,EAAE,MAAM,EAAE,EACpB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC,EACtC,eAAe,CAAC,EAAE,MAAM,CACtB,MAAM,EACN;QACE,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CACF;CAwGJ;AASD,eAAO,MAAM,MAAM,eAGjB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,GAAG,CAAC"}

package/lib/services/bullmq.service.js

@@ -1,8 +1,12 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.bullmq = void 0;
 // libs/bullmqService.ts
 const bullmq_1 = require("bullmq");
+const ioredis_1 = __importDefault(require("ioredis"));
 const logger_1 = require("../utils/logger");
 const redisOptions_1 = require("./redisOptions");
 class BullMQService {
@@ -20,19 +24,44 @@ class BullMQService {
         }
         return BullMQService.instance;
     }
+    /**
+     * Lazily create and return the single shared ioredis connection for this
+     * process. ALL Queues and Workers must use this instance rather than a plain
+     * options object.
+     *
+     * WHY THIS EXISTS — Redis client exhaustion:
+     * Passing a plain RedisOptions object to `new Queue`/`new Worker` makes
+     * BullMQ open a BRAND-NEW connection for every component, so total
+     * connections scale as (instances × queues). With 7 queues across N warm
+     * Cloud Run instances + the worker fleet, this blew past the Redis
+     * `maxclients` cap → "ERR max number of clients reached".
+     *
+     * Passing a shared IORedis instance instead:
+     *   - Queues reuse this one connection (0 extra per queue).
+     *   - Workers reuse it for non-blocking commands and BullMQ internally
+     *     `.duplicate()`s it for each worker's MANDATORY blocking client
+     *     (one blocking conn per worker is unavoidable by BullMQ design).
+     * Net: a producer (Functions) holds exactly 1 connection regardless of how
+     * many queues it enqueues to; a worker host holds 1 + (1 per worker).
+     *
+     * `maxRetriesPerRequest: null` (in sharedRedisOptions) is REQUIRED by BullMQ
+     * for the connection it blocks on.
+     */
+    getConnection() {
+        if (this.connection)
+            return this.connection;
+        this.connection = new ioredis_1.default(this.redisOptions);
+        (0, redisOptions_1.attachRedisListeners)(this.connection, "BullMQ");
+        return this.connection;
+    }
     /** Initialize BullMQ service (like your initRedis) */
     async init() {
         if (this.initialized)
             return;
         try {
-            // Create a temporary test queue
-            const testQueue = new bullmq_1.Queue("__init__", {
-                connection: this.redisOptions,
-            });
-            // Ping Redis via ioredis client
-            const testQueueClient = await testQueue.client;
-            testQueueClient.ping();
-            await testQueue.close();
+            // Ping the shared connection directly — no throwaway Queue (which would
+            // open, then close, an extra connection on every cold start).
+            await this.getConnection().ping();
             this.initialized = true;
             logger_1.logger.info("BullMQService initialized and connected to Redis");
         }
@@ -48,7 +77,7 @@ class BullMQService {
         if (!this.initialized)
             await this.init();
         const queue = this.queues.get(queueName) ??
-            new bullmq_1.Queue(queueName, { connection: this.redisOptions });
+            new bullmq_1.Queue(queueName, { connection: this.getConnection() });
         this.queues.set(queueName, queue);
         await queue.add(opts?.jobId || "default", data, { delay: opts?.delay });
     }
@@ -58,11 +87,11 @@ class BullMQService {
                 await this.init();
             for (const queueName of queueNames) {
                 const queue = this.queues.get(queueName) ??
-                    new bullmq_1.Queue(queueName, { connection: this.redisOptions });
+                    new bullmq_1.Queue(queueName, { connection: this.getConnection() });
                 this.queues.set(queueName, queue);
                 const overrides = perQueueOptions?.[queueName] ?? {};
                 const worker = new bullmq_1.Worker(queueName, async (job) => processor(job), {
-                    connection: this.redisOptions,
+                    connection: this.getConnection(),
                     concurrency: overrides.concurrency ?? this.concurrency,
                     // N5 (agent refactor Stage 2). Explicit lock semantics:
                     //   - lockDuration: 60s default. BullMQ auto-renews while the
@@ -122,6 +151,15 @@ class BullMQService {
                     await worker.close();
                     logger_1.logger.info(`Worker for queue closed`, { queueName: worker.name });
                 }
+                // Close queues and the shared connection so blocking-client dups are
+                // released cleanly (prevents lingering clients on the Redis side after
+                // a rolling deploy / SIGTERM on Railway).
+                for (const queue of this.queues.values()) {
+                    await queue.close();
+                }
+                if (this.connection) {
+                    await this.connection.quit();
+                }
                 logger_1.logger.info("All workers closed. Exiting process.");
                 process.exit(0);
             };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vidspotai-shared",
-  "version": "1.0.72",
+  "version": "1.0.73",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "exports": {