pg-boss 12.5.3 → 12.6.0

package/dist/manager.js

@@ -26,6 +26,8 @@ class Manager extends EventEmitter {
     queues;
     pendingOffWorkCleanups;
     #spies;
+    #localGroupActive;
+    #localGroupConfig;
     constructor(db, config) {
         super();
         this.config = config;
@@ -35,6 +37,8 @@ class Manager extends EventEmitter {
         this.queues = null;
         this.pendingOffWorkCleanups = new Set();
         this.#spies = new Map();
+        this.#localGroupActive = new Map();
+        this.#localGroupConfig = new Map();
     }
     getSpy(name) {
         if (!this.config.__test__enableSpies) {
@@ -53,6 +57,138 @@ class Manager extends EventEmitter {
         }
         this.#spies.clear();
     }
+    #getLocalGroupLimit(queueName, groupTier) {
+        const config = this.#localGroupConfig.get(queueName);
+        if (!config)
+            return Infinity;
+        if (groupTier && config.tiers && groupTier in config.tiers) {
+            return config.tiers[groupTier];
+        }
+        return config.default;
+    }
+    #getGroupsAtLocalCapacity(queueName) {
+        const config = this.#localGroupConfig.get(queueName);
+        if (!config)
+            return [];
+        const queueGroups = this.#localGroupActive.get(queueName);
+        if (!queueGroups)
+            return [];
+        const atCapacity = [];
+        for (const [groupId, activeCount] of queueGroups.entries()) {
+            // We don't have tier info here, so use default limit
+            // Jobs with tiers will be checked individually after fetch
+            const limit = config.default;
+            if (activeCount >= limit) {
+                atCapacity.push(groupId);
+            }
+        }
+        return atCapacity;
+    }
+    #incrementLocalGroupCount(queueName, groupId) {
+        let queueGroups = this.#localGroupActive.get(queueName);
+        if (!queueGroups) {
+            queueGroups = new Map();
+            this.#localGroupActive.set(queueName, queueGroups);
+        }
+        const current = queueGroups.get(groupId) || 0;
+        queueGroups.set(groupId, current + 1);
+    }
+    #decrementLocalGroupCount(queueName, groupId) {
+        const queueGroups = this.#localGroupActive.get(queueName);
+        if (!queueGroups)
+            return;
+        const current = queueGroups.get(groupId) || 0;
+        if (current <= 1) {
+            queueGroups.delete(groupId);
+        }
+        else {
+            queueGroups.set(groupId, current - 1);
+        }
+    }
+    #trackJobsActive(name, jobs) {
+        const spy = this.config.__test__enableSpies ? this.#spies.get(name) : undefined;
+        if (spy) {
+            for (const job of jobs) {
+                spy.addJob(job.id, name, job.data, 'active');
+            }
+        }
+    }
+    #trackJobsCompleted(name, jobs, result) {
+        const spy = this.config.__test__enableSpies ? this.#spies.get(name) : undefined;
+        if (spy) {
+            const output = jobs.length === 1 ? result : undefined;
+            for (const job of jobs) {
+                spy.addJob(job.id, name, job.data, 'completed', output);
+            }
+        }
+    }
+    #trackJobsFailed(name, jobs, err) {
+        const spy = this.config.__test__enableSpies ? this.#spies.get(name) : undefined;
+        if (spy) {
+            for (const job of jobs) {
+                spy.addJob(job.id, name, job.data, 'failed', { message: err?.message, stack: err?.stack });
+            }
+        }
+    }
+    #storeLocalGroupConfig(name, localGroupConcurrency) {
+        const config = typeof localGroupConcurrency === 'number'
+            ? { default: localGroupConcurrency }
+            : localGroupConcurrency;
+        this.#localGroupConfig.set(name, config);
+    }
+    #cleanupLocalGroupTracking(name) {
+        // Only cleanup if no more workers exist for this queue
+        const hasWorkersForQueue = this.getWorkers().some(w => w.name === name && !w.stopping && !w.stopped);
+        if (!hasWorkersForQueue) {
+            this.#localGroupConfig.delete(name);
+            this.#localGroupActive.delete(name);
+        }
+    }
+    #trackLocalGroupStart(name, jobs) {
+        const allowed = [];
+        const excess = [];
+        const groupedJobs = [];
+        for (const job of jobs) {
+            if (!job.groupId) {
+                // Jobs without group bypass local group limits
+                allowed.push(job);
+                continue;
+            }
+            const currentCount = this.#localGroupActive.get(name)?.get(job.groupId) || 0;
+            const limit = this.#getLocalGroupLimit(name, job.groupTier);
+            if (currentCount < limit) {
+                this.#incrementLocalGroupCount(name, job.groupId);
+                allowed.push(job);
+                groupedJobs.push(job);
+            }
+            else {
+                excess.push(job);
+            }
+        }
+        return { allowed, excess, groupedJobs };
+    }
+    #trackLocalGroupEnd(name, groupedJobs) {
+        for (const job of groupedJobs) {
+            if (job.groupId) {
+                this.#decrementLocalGroupCount(name, job.groupId);
+            }
+        }
+    }
+    async #processJobs(name, jobs, callback) {
+        const jobIds = jobs.map(job => job.id);
+        const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expireInSeconds), 0);
+        const ac = new AbortController();
+        jobs.forEach(job => { job.signal = ac.signal; });
+        try {
+            const result = await resolveWithinSeconds(callback(jobs), maxExpiration, `handler execution exceeded ${maxExpiration}s`, ac);
+            await this.complete(name, jobIds, jobIds.length === 1 ? result : undefined);
+            this.#trackJobsCompleted(name, jobs, result);
+        }
+        catch (err) {
+            await this.fail(name, jobIds, err);
+            this.#trackJobsFailed(name, jobs, err);
+        }
+    }
     async start() {
         this.stopped = false;
         this.queueCacheInterval = setInterval(() => this.onCacheQueues({ emit: true }), this.config.queueCacheIntervalSeconds * 1000);
@@ -87,6 +223,9 @@ class Manager extends EventEmitter {
         await Promise.allSettled([...this.workers.values()]
             .filter(worker => !INTERNAL_QUEUES[worker.name])
             .map(async (worker) => await this.offWork(worker.name, { wait: false })));
+        // Clean up all local group tracking on full stop
+        this.#localGroupConfig.clear();
+        this.#localGroupActive.clear();
     }
     async failWip() {
         for (const worker of this.workers.values()) {
@@ -101,53 +240,59 @@ class Manager extends EventEmitter {
         if (this.stopped) {
             throw new Error('Workers are disabled. pg-boss is stopped');
         }
-        const { pollingInterval: interval, batchSize = 1, includeMetadata = false, priority = true } = options;
-        const id = randomUUID({ disableEntropyCache: true });
-        const fetch = () => this.fetch(name, { batchSize, includeMetadata, priority });
-        const onFetch = async (jobs) => {
-            if (!jobs.length) {
-                return;
-            }
-            if (this.config.__test__throw_worker) {
-                throw new Error('__test__throw_worker');
-            }
-            this.emitWip(name);
-            const spy = this.config.__test__enableSpies ? this.#spies.get(name) : undefined;
-            if (spy) {
-                for (const job of jobs) {
-                    spy.addJob(job.id, name, job.data, 'active');
+        const { pollingInterval: interval, batchSize = 1, includeMetadata = false, priority = true, localConcurrency = 1, localGroupConcurrency, groupConcurrency } = options;
+        if (localGroupConcurrency != null) {
+            this.#storeLocalGroupConfig(name, localGroupConcurrency);
+        }
+        const firstWorkerId = randomUUID({ disableEntropyCache: true });
+        const createWorker = (workerId) => {
+            const fetch = () => {
+                const ignoreGroups = localGroupConcurrency != null
+                    ? this.#getGroupsAtLocalCapacity(name)
+                    : undefined;
+                return this.fetch(name, { batchSize, includeMetadata, priority, groupConcurrency, ignoreGroups });
+            };
+            const onFetch = async (jobs) => {
+                if (!jobs.length)
+                    return;
+                if (this.config.__test__throw_worker)
+                    throw new Error('__test__throw_worker');
+                this.emitWip(name);
+                this.#trackJobsActive(name, jobs);
+                // Skip all in-memory group tracking when localGroupConcurrency is not enabled
+                if (localGroupConcurrency == null) {
+                    await this.#processJobs(name, jobs, callback);
                 }
-            }
-            const maxExpiration = jobs.reduce((acc, i) => Math.max(acc, i.expireInSeconds), 0);
-            const jobIds = jobs.map(job => job.id);
-            const ac = new AbortController();
-            jobs.forEach(job => { job.signal = ac.signal; });
-            try {
-                const result = await resolveWithinSeconds(callback(jobs), maxExpiration, `handler execution exceeded ${maxExpiration}s`, ac);
-                await this.complete(name, jobIds, jobIds.length === 1 ? result : undefined);
-                if (spy) {
-                    for (const job of jobs) {
-                        spy.addJob(job.id, name, job.data, 'completed', jobIds.length === 1 ? result : undefined);
+                else {
+                    const { allowed, excess, groupedJobs } = this.#trackLocalGroupStart(name, jobs);
+                    if (excess.length > 0) {
+                        const excessIds = excess.map(job => job.id);
+                        await this.restore(name, excessIds);
                     }
-                }
-            }
-            catch (err) {
-                await this.fail(name, jobIds, err);
-                if (spy) {
-                    for (const job of jobs) {
-                        spy.addJob(job.id, name, job.data, 'failed', { message: err?.message, stack: err?.stack });
+                    if (allowed.length > 0) {
+                        try {
+                            await this.#processJobs(name, allowed, callback);
+                        }
+                        finally {
+                            this.#trackLocalGroupEnd(name, groupedJobs);
+                        }
                     }
                 }
-            }
-            this.emitWip(name);
-        };
-        const onError = (error) => {
-            this.emit(events.error, { ...error, message: error.message, stack: error.stack, queue: name, worker: id });
+                this.emitWip(name);
+            };
+            const onError = (error) => {
+                this.emit(events.error, { ...error, message: error.message, stack: error.stack, queue: name, worker: workerId });
+            };
+            return new Worker({ id: workerId, name, options, interval, fetch, onFetch, onError });
         };
-        const worker = new Worker({ id, name, options, interval, fetch, onFetch, onError });
-        this.addWorker(worker);
-        worker.start();
-        return id;
+        // Spawn workers based on localConcurrency setting
+        for (let i = 0; i < localConcurrency; i++) {
+            const workerId = i === 0 ? firstWorkerId : randomUUID({ disableEntropyCache: true });
+            const worker = createWorker(workerId);
+            this.addWorker(worker);
+            worker.start();
+        }
+        return firstWorkerId;
     }
     addWorker(worker) {
         this.workers.set(worker.id, worker);
@@ -191,10 +336,14 @@ class Manager extends EventEmitter {
         }));
         if (options.wait) {
             await cleanupPromise;
+            this.#cleanupLocalGroupTracking(name);
         }
         else {
             this.pendingOffWorkCleanups.add(cleanupPromise);
-            cleanupPromise.finally(() => this.pendingOffWorkCleanups.delete(cleanupPromise));
+            cleanupPromise.finally(() => {
+                this.pendingOffWorkCleanups.delete(cleanupPromise);
+                this.#cleanupLocalGroupTracking(name);
+            });
         }
     }
     notifyWorker(workerId) {
@@ -246,7 +395,7 @@ class Manager extends EventEmitter {
     }
     async createJob(request) {
         const { name, data = null, options = {} } = request;
-        const { id = null, db: wrapper, priority, startAfter, singletonKey = null, singletonSeconds, singletonNextSlot, expireInSeconds, deleteAfterSeconds, retentionSeconds, keepUntil, retryLimit, retryDelay, retryBackoff, retryDelayMax } = options;
+        const { id = null, db: wrapper, priority, startAfter, singletonKey = null, singletonSeconds, singletonNextSlot, expireInSeconds, deleteAfterSeconds, retentionSeconds, keepUntil, retryLimit, retryDelay, retryBackoff, retryDelayMax, group } = options;
         const job = {
             id,
             name,
@@ -256,6 +405,8 @@ class Manager extends EventEmitter {
             singletonKey,
             singletonSeconds,
             singletonOffset: 0,
+            groupId: group?.id ?? null,
+            groupTier: group?.tier ?? null,
             expireInSeconds,
             deleteAfterSeconds,
             retentionSeconds,
@@ -391,21 +542,21 @@ class Manager extends EventEmitter {
         const result = await db.executeSql(sql, [name, ids, this.mapCompletionDataArg(data)]);
         return this.mapCommandResponse(ids, result);
     }
-    async cancel(name, id, options = {}) {
+    async deleteJob(name, id, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'cancel');
+        const ids = this.mapCompletionIdArg(id, 'deleteJob');
         const { table } = await this.getQueueCache(name);
-        const sql = plans.cancelJobs(this.config.schema, table);
+        const sql = plans.deleteJobsById(this.config.schema, table);
         const result = await db.executeSql(sql, [name, ids]);
         return this.mapCommandResponse(ids, result);
     }
-    async deleteJob(name, id, options = {}) {
+    async cancel(name, id, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'deleteJob');
+        const ids = this.mapCompletionIdArg(id, 'cancel');
         const { table } = await this.getQueueCache(name);
-        const sql = plans.deleteJobsById(this.config.schema, table);
+        const sql = plans.cancelJobs(this.config.schema, table);
         const result = await db.executeSql(sql, [name, ids]);
         return this.mapCommandResponse(ids, result);
     }
@@ -418,6 +569,14 @@ class Manager extends EventEmitter {
         const result = await db.executeSql(sql, [name, ids]);
         return this.mapCommandResponse(ids, result);
     }
+    async restore(name, id, options = {}) {
+        Attorney.assertQueueName(name);
+        const db = this.assertDb(options);
+        const ids = this.mapCompletionIdArg(id, 'restore');
+        const { table } = await this.getQueueCache(name);
+        const sql = plans.restoreJobs(this.config.schema, table);
+        await db.executeSql(sql, [name, ids]);
+    }
     async retry(name, id, options = {}) {
         Attorney.assertQueueName(name);
         const db = options.db || this.db;
@@ -540,6 +699,27 @@ class Manager extends EventEmitter {
             return null;
         }
     }
+    async findJobs(name, options = {}) {
+        Attorney.assertQueueName(name);
+        const db = this.assertDb(options);
+        const { table } = await this.getQueueCache(name);
+        const { id, key, data, queued = false } = options;
+        const sql = plans.findJobs(this.config.schema, table, {
+            byId: id !== undefined,
+            byKey: key !== undefined,
+            byData: data !== undefined,
+            queued
+        });
+        const values = [name];
+        if (id !== undefined)
+            values.push(id);
+        if (key !== undefined)
+            values.push(key);
+        if (data !== undefined)
+            values.push(JSON.stringify(data));
+        const result = await db.executeSql(sql, values);
+        return result?.rows || [];
+    }
     assertDb(options) {
         if (options.db) {
             return options.db;

package/dist/migrationStore.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"migrationStore.d.ts","sourceRoot":"","sources":["../src/migrationStore.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AASnC,iBAAS,QAAQ,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,UAQjF;AAED,iBAAS,IAAI,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,SAAS,UAQxF;AAED,iBAAS,OAAO,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,UAehF;AAED,iBAAS,MAAM,CAAE,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CA0FlD;AAED,OAAO,EACL,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,MAAM,GACP,CAAA"}
+{"version":3,"file":"migrationStore.d.ts","sourceRoot":"","sources":["../src/migrationStore.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AASnC,iBAAS,QAAQ,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,UAQjF;AAED,iBAAS,IAAI,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,SAAS,UAQxF;AAED,iBAAS,OAAO,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,UAuBhF;AAED,iBAAS,MAAM,CAAE,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAkYlD;AAED,OAAO,EACL,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,MAAM,GACP,CAAA"}