pg-boss 12.34.0 → 12.35.0

package/dist/manager.js

@@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto';
 import EventEmitter from 'node:events';
 import { serializeError as stringify } from 'serialize-error';
 import * as Attorney from "./attorney.js";
+import { untracked } from "./activity.js";
 import { TRANSACTION_ROLLBACK_TIMEOUT_MS } from "./db.js";
 import * as plans from "./plans.js";
 import * as timekeeper from "./timekeeper.js";
@@ -89,10 +90,16 @@ const NUMERIC_QUEUE_FIELDS = [
     'deferredCount',
     'warningQueueSize',
     'queuedCount',
+    'readyCount',
     'activeCount',
-    'totalCount'
+    'failedCount',
+    'totalCount',
+    'createdDelta',
+    'completedDelta',
+    'failedDelta',
+    'deltaSeconds'
 ];
-// The count columns shared by live stats and recorded snapshots (the QueueStats shape).
+// The gauges shared by live stats and recorded snapshots (the QueueStats shape).
 const STATS_COUNT_FIELDS = [
     'deferredCount',
     'queuedCount',
@@ -101,6 +108,14 @@ const STATS_COUNT_FIELDS = [
     'failedCount',
     'totalCount'
 ];
+// The throughput counters and the seconds they cover. Only recorded snapshots carry them; see
+// getQueueStats.
+const STATS_DELTA_FIELDS = [
+    'completedDelta',
+    'failedDelta',
+    'createdDelta',
+    'deltaSeconds'
+];
 // Stale-cache budget for getQueueStats when persistQueueStats is off. A queue-table cache older than
 // this means monitoring isn't keeping it current (e.g. supervise was enabled once but isn't now), so
 // the counts are recomputed and re-cached instead of returned. Defaults to one hour, raised to the
@@ -152,6 +167,9 @@ class Manager extends EventEmitter {
     // that transaction's db. #processJobs reads it to interpret its own completion; see the check
     // there. Weak because the key is the transaction, so an entry goes away with it.
     #handlerSettledJobs;
+    // The retryCount each job of a transactional batch was claimed at, keyed by the transaction the
+    // handler was handed, so the handler's own settles through it can be fenced (#handlerAttempts).
+    #handlerClaims;
     // Which GUC this server bounds a transaction with, resolved on the first transactional batch and
     // kept for the life of the process. Resolving to null means neither is recognised, or the probe
     // itself failed; a failed probe clears the memo and sets the instant before which no batch asks
@@ -175,6 +193,7 @@ class Manager extends EventEmitter {
         this.pendingOffWorkCleanups = new Set();
         this.#spies = new Map();
         this.#handlerSettledJobs = new WeakMap();
+        this.#handlerClaims = new WeakMap();
         this.#transactionTimeoutGuc = null;
         this.#transactionTimeoutProbeRetryAt = 0;
         this.#transactionTimeoutProbeCooldownMs = config.__test__transactionTimeoutProbeCooldownMs ?? TRANSACTION_TIMEOUT_PROBE_COOLDOWN_MS;
@@ -334,7 +353,7 @@ class Manager extends EventEmitter {
             // The handler opted into perJobResults but did not return an array: a contract violation.
             // Fail the whole batch so the mistake surfaces and the jobs are retried.
             const err = new Error('perJobResults handler must resolve with an array of job results');
-            await this.fail(name, jobs.map(job => job.id), err);
+            await this.fail(name, jobs, err);
             await this.#trackJobsFailed(name, jobs, err);
             return;
         }
@@ -367,31 +386,33 @@ class Manager extends EventEmitter {
                 failed.push({ job, output: new Error('no disposition returned by handler') });
             }
         }
-        if (completed.length > 0) {
-            await this.#completeWithOutputs(name, completed.map(c => ({ id: c.job.id, output: c.output })));
-        }
-        if (failed.length > 0) {
-            await this.#failWithOutputs(name, failed.map(f => ({ id: f.job.id, output: f.output })));
-        }
-        if (deadLettered.length > 0) {
-            await this.#failWithOutputs(name, deadLettered.map(d => ({ id: d.job.id, output: d.output })), true);
-        }
+        const items = (entries) => entries.map(({ job, output }) => ({ id: job.id, retryCount: job.retryCount, output }));
+        const completedIds = completed.length > 0 ? await this.#completeWithOutputs(name, items(completed)) : null;
+        const failedIds = failed.length > 0 ? await this.#failWithOutputs(name, items(failed)) : null;
+        const deadLetteredIds = deadLettered.length > 0 ? await this.#failWithOutputs(name, items(deadLettered), true) : null;
+        // Only the jobs each statement actually settled: the attempt fence leaves a job whose claim
+        // lapsed alone, and recording it would tell a spy it settled when another attempt holds it.
+        const landed = (entries, result) => {
+            const settled = new Set(result?.settled ?? []);
+            return entries.filter(entry => settled.has(entry.job.id));
+        };
         // Dead lettered jobs end in the same terminal `failed` state as failed jobs on the source queue.
-        this.#trackJobsSettled(name, completed, [...failed, ...deadLettered]);
+        this.#trackJobsSettled(name, landed(completed, completedIds), [...landed(failed, failedIds), ...landed(deadLettered, deadLetteredIds)]);
     }
     // Complete a set of active jobs, each with its own output, in a constant number of statements
     // (one on Postgres, two on a distributed backend). Outputs are serialized like complete()/fail()
     // and passed as a JSON recordset so the batch size doesn't drive the statement count.
+    // Each item carries the retryCount its job was fetched with, which the statement fences on.
     async #completeWithOutputs(name, items) {
         const { table } = await this.getQueueCache(name);
-        const payload = items.map(item => ({ id: item.id, output: this.mapCompletionDataArg(item.output) }));
+        const payload = items.map(item => ({ id: item.id, retry_count: item.retryCount, output: this.mapCompletionDataArg(item.output) }));
         const ids = items.map(item => item.id);
         if (this.config.noMultiMutationCte) {
             // Dependency unblocking is handled out of band by the background resolver (Navigator), so
             // completion is a single statement here too.
             const sql = plans.completeJobsWithOutputsDistributed(this.config.schema, table);
             const { rows } = await this.db.executeSql(sql, [name, JSON.stringify(payload)]);
-            return { jobs: ids, requested: ids.length, affected: rows.length };
+            return { jobs: ids, requested: ids.length, affected: rows.length, settled: rows.map(row => row.id) };
         }
         const sql = plans.completeJobsWithOutputs(this.config.schema, table);
         const result = await this.db.executeSql(sql, [name, JSON.stringify(payload)]);
@@ -407,18 +428,19 @@ class Manager extends EventEmitter {
         if (this.config.noMultiMutationCte) {
             const outputById = new Map(items.map(item => [item.id, this.mapCompletionDataArg(item.output)]));
             return this.ensureTransaction(this.db, async (tx) => {
-                const selectQuery = plans.selectJobsToFailById(this.config.schema, table);
-                const { rows: jobs } = await tx.executeSql(selectQuery.text, [name, ids]);
+                const selectQuery = plans.selectJobsToFailById(this.config.schema, table, true);
+                const { rows: jobs } = await tx.executeSql(selectQuery.text, [name, ids, plans.attemptPairs(ids, items.map(item => item.retryCount))]);
                 if (jobs.length === 0) {
-                    return { jobs: ids, requested: ids.length, affected: 0 };
+                    return { jobs: ids, requested: ids.length, affected: 0, settled: [] };
                 }
+                // Only the rows the select matched: see failDistributed.
                 const deleteQuery = plans.deleteJobsToFail(this.config.schema, table);
-                await tx.executeSql(deleteQuery.text, [name, ids]);
+                await tx.executeSql(deleteQuery.text, [name, jobs.map(job => job.id)]);
                 const count = await this.reinsertFailedJobs(tx, table, jobs, null, outputById, forceTerminal);
-                return { jobs: ids, requested: ids.length, affected: count };
+                return { jobs: ids, requested: ids.length, affected: count, settled: jobs.map(job => job.id) };
             });
         }
-        const payload = items.map(item => ({ id: item.id, output: this.mapCompletionDataArg(item.output) }));
+        const payload = items.map(item => ({ id: item.id, retry_count: item.retryCount, output: this.mapCompletionDataArg(item.output) }));
         const sql = forceTerminal
             ? plans.deadLetterJobsByIdWithOutputs(this.config.schema, table)
             : plans.failJobsByIdWithOutputs(this.config.schema, table);
@@ -494,7 +516,14 @@ class Manager extends EventEmitter {
         const heartbeatCandidates = jobs.map(j => j.heartbeatSeconds || 0).filter(s => s > 0);
         const heartbeatSeconds = heartbeatCandidates.length ? Math.min(...heartbeatCandidates) : 0;
         const ac = new AbortController();
-        jobs.forEach(job => { job.signal = ac.signal; });
+        // Each job also gets a signal of its own, aborted when the heartbeat finds that job's claim gone
+        // (see the timer below), so a handler can stop work whose result can no longer land. Per job
+        // rather than per batch: a short heartbeat does not say why a job went missing, and the handler
+        // settling one of its own jobs early looks the same as another worker taking it. Aborting only
+        // the missing job makes that harmless, since a handler that settled a job no longer holds its
+        // claim either, and the rest of the batch runs on untouched.
+        const claims = new Map(jobs.map(job => [job.id, new AbortController()]));
+        jobs.forEach(job => { job.signal = AbortSignal.any([ac.signal, claims.get(job.id).signal]); });
         // Store AbortController on worker so it can be aborted after graceful shutdown
         if (worker) {
             worker.abortController = ac;
@@ -505,11 +534,30 @@ class Manager extends EventEmitter {
             const refreshSeconds = heartbeatRefreshSeconds ?? (heartbeatSeconds / 2);
             const intervalMs = refreshSeconds * 1000;
             heartbeatTimer = this.config.clock.setInterval(async () => {
+                const held = jobs.filter(job => !claims.get(job.id).signal.aborted);
+                if (held.length === 0)
+                    return;
+                let touched;
                 try {
-                    await this.touch(name, jobIds);
+                    touched = await this.touch(name, held);
                 }
                 catch (err) {
+                    // A heartbeat that could not run says nothing about the claim, so nothing is aborted.
                     this.emit(events.error, err);
+                    return;
+                }
+                // Without the ids it refreshed, a heartbeat cannot tell which job went missing, so it aborts
+                // nothing rather than everything.
+                if (!touched.settled)
+                    return;
+                // touch() is fenced to the attempts this worker fetched, so a job it did not refresh is no
+                // longer active under this worker's claim: retried or settled elsewhere, or settled by the
+                // handler itself.
+                const refreshed = new Set(touched.settled);
+                for (const job of held) {
+                    if (!refreshed.has(job.id)) {
+                        claims.get(job.id).abort(new Error(`job ${job.id} is no longer active under this worker's claim`));
+                    }
                 }
             }, intervalMs);
         }
@@ -537,13 +585,19 @@ class Manager extends EventEmitter {
                 // the bound lost for the whole cooldown after. Memoized after the first batch either way.
                 const timeoutGuc = timeoutMs > 0 ? await this.#resolveTransactionTimeoutGuc() : null;
                 transaction = await this.db.beginTransaction();
-                this.#handlerSettledJobs.set(transaction.db, new Set());
+                // The handler's view shares these, so a settle it runs on { db: tx } is still recognized and fenced.
+                const settled = new Set();
+                const claims = new Map(jobs.map(job => [job.id, job.retryCount]));
+                for (const db of [transaction.db, untracked(transaction.db)]) {
+                    this.#handlerSettledJobs.set(db, settled);
+                    this.#handlerClaims.set(db, claims);
+                }
                 if (timeoutGuc) {
                     await this.#applyTransactionTimeout(transaction, timeoutGuc, timeoutMs);
                 }
             }
             const handling = transaction
-                ? callback(jobs, transaction.db)
+                ? callback(jobs, untracked(transaction.db))
                 : callback(jobs);
             const result = await resolveWithinSeconds(this.config.clock, handling, maxExpiration, `handler execution exceeded ${maxExpiration}s`, ac);
             // An abort resolves the race rather than rejecting it, so on this line an abandoned handler
@@ -567,7 +621,7 @@ class Manager extends EventEmitter {
                 // Read out before the completion below, which goes through the same complete() and would
                 // otherwise record pg-boss's own settle as one the handler made.
                 const settledByHandler = transaction ? this.#takeHandlerSettles(transaction) : null;
-                const completion = await this.complete(name, jobIds, jobIds.length === 1 ? result : undefined, transaction ? { db: transaction.db } : undefined);
+                const completion = await this.complete(name, jobs, jobIds.length === 1 ? result : undefined, transaction ? { db: transaction.db } : undefined);
                 completedResult = result;
                 completedAffected = completion.affected;
                 if (settledByHandler) {
@@ -591,7 +645,7 @@ class Manager extends EventEmitter {
                     err.message = `${err.message} (the handler left its transaction aborted: let a SQL error raised through tx propagate out of the handler, or isolate the statement with a SAVEPOINT)`;
                 }
             }
-            await this.fail(name, jobIds, err);
+            await this.fail(name, jobs, err);
             failedError = err;
             didFail = true;
         }
@@ -726,6 +780,7 @@ class Manager extends EventEmitter {
     #takeHandlerSettles(transaction) {
         const settled = this.#handlerSettledJobs.get(transaction.db) ?? new Set();
         this.#handlerSettledJobs.delete(transaction.db);
+        this.#handlerSettledJobs.delete(untracked(transaction.db));
         return settled;
     }
     // Records a settle a transactional handler ran through the transaction it was handed. A no-op for
@@ -749,6 +804,18 @@ class Manager extends EventEmitter {
             settled.add(id);
         }
     }
+    // The attempt fence for a transactional handler settling its own batch through the transaction it
+    // was handed. Those settles go through the public complete(), fail(), cancel() and deleteJob(),
+    // which carry no fence of their own, and #assertClaimHeld counts whatever they touch as the
+    // handler's: unfenced, a handler whose claim lapsed would settle the newer attempt, the counts would
+    // agree, and the transaction would commit. Only when every id is one of the batch's; a settle that
+    // reaches beyond the batch is not a claim this worker holds, so there is nothing to fence it to.
+    #handlerAttempts(options, ids) {
+        const claims = options.db && this.#handlerClaims.get(options.db);
+        if (!claims || !ids.every(id => claims.has(id)))
+            return undefined;
+        return ids.map(id => claims.get(id));
+    }
     async start() {
         this.stopped = false;
         this.queueCacheInterval = this.config.clock.setInterval(() => this.onCacheQueues({ emit: true }), this.config.queueCacheIntervalSeconds * 1000);
@@ -818,7 +885,7 @@ class Manager extends EventEmitter {
                 // jobs active and its completion finds nothing to settle. Its own failure is reported
                 // rather than thrown: the remaining workers still have to be aborted.
                 try {
-                    await this.fail(worker.name, jobIds, 'pg-boss shut down while active');
+                    await this.fail(worker.name, worker.jobs, 'pg-boss shut down while active');
                 }
                 catch (err) {
                     try {
@@ -902,6 +969,11 @@ class Manager extends EventEmitter {
                         if (excess.length > 0) {
                             const excessIds = excess.map(job => job.id);
                             await this.restore(name, excessIds);
+                            // A restored job is claimable again at the same retryCount (restore clears started_on,
+                            // so the next claim does not increment it), and failWip() fails whatever is left in
+                            // worker.jobs: left there, a shutdown would fail another worker's claim on it.
+                            if (worker)
+                                worker.jobs = allowed;
                         }
                         if (allowed.length > 0) {
                             await this.#processJobs(name, allowed, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds);
@@ -1508,6 +1580,20 @@ class Manager extends EventEmitter {
         // heartbeatSeconds, so normalize regardless of includeMetadata.
         return this.#numericJobFields(rows);
     }
+    // The id argument of the calls that settle or refresh a claim. Jobs passed as { id, retryCount }
+    // (what fetch() and work() hand out) come back with their attempts, which fence the call to the
+    // attempt that was fetched (plans.attemptFence). Plain ids come back without, and are not fenced.
+    mapAttemptArg(id, funcName) {
+        assert(id, `${funcName}() requires an id`);
+        const items = Array.isArray(id) ? id : [id];
+        assert(items.length, `${funcName}() requires an id`);
+        if (items.every(item => typeof item === 'string')) {
+            return { ids: items };
+        }
+        const jobs = items;
+        assert(jobs.every(job => job && typeof job === 'object' && typeof job.id === 'string' && Number.isInteger(job.retryCount) && job.retryCount >= 0), `${funcName}() requires either ids or jobs with an id and an integer retryCount, not a mix`);
+        return { ids: jobs.map(job => job.id), attempts: jobs.map(job => job.retryCount) };
+    }
     mapCompletionIdArg(id, funcName) {
         const errorMessage = `${funcName}() requires an id`;
         assert(id, errorMessage);
@@ -1529,27 +1615,30 @@ class Manager extends EventEmitter {
             jobs: ids,
             requested: ids.length,
             affected: result && result.rows ? parseInt(result.rows[0].count) : 0,
-            // The settle statements aggregate the ids they touched beside the count; the read-only
-            // mutators that share this mapper (resume, restore, retry, touch) do not, so it stays
-            // optional and #trackHandlerSettle falls back to the count when it is absent.
+            // The settle statements and touch() aggregate the ids they touched beside the count; the
+            // mutators that share this mapper without doing so (resume, restore, retry) leave it unset, so
+            // it stays optional and #trackHandlerSettle falls back to the count when it is absent.
             settled: result?.rows?.[0]?.ids
         };
     }
+    // Fenced to the fetched attempt (plans.attemptFence) when `id` carries jobs, as the worker's own
+    // settles do, or when a transactional handler settles its own batch through its transaction.
     async complete(name, id, data, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'complete');
+        const { ids, attempts: fetched } = this.mapAttemptArg(id, 'complete');
+        const attempts = fetched ?? this.#handlerAttempts(options, ids);
         const { table } = await this.getQueueCache(name);
         const outputData = this.mapCompletionDataArg(data);
         let response;
         // noMultiMutationCte: split the dependency-unblocking into a separate statement to
         // avoid CockroachDB's multi-mutation CTE limitation (completeJobs updates two tables).
         if (this.config.noMultiMutationCte) {
-            response = await this.completeDistributed(name, ids, outputData, table, db, options.includeQueued);
+            response = await this.completeDistributed(name, ids, outputData, table, db, options.includeQueued, attempts);
         }
         else {
-            const sql = plans.completeJobs(this.config.schema, table, options.includeQueued);
-            const result = await db.executeSql(sql, [name, ids, outputData]);
+            const sql = plans.completeJobs(this.config.schema, table, options.includeQueued, !!attempts);
+            const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
             response = this.mapCommandResponse(ids, result);
         }
         this.#trackHandlerSettle(options, response);
@@ -1565,17 +1654,19 @@ class Manager extends EventEmitter {
         }
         return fn(db);
     }
-    async completeDistributed(name, ids, outputData, table, db, includeQueued) {
+    async completeDistributed(name, ids, outputData, table, db, includeQueued, attempts) {
         // Dependency unblocking is handled out of band by the background resolver (Navigator), so
         // completion is a single statement on every backend.
-        const sql = plans.completeJobsDistributed(this.config.schema, table, includeQueued);
-        const { rows } = await db.executeSql(sql, [name, ids, outputData]);
+        const sql = plans.completeJobsDistributed(this.config.schema, table, includeQueued, !!attempts);
+        const { rows } = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
         return { jobs: ids, requested: ids.length, affected: rows.length, settled: rows.map(row => row.id) };
     }
+    // Fenced to the fetched attempt as in complete().
     async fail(name, id, data, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'fail');
+        const { ids, attempts: fetched } = this.mapAttemptArg(id, 'fail');
+        const attempts = fetched ?? this.#handlerAttempts(options, ids);
         const { table } = await this.getQueueCache(name);
         const outputData = this.mapCompletionDataArg(data);
         let response;
@@ -1583,29 +1674,31 @@ class Manager extends EventEmitter {
         // The delete and re-insert run in a single transaction (see ensureTransaction) so the
         // job cannot be lost between the two statements.
         if (this.config.noMultiMutationCte) {
-            response = await this.failDistributed(name, ids, outputData, table, db);
+            response = await this.failDistributed(name, ids, outputData, table, db, attempts);
         }
         else {
-            const sql = plans.failJobsById(this.config.schema, table);
-            const result = await db.executeSql(sql, [name, ids, outputData]);
+            const sql = plans.failJobsById(this.config.schema, table, !!attempts);
+            const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
             response = this.mapCommandResponse(ids, result);
         }
         this.#trackHandlerSettle(options, response);
         return response;
     }
-    async failDistributed(name, ids, outputData, table, db) {
+    async failDistributed(name, ids, outputData, table, db, attempts) {
         // CockroachDB doesn't support multi-mutation CTEs, but does support transactions, so the
         // delete + re-insert is split into separate statements run atomically.
         return this.ensureTransaction(db, async (tx) => {
             // Step 1: Select jobs to fail
-            const selectQuery = plans.selectJobsToFailById(this.config.schema, table);
-            const { rows: jobs } = await tx.executeSql(selectQuery.text, [name, ids]);
+            const selectQuery = plans.selectJobsToFailById(this.config.schema, table, !!attempts);
+            const { rows: jobs } = await tx.executeSql(selectQuery.text, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
             if (jobs.length === 0) {
                 return { jobs: ids, requested: ids.length, affected: 0, settled: [] };
             }
-            // Step 2: Delete the jobs
+            // Step 2: Delete the jobs the select matched, not every id asked about. The re-insert below
+            // only restores the selected rows, so deleting by the requested ids would drop any of them
+            // the select skipped (an already completed or cancelled job) for good.
             const deleteQuery = plans.deleteJobsToFail(this.config.schema, table);
-            await tx.executeSql(deleteQuery.text, [name, ids]);
+            await tx.executeSql(deleteQuery.text, [name, jobs.map(job => job.id)]);
             // Step 3: Re-insert jobs with updated state
             const count = await this.reinsertFailedJobs(tx, table, jobs, outputData);
             return { jobs: ids, requested: ids.length, affected: count, settled: jobs.map(job => job.id) };
@@ -1720,7 +1813,7 @@ class Manager extends EventEmitter {
                     job.deletion_seconds, createdOn, null, keepUntil, job.policy,
                     jobOutput, job.dead_letter,
                     null, job.heartbeat_seconds, job.blocked, job.blocking, job.pending_dependencies,
-                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count
+                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, job.source_output, job.source_root_id
                 ]);
                 // The retry insert can be dropped by ON CONFLICT when the queue policy (e.g. stately,
                 // singleton, key_strict_fifo) already has a non-terminal job. Mirror the failed_jobs
@@ -1735,24 +1828,26 @@ class Manager extends EventEmitter {
                     job.deletion_seconds, createdOn, new Date(this.config.clock.now()), keepUntil, job.policy,
                     jobOutput, job.dead_letter,
                     null, job.heartbeat_seconds, job.blocked, job.blocking, job.pending_dependencies,
-                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count
+                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, job.source_output, job.source_root_id
                 ]);
                 // Insert to dead letter queue if failed and has dead_letter configured
                 if (job.dead_letter) {
-                    await tx.executeSql(dlqSql, [job.dead_letter, job.data, jobOutput, job.name, job.id, createdOn, job.retry_count, job.singleton_key, job.priority, job.group_id, job.group_tier]);
+                    await tx.executeSql(dlqSql, [job.dead_letter, job.data, jobOutput, job.name, job.id, createdOn, job.retry_count, job.singleton_key, job.priority, job.group_id, job.group_tier, job.source_root_id]);
                 }
             }
             count++;
         }
         return count;
     }
+    // Fenced to the fetched attempt as in complete().
     async deleteJob(name, id, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'deleteJob');
+        const { ids, attempts: fetched } = this.mapAttemptArg(id, 'deleteJob');
+        const attempts = fetched ?? this.#handlerAttempts(options, ids);
         const { table } = await this.getQueueCache(name);
-        const sql = plans.deleteJobsById(this.config.schema, table);
-        const result = await db.executeSql(sql, [name, ids]);
+        const sql = plans.deleteJobsById(this.config.schema, table, !!attempts);
+        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
         const response = this.mapCommandResponse(ids, result);
         this.#trackHandlerSettle(options, response);
         return response;
@@ -1793,16 +1888,23 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { table } = await this.getQueueCache(name);
         // CockroachDB rejects the single-statement version (a DELETE and an INSERT on one table), so
-        // the same move runs as three statements in one transaction. See plans.selectRedriveCandidates.
+        // the same move runs as statements in one transaction. See plans.selectRedriveCandidates.
         if (this.config.noMultiMutationCte) {
             return this.ensureTransaction(db, async (tx) => {
                 const { rows } = await tx.executeSql(plans.selectRedriveCandidates(this.config.schema, table), [name, ...filter, limit]);
                 if (rows.length === 0)
                     return 0;
                 const ids = rows.map((row) => row.id);
-                const { rows: inserted } = await tx.executeSql(plans.insertRedrivenJobs(this.config.schema, table), [ids, filter[0]]);
-                await tx.executeSql(plans.deleteJobsByIds(this.config.schema, table).text, [ids]);
-                return inserted.length;
+                const newIds = rows.map((row) => row.new_id);
+                const { rows: inserted } = await tx.executeSql(plans.insertRedrivenJobs(this.config.schema, table), [ids, filter[0], newIds]);
+                const created = new Set(inserted.map((row) => row.id));
+                const moved = ids.filter((_, i) => created.has(newIds[i]));
+                const conflicted = ids.filter((_, i) => !created.has(newIds[i]));
+                if (moved.length)
+                    await tx.executeSql(plans.deleteJobsByIds(this.config.schema, table).text, [moved]);
+                if (conflicted.length)
+                    await tx.executeSql(plans.failRedriveConflicts(this.config.schema, table), [conflicted, filter[0]]);
+                return moved.length;
             });
         }
         const sql = plans.redriveJobs(this.config.schema, table);
@@ -1833,13 +1935,15 @@ class Manager extends EventEmitter {
             unroutable
         };
     }
+    // Fenced to the fetched attempt as in complete().
     async cancel(name, id, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'cancel');
+        const { ids, attempts: fetched } = this.mapAttemptArg(id, 'cancel');
+        const attempts = fetched ?? this.#handlerAttempts(options, ids);
         const { table } = await this.getQueueCache(name);
-        const sql = plans.cancelJobs(this.config.schema, table);
-        const result = await db.executeSql(sql, [name, ids]);
+        const sql = plans.cancelJobs(this.config.schema, table, !!attempts);
+        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
         const response = this.mapCommandResponse(ids, result);
         this.#trackHandlerSettle(options, response);
         return response;
@@ -1870,13 +1974,15 @@ class Manager extends EventEmitter {
         const result = await db.executeSql(sql, [name, ids]);
         return this.mapCommandResponse(ids, result);
     }
+    // Fenced to the fetched attempt as in complete(), so a worker that lost its claim cannot keep a
+    // newer attempt's heartbeat fresh.
     async touch(name, id, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'touch');
+        const { ids, attempts } = this.mapAttemptArg(id, 'touch');
         const { table } = await this.getQueueCache(name);
-        const sql = plans.touchJobs(this.config.schema, table);
-        const result = await db.executeSql(sql, [name, ids]);
+        const sql = plans.touchJobs(this.config.schema, table, !!attempts);
+        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
         return this.mapCommandResponse(ids, result);
     }
     async createQueue(name, options = {}) {
@@ -2005,7 +2111,9 @@ class Manager extends EventEmitter {
     async getQueueStats(name, options = {}) {
         Attorney.assertQueueName(name);
         const isCockroach = this.config.backend === 'cockroachdb';
-        const toSnapshot = (row) => {
+        // `counted` is true for recorded snapshots. The cache path serves only gauges: the queue table's
+        // counters describe the last pass that counted, not this reading.
+        const toSnapshot = (row, counted = false) => {
             const snapshot = {
                 name,
                 deferredCount: 0,
@@ -2014,14 +2122,25 @@ class Manager extends EventEmitter {
                 activeCount: 0,
                 failedCount: 0,
                 totalCount: 0,
+                // Null, not zero, until something counted them: with persistQueueStats off
+                // nobody does, and a snapshot captured before 12.35 predates the columns.
+                // Zero would claim the queue was idle.
+                completedDelta: null,
+                failedDelta: null,
+                createdDelta: null,
+                deltaSeconds: null,
+                deltaOn: null,
                 capturedOn: row?.capturedOn ?? new Date(this.config.clock.now())
             };
-            for (const field of STATS_COUNT_FIELDS) {
+            for (const field of counted ? [...STATS_COUNT_FIELDS, ...STATS_DELTA_FIELDS] : STATS_COUNT_FIELDS) {
                 const value = row?.[field];
                 // CockroachDB returns integer columns as strings; normalize the counts.
                 if (value !== undefined && value !== null)
                     snapshot[field] = isCockroach ? Number(value) : value;
             }
+            // The end of the interval the counters cover, handed on as the row holds it, like capturedOn.
+            if (counted && row?.deltaOn != null)
+                snapshot.deltaOn = row.deltaOn;
             return snapshot;
         };
         if (this.config.persistQueueStats) {
@@ -2039,11 +2158,11 @@ class Manager extends EventEmitter {
                 assert(Number.isInteger(width) && width >= 1, `getQueueStats: ${mode === 'bucket' ? 'bucketSeconds' : 'maxDataPoints'} must be a positive integer`);
                 const sql = plans.getQueueStatsHistoryBucketed(this.config.schema, aggregate, mode);
                 const { rows } = await this.db.executeSql(sql, [name, from, to, limit, width]);
-                return rows.map(toSnapshot);
+                return rows.map(row => toSnapshot(row, true));
             }
             const sql = plans.getQueueStatsHistory(this.config.schema);
             const { rows } = await this.db.executeSql(sql, [name, from, to, limit]);
-            return rows.map(toSnapshot);
+            return rows.map(row => toSnapshot(row, true));
         }
         // persistQueueStats disabled: serve the cached counts the monitor keeps on the queue table.
         // capturedOn is monitor_on, NULL if never monitored, or old if monitoring has since been turned

package/dist/migrationStore.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"migrationStore.d.ts","sourceRoot":"","sources":["../src/migrationStore.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAKnC,UAAU,cAAc;IACtB,WAAW,CAAC,EAAE,OAAO,CAAA;IAKrB,eAAe,CAAC,EAAE,KAAK,CAAC,kBAAkB,EAAE,CAAA;CAC7C;AAoGD,UAAU,iBAAiB;IACzB,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,EAAE,CAAA;CACrB;AASD,iBAAS,QAAQ,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,UAsB5G;AAED,iBAAS,IAAI,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,UAQxG;AAKD,iBAAS,eAAe,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,cAAmB,GAAG,iBAAiB,CAiDrK;AAKD,iBAAS,OAAO,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,cAAmB,UAIzI;AAoiCD,iBAAS,aAAa,CAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAE9C;AAID,iBAAS,eAAe,CAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,0BAA0B,EAAE,QAAQ,GAAG,qBAAqB,GAAG,mBAAmB,GAAG,qBAAqB,CAAC,GAAG,KAAK,CAAC,SAAS,EAAE,CAE3K;AAOD,iBAAS,MAAM,CAAE,MAAM,EAAE,MAAM,EAAE,cAAc,UAAQ,EAAE,UAAU,UAAQ,EAAE,mBAAmB,UAAQ,GAAG,KAAK,CAAC,SAAS,EAAE,CAqkB3H;AAED,OAAO,EACL,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,eAAe,EACf,MAAM,EACN,eAAe,EACf,aAAa,GACd,CAAA"}
+{"version":3,"file":"migrationStore.d.ts","sourceRoot":"","sources":["../src/migrationStore.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAKnC,UAAU,cAAc;IACtB,WAAW,CAAC,EAAE,OAAO,CAAA;IAKrB,eAAe,CAAC,EAAE,KAAK,CAAC,kBAAkB,EAAE,CAAA;CAC7C;AAoGD,UAAU,iBAAiB;IACzB,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,EAAE,CAAA;CACrB;AASD,iBAAS,QAAQ,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,UAsB5G;AAED,iBAAS,IAAI,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,UAQxG;AAKD,iBAAS,eAAe,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,cAAmB,GAAG,iBAAiB,CAiDrK;AAKD,iBAAS,OAAO,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,cAAmB,UAIzI;AAgoCD,iBAAS,aAAa,CAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAE9C;AAID,iBAAS,eAAe,CAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,0BAA0B,EAAE,QAAQ,GAAG,qBAAqB,GAAG,mBAAmB,GAAG,qBAAqB,CAAC,GAAG,KAAK,CAAC,SAAS,EAAE,CAE3K;AAOD,iBAAS,MAAM,CAAE,MAAM,EAAE,MAAM,EAAE,cAAc,UAAQ,EAAE,UAAU,UAAQ,EAAE,mBAAmB,UAAQ,GAAG,KAAK,CAAC,SAAS,EAAE,CA4oB3H;AAED,OAAO,EACL,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,eAAe,EACf,MAAM,EACN,eAAe,EACf,aAAa,GACd,CAAA"}