pg-boss 12.34.0 → 12.35.1

package/dist/manager.js

@@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto';
 import EventEmitter from 'node:events';
 import { serializeError as stringify } from 'serialize-error';
 import * as Attorney from "./attorney.js";
+import { untracked } from "./activity.js";
 import { TRANSACTION_ROLLBACK_TIMEOUT_MS } from "./db.js";
 import * as plans from "./plans.js";
 import * as timekeeper from "./timekeeper.js";
@@ -89,10 +90,16 @@ const NUMERIC_QUEUE_FIELDS = [
     'deferredCount',
     'warningQueueSize',
     'queuedCount',
+    'readyCount',
     'activeCount',
-    'totalCount'
+    'failedCount',
+    'totalCount',
+    'createdDelta',
+    'completedDelta',
+    'failedDelta',
+    'deltaSeconds'
 ];
-// The count columns shared by live stats and recorded snapshots (the QueueStats shape).
+// The gauges shared by live stats and recorded snapshots (the QueueStats shape).
 const STATS_COUNT_FIELDS = [
     'deferredCount',
     'queuedCount',
@@ -101,6 +108,14 @@ const STATS_COUNT_FIELDS = [
     'failedCount',
     'totalCount'
 ];
+// The throughput counters and the seconds they cover. Only recorded snapshots carry them; see
+// getQueueStats.
+const STATS_DELTA_FIELDS = [
+    'completedDelta',
+    'failedDelta',
+    'createdDelta',
+    'deltaSeconds'
+];
 // Stale-cache budget for getQueueStats when persistQueueStats is off. A queue-table cache older than
 // this means monitoring isn't keeping it current (e.g. supervise was enabled once but isn't now), so
 // the counts are recomputed and re-cached instead of returned. Defaults to one hour, raised to the
@@ -132,6 +147,10 @@ function rethrowWriteError(err) {
     }
     throw err;
 }
+// For a json value read back from a row and bound again behind `::text::jsonb`. Null stays SQL NULL.
+function toJsonText(value) {
+    return value == null ? null : JSON.stringify(value);
+}
 class Manager extends EventEmitter {
     events = events;
     // Warn once per option per instance, not once per fetch.
@@ -152,6 +171,9 @@ class Manager extends EventEmitter {
     // that transaction's db. #processJobs reads it to interpret its own completion; see the check
     // there. Weak because the key is the transaction, so an entry goes away with it.
     #handlerSettledJobs;
+    // The retryCount each job of a transactional batch was claimed at, keyed by the transaction the
+    // handler was handed, so the handler's own settles through it can be fenced (#handlerAttempts).
+    #handlerClaims;
     // Which GUC this server bounds a transaction with, resolved on the first transactional batch and
     // kept for the life of the process. Resolving to null means neither is recognised, or the probe
     // itself failed; a failed probe clears the memo and sets the instant before which no batch asks
@@ -175,6 +197,7 @@ class Manager extends EventEmitter {
         this.pendingOffWorkCleanups = new Set();
         this.#spies = new Map();
         this.#handlerSettledJobs = new WeakMap();
+        this.#handlerClaims = new WeakMap();
         this.#transactionTimeoutGuc = null;
         this.#transactionTimeoutProbeRetryAt = 0;
         this.#transactionTimeoutProbeCooldownMs = config.__test__transactionTimeoutProbeCooldownMs ?? TRANSACTION_TIMEOUT_PROBE_COOLDOWN_MS;
@@ -334,7 +357,7 @@ class Manager extends EventEmitter {
             // The handler opted into perJobResults but did not return an array: a contract violation.
             // Fail the whole batch so the mistake surfaces and the jobs are retried.
             const err = new Error('perJobResults handler must resolve with an array of job results');
-            await this.fail(name, jobs.map(job => job.id), err);
+            await this.fail(name, jobs, err);
             await this.#trackJobsFailed(name, jobs, err);
             return;
         }
@@ -367,31 +390,33 @@ class Manager extends EventEmitter {
                 failed.push({ job, output: new Error('no disposition returned by handler') });
             }
         }
-        if (completed.length > 0) {
-            await this.#completeWithOutputs(name, completed.map(c => ({ id: c.job.id, output: c.output })));
-        }
-        if (failed.length > 0) {
-            await this.#failWithOutputs(name, failed.map(f => ({ id: f.job.id, output: f.output })));
-        }
-        if (deadLettered.length > 0) {
-            await this.#failWithOutputs(name, deadLettered.map(d => ({ id: d.job.id, output: d.output })), true);
-        }
+        const items = (entries) => entries.map(({ job, output }) => ({ id: job.id, retryCount: job.retryCount, output }));
+        const completedIds = completed.length > 0 ? await this.#completeWithOutputs(name, items(completed)) : null;
+        const failedIds = failed.length > 0 ? await this.#failWithOutputs(name, items(failed)) : null;
+        const deadLetteredIds = deadLettered.length > 0 ? await this.#failWithOutputs(name, items(deadLettered), true) : null;
+        // Only the jobs each statement actually settled: the attempt fence leaves a job whose claim
+        // lapsed alone, and recording it would tell a spy it settled when another attempt holds it.
+        const landed = (entries, result) => {
+            const settled = new Set(result?.settled ?? []);
+            return entries.filter(entry => settled.has(entry.job.id));
+        };
         // Dead lettered jobs end in the same terminal `failed` state as failed jobs on the source queue.
-        this.#trackJobsSettled(name, completed, [...failed, ...deadLettered]);
+        this.#trackJobsSettled(name, landed(completed, completedIds), [...landed(failed, failedIds), ...landed(deadLettered, deadLetteredIds)]);
     }
     // Complete a set of active jobs, each with its own output, in a constant number of statements
     // (one on Postgres, two on a distributed backend). Outputs are serialized like complete()/fail()
     // and passed as a JSON recordset so the batch size doesn't drive the statement count.
+    // Each item carries the retryCount its job was fetched with, which the statement fences on.
     async #completeWithOutputs(name, items) {
         const { table } = await this.getQueueCache(name);
-        const payload = items.map(item => ({ id: item.id, output: this.mapCompletionDataArg(item.output) }));
+        const payload = items.map(item => ({ id: item.id, retry_count: item.retryCount, output: this.mapCompletionDataArg(item.output) }));
         const ids = items.map(item => item.id);
         if (this.config.noMultiMutationCte) {
             // Dependency unblocking is handled out of band by the background resolver (Navigator), so
             // completion is a single statement here too.
             const sql = plans.completeJobsWithOutputsDistributed(this.config.schema, table);
             const { rows } = await this.db.executeSql(sql, [name, JSON.stringify(payload)]);
-            return { jobs: ids, requested: ids.length, affected: rows.length };
+            return { jobs: ids, requested: ids.length, affected: rows.length, settled: rows.map(row => row.id) };
         }
         const sql = plans.completeJobsWithOutputs(this.config.schema, table);
         const result = await this.db.executeSql(sql, [name, JSON.stringify(payload)]);
@@ -407,18 +432,19 @@ class Manager extends EventEmitter {
         if (this.config.noMultiMutationCte) {
             const outputById = new Map(items.map(item => [item.id, this.mapCompletionDataArg(item.output)]));
             return this.ensureTransaction(this.db, async (tx) => {
-                const selectQuery = plans.selectJobsToFailById(this.config.schema, table);
-                const { rows: jobs } = await tx.executeSql(selectQuery.text, [name, ids]);
+                const selectQuery = plans.selectJobsToFailById(this.config.schema, table, true);
+                const { rows: jobs } = await tx.executeSql(selectQuery.text, [name, ids, plans.attemptPairs(ids, items.map(item => item.retryCount))]);
                 if (jobs.length === 0) {
-                    return { jobs: ids, requested: ids.length, affected: 0 };
+                    return { jobs: ids, requested: ids.length, affected: 0, settled: [] };
                 }
+                // Only the rows the select matched: see failDistributed.
                 const deleteQuery = plans.deleteJobsToFail(this.config.schema, table);
-                await tx.executeSql(deleteQuery.text, [name, ids]);
+                await tx.executeSql(deleteQuery.text, [name, jobs.map(job => job.id)]);
                 const count = await this.reinsertFailedJobs(tx, table, jobs, null, outputById, forceTerminal);
-                return { jobs: ids, requested: ids.length, affected: count };
+                return { jobs: ids, requested: ids.length, affected: count, settled: jobs.map(job => job.id) };
             });
         }
-        const payload = items.map(item => ({ id: item.id, output: this.mapCompletionDataArg(item.output) }));
+        const payload = items.map(item => ({ id: item.id, retry_count: item.retryCount, output: this.mapCompletionDataArg(item.output) }));
         const sql = forceTerminal
             ? plans.deadLetterJobsByIdWithOutputs(this.config.schema, table)
             : plans.failJobsByIdWithOutputs(this.config.schema, table);
@@ -494,7 +520,14 @@ class Manager extends EventEmitter {
         const heartbeatCandidates = jobs.map(j => j.heartbeatSeconds || 0).filter(s => s > 0);
         const heartbeatSeconds = heartbeatCandidates.length ? Math.min(...heartbeatCandidates) : 0;
         const ac = new AbortController();
-        jobs.forEach(job => { job.signal = ac.signal; });
+        // Each job also gets a signal of its own, aborted when the heartbeat finds that job's claim gone
+        // (see the timer below), so a handler can stop work whose result can no longer land. Per job
+        // rather than per batch: a short heartbeat does not say why a job went missing, and the handler
+        // settling one of its own jobs early looks the same as another worker taking it. Aborting only
+        // the missing job makes that harmless, since a handler that settled a job no longer holds its
+        // claim either, and the rest of the batch runs on untouched.
+        const claims = new Map(jobs.map(job => [job.id, new AbortController()]));
+        jobs.forEach(job => { job.signal = AbortSignal.any([ac.signal, claims.get(job.id).signal]); });
         // Store AbortController on worker so it can be aborted after graceful shutdown
         if (worker) {
             worker.abortController = ac;
@@ -505,11 +538,30 @@ class Manager extends EventEmitter {
             const refreshSeconds = heartbeatRefreshSeconds ?? (heartbeatSeconds / 2);
             const intervalMs = refreshSeconds * 1000;
             heartbeatTimer = this.config.clock.setInterval(async () => {
+                const held = jobs.filter(job => !claims.get(job.id).signal.aborted);
+                if (held.length === 0)
+                    return;
+                let touched;
                 try {
-                    await this.touch(name, jobIds);
+                    touched = await this.touch(name, held);
                 }
                 catch (err) {
+                    // A heartbeat that could not run says nothing about the claim, so nothing is aborted.
                     this.emit(events.error, err);
+                    return;
+                }
+                // Without the ids it refreshed, a heartbeat cannot tell which job went missing, so it aborts
+                // nothing rather than everything.
+                if (!touched.settled)
+                    return;
+                // touch() is fenced to the attempts this worker fetched, so a job it did not refresh is no
+                // longer active under this worker's claim: retried or settled elsewhere, or settled by the
+                // handler itself.
+                const refreshed = new Set(touched.settled);
+                for (const job of held) {
+                    if (!refreshed.has(job.id)) {
+                        claims.get(job.id).abort(new Error(`job ${job.id} is no longer active under this worker's claim`));
+                    }
                 }
             }, intervalMs);
         }
@@ -537,13 +589,19 @@ class Manager extends EventEmitter {
                 // the bound lost for the whole cooldown after. Memoized after the first batch either way.
                 const timeoutGuc = timeoutMs > 0 ? await this.#resolveTransactionTimeoutGuc() : null;
                 transaction = await this.db.beginTransaction();
-                this.#handlerSettledJobs.set(transaction.db, new Set());
+                // The handler's view shares these, so a settle it runs on { db: tx } is still recognized and fenced.
+                const settled = new Set();
+                const claims = new Map(jobs.map(job => [job.id, job.retryCount]));
+                for (const db of [transaction.db, untracked(transaction.db)]) {
+                    this.#handlerSettledJobs.set(db, settled);
+                    this.#handlerClaims.set(db, claims);
+                }
                 if (timeoutGuc) {
                     await this.#applyTransactionTimeout(transaction, timeoutGuc, timeoutMs);
                 }
             }
             const handling = transaction
-                ? callback(jobs, transaction.db)
+                ? callback(jobs, untracked(transaction.db))
                 : callback(jobs);
             const result = await resolveWithinSeconds(this.config.clock, handling, maxExpiration, `handler execution exceeded ${maxExpiration}s`, ac);
             // An abort resolves the race rather than rejecting it, so on this line an abandoned handler
@@ -567,7 +625,7 @@ class Manager extends EventEmitter {
                 // Read out before the completion below, which goes through the same complete() and would
                 // otherwise record pg-boss's own settle as one the handler made.
                 const settledByHandler = transaction ? this.#takeHandlerSettles(transaction) : null;
-                const completion = await this.complete(name, jobIds, jobIds.length === 1 ? result : undefined, transaction ? { db: transaction.db } : undefined);
+                const completion = await this.complete(name, jobs, jobIds.length === 1 ? result : undefined, transaction ? { db: transaction.db } : undefined);
                 completedResult = result;
                 completedAffected = completion.affected;
                 if (settledByHandler) {
@@ -591,7 +649,7 @@ class Manager extends EventEmitter {
                     err.message = `${err.message} (the handler left its transaction aborted: let a SQL error raised through tx propagate out of the handler, or isolate the statement with a SAVEPOINT)`;
                 }
             }
-            await this.fail(name, jobIds, err);
+            await this.fail(name, jobs, err);
             failedError = err;
             didFail = true;
         }
@@ -726,6 +784,7 @@ class Manager extends EventEmitter {
     #takeHandlerSettles(transaction) {
         const settled = this.#handlerSettledJobs.get(transaction.db) ?? new Set();
         this.#handlerSettledJobs.delete(transaction.db);
+        this.#handlerSettledJobs.delete(untracked(transaction.db));
         return settled;
     }
     // Records a settle a transactional handler ran through the transaction it was handed. A no-op for
@@ -749,6 +808,18 @@ class Manager extends EventEmitter {
             settled.add(id);
         }
     }
+    // The attempt fence for a transactional handler settling its own batch through the transaction it
+    // was handed. Those settles go through the public complete(), fail(), cancel() and deleteJob(),
+    // which carry no fence of their own, and #assertClaimHeld counts whatever they touch as the
+    // handler's: unfenced, a handler whose claim lapsed would settle the newer attempt, the counts would
+    // agree, and the transaction would commit. Only when every id is one of the batch's; a settle that
+    // reaches beyond the batch is not a claim this worker holds, so there is nothing to fence it to.
+    #handlerAttempts(options, ids) {
+        const claims = options.db && this.#handlerClaims.get(options.db);
+        if (!claims || !ids.every(id => claims.has(id)))
+            return undefined;
+        return ids.map(id => claims.get(id));
+    }
     async start() {
         this.stopped = false;
         this.queueCacheInterval = this.config.clock.setInterval(() => this.onCacheQueues({ emit: true }), this.config.queueCacheIntervalSeconds * 1000);
@@ -818,7 +889,7 @@ class Manager extends EventEmitter {
                 // jobs active and its completion finds nothing to settle. Its own failure is reported
                 // rather than thrown: the remaining workers still have to be aborted.
                 try {
-                    await this.fail(worker.name, jobIds, 'pg-boss shut down while active');
+                    await this.fail(worker.name, worker.jobs, 'pg-boss shut down while active');
                 }
                 catch (err) {
                     try {
@@ -902,6 +973,11 @@ class Manager extends EventEmitter {
                         if (excess.length > 0) {
                             const excessIds = excess.map(job => job.id);
                             await this.restore(name, excessIds);
+                            // A restored job is claimable again at the same retryCount (restore clears started_on,
+                            // so the next claim does not increment it), and failWip() fails whatever is left in
+                            // worker.jobs: left there, a shutdown would fail another worker's claim on it.
+                            if (worker)
+                                worker.jobs = allowed;
                         }
                         if (allowed.length > 0) {
                             await this.#processJobs(name, allowed, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds);
@@ -1004,8 +1080,17 @@ class Manager extends EventEmitter {
             .filter(i => i.state !== 'stopped' && (!INTERNAL_QUEUES[i.name] || includeInternal));
         return data;
     }
-    hasPendingCleanups() {
-        return this.pendingOffWorkCleanups.size > 0;
+    trackCleanup(cleanup) {
+        this.pendingOffWorkCleanups.add(cleanup);
+        const settled = () => { this.pendingOffWorkCleanups.delete(cleanup); };
+        cleanup.then(settled, settled);
+    }
+    // Resolves once no cleanup is pending, including any a settling cleanup registers, so a graceful
+    // stop() carries on the moment its last worker has stopped.
+    async settleCleanups() {
+        while (this.pendingOffWorkCleanups.size > 0) {
+            await Promise.allSettled([...this.pendingOffWorkCleanups]);
+        }
     }
     async offWork(name, options = { wait: true }) {
         assert(name, 'queue name is required');
@@ -1034,11 +1119,8 @@ class Manager extends EventEmitter {
             this.#cleanupLocalGroupTracking(name);
         }
         else {
-            this.pendingOffWorkCleanups.add(cleanupPromise);
-            cleanupPromise.finally(() => {
-                this.pendingOffWorkCleanups.delete(cleanupPromise);
-                this.#cleanupLocalGroupTracking(name);
-            });
+            this.trackCleanup(cleanupPromise);
+            cleanupPromise.finally(() => this.#cleanupLocalGroupTracking(name));
         }
     }
     notifyWorker(workerId) {
@@ -1508,6 +1590,20 @@ class Manager extends EventEmitter {
         // heartbeatSeconds, so normalize regardless of includeMetadata.
         return this.#numericJobFields(rows);
     }
+    // The id argument of the calls that settle or refresh a claim. Jobs passed as { id, retryCount }
+    // (what fetch() and work() hand out) come back with their attempts, which fence the call to the
+    // attempt that was fetched (plans.attemptFence). Plain ids come back without, and are not fenced.
+    mapAttemptArg(id, funcName) {
+        assert(id, `${funcName}() requires an id`);
+        const items = Array.isArray(id) ? id : [id];
+        assert(items.length, `${funcName}() requires an id`);
+        if (items.every(item => typeof item === 'string')) {
+            return { ids: items };
+        }
+        const jobs = items;
+        assert(jobs.every(job => job && typeof job === 'object' && typeof job.id === 'string' && Number.isInteger(job.retryCount) && job.retryCount >= 0), `${funcName}() requires either ids or jobs with an id and an integer retryCount, not a mix`);
+        return { ids: jobs.map(job => job.id), attempts: jobs.map(job => job.retryCount) };
+    }
     mapCompletionIdArg(id, funcName) {
         const errorMessage = `${funcName}() requires an id`;
         assert(id, errorMessage);
@@ -1529,27 +1625,30 @@ class Manager extends EventEmitter {
             jobs: ids,
             requested: ids.length,
             affected: result && result.rows ? parseInt(result.rows[0].count) : 0,
-            // The settle statements aggregate the ids they touched beside the count; the read-only
-            // mutators that share this mapper (resume, restore, retry, touch) do not, so it stays
-            // optional and #trackHandlerSettle falls back to the count when it is absent.
+            // The settle statements and touch() aggregate the ids they touched beside the count; the
+            // mutators that share this mapper without doing so (resume, restore, retry) leave it unset, so
+            // it stays optional and #trackHandlerSettle falls back to the count when it is absent.
             settled: result?.rows?.[0]?.ids
         };
     }
+    // Fenced to the fetched attempt (plans.attemptFence) when `id` carries jobs, as the worker's own
+    // settles do, or when a transactional handler settles its own batch through its transaction.
     async complete(name, id, data, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'complete');
+        const { ids, attempts: fetched } = this.mapAttemptArg(id, 'complete');
+        const attempts = fetched ?? this.#handlerAttempts(options, ids);
         const { table } = await this.getQueueCache(name);
         const outputData = this.mapCompletionDataArg(data);
         let response;
         // noMultiMutationCte: split the dependency-unblocking into a separate statement to
         // avoid CockroachDB's multi-mutation CTE limitation (completeJobs updates two tables).
         if (this.config.noMultiMutationCte) {
-            response = await this.completeDistributed(name, ids, outputData, table, db, options.includeQueued);
+            response = await this.completeDistributed(name, ids, outputData, table, db, options.includeQueued, attempts);
         }
         else {
-            const sql = plans.completeJobs(this.config.schema, table, options.includeQueued);
-            const result = await db.executeSql(sql, [name, ids, outputData]);
+            const sql = plans.completeJobs(this.config.schema, table, options.includeQueued, !!attempts);
+            const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
             response = this.mapCommandResponse(ids, result);
         }
         this.#trackHandlerSettle(options, response);
@@ -1565,17 +1664,19 @@ class Manager extends EventEmitter {
         }
         return fn(db);
     }
-    async completeDistributed(name, ids, outputData, table, db, includeQueued) {
+    async completeDistributed(name, ids, outputData, table, db, includeQueued, attempts) {
         // Dependency unblocking is handled out of band by the background resolver (Navigator), so
         // completion is a single statement on every backend.
-        const sql = plans.completeJobsDistributed(this.config.schema, table, includeQueued);
-        const { rows } = await db.executeSql(sql, [name, ids, outputData]);
+        const sql = plans.completeJobsDistributed(this.config.schema, table, includeQueued, !!attempts);
+        const { rows } = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
         return { jobs: ids, requested: ids.length, affected: rows.length, settled: rows.map(row => row.id) };
     }
+    // Fenced to the fetched attempt as in complete().
     async fail(name, id, data, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'fail');
+        const { ids, attempts: fetched } = this.mapAttemptArg(id, 'fail');
+        const attempts = fetched ?? this.#handlerAttempts(options, ids);
         const { table } = await this.getQueueCache(name);
         const outputData = this.mapCompletionDataArg(data);
         let response;
@@ -1583,29 +1684,31 @@ class Manager extends EventEmitter {
         // The delete and re-insert run in a single transaction (see ensureTransaction) so the
         // job cannot be lost between the two statements.
         if (this.config.noMultiMutationCte) {
-            response = await this.failDistributed(name, ids, outputData, table, db);
+            response = await this.failDistributed(name, ids, outputData, table, db, attempts);
         }
         else {
-            const sql = plans.failJobsById(this.config.schema, table);
-            const result = await db.executeSql(sql, [name, ids, outputData]);
+            const sql = plans.failJobsById(this.config.schema, table, !!attempts);
+            const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
             response = this.mapCommandResponse(ids, result);
         }
         this.#trackHandlerSettle(options, response);
         return response;
     }
-    async failDistributed(name, ids, outputData, table, db) {
+    async failDistributed(name, ids, outputData, table, db, attempts) {
         // CockroachDB doesn't support multi-mutation CTEs, but does support transactions, so the
         // delete + re-insert is split into separate statements run atomically.
         return this.ensureTransaction(db, async (tx) => {
             // Step 1: Select jobs to fail
-            const selectQuery = plans.selectJobsToFailById(this.config.schema, table);
-            const { rows: jobs } = await tx.executeSql(selectQuery.text, [name, ids]);
+            const selectQuery = plans.selectJobsToFailById(this.config.schema, table, !!attempts);
+            const { rows: jobs } = await tx.executeSql(selectQuery.text, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
             if (jobs.length === 0) {
                 return { jobs: ids, requested: ids.length, affected: 0, settled: [] };
             }
-            // Step 2: Delete the jobs
+            // Step 2: Delete the jobs the select matched, not every id asked about. The re-insert below
+            // only restores the selected rows, so deleting by the requested ids would drop any of them
+            // the select skipped (an already completed or cancelled job) for good.
             const deleteQuery = plans.deleteJobsToFail(this.config.schema, table);
-            await tx.executeSql(deleteQuery.text, [name, ids]);
+            await tx.executeSql(deleteQuery.text, [name, jobs.map(job => job.id)]);
             // Step 3: Re-insert jobs with updated state
             const count = await this.reinsertFailedJobs(tx, table, jobs, outputData);
             return { jobs: ids, requested: ids.length, affected: count, settled: jobs.map(job => job.id) };
@@ -1692,6 +1795,11 @@ class Manager extends EventEmitter {
             // Dead-letter provenance, carried through so a job in a dead letter queue that fails here
             // still knows where to be redriven. See failJobsBody for the single-statement path.
             const sourceCreatedOn = job.source_created_on_text ?? job.source_created_on;
+            // The json columns go back as text. Bound as read, a job whose data is an array reaches pg as
+            // a Postgres array literal and a string as bare text, and neither parses as json.
+            const data = toJsonText(job.data);
+            const output = toJsonText(jobOutput);
+            const sourceOutput = toJsonText(job.source_output);
             // forceTerminal (perJobResults `deadletter`) skips retries so the job fails terminally and
             // routes straight to the dead letter queue below.
             const canRetry = !forceTerminal && retryCount < retryLimit;
@@ -1714,13 +1822,13 @@ class Manager extends EventEmitter {
                 // pending_dependencies are preserved so flows and heartbeat detection survive a retry
                 // (matches the non-distributed failJobs() CTE).
                 const { rows } = await tx.executeSql(insertSql, [
-                    job.id, job.name, job.priority, job.data, 'retry', job.retry_limit, job.retry_count,
+                    job.id, job.name, job.priority, data, 'retry', job.retry_limit, job.retry_count,
                     job.retry_delay, job.retry_backoff, job.retry_delay_max, startAfter, startedOn,
                     job.singleton_key, singletonOn, job.group_id, job.group_tier, job.expire_seconds,
                     job.deletion_seconds, createdOn, null, keepUntil, job.policy,
-                    jobOutput, job.dead_letter,
+                    output, job.dead_letter,
                     null, job.heartbeat_seconds, job.blocked, job.blocking, job.pending_dependencies,
-                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count
+                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id
                 ]);
                 // The retry insert can be dropped by ON CONFLICT when the queue policy (e.g. stately,
                 // singleton, key_strict_fifo) already has a non-terminal job. Mirror the failed_jobs
@@ -1729,30 +1837,32 @@ class Manager extends EventEmitter {
             }
             if (!retried) {
                 await tx.executeSql(insertSql, [
-                    job.id, job.name, job.priority, job.data, 'failed', job.retry_limit, job.retry_count,
+                    job.id, job.name, job.priority, data, 'failed', job.retry_limit, job.retry_count,
                     job.retry_delay, job.retry_backoff, job.retry_delay_max, startAfterColumn, startedOn,
                     job.singleton_key, singletonOn, job.group_id, job.group_tier, job.expire_seconds,
                     job.deletion_seconds, createdOn, new Date(this.config.clock.now()), keepUntil, job.policy,
-                    jobOutput, job.dead_letter,
+                    output, job.dead_letter,
                     null, job.heartbeat_seconds, job.blocked, job.blocking, job.pending_dependencies,
-                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count
+                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id
                 ]);
                 // Insert to dead letter queue if failed and has dead_letter configured
                 if (job.dead_letter) {
-                    await tx.executeSql(dlqSql, [job.dead_letter, job.data, jobOutput, job.name, job.id, createdOn, job.retry_count, job.singleton_key, job.priority, job.group_id, job.group_tier]);
+                    await tx.executeSql(dlqSql, [job.dead_letter, data, output, job.name, job.id, createdOn, job.retry_count, job.singleton_key, job.priority, job.group_id, job.group_tier, job.source_root_id]);
                 }
             }
             count++;
         }
         return count;
     }
+    // Fenced to the fetched attempt as in complete().
     async deleteJob(name, id, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'deleteJob');
+        const { ids, attempts: fetched } = this.mapAttemptArg(id, 'deleteJob');
+        const attempts = fetched ?? this.#handlerAttempts(options, ids);
         const { table } = await this.getQueueCache(name);
-        const sql = plans.deleteJobsById(this.config.schema, table);
-        const result = await db.executeSql(sql, [name, ids]);
+        const sql = plans.deleteJobsById(this.config.schema, table, !!attempts);
+        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
         const response = this.mapCommandResponse(ids, result);
         this.#trackHandlerSettle(options, response);
         return response;
@@ -1793,16 +1903,23 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { table } = await this.getQueueCache(name);
         // CockroachDB rejects the single-statement version (a DELETE and an INSERT on one table), so
-        // the same move runs as three statements in one transaction. See plans.selectRedriveCandidates.
+        // the same move runs as statements in one transaction. See plans.selectRedriveCandidates.
         if (this.config.noMultiMutationCte) {
             return this.ensureTransaction(db, async (tx) => {
                 const { rows } = await tx.executeSql(plans.selectRedriveCandidates(this.config.schema, table), [name, ...filter, limit]);
                 if (rows.length === 0)
                     return 0;
                 const ids = rows.map((row) => row.id);
-                const { rows: inserted } = await tx.executeSql(plans.insertRedrivenJobs(this.config.schema, table), [ids, filter[0]]);
-                await tx.executeSql(plans.deleteJobsByIds(this.config.schema, table).text, [ids]);
-                return inserted.length;
+                const newIds = rows.map((row) => row.new_id);
+                const { rows: inserted } = await tx.executeSql(plans.insertRedrivenJobs(this.config.schema, table), [ids, filter[0], newIds]);
+                const created = new Set(inserted.map((row) => row.id));
+                const moved = ids.filter((_, i) => created.has(newIds[i]));
+                const conflicted = ids.filter((_, i) => !created.has(newIds[i]));
+                if (moved.length)
+                    await tx.executeSql(plans.deleteJobsByIds(this.config.schema, table).text, [moved]);
+                if (conflicted.length)
+                    await tx.executeSql(plans.failRedriveConflicts(this.config.schema, table), [conflicted, filter[0]]);
+                return moved.length;
             });
         }
         const sql = plans.redriveJobs(this.config.schema, table);
@@ -1833,13 +1950,15 @@ class Manager extends EventEmitter {
             unroutable
         };
     }
+    // Fenced to the fetched attempt as in complete().
     async cancel(name, id, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'cancel');
+        const { ids, attempts: fetched } = this.mapAttemptArg(id, 'cancel');
+        const attempts = fetched ?? this.#handlerAttempts(options, ids);
         const { table } = await this.getQueueCache(name);
-        const sql = plans.cancelJobs(this.config.schema, table);
-        const result = await db.executeSql(sql, [name, ids]);
+        const sql = plans.cancelJobs(this.config.schema, table, !!attempts);
+        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
         const response = this.mapCommandResponse(ids, result);
         this.#trackHandlerSettle(options, response);
         return response;
@@ -1870,13 +1989,15 @@ class Manager extends EventEmitter {
         const result = await db.executeSql(sql, [name, ids]);
         return this.mapCommandResponse(ids, result);
     }
+    // Fenced to the fetched attempt as in complete(), so a worker that lost its claim cannot keep a
+    // newer attempt's heartbeat fresh.
     async touch(name, id, options = {}) {
         Attorney.assertQueueName(name);
         const db = this.assertDb(options);
-        const ids = this.mapCompletionIdArg(id, 'touch');
+        const { ids, attempts } = this.mapAttemptArg(id, 'touch');
         const { table } = await this.getQueueCache(name);
-        const sql = plans.touchJobs(this.config.schema, table);
-        const result = await db.executeSql(sql, [name, ids]);
+        const sql = plans.touchJobs(this.config.schema, table, !!attempts);
+        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
         return this.mapCommandResponse(ids, result);
     }
     async createQueue(name, options = {}) {
@@ -1976,10 +2097,14 @@ class Manager extends EventEmitter {
         const sql = plans.deleteStoredJobs(this.config.schema, table);
         await this.db.executeSql(sql, [name]);
     }
+    // A truncate leaves nothing to count, so it zeroes the cached counts after itself. A monitor pass
+    // that read the table first holds it until done, so the truncate and then the zeroes land after
+    // that pass's write.
     async deleteAllJobs(name) {
         if (!name) {
             const sql = plans.truncateTable(this.config.schema, plans.BASE_JOB_TABLE);
             await this.db.executeSql(sql);
+            await this.db.executeSql(plans.zeroQueueStats(this.config.schema));
             return;
         }
         Attorney.assertQueueName(name);
@@ -1987,6 +2112,7 @@ class Manager extends EventEmitter {
         if (partition) {
             const sql = plans.truncateTable(this.config.schema, table);
             await this.db.executeSql(sql);
+            await this.db.executeSql(plans.zeroQueueStats(this.config.schema, true), [name]);
         }
         else {
             const sql = plans.deleteAllJobs(this.config.schema, table);
@@ -2005,7 +2131,9 @@ class Manager extends EventEmitter {
     async getQueueStats(name, options = {}) {
         Attorney.assertQueueName(name);
         const isCockroach = this.config.backend === 'cockroachdb';
-        const toSnapshot = (row) => {
+        // `counted` is true for recorded snapshots. The cache path serves only gauges: the queue table's
+        // counters describe the last pass that counted, not this reading.
+        const toSnapshot = (row, counted = false) => {
             const snapshot = {
                 name,
                 deferredCount: 0,
@@ -2014,14 +2142,25 @@ class Manager extends EventEmitter {
                 activeCount: 0,
                 failedCount: 0,
                 totalCount: 0,
+                // Null, not zero, until something counted them: with persistQueueStats off
+                // nobody does, and a snapshot captured before 12.35 predates the columns.
+                // Zero would claim the queue was idle.
+                completedDelta: null,
+                failedDelta: null,
+                createdDelta: null,
+                deltaSeconds: null,
+                deltaOn: null,
                 capturedOn: row?.capturedOn ?? new Date(this.config.clock.now())
             };
-            for (const field of STATS_COUNT_FIELDS) {
+            for (const field of counted ? [...STATS_COUNT_FIELDS, ...STATS_DELTA_FIELDS] : STATS_COUNT_FIELDS) {
                 const value = row?.[field];
                 // CockroachDB returns integer columns as strings; normalize the counts.
                 if (value !== undefined && value !== null)
                     snapshot[field] = isCockroach ? Number(value) : value;
             }
+            // The end of the interval the counters cover, handed on as the row holds it, like capturedOn.
+            if (counted && row?.deltaOn != null)
+                snapshot.deltaOn = row.deltaOn;
             return snapshot;
         };
         if (this.config.persistQueueStats) {
@@ -2039,11 +2178,11 @@ class Manager extends EventEmitter {
                 assert(Number.isInteger(width) && width >= 1, `getQueueStats: ${mode === 'bucket' ? 'bucketSeconds' : 'maxDataPoints'} must be a positive integer`);
                 const sql = plans.getQueueStatsHistoryBucketed(this.config.schema, aggregate, mode);
                 const { rows } = await this.db.executeSql(sql, [name, from, to, limit, width]);
-                return rows.map(toSnapshot);
+                return rows.map(row => toSnapshot(row, true));
             }
             const sql = plans.getQueueStatsHistory(this.config.schema);
             const { rows } = await this.db.executeSql(sql, [name, from, to, limit]);
-            return rows.map(toSnapshot);
+            return rows.map(row => toSnapshot(row, true));
         }
         // persistQueueStats disabled: serve the cached counts the monitor keeps on the queue table.
         // capturedOn is monitor_on, NULL if never monitored, or old if monitoring has since been turned