pg-boss 12.35.0 → 12.36.0

package/dist/manager.js

@@ -6,11 +6,13 @@ import * as Attorney from "./attorney.js";
 import { untracked } from "./activity.js";
 import { TRANSACTION_ROLLBACK_TIMEOUT_MS } from "./db.js";
 import * as plans from "./plans.js";
+import { percentile } from "./latency.js";
 import * as timekeeper from "./timekeeper.js";
 import { resolveWithinSeconds } from "./tools.js";
 import * as types from "./types.js";
 import Worker from "./worker.js";
 import { JobSpy } from "./spy.js";
+import Telemetry, {} from "./telemetry.js";
 const INTERNAL_QUEUES = Object.values(timekeeper.QUEUES).reduce((acc, i) => ({ ...acc, [i]: i }), {});
 // postgres: current transaction is aborted, commands ignored until end of transaction block
 const TRANSACTION_ABORTED = '25P02';
@@ -80,6 +82,7 @@ const NUMERIC_METADATA_FIELDS = [
 ];
 // Queue rows (plans.getQueues) return these integer columns as strings on CockroachDB too.
 const NUMERIC_QUEUE_FIELDS = [
+    'blockedCount',
     'retryLimit',
     'retryDelay',
     'retryDelayMax',
@@ -97,7 +100,8 @@ const NUMERIC_QUEUE_FIELDS = [
     'createdDelta',
     'completedDelta',
     'failedDelta',
-    'deltaSeconds'
+    'deltaSeconds',
+    'readyOldestSeconds'
 ];
 // The gauges shared by live stats and recorded snapshots (the QueueStats shape).
 const STATS_COUNT_FIELDS = [
@@ -108,6 +112,12 @@ const STATS_COUNT_FIELDS = [
     'failedCount',
     'totalCount'
 ];
+// A snapshot's histogram, LATENCY_SLOTS counts from a recorded pass or added up over a bucket, with
+// the slots stored as null (no job landed there) handed out as 0. Null when no pass counted it; all
+// zeros when one did and nothing finished. CockroachDB hands integers over as strings.
+function toBins(bins) {
+    return Array.isArray(bins) ? bins.map(n => (n == null ? 0 : Number(n))) : null;
+}
 // The throughput counters and the seconds they cover. Only recorded snapshots carry them; see
 // getQueueStats.
 const STATS_DELTA_FIELDS = [
@@ -147,6 +157,10 @@ function rethrowWriteError(err) {
     }
     throw err;
 }
+// For a json value read back from a row and bound again behind `::text::jsonb`. Null stays SQL NULL.
+function toJsonText(value) {
+    return value == null ? null : JSON.stringify(value);
+}
 class Manager extends EventEmitter {
     events = events;
     // Warn once per option per instance, not once per fetch.
@@ -183,6 +197,9 @@ class Manager extends EventEmitter {
     #localGroupActive;
     #localGroupConfig;
     #localGroupMaxLimit;
+    #telemetry;
+    // The trace context each fetched job was sent with, kept off the job object handed to the handler.
+    #traceContexts;
     constructor(db, config) {
         super();
         this.config = config;
@@ -201,6 +218,8 @@ class Manager extends EventEmitter {
         this.#localGroupActive = new Map();
         this.#localGroupConfig = new Map();
         this.#localGroupMaxLimit = new Map();
+        this.#telemetry = new Telemetry(config.openTelemetry, config.schema, () => this.queues);
+        this.#traceContexts = new WeakMap();
     }
     getSpy(name) {
         if (!this.config.__test__enableSpies) {
@@ -348,6 +367,7 @@ class Manager extends EventEmitter {
     // (each output carried per-id via a JSON recordset), so batch size never drives the statement
     // count. Any batch job the handler omits (or returns with an invalid shape) is failed with a
     // descriptive error so it retries / dead-letters per queue config.
+    // Resolves with the error the whole batch was failed with, or undefined.
     async #settlePerJob(name, jobs, result) {
         if (!Array.isArray(result)) {
             // The handler opted into perJobResults but did not return an array: a contract violation.
@@ -355,7 +375,7 @@ class Manager extends EventEmitter {
             const err = new Error('perJobResults handler must resolve with an array of job results');
             await this.fail(name, jobs, err);
             await this.#trackJobsFailed(name, jobs, err);
-            return;
+            return err;
         }
         // Index the handler's dispositions by job id, keeping only valid entries that reference a job
         // from this batch. Last write wins on duplicate ids.
@@ -387,9 +407,16 @@ class Manager extends EventEmitter {
             }
         }
         const items = (entries) => entries.map(({ job, output }) => ({ id: job.id, retryCount: job.retryCount, output }));
-        const completedIds = completed.length > 0 ? await this.#completeWithOutputs(name, items(completed)) : null;
-        const failedIds = failed.length > 0 ? await this.#failWithOutputs(name, items(failed)) : null;
-        const deadLetteredIds = deadLettered.length > 0 ? await this.#failWithOutputs(name, items(deadLettered), true) : null;
+        const ids = (entries) => entries.map(({ job }) => job.id);
+        const completedIds = completed.length > 0
+            ? await this.#telemetry.settle('complete', name, ids(completed), () => this.#completeWithOutputs(name, items(completed)))
+            : null;
+        const failedIds = failed.length > 0
+            ? await this.#telemetry.settle('fail', name, ids(failed), () => this.#failWithOutputs(name, items(failed)))
+            : null;
+        const deadLetteredIds = deadLettered.length > 0
+            ? await this.#telemetry.settle('fail', name, ids(deadLettered), () => this.#failWithOutputs(name, items(deadLettered), true))
+            : null;
         // Only the jobs each statement actually settled: the attempt fence leaves a job whose claim
         // lapsed alone, and recording it would tell a spy it settled when another attempt holds it.
         const landed = (entries, result) => {
@@ -505,6 +532,8 @@ class Manager extends EventEmitter {
      * same as any other worker's. Because the claim is outside the transaction, the jobs stay
      * visibly `active` throughout, which is what keeps heartbeats, `expireInSeconds`, and another
      * instance's supervisor working on them as usual.
+     *
+     * Resolves with the error the batch was failed with, or undefined when it completed.
      */
     async #processJobs(name, jobs, callback, worker, heartbeatRefreshSeconds, perJobResults = false, transactional = false, transactionTimeoutSeconds) {
         const jobIds = jobs.map(job => job.id);
@@ -565,6 +594,9 @@ class Manager extends EventEmitter {
         let completedAffected = 0;
         let failedError;
         let didFail = false;
+        // A perJobResults batch #settlePerJob failed as a whole. Kept apart from failedError, since
+        // #settlePerJob has already failed and tracked those jobs itself.
+        let perJobError;
         // Only for a transactional worker, and only from the begin below until it settles. rollback()
         // is idempotent, so the catch can settle it without tracking whether the commit got there
         // first.
@@ -615,7 +647,7 @@ class Manager extends EventEmitter {
                 // #settlePerJob settles each job individually and does its own (synchronous,
                 // lookup-free) spy tracking via #trackJobsSettled, so the deferred tracker below
                 // is skipped for this path.
-                await this.#settlePerJob(name, jobs, result);
+                perJobError = await this.#settlePerJob(name, jobs, result);
             }
             else {
                 // Read out before the completion below, which goes through the same complete() and would
@@ -672,6 +704,7 @@ class Manager extends EventEmitter {
                 await this.#trackJobsCompleted(name, jobs, completedResult, completedAffected);
             }
         }
+        return didFail ? (failedError ?? new Error('handler rejected without a reason')) : perJobError;
     }
     /**
      * Gives the handler's transaction a deadline the database enforces.
@@ -818,6 +851,10 @@ class Manager extends EventEmitter {
     }
     async start() {
         this.stopped = false;
+        if (this.#telemetry.enabled) {
+            const { rows } = await this.db.executeSql(plans.currentDatabase());
+            this.#telemetry.setDatabase(rows[0].name);
+        }
         this.queueCacheInterval = this.config.clock.setInterval(() => this.onCacheQueues({ emit: true }), this.config.queueCacheIntervalSeconds * 1000);
         this.wipInterval = this.config.clock.setInterval(() => {
             const now = this.config.clock.now();
@@ -831,12 +868,14 @@ class Manager extends EventEmitter {
             }
         }, 2000);
         await this.onCacheQueues();
+        this.#telemetry.observeQueues();
     }
     async onCacheQueues({ emit = false } = {}) {
         try {
             assert(!this.config.__test__throw_queueCache, 'test error');
             const queues = await this.getQueues();
             this.queues = queues.reduce((acc, i) => { acc[i.name] = i; return acc; }, {});
+            this.#telemetry.refreshInstruments();
         }
         catch (error) {
             emit && this.emit(events.error, { ...error, message: error.message, stack: error.stack });
@@ -859,10 +898,24 @@ class Manager extends EventEmitter {
         if (this.queues)
             delete this.queues[name];
     }
+    // Replaces a queue's cache entry with its row as it stands, rather than evicting it, so a queue
+    // created again or updated stays in the cache and the queue gauge while still picking up what
+    // changed: new options, or a table that changed under it (deleted and recreated elsewhere with
+    // another partition setting).
+    async #reloadQueueCache(name) {
+        if (!this.queues)
+            return;
+        const queue = await this.getQueue(name);
+        if (queue)
+            this.queues[name] = queue;
+        else
+            this.#evictQueueCache(name);
+    }
     async stop() {
         this.stopped = true;
         this.config.clock.clearInterval(this.queueCacheInterval);
         this.config.clock.clearInterval(this.wipInterval);
+        this.#telemetry.unobserveQueues();
         // offWork stops every worker on a queue, so iterate queue names rather than workers - otherwise
         // a localConcurrency of N re-stops all N workers N times and registers N pending cleanups.
         const names = new Set([...this.workers.values()]
@@ -948,7 +1001,12 @@ class Manager extends EventEmitter {
                 const ignoreGroups = localGroupConcurrency != null
                     ? this.#getGroupsAtLocalCapacity(name)
                     : undefined;
-                return this.fetch(name, { batchSize, includeMetadata, priority, orderByCreatedOn, groupConcurrency, ignoreGroups, minPriority, maxPriority });
+                return this.#fetch(name, { batchSize, includeMetadata, priority, orderByCreatedOn, groupConcurrency, ignoreGroups, minPriority, maxPriority });
+            };
+            // Counted here rather than on fetch: jobs past localGroupConcurrency are restored, not delivered.
+            const processBatch = (batch, worker) => {
+                this.#telemetry.consumed(name, batch.length);
+                return this.#telemetry.process(name, batch, job => this.#traceContexts.get(job), () => this.#processJobs(name, batch, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds));
             };
             const onFetch = async (jobs) => {
                 if (!jobs.length)
@@ -961,7 +1019,7 @@ class Manager extends EventEmitter {
                 const worker = this.workers.get(workerId);
                 // Skip all in-memory group tracking when localGroupConcurrency is not enabled
                 if (localGroupConcurrency == null) {
-                    await this.#processJobs(name, jobs, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds);
+                    await processBatch(jobs, worker);
                 }
                 else {
                     const { allowed, excess, groupedJobs } = this.#trackLocalGroupStart(name, jobs);
@@ -976,7 +1034,7 @@ class Manager extends EventEmitter {
                                 worker.jobs = allowed;
                         }
                         if (allowed.length > 0) {
-                            await this.#processJobs(name, allowed, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds);
+                            await processBatch(allowed, worker);
                         }
                     }
                     finally {
@@ -1076,8 +1134,17 @@ class Manager extends EventEmitter {
             .filter(i => i.state !== 'stopped' && (!INTERNAL_QUEUES[i.name] || includeInternal));
         return data;
     }
-    hasPendingCleanups() {
-        return this.pendingOffWorkCleanups.size > 0;
+    trackCleanup(cleanup) {
+        this.pendingOffWorkCleanups.add(cleanup);
+        const settled = () => { this.pendingOffWorkCleanups.delete(cleanup); };
+        cleanup.then(settled, settled);
+    }
+    // Resolves once no cleanup is pending, including any a settling cleanup registers, so a graceful
+    // stop() carries on the moment its last worker has stopped.
+    async settleCleanups() {
+        while (this.pendingOffWorkCleanups.size > 0) {
+            await Promise.allSettled([...this.pendingOffWorkCleanups]);
+        }
     }
     async offWork(name, options = { wait: true }) {
         assert(name, 'queue name is required');
@@ -1106,11 +1173,8 @@ class Manager extends EventEmitter {
             this.#cleanupLocalGroupTracking(name);
         }
         else {
-            this.pendingOffWorkCleanups.add(cleanupPromise);
-            cleanupPromise.finally(() => {
-                this.pendingOffWorkCleanups.delete(cleanupPromise);
-                this.#cleanupLocalGroupTracking(name);
-            });
+            this.trackCleanup(cleanupPromise);
+            cleanupPromise.finally(() => this.#cleanupLocalGroupTracking(name));
         }
     }
     notifyWorker(workerId) {
@@ -1154,6 +1218,10 @@ class Manager extends EventEmitter {
     }
     async publish(event, data, options) {
         assert(event, 'Missing required argument');
+        // Counts no jobs of its own: each send() below records the job it creates.
+        await this.#telemetry.send('publish', event, 0, () => this.#publish(event, data, options));
+    }
+    async #publish(event, data, options) {
         const sql = plans.getQueuesForEvent(this.config.schema);
         const { rows } = await this.db.executeSql(sql, [event]);
         const results = await Promise.allSettled(rows.map(({ name }) => this.send(name, data, options)));
@@ -1220,9 +1288,12 @@ class Manager extends EventEmitter {
         };
     }
     async createJob(request) {
+        return this.#telemetry.send('send', request.name, 1, carrier => this.#createJob(request, carrier), id => id ? [id] : null);
+    }
+    async #createJob(request, traceContext) {
         const { name, data = null, options = {} } = request;
         const { db: wrapper, singletonSeconds, singletonNextSlot } = options;
-        const job = this.#toJobPayload(name, data, options);
+        const job = { ...this.#toJobPayload(name, data, options), __traceContext: traceContext };
         const db = wrapper || this.db;
         const { table, policy, notify } = await this.getQueueCache(name);
         if (policy === plans.QUEUE_POLICIES.key_strict_fifo && !job.singletonKey) {
@@ -1299,9 +1370,14 @@ class Manager extends EventEmitter {
     }
     async upsert(...args) {
         const request = Attorney.checkUpdateArgs(args, { upsert: true });
+        Attorney.assertQueueName(request.name);
+        // Only an insert counts as a send, and only an inserted job stores the trace context: an
+        // updated job keeps the trace of the send that created it.
+        return this.#telemetry.send('upsert', request.name, result => result.inserted, carrier => this.#upsert(request, carrier), result => result.jobs);
+    }
+    async #upsert(request, traceContext) {
         const { name, data } = request;
         const opts = (request.options ?? {});
-        Attorney.assertQueueName(name);
         const db = this.assertDb(opts);
         const { table, policy, notify } = await this.getQueueCache(name);
         const by = opts.id ? 'id' : 'singletonKey';
@@ -1316,7 +1392,7 @@ class Manager extends EventEmitter {
         const insertSql = plans.insertJobs(this.config.schema, { table, name, returnId: true, notify: notifyEnabled });
         const job = this.#toUpdatePayload(data, opts);
         const updatePayload = JSON.stringify(job);
-        const insertPayload = JSON.stringify([job]);
+        const insertPayload = JSON.stringify([{ ...job, __traceContext: traceContext }]);
         const result = await this.ensureTransaction(db, async (tx) => {
             const { rows: updated } = await tx.executeSql(updateSql, [updatePayload]);
             if (updated.length) {
@@ -1353,6 +1429,9 @@ class Manager extends EventEmitter {
     // statement nor kept on the objects below, so this stays the raw path it has always been.
     options = {}) {
         assert(Array.isArray(jobs), 'jobs argument should be an array');
+        return this.#telemetry.send('insert', name, jobs.length, carrier => this.#insert(name, jobs, options, carrier));
+    }
+    async #insert(name, jobs, options, traceContext) {
         const slots = options.__singletonSlots === true;
         const seenIds = new Set();
         for (const job of jobs) {
@@ -1384,6 +1463,8 @@ class Manager extends EventEmitter {
         const dataById = spy ? new Map() : undefined;
         const insertPayload = jobs.map(j => {
             const { blocked, blocking, pendingDependencies, group, __singletonSlot, ...rest } = j;
+            // Overwrites any __traceContext the caller passed.
+            Object.assign(rest, { __traceContext: traceContext });
             // Reattached only for the caller that asked for the column, so a public insert() drops the
             // field rather than handing an unvalidated value to a timestamp cast.
             if (slots && __singletonSlot !== undefined) {
@@ -1431,6 +1512,11 @@ class Manager extends EventEmitter {
     }
     async flow(jobs, options = {}) {
         Attorney.validateFlowJobs(jobs);
+        const queues = new Set(jobs.map(job => job.name));
+        const destination = queues.size === 1 ? jobs[0].name : null;
+        return this.#telemetry.send('flow', destination, jobs.length, carrier => this.#flow(jobs, options, carrier), refToId => Object.values(refToId));
+    }
+    async #flow(jobs, options, traceContext) {
         // validate and normalize each job's options the same way send()/insert() do
         const flowJobs = jobs.map(job => ({
             ...job,
@@ -1493,7 +1579,8 @@ class Manager extends EventEmitter {
                     deadLetter: j.options?.deadLetter ?? undefined,
                     blocked: dependencyCount > 0 || undefined,
                     blocking: parentRefs.has(j.ref) || undefined,
-                    pendingDependencies: dependencyCount || undefined
+                    pendingDependencies: dependencyCount || undefined,
+                    __traceContext: traceContext
                 };
             });
             statements.push(plans.insertFlowJobs(this.config.schema, { table, name: queueName }, insertPayload));
@@ -1549,6 +1636,9 @@ class Manager extends EventEmitter {
     }
     async fetch(name, options = {}) {
         Attorney.checkFetchArgs(name, options);
+        return this.#telemetry.receive(name, () => this.#fetch(name, options), job => this.#traceContexts.get(job));
+    }
+    async #fetch(name, options) {
         this.#warnDeprecatedFetchOptions(options);
         const db = this.assertDb(options);
         const { table, policy, singletonsActive } = await this.getQueueCache(name);
@@ -1559,7 +1649,8 @@ class Manager extends EventEmitter {
             name,
             policy,
             limit: options.batchSize || 1,
-            ignoreSingletons: singletonsActive
+            ignoreSingletons: singletonsActive,
+            includeTraceContext: this.#telemetry.enabled
         };
         const query = plans.fetchNextJob(fetchOptions, this.config.noSkipLocked);
         let result;
@@ -1576,6 +1667,13 @@ class Manager extends EventEmitter {
                 throw err;
         }
         const rows = result?.rows || [];
+        for (const row of rows) {
+            // A db adapter may hand jsonb back unparsed.
+            const carrier = typeof row.__traceContext === 'string' ? JSON.parse(row.__traceContext) : row.__traceContext;
+            delete row.__traceContext;
+            if (carrier)
+                this.#traceContexts.set(row, carrier);
+        }
         // Even a minimal fetch (JOB_COLUMNS_MIN) returns numeric fields like expireInSeconds and
         // heartbeatSeconds, so normalize regardless of includeMetadata.
         return this.#numericJobFields(rows);
@@ -1628,21 +1726,23 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'complete');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const outputData = this.mapCompletionDataArg(data);
-        let response;
-        // noMultiMutationCte: split the dependency-unblocking into a separate statement to
-        // avoid CockroachDB's multi-mutation CTE limitation (completeJobs updates two tables).
-        if (this.config.noMultiMutationCte) {
-            response = await this.completeDistributed(name, ids, outputData, table, db, options.includeQueued, attempts);
-        }
-        else {
-            const sql = plans.completeJobs(this.config.schema, table, options.includeQueued, !!attempts);
-            const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
-            response = this.mapCommandResponse(ids, result);
-        }
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('complete', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const outputData = this.mapCompletionDataArg(data);
+            let response;
+            // noMultiMutationCte: split the dependency-unblocking into a separate statement to
+            // avoid CockroachDB's multi-mutation CTE limitation (completeJobs updates two tables).
+            if (this.config.noMultiMutationCte) {
+                response = await this.completeDistributed(name, ids, outputData, table, db, options.includeQueued, attempts);
+            }
+            else {
+                const sql = plans.completeJobs(this.config.schema, table, options.includeQueued, !!attempts);
+                const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
+                response = this.mapCommandResponse(ids, result);
+            }
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     // Distributed complete/fail need several statements run atomically. When we own the pooled
     // connection we pin a single client via withTransaction(); when the caller supplied their own
@@ -1667,22 +1767,24 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'fail');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const outputData = this.mapCompletionDataArg(data);
-        let response;
-        // noMultiMutationCte: use separate queries to avoid CockroachDB's multi-mutation CTE limitation.
-        // The delete and re-insert run in a single transaction (see ensureTransaction) so the
-        // job cannot be lost between the two statements.
-        if (this.config.noMultiMutationCte) {
-            response = await this.failDistributed(name, ids, outputData, table, db, attempts);
-        }
-        else {
-            const sql = plans.failJobsById(this.config.schema, table, !!attempts);
-            const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
-            response = this.mapCommandResponse(ids, result);
-        }
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('fail', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const outputData = this.mapCompletionDataArg(data);
+            let response;
+            // noMultiMutationCte: use separate queries to avoid CockroachDB's multi-mutation CTE limitation.
+            // The delete and re-insert run in a single transaction (see ensureTransaction) so the
+            // job cannot be lost between the two statements.
+            if (this.config.noMultiMutationCte) {
+                response = await this.failDistributed(name, ids, outputData, table, db, attempts);
+            }
+            else {
+                const sql = plans.failJobsById(this.config.schema, table, !!attempts);
+                const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
+                response = this.mapCommandResponse(ids, result);
+            }
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     async failDistributed(name, ids, outputData, table, db, attempts) {
         // CockroachDB doesn't support multi-mutation CTEs, but does support transactions, so the
@@ -1785,6 +1887,12 @@ class Manager extends EventEmitter {
             // Dead-letter provenance, carried through so a job in a dead letter queue that fails here
             // still knows where to be redriven. See failJobsBody for the single-statement path.
             const sourceCreatedOn = job.source_created_on_text ?? job.source_created_on;
+            // The json columns go back as text. Bound as read, a job whose data is an array reaches pg as
+            // a Postgres array literal and a string as bare text, and neither parses as json.
+            const data = toJsonText(job.data);
+            const output = toJsonText(jobOutput);
+            const sourceOutput = toJsonText(job.source_output);
+            const traceContext = toJsonText(job.trace_context);
             // forceTerminal (perJobResults `deadletter`) skips retries so the job fails terminally and
             // routes straight to the dead letter queue below.
             const canRetry = !forceTerminal && retryCount < retryLimit;
@@ -1807,13 +1915,13 @@ class Manager extends EventEmitter {
                 // pending_dependencies are preserved so flows and heartbeat detection survive a retry
                 // (matches the non-distributed failJobs() CTE).
                 const { rows } = await tx.executeSql(insertSql, [
-                    job.id, job.name, job.priority, job.data, 'retry', job.retry_limit, job.retry_count,
+                    job.id, job.name, job.priority, data, 'retry', job.retry_limit, job.retry_count,
                     job.retry_delay, job.retry_backoff, job.retry_delay_max, startAfter, startedOn,
                     job.singleton_key, singletonOn, job.group_id, job.group_tier, job.expire_seconds,
                     job.deletion_seconds, createdOn, null, keepUntil, job.policy,
-                    jobOutput, job.dead_letter,
+                    output, job.dead_letter,
                     null, job.heartbeat_seconds, job.blocked, job.blocking, job.pending_dependencies,
-                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, job.source_output, job.source_root_id
+                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id, traceContext
                 ]);
                 // The retry insert can be dropped by ON CONFLICT when the queue policy (e.g. stately,
                 // singleton, key_strict_fifo) already has a non-terminal job. Mirror the failed_jobs
@@ -1822,17 +1930,17 @@ class Manager extends EventEmitter {
             }
             if (!retried) {
                 await tx.executeSql(insertSql, [
-                    job.id, job.name, job.priority, job.data, 'failed', job.retry_limit, job.retry_count,
+                    job.id, job.name, job.priority, data, 'failed', job.retry_limit, job.retry_count,
                     job.retry_delay, job.retry_backoff, job.retry_delay_max, startAfterColumn, startedOn,
                     job.singleton_key, singletonOn, job.group_id, job.group_tier, job.expire_seconds,
                     job.deletion_seconds, createdOn, new Date(this.config.clock.now()), keepUntil, job.policy,
-                    jobOutput, job.dead_letter,
+                    output, job.dead_letter,
                     null, job.heartbeat_seconds, job.blocked, job.blocking, job.pending_dependencies,
-                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, job.source_output, job.source_root_id
+                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id, traceContext
                 ]);
                 // Insert to dead letter queue if failed and has dead_letter configured
                 if (job.dead_letter) {
-                    await tx.executeSql(dlqSql, [job.dead_letter, job.data, jobOutput, job.name, job.id, createdOn, job.retry_count, job.singleton_key, job.priority, job.group_id, job.group_tier, job.source_root_id]);
+                    await tx.executeSql(dlqSql, [job.dead_letter, data, output, job.name, job.id, createdOn, job.retry_count, job.singleton_key, job.priority, job.group_id, job.group_tier, job.source_root_id, traceContext]);
                 }
             }
             count++;
@@ -1845,12 +1953,14 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'deleteJob');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const sql = plans.deleteJobsById(this.config.schema, table, !!attempts);
-        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
-        const response = this.mapCommandResponse(ids, result);
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('delete', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const sql = plans.deleteJobsById(this.config.schema, table, !!attempts);
+            const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
+            const response = this.mapCommandResponse(ids, result);
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     // The filter half of redrive and previewRedrive, validated once and in the parameter order
     // plans.redriveWhere expects ($2 through $6).
@@ -1941,12 +2051,14 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'cancel');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const sql = plans.cancelJobs(this.config.schema, table, !!attempts);
-        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
-        const response = this.mapCommandResponse(ids, result);
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('cancel', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const sql = plans.cancelJobs(this.config.schema, table, !!attempts);
+            const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
+            const response = this.mapCommandResponse(ids, result);
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     async resume(name, id, options = {}) {
         Attorney.assertQueueName(name);
@@ -1998,7 +2110,7 @@ class Manager extends EventEmitter {
         }
         const sql = plans.createQueue(this.config.schema, name, { ...options, policy }, this.config.noAdvisoryLocks);
         await this.db.executeSql(sql);
-        this.#evictQueueCache(name);
+        await this.#reloadQueueCache(name);
     }
     async getBlockedKeys(name) {
         Attorney.assertQueueName(name);
@@ -2028,6 +2140,11 @@ class Manager extends EventEmitter {
                 }
             }
         }
+        // Every backend: the histograms' empty slots are stored as null and handed out as 0.
+        for (const row of rows) {
+            row.waitBins = toBins(row.waitBins);
+            row.runBins = toBins(row.runBins);
+        }
         return rows;
     }
     async updateQueue(name, options = {}) {
@@ -2049,7 +2166,7 @@ class Manager extends EventEmitter {
         }
         const sql = plans.updateQueue(this.config.schema);
         await this.db.executeSql(sql, [name, options]);
-        this.#evictQueueCache(name);
+        await this.#reloadQueueCache(name);
     }
     async getQueue(name) {
         const rows = await this.getQueues([name]);
@@ -2082,10 +2199,14 @@ class Manager extends EventEmitter {
         const sql = plans.deleteStoredJobs(this.config.schema, table);
         await this.db.executeSql(sql, [name]);
     }
+    // A truncate leaves nothing to count, so it zeroes the cached counts after itself. A monitor pass
+    // that read the table first holds it until done, so the truncate and then the zeroes land after
+    // that pass's write.
     async deleteAllJobs(name) {
         if (!name) {
             const sql = plans.truncateTable(this.config.schema, plans.BASE_JOB_TABLE);
             await this.db.executeSql(sql);
+            await this.db.executeSql(plans.zeroQueueStats(this.config.schema));
             return;
         }
         Attorney.assertQueueName(name);
@@ -2093,6 +2214,7 @@ class Manager extends EventEmitter {
         if (partition) {
             const sql = plans.truncateTable(this.config.schema, table);
             await this.db.executeSql(sql);
+            await this.db.executeSql(plans.zeroQueueStats(this.config.schema, true), [name]);
         }
         else {
             const sql = plans.deleteAllJobs(this.config.schema, table);
@@ -2110,6 +2232,10 @@ class Manager extends EventEmitter {
     // getQueue(name).
     async getQueueStats(name, options = {}) {
         Attorney.assertQueueName(name);
+        assert(options.percentiles === undefined || (Array.isArray(options.percentiles) && options.percentiles.length > 0 &&
+            options.percentiles.every(p => typeof p === 'number' && p >= 1 && p <= 100)), 'getQueueStats: percentiles must be a non-empty array of percents from 1 to 100, such as [50, 95]');
+        // Each value once, in the order first asked for.
+        const percentiles = options.percentiles && [...new Set(options.percentiles)];
         const isCockroach = this.config.backend === 'cockroachdb';
         // `counted` is true for recorded snapshots. The cache path serves only gauges: the queue table's
         // counters describe the last pass that counted, not this reading.
@@ -2130,6 +2256,9 @@ class Manager extends EventEmitter {
                 createdDelta: null,
                 deltaSeconds: null,
                 deltaOn: null,
+                waitBins: null,
+                runBins: null,
+                readyOldestSeconds: null,
                 capturedOn: row?.capturedOn ?? new Date(this.config.clock.now())
             };
             for (const field of counted ? [...STATS_COUNT_FIELDS, ...STATS_DELTA_FIELDS] : STATS_COUNT_FIELDS) {
@@ -2141,6 +2270,20 @@ class Manager extends EventEmitter {
             // The end of the interval the counters cover, handed on as the row holds it, like capturedOn.
             if (counted && row?.deltaOn != null)
                 snapshot.deltaOn = row.deltaOn;
+            if (counted) {
+                snapshot.waitBins = toBins(row?.waitBins);
+                snapshot.runBins = toBins(row?.runBins);
+                if (row?.readyOldestSeconds != null)
+                    snapshot.readyOldestSeconds = Number(row.readyOldestSeconds);
+            }
+            // Read from this snapshot's (or bucket's) own histograms, so they are null wherever those are.
+            if (percentiles) {
+                snapshot.percentiles = percentiles.map(p => ({
+                    p,
+                    waitSeconds: percentile(snapshot.waitBins, p),
+                    runSeconds: percentile(snapshot.runBins, p)
+                }));
+            }
             return snapshot;
         };
         if (this.config.persistQueueStats) {