pg-boss 12.35.1 → 12.37.0

package/dist/manager.js

@@ -6,14 +6,24 @@ import * as Attorney from "./attorney.js";
 import { untracked } from "./activity.js";
 import { TRANSACTION_ROLLBACK_TIMEOUT_MS } from "./db.js";
 import * as plans from "./plans.js";
+import { percentile } from "./latency.js";
 import * as timekeeper from "./timekeeper.js";
-import { resolveWithinSeconds } from "./tools.js";
+import { delay, resolveWithinSeconds } from "./tools.js";
 import * as types from "./types.js";
 import Worker from "./worker.js";
 import { JobSpy } from "./spy.js";
+import Telemetry, {} from "./telemetry.js";
 const INTERNAL_QUEUES = Object.values(timekeeper.QUEUES).reduce((acc, i) => ({ ...acc, [i]: i }), {});
 // postgres: current transaction is aborted, commands ignored until end of transaction block
 const TRANSACTION_ABORTED = '25P02';
+// serialization_failure, which CockroachDB raises for a transaction it has to restart.
+const SERIALIZATION_FAILURE = '40001';
+const UPSERT_ATTEMPTS = 3;
+const QUEUE_NOT_FOUND = 'PGBOSS_QUEUE_NOT_FOUND';
+// How long deleteQueue() keeps trying for the locks a queue with its own table needs: 25 ms doubling
+// to 400 ms between tries, about 3 seconds in all.
+const DELETE_QUEUE_ATTEMPTS = 12;
+const DELETE_QUEUE_MAX_DELAY_MS = 400;
 // pg's own default when the pool size is not configured. Used to tell a transactional worker how
 // much room it actually has, since each handler in flight holds a connection of its own.
 const DEFAULT_POOL_MAX = 10;
@@ -80,6 +90,7 @@ const NUMERIC_METADATA_FIELDS = [
 ];
 // Queue rows (plans.getQueues) return these integer columns as strings on CockroachDB too.
 const NUMERIC_QUEUE_FIELDS = [
+    'blockedCount',
     'retryLimit',
     'retryDelay',
     'retryDelayMax',
@@ -97,7 +108,8 @@ const NUMERIC_QUEUE_FIELDS = [
     'createdDelta',
     'completedDelta',
     'failedDelta',
-    'deltaSeconds'
+    'deltaSeconds',
+    'readyOldestSeconds'
 ];
 // The gauges shared by live stats and recorded snapshots (the QueueStats shape).
 const STATS_COUNT_FIELDS = [
@@ -108,6 +120,12 @@ const STATS_COUNT_FIELDS = [
     'failedCount',
     'totalCount'
 ];
+// A snapshot's histogram, LATENCY_SLOTS counts from a recorded pass or added up over a bucket, with
+// the slots stored as null (no job landed there) handed out as 0. Null when no pass counted it; all
+// zeros when one did and nothing finished. CockroachDB hands integers over as strings.
+function toBins(bins) {
+    return Array.isArray(bins) ? bins.map(n => (n == null ? 0 : Number(n))) : null;
+}
 // The throughput counters and the seconds they cover. Only recorded snapshots carry them; see
 // getQueueStats.
 const STATS_DELTA_FIELDS = [
@@ -147,6 +165,16 @@ function rethrowWriteError(err) {
     }
     throw err;
 }
+// The foreign key a job insert broke, from the error's constraint field where the driver reports one
+// and from the message otherwise, and the key value it could not find. Bun.SQL carries the SQLSTATE
+// in errno.
+function brokenForeignKey(err) {
+    if (err?.code !== plans.PG_ERROR.foreignKeyViolation && err?.errno !== plans.PG_ERROR.foreignKeyViolation)
+        return;
+    const constraint = err.constraint ?? err.constraint_name ?? /foreign key constraint "(\w+)"/.exec(err.message)?.[1];
+    const value = /\)=\('?(.*?)'?\) is not present/.exec(err.detail ?? err.message)?.[1];
+    return { constraint, value };
+}
 // For a json value read back from a row and bound again behind `::text::jsonb`. Null stays SQL NULL.
 function toJsonText(value) {
     return value == null ? null : JSON.stringify(value);
@@ -187,6 +215,9 @@ class Manager extends EventEmitter {
     #localGroupActive;
     #localGroupConfig;
     #localGroupMaxLimit;
+    #telemetry;
+    // The trace context each fetched job was sent with, kept off the job object handed to the handler.
+    #traceContexts;
     constructor(db, config) {
         super();
         this.config = config;
@@ -205,6 +236,8 @@ class Manager extends EventEmitter {
         this.#localGroupActive = new Map();
         this.#localGroupConfig = new Map();
         this.#localGroupMaxLimit = new Map();
+        this.#telemetry = new Telemetry(config.openTelemetry, config.schema, () => this.queues);
+        this.#traceContexts = new WeakMap();
     }
     getSpy(name) {
         if (!this.config.__test__enableSpies) {
@@ -352,6 +385,7 @@ class Manager extends EventEmitter {
     // (each output carried per-id via a JSON recordset), so batch size never drives the statement
     // count. Any batch job the handler omits (or returns with an invalid shape) is failed with a
     // descriptive error so it retries / dead-letters per queue config.
+    // Resolves with the error the whole batch was failed with, or undefined.
     async #settlePerJob(name, jobs, result) {
         if (!Array.isArray(result)) {
             // The handler opted into perJobResults but did not return an array: a contract violation.
@@ -359,7 +393,7 @@ class Manager extends EventEmitter {
             const err = new Error('perJobResults handler must resolve with an array of job results');
             await this.fail(name, jobs, err);
             await this.#trackJobsFailed(name, jobs, err);
-            return;
+            return err;
         }
         // Index the handler's dispositions by job id, keeping only valid entries that reference a job
         // from this batch. Last write wins on duplicate ids.
@@ -391,9 +425,16 @@ class Manager extends EventEmitter {
             }
         }
         const items = (entries) => entries.map(({ job, output }) => ({ id: job.id, retryCount: job.retryCount, output }));
-        const completedIds = completed.length > 0 ? await this.#completeWithOutputs(name, items(completed)) : null;
-        const failedIds = failed.length > 0 ? await this.#failWithOutputs(name, items(failed)) : null;
-        const deadLetteredIds = deadLettered.length > 0 ? await this.#failWithOutputs(name, items(deadLettered), true) : null;
+        const ids = (entries) => entries.map(({ job }) => job.id);
+        const completedIds = completed.length > 0
+            ? await this.#telemetry.settle('complete', name, ids(completed), () => this.#completeWithOutputs(name, items(completed)))
+            : null;
+        const failedIds = failed.length > 0
+            ? await this.#telemetry.settle('fail', name, ids(failed), () => this.#failWithOutputs(name, items(failed)))
+            : null;
+        const deadLetteredIds = deadLettered.length > 0
+            ? await this.#telemetry.settle('fail', name, ids(deadLettered), () => this.#failWithOutputs(name, items(deadLettered), true))
+            : null;
         // Only the jobs each statement actually settled: the attempt fence leaves a job whose claim
         // lapsed alone, and recording it would tell a spy it settled when another attempt holds it.
         const landed = (entries, result) => {
@@ -509,6 +550,8 @@ class Manager extends EventEmitter {
      * same as any other worker's. Because the claim is outside the transaction, the jobs stay
      * visibly `active` throughout, which is what keeps heartbeats, `expireInSeconds`, and another
      * instance's supervisor working on them as usual.
+     *
+     * Resolves with the error the batch was failed with, or undefined when it completed.
      */
     async #processJobs(name, jobs, callback, worker, heartbeatRefreshSeconds, perJobResults = false, transactional = false, transactionTimeoutSeconds) {
         const jobIds = jobs.map(job => job.id);
@@ -569,6 +612,9 @@ class Manager extends EventEmitter {
         let completedAffected = 0;
         let failedError;
         let didFail = false;
+        // A perJobResults batch #settlePerJob failed as a whole. Kept apart from failedError, since
+        // #settlePerJob has already failed and tracked those jobs itself.
+        let perJobError;
         // Only for a transactional worker, and only from the begin below until it settles. rollback()
         // is idempotent, so the catch can settle it without tracking whether the commit got there
         // first.
@@ -600,6 +646,10 @@ class Manager extends EventEmitter {
                     await this.#applyTransactionTimeout(transaction, timeoutGuc, timeoutMs);
                 }
             }
+            // A fetch or transaction begin can finish after stop() has exhausted its active-handler grace.
+            if (worker?.graceExpired) {
+                throw new Error('pg-boss shut down before the handler started');
+            }
             const handling = transaction
                 ? callback(jobs, untracked(transaction.db))
                 : callback(jobs);
@@ -619,7 +669,7 @@ class Manager extends EventEmitter {
                 // #settlePerJob settles each job individually and does its own (synchronous,
                 // lookup-free) spy tracking via #trackJobsSettled, so the deferred tracker below
                 // is skipped for this path.
-                await this.#settlePerJob(name, jobs, result);
+                perJobError = await this.#settlePerJob(name, jobs, result);
             }
             else {
                 // Read out before the completion below, which goes through the same complete() and would
@@ -676,6 +726,7 @@ class Manager extends EventEmitter {
                 await this.#trackJobsCompleted(name, jobs, completedResult, completedAffected);
             }
         }
+        return didFail ? (failedError ?? new Error('handler rejected without a reason')) : perJobError;
     }
     /**
      * Gives the handler's transaction a deadline the database enforces.
@@ -822,6 +873,10 @@ class Manager extends EventEmitter {
     }
     async start() {
         this.stopped = false;
+        if (this.#telemetry.enabled) {
+            const { rows } = await this.db.executeSql(plans.currentDatabase());
+            this.#telemetry.setDatabase(rows[0].name);
+        }
         this.queueCacheInterval = this.config.clock.setInterval(() => this.onCacheQueues({ emit: true }), this.config.queueCacheIntervalSeconds * 1000);
         this.wipInterval = this.config.clock.setInterval(() => {
             const now = this.config.clock.now();
@@ -835,12 +890,14 @@ class Manager extends EventEmitter {
             }
         }, 2000);
         await this.onCacheQueues();
+        this.#telemetry.observeQueues();
     }
     async onCacheQueues({ emit = false } = {}) {
         try {
             assert(!this.config.__test__throw_queueCache, 'test error');
             const queues = await this.getQueues();
             this.queues = queues.reduce((acc, i) => { acc[i.name] = i; return acc; }, {});
+            this.#telemetry.refreshInstruments();
         }
         catch (error) {
             emit && this.emit(events.error, { ...error, message: error.message, stack: error.stack });
@@ -854,7 +911,7 @@ class Manager extends EventEmitter {
         }
         queue = await this.getQueue(name);
         if (!queue) {
-            throw new Error(`Queue ${name} does not exist`);
+            throw Object.assign(new Error(`Queue ${name} does not exist`), { code: QUEUE_NOT_FOUND });
         }
         this.queues[name] = queue;
         return queue;
@@ -863,10 +920,60 @@ class Manager extends EventEmitter {
         if (this.queues)
             delete this.queues[name];
     }
+    // A job insert is what notices a queue deleted after this instance cached it: q_fkey fails for a
+    // queue in the shared table, and a queue with its own table (partition: true) fails on the dropped
+    // table (42P01), dlq_fkey for its dead letter queue. Each is rewritten into the error a missing
+    // queue gets up front, and a missing queue loses its cache entry so the next call fails before
+    // reaching the database. `names` are the queues the statement wrote to.
+    async #rethrowInsertError(err, names, owned) {
+        const broken = brokenForeignKey(err);
+        if (broken?.constraint === 'q_fkey') {
+            this.#throwQueueNotFound(broken.value && names.includes(broken.value) ? [broken.value] : names, err, owned);
+        }
+        // A missing table can also mean a queue deleted and created again elsewhere under its own table, and
+        // a partition constraint (23514) one created again with its own table while this instance still
+        // writes to the shared one. So the queues are read again: a gone one is reported as missing, and the
+        // others take their current row, so the next call writes to the right table. A failed read leaves
+        // the original error.
+        if (err?.code === plans.PG_ERROR.undefinedTable || err?.code === plans.PG_ERROR.checkViolation) {
+            const queues = await Promise.all(names.map(name => this.getQueue(name))).catch(() => null);
+            if (queues) {
+                const missing = names.filter((name, i) => !queues[i]);
+                if (missing.length)
+                    this.#throwQueueNotFound(missing, err, owned);
+                queues.forEach(queue => { if (queue && this.queues)
+                    this.queues[queue.name] = queue; });
+            }
+        }
+        if (broken?.constraint === 'dlq_fkey') {
+            throw new Error(broken.value ? `Dead letter queue ${broken.value} does not exist` : 'Dead letter queue does not exist', { cause: err });
+        }
+        rethrowWriteError(err);
+    }
+    // Coded, so publish() skips it, only when the statement ran on pg-boss's own connection: a failed
+    // statement in a caller's transaction has aborted it, and that has to reach the caller.
+    #throwQueueNotFound(missing, cause, owned) {
+        missing.forEach(name => this.#evictQueueCache(name));
+        throw Object.assign(new Error(`Queue ${missing.join(', ')} does not exist`, { cause }), owned ? { code: QUEUE_NOT_FOUND } : {});
+    }
+    // Replaces a queue's cache entry with its row as it stands, rather than evicting it, so a queue
+    // created again or updated stays in the cache and the queue gauge while still picking up what
+    // changed: new options, or a table that changed under it (deleted and recreated elsewhere with
+    // another partition setting).
+    async #reloadQueueCache(name) {
+        if (!this.queues)
+            return;
+        const queue = await this.getQueue(name);
+        if (queue)
+            this.queues[name] = queue;
+        else
+            this.#evictQueueCache(name);
+    }
     async stop() {
         this.stopped = true;
         this.config.clock.clearInterval(this.queueCacheInterval);
         this.config.clock.clearInterval(this.wipInterval);
+        this.#telemetry.unobserveQueues();
         // offWork stops every worker on a queue, so iterate queue names rather than workers - otherwise
         // a localConcurrency of N re-stops all N workers N times and registers N pending cleanups.
         const names = new Set([...this.workers.values()]
@@ -882,6 +989,11 @@ class Manager extends EventEmitter {
     // `stopped` all wait on this, so every worker has to be reached even when one of them cannot be
     // failed.
     async failWip() {
+        // Every worker is marked before any fail below is awaited, so a claim landing on a later worker
+        // while an earlier one's fail is in flight is refused too.
+        for (const worker of this.workers.values()) {
+            worker.graceExpired = true;
+        }
         for (const worker of this.workers.values()) {
             const jobIds = worker.jobs.map(j => j.id);
             if (jobIds.length) {
@@ -919,6 +1031,10 @@ class Manager extends EventEmitter {
         if (transactional) {
             assert(typeof this.db.beginTransaction === 'function', 'transactional workers require a database connection pg-boss can open a transaction on: the built-in pool, or a db adapter implementing beginTransaction');
             await this.#assertTransactionalHeartbeatSupported(name);
+            // A stop() during that await has already looked for workers, and would not see this one.
+            if (this.stopped) {
+                throw new Error('Workers are disabled. pg-boss is stopped');
+            }
             this.#warnOnTransactionalPoolHeadroom(localConcurrency);
         }
         if (localGroupConcurrency != null) {
@@ -952,7 +1068,12 @@ class Manager extends EventEmitter {
                 const ignoreGroups = localGroupConcurrency != null
                     ? this.#getGroupsAtLocalCapacity(name)
                     : undefined;
-                return this.fetch(name, { batchSize, includeMetadata, priority, orderByCreatedOn, groupConcurrency, ignoreGroups, minPriority, maxPriority });
+                return this.#fetch(name, { batchSize, includeMetadata, priority, orderByCreatedOn, groupConcurrency, ignoreGroups, minPriority, maxPriority });
+            };
+            // Counted here rather than on fetch: jobs past localGroupConcurrency are restored, not delivered.
+            const processBatch = (batch, worker) => {
+                this.#telemetry.consumed(name, batch.length);
+                return this.#telemetry.process(name, batch, job => this.#traceContexts.get(job), () => this.#processJobs(name, batch, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds));
             };
             const onFetch = async (jobs) => {
                 if (!jobs.length)
@@ -965,7 +1086,7 @@ class Manager extends EventEmitter {
                 const worker = this.workers.get(workerId);
                 // Skip all in-memory group tracking when localGroupConcurrency is not enabled
                 if (localGroupConcurrency == null) {
-                    await this.#processJobs(name, jobs, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds);
+                    await processBatch(jobs, worker);
                 }
                 else {
                     const { allowed, excess, groupedJobs } = this.#trackLocalGroupStart(name, jobs);
@@ -980,7 +1101,7 @@ class Manager extends EventEmitter {
                                 worker.jobs = allowed;
                         }
                         if (allowed.length > 0) {
-                            await this.#processJobs(name, allowed, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds);
+                            await processBatch(allowed, worker);
                         }
                     }
                     finally {
@@ -1164,12 +1285,17 @@ class Manager extends EventEmitter {
     }
     async publish(event, data, options) {
         assert(event, 'Missing required argument');
+        // Counts no jobs of its own: each send() below records the job it creates.
+        await this.#telemetry.send('publish', event, 0, () => this.#publish(event, data, options));
+    }
+    async #publish(event, data, options) {
         const sql = plans.getQueuesForEvent(this.config.schema);
         const { rows } = await this.db.executeSql(sql, [event]);
         const results = await Promise.allSettled(rows.map(({ name }) => this.send(name, data, options)));
         const failures = results
             .map((result, index) => ({ result, name: rows[index].name }))
-            .filter((entry) => entry.result.status === 'rejected');
+            // A deleted subscriber can still be in the selection, just as a cached send can insert no job.
+            .filter((entry) => entry.result.status === 'rejected' && entry.result.reason?.code !== QUEUE_NOT_FOUND);
         if (failures.length > 0) {
             // Each entry names its own queue, so attribution doesn't depend on lining errors[] up with
             // the queue order positionally, and survives two subscribers failing with identical text.
@@ -1230,16 +1356,20 @@ class Manager extends EventEmitter {
         };
     }
     async createJob(request) {
+        return this.#telemetry.send('send', request.name, 1, carrier => this.#createJob(request, carrier), id => id ? [id] : null);
+    }
+    async #createJob(request, traceContext) {
         const { name, data = null, options = {} } = request;
         const { db: wrapper, singletonSeconds, singletonNextSlot } = options;
-        const job = this.#toJobPayload(name, data, options);
+        const job = { ...this.#toJobPayload(name, data, options), __traceContext: traceContext };
         const db = wrapper || this.db;
         const { table, policy, notify } = await this.getQueueCache(name);
         if (policy === plans.QUEUE_POLICIES.key_strict_fifo && !job.singletonKey) {
             throw new Error(`${plans.QUEUE_POLICIES.key_strict_fifo} queues require a singletonKey`);
         }
         const sql = plans.insertJobs(this.config.schema, { table, name, returnId: true, notify: this.#notifyEnabled(notify) });
-        const { rows: try1 } = await db.executeSql(sql, [JSON.stringify([job])]);
+        const insert = () => db.executeSql(sql, [JSON.stringify([job])]).catch(err => this.#rethrowInsertError(err, [name], !wrapper));
+        const { rows: try1 } = await insert();
         if (try1.length === 1) {
             const jobId = try1[0].id;
             if (this.config.__test__enableSpies) {
@@ -1254,7 +1384,7 @@ class Manager extends EventEmitter {
             // delay starting by the offset to honor throttling config
             job.startAfter = this.getDebounceStartAfter(singletonSeconds, this.timekeeper.clockSkew);
             job.singletonOffset = singletonSeconds;
-            const { rows: try2 } = await db.executeSql(sql, [JSON.stringify([job])]);
+            const { rows: try2 } = await insert();
             if (try2.length === 1) {
                 const jobId = try2[0].id;
                 if (this.config.__test__enableSpies) {
@@ -1309,9 +1439,14 @@ class Manager extends EventEmitter {
     }
     async upsert(...args) {
         const request = Attorney.checkUpdateArgs(args, { upsert: true });
+        Attorney.assertQueueName(request.name);
+        // Only an insert counts as a send, and only an inserted job stores the trace context: an
+        // updated job keeps the trace of the send that created it.
+        return this.#telemetry.send('upsert', request.name, result => result.inserted, carrier => this.#upsert(request, carrier), result => result.jobs);
+    }
+    async #upsert(request, traceContext) {
         const { name, data } = request;
         const opts = (request.options ?? {});
-        Attorney.assertQueueName(name);
         const db = this.assertDb(opts);
         const { table, policy, notify } = await this.getQueueCache(name);
         const by = opts.id ? 'id' : 'singletonKey';
@@ -1323,11 +1458,12 @@ class Manager extends EventEmitter {
         }
         const notifyEnabled = this.#notifyEnabled(notify);
         const updateSql = plans.updateJob(this.config.schema, table, name, by, match, notifyEnabled);
-        const insertSql = plans.insertJobs(this.config.schema, { table, name, returnId: true, notify: notifyEnabled });
+        const insertSql = plans.insertJobs(this.config.schema, { table, name, returnId: true, notify: notifyEnabled, upsertByKey: by === 'singletonKey' });
         const job = this.#toUpdatePayload(data, opts);
         const updatePayload = JSON.stringify(job);
-        const insertPayload = JSON.stringify([job]);
-        const result = await this.ensureTransaction(db, async (tx) => {
+        const insertPayload = JSON.stringify([{ ...job, __traceContext: traceContext }]);
+        // The catch is outside the transaction: a deferred q_fkey reports at its COMMIT.
+        const attempt = () => this.ensureTransaction(db, async (tx) => {
             const { rows: updated } = await tx.executeSql(updateSql, [updatePayload]);
             if (updated.length) {
                 const jobs = updated.map(row => row.id);
@@ -1343,7 +1479,22 @@ class Manager extends EventEmitter {
             const { rows: retry } = await tx.executeSql(updateSql, [updatePayload]);
             const jobs = retry.map(row => row.id);
             return { jobs, updated: jobs.length, inserted: 0 };
-        });
+        }).catch(err => this.#rethrowInsertError(err, [name], !opts.db));
+        // Where every transaction is serializable (CockroachDB), two upserts of one key that both miss
+        // conflict and one fails with 40001, rather than waiting on job_i13 as under READ COMMITTED. The
+        // transaction is pg-boss's own unless the caller passed a db, so the upsert runs again and finds
+        // the other's job. A caller's transaction is theirs to retry.
+        let result;
+        for (let attempts = 1;; attempts++) {
+            try {
+                result = await attempt();
+                break;
+            }
+            catch (err) {
+                if (opts.db || err?.code !== SERIALIZATION_FAILURE || attempts === UPSERT_ATTEMPTS)
+                    throw err;
+            }
+        }
         // Track inserted (newly created) jobs for spies, matching createJob/insert. Runs after the
         // transaction commits so a rolled-back insert never leaves a phantom spy entry.
         if (result.inserted && this.config.__test__enableSpies) {
@@ -1363,6 +1514,9 @@ class Manager extends EventEmitter {
     // statement nor kept on the objects below, so this stays the raw path it has always been.
     options = {}) {
         assert(Array.isArray(jobs), 'jobs argument should be an array');
+        return this.#telemetry.send('insert', name, jobs.length, carrier => this.#insert(name, jobs, options, carrier));
+    }
+    async #insert(name, jobs, options, traceContext) {
         const slots = options.__singletonSlots === true;
         const seenIds = new Set();
         for (const job of jobs) {
@@ -1394,6 +1548,8 @@ class Manager extends EventEmitter {
         const dataById = spy ? new Map() : undefined;
         const insertPayload = jobs.map(j => {
             const { blocked, blocking, pendingDependencies, group, __singletonSlot, ...rest } = j;
+            // Overwrites any __traceContext the caller passed.
+            Object.assign(rest, { __traceContext: traceContext });
             // Reattached only for the caller that asked for the column, so a public insert() drops the
             // field rather than handing an unvalidated value to a timestamp cast.
             if (slots && __singletonSlot !== undefined) {
@@ -1427,7 +1583,7 @@ class Manager extends EventEmitter {
         // Return IDs if spy is active for this queue (needed for job tracking)
         const returnId = !!spy || !!options.returnId;
         const sql = plans.insertJobs(this.config.schema, { table, name, returnId, notify: this.#notifyEnabled(notify), slots });
-        const { rows } = await db.executeSql(sql, [JSON.stringify(insertPayload)]);
+        const { rows } = await db.executeSql(sql, [JSON.stringify(insertPayload)]).catch(err => this.#rethrowInsertError(err, [name], !options.db));
         if (rows.length) {
             if (spy) {
                 // dataById is populated for every job when a spy is active
@@ -1441,6 +1597,11 @@ class Manager extends EventEmitter {
     }
     async flow(jobs, options = {}) {
         Attorney.validateFlowJobs(jobs);
+        const queues = new Set(jobs.map(job => job.name));
+        const destination = queues.size === 1 ? jobs[0].name : null;
+        return this.#telemetry.send('flow', destination, jobs.length, carrier => this.#flow(jobs, options, carrier), refToId => Object.values(refToId));
+    }
+    async #flow(jobs, options, traceContext) {
         // validate and normalize each job's options the same way send()/insert() do
         const flowJobs = jobs.map(job => ({
             ...job,
@@ -1503,7 +1664,8 @@ class Manager extends EventEmitter {
                     deadLetter: j.options?.deadLetter ?? undefined,
                     blocked: dependencyCount > 0 || undefined,
                     blocking: parentRefs.has(j.ref) || undefined,
-                    pendingDependencies: dependencyCount || undefined
+                    pendingDependencies: dependencyCount || undefined,
+                    __traceContext: traceContext
                 };
             });
             statements.push(plans.insertFlowJobs(this.config.schema, { table, name: queueName }, insertPayload));
@@ -1526,7 +1688,7 @@ class Manager extends EventEmitter {
             await db.executeSql(sql);
         }
         catch (err) {
-            rethrowWriteError(err);
+            await this.#rethrowInsertError(err, [...byQueue.keys()], !options.db);
         }
         return refToId;
     }
@@ -1559,6 +1721,9 @@ class Manager extends EventEmitter {
     }
     async fetch(name, options = {}) {
         Attorney.checkFetchArgs(name, options);
+        return this.#telemetry.receive(name, () => this.#fetch(name, options), job => this.#traceContexts.get(job));
+    }
+    async #fetch(name, options) {
         this.#warnDeprecatedFetchOptions(options);
         const db = this.assertDb(options);
         const { table, policy, singletonsActive } = await this.getQueueCache(name);
@@ -1569,7 +1734,8 @@ class Manager extends EventEmitter {
             name,
             policy,
             limit: options.batchSize || 1,
-            ignoreSingletons: singletonsActive
+            ignoreSingletons: singletonsActive,
+            includeTraceContext: this.#telemetry.enabled
         };
         const query = plans.fetchNextJob(fetchOptions, this.config.noSkipLocked);
         let result;
@@ -1586,6 +1752,13 @@ class Manager extends EventEmitter {
                 throw err;
         }
         const rows = result?.rows || [];
+        for (const row of rows) {
+            // A db adapter may hand jsonb back unparsed.
+            const carrier = typeof row.__traceContext === 'string' ? JSON.parse(row.__traceContext) : row.__traceContext;
+            delete row.__traceContext;
+            if (carrier)
+                this.#traceContexts.set(row, carrier);
+        }
         // Even a minimal fetch (JOB_COLUMNS_MIN) returns numeric fields like expireInSeconds and
         // heartbeatSeconds, so normalize regardless of includeMetadata.
         return this.#numericJobFields(rows);
@@ -1638,21 +1811,23 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'complete');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const outputData = this.mapCompletionDataArg(data);
-        let response;
-        // noMultiMutationCte: split the dependency-unblocking into a separate statement to
-        // avoid CockroachDB's multi-mutation CTE limitation (completeJobs updates two tables).
-        if (this.config.noMultiMutationCte) {
-            response = await this.completeDistributed(name, ids, outputData, table, db, options.includeQueued, attempts);
-        }
-        else {
-            const sql = plans.completeJobs(this.config.schema, table, options.includeQueued, !!attempts);
-            const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
-            response = this.mapCommandResponse(ids, result);
-        }
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('complete', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const outputData = this.mapCompletionDataArg(data);
+            let response;
+            // noMultiMutationCte: split the dependency-unblocking into a separate statement to
+            // avoid CockroachDB's multi-mutation CTE limitation (completeJobs updates two tables).
+            if (this.config.noMultiMutationCte) {
+                response = await this.completeDistributed(name, ids, outputData, table, db, options.includeQueued, attempts);
+            }
+            else {
+                const sql = plans.completeJobs(this.config.schema, table, options.includeQueued, !!attempts);
+                const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
+                response = this.mapCommandResponse(ids, result);
+            }
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     // Distributed complete/fail need several statements run atomically. When we own the pooled
     // connection we pin a single client via withTransaction(); when the caller supplied their own
@@ -1677,22 +1852,24 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'fail');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const outputData = this.mapCompletionDataArg(data);
-        let response;
-        // noMultiMutationCte: use separate queries to avoid CockroachDB's multi-mutation CTE limitation.
-        // The delete and re-insert run in a single transaction (see ensureTransaction) so the
-        // job cannot be lost between the two statements.
-        if (this.config.noMultiMutationCte) {
-            response = await this.failDistributed(name, ids, outputData, table, db, attempts);
-        }
-        else {
-            const sql = plans.failJobsById(this.config.schema, table, !!attempts);
-            const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
-            response = this.mapCommandResponse(ids, result);
-        }
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('fail', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const outputData = this.mapCompletionDataArg(data);
+            let response;
+            // noMultiMutationCte: use separate queries to avoid CockroachDB's multi-mutation CTE limitation.
+            // The delete and re-insert run in a single transaction (see ensureTransaction) so the
+            // job cannot be lost between the two statements.
+            if (this.config.noMultiMutationCte) {
+                response = await this.failDistributed(name, ids, outputData, table, db, attempts);
+            }
+            else {
+                const sql = plans.failJobsById(this.config.schema, table, !!attempts);
+                const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
+                response = this.mapCommandResponse(ids, result);
+            }
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     async failDistributed(name, ids, outputData, table, db, attempts) {
         // CockroachDB doesn't support multi-mutation CTEs, but does support transactions, so the
@@ -1800,6 +1977,7 @@ class Manager extends EventEmitter {
             const data = toJsonText(job.data);
             const output = toJsonText(jobOutput);
             const sourceOutput = toJsonText(job.source_output);
+            const traceContext = toJsonText(job.trace_context);
             // forceTerminal (perJobResults `deadletter`) skips retries so the job fails terminally and
             // routes straight to the dead letter queue below.
             const canRetry = !forceTerminal && retryCount < retryLimit;
@@ -1828,7 +2006,7 @@ class Manager extends EventEmitter {
                     job.deletion_seconds, createdOn, null, keepUntil, job.policy,
                     output, job.dead_letter,
                     null, job.heartbeat_seconds, job.blocked, job.blocking, job.pending_dependencies,
-                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id
+                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id, traceContext
                 ]);
                 // The retry insert can be dropped by ON CONFLICT when the queue policy (e.g. stately,
                 // singleton, key_strict_fifo) already has a non-terminal job. Mirror the failed_jobs
@@ -1843,11 +2021,11 @@ class Manager extends EventEmitter {
                     job.deletion_seconds, createdOn, new Date(this.config.clock.now()), keepUntil, job.policy,
                     output, job.dead_letter,
                     null, job.heartbeat_seconds, job.blocked, job.blocking, job.pending_dependencies,
-                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id
+                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id, traceContext
                 ]);
                 // Insert to dead letter queue if failed and has dead_letter configured
                 if (job.dead_letter) {
-                    await tx.executeSql(dlqSql, [job.dead_letter, data, output, job.name, job.id, createdOn, job.retry_count, job.singleton_key, job.priority, job.group_id, job.group_tier, job.source_root_id]);
+                    await tx.executeSql(dlqSql, [job.dead_letter, data, output, job.name, job.id, createdOn, job.retry_count, job.singleton_key, job.priority, job.group_id, job.group_tier, job.source_root_id, traceContext]);
                 }
             }
             count++;
@@ -1860,12 +2038,14 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'deleteJob');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const sql = plans.deleteJobsById(this.config.schema, table, !!attempts);
-        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
-        const response = this.mapCommandResponse(ids, result);
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('delete', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const sql = plans.deleteJobsById(this.config.schema, table, !!attempts);
+            const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
+            const response = this.mapCommandResponse(ids, result);
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     // The filter half of redrive and previewRedrive, validated once and in the parameter order
     // plans.redriveWhere expects ($2 through $6).
@@ -1956,12 +2136,14 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'cancel');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const sql = plans.cancelJobs(this.config.schema, table, !!attempts);
-        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
-        const response = this.mapCommandResponse(ids, result);
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('cancel', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const sql = plans.cancelJobs(this.config.schema, table, !!attempts);
+            const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
+            const response = this.mapCommandResponse(ids, result);
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     async resume(name, id, options = {}) {
         Attorney.assertQueueName(name);
@@ -2013,7 +2195,7 @@ class Manager extends EventEmitter {
         }
         const sql = plans.createQueue(this.config.schema, name, { ...options, policy }, this.config.noAdvisoryLocks);
         await this.db.executeSql(sql);
-        this.#evictQueueCache(name);
+        await this.#reloadQueueCache(name);
     }
     async getBlockedKeys(name) {
         Attorney.assertQueueName(name);
@@ -2043,6 +2225,11 @@ class Manager extends EventEmitter {
                 }
             }
         }
+        // Every backend: the histograms' empty slots are stored as null and handed out as 0.
+        for (const row of rows) {
+            row.waitBins = toBins(row.waitBins);
+            row.runBins = toBins(row.runBins);
+        }
         return rows;
     }
     async updateQueue(name, options = {}) {
@@ -2064,7 +2251,7 @@ class Manager extends EventEmitter {
         }
         const sql = plans.updateQueue(this.config.schema);
         await this.db.executeSql(sql, [name, options]);
-        this.#evictQueueCache(name);
+        await this.#reloadQueueCache(name);
     }
     async getQueue(name) {
         const rows = await this.getQueues([name]);
@@ -2081,8 +2268,21 @@ class Manager extends EventEmitter {
         catch {
             return;
         }
-        const sql = plans.deleteQueue(this.config.schema, name, this.config.noAdvisoryLocks);
-        await this.db.executeSql(sql);
+        const sql = plans.deleteQueue(this.config.schema, name, this.config.noAdvisoryLocks, !this.config.noTablePartitioning);
+        // Only a queue with its own table takes locks that can be busy, and only those are tried again.
+        for (let attempt = 1;; attempt++) {
+            try {
+                await this.db.executeSql(sql);
+                break;
+            }
+            catch (err) {
+                if (err?.code !== plans.PG_ERROR.lockNotAvailable)
+                    throw err;
+                if (attempt === DELETE_QUEUE_ATTEMPTS)
+                    throw new Error(`Queue ${name} was not deleted: its tables stayed locked through ${attempt} tries`, { cause: err });
+                await delay(Math.min(25 * 2 ** (attempt - 1), DELETE_QUEUE_MAX_DELAY_MS) * (0.5 + Math.random() / 2));
+            }
+        }
         this.#evictQueueCache(name);
     }
     async deleteQueuedJobs(name) {
@@ -2130,6 +2330,10 @@ class Manager extends EventEmitter {
     // getQueue(name).
     async getQueueStats(name, options = {}) {
         Attorney.assertQueueName(name);
+        assert(options.percentiles === undefined || (Array.isArray(options.percentiles) && options.percentiles.length > 0 &&
+            options.percentiles.every(p => typeof p === 'number' && p >= 1 && p <= 100)), 'getQueueStats: percentiles must be a non-empty array of percents from 1 to 100, such as [50, 95]');
+        // Each value once, in the order first asked for.
+        const percentiles = options.percentiles && [...new Set(options.percentiles)];
         const isCockroach = this.config.backend === 'cockroachdb';
         // `counted` is true for recorded snapshots. The cache path serves only gauges: the queue table's
         // counters describe the last pass that counted, not this reading.
@@ -2150,6 +2354,9 @@ class Manager extends EventEmitter {
                 createdDelta: null,
                 deltaSeconds: null,
                 deltaOn: null,
+                waitBins: null,
+                runBins: null,
+                readyOldestSeconds: null,
                 capturedOn: row?.capturedOn ?? new Date(this.config.clock.now())
             };
             for (const field of counted ? [...STATS_COUNT_FIELDS, ...STATS_DELTA_FIELDS] : STATS_COUNT_FIELDS) {
@@ -2161,6 +2368,20 @@ class Manager extends EventEmitter {
             // The end of the interval the counters cover, handed on as the row holds it, like capturedOn.
             if (counted && row?.deltaOn != null)
                 snapshot.deltaOn = row.deltaOn;
+            if (counted) {
+                snapshot.waitBins = toBins(row?.waitBins);
+                snapshot.runBins = toBins(row?.runBins);
+                if (row?.readyOldestSeconds != null)
+                    snapshot.readyOldestSeconds = Number(row.readyOldestSeconds);
+            }
+            // Read from this snapshot's (or bucket's) own histograms, so they are null wherever those are.
+            if (percentiles) {
+                snapshot.percentiles = percentiles.map(p => ({
+                    p,
+                    waitSeconds: percentile(snapshot.waitBins, p),
+                    runSeconds: percentile(snapshot.runBins, p)
+                }));
+            }
             return snapshot;
         };
         if (this.config.persistQueueStats) {
@@ -2293,7 +2514,7 @@ class Manager extends EventEmitter {
             return options.db;
         }
         if (this.db._pgbdb) {
-            assert(this.db.opened, 'Database connection is not opened');
+            assert(this.db.opened, 'Database not opened. Call start() before using pg-boss, or again after stop().');
         }
         return this.db;
     }