pg-boss 12.35.1 → 12.36.0

package/dist/manager.js

@@ -6,11 +6,13 @@ import * as Attorney from "./attorney.js";
 import { untracked } from "./activity.js";
 import { TRANSACTION_ROLLBACK_TIMEOUT_MS } from "./db.js";
 import * as plans from "./plans.js";
+import { percentile } from "./latency.js";
 import * as timekeeper from "./timekeeper.js";
 import { resolveWithinSeconds } from "./tools.js";
 import * as types from "./types.js";
 import Worker from "./worker.js";
 import { JobSpy } from "./spy.js";
+import Telemetry, {} from "./telemetry.js";
 const INTERNAL_QUEUES = Object.values(timekeeper.QUEUES).reduce((acc, i) => ({ ...acc, [i]: i }), {});
 // postgres: current transaction is aborted, commands ignored until end of transaction block
 const TRANSACTION_ABORTED = '25P02';
@@ -80,6 +82,7 @@ const NUMERIC_METADATA_FIELDS = [
 ];
 // Queue rows (plans.getQueues) return these integer columns as strings on CockroachDB too.
 const NUMERIC_QUEUE_FIELDS = [
+    'blockedCount',
     'retryLimit',
     'retryDelay',
     'retryDelayMax',
@@ -97,7 +100,8 @@ const NUMERIC_QUEUE_FIELDS = [
     'createdDelta',
     'completedDelta',
     'failedDelta',
-    'deltaSeconds'
+    'deltaSeconds',
+    'readyOldestSeconds'
 ];
 // The gauges shared by live stats and recorded snapshots (the QueueStats shape).
 const STATS_COUNT_FIELDS = [
@@ -108,6 +112,12 @@ const STATS_COUNT_FIELDS = [
     'failedCount',
     'totalCount'
 ];
+// A snapshot's histogram, LATENCY_SLOTS counts from a recorded pass or added up over a bucket, with
+// the slots stored as null (no job landed there) handed out as 0. Null when no pass counted it; all
+// zeros when one did and nothing finished. CockroachDB hands integers over as strings.
+function toBins(bins) {
+    return Array.isArray(bins) ? bins.map(n => (n == null ? 0 : Number(n))) : null;
+}
 // The throughput counters and the seconds they cover. Only recorded snapshots carry them; see
 // getQueueStats.
 const STATS_DELTA_FIELDS = [
@@ -187,6 +197,9 @@ class Manager extends EventEmitter {
     #localGroupActive;
     #localGroupConfig;
     #localGroupMaxLimit;
+    #telemetry;
+    // The trace context each fetched job was sent with, kept off the job object handed to the handler.
+    #traceContexts;
     constructor(db, config) {
         super();
         this.config = config;
@@ -205,6 +218,8 @@ class Manager extends EventEmitter {
         this.#localGroupActive = new Map();
         this.#localGroupConfig = new Map();
         this.#localGroupMaxLimit = new Map();
+        this.#telemetry = new Telemetry(config.openTelemetry, config.schema, () => this.queues);
+        this.#traceContexts = new WeakMap();
     }
     getSpy(name) {
         if (!this.config.__test__enableSpies) {
@@ -352,6 +367,7 @@ class Manager extends EventEmitter {
     // (each output carried per-id via a JSON recordset), so batch size never drives the statement
     // count. Any batch job the handler omits (or returns with an invalid shape) is failed with a
     // descriptive error so it retries / dead-letters per queue config.
+    // Resolves with the error the whole batch was failed with, or undefined.
     async #settlePerJob(name, jobs, result) {
         if (!Array.isArray(result)) {
             // The handler opted into perJobResults but did not return an array: a contract violation.
@@ -359,7 +375,7 @@ class Manager extends EventEmitter {
             const err = new Error('perJobResults handler must resolve with an array of job results');
             await this.fail(name, jobs, err);
             await this.#trackJobsFailed(name, jobs, err);
-            return;
+            return err;
         }
         // Index the handler's dispositions by job id, keeping only valid entries that reference a job
         // from this batch. Last write wins on duplicate ids.
@@ -391,9 +407,16 @@ class Manager extends EventEmitter {
             }
         }
         const items = (entries) => entries.map(({ job, output }) => ({ id: job.id, retryCount: job.retryCount, output }));
-        const completedIds = completed.length > 0 ? await this.#completeWithOutputs(name, items(completed)) : null;
-        const failedIds = failed.length > 0 ? await this.#failWithOutputs(name, items(failed)) : null;
-        const deadLetteredIds = deadLettered.length > 0 ? await this.#failWithOutputs(name, items(deadLettered), true) : null;
+        const ids = (entries) => entries.map(({ job }) => job.id);
+        const completedIds = completed.length > 0
+            ? await this.#telemetry.settle('complete', name, ids(completed), () => this.#completeWithOutputs(name, items(completed)))
+            : null;
+        const failedIds = failed.length > 0
+            ? await this.#telemetry.settle('fail', name, ids(failed), () => this.#failWithOutputs(name, items(failed)))
+            : null;
+        const deadLetteredIds = deadLettered.length > 0
+            ? await this.#telemetry.settle('fail', name, ids(deadLettered), () => this.#failWithOutputs(name, items(deadLettered), true))
+            : null;
         // Only the jobs each statement actually settled: the attempt fence leaves a job whose claim
         // lapsed alone, and recording it would tell a spy it settled when another attempt holds it.
         const landed = (entries, result) => {
@@ -509,6 +532,8 @@ class Manager extends EventEmitter {
      * same as any other worker's. Because the claim is outside the transaction, the jobs stay
      * visibly `active` throughout, which is what keeps heartbeats, `expireInSeconds`, and another
      * instance's supervisor working on them as usual.
+     *
+     * Resolves with the error the batch was failed with, or undefined when it completed.
      */
     async #processJobs(name, jobs, callback, worker, heartbeatRefreshSeconds, perJobResults = false, transactional = false, transactionTimeoutSeconds) {
         const jobIds = jobs.map(job => job.id);
@@ -569,6 +594,9 @@ class Manager extends EventEmitter {
         let completedAffected = 0;
         let failedError;
         let didFail = false;
+        // A perJobResults batch #settlePerJob failed as a whole. Kept apart from failedError, since
+        // #settlePerJob has already failed and tracked those jobs itself.
+        let perJobError;
         // Only for a transactional worker, and only from the begin below until it settles. rollback()
         // is idempotent, so the catch can settle it without tracking whether the commit got there
         // first.
@@ -619,7 +647,7 @@ class Manager extends EventEmitter {
                 // #settlePerJob settles each job individually and does its own (synchronous,
                 // lookup-free) spy tracking via #trackJobsSettled, so the deferred tracker below
                 // is skipped for this path.
-                await this.#settlePerJob(name, jobs, result);
+                perJobError = await this.#settlePerJob(name, jobs, result);
             }
             else {
                 // Read out before the completion below, which goes through the same complete() and would
@@ -676,6 +704,7 @@ class Manager extends EventEmitter {
                 await this.#trackJobsCompleted(name, jobs, completedResult, completedAffected);
             }
         }
+        return didFail ? (failedError ?? new Error('handler rejected without a reason')) : perJobError;
     }
     /**
      * Gives the handler's transaction a deadline the database enforces.
@@ -822,6 +851,10 @@ class Manager extends EventEmitter {
     }
     async start() {
         this.stopped = false;
+        if (this.#telemetry.enabled) {
+            const { rows } = await this.db.executeSql(plans.currentDatabase());
+            this.#telemetry.setDatabase(rows[0].name);
+        }
         this.queueCacheInterval = this.config.clock.setInterval(() => this.onCacheQueues({ emit: true }), this.config.queueCacheIntervalSeconds * 1000);
         this.wipInterval = this.config.clock.setInterval(() => {
             const now = this.config.clock.now();
@@ -835,12 +868,14 @@ class Manager extends EventEmitter {
             }
         }, 2000);
         await this.onCacheQueues();
+        this.#telemetry.observeQueues();
     }
     async onCacheQueues({ emit = false } = {}) {
         try {
             assert(!this.config.__test__throw_queueCache, 'test error');
             const queues = await this.getQueues();
             this.queues = queues.reduce((acc, i) => { acc[i.name] = i; return acc; }, {});
+            this.#telemetry.refreshInstruments();
         }
         catch (error) {
             emit && this.emit(events.error, { ...error, message: error.message, stack: error.stack });
@@ -863,10 +898,24 @@ class Manager extends EventEmitter {
         if (this.queues)
             delete this.queues[name];
     }
+    // Replaces a queue's cache entry with its row as it stands, rather than evicting it, so a queue
+    // created again or updated stays in the cache and the queue gauge while still picking up what
+    // changed: new options, or a table that changed under it (deleted and recreated elsewhere with
+    // another partition setting).
+    async #reloadQueueCache(name) {
+        if (!this.queues)
+            return;
+        const queue = await this.getQueue(name);
+        if (queue)
+            this.queues[name] = queue;
+        else
+            this.#evictQueueCache(name);
+    }
     async stop() {
         this.stopped = true;
         this.config.clock.clearInterval(this.queueCacheInterval);
         this.config.clock.clearInterval(this.wipInterval);
+        this.#telemetry.unobserveQueues();
         // offWork stops every worker on a queue, so iterate queue names rather than workers - otherwise
         // a localConcurrency of N re-stops all N workers N times and registers N pending cleanups.
         const names = new Set([...this.workers.values()]
@@ -952,7 +1001,12 @@ class Manager extends EventEmitter {
                 const ignoreGroups = localGroupConcurrency != null
                     ? this.#getGroupsAtLocalCapacity(name)
                     : undefined;
-                return this.fetch(name, { batchSize, includeMetadata, priority, orderByCreatedOn, groupConcurrency, ignoreGroups, minPriority, maxPriority });
+                return this.#fetch(name, { batchSize, includeMetadata, priority, orderByCreatedOn, groupConcurrency, ignoreGroups, minPriority, maxPriority });
+            };
+            // Counted here rather than on fetch: jobs past localGroupConcurrency are restored, not delivered.
+            const processBatch = (batch, worker) => {
+                this.#telemetry.consumed(name, batch.length);
+                return this.#telemetry.process(name, batch, job => this.#traceContexts.get(job), () => this.#processJobs(name, batch, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds));
             };
             const onFetch = async (jobs) => {
                 if (!jobs.length)
@@ -965,7 +1019,7 @@ class Manager extends EventEmitter {
                 const worker = this.workers.get(workerId);
                 // Skip all in-memory group tracking when localGroupConcurrency is not enabled
                 if (localGroupConcurrency == null) {
-                    await this.#processJobs(name, jobs, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds);
+                    await processBatch(jobs, worker);
                 }
                 else {
                     const { allowed, excess, groupedJobs } = this.#trackLocalGroupStart(name, jobs);
@@ -980,7 +1034,7 @@ class Manager extends EventEmitter {
                                 worker.jobs = allowed;
                         }
                         if (allowed.length > 0) {
-                            await this.#processJobs(name, allowed, callback, worker, heartbeatRefreshSeconds, perJobResults, transactional, transactionTimeoutSeconds);
+                            await processBatch(allowed, worker);
                         }
                     }
                     finally {
@@ -1164,6 +1218,10 @@ class Manager extends EventEmitter {
     }
     async publish(event, data, options) {
         assert(event, 'Missing required argument');
+        // Counts no jobs of its own: each send() below records the job it creates.
+        await this.#telemetry.send('publish', event, 0, () => this.#publish(event, data, options));
+    }
+    async #publish(event, data, options) {
         const sql = plans.getQueuesForEvent(this.config.schema);
         const { rows } = await this.db.executeSql(sql, [event]);
         const results = await Promise.allSettled(rows.map(({ name }) => this.send(name, data, options)));
@@ -1230,9 +1288,12 @@ class Manager extends EventEmitter {
         };
     }
     async createJob(request) {
+        return this.#telemetry.send('send', request.name, 1, carrier => this.#createJob(request, carrier), id => id ? [id] : null);
+    }
+    async #createJob(request, traceContext) {
         const { name, data = null, options = {} } = request;
         const { db: wrapper, singletonSeconds, singletonNextSlot } = options;
-        const job = this.#toJobPayload(name, data, options);
+        const job = { ...this.#toJobPayload(name, data, options), __traceContext: traceContext };
         const db = wrapper || this.db;
         const { table, policy, notify } = await this.getQueueCache(name);
         if (policy === plans.QUEUE_POLICIES.key_strict_fifo && !job.singletonKey) {
@@ -1309,9 +1370,14 @@ class Manager extends EventEmitter {
     }
     async upsert(...args) {
         const request = Attorney.checkUpdateArgs(args, { upsert: true });
+        Attorney.assertQueueName(request.name);
+        // Only an insert counts as a send, and only an inserted job stores the trace context: an
+        // updated job keeps the trace of the send that created it.
+        return this.#telemetry.send('upsert', request.name, result => result.inserted, carrier => this.#upsert(request, carrier), result => result.jobs);
+    }
+    async #upsert(request, traceContext) {
         const { name, data } = request;
         const opts = (request.options ?? {});
-        Attorney.assertQueueName(name);
         const db = this.assertDb(opts);
         const { table, policy, notify } = await this.getQueueCache(name);
         const by = opts.id ? 'id' : 'singletonKey';
@@ -1326,7 +1392,7 @@ class Manager extends EventEmitter {
         const insertSql = plans.insertJobs(this.config.schema, { table, name, returnId: true, notify: notifyEnabled });
         const job = this.#toUpdatePayload(data, opts);
         const updatePayload = JSON.stringify(job);
-        const insertPayload = JSON.stringify([job]);
+        const insertPayload = JSON.stringify([{ ...job, __traceContext: traceContext }]);
         const result = await this.ensureTransaction(db, async (tx) => {
             const { rows: updated } = await tx.executeSql(updateSql, [updatePayload]);
             if (updated.length) {
@@ -1363,6 +1429,9 @@ class Manager extends EventEmitter {
     // statement nor kept on the objects below, so this stays the raw path it has always been.
     options = {}) {
         assert(Array.isArray(jobs), 'jobs argument should be an array');
+        return this.#telemetry.send('insert', name, jobs.length, carrier => this.#insert(name, jobs, options, carrier));
+    }
+    async #insert(name, jobs, options, traceContext) {
         const slots = options.__singletonSlots === true;
         const seenIds = new Set();
         for (const job of jobs) {
@@ -1394,6 +1463,8 @@ class Manager extends EventEmitter {
         const dataById = spy ? new Map() : undefined;
         const insertPayload = jobs.map(j => {
             const { blocked, blocking, pendingDependencies, group, __singletonSlot, ...rest } = j;
+            // Overwrites any __traceContext the caller passed.
+            Object.assign(rest, { __traceContext: traceContext });
             // Reattached only for the caller that asked for the column, so a public insert() drops the
             // field rather than handing an unvalidated value to a timestamp cast.
             if (slots && __singletonSlot !== undefined) {
@@ -1441,6 +1512,11 @@ class Manager extends EventEmitter {
     }
     async flow(jobs, options = {}) {
         Attorney.validateFlowJobs(jobs);
+        const queues = new Set(jobs.map(job => job.name));
+        const destination = queues.size === 1 ? jobs[0].name : null;
+        return this.#telemetry.send('flow', destination, jobs.length, carrier => this.#flow(jobs, options, carrier), refToId => Object.values(refToId));
+    }
+    async #flow(jobs, options, traceContext) {
         // validate and normalize each job's options the same way send()/insert() do
         const flowJobs = jobs.map(job => ({
             ...job,
@@ -1503,7 +1579,8 @@ class Manager extends EventEmitter {
                     deadLetter: j.options?.deadLetter ?? undefined,
                     blocked: dependencyCount > 0 || undefined,
                     blocking: parentRefs.has(j.ref) || undefined,
-                    pendingDependencies: dependencyCount || undefined
+                    pendingDependencies: dependencyCount || undefined,
+                    __traceContext: traceContext
                 };
             });
             statements.push(plans.insertFlowJobs(this.config.schema, { table, name: queueName }, insertPayload));
@@ -1559,6 +1636,9 @@ class Manager extends EventEmitter {
     }
     async fetch(name, options = {}) {
         Attorney.checkFetchArgs(name, options);
+        return this.#telemetry.receive(name, () => this.#fetch(name, options), job => this.#traceContexts.get(job));
+    }
+    async #fetch(name, options) {
         this.#warnDeprecatedFetchOptions(options);
         const db = this.assertDb(options);
         const { table, policy, singletonsActive } = await this.getQueueCache(name);
@@ -1569,7 +1649,8 @@ class Manager extends EventEmitter {
             name,
             policy,
             limit: options.batchSize || 1,
-            ignoreSingletons: singletonsActive
+            ignoreSingletons: singletonsActive,
+            includeTraceContext: this.#telemetry.enabled
         };
         const query = plans.fetchNextJob(fetchOptions, this.config.noSkipLocked);
         let result;
@@ -1586,6 +1667,13 @@ class Manager extends EventEmitter {
                 throw err;
         }
         const rows = result?.rows || [];
+        for (const row of rows) {
+            // A db adapter may hand jsonb back unparsed.
+            const carrier = typeof row.__traceContext === 'string' ? JSON.parse(row.__traceContext) : row.__traceContext;
+            delete row.__traceContext;
+            if (carrier)
+                this.#traceContexts.set(row, carrier);
+        }
         // Even a minimal fetch (JOB_COLUMNS_MIN) returns numeric fields like expireInSeconds and
         // heartbeatSeconds, so normalize regardless of includeMetadata.
         return this.#numericJobFields(rows);
@@ -1638,21 +1726,23 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'complete');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const outputData = this.mapCompletionDataArg(data);
-        let response;
-        // noMultiMutationCte: split the dependency-unblocking into a separate statement to
-        // avoid CockroachDB's multi-mutation CTE limitation (completeJobs updates two tables).
-        if (this.config.noMultiMutationCte) {
-            response = await this.completeDistributed(name, ids, outputData, table, db, options.includeQueued, attempts);
-        }
-        else {
-            const sql = plans.completeJobs(this.config.schema, table, options.includeQueued, !!attempts);
-            const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
-            response = this.mapCommandResponse(ids, result);
-        }
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('complete', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const outputData = this.mapCompletionDataArg(data);
+            let response;
+            // noMultiMutationCte: split the dependency-unblocking into a separate statement to
+            // avoid CockroachDB's multi-mutation CTE limitation (completeJobs updates two tables).
+            if (this.config.noMultiMutationCte) {
+                response = await this.completeDistributed(name, ids, outputData, table, db, options.includeQueued, attempts);
+            }
+            else {
+                const sql = plans.completeJobs(this.config.schema, table, options.includeQueued, !!attempts);
+                const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
+                response = this.mapCommandResponse(ids, result);
+            }
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     // Distributed complete/fail need several statements run atomically. When we own the pooled
     // connection we pin a single client via withTransaction(); when the caller supplied their own
@@ -1677,22 +1767,24 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'fail');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const outputData = this.mapCompletionDataArg(data);
-        let response;
-        // noMultiMutationCte: use separate queries to avoid CockroachDB's multi-mutation CTE limitation.
-        // The delete and re-insert run in a single transaction (see ensureTransaction) so the
-        // job cannot be lost between the two statements.
-        if (this.config.noMultiMutationCte) {
-            response = await this.failDistributed(name, ids, outputData, table, db, attempts);
-        }
-        else {
-            const sql = plans.failJobsById(this.config.schema, table, !!attempts);
-            const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
-            response = this.mapCommandResponse(ids, result);
-        }
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('fail', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const outputData = this.mapCompletionDataArg(data);
+            let response;
+            // noMultiMutationCte: use separate queries to avoid CockroachDB's multi-mutation CTE limitation.
+            // The delete and re-insert run in a single transaction (see ensureTransaction) so the
+            // job cannot be lost between the two statements.
+            if (this.config.noMultiMutationCte) {
+                response = await this.failDistributed(name, ids, outputData, table, db, attempts);
+            }
+            else {
+                const sql = plans.failJobsById(this.config.schema, table, !!attempts);
+                const result = await db.executeSql(sql, attempts ? [name, ids, outputData, plans.attemptPairs(ids, attempts)] : [name, ids, outputData]);
+                response = this.mapCommandResponse(ids, result);
+            }
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     async failDistributed(name, ids, outputData, table, db, attempts) {
         // CockroachDB doesn't support multi-mutation CTEs, but does support transactions, so the
@@ -1800,6 +1892,7 @@ class Manager extends EventEmitter {
             const data = toJsonText(job.data);
             const output = toJsonText(jobOutput);
             const sourceOutput = toJsonText(job.source_output);
+            const traceContext = toJsonText(job.trace_context);
             // forceTerminal (perJobResults `deadletter`) skips retries so the job fails terminally and
             // routes straight to the dead letter queue below.
             const canRetry = !forceTerminal && retryCount < retryLimit;
@@ -1828,7 +1921,7 @@ class Manager extends EventEmitter {
                     job.deletion_seconds, createdOn, null, keepUntil, job.policy,
                     output, job.dead_letter,
                     null, job.heartbeat_seconds, job.blocked, job.blocking, job.pending_dependencies,
-                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id
+                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id, traceContext
                 ]);
                 // The retry insert can be dropped by ON CONFLICT when the queue policy (e.g. stately,
                 // singleton, key_strict_fifo) already has a non-terminal job. Mirror the failed_jobs
@@ -1843,11 +1936,11 @@ class Manager extends EventEmitter {
                     job.deletion_seconds, createdOn, new Date(this.config.clock.now()), keepUntil, job.policy,
                     output, job.dead_letter,
                     null, job.heartbeat_seconds, job.blocked, job.blocking, job.pending_dependencies,
-                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id
+                    job.source_name, job.source_id, sourceCreatedOn, job.source_retry_count, sourceOutput, job.source_root_id, traceContext
                 ]);
                 // Insert to dead letter queue if failed and has dead_letter configured
                 if (job.dead_letter) {
-                    await tx.executeSql(dlqSql, [job.dead_letter, data, output, job.name, job.id, createdOn, job.retry_count, job.singleton_key, job.priority, job.group_id, job.group_tier, job.source_root_id]);
+                    await tx.executeSql(dlqSql, [job.dead_letter, data, output, job.name, job.id, createdOn, job.retry_count, job.singleton_key, job.priority, job.group_id, job.group_tier, job.source_root_id, traceContext]);
                 }
             }
             count++;
@@ -1860,12 +1953,14 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'deleteJob');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const sql = plans.deleteJobsById(this.config.schema, table, !!attempts);
-        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
-        const response = this.mapCommandResponse(ids, result);
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('delete', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const sql = plans.deleteJobsById(this.config.schema, table, !!attempts);
+            const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
+            const response = this.mapCommandResponse(ids, result);
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     // The filter half of redrive and previewRedrive, validated once and in the parameter order
     // plans.redriveWhere expects ($2 through $6).
@@ -1956,12 +2051,14 @@ class Manager extends EventEmitter {
         const db = this.assertDb(options);
         const { ids, attempts: fetched } = this.mapAttemptArg(id, 'cancel');
         const attempts = fetched ?? this.#handlerAttempts(options, ids);
-        const { table } = await this.getQueueCache(name);
-        const sql = plans.cancelJobs(this.config.schema, table, !!attempts);
-        const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
-        const response = this.mapCommandResponse(ids, result);
-        this.#trackHandlerSettle(options, response);
-        return response;
+        return this.#telemetry.settle('cancel', name, ids, async () => {
+            const { table } = await this.getQueueCache(name);
+            const sql = plans.cancelJobs(this.config.schema, table, !!attempts);
+            const result = await db.executeSql(sql, attempts ? [name, ids, plans.attemptPairs(ids, attempts)] : [name, ids]);
+            const response = this.mapCommandResponse(ids, result);
+            this.#trackHandlerSettle(options, response);
+            return response;
+        });
     }
     async resume(name, id, options = {}) {
         Attorney.assertQueueName(name);
@@ -2013,7 +2110,7 @@ class Manager extends EventEmitter {
         }
         const sql = plans.createQueue(this.config.schema, name, { ...options, policy }, this.config.noAdvisoryLocks);
         await this.db.executeSql(sql);
-        this.#evictQueueCache(name);
+        await this.#reloadQueueCache(name);
     }
     async getBlockedKeys(name) {
         Attorney.assertQueueName(name);
@@ -2043,6 +2140,11 @@ class Manager extends EventEmitter {
                 }
             }
         }
+        // Every backend: the histograms' empty slots are stored as null and handed out as 0.
+        for (const row of rows) {
+            row.waitBins = toBins(row.waitBins);
+            row.runBins = toBins(row.runBins);
+        }
         return rows;
     }
     async updateQueue(name, options = {}) {
@@ -2064,7 +2166,7 @@ class Manager extends EventEmitter {
         }
         const sql = plans.updateQueue(this.config.schema);
         await this.db.executeSql(sql, [name, options]);
-        this.#evictQueueCache(name);
+        await this.#reloadQueueCache(name);
     }
     async getQueue(name) {
         const rows = await this.getQueues([name]);
@@ -2130,6 +2232,10 @@ class Manager extends EventEmitter {
     // getQueue(name).
     async getQueueStats(name, options = {}) {
         Attorney.assertQueueName(name);
+        assert(options.percentiles === undefined || (Array.isArray(options.percentiles) && options.percentiles.length > 0 &&
+            options.percentiles.every(p => typeof p === 'number' && p >= 1 && p <= 100)), 'getQueueStats: percentiles must be a non-empty array of percents from 1 to 100, such as [50, 95]');
+        // Each value once, in the order first asked for.
+        const percentiles = options.percentiles && [...new Set(options.percentiles)];
         const isCockroach = this.config.backend === 'cockroachdb';
         // `counted` is true for recorded snapshots. The cache path serves only gauges: the queue table's
         // counters describe the last pass that counted, not this reading.
@@ -2150,6 +2256,9 @@ class Manager extends EventEmitter {
                 createdDelta: null,
                 deltaSeconds: null,
                 deltaOn: null,
+                waitBins: null,
+                runBins: null,
+                readyOldestSeconds: null,
                 capturedOn: row?.capturedOn ?? new Date(this.config.clock.now())
             };
             for (const field of counted ? [...STATS_COUNT_FIELDS, ...STATS_DELTA_FIELDS] : STATS_COUNT_FIELDS) {
@@ -2161,6 +2270,20 @@ class Manager extends EventEmitter {
             // The end of the interval the counters cover, handed on as the row holds it, like capturedOn.
             if (counted && row?.deltaOn != null)
                 snapshot.deltaOn = row.deltaOn;
+            if (counted) {
+                snapshot.waitBins = toBins(row?.waitBins);
+                snapshot.runBins = toBins(row?.runBins);
+                if (row?.readyOldestSeconds != null)
+                    snapshot.readyOldestSeconds = Number(row.readyOldestSeconds);
+            }
+            // Read from this snapshot's (or bucket's) own histograms, so they are null wherever those are.
+            if (percentiles) {
+                snapshot.percentiles = percentiles.map(p => ({
+                    p,
+                    waitSeconds: percentile(snapshot.waitBins, p),
+                    runSeconds: percentile(snapshot.runBins, p)
+                }));
+            }
             return snapshot;
         };
         if (this.config.persistQueueStats) {

package/dist/migrationStore.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"migrationStore.d.ts","sourceRoot":"","sources":["../src/migrationStore.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAKnC,UAAU,cAAc;IACtB,WAAW,CAAC,EAAE,OAAO,CAAA;IAKrB,eAAe,CAAC,EAAE,KAAK,CAAC,kBAAkB,EAAE,CAAA;CAC7C;AAoGD,UAAU,iBAAiB;IACzB,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,EAAE,CAAA;CACrB;AASD,iBAAS,QAAQ,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,UAsB5G;AAED,iBAAS,IAAI,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,UAQxG;AAKD,iBAAS,eAAe,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,cAAmB,GAAG,iBAAiB,CAiDrK;AAKD,iBAAS,OAAO,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,cAAmB,UAIzI;AAgoCD,iBAAS,aAAa,CAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAE9C;AAID,iBAAS,eAAe,CAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,0BAA0B,EAAE,QAAQ,GAAG,qBAAqB,GAAG,mBAAmB,GAAG,qBAAqB,CAAC,GAAG,KAAK,CAAC,SAAS,EAAE,CAE3K;AAOD,iBAAS,MAAM,CAAE,MAAM,EAAE,MAAM,EAAE,cAAc,UAAQ,EAAE,UAAU,UAAQ,EAAE,mBAAmB,UAAQ,GAAG,KAAK,CAAC,SAAS,EAAE,CA4oB3H;AAED,OAAO,EACL,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,eAAe,EACf,MAAM,EACN,eAAe,EACf,aAAa,GACd,CAAA"}
+{"version":3,"file":"migrationStore.d.ts","sourceRoot":"","sources":["../src/migrationStore.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAKnC,UAAU,cAAc;IACtB,WAAW,CAAC,EAAE,OAAO,CAAA;IAKrB,eAAe,CAAC,EAAE,KAAK,CAAC,kBAAkB,EAAE,CAAA;CAC7C;AAoGD,UAAU,iBAAiB;IACzB,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,EAAE,CAAA;CACrB;AASD,iBAAS,QAAQ,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,UAsB5G;AAED,iBAAS,IAAI,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,UAQxG;AAKD,iBAAS,eAAe,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,cAAmB,GAAG,iBAAiB,CAiDrK;AAKD,iBAAS,OAAO,CAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,cAAmB,UAIzI;AAgoCD,iBAAS,aAAa,CAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAE9C;AAID,iBAAS,eAAe,CAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,0BAA0B,EAAE,QAAQ,GAAG,qBAAqB,GAAG,mBAAmB,GAAG,qBAAqB,CAAC,GAAG,KAAK,CAAC,SAAS,EAAE,CAE3K;AAOD,iBAAS,MAAM,CAAE,MAAM,EAAE,MAAM,EAAE,cAAc,UAAQ,EAAE,UAAU,UAAQ,EAAE,mBAAmB,UAAQ,GAAG,KAAK,CAAC,SAAS,EAAE,CA+sB3H;AAED,OAAO,EACL,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,eAAe,EACf,MAAM,EACN,eAAe,EACf,aAAa,GACd,CAAA"}

package/dist/migrationStore.js

@@ -1963,6 +1963,73 @@ AS $function$
           DROP COLUMN source_output,
           DROP COLUMN source_root_id`
             ]
+        },
+        {
+            release: '12.36.0',
+            version: 44,
+            previous: 43,
+            // The instance table starts empty: each instance registers on its next start().
+            // blocked_count's constant default adds it without a table rewrite, and existing queues read 0
+            // until the next monitor pass counts them. It goes on the queue row only, not queue_stats, so
+            // the covering index on queue_stats needs no rebuild.
+            // The histogram and ready_oldest_seconds columns are nullable with no default, like the v43
+            // deltas, so no statement rewrites a table, and a snapshot captured before them reads as not
+            // counted rather than as a queue with no waits.
+            // trace_context is nullable with no default too, so adding it rewrites no rows.
+            install: [
+                /* eslint-disable no-restricted-syntax -- column defaults stay on the real clock: every pg-boss write names its timestamps through job_now() */
+                `CREATE TABLE ${schema}.instance (
+          id uuid PRIMARY KEY,
+          name text,
+          host text NOT NULL,
+          pid int NOT NULL,
+          version text NOT NULL,
+          node_version text NOT NULL,
+          application_name text,
+          heartbeat_seconds int NOT NULL,
+          supervise bool NOT NULL,
+          schedule bool NOT NULL,
+          migrate bool NOT NULL,
+          persist_queue_stats bool NOT NULL,
+          persist_warnings bool NOT NULL,
+          pool_max int,
+          pool_total int,
+          pool_idle int,
+          pool_waiting int,
+          workers jsonb NOT NULL DEFAULT '[]'::jsonb,
+          metrics jsonb,
+          config jsonb NOT NULL DEFAULT '{}'::jsonb,
+          crash_restarts int NOT NULL DEFAULT 0,
+          crash_restarts_since timestamptz,
+          started_on timestamptz NOT NULL DEFAULT now(),
+          heartbeat_on timestamptz NOT NULL DEFAULT now(),
+          stopped_on timestamptz
+        )`,
+                /* eslint-enable no-restricted-syntax */
+                `ALTER TABLE ${schema}.queue
+          ADD COLUMN blocked_count int NOT NULL DEFAULT 0,
+          ADD COLUMN wait_bins int[],
+          ADD COLUMN run_bins int[],
+          ADD COLUMN ready_oldest_seconds int`,
+                `ALTER TABLE ${schema}.queue_stats
+          ADD COLUMN wait_bins int[],
+          ADD COLUMN run_bins int[],
+          ADD COLUMN ready_oldest_seconds int`,
+                `ALTER TABLE ${schema}.job ADD COLUMN IF NOT EXISTS trace_context jsonb`
+            ],
+            uninstall: [
+                `DROP TABLE ${schema}.instance`,
+                `ALTER TABLE ${schema}.queue
+          DROP COLUMN blocked_count,
+          DROP COLUMN wait_bins,
+          DROP COLUMN run_bins,
+          DROP COLUMN ready_oldest_seconds`,
+                `ALTER TABLE ${schema}.queue_stats
+          DROP COLUMN wait_bins,
+          DROP COLUMN run_bins,
+          DROP COLUMN ready_oldest_seconds`,
+                `ALTER TABLE ${schema}.job DROP COLUMN trace_context`
+            ]
         }
     ];
 }