pg-boss 12.37.0 → 12.38.0

package/dist/activity.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"activity.d.ts","sourceRoot":"","sources":["../src/activity.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAqB,MAAM,YAAY,CAAA;AAI9D;;;GAGG;AACH,wBAAgB,SAAS,CAAC,CAAC,SAAS,SAAS,EAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAExD;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,CAAC,SAAS,SAAS,EAAG,EAAE,EAAE,CAAC,GAAG;IAAE,EAAE,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;CAAE,CAmFlG"}
+{"version":3,"file":"activity.d.ts","sourceRoot":"","sources":["../src/activity.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAqB,MAAM,YAAY,CAAA;AAI9D;;;GAGG;AACH,wBAAgB,SAAS,CAAC,CAAC,SAAS,SAAS,EAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAExD;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,CAAC,SAAS,SAAS,EAAG,EAAE,EAAE,CAAC,GAAG;IAAE,EAAE,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;CAAE,CAoFlG"}

package/dist/activity.js

@@ -51,7 +51,8 @@ export function trackActivity(db) {
     });
     const wrapDb = (inner) => {
         const overrides = {
-            executeSql: track((text, values) => inner.executeSql(text, values))
+            // Every argument, not only text and values: pg-boss's own Db takes BAM's stop signals as a third.
+            executeSql: track((...args) => inner.executeSql(...args))
         };
         // Only when present: callers detect the capability with typeof.
         if (typeof inner.beginTransaction === 'function') {

package/dist/attorney.d.ts

@@ -20,7 +20,10 @@ declare function checkWorkArgs(name: string, args: any[]): {
 declare function checkFetchArgs(name: string, options: any): void;
 declare function getConfig(value: string | types.ConstructorOptions): types.ResolvedConstructorOptions;
 declare function assertPostgresObjectName(name: string): void;
+declare function assertListLimit(method: string, limit?: number): number;
+declare function assertListAfter(method: string, after: unknown, id?: boolean): void;
+declare function assertListAfterRow(method: string, after: unknown, fields: [string, string], id?: string): void;
 declare function assertQueueName(name: string): void;
 declare function assertKey(key: string): void;
-export { assertKey, COMPATIBILITY_FLAGS, assertPostgresObjectName, assertQueueName, checkFetchArgs, checkSendArgs, checkUpdateArgs, checkWorkArgs, getConfig, pinZonelessDateTime, POLICY, validateFlowJobs, validateGroupConfig, validateQueueArgs };
+export { assertKey, assertListAfter, assertListAfterRow, assertListLimit, COMPATIBILITY_FLAGS, assertPostgresObjectName, assertQueueName, checkFetchArgs, checkSendArgs, checkUpdateArgs, checkWorkArgs, getConfig, pinZonelessDateTime, POLICY, validateFlowJobs, validateGroupConfig, validateQueueArgs };
 //# sourceMappingURL=attorney.d.ts.map

package/dist/attorney.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"attorney.d.ts","sourceRoot":"","sources":["../src/attorney.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,KAAK,KAAK,MAAM,YAAY,CAAA;AAExC,QAAA,MAAM,MAAM;;;;CAIX,CAAA;AAKD,QAAA,MAAM,mBAAmB,6RAcf,CAAA;AA+EV,iBAAS,iBAAiB,CAAE,MAAM,GAAE,GAAQ,QAc3C;AAiBD,iBAAS,mBAAmB,CAAE,KAAK,EAAE,MAAM,UAE1C;AAED,iBAAS,aAAa,CAAE,IAAI,EAAE,GAAG,GAAG,KAAK,CAAC,OAAO,CAgDhD;AAOD,iBAAS,eAAe,CAAE,IAAI,EAAE,GAAG,EAAE,EAAE,MAAc,EAAE;;CAAK,GAAG,KAAK,CAAC,OAAO,CA4D3E;AAED,iBAAS,mBAAmB,CAAE,MAAM,EAAE,GAAG,QAOxC;AAED,iBAAS,gBAAgB,CAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,QA2D/C;AA2GD,iBAAS,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG;IAClD,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAA;IAClC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;CACjC,CAgCA;AAqBD,iBAAS,cAAc,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,QAQlD;AAED,iBAAS,SAAS,CAAE,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,kBAAkB,GAAG,KAAK,CAAC,0BAA0B,CA4B9F;AA4JD,iBAAS,wBAAwB,CAAE,IAAI,EAAE,MAAM,QAiC9C;AAED,iBAAS,eAAe,CAAE,IAAI,EAAE,MAAM,QAIrC;AAED,iBAAS,SAAS,CAAE,GAAG,EAAE,MAAM,QAI9B;AA+LD,OAAO,EACL,SAAS,EACT,mBAAmB,EACnB,wBAAwB,EACxB,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,EACf,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,MAAM,EACN,gBAAgB,EAChB,mBAAmB,EACnB,iBAAiB,EAClB,CAAA"}
+{"version":3,"file":"attorney.d.ts","sourceRoot":"","sources":["../src/attorney.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,KAAK,KAAK,MAAM,YAAY,CAAA;AAExC,QAAA,MAAM,MAAM;;;;CAIX,CAAA;AAKD,QAAA,MAAM,mBAAmB,6RAcf,CAAA;AA+EV,iBAAS,iBAAiB,CAAE,MAAM,GAAE,GAAQ,QAc3C;AAiBD,iBAAS,mBAAmB,CAAE,KAAK,EAAE,MAAM,UAE1C;AAED,iBAAS,aAAa,CAAE,IAAI,EAAE,GAAG,GAAG,KAAK,CAAC,OAAO,CAgDhD;AAOD,iBAAS,eAAe,CAAE,IAAI,EAAE,GAAG,EAAE,EAAE,MAAc,EAAE;;CAAK,GAAG,KAAK,CAAC,OAAO,CA4D3E;AAED,iBAAS,mBAAmB,CAAE,MAAM,EAAE,GAAG,QAOxC;AAED,iBAAS,gBAAgB,CAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,QA2D/C;AA2GD,iBAAS,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG;IAClD,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAA;IAClC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;CACjC,CAgCA;AAqBD,iBAAS,cAAc,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,QAQlD;AAED,iBAAS,SAAS,CAAE,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,kBAAkB,GAAG,KAAK,CAAC,0BAA0B,CA4B9F;AA4JD,iBAAS,wBAAwB,CAAE,IAAI,EAAE,MAAM,QAiC9C;AAOD,iBAAS,eAAe,CAAE,MAAM,EAAE,MAAM,EAAE,KAAK,GAAE,MAA2B,GAAG,MAAM,CAIpF;AAYD,iBAAS,eAAe,CAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,UAAQ,GAAG,IAAI,CAI1E;AAID,iBAAS,kBAAkB,CAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAKxG;AAED,iBAAS,eAAe,CAAE,IAAI,EAAE,MAAM,QAIrC;AAED,iBAAS,SAAS,CAAE,GAAG,EAAE,MAAM,QAI9B;AA+LD,OAAO,EACL,SAAS,EACT,eAAe,EACf,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,wBAAwB,EACxB,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,EACf,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,MAAM,EACN,gBAAgB,EAChB,mBAAmB,EACnB,iBAAiB,EAClB,CAAA"}

package/dist/attorney.js

@@ -572,6 +572,33 @@ function assertPostgresObjectName(name) {
     assert(!/\W/.test(name), `Schema name ${JSON.stringify(name)} can only contain alphanumeric characters or underscores when unquoted.${remedy}`);
     assert(!/^\d/.test(name), `Schema name ${JSON.stringify(name)} cannot start with a number when unquoted.${remedy}`);
 }
+const LIST_LIMIT_DEFAULT = 1000;
+const LIST_LIMIT_MAX = 100_000;
+// The limit of a public list read: an integer from 1 to LIST_LIMIT_MAX, LIST_LIMIT_DEFAULT when
+// left out. Internal callers that need every row call below the public method instead.
+function assertListLimit(method, limit = LIST_LIMIT_DEFAULT) {
+    assert(Number.isInteger(limit) && limit >= 1 && limit <= LIST_LIMIT_MAX, `${method}: limit must be an integer between 1 and ${LIST_LIMIT_MAX}`);
+    return limit;
+}
+// Any spelling the uuid type accepts, in either case: 32 hex digits, a hyphen allowed after any group
+// of four but the last, and the whole optionally inside one pair of braces. A read that pages by id
+// checks it here, so a malformed one is a usage error rather than the database's.
+const UUID_INPUT = /^(?:\{[0-9a-fA-F]{4}(?:-?[0-9a-fA-F]{4}){7}\}|[0-9a-fA-F]{4}(?:-?[0-9a-fA-F]{4}){7})$/;
+function isUuid(value) {
+    return UUID_INPUT.test(value);
+}
+// The `after` of a list read that pages by one value: a queue name, a key or, with `id`, an id.
+function assertListAfter(method, after, id = false) {
+    assert(after === undefined || (typeof after === 'string' && after.length > 0), `${method}: after must be a non-empty string`);
+    assert(!id || after === undefined || isUuid(after), `${method}: after must be a uuid`);
+}
+// The `after` of a list read that pages by a row: the previous page's last row, of which only the
+// two named fields are read. With `id`, that field must be a uuid.
+function assertListAfterRow(method, after, fields, id) {
+    assert(after === undefined || (typeof after === 'object' && after !== null &&
+        fields.every(field => typeof after[field] === 'string')), `${method}: after must be an object with string ${fields.join(' and ')}`);
+    assert(!id || after === undefined || isUuid(after[id]), `${method}: after.${id} must be a uuid`);
+}
 function assertQueueName(name) {
     assert(name, 'Name is required');
     assert(typeof name === 'string', 'Name must be a string');
@@ -694,4 +721,4 @@ function applyFlowConfig(config) {
     assert(!('flowIntervalSeconds' in config) || config.flowIntervalSeconds >= minInterval, `configuration assert: flowIntervalSeconds must be at least ${minInterval} seconds`);
     config.flowIntervalSeconds = config.flowIntervalSeconds || 5;
 }
-export { assertKey, COMPATIBILITY_FLAGS, assertPostgresObjectName, assertQueueName, checkFetchArgs, checkSendArgs, checkUpdateArgs, checkWorkArgs, getConfig, pinZonelessDateTime, POLICY, validateFlowJobs, validateGroupConfig, validateQueueArgs };
+export { assertKey, assertListAfter, assertListAfterRow, assertListLimit, COMPATIBILITY_FLAGS, assertPostgresObjectName, assertQueueName, checkFetchArgs, checkSendArgs, checkUpdateArgs, checkWorkArgs, getConfig, pinZonelessDateTime, POLICY, validateFlowJobs, validateGroupConfig, validateQueueArgs };

package/dist/bam.d.ts

@@ -9,7 +9,7 @@ declare class Bam extends EventEmitter implements types.EventsMixin {
     constructor(db: types.IDatabase, config: types.ResolvedConstructorOptions);
     get working(): boolean;
     start(): Promise<void>;
-    stop(): Promise<void>;
+    stop(deadline?: Promise<void>): Promise<void>;
 }
 export default Bam;
 //# sourceMappingURL=bam.d.ts.map

package/dist/bam.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bam.d.ts","sourceRoot":"","sources":["../src/bam.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,aAAa,CAAA;AAItC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAgBnC,cAAM,GAAI,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;;IAOzD,MAAM;;;MAAS;gBAGb,EAAE,EAAE,KAAK,CAAC,SAAS,EACnB,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAU1C,IAAI,OAAO,IAAK,OAAO,CAEtB;IAEK,KAAK;IAcL,IAAI;CAwLX;AAED,eAAe,GAAG,CAAA"}
+{"version":3,"file":"bam.d.ts","sourceRoot":"","sources":["../src/bam.ts"],"names":[],"mappings":"AACA,OAAO,YAAY,MAAM,aAAa,CAAA;AAKtC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAmDnC,cAAM,GAAI,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;;IAQzD,MAAM;;;MAAS;gBAGb,EAAE,EAAE,KAAK,CAAC,SAAS,EACnB,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAU1C,IAAI,OAAO,IAAK,OAAO,CAEtB;IAEK,KAAK;IAiBL,IAAI,CAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC;CA+UrC;AAED,eAAe,GAAG,CAAA"}

package/dist/bam.js

@@ -1,16 +1,23 @@
+import assert from 'node:assert';
 import EventEmitter from 'node:events';
 import { ClaimTimer } from "./claimTimer.js";
+import Db from "./db.js";
 import * as plans from "./plans.js";
-import { delay } from "./tools.js";
+import { delay, unwrapSQLResult } from "./tools.js";
 import * as types from "./types.js";
 const events = {
     error: 'error',
     bam: 'bam'
 };
+// How often stop() repeats a cancel that found nothing to cancel. It can land between two statements
+// of one row, or before the statement reaches the server. It also finds nothing when the role cannot
+// see the statement in pg_stat_activity, and then stop() keeps trying until the statement ends.
+const CANCEL_RETRY_MS = 1000;
 class Bam extends EventEmitter {
     #stopped;
     #working;
     #pollTimer;
+    #running;
     #db;
     #config;
     events = events;
@@ -32,16 +39,95 @@ class Bam extends EventEmitter {
         setImmediate(() => this.#onPoll());
         this.#pollTimer.start();
     }
-    async stop() {
+    // Waits for a running command until deadline resolves, then ends it (see Ending) and waits for
+    // that to land, so the row is recorded before index.ts closes the pool. Without a deadline it waits
+    // for the command to finish.
+    async stop(deadline) {
         if (this.#stopped)
             return;
         this.#stopped = true;
         this.#pollTimer.stop();
         this.#pollTimer = undefined;
+        let expired = false;
+        deadline?.then(() => { expired = true; });
+        let endedAt = 0;
         while (this.#working) {
+            const running = this.#running;
+            if (expired && running) {
+                running.stopped = true;
+                if (!running.ended && running.ending !== 'wait' && Date.now() - endedAt >= CANCEL_RETRY_MS) {
+                    endedAt = Date.now();
+                    await this.#end(running);
+                }
+            }
             await delay(10);
         }
     }
+    async #end(running) {
+        if (running.ending === 'abandon' || running.ending === 'drop') {
+            running.abort.abort();
+            running.ended = true;
+            return;
+        }
+        // Before the cancel reads pg_stat_activity, so a statement that ends after the read closes its
+        // connection rather than returning it to the pool with the cancel still on its way.
+        running.abort.abort();
+        try {
+            const { rows } = await this.#db.executeSql(plans.cancelBamCommand(running.id));
+            // Only ever set here: a statement that stop() kept from starting may have set it while the
+            // cancel was in flight, and the cancel then finds nothing.
+            if (rows.some(row => row.cancelled))
+                running.ended = true;
+        }
+        catch (err) {
+            // Usually a role that can see the statement but not signal its backend, which a retry will not
+            // change. Report it once and wait for the statement.
+            running.ending = 'wait';
+            this.emit(events.error, err);
+        }
+    }
+    // Runs one of a claimed row's statements so stop() can end it. See Ending.
+    async #execute(running, sql) {
+        if (running.stopped) {
+            this.#notStarted(running);
+        }
+        const cockroach = this.#config.backend === 'cockroachdb';
+        if (!(this.#db instanceof Db)) {
+            running.ending = cockroach ? 'wait' : 'cancel';
+        }
+        else if (cockroach) {
+            running.ending = 'drop';
+        }
+        else {
+            running.ending = !this.#config.noIndexProgressView && plans.isConcurrentIndexCommand(sql) ? 'abandon' : 'cancel';
+        }
+        // CockroachDB drops comments from the statement text it reports, so the tag only serves Postgres.
+        const text = cockroach ? sql : `${plans.bamCommandTag(running.id)}\n${sql}`;
+        try {
+            const result = !(this.#db instanceof Db)
+                ? await this.#db.executeSql(text)
+                : await this.#db.executeSql(text, undefined, running.ending === 'cancel'
+                    ? { retire: running.abort.signal }
+                    : { signal: running.abort.signal });
+            // The statement is over. Until the next one starts, which checks stopped first, stop() has
+            // nothing to end, and a failure of the row's other statements is not the stop's doing.
+            running.ending = 'wait';
+            return result;
+        }
+        catch (err) {
+            // Db throws the abort reason when stop() aborted before the statement reached a connection.
+            if (running.abort.signal.aborted && err === running.abort.signal.reason) {
+                this.#notStarted(running);
+            }
+            throw err;
+        }
+    }
+    #notStarted(running) {
+        // Nothing is left running for the row to wait on, whatever the previous statement was.
+        running.ending = 'wait';
+        running.ended = true;
+        throw new Error('stop() ended the command before it ran');
+    }
     async #onPoll() {
         if (this.#stopped || this.#working || !this.#config.migrate)
             return;
@@ -69,12 +155,20 @@ class Bam extends EventEmitter {
             this.#working = false;
         }
     }
+    // Runs commands back to back once a poll finds one, rather than one per bamIntervalSeconds, so a
+    // migration's builds land one after another. The first claim may retry a failed or stale row; the
+    // rest take pending rows only, and the burst ends when none is left or the runner stops.
     async #processCommands() {
-        if (this.#stopped)
-            return;
-        const entry = await this.#getNextCommand();
+        let pendingOnly = false;
+        while (!this.#stopped && await this.#processNextCommand(pendingOnly)) {
+            pendingOnly = true;
+        }
+    }
+    // Claims and runs one command. Returns whether a command was claimed, whatever its outcome.
+    async #processNextCommand(pendingOnly) {
+        const entry = await this.#getNextCommand(pendingOnly);
         if (!entry)
-            return;
+            return false;
         if (this.#config.__test__delay_bam_claim_ms) {
             await delay(this.#config.__test__delay_bam_claim_ms);
         }
@@ -90,8 +184,10 @@ class Bam extends EventEmitter {
             catch (err) {
                 this.emit(events.error, err);
             }
-            return;
+            return false;
         }
+        const running = { id: entry.id, stopped: false, ended: false, ending: 'wait', abort: new AbortController() };
+        this.#running = running;
         this.emit(events.bam, {
             id: entry.id,
             name: entry.name,
@@ -99,14 +195,19 @@ class Bam extends EventEmitter {
             queue: entry.queue,
             table: entry.table
         });
+        // Set when a CockroachDB build returns without its index because an earlier build of it is still
+        // running as a schema change job. The row fails until that job ends, which is not an error.
+        let buildPending = false;
         try {
             let alreadyBuilt = false;
+            // Non-null for a CONCURRENTLY index build, which is checked against the catalog before a
+            // re-attempt and after the build returns.
+            const probeSql = plans.bamHealProbe(this.#config.schema, entry.command);
             // A re-attempted command (a stale in_progress reclaim, or a retry of a prior 'failed', including
             // failed rows left by older releases) needs the catalog consulted before the command is re-run,
             // because the row keeps the text it was enqueued with and that text may not be idempotent.
             // Probe indisvalid on every backend: pg_index is readable everywhere, and both outcomes matter.
             if (entry.reattempt) {
-                const probeSql = plans.bamHealProbe(this.#config.schema, entry.command);
                 if (probeSql) {
                     const { rows } = await this.#db.executeSql(probeSql);
                     if (rows[0]?.invalid) {
@@ -117,7 +218,7 @@ class Bam extends EventEmitter {
                         // DROP ... CONCURRENTLY isn't their model.
                         if (!this.#config.noIndexProgressView) {
                             // Non-null wherever the probe was: both recognise the same CONCURRENTLY commands.
-                            await this.#db.executeSql(plans.bamHealDrop(this.#config.schema, entry.command));
+                            await this.#execute(running, plans.bamHealDrop(this.#config.schema, entry.command));
                         }
                     }
                     else if (rows[0]) {
@@ -134,7 +235,25 @@ class Bam extends EventEmitter {
                 }
             }
             if (!alreadyBuilt) {
-                await this.#db.executeSql(entry.command);
+                await this.#execute(running, entry.command);
+                // A build can return without leaving a valid index. On CockroachDB, a retry of a build whose
+                // connection stop() dropped returns at once while the first build still runs as a schema
+                // change job, because IF NOT EXISTS matches the index it is adding, which pg_index does not
+                // list yet. On Postgres, IF NOT EXISTS skips over an INVALID index of the same name. Marking
+                // the row completed then would release the commands behind it, and nothing would build the
+                // index again if that job failed. Failing the row has a later attempt check again, where the
+                // probe completes it or heals the INVALID index. Without liveness there is no heal, so an
+                // INVALID index would fail every retry, and the row completes as before.
+                if (probeSql) {
+                    const { rows } = await this.#db.executeSql(probeSql);
+                    if (!rows[0] || (rows[0].invalid && !this.#config.noIndexProgressView)) {
+                        buildPending = !rows[0] && this.#config.backend === 'cockroachdb';
+                        const name = plans.bamCommandIndexName(entry.command);
+                        throw new Error(rows[0]
+                            ? `index ${name} is invalid after the build`
+                            : `index ${name} does not exist after the build. An earlier build of it may still be running.`);
+                    }
+                }
             }
             // Record the outcome even when a stop landed while the command was running. stop() waits out
             // #working and index.ts closes the pool only after #bam.stop() resolves, so the UPDATE is safe
@@ -154,6 +273,11 @@ class Bam extends EventEmitter {
             });
         }
         catch (err) {
+            // The one failure that leaves the row in_progress: stop() dropped the connection under a
+            // concurrent index statement that the server is still running. See Ending.
+            if (running.ended && running.ending === 'abandon') {
+                return true;
+            }
             // Same reasoning as the completed path: a stop must not leave the row in_progress. This write
             // is guarded because it commonly runs on the connection that just failed - a terminated backend
             // or dropped connection fails the command AND the UPDATE that would record it. Letting that
@@ -165,7 +289,11 @@ class Bam extends EventEmitter {
             catch (markErr) {
                 this.emit(events.error, markErr);
             }
-            this.emit(events.error, err);
+            // A failure stop() caused is not an error. The row still records it, so the next attempt runs
+            // as a reattempt and heals what the cancel left.
+            if (!running.ended && !buildPending) {
+                this.emit(events.error, err);
+            }
             this.emit(events.bam, {
                 id: entry.id,
                 name: entry.name,
@@ -175,11 +303,22 @@ class Bam extends EventEmitter {
                 error: String(err)
             });
         }
+        finally {
+            this.#running = undefined;
+        }
+        return true;
     }
-    async #getNextCommand() {
-        const sql = plans.getNextBamCommand(this.#config.schema, { useLiveness: !this.#config.noIndexProgressView });
-        const { rows } = await this.#db.executeSql(sql);
-        return rows[0] || null;
+    async #getNextCommand(pendingOnly) {
+        const sql = plans.claimBamCommand(this.#config.schema, {
+            useLiveness: !this.#config.noIndexProgressView,
+            pendingOnly,
+            noAdvisoryLocks: this.#config.noAdvisoryLocks
+        });
+        const { rows } = unwrapSQLResult(await this.#db.executeSql(sql));
+        // The lock's own row comes first, so the claim is told apart by a column only it returns. Without
+        // the lock's row the db returned only the last statement, COMMIT, and a claim it made is lost.
+        assert(this.#config.noAdvisoryLocks || rows.some(row => row && 'pg_advisory_xact_lock' in row), 'bam claim returned no lock row: a custom db must return the rows of every statement in a multi-statement query');
+        return rows.find(row => row?.claimedStartedOn !== undefined) ?? null;
     }
     async #releaseCommand(entry) {
         const sql = plans.releaseBamCommand(this.#config.schema, entry.id, entry.priorStatus, entry.priorStartedOn ?? null, entry.claimedStartedOn);

package/dist/boss.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"boss.d.ts","sourceRoot":"","sources":["../src/boss.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,aAAa,CAAA;AAEtC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AAGvC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAkInC,cAAM,IAAK,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;;IAsC1D,MAAM;;;MAAS;gBAGb,EAAE,EAAE,KAAK,CAAC,SAAS,EACnB,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAa1C,IAAI,WAAW,IAAK,OAAO,CAE1B;IAIK,KAAK;IAiBL,IAAI;IAoHJ,SAAS,CAAE,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,gBAAgB;IA2tBvF;;;;OAIG;IACG,kBAAkB,CAAE,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAwB7E;AAED,eAAe,IAAI,CAAA"}
+{"version":3,"file":"boss.d.ts","sourceRoot":"","sources":["../src/boss.ts"],"names":[],"mappings":"AACA,OAAO,YAAY,MAAM,aAAa,CAAA;AAEtC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AAGvC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAkInC,cAAM,IAAK,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;;IAsC1D,MAAM;;;MAAS;gBAGb,EAAE,EAAE,KAAK,CAAC,SAAS,EACnB,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAa1C,IAAI,WAAW,IAAK,OAAO,CAE1B;IAIK,KAAK;IAiBL,IAAI;IAwHJ,SAAS,CAAE,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,gBAAgB;IAizBvF;;;;OAIG;IACG,kBAAkB,CAAE,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAwB7E;AAED,eAAe,IAAI,CAAA"}

package/dist/boss.js

@@ -1,3 +1,4 @@
+import assert from 'node:assert';
 import EventEmitter from 'node:events';
 import { ClaimTimer } from "./claimTimer.js";
 import * as plans from "./plans.js";
@@ -242,10 +243,14 @@ class Boss extends EventEmitter {
         if (!this.#config.persistQueueStats || !this.#config.queueStatRetentionDays) {
             return;
         }
-        const sql = this.#config.noTablePartitioning
-            ? plans.deleteOldQueueStats(this.#config.schema, this.#config.queueStatRetentionDays)
-            : plans.dropOldQueueStatsPartitions(this.#config.schema, this.#config.queueStatRetentionDays);
-        await this.#executeQuery(sql);
+        const { schema, queueStatRetentionDays: days } = this.#config;
+        if (this.#config.noTablePartitioning) {
+            const batchSize = this.#config.__test__walk_batch_size;
+            await this.#walk((after) => plans.deleteOldQueueStats(schema, days, { after, batchSize }));
+        }
+        else {
+            await this.#executeQuery(plans.dropOldQueueStatsPartitions(schema, days));
+        }
     }
     // Waits for a pass in flight rather than skipping like #onSupervise, since the caller asked for
     // this one. Holding #maintaining keeps it from overlapping the background pass and makes stop()
@@ -392,38 +397,117 @@ class Boss extends EventEmitter {
                 return;
             // CockroachDB rejects the multi-mutation failJobs() CTE these use, so under noMultiMutationCte
             // route expiry through the manager's split select/delete/re-insert variants instead.
-            if (this.#config.noMultiMutationCte) {
-                await this.#manager.failJobsByTimeoutDistributed(table, queues);
-            }
-            else {
-                const sql = plans.failJobsByTimeout(this.#config.schema, table, queues, this.#config.noAdvisoryLocks);
-                await this.#executeQuery(sql);
-            }
+            const limit = this.#config.__test__expiry_batch_size ?? plans.EXPIRY_BATCH_SIZE;
+            await this.#repeatWhileFull(limit, () => this.#config.noMultiMutationCte
+                ? this.#manager.failJobsByTimeoutDistributed(table, queues, limit)
+                : this.#expiryBatch(plans.failJobsByTimeout(this.#config.schema, table, queues, this.#config.noAdvisoryLocks, limit)));
             if (this.#stopping)
                 return;
-            if (this.#config.noMultiMutationCte) {
-                await this.#manager.failJobsByHeartbeatDistributed(table, queues);
-            }
-            else {
-                const heartbeatSql = plans.failJobsByHeartbeat(this.#config.schema, table, queues, this.#config.noAdvisoryLocks);
-                await this.#executeQuery(heartbeatSql);
-            }
+            await this.#repeatWhileFull(limit, () => this.#config.noMultiMutationCte
+                ? this.#manager.failJobsByHeartbeatDistributed(table, queues, limit)
+                : this.#expiryBatch(plans.failJobsByHeartbeat(this.#config.schema, table, queues, this.#config.noAdvisoryLocks, limit)));
         }
     }
+    // Runs a maintenance sweep a batch at a time until a batch picks fewer than `limit` rows. A stop
+    // between batches leaves the rest for the next pass.
+    async #repeatWhileFull(limit, batch) {
+        while (!this.#stopping) {
+            if (await batch() < limit)
+                return;
+        }
+    }
+    // CockroachDB returns the count as a string.
+    async #expiryBatch(sql) {
+        const { rows } = await this.#executeQuery(sql);
+        return Number(rows.find((row) => row.expiryPicked !== undefined)?.expiryPicked ?? 0);
+    }
     async #maintain(table, names) {
         if (this.#stopping)
             return;
-        const command = plans.trySetQueueDeletionTime(this.#config.schema, names, this.#config.maintenanceIntervalSeconds, plans.queueClaimSkipLocked(this.#config.backend, this.#config.noSkipLocked));
+        const skipLocked = plans.queueClaimSkipLocked(this.#config.backend, this.#config.noSkipLocked);
+        const command = plans.trySetQueueDeletionTime(this.#config.schema, names, this.#config.maintenanceIntervalSeconds, skipLocked);
         const { rows } = await this.#executeQuery(command);
+        const queues = rows.map((q) => q.name);
+        // One statement stamps every queue it claims with the same time.
+        const claimedOn = rows[0]?.claimedOn;
+        let finished = 0;
+        try {
+            while (finished < queues.length && await this.#maintainQueue(table, queues[finished])) {
+                finished++;
+            }
+        }
+        finally {
+            // The claim covers every queue up front, so a pass that a stop or a failure ends early hands
+            // back the ones it did not finish. Otherwise they would wait a whole maintenance interval.
+            if (finished < queues.length) {
+                try {
+                    await this.#executeQuery(plans.releaseQueueDeletionTime(this.#config.schema, queues.slice(finished), claimedOn, skipLocked));
+                }
+                catch (err) {
+                    this.emit(events.error, err);
+                }
+            }
+        }
+    }
+    // The retention sweep and the dependency cleanup for one queue. Returns whether every walk covered
+    // the whole queue.
+    async #maintainQueue(table, name) {
         if (this.#stopping)
-            return;
-        if (rows.length) {
-            const queues = rows.map((q) => q.name);
-            const sql = plans.deletion(this.#config.schema, table, queues, this.#config.noAdvisoryLocks);
-            await this.#executeQuery(sql);
-            const depSql = plans.cleanupDependencies(this.#config.schema, table, queues, this.#config.noAdvisoryLocks);
-            await this.#executeQuery(depSql);
+            return false;
+        const { schema, noAdvisoryLocks, backend } = this.#config;
+        const batchSize = this.#config.__test__walk_batch_size;
+        // The walk counts the completed and cancelled jobs that finished by DELTA_LAG before it
+        // started, so a completion committed late is still ahead of the watermark, and the monitor
+        // counts the rest after that (see plans.createTableQueue).
+        const { rows: [{ time }] } = await this.#executeQuery(plans.getTime(schema));
+        const retainedBefore = Number(time) - plans.DELTA_LAG_SECONDS * 1000;
+        const retention = { retained: 0, deleted: 0, deletedRetained: 0 };
+        let done = false;
+        try {
+            done = await this.#walk((after) => plans.deletion(schema, table, name, { after, batchSize, backend, noAdvisoryLocks, retainedBefore }), retention);
         }
+        finally {
+            // A walk that a stop or a failed batch cut short has not counted what it kept, so it only
+            // takes what it deleted off the counts, as a bulk delete does. A failed update is reported so
+            // that it does not hide the batch's own error, and the next full walk sets the count.
+            if (!done && retention.deleted) {
+                try {
+                    await this.#executeQuery({ text: plans.reduceRetainedCount(schema), values: [name, retention.deletedRetained, retention.deleted] });
+                }
+                catch (err) {
+                    this.emit(events.error, err);
+                }
+            }
+        }
+        // Only a walk that covered the whole queue has counted all of what it kept.
+        if (done) {
+            await this.#executeQuery({ text: plans.setRetainedCount(schema, table), values: [name, retention.retained, retainedBefore] });
+        }
+        const children = await this.#walk((after) => plans.cleanupDependencies(schema, table, name, 'child', { after, batchSize, backend, noAdvisoryLocks }));
+        const parents = await this.#walk((after) => plans.cleanupDependencies(schema, table, name, 'parent', { after, batchSize, backend, noAdvisoryLocks }));
+        return done && children && parents;
+    }
+    // Walks one queue's keys a batch at a time (see plans.deletion), carrying each batch's cursor into
+    // the next until a batch comes back without one, and returns whether it got there. A stop between
+    // batches leaves the rest for the next pass. `tally` totals the batches' counts as they commit, so
+    // it holds them even when a batch throws. CockroachDB returns counts as strings. A batch with no
+    // result row fails the walk rather than ending it, since a walk taken as done records its count.
+    async #walk(batch, tally) {
+        let after;
+        while (!this.#stopping) {
+            const { rows } = await this.#executeQuery(batch(after));
+            const row = rows.find((row) => row.walkCursor !== undefined);
+            assert(row, 'maintenance batch returned no result row: a custom db must return the rows of every statement in a multi-statement query');
+            if (tally) {
+                tally.retained += Number(row.retained ?? 0);
+                tally.deleted += Number(row.deleted ?? 0);
+                tally.deletedRetained += Number(row.deletedRetained ?? 0);
+            }
+            if (!row.walkCursor)
+                return true;
+            after = row.walkCursor;
+        }
+        return false;
     }
     /**
      * Defer the next queue-stats aggregate when this pass spent long enough inside it to threaten
@@ -712,11 +796,10 @@ class Boss extends EventEmitter {
             // because "no rows" from an unreadable view must not read as "horizon is healthy".
             const readable = await this.#narrowXminHorizonSources(attempted, lastVacuum);
             this.#xminHorizonSources = readable;
+            // Persisted like the horizon warnings themselves, so a table with none of them does not read
+            // as a healthy horizon on an instance that cannot check it.
             if (!readable.length) {
-                this.emit(events.warning, {
-                    message: 'Unable to read the transaction horizon; the xmin_horizon check is disabled for this instance.',
-                    data: { type: WARNING_TYPES.XMIN_HORIZON, error: err?.message }
-                });
+                await emitAndPersistWarning(this.#warningContext, WARNING_TYPES.XMIN_HORIZON, 'Unable to read the transaction horizon; the xmin_horizon check is disabled for this instance.', { type: WARNING_TYPES.XMIN_HORIZON, instance: this.#manager.instanceId, error: err?.message });
             }
             return null;
         }

package/dist/cli.js

@@ -331,9 +331,20 @@ async function cmdMigrate(args) {
         const partitionTables = await getPartitionTables(db, schema);
         const { sql, concurrent } = migrationStore.migrateCommands(schema, version, migrationStore.getAllForConfig(config), config.noAdvisoryLocks, { inlineAsync: true, partitionTables });
         await db.executeSql(sql);
-        // CONCURRENTLY index builds must run outside the migration transaction, one at a time.
-        for (const statement of concurrent) {
-            await db.executeSql(statement);
+        // CONCURRENTLY index builds must run outside the migration transaction, one at a time. The
+        // version is already committed by then and nothing enqueues them for BAM, so a failed build
+        // would never be retried: hand the rest to the operator instead.
+        for (const [i, statement] of concurrent.entries()) {
+            try {
+                await db.executeSql(statement);
+            }
+            catch (err) {
+                console.error(`pg-boss schema "${schema}" is at version ${schemaVersion}, but an index build failed: ${err.message}`);
+                console.error('Drop any invalid index the failed build left, then run these statements one at a time:\n');
+                console.error(concurrent.slice(i).map(rest => `${rest};`).join('\n\n'));
+                process.exitCode = 1;
+                return;
+            }
         }
         console.log(`Successfully migrated pg-boss schema "${schema}" to version ${schemaVersion}`);
     }

package/dist/db.d.ts

@@ -17,7 +17,10 @@ declare class Db extends EventEmitter implements types.IDatabase, types.EventsMi
     close(): Promise<void>;
     poolCounts(): types.PoolCounts | null;
     setSessionStatements(statements: string[]): Promise<void>;
-    executeSql(text: string, values?: unknown[]): Promise<import("pg").QueryResult<any>>;
+    executeSql(text: string, values?: unknown[], { signal, retire }?: {
+        signal?: AbortSignal;
+        retire?: AbortSignal;
+    }): Promise<import("pg").QueryResult<any>>;
     listen(channel: string, onNotification: (payload: string) => void, onReconnect: () => void): Promise<types.ListenHandle>;
     beginTransaction(): Promise<types.TransactionHandle>;
     withTransaction<T>(fn: (db: types.IDatabase) => Promise<T>): Promise<T>;

package/dist/db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,aAAa,CAAA;AAItC,OAAO,KAAK,KAAK,KAAK,MAAM,YAAY,CAAA;AAaxC,eAAO,MAAM,+BAA+B,OAAO,CAAA;AAEnD,cAAM,EAAG,SAAQ,YAAa,YAAW,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,WAAW;IACzE,OAAO,CAAC,IAAI,CAAU;IACtB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,KAAK,CAAa;IAG1B,OAAO,CAAC,iBAAiB,CAAe;IAGxC,MAAM,EAAE,OAAO,CAAA;gBAEF,MAAM,EAAE,KAAK,CAAC,eAAe,GAAG;QAAE,KAAK,CAAC,EAAE,KAAK,CAAC,KAAK,CAAA;KAAE;IAcpE,MAAM;;MAEL;IAEK,IAAI;IAgBJ,KAAK;IAQX,UAAU,IAAK,KAAK,CAAC,UAAU,GAAG,IAAI;IAUhC,oBAAoB,CAAE,UAAU,EAAE,MAAM,EAAE;IAoC1C,UAAU,CAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE;IAqB5C,MAAM,CACV,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EACzC,WAAW,EAAE,MAAM,IAAI,GACtB,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC;IA2JxB,gBAAgB,IAAK,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC;IAsFrD,eAAe,CAAC,CAAC,EAAG,EAAE,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAY/E;AAED,eAAe,EAAE,CAAA"}
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,aAAa,CAAA;AAItC,OAAO,KAAK,KAAK,KAAK,MAAM,YAAY,CAAA;AAaxC,eAAO,MAAM,+BAA+B,OAAO,CAAA;AAEnD,cAAM,EAAG,SAAQ,YAAa,YAAW,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,WAAW;IACzE,OAAO,CAAC,IAAI,CAAU;IACtB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,KAAK,CAAa;IAG1B,OAAO,CAAC,iBAAiB,CAAe;IAGxC,MAAM,EAAE,OAAO,CAAA;gBAEF,MAAM,EAAE,KAAK,CAAC,eAAe,GAAG;QAAE,KAAK,CAAC,EAAE,KAAK,CAAC,KAAK,CAAA;KAAE;IAcpE,MAAM;;MAEL;IAEK,IAAI;IAgBJ,KAAK;IAQX,UAAU,IAAK,KAAK,CAAC,UAAU,GAAG,IAAI;IAUhC,oBAAoB,CAAE,UAAU,EAAE,MAAM,EAAE;IA2C1C,UAAU,CAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAO;IA4DrH,MAAM,CACV,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EACzC,WAAW,EAAE,MAAM,IAAI,GACtB,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC;IA2JxB,gBAAgB,IAAK,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC;IAsFrD,eAAe,CAAC,CAAC,EAAG,EAAE,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAY/E;AAED,eAAe,EAAE,CAAA"}