pg-boss 12.33.2 → 12.33.4

package/dist/plans.js

@@ -15,7 +15,7 @@ const SEVEN_DAYS = 60 * 60 * 24 * 7;
 // A bam row stuck at 'in_progress' past this means the process that claimed it died or was
 // stopped mid-command (bam.ts #processCommands returns without marking the row on #stopped or a
 // crash), and getNextBamCommand needs to reclaim it or every future async migration wedges forever.
-// This is the *fallback* backstop only — deliberately long (24h) because false-reclaim is the worse
+// This is the *fallback* backstop only, deliberately long (24h) because false-reclaim is the worse
 // failure: reclaiming a still-running CREATE INDEX CONCURRENTLY runs two builds on the same index at
 // once, and an interrupted CONCURRENTLY leaves an INVALID index needing manual cleanup. The intended
 // *primary* trigger is a liveness check against pg_stat_progress_create_index (reclaim as soon as no
@@ -25,7 +25,7 @@ const BAM_STALE_SECONDS = 60 * 60 * 24;
 // Liveness grace window: on the native-Postgres path we don't trust a "no build running" reading
 // until a claimed command has had time to actually start and register in pg_stat_progress_create_index
 // (pool latency between claiming the row and issuing the build). Below this age, an in_progress row is
-// only reclaimed via the 24h fallback, never via liveness — so a genuinely-running build is never
+// only reclaimed via the 24h fallback, never via liveness, so a genuinely-running build is never
 // yanked out from under itself.
 const BAM_LIVENESS_GRACE_SECONDS = 60 * 5;
 export const JOB_STATES = Object.freeze({
@@ -323,7 +323,7 @@ function createTableSubscription(schema) {
   `;
 }
 // created_on defaults to clock_timestamp(), not ${schema}.job_now(), so multiple job_table_run_async() enqueues
-// within a single migration transaction keep their insertion order — BAM applies queued commands in
+// within a single migration transaction keep their insertion order, BAM applies queued commands in
 // created_on order, and some migrations enqueue an ordered drop-then-rebuild pair (see v33).
 /* eslint-enable no-restricted-syntax */
 /* eslint-disable no-restricted-syntax -- bam.created_on orders several enqueues within one transaction; the schema clock would tie them */
@@ -825,14 +825,14 @@ function createIndexJobFetch(schema, noCoveringIndex = false) {
     //    linearly. INCLUDE cannot help: FOR UPDATE forces an Index Scan, never an Index Only Scan.
     //
     //  - `id` is deliberately NOT a key column. The obvious shape, (name, priority DESC, created_on,
-    //    id), also satisfies the ordering — but id is a random uuid, so it defeats btree
+    //    id), also satisfies the ordering, but id is a random uuid, so it defeats btree
     //    deduplication and dominates size: 24 MB against 19 MB for this shape at 500k rows, on the
     //    hot insert path and on every vacuum's bulkdelete. It buys nothing, because fetchNextJob
     //    dropped id from its ORDER BY for the same reason (a random uuid was never creation order);
     //    with the ordering ending at created_on, this index satisfies it outright and the plan
     //    carries no sort node at all.
     //
-    // Also measured: neutral for key_strict_fifo (that fetch never reaches this index — the
+    // Also measured: neutral for key_strict_fifo (that fetch never reaches this index. The
     // strict_fifo_heads CTE on job_i10 feeds the outer CTE), and a ~7x win on CockroachDB, which
     // takes the noSkipLocked path and so does not depend on Incremental Sort at all.
     //
@@ -996,23 +996,29 @@ export function setMonitorBackoff(schema, elapsedSeconds) {
 export function trySetQueueDeletionTime(schema, queues, seconds) {
     return trySetQueueTimestamp(schema, queues, 'maintain_on', seconds);
 }
-// The cron claim, which also answers with the timestamp it replaced. That timestamp is when an
-// instance last ran a pass, so the pass reads it to find out how long scheduling was off, which is
-// the window a schedule's `missed` policy catches up over. Null on a database no pass has ever run
-// against, where there is no gap to catch up on.
+// The cron claim, which also answers with the timestamp it replaced and how old that timestamp was.
+// The timestamp is when an instance last ran a pass, so the pass reads it to find out how long
+// scheduling was off, which is the window a schedule's `missed` policy catches up over. Null on a
+// database no pass has ever run against, where there is no gap to catch up on.
 //
-// The prior value has to come from a CTE of its own: RETURNING sees the row as the UPDATE leaves
+// Both answers have to come from a CTE of their own: RETURNING sees the row as the UPDATE leaves
 // it, while a WITH sub-statement reads the snapshot the whole statement was planned against, so it
-// sees the value the UPDATE is replacing. Zero rows still means another instance holds the claim,
-// which is all the caller checked before.
+// sees the value the UPDATE is replacing.
+//
+// This answers on both outcomes rather than on a claim alone, which is why `claimed` is a column
+// and not the row count: an instance that was refused needs to know when the row it lost to comes
+// due, so its next attempt can be measured from the claim that beat it rather than from its own
+// failure. See Timekeeper.onCron(). Reading `claim` through EXISTS does not make the UPDATE
+// conditional - a data-modifying CTE always runs, exactly once, whether or not the outer query
+// reads it.
 export function trySetCronTime(schema, seconds) {
     return `
     WITH prior AS (
-      SELECT cron_on FROM ${schema}.version
+      SELECT cron_on, EXTRACT( EPOCH FROM (${schema}.job_now() - cron_on) ) as elapsed FROM ${schema}.version
     ), claim AS (
       ${trySetTimestamp(schema, 'cron_on', seconds)}
     )
-    SELECT prior.cron_on as "priorCronOn" FROM prior, claim
+    SELECT prior.cron_on as "priorCronOn", prior.elapsed as "elapsed", EXISTS (SELECT 1 FROM claim) as "claimed" FROM prior
   `;
 }
 export function trySetBamTime(schema, seconds) {
@@ -1318,8 +1324,8 @@ export function createIndexQueueStats(schema, noCoveringIndex = false) {
 // explicit `+00` timestamptz literals. This keeps partitioning correct regardless of the database
 // session TimeZone (a bare date literal like '2026-06-25' would otherwise be cast to timestamptz in
 // the session TZ, so rows written near UTC midnight could fall outside every existing partition).
-// Computing the date in SQL (rather than interpolating new Date()) also keeps emitted DDL — including
-// the v35 migration and exported create plans — deterministic and apply-time accurate.
+// Computing the date in SQL (rather than interpolating new Date()) also keeps emitted DDL, including
+// the v35 migration and exported create plans, deterministic and apply-time accurate.
 export function ensureQueueStatsPartitions(schema) {
     return `
     DO $$
@@ -1394,7 +1400,7 @@ export function insertQueueStats(schema, queues, noAdvisoryLocks) {
     return locked(schema, sql, 'queue-stats-insert', noAdvisoryLocks);
 }
 // Cheap single-row read of the cached counts the monitor maintains on the queue table. capturedOn
-// is monitor_on — the moment those counts were last refreshed, or NULL if the queue has never been
+// is monitor_on. The moment those counts were last refreshed, or NULL if the queue has never been
 // monitored (so the caller knows to recompute rather than trust default-zero counts).
 //
 // monitorBackoff rides along because the caller must serve this cache even when it is stale while
@@ -1438,7 +1444,7 @@ export function getQueueStatsHistory(schema) {
   `;
 }
 // Per-bucket aggregate over a count column. The function name can't be a bind parameter, so it's
-// interpolated — safe because the manager validates `aggregate` against this whitelist first. Every
+// interpolated, safe because the manager validates `aggregate` against this whitelist first. Every
 // result is cast back to int: it honors the int count contract (avg rounds) and keeps Postgres
 // returning the value as a JS number rather than a numeric string.
 const STATS_AGG = {
@@ -1450,8 +1456,8 @@ const STATS_AGG = {
 // bucket's counts with `aggregate`, so a wide window returns a manageable, representative sample
 // instead of just the newest `limit` raw rows.
 //
-//   mode 'bucket' — $5 is the bucket width in seconds (explicit resolution).
-//   mode 'auto'   — $5 is maxDataPoints; the width is derived so the series fits in $5 points.
+//   mode 'bucket', $5 is the bucket width in seconds (explicit resolution).
+//   mode 'auto', $5 is maxDataPoints; the width is derived so the series fits in $5 points.
 //                   from/to sets the range, but they cannot exceed the data's own min/max values.
 //
 // The bucket key avoids date_bin() (PG14+): pg-boss supports PostgreSQL 13+ and CockroachDB/
@@ -1668,7 +1674,7 @@ export function fetchNextJob(options, noSkipLocked = false) {
         : '';
     // state DESC (retry > created in the enum) makes a retry job its key's head. job_i8
     // guarantees at most one job per key in active/retry/failed, and the NOT EXISTS blocker
-    // below rejects every sibling of a retry job — so if created_on picked the head, an older
+    // below rejects every sibling of a retry job, so if created_on picked the head, an older
     // deferred job whose start_after has since arrived would claim the head slot while being
     // unfetchable, and the retry job (fetchable but not the head) would deadlock the key.
     const strictFifoHeadsCte = keyStrictFifo
@@ -1715,13 +1721,13 @@ export function fetchNextJob(options, noSkipLocked = false) {
     //
     // The two conditionals this replaces existed only to serve the `priority` and `orderByCreatedOn`
     // fetch options, which were requested as performance escapes from this sort. job_i11 now satisfies
-    // the ordering directly, so the escapes have nothing to escape — see the deprecation in
+    // the ordering directly, so the escapes have nothing to escape. See the deprecation in
     // manager.fetch(). Dropping them also deletes two branches from the hottest SQL builder here.
     //
     // `id` goes too. It is a random uuid, so it never provided creation order: a batch insert shares
     // one ${schema}.job_now() and therefore ties on created_on, and those ties resolve today in random uuid order,
     // not insertion order. Without it the index satisfies the ordering outright rather than through
-    // an Incremental Sort — 0.097 -> 0.031 ms and 50 -> 5 buffers at limit=1 — and ties fall back to
+    // an Incremental Sort (0.097 -> 0.031 ms and 50 -> 5 buffers at limit=1) and ties fall back to
     // index order, which tracks insertion order better than a uuid does.
     //
     // key_strict_fifo keeps its own id tiebreak in strict_fifo_heads, where DISTINCT ON does need a
@@ -1929,7 +1935,7 @@ export function restoreJobs(schema, table) {
 // A `startAfter` string is either an absolute date time or a delay expressed as a Postgres
 // interval. A trailing 'Z' has always marked a date time; a leading ISO 8601 calendar date
 // (YYYY-MM-DD) marks one as well, which is what lets the other 8601 zone designators through
-// ('+00:00', '+05:30', '-08:00') along with zone-less and date-only strings — all of which
+// ('+00:00', '+05:30', '-08:00') along with zone-less and date-only strings, all of which
 // used to reach the interval cast and fail as 'invalid input syntax for type interval'.
 //
 // Recognition is only ever widened, so anything that resolves as an interval today still
@@ -2526,17 +2532,17 @@ export function insertDeadLetterJob(schema) {
 // Dead-letter redrive. Moves un-started jobs out of a dead-letter queue and
 // re-creates them as fresh jobs on their original source queue (or $2 destination override),
 // oldest-first, capped at $4. The JOIN in `candidates` only matches jobs whose destination queue
-// exists, so legacy/orphaned jobs (NULL source_name, no override) are never deleted — they stay
+// exists, so legacy/orphaned jobs (NULL source_name, no override) are never deleted. They stay
 // in the DLQ rather than being lost. Re-created jobs get a new id, `created` state, retry_count 0,
 // cleared output, NULL source_*, and every queue-config column (retry/retention/policy/expiry/
-// heartbeat/dead_letter) from the destination queue as it is configured now — per-job overrides
+// heartbeat/dead_letter) from the destination queue as it is configured now, per-job overrides
 // from the original send() are not preserved, since the DLQ copy never stored them. `dead_letter`
 // is the same value send() falls back to, so a second terminal failure re-enters the DLQ.
 // Job-identity columns (priority, singleton_key, group_id, group_tier) are carried over instead.
 //
 // The insert's ON CONFLICT DO NOTHING is load-bearing: a destination queue's short/stately policy
 // can still collide on (name, singleton_key) if two redriven jobs share a key (job_i1/job_i3), and
-// dropping just that row — matching retried_jobs' ON CONFLICT DO NOTHING elsewhere — is preferable
+// dropping just that row (matching retried_jobs' ON CONFLICT DO NOTHING elsewhere) is preferable
 // to aborting the whole batch. The dropped job has already been deleted from the DLQ by the moved
 // CTE and is not restored.
 export function redriveJobs(schema, table) {
@@ -2711,7 +2717,7 @@ export function getQueueStats(schema, table, queues) {
 // Length of the recent-ready-count sliding window kept on queue.ready_history for the dashboard
 // sparkline. One sample is appended per monitor cycle (default 60s), so this is roughly the last
 // READY_HISTORY_SIZE minutes of trend. Sized to comfortably render the sparkline (the widest is the
-// ~160px detail card) without over-collecting — more points than pixels add nothing visible.
+// ~160px detail card) without over-collecting, more points than pixels add nothing visible.
 export const READY_HISTORY_SIZE = 60;
 /* eslint-disable no-restricted-syntax -- how long this transaction has held its snapshot: real elapsed time, not job time */
 const PIN_SECONDS_SQL = 'EXTRACT(EPOCH FROM (clock_timestamp() - transaction_timestamp()))::float8';
@@ -2889,7 +2895,7 @@ function advisoryLock(schema, key) {
 // winning the race to duplicate a scan nobody asked for costs the job table.
 //
 // The guard is a one-time filter, so a lost race skips the scan rather than running and discarding
-// it — verified on the plan: `Parallel Seq Scan on job (never executed)`. It also has to gate the
+// it, verified on the plan: `Parallel Seq Scan on job (never executed)`. It also has to gate the
 // UPDATE, not just the aggregate: an empty stats CTE still LEFT JOINs, and the COALESCE(…, 0) would
 // write every count to zero.
 function tryAdvisoryLock(schema, key, noAdvisoryLocks) {
@@ -3004,48 +3010,56 @@ export function getNextBamCommand(schema, { useLiveness = false } = {}) {
         // a stuck in_progress row is reclaimed purely on the 24h fallback. No CONCURRENTLY healing, so no
         // reclaimed flag is emitted (bam.ts skips healing when noIndexProgressView is set anyway).
         return `
-      UPDATE ${schema}.bam
-      SET status = 'in_progress', started_on = ${schema}.job_now()
-      WHERE id = (
-        SELECT id FROM ${schema}.bam
+      WITH candidate AS (
+        SELECT c.id, c.status AS prior_status, c.started_on AS prior_started_on
+        FROM ${schema}.bam c
         WHERE (
-          status IN ('pending', 'failed')
-          OR (status = 'in_progress' AND started_on < ${schema}.job_now() - interval '${BAM_STALE_SECONDS} seconds')
+          c.status IN ('pending', 'failed')
+          OR (c.status = 'in_progress' AND c.started_on < ${schema}.job_now() - interval '${BAM_STALE_SECONDS} seconds')
         )
         AND NOT EXISTS (
-          SELECT 1 FROM ${schema}.bam
-          WHERE status = 'in_progress' AND started_on >= ${schema}.job_now() - interval '${BAM_STALE_SECONDS} seconds'
+          SELECT 1 FROM ${schema}.bam g
+          WHERE g.status = 'in_progress' AND g.started_on >= ${schema}.job_now() - interval '${BAM_STALE_SECONDS} seconds'
         )
-        ORDER BY (status != 'pending'), created_on
+        ORDER BY (c.status != 'pending'), c.created_on
         LIMIT 1
       )
-      RETURNING id, name, version, status, queue, table_name as "table", command, error,
-                created_on as "createdOn", started_on as "startedOn", completed_on as "completedOn"
+      UPDATE ${schema}.bam b
+      SET status = 'in_progress', started_on = ${schema}.job_now()
+      FROM candidate
+      WHERE b.id = candidate.id
+        AND b.status = candidate.prior_status
+        AND b.started_on IS NOT DISTINCT FROM candidate.prior_started_on
+      RETURNING b.id, b.name, b.version, b.status, b.queue, b.table_name as "table", b.command, b.error,
+                b.created_on as "createdOn", b.started_on as "startedOn", b.completed_on as "completedOn",
+                (candidate.prior_status <> 'pending') as reattempt,
+                candidate.prior_status as "priorStatus", candidate.prior_started_on::text as "priorStartedOn",
+                b.started_on::text as "claimedStartedOn"
     `;
     }
     // Native-Postgres liveness path. An in_progress row counts as "stale" (reclaimable) when it is past
     // the grace window AND no backend is actually building its index right now. The same predicate,
-    // negated, defines a genuinely-live command that must still block the queue — so a running build is
+    // negated, defines a genuinely-live command that must still block the queue, so a running build is
     // NEVER reclaimed (no matter how long it runs), and a dead one recovers within the grace window.
     // There is deliberately no 24h absolute cap here: liveBuild=true always means a build is in flight, so
     // capping on elapsed time would reclaim a genuinely-running build and start a second
-    // CREATE INDEX CONCURRENTLY on the same index — the exact double-build this path exists to prevent.
+    // CREATE INDEX CONCURRENTLY on the same index. The exact double-build this path exists to prevent.
     // (The timeout-only path's BAM_STALE_SECONDS fallback covers engines with no way to detect liveness.)
     //
     // liveBuild(tableCol): is a CREATE INDEX CONCURRENTLY actively building this table's index right now?
-    // Detected via pg_locks, NOT pg_stat_progress_create_index — and that choice is load-bearing for
+    // Detected via pg_locks, NOT pg_stat_progress_create_index, and that choice is load-bearing for
     // multi-instance safety. pg_stat_progress_* is filtered to the querying role's OWN backends (only a
     // superuser or a member of pg_read_all_stats sees another role's builds), so a progress-view check
     // silently reads a peer's live build as "dead" whenever pg-boss instances connect under different DB
-    // roles — and the heal step (bamHealProbe/bamHealDrop in bam.ts) would then DROP INDEX CONCURRENTLY a
+    // roles, and the heal step (bamHealProbe/bamHealDrop in bam.ts) would then DROP INDEX CONCURRENTLY a
     // live index mid-build, racing the builder into a double CREATE. pg_locks, by contrast, is cluster-wide
     // and visible to every role (verified empirically). CREATE INDEX CONCURRENTLY holds a
     // ShareUpdateExclusiveLock on the target table for the ENTIRE build and releases it the instant the
     // statement finishes or the backend dies, so a granted SUExclusive lock on the row's table is a
-    // crash-safe, role-agnostic "build in flight" signal — instances may run under different roles with no
+    // crash-safe, role-agnostic "build in flight" signal, instances may run under different roles with no
     // loss of safety. Ordinary queue DML never takes SUExclusive (it uses AccessShare/RowShare/
     // RowExclusive), so it can't false-trigger; a concurrent autovacuum/ANALYZE on the same table DOES take
-    // SUExclusive and reads as "live", but that false positive is in the SAFE direction — it only briefly
+    // SUExclusive and reads as "live", but that false positive is in the SAFE direction. It only briefly
     // DEFERS a reclaim, never drops a live index. Scoped to the current database (l.database) because
     // pg_locks is cluster-wide while relation OIDs are only unique per database. Correlating on the table
     // is enough because the queue runs only one in_progress command at a time.
@@ -3077,7 +3091,7 @@ export function getNextBamCommand(schema, { useLiveness = false } = {}) {
     // 'pending' rows have nothing to heal.
     return `
     WITH candidate AS (
-      SELECT c.id, c.status AS prior_status
+      SELECT c.id, c.status AS prior_status, c.started_on AS prior_started_on
       FROM ${schema}.bam c
       WHERE (
         c.status IN ('pending', 'failed')
@@ -3095,9 +3109,13 @@ export function getNextBamCommand(schema, { useLiveness = false } = {}) {
     SET status = 'in_progress', started_on = ${schema}.job_now()
     FROM candidate
     WHERE b.id = candidate.id
+      AND b.status = candidate.prior_status
+      AND b.started_on IS NOT DISTINCT FROM candidate.prior_started_on
     RETURNING b.id, b.name, b.version, b.status, b.queue, b.table_name as "table", b.command, b.error,
               b.created_on as "createdOn", b.started_on as "startedOn", b.completed_on as "completedOn",
-              (candidate.prior_status <> 'pending') as reattempt
+              (candidate.prior_status <> 'pending') as reattempt,
+              candidate.prior_status as "priorStatus", candidate.prior_started_on::text as "priorStartedOn",
+              b.started_on::text as "claimedStartedOn"
   `;
 }
 // Derives the drop-then-rebuild heal step for a re-attempted BAM index build: an interrupted
@@ -3114,8 +3132,8 @@ export function bamHealDrop(schema, command) {
 }
 // Probe run before bamHealDrop: returns `invalid = true` only when the index the re-attempted command
 // would build already exists AND is INVALID (an interrupted CREATE INDEX CONCURRENTLY left a stub).
-// A build that actually finished but whose BAM row was never marked completed — e.g. a graceful stop
-// landed between the CREATE succeeding and markCompleted — leaves a VALID index; dropping that would
+// A build that actually finished but whose BAM row was never marked completed, e.g. a graceful stop
+// landed between the CREATE succeeding and markCompleted, leaves a VALID index; dropping that would
 // tear down a live production index for the whole rebuild window, so the caller must NOT heal it (the
 // re-run's own IF NOT EXISTS then no-ops and just marks the row done). Returns null for non-index
 // commands and for an absent index (no row), where there is nothing to drop. Mirrors bamHealDrop's
@@ -3132,10 +3150,39 @@ export function bamHealProbe(schema, command) {
     WHERE n.nspname = '${resolveSchemaName(schema).replace(SINGLE_QUOTE_REGEX, "''")}' AND c.relname = '${match[1].replace(SINGLE_QUOTE_REGEX, "''")}'
   `;
 }
+// Undoes a claim that never ran: hands the row back exactly as getNextBamCommand found it, restoring
+// both the status and the started_on the claim overwrote. Used when a stop lands between the claim
+// UPDATE returning and the runner picking the command up. Leaving the row in_progress instead would
+// block every other BAM command until it went stale - BAM_LIVENESS_GRACE_SECONDS on native Postgres,
+// BAM_STALE_SECONDS (24 hours) on the timeout-only backends - and restoring started_on is what makes a
+// released stale-in_progress row immediately reclaimable again rather than starting that clock over.
+export function releaseBamCommand(schema, id, priorStatus, priorStartedOn, claimedStartedOn) {
+    // priorStartedOn arrives as the text rendering of the original timestamptz (the claim casts it in
+    // SQL rather than round-tripping a JS Date), so it carries its offset and full precision back.
+    const startedOn = priorStartedOn == null
+        ? 'NULL'
+        : `'${priorStartedOn.replace(SINGLE_QUOTE_REGEX, "''")}'::timestamptz`;
+    // Compare-and-swap on the claim this runner actually took, not on the id alone. The timeout-only
+    // claim has no SKIP LOCKED (see getNextBamCommand), so two overlapping claims can both return the
+    // same row - verified: both get rowCount 1 and the same prior_status. Releasing on the id alone
+    // would then reset a row a peer is actively building back to 'pending', and the next poll would
+    // start a second CREATE INDEX CONCURRENTLY on the same index. Matching started_on makes the release
+    // a no-op unless the claim is still ours.
+    return `
+    UPDATE ${schema}.bam
+    SET status = '${priorStatus.replace(SINGLE_QUOTE_REGEX, "''")}', started_on = ${startedOn}
+    WHERE id = '${id}'
+      AND status = 'in_progress'
+      AND started_on = '${claimedStartedOn.replace(SINGLE_QUOTE_REGEX, "''")}'::timestamptz
+  `;
+}
 export function setBamCompleted(schema, id) {
+    // error is cleared: a row reaching 'completed' is commonly a retry of a prior 'failed' attempt, and
+    // getBamEntries() returns error for every status - a stale message would keep reading as a failure
+    // on a command that succeeded.
     return `
     UPDATE ${schema}.bam
-    SET status = 'completed', completed_on = ${schema}.job_now()
+    SET status = 'completed', completed_on = ${schema}.job_now(), error = NULL
     WHERE id = '${id}'
   `;
 }
@@ -3176,7 +3223,7 @@ export function jobCommonExists(schema) {
 export function getManagedQueuePartitions(schema) {
     return `SELECT table_name as "table", policy FROM ${schema}.queue WHERE partition = true`;
 }
-// The CREATE INDEX command text of every BAM row not yet completed — used to tell a genuinely-missing
+// The CREATE INDEX command text of every BAM row not yet completed, used to tell a genuinely-missing
 // index apart from one an async build is still working on (or retrying after a failure).
 export function getIncompleteBamCommands(schema) {
     return `SELECT command FROM ${schema}.bam WHERE status <> 'completed'`;
@@ -3199,7 +3246,7 @@ const POLICY_JOB_INDEXES = {
     8: QUEUE_POLICIES.key_strict_fifo,
     10: QUEUE_POLICIES.key_strict_fifo
 };
-// job_iN indexes with no policy gate — created on every job table regardless of policy
+// job_iN indexes with no policy gate, created on every job table regardless of policy
 // (throttle i4, fetch i11, group-concurrency i7, blocking i9). 5 is absent, not missing: the fetch
 // index was replaced in v40 and the retired number is not reused.
 const BASE_JOB_INDEXES = [4, 7, 9, 11];
@@ -3214,7 +3261,7 @@ function applyManifestSchema(text, schema) {
     return text.split(schemaManifest.schemaToken).join(schema);
 }
 // The job_state enum values in declaration order, from the manifest (both sections carry the same enum).
-// Order is significant — the numeric base type makes created < retry < … < failed load-bearing.
+// Order is significant. The numeric base type makes created < retry < … < failed load-bearing.
 export const EXPECTED_JOB_STATES = schemaManifest.partitioned.enum;
 // The tables pg-boss expects to exist. The manifest lists the fixed tables plus job (and job_common in
 // partitioned mode); per-queue partition tables are dynamic, so they are appended from the live set.
@@ -3228,7 +3275,7 @@ export function expectedManagedTables(schema, partitioned, partitions = []) {
 // The columns pg-boss expects on each managed table, taken from the manifest (introspected column type,
 // nullability, and default). Fixed tables carry the full defaults + types maps so column defaults, data
 // types, and nullability are diffed; the job table (shared by job_common and every per-queue partition
-// in partitioned mode) is name-only — its FKs are profile-dependent and keep_until's interval default is
+// in partitioned mode) is name-only. Its FKs are profile-dependent and keep_until's interval default is
 // not worth pinning per-partition. A table with no live columns is skipped by the diff, so listing one
 // that does not exist yet is harmless.
 export function expectedManagedColumns(schema, partitioned, partitions = []) {
@@ -3296,7 +3343,7 @@ export function expectedManagedFunctions(schema, partitioned, options = {}) {
 // `partitioned` reflects the live database (job_common present ⇒ partitioned architecture), so this
 // needs no boss config. The manifest holds every fixed index (the static ones plus the job_iN set on
 // job/job_common); each per-queue partition is dynamic, so its indexes are templated here from the
-// job_common set — the base indexes always, plus the one for the queue's policy. keys/predicate/
+// job_common set. The base indexes always, plus the one for the queue's policy. keys/predicate/
 // definition are derived from the catalog-canonical pg_get_indexdef the manifest stores.
 export function expectedManagedIndexes(schema, partitioned, partitions = []) {
     const managed = (name, table, indexdef) => {
@@ -3330,13 +3377,13 @@ export function expectedManagedIndexes(schema, partitioned, partitions = []) {
 //
 // Autovacuum reclaims heap space but never returns btree pages to the OS, so a job index sizes
 // itself to the largest backlog its queue has ever held and stays there. A drained 1M-job peak
-// leaves job_common_i11 at 38 MB and job_common_pkey at 51 MB holding 1k live rows — and every
+// leaves job_common_i11 at 38 MB and job_common_pkey at 51 MB holding 1k live rows, and every
 // subsequent vacuum with dead tuples to clean does a full physical scan of every one of those
 // pages (measured: 17,969 buffer hits + 3,892 reads vs 750 hits / 0 reads after a rebuild), which
 // is what drains IO burst credits on managed Postgres.
 //
-// The bloat is a high-water mark rather than an unbounded leak — empty pages ARE recycled by later
-// inserts — so the trigger is density, not elapsed time.
+// The bloat is a high-water mark rather than an unbounded leak, empty pages ARE recycled by later
+// inserts, so the trigger is density, not elapsed time.
 export const REINDEX_DEFAULTS = {
     // 1 MB. Below this the absolute waste is irrelevant and reltuples/relpages is noisy.
     minPages: 128,
@@ -3365,8 +3412,8 @@ export const REINDEX_DEFAULTS = {
 // int4 is not an option - the index that exceeds maxIndexBytes is exactly the one that overflows it.
 // float8 is exact to 2^53 bytes, and reltuples is a float4 estimate to begin with.
 // The pages the index's *live* entries actually need, from the average width of what it indexes.
-// pg_stats carries that per column, and — the part that matters here, since i1-i4 and i6 key on
-// `COALESCE(singleton_key, '')` — ANALYZE also collects it for an index's expression columns, keyed
+// pg_stats carries that per column, and. The part that matters here, since i1-i4 and i6 key on
+// `COALESCE(singleton_key, '')`, ANALYZE also collects it for an index's expression columns, keyed
 // by the index relation. Plain attnums read the table's stats, attnum 0 (an expression) falls back
 // to the index's own. 20 bytes covers the btree tuple header and line pointer.
 const INDEX_WIDTH_JOIN = `LEFT JOIN LATERAL (
@@ -3409,15 +3456,15 @@ function jobTableScope(schema, tables) {
  * healthy one by three orders of magnitude on their own.
  *
  * PostgreSQL-only (including PGlite, where both the bloat and REINDEX CONCURRENTLY behave normally).
- * Callers must gate on noReindex — see the note in boss.ts #reindex.
+ * Callers must gate on noReindex. See the note in boss.ts #reindex.
  *
  * `owned` is returned rather than filtered on, so an installation whose role cannot reindex still
- * gets told what is bloated. Only leaf indexes (relkind 'i') are candidates — pg-boss creates
+ * gets told what is bloated. Only leaf indexes (relkind 'i') are candidates. pg-boss creates
  * i1..i10 directly on each partition, and a partitioned index (relkind 'I') cannot be reindexed
  * except through its leaves anyway.
  *
  * Caveat: relpages/reltuples are refreshed by VACUUM/ANALYZE, so they go stale where autovacuum is
- * disabled for the table. That is acceptable — the entire failure mode assumes autovacuum runs.
+ * disabled for the table. That is acceptable. The entire failure mode assumes autovacuum runs.
  */
 export function getBloatedIndexes(schema, tables, options) {
     // These are interpolated into the statement, and supervise()/getReindexCommands() accept them
@@ -3454,7 +3501,7 @@ export function getBloatedIndexes(schema, tables, options) {
   `;
 }
 /**
- * Every owned leaf index on the job tables, ignoring the density gate — the target list for
+ * Every owned leaf index on the job tables, ignoring the density gate. The target list for
  * `{ force: true }`.
  */
 export function getJobIndexes(schema, tables) {
@@ -3528,13 +3575,13 @@ function quoteIdentifier(name) {
  *
  * `pg_stat_activity.backend_xmin` is readable by an unprivileged role (it is not one of the columns
  * restricted to the owning user), as are the other three views, so the degraded path is defensive
- * rather than expected — a managed provider may still revoke them.
+ * rather than expected. A managed provider may still revoke them.
  */
 /* eslint-disable no-restricted-syntax -- these measure real backend and vacuum age against pg_stat_* timestamps Postgres wrote; a fake clock would compare two different clocks */
 export const XMIN_HORIZON_SOURCES = {
     // Restricted to backends whose transaction was already open when the failed vacuum ran ($1).
     // Every backend executing a query advertises a backend_xmin, including the one asking this
-    // question, so presence alone is not evidence — the ordering against a vacuum that reclaimed
+    // question, so presence alone is not evidence. The ordering against a vacuum that reclaimed
     // nothing is. Compared as timestamps inside one statement rather than as two ages read from two
     // queries a moment apart, which would let a connection that started after the vacuum qualify.
     //
@@ -3543,7 +3590,7 @@ export const XMIN_HORIZON_SOURCES = {
     // ordinary role only pid, application_name, usename and backend_xmin for a backend owned by a
     // *different* role; xact_start reads NULL and query reads '<insufficient privilege>' (measured on
     // PG18). Since `NULL < $1` is NULL, the strict form silently discarded exactly the holder an
-    // operator most needs named — another application's analytical query — leaving every source NULL,
+    // operator most needs named (another application's analytical query) leaving every source NULL,
     // no holder attributable, and the warning withheld. Admitting them costs a possible false
     // positive from a young foreign backend, which in practice loses the max(age()) to any real
     // holder, and opaqueBackends below reports how many could not be timed.
@@ -3560,7 +3607,7 @@ export const XMIN_HORIZON_QUERY_SOURCES = Object.keys(XMIN_HORIZON_SOURCES);
  * `age()` measures transactions elapsed since the pinned xid, not wall time, which is the metric
  * that matters: dead tuples accumulate per transaction, so a horizon pinned across an idle night
  * costs nothing while the same age on a busy queue is a backlog of rows autovacuum cannot reclaim.
- * Measured directly — a REPEATABLE READ holder open on an idle database reports 0, and 2,000 after
+ * Measured directly. A REPEATABLE READ holder open on an idle database reports 0, and 2,000 after
  * 2,000 transactions with the same holder still open.
  *
  * Also returns the oldest in-transaction backend's wall-clock age, purely so the warning can say
@@ -3687,7 +3734,7 @@ export function dropIndexConcurrently(schema, name) {
  * The statement list an operator runs by hand: every stale stub dropped first, then the rebuilds.
  *
  * Stubs are emitted whole rather than paired to the index they came from. Postgres does not append
- * `_ccnew` — it truncates the *base* so the result fits in 63 bytes, so `j<sha224>_i11` (61 chars)
+ * `_ccnew`. It truncates the *base* so the result fits in 63 bytes, so `j<sha224>_i11` (61 chars)
  * leaves behind `j<sha224>__ccnew` with the `i11` gone, and a prefix match never fires on a
  * partitioned queue (every partition table name is `'j' || sha224(queue_name)`, 57 chars). Dropping
  * them unconditionally is also what the background pass does: an invalid stub is dead weight that

package/dist/timekeeper.d.ts

@@ -31,6 +31,28 @@ declare class Timekeeper extends EventEmitter implements types.EventsMixin {
     private scheduleSkewMonitor;
     cacheClockSkew(): Promise<void>;
     onCron(): Promise<void>;
+    /**
+     * How long an instance whose claim was refused waits before trying again: until the row it lost
+     * to comes due, plus a beat.
+     *
+     * The beat keeps the retry off the due instant itself, where the holder's own attempt lands and
+     * where the row can still be a microsecond short of the interval. A quarter of the interval,
+     * capped at a second, so it stays a beat at every configurable interval instead of doubling the
+     * smallest one.
+     *
+     * No jitter. What it would buy is a spread the retries already have: each is measured from its
+     * own statement's snapshot and its own round trip, so several instances do not land on one
+     * instant, and a refused claim is a single-row UPDATE whose predicate fails. What it would cost
+     * is the one thing in this path a TestClock cannot drive to a known instant, which is what the
+     * regression test for this needs.
+     *
+     * `elapsed` is how old the row was in the snapshot the claim read, so a winner that commits
+     * between that snapshot and this statement's UPDATE leaves it measured against the previous
+     * stamp: the retry then comes back early, is refused, and measures again. Self-correcting, and it
+     * costs one extra statement in a race that has to happen inside a round trip. A null reads as a
+     * full interval, which is the row a pass has never stamped - a state no claim is refused in.
+     */
+    private claimRetryDelay;
     /**
      * `priorCronOn` is when an instance last ran a pass, as the claim in onCron() read it off the row
      * it advanced. Null on a database no pass has run against, and left null by a caller that does
@@ -63,14 +85,18 @@ declare class Timekeeper extends EventEmitter implements types.EventsMixin {
      * The gap is (lastPass, windowStart]: older than the due window, so no pass has sent it, and
      * newer than the moment an instance last ran a pass, so no pass has skipped it either. A pass
      * claims `cronMonitorIntervalSeconds` after the one before it plus the round trip the claim
-     * itself cost, 45 seconds and change at the configurable ceiling, against a 60-second window, so
-     * the range is empty while passes keep running and fills up when they stop: a deployment that is
-     * down, between deploys, or running with scheduling switched off.
-     *
-     * That the spacing is an interval and not two of them is the claim timer's doing: it schedules
-     * each attempt from the moment the previous claim stamped the row rather than on a grid, so a
-     * statement that lands quickly cannot measure the row short of the interval and lose the claim
-     * for a whole one. See ClaimTimer.
+     * itself cost and the beat a refused instance waits past the due moment, 46 seconds and change at
+     * the configurable ceiling, against a 60-second window, so the range is empty while passes keep
+     * running and fills up when they stop: a deployment that is down, between deploys, or running
+     * with scheduling switched off.
+     *
+     * That the spacing is an interval and not two of them takes both halves of onCron()'s answer. The
+     * claim timer schedules a winner's next attempt from the moment its claim stamped the row rather
+     * than on a grid, so a statement that lands quickly cannot measure the row short of the interval
+     * and lose the claim for a whole one. A loser schedules its next attempt for when the row it lost
+     * to comes due rather than an interval after its own failure, so the deployment keeps the spacing
+     * when the instance holding the claim stops rather than waiting out that instance's phase as
+     * well. See ClaimTimer and claimRetryDelay().
      *
      * The most recent occurrence rather than all of them, which is the whole of what `once` promises:
      * a job carries the schedule's `data` and nothing else, so a job per missed occurrence would be

package/dist/timekeeper.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"timekeeper.d.ts","sourceRoot":"","sources":["../src/timekeeper.ts"],"names":[],"mappings":"AAEA,OAAO,YAAY,MAAM,aAAa,CAAA;AAItC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AAKvC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAGnC,eAAO,MAAM,MAAM;;CAElB,CAAA;AAwBD,kGAAkG;AAClG,eAAO,MAAM,iBAAiB,OAAO,CAAA;AA8JrC,cAAM,UAAW,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;IAChE,EAAE,EAAE,KAAK,CAAC,SAAS,CAAA;IACnB,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAA;IACxC,OAAO,EAAE,OAAO,CAAA;IAEhB,OAAO,CAAC,OAAO,CAAO;IACtB,OAAO,CAAC,gBAAgB,CAA+B;IACvD,OAAO,CAAC,gBAAgB,CAAqC;IAC7D,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,aAAa,CAAQ;IAS7B,OAAO,CAAC,eAAe,CAAoB;IAE3C,SAAS,SAAI;IACb,MAAM;;;;MAAS;gBAEF,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAQ5F,IAAI,YAAY,IAAK,OAAO,CAE3B;IAMD,OAAO,KAAK,YAAY,GAEvB;IAED,OAAO,KAAK,cAAc,GASzB;IAEK,KAAK;IA4BL,IAAI;IAmCV,OAAO,CAAC,mBAAmB;IAUrB,cAAc;IA8Cd,MAAM;IAkCZ;;;;;OAKG;IACG,IAAI,CAAE,WAAW,GAAE,OAAc;IA0IvC,YAAY,CAAE,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,GAAE,KAAK,CAAC,YAAwC;IAIlG;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,cAAc;IAgBtB;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,OAAO,CAAC,gBAAgB;IAgBxB;;;;;;;OAOG;IACH,OAAO,CAAC,sBAAsB;IAgB9B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,eAAe;IAkBvB,OAAO,CAAC,iBAAiB;YAQX,QAAQ;YAsDR,aAAa;IAerB,YAAY,CAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;IAiBrE,WAAW,CAAE,IAAI,EAAE,MAAM,EAAE,GAAG,SAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;IAe1E;;;;;;;OAOG;IACH,OAAO,CAAC,gBAAgB;IAYxB;;;;;;;;;;;;;;;;OAgBG;IACH,eAAe,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,sBAA2B,GAAG,IAAI,EAAE;IA0C5E,QAAQ,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,KAAK,CAAC,eAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IA6CzG,UAAU,CAAE,IAAI,EAAE,MAAM,EAAE,GAAG,SAAK,GAAG,OAAO,CAAC,IAAI,CAAC;CAIzD;AAED,eAAe,UAAU,CAAA"}
+{"version":3,"file":"timekeeper.d.ts","sourceRoot":"","sources":["../src/timekeeper.ts"],"names":[],"mappings":"AAEA,OAAO,YAAY,MAAM,aAAa,CAAA;AAItC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AAKvC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAGnC,eAAO,MAAM,MAAM;;CAElB,CAAA;AAwBD,kGAAkG;AAClG,eAAO,MAAM,iBAAiB,OAAO,CAAA;AAgLrC,cAAM,UAAW,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;IAChE,EAAE,EAAE,KAAK,CAAC,SAAS,CAAA;IACnB,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAA;IACxC,OAAO,EAAE,OAAO,CAAA;IAEhB,OAAO,CAAC,OAAO,CAAO;IACtB,OAAO,CAAC,gBAAgB,CAA+B;IACvD,OAAO,CAAC,gBAAgB,CAAqC;IAC7D,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,aAAa,CAAQ;IAS7B,OAAO,CAAC,eAAe,CAAoB;IAE3C,SAAS,SAAI;IACb,MAAM;;;;MAAS;gBAEF,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAQ5F,IAAI,YAAY,IAAK,OAAO,CAE3B;IAMD,OAAO,KAAK,YAAY,GAEvB;IAED,OAAO,KAAK,cAAc,GASzB;IAEK,KAAK;IA4BL,IAAI;IAmCV,OAAO,CAAC,mBAAmB;IAUrB,cAAc;IA8Cd,MAAM;IAgDZ;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,OAAO,CAAC,eAAe;IAQvB;;;;;OAKG;IACG,IAAI,CAAE,WAAW,GAAE,OAAc;IA0IvC,YAAY,CAAE,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,GAAE,KAAK,CAAC,YAAwC;IAIlG;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,cAAc;IAgBtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,OAAO,CAAC,gBAAgB;IAgBxB;;;;;;;OAOG;IACH,OAAO,CAAC,sBAAsB;IAgB9B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,eAAe;IAkBvB,OAAO,CAAC,iBAAiB;YAQX,QAAQ;YAsDR,aAAa;IAerB,YAAY,CAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;IAiBrE,WAAW,CAAE,IAAI,EAAE,MAAM,EAAE,GAAG,SAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;IAe1E;;;;;;;OAOG;IACH,OAAO,CAAC,gBAAgB;IAYxB;;;;;;;;;;;;;;;;OAgBG;IACH,eAAe,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,sBAA2B,GAAG,IAAI,EAAE;IA0C5E,QAAQ,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,KAAK,CAAC,eAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IA6CzG,UAAU,CAAE,IAAI,EAAE,MAAM,EAAE,GAAG,SAAK,GAAG,OAAO,CAAC,IAAI,CAAC;CAIzD;AAED,eAAe,UAAU,CAAA"}