pipework 0.252.0 → 0.253.0

package/src/async/jobs/queue.ts

@@ -0,0 +1,1012 @@
+import { sql } from '../../data/db/sql.js'
+import type { DB, SQL } from '../../data/db/index.js'
+import { assertValidIdentifier, quoteIdentifier } from '../../data/db/identifiers.js'
+import { serializable, isTransientConcurrencyError } from '../../data/db/serializable.js'
+import type { DefinedTable } from '../../data/domain/types.js'
+import { abortableDelay } from './abortable-delay.js'
+import { JobListener } from './listener.js'
+import type { ListenClient, WaitOptions, WaitResult } from './listener.js'
+import { createScheduler } from './scheduler.js'
+import type { ScheduleSpec, ScheduleInfo, SchedulerOptions } from './scheduler.js'
+import { resolveExclusion, exclusionClaimClause } from './exclusion.js'
+import type { ExclusionConfig } from './exclusion.js'
+import type { JobLaterError } from './errors.js'
+import { openClaim, closeClaim, parseClaims } from './claims.js'
+import type { JobClaim, JobClaimReason } from './claims.js'
+import { serializeError } from './serialize-error.js'
+import { queueDefinition, registerProgressColumn, liveKeyIndexName, LIVE_KEY_TENANT_EXPR, LIVE_KEY_PREDICATE, pendingKeyIndexName, PENDING_KEY_TENANT_EXPR, PENDING_KEY_PREDICATE } from './table.js'
+import { queueDoorName } from './queue-door.js'
+import { unscopedExecutor, requireUnscopedExecutor } from '../../auth/tenant/scoped-db.js'
+import { getContext } from '../../request/context/store.js'
+import { resolveActorId } from '../../data/write/audit/actor.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import { recordFault, faultMessage } from '../../infra/alerts/faults.js'
+import type { ParentTraceRef } from '../../trace/entry/open-trace.js'
+
+export interface BackoffConfig {
+  readonly type: 'exponential' | 'fixed'
+  readonly delayMs: number
+  readonly maxDelayMs?: number
+}
+
+/** Queue configuration — table name, stale timeout, max attempts, backoff strategy, cross-job exclusion. */
+export interface QueueConfig {
+  readonly table?: string
+  readonly staleTimeoutMs?: number
+  readonly notifyChannel?: string
+  readonly claimFilter?: (candidateAlias: string) => string
+  readonly maxAttempts?: number
+  readonly backoff?: BackoffConfig
+  /** An app-added column on the queue table whose value is recorded as each claim's `progressMark` when the claim is released. */
+  readonly progressColumn?: string
+  /** Cross-job exclusion gate — overlap/blocks-all/tenant columns checked against claimed/running jobs at claim time. */
+  readonly exclusion?: ExclusionConfig
+}
+
+/** Exclusion values for one job — mapped onto the configured exclusion columns. */
+export interface EnqueueExclusion {
+  /** Conflict set for the configured arrayColumn; null/omitted = non-participating. */
+  readonly array?: readonly string[] | null
+  /** TRUE makes this job exclusive against every participating job in its scope. */
+  readonly blocksAll?: boolean
+  /** Tenant value for the configured tenantColumn — required when one is configured. */
+  readonly tenant?: string
+}
+
+/** Options for enqueuing a job — type, payload, priority, max attempts. */
+export interface EnqueueOptions {
+  readonly type: string
+  readonly priority?: number
+  readonly payload: unknown
+  readonly maxAttempts?: number
+  readonly exclusion?: EnqueueExclusion
+  /**
+   * Record the enqueuing Flow's trace, open step and tenant so the worker runs
+   * the job as a child of that trace under that tenant. Default true; false
+   * records none of them (the job runs as a root trace with no tenant).
+   */
+  readonly continueFlow?: boolean
+  /** Run the job under this tenant instead of the Flow's; `null` runs it without a tenant. */
+  readonly tenant?: string | null
+  /** Also record the Flow's actor (`auth.userId`); the job's `auth` is `{ userId }` when set. Default false. */
+  readonly carryActor?: boolean
+  /** Record this actor instead of the Flow's (`null`: none); overrides `carryActor`. Ignored with `continueFlow: false`. */
+  readonly actor?: string | null
+  /**
+   * The trace (and open step) the job continues, in place of the Flow's own.
+   * `null` records no trace parent at all. Tenant and actor still follow `tenant` and `carryActor`.
+   */
+  readonly parent?: ParentTraceRef | null
+  /**
+   * While a job on this queue with the same key and the same tenant is live (pending, claimed or running),
+   * enqueue returns that job's id and inserts nothing — the payload and other options of this call are
+   * ignored. Once that job completes, fails for good or is cancelled, the next enqueue inserts a new job.
+   * A non-empty string.
+   */
+  readonly liveKey?: string
+  /**
+   * While a job on this queue with the same key and the same tenant is pending (not yet claimed), enqueue
+   * replaces that job's payload with this call's and returns its id; every other option of this call is
+   * ignored and the job keeps its place in line. Once the job is claimed, the next enqueue inserts a new job.
+   * A non-empty string; not combinable with `liveKey`.
+   */
+  readonly pendingKey?: string
+}
+
+/** A job that has just been dead-lettered — handed to every `onDeadLetter` listener inside the transaction that did it. */
+export interface DeadLetterInfo {
+  readonly id: string
+  readonly type: string
+  readonly tenant: string | null
+  readonly payload: unknown
+  /** The serialized error of the last attempt. */
+  readonly error: unknown
+  /** The Flow that enqueued the job. */
+  readonly flow: ClaimedFlow
+}
+
+/** The enqueuing Flow a claimed job continues — every field null when none was recorded. */
+export interface ClaimedFlow {
+  readonly traceId: string | null
+  readonly rootId: string | null
+  readonly stepId: string | null
+  readonly tenant: string | null
+  readonly actor: string | null
+}
+
+/** The exclusion values a claimed job carries — present when the queue configures exclusion. */
+export interface ClaimedExclusion {
+  readonly array: readonly string[] | null
+  readonly blocksAll: boolean
+  readonly tenant: string | null
+}
+
+export type { JobClaim, JobClaimReason }
+
+/** A job claimed by a worker — includes ID, type, payload, and created LSN. */
+export interface ClaimedJob {
+  readonly id: string
+  readonly type: string
+  readonly priority: number
+  readonly payload: unknown
+  readonly createdLsn: string
+  /** Minted by this claim's own UPDATE; nulled when the claim ends. Compare it exactly to fence writes made under this claim. */
+  readonly claimToken: string
+  /** 1-based number of the attempt this claim starts. */
+  readonly attempt: number
+  readonly maxAttempts: number
+  /** Why the previous claim ended without a failure of its own — a reap or a hand-back — and the worker that held it; null otherwise. */
+  readonly previous: { readonly reason: 'reaped' | 'handed-back' | 'later'; readonly workerId: string | null } | null
+  readonly exclusion?: ClaimedExclusion
+  /** The Flow that enqueued this job; the worker surface restores its trace, tenant and actor. */
+  readonly flow: ClaimedFlow
+  /** The `liveKey` the job was enqueued with; null when none. */
+  readonly liveKey: string | null
+  /** The `pendingKey` the job was enqueued with; null when none. */
+  readonly pendingKey: string | null
+  /** The claims before this one, oldest first, each closed with its release time, reason and progress mark. */
+  readonly claims: readonly JobClaim[]
+}
+
+/** One job `reap` took from a lost worker. `attempt` is the 1-based number of the attempt that was lost. */
+export interface ReapedJob {
+  readonly jobId: string
+  readonly type: string
+  readonly attempt: number
+  readonly outcome: 'retry' | 'dead_letter' | 'superseded'
+  readonly maxAttempts: number
+  readonly flow: ClaimedFlow
+}
+
+/** What `failAttempt` did: `outcome` and the error as stored in the row's `error_payload`. */
+export interface FailedAttempt {
+  readonly outcome: 'retry' | 'dead_letter' | 'superseded'
+  readonly error: unknown
+}
+
+export interface JobResult {
+  readonly status: 'completed' | 'failed' | 'cancelled' | 'timeout' | 'dead_letter'
+  readonly output?: unknown
+  readonly error?: unknown
+}
+
+/** Jobs ready to claim, jobs claimed or running, and how long the oldest ready job has waited since it was queued (null when none is ready). */
+export interface QueueTypeTotals {
+  readonly ready: number
+  readonly running: number
+  readonly oldestReadyWaitMs: number | null
+}
+
+/** The whole queue's totals; `byType` omits a type with nothing ready and nothing running. */
+export interface QueueTotals {
+  readonly overall: QueueTypeTotals
+  readonly byType: Readonly<Record<string, QueueTypeTotals>>
+}
+
+/** Job queue instance with claim/complete/fail lifecycle, LISTEN/NOTIFY, and cron scheduling. */
+export interface Queue {
+  /** The queue table as a pipe.define() definition — `pipework generate` emits its DDL. */
+  readonly definition: DefinedTable
+  /**
+   * The framework door (`pipework.queue.<table>`) that claim, reap, heartbeat and complete need on a platform-mode
+   * database; null when the queue has no `exclusion.tenantColumn`, whose table is not row-secured by tenant.
+   */
+  readonly door: string | null
+  enqueue(db: DB, opts: EnqueueOptions): Promise<string>
+  /** Claims the next eligible job (of one of `types`, when given), retrying transient conflicts until clean. The signal ends the retry loop on shutdown — claim resolves null promptly. */
+  claim(db: DB, workerId: string, signal?: AbortSignal, types?: readonly string[], release?: string | null): Promise<ClaimedJob | null>
+  /**
+   * Each of heartbeat, complete and fail takes the claim's token and applies only to that claim:
+   * once the row was reaped, handed back or claimed again they change nothing and return false.
+   * Without a token they match on the id alone.
+   */
+  heartbeat(db: DB, jobId: string, claimToken?: string): Promise<boolean>
+  complete(db: DB, jobId: string, output?: unknown, claimToken?: string): Promise<boolean>
+  /** Counts the attempt as failed: retried with backoff while attempts remain, dead-lettered after. One transaction, row locked. */
+  fail(db: DB, jobId: string, error: unknown, claimToken?: string, opts?: { permanent?: boolean }): Promise<boolean>
+  /** `fail` that says what it did: null when the claim no longer holds the row, else the outcome and the stored error. */
+  failAttempt(db: DB, jobId: string, error: unknown, claimToken?: string, opts?: { permanent?: boolean }): Promise<FailedAttempt | null>
+  /** Returns a claimed job to pending after the error's `delayMs` (else the backoff, at least 1 s) without counting the attempt. False when the claim no longer holds the row. */
+  later(db: DB, jobId: string, error: JobLaterError, claimToken: string): Promise<boolean>
+  /** Returns a claimed job to pending without counting the attempt. False when the claim no longer holds the row. */
+  handBack(db: DB, jobId: string, claimToken: string): Promise<boolean>
+  /**
+   * Cancels a pending or claimed job; no-op on any other status. `error_payload` becomes exactly
+   * `{ reason, message }`, replacing what an earlier attempt left, and a held claim is closed 'cancelled'
+   * with the same object as its `cancel`. An empty `reason.reason` throws.
+   */
+  cancel(db: DB, jobId: string, reason: { reason: string; message: string }): Promise<void>
+  /** For tests and support tools: makes pending jobs waiting on a retry time due now — the one job, or every one. Returns how many it changed. */
+  retryNow(db: DB, jobId?: string): Promise<number>
+  /** For tests and support tools: the Flow a job was enqueued under, by its id; null when no such job. */
+  flowOf(db: DB, jobId: string): Promise<ClaimedFlow | null>
+  /** Ready (pending and not waiting on a retry time), running (claimed or running) and the oldest ready job's wait, per job type and overall. One query. */
+  totals(db: DB): Promise<QueueTotals>
+  /** Fails every stale claimed job through the same retry/dead-letter rule as `fail`, and reports each. */
+  reap(db: DB): Promise<ReapedJob[]>
+  /**
+   * Registers a listener called, inside the transaction that dead-letters a job (by `fail` or `reap`), after the
+   * row is updated. A listener's throw is logged and rolled back to a savepoint; it never fails the dead-lettering.
+   */
+  onDeadLetter(listener: (tx: DB, job: DeadLetterInfo) => Promise<void>): () => void
+  requeueDeadLetter(db: DB, jobId: string): Promise<void>
+  startListener(client: ListenClient): Promise<void>
+  stopListener(): Promise<void>
+  waitFor(db: DB, jobId: string, opts: WaitOptions): Promise<WaitResult>
+  enqueueAndWait(db: DB, opts: EnqueueOptions, waitOpts: WaitOptions): Promise<WaitResult>
+  schedule(db: DB, spec: ScheduleSpec): Promise<void>
+  /** Upserts every declared schedule and removes schedules named in `owned` that are not declared. Returns the removed names. */
+  syncSchedules(db: DB, declared: readonly ScheduleSpec[], owned: readonly string[]): Promise<string[]>
+  /** Test hook: enqueues the named schedule's job now, as a scheduler tick would, without moving its next run. Returns the job id. */
+  fireSchedule(db: DB, name: string): Promise<string>
+  /** Fires every due schedule once (one transaction each, so processes sharing a database fire it once between them). Returns the count fired. */
+  runDueSchedules(db: DB, names?: readonly string[]): Promise<number>
+  unschedule(db: DB, name: string): Promise<boolean>
+  listSchedules(db: DB): Promise<ScheduleInfo[]>
+  startScheduler(db: DB, opts?: SchedulerOptions): void
+  stopScheduler(): void
+}
+
+/** Whether the error (or a cause in its chain) is the unique violation of the queue's live-key index. */
+function isLiveKeyViolation(err: unknown, constraint: string): boolean {
+  let current: unknown = err
+  for (let depth = 0; depth < 4 && typeof current === 'object' && current !== null; depth++) {
+    const e = current as { code?: unknown; constraint_name?: unknown }
+    if (e.code === '23505' && e.constraint_name === constraint) return true
+    current = (current as { cause?: unknown }).cause
+  }
+  return false
+}
+
+/** Serializes values into a Postgres array literal (`{"a","b"}`) — sent as one text param and cast to the element type. */
+function pgArrayLiteral(values: readonly string[]): string {
+  const escaped = values.map(v => `"${v.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`)
+  return `{${escaped.join(',')}}`
+}
+
+/** The fields of a locked job row that the retry/dead-letter rule needs. */
+interface JobRow {
+  readonly jobId: string
+  readonly type: string
+  readonly payload: unknown
+  readonly tenant: string | null
+  readonly attemptCount: number
+  readonly maxAttempts: number
+  readonly flow: ClaimedFlow
+}
+
+function flowOf(row: Record<string, unknown>): ClaimedFlow {
+  return {
+    traceId: (row['flow_trace_id'] as string | null) ?? null,
+    rootId: (row['flow_root_id'] as string | null) ?? null,
+    stepId: (row['flow_step_id'] as string | null) ?? null,
+    tenant: (row['flow_tenant'] as string | null) ?? null,
+    actor: (row['flow_actor'] as string | null) ?? null,
+  }
+}
+
+export function createQueue(config: QueueConfig = {}): Queue {
+  const table = config.table ?? 'pipework_jobs'
+  const staleTimeoutMs = config.staleTimeoutMs ?? 120_000
+  const channel = config.notifyChannel ?? 'pipework_job_complete'
+  const maxAttempts = config.maxAttempts ?? 1
+  const backoff = config.backoff
+
+  assertValidIdentifier(table, 'queue table')
+  assertValidIdentifier(channel, 'notify channel')
+  if (config.progressColumn !== undefined) assertValidIdentifier(config.progressColumn, 'queue progress column')
+  const progressQ = config.progressColumn === undefined ? null : quoteIdentifier(config.progressColumn, 'queue progress column')
+  registerProgressColumn(table, config.progressColumn)
+
+  const tableQ = quoteIdentifier(table, 'queue table')
+  const listener = new JobListener(channel, tableQ)
+
+  const deadLetterListeners = new Set<(tx: DB, job: DeadLetterInfo) => Promise<void>>()
+
+  const exclusion = resolveExclusion(config.exclusion)
+  const definition = queueDefinition(table, exclusion)
+
+  /** What the active Flow contributes to a job: its trace, open step, tenant and (on request) actor. */
+  function enqueuingFlow(opts: EnqueueOptions): ClaimedFlow {
+    const none: ClaimedFlow = { traceId: null, rootId: null, stepId: null, tenant: null, actor: null }
+    if (opts.continueFlow === false) return none
+    const flow = getContext()
+    if (flow === undefined) {
+      const parent = opts.parent ?? null
+      return {
+        ...none,
+        traceId: parent?.id ?? null,
+        rootId: parent?.rootId ?? null,
+        stepId: parent?.stepId ?? null,
+        ...(opts.tenant !== undefined ? { tenant: opts.tenant } : {}),
+        actor: opts.actor ?? null,
+      }
+    }
+
+    const trace = flow.trace !== null && !flow.trace.buffer.truncated ? flow.trace : null
+    const parent = opts.parent !== undefined ? opts.parent : null
+    return {
+      traceId: opts.parent !== undefined ? (parent?.id ?? null) : (trace?.id ?? null),
+      rootId: opts.parent !== undefined ? (parent?.rootId ?? null) : (trace?.rootId ?? null),
+      stepId: opts.parent !== undefined ? (parent?.stepId ?? null) : (trace?.buffer.openStepId ?? null),
+      tenant: opts.tenant !== undefined ? opts.tenant : flow.tenant,
+      actor: opts.actor !== undefined ? opts.actor : opts.carryActor === true ? resolveActorId(flow) : null,
+    }
+  }
+
+  /** `AND flow_tenant = T` for a tenant-scoped caller, nothing otherwise. */
+  function tenantAnd(tenant: string | null): SQL {
+    return tenant === null ? sql`` : sql` AND flow_tenant = ${tenant}::text`
+  }
+
+  /** Worker and cross-tenant ops refuse a tenant-scoped handle up front and run on the handle they were given. */
+  function crossTenant(db: DB, op: string): DB {
+    return requireUnscopedExecutor(db, `queue ${op}()`, `${op} acts across tenants on queue "${table}"`)
+  }
+
+  function foreignTenant(op: string, scoped: string, named: string, what: string): never {
+    throw new Error(
+      `[pipework] ${op}() on queue "${table}" was given a handle scoped to tenant "${scoped}" but ${what} "${named}".\n\n` +
+      '  A tenant-scoped connection enqueues only its own tenant\'s jobs. Enqueue for another tenant\n' +
+      '  from an unscoped handle (pipe.system(), or pool.getOrCreate(...).drizzle).\n',
+    )
+  }
+
+  async function enqueue(handle: DB, opts: EnqueueOptions): Promise<string> {
+    return (await enqueueJob(handle, opts)).id
+  }
+
+  /** The tries an enqueue with a liveKey makes while the live job it conflicts with keeps finishing before it can be read. */
+  const LIVE_KEY_TRIES = 3
+
+  /** Enqueues, or with a liveKey returns the live job holding the key (`created: false`). */
+  async function enqueueJob(handle: DB, opts: EnqueueOptions): Promise<{ id: string; created: boolean }> {
+    if (opts.liveKey !== undefined && opts.liveKey === '') {
+      throw new Error(
+        `[pipework] enqueue() on queue "${table}" received an empty liveKey.\n\n` +
+        '  An empty key would make every such job share one slot. Pass a non-empty string, or omit liveKey.\n',
+      )
+    }
+    if (opts.pendingKey !== undefined && opts.pendingKey === '') {
+      throw new Error(
+        `[pipework] enqueue() on queue "${table}" received an empty pendingKey.\n\n` +
+        '  An empty key would make every such job share one slot. Pass a non-empty string, or omit pendingKey.\n',
+      )
+    }
+    if (opts.pendingKey !== undefined && opts.liveKey !== undefined) {
+      throw new Error(
+        `[pipework] enqueue() on queue "${table}" received both liveKey and pendingKey.\n\n` +
+        '  liveKey returns the live job and keeps its payload; pendingKey replaces the pending job\'s payload. Pass one.\n',
+      )
+    }
+    const { db, tenant: scopedTenant } = unscopedExecutor(handle)
+    const priority = opts.priority ?? 0
+    const payload = JSON.stringify(opts.payload)
+    const attempts = opts.maxAttempts ?? maxAttempts
+
+    const columns = ['job_type', 'priority', 'payload', 'max_attempts']
+    const values: SQL[] = [sql`${opts.type}`, sql`${priority}`, sql`${payload}::jsonb`, sql`${attempts}`]
+
+    const flow: { -readonly [K in keyof ClaimedFlow]: ClaimedFlow[K] } = { ...enqueuingFlow(opts) }
+    if (scopedTenant !== null) {
+      if (flow.tenant !== null && flow.tenant !== scopedTenant) {
+        foreignTenant('enqueue', scopedTenant, flow.tenant, opts.tenant !== undefined ? 'opts.tenant names' : 'the flow carries tenant')
+      }
+      flow.tenant = scopedTenant
+    }
+    columns.push('flow_trace_id', 'flow_root_id', 'flow_step_id', 'flow_tenant', 'flow_actor')
+    values.push(
+      sql`${flow.traceId}::uuid`, sql`${flow.rootId}::uuid`, sql`${flow.stepId}::uuid`,
+      sql`${flow.tenant}::text`, sql`${flow.actor}::text`,
+    )
+
+    if (exclusion === undefined) {
+      if (opts.exclusion !== undefined) {
+        throw new Error(
+          `[pipework] enqueue() received exclusion values, but queue "${table}" has no exclusion config.\n\n` +
+          '  Silently dropping them would enqueue a job without its conflict data.\n' +
+          '  Configure the queue: createQueue({ exclusion: { arrayColumn: ..., blocksAllColumn: ..., tenantColumn: ... } }).\n',
+        )
+      }
+    } else {
+      const given = opts.exclusion ?? {}
+      if (given.array !== undefined && given.array !== null && exclusion.arrayColumn === undefined) {
+        throw new Error(
+          `[pipework] enqueue() received exclusion.array, but queue "${table}" configures no arrayColumn.\n\n` +
+          '  Add arrayColumn to the queue exclusion config, or drop the value.\n',
+        )
+      }
+      if (given.blocksAll !== undefined && exclusion.blocksAllColumn === undefined) {
+        throw new Error(
+          `[pipework] enqueue() received exclusion.blocksAll, but queue "${table}" configures no blocksAllColumn.\n\n` +
+          '  Add blocksAllColumn to the queue exclusion config, or drop the value.\n',
+        )
+      }
+      if (given.tenant !== undefined && exclusion.tenantColumn === undefined) {
+        throw new Error(
+          `[pipework] enqueue() received exclusion.tenant, but queue "${table}" configures no tenantColumn.\n\n` +
+          '  Add tenantColumn to the queue exclusion config, or drop the value.\n',
+        )
+      }
+      if (exclusion.tenantColumn !== undefined) {
+        if (given.tenant === undefined) {
+          throw new Error(
+            `[pipework] enqueue() requires exclusion.tenant — queue "${table}" scopes exclusion by "${exclusion.tenantColumn}".\n\n` +
+            '  Every job needs a tenant: enqueue(db, { ..., exclusion: { tenant: <id> } }).\n',
+          )
+        }
+        if (scopedTenant !== null && given.tenant !== scopedTenant) {
+          foreignTenant('enqueue', scopedTenant, String(given.tenant), 'exclusion.tenant names')
+        }
+        columns.push(quoteIdentifier(exclusion.tenantColumn, 'exclusion column'))
+        values.push(sql`${given.tenant}::uuid`)
+      }
+      if (exclusion.arrayColumn !== undefined && given.array !== undefined && given.array !== null) {
+        columns.push(quoteIdentifier(exclusion.arrayColumn, 'exclusion column'))
+        values.push(sql`${pgArrayLiteral(given.array)}::${sql.raw(exclusion.arrayElementType)}[]`)
+      }
+      if (exclusion.blocksAllColumn !== undefined && given.blocksAll !== undefined) {
+        columns.push(quoteIdentifier(exclusion.blocksAllColumn, 'exclusion column'))
+        values.push(sql`${given.blocksAll}`)
+      }
+    }
+
+    if (opts.pendingKey !== undefined) {
+      columns.push('pending_key')
+      values.push(sql`${opts.pendingKey}::text`)
+      const pendingKey = opts.pendingKey
+      const rows = await db.transaction(async (tx) => {
+        await tx.execute(sql`SELECT ${pendingKeyLock(sql`${pendingKey}::text`, sql`coalesce(${flow.tenant}::text, '')`)}`)
+        return tx.execute(
+          sql`INSERT INTO ${sql.raw(tableQ)} (${sql.raw(columns.join(', '))})
+                   VALUES (${sql.join(values, sql`, `)})
+                   ON CONFLICT (pending_key, (${sql.raw(PENDING_KEY_TENANT_EXPR)})) WHERE ${sql.raw(PENDING_KEY_PREDICATE)}
+                   DO UPDATE SET payload = EXCLUDED.payload
+                   RETURNING id::text, (xmax = 0) AS created`
+        )
+      })
+      return { id: rows[0]!['id'] as string, created: rows[0]!['created'] === true }
+    }
+
+    if (opts.liveKey === undefined) {
+      const rows = await db.execute(
+        sql`INSERT INTO ${sql.raw(tableQ)} (${sql.raw(columns.join(', '))})
+                 VALUES (${sql.join(values, sql`, `)})
+                 RETURNING id::text`
+      )
+      return { id: rows[0]!['id'] as string, created: true }
+    }
+
+    columns.push('live_key')
+    values.push(sql`${opts.liveKey}::text`)
+    const insert = sql`INSERT INTO ${sql.raw(tableQ)} (${sql.raw(columns.join(', '))})
+               VALUES (${sql.join(values, sql`, `)})
+               ON CONFLICT (live_key, (${sql.raw(LIVE_KEY_TENANT_EXPR)})) WHERE ${sql.raw(LIVE_KEY_PREDICATE)} DO NOTHING
+               RETURNING id::text`
+    // A separate statement for the lookup: a CTE would share the insert's snapshot and could not see
+    // a conflicting row a concurrent enqueuer committed after it began.
+    const lookup = sql`SELECT id::text FROM ${sql.raw(tableQ)}
+               WHERE live_key = ${opts.liveKey}::text AND ${sql.raw(LIVE_KEY_TENANT_EXPR)} = coalesce(${flow.tenant}::text, '')
+                 AND status IN ('pending', 'claimed', 'running')
+               LIMIT 1`
+    for (let attempt = 0; attempt < LIVE_KEY_TRIES; attempt++) {
+      const inserted = await db.execute(insert)
+      if (inserted.length > 0) return { id: inserted[0]!['id'] as string, created: true }
+      const live = await db.execute(lookup)
+      if (live.length > 0) return { id: live[0]!['id'] as string, created: false }
+    }
+    throw new Error(
+      `[pipework] enqueue() on queue "${table}" could not settle liveKey "${opts.liveKey}" after ${LIVE_KEY_TRIES} tries.\n\n` +
+      '  Each try found the key held by a live job that finished before it could be read, then lost the\n' +
+      '  insert again. Retry the enqueue.\n',
+    )
+  }
+
+  function buildClaimQuery(workerId: string, types?: readonly string[], release?: string | null): SQL {
+    const typeClause = types !== undefined ? sql` AND candidate.job_type = ANY(${pgArrayLiteral(types)}::text[])` : sql``
+    const claimFilterClause = config.claimFilter
+      ? ` AND (${config.claimFilter('candidate')})`
+      : ''
+    const exclusionClause = exclusion !== undefined
+      ? ` AND ${exclusionClaimClause(tableQ, exclusion)}`
+      : ''
+    const exclusionReturning = exclusion !== undefined
+      ? [exclusion.tenantColumn, exclusion.arrayColumn, exclusion.blocksAllColumn]
+        .filter((c): c is string => c !== undefined)
+        .map(c => `, ${quoteIdentifier(c, 'exclusion column')}`)
+        .join('')
+      : ''
+
+    return sql`
+      UPDATE ${sql.raw(tableQ)} SET
+        ${openClaim(workerId, release ?? null)},
+        status = 'claimed',
+        claimed_at = now(),
+        heartbeat_at = now(),
+        worker_id = ${workerId},
+        worker_release = ${release ?? null},
+        claim_token = gen_random_uuid()
+      WHERE id = (
+        SELECT candidate.id FROM ${sql.raw(tableQ)} candidate
+        WHERE candidate.status = 'pending' AND (candidate.next_retry_at IS NULL OR candidate.next_retry_at <= now())${sql.raw(exclusionClause)}${sql.raw(claimFilterClause)}${typeClause}
+        ORDER BY candidate.priority DESC, candidate.queued_at ASC
+        FOR UPDATE SKIP LOCKED
+        LIMIT 1
+      )
+      RETURNING id::text, job_type, priority, payload, created_lsn::text, claim_token::text, attempt_count, max_attempts, error_payload->>'reason' AS previous_reason, error_payload->>'workerId' AS previous_worker_id, flow_trace_id::text, flow_root_id::text, flow_step_id::text, flow_tenant, flow_actor, live_key, pending_key, claims #- ARRAY[(jsonb_array_length(claims) - 1)::text] AS claims_before${sql.raw(exclusionReturning)}`
+  }
+
+  function previousOf(row: Record<string, unknown>): ClaimedJob['previous'] {
+    const reason = row['previous_reason']
+    if (reason !== 'reaped' && reason !== 'handed-back' && reason !== 'later') return null
+    return { reason, workerId: (row['previous_worker_id'] as string | null | undefined) ?? null }
+  }
+
+  function mapClaimedRow(row: Record<string, unknown>): ClaimedJob {
+    const base = {
+      id: row['id'] as string,
+      type: row['job_type'] as string,
+      priority: row['priority'] as number,
+      payload: row['payload'],
+      createdLsn: row['created_lsn'] as string,
+      claimToken: row['claim_token'] as string,
+      attempt: (row['attempt_count'] as number) + 1,
+      maxAttempts: row['max_attempts'] as number,
+      previous: previousOf(row),
+      claims: parseClaims(row['claims_before']),
+      liveKey: (row['live_key'] as string | null | undefined) ?? null,
+      pendingKey: (row['pending_key'] as string | null | undefined) ?? null,
+      flow: {
+        traceId: (row['flow_trace_id'] as string | null | undefined) ?? null,
+        rootId: (row['flow_root_id'] as string | null | undefined) ?? null,
+        stepId: (row['flow_step_id'] as string | null | undefined) ?? null,
+        tenant: (row['flow_tenant'] as string | null | undefined) ?? null,
+        actor: (row['flow_actor'] as string | null | undefined) ?? null,
+      },
+    }
+    if (exclusion === undefined) return base
+    return {
+      ...base,
+      exclusion: {
+        array: exclusion.arrayColumn !== undefined ? (row[exclusion.arrayColumn] as string[] | null) : null,
+        blocksAll: exclusion.blocksAllColumn !== undefined ? (row[exclusion.blocksAllColumn] as boolean) : false,
+        tenant: exclusion.tenantColumn !== undefined ? (row[exclusion.tenantColumn] as string) : null,
+      },
+    }
+  }
+
+  /** Full-jitter backoff for transient claim conflicts — identical schedules re-collide, jitter decorrelates the herd. */
+  function conflictDelay(attempt: number): number {
+    return Math.random() * Math.min(1000, 50 * 2 ** attempt)
+  }
+
+  async function claim(handle: DB, workerId: string, signal?: AbortSignal, types?: readonly string[], release?: string | null): Promise<ClaimedJob | null> {
+    const db = crossTenant(handle, 'claim')
+    const query = buildClaimQuery(workerId, types, release)
+
+    // Serialization failures (40001) carry Postgres's safe-retry guarantee:
+    // the conflicting transaction committed, so every abort means another
+    // claimer made progress and a retry cannot fail the same way again.
+    // Retrying until a clean attempt keeps the contract exact — null means
+    // "nothing claimable", never "gave up under contention". A fixed retry
+    // budget here starves non-conflicting claimers under sustained load.
+    // The loop's only other exit is shutdown: the signal cancels the backoff
+    // sleep and the next attempt, returning null — without it a worker stuck
+    // under sustained contention can never observe shutdown.
+    for (let attempt = 0; ; attempt++) {
+      if (signal?.aborted === true) return null
+      try {
+        // Two workers evaluating the exclusion NOT EXISTS checks against the
+        // same MVCC snapshot can both pass and claim conflicting jobs (write
+        // skew) — the exclusion claim query must run under SERIALIZABLE.
+        const rows = exclusion !== undefined
+          ? await serializable(db, async (tx) => tx.execute(query), { maxAttempts: 1 })
+          : await db.execute(query)
+        if (rows.length === 0) return null
+        return mapClaimedRow(rows[0]!)
+      } catch (err: unknown) {
+        if (!isTransientConcurrencyError(err)) throw err
+        await abortableDelay(conflictDelay(attempt), signal)
+      }
+    }
+  }
+
+  /** Rows a claim-scoped statement may touch: the claim's own token while it is still held. */
+  function claimScope(claimToken: string | undefined): SQL {
+    return claimToken === undefined
+      ? sql``
+      : sql` AND claim_token = ${claimToken}::uuid AND status IN ('claimed', 'running')`
+  }
+
+  async function heartbeat(handle: DB, jobId: string, claimToken?: string): Promise<boolean> {
+    const db = crossTenant(handle, 'heartbeat')
+    const rows = await db.execute(
+      sql`UPDATE ${sql.raw(tableQ)} SET heartbeat_at = now() WHERE id = ${jobId}::uuid${claimScope(claimToken)} RETURNING id`
+    )
+    return rows.length > 0
+  }
+
+  async function complete(handle: DB, jobId: string, output?: unknown, claimToken?: string): Promise<boolean> {
+    const db = crossTenant(handle, 'complete')
+    const outputJson = output !== undefined ? JSON.stringify(output) : null
+    const rows = await db.execute(
+      sql`UPDATE ${sql.raw(tableQ)} SET ${closeClaim('completed', progressQ)}, status = 'completed', completed_at = now(), claim_token = NULL, output = ${outputJson !== null ? sql`${outputJson}::jsonb` : sql`NULL`}
+               WHERE id = ${jobId}::uuid${claimScope(claimToken)} RETURNING id`
+    )
+    if (rows.length === 0) return false
+    await db.execute(
+      sql`SELECT pg_notify(${channel}, ${jobId})`
+    )
+    return true
+  }
+
+  function calculateDelay(attemptCount: number): number {
+    if (!backoff) return 0
+    if (backoff.type === 'fixed') return backoff.delayMs
+    const delay = backoff.delayMs * Math.pow(2, attemptCount)
+    return backoff.maxDelayMs !== undefined ? Math.min(delay, backoff.maxDelayMs) : delay
+  }
+
+  /** The advisory lock an enqueue and a return-to-pending both take, per (table, pendingKey, tenant), before touching the index. */
+  function pendingKeyLock(key: SQL, tenant: SQL): SQL {
+    return sql`pg_advisory_xact_lock(hashtextextended(${table}::text || chr(31) || ${key} || chr(31) || ${tenant}, 0))`
+  }
+
+  /** Takes the lock for a job's own key and tenant; nothing when the job has no pendingKey. */
+  async function lockPendingKeyOf(tx: DB, jobId: string): Promise<void> {
+    await tx.execute(
+      sql`SELECT ${pendingKeyLock(sql`pending_key`, sql`coalesce(flow_tenant, '')`)} FROM ${sql.raw(tableQ)}
+           WHERE id = ${jobId}::uuid AND pending_key IS NOT NULL`
+    )
+  }
+
+  /**
+   * The SET fragments that put a claimed job back to pending — unless a newer pending job holds its
+   * pendingKey, in which case the job is cancelled instead (claim closed as 'cancelled' with `cancel`
+   * `{ reason: 'superseded', message }`, error_payload `{ reason: 'superseded', by: <twin id> }`), so the twin with the newer payload is the one that runs.
+   * Decided inside the UPDATE, from the row as it is when the statement runs. Callers must hold
+   * `lockPendingKeyOf` for the row first, in the same transaction, so a racing enqueue cannot add a twin unseen.
+   */
+  function returnToPending(reason: JobClaimReason, errorPayload: SQL): SQL {
+    const twinId = sql`(SELECT twin.id::text FROM ${sql.raw(tableQ)} AS twin
+        WHERE twin.pending_key = ${sql.raw(tableQ)}.pending_key
+          AND coalesce(twin.flow_tenant, '') = coalesce(${sql.raw(tableQ)}.flow_tenant, '')
+          AND twin.status = 'pending' AND twin.id <> ${sql.raw(tableQ)}.id
+        LIMIT 1)`
+    const superseded = sql`(pending_key IS NOT NULL AND ${twinId} IS NOT NULL)`
+    const supersededClaim = sql`CASE WHEN ${superseded}
+        THEN jsonb_build_object('cancel', jsonb_build_object('reason', 'superseded', 'message', 'superseded by ' || ${twinId}))
+        ELSE '{}'::jsonb END`
+    return sql`${closeClaim(sql`CASE WHEN ${superseded} THEN 'cancelled' ELSE ${reason}::text END`, progressQ, supersededClaim)},
+              status = CASE WHEN ${superseded} THEN 'cancelled' ELSE 'pending' END,
+              completed_at = CASE WHEN ${superseded} THEN now() ELSE completed_at END,
+              error_payload = CASE WHEN ${superseded} THEN jsonb_build_object('reason', 'superseded', 'by', ${twinId}) ELSE ${errorPayload} END`
+  }
+
+  /** Wakes waitFor callers of a job `returnToPending` cancelled. */
+  async function notifyIfSuperseded(tx: DB, jobId: string, rows: readonly Record<string, unknown>[]): Promise<boolean> {
+    const superseded = rows[0]?.['status'] === 'cancelled'
+    if (superseded) await tx.execute(sql`SELECT pg_notify(${channel}, ${jobId})`)
+    return superseded
+  }
+
+  /**
+   * Counts the attempt as failed: back to pending with backoff while attempts
+   * remain, dead letter (and a notify) after. The row must already be locked by
+   * the caller's transaction. Shared by fail and reap so the two cannot drift.
+   */
+  async function retryOrDeadLetter(
+    tx: DB, job: JobRow, errorJson: string, reason: 'failed' | 'reaped-stale', permanent = false,
+  ): Promise<'retry' | 'dead_letter' | 'superseded'> {
+    const { jobId, attemptCount, maxAttempts: jobMaxAttempts } = job
+    const nextAttempt = attemptCount + 1
+
+    if (!permanent && nextAttempt < jobMaxAttempts) {
+      const delayMs = calculateDelay(attemptCount)
+      const delaySeconds = `${Math.ceil(delayMs / 1000)} seconds`
+      await lockPendingKeyOf(tx, jobId)
+      const rows = await tx.execute(
+        sql`UPDATE ${sql.raw(tableQ)} SET
+              ${returnToPending(reason, sql`${errorJson}::jsonb`)},
+              attempt_count = ${nextAttempt},
+              next_retry_at = ${delayMs > 0 ? sql`now() + ${delaySeconds}::interval` : sql`NULL`},
+              claimed_at = NULL,
+              heartbeat_at = NULL,
+              worker_id = NULL,
+              claim_token = NULL
+            WHERE id = ${jobId}::uuid
+            RETURNING status`
+      )
+      return (await notifyIfSuperseded(tx, jobId, rows)) ? 'superseded' : 'retry'
+    }
+    await tx.execute(
+      sql`UPDATE ${sql.raw(tableQ)} SET
+            ${closeClaim(reason, progressQ)},
+            status = 'dead_letter',
+            attempt_count = ${nextAttempt},
+            completed_at = now(),
+            claim_token = NULL,
+            error_payload = ${errorJson}::jsonb
+          WHERE id = ${jobId}::uuid`
+    )
+    await tx.execute(
+      sql`SELECT pg_notify(${channel}, ${jobId})`
+    )
+    for (const listener of deadLetterListeners) {
+      try {
+        await tx.transaction(async (savepoint) => {
+          await listener(savepoint, {
+            id: jobId, type: job.type, tenant: job.tenant, payload: job.payload, error: JSON.parse(errorJson) as unknown, flow: job.flow,
+          })
+        })
+      } catch (err) {
+        getBaseLogger().error({ err, jobId, jobType: job.type }, 'Dead-letter listener failed')
+        void recordFault('queue-listener', faultMessage(err), { jobId, jobType: job.type })
+      }
+    }
+    return 'dead_letter'
+  }
+
+  async function fail(db: DB, jobId: string, error: unknown, claimToken?: string, opts?: { permanent?: boolean }): Promise<boolean> {
+    crossTenant(db, 'fail')
+    return (await failAttempt(db, jobId, error, claimToken, opts)) !== null
+  }
+
+  async function failAttempt(
+    handle: DB, jobId: string, error: unknown, claimToken?: string, opts?: { permanent?: boolean },
+  ): Promise<FailedAttempt | null> {
+    const db = crossTenant(handle, 'failAttempt')
+    const errorJson = JSON.stringify(serializeError(error))
+
+    return db.transaction(async (tx) => {
+      const jobRows = await tx.execute(
+        sql`SELECT job_type, payload, flow_tenant, attempt_count, max_attempts, flow_trace_id::text, flow_root_id::text,
+                flow_step_id::text, flow_actor FROM ${sql.raw(tableQ)} WHERE id = ${jobId}::uuid${claimScope(claimToken)} FOR UPDATE`
+      )
+      const job = jobRows[0]
+      if (!job) {
+        if (claimToken !== undefined) return null
+        throw new Error(`[pipework] Cannot fail job "${jobId}" — job not found.`)
+      }
+      const outcome = await retryOrDeadLetter(tx, {
+        jobId,
+        type: job['job_type'] as string,
+        payload: job['payload'],
+        tenant: (job['flow_tenant'] as string | null) ?? null,
+        attemptCount: job['attempt_count'] as number,
+        maxAttempts: job['max_attempts'] as number,
+        flow: flowOf(job),
+      }, errorJson, 'failed', opts?.permanent === true)
+      return { outcome, error: JSON.parse(errorJson) as unknown }
+    })
+  }
+
+  async function handBack(handle: DB, jobId: string, claimToken: string): Promise<boolean> {
+    const db = crossTenant(handle, 'handBack')
+    return db.transaction(async (tx) => {
+      await lockPendingKeyOf(tx, jobId)
+      const rows = await tx.execute(
+        sql`UPDATE ${sql.raw(tableQ)} SET
+              ${returnToPending('shutdown-returned', sql`jsonb_build_object('reason', 'handed-back', 'workerId', worker_id)`)},
+              next_retry_at = NULL,
+              claimed_at = NULL,
+              heartbeat_at = NULL,
+              claim_token = NULL,
+              worker_id = NULL
+            WHERE id = ${jobId}::uuid${claimScope(claimToken)}
+            RETURNING status`
+      )
+      await notifyIfSuperseded(tx, jobId, rows)
+      return rows.length > 0
+    })
+  }
+
+  async function later(handle: DB, jobId: string, error: JobLaterError, claimToken: string): Promise<boolean> {
+    const db = crossTenant(handle, 'later')
+    return db.transaction(async (tx) => {
+      const found = await tx.execute(
+        sql`SELECT attempt_count FROM ${sql.raw(tableQ)} WHERE id = ${jobId}::uuid${claimScope(claimToken)} FOR UPDATE`
+      )
+      const row = found[0]
+      if (!row) return false
+      const delayMs = error.delayMs !== undefined
+        ? Math.max(0, error.delayMs)
+        : Math.max(1000, calculateDelay(row['attempt_count'] as number))
+      await lockPendingKeyOf(tx, jobId)
+      const updated = await tx.execute(
+        sql`UPDATE ${sql.raw(tableQ)} SET
+              ${returnToPending('later', sql`jsonb_build_object('reason', 'later', 'message', ${error.message}::text, 'workerId', worker_id)`)},
+              next_retry_at = now() + make_interval(secs => ${delayMs / 1000}::double precision),
+              claimed_at = NULL,
+              heartbeat_at = NULL,
+              claim_token = NULL,
+              worker_id = NULL
+            WHERE id = ${jobId}::uuid
+            RETURNING status`
+      )
+      await notifyIfSuperseded(tx, jobId, updated)
+      return true
+    })
+  }
+
+  async function cancel(handle: DB, jobId: string, why: { reason: string; message: string }): Promise<void> {
+    if (why.reason === '') {
+      throw new Error(
+        `[pipework] cancel() on queue "${table}" received an empty reason.\n\n` +
+        '  The reason is stored in error_payload so a cancelled job says why. Pass a non-empty reason, e.g. { reason: "operator", message: "..." }.\n',
+      )
+    }
+    const { db, tenant } = unscopedExecutor(handle)
+    const cancel = sql`jsonb_build_object('reason', ${why.reason}::text, 'message', ${why.message}::text)`
+    await db.execute(
+      sql`UPDATE ${sql.raw(tableQ)} SET ${closeClaim('cancelled', progressQ, sql`jsonb_build_object('cancel', ${cancel})`)}, status = 'cancelled', completed_at = now(), claim_token = NULL,
+               error_payload = ${cancel}
+               WHERE id = ${jobId}::uuid AND status IN ('pending', 'claimed')${tenantAnd(tenant)}`
+    )
+  }
+
+  async function retryNow(handle: DB, jobId?: string): Promise<number> {
+    const { db, tenant } = unscopedExecutor(handle)
+    const rows = await db.execute(
+      sql`UPDATE ${sql.raw(tableQ)} SET next_retry_at = now()
+           WHERE status = 'pending' AND next_retry_at > now()${jobId === undefined ? sql`` : sql` AND id = ${jobId}::uuid`}${tenantAnd(tenant)}
+           RETURNING id`
+    )
+    return rows.length
+  }
+
+  async function flowOfJob(handle: DB, jobId: string): Promise<ClaimedFlow | null> {
+    const { db, tenant } = unscopedExecutor(handle)
+    const rows = await db.execute(
+      sql`SELECT flow_trace_id::text, flow_root_id::text, flow_step_id::text, flow_tenant, flow_actor
+            FROM ${sql.raw(tableQ)} WHERE id = ${jobId}::uuid${tenantAnd(tenant)}`
+    )
+    return rows.length === 0 ? null : flowOf(rows[0] as Record<string, unknown>)
+  }
+
+  async function totals(handle: DB): Promise<QueueTotals> {
+    const { db, tenant } = unscopedExecutor(handle)
+    const rows = await db.execute(
+      sql`SELECT job_type,
+                 (count(*) FILTER (WHERE status = 'pending' AND (next_retry_at IS NULL OR next_retry_at <= clock_timestamp())))::int AS ready,
+                 (count(*) FILTER (WHERE status IN ('claimed', 'running')))::int AS running,
+                 (extract(epoch FROM clock_timestamp() - min(queued_at) FILTER (WHERE status = 'pending' AND (next_retry_at IS NULL OR next_retry_at <= clock_timestamp()))) * 1000)::bigint AS oldest_ms
+            FROM ${sql.raw(tableQ)}
+           WHERE status IN ('pending', 'claimed', 'running')${tenantAnd(tenant)}
+           GROUP BY GROUPING SETS ((job_type), ())`
+    )
+    let overall: QueueTypeTotals = { ready: 0, running: 0, oldestReadyWaitMs: null }
+    const byType: Record<string, QueueTypeTotals> = {}
+    for (const r of rows) {
+      const t: QueueTypeTotals = {
+        ready: Number(r['ready']),
+        running: Number(r['running']),
+        oldestReadyWaitMs: r['oldest_ms'] === null ? null : Number(r['oldest_ms']),
+      }
+      if (r['job_type'] === null) overall = t
+      else if (t.ready > 0 || t.running > 0) byType[String(r['job_type'])] = t
+    }
+    return { overall, byType }
+  }
+
+  async function reap(handle: DB): Promise<ReapedJob[]> {
+    const db = crossTenant(handle, 'reap')
+    const staleInterval = `${Math.floor(staleTimeoutMs / 1000)} seconds`
+    return db.transaction(async (tx) => {
+      const stale = await tx.execute(
+        sql`SELECT id::text, job_type, payload, attempt_count, max_attempts, worker_id, flow_trace_id::text, flow_root_id::text,
+                   flow_step_id::text, flow_tenant, flow_actor
+              FROM ${sql.raw(tableQ)}
+             WHERE status IN ('claimed', 'running')
+               AND heartbeat_at < now() - ${staleInterval}::interval
+             FOR UPDATE SKIP LOCKED`
+      )
+      const reaped: ReapedJob[] = []
+      for (const row of stale) {
+        const id = row['id'] as string
+        const attemptCount = row['attempt_count'] as number
+        const jobMaxAttempts = row['max_attempts'] as number
+        const errorJson = JSON.stringify({
+          reason: 'reaped',
+          message: 'worker lost: heartbeat timeout',
+          workerId: (row['worker_id'] as string | null) ?? null,
+        })
+        const outcome = await retryOrDeadLetter(tx, {
+          jobId: id,
+          type: row['job_type'] as string,
+          payload: row['payload'],
+          tenant: (row['flow_tenant'] as string | null) ?? null,
+          attemptCount,
+          maxAttempts: jobMaxAttempts,
+          flow: flowOf(row),
+        }, errorJson, 'reaped-stale')
+        reaped.push({
+          jobId: id,
+          type: row['job_type'] as string,
+          attempt: attemptCount + 1,
+          outcome,
+          maxAttempts: jobMaxAttempts,
+          flow: flowOf(row),
+        })
+      }
+      return reaped
+    })
+  }
+
+  async function requeueDeadLetter(handle: DB, jobId: string): Promise<void> {
+    const { db, tenant } = unscopedExecutor(handle)
+    let rows: readonly Record<string, unknown>[]
+    try {
+      rows = await db.execute(
+        sql`UPDATE ${sql.raw(tableQ)} SET
+            status = 'pending', attempt_count = 0, max_attempts = ${maxAttempts},
+            completed_at = NULL, next_retry_at = NULL, claimed_at = NULL,
+            heartbeat_at = NULL, worker_id = NULL, claim_token = NULL
+          WHERE id = ${jobId}::uuid AND status = 'dead_letter'${tenantAnd(tenant)}
+          RETURNING id`
+      )
+    } catch (err) {
+      if (isLiveKeyViolation(err, pendingKeyIndexName(table))) {
+        const twin = await db.execute(
+          sql`SELECT t.pending_key, (SELECT p.id::text FROM ${sql.raw(tableQ)} p
+                 WHERE p.pending_key = t.pending_key AND coalesce(p.flow_tenant, '') = coalesce(t.flow_tenant, '') AND p.status = 'pending' LIMIT 1) AS twin
+                FROM ${sql.raw(tableQ)} t WHERE t.id = ${jobId}::uuid`
+        )
+        throw new Error(
+          `[pipework] Cannot requeue job "${jobId}" — pending job "${String(twin[0]?.['twin'])}" already holds its pendingKey "${String(twin[0]?.['pending_key'])}".\n\n` +
+          '  Requeueing would make a second pending job with the same key. Let the pending job run or cancel it, then requeue.\n',
+          { cause: err },
+        )
+      }
+      if (!isLiveKeyViolation(err, liveKeyIndexName(table))) throw err
+      const held = await db.execute(sql`SELECT live_key FROM ${sql.raw(tableQ)} WHERE id = ${jobId}::uuid`)
+      throw new Error(
+        `[pipework] Cannot requeue job "${jobId}" — a live job already holds its liveKey "${String(held[0]?.['live_key'])}".\n\n` +
+        '  Requeueing would make a second live job with the same key. Let the live job finish or cancel it, then requeue.\n',
+        { cause: err },
+      )
+    }
+    if (rows.length === 0) {
+      throw new Error(`[pipework] Cannot requeue job "${jobId}" — job not found or not in dead_letter status.`)
+    }
+  }
+
+  async function startListener(client: ListenClient): Promise<void> {
+    await listener.start(client)
+  }
+
+  async function stopListener(): Promise<void> {
+    await listener.stop()
+  }
+
+  async function waitFor(handle: DB, jobId: string, opts: WaitOptions): Promise<WaitResult> {
+    const { db, tenant } = unscopedExecutor(handle)
+    return listener.waitFor(db, jobId, opts, tenant)
+  }
+
+  async function enqueueAndWait(db: DB, opts: EnqueueOptions, waitOpts: WaitOptions): Promise<WaitResult> {
+    const jobId = await enqueue(db, opts)
+    return waitFor(db, jobId, waitOpts)
+  }
+
+  const sched = createScheduler({ enqueue, enqueueJob }, tableQ)
+
+  return {
+    definition,
+    door: exclusion?.tenantColumn !== undefined ? queueDoorName(table) : null,
+    onDeadLetter(listener) {
+      deadLetterListeners.add(listener)
+      return () => { deadLetterListeners.delete(listener) }
+    },
+    enqueue, claim, heartbeat, complete, fail, failAttempt, later, handBack, cancel, retryNow, flowOf: flowOfJob, totals, reap, requeueDeadLetter,
+    startListener, stopListener, waitFor, enqueueAndWait,
+    schedule: (db, spec) => sched.schedule(crossTenant(db, 'schedule'), spec),
+    syncSchedules: (db, declared, owned) => sched.sync(crossTenant(db, 'syncSchedules'), declared, owned),
+    fireSchedule: (db, name) => sched.fire(crossTenant(db, 'fireSchedule'), name),
+    runDueSchedules: (db, names) => sched.tick(crossTenant(db, 'runDueSchedules'), names),
+    unschedule: (db, name) => sched.unschedule(crossTenant(db, 'unschedule'), name),
+    listSchedules: (db) => sched.listSchedules(crossTenant(db, 'listSchedules')),
+    startScheduler: (db, opts) => sched.start(crossTenant(db, 'startScheduler'), opts),
+    stopScheduler: () => sched.stop(),
+  }
+}