pipework 0.252.0 → 0.252.1

package/src/infra/join-requests/handle.ts

@@ -0,0 +1,312 @@
+import { sql } from '../../data/db/sql.js'
+import { pipe, pipeRaw, pipeSystem } from '../../data/db/db.js'
+import type { DB } from '../../data/db/index.js'
+import { requireContext } from '../../request/context/store.js'
+import type { Flow } from '../../request/context/types.js'
+import { writeInfraAudit } from '../../data/write/audit/infra-audit.js'
+import { ValidationError, ForbiddenError } from '../../core/errors/index.js'
+import { inTenant as runAsTenant } from '../../auth/tenant/in-tenant.js'
+import { JOIN_REQUEST_STATUSES } from './schema.js'
+
+export type JoinRequestStatus = typeof JOIN_REQUEST_STATUSES[number]
+
+export interface JoinRequest {
+  readonly id: string
+  readonly orgId: string
+  readonly userId: string
+  readonly status: JoinRequestStatus
+  readonly reason: string | null
+  /** App data given to `request`; may be personal data, so the audit record leaves it out. */
+  readonly evidence: unknown
+  readonly requestedAt: Date
+  readonly expiresAt: Date
+  readonly settledAt: Date | null
+  /** Who settled it; `null` while open, and for an expiry. */
+  readonly settledBy: string | null
+  /** The role given on approval. */
+  readonly role: string | null
+  /** The note given on refusal. */
+  readonly note: string | null
+}
+
+export interface JoinRequestsConfig {
+  /** The app's owner check; roles are the app's. Called inside the org's transaction. */
+  isOwner(tx: DB, args: { orgId: string; userId: string }): Promise<boolean>
+  /** Adds the member; called inside the approve transaction, so a throw rolls the approval back. */
+  addMember(tx: DB, args: { orgId: string; userId: string; role: string | undefined; requestId: string; by: string }): Promise<void>
+  /** How long a request stays open. Default 30 days. */
+  readonly expiresInMs?: number
+  /** Awaited inside the transaction that created the request; a throw rolls it back. */
+  onRequested?(tx: DB, request: JoinRequest): void | Promise<void>
+  /** Awaited inside the transaction that settled the request (approved, refused, withdrawn, expired); a throw rolls it back. */
+  onSettled?(tx: DB, request: JoinRequest): void | Promise<void>
+}
+
+export type SettleResult =
+  | { readonly ok: true; readonly request: JoinRequest }
+  | { readonly ok: false; readonly reason: 'not-found' | 'expired' | 'already-settled' }
+
+export interface JoinRequests {
+  /**
+   * Asks to join `orgId` as `userId`. Runs in the named org (the joiner is not in it). One open request
+   * per (org, user): while one is open and unexpired it is returned with `created: false`, with no audit
+   * record and no hook. An open one past its expiry is marked expired first. Writes a `JoinRequest:request`
+   * audit record when the Flow has a trace with an open step.
+   */
+  request(orgId: string, userId: string, opts?: { reason?: string; evidence?: unknown }): Promise<{ request: JoinRequest; created: boolean }>
+  /** The Flow tenant's owner (`isOwner`) approves: marks it approved and calls `addMember`, in one transaction. A non-owner gets `ForbiddenError`. */
+  approve(requestId: string, opts: { role?: string }, by: string): Promise<SettleResult>
+  /** The Flow tenant's owner refuses it. A non-owner gets `ForbiddenError`. */
+  refuse(requestId: string, opts: { note?: string }, by: string): Promise<SettleResult>
+  /** The joiner withdraws their own open request; another user's id is `not-found`. */
+  withdraw(requestId: string, userId: string): Promise<SettleResult>
+  /** The Flow tenant's requests with `status` (default `open`), newest first. */
+  list(opts?: { status?: JoinRequestStatus }): Promise<JoinRequest[]>
+  /** One user's requests across orgs, read through the system channel, newest first. */
+  mine(userId: string): Promise<JoinRequest[]>
+  /** Marks every open request past its expiry as expired; schedule it (a cron job). */
+  sweep(): Promise<{ expired: number }>
+}
+
+const DEFAULT_EXPIRES_IN_MS = 30 * 24 * 60 * 60 * 1000
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
+const MAX_INSERT_ATTEMPTS = 3
+
+type Row = Record<string, unknown>
+
+function toDate(value: unknown): Date {
+  return value instanceof Date ? value : new Date(String(value))
+}
+
+function toDateOrNull(value: unknown): Date | null {
+  return value === null || value === undefined ? null : toDate(value)
+}
+
+function rowToRequest(row: Row): JoinRequest {
+  return {
+    id: row['id'] as string,
+    orgId: row['tenant_id'] as string,
+    userId: row['user_id'] as string,
+    status: row['status'] as JoinRequestStatus,
+    reason: (row['reason'] as string | null) ?? null,
+    evidence: row['evidence'] ?? null,
+    requestedAt: toDate(row['requested_at']),
+    expiresAt: toDate(row['expires_at']),
+    settledAt: toDateOrNull(row['settled_at']),
+    settledBy: (row['settled_by'] as string | null) ?? null,
+    role: (row['role'] as string | null) ?? null,
+    note: (row['note'] as string | null) ?? null,
+  }
+}
+
+function isUniqueViolation(err: unknown): boolean {
+  let current: unknown = err
+  for (let depth = 0; depth < 4 && typeof current === 'object' && current !== null; depth++) {
+    if ((current as { code?: unknown }).code === '23505') return true
+    current = (current as { cause?: unknown }).cause
+  }
+  return false
+}
+
+function requireUuid(value: unknown, name: string): string {
+  if (typeof value !== 'string' || !UUID.test(value)) {
+    throw new ValidationError({ [name]: ['must be a uuid'] }, `joinRequests: ${name} must be a uuid, got ${JSON.stringify(value)}.`)
+  }
+  return value
+}
+
+function requireText(value: unknown, name: string): string {
+  if (typeof value !== 'string' || value === '') {
+    throw new ValidationError({ [name]: ['must be a non-empty string'] }, `joinRequests: ${name} must be a non-empty string.`)
+  }
+  return value
+}
+
+// The audit record leaves out `evidence`, which may be personal data.
+function auditProjection(request: JoinRequest): Record<string, unknown> {
+  const { id, orgId, userId, status, reason, requestedAt, expiresAt, settledAt, settledBy, role, note } = request
+  return { id, orgId, userId, status, reason, requestedAt, expiresAt, settledAt, settledBy, role, note }
+}
+
+export function createJoinRequests(config: JoinRequestsConfig): JoinRequests {
+  if (typeof config.isOwner !== 'function' || typeof config.addMember !== 'function') {
+    throw new Error('[pipework] joinRequests.create needs `isOwner` and `addMember`.\n\n  Roles are the app\'s: isOwner(tx, { orgId, userId }) says who may settle a request, and\n  addMember(tx, { orgId, userId, role, requestId, by }) adds the member inside the approval.\n')
+  }
+  const expiresInMs = config.expiresInMs ?? DEFAULT_EXPIRES_IN_MS
+  if (!Number.isFinite(expiresInMs) || expiresInMs <= 0) {
+    throw new Error('[pipework] joinRequests expiresInMs must be a positive number of milliseconds.\n')
+  }
+
+  function requireTenant(): { flow: Flow; tenantId: string } {
+    const flow = requireContext()
+    if (flow.tenant === null) {
+      throw new Error(
+        '[pipework] This joinRequests method needs a tenant in the Flow.\n\n' +
+        '  approve/refuse/list act for one org. Call them from an authenticated tenant route;\n' +
+        '  request/withdraw/mine are the ones for the joiner, who is not in the org.\n',
+      )
+    }
+    return { flow, tenantId: flow.tenant }
+  }
+
+  function inTenant<T>(flow: Flow, tenantId: string, fn: (tx: DB) => Promise<T>): Promise<T> {
+    return runAsTenant(flow, tenantId, fn)
+  }
+
+  function audit(flow: Flow, tx: DB, verb: string, request: JoinRequest): Promise<void> {
+    return writeInfraAudit(flow, tx, {
+      tenantId: request.orgId, entity: 'JoinRequest', verb, entityId: request.id, after: auditProjection(request),
+    })
+  }
+
+  async function settled(flow: Flow, tx: DB, verb: string, request: JoinRequest): Promise<void> {
+    await audit(flow, tx, verb, request)
+    await config.onSettled?.(tx, request)
+  }
+
+  // Says why an open row was not there to settle.
+  async function whyNot(tx: DB, requestId: string, orgId: string): Promise<'not-found' | 'expired' | 'already-settled'> {
+    const rows = await tx.execute(sql`
+      SELECT status, (expires_at <= now()) AS past FROM join_request WHERE id = ${requestId} AND tenant_id = ${orgId}`)
+    const row = rows[0]
+    if (row === undefined) return 'not-found'
+    if (row['status'] === 'expired' || (row['status'] === 'open' && row['past'] === true)) return 'expired'
+    return 'already-settled'
+  }
+
+  async function settleAsOwner(kind: 'approve' | 'refuse', requestId: string, by: string, opts: { role?: string; note?: string }): Promise<SettleResult> {
+    requireUuid(requestId, 'requestId')
+    requireText(by, 'by')
+    const { flow, tenantId } = requireTenant()
+    return pipe.transaction(async (tx: DB): Promise<SettleResult> => {
+      if (!(await config.isOwner(tx, { orgId: tenantId, userId: by }))) {
+        throw new ForbiddenError(`Only an owner of the organisation can ${kind} a join request.`, { orgId: tenantId, userId: by })
+      }
+      const rows = kind === 'approve'
+        ? await tx.execute(sql`
+            UPDATE join_request SET status = 'approved', settled_at = now(), settled_by = ${by}, role = ${opts.role ?? null}
+            WHERE id = ${requestId} AND tenant_id = ${tenantId} AND status = 'open' AND expires_at > now()
+            RETURNING *`)
+        : await tx.execute(sql`
+            UPDATE join_request SET status = 'refused', settled_at = now(), settled_by = ${by}, note = ${opts.note ?? null}
+            WHERE id = ${requestId} AND tenant_id = ${tenantId} AND status = 'open' AND expires_at > now()
+            RETURNING *`)
+      if (rows.length === 0) return { ok: false, reason: await whyNot(tx, requestId, tenantId) }
+      const request = rowToRequest(rows[0]!)
+      if (kind === 'approve') {
+        await config.addMember(tx, { orgId: tenantId, userId: request.userId, role: opts.role, requestId, by })
+      }
+      await settled(flow, tx, kind, request)
+      return { ok: true, request }
+    })
+  }
+
+  // Marks the open requests of one org that are past expiry (optionally one user's) as expired.
+  async function expireOpen(flow: Flow, tx: DB, orgId: string, userId?: string): Promise<number> {
+    const rows = await tx.execute(sql`
+      UPDATE join_request SET status = 'expired', settled_at = now()
+      WHERE tenant_id = ${orgId} AND status = 'open' AND expires_at <= now()
+        ${userId === undefined ? sql`` : sql`AND user_id = ${userId}`}
+      RETURNING *`)
+    for (const row of rows) await settled(flow, tx, 'expire', rowToRequest(row))
+    return rows.length
+  }
+
+  const handle: JoinRequests = {
+    async request(orgId, userId, opts) {
+      requireUuid(orgId, 'orgId')
+      requireText(userId, 'userId')
+      const flow = requireContext()
+      const evidence = opts?.evidence === undefined || opts.evidence === null ? null : JSON.stringify(opts.evidence)
+      return inTenant(flow, orgId, async (tx) => {
+        for (let attempt = 1; attempt <= MAX_INSERT_ATTEMPTS; attempt++) {
+          const open = await tx.execute(sql`
+            SELECT *, (expires_at <= now()) AS past FROM join_request
+            WHERE tenant_id = ${orgId} AND user_id = ${userId} AND status = 'open'`)
+          if (open.length > 0) {
+            if (open[0]!['past'] !== true) return { request: rowToRequest(open[0]!), created: false }
+            await expireOpen(flow, tx, orgId, userId)
+          }
+          try {
+            // A savepoint: losing the race to another request aborts only this insert.
+            const rows = await tx.transaction((sp: DB) => sp.execute(sql`
+              INSERT INTO join_request (tenant_id, user_id, status, reason, evidence, expires_at)
+              VALUES (${orgId}, ${userId}, 'open', ${opts?.reason ?? null},
+                ${evidence === null ? sql`NULL` : sql`${evidence}::jsonb`},
+                now() + ${expiresInMs}::float8 * interval '1 millisecond')
+              RETURNING *`))
+            const request = rowToRequest(rows[0]!)
+            await audit(flow, tx, 'request', request)
+            await config.onRequested?.(tx, request)
+            return { request, created: true }
+          } catch (err) {
+            if (!isUniqueViolation(err)) throw err
+          }
+        }
+        throw new Error('[pipework] joinRequests.request could not settle on an open request after repeated conflicts; retry.\n')
+      })
+    },
+
+    approve(requestId, opts, by) {
+      return settleAsOwner('approve', requestId, by, { ...(opts.role !== undefined ? { role: opts.role } : {}) })
+    },
+
+    refuse(requestId, opts, by) {
+      return settleAsOwner('refuse', requestId, by, { ...(opts.note !== undefined ? { note: opts.note } : {}) })
+    },
+
+    async withdraw(requestId, userId) {
+      requireUuid(requestId, 'requestId')
+      requireText(userId, 'userId')
+      const flow = requireContext()
+      const found = await pipeSystem().execute(sql`SELECT tenant_id FROM join_request WHERE id = ${requestId}`)
+      if (found.length === 0) return { ok: false, reason: 'not-found' }
+      const orgId = found[0]!['tenant_id'] as string
+      return inTenant(flow, orgId, async (tx): Promise<SettleResult> => {
+        const rows = await tx.execute(sql`
+          UPDATE join_request SET status = 'withdrawn', settled_at = now(), settled_by = ${userId}
+          WHERE id = ${requestId} AND tenant_id = ${orgId} AND user_id = ${userId} AND status = 'open' AND expires_at > now()
+          RETURNING *`)
+        if (rows.length === 0) {
+          const own = await tx.execute(sql`SELECT 1 FROM join_request WHERE id = ${requestId} AND tenant_id = ${orgId} AND user_id = ${userId}`)
+          return { ok: false, reason: own.length === 0 ? 'not-found' : await whyNot(tx, requestId, orgId) }
+        }
+        const request = rowToRequest(rows[0]!)
+        await settled(flow, tx, 'withdraw', request)
+        return { ok: true, request }
+      })
+    },
+
+    async list(opts) {
+      const { tenantId } = requireTenant()
+      const status = opts?.status ?? 'open'
+      if (!(JOIN_REQUEST_STATUSES as readonly string[]).includes(status)) {
+        throw new ValidationError({ status: [`must be one of ${JOIN_REQUEST_STATUSES.join(', ')}`] }, 'joinRequests list status is not valid.')
+      }
+      const rows = await pipeRaw().execute(sql`
+        SELECT * FROM join_request WHERE tenant_id = ${tenantId} AND status = ${status}
+        ORDER BY requested_at DESC, id DESC`)
+      return rows.map(rowToRequest)
+    },
+
+    async mine(userId) {
+      requireText(userId, 'userId')
+      const rows = await pipeSystem().execute(sql`
+        SELECT * FROM join_request WHERE user_id = ${userId} ORDER BY requested_at DESC, id DESC`)
+      return rows.map(rowToRequest)
+    },
+
+    async sweep() {
+      const flow = requireContext()
+      const orgs = await pipeSystem().execute(sql`
+        SELECT DISTINCT tenant_id FROM join_request WHERE status = 'open' AND expires_at <= now()`)
+      let expired = 0
+      for (const org of orgs) {
+        const orgId = org['tenant_id'] as string
+        expired += await inTenant(flow, orgId, tx => expireOpen(flow, tx, orgId))
+      }
+      return { expired }
+    },
+  }
+  return handle
+}

package/src/infra/join-requests/index.ts

@@ -0,0 +1,2 @@
+export { createJoinRequests } from './handle.js'
+export type { JoinRequests, JoinRequestsConfig, JoinRequest, JoinRequestStatus, SettleResult } from './handle.js'

package/src/infra/join-requests/namespace.ts

@@ -0,0 +1,7 @@
+import { createJoinRequests } from './handle.js'
+
+/** Join requests namespace — a user asks to join an existing org and one of its owners settles it. */
+export const joinRequests = {
+  /** Creates a join-request handle: `request`, `approve`, `refuse`, `withdraw`, `list`, `mine` and `sweep`. Needs blueprint `joinRequests: {}`. */
+  create: createJoinRequests,
+} as const

package/src/infra/join-requests/schema.ts

@@ -0,0 +1,33 @@
+// The table behind join requests. `join_request` carries a `.tenant()` key (the
+// org the user asks to join), so generated migrations put it under forced
+// row-level security. One open request per (org, user) is enforced by a
+// partial unique index.
+
+import { define } from '../../data/domain/index.js'
+import { field } from '../../data/domain/field.js'
+import { schema } from '../../data/schema/namespace.js'
+import { sql } from '../../data/db/sql.js'
+import { index, uniqueIndex } from '../../data/db/idx/index.js'
+
+export const JOIN_REQUEST_STATUSES = ['open', 'approved', 'refused', 'withdrawn', 'expired'] as const
+
+export const joinRequest = define('join_request', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  tenantId: field.uuid().tenant(),
+  userId: field.text(),
+  status: field.enum(JOIN_REQUEST_STATUSES),
+  reason: field.text().nullable(),
+  evidence: field.jsonb(schema.check.json()).nullable(),
+  requestedAt: field.timestamp().default('now'),
+  expiresAt: field.timestamp(),
+  settledAt: field.timestamp().nullable(),
+  settledBy: field.text().nullable(),
+  role: field.text().nullable(),
+  note: field.text().nullable(),
+}, {
+  indexes: (t) => ({
+    oneOpen: uniqueIndex('uq_join_request_open').on(t['tenantId'], t['userId']).where(sql.raw(`status = 'open'`)),
+    byUser: index('idx_join_request_user').on(t['userId']),
+    byStatus: index('idx_join_request_status').on(t['tenantId'], t['status']),
+  }),
+})

package/src/infra/outbox/core.ts

@@ -0,0 +1,165 @@
+// The delivery core shared by alerts and support events: rows in an outbox table are leased with
+// SKIP LOCKED, sent in order per (group, sink), retried with backoff and given up after maxAttempts.
+// Table and column names are the caller's own constants, never user input.
+
+import { sql } from '../../data/db/sql.js'
+import { pipeSystem } from '../../data/db/db.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import { describeError } from '../../core/logging/error-chain.js'
+import { ValidationError } from '../../core/errors/index.js'
+
+const SEND_TIMEOUT_MS = 10_000
+const LEASE_MS = 30_000
+const BATCH = 50
+const MAX_BATCHES = 20
+const MAX_BACKOFF_MS = 3_600_000
+
+export interface OutboxTotals {
+  delivered: number
+  retried: number
+  failed: number
+}
+
+/** What a sink receives. `kind` is the payload's own kind; `key` the group the row is ordered in (alert key, ticket id). */
+export interface SinkDelivery<P extends { readonly kind: string }> {
+  readonly id: string
+  readonly kind: P['kind']
+  readonly key: string
+  readonly sink: string
+  readonly payload: P
+  /** 1 on the first try. */
+  readonly attempt: number
+}
+
+/**
+ * A place events are sent. `send` throwing is a failed attempt; so is running past 10 s, which is all the
+ * outbox gives a send (the row is leased for 30 s). `delivery.id` is the same on every attempt, so a receiver can dedupe on it.
+ */
+export interface Sink<P extends { readonly kind: string }> {
+  readonly name: string
+  send(delivery: SinkDelivery<P>): Promise<void>
+}
+
+export interface OutboxSink<D> {
+  readonly name: string
+  send(delivery: D): Promise<void>
+}
+
+export interface OutboxOptions<D> {
+  /** The outbox table: `id`, the group column, `kind`, `sink`, `payload`, `attempts`, `next_attempt_at`, `delivered_at`, `failed_at`, `last_error`, `created_at`. */
+  readonly table: string
+  /** The column holding what a row is ordered within, together with its sink. */
+  readonly groupColumn: string
+  readonly database: string | undefined
+  readonly sinks: ReadonlyMap<string, OutboxSink<D>>
+  readonly toDelivery: (row: Record<string, unknown>, attempt: number) => D
+  readonly maxAttempts: number
+  readonly backoffMs: number
+  /** Names the log line written when a delivery is given up. */
+  readonly label: string
+  readonly onDelivered: (row: Record<string, unknown>, attempt: number) => Promise<void>
+  readonly onGivenUp: (row: Record<string, unknown>, attempt: number, error: string) => Promise<void>
+}
+
+export interface Outbox {
+  /** Delivers what is due now, up to 20 batches; rows that fail wait for the next call. */
+  drain(): Promise<OutboxTotals>
+  /**
+   * Puts given-up rows (`failed_at` set, never delivered) back on the queue: `failed_at` cleared, `attempts` 0, due now.
+   * `last_error` stays until the next attempt. Ids are unchanged. Returns each requeued row as it now is, plus
+   * `prev_attempts`, the attempts it had when given up.
+   */
+  requeue(options: { sink?: string; failedSince?: Date }): Promise<Record<string, unknown>[]>
+}
+
+function withTimeout(promise: Promise<void>, ms: number): Promise<void> {
+  let timer: NodeJS.Timeout | undefined
+  const timeout = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`sink did not answer within ${String(ms)} ms`)), ms)
+  })
+  return Promise.race([promise, timeout]).finally(() => { clearTimeout(timer) })
+}
+
+export function createOutbox<D>(options: OutboxOptions<D>): Outbox {
+  const table = sql.raw(`"${options.table}"`)
+  const group = sql.raw(`"${options.groupColumn}"`)
+  const log = getBaseLogger()
+  const db = () => pipeSystem(options.database)
+
+  // Leases the due rows by pushing next_attempt_at out, so no row lock is held across a send. A row is
+  // due only if no older undelivered row of the same group and sink exists: a later event never overtakes an earlier one.
+  async function claimDue(dueBefore: string): Promise<Record<string, unknown>[]> {
+    return [...await db().execute(sql`
+      UPDATE ${table} SET next_attempt_at = clock_timestamp() + ${LEASE_MS} * interval '1 millisecond'
+      WHERE id IN (
+        SELECT d.id FROM ${table} d
+        WHERE d.delivered_at IS NULL AND d.failed_at IS NULL AND d.next_attempt_at <= ${dueBefore}::timestamptz
+          AND NOT EXISTS (
+            SELECT 1 FROM ${table} o
+            WHERE o.${group} = d.${group} AND o.sink = d.sink AND o.delivered_at IS NULL AND o.failed_at IS NULL
+              AND (o.created_at, o.id) < (d.created_at, d.id))
+        ORDER BY d.next_attempt_at LIMIT ${BATCH} FOR UPDATE OF d SKIP LOCKED)
+      RETURNING *`)]
+  }
+
+  async function deliverBatch(batch: Record<string, unknown>[], totals: OutboxTotals): Promise<void> {
+    await Promise.all(batch.map(async (row) => {
+      const id = row['id'] as string
+      const sinkName = row['sink'] as string
+      const attempt = Number(row['attempts']) + 1
+      const sink = options.sinks.get(sinkName)
+      let error: string | null = null
+      if (sink === undefined) {
+        error = `sink "${sinkName}" is not configured`
+      } else {
+        try {
+          await withTimeout(sink.send(options.toDelivery(row, attempt)), SEND_TIMEOUT_MS)
+        } catch (err) {
+          error = describeError(err).message
+        }
+      }
+      if (error === null) {
+        await db().execute(sql`UPDATE ${table} SET delivered_at = clock_timestamp(), attempts = ${attempt}, last_error = NULL WHERE id = ${id}::uuid`)
+        await options.onDelivered(row, attempt)
+        totals.delivered++
+      } else if (sink === undefined || attempt >= options.maxAttempts) {
+        await db().execute(sql`UPDATE ${table} SET failed_at = clock_timestamp(), attempts = ${attempt}, last_error = ${error} WHERE id = ${id}::uuid`)
+        log.error({ id, sink: sinkName, kind: row['kind'], attempt, error }, options.label)
+        await options.onGivenUp(row, attempt, error)
+        totals.failed++
+      } else {
+        const delay = Math.min(options.backoffMs * 2 ** (attempt - 1), MAX_BACKOFF_MS)
+        await db().execute(sql`UPDATE ${table} SET attempts = ${attempt}, last_error = ${error},
+          next_attempt_at = clock_timestamp() + ${delay} * interval '1 millisecond' WHERE id = ${id}::uuid`)
+        totals.retried++
+      }
+    }))
+  }
+
+  return {
+    async requeue(opts) {
+      if (opts.failedSince !== undefined && !(opts.failedSince instanceof Date && !Number.isNaN(opts.failedSince.getTime()))) {
+        throw new ValidationError({ failedSince: ['must be a valid Date'] }, 'Outbox failedSince must be a Date.')
+      }
+      const sinkFilter = opts.sink === undefined ? sql`` : sql`AND sink = ${opts.sink}`
+      const sinceFilter = opts.failedSince === undefined ? sql`` : sql`AND failed_at >= ${opts.failedSince.toISOString()}::timestamptz`
+      return [...await db().execute(sql`
+        UPDATE ${table} t SET failed_at = NULL, attempts = 0, next_attempt_at = clock_timestamp()
+        FROM (SELECT id, attempts AS prev_attempts FROM ${table}
+          WHERE failed_at IS NOT NULL AND delivered_at IS NULL ${sinkFilter} ${sinceFilter} FOR UPDATE) old
+        WHERE t.id = old.id
+        RETURNING t.*, old.prev_attempts`)]
+    },
+    async drain() {
+      const totals: OutboxTotals = { delivered: 0, retried: 0, failed: 0 }
+      // Rows that fail during this call wait for the next one, however short their backoff.
+      const dueBefore = String((await db().execute(sql`SELECT clock_timestamp()::text AS t`))[0]!['t'])
+      for (let i = 0; i < MAX_BATCHES; i++) {
+        const batch = await claimDue(dueBefore)
+        if (batch.length === 0) break
+        await deliverBatch(batch, totals)
+      }
+      return totals
+    },
+  }
+}

package/src/infra/quota/errors.ts

@@ -0,0 +1,32 @@
+import { PipeworkError } from '../../core/errors/base.js'
+
+/**
+ * Thrown by `quota.call` before sending, when the upstream's recorded budget for `key`
+ * has fallen to the reserve kept for other classes. Inside a job it is handed back like
+ * `jobs.later` until `resetAt`.
+ */
+export class QuotaExhaustedError extends PipeworkError {
+  readonly code = 'QUOTA_EXHAUSTED' as const
+  readonly httpStatus = 429 as const
+  readonly key: string
+  readonly class: string
+  readonly remaining: number
+  readonly reserve: number
+  readonly resetAt: Date
+  /** Whole seconds from now to `resetAt`; sent as Retry-After and in the 429 body. */
+  readonly retryAfterSeconds: number
+
+  constructor(info: { key: string; class: string; remaining: number; reserve: number; resetAt: Date }) {
+    super(
+      `[pipework] Quota for "${info.key}" is exhausted for class "${info.class}": ` +
+      `${String(info.remaining)} remaining, ${String(info.reserve)} held in reserve; resets at ${info.resetAt.toISOString()}.`,
+    )
+    this.name = 'QuotaExhaustedError'
+    this.key = info.key
+    this.class = info.class
+    this.remaining = info.remaining
+    this.reserve = info.reserve
+    this.resetAt = info.resetAt
+    this.retryAfterSeconds = Math.max(0, Math.ceil((info.resetAt.getTime() - Date.now()) / 1000))
+  }
+}