pipework 0.252.0 → 0.253.0

package/src/async/jobs/run-script.ts

@@ -0,0 +1,67 @@
+import type { Manifold } from '../../core/pipework.js'
+import { runInContext } from '../../request/context/store.js'
+import { createJobContext } from '../../request/context/create.js'
+import { executeInTransaction } from '../../request/context/run-in-transaction.js'
+import { isTracingEnabled } from '../../trace/entry/tracing-enabled.js'
+import { openTrace } from '../../trace/entry/open-trace.js'
+import { parentTraceRef } from '../../trace/entry/parent-trace.js'
+import { closeTrace } from '../../trace/entry/close-trace.js'
+import { stageProcessStart } from '../../trace/entry/process-start.js'
+
+const SCRIPT_TRANSACTION_TIMEOUT_MS = 5000
+
+export interface RunScriptOptions {
+  tenant?: string | null
+  auth?: unknown
+  jobType?: string
+  /** Names the process in the trace: stages the opening `external:pipework.process.start` step. */
+  process?: { phase: 'preStart' | 'script'; surfaces: readonly string[] }
+}
+
+/**
+ * Runs an operator script in the flow a job gets: a trace (entry kind `job`) that
+ * is closed `closed` or `error`, and, with a tenant, the default database's
+ * transaction bound to the flow with the tenant session variables set. The
+ * transaction commits when `fn` returns and rolls back when it throws. Without a
+ * tenant nothing is bound, as for a tenantless job.
+ *
+ * The trace close is awaited, so the row is written before the caller closes the pool.
+ */
+export async function runScript<T>(
+  instance: Manifold,
+  fn: () => T | Promise<T>,
+  options: RunScriptOptions = {},
+): Promise<T> {
+  const tenant = options.tenant ?? null
+  const auth = options.auth ?? null
+  const jobType = options.jobType ?? 'script'
+  const databases = tenant === null ? [] : [instance.config.defaultDatabaseName()]
+  const trace = isTracingEnabled(instance)
+    ? openTrace({
+      instance,
+      entryKind: 'job',
+      parent: parentTraceRef(),
+      tenant,
+      auth,
+      requestId: null,
+      sessionId: null,
+    })
+    : null
+  if (trace !== null && options.process !== undefined) {
+    stageProcessStart(trace, { ...options.process, release: instance.release })
+  }
+
+  const ctx = createJobContext({ instance, jobType, auth, tenant, ...(trace !== null ? { trace } : {}) })
+  const run = async (): Promise<T> => fn()
+
+  try {
+    const result = tenant === null
+      ? await runInContext(ctx, run)
+      : await executeInTransaction(ctx, databases, null, SCRIPT_TRANSACTION_TIMEOUT_MS, run)
+    await closeTrace(instance, trace, 'closed')
+    return result
+  } catch (err) {
+    await closeTrace(instance, trace, 'error')
+    throw err
+  }
+}

package/src/async/jobs/scheduler.ts

@@ -0,0 +1,218 @@
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+import { parseCron, nextTick } from './cron.js'
+import type { EnqueueOptions } from './queue.js'
+import { describeError } from '../../core/logging/error-chain.js'
+
+export interface Enqueuer {
+  enqueue(db: DB, opts: EnqueueOptions): Promise<string>
+  /** `enqueue` that also says whether it inserted a job (false: it returned the live job holding the `liveKey`). */
+  enqueueJob(db: DB, opts: EnqueueOptions): Promise<{ id: string; created: boolean }>
+}
+
+export interface ScheduleSpec {
+  readonly name: string
+  readonly cron: string
+  readonly timezone?: string
+  readonly payload?: unknown
+}
+
+export interface ScheduleInfo {
+  readonly name: string
+  readonly cronExpression: string
+  readonly timezone: string
+  readonly payload: unknown
+  readonly lastRunAt: Date | null
+  readonly nextRunAt: Date
+}
+
+export interface SchedulerOptions {
+  readonly checkIntervalMs?: number
+  /** Fire only these schedules (by name). Omitted: every schedule in the table. */
+  readonly names?: readonly string[]
+  /** Called when a tick throws; without it the error is swallowed so the next tick still runs. */
+  readonly onError?: (err: unknown) => void
+  /** Called after every check, whether it succeeded or threw. */
+  readonly onCheck?: (outcome: { readonly ok: boolean; readonly error?: string }) => void
+}
+
+export interface Scheduler {
+  /** Upserts a schedule. An unchanged cron and timezone keep `next_run_at` and `last_run_at`. */
+  schedule(db: DB, spec: ScheduleSpec): Promise<void>
+  /** Upserts every declared spec and removes schedules named in `owned` that are not declared. Returns the removed names. */
+  sync(db: DB, declared: readonly ScheduleSpec[], owned: readonly string[]): Promise<string[]>
+  /** Enqueues the schedule's job now, as a tick would, without moving `next_run_at`. Returns the job id. */
+  fire(db: DB, name: string): Promise<string>
+  /**
+   * Fires every due schedule once and moves its `next_run_at` to the next future
+   * tick (a schedule missed by many ticks fires once, not once per missed tick).
+   * Each schedule is handled in its own transaction under `FOR UPDATE SKIP LOCKED`,
+   * so across processes a due schedule enqueues exactly one job. Returns the count fired.
+   */
+  tick(db: DB, names?: readonly string[]): Promise<number>
+  unschedule(db: DB, name: string): Promise<boolean>
+  listSchedules(db: DB): Promise<ScheduleInfo[]>
+  start(db: DB, opts?: SchedulerOptions): void
+  stop(): void
+}
+
+export function createScheduler(enqueuer: Enqueuer, jobsTableQ: string): Scheduler {
+  let intervalHandle: ReturnType<typeof setInterval> | null = null
+  let running = false
+
+  async function schedule(db: DB, spec: ScheduleSpec): Promise<void> {
+    const parsed = parseCron(spec.cron)
+    const tz = spec.timezone ?? 'UTC'
+    const nextRunAt = nextTick(parsed, new Date(), tz)
+    const payloadJson = spec.payload !== undefined ? JSON.stringify(spec.payload) : null
+
+    await db.execute(sql`
+      INSERT INTO pipework_schedules (name, cron_expression, timezone, payload, next_run_at)
+      VALUES (${spec.name}, ${spec.cron}, ${tz}, ${payloadJson !== null ? sql`${payloadJson}::jsonb` : sql`NULL`}, ${nextRunAt.toISOString()}::timestamptz)
+      ON CONFLICT (name) DO UPDATE SET
+        next_run_at = CASE
+          WHEN pipework_schedules.cron_expression IS DISTINCT FROM EXCLUDED.cron_expression
+            OR pipework_schedules.timezone IS DISTINCT FROM EXCLUDED.timezone
+          THEN EXCLUDED.next_run_at
+          ELSE pipework_schedules.next_run_at
+        END,
+        cron_expression = EXCLUDED.cron_expression,
+        timezone = EXCLUDED.timezone,
+        payload = EXCLUDED.payload,
+        updated_at = now()
+    `)
+  }
+
+  async function unschedule(db: DB, name: string): Promise<boolean> {
+    const rows = await db.execute(
+      sql`DELETE FROM pipework_schedules WHERE name = ${name} RETURNING name`,
+    )
+    return rows.length > 0
+  }
+
+  async function listSchedules(db: DB): Promise<ScheduleInfo[]> {
+    const rows = await db.execute(
+      sql`SELECT name, cron_expression, timezone, payload, last_run_at, next_run_at
+          FROM pipework_schedules ORDER BY name`,
+    )
+    return rows.map(row => ({
+      name: row['name'] as string,
+      cronExpression: row['cron_expression'] as string,
+      timezone: row['timezone'] as string,
+      payload: row['payload'] ?? null,
+      lastRunAt: row['last_run_at'] !== null && row['last_run_at'] !== undefined ? new Date(row['last_run_at'] as string) : null,
+      nextRunAt: new Date(row['next_run_at'] as string),
+    }))
+  }
+
+  async function sync(db: DB, declared: readonly ScheduleSpec[], owned: readonly string[]): Promise<string[]> {
+    for (const spec of declared) await schedule(db, spec)
+    const keep = new Set(declared.map(spec => spec.name))
+    const removed: string[] = []
+    for (const info of await listSchedules(db)) {
+      if (owned.includes(info.name) && !keep.has(info.name)) {
+        if (await unschedule(db, info.name)) removed.push(info.name)
+      }
+    }
+    return removed
+  }
+
+  async function fire(db: DB, name: string): Promise<string> {
+    const rows = await db.execute(sql`SELECT payload FROM pipework_schedules WHERE name = ${name}`)
+    if (rows.length === 0) {
+      throw new Error(
+        `[pipework] No schedule named "${name}".\n\n` +
+        '  fireSchedule() enqueues the job of a registered schedule; register it with\n' +
+        '  queue.schedule() or declare it on the worker surface.\n',
+      )
+    }
+    return enqueuer.enqueue(db, { type: name, payload: rows[0]!['payload'] ?? {}, continueFlow: false })
+  }
+
+  // One due schedule, locked for the transaction: a second process skips the
+  // locked row, and once the first commits next_run_at is in the future.
+  async function fireOneDue(db: DB, names: readonly string[] | undefined): Promise<'none' | 'skipped' | 'fired'> {
+    return db.transaction(async (tx: DB) => {
+      const filter = names !== undefined
+        ? sql`AND name ${sql.inList(names)}`
+        : sql``
+      const due = await tx.execute(sql`
+        SELECT name, cron_expression, timezone, payload
+        FROM pipework_schedules
+        WHERE next_run_at <= now() ${filter}
+        ORDER BY next_run_at
+        LIMIT 1
+        FOR UPDATE SKIP LOCKED
+      `)
+      if (due.length === 0) return 'none'
+
+      const row = due[0]!
+      const name = row['name'] as string
+      const next = nextTick(parseCron(row['cron_expression'] as string), new Date(), row['timezone'] as string)
+
+      // A job of this type queued by other means (no key) also holds the schedule back. The key then
+      // makes concurrent ticks of this schedule atomic where the check alone is not.
+      const queued = await tx.execute(sql`
+        SELECT 1 FROM ${sql.raw(jobsTableQ)}
+        WHERE job_type = ${name} AND status IN ('pending', 'claimed')
+        LIMIT 1
+      `)
+      const created = queued.length === 0 && (await enqueuer.enqueueJob(tx, {
+        type: name, payload: row['payload'] ?? {}, continueFlow: false, liveKey: `schedule:${name}`,
+      })).created
+      if (!created) {
+        await tx.execute(sql`
+          UPDATE pipework_schedules SET next_run_at = ${next.toISOString()}::timestamptz, updated_at = now()
+          WHERE name = ${name}
+        `)
+        return 'skipped'
+      }
+
+      await tx.execute(sql`
+        UPDATE pipework_schedules SET
+          last_run_at = now(),
+          next_run_at = ${next.toISOString()}::timestamptz,
+          updated_at = now()
+        WHERE name = ${name}
+      `)
+      return 'fired'
+    })
+  }
+
+  async function tick(db: DB, names?: readonly string[]): Promise<number> {
+    let fired = 0
+    // Every pass moves one schedule's next_run_at into the future, so the loop ends.
+    for (;;) {
+      const outcome = await fireOneDue(db, names)
+      if (outcome === 'none') return fired
+      if (outcome === 'fired') fired++
+    }
+  }
+
+  function start(db: DB, opts?: SchedulerOptions): void {
+    if (running) return
+    running = true
+    const intervalMs = opts?.checkIntervalMs ?? 60_000
+    let inTick = false
+    intervalHandle = setInterval(() => {
+      if (inTick) return
+      inTick = true
+      tick(db, opts?.names).then(
+        () => { opts?.onCheck?.({ ok: true }) },
+        (err: unknown) => {
+          opts?.onError?.(err)
+          opts?.onCheck?.({ ok: false, error: describeError(err).message })
+        },
+      ).finally(() => { inTick = false })
+    }, intervalMs)
+  }
+
+  function stop(): void {
+    if (!running || intervalHandle === null) return
+    clearInterval(intervalHandle)
+    intervalHandle = null
+    running = false
+  }
+
+  return { schedule, sync, fire, tick, unschedule, listSchedules, start, stop }
+}

package/src/async/jobs/schedules-table.ts

@@ -0,0 +1,37 @@
+// The schedules table as a pipe.define() definition. The table used to be a
+// hand-written schedules.schema.sql that `pipework generate` could not emit.
+// A worker surface that declares schedules registers the definition at
+// construction; the migration generator merges it through internalDefinitions.
+// Its CREATE statements are IF NOT EXISTS (SCHEDULE_TABLE_NAMES in generate.ts),
+// so apps that applied the old .sql by hand migrate cleanly.
+
+import { define } from '../../data/domain/index.js'
+import { field } from '../../data/domain/field.js'
+import type { DefinedTable, FieldDescriptor } from '../../data/domain/types.js'
+import { schema } from '../../data/schema/namespace.js'
+
+export const SCHEDULE_TABLE_NAMES: ReadonlySet<string> = new Set(['pipework_schedules'])
+
+const scheduleFields: Record<string, FieldDescriptor> = {
+  name: field.text().primaryKey(),
+  cronExpression: field.text(),
+  timezone: field.text().default('UTC'),
+  payload: field.jsonb(schema.check.json()).nullable(),
+  lastRunAt: field.timestamp().nullable(),
+  nextRunAt: field.timestamp(),
+  createdAt: field.timestamp().default('now'),
+  updatedAt: field.timestamp().default('now'),
+}
+
+export const schedules: DefinedTable = define('pipework_schedules', scheduleFields)
+
+let registered = false
+
+/** Marks the schedules table as in use, so `pipework generate` emits it. */
+export function registerSchedulesDefinition(): void {
+  registered = true
+}
+
+export function schedulesDefinitions(): ReadonlyMap<string, DefinedTable> {
+  return registered ? new Map([['pipework_schedules', schedules]]) : new Map()
+}

package/src/async/jobs/serialize-error.ts

@@ -0,0 +1,48 @@
+/** Serialize an arbitrary thrown value for durable storage in a job's `error_payload`.
+ *
+ * A bare `{ message, stack }` capture is a diagnostics trap: wrapper errors hide the
+ * root cause. Drizzle throws `DrizzleQueryError` whose `.message` is only the failed
+ * SQL text; the real `PostgresError` — carrying `code`/`detail`/`constraint`/`table`
+ * — lives on `.cause`. This walks the full `cause` chain and copies each error's
+ * own-enumerable properties (where both Drizzle's `query` and Postgres' fields live),
+ * so a dead-lettered job records *why* it failed, not just *which query* failed.
+ *
+ * Parameters are never stored (`params`, `parameters`, `args`, `types`); `query` is cut
+ * to 2,000 characters and any other string to 4,000, ending `…[truncated N chars]`.
+ * Non-Error values pass through unchanged. Circular `cause` references are broken. */
+export function serializeError(error: unknown): unknown {
+  return serialize(error, new Set<Error>())
+}
+
+const DROPPED_KEYS = new Set(['params', 'parameters', 'args', 'types'])
+const MAX_QUERY_CHARS = 2_000
+const MAX_STRING_CHARS = 4_000
+
+function cut(text: string, max: number): string {
+  return text.length <= max ? text : `${text.slice(0, max)}…[truncated ${String(text.length - max)} chars]`
+}
+
+function serialize(value: unknown, seen: Set<Error>): unknown {
+  if (!(value instanceof Error)) return value
+  if (seen.has(value)) return { message: '[circular error reference]' }
+  seen.add(value)
+
+  // Drizzle's message (and so its stack) is `Failed query: <sql>\nparams: <values>`: keep the text before the values.
+  const paramsAt = Object.hasOwn(value, 'params') ? value.message.indexOf('\nparams: ') : -1
+  const message = paramsAt === -1 ? value.message : value.message.slice(0, paramsAt)
+  const stack = typeof value.stack === 'string' && paramsAt !== -1 ? value.stack.replace(value.message, message) : value.stack
+  const out: Record<string, unknown> = {
+    name: value.name,
+    message: cut(message, MAX_STRING_CHARS),
+    stack: typeof stack === 'string' ? cut(stack, MAX_STRING_CHARS) : stack,
+  }
+  for (const key of Object.keys(value)) {
+    if (key === 'name' || key === 'message' || key === 'stack' || key === 'cause' || DROPPED_KEYS.has(key)) continue
+    const v = (value as unknown as Record<string, unknown>)[key]
+    out[key] = typeof v === 'string' ? cut(v, key === 'query' ? MAX_QUERY_CHARS : MAX_STRING_CHARS) : v
+  }
+  if (value.cause !== undefined) {
+    out['cause'] = serialize(value.cause, seen)
+  }
+  return out
+}

package/src/async/jobs/table.ts

@@ -0,0 +1,191 @@
+// The queue table as a pipe.define() definition, built from queue config.
+//
+// The table shape used to live in a hand-written schema.sql that no TS code
+// referenced, so `pipework generate` could not emit it and a regenerated
+// baseline silently dropped the queue migration. Expressing it as a
+// DefinedTable puts it on the same generate path as every other table:
+// createQueue() registers the definition here, and the migration generator
+// merges this registry through its internalDefinitions hook.
+
+import { define } from '../../data/domain/index.js'
+import { field } from '../../data/domain/field.js'
+import type { DefinedTable, FieldDescriptor } from '../../data/domain/types.js'
+import { schema } from '../../data/schema/namespace.js'
+import { index, uniqueIndex } from '../../data/db/idx/index.js'
+import { quoteIdentifier } from '../../data/db/identifiers.js'
+import { sql } from '../../data/db/sql.js'
+import type { ResolvedExclusion } from './exclusion.js'
+
+export const QUEUE_STATUSES = ['pending', 'claimed', 'running', 'completed', 'failed', 'cancelled', 'dead_letter'] as const
+
+/** Column names every queue table owns — exclusion columns must not collide with these. */
+export const QUEUE_BASE_COLUMNS: ReadonlySet<string> = new Set([
+  'id', 'job_type', 'priority', 'status', 'payload', 'output', 'error_payload',
+  'created_lsn', 'queued_at', 'claimed_at', 'heartbeat_at', 'completed_at',
+  'worker_id', 'worker_release', 'claims', 'attempt_count', 'max_attempts', 'next_retry_at', 'claim_token',
+  'flow_trace_id', 'flow_root_id', 'flow_step_id', 'flow_tenant', 'flow_actor', 'live_key', 'pending_key',
+])
+
+/** The live-key index's tenant expression and predicate; enqueue's ON CONFLICT must repeat them exactly. */
+export const LIVE_KEY_TENANT_EXPR = `coalesce(flow_tenant, '')`
+export const LIVE_KEY_PREDICATE = `live_key IS NOT NULL AND status IN ('pending', 'claimed', 'running')`
+export function liveKeyIndexName(tableName: string): string {
+  return `uq_${tableName}_live_key`
+}
+
+/** The pending-key index's tenant expression and predicate; enqueue's ON CONFLICT must repeat them exactly. */
+export const PENDING_KEY_TENANT_EXPR = LIVE_KEY_TENANT_EXPR
+export const PENDING_KEY_PREDICATE = `pending_key IS NOT NULL AND status = 'pending'`
+export function pendingKeyIndexName(tableName: string): string {
+  return `uq_${tableName}_pending_key`
+}
+
+function queueFields(exclusion: ResolvedExclusion | undefined): Record<string, FieldDescriptor> {
+  const fields: Record<string, FieldDescriptor> = {
+    id: field.uuid().primaryKey().defaultRandom(),
+    jobType: field.text(),
+    priority: field.integer().default(0),
+    status: field.enum(QUEUE_STATUSES).default('pending'),
+    payload: field.jsonb(schema.check.json()),
+    output: field.jsonb(schema.check.json()).nullable(),
+    errorPayload: field.jsonb(schema.check.json()).nullable(),
+    createdLsn: field.pgLsn().default('currentWal'),
+    queuedAt: field.timestamp().default('now'),
+    claimedAt: field.timestamp().nullable(),
+    heartbeatAt: field.timestamp().nullable(),
+    completedAt: field.timestamp().nullable(),
+    workerId: field.text().nullable(),
+    // The release of the worker that last claimed the job. Unlike worker_id it
+    // survives completion, failure, hand-back and later.
+    workerRelease: field.text().nullable(),
+    // Append-only history of the job's claims: claim()'s UPDATE appends one
+    // element, every statement that ends a claim closes the last one.
+    claims: field.jsonb(schema.check.json()).default([]),
+    attemptCount: field.integer().default(0),
+    maxAttempts: field.integer().default(1),
+    nextRetryAt: field.timestamp().nullable(),
+    // Minted only inside claim()'s own UPDATE and nulled by every release, so
+    // a consumer can fence a claimant's writes on an exact uuid comparison.
+    claimToken: field.uuid().nullable(),
+    // The flow that enqueued the job, restored when a worker runs it: the trace
+    // (and open step) it continues, the tenant it runs under, and optionally
+    // the actor. All null for a job enqueued outside any Flow.
+    flowTraceId: field.uuid().nullable(),
+    flowRootId: field.uuid().nullable(),
+    flowStepId: field.uuid().nullable(),
+    flowTenant: field.text().nullable(),
+    flowActor: field.text().nullable(),
+    // enqueue's liveKey: at most one live job per (key, tenant), enforced by the partial unique index.
+    liveKey: field.text().nullable(),
+    // enqueue's pendingKey: at most one pending job per (key, tenant), enforced by the partial unique index.
+    pendingKey: field.text().nullable(),
+  }
+
+  if (exclusion?.tenantColumn !== undefined) {
+    fields[exclusion.tenantColumn] = field.uuid().tenant()
+  }
+  if (exclusion?.arrayColumn !== undefined) {
+    const column = exclusion.arrayElementType === 'text'
+      ? schema.col.text(exclusion.arrayColumn).array()
+      : schema.col.uuid(exclusion.arrayColumn).array()
+    fields[exclusion.arrayColumn] = field.custom(column, schema.check.array(schema.check.string())).nullable()
+  }
+  if (exclusion?.blocksAllColumn !== undefined) {
+    fields[exclusion.blocksAllColumn] = field.boolean().default(false)
+  }
+
+  return fields
+}
+
+interface QueueTableEntry {
+  readonly definition: DefinedTable
+  readonly configKey: string
+}
+
+const queueTables = new Map<string, QueueTableEntry>()
+const progressColumns = new Map<string, string>()
+
+/** Records (or clears) the queue's configured progress column so startup validation can check the table has it. */
+export function registerProgressColumn(tableName: string, column: string | undefined): void {
+  if (column === undefined) progressColumns.delete(tableName)
+  else progressColumns.set(tableName, column)
+}
+
+/** Progress columns configured per queue table. */
+export function queueProgressColumns(): ReadonlyMap<string, string> {
+  return progressColumns
+}
+
+/**
+ * Builds (or reuses) the DefinedTable for a queue. Cached per table name so
+ * repeated createQueue() calls share one definition; the same name with a
+ * different shape is a config error, not a silent overwrite.
+ */
+export function queueDefinition(tableName: string, exclusion: ResolvedExclusion | undefined): DefinedTable {
+  const configKey = JSON.stringify(exclusion ?? null)
+
+  const existing = queueTables.get(tableName)
+  if (existing !== undefined) {
+    if (existing.configKey !== configKey) {
+      throw new Error(
+        `[pipework] Queue table "${tableName}" is already defined with a different exclusion config.\n\n` +
+        `  Existing: ${existing.configKey}\n` +
+        `  New:      ${configKey}\n` +
+        '  One table, one shape — give the second queue its own table name.\n',
+      )
+    }
+    return existing.definition
+  }
+
+  const definition = define(tableName, queueFields(exclusion), {
+    indexes: (t) => ({
+      claimable: index(`idx_${tableName}_claimable`)
+        .on(t['priority'].desc(), t['queuedAt'].asc())
+        .where(sql.raw(`status = 'pending'`)),
+      liveKey: uniqueIndex(liveKeyIndexName(tableName))
+        .on(t['liveKey'], sql.raw(LIVE_KEY_TENANT_EXPR))
+        .where(sql.raw(LIVE_KEY_PREDICATE)),
+      pendingKey: uniqueIndex(pendingKeyIndexName(tableName))
+        .on(t['pendingKey'], sql.raw(PENDING_KEY_TENANT_EXPR))
+        .where(sql.raw(PENDING_KEY_PREDICATE)),
+      stale: index(`idx_${tableName}_stale`)
+        .on(t['heartbeatAt'])
+        .where(sql.raw(`status IN ('claimed', 'running')`)),
+      retryable: index(`idx_${tableName}_retryable`)
+        .on(t['nextRetryAt'])
+        .where(sql.raw(`status = 'pending' AND next_retry_at IS NOT NULL`)),
+      // The claim query's exclusion checks run under SERIALIZABLE; without a
+      // matching index the overlap NOT EXISTS scans the whole claimed/running
+      // set and takes coarse predicate locks, so claimers with disjoint arrays
+      // SSI-conflict and starve. Partial GIN keeps the locks at the key level;
+      // fastupdate must be off — a GIN pending list forces relation-level
+      // predicate locks, voiding the granularity.
+      ...(exclusion?.arrayColumn !== undefined ? {
+        exclusionArray: index(`idx_${tableName}_excl_arr`)
+          .using('gin', t[exclusion.arrayColumn]!)
+          .with({ fastupdate: 'off' })
+          .where(sql.raw(`status IN ('claimed', 'running')`)),
+      } : {}),
+      // Same starvation mechanism for the blocks-all NOT EXISTS: without its
+      // own index that check scans every claimed/running row. This partial
+      // index contains only active blocks-all jobs (normally none), so
+      // ordinary claims neither read beyond it nor insert into it — only a
+      // genuine blocks-all claim conflicts, which is the intended semantics.
+      ...(exclusion?.blocksAllColumn !== undefined ? {
+        exclusionBlocks: index(`idx_${tableName}_excl_blocks`)
+          .on(t[exclusion.tenantColumn ?? exclusion.blocksAllColumn]!)
+          .where(sql.raw(
+            `status IN ('claimed', 'running') AND ${quoteIdentifier(exclusion.blocksAllColumn, 'exclusion column')} = true`,
+          )),
+      } : {}),
+    }),
+  })
+
+  queueTables.set(tableName, { definition, configKey })
+  return definition
+}
+
+/** Registered queue definitions — merged into `pipework generate` via internalDefinitions. */
+export function queueDefinitions(): ReadonlyMap<string, DefinedTable> {
+  return new Map([...queueTables].map(([name, entry]) => [name, entry.definition]))
+}

package/src/async/pipeline/define.ts

@@ -0,0 +1,53 @@
+import type { DB } from '../../data/db/index.js'
+import { sql } from '../../data/db/sql.js'
+import type { PipelineConfig, Pipeline, PipelineRun, PipelineExecuteContext } from './types.js'
+import { executePipeline } from './execute.js'
+import { resumePipeline } from './resume.js'
+import { refuseScopedHandle } from '../../auth/tenant/scoped-db.js'
+
+export function definePipeline(config: PipelineConfig): Pipeline {
+  if (config.steps.length === 0) {
+    throw new Error(`[pipework] Pipeline "${config.name}" must have at least one step.`)
+  }
+
+  const stepNames = new Set<string>()
+  for (const step of config.steps) {
+    if (stepNames.has(step.name)) {
+      throw new Error(`[pipework] Pipeline "${config.name}" has duplicate step name "${step.name}".`)
+    }
+    stepNames.add(step.name)
+  }
+
+  return refuseScopedHandle({
+    async execute(db: DB, input: unknown, context: PipelineExecuteContext): Promise<PipelineRun> {
+      return executePipeline(db, config, input, context)
+    },
+
+    async resume(db: DB, runId: string, stepOutput: unknown): Promise<PipelineRun> {
+      return resumePipeline(db, config, runId, stepOutput)
+    },
+
+    async getStatus(db: DB, runId: string): Promise<PipelineRun | null> {
+      const [row] = await db.execute(sql`
+        SELECT id, pipeline_name, status, current_step, input, started_at, completed_at, error
+        FROM pipework_pipeline_runs
+        WHERE id = ${runId}::uuid
+      `)
+
+      if (row === undefined) return null
+
+      return {
+        id: String(row['id']),
+        pipelineName: String(row['pipeline_name']),
+        status: String(row['status']) as PipelineRun['status'],
+        currentStep: Number(row['current_step']),
+        input: row['input'],
+        startedAt: new Date(String(row['started_at'])),
+        completedAt: row['completed_at'] !== null ? new Date(String(row['completed_at'])) : null,
+        error: row['error'] ?? null,
+      }
+    },
+
+    definition: config,
+  }, `pipeline "${config.name}"`, 'its step handlers receive the connection it runs on and would bypass the tenant filter')
+}

package/src/async/pipeline/errors.ts

@@ -0,0 +1,35 @@
+import { PipeworkError } from '../../core/errors/base.js'
+
+export class PipelineStepError extends PipeworkError {
+  readonly code = 'PIPELINE_STEP_FAILED' as const
+  readonly httpStatus = 500 as const
+  readonly pipelineName: string
+  readonly stepName: string
+  readonly stepIndex: number
+
+  constructor(pipelineName: string, stepName: string, stepIndex: number, cause: unknown) {
+    const causeMsg = cause instanceof Error ? cause.message : String(cause)
+    super(
+      `Pipeline "${pipelineName}" failed at step ${stepIndex} ("${stepName}"): ${causeMsg}. ` +
+      `Prior steps may have committed. Check pipeline run status for details.`,
+    )
+    this.pipelineName = pipelineName
+    this.stepName = stepName
+    this.stepIndex = stepIndex
+  }
+}
+
+export class PipelineResumeError extends PipeworkError {
+  readonly code = 'PIPELINE_RESUME_FAILED' as const
+  readonly httpStatus = 409 as const
+  readonly runId: string
+  readonly reason: string
+
+  constructor(runId: string, reason: string) {
+    super(
+      `Cannot resume pipeline run "${runId}": ${reason}`,
+    )
+    this.runId = runId
+    this.reason = reason
+  }
+}