pipework 0.252.0 → 0.252.1

package/src/async/steps/graph.ts

@@ -0,0 +1,1214 @@
+// Step graphs over a jobs Queue. A graph is defined once; each request names the
+// results it wants for an item, and only the steps whose result is not already
+// recorded for their inputs run, each as an ordinary queued job.
+//
+// A result's identity is its content key: sha256 of (graph, step, version, the
+// element, the hash of every need). A changed input changes the keys that depend
+// on it and nothing else, so invalidation needs no bookkeeping.
+//
+// Run state changes only under a row lock on the run (`advance` and every writer
+// take it before touching tasks), so steps finishing together serialize.
+
+import { randomUUID } from 'node:crypto'
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+import { scopedExecutor } from '../../auth/tenant/scoped-db.js'
+import { schema } from '../../data/schema/namespace.js'
+import { fitting } from '../../request/di/index.js'
+import type { Handler } from '../../request/di/types.js'
+import { resolveActorId } from '../../data/write/audit/actor.js'
+import { getContext, runInContext } from '../../request/context/store.js'
+import { createJobContext } from '../../request/context/create.js'
+import { bindTransaction } from '../../request/context/guard.js'
+import { propagateLocals } from '../../request/context/run-in-transaction.js'
+import { isTracingEnabled } from '../../trace/entry/tracing-enabled.js'
+import { openTrace } from '../../trace/entry/open-trace.js'
+import { finalizeTrace } from '../../trace/entry/finalize-trace.js'
+import { __flowStep } from '../../trace/runtime/flow-step.js'
+import { parentTraceRef } from '../../trace/entry/parent-trace.js'
+import type { ParentTraceRef } from '../../trace/entry/open-trace.js'
+import { serializeForTrace } from '../../trace/capture/bounded-serializer.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import { recordFaultFor, faultMessage } from '../../infra/alerts/faults.js'
+import type { DeadLetterInfo } from '../jobs/queue.js'
+import type { Manifold } from '../../core/pipework.js'
+import { projectStepError } from '../../trace/runtime/step-error.js'
+import { StepItemRunningError, StepTimeoutError } from './errors.js'
+import { canonicalJson, inputHash, sha256Hex, valueHash } from './hash.js'
+import { SINGLE, registerStepTables } from './table.js'
+import type {
+  DecideOptions, EndedRun, EndedRunsOptions, GraphConfig, GraphFailure, GraphSelection, ScopeOptions, GraphForEachContext, GraphHost, GraphRunStatus, GraphStepContext, GraphStepDefinition,
+  PlanOptions, PlanResult, RemoveItemOptions, RemoveItemResult, RequestOptions, RequestResult, RunStatus, RunTask, StepCounts, StepGraph, StepPlan, StepResultRecord, TaskState,
+} from './types.js'
+
+const RUN = 'pipework_step_run'
+const TASK = 'pipework_step_task'
+const RESULT = 'pipework_step_result'
+const NAME_RE = /^[A-Za-z][A-Za-z0-9_.-]*$/
+const CLASS_RE = /^[A-Za-z][A-Za-z0-9_-]*$/
+const DEFAULT_CLASS = 'default'
+const MS_PER_DAY = 86_400_000
+
+type TaskStatus = 'blocked' | 'queued' | 'waiting' | 'done' | 'reused' | 'failed' | 'cancelled'
+
+interface Task {
+  readonly id: string
+  readonly step: string
+  readonly instance: number
+  status: TaskStatus
+  jobId: string | null
+  inputKey: string | null
+  resultId: string | null
+  readonly element: unknown
+}
+
+interface RunRow {
+  readonly id: string
+  readonly tenant: string
+  readonly item: string
+  readonly status: GraphRunStatus
+  readonly priority: number
+  readonly stepPriority: Readonly<Record<string, number>> | null
+  readonly targets: readonly string[]
+  readonly inputs: Readonly<Record<string, unknown>>
+  readonly parent: ParentTraceRef | null
+  readonly actor: string | null
+}
+
+interface ResultValue { readonly value: unknown; readonly hash: string }
+
+interface Outcome { reused: number; queued: number; completed: boolean }
+
+function fail(graph: string, message: string): never {
+  throw new Error(`[pipework] steps.define("${graph}"): ${message}\n`)
+}
+
+function toJson(value: unknown): unknown {
+  return value === undefined ? null : JSON.parse(JSON.stringify(value))
+}
+
+function stepDefinitionChecks(config: GraphConfig): { order: string[]; inputs: ReadonlySet<string> } {
+  const graph = config.name
+  if (typeof graph !== 'string' || !NAME_RE.test(graph)) {
+    throw new Error(`[pipework] steps.define: graph name "${String(graph)}" must start with a letter and use letters, digits, "_", "." or "-".\n`)
+  }
+  const inputs = new Set(config.inputs ?? [])
+  const names = Object.keys(config.steps)
+  if (names.length === 0) fail(graph, 'no steps.')
+  for (const name of names) {
+    const def = config.steps[name]!
+    if (!NAME_RE.test(name)) fail(graph, `step name "${name}" must start with a letter and use letters, digits, "_", "." or "-".`)
+    if (inputs.has(name)) fail(graph, `"${name}" is both a request input and a step.`)
+    if (!Number.isInteger(def.version)) fail(graph, `step "${name}" needs an integer version.`)
+    for (const need of def.needs ?? []) {
+      if (!inputs.has(need) && config.steps[need] === undefined) {
+        fail(graph, `step "${name}" needs "${need}", which is neither a step nor a declared input (steps: ${names.join(', ')}; inputs: ${[...inputs].join(', ') || '(none)'}).`)
+      }
+    }
+    if (def.workers !== undefined && (typeof def.workers !== 'string' || !CLASS_RE.test(def.workers))) {
+      fail(graph, `step "${name}" names worker class "${String(def.workers)}", which must start with a letter and use letters, digits, "_" or "-".`)
+    }
+    if (def.priority !== undefined && !Number.isInteger(def.priority)) {
+      fail(graph, `step "${name}" has priority ${String(def.priority)}, which must be an integer.`)
+    }
+    if (def.timeoutMs !== undefined) {
+      if (!Number.isInteger(def.timeoutMs) || def.timeoutMs <= 0) fail(graph, `step "${name}" has timeoutMs ${String(def.timeoutMs)}, which must be a positive integer (milliseconds).`)
+      if (def.person === true) fail(graph, `step "${name}" is a person step and cannot have timeoutMs: it holds no worker.`)
+    }
+    if (def.forEachOnly !== undefined) {
+      if (!Array.isArray(def.forEachOnly)) fail(graph, `step "${name}" has forEachOnly ${String(def.forEachOnly)}, which must be an array of need names.`)
+      for (const n of def.forEachOnly) {
+        if (!(def.needs ?? []).includes(n)) fail(graph, `step "${name}" lists "${String(n)}" in forEachOnly, which is not one of its needs (${(def.needs ?? []).join(', ') || '(none)'}).`)
+      }
+      if (def.forEachOnly.length > 0 && def.forEach === undefined) fail(graph, `step "${name}" has forEachOnly "${def.forEachOnly.join('", "')}" but no forEach.`)
+    }
+    if (def.byElement !== undefined) {
+      const by = def.byElement
+      if (!Array.isArray(by) || by.length === 0) fail(graph, `step "${name}" has byElement ${String(by)}, which must be a non-empty array of need names.`)
+      if (def.forEach !== undefined) fail(graph, `step "${name}" has both forEach and byElement; byElement takes its list from the steps it names.`)
+      if (def.forEachOnly !== undefined) fail(graph, `step "${name}" has forEachOnly but no forEach.`)
+      if (def.person === true) fail(graph, `step "${name}" is a person step and cannot have byElement.`)
+      for (const n of by) {
+        if (!(def.needs ?? []).includes(n)) fail(graph, `step "${name}" lists "${String(n)}" in byElement, which is not one of its needs (${(def.needs ?? []).join(', ') || '(none)'}).`)
+        if (inputs.has(n) || config.steps[n] === undefined) fail(graph, `step "${name}" lists "${n}" in byElement, but it is a request input, not a fan-out step.`)
+        if (config.steps[n]!.forEach === undefined && config.steps[n]!.byElement === undefined) {
+          fail(graph, `step "${name}" lists "${n}" in byElement, but "${n}" is not a fan-out step (it has neither forEach nor byElement).`)
+        }
+      }
+    }
+    if (def.person === true) {
+      if (def.forEach !== undefined) fail(graph, `step "${name}" is a person step and cannot have forEach.`)
+      if (def.run !== undefined) fail(graph, `step "${name}" is a person step and cannot have run: its value comes from decide() or autoAccept.`)
+    } else {
+      if (def.run === undefined) fail(graph, `step "${name}" has no run.`)
+      if (def.autoAccept !== undefined) fail(graph, `step "${name}" has autoAccept but is not a person step.`)
+    }
+  }
+  // Topological order; a back edge is a cycle.
+  const order: string[] = []
+  const state = new Map<string, 'visiting' | 'done'>()
+  const visit = (name: string, path: string[]): void => {
+    const seen = state.get(name)
+    if (seen === 'done') return
+    if (seen === 'visiting') fail(graph, `steps form a cycle: ${[...path.slice(path.indexOf(name)), name].join(' -> ')}.`)
+    state.set(name, 'visiting')
+    for (const need of config.steps[name]!.needs ?? []) {
+      if (config.steps[need] !== undefined) visit(need, [...path, name])
+    }
+    state.set(name, 'done')
+    order.push(name)
+  }
+  for (const name of names) visit(name, [])
+  // byElement is a subset of needs, so with no cycle every chain of byElement ends at a forEach step.
+  const rootOf = (name: string): string => {
+    const def = config.steps[name]!
+    return def.forEach !== undefined ? name : rootOf(def.byElement![0]!)
+  }
+  for (const name of names) {
+    const by = config.steps[name]!.byElement
+    if (by === undefined) continue
+    const roots = by.map(rootOf)
+    if (new Set(roots).size > 1) {
+      fail(graph, `step "${name}" lists byElement steps over different lists: ${by.map((n, i) => `"${n}" (list of "${roots[i]!}")`).join(', ')}.`)
+    }
+  }
+  return { order, inputs }
+}
+
+export function defineGraph(config: GraphConfig): StepGraph {
+  const { order, inputs: inputNames } = stepDefinitionChecks(config)
+  registerStepTables()
+
+  const graph = config.name
+  const jobType = `pipework.step.${graph}`
+  const steps = config.steps
+  const classOf = (name: string): string => steps[name]!.workers ?? DEFAULT_CLASS
+  const typeOfClass = (cls: string): string => cls === DEFAULT_CLASS ? jobType : `${jobType}:${cls}`
+  const classes = [...new Set(order.map(classOf))]
+  const jobTypes = classes.map(typeOfClass)
+  const queue = config.queue
+  const isFanOut = (name: string): boolean => steps[name]!.forEach !== undefined || steps[name]!.byElement !== undefined
+  const running = new Map<string, Set<AbortController>>()
+
+  /** Every step the targets need, transitively, in dependency order. */
+  function closureOf(targets: readonly string[]): string[] {
+    const wanted = new Set<string>()
+    const add = (name: string): void => {
+      if (wanted.has(name)) return
+      wanted.add(name)
+      for (const need of steps[name]!.needs ?? []) if (steps[need] !== undefined) add(need)
+    }
+    for (const t of targets) add(t)
+    return order.filter(n => wanted.has(n))
+  }
+
+  /** The steps that depend on `step`, directly or not. */
+  function dependentsOf(step: string): Set<string> {
+    const out = new Set<string>()
+    const grow = (name: string): void => {
+      for (const other of order) {
+        if (!out.has(other) && (steps[other]!.needs ?? []).includes(name)) { out.add(other); grow(other) }
+      }
+    }
+    grow(step)
+    return out
+  }
+
+  function checkRequest(opts: { item: string; targets: readonly string[]; inputs?: Readonly<Record<string, unknown>>; stepPriority?: Readonly<Record<string, number>> }, what: string): string[] {
+    if (typeof opts.item !== 'string' || opts.item === '') throw new Error(`[pipework] ${graph}.${what}: item must be a non-empty string.\n`)
+    if (!Array.isArray(opts.targets) || opts.targets.length === 0) throw new Error(`[pipework] ${graph}.${what}: targets must name at least one step.\n`)
+    for (const t of opts.targets) {
+      if (steps[t] === undefined) {
+        throw new Error(`[pipework] ${graph}.${what}: unknown target step "${t}" (steps: ${order.join(', ')}).\n`)
+      }
+    }
+    for (const name of Object.keys(opts.inputs ?? {})) {
+      if (!inputNames.has(name)) {
+        throw new Error(`[pipework] ${graph}.${what}: unknown input "${name}" (inputs: ${[...inputNames].join(', ') || '(none)'}).\n`)
+      }
+    }
+    if (opts.stepPriority !== undefined) {
+      for (const [name, p] of Object.entries(opts.stepPriority)) {
+        if (steps[name] === undefined) {
+          throw new Error(`[pipework] ${graph}.${what}: stepPriority names unknown step "${name}" (steps: ${order.join(', ')}).\n`)
+        }
+        if (!Number.isInteger(p)) throw new Error(`[pipework] ${graph}.${what}: stepPriority for step "${name}" is ${String(p)}, which must be an integer.\n`)
+      }
+    }
+    const closure = closureOf(opts.targets)
+    for (const name of closure) {
+      for (const need of steps[name]!.needs ?? []) {
+        if (inputNames.has(need) && (opts.inputs ?? {})[need] === undefined) {
+          throw new Error(`[pipework] ${graph}.${what}: step "${name}" needs input "${need}", which the request does not give.\n`)
+        }
+      }
+    }
+    return closure
+  }
+
+  function tenantOf(explicit: string | undefined, what: string): string {
+    const tenant = explicit ?? getContext()?.tenant ?? null
+    if (tenant === null || tenant === '') {
+      throw new Error(`[pipework] ${graph}.${what}: no tenant. Pass { tenant } or call inside a tenant Flow; results are recorded per tenant.\n`)
+    }
+    return tenant
+  }
+
+  /** The tenant of a tenant-scoped handle, which stands in when a call names none. */
+  const handleTenant = (db: DB): string | undefined => scopedExecutor(db, null, graph).tenant ?? undefined
+
+  let attached: Manifold | undefined
+
+  /** The session variable the tenant policies read: the graph's own, else the active or attached Manifold's, else the default. */
+  function sessionVarOf(instance?: Manifold): string {
+    if (config.sessionVar !== undefined) return config.sessionVar
+    const m = instance ?? getContext()?.instance ?? attached
+    if (m === undefined) return 'pipework.tenant_id'
+    return m.config.database(m.config.defaultDatabaseName()).rls?.sessionVar ?? 'pipework.tenant_id'
+  }
+
+  /** Sets the tenant session variable for the rest of the transaction `tx` is in, and puts back what was there. */
+  async function asTenant<T>(tx: DB, tenant: string | null, fn: () => Promise<T>, instance?: Manifold): Promise<T> {
+    if (tenant === null) return fn()
+    const sessionVar = sessionVarOf(instance)
+    const before = await tx.execute(sql`SELECT current_setting(${sessionVar}, true) AS v`)
+    await tx.execute(sql`SELECT set_config(${sessionVar}, ${tenant}, true)`)
+    const result = await fn()
+    await tx.execute(sql`SELECT set_config(${sessionVar}, ${(before[0]?.['v'] as string | null) ?? ''}, true)`)
+    return result
+  }
+
+  async function inTenantTx<T>(handle: DB, scope: string | null, fn: (tx: DB) => Promise<T>): Promise<T> {
+    const { db, tenant } = scopedExecutor(handle, scope, `${graph} step graph`)
+    return db.transaction(async (tx) => asTenant(tx, tenant, () => fn(tx)))
+  }
+
+  /** The tenant a call acts for: the one given, else the Flow's, else none (an owner connection needs none). */
+  const scopeOf = (explicit?: string): string | null => explicit ?? getContext()?.tenant ?? null
+
+  /** Cancels the jobs that are still pending in one statement, skipping any row another transaction holds: never waits on a job while the run lock is held. */
+  async function cancelPending(tx: DB, jobIds: readonly string[], why: { reason: string; message: string }): Promise<void> {
+    if (jobIds.length === 0) return
+    const table = quoted(queueTable())
+    await tx.execute(sql`
+      UPDATE ${sql.raw(table)} SET status = 'cancelled', completed_at = now(), claim_token = NULL,
+             error_payload = jsonb_build_object('reason', ${why.reason}::text, 'message', ${why.message}::text)
+       WHERE id IN (SELECT id FROM ${sql.raw(table)}
+                     WHERE id ${sql.inList(jobIds, 'uuid')} AND status = 'pending'
+                     FOR UPDATE SKIP LOCKED)`)
+  }
+
+  async function lockRun(tx: DB, runId: string): Promise<RunRow | null> {
+    const rows = await tx.execute(sql`
+      SELECT id::text, tenant_id::text AS tenant, item, status, priority, step_priority, targets, inputs,
+             trace_id::text AS trace_id, trace_root_id::text AS trace_root_id, trace_step_id::text AS trace_step_id, actor
+        FROM ${sql.raw(RUN)} WHERE id = ${runId}::uuid AND graph = ${graph} FOR UPDATE`)
+    const row = rows[0]
+    if (row === undefined) return null
+    const traceId = row['trace_id'] as string | null
+    const rootId = row['trace_root_id'] as string | null
+    return {
+      id: row['id'] as string,
+      tenant: row['tenant'] as string,
+      item: row['item'] as string,
+      status: row['status'] as GraphRunStatus,
+      priority: row['priority'] as number,
+      stepPriority: row['step_priority'] as Record<string, number> | null,
+      targets: row['targets'] as string[],
+      inputs: row['inputs'] as Record<string, unknown>,
+      parent: traceId !== null && rootId !== null ? { id: traceId, rootId, stepId: row['trace_step_id'] as string | null } : null,
+      actor: row['actor'] as string | null,
+    }
+  }
+
+  async function loadTasks(tx: DB, runId: string): Promise<Task[]> {
+    const rows = await tx.execute(sql`
+      SELECT id::text, step, instance, status, job_id::text AS job_id, input_key, result_id::text AS result_id, element
+        FROM ${sql.raw(TASK)} WHERE run_id = ${runId}::uuid ORDER BY step, instance`)
+    return rows.map(r => ({
+      id: r['id'] as string,
+      step: r['step'] as string,
+      instance: r['instance'] as number,
+      status: r['status'] as TaskStatus,
+      jobId: r['job_id'] as string | null,
+      inputKey: r['input_key'] as string | null,
+      resultId: r['result_id'] as string | null,
+      element: r['element'],
+    }))
+  }
+
+  async function loadResults(tx: DB, ids: readonly string[]): Promise<Map<string, ResultValue>> {
+    const out = new Map<string, ResultValue>()
+    if (ids.length === 0) return out
+    const rows = await tx.execute(sql`
+      SELECT id::text, value, value_hash FROM ${sql.raw(RESULT)}
+       WHERE id ${sql.inList(ids, 'uuid')}`)
+    for (const r of rows) out.set(r['id'] as string, { value: r['value'], hash: r['value_hash'] as string })
+    return out
+  }
+
+  async function findRecorded(
+    tx: DB, tenant: string, item: string, step: string, key: string, instance: number,
+  ): Promise<{ id: string; value: unknown; hash: string } | null> {
+    const rows = await tx.execute(sql`
+      SELECT id::text, value, value_hash FROM ${sql.raw(RESULT)}
+       WHERE tenant_id = ${tenant}::uuid AND graph = ${graph} AND item = ${item} AND step = ${step}
+         AND version = ${steps[step]!.version} AND input_key = ${key}
+       ORDER BY (instance = ${instance}) DESC, created_at ASC LIMIT 1`)
+    const row = rows[0]
+    return row === undefined ? null : { id: row['id'] as string, value: row['value'], hash: row['value_hash'] as string }
+  }
+
+  /** Records a result; when one already holds the key (a concurrent run got there) that one is returned. */
+  async function recordResult(
+    tx: DB, run: RunRow, step: string, key: string, instance: number, value: unknown, replace: boolean,
+  ): Promise<{ id: string; hash: string; value: unknown }> {
+    const json = JSON.stringify(value)
+    const hash = valueHash(value)
+    const conflict = replace
+      ? sql`ON CONFLICT (tenant_id, graph, item, step, version, input_key, instance)
+              DO UPDATE SET value = EXCLUDED.value, value_hash = EXCLUDED.value_hash, created_at = now()`
+      : sql`ON CONFLICT (tenant_id, graph, item, step, version, input_key, instance) DO NOTHING`
+    const inserted = await tx.execute(sql`
+      INSERT INTO ${sql.raw(RESULT)} (tenant_id, graph, item, step, version, input_key, instance, value, value_hash)
+      VALUES (${run.tenant}::uuid, ${graph}, ${run.item}, ${step}, ${steps[step]!.version}, ${key}, ${instance}, ${json}::jsonb, ${hash})
+      ${conflict} RETURNING id::text`)
+    if (inserted[0] !== undefined) return { id: inserted[0]['id'] as string, hash, value }
+    const rows = await tx.execute(sql`
+      SELECT id::text, value, value_hash FROM ${sql.raw(RESULT)}
+       WHERE tenant_id = ${run.tenant}::uuid AND graph = ${graph} AND item = ${run.item} AND step = ${step}
+         AND version = ${steps[step]!.version} AND input_key = ${key} AND instance = ${instance}`)
+    const row = rows[0]!
+    return { id: row['id'] as string, hash: row['value_hash'] as string, value: row['value'] }
+  }
+
+  function stageReused(step: string, instance: number, resultId: string): void {
+    const buffer = getContext()?.trace?.buffer
+    if (buffer === undefined) return
+    const now = new Date()
+    buffer.stageExternal({
+      id: randomUUID(),
+      parentStepId: null,
+      flowStepRef: 'external:pipework.step.reused',
+      status: 'ok',
+      startedAt: now,
+      completedAt: now,
+      durationMicros: 0n,
+      capturedInputs: [serializeForTrace({ step, instance: instance === SINGLE ? null : instance, resultId })],
+      error: null,
+    })
+  }
+
+  function stageTimeout(step: string, instance: number, timeoutMs: number, startedAt: Date, error: StepTimeoutError): void {
+    const buffer = getContext()?.trace?.buffer
+    if (buffer === undefined) return
+    const now = new Date()
+    buffer.stageExternal({
+      id: randomUUID(),
+      parentStepId: null,
+      flowStepRef: 'external:pipework.step.timeout',
+      status: 'error',
+      startedAt,
+      completedAt: now,
+      durationMicros: BigInt(Math.max(0, now.getTime() - startedAt.getTime())) * 1000n,
+      capturedInputs: [serializeForTrace({ step, instance: instance === SINGLE ? null : instance, timeoutMs })],
+      error: projectStepError(error),
+    })
+  }
+
+  /** The state of one run during an `advance`: its tasks and the results they hold. */
+  class RunState {
+    constructor(
+      readonly run: RunRow,
+      readonly tasks: Task[],
+      readonly results: Map<string, ResultValue>,
+      readonly inputHashes: Record<string, string>,
+    ) {}
+
+    of(step: string): Task[] { return this.tasks.filter(t => t.step === step) }
+    single(step: string): Task | undefined { return this.tasks.find(t => t.step === step && t.instance === SINGLE) }
+    instances(step: string): Task[] {
+      return this.tasks.filter(t => t.step === step && t.instance >= 0).sort((a, b) => a.instance - b.instance)
+    }
+    instanceOf(step: string, i: number): Task { return this.instances(step).find(t => t.instance === i)! }
+    good(t: Task): boolean { return t.status === 'done' || t.status === 'reused' }
+
+    complete(step: string): boolean {
+      const single = this.single(step)
+      if (single === undefined) return false
+      if (!isFanOut(step)) return this.good(single)
+      return single.status === 'done' && this.instances(step).every(t => this.good(t))
+    }
+
+    resultOf(t: Task): ResultValue {
+      const r = t.resultId === null ? undefined : this.results.get(t.resultId)
+      if (r === undefined) throw new Error(`[pipework] ${graph}: task ${t.id} (${t.step}) has no recorded result.\n`)
+      return r
+    }
+
+    /** What a step reads: the value and hash of each need, and the request inputs among them. */
+    gather(def: GraphStepDefinition, task?: Task): { values: Record<string, unknown>; hashes: Record<string, string | string[]>; inputs: Record<string, unknown> } {
+      const values: Record<string, unknown> = {}
+      const hashes: Record<string, string | string[]> = {}
+      const inputs: Record<string, unknown> = {}
+      for (const need of def.needs ?? []) {
+        if (inputNames.has(need)) {
+          values[need] = this.run.inputs[need]
+          inputs[need] = this.run.inputs[need]
+          hashes[need] = this.inputHashes[need]!
+        } else if (def.byElement?.includes(need) === true && task !== undefined) {
+          const r = this.resultOf(this.instanceOf(need, task.instance))
+          values[need] = r.value
+          hashes[need] = r.hash
+        } else if (isFanOut(need)) {
+          const rs = this.instances(need).map(t => this.resultOf(t))
+          values[need] = rs.map(r => r.value)
+          hashes[need] = rs.map(r => r.hash)
+        } else {
+          const r = this.resultOf(this.single(need)!)
+          values[need] = r.value
+          hashes[need] = r.hash
+        }
+      }
+      return { values, hashes, inputs }
+    }
+  }
+
+  function inputHashesOf(inputs: Readonly<Record<string, unknown>>): Record<string, string> {
+    return Object.fromEntries(Object.entries(inputs).map(([k, v]) => [k, inputHash(v)]))
+  }
+
+  /** A step's needs (values or hashes) without those that only choose its list: what its key and `run` cover. */
+  function without<T>(def: GraphStepDefinition, values: Record<string, T>): Record<string, T> {
+    const only = def.forEachOnly
+    if (only === undefined || only.length === 0) return values
+    return Object.fromEntries(Object.entries(values).filter(([n]) => !only.includes(n)))
+  }
+
+  function contentKey(step: string, element: unknown, fanOut: boolean, hashes: Record<string, string | string[]>): string {
+    return sha256Hex(canonicalJson({
+      graph, step, version: steps[step]!.version, element: fanOut ? valueHash(element) : null, needs: hashes,
+    }))
+  }
+
+  async function loadState(tx: DB, run: RunRow): Promise<RunState> {
+    const tasks = await loadTasks(tx, run.id)
+    const results = await loadResults(tx, tasks.flatMap(t => t.resultId === null ? [] : [t.resultId]))
+    return new RunState(run, tasks, results, inputHashesOf(run.inputs))
+  }
+
+  /** What a step's context carries besides its data: the database handles `run` is given. */
+  interface StepIo {
+    checkpoint<R>(fn: (tx: DB) => Promise<R>): Promise<R>
+    withResult(fn: (tx: DB) => Promise<void>): void
+  }
+
+  function stepContext(state: RunState, def: GraphStepDefinition, task: Task, fanOut: boolean, signal: AbortSignal, io: StepIo): Omit<GraphStepContext, 'attempt' | 'maxAttempts'> {
+    const g = state.gather(def, task)
+    return {
+      needs: without(def, g.values),
+      element: fanOut ? task.element : undefined,
+      index: fanOut ? task.instance : null,
+      item: state.run.item,
+      inputs: without(def, g.inputs),
+      runId: state.run.id,
+      signal,
+      checkpoint: io.checkpoint,
+      withResult: io.withResult,
+    }
+  }
+
+  /** The priority of a step's jobs: the request's `stepPriority` for it, else its definition's `priority`, else the run's. */
+  function stepJobPriority(run: RunRow, def: GraphStepDefinition, step: string): number {
+    return run.stepPriority?.[step] ?? def.priority ?? run.priority
+  }
+
+  async function setTask(tx: DB, t: Task, patch: { status: TaskStatus; jobId?: string; inputKey?: string; resultId?: string; error?: unknown }): Promise<void> {
+    await tx.execute(sql`
+      UPDATE ${sql.raw(TASK)} SET status = ${patch.status},
+             job_id = COALESCE(${patch.jobId ?? null}::uuid, job_id),
+             input_key = COALESCE(${patch.inputKey ?? null}, input_key),
+             result_id = COALESCE(${patch.resultId ?? null}::uuid, result_id),
+             error = COALESCE(${patch.error === undefined ? null : JSON.stringify(patch.error)}::jsonb, error)
+       WHERE id = ${t.id}::uuid`)
+    t.status = patch.status
+    if (patch.jobId !== undefined) t.jobId = patch.jobId
+    if (patch.inputKey !== undefined) t.inputKey = patch.inputKey
+    if (patch.resultId !== undefined) t.resultId = patch.resultId
+  }
+
+  /**
+   * Moves a run as far as its recorded results allow, in one pass over the steps in
+   * dependency order: expands fan-out steps whose list is now known, reuses recorded
+   * results, queues the rest, and marks the run done when nothing is left. The run
+   * row is locked first, so concurrent finishes advance it one at a time.
+   */
+  async function advance(tx: DB, runId: string): Promise<Outcome & { run: RunRow | null }> {
+    const outcome: Outcome = { reused: 0, queued: 0, completed: false }
+    const run = await lockRun(tx, runId)
+    if (run === null || run.status !== 'running') return { ...outcome, run }
+    const state = await loadState(tx, run)
+    const closure = closureOf(run.targets)
+    const signal = new AbortController().signal
+
+    const process = async (task: Task, def: GraphStepDefinition, fanOut: boolean): Promise<void> => {
+      const g = state.gather(def, task)
+      const key = contentKey(task.step, task.element, fanOut, without(def, g.hashes))
+      const recorded = await findRecorded(tx, run.tenant, run.item, task.step, key, task.instance)
+      if (recorded !== null) {
+        state.results.set(recorded.id, { value: recorded.value, hash: recorded.hash })
+        await setTask(tx, task, { status: 'reused', inputKey: key, resultId: recorded.id })
+        stageReused(task.step, task.instance, recorded.id)
+        outcome.reused++
+        return
+      }
+      if (def.person === true) {
+        const accepted = def.autoAccept === undefined ? undefined : await def.autoAccept(stepContext(state, def, task, fanOut, signal, {
+          checkpoint: (fn) => fn(tx),
+          withResult: () => {
+            throw new Error(`[pipework] ${graph}: step "${task.step}" is a person step; withResult is for steps that run (a person step records through decide).\n`)
+          },
+        }))
+        if (accepted !== undefined) {
+          const rec = await recordResult(tx, run, task.step, key, task.instance, toJson(accepted), false)
+          state.results.set(rec.id, { value: rec.value, hash: rec.hash })
+          await setTask(tx, task, { status: 'done', inputKey: key, resultId: rec.id })
+        } else {
+          await setTask(tx, task, { status: 'waiting', inputKey: key })
+        }
+        return
+      }
+      const jobId = await queue.enqueue(tx, {
+        type: typeOfClass(classOf(task.step)),
+        payload: { kind: 'step', runId: run.id, taskId: task.id },
+        priority: stepJobPriority(run, def, task.step),
+        ...(def.maxAttempts !== undefined ? { maxAttempts: def.maxAttempts } : {}),
+        tenant: run.tenant,
+        parent: run.parent,
+        actor: run.actor,
+      })
+      await setTask(tx, task, { status: 'queued', inputKey: key, jobId })
+      outcome.queued++
+    }
+
+    for (const name of closure) {
+      if (state.complete(name)) continue
+      const def = steps[name]!
+      const by = def.byElement
+      const ready = (def.needs ?? []).every(n => steps[n] === undefined
+        || (by?.includes(n) === true ? state.single(n)?.status === 'done' : state.complete(n)))
+      if (!ready) continue
+      const expander = state.single(name)
+      if (expander === undefined) continue
+
+      if (!isFanOut(name)) {
+        if (expander.status === 'blocked') await process(expander, def, false)
+        continue
+      }
+
+      const expand = async (elements: readonly unknown[]): Promise<void> => {
+        if (elements.length > 0) {
+          const rows = await tx.execute(sql`
+            INSERT INTO ${sql.raw(TASK)} (tenant_id, run_id, step, instance, element, status)
+            VALUES ${sql.join(elements.map((el, i) =>
+              sql`(${run.tenant}::uuid, ${run.id}::uuid, ${name}, ${i}, ${JSON.stringify(el)}::jsonb, 'blocked')`), sql`, `)}
+            RETURNING id::text, instance`)
+          for (const r of rows) {
+            const i = r['instance'] as number
+            state.tasks.push({
+              id: r['id'] as string, step: name, instance: i, status: 'blocked', jobId: null, inputKey: null, resultId: null, element: elements[i],
+            })
+          }
+        }
+        await setTask(tx, expander, { status: 'done' })
+      }
+
+      if (expander.status === 'blocked') {
+        if (by !== undefined) {
+          await expand(state.instances(by[0]!).map(t => t.element))
+        } else {
+          const g = state.gather(def)
+          const forEachCtx: GraphForEachContext = { needs: g.values, item: run.item, inputs: g.inputs, runId: run.id }
+          const list = await def.forEach!(forEachCtx)
+          if (!Array.isArray(list)) {
+            throw new Error(`[pipework] ${graph}: forEach of step "${name}" must return an array, got ${typeof list}.\n`)
+          }
+          await expand(list.map(toJson))
+        }
+      }
+      for (const t of state.instances(name)) {
+        if (t.status !== 'blocked') continue
+        if (by !== undefined && !by.every(n => state.good(state.instanceOf(n, t.instance)))) continue
+        await process(t, def, true)
+      }
+    }
+
+    if (closure.every(n => state.complete(n))) {
+      const done = await tx.execute(sql`
+        UPDATE ${sql.raw(RUN)} SET status = 'done', finished_at = now(), notified = ${config.onDone === undefined}
+         WHERE id = ${run.id}::uuid AND status = 'running' RETURNING id`)
+      if (done.length > 0) {
+        outcome.completed = true
+        await tx.execute(sql`SELECT pg_notify('pipework_step_run', ${run.id})`)
+      }
+    }
+    return { ...outcome, run }
+  }
+
+  /** Marks the run's done callback delivered and calls it; false when it already was. */
+  async function deliverDone(tx: DB, runId: string): Promise<void> {
+    const rows = await tx.execute(sql`
+      UPDATE ${sql.raw(RUN)} SET notified = true
+       WHERE id = ${runId}::uuid AND status = 'done' AND notified = false
+       RETURNING item, tenant_id::text AS tenant`)
+    const row = rows[0]
+    if (row === undefined || config.onDone === undefined) return
+    await config.onDone({ runId, item: row['item'] as string, tenant: row['tenant'] as string })
+  }
+
+  /** After an advance outside a step job: a finished run's callback is delivered by a job, which has a Flow to run it in. */
+  async function scheduleDone(tx: DB, run: RunRow | null, outcome: Outcome): Promise<void> {
+    if (run === null || !outcome.completed || config.onDone === undefined) return
+    await queue.enqueue(tx, {
+      type: typeOfClass(classOf(run.targets[0]!)), payload: { kind: 'finish', runId: run.id, taskId: run.id }, priority: run.priority, tenant: run.tenant, parent: run.parent, actor: run.actor,
+    })
+  }
+
+  async function request(db: DB, opts: RequestOptions): Promise<RequestResult> {
+    const closure = checkRequest(opts, 'request')
+    const tenant = tenantOf(opts.tenant ?? handleTenant(db), 'request')
+    const priority = opts.priority ?? config.priority ?? 0
+    const stepPriority = opts.stepPriority === undefined || Object.keys(opts.stepPriority).length === 0 ? null : opts.stepPriority
+    const parent = parentTraceRef()
+    const flow = getContext()
+    const actor = opts.carryActor === false || flow === undefined ? null : resolveActorId(flow)
+    const inputs = toJson(opts.inputs ?? {})
+
+    return inTenantTx(db, tenant, async (tx) => {
+      const created = await tx.execute(sql`
+        INSERT INTO ${sql.raw(RUN)} (tenant_id, graph, item, targets, priority, step_priority, inputs, trace_id, trace_root_id, trace_step_id, actor)
+        VALUES (${tenant}::uuid, ${graph}, ${opts.item}, ${JSON.stringify(opts.targets)}::jsonb, ${priority}, ${stepPriority === null ? null : JSON.stringify(stepPriority)}::jsonb, ${JSON.stringify(inputs)}::jsonb,
+                ${parent?.id ?? null}::uuid, ${parent?.rootId ?? null}::uuid, ${parent?.stepId ?? null}::uuid, ${actor}::text)
+        RETURNING id::text`)
+      const runId = created[0]!['id'] as string
+      await tx.execute(sql`
+        INSERT INTO ${sql.raw(TASK)} (tenant_id, run_id, step, instance, status)
+        VALUES ${sql.join(closure.map(step => sql`(${tenant}::uuid, ${runId}::uuid, ${step}, ${SINGLE}, 'blocked')`), sql`, `)}`)
+      const outcome = await advance(tx, runId)
+      await scheduleDone(tx, outcome.run, outcome)
+      return { runId, reused: outcome.reused, queued: outcome.queued }
+    })
+  }
+
+  async function plan(db: DB, opts: PlanOptions): Promise<PlanResult> {
+    const closure = checkRequest(opts, 'plan')
+    const tenant = tenantOf(opts.tenant ?? handleTenant(db), 'plan')
+    const inputs = toJson(opts.inputs ?? {}) as Record<string, unknown>
+    const hashes = inputHashesOf(inputs)
+
+    return inTenantTx(db, tenant, async (tx) => {
+      // What each step would hold if the run were made now: its list and, per instance, the recorded hash and value (null where none is).
+      interface Known { elements: unknown[] | null; hashes: Array<string | null>; values: unknown[] }
+      const known = new Map<string, Known>()
+      const full = (k: Known | undefined): k is Known & { elements: unknown[] } =>
+        k !== undefined && k.elements !== null && k.hashes.every(h => h !== null)
+      const result: Record<string, StepPlan> = {}
+      for (const name of closure) {
+        const def = steps[name]!
+        const fanOut = isFanOut(name)
+        const needs = def.needs ?? []
+        const by = def.byElement
+        const unknownList = (): void => { result[name] = { reusable: 0, toRun: fanOut ? null : 1 } }
+        const stepNeeds = needs.filter(n => steps[n] !== undefined && by?.includes(n) !== true)
+        if (!stepNeeds.every(n => full(known.get(n)))) { unknownList(); continue }
+        if (by !== undefined && known.get(by[0]!)?.elements == null) { unknownList(); continue }
+
+        const needValues: Record<string, unknown> = {}
+        const needHashes: Record<string, string | string[]> = {}
+        const given: Record<string, unknown> = {}
+        for (const n of needs) {
+          if (inputNames.has(n)) { needValues[n] = inputs[n]; given[n] = inputs[n]; needHashes[n] = hashes[n]! } else if (by?.includes(n) !== true) {
+            const k = known.get(n)!
+            needValues[n] = isFanOut(n) ? k.values : k.values[0]
+            needHashes[n] = isFanOut(n) ? k.hashes as string[] : k.hashes[0]!
+          }
+        }
+        const elements: unknown[] = by !== undefined
+          ? known.get(by[0]!)!.elements!
+          : fanOut ? (await def.forEach!({ needs: needValues, item: opts.item, inputs: given, runId: '' })).map(toJson) : [undefined]
+        let reusable = 0
+        const values: unknown[] = []
+        const valueHashes: Array<string | null> = []
+        for (const [i, element] of elements.entries()) {
+          let own = needHashes
+          let checkable = true
+          if (by !== undefined) {
+            own = { ...needHashes }
+            for (const n of by) {
+              const h = known.get(n)?.hashes[i] ?? null
+              if (h === null) checkable = false
+              else own[n] = h
+            }
+          }
+          const rec = checkable
+            ? await findRecorded(tx, tenant, opts.item, name, contentKey(name, element, fanOut, without(def, own)), fanOut ? i : SINGLE)
+            : null
+          if (rec !== null) reusable++
+          values.push(rec?.value)
+          valueHashes.push(rec?.hash ?? null)
+        }
+        result[name] = { reusable, toRun: elements.length - reusable }
+        known.set(name, { elements, hashes: valueHashes, values })
+      }
+      return { steps: result }
+    })
+  }
+
+  async function status(db: DB, runId: string, opts?: ScopeOptions): Promise<RunStatus> {
+    return inTenantTx(db, scopeOf(opts?.tenant), tx => statusIn(tx, runId))
+  }
+
+  function stateOf(status: TaskStatus, jobStatus: string | null): TaskState {
+    if (status === 'done' || status === 'reused' || status === 'failed' || status === 'cancelled') return status
+    return status === 'queued' && (jobStatus === 'claimed' || jobStatus === 'running') ? 'running' : 'waiting'
+  }
+
+  async function statusIn(db: DB, runId: string): Promise<RunStatus> {
+    const runs = await db.execute(sql`SELECT status, inputs, finished_at FROM ${sql.raw(RUN)} WHERE id = ${runId}::uuid AND graph = ${graph}`)
+    if (runs[0] === undefined) throw new Error(`[pipework] ${graph}.status: no run "${runId}".\n`)
+    const tasks = await db.execute(sql`
+      SELECT t.step, t.instance, t.status, t.element, j.status AS job_status
+        FROM ${sql.raw(TASK)} t LEFT JOIN ${sql.raw(quoted(queueTable()))} j ON j.id = t.job_id
+       WHERE t.run_id = ${runId}::uuid`)
+    const zero = (): { total: number; done: number; reused: number; running: number; waiting: number; failed: number; cancelled: number } =>
+      ({ total: 0, done: 0, reused: 0, running: 0, waiting: 0, failed: 0, cancelled: 0 })
+    const counts = new Map<string, ReturnType<typeof zero>>()
+    for (const t of tasks) {
+      const step = t['step'] as string
+      const fanOut = steps[step] !== undefined && isFanOut(step)
+      const c = counts.get(step) ?? zero()
+      counts.set(step, c)
+      if (fanOut && (t['instance'] as number) < 0) continue
+      c.total++
+      c[stateOf(t['status'] as TaskStatus, t['job_status'] as string | null)]++
+    }
+    const listDone = new Set<string>()
+    for (const t of tasks) if ((t['instance'] as number) < 0 && t['status'] === 'done') listDone.add(t['step'] as string)
+    const out: Record<string, StepCounts> = {}
+    const list: RunTask[] = []
+    for (const name of order) {
+      const c = counts.get(name)
+      if (c === undefined) continue
+      out[name] = steps[name] !== undefined && isFanOut(name) ? { ...c, listKnown: listDone.has(name) } : c
+      const mine = tasks.filter(t => t['step'] === name).sort((a, b) => (a['instance'] as number) - (b['instance'] as number))
+      for (const t of mine) {
+        list.push({
+          step: name, instance: t['instance'] as number, element: t['element'] ?? null,
+          state: stateOf(t['status'] as TaskStatus, t['job_status'] as string | null),
+        })
+      }
+    }
+    const finished = runs[0]['finished_at'] as string | Date | null
+    const raw = runs[0]['inputs']
+    const inputs = (typeof raw === 'string' ? JSON.parse(raw) : raw ?? {}) as Record<string, unknown>
+    return {
+      status: runs[0]['status'] as GraphRunStatus, steps: out, tasks: list, inputs,
+      finishedAt: finished === null ? null : new Date(finished),
+    }
+  }
+
+  async function ended(db: DB, opts?: EndedRunsOptions): Promise<readonly EndedRun[]> {
+    const tenant = scopedExecutor(db, scopeOf(opts?.tenant), `${graph}.ended`).tenant
+    const limit = opts?.limit ?? 1000
+    if (!Number.isInteger(limit) || limit < 1) throw new Error(`[pipework] ${graph}.ended: limit must be a positive integer.\n`)
+    const only = tenant === null ? sql`` : sql` AND tenant_id = ${tenant}::uuid`
+    const byIds = opts?.ids === undefined ? sql`` : opts.ids.length === 0 ? sql` AND false`
+      : sql` AND id ${sql.inList(opts.ids, 'uuid')}`
+    const before = opts?.finishedBefore === undefined ? sql`` : sql` AND finished_at < ${opts.finishedBefore.toISOString()}::timestamptz`
+    const rows = await inTenantTx(db, tenant, tx => tx.execute(sql`
+      SELECT id::text, tenant_id::text, item, status, finished_at, notified FROM ${sql.raw(RUN)}
+       WHERE graph = ${graph} AND status <> 'running' AND finished_at IS NOT NULL${only}${byIds}${before}
+       ORDER BY finished_at, id LIMIT ${limit}`))
+    return rows.map(r => ({
+      id: r['id'] as string,
+      tenant: r['tenant_id'] as string,
+      item: r['item'] as string,
+      status: r['status'] as EndedRun['status'],
+      finishedAt: new Date(r['finished_at'] as string | Date),
+      notified: r['notified'] as boolean,
+    }))
+  }
+
+  /** The queue's table name, read from its definition. */
+  function queueTable(): string {
+    return (queue.definition as unknown as { $name: string }).$name
+  }
+  const quoted = (name: string): string => `"${name.replace(/"/g, '""')}"`
+
+  async function cancelRun(tx: DB, runId: string): Promise<void> {
+    const run = await lockRun(tx, runId)
+    if (run === null || run.status !== 'running') return
+    await tx.execute(sql`UPDATE ${sql.raw(RUN)} SET status = 'cancelled', finished_at = now() WHERE id = ${runId}::uuid`)
+    const live = await tx.execute(sql`
+      UPDATE ${sql.raw(TASK)} SET status = 'cancelled'
+       WHERE run_id = ${runId}::uuid AND status IN ('blocked', 'queued', 'waiting') RETURNING job_id::text`)
+    await cancelPending(tx, live.flatMap(r => r['job_id'] === null ? [] : [r['job_id'] as string]),
+      { reason: 'run-cancelled', message: `${graph} run ${runId} was cancelled` })
+    await tx.execute(sql`SELECT pg_notify('pipework_step_run', ${runId})`)
+    for (const c of running.get(runId) ?? []) c.abort(new Error(`[pipework] ${graph} run ${runId} was cancelled.`))
+  }
+
+  async function cancel(db: DB, runId: string, opts?: ScopeOptions): Promise<void> {
+    await inTenantTx(db, scopeOf(opts?.tenant), async (tx) => { await cancelRun(tx, runId) })
+  }
+
+  async function decide(db: DB, opts: DecideOptions): Promise<void> {
+    const def = steps[opts.step]
+    if (def === undefined || def.person !== true) {
+      throw new Error(`[pipework] ${graph}.decide: "${opts.step}" is not a person step.\n`)
+    }
+    const instance = opts.index ?? SINGLE
+    const value = toJson(opts.value)
+    await inTenantTx(db, scopeOf(opts.tenant), async (tx) => {
+      const run = await lockRun(tx, opts.runId)
+      if (run === null) throw new Error(`[pipework] ${graph}.decide: no run "${opts.runId}".\n`)
+      const rows = await tx.execute(sql`
+        SELECT id::text, status, input_key, result_id::text AS result_id FROM ${sql.raw(TASK)}
+         WHERE run_id = ${run.id}::uuid AND step = ${opts.step} AND instance = ${instance} FOR UPDATE`)
+      const task = rows[0]
+      if (task === undefined) {
+        throw new Error(`[pipework] ${graph}.decide: run ${opts.runId} has no task for step "${opts.step}"${opts.index === undefined ? '' : ` instance ${String(opts.index)}`}.\n`)
+      }
+      const taskStatus = task['status'] as TaskStatus
+      if (taskStatus === 'waiting') {
+        const rec = await recordResult(tx, run, opts.step, task['input_key'] as string, instance, value, true)
+        await tx.execute(sql`UPDATE ${sql.raw(TASK)} SET status = 'done', result_id = ${rec.id}::uuid WHERE id = ${task['id'] as string}::uuid`)
+        const outcome = await advance(tx, run.id)
+        await scheduleDone(tx, outcome.run, outcome)
+        return
+      }
+      if (taskStatus === 'done' || taskStatus === 'reused') {
+        await tx.execute(sql`
+          UPDATE ${sql.raw(RESULT)} SET value = ${JSON.stringify(value)}::jsonb, value_hash = ${valueHash(value)}, created_at = now()
+           WHERE id = ${task['result_id'] as string}::uuid`)
+        return
+      }
+      throw new Error(`[pipework] ${graph}.decide: step "${opts.step}" of run ${opts.runId} is ${taskStatus}, not waiting for a decision.\n`)
+    })
+  }
+
+  async function wait(db: DB, runId: string, opts: { readonly timeoutMs: number; readonly tenant?: string }): Promise<GraphRunStatus | 'timeout'> {
+    const deadline = Date.now() + opts.timeoutMs
+    for (;;) {
+      const rows = await inTenantTx(db, scopeOf(opts.tenant), tx =>
+        tx.execute(sql`SELECT status FROM ${sql.raw(RUN)} WHERE id = ${runId}::uuid AND graph = ${graph}`))
+      if (rows[0] === undefined) throw new Error(`[pipework] ${graph}.wait: no run "${runId}".\n`)
+      const s = rows[0]['status'] as GraphRunStatus
+      if (s !== 'running') return s
+      if (Date.now() >= deadline) return 'timeout'
+      await new Promise(r => setTimeout(r, Math.min(50, Math.max(1, deadline - Date.now()))))
+    }
+  }
+
+  async function sweep(db: DB, opts?: ScopeOptions): Promise<number> {
+    if (config.resultRetentionDays === undefined) return 0
+    const cutoff = new Date(Date.now() - config.resultRetentionDays * MS_PER_DAY).toISOString()
+    const tenant = scopedExecutor(db, scopeOf(opts?.tenant), `${graph}.sweep`).tenant
+    const only = tenant === null ? sql`` : sql` AND tenant_id = ${tenant}::uuid`
+    return inTenantTx(db, tenant, async (tx) => {
+      const old = await tx.execute(sql`
+        SELECT id::text FROM ${sql.raw(RUN)}
+         WHERE graph = ${graph} AND status <> 'running' AND finished_at < ${cutoff}::timestamptz${only}`)
+      if (old.length > 0) {
+        const ids = sql.inList(old.map(r => r['id'] as string), 'uuid')
+        await tx.execute(sql`DELETE FROM ${sql.raw(TASK)} WHERE run_id ${ids}`)
+        await tx.execute(sql`DELETE FROM ${sql.raw(RUN)} WHERE id ${ids}`)
+      }
+      const rows = await tx.execute(sql`
+        DELETE FROM ${sql.raw(RESULT)} WHERE graph = ${graph} AND created_at < ${cutoff}::timestamptz${only} RETURNING id`)
+      return rows.length
+    })
+  }
+
+  async function removeTenant(db: DB, tenantId: string): Promise<void> {
+    if (typeof tenantId !== 'string' || tenantId.trim() === '') {
+      throw new Error(`[pipework] ${graph}.removeTenant: tenantId must be a non-empty string.\n`)
+    }
+    await inTenantTx(db, tenantId, async (tx) => {
+      await tx.execute(sql`DELETE FROM ${sql.raw(TASK)} WHERE tenant_id = ${tenantId}::uuid`)
+      await tx.execute(sql`DELETE FROM ${sql.raw(RUN)} WHERE tenant_id = ${tenantId}::uuid`)
+      await tx.execute(sql`DELETE FROM ${sql.raw(RESULT)} WHERE tenant_id = ${tenantId}::uuid`)
+    })
+  }
+
+  function itemScope(method: string, item: string, explicit?: string): string {
+    if (typeof item !== 'string' || item.trim() === '') {
+      throw new Error(`[pipework] ${graph}.${method}: item must be a non-empty string.\n`)
+    }
+    const tenant = scopeOf(explicit)
+    if (tenant === null) {
+      throw new Error(`[pipework] ${graph}.${method}: no tenant — pass options.tenant or call inside a Flow with one.\n`)
+    }
+    return tenant
+  }
+
+  async function results(db: DB, item: string, opts?: ScopeOptions): Promise<readonly StepResultRecord[]> {
+    const tenant = itemScope('results', item, opts?.tenant ?? handleTenant(db))
+    return inTenantTx(db, tenant, async (tx) => {
+      const rows = await tx.execute(sql`
+        SELECT id::text, step, version, instance, value, created_at FROM ${sql.raw(RESULT)}
+         WHERE graph = ${graph} AND item = ${item} AND tenant_id = ${tenant}::uuid
+         ORDER BY step, instance, created_at`)
+      return rows.map(r => ({
+        id: r['id'] as string,
+        step: r['step'] as string,
+        version: Number(r['version']),
+        instance: Number(r['instance']),
+        value: r['value'],
+        createdAt: new Date(r['created_at'] as string | Date),
+      }))
+    })
+  }
+
+  async function removeItem(db: DB, item: string, opts?: RemoveItemOptions): Promise<RemoveItemResult> {
+    const tenant = itemScope('removeItem', item, opts?.tenant ?? handleTenant(db))
+    return inTenantTx(db, tenant, async (tx) => {
+      const runs = await tx.execute(sql`
+        SELECT id::text, status FROM ${sql.raw(RUN)}
+         WHERE graph = ${graph} AND item = ${item} AND tenant_id = ${tenant}::uuid ORDER BY id FOR UPDATE`)
+      const live = runs.filter(r => r['status'] === 'running').map(r => r['id'] as string)
+      if (live.length > 0) {
+        if (opts?.running !== 'cancel') throw new StepItemRunningError(graph, item, live)
+        for (const id of live) await cancelRun(tx, id)
+      }
+      let tasks = 0
+      if (runs.length > 0) {
+        const ids = sql.inList(runs.map(r => r['id'] as string), 'uuid')
+        tasks = (await tx.execute(sql`DELETE FROM ${sql.raw(TASK)} WHERE run_id ${ids} RETURNING id`)).length
+        await tx.execute(sql`DELETE FROM ${sql.raw(RUN)} WHERE id ${ids}`)
+      }
+      const gone = await tx.execute(sql`
+        DELETE FROM ${sql.raw(RESULT)} WHERE graph = ${graph} AND item = ${item} AND tenant_id = ${tenant}::uuid RETURNING id`)
+      return { runs: runs.length, tasks, results: gone.length }
+    })
+  }
+
+  type Checkpoint = <R>(fn: (tx: DB) => Promise<R>) => Promise<R>
+
+  /**
+   * The step job's attempt and limit, from the Flow the worker surface built; a job run
+   * outside it has neither, so they are read from the job's row.
+   */
+  async function triesOf(checkpoint: Checkpoint, taskId: string): Promise<{ attempt: number; maxAttempts: number }> {
+    const flow = getContext()
+    if (flow?.jobAttempt != null && flow.jobMaxAttempts != null) return { attempt: flow.jobAttempt, maxAttempts: flow.jobMaxAttempts }
+    return checkpoint(async (tx) => {
+      const rows = await tx.execute(sql`
+        SELECT j.attempt_count, j.max_attempts FROM ${sql.raw(TASK)} t JOIN ${sql.raw(quoted(queueTable()))} j ON j.id = t.job_id
+         WHERE t.id = ${taskId}::uuid`)
+      const row = rows[0]
+      return { attempt: ((row?.['attempt_count'] as number | undefined) ?? 0) + 1, maxAttempts: (row?.['max_attempts'] as number | undefined) ?? 1 }
+    })
+  }
+
+  /**
+   * The step job. The job holds no transaction while `run` waits: one short checkpoint
+   * checks the task and reads the run's state, `run` executes outside any transaction,
+   * and a second checkpoint records the result (with whatever `withResult` callbacks
+   * asked to write) and advances the run.
+   */
+  async function handleJob(checkpoint: Checkpoint, payload: { kind: string; runId: string; taskId: string }): Promise<unknown> {
+    if (payload.kind === 'finish') {
+      await checkpoint((tx) => deliverDone(tx, payload.runId))
+      return { finished: payload.runId }
+    }
+
+    const loaded = await checkpoint(async (tx) => {
+      const live = await tx.execute(sql`
+        SELECT 1 FROM ${sql.raw(TASK)} t JOIN ${sql.raw(RUN)} r ON r.id = t.run_id
+         WHERE t.id = ${payload.taskId}::uuid AND t.run_id = ${payload.runId}::uuid AND t.status = 'queued' AND r.status = 'running'`)
+      if (live.length === 0) return null
+
+      // Read the run's state without the lock: the step may take a long time, and other steps of the run must proceed meanwhile.
+      const runRows = await tx.execute(sql`
+        SELECT id::text, tenant_id::text AS tenant, item, status, priority, step_priority, targets, inputs, actor FROM ${sql.raw(RUN)} WHERE id = ${payload.runId}::uuid`)
+      const r = runRows[0]!
+      const run: RunRow = {
+        id: r['id'] as string, tenant: r['tenant'] as string, item: r['item'] as string, status: r['status'] as GraphRunStatus,
+        priority: r['priority'] as number, stepPriority: r['step_priority'] as Record<string, number> | null, targets: r['targets'] as string[], inputs: r['inputs'] as Record<string, unknown>, parent: null, actor: r['actor'] as string | null,
+      }
+      return { run, state: await loadState(tx, run) }
+    })
+    if (loaded === null) return { skipped: payload.taskId }
+    const { run, state } = loaded
+    const task = state.tasks.find(t => t.id === payload.taskId)!
+    const def = steps[task.step]!
+    const fanOut = isFanOut(task.step)
+
+    const tries = await triesOf(checkpoint, payload.taskId)
+    const resultWrites: Array<(tx: DB) => Promise<void>> = []
+    const controller = new AbortController()
+    let controllers = running.get(run.id)
+    if (controllers === undefined) { controllers = new Set(); running.set(run.id, controllers) }
+    controllers.add(controller)
+    let value: unknown
+    let timer: NodeJS.Timeout | undefined
+    const startedAt = new Date()
+    try {
+      const work = Promise.resolve(def.run!({
+        ...stepContext(state, def, task, fanOut, controller.signal, {
+          checkpoint,
+          withResult: (fn) => { resultWrites.push(fn) },
+        }),
+        ...tries,
+      }))
+      if (def.timeoutMs === undefined) {
+        value = toJson(await work)
+      } else {
+        const limit = def.timeoutMs
+        const expired = new Promise<never>((_, reject) => {
+          timer = setTimeout(() => {
+            const timeout = new StepTimeoutError(graph, task.step, limit)
+            controller.abort(timeout)
+            reject(timeout)
+          }, limit)
+          timer.unref()
+        })
+        work.catch(() => {})
+        try {
+          value = toJson(await Promise.race([work, expired]))
+        } catch (err) {
+          if (err instanceof StepTimeoutError) stageTimeout(task.step, task.instance, limit, startedAt, err)
+          throw err
+        }
+      }
+    } finally {
+      if (timer !== undefined) clearTimeout(timer)
+      controllers.delete(controller)
+      if (controllers.size === 0) running.delete(run.id)
+    }
+
+    return checkpoint(async (tx) => {
+      const locked = await lockRun(tx, run.id)
+      if (locked === null) return { skipped: payload.taskId }
+      const current = await tx.execute(sql`SELECT status FROM ${sql.raw(TASK)} WHERE id = ${task.id}::uuid FOR UPDATE`)
+      for (const write of resultWrites) await write(tx)
+      const rec = await recordResult(tx, locked, task.step, task.inputKey!, task.instance, value, false)
+      if (current[0]?.['status'] !== 'queued' || locked.status !== 'running') {
+        // Cancelled or failed while it ran: the result is kept for later requests, the run does not move.
+        return { recorded: rec.id }
+      }
+      await tx.execute(sql`UPDATE ${sql.raw(TASK)} SET status = 'done', result_id = ${rec.id}::uuid WHERE id = ${task.id}::uuid`)
+      const outcome = await advance(tx, run.id)
+      if (outcome.completed) await deliverDone(tx, run.id)
+      return { step: task.step, resultId: rec.id }
+    })
+  }
+
+  function handlerFor(type: string, database: string | undefined): Handler<Record<string, unknown>, unknown> {
+    const builder = database === undefined ? fitting.use() : fitting.use(database)
+    const key = database ?? 'db'
+    return builder.job(type)
+      .transaction('manual')
+      .input(schema.check.object({ kind: schema.check.string(), runId: schema.check.string(), taskId: schema.check.string() }))
+      .fit(async ({ input, checkpoint }: {
+        input: { kind: string; runId: string; taskId: string }
+        checkpoint: <R>(fn: (dbs: Record<string, DB>) => Promise<R>) => Promise<R>
+      }) => {
+        const tenant = getContext()?.tenant ?? null
+        const atTenant: Checkpoint = (fn) => checkpoint((dbs) => {
+          const tx = dbs[key]!
+          return asTenant(tx, tenant, () => fn(tx))
+        })
+        return handleJob(atTenant, input)
+      }) as unknown as Handler<Record<string, unknown>, unknown>
+  }
+
+  function handlers(database: string | undefined, selected?: readonly string[]): Handler<Record<string, unknown>, unknown>[] {
+    return (selected ?? classes).map(cls => handlerFor(typeOfClass(cls), database))
+  }
+
+  function workers(...wanted: string[]): GraphSelection {
+    if (wanted.length === 0) fail(graph, 'workers() needs at least one class.')
+    for (const cls of wanted) {
+      if (typeof cls !== 'string' || !CLASS_RE.test(cls)) {
+        fail(graph, `worker class "${String(cls)}" must start with a letter and use letters, digits, "_" or "-".`)
+      }
+      if (!classes.includes(cls)) fail(graph, `no step uses worker class "${cls}" (classes: ${classes.join(', ')}).`)
+    }
+    return { graph: self, classes: [...new Set(wanted)] }
+  }
+
+  /** A step job of this graph dead-lettered: fail the task, what depends on it and the run, cancel the rest, tell the app once. */
+  async function onDeadLetter(tx: DB, job: DeadLetterInfo, host: GraphHost): Promise<void> {
+    if (!jobTypes.includes(job.type)) return
+    const payload = job.payload as { kind?: string; runId?: string; taskId?: string }
+    if (payload.kind !== 'step' || payload.runId === undefined || payload.taskId === undefined) return
+    attached = host.instance
+    await asTenant(tx, job.tenant, () => failRun(tx, job, payload.runId!, payload.taskId!, host), host.instance)
+  }
+
+  /** The graph's `onFailed` inside its trace step; the step captures the failure as its input. */
+  const onFailedStep = __flowStep({ ref: 'pipework/steps:on-failed', capture: [(failure) => failure] }, async (failure: GraphFailure) => { await config.onFailed!(failure) })
+
+  async function failRun(tx: DB, job: DeadLetterInfo, runId: string, taskId: string, host: GraphHost): Promise<void> {
+    const run = await lockRun(tx, runId)
+    if (run === null || run.status !== 'running') return
+    const tasks = await loadTasks(tx, run.id)
+    const dead = tasks.find(t => t.id === taskId)
+    if (dead === undefined || dead.status !== 'queued') return
+
+    const dependents = dependentsOf(dead.step)
+    await setTask(tx, dead, { status: 'failed', error: job.error })
+    const cancelJobs: string[] = []
+    for (const t of tasks) {
+      if (t === dead || (t.status !== 'blocked' && t.status !== 'queued' && t.status !== 'waiting')) continue
+      if (t.jobId !== null) cancelJobs.push(t.jobId)
+      await setTask(tx, t, { status: dependents.has(t.step) ? 'failed' : 'cancelled' })
+    }
+    await cancelPending(tx, cancelJobs, { reason: 'run-failed', message: `${graph} run ${runId} failed at step ${dead.step}` })
+    await tx.execute(sql`
+      UPDATE ${sql.raw(RUN)} SET status = 'failed', finished_at = now(), notified = true,
+             error = ${JSON.stringify({ step: dead.step, error: job.error })}::jsonb
+       WHERE id = ${run.id}::uuid`)
+    await tx.execute(sql`SELECT pg_notify('pipework_step_run', ${run.id})`)
+
+    if (config.onFailed === undefined) return
+    const failure = { runId: run.id, item: run.item, tenant: run.tenant, step: dead.step, error: job.error }
+    try {
+      await tx.transaction(async (savepoint) => {
+        const dbName = host.database ?? host.instance.config.defaultDatabaseName()
+        const trace = isTracingEnabled(host.instance) ? openTrace({
+          instance: host.instance,
+          entryKind: 'job',
+          parent: job.flow.traceId !== null && job.flow.rootId !== null
+            ? { id: job.flow.traceId, rootId: job.flow.rootId, stepId: job.flow.stepId }
+            : null,
+          tenant: run.tenant,
+          auth: run.actor !== null ? { userId: run.actor } : null,
+          requestId: job.id,
+          sessionId: null,
+        }) : null
+        const base = createJobContext({
+          instance: host.instance, jobId: job.id, jobType: job.type, tenant: run.tenant, auth: run.actor !== null ? { userId: run.actor } : null, ...(trace !== null ? { trace } : {}),
+        })
+        const ctx = { ...base, transactions: new Map([[dbName, bindTransaction(base, dbName, savepoint)]]) }
+        await propagateLocals(savepoint, ctx, null)
+        await finalizeTrace(host.instance, trace, runInContext(ctx, async () => { await onFailedStep(failure) }))
+      })
+    } catch (err) {
+      getBaseLogger().error({ err, runId: run.id, graph }, 'Step graph onFailed callback failed; the run is already marked failed')
+      void recordFaultFor(host.instance, 'step-graph-on-failed', faultMessage(err), { runId: run.id, graph })
+    }
+  }
+
+  const self: StepGraph = {
+    name: graph,
+    jobType,
+    jobTypes,
+    queue,
+    request, plan, status, ended, cancel, decide, wait, sweep, removeTenant, results, removeItem, workers,
+    worker: { handlers, onDeadLetter },
+  }
+  return self
+}