pipework 0.84.0 → 0.85.0

package/dist/async/steps/graph.js

@@ -0,0 +1,850 @@
+// Step graphs over a jobs Queue. A graph is defined once; each request names the
+// results it wants for an item, and only the steps whose result is not already
+// recorded for their inputs run, each as an ordinary queued job.
+//
+// A result's identity is its content key: sha256 of (graph, step, version, the
+// element, the hash of every need). A changed input changes the keys that depend
+// on it and nothing else, so invalidation needs no bookkeeping.
+//
+// Run state changes only under a row lock on the run (`advance` and every writer
+// take it before touching tasks), so steps finishing together serialize.
+import { randomUUID } from 'node:crypto';
+import { sql } from '../../data/db/sql.js';
+import { schema } from '../../data/schema/namespace.js';
+import { fitting } from '../../request/di/index.js';
+import { pipeRaw } from '../../data/db/db.js';
+import { getContext, runInContext } from '../../request/context/store.js';
+import { createJobContext } from '../../request/context/create.js';
+import { bindTransaction } from '../../request/context/guard.js';
+import { propagateLocals } from '../../request/context/run-in-transaction.js';
+import { parentTraceRef } from '../../trace/entry/parent-trace.js';
+import { serializeForTrace } from '../../trace/capture/bounded-serializer.js';
+import { getBaseLogger } from '../../core/logging/index.js';
+import { canonicalJson, inputHash, sha256Hex, valueHash } from './hash.js';
+import { SINGLE, registerStepTables } from './table.js';
+const RUN = 'pipework_step_run';
+const TASK = 'pipework_step_task';
+const RESULT = 'pipework_step_result';
+const NAME_RE = /^[A-Za-z][A-Za-z0-9_.-]*$/;
+const MS_PER_DAY = 86_400_000;
+function fail(graph, message) {
+    throw new Error(`[pipework] steps.define("${graph}"): ${message}\n`);
+}
+function toJson(value) {
+    return value === undefined ? null : JSON.parse(JSON.stringify(value));
+}
+function stepDefinitionChecks(config) {
+    const graph = config.name;
+    if (typeof graph !== 'string' || !NAME_RE.test(graph)) {
+        throw new Error(`[pipework] steps.define: graph name "${String(graph)}" must start with a letter and use letters, digits, "_", "." or "-".\n`);
+    }
+    const inputs = new Set(config.inputs ?? []);
+    const names = Object.keys(config.steps);
+    if (names.length === 0)
+        fail(graph, 'no steps.');
+    for (const name of names) {
+        const def = config.steps[name];
+        if (!NAME_RE.test(name))
+            fail(graph, `step name "${name}" must start with a letter and use letters, digits, "_", "." or "-".`);
+        if (inputs.has(name))
+            fail(graph, `"${name}" is both a request input and a step.`);
+        if (!Number.isInteger(def.version))
+            fail(graph, `step "${name}" needs an integer version.`);
+        for (const need of def.needs ?? []) {
+            if (!inputs.has(need) && config.steps[need] === undefined) {
+                fail(graph, `step "${name}" needs "${need}", which is neither a step nor a declared input (steps: ${names.join(', ')}; inputs: ${[...inputs].join(', ') || '(none)'}).`);
+            }
+        }
+        if (def.person === true) {
+            if (def.forEach !== undefined)
+                fail(graph, `step "${name}" is a person step and cannot have forEach.`);
+            if (def.run !== undefined)
+                fail(graph, `step "${name}" is a person step and cannot have run: its value comes from decide() or autoAccept.`);
+        }
+        else {
+            if (def.run === undefined)
+                fail(graph, `step "${name}" has no run.`);
+            if (def.autoAccept !== undefined)
+                fail(graph, `step "${name}" has autoAccept but is not a person step.`);
+        }
+    }
+    // Topological order; a back edge is a cycle.
+    const order = [];
+    const state = new Map();
+    const visit = (name, path) => {
+        const seen = state.get(name);
+        if (seen === 'done')
+            return;
+        if (seen === 'visiting')
+            fail(graph, `steps form a cycle: ${[...path.slice(path.indexOf(name)), name].join(' -> ')}.`);
+        state.set(name, 'visiting');
+        for (const need of config.steps[name].needs ?? []) {
+            if (config.steps[need] !== undefined)
+                visit(need, [...path, name]);
+        }
+        state.set(name, 'done');
+        order.push(name);
+    };
+    for (const name of names)
+        visit(name, []);
+    return { order, inputs };
+}
+export function defineGraph(config) {
+    const { order, inputs: inputNames } = stepDefinitionChecks(config);
+    registerStepTables();
+    const graph = config.name;
+    const jobType = `pipework.step.${graph}`;
+    const steps = config.steps;
+    const queue = config.queue;
+    const isFanOut = (name) => steps[name].forEach !== undefined;
+    const running = new Map();
+    /** Every step the targets need, transitively, in dependency order. */
+    function closureOf(targets) {
+        const wanted = new Set();
+        const add = (name) => {
+            if (wanted.has(name))
+                return;
+            wanted.add(name);
+            for (const need of steps[name].needs ?? [])
+                if (steps[need] !== undefined)
+                    add(need);
+        };
+        for (const t of targets)
+            add(t);
+        return order.filter(n => wanted.has(n));
+    }
+    /** The steps that depend on `step`, directly or not. */
+    function dependentsOf(step) {
+        const out = new Set();
+        const grow = (name) => {
+            for (const other of order) {
+                if (!out.has(other) && (steps[other].needs ?? []).includes(name)) {
+                    out.add(other);
+                    grow(other);
+                }
+            }
+        };
+        grow(step);
+        return out;
+    }
+    function checkRequest(opts, what) {
+        if (typeof opts.item !== 'string' || opts.item === '')
+            throw new Error(`[pipework] ${graph}.${what}: item must be a non-empty string.\n`);
+        if (!Array.isArray(opts.targets) || opts.targets.length === 0)
+            throw new Error(`[pipework] ${graph}.${what}: targets must name at least one step.\n`);
+        for (const t of opts.targets) {
+            if (steps[t] === undefined) {
+                throw new Error(`[pipework] ${graph}.${what}: unknown target step "${t}" (steps: ${order.join(', ')}).\n`);
+            }
+        }
+        for (const name of Object.keys(opts.inputs ?? {})) {
+            if (!inputNames.has(name)) {
+                throw new Error(`[pipework] ${graph}.${what}: unknown input "${name}" (inputs: ${[...inputNames].join(', ') || '(none)'}).\n`);
+            }
+        }
+        const closure = closureOf(opts.targets);
+        for (const name of closure) {
+            for (const need of steps[name].needs ?? []) {
+                if (inputNames.has(need) && (opts.inputs ?? {})[need] === undefined) {
+                    throw new Error(`[pipework] ${graph}.${what}: step "${name}" needs input "${need}", which the request does not give.\n`);
+                }
+            }
+        }
+        return closure;
+    }
+    function tenantOf(explicit, what) {
+        const tenant = explicit ?? getContext()?.tenant ?? null;
+        if (tenant === null || tenant === '') {
+            throw new Error(`[pipework] ${graph}.${what}: no tenant. Pass { tenant } or call inside a tenant Flow; results are recorded per tenant.\n`);
+        }
+        return tenant;
+    }
+    let attached;
+    /** The session variable the tenant policies read: the graph's own, else the active or attached Manifold's, else the default. */
+    function sessionVarOf(instance) {
+        if (config.sessionVar !== undefined)
+            return config.sessionVar;
+        const m = instance ?? getContext()?.instance ?? attached;
+        if (m === undefined)
+            return 'pipework.tenant_id';
+        return m.config.database(m.config.defaultDatabaseName()).rls?.sessionVar ?? 'pipework.tenant_id';
+    }
+    /** Sets the tenant session variable for the rest of the transaction `tx` is in, and puts back what was there. */
+    async function asTenant(tx, tenant, fn, instance) {
+        if (tenant === null)
+            return fn();
+        const sessionVar = sessionVarOf(instance);
+        const before = await tx.execute(sql `SELECT current_setting(${sessionVar}, true) AS v`);
+        await tx.execute(sql `SELECT set_config(${sessionVar}, ${tenant}, true)`);
+        const result = await fn();
+        await tx.execute(sql `SELECT set_config(${sessionVar}, ${before[0]?.['v'] ?? ''}, true)`);
+        return result;
+    }
+    async function inTenantTx(db, tenant, fn) {
+        return db.transaction(async (tx) => asTenant(tx, tenant, () => fn(tx)));
+    }
+    /** The tenant a call acts for: the one given, else the Flow's, else none (an owner connection needs none). */
+    const scopeOf = (explicit) => explicit ?? getContext()?.tenant ?? null;
+    /** Cancels the jobs that are still pending in one statement, skipping any row another transaction holds: never waits on a job while the run lock is held. */
+    async function cancelPending(tx, jobIds) {
+        if (jobIds.length === 0)
+            return;
+        const table = quoted(queueTable());
+        await tx.execute(sql `
+      UPDATE ${sql.raw(table)} SET status = 'cancelled', completed_at = now(), claim_token = NULL
+       WHERE id IN (SELECT id FROM ${sql.raw(table)}
+                     WHERE id IN (${sql.join(jobIds.map(id => sql `${id}::uuid`), sql `, `)}) AND status = 'pending'
+                     FOR UPDATE SKIP LOCKED)`);
+    }
+    async function lockRun(tx, runId) {
+        const rows = await tx.execute(sql `
+      SELECT id::text, tenant_id::text AS tenant, item, status, priority, targets, inputs,
+             trace_id::text AS trace_id, trace_root_id::text AS trace_root_id, trace_step_id::text AS trace_step_id
+        FROM ${sql.raw(RUN)} WHERE id = ${runId}::uuid AND graph = ${graph} FOR UPDATE`);
+        const row = rows[0];
+        if (row === undefined)
+            return null;
+        const traceId = row['trace_id'];
+        const rootId = row['trace_root_id'];
+        return {
+            id: row['id'],
+            tenant: row['tenant'],
+            item: row['item'],
+            status: row['status'],
+            priority: row['priority'],
+            targets: row['targets'],
+            inputs: row['inputs'],
+            parent: traceId !== null && rootId !== null ? { id: traceId, rootId, stepId: row['trace_step_id'] } : null,
+        };
+    }
+    async function loadTasks(tx, runId) {
+        const rows = await tx.execute(sql `
+      SELECT id::text, step, instance, status, job_id::text AS job_id, input_key, result_id::text AS result_id, element
+        FROM ${sql.raw(TASK)} WHERE run_id = ${runId}::uuid ORDER BY step, instance`);
+        return rows.map(r => ({
+            id: r['id'],
+            step: r['step'],
+            instance: r['instance'],
+            status: r['status'],
+            jobId: r['job_id'],
+            inputKey: r['input_key'],
+            resultId: r['result_id'],
+            element: r['element'],
+        }));
+    }
+    async function loadResults(tx, ids) {
+        const out = new Map();
+        if (ids.length === 0)
+            return out;
+        const rows = await tx.execute(sql `
+      SELECT id::text, value, value_hash FROM ${sql.raw(RESULT)}
+       WHERE id IN (${sql.join(ids.map(id => sql `${id}::uuid`), sql `, `)})`);
+        for (const r of rows)
+            out.set(r['id'], { value: r['value'], hash: r['value_hash'] });
+        return out;
+    }
+    async function findRecorded(tx, tenant, item, step, key, instance) {
+        const rows = await tx.execute(sql `
+      SELECT id::text, value, value_hash FROM ${sql.raw(RESULT)}
+       WHERE tenant_id = ${tenant}::uuid AND graph = ${graph} AND item = ${item} AND step = ${step}
+         AND version = ${steps[step].version} AND input_key = ${key}
+       ORDER BY (instance = ${instance}) DESC, created_at ASC LIMIT 1`);
+        const row = rows[0];
+        return row === undefined ? null : { id: row['id'], value: row['value'], hash: row['value_hash'] };
+    }
+    /** Records a result; when one already holds the key (a concurrent run got there) that one is returned. */
+    async function recordResult(tx, run, step, key, instance, value, replace) {
+        const json = JSON.stringify(value);
+        const hash = valueHash(value);
+        const conflict = replace
+            ? sql `ON CONFLICT (tenant_id, graph, item, step, version, input_key, instance)
+              DO UPDATE SET value = EXCLUDED.value, value_hash = EXCLUDED.value_hash, created_at = now()`
+            : sql `ON CONFLICT (tenant_id, graph, item, step, version, input_key, instance) DO NOTHING`;
+        const inserted = await tx.execute(sql `
+      INSERT INTO ${sql.raw(RESULT)} (tenant_id, graph, item, step, version, input_key, instance, value, value_hash)
+      VALUES (${run.tenant}::uuid, ${graph}, ${run.item}, ${step}, ${steps[step].version}, ${key}, ${instance}, ${json}::jsonb, ${hash})
+      ${conflict} RETURNING id::text`);
+        if (inserted[0] !== undefined)
+            return { id: inserted[0]['id'], hash, value };
+        const rows = await tx.execute(sql `
+      SELECT id::text, value, value_hash FROM ${sql.raw(RESULT)}
+       WHERE tenant_id = ${run.tenant}::uuid AND graph = ${graph} AND item = ${run.item} AND step = ${step}
+         AND version = ${steps[step].version} AND input_key = ${key} AND instance = ${instance}`);
+        const row = rows[0];
+        return { id: row['id'], hash: row['value_hash'], value: row['value'] };
+    }
+    function stageReused(step, instance, resultId) {
+        const buffer = getContext()?.trace?.buffer;
+        if (buffer === undefined)
+            return;
+        const now = new Date();
+        buffer.stageExternal({
+            id: randomUUID(),
+            parentStepId: null,
+            flowStepRef: 'external:pipework.step.reused',
+            status: 'ok',
+            startedAt: now,
+            completedAt: now,
+            durationMicros: 0n,
+            capturedInputs: [serializeForTrace({ step, instance: instance === SINGLE ? null : instance, resultId })],
+            error: null,
+        });
+    }
+    /** The state of one run during an `advance`: its tasks and the results they hold. */
+    class RunState {
+        run;
+        tasks;
+        results;
+        inputHashes;
+        constructor(run, tasks, results, inputHashes) {
+            this.run = run;
+            this.tasks = tasks;
+            this.results = results;
+            this.inputHashes = inputHashes;
+        }
+        of(step) { return this.tasks.filter(t => t.step === step); }
+        single(step) { return this.tasks.find(t => t.step === step && t.instance === SINGLE); }
+        instances(step) {
+            return this.tasks.filter(t => t.step === step && t.instance >= 0).sort((a, b) => a.instance - b.instance);
+        }
+        good(t) { return t.status === 'done' || t.status === 'reused'; }
+        complete(step) {
+            const single = this.single(step);
+            if (single === undefined)
+                return false;
+            if (!isFanOut(step))
+                return this.good(single);
+            return single.status === 'done' && this.instances(step).every(t => this.good(t));
+        }
+        resultOf(t) {
+            const r = t.resultId === null ? undefined : this.results.get(t.resultId);
+            if (r === undefined)
+                throw new Error(`[pipework] ${graph}: task ${t.id} (${t.step}) has no recorded result.\n`);
+            return r;
+        }
+        /** What a step reads: the value and hash of each need, and the request inputs among them. */
+        gather(def) {
+            const values = {};
+            const hashes = {};
+            const inputs = {};
+            for (const need of def.needs ?? []) {
+                if (inputNames.has(need)) {
+                    values[need] = this.run.inputs[need];
+                    inputs[need] = this.run.inputs[need];
+                    hashes[need] = this.inputHashes[need];
+                }
+                else if (isFanOut(need)) {
+                    const rs = this.instances(need).map(t => this.resultOf(t));
+                    values[need] = rs.map(r => r.value);
+                    hashes[need] = rs.map(r => r.hash);
+                }
+                else {
+                    const r = this.resultOf(this.single(need));
+                    values[need] = r.value;
+                    hashes[need] = r.hash;
+                }
+            }
+            return { values, hashes, inputs };
+        }
+    }
+    function inputHashesOf(inputs) {
+        return Object.fromEntries(Object.entries(inputs).map(([k, v]) => [k, inputHash(v)]));
+    }
+    function contentKey(step, element, fanOut, hashes) {
+        return sha256Hex(canonicalJson({
+            graph, step, version: steps[step].version, element: fanOut ? valueHash(element) : null, needs: hashes,
+        }));
+    }
+    async function loadState(tx, run) {
+        const tasks = await loadTasks(tx, run.id);
+        const results = await loadResults(tx, tasks.flatMap(t => t.resultId === null ? [] : [t.resultId]));
+        return new RunState(run, tasks, results, inputHashesOf(run.inputs));
+    }
+    function stepContext(state, def, task, fanOut, signal) {
+        const g = state.gather(def);
+        return {
+            needs: g.values,
+            element: fanOut ? task.element : undefined,
+            index: fanOut ? task.instance : null,
+            item: state.run.item,
+            inputs: g.inputs,
+            runId: state.run.id,
+            signal,
+        };
+    }
+    async function setTask(tx, t, patch) {
+        await tx.execute(sql `
+      UPDATE ${sql.raw(TASK)} SET status = ${patch.status},
+             job_id = COALESCE(${patch.jobId ?? null}::uuid, job_id),
+             input_key = COALESCE(${patch.inputKey ?? null}, input_key),
+             result_id = COALESCE(${patch.resultId ?? null}::uuid, result_id),
+             error = COALESCE(${patch.error === undefined ? null : JSON.stringify(patch.error)}::jsonb, error)
+       WHERE id = ${t.id}::uuid`);
+        t.status = patch.status;
+        if (patch.jobId !== undefined)
+            t.jobId = patch.jobId;
+        if (patch.inputKey !== undefined)
+            t.inputKey = patch.inputKey;
+        if (patch.resultId !== undefined)
+            t.resultId = patch.resultId;
+    }
+    /**
+     * Moves a run as far as its recorded results allow, in one pass over the steps in
+     * dependency order: expands fan-out steps whose list is now known, reuses recorded
+     * results, queues the rest, and marks the run done when nothing is left. The run
+     * row is locked first, so concurrent finishes advance it one at a time.
+     */
+    async function advance(tx, runId) {
+        const outcome = { reused: 0, queued: 0, completed: false };
+        const run = await lockRun(tx, runId);
+        if (run === null || run.status !== 'running')
+            return { ...outcome, run };
+        const state = await loadState(tx, run);
+        const closure = closureOf(run.targets);
+        const signal = new AbortController().signal;
+        const process = async (task, def, fanOut) => {
+            const g = state.gather(def);
+            const key = contentKey(task.step, task.element, fanOut, g.hashes);
+            const recorded = await findRecorded(tx, run.tenant, run.item, task.step, key, task.instance);
+            if (recorded !== null) {
+                state.results.set(recorded.id, { value: recorded.value, hash: recorded.hash });
+                await setTask(tx, task, { status: 'reused', inputKey: key, resultId: recorded.id });
+                stageReused(task.step, task.instance, recorded.id);
+                outcome.reused++;
+                return;
+            }
+            if (def.person === true) {
+                const accepted = def.autoAccept === undefined ? undefined : await def.autoAccept(stepContext(state, def, task, fanOut, signal));
+                if (accepted !== undefined) {
+                    const rec = await recordResult(tx, run, task.step, key, task.instance, toJson(accepted), false);
+                    state.results.set(rec.id, { value: rec.value, hash: rec.hash });
+                    await setTask(tx, task, { status: 'done', inputKey: key, resultId: rec.id });
+                }
+                else {
+                    await setTask(tx, task, { status: 'waiting', inputKey: key });
+                }
+                return;
+            }
+            const jobId = await queue.enqueue(tx, {
+                type: jobType,
+                payload: { kind: 'step', runId: run.id, taskId: task.id },
+                priority: run.priority,
+                ...(def.maxAttempts !== undefined ? { maxAttempts: def.maxAttempts } : {}),
+                tenant: run.tenant,
+                parent: run.parent,
+            });
+            await setTask(tx, task, { status: 'queued', inputKey: key, jobId });
+            outcome.queued++;
+        };
+        for (const name of closure) {
+            if (state.complete(name))
+                continue;
+            const def = steps[name];
+            const ready = (def.needs ?? []).every(n => steps[n] === undefined || state.complete(n));
+            if (!ready)
+                continue;
+            const expander = state.single(name);
+            if (expander === undefined)
+                continue;
+            if (!isFanOut(name)) {
+                if (expander.status === 'blocked')
+                    await process(expander, def, false);
+                continue;
+            }
+            if (expander.status === 'blocked') {
+                const g = state.gather(def);
+                const forEachCtx = { needs: g.values, item: run.item, inputs: g.inputs, runId: run.id };
+                const list = await def.forEach(forEachCtx);
+                if (!Array.isArray(list)) {
+                    throw new Error(`[pipework] ${graph}: forEach of step "${name}" must return an array, got ${typeof list}.\n`);
+                }
+                const elements = list.map(toJson);
+                if (elements.length > 0) {
+                    const rows = await tx.execute(sql `
+            INSERT INTO ${sql.raw(TASK)} (tenant_id, run_id, step, instance, element, status)
+            VALUES ${sql.join(elements.map((el, i) => sql `(${run.tenant}::uuid, ${run.id}::uuid, ${name}, ${i}, ${JSON.stringify(el)}::jsonb, 'blocked')`), sql `, `)}
+            RETURNING id::text, instance`);
+                    for (const r of rows) {
+                        const i = r['instance'];
+                        state.tasks.push({
+                            id: r['id'], step: name, instance: i, status: 'blocked', jobId: null, inputKey: null, resultId: null, element: elements[i],
+                        });
+                    }
+                }
+                await setTask(tx, expander, { status: 'done' });
+            }
+            for (const t of state.instances(name))
+                if (t.status === 'blocked')
+                    await process(t, def, true);
+        }
+        if (closure.every(n => state.complete(n))) {
+            const done = await tx.execute(sql `
+        UPDATE ${sql.raw(RUN)} SET status = 'done', finished_at = now(), notified = ${config.onDone === undefined}
+         WHERE id = ${run.id}::uuid AND status = 'running' RETURNING id`);
+            if (done.length > 0) {
+                outcome.completed = true;
+                await tx.execute(sql `SELECT pg_notify('pipework_step_run', ${run.id})`);
+            }
+        }
+        return { ...outcome, run };
+    }
+    /** Marks the run's done callback delivered and calls it; false when it already was. */
+    async function deliverDone(tx, runId) {
+        const rows = await tx.execute(sql `
+      UPDATE ${sql.raw(RUN)} SET notified = true
+       WHERE id = ${runId}::uuid AND status = 'done' AND notified = false
+       RETURNING item, tenant_id::text AS tenant`);
+        const row = rows[0];
+        if (row === undefined || config.onDone === undefined)
+            return;
+        await config.onDone({ runId, item: row['item'], tenant: row['tenant'] });
+    }
+    /** After an advance outside a step job: a finished run's callback is delivered by a job, which has a Flow to run it in. */
+    async function scheduleDone(tx, run, outcome) {
+        if (run === null || !outcome.completed || config.onDone === undefined)
+            return;
+        await queue.enqueue(tx, {
+            type: jobType, payload: { kind: 'finish', runId: run.id, taskId: run.id }, priority: run.priority, tenant: run.tenant, parent: run.parent,
+        });
+    }
+    async function request(db, opts) {
+        const closure = checkRequest(opts, 'request');
+        const tenant = tenantOf(opts.tenant, 'request');
+        const priority = opts.priority ?? config.priority ?? 0;
+        const parent = parentTraceRef();
+        const inputs = toJson(opts.inputs ?? {});
+        return inTenantTx(db, tenant, async (tx) => {
+            const created = await tx.execute(sql `
+        INSERT INTO ${sql.raw(RUN)} (tenant_id, graph, item, targets, priority, inputs, trace_id, trace_root_id, trace_step_id)
+        VALUES (${tenant}::uuid, ${graph}, ${opts.item}, ${JSON.stringify(opts.targets)}::jsonb, ${priority}, ${JSON.stringify(inputs)}::jsonb,
+                ${parent?.id ?? null}::uuid, ${parent?.rootId ?? null}::uuid, ${parent?.stepId ?? null}::uuid)
+        RETURNING id::text`);
+            const runId = created[0]['id'];
+            await tx.execute(sql `
+        INSERT INTO ${sql.raw(TASK)} (tenant_id, run_id, step, instance, status)
+        VALUES ${sql.join(closure.map(step => sql `(${tenant}::uuid, ${runId}::uuid, ${step}, ${SINGLE}, 'blocked')`), sql `, `)}`);
+            const outcome = await advance(tx, runId);
+            await scheduleDone(tx, outcome.run, outcome);
+            return { runId, reused: outcome.reused, queued: outcome.queued };
+        });
+    }
+    async function plan(db, opts) {
+        const closure = checkRequest(opts, 'plan');
+        const tenant = tenantOf(opts.tenant, 'plan');
+        const inputs = toJson(opts.inputs ?? {});
+        const hashes = inputHashesOf(inputs);
+        return inTenantTx(db, tenant, async (tx) => {
+            // What each step would hold if the run were made now: its values, when every instance is recorded.
+            const known = new Map();
+            const result = {};
+            for (const name of closure) {
+                const def = steps[name];
+                const fanOut = isFanOut(name);
+                const needs = def.needs ?? [];
+                if (!needs.every(n => steps[n] === undefined || known.has(n))) {
+                    result[name] = { reusable: 0, toRun: fanOut ? null : 1 };
+                    continue;
+                }
+                const needValues = {};
+                const needHashes = {};
+                const given = {};
+                for (const n of needs) {
+                    if (inputNames.has(n)) {
+                        needValues[n] = inputs[n];
+                        given[n] = inputs[n];
+                        needHashes[n] = hashes[n];
+                    }
+                    else {
+                        const k = known.get(n);
+                        needValues[n] = isFanOut(n) ? k.values : k.values[0];
+                        needHashes[n] = isFanOut(n) ? k.hashes : k.hashes[0];
+                    }
+                }
+                const elements = fanOut
+                    ? (await def.forEach({ needs: needValues, item: opts.item, inputs: given, runId: '' })).map(toJson)
+                    : [undefined];
+                let reusable = 0;
+                const values = [];
+                const valueHashes = [];
+                for (const [i, element] of elements.entries()) {
+                    const key = contentKey(name, element, fanOut, needHashes);
+                    const rec = await findRecorded(tx, tenant, opts.item, name, key, fanOut ? i : SINGLE);
+                    if (rec !== null) {
+                        reusable++;
+                        values.push(rec.value);
+                        valueHashes.push(rec.hash);
+                    }
+                }
+                result[name] = { reusable, toRun: elements.length - reusable };
+                if (reusable === elements.length)
+                    known.set(name, { values, hashes: valueHashes });
+            }
+            return { steps: result };
+        });
+    }
+    async function status(db, runId, opts) {
+        return inTenantTx(db, scopeOf(opts?.tenant), tx => statusIn(tx, runId));
+    }
+    async function statusIn(db, runId) {
+        const runs = await db.execute(sql `SELECT status FROM ${sql.raw(RUN)} WHERE id = ${runId}::uuid AND graph = ${graph}`);
+        if (runs[0] === undefined)
+            throw new Error(`[pipework] ${graph}.status: no run "${runId}".\n`);
+        const tasks = await db.execute(sql `
+      SELECT t.step, t.instance, t.status, j.status AS job_status
+        FROM ${sql.raw(TASK)} t LEFT JOIN ${sql.raw(quoted(queueTable()))} j ON j.id = t.job_id
+       WHERE t.run_id = ${runId}::uuid`);
+        const zero = () => ({ total: 0, done: 0, reused: 0, running: 0, waiting: 0, failed: 0, cancelled: 0 });
+        const counts = new Map();
+        for (const t of tasks) {
+            const step = t['step'];
+            const fanOut = steps[step] !== undefined && isFanOut(step);
+            const c = counts.get(step) ?? zero();
+            counts.set(step, c);
+            if (fanOut && t['instance'] < 0)
+                continue;
+            c.total++;
+            const s = t['status'];
+            const job = t['job_status'];
+            if (s === 'done')
+                c.done++;
+            else if (s === 'reused')
+                c.reused++;
+            else if (s === 'failed')
+                c.failed++;
+            else if (s === 'cancelled')
+                c.cancelled++;
+            else if (s === 'queued' && (job === 'claimed' || job === 'running'))
+                c.running++;
+            else
+                c.waiting++;
+        }
+        const out = {};
+        for (const name of order) {
+            const c = counts.get(name);
+            if (c !== undefined)
+                out[name] = c;
+        }
+        return { status: runs[0]['status'], steps: out };
+    }
+    /** The queue's table name, read from its definition. */
+    function queueTable() {
+        return queue.definition.$name;
+    }
+    const quoted = (name) => `"${name.replace(/"/g, '""')}"`;
+    async function cancelRun(tx, runId) {
+        const run = await lockRun(tx, runId);
+        if (run === null || run.status !== 'running')
+            return;
+        await tx.execute(sql `UPDATE ${sql.raw(RUN)} SET status = 'cancelled', finished_at = now() WHERE id = ${runId}::uuid`);
+        const live = await tx.execute(sql `
+      UPDATE ${sql.raw(TASK)} SET status = 'cancelled'
+       WHERE run_id = ${runId}::uuid AND status IN ('blocked', 'queued', 'waiting') RETURNING job_id::text`);
+        await cancelPending(tx, live.flatMap(r => r['job_id'] === null ? [] : [r['job_id']]));
+        await tx.execute(sql `SELECT pg_notify('pipework_step_run', ${runId})`);
+        for (const c of running.get(runId) ?? [])
+            c.abort(new Error(`[pipework] ${graph} run ${runId} was cancelled.`));
+    }
+    async function cancel(db, runId, opts) {
+        await inTenantTx(db, scopeOf(opts?.tenant), async (tx) => { await cancelRun(tx, runId); });
+    }
+    async function decide(db, opts) {
+        const def = steps[opts.step];
+        if (def === undefined || def.person !== true) {
+            throw new Error(`[pipework] ${graph}.decide: "${opts.step}" is not a person step.\n`);
+        }
+        const instance = opts.index ?? SINGLE;
+        const value = toJson(opts.value);
+        await inTenantTx(db, scopeOf(opts.tenant), async (tx) => {
+            const run = await lockRun(tx, opts.runId);
+            if (run === null)
+                throw new Error(`[pipework] ${graph}.decide: no run "${opts.runId}".\n`);
+            const rows = await tx.execute(sql `
+        SELECT id::text, status, input_key, result_id::text AS result_id FROM ${sql.raw(TASK)}
+         WHERE run_id = ${run.id}::uuid AND step = ${opts.step} AND instance = ${instance} FOR UPDATE`);
+            const task = rows[0];
+            if (task === undefined) {
+                throw new Error(`[pipework] ${graph}.decide: run ${opts.runId} has no task for step "${opts.step}"${opts.index === undefined ? '' : ` instance ${String(opts.index)}`}.\n`);
+            }
+            const taskStatus = task['status'];
+            if (taskStatus === 'waiting') {
+                const rec = await recordResult(tx, run, opts.step, task['input_key'], instance, value, true);
+                await tx.execute(sql `UPDATE ${sql.raw(TASK)} SET status = 'done', result_id = ${rec.id}::uuid WHERE id = ${task['id']}::uuid`);
+                const outcome = await advance(tx, run.id);
+                await scheduleDone(tx, outcome.run, outcome);
+                return;
+            }
+            if (taskStatus === 'done' || taskStatus === 'reused') {
+                await tx.execute(sql `
+          UPDATE ${sql.raw(RESULT)} SET value = ${JSON.stringify(value)}::jsonb, value_hash = ${valueHash(value)}, created_at = now()
+           WHERE id = ${task['result_id']}::uuid`);
+                return;
+            }
+            throw new Error(`[pipework] ${graph}.decide: step "${opts.step}" of run ${opts.runId} is ${taskStatus}, not waiting for a decision.\n`);
+        });
+    }
+    async function wait(db, runId, opts) {
+        const deadline = Date.now() + opts.timeoutMs;
+        for (;;) {
+            const rows = await inTenantTx(db, scopeOf(opts.tenant), tx => tx.execute(sql `SELECT status FROM ${sql.raw(RUN)} WHERE id = ${runId}::uuid AND graph = ${graph}`));
+            if (rows[0] === undefined)
+                throw new Error(`[pipework] ${graph}.wait: no run "${runId}".\n`);
+            const s = rows[0]['status'];
+            if (s !== 'running')
+                return s;
+            if (Date.now() >= deadline)
+                return 'timeout';
+            await new Promise(r => setTimeout(r, Math.min(50, Math.max(1, deadline - Date.now()))));
+        }
+    }
+    async function sweep(db, opts) {
+        if (config.resultRetentionDays === undefined)
+            return 0;
+        const cutoff = new Date(Date.now() - config.resultRetentionDays * MS_PER_DAY).toISOString();
+        const tenant = scopeOf(opts?.tenant);
+        const only = tenant === null ? sql `` : sql ` AND tenant_id = ${tenant}::uuid`;
+        return inTenantTx(db, tenant, async (tx) => {
+            const old = await tx.execute(sql `
+        SELECT id::text FROM ${sql.raw(RUN)}
+         WHERE graph = ${graph} AND status <> 'running' AND finished_at < ${cutoff}::timestamptz${only}`);
+            if (old.length > 0) {
+                const ids = sql.join(old.map(r => sql `${r['id']}::uuid`), sql `, `);
+                await tx.execute(sql `DELETE FROM ${sql.raw(TASK)} WHERE run_id IN (${ids})`);
+                await tx.execute(sql `DELETE FROM ${sql.raw(RUN)} WHERE id IN (${ids})`);
+            }
+            const rows = await tx.execute(sql `
+        DELETE FROM ${sql.raw(RESULT)} WHERE graph = ${graph} AND created_at < ${cutoff}::timestamptz${only} RETURNING id`);
+            return rows.length;
+        });
+    }
+    async function removeTenant(db, tenantId) {
+        if (typeof tenantId !== 'string' || tenantId.trim() === '') {
+            throw new Error(`[pipework] ${graph}.removeTenant: tenantId must be a non-empty string.\n`);
+        }
+        await inTenantTx(db, tenantId, async (tx) => {
+            await tx.execute(sql `DELETE FROM ${sql.raw(TASK)} WHERE tenant_id = ${tenantId}::uuid`);
+            await tx.execute(sql `DELETE FROM ${sql.raw(RUN)} WHERE tenant_id = ${tenantId}::uuid`);
+            await tx.execute(sql `DELETE FROM ${sql.raw(RESULT)} WHERE tenant_id = ${tenantId}::uuid`);
+        });
+    }
+    /** The step job: runs one task's step, records its result and advances the run. */
+    async function handleJob(tx, payload) {
+        if (payload.kind === 'finish') {
+            await deliverDone(tx, payload.runId);
+            return { finished: payload.runId };
+        }
+        const live = await tx.execute(sql `
+      SELECT 1 FROM ${sql.raw(TASK)} t JOIN ${sql.raw(RUN)} r ON r.id = t.run_id
+       WHERE t.id = ${payload.taskId}::uuid AND t.run_id = ${payload.runId}::uuid AND t.status = 'queued' AND r.status = 'running'`);
+        if (live.length === 0)
+            return { skipped: payload.taskId };
+        // Read the run's state without the lock: the step may take a long time, and other steps of the run must proceed meanwhile.
+        const runRows = await tx.execute(sql `
+      SELECT id::text, tenant_id::text AS tenant, item, status, priority, targets, inputs FROM ${sql.raw(RUN)} WHERE id = ${payload.runId}::uuid`);
+        const r = runRows[0];
+        const run = {
+            id: r['id'], tenant: r['tenant'], item: r['item'], status: r['status'],
+            priority: r['priority'], targets: r['targets'], inputs: r['inputs'], parent: null,
+        };
+        const state = await loadState(tx, run);
+        const task = state.tasks.find(t => t.id === payload.taskId);
+        const def = steps[task.step];
+        const fanOut = isFanOut(task.step);
+        const controller = new AbortController();
+        let controllers = running.get(run.id);
+        if (controllers === undefined) {
+            controllers = new Set();
+            running.set(run.id, controllers);
+        }
+        controllers.add(controller);
+        let value;
+        try {
+            value = toJson(await def.run(stepContext(state, def, task, fanOut, controller.signal)));
+        }
+        finally {
+            controllers.delete(controller);
+            if (controllers.size === 0)
+                running.delete(run.id);
+        }
+        const locked = await lockRun(tx, run.id);
+        if (locked === null)
+            return { skipped: payload.taskId };
+        const current = await tx.execute(sql `SELECT status FROM ${sql.raw(TASK)} WHERE id = ${task.id}::uuid FOR UPDATE`);
+        const rec = await recordResult(tx, locked, task.step, task.inputKey, task.instance, value, false);
+        if (current[0]?.['status'] !== 'queued' || locked.status !== 'running') {
+            // Cancelled or failed while it ran: the result is kept for later requests, the run does not move.
+            return { recorded: rec.id };
+        }
+        await tx.execute(sql `UPDATE ${sql.raw(TASK)} SET status = 'done', result_id = ${rec.id}::uuid WHERE id = ${task.id}::uuid`);
+        const outcome = await advance(tx, run.id);
+        if (outcome.completed)
+            await deliverDone(tx, run.id);
+        return { step: task.step, resultId: rec.id };
+    }
+    function handler(database) {
+        const builder = database === undefined ? fitting.use() : fitting.use(database);
+        return builder.job(jobType)
+            .input(schema.check.object({ kind: schema.check.string(), runId: schema.check.string(), taskId: schema.check.string() }))
+            .fit(async ({ input }) => {
+            const tx = pipeRaw(database);
+            return asTenant(tx, getContext()?.tenant ?? null, () => handleJob(tx, input));
+        });
+    }
+    /** A step job of this graph dead-lettered: fail the task, what depends on it and the run, cancel the rest, tell the app once. */
+    async function onDeadLetter(tx, job, host) {
+        if (job.type !== jobType)
+            return;
+        const payload = job.payload;
+        if (payload.kind !== 'step' || payload.runId === undefined || payload.taskId === undefined)
+            return;
+        attached = host.instance;
+        await asTenant(tx, job.tenant, () => failRun(tx, job, payload.runId, payload.taskId, host), host.instance);
+    }
+    async function failRun(tx, job, runId, taskId, host) {
+        const run = await lockRun(tx, runId);
+        if (run === null || run.status !== 'running')
+            return;
+        const tasks = await loadTasks(tx, run.id);
+        const dead = tasks.find(t => t.id === taskId);
+        if (dead === undefined || dead.status !== 'queued')
+            return;
+        const dependents = dependentsOf(dead.step);
+        await setTask(tx, dead, { status: 'failed', error: job.error });
+        const cancelJobs = [];
+        for (const t of tasks) {
+            if (t === dead || (t.status !== 'blocked' && t.status !== 'queued' && t.status !== 'waiting'))
+                continue;
+            if (t.jobId !== null)
+                cancelJobs.push(t.jobId);
+            await setTask(tx, t, { status: dependents.has(t.step) ? 'failed' : 'cancelled' });
+        }
+        await cancelPending(tx, cancelJobs);
+        await tx.execute(sql `
+      UPDATE ${sql.raw(RUN)} SET status = 'failed', finished_at = now(), notified = true,
+             error = ${JSON.stringify({ step: dead.step, error: job.error })}::jsonb
+       WHERE id = ${run.id}::uuid`);
+        await tx.execute(sql `SELECT pg_notify('pipework_step_run', ${run.id})`);
+        if (config.onFailed === undefined)
+            return;
+        const failure = { runId: run.id, item: run.item, tenant: run.tenant, step: dead.step, error: job.error };
+        try {
+            await tx.transaction(async (savepoint) => {
+                const dbName = host.database ?? host.instance.config.defaultDatabaseName();
+                const base = createJobContext({ instance: host.instance, jobId: job.id, jobType: job.type, tenant: run.tenant });
+                const ctx = { ...base, transactions: new Map([[dbName, bindTransaction(base, dbName, savepoint)]]) };
+                await propagateLocals(savepoint, ctx, null);
+                await runInContext(ctx, async () => { await config.onFailed(failure); });
+            });
+        }
+        catch (err) {
+            getBaseLogger().error({ err, runId: run.id, graph }, 'Step graph onFailed callback failed; the run is already marked failed');
+        }
+    }
+    return {
+        name: graph,
+        jobType,
+        queue,
+        request, plan, status, cancel, decide, wait, sweep, removeTenant,
+        worker: { handler, onDeadLetter },
+    };
+}
+//# sourceMappingURL=graph.js.map