pipework 0.251.0 → 0.252.1

package/src/async/pipeline/execute.ts

@@ -0,0 +1,53 @@
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+import type { PipelineConfig, PipelineRun, PipelineExecuteContext } from './types.js'
+import { executeStepsFrom } from './run-steps.js'
+
+export async function executePipeline(
+  db: DB,
+  config: PipelineConfig,
+  input: unknown,
+  context: PipelineExecuteContext,
+): Promise<PipelineRun> {
+  const [runRow] = await db.execute(sql`
+    INSERT INTO pipework_pipeline_runs (pipeline_name, tenant_id, status, current_step, input, metadata)
+    VALUES (${config.name}, ${context.tenant}, 'running', 0, ${JSON.stringify(input)}::jsonb, ${JSON.stringify(context.metadata ?? {})}::jsonb)
+    RETURNING id, started_at
+  `)
+
+  const runId = String(runRow!['id'])
+  const startedAt = new Date(String(runRow!['started_at']))
+
+  for (let i = 0; i < config.steps.length; i++) {
+    const step = config.steps[i]!
+
+    await db.execute(sql`
+      INSERT INTO pipework_pipeline_steps (run_id, step_index, step_name, status)
+      VALUES (${runId}::uuid, ${i}, ${step.name}, 'pending')
+    `)
+  }
+
+  const result = await executeStepsFrom(
+    { db, config, runId, input, startedAt, tenant: context.tenant, metadata: context.metadata ?? {} },
+    0,
+    input,
+  )
+
+  if (!result.done) return result.waitingRun
+
+  await db.execute(sql`
+    UPDATE pipework_pipeline_runs SET status = 'completed', completed_at = now()
+    WHERE id = ${runId}::uuid
+  `)
+
+  return {
+    id: runId,
+    pipelineName: config.name,
+    status: 'completed',
+    currentStep: config.steps.length,
+    input,
+    startedAt,
+    completedAt: new Date(),
+    error: null,
+  }
+}

package/src/async/pipeline/index.ts

@@ -0,0 +1,13 @@
+export { definePipeline } from './define.js'
+export { PipelineStepError, PipelineResumeError } from './errors.js'
+export type {
+  PipelineConfig,
+  Pipeline,
+  PipelineRun,
+  PipelineExecuteContext,
+  StepDefinition,
+  StepContext,
+  StepRunResult,
+  PipelineStatus,
+  StepStatus,
+} from './types.js'

package/src/async/pipeline/namespace.ts

@@ -0,0 +1,12 @@
+import { definePipeline } from './define.js'
+import { PipelineStepError, PipelineResumeError } from './errors.js'
+
+/** Multi-step pipeline namespace — define, execute, pause, and resume ordered step sequences. */
+export const pipeline = {
+  /** Defines a pipeline from a config of named steps with handlers. Returns a Pipeline with execute/resume/getStatus. */
+  define: definePipeline,
+  /** Thrown when an individual pipeline step fails. */
+  PipelineStepError,
+  /** Thrown when resuming a pipeline from an invalid state. */
+  PipelineResumeError,
+}

package/src/async/pipeline/resume.ts

@@ -0,0 +1,67 @@
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+import type { PipelineConfig, PipelineRun } from './types.js'
+import { PipelineResumeError } from './errors.js'
+import { executeStepsFrom } from './run-steps.js'
+
+export async function resumePipeline(
+  db: DB,
+  config: PipelineConfig,
+  runId: string,
+  stepOutput: unknown,
+): Promise<PipelineRun> {
+  const [runRow] = await db.execute(sql`
+    UPDATE pipework_pipeline_runs
+    SET status = 'running'
+    WHERE id = ${runId}::uuid AND status = 'waiting'
+    RETURNING id, current_step, input, tenant_id, metadata, started_at
+  `)
+
+  if (runRow === undefined) {
+    const [existingRow] = await db.execute(sql`
+      SELECT status FROM pipework_pipeline_runs WHERE id = ${runId}::uuid
+    `)
+    if (existingRow === undefined) {
+      throw new PipelineResumeError(runId, 'Pipeline run not found.')
+    }
+    throw new PipelineResumeError(
+      runId,
+      `Expected status "waiting" but found "${String(existingRow['status'])}".`,
+    )
+  }
+
+  const currentStep = Number(runRow['current_step'])
+  const input = runRow['input']
+  const tenant = String(runRow['tenant_id'])
+  const metadata = (runRow['metadata'] as Record<string, unknown> | null) ?? {}
+  const startedAt = new Date(String(runRow['started_at']))
+
+  await db.execute(sql`
+    UPDATE pipework_pipeline_steps SET status = 'completed', output = ${JSON.stringify(stepOutput ?? null)}::jsonb, completed_at = now()
+    WHERE run_id = ${runId}::uuid AND step_index = ${currentStep}
+  `)
+
+  const result = await executeStepsFrom(
+    { db, config, runId, input, startedAt, tenant, metadata },
+    currentStep + 1,
+    stepOutput,
+  )
+
+  if (!result.done) return result.waitingRun
+
+  await db.execute(sql`
+    UPDATE pipework_pipeline_runs SET status = 'completed', current_step = ${config.steps.length}, completed_at = now()
+    WHERE id = ${runId}::uuid
+  `)
+
+  return {
+    id: runId,
+    pipelineName: config.name,
+    status: 'completed',
+    currentStep: config.steps.length,
+    input,
+    startedAt,
+    completedAt: new Date(),
+    error: null,
+  }
+}

package/src/async/pipeline/run-steps.ts

@@ -0,0 +1,98 @@
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+import type { PipelineConfig, PipelineRun, StepContext } from './types.js'
+import { PipelineStepError } from './errors.js'
+
+export interface StepLoopContext {
+  readonly db: DB
+  readonly config: PipelineConfig
+  readonly runId: string
+  readonly input: unknown
+  readonly startedAt: Date
+  readonly tenant: string
+  readonly metadata: Record<string, unknown>
+}
+
+export type StepLoopResult =
+  | { done: true; lastOutput: unknown }
+  | { done: false; waitingRun: PipelineRun }
+
+export async function executeStepsFrom(
+  ctx: StepLoopContext,
+  startIndex: number,
+  initialOutput: unknown,
+): Promise<StepLoopResult> {
+  const { db, config, runId, input, startedAt, tenant, metadata } = ctx
+  let lastOutput: unknown = initialOutput
+
+  for (let i = startIndex; i < config.steps.length; i++) {
+    const step = config.steps[i]!
+
+    if (step.async === true) {
+      await db.execute(sql`
+        UPDATE pipework_pipeline_runs SET status = 'waiting', current_step = ${i}
+        WHERE id = ${runId}::uuid
+      `)
+
+      return {
+        done: false,
+        waitingRun: {
+          id: runId,
+          pipelineName: config.name,
+          status: 'waiting',
+          currentStep: i,
+          input,
+          startedAt,
+          completedAt: null,
+          error: null,
+        },
+      }
+    }
+
+    await db.execute(sql`
+      UPDATE pipework_pipeline_steps SET status = 'running', started_at = now()
+      WHERE run_id = ${runId}::uuid AND step_index = ${i}
+    `)
+
+    try {
+      const stepCtx: StepContext = {
+        db,
+        input: lastOutput,
+        runId,
+        stepIndex: i,
+        tenant,
+        metadata,
+      }
+
+      lastOutput = await step.handler(stepCtx)
+
+      await db.execute(sql`
+        UPDATE pipework_pipeline_steps SET status = 'completed', output = ${JSON.stringify(lastOutput ?? null)}::jsonb, completed_at = now()
+        WHERE run_id = ${runId}::uuid AND step_index = ${i}
+      `)
+
+      await db.execute(sql`
+        UPDATE pipework_pipeline_runs SET current_step = ${i + 1}
+        WHERE id = ${runId}::uuid
+      `)
+    } catch (error) {
+      const errorJson = error instanceof Error
+        ? { message: error.message, stack: error.stack }
+        : { message: String(error) }
+
+      await db.execute(sql`
+        UPDATE pipework_pipeline_steps SET status = 'failed', error = ${JSON.stringify(errorJson)}::jsonb, completed_at = now()
+        WHERE run_id = ${runId}::uuid AND step_index = ${i}
+      `)
+
+      await db.execute(sql`
+        UPDATE pipework_pipeline_runs SET status = 'failed', error = ${JSON.stringify(errorJson)}::jsonb, completed_at = now()
+        WHERE id = ${runId}::uuid
+      `)
+
+      throw new PipelineStepError(config.name, step.name, i, error)
+    }
+  }
+
+  return { done: true, lastOutput }
+}

package/src/async/pipeline/types.ts

@@ -0,0 +1,57 @@
+import type { DB } from '../../data/db/index.js'
+
+export interface StepDefinition<TInput = unknown, TOutput = unknown> {
+  readonly name: string
+  readonly async?: boolean
+  readonly handler: (ctx: StepContext<TInput>) => Promise<TOutput>
+}
+
+export interface StepContext<TInput = unknown> {
+  readonly db: DB
+  readonly input: TInput
+  readonly runId: string
+  readonly stepIndex: number
+  readonly tenant: string
+  readonly metadata: Record<string, unknown>
+}
+
+export interface PipelineConfig {
+  readonly name: string
+  readonly database: string
+  readonly steps: readonly StepDefinition[]
+}
+
+export type PipelineStatus = 'running' | 'waiting' | 'completed' | 'failed'
+export type StepStatus = 'pending' | 'running' | 'completed' | 'failed'
+
+export interface PipelineRun {
+  readonly id: string
+  readonly pipelineName: string
+  readonly status: PipelineStatus
+  readonly currentStep: number
+  readonly input: unknown
+  readonly startedAt: Date
+  readonly completedAt: Date | null
+  readonly error: unknown | null
+}
+
+export interface StepRunResult {
+  readonly stepName: string
+  readonly status: StepStatus
+  readonly output: unknown | null
+  readonly startedAt: Date | null
+  readonly completedAt: Date | null
+  readonly error: unknown | null
+}
+
+export interface PipelineExecuteContext {
+  readonly tenant: string
+  readonly metadata?: Record<string, unknown>
+}
+
+export interface Pipeline {
+  execute(db: DB, input: unknown, context: PipelineExecuteContext): Promise<PipelineRun>
+  resume(db: DB, runId: string, stepOutput: unknown): Promise<PipelineRun>
+  getStatus(db: DB, runId: string): Promise<PipelineRun | null>
+  readonly definition: PipelineConfig
+}

package/src/async/state-machine/define.ts

@@ -0,0 +1,64 @@
+import type { StateMachineConfig, StateMachine, TransitionContext, TransitionResult } from './types.js'
+import { InvalidStateMachineError } from './errors.js'
+import { buildTransitionMap, getValidTargets, attemptTransition, evaluateGuard } from './transition.js'
+
+export function defineStateMachine(config: StateMachineConfig): StateMachine {
+  validate(config)
+
+  const transitionMap = buildTransitionMap(config.transitions)
+
+  return {
+    validTargets(from: string): readonly string[] {
+      return getValidTargets(transitionMap, from)
+    },
+
+    hasTransition(from: string, to: string): boolean {
+      return getValidTargets(transitionMap, from).includes(to)
+    },
+
+    async canTransition(context: TransitionContext): Promise<boolean> {
+      return evaluateGuard(transitionMap, context)
+    },
+
+    async transition(context: TransitionContext): Promise<TransitionResult> {
+      return attemptTransition(context, transitionMap)
+    },
+
+    definition: config,
+  }
+}
+
+function validate(config: StateMachineConfig): void {
+  if (config.states.length === 0) {
+    throw new InvalidStateMachineError('Must define at least one state.')
+  }
+
+  const stateSet = new Set(config.states)
+
+  if (stateSet.size !== config.states.length) {
+    const dupes = config.states.filter((s, i) => config.states.indexOf(s) !== i)
+    throw new InvalidStateMachineError(`Duplicate states: ${dupes.join(', ')}.`)
+  }
+
+  if (config.initial !== undefined && !stateSet.has(config.initial)) {
+    throw new InvalidStateMachineError(
+      `Initial state "${config.initial}" is not in the states list.`,
+    )
+  }
+
+  for (const t of config.transitions) {
+    if (!stateSet.has(t.from)) {
+      throw new InvalidStateMachineError(
+        `Transition references unknown source state "${t.from}".`,
+      )
+    }
+    const targets = typeof t.to === 'string' ? [t.to] : t.to
+    for (const target of targets) {
+      if (!stateSet.has(target)) {
+        throw new InvalidStateMachineError(
+          `Transition from "${t.from}" references unknown target state "${target}".`,
+        )
+      }
+    }
+  }
+}

package/src/async/state-machine/errors.ts

@@ -0,0 +1,29 @@
+import { PipeworkError } from '../../core/errors/base.js'
+
+export class InvalidTransitionError extends PipeworkError {
+  readonly code = 'INVALID_TRANSITION' as const
+  readonly httpStatus = 409 as const
+
+  constructor(
+    entityType: string,
+    from: string,
+    to: string,
+    validTargets: readonly string[],
+  ) {
+    const targets = validTargets.length > 0
+      ? `Valid targets from "${from}": ${validTargets.join(', ')}.`
+      : `No transitions defined from "${from}".`
+    super(
+      `Cannot transition ${entityType} from "${from}" to "${to}". ${targets}`,
+    )
+  }
+}
+
+export class InvalidStateMachineError extends PipeworkError {
+  readonly code = 'INVALID_STATE_MACHINE' as const
+  readonly httpStatus = 500 as const
+
+  constructor(message: string) {
+    super(`[pipework] Invalid state machine definition: ${message}`)
+  }
+}

package/src/async/state-machine/index.ts

@@ -0,0 +1,3 @@
+export { defineStateMachine } from './define.js'
+export { InvalidTransitionError, InvalidStateMachineError } from './errors.js'
+export type { StateMachineConfig, TransitionDef, TransitionContext, TransitionResult, StateMachine } from './types.js'

package/src/async/state-machine/namespace.ts

@@ -0,0 +1,12 @@
+import { defineStateMachine } from './define.js'
+import { InvalidTransitionError, InvalidStateMachineError } from './errors.js'
+
+/** State machine namespace — define valid transitions with guards and side effects. */
+export const state = {
+  /** Defines a state machine from states, transitions, guards, and effects. Returns a StateMachine with transition/canTransition. */
+  define: defineStateMachine,
+  /** Thrown when a transition is not allowed from the current state. */
+  InvalidTransitionError,
+  /** Thrown when the state machine definition is internally inconsistent. */
+  InvalidStateMachineError,
+}

package/src/async/state-machine/transition.ts

@@ -0,0 +1,90 @@
+import type { TransitionDef, TransitionContext, TransitionResult } from './types.js'
+import { InvalidTransitionError } from './errors.js'
+
+export function buildTransitionMap(transitions: readonly TransitionDef[]): Map<string, TransitionDef[]> {
+  const map = new Map<string, TransitionDef[]>()
+  for (const t of transitions) {
+    const existing = map.get(t.from) ?? []
+    existing.push(t)
+    map.set(t.from, existing)
+  }
+  return map
+}
+
+export function getValidTargets(transitionMap: Map<string, TransitionDef[]>, from: string): string[] {
+  const defs = transitionMap.get(from) ?? []
+  const targets: string[] = []
+  for (const def of defs) {
+    if (typeof def.to === 'string') {
+      targets.push(def.to)
+    } else {
+      targets.push(...def.to)
+    }
+  }
+  return targets
+}
+
+export function findMatchingDef(
+  transitionMap: Map<string, TransitionDef[]>,
+  from: string,
+  to: string,
+): TransitionDef | undefined {
+  const defs = transitionMap.get(from) ?? []
+  return defs.find(d =>
+    typeof d.to === 'string' ? d.to === to : d.to.includes(to),
+  )
+}
+
+export async function evaluateGuard(
+  transitionMap: Map<string, TransitionDef[]>,
+  context: TransitionContext,
+): Promise<boolean> {
+  const validTargets = getValidTargets(transitionMap, context.from)
+  if (!validTargets.includes(context.to)) {
+    return false
+  }
+
+  const matchingDef = findMatchingDef(transitionMap, context.from, context.to)
+
+  if (matchingDef?.guard !== undefined) {
+    return matchingDef.guard(context)
+  }
+
+  return true
+}
+
+export async function attemptTransition(
+  context: TransitionContext,
+  transitionMap: Map<string, TransitionDef[]>,
+): Promise<TransitionResult> {
+  const validTargets = getValidTargets(transitionMap, context.from)
+
+  if (!validTargets.includes(context.to)) {
+    throw new InvalidTransitionError(
+      context.entityType,
+      context.from,
+      context.to,
+      validTargets,
+    )
+  }
+
+  const matchingDef = findMatchingDef(transitionMap, context.from, context.to)
+
+  if (matchingDef?.guard !== undefined) {
+    const allowed = await matchingDef.guard(context)
+    if (!allowed) {
+      throw new InvalidTransitionError(
+        context.entityType,
+        context.from,
+        context.to,
+        validTargets,
+      )
+    }
+  }
+
+  return {
+    from: context.from,
+    to: context.to,
+    entityId: context.entityId,
+  }
+}

package/src/async/state-machine/types.ts

@@ -0,0 +1,37 @@
+import type { Audit } from '../../infra/audit/types.js'
+
+export interface TransitionDef {
+  readonly from: string
+  readonly to: string | readonly string[]
+  readonly guard?: (context: TransitionContext) => boolean | Promise<boolean>
+}
+
+export interface StateMachineConfig {
+  readonly states: readonly string[]
+  readonly transitions: readonly TransitionDef[]
+  readonly initial?: string
+  readonly audit?: Audit
+}
+
+export interface TransitionContext {
+  readonly entityId: string
+  readonly entityType: string
+  readonly from: string
+  readonly to: string
+  readonly userId?: string
+  readonly metadata?: Record<string, unknown>
+}
+
+export interface TransitionResult {
+  readonly from: string
+  readonly to: string
+  readonly entityId: string
+}
+
+export interface StateMachine {
+  validTargets(from: string): readonly string[]
+  hasTransition(from: string, to: string): boolean
+  canTransition(context: TransitionContext): Promise<boolean>
+  transition(context: TransitionContext): Promise<TransitionResult>
+  readonly definition: StateMachineConfig
+}

package/src/async/steps/errors.ts

@@ -0,0 +1,31 @@
+import { PipeworkError } from '../../core/errors/base.js'
+
+/** The reason a step's `signal` aborts with, and what the try fails with, when one try exceeds the step's `timeoutMs`. */
+export class StepTimeoutError extends PipeworkError {
+  readonly code = 'STEP_TIMEOUT' as const
+  readonly httpStatus = 504 as const
+
+  constructor(graph: string, step: string, timeoutMs: number) {
+    super(
+      `[pipework] ${graph}: step "${step}" timed out after ${timeoutMs}ms.\n\n` +
+      '  One try of the step did not finish within its timeoutMs. Raise timeoutMs on the step if the work legitimately takes longer.\n',
+      { graph, step, timeoutMs },
+    )
+    this.name = 'StepTimeoutError'
+  }
+}
+
+/** Thrown by `removeItem` when a run for the item is still in progress and `running: 'cancel'` was not passed. */
+export class StepItemRunningError extends PipeworkError {
+  readonly code = 'STEP_ITEM_RUNNING' as const
+  readonly httpStatus = 409 as const
+
+  constructor(graph: string, item: string, runIds: readonly string[]) {
+    super(
+      `[pipework] ${graph}.removeItem: item "${item}" has ${runIds.length} run(s) in progress (${runIds.join(', ')}).\n\n` +
+      "  Nothing was deleted. Pass { running: 'cancel' } to cancel them and delete, or wait for them to finish.\n",
+      { graph, item, runIds },
+    )
+    this.name = 'StepItemRunningError'
+  }
+}