pipework 0.252.0 → 0.253.0

package/src/core/surface/validate.ts

@@ -0,0 +1,88 @@
+import type { ResolvedConfig } from '../config/load.js'
+import type { Surface, SurfaceMap } from './types.js'
+import { DEFAULT_DB } from '../../request/di/types.js'
+
+export function validateSurfaces(config: ResolvedConfig, surfaces: SurfaceMap): void {
+  const problems: string[] = []
+  const configuredDbs = new Set(config.databaseNames())
+
+  for (const [name, surface] of Object.entries(surfaces)) {
+    for (const db of surface.manifest.databases) {
+      if (db === DEFAULT_DB) {
+        if (configuredDbs.size > 1) {
+          try {
+            config.defaultDatabaseName()
+          } catch {
+            problems.push(
+              `Surface "${name}" uses the default database, but multiple databases are configured.\n` +
+              `    Configured: ${[...configuredDbs].join(', ')}\n` +
+              `    Fix: Pass an explicit database name — e.g., surface.worker({ database: "${[...configuredDbs][0]}" })`,
+            )
+          }
+        }
+      } else if (!configuredDbs.has(db)) {
+        const available = [...configuredDbs].join(', ')
+        problems.push(
+          `Surface "${name}" requires database "${db}" which is not configured.\n` +
+          `    Configured databases: ${available || '(none)'}\n` +
+          `    Fix: Add "${db}" to your databases config, or correct the name in the surface definition.`,
+        )
+      }
+    }
+
+    validateSurfaceKind(name, surface, problems)
+  }
+
+  if (problems.length > 0) {
+    throw new Error(
+      `[pipework] Surface validation failed:\n\n${problems.map((p, i) => `  ${i + 1}. ${p}`).join('\n\n')}\n`,
+    )
+  }
+}
+
+function validateSurfaceKind(name: string, surface: Surface, problems: string[]): void {
+  if (surface.kind === 'worker') {
+    if (surface.manifest.jobTypes.length === 0) {
+      problems.push(
+        `Surface "${name}" is a worker with no job handlers.\n` +
+        '    Fix: Pass at least one job handler — surface.worker({ jobs: [myJob] })',
+      )
+    }
+
+    const seen = new Set<string>()
+    for (const jobType of surface.manifest.jobTypes) {
+      if (seen.has(jobType)) {
+        problems.push(
+          `Surface "${name}" has duplicate job type "${jobType}".\n` +
+          '    Each job type must have exactly one handler.',
+        )
+      }
+      seen.add(jobType)
+    }
+  }
+}
+
+export function validateNoJobTypeDuplicates(surfaces: SurfaceMap): void {
+  const owners = new Map<string, string>()
+  const problems: string[] = []
+
+  for (const [name, surface] of Object.entries(surfaces)) {
+    for (const jobType of surface.manifest.jobTypes) {
+      const existing = owners.get(jobType)
+      if (existing !== undefined) {
+        problems.push(
+          `Job type "${jobType}" is handled by both "${existing}" and "${name}".\n` +
+          '    Each job type must be handled by exactly one worker surface.',
+        )
+      } else {
+        owners.set(jobType, name)
+      }
+    }
+  }
+
+  if (problems.length > 0) {
+    throw new Error(
+      `[pipework] Job type conflict:\n\n${problems.map((p, i) => `  ${i + 1}. ${p}`).join('\n\n')}\n`,
+    )
+  }
+}

package/src/core/surface/worker.ts

@@ -0,0 +1,528 @@
+import { randomUUID } from 'node:crypto'
+import type { SurfaceBuilder, SurfaceHandle, WorkerSurfaceConfig, SurfaceContext, JobEvent } from './types.js'
+import type { Handler } from '../../request/di/types.js'
+import { DEFAULT_DB } from '../../request/di/types.js'
+import { createQueue } from '../../async/jobs/queue.js'
+import type { ClaimedFlow, ClaimedJob } from '../../async/jobs/queue.js'
+import { abortableDelay } from '../../async/jobs/abortable-delay.js'
+import type { ListenClient, ListenHandle } from '../../async/jobs/listener.js'
+import { executeJob } from '../../async/jobs/execute.js'
+import { JobLaterError, JobPermanentError } from '../../async/jobs/errors.js'
+import { QuotaExhaustedError } from '../../infra/quota/errors.js'
+import { executeWithTimeout } from './job-timeout.js'
+import { createJobContext } from '../../request/context/create.js'
+import { runInContext } from '../../request/context/store.js'
+import { executeInTransaction } from '../../request/context/run-in-transaction.js'
+import { isTracingEnabled } from '../../trace/entry/tracing-enabled.js'
+import { openTrace } from '../../trace/entry/open-trace.js'
+import { finalizeTrace } from '../../trace/entry/finalize-trace.js'
+import type { Manifold } from '../pipework.js'
+import { getBaseLogger } from '../logging/index.js'
+import { pgErrorCode } from '../../data/db/serializable.js'
+import { sql } from '../../data/db/sql.js'
+import { frameworkDoorDb } from '../../data/db/db.js'
+import { recordBeat, beatsEnabled } from '../../infra/alerts/beat.js'
+import { recordFault, faultMessage, useFaultTarget } from '../../infra/alerts/faults.js'
+import { parseCron } from '../../async/jobs/cron.js'
+import { registerSchedulesDefinition } from '../../async/jobs/schedules-table.js'
+import { describeError } from '../logging/error-chain.js'
+
+export function createWorkerSurface(config: WorkerSurfaceConfig): SurfaceBuilder {
+  const selections = (config.graphs ?? []).map(g => 'graph' in g ? g : { graph: g, classes: undefined })
+  const graphs = [...new Set(selections.map(g => g.graph))]
+  const graphHandlers = selections.flatMap(g => g.graph.worker.handlers(config.database, g.classes))
+  const allJobs = [...config.jobs, ...graphHandlers]
+  const databases = collectWorkerDatabases({ ...config, jobs: allJobs })
+  const jobTypes = collectJobTypes(allJobs)
+  validateJobHandlers(allJobs)
+  const deps: Record<string, string> = {}
+  // Created at surface construction, not start(): createQueue() registers the
+  // queue table definition, and `pipework generate` discovers surfaces without
+  // starting them — construction time is what makes the DDL reach migrations.
+  const queue = createQueue(config.queue)
+  for (const g of graphs) {
+    const graphTable = (g.queue.definition as unknown as { $name: string }).$name
+    const workerTable = (queue.definition as unknown as { $name: string }).$name
+    if (graphTable !== workerTable) {
+      throw new Error(
+        `[pipework] Step graph "${g.name}" enqueues on queue table "${graphTable}", but this worker surface reads "${workerTable}".\n\n` +
+        `  Set the worker's queue to the graph's: createWorkerSurface({ graphs, queue: { table: "${graphTable}" } }).\n`,
+      )
+    }
+  }
+  const schedules = config.schedules
+  if (schedules !== undefined) {
+    validateSchedules(schedules, jobTypes)
+    registerSchedulesDefinition()
+  }
+  const scheduledJobs = new Set((schedules ?? []).map(s => s.job))
+
+  const surface: SurfaceBuilder = {
+    kind: 'worker',
+    manifest: { databases, jobTypes, dependencies: deps },
+
+    dependencies(d: Record<string, string>) {
+      Object.assign(deps, d)
+      return surface
+    },
+
+    async start(context: SurfaceContext): Promise<SurfaceHandle> {
+      const logger = getBaseLogger()
+      const dbName = config.database ?? context.config.defaultDatabaseName()
+      const dbConfig = context.config.database(dbName)
+      const conn = context.pool.getOrCreate(dbConfig)
+      const db = conn.drizzle
+
+      await db.execute(sql`SELECT 1`)
+
+      // A tenant-column queue is row-secured: on a platform-mode database its claim, reap, heartbeat and
+      // completion run through the queue's framework door (generated by `pipework generate`), which opens
+      // that one table. Everything else the surface does (beats, faults, schedules) needs no door.
+      const queueDb = (dbConfig as { readonly rls?: { readonly platformRole?: string } }).rls?.platformRole !== undefined && queue.door !== null && (context.pool as { isOverridden?(name: string): boolean }).isOverridden?.(dbName) !== true
+        ? frameworkDoorDb(db, queue.door)
+        : db
+
+      // Loop beats go on the loop's own connection, outside any job transaction, and only where the blueprint has `alerts`.
+      const beatsOn = beatsEnabled(context.config)
+      const beat = (name: string, outcome: { ok: boolean; error?: string }, everyMs: number): void => {
+        if (beatsOn) void recordBeat(db, name, { ...outcome, everyMs })
+      }
+      // Failures this surface only logs are recorded for `alerts.source.faults`, again only where the blueprint has `alerts`.
+      const releaseFaults = beatsOn ? useFaultTarget(db) : (): void => undefined
+      const CLAIM_BEAT_EVERY_MS = 60_000
+      let lastClaimBeatAt = 0
+      const claimBeat = (outcome: { ok: boolean; error?: string }): void => {
+        const now = Date.now()
+        if (!beatsOn || now - lastClaimBeatAt < CLAIM_BEAT_EVERY_MS) return
+        lastClaimBeatAt = now
+        beat('jobs.claim', outcome, CLAIM_BEAT_EVERY_MS)
+      }
+
+      if (config.listen === true) {
+        const rawConn = conn as unknown as { readonly client: { listen(channel: string, cb: (payload: string) => void): Promise<{ unlisten(): Promise<void> }> } }
+        const listenClient: ListenClient = {
+          async listen(channel: string, cb: (payload: string) => void): Promise<ListenHandle> {
+            const sub = await rawConn.client.listen(channel, cb)
+            return { unlisten: () => sub.unlisten() }
+          },
+        }
+        await queue.startListener(listenClient)
+        logger.info('LISTEN/NOTIFY active for job completion')
+      }
+
+      // Schedules are in place before any claim loop runs, so a job the scheduler
+      // enqueues is never claimed ahead of its registration.
+      if (schedules !== undefined) {
+        const removed = await queue.syncSchedules(
+          db,
+          schedules.map(s => ({
+            name: s.job,
+            cron: s.cron,
+            ...(s.timezone !== undefined ? { timezone: s.timezone } : {}),
+            ...(s.payload !== undefined ? { payload: s.payload } : {}),
+          })),
+          jobTypes,
+        )
+        if (removed.length > 0) logger.info({ removed }, 'Removed schedules no longer declared')
+        queue.startScheduler(db, {
+          checkIntervalMs: config.schedulerIntervalMs ?? 15_000,
+          names: [...scheduledJobs],
+          onError: (err) => { logger.error({ err }, 'Scheduler tick failed'); void recordFault('scheduler', faultMessage(err)) },
+          onCheck: (outcome) => { beat('jobs.scheduler', outcome, config.schedulerIntervalMs ?? 15_000) },
+        })
+      }
+
+      const dispatch = buildDispatchTable(allJobs)
+      const handledTypes = [...dispatch.keys()]
+      const instance = context as unknown as Manifold
+      const releaseGraphs = graphs.map(g => queue.onDeadLetter(
+        (tx, job) => g.worker.onDeadLetter(tx, job, { instance, database: config.database }),
+      ))
+      const concurrency = config.concurrency ?? 1
+      const pollIntervalMs = config.pollIntervalMs ?? 1000
+      const shutdownDeadlineMs = config.shutdownDeadlineMs ?? 30_000
+      const heartbeatIntervalMs = config.heartbeatIntervalMs ?? 30_000
+      const priorityTimeouts = config.timeouts ?? {}
+
+      const maxConsecutiveClaimFailures = config.maxConsecutiveClaimFailures ?? 10
+
+      const shouldRun = { value: true }
+      // claim() retries transient conflicts until clean (unbounded), so
+      // shouldRun alone cannot end a loop stuck under contention — the abort
+      // signal is what makes shutdown observable inside claim and the sleeps.
+      const shutdown = new AbortController()
+      // Jobs this worker is running: the claim each holds and the controller that ends its handler.
+      // `endedBy` is set before the abort, so the loop knows the job was taken from it and must not fail it.
+      interface Running { readonly job: ClaimedJob; readonly controller: AbortController; endedBy: 'handed-back' | 'claim-lost' | null }
+      const running = new Map<string, Running>()
+      const eventJob = (job: ClaimedJob): JobEvent['job'] => ({
+        id: job.id, type: job.type, attempt: job.attempt, maxAttempts: job.maxAttempts, tenant: job.flow.tenant,
+      })
+
+      // Runs the app's callback as the job's tenant and actor, in its own transaction, as a child of the job's
+      // trace. Nothing it does reaches the caller: a throw is logged.
+      const emit = async (event: JobEvent, flow: ClaimedFlow): Promise<void> => {
+        const callback = config.onJobEvent
+        if (callback === undefined) return
+        try {
+          const auth = flow.actor !== null ? { userId: flow.actor } : null
+          const trace = isTracingEnabled(instance)
+            ? openTrace({
+              instance,
+              entryKind: 'job',
+              parent: flow.traceId !== null && flow.rootId !== null
+                ? { id: flow.traceId, rootId: flow.rootId, stepId: flow.stepId }
+                : null,
+              tenant: flow.tenant,
+              auth,
+              requestId: event.job.id,
+              sessionId: null,
+            })
+            : null
+          const ctx = createJobContext({
+            instance, jobId: event.job.id, jobType: event.job.type, attempt: event.job.attempt, maxAttempts: event.job.maxAttempts, tenant: flow.tenant, auth, trace,
+          })
+          const run = async (): Promise<void> => { await callback(event) }
+          const work = ctx.tenant === null
+            ? runInContext(ctx, run)
+            : executeInTransaction(ctx, [dbName], null, 5000, run)
+          await finalizeTrace(instance, ctx.trace, work)
+        } catch (err) {
+          logger.error({ err, kind: event.kind, jobId: event.job.id }, 'onJobEvent callback failed')
+          void recordFault('job-event-callback', faultMessage(err), { kind: event.kind, jobId: event.job.id })
+        }
+      }
+      // A claim that throws is never a quiet queue: it is logged with its
+      // Postgres code and counted. A successful claim clears the count; enough
+      // consecutive failures end the surface rather than let it poll forever.
+      let consecutiveClaimFailures = 0
+
+      const claimLoop = async () => {
+        const workerId = randomUUID()
+        while (shouldRun.value) {
+          try {
+            const claimed = await queue.claim(queueDb, workerId, shutdown.signal, handledTypes, instance.release)
+            consecutiveClaimFailures = 0
+            claimBeat({ ok: true })
+            if (claimed === null) {
+              await abortableDelay(pollIntervalMs, shutdown.signal)
+              continue
+            }
+
+            const handler = dispatch.get(claimed.type)
+
+            if (handler === undefined) {
+              const failedNoHandler = await queue.failAttempt(queueDb, claimed.id, new Error(
+                `[pipework] No handler registered for job type "${claimed.type}".\n\n` +
+                `  Registered types: ${[...dispatch.keys()].join(', ')}\n` +
+                '  This job was enqueued with a type that no handler recognizes.\n'
+              ), claimed.claimToken)
+              if (failedNoHandler?.outcome === 'dead_letter') {
+                await emit({ kind: 'dead-lettered', cause: 'no-handler', error: failedNoHandler.error, job: eventJob(claimed) }, claimed.flow)
+              }
+              continue
+            }
+
+            const entry: Running = { job: claimed, controller: new AbortController(), endedBy: null }
+            running.set(claimed.id, entry)
+            const heartbeatInterval = setInterval(() => {
+              queue.heartbeat(queueDb, claimed.id, claimed.claimToken).then((held) => {
+                // A loop busy with a long job is alive.
+                claimBeat({ ok: true })
+                if (held || entry.endedBy !== null) return
+                entry.endedBy = 'claim-lost'
+                entry.controller.abort(new Error(
+                  `[pipework] Job "${claimed.type}" (${claimed.id}) lost its claim: the row was reaped or taken by another worker.`,
+                ))
+                return emit({ kind: 'claim-lost', job: eventJob(claimed) }, claimed.flow)
+              }).catch((err: unknown) => {
+                logger.error({ err, jobId: claimed.id }, 'Job heartbeat failed')
+                void recordFault('queue-heartbeat', faultMessage(err), { jobId: claimed.id })
+              })
+            }, heartbeatIntervalMs)
+
+            const timedOut = { value: false }
+            try {
+              const timeoutMs = resolvePriorityTimeout(claimed, handler, priorityTimeouts)
+              const result = await executeWithTimeout(
+                (timeoutSignal) => {
+                  if (timeoutSignal !== undefined) {
+                    timeoutSignal.addEventListener('abort', () => {
+                      timedOut.value = true
+                      entry.controller.abort(timeoutSignal.reason)
+                    }, { once: true })
+                  }
+                  return executeJob(handler, {
+                    instance,
+                    jobId: claimed.id,
+                    jobType: claimed.type,
+                    attempt: claimed.attempt,
+                    maxAttempts: claimed.maxAttempts,
+                    // A scheduled type enqueued with no flow came from the scheduler: its own root trace.
+                    entryKind: scheduledJobs.has(claimed.type) && claimed.flow.traceId === null && claimed.flow.tenant === null
+                      ? 'cron'
+                      : 'job',
+                    input: claimed.payload,
+                    tenant: claimed.flow.tenant,
+                    parent: claimed.flow.traceId !== null && claimed.flow.rootId !== null
+                      ? { id: claimed.flow.traceId, rootId: claimed.flow.rootId, stepId: claimed.flow.stepId }
+                      : null,
+                    ...(claimed.flow.actor !== null ? { auth: { userId: claimed.flow.actor } } : {}),
+                    ...(claimed.previous !== null
+                      ? { restart: { reason: claimed.previous.reason, previousWorkerId: claimed.previous.workerId } }
+                      : {}),
+                    signal: entry.controller.signal,
+                  })
+                },
+                timeoutMs,
+                claimed.type,
+                claimed.id,
+              )
+              await queue.complete(queueDb, claimed.id, result, claimed.claimToken)
+            } catch (err) {
+              // A job taken from this worker (handed back, claim lost) is not its failure to record.
+              if (entry.endedBy === null) {
+                if (err instanceof JobLaterError) await queue.later(queueDb, claimed.id, err, claimed.claimToken)
+                else if (err instanceof QuotaExhaustedError) {
+                  const delayMs = Math.max(0, err.resetAt.getTime() - Date.now())
+                  await queue.later(queueDb, claimed.id, new JobLaterError(err.message, { delayMs, cause: err }), claimed.claimToken)
+                }
+                else {
+                  const permanent = err instanceof JobPermanentError
+                  const failed = await queue.failAttempt(queueDb, claimed.id, err, claimed.claimToken, permanent ? { permanent: true } : undefined)
+                  if (failed?.outcome === 'dead_letter') {
+                    const cause = permanent ? 'permanent' : timedOut.value ? 'timeout' : 'failed'
+                    await emit({ kind: 'dead-lettered', cause, error: failed.error, job: eventJob(claimed) }, claimed.flow)
+                  }
+                }
+              }
+            } finally {
+              clearInterval(heartbeatInterval)
+              running.delete(claimed.id)
+            }
+          } catch (err) {
+            if (shutdown.signal.aborted) return
+
+            consecutiveClaimFailures++
+            claimBeat({ ok: false, error: describeError(err).message })
+            logger.error(
+              { err, pgCode: pgErrorCode(err), consecutiveFailures: consecutiveClaimFailures, maxConsecutiveClaimFailures },
+              'Job claim failed',
+            )
+
+            if (consecutiveClaimFailures >= maxConsecutiveClaimFailures) {
+              logger.fatal(
+                { err, pgCode: pgErrorCode(err), consecutiveFailures: consecutiveClaimFailures },
+                `Worker surface stopping: ${String(consecutiveClaimFailures)} consecutive claim failures`,
+              )
+              shouldRun.value = false
+              shutdown.abort()
+              process.exitCode = 1
+              return
+            }
+
+            await abortableDelay(pollIntervalMs, shutdown.signal)
+          }
+        }
+      }
+
+      const workers = Array.from({ length: concurrency }, () => claimLoop())
+
+      let reaperTimer: ReturnType<typeof setInterval> | null = null
+      const reaperByDefault = config.reaper === undefined && scheduledJobs.size > 0
+      if (reaperByDefault) logger.info('Reaper on: this worker declares schedules (set reaper: false to turn it off)')
+      if (reaperByDefault || (config.reaper !== undefined && config.reaper !== false)) {
+        const reaperIntervalMs = typeof config.reaper === 'object'
+          ? (config.reaper.intervalMs ?? 60_000)
+          : 60_000
+        reaperTimer = setInterval(() => {
+          void queue.reap(queueDb).then(async (reaped) => {
+            beat('jobs.reaper', { ok: true }, reaperIntervalMs)
+            if (reaped.length === 0) return
+            logger.info({ reaped: reaped.length }, `Reaped ${String(reaped.length)} stale job(s)`)
+            for (const r of reaped) {
+              await emit({
+                kind: 'reaped',
+                outcome: r.outcome,
+                job: { id: r.jobId, type: r.type, attempt: r.attempt, maxAttempts: r.maxAttempts, tenant: r.flow.tenant },
+              }, r.flow)
+            }
+          }).catch((err) => {
+            beat('jobs.reaper', { ok: false, error: describeError(err).message }, reaperIntervalMs)
+            logger.error({ err }, 'Reaper failed')
+            void recordFault('queue-reaper', faultMessage(err))
+          })
+        }, reaperIntervalMs)
+        reaperTimer.unref()
+      }
+
+      return {
+        kind: 'worker',
+
+        fireSchedule(job: string): Promise<string> {
+          if (!scheduledJobs.has(job)) {
+            return Promise.reject(new Error(
+              `[pipework] No schedule declared for job "${job}" on this worker surface.\n\n` +
+              `  Declared: ${[...scheduledJobs].join(', ') || '(none)'}\n`,
+            ))
+          }
+          return queue.fireSchedule(db, job)
+        },
+
+        async stop(force?: AbortSignal) {
+          logger.info({ inFlight: running.size, deadlineMs: shutdownDeadlineMs }, 'Worker surface draining')
+          queue.stopScheduler()
+          shouldRun.value = false
+          shutdown.abort()
+
+          if (reaperTimer !== null) {
+            clearInterval(reaperTimer)
+            reaperTimer = null
+          }
+
+          if (config.listen === true) {
+            await queue.stopListener()
+          }
+
+          const deadline = Date.now() + shutdownDeadlineMs
+
+          // Defense in depth behind the abort signal: a claim loop that
+          // somehow ignores cancellation must not wedge stop(). The deadline
+          // leg is aborted when the loops win so its timer cannot hold the
+          // event loop open after a clean shutdown.
+          const deadlineAc = new AbortController()
+          const waitEnd = AbortSignal.any(force === undefined ? [deadlineAc.signal] : [deadlineAc.signal, force])
+          const loopsExited = await Promise.race([
+            Promise.allSettled(workers).then(() => { deadlineAc.abort(); return true }),
+            abortableDelay(shutdownDeadlineMs, waitEnd).then(() => false),
+          ])
+          if (!loopsExited) {
+            logger.warn('Claim loops did not exit within the shutdown deadline')
+          }
+
+          while (running.size > 0 && Date.now() < deadline && force?.aborted !== true) {
+            logger.info({ inFlight: running.size, remainingMs: deadline - Date.now() }, 'Waiting for in-flight jobs')
+            await abortableDelay(Math.min(500, deadline - Date.now()), force)
+          }
+
+          if (running.size > 0) {
+            const stranded = [...running.values()]
+            logger.warn({ inFlight: stranded.length }, 'Shutdown deadline reached with in-flight jobs; handing them back')
+            for (const entry of stranded) {
+              entry.endedBy = 'handed-back'
+              entry.controller.abort(new Error(
+                `[pipework] Job "${entry.job.type}" (${entry.job.id}) aborted: the worker is stopping and handed it back to the queue.`,
+              ))
+            }
+            for (const { job } of stranded) {
+              try {
+                if (await queue.handBack(queueDb, job.id, job.claimToken)) {
+                  await emit({ kind: 'handed-back', job: eventJob(job) }, job.flow)
+                }
+              } catch (err) {
+                logger.error({ err, jobId: job.id }, 'Handing back a job failed')
+                void recordFault('queue-handback', faultMessage(err), { jobId: job.id })
+              }
+            }
+          } else {
+            logger.info('Worker surface drained cleanly')
+          }
+          for (const release of releaseGraphs) release()
+          releaseFaults()
+        },
+      }
+    },
+  }
+
+  return surface
+}
+
+function buildDispatchTable(
+  jobs: readonly Handler<Record<string, unknown>, unknown>[],
+): Map<string, Handler<Record<string, unknown>, unknown>> {
+  const table = new Map<string, Handler<Record<string, unknown>, unknown>>()
+  for (const job of jobs) {
+    const jobType = job.meta.jobType
+    if (typeof jobType === 'string') {
+      table.set(jobType, job)
+    }
+  }
+  return table
+}
+
+function validateJobHandlers(jobs: readonly Handler<Record<string, unknown>, unknown>[]): void {
+  const seen = new Set<string>()
+  for (const job of jobs) {
+    if (!job.meta.isJob || job.meta.jobType === null) {
+      throw new Error(
+        '[pipework] All handlers passed to surface.worker({ jobs }) must be created with .job("type-name").\n\n' +
+        '  Example:\n' +
+        '    fitting().use().job("send-email").input(schema).fit(handler)\n'
+      )
+    }
+    if (seen.has(job.meta.jobType)) {
+      throw new Error(
+        `[pipework] Duplicate job type "${job.meta.jobType}" in worker surface.\n\n` +
+        '  Each job type must have exactly one handler. If you need multiple handlers,\n' +
+        '  create separate worker surfaces.\n'
+      )
+    }
+    seen.add(job.meta.jobType)
+  }
+}
+
+function validateSchedules(
+  schedules: readonly { readonly job: string; readonly cron: string }[],
+  jobTypes: readonly string[],
+): void {
+  const seen = new Set<string>()
+  for (const s of schedules) {
+    if (!jobTypes.includes(s.job)) {
+      throw new Error(
+        `[pipework] Schedule for job "${s.job}", but this worker surface handles no such job type.\n\n` +
+        `  Handled types: ${jobTypes.join(', ') || '(none)'}\n` +
+        '  A schedule fires a job this surface runs; put the schedule on the surface that handles it.\n',
+      )
+    }
+    if (seen.has(s.job)) {
+      throw new Error(`[pipework] Job "${s.job}" is scheduled twice on this worker surface; a job type has one schedule.\n`)
+    }
+    seen.add(s.job)
+    parseCron(s.cron)
+  }
+}
+
+function collectWorkerDatabases(config: WorkerSurfaceConfig): string[] {
+  const dbs = new Set<string>()
+  if (config.database !== undefined) {
+    dbs.add(config.database)
+  } else {
+    dbs.add(DEFAULT_DB)
+  }
+  for (const job of config.jobs) {
+    for (const db of job.meta.databases) {
+      dbs.add(db)
+    }
+  }
+  return [...dbs]
+}
+
+function collectJobTypes(jobs: readonly Handler<Record<string, unknown>, unknown>[]): string[] {
+  const types: string[] = []
+  for (const j of jobs) {
+    if (typeof j.meta.jobType === 'string') {
+      types.push(j.meta.jobType)
+    }
+  }
+  return types
+}
+
+function resolvePriorityTimeout(
+  claimed: { readonly priority: number },
+  handler: Handler<Record<string, unknown>, unknown>,
+  priorityTimeouts: Readonly<Record<number, number>>,
+): number | undefined {
+  const fromPriority = priorityTimeouts[claimed.priority]
+  if (fromPriority !== undefined) return fromPriority
+  return handler.meta.jobTimeout ?? undefined
+}

package/src/core/validation/connectivity.ts

@@ -0,0 +1,49 @@
+import type { ValidationIssue } from './types.js'
+import type { ResolvedDatabase } from '../config/load.js'
+
+export async function validateConnectivity(
+  databases: ReadonlyMap<string, ResolvedDatabase>,
+  timeoutMs = 5000,
+): Promise<ValidationIssue[]> {
+  const issues: ValidationIssue[] = []
+
+  const checks = [...databases.entries()].map(async ([name, db]) => {
+    try {
+      // A platform-mode database is reached at runtime as the platform role; the owner is not dialled.
+      await testConnection(db.rls?.platformRole !== undefined && db.systemUrl !== undefined ? db.systemUrl : db.url, timeoutMs)
+    } catch (err) {
+      issues.push({
+        phase: 'database',
+        message: `cannot connect to database "${name}": ${err instanceof Error ? err.message : String(err)}`,
+      })
+    }
+  })
+
+  await Promise.all(checks)
+  return issues
+}
+
+async function testConnection(url: string, timeoutMs: number): Promise<void> {
+  const { default: postgres } = await import('postgres')
+
+  const sql = postgres(url, {
+    max: 1,
+    idle_timeout: 1,
+    connect_timeout: Math.ceil(timeoutMs / 1000),
+    onnotice: () => {},
+  })
+
+  try {
+    await sql`SELECT 1`
+  } finally {
+    await sql.end({ timeout: 1 })
+  }
+}
+
+/**
+ * The probe's `connect_timeout` option. A number pins it; `null` omits it so
+ * postgres.js reads PGCONNECT_TIMEOUT (seconds), else its 30 s default.
+ */
+export function probeConnectTimeout(timeoutMs: number | null): { connect_timeout?: number } {
+  return timeoutMs === null ? {} : { connect_timeout: Math.ceil(timeoutMs / 1000) }
+}