pipework 0.252.0 → 0.252.1

package/src/async/jobs/execute.ts

@@ -0,0 +1,219 @@
+import type { Manifold } from '../../core/pipework.js'
+import type { Handler } from '../../request/di/types.js'
+import type { TenantConfig } from '../../auth/tenant/types.js'
+import { runInContext } from '../../request/context/store.js'
+import { createJobContext } from '../../request/context/create.js'
+import { executeInTransaction } from '../../request/context/run-in-transaction.js'
+import { resolveAndExecute, resolveHandlerDatabases, buildCheckpoint } from '../../request/di/resolve.js'
+import { isTracingEnabled } from '../../trace/entry/tracing-enabled.js'
+import { openTrace } from '../../trace/entry/open-trace.js'
+import { parentTraceRef } from '../../trace/entry/parent-trace.js'
+import type { ParentTraceRef } from '../../trace/entry/open-trace.js'
+import { finalizeTrace } from '../../trace/entry/finalize-trace.js'
+import { JobAbortedError } from './errors.js'
+import { randomUUID } from 'node:crypto'
+import { serializeForTrace } from '../../trace/capture/bounded-serializer.js'
+
+/** Default timeout for setting up a tenant job's database transaction(s). */
+const DEFAULT_JOB_TRANSACTION_TIMEOUT_MS = 5000
+
+export interface ExecuteJobOptions {
+  instance: Manifold
+  jobId?: string
+  jobType?: string
+  /** 1-based attempt number, exposed as `flow.jobAttempt`. */
+  attempt?: number
+  /** The job's attempt limit, exposed as `flow.jobMaxAttempts`. */
+  maxAttempts?: number
+  auth?: unknown
+  tenant?: string | null
+  /**
+   * Tenant config carrying custom `sessionVars`. When provided alongside
+   * `tenant`, the custom session variables are propagated in addition to
+   * `pipework.tenant_id` — matching the HTTP surface's behaviour.
+   */
+  tenantConfig?: TenantConfig | null
+  input?: unknown
+  /** Timeout for multi-database transaction setup when running a tenant job. */
+  transactionTimeoutMs?: number
+  /**
+   * The trace this job continues. Omitted: the trace on the active Flow, if any.
+   * `null`: a root trace even inside a Flow. The worker surface passes the
+   * trace recorded when the job was enqueued.
+   */
+  parent?: ParentTraceRef | null
+  /** The ingress kind for the opened trace — `'cron'` for schedule-fired jobs, `'job'` otherwise. */
+  entryKind?: 'job' | 'cron'
+  /**
+   * Set when this attempt follows one that ended without failing on its own (a
+   * reap or a hand-back): the attempt's trace opens with a `pipework.job.restart`
+   * external step naming the reason and the worker that held the previous claim.
+   */
+  restart?: { readonly reason: 'reaped' | 'handed-back' | 'later'; readonly previousWorkerId: string | null }
+  /**
+   * Ends the job at the caller's deadline. On abort the job's in-flight query is
+   * cancelled server-side so its transaction rolls back and its connection
+   * returns to the pool, and `executeJob` rejects — with `signal.reason` when
+   * that is an `Error`, otherwise a `JobAbortedError` (`name: 'AbortError'`)
+   * naming the job. A handler that ignores the cancelled query keeps running;
+   * the rejection is still delivered on time.
+   */
+  signal?: AbortSignal
+}
+
+export async function executeJob<TDeps extends Record<string, unknown>, TReturn>(
+  handler: Handler<TDeps, TReturn>,
+  options: ExecuteJobOptions,
+): Promise<TReturn> {
+  const label = jobLabel(options)
+  if (options.signal?.aborted === true) throw abortReason(options.signal, label)
+
+  const jobCtxOptions: Parameters<typeof createJobContext>[0] = {
+    instance: options.instance,
+  }
+  if (options.jobId !== undefined) jobCtxOptions.jobId = options.jobId
+  if (options.jobType !== undefined) jobCtxOptions.jobType = options.jobType
+  if (options.attempt !== undefined) jobCtxOptions.attempt = options.attempt
+  if (options.maxAttempts !== undefined) jobCtxOptions.maxAttempts = options.maxAttempts
+  if (options.auth !== undefined) jobCtxOptions.auth = options.auth
+  if (options.tenant !== undefined) jobCtxOptions.tenant = options.tenant
+  if (options.tenantConfig !== undefined) jobCtxOptions.tenantConfig = options.tenantConfig
+  if (isTracingEnabled(options.instance)) {
+    jobCtxOptions.trace = openTrace({
+      instance: options.instance,
+      entryKind: options.entryKind ?? 'job',
+      parent: options.parent !== undefined ? options.parent : parentTraceRef(),
+      tenant: options.tenant ?? null,
+      auth: options.auth ?? null,
+      requestId: options.jobId ?? null,
+      sessionId: null,
+    })
+  }
+
+  if (options.restart !== undefined && jobCtxOptions.trace !== undefined && jobCtxOptions.trace !== null) {
+    const now = new Date()
+    jobCtxOptions.trace.buffer.stageExternal({
+      id: randomUUID(),
+      parentStepId: null,
+      flowStepRef: 'external:pipework.job.restart',
+      status: 'ok',
+      startedAt: now,
+      completedAt: now,
+      durationMicros: 0n,
+      capturedInputs: [serializeForTrace({ reason: options.restart.reason, previousWorkerId: options.restart.previousWorkerId })],
+      error: null,
+    })
+  }
+
+  // The handler's own signal: aborted, with the error `executeJob` rejects with,
+  // whenever the job stops being the handler's while it may still be running.
+  const own = new AbortController()
+  const ctx = createJobContext(jobCtxOptions)
+  const databases = resolveHandlerDatabases(handler, options.instance)
+  const timeoutMs = options.transactionTimeoutMs ?? DEFAULT_JOB_TRANSACTION_TIMEOUT_MS
+
+  // Manual transaction mode: the framework opens no outer transaction. The
+  // handler owns its commit boundaries through `checkpoint`, which opens a fresh
+  // tenant-scoped transaction per call (re-establishing the tenant session vars
+  // so database-side RLS sees the tenant) and commits it independently.
+  if (handler.meta.jobTransaction === 'manual') {
+    const checkpoint = buildCheckpoint(
+      handler,
+      ctx,
+      databases,
+      options.tenantConfig ?? null,
+      timeoutMs,
+      options.signal,
+      own.signal,
+    )
+    return finalizeTrace(
+      options.instance,
+      ctx.trace,
+      untilAborted(
+        runInContext(ctx, () => resolveAndExecute(handler, options.input, undefined, undefined, { checkpoint, signal: own.signal })),
+        options.signal,
+        label,
+        own,
+      ),
+    )
+  }
+
+  const run = (): Promise<TReturn> => resolveAndExecute(handler, options.input, undefined, undefined, { signal: own.signal })
+
+  // No tenant: app-level scoping does not apply, so there is nothing to
+  // propagate to Postgres — run the handler directly as before.
+  if (ctx.tenant === null || databases.length === 0) {
+    return finalizeTrace(options.instance, ctx.trace, untilAborted(runInContext(ctx, run), options.signal, label, own))
+  }
+
+  // Tenant job: wrap in a transaction and propagate session vars so
+  // database-side RLS sees the tenant — consistent with the HTTP surface.
+  return finalizeTrace(
+    options.instance,
+    ctx.trace,
+    untilAborted(
+      executeInTransaction(
+        ctx,
+        databases,
+        options.tenantConfig ?? null,
+        timeoutMs,
+        run,
+        options.signal,
+      ),
+      options.signal,
+      label,
+      own,
+    ),
+  )
+}
+
+/** `[type] id`, `[type]`, `id`, or `(anonymous)` — whatever the caller gave us to name the job by. */
+function jobLabel(options: ExecuteJobOptions): string {
+  const parts: string[] = []
+  if (options.jobType !== undefined) parts.push(`"${options.jobType}"`)
+  if (options.jobId !== undefined) parts.push(`(${options.jobId})`)
+  return parts.length === 0 ? '(anonymous)' : parts.join(' ')
+}
+
+/**
+ * The error an abort rejects with: the caller's own `Error` reason when they set
+ * one (the worker passes its timeout error this way), otherwise a
+ * `JobAbortedError` naming the job. `abort()` with no reason leaves a generic
+ * DOMException that names nothing — that is the case worth replacing.
+ */
+function abortReason(signal: AbortSignal, label: string): Error {
+  const reason: unknown = signal.reason
+  const isDefaultReason =
+    typeof DOMException === 'function' && reason instanceof DOMException && reason.name === 'AbortError'
+
+  return reason instanceof Error && !isDefaultReason
+    ? reason
+    : new JobAbortedError(label, reason)
+}
+
+/**
+ * Rejects when the signal aborts, even if the handler is still running — the
+ * caller's deadline is the caller's, not the handler's. The work promise's own
+ * rejection is swallowed once we have rejected, so a later unwind (the cancelled
+ * query surfacing) does not become an unhandled rejection. The handler's own
+ * signal aborts with that same rejection first, so it sees the abort while it runs on.
+ */
+function untilAborted<T>(work: Promise<T>, signal: AbortSignal | undefined, label: string, own: AbortController): Promise<T> {
+  if (signal === undefined) return work
+
+  return new Promise<T>((resolve, reject) => {
+    const onAbort = (): void => {
+      work.catch(() => {})
+      const reason = abortReason(signal, label)
+      own.abort(reason)
+      reject(reason)
+    }
+    if (signal.aborted) {
+      onAbort()
+      return
+    }
+    signal.addEventListener('abort', onAbort, { once: true })
+    work.then(resolve, reject).finally(() => { signal.removeEventListener('abort', onAbort) })
+  })
+}
+

package/src/async/jobs/index.ts

@@ -0,0 +1,10 @@
+export { executeJob } from './execute.js'
+export type { ExecuteJobOptions } from './execute.js'
+export { JobAbortedError, JobLaterError, JobPermanentError } from './errors.js'
+export { createQueue } from './queue.js'
+export type { Queue, QueueConfig, EnqueueOptions, EnqueueExclusion, ClaimedJob, JobClaim, JobClaimReason, ClaimedExclusion, DeadLetterInfo, ReapedJob, FailedAttempt, QueueTotals, QueueTypeTotals, JobResult, BackoffConfig } from './queue.js'
+export type { ExclusionConfig } from './exclusion.js'
+export type { WaitOptions, WaitResult, ListenClient, ListenHandle } from './listener.js'
+export { parseCron, nextTick } from './cron.js'
+export type { CronSchedule } from './cron.js'
+export type { ScheduleSpec, ScheduleInfo, SchedulerOptions } from './scheduler.js'

package/src/async/jobs/listener.ts

@@ -0,0 +1,135 @@
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+import type { JobClaim } from './claims.js'
+import { parseClaims } from './claims.js'
+
+export interface WaitOptions {
+  readonly timeoutMs: number
+}
+
+export interface WaitResult {
+  readonly jobId: string
+  readonly status: 'completed' | 'failed' | 'dead_letter' | 'cancelled' | 'timeout'
+  readonly output?: unknown
+  readonly error?: unknown
+  /** Every claim the job has had, oldest first. Absent on a timeout. */
+  readonly claims?: readonly JobClaim[]
+}
+
+export interface ListenHandle {
+  unlisten(): Promise<void>
+}
+
+export interface ListenClient {
+  listen(channel: string, cb: (payload: string) => void): Promise<ListenHandle>
+}
+
+interface Waiter {
+  readonly resolve: (result: WaitResult) => void
+  readonly timer: ReturnType<typeof setTimeout>
+  readonly db: DB
+  readonly tenant: string | null
+}
+
+export class JobListener {
+  private readonly channel: string
+  private readonly tableQ: string
+  private readonly waiters = new Map<string, Waiter>()
+  private handle: ListenHandle | null = null
+  private listening = false
+
+  constructor(channel: string, tableQ: string) {
+    this.channel = channel
+    this.tableQ = tableQ
+  }
+
+  async start(client: ListenClient): Promise<void> {
+    if (this.listening) return
+    this.handle = await client.listen(this.channel, (payload: string) => {
+      this.dispatch(payload)
+    })
+    this.listening = true
+  }
+
+  async stop(): Promise<void> {
+    if (!this.listening || !this.handle) return
+    await this.handle.unlisten()
+    this.handle = null
+    this.listening = false
+
+    for (const [jobId, waiter] of this.waiters) {
+      clearTimeout(waiter.timer)
+      waiter.resolve({ jobId, status: 'timeout' })
+    }
+    this.waiters.clear()
+  }
+
+  get isListening(): boolean {
+    return this.listening
+  }
+
+  async waitFor(db: DB, jobId: string, opts: WaitOptions, tenant: string | null = null): Promise<WaitResult> {
+    if (!this.listening) {
+      const existing = await this.pollJob(db, jobId, tenant)
+      if (existing !== null) return existing
+      return this.pollUntil(db, jobId, opts.timeoutMs, tenant)
+    }
+
+    return new Promise<WaitResult>((resolve) => {
+      const timer = setTimeout(() => {
+        this.waiters.delete(jobId)
+        resolve({ jobId, status: 'timeout' })
+      }, opts.timeoutMs)
+
+      const waiter: Waiter = { resolve, timer, db, tenant }
+      this.waiters.set(jobId, waiter)
+
+      void this.pollJob(db, jobId, tenant).then((result) => {
+        if (result !== null && this.waiters.has(jobId)) {
+          this.waiters.delete(jobId)
+          clearTimeout(timer)
+          resolve(result)
+        }
+      })
+    })
+  }
+
+  private dispatch(jobId: string): void {
+    const waiter = this.waiters.get(jobId)
+    if (!waiter) return
+    this.waiters.delete(jobId)
+    clearTimeout(waiter.timer)
+
+    void this.pollJob(waiter.db, jobId, waiter.tenant).then((result) => {
+      waiter.resolve(result ?? { jobId, status: 'completed' })
+    })
+  }
+
+  async pollJob(db: DB, jobId: string, tenant: string | null = null): Promise<WaitResult | null> {
+    const rows = await db.execute(
+      sql`SELECT status, output, error_payload, claims FROM ${sql.raw(this.tableQ)} WHERE id = ${jobId}::uuid${tenant === null ? sql`` : sql` AND flow_tenant = ${tenant}::text`}`
+    )
+    if (rows.length === 0) return null
+    const row = rows[0]!
+    const status = row['status'] as string
+    if (status === 'pending' || status === 'claimed' || status === 'running') return null
+    return {
+      jobId,
+      status: status as WaitResult['status'],
+      output: row['output'] ?? undefined,
+      error: row['error_payload'] ?? undefined,
+      claims: parseClaims(row['claims']),
+    }
+  }
+
+  private async pollUntil(db: DB, jobId: string, timeoutMs: number, tenant: string | null): Promise<WaitResult> {
+    const deadline = Date.now() + timeoutMs
+    const interval = 500
+    while (Date.now() < deadline) {
+      const result = await this.pollJob(db, jobId, tenant)
+      if (result !== null) return result
+      await new Promise(r => setTimeout(r, Math.min(interval, deadline - Date.now())))
+    }
+    return { jobId, status: 'timeout' }
+  }
+}

package/src/async/jobs/namespace.ts

@@ -0,0 +1,20 @@
+import { createQueue } from './queue.js'
+import { executeJob } from './execute.js'
+import { JobLaterError, JobPermanentError } from './errors.js'
+import { parseCron, nextTick } from './cron.js'
+
+/** Background job queue namespace — Postgres-backed with SKIP LOCKED, LISTEN/NOTIFY, cron scheduling. */
+export const jobs = {
+  /** Creates a job queue with claim/complete/fail lifecycle, heartbeat, and dead letter requeue. */
+  createQueue,
+  /** Executes a fitting handler as a background job with its own Flow context. */
+  execute: executeJob,
+  /** The error to throw to end an attempt as "later": pending again after `delayMs`, no attempt counted. */
+  later: (message: string, opts?: { delayMs?: number; cause?: unknown }): JobLaterError => new JobLaterError(message, opts),
+  /** The error to throw to dead-letter the job now, whatever attempts remain. */
+  permanent: (message: string, opts?: { cause?: unknown }): JobPermanentError => new JobPermanentError(message, opts),
+  /** Parses a 5-field cron expression into a CronSchedule (minute, hour, dom, month, dow). */
+  parseCron,
+  /** Computes the next fire time for a CronSchedule after a given date, with optional timezone. */
+  nextTick,
+}

package/src/async/jobs/queue-door.ts

@@ -0,0 +1,44 @@
+import type { ResolvedDoor } from '../../core/config/load.js'
+import type { DefinedTable } from '../../data/domain/types.js'
+import { queueDefinitions } from './table.js'
+
+/** The framework door a tenant-column queue table's claim, reap, heartbeat and complete run through. */
+export function queueDoorName(table: string): string {
+  return `pipework.queue.${table}`
+}
+
+/** A queue only selects and updates its rows: it never deletes one. */
+export function queueFrameworkDoor(table: string, database: string): ResolvedDoor {
+  return Object.freeze({
+    name: queueDoorName(table), table, database,
+    reason: 'the queue claims, reaps and completes jobs across tenants',
+    framework: true as const, ops: ['select', 'update'] as const,
+  })
+}
+
+const hasTenant = (definition: DefinedTable): boolean =>
+  Object.values(definition.$fields).some(field => field._meta.tenant === true)
+
+/**
+ * The queue doors pipework declares: one per registered queue table with a
+ * tenant column, where the default database sets `rls.platformRole`. Queue
+ * tables register when a worker surface or graph is constructed, so ask after
+ * the app's modules are loaded.
+ */
+export function queueFrameworkDoors(config: {
+  defaultDatabaseName(): string
+  database(name?: string): { readonly rls?: { readonly platformRole?: string } | undefined }
+}): Record<string, ResolvedDoor> {
+  const out: Record<string, ResolvedDoor> = {}
+  let database: string
+  try {
+    database = config.defaultDatabaseName()
+  } catch {
+    return out
+  }
+  if (config.database(database).rls?.platformRole === undefined) return out
+  for (const [table, definition] of queueDefinitions()) {
+    if (hasTenant(definition)) out[queueDoorName(table)] = queueFrameworkDoor(table, database)
+  }
+  return out
+}