pipework 0.251.0 → 0.252.1

package/src/trace/partition/tables.ts

@@ -0,0 +1,53 @@
+// The partitioned tables the substrate manages, plus the retention window each
+// one obeys. Three retention classes share one partition lifecycle:
+//
+//   - **hot** (`trace`, `trace_step`) — `hotRetentionDays`, short-lived.
+//   - **cold** (`trace_retained`, `trace_step_retained`) — `errorRetentionDays`,
+//     copy-on-error survives past hot retention.
+//   - **audit** (`audit_record`) — `auditRetentionDays`, long / indefinite; the
+//     audit projection lives independently of trace storage.
+//
+// Partition creation uses the same `partitionLeadDays` for all three classes —
+// inserts on any class must never race partition creation.
+
+import { trace } from '../schema/trace.js'
+import { traceStep } from '../schema/trace-step.js'
+import { traceRetained } from '../schema/trace-retained.js'
+import { traceStepRetained } from '../schema/trace-step-retained.js'
+import { auditRecord } from '../../audit/schema/audit-record.js'
+
+export type RetentionClass = 'hot' | 'cold' | 'audit'
+
+export interface PartitionedTraceTable {
+  /** The parent table's wire name — partitions are children of this. */
+  readonly parent: string
+  readonly retention: RetentionClass
+}
+
+export const PARTITIONED_TRACE_TABLES: readonly PartitionedTraceTable[] = [
+  { parent: trace.$name, retention: 'hot' },
+  { parent: traceStep.$name, retention: 'hot' },
+  { parent: traceRetained.$name, retention: 'cold' },
+  { parent: traceStepRetained.$name, retention: 'cold' },
+  { parent: auditRecord.$name, retention: 'audit' },
+]
+
+/**
+ * Wire names of the framework-owned partitioned tables. A `pipework generate`
+ * must not emit these into an application migration (a bare `CREATE TABLE` of a
+ * `PARTITION BY RANGE` table accepts no inserts), and the test harness
+ * provisions them — parent plus day partitions — itself. Issue #313.
+ */
+export const PARTITIONED_TRACE_TABLE_NAMES: ReadonlySet<string> = new Set(
+  PARTITIONED_TRACE_TABLES.map(t => t.parent),
+)
+
+/**
+ * Each step table with the trace table its foreign key points at. A day's step
+ * partition must go before the trace partition it references, so retention
+ * drops the two together.
+ */
+export const STEP_TABLE_PARENTS: ReadonlyMap<string, string> = new Map([
+  [traceStep.$name, trace.$name],
+  [traceStepRetained.$name, traceRetained.$name],
+])

package/src/trace/read/tree.ts

@@ -0,0 +1,83 @@
+// `trace.tree`: every hot trace sharing a root, with its steps.
+
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+
+/** One step of a trace. `ref` is the flow step's `path:symbol`, or `external:<name>` for a recorded or point-event step. */
+export interface TraceStepNode {
+  readonly id: string
+  readonly parentStepId: string | null
+  readonly ref: string
+  readonly status: 'ok' | 'error'
+  readonly startedAt: Date
+  readonly completedAt: Date | null
+  /** The step's captured inputs (one entry per captured argument), or `null` when none were captured. */
+  readonly capturedInputs: unknown
+  readonly error: unknown
+}
+
+/** One trace of the tree, with its steps ordered by `seq`. */
+export interface TraceNode {
+  readonly id: string
+  readonly parentId: string | null
+  readonly entryKind: 'http' | 'job' | 'cron' | 'test'
+  readonly tenant: string | null
+  readonly requestId: string | null
+  readonly startedAt: Date
+  readonly steps: readonly TraceStepNode[]
+}
+
+/** The traces that share a root id, ordered by when they opened. */
+export interface TraceTree {
+  readonly traces: readonly TraceNode[]
+}
+
+const str = (v: unknown): string | null => (v === null || v === undefined ? null : String(v))
+const date = (v: unknown): Date => (v instanceof Date ? v : new Date(String(v)))
+
+/**
+ * Reads the hot `trace` and `trace_step` rows whose root is `rootId`: the
+ * request's trace and the traces of the jobs it queued. Rows already moved to
+ * the retained tables are not read. The read runs on `db` as given, so the
+ * tables' tenant policies apply: a connection with a tenant setting sees that
+ * tenant's rows, and only an unrestricted one (the table owner or a role that
+ * bypasses row-level security, as in tests) sees every tenant's. An unknown
+ * root returns no traces.
+ */
+export async function tree(db: DB, rootId: string): Promise<TraceTree> {
+  const traces = await db.execute(sql`
+    SELECT "id", "parent_id", "entry_kind", "tenant_id", "request_id", "opened_at"
+    FROM "trace" WHERE "root_id" = ${rootId}::uuid ORDER BY "opened_at", "id"`)
+  if (traces.length === 0) return { traces: [] }
+  const steps = await db.execute(sql`
+    SELECT s."trace_id", s."id", s."parent_step_id", s."flow_step_ref", s."status", s."started_at",
+           s."completed_at", s."captured_inputs", s."error"
+    FROM "trace_step" s JOIN "trace" t ON t."id" = s."trace_id" AND t."opened_day" = s."trace_day"
+    WHERE t."root_id" = ${rootId}::uuid ORDER BY s."trace_id", s."seq"`)
+  const byTrace = new Map<string, TraceStepNode[]>()
+  for (const r of steps) {
+    const list = byTrace.get(String(r['trace_id'])) ?? []
+    list.push({
+      id: String(r['id']),
+      parentStepId: str(r['parent_step_id']),
+      ref: String(r['flow_step_ref']),
+      status: r['status'] as 'ok' | 'error',
+      startedAt: date(r['started_at']),
+      completedAt: r['completed_at'] === null ? null : date(r['completed_at']),
+      capturedInputs: r['captured_inputs'],
+      error: r['error'],
+    })
+    byTrace.set(String(r['trace_id']), list)
+  }
+  return {
+    traces: traces.map(r => ({
+      id: String(r['id']),
+      parentId: str(r['parent_id']),
+      entryKind: r['entry_kind'] as TraceNode['entryKind'],
+      tenant: str(r['tenant_id']),
+      requestId: str(r['request_id']),
+      startedAt: date(r['opened_at']),
+      steps: byTrace.get(String(r['id'])) ?? [],
+    })),
+  }
+}

package/src/trace/retain/copy-on-error.ts

@@ -0,0 +1,76 @@
+// Copy-on-error: when a trace ends badly (error / truncated / expired), the hot
+// `trace` / `trace_step` partitions it lives in will be dropped on the normal
+// `hotRetentionDays` schedule. Before that happens the errored trace — and its
+// ancestry spine up to the root — is copied into the cold `trace_retained` /
+// `trace_step_retained` tables, which keep the longer `errorRetentionDays`
+// window.
+//
+// Only the parent chain is followed (`parent_id`): not siblings, not the step
+// tree (`parent_step_id`). The spine is what you need to read an errored trace
+// in context. Each copied trace brings all of its own steps.
+//
+// The copy is idempotent — `ON CONFLICT DO NOTHING` against each cold table's
+// composite primary key — so re-running after a crash, or copying a trace whose
+// ancestor was already retained by an earlier error, is a no-op.
+
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+import { quoteIdentifier } from '../../data/db/identifiers.js'
+import { withMaintenance } from '../../data/db/maintenance.js'
+import { camelToSnake } from '../../data/domain/naming.js'
+import { traceFields } from '../schema/trace-fields.js'
+import { traceStepFields } from '../schema/trace-step-fields.js'
+
+export interface CopyOnErrorResult {
+  /** Rows newly inserted into `trace_retained` (rows already retained excluded). */
+  readonly traces: number
+  /** Rows newly inserted into `trace_step_retained`. */
+  readonly steps: number
+}
+
+function columnList(keys: string[], label: string, prefix = ''): string {
+  return keys.map(k => `${prefix}${quoteIdentifier(camelToSnake(k), label)}`).join(', ')
+}
+
+const TRACE_COLUMNS = columnList(Object.keys(traceFields), 'trace column')
+const TRACE_STEP_COLUMNS = columnList(Object.keys(traceStepFields), 'trace step column')
+const TRACE_STEP_COLUMNS_FROM_STEP = columnList(Object.keys(traceStepFields), 'trace step column', 's.')
+
+/**
+ * Copy an errored trace and its parent spine into the cold retained tables.
+ *
+ * `traceId` is the trace that ended badly. Its ancestry is walked through
+ * `parent_id` to the root; every trace on that spine, plus all of their steps,
+ * is inserted into `trace_retained` / `trace_step_retained`. Returns the count
+ * of rows actually inserted — zero for both when everything was already
+ * retained, or when `traceId` does not exist.
+ */
+export async function copyTraceOnError(db: DB, traceId: string): Promise<CopyOnErrorResult> {
+  // Reads the whole parent spine, which may cross tenants, so the copy is a maintenance pass.
+  const rows = await withMaintenance(db, tx => tx.execute(sql`
+    WITH RECURSIVE spine AS (
+      SELECT * FROM "trace" WHERE "id" = ${traceId}
+      UNION
+      SELECT t.* FROM "trace" t JOIN spine ON t."id" = spine."parent_id"
+    ),
+    copied_traces AS (
+      INSERT INTO "trace_retained" (${sql.raw(TRACE_COLUMNS)})
+      SELECT ${sql.raw(TRACE_COLUMNS)} FROM spine
+      ON CONFLICT ("id", "opened_day") DO NOTHING
+      RETURNING "id"
+    ),
+    copied_steps AS (
+      INSERT INTO "trace_step_retained" (${sql.raw(TRACE_STEP_COLUMNS)})
+      SELECT ${sql.raw(TRACE_STEP_COLUMNS_FROM_STEP)}
+      FROM "trace_step" s JOIN spine ON s."trace_id" = spine."id"
+      ON CONFLICT ("id", "trace_day") DO NOTHING
+      RETURNING "id"
+    )
+    SELECT
+      (SELECT count(*) FROM copied_traces)::int AS traces,
+      (SELECT count(*) FROM copied_steps)::int AS steps
+  `))
+
+  const row = rows[0]!
+  return { traces: row['traces'] as number, steps: row['steps'] as number }
+}

package/src/trace/runtime/expire-stale-traces.ts

@@ -0,0 +1,54 @@
+// Q7 expiry sweep. A trace must not outlive its own oldest day-partition: if
+// a trace is still `open` past `expires_at` (stamped as `opened_at +
+// hotRetentionDays` at entry), the entry leaked or the job hung — the process
+// that opened the trace may be long dead, so there is no in-memory buffer to
+// flush. The sweep force-closes those orphans at the database level: stamps
+// `status = 'expired'` and `closed_at`, then copies the trace to the cold set
+// (treated as an error for Q4) before the next partition retirement drops the
+// hot row.
+//
+// `force_now` is injectable so the sweep is testable without mocking the
+// clock — production calls pass nothing and use `NOW()`.
+
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import { withMaintenance } from '../../data/db/maintenance.js'
+import { copyTraceOnError } from '../retain/copy-on-error.js'
+
+export interface ExpireStaleTracesResult {
+  /** Trace rows force-closed by this sweep (`open` past `expires_at`). */
+  readonly expired: number
+  /** Rows newly inserted into `trace_retained` across all expired traces. */
+  readonly retained: number
+}
+
+/** Force-closes every trace whose `expires_at` has passed; copies each to the cold set. */
+export async function expireStaleTraces(
+  db: DB,
+  now: Date = new Date(),
+): Promise<ExpireStaleTracesResult> {
+  // postgres.js's raw `sql` parameters carry no column-type hint, so a Date
+  // binds as an unsupported type for `timestamptz`. The boundary cast to an
+  // ISO string keeps the production signature on `Date` without losing
+  // precision (compared as a literal, Postgres parses it back to `timestamptz`).
+  const nowIso = now.toISOString()
+  // Every tenant's traces, so the sweep is a maintenance pass.
+  const rows = await withMaintenance(db, tx => tx.execute(sql`
+    UPDATE "trace"
+    SET "status" = 'expired', "closed_at" = ${nowIso}
+    WHERE "status" = 'open' AND "expires_at" <= ${nowIso}
+    RETURNING "id"
+  `))
+  const ids = rows.map(r => r['id'] as string)
+  if (ids.length === 0) return { expired: 0, retained: 0 }
+
+  getBaseLogger().error(
+    { count: ids.length, traceIds: ids },
+    'Force-closed expired traces — entries leaked or hung past their hot retention window',
+  )
+
+  const copyResults = await Promise.all(ids.map(id => copyTraceOnError(db, id)))
+  const retained = copyResults.reduce((sum, r) => sum + r.traces, 0)
+  return { expired: ids.length, retained }
+}

package/src/trace/runtime/flow-step.ts

@@ -0,0 +1,147 @@
+// `__flowStep` — the value-wrapping runtime for a flow step. The build
+// transformer (component 5) renames each documented function and rebinds its
+// public name through this wrapper: `export const runJob = __flowStep(meta,
+// runJobImpl)`. No body is rewritten; the wrapper is the whole instrumentation.
+//
+// Off the traced path it is a pass-through with one `getContext()` lookup and a
+// branch — there is no active Flow, or the Flow opened no trace, so there is
+// nothing to record. On the traced path it brackets the implementation with a
+// `trace_step`: a `seq`, a parent link, start/end timing, and the outcome.
+// Once a trace truncates (Q7), every later step takes the pass-through path
+// too, so a truncated trace costs exactly what an untraced one does.
+//
+// Tier-2 input capture (Q5) runs at `begin` time, before the implementation
+// gets a chance to mutate the arguments. Each parameter's projection — baked
+// into `meta.capture` by the transformer — turns its argument into a
+// trace-safe payload, which the bounded serializer caps for depth, fan-out,
+// string length, and redaction before staging onto `trace_step.captured_inputs`.
+// The trace's `captureInputs` kill-switch (resolved at trace open from
+// `config.trace.captureInputs`) suppresses the whole capture path while
+// leaving Tier-1 (timing, status, error) untouched.
+//
+// `meta` is a literal baked in at transform time — the wrapper never computes
+// the step's identity.
+
+import { getContext } from '../../request/context/store.js'
+import { serializeForTrace } from '../capture/bounded-serializer.js'
+import { isThenable } from './thenable.js'
+import { truncateTrace } from './truncate-trace.js'
+import { emitReadTouches } from '../../data/write/audit/read-touches.js'
+import type { CaptureProjection, FlowStepMeta } from './step-meta.js'
+import type { BegunStep } from './step-buffer.js'
+import type { Flow } from '../../request/context/types.js'
+
+/* eslint-disable @typescript-eslint/no-explicit-any -- a transparent wrapper must accept and return the implementation's exact signature */
+type AnyFunction = (...args: any[]) => any
+
+/**
+ * Wraps `impl` so that, under an active traced Flow, each call records a `trace_step`.
+ *
+ * A function documented with `@flow_step` (and `@touches Entity:verb`) is wrapped automatically: under `pipework test` and the vitest preset, and under `pipework dev` and `pipework serve`. An app built with plain `tsc` gets no wrapping: run it through `pipework serve`, or call `__flowStep` by hand.
+ *
+ * A write under a step with `@touches` is audited only under the touches whose entity is the written table, one `audit_record` per affected row per selected touch. A row's verb is `create` (an insert, or an upsert row that did not exist), `update` or `delete`; it is recorded under the touches with exactly that verb, else under the touches with a custom verb (not create/update/delete), else under all the entity's touches. A write also counts as written for the step's other create/update/delete touches on the entity, and for the same touch on enclosing steps. A touch's `Entity` matches a table when the definition declares `define(name, fields, { auditEntity })` equal to `Entity` exactly; otherwise when the names agree ignoring case and underscores, with the table name optionally plural (`Version` matches `version` and `versions`; `DocumentVersion` matches `document_versions`). A write to a table no touch matches is not audited. A touch that no write of the step was audited under gets a read row (`before` and `after` null) when the step completes. A write under a step with a touch whose entity matches no defined table throws, naming the closest table names. `.returning()` returns the same rows with or without an audited step.
+ */
+export function __flowStep<F extends AnyFunction>(meta: FlowStepMeta, impl: F): F {
+  function traced(this: unknown, ...args: Parameters<F>): ReturnType<F> {
+    const flow = getContext()
+    if (flow === undefined || flow.trace === null) {
+      return impl.apply(this, args) as ReturnType<F>
+    }
+    const ctx = flow.trace
+    const buffer = ctx.buffer
+    if (buffer.truncated) {
+      return impl.apply(this, args) as ReturnType<F>
+    }
+
+    const capturedInputs = buffer.captureInputs && meta.capture !== undefined
+      ? captureArgs(args, meta.capture)
+      : null
+
+    const begun = buffer.begin(meta.ref, capturedInputs, meta.touches ?? [])
+    // `begin` returns null for the call that blows the step cap — that call,
+    // and every step after it, runs untraced. The transition fires `truncateTrace`
+    // exactly once: the `buffer.truncated` short-circuit above catches every
+    // subsequent call before it reaches `begin`.
+    if (begun === null) {
+      void truncateTrace(flow.instance, ctx)
+      return impl.apply(this, args) as ReturnType<F>
+    }
+
+    let result: unknown
+    try {
+      result = impl.apply(this, args)
+    } catch (err) {
+      buffer.complete(begun, 'error', err)
+      throw err
+    }
+
+    if (isThenable(result)) {
+      return result.then(
+        async (value: unknown) => {
+          buffer.complete(begun, 'ok', null)
+          await emitReadTouchesIfNeeded(flow, begun)
+          return value
+        },
+        (err: unknown) => {
+          buffer.complete(begun, 'error', err)
+          throw err
+        },
+      ) as ReturnType<F>
+    }
+
+    buffer.complete(begun, 'ok', null)
+    // Sync impl: can't change the wrapper's return type to Promise, so fire-and-
+    // forget. The audit emitter swallows its own errors, so this won't yield an
+    // unhandled rejection; the trade-off is that a sync `@touches` step's audit
+    // row may land after control has returned to the caller. Sync impls under
+    // `@touches` are rare anyway (pipe.* writes are all async).
+    void emitReadTouchesIfNeeded(flow, begun)
+    return result as ReturnType<F>
+  }
+
+  return traced as F
+}
+/* eslint-enable @typescript-eslint/no-explicit-any */
+
+/**
+ * Read-touches gate. The CTE-form write path already covered the touches it wrote
+ * (recorded via `markWrote(touch)`); this fires for the rest — touches
+ * no `pipe.*` write of the step was audited under. Empty
+ * `touches` means no audit projection at all and short-circuits earlier in
+ * the emitter.
+ */
+function emitReadTouchesIfNeeded(flow: Flow, begun: BegunStep): Promise<void> {
+  if (begun.touches.every(t => begun.wroteTouches.has(t))) return Promise.resolve()
+  return emitReadTouches(flow, begun)
+}
+
+/**
+ * Projects + bounded-serializes one captured payload per parameter, in
+ * declaration order. A `null` projection slot means the parameter has no
+ * registry-derived projection (primitive, `any`, or no `.traceSafe()` entry)
+ * and the raw arg is handed straight to the serializer. A projection that
+ * returns `undefined` is preserved as a `null` slot — fail-closed: an unsafe
+ * structured type with no marked-safe fields contributes no payload.
+ */
+function captureArgs(
+  args: readonly unknown[],
+  projections: ReadonlyArray<CaptureProjection | null>,
+): unknown[] {
+  const out: unknown[] = []
+  // Pair by parameter index. Extra args (a variadic tail without a matching
+  // projection) fall through to the raw-serializer path; extra projections
+  // (impl invoked with fewer than declared args) project `undefined` — both
+  // paths converge on the bounded serializer's `undefined` handling.
+  const count = Math.max(args.length, projections.length)
+  for (let i = 0; i < count; i++) {
+    const projection = projections[i] ?? null
+    const raw = args[i]
+    const projected = projection === null ? raw : projection(raw)
+    if (projected === undefined) {
+      out.push(null)
+      continue
+    }
+    out.push(serializeForTrace(projected))
+  }
+  return out
+}

package/src/trace/runtime/flush-steps.ts

@@ -0,0 +1,31 @@
+// Batch-writes a buffered set of `trace_step` rows. The write is fire-and-
+// forget — the close path doesn't block on it, and a failure must not break
+// the request: a missing step row is a degraded trace, not a degraded user
+// flow. Errors land in the base logger so an operator can see what was lost.
+//
+// The write goes through `instance.pool` directly, never the request
+// transaction — an errored request rolls back, but its trace steps must
+// survive to be inspected.
+
+import type { Manifold } from '../../core/pipework.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import { recordFaultFor, faultMessage } from '../../infra/alerts/faults.js'
+import { traceStep } from '../schema/trace-step.js'
+import { trackTraceWrite } from './pending-writes.js'
+import type { PendingStep } from './step-buffer.js'
+
+/** Inserts `steps` into `trace_step` in a single batch. Resolves once the write settles. */
+export function flushSteps(instance: Manifold, steps: readonly PendingStep[]): Promise<void> {
+  if (steps.length === 0) return Promise.resolve()
+  const db = instance.pool.getOrCreate(instance.config.database()).drizzle
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Drizzle's insert needs the concrete runtime table type
+  return trackTraceWrite((db as any).insert(traceStep.table()).values([...steps])
+    .then(() => undefined)
+    .catch((err: unknown) => {
+      getBaseLogger().error(
+        { err, traceId: steps[0]!.traceId, steps: steps.length },
+        'Failed to persist trace steps — the trace is missing step rows',
+      )
+      void recordFaultFor(instance, 'trace-steps-lost', `${String(steps.length)} trace step(s) not persisted: ${faultMessage(err)}`, { traceId: steps[0]!.traceId, steps: steps.length })
+    }))
+}

package/src/trace/runtime/pending-writes.ts

@@ -0,0 +1,54 @@
+// Trace writes are fire-and-forget; this tracks the ones in flight so the test
+// harness can wait for them before it ends the connection they write through.
+
+import { getBaseLogger } from '../../core/logging/index.js'
+
+// Process-global so writes tracked by any copy of this module (loaded by
+// `vi.resetModules()` / `vi.isolateModules`) are visible to the harness's settle.
+const SLOT = Symbol.for('pipework.traceWrites')
+const slots = globalThis as unknown as Record<symbol, Set<Promise<void>> | undefined>
+const pending: Set<Promise<void>> = (slots[SLOT] ??= new Set<Promise<void>>())
+
+/** Records `write` as in flight until it settles; returns it unchanged. */
+export function trackTraceWrite(write: Promise<void>): Promise<void> {
+  pending.add(write)
+  const forget = (): void => { pending.delete(write) }
+  write.then(forget, forget)
+  return write
+}
+
+/** Resolves once no trace write is in flight, including writes started by earlier ones. */
+export async function settleTraceWrites(): Promise<void> {
+  while (pending.size > 0) await Promise.allSettled([...pending])
+}
+
+/** How many trace writes are in flight now. */
+export function pendingTraceWriteCount(): number {
+  return pending.size
+}
+
+export const TRACE_SETTLE_TIMEOUT_MS = 30_000
+
+/** The harness's teardown hooks' own timeout: room for a full settle plus the teardown itself. */
+export const TEARDOWN_HOOK_TIMEOUT_MS = TRACE_SETTLE_TIMEOUT_MS + 30_000
+
+/**
+ * Waits for in-flight trace writes for at most `timeoutMs`. On expiry it warns once with
+ * the number still in flight and returns, so teardown proceeds anyway.
+ */
+export async function settleTraceWritesBounded(timeoutMs: number = TRACE_SETTLE_TIMEOUT_MS): Promise<void> {
+  let timer: NodeJS.Timeout | undefined
+  const expired = Symbol('expired')
+  const timeout = new Promise<typeof expired>(resolve => { timer = setTimeout(() => resolve(expired), timeoutMs) })
+  try {
+    const outcome = await Promise.race([settleTraceWrites(), timeout])
+    if (outcome === expired) {
+      getBaseLogger().warn(
+        { inFlight: pendingTraceWriteCount(), timeoutMs },
+        `${pendingTraceWriteCount()} trace write(s) still in flight after ${timeoutMs} ms; test teardown proceeds anyway`,
+      )
+    }
+  } finally {
+    clearTimeout(timer)
+  }
+}