pipework 0.252.0 → 0.253.0

package/src/test/setup.ts

@@ -0,0 +1,1422 @@
+import { randomBytes } from 'node:crypto'
+import postgres from 'postgres'
+import { drizzle } from '../data/query/postgres-js/index.js'
+import { sql } from '../data/query/index.js'
+import type { Manifold } from '../core/pipework.js'
+import type { Blueprint, ResolvedDatabase } from '../core/config/index.js'
+import { quoteIdentifier } from '../data/db/identifiers.js'
+import { applyMigrations } from '../data/migrate/apply.js'
+import { queueTableProblems, queueTableProblemMessage } from '../core/validation/queue-tables.js'
+import { probeRlsRole, rlsProbeTarget, rlsRoleMessage } from '../core/validation/rls-role.js'
+import { provisionFrameworkTables } from '../data/migrate/provision-framework.js'
+import { getAdminUrl, reapOrphanTestDatabases, runLockKey, templateBuildKey, templateUseKey } from './reap.js'
+import { testDbPrefix, appRolePrefix, templateDbPrefix } from './namespace.js'
+import { templateFingerprint, templateInputs } from './fingerprint.js'
+import {
+  PhaseTracker,
+  withStallWatchdog,
+  DEFAULT_SETUP_STALL_MS,
+  type SetupProgressListener,
+} from './progress.js'
+import { describeError } from '../core/logging/error-chain.js'
+import { captureBaseline, resetDatabase, TEST_RESET_LOG_PREFIX, type Baseline } from './reset.js'
+
+export interface TestDatabase {
+  name: string
+  url: string
+  appUrl: string
+  appRole: string
+  /** The per-file database that gets dropped on teardown. */
+  database: string
+}
+
+// A template per configured database, built once — migrations, extensions,
+// framework tables, role grants — then every test file's database is a
+// `CREATE DATABASE ... WITH TEMPLATE` clone of it. A clone is a filesystem copy
+// (tens of ms); replaying the migration stack per file is seconds. The
+// template's app role and its table/schema/default-privilege grants are
+// catalog rows that the clone copies for free. Issue #321.
+//
+// The template is shared across PROCESSES, keyed by the hash of everything that
+// determines its contents. A consumer running vitest with `pool: 'forks'` and
+// `isolate: true` gets a fresh process per test file; when the template was
+// keyed by pid and run nonce, every one of those processes replayed the entire
+// migration stack for a byte-identical result. Now the first process to want a
+// given content hash builds it and the rest clone from it, across processes and
+// across runs. Per-file CLONES stay pid+nonce-keyed exactly as before — nothing
+// about isolation between test files changes. Issue #340.
+interface Template {
+  templateDb: string
+  appRole: string
+  fingerprint: string
+}
+
+/** One database's template identity, computed before any connection is opened. */
+interface TemplatePlan {
+  name: string
+  dbConfig: ResolvedDatabase
+  fingerprint: string
+  templateDb: string
+  appRole: string
+  isDefaultDatabase: boolean
+}
+
+// Marker table written inside a template as the last act of a successful build.
+// A template database that exists without this row is the wreckage of a builder
+// that died mid-migration: it must be rebuilt, never cloned.
+const TEMPLATE_MARKER_TABLE = 'pipework_test_template'
+
+const testDatabases = new Map<string, TestDatabase>()
+
+// A per-file database this process keeps across files, with the state it is
+// returned to between them (`reset.ts`). Keyed by database name.
+interface RetainedDatabase {
+  testDb: TestDatabase
+  baseline: Baseline
+  /** Which template, server and clone settings it was made from. */
+  key: string
+  inUse: boolean
+  /** Its file has ended; the next setup resets it before handing it out. */
+  needsReset: boolean
+}
+const retained = new Map<string, RetainedDatabase>()
+let databasePerFile = false
+let runAdminUrl = ''
+let adminIsSuperuser: boolean | null = null
+
+async function readSuperuser(): Promise<boolean> {
+  const rows = await adminClient!`SELECT rolsuper FROM pg_roles WHERE rolname = current_user`
+  return rows[0]?.['rolsuper'] === true
+}
+const templates = new Map<string, Template>()
+let adminClient: postgres.Sql | null = null
+
+// A random per-process fingerprint, embedded in every database and role name
+// this run creates. It keys the advisory lock the run holds for its lifetime,
+// letting another run's reaper tell this run's databases apart from orphans.
+const runNonce = randomBytes(6).toString('hex')
+let lockClient: postgres.Sql | null = null
+let reaped = false
+let workerShutdownArmed = false
+
+function templateDbName(baseName: string, fingerprint: string): string {
+  return `${templateDbPrefix()}${baseName}_${fingerprint}`
+}
+
+/** Log prefix for what the orphan reaper left alone (#391). */
+export const TEST_REAP_LOG_PREFIX = 'pipework:test-reap'
+
+// One name per CLONE, not per process. A finished database may still be queued
+// for drop (see `pendingDrops`), so the next clone must not ask for its name.
+let cloneSeq = 0
+
+function testDbName(baseName: string): string {
+  return `${testDbPrefix()}${baseName}_${cloneSeq++}_${process.pid}_${runNonce}`
+}
+
+// The app role is named to match what the orphan reaper derives from the
+// *template* database name (`reap.ts` `pairedTemplateRoleName`): the template
+// outlives every per-file clone and every run, so when the reaper finally drops
+// a stale template it drops this role with it. The role must be content-keyed
+// too — its grants are baked into the template's catalog by OID, so a template
+// shared across processes needs a role shared across processes.
+function appRoleName(baseName: string, fingerprint: string): string {
+  return `${appRolePrefix()}${baseName}_${fingerprint}`
+}
+
+export interface SetupOptions {
+  /** Told about every step setup reaches, in order. Issue #380. */
+  onProgress?: SetupProgressListener
+  /** Overrides `test.setupStallMs`. Injectable for tests. */
+  stallMs?: number
+  /** How often the stall watchdog looks. Injectable for tests. */
+  stallPollMs?: number
+}
+
+export async function setupTestDatabases(
+  instance: Manifold,
+  options: SetupOptions = {},
+): Promise<Map<string, TestDatabase>> {
+  const tracker = new PhaseTracker(options.onProgress)
+  return withStallWatchdog(
+    () => runSetup(instance, tracker),
+    tracker,
+    {
+      stallMs: options.stallMs ?? instance.config.raw.test?.setupStallMs ?? DEFAULT_SETUP_STALL_MS,
+      ...(options.stallPollMs === undefined ? {} : { pollMs: options.stallPollMs }),
+    },
+  )
+}
+
+async function runSetup(instance: Manifold, tracker: PhaseTracker): Promise<Map<string, TestDatabase>> {
+  const databases = [...instance.config.databases()]
+
+  // D4b at test bootstrap: for an rls-configured database, the role tests
+  // actually execute on must not bypass RLS — otherwise every policy is inert
+  // and the suite proves nothing about isolation. Which connection that is
+  // depends on the isolation strategy: transaction isolation collapses every
+  // role onto the resolved `url` connection; database/container isolation
+  // resolves roles normally (`appUrl ?? url`). Runs before prepareRun so a
+  // refused bootstrap opens no admin connection and leaves no state behind.
+  for (const [name, dbConfig] of databases) {
+    if (dbConfig.rls === undefined) continue
+    tracker.reached('rls-probe', name)
+    const target = rlsProbeTarget(dbConfig, instance.config.isolationStrategy(name))
+    const probe = await namingDatabase(name, 'rls-probe', () => probeRlsRole(target, null))
+    if (probe.superuser || probe.bypassRls) {
+      throw new Error(
+        `[pipework] Test bootstrap failed:\n\n  ${rlsRoleMessage(name, probe)}\n`,
+      )
+    }
+  }
+
+  // Template identities are pure functions of config and migration content, so
+  // they are computed before any connection opens — the reaper needs the set of
+  // templates this run intends to use before it decides which templates are
+  // stale enough to drop.
+  const today = new Date()
+  const defaultName = instance.config.databaseNames()[0]
+  const plans: TemplatePlan[] = databases.map(([name, dbConfig]) => {
+    const fingerprint = templateFingerprint(
+      templateInputs(name, dbConfig, instance.config.raw, name === defaultName, today),
+    )
+    return {
+      name,
+      dbConfig,
+      fingerprint,
+      templateDb: templateDbName(name, fingerprint),
+      appRole: appRoleName(name, fingerprint),
+      isDefaultDatabase: name === defaultName,
+    }
+  })
+
+  dropBatchSize = instance.config.raw.test?.dropBatchSize ?? DEFAULT_DROP_BATCH_SIZE
+  databasePerFile = instance.config.raw.test?.databasePerFile ?? false
+  const cloneGuards = resolveCloneGuards(instance.config.raw)
+
+  const firstDb = databases[0]
+  if (firstDb !== undefined) {
+    tracker.reached('run-prepare')
+    await prepareRun(getAdminUrl(firstDb[1].url), new Set(plans.map(plan => plan.templateDb)), tracker)
+  }
+
+  for (const plan of plans) {
+    const template = await ensureTemplate(instance, plan, tracker)
+    const key = JSON.stringify([plan.name, plan.dbConfig.url, template.templateDb, cloneGuards])
+    const kept = databasePerFile ? undefined : await takeKept(key)
+    if (kept !== undefined) {
+      testDatabases.set(plan.name, kept.testDb)
+      continue
+    }
+    tracker.reached('clone-start', plan.name)
+    const testDb = await cloneTestDatabase(plan.name, plan.dbConfig, template, cloneGuards,
+      n => tracker.reached('clone-start', plan.name, `retry ${n}`))
+    tracker.reached('clone-done', plan.name)
+    if (!databasePerFile) {
+      const baseline = await captureBaseline(withDatabase(runAdminUrl, testDb.database))
+      if (baseline.hidden.length === 0) {
+        retained.set(testDb.database, { testDb, baseline, key, inUse: true, needsReset: false })
+      } else {
+        // Rows the admin role cannot read cannot be restored: this file's clone is dropped, as with databasePerFile.
+        console.warn(`${TEST_RESET_LOG_PREFIX} db=${testDb.database} not kept reason=rls tables=${baseline.hidden.join(',')}`)
+      }
+    }
+    testDatabases.set(plan.name, testDb)
+  }
+
+  tracker.reached('ready')
+  return testDatabases
+}
+
+// Acquire this run's advisory lock and reap orphans before provisioning. The
+// lock is taken before the first CREATE DATABASE so a concurrent run's reaper
+// can never mistake this run's freshly-created, backend-less databases — the
+// per-run clones included — for orphans. It is held on a dedicated
+// connection (idle/lifetime timeouts off) so Postgres releases it only when
+// this process dies, and is never unlocked per file: the template outlives
+// every file's database, so the lock that protects it must outlive them too.
+async function prepareRun(adminUrl: string, liveTemplates: ReadonlySet<string>, tracker: PhaseTracker): Promise<void> {
+  runAdminUrl = adminUrl
+  if (adminClient === null) {
+    adminClient = postgres(adminUrl, { max: 1, onnotice: () => {} })
+  }
+  if (lockClient === null) {
+    lockClient = postgres(adminUrl, { max: 1, idle_timeout: 0, max_lifetime: 0, onnotice: () => {} })
+    await lockClient`SELECT pg_advisory_lock(${runLockKey(runNonce).toString()}::bigint)`
+  }
+  armWorkerShutdown()
+  if (!reaped) {
+    reaped = true
+    try {
+      const result = await reapOrphanTestDatabases(adminClient, liveTemplates, {
+        onReap: (datname, i, total) => tracker.reached('run-reap', null, `${i}/${total} ${datname}`),
+      })
+      // #391: databases sharing this project's prefix that pipework did not create
+      // are left alone. Naming them once per run is what keeps a deliberate skip
+      // from looking like the reaper having nothing to do.
+      if (result.skippedDatabases.length > 0) {
+        process.stderr.write(
+          `${TEST_REAP_LOG_PREFIX} left alone (not created by pipework): ${result.skippedDatabases.join(', ')}\n`,
+        )
+      }
+    } catch {
+      // Reaping is best effort — never let stale-orphan cleanup break setup.
+    }
+  }
+}
+
+/**
+ * Get the shared template for one database, building it only if no process has
+ * built it yet.
+ *
+ * The protocol, in order, and the order is the point:
+ *
+ *  1. Take a SHARED advisory lock on the template's content key, on this run's
+ *     lifetime connection. That pins the template: the reaper drops a template
+ *     only under an EXCLUSIVE lock, so from here until this process dies, no
+ *     reaper can pull the template out from under a clone.
+ *  2. Ask whether the template is ready. Because the pin is already held, a
+ *     `yes` cannot go stale.
+ *  3. If not ready, TRY the separate BUILD key without blocking. Winning it
+ *     means nobody else is building: ask once more (a builder may have finished
+ *     between step 2 and the lock) and then build. Losing it means someone else
+ *     is building right now — so wait by re-asking whether the template is
+ *     ready, never by queueing on the exclusive key. Under a batch of forks
+ *     that is the difference between one lock queue everyone traverses in turn
+ *     and one builder with N cheap pollers: waiters proceed on the shared
+ *     use-lock alone, with no replay and no exclusive acquisition. The wait is
+ *     bounded; overrunning it fails here, by name, instead of surfacing as a
+ *     hook timeout somewhere else. Issue #362.
+ *
+ * There is deliberately no fallback: if the shared template cannot be built or
+ * validated, setup fails loudly. Quietly reverting to a private per-process
+ * build would re-hide exactly the cost this exists to remove. Issue #340.
+ */
+async function ensureTemplate(instance: Manifold, plan: TemplatePlan, tracker: PhaseTracker): Promise<Template> {
+  const cached = templates.get(plan.name)
+  if (cached !== undefined) return cached
+
+  const useKey = templateUseKey(plan.fingerprint).toString()
+  const buildKey = templateBuildKey(plan.fingerprint).toString()
+
+  tracker.reached('template-pin', plan.name)
+  await lockClient!`SELECT pg_advisory_lock_shared(${useKey}::bigint)`
+
+  await awaitTemplateBuild({
+    buildKey,
+    timeoutMs: templateBuildWaitMs(instance),
+    isReady: () => templateIsReady(plan),
+    tryLock: async () => {
+      const rows = await lockClient!<{ locked: boolean }[]>`
+        SELECT pg_try_advisory_lock(${buildKey}::bigint) AS locked
+      `
+      return rows[0]?.locked === true
+    },
+    unlock: async () => {
+      await lockClient!`SELECT pg_advisory_unlock(${buildKey}::bigint)`
+    },
+    build: () => {
+      tracker.reached('template-build', plan.name)
+      return buildTemplate(instance, plan, tracker)
+    },
+    builderPid: () => advisoryLockHolder(buildKey),
+    onWait: polls => tracker.reached('template-wait', plan.name, `polls=${polls}`),
+  })
+
+  tracker.reached('template-ready', plan.name)
+
+  // Checked on every process, not just the one that happened to build. A
+  // template is keyed by the migrations that made it; the QUEUES a manifold
+  // registers are not part of that key, so two instances can legitimately share
+  // one template while disagreeing about which queue tables must exist. Running
+  // this only inside the build would make the check depend on who got there
+  // first — which is to say, it would silently stop running.
+  tracker.reached('queue-check', plan.name)
+  await assertQueueTables(instance, plan)
+
+  const template: Template = {
+    templateDb: plan.templateDb,
+    appRole: plan.appRole,
+    fingerprint: plan.fingerprint,
+  }
+  templates.set(plan.name, template)
+  return template
+}
+
+// D2 loudness at test bootstrap: a registered queue whose table did not come
+// out of the migrations — or came out of an older one, missing a column the
+// current definition requires — would otherwise fail at first insert or first
+// claim, deep in a test.
+// Checked on the database queue migrations are emitted into — the sole
+// database, or the declared defaultDatabase (same rule as startup validation).
+async function assertQueueTables(instance: Manifold, plan: TemplatePlan): Promise<void> {
+  const declaredNames = instance.config.databaseNames()
+  const queueDbName = declaredNames.length === 1 ? declaredNames[0] : instance.config.raw.defaultDatabase
+  if (plan.name !== queueDbName) return
+
+  const problems = await namingDatabase(
+    plan.name, 'queue-check', () => queueTableProblems(siblingUrl(plan.dbConfig, plan.templateDb), null),
+  )
+  if (problems.length > 0) {
+    throw new Error(
+      `[pipework] Test bootstrap failed:\n\n` +
+      problems.map(p => `  ${queueTableProblemMessage(p, plan.name)}`).join('\n') + '\n',
+    )
+  }
+}
+
+/**
+ * Setup's probes leave the connect timeout to the driver (PGCONNECT_TIMEOUT,
+ * else 30 s), so a loaded server can be given longer than a pinned 5 s. When
+ * the probe still cannot connect, the error names the database and the step.
+ */
+async function namingDatabase<T>(database: string, step: string, probe: () => Promise<T>): Promise<T> {
+  try {
+    return await probe()
+  } catch (err) {
+    if (!isTransientProbeFailure(err)) throw err
+    const reason = describeError(err).message
+    throw new Error(
+      `[pipework] Test bootstrap could not reach database "${database}" (${step}): ${reason}. ` +
+      `The connect timeout is PGCONNECT_TIMEOUT seconds (default 30).`,
+      { cause: err },
+    )
+  }
+}
+
+/** Connection URL for a database on the same server as `dbConfig.url`. */
+function siblingUrl(dbConfig: ResolvedDatabase, database: string): string {
+  return withDatabase(dbConfig.url, database)
+}
+
+/** The same connection — role, password, server, parameters — pointed at `database`. */
+function withDatabase(connectionUrl: string, database: string): string {
+  const url = new URL(connectionUrl)
+  url.pathname = `/${database}`
+  return url.toString()
+}
+
+/**
+ * Is this template complete and usable?
+ *
+ * `false` means absent or unbuilt — no database (3D000), no marker table
+ * (42P01), a fingerprint that is not ours, or a cluster missing the app or
+ * platform role the build makes — and the caller rebuilds under
+ * the exclusive build lock. A connection that merely failed (57P01 from a peer's
+ * terminate, ECONNRESET, refused, timed out) is NOT an answer: it is retried,
+ * and if it never answers it throws. Answering `false` there would let one
+ * fork's transient blip drop and rebuild the template every other fork is
+ * cloning from. Issue #340.
+ */
+async function templateIsReady(plan: TemplatePlan): Promise<boolean> {
+  return probeTemplateReady(() => readTemplateMarker(plan))
+}
+
+/** One probe: connect, read the marker, disconnect. Throws on any failure. */
+async function readTemplateMarker(plan: TemplatePlan): Promise<boolean> {
+  const client = postgres(siblingUrl(plan.dbConfig, plan.templateDb), {
+    max: 1,
+    onnotice: () => {},
+  })
+  try {
+    const rows = await client<{ fingerprint: string }[]>`
+      SELECT fingerprint FROM ${client(TEMPLATE_MARKER_TABLE)} LIMIT 1
+    `
+    if (rows[0]?.fingerprint !== plan.fingerprint) return false
+    // Roles are cluster-global and do not travel with the database, so a
+    // marked template whose roles are gone cannot be cloned. Issue #812.
+    const wanted = templateRoles(plan)
+    const present = await client<{ rolname: string }[]>`
+      SELECT rolname FROM pg_roles WHERE rolname IN ${client(wanted)}
+    `
+    return present.length === wanted.length
+  } finally {
+    await client.end().catch(() => {})
+  }
+}
+
+/** The cluster-global roles a clone of this template depends on. */
+function templateRoles(plan: TemplatePlan): string[] {
+  const platformRole = plan.dbConfig.rls?.platformRole
+  return platformRole === undefined ? [plan.appRole] : [...new Set([plan.appRole, platformRole])]
+}
+
+/** The template is absent or unbuilt — a real `no`, and grounds for a rebuild. */
+const TEMPLATE_ABSENT = new Set(['3D000', '42P01'])
+
+/**
+ * The connection failed rather than answered. 57P01 is `admin_shutdown`: a peer
+ * terminated this backend. The rest are socket-level failures of the same kind.
+ */
+const TEMPLATE_TRANSIENT = new Set([
+  '57P01', '57P02', '57P03', '08006', '08003', '08001',
+  'ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'EPIPE', 'CONNECT_TIMEOUT', 'CONNECTION_CLOSED',
+])
+
+function errCode(err: unknown): string | undefined {
+  const code = (err as { code?: unknown } | null)?.code
+  return typeof code === 'string' ? code : undefined
+}
+
+function isTransientProbeFailure(err: unknown): boolean {
+  if (err === null || typeof err !== 'object') return false
+  const code = errCode(err)
+  if (code !== undefined && TEMPLATE_TRANSIENT.has(code)) return true
+  return isTransientProbeFailure((err as { cause?: unknown }).cause)
+}
+
+/** A probe that never answered. Never a reason to rebuild. */
+export class TemplateProbeFailed extends Error {
+  constructor(attempts: number, cause: unknown) {
+    super(`[pipework] Could not reach the shared test template after ${attempts} attempts`, { cause })
+    this.name = 'TemplateProbeFailed'
+  }
+}
+
+const PROBE_RETRY_DELAYS_MS: readonly number[] = [50, 150, 400, 1000]
+
+/**
+ * Run `probe` until it answers. An absent/unbuilt template answers `false`; a
+ * transient connection failure is retried and then thrown as
+ * `TemplateProbeFailed`; anything else propagates. Injectable for tests.
+ */
+export async function probeTemplateReady(
+  probe: () => Promise<boolean>,
+  sleep: (ms: number) => Promise<void> = ms => new Promise(resolve => setTimeout(resolve, ms)),
+  delaysMs: readonly number[] = PROBE_RETRY_DELAYS_MS,
+): Promise<boolean> {
+  for (let i = 0; ; i++) {
+    try {
+      return await probe()
+    } catch (err) {
+      const code = errCode(err)
+      if (code !== undefined && TEMPLATE_ABSENT.has(code)) return false
+      if (!isTransientProbeFailure(err)) throw err
+      if (i >= delaysMs.length) throw new TemplateProbeFailed(i + 1, err)
+      await sleep(delaysMs[i]!)
+    }
+  }
+}
+
+/** Default bound on waiting for another process's template build. */
+const DEFAULT_TEMPLATE_BUILD_WAIT_MS = 120_000
+
+/** Base gap between readiness re-asks while another process builds. */
+const BUILD_POLL_INTERVAL_MS = 500
+
+function templateBuildWaitMs(instance: Manifold): number {
+  return instance.config.raw.test?.templateBuildWaitMs ?? DEFAULT_TEMPLATE_BUILD_WAIT_MS
+}
+
+/** Waited out another process's template build. Names the key, the builder, the wait. */
+export class TemplateBuildWaitTimeout extends Error {
+  constructor(buildKey: string, builderPid: number | null, waitedMs: number) {
+    const who = builderPid === null
+      ? 'unknown — no backend currently holds the key'
+      : String(builderPid)
+    super(
+      `[pipework] Test bootstrap failed:\n\n` +
+      `  Waited ${(waitedMs / 1000).toFixed(1)}s for another process to finish building the shared\n` +
+      `  test template, and it has not finished.\n\n` +
+      `  build lock key: ${buildKey}\n` +
+      `  builder pid:    ${who}\n\n` +
+      `  Inspect that backend (pg_stat_activity), or raise the bound with\n` +
+      `  test.templateBuildWaitMs in pipework.config.ts.\n`,
+    )
+    this.name = 'TemplateBuildWaitTimeout'
+  }
+}
+
+export interface TemplateBuildWait {
+  buildKey: string
+  timeoutMs: number
+  /** Is the template built and ours? Never touches the build key. */
+  isReady: () => Promise<boolean>
+  /** Non-blocking `pg_try_advisory_lock` on the build key. */
+  tryLock: () => Promise<boolean>
+  unlock: () => Promise<void>
+  build: () => Promise<void>
+  builderPid: () => Promise<number | null>
+  sleep?: (ms: number) => Promise<void>
+  pollIntervalMs?: number
+  now?: () => number
+  /** Called before each wait for another process's build, with the poll count. */
+  onWait?: (polls: number) => void
+}
+
+/**
+ * Make sure the template exists, building it here only if no other process is
+ * already building it.
+ *
+ * Exactly one process per fingerprint builds: the one that wins the build key.
+ * Everyone else re-asks `isReady` on an interval — a poll, not a lock queue —
+ * so a batch of forks costs one build and N cheap probes rather than N serial
+ * acquisitions of the exclusive key. The try-lock in the loop is what recovers
+ * a builder that died: its lock goes with its connection, and the next poller
+ * wins the key and builds. Injectable for tests. Issue #362.
+ */
+export async function awaitTemplateBuild(wait: TemplateBuildWait): Promise<'ready' | 'built'> {
+  const sleep = wait.sleep ?? (ms => new Promise<void>(resolve => setTimeout(resolve, ms)))
+  const now = wait.now ?? (() => Date.now())
+  const interval = wait.pollIntervalMs ?? BUILD_POLL_INTERVAL_MS
+  const start = now()
+  let polls = 0
+
+  for (;;) {
+    if (await wait.isReady()) return 'ready'
+
+    if (await wait.tryLock()) {
+      try {
+        // A builder may have finished between the probe above and this lock.
+        if (await wait.isReady()) return 'ready'
+        await wait.build()
+        return 'built'
+      } finally {
+        await wait.unlock()
+      }
+    }
+
+    const waited = now() - start
+    if (waited >= wait.timeoutMs) {
+      throw new TemplateBuildWaitTimeout(wait.buildKey, await wait.builderPid(), waited)
+    }
+    // Jittered so a batch of forks does not re-probe in lockstep. Each wait is
+    // reported: a process queued behind a peer's build is making progress, and
+    // setup's stall watchdog must see that rather than count it as silence.
+    polls++
+    wait.onWait?.(polls)
+    await sleep(interval + Math.floor(Math.random() * interval))
+  }
+}
+
+/**
+ * The backend holding an advisory lock, or null. Diagnostic only: a failure to
+ * answer must never be the thing that fails a bootstrap — which is exactly why
+ * the predicate is covered by a real-Postgres test.
+ */
+export async function advisoryLockHolder(
+  key: string,
+  client: postgres.Sql = adminClient!,
+): Promise<number | null> {
+  try {
+    const rows = await client<{ pid: number }[]>`
+      SELECT pid FROM pg_locks
+      WHERE locktype = 'advisory'
+        AND granted
+        AND objsubid = 1
+        AND ((classid::bigint << 32) | objid::bigint) = ${key}::bigint
+      LIMIT 1
+    `
+    return rows[0]?.pid ?? null
+  } catch {
+    return null
+  }
+}
+
+// Build one shared template from scratch. Runs under the exclusive build lock,
+// with this process's shared use-lock already held. Issue #321, #340.
+async function buildTemplate(instance: Manifold, plan: TemplatePlan, tracker: PhaseTracker): Promise<void> {
+  const { dbConfig } = plan
+  const templateDb = plan.templateDb
+  const templateIdent = quoteIdentifier(templateDb, 'database')
+  const admin = drizzle(adminClient!)
+
+  // A template database that exists at this point failed its readiness check —
+  // the leftovers of a builder that died. Drop and start over.
+  tracker.reached('template-build', plan.name, 'drop')
+  await closeTemplateClients(admin, templateDb)
+  await withTemplateRetry(
+    () => admin.execute(sql.raw(`DROP DATABASE IF EXISTS ${templateIdent}`)),
+    { prefix: TEMPLATE_DROP_LOG_PREFIX, onRetry: n => tracker.reached('template-build', plan.name, `drop retry ${n}`) },
+  )
+  tracker.reached('template-build', plan.name, 'create')
+  await admin.execute(sql.raw(`CREATE DATABASE ${templateIdent}`))
+  // #325: a test database is dropped or rolled back — durable commits are pure
+  // overhead. `synchronous_commit` is a per-database setting and does not copy
+  // with `WITH TEMPLATE`, so it is set on the template and on every clone.
+  await admin.execute(sql.raw(`ALTER DATABASE ${templateIdent} SET synchronous_commit = off`))
+
+  const templateUrlStr = siblingUrl(dbConfig, templateDb)
+
+  const templateClient = postgres(templateUrlStr, { max: 1, onnotice: () => {} })
+  const templateDbHandle = drizzle(templateClient)
+
+  // The template's migrations run as the configured migration owner, so every
+  // object a migration creates is owned by it — the one thing that decides
+  // what a `SECURITY DEFINER` function made by a migration can read, and the
+  // one thing test mode used to get differently from production. Everything
+  // else stays on this run's own connection: the template is created, cloned
+  // and connected to as `url`, which is what keeps the RLS bootstrap probe and
+  // tenant isolation biting exactly as before.
+  const migrateUrlStr = dbConfig.testOwnerUrl !== undefined
+    ? withDatabase(dbConfig.testOwnerUrl, templateDb)
+    : undefined
+  if (migrateUrlStr !== undefined) {
+    tracker.reached('template-build', plan.name, 'owner')
+    const ownerIdent = quoteIdentifier(new URL(migrateUrlStr).username, 'role')
+    await admin.execute(sql.raw(`GRANT CONNECT ON DATABASE ${templateIdent} TO ${ownerIdent}`))
+    // PG15+: `public` belongs to the database owner and grants CREATE to
+    // nobody. Without this the first migration fails on its first CREATE.
+    await templateDbHandle.execute(sql.raw(`GRANT CREATE, USAGE ON SCHEMA public TO ${ownerIdent}`))
+  }
+
+  // Door policies in the migrations say `TO <platform role>`, so the role must
+  // exist before they run.
+  if (dbConfig.rls?.platformRole !== undefined) {
+    tracker.reached('template-build', plan.name, 'platform role')
+    await provisionPlatformRole(adminClient!, dbConfig.rls.platformRole, new URL(templateUrlStr).username)
+  }
+
+  if (dbConfig.extensions.length > 0) {
+    for (const ext of dbConfig.extensions) {
+      tracker.reached('template-build', plan.name, `extension ${ext}`)
+      await templateDbHandle.execute(sql.raw(`CREATE EXTENSION IF NOT EXISTS ${quoteIdentifier(ext, 'extension')}`))
+    }
+  }
+
+  if (dbConfig.migrations !== undefined) {
+    tracker.reached('template-migrate', plan.name, 'start')
+    await applyMigrations({
+      migrationsFolder: dbConfig.migrations,
+      migrationsTable: dbConfig.migrationTable,
+      connectionUrl: migrateUrlStr ?? templateUrlStr,
+      stripConcurrently: true,
+      unloggedTables: true,
+      // A migration stack is the one step long enough to look like a stall.
+      // Each file applied is an advance. Issue #380.
+      onMigration: (tag, index, total) =>
+        tracker.reached('template-migrate', plan.name, `${index}/${total} ${tag}`),
+      // Statements inside a long file, and the steps before the first file,
+      // are advances too. Issue #871.
+      onStatement: (tag, statement, statements, index, total) =>
+        tracker.reached('template-migrate', plan.name, `${index}/${total} ${tag} stmt ${statement}/${statements}`),
+      onPrepare: detail => tracker.reached('template-migrate', plan.name, `prepare ${detail}`),
+    })
+  }
+
+  // Framework-owned partitioned tables (trace family) are kept out of app
+  // migrations (#313) — pipework creates them and their day partitions here so
+  // a traced test never hits a missing relation or a missing partition.
+  tracker.reached('template-framework', plan.name)
+  await provisionFrameworkTables(
+    templateDbHandle,
+    instance.config.raw,
+    {
+      hotRetentionDays: instance.config.trace.hotRetentionDays,
+      errorRetentionDays: instance.config.trace.errorRetentionDays,
+      auditRetentionDays: instance.config.audit.retentionDays === 'keep' ? null : instance.config.audit.retentionDays,
+      partitionLeadDays: instance.config.trace.partitionLeadDays,
+    },
+    plan.isDefaultDatabase,
+    {
+      unlogged: true,
+      rls: dbConfig.rls,
+      // Every partition made is an advance: a year of audit days is hundreds
+      // of CREATEs, each of which can wait on the disk. Issue #806.
+      onProgress: (step, detail) => step === 'partition'
+        ? tracker.reached('template-partitions', plan.name, detail)
+        : tracker.reached('template-framework', plan.name, detail === undefined ? step : `${step} ${detail}`),
+    },
+  )
+
+  const grantBeat = (detail: string): void => tracker.reached('template-grants', plan.name, detail)
+  grantBeat('role')
+  await provisionTemplateRole(adminClient!, templateUrlStr, plan.appRole, migrateUrlStr, grantBeat)
+  if (dbConfig.rls !== undefined) await provisionTestSystemPolicy(templateUrlStr, migrateUrlStr, grantBeat)
+
+  // The marker is written last, and its presence is what makes this template
+  // clonable by any other process. The roles, grants and policies above must
+  // have succeeded first; a builder that dies before this point leaves a
+  // database that every reader correctly refuses to clone. Issue #812.
+  tracker.reached('template-framework', plan.name, 'marker')
+  await templateDbHandle.execute(sql.raw(
+    `CREATE TABLE ${quoteIdentifier(TEMPLATE_MARKER_TABLE, 'table')} (` +
+    `fingerprint text PRIMARY KEY, built_at timestamptz NOT NULL DEFAULT now(), builder_pid integer NOT NULL)`,
+  ))
+  await templateDbHandle.execute(sql`
+    INSERT INTO ${sql.identifier(TEMPLATE_MARKER_TABLE)} (fingerprint, builder_pid)
+    VALUES (${plan.fingerprint}, ${process.pid})
+  `)
+
+  await templateClient.end()
+}
+
+/**
+ * Create the platform role if the cluster lacks it: NOLOGIN, and never
+ * SUPERUSER or BYPASSRLS, which would make its door policies inert. The test
+ * login becomes a member so a test can `SET ROLE` to it. A role that already
+ * exists with either flag is refused rather than altered.
+ */
+async function provisionPlatformRole(client: postgres.Sql, roleName: string, testLogin: string): Promise<void> {
+  const roleIdent = quoteIdentifier(roleName, 'role')
+  await refusingWithoutCreateRole(client, roleName, () => client.unsafe(
+    `DO $$ BEGIN ` +
+    `IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${roleName}') THEN ` +
+    `CREATE ROLE ${roleIdent} NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE; ` +
+    `END IF; ` +
+    `EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
+  ))
+  const [flags] = await client<{ rolsuper: boolean; rolbypassrls: boolean }[]>`
+    SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname = ${roleName}
+  `
+  if (flags === undefined || flags.rolsuper || flags.rolbypassrls) {
+    throw new Error(
+      `[pipework] rls.platformRole "${roleName}" is ${flags?.rolsuper === true ? 'SUPERUSER' : 'BYPASSRLS'} on the test cluster — ` +
+      `row-level security never applies to it, so its doors would mean nothing. Use a role without either flag, or drop this one.`,
+    )
+  }
+  if (testLogin !== roleName) await client.unsafe(`GRANT ${roleIdent} TO ${quoteIdentifier(testLogin, 'role')}`)
+}
+
+/**
+ * Test template only: a permissive policy on every row-secured table that
+ * opens while `pipework.test_system` is `on`, which the test harness sets
+ * around statements sent through `pipe.system()`. It stands in for the owner's
+ * BYPASSRLS that production gives the system channel and a test login cannot
+ * have. Each policy is created by the table's owner, so the two grantors each
+ * take what they own.
+ */
+async function provisionTestSystemPolicy(
+  templateUrl: string,
+  migrateUrl: string | undefined,
+  beat: (detail: string) => void,
+): Promise<void> {
+  for (const url of migrateUrl === undefined ? [templateUrl] : [templateUrl, migrateUrl]) {
+    const client = postgres(url, { max: 1, onnotice: () => {} })
+    try {
+      const tables = await client<{ ident: string; name: string }[]>`
+        SELECT c.oid::regclass::text AS ident, c.relname AS name
+        FROM pg_class c
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relkind IN ('r', 'p')
+          AND c.relrowsecurity
+          AND NOT c.relispartition
+          AND n.nspname NOT IN ('pg_catalog', 'information_schema')
+          AND n.nspname NOT LIKE 'pg\_toast%'
+          AND pg_get_userbyid(c.relowner) = current_user
+          AND NOT EXISTS (SELECT FROM pg_policy p WHERE p.polrelid = c.oid AND p.polname = 'pipework_test_system')
+      `
+      const open = `current_setting('pipework.test_system', true) = 'on'`
+      for (const [i, t] of tables.entries()) {
+        beat(`policy ${i + 1}/${tables.length} ${t.name}`)
+        await client.unsafe(`CREATE POLICY pipework_test_system ON ${t.ident} AS PERMISSIVE FOR ALL USING (${open}) WITH CHECK (${open})`)
+      }
+    } finally {
+      await client.end()
+    }
+  }
+}
+
+// Clone one test file's database from the shared template. A clone copies
+// the template's tables, partitions, and in-database grants; only the cluster
+// catalog's CONNECT privilege has to be re-granted on the new database. Issue #321.
+// Session guards set on every per-test database. Teardown terminates a file's
+// backends before dropping (#379), which covers an orderly finish. A process
+// killed by pid is not covered: its backends keep running whatever statement
+// they are inside, because Postgres learns the client is gone only when it
+// next writes to the socket — a long statement never gets there. The orphan
+// holds locks, raises 55006 against the template, and blocks the drop. These
+// are set on the CLONE only, never on the template: a migration replay on the
+// template is legitimately long, and per-database settings do not copy with
+// `WITH TEMPLATE`. Issue #384.
+interface CloneGuards {
+  readonly statementTimeoutMs: number
+  readonly idleInTransactionTimeoutMs: number
+  readonly clientCheckIntervalMs: number
+}
+
+const DEFAULT_STATEMENT_TIMEOUT_MS = 120_000
+const DEFAULT_IDLE_IN_TRANSACTION_TIMEOUT_MS = 120_000
+const DEFAULT_CLIENT_CHECK_INTERVAL_MS = 1_000
+
+function resolveCloneGuards(raw: Blueprint): CloneGuards {
+  return {
+    statementTimeoutMs: raw.test?.statementTimeoutMs ?? DEFAULT_STATEMENT_TIMEOUT_MS,
+    idleInTransactionTimeoutMs: raw.test?.idleInTransactionTimeoutMs ?? DEFAULT_IDLE_IN_TRANSACTION_TIMEOUT_MS,
+    clientCheckIntervalMs: raw.test?.clientCheckIntervalMs ?? DEFAULT_CLIENT_CHECK_INTERVAL_MS,
+  }
+}
+
+/** The `ALTER DATABASE ... SET` statements for one clone. 0 means "leave it alone". */
+export function cloneGuardStatements(dbIdent: string, guards: CloneGuards): string[] {
+  const settings: [string, number][] = [
+    ['statement_timeout', guards.statementTimeoutMs],
+    ['idle_in_transaction_session_timeout', guards.idleInTransactionTimeoutMs],
+    ['client_connection_check_interval', guards.clientCheckIntervalMs],
+  ]
+  return settings
+    .filter(([, ms]) => ms > 0)
+    .map(([name, ms]) => `ALTER DATABASE ${dbIdent} SET ${name} = ${ms}`)
+}
+
+async function cloneTestDatabase(name: string, dbConfig: ResolvedDatabase, template: Template, guards: CloneGuards, onRetry: (n: number) => void): Promise<TestDatabase> {
+  const dbName = testDbName(name)
+  const dbIdent = quoteIdentifier(dbName, 'database')
+  const templateIdent = quoteIdentifier(template.templateDb, 'database')
+  const roleIdent = quoteIdentifier(template.appRole, 'role')
+  const admin = drizzle(adminClient!)
+
+  // No `DROP DATABASE IF EXISTS` here. The name carries a per-clone sequence
+  // number, so it has never existed before — and a DROP of a database that DOES
+  // exist would force a checkpoint on the create path too (#379).
+  // CREATE ... WITH TEMPLATE refuses to run while any backend is connected to
+  // the template; close out stragglers, then create with a retry that waits out
+  // anything we couldn't (an autovacuum worker we have no right to terminate).
+  await closeTemplateClients(admin, template.templateDb)
+  await withTemplateRetry(() =>
+    admin.execute(sql.raw(`CREATE DATABASE ${dbIdent} WITH TEMPLATE ${templateIdent}`)),
+    { onRetry },
+  )
+  await admin.execute(sql.raw(`ALTER DATABASE ${dbIdent} SET synchronous_commit = off`))
+  for (const statement of cloneGuardStatements(dbIdent, guards)) {
+    await admin.execute(sql.raw(statement))
+  }
+  await admin.execute(sql.raw(`GRANT CONNECT ON DATABASE ${dbIdent} TO ${roleIdent}`))
+
+  const testUrl = new URL(dbConfig.url)
+  testUrl.pathname = `/${dbName}`
+
+  const appUrl = new URL(testUrl.toString())
+  appUrl.username = template.appRole
+  appUrl.password = 'pipework_test'
+
+  return {
+    name,
+    url: testUrl.toString(),
+    appUrl: appUrl.toString(),
+    appRole: template.appRole,
+    database: dbName,
+  }
+}
+
+// Close straggler client connections to the template so the clone can proceed.
+// Three guards keep this from aborting the clone — or from sabotaging a peer:
+//   - `backend_type = 'client backend'` excludes background workers. An
+//     autovacuum worker attaches to every database and runs as superuser, and a
+//     non-superuser test role cannot signal it — targeting it would raise
+//     `permission denied to terminate process` (42501) and abort the statement.
+//   - `state = 'idle'` AND idle for 5s excludes backends doing work AND live
+//     peers. A peer fork's readiness probe is an idle client backend for a few
+//     ms between connecting and disconnecting; terminating it made the peer
+//     read the blip as "template absent" and rebuild the shared template out
+//     from under this clone. Only a backend idle for seconds is a straggler.
+//   - best-effort: if a client backend is still unkillable (owned by a role we
+//     cannot signal), swallow it. The `CREATE` retry waits out whatever remains.
+async function closeTemplateClients(admin: ReturnType<typeof drizzle>, templateDb: string): Promise<void> {
+  try {
+    await admin.execute(sql`
+      SELECT pg_terminate_backend(pid)
+      FROM pg_stat_activity
+      WHERE datname = ${templateDb}
+        AND pid <> pg_backend_pid()
+        AND backend_type = 'client backend'
+        AND state = 'idle'
+        AND state_change < now() - interval '5 seconds'
+    `)
+  } catch {
+    // best-effort — withTemplateRetry tolerates any backend we couldn't close
+  }
+}
+
+// SQLSTATE 55006 (object_in_use): `CREATE DATABASE ... WITH TEMPLATE` raises this
+// while any session is still attached to the template — including a transient
+// autovacuum worker, which a non-superuser test role cannot terminate. The
+// worker detaches in well under a second, so retry with capped backoff rather
+// than depend on superuser termination rights.
+const OBJECT_IN_USE = '55006'
+
+// The query builder wraps the PostgresError, so the SQLSTATE may sit on `.cause`.
+function isObjectInUse(err: unknown): boolean {
+  if (err === null || typeof err !== 'object') return false
+  return errCode(err) === OBJECT_IN_USE || isObjectInUse((err as { cause?: unknown }).cause)
+}
+
+// ~9.6 s in total: long enough to wait out real contention on a wide run,
+// where several forks clone from one template and each probe attaches to it
+// for a few ms.
+const TEMPLATE_RETRY_DELAYS_MS: readonly number[] = [50, 100, 200, 400, 800, 1600, 1600, 1600, 1600, 1600]
+
+// src/test has no logger — it runs before a Manifold exists — so retry lines go
+// to stderr. Injectable so a test can read them without a global. Every line
+// starts with the operation's prefix and carries key=value fields, so an infra
+// script can grep the prefix and pull the numbers out without parsing prose.
+// The two operations that retry 55006 get separate prefixes rather than one
+// prefix and an op= field: a grep for template-clone then tallies clone
+// contention only, and never a drop that a dead builder's leftovers forced.
+export type RetryLog = (message: string) => void
+
+export const TEMPLATE_CLONE_LOG_PREFIX = 'pipework:template-clone'
+export const TEMPLATE_DROP_LOG_PREFIX = 'pipework:template-drop'
+
+export interface TemplateRetryOptions {
+  /** Injectable for tests. */
+  sleep?: (ms: number) => Promise<void>
+  /** Injectable for tests. */
+  delaysMs?: readonly number[]
+  /** Injectable for tests. */
+  log?: RetryLog
+  /** Injectable for tests. */
+  now?: () => number
+  /** Names the operation the lines are about. Defaults to the clone prefix. */
+  prefix?: string
+  /** Called with the retry number before each backoff sleep. */
+  onRetry?: (n: number) => void
+}
+
+/**
+ * Run `attempt`, retrying only on SQLSTATE 55006 with capped exponential
+ * backoff. Any other error propagates immediately; 55006 past the last delay
+ * propagates too. Every retry, and the eventual recovery or exhaustion, is
+ * logged under `prefix`, so contention on the shared template is visible in a
+ * run's output and exhaustion is stated rather than inferred from a missing
+ * line; an attempt that succeeds first try says nothing. Reported elapsed is
+ * wall clock from before the first attempt, so it counts the failed
+ * round-trips, not just the sleeps.
+ */
+export async function withTemplateRetry<T>(
+  attempt: () => Promise<T>,
+  options: TemplateRetryOptions = {},
+): Promise<T> {
+  const {
+    sleep = (ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms)),
+    delaysMs = TEMPLATE_RETRY_DELAYS_MS,
+    log = (message: string) => console.warn(message),
+    now = () => Date.now(),
+    prefix = TEMPLATE_CLONE_LOG_PREFIX,
+    onRetry,
+  } = options
+
+  const start = now()
+  for (let i = 0; ; i++) {
+    try {
+      const result = await attempt()
+      if (i > 0) {
+        log(`${prefix} recovered sqlstate=${OBJECT_IN_USE} retries=${i} elapsed_ms=${now() - start}`)
+      }
+      return result
+    } catch (err) {
+      // A non-55006 error is not a contention event and stays out of this
+      // channel; an exhausted ladder says so, so a tally never has to read a
+      // fork killed mid-retry as an exhaustion.
+      if (!isObjectInUse(err)) throw err
+      if (i >= delaysMs.length) {
+        log(`${prefix} exhausted sqlstate=${OBJECT_IN_USE} retries=${i} elapsed_ms=${now() - start}`)
+        throw err
+      }
+      const delay = delaysMs[i]!
+      log(
+        `${prefix} retry sqlstate=${OBJECT_IN_USE} ` +
+        `retry=${i + 1}/${delaysMs.length} delay_ms=${delay} elapsed_ms=${now() - start}`,
+      )
+      onRetry?.(i + 1)
+      await sleep(delay)
+    }
+  }
+}
+
+const INSUFFICIENT_PRIVILEGE = '42501'
+
+function hasSqlState(err: unknown, code: string): boolean {
+  if (err === null || typeof err !== 'object') return false
+  return errCode(err) === code || hasSqlState((err as { cause?: unknown }).cause, code)
+}
+
+/** Run a CREATE ROLE; a refusal for lack of CREATEROLE names the login and the fix. Issue #812. */
+async function refusingWithoutCreateRole(client: postgres.Sql, roleName: string, create: () => PromiseLike<unknown>): Promise<void> {
+  try {
+    await create()
+  } catch (err) {
+    if (!hasSqlState(err, INSUFFICIENT_PRIVILEGE)) throw err
+    const [who] = await client<{ login: string }[]>`SELECT current_user AS login`.catch(() => [])
+    const login = who?.login ?? 'the test login'
+    throw new Error(
+      `[pipework] The test login "${login}" needs CREATEROLE on the test cluster to make role "${roleName}" ` +
+      `(ALTER ROLE "${login}" CREATEROLE).`,
+      { cause: err },
+    )
+  }
+}
+
+// Create the app role (cluster-global, once per process) and grant it on the
+// template's tables, sequences, schema, and future objects. Those grants are
+// in-database catalog rows, so every clone of the template inherits them — the
+// role keeps a stable OID for the process lifetime, so the inherited ACL
+// entries stay valid. Issue #321.
+async function provisionTemplateRole(
+  client: postgres.Sql,
+  templateUrl: string,
+  roleName: string,
+  migrateUrl: string | undefined,
+  beat: (detail: string) => void,
+): Promise<void> {
+  const admin = drizzle(client)
+  const roleIdent = quoteIdentifier(roleName, 'role')
+
+  await refusingWithoutCreateRole(client, roleName, () => admin.execute(sql.raw(
+    `DO $$ BEGIN ` +
+    `IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${roleName}') THEN ` +
+    `CREATE ROLE ${roleIdent} WITH LOGIN PASSWORD 'pipework_test' NOSUPERUSER NOCREATEDB NOCREATEROLE; ` +
+    `END IF; ` +
+    `END $$`,
+  )))
+
+  const templateClient = postgres(templateUrl, { max: 1, onnotice: () => {} })
+  const templateDb = drizzle(templateClient)
+
+  await templateDb.execute(sql.raw(`GRANT USAGE ON SCHEMA public TO ${roleIdent}`))
+
+  if (migrateUrl === undefined) {
+    beat('tables')
+    await templateDb.execute(sql.raw(`GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO ${roleIdent}`))
+    await templateDb.execute(sql.raw(`GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ${roleIdent}`))
+    await templateDb.execute(sql.raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ${roleIdent}`))
+    await templateDb.execute(sql.raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO ${roleIdent}`))
+    // Routines are granted here, after the migrations have run, because this is
+    // the one place where roles are created AFTER the objects they must reach:
+    // a migration that creates a function and revokes EXECUTE from PUBLIC would
+    // otherwise leave the app role unable to call it.
+    beat('routines')
+    await templateDb.execute(sql.raw(`GRANT EXECUTE ON ALL ROUTINES IN SCHEMA public TO ${roleIdent}`))
+    await templateDb.execute(sql.raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT EXECUTE ON ROUTINES TO ${roleIdent}`))
+    await templateClient.end()
+    return
+  }
+
+  // Two owners in one template: the migration owner made everything the
+  // migrations made, this run's own role made the extensions' objects, the
+  // framework tables. `GRANT ... ON ALL TABLES` refuses on the
+  // first object its grantor does not own instead of skipping it, so each
+  // grantor grants what it owns and nothing else.
+  await grantOwnedObjects(templateClient, roleIdent, APP_ROLE_GRANTS, beat)
+
+  const migrateClient = postgres(migrateUrl, { max: 1, onnotice: () => {} })
+  try {
+    await grantOwnedObjects(migrateClient, roleIdent, APP_ROLE_GRANTS, beat)
+    // The test login is the role tests connect as, and it no longer owns what
+    // the migrations made. Everything short of ownership is granted back to
+    // it; DDL that requires ownership (ALTER, DROP) needs the test login to be
+    // a MEMBER of the migration owner, which is the venue's business — pipework
+    // has no right to hand out role membership.
+    await grantOwnedObjects(migrateClient, quoteIdentifier(new URL(templateUrl).username, 'role'), TEST_LOGIN_GRANTS, beat)
+  } finally {
+    await migrateClient.end()
+  }
+
+  await templateClient.end()
+}
+
+/** The privileges one grantor hands one role, by object kind. */
+interface GrantSet {
+  readonly tables: string
+  readonly sequences: string
+  readonly routines: string
+}
+
+const APP_ROLE_GRANTS: GrantSet = {
+  tables: 'SELECT, INSERT, UPDATE, DELETE',
+  sequences: 'USAGE, SELECT',
+  routines: 'EXECUTE',
+}
+
+const TEST_LOGIN_GRANTS: GrantSet = {
+  tables: 'ALL PRIVILEGES',
+  sequences: 'ALL PRIVILEGES',
+  routines: 'EXECUTE',
+}
+
+// Grant on the objects in `public` that THIS connection's role owns, plus the
+// default privileges for what it creates later. Ownership is read from the
+// catalog rather than assumed, so a template with two object owners is granted
+// correctly from either side and neither one's sweep can fail on the other's
+// objects. Object names come back as `regclass`/`regprocedure` text, already
+// quoted by Postgres and carrying a routine's argument types.
+async function grantOwnedObjects(
+  client: postgres.Sql,
+  roleIdent: string,
+  grants: GrantSet,
+  beat: (detail: string) => void,
+): Promise<void> {
+  const relations = await client<{ ident: string; kind: string }[]>`
+    SELECT c.oid::regclass::text AS ident, c.relkind AS kind
+    FROM pg_class c
+    JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname = 'public'
+      AND c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S')
+      AND pg_get_userbyid(c.relowner) = current_user
+  `
+  const tables = relations.filter(r => r.kind !== 'S').map(r => r.ident)
+  const sequences = relations.filter(r => r.kind === 'S').map(r => r.ident)
+
+  const routines = await client<{ ident: string }[]>`
+    SELECT p.oid::regprocedure::text AS ident
+    FROM pg_proc p
+    JOIN pg_namespace n ON n.oid = p.pronamespace
+    WHERE n.nspname = 'public'
+      AND pg_get_userbyid(p.proowner) = current_user
+  `
+
+  if (tables.length > 0) {
+    await client.unsafe(`GRANT ${grants.tables} ON TABLE ${tables.join(', ')} TO ${roleIdent}`)
+  }
+  if (sequences.length > 0) {
+    await client.unsafe(`GRANT ${grants.sequences} ON SEQUENCE ${sequences.join(', ')} TO ${roleIdent}`)
+  }
+  for (const [i, routine] of routines.entries()) {
+    beat(`routine ${i + 1}/${routines.length} ${routine.ident}`)
+    await client.unsafe(`GRANT ${grants.routines} ON ROUTINE ${routine.ident} TO ${roleIdent}`)
+  }
+
+  await client.unsafe(`ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ${grants.tables} ON TABLES TO ${roleIdent}`)
+  await client.unsafe(`ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ${grants.sequences} ON SEQUENCES TO ${roleIdent}`)
+  await client.unsafe(`ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ${grants.routines} ON ROUTINES TO ${roleIdent}`)
+}
+
+// Close this run's persistent admin and lock connections. Releasing
+// `lockClient` drops the run's advisory lock — so this runs exactly once, at
+// worker shutdown, never per file: these connections hold the run's advisory
+// lock and this run's shared pin on every template it uses, and both must
+// outlive every file in the worker.
+async function closeRunConnections(): Promise<void> {
+  if (adminClient !== null) {
+    await adminClient.end()
+    adminClient = null
+    adminIsSuperuser = null
+  }
+  if (lockClient !== null) {
+    await lockClient.end()
+    lockClient = null
+  }
+}
+
+// Arm a one-time close of the persistent connections at worker shutdown.
+// Under `pool: 'forks'` each vitest worker is a forked child with an IPC
+// channel; vitest disconnects that channel when the worker has no more files,
+// firing `disconnect` — the only once-per-worker signal, since a per-file
+// `afterAll` cannot know which file is the worker's last. The admin and lock
+// connections are opened with no idle timeout, so their open sockets keep the
+// worker's event loop alive; closing them here lets the worker exit naturally
+// (#774). Holding them until then keeps the advisory lock for the worker's
+// whole life, so a sibling worker's reaper can never mistake this run's live
+// template for an orphan. Issue #321. Vitest 4 never sends `disconnect` to a
+// fork (it SIGTERMs it), so there the plugin's `reap-after-run` drops the
+// remainder from the parent instead (#810).
+function armWorkerShutdown(): void {
+  if (workerShutdownArmed) return
+  workerShutdownArmed = true
+  if (typeof process.send === 'function') {
+    process.once('disconnect', () => {
+      // Flush FIRST: the drops need the admin connection this then closes.
+      void flushPendingDrops('shutdown').finally(() => closeRunConnections())
+    })
+  }
+}
+
+// Drop this file's clones. The shared template and its app role deliberately
+// outlive teardown — the next file, the next process, and the next run all
+// reuse it, and a reaper drops it only once its content key goes stale (#340). The run's admin and lock
+// connections also survive: `lockClient` holds the advisory lock that protects
+// the live template from a sibling worker's reaper, so it must stay open for
+// the worker's whole life — `armWorkerShutdown` closes both once, at worker
+// exit. A non-forked run has no IPC channel and so no `disconnect` signal, but
+// also no sibling worker and so no concurrent reaper: closing the connections
+// per file is then both safe and necessary for the process to exit. Issue #321.
+export async function teardownTestDatabases(): Promise<void> {
+  if (adminClient === null) {
+    testDatabases.clear()
+    return
+  }
+
+  const admin = drizzle(adminClient)
+
+  // Kept databases are reset by the next setup that takes them, not here: a
+  // process that ends after one file never pays for it. Without an IPC channel
+  // nothing follows this teardown (see below): drop instead.
+  const canReuse = typeof process.send === 'function'
+  for (const [, testDb] of testDatabases) {
+    try {
+      await admin.execute(sql`SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = ${testDb.database} AND pid != pg_backend_pid()`)
+      const keep = retained.get(testDb.database)
+      if (keep !== undefined && canReuse) {
+        keep.inUse = false
+        keep.needsReset = true
+      } else {
+        retained.delete(testDb.database)
+        pendingDrops.push(testDb.database)
+      }
+    } catch {
+      // Best effort cleanup — a leftover clone is reaped by the next run.
+    }
+  }
+
+  testDatabases.clear()
+
+  if (pendingDrops.length >= dropBatchSize) {
+    await flushPendingDrops('batch')
+  }
+
+  if (typeof process.send !== 'function') {
+    // No IPC channel means no `disconnect` signal and so no later flush: this
+    // teardown is the last chance to settle what is queued.
+    await flushPendingDrops('shutdown')
+    await closeRunConnections()
+  }
+}
+
+/**
+ * An idle kept database for this key, reset and marked in use. One whose reset
+ * fails is queued for drop and the next candidate tried; none left means the
+ * caller clones.
+ */
+async function takeKept(key: string): Promise<RetainedDatabase | undefined> {
+  for (const keep of retained.values()) {
+    if (keep.key !== key || keep.inUse) continue
+    if (keep.needsReset && !(await renameKept(keep) && await resetKept(keep))) {
+      retained.delete(keep.testDb.database)
+      pendingDrops.push(keep.testDb.database)
+      continue
+    }
+    keep.needsReset = false
+    keep.inUse = true
+    return keep
+  }
+  return undefined
+}
+
+/**
+ * Give a kept database a new name before its next file, so that anything the
+ * last file left running — a timer, a job loop, a pool — and that connects by
+ * the old URL finds no database, as it would with a dropped clone, instead of
+ * writing into the next file's. Sessions still attached are ended first; the
+ * rename waits out one that attaches in between. False: it cannot be renamed
+ * and needs a fresh clone.
+ */
+async function renameKept(keep: RetainedDatabase): Promise<boolean> {
+  const oldName = keep.testDb.database
+  const newName = testDbName(keep.testDb.name)
+  const admin = drizzle(adminClient!)
+  for (let attempt = 0; attempt < RENAME_ATTEMPTS; attempt++) {
+    try {
+      await admin.execute(sql`SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = ${oldName} AND pid <> pg_backend_pid()`)
+      await admin.execute(sql.raw(`ALTER DATABASE ${quoteIdentifier(oldName, 'database')} RENAME TO ${quoteIdentifier(newName, 'database')}`))
+    } catch {
+      await new Promise(resolve => setTimeout(resolve, 50 * (attempt + 1)))
+      continue
+    }
+    retained.delete(oldName)
+    keep.testDb = {
+      ...keep.testDb,
+      database: newName,
+      url: withDatabase(keep.testDb.url, newName),
+      appUrl: withDatabase(keep.testDb.appUrl, newName),
+    }
+    retained.set(newName, keep)
+    return true
+  }
+  console.warn(`${TEST_RESET_LOG_PREFIX} db=${oldName} fresh-clone reason=rename`)
+  return false
+}
+
+const RENAME_ATTEMPTS = 5
+
+/** Reset a kept database to its baseline and say so. False: it needs a fresh clone. */
+async function resetKept(keep: RetainedDatabase): Promise<boolean> {
+  const start = Date.now()
+  adminIsSuperuser ??= await readSuperuser()
+  const outcome = await resetDatabase(withDatabase(runAdminUrl, keep.testDb.database), keep.baseline, adminIsSuperuser)
+  const db = keep.testDb.database
+  console.warn(outcome.ok
+    ? `${TEST_RESET_LOG_PREFIX} db=${db} cleared=${outcome.cleared} reloaded=${outcome.reloaded} elapsed_ms=${Date.now() - start}`
+    : `${TEST_RESET_LOG_PREFIX} db=${db} fresh-clone reason=${outcome.reason}`)
+  return outcome.ok
+}
+
+// Per-file databases that are done with, waiting to be dropped together.
+//
+// A worker keeps its databases across files (`test.databasePerFile` off), so
+// what reaches this queue is a database whose reset fell back to a fresh clone,
+// every kept database at worker shutdown, and every database with
+// `databasePerFile`.
+//
+// Every `DROP DATABASE` forces an IMMEDIATE cluster-wide checkpoint and waits
+// for it — `RequestCheckpoint(CHECKPOINT_IMMEDIATE | FORCE | WAIT)` in
+// Postgres's own `dropdb()`. `CREATE DATABASE` does not: since PG15 its default
+// strategy is WAL_LOG, which needs no checkpoint at all. So the drop, not the
+// clone, is what a test file pays for, and it pays SYNCHRONOUSLY inside the
+// teardown hook: on a saturated cluster those checkpoints were measured at
+// 20-47 s apiece, which is the teardown hook timing out.
+//
+// The count of checkpoints is fixed by the count of databases — batching cannot
+// change it. What batching changes is WHERE the wait lands: one teardown in
+// `dropBatchSize` pays it instead of every one, and the rest of a worker's
+// drops are settled at worker shutdown, off the critical path of a wide start
+// burst. A worker that dies with drops still queued leaks nothing permanent —
+// its run lock dies with it and the next run's reaper drops them. Issue #379.
+const pendingDrops: string[] = []
+
+const DEFAULT_DROP_BATCH_SIZE = 8
+
+let dropBatchSize = DEFAULT_DROP_BATCH_SIZE
+
+export const TEST_DROP_LOG_PREFIX = 'pipework:test-drop'
+
+/**
+ * Drop every queued database, back to back. Best effort per database: one that
+ * refuses (a backend attached between teardown and here) stays for the reaper
+ * rather than failing the flush for the rest.
+ */
+export async function flushPendingDrops(
+  reason: 'batch' | 'shutdown',
+  log: RetryLog = message => console.warn(message),
+  now: () => number = () => Date.now(),
+): Promise<number> {
+  if (reason === 'shutdown') {
+    pendingDrops.push(...retained.keys())
+    retained.clear()
+  }
+  if (adminClient === null || pendingDrops.length === 0) return 0
+
+  const admin = drizzle(adminClient)
+  const batch = pendingDrops.splice(0, pendingDrops.length)
+  const start = now()
+  let dropped = 0
+
+  for (const database of batch) {
+    try {
+      await admin.execute(sql.raw(`DROP DATABASE IF EXISTS ${quoteIdentifier(database, 'database')}`))
+      dropped++
+    } catch {
+      // Left for the next run's reaper.
+    }
+  }
+
+  // Each drop is a forced checkpoint the cluster had to wait out, so what the
+  // batch cost is worth stating — an infra script greps the prefix and reads
+  // the numbers without parsing prose, as the template-retry lines do.
+  log(`${TEST_DROP_LOG_PREFIX} ${reason} queued=${batch.length} dropped=${dropped} elapsed_ms=${now() - start}`)
+  return dropped
+}
+
+/** Test-only: how many databases are waiting to be dropped. */
+export function pendingDropCount(): number {
+  return pendingDrops.length
+}
+
+export function getTestDatabase(name: string): TestDatabase | undefined {
+  return testDatabases.get(name)
+}
+
+export function getTestDatabases(): ReadonlyMap<string, TestDatabase> {
+  return testDatabases
+}
+
+/**
+ * Forget the in-process template cache, so the next `setupTestDatabases` takes
+ * the same path a freshly-forked worker takes: shared lock, readiness check,
+ * clone. This is how a single process can prove the cross-process claim — that
+ * a second fork reuses the template rather than rebuilding it — without
+ * actually forking. Test-only. Issue #340.
+ */
+export function forgetTemplateCacheForTests(): void {
+  templates.clear()
+  reaped = false
+}