pipework 0.252.0 → 0.252.1

package/src/test/progress.ts

@@ -0,0 +1,185 @@
+// Test setup is one awaited call from a vitest `beforeAll`. When forty forks
+// start at once, a healthy setup can spend a minute queued behind its peers —
+// and the consumer's hook timeout fires with nothing to say but the number of
+// milliseconds it waited. So setup narrates: every step it reaches is reported
+// to the caller, and the bound it enforces itself is on PHASE ADVANCE, never on
+// total elapsed time. A queued setup that keeps advancing finishes late and
+// green; a genuinely stuck one fails naming the phase it reached. Issue #380.
+
+/** Steps `setupTestDatabases` passes through, in the order it reaches them. */
+export type SetupPhase =
+  | 'rls-probe'
+  | 'run-prepare'
+  | 'run-reap'
+  | 'template-pin'
+  | 'template-wait'
+  | 'template-build'
+  | 'template-migrate'
+  | 'template-framework'
+  | 'template-partitions'
+  | 'template-grants'
+  | 'template-ready'
+  | 'queue-check'
+  | 'clone-start'
+  | 'clone-done'
+  | 'ready'
+
+export interface SetupProgress {
+  readonly phase: SetupPhase
+  /** The configured database this step is about, or null for whole-run steps. */
+  readonly database: string | null
+  /** Free-form detail — a migration tag, a poll count. Never parsed. */
+  readonly detail?: string
+  /** Milliseconds since setup started. */
+  readonly elapsedMs: number
+  /** The phase this one displaced, and how long it held. Null on the first. */
+  readonly previous: { readonly phase: SetupPhase, readonly database: string | null, readonly heldMs: number } | null
+}
+
+export type SetupProgressListener = (progress: SetupProgress) => void
+
+/** Setup made no progress for the configured bound. Names the last phase reached. */
+export class SetupStalled extends Error {
+  constructor(
+    readonly phase: SetupPhase,
+    readonly database: string | null,
+    readonly stalledMs: number,
+    readonly elapsedMs: number,
+  ) {
+    const where = database === null ? phase : `${phase} (database "${database}")`
+    super(
+      `[pipework] Test setup failed:\n\n` +
+      `  No progress for ${(stalledMs / 1000).toFixed(1)}s.\n\n` +
+      `  last phase reached: ${where}\n` +
+      `  total elapsed:      ${(elapsedMs / 1000).toFixed(1)}s\n\n` +
+      `  This bound is on phase advance, not on total time — setup queued behind\n` +
+      `  other processes keeps advancing and does not trip it. Something at the\n` +
+      `  phase above is stuck. Raise the bound with test.setupStallMs in\n` +
+      `  pipework.config.ts if the step is legitimately this slow.\n`,
+    )
+    this.name = 'SetupStalled'
+  }
+}
+
+/**
+ * Records the phase setup is in and when it got there. The watchdog reads it;
+ * the listener is told about every advance.
+ */
+export class PhaseTracker {
+  phase: SetupPhase = 'run-prepare'
+  database: string | null = null
+  private started = false
+  private at: number
+  private readonly start: number
+
+  constructor(
+    private readonly listener: SetupProgressListener = () => {},
+    private readonly clock: () => number = () => Date.now(),
+  ) {
+    this.start = this.clock()
+    this.at = this.start
+  }
+
+  /** How long the current phase has held. */
+  idleMs(): number {
+    return this.clock() - this.at
+  }
+
+  elapsedMs(): number {
+    return this.clock() - this.start
+  }
+
+  reached(phase: SetupPhase, database: string | null = null, detail?: string): void {
+    const now = this.clock()
+    const previous = this.started
+      ? { phase: this.phase, database: this.database, heldMs: now - this.at }
+      : null
+    this.started = true
+    this.phase = phase
+    this.database = database
+    this.at = now
+    this.listener({
+      phase,
+      database,
+      ...(detail === undefined ? {} : { detail }),
+      elapsedMs: now - this.start,
+      previous,
+    })
+  }
+}
+
+export const DEFAULT_SETUP_STALL_MS = 120_000
+
+/**
+ * Timeout for the `beforeAll` that runs setup: setTimeout's ceiling, so vitest's
+ * hookTimeout never bounds a setup that is still advancing. Only the stall
+ * watchdog fails setup.
+ */
+export const SETUP_HOOK_TIMEOUT_MS = 2 ** 31 - 1
+
+export interface StallWatchOptions {
+  stallMs?: number
+  /** How often the watchdog looks. Defaults to a tenth of the bound. */
+  pollMs?: number
+}
+
+/**
+ * Run `work`, failing it if `tracker` reports no phase advance for `stallMs`.
+ *
+ * The watchdog only ever REJECTS earlier than `work` would settle — it cannot
+ * make a completed setup fail, and its timer is unref'd so it never holds the
+ * process open.
+ */
+export async function withStallWatchdog<T>(
+  work: () => Promise<T>,
+  tracker: PhaseTracker,
+  options: StallWatchOptions = {},
+): Promise<T> {
+  const stallMs = options.stallMs ?? DEFAULT_SETUP_STALL_MS
+  const pollMs = options.pollMs ?? Math.max(50, Math.floor(stallMs / 10))
+
+  let timer: ReturnType<typeof setInterval> | undefined
+  const stalled = new Promise<never>((_resolve, reject) => {
+    timer = setInterval(() => {
+      const idle = tracker.idleMs()
+      if (idle >= stallMs) {
+        reject(new SetupStalled(tracker.phase, tracker.database, idle, tracker.elapsedMs()))
+      }
+    }, pollMs)
+    timer.unref()
+  })
+
+  try {
+    return await Promise.race([work(), stalled])
+  } finally {
+    if (timer !== undefined) clearInterval(timer)
+  }
+}
+
+/** Prefix for the stderr lines `auto-setup` writes about slow phases. */
+export const SETUP_PROGRESS_LOG_PREFIX = 'pipework:test-setup'
+
+/**
+ * A phase that held this long is worth a line. Below it setup is silent: a
+ * healthy run must not spray forty lines per test file.
+ */
+export const SLOW_PHASE_MS = 10_000
+
+/**
+ * The listener `auto-setup` installs. Silent while setup is quick, one
+ * key=value line per phase that took long enough to explain a late hook.
+ */
+export function logSlowPhases(
+  write: (line: string) => void = message => console.warn(message),
+  slowMs: number = SLOW_PHASE_MS,
+): SetupProgressListener {
+  return progress => {
+    const previous = progress.previous
+    if (previous === null || previous.heldMs < slowMs) return
+    const db = previous.database === null ? '' : ` db=${previous.database}`
+    write(
+      `${SETUP_PROGRESS_LOG_PREFIX} slow phase=${previous.phase}${db} ` +
+      `held_ms=${previous.heldMs} next=${progress.phase} elapsed_ms=${progress.elapsedMs}`,
+    )
+  }
+}

package/src/test/reap-after-run.ts

@@ -0,0 +1,28 @@
+import { loadEnvFiles } from '../core/config/env.js'
+import { discoverInstance } from '../core/config/discover.js'
+import { reapConfiguredTestDatabases } from './reap.js'
+import { TEST_REAP_LOG_PREFIX } from './setup.js'
+
+// Vitest globalSetup, added by the pipework plugin. `teardown` runs in the
+// parent after every worker has stopped, so the clones a SIGTERMed fork left
+// queued are orphans by then: their run locks died with the workers. Clones
+// only (templates outlive runs), and only those whose run lock is free, so a
+// concurrent run's live clones are never touched. Best effort, never fails a run.
+export function setup(): void {}
+
+export async function teardown(): Promise<void> {
+  try {
+    loadEnvFiles()
+    const instance = await discoverInstance()
+    try {
+      const { droppedDatabases } = await reapConfiguredTestDatabases(instance, { clonesOnly: true })
+      if (droppedDatabases.length > 0) {
+        process.stderr.write(`${TEST_REAP_LOG_PREFIX} dropped ${String(droppedDatabases.length)} leftover clone(s): ${droppedDatabases.join(', ')}\n`)
+      }
+    } finally {
+      await instance.pool.closeAll()
+    }
+  } catch {
+    // Best effort — the next run's reaper drops what this missed.
+  }
+}

package/src/test/reap.ts

@@ -0,0 +1,286 @@
+import { createHash } from 'node:crypto'
+import postgres from 'postgres'
+import type { Manifold } from '../core/pipework.js'
+import { quoteIdentifier } from '../data/db/identifiers.js'
+import {
+  testDbPrefix,
+  appRolePrefix,
+  testDbLikePattern,
+  templateDbPrefix,
+  templateDbLikePattern,
+} from './namespace.js'
+
+
+// The run nonce every name `testDbName` writes ends with: 12 random hex chars.
+// Carrying one is the reaper's evidence that it created the database, and the nonce
+// rebuilds the owning run's advisory-lock key. Deliberately not the fuller
+// `_<seq>_<pid>_<nonce>` tail: an orphan left by an older pipework whose names had
+// no clone sequence would then never be reaped by any later version.
+//
+// The residual limit, stated rather than hidden: a foreign database inside this
+// project's prefix whose name happens to end in exactly 12 hex characters is
+// indistinguishable from a clone and is still eligible. Nothing in the name can
+// separate them; `PIPEWORK_TEST_NAMESPACE` is the way out of a shared prefix.
+const CLONE_SUFFIX = /_([0-9a-f]{12})$/
+
+export interface ReapOptions {
+  /**
+   * Called before each candidate database is handled, `index` counting 1..`total`
+   * across the clones and the stale templates together, so a hung drop is named.
+   */
+  onReap?: (datname: string, index: number, total: number) => void
+  /** Leave shared templates alone: only per-run clones are candidates. */
+  clonesOnly?: boolean
+}
+
+export interface ReapResult {
+  readonly droppedDatabases: readonly string[]
+  /**
+   * Databases inside this project's prefix that the reaper did NOT create and so
+   * will not drop. Reported rather than dropped: sharing a prefix is not evidence
+   * of ownership, and a name pipework never wrote belongs to somebody else (#391).
+   */
+  readonly skippedDatabases: readonly string[]
+}
+
+/**
+ * Advisory-lock key identifying a test run. A live run holds this lock on a
+ * dedicated connection for its entire lifetime; Postgres releases it
+ * automatically when that connection ends by any means (clean exit, crash,
+ * SIGKILL, reboot). The reaper reads "lock is free" as "owning run is dead".
+ */
+export function runLockKey(nonce: string): bigint {
+  return BigInt('0x' + nonce)
+}
+
+/**
+ * Advisory-lock keys for a shared template, derived from its content
+ * fingerprint. Both are salted hashes truncated to 60 bits, which keeps them
+ * inside a signed `bigint` and makes an accidental collision with a 48-bit run
+ * lock key or with each other a non-event.
+ *
+ * Two keys, because the two things being coordinated are different:
+ *
+ *  - the USE key is taken in SHARE mode by every process that intends to clone
+ *    the template, for that process's whole life, and in EXCLUSIVE mode by the
+ *    reaper. That is what stops a reaper from dropping a template between
+ *    another process's readiness check and its `CREATE DATABASE ... WITH
+ *    TEMPLATE`: the reaper simply cannot get the lock while anyone holds it.
+ *  - the BUILD key is taken in EXCLUSIVE mode only while building, so that N
+ *    processes starting at once produce one template and one migration replay
+ *    rather than N. It is separate because advisory locks do not upgrade from
+ *    share to exclusive, and a builder must keep its own pin while it builds.
+ *
+ * Issue #340.
+ */
+function templateLockKey(salt: string, fingerprint: string): bigint {
+  return BigInt('0x' + createHash('sha256').update(`${salt}:${fingerprint}`).digest('hex').slice(0, 15))
+}
+
+export function templateUseKey(fingerprint: string): bigint {
+  return templateLockKey('pipework-template-use', fingerprint)
+}
+
+export function templateBuildKey(fingerprint: string): bigint {
+  return templateLockKey('pipework-template-build', fingerprint)
+}
+
+/** Admin (`/postgres`) connection URL for a database URL. */
+export function getAdminUrl(databaseUrl: string): string {
+  const url = new URL(databaseUrl)
+  url.pathname = '/postgres'
+  return url.toString()
+}
+
+function pairedRoleName(datname: string): string {
+  return appRolePrefix() + datname.slice(testDbPrefix().length)
+}
+
+/** A shared template's app role is content-keyed the same way the template is. */
+function pairedTemplateRoleName(datname: string): string {
+  return appRolePrefix() + datname.slice(templateDbPrefix().length)
+}
+
+/** Content fingerprint trailing a shared template's name: 16 lowercase hex. */
+const TEMPLATE_FINGERPRINT_SUFFIX = /_([0-9a-f]{16})$/
+
+/**
+ * Drop test databases left behind by runs that never reached teardown
+ * (Ctrl-C, timeout, crash). A database is reaped only when ALL THREE hold:
+ *
+ *   - the reaper created it — its name carries the 12-hex run nonce every name
+ *     `testDbName` writes. Sharing the project prefix is not evidence of
+ *     ownership: a cluster carrying `<project>_test_<slot>` control databases
+ *     lost all of them at once to a prefix-only sweep (#391). Anything else in
+ *     the prefix is reported in `skippedDatabases` and left alone;
+ *   - its run's advisory lock is free — the owning run's process is gone; and
+ *   - `numbackends = 0` — no connections remain, so reaping cannot pull a
+ *     database out from under workers orphaned by a SIGKILL'd parent.
+ *
+ * All three only ever make the reaper more conservative, so it can never drop a
+ * database belonging to a live or still-provisioning run, or to anyone else.
+ */
+export async function reapOrphanTestDatabases(
+  admin: postgres.Sql,
+  liveTemplates: ReadonlySet<string> = new Set(),
+  options: ReapOptions = {},
+): Promise<ReapResult> {
+  const rows = await admin<{ datname: string, numbackends: number }[]>`
+    SELECT datname, numbackends
+    FROM pg_stat_database
+    WHERE datname LIKE ${testDbLikePattern()}
+  `
+  const templateRows = options.clonesOnly === true ? [] : await staleTemplateCandidates(admin)
+  const total = rows.length + templateRows.length
+  const { onReap } = options
+  const droppedDatabases: string[] = []
+  const skippedDatabases: string[] = []
+  for (const [i, { datname, numbackends }] of rows.entries()) {
+    onReap?.(datname, i + 1, total)
+    // A name in this prefix that pipework did not write. It could be anything —
+    // a control database, a hand-made fixture, another tool's namespace that
+    // happens to collide. Dropping it because the prefix matched is what #391
+    // reported; say what was left alone instead.
+    const nonce = CLONE_SUFFIX.exec(datname)?.[1]
+    if (nonce === undefined) {
+      skippedDatabases.push(datname)
+      continue
+    }
+
+    if (numbackends > 0) continue
+
+    const key = runLockKey(nonce)
+    if (!(await tryAcquire(admin, key))) continue
+
+    try {
+      if (await dropDatabase(admin, datname)) {
+        droppedDatabases.push(datname)
+        await dropRole(admin, pairedRoleName(datname))
+      }
+    } finally {
+      await admin`SELECT pg_advisory_unlock(${key.toString()}::bigint)`
+    }
+  }
+  const templates = await reapStaleTemplates(admin, liveTemplates, templateRows, (datname, i) => onReap?.(datname, rows.length + i, total))
+  droppedDatabases.push(...templates.droppedDatabases)
+  skippedDatabases.push(...templates.skippedDatabases)
+  return { droppedDatabases, skippedDatabases }
+}
+
+/**
+ * Drop shared templates nobody wants any more.
+ *
+ * A template is deliberately long-lived — it survives the run that built it, so
+ * the next run clones instead of replaying migrations (#340) — which means
+ * "left behind by a dead run" is no longer a reason to drop one. The two
+ * reasons that remain:
+ *
+ *   - it is not in `liveTemplates`, the set of content keys the calling run
+ *     actually intends to use. Editing a migration changes the key, so the
+ *     template built from the old migrations is named something nobody will ask
+ *     for again. That is what makes invalidation automatic rather than manual.
+ *   - nobody holds its use lock, in any mode. Every process that intends to
+ *     clone a template holds that lock in SHARE mode for its whole life, so
+ *     failing to take it EXCLUSIVE means no live process is cloning from it —
+ *     including no process sitting between a readiness check and its `CREATE
+ *     DATABASE ... WITH TEMPLATE`.
+ *
+ * `numbackends = 0` is required as well, for the same conservatism as clones.
+ *
+ * An empty `liveTemplates` — what `pipework test --reap` passes — therefore
+ * means "drop every template not currently in use", which is the right reading
+ * of an explicit cleanup command.
+ */
+async function reapStaleTemplates(
+  admin: postgres.Sql,
+  liveTemplates: ReadonlySet<string>,
+  rows: readonly { datname: string, numbackends: number }[],
+  onEach: (datname: string, index: number) => void,
+): Promise<ReapResult> {
+  const droppedDatabases: string[] = []
+  const skippedDatabases: string[] = []
+  for (const [i, { datname, numbackends }] of rows.entries()) {
+    onEach(datname, i + 1)
+    const fingerprint = TEMPLATE_FINGERPRINT_SUFFIX.exec(datname)?.[1]
+    // A name in the template namespace without a fingerprint was not written by
+    // this scheme. Leave it alone rather than guess at its owner — and say so.
+    if (fingerprint === undefined) {
+      skippedDatabases.push(datname)
+      continue
+    }
+
+    if (numbackends > 0) continue
+    if (liveTemplates.has(datname)) continue
+
+    const key = templateUseKey(fingerprint)
+    if (!(await tryAcquire(admin, key))) continue
+
+    try {
+      if (await dropDatabase(admin, datname)) {
+        droppedDatabases.push(datname)
+        await dropRole(admin, pairedTemplateRoleName(datname))
+      }
+    } finally {
+      await admin`SELECT pg_advisory_unlock(${key.toString()}::bigint)`
+    }
+  }
+  return { droppedDatabases, skippedDatabases }
+}
+
+async function staleTemplateCandidates(admin: postgres.Sql): Promise<{ datname: string, numbackends: number }[]> {
+  return [...await admin<{ datname: string, numbackends: number }[]>`
+    SELECT datname, numbackends
+    FROM pg_stat_database
+    WHERE datname LIKE ${templateDbLikePattern()}
+  `]
+}
+
+/**
+ * Reap orphan test databases across every Postgres server referenced by the
+ * manifold's database config. Used by `pipework test --reap`.
+ */
+export async function reapConfiguredTestDatabases(instance: Manifold, options: ReapOptions = {}): Promise<ReapResult> {
+  const adminUrls = new Set<string>()
+  for (const [, dbConfig] of instance.config.databases()) {
+    adminUrls.add(getAdminUrl(dbConfig.url))
+  }
+  const droppedDatabases: string[] = []
+  const skippedDatabases: string[] = []
+  for (const adminUrl of adminUrls) {
+    const client = postgres(adminUrl, { max: 1, onnotice: () => {} })
+    try {
+      const result = await reapOrphanTestDatabases(client, new Set(), options)
+      droppedDatabases.push(...result.droppedDatabases)
+      skippedDatabases.push(...result.skippedDatabases)
+    } finally {
+      await client.end()
+    }
+  }
+  return { droppedDatabases, skippedDatabases }
+}
+
+async function tryAcquire(admin: postgres.Sql, key: bigint): Promise<boolean> {
+  const rows = await admin<{ locked: boolean }[]>`
+    SELECT pg_try_advisory_lock(${key.toString()}::bigint) AS locked
+  `
+  return rows[0]?.locked === true
+}
+
+async function dropDatabase(admin: postgres.Sql, datname: string): Promise<boolean> {
+  try {
+    await admin.unsafe(`DROP DATABASE IF EXISTS ${quoteIdentifier(datname, 'database')}`)
+    return true
+  } catch {
+    // A backend connected between the numbackends check and the DROP — leave
+    // this database for the next run's reaper.
+    return false
+  }
+}
+
+async function dropRole(admin: postgres.Sql, role: string): Promise<void> {
+  try {
+    await admin.unsafe(`DROP ROLE IF EXISTS ${quoteIdentifier(role, 'role')}`)
+  } catch {
+    // Role still owns objects in another database — best effort.
+  }
+}