pipework 0.252.0 → 0.252.1

package/src/data/migrate/applied.ts

@@ -0,0 +1,47 @@
+import { sql } from '../query/index.js'
+import type { DB } from '../db/index.js'
+import { quoteIdentifier } from '../db/identifiers.js'
+
+/**
+ * What the tracker table says has been applied, in whichever vocabulary the
+ * table currently speaks:
+ *
+ * - `tags`: the upgraded shape — applied = the exact set of recorded migration
+ *   tags (see `applyMigrations`, which owns the membership semantics and the
+ *   legacy-table upgrade).
+ * - `legacy-threshold`: a not-yet-upgraded table (no `tag` column) can only
+ *   express the old high-water mark, `max(created_at)`. It is read for exactly
+ *   one purpose: scoping the pre-apply SAFETY check to pending migrations, so
+ *   a fully-current database with a destructive op in its applied history can
+ *   still run a clean no-op `migrate`. `applyMigrations` upgrades the table on
+ *   its next run, after which this variant never appears again.
+ *
+ * `null` — the table does not exist; nothing has been applied.
+ */
+export type AppliedState =
+  | { kind: 'tags'; tags: ReadonlySet<string> }
+  | { kind: 'legacy-threshold'; threshold: number }
+
+export async function readAppliedState(db: DB, migrationsTable: string): Promise<AppliedState | null> {
+  const quoted = quoteIdentifier(migrationsTable, 'migration table')
+
+  const [reg] = await db.execute(sql`SELECT to_regclass(${quoted}) AS reg`)
+  if (reg === undefined || reg['reg'] === null) return null
+
+  const [tagColumn] = await db.execute(
+    sql`SELECT 1 AS present FROM pg_attribute
+        WHERE attrelid = to_regclass(${quoted}) AND attname = 'tag' AND NOT attisdropped`,
+  )
+
+  if (tagColumn === undefined) {
+    const [row] = await db.execute(sql.raw(`SELECT max(created_at) AS last_applied FROM ${quoted}`))
+    const lastApplied = row?.['last_applied']
+    return {
+      kind: 'legacy-threshold',
+      threshold: lastApplied === null || lastApplied === undefined ? -1 : Number(lastApplied),
+    }
+  }
+
+  const rows = await db.execute(sql.raw(`SELECT tag FROM ${quoted} WHERE tag IS NOT NULL`))
+  return { kind: 'tags', tags: new Set(rows.map((row) => String(row['tag']))) }
+}

package/src/data/migrate/apply.ts

@@ -0,0 +1,309 @@
+import { readFileSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import postgres from 'postgres'
+import { scanStatements } from './statements.js'
+import { stripPragmaLines } from './pragma.js'
+import { listMigrations } from './history.js'
+
+const NO_TRANSACTION_PRAGMA = '-- pipework:no-transaction'
+
+export interface ApplyOptions {
+  migrationsFolder: string
+  migrationsTable: string
+  connectionUrl: string
+  stripConcurrently?: boolean
+  /**
+   * Rewrite `CREATE TABLE` to `CREATE UNLOGGED TABLE`. A test database is
+   * dropped or rolled back and never needs crash durability, so skipping the
+   * WAL for its tables is free speed with no semantic change. Issue #325.
+   */
+  unloggedTables?: boolean
+  /**
+   * Called after each migration file is applied and tracked. Replaying a stack
+   * is the longest single step of test setup; without this it is one silent
+   * await and a caller watching for progress sees none. Issue #380.
+   */
+  onMigration?: (tag: string, index: number, total: number) => void
+  /**
+   * Called after each statement of a migration file lands (inside the file's
+   * transaction, when it has one), with the 1-based statement and the file's
+   * statement count, then the 1-based position of the file among those being
+   * applied and their count. A long file is otherwise silent until it commits.
+   * Issue #871.
+   */
+  onStatement?: (tag: string, statement: number, statements: number, index: number, total: number) => void
+  /**
+   * Called as each step before the first file finishes (tracker table, legacy
+   * tag recovery per file, the unique index, the applied read), with a detail
+   * naming it. Issue #871.
+   */
+  onPrepare?: (detail: string) => void
+  /**
+   * Called when a no-transaction migration resumes a half-applied file, with
+   * the statement it starts from (1-based) and how many the file holds.
+   * Issue #378.
+   */
+  onResume?: (tag: string, fromStatement: number, total: number) => void
+}
+
+export interface Migration {
+  tag: string
+  hash: string
+  /** Hash of the file with `expect-*` pragma lines removed; also accepted as the recorded one. */
+  pragmaFreeHash?: string
+  folderMillis: number
+  statements: string[]
+  noTransaction: boolean
+}
+
+export async function applyMigrations(options: ApplyOptions): Promise<number> {
+  const migrations = readMigrations(options)
+  if (migrations.length === 0) return 0
+
+  const client = postgres(options.connectionUrl, { max: 1, onnotice: () => {} })
+
+  try {
+    const table = quoteIdent(options.migrationsTable)
+    const prepared = (detail: string): void => options.onPrepare?.(detail)
+
+    // "Applied" is MEMBERSHIP — this exact migration (by tag, verified by
+    // content hash) has a tracker row — never a timestamp high-water mark.
+    // A threshold silently skips any unapplied migration dated earlier than
+    // the newest applied one: a branch merged late, or a second manifold
+    // sharing this tracker table whose whole journal predates the first
+    // manifold's newest entry (a real production incident for both).
+    await client.unsafe(`
+      CREATE TABLE IF NOT EXISTS ${table} (
+        id SERIAL PRIMARY KEY,
+        hash text NOT NULL,
+        created_at bigint,
+        tag text
+      )
+    `)
+    prepared('tracker table')
+    // Deployed databases predate the tag column: add it, then recover each
+    // legacy row's identity by matching its recorded content hash against the
+    // journal's files (min(id) claims one row per tag, so a duplicated hash
+    // can't double-claim). Rows matching no current journal entry keep a NULL
+    // tag and never participate in membership.
+    await client.unsafe(`ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS tag text`)
+    for (const [i, migration] of migrations.entries()) {
+      await client.unsafe(
+        `UPDATE ${table} SET tag = $1
+         WHERE id = (SELECT min(id) FROM ${table} WHERE hash = $2 AND tag IS NULL)`,
+        [migration.tag, migration.hash],
+      )
+      prepared(`tags ${i + 1}/${migrations.length}`)
+    }
+    await client.unsafe(
+      `CREATE UNIQUE INDEX IF NOT EXISTS ${quoteIdent(`${options.migrationsTable}_tag_key`)} ON ${table} (tag)`,
+    )
+    prepared('tag index')
+
+    const rows = await client.unsafe<{ tag: string | null; hash: string }[]>(
+      `SELECT tag, hash FROM ${table} WHERE tag IS NOT NULL`,
+    )
+    const appliedHashByTag = new Map(rows.map((row) => [row.tag!, row.hash]))
+    prepared('applied read')
+
+    // A no-transaction file has no rollback: interrupted between statements it
+    // leaves the database half-changed with no tracker row, and the rerun
+    // replays from the top and dies on the first already-applied statement.
+    // Nor can the file be edited to add guards — its content is immutable
+    // history. So each statement's completion is recorded as it lands, and a
+    // rerun starts from the first one not recorded. The table exists only for
+    // databases that actually have a no-transaction migration. Issue #378.
+    const progress = quoteIdent(progressTableName(options.migrationsTable))
+    const hasNoTransaction = migrations.some((migration) => migration.noTransaction)
+    const progressByTag = new Map<string, { hash: string; done: number }>()
+    if (hasNoTransaction) {
+      await client.unsafe(`
+        CREATE TABLE IF NOT EXISTS ${progress} (
+          tag text PRIMARY KEY,
+          hash text NOT NULL,
+          statements_done integer NOT NULL,
+          updated_at timestamptz NOT NULL DEFAULT now()
+        )
+      `)
+      const partial = await client.unsafe<{ tag: string; hash: string; statements_done: number }[]>(
+        `SELECT tag, hash, statements_done FROM ${progress}`,
+      )
+      for (const row of partial) {
+        progressByTag.set(row.tag, { hash: row.hash, done: row.statements_done })
+      }
+      prepared('progress table')
+    }
+
+    let applied = 0
+
+    for (const migration of migrations) {
+      const appliedHash = appliedHashByTag.get(migration.tag)
+      if (appliedHash !== undefined) {
+        if (!hashMatches(appliedHash, migration)) {
+          throw new Error(
+            `Migration "${migration.tag}" is recorded as applied in ${options.migrationsTable}, but its ` +
+              `file content has changed (recorded sha256 ${appliedHash.slice(0, 12)}…, file is now ` +
+              `${migration.hash.slice(0, 12)}…). Applied migrations are immutable history — put schema ` +
+              `changes in a NEW migration (\`pipework generate\`) instead of editing an applied one (adding "-- pipework:expect-*" lines is allowed). If two ` +
+              `manifolds share this database and both declare a migration tagged "${migration.tag}", that is ` +
+              `a schema-level collision no tracker can reconcile — give them separate databases or distinct ` +
+              `migrationTable names.`,
+          )
+        }
+        continue
+      }
+
+      if (migration.noTransaction) {
+        const from = resumePoint(migration, progressByTag.get(migration.tag), options.migrationsTable)
+        if (from > 0) options.onResume?.(migration.tag, from + 1, migration.statements.length)
+
+        for (let i = from; i < migration.statements.length; i++) {
+          await client.unsafe(migration.statements[i]!)
+          // Recorded AFTER the statement it describes: marking first would let
+          // a death between the mark and the statement skip work silently,
+          // which is worse than replaying one statement. The window that
+          // remains is exactly one statement wide — the one whose mark had not
+          // landed yet — instead of the whole file.
+          await client.unsafe(
+            `INSERT INTO ${progress} (tag, hash, statements_done) VALUES ($1, $2, $3)
+             ON CONFLICT (tag) DO UPDATE SET statements_done = EXCLUDED.statements_done, updated_at = now()`,
+            [migration.tag, migration.hash, i + 1],
+          )
+          options.onStatement?.(migration.tag, i + 1, migration.statements.length, applied + 1, migrations.length)
+        }
+        await client.unsafe(
+          `INSERT INTO ${table} ("hash", "created_at", "tag") VALUES ($1, $2, $3)`,
+          [migration.hash, migration.folderMillis, migration.tag],
+        )
+        await client.unsafe(`DELETE FROM ${progress} WHERE tag = $1`, [migration.tag])
+      } else {
+        await client.begin(async (tx) => {
+          for (const [i, stmt] of migration.statements.entries()) {
+            await tx.unsafe(stmt)
+            options.onStatement?.(migration.tag, i + 1, migration.statements.length, applied + 1, migrations.length)
+          }
+          await tx.unsafe(
+            `INSERT INTO ${table} ("hash", "created_at", "tag") VALUES ($1, $2, $3)`,
+            [migration.hash, migration.folderMillis, migration.tag],
+          )
+        })
+      }
+
+      applied++
+      options.onMigration?.(migration.tag, applied, migrations.length)
+    }
+
+    return applied
+  } finally {
+    await client.end()
+  }
+}
+
+export function readMigrations(options: ApplyOptions): Migration[] {
+  const migrations: Migration[] = []
+
+  for (const entry of listMigrations(options.migrationsFolder)) {
+    const filePath = `${options.migrationsFolder}/${entry.tag}.sql`
+    let raw = readFileSync(filePath, 'utf-8')
+    const noTransaction = raw.includes(NO_TRANSACTION_PRAGMA)
+
+    if (options.stripConcurrently === true) {
+      raw = raw.replace(/\bCONCURRENTLY\s+/gi, '')
+    }
+
+    if (options.unloggedTables === true) {
+      raw = makeTablesUnlogged(raw)
+    }
+
+    const statements = parseStatements(raw)
+
+    const fileText = readFileSync(filePath, 'utf-8')
+    migrations.push({
+      tag: entry.tag,
+      hash: sha256(fileText),
+      pragmaFreeHash: sha256(stripPragmaLines(fileText)),
+      folderMillis: entry.when,
+      statements,
+      noTransaction,
+    })
+  }
+
+  return migrations
+}
+
+function sha256(text: string): string {
+  return createHash('sha256').update(text).digest('hex')
+}
+
+/**
+ * True when a recorded hash is this migration's: the raw file hash, or the
+ * hash with `expect-*` pragma lines removed (those lines may be added to an
+ * applied file; nothing else may change).
+ */
+function hashMatches(recorded: string, migration: Migration): boolean {
+  return recorded === migration.hash || recorded === migration.pragmaFreeHash
+}
+
+/**
+ * Split migration SQL into individual statements, honoring drizzle's
+ * `--> statement-breakpoint` as an explicit separator.
+ *
+ * The scanning rules live in `statements.ts`, shared with the safety gate so
+ * the runner and the gate cannot disagree about where a statement begins —
+ * they did, and the gate was the one that was wrong (#334).
+ */
+export function parseStatements(raw: string): string[] {
+  return scanStatements(raw, { breakpointSeparator: true })
+}
+
+/** Companion to the tracker table, holding half-applied no-transaction files. */
+export function progressTableName(migrationsTable: string): string {
+  return `${migrationsTable}_progress`
+}
+
+/**
+ * The statement index a half-applied no-transaction file resumes from. Zero
+ * when there is no record of one.
+ *
+ * A recorded partial whose hash no longer matches the file is refused rather
+ * than resumed: the mark counts statements in the file as it was, and the file
+ * on disk is now a different one, so no index into it means anything.
+ */
+export function resumePoint(
+  migration: Migration,
+  recorded: { hash: string; done: number } | undefined,
+  migrationsTable: string,
+): number {
+  if (recorded === undefined) return 0
+  if (!hashMatches(recorded.hash, migration)) {
+    throw new Error(
+      `Migration "${migration.tag}" is half applied — ${recorded.done} of its statements ran before the ` +
+        `runner stopped, recorded in ${progressTableName(migrationsTable)} — and its file content has ` +
+        `changed since (recorded sha256 ${recorded.hash.slice(0, 12)}…, file is now ` +
+        `${migration.hash.slice(0, 12)}…). The recorded count indexes into the file as it was, so it ` +
+        `cannot be carried into the new one. Restore the file to the content that was running, let the ` +
+        `resume finish, and put the change in a NEW migration; or, if the half-applied statements have ` +
+        `been undone by hand, delete the "${migration.tag}" row from ` +
+        `${progressTableName(migrationsTable)} to start the file over.`,
+    )
+  }
+  // A count past the end means every statement landed and only the tracker row
+  // was missed: nothing left to run, and the caller writes the row.
+  return Math.min(recorded.done, migration.statements.length)
+}
+
+function quoteIdent(name: string): string {
+  return `"${name.replace(/"/g, '""')}"`
+}
+
+/**
+ * Rewrite every `CREATE TABLE` (including the `IF NOT EXISTS` form) to
+ * `CREATE UNLOGGED TABLE`. Used only when applying schema to a test database —
+ * a test database is dropped or rolled back, so the WAL its tables would
+ * generate is pure overhead. `CREATE UNLOGGED TABLE` is otherwise identical
+ * SQL: same columns, constraints, RLS, isolation. `TEMPORARY` and
+ * already-`UNLOGGED` tables are left alone. Issue #325.
+ */
+export function makeTablesUnlogged(sql: string): string {
+  return sql.replace(/\bCREATE\s+TABLE\b/gi, 'CREATE UNLOGGED TABLE')
+}

package/src/data/migrate/check.ts

@@ -0,0 +1,119 @@
+import { readFileSync, existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { analyzeSQL, formatHazards, type MigrationHazard, type Severity } from './safety.js'
+import { hasPragma, PRAGMA_FOR, type PragmaKind } from './pragma.js'
+import type { AppliedState } from './applied.js'
+import { listMigrations, type MigrationEntry } from './history.js'
+
+export interface FileHazards {
+  file: string
+  hazards: MigrationHazard[]
+  /** Hazard classes this file raises but does not declare a pragma for. */
+  undeclared: PragmaKind[]
+}
+
+export interface MigrationCheckResult {
+  safe: boolean
+  hasDestructive: boolean
+  hasLocking: boolean
+  hasDynamic: boolean
+  /** True when every hazard raised is declared by its own migration. */
+  declared: boolean
+  fileHazards: FileHazards[]
+}
+
+/**
+ * Scans pending migrations for destructive and locking operations.
+ *
+ * `applied` is what the tracker table records as already applied (see
+ * `readAppliedState`); applied entries are skipped — their hazards are
+ * history, not a decision the operator still gets to make. Membership is by
+ * tag on an upgraded tracker, or by the old `when`-timestamp high-water mark
+ * on a tracker `applyMigrations` has not upgraded yet. Omitting `applied`
+ * (or passing null — no tracker table) scans every entry, the correct
+ * behaviour when nothing is known to be applied.
+ */
+export function checkMigrations(migrationsFolder: string, applied?: AppliedState | null): MigrationCheckResult {
+  const entries = listMigrations(migrationsFolder)
+  const fileHazards: FileHazards[] = []
+  let hasDestructive = false
+  let hasLocking = false
+  let hasDynamic = false
+  let declared = true
+
+  const isApplied = (entry: MigrationEntry): boolean => {
+    if (applied === undefined || applied === null) return false
+    if (applied.kind === 'tags') return applied.tags.has(entry.tag)
+    return entry.when <= applied.threshold
+  }
+
+  for (const entry of entries) {
+    if (isApplied(entry)) continue
+
+    const sqlPath = join(migrationsFolder, `${entry.tag}.sql`)
+    if (!existsSync(sqlPath)) continue
+
+    const sql = readFileSync(sqlPath, 'utf-8')
+    const hazards = analyzeSQL(sql)
+
+    if (hazards.length > 0) {
+      // A hazard is authorized by a pragma in the migration that raises it —
+      // never by a flag on the deploy command, which would authorize this
+      // migration and every migration after it. Checked per file, per class.
+      const undeclared: PragmaKind[] = []
+      for (const kind of ['destructive', 'locking', 'dynamic'] as const) {
+        if (!hazards.some(h => h.severity === kind)) continue
+        if (kind === 'destructive') hasDestructive = true
+        if (kind === 'locking') hasLocking = true
+        if (kind === 'dynamic') hasDynamic = true
+        if (!hasPragma(sql, kind)) {
+          undeclared.push(kind)
+          declared = false
+        }
+      }
+      fileHazards.push({ file: `${entry.tag}.sql`, hazards, undeclared })
+    }
+  }
+
+  return {
+    safe: !hasDestructive && !hasLocking && !hasDynamic,
+    hasDestructive,
+    hasLocking,
+    hasDynamic,
+    declared,
+    fileHazards,
+  }
+}
+
+export function formatCheckResult(result: MigrationCheckResult): string {
+  if (result.declared) return ''
+
+  const lines: string[] = []
+  const undeclaredFiles = result.fileHazards.filter(f => f.undeclared.length > 0)
+  const n = undeclaredFiles.length
+
+  lines.push(
+    `Migration safety check: ${n} migration${n > 1 ? 's' : ''} raise${n > 1 ? '' : 's'} ` +
+    `hazards that ${n > 1 ? 'their own files do' : 'its own file does'} not declare.\n`,
+  )
+
+  for (const { file, hazards } of undeclaredFiles) {
+    lines.push(formatHazards(hazards, file))
+  }
+
+  lines.push('')
+  lines.push('Authorization belongs in the migration, not the deploy command.')
+  lines.push('Add the matching line to the top of each file above and commit it,')
+  lines.push('so the approval is reviewed in the diff and recorded in history:')
+  lines.push('')
+
+  for (const { file, undeclared } of undeclaredFiles) {
+    for (const kind of undeclared) {
+      lines.push(`  ${file}:  -- ${PRAGMA_FOR[kind]}`)
+    }
+  }
+
+  return lines.join('\n')
+}
+
+export type { MigrationHazard, Severity }