turbine-orm 0.45.0 → 0.46.0

package/dist/cli/migrate.js

@@ -121,21 +121,37 @@ export function listMigrationFiles(migrationsDir) {
     return files;
 }
 /**
- * Parse migration content string into UP and DOWN sections.
+ * The `-- turbine:no-transaction` directive: when present in a migration's
+ * header (before `-- UP`), the runner applies the file WITHOUT wrapping it in
+ * BEGIN/COMMIT and runs one statement per `client.query()` call. Required for
+ * `CREATE INDEX CONCURRENTLY`, which Postgres forbids inside any transaction,
+ * including the implicit transaction a multi-statement simple query creates.
+ */
+const NO_TRANSACTION_DIRECTIVE = /^--\s*turbine:no-transaction\s*$/i;
+/**
+ * Parse migration content string into UP and DOWN sections plus directives.
  * Exported for unit testing.
  */
 export function parseMigrationContent(content) {
     const lines = content.split('\n');
     let section = 'none';
+    let noTransaction = false;
     const upLines = [];
     const downLines = [];
     for (const line of lines) {
-        const trimmed = line.trim().toUpperCase();
-        if (trimmed === '-- UP') {
+        const trimmed = line.trim();
+        const upper = trimmed.toUpperCase();
+        // The directive is only honored in the header (before -- UP), so it can
+        // never be smuggled in via a DOWN-section comment.
+        if (section === 'none' && NO_TRANSACTION_DIRECTIVE.test(trimmed)) {
+            noTransaction = true;
+            continue;
+        }
+        if (upper === '-- UP') {
             section = 'up';
             continue;
         }
-        if (trimmed === '-- DOWN') {
+        if (upper === '-- DOWN') {
             section = 'down';
             continue;
         }
@@ -147,8 +163,144 @@ export function parseMigrationContent(content) {
     return {
         up: upLines.join('\n').trim(),
         down: downLines.join('\n').trim(),
+        noTransaction,
     };
 }
+/**
+ * Split a SQL script into individual statements on top-level semicolons.
+ *
+ * A correct tokenizer, not a `split(';')`: a semicolon inside a single-quoted
+ * string, a double-quoted identifier, a dollar-quoted body, a line comment
+ * (`--`), or a block comment (`/* *\/`, which Postgres allows to nest) must NOT
+ * split. This is the one production-destroying failure mode of no-transaction
+ * migrations (a partial statement executed against production), so the behavior
+ * is pinned by exhaustive unit tests.
+ *
+ * Comment-only fragments are dropped; every returned statement is trimmed and
+ * carries no trailing semicolon.
+ */
+export function splitSqlStatements(sql) {
+    const statements = [];
+    let current = '';
+    let i = 0;
+    const n = sql.length;
+    while (i < n) {
+        const ch = sql[i];
+        const next = sql[i + 1];
+        // Line comment: consume to end of line (kept verbatim in the statement).
+        if (ch === '-' && next === '-') {
+            let j = i;
+            while (j < n && sql[j] !== '\n')
+                j++;
+            current += sql.slice(i, j);
+            i = j;
+            continue;
+        }
+        // Block comment (Postgres allows nesting: /* /* */ */).
+        if (ch === '/' && next === '*') {
+            let depth = 1;
+            let j = i + 2;
+            current += '/*';
+            while (j < n && depth > 0) {
+                if (sql[j] === '/' && sql[j + 1] === '*') {
+                    depth++;
+                    current += '/*';
+                    j += 2;
+                }
+                else if (sql[j] === '*' && sql[j + 1] === '/') {
+                    depth--;
+                    current += '*/';
+                    j += 2;
+                }
+                else {
+                    current += sql[j];
+                    j++;
+                }
+            }
+            i = j;
+            continue;
+        }
+        // Single-quoted string ('' is an escaped quote, stays inside the string).
+        if (ch === "'") {
+            let j = i + 1;
+            current += "'";
+            while (j < n) {
+                if (sql[j] === "'" && sql[j + 1] === "'") {
+                    current += "''";
+                    j += 2;
+                    continue;
+                }
+                if (sql[j] === "'") {
+                    current += "'";
+                    j++;
+                    break;
+                }
+                current += sql[j];
+                j++;
+            }
+            i = j;
+            continue;
+        }
+        // Double-quoted identifier ("" is an escaped quote).
+        if (ch === '"') {
+            let j = i + 1;
+            current += '"';
+            while (j < n) {
+                if (sql[j] === '"' && sql[j + 1] === '"') {
+                    current += '""';
+                    j += 2;
+                    continue;
+                }
+                if (sql[j] === '"') {
+                    current += '"';
+                    j++;
+                    break;
+                }
+                current += sql[j];
+                j++;
+            }
+            i = j;
+            continue;
+        }
+        // Dollar-quoted body ($tag$ ... $tag$; tag is empty or an identifier, never
+        // digit-leading, so a `$1` parameter placeholder is not mistaken for one).
+        if (ch === '$') {
+            const tagMatch = /^\$([A-Za-z_][A-Za-z_0-9]*)?\$/.exec(sql.slice(i));
+            if (tagMatch) {
+                const tag = tagMatch[0];
+                const end = sql.indexOf(tag, i + tag.length);
+                if (end === -1) {
+                    current += sql.slice(i);
+                    i = n;
+                    continue;
+                }
+                current += sql.slice(i, end + tag.length);
+                i = end + tag.length;
+                continue;
+            }
+        }
+        // Top-level statement terminator.
+        if (ch === ';') {
+            const trimmed = current.trim();
+            if (trimmed)
+                statements.push(trimmed);
+            current = '';
+            i++;
+            continue;
+        }
+        current += ch;
+        i++;
+    }
+    const tail = current.trim();
+    if (tail)
+        statements.push(tail);
+    return statements.filter((s) => !isCommentOnlyStatement(s));
+}
+/** True when a fragment contains nothing but comments and whitespace. */
+function isCommentOnlyStatement(stmt) {
+    const withoutComments = stmt.replace(/\/\*[\s\S]*?\*\//g, ' ').replace(/--[^\n]*/g, ' ');
+    return withoutComments.trim().length === 0;
+}
 /**
  * Parse a migration file into UP and DOWN sections.
  */
@@ -307,6 +459,9 @@ export function buildDiffMigrationBody(diff) {
  * - `autoContent`: pre-populate UP/DOWN from a schema diff.
  * - `options.recipe`: scaffold a named recipe (see {@link MIGRATION_RECIPES}).
  *   Mutually exclusive with `autoContent`; an unknown recipe throws.
+ * - `options.header`: extra header line(s) injected BEFORE `-- UP` (the only
+ *   place a `-- turbine:no-transaction` directive is honored). Used with
+ *   `autoContent`.
  */
 export function createMigration(migrationsDir, name, autoContent, options) {
     mkdirSync(migrationsDir, { recursive: true });
@@ -335,10 +490,11 @@ ${body.down}
 `;
     }
     else if (autoContent) {
-        template = `-- Migration: ${name} (auto-generated from schema diff)
+        const headerBlock = options?.header ? `${options.header}\n` : '';
+        template = `-- Migration: ${name} (auto-generated)
 -- Created: ${now.toISOString()}
 -- Review this file before running: npx turbine migrate up
-
+${headerBlock}
 -- UP
 ${autoContent.up}
 
@@ -621,25 +777,54 @@ export async function migrateUp(connectionString, migrationsDir, options) {
             const results = [];
             const errors = [];
             const outOfOrder = [];
+            const noTransactionApplied = [];
+            const flagOutOfOrder = (file) => {
+                // Flag an out-of-order apply: this file's timestamp is older than a
+                // migration that was already applied before this run started.
+                if (newestPrior && file.timestamp && file.timestamp < newestPrior.ts) {
+                    outOfOrder.push({ applied: file.filename, newestPrior: `${newestPrior.name}.sql` });
+                }
+            };
             for (const file of pending) {
-                const { up } = parseMigrationSQL(file.path);
+                const parsed = parseMigrationSQL(file.path);
+                const up = parsed.up;
                 if (!up) {
                     errors.push({ file, error: 'No UP section found in migration file' });
                     continue;
                 }
                 const content = readFileSync(file.path, 'utf-8');
                 const hash = checksum(content);
+                const insertApplied = dialect.buildMigrationInsertApplied(quotedTrackingTable(dialect));
+                if (parsed.noTransaction) {
+                    // No BEGIN/COMMIT: run ONE statement per query() call (a multi-statement
+                    // simple query would be wrapped in an implicit transaction, breaking
+                    // CREATE INDEX CONCURRENTLY). Recording happens only after all statements
+                    // succeed: a mid-file failure leaves earlier (idempotent) statements
+                    // applied and the migration unrecorded, so a rerun resumes it.
+                    options?.onNoTransaction?.(file);
+                    noTransactionApplied.push(file);
+                    try {
+                        for (const stmt of splitSqlStatements(up)) {
+                            await client.query(stmt);
+                        }
+                        await client.query(insertApplied, [file.name, hash]);
+                        results.push(file);
+                        flagOutOfOrder(file);
+                    }
+                    catch (err) {
+                        const msg = err instanceof Error ? err.message : String(err);
+                        errors.push({ file, error: msg });
+                        break;
+                    }
+                    continue;
+                }
                 try {
                     await client.query('BEGIN');
                     await client.query(up);
-                    await client.query(dialect.buildMigrationInsertApplied(quotedTrackingTable(dialect)), [file.name, hash]);
+                    await client.query(insertApplied, [file.name, hash]);
                     await client.query('COMMIT');
                     results.push(file);
-                    // Flag an out-of-order apply: this file's timestamp is older than a
-                    // migration that was already applied before this run started.
-                    if (newestPrior && file.timestamp && file.timestamp < newestPrior.ts) {
-                        outOfOrder.push({ applied: file.filename, newestPrior: `${newestPrior.name}.sql` });
-                    }
+                    flagOutOfOrder(file);
                 }
                 catch (err) {
                     await client.query('ROLLBACK');
@@ -649,7 +834,7 @@ export async function migrateUp(connectionString, migrationsDir, options) {
                     break;
                 }
             }
-            return { applied: results, errors, destructive, outOfOrder };
+            return { applied: results, errors, destructive, outOfOrder, noTransaction: noTransactionApplied };
         }
         finally {
             await releaseLock(client, lockId, adapter);
@@ -746,15 +931,34 @@ export async function migrateDown(connectionString, migrationsDir, options) {
                     });
                     continue;
                 }
-                const { down } = parseMigrationSQL(file.path);
+                const parsed = parseMigrationSQL(file.path);
+                const down = parsed.down;
                 if (!down) {
                     errors.push({ file, error: 'No DOWN section found in migration file' });
                     continue;
                 }
+                const deleteApplied = dialect.buildMigrationDeleteApplied(quotedTrackingTable(dialect));
+                if (parsed.noTransaction) {
+                    // Untransacted rollback (DROP INDEX CONCURRENTLY IF EXISTS), one
+                    // statement per query() call: same contract as the untransacted UP.
+                    try {
+                        for (const stmt of splitSqlStatements(down)) {
+                            await client.query(stmt);
+                        }
+                        await client.query(deleteApplied, [migration.name]);
+                        results.push(file);
+                    }
+                    catch (err) {
+                        const msg = err instanceof Error ? err.message : String(err);
+                        errors.push({ file, error: msg });
+                        break;
+                    }
+                    continue;
+                }
                 try {
                     await client.query('BEGIN');
                     await client.query(down);
-                    await client.query(dialect.buildMigrationDeleteApplied(quotedTrackingTable(dialect)), [migration.name]);
+                    await client.query(deleteApplied, [migration.name]);
                     await client.query('COMMIT');
                     results.push(file);
                 }

package/dist/index-advisor.d.ts

@@ -38,6 +38,36 @@ export interface MissingRelationIndex {
     /** DROP INDEX statement for the fix migration's DOWN section */
     dropSql: string;
 }
+/** Options for {@link buildCreateIndexSql}. All default to the plain, in-transaction form. */
+export interface CreateIndexSqlOptions {
+    /**
+     * Emit `CREATE INDEX CONCURRENTLY`. A concurrent build never holds a write
+     * lock, but it CANNOT run inside a transaction block, so a migration carrying
+     * it must also carry the `-- turbine:no-transaction` directive.
+     */
+    concurrently?: boolean;
+    /** Emit `IF NOT EXISTS` (default true — required for idempotent no-transaction migrations). */
+    ifNotExists?: boolean;
+    /**
+     * Emit a partial index `... WHERE <col> IS NOT NULL`. Only applied for a
+     * single-column index (a relation probe correlates `child.fk = parent.pk`, and
+     * NULL never equals anything, so a mostly-NULL FK is fully served by a partial
+     * index at a fraction of the size). Ignored for composite indexes.
+     */
+    partialNotNull?: boolean;
+}
+/**
+ * Build a `CREATE INDEX` statement for a relation-probe fix. Pure string
+ * assembly (no stats, no DB read): the caller decides whether to pass
+ * `concurrently`/`partialNotNull` based on collected statistics, keeping this
+ * module topology-only.
+ */
+export declare function buildCreateIndexSql(table: string, columns: string[], indexName: string, options?: CreateIndexSqlOptions): string;
+/** Build the matching `DROP INDEX` statement. `concurrently` requires no-transaction execution. */
+export declare function buildDropIndexSql(indexName: string, options?: {
+    concurrently?: boolean;
+    ifExists?: boolean;
+}): string;
 /**
  * Whether an equality probe on `columns` is served by the table's indexes.
  *

package/dist/index-stats.d.ts

@@ -0,0 +1,208 @@
+/**
+ * Index statistics - cost-aware triage for the missing-index advisor.
+ *
+ * `index-advisor.ts` decides WHICH relation probes lack an index from pure
+ * schema topology. This module decides whether adding each one is worth it,
+ * reading live Postgres statistics (table size, write volume, existing indexes,
+ * HOT-update ratio, column null fraction) and scoring every finding into a tier:
+ *
+ *   - take-freely       large table, low write rate, few existing indexes;
+ *   - take-deliberately real write rate, many existing indexes, or HOT at risk;
+ *   - scrutinize        tiny table, append-only log shape, or stats too young.
+ *
+ * DESIGN: the file splits into two halves.
+ *   - The COLLECTOR half (`collectStatsSnapshot`) reads pg catalogs. It uses a
+ *     single one-connection pool, sets a statement_timeout, and treats every
+ *     catalog read as INDIVIDUALLY OPTIONAL: a read that fails (privileges,
+ *     CockroachDB/YugabyteDB catalog gaps, missing view) degrades that one
+ *     signal and records a notice rather than aborting the whole snapshot.
+ *   - The PURE half (`scoreMissingIndex`, `findInvalidIndexes`, and the exported
+ *     threshold constants) takes a typed {@link StatsSnapshot} and imports NO pg.
+ *     It is fully unit-testable with hand-built snapshots and is what carries the
+ *     thresholds the CLI prints alongside every verdict.
+ *
+ * All statistics features are Postgres-only. On a non-Postgres engine the
+ * collector returns an unavailable snapshot and the caller falls back to the
+ * topology-only report.
+ */
+/**
+ * Heuristic thresholds behind the tier decision. Exported and surfaced in the
+ * doctor output so the reasoning is never a black box: if a threshold is wrong
+ * for a workload, the user can see the exact number that drove the verdict.
+ */
+export declare const STATS_THRESHOLDS: {
+    /** reltuples below this = tiny table; a missing index there is noise → scrutinize. */
+    readonly tinyTableRows: 1000;
+    /**
+     * Writes/day (n_tup_ins+upd+del normalized by stats age) at or above this is a
+     * "real" write rate: the per-index write tax is worth a second look → take deliberately.
+     */
+    readonly highWritesPerDay: 50000;
+    /**
+     * Existing index count at or above this: every extra index compounds the write
+     * and storage tax, so a new one is no longer free → take deliberately.
+     */
+    readonly manyIndexes: 4;
+    /**
+     * HOT ratio (n_tup_hot_upd / n_tup_upd) at or above this, WITH real update
+     * volume, means the table currently relies on heap-only-tuple updates. A new
+     * index can disqualify HOT and amplify write cost → warning + take deliberately.
+     */
+    readonly hotRatioAtRisk: 0.3;
+    /** Minimum n_tup_upd for the HOT signal to be statistically meaningful. */
+    readonly hotMinUpdates: 1000;
+    /**
+     * null_frac at or above this on a probed FK column: the column is mostly NULL,
+     * so a partial `WHERE col IS NOT NULL` index covers every relation probe at a
+     * fraction of the size.
+     */
+    readonly partialNullFrac: 0.9;
+    /**
+     * Stats younger than this (days since stats_reset) cannot normalize a write
+     * rate; the whole report degrades to topology-only output.
+     */
+    readonly minStatsAgeDays: 1;
+    /** Append-heavy log shape: n_tup_ins at or above this ... */
+    readonly appendHeavyMinInserts: 100000;
+    /** ... with inserts making up at or above this fraction of all writes ... */
+    readonly appendHeavyInsertRatio: 0.95;
+    /** ... and seq_scan at or below this ("near-zero probe reads") → scrutinize. */
+    readonly appendHeavyMaxSeqScan: 5;
+};
+/** Per-table live statistics. Any field may be absent when its catalog read degraded. */
+export interface TableStats {
+    table: string;
+    /** pg_class.reltuples. 0 or -1 means never-analyzed → treated as UNKNOWN (null rows). */
+    reltuples: number;
+    /** pg_stat_user_tables.n_live_tup, a cross-check for reltuples. */
+    nLiveTup?: number;
+    nTupIns?: number;
+    nTupUpd?: number;
+    nTupDel?: number;
+    nTupHotUpd?: number;
+    seqScan?: number;
+    seqTupRead?: number;
+    /** pg_total_relation_size (table + indexes + toast), bytes. */
+    totalSizeBytes?: number;
+    /** pg_relation_size (heap only), bytes. */
+    tableSizeBytes?: number;
+    /** Count of indexes already on the table (pg_index). */
+    existingIndexCount?: number;
+}
+/** A single index's identity + validity, from pg_index / pg_stat_user_indexes. */
+export interface IndexStat {
+    table: string;
+    indexName: string;
+    columns: string[];
+    idxScan?: number;
+    isValid: boolean;
+    isUnique: boolean;
+    isPrimary: boolean;
+    isReplicaIdent: boolean;
+}
+/**
+ * A point-in-time read of the statistics the triage needs. Every part is
+ * optional at the field level so the pure scorer degrades honestly.
+ */
+export interface StatsSnapshot {
+    /** True when at least the core table statistics were readable. */
+    available: boolean;
+    /** pg_stat_database.stats_reset for the current DB. NULL when never reset. */
+    statsReset: Date | null;
+    /** Days since stats_reset, or null when stats_reset is NULL / in the future. */
+    statsAgeDays: number | null;
+    /** Per-table stats, keyed by table name. */
+    tables: Record<string, TableStats>;
+    /** All indexes read (used for invalid-index detection). */
+    indexes: IndexStat[];
+    /** null_frac per probed column, keyed `table.column`. */
+    nullFrac: Record<string, number>;
+    /** Per-signal degradation notices (privileges, catalog gaps, timeouts). */
+    notices: string[];
+}
+/** Build an empty (fully unavailable) snapshot - the honest "no stats" baseline. */
+export declare function emptyStatsSnapshot(notices?: string[]): StatsSnapshot;
+export type IndexTier = 'take-freely' | 'take-deliberately' | 'scrutinize';
+/** The numbers behind a verdict, all nullable so unknowns never masquerade as zero. */
+export interface FindingMetrics {
+    /** reltuples, or null when never-analyzed (0/-1). */
+    rows: number | null;
+    /** pg_total_relation_size in bytes. */
+    sizeBytes: number | null;
+    /** Writes/day since stats reset, or null when the rate cannot be normalized. */
+    writesPerDay: number | null;
+    existingIndexCount: number | null;
+    probingRelations: number;
+    /** n_tup_hot_upd / n_tup_upd, or null. */
+    hotRatio: number | null;
+    /** Highest null_frac across the probed columns, or null when unknown. */
+    nullFrac: number | null;
+    statsAgeDays: number | null;
+}
+export interface ScoredMissingIndex {
+    table: string;
+    columns: string[];
+    tier: IndexTier;
+    /** Human-readable reasons, each carrying the number that drove it. */
+    reasons: string[];
+    metrics: FindingMetrics;
+    /** The HOT-update caveat when the table is at risk of losing HOT, else null. */
+    hotWarning: string | null;
+    /** When true, the emitted index should be a partial `WHERE col IS NOT NULL`. */
+    partialNotNull: boolean;
+    /** Sort key within a tier: bigger, more-probed tables first. */
+    benefitScore: number;
+}
+/** The subset of a topology finding the scorer needs. */
+export interface ScorableMissingIndex {
+    table: string;
+    columns: string[];
+    probes: unknown[];
+}
+/** Format a byte count as a short human string (KB/MB/GB). */
+export declare function formatBytes(bytes: number | null | undefined): string;
+/**
+ * Score a single missing-index finding against a snapshot. Pure and total:
+ * every unknown degrades to a caveat rather than a fabricated number.
+ */
+export declare function scoreMissingIndex(missing: ScorableMissingIndex, snapshot: StatsSnapshot): ScoredMissingIndex;
+export interface InvalidIndex {
+    table: string;
+    indexName: string;
+    columns: string[];
+    /** `DROP INDEX CONCURRENTLY IF EXISTS` - the repair for an INVALID corpse. */
+    dropSql: string;
+}
+/**
+ * Indexes left INVALID (indisvalid = false) - the failure artifact of a
+ * `CREATE INDEX CONCURRENTLY` that errored. `IF NOT EXISTS` on a retry silently
+ * skips the corpse, so the fix is DROP INDEX CONCURRENTLY then rerun.
+ */
+export declare function findInvalidIndexes(snapshot: StatsSnapshot): InvalidIndex[];
+/**
+ * Whether the snapshot is trustworthy enough to render tier verdicts. Empty,
+ * unavailable, or too-young stats degrade to the topology-only report.
+ */
+export declare function isSnapshotUsable(snapshot: StatsSnapshot): boolean;
+/** A tuple identifying a probed column whose null_frac the collector should read. */
+export interface ProbedColumn {
+    table: string;
+    column: string;
+}
+export interface CollectSnapshotOptions {
+    connectionString: string;
+    schema: string;
+    /** Tables to read table-level stats + sizes for (the probed tables). */
+    tables: string[];
+    /** Columns to read null_frac for (single-column probes). */
+    columns: ProbedColumn[];
+    /** statement_timeout for each catalog read. Default 5000ms. */
+    statementTimeoutMs?: number;
+}
+/**
+ * Read a live statistics snapshot from Postgres. Each catalog read is wrapped
+ * individually: a failure records a notice and leaves that signal absent, so a
+ * privilege gap or a CockroachDB/YugabyteDB catalog difference degrades one
+ * signal rather than the whole snapshot.
+ */
+export declare function collectStatsSnapshot(options: CollectSnapshotOptions): Promise<StatsSnapshot>;