turbine-orm 0.45.0 → 0.46.0

package/dist/cjs/cli/index.js

@@ -15,7 +15,7 @@
  *   turbine migrate status        — Show migration status
  *   turbine seed                  — Run seed file
  *   turbine status                — Show schema summary
- *   turbine doctor                — Check relations for missing FK indexes (--fix emits migration)
+ *   turbine doctor                - Cost-aware missing-FK-index triage (--fix, --json, --no-concurrently)
  *   turbine studio                : Launch local read-only web UI (--demo for a seeded sample DB)
  *   turbine mcp                   — Start read-only MCP server over JSON-RPC stdio
  *   turbine observe               — Launch metrics dashboard (requires TURBINE_OBSERVE_URL)
@@ -73,6 +73,7 @@ const node_path_1 = require("node:path");
 const node_url_1 = require("node:url");
 const generate_js_1 = require("../generate.js");
 const index_advisor_js_1 = require("../index-advisor.js");
+const index_stats_js_1 = require("../index-stats.js");
 const introspect_js_1 = require("../introspect.js");
 const schema_sql_js_1 = require("../schema-sql.js");
 const config_js_1 = require("./config.js");
@@ -164,6 +165,12 @@ function parseArgs(argv = process.argv.slice(2)) {
             case '--fix':
                 result.fix = true;
                 break;
+            case '--json':
+                result.json = true;
+                break;
+            case '--no-concurrently':
+                result.noConcurrently = true;
+                break;
             case '--zod':
                 result.zod = true;
                 break;
@@ -1571,12 +1578,24 @@ async function cmdMigrateUp(args, config) {
         (0, ui_js_1.newline)();
     }
     const spinner = new ui_js_1.Spinner('Applying migrations').start();
+    // A no-transaction migration (CREATE INDEX CONCURRENTLY) can wait a long time
+    // on other transactions and otherwise looks hung. Stop the spinner and print
+    // a loud notice the moment one is about to run.
+    const onNoTransaction = (file) => {
+        spinner.stop();
+        (0, ui_js_1.warn)(`Running ${(0, ui_js_1.bold)(file.filename)} WITHOUT a transaction (-- turbine:no-transaction).`);
+        console.log(`  ${(0, ui_js_1.dim)('Each statement runs on its own. A mid-file failure leaves earlier statements')}`);
+        console.log(`  ${(0, ui_js_1.dim)('applied and the migration UNRECORDED, so every statement must be idempotent.')}`);
+        console.log(`  ${(0, ui_js_1.dim)('CREATE INDEX CONCURRENTLY can wait on long-running transactions: not a hang.')}`);
+        (0, ui_js_1.newline)();
+    };
     let result;
     try {
         result = await (0, migrate_js_1.migrateUp)(url, config.migrationsDir, {
             step: args.step,
             allowDrift: args.allowDrift,
             allowDestructive: args.allowDestructive,
+            onNoTransaction,
         });
     }
     catch (err) {
@@ -1593,6 +1612,7 @@ async function cmdMigrateUp(args, config) {
             step: args.step,
             allowDrift: args.allowDrift,
             allowDestructive: true,
+            onNoTransaction,
         });
     }
     if (result.applied.length === 0 && result.errors.length === 0) {
@@ -2038,16 +2058,50 @@ async function cmdStatus(_args, config) {
         (0, ui_js_1.newline)();
     }
 }
-// ---------------------------------------------------------------------------
-// Command: doctor — relation/index health check
-// ---------------------------------------------------------------------------
+/** Human labels + tier ordering for the three triage buckets. */
+const TIER_ORDER = ['take-freely', 'take-deliberately', 'scrutinize'];
+const TIER_LABEL = {
+    'take-freely': 'TAKE FREELY',
+    'take-deliberately': 'TAKE DELIBERATELY',
+    scrutinize: 'SCRUTINIZE',
+};
+/** Build the `CREATE INDEX` for a finding, honoring the partial-null suggestion. */
+function doctorCreateSql(f, opts) {
+    return (0, index_advisor_js_1.buildCreateIndexSql)(f.missing.table, f.missing.columns, f.missing.indexName, {
+        concurrently: opts.concurrently,
+        partialNotNull: f.score.partialNotNull,
+    });
+}
+/** The commented recipe prepended to a CONCURRENTLY fix migration's UP body. */
+const CONCURRENTLY_RECIPE_COMMENT = [
+    '-- CREATE INDEX CONCURRENTLY builds without holding a write lock, but it cannot',
+    '-- run inside a transaction (that is why this file carries the',
+    '-- "-- turbine:no-transaction" directive above). Read before applying:',
+    '--',
+    '--   1. Idempotency is required. This migration is recorded only after ALL',
+    '--      statements succeed; a mid-file failure leaves earlier indexes built and',
+    '--      the migration unrecorded, so a rerun must be safe. IF NOT EXISTS keeps',
+    '--      each CREATE idempotent.',
+    '--   2. The INVALID-index trap. A CREATE INDEX CONCURRENTLY that fails partway',
+    '--      leaves an INVALID index behind. On rerun, IF NOT EXISTS SKIPS that',
+    '--      corpse (the name already exists), so the index is never actually built.',
+    '--      Fix: DROP INDEX CONCURRENTLY the invalid index, then rerun. Run',
+    '--      "turbine doctor" to list invalid indexes.',
+    '--   3. Locking. CREATE INDEX CONCURRENTLY waits for every transaction that can',
+    '--      see the table to finish. A long-running transaction makes it wait and',
+    '--      makes "migrate up" look hung. For bounded waits, SET lock_timeout /',
+    '--      statement_timeout in a psql session.',
+].join('\n');
 async function cmdDoctor(args, config) {
-    (0, ui_js_1.banner)();
+    const jsonMode = args.json === true;
     const url = requireUrl(config);
-    (0, ui_js_1.label)('Database', (0, ui_js_1.redactUrl)(url));
-    (0, ui_js_1.label)('Schema', config.schema);
-    (0, ui_js_1.newline)();
-    const spinner = new ui_js_1.Spinner('Introspecting database').start();
+    if (!jsonMode) {
+        (0, ui_js_1.banner)();
+        (0, ui_js_1.label)('Database', (0, ui_js_1.redactUrl)(url));
+        (0, ui_js_1.label)('Schema', config.schema);
+        (0, ui_js_1.newline)();
+    }
+    const spinner = jsonMode ? null : new ui_js_1.Spinner('Introspecting database').start();
     const schema = await (0, introspect_js_1.introspect)({
         connectionString: url,
         schema: config.schema,
@@ -2055,62 +2109,201 @@ async function cmdDoctor(args, config) {
         exclude: config.exclude.length ? config.exclude : undefined,
     });
     const missing = (0, index_advisor_js_1.findMissingRelationIndexes)(schema);
-    if (missing.length === 0) {
-        spinner.succeed('Every relation probe is backed by an index');
-        (0, ui_js_1.newline)();
+    // Collect live statistics. The collector reads whole-schema indexes (for
+    // invalid-index detection) plus per-table stats + probed-column null_frac.
+    const probedTables = [...new Set(missing.map((m) => m.table))];
+    const probedColumns = [];
+    for (const m of missing) {
+        if (m.columns.length === 1 && m.columns[0] !== undefined) {
+            probedColumns.push({ table: m.table, column: m.columns[0] });
+        }
+    }
+    let snapshot;
+    try {
+        snapshot = await (0, index_stats_js_1.collectStatsSnapshot)({
+            connectionString: url,
+            schema: config.schema,
+            tables: probedTables,
+            columns: probedColumns,
+        });
+    }
+    catch (err) {
+        snapshot = {
+            available: false,
+            statsReset: null,
+            statsAgeDays: null,
+            tables: {},
+            indexes: [],
+            nullFrac: {},
+            notices: [`statistics collection failed: ${err instanceof Error ? err.message : String(err)}`],
+        };
+    }
+    const invalid = (0, index_stats_js_1.findInvalidIndexes)(snapshot);
+    const usable = (0, index_stats_js_1.isSnapshotUsable)(snapshot);
+    const findings = missing.map((m) => ({ missing: m, score: (0, index_stats_js_1.scoreMissingIndex)(m, snapshot) }));
+    if (jsonMode) {
+        spinner?.stop();
+        console.log(JSON.stringify(buildDoctorJson({ schema, findings, invalid, snapshot, usable, args }), null, 2));
         return;
     }
+    await renderDoctorHuman({ spinner: spinner, schema, findings, invalid, snapshot, usable, args, config });
+}
+/** The stable, versioned JSON contract (schemaVersion: 1). First external consumer: BataDB import. */
+function buildDoctorJson(ctx) {
+    const concurrently = ctx.args.noConcurrently !== true;
+    return {
+        schemaVersion: 1,
+        scannedTables: Object.keys(ctx.schema.tables).length,
+        stats: {
+            available: ctx.snapshot.available,
+            usable: ctx.usable,
+            statsReset: ctx.snapshot.statsReset ? ctx.snapshot.statsReset.toISOString() : null,
+            statsAgeDays: ctx.snapshot.statsAgeDays,
+            notices: ctx.snapshot.notices,
+        },
+        thresholds: index_stats_js_1.STATS_THRESHOLDS,
+        findings: ctx.findings.map((f) => ({
+            table: f.missing.table,
+            columns: f.missing.columns,
+            indexName: f.missing.indexName,
+            tier: f.score.tier,
+            reasons: f.score.reasons,
+            metrics: f.score.metrics,
+            hotWarning: f.score.hotWarning,
+            partialNotNull: f.score.partialNotNull,
+            probes: f.missing.probes,
+            createSql: doctorCreateSql(f, { concurrently }),
+            dropSql: (0, index_advisor_js_1.buildDropIndexSql)(f.missing.indexName, { concurrently }),
+        })),
+        invalidIndexes: ctx.invalid,
+    };
+}
+async function renderDoctorHuman(ctx) {
+    const { spinner, schema, findings, invalid, snapshot, usable, args, config } = ctx;
     spinner.succeed(`Scanned ${(0, ui_js_1.bold)(String(Object.keys(schema.tables).length))} tables`);
-    (0, ui_js_1.warn)(`Found ${(0, ui_js_1.bold)(String(missing.length))} unindexed relation probe(s)`);
-    (0, ui_js_1.newline)();
-    // Row counts put the findings in severity order: a missing index on a 300-row
-    // table is noise; on a 300K-row table it is the whole page load.
-    const rowCounts = new Map();
-    {
-        const { Pool } = (await Promise.resolve().then(() => __importStar(require('pg')))).default;
-        const pool = new Pool({ connectionString: url, max: 1 });
-        try {
-            const tables = [...new Set(missing.map((m) => m.table))];
-            const res = await pool.query(`SELECT c.relname, c.reltuples::bigint::text AS reltuples
-           FROM pg_class c
-           JOIN pg_namespace n ON n.oid = c.relnamespace
-          WHERE n.nspname = $1 AND c.relname = ANY($2)`, [config.schema, tables]);
-            for (const row of res.rows)
-                rowCounts.set(row.relname, Math.max(0, Number(row.reltuples)));
+    if (findings.length === 0 && invalid.length === 0) {
+        (0, ui_js_1.success)('Every relation probe is backed by an index, and no invalid indexes were found');
+        (0, ui_js_1.newline)();
+        return;
+    }
+    if (findings.length > 0) {
+        (0, ui_js_1.warn)(`Found ${(0, ui_js_1.bold)(String(findings.length))} unindexed relation probe(s)`);
+        (0, ui_js_1.newline)();
+        console.log(`  ${(0, ui_js_1.dim)('Turbine loads relations as correlated subqueries: the child table is probed')}`);
+        console.log(`  ${(0, ui_js_1.dim)('once per parent row, so an unindexed FK costs a full table scan PER PARENT.')}`);
+        (0, ui_js_1.newline)();
+        if (usable) {
+            renderTiers(findings, snapshot, args);
         }
-        finally {
-            await pool.end();
+        else {
+            renderTopologyFallback(findings, snapshot);
         }
     }
-    missing.sort((a, b) => (rowCounts.get(b.table) ?? 0) - (rowCounts.get(a.table) ?? 0));
-    console.log(`  ${(0, ui_js_1.dim)('Turbine loads relations as correlated subqueries — the child table is probed')}`);
-    console.log(`  ${(0, ui_js_1.dim)('once per parent row, so an unindexed FK costs a full table scan PER PARENT.')}`);
-    (0, ui_js_1.newline)();
-    for (const m of missing) {
-        const rows = rowCounts.get(m.table);
-        const rowsLabel = rows !== undefined ? `~${rows.toLocaleString()} rows` : 'row count unknown';
-        console.log(`  ${(0, ui_js_1.yellow)(ui_js_1.symbols.warning)} ${(0, ui_js_1.bold)((0, ui_js_1.cyan)(m.table))} ${(0, ui_js_1.dim)(`(${m.columns.join(', ')})`)}  ${(0, ui_js_1.gray)(rowsLabel)}`);
-        for (const p of m.probes) {
-            console.log(`    ${(0, ui_js_1.dim)(ui_js_1.symbols.tee)} probed by ${p.from}.${(0, ui_js_1.blue)(p.relation)} ${(0, ui_js_1.dim)(`(${p.type})`)}`);
+    renderInvalidIndexes(invalid);
+    if (findings.length > 0) {
+        if (args.fix) {
+            renderFixMigration(findings, config, args);
         }
-        console.log(`    ${(0, ui_js_1.dim)(ui_js_1.symbols.teeEnd)} ${(0, ui_js_1.green)(m.createSql)}`);
+        else {
+            console.log(`  ${(0, ui_js_1.dim)('Generate a fix migration with:')} ${(0, ui_js_1.cyan)('npx turbine doctor --fix')}`);
+            console.log(`  ${(0, ui_js_1.dim)('Machine-readable report:')} ${(0, ui_js_1.cyan)('npx turbine doctor --json')}`);
+            (0, ui_js_1.newline)();
+        }
+    }
+}
+/** Cost-aware tiered output: three sections, each finding annotated with its numbers. */
+function renderTiers(findings, snapshot, _args) {
+    const ageLabel = snapshot.statsAgeDays !== null ? `${Math.round(snapshot.statsAgeDays)}d` : 'unknown';
+    console.log(`  ${(0, ui_js_1.dim)(`Cost triage based on live statistics (stats reset ${ageLabel} ago). Thresholds: tiny < ${index_stats_js_1.STATS_THRESHOLDS.tinyTableRows.toLocaleString()} rows,`)}`);
+    console.log(`  ${(0, ui_js_1.dim)(`"real" write rate >= ${index_stats_js_1.STATS_THRESHOLDS.highWritesPerDay.toLocaleString()}/day, "many" indexes >= ${index_stats_js_1.STATS_THRESHOLDS.manyIndexes}.`)}`);
+    (0, ui_js_1.newline)();
+    const tierColor = {
+        'take-freely': ui_js_1.green,
+        'take-deliberately': ui_js_1.yellow,
+        scrutinize: ui_js_1.red,
+    };
+    for (const tier of TIER_ORDER) {
+        const inTier = findings
+            .filter((f) => f.score.tier === tier)
+            .sort((a, b) => b.score.benefitScore - a.score.benefitScore);
+        if (inTier.length === 0)
+            continue;
+        console.log(`  ${(0, ui_js_1.bold)(tierColor[tier](`${TIER_LABEL[tier]} (${inTier.length})`))}`);
         (0, ui_js_1.newline)();
+        for (const f of inTier) {
+            renderFinding(f, { concurrently: true, withReasons: true });
+        }
     }
-    if (args.fix) {
-        const up = missing.map((m) => m.createSql).join('\n');
-        const down = missing.map((m) => m.dropSql).join('\n');
-        const file = (0, migrate_js_1.createMigration)(config.migrationsDir, 'add_relation_fk_indexes', { up, down });
-        (0, ui_js_1.success)(`Created migration: ${(0, ui_js_1.bold)(file.filename)}`);
+}
+/** Degraded output: today's size-sorted topology report when stats are absent/young. */
+function renderTopologyFallback(findings, snapshot) {
+    (0, ui_js_1.warn)('Statistics unavailable or too young to score cost: showing size-sorted topology only.');
+    for (const notice of snapshot.notices)
+        console.log(`  ${(0, ui_js_1.dim)(`- ${notice}`)}`);
+    if (snapshot.statsAgeDays !== null && snapshot.statsAgeDays < index_stats_js_1.STATS_THRESHOLDS.minStatsAgeDays) {
+        console.log(`  ${(0, ui_js_1.dim)(`- statistics were reset less than ${index_stats_js_1.STATS_THRESHOLDS.minStatsAgeDays} day(s) ago; write rates are not yet meaningful.`)}`);
+    }
+    (0, ui_js_1.newline)();
+    const sorted = [...findings].sort((a, b) => (b.score.metrics.rows ?? 0) - (a.score.metrics.rows ?? 0));
+    for (const f of sorted) {
+        renderFinding(f, { concurrently: false, withReasons: false });
+    }
+}
+/** Render one finding: table + columns, probing relations, reasons, and the create SQL. */
+function renderFinding(f, opts) {
+    const m = f.missing;
+    const rows = f.score.metrics.rows;
+    const rowsLabel = rows !== null ? `~${rows.toLocaleString()} rows` : 'row count unknown';
+    const sizeLabel = f.score.metrics.sizeBytes !== null ? `, ${(0, index_stats_js_1.formatBytes)(f.score.metrics.sizeBytes)}` : '';
+    console.log(`  ${(0, ui_js_1.yellow)(ui_js_1.symbols.warning)} ${(0, ui_js_1.bold)((0, ui_js_1.cyan)(m.table))} ${(0, ui_js_1.dim)(`(${m.columns.join(', ')})`)}  ${(0, ui_js_1.gray)(`${rowsLabel}${sizeLabel}`)}`);
+    for (const p of m.probes) {
+        console.log(`    ${(0, ui_js_1.dim)(ui_js_1.symbols.tee)} probed by ${p.from}.${(0, ui_js_1.blue)(p.relation)} ${(0, ui_js_1.dim)(`(${p.type})`)}`);
+    }
+    if (opts.withReasons) {
+        for (const reason of f.score.reasons) {
+            console.log(`    ${(0, ui_js_1.dim)(ui_js_1.symbols.tee)} ${(0, ui_js_1.dim)(reason)}`);
+        }
+    }
+    console.log(`    ${(0, ui_js_1.dim)(ui_js_1.symbols.teeEnd)} ${(0, ui_js_1.green)(doctorCreateSql(f, { concurrently: opts.concurrently }))}`);
+    (0, ui_js_1.newline)();
+}
+/** The invalid-index report section (a failed CONCURRENTLY build leaves these behind). */
+function renderInvalidIndexes(invalid) {
+    if (invalid.length === 0)
+        return;
+    (0, ui_js_1.warn)(`Found ${(0, ui_js_1.bold)(String(invalid.length))} INVALID index(es) (a failed CREATE INDEX CONCURRENTLY leaves these).`);
+    console.log(`  ${(0, ui_js_1.dim)('IF NOT EXISTS skips an invalid index on rerun, so it never rebuilds. Drop it, then rerun:')}`);
+    (0, ui_js_1.newline)();
+    for (const idx of invalid) {
+        console.log(`  ${(0, ui_js_1.red)(ui_js_1.symbols.warning)} ${(0, ui_js_1.bold)((0, ui_js_1.cyan)(idx.table))} ${(0, ui_js_1.dim)(`(${idx.columns.join(', ') || '?'})`)}  ${(0, ui_js_1.gray)(idx.indexName)}`);
+        console.log(`    ${(0, ui_js_1.dim)(ui_js_1.symbols.teeEnd)} ${(0, ui_js_1.green)(idx.dropSql)}`);
         (0, ui_js_1.newline)();
+    }
+}
+/** Write the --fix migration (CONCURRENTLY + directive by default; plain with --no-concurrently). */
+function renderFixMigration(findings, config, args) {
+    const concurrently = args.noConcurrently !== true;
+    const up = findings.map((f) => doctorCreateSql(f, { concurrently })).join('\n');
+    const down = findings.map((f) => (0, index_advisor_js_1.buildDropIndexSql)(f.missing.indexName, { concurrently })).join('\n');
+    let file;
+    if (concurrently) {
+        file = (0, migrate_js_1.createMigration)(config.migrationsDir, 'add_relation_fk_indexes', { up: `${CONCURRENTLY_RECIPE_COMMENT}\n\n${up}`, down }, { header: '-- turbine:no-transaction' });
+    }
+    else {
+        file = (0, migrate_js_1.createMigration)(config.migrationsDir, 'add_relation_fk_indexes', { up, down });
+    }
+    (0, ui_js_1.success)(`Created migration: ${(0, ui_js_1.bold)(file.filename)}`);
+    (0, ui_js_1.newline)();
+    if (concurrently) {
+        console.log(`  ${(0, ui_js_1.dim)('This is a')} ${(0, ui_js_1.cyan)('no-transaction')} ${(0, ui_js_1.dim)('migration (CREATE INDEX CONCURRENTLY).')}`);
         console.log(`  ${(0, ui_js_1.dim)('Review it, then apply with:')} ${(0, ui_js_1.cyan)('npx turbine migrate up')}`);
-        console.log(`  ${(0, ui_js_1.dim)('Large, hot tables: consider running the statements manually with')} ${(0, ui_js_1.cyan)('CREATE INDEX CONCURRENTLY')}`);
-        console.log(`  ${(0, ui_js_1.dim)('(cannot run inside a transaction, so it is not emitted in the migration).')}`);
-        (0, ui_js_1.newline)();
+        console.log(`  ${(0, ui_js_1.dim)('For a plain in-transaction migration instead, use')} ${(0, ui_js_1.cyan)('doctor --fix --no-concurrently')}`);
     }
     else {
-        console.log(`  ${(0, ui_js_1.dim)('Generate a fix migration with:')} ${(0, ui_js_1.cyan)('npx turbine doctor --fix')}`);
-        (0, ui_js_1.newline)();
+        console.log(`  ${(0, ui_js_1.dim)('Review it, then apply with:')} ${(0, ui_js_1.cyan)('npx turbine migrate up')}`);
+        console.log(`  ${(0, ui_js_1.dim)('For large, hot tables prefer the CONCURRENTLY form (drop')} ${(0, ui_js_1.cyan)('--no-concurrently')}${(0, ui_js_1.dim)(').')}`);
     }
+    (0, ui_js_1.newline)();
 }
 // ---------------------------------------------------------------------------
 // Loopback host gate (Studio / Observe)
@@ -2542,7 +2735,7 @@ function showHelp() {
     console.log(`      ${(0, ui_js_1.dim)('status')}           Show applied/pending migrations`);
     console.log(`    ${(0, ui_js_1.cyan)('seed')}               Run seed file`);
     console.log(`    ${(0, ui_js_1.cyan)('status')} ${(0, ui_js_1.dim)('| info')}      Show schema summary`);
-    console.log(`    ${(0, ui_js_1.cyan)('doctor')}             Check relations for missing FK indexes ${(0, ui_js_1.dim)('(--fix emits migration)')}`);
+    console.log(`    ${(0, ui_js_1.cyan)('doctor')}             Cost-aware missing-FK-index triage ${(0, ui_js_1.dim)('(--fix, --json, --no-concurrently)')}`);
     console.log(`    ${(0, ui_js_1.cyan)('studio')}             Launch local read-only web UI ${(0, ui_js_1.dim)('(--write for writes, --demo for a sample DB)')}`);
     console.log(`    ${(0, ui_js_1.cyan)('mcp')}                Start read-only MCP server over stdio`);
     console.log(`    ${(0, ui_js_1.cyan)('observe')}            Launch metrics dashboard ${(0, ui_js_1.dim)('(requires TURBINE_OBSERVE_URL)')}`);