turbine-orm 0.46.0 → 0.47.0

package/dist/cjs/observe.js

@@ -1,17 +1,23 @@
 "use strict";
 /**
- * turbine-orm — Observability module
+ * turbine-orm: Observability module
  *
  * Buffers query metrics in memory (keyed by model:action per minute bucket),
- * then periodically flushes aggregates (count, avg, p50, p95, p99, errors)
- * to a dedicated _turbine_metrics table. Uses a separate 1-connection pool
- * so metrics writes never contend with the application pool.
+ * then periodically flushes aggregates (count, avg, p50, p95, p99, errors) to a
+ * pluggable {@link ObserveSink}. The default sink writes to a dedicated
+ * `_turbine_metrics` Postgres table over its own 1-connection pool, so metrics
+ * writes never contend with the application pool; alternative sinks (for example
+ * {@link HttpJsonSink}) can forward the same aggregates elsewhere.
+ *
+ * The aggregation privacy posture is deliberate: a batch carries only the
+ * model/action identity, the counts, and the latency percentiles. It never
+ * carries SQL text or bound parameter values.
  */
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ObserveEngine = void 0;
+exports.ObserveEngine = exports.HttpJsonSink = exports.PgMetricsSink = void 0;
 exports.floorToMinute = floorToMinute;
 exports.percentile = percentile;
 const pg_1 = __importDefault(require("pg"));
@@ -27,7 +33,7 @@ function percentile(sorted, p) {
     return sorted[Math.max(0, idx)];
 }
 // ---------------------------------------------------------------------------
-// Schema DDL
+// Schema DDL + statements (default Postgres sink)
 // ---------------------------------------------------------------------------
 const SCHEMA_DDL = `
 CREATE TABLE IF NOT EXISTS _turbine_metrics (
@@ -58,22 +64,104 @@ ON CONFLICT (bucket, model, action) DO UPDATE SET
   error_count = _turbine_metrics.error_count + EXCLUDED.error_count
 `;
 const RETENTION_SQL = `DELETE FROM _turbine_metrics WHERE bucket < NOW() - INTERVAL '1 day' * $1`;
+/**
+ * The default flush target: upserts each aggregate row into `_turbine_metrics`
+ * and prunes rows older than `retentionDays`. The SQL and per-row/retention
+ * ordering are byte-identical to the pre-sink `ObserveEngine.flush` writer.
+ */
+class PgMetricsSink {
+    pool;
+    retentionDays;
+    constructor(options) {
+        this.pool = new pg_1.default.Pool({ connectionString: options.connectionString, max: 1 });
+        this.retentionDays = options.retentionDays ?? 30;
+    }
+    async init() {
+        await this.pool.query(SCHEMA_DDL);
+    }
+    async flush(batch) {
+        if (batch.rows.length === 0)
+            return;
+        for (const row of batch.rows) {
+            try {
+                await this.pool.query(UPSERT_SQL, [
+                    row.bucket,
+                    row.model,
+                    row.action,
+                    row.count,
+                    row.avg,
+                    row.p50,
+                    row.p95,
+                    row.p99,
+                    row.errors,
+                ]);
+            }
+            catch {
+                // Fire-and-forget: never throw from flush
+            }
+        }
+        try {
+            await this.pool.query(RETENTION_SQL, [this.retentionDays]);
+        }
+        catch {
+            // Best effort
+        }
+    }
+    async stop() {
+        await this.pool.end();
+    }
+}
+exports.PgMetricsSink = PgMetricsSink;
+/**
+ * Forwards each aggregate batch to an HTTP endpoint as a JSON POST. Fire-and-
+ * forget: a failed request is swallowed and never throws, and there are no
+ * retries beyond the engine's next scheduled flush. Aggregates only: the body
+ * carries no SQL text and no parameter values.
+ */
+class HttpJsonSink {
+    url;
+    headers;
+    fetchFn;
+    constructor(options) {
+        this.url = options.url;
+        this.headers = options.headers ?? {};
+        this.fetchFn = options.fetchFunction ?? fetch;
+    }
+    async flush(batch) {
+        if (batch.rows.length === 0)
+            return;
+        try {
+            await this.fetchFn(this.url, {
+                method: 'POST',
+                headers: { 'content-type': 'application/json', ...this.headers },
+                body: JSON.stringify(batch),
+            });
+        }
+        catch {
+            // Fire-and-forget: a failing collector never affects the application.
+        }
+    }
+}
+exports.HttpJsonSink = HttpJsonSink;
 // ---------------------------------------------------------------------------
 // Observe engine
 // ---------------------------------------------------------------------------
 class ObserveEngine {
-    pool;
+    sink;
     buffer = new Map();
     currentBucket;
     flushIntervalMs;
-    retentionDays;
     timer;
     listener;
     stopped = false;
     constructor(config) {
-        this.pool = new pg_1.default.Pool({ connectionString: config.connectionString, max: 1 });
+        if (!config.sink && !config.connectionString) {
+            throw new Error('ObserveEngine requires either a connectionString or a sink');
+        }
+        this.sink =
+            config.sink ??
+                new PgMetricsSink({ connectionString: config.connectionString, retentionDays: config.retentionDays ?? 30 });
         this.flushIntervalMs = config.flushIntervalMs ?? 60_000;
-        this.retentionDays = config.retentionDays ?? 30;
         this.currentBucket = floorToMinute(new Date());
         this.listener = (event) => {
             if (this.stopped)
@@ -97,7 +185,7 @@ class ObserveEngine {
         return this.listener;
     }
     async init() {
-        await this.pool.query(SCHEMA_DDL);
+        await this.sink.init?.();
         this.timer = setInterval(() => {
             this.flush().catch(() => { });
         }, this.flushIntervalMs);
@@ -112,34 +200,32 @@ class ObserveEngine {
         const bucket = this.currentBucket;
         const entries = new Map(this.buffer);
         this.buffer.clear();
+        const rows = [];
         for (const [key, entry] of entries) {
             const [model, action] = key.split(':');
             const sorted = entry.durations.slice().sort((a, b) => a - b);
             const count = sorted.length;
             const avg = sorted.reduce((s, v) => s + v, 0) / count;
-            const p50 = percentile(sorted, 0.5);
-            const p95 = percentile(sorted, 0.95);
-            const p99 = percentile(sorted, 0.99);
-            try {
-                await this.pool.query(UPSERT_SQL, [bucket, model, action, count, avg, p50, p95, p99, entry.errors]);
-            }
-            catch {
-                // Fire-and-forget — never throw from flush
-            }
-        }
-        try {
-            await this.pool.query(RETENTION_SQL, [this.retentionDays]);
-        }
-        catch {
-            // Best effort
+            rows.push({
+                bucket,
+                model: model ?? '',
+                action: action ?? '',
+                count,
+                avg,
+                p50: percentile(sorted, 0.5),
+                p95: percentile(sorted, 0.95),
+                p99: percentile(sorted, 0.99),
+                errors: entry.errors,
+            });
         }
+        await this.sink.flush({ rows });
     }
     async stop() {
         this.stopped = true;
         if (this.timer)
             clearInterval(this.timer);
         await this.flush();
-        await this.pool.end();
+        await this.sink.stop?.();
     }
 }
 exports.ObserveEngine = ObserveEngine;

package/dist/cli/index.d.ts

@@ -14,7 +14,7 @@
  *   turbine migrate status        — Show migration status
  *   turbine seed                  — Run seed file
  *   turbine status                — Show schema summary
- *   turbine doctor                - Cost-aware missing-FK-index triage (--fix, --json, --no-concurrently)
+ *   turbine doctor                - Cost-aware missing-FK-index triage (--fix, --json, --no-concurrently, --unused, --audit)
  *   turbine studio                : Launch local read-only web UI (--demo for a seeded sample DB)
  *   turbine mcp                   — Start read-only MCP server over JSON-RPC stdio
  *   turbine observe               — Launch metrics dashboard (requires TURBINE_OBSERVE_URL)
@@ -51,6 +51,14 @@ export interface CliArgs {
     json?: boolean;
     /** `doctor --fix --no-concurrently`: emit plain CREATE INDEX instead of the CONCURRENTLY + no-transaction form. */
     noConcurrently?: boolean;
+    /** `doctor --unused`: report-only never-scanned / redundant / invalid indexes with DROP suggestions. */
+    unused?: boolean;
+    /** `doctor --audit`: unused report scoped to doctor's own previously-suggested index names. */
+    audit?: boolean;
+    /** `doctor --min-scans <n>`: idx_scan below this counts as never-scanned (default 1 = idx_scan 0). */
+    minScans?: number;
+    /** `doctor --metrics-url <url>`: read _turbine_metrics for the table-heat boost from a separate DB. */
+    metricsUrl?: string;
     /** `init --yes`/`-y`: accept every step's default non-interactively. */
     yes?: boolean;
     /** `init --skip-schema`: don't scaffold the schema file. */

package/dist/cli/index.js

@@ -14,7 +14,7 @@
  *   turbine migrate status        — Show migration status
  *   turbine seed                  — Run seed file
  *   turbine status                — Show schema summary
- *   turbine doctor                - Cost-aware missing-FK-index triage (--fix, --json, --no-concurrently)
+ *   turbine doctor                - Cost-aware missing-FK-index triage (--fix, --json, --no-concurrently, --unused, --audit)
  *   turbine studio                : Launch local read-only web UI (--demo for a seeded sample DB)
  *   turbine mcp                   — Start read-only MCP server over JSON-RPC stdio
  *   turbine observe               — Launch metrics dashboard (requires TURBINE_OBSERVE_URL)
@@ -29,8 +29,8 @@ import { tmpdir } from 'node:os';
 import { basename, dirname, extname, join, relative, resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { generate, generatePrismaMap } from '../generate.js';
-import { buildCreateIndexSql, buildDropIndexSql, findMissingRelationIndexes, } from '../index-advisor.js';
-import { collectStatsSnapshot, findInvalidIndexes, formatBytes, isSnapshotUsable, STATS_THRESHOLDS, scoreMissingIndex, } from '../index-stats.js';
+import { buildCreateIndexSql, buildDropIndexSql, collectDoctorProbeIndexNames, findMissingRelationIndexes, } from '../index-advisor.js';
+import { auditDoctorIndexes, collectStatsSnapshot, collectTableHeat, findInvalidIndexes, findRedundantIndexes, findUnusedIndexes, formatBytes, isSnapshotUsable, STATS_THRESHOLDS, scoreMissingIndex, } from '../index-stats.js';
 import { introspect } from '../introspect.js';
 import { DestructivePushRefusal, schemaDiff, schemaPush } from '../schema-sql.js';
 import { configTemplate, findConfigFile, loadConfigResult, looksLikeSchemaFilePath, resolveConfig, resolveSeedFile, unwrapModuleDefault, } from './config.js';
@@ -128,6 +128,20 @@ export function parseArgs(argv = process.argv.slice(2)) {
             case '--no-concurrently':
                 result.noConcurrently = true;
                 break;
+            case '--unused':
+                result.unused = true;
+                break;
+            case '--audit':
+                result.audit = true;
+                break;
+            case '--min-scans':
+                result.minScans = next ? Number.parseInt(next, 10) : undefined;
+                i++;
+                break;
+            case '--metrics-url':
+                result.metricsUrl = next;
+                i++;
+                break;
             case '--zod':
                 result.zod = true;
                 break;
@@ -2095,20 +2109,64 @@ async function cmdDoctor(args, config) {
             notices: [`statistics collection failed: ${err instanceof Error ? err.message : String(err)}`],
         };
     }
+    // Table-heat boost: read _turbine_metrics (app DB or --metrics-url) and use
+    // per-model heat as an extra benefit signal. Best-effort; a missing table just
+    // means "heat boosting unavailable" and the triage continues without it.
+    const heat = probedTables.length > 0
+        ? await collectTableHeatSafe(args.metricsUrl ?? url, probedTables)
+        : { available: false, tables: {}, notice: null };
     const invalid = findInvalidIndexes(snapshot);
     const usable = isSnapshotUsable(snapshot);
-    const findings = missing.map((m) => ({ missing: m, score: scoreMissingIndex(m, snapshot) }));
+    const findings = missing.map((m) => ({
+        missing: m,
+        score: scoreMissingIndex(m, snapshot, heat.tables[m.table]),
+    }));
+    // "doctor learns to subtract": report-only drop suggestions, never a migration.
+    const unusedRan = args.unused === true;
+    const auditRan = args.audit === true;
+    const minScans = args.minScans;
+    const unused = unusedRan ? findUnusedIndexes(snapshot, { minScans }) : [];
+    const redundant = unusedRan ? findRedundantIndexes(snapshot) : [];
+    const audit = auditRan ? auditDoctorIndexes(snapshot, collectDoctorProbeIndexNames(schema), { minScans }) : [];
+    const subtract = { unusedRan, auditRan, minScans, unused, redundant, audit };
     if (jsonMode) {
         spinner?.stop();
-        console.log(JSON.stringify(buildDoctorJson({ schema, findings, invalid, snapshot, usable, args }), null, 2));
+        console.log(JSON.stringify(buildDoctorJson({ schema, findings, invalid, snapshot, usable, heat, subtract, args }), null, 2));
         return;
     }
-    await renderDoctorHuman({ spinner: spinner, schema, findings, invalid, snapshot, usable, args, config });
+    await renderDoctorHuman({
+        spinner: spinner,
+        schema,
+        findings,
+        invalid,
+        snapshot,
+        usable,
+        heat,
+        subtract,
+        args,
+        config,
+    });
+}
+/** Best-effort table-heat read: any failure degrades to unavailable, never throws. */
+async function collectTableHeatSafe(connectionString, models) {
+    try {
+        return await collectTableHeat({ connectionString, models });
+    }
+    catch (err) {
+        return {
+            available: false,
+            tables: {},
+            notice: `heat boosting is unavailable (reading _turbine_metrics failed: ${err instanceof Error ? err.message.split('\n')[0] : String(err)}).`,
+        };
+    }
 }
-/** The stable, versioned JSON contract (schemaVersion: 1). First external consumer: BataDB import. */
+/**
+ * The stable, versioned JSON contract (schemaVersion: 1). Fields are only ever
+ * added, never removed or repurposed, so a parser never breaks on an upgrade.
+ */
 function buildDoctorJson(ctx) {
     const concurrently = ctx.args.noConcurrently !== true;
-    return {
+    const out = {
         schemaVersion: 1,
         scannedTables: Object.keys(ctx.schema.tables).length,
         stats: {
@@ -2118,6 +2176,10 @@ function buildDoctorJson(ctx) {
             statsAgeDays: ctx.snapshot.statsAgeDays,
             notices: ctx.snapshot.notices,
         },
+        heat: {
+            available: ctx.heat.available,
+            notice: ctx.heat.notice,
+        },
         thresholds: STATS_THRESHOLDS,
         findings: ctx.findings.map((f) => ({
             table: f.missing.table,
@@ -2128,18 +2190,37 @@ function buildDoctorJson(ctx) {
             metrics: f.score.metrics,
             hotWarning: f.score.hotWarning,
             partialNotNull: f.score.partialNotNull,
+            heatBoosted: f.score.heatBoosted,
             probes: f.missing.probes,
             createSql: doctorCreateSql(f, { concurrently }),
             dropSql: buildDropIndexSql(f.missing.indexName, { concurrently }),
         })),
         invalidIndexes: ctx.invalid,
     };
+    // Additive: the drop-suggestion arrays appear only when --unused / --audit ran.
+    if (ctx.subtract.unusedRan) {
+        out.unused = ctx.subtract.unused;
+        out.redundant = ctx.subtract.redundant;
+        out.invalid = ctx.invalid;
+    }
+    if (ctx.subtract.auditRan) {
+        out.audit = ctx.subtract.audit;
+    }
+    return out;
 }
 async function renderDoctorHuman(ctx) {
-    const { spinner, schema, findings, invalid, snapshot, usable, args, config } = ctx;
+    const { spinner, schema, findings, invalid, snapshot, usable, heat, subtract, args, config } = ctx;
     spinner.succeed(`Scanned ${bold(String(Object.keys(schema.tables).length))} tables`);
-    if (findings.length === 0 && invalid.length === 0) {
-        success('Every relation probe is backed by an index, and no invalid indexes were found');
+    const subtractRan = subtract.unusedRan || subtract.auditRan;
+    const nothingToAdd = findings.length === 0 && invalid.length === 0;
+    const nothingToSubtract = subtract.unused.length === 0 && subtract.redundant.length === 0 && subtract.audit.length === 0;
+    if (nothingToAdd && (!subtractRan || nothingToSubtract)) {
+        if (subtractRan) {
+            success('No never-scanned or redundant indexes found, and no invalid indexes were found');
+        }
+        else {
+            success('Every relation probe is backed by an index, and no invalid indexes were found');
+        }
         newline();
         return;
     }
@@ -2155,8 +2236,20 @@ async function renderDoctorHuman(ctx) {
         else {
             renderTopologyFallback(findings, snapshot);
         }
+        // Heat honesty: one line when the workload-heat boost could not be sourced.
+        if (!heat.available && heat.notice) {
+            console.log(`  ${dim(`Note: ${heat.notice}`)}`);
+            newline();
+        }
     }
     renderInvalidIndexes(invalid);
+    if (subtract.unusedRan) {
+        renderUnusedIndexes(subtract.unused, subtract.minScans, snapshot);
+        renderRedundantIndexes(subtract.redundant);
+    }
+    if (subtract.auditRan) {
+        renderDoctorAudit(subtract.audit, subtract.minScans, snapshot);
+    }
     if (findings.length > 0) {
         if (args.fix) {
             renderFixMigration(findings, config, args);
@@ -2168,6 +2261,65 @@ async function renderDoctorHuman(ctx) {
         }
     }
 }
+/** Shared caveat block: what an idx_scan of zero does and does not prove. */
+function renderUnusedCaveats(minScans, snapshot) {
+    const ageLabel = snapshot.statsAgeDays !== null ? `${Math.round(snapshot.statsAgeDays)}d` : 'unknown';
+    const threshold = minScans ?? STATS_THRESHOLDS.unusedMinScans;
+    console.log(`  ${dim(`Usage counters are since the last stats reset (${ageLabel} ago).`)}`);
+    console.log(`  ${dim(`Caveats: counters zero on a stats reset or crash; a read replica's index scans NEVER feed`)}`);
+    console.log(`  ${dim(`the primary's counters, so an index only a replica uses looks dead here. Threshold: idx_scan < ${threshold}.`)}`);
+    console.log(`  ${dim('Primary-key, unique, exclusion, and replica-identity indexes are excluded. Nothing here is auto-dropped.')}`);
+    newline();
+}
+/** doctor --unused: never-scanned indexes with DROP suggestions (report-only). */
+function renderUnusedIndexes(unused, minScans, snapshot) {
+    if (unused.length === 0)
+        return;
+    const total = unused.reduce((sum, u) => sum + (u.sizeBytes ?? 0), 0);
+    warn(`Found ${bold(String(unused.length))} never-scanned index(es) (${formatBytes(total)} reclaimable).`);
+    renderUnusedCaveats(minScans, snapshot);
+    for (const u of unused) {
+        console.log(`  ${yellow(symbols.warning)} ${bold(cyan(u.table))} ${dim(`(${u.columns.join(', ') || '?'})`)}  ${gray(`${u.indexName} · ${u.idxScan} scans · ${formatBytes(u.sizeBytes)}`)}`);
+        console.log(`    ${dim(symbols.teeEnd)} ${green(u.dropSql)}`);
+        newline();
+    }
+}
+/** doctor --unused: redundant leading-prefix indexes with DROP suggestions (report-only). */
+function renderRedundantIndexes(redundant) {
+    if (redundant.length === 0)
+        return;
+    const total = redundant.reduce((sum, r) => sum + (r.sizeBytes ?? 0), 0);
+    warn(`Found ${bold(String(redundant.length))} redundant index(es) (${formatBytes(total)} reclaimable).`);
+    console.log(`  ${dim('Each is a leading prefix of a wider index that already serves the same lookups. Report-only, never auto-dropped.')}`);
+    newline();
+    for (const r of redundant) {
+        console.log(`  ${yellow(symbols.warning)} ${bold(cyan(r.table))} ${dim(`(${r.columns.join(', ')})`)}  ${gray(`${r.indexName} · ${formatBytes(r.sizeBytes)}`)}`);
+        console.log(`    ${dim(symbols.tee)} covered by ${blue(r.coveredBy)} ${dim(`(${r.coveredByColumns.join(', ')})`)}`);
+        console.log(`    ${dim(symbols.teeEnd)} ${green(r.dropSql)}`);
+        newline();
+    }
+}
+/** doctor --audit: doctor's own previously-suggested indexes now never scanned. */
+function renderDoctorAudit(audit, minScans, snapshot) {
+    if (audit.length === 0) {
+        success('No doctor-suggested index is going unused');
+        newline();
+        return;
+    }
+    warn(`doctor previously suggested these indexes; ${bold(String(audit.length))} have never been scanned since the stats reset.`);
+    renderUnusedCaveats(minScans, snapshot);
+    for (const a of audit) {
+        const tag = a.ambiguous ? red(' [ambiguous: truncated name collides with another column set]') : '';
+        console.log(`  ${yellow(symbols.warning)} ${bold(cyan(a.table))} ${dim(`(${a.columns.join(', ') || '?'})`)}  ${gray(`${a.indexName} · ${a.idxScan} scans · ${formatBytes(a.sizeBytes)}`)}${tag}`);
+        if (a.ambiguous) {
+            console.log(`    ${dim(symbols.tee)} ${dim('the 63-byte name maps to more than one probe column set; confirm before dropping')}`);
+        }
+        console.log(`    ${dim(symbols.teeEnd)} ${green(a.dropSql)}`);
+        newline();
+    }
+    console.log(`  ${dim('Consider dropping the ones you confirm are unused. Nothing here is auto-dropped.')}`);
+    newline();
+}
 /** Cost-aware tiered output: three sections, each finding annotated with its numbers. */
 function renderTiers(findings, snapshot, _args) {
     const ageLabel = snapshot.statsAgeDays !== null ? `${Math.round(snapshot.statsAgeDays)}d` : 'unknown';
@@ -2692,7 +2844,7 @@ function showHelp() {
     console.log(`      ${dim('status')}           Show applied/pending migrations`);
     console.log(`    ${cyan('seed')}               Run seed file`);
     console.log(`    ${cyan('status')} ${dim('| info')}      Show schema summary`);
-    console.log(`    ${cyan('doctor')}             Cost-aware missing-FK-index triage ${dim('(--fix, --json, --no-concurrently)')}`);
+    console.log(`    ${cyan('doctor')}             Cost-aware missing-FK-index triage ${dim('(--fix, --json, --unused, --audit)')}`);
     console.log(`    ${cyan('studio')}             Launch local read-only web UI ${dim('(--write for writes, --demo for a sample DB)')}`);
     console.log(`    ${cyan('mcp')}                Start read-only MCP server over stdio`);
     console.log(`    ${cyan('observe')}            Launch metrics dashboard ${dim('(requires TURBINE_OBSERVE_URL)')}`);

package/dist/index-advisor.d.ts

@@ -79,6 +79,25 @@ export declare function buildDropIndexSql(indexName: string, options?: {
  * counts as an index.
  */
 export declare function isProbeIndexed(meta: TableMetadata, columns: string[]): boolean;
+/**
+ * The deterministic index name `doctor --fix` uses for a relation-probe index:
+ * `idx_<table>_<cols>` truncated to Postgres's 63-byte identifier limit. `doctor
+ * --audit` recomputes this to recognize its own previously-suggested indexes.
+ */
+export declare function doctorIndexName(table: string, columns: string[]): string;
+/** A (table, columns) pair whose probe maps to a given deterministic name. */
+export interface DoctorProbeColumns {
+    table: string;
+    columns: string[];
+}
+/**
+ * Map every deterministic doctor index name the schema's relation probes would
+ * generate to the distinct column sets that produce it. A name that maps to more
+ * than one distinct column set is a post-truncation collision (very long column
+ * names slicing to the same 63 bytes): `doctor --audit` reports those as
+ * ambiguous rather than issuing a confident drop verdict.
+ */
+export declare function collectDoctorProbeIndexNames(schema: SchemaMetadata): Map<string, DoctorProbeColumns[]>;
 /**
  * Scan every relation in the schema and return the probes with no index support,
  * deduplicated by (table, column set) with all contributing relations attached.