turbine-orm 0.48.0 → 0.49.0

package/dist/cli/index.js

@@ -29,7 +29,7 @@ import { tmpdir } from 'node:os';
 import { basename, dirname, extname, join, relative, resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { generate, generatePrismaMap } from '../generate.js';
-import { buildCreateIndexSql, buildDropIndexSql, collectDoctorProbeIndexNames, findMissingRelationIndexes, } from '../index-advisor.js';
+import { buildCreateIndexSql, buildDropIndexSql, collectDoctorProbeIndexNames, collectRelationProbeColumns, findMissingRelationIndexes, } from '../index-advisor.js';
 import { auditDoctorIndexes, collectStatsSnapshot, collectTableHeat, findInvalidIndexes, findRedundantIndexes, findUnusedIndexes, formatBytes, isSnapshotUsable, STATS_THRESHOLDS, scoreMissingIndex, } from '../index-stats.js';
 import { introspect } from '../introspect.js';
 import { DestructivePushRefusal, schemaDiff, schemaPush } from '../schema-sql.js';
@@ -2125,9 +2125,14 @@ async function cmdDoctor(args, config) {
     const unusedRan = args.unused === true;
     const auditRan = args.audit === true;
     const minScans = args.minScans;
-    const unused = unusedRan ? findUnusedIndexes(snapshot, { minScans }) : [];
+    // Never suggest dropping an index that still serves a relation probe: the
+    // missing-index half of this very report demands it.
+    const relationProbes = collectRelationProbeColumns(schema);
+    const unused = unusedRan ? findUnusedIndexes(snapshot, { minScans, relationProbes }) : [];
     const redundant = unusedRan ? findRedundantIndexes(snapshot) : [];
-    const audit = auditRan ? auditDoctorIndexes(snapshot, collectDoctorProbeIndexNames(schema), { minScans }) : [];
+    const audit = auditRan
+        ? auditDoctorIndexes(snapshot, collectDoctorProbeIndexNames(schema), { minScans, relationProbes })
+        : [];
     const subtract = { unusedRan, auditRan, minScans, unused, redundant, audit };
     if (jsonMode) {
         spinner?.stop();
@@ -2197,15 +2202,19 @@ function buildDoctorJson(ctx) {
         })),
         invalidIndexes: ctx.invalid,
     };
-    // Additive: the drop-suggestion arrays appear only when --unused / --audit ran.
-    if (ctx.subtract.unusedRan) {
-        out.unused = ctx.subtract.unused;
-        out.redundant = ctx.subtract.redundant;
-        out.invalid = ctx.invalid;
-    }
-    if (ctx.subtract.auditRan) {
-        out.audit = ctx.subtract.audit;
-    }
+    // The keys below are ALWAYS present under `schemaVersion: 1`: a declared
+    // schema version that changes shape by flag forces every consumer to write
+    // `json.unused ?? []`. Which subtraction scans actually ran is reported as
+    // data (`ran`), not as the presence or absence of a key.
+    out.subtraction = {
+        unusedRan: ctx.subtract.unusedRan,
+        auditRan: ctx.subtract.auditRan,
+        minScans: ctx.subtract.minScans ?? null,
+    };
+    out.unused = ctx.subtract.unusedRan ? ctx.subtract.unused : [];
+    out.redundant = ctx.subtract.unusedRan ? ctx.subtract.redundant : [];
+    out.audit = ctx.subtract.auditRan ? ctx.subtract.audit : [];
+    out.invalid = ctx.invalid;
     return out;
 }
 async function renderDoctorHuman(ctx) {
@@ -2269,6 +2278,15 @@ function renderUnusedCaveats(minScans, snapshot) {
     console.log(`  ${dim(`Caveats: counters zero on a stats reset or crash; a read replica's index scans NEVER feed`)}`);
     console.log(`  ${dim(`the primary's counters, so an index only a replica uses looks dead here. Threshold: idx_scan < ${threshold}.`)}`);
     console.log(`  ${dim('Primary-key, unique, exclusion, and replica-identity indexes are excluded. Nothing here is auto-dropped.')}`);
+    console.log(`  ${dim('Indexes that still serve a relation Turbine probes are withheld: this report demands those.')}`);
+    // The cost section refuses to SCORE on stats this young; prescribing drops off
+    // the same counters in the next section would be the report contradicting
+    // itself. Say so where the advice is, not only where the scoring is.
+    if (snapshot.statsAgeDays !== null && snapshot.statsAgeDays < STATS_THRESHOLDS.minStatsAgeDays) {
+        console.log(`  ${yellow(`Statistics are only ${ageLabel} old, below the ${STATS_THRESHOLDS.minStatsAgeDays}d floor this report uses to score cost.`)}`);
+        console.log(`  ${yellow('Treat everything below as a list to investigate, not advice to act on: an index your')}`);
+        console.log(`  ${yellow('workload simply has not reached yet looks identical to a dead one.')}`);
+    }
     newline();
 }
 /** doctor --unused: never-scanned indexes with DROP suggestions (report-only). */
@@ -2280,6 +2298,10 @@ function renderUnusedIndexes(unused, minScans, snapshot) {
     renderUnusedCaveats(minScans, snapshot);
     for (const u of unused) {
         console.log(`  ${yellow(symbols.warning)} ${bold(cyan(u.table))} ${dim(`(${u.columns.join(', ') || '?'})`)}  ${gray(`${u.indexName} · ${u.idxScan} scans · ${formatBytes(u.sizeBytes)}`)}`);
+        // A functional / partial / non-btree index is not a plain `(col)` rebuild:
+        // say what it actually is before anyone runs the DROP.
+        if (u.caveat)
+            console.log(`    ${dim(symbols.tee)} ${yellow(u.caveat)}`);
         console.log(`    ${dim(symbols.teeEnd)} ${green(u.dropSql)}`);
         newline();
     }
@@ -2314,7 +2336,13 @@ function renderDoctorAudit(audit, minScans, snapshot) {
         if (a.ambiguous) {
             console.log(`    ${dim(symbols.tee)} ${dim('the 63-byte name maps to more than one probe column set; confirm before dropping')}`);
         }
-        console.log(`    ${dim(symbols.teeEnd)} ${green(a.dropSql)}`);
+        if (a.stillProbed) {
+            console.log(`    ${dim(symbols.teeEnd)} ${dim('kept: a relation in your schema still probes these columns, so dropping it would')}`);
+            console.log(`      ${dim('reappear as a missing-index finding in this same report.')}`);
+        }
+        else if (a.dropSql) {
+            console.log(`    ${dim(symbols.teeEnd)} ${green(a.dropSql)}`);
+        }
         newline();
     }
     console.log(`  ${dim('Consider dropping the ones you confirm are unused. Nothing here is auto-dropped.')}`);
@@ -2472,6 +2500,7 @@ async function cmdStudio(args, config) {
             write: args.write === true,
             showPii: args.showPii === true,
             demo,
+            metadataDir: config.out,
         });
         spinner.succeed(demo ? 'Demo Studio is running' : 'Studio is running');
     }
@@ -2506,6 +2535,20 @@ async function cmdStudio(args, config) {
             newline();
             console.log(warn('--show-pii is ON. PII-tagged column values are shown UNREDACTED in Studio.'));
         }
+        // PII tags are a code-first declaration; introspection never infers them.
+        // Say plainly whether any reached this session, so nobody assumes a
+        // redaction guarantee that has nothing to act on.
+        if (!args.showPii) {
+            newline();
+            if (studio.piiTags && studio.piiTags.applied > 0) {
+                console.log(`  ${dim('PII redaction:')} ${studio.piiTags.applied} tagged column(s) from ${dim(studio.piiTags.path)}`);
+            }
+            else {
+                console.log(warn('No PII-tagged columns found, so nothing will be redacted. Tags are declared in code ' +
+                    `(defineSchema \`pii: true\`) and read from generated metadata in ${config.out}; ` +
+                    'introspection alone never infers them. Run `turbine generate` after tagging.'));
+            }
+        }
         newline();
         console.log(box([
             `${bold('Turbine Studio')}  ${dim(args.write ? 'local UI (WRITE MODE)' : 'local read-only UI')}`,
@@ -2546,6 +2589,7 @@ async function cmdMcp(_args, config) {
         url,
         schema: config.schema,
         migrationsDir: config.migrationsDir,
+        metadataDir: config.out,
         include: config.include.length ? config.include : undefined,
         exclude: config.exclude.length ? config.exclude : undefined,
     });

package/dist/cli/mcp.d.ts

@@ -6,6 +6,13 @@ export interface McpServerOptions {
     migrationsDir: string;
     include?: string[];
     exclude?: string[];
+    /**
+     * Directory holding generated Turbine metadata (`turbine generate`'s `out`).
+     * PII tags are code-first declarations that introspection never sets, so
+     * without this the server has nothing to redact against. Read as text;
+     * nothing from it is executed. See `pii-tags.ts`.
+     */
+    metadataDir?: string;
 }
 export interface McpTransport {
     input?: Readable;

package/dist/cli/mcp.js

@@ -7,6 +7,7 @@ import { addAutoManyToManyRelations, buildRelationsFromForeignKeys, isUnknownTsT
 import { QueryInterface, quoteIdent } from '../query/index.js';
 import { isDateType, pgArrayType, pgTypeToTs, snakeToCamel, } from '../schema.js';
 import { listMigrationFiles } from './migrate.js';
+import { applyPiiTags, loadPiiTags } from './pii-tags.js';
 /**
  * Walk up from the running script to find turbine-orm's own package.json.
  * Uses process.argv[1] instead of import.meta.url so the same code compiles
@@ -402,15 +403,27 @@ async function sampleRows(ctx, tableName, limit) {
         const table = requireTable(metadata, tableName);
         const qualifiedTable = `${quoteIdent(ctx.options.schema)}.${quoteIdent(table.name)}`;
         const result = await client.query(`SELECT * FROM ${qualifiedTable} LIMIT $1`, [limit]);
+        // Sample rows go straight into an LLM context, so PII-tagged values are
+        // replaced before serialization, the same stance Studio's Data tab takes.
+        const piiColumns = new Set(table.columns.filter((c) => c.pii).map((c) => c.name));
         return {
             table: table.name,
             limit,
+            redactedColumns: [...piiColumns],
             columns: result.fields.map((field) => ({ name: field.name, dataTypeID: field.dataTypeID })),
-            rows: result.rows,
+            rows: piiColumns.size === 0 ? result.rows : result.rows.map((row) => redactRow(row, piiColumns)),
             rowCount: result.rowCount ?? result.rows.length,
         };
     });
 }
+/** Replace PII-tagged cells with a fixed marker (never the value, never null). */
+function redactRow(row, piiColumns) {
+    const out = {};
+    for (const [key, value] of Object.entries(row)) {
+        out[key] = piiColumns.has(key) ? '•• redacted ••' : value;
+    }
+    return out;
+}
 async function withReadOnly(ctx, fn) {
     const client = await ctx.pool.connect();
     try {
@@ -588,7 +601,15 @@ async function loadSchemaMetadata(client, options) {
             indexes: indexesByTable.get(tableName) ?? [],
         };
     }
-    return { tables, enums };
+    const metadata = { tables, enums };
+    // Code-first PII tags, layered onto the live catalog. Without this the
+    // redaction below has nothing to act on (introspection never infers a tag).
+    if (options.metadataDir) {
+        const source = loadPiiTags(options.metadataDir);
+        if (source)
+            applyPiiTags(metadata, source.tags);
+    }
+    return metadata;
 }
 /**
  * Group raw FK rows into constraint-level entries and delegate relation

package/dist/cli/migrate.d.ts

@@ -118,7 +118,8 @@ export declare function parseMigrationContent(content: string): ParsedMigration;
  * Split a SQL script into individual statements on top-level semicolons.
  *
  * A correct tokenizer, not a `split(';')`: a semicolon inside a single-quoted
- * string, a double-quoted identifier, a dollar-quoted body, a line comment
+ * string (including a backslash-escaping `E'...'` string), a double-quoted
+ * identifier, a dollar-quoted body, a line comment
  * (`--`), or a block comment (`/* *\/`, which Postgres allows to nest) must NOT
  * split. This is the one production-destroying failure mode of no-transaction
  * migrations (a partial statement executed against production), so the behavior

package/dist/cli/migrate.js

@@ -170,7 +170,8 @@ export function parseMigrationContent(content) {
  * Split a SQL script into individual statements on top-level semicolons.
  *
  * A correct tokenizer, not a `split(';')`: a semicolon inside a single-quoted
- * string, a double-quoted identifier, a dollar-quoted body, a line comment
+ * string (including a backslash-escaping `E'...'` string), a double-quoted
+ * identifier, a dollar-quoted body, a line comment
  * (`--`), or a block comment (`/* *\/`, which Postgres allows to nest) must NOT
  * split. This is the one production-destroying failure mode of no-transaction
  * migrations (a partial statement executed against production), so the behavior
@@ -221,10 +222,19 @@ export function splitSqlStatements(sql) {
             continue;
         }
         // Single-quoted string ('' is an escaped quote, stays inside the string).
+        // An E-prefixed string (E'...') additionally honors backslash escapes, so
+        // `E'p\'q'` is ONE string: treating the `\'` as a terminator would close the
+        // string early and let the next quote swallow a real statement terminator.
         if (ch === "'") {
+            const backslashEscapes = isEscapeStringPrefix(sql, i);
             let j = i + 1;
             current += "'";
             while (j < n) {
+                if (backslashEscapes && sql[j] === '\\' && j + 1 < n) {
+                    current += sql[j] + sql[j + 1];
+                    j += 2;
+                    continue;
+                }
                 if (sql[j] === "'" && sql[j + 1] === "'") {
                     current += "''";
                     j += 2;
@@ -296,6 +306,23 @@ export function splitSqlStatements(sql) {
         statements.push(tail);
     return statements.filter((s) => !isCommentOnlyStatement(s));
 }
+/**
+ * True when the quote at `quoteAt` opens a Postgres escape string (`E'...'`),
+ * whose body treats a backslash as an escape character.
+ *
+ * The `E` must be a standalone token: an identifier that merely ends in `e`
+ * (`some_table` cannot be followed by a quote in valid SQL, but the check keeps
+ * the tokenizer honest) does not turn the following literal into an E-string.
+ * Ordinary literals are left alone on purpose: with the modern
+ * `standard_conforming_strings = on` default, `'a\'` IS a complete string.
+ */
+function isEscapeStringPrefix(sql, quoteAt) {
+    const prev = sql[quoteAt - 1];
+    if (prev !== 'E' && prev !== 'e')
+        return false;
+    const before = sql[quoteAt - 2];
+    return before === undefined || !/[A-Za-z0-9_$"]/.test(before);
+}
 /** True when a fragment contains nothing but comments and whitespace. */
 function isCommentOnlyStatement(stmt) {
     const withoutComments = stmt.replace(/\/\*[\s\S]*?\*\//g, ' ').replace(/--[^\n]*/g, ' ');

package/dist/cli/pii-tags.d.ts

@@ -0,0 +1,53 @@
+/**
+ * PII tags for tools that build their schema from live introspection.
+ *
+ * `ColumnMetadata.pii` is a CODE-FIRST declaration: it comes from
+ * `defineSchema({ pii: true })` or the fluent `.pii()`, and `introspect.ts`
+ * never sets it (there is no reliable way to infer "this column holds personal
+ * data" from a Postgres catalog, and guessing would be worse than not trying).
+ *
+ * Studio and the MCP server introspect a live database, so on their own they
+ * see NO tags at all and their redaction is inert. This module closes that gap
+ * by reading the tags out of the generated `metadata.ts` that `turbine
+ * generate` writes, and handing back a table → column-name map the caller
+ * layers onto its introspected metadata.
+ *
+ * The generated file is TypeScript in the user's project, so it cannot simply
+ * be imported by a compiled CLI. It is read as TEXT and scanned for the exact
+ * shapes `generate.ts` emits (`serializeColumn`: one column object per line,
+ * `pii: true` only when tagged). Nothing is executed. A file that does not
+ * parse yields no tags, and the caller decides what to say about that.
+ */
+/** Table name → the snake_case column names tagged `pii: true`. */
+export type PiiTagMap = Record<string, string[]>;
+export interface PiiTagSource {
+    /** Absolute path that was read. */
+    path: string;
+    tags: PiiTagMap;
+    /** Total tagged columns, across tables. */
+    count: number;
+}
+/**
+ * Scan generated-metadata source text for PII-tagged columns.
+ *
+ * Exported for testing; callers normally use {@link loadPiiTags}.
+ */
+export declare function parsePiiTags(source: string): PiiTagMap;
+/**
+ * Read PII tags from the generated metadata in `outDir`, or return `null` when
+ * there is no readable generated metadata there. Never throws.
+ */
+export declare function loadPiiTags(outDir: string): PiiTagSource | null;
+/**
+ * Apply `tags` to introspected metadata, in place. Only columns that exist in
+ * the live schema are tagged, so a stale generated file can never invent one.
+ * Returns the number of columns actually tagged.
+ */
+export declare function applyPiiTags(metadata: {
+    tables: Record<string, {
+        columns: {
+            name: string;
+            pii?: boolean;
+        }[];
+    }>;
+}, tags: PiiTagMap): number;

package/dist/cli/pii-tags.js

@@ -0,0 +1,106 @@
+/**
+ * PII tags for tools that build their schema from live introspection.
+ *
+ * `ColumnMetadata.pii` is a CODE-FIRST declaration: it comes from
+ * `defineSchema({ pii: true })` or the fluent `.pii()`, and `introspect.ts`
+ * never sets it (there is no reliable way to infer "this column holds personal
+ * data" from a Postgres catalog, and guessing would be worse than not trying).
+ *
+ * Studio and the MCP server introspect a live database, so on their own they
+ * see NO tags at all and their redaction is inert. This module closes that gap
+ * by reading the tags out of the generated `metadata.ts` that `turbine
+ * generate` writes, and handing back a table → column-name map the caller
+ * layers onto its introspected metadata.
+ *
+ * The generated file is TypeScript in the user's project, so it cannot simply
+ * be imported by a compiled CLI. It is read as TEXT and scanned for the exact
+ * shapes `generate.ts` emits (`serializeColumn`: one column object per line,
+ * `pii: true` only when tagged). Nothing is executed. A file that does not
+ * parse yields no tags, and the caller decides what to say about that.
+ */
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+/** `    <table>: {` at the table-entry indentation `generateMetadata` emits. */
+const TABLE_HEAD = /^ {4}(?:'([^']+)'|([A-Za-z_$][\w$]*)): \{$/;
+/** A serialized column object; `pii: true` is emitted only when tagged. */
+const COLUMN_LINE = /^ {8}\{ name: '([^']+)'.*\bpii: true\b/;
+/** `      columns: [` opens the column list; `      ],` closes it. */
+const COLUMNS_OPEN = /^ {6}columns: \[$/;
+const COLUMNS_CLOSE = /^ {6}\],$/;
+/**
+ * Scan generated-metadata source text for PII-tagged columns.
+ *
+ * Exported for testing; callers normally use {@link loadPiiTags}.
+ */
+export function parsePiiTags(source) {
+    const tags = {};
+    let table = null;
+    let inColumns = false;
+    for (const line of source.split('\n')) {
+        const head = TABLE_HEAD.exec(line);
+        if (head) {
+            table = head[1] ?? head[2] ?? null;
+            inColumns = false;
+            continue;
+        }
+        if (!table)
+            continue;
+        if (COLUMNS_OPEN.test(line)) {
+            inColumns = true;
+            continue;
+        }
+        if (inColumns && COLUMNS_CLOSE.test(line)) {
+            inColumns = false;
+            continue;
+        }
+        if (!inColumns)
+            continue;
+        const col = COLUMN_LINE.exec(line);
+        if (col?.[1]) {
+            const list = tags[table] ?? [];
+            list.push(col[1]);
+            tags[table] = list;
+        }
+    }
+    return tags;
+}
+/**
+ * Read PII tags from the generated metadata in `outDir`, or return `null` when
+ * there is no readable generated metadata there. Never throws.
+ */
+export function loadPiiTags(outDir) {
+    for (const file of ['metadata.ts', 'metadata.js']) {
+        const path = join(outDir, file);
+        let source;
+        try {
+            source = readFileSync(path, 'utf8');
+        }
+        catch {
+            continue;
+        }
+        const tags = parsePiiTags(source);
+        const count = Object.values(tags).reduce((n, cols) => n + cols.length, 0);
+        return { path, tags, count };
+    }
+    return null;
+}
+/**
+ * Apply `tags` to introspected metadata, in place. Only columns that exist in
+ * the live schema are tagged, so a stale generated file can never invent one.
+ * Returns the number of columns actually tagged.
+ */
+export function applyPiiTags(metadata, tags) {
+    let applied = 0;
+    for (const [tableName, columns] of Object.entries(tags)) {
+        const table = Object.hasOwn(metadata.tables, tableName) ? metadata.tables[tableName] : undefined;
+        if (!table)
+            continue;
+        for (const col of table.columns) {
+            if (columns.includes(col.name)) {
+                col.pii = true;
+                applied++;
+            }
+        }
+    }
+    return applied;
+}