turbine-orm 0.46.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/cli/index.js +162 -10
- package/dist/cjs/index-advisor.js +0 -0
- package/dist/cjs/index-stats.js +219 -3
- package/dist/cjs/index.js +6 -2
- package/dist/cjs/observe.js +113 -27
- package/dist/cli/index.d.ts +9 -1
- package/dist/cli/index.js +164 -12
- package/dist/index-advisor.d.ts +19 -0
- package/dist/index-advisor.js +0 -0
- package/dist/index-stats.d.ts +127 -2
- package/dist/index-stats.js +215 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -0
- package/dist/observe.d.ts +88 -7
- package/dist/observe.js +110 -26
- package/package.json +1 -1
package/dist/index-stats.d.ts
CHANGED
|
@@ -68,6 +68,18 @@ export declare const STATS_THRESHOLDS: {
|
|
|
68
68
|
readonly appendHeavyInsertRatio: 0.95;
|
|
69
69
|
/** ... and seq_scan at or below this ("near-zero probe reads") → scrutinize. */
|
|
70
70
|
readonly appendHeavyMaxSeqScan: 5;
|
|
71
|
+
/**
|
|
72
|
+
* Default `--min-scans` for the unused-index report: an index with fewer than
|
|
73
|
+
* this many scans since stats_reset is flagged never-scanned. Default 1 makes
|
|
74
|
+
* the bare report mean exactly "idx_scan = 0".
|
|
75
|
+
*/
|
|
76
|
+
readonly unusedMinScans: 1;
|
|
77
|
+
/**
|
|
78
|
+
* Queries/min (from _turbine_metrics) at or above this makes a table "hot in
|
|
79
|
+
* your workload": a benefit signal that boosts the finding's priority and
|
|
80
|
+
* annotates it. It never downgrades a cost tier (write cost is a separate axis).
|
|
81
|
+
*/
|
|
82
|
+
readonly heatMinQueriesPerMin: 1;
|
|
71
83
|
};
|
|
72
84
|
/** Per-table live statistics. Any field may be absent when its catalog read degraded. */
|
|
73
85
|
export interface TableStats {
|
|
@@ -99,6 +111,10 @@ export interface IndexStat {
|
|
|
99
111
|
isUnique: boolean;
|
|
100
112
|
isPrimary: boolean;
|
|
101
113
|
isReplicaIdent: boolean;
|
|
114
|
+
/** Backs an exclusion constraint (pg_constraint.contype = 'x'). */
|
|
115
|
+
isExclusion?: boolean;
|
|
116
|
+
/** pg_relation_size of the index heap, bytes. Size reclaimed by a drop. */
|
|
117
|
+
sizeBytes?: number;
|
|
102
118
|
}
|
|
103
119
|
/**
|
|
104
120
|
* A point-in-time read of the statistics the triage needs. Every part is
|
|
@@ -150,9 +166,18 @@ export interface ScoredMissingIndex {
|
|
|
150
166
|
hotWarning: string | null;
|
|
151
167
|
/** When true, the emitted index should be a partial `WHERE col IS NOT NULL`. */
|
|
152
168
|
partialNotNull: boolean;
|
|
153
|
-
/**
|
|
169
|
+
/** True when workload heat lifted this finding's priority (see {@link TableHeatEntry}). */
|
|
170
|
+
heatBoosted: boolean;
|
|
171
|
+
/** Sort key within a tier: bigger, more-probed tables first; hot tables first of all. */
|
|
154
172
|
benefitScore: number;
|
|
155
173
|
}
|
|
174
|
+
/** Per-table workload heat, derived from the _turbine_metrics per-minute aggregates. */
|
|
175
|
+
export interface TableHeatEntry {
|
|
176
|
+
/** Average queries per minute across the observed window. */
|
|
177
|
+
queriesPerMin: number;
|
|
178
|
+
/** Worst observed p95 latency (ms) across the window. */
|
|
179
|
+
p95Ms: number;
|
|
180
|
+
}
|
|
156
181
|
/** The subset of a topology finding the scorer needs. */
|
|
157
182
|
export interface ScorableMissingIndex {
|
|
158
183
|
table: string;
|
|
@@ -165,7 +190,7 @@ export declare function formatBytes(bytes: number | null | undefined): string;
|
|
|
165
190
|
* Score a single missing-index finding against a snapshot. Pure and total:
|
|
166
191
|
* every unknown degrades to a caveat rather than a fabricated number.
|
|
167
192
|
*/
|
|
168
|
-
export declare function scoreMissingIndex(missing: ScorableMissingIndex, snapshot: StatsSnapshot): ScoredMissingIndex;
|
|
193
|
+
export declare function scoreMissingIndex(missing: ScorableMissingIndex, snapshot: StatsSnapshot, heat?: TableHeatEntry): ScoredMissingIndex;
|
|
169
194
|
export interface InvalidIndex {
|
|
170
195
|
table: string;
|
|
171
196
|
indexName: string;
|
|
@@ -179,6 +204,81 @@ export interface InvalidIndex {
|
|
|
179
204
|
* skips the corpse, so the fix is DROP INDEX CONCURRENTLY then rerun.
|
|
180
205
|
*/
|
|
181
206
|
export declare function findInvalidIndexes(snapshot: StatsSnapshot): InvalidIndex[];
|
|
207
|
+
/** A never-scanned index and the DROP statement that would reclaim it. */
|
|
208
|
+
export interface UnusedIndex {
|
|
209
|
+
table: string;
|
|
210
|
+
indexName: string;
|
|
211
|
+
columns: string[];
|
|
212
|
+
/** idx_scan since the last stats reset (0 or below --min-scans). */
|
|
213
|
+
idxScan: number;
|
|
214
|
+
/** Size reclaimed by the drop, bytes (null when the size read degraded). */
|
|
215
|
+
sizeBytes: number | null;
|
|
216
|
+
/** `DROP INDEX CONCURRENTLY IF EXISTS`: printed only, never written to a migration. */
|
|
217
|
+
dropSql: string;
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Indexes never (or barely) scanned since the last stats reset. Report-only:
|
|
221
|
+
* counters reset on a crash/reset and REPLICA READS NEVER FEED PRIMARY COUNTERS,
|
|
222
|
+
* so an index only a read replica uses looks dead here. Constraint-backing and
|
|
223
|
+
* replica-identity indexes are excluded by construction. An index whose idx_scan
|
|
224
|
+
* could not be read (no pg_stat row) is skipped rather than guessed.
|
|
225
|
+
*/
|
|
226
|
+
export declare function findUnusedIndexes(snapshot: StatsSnapshot, options?: {
|
|
227
|
+
minScans?: number;
|
|
228
|
+
}): UnusedIndex[];
|
|
229
|
+
/** A redundant index whose columns are a leading prefix of a wider index. */
|
|
230
|
+
export interface RedundantIndex {
|
|
231
|
+
table: string;
|
|
232
|
+
indexName: string;
|
|
233
|
+
columns: string[];
|
|
234
|
+
/** The wider index that already covers this one's leading-prefix lookups. */
|
|
235
|
+
coveredBy: string;
|
|
236
|
+
coveredByColumns: string[];
|
|
237
|
+
sizeBytes: number | null;
|
|
238
|
+
dropSql: string;
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Non-unique indexes whose column list is a leading prefix of a WIDER index on
|
|
242
|
+
* the same table. A btree serves any leading-prefix lookup, so the narrow index
|
|
243
|
+
* is redundant. Pure metadata (no idx_scan needed).
|
|
244
|
+
*
|
|
245
|
+
* Uniqueness compatibility: only a NON-unique index is ever reported. A unique
|
|
246
|
+
* or primary-key prefix is load-bearing (it enforces a constraint), so it is
|
|
247
|
+
* never called redundant even when a wider index shares its leading columns.
|
|
248
|
+
*/
|
|
249
|
+
export declare function findRedundantIndexes(snapshot: StatsSnapshot): RedundantIndex[];
|
|
250
|
+
/**
|
|
251
|
+
* A `doctor --audit` finding: an existing index that matches doctor's own
|
|
252
|
+
* deterministic naming AND has not been scanned since the stats reset.
|
|
253
|
+
*/
|
|
254
|
+
export interface DoctorIndexAudit {
|
|
255
|
+
table: string;
|
|
256
|
+
indexName: string;
|
|
257
|
+
columns: string[];
|
|
258
|
+
idxScan: number;
|
|
259
|
+
sizeBytes: number | null;
|
|
260
|
+
dropSql: string;
|
|
261
|
+
/**
|
|
262
|
+
* True when the truncated (63-byte) name collides across DIFFERENT column sets
|
|
263
|
+
* in the schema's probes, so which suggestion this index realizes is
|
|
264
|
+
* ambiguous. Reported as ambiguous instead of a confident drop verdict.
|
|
265
|
+
*/
|
|
266
|
+
ambiguous: boolean;
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* The unused-index machinery scoped to doctor's OWN previously-suggested indexes:
|
|
270
|
+
* existing indexes whose name matches the `idx_<table>_<cols>` (63-byte truncated)
|
|
271
|
+
* shape doctor emits, never scanned since the stats reset. `doctorNames` maps each
|
|
272
|
+
* deterministic name to the distinct column sets that truncate to it (see
|
|
273
|
+
* `collectDoctorProbeIndexNames`); a name with more than one column set is a
|
|
274
|
+
* post-truncation collision, reported as ambiguous.
|
|
275
|
+
*/
|
|
276
|
+
export declare function auditDoctorIndexes(snapshot: StatsSnapshot, doctorNames: Map<string, Array<{
|
|
277
|
+
table: string;
|
|
278
|
+
columns: string[];
|
|
279
|
+
}>>, options?: {
|
|
280
|
+
minScans?: number;
|
|
281
|
+
}): DoctorIndexAudit[];
|
|
182
282
|
/**
|
|
183
283
|
* Whether the snapshot is trustworthy enough to render tier verdicts. Empty,
|
|
184
284
|
* unavailable, or too-young stats degrade to the topology-only report.
|
|
@@ -206,3 +306,28 @@ export interface CollectSnapshotOptions {
|
|
|
206
306
|
* signal rather than the whole snapshot.
|
|
207
307
|
*/
|
|
208
308
|
export declare function collectStatsSnapshot(options: CollectSnapshotOptions): Promise<StatsSnapshot>;
|
|
309
|
+
export interface CollectTableHeatOptions {
|
|
310
|
+
/** Where _turbine_metrics lives (the app DB, or a separate --metrics-url DB). */
|
|
311
|
+
connectionString: string;
|
|
312
|
+
/** Model names to read heat for (the probed table names). */
|
|
313
|
+
models: string[];
|
|
314
|
+
/** Trailing window of per-minute buckets to average, minutes. Default 60. */
|
|
315
|
+
windowMinutes?: number;
|
|
316
|
+
statementTimeoutMs?: number;
|
|
317
|
+
}
|
|
318
|
+
export interface TableHeatResult {
|
|
319
|
+
/** True when _turbine_metrics existed and was read. False = heat boosting off. */
|
|
320
|
+
available: boolean;
|
|
321
|
+
/** Per-model heat, keyed by model (= table) name. */
|
|
322
|
+
tables: Record<string, TableHeatEntry>;
|
|
323
|
+
/** One honesty line when heat could not be sourced (absent table / non-pg sink). */
|
|
324
|
+
notice: string | null;
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Read per-model workload heat from a `_turbine_metrics` table (written by
|
|
328
|
+
* `db.$observe()` with the default Postgres sink). When the table is absent, the
|
|
329
|
+
* result is `available: false` with a notice: this is the expected state when
|
|
330
|
+
* observe uses a non-Postgres sink, or when no metrics have been collected yet.
|
|
331
|
+
* Every read is best-effort; a failure degrades to unavailable, never throws.
|
|
332
|
+
*/
|
|
333
|
+
export declare function collectTableHeat(options: CollectTableHeatOptions): Promise<TableHeatResult>;
|
package/dist/index-stats.js
CHANGED
|
@@ -72,6 +72,18 @@ export const STATS_THRESHOLDS = {
|
|
|
72
72
|
appendHeavyInsertRatio: 0.95,
|
|
73
73
|
/** ... and seq_scan at or below this ("near-zero probe reads") → scrutinize. */
|
|
74
74
|
appendHeavyMaxSeqScan: 5,
|
|
75
|
+
/**
|
|
76
|
+
* Default `--min-scans` for the unused-index report: an index with fewer than
|
|
77
|
+
* this many scans since stats_reset is flagged never-scanned. Default 1 makes
|
|
78
|
+
* the bare report mean exactly "idx_scan = 0".
|
|
79
|
+
*/
|
|
80
|
+
unusedMinScans: 1,
|
|
81
|
+
/**
|
|
82
|
+
* Queries/min (from _turbine_metrics) at or above this makes a table "hot in
|
|
83
|
+
* your workload": a benefit signal that boosts the finding's priority and
|
|
84
|
+
* annotates it. It never downgrades a cost tier (write cost is a separate axis).
|
|
85
|
+
*/
|
|
86
|
+
heatMinQueriesPerMin: 1,
|
|
75
87
|
};
|
|
76
88
|
/** Build an empty (fully unavailable) snapshot - the honest "no stats" baseline. */
|
|
77
89
|
export function emptyStatsSnapshot(notices = []) {
|
|
@@ -107,7 +119,7 @@ function formatInt(n) {
|
|
|
107
119
|
* Score a single missing-index finding against a snapshot. Pure and total:
|
|
108
120
|
* every unknown degrades to a caveat rather than a fabricated number.
|
|
109
121
|
*/
|
|
110
|
-
export function scoreMissingIndex(missing, snapshot) {
|
|
122
|
+
export function scoreMissingIndex(missing, snapshot, heat) {
|
|
111
123
|
const t = STATS_THRESHOLDS;
|
|
112
124
|
const stats = snapshot.tables[missing.table];
|
|
113
125
|
const probingRelations = missing.probes.length;
|
|
@@ -219,8 +231,17 @@ export function scoreMissingIndex(missing, snapshot) {
|
|
|
219
231
|
if (partialNotNull) {
|
|
220
232
|
reasons.push(`column is ${Math.round((nullFrac ?? 0) * 100)}% NULL: suggesting a partial "WHERE ${missing.columns[0]} IS NOT NULL" index (caveat: a user-written where: { ${missing.columns[0]}: null } filter will NOT use it)`);
|
|
221
233
|
}
|
|
234
|
+
// Workload heat is a BENEFIT signal, not a cost one: a table your app hits hard
|
|
235
|
+
// is a table where a missing index hurts most. It never downgrades a cost tier
|
|
236
|
+
// (write cost is decided above); it re-prioritizes and annotates.
|
|
237
|
+
const heatBoosted = heat !== undefined && heat.queriesPerMin >= t.heatMinQueriesPerMin;
|
|
238
|
+
if (heatBoosted && heat !== undefined) {
|
|
239
|
+
reasons.push(`hot in your workload: ${formatInt(heat.queriesPerMin)} queries/min, p95 ${heat.p95Ms >= 10 ? Math.round(heat.p95Ms) : heat.p95Ms.toFixed(1)} ms`);
|
|
240
|
+
}
|
|
222
241
|
// Benefit sort key: bigger, more-probed tables first. Unknown rows sort last.
|
|
223
|
-
|
|
242
|
+
// A heat-boosted finding is lifted above every non-hot finding in its tier.
|
|
243
|
+
const base = (rows ?? 0) * Math.max(1, probingRelations);
|
|
244
|
+
const benefitScore = heatBoosted && heat !== undefined ? base + (heat.queriesPerMin + 1) * 1e12 : base;
|
|
224
245
|
return {
|
|
225
246
|
table: missing.table,
|
|
226
247
|
columns: missing.columns,
|
|
@@ -229,6 +250,7 @@ export function scoreMissingIndex(missing, snapshot) {
|
|
|
229
250
|
metrics,
|
|
230
251
|
hotWarning,
|
|
231
252
|
partialNotNull,
|
|
253
|
+
heatBoosted,
|
|
232
254
|
benefitScore,
|
|
233
255
|
};
|
|
234
256
|
}
|
|
@@ -255,6 +277,123 @@ export function findInvalidIndexes(snapshot) {
|
|
|
255
277
|
}))
|
|
256
278
|
.sort((a, b) => a.indexName.localeCompare(b.indexName));
|
|
257
279
|
}
|
|
280
|
+
// ---------------------------------------------------------------------------
|
|
281
|
+
// Unused-index detection (pure): doctor learns to subtract
|
|
282
|
+
// ---------------------------------------------------------------------------
|
|
283
|
+
/**
|
|
284
|
+
* An index constraint-backing indexes are NEVER candidates for a drop suggestion:
|
|
285
|
+
* a primary key, a unique constraint, or an exclusion constraint owns its index,
|
|
286
|
+
* and a replica-identity index is load-bearing for logical replication. Dropping
|
|
287
|
+
* any of these changes semantics, not just performance.
|
|
288
|
+
*/
|
|
289
|
+
function isConstraintBacking(idx) {
|
|
290
|
+
return idx.isPrimary || idx.isUnique || idx.isExclusion === true || idx.isReplicaIdent;
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* Indexes never (or barely) scanned since the last stats reset. Report-only:
|
|
294
|
+
* counters reset on a crash/reset and REPLICA READS NEVER FEED PRIMARY COUNTERS,
|
|
295
|
+
* so an index only a read replica uses looks dead here. Constraint-backing and
|
|
296
|
+
* replica-identity indexes are excluded by construction. An index whose idx_scan
|
|
297
|
+
* could not be read (no pg_stat row) is skipped rather than guessed.
|
|
298
|
+
*/
|
|
299
|
+
export function findUnusedIndexes(snapshot, options = {}) {
|
|
300
|
+
const minScans = options.minScans ?? STATS_THRESHOLDS.unusedMinScans;
|
|
301
|
+
return snapshot.indexes
|
|
302
|
+
.filter((idx) => idx.isValid && !isConstraintBacking(idx))
|
|
303
|
+
.filter((idx) => idx.idxScan !== undefined && idx.idxScan < minScans)
|
|
304
|
+
.map((idx) => ({
|
|
305
|
+
table: idx.table,
|
|
306
|
+
indexName: idx.indexName,
|
|
307
|
+
columns: idx.columns,
|
|
308
|
+
idxScan: idx.idxScan ?? 0,
|
|
309
|
+
sizeBytes: idx.sizeBytes ?? null,
|
|
310
|
+
dropSql: buildDropIndexSql(idx.indexName, { concurrently: true }),
|
|
311
|
+
}))
|
|
312
|
+
.sort((a, b) => (b.sizeBytes ?? 0) - (a.sizeBytes ?? 0) || a.indexName.localeCompare(b.indexName));
|
|
313
|
+
}
|
|
314
|
+
/** Whether `prefix` is a strict leading prefix of `columns`. */
|
|
315
|
+
function isLeadingPrefix(prefix, columns) {
|
|
316
|
+
if (prefix.length === 0 || prefix.length >= columns.length)
|
|
317
|
+
return false;
|
|
318
|
+
return prefix.every((c, i) => columns[i] === c);
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* Non-unique indexes whose column list is a leading prefix of a WIDER index on
|
|
322
|
+
* the same table. A btree serves any leading-prefix lookup, so the narrow index
|
|
323
|
+
* is redundant. Pure metadata (no idx_scan needed).
|
|
324
|
+
*
|
|
325
|
+
* Uniqueness compatibility: only a NON-unique index is ever reported. A unique
|
|
326
|
+
* or primary-key prefix is load-bearing (it enforces a constraint), so it is
|
|
327
|
+
* never called redundant even when a wider index shares its leading columns.
|
|
328
|
+
*/
|
|
329
|
+
export function findRedundantIndexes(snapshot) {
|
|
330
|
+
const byTable = new Map();
|
|
331
|
+
for (const idx of snapshot.indexes) {
|
|
332
|
+
if (!idx.isValid)
|
|
333
|
+
continue;
|
|
334
|
+
let list = byTable.get(idx.table);
|
|
335
|
+
if (!list) {
|
|
336
|
+
list = [];
|
|
337
|
+
byTable.set(idx.table, list);
|
|
338
|
+
}
|
|
339
|
+
list.push(idx);
|
|
340
|
+
}
|
|
341
|
+
const out = [];
|
|
342
|
+
for (const list of byTable.values()) {
|
|
343
|
+
for (const narrow of list) {
|
|
344
|
+
// Candidate must be a plain, non-constraint index: dropping it must not
|
|
345
|
+
// remove a uniqueness/PK/exclusion guarantee or a replica identity.
|
|
346
|
+
if (isConstraintBacking(narrow))
|
|
347
|
+
continue;
|
|
348
|
+
if (narrow.columns.length === 0)
|
|
349
|
+
continue;
|
|
350
|
+
const wider = list.find((w) => w.indexName !== narrow.indexName && isLeadingPrefix(narrow.columns, w.columns));
|
|
351
|
+
if (!wider)
|
|
352
|
+
continue;
|
|
353
|
+
out.push({
|
|
354
|
+
table: narrow.table,
|
|
355
|
+
indexName: narrow.indexName,
|
|
356
|
+
columns: narrow.columns,
|
|
357
|
+
coveredBy: wider.indexName,
|
|
358
|
+
coveredByColumns: wider.columns,
|
|
359
|
+
sizeBytes: narrow.sizeBytes ?? null,
|
|
360
|
+
dropSql: buildDropIndexSql(narrow.indexName, { concurrently: true }),
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
return out.sort((a, b) => a.table.localeCompare(b.table) || a.indexName.localeCompare(b.indexName));
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* The unused-index machinery scoped to doctor's OWN previously-suggested indexes:
|
|
368
|
+
* existing indexes whose name matches the `idx_<table>_<cols>` (63-byte truncated)
|
|
369
|
+
* shape doctor emits, never scanned since the stats reset. `doctorNames` maps each
|
|
370
|
+
* deterministic name to the distinct column sets that truncate to it (see
|
|
371
|
+
* `collectDoctorProbeIndexNames`); a name with more than one column set is a
|
|
372
|
+
* post-truncation collision, reported as ambiguous.
|
|
373
|
+
*/
|
|
374
|
+
export function auditDoctorIndexes(snapshot, doctorNames, options = {}) {
|
|
375
|
+
const minScans = options.minScans ?? STATS_THRESHOLDS.unusedMinScans;
|
|
376
|
+
const out = [];
|
|
377
|
+
for (const idx of snapshot.indexes) {
|
|
378
|
+
if (!idx.isValid || isConstraintBacking(idx))
|
|
379
|
+
continue;
|
|
380
|
+
const candidates = doctorNames.get(idx.indexName);
|
|
381
|
+
if (!candidates || candidates.length === 0)
|
|
382
|
+
continue;
|
|
383
|
+
if (idx.idxScan === undefined || idx.idxScan >= minScans)
|
|
384
|
+
continue;
|
|
385
|
+
out.push({
|
|
386
|
+
table: idx.table,
|
|
387
|
+
indexName: idx.indexName,
|
|
388
|
+
columns: idx.columns,
|
|
389
|
+
idxScan: idx.idxScan,
|
|
390
|
+
sizeBytes: idx.sizeBytes ?? null,
|
|
391
|
+
dropSql: buildDropIndexSql(idx.indexName, { concurrently: true }),
|
|
392
|
+
ambiguous: candidates.length > 1,
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
return out.sort((a, b) => a.indexName.localeCompare(b.indexName));
|
|
396
|
+
}
|
|
258
397
|
/**
|
|
259
398
|
* Whether the snapshot is trustworthy enough to render tier verdicts. Empty,
|
|
260
399
|
* unavailable, or too-young stats degrade to the topology-only report.
|
|
@@ -356,8 +495,11 @@ export async function collectStatsSnapshot(options) {
|
|
|
356
495
|
const indexRows = await run('pg_index', `SELECT c.relname AS table_name,
|
|
357
496
|
ic.relname AS index_name,
|
|
358
497
|
i.indisvalid, i.indisunique, i.indisprimary, i.indisreplident,
|
|
498
|
+
EXISTS (SELECT 1 FROM pg_constraint con
|
|
499
|
+
WHERE con.conindid = i.indexrelid AND con.contype = 'x') AS is_exclusion,
|
|
359
500
|
s.idx_scan::text AS idx_scan,
|
|
360
|
-
(
|
|
501
|
+
pg_relation_size(i.indexrelid)::text AS index_size,
|
|
502
|
+
(SELECT array_agg(a.attname::text ORDER BY k.ord)
|
|
361
503
|
FROM unnest(i.indkey) WITH ORDINALITY AS k(attnum, ord)
|
|
362
504
|
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum) AS columns
|
|
363
505
|
FROM pg_index i
|
|
@@ -377,6 +519,8 @@ export async function collectStatsSnapshot(options) {
|
|
|
377
519
|
isUnique: row.indisunique,
|
|
378
520
|
isPrimary: row.indisprimary,
|
|
379
521
|
isReplicaIdent: row.indisreplident,
|
|
522
|
+
isExclusion: row.is_exclusion,
|
|
523
|
+
sizeBytes: row.index_size == null ? undefined : Number(row.index_size),
|
|
380
524
|
});
|
|
381
525
|
}
|
|
382
526
|
}
|
|
@@ -406,3 +550,71 @@ export async function collectStatsSnapshot(options) {
|
|
|
406
550
|
}
|
|
407
551
|
return snapshot;
|
|
408
552
|
}
|
|
553
|
+
/**
|
|
554
|
+
* Read per-model workload heat from a `_turbine_metrics` table (written by
|
|
555
|
+
* `db.$observe()` with the default Postgres sink). When the table is absent, the
|
|
556
|
+
* result is `available: false` with a notice: this is the expected state when
|
|
557
|
+
* observe uses a non-Postgres sink, or when no metrics have been collected yet.
|
|
558
|
+
* Every read is best-effort; a failure degrades to unavailable, never throws.
|
|
559
|
+
*/
|
|
560
|
+
export async function collectTableHeat(options) {
|
|
561
|
+
const windowMinutes = options.windowMinutes ?? 60;
|
|
562
|
+
const timeout = options.statementTimeoutMs ?? 5000;
|
|
563
|
+
const result = { available: false, tables: {}, notice: null };
|
|
564
|
+
if (options.models.length === 0) {
|
|
565
|
+
result.notice = 'no probed tables to correlate against workload heat.';
|
|
566
|
+
return result;
|
|
567
|
+
}
|
|
568
|
+
const { Pool } = (await import('pg')).default;
|
|
569
|
+
const pool = new Pool({ connectionString: options.connectionString, max: 1 });
|
|
570
|
+
try {
|
|
571
|
+
try {
|
|
572
|
+
await pool.query(`SET statement_timeout = ${Number(timeout)}`);
|
|
573
|
+
}
|
|
574
|
+
catch {
|
|
575
|
+
/* best-effort */
|
|
576
|
+
}
|
|
577
|
+
const exists = await pool
|
|
578
|
+
.query(`SELECT to_regclass('_turbine_metrics')::text AS reg`)
|
|
579
|
+
.then((r) => r.rows[0]?.reg ?? null)
|
|
580
|
+
.catch(() => null);
|
|
581
|
+
if (!exists) {
|
|
582
|
+
result.notice =
|
|
583
|
+
'the _turbine_metrics table was not found: heat boosting is unavailable (observe may be using a non-Postgres sink, or no metrics have been collected yet).';
|
|
584
|
+
return result;
|
|
585
|
+
}
|
|
586
|
+
const rows = await pool
|
|
587
|
+
.query(`SELECT model,
|
|
588
|
+
sum(count)::float8::text AS total_count,
|
|
589
|
+
max(p95_ms)::text AS p95,
|
|
590
|
+
count(*)::text AS buckets
|
|
591
|
+
FROM _turbine_metrics
|
|
592
|
+
WHERE bucket >= NOW() - (INTERVAL '1 minute' * $1) AND model = ANY($2)
|
|
593
|
+
GROUP BY model`, [windowMinutes, options.models])
|
|
594
|
+
.then((r) => r.rows)
|
|
595
|
+
.catch(() => null);
|
|
596
|
+
if (rows === null) {
|
|
597
|
+
result.notice = 'reading _turbine_metrics failed: heat boosting is unavailable.';
|
|
598
|
+
return result;
|
|
599
|
+
}
|
|
600
|
+
result.available = true;
|
|
601
|
+
for (const row of rows) {
|
|
602
|
+
const total = Number(row.total_count);
|
|
603
|
+
if (!Number.isFinite(total))
|
|
604
|
+
continue;
|
|
605
|
+
result.tables[row.model] = {
|
|
606
|
+
queriesPerMin: total / Math.max(1, windowMinutes),
|
|
607
|
+
p95Ms: row.p95 == null ? 0 : Number(row.p95),
|
|
608
|
+
};
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
finally {
|
|
612
|
+
try {
|
|
613
|
+
await pool.end();
|
|
614
|
+
}
|
|
615
|
+
catch {
|
|
616
|
+
/* best-effort */
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
return result;
|
|
620
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -41,7 +41,7 @@ export { CheckConstraintError, CircularRelationError, ConnectionError, DeadlockE
|
|
|
41
41
|
export { type GenerateOptions, generate } from './generate.js';
|
|
42
42
|
export { type IntrospectOptions, introspect } from './introspect.js';
|
|
43
43
|
export { executeNestedCreate, executeNestedUpdate, hasRelationFields, type NestedWriteContext, } from './nested-write.js';
|
|
44
|
-
export type
|
|
44
|
+
export { HttpJsonSink, type HttpJsonSinkOptions, type MetricsFlushBatch, type MetricsFlushRow, type ObserveConfig, type ObserveHandle, type ObserveSink, PgMetricsSink, type PgMetricsSinkOptions, } from './observe.js';
|
|
45
45
|
export { executePipeline, type PipelineOptions, type PipelineResults, pipelineSupported } from './pipeline.js';
|
|
46
46
|
export { type AggregateArgs, type AggregateResult, type ArrayFilter, type ColumnRef, type ConnectOrCreateOp, type CountArgs, type CreateArgs, type CreateDataInput, type CreateManyArgs, type DeferredQuery, type DeleteArgs, type DeleteManyArgs, type FieldResult, type FindManyArgs, type FindManyStreamArgs, type FindUniqueArgs, type GlobalFilters, type GroupByAggregateSpec, type GroupByArgs, type GroupByDistinctOn, type GroupByResult, type HavingClause, type JsonFilter, type JsonPathAggregateTarget, type JsonPathGroupKey, type JsonPathOrderBy, type MiddlewareFn, type NestedCreateOp, type NestedUpdateOp, type NestedUpdateOpItem, type NestedUpsertOpItem, type OmitResult, type OrderByClause, type OrderByObject, type OrderDirection, type QueryEvent, type QueryEventListener, QueryInterface, type QueryResult, type RelationDescriptor, type RelationFilter, type RelationLoadStrategy, type RelationPickBy, type RelationPickOrderBy, type SelectResult, type SkipGlobalFilters, type TextSearchFilter, type TypedWithClause, type UpdateArgs, type UpdateDataInput, type UpdateInput, type UpdateManyArgs, type UpdateOperatorInput, type UpsertArgs, type VectorDistanceFilter, type VectorFilter, type VectorMetric, type VectorOrderBy, type VectorOrderByDistance, type WhereClause, type WhereOperator, type WhereValue, type WithClause, type WithOptions, type WithOrderByObject, type WithResult, } from './query/index.js';
|
|
47
47
|
export { type ActiveSubscription, type NotificationHandler, type Subscription, validateChannel } from './realtime.js';
|
package/dist/index.js
CHANGED
|
@@ -44,6 +44,8 @@ export { generate } from './generate.js';
|
|
|
44
44
|
export { introspect } from './introspect.js';
|
|
45
45
|
// Nested writes
|
|
46
46
|
export { executeNestedCreate, executeNestedUpdate, hasRelationFields, } from './nested-write.js';
|
|
47
|
+
// Observability
|
|
48
|
+
export { HttpJsonSink, PgMetricsSink, } from './observe.js';
|
|
47
49
|
// Pipeline
|
|
48
50
|
export { executePipeline, pipelineSupported } from './pipeline.js';
|
|
49
51
|
// Query builder
|
package/dist/observe.d.ts
CHANGED
|
@@ -1,28 +1,109 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* turbine-orm
|
|
2
|
+
* turbine-orm: Observability module
|
|
3
3
|
*
|
|
4
4
|
* Buffers query metrics in memory (keyed by model:action per minute bucket),
|
|
5
|
-
* then periodically flushes aggregates (count, avg, p50, p95, p99, errors)
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* then periodically flushes aggregates (count, avg, p50, p95, p99, errors) to a
|
|
6
|
+
* pluggable {@link ObserveSink}. The default sink writes to a dedicated
|
|
7
|
+
* `_turbine_metrics` Postgres table over its own 1-connection pool, so metrics
|
|
8
|
+
* writes never contend with the application pool; alternative sinks (for example
|
|
9
|
+
* {@link HttpJsonSink}) can forward the same aggregates elsewhere.
|
|
10
|
+
*
|
|
11
|
+
* The aggregation privacy posture is deliberate: a batch carries only the
|
|
12
|
+
* model/action identity, the counts, and the latency percentiles. It never
|
|
13
|
+
* carries SQL text or bound parameter values.
|
|
8
14
|
*/
|
|
9
15
|
import type { QueryEventListener } from './query/index.js';
|
|
10
16
|
export interface ObserveConfig {
|
|
11
|
-
|
|
17
|
+
/**
|
|
18
|
+
* Metrics database connection string for the default Postgres sink. Optional
|
|
19
|
+
* when a {@link ObserveConfig.sink} is supplied; at least one of the two must
|
|
20
|
+
* be present.
|
|
21
|
+
*/
|
|
22
|
+
connectionString?: string;
|
|
12
23
|
flushIntervalMs?: number;
|
|
13
24
|
retentionDays?: number;
|
|
25
|
+
/**
|
|
26
|
+
* A custom flush target. When omitted, the engine writes to `_turbine_metrics`
|
|
27
|
+
* via the default Postgres sink (byte-identical to the pre-sink writer).
|
|
28
|
+
*/
|
|
29
|
+
sink?: ObserveSink;
|
|
14
30
|
}
|
|
15
31
|
export interface ObserveHandle {
|
|
16
32
|
stop(): Promise<void>;
|
|
17
33
|
}
|
|
34
|
+
/** One per-bucket aggregate row. Identity + numbers only: no SQL, no params. */
|
|
35
|
+
export interface MetricsFlushRow {
|
|
36
|
+
/** Minute bucket the aggregate belongs to. */
|
|
37
|
+
bucket: Date;
|
|
38
|
+
model: string;
|
|
39
|
+
action: string;
|
|
40
|
+
count: number;
|
|
41
|
+
avg: number;
|
|
42
|
+
p50: number;
|
|
43
|
+
p95: number;
|
|
44
|
+
p99: number;
|
|
45
|
+
errors: number;
|
|
46
|
+
}
|
|
47
|
+
/** A batch of aggregate rows handed to a sink on each flush. */
|
|
48
|
+
export interface MetricsFlushBatch {
|
|
49
|
+
rows: MetricsFlushRow[];
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* A pluggable flush target for the observe engine. `init` runs once at startup
|
|
53
|
+
* (create tables, open connections); `flush` receives each aggregate batch;
|
|
54
|
+
* `stop` tears down on shutdown. A sink must never let a flush error escape into
|
|
55
|
+
* the application: metrics are best-effort by contract.
|
|
56
|
+
*/
|
|
57
|
+
export interface ObserveSink {
|
|
58
|
+
init?(): Promise<void>;
|
|
59
|
+
flush(batch: MetricsFlushBatch): Promise<void>;
|
|
60
|
+
stop?(): Promise<void>;
|
|
61
|
+
}
|
|
18
62
|
declare function floorToMinute(date: Date): Date;
|
|
19
63
|
declare function percentile(sorted: number[], p: number): number;
|
|
20
|
-
export
|
|
64
|
+
export interface PgMetricsSinkOptions {
|
|
65
|
+
connectionString: string;
|
|
66
|
+
retentionDays?: number;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* The default flush target: upserts each aggregate row into `_turbine_metrics`
|
|
70
|
+
* and prunes rows older than `retentionDays`. The SQL and per-row/retention
|
|
71
|
+
* ordering are byte-identical to the pre-sink `ObserveEngine.flush` writer.
|
|
72
|
+
*/
|
|
73
|
+
export declare class PgMetricsSink implements ObserveSink {
|
|
21
74
|
private readonly pool;
|
|
75
|
+
private readonly retentionDays;
|
|
76
|
+
constructor(options: PgMetricsSinkOptions);
|
|
77
|
+
init(): Promise<void>;
|
|
78
|
+
flush(batch: MetricsFlushBatch): Promise<void>;
|
|
79
|
+
stop(): Promise<void>;
|
|
80
|
+
}
|
|
81
|
+
export interface HttpJsonSinkOptions {
|
|
82
|
+
/** Endpoint that receives POSTed JSON batches. */
|
|
83
|
+
url: string;
|
|
84
|
+
/** Extra request headers (for example an auth token). */
|
|
85
|
+
headers?: Record<string, string>;
|
|
86
|
+
/** Override the fetch implementation (defaults to the global `fetch`). */
|
|
87
|
+
fetchFunction?: typeof fetch;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Forwards each aggregate batch to an HTTP endpoint as a JSON POST. Fire-and-
|
|
91
|
+
* forget: a failed request is swallowed and never throws, and there are no
|
|
92
|
+
* retries beyond the engine's next scheduled flush. Aggregates only: the body
|
|
93
|
+
* carries no SQL text and no parameter values.
|
|
94
|
+
*/
|
|
95
|
+
export declare class HttpJsonSink implements ObserveSink {
|
|
96
|
+
private readonly url;
|
|
97
|
+
private readonly headers;
|
|
98
|
+
private readonly fetchFn;
|
|
99
|
+
constructor(options: HttpJsonSinkOptions);
|
|
100
|
+
flush(batch: MetricsFlushBatch): Promise<void>;
|
|
101
|
+
}
|
|
102
|
+
export declare class ObserveEngine {
|
|
103
|
+
private readonly sink;
|
|
22
104
|
private readonly buffer;
|
|
23
105
|
private currentBucket;
|
|
24
106
|
private readonly flushIntervalMs;
|
|
25
|
-
private readonly retentionDays;
|
|
26
107
|
private timer;
|
|
27
108
|
private readonly listener;
|
|
28
109
|
private stopped;
|