postgresai 0.15.0 → 0.16.0-dev.0

package/lib/checkup.ts

@@ -243,6 +243,58 @@ export interface RedundantIndex {
   redundant_to_parse_error?: string;
 }
 
+/**
+ * Dead tuples table entry (F003) - matches F003.schema.json deadTuplesTable
+ *
+ * Sourced from pg_stat_user_tables live counters (n_dead_tup / n_live_tup),
+ * so dead tuples that have never been vacuumed are visible - unlike the
+ * statistical bloat estimators (F004/F005), which miss them entirely.
+ */
+export interface DeadTuplesTable {
+  schema_name: string;
+  table_name: string;
+  n_live_tup: number;
+  n_dead_tup: number;
+  /** Dead tuples as percentage of all tuples: n_dead_tup / (n_live_tup + n_dead_tup) * 100 */
+  dead_pct: number;
+  last_autovacuum: string | null;
+  /** Epoch seconds of the last autovacuum; 0 = never */
+  last_autovacuum_epoch: number;
+  last_vacuum: string | null;
+  /** Epoch seconds of the last manual vacuum; 0 = never */
+  last_vacuum_epoch: number;
+  autovacuum_count: number;
+  vacuum_count: number;
+  /** True when autovacuum is disabled per-table via reloptions (autovacuum_enabled=off/false/0/...) */
+  autovacuum_disabled: boolean;
+  table_size_bytes: number;
+  table_size_pretty: string;
+  /** True when BOTH F003_DEAD_TUPLES_MIN and F003_DEAD_PCT_MIN thresholds are exceeded */
+  exceeds_dead_tuple_thresholds: boolean;
+  /** True when autovacuum is disabled per-table on a non-tiny table (>= F003_AUTOVACUUM_DISABLED_MIN_ROWS tuples) */
+  autovacuum_disabled_flagged: boolean;
+}
+
+/**
+ * F003 thresholds.
+ *
+ * A table's dead-tuple accumulation is flagged only when it is high in BOTH
+ * absolute and relative terms:
+ * - F003_DEAD_TUPLES_MIN keeps small/noisy tables out (100k dead tuples is
+ *   real work for vacuum regardless of table size);
+ * - F003_DEAD_PCT_MIN = 20 mirrors the default autovacuum_vacuum_scale_factor
+ *   of 0.2: with default settings autovacuum should have fired well before a
+ *   table is 20% dead, so reaching this level in a snapshot is an unambiguous
+ *   signal that vacuum is not keeping up (lagging, blocked, or disabled).
+ *
+ * Per-table disabled autovacuum is a classic footgun and is always flagged on
+ * non-tiny tables (>= F003_AUTOVACUUM_DISABLED_MIN_ROWS total tuples; same
+ * 10k-row "non-tiny" cutoff the classic postgres-checkup F003 uses).
+ */
+export const F003_DEAD_TUPLES_MIN = 100_000;
+export const F003_DEAD_PCT_MIN = 20;
+export const F003_AUTOVACUUM_DISABLED_MIN_ROWS = 10_000;
+
 /**
  * I/O statistics by backend type (I001) - matches I001.schema.json backendIOStats
  */
@@ -257,7 +309,7 @@ export interface BackendIOStats {
   write_bytes_mb: number;
   write_time_ms: number;
   writebacks: number;
-  /** Writeback MiB. The historical `_mb` suffix is retained for schema compatibility. */
+  /** Writeback MiB. Always 0 on PG18+ (op_bytes removed, no writeback byte counts exposed). The historical `_mb` suffix is retained for schema compatibility. */
   writeback_bytes_mb: number;
   writeback_time_ms: number;
   fsyncs: number;
@@ -361,11 +413,11 @@ function formatSettingPrettyValue(
   if (settingNormalized === null || unitNormalized === null) {
     return rawValue;
   }
-  
+
   if (unitNormalized === "bytes") {
     return formatBytes(settingNormalized);
   }
-  
+
   if (unitNormalized === "seconds") {
     // Format time values with appropriate units based on magnitude:
     // - Sub-second values (< 1s): show in milliseconds for precision
@@ -380,7 +432,7 @@ function formatSettingPrettyValue(
       return `${(settingNormalized / SECONDS_PER_MINUTE).toFixed(1)} min`;
     }
   }
-  
+
   return rawValue;
 }
 
@@ -436,7 +488,7 @@ export async function getSettings(client: Client, pgMajorVersion: number = 16):
     const vartype = row.tag_vartype || "";
     const settingNormalized = row.setting_normalized !== null ? parseFloat(row.setting_normalized) : null;
     const unitNormalized = row.unit_normalized || null;
-    
+
     settings[name] = {
       setting: settingValue,
       unit,
@@ -468,7 +520,7 @@ export async function getAlteredSettings(client: Client, pgMajorVersion: number
       const category = row.tag_category || "";
       const settingNormalized = row.setting_normalized !== null ? parseFloat(row.setting_normalized) : null;
       const unitNormalized = row.unit_normalized || null;
-      
+
       settings[name] = {
         value: settingValue,
         unit,
@@ -737,22 +789,22 @@ export async function getStatsReset(client: Client, pgMajorVersion: number = 16)
   const sql = getMetricSql(METRIC_NAMES.statsReset, pgMajorVersion);
   const result = await client.query(sql);
   const row = result.rows[0] || {};
-  
+
   // The stats_reset metric returns stats_reset_epoch and seconds_since_reset
   // We need to calculate additional fields
   const statsResetEpoch = row.stats_reset_epoch ? parseFloat(row.stats_reset_epoch) : null;
   const secondsSinceReset = row.seconds_since_reset ? parseInt(row.seconds_since_reset, 10) : null;
-  
+
   // Calculate stats_reset_time from epoch
-  const statsResetTime = statsResetEpoch 
+  const statsResetTime = statsResetEpoch
     ? new Date(statsResetEpoch * 1000).toISOString()
     : null;
-  
+
   // Calculate days since reset
   const daysSinceReset = secondsSinceReset !== null
     ? Math.floor(secondsSinceReset / SECONDS_PER_DAY)
     : null;
-  
+
   // Get postmaster startup time separately (simple inline SQL)
   // This is supplementary data - errors are captured in output, not propagated
   let postmasterStartupEpoch: number | null = null;
@@ -765,8 +817,8 @@ export async function getStatsReset(client: Client, pgMajorVersion: number = 16)
         pg_postmaster_start_time()::text as postmaster_startup_time
     `);
     if (pmResult.rows.length > 0) {
-      postmasterStartupEpoch = pmResult.rows[0].postmaster_startup_epoch 
-        ? parseFloat(pmResult.rows[0].postmaster_startup_epoch) 
+      postmasterStartupEpoch = pmResult.rows[0].postmaster_startup_epoch
+        ? parseFloat(pmResult.rows[0].postmaster_startup_epoch)
         : null;
       postmasterStartupTime = pmResult.rows[0].postmaster_startup_time || null;
     }
@@ -775,7 +827,7 @@ export async function getStatsReset(client: Client, pgMajorVersion: number = 16)
     postmasterStartupError = `Failed to query postmaster start time: ${errorMsg}`;
     console.error(`[getStatsReset] Warning: ${postmasterStartupError}`);
   }
-  
+
   const statsResult: StatsReset = {
     stats_reset_epoch: statsResetEpoch,
     stats_reset_time: statsResetTime,
@@ -783,12 +835,12 @@ export async function getStatsReset(client: Client, pgMajorVersion: number = 16)
     postmaster_startup_epoch: postmasterStartupEpoch,
     postmaster_startup_time: postmasterStartupTime,
   };
-  
+
   // Only include error field if there was an error (keeps output clean)
   if (postmasterStartupError) {
     statsResult.postmaster_startup_error = postmasterStartupError;
   }
-  
+
   return statsResult;
 }
 
@@ -800,7 +852,7 @@ export async function getCurrentDatabaseInfo(client: Client, pgMajorVersion: num
   const sql = getMetricSql(METRIC_NAMES.dbSize, pgMajorVersion);
   const result = await client.query(sql);
   const row = result.rows[0] || {};
-  
+
   // db_size metric returns tag_datname and size_b
   return {
     datname: row.tag_datname || "postgres",
@@ -831,7 +883,7 @@ export async function getRedundantIndexes(client: Client, pgMajorVersion: number
     const transformed = transformMetricRow(row);
     const indexSizeBytes = parseInt(String(transformed.index_size_bytes || 0), 10);
     const tableSizeBytes = parseInt(String(transformed.table_size_bytes || 0), 10);
-    
+
     // Parse redundant_to JSON array (indexes that make this one redundant)
     let redundantTo: RedundantToIndex[] = [];
     let parseError: string | undefined;
@@ -857,7 +909,7 @@ export async function getRedundantIndexes(client: Client, pgMajorVersion: number
       parseError = `Failed to parse redundant_to_json: ${errorMsg}`;
       console.error(`[H004] Warning: ${parseError} for index "${indexName}"`);
     }
-    
+
     const result: RedundantIndex = {
       schema_name: String(transformed.schema_name || ""),
       table_name: String(transformed.table_name || ""),
@@ -874,16 +926,117 @@ export async function getRedundantIndexes(client: Client, pgMajorVersion: number
       table_size_pretty: formatBytes(tableSizeBytes),
       redundant_to: redundantTo,
     };
-    
+
     // Only include parse error field if there was an error (keeps output clean)
     if (parseError) {
       result.redundant_to_parse_error = parseError;
     }
-    
+
     return result;
   });
 }
 
+/**
+ * Get per-table dead-tuple stats and per-table autovacuum overrides (F003).
+ * SQL loaded from config/pgwatch-prometheus/metrics.yml (pg_dead_tuples metric).
+ *
+ * Returns tables that carry dead tuples or have autovacuum disabled per-table,
+ * with threshold flags precomputed (see F003_* constants).
+ *
+ * @param client - Connected PostgreSQL client
+ * @param pgMajorVersion - PostgreSQL major version (default: 16)
+ * @throws {Error} If database query fails (propagating - critical data)
+ */
+export async function getDeadTuples(client: Client, pgMajorVersion: number = 16): Promise<DeadTuplesTable[]> {
+  const sql = getMetricSql(METRIC_NAMES.F003, pgMajorVersion);
+  const result = await client.query(sql);
+  return result.rows.map((row) => {
+    const t = transformMetricRow(row);
+    const nLive = parseInt(String(t.n_live_tup || 0), 10);
+    const nDead = parseInt(String(t.n_dead_tup || 0), 10);
+    const deadPct = parseFloat(String(t.dead_pct)) || 0;
+    const lastAutovacuumEpoch = parseInt(String(t.last_autovacuum || 0), 10);
+    const lastVacuumEpoch = parseInt(String(t.last_vacuum || 0), 10);
+    // The metric emits 0/1; be liberal in what we accept (driver may return strings)
+    const autovacuumDisabled = parseInt(String(t.autovacuum_disabled || 0), 10) === 1 || toBool(t.autovacuum_disabled);
+    const tableSizeBytes = parseInt(String(t.table_size_b || 0), 10);
+
+    return {
+      schema_name: String(t.schemaname || ""),
+      table_name: String(t.relname || ""),
+      n_live_tup: nLive,
+      n_dead_tup: nDead,
+      dead_pct: deadPct,
+      last_autovacuum: lastAutovacuumEpoch > 0 ? new Date(lastAutovacuumEpoch * 1000).toISOString() : null,
+      last_autovacuum_epoch: lastAutovacuumEpoch,
+      last_vacuum: lastVacuumEpoch > 0 ? new Date(lastVacuumEpoch * 1000).toISOString() : null,
+      last_vacuum_epoch: lastVacuumEpoch,
+      autovacuum_count: parseInt(String(t.autovacuum_count || 0), 10),
+      vacuum_count: parseInt(String(t.vacuum_count || 0), 10),
+      autovacuum_disabled: autovacuumDisabled,
+      table_size_bytes: tableSizeBytes,
+      table_size_pretty: formatBytes(tableSizeBytes),
+      exceeds_dead_tuple_thresholds: nDead >= F003_DEAD_TUPLES_MIN && deadPct >= F003_DEAD_PCT_MIN,
+      autovacuum_disabled_flagged: autovacuumDisabled && nLive + nDead >= F003_AUTOVACUUM_DISABLED_MIN_ROWS,
+    };
+  });
+}
+
+/**
+ * Build concrete, human-readable conclusions and recommendations for F003.
+ *
+ * Exported separately so the wording (which the console surfaces verbatim in
+ * auto-created issues) can be unit-tested without a database.
+ */
+export function buildDeadTuplesConclusions(tables: DeadTuplesTable[]): {
+  conclusions: string[];
+  recommendations: string[];
+} {
+  const conclusions: string[] = [];
+  const recommendations: string[] = [];
+
+  const fmt = (n: number) => n.toLocaleString("en-US");
+
+  for (const t of tables) {
+    const rel = `"${t.schema_name}"."${t.table_name}"`;
+    const lastAv = t.last_autovacuum
+      ? `last autovacuum: ${t.last_autovacuum}`
+      : "autovacuum has never vacuumed it";
+
+    if (t.exceeds_dead_tuple_thresholds && t.autovacuum_disabled) {
+      conclusions.push(
+        `Table ${rel} has ${fmt(t.n_dead_tup)} dead tuples (${t.dead_pct}% of all tuples) ` +
+        `and autovacuum is disabled on it via reloptions (${lastAv}).`
+      );
+      recommendations.push(
+        `Re-enable autovacuum on ${rel}: alter table ${rel} reset (autovacuum_enabled); ` +
+        `then run: vacuum (analyze) ${rel}; to clean up the accumulated dead tuples.`
+      );
+    } else if (t.exceeds_dead_tuple_thresholds) {
+      conclusions.push(
+        `Table ${rel} has ${fmt(t.n_dead_tup)} dead tuples (${t.dead_pct}% of all tuples; ${lastAv}).`
+      );
+      recommendations.push(
+        `Run: vacuum (analyze) ${rel}; and review autovacuum settings ` +
+        `(autovacuum_vacuum_scale_factor, autovacuum_vacuum_cost_delay, autovacuum_max_workers) ` +
+        `if dead tuples keep accumulating on ${rel}.`
+      );
+    } else if (t.autovacuum_disabled_flagged) {
+      conclusions.push(
+        `Autovacuum is disabled via reloptions on table ${rel} ` +
+        `(~${fmt(t.n_live_tup + t.n_dead_tup)} tuples); dead tuples and transaction ID age ` +
+        `will accumulate unchecked.`
+      );
+      recommendations.push(
+        `Re-enable autovacuum on ${rel}: alter table ${rel} reset (autovacuum_enabled); ` +
+        `unless this table is managed by a carefully scheduled manual vacuum job.`
+      );
+    }
+  }
+
+  return { conclusions, recommendations };
+}
+
 /**
  * Create base report structure
  */
@@ -1286,6 +1439,58 @@ async function generateF001(client: Client, nodeName: string): Promise<Report> {
   return report;
 }
 
+/**
+ * Generate F003 report - Autovacuum: dead tuples
+ *
+ * Reads per-table dead-tuple counters from pg_stat_user_tables and per-table
+ * autovacuum overrides from pg_class.reloptions. Flags tables where dead
+ * tuples are high both in absolute terms and relative to live tuples, and
+ * tables where autovacuum is disabled per-table (a classic footgun).
+ *
+ * Unlike F004/F005 (statistical bloat estimators), this check sees dead
+ * tuples that have never been vacuumed.
+ * SQL loaded from config/pgwatch-prometheus/metrics.yml (pg_dead_tuples metric).
+ */
+async function generateF003(client: Client, nodeName: string): Promise<Report> {
+  const report = createBaseReport("F003", "Autovacuum: dead tuples", nodeName);
+  const postgresVersion = await getPostgresVersion(client);
+  const pgMajorVersion = parseInt(postgresVersion.server_major_ver, 10) || 16;
+
+  const tables = await getDeadTuples(client, pgMajorVersion);
+  const { datname: dbName, size_bytes: dbSizeBytes } = await getCurrentDatabaseInfo(client, pgMajorVersion);
+
+  const flaggedCount = tables.filter((t) => t.exceeds_dead_tuple_thresholds).length;
+  const autovacuumDisabledCount = tables.filter((t) => t.autovacuum_disabled).length;
+  const autovacuumDisabledFlaggedCount = tables.filter((t) => t.autovacuum_disabled_flagged).length;
+  const totalDeadTuples = tables.reduce((sum, t) => sum + t.n_dead_tup, 0);
+  const { conclusions, recommendations } = buildDeadTuplesConclusions(tables);
+
+  const dbEntry = {
+    dead_tuples_tables: tables,
+    total_count: tables.length,
+    flagged_count: flaggedCount,
+    autovacuum_disabled_count: autovacuumDisabledCount,
+    autovacuum_disabled_flagged_count: autovacuumDisabledFlaggedCount,
+    total_dead_tuples: totalDeadTuples,
+    thresholds: {
+      dead_tuples_min: F003_DEAD_TUPLES_MIN,
+      dead_pct_min: F003_DEAD_PCT_MIN,
+      autovacuum_disabled_min_rows: F003_AUTOVACUUM_DISABLED_MIN_ROWS,
+    },
+    conclusions,
+    recommendations,
+    database_size_bytes: dbSizeBytes,
+    database_size_pretty: formatBytes(dbSizeBytes),
+  };
+
+  report.results[nodeName] = {
+    data: { [dbName]: dbEntry },
+    postgres_version: postgresVersion,
+  };
+
+  return report;
+}
+
 /**
  * Generate F004 report - Autovacuum: heap bloat (estimated)
  *
@@ -1900,6 +2105,7 @@ export const REPORT_GENERATORS: Record<string, (client: Client, nodeName: string
   D001: generateD001,
   D004: generateD004,
   F001: generateF001,
+  F003: generateF003,
   F004: generateF004,
   F005: generateF005,
   G001: generateG001,

package/lib/metrics-loader.ts

@@ -17,25 +17,25 @@ import { METRICS, MetricDefinition } from "./metrics-embedded";
  */
 export function getMetricSql(metricName: string, pgMajorVersion: number = 16): string {
   const metric = METRICS[metricName];
-  
+
   if (!metric) {
     throw new Error(`Metric "${metricName}" not found. Available metrics: ${Object.keys(METRICS).join(", ")}`);
   }
-  
+
   // Find the best matching version: highest version <= pgMajorVersion
   const availableVersions = Object.keys(metric.sqls)
     .map(v => parseInt(v, 10))
     .sort((a, b) => b - a); // Sort descending
-  
+
   const matchingVersion = availableVersions.find(v => v <= pgMajorVersion);
-  
+
   if (matchingVersion === undefined) {
     throw new Error(
       `No compatible SQL version for metric "${metricName}" with PostgreSQL ${pgMajorVersion}. ` +
       `Available versions: ${availableVersions.join(", ")}`
     );
   }
-  
+
   return metric.sqls[matchingVersion];
 }
 
@@ -65,6 +65,8 @@ export const METRIC_NAMES = {
   H001: "pg_invalid_indexes",
   H002: "unused_indexes",
   H004: "redundant_indexes",
+  // Dead tuples and per-table autovacuum overrides
+  F003: "pg_dead_tuples",
   // Bloat estimation
   F004: "pg_table_bloat",
   F005: "pg_btree_bloat",
@@ -86,18 +88,18 @@ export const METRIC_NAMES = {
  */
 export function transformMetricRow(row: Record<string, unknown>): Record<string, unknown> {
   const result: Record<string, unknown> = {};
-  
+
   for (const [key, value] of Object.entries(row)) {
     // Skip Prometheus-specific fields
     if (key === "epoch_ns" || key === "num" || key === "tag_datname") {
       continue;
     }
-    
+
     // Strip tag_ prefix
     const newKey = key.startsWith("tag_") ? key.slice(4) : key;
     result[newKey] = value;
   }
-  
+
   return result;
 }
 

package/lib/util.ts

@@ -22,19 +22,27 @@ function isHtmlContent(text: string): boolean {
   return trimmed.startsWith("<!DOCTYPE") || trimmed.startsWith("<html") || trimmed.startsWith("<HTML");
 }
 
+/**
+ * Remediation hint appended to 401 errors so both humans and AI agents
+ * (MCP tool callers) know how to recover from an invalid/stale API key.
+ */
+const AUTH_REMEDIATION_HINT = "Run 'postgresai auth' to (re)authenticate, or set/update PGAI_API_KEY.";
+
 /**
  * Format an HTTP error response into a clean, developer-friendly message.
  * Handles HTML error pages (e.g., from Cloudflare) by showing just the status code and message.
+ * For 401 responses, appends a remediation hint pointing at `postgresai auth`.
  */
 export function formatHttpError(operation: string, status: number, responseBody?: string): string {
   const statusMessage = HTTP_STATUS_MESSAGES[status] || "Request failed";
   let errMsg = `${operation}: HTTP ${status} - ${statusMessage}`;
+  const remediation = status === 401 ? `\n${AUTH_REMEDIATION_HINT}` : "";
 
   if (responseBody) {
     // If it's HTML (like Cloudflare error pages), don't dump the raw HTML
     if (isHtmlContent(responseBody)) {
       // Just use the status message, don't append HTML
-      return errMsg;
+      return errMsg + remediation;
     }
 
     // Try to parse as JSON for structured error info
@@ -56,7 +64,7 @@ export function formatHttpError(operation: string, status: number, responseBody?
     }
   }
 
-  return errMsg;
+  return errMsg + remediation;
 }
 
 export function maskSecret(secret: string): string {
@@ -124,4 +132,3 @@ export function resolveBaseUrls(
     storageBaseUrl: normalizeBaseUrl(storageCandidate),
   };
 }
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "postgresai",
-  "version": "0.15.0",
+  "version": "0.16.0-dev.0",
   "description": "postgres_ai CLI",
   "license": "Apache-2.0",
   "private": false,

package/scripts/embed-metrics.ts

@@ -1,12 +1,12 @@
 #!/usr/bin/env bun
 /**
  * Build script to embed metrics.yml into the CLI bundle.
- * 
+ *
  * This script reads config/pgwatch-prometheus/metrics.yml and generates
  * cli/lib/metrics-embedded.ts with the metrics data embedded as TypeScript.
- * 
+ *
  * The generated file is NOT committed to git - it's regenerated at build time.
- * 
+ *
  * Usage: bun run scripts/embed-metrics.ts
  */
 
@@ -46,6 +46,8 @@ const REQUIRED_METRICS = [
   "redundant_indexes",
   // Stats reset info (H002)
   "stats_reset",
+  // Dead tuples and per-table autovacuum overrides (F003)
+  "pg_dead_tuples",
   // Bloat estimation (F004, F005)
   "pg_table_bloat",
   "pg_btree_bloat",
@@ -55,7 +57,7 @@ const REQUIRED_METRICS = [
 
 function main() {
   console.log(`Reading metrics from: ${METRICS_YML_PATH}`);
-  
+
   if (!fs.existsSync(METRICS_YML_PATH)) {
     console.error(`ERROR: metrics.yml not found at ${METRICS_YML_PATH}`);
     process.exit(1);
@@ -120,7 +122,7 @@ function generateTypeScript(metrics: Record<string, MetricDefinition>): string {
 
   for (const [name, metric] of Object.entries(metrics)) {
     lines.push(`  ${JSON.stringify(name)}: {`);
-    
+
     if (metric.description) {
       // Escape description for TypeScript string
       const desc = metric.description.trim().replace(/\n/g, " ").replace(/\s+/g, " ");
@@ -156,4 +158,3 @@ function generateTypeScript(metrics: Record<string, MetricDefinition>): string {
 }
 
 main();
-