postgresai 0.14.0-beta.12 → 0.14.0-beta.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "postgresai",
-  "version": "0.14.0-beta.12",
+  "version": "0.14.0-beta.14",
   "description": "postgres_ai CLI",
   "license": "Apache-2.0",
   "private": false,

package/sql/02.extensions.sql

@@ -0,0 +1,8 @@
+-- Extensions required for postgres_ai monitoring
+
+-- Enable pg_stat_statements for query performance monitoring
+-- Note: Uses IF NOT EXISTS because extension may already be installed.
+-- We do NOT drop this extension in unprepare-db since it may have been pre-existing.
+create extension if not exists pg_stat_statements;
+
+

package/sql/{02.permissions.sql → 03.permissions.sql}

@@ -8,6 +8,7 @@ grant pg_monitor to {{ROLE_IDENT}};
 grant select on pg_catalog.pg_index to {{ROLE_IDENT}};
 
 -- Create postgres_ai schema for our objects
+-- Using IF NOT EXISTS for idempotency - prepare-db can be run multiple times
 create schema if not exists postgres_ai;
 grant usage on schema postgres_ai to {{ROLE_IDENT}};
 

package/sql/uninit/01.helpers.sql

@@ -0,0 +1,5 @@
+-- Drop helper functions created by prepare-db (template-filled by cli/lib/init.ts)
+-- Run before dropping the postgres_ai schema.
+
+drop function if exists postgres_ai.explain_generic(text, text, text);
+drop function if exists postgres_ai.table_describe(text);

package/sql/uninit/02.permissions.sql

@@ -0,0 +1,30 @@
+-- Revoke permissions and drop objects created by prepare-db (template-filled by cli/lib/init.ts)
+
+-- Drop the postgres_ai.pg_statistic view
+drop view if exists postgres_ai.pg_statistic;
+
+-- Drop the postgres_ai schema (CASCADE to handle any remaining objects)
+drop schema if exists postgres_ai cascade;
+
+-- Revoke permissions from the monitoring role
+-- Use a DO block to handle the case where the role doesn't exist
+do $$ begin
+  revoke pg_monitor from {{ROLE_IDENT}};
+exception when undefined_object then
+  null; -- Role doesn't exist, nothing to revoke
+end $$;
+
+do $$ begin
+  revoke select on pg_catalog.pg_index from {{ROLE_IDENT}};
+exception when undefined_object then
+  null; -- Role doesn't exist
+end $$;
+
+do $$ begin
+  revoke connect on database {{DB_IDENT}} from {{ROLE_IDENT}};
+exception when undefined_object then
+  null; -- Role doesn't exist
+end $$;
+
+-- Note: USAGE on public is typically granted by default; we don't revoke it
+-- to avoid breaking other applications that may rely on it.

package/sql/uninit/03.role.sql

@@ -0,0 +1,27 @@
+-- Drop the monitoring role created by prepare-db (template-filled by cli/lib/init.ts)
+-- This must run after revoking all permissions from the role.
+
+-- Use a DO block to handle the case where the role doesn't exist
+do $$ begin
+  -- Reassign owned objects to current user before dropping
+  -- This handles any objects that might have been created by the role
+  begin
+    execute format('reassign owned by %I to current_user', {{ROLE_LITERAL}});
+  exception when undefined_object then
+    null; -- Role doesn't exist, nothing to reassign
+  end;
+
+  -- Drop owned objects (in case reassign didn't work for some objects)
+  begin
+    execute format('drop owned by %I', {{ROLE_LITERAL}});
+  exception when undefined_object then
+    null; -- Role doesn't exist
+  end;
+
+  -- Drop the role
+  begin
+    execute format('drop role %I', {{ROLE_LITERAL}});
+  exception when undefined_object then
+    null; -- Role doesn't exist, that's fine
+  end;
+end $$;

package/test/checkup.test.ts

@@ -287,27 +287,28 @@ describe("Report generators with mock client", () => {
           },
         ],
         databaseSizesRows: [{ datname: "postgres", size_bytes: "1073741824" }],
-        dbStatsRows: [{ 
-          numbackends: 5, 
-          xact_commit: 100, 
-          xact_rollback: 1, 
-          blks_read: 1000, 
-          blks_hit: 9000, 
-          tup_returned: 500, 
-          tup_fetched: 400, 
-          tup_inserted: 50, 
-          tup_updated: 30, 
-          tup_deleted: 10, 
-          deadlocks: 0, 
-          temp_files: 0, 
+        dbStatsRows: [{
+          numbackends: 5,
+          xact_commit: 100,
+          xact_rollback: 1,
+          blks_read: 1000,
+          blks_hit: 9000,
+          tup_returned: 500,
+          tup_fetched: 400,
+          tup_inserted: 50,
+          tup_updated: 30,
+          tup_deleted: 10,
+          deadlocks: 0,
+          temp_files: 0,
           temp_bytes: 0,
-          postmaster_uptime_s: 864000 
+          postmaster_uptime_s: 864000
         }],
         connectionStatesRows: [{ state: "active", count: 2 }, { state: "idle", count: 3 }],
         uptimeRows: [{ start_time: new Date("2024-01-01T00:00:00Z"), uptime: "10 days" }],
         invalidIndexesRows: [],
         unusedIndexesRows: [],
         redundantIndexesRows: [],
+        sensitiveColumnsRows: [],
       }
     );
 
@@ -320,6 +321,7 @@ describe("Report generators with mock client", () => {
     expect("H001" in reports).toBe(true);
     expect("H002" in reports).toBe(true);
     expect("H004" in reports).toBe(true);
+    // S001 is only available in Python reporter, not in CLI express mode
     expect(reports.A002.checkId).toBe("A002");
     expect(reports.A003.checkId).toBe("A003");
     expect(reports.A004.checkId).toBe("A004");
@@ -525,9 +527,233 @@ describe("H001 - Invalid indexes", () => {
     expect(dbData.database_size_pretty).toBeTruthy();
     expect(report.results["test-node"].postgres_version).toBeTruthy();
   });
+
+  test("getInvalidIndexes returns decision tree fields including valid_duplicate_definition", async () => {
+    const mockClient = createMockClient({
+      invalidIndexesRows: [
+        {
+          schema_name: "public",
+          table_name: "users",
+          index_name: "users_email_idx_invalid",
+          relation_name: "users",
+          index_size_bytes: "1048576",
+          index_definition: "CREATE INDEX users_email_idx_invalid ON public.users USING btree (email)",
+          supports_fk: false,
+          is_pk: false,
+          is_unique: false,
+          constraint_name: null,
+          table_row_estimate: "5000",
+          has_valid_duplicate: true,
+          valid_index_name: "users_email_idx",
+          valid_index_definition: "CREATE INDEX users_email_idx ON public.users USING btree (email)",
+        },
+      ],
+    });
+
+    const indexes = await checkup.getInvalidIndexes(mockClient as any);
+    expect(indexes.length).toBe(1);
+    expect(indexes[0].is_pk).toBe(false);
+    expect(indexes[0].is_unique).toBe(false);
+    expect(indexes[0].constraint_name).toBeNull();
+    expect(indexes[0].table_row_estimate).toBe(5000);
+    expect(indexes[0].has_valid_duplicate).toBe(true);
+    expect(indexes[0].valid_duplicate_name).toBe("users_email_idx");
+    expect(indexes[0].valid_duplicate_definition).toBe("CREATE INDEX users_email_idx ON public.users USING btree (email)");
+  });
+
+  test("getInvalidIndexes handles has_valid_duplicate: false with null values", async () => {
+    const mockClient = createMockClient({
+      invalidIndexesRows: [
+        {
+          schema_name: "public",
+          table_name: "orders",
+          index_name: "orders_status_idx_invalid",
+          relation_name: "orders",
+          index_size_bytes: "524288",
+          index_definition: "CREATE INDEX orders_status_idx_invalid ON public.orders USING btree (status)",
+          supports_fk: false,
+          is_pk: false,
+          is_unique: false,
+          constraint_name: null,
+          table_row_estimate: "100000",
+          has_valid_duplicate: false,
+          valid_index_name: null,
+          valid_index_definition: null,
+        },
+      ],
+    });
+
+    const indexes = await checkup.getInvalidIndexes(mockClient as Client);
+    expect(indexes.length).toBe(1);
+    expect(indexes[0].has_valid_duplicate).toBe(false);
+    expect(indexes[0].valid_duplicate_name).toBeNull();
+    expect(indexes[0].valid_duplicate_definition).toBeNull();
+  });
+
+  test("getInvalidIndexes handles is_pk: true with constraint", async () => {
+    const mockClient = createMockClient({
+      invalidIndexesRows: [
+        {
+          schema_name: "public",
+          table_name: "accounts",
+          index_name: "accounts_pkey_invalid",
+          relation_name: "accounts",
+          index_size_bytes: "262144",
+          index_definition: "CREATE UNIQUE INDEX accounts_pkey_invalid ON public.accounts USING btree (id)",
+          supports_fk: true,
+          is_pk: true,
+          is_unique: true,
+          constraint_name: "accounts_pkey",
+          table_row_estimate: "500",
+          has_valid_duplicate: false,
+          valid_index_name: null,
+          valid_index_definition: null,
+        },
+      ],
+    });
+
+    const indexes = await checkup.getInvalidIndexes(mockClient as Client);
+    expect(indexes.length).toBe(1);
+    expect(indexes[0].is_pk).toBe(true);
+    expect(indexes[0].is_unique).toBe(true);
+    expect(indexes[0].constraint_name).toBe("accounts_pkey");
+    expect(indexes[0].supports_fk).toBe(true);
+  });
+
+  test("getInvalidIndexes handles is_unique: true without PK", async () => {
+    const mockClient = createMockClient({
+      invalidIndexesRows: [
+        {
+          schema_name: "public",
+          table_name: "users",
+          index_name: "users_email_unique_invalid",
+          relation_name: "users",
+          index_size_bytes: "131072",
+          index_definition: "CREATE UNIQUE INDEX users_email_unique_invalid ON public.users USING btree (email)",
+          supports_fk: false,
+          is_pk: false,
+          is_unique: true,
+          constraint_name: "users_email_unique",
+          table_row_estimate: "25000",
+          has_valid_duplicate: true,
+          valid_index_name: "users_email_unique_idx",
+          valid_index_definition: "CREATE UNIQUE INDEX users_email_unique_idx ON public.users USING btree (email)",
+        },
+      ],
+    });
+
+    const indexes = await checkup.getInvalidIndexes(mockClient as Client);
+    expect(indexes.length).toBe(1);
+    expect(indexes[0].is_pk).toBe(false);
+    expect(indexes[0].is_unique).toBe(true);
+    expect(indexes[0].constraint_name).toBe("users_email_unique");
+    expect(indexes[0].has_valid_duplicate).toBe(true);
+  });
   // Top-level structure tests removed - covered by schema-validation.test.ts
 });
 
+// Tests for H001 decision tree recommendation logic
+describe("H001 - Decision tree recommendations", () => {
+  // Helper to create a minimal InvalidIndex for testing
+  const createTestIndex = (overrides: Partial<checkup.InvalidIndex> = {}): checkup.InvalidIndex => ({
+    schema_name: "public",
+    table_name: "test_table",
+    index_name: "test_idx",
+    relation_name: "public.test_table",
+    index_size_bytes: 1024,
+    index_size_pretty: "1 KiB",
+    index_definition: "CREATE INDEX test_idx ON public.test_table USING btree (col)",
+    supports_fk: false,
+    is_pk: false,
+    is_unique: false,
+    constraint_name: null,
+    table_row_estimate: 100000, // Large table by default
+    has_valid_duplicate: false,
+    valid_duplicate_name: null,
+    valid_duplicate_definition: null,
+    ...overrides,
+  });
+
+  test("returns DROP when has_valid_duplicate is true", () => {
+    const index = createTestIndex({ has_valid_duplicate: true, valid_duplicate_name: "existing_idx" });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("DROP");
+  });
+
+  test("returns DROP even when is_pk is true if has_valid_duplicate is true", () => {
+    // has_valid_duplicate takes precedence over is_pk
+    const index = createTestIndex({
+      has_valid_duplicate: true,
+      is_pk: true,
+      is_unique: true,
+    });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("DROP");
+  });
+
+  test("returns RECREATE when is_pk is true and no valid duplicate", () => {
+    const index = createTestIndex({
+      is_pk: true,
+      is_unique: true,
+      constraint_name: "test_pkey",
+    });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("RECREATE");
+  });
+
+  test("returns RECREATE when is_unique is true (non-PK) and no valid duplicate", () => {
+    const index = createTestIndex({
+      is_unique: true,
+      constraint_name: "test_unique",
+    });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("RECREATE");
+  });
+
+  test("returns RECREATE for small table (< 10K rows) without valid duplicate", () => {
+    const index = createTestIndex({ table_row_estimate: 5000 });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("RECREATE");
+  });
+
+  test("returns RECREATE for table at threshold boundary (9999 rows)", () => {
+    const index = createTestIndex({ table_row_estimate: 9999 });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("RECREATE");
+  });
+
+  test("returns UNCERTAIN for large table (>= 10K rows) at threshold boundary", () => {
+    const index = createTestIndex({ table_row_estimate: 10000 });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("UNCERTAIN");
+  });
+
+  test("returns UNCERTAIN for large table without valid duplicate or constraint", () => {
+    const index = createTestIndex({ table_row_estimate: 1000000 });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("UNCERTAIN");
+  });
+
+  test("returns UNCERTAIN for empty table (0 rows) with no valid duplicate - edge case", () => {
+    // Empty table should be RECREATE (< 10K threshold)
+    const index = createTestIndex({ table_row_estimate: 0 });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("RECREATE");
+  });
+
+  test("decision tree priority: has_valid_duplicate > is_pk > small_table", () => {
+    // Even with PK and small table, has_valid_duplicate should win
+    const index = createTestIndex({
+      has_valid_duplicate: true,
+      is_pk: true,
+      is_unique: true,
+      table_row_estimate: 100,
+    });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("DROP");
+  });
+
+  test("decision tree priority: is_pk > small_table", () => {
+    // is_pk should return RECREATE regardless of table size
+    const index = createTestIndex({
+      is_pk: true,
+      is_unique: true,
+      table_row_estimate: 1000000, // Large table
+    });
+    expect(checkup.getInvalidIndexRecommendation(index)).toBe("RECREATE");
+  });
+});
+
 // Tests for H002 (Unused indexes)
 describe("H002 - Unused indexes", () => {
   test("getUnusedIndexes returns unused indexes", async () => {

package/test/config-consistency.test.ts

@@ -0,0 +1,36 @@
+/**
+ * Tests that config files are consistent with what the CLI expects.
+ * Catches schema mismatches like pg_statistic in wrong schema.
+ */
+import { describe, test, expect } from "bun:test";
+import { readFileSync } from "fs";
+import { resolve } from "path";
+
+const configDir = resolve(import.meta.dir, "../../config");
+
+describe("Config consistency", () => {
+  test("target-db/init.sql creates pg_statistic in postgres_ai schema", () => {
+    const initSql = readFileSync(resolve(configDir, "target-db/init.sql"), "utf8");
+
+    // Must create postgres_ai schema
+    expect(initSql).toMatch(/create\s+schema\s+if\s+not\s+exists\s+postgres_ai/i);
+
+    // Must create view in postgres_ai schema, not public
+    expect(initSql).toMatch(/create\s+or\s+replace\s+view\s+postgres_ai\.pg_statistic/i);
+    expect(initSql).not.toMatch(/create\s+or\s+replace\s+view\s+public\.pg_statistic/i);
+
+    // Must grant on postgres_ai.pg_statistic
+    expect(initSql).toMatch(/grant\s+select\s+on\s+postgres_ai\.pg_statistic/i);
+  });
+
+  test("pgwatch metrics.yml uses postgres_ai.pg_statistic", () => {
+    const metricsYml = readFileSync(
+      resolve(configDir, "pgwatch-prometheus/metrics.yml"),
+      "utf8"
+    );
+
+    // Should reference postgres_ai.pg_statistic, not public.pg_statistic
+    expect(metricsYml).not.toMatch(/public\.pg_statistic/);
+    expect(metricsYml).toMatch(/postgres_ai\.pg_statistic/);
+  });
+});

package/test/init.integration.test.ts

@@ -241,70 +241,76 @@ describe.skipIf(skipTests)("integration: prepare-db", () => {
     }
   });
 
-  test("fixes slightly-off permissions idempotently", async () => {
-    pg = await createTempPostgres();
+  test(
+    "fixes slightly-off permissions idempotently",
+    async () => {
+      pg = await createTempPostgres();
 
-    try {
-      // Create monitoring role with wrong password, no grants.
-      {
-        const c = new Client({ connectionString: pg.adminUri });
-        await c.connect();
-        await c.query(
-          "do $$ begin if not exists (select 1 from pg_roles where rolname='postgres_ai_mon') then create role postgres_ai_mon login password 'wrong'; end if; end $$;"
-        );
-        await c.end();
-      }
+      try {
+        // Create monitoring role with wrong password, no grants.
+        {
+          const c = new Client({ connectionString: pg.adminUri });
+          await c.connect();
+          await c.query(
+            "do $$ begin if not exists (select 1 from pg_roles where rolname='postgres_ai_mon') then create role postgres_ai_mon login password 'wrong'; end if; end $$;"
+          );
+          await c.end();
+        }
 
-      // Run init (should grant everything).
-      {
-        const r = runCliInit([pg.adminUri, "--password", "correctpw", "--skip-optional-permissions"]);
-        expect(r.status).toBe(0);
-      }
+        // Run init (should grant everything).
+        {
+          const r = runCliInit([pg.adminUri, "--password", "correctpw", "--skip-optional-permissions"]);
+          expect(r.status).toBe(0);
+        }
 
-      // Verify privileges.
-      {
-        const c = new Client({ connectionString: pg.adminUri });
-        await c.connect();
-        const dbOk = await c.query(
-          "select has_database_privilege('postgres_ai_mon', current_database(), 'CONNECT') as ok"
-        );
-        expect(dbOk.rows[0].ok).toBe(true);
-        const roleOk = await c.query("select pg_has_role('postgres_ai_mon', 'pg_monitor', 'member') as ok");
-        expect(roleOk.rows[0].ok).toBe(true);
-        const idxOk = await c.query(
-          "select has_table_privilege('postgres_ai_mon', 'pg_catalog.pg_index', 'SELECT') as ok"
-        );
-        expect(idxOk.rows[0].ok).toBe(true);
-        const viewOk = await c.query(
-          "select has_table_privilege('postgres_ai_mon', 'postgres_ai.pg_statistic', 'SELECT') as ok"
-        );
-        expect(viewOk.rows[0].ok).toBe(true);
-        const sp = await c.query("select rolconfig from pg_roles where rolname='postgres_ai_mon'");
-        expect(Array.isArray(sp.rows[0].rolconfig)).toBe(true);
-        expect(sp.rows[0].rolconfig.some((v: string) => String(v).includes("search_path="))).toBe(true);
-        await c.end();
-      }
+        // Verify privileges.
+        {
+          const c = new Client({ connectionString: pg.adminUri });
+          await c.connect();
+          const dbOk = await c.query(
+            "select has_database_privilege('postgres_ai_mon', current_database(), 'CONNECT') as ok"
+          );
+          expect(dbOk.rows[0].ok).toBe(true);
+          const roleOk = await c.query("select pg_has_role('postgres_ai_mon', 'pg_monitor', 'member') as ok");
+          expect(roleOk.rows[0].ok).toBe(true);
+          const idxOk = await c.query(
+            "select has_table_privilege('postgres_ai_mon', 'pg_catalog.pg_index', 'SELECT') as ok"
+          );
+          expect(idxOk.rows[0].ok).toBe(true);
+          const viewOk = await c.query(
+            "select has_table_privilege('postgres_ai_mon', 'postgres_ai.pg_statistic', 'SELECT') as ok"
+          );
+          expect(viewOk.rows[0].ok).toBe(true);
+          const sp = await c.query("select rolconfig from pg_roles where rolname='postgres_ai_mon'");
+          expect(Array.isArray(sp.rows[0].rolconfig)).toBe(true);
+          expect(sp.rows[0].rolconfig.some((v: string) => String(v).includes("search_path="))).toBe(true);
+          await c.end();
+        }
 
-      // Run init again (idempotent).
-      {
-        const r = runCliInit([pg.adminUri, "--password", "correctpw", "--skip-optional-permissions"]);
-        expect(r.status).toBe(0);
+        // Run init again (idempotent).
+        {
+          const r = runCliInit([pg.adminUri, "--password", "correctpw", "--skip-optional-permissions"]);
+          expect(r.status).toBe(0);
+        }
+      } finally {
+        await pg.cleanup();
       }
-    } finally {
-      await pg.cleanup();
-    }
-  });
+    },
+    { timeout: 15000 }
+  );
 
-  test("reports nicely when lacking permissions", async () => {
-    pg = await createTempPostgres();
+  test(
+    "reports nicely when lacking permissions",
+    async () => {
+      pg = await createTempPostgres();
 
-    try {
-      // Create limited user that can connect but cannot create roles / grant.
-      const limitedPw = "limitedpw";
-      {
-        const c = new Client({ connectionString: pg.adminUri });
-        await c.connect();
-        await c.query(`do $$ begin
+      try {
+        // Create limited user that can connect but cannot create roles / grant.
+        const limitedPw = "limitedpw";
+        {
+          const c = new Client({ connectionString: pg.adminUri });
+          await c.connect();
+          await c.query(`do $$ begin
           if not exists (select 1 from pg_roles where rolname='limited') then
             begin
               create role limited login password ${sqlLiteral(limitedPw)};
@@ -313,20 +319,22 @@ describe.skipIf(skipTests)("integration: prepare-db", () => {
             end;
           end if;
         end $$;`);
-        await c.query("grant connect on database testdb to limited");
-        await c.end();
-      }
+          await c.query("grant connect on database testdb to limited");
+          await c.end();
+        }
 
-      const limitedUri = `postgresql://limited:${limitedPw}@127.0.0.1:${pg.port}/testdb`;
-      const r = runCliInit([limitedUri, "--password", "monpw", "--skip-optional-permissions"]);
-      expect(r.status).not.toBe(0);
-      expect(r.stderr).toMatch(/Error: prepare-db:/);
-      expect(r.stderr).toMatch(/Failed at step "/);
-      expect(r.stderr).toMatch(/Fix: connect as a superuser/i);
-    } finally {
-      await pg.cleanup();
-    }
-  });
+        const limitedUri = `postgresql://limited:${limitedPw}@127.0.0.1:${pg.port}/testdb`;
+        const r = runCliInit([limitedUri, "--password", "monpw", "--skip-optional-permissions"]);
+        expect(r.status).not.toBe(0);
+        expect(r.stderr).toMatch(/Error: prepare-db:/);
+        expect(r.stderr).toMatch(/Failed at step "/);
+        expect(r.stderr).toMatch(/Fix: connect as a superuser/i);
+      } finally {
+        await pg.cleanup();
+      }
+    },
+    { timeout: 15000 }
+  );
 
   test(
     "--verify returns 0 when ok and non-zero when missing",
@@ -399,6 +407,7 @@ describe.skipIf(skipTests)("integration: prepare-db", () => {
     }
   });
 
+  // 60s timeout for PostgreSQL startup + multiple SQL queries in slow CI
   test("explain_generic validates input and prevents SQL injection", async () => {
     pg = await createTempPostgres();
 
@@ -495,5 +504,5 @@ describe.skipIf(skipTests)("integration: prepare-db", () => {
     } finally {
       await pg.cleanup();
     }
-  });
+  }, { timeout: 60000 });
 });