queen-mq 0.1.0 → 0.1.2

package/src/config.js

@@ -23,12 +23,17 @@ export const DATABASE = {
   PASSWORD: process.env.PG_PASSWORD || 'postgres',
   PORT: parseInt(process.env.PG_PORT) || 5432,
   
+  // SSL configuration
+  USE_SSL: process.env.PG_USE_SSL === 'true',
+  SSL_REJECT_UNAUTHORIZED: process.env.PG_SSL_REJECT_UNAUTHORIZED !== 'false',
+  
   // Pool configuration
-  POOL_SIZE: parseInt(process.env.DB_POOL_SIZE) || 20,
+  POOL_SIZE: parseInt(process.env.DB_POOL_SIZE) || 150,
   IDLE_TIMEOUT: parseInt(process.env.DB_IDLE_TIMEOUT) || 30000, // 30 seconds
   CONNECTION_TIMEOUT: parseInt(process.env.DB_CONNECTION_TIMEOUT) || 2000, // 2 seconds
   STATEMENT_TIMEOUT: parseInt(process.env.DB_STATEMENT_TIMEOUT) || 30000, // 30 seconds
   QUERY_TIMEOUT: parseInt(process.env.DB_QUERY_TIMEOUT) || 30000, // 30 seconds
+  LOCK_TIMEOUT: parseInt(process.env.DB_LOCK_TIMEOUT) || 10000, // 10 seconds
   
   // Pool manager settings
   MAX_RETRIES: parseInt(process.env.DB_MAX_RETRIES) || 3
@@ -43,8 +48,14 @@ export const QUEUE = {
   BATCH_INSERT_SIZE: parseInt(process.env.BATCH_INSERT_SIZE) || 1000,
   
   // Long polling
-  POLL_INTERVAL: parseInt(process.env.QUEUE_POLL_INTERVAL) || 100, // 100ms
-  POLL_INTERVAL_FILTERED: parseInt(process.env.QUEUE_POLL_INTERVAL_FILTERED) || 1000, // 1 second
+  POLL_INTERVAL: parseInt(process.env.QUEUE_POLL_INTERVAL) || 100, // 100ms - initial poll interval
+  POLL_INTERVAL_FILTERED: parseInt(process.env.QUEUE_POLL_INTERVAL_FILTERED) || 50, // 50ms for better distribution across consumers
+  MAX_POLL_INTERVAL: parseInt(process.env.QUEUE_MAX_POLL_INTERVAL) || 2000, // 2000ms - max poll interval after backoff
+  BACKOFF_THRESHOLD: parseInt(process.env.QUEUE_BACKOFF_THRESHOLD) || 5, // Number of empty polls before backoff starts
+  BACKOFF_MULTIPLIER: parseFloat(process.env.QUEUE_BACKOFF_MULTIPLIER) || 2, // Exponential backoff multiplier
+  
+  // Partition selection for filtered pops (namespace/task)
+  MAX_PARTITION_CANDIDATES: parseInt(process.env.MAX_PARTITION_CANDIDATES) || 100, // Number of candidate partitions to fetch for lease acquisition
   
   // Queue defaults
   DEFAULT_LEASE_TIME: parseInt(process.env.DEFAULT_LEASE_TIME) || 300, // 5 minutes
@@ -72,7 +83,7 @@ export const QUEUE = {
 // System Events Configuration
 export const SYSTEM_EVENTS = {
   // Enable/disable system event propagation
-  ENABLED: process.env.QUEEN_SYSTEM_EVENTS_ENABLED === 'true' || true,
+  ENABLED: process.env.QUEEN_SYSTEM_EVENTS_ENABLED === 'true' || false,
   
   // Batching window for event publishing (milliseconds)
   BATCH_MS: parseInt(process.env.QUEEN_SYSTEM_EVENTS_BATCH_MS) || 10,
@@ -91,6 +102,9 @@ export const JOBS = {
   RETENTION_BATCH_SIZE: parseInt(process.env.RETENTION_BATCH_SIZE) || 1000,
   PARTITION_CLEANUP_DAYS: parseInt(process.env.PARTITION_CLEANUP_DAYS) || 7,
   
+  // Metrics retention (messages_consumed table)
+  METRICS_RETENTION_DAYS: parseInt(process.env.METRICS_RETENTION_DAYS) || 90, // Keep 90 days of metrics by default
+  
   // Eviction service
   EVICTION_INTERVAL: parseInt(process.env.EVICTION_INTERVAL) || 60000, // 1 minute
   EVICTION_BATCH_SIZE: parseInt(process.env.EVICTION_BATCH_SIZE) || 1000,
@@ -129,7 +143,7 @@ export const CLIENT = {
 
 // API Configuration
 export const API = {
-  MAX_BODY_SIZE: parseInt(process.env.MAX_BODY_SIZE) || 10 * 1024 * 1024, // 10MB max body size
+  MAX_BODY_SIZE: parseInt(process.env.MAX_BODY_SIZE) || 100 * 1024 * 1024, // 100MB max body size
   // Pagination
   DEFAULT_LIMIT: parseInt(process.env.API_DEFAULT_LIMIT) || 100,
   MAX_LIMIT: parseInt(process.env.API_MAX_LIMIT) || 1000,

package/src/database/connection.js

@@ -2,22 +2,44 @@ import pg from 'pg';
 import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
+import config from '../config.js';
 
 const { Pool } = pg;
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
-// Create connection pool
+/**
+ * Create a PostgreSQL connection pool with proper configuration
+ * This is the single source of truth for pool creation across the application
+ */
 export const createPool = () => {
-  return new Pool({
-    user: process.env.PG_USER || 'postgres',
-    host: process.env.PG_HOST || 'localhost',
-    database: process.env.PG_DB || 'postgres',
-    password: process.env.PG_PASSWORD || 'postgres',
-    port: process.env.PG_PORT || 5432,
-    max: process.env.DB_POOL_SIZE || 20,
-    idleTimeoutMillis: 30000,
-    connectionTimeoutMillis: 2000,
-  });
+  const poolConfig = {
+    user: config.DATABASE.USER,
+    host: config.DATABASE.HOST,
+    database: config.DATABASE.DATABASE,
+    password: config.DATABASE.PASSWORD,
+    port: config.DATABASE.PORT,
+    max: config.DATABASE.POOL_SIZE,
+    idleTimeoutMillis: config.DATABASE.IDLE_TIMEOUT,
+    connectionTimeoutMillis: config.DATABASE.CONNECTION_TIMEOUT,
+    // Note: statement_timeout and query_timeout cannot be set as startup parameters
+    // They should be set per-query or per-transaction using SET commands
+    application_name: config.SERVER.APPLICATION_NAME,
+    // PERFORMANCE NOTE: TCP_NODELAY for latency reduction
+    // The node-postgres driver enables TCP_NODELAY by default on sockets
+    // For server-side: Edit postgresql.conf and add: tcp_nodelay = on
+    // Reference: https://stackoverflow.com/questions/60634455/how-does-one-configure-tcp-nodelay-for-libpq-and-postgres-server
+    keepAlive: true,
+    keepAliveInitialDelayMillis: 10000
+  };
+
+  // Add SSL configuration if enabled
+  if (config.DATABASE.USE_SSL) {
+    poolConfig.ssl = {
+      rejectUnauthorized: config.DATABASE.SSL_REJECT_UNAUTHORIZED
+    };
+  }
+
+  return new Pool(poolConfig);
 };
 
 // Initialize database schema
@@ -74,8 +96,8 @@ export const withTransaction = async (pool, callback, isolationLevel = 'READ COM
       }
       
       // Set timeouts to prevent long blocks
-      await client.query('SET LOCAL statement_timeout = 30000'); // 30 seconds
-      await client.query('SET LOCAL lock_timeout = 5000'); // 5 seconds
+      await client.query(`SET LOCAL statement_timeout = ${config.DATABASE.STATEMENT_TIMEOUT}`);
+      await client.query(`SET LOCAL lock_timeout = ${config.DATABASE.LOCK_TIMEOUT}`);
       
       const result = await callback(client);
       await client.query('COMMIT');
@@ -83,13 +105,17 @@ export const withTransaction = async (pool, callback, isolationLevel = 'READ COM
     } catch (error) {
       await client.query('ROLLBACK');
       
-      // Check for serialization failure or deadlock
-      if ((error.code === '40001' || error.code === '40P01') && attempt < maxAttempts) {
+      // Check for retriable errors:
+      // 40001: serialization_failure
+      // 40P01: deadlock_detected
+      // 55P03: lock_not_available (lock timeout)
+      // 25P02: in_failed_sql_transaction (transaction already aborted - retry whole transaction)
+      if ((error.code === '40001' || error.code === '40P01' || error.code === '55P03' || error.code === '25P02') && attempt < maxAttempts) {
         // Exponential backoff: 100ms, 200ms, 400ms
         const delay = Math.min(100 * Math.pow(2, attempt - 1), 1000);
         await new Promise(resolve => setTimeout(resolve, delay));
         attempt++;
-        console.log(`Transaction retry attempt ${attempt}/${maxAttempts} after ${error.code}`);
+        console.log(`Transaction retry attempt ${attempt}/${maxAttempts} after ${error.code} (${error.message})`);
         continue;
       }
       

package/src/database/poolManager.js

@@ -22,6 +22,13 @@ export class PoolManager {
       connectionTimeoutMillis: appConfig.DATABASE.CONNECTION_TIMEOUT,
       ...config
     };
+
+    // Add SSL configuration if enabled
+    if (appConfig.DATABASE.USE_SSL) {
+      this.config.ssl = {
+        rejectUnauthorized: appConfig.DATABASE.SSL_REJECT_UNAUTHORIZED
+      };
+    }
     
     this.pool = new Pool(this.config);
     this.waitingQueue = [];

package/src/database/schema-v2.sql

@@ -1,17 +1,21 @@
--- Queen Message Queue Schema V2 with Bus/Consumer Group Support
--- Structure: Queues → Partitions → Messages → Status (per consumer group)
+-- ════════════════════════════════════════════════════════════════
+-- Queen Message Queue Schema V3
+-- Simplified: Merged cursor+lease, removed messages_status
+-- ════════════════════════════════════════════════════════════════
 
--- Create schema only if it doesn't exist (preserves existing data)
 CREATE SCHEMA IF NOT EXISTS queen;
 
--- Queues table (top level, with all configuration options)
+-- ────────────────────────────────────────────────────────────────
+-- 1. QUEUES - Top-level configuration
+-- ────────────────────────────────────────────────────────────────
 CREATE TABLE IF NOT EXISTS queen.queues (
     id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
     name VARCHAR(255) UNIQUE NOT NULL,
-    namespace VARCHAR(255),  -- optional grouping
-    task VARCHAR(255),       -- optional grouping
-    priority INTEGER DEFAULT 0,  -- Queue priority (higher = processed first)
-    -- Configuration options (all at queue level now)
+    namespace VARCHAR(255),
+    task VARCHAR(255),
+    priority INTEGER DEFAULT 0,
+    
+    -- Processing configuration
     lease_time INTEGER DEFAULT 300,
     retry_limit INTEGER DEFAULT 3,
     retry_delay INTEGER DEFAULT 1000,
@@ -21,16 +25,30 @@ CREATE TABLE IF NOT EXISTS queen.queues (
     dlq_after_max_retries BOOLEAN DEFAULT FALSE,
     delayed_processing INTEGER DEFAULT 0,
     window_buffer INTEGER DEFAULT 0,
+    
+    -- Enterprise features
     retention_seconds INTEGER DEFAULT 0,
     completed_retention_seconds INTEGER DEFAULT 0,
     retention_enabled BOOLEAN DEFAULT FALSE,
     encryption_enabled BOOLEAN DEFAULT FALSE,
     max_wait_time_seconds INTEGER DEFAULT 0,
-    max_queue_size INTEGER DEFAULT 0,  -- 0 = unlimited queue size
+    max_queue_size INTEGER DEFAULT 0,
+    
     created_at TIMESTAMPTZ DEFAULT NOW()
 );
 
--- Partitions table (subdivisions of queues, where FIFO happens)
+-- Indexes for queues
+CREATE INDEX IF NOT EXISTS idx_queues_name ON queen.queues(name);
+CREATE INDEX IF NOT EXISTS idx_queues_priority ON queen.queues(priority DESC);
+CREATE INDEX IF NOT EXISTS idx_queues_namespace ON queen.queues(namespace) WHERE namespace IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_queues_task ON queen.queues(task) WHERE task IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_queues_namespace_task ON queen.queues(namespace, task) 
+WHERE namespace IS NOT NULL AND task IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_queues_retention_enabled ON queen.queues(retention_enabled) WHERE retention_enabled = true;
+
+-- ────────────────────────────────────────────────────────────────
+-- 2. PARTITIONS - FIFO subdivisions within queues
+-- ────────────────────────────────────────────────────────────────
 CREATE TABLE IF NOT EXISTS queen.partitions (
     id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
     queue_id UUID REFERENCES queen.queues(id) ON DELETE CASCADE,
@@ -40,131 +58,169 @@ CREATE TABLE IF NOT EXISTS queen.partitions (
     UNIQUE(queue_id, name)
 );
 
--- Messages table (immutable message data only)
+CREATE INDEX IF NOT EXISTS idx_partitions_queue_name 
+ON queen.partitions(queue_id, name);
+
+CREATE INDEX IF NOT EXISTS idx_partitions_last_activity 
+ON queen.partitions(last_activity);
+
+-- ────────────────────────────────────────────────────────────────
+-- 3. MESSAGES - Immutable message data
+-- ────────────────────────────────────────────────────────────────
 CREATE TABLE IF NOT EXISTS queen.messages (
     id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    transaction_id VARCHAR(255) UNIQUE NOT NULL,  -- Changed from UUID to string for flexibility
-    trace_id UUID DEFAULT gen_random_uuid(),  -- For tracking multi-step pipelines
+    transaction_id VARCHAR(255) UNIQUE NOT NULL,
+    trace_id UUID DEFAULT gen_random_uuid(),
     partition_id UUID REFERENCES queen.partitions(id) ON DELETE CASCADE,
     payload JSONB NOT NULL,
     created_at TIMESTAMPTZ DEFAULT NOW(),
     is_encrypted BOOLEAN DEFAULT FALSE
-    -- Removed all status fields - moved to messages_status table
 );
 
--- Messages status table (tracks consumption per consumer group)
-CREATE TABLE IF NOT EXISTS queen.messages_status (
+-- Indexes for messages (critical for cursor-based queries!)
+CREATE INDEX IF NOT EXISTS idx_messages_partition_created_id 
+ON queen.messages(partition_id, created_at, id);
+
+CREATE INDEX IF NOT EXISTS idx_messages_transaction_id 
+ON queen.messages(transaction_id);
+
+CREATE INDEX IF NOT EXISTS idx_messages_trace_id 
+ON queen.messages(trace_id);
+
+CREATE INDEX IF NOT EXISTS idx_messages_created_at 
+ON queen.messages(created_at);
+
+-- ────────────────────────────────────────────────────────────────
+-- 4. PARTITION_CONSUMERS - Unified cursor + lease tracking
+-- ────────────────────────────────────────────────────────────────
+CREATE TABLE IF NOT EXISTS queen.partition_consumers (
     id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    message_id UUID REFERENCES queen.messages(id) ON DELETE CASCADE,
-    consumer_group VARCHAR(255),  -- NULL for queue mode (competing consumers)
-    status VARCHAR(20) DEFAULT 'pending',
-    worker_id VARCHAR(255),
-    locked_at TIMESTAMPTZ,
-    completed_at TIMESTAMPTZ,
-    failed_at TIMESTAMPTZ,
-    error_message TEXT,
-    retry_count INTEGER DEFAULT 0,
+    partition_id UUID REFERENCES queen.partitions(id) ON DELETE CASCADE,
+    consumer_group VARCHAR(255) DEFAULT '__QUEUE_MODE__',
+    
+    -- ═══════════════════════════════════════════════════════════
+    -- CURSOR STATE (Persistent - consumption progress)
+    -- ═══════════════════════════════════════════════════════════
+    last_consumed_id UUID DEFAULT '00000000-0000-0000-0000-000000000000',
+    last_consumed_created_at TIMESTAMPTZ,
+    total_messages_consumed BIGINT DEFAULT 0,
+    total_batches_consumed BIGINT DEFAULT 0,
+    last_consumed_at TIMESTAMPTZ,
+    
+    -- ═══════════════════════════════════════════════════════════
+    -- LEASE STATE (Ephemeral - active processing lock)
+    -- ═══════════════════════════════════════════════════════════
     lease_expires_at TIMESTAMPTZ,
-    processing_at TIMESTAMPTZ,
+    lease_acquired_at TIMESTAMPTZ,
+    message_batch JSONB,
+    batch_size INTEGER DEFAULT 0,
+    acked_count INTEGER DEFAULT 0,
+    worker_id VARCHAR(255),
+    
+    -- ═══════════════════════════════════════════════════════════
+    -- STATISTICS (for dashboard - avoids expensive COUNT queries)
+    -- ═══════════════════════════════════════════════════════════
+    pending_estimate BIGINT DEFAULT 0,
+    last_stats_update TIMESTAMPTZ,
+    
+    -- ═══════════════════════════════════════════════════════════
+    -- METADATA
+    -- ═══════════════════════════════════════════════════════════
     created_at TIMESTAMPTZ DEFAULT NOW(),
-    UNIQUE(message_id, consumer_group)  -- One status per message per consumer group
+    
+    UNIQUE(partition_id, consumer_group),
+    
+    -- Constraint: cursor consistency
+    CHECK (
+        (last_consumed_id = '00000000-0000-0000-0000-000000000000' 
+         AND last_consumed_created_at IS NULL)
+        OR 
+        (last_consumed_id != '00000000-0000-0000-0000-000000000000' 
+         AND last_consumed_created_at IS NOT NULL)
+    )
 );
 
--- Consumer groups registry (tracks subscription preferences)
-CREATE TABLE IF NOT EXISTS queen.consumer_groups (
+-- Indexes for partition_consumers (critical for performance!)
+CREATE INDEX IF NOT EXISTS idx_partition_consumers_lookup
+ON queen.partition_consumers(partition_id, consumer_group);
+
+CREATE INDEX IF NOT EXISTS idx_partition_consumers_active_leases
+ON queen.partition_consumers(partition_id, consumer_group, lease_expires_at)
+WHERE lease_expires_at IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_partition_consumers_expired_leases
+ON queen.partition_consumers(lease_expires_at)
+WHERE lease_expires_at IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_partition_consumers_progress
+ON queen.partition_consumers(last_consumed_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_partition_consumers_idle
+ON queen.partition_consumers(partition_id, consumer_group)
+WHERE lease_expires_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_partition_consumers_consumer_group
+ON queen.partition_consumers(consumer_group);
+
+-- ────────────────────────────────────────────────────────────────
+-- 5. MESSAGES_CONSUMED - ACK tracking for metrics (time-series)
+-- ────────────────────────────────────────────────────────────────
+CREATE TABLE IF NOT EXISTS queen.messages_consumed (
     id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    queue_id UUID REFERENCES queen.queues(id) ON DELETE CASCADE,
-    name VARCHAR(255) NOT NULL,
-    created_at TIMESTAMPTZ DEFAULT NOW(),
-    subscription_start_from TIMESTAMPTZ,  -- NULL = consume all, timestamp = consume from this point
-    active BOOLEAN DEFAULT true,
-    last_seen_at TIMESTAMPTZ DEFAULT NOW(),
-    UNIQUE(queue_id, name)
+    partition_id UUID REFERENCES queen.partitions(id) ON DELETE CASCADE,
+    consumer_group VARCHAR(255) NOT NULL,
+    
+    -- Metrics for this ACK batch
+    messages_completed INTEGER DEFAULT 0,
+    messages_failed INTEGER DEFAULT 0,
+    
+    -- Timestamp for time-series aggregation
+    acked_at TIMESTAMPTZ DEFAULT NOW()
 );
 
--- Partition leases table (tracks which partition is locked for which consumer group)
-CREATE TABLE IF NOT EXISTS queen.partition_leases (
+-- Indexes for time-series queries
+CREATE INDEX IF NOT EXISTS idx_messages_consumed_acked_at 
+ON queen.messages_consumed(acked_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_messages_consumed_partition_acked 
+ON queen.messages_consumed(partition_id, acked_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_messages_consumed_consumer_acked 
+ON queen.messages_consumed(consumer_group, acked_at DESC);
+
+-- Composite index for filtering by queue (via partition join)
+CREATE INDEX IF NOT EXISTS idx_messages_consumed_partition_id
+ON queen.messages_consumed(partition_id);
+
+-- ────────────────────────────────────────────────────────────────
+-- 6. DEAD_LETTER_QUEUE - Failed messages
+-- ────────────────────────────────────────────────────────────────
+CREATE TABLE IF NOT EXISTS queen.dead_letter_queue (
     id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    message_id UUID REFERENCES queen.messages(id) ON DELETE CASCADE,
     partition_id UUID REFERENCES queen.partitions(id) ON DELETE CASCADE,
-    consumer_group VARCHAR(255) DEFAULT '__QUEUE_MODE__',  -- '__QUEUE_MODE__' for queue mode
-    lease_expires_at TIMESTAMPTZ NOT NULL,
-    message_batch JSONB,  -- Store IDs of messages in this lease
-    created_at TIMESTAMPTZ DEFAULT NOW(),
-    released_at TIMESTAMPTZ,  -- When lease was released (NULL if active)
-    UNIQUE(partition_id, consumer_group)  -- One active lease per partition per consumer group
+    consumer_group VARCHAR(255),
+    error_message TEXT,
+    retry_count INTEGER DEFAULT 0,
+    original_created_at TIMESTAMPTZ,
+    failed_at TIMESTAMPTZ DEFAULT NOW()
 );
 
--- Indexes for queues
-CREATE INDEX IF NOT EXISTS idx_queues_name ON queen.queues(name);
-CREATE INDEX IF NOT EXISTS idx_queues_priority ON queen.queues(priority DESC);
-CREATE INDEX IF NOT EXISTS idx_queues_namespace ON queen.queues(namespace) WHERE namespace IS NOT NULL;
-CREATE INDEX IF NOT EXISTS idx_queues_task ON queen.queues(task) WHERE task IS NOT NULL;
-CREATE INDEX IF NOT EXISTS idx_queues_namespace_task ON queen.queues(namespace, task) WHERE namespace IS NOT NULL AND task IS NOT NULL;
-CREATE INDEX IF NOT EXISTS idx_queues_namespace_priority ON queen.queues(namespace, priority DESC) WHERE namespace IS NOT NULL;
-CREATE INDEX IF NOT EXISTS idx_queues_task_priority ON queen.queues(task, priority DESC) WHERE task IS NOT NULL;
-CREATE INDEX IF NOT EXISTS idx_queues_retention_enabled ON queen.queues(retention_enabled) WHERE retention_enabled = true;
-CREATE INDEX IF NOT EXISTS idx_queues_lease_time ON queen.queues(lease_time);
-CREATE INDEX IF NOT EXISTS idx_queues_ttl ON queen.queues(ttl);
-
--- Indexes for partitions
-CREATE INDEX IF NOT EXISTS idx_partitions_queue_name ON queen.partitions(queue_id, name);
-
--- Indexes for messages (simplified - no status fields)
-CREATE INDEX IF NOT EXISTS idx_messages_partition_created ON queen.messages(partition_id, created_at);
-CREATE INDEX IF NOT EXISTS idx_messages_transaction_id ON queen.messages(transaction_id);
-CREATE INDEX IF NOT EXISTS idx_messages_trace_id ON queen.messages(trace_id);
-CREATE INDEX IF NOT EXISTS idx_messages_created_at ON queen.messages(created_at);
-
--- Indexes for messages_status (critical for performance)
--- Note: Removed the partial unique index idx_status_queue_mode as it conflicts with 
--- the table-level UNIQUE(message_id, consumer_group) constraint
-
--- Bus mode index (consumer_group IS NOT NULL)
-CREATE INDEX IF NOT EXISTS idx_status_bus_mode 
-ON queen.messages_status(message_id, consumer_group, status) 
-WHERE consumer_group IS NOT NULL;
-
--- Index for finding pending messages by consumer group
-CREATE INDEX IF NOT EXISTS idx_status_pending 
-ON queen.messages_status(consumer_group, status, message_id) 
-WHERE status = 'pending';
-
--- Index for lease expiration checks
-CREATE INDEX IF NOT EXISTS idx_status_lease_expires 
-ON queen.messages_status(lease_expires_at, status, consumer_group) 
-WHERE status = 'processing';
-
--- Index for fast queue depth checks
-CREATE INDEX IF NOT EXISTS idx_messages_status_pending_processing 
-ON queen.messages_status(message_id, status) 
-WHERE status IN ('pending', 'processing') OR status IS NULL;
-
--- Index for efficient pop operations (find messages without status for a consumer group)
-CREATE INDEX IF NOT EXISTS idx_status_message_consumer 
-ON queen.messages_status(message_id, consumer_group);
-
--- Indexes for consumer groups
-CREATE INDEX IF NOT EXISTS idx_consumer_groups_queue_name 
-ON queen.consumer_groups(queue_id, name);
-
-CREATE INDEX IF NOT EXISTS idx_consumer_groups_active 
-ON queen.consumer_groups(queue_id, active) 
-WHERE active = true;
-
--- Indexes for partition leases
-CREATE INDEX IF NOT EXISTS idx_partition_leases_expires 
-ON queen.partition_leases(lease_expires_at) 
-WHERE released_at IS NULL;
-
-CREATE INDEX IF NOT EXISTS idx_partition_leases_active 
-ON queen.partition_leases(partition_id, consumer_group) 
-WHERE released_at IS NULL;
-
-CREATE INDEX IF NOT EXISTS idx_partition_leases_active_lookup 
-ON queen.partition_leases(partition_id, consumer_group, lease_expires_at) 
-WHERE released_at IS NULL;
-
--- Retention history table for tracking deletions
+CREATE INDEX IF NOT EXISTS idx_dlq_partition
+ON queen.dead_letter_queue(partition_id);
+
+CREATE INDEX IF NOT EXISTS idx_dlq_consumer_group
+ON queen.dead_letter_queue(consumer_group);
+
+CREATE INDEX IF NOT EXISTS idx_dlq_failed_at
+ON queen.dead_letter_queue(failed_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_dlq_message_consumer
+ON queen.dead_letter_queue(message_id, consumer_group);
+
+-- ────────────────────────────────────────────────────────────────
+-- 7. RETENTION_HISTORY - Optional audit of cleanup operations
+-- ────────────────────────────────────────────────────────────────
 CREATE TABLE IF NOT EXISTS queen.retention_history (
     id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
     partition_id UUID REFERENCES queen.partitions(id) ON DELETE CASCADE,
@@ -179,7 +235,11 @@ ON queen.retention_history(partition_id);
 CREATE INDEX IF NOT EXISTS idx_retention_history_executed 
 ON queen.retention_history(executed_at);
 
--- Trigger to update partition last_activity
+-- ────────────────────────────────────────────────────────────────
+-- TRIGGERS
+-- ────────────────────────────────────────────────────────────────
+
+-- Update partition last_activity on message insert
 CREATE OR REPLACE FUNCTION update_partition_last_activity()
 RETURNS TRIGGER AS $$
 BEGIN
@@ -196,19 +256,23 @@ AFTER INSERT ON queen.messages
 FOR EACH ROW
 EXECUTE FUNCTION update_partition_last_activity();
 
--- Function to clean up old completed statuses in bus mode
-CREATE OR REPLACE FUNCTION cleanup_completed_statuses()
-RETURNS void AS $$
+-- Update pending_estimate on message insert
+CREATE OR REPLACE FUNCTION update_pending_on_push()
+RETURNS TRIGGER AS $$
 BEGIN
-    -- Delete completed statuses older than retention period for bus mode
-    DELETE FROM queen.messages_status ms
-    USING queen.messages m, queen.partitions p, queen.queues q
-    WHERE ms.message_id = m.id
-      AND m.partition_id = p.id
-      AND p.queue_id = q.id
-      AND ms.status IN ('completed', 'failed')
-      AND ms.consumer_group IS NOT NULL  -- Only for bus mode
-      AND q.completed_retention_seconds > 0
-      AND ms.completed_at < NOW() - INTERVAL '1 second' * q.completed_retention_seconds;
+    -- Increment pending estimate for all consumers on this partition
+    UPDATE queen.partition_consumers
+    SET pending_estimate = pending_estimate + 1,
+        last_stats_update = NOW()
+    WHERE partition_id = NEW.partition_id;
+    
+    -- If no consumers exist yet, that's OK - they'll be created on first POP
+    RETURN NEW;
 END;
-$$ LANGUAGE plpgsql;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS trigger_update_pending_on_push ON queen.messages;
+CREATE TRIGGER trigger_update_pending_on_push
+AFTER INSERT ON queen.messages
+FOR EACH ROW
+EXECUTE FUNCTION update_pending_on_push();