queen-mq 0.1.0 → 0.1.2

package/src/managers/queueManagerOptimized.js

@@ -10,6 +10,83 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
   // Batch size for database operations
   const BATCH_INSERT_SIZE = config.QUEUE.BATCH_INSERT_SIZE;
   
+  // Performance metrics tracking for filtered pops
+  const popMetrics = {
+    partitionQueryTimes: [],
+    leaseAttempts: 0,
+    leaseSuccesses: 0,
+    messageQueryTimes: [],
+    totalPopTimes: [],
+    batchSizes: [],
+    lastLogTime: Date.now()
+  };
+  
+  // Performance metrics for direct pops (queue/partition)
+  const directPopMetrics = {
+    leaseAcquisitionTimes: [],
+    messageQueryTimes: [],
+    statusUpdateTimes: [],
+    leaseUpdateTimes: [],
+    decryptionTimes: [],
+    totalPopTimes: [],
+    batchSizes: [],
+    lastLogTime: Date.now()
+  };
+  
+  // Log metrics periodically (every 5 seconds OR after 10 samples)
+  const logMetrics = () => {
+    const now = Date.now();
+    const timeSinceLastLog = now - Math.max(popMetrics.lastLogTime, directPopMetrics.lastLogTime);
+    const hasEnoughSamples = popMetrics.totalPopTimes.length >= 10 || directPopMetrics.totalPopTimes.length >= 10;
+    
+    // Only log if enough time passed OR we have enough samples
+    if (timeSinceLastLog < 5000 && !hasEnoughSamples) return;
+    
+    if (popMetrics.totalPopTimes.length === 0 && directPopMetrics.totalPopTimes.length === 0) return; // No data yet
+    
+    const avg = arr => arr.length > 0 ? (arr.reduce((a,b) => a+b, 0) / arr.length).toFixed(2) : 0;
+    const max = arr => arr.length > 0 ? Math.max(...arr).toFixed(2) : 0;
+    const min = arr => arr.length > 0 ? Math.min(...arr).toFixed(2) : 0;
+    
+    if (popMetrics.totalPopTimes.length > 0) {
+      log(`📊 PERF METRICS (filtered pop) | Samples: ${popMetrics.totalPopTimes.length}`);
+      log(`  - Partition Query: avg=${avg(popMetrics.partitionQueryTimes)}ms max=${max(popMetrics.partitionQueryTimes)}ms`);
+      log(`  - Message Query: avg=${avg(popMetrics.messageQueryTimes)}ms max=${max(popMetrics.messageQueryTimes)}ms`);
+      log(`  - Total Pop: avg=${avg(popMetrics.totalPopTimes)}ms max=${max(popMetrics.totalPopTimes)}ms`);
+      log(`  - Lease Success Rate: ${popMetrics.leaseSuccesses}/${popMetrics.leaseAttempts} (${(popMetrics.leaseSuccesses/popMetrics.leaseAttempts*100).toFixed(1)}%)`);
+      log(`  - Avg Batch Size: ${avg(popMetrics.batchSizes)} messages`);
+    }
+    
+    if (directPopMetrics.totalPopTimes.length > 0) {
+      log(`📊 PERF METRICS (direct pop) | Samples: ${directPopMetrics.totalPopTimes.length}`);
+      log(`  - Lease Acquisition: avg=${avg(directPopMetrics.leaseAcquisitionTimes)}ms max=${max(directPopMetrics.leaseAcquisitionTimes)}ms`);
+      log(`  - Message Query: avg=${avg(directPopMetrics.messageQueryTimes)}ms max=${max(directPopMetrics.messageQueryTimes)}ms`);
+      log(`  - Status Update: avg=${avg(directPopMetrics.statusUpdateTimes)}ms max=${max(directPopMetrics.statusUpdateTimes)}ms`);
+      log(`  - Lease Update: avg=${avg(directPopMetrics.leaseUpdateTimes)}ms max=${max(directPopMetrics.leaseUpdateTimes)}ms`);
+      log(`  - Decryption/Format: avg=${avg(directPopMetrics.decryptionTimes)}ms max=${max(directPopMetrics.decryptionTimes)}ms`);
+      log(`  - Total Pop: avg=${avg(directPopMetrics.totalPopTimes)}ms max=${max(directPopMetrics.totalPopTimes)}ms`);
+      log(`  - Avg Batch Size: ${avg(directPopMetrics.batchSizes)} messages`);
+    }
+    
+    // Reset metrics after logging
+    popMetrics.partitionQueryTimes = [];
+    popMetrics.leaseAttempts = 0;
+    popMetrics.leaseSuccesses = 0;
+    popMetrics.messageQueryTimes = [];
+    popMetrics.totalPopTimes = [];
+    popMetrics.batchSizes = [];
+    popMetrics.lastLogTime = now;
+    
+    directPopMetrics.leaseAcquisitionTimes = [];
+    directPopMetrics.messageQueryTimes = [];
+    directPopMetrics.statusUpdateTimes = [];
+    directPopMetrics.leaseUpdateTimes = [];
+    directPopMetrics.decryptionTimes = [];
+    directPopMetrics.totalPopTimes = [];
+    directPopMetrics.batchSizes = [];
+    directPopMetrics.lastLogTime = now;
+  };
+  
   // Ensure queue exists and partition exist (with caching)
   // Queue must be created via configure endpoint, but partitions can be created on-demand
   const ensureResources = async (client, queueName, partitionName = 'Default', namespace = null, task = null) => {
@@ -126,6 +203,8 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
   
   // Optimized batch insert for high throughput (simplified - no status)
   const pushMessagesBatch = async (items) => {
+    const pushStartTime = Date.now();
+    
     // Group items by queue and partition for efficient resource lookup
     const partitionGroups = {};
     for (const item of items) {
@@ -139,6 +218,45 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
     
     const allResults = [];
     
+    // OPTIMIZATION: Check queue capacity once per queue, not per partition
+    // Build a map of queue -> total batch size across all partitions
+    const queueBatchSizes = {};
+    for (const [partitionKey, partitionItems] of Object.entries(partitionGroups)) {
+      const [queueName] = partitionKey.split(':');
+      queueBatchSizes[queueName] = (queueBatchSizes[queueName] || 0) + partitionItems.length;
+    }
+    
+    // Check capacity for all queues with max_queue_size set (single query)
+    const queuesToCheck = Object.keys(queueBatchSizes);
+    if (queuesToCheck.length > 0) {
+      await withTransaction(pool, async (client) => {
+        const capacityCheck = await client.query(
+          `SELECT 
+             q.name,
+             q.max_queue_size,
+             COALESCE(SUM(pc.pending_estimate), 0)::integer as current_depth
+           FROM queen.queues q
+           LEFT JOIN queen.partitions p ON p.queue_id = q.id
+           LEFT JOIN queen.partition_consumers pc ON pc.partition_id = p.id
+             AND pc.consumer_group = '__QUEUE_MODE__'
+           WHERE q.name = ANY($1::varchar[])
+             AND q.max_queue_size > 0
+           GROUP BY q.name, q.max_queue_size`,
+          [queuesToCheck]
+        );
+        
+        // Check each queue's capacity
+        for (const row of capacityCheck.rows) {
+          const currentDepth = parseInt(row.current_depth);
+          const batchSize = queueBatchSizes[row.name];
+          const maxSize = parseInt(row.max_queue_size);
+          if (currentDepth + batchSize > maxSize) {
+            throw new Error(`Queue '${row.name}' would exceed max capacity (${maxSize}). Current: ${currentDepth}, Batch: ${batchSize}`);
+          }
+        }
+      });
+    }
+    
     // Process each partition group
     for (const [partitionKey, partitionItems] of Object.entries(partitionGroups)) {
       const [queueName, partitionName] = partitionKey.split(':');
@@ -154,27 +272,6 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
             const resources = await ensureResources(client, queueName, partitionName);
             const { partitionId, encryptionEnabled, queueConfig } = resources;
         
-        // Check queue capacity if max_queue_size is set
-        if (queueConfig.maxQueueSize > 0) {
-          const capacityCheck = await client.query(
-            `SELECT COUNT(m.id) as current_depth
-             FROM queen.messages m
-             JOIN queen.partitions p ON m.partition_id = p.id
-             JOIN queen.queues q ON p.queue_id = q.id
-             LEFT JOIN queen.messages_status ms ON m.id = ms.message_id 
-               AND ms.consumer_group = '__QUEUE_MODE__'
-             WHERE q.name = $1
-               AND (ms.status IS NULL OR ms.status IN ('pending', 'processing'))`,
-            [queueName]
-          );
-          
-          const currentDepth = parseInt(capacityCheck.rows[0].current_depth);
-          const batchSize = partitionItems.length;
-          if (currentDepth + batchSize > queueConfig.maxQueueSize) {
-            throw new Error(`Queue '${queueName}' would exceed max capacity (${queueConfig.maxQueueSize}). Current: ${currentDepth}, Batch: ${batchSize}`);
-          }
-        }
-        
         // Process items in batches
         const batchResults = [];
         
@@ -189,29 +286,39 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
           const encryptedFlags = [];
           const duplicates = [];
           
-          for (const item of batch) {
-            const messageId = generateUUID();
-            const transactionId = item.transactionId || generateUUID();
-            const traceId = item.traceId || null;
-            
-            // Check for duplicate transaction ID
+          // ⚡ OPTIMIZATION #1: Batch duplicate check - single query for all transaction IDs
+          const allTransactionIds = batch.map(item => item.transactionId || generateUUID());
+          
+          // Skip duplicate check if no transaction IDs were explicitly provided (all generated)
+          const hasExplicitTxnIds = batch.some(item => item.transactionId);
+          let existingTxnIds = new Map();
+          
+          if (hasExplicitTxnIds) {
+            // Use more efficient LEFT JOIN approach instead of ANY() for better index usage
             const dupCheck = await client.query(
-              'SELECT id FROM queen.messages WHERE transaction_id = $1',
-              [transactionId]
+              `SELECT t.txn_id as transaction_id, m.id
+               FROM UNNEST($1::varchar[]) AS t(txn_id)
+               LEFT JOIN queen.messages m ON m.transaction_id = t.txn_id
+               WHERE m.id IS NOT NULL`,
+              [allTransactionIds]
             );
             
-            if (dupCheck.rows.length > 0) {
-              duplicates.push({
-                id: dupCheck.rows[0].id,
-                transactionId: transactionId,
-                status: 'duplicate'
-              });
-              continue;
-            }
+            // Create a map of existing transaction IDs for O(1) lookup
+            existingTxnIds = new Map(dupCheck.rows.map(row => [row.transaction_id, row.id]));
+          }
+          
+          // ⚡ OPTIMIZATION #2: Parallel encryption - encrypt all payloads concurrently
+          const encryptionTasks = batch.map(async (item, idx) => {
+            const transactionId = allTransactionIds[idx];
             
-            messageIds.push(messageId);
-            transactionIds.push(transactionId);
-            traceIds.push(traceId);
+            // Check if this transaction ID already exists
+            if (existingTxnIds.has(transactionId)) {
+              return {
+                isDuplicate: true,
+                id: existingTxnIds.get(transactionId),
+                transactionId
+              };
+            }
             
             // Handle encryption if enabled
             let payload = item.payload;
@@ -222,34 +329,61 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
                 payload = await encryption.encryptPayload(item.payload);
                 isEncrypted = true;
               } catch (error) {
-                log(`ERROR: Encryption failed for message ${messageId}:`, error);
+                log(`ERROR: Encryption failed for transaction ${transactionId}:`, error);
                 // Continue with unencrypted payload
               }
             }
             
+            return {
+              isDuplicate: false,
+              messageId: generateUUID(),
+              transactionId,
+              traceId: item.traceId || null,
+              payload,
+              isEncrypted
+            };
+          });
+          
+          // Wait for all encryption tasks to complete in parallel
+          const encryptionResults = await Promise.all(encryptionTasks);
+          
+          // Process results
+          for (const result of encryptionResults) {
+            if (result.isDuplicate) {
+              duplicates.push({
+                id: result.id,
+                transactionId: result.transactionId,
+                status: 'duplicate'
+              });
+              continue;
+            }
+            
             // Handle large payloads and special cases
             let jsonPayload;
             try {
               // Handle null/undefined payloads - convert to JSON null string
-              if (payload === null || payload === undefined) {
+              if (result.payload === null || result.payload === undefined) {
                 jsonPayload = 'null';  // JSONB expects the string 'null' for null values
               } else {
                 // Try to stringify the payload
-                jsonPayload = JSON.stringify(payload);
+                jsonPayload = JSON.stringify(result.payload);
               }
             } catch (error) {
               // If JSON.stringify fails (e.g., circular references, too large), 
               // store as a string representation
-              log(`WARN: Failed to stringify payload for message ${messageId}:`, error.message);
+              log(`WARN: Failed to stringify payload for message ${result.messageId}:`, error.message);
               jsonPayload = JSON.stringify({ 
                 error: 'Payload serialization failed', 
-                type: typeof payload,
+                type: typeof result.payload,
                 message: error.message 
               });
             }
             
+            messageIds.push(result.messageId);
+            transactionIds.push(result.transactionId);
+            traceIds.push(result.traceId);
             payloads.push(jsonPayload);
-            encryptedFlags.push(isEncrypted);
+            encryptedFlags.push(result.isEncrypted);
           }
           
           // Batch insert messages with trace_id
@@ -316,601 +450,24 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
       }
     }
     
+    const duration = Date.now() - pushStartTime;
+    log(`[PUSH] Pushed ${items.length} items → ${allResults.length} results in ${duration}ms (${(items.length / (duration / 1000)).toFixed(0)} msg/s)`);
     return allResults;
   };
 
-  // Helper function to ensure consumer group exists
-const ensureConsumerGroup = async (client, queueId, consumerGroup, subscriptionMode, subscriptionFrom) => {
-  // Handle special subscription values
-  let effectiveSubscriptionFrom = subscriptionFrom;
-  if (subscriptionFrom === 'now' || subscriptionMode === 'new' || subscriptionMode === 'new-only') {
-    effectiveSubscriptionFrom = new Date();
-  } else if (!subscriptionFrom || subscriptionFrom === 'all') {
-    effectiveSubscriptionFrom = null; // Will consume all messages
-  }
-  
-  const result = await client.query(`
-    INSERT INTO queen.consumer_groups (queue_id, name, subscription_start_from)
-    VALUES ($1, $2, $3)
-    ON CONFLICT (queue_id, name) DO UPDATE SET
-      subscription_start_from = CASE 
-        WHEN queen.consumer_groups.subscription_start_from IS NULL 
-        THEN EXCLUDED.subscription_start_from
-        ELSE queen.consumer_groups.subscription_start_from
-      END,
-      last_seen_at = NOW()
-    RETURNING *
-  `, [queueId, consumerGroup, effectiveSubscriptionFrom]);
-  
-  return result.rows[0];
-};
-
-// Helper function for eviction on pop
-const evictOnPop = async (client, queueName) => {
-  const evictionResult = await client.query(`
-    WITH queue_config AS (
-      SELECT id, max_wait_time_seconds
-      FROM queen.queues
-      WHERE name = $1 AND max_wait_time_seconds > 0
-    ),
-    evicted AS (
-      DELETE FROM queen.messages
-      WHERE id IN (
-        SELECT m.id
-        FROM queen.messages m
-        JOIN queen.partitions p ON m.partition_id = p.id
-        JOIN queue_config qc ON p.queue_id = qc.id
-        WHERE m.created_at < NOW() - INTERVAL '1 second' * qc.max_wait_time_seconds
-      )
-      RETURNING id
-    )
-    SELECT COUNT(*) as count FROM evicted
-  `, [queueName]);
-  
-  if (evictionResult.rows[0].count > 0) {
-    log(`${LogTypes.POP} | EVICTION | Queue: ${queueName} | Count: ${evictionResult.rows[0].count}`);
-  }
-};
-
-/**
- * Pop messages with simplified transaction-based approach
- * This is a drop-in replacement for the original popMessages function
- */
-const popMessagesV2 = async (scope, options = {}) => {
-  const { queue, partition, consumerGroup } = scope;
-  const { 
-    wait = false, 
-    timeout = config.QUEUE.DEFAULT_TIMEOUT, 
-    batch = config.QUEUE.DEFAULT_BATCH_SIZE,
-    subscriptionMode = null,
-    subscriptionFrom = null
-  } = options;
-  
-  return withTransaction(pool, async (client) => {
-    // Set transaction isolation level to prevent dirty reads
-    await client.query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ');
-    
-    // Step 1: Reclaim expired leases
-    const reclaimResult = await client.query(`
-      UPDATE queen.messages_status 
-      SET status = 'pending', 
-          lease_expires_at = NULL, 
-          worker_id = NULL, 
-          processing_at = NULL
-      WHERE status = 'processing' 
-        AND lease_expires_at < NOW()
-        AND consumer_group = $1
-      RETURNING message_id
-    `, [consumerGroup || '__QUEUE_MODE__']);
-    
-    if (reclaimResult.rowCount > 0) {
-      log(`Reclaimed ${reclaimResult.rowCount} expired leases`);
-    }
-    
-    // Step 2: Evict expired messages
-    await evictOnPop(client, queue);
-    
-    // Step 3: Get queue information
-    const queueResult = await client.query(
-      'SELECT * FROM queen.queues WHERE name = $1',
-      [queue]
-    );
-    
-    if (queueResult.rows.length === 0) {
-      return { messages: [] };
-    }
-    
-    const queueInfo = queueResult.rows[0];
-    
-    // Step 4: Handle consumer group if specified
-    let consumerGroupInfo = null;
-    if (consumerGroup) {
-      consumerGroupInfo = await ensureConsumerGroup(
-        client, 
-        queueInfo.id, 
-        consumerGroup,
-        subscriptionMode,
-        subscriptionFrom
-      );
-    }
-    
-    const actualConsumerGroup = consumerGroup || '__QUEUE_MODE__';
-    let subscriptionStart = consumerGroupInfo?.subscription_start_from;
-    
-    // Handle subscription start date
-    if (consumerGroup) {
-      if (!subscriptionStart) {
-        subscriptionStart = '1970-01-01';
-      } else if (typeof subscriptionStart === 'object' && subscriptionStart instanceof Date) {
-        subscriptionStart = subscriptionStart.toISOString();
-      }
-    }
-    
-    // Step 5: Get or create partition
-    let partitionInfo;
-    if (partition) {
-      // Specific partition requested
-      const partitionResult = await client.query(`
-        SELECT p.* FROM queen.partitions p
-        WHERE p.queue_id = $1 AND p.name = $2
-      `, [queueInfo.id, partition]);
-      
-      if (partitionResult.rows.length === 0) {
-        // Create partition if it doesn't exist
-        const createPartitionResult = await client.query(`
-          INSERT INTO queen.partitions (id, queue_id, name)
-          VALUES ($1, $2, $3)
-          ON CONFLICT (queue_id, name) DO UPDATE SET name = EXCLUDED.name
-          RETURNING *
-        `, [generateUUID(), queueInfo.id, partition]);
-        partitionInfo = createPartitionResult.rows[0];
-      } else {
-        partitionInfo = partitionResult.rows[0];
-      }
-    } else {
-      // Find any available partition with messages that is not currently leased
-      // This applies to both QUEUE MODE and BUS MODE for partition isolation
-      const partitionResult = await client.query(`
-        SELECT DISTINCT p.* 
-        FROM queen.partitions p
-        JOIN queen.messages m ON m.partition_id = p.id
-        LEFT JOIN queen.messages_status ms ON m.id = ms.message_id 
-          AND ms.consumer_group = $2
-        LEFT JOIN queen.partition_leases pl ON p.id = pl.partition_id 
-          AND pl.consumer_group = $2
-          AND pl.released_at IS NULL
-          AND pl.lease_expires_at > NOW()
-        WHERE p.queue_id = $1
-          AND (ms.id IS NULL OR ms.status IN ('pending', 'failed'))
-          AND pl.id IS NULL  -- No active lease for this partition
-        ORDER BY p.created_at
-        LIMIT 1
-      `, [queueInfo.id, actualConsumerGroup]);
-      
-      if (partitionResult.rows.length === 0) {
-        // No partitions with available messages, check if we need to create default
-        const anyPartitionResult = await client.query(`
-          SELECT * FROM queen.partitions WHERE queue_id = $1 LIMIT 1
-        `, [queueInfo.id]);
-        
-        if (anyPartitionResult.rows.length === 0) {
-          // Create default partition
-          const createPartitionResult = await client.query(`
-            INSERT INTO queen.partitions (id, queue_id, name)
-            VALUES ($1, $2, 'Default')
-            ON CONFLICT (queue_id, name) DO UPDATE SET name = EXCLUDED.name
-            RETURNING *
-          `, [generateUUID(), queueInfo.id]);
-          partitionInfo = createPartitionResult.rows[0];
-        } else {
-          return { messages: [] };
-        }
-      } else {
-        partitionInfo = partitionResult.rows[0];
-      }
-    }
-    
-    // Step 6: Try to acquire or verify partition lease (for both QUEUE MODE and BUS MODE)
-    // This ensures partition isolation between consumers in the same group
-    
-    // First, try to acquire a new lease or check if we can take over an expired one
-    const leaseResult = await client.query(`
-      INSERT INTO queen.partition_leases (
-        partition_id, 
-        consumer_group, 
-        lease_expires_at
-      ) VALUES ($1, $2, NOW() + INTERVAL '1 second' * $3)
-      ON CONFLICT (partition_id, consumer_group) 
-      DO UPDATE SET 
-        lease_expires_at = CASE
-          WHEN queen.partition_leases.released_at IS NOT NULL 
-            OR queen.partition_leases.lease_expires_at <= NOW()
-          THEN EXCLUDED.lease_expires_at
-          ELSE queen.partition_leases.lease_expires_at
-        END,
-        released_at = CASE
-          WHEN queen.partition_leases.released_at IS NOT NULL 
-            OR queen.partition_leases.lease_expires_at <= NOW()
-          THEN NULL
-          ELSE queen.partition_leases.released_at
-        END
-      RETURNING 
-        lease_expires_at,
-        (lease_expires_at = NOW() + INTERVAL '1 second' * $3) as acquired
-    `, [partitionInfo.id, actualConsumerGroup, queueInfo.lease_time]);
-    
-    // Check if we actually acquired the lease
-    if (!leaseResult.rows[0].acquired) {
-      // Another consumer still has an active lease on this partition
-      return { messages: [] };
-    }
-    
-    // Step 7: Build WHERE clause based on mode
-    let whereClause;
-    let queryParams;
-    
-    if (consumerGroup) {
-      // BUS MODE with consumer group
-      whereClause = `
-        m.partition_id = $2
-        AND (ms.id IS NULL OR ms.status IN ('pending', 'failed'))
-        AND m.created_at <= NOW() - INTERVAL '1 second' * $3
-        AND m.created_at >= $4::timestamp
-        AND ($5 = 0 OR m.created_at > NOW() - INTERVAL '1 second' * $5)
-      `;
-      queryParams = [
-        actualConsumerGroup,
-        partitionInfo.id,
-        queueInfo.delayed_processing || 0,
-        subscriptionStart,
-        queueInfo.max_wait_time_seconds || 0,
-        batch
-      ];
-    } else {
-      // QUEUE MODE
-      whereClause = `
-        m.partition_id = $2
-        AND (ms.id IS NULL OR ms.status IN ('pending', 'failed'))
-        AND m.created_at <= NOW() - INTERVAL '1 second' * $3
-        AND ($4 = 0 OR m.created_at > NOW() - INTERVAL '1 second' * $4)
-        AND ($5 = 0 OR NOT EXISTS (
-          SELECT 1 FROM queen.messages m2 
-          WHERE m2.partition_id = p.id 
-            AND m2.created_at > NOW() - INTERVAL '1 second' * $5
-        ))
-      `;
-      queryParams = [
-        actualConsumerGroup,
-        partitionInfo.id,
-        queueInfo.delayed_processing || 0,
-        queueInfo.max_wait_time_seconds || 0,
-        queueInfo.window_buffer || 0,
-        batch
-      ];
-    }
-    
-    // Step 8: Select and lock messages
-    const messagesResult = await client.query(`
-      SELECT 
-        m.id,
-        m.transaction_id,
-        m.trace_id,
-        m.payload,
-        m.is_encrypted,
-        m.created_at,
-        p.name as partition_name,
-        q.name as queue_name,
-        q.priority,
-        COALESCE(ms.retry_count, 0) as retry_count
-      FROM queen.messages m
-      JOIN queen.partitions p ON m.partition_id = p.id
-      JOIN queen.queues q ON p.queue_id = q.id
-      LEFT JOIN queen.messages_status ms ON m.id = ms.message_id 
-        AND ms.consumer_group = $1
-      WHERE ${whereClause}
-      ORDER BY m.created_at ASC
-      LIMIT $6
-      FOR UPDATE OF m NOWAIT
-    `, queryParams).catch(err => {
-      if (err.code === '55P03') { // Lock not available
-        return { rows: [] };
-      }
-      throw err;
-    });
-    
-    const messages = messagesResult.rows;
-    
-    if (messages.length === 0) {
-      // Release partition lease if no messages
-      await client.query(`
-        UPDATE queen.partition_leases
-        SET released_at = NOW()
-        WHERE partition_id = $1 AND consumer_group = $2
-      `, [partitionInfo.id, actualConsumerGroup]);
-      return { messages: [] };
-    }
-    
-    // Step 9: Update or insert message status for selected messages
-    const leaseExpiresAt = new Date(Date.now() + (queueInfo.lease_time * 1000));
-    
-    for (const message of messages) {
-      await client.query(`
-        INSERT INTO queen.messages_status (
-          message_id,
-          consumer_group,
-          status,
-          lease_expires_at,
-          processing_at,
-          worker_id,
-          retry_count
-        ) VALUES ($1, $2, 'processing', $3, NOW(), $4, $5)
-        ON CONFLICT (message_id, consumer_group) 
-        DO UPDATE SET
-          status = 'processing',
-          lease_expires_at = EXCLUDED.lease_expires_at,
-          processing_at = NOW(),
-          worker_id = EXCLUDED.worker_id
-          -- Keep existing retry_count when re-processing failed messages
-      `, [
-        message.id,
-        actualConsumerGroup,
-        leaseExpiresAt,
-        config.WORKER_ID,
-        message.retry_count || 0
-      ]);
-    }
-    
-    // Step 10: Update partition lease with message batch
-    if (messages.length > 0) {
-      const messageIds = messages.map(m => m.id);
-      await client.query(`
-        UPDATE queen.partition_leases
-        SET message_batch = $1::jsonb
-        WHERE partition_id = $2 AND consumer_group = $3
-      `, [JSON.stringify(messageIds), partitionInfo.id, actualConsumerGroup]);
-    }
-    
-    // Step 11: Decrypt payloads if needed and format response
-    const decryptedMessages = await Promise.all(messages.map(async (msg) => {
-      let payload = msg.payload;
-      if (msg.is_encrypted && encryption.isEncryptionEnabled()) {
-        try {
-          payload = await encryption.decryptPayload(msg.payload);
-        } catch (error) {
-          log(`${LogTypes.ERROR} | Failed to decrypt message ${msg.transaction_id}: ${error.message}`);
-        }
-      }
-      
-      return {
-        id: msg.id,
-        transactionId: msg.transaction_id,
-        traceId: msg.trace_id,
-        queue: msg.queue_name,
-        partition: msg.partition_name,
-        data: payload,
-        payload: payload,
-        retryCount: msg.retry_count || 0,
-        priority: msg.priority || 0,
-        createdAt: msg.created_at,
-        consumerGroup: consumerGroup || null
-      };
-    }));
-    
-    log(`${LogTypes.POP} | Count: ${decryptedMessages.length} | ConsumerGroup: ${actualConsumerGroup === '__QUEUE_MODE__' ? 'QUEUE_MODE' : actualConsumerGroup}`);
-    
-    return { messages: decryptedMessages };
-  });
-};
-
-
-const popMessages = async (scope, options = {}) => {
-  return popMessagesV2(scope, options);
-};
 
   // Acknowledge messages (now per consumer group)
+  // CURSOR-BASED: Single ACK now routes through batch ACK to advance cursor properly
   const acknowledgeMessage = async (transactionId, status = 'completed', error = null, consumerGroup = null) => {
-    return withTransaction(pool, async (client) => {
-      // Find the message and status entry (must be in 'processing' status to be acknowledged)
-      const findQuery = consumerGroup 
-        ? `SELECT ms.retry_count, ms.status, ms.message_id, m.partition_id, 
-                  q.retry_limit, q.dlq_after_max_retries
-           FROM queen.messages_status ms
-           JOIN queen.messages m ON ms.message_id = m.id
-           JOIN queen.partitions p ON m.partition_id = p.id
-           JOIN queen.queues q ON p.queue_id = q.id
-           WHERE m.transaction_id = $1 AND ms.consumer_group = $2 AND ms.status = 'processing'`
-        : `SELECT ms.retry_count, ms.status, ms.message_id, m.partition_id, 
-                  q.retry_limit, q.dlq_after_max_retries
-           FROM queen.messages_status ms
-           JOIN queen.messages m ON ms.message_id = m.id
-           JOIN queen.partitions p ON m.partition_id = p.id
-           JOIN queen.queues q ON p.queue_id = q.id
-           WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__' AND ms.status = 'processing'`;
-      
-      const params = consumerGroup ? [transactionId, consumerGroup] : [transactionId];
-      const result = await client.query(findQuery, params);
-      
-      if (result.rows.length === 0) {
-        log(`WARN: Message not found for acknowledgment: ${transactionId}, consumerGroup: ${consumerGroup || 'QUEUE_MODE'}`);
-        
-        // Try to find the message regardless of status for debugging
-        const debugQuery = consumerGroup
-          ? `SELECT ms.retry_count, ms.status FROM queen.messages_status ms
-             JOIN queen.messages m ON ms.message_id = m.id
-             WHERE m.transaction_id = $1 AND ms.consumer_group = $2`
-          : `SELECT ms.retry_count, ms.status FROM queen.messages_status ms
-             JOIN queen.messages m ON ms.message_id = m.id
-             WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__'`;
-        const debugResult = await client.query(debugQuery, params);
-        if (debugResult.rows.length > 0) {
-          log(`DEBUG: Found message with status=${debugResult.rows[0].status}, retry_count=${debugResult.rows[0].retry_count}`);
-        }
-        
-        return { status: 'not_found', transaction_id: transactionId };
-      }
-      
-      const messageStatus = result.rows[0];
-      log(`DEBUG: Found message for ack with retry_count=${messageStatus.retry_count}, status=${messageStatus.status}, dlq=${messageStatus.dlq_after_max_retries}, limit=${messageStatus.retry_limit}`);
-      
-      if (status === 'completed') {
-        // Mark as completed and release partition lease
-        const actualConsumerGroup = consumerGroup || '__QUEUE_MODE__';
-        const updateParams = consumerGroup 
-          ? [transactionId, consumerGroup]
-          : [transactionId];
-        
-        const updateQuery = consumerGroup
-          ? `UPDATE queen.messages_status ms
-             SET status = 'completed', completed_at = NOW()
-             FROM queen.messages m
-             WHERE ms.message_id = m.id 
-               AND m.transaction_id = $1 
-               AND ms.consumer_group = $2`
-          : `UPDATE queen.messages_status ms
-             SET status = 'completed', completed_at = NOW()
-             FROM queen.messages m
-             WHERE ms.message_id = m.id 
-               AND m.transaction_id = $1 
-               AND ms.consumer_group = '__QUEUE_MODE__'`;
-        
-        await client.query(updateQuery, updateParams);
-        
-        // Release partition lease if this was the last message in the batch
-        const releaseQuery = `
-          WITH message_partition AS (
-            -- Get the partition for this message
-            SELECT DISTINCT m.partition_id
-            FROM queen.messages m
-            WHERE m.transaction_id = $1
-          ),
-          check_remaining AS (
-            -- Check if there are any other processing messages in this partition
-            SELECT 
-              mp.partition_id,
-              COUNT(ms.message_id) as processing_count
-            FROM message_partition mp
-            LEFT JOIN queen.messages m ON m.partition_id = mp.partition_id
-            LEFT JOIN queen.messages_status ms ON ms.message_id = m.id
-              AND ms.consumer_group = $2
-              AND ms.status = 'processing'
-            GROUP BY mp.partition_id
-          )
-          -- Release the lease only if no messages are processing
-          UPDATE queen.partition_leases pl
-          SET released_at = NOW()
-          FROM check_remaining cr
-          WHERE pl.partition_id = cr.partition_id
-            AND pl.consumer_group = $2
-            AND pl.released_at IS NULL
-            AND cr.processing_count = 0`;
-        
-        await client.query(releaseQuery, [transactionId, actualConsumerGroup]);
-        
-        log(`${LogTypes.ACK} | TransactionId: ${transactionId} | Status: completed | ConsumerGroup: ${consumerGroup || 'QUEUE_MODE'}`);
-      } else if (status === 'failed') {
-        // Handle failure with retry logic
-        const actualConsumerGroup = consumerGroup || '__QUEUE_MODE__';
-        const currentRetryCount = messageStatus.retry_count || 0;
-        const nextRetryCount = currentRetryCount + 1;
-        const retryLimit = messageStatus.retry_limit || config.QUEUE.DEFAULT_RETRY_LIMIT;
-        const dlqEnabled = messageStatus.dlq_after_max_retries;
-        
-        log(`${LogTypes.ACK} | Retry check: current=${currentRetryCount}, next=${nextRetryCount}, limit=${retryLimit}, dlq=${dlqEnabled}`);
-        
-        if (nextRetryCount > retryLimit && dlqEnabled) {
-          // Move to dead letter queue after exceeding retry limit
-          const updateParams = consumerGroup 
-            ? [transactionId, error, consumerGroup]
-            : [transactionId, error];
-          
-          const updateQuery = consumerGroup
-            ? `UPDATE queen.messages_status ms
-               SET status = 'dead_letter', failed_at = NOW(), error_message = $2
-               FROM queen.messages m
-               WHERE ms.message_id = m.id 
-                 AND m.transaction_id = $1 
-                 AND ms.consumer_group = $3`
-            : `UPDATE queen.messages_status ms
-               SET status = 'dead_letter', failed_at = NOW(), error_message = $2
-               FROM queen.messages m
-               WHERE ms.message_id = m.id 
-                 AND m.transaction_id = $1 
-                 AND ms.consumer_group = '__QUEUE_MODE__'`;
-          
-          await client.query(updateQuery, updateParams);
-          
-          log(`${LogTypes.ACK} | TransactionId: ${transactionId} | Status: dead_letter | ConsumerGroup: ${consumerGroup || 'QUEUE_MODE'} | Error: ${error}`);
-          
-          return { status: 'dead_letter', transaction_id: transactionId };
-        } else {
-          // Mark as failed and increment retry count
-          const updateParams = consumerGroup 
-            ? [transactionId, error, consumerGroup]
-            : [transactionId, error];
-          
-          const updateQuery = consumerGroup
-            ? `UPDATE queen.messages_status ms
-               SET status = 'failed', 
-                   failed_at = NOW(), 
-                   error_message = $2, 
-                   lease_expires_at = NULL,
-                   retry_count = COALESCE(retry_count, 0) + 1
-               FROM queen.messages m
-               WHERE ms.message_id = m.id 
-                 AND m.transaction_id = $1 
-                 AND ms.consumer_group = $3`
-            : `UPDATE queen.messages_status ms
-               SET status = 'failed', 
-                   failed_at = NOW(), 
-                   error_message = $2, 
-                   lease_expires_at = NULL,
-                   retry_count = COALESCE(retry_count, 0) + 1
-               FROM queen.messages m
-               WHERE ms.message_id = m.id 
-                 AND m.transaction_id = $1 
-                 AND ms.consumer_group = '__QUEUE_MODE__'`;
-          
-          await client.query(updateQuery, updateParams);
-          
-          // Release partition lease on failure as well (so partition can be retried)
-          const releaseOnFailQuery = `
-            WITH message_partition AS (
-              SELECT DISTINCT m.partition_id
-              FROM queen.messages m
-              WHERE m.transaction_id = $1
-            ),
-            check_remaining AS (
-              SELECT 
-                mp.partition_id,
-                COUNT(ms.message_id) as processing_count
-              FROM message_partition mp
-              LEFT JOIN queen.messages m ON m.partition_id = mp.partition_id
-              LEFT JOIN queen.messages_status ms ON ms.message_id = m.id
-                AND ms.consumer_group = $2
-                AND ms.status = 'processing'
-              GROUP BY mp.partition_id
-            )
-            UPDATE queen.partition_leases pl
-            SET released_at = NOW()
-            FROM check_remaining cr
-            WHERE pl.partition_id = cr.partition_id
-              AND pl.consumer_group = $2
-              AND pl.released_at IS NULL
-              AND cr.processing_count = 0`;
-          
-          await client.query(releaseOnFailQuery, [transactionId, actualConsumerGroup]);
-          
-          log(`${LogTypes.ACK} | TransactionId: ${transactionId} | Status: failed (retry ${nextRetryCount}/${retryLimit}) | ConsumerGroup: ${consumerGroup || 'QUEUE_MODE'} | Error: ${error}`);
-        }
-      }
-      
-      return { status, transaction_id: transactionId };
-    });
+    const batchResult = await acknowledgeMessages([{ transactionId, status, error }], consumerGroup);
+    return batchResult[0] || { status: 'not_found', transaction_id: transactionId };
   };
   
-  // Batch acknowledge messages
+  // Batch acknowledge messages with cursor-based advancement
   const acknowledgeMessages = async (acknowledgments, consumerGroup = null) => {
+    const ackStartTime = Date.now();
     const results = [];
+    const actualConsumerGroup = consumerGroup || '__QUEUE_MODE__';
     
     // Group by status for efficient processing
     const grouped = {
@@ -924,113 +481,190 @@ const popMessages = async (scope, options = {}) => {
       }
     });
     
+    const totalMessages = acknowledgments.length;
+    const failedCount = grouped.failed.length;
+    const successCount = grouped.completed.length;
+    
+    // Determine if this is a TOTAL batch failure or partial failure
+    // Only treat as "total batch failure" if it's a real batch (>1 message)
+    // Single-message failures always go to DLQ
+    const isTotalBatchFailure = failedCount === totalMessages && totalMessages > 1;
+    
     await withTransaction(pool, async (client) => {
-      // Process completed messages
-      if (grouped.completed.length > 0) {
-        const ids = grouped.completed.map(a => a.transactionId);
-        const actualConsumerGroup = consumerGroup || '__QUEUE_MODE__';
-        
-        const updateQuery = consumerGroup
-          ? `UPDATE queen.messages_status ms
-             SET status = 'completed', completed_at = NOW()
-             FROM queen.messages m
-             WHERE ms.message_id = m.id 
-               AND m.transaction_id = ANY($1::varchar[])
-               AND ms.consumer_group = $2`
-          : `UPDATE queen.messages_status ms
-             SET status = 'completed', completed_at = NOW()
-             FROM queen.messages m
-             WHERE ms.message_id = m.id 
-               AND m.transaction_id = ANY($1::varchar[])
-               AND ms.consumer_group = '__QUEUE_MODE__'`;
+      // CURSOR-BASED APPROACH: Handle total batch failure vs partial failure differently
+      
+      if (isTotalBatchFailure) {
+        // ═══════════════════════════════════════════════════════════════
+        // TOTAL BATCH FAILURE: Don't advance cursor, allow retry
+        // ═══════════════════════════════════════════════════════════════
+        log(`[CURSOR] Total batch failure - cursor NOT advanced, same batch will retry on next POP`);
+        const ackTime = Date.now() - ackStartTime;
+        const throughput = ackTime > 0 ? (totalMessages / (ackTime / 1000)).toFixed(0) : 0;
         
-        const params = consumerGroup ? [ids, consumerGroup] : [ids];
-        await client.query(updateQuery, params);
+        log(`[${LogTypes.ACK_BATCH}] Total batch failure: ${failedCount} msgs | ${throughput} msg/s | ${ackTime}ms | ConsumerGroup: ${consumerGroup || 'QUEUE_MODE'}`);
         
-        // Release partition leases for partitions with no more processing messages
-        const releaseQuery = `
+        // Just release the lease - next POP will get same batch
+        const allIds = acknowledgments.map(a => a.transactionId);
+        await client.query(`
           WITH affected_partitions AS (
             SELECT DISTINCT m.partition_id
             FROM queen.messages m
             WHERE m.transaction_id = ANY($1::varchar[])
           )
-          UPDATE queen.partition_leases pl
-          SET released_at = NOW()
+          UPDATE queen.partition_consumers pc
+          SET lease_expires_at = NULL,
+              lease_acquired_at = NULL,
+              message_batch = NULL,
+              batch_size = 0,
+              acked_count = 0
           FROM affected_partitions ap
-          WHERE pl.partition_id = ap.partition_id
-            AND pl.consumer_group = $2
-            AND pl.released_at IS NULL
-            AND NOT EXISTS (
-              SELECT 1 
-              FROM queen.messages_status ms
-              JOIN queen.messages m2 ON ms.message_id = m2.id
-              WHERE m2.partition_id = ap.partition_id
-                AND ms.consumer_group = $2
-                AND ms.status = 'processing'
-            )`;
+          WHERE pc.partition_id = ap.partition_id
+            AND pc.consumer_group = $2
+        `, [allIds, actualConsumerGroup]);
+        
+        grouped.failed.forEach(ack => {
+          results.push({ transactionId: ack.transactionId, status: 'failed_retry' });
+        });
+        
+      } else {
+        // ═══════════════════════════════════════════════════════════════
+        // PARTIAL SUCCESS: Advance cursor, move failed to DLQ
+        // ═══════════════════════════════════════════════════════════════
+        
+        // Get all messages in this batch (successful and failed) to find cursor position
+        const allIds = acknowledgments.map(a => a.transactionId);
+        const batchInfo = await client.query(`
+               SELECT 
+            m.id,
+                 m.transaction_id,
+            m.created_at,
+            m.partition_id,
+            p.name as partition_name
+               FROM queen.messages m
+          JOIN queen.partitions p ON p.id = m.partition_id
+               WHERE m.transaction_id = ANY($1::varchar[])
+          ORDER BY m.created_at DESC, m.id DESC
+          LIMIT 1
+        `, [allIds]);
+        
+        if (batchInfo.rows.length === 0) {
+          throw new Error('No messages found for acknowledgment');
+        }
         
-        await client.query(releaseQuery, [ids, actualConsumerGroup]);
+        const lastMessage = batchInfo.rows[0];
+        const partitionId = lastMessage.partition_id;
+        const partitionName = lastMessage.partition_name;
         
-        log(`${LogTypes.ACK_BATCH} | Status: completed | Count: ${ids.length} | ConsumerGroup: ${consumerGroup || 'QUEUE_MODE'} | TransactionIds: [${ids.join(', ')}]`);
+        // Advance cursor + update lease atomically in partition_consumers
+        const cursorUpdateResult = await client.query(`
+          UPDATE queen.partition_consumers
+          SET 
+              -- Advance cursor (regardless of success/failure)
+              last_consumed_created_at = $1,
+              last_consumed_id = $2,
+              total_messages_consumed = total_messages_consumed + $3,
+              total_batches_consumed = total_batches_consumed + 1,
+              last_consumed_at = NOW(),
+              
+              -- Update pending estimate
+              pending_estimate = GREATEST(0, pending_estimate - $3),
+              last_stats_update = NOW(),
+              
+              -- Update/release lease
+              acked_count = acked_count + $3,
+              lease_expires_at = CASE 
+                  WHEN acked_count + $3 >= batch_size AND batch_size > 0 
+                  THEN NULL 
+                  ELSE lease_expires_at 
+              END,
+              lease_acquired_at = CASE 
+                  WHEN acked_count + $3 >= batch_size AND batch_size > 0 
+                  THEN NULL 
+                  ELSE lease_acquired_at 
+              END,
+              message_batch = CASE 
+                  WHEN acked_count + $3 >= batch_size AND batch_size > 0 
+                  THEN NULL 
+                  ELSE message_batch 
+              END,
+              batch_size = CASE 
+                  WHEN acked_count + $3 >= batch_size AND batch_size > 0 
+                  THEN 0 
+                  ELSE batch_size 
+              END
+          WHERE partition_id = $4 
+            AND consumer_group = $5
+          RETURNING total_messages_consumed, total_batches_consumed, 
+                    (lease_expires_at IS NULL) as lease_released
+        `, [lastMessage.created_at, lastMessage.id, totalMessages, partitionId, actualConsumerGroup]);
         
+        const newTotalConsumed = cursorUpdateResult.rows[0]?.total_messages_consumed || 0;
+        const newTotalBatches = cursorUpdateResult.rows[0]?.total_batches_consumed || 0;
+        const leaseReleased = cursorUpdateResult.rows[0]?.lease_released || false;
+        
+        const ackTime = Date.now() - ackStartTime;
+        const throughput = ackTime > 0 ? (totalMessages / (ackTime / 1000)).toFixed(0) : 0;
+        
+        log(`[${LogTypes.ACK_BATCH}] Success: ${successCount} | Failed: ${failedCount} | ${throughput} msg/s | ${ackTime}ms | Cursor: ${newTotalConsumed} msgs | Lease: ${leaseReleased ? 'Released' : 'Active'} | ConsumerGroup: ${consumerGroup || 'QUEUE_MODE'}`);
+        
+        // Move failed messages to DLQ (individual failures, not batch retry)
+        if (grouped.failed.length > 0) {
+          const failedIds = grouped.failed.map(a => a.transactionId);
+          const failedErrors = grouped.failed.map(a => a.error || 'Unknown error');
+          
+            await client.query(`
+            INSERT INTO queen.dead_letter_queue (message_id, partition_id, consumer_group, error_message, original_created_at)
+            SELECT 
+              m.id,
+              m.partition_id,
+              $2,
+              e.error_message,
+              m.created_at
+            FROM queen.messages m
+            CROSS JOIN LATERAL UNNEST($1::varchar[], $3::text[]) AS e(txn_id, error_message)
+            WHERE m.transaction_id = e.txn_id
+              AND m.transaction_id = ANY($1::varchar[])
+            ON CONFLICT DO NOTHING
+          `, [failedIds, actualConsumerGroup, failedErrors]);
+          
+          log(`[${LogTypes.ACK_BATCH}] Moved ${failedCount} failed messages to DLQ | ConsumerGroup: ${consumerGroup || 'QUEUE_MODE'}`);
+        }
+        
+        // Track consumption metrics for time-series analytics
+        await client.query(`
+          INSERT INTO queen.messages_consumed (partition_id, consumer_group, messages_completed, messages_failed)
+          VALUES ($1, $2, $3, $4)
+        `, [partitionId, actualConsumerGroup, successCount, failedCount]);
+        
+        // Add results
         grouped.completed.forEach(ack => {
           results.push({ transactionId: ack.transactionId, status: 'completed' });
         });
-      }
-      
-      // Process failed messages (needs individual handling for retry logic)
-      for (const ack of grouped.failed) {
-        const result = await acknowledgeMessage(ack.transactionId, 'failed', ack.error, consumerGroup);
-        results.push({ transactionId: ack.transactionId, ...result });
+        
+        grouped.failed.forEach(ack => {
+          results.push({ transactionId: ack.transactionId, status: 'failed_dlq' });
+        });
       }
     });
     
     return results;
   };
   
-  // Reclaim expired leases (now includes partition leases)
+  // Reclaim expired leases from partition_consumers
   const reclaimExpiredLeases = async () => {
     return withTransaction(pool, async (client) => {
-      // First, handle expired partition leases
-      const partitionLeaseResult = await client.query(
-        `WITH expired_leases AS (
-          UPDATE queen.partition_leases
-          SET released_at = NOW()
-          WHERE lease_expires_at < NOW()
-            AND released_at IS NULL
-          RETURNING partition_id, consumer_group, message_batch
-        )
-        -- Reset message status for messages in expired leases
-        UPDATE queen.messages_status ms
-        SET status = 'pending',
-            retry_count = COALESCE(retry_count, 0) + 1,
-            failed_at = NOW(),
-            error_message = 'Partition lease expired',
-            lease_expires_at = NULL,
-            worker_id = NULL,
-            processing_at = NULL
-        FROM expired_leases el
-        WHERE ms.message_id = ANY(
-          SELECT jsonb_array_elements_text(el.message_batch)::uuid
-        )
-        AND ms.consumer_group = el.consumer_group
-        RETURNING ms.message_id`
-      );
-      
-      if (partitionLeaseResult.rows.length > 0) {
-        log(`${LogTypes.RECLAIM} | Expired partition leases reclaimed | Message count: ${partitionLeaseResult.rows.length}`);
-      }
-      
-      // Then handle individual message lease expiration
+      // Clear expired leases from partition_consumers
       const result = await client.query(
-        `UPDATE queen.messages_status 
-         SET status = 'pending', 
-             lease_expires_at = NULL,
-             worker_id = NULL,
-             processing_at = NULL
-         WHERE status = 'processing' 
+        `UPDATE queen.partition_consumers
+         SET lease_expires_at = NULL,
+             lease_acquired_at = NULL,
+             message_batch = NULL,
+             batch_size = 0,
+             acked_count = 0,
+             worker_id = NULL
+         WHERE lease_expires_at IS NOT NULL
            AND lease_expires_at < NOW()
-         RETURNING message_id, consumer_group`
+         RETURNING partition_id, consumer_group`
       );
       
       if (result.rows.length > 0) {
@@ -1041,7 +675,7 @@ const popMessages = async (scope, options = {}) => {
         });
         
         Object.entries(byGroup).forEach(([group, count]) => {
-          log(`Reclaimed ${count} expired leases for consumer group: ${group}`);
+          log(`${LogTypes.RECLAIM} | Reclaimed ${count} expired leases for consumer group: ${group}`);
         });
       }
       
@@ -1049,7 +683,7 @@ const popMessages = async (scope, options = {}) => {
     });
   };
   
-  // Get queue statistics (updated for new structure)
+  // Get queue statistics (using partition_consumers for fast estimates)
   const getQueueStats = async (filters = {}) => {
     const { queue, namespace, task, fromDateTime, toDateTime } = filters;
     
@@ -1059,19 +693,26 @@ const popMessages = async (scope, options = {}) => {
         q.namespace,
         q.task,
         p.name as partition,
-        cg.name as consumer_group,
-        COUNT(CASE WHEN ms.status = 'pending' THEN 1 END) as pending,
-        COUNT(CASE WHEN ms.status = 'processing' THEN 1 END) as processing,
-        COUNT(CASE WHEN ms.status = 'completed' THEN 1 END) as completed,
-        COUNT(CASE WHEN ms.status = 'failed' THEN 1 END) as failed,
-        COUNT(CASE WHEN ms.status = 'dead_letter' THEN 1 END) as dead_letter,
-        COUNT(m.id) as total_messages,
-        COUNT(DISTINCT cg.name) as consumer_groups_count
+        COALESCE(pc.pending_estimate, 0) as pending,
+        CASE WHEN pc.lease_expires_at IS NOT NULL THEN 1 ELSE 0 END as processing,
+        COALESCE(pc.total_messages_consumed, 0) as completed,
+        0 as failed,
+        (SELECT COUNT(*) FROM queen.dead_letter_queue dlq 
+         WHERE dlq.partition_id = p.id 
+         AND dlq.consumer_group = COALESCE(pc.consumer_group, '__QUEUE_MODE__')) as dead_letter,
+        (SELECT COUNT(*) FROM queen.messages m 
+         WHERE m.partition_id = p.id
+         ${fromDateTime ? 'AND m.created_at >= $' + (1 + (queue ? 1 : 0) + (namespace ? 1 : 0) + (task ? 1 : 0)) + '::timestamp' : ''}
+         ${toDateTime ? 'AND m.created_at <= $' + (1 + (queue ? 1 : 0) + (namespace ? 1 : 0) + (task ? 1 : 0) + (fromDateTime ? 1 : 0)) + '::timestamp' : ''}
+        ) as total_messages,
+        (SELECT COUNT(DISTINCT consumer_group) 
+         FROM queen.partition_consumers 
+         WHERE partition_id = p.id 
+         AND consumer_group != '__QUEUE_MODE__') as consumer_groups_count
       FROM queen.queues q
       LEFT JOIN queen.partitions p ON p.queue_id = q.id
-      LEFT JOIN queen.messages m ON m.partition_id = p.id
-      LEFT JOIN queen.messages_status ms ON ms.message_id = m.id
-      LEFT JOIN queen.consumer_groups cg ON cg.queue_id = q.id AND cg.active = true
+      LEFT JOIN queen.partition_consumers pc ON pc.partition_id = p.id 
+        AND pc.consumer_group = '__QUEUE_MODE__'
     `;
     
     const conditions = [];
@@ -1092,14 +733,12 @@ const popMessages = async (scope, options = {}) => {
       params.push(task);
     }
     
-    // Add date/time filtering
+    // Add date params for subqueries
     if (fromDateTime) {
-      conditions.push(`m.created_at >= $${params.length + 1}::timestamp`);
       params.push(fromDateTime);
     }
     
     if (toDateTime) {
-      conditions.push(`m.created_at <= $${params.length + 1}::timestamp`);
       params.push(toDateTime);
     }
     
@@ -1107,14 +746,13 @@ const popMessages = async (scope, options = {}) => {
       query += ` WHERE ${conditions.join(' AND ')}`;
     }
     
-    query += ` GROUP BY q.name, q.namespace, q.task, p.name, cg.name
-               ORDER BY q.name, p.name, cg.name`;
+    query += ` ORDER BY q.name, p.name`;
     
     const result = await pool.query(query, params);
     return result.rows;
   };
   
-  // Get queue lag statistics
+  // Get queue lag statistics (simplified with partition_consumers)
   const getQueueLag = async (filters = {}) => {
     const { queue, namespace, task, fromDateTime, toDateTime } = filters;
     
@@ -1125,49 +763,32 @@ const popMessages = async (scope, options = {}) => {
           q.namespace,
           q.task,
           p.name as partition,
-          COUNT(CASE WHEN ms.status = 'pending' THEN 1 END) as pending_count,
-          COUNT(CASE WHEN ms.status = 'processing' THEN 1 END) as processing_count,
-          COUNT(CASE WHEN ms.status IN ('pending', 'processing') THEN 1 END) as total_backlog,
-          COUNT(CASE 
-            WHEN ms.status = 'completed' 
-            AND ms.completed_at >= COALESCE($1::timestamp, NOW() - INTERVAL '1 hour')
-            AND ms.completed_at <= COALESCE($2::timestamp, NOW())
-            THEN 1 
-          END) as completed_messages,
-          AVG(CASE 
-            WHEN ms.status = 'completed' AND ms.completed_at IS NOT NULL AND m.created_at IS NOT NULL
-            THEN EXTRACT(EPOCH FROM (ms.completed_at - m.created_at))
-            ELSE NULL
-          END) as avg_processing_time_seconds,
-          PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY 
-            CASE 
-              WHEN ms.status = 'completed' AND ms.completed_at IS NOT NULL AND m.created_at IS NOT NULL
-              THEN EXTRACT(EPOCH FROM (ms.completed_at - m.created_at))
-              ELSE NULL
-            END
-          ) as median_processing_time_seconds,
-          PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY 
-            CASE 
-              WHEN ms.status = 'completed' AND ms.completed_at IS NOT NULL AND m.created_at IS NOT NULL
-              THEN EXTRACT(EPOCH FROM (ms.completed_at - m.created_at))
-              ELSE NULL
-            END
-          ) as p95_processing_time_seconds,
-          MIN(m.created_at) FILTER (WHERE ms.status IN ('pending', 'processing')) as oldest_unprocessed,
-          MAX(m.created_at) FILTER (WHERE ms.status = 'completed') as newest_completed
+          COALESCE(pc.pending_estimate, 0) as pending_count,
+          CASE WHEN pc.lease_expires_at IS NOT NULL THEN 1 ELSE 0 END as processing_count,
+          COALESCE(pc.pending_estimate, 0) + CASE WHEN pc.lease_expires_at IS NOT NULL THEN 1 ELSE 0 END as total_backlog,
+          COALESCE(pc.total_messages_consumed, 0) as completed_messages,
+          0 as avg_processing_time_seconds,
+          0 as median_processing_time_seconds,
+          0 as p95_processing_time_seconds,
+          (SELECT MIN(created_at) FROM queen.messages m 
+           WHERE m.partition_id = p.id 
+           AND (m.created_at, m.id) > (COALESCE(pc.last_consumed_created_at, '1970-01-01'::timestamptz), 
+                                        COALESCE(pc.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid))
+          ) as oldest_unprocessed,
+          (SELECT MAX(created_at) FROM queen.messages m
+           WHERE m.partition_id = p.id
+           AND (m.created_at, m.id) <= (COALESCE(pc.last_consumed_created_at, NOW()::timestamptz), 
+                                         COALESCE(pc.last_consumed_id, gen_random_uuid()))
+          ) as newest_completed
         FROM queen.queues q
         LEFT JOIN queen.partitions p ON p.queue_id = q.id
-        LEFT JOIN queen.messages m ON m.partition_id = p.id
-        LEFT JOIN queen.messages_status ms ON ms.message_id = m.id AND ms.consumer_group = '__QUEUE_MODE__'
+        LEFT JOIN queen.partition_consumers pc ON pc.partition_id = p.id 
+          AND pc.consumer_group = '__QUEUE_MODE__'
     `;
     
     const conditions = [];
     const params = [];
     
-    // Add date params first (they're referenced in the query above as $1 and $2)
-    params.push(fromDateTime || null);
-    params.push(toDateTime || null);
-    
     if (queue) {
       conditions.push(`q.name = $${params.length + 1}`);
       params.push(queue);
@@ -1187,7 +808,9 @@ const popMessages = async (scope, options = {}) => {
       query += ` WHERE ${conditions.join(' AND ')}`;
     }
     
-    query += ` GROUP BY q.name, q.namespace, q.task, p.name
+    query += ` GROUP BY q.name, q.namespace, q.task, p.name, pc.pending_estimate, 
+                       pc.lease_expires_at, pc.total_messages_consumed, 
+                       pc.last_consumed_created_at, pc.last_consumed_id, p.id
       )
       SELECT 
         queue,
@@ -1199,9 +822,9 @@ const popMessages = async (scope, options = {}) => {
           'processingCount', processing_count,
           'totalBacklog', total_backlog,
           'completedMessages', completed_messages,
-          'avgProcessingTimeSeconds', COALESCE(avg_processing_time_seconds, 0),
-          'medianProcessingTimeSeconds', COALESCE(median_processing_time_seconds, 0),
-          'p95ProcessingTimeSeconds', COALESCE(p95_processing_time_seconds, 0),
+          'avgProcessingTimeSeconds', avg_processing_time_seconds,
+          'medianProcessingTimeSeconds', median_processing_time_seconds,
+          'p95ProcessingTimeSeconds', p95_processing_time_seconds,
           'estimatedLagSeconds', CASE 
             WHEN oldest_unprocessed IS NOT NULL 
             THEN EXTRACT(EPOCH FROM (NOW() - oldest_unprocessed))
@@ -1225,165 +848,6 @@ const popMessages = async (scope, options = {}) => {
     return result.rows;
   };
 
-  /**
-   * Build query for popping messages with filters (namespace/task)
-   * This is a helper function to construct the complex query used in popMessagesWithFilters
-   */
-  const buildPopMessagesWithFiltersQuery = (actualConsumerGroup, namespace, task, batch) => {
-    let query = `
-      WITH available_messages AS (
-        SELECT m.id, m.transaction_id, m.trace_id, m.payload, m.is_encrypted, m.created_at,
-               p.id as partition_id, p.name as partition_name, q.name as queue_name,
-               q.priority, q.lease_time, q.retry_limit, q.delayed_processing,
-               q.window_buffer, q.max_wait_time_seconds, q.namespace, q.task
-        FROM queen.messages m
-        JOIN queen.partitions p ON m.partition_id = p.id
-        JOIN queen.queues q ON p.queue_id = q.id
-        LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = $1
-        WHERE (ms.id IS NULL OR ms.status = 'pending')
-          AND m.created_at <= NOW() - INTERVAL '1 second' * q.delayed_processing
-          AND (q.max_wait_time_seconds = 0 OR 
-               m.created_at > NOW() - INTERVAL '1 second' * q.max_wait_time_seconds)
-          AND (q.window_buffer = 0 OR NOT EXISTS (
-            SELECT 1 FROM queen.messages m2 
-            WHERE m2.partition_id = p.id 
-              AND m2.created_at > NOW() - INTERVAL '1 second' * q.window_buffer
-          ))
-          AND NOT EXISTS (
-            -- Check for active partition leases
-            SELECT 1 FROM queen.partition_leases pl
-            WHERE pl.partition_id = p.id
-              AND pl.consumer_group = $1
-              AND pl.released_at IS NULL
-              AND pl.lease_expires_at > NOW()
-          )
-    `;
-    
-    const params = [actualConsumerGroup];
-    
-    if (namespace) {
-      params.push(namespace);
-      query += ` AND q.namespace = $${params.length}`;
-    }
-    
-    if (task) {
-      params.push(task);
-      query += ` AND q.task = $${params.length}`;
-    }
-    
-    params.push(batch);
-    query += ` ORDER BY q.priority DESC, m.created_at ASC LIMIT $${params.length} FOR UPDATE OF m SKIP LOCKED`;
-    
-    // Add the INSERT part
-    params.push(config.WORKER_ID);
-    query += `)
-      INSERT INTO queen.messages_status (message_id, consumer_group, status, lease_expires_at, processing_at, worker_id)
-      SELECT id, $1, 'processing', NOW() + INTERVAL '1 second' * lease_time, NOW(), $${params.length}
-      FROM available_messages
-      ON CONFLICT (message_id, consumer_group) 
-      DO UPDATE SET 
-        status = 'processing',
-        lease_expires_at = EXCLUDED.lease_expires_at,
-        processing_at = EXCLUDED.processing_at,
-        worker_id = EXCLUDED.worker_id,
-        retry_count = queen.messages_status.retry_count + 1
-      RETURNING message_id,
-        (SELECT transaction_id FROM queen.messages WHERE id = message_id),
-        (SELECT trace_id FROM queen.messages WHERE id = message_id),
-        (SELECT payload FROM queen.messages WHERE id = message_id),
-        (SELECT is_encrypted FROM queen.messages WHERE id = message_id),
-        (SELECT created_at FROM queen.messages WHERE id = message_id),
-        (SELECT p.id FROM queen.messages m2 JOIN queen.partitions p ON m2.partition_id = p.id WHERE m2.id = message_id) as partition_id,
-        (SELECT p.name FROM queen.messages m2 JOIN queen.partitions p ON m2.partition_id = p.id WHERE m2.id = message_id) as partition_name,
-        (SELECT q.name FROM queen.messages m2 JOIN queen.partitions p ON m2.partition_id = p.id JOIN queen.queues q ON p.queue_id = q.id WHERE m2.id = message_id) as queue_name,
-        (SELECT q.priority FROM queen.messages m2 JOIN queen.partitions p ON m2.partition_id = p.id JOIN queen.queues q ON p.queue_id = q.id WHERE m2.id = message_id) as priority,
-        (SELECT q.lease_time FROM queen.messages m2 JOIN queen.partitions p ON m2.partition_id = p.id JOIN queen.queues q ON p.queue_id = q.id WHERE m2.id = message_id) as lease_time,
-        retry_count
-    `;
-    
-    return { query, params };
-  };
-
-  /**
-   * Pop messages with namespace/task filtering
-   * Supports filtering by namespace and/or task in addition to consumer groups
-   */
-  const popMessagesWithFilters = async (filters, options = {}) => {
-    const { namespace, task, consumerGroup } = filters;
-    const { wait = false, timeout = config.QUEUE.DEFAULT_TIMEOUT, batch = config.QUEUE.DEFAULT_BATCH_SIZE } = options;
-    
-    return withTransaction(pool, async (client) => {
-      // Determine the actual consumer group to use
-      const actualConsumerGroup = consumerGroup || '__QUEUE_MODE__';
-      
-      // Use the helper function to build the query
-      const { query, params } = buildPopMessagesWithFiltersQuery(actualConsumerGroup, namespace, task, batch);
-      
-      const result = await client.query(query, params);
-      
-      // Lock all partitions that we got messages from
-      if (result.rows.length > 0) {
-        const partitionIds = [...new Set(result.rows.map(r => r.partition_id))];
-        const leaseTime = Math.max(...result.rows.map(r => r.lease_time || 30));
-        
-        // Acquire partition leases for all partitions we got messages from
-        for (const partitionId of partitionIds) {
-          await client.query(`
-            INSERT INTO queen.partition_leases (
-              partition_id, 
-              consumer_group, 
-              lease_expires_at
-            ) VALUES ($1, $2, NOW() + INTERVAL '1 second' * $3)
-            ON CONFLICT (partition_id, consumer_group) 
-            DO UPDATE SET 
-              lease_expires_at = EXCLUDED.lease_expires_at,
-              released_at = NULL
-          `, [partitionId, actualConsumerGroup, leaseTime]);
-        }
-      }
-      
-      if (result.rows.length === 0 && wait) {
-        // For namespace/task filtering, we can't easily wait on specific queue paths
-        // So we'll use simple polling
-        return { messages: [] };
-      }
-      
-      // Format and decrypt messages
-      const messages = await Promise.all(
-        result.rows.map(async (row) => {
-          let decryptedPayload = row.payload;
-          
-          if (row.is_encrypted && encryption.isEncryptionEnabled()) {
-            try {
-              decryptedPayload = await encryption.decryptPayload(row.payload);
-            } catch (error) {
-              log('ERROR: Decryption failed:', error);
-            }
-          }
-          
-          return {
-            id: row.message_id,
-            transactionId: row.transaction_id,
-            traceId: row.trace_id,
-            queue: row.queue_name,
-            partition: row.partition_name,
-            data: decryptedPayload,
-            payload: decryptedPayload,
-            retryCount: row.retry_count || 0,
-            priority: row.priority || 0,
-            createdAt: row.created_at
-          };
-        })
-      );
-      
-      if (messages.length > 0) {
-        log(`${LogTypes.POP} | Count: ${messages.length} | Namespace: ${namespace || 'ANY'} | Task: ${task || 'ANY'}`);
-      }
-      
-      return { messages };
-    });
-  };
-
   /**
    * Configure queue settings with namespace and task support
    * Updates queue configuration options and optionally sets namespace/task
@@ -1495,18 +959,444 @@ const popMessages = async (scope, options = {}) => {
     });
   };
 
+  /**
+   * ═══════════════════════════════════════════════════════════════════════════
+   * UNIFIED POP FUNCTION - Handles all 3 access patterns in one implementation
+   * ═══════════════════════════════════════════════════════════════════════════
+   * 
+   * This function unifies:
+   * 1. Direct partition access (queue + partition specified)
+   * 2. Queue-level access (queue only, find unlocked partition)
+   * 3. Filtered access (namespace/task, find matching queues + partitions)
+   * 
+   * Benefits over separate functions:
+   * - Single source of truth for lease acquisition
+   * - Consistent multi-candidate retry logic
+   * - Easier to maintain and optimize
+   * - Shared backpressure and metrics
+   */
+  const uniquePop = async (scope, options = {}) => {
+    const { 
+      queue, 
+      partition, 
+      namespace, 
+      task, 
+      consumerGroup 
+    } = scope;
+    
+    const { 
+      wait = false, 
+      timeout = config.QUEUE.DEFAULT_TIMEOUT, 
+      batch = config.QUEUE.DEFAULT_BATCH_SIZE,
+      subscriptionMode = null,
+      subscriptionFrom = null
+    } = options;
+    
+    const startTime = Date.now();
+    const actualConsumerGroup = consumerGroup || '__QUEUE_MODE__';
+    
+    // Determine access mode
+    const accessMode = partition ? 'direct' : (namespace || task ? 'filtered' : 'queue');
+    
+    return withTransaction(pool, async (client) => {
+      const connectionWaitTime = Date.now() - startTime;
+      if (connectionWaitTime > 50) {
+        log(`⚠️  : ${connectionWaitTime}ms`);
+      }
+      
+      // ─────────────────────────────────────────────────────────────────────
+      // PHASE 1: Find Candidate Partitions (mode-specific queries)
+      // ─────────────────────────────────────────────────────────────────────
+      
+      const candidateQueryStart = Date.now();
+      let candidatePartitions = [];
+      let queueInfo = null;
+      
+      if (accessMode === 'direct') {
+        // CASE 1: Direct partition access - single partition, no search needed
+        const result = await client.query(`
+          SELECT 
+            q.id as queue_id,
+            q.name as queue_name,
+            q.lease_time,
+            q.retry_limit,
+            q.delayed_processing,
+            q.max_wait_time_seconds,
+            q.window_buffer,
+            q.priority,
+            p.id as partition_id,
+            p.name as partition_name
+          FROM queen.queues q
+          JOIN queen.partitions p ON p.queue_id = q.id
+          WHERE q.name = $1 AND p.name = $2
+        `, [queue, partition]);
+        
+        if (result.rows.length > 0) {
+          candidatePartitions = [result.rows[0]];
+          queueInfo = result.rows[0];
+        }
+        
+      } else if (accessMode === 'queue') {
+        // CASE 2: Queue-level access - find unlocked partitions
+        const result = await client.query(`
+          SELECT 
+            q.id as queue_id,
+            q.name as queue_name,
+            q.lease_time,
+            q.retry_limit,
+            q.delayed_processing,
+            q.max_wait_time_seconds,
+            q.window_buffer,
+            q.priority,
+            p.id as partition_id,
+            p.name as partition_name
+          FROM queen.queues q
+          JOIN queen.partitions p ON p.queue_id = q.id
+          WHERE q.name = $2
+            AND NOT EXISTS (
+              SELECT 1 FROM queen.partition_consumers pc
+              WHERE pc.partition_id = p.id
+                AND pc.consumer_group = $1
+                AND pc.lease_expires_at IS NOT NULL
+                AND pc.lease_expires_at > NOW()
+            )
+          ORDER BY RANDOM()
+          LIMIT $3
+        `, [actualConsumerGroup, queue, config.QUEUE.MAX_PARTITION_CANDIDATES]);
+        
+        if (result.rows.length > 0) {
+          candidatePartitions = result.rows;
+          queueInfo = result.rows[0];
+        }
+        
+      } else {
+        // CASE 3: Filtered access - namespace/task
+        let filterQuery = `
+          SELECT 
+            q.id as queue_id,
+            q.name as queue_name,
+            q.lease_time,
+            q.retry_limit,
+            q.delayed_processing,
+            q.max_wait_time_seconds,
+            q.window_buffer,
+            q.priority,
+            p.id as partition_id,
+            p.name as partition_name,
+            MIN(m.created_at) as oldest_message
+          FROM queen.messages m
+          JOIN queen.partitions p ON m.partition_id = p.id
+          JOIN queen.queues q ON p.queue_id = q.id
+          LEFT JOIN queen.partition_consumers pc ON p.id = pc.partition_id
+            AND pc.consumer_group = $1
+          WHERE m.id > COALESCE(pc.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid)
+            AND (q.delayed_processing = 0 OR m.created_at <= NOW() - INTERVAL '1 second' * q.delayed_processing)
+            AND (q.max_wait_time_seconds = 0 OR 
+                 m.created_at > NOW() - INTERVAL '1 second' * q.max_wait_time_seconds)
+            AND (q.window_buffer = 0 OR NOT EXISTS (
+              SELECT 1 FROM queen.messages m2 
+              WHERE m2.partition_id = p.id 
+                AND m2.created_at > NOW() - INTERVAL '1 second' * q.window_buffer
+            ))
+            AND NOT EXISTS (
+              SELECT 1 FROM queen.partition_consumers pc2
+              WHERE pc2.partition_id = p.id
+                AND pc2.consumer_group = $1
+                AND pc2.lease_expires_at IS NOT NULL
+                AND pc2.lease_expires_at > NOW()
+            )
+        `;
+        
+        const params = [actualConsumerGroup];
+        
+        if (namespace) {
+          params.push(namespace);
+          filterQuery += ` AND q.namespace = $${params.length}`;
+        }
+        
+        if (task) {
+          params.push(task);
+          filterQuery += ` AND q.task = $${params.length}`;
+        }
+        
+        params.push(config.QUEUE.MAX_PARTITION_CANDIDATES);
+        filterQuery += `
+          GROUP BY q.id, q.name, q.lease_time, q.retry_limit, q.delayed_processing,
+                   q.max_wait_time_seconds, q.window_buffer, q.priority,
+                   p.id, p.name
+          HAVING COUNT(*) > 0
+          ORDER BY q.priority DESC, MIN(m.created_at) ASC, RANDOM()
+          LIMIT $${params.length}
+        `;
+        
+        const result = await client.query(filterQuery, params);
+        
+        if (result.rows.length > 0) {
+          candidatePartitions = result.rows;
+          queueInfo = result.rows[0];
+        }
+      }
+      
+      const candidateQueryTime = Date.now() - candidateQueryStart;
+      
+      if (candidatePartitions.length === 0) {
+        return { messages: [] };
+      }
+      
+      // ─────────────────────────────────────────────────────────────────────
+      // PHASE 2: Try to Acquire Lease on a Candidate Partition (shared logic)
+      // ─────────────────────────────────────────────────────────────────────
+      
+      let acquiredPartition = null;
+      const MAX_ATTEMPTS = accessMode === 'direct' ? 1 : 3;
+      
+      for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+        // Shuffle candidates to distribute across consumers
+        const shuffled = [...candidatePartitions];
+        for (let i = shuffled.length - 1; i > 0; i--) {
+          const j = Math.floor(Math.random() * (i + 1));
+          [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+        }
+        
+        // Try to acquire lease on each candidate
+        for (const candidate of shuffled) {
+          const leaseAcqStart = Date.now();
+          
+          // Try to acquire lease - check if lease is available based on DB state
+          // The key insight: released_at IS NULL means someone holds the lease
+          // We can only acquire if: released_at IS NOT NULL OR lease expired
+          try {
+            // Get or create cursor position for this partition/consumer group
+            // Handle subscription modes for bus mode (new-only, from-timestamp, etc.)
+            let initialCursorId = '00000000-0000-0000-0000-000000000000';
+            let initialCursorTimestamp = null;
+            
+            // For bus mode with subscription preferences
+            if (consumerGroup && (subscriptionMode || subscriptionFrom)) {
+              if (subscriptionMode === 'new' || subscriptionMode === 'new-only' || subscriptionFrom === 'now') {
+                // Start from NOW - get the latest message ID to use as cursor
+                const latestMsg = await client.query(`
+                  SELECT id, created_at FROM queen.messages
+                  WHERE partition_id = $1
+                  ORDER BY created_at DESC, id DESC
+                  LIMIT 1
+                `, [candidate.partition_id]);
+                
+                if (latestMsg.rows.length > 0) {
+                  initialCursorId = latestMsg.rows[0].id;
+                  initialCursorTimestamp = latestMsg.rows[0].created_at;
+                }
+              }
+              // else: subscriptionFrom='all' or no preference → start from beginning (00000000...)
+            }
+            
+            // Step 1: Ensure cursor exists (idempotent)
+            await client.query(`
+              INSERT INTO queen.partition_consumers (
+                partition_id, 
+                consumer_group, 
+                last_consumed_id, 
+                last_consumed_created_at
+              )
+              VALUES ($1, $2, $3, $4)
+              ON CONFLICT (partition_id, consumer_group) DO NOTHING
+            `, [candidate.partition_id, actualConsumerGroup, initialCursorId, initialCursorTimestamp]);
+            
+            // Step 2: Try to acquire lease and get cursor position
+            const leaseResult = await client.query(`
+              UPDATE queen.partition_consumers
+              SET lease_expires_at = NOW() + INTERVAL '1 second' * $3,
+                  lease_acquired_at = NOW(),
+                  message_batch = NULL,
+                  batch_size = 0,
+                  acked_count = 0
+              WHERE partition_id = $1 
+                AND consumer_group = $2
+                AND (lease_expires_at IS NULL OR lease_expires_at <= NOW())
+              RETURNING partition_id, last_consumed_id, last_consumed_created_at, 
+                        total_messages_consumed, lease_expires_at
+            `, [candidate.partition_id, actualConsumerGroup, candidate.lease_time]);
+            
+            // Check if we acquired the lease
+            const acquired = leaseResult.rows.length > 0;
+            const result = leaseResult.rows[0];
+            
+            if (acquired) {
+              // Lease acquired! Now check if this partition has messages
+              // If not, release and try next candidate
+              const messageQueryStart = Date.now();
+              
+              const cursor = result;
+              const cursorId = cursor.last_consumed_id || '00000000-0000-0000-0000-000000000000';
+              const totalConsumed = cursor.total_messages_consumed || 0;
+              
+              // Cursor-based query using ONLY UUID comparison (UUIDv7 is time-ordered)
+              // Conditionally add timestamp filters only when configured
+              const hasDelayedProcessing = candidate.delayed_processing > 0;
+              const hasEviction = candidate.max_wait_time_seconds > 0;
+              
+              let whereClause = 'm.partition_id = $4 AND m.id > $5::uuid';
+              let queryParams = [
+                candidate.partition_name,
+                candidate.queue_name,
+                candidate.priority || 0,
+                candidate.partition_id,
+                cursorId
+              ];
+              let paramIndex = 6;
+              
+              if (hasDelayedProcessing) {
+                whereClause += ` AND m.created_at <= NOW() - INTERVAL '1 second' * $${paramIndex}`;
+                queryParams.push(candidate.delayed_processing);
+                paramIndex++;
+              }
+              
+              if (hasEviction) {
+                whereClause += ` AND m.created_at > NOW() - INTERVAL '1 second' * $${paramIndex}`;
+                queryParams.push(candidate.max_wait_time_seconds);
+                paramIndex++;
+              }
+              
+              queryParams.push(batch);
+              const limitParam = `$${paramIndex}`;
+              
+              const messagesResult = await client.query(`
+                SELECT 
+                  m.id,
+                  m.transaction_id,
+                  m.trace_id,
+                  m.payload,
+                  m.is_encrypted,
+                  m.created_at,
+                  $1 as partition_name,
+                  $2 as queue_name,
+                  $3 as priority,
+                  0 as retry_count
+                FROM queen.messages m
+                WHERE ${whereClause}
+                ORDER BY m.created_at ASC, m.id ASC
+                LIMIT ${limitParam}
+                FOR UPDATE OF m SKIP LOCKED
+              `, queryParams)
+              .catch(err => {
+                if (err.code === '55P03') { // Lock not available
+                  return { rows: [] };
+                }
+                throw err;
+              });
+              
+              const messageQueryTime = Date.now() - messageQueryStart;
+              const messages = messagesResult.rows;
+              
+              // Check if we found messages
+              if (messages.length === 0) {
+                // No messages in this partition (cursor exhausted) - release lease and try next candidate
+                log(`[CURSOR] Partition ${candidate.partition_name} exhausted after ${totalConsumed} messages`);
+                await client.query(`
+                  UPDATE queen.partition_consumers
+                  SET lease_expires_at = NULL,
+                      lease_acquired_at = NULL
+                  WHERE partition_id = $1 AND consumer_group = $2
+                `, [candidate.partition_id, actualConsumerGroup]);
+                continue; // Try next candidate partition
+              }
+              
+              // Success! We have messages from this partition
+              acquiredPartition = candidate;
+              
+              // ─────────────────────────────────────────────────────────────────────
+              // PHASE 4: Update Partition Lease with Message Batch
+              // ─────────────────────────────────────────────────────────────────────
+              
+              const leaseUpdateStart = Date.now();
+              const messageIds = messages.map(m => m.id);
+              await client.query(`
+                UPDATE queen.partition_consumers
+                SET message_batch = $1::jsonb,
+                    batch_size = $2::integer,
+                    acked_count = 0
+                WHERE partition_id = $3 AND consumer_group = $4
+              `, [JSON.stringify(messageIds), messageIds.length, acquiredPartition.partition_id, actualConsumerGroup]);
+              const leaseUpdateTime = Date.now() - leaseUpdateStart;
+              
+              log(`[CURSOR-LEASE] Partition: ${acquiredPartition.partition_name} | Leased ${messageIds.length} messages for processing`);
+              
+              // ─────────────────────────────────────────────────────────────────────
+              // PHASE 5: Decrypt and Format Messages
+              // ─────────────────────────────────────────────────────────────────────
+              
+              const decryptStart = Date.now();
+              const formattedMessages = await Promise.all(messages.map(async (msg) => {
+                let payload = msg.payload;
+                
+                if (msg.is_encrypted && encryption.isEncryptionEnabled()) {
+                  try {
+                    payload = await encryption.decryptPayload(msg.payload);
+                  } catch (error) {
+                    log(`${LogTypes.ERROR} | Failed to decrypt message ${msg.transaction_id}: ${error.message}`);
+                  }
+                }
+                
+                return {
+                  id: msg.id,
+                  transactionId: msg.transaction_id,
+                  traceId: msg.trace_id,
+                  queue: msg.queue_name,
+                  partition: msg.partition_name,
+                  data: payload,
+                  payload: payload,
+                  retryCount: msg.retry_count || 0,
+                  priority: msg.priority || 0,
+                  createdAt: msg.created_at,
+                  consumerGroup: consumerGroup || null
+                };
+              }));
+              const decryptTime = Date.now() - decryptStart;
+              
+              // ─────────────────────────────────────────────────────────────────────
+              // PHASE 6: Metrics and Logging
+              // ─────────────────────────────────────────────────────────────────────
+              
+              const totalTime = Date.now() - startTime;
+              const overhead = totalTime - (connectionWaitTime + candidateQueryTime + messageQueryTime + leaseUpdateTime + decryptTime);
+              const throughput = totalTime > 0 ? (formattedMessages.length / (totalTime / 1000)).toFixed(0) : 0;
+              
+              log(`[${LogTypes.POP}] Mode: ${accessMode} | Count: ${formattedMessages.length} | ${throughput} msg/s | ConsumerGroup: ${actualConsumerGroup === '__QUEUE_MODE__' ? 'QUEUE_MODE' : actualConsumerGroup}`);
+              log(`  ⏱️  ConnWait:${connectionWaitTime.toFixed(0)}ms CandidateQ:${candidateQueryTime.toFixed(0)}ms MsgQuery:${messageQueryTime.toFixed(0)}ms LeaseUpd:${leaseUpdateTime.toFixed(0)}ms Decrypt:${decryptTime.toFixed(0)}ms Overhead:${overhead.toFixed(0)}ms Total:${totalTime.toFixed(0)}ms`);
+              
+              return { messages: formattedMessages };
+            }
+          } catch (err) {
+            // Unexpected error
+            if (!candidate.queue_name.startsWith('__')) {
+              log(`LEASE ERROR: Partition=${candidate.partition_name} | Error: ${err.message}`);
+            }
+            throw err;
+          }
+        }
+        
+        if (acquiredPartition) break;
+      }
+      
+      // If we exhausted all attempts and couldn't get messages
+      if (!acquiredPartition) {
+        log(`WARN: Failed to acquire any partition with messages after ${MAX_ATTEMPTS} attempts`);
+        return { messages: [] };
+      }
+    });
+  };
+
   // Export all functions
   return {
     pushMessages: pushMessagesBatch,
     pushMessagesBatch,
-    popMessages,
     acknowledgeMessage,
     acknowledgeMessages,
     ackMessage: acknowledgeMessage,  // Alias for compatibility
     reclaimExpiredLeases,
     getQueueStats,
     getQueueLag,
-    popMessagesWithFilters,
+    uniquePop,  // ← New unified function
     configureQueue
   };
 };