queen-mq 0.1.1 → 0.1.2

package/src/services/evictionService.js

@@ -5,37 +5,66 @@ import config from '../config.js';
 const EVICTION_INTERVAL = config.JOBS.EVICTION_INTERVAL;
 
 // Evict messages that exceeded max wait time for a queue
+// Note: Eviction is already handled by POP query filtering old messages
+// This service optionally moves them to DLQ for visibility
 const evictMessages = async (client, queueName, maxWaitTimeSeconds) => {
   if (!maxWaitTimeSeconds || maxWaitTimeSeconds <= 0) return 0;
   
-  // For the new schema, we need to update or create status entries for evicted messages
-  const result = await client.query(`
-    WITH queue_partitions AS (
-      SELECT p.id
-      FROM queen.partitions p
-      JOIN queen.queues q ON p.queue_id = q.id
-      WHERE q.name = $1
-    ),
-    eligible_messages AS (
-      SELECT m.id
-      FROM queen.messages m
-      JOIN queue_partitions qp ON m.partition_id = qp.id
-      LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group IS NULL
-      WHERE m.created_at < NOW() - INTERVAL '1 second' * $2
-        AND (ms.status = 'pending' OR ms.status IS NULL)
-    )
-    INSERT INTO queen.messages_status (message_id, consumer_group, status, completed_at, error_message)
-    SELECT id, NULL, 'evicted', NOW(), 'Message exceeded maximum wait time'
-    FROM eligible_messages
-    ON CONFLICT (message_id, consumer_group) 
-    DO UPDATE SET 
-      status = 'evicted',
-      completed_at = NOW(),
-      error_message = 'Message exceeded maximum wait time'
-    RETURNING message_id
-  `, [queueName, maxWaitTimeSeconds]);
-  
-  return result.rowCount || 0;
+  try {
+    // Find old messages that haven't been consumed and move to DLQ
+    // Use a WHERE EXISTS check to ensure message still exists before inserting into DLQ
+    const result = await client.query(`
+      WITH queue_partitions AS (
+        SELECT p.id, p.queue_id
+        FROM queen.partitions p
+        JOIN queen.queues q ON p.queue_id = q.id
+        WHERE q.name = $1
+      ),
+      old_unconsumed_messages AS (
+        SELECT m.id, m.partition_id, m.created_at
+        FROM queen.messages m
+        JOIN queue_partitions qp ON m.partition_id = qp.id
+        LEFT JOIN queen.partition_consumers pc ON pc.partition_id = m.partition_id
+          AND pc.consumer_group = '__QUEUE_MODE__'
+        WHERE m.created_at < NOW() - INTERVAL '1 second' * $2
+          -- Message has not been consumed yet
+          AND ((m.created_at, m.id) > (COALESCE(pc.last_consumed_created_at, '1970-01-01'::timestamptz),
+                                         COALESCE(pc.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid))
+               OR pc.last_consumed_id IS NULL)
+      )
+      INSERT INTO queen.dead_letter_queue (
+          message_id, 
+          partition_id, 
+          consumer_group, 
+          error_message, 
+          original_created_at
+      )
+      SELECT 
+          om.id,
+          om.partition_id,
+          '__QUEUE_MODE__',
+          'Message exceeded maximum wait time',
+          om.created_at
+      FROM old_unconsumed_messages om
+      WHERE EXISTS (
+          SELECT 1 FROM queen.messages m WHERE m.id = om.id
+      )
+      ON CONFLICT DO NOTHING
+      RETURNING message_id
+    `, [queueName, maxWaitTimeSeconds]);
+    
+    return result.rowCount || 0;
+  } catch (error) {
+    // Catch foreign key violations (messages deleted between SELECT and INSERT)
+    if (error.code === '23503') {
+      // Foreign key constraint violation - message was deleted
+      // This is a harmless race condition, just log it
+      log(`${LogTypes.EVICTION} | Queue: ${queueName} | Warning: Message deleted during eviction (race condition)`);
+      return 0;
+    }
+    // Re-throw other errors
+    throw error;
+  }
 };
 
 // Evict messages during pop operation (inline eviction)

package/src/services/retentionService.js

@@ -16,30 +16,44 @@ const retentionQueue = async (client, queue) => {
   
   let totalDeleted = 0;
   
-  // Delete old pending messages from all partitions in this queue
+  // Delete old unconsumed messages from all partitions in this queue
   if (retentionSeconds > 0) {
     const result = await client.query(`
-      DELETE FROM queen.messages
-      WHERE partition_id IN (
+      DELETE FROM queen.messages m
+      WHERE m.partition_id IN (
         SELECT id FROM queen.partitions WHERE queue_id = $1
       )
-        AND status = 'pending'
-        AND created_at < NOW() - INTERVAL '1 second' * $2
+      AND m.created_at < NOW() - INTERVAL '1 second' * $2
+      AND NOT EXISTS (
+          -- Message has been consumed by at least one consumer
+          SELECT 1 
+          FROM queen.partition_consumers pc
+          WHERE pc.partition_id = m.partition_id
+            AND (m.created_at, m.id) <= (pc.last_consumed_created_at, pc.last_consumed_id)
+      )
       RETURNING id
     `, [queue.id, retentionSeconds]);
     
     totalDeleted += result.rowCount || 0;
   }
   
-  // Delete completed/failed/evicted messages from all partitions in this queue
+  // Delete old consumed messages from all partitions in this queue
+  // (messages consumed by ALL consumer groups on that partition)
   if (completedRetentionSeconds > 0) {
     const result = await client.query(`
-      DELETE FROM queen.messages
-      WHERE partition_id IN (
+      DELETE FROM queen.messages m
+      WHERE m.partition_id IN (
         SELECT id FROM queen.partitions WHERE queue_id = $1
       )
-        AND status IN ('completed', 'failed', 'evicted')
-        AND COALESCE(completed_at, failed_at, created_at) < NOW() - INTERVAL '1 second' * $2
+      AND m.created_at < NOW() - INTERVAL '1 second' * $2
+      AND NOT EXISTS (
+          -- No consumer group that hasn't consumed this message yet
+          SELECT 1 
+          FROM queen.partition_consumers pc
+          WHERE pc.partition_id = m.partition_id
+            AND ((m.created_at, m.id) > (pc.last_consumed_created_at, pc.last_consumed_id)
+                 OR pc.last_consumed_id IS NULL)
+      )
       RETURNING id
     `, [queue.id, completedRetentionSeconds]);
     
@@ -48,7 +62,6 @@ const retentionQueue = async (client, queue) => {
   
   // Log retention if messages were deleted
   if (totalDeleted > 0) {
-    // Note: We might need to update retention_history table to use queue_id instead
     log(`${LogTypes.RETENTION} | Queue: ${queue.name} | Count: ${totalDeleted} | RetentionSeconds: ${retentionSeconds} | CompletedRetentionSeconds: ${completedRetentionSeconds}`);
   }
   
@@ -76,6 +89,23 @@ const cleanupEmptyPartitions = async (client) => {
   return result.rowCount;
 };
 
+// Clean up old metrics data (messages_consumed table)
+const cleanupOldMetrics = async (client) => {
+  const retentionDays = config.JOBS.METRICS_RETENTION_DAYS;
+  
+  // Delete metrics data older than the retention period
+  const result = await client.query(`
+    DELETE FROM queen.messages_consumed
+    WHERE acked_at < NOW() - INTERVAL '1 day' * $1
+  `, [retentionDays]);
+  
+  if (result.rowCount > 0) {
+    log(`📊 Deleted ${result.rowCount} old metrics records (older than ${retentionDays} days)`);
+  }
+  
+  return result.rowCount;
+};
+
 // Main retention function
 const performRetention = async (pool) => {
   const client = await pool.connect();
@@ -97,6 +127,9 @@ const performRetention = async (pool) => {
     // Cleanup empty partitions
     await cleanupEmptyPartitions(client);
     
+    // Cleanup old metrics data
+    await cleanupOldMetrics(client);
+    
     return totalDeleted;
   } catch (error) {
     log('Retention error:', error);

package/src/test/advanced-pattern-tests.js

@@ -217,12 +217,12 @@ export async function testDeadLetterQueuePattern(client) {
       
       for (const msg of messages) {
         if (msg.data.shouldFail) {
-          // Check retry count
+          // Check if message is in DLQ
           const retryCheck = await dbPool.query(`
-            SELECT ms.retry_count 
-            FROM queen.messages_status ms
-            JOIN queen.messages m ON ms.message_id = m.id
-            WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__'
+            SELECT dlq.retry_count 
+            FROM queen.dead_letter_queue dlq
+            JOIN queen.messages m ON dlq.message_id = m.id
+            WHERE m.transaction_id = $1 AND dlq.consumer_group = '__QUEUE_MODE__'
           `, [msg.transactionId]);
           
           const retryCount = retryCheck.rows[0]?.retry_count || 0;

package/src/test/bus-mode-tests.js

@@ -320,24 +320,37 @@ export async function testConsumerGroupIsolation(client) {
       }
     }
     
-    // Verify isolation: Check that group1's first message is completed for group1 but failed for group2
+    // Verify isolation: Check that group1's first message is completed (cursor advanced) and group2 failed (in DLQ)
     const statusCheck = await dbPool.query(`
-      SELECT ms.status, ms.consumer_group
-      FROM queen.messages_status ms
-      JOIN queen.messages m ON ms.message_id = m.id
+      SELECT 
+        m.id as message_id,
+        m.transaction_id,
+        CASE 
+          WHEN dlq1.id IS NOT NULL THEN 'failed'
+          WHEN m.id <= COALESCE(pc1.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid) THEN 'completed'
+          ELSE 'pending'
+        END as group1_status,
+        CASE
+          WHEN dlq2.id IS NOT NULL THEN 'failed'
+          WHEN m.id <= COALESCE(pc2.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid) THEN 'completed'
+          ELSE 'pending'
+        END as group2_status
+      FROM queen.messages m
+      LEFT JOIN queen.partition_consumers pc1 ON pc1.partition_id = m.partition_id AND pc1.consumer_group = 'group1'
+      LEFT JOIN queen.partition_consumers pc2 ON pc2.partition_id = m.partition_id AND pc2.consumer_group = 'group2'
+      LEFT JOIN queen.dead_letter_queue dlq1 ON dlq1.message_id = m.id AND dlq1.consumer_group = 'group1'
+      LEFT JOIN queen.dead_letter_queue dlq2 ON dlq2.message_id = m.id AND dlq2.consumer_group = 'group2'
       WHERE m.transaction_id = $1
-      ORDER BY ms.consumer_group
     `, [group1Messages[0].transactionId]);
     
-    const group1Status = statusCheck.rows.find(r => r.consumer_group === 'group1');
-    const group2Status = statusCheck.rows.find(r => r.consumer_group === 'group2');
+    const result = statusCheck.rows[0];
     
-    if (group1Status?.status !== 'completed') {
-      throw new Error(`Group1 status should be completed, got ${group1Status?.status}`);
+    if (result?.group1_status !== 'completed') {
+      throw new Error(`Group1 status should be completed, got ${result?.group1_status}`);
     }
     
-    if (group2Status?.status !== 'failed') {
-      throw new Error(`Group2 status should be failed, got ${group2Status?.status}`);
+    if (result?.group2_status !== 'failed') {
+      throw new Error(`Group2 status should be failed, got ${result?.group2_status}`);
     }
     
     passTest('Consumer groups are properly isolated');

package/src/test/edge-case-tests.js

@@ -268,12 +268,19 @@ export async function testLeaseExpiration(client) {
     // Wait for lease to expire (2 seconds + buffer)
     await sleep(2500);
     
-    // Check message status in database
+    // Check partition lease status in database
     const statusCheck = await dbPool.query(`
-      SELECT status, lease_expires_at 
-      FROM queen.messages_status ms
-      JOIN queen.messages m ON ms.message_id = m.id
-      WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__'
+      SELECT pc.lease_expires_at,
+             CASE 
+               WHEN (m.created_at, m.id) <= (pc.last_consumed_created_at, COALESCE(pc.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid)) 
+               THEN 'completed'
+               WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() 
+               THEN 'processing'
+               ELSE 'pending'
+             END as status
+      FROM queen.messages m
+      LEFT JOIN queen.partition_consumers pc ON pc.partition_id = m.partition_id AND pc.consumer_group = '__QUEUE_MODE__'
+      WHERE m.transaction_id = $1
     `, [firstTransactionId]);
     
     if (statusCheck.rows.length > 0) {

package/src/test/enterprise-tests.js

@@ -184,13 +184,16 @@ export async function testMessageEviction(client) {
       timestamp: Date.now()
     });
     
-    // Verify message exists
+    // Verify message exists and is pending
     const pendingCheck = await dbPool.query(`
-      SELECT m.id, m.created_at, ms.status
+      SELECT m.id, m.created_at,
+             pc.last_consumed_id, pc.last_consumed_created_at,
+             dlq.id as dlq_id
       FROM queen.messages m
       JOIN queen.partitions p ON m.partition_id = p.id
       JOIN queen.queues q ON p.queue_id = q.id
-      LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
+      LEFT JOIN queen.partition_consumers pc ON pc.partition_id = p.id AND pc.consumer_group = '__QUEUE_MODE__'
+      LEFT JOIN queen.dead_letter_queue dlq ON dlq.message_id = m.id AND dlq.consumer_group = '__QUEUE_MODE__'
       WHERE q.name = $1 AND p.name = 'eviction-partition'
       ORDER BY m.created_at DESC
       LIMIT 1
@@ -200,7 +203,14 @@ export async function testMessageEviction(client) {
       throw new Error('Message not found');
     }
     
-    const status = pendingCheck.rows[0].status || 'pending';
+    // Derive status from cursor position
+    const msg = pendingCheck.rows[0];
+    const isDLQ = msg.dlq_id != null;
+    const isConsumed = msg.last_consumed_id && 
+                       ((msg.created_at < msg.last_consumed_created_at) || 
+                        (msg.created_at.getTime() === msg.last_consumed_created_at?.getTime() && msg.id <= msg.last_consumed_id));
+    const status = isDLQ ? 'dead_letter' : (isConsumed ? 'completed' : 'pending');
+    
     if (status !== 'pending') {
       throw new Error(`Message not in pending status, got: ${status}`);
     }
@@ -215,13 +225,14 @@ export async function testMessageEviction(client) {
       gotMessage = true;
     }
     
-    // Check if message was evicted
+    // Check if message was evicted (moved to DLQ)
     const evictionCheck = await dbPool.query(`
-      SELECT m.id, ms.status, ms.error_message
+      SELECT m.id, dlq.error_message,
+             CASE WHEN dlq.id IS NOT NULL THEN 'evicted' ELSE 'pending' END as status
       FROM queen.messages m
       JOIN queen.partitions p ON m.partition_id = p.id
       JOIN queen.queues q ON p.queue_id = q.id
-      LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
+      LEFT JOIN queen.dead_letter_queue dlq ON dlq.message_id = m.id AND dlq.consumer_group = '__QUEUE_MODE__'
       WHERE q.name = $1 AND p.name = 'eviction-partition'
       ORDER BY m.created_at DESC
       LIMIT 1
@@ -382,16 +393,32 @@ export async function testRetentionCompletedMessages(client) {
       throw new Error('Failed to take message for completion');
     }
     
-    // Verify message is marked as completed
+    // Verify message is marked as completed (cursor-based check using UUIDv7 ordering)
     const statusCheck = await dbPool.query(`
-      SELECT ms.status 
-      FROM queen.messages_status ms
-      JOIN queen.messages m ON ms.message_id = m.id
-      WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__'
+      SELECT 
+        m.id, m.created_at,
+        pc.last_consumed_id, pc.last_consumed_created_at,
+        CASE 
+          WHEN pc.last_consumed_id IS NOT NULL AND m.id <= pc.last_consumed_id
+          THEN 'completed'
+          ELSE 'pending'
+        END as status
+      FROM queen.messages m
+      LEFT JOIN queen.partition_consumers pc 
+        ON pc.partition_id = m.partition_id 
+        AND pc.consumer_group = '__QUEUE_MODE__'
+      WHERE m.transaction_id = $1
     `, [transactionId]);
     
     if (!statusCheck.rows[0] || statusCheck.rows[0].status !== 'completed') {
-      throw new Error('Message not marked as completed');
+      const row = statusCheck.rows[0];
+      throw new Error(
+        `Message not marked as completed.\n` +
+        `  Status: ${row?.status}\n` +
+        `  Message ID: ${row?.id}\n` +
+        `  Cursor ID: ${row?.last_consumed_id}\n` +
+        `  Has cursor: ${row?.last_consumed_id != null}`
+      );
     }
     
     // Wait for retention period
@@ -405,9 +432,15 @@ export async function testRetentionCompletedMessages(client) {
     
     // Check if completed message still exists
     const retainedCheck = await dbPool.query(`
-      SELECT m.id, ms.status, ms.completed_at
+      SELECT m.id, 
+             CASE 
+               WHEN (m.created_at, m.id) <= (pc.last_consumed_created_at, COALESCE(pc.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid)) 
+               THEN 'completed' 
+               ELSE 'pending' 
+             END as status,
+             pc.last_consumed_at as completed_at
       FROM queen.messages m
-      LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
+      LEFT JOIN queen.partition_consumers pc ON pc.partition_id = m.partition_id AND pc.consumer_group = '__QUEUE_MODE__'
       WHERE m.transaction_id = $1
     `, [transactionId]);
     

package/src/test/utils.js

@@ -6,7 +6,7 @@ import pg from 'pg';
 
 // Test configuration
 export const TEST_CONFIG = {
-  baseUrls: ['http://localhost:6632', 'http://localhost:6633'],
+  baseUrls: ['http://localhost:6632'],
   dbConfig: {
     host: process.env.PG_HOST || 'localhost',
     port: process.env.PG_PORT || 5432,

package/src/utils/streaming.js

@@ -0,0 +1,231 @@
+/**
+ * Streaming utilities for uWebSockets.js with backpressure handling
+ * 
+ * When sending large responses (>1MB), we need to handle backpressure to prevent:
+ * - Event loop blocking
+ * - Memory buildup
+ * - Slow client stalling the server
+ * 
+ * uWebSockets.js provides efficient backpressure handling via:
+ * - res.write() - Returns false if buffer is full
+ * - res.onWritable() - Called when buffer has space again
+ * - res.cork() - Batches syscalls for efficiency
+ */
+
+import { log } from './logger.js';
+
+// Thresholds for streaming decisions
+const SMALL_RESPONSE_THRESHOLD = 128 * 1024;  // 128KB - send directly
+const CHUNK_SIZE = 64 * 1024;                  // 64KB - stream in chunks
+
+/**
+ * Send JSON response with automatic backpressure handling
+ * 
+ * For small responses (<128KB): Sends directly via res.end()
+ * For large responses (>=128KB): Streams with backpressure via res.write() + res.onWritable()
+ * 
+ * @param {Object} res - uWebSockets.js response object
+ * @param {Object} data - Data to serialize and send
+ * @param {Object} abortedRef - Reference object with 'aborted' flag
+ * @param {Object} options - Optional settings
+ * @returns {boolean} - True if sent successfully, false if aborted
+ */
+export const streamJSON = (res, data, abortedRef = null, options = {}) => {
+  // Check if already aborted
+  if (abortedRef && abortedRef.aborted) {
+    return false;
+  }
+  
+  const jsonStr = JSON.stringify(data);
+  const dataSize = jsonStr.length;
+  
+  // Small response - send directly (fast path)
+  if (dataSize < SMALL_RESPONSE_THRESHOLD) {
+    try {
+      res.cork(() => {
+        res.end(jsonStr);
+      });
+      return true;
+    } catch (error) {
+      log(`Error sending small response: ${error.message}`);
+      return false;
+    }
+  }
+  
+  // Large response - stream with backpressure handling
+  // uWebSockets.js requires all writes to be synchronous within cork callbacks
+  log(`Streaming large response: ${(dataSize / 1024 / 1024).toFixed(2)}MB`);
+  
+  let offset = 0;
+  
+  // Start sending - write as much as possible in first cork
+  res.cork(() => {
+    // Write chunks until buffer is full or done
+    while (offset < dataSize) {
+      const chunkEnd = Math.min(offset + CHUNK_SIZE, dataSize);
+      const chunk = jsonStr.slice(offset, chunkEnd);
+      const isLastChunk = chunkEnd >= dataSize;
+      
+      if (isLastChunk) {
+        // Last chunk - end the response
+        res.end(chunk);
+        log(`Streamed response complete: ${(dataSize / 1024).toFixed(2)}KB sent`);
+        return; // All done!
+      }
+      
+      const ok = res.write(chunk);
+      offset = chunkEnd;
+      
+      if (!ok) {
+        // Buffer full - stop and wait for drain
+        log(`Backpressure detected at offset ${offset}/${dataSize} (${((offset/dataSize)*100).toFixed(1)}%)`);
+        break; // Exit while loop, set up onWritable below
+      }
+      
+      // Successfully wrote, continue to next chunk
+    }
+    
+    // If we exited due to backpressure (offset < dataSize), set up handler
+    if (offset < dataSize) {
+      // Set up backpressure handler for remaining chunks
+      // onWritable is called when the socket buffer has space again
+      res.onWritable((writeOffset) => {
+        if (abortedRef && abortedRef.aborted) {
+          return true; // true = done, stop calling
+        }
+        
+        // CRITICAL: All writes inside onWritable must also be corked!
+        let isDone = true; // Will be set to false if we need to continue
+        
+        res.cork(() => {
+          // Write as many chunks as possible while buffer has space
+          while (offset < dataSize) {
+            const chunkEnd = Math.min(offset + CHUNK_SIZE, dataSize);
+            const chunk = jsonStr.slice(offset, chunkEnd);
+            const isLastChunk = chunkEnd >= dataSize;
+            
+            if (isLastChunk) {
+              // Last chunk - end the response
+              res.end(chunk);
+              log(`Streamed response complete: ${(dataSize / 1024).toFixed(2)}KB sent`);
+              isDone = true; // true = done, stop calling onWritable
+              return;
+            }
+            
+            const ok = res.write(chunk);
+            offset = chunkEnd;
+            
+            if (!ok) {
+              // Buffer full again - stop and wait for next onWritable call
+              log(`Backpressure detected at offset ${offset}/${dataSize} (${((offset/dataSize)*100).toFixed(1)}%)`);
+              isDone = false; // false = not done, call me again when writable
+              return;
+            }
+            
+            // Successfully wrote, try next chunk in this iteration
+          }
+          
+          // If we get here, all chunks sent
+          isDone = true; // Done
+        });
+        
+        return isDone;
+      });
+    }
+  });
+  
+  return true;
+};
+
+/**
+ * Send a simple JSON response (for small data)
+ * Always uses res.cork() for efficiency
+ */
+export const sendJSON = (res, data, statusCode = 200) => {
+  res.cork(() => {
+    res.writeStatus(statusCode.toString());
+    res.end(JSON.stringify(data));
+  });
+};
+
+/**
+ * Send error response with proper status code
+ * Note: CORS headers should be set by caller before calling this
+ */
+export const sendError = (res, error, statusCode = 500) => {
+  res.cork(() => {
+    res.writeStatus(statusCode.toString());
+    res.end(JSON.stringify({ 
+      error: error.message || error,
+      code: error.code || 'INTERNAL_ERROR'
+    }));
+  });
+};
+
+/**
+ * Stream large array in chunks (alternative approach for very large arrays)
+ * Serializes chunks separately to avoid large JSON.stringify() blocking
+ */
+export const streamJSONArray = (res, items, abortedRef = null) => {
+  if (abortedRef && abortedRef.aborted) return false;
+  
+  const ITEMS_PER_CHUNK = 1000;
+  let sent = 0;
+  
+  const sendNextChunk = () => {
+    if (abortedRef && abortedRef.aborted) return true;
+    
+    const chunk = items.slice(sent, sent + ITEMS_PER_CHUNK);
+    if (chunk.length === 0) {
+      // All sent
+      res.end(']');
+      return true;
+    }
+    
+    const isFirst = sent === 0;
+    const isLast = sent + chunk.length >= items.length;
+    
+    // Build chunk JSON
+    let chunkStr;
+    if (isFirst) {
+      chunkStr = '[' + chunk.map(item => JSON.stringify(item)).join(',');
+    } else {
+      chunkStr = ',' + chunk.map(item => JSON.stringify(item)).join(',');
+    }
+    
+    const ok = isLast 
+      ? res.end(chunkStr + ']')
+      : res.write(chunkStr);
+    
+    sent += chunk.length;
+    
+    if (!ok) {
+      // Backpressure
+      res.onWritable(() => sendNextChunk());
+      return false;
+    }
+    
+    if (!isLast) {
+      setImmediate(sendNextChunk);
+    }
+    
+    return true;
+  };
+  
+  res.cork(() => {
+    const ok = sendNextChunk();
+    if (!ok) {
+      res.onWritable(sendNextChunk);
+    }
+  });
+  
+  return true;
+};
+
+export default {
+  streamJSON,
+  sendJSON,
+  sendError,
+  streamJSONArray
+};
+

package/src/utils/uuid.js

@@ -1,5 +1,5 @@
-import { v4 as uuidv4 } from 'uuid';
+import { v7 as uuidv7 } from 'uuid';
 
 export const generateUUID = () => {
-  return uuidv4();
+  return uuidv7();
 };

package/src/websocket/wsServer.js

@@ -191,10 +191,17 @@ export const createWebSocketServer = (app, eventManager) => {
     try {
       const result = await pool.query(`
         SELECT
-          (SELECT COUNT(*) FROM queen.messages_status WHERE status = 'pending' AND consumer_group IS NULL) as pending,
-          (SELECT COUNT(*) FROM queen.messages_status WHERE status = 'processing' AND consumer_group IS NULL) as processing,
+          (SELECT COALESCE(SUM(pending_estimate), 0)::integer FROM queen.partition_consumers 
+           WHERE consumer_group = '__QUEUE_MODE__') as pending,
+          (SELECT COUNT(*) FROM queen.partition_consumers 
+           WHERE consumer_group = '__QUEUE_MODE__' 
+           AND lease_expires_at IS NOT NULL 
+           AND lease_expires_at > NOW()) as processing,
           (SELECT COUNT(*) FROM queen.messages WHERE created_at > NOW() - INTERVAL '1 minute') as recent_created,
-          (SELECT COUNT(*) FROM queen.messages_status WHERE completed_at > NOW() - INTERVAL '1 minute' AND consumer_group IS NULL) as recent_completed
+          (SELECT COALESCE(SUM(CASE WHEN last_consumed_at > NOW() - INTERVAL '1 minute' 
+                                    THEN total_batches_consumed ELSE 0 END), 0)::integer 
+           FROM queen.partition_consumers 
+           WHERE consumer_group = '__QUEUE_MODE__') as recent_completed
       `);
       
       const stats = result.rows[0];