queen-mq 0.1.0

package/src/utils/logger.js

@@ -0,0 +1,44 @@
+/**
+ * Centralized logging utility for Queen message queue
+ * Logs all state-changing operations with consistent formatting
+ */
+
+/**
+ * Log a message with timestamp and process ID
+ * @param {...any} args - Arguments to log (objects will be JSON stringified)
+ */
+export function log(...args) {
+  args = args.map(arg => typeof arg === 'object' ? JSON.stringify(arg) : arg);
+  const msg = args.join(' ');
+  console.log(new Date(), `| ${process.pid} | #> ${msg}`);
+}
+
+/**
+ * Log operation types for consistency
+ */
+export const LogTypes = {
+  // Message operations
+  PUSH: 'PUSH',
+  POP: 'POP',
+  POP_FILTERED: 'POP_FILTERED',
+  ACK: 'ACK',
+  ACK_BATCH: 'ACK_BATCH',
+  
+  // Message state changes
+  DELETE: 'DELETE',
+  RETRY: 'RETRY',
+  RETRY_SCHEDULED: 'RETRY_SCHEDULED',
+  MOVED_TO_DLQ: 'MOVED_TO_DLQ',
+  MANUAL_DLQ: 'MANUAL_DLQ',
+  
+  // Queue operations
+  CLEAR_QUEUE: 'CLEAR_QUEUE',
+  CONFIGURE_QUEUE: 'CONFIGURE_QUEUE',
+  
+  // System operations
+  EVICTION: 'EVICTION',
+  RETENTION: 'RETENTION',
+  LEASE_EXPIRED: 'LEASE_EXPIRED'
+};
+
+export default log;

package/src/utils/uuid.js

@@ -0,0 +1,5 @@
+import { v4 as uuidv4 } from 'uuid';
+
+export const generateUUID = () => {
+  return uuidv4();
+};

package/src/websocket/wsServer.js

@@ -0,0 +1,221 @@
+import { generateUUID } from '../utils/uuid.js';
+import config from '../config.js';
+
+export const createWebSocketServer = (app, eventManager) => {
+  const connections = new Map();
+  
+  // Broadcast event to all connected clients
+  const broadcast = (event, data) => {
+    const message = JSON.stringify({ event, data, timestamp: new Date().toISOString() });
+    
+    for (const [id, ws] of connections) {
+      try {
+        ws.send(message);
+      } catch (error) {
+        console.error(`Failed to send to client ${id}:`, error);
+        connections.delete(id);
+      }
+    }
+  };
+  
+  // Setup WebSocket route
+  app.ws('/ws/dashboard', {
+    compression: config.WEBSOCKET.COMPRESSION,
+    maxPayloadLength: config.WEBSOCKET.MAX_PAYLOAD_LENGTH,
+    idleTimeout: config.WEBSOCKET.IDLE_TIMEOUT,
+    
+    open: (ws) => {
+      const connectionId = generateUUID();
+      connections.set(connectionId, ws);
+      ws.connectionId = connectionId;
+      
+      console.log(`WebSocket client connected: ${connectionId}`);
+      
+      // Send welcome message
+      ws.send(JSON.stringify({
+        event: 'connected',
+        data: { 
+          connectionId,
+          version: 'v2'
+        },
+        timestamp: new Date().toISOString()
+      }));
+      
+      broadcast('client.connected', { clientId: connectionId });
+    },
+    
+    message: (ws, message, isBinary) => {
+      // Handle ping/pong
+      const msg = Buffer.from(message).toString();
+      if (msg === 'ping') {
+        ws.send('pong');
+      } else {
+        // Handle subscription requests
+        try {
+          const data = JSON.parse(msg);
+          if (data.type === 'subscribe') {
+            // Store subscription preferences (could be used for filtering)
+            ws.subscriptions = data.queues || [];
+            ws.send(JSON.stringify({
+              event: 'subscribed',
+              data: { queues: ws.subscriptions },
+              timestamp: new Date().toISOString()
+            }));
+          }
+        } catch (error) {
+          // Invalid JSON, ignore
+        }
+      }
+    },
+    
+    close: (ws, code, message) => {
+      const connectionId = ws.connectionId;
+      connections.delete(connectionId);
+      console.log(`WebSocket client disconnected: ${connectionId}`);
+      broadcast('client.disconnected', { clientId: connectionId });
+    }
+  });
+  
+  // Subscribe to queue events - Updated for V2 structure
+  eventManager.on('message.pushed', (data) => {
+    broadcast('message.pushed', {
+      queue: data.queue,
+      partition: data.partition,
+      transactionId: data.transactionId
+    });
+  });
+  
+  eventManager.on('message.processing', (data) => {
+    broadcast('message.processing', {
+      queue: data.queue,
+      partition: data.partition,
+      transactionId: data.transactionId,
+      workerId: data.workerId
+    });
+  });
+  
+  eventManager.on('message.completed', (data) => {
+    broadcast('message.completed', {
+      transactionId: data.transactionId
+    });
+  });
+  
+  eventManager.on('message.failed', (data) => {
+    broadcast('message.failed', {
+      transactionId: data.transactionId,
+      error: data.error
+    });
+  });
+  
+  eventManager.on('queue.created', (data) => {
+    broadcast('queue.created', {
+      queue: data.queue,
+      partition: data.partition ?? 'Default'
+    });
+  });
+  
+  // Periodic queue depth updates - Updated for V2
+  const updateQueueDepths = async (queueManager) => {
+    try {
+      const stats = await queueManager.getQueueStats();
+      
+      // Group by queue for aggregated depths
+      const queueDepths = {};
+      
+      for (const stat of stats) {
+        if (!queueDepths[stat.queue]) {
+          queueDepths[stat.queue] = {
+            queue: stat.queue,
+            namespace: stat.namespace,
+            task: stat.task,
+            totalDepth: 0,
+            totalProcessing: 0,
+            partitions: {}
+          };
+        }
+        
+        // Skip consumer group specific stats for queue depth calculation
+        // Only count NULL consumer_group (queue mode) stats
+        if (stat.consumer_group) continue;
+        
+        // Initialize partition if not exists
+        if (!queueDepths[stat.queue].partitions[stat.partition]) {
+          queueDepths[stat.queue].partitions[stat.partition] = {
+            depth: 0,
+            processing: 0,
+            completed: 0,
+            failed: 0
+          };
+        }
+        
+        // Add partition stats (using direct field names from new schema)
+        const partitionStats = queueDepths[stat.queue].partitions[stat.partition];
+        partitionStats.depth += parseInt(stat.pending || 0);
+        partitionStats.processing += parseInt(stat.processing || 0);
+        partitionStats.completed += parseInt(stat.completed || 0);
+        partitionStats.failed += parseInt(stat.failed || 0);
+        
+        // Update totals
+        queueDepths[stat.queue].totalDepth += parseInt(stat.pending || 0);
+        queueDepths[stat.queue].totalProcessing += parseInt(stat.processing || 0);
+      }
+      
+      // Broadcast aggregated queue depths
+      for (const queueData of Object.values(queueDepths)) {
+        broadcast('queue.depth', queueData);
+      }
+      
+      // Also broadcast individual partition depths for detailed monitoring
+      for (const stat of stats) {
+        // Skip consumer group specific stats
+        if (stat.consumer_group) continue;
+        
+        broadcast('partition.depth', {
+          queue: stat.queue,
+          partition: stat.partition,
+          depth: parseInt(stat.pending || 0),
+          processing: parseInt(stat.processing || 0),
+          completed: parseInt(stat.completed || 0),
+          failed: parseInt(stat.failed || 0),
+          total: parseInt(stat.total_messages || 0)
+        });
+      }
+      
+    } catch (error) {
+      console.error('Error updating queue depths:', error);
+    }
+  };
+  
+  // Send system stats periodically
+  const sendSystemStats = async (pool) => {
+    try {
+      const result = await pool.query(`
+        SELECT
+          (SELECT COUNT(*) FROM queen.messages_status WHERE status = 'pending' AND consumer_group IS NULL) as pending,
+          (SELECT COUNT(*) FROM queen.messages_status WHERE status = 'processing' AND consumer_group IS NULL) as processing,
+          (SELECT COUNT(*) FROM queen.messages WHERE created_at > NOW() - INTERVAL '1 minute') as recent_created,
+          (SELECT COUNT(*) FROM queen.messages_status WHERE completed_at > NOW() - INTERVAL '1 minute' AND consumer_group IS NULL) as recent_completed
+      `);
+      
+      const stats = result.rows[0];
+      
+      broadcast('system.stats', {
+        pending: parseInt(stats.pending || 0),
+        processing: parseInt(stats.processing || 0),
+        recentCreated: parseInt(stats.recent_created || 0),
+        recentCompleted: parseInt(stats.recent_completed || 0),
+        connections: connections.size
+      });
+    } catch (error) {
+      console.error('Error sending system stats:', error);
+    }
+  };
+  
+  return {
+    broadcast,
+    updateQueueDepths,
+    sendSystemStats,
+    getConnectionCount: () => connections.size,
+    getConnections: () => Array.from(connections.keys())
+  };
+};