queen-mq 0.1.0 → 0.1.2

package/src/client/client.js

@@ -55,6 +55,14 @@ export class Queen {
     this.#connected = true;
   }
   
+  /**
+   * Validate if a string is a valid UUID v4
+   */
+  #isValidUUID(str) {
+    const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+    return uuidRegex.test(str);
+  }
+  
   /**
    * Parse address string into components
    * Examples:
@@ -62,9 +70,37 @@ export class Queen {
    * - "myqueue/urgent" -> { queue: "myqueue", partition: "urgent" }
    * - "myqueue@workers" -> { queue: "myqueue", consumerGroup: "workers" }
    * - "myqueue/urgent@workers" -> { queue: "myqueue", partition: "urgent", consumerGroup: "workers" }
+   * - "namespace:billing" -> { namespace: "billing" }
+   * - "task:process" -> { task: "process" }
+   * - "namespace:billing/task:process" -> { namespace: "billing", task: "process" }
    */
   #parseAddress(address) {
-    // Match: queue[/partition][@group]
+    // Check for namespace/task pattern
+    if (address.includes('namespace:') || address.includes('task:')) {
+      const parts = {};
+      
+      // Extract consumer group if present (after @)
+      let workingAddress = address;
+      const atIndex = address.lastIndexOf('@');
+      if (atIndex > 0) {
+        parts.consumerGroup = address.substring(atIndex + 1);
+        workingAddress = address.substring(0, atIndex);
+      }
+      
+      const segments = workingAddress.split('/');
+      
+      for (const segment of segments) {
+        if (segment.startsWith('namespace:')) {
+          parts.namespace = segment.substring(10);
+        } else if (segment.startsWith('task:')) {
+          parts.task = segment.substring(5);
+        }
+      }
+      
+      return parts;
+    }
+    
+    // Standard queue[/partition][@group] pattern
     const match = address.match(/^([^/@]+)(?:\/([^@]+))?(?:@(.+))?$/);
     if (!match) {
       throw new Error(`Invalid address format: ${address}`);
@@ -81,13 +117,18 @@ export class Queen {
    * Configure a queue with options
    * @param {string} name - Queue name
    * @param {Object} options - Queue configuration options
+   * @param {Object} metadata - Optional metadata { namespace, task }
    */
-  async queue(name, options = {}) {
+  async queue(name, options = {}, metadata = {}) {
     await this.#ensureConnected();
     
+    const { namespace, task } = metadata;
+    
     const result = await withRetry(
       () => this.#http.post('/api/v1/configure', { 
         queue: name,
+        namespace: namespace || null,
+        task: task || null,
         options 
       }),
       this.#config.retryAttempts,
@@ -105,8 +146,9 @@ export class Queen {
    * Push messages to a queue
    * @param {string} address - Queue address (e.g., "myqueue" or "myqueue/partition")
    * @param {Object|Array} payload - Single message or array of messages
+   * @param {Object} options - Optional message properties { transactionId, traceId }
    */
-  async push(address, payload) {
+  async push(address, payload, options = {}) {
     await this.#ensureConnected();
     
     const { queue, partition } = this.#parseAddress(address);
@@ -115,12 +157,44 @@ export class Queen {
     const items = Array.isArray(payload) ? payload : [payload];
     
     // Format items for the API
-    const formattedItems = items.map(item => ({
-      queue,
-      partition,
-      payload: item,
-      transactionId: item.transactionId // Optional, for idempotency
-    }));
+    const formattedItems = items.map(item => {
+      // If item is an object with special properties, extract them
+      const isMessageObject = typeof item === 'object' && item !== null &&
+        (item._payload || item._transactionId || item._traceId);
+      
+      if (isMessageObject) {
+        const result = {
+          queue,
+          partition,
+          payload: item._payload || item,
+          transactionId: item._transactionId || item.transactionId
+        };
+        
+        // Include traceId if provided and valid UUID
+        const traceId = item._traceId || item.traceId;
+        if (traceId && this.#isValidUUID(traceId)) {
+          result.traceId = traceId;
+        }
+        
+        return result;
+      }
+      
+      // Otherwise use the item as payload and apply options
+      const result = {
+        queue,
+        partition,
+        payload: item,
+        transactionId: options.transactionId || (item && typeof item === 'object' ? item.transactionId : undefined)
+      };
+      
+      // Include traceId if provided and valid UUID
+      const traceId = options.traceId || (item && typeof item === 'object' ? item.traceId : undefined);
+      if (traceId && this.#isValidUUID(traceId)) {
+        result.traceId = traceId;
+      }
+      
+      return result;
+    });
     
     const result = await withRetry(
       () => this.#http.post('/api/v1/push', { 
@@ -138,38 +212,52 @@ export class Queen {
   }
   
   /**
-   * Take messages from a queue (async iterator)
-   * @param {string} address - Queue address (e.g., "myqueue", "myqueue/partition", "myqueue@group")
+   * Internal method that yields batches of messages (arrays)
+   * @private
+   * @param {string} address - Queue address
    * @param {Object} options - Options for taking messages
-   * @yields {Object} Message objects
+   * @yields {Array} Arrays of message objects
    */
-  async *take(address, options = {}) {
+  async *#takeInternal(address, options = {}) {
     await this.#ensureConnected();
     
-    const { queue, partition, consumerGroup } = this.#parseAddress(address);
+    const { queue, partition, consumerGroup, namespace, task } = this.#parseAddress(address);
     const { 
       limit = null,
       batch = 1, 
       wait = false, 
       timeout = 30000,
       subscriptionMode = null,
-      subscriptionFrom = null
+      subscriptionFrom = null,
+      idleTimeout = null
     } = options;
     
-    let count = 0;
+    let totalCount = 0;
     let consecutiveEmptyResponses = 0;
     const maxConsecutiveEmpty = 3;
+    let lastMessageTime = idleTimeout ? Date.now() : null;
     
     while (true) {
       // Check if we've reached the limit
-      if (limit && count >= limit) break;
+      if (limit && totalCount >= limit) break;
+      
+      // Check idle timeout
+      if (idleTimeout && lastMessageTime) {
+        const idleTime = Date.now() - lastMessageTime;
+        if (idleTime >= idleTimeout) {
+          break; // Exit if idle time exceeded
+        }
+      }
+      
+      // Calculate batch size respecting the limit
+      const effectiveBatch = limit ? Math.min(batch, limit - totalCount) : batch;
       
       // Build the request path and parameters
       let path;
       const params = new URLSearchParams({
         wait: wait.toString(),
         timeout: timeout.toString(),
-        batch: Math.min(batch, limit ? limit - count : batch).toString()
+        batch: effectiveBatch.toString()
       });
       
       // Add consumer group parameters if provided
@@ -178,10 +266,19 @@ export class Queen {
       if (subscriptionFrom) params.append('subscriptionFrom', subscriptionFrom);
       
       // Determine the endpoint based on parameters
-      if (partition && partition !== 'Default') {
-        path = `/api/v1/pop/queue/${queue}/partition/${partition}`;
+      if (queue) {
+        if (partition && partition !== 'Default') {
+          path = `/api/v1/pop/queue/${queue}/partition/${partition}`;
+        } else {
+          path = `/api/v1/pop/queue/${queue}`;
+        }
+      } else if (namespace || task) {
+        // Pop by namespace/task
+        path = '/api/v1/pop';
+        if (namespace) params.append('namespace', namespace);
+        if (task) params.append('task', task);
       } else {
-        path = `/api/v1/pop/queue/${queue}`;
+        throw new Error('Must specify either queue, namespace, or task');
       }
       
       try {
@@ -210,11 +307,22 @@ export class Queen {
         // Reset empty counter on successful fetch
         consecutiveEmptyResponses = 0;
         
-        // Yield each message
-        for (const message of result.messages) {
-          yield message;
-          count++;
-          if (limit && count >= limit) return;
+        // Update last message time if tracking idle timeout
+        if (idleTimeout) {
+          lastMessageTime = Date.now();
+        }
+        
+        // Filter out null/undefined messages
+        const messages = result.messages.filter(msg => msg != null);
+        
+        if (messages.length > 0) {
+          totalCount += messages.length;
+          yield messages;
+          
+          // If we've hit the limit, stop
+          if (limit && totalCount >= limit) {
+            break;
+          }
         }
         
       } catch (error) {
@@ -235,18 +343,154 @@ export class Queen {
   }
   
   /**
-   * Acknowledge a message
-   * @param {Object|string} message - Message object or transaction ID
+   * Take messages from a queue one at a time (async iterator)
+   * @param {string} address - Queue address (e.g., "myqueue", "myqueue/partition", "myqueue@group")
+   * @param {Object} options - Options for taking messages
+   * @yields {Object} Individual message objects
+   */
+  async *take(address, options = {}) {
+    let count = 0;
+    const { limit = null } = options;
+    
+    for await (const messages of this.#takeInternal(address, options)) {
+      for (const message of messages) {
+        yield message;
+        count++;
+        
+        // Double-check limit (internal method also checks, but this ensures exact limit)
+        if (limit && count >= limit) {
+          return;
+        }
+      }
+    }
+  }
+  
+  /**
+   * Take messages from a queue in batches (async iterator)
+   * @param {string} address - Queue address (e.g., "myqueue", "myqueue/partition", "myqueue@group")
+   * @param {Object} options - Options for taking messages
+   * @yields {Array} Arrays of message objects
+   */
+  async *takeBatch(address, options = {}) {
+    for await (const messages of this.#takeInternal(address, options)) {
+      // Only yield non-empty batches
+      if (messages && messages.length > 0) {
+        yield messages;
+      }
+    }
+  }
+  
+  /**
+   * Acknowledge a message or batch of messages
+   * @param {Object|string|Array} message - Message object, transaction ID, or array of messages
    * @param {boolean|string} status - true for success, false for failure, or 'retry'
    * @param {Object} context - Optional context (e.g., { group: 'workers', error: 'reason' })
    */
   async ack(message, status = true, context = {}) {
     await this.#ensureConnected();
     
+    // Handle batch acknowledgment
+    if (Array.isArray(message)) {
+      if (message.length === 0) {
+        return { processed: 0, results: [] };
+      }
+      
+      // Check if messages have individual status (Option B pattern)
+      const hasIndividualStatus = message.some(msg => 
+        typeof msg === 'object' && msg !== null && ('_status' in msg || '_error' in msg)
+      );
+      
+      let acknowledgments;
+      
+      if (hasIndividualStatus) {
+        // Option B: Each message has its own status
+        acknowledgments = message.map(msg => {
+          // Extract transaction ID
+          let transactionId;
+          if (typeof msg === 'string') {
+            transactionId = msg;
+          } else if (typeof msg === 'object' && msg !== null) {
+            transactionId = msg.transactionId || msg.id;
+            if (!transactionId) {
+              throw new Error('Message object must have transactionId or id property');
+            }
+          } else {
+            throw new Error('Invalid message in batch');
+          }
+          
+          // Get individual status or fall back to parameter
+          let msgStatus = msg._status !== undefined ? msg._status : status;
+          let statusStr;
+          if (typeof msgStatus === 'boolean') {
+            statusStr = msgStatus ? 'completed' : 'failed';
+          } else {
+            statusStr = msgStatus;
+          }
+          
+          return {
+            transactionId,
+            status: statusStr,
+            error: msg._error || context.error || null
+          };
+        });
+      } else {
+        // Option A: Same status for all messages
+        const statusStr = typeof status === 'boolean' 
+          ? (status ? 'completed' : 'failed')
+          : status;
+        
+        acknowledgments = message.map(msg => {
+          // Extract transaction ID
+          let transactionId;
+          if (typeof msg === 'string') {
+            transactionId = msg;
+          } else if (typeof msg === 'object' && msg !== null) {
+            transactionId = msg.transactionId || msg.id;
+            if (!transactionId) {
+              throw new Error('Message object must have transactionId or id property');
+            }
+          } else {
+            throw new Error('Invalid message in batch');
+          }
+          
+          return {
+            transactionId,
+            status: statusStr,
+            error: context.error || null
+          };
+        });
+      }
+      
+      // Call batch ack endpoint
+      const result = await withRetry(
+        () => this.#http.post('/api/v1/ack/batch', { 
+          acknowledgments,
+          consumerGroup: context.group || null
+        }),
+        this.#config.retryAttempts,
+        this.#config.retryDelay
+      );
+      
+      if (result && result.error) {
+        throw new Error(result.error);
+      }
+      
+      return result;
+    }
+    
+    // Handle single message acknowledgment
     // Extract transaction ID
-    const transactionId = typeof message === 'object' ? 
-      (message.transactionId || message.id) : 
-      message;
+    let transactionId;
+    if (typeof message === 'string') {
+      transactionId = message;
+    } else if (typeof message === 'object' && message !== null) {
+      transactionId = message.transactionId || message.id;
+      if (!transactionId) {
+        throw new Error('Message object must have transactionId or id property');
+      }
+    } else {
+      throw new Error('Message must be a string (transaction ID) or object with transactionId');
+    }
     
     // Determine status string
     let statusStr;
@@ -277,6 +521,31 @@ export class Queen {
     return result;
   }
   
+  /**
+   * Delete a queue (removes queue and all its messages/partitions)
+   * @param {string} name - Queue name to delete
+   * @returns {Promise<Object>} Deletion result
+   */
+  async queueDelete(name) {
+    await this.#ensureConnected();
+    
+    if (typeof name !== 'string' || !name) {
+      throw new Error('Queue name must be a non-empty string');
+    }
+    
+    const result = await withRetry(
+      () => this.#http.delete(`/api/v1/resources/queues/${encodeURIComponent(name)}`),
+      this.#config.retryAttempts,
+      this.#config.retryDelay
+    );
+    
+    if (result && result.error) {
+      throw new Error(result.error);
+    }
+    
+    return result;
+  }
+  
   /**
    * Close the client connection
    */

package/src/client/queenClient.js

@@ -212,6 +212,11 @@ export const createQueenClient = (options = {}) => {
     clear: async (queue, partition = null) => {
       const params = partition ? `?partition=${partition}` : '';
       return http.delete(`/api/v1/queues/${queue}/clear${params}`);
+    },
+    
+    // Delete a queue (removes queue and all its messages/partitions)
+    delete: async (queue) => {
+      return http.delete(`/api/v1/resources/queues/${encodeURIComponent(queue)}`);
     }
   };
   

package/src/cluster-server.js

@@ -0,0 +1,242 @@
+#!/usr/bin/env node
+
+/**
+ * Queen Message Queue - Clustered Server
+ * 
+ * This module enables multi-core utilization using Node.js cluster module.
+ * Each worker runs an independent instance of the Queen server, sharing the same port.
+ * 
+ * Benefits:
+ * - 10x throughput improvement (utilizes all CPU cores)
+ * - Automatic load balancing across workers
+ * - Fault tolerance (automatic worker restart on crash)
+ * - Zero-downtime deploys (rolling worker restarts)
+ * 
+ * Usage:
+ *   node src/cluster-server.js
+ *   
+ * Environment Variables:
+ *   QUEEN_WORKERS=N    - Number of workers (default: CPU count)
+ *   WORKER_ID=name     - Base worker ID prefix (default: queen-worker)
+ */
+
+import cluster from 'cluster';
+import os from 'os';
+import { log } from './utils/logger.js';
+
+// Configuration
+const numWorkers = parseInt(process.env.QUEEN_WORKERS) || os.cpus().length;
+const workerIdPrefix = process.env.WORKER_ID || 'queen-worker';
+
+if (cluster.isPrimary) {
+  console.log('');
+  console.log('═══════════════════════════════════════════════════════════');
+  console.log('🚀 Queen Message Queue - Clustered Mode');
+  console.log('═══════════════════════════════════════════════════════════');
+  console.log(`📊 System: ${os.cpus()[0].model}`);
+  console.log(`🔢 CPU Cores: ${os.cpus().length}`);
+  console.log(`👷 Starting Workers: ${numWorkers}`);
+  console.log(`🆔 Worker ID Prefix: ${workerIdPrefix}`);
+  console.log('═══════════════════════════════════════════════════════════');
+  console.log('');
+
+  // Track worker statistics
+  const workerStats = {
+    startTime: Date.now(),
+    totalRestarts: 0,
+    workers: new Map()
+  };
+
+  // Graceful shutdown state (declared early for use in event handlers)
+  let isShuttingDown = false;
+  let forceExitTimeout = null;
+  let statusInterval = null;
+
+  // Fork workers
+  for (let i = 0; i < numWorkers; i++) {
+    forkWorker(i, workerStats);
+  }
+
+  // Handle worker exit
+  cluster.on('exit', (worker, code, signal) => {
+    const workerInfo = workerStats.workers.get(worker.id);
+    const workerId = workerInfo ? workerInfo.workerId : `worker-${worker.id}`;
+    const workerIndex = workerInfo ? workerInfo.index : 0;
+    
+    if (code !== 0 && !worker.exitedAfterDisconnect) {
+      // Unexpected crash
+      console.log('');
+      console.log(`⚠️  Worker ${workerId} (PID: ${worker.process.pid}) crashed!`);
+      console.log(`   Exit code: ${code}, Signal: ${signal}`);
+      console.log(`   Restarting worker...`);
+      
+      workerStats.totalRestarts++;
+      
+      // Remove from stats
+      workerStats.workers.delete(worker.id);
+      
+      // Restart with same index
+      setTimeout(() => {
+        forkWorker(workerIndex, workerStats);
+      }, 1000); // Wait 1 second before restart
+    } else {
+      // Graceful shutdown
+      console.log(`✅ Worker ${workerId} (PID: ${worker.process.pid}) exited gracefully`);
+      workerStats.workers.delete(worker.id);
+      
+      // Check if all workers have exited during shutdown
+      if (isShuttingDown && Object.keys(cluster.workers).length === 0) {
+        console.log('✅ All workers exited, shutting down master process');
+        clearInterval(statusInterval);
+        clearTimeout(forceExitTimeout);
+        process.exit(0);
+      }
+    }
+  });
+
+  // Handle worker online
+  cluster.on('online', (worker) => {
+    const workerInfo = workerStats.workers.get(worker.id);
+    if (workerInfo) {
+      // Update PID now that it's available
+      workerInfo.pid = worker.process.pid;
+      console.log(`✅ Worker ${workerInfo.workerId} (PID: ${worker.process.pid}) online and ready`);
+    } else {
+      console.log(`✅ Worker ${worker.id} (PID: ${worker.process.pid}) online`);
+    }
+  });
+
+  // Handle worker listening
+  cluster.on('listening', (worker, address) => {
+    const workerInfo = workerStats.workers.get(worker.id);
+    const workerId = workerInfo ? workerInfo.workerId : `worker-${worker.id}`;
+    console.log(`🎧 Worker ${workerId} listening on ${address.address}:${address.port}`);
+  });
+  
+  // Handle cluster errors
+  cluster.on('error', (error) => {
+    console.error('⚠️  Cluster error:', error.message);
+  });
+  
+  // Handle uncaught exceptions in master
+  process.on('uncaughtException', (error) => {
+    console.error('❌ Uncaught exception in master process:', error);
+    console.error(error.stack);
+    // Don't exit - try to keep the cluster running
+  });
+  
+  process.on('unhandledRejection', (reason, promise) => {
+    console.error('❌ Unhandled rejection in master process:', reason);
+    // Don't exit - try to keep the cluster running
+  });
+
+  // Graceful shutdown handling
+  const gracefulShutdown = (signal) => {
+    if (isShuttingDown) return;
+    isShuttingDown = true;
+    
+    console.log('');
+    console.log(`🛑 Received ${signal}, shutting down gracefully...`);
+    console.log(`📊 Total worker restarts during uptime: ${workerStats.totalRestarts}`);
+    console.log(`⏱️  Uptime: ${((Date.now() - workerStats.startTime) / 1000 / 60).toFixed(2)} minutes`);
+    
+    // Stop status reporting during shutdown
+    clearInterval(statusInterval);
+    
+    // Check if we already have no workers
+    if (Object.keys(cluster.workers).length === 0) {
+      console.log('✅ No active workers, exiting immediately');
+      process.exit(0);
+    }
+    
+    // Disconnect all workers
+    for (const id in cluster.workers) {
+      cluster.workers[id].disconnect();
+    }
+    
+    // Force exit if workers don't shut down in 10 seconds (reduced from 30)
+    forceExitTimeout = setTimeout(() => {
+      console.log('⚠️  Forcing shutdown after 10 second timeout');
+      console.log(`   Remaining workers: ${Object.keys(cluster.workers).length}`);
+      process.exit(0);
+    }, 10000);
+  };
+
+  process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
+  process.on('SIGINT', () => gracefulShutdown('SIGINT'));
+
+  // Status reporting every 60 seconds
+  statusInterval = setInterval(() => {
+    // Don't show status during shutdown
+    if (isShuttingDown) return;
+    
+    const aliveWorkers = Object.keys(cluster.workers).length;
+    const uptime = ((Date.now() - workerStats.startTime) / 1000 / 60).toFixed(2);
+    
+    console.log('');
+    console.log(`📊 Cluster Status (${new Date().toISOString()})`);
+    console.log(`   Active Workers: ${aliveWorkers}/${numWorkers}`);
+    console.log(`   Total Restarts: ${workerStats.totalRestarts}`);
+    console.log(`   Uptime: ${uptime} minutes`);
+  }, 60000);
+
+} else {
+  // Worker process - run the normal server
+  const workerId = process.env.WORKER_ID || `worker-${process.pid}`;
+  const workerIndex = process.env.WORKER_INDEX || '0';
+  
+  console.log(`👷 Worker ${workerId} (index: ${workerIndex}, PID: ${process.pid}) starting...`);
+  
+  try {
+    // Import and run the main server
+    await import('./server.js');
+    
+    // Worker-specific metrics could be added here
+    // e.g., track messages processed, errors, etc.
+  } catch (error) {
+    console.error(`❌ Worker ${workerId} failed to start:`, error.message);
+    console.error(error.stack);
+    process.exit(1);
+  }
+  
+  // Handle uncaught exceptions in worker
+  process.on('uncaughtException', (error) => {
+    console.error(`❌ Uncaught exception in worker ${workerId}:`, error);
+    process.exit(1);
+  });
+  
+  process.on('unhandledRejection', (reason, promise) => {
+    console.error(`❌ Unhandled rejection in worker ${workerId}:`, reason);
+    process.exit(1);
+  });
+}
+
+/**
+ * Fork a new worker with proper environment setup
+ */
+function forkWorker(index, stats) {
+  const workerId = `${workerIdPrefix}-${index}`;
+  
+  try {
+    const worker = cluster.fork({
+      WORKER_ID: workerId,
+      WORKER_INDEX: index.toString(),
+      // Each worker gets a unique ID for system events
+      SYSTEM_WORKER_ID: `${workerIdPrefix}-${index}-${Date.now()}`
+    });
+    
+    // Store worker info immediately (PID will be updated in 'online' event)
+    stats.workers.set(worker.id, {
+      workerId,
+      index,
+      startTime: Date.now(),
+      pid: null  // Will be set when worker comes online
+    });
+    
+    return worker;
+  } catch (error) {
+    console.error(`❌ Failed to fork worker ${workerId}:`, error.message);
+    return null;
+  }
+}
+