queen-mq 0.1.1 → 0.1.2

package/src/benchmark/producer.js

@@ -0,0 +1,75 @@
+import { Queen } from '../client/client.js'; 
+import fs from 'fs';
+
+const TOTAL_MESSAGES = 100000;
+const PARTITIONS = 10; 
+const MAX_BATCH_SIZE = 1000;  // Reduced from 10000
+const PUSH_PARALLEL = 10;      // Reduced from 10 to avoid lock contention
+const QUEUE_NAME = 'benchmark-queue-001';
+
+const q = new Queen({
+  baseUrls: ['http://localhost:6632'],
+  timeout: 120000
+});
+
+//await q.queueDelete(QUEUE_NAME);
+await q.queue(QUEUE_NAME, {
+  leaseTime: 10,
+  retryLimit: 3
+}, { namespace: 'benchmark' });
+
+
+
+let batches = []
+const messagesPerPartition = Math.floor(TOTAL_MESSAGES / PARTITIONS);
+
+// Create batches for each partition
+const partitionBatches = [];
+for (let i = 0; i < PARTITIONS; i++) {
+  partitionBatches[i] = [];
+  let messages = [];
+  for (let j = 0; j < messagesPerPartition; j++) {
+    messages.push({
+      id: j,
+      //message: `Message ${globalMessageId}`,
+      //timestamp: new Date().toISOString()
+    });
+    if (messages.length >= MAX_BATCH_SIZE) {
+      console.log(`Pushing ${messages.length} messages for partition ${i}`);
+      partitionBatches[i].push({
+        partition: i,
+        messages: messages
+      });
+      messages = [];
+    }
+  }
+  // Only add the remaining batch if it has messages
+  if (messages.length > 0) {
+    console.log(`Pushing ${messages.length} messages for partition ${i}`);
+    partitionBatches[i].push({
+      partition: i,
+      messages: messages
+    });
+  }
+}
+
+// Interleave batches from different partitions to avoid lock contention
+// This ensures parallel pushes go to different partitions
+const maxBatchesPerPartition = Math.max(...partitionBatches.map(pb => pb.length));
+for (let batchIndex = 0; batchIndex < maxBatchesPerPartition; batchIndex++) {
+  for (let partition = 0; partition < PARTITIONS; partition++) {
+    if (partitionBatches[partition][batchIndex]) {
+      batches.push(partitionBatches[partition][batchIndex]);
+    }
+  }
+}
+
+const start = Date.now();
+console.log(`Total batches: ${batches.length}, pushing ${PUSH_PARALLEL} at a time`);
+for (let i = 0; i < batches.length; i += PUSH_PARALLEL) {
+  const batchGroup = batches.slice(i, i + PUSH_PARALLEL);
+  console.log(`Pushing batch group ${Math.floor(i/PUSH_PARALLEL) + 1}/${Math.ceil(batches.length/PUSH_PARALLEL)}: ${batchGroup.map(b => `partition ${b.partition} (${b.messages.length} msgs)`).join(', ')}`);
+  await Promise.all(batchGroup.map(batch => q.push(`${QUEUE_NAME}/${batch.partition}`, batch.messages)));
+}
+const end = Date.now();
+console.log(`Pushed ${TOTAL_MESSAGES} messages in ${(end - start) / 1000}s, ${(TOTAL_MESSAGES / ((end - start) / 1000)).toFixed(2)} msg/s`);

package/src/benchmark/producer_multi.js

@@ -0,0 +1,115 @@
+import { Queen } from '../client/client.js'; 
+import fs from 'fs';
+
+const TOTAL_MESSAGES = 100000;
+const NUM_QUEUES = 50;          // X: Number of queues
+const PARTITIONS_PER_QUEUE = 10; // Y: Partitions per queue
+const MAX_BATCH_SIZE = 10;  
+const PUSH_PARALLEL = 10;      
+
+const q = new Queen({
+  baseUrls: ['http://localhost:6632'],
+  timeout: 120000
+});
+
+// Create X queues, each with Y partitions, all in 'benchmark' namespace
+console.log(`Creating ${NUM_QUEUES} queues with ${PARTITIONS_PER_QUEUE} partitions each...`);
+for (let queueIdx = 0; queueIdx < NUM_QUEUES; queueIdx++) {
+  const queueName = `benchmark-queue-${String(queueIdx + 1).padStart(2, '0')}`;
+  try {
+    //await q.queueDelete(queueName);
+  } catch (e) {
+    // Queue might not exist
+  }
+  await q.queue(queueName, {
+    leaseTime: 10,
+    retryLimit: 3
+  }, { namespace: 'benchmark' });
+  console.log(`  Created queue: ${queueName}`);
+}
+
+
+// Calculate total partitions across all queues
+const TOTAL_PARTITIONS = NUM_QUEUES * PARTITIONS_PER_QUEUE;
+const messagesPerPartition = Math.floor(TOTAL_MESSAGES / TOTAL_PARTITIONS);
+
+console.log(`\nDistributing ${TOTAL_MESSAGES} messages across ${NUM_QUEUES} queues × ${PARTITIONS_PER_QUEUE} partitions = ${TOTAL_PARTITIONS} total partitions`);
+console.log(`Messages per partition: ${messagesPerPartition}\n`);
+
+// Create batches for each queue/partition combination
+const queuePartitionBatches = [];
+for (let queueIdx = 0; queueIdx < NUM_QUEUES; queueIdx++) {
+  for (let partitionIdx = 0; partitionIdx < PARTITIONS_PER_QUEUE; partitionIdx++) {
+    const key = `q${queueIdx}_p${partitionIdx}`;
+    queuePartitionBatches.push({ queueIdx, partitionIdx, batches: [] });
+    
+    let messages = [];
+    for (let msgIdx = 0; msgIdx < messagesPerPartition; msgIdx++) {
+      messages.push({
+        id: msgIdx,
+        queue: queueIdx,
+        partition: partitionIdx
+      });
+      
+      if (messages.length >= MAX_BATCH_SIZE) {
+        queuePartitionBatches[queuePartitionBatches.length - 1].batches.push([...messages]);
+        messages = [];
+      }
+    }
+    
+    // Add remaining messages
+    if (messages.length > 0) {
+      queuePartitionBatches[queuePartitionBatches.length - 1].batches.push([...messages]);
+    }
+  }
+}
+
+// Flatten and interleave batches to distribute load evenly
+let batches = [];
+const maxBatchesPerQueuePartition = Math.max(...queuePartitionBatches.map(qp => qp.batches.length));
+for (let batchIdx = 0; batchIdx < maxBatchesPerQueuePartition; batchIdx++) {
+  for (const qp of queuePartitionBatches) {
+    if (qp.batches[batchIdx]) {
+      batches.push({
+        queueIdx: qp.queueIdx,
+        partitionIdx: qp.partitionIdx,
+        messages: qp.batches[batchIdx]
+      });
+    }
+  }
+}
+
+const start = Date.now();
+console.log(`Total batches: ${batches.length}, pushing ${PUSH_PARALLEL} at a time\n`);
+
+for (let i = 0; i < batches.length; i += PUSH_PARALLEL) {
+  const batchGroup = batches.slice(i, i + PUSH_PARALLEL);
+  const batchGroupNum = Math.floor(i/PUSH_PARALLEL) + 1;
+  const totalBatchGroups = Math.ceil(batches.length/PUSH_PARALLEL);
+  
+  console.log(`[${batchGroupNum}/${totalBatchGroups}] Pushing: ${batchGroup.map(b => {
+    const queueName = `benchmark-queue-${String(b.queueIdx + 1).padStart(2, '0')}`;
+    return `${queueName}/p${b.partitionIdx} (${b.messages.length} msgs)`;
+  }).join(', ')}`);
+  
+  await Promise.all(batchGroup.map(batch => {
+    const queueName = `benchmark-queue-${String(batch.queueIdx + 1).padStart(2, '0')}`;
+    return q.push(`${queueName}/${batch.partitionIdx}`, batch.messages);
+  }));
+}
+
+const end = Date.now();
+const duration = (end - start) / 1000;
+const throughput = (TOTAL_MESSAGES / duration).toFixed(2);
+
+console.log('\n' + '='.repeat(80));
+console.log('📤 PRODUCER RESULTS');
+console.log('='.repeat(80));
+console.log(`Total Messages Pushed: ${TOTAL_MESSAGES.toLocaleString()}`);
+console.log(`Queues: ${NUM_QUEUES}`);
+console.log(`Partitions per Queue: ${PARTITIONS_PER_QUEUE}`);
+console.log(`Total Partitions: ${TOTAL_PARTITIONS}`);
+console.log(`Duration: ${duration.toFixed(2)}s`);
+console.log(`Throughput: ${throughput} msg/s`);
+console.log('='.repeat(80));
+

package/src/client/client.js

@@ -212,12 +212,13 @@ export class Queen {
   }
   
   /**
-   * Take messages from a queue (async iterator)
-   * @param {string} address - Queue address (e.g., "myqueue", "myqueue/partition", "myqueue@group")
+   * Internal method that yields batches of messages (arrays)
+   * @private
+   * @param {string} address - Queue address
    * @param {Object} options - Options for taking messages
-   * @yields {Object} Message objects
+   * @yields {Array} Arrays of message objects
    */
-  async *take(address, options = {}) {
+  async *#takeInternal(address, options = {}) {
     await this.#ensureConnected();
     
     const { queue, partition, consumerGroup, namespace, task } = this.#parseAddress(address);
@@ -227,23 +228,36 @@ export class Queen {
       wait = false, 
       timeout = 30000,
       subscriptionMode = null,
-      subscriptionFrom = null
+      subscriptionFrom = null,
+      idleTimeout = null
     } = options;
     
-    let count = 0;
+    let totalCount = 0;
     let consecutiveEmptyResponses = 0;
     const maxConsecutiveEmpty = 3;
+    let lastMessageTime = idleTimeout ? Date.now() : null;
     
     while (true) {
       // Check if we've reached the limit
-      if (limit && count >= limit) break;
+      if (limit && totalCount >= limit) break;
+      
+      // Check idle timeout
+      if (idleTimeout && lastMessageTime) {
+        const idleTime = Date.now() - lastMessageTime;
+        if (idleTime >= idleTimeout) {
+          break; // Exit if idle time exceeded
+        }
+      }
+      
+      // Calculate batch size respecting the limit
+      const effectiveBatch = limit ? Math.min(batch, limit - totalCount) : batch;
       
       // Build the request path and parameters
       let path;
       const params = new URLSearchParams({
         wait: wait.toString(),
         timeout: timeout.toString(),
-        batch: Math.min(batch, limit ? limit - count : batch).toString()
+        batch: effectiveBatch.toString()
       });
       
       // Add consumer group parameters if provided
@@ -293,12 +307,21 @@ export class Queen {
         // Reset empty counter on successful fetch
         consecutiveEmptyResponses = 0;
         
-        // Yield each message (filter out any null/undefined values)
-        for (const message of result.messages) {
-          if (message) { // Skip null/undefined messages
-            yield message;
-            count++;
-            if (limit && count >= limit) return;
+        // Update last message time if tracking idle timeout
+        if (idleTimeout) {
+          lastMessageTime = Date.now();
+        }
+        
+        // Filter out null/undefined messages
+        const messages = result.messages.filter(msg => msg != null);
+        
+        if (messages.length > 0) {
+          totalCount += messages.length;
+          yield messages;
+          
+          // If we've hit the limit, stop
+          if (limit && totalCount >= limit) {
+            break;
           }
         }
         
@@ -320,14 +343,142 @@ export class Queen {
   }
   
   /**
-   * Acknowledge a message
-   * @param {Object|string} message - Message object or transaction ID
+   * Take messages from a queue one at a time (async iterator)
+   * @param {string} address - Queue address (e.g., "myqueue", "myqueue/partition", "myqueue@group")
+   * @param {Object} options - Options for taking messages
+   * @yields {Object} Individual message objects
+   */
+  async *take(address, options = {}) {
+    let count = 0;
+    const { limit = null } = options;
+    
+    for await (const messages of this.#takeInternal(address, options)) {
+      for (const message of messages) {
+        yield message;
+        count++;
+        
+        // Double-check limit (internal method also checks, but this ensures exact limit)
+        if (limit && count >= limit) {
+          return;
+        }
+      }
+    }
+  }
+  
+  /**
+   * Take messages from a queue in batches (async iterator)
+   * @param {string} address - Queue address (e.g., "myqueue", "myqueue/partition", "myqueue@group")
+   * @param {Object} options - Options for taking messages
+   * @yields {Array} Arrays of message objects
+   */
+  async *takeBatch(address, options = {}) {
+    for await (const messages of this.#takeInternal(address, options)) {
+      // Only yield non-empty batches
+      if (messages && messages.length > 0) {
+        yield messages;
+      }
+    }
+  }
+  
+  /**
+   * Acknowledge a message or batch of messages
+   * @param {Object|string|Array} message - Message object, transaction ID, or array of messages
    * @param {boolean|string} status - true for success, false for failure, or 'retry'
    * @param {Object} context - Optional context (e.g., { group: 'workers', error: 'reason' })
    */
   async ack(message, status = true, context = {}) {
     await this.#ensureConnected();
     
+    // Handle batch acknowledgment
+    if (Array.isArray(message)) {
+      if (message.length === 0) {
+        return { processed: 0, results: [] };
+      }
+      
+      // Check if messages have individual status (Option B pattern)
+      const hasIndividualStatus = message.some(msg => 
+        typeof msg === 'object' && msg !== null && ('_status' in msg || '_error' in msg)
+      );
+      
+      let acknowledgments;
+      
+      if (hasIndividualStatus) {
+        // Option B: Each message has its own status
+        acknowledgments = message.map(msg => {
+          // Extract transaction ID
+          let transactionId;
+          if (typeof msg === 'string') {
+            transactionId = msg;
+          } else if (typeof msg === 'object' && msg !== null) {
+            transactionId = msg.transactionId || msg.id;
+            if (!transactionId) {
+              throw new Error('Message object must have transactionId or id property');
+            }
+          } else {
+            throw new Error('Invalid message in batch');
+          }
+          
+          // Get individual status or fall back to parameter
+          let msgStatus = msg._status !== undefined ? msg._status : status;
+          let statusStr;
+          if (typeof msgStatus === 'boolean') {
+            statusStr = msgStatus ? 'completed' : 'failed';
+          } else {
+            statusStr = msgStatus;
+          }
+          
+          return {
+            transactionId,
+            status: statusStr,
+            error: msg._error || context.error || null
+          };
+        });
+      } else {
+        // Option A: Same status for all messages
+        const statusStr = typeof status === 'boolean' 
+          ? (status ? 'completed' : 'failed')
+          : status;
+        
+        acknowledgments = message.map(msg => {
+          // Extract transaction ID
+          let transactionId;
+          if (typeof msg === 'string') {
+            transactionId = msg;
+          } else if (typeof msg === 'object' && msg !== null) {
+            transactionId = msg.transactionId || msg.id;
+            if (!transactionId) {
+              throw new Error('Message object must have transactionId or id property');
+            }
+          } else {
+            throw new Error('Invalid message in batch');
+          }
+          
+          return {
+            transactionId,
+            status: statusStr,
+            error: context.error || null
+          };
+        });
+      }
+      
+      // Call batch ack endpoint
+      const result = await withRetry(
+        () => this.#http.post('/api/v1/ack/batch', { 
+          acknowledgments,
+          consumerGroup: context.group || null
+        }),
+        this.#config.retryAttempts,
+        this.#config.retryDelay
+      );
+      
+      if (result && result.error) {
+        throw new Error(result.error);
+      }
+      
+      return result;
+    }
+    
+    // Handle single message acknowledgment
     // Extract transaction ID
     let transactionId;
     if (typeof message === 'string') {
@@ -370,6 +521,31 @@ export class Queen {
     return result;
   }
   
+  /**
+   * Delete a queue (removes queue and all its messages/partitions)
+   * @param {string} name - Queue name to delete
+   * @returns {Promise<Object>} Deletion result
+   */
+  async queueDelete(name) {
+    await this.#ensureConnected();
+    
+    if (typeof name !== 'string' || !name) {
+      throw new Error('Queue name must be a non-empty string');
+    }
+    
+    const result = await withRetry(
+      () => this.#http.delete(`/api/v1/resources/queues/${encodeURIComponent(name)}`),
+      this.#config.retryAttempts,
+      this.#config.retryDelay
+    );
+    
+    if (result && result.error) {
+      throw new Error(result.error);
+    }
+    
+    return result;
+  }
+  
   /**
    * Close the client connection
    */

package/src/client/queenClient.js

@@ -212,6 +212,11 @@ export const createQueenClient = (options = {}) => {
     clear: async (queue, partition = null) => {
       const params = partition ? `?partition=${partition}` : '';
       return http.delete(`/api/v1/queues/${queue}/clear${params}`);
+    },
+    
+    // Delete a queue (removes queue and all its messages/partitions)
+    delete: async (queue) => {
+      return http.delete(`/api/v1/resources/queues/${encodeURIComponent(queue)}`);
     }
   };