queen-mq 0.1.0 → 0.1.2

package/src/benchmark/consumer.js

@@ -0,0 +1,207 @@
+import { Queen } from '../client/client.js';
+import fs from 'fs';
+
+const QUEUE_NAME = 'benchmark-queue-001';
+const NUMBER_OF_CONSUMERS = 10;
+const BATCH_SIZE = 10;
+const CONSUME_MODE = 'partition';
+
+// Global metrics tracking
+const metrics = {
+  startTime: null,
+  endTime: null,
+  totalProcessed: 0,
+  consumers: [],
+  batchTimes: [],
+  ackTimes: [],
+  popTimes: [],
+  firstMessageTime: null,
+  lastMessageTime: null
+};
+
+async function consumer(consumerId, partition) {
+  const q = new Queen({
+    baseUrls: ['http://localhost:6632'],
+    timeout: 30000
+  });
+  
+  const consumerMetrics = {
+    consumerId,
+    partition,
+    messagesProcessed: 0,
+    batchesProcessed: 0,
+    batchTimes: [],
+    ackTimes: [],
+    popTimes: [],
+    startTime: Date.now(),
+    endTime: null
+  };
+
+  const target = CONSUME_MODE === 'partition' ? `${QUEUE_NAME}/${partition}` : `${QUEUE_NAME}`;
+  
+  metrics.consumers.push(consumerMetrics);
+  if (target === `${QUEUE_NAME}`) {
+    console.log(`[Consumer ${consumerId}] Started, consuming all partitions`);
+  } else {
+    console.log(`[Consumer ${consumerId}] Started, consuming partition ${partition}`);
+  }
+  
+  try {
+    for await (const messages of q.takeBatch(target, { //`namespace:benchmark`, 
+      wait: true,
+      timeout: 15000,
+      batch: BATCH_SIZE,
+      idleTimeout: 5000
+    })) {
+      if (!metrics.firstMessageTime) {
+        metrics.firstMessageTime = Date.now();
+      }
+      metrics.lastMessageTime = Date.now();
+      const batchStartTime = Date.now();
+      const popTime = batchStartTime - (consumerMetrics.endTime || consumerMetrics.startTime);
+      consumerMetrics.popTimes.push(popTime);
+      metrics.popTimes.push(popTime);
+      
+      try {
+        // Validate message ordering
+        const partitionLastIds = new Map();
+        
+        for (const message of messages) {
+          if (!message.data || typeof message.data.id !== 'number') {
+            throw new Error(`Message missing or invalid data.id: ${JSON.stringify(message)}`);
+          }
+          
+          const partition = message.partition || 'default';
+          const messageId = message.data.id;
+          
+          if (partitionLastIds.has(partition)) {
+            const lastId = partitionLastIds.get(partition);
+            if (messageId !== lastId + 1) {
+              console.log(message.partition, message);
+              throw new Error(`Message ordering violation in partition '${partition}': expected id ${lastId + 1}, got ${messageId}`);
+            }
+          }
+          
+          partitionLastIds.set(partition, messageId);
+        }
+
+        // Acknowledge all messages
+        const ackStartTime = Date.now();
+        await q.ack(messages);
+        const ackTime = Date.now() - ackStartTime;
+        consumerMetrics.ackTimes.push(ackTime);
+        metrics.ackTimes.push(ackTime);
+        
+        const batchEndTime = Date.now();
+        const batchDuration = batchEndTime - batchStartTime;
+        
+        consumerMetrics.messagesProcessed += messages.length;
+        consumerMetrics.batchesProcessed++;
+        consumerMetrics.batchTimes.push(batchDuration);
+        consumerMetrics.endTime = batchEndTime;
+        
+        metrics.totalProcessed += messages.length;
+        metrics.batchTimes.push(batchDuration);
+        
+        const overallDuration = (batchEndTime - metrics.startTime) / 1000;
+        const currentRate = (metrics.totalProcessed / overallDuration).toFixed(0);
+        
+        console.log(new Date().toISOString(), `[C${consumerId}] Batch: ${messages.length} msgs | Time: ${(batchDuration/1000).toFixed(3)}s (pop:${(popTime/1000).toFixed(3)}s ack:${(ackTime/1000).toFixed(3)}s) | Total: ${metrics.totalProcessed} | Rate: ${currentRate} msg/s`);
+        metrics.lastMessageTime = Date.now();
+      } catch (error) {
+        console.error(new Date().toISOString(), `[Consumer ${consumerId}] Error processing batch:`, error);
+        await q.ack(messages, false, { error: error.message });
+      }
+    }
+  } catch (error) {
+    console.error(`[Consumer ${consumerId}] Fatal error:`, error);
+  }
+  
+  consumerMetrics.endTime = Date.now();
+  console.log(`[Consumer ${consumerId}] Finished. Processed ${consumerMetrics.messagesProcessed} messages in ${consumerMetrics.batchesProcessed} batches`);
+}
+
+// Start metrics
+metrics.startTime = Date.now();
+
+// Launch all consumers
+const consumerPromises = [];
+for (let i = 0; i < NUMBER_OF_CONSUMERS; i++) {
+  consumerPromises.push(consumer(i, i));
+}
+
+// Wait for all consumers to complete
+await Promise.all(consumerPromises);
+metrics.endTime = Date.now();
+
+// Calculate actual processing time (excluding idle timeout)
+// Find the time when the last batch was actually completed
+let lastBatchCompletedTime = metrics.startTime;
+metrics.consumers.forEach(c => {
+  if (c.endTime && c.batchesProcessed > 0) {
+    // c.endTime is when consumer exited (after idle timeout)
+    // Actual last batch time = c.endTime - (idle timeout we didn't track)
+    // Better: use the sum of all processing times
+    const totalBatchTime = c.batchTimes.reduce((a,b)=>a+b,0);
+    const totalPopTime = c.popTimes.reduce((a,b)=>a+b,0);
+    const actualEndTime = c.startTime + totalBatchTime + totalPopTime;
+    if (actualEndTime > lastBatchCompletedTime) {
+      lastBatchCompletedTime = actualEndTime;
+    }
+  }
+});
+
+const totalDuration = (metrics.endTime - metrics.startTime) / 1000;
+const actualProcessingDuration = (lastBatchCompletedTime - metrics.startTime) / 1000;
+const avgThroughput = (metrics.totalProcessed / actualProcessingDuration).toFixed(0);
+
+console.log('\n' + '='.repeat(80));
+console.log('📊 BENCHMARK RESULTS');
+console.log('='.repeat(80));
+console.log(`Total Messages Processed: ${metrics.totalProcessed.toLocaleString()}`);
+console.log(`Total Wall Clock Time: ${totalDuration.toFixed(2)}s (includes idle timeout)`);
+console.log(`Actual Processing Time: ${actualProcessingDuration.toFixed(2)}s`);
+console.log(`Idle/Wait Time: ${(totalDuration - actualProcessingDuration).toFixed(2)}s`);
+console.log(`Average Throughput: ${avgThroughput} msg/s (based on processing time)`);
+console.log(`Number of Consumers: ${NUMBER_OF_CONSUMERS}`);
+console.log(`Total Batches: ${metrics.batchTimes.length}`);
+console.log(`Total Time: ${metrics.lastMessageTime - metrics.firstMessageTime}ms`);
+console.log(`Total msg/s: ${metrics.totalProcessed / (metrics.lastMessageTime - metrics.firstMessageTime) * 1000} msg/s`);
+
+if (metrics.batchTimes.length > 0) {
+  const avgBatchTime = (metrics.batchTimes.reduce((a, b) => a + b, 0) / metrics.batchTimes.length / 1000).toFixed(3);
+  const minBatchTime = (Math.min(...metrics.batchTimes) / 1000).toFixed(3);
+  const maxBatchTime = (Math.max(...metrics.batchTimes) / 1000).toFixed(3);
+  
+  console.log(`\nBatch Times:`);
+  console.log(`  Average: ${avgBatchTime}s`);
+  console.log(`  Min: ${minBatchTime}s`);
+  console.log(`  Max: ${maxBatchTime}s`);
+}
+
+if (metrics.popTimes.length > 0) {
+  const avgPopTime = (metrics.popTimes.reduce((a, b) => a + b, 0) / metrics.popTimes.length / 1000).toFixed(3);
+  console.log(`\nAverage POP Time: ${avgPopTime}s`);
+}
+
+if (metrics.ackTimes.length > 0) {
+  const avgAckTime = (metrics.ackTimes.reduce((a, b) => a + b, 0) / metrics.ackTimes.length / 1000).toFixed(3);
+  console.log(`Average ACK Time: ${avgAckTime}s`);
+}
+
+console.log(`\nPer-Consumer Stats:`);
+metrics.consumers.forEach(c => {
+  // Calculate actual processing time (sum of all batch times + pop times)
+  const totalBatchTime = c.batchTimes.reduce((a,b)=>a+b,0);
+  const totalPopTime = c.popTimes.reduce((a,b)=>a+b,0);
+  const actualProcessingTime = (totalBatchTime + totalPopTime) / 1000;
+  
+  const wallClockTime = ((c.endTime - c.startTime) / 1000).toFixed(2);
+  const rate = (c.messagesProcessed / actualProcessingTime).toFixed(0);
+  const avgBatch = c.batchTimes.length > 0 ? (totalBatchTime / c.batchTimes.length / 1000).toFixed(3) : 'N/A';
+  const idleTime = (wallClockTime - actualProcessingTime).toFixed(2);
+  
+  console.log(`  [C${c.consumerId}] ${c.messagesProcessed.toLocaleString()} msgs | Active: ${actualProcessingTime.toFixed(2)}s (${rate} msg/s) | Idle: ${idleTime}s | ${c.batchesProcessed} batches | avg: ${avgBatch}s`);
+});
+
+console.log('='.repeat(80));

package/src/benchmark/consumer_multi.js

@@ -0,0 +1,216 @@
+import { Queen } from '../client/client.js';
+import fs from 'fs';
+
+const NAMESPACE = 'benchmark';
+const NUMBER_OF_CONSUMERS = 10;
+const BATCH_SIZE = 1;
+
+// Global metrics tracking
+const metrics = {
+  startTime: null,
+  endTime: null,
+  totalProcessed: 0,
+  consumers: [],
+  batchTimes: [],
+  ackTimes: [],
+  popTimes: [],
+  firstMessageTime: null,
+  lastMessageTime: null
+};
+
+async function consumer(consumerId) {
+  const q = new Queen({
+    baseUrls: ['http://localhost:6632'],
+    timeout: 30000
+  });
+  
+  const consumerMetrics = {
+    consumerId,
+    messagesProcessed: 0,
+    batchesProcessed: 0,
+    batchTimes: [],
+    ackTimes: [],
+    popTimes: [],
+    startTime: Date.now(),
+    endTime: null,
+    queuesConsumed: new Set(),
+    partitionsConsumed: new Set()
+  };
+
+  const target = `namespace:${NAMESPACE}`;
+  
+  metrics.consumers.push(consumerMetrics);
+  console.log(`[Consumer ${consumerId}] Started, consuming from namespace '${NAMESPACE}'`);
+  
+  try {
+    for await (const messages of q.takeBatch(target, {
+      wait: true,
+      timeout: 15000,
+      batch: BATCH_SIZE,
+      idleTimeout: 5000
+    })) {
+      if (!metrics.firstMessageTime) {
+        metrics.firstMessageTime = Date.now();
+      }
+      metrics.lastMessageTime = Date.now();
+      const batchStartTime = Date.now();
+      const popTime = batchStartTime - (consumerMetrics.endTime || consumerMetrics.startTime);
+      consumerMetrics.popTimes.push(popTime);
+      metrics.popTimes.push(popTime);
+      
+      try {
+        // Track which queues and partitions this consumer is consuming from
+        for (const message of messages) {
+          if (message.queue) consumerMetrics.queuesConsumed.add(message.queue);
+          if (message.partition !== undefined) consumerMetrics.partitionsConsumed.add(`${message.queue}:${message.partition}`);
+        }
+        
+        // Validate message ordering per queue/partition
+        const queuePartitionLastIds = new Map();
+        
+        for (const message of messages) {
+          if (!message.data || typeof message.data.id !== 'number') {
+            throw new Error(`Message missing or invalid data.id: ${JSON.stringify(message)}`);
+          }
+          
+          // Use queue and partition to ensure proper ordering within each queue/partition
+          const queuePartitionKey = `${message.queue || 'default'}:${message.partition || 'default'}`;
+          const messageId = message.data.id;
+          
+          if (queuePartitionLastIds.has(queuePartitionKey)) {
+            const lastId = queuePartitionLastIds.get(queuePartitionKey);
+            if (messageId !== lastId + 1) {
+              console.log(`Queue: ${message.queue}, Partition: ${message.partition}, Message:`, message);
+              throw new Error(`Message ordering violation in '${queuePartitionKey}': expected id ${lastId + 1}, got ${messageId}`);
+            }
+          }
+          
+          queuePartitionLastIds.set(queuePartitionKey, messageId);
+        }
+
+        // Acknowledge all messages
+        const ackStartTime = Date.now();
+        await q.ack(messages);
+        const ackTime = Date.now() - ackStartTime;
+        consumerMetrics.ackTimes.push(ackTime);
+        metrics.ackTimes.push(ackTime);
+        
+        const batchEndTime = Date.now();
+        const batchDuration = batchEndTime - batchStartTime;
+        
+        consumerMetrics.messagesProcessed += messages.length;
+        consumerMetrics.batchesProcessed++;
+        consumerMetrics.batchTimes.push(batchDuration);
+        consumerMetrics.endTime = batchEndTime;
+        
+        metrics.totalProcessed += messages.length;
+        metrics.batchTimes.push(batchDuration);
+        
+        const overallDuration = (batchEndTime - metrics.startTime) / 1000;
+        const currentRate = (metrics.totalProcessed / overallDuration).toFixed(0);
+        
+        console.log(new Date().toISOString(), `[C${consumerId}] Batch: ${messages.length} msgs | Time: ${(batchDuration/1000).toFixed(3)}s (pop:${(popTime/1000).toFixed(3)}s ack:${(ackTime/1000).toFixed(3)}s) | Total: ${metrics.totalProcessed} | Rate: ${currentRate} msg/s`);
+        metrics.lastMessageTime = Date.now();
+      } catch (error) {
+        console.error(new Date().toISOString(), `[Consumer ${consumerId}] Error processing batch:`, error);
+        await q.ack(messages, false, { error: error.message });
+      }
+    }
+  } catch (error) {
+    console.error(`[Consumer ${consumerId}] Fatal error:`, error);
+  }
+  
+  consumerMetrics.endTime = Date.now();
+  console.log(`[Consumer ${consumerId}] Finished. Processed ${consumerMetrics.messagesProcessed} messages in ${consumerMetrics.batchesProcessed} batches`);
+}
+
+// Start metrics
+metrics.startTime = Date.now();
+
+// Launch all consumers (they will all consume from the namespace)
+console.log(`Launching ${NUMBER_OF_CONSUMERS} consumers for namespace '${NAMESPACE}'...\n`);
+const consumerPromises = [];
+for (let i = 0; i < NUMBER_OF_CONSUMERS; i++) {
+  consumerPromises.push(consumer(i));
+}
+
+// Wait for all consumers to complete
+await Promise.all(consumerPromises);
+metrics.endTime = Date.now();
+
+// Calculate actual processing time (excluding idle timeout)
+// Find the time when the last batch was actually completed
+let lastBatchCompletedTime = metrics.startTime;
+metrics.consumers.forEach(c => {
+  if (c.endTime && c.batchesProcessed > 0) {
+    // c.endTime is when consumer exited (after idle timeout)
+    // Actual last batch time = c.endTime - (idle timeout we didn't track)
+    // Better: use the sum of all processing times
+    const totalBatchTime = c.batchTimes.reduce((a,b)=>a+b,0);
+    const totalPopTime = c.popTimes.reduce((a,b)=>a+b,0);
+    const actualEndTime = c.startTime + totalBatchTime + totalPopTime;
+    if (actualEndTime > lastBatchCompletedTime) {
+      lastBatchCompletedTime = actualEndTime;
+    }
+  }
+});
+
+const totalDuration = (metrics.endTime - metrics.startTime) / 1000;
+const actualProcessingDuration = (lastBatchCompletedTime - metrics.startTime) / 1000;
+const avgThroughput = (metrics.totalProcessed / actualProcessingDuration).toFixed(0);
+
+console.log('\n' + '='.repeat(80));
+console.log('📊 BENCHMARK RESULTS');
+console.log('='.repeat(80));
+console.log(`Total Messages Processed: ${metrics.totalProcessed.toLocaleString()}`);
+console.log(`Total Wall Clock Time: ${totalDuration.toFixed(2)}s (includes idle timeout)`);
+console.log(`Actual Processing Time: ${actualProcessingDuration.toFixed(2)}s`);
+console.log(`Idle/Wait Time: ${(totalDuration - actualProcessingDuration).toFixed(2)}s`);
+console.log(`Average Throughput: ${avgThroughput} msg/s (based on processing time)`);
+console.log(`Number of Consumers: ${NUMBER_OF_CONSUMERS}`);
+console.log(`Total Batches: ${metrics.batchTimes.length}`);
+console.log(`Total Time: ${metrics.lastMessageTime - metrics.firstMessageTime}ms`);
+console.log(`Total msg/s: ${metrics.totalProcessed / (metrics.lastMessageTime - metrics.firstMessageTime) * 1000} msg/s`);
+
+if (metrics.batchTimes.length > 0) {
+  const avgBatchTime = (metrics.batchTimes.reduce((a, b) => a + b, 0) / metrics.batchTimes.length / 1000).toFixed(3);
+  const minBatchTime = (Math.min(...metrics.batchTimes) / 1000).toFixed(3);
+  const maxBatchTime = (Math.max(...metrics.batchTimes) / 1000).toFixed(3);
+  
+  console.log(`\nBatch Times:`);
+  console.log(`  Average: ${avgBatchTime}s`);
+  console.log(`  Min: ${minBatchTime}s`);
+  console.log(`  Max: ${maxBatchTime}s`);
+}
+
+if (metrics.popTimes.length > 0) {
+  const avgPopTime = (metrics.popTimes.reduce((a, b) => a + b, 0) / metrics.popTimes.length / 1000).toFixed(3);
+  console.log(`\nAverage POP Time: ${avgPopTime}s`);
+}
+
+if (metrics.ackTimes.length > 0) {
+  const avgAckTime = (metrics.ackTimes.reduce((a, b) => a + b, 0) / metrics.ackTimes.length / 1000).toFixed(3);
+  console.log(`Average ACK Time: ${avgAckTime}s`);
+}
+
+console.log(`\nPer-Consumer Stats:`);
+metrics.consumers.forEach(c => {
+  // Calculate actual processing time (sum of all batch times + pop times)
+  const totalBatchTime = c.batchTimes.reduce((a,b)=>a+b,0);
+  const totalPopTime = c.popTimes.reduce((a,b)=>a+b,0);
+  const actualProcessingTime = (totalBatchTime + totalPopTime) / 1000;
+  
+  const wallClockTime = ((c.endTime - c.startTime) / 1000).toFixed(2);
+  const rate = (c.messagesProcessed / actualProcessingTime).toFixed(0);
+  const avgBatch = c.batchTimes.length > 0 ? (totalBatchTime / c.batchTimes.length / 1000).toFixed(3) : 'N/A';
+  const idleTime = (wallClockTime - actualProcessingTime).toFixed(2);
+  
+  const queuesStr = c.queuesConsumed.size > 0 ? `queues:[${Array.from(c.queuesConsumed).sort().join(',')}]` : '';
+  const partitionsStr = c.partitionsConsumed.size > 0 ? `partitions:${c.partitionsConsumed.size}` : '';
+  const consumedFrom = [queuesStr, partitionsStr].filter(Boolean).join(' ');
+  
+  console.log(`  [C${c.consumerId}] ${c.messagesProcessed.toLocaleString()} msgs | Active: ${actualProcessingTime.toFixed(2)}s (${rate} msg/s) | Idle: ${idleTime}s | ${c.batchesProcessed} batches | avg: ${avgBatch}s | ${consumedFrom}`);
+});
+
+console.log('='.repeat(80));
+

package/src/benchmark/producer.js

@@ -0,0 +1,75 @@
+import { Queen } from '../client/client.js'; 
+import fs from 'fs';
+
+const TOTAL_MESSAGES = 100000;
+const PARTITIONS = 10; 
+const MAX_BATCH_SIZE = 1000;  // Reduced from 10000
+const PUSH_PARALLEL = 10;      // Reduced from 10 to avoid lock contention
+const QUEUE_NAME = 'benchmark-queue-001';
+
+const q = new Queen({
+  baseUrls: ['http://localhost:6632'],
+  timeout: 120000
+});
+
+//await q.queueDelete(QUEUE_NAME);
+await q.queue(QUEUE_NAME, {
+  leaseTime: 10,
+  retryLimit: 3
+}, { namespace: 'benchmark' });
+
+
+
+let batches = []
+const messagesPerPartition = Math.floor(TOTAL_MESSAGES / PARTITIONS);
+
+// Create batches for each partition
+const partitionBatches = [];
+for (let i = 0; i < PARTITIONS; i++) {
+  partitionBatches[i] = [];
+  let messages = [];
+  for (let j = 0; j < messagesPerPartition; j++) {
+    messages.push({
+      id: j,
+      //message: `Message ${globalMessageId}`,
+      //timestamp: new Date().toISOString()
+    });
+    if (messages.length >= MAX_BATCH_SIZE) {
+      console.log(`Pushing ${messages.length} messages for partition ${i}`);
+      partitionBatches[i].push({
+        partition: i,
+        messages: messages
+      });
+      messages = [];
+    }
+  }
+  // Only add the remaining batch if it has messages
+  if (messages.length > 0) {
+    console.log(`Pushing ${messages.length} messages for partition ${i}`);
+    partitionBatches[i].push({
+      partition: i,
+      messages: messages
+    });
+  }
+}
+
+// Interleave batches from different partitions to avoid lock contention
+// This ensures parallel pushes go to different partitions
+const maxBatchesPerPartition = Math.max(...partitionBatches.map(pb => pb.length));
+for (let batchIndex = 0; batchIndex < maxBatchesPerPartition; batchIndex++) {
+  for (let partition = 0; partition < PARTITIONS; partition++) {
+    if (partitionBatches[partition][batchIndex]) {
+      batches.push(partitionBatches[partition][batchIndex]);
+    }
+  }
+}
+
+const start = Date.now();
+console.log(`Total batches: ${batches.length}, pushing ${PUSH_PARALLEL} at a time`);
+for (let i = 0; i < batches.length; i += PUSH_PARALLEL) {
+  const batchGroup = batches.slice(i, i + PUSH_PARALLEL);
+  console.log(`Pushing batch group ${Math.floor(i/PUSH_PARALLEL) + 1}/${Math.ceil(batches.length/PUSH_PARALLEL)}: ${batchGroup.map(b => `partition ${b.partition} (${b.messages.length} msgs)`).join(', ')}`);
+  await Promise.all(batchGroup.map(batch => q.push(`${QUEUE_NAME}/${batch.partition}`, batch.messages)));
+}
+const end = Date.now();
+console.log(`Pushed ${TOTAL_MESSAGES} messages in ${(end - start) / 1000}s, ${(TOTAL_MESSAGES / ((end - start) / 1000)).toFixed(2)} msg/s`);

package/src/benchmark/producer_multi.js

@@ -0,0 +1,115 @@
+import { Queen } from '../client/client.js'; 
+import fs from 'fs';
+
+const TOTAL_MESSAGES = 100000;
+const NUM_QUEUES = 50;          // X: Number of queues
+const PARTITIONS_PER_QUEUE = 10; // Y: Partitions per queue
+const MAX_BATCH_SIZE = 10;  
+const PUSH_PARALLEL = 10;      
+
+const q = new Queen({
+  baseUrls: ['http://localhost:6632'],
+  timeout: 120000
+});
+
+// Create X queues, each with Y partitions, all in 'benchmark' namespace
+console.log(`Creating ${NUM_QUEUES} queues with ${PARTITIONS_PER_QUEUE} partitions each...`);
+for (let queueIdx = 0; queueIdx < NUM_QUEUES; queueIdx++) {
+  const queueName = `benchmark-queue-${String(queueIdx + 1).padStart(2, '0')}`;
+  try {
+    //await q.queueDelete(queueName);
+  } catch (e) {
+    // Queue might not exist
+  }
+  await q.queue(queueName, {
+    leaseTime: 10,
+    retryLimit: 3
+  }, { namespace: 'benchmark' });
+  console.log(`  Created queue: ${queueName}`);
+}
+
+
+// Calculate total partitions across all queues
+const TOTAL_PARTITIONS = NUM_QUEUES * PARTITIONS_PER_QUEUE;
+const messagesPerPartition = Math.floor(TOTAL_MESSAGES / TOTAL_PARTITIONS);
+
+console.log(`\nDistributing ${TOTAL_MESSAGES} messages across ${NUM_QUEUES} queues × ${PARTITIONS_PER_QUEUE} partitions = ${TOTAL_PARTITIONS} total partitions`);
+console.log(`Messages per partition: ${messagesPerPartition}\n`);
+
+// Create batches for each queue/partition combination
+const queuePartitionBatches = [];
+for (let queueIdx = 0; queueIdx < NUM_QUEUES; queueIdx++) {
+  for (let partitionIdx = 0; partitionIdx < PARTITIONS_PER_QUEUE; partitionIdx++) {
+    const key = `q${queueIdx}_p${partitionIdx}`;
+    queuePartitionBatches.push({ queueIdx, partitionIdx, batches: [] });
+    
+    let messages = [];
+    for (let msgIdx = 0; msgIdx < messagesPerPartition; msgIdx++) {
+      messages.push({
+        id: msgIdx,
+        queue: queueIdx,
+        partition: partitionIdx
+      });
+      
+      if (messages.length >= MAX_BATCH_SIZE) {
+        queuePartitionBatches[queuePartitionBatches.length - 1].batches.push([...messages]);
+        messages = [];
+      }
+    }
+    
+    // Add remaining messages
+    if (messages.length > 0) {
+      queuePartitionBatches[queuePartitionBatches.length - 1].batches.push([...messages]);
+    }
+  }
+}
+
+// Flatten and interleave batches to distribute load evenly
+let batches = [];
+const maxBatchesPerQueuePartition = Math.max(...queuePartitionBatches.map(qp => qp.batches.length));
+for (let batchIdx = 0; batchIdx < maxBatchesPerQueuePartition; batchIdx++) {
+  for (const qp of queuePartitionBatches) {
+    if (qp.batches[batchIdx]) {
+      batches.push({
+        queueIdx: qp.queueIdx,
+        partitionIdx: qp.partitionIdx,
+        messages: qp.batches[batchIdx]
+      });
+    }
+  }
+}
+
+const start = Date.now();
+console.log(`Total batches: ${batches.length}, pushing ${PUSH_PARALLEL} at a time\n`);
+
+for (let i = 0; i < batches.length; i += PUSH_PARALLEL) {
+  const batchGroup = batches.slice(i, i + PUSH_PARALLEL);
+  const batchGroupNum = Math.floor(i/PUSH_PARALLEL) + 1;
+  const totalBatchGroups = Math.ceil(batches.length/PUSH_PARALLEL);
+  
+  console.log(`[${batchGroupNum}/${totalBatchGroups}] Pushing: ${batchGroup.map(b => {
+    const queueName = `benchmark-queue-${String(b.queueIdx + 1).padStart(2, '0')}`;
+    return `${queueName}/p${b.partitionIdx} (${b.messages.length} msgs)`;
+  }).join(', ')}`);
+  
+  await Promise.all(batchGroup.map(batch => {
+    const queueName = `benchmark-queue-${String(batch.queueIdx + 1).padStart(2, '0')}`;
+    return q.push(`${queueName}/${batch.partitionIdx}`, batch.messages);
+  }));
+}
+
+const end = Date.now();
+const duration = (end - start) / 1000;
+const throughput = (TOTAL_MESSAGES / duration).toFixed(2);
+
+console.log('\n' + '='.repeat(80));
+console.log('📤 PRODUCER RESULTS');
+console.log('='.repeat(80));
+console.log(`Total Messages Pushed: ${TOTAL_MESSAGES.toLocaleString()}`);
+console.log(`Queues: ${NUM_QUEUES}`);
+console.log(`Partitions per Queue: ${PARTITIONS_PER_QUEUE}`);
+console.log(`Total Partitions: ${TOTAL_PARTITIONS}`);
+console.log(`Duration: ${duration.toFixed(2)}s`);
+console.log(`Throughput: ${throughput} msg/s`);
+console.log('='.repeat(80));
+