queen-mq 0.1.5 → 0.2.0

package/examples/batch-processing.js

@@ -1,58 +0,0 @@
-import { Queen } from '../src/client/client.js';
-
-const q = new Queen({ baseUrl: 'http://localhost:6632' });
-
-await q.queue('demo-queue', {
-  leaseTime: 30,
-  retryLimit: 3
-});
-
-console.log('=== Example 1: take() - Individual message processing ===');
-let count1 = 0;
-for await (const message of q.take('demo-queue', { 
-  batch: 10,      // Fetch 10 at a time internally
-  limit: 25,      // Process 25 total
-  wait: false 
-})) {
-  console.log(`Message ${++count1}:`, message.data);
-  await q.ack(message);
-}
-
-console.log('\n=== Example 2: takeBatch() - Batch processing ===');
-let count2 = 0;
-for await (const messages of q.takeBatch('demo-queue', { 
-  batch: 10,      // Fetch 10 at a time
-  limit: 25,      // Process 25 total (across batches)
-  wait: false 
-})) {
-  console.log(`Batch ${++count2}: ${messages.length} messages`);
-  
-  // Process entire batch
-  for (const msg of messages) {
-    console.log(`  - ${msg.data}`);
-  }
-  
-  // Acknowledge entire batch at once
-  await q.ack(messages);
-}
-
-console.log('\n=== Example 3: High-throughput batch processing ===');
-let totalProcessed = 0;
-const startTime = Date.now();
-
-for await (const messages of q.takeBatch('demo-queue', { 
-  batch: 1000,       // Large batches for throughput
-  wait: true,        // Long polling
-  idleTimeout: 5000  // Exit after 5s of no messages
-})) {
-  totalProcessed += messages.length;
-  await q.ack(messages);
-  
-  const rate = totalProcessed / ((Date.now() - startTime) / 1000);
-  console.log(`Processed ${messages.length} | Total: ${totalProcessed} | Rate: ${rate.toFixed(0)} msg/s`);
-}
-
-console.log(`\nCompleted: ${totalProcessed} messages`);
-
-await q.close();
-

package/examples/bus-mode.js

@@ -1,239 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Bus Mode Example for Queen V3
- * 
- * This example demonstrates the new bus/pub-sub functionality where
- * multiple consumer groups can receive ALL messages independently,
- * similar to Kafka's consumer groups.
- * 
- * Each consumer group maintains its own progress through the message stream.
- */
-
-import { createQueenClient } from '../src/client/index.js';
-
-async function main() {
-  const client = createQueenClient({
-    baseUrl: process.env.QUEEN_URL || 'http://localhost:6632',
-    timeout: 35000
-  });
-
-  const queue = 'events';
-  
-  console.log('🚀 Starting Bus Mode Example');
-  console.log('📦 Queue:', queue);
-  console.log('');
-  console.log('This example will:');
-  console.log('1. Push some test messages to the queue');
-  console.log('2. Start 3 different consumer groups');
-  console.log('3. Show how each group receives ALL messages independently');
-  console.log('');
-  
-  // First, push some test messages
-  console.log('📤 Pushing test messages...');
-  const testMessages = [];
-  for (let i = 1; i <= 5; i++) {
-    testMessages.push({
-      queue,
-      payload: {
-        id: i,
-        type: 'test-event',
-        timestamp: new Date().toISOString(),
-        data: `Message ${i}`
-      }
-    });
-  }
-  
-  await client.push({ items: testMessages });
-  console.log(`✅ Pushed ${testMessages.length} messages\n`);
-  
-  // Track messages received by each consumer group
-  const received = {
-    analytics: [],
-    notifications: [],
-    audit: []
-  };
-  
-  // Consumer Group 1: Analytics Service
-  console.log('🔵 Starting Analytics Consumer Group...');
-  const analyticsConsumer = client.consume({
-    queue,
-    consumerGroup: 'analytics-service',
-    handler: async (message) => {
-      console.log(`  [Analytics] Received message ${message.data.id}: ${message.data.data}`);
-      received.analytics.push(message.data.id);
-      
-      // Simulate processing
-      await new Promise(resolve => setTimeout(resolve, 100));
-    },
-    options: {
-      batch: 2,
-      wait: false  // Don't wait for new messages in this example
-    }
-  });
-  
-  // Consumer Group 2: Notification Service
-  console.log('🟢 Starting Notifications Consumer Group...');
-  const notificationConsumer = client.consume({
-    queue,
-    consumerGroup: 'notification-service',
-    handler: async (message) => {
-      console.log(`  [Notifications] Received message ${message.data.id}: ${message.data.data}`);
-      received.notifications.push(message.data.id);
-      
-      // Simulate processing
-      await new Promise(resolve => setTimeout(resolve, 150));
-    },
-    options: {
-      batch: 1,
-      wait: false
-    }
-  });
-  
-  // Consumer Group 3: Audit Log Service
-  console.log('🟡 Starting Audit Log Consumer Group...');
-  const auditConsumer = client.consume({
-    queue,
-    consumerGroup: 'audit-log',
-    handler: async (message) => {
-      console.log(`  [Audit] Received message ${message.data.id}: ${message.data.data}`);
-      received.audit.push(message.data.id);
-      
-      // Simulate processing
-      await new Promise(resolve => setTimeout(resolve, 50));
-    },
-    options: {
-      batch: 3,
-      wait: false
-    }
-  });
-  
-  // Wait for all messages to be processed
-  console.log('\n⏳ Processing messages...\n');
-  await new Promise(resolve => setTimeout(resolve, 3000));
-  
-  // Stop all consumers
-  analyticsConsumer();
-  notificationConsumer();
-  auditConsumer();
-  
-  // Display results
-  console.log('\n📊 Results:');
-  console.log('═══════════════════════════════════════════');
-  console.log(`Analytics Service received: [${received.analytics.sort().join(', ')}]`);
-  console.log(`Notification Service received: [${received.notifications.sort().join(', ')}]`);
-  console.log(`Audit Log Service received: [${received.audit.sort().join(', ')}]`);
-  console.log('');
-  
-  // Verify all groups received all messages
-  const allReceived = 
-    received.analytics.length === testMessages.length &&
-    received.notifications.length === testMessages.length &&
-    received.audit.length === testMessages.length;
-  
-  if (allReceived) {
-    console.log('✅ SUCCESS: All consumer groups received ALL messages!');
-    console.log('   This demonstrates bus/pub-sub mode where each consumer group');
-    console.log('   gets its own copy of every message, independently.');
-  } else {
-    console.log('⚠️  Some messages were not received by all groups.');
-    console.log('   This might be due to timing. Try running again.');
-  }
-  
-  console.log('\n🎯 Key Differences from Queue Mode:');
-  console.log('  • In Queue Mode: Messages are consumed by ONE worker');
-  console.log('  • In Bus Mode: Messages are consumed by ALL consumer groups');
-  console.log('  • Each group maintains its own progress/offset');
-  console.log('  • Perfect for: analytics, notifications, audit logs, etc.');
-}
-
-// Demonstrate subscription modes
-async function demonstrateSubscriptionModes() {
-  const client = createQueenClient({
-    baseUrl: process.env.QUEEN_URL || 'http://localhost:6632'
-  });
-  
-  const queue = 'subscription-test';
-  
-  console.log('\n\n📅 Demonstrating Subscription Modes');
-  console.log('════════════════════════════════════════════');
-  
-  // Push some historical messages
-  console.log('📤 Pushing historical messages...');
-  for (let i = 1; i <= 3; i++) {
-    await client.push({
-      items: [{
-        queue,
-        payload: { id: i, type: 'historical', data: `Historical message ${i}` }
-      }]
-    });
-  }
-  
-  console.log('✅ Historical messages pushed\n');
-  
-  // Consumer 1: Consume all messages (default)
-  console.log('🔵 Consumer Group "all-messages" (default mode):');
-  const allMessages = [];
-  const consumer1 = client.consume({
-    queue,
-    consumerGroup: 'all-messages',
-    handler: async (msg) => {
-      console.log(`  Received: ${msg.data.type} - ${msg.data.data}`);
-      allMessages.push(msg.data.id);
-    },
-    options: { wait: false }
-  });
-  
-  await new Promise(resolve => setTimeout(resolve, 1000));
-  consumer1();
-  
-  // Consumer 2: Only new messages from subscription time
-  console.log('\n🟢 Consumer Group "new-only" (subscriptionMode: "new"):');
-  const newOnly = [];
-  const consumer2 = client.consume({
-    queue,
-    consumerGroup: 'new-only',
-    handler: async (msg) => {
-      console.log(`  Received: ${msg.data.type} - ${msg.data.data}`);
-      newOnly.push(msg.data.id);
-    },
-    options: { 
-      wait: false,
-      subscriptionMode: 'new'
-    }
-  });
-  
-  // Push new messages after subscription
-  console.log('\n📤 Pushing new messages...');
-  for (let i = 4; i <= 6; i++) {
-    await client.push({
-      items: [{
-        queue,
-        payload: { id: i, type: 'new', data: `New message ${i}` }
-      }]
-    });
-  }
-  
-  await new Promise(resolve => setTimeout(resolve, 1000));
-  consumer2();
-  
-  // Display results
-  console.log('\n📊 Subscription Mode Results:');
-  console.log('════════════════════════════════════');
-  console.log(`"all-messages" group received: [${allMessages.sort().join(', ')}] (all 6 messages)`);
-  console.log(`"new-only" group received: [${newOnly.sort().join(', ')}] (only new messages)`);
-  console.log('');
-  console.log('✅ This demonstrates how consumer groups can choose when to start consuming!');
-}
-
-// Run the examples
-main()
-  .then(() => demonstrateSubscriptionModes())
-  .then(() => {
-    console.log('\n✨ Bus mode examples completed!');
-    process.exit(0);
-  })
-  .catch(error => {
-    console.error('❌ Error:', error);
-    process.exit(1);
-  });

package/examples/continuous-consumer-optimized.js

@@ -1,215 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Optimized Continuous Consumer Example
- * 
- * This optimized version processes messages in parallel batches
- * with batch acknowledgments for maximum throughput.
- */
-
-import { createQueenClient } from '../src/client/index.js';
-
-async function main() {
-  // Create client with configuration
-  const client = createQueenClient({
-    baseUrl: process.env.QUEEN_URL || 'http://localhost:6632',
-    timeout: 35000  // Client timeout for non-polling requests
-  });
-
-  const namespace = process.env.NS || 'smartchat';
-  const task = process.env.TASK || 'translations';
-  const queue = process.env.QUEUE || 'email';
-  
-  // Optimized settings for high throughput
-  const batchSize = parseInt(process.env.BATCH_SIZE) || 50;  // Reasonable batch size
-  const parallelWorkers = parseInt(process.env.WORKERS) || 10;  // Process in parallel
-  const pollTimeout = parseInt(process.env.POLL_TIMEOUT) || 5000;  // Shorter timeout for faster response
-
-  console.log('🚀 Starting OPTIMIZED continuous consumer');
-  console.log(`📦 Queue: ${namespace}/${task}/${queue}`);
-  console.log('⚡ Performance settings:');
-  console.log(`   - Batch size: ${batchSize}`);
-  console.log(`   - Parallel workers: ${parallelWorkers}`);
-  console.log(`   - Poll timeout: ${pollTimeout}ms`);
-  console.log('');
-
-  let stats = {
-    messages: 0,
-    errors: 0,
-    startTime: Date.now(),
-    lastMessageTime: null,
-    processingTimes: []
-  };
-
-  // Process message function
-  async function processMessage(message) {
-    // Your actual processing logic here
-    // For demo, just simulate minimal work
-    return new Promise(resolve => setImmediate(resolve));
-  }
-
-  // Optimized consume function with parallel processing
-  async function consumeOptimized() {
-    let running = true;
-    const activeWorkers = new Set();
-    
-    // Handle shutdown
-    process.on('SIGINT', () => {
-      console.log('\n\n🛑 Shutting down consumer...');
-      running = false;
-    });
-
-    while (running) {
-      try {
-        // Wait if we have too many active workers
-        while (activeWorkers.size >= parallelWorkers && running) {
-          await new Promise(resolve => setTimeout(resolve, 10));
-        }
-
-        if (!running) break;
-
-        // Start a new worker
-        const worker = (async () => {
-          try {
-            // Pop messages with optimized settings
-            const result = await client.pop({ 
-              ns: namespace,
-              task: task,
-              queue: queue,
-              wait: true,
-              timeout: pollTimeout,
-              batch: batchSize
-            });
-            
-            if (result.messages && result.messages.length > 0) {
-              const startTime = Date.now();
-              
-              // Process messages in parallel within the batch
-              const processPromises = result.messages.map(async (message) => {
-                try {
-                  await processMessage(message);
-                  stats.messages++;
-                  stats.lastMessageTime = Date.now();
-                  return { transactionId: message.transactionId, status: 'completed' };
-                } catch (error) {
-                  stats.errors++;
-                  console.error(`Error processing message ${message.id}:`, error.message);
-                  return { transactionId: message.transactionId, status: 'failed', error: error.message };
-                }
-              });
-
-              // Wait for all messages in batch to process
-              const acknowledgments = await Promise.all(processPromises);
-              
-              // Batch acknowledge all messages at once
-              try {
-                await client.ackBatch(acknowledgments);
-              } catch (error) {
-                console.error('Batch ACK failed:', error);
-                // Fall back to individual ACKs if batch fails
-                for (const ack of acknowledgments) {
-                  try {
-                    await client.ack(ack.transactionId, ack.status, ack.error);
-                  } catch (e) {
-                    console.error(`Individual ACK failed for ${ack.transactionId}:`, e.message);
-                  }
-                }
-              }
-              
-              const processingTime = Date.now() - startTime;
-              stats.processingTimes.push(processingTime);
-              
-              // Keep only last 100 processing times for average calculation
-              if (stats.processingTimes.length > 100) {
-                stats.processingTimes.shift();
-              }
-              
-              console.log(`✅ Processed batch of ${result.messages.length} messages in ${processingTime}ms`);
-            }
-          } catch (error) {
-            if (error.name !== 'AbortError') {
-              console.error('Worker error:', error);
-              await new Promise(resolve => setTimeout(resolve, 1000));
-            }
-          } finally {
-            activeWorkers.delete(worker);
-          }
-        })();
-
-        activeWorkers.add(worker);
-      } catch (error) {
-        console.error('Main loop error:', error);
-        await new Promise(resolve => setTimeout(resolve, 1000));
-      }
-    }
-
-    // Wait for all workers to complete
-    await Promise.all(Array.from(activeWorkers));
-    
-    // Final stats
-    const uptime = Math.floor((Date.now() - stats.startTime) / 1000);
-    const avgProcessingTime = stats.processingTimes.length > 0 
-      ? stats.processingTimes.reduce((a, b) => a + b, 0) / stats.processingTimes.length
-      : 0;
-    
-    console.log('\n📈 Final Statistics:');
-    console.log(`  Total runtime: ${formatUptime(uptime)}`);
-    console.log(`  Messages processed: ${stats.messages}`);
-    console.log(`  Throughput: ${(stats.messages / uptime).toFixed(2)} msg/s`);
-    console.log(`  Average batch processing time: ${avgProcessingTime.toFixed(2)}ms`);
-    console.log(`  Errors encountered: ${stats.errors}`);
-    
-    process.exit(0);
-  }
-
-  // Print statistics every 10 seconds
-  const statsInterval = setInterval(() => {
-    if (stats.messages === 0) return;
-    
-    const uptime = Math.floor((Date.now() - stats.startTime) / 1000);
-    const throughput = stats.messages / uptime;
-    const avgProcessingTime = stats.processingTimes.length > 0 
-      ? stats.processingTimes.reduce((a, b) => a + b, 0) / stats.processingTimes.length
-      : 0;
-    
-    console.log('\n📊 Statistics:');
-    console.log(`  Uptime: ${formatUptime(uptime)}`);
-    console.log(`  Messages processed: ${stats.messages}`);
-    console.log(`  Throughput: ${throughput.toFixed(2)} msg/s`);
-    console.log(`  Average batch processing: ${avgProcessingTime.toFixed(2)}ms`);
-    console.log(`  Errors: ${stats.errors}`);
-    
-    if (stats.lastMessageTime) {
-      const idle = Math.floor((Date.now() - stats.lastMessageTime) / 1000);
-      if (idle > 2) {
-        console.log(`  ⚠️ Idle time: ${idle}s`);
-      }
-    }
-    console.log('');
-  }, 10000);
-
-  console.log('✅ Consumer started. Press Ctrl+C to stop.\n');
-  
-  // Start consuming
-  await consumeOptimized();
-  clearInterval(statsInterval);
-}
-
-function formatUptime(seconds) {
-  const hours = Math.floor(seconds / 3600);
-  const minutes = Math.floor((seconds % 3600) / 60);
-  const secs = seconds % 60;
-  
-  const parts = [];
-  if (hours > 0) parts.push(`${hours}h`);
-  if (minutes > 0) parts.push(`${minutes}m`);
-  parts.push(`${secs}s`);
-  
-  return parts.join(' ');
-}
-
-// Run the consumer
-main().catch(error => {
-  console.error('❌ Fatal error:', error);
-  process.exit(1);
-});

package/examples/continuous-consumer.js

@@ -1,159 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Continuous Consumer Example for Queen V2
- * 
- * This example demonstrates how the Queen client performs continuous
- * long polling without delays when timeouts occur. The consumer will
- * immediately start a new request when the previous one times out,
- * ensuring no messages are missed and minimizing latency.
- */
-
-import { createQueenClient } from '../src/client/index.js';
-
-async function main() {
-  // Create client with configuration
-  const client = createQueenClient({
-    baseUrl: process.env.QUEEN_URL || 'http://localhost:6632',
-    timeout: 35000  // Client timeout for non-polling requests
-  });
-
-  // V2: Using queue and optional partition
-  const queue = process.env.QUEUE || 'email-translations';
-  const partition = process.env.PARTITION;  // Optional: if not set, consumes from any partition
-  const namespace = process.env.NAMESPACE;  // Optional: for filtering
-  const task = process.env.TASK;           // Optional: for filtering
-
-  console.log('🚀 Starting continuous consumer (V2)');
-  if (queue) {
-    console.log(`📦 Queue: ${queue}${partition ? '/' + partition : ' (any partition)'}`);
-  } else if (namespace || task) {
-    console.log(`🔍 Filtering by: ${namespace ? 'namespace=' + namespace : ''} ${task ? 'task=' + task : ''}`);
-  }
-  console.log('⏱️  Long polling timeout: 30 seconds');
-  console.log('🔄 Will immediately retry on timeout (no delays)\n');
-
-  let stats = {
-    messages: 0,
-    errors: 0,
-    startTime: Date.now(),
-    lastMessageTime: null,
-    partitionCounts: {}
-  };
-
-  // Start the consumer
-  const stop = client.consume({
-    queue: queue,
-    partition: partition,
-    namespace: namespace,
-    task: task,
-    handler: async (message) => {
-      stats.messages++;
-      stats.lastMessageTime = Date.now();
-      
-      // Track messages per partition
-      const partitionKey = `${message.queue}/${message.partition}`;
-      stats.partitionCounts[partitionKey] = (stats.partitionCounts[partitionKey] || 0) + 1;
-      
-      console.log(`✉️  [${new Date().toISOString()}] Message received:`, {
-        queue: message.queue,
-        partition: message.partition,
-        transactionId: message.transactionId,
-        retryCount: message.retryCount,
-        data: message.data
-      });
-
-      // Simulate processing
-      await processMessage(message);
-    },
-    options: {
-      wait: true,         // Enable long polling
-      timeout: 30000,     // 30 second server timeout
-      batch: 10,          // Process up to 10 messages at once
-      stopOnError: false  // Continue processing on errors
-    }
-  });
-
-  // Print statistics every minute
-  const statsInterval = setInterval(() => {
-    const uptime = Math.floor((Date.now() - stats.startTime) / 1000);
-    const rate = stats.messages / (uptime / 60);
-    
-    console.log('\n📊 Statistics:');
-    console.log(`  Uptime: ${formatUptime(uptime)}`);
-    console.log(`  Messages processed: ${stats.messages}`);
-    console.log(`  Processing rate: ${rate.toFixed(2)} msg/min`);
-    console.log(`  Errors: ${stats.errors}`);
-    
-    // Show partition distribution
-    if (Object.keys(stats.partitionCounts).length > 0) {
-      console.log('  Distribution by partition:');
-      for (const [partition, count] of Object.entries(stats.partitionCounts)) {
-        const percentage = ((count / stats.messages) * 100).toFixed(1);
-        console.log(`    ${partition}: ${count} messages (${percentage}%)`);
-      }
-    }
-    
-    if (stats.lastMessageTime) {
-      const idle = Math.floor((Date.now() - stats.lastMessageTime) / 1000);
-      console.log(`  Idle time: ${idle}s`);
-    }
-    console.log('');
-  }, 60000);
-
-  // Handle shutdown
-  process.on('SIGINT', () => {
-    console.log('\n\n🛑 Shutting down consumer...');
-    
-    clearInterval(statsInterval);
-    stop();
-    
-    const uptime = Math.floor((Date.now() - stats.startTime) / 1000);
-    console.log('\n📈 Final Statistics:');
-    console.log(`  Total runtime: ${formatUptime(uptime)}`);
-    console.log(`  Messages processed: ${stats.messages}`);
-    console.log(`  Errors encountered: ${stats.errors}`);
-    
-    if (Object.keys(stats.partitionCounts).length > 0) {
-      console.log('  Final partition distribution:');
-      for (const [partition, count] of Object.entries(stats.partitionCounts)) {
-        const percentage = ((count / stats.messages) * 100).toFixed(1);
-        console.log(`    ${partition}: ${count} messages (${percentage}%)`);
-      }
-    }
-    
-    process.exit(0);
-  });
-
-  console.log('✅ Consumer started. Press Ctrl+C to stop.\n');
-}
-
-async function processMessage(message) {
-  // Simulate some processing work
-  const processingTime = Math.random() * 1000 + 500; // 0.5-1.5 seconds
-  await new Promise(resolve => setTimeout(resolve, processingTime));
-  
-  // Randomly simulate occasional processing errors (5% chance)
-  if (Math.random() < 0.05) {
-    throw new Error('Simulated processing error');
-  }
-}
-
-function formatUptime(seconds) {
-  const hours = Math.floor(seconds / 3600);
-  const minutes = Math.floor((seconds % 3600) / 60);
-  const secs = seconds % 60;
-  
-  const parts = [];
-  if (hours > 0) parts.push(`${hours}h`);
-  if (minutes > 0) parts.push(`${minutes}m`);
-  parts.push(`${secs}s`);
-  
-  return parts.join(' ');
-}
-
-// Run the consumer
-main().catch(error => {
-  console.error('❌ Fatal error:', error);
-  process.exit(1);
-});