queen-mq 0.1.1 → 0.1.3

package/examples/test-connection-recovery.js

@@ -0,0 +1,66 @@
+/**
+ * Test connection recovery during take operations
+ * This demonstrates how the client handles server disconnections gracefully
+ */
+
+import { Queen } from '../src/client/index.js';
+
+async function testConnectionRecovery() {
+  const client = new Queen({
+    baseUrl: 'http://localhost:6632',
+    retryDelay: 1000,  // 1 second base retry delay
+    retryAttempts: 3
+  });
+
+  try {
+    // Configure a test queue
+    await client.queue('test-recovery-queue');
+    console.log('Queue configured');
+
+    // Push some test messages
+    await client.push('test-recovery-queue', [
+      { message: 'Test message 1' },
+      { message: 'Test message 2' },
+      { message: 'Test message 3' }
+    ]);
+    console.log('Pushed 3 test messages');
+
+    console.log('\nStarting consumer...');
+    console.log('To test: Stop the server after receiving some messages');
+    console.log('Expected behavior: Client will retry with exponential backoff indefinitely (max 30s between retries)');
+    console.log('                   When server comes back online, it will automatically reconnect\n');
+
+    let messageCount = 0;
+
+    // Consume messages with takeBatch
+    for await (const batch of client.takeBatch('test-recovery-queue', { 
+      batch: 10, 
+      wait: true,
+      timeout: 5000 
+    })) {
+      console.log(`\nReceived batch of ${batch.length} messages:`);
+      
+      for (const message of batch) {
+        messageCount++;
+        console.log(`  ${messageCount}. ${JSON.stringify(message.payload)}`);
+        
+        // Acknowledge the message
+        await client.ack(message, true);
+      }
+    }
+
+    console.log('\nConsumer loop completed');
+    console.log(`Total messages processed: ${messageCount}`);
+
+  } catch (error) {
+    console.error('\nError in consumer:', error.message);
+    console.error('Stack:', error.stack);
+  } finally {
+    await client.close();
+    console.log('\nClient closed');
+  }
+}
+
+console.log('=== Connection Recovery Test ===\n');
+testConnectionRecovery().catch(console.error);
+

package/examples/test-dashboard-api.js

@@ -0,0 +1,200 @@
+/**
+ * Test script for the new Dashboard Status API endpoints
+ * 
+ * This script demonstrates all 5 new endpoints:
+ * 1. GET /api/v1/status - Dashboard overview
+ * 2. GET /api/v1/status/queues - List queues with stats
+ * 3. GET /api/v1/status/queues/:queue - Queue detail
+ * 4. GET /api/v1/status/queues/:queue/messages - Queue messages
+ * 5. GET /api/v1/status/analytics - Advanced analytics
+ */
+
+import { createQueenClient } from '../src/client/queenClient.js';
+
+const BASE_URL = process.env.QUEEN_URL || 'http://localhost:3000';
+
+// Create client
+const client = createQueenClient({ baseUrl: BASE_URL });
+
+// Helper to make GET requests
+async function get(path) {
+  const response = await fetch(`${BASE_URL}${path}`, {
+    method: 'GET',
+    headers: {
+      'Content-Type': 'application/json'
+    }
+  });
+  
+  if (!response.ok) {
+    const error = await response.text();
+    throw new Error(`HTTP ${response.status}: ${error}`);
+  }
+  
+  return await response.json();
+}
+
+// Helper to format JSON output
+function printJson(label, data) {
+  console.log(`\n${'='.repeat(80)}`);
+  console.log(`📊 ${label}`);
+  console.log('='.repeat(80));
+  console.log(JSON.stringify(data, null, 2));
+}
+
+async function runTests() {
+  console.log('🚀 Testing Queen Dashboard Status API\n');
+  
+  try {
+    // First, let's create some test data
+    console.log('📝 Setting up test data...');
+    
+    // Configure a test queue
+    await client.configure({
+      queue: 'dashboard-test-queue',
+      namespace: 'test-namespace',
+      task: 'test-task',
+      partitions: 3
+    });
+    
+    // Push some test messages
+    const messages = [];
+    for (let i = 0; i < 20; i++) {
+      messages.push({
+        queue: 'dashboard-test-queue',
+        partition: `partition-${i % 3}`,
+        payload: {
+          messageNumber: i,
+          timestamp: new Date().toISOString(),
+          data: `Test message ${i}`
+        }
+      });
+    }
+    
+    await client.pushBatch(messages);
+    console.log('✅ Created test queue with 20 messages\n');
+    
+    // Wait a bit for messages to be processed
+    await new Promise(resolve => setTimeout(resolve, 1000));
+    
+    // Test 1: Dashboard Status
+    console.log('🧪 Test 1: Dashboard Status (GET /api/v1/status)');
+    const status = await get('/api/v1/status');
+    printJson('Dashboard Status', {
+      timeRange: status.timeRange,
+      queuesCount: status.queues.length,
+      messages: status.messages,
+      leases: status.leases,
+      throughputSamples: status.throughput.length,
+      dlqTotal: status.deadLetterQueue.totalMessages
+    });
+    
+    // Test 2: Queues List
+    console.log('\n🧪 Test 2: List Queues (GET /api/v1/status/queues)');
+    const queues = await get('/api/v1/status/queues?limit=10');
+    printJson('Queues List', {
+      total: queues.pagination.total,
+      queues: queues.queues.map(q => ({
+        name: q.name,
+        namespace: q.namespace,
+        partitions: q.partitions,
+        pending: q.messages.pending,
+        completed: q.messages.completed
+      }))
+    });
+    
+    // Test 3: Queue Detail
+    console.log('\n🧪 Test 3: Queue Detail (GET /api/v1/status/queues/:queue)');
+    const queueDetail = await get('/api/v1/status/queues/dashboard-test-queue');
+    printJson('Queue Detail', {
+      queue: queueDetail.queue,
+      totals: queueDetail.totals,
+      partitionCount: queueDetail.partitions.length,
+      partitions: queueDetail.partitions.map(p => ({
+        name: p.name,
+        messages: p.messages,
+        cursor: p.cursor
+      }))
+    });
+    
+    // Test 4: Queue Messages
+    console.log('\n🧪 Test 4: Queue Messages (GET /api/v1/status/queues/:queue/messages)');
+    const messages_result = await get('/api/v1/status/queues/dashboard-test-queue/messages?limit=5');
+    printJson('Queue Messages', {
+      total: messages_result.pagination.total,
+      showing: messages_result.messages.length,
+      messages: messages_result.messages.map(m => ({
+        transactionId: m.transactionId,
+        partition: m.partition,
+        status: m.status,
+        createdAt: m.createdAt
+      }))
+    });
+    
+    // Test 5: Filter by status
+    console.log('\n🧪 Test 5: Filter Messages by Status (GET /api/v1/status/queues/:queue/messages?status=pending)');
+    const pendingMessages = await get('/api/v1/status/queues/dashboard-test-queue/messages?status=pending&limit=5');
+    printJson('Pending Messages', {
+      total: pendingMessages.pagination.total,
+      messages: pendingMessages.messages.map(m => ({
+        transactionId: m.transactionId,
+        status: m.status,
+        age: m.age
+      }))
+    });
+    
+    // Test 6: Analytics (last hour with minute interval)
+    console.log('\n🧪 Test 6: Analytics (GET /api/v1/status/analytics?interval=minute)');
+    const analytics = await get('/api/v1/status/analytics?interval=minute');
+    printJson('Analytics Overview', {
+      timeRange: analytics.timeRange,
+      throughput: {
+        totals: analytics.throughput.totals,
+        recentSamples: analytics.throughput.timeSeries.slice(0, 3)
+      },
+      latency: analytics.latency.overall,
+      errorRates: analytics.errorRates.overall,
+      currentDepth: analytics.queueDepths.current,
+      topQueues: analytics.topQueues.slice(0, 3),
+      dlqTotal: analytics.deadLetterQueue.total
+    });
+    
+    // Test 7: Filter by namespace
+    console.log('\n🧪 Test 7: Dashboard Status filtered by namespace (GET /api/v1/status?namespace=test-namespace)');
+    const statusFiltered = await get('/api/v1/status?namespace=test-namespace');
+    printJson('Filtered Status', {
+      queuesCount: statusFiltered.queues.length,
+      queues: statusFiltered.queues.map(q => q.name),
+      messages: statusFiltered.messages
+    });
+    
+    // Test 8: Analytics with time range
+    console.log('\n🧪 Test 8: Analytics with custom time range (last 30 minutes)');
+    const thirtyMinutesAgo = new Date(Date.now() - 30 * 60 * 1000).toISOString();
+    const now = new Date().toISOString();
+    const analyticsTimeRange = await get(`/api/v1/status/analytics?from=${thirtyMinutesAgo}&to=${now}&interval=minute`);
+    printJson('Analytics (Last 30 min)', {
+      timeRange: analyticsTimeRange.timeRange,
+      throughputSamples: analyticsTimeRange.throughput.timeSeries.length,
+      totals: analyticsTimeRange.throughput.totals
+    });
+    
+    console.log('\n' + '='.repeat(80));
+    console.log('✅ All tests completed successfully!');
+    console.log('='.repeat(80));
+    
+  } catch (error) {
+    console.error('\n❌ Test failed:', error.message);
+    console.error(error.stack);
+    process.exit(1);
+  }
+}
+
+// Run tests
+runTests().then(() => {
+  console.log('\n✨ Dashboard API test complete!');
+  process.exit(0);
+}).catch(error => {
+  console.error('Fatal error:', error);
+  process.exit(1);
+});
+

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "queen-mq",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "type": "module",
   "description": "High-performance message queue backed by PostgreSQL",
   "main": "src/client/index.js",
-
   "scripts": {
     "start": "node src/server.js",
+    "start:cluster": "node src/cluster-server.js",
     "test:produce": "node test.js produce",
     "test:consume": "node test.js consume",
     "test": "node src/test/test.js",
@@ -23,6 +23,8 @@
     "init": "node init-db.js"
   },
   "dependencies": {
+    "axios": "^1.12.2",
+    "chart.js": "^4.5.1",
     "pg": "^8.13.3",
     "uuid": "^13.0.0",
     "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.52.0",

package/server.log

@@ -0,0 +1 @@
+zsh: command not found: node

package/src/benchmark/consumer.js

@@ -0,0 +1,207 @@
+import { Queen } from '../client/client.js';
+import fs from 'fs';
+
+const QUEUE_NAME = 'benchmark-queue-001';
+const NUMBER_OF_CONSUMERS = 10;
+const BATCH_SIZE = 10;
+const CONSUME_MODE = 'partition';
+
+// Global metrics tracking
+const metrics = {
+  startTime: null,
+  endTime: null,
+  totalProcessed: 0,
+  consumers: [],
+  batchTimes: [],
+  ackTimes: [],
+  popTimes: [],
+  firstMessageTime: null,
+  lastMessageTime: null
+};
+
+async function consumer(consumerId, partition) {
+  const q = new Queen({
+    baseUrls: ['http://localhost:6632'],
+    timeout: 30000
+  });
+  
+  const consumerMetrics = {
+    consumerId,
+    partition,
+    messagesProcessed: 0,
+    batchesProcessed: 0,
+    batchTimes: [],
+    ackTimes: [],
+    popTimes: [],
+    startTime: Date.now(),
+    endTime: null
+  };
+
+  const target = CONSUME_MODE === 'partition' ? `${QUEUE_NAME}/${partition}` : `${QUEUE_NAME}`;
+  
+  metrics.consumers.push(consumerMetrics);
+  if (target === `${QUEUE_NAME}`) {
+    console.log(`[Consumer ${consumerId}] Started, consuming all partitions`);
+  } else {
+    console.log(`[Consumer ${consumerId}] Started, consuming partition ${partition}`);
+  }
+  
+  try {
+    for await (const messages of q.takeBatch(target, { //`namespace:benchmark`, 
+      wait: true,
+      timeout: 15000,
+      batch: BATCH_SIZE,
+      idleTimeout: 5000
+    })) {
+      if (!metrics.firstMessageTime) {
+        metrics.firstMessageTime = Date.now();
+      }
+      metrics.lastMessageTime = Date.now();
+      const batchStartTime = Date.now();
+      const popTime = batchStartTime - (consumerMetrics.endTime || consumerMetrics.startTime);
+      consumerMetrics.popTimes.push(popTime);
+      metrics.popTimes.push(popTime);
+      
+      try {
+        // Validate message ordering
+        const partitionLastIds = new Map();
+        
+        for (const message of messages) {
+          if (!message.data || typeof message.data.id !== 'number') {
+            throw new Error(`Message missing or invalid data.id: ${JSON.stringify(message)}`);
+          }
+          
+          const partition = message.partition || 'default';
+          const messageId = message.data.id;
+          
+          if (partitionLastIds.has(partition)) {
+            const lastId = partitionLastIds.get(partition);
+            if (messageId !== lastId + 1) {
+              console.log(message.partition, message);
+              throw new Error(`Message ordering violation in partition '${partition}': expected id ${lastId + 1}, got ${messageId}`);
+            }
+          }
+          
+          partitionLastIds.set(partition, messageId);
+        }
+
+        // Acknowledge all messages
+        const ackStartTime = Date.now();
+        await q.ack(messages);
+        const ackTime = Date.now() - ackStartTime;
+        consumerMetrics.ackTimes.push(ackTime);
+        metrics.ackTimes.push(ackTime);
+        
+        const batchEndTime = Date.now();
+        const batchDuration = batchEndTime - batchStartTime;
+        
+        consumerMetrics.messagesProcessed += messages.length;
+        consumerMetrics.batchesProcessed++;
+        consumerMetrics.batchTimes.push(batchDuration);
+        consumerMetrics.endTime = batchEndTime;
+        
+        metrics.totalProcessed += messages.length;
+        metrics.batchTimes.push(batchDuration);
+        
+        const overallDuration = (batchEndTime - metrics.startTime) / 1000;
+        const currentRate = (metrics.totalProcessed / overallDuration).toFixed(0);
+        
+        console.log(new Date().toISOString(), `[C${consumerId}] Batch: ${messages.length} msgs | Time: ${(batchDuration/1000).toFixed(3)}s (pop:${(popTime/1000).toFixed(3)}s ack:${(ackTime/1000).toFixed(3)}s) | Total: ${metrics.totalProcessed} | Rate: ${currentRate} msg/s`);
+        metrics.lastMessageTime = Date.now();
+      } catch (error) {
+        console.error(new Date().toISOString(), `[Consumer ${consumerId}] Error processing batch:`, error);
+        await q.ack(messages, false, { error: error.message });
+      }
+    }
+  } catch (error) {
+    console.error(`[Consumer ${consumerId}] Fatal error:`, error);
+  }
+  
+  consumerMetrics.endTime = Date.now();
+  console.log(`[Consumer ${consumerId}] Finished. Processed ${consumerMetrics.messagesProcessed} messages in ${consumerMetrics.batchesProcessed} batches`);
+}
+
+// Start metrics
+metrics.startTime = Date.now();
+
+// Launch all consumers
+const consumerPromises = [];
+for (let i = 0; i < NUMBER_OF_CONSUMERS; i++) {
+  consumerPromises.push(consumer(i, i));
+}
+
+// Wait for all consumers to complete
+await Promise.all(consumerPromises);
+metrics.endTime = Date.now();
+
+// Calculate actual processing time (excluding idle timeout)
+// Find the time when the last batch was actually completed
+let lastBatchCompletedTime = metrics.startTime;
+metrics.consumers.forEach(c => {
+  if (c.endTime && c.batchesProcessed > 0) {
+    // c.endTime is when consumer exited (after idle timeout)
+    // Actual last batch time = c.endTime - (idle timeout we didn't track)
+    // Better: use the sum of all processing times
+    const totalBatchTime = c.batchTimes.reduce((a,b)=>a+b,0);
+    const totalPopTime = c.popTimes.reduce((a,b)=>a+b,0);
+    const actualEndTime = c.startTime + totalBatchTime + totalPopTime;
+    if (actualEndTime > lastBatchCompletedTime) {
+      lastBatchCompletedTime = actualEndTime;
+    }
+  }
+});
+
+const totalDuration = (metrics.endTime - metrics.startTime) / 1000;
+const actualProcessingDuration = (lastBatchCompletedTime - metrics.startTime) / 1000;
+const avgThroughput = (metrics.totalProcessed / actualProcessingDuration).toFixed(0);
+
+console.log('\n' + '='.repeat(80));
+console.log('📊 BENCHMARK RESULTS');
+console.log('='.repeat(80));
+console.log(`Total Messages Processed: ${metrics.totalProcessed.toLocaleString()}`);
+console.log(`Total Wall Clock Time: ${totalDuration.toFixed(2)}s (includes idle timeout)`);
+console.log(`Actual Processing Time: ${actualProcessingDuration.toFixed(2)}s`);
+console.log(`Idle/Wait Time: ${(totalDuration - actualProcessingDuration).toFixed(2)}s`);
+console.log(`Average Throughput: ${avgThroughput} msg/s (based on processing time)`);
+console.log(`Number of Consumers: ${NUMBER_OF_CONSUMERS}`);
+console.log(`Total Batches: ${metrics.batchTimes.length}`);
+console.log(`Total Time: ${metrics.lastMessageTime - metrics.firstMessageTime}ms`);
+console.log(`Total msg/s: ${metrics.totalProcessed / (metrics.lastMessageTime - metrics.firstMessageTime) * 1000} msg/s`);
+
+if (metrics.batchTimes.length > 0) {
+  const avgBatchTime = (metrics.batchTimes.reduce((a, b) => a + b, 0) / metrics.batchTimes.length / 1000).toFixed(3);
+  const minBatchTime = (Math.min(...metrics.batchTimes) / 1000).toFixed(3);
+  const maxBatchTime = (Math.max(...metrics.batchTimes) / 1000).toFixed(3);
+  
+  console.log(`\nBatch Times:`);
+  console.log(`  Average: ${avgBatchTime}s`);
+  console.log(`  Min: ${minBatchTime}s`);
+  console.log(`  Max: ${maxBatchTime}s`);
+}
+
+if (metrics.popTimes.length > 0) {
+  const avgPopTime = (metrics.popTimes.reduce((a, b) => a + b, 0) / metrics.popTimes.length / 1000).toFixed(3);
+  console.log(`\nAverage POP Time: ${avgPopTime}s`);
+}
+
+if (metrics.ackTimes.length > 0) {
+  const avgAckTime = (metrics.ackTimes.reduce((a, b) => a + b, 0) / metrics.ackTimes.length / 1000).toFixed(3);
+  console.log(`Average ACK Time: ${avgAckTime}s`);
+}
+
+console.log(`\nPer-Consumer Stats:`);
+metrics.consumers.forEach(c => {
+  // Calculate actual processing time (sum of all batch times + pop times)
+  const totalBatchTime = c.batchTimes.reduce((a,b)=>a+b,0);
+  const totalPopTime = c.popTimes.reduce((a,b)=>a+b,0);
+  const actualProcessingTime = (totalBatchTime + totalPopTime) / 1000;
+  
+  const wallClockTime = ((c.endTime - c.startTime) / 1000).toFixed(2);
+  const rate = (c.messagesProcessed / actualProcessingTime).toFixed(0);
+  const avgBatch = c.batchTimes.length > 0 ? (totalBatchTime / c.batchTimes.length / 1000).toFixed(3) : 'N/A';
+  const idleTime = (wallClockTime - actualProcessingTime).toFixed(2);
+  
+  console.log(`  [C${c.consumerId}] ${c.messagesProcessed.toLocaleString()} msgs | Active: ${actualProcessingTime.toFixed(2)}s (${rate} msg/s) | Idle: ${idleTime}s | ${c.batchesProcessed} batches | avg: ${avgBatch}s`);
+});
+
+console.log('='.repeat(80));