queen-mq 0.1.0 → 0.1.2

package/src/test/test-new.js

@@ -0,0 +1,278 @@
+#!/usr/bin/env node
+
+/**
+ * Complete Test Suite for Queen Message Queue System
+ * Using the new minimalist Queen client interface
+ * 
+ * Test Categories:
+ * 1. Core Features
+ * 2. Partition Locking
+ * 3. Enterprise Features (coming soon)
+ * 4. Bus Mode Features (coming soon)
+ * 5. Edge Cases (coming soon)
+ * 6. Advanced Scenarios & Patterns (coming soon)
+ */
+
+import { Queen } from '../client/client.js';
+import { 
+  TEST_CONFIG, 
+  initDb, 
+  closeDb, 
+  cleanupTestData, 
+  printResults, 
+  log,
+  resetResults
+} from './utils.js';
+
+// Import test modules
+import {
+  testQueueCreationPolicy,
+  testSingleMessagePush,
+  testBatchMessagePush,
+  testQueueConfiguration,
+  testTakeAndAcknowledgment,
+  testDelayedProcessing,
+  testPartitionFIFOOrdering
+} from './core-tests.js';
+
+import {
+  testPartitionLocking,
+  testBusPartitionLocking,
+  testSpecificPartitionLocking,
+  testNamespaceTaskFiltering,
+  testNamespaceTaskBusMode
+} from './partition-locking-tests.js';
+
+import {
+  testMessageEncryption,
+  testRetentionPendingMessages,
+  testRetentionCompletedMessages,
+  testPartitionRetention,
+  testConsumerEncryption,
+  testMessageEviction,
+  testCombinedEnterpriseFeatures,
+  testEnterpriseErrorHandling
+} from './enterprise-tests.js';
+
+import {
+  testBusConsumerGroups,
+  testMixedMode,
+  testConsumerGroupSubscriptionModes,
+  testConsumerGroupIsolation
+} from './bus-mode-tests.js';
+
+import {
+  testEmptyAndNullPayloads,
+  testVeryLargePayloads,
+  testConcurrentPushOperations,
+  testConcurrentTakeOperations,
+  testManyConsumerGroups,
+  testRetryLimitExhaustion,
+  testLeaseExpiration,
+  testSQLInjectionPrevention,
+  testXSSPrevention
+} from './edge-case-tests.js';
+
+import {
+  testMultiStagePipeline,
+  testFanOutFanIn,
+  testComplexPriorityScenarios,
+  testDynamicPriorityAdjustment,
+  testSagaPattern,
+  testRateLimiting,
+  testTimeBatchProcessing,
+  testEventSourcing,
+  testQueueMetrics,
+  testMultipleConsumersSinglePartitionOrdering,
+  testDeadLetterQueuePattern,
+  testCircuitBreaker,
+  testMessageDeduplication
+} from './advanced-pattern-tests.js';
+
+// Helper to run a test with error handling and delay
+async function runTest(testFn) {
+  try {
+    await testFn();
+  } catch (error) {
+    log(`Unexpected error in test: ${error.message}`, 'error');
+  }
+  await new Promise(resolve => setTimeout(resolve, 100)); // Small delay between tests
+}
+
+// Parse command-line arguments
+const args = process.argv.slice(2);
+const testFilter = args[0]; // e.g., 'core', 'bus', 'edge', 'enterprise', 'partition', 'advanced'
+
+// Show help if requested
+if (testFilter === 'help' || testFilter === '--help' || testFilter === '-h') {
+  console.log('Usage: node src/test/test-new.js [category]');
+  console.log('\nAvailable test categories:');
+  console.log('  core       - Core features (queue creation, push, take, ack, etc.)');
+  console.log('  partition  - Partition locking tests');
+  console.log('  enterprise - Enterprise features (encryption, retention, eviction)');
+  console.log('  bus        - Bus mode features (consumer groups, mixed mode)');
+  console.log('  edge       - Edge cases (null payloads, large payloads, SQL injection, etc.)');
+  console.log('  advanced   - Advanced patterns (pipeline, fan-out, DLQ, circuit breaker)');
+  console.log('\nExamples:');
+  console.log('  node src/test/test-new.js              # Run all tests');
+  console.log('  node src/test/test-new.js core         # Run only core tests');
+  console.log('  node src/test/test-new.js bus          # Run only bus mode tests');
+  console.log('  node src/test/test-new.js edge         # Run only edge case tests');
+  process.exit(0);
+}
+
+async function runTests() {
+  console.log('🚀 Starting Queen Message Queue Test Suite');
+  console.log('   Using the new minimalist Queen client interface\n');
+  if (testFilter) {
+    console.log(`   🎯 Running only: ${testFilter} tests\n`);
+  }
+  console.log('='.repeat(80));
+  
+  let client;
+  
+  try {
+    // Initialize database connection
+    await initDb();
+    log('Database connection established');
+    
+    // Initialize client
+    client = new Queen({ 
+      baseUrls: TEST_CONFIG.baseUrls,
+      timeout: 30000,
+      retryAttempts: 3
+    });
+    log('Queen client initialized');
+    
+    // Check for encryption key
+    if (process.env.QUEEN_ENCRYPTION_KEY) {
+      log('Encryption key detected - encryption tests will run', 'info');
+    } else {
+      log('No encryption key found - encryption tests will be skipped', 'warning');
+    }
+    
+    // Clean up any existing test data
+    await cleanupTestData();
+    
+    // ============================================
+    // CORE FEATURE TESTS
+    // ============================================
+    if (!testFilter || testFilter === 'core') {
+      console.log('\n📦 CORE FEATURES');
+      console.log('-'.repeat(40));
+      
+      await runTest(() => testQueueCreationPolicy(client));
+      await runTest(() => testSingleMessagePush(client));
+      await runTest(() => testBatchMessagePush(client));
+      await runTest(() => testQueueConfiguration(client));
+      await runTest(() => testTakeAndAcknowledgment(client));
+      await runTest(() => testDelayedProcessing(client));
+      await runTest(() => testPartitionFIFOOrdering(client));
+    }
+    
+    // ============================================
+    // PARTITION LOCKING TESTS
+    // ============================================
+    if (!testFilter || testFilter === 'partition' || testFilter === 'locking') {
+      console.log('\n🔒 PARTITION LOCKING');
+      console.log('-'.repeat(40));
+      
+      await runTest(() => testPartitionLocking(client));
+      await runTest(() => testBusPartitionLocking(client));
+      await runTest(() => testSpecificPartitionLocking(client));
+      await runTest(() => testNamespaceTaskFiltering(client));
+      await runTest(() => testNamespaceTaskBusMode(client));
+    }
+    
+    // ============================================
+    // ENTERPRISE FEATURE TESTS
+    // ============================================
+    if (!testFilter || testFilter === 'enterprise') {
+      console.log('\n🏢 ENTERPRISE FEATURES');
+      console.log('-'.repeat(40));
+      
+      await runTest(() => testMessageEncryption(client));
+      await runTest(() => testRetentionPendingMessages(client));
+      await runTest(() => testRetentionCompletedMessages(client));
+      await runTest(() => testPartitionRetention(client));
+      await runTest(() => testConsumerEncryption(client));
+      await runTest(() => testMessageEviction(client));
+      await runTest(() => testCombinedEnterpriseFeatures(client));
+      await runTest(() => testEnterpriseErrorHandling(client));
+    }
+    
+    // ============================================
+    // BUS MODE FEATURE TESTS
+    // ============================================
+    if (!testFilter || testFilter === 'bus') {
+      console.log('\n🚌 BUS MODE FEATURES');
+      console.log('-'.repeat(40));
+      
+      await runTest(() => testBusConsumerGroups(client));
+      await runTest(() => testMixedMode(client));
+      await runTest(() => testConsumerGroupSubscriptionModes(client));
+      await runTest(() => testConsumerGroupIsolation(client));
+    }
+    
+    // ============================================
+    // EDGE CASE TESTS
+    // ============================================
+    if (!testFilter || testFilter === 'edge') {
+      console.log('\n🔍 EDGE CASES');
+      console.log('-'.repeat(40));
+      
+      await runTest(() => testEmptyAndNullPayloads(client));
+      await runTest(() => testVeryLargePayloads(client));
+      await runTest(() => testConcurrentPushOperations(client));
+      await runTest(() => testConcurrentTakeOperations(client));
+      await runTest(() => testManyConsumerGroups(client));
+      await runTest(() => testRetryLimitExhaustion(client));
+      await runTest(() => testLeaseExpiration(client));
+      await runTest(() => testSQLInjectionPrevention(client));
+      await runTest(() => testXSSPrevention(client));
+    }
+    
+    // ============================================
+    // ADVANCED SCENARIO TESTS
+    // ============================================
+    if (!testFilter || testFilter === 'advanced' || testFilter === 'pattern') {
+      console.log('\n🎯 ADVANCED SCENARIOS');
+      console.log('-'.repeat(40));
+      
+      await runTest(() => testMultiStagePipeline(client));
+      await runTest(() => testFanOutFanIn(client));
+      await runTest(() => testComplexPriorityScenarios(client));
+      await runTest(() => testDynamicPriorityAdjustment(client));
+      await runTest(() => testDeadLetterQueuePattern(client));
+      await runTest(() => testSagaPattern(client));
+      await runTest(() => testRateLimiting(client));
+      await runTest(() => testMessageDeduplication(client));
+      await runTest(() => testTimeBatchProcessing(client));
+      await runTest(() => testEventSourcing(client));
+      await runTest(() => testQueueMetrics(client));
+      await runTest(() => testCircuitBreaker(client));
+      await runTest(() => testMultipleConsumersSinglePartitionOrdering(client));
+    }
+    
+  } catch (error) {
+    log(`Test suite setup failed: ${error.message}`, 'error');
+    console.error(error);
+  } finally {
+    // Cleanup
+    if (client) {
+      await client.close();
+    }
+    await cleanupTestData();
+    await closeDb();
+  }
+  
+  // Print results
+  printResults();
+}
+
+// Run the test suite
+runTests().catch(error => {
+  console.error('💥 Test suite crashed:', error);
+  process.exit(1);
+});
+

package/src/test/test.js

@@ -12,7 +12,7 @@
  * 5. Advanced Scenarios & Patterns
  */
 
-import { Queen } from '../client/client.js';
+import { createQueenClient } from '../client/queenClient.js';
 import pg from 'pg';
 import crypto from 'crypto';
 
@@ -124,7 +124,10 @@ const getMessageCount = async (queueName, partitionName = null) => {
 // Helper function to ensure queue exists
 const ensureQueue = async (queueName, options = {}) => {
   try {
-    await client.queue(queueName, options);
+    await client.configure({
+      queue: queueName,
+      options: options
+    });
   } catch (error) {
     // Queue might already exist, that's okay
     if (!error.message.includes('already exists')) {
@@ -4301,7 +4304,7 @@ async function runAllTests() {
   
   try {
     // Initialize client and database connection
-    client = new Queen({ baseUrls: TEST_CONFIG.baseUrls });
+    client = createQueenClient({ baseUrls: TEST_CONFIG.baseUrls });
     dbPool = new pg.Pool(TEST_CONFIG.dbConfig);
     
     // Test database connection

package/src/test/utils.js

@@ -0,0 +1,169 @@
+/**
+ * Shared test utilities for Queen test suite
+ */
+
+import pg from 'pg';
+
+// Test configuration
+export const TEST_CONFIG = {
+  baseUrls: ['http://localhost:6632'],
+  dbConfig: {
+    host: process.env.PG_HOST || 'localhost',
+    port: process.env.PG_PORT || 5432,
+    database: process.env.PG_DB || 'postgres',
+    user: process.env.PG_USER || 'postgres',
+    password: process.env.PG_PASSWORD || 'postgres'
+  }
+};
+
+// Global test state
+export let dbPool;
+export let testResults = [];
+export let currentTest = '';
+
+// Initialize database pool
+export async function initDb() {
+  dbPool = new pg.Pool(TEST_CONFIG.dbConfig);
+  await dbPool.query('SELECT 1');
+  return dbPool;
+}
+
+// Close database pool
+export async function closeDb() {
+  if (dbPool) {
+    await dbPool.end();
+  }
+}
+
+// Test logging utilities
+export const log = (message, type = 'info') => {
+  const timestamp = new Date().toISOString().substring(11, 23);
+  const prefix = {
+    info: '📝',
+    success: '✅',
+    error: '❌',
+    warning: '⚠️',
+    test: '🧪',
+    enterprise: '🏢',
+    edge: '🔍',
+    pattern: '🎯',
+    workflow: '🔄',
+    priority: '⚡'
+  }[type] || '📝';
+  
+  console.log(`[${timestamp}] ${prefix} ${message}`);
+};
+
+export const startTest = (testName, category = 'test') => {
+  currentTest = testName;
+  log(`Starting: ${testName}`, category);
+};
+
+export const passTest = (message = '') => {
+  const result = { test: currentTest, status: 'PASS', message };
+  testResults.push(result);
+  log(`PASS: ${currentTest} ${message}`, 'success');
+};
+
+export const failTest = (error) => {
+  const result = { test: currentTest, status: 'FAIL', error: error.message };
+  testResults.push(result);
+  log(`FAIL: ${currentTest} - ${error.message}`, 'error');
+};
+
+export const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+// Database utilities
+export const cleanupTestData = async () => {
+  try {
+    await dbPool.query(`
+      DELETE FROM queen.messages 
+      WHERE partition_id IN (
+        SELECT p.id FROM queen.partitions p
+        JOIN queen.queues q ON p.queue_id = q.id
+        WHERE q.name LIKE 'test-%' OR q.name LIKE 'edge-%' OR q.name LIKE 'pattern-%' OR q.name LIKE 'workflow-%'
+      )
+    `);
+    
+    await dbPool.query(`
+      DELETE FROM queen.partitions 
+      WHERE queue_id IN (
+        SELECT id FROM queen.queues WHERE name LIKE 'test-%' OR name LIKE 'edge-%' OR name LIKE 'pattern-%' OR name LIKE 'workflow-%'
+      )
+    `);
+    
+    await dbPool.query(`DELETE FROM queen.queues WHERE name LIKE 'test-%' OR name LIKE 'edge-%' OR name LIKE 'pattern-%' OR name LIKE 'workflow-%'`);
+    
+    await dbPool.query(`
+      DELETE FROM queen.retention_history
+      WHERE partition_id NOT IN (SELECT id FROM queen.partitions)
+    `);
+    
+    log('Test data cleaned up');
+  } catch (error) {
+    log(`Cleanup error: ${error.message}`, 'warning');
+  }
+};
+
+export const getMessageCount = async (queueName, partitionName = null) => {
+  const query = partitionName 
+    ? `SELECT COUNT(*) as count FROM queen.messages m
+       JOIN queen.partitions p ON m.partition_id = p.id
+       JOIN queen.queues q ON p.queue_id = q.id
+       WHERE q.name = $1 AND p.name = $2`
+    : `SELECT COUNT(*) as count FROM queen.messages m
+       JOIN queen.partitions p ON m.partition_id = p.id
+       JOIN queen.queues q ON p.queue_id = q.id
+       WHERE q.name = $1`;
+       
+  const params = partitionName ? [queueName, partitionName] : [queueName];
+  const result = await dbPool.query(query, params);
+  return parseInt(result.rows[0].count);
+};
+
+// Print test results summary
+export function printResults() {
+  console.log('\n' + '='.repeat(80));
+  console.log('📊 TEST RESULTS SUMMARY');
+  console.log('='.repeat(80));
+  
+  const passed = testResults.filter(r => r.status === 'PASS').length;
+  const failed = testResults.filter(r => r.status === 'FAIL').length;
+  const total = testResults.length;
+  
+  console.log(`\n📈 Overall Results: ${passed}/${total} tests passed`);
+  
+  if (failed > 0) {
+    console.log('\n❌ Failed Tests:');
+    testResults
+      .filter(r => r.status === 'FAIL')
+      .forEach(r => {
+        console.log(`   • ${r.test}: ${r.error}`);
+      });
+  }
+  
+  if (passed > 0) {
+    console.log('\n✅ Passed Tests:');
+    testResults
+      .filter(r => r.status === 'PASS')
+      .forEach(r => {
+        console.log(`   • ${r.test}${r.message ? ': ' + r.message : ''}`);
+      });
+  }
+  
+  console.log('\n' + '='.repeat(80));
+  
+  if (failed === 0) {
+    console.log('🎉 ALL TESTS PASSED! Queen Message Queue System is working correctly.');
+    console.log('   All features are operational.');
+  } else {
+    console.log(`⚠️  ${failed} test(s) failed. Please review the failures above.`);
+    process.exit(1);
+  }
+}
+
+// Reset test results (useful for running tests in batches)
+export function resetResults() {
+  testResults = [];
+}
+

package/src/utils/streaming.js

@@ -0,0 +1,231 @@
+/**
+ * Streaming utilities for uWebSockets.js with backpressure handling
+ * 
+ * When sending large responses (>1MB), we need to handle backpressure to prevent:
+ * - Event loop blocking
+ * - Memory buildup
+ * - Slow client stalling the server
+ * 
+ * uWebSockets.js provides efficient backpressure handling via:
+ * - res.write() - Returns false if buffer is full
+ * - res.onWritable() - Called when buffer has space again
+ * - res.cork() - Batches syscalls for efficiency
+ */
+
+import { log } from './logger.js';
+
+// Thresholds for streaming decisions
+const SMALL_RESPONSE_THRESHOLD = 128 * 1024;  // 128KB - send directly
+const CHUNK_SIZE = 64 * 1024;                  // 64KB - stream in chunks
+
+/**
+ * Send JSON response with automatic backpressure handling
+ * 
+ * For small responses (<128KB): Sends directly via res.end()
+ * For large responses (>=128KB): Streams with backpressure via res.write() + res.onWritable()
+ * 
+ * @param {Object} res - uWebSockets.js response object
+ * @param {Object} data - Data to serialize and send
+ * @param {Object} abortedRef - Reference object with 'aborted' flag
+ * @param {Object} options - Optional settings
+ * @returns {boolean} - True if sent successfully, false if aborted
+ */
+export const streamJSON = (res, data, abortedRef = null, options = {}) => {
+  // Check if already aborted
+  if (abortedRef && abortedRef.aborted) {
+    return false;
+  }
+  
+  const jsonStr = JSON.stringify(data);
+  const dataSize = jsonStr.length;
+  
+  // Small response - send directly (fast path)
+  if (dataSize < SMALL_RESPONSE_THRESHOLD) {
+    try {
+      res.cork(() => {
+        res.end(jsonStr);
+      });
+      return true;
+    } catch (error) {
+      log(`Error sending small response: ${error.message}`);
+      return false;
+    }
+  }
+  
+  // Large response - stream with backpressure handling
+  // uWebSockets.js requires all writes to be synchronous within cork callbacks
+  log(`Streaming large response: ${(dataSize / 1024 / 1024).toFixed(2)}MB`);
+  
+  let offset = 0;
+  
+  // Start sending - write as much as possible in first cork
+  res.cork(() => {
+    // Write chunks until buffer is full or done
+    while (offset < dataSize) {
+      const chunkEnd = Math.min(offset + CHUNK_SIZE, dataSize);
+      const chunk = jsonStr.slice(offset, chunkEnd);
+      const isLastChunk = chunkEnd >= dataSize;
+      
+      if (isLastChunk) {
+        // Last chunk - end the response
+        res.end(chunk);
+        log(`Streamed response complete: ${(dataSize / 1024).toFixed(2)}KB sent`);
+        return; // All done!
+      }
+      
+      const ok = res.write(chunk);
+      offset = chunkEnd;
+      
+      if (!ok) {
+        // Buffer full - stop and wait for drain
+        log(`Backpressure detected at offset ${offset}/${dataSize} (${((offset/dataSize)*100).toFixed(1)}%)`);
+        break; // Exit while loop, set up onWritable below
+      }
+      
+      // Successfully wrote, continue to next chunk
+    }
+    
+    // If we exited due to backpressure (offset < dataSize), set up handler
+    if (offset < dataSize) {
+      // Set up backpressure handler for remaining chunks
+      // onWritable is called when the socket buffer has space again
+      res.onWritable((writeOffset) => {
+        if (abortedRef && abortedRef.aborted) {
+          return true; // true = done, stop calling
+        }
+        
+        // CRITICAL: All writes inside onWritable must also be corked!
+        let isDone = true; // Will be set to false if we need to continue
+        
+        res.cork(() => {
+          // Write as many chunks as possible while buffer has space
+          while (offset < dataSize) {
+            const chunkEnd = Math.min(offset + CHUNK_SIZE, dataSize);
+            const chunk = jsonStr.slice(offset, chunkEnd);
+            const isLastChunk = chunkEnd >= dataSize;
+            
+            if (isLastChunk) {
+              // Last chunk - end the response
+              res.end(chunk);
+              log(`Streamed response complete: ${(dataSize / 1024).toFixed(2)}KB sent`);
+              isDone = true; // true = done, stop calling onWritable
+              return;
+            }
+            
+            const ok = res.write(chunk);
+            offset = chunkEnd;
+            
+            if (!ok) {
+              // Buffer full again - stop and wait for next onWritable call
+              log(`Backpressure detected at offset ${offset}/${dataSize} (${((offset/dataSize)*100).toFixed(1)}%)`);
+              isDone = false; // false = not done, call me again when writable
+              return;
+            }
+            
+            // Successfully wrote, try next chunk in this iteration
+          }
+          
+          // If we get here, all chunks sent
+          isDone = true; // Done
+        });
+        
+        return isDone;
+      });
+    }
+  });
+  
+  return true;
+};
+
+/**
+ * Send a simple JSON response (for small data)
+ * Always uses res.cork() for efficiency
+ */
+export const sendJSON = (res, data, statusCode = 200) => {
+  res.cork(() => {
+    res.writeStatus(statusCode.toString());
+    res.end(JSON.stringify(data));
+  });
+};
+
+/**
+ * Send error response with proper status code
+ * Note: CORS headers should be set by caller before calling this
+ */
+export const sendError = (res, error, statusCode = 500) => {
+  res.cork(() => {
+    res.writeStatus(statusCode.toString());
+    res.end(JSON.stringify({ 
+      error: error.message || error,
+      code: error.code || 'INTERNAL_ERROR'
+    }));
+  });
+};
+
+/**
+ * Stream large array in chunks (alternative approach for very large arrays)
+ * Serializes chunks separately to avoid large JSON.stringify() blocking
+ */
+export const streamJSONArray = (res, items, abortedRef = null) => {
+  if (abortedRef && abortedRef.aborted) return false;
+  
+  const ITEMS_PER_CHUNK = 1000;
+  let sent = 0;
+  
+  const sendNextChunk = () => {
+    if (abortedRef && abortedRef.aborted) return true;
+    
+    const chunk = items.slice(sent, sent + ITEMS_PER_CHUNK);
+    if (chunk.length === 0) {
+      // All sent
+      res.end(']');
+      return true;
+    }
+    
+    const isFirst = sent === 0;
+    const isLast = sent + chunk.length >= items.length;
+    
+    // Build chunk JSON
+    let chunkStr;
+    if (isFirst) {
+      chunkStr = '[' + chunk.map(item => JSON.stringify(item)).join(',');
+    } else {
+      chunkStr = ',' + chunk.map(item => JSON.stringify(item)).join(',');
+    }
+    
+    const ok = isLast 
+      ? res.end(chunkStr + ']')
+      : res.write(chunkStr);
+    
+    sent += chunk.length;
+    
+    if (!ok) {
+      // Backpressure
+      res.onWritable(() => sendNextChunk());
+      return false;
+    }
+    
+    if (!isLast) {
+      setImmediate(sendNextChunk);
+    }
+    
+    return true;
+  };
+  
+  res.cork(() => {
+    const ok = sendNextChunk();
+    if (!ok) {
+      res.onWritable(sendNextChunk);
+    }
+  });
+  
+  return true;
+};
+
+export default {
+  streamJSON,
+  sendJSON,
+  sendError,
+  streamJSONArray
+};
+