queen-mq 0.1.0 → 0.1.2

package/src/test/bus-mode-tests.js

@@ -0,0 +1,361 @@
+/**
+ * Bus Mode Tests for Queen Message Queue
+ * Tests: Consumer groups, mixed mode, subscription modes, consumer group isolation
+ */
+
+import { startTest, passTest, failTest, sleep, dbPool, log } from './utils.js';
+
+/**
+ * Test: Bus Mode - Consumer Groups
+ */
+export async function testBusConsumerGroups(client) {
+  startTest('Bus Mode - Consumer Groups', 'enterprise');
+  
+  try {
+    const queue = 'test-bus-mode';
+    
+    // Configure queue
+    await client.queue(queue, {});
+    
+    // Push test messages
+    const messages = Array.from({ length: 5 }, (_, i) => ({
+      id: i + 1,
+      data: `Message ${i + 1}`
+    }));
+    
+    await client.push(queue, messages);
+    await sleep(100);
+    
+    // Consumer Group 1: Take messages
+    const group1Messages = [];
+    for await (const msg of client.take(`${queue}@group1`, { limit: 3, batch: 3 })) {
+      group1Messages.push(msg);
+    }
+    
+    if (group1Messages.length !== 3) {
+      throw new Error(`Group1 expected 3 messages, got ${group1Messages.length}`);
+    }
+    
+    // Consumer Group 2: Should get the same messages
+    const group2Messages = [];
+    for await (const msg of client.take(`${queue}@group2`, { limit: 3, batch: 3 })) {
+      group2Messages.push(msg);
+    }
+    
+    if (group2Messages.length !== 3) {
+      throw new Error(`Group2 expected 3 messages, got ${group2Messages.length}`);
+    }
+    
+    // Verify both groups got the same message IDs
+    const group1Ids = group1Messages.map(m => m.payload.id).sort();
+    const group2Ids = group2Messages.map(m => m.payload.id).sort();
+    
+    if (JSON.stringify(group1Ids) !== JSON.stringify(group2Ids)) {
+      throw new Error('Consumer groups did not receive the same messages');
+    }
+    
+    // Acknowledge messages for both groups
+    for (const msg of group1Messages) {
+      await client.ack(msg, true, { group: 'group1' });
+    }
+    
+    for (const msg of group2Messages) {
+      await client.ack(msg, true, { group: 'group2' });
+    }
+    
+    // Take remaining messages
+    const group1Remaining = [];
+    for await (const msg of client.take(`${queue}@group1`, { limit: 5 })) {
+      group1Remaining.push(msg);
+      await client.ack(msg, true, { group: 'group1' });
+    }
+    
+    if (group1Remaining.length !== 2) {
+      throw new Error(`Expected 2 remaining messages for group1, got ${group1Remaining.length}`);
+    }
+    
+    passTest('Consumer groups receive all messages independently');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test: Mixed Mode - Queue and Bus Together
+ */
+export async function testMixedMode(client) {
+  startTest('Mixed Mode - Queue and Bus Together', 'enterprise');
+  
+  try {
+    const queue = 'test-mixed-mode';
+    
+    // Configure queue
+    await client.queue(queue, {});
+    
+    // Push test messages
+    const messages = Array.from({ length: 6 }, (_, i) => ({
+      id: i + 1,
+      data: `Mixed message ${i + 1}`
+    }));
+    
+    await client.push(queue, messages);
+    await sleep(100);
+    
+    // Queue Mode: Workers compete for messages
+    const worker1Messages = [];
+    for await (const msg of client.take(queue, { limit: 2, batch: 2 })) {
+      worker1Messages.push(msg);
+      await client.ack(msg);
+    }
+    
+    const worker2Messages = [];
+    for await (const msg of client.take(queue, { limit: 2, batch: 2 })) {
+      worker2Messages.push(msg);
+      await client.ack(msg);
+    }
+    
+    if (!worker1Messages.length || !worker2Messages.length) {
+      throw new Error('Workers did not receive messages');
+    }
+    
+    // Workers should get different messages (competing)
+    const worker1Ids = worker1Messages.map(m => m.payload.id);
+    const worker2Ids = worker2Messages.map(m => m.payload.id);
+    const intersection = worker1Ids.filter(id => worker2Ids.includes(id));
+    
+    if (intersection.length > 0) {
+      throw new Error('Workers received the same messages - should be competing');
+    }
+    
+    // Bus Mode: Consumer groups get all messages
+    const analyticsMessages = [];
+    for await (const msg of client.take(`${queue}@analytics`, { limit: 10 })) {
+      analyticsMessages.push(msg);
+      await client.ack(msg, true, { group: 'analytics' });
+    }
+    
+    const auditMessages = [];
+    for await (const msg of client.take(`${queue}@audit`, { limit: 10 })) {
+      auditMessages.push(msg);
+      await client.ack(msg, true, { group: 'audit' });
+    }
+    
+    // Both groups should get all 6 messages
+    if (analyticsMessages.length !== 6) {
+      throw new Error(`Analytics group expected 6 messages, got ${analyticsMessages.length}`);
+    }
+    
+    if (auditMessages.length !== 6) {
+      throw new Error(`Audit group expected 6 messages, got ${auditMessages.length}`);
+    }
+    
+    // Verify groups got all messages
+    const analyticsIds = analyticsMessages.map(m => m.payload.id).sort();
+    const auditIds = auditMessages.map(m => m.payload.id).sort();
+    
+    if (JSON.stringify(analyticsIds) !== JSON.stringify([1, 2, 3, 4, 5, 6])) {
+      throw new Error('Analytics group did not receive all messages');
+    }
+    
+    if (JSON.stringify(auditIds) !== JSON.stringify([1, 2, 3, 4, 5, 6])) {
+      throw new Error('Audit group did not receive all messages');
+    }
+    
+    passTest('Mixed mode works: workers compete, groups get all messages');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test: Consumer Group Subscription Modes
+ */
+export async function testConsumerGroupSubscriptionModes(client) {
+  startTest('Consumer Group Subscription Modes', 'enterprise');
+  
+  try {
+    const queue = 'test-subscription-modes';
+    
+    // Configure queue
+    await client.queue(queue, {});
+    
+    // Push historical messages
+    const historicalMessages = Array.from({ length: 3 }, (_, i) => ({
+      id: i + 1,
+      type: 'historical',
+      data: `Historical ${i + 1}`
+    }));
+    
+    await client.push(queue, historicalMessages);
+    await sleep(100);
+    
+    // Group 1: Default mode (consume all)
+    const allMessages = [];
+    for await (const msg of client.take(`${queue}@all-messages`, { limit: 10 })) {
+      allMessages.push(msg);
+      // ACK immediately to release partition lock
+      await client.ack(msg, true, { group: 'all-messages' });
+    }
+    
+    if (allMessages.length !== 3) {
+      throw new Error(`All-messages group expected 3 messages, got ${allMessages.length}`);
+    }
+    
+    // Group 2: New messages only (subscription mode = 'new')
+    const newOnlyMessages = [];
+    for await (const msg of client.take(`${queue}@new-only`, { 
+      limit: 10,
+      subscriptionMode: 'new'
+    })) {
+      newOnlyMessages.push(msg);
+    }
+    
+    if (newOnlyMessages.length !== 0) {
+      throw new Error(`New-only group should get 0 historical messages, got ${newOnlyMessages.length}`);
+    }
+    
+    // Wait to ensure subscription timestamp is in the past
+    await sleep(1000);
+    
+    // Push new messages
+    const newMessages = Array.from({ length: 3 }, (_, i) => ({
+      id: i + 4,
+      type: 'new',
+      data: `New ${i + 4}`
+    }));
+    
+    await client.push(queue, newMessages);
+    await sleep(100);
+    
+    // All-messages group should get the new messages
+    const allMessagesNew = [];
+    for await (const msg of client.take(`${queue}@all-messages`, { limit: 10 })) {
+      allMessagesNew.push(msg);
+      // ACK immediately
+      await client.ack(msg, true, { group: 'all-messages' });
+    }
+    
+    if (allMessagesNew.length !== 3) {
+      throw new Error(`All-messages group expected 3 new messages, got ${allMessagesNew.length}`);
+    }
+    
+    // New-only group should also get the new messages
+    const newOnlyNew = [];
+    for await (const msg of client.take(`${queue}@new-only`, { limit: 10 })) {
+      newOnlyNew.push(msg);
+      await client.ack(msg, true, { group: 'new-only' });
+    }
+    
+    if (newOnlyNew.length !== 3) {
+      throw new Error(`New-only group expected 3 new messages, got ${newOnlyNew.length}`);
+    }
+    
+    passTest('Consumer group subscription modes work correctly');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test: Consumer Group Isolation
+ */
+export async function testConsumerGroupIsolation(client) {
+  startTest('Consumer Group Isolation', 'enterprise');
+  
+  try {
+    const queue = 'test-group-isolation';
+    
+    // Configure queue
+    await client.queue(queue, {});
+    
+    // Push messages
+    const messages = Array.from({ length: 4 }, (_, i) => ({
+      id: i + 1,
+      data: `Isolation test ${i + 1}`
+    }));
+    
+    await client.push(queue, messages);
+    await sleep(100);
+    
+    // Group 1: Take some messages first
+    const group1Messages = [];
+    for await (const msg of client.take(`${queue}@group1`, { limit: 2, batch: 2 })) {
+      group1Messages.push(msg);
+      await client.ack(msg, true, { group: 'group1' });
+    }
+    
+    if (group1Messages.length !== 2) {
+      throw new Error(`Group1 expected 2 messages, got ${group1Messages.length}`);
+    }
+    
+    // Group 2: Should get ALL 4 messages (different consumer group, independent)
+    // Consumer groups don't race - they each get all messages
+    const group2Messages = [];
+    for await (const msg of client.take(`${queue}@group2`, { limit: 10, batch: 10 })) {
+      group2Messages.push(msg);
+    }
+    
+    if (group2Messages.length !== 4) {
+      log(`Group2 got ${group2Messages.length} messages, expected 4 - consumer group isolation may not be working`, 'warning');
+      throw new Error(`Group2 expected all 4 messages, got ${group2Messages.length}`);
+    }
+    
+    // Group 1: Take remaining messages
+    const group1Remaining = [];
+    for await (const msg of client.take(`${queue}@group1`, { limit: 10, batch: 10 })) {
+      group1Remaining.push(msg);
+      await client.ack(msg, true, { group: 'group1' });
+    }
+    
+    if (group1Remaining.length !== 2) {
+      throw new Error(`Group1 expected 2 remaining messages, got ${group1Remaining.length}`);
+    }
+    
+    // Verify isolation: Fail first 2 messages for group2, complete the rest
+    for (let i = 0; i < group2Messages.length; i++) {
+      if (i < 2) {
+        await client.ack(group2Messages[i], false, { error: 'Test failure', group: 'group2' });
+      } else {
+        await client.ack(group2Messages[i], true, { group: 'group2' });
+      }
+    }
+    
+    // Verify isolation: Check that group1's first message is completed (cursor advanced) and group2 failed (in DLQ)
+    const statusCheck = await dbPool.query(`
+      SELECT 
+        m.id as message_id,
+        m.transaction_id,
+        CASE 
+          WHEN dlq1.id IS NOT NULL THEN 'failed'
+          WHEN m.id <= COALESCE(pc1.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid) THEN 'completed'
+          ELSE 'pending'
+        END as group1_status,
+        CASE
+          WHEN dlq2.id IS NOT NULL THEN 'failed'
+          WHEN m.id <= COALESCE(pc2.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid) THEN 'completed'
+          ELSE 'pending'
+        END as group2_status
+      FROM queen.messages m
+      LEFT JOIN queen.partition_consumers pc1 ON pc1.partition_id = m.partition_id AND pc1.consumer_group = 'group1'
+      LEFT JOIN queen.partition_consumers pc2 ON pc2.partition_id = m.partition_id AND pc2.consumer_group = 'group2'
+      LEFT JOIN queen.dead_letter_queue dlq1 ON dlq1.message_id = m.id AND dlq1.consumer_group = 'group1'
+      LEFT JOIN queen.dead_letter_queue dlq2 ON dlq2.message_id = m.id AND dlq2.consumer_group = 'group2'
+      WHERE m.transaction_id = $1
+    `, [group1Messages[0].transactionId]);
+    
+    const result = statusCheck.rows[0];
+    
+    if (result?.group1_status !== 'completed') {
+      throw new Error(`Group1 status should be completed, got ${result?.group1_status}`);
+    }
+    
+    if (result?.group2_status !== 'failed') {
+      throw new Error(`Group2 status should be failed, got ${result?.group2_status}`);
+    }
+    
+    passTest('Consumer groups are properly isolated');
+  } catch (error) {
+    failTest(error);
+  }
+}
+

package/src/test/core-tests.js

@@ -0,0 +1,342 @@
+/**
+ * Core Feature Tests for Queen Message Queue
+ * Tests: Queue configuration, push, take, acknowledgment, delayed processing, FIFO ordering
+ */
+
+import { startTest, passTest, failTest, sleep, dbPool, getMessageCount, log } from './utils.js';
+
+/**
+ * Test 1: Queue Creation Policy
+ * Verifies that queues must be created via queue() method
+ */
+export async function testQueueCreationPolicy(client) {
+  startTest('Queue Creation Policy');
+  
+  try {
+    // Test 1: Push to non-existent queue should fail
+    const nonExistentQueue = 'queue-that-does-not-exist-' + Date.now();
+    try {
+      await client.push(nonExistentQueue, { test: 'should fail' });
+      throw new Error('Push to non-existent queue should have failed');
+    } catch (error) {
+      if (!error.message.includes('does not exist')) {
+        throw error;
+      }
+      log('Push to non-existent queue correctly failed', 'success');
+    }
+    
+    // Test 2: Create queue via queue() method
+    const testQueue = 'test-creation-policy-' + Date.now();
+    await client.queue(testQueue, {
+      retryLimit: 3,
+      priority: 5
+    }, {
+      namespace: 'test',
+      task: 'creation-policy'
+    });
+    log('Queue created via queue() method', 'success');
+    
+    // Test 3: Push to configured queue should succeed
+    await client.push(testQueue, { test: 'message 1' });
+    log('Push to configured queue succeeded', 'success');
+    
+    // Test 4: Push to new partition (should create on-demand)
+    await client.push(`${testQueue}/NewPartition`, { test: 'message 2' });
+    log('Partition created on-demand', 'success');
+    
+    passTest('Queue creation policy working correctly');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test 2: Single Message Push
+ */
+export async function testSingleMessagePush(client) {
+  startTest('Single Message Push');
+  
+  try {
+    const queue = 'test-single-push';
+    
+    // Configure queue first
+    await client.queue(queue, {});
+    
+    // Push single message
+    await client.push(queue, { 
+      message: 'Single test message', 
+      timestamp: Date.now() 
+    });
+    
+    // Verify message was stored
+    const count = await getMessageCount(queue);
+    if (count !== 1) {
+      throw new Error(`Expected 1 message in database, got ${count}`);
+    }
+    
+    // Take and verify the message
+    let messageReceived = false;
+    for await (const msg of client.take(queue, { limit: 1 })) {
+      if (msg.data && msg.data.message === 'Single test message') {
+        messageReceived = true;
+        await client.ack(msg);
+      }
+    }
+    
+    if (!messageReceived) {
+      throw new Error('Failed to receive pushed message');
+    }
+    
+    passTest('Single message pushed successfully');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test 3: Batch Message Push
+ */
+export async function testBatchMessagePush(client) {
+  startTest('Batch Message Push');
+  
+  try {
+    const queue = 'test-batch-push';
+    
+    // Configure queue first
+    await client.queue(queue, {});
+    
+    const batchSize = 5;
+    const messages = Array.from({ length: batchSize }, (_, i) => ({
+      message: `Batch message ${i + 1}`,
+      index: i + 1
+    }));
+    
+    // Push batch
+    await client.push(`${queue}/batch-partition`, messages);
+    
+    // Verify messages were stored
+    const count = await getMessageCount(queue, 'batch-partition');
+    if (count !== batchSize) {
+      throw new Error(`Expected ${batchSize} messages in database, got ${count}`);
+    }
+    
+    // Take all messages
+    const received = [];
+    for await (const msg of client.take(`${queue}/batch-partition`, { limit: batchSize })) {
+      received.push(msg);
+      await client.ack(msg);
+    }
+    
+    if (received.length !== batchSize) {
+      throw new Error(`Expected ${batchSize} messages, got ${received.length}`);
+    }
+    
+    passTest(`Batch of ${batchSize} messages pushed successfully`);
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test 4: Queue Configuration
+ */
+export async function testQueueConfiguration(client) {
+  startTest('Queue Configuration');
+  
+  try {
+    const queue = 'test-config-queue';
+    
+    await client.queue(queue, {
+      leaseTime: 600,
+      retryLimit: 5,
+      priority: 8,
+      delayedProcessing: 2,
+      windowBuffer: 3
+    });
+    
+    // Verify configuration was applied by checking database
+    const result = await dbPool.query(`
+      SELECT lease_time, retry_limit, priority, delayed_processing, window_buffer
+      FROM queen.queues
+      WHERE name = $1
+    `, [queue]);
+    
+    if (result.rows.length === 0) {
+      throw new Error('Configured queue not found in database');
+    }
+    
+    const row = result.rows[0];
+    if (row.lease_time !== 600 || row.retry_limit !== 5) {
+      throw new Error('Configuration options not saved correctly');
+    }
+    
+    passTest('Queue configuration applied successfully');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test 5: Take and Acknowledgment
+ */
+export async function testTakeAndAcknowledgment(client) {
+  startTest('Take and Acknowledgment');
+  
+  try {
+    const queue = 'test-take-ack';
+    
+    // Configure queue first
+    await client.queue(queue, {});
+    
+    // Push test messages
+    await client.push(`${queue}/ack-partition`, [
+      { message: 'Message to complete' },
+      { message: 'Message to fail' }
+    ]);
+    
+    await sleep(100);
+    
+    // Take messages
+    const messages = [];
+    for await (const msg of client.take(`${queue}/ack-partition`, { limit: 2, batch: 2 })) {
+      messages.push(msg);
+    }
+    
+    if (messages.length !== 2) {
+      throw new Error('Expected 2 messages from take');
+    }
+    
+    // Acknowledge first as completed
+    await client.ack(messages[0], true);
+    
+    // Acknowledge second as failed
+    await client.ack(messages[1], false, { 
+      error: 'Test failure' 
+    });
+    
+    passTest('Take and acknowledgment work correctly');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test 6: Delayed Processing
+ */
+export async function testDelayedProcessing(client) {
+  startTest('Delayed Processing');
+  
+  try {
+    const queue = 'test-delayed-queue';
+    
+    // Configure queue with delayed processing
+    await client.queue(queue, {
+      delayedProcessing: 2 // 2 seconds delay
+    });
+    
+    const startTime = Date.now();
+    
+    // Push message
+    await client.push(queue, { 
+      message: 'Delayed message', 
+      sentAt: startTime 
+    });
+    
+    // Try to take immediately (should get nothing)
+    let gotImmediate = false;
+    for await (const msg of client.take(queue, { limit: 1 })) {
+      gotImmediate = true;
+    }
+    
+    if (gotImmediate) {
+      throw new Error('Got message immediately when it should be delayed');
+    }
+    
+    // Wait for delay period plus buffer
+    await sleep(2500);
+    
+    // Try to take again (should get the message now)
+    let gotDelayed = false;
+    let processingDelay = 0;
+    
+    for await (const msg of client.take(queue, { limit: 1 })) {
+      gotDelayed = true;
+      processingDelay = Date.now() - startTime;
+      await client.ack(msg);
+    }
+    
+    if (!gotDelayed) {
+      throw new Error('Did not get delayed message after delay period');
+    }
+    
+    if (processingDelay < 2000) {
+      throw new Error(`Message processed too early: ${processingDelay}ms`);
+    }
+    
+    passTest(`Delayed processing works correctly (${processingDelay}ms delay)`);
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test 7: FIFO Ordering Within Partitions
+ */
+export async function testPartitionFIFOOrdering(client) {
+  startTest('FIFO Ordering Within Partitions');
+  
+  try {
+    const queue = 'test-partition-fifo';
+    
+    // Configure queue
+    await client.queue(queue, {
+      priority: 0
+    });
+    
+    const partitions = [
+      { name: 'ultra-high', priority: 100 },
+      { name: 'high', priority: 10 },
+      { name: 'medium', priority: 5 },
+      { name: 'low', priority: 1 }
+    ];
+    
+    // Create partitions manually in DB
+    for (const partition of partitions) {
+      await dbPool.query(`
+        INSERT INTO queen.partitions (queue_id, name)
+        SELECT q.id, $1
+        FROM queen.queues q
+        WHERE q.name = $2
+        ON CONFLICT (queue_id, name) DO NOTHING
+      `, [partition.name, queue]);
+    }
+    
+    // Push messages to partitions
+    for (const p of partitions) {
+      await client.push(`${queue}/${p.name}`, {
+        message: `${p.name} priority message`,
+        priority: p.priority
+      });
+    }
+    
+    await sleep(100);
+    
+    // Take messages from different partitions
+    const allMessages = [];
+    for (let i = 0; i < partitions.length; i++) {
+      for await (const msg of client.take(queue, { limit: 1, batch: 1 })) {
+        allMessages.push(msg);
+        await client.ack(msg);
+      }
+    }
+    
+    if (allMessages.length !== partitions.length) {
+      throw new Error(`Expected ${partitions.length} messages, got ${allMessages.length}`);
+    }
+    
+    passTest('FIFO ordering within partitions works correctly');
+  } catch (error) {
+    failTest(error);
+  }
+}
+