queen-mq 0.1.6 → 0.2.11

package/{src → client-js}/test/advanced-pattern-tests.js

@@ -401,8 +401,8 @@ export async function testComplexPriorityScenarios(client) {
     }
     
     // Calculate average priority of first vs last 3 messages
-    const firstThree = received.slice(0, 3).map(m => m.payload.priority);
-    const lastThree = received.slice(-3).map(m => m.payload.priority);
+    const firstThree = received.slice(0, 3).map(m => m.data.priority);
+    const lastThree = received.slice(-3).map(m => m.data.priority);
     
     const avgFirst = firstThree.reduce((a, b) => a + b, 0) / 3;
     const avgLast = lastThree.reduce((a, b) => a + b, 0) / 3;

package/{src → client-js}/test/bus-mode-tests.js

@@ -47,8 +47,8 @@ export async function testBusConsumerGroups(client) {
     }
     
     // Verify both groups got the same message IDs
-    const group1Ids = group1Messages.map(m => m.payload.id).sort();
-    const group2Ids = group2Messages.map(m => m.payload.id).sort();
+    const group1Ids = group1Messages.map(m => m.data.id).sort();
+    const group2Ids = group2Messages.map(m => m.data.id).sort();
     
     if (JSON.stringify(group1Ids) !== JSON.stringify(group2Ids)) {
       throw new Error('Consumer groups did not receive the same messages');
@@ -119,8 +119,8 @@ export async function testMixedMode(client) {
     }
     
     // Workers should get different messages (competing)
-    const worker1Ids = worker1Messages.map(m => m.payload.id);
-    const worker2Ids = worker2Messages.map(m => m.payload.id);
+    const worker1Ids = worker1Messages.map(m => m.data.id);
+    const worker2Ids = worker2Messages.map(m => m.data.id);
     const intersection = worker1Ids.filter(id => worker2Ids.includes(id));
     
     if (intersection.length > 0) {
@@ -150,8 +150,8 @@ export async function testMixedMode(client) {
     }
     
     // Verify groups got all messages
-    const analyticsIds = analyticsMessages.map(m => m.payload.id).sort();
-    const auditIds = auditMessages.map(m => m.payload.id).sort();
+    const analyticsIds = analyticsMessages.map(m => m.data.id).sort();
+    const auditIds = auditMessages.map(m => m.data.id).sort();
     
     if (JSON.stringify(analyticsIds) !== JSON.stringify([1, 2, 3, 4, 5, 6])) {
       throw new Error('Analytics group did not receive all messages');

package/{src → client-js}/test/core-tests.js

@@ -113,19 +113,23 @@ export async function testBatchMessagePush(client) {
     
     // Push batch
     await client.push(`${queue}/batch-partition`, messages);
-    
+
     // Verify messages were stored
     const count = await getMessageCount(queue, 'batch-partition');
+
     if (count !== batchSize) {
       throw new Error(`Expected ${batchSize} messages in database, got ${count}`);
     }
-    
+
     // Take all messages
     const received = [];
     for await (const msg of client.take(`${queue}/batch-partition`, { limit: batchSize })) {
       received.push(msg);
+
       await client.ack(msg);
     }
+
+
     
     if (received.length !== batchSize) {
       throw new Error(`Expected ${batchSize} messages, got ${received.length}`);
@@ -233,6 +237,7 @@ export async function testDelayedProcessing(client) {
     await client.queue(queue, {
       delayedProcessing: 2 // 2 seconds delay
     });
+
     
     const startTime = Date.now();
     
@@ -418,7 +423,7 @@ export async function testWindowBuffer(client) {
     const task = `test-task-${timestamp}`;
     const queue3 = 'test-window-buffer-filtered-' + timestamp;
     
-    await client.queue(queue3, { windowBuffer: 5 }, ns, task);
+    await client.queue(queue3, { windowBuffer: 5 }, { namespace: ns, task: task });
     await client.push(`${queue3}/p1`, { test: 'case3' });
     
     let gotCase3Immediate = false;

package/{src → client-js}/test/edge-case-tests.js

@@ -105,10 +105,11 @@ export async function testVeryLargePayloads(client) {
     
     // Take and verify
     let received = null;
-    for await (const msg of client.take(queue, { limit: 1 })) {
+    for await (const msg of client.take(queue, { limit: 1, timeout: 20000 })) {
       received = msg.data;
       await client.ack(msg);
     }
+
     
     if (!received || !received.array || received.array.length !== 10000) {
       throw new Error('Large payload corrupted');
@@ -487,10 +488,10 @@ export async function testManyConsumerGroups(client) {
     }
     
     // Verify all groups got the same messages
-    const firstGroupIds = groupResults[0].map(m => m.payload.id).sort();
+    const firstGroupIds = groupResults[0].map(m => m.data.id).sort();
     
     for (let g = 1; g < numGroups; g++) {
-      const groupIds = groupResults[g].map(m => m.payload.id).sort();
+      const groupIds = groupResults[g].map(m => m.data.id).sort();
       if (JSON.stringify(groupIds) !== JSON.stringify(firstGroupIds)) {
         throw new Error(`Group ${g} got different messages`);
       }

package/client-js/test/human.js

@@ -0,0 +1,162 @@
+/**
+ * Test written by an human
+ */
+
+import { startTest, passTest, failTest, sleep, log } from './utils.js';
+
+/**
+ * Test: Pipeline API - Basic Processing
+ */
+export async function testCreateQueue(client) {
+    startTest('Create Queue', 'human');
+
+    try {
+        const queue = 'test-create-queue-hm';
+        await client.queue(queue, { leaseTime: 30 });
+        passTest('Queue created successfully');
+    } catch (error) {
+        failTest('Create Queue', error  );
+    }
+}
+
+export async function workerQueueTest(client) {
+    startTest('Worker Queue', 'human');
+
+    try {
+        const queue = 'test-worker-queue-hm';
+        const processedQueue = queue + '-processed';
+        await client.queueDelete(queue);
+        await client.queueDelete(processedQueue); 
+        await client.queue(queue, { leaseTime: 30 });
+        await client.queue(processedQueue, { leaseTime: 30 });
+        const pushMessages = async (count) => {
+            let messages = [];
+            for (let i = 0; i < count; i++) {
+                messages.push({ id: i, data: `Message ${i}` });
+            }
+            await client.push(queue, messages); 
+        }
+
+
+        // Test 1
+        await pushMessages(10);
+
+        let counter1 = 0;
+        await client
+        .pipeline(queue)
+        .take(10)
+        .process(async (message) => {
+            counter1++;
+            return { id: message.data.id, data: message.data.data };
+        })
+        .execute();
+
+        if (counter1 !== 10) {
+            throw new Error('Expected 10 messages to be processed, got ' + counter1);
+        }
+
+        // Test 2
+        await pushMessages(100); 
+
+        let counter2 = 0;
+        await client 
+        .pipeline(queue)
+        .take(10)
+        .processBatch(async (messages) => {
+            counter2 += messages.length;
+            return messages;
+        })
+        .repeat({ maxIterations: 1, continuous: false })
+        .execute();
+
+        if (counter2 !== 10) {
+            throw new Error('Expected 10 messages to be processed, got ' + counter2);
+        }
+
+        // Test 3
+        let counter3 = 0;
+        await client 
+        .pipeline(queue)
+        .take(10)
+        .processBatch(async (messages) => {
+            counter3 += messages.length;
+            return messages;
+        })
+        .repeat({ maxIterations: 9, continuous: false })
+        .execute();        
+
+        if (counter3 !== 90) {
+            throw new Error('Expected 90 messages to be processed, got ' + counter3);
+        }
+
+        await pushMessages(100);
+
+        await client
+        .pipeline(queue)
+        .take(10)
+        .processBatch(async (messages) => {
+            return messages;
+        })
+        .atomically((tx, originalMessages, processedMessages) => {
+            tx.ack(originalMessages);
+        })
+        .repeat({ maxIterations: 10, continuous: true })
+        .execute();
+
+        const messagesPopped = await client.takeSingleBatch(queue);
+        let counter4 = messagesPopped.length;
+        if (counter4 !== 0) {
+            throw new Error('Expected 0 message to be processed, got ' + counter4);
+        }
+
+        // Test transaction that works correctly
+        await pushMessages(100);
+
+        await client
+        .pipeline(queue)
+        .take(10)
+        .processBatch(async (messages) => {
+            return messages;
+        })
+        .atomically((tx, originalMessages, processedMessages) => {
+            tx.ack(originalMessages);
+            tx.push(processedQueue, processedMessages);
+        })
+        .repeat({ maxIterations: 10, continuous: true })
+        .execute();
+
+        const messagesPopped2 = [];
+        for await (const messages of client.takeBatch(processedQueue, { batch: 100 })) {
+            messagesPopped2.push(...messages);
+        }
+        if (messagesPopped2.length !== 100) {
+            throw new Error('Expected 100 message to be processed, got ' + counter5);
+        }
+
+        // Test transaction that fails
+        
+        await pushMessages(100);
+
+        try {
+        await client
+        .pipeline(queue)
+        .take(10)
+        .processBatch(async (messages) => {
+            return messages;
+        })
+        .atomically((tx, originalMessages, processedMessages) => {
+            originalMessages.map(x => x.leaseId = 'invalid-lease-id-12345');
+            tx.ack(originalMessages);
+            tx.push(processedQueue, processedMessages);
+        })
+        .repeat({ maxIterations: 10, continuous: true })
+        .execute();
+        failTest('Expected transaction to fail, got ' + error);
+        } catch (error) {}
+
+        passTest('Worker Queue test passed');
+    } catch (error) {
+        console.error(error);
+        failTest('Worker Queue test failed', error);
+    }
+}

package/client-js/test/qos0-tests.js

@@ -0,0 +1,334 @@
+import { startTest, passTest, failTest } from './utils.js';
+
+/**
+ * Test server-side buffering
+ * Uses file buffer for fast writes, server flushes every 100ms (configurable via FILE_BUFFER_FLUSH_MS)
+ */
+export async function testQoS0Buffering(client) {
+  startTest('Server-side buffering', 'qos0');
+  
+  try {
+    const queue = 'test-qos0-buffer-' + Date.now();
+    await client.queue(queue);
+    
+    // Push 10 events with server-side buffering
+    const start = Date.now();
+    for (let i = 0; i < 10; i++) {
+      await client.push(queue, { index: i }, { buffer: true });
+    }
+    const pushTime = Date.now() - start;
+    
+    // Should return quickly (buffered to file, server-side batching)
+    // With file buffering, push should be fast (< 500ms for 10 messages)
+    if (pushTime >= 500) {
+      throw new Error(`Push should be fast (file buffered), took ${pushTime}ms`);
+    }
+    
+    // Wait for server to flush buffer (FILE_BUFFER_FLUSH_MS default is 100ms)
+    // Add extra margin for processing
+    await new Promise(resolve => setTimeout(resolve, 300));
+    
+    // Consume all events
+    const messages = [];
+    for await (const msg of client.take(queue, { batch: 20 })) {
+      messages.push(msg);
+      await client.ack(msg);
+      if (messages.length >= 10) break;
+    }
+    
+    if (messages.length !== 10) {
+      throw new Error(`Should receive all 10 events, got ${messages.length}`);
+    }
+    
+    // Verify all events received
+    const indices = messages.map(m => m.data.index).sort((a, b) => a - b);
+    for (let i = 0; i < 10; i++) {
+      if (!indices.includes(i)) {
+        throw new Error(`Should have event with index ${i}`);
+      }
+    }
+    
+    passTest('Server-side buffering works correctly');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test auto-acknowledgment
+ */
+export async function testAutoAck(client) {
+  startTest('Auto-acknowledgment', 'qos0');
+  
+  try {
+    const queue = 'test-autoack-' + Date.now();
+    await client.queue(queue);
+    
+    // Push message
+    await client.push(queue, { test: 'autoack-data' });
+    
+    // Take with auto-ack
+    let msg;
+    for await (const m of client.take(queue, { autoAck: true })) {
+      msg = m;
+      break;
+    }
+    
+    if (msg === undefined) {
+      throw new Error('Should receive message');
+    }
+    if (msg.data.test !== 'autoack-data') {
+      throw new Error('Should have correct data');
+    }
+    
+    // Message should NOT be available again (already auto-acked)
+    await new Promise(resolve => setTimeout(resolve, 100));
+    
+    let count = 0;
+    for await (const m of client.take(queue, { wait: false, batch: 10 })) {
+      count++;
+      await client.ack(m);
+      break;
+    }
+    
+    if (count !== 0) {
+      throw new Error('Message should not be available again (auto-acked)');
+    }
+    
+    passTest('Auto-ack works correctly');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test consumer group with auto-ack
+ */
+export async function testConsumerGroupAutoAck(client) {
+  startTest('Consumer group with auto-ack', 'qos0');
+  
+  try {
+    const queue = 'test-cg-autoack-' + Date.now();
+    await client.queue(queue);
+    
+    // Push message
+    await client.push(queue, { value: 42, type: 'test' });
+    
+    // Wait a bit for message to be available
+    await new Promise(resolve => setTimeout(resolve, 100));
+    
+    // Two consumer groups consume same message
+    let msgA, msgB;
+    
+    for await (const msg of client.take(`${queue}@groupA`, { autoAck: true })) {
+      msgA = msg;
+      break;
+    }
+    
+    for await (const msg of client.take(`${queue}@groupB`, { autoAck: true })) {
+      msgB = msg;
+      break;
+    }
+    
+    if (msgA === undefined) {
+      throw new Error('Group A should receive message');
+    }
+    if (msgB === undefined) {
+      throw new Error('Group B should receive message');
+    }
+    if (msgA.data.value !== 42) {
+      throw new Error('Group A should have correct data');
+    }
+    if (msgB.data.value !== 42) {
+      throw new Error('Group B should have correct data');
+    }
+    
+    // Both groups should have auto-acked - message not available again
+    await new Promise(resolve => setTimeout(resolve, 100));
+    
+    let countA = 0, countB = 0;
+    
+    for await (const msg of client.take(`${queue}@groupA`, { wait: false })) {
+      countA++;
+      await client.ack(msg, true, { group: 'groupA' });
+      break;
+    }
+    
+    for await (const msg of client.take(`${queue}@groupB`, { wait: false })) {
+      countB++;
+      await client.ack(msg, true, { group: 'groupB' });
+      break;
+    }
+    
+    if (countA !== 0) {
+      throw new Error('Group A should have auto-acked');
+    }
+    if (countB !== 0) {
+      throw new Error('Group B should have auto-acked');
+    }
+    
+    passTest('Consumer group auto-ack works correctly');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test normal push preserves FIFO ordering
+ */
+export async function testFIFOOrdering(client) {
+  startTest('Normal push preserves FIFO', 'qos0');
+  
+  try {
+    const queue = 'test-fifo-' + Date.now();
+    await client.queue(queue);
+    
+    // Push without buffering (normal mode)
+    for (let i = 0; i < 20; i++) {
+      await client.push(queue, { order: i });
+    }
+    
+    // Consume and verify order
+    const messages = [];
+    for await (const msg of client.take(queue, { batch: 20 })) {
+      messages.push(msg);
+      await client.ack(msg);
+      if (messages.length >= 20) break;
+    }
+    
+    if (messages.length !== 20) {
+      throw new Error(`Should have 20 messages, got ${messages.length}`);
+    }
+    
+    // Verify FIFO ordering
+    for (let i = 0; i < 20; i++) {
+      if (messages[i].data.order !== i) {
+        throw new Error(`Message ${i} out of order: got ${messages[i].data.order}`);
+      }
+    }
+    
+    passTest('FIFO ordering preserved for normal push');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test mixed buffered and non-buffered operations
+ */
+export async function testMixedOperations(client) {
+  startTest('Mixed buffered and normal operations', 'qos0');
+  
+  try {
+    const queue = 'test-mixed-' + Date.now();
+    await client.queue(queue);
+    
+    // Push with buffering (file buffered)
+    await client.push(queue, { type: 'buffered', id: 1 }, { buffer: true });
+    
+    // Push normal (direct DB)
+    await client.push(queue, { type: 'normal', id: 2 });
+    
+    // Wait for buffer flush (100ms + margin)
+    await new Promise(resolve => setTimeout(resolve, 200));
+    
+    // Should have both messages
+    const messages = [];
+    for await (const msg of client.take(queue, { batch: 10 })) {
+      messages.push(msg);
+      await client.ack(msg);
+      if (messages.length >= 2) break;
+    }
+    
+    if (messages.length !== 2) {
+      throw new Error(`Should have 2 messages, got ${messages.length}`);
+    }
+    
+    const types = messages.map(m => m.data.type);
+    if (!types.includes('buffered')) {
+      throw new Error('Should have buffered message');
+    }
+    if (!types.includes('normal')) {
+      throw new Error('Should have normal message');
+    }
+    
+    passTest('Mixed buffered and normal operations work correctly');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Test buffered events are stored individually
+ * Note: Buffering is a server-side optimization for DB writes,
+ * not for combining events into a single message
+ */
+export async function testBatchedPayload(client) {
+  startTest('Buffered events stored individually', 'qos0');
+  
+  try {
+    const queue = 'test-batch-payload-' + Date.now();
+    await client.queue(queue);
+    
+    // Push multiple events with buffering
+    for (let i = 0; i < 5; i++) {
+      await client.push(queue, { index: i, timestamp: Date.now() }, { buffer: true });
+    }
+    
+    // Wait for server flush (100ms + margin)
+    await new Promise(resolve => setTimeout(resolve, 200));
+    
+    // Consume all messages - each event should be a separate message
+    const messages = [];
+    for await (const msg of client.take(queue, { batch: 10 })) {
+      messages.push(msg);
+      await client.ack(msg);
+      if (messages.length >= 5) break;
+    }
+    
+    if (messages.length !== 5) {
+      throw new Error(`Should receive 5 separate messages, got ${messages.length}`);
+    }
+    
+    // Verify each message has individual payload (not batched into array)
+    for (const msg of messages) {
+      if (typeof msg.data !== 'object' || Array.isArray(msg.data)) {
+        throw new Error(`Each message should have object payload, got ${typeof msg.data}`);
+      }
+      if (typeof msg.data.index !== 'number') {
+        throw new Error('Each message should have index property');
+      }
+    }
+    
+    // Verify all indices are present
+    const indices = messages.map(m => m.data.index).sort((a, b) => a - b);
+    for (let i = 0; i < 5; i++) {
+      if (!indices.includes(i)) {
+        throw new Error(`Should have message with index ${i}`);
+      }
+    }
+    
+    passTest('Buffered events stored and retrieved individually');
+  } catch (error) {
+    failTest(error);
+  }
+}
+
+/**
+ * Run all QoS 0 tests
+ */
+export async function runQoS0Tests(client) {
+  console.log('\n╔════════════════════════════════════════╗');
+  console.log('║     Queen QoS 0 Test Suite            ║');
+  console.log('╚════════════════════════════════════════╝\n');
+  
+  await testQoS0Buffering(client);
+  await testAutoAck(client);
+  await testConsumerGroupAutoAck(client);
+  await testFIFOOrdering(client);
+  await testMixedOperations(client);
+  await testBatchedPayload(client);
+  
+  console.log('\n✅ All QoS 0 tests passed!\n');
+}
+