queen-mq 0.2.23 → 0.3.0

package/client-js/test-v2/dlq.js

@@ -0,0 +1,82 @@
+export async function testDLQ(client) { 
+    const queueName = 'test-queue-v2-dlq'
+    
+    // 1. Create queue with low retry limit
+    const queue = await client
+        .queue(queueName)
+        .config({
+            retryLimit: 1,  // Only 1 retry before DLQ
+            dlqAfterMaxRetries: true  // Enable DLQ after max retries
+        })
+        .create()
+    
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // 2. Push a test message
+    const testMessage = { message: 'Test DLQ message', timestamp: Date.now() }
+    await client
+        .queue(queueName)
+        .push([{ data: testMessage }])
+
+    // 3. Consume and fail the message (will trigger retry then DLQ)
+    await client
+        .queue(queueName)
+        .batch(1)
+        .wait(false)
+        .limit(1)  // Process up to 2 messages (original + retry)
+        .each()
+        .consume(async msg => {
+            // Always fail to trigger DLQ
+            throw new Error('Test error - triggering DLQ')
+        })
+        .onError(async (msg, error) => {
+            // Ack as failed to trigger DLQ logic
+            await client.ack(msg, false, { error: error.message })
+        })
+
+    // 4. Wait a moment for DLQ processing
+    await new Promise(resolve => setTimeout(resolve, 500))
+    // 5. Check DLQ for the message
+    const dlqResult = await client
+        .queue(queueName)
+        .dlq()
+        .limit(10)
+        .get()
+
+    console.log('DLQ result: ', JSON.stringify(dlqResult, null, 2))
+
+    if (!dlqResult || !dlqResult.messages) {
+        return { success: false, message: 'Failed to query DLQ' }
+    }
+
+    if (dlqResult.messages.length === 0) {
+        return { 
+            success: false, 
+            message: `No messages in DLQ. Expected at least 1 message. Total: ${dlqResult.total}` 
+        }
+    }
+
+    // 6. Verify the message content
+    const dlqMessage = dlqResult.messages[0]
+    
+    if (!dlqMessage.data || dlqMessage.data.message !== testMessage.message) {
+        return { 
+            success: false, 
+            message: 'DLQ message data does not match expected content' 
+        }
+    }
+
+    if (!dlqMessage.errorMessage || !dlqMessage.errorMessage.includes('Test error')) {
+        return { 
+            success: false, 
+            message: 'DLQ message missing or incorrect error message' 
+        }
+    }
+
+    return { 
+        success: true, 
+        message: `DLQ test completed successfully. Found ${dlqResult.total} message(s) in DLQ` 
+    }
+}

package/client-js/test-v2/load.js

@@ -0,0 +1,177 @@
+export async function testLoad(client) { 
+    const queue = await client
+    .queue('test-queue-v2-load')
+    .create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    const messagesToPush = 100000;
+
+    let messages = [];
+    for (let i = 0; i < messagesToPush; i++) {
+        messages.push({ data: { id: i } })
+        if (messages.length >= 10000) {
+            await client.queue('test-queue-v2-load').push(messages)
+            messages = [];
+        }
+    }
+    await client.queue('test-queue-v2-load').push(messages)    
+
+    let uniqueIds = new Set();
+    let lastId = null;
+
+    await client
+    .queue('test-queue-v2-load')
+    .concurrency(10)
+    .batch(10000)
+    .wait(false)
+    .limit(1)
+    .consume(async msgs => {
+        console.log(`Messages: ${msgs.length}`)
+        for (const msg of msgs) {
+            if (lastId === null) {
+                lastId = msg.data.id
+            } else {
+                if (msg.data.id !== lastId + 1) {
+                    throw new Error('Message ordering violation')
+                }
+                lastId = msg.data.id
+            }
+            uniqueIds.add(msg.data.id)
+        }
+    })
+
+    const uniqueIdsCount = uniqueIds.size;
+    console.log(`Unique IDs count: ${uniqueIdsCount}`)
+
+    return { success: uniqueIdsCount === messagesToPush, message: 'Load test completed successfully' }
+}
+
+export async function testLoadPartition(client) { 
+    const queue = await client
+    .queue('test-queue-v2-load-partition')
+    .create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    const messagesToPush = 100000;
+
+    let messages = [];
+    let k = 0
+    for (let i = 0; i < messagesToPush; i++) {
+        messages.push({ data: { id: i } })
+        if (messages.length >= 10000) {
+            await client
+            .queue('test-queue-v2-load-partition')
+            .partition(k.toString())
+            .push(messages)
+            messages = [];
+            k++;
+        }
+    }
+    await client.queue('test-queue-v2-load-partition').push(messages)    
+
+    let uniqueIds = new Set();
+    let lastId = null;
+    
+    for (let i = 0; i < 10; i++) {
+    await client
+    .queue('test-queue-v2-load-partition')
+    .partition(i.toString())
+    .batch(10000)
+    .wait(false)
+    .limit(1)
+    .consume(async msgs => {
+        for (const msg of msgs) {
+            if (lastId === null) {
+                lastId = msg.data.id
+            } else {
+                if (msg.data.id !== lastId + 1) {
+                    throw new Error('Message ordering violation')
+                }
+                lastId = msg.data.id
+            }
+            uniqueIds.add(msg.data.id)
+        }
+    })
+    }
+
+    const uniqueIdsCount = uniqueIds.size;
+
+    return { success: uniqueIdsCount === messagesToPush, message: 'Load test completed successfully' }
+}
+
+export async function testLoadConsumerGroup(client) { 
+    const queue = await client
+    .queue('test-queue-v2-load-consumer-group')
+    .create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    const messagesToPush = 100000;
+
+    let messages = [];
+    for (let i = 0; i < messagesToPush; i++) {
+        messages.push({ data: { id: i } })
+        if (messages.length >= 10000) {
+            await client.queue('test-queue-v2-load-consumer-group').push(messages)
+            messages = [];
+        }
+    }
+    await client.queue('test-queue-v2-load-consumer-group').push(messages)    
+
+    let uniqueIdsA = new Set();
+    let lastIdA = null;
+
+    await client
+    .queue('test-queue-v2-load-consumer-group')
+    .group('test-consumer-group-a')
+    .concurrency(10)
+    .batch(10000)
+    .wait(false)
+    .limit(1)
+    .consume(async msgs => {
+        for (const msg of msgs) {
+            if (lastIdA === null) {
+                lastIdA = msg.data.id
+            } else {
+                if (msg.data.id !== lastIdA + 1) {
+                    throw new Error('Message ordering violation')
+                }
+                lastIdA = msg.data.id
+            }
+            uniqueIdsA.add(msg.data.id)
+        }
+    })
+    
+    let uniqueIdsB = new Set();
+    let lastIdB = null;
+
+    await client
+    .queue('test-queue-v2-load-consumer-group')
+    .group('test-consumer-group-b')
+    .concurrency(10)
+    .batch(10000)
+    .wait(false)
+    .limit(1)
+    .consume(async msgs => {
+        for (const msg of msgs) {
+            if (lastIdB === null) {
+                lastIdB = msg.data.id
+            } else {
+                if (msg.data.id !== lastIdB + 1) {
+                    throw new Error('Message ordering violation')
+                }
+                lastIdB = msg.data.id
+            }
+            uniqueIdsB.add(msg.data.id)
+        }
+    })    
+
+    const uniqueIdsCountA = uniqueIdsA.size;
+    const uniqueIdsCountB = uniqueIdsB.size;
+    return { success: uniqueIdsCountA === messagesToPush && uniqueIdsCountB === messagesToPush, message: 'Load test completed successfully' }
+}

package/client-js/test-v2/pop.js

@@ -0,0 +1,114 @@
+export async function popEmptyQueue(client) {
+    const queue = await client.queue('test-queue-v2-pop-empty').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client
+    .queue('test-queue-v2-pop-empty')
+    .batch(1)      // Fetch 1 message
+    .wait(false)
+    .pop()
+    return { success: res.length === 0 }
+}
+
+export async function popNonEmptyQueue(client) {
+    const queue = await client.queue('test-queue-v2-pop-non-empty').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    await client
+    .queue('test-queue-v2-pop-non-empty')
+    .push([{ data: { message: 'Hello, world!' } }])
+
+    const res = await client
+    .queue('test-queue-v2-pop-non-empty')
+    .batch(1)
+    .wait(false)
+    .pop()
+    return { success: res.length === 1 }
+}
+
+export async function popWithWait(client) {
+    const queue = await client.queue('test-queue-v2-pop-with-wait').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    setTimeout(async () => {
+        await client
+        .queue('test-queue-v2-pop-with-wait')
+        .push([{ data: { message: 'Hello, world!' } }])
+    }, 2000);
+
+    const res = await client
+    .queue('test-queue-v2-pop-with-wait')
+    .batch(1)
+    .wait(true)
+    .pop()
+
+    return { success: res.length === 1 }
+}
+
+export async function popWithAck(client) {
+    const queue = await client.queue('test-queue-v2-pop-with-ack').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    await client
+    .queue('test-queue-v2-pop-with-ack')
+    .push([{ data: { message: 'Hello, world!' } }]) 
+
+
+    const res = await client
+    .queue('test-queue-v2-pop-with-ack')
+    .batch(1)
+    .wait(false)
+    .pop()
+
+    const resAck = await client.ack(res[0])
+
+    if (!resAck.success) {
+        return { success: false, message: `Ack failed: ${resAck.error}` }
+    }
+
+    return { success: true, message: 'Message popped and acked successfully' }
+}
+
+export async function popWithAckReconsume(client) {
+    const queue = await client
+    .queue('test-queue-v2-pop-with-ack-reconsume')
+    .config({
+        leaseTime: 1,  // 1 second lease
+    })
+    .create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    
+    await client
+    .queue('test-queue-v2-pop-with-ack-reconsume')
+    .push([{ data: { message: 'Hello, world!' } }])
+
+    const res = await client
+    .queue('test-queue-v2-pop-with-ack-reconsume')
+    .batch(1)
+    .wait(false)
+    .pop()
+
+    if (res.length !== 1) {
+        return { success: false, message: `Expected 1 message, got ${res.length}` }
+    }
+
+    await new Promise(r => setTimeout(r, 2000))
+
+    const res2 = await client
+    .queue('test-queue-v2-pop-with-ack-reconsume')
+    .batch(1)
+    .wait(true)
+    .pop()
+
+    if (res2.length === 1) {
+        return { success: true, message: 'Message consumed and reconsumed after lease expiry' }
+    }
+    return { success: false, message: `Expected 1 message, got ${res2.length}` }
+}
+

package/client-js/test-v2/push.js

@@ -0,0 +1,333 @@
+import crypto from 'crypto';
+
+export async function pushMessage(client) {
+    const queue = await client.queue('test-queue-v2').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client
+    .queue('test-queue-v2')
+    .push([{ data: { message: 'Hello, world!' } }])
+
+    return { success: res[0].status === 'queued' }
+}
+
+export async function pushDuplicateMessage(client) {
+    const queue = await client.queue('test-queue-v2').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res1 = await client
+    .queue('test-queue-v2')
+    .push([{ transactionId: 'test-transaction-id', data: { message: 'Hello, world!' } }])
+    
+    const res2 = await client
+    .queue('test-queue-v2')
+    .push([{ transactionId: 'test-transaction-id', data: { message: 'Hello, world!' } }])
+    
+    return { success: res1[0].status === 'queued' && res2[0].status === 'duplicate' }
+}
+
+export async function pushDuplicateMessageOnSpecificPartition(client) {
+    const queue = await client.queue('test-queue-partition-duplicate').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res1 = await client
+    .queue('test-queue-partition-duplicate')
+    .partition('0')
+    .push([{ transactionId: 'test-transaction-id', data: { message: 'Hello, world!' } }])
+    
+    const res2 = await client
+    .queue('test-queue-partition-duplicate')
+    .partition('0')
+    .push([{ transactionId: 'test-transaction-id', data: { message: 'Hello, world!' } }])
+
+    return { success: res1[0].status === 'queued' && res2[0].status === 'duplicate' }
+}
+
+export async function pushDuplicateMessageOnDifferentPartition(client) {
+    const queue = await client.queue('test-queue-partition-duplicate-different').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res1 = await client
+    .queue('test-queue-partition-duplicate-different')
+    .partition('0')
+    .push([{ transactionId: 'test-transaction-id', data: { message: 'Hello, world!' } }])
+    
+    const res2 = await client
+    .queue('test-queue-partition-duplicate-different')
+    .partition('1')
+    .push([{ transactionId: 'test-transaction-id', data: { message: 'Hello, world!' } }])
+    const success = res1[0].status === 'queued' && res2[0].status === 'queued'
+
+    return { success: success }
+}
+
+export async function pushMessageWithTransactionId(client) {
+    const queue = await client.queue('test-queue-transaction-id').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client
+    .queue('test-queue-transaction-id')
+    .push([{ transactionId: 'test-transaction-id', data: { message: 'Hello, world!' } }])
+
+    if (res[0].status !== 'queued' && res[0].transactionId !== 'test-transaction-id') {
+        return { success: false, message: 'Transaction ID mismatch' }
+    }
+    return { success: true }
+}
+
+export async function pushBufferedMessage(client) {
+    const queue = await client.queue('test-queue-buffered').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client
+    .queue('test-queue-buffered')
+    .buffer({ messageCount: 10, timeMillis: 1000 })
+    .push([{ data: { message: 'Hello, world!' } }])
+
+    const pop = await client
+    .queue('test-queue-buffered')
+    .batch(1)
+    .wait(false)
+    .pop() 
+
+    if (pop.length !== 0) {
+        return { success: false, message: 'Message not buffered' }
+    }
+
+    await new Promise(r => setTimeout(r, 2000));
+
+    const pop2 = await client
+    .queue('test-queue-buffered')
+    .batch(1)
+    .wait(false)
+    .pop()     
+
+    if (pop2.length === 0) {
+        return { success: false, message: 'Message not popped' }
+    }
+
+    return { success: true }
+}
+
+export async function pushMaxQueueSize(client) {
+    const queue = await client.queue('test-queue-max-size').config({ maxSize: 10 }).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // Necessary to create a partition consumer to trigger the capacity check
+    const popToCreateParititonConsumer = await client
+    .queue('test-queue-max-size')
+    .batch(1)
+    .wait(false)
+    .pop()    
+
+    try {
+        for (let i = 0; i < 20; i++) {
+        const result = await client
+        .queue('test-queue-max-size')
+        .push([{ data: { message: i } }])
+        }
+    } catch (error) {
+        if (!error.message.includes(' Queue "test-queue-max-size" would exceed max capacity (10).')) {
+            return { success: true, message: 'Correct error message' }
+        } else {
+            return { success: false, message: 'Incorrect error message' }
+        }
+    }
+    return { success: false }
+}
+
+export async function pushDelayedMessage(client) {
+    const queue = await client.queue('test-queue-delayed')
+    .config({ delayedProcessing: 2 })
+    .create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client
+    .queue('test-queue-delayed')
+    .push([{ transactionId: 'test-transaction-dealyed-id', data: { message: 'Hello, world!', aaa: '1' } }])
+
+    const pop = await client
+    .queue('test-queue-delayed')
+    .batch(1)
+    .wait(false)
+    .pop()
+
+    
+
+    if (pop.length !== 0) {
+        return { success: false, message: 'Message not delayed' }
+    }
+
+    await new Promise(r => setTimeout(r, 2500));
+
+    const pop2 = await client
+    .queue('test-queue-delayed')
+    .batch(1)
+    .wait(true)
+    .pop()
+
+    if (pop2.length === 1) {
+        return { success: true, message: 'Message delayed' }
+    }
+
+    return { success: false, message: 'Message not delayed' }
+}
+
+export async function pushWindowBuffer(client) {
+    const queue = await client.queue('test-queue-window-buffer')
+    .config({ windowBuffer: 2 })
+    .create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client
+    .queue('test-queue-window-buffer')
+    .push(
+        [
+            { data: { message: 'Hello, world 1!' } },
+            { data: { message: 'Hello, world 2!' } },
+            { data: { message: 'Hello, world 3!' } }
+        ])
+
+    const pop = await client
+    .queue('test-queue-window-buffer')
+    .batch(1)
+    .wait(false)
+    .pop()
+
+    if (pop.length !== 0) {
+        return { success: false, message: 'Message not delayed' }
+    }
+
+    await new Promise(r => setTimeout(r, 2500));
+
+    const pop2 = await client
+    .queue('test-queue-window-buffer')
+    .batch(4)
+    .wait(false)
+    .pop()
+    if (pop2.length === 3) {
+        return { success: true, message: 'Window buffer works correctly' }
+    }
+
+    return { success: false, message: 'Window buffer does not work correctly' }
+}
+
+export async function pushLargePayload(client) {
+    const queue = await client.queue('test-queue-large-payload').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    // Create a large payload (1MB of data)
+    const largeArray = new Array(10000).fill({
+        id: crypto.randomBytes(16).toString('hex'),
+        data: 'x'.repeat(100),
+        nested: {
+          field1: 'value1',
+          field2: 'value2',
+          field3: 'value3'
+        }
+    });
+      
+    const largePayload = {
+        array: largeArray,
+        metadata: {
+          size: JSON.stringify(largeArray).length,
+          timestamp: Date.now()
+        }
+    };
+
+    const res = await client
+    .queue('test-queue-large-payload')
+    .push([{ data: largePayload }])
+
+    const received = await client
+    .queue('test-queue-large-payload')
+    .batch(1)
+    .wait(false)
+    .pop()
+    
+    if (!received || !received[0].data || !received[0].data.array || received[0].data.array.length !== 10000) {
+        return { success: false, message: 'Large payload corrupted' }
+    }
+
+    return { success: true }
+}
+
+export async function pushNullPayload(client) {
+    const queue = await client.queue('test-queue-null-payload').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client
+    .queue('test-queue-null-payload')
+    .push([{ data: null }])
+
+    const received = await client
+    .queue('test-queue-null-payload')
+    .batch(1)
+    .wait(false)
+    .pop()
+
+    if (!received || received[0].data !== null) {
+        return { success: false, message: 'Null data corrupted' }
+    }
+
+    return { success: true }
+}
+
+export async function pushEmptyPayload(client) {
+    const queue = await client.queue('test-queue-empty-payload').create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client
+    .queue('test-queue-empty-payload')
+    .push([{ data: {} }])
+
+    const received = await client
+    .queue('test-queue-empty-payload')
+    .batch(1)
+    .wait(false)
+    .pop()
+    
+    if (!received || Object.keys(received[0].data).length !== 0) {
+        return { success: false, message: 'Empty object payload corrupted' }
+    }
+
+    return { success: true }
+}
+
+export async function pushEncryptedPayload(client) {
+    const queue = await client.queue('test-queue-encrypted-payload')
+    .config({ encryptionEnabled: true })
+    .create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client
+    .queue('test-queue-encrypted-payload')
+    .push([{ data: { message: 'Hello, world!' } }])
+
+    const received = await client
+    .queue('test-queue-encrypted-payload')
+    .batch(1)
+    .wait(false)
+    .pop()
+
+    
+    if (!received || received[0].data.message !== 'Hello, world!') {
+        return { success: false, message: 'Encrypted payload corrupted' }
+    }
+
+    return { success: true }
+}