queen-mq 0.2.0 → 0.2.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/README.md +186 -763
  2. package/{src → client-js}/benchmark/consumer.js +6 -4
  3. package/{src → client-js}/benchmark/producer.js +9 -4
  4. package/{src → client-js}/client/client.js +277 -7
  5. package/{src → client-js}/client/index.js +1 -4
  6. package/{src → client-js}/client/utils/http.js +3 -2
  7. package/{src → client-js}/client/utils/retry.js +7 -1
  8. package/{src → client-js}/test/README.md +30 -9
  9. package/{src → client-js}/test/core-tests.js +7 -2
  10. package/{src → client-js}/test/edge-case-tests.js +2 -1
  11. package/client-js/test/human.js +162 -0
  12. package/client-js/test/partition-transaction-tests.js +482 -0
  13. package/client-js/test/qos0-tests.js +334 -0
  14. package/{src → client-js}/test/test-new.js +55 -0
  15. package/package.json +8 -17
  16. package/init-db.js +0 -20
  17. package/src/cluster-server.js +0 -242
  18. package/src/config.js +0 -229
  19. package/src/database/connection.js +0 -129
  20. package/src/database/poolManager.js +0 -199
  21. package/src/database/schema-v2.sql +0 -278
  22. package/src/managers/eventManager.js +0 -59
  23. package/src/managers/queueManagerOptimized.js +0 -1634
  24. package/src/managers/resourceCache.js +0 -96
  25. package/src/managers/systemEventManager.js +0 -132
  26. package/src/routes/ack.js +0 -26
  27. package/src/routes/configure.js +0 -46
  28. package/src/routes/messages.js +0 -368
  29. package/src/routes/pop.js +0 -69
  30. package/src/routes/push.js +0 -28
  31. package/src/routes/resources.js +0 -330
  32. package/src/routes/status.js +0 -1037
  33. package/src/server.js +0 -1633
  34. package/src/test/MIGRATION_ISSUES.md +0 -174
  35. package/src/test/test.js +0 -4524
  36. package/src/utils/streaming.js +0 -231
  37. package/src/webapp-dist/assets/Analytics-DJrE4Rz2.css +0 -1
  38. package/src/webapp-dist/assets/Analytics-HEMXKmfJ.js +0 -1
  39. package/src/webapp-dist/assets/ConfirmDialog-YEoCdE7x.js +0 -1
  40. package/src/webapp-dist/assets/ConsumerGroups-CC5VJywp.css +0 -1
  41. package/src/webapp-dist/assets/ConsumerGroups-D8Z20HhZ.js +0 -1
  42. package/src/webapp-dist/assets/Dashboard-BRFOz6it.css +0 -1
  43. package/src/webapp-dist/assets/Dashboard-Bpx9I32h.js +0 -1
  44. package/src/webapp-dist/assets/LoadingSpinner-BPttXcbs.js +0 -1
  45. package/src/webapp-dist/assets/Messages-BOfVHKma.css +0 -1
  46. package/src/webapp-dist/assets/Messages-EKURCbLl.js +0 -1
  47. package/src/webapp-dist/assets/QueueDetail-BXXA7xoZ.css +0 -1
  48. package/src/webapp-dist/assets/QueueDetail-Bmxg0KiA.js +0 -1
  49. package/src/webapp-dist/assets/Queues-CvrJkrZT.css +0 -1
  50. package/src/webapp-dist/assets/Queues-yUx9_3-9.js +0 -1
  51. package/src/webapp-dist/assets/StatusBadge-C1bBtXHh.css +0 -1
  52. package/src/webapp-dist/assets/StatusBadge-C6N84XmO.js +0 -1
  53. package/src/webapp-dist/assets/analytics-BapTekLM.js +0 -1
  54. package/src/webapp-dist/assets/colors-5z6Q_kUr.js +0 -18
  55. package/src/webapp-dist/assets/index-B7w2bKT6.css +0 -1
  56. package/src/webapp-dist/assets/index-CUSnwYTH.js +0 -31
  57. package/src/webapp-dist/assets/messages-CRV6E-xU.js +0 -1
  58. package/src/webapp-dist/assets/queen-logo-blue.svg +0 -210
  59. package/src/webapp-dist/assets/queen-logo-cyan.svg +0 -210
  60. package/src/webapp-dist/assets/queen-logo-indigo.svg +0 -210
  61. package/src/webapp-dist/assets/queen-logo-orange.svg +0 -210
  62. package/src/webapp-dist/assets/queen-logo-pink.svg +0 -210
  63. package/src/webapp-dist/assets/queen-logo-purple.svg +0 -210
  64. package/src/webapp-dist/assets/queen-logo-rose.svg +0 -239
  65. package/src/webapp-dist/assets/queen-logo.svg +0 -263
  66. package/src/webapp-dist/assets/queues-BdUbxfBC.js +0 -1
  67. package/src/webapp-dist/assets/resources-Bs9U-ceF.js +0 -1
  68. package/src/webapp-dist/index.html +0 -15
  69. package/src/websocket/wsServer.js +0 -228
  70. /package/{src → client-js}/benchmark/consumer_multi.js +0 -0
  71. /package/{src → client-js}/benchmark/producer_multi.js +0 -0
  72. /package/{src → client-js}/client/utils/loadBalancer.js +0 -0
  73. /package/{src → client-js}/services/encryptionService.js +0 -0
  74. /package/{src → client-js}/services/evictionService.js +0 -0
  75. /package/{src → client-js}/services/retentionService.js +0 -0
  76. /package/{src → client-js}/services/startupSync.js +0 -0
  77. /package/{src → client-js}/test/advanced-client-tests.js +0 -0
  78. /package/{src → client-js}/test/advanced-pattern-tests.js +0 -0
  79. /package/{src → client-js}/test/bus-mode-tests.js +0 -0
  80. /package/{src → client-js}/test/enterprise-tests.js +0 -0
  81. /package/{src → client-js}/test/partition-locking-tests.js +0 -0
  82. /package/{src → client-js}/test/utils.js +0 -0
  83. /package/{src → client-js}/test/window-buffer-test.js +0 -0
  84. /package/{src → client-js}/utils/logger.js +0 -0
  85. /package/{src → client-js}/utils/uuid.js +0 -0
package/src/test/test.js DELETED
@@ -1,4524 +0,0 @@
1
- #!/usr/bin/env node
2
-
3
- /**
4
- * Comprehensive Test Suite for Queen Message Queue System
5
- * Consolidated from all test files
6
- *
7
- * Test Categories:
8
- * 1. Core Features
9
- * 2. Enterprise Features
10
- * 3. Bus Mode Features
11
- * 4. Edge Cases
12
- * 5. Advanced Scenarios & Patterns
13
- */
14
-
15
- import { createQueenClient } from '../client/queenClient.js';
16
- import pg from 'pg';
17
- import crypto from 'crypto';
18
-
19
- // Test configuration
20
- const TEST_CONFIG = {
21
- baseUrls: ['http://localhost:6632', 'http://localhost:6633'],
22
- dbConfig: {
23
- host: process.env.PG_HOST || 'localhost',
24
- port: process.env.PG_PORT || 5432,
25
- database: process.env.PG_DB || 'postgres',
26
- user: process.env.PG_USER || 'postgres',
27
- password: process.env.PG_PASSWORD || 'postgres'
28
- }
29
- };
30
-
31
- // Global test state
32
- let client;
33
- let dbPool;
34
- let testResults = [];
35
- let currentTest = '';
36
-
37
- // Test utilities
38
- const log = (message, type = 'info') => {
39
- const timestamp = new Date().toISOString().substring(11, 23);
40
- const prefix = {
41
- info: '📝',
42
- success: '✅',
43
- error: '❌',
44
- warning: '⚠️',
45
- test: '🧪',
46
- enterprise: '🏢',
47
- edge: '🔍',
48
- pattern: '🎯',
49
- workflow: '🔄',
50
- priority: '⚡'
51
- }[type] || '📝';
52
-
53
- console.log(`[${timestamp}] ${prefix} ${message}`);
54
- };
55
-
56
- const startTest = (testName, category = 'test') => {
57
- currentTest = testName;
58
- log(`Starting: ${testName}`, category);
59
- };
60
-
61
- const passTest = (message = '') => {
62
- const result = { test: currentTest, status: 'PASS', message };
63
- testResults.push(result);
64
- log(`PASS: ${currentTest} ${message}`, 'success');
65
- };
66
-
67
- const failTest = (error) => {
68
- const result = { test: currentTest, status: 'FAIL', error: error.message };
69
- testResults.push(result);
70
- log(`FAIL: ${currentTest} - ${error.message}`, 'error');
71
- };
72
-
73
- const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
74
-
75
- // Database utilities
76
- const cleanupTestData = async () => {
77
- try {
78
- await dbPool.query(`
79
- DELETE FROM queen.messages
80
- WHERE partition_id IN (
81
- SELECT p.id FROM queen.partitions p
82
- JOIN queen.queues q ON p.queue_id = q.id
83
- WHERE q.name LIKE 'test-%' OR q.name LIKE 'edge-%' OR q.name LIKE 'pattern-%' OR q.name LIKE 'workflow-%'
84
- )
85
- `);
86
-
87
- await dbPool.query(`
88
- DELETE FROM queen.partitions
89
- WHERE queue_id IN (
90
- SELECT id FROM queen.queues WHERE name LIKE 'test-%' OR name LIKE 'edge-%' OR name LIKE 'pattern-%' OR name LIKE 'workflow-%'
91
- )
92
- `);
93
-
94
- await dbPool.query(`DELETE FROM queen.queues WHERE name LIKE 'test-%' OR name LIKE 'edge-%' OR name LIKE 'pattern-%' OR name LIKE 'workflow-%'`);
95
-
96
- // Also clean up retention history for test partitions
97
- await dbPool.query(`
98
- DELETE FROM queen.retention_history
99
- WHERE partition_id NOT IN (SELECT id FROM queen.partitions)
100
- `);
101
-
102
- log('Test data cleaned up');
103
- } catch (error) {
104
- log(`Cleanup error: ${error.message}`, 'warning');
105
- }
106
- };
107
-
108
- const getMessageCount = async (queueName, partitionName = null) => {
109
- const query = partitionName
110
- ? `SELECT COUNT(*) as count FROM queen.messages m
111
- JOIN queen.partitions p ON m.partition_id = p.id
112
- JOIN queen.queues q ON p.queue_id = q.id
113
- WHERE q.name = $1 AND p.name = $2`
114
- : `SELECT COUNT(*) as count FROM queen.messages m
115
- JOIN queen.partitions p ON m.partition_id = p.id
116
- JOIN queen.queues q ON p.queue_id = q.id
117
- WHERE q.name = $1`;
118
-
119
- const params = partitionName ? [queueName, partitionName] : [queueName];
120
- const result = await dbPool.query(query, params);
121
- return parseInt(result.rows[0].count);
122
- };
123
-
124
- // Helper function to ensure queue exists
125
- const ensureQueue = async (queueName, options = {}) => {
126
- try {
127
- await client.configure({
128
- queue: queueName,
129
- options: options
130
- });
131
- } catch (error) {
132
- // Queue might already exist, that's okay
133
- if (!error.message.includes('already exists')) {
134
- log(`Warning: Failed to configure queue ${queueName}: ${error.message}`, 'warning');
135
- }
136
- }
137
- };
138
-
139
- // ============================================
140
- // CORE FEATURE TESTS
141
- // ============================================
142
-
143
- /**
144
- * Test 1: Queue Creation Policy
145
- * Verifies that queues must be created via configure endpoint
146
- */
147
- async function testQueueCreationPolicy() {
148
- startTest('Queue Creation Policy');
149
-
150
- try {
151
- // Test 1: Push to non-existent queue should fail
152
- const nonExistentQueue = 'queue-that-does-not-exist-' + Date.now();
153
- try {
154
- await client.push({
155
- items: [{
156
- queue: nonExistentQueue,
157
- partition: 'Default',
158
- payload: { test: 'should fail' }
159
- }]
160
- });
161
- throw new Error('Push to non-existent queue should have failed');
162
- } catch (error) {
163
- if (!error.message.includes('does not exist')) {
164
- throw error;
165
- }
166
- log('Push to non-existent queue correctly failed', 'success');
167
- }
168
-
169
- // Test 2: Create queue via configure
170
- const testQueue = 'test-creation-policy-' + Date.now();
171
- await client.configure({
172
- queue: testQueue,
173
- namespace: 'test',
174
- task: 'creation-policy',
175
- options: {
176
- retryLimit: 3,
177
- priority: 5
178
- }
179
- });
180
- log('Queue created via configure endpoint', 'success');
181
-
182
- // Test 3: Push to configured queue should succeed
183
- await client.push({
184
- items: [{
185
- queue: testQueue,
186
- partition: 'Default',
187
- payload: { test: 'message 1' }
188
- }]
189
- });
190
- log('Push to configured queue succeeded', 'success');
191
-
192
- // Test 4: Push to new partition (should create on-demand)
193
- await client.push({
194
- items: [{
195
- queue: testQueue,
196
- partition: 'NewPartition',
197
- payload: { test: 'message 2' }
198
- }]
199
- });
200
- log('Partition created on-demand', 'success');
201
-
202
- // Clean up
203
- await client.queues.clear(testQueue);
204
-
205
- passTest('Queue creation policy working correctly');
206
- } catch (error) {
207
- failTest(error);
208
- }
209
- }
210
-
211
- /**
212
- * Test 2: Single Message Push
213
- */
214
- async function testSingleMessagePush() {
215
- startTest('Single Message Push');
216
-
217
- try {
218
- // Configure queue first
219
- await client.configure({
220
- queue: 'test-single-push',
221
- options: {}
222
- });
223
-
224
- const result = await client.push({
225
- items: [{
226
- queue: 'test-single-push',
227
- partition: 'default',
228
- payload: { message: 'Single test message', timestamp: Date.now() }
229
- }]
230
- });
231
-
232
- if (!result.messages || result.messages.length !== 1) {
233
- throw new Error('Expected 1 message in response');
234
- }
235
-
236
- const message = result.messages[0];
237
- if (!message.id || !message.transactionId || message.status !== 'queued') {
238
- throw new Error('Invalid message response format');
239
- }
240
-
241
- // Verify message was stored
242
- const count = await getMessageCount('test-single-push');
243
- if (count !== 1) {
244
- throw new Error(`Expected 1 message in database, got ${count}`);
245
- }
246
-
247
- passTest('Single message pushed successfully');
248
- } catch (error) {
249
- failTest(error);
250
- }
251
- }
252
-
253
- /**
254
- * Test 3: Batch Message Push
255
- */
256
- async function testBatchMessagePush() {
257
- startTest('Batch Message Push');
258
-
259
- try {
260
- // Configure queue first
261
- await client.configure({
262
- queue: 'test-batch-push',
263
- options: {}
264
- });
265
-
266
- const batchSize = 5;
267
- const items = Array.from({ length: batchSize }, (_, i) => ({
268
- queue: 'test-batch-push',
269
- partition: 'batch-partition',
270
- payload: { message: `Batch message ${i + 1}`, index: i + 1 }
271
- }));
272
-
273
- const result = await client.push({ items });
274
-
275
- if (!result.messages || result.messages.length !== batchSize) {
276
- throw new Error(`Expected ${batchSize} messages in response, got ${result.messages?.length}`);
277
- }
278
-
279
- // Verify all messages have unique transaction IDs
280
- const transactionIds = result.messages.map(m => m.transactionId);
281
- const uniqueIds = new Set(transactionIds);
282
- if (uniqueIds.size !== batchSize) {
283
- throw new Error('Transaction IDs are not unique');
284
- }
285
-
286
- // Verify messages were stored
287
- const count = await getMessageCount('test-batch-push', 'batch-partition');
288
- if (count !== batchSize) {
289
- throw new Error(`Expected ${batchSize} messages in database, got ${count}`);
290
- }
291
-
292
- passTest(`Batch of ${batchSize} messages pushed successfully`);
293
- } catch (error) {
294
- failTest(error);
295
- }
296
- }
297
-
298
- /**
299
- * Test 3: Queue Configuration
300
- */
301
- async function testQueueConfiguration() {
302
- startTest('Queue Configuration');
303
-
304
- try {
305
- const configResult = await client.configure({
306
- queue: 'test-config-queue',
307
- options: {
308
- leaseTime: 600,
309
- retryLimit: 5,
310
- priority: 8,
311
- delayedProcessing: 2,
312
- windowBuffer: 3
313
- }
314
- });
315
-
316
- if (!configResult.configured) {
317
- throw new Error('Configuration failed');
318
- }
319
-
320
- if (configResult.queue !== 'test-config-queue') {
321
- throw new Error('Configuration response has wrong queue name');
322
- }
323
-
324
- // Verify configuration was applied by checking database
325
- const result = await dbPool.query(`
326
- SELECT lease_time, retry_limit, priority, delayed_processing, window_buffer
327
- FROM queen.queues
328
- WHERE name = 'test-config-queue'
329
- `);
330
-
331
- if (result.rows.length === 0) {
332
- throw new Error('Configured queue not found in database');
333
- }
334
-
335
- const row = result.rows[0];
336
- if (row.lease_time !== 600 || row.retry_limit !== 5) {
337
- throw new Error('Configuration options not saved correctly');
338
- }
339
-
340
- passTest('Queue configuration applied successfully');
341
- } catch (error) {
342
- failTest(error);
343
- }
344
- }
345
-
346
- /**
347
- * Test 4: Pop and Acknowledgment
348
- */
349
- async function testPopAndAcknowledgment() {
350
- startTest('Pop and Acknowledgment');
351
-
352
- try {
353
- // Configure queue first
354
- await client.configure({
355
- queue: 'test-pop-ack',
356
- options: {}
357
- });
358
-
359
- // Push test messages
360
- const pushResult = await client.push({
361
- items: [
362
- {
363
- queue: 'test-pop-ack',
364
- partition: 'ack-partition',
365
- payload: { message: 'Message to complete' }
366
- },
367
- {
368
- queue: 'test-pop-ack',
369
- partition: 'ack-partition',
370
- payload: { message: 'Message to fail' }
371
- }
372
- ]
373
- });
374
-
375
- await sleep(100);
376
-
377
- // Pop messages
378
- const popResult = await client.pop({
379
- queue: 'test-pop-ack',
380
- partition: 'ack-partition',
381
- batch: 2
382
- });
383
-
384
- if (!popResult.messages || popResult.messages.length !== 2) {
385
- throw new Error('Expected 2 messages from pop');
386
- }
387
-
388
- // Acknowledge first as completed
389
- const completeResult = await client.ack(
390
- popResult.messages[0].transactionId,
391
- 'completed'
392
- );
393
-
394
- if (!completeResult || !completeResult.acknowledgedAt) {
395
- console.log('Complete ACK result:', completeResult);
396
- throw new Error('Failed to acknowledge message as completed');
397
- }
398
-
399
- // Acknowledge second as failed
400
- const failResult = await client.ack(
401
- popResult.messages[1].transactionId,
402
- 'failed',
403
- 'Test failure'
404
- );
405
-
406
- if (!failResult || !failResult.acknowledgedAt) {
407
- console.log('Fail ACK result:', failResult);
408
- console.log('Transaction ID:', popResult.messages[1].transactionId);
409
- throw new Error('Failed to acknowledge message as failed');
410
- }
411
-
412
- passTest('Pop and acknowledgment work correctly');
413
- } catch (error) {
414
- failTest(error);
415
- }
416
- }
417
-
418
- /**
419
- * Test 5: Delayed Processing
420
- */
421
- async function testDelayedProcessing() {
422
- startTest('Delayed Processing');
423
-
424
- try {
425
- // Configure queue with delayed processing
426
- await client.configure({
427
- queue: 'test-delayed-queue',
428
- options: {
429
- delayedProcessing: 2 // 2 seconds delay
430
- }
431
- });
432
-
433
- const startTime = Date.now();
434
-
435
- // Push message
436
- await client.push({
437
- items: [{
438
- queue: 'test-delayed-queue',
439
- payload: { message: 'Delayed message', sentAt: startTime }
440
- }]
441
- });
442
-
443
- // Try to pop immediately (should get nothing)
444
- const immediateResult = await client.pop({
445
- queue: 'test-delayed-queue',
446
- batch: 1
447
- });
448
-
449
- if (immediateResult.messages && immediateResult.messages.length > 0) {
450
- throw new Error('Got message immediately when it should be delayed');
451
- }
452
-
453
- // Wait for delay period plus buffer
454
- await sleep(2500);
455
-
456
- // Try to pop again (should get the message now)
457
- const delayedResult = await client.pop({
458
- queue: 'test-delayed-queue',
459
- batch: 1
460
- });
461
-
462
- if (!delayedResult.messages || delayedResult.messages.length !== 1) {
463
- throw new Error('Did not get delayed message after delay period');
464
- }
465
-
466
- const processingDelay = Date.now() - startTime;
467
- if (processingDelay < 2000) {
468
- throw new Error(`Message processed too early: ${processingDelay}ms`);
469
- }
470
-
471
- await client.ack(delayedResult.messages[0].transactionId, 'completed');
472
-
473
- passTest(`Delayed processing works correctly (${processingDelay}ms delay)`);
474
- } catch (error) {
475
- failTest(error);
476
- }
477
- }
478
-
479
- /**
480
- * Test 6: FIFO Ordering Within Partitions
481
- */
482
- async function testPartitionPriorityOrdering() {
483
- startTest('FIFO Ordering Within Partitions');
484
-
485
- try {
486
- // Configure queue
487
- await client.configure({
488
- queue: 'test-partition-priority',
489
- options: {
490
- priority: 0
491
- }
492
- });
493
-
494
- const partitions = [
495
- { name: 'ultra-high', priority: 100 },
496
- { name: 'high', priority: 10 },
497
- { name: 'medium', priority: 5 },
498
- { name: 'low', priority: 1 }
499
- ];
500
-
501
- // Create partitions
502
- for (const partition of partitions) {
503
- await dbPool.query(`
504
- INSERT INTO queen.partitions (queue_id, name)
505
- SELECT q.id, $1
506
- FROM queen.queues q
507
- WHERE q.name = 'test-partition-priority'
508
- ON CONFLICT (queue_id, name) DO NOTHING
509
- `, [partition.name]);
510
- }
511
-
512
- // Push messages to partitions
513
- const messages = partitions.map(p => ({
514
- queue: 'test-partition-priority',
515
- partition: p.name,
516
- payload: {
517
- message: `${p.name} priority message`,
518
- priority: p.priority
519
- }
520
- }));
521
-
522
- await client.push({ items: messages });
523
- await sleep(100);
524
-
525
- // With partition locking, we need to pop from each partition separately
526
- // or pop multiple times to get messages from different partitions
527
- const allMessages = [];
528
-
529
- // Pop messages multiple times to get from different partitions
530
- for (let i = 0; i < partitions.length; i++) {
531
- const result = await client.pop({
532
- queue: 'test-partition-priority',
533
- batch: 1
534
- });
535
-
536
- if (result.messages && result.messages.length > 0) {
537
- allMessages.push(...result.messages);
538
- }
539
- }
540
-
541
- if (allMessages.length !== partitions.length) {
542
- throw new Error(`Expected ${partitions.length} messages, got ${allMessages.length}`);
543
- }
544
-
545
- // Verify we got all messages
546
- const receivedMessages = allMessages.map(m => m.payload.message);
547
- const expectedMessages = messages.map(m => m.payload.message).sort();
548
- const actualMessages = receivedMessages.sort();
549
-
550
- if (JSON.stringify(expectedMessages) !== JSON.stringify(actualMessages)) {
551
- throw new Error(`Messages don't match. Expected: ${expectedMessages}, Got: ${actualMessages}`);
552
- }
553
-
554
- // Acknowledge all messages
555
- for (const message of allMessages) {
556
- await client.ack(message.transactionId, 'completed');
557
- }
558
-
559
- passTest('FIFO ordering within partitions works correctly');
560
- } catch (error) {
561
- failTest(error);
562
- }
563
- }
564
-
565
- // ============================================
566
- // PARTITION LOCKING TESTS
567
- // ============================================
568
-
569
- /**
570
- * Test 7: Partition Locking in Queue Mode
571
- * Tests that partitions are properly locked when consumed
572
- */
573
- async function testPartitionLocking() {
574
- startTest('Partition Locking in Queue Mode');
575
-
576
- const QUEUE_NAME = 'test-partition-lock';
577
- const PARTITION1 = 'partition-A';
578
- const PARTITION2 = 'partition-B';
579
-
580
- try {
581
- // Configure queue
582
- await client.configure({
583
- queue: QUEUE_NAME,
584
- options: {}
585
- });
586
-
587
- // Setup: Push 3 messages to each partition
588
- const messages = [];
589
-
590
- for (let i = 1; i <= 3; i++) {
591
- messages.push({
592
- queue: QUEUE_NAME,
593
- partition: PARTITION1,
594
- payload: { data: `Message ${i} for ${PARTITION1}` },
595
- transactionId: `txn-p1-${i}`
596
- });
597
- }
598
-
599
- for (let i = 1; i <= 3; i++) {
600
- messages.push({
601
- queue: QUEUE_NAME,
602
- partition: PARTITION2,
603
- payload: { data: `Message ${i} for ${PARTITION2}` },
604
- transactionId: `txn-p2-${i}`
605
- });
606
- }
607
-
608
- // Push all messages
609
- await client.push({ items: messages });
610
- log(`Pushed ${messages.length} messages (3 to each partition)`);
611
-
612
- // Consumer 1: Pop without specifying partition
613
- log('Consumer 1: Popping messages (no partition specified)...');
614
- const result1 = await client.pop({ queue: QUEUE_NAME, batch: 2 });
615
-
616
- if (result1.messages.length === 0) {
617
- throw new Error('Consumer 1 got no messages');
618
- }
619
-
620
- const partition1 = result1.messages[0].partition;
621
- log(`Consumer 1: Got ${result1.messages.length} messages from partition: ${partition1}`);
622
-
623
- // Consumer 2: Pop without specifying partition (should get different partition)
624
- log('Consumer 2: Popping messages (no partition specified)...');
625
- const result2 = await client.pop({ queue: QUEUE_NAME, batch: 2 });
626
-
627
- if (result2.messages.length === 0) {
628
- throw new Error('Consumer 2 got no messages');
629
- }
630
-
631
- const partition2 = result2.messages[0].partition;
632
- log(`Consumer 2: Got ${result2.messages.length} messages from partition: ${partition2}`);
633
-
634
- // Verify they got different partitions
635
- if (partition1 === partition2) {
636
- throw new Error('Both consumers got the same partition - partition locking not working!');
637
- }
638
-
639
- log(`SUCCESS: Consumers got different partitions (${partition1} vs ${partition2})`);
640
-
641
- // Consumer 3: Try to pop again (should get no messages as both partitions are locked)
642
- log('Consumer 3: Trying to pop (both partitions should be locked)...');
643
- const result3 = await client.pop({ queue: QUEUE_NAME, batch: 2 });
644
-
645
- if (result3.messages.length !== 0) {
646
- throw new Error(`Consumer 3 got ${result3.messages.length} messages (should have been 0)`);
647
- }
648
-
649
- log('Consumer 3: Got no messages (correct - both partitions are locked)');
650
-
651
- // Now ACK messages from Consumer 1 to release its partition
652
- log('Consumer 1: ACKing messages to release partition...');
653
- for (const msg of result1.messages) {
654
- await client.ack(msg.transactionId, 'completed');
655
- }
656
-
657
- // Consumer 4: Should now be able to get remaining message from Consumer 1's partition
658
- log('Consumer 4: Popping after Consumer 1 released...');
659
- const result4 = await client.pop({ queue: QUEUE_NAME, batch: 2 });
660
-
661
- if (result4.messages.length > 0) {
662
- const partition4 = result4.messages[0].partition;
663
- log(`Consumer 4: Got ${result4.messages.length} messages from partition: ${partition4}`);
664
-
665
- // Should be from the same partition as Consumer 1 had
666
- if (partition4 !== partition1) {
667
- throw new Error('Consumer 4 did not get messages from released partition');
668
- }
669
- }
670
-
671
- // Clean up: ACK all remaining messages
672
- log('Cleaning up: ACKing all remaining messages...');
673
- for (const msg of result2.messages) {
674
- await client.ack(msg.transactionId, 'completed');
675
- }
676
- for (const msg of result4.messages) {
677
- await client.ack(msg.transactionId, 'completed');
678
- }
679
-
680
- passTest('Partition locking works correctly');
681
- } catch (error) {
682
- failTest(error);
683
- }
684
- }
685
-
686
- /**
687
- * Test 8: Partition Locking in Bus Mode
688
- * Tests that partitions are properly locked per consumer group
689
- */
690
- async function testBusPartitionLocking() {
691
- startTest('Partition Locking in Bus Mode');
692
-
693
- const QUEUE_NAME = 'test-bus-partition-lock';
694
- const PARTITION1 = 'partition-X';
695
- const PARTITION2 = 'partition-Y';
696
- const CONSUMER_GROUP1 = 'group-alpha';
697
- const CONSUMER_GROUP2 = 'group-beta';
698
-
699
- try {
700
- // Configure queue
701
- await client.configure({
702
- queue: QUEUE_NAME,
703
- options: {}
704
- });
705
-
706
- // Setup: Push 3 messages to each partition
707
- const messages = [];
708
-
709
- for (let i = 1; i <= 3; i++) {
710
- messages.push({
711
- queue: QUEUE_NAME,
712
- partition: PARTITION1,
713
- payload: { data: `Message ${i} for ${PARTITION1}` },
714
- transactionId: `txn-px-${i}`
715
- });
716
- }
717
-
718
- for (let i = 1; i <= 3; i++) {
719
- messages.push({
720
- queue: QUEUE_NAME,
721
- partition: PARTITION2,
722
- payload: { data: `Message ${i} for ${PARTITION2}` },
723
- transactionId: `txn-py-${i}`
724
- });
725
- }
726
-
727
- // Push all messages
728
- await client.push({ items: messages });
729
- log(`Pushed ${messages.length} messages (3 to each partition)`);
730
-
731
- // Group Alpha, Consumer 1: Pop without specifying partition
732
- log('Group Alpha, Consumer 1: Popping messages...');
733
- const result1 = await client.pop({
734
- queue: QUEUE_NAME,
735
- batch: 2,
736
- consumerGroup: CONSUMER_GROUP1
737
- });
738
-
739
- if (result1.messages.length === 0) {
740
- throw new Error('Group Alpha Consumer 1 got no messages');
741
- }
742
-
743
- const partition1 = result1.messages[0].partition;
744
- log(`Group Alpha Consumer 1: Got ${result1.messages.length} messages from partition: ${partition1}`);
745
-
746
- // Group Alpha, Consumer 2: Pop without specifying partition (should get different partition if locking works)
747
- log('Group Alpha, Consumer 2: Popping messages...');
748
- const result2 = await client.pop({
749
- queue: QUEUE_NAME,
750
- batch: 2,
751
- consumerGroup: CONSUMER_GROUP1
752
- });
753
-
754
- if (result2.messages.length === 0) {
755
- log('Group Alpha Consumer 2 got no messages (partition locking might be working)');
756
- } else {
757
- const partition2 = result2.messages[0].partition;
758
- log(`Group Alpha Consumer 2: Got ${result2.messages.length} messages from partition: ${partition2}`);
759
-
760
- if (partition1 === partition2) {
761
- log('WARNING: Both consumers in same group got same partition!');
762
- log('This means partition locking is NOT implemented for bus mode.');
763
- } else {
764
- log('Consumers in same group got different partitions');
765
- }
766
- }
767
-
768
- // Group Beta, Consumer 1: Should be able to get messages from any partition (different consumer group)
769
- log('Group Beta, Consumer 1: Popping messages (different consumer group)...');
770
- const result3 = await client.pop({
771
- queue: QUEUE_NAME,
772
- batch: 2,
773
- consumerGroup: CONSUMER_GROUP2
774
- });
775
-
776
- if (result3.messages.length > 0) {
777
- const partition3 = result3.messages[0].partition;
778
- log(`Group Beta Consumer 1: Got ${result3.messages.length} messages from partition: ${partition3}`);
779
- log('Different consumer groups can access same partitions independently (correct)');
780
- }
781
-
782
- // Clean up: ACK all messages
783
- log('Cleaning up: ACKing all messages...');
784
- for (const msg of result1.messages) {
785
- await client.ack(msg.transactionId, 'completed', null, CONSUMER_GROUP1);
786
- }
787
- for (const msg of result2.messages) {
788
- await client.ack(msg.transactionId, 'completed', null, CONSUMER_GROUP1);
789
- }
790
- for (const msg of result3.messages) {
791
- await client.ack(msg.transactionId, 'completed', null, CONSUMER_GROUP2);
792
- }
793
-
794
- passTest('Bus mode partition locking test completed');
795
- } catch (error) {
796
- failTest(error);
797
- }
798
- }
799
-
800
- /**
801
- * Test 9: Specific Partition Request with Locking
802
- * Tests that requesting a specific partition respects locking
803
- */
804
- async function testSpecificPartitionLocking() {
805
- startTest('Specific Partition Request with Locking');
806
-
807
- const QUEUE_NAME = 'test-specific-partition';
808
- const PARTITION_NAME = 'specific-partition';
809
- const CONSUMER_GROUP = 'test-group';
810
-
811
- try {
812
- // Configure queue
813
- await client.configure({
814
- queue: QUEUE_NAME,
815
- options: {}
816
- });
817
-
818
- // Part 1: Test Queue Mode
819
- log('=== Testing Queue Mode with Specific Partition ===');
820
-
821
- // Push test messages
822
- const messages = [];
823
- for (let i = 1; i <= 5; i++) {
824
- messages.push({
825
- queue: QUEUE_NAME,
826
- partition: PARTITION_NAME,
827
- payload: { data: `Message ${i}` },
828
- transactionId: `txn-${i}`
829
- });
830
- }
831
-
832
- await client.push({ items: messages });
833
- log(`Pushed ${messages.length} messages to ${PARTITION_NAME}`);
834
-
835
- // Consumer 1: Request specific partition
836
- log('Consumer 1: Requesting specific partition...');
837
- const result1 = await client.pop({
838
- queue: QUEUE_NAME,
839
- batch: 2,
840
- partition: PARTITION_NAME
841
- });
842
-
843
- if (result1.messages.length === 0) {
844
- throw new Error('Consumer 1 got no messages');
845
- }
846
-
847
- log(`Consumer 1: Got ${result1.messages.length} messages from partition: ${result1.messages[0].partition}`);
848
-
849
- // Consumer 2: Try to request same specific partition (should get nothing due to lock)
850
- log('Consumer 2: Requesting same specific partition...');
851
- const result2 = await client.pop({
852
- queue: QUEUE_NAME,
853
- batch: 2,
854
- partition: PARTITION_NAME
855
- });
856
-
857
- if (result2.messages.length !== 0) {
858
- throw new Error(`Consumer 2 got ${result2.messages.length} messages (should have been 0 due to lock)`);
859
- }
860
-
861
- log('Consumer 2: Got no messages (correct - partition is locked by Consumer 1)');
862
-
863
- // ACK messages from Consumer 1
864
- log('Consumer 1: ACKing messages...');
865
- for (const msg of result1.messages) {
866
- await client.ack(msg.transactionId, 'completed');
867
- }
868
-
869
- // Consumer 3: Should now get messages from the specific partition
870
- log('Consumer 3: Requesting specific partition after release...');
871
- const result3 = await client.pop({
872
- queue: QUEUE_NAME,
873
- batch: 2,
874
- partition: PARTITION_NAME
875
- });
876
-
877
- if (result3.messages.length === 0) {
878
- throw new Error('Consumer 3 got no messages (should have gotten remaining messages)');
879
- }
880
-
881
- log(`Consumer 3: Got ${result3.messages.length} messages after lease release`);
882
-
883
- // Clean up queue mode messages
884
- for (const msg of result3.messages) {
885
- await client.ack(msg.transactionId, 'completed');
886
- }
887
-
888
- // Part 2: Test Bus Mode
889
- log('=== Testing Bus Mode with Specific Partition ===');
890
-
891
- // Push fresh messages
892
- const busMessages = [];
893
- for (let i = 1; i <= 5; i++) {
894
- busMessages.push({
895
- queue: QUEUE_NAME,
896
- partition: PARTITION_NAME,
897
- payload: { data: `Bus Message ${i}` },
898
- transactionId: `bus-txn-${i}`
899
- });
900
- }
901
-
902
- await client.push({ items: busMessages });
903
-
904
- // Group Alpha, Consumer 1: Request specific partition
905
- log('Group Alpha, Consumer 1: Requesting specific partition...');
906
- const busResult1 = await client.pop({
907
- queue: QUEUE_NAME,
908
- batch: 2,
909
- partition: PARTITION_NAME,
910
- consumerGroup: CONSUMER_GROUP
911
- });
912
-
913
- if (busResult1.messages.length === 0) {
914
- throw new Error('Group Alpha Consumer 1 got no messages');
915
- }
916
-
917
- log(`Group Alpha Consumer 1: Got ${busResult1.messages.length} messages`);
918
-
919
- // Group Alpha, Consumer 2: Try same specific partition (should be locked)
920
- log('Group Alpha, Consumer 2: Requesting same specific partition...');
921
- const busResult2 = await client.pop({
922
- queue: QUEUE_NAME,
923
- batch: 2,
924
- partition: PARTITION_NAME,
925
- consumerGroup: CONSUMER_GROUP
926
- });
927
-
928
- if (busResult2.messages.length !== 0) {
929
- throw new Error(`Group Alpha Consumer 2: Got ${busResult2.messages.length} messages (should be 0)`);
930
- }
931
-
932
- log('Group Alpha Consumer 2: Got no messages (correct - partition locked by Consumer 1)');
933
-
934
- // Different Group, Consumer 1: Should get messages (different consumer group)
935
- log('Group Beta, Consumer 1: Requesting same partition (different group)...');
936
- const busResult3 = await client.pop({
937
- queue: QUEUE_NAME,
938
- batch: 2,
939
- partition: PARTITION_NAME,
940
- consumerGroup: 'group-beta'
941
- });
942
-
943
- if (busResult3.messages.length === 0) {
944
- throw new Error('Group Beta should be able to access partition (different group)');
945
- }
946
-
947
- log(`Group Beta Consumer 1: Got ${busResult3.messages.length} messages (different group works)`);
948
-
949
- // Clean up
950
- log('Cleaning up...');
951
- for (const msg of busResult1.messages) {
952
- await client.ack(msg.transactionId, 'completed', null, CONSUMER_GROUP);
953
- }
954
- for (const msg of busResult3.messages) {
955
- await client.ack(msg.transactionId, 'completed', null, 'group-beta');
956
- }
957
-
958
- passTest('Specific partition locking works correctly');
959
- } catch (error) {
960
- failTest(error);
961
- }
962
- }
963
-
964
- /**
965
- * Test 10: Namespace/Task Filtering with Partition Locking
966
- * Tests that namespace/task filtering properly locks partitions
967
- */
968
- async function testNamespaceTaskFiltering() {
969
- startTest('Namespace/Task Filtering with Partition Locking');
970
-
971
- try {
972
- // Setup: Create multiple queues with same namespace but different tasks
973
- const namespace = 'test-namespace';
974
- const queues = [
975
- { name: 'test-queue-task-1', task: 'process', partition: 'p1' },
976
- { name: 'test-queue-task-2', task: 'process', partition: 'p2' },
977
- { name: 'test-queue-task-3', task: 'analyze', partition: 'p3' }
978
- ];
979
-
980
- // Configure queues with namespace and task
981
- for (const q of queues) {
982
- // Configure queue with namespace and task
983
- await client.configure({
984
- queue: q.name,
985
- namespace: namespace,
986
- task: q.task,
987
- options: { leaseTime: 30 }
988
- });
989
- // Push messages to each queue
990
- await client.push({
991
- items: [
992
- {
993
- queue: q.name,
994
- partition: q.partition,
995
- payload: { data: `Message for ${q.name}` },
996
- transactionId: `txn-${q.name}-1`
997
- },
998
- {
999
- queue: q.name,
1000
- partition: q.partition,
1001
- payload: { data: `Message 2 for ${q.name}` },
1002
- transactionId: `txn-${q.name}-2`
1003
- }
1004
- ]
1005
- });
1006
- }
1007
-
1008
- log('Pushed messages to 3 queues with namespace/task metadata');
1009
-
1010
- // Test 1: Pop by namespace only (should get messages from all queues)
1011
- log('Consumer 1: Popping by namespace only...');
1012
- const result1 = await client.pop({
1013
- namespace: namespace
1014
- }, { batch: 3 });
1015
-
1016
- if (result1.messages.length === 0) {
1017
- throw new Error('Consumer 1 got no messages');
1018
- }
1019
-
1020
- log(`Consumer 1: Got ${result1.messages.length} messages from namespace ${namespace}`);
1021
- const partitions1 = [...new Set(result1.messages.map(m => m.partition))];
1022
- log(`Consumer 1: Locked partitions: ${partitions1.join(', ')}`);
1023
-
1024
- // Test 2: Another consumer tries to pop from same namespace (should get from unlocked partitions only)
1025
- log('Consumer 2: Trying to pop from same namespace...');
1026
- const result2 = await client.pop({
1027
- namespace: namespace
1028
- }, { batch: 3 });
1029
-
1030
- if (result2.messages.length > 0) {
1031
- const partitions2 = [...new Set(result2.messages.map(m => m.partition))];
1032
- log(`Consumer 2: Got ${result2.messages.length} messages from partitions: ${partitions2.join(', ')}`);
1033
-
1034
- // Verify no overlap in partitions
1035
- const overlap = partitions1.filter(p => partitions2.includes(p));
1036
- if (overlap.length > 0) {
1037
- throw new Error(`Partition locking failed! Both consumers got partition(s): ${overlap.join(', ')}`);
1038
- }
1039
- log('SUCCESS: No partition overlap between consumers');
1040
- } else {
1041
- log('Consumer 2: Got no messages (all partitions locked)');
1042
- }
1043
-
1044
- // Test 3: Pop by specific task (should only get from matching queues)
1045
- log('Consumer 3: Popping by specific task "analyze"...');
1046
- const result3 = await client.pop({
1047
- namespace: namespace,
1048
- task: 'analyze'
1049
- }, { batch: 2 });
1050
-
1051
- if (result3.messages.length > 0) {
1052
- log(`Consumer 3: Got ${result3.messages.length} messages for task "analyze"`);
1053
- // Verify all messages are from the analyze task
1054
- const allAnalyze = result3.messages.every(m => m.queue === 'test-queue-task-3');
1055
- if (!allAnalyze) {
1056
- throw new Error('Got messages from wrong task!');
1057
- }
1058
- }
1059
-
1060
- // Clean up: ACK all messages
1061
- for (const msg of [...result1.messages, ...result2.messages, ...result3.messages]) {
1062
- await client.ack(msg.transactionId, 'completed');
1063
- }
1064
-
1065
- passTest('Namespace/Task filtering with partition locking works correctly');
1066
- } catch (error) {
1067
- failTest(error);
1068
- }
1069
- }
1070
-
1071
- /**
1072
- * Test 11: Namespace/Task Filtering in Bus Mode
1073
- * Tests that namespace/task filtering works with consumer groups
1074
- */
1075
- async function testNamespaceTaskBusMode() {
1076
- startTest('Namespace/Task Filtering in Bus Mode');
1077
-
1078
- try {
1079
- const namespace = 'bus-namespace';
1080
- const task = 'bus-task';
1081
- const queues = [
1082
- { name: 'test-bus-queue-1', partition: 'bus-p1' },
1083
- { name: 'test-bus-queue-2', partition: 'bus-p2' }
1084
- ];
1085
-
1086
- // Setup queues and push messages
1087
- for (const q of queues) {
1088
- // Configure queue with namespace and task
1089
- await client.configure({
1090
- queue: q.name,
1091
- namespace: namespace,
1092
- task: task,
1093
- options: { leaseTime: 30 }
1094
- });
1095
- await client.push({
1096
- items: [
1097
- {
1098
- queue: q.name,
1099
- partition: q.partition,
1100
- payload: { data: `Bus message for ${q.name}` },
1101
- transactionId: `bus-txn-${q.name}-1`
1102
- },
1103
- {
1104
- queue: q.name,
1105
- partition: q.partition,
1106
- payload: { data: `Bus message 2 for ${q.name}` },
1107
- transactionId: `bus-txn-${q.name}-2`
1108
- }
1109
- ]
1110
- });
1111
- }
1112
-
1113
- log('Pushed messages to 2 queues for bus mode testing');
1114
-
1115
- // Test 1: Consumer Group A pops messages
1116
- log('Group A, Consumer 1: Popping with namespace/task filter...');
1117
- const resultA1 = await client.pop({
1118
- namespace: namespace,
1119
- task: task,
1120
- consumerGroup: 'group-a'
1121
- }, { batch: 2 });
1122
-
1123
- if (resultA1.messages.length === 0) {
1124
- throw new Error('Group A Consumer 1 got no messages');
1125
- }
1126
-
1127
- const partitionsA1 = [...new Set(resultA1.messages.map(m => m.partition))];
1128
- log(`Group A Consumer 1: Got ${resultA1.messages.length} messages from partitions: ${partitionsA1.join(', ')}`);
1129
-
1130
- // Test 2: Another consumer in Group A (should get from different partitions due to locking)
1131
- log('Group A, Consumer 2: Trying same namespace/task filter...');
1132
- const resultA2 = await client.pop({
1133
- namespace: namespace,
1134
- task: task,
1135
- consumerGroup: 'group-a'
1136
- }, { batch: 2 });
1137
-
1138
- if (resultA2.messages.length > 0) {
1139
- const partitionsA2 = [...new Set(resultA2.messages.map(m => m.partition))];
1140
- log(`Group A Consumer 2: Got ${resultA2.messages.length} messages from partitions: ${partitionsA2.join(', ')}`);
1141
-
1142
- // Check for partition overlap
1143
- const overlapA = partitionsA1.filter(p => partitionsA2.includes(p));
1144
- if (overlapA.length > 0) {
1145
- log(`WARNING: Partition overlap in same group: ${overlapA.join(', ')}`);
1146
- }
1147
- } else {
1148
- log('Group A Consumer 2: No messages (partitions locked by Consumer 1)');
1149
- }
1150
-
1151
- // Test 3: Consumer Group B (should be independent)
1152
- log('Group B, Consumer 1: Popping with same namespace/task filter...');
1153
- const resultB1 = await client.pop({
1154
- namespace: namespace,
1155
- task: task,
1156
- consumerGroup: 'group-b'
1157
- }, { batch: 2 });
1158
-
1159
- if (resultB1.messages.length > 0) {
1160
- const partitionsB1 = [...new Set(resultB1.messages.map(m => m.partition))];
1161
- log(`Group B Consumer 1: Got ${resultB1.messages.length} messages from partitions: ${partitionsB1.join(', ')}`);
1162
- log('SUCCESS: Different consumer groups can access same partitions independently');
1163
- } else {
1164
- throw new Error('Group B should be able to get messages (different consumer group)');
1165
- }
1166
-
1167
- // Clean up: ACK all messages
1168
- for (const msg of resultA1.messages) {
1169
- await client.ack(msg.transactionId, 'completed', null, 'group-a');
1170
- }
1171
- for (const msg of resultA2.messages) {
1172
- await client.ack(msg.transactionId, 'completed', null, 'group-a');
1173
- }
1174
- for (const msg of resultB1.messages) {
1175
- await client.ack(msg.transactionId, 'completed', null, 'group-b');
1176
- }
1177
-
1178
- passTest('Namespace/Task filtering in bus mode works correctly');
1179
- } catch (error) {
1180
- failTest(error);
1181
- }
1182
- }
1183
-
1184
- // ============================================
1185
- // ENTERPRISE FEATURE TESTS
1186
- // ============================================
1187
-
1188
- /**
1189
- * Test 12: Message Encryption
1190
- */
1191
- async function testMessageEncryption() {
1192
- startTest('Message Encryption', 'enterprise');
1193
-
1194
- try {
1195
- // Check if encryption is enabled
1196
- const encryptionKey = process.env.QUEEN_ENCRYPTION_KEY;
1197
- if (!encryptionKey) {
1198
- log('Skipping encryption test - QUEEN_ENCRYPTION_KEY not set', 'warning');
1199
- passTest('Skipped - encryption key not configured');
1200
- return;
1201
- }
1202
-
1203
- // Configure queue with encryption
1204
- await client.configure({
1205
- queue: 'test-encryption-queue',
1206
- options: {
1207
- encryptionEnabled: true
1208
- }
1209
- });
1210
-
1211
- // Verify encryption is enabled in database
1212
- const queueResult = await dbPool.query(`
1213
- SELECT encryption_enabled
1214
- FROM queen.queues
1215
- WHERE name = 'test-encryption-queue'
1216
- `);
1217
-
1218
- if (queueResult.rows.length === 0) {
1219
- throw new Error('Queue not found in database after configuration');
1220
- }
1221
-
1222
- if (!queueResult.rows[0].encryption_enabled) {
1223
- throw new Error(`Encryption not enabled on queue`);
1224
- }
1225
-
1226
- // Push sensitive data
1227
- const sensitiveData = {
1228
- creditCard: '4111111111111111',
1229
- ssn: '123-45-6789',
1230
- password: 'super-secret-password',
1231
- timestamp: Date.now()
1232
- };
1233
-
1234
- const pushResult = await client.push({
1235
- items: [{
1236
- queue: 'test-encryption-queue',
1237
- payload: sensitiveData
1238
- }]
1239
- });
1240
-
1241
- if (!pushResult.messages || pushResult.messages.length !== 1) {
1242
- throw new Error('Failed to push encrypted message');
1243
- }
1244
-
1245
- await sleep(100);
1246
-
1247
- // Check that the message is encrypted in database
1248
- const dbResult = await dbPool.query(`
1249
- SELECT m.payload, m.is_encrypted
1250
- FROM queen.messages m
1251
- JOIN queen.partitions p ON m.partition_id = p.id
1252
- JOIN queen.queues q ON p.queue_id = q.id
1253
- WHERE q.name = 'test-encryption-queue'
1254
- ORDER BY m.created_at DESC
1255
- LIMIT 1
1256
- `);
1257
-
1258
- if (dbResult.rows.length === 0) {
1259
- throw new Error('No message found in database after push');
1260
- }
1261
-
1262
- if (!dbResult.rows[0].is_encrypted) {
1263
- throw new Error('Message not marked as encrypted in database');
1264
- }
1265
-
1266
- const storedData = dbResult.rows[0].payload;
1267
-
1268
- // Verify the stored data is encrypted
1269
- if (!storedData.encrypted || !storedData.iv || !storedData.authTag) {
1270
- throw new Error('Message not properly encrypted in database');
1271
- }
1272
-
1273
- // Verify the stored data doesn't contain plain text sensitive info
1274
- const storedStr = JSON.stringify(storedData);
1275
- if (storedStr.includes('4111111111111111') ||
1276
- storedStr.includes('123-45-6789') ||
1277
- storedStr.includes('super-secret-password')) {
1278
- throw new Error('Sensitive data found in plain text - encryption failed!');
1279
- }
1280
-
1281
- // Pop the message (should be decrypted automatically)
1282
- const popResult = await client.pop({
1283
- queue: 'test-encryption-queue',
1284
- batch: 1
1285
- });
1286
-
1287
- if (!popResult.messages || popResult.messages.length !== 1) {
1288
- throw new Error('Failed to pop encrypted message');
1289
- }
1290
-
1291
- const decryptedData = popResult.messages[0].data;
1292
-
1293
- // Verify decrypted data matches original
1294
- if (decryptedData.creditCard !== sensitiveData.creditCard ||
1295
- decryptedData.ssn !== sensitiveData.ssn ||
1296
- decryptedData.password !== sensitiveData.password ||
1297
- decryptedData.timestamp !== sensitiveData.timestamp) {
1298
- throw new Error('Decrypted data does not match original');
1299
- }
1300
-
1301
- await client.ack(popResult.messages[0].transactionId, 'completed');
1302
-
1303
- passTest('Message encryption/decryption works correctly');
1304
- } catch (error) {
1305
- failTest(error);
1306
- }
1307
- }
1308
-
1309
- /**
1310
- * Test 13: Retention Policy - Pending Messages
1311
- */
1312
- async function testRetentionPendingMessages() {
1313
- startTest('Retention Policy - Pending Messages', 'enterprise');
1314
-
1315
- try {
1316
- // Configure queue with retention for pending messages
1317
- await client.configure({
1318
- queue: 'test-retention-pending',
1319
- options: {
1320
- retentionEnabled: true,
1321
- retentionSeconds: 3 // 3 seconds retention for pending messages
1322
- }
1323
- });
1324
-
1325
- // Push messages
1326
- await client.push({
1327
- items: [
1328
- {
1329
- queue: 'test-retention-pending',
1330
- payload: { message: 'Message to be retained', order: 1 }
1331
- },
1332
- {
1333
- queue: 'test-retention-pending',
1334
- payload: { message: 'Another message to be retained', order: 2 }
1335
- }
1336
- ]
1337
- });
1338
-
1339
- // Verify messages are in database
1340
- const initialCount = await getMessageCount('test-retention-pending', 'Default');
1341
- if (initialCount !== 2) {
1342
- throw new Error(`Expected 2 messages, got ${initialCount}`);
1343
- }
1344
-
1345
- // Wait for retention period
1346
- log('Waiting for retention period (3 seconds)...', 'info');
1347
- await sleep(3500);
1348
-
1349
- // Trigger retention by attempting a pop
1350
- await client.pop({
1351
- queue: 'test-retention-pending',
1352
- batch: 1
1353
- });
1354
-
1355
- // Check if messages were retained
1356
- const finalCount = await getMessageCount('test-retention-pending', 'Default');
1357
-
1358
- log(`Messages after retention period: ${finalCount} (from ${initialCount})`, 'info');
1359
-
1360
- // Check retention history
1361
- const retentionHistory = await dbPool.query(`
1362
- SELECT rh.messages_deleted, rh.retention_type
1363
- FROM queen.retention_history rh
1364
- JOIN queen.partitions p ON rh.partition_id = p.id
1365
- JOIN queen.queues q ON p.queue_id = q.id
1366
- WHERE q.name = 'test-retention-pending'
1367
- ORDER BY rh.executed_at DESC
1368
- LIMIT 1
1369
- `);
1370
-
1371
- if (retentionHistory.rows.length > 0) {
1372
- log(`Retention history: ${retentionHistory.rows[0].messages_deleted} messages deleted (${retentionHistory.rows[0].retention_type})`, 'info');
1373
- }
1374
-
1375
- passTest('Retention policy for pending messages configured successfully');
1376
- } catch (error) {
1377
- failTest(error);
1378
- }
1379
- }
1380
-
1381
- /**
1382
- * Test 14: Retention Policy - Completed Messages
1383
- */
1384
- async function testRetentionCompletedMessages() {
1385
- startTest('Retention Policy - Completed Messages', 'enterprise');
1386
-
1387
- try {
1388
- // Configure queue with retention for completed messages
1389
- await client.configure({
1390
- queue: 'test-retention-completed',
1391
- options: {
1392
- retentionEnabled: true,
1393
- completedRetentionSeconds: 2 // 2 seconds retention for completed messages
1394
- }
1395
- });
1396
-
1397
- // Push messages
1398
- const pushResult = await client.push({
1399
- items: [
1400
- {
1401
- queue: 'test-retention-completed',
1402
- payload: { message: 'Message to complete and retain' }
1403
- }
1404
- ]
1405
- });
1406
-
1407
- await sleep(100);
1408
-
1409
- // Pop and complete the message
1410
- const popResult = await client.pop({
1411
- queue: 'test-retention-completed',
1412
- batch: 1
1413
- });
1414
-
1415
- if (!popResult.messages || popResult.messages.length !== 1) {
1416
- throw new Error('Failed to pop message for completion');
1417
- }
1418
-
1419
- await client.ack(popResult.messages[0].transactionId, 'completed');
1420
-
1421
- // Verify message is marked as completed
1422
- const statusCheck = await dbPool.query(`
1423
- SELECT ms.status
1424
- FROM queen.messages_status ms
1425
- JOIN queen.messages m ON ms.message_id = m.id
1426
- WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__'
1427
- `, [popResult.messages[0].transactionId]);
1428
-
1429
- if (!statusCheck.rows[0] || statusCheck.rows[0].status !== 'completed') {
1430
- throw new Error('Message not marked as completed');
1431
- }
1432
-
1433
- // Wait for retention period
1434
- log('Waiting for completed retention period (2 seconds)...', 'info');
1435
- await sleep(2500);
1436
-
1437
- // Trigger retention check
1438
- await client.pop({
1439
- queue: 'test-retention-completed',
1440
- batch: 1
1441
- });
1442
-
1443
- // Check if completed message still exists
1444
- const retainedCheck = await dbPool.query(`
1445
- SELECT m.id, ms.status, ms.completed_at
1446
- FROM queen.messages m
1447
- LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
1448
- WHERE m.transaction_id = $1
1449
- `, [popResult.messages[0].transactionId]);
1450
-
1451
- const messageExists = retainedCheck.rows.length > 0;
1452
- const isCompleted = retainedCheck.rows[0]?.status === 'completed';
1453
-
1454
- if (!messageExists) {
1455
- throw new Error('Message was deleted, but should be retained with completed status');
1456
- }
1457
-
1458
- if (!isCompleted) {
1459
- throw new Error('Message status is not completed');
1460
- }
1461
-
1462
- log(`Message retained with completed status`, 'info');
1463
- passTest('Retention policy for completed messages works correctly');
1464
- } catch (error) {
1465
- failTest(error);
1466
- }
1467
- }
1468
-
1469
- /**
1470
- * Test 15: Partition Management
1471
- */
1472
- async function testPartitionRetention() {
1473
- startTest('Partition Management', 'enterprise');
1474
-
1475
- try {
1476
- // Configure queue
1477
- await client.configure({
1478
- queue: 'test-partition-management',
1479
- options: {
1480
- retentionEnabled: false
1481
- }
1482
- });
1483
-
1484
- // Push a message to create Default partition
1485
- await client.push({
1486
- items: [{
1487
- queue: 'test-partition-management',
1488
- payload: { message: 'Test message' }
1489
- }]
1490
- });
1491
-
1492
- // Verify partition exists
1493
- const partitionCheck = await dbPool.query(`
1494
- SELECT p.id, p.last_activity
1495
- FROM queen.partitions p
1496
- JOIN queen.queues q ON p.queue_id = q.id
1497
- WHERE q.name = 'test-partition-management' AND p.name = 'Default'
1498
- `);
1499
-
1500
- if (partitionCheck.rows.length === 0) {
1501
- throw new Error('Default partition not created');
1502
- }
1503
-
1504
- const partitionId = partitionCheck.rows[0].id;
1505
- log(`Partition created with ID: ${partitionId}`, 'info');
1506
-
1507
- // Push another message to update last_activity
1508
- await client.push({
1509
- items: [{
1510
- queue: 'test-partition-management',
1511
- payload: { message: 'Another message' }
1512
- }]
1513
- });
1514
-
1515
- // Pop and complete the message
1516
- const popResult = await client.pop({
1517
- queue: 'test-partition-management',
1518
- batch: 2
1519
- });
1520
-
1521
- if (popResult.messages && popResult.messages.length > 0) {
1522
- await client.ack(popResult.messages[0].transactionId, 'completed');
1523
- }
1524
-
1525
- // Check partition still exists
1526
- const stillExists = await dbPool.query(`
1527
- SELECT COUNT(*) as count FROM queen.partitions WHERE id = $1
1528
- `, [partitionId]);
1529
-
1530
- if (stillExists.rows[0].count === '0') {
1531
- throw new Error('Partition deleted too early');
1532
- }
1533
-
1534
- passTest('Partition management works correctly');
1535
- } catch (error) {
1536
- failTest(error);
1537
- }
1538
- }
1539
-
1540
- /**
1541
- * Test 16: Message Eviction
1542
- */
1543
- async function testMessageEviction() {
1544
- startTest('Message Eviction', 'enterprise');
1545
-
1546
- try {
1547
- // Configure queue with max wait time
1548
- await client.configure({
1549
- queue: 'test-eviction-queue',
1550
- options: {
1551
- maxWaitTimeSeconds: 3
1552
- }
1553
- });
1554
-
1555
- // Push messages
1556
- await client.push({
1557
- items: [
1558
- {
1559
- queue: 'test-eviction-queue',
1560
- partition: 'eviction-partition',
1561
- payload: { message: 'Message to be evicted', timestamp: Date.now() }
1562
- }
1563
- ]
1564
- });
1565
-
1566
- // Verify message exists
1567
- const pendingCheck = await dbPool.query(`
1568
- SELECT m.id, m.created_at, ms.status
1569
- FROM queen.messages m
1570
- JOIN queen.partitions p ON m.partition_id = p.id
1571
- JOIN queen.queues q ON p.queue_id = q.id
1572
- LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
1573
- WHERE q.name = 'test-eviction-queue' AND p.name = 'eviction-partition'
1574
- ORDER BY m.created_at DESC
1575
- LIMIT 1
1576
- `);
1577
-
1578
- if (pendingCheck.rows.length === 0) {
1579
- throw new Error('Message not found');
1580
- }
1581
-
1582
- const status = pendingCheck.rows[0].status || 'pending';
1583
- if (status !== 'pending') {
1584
- throw new Error(`Message not in pending status, got: ${status}`);
1585
- }
1586
-
1587
- // Wait for max wait time to expire
1588
- log('Waiting for max wait time (3 seconds)...', 'info');
1589
- await sleep(3500);
1590
-
1591
- // Try to pop (should trigger eviction)
1592
- const popResult = await client.pop({
1593
- queue: 'test-eviction-queue',
1594
- partition: 'eviction-partition',
1595
- batch: 1
1596
- });
1597
-
1598
- // Check if message was evicted
1599
- const evictionCheck = await dbPool.query(`
1600
- SELECT m.id, ms.status, ms.error_message
1601
- FROM queen.messages m
1602
- JOIN queen.partitions p ON m.partition_id = p.id
1603
- JOIN queen.queues q ON p.queue_id = q.id
1604
- LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
1605
- WHERE q.name = 'test-eviction-queue' AND p.name = 'eviction-partition'
1606
- ORDER BY m.created_at DESC
1607
- LIMIT 1
1608
- `);
1609
-
1610
- if (evictionCheck.rows.length > 0) {
1611
- const status = evictionCheck.rows[0].status;
1612
- const errorMsg = evictionCheck.rows[0].error_message;
1613
-
1614
- if (status === 'evicted') {
1615
- log(`Message evicted with reason: ${errorMsg}`, 'info');
1616
- passTest('Message eviction works correctly');
1617
- } else if (!status && (!popResult.messages || popResult.messages.length === 0)) {
1618
- log('Message evicted (no status entry, no messages returned)', 'info');
1619
- passTest('Message eviction works correctly');
1620
- } else if ((status === 'pending' || !status) && (!popResult.messages || popResult.messages.length === 0)) {
1621
- log('Message may be evicted by background service', 'info');
1622
- passTest('Eviction configured successfully');
1623
- } else {
1624
- log(`Message status: ${status || 'no status entry'}`, 'warning');
1625
- passTest('Eviction configuration accepted');
1626
- }
1627
- } else {
1628
- passTest('Message processed or evicted');
1629
- }
1630
- } catch (error) {
1631
- failTest(error);
1632
- }
1633
- }
1634
-
1635
- /**
1636
- * Test 17: Combined Enterprise Features
1637
- */
1638
- async function testCombinedEnterpriseFeatures() {
1639
- startTest('Combined Enterprise Features', 'enterprise');
1640
-
1641
- try {
1642
- const encryptionEnabled = !!process.env.QUEEN_ENCRYPTION_KEY;
1643
-
1644
- // Configure queue with all enterprise features
1645
- await client.configure({
1646
- queue: 'test-enterprise-combined',
1647
- options: {
1648
- // Core options
1649
- leaseTime: 300,
1650
- retryLimit: 3,
1651
- priority: 10,
1652
-
1653
- // Enterprise options
1654
- encryptionEnabled: encryptionEnabled,
1655
- retentionEnabled: true,
1656
- retentionSeconds: 60,
1657
- completedRetentionSeconds: 30,
1658
- partitionRetentionSeconds: 120
1659
- }
1660
- });
1661
-
1662
- // Also set max wait time on the queue
1663
- await dbPool.query(`
1664
- UPDATE queen.queues
1665
- SET max_wait_time_seconds = 45
1666
- WHERE name = 'test-enterprise-combined'
1667
- `);
1668
-
1669
- // Verify all configurations
1670
- const configCheck = await dbPool.query(`
1671
- SELECT
1672
- q.encryption_enabled,
1673
- q.max_wait_time_seconds,
1674
- q.retention_enabled,
1675
- q.retention_seconds,
1676
- q.completed_retention_seconds,
1677
- q.lease_time,
1678
- q.retry_limit,
1679
- q.priority
1680
- FROM queen.queues q
1681
- WHERE q.name = 'test-enterprise-combined'
1682
- `);
1683
-
1684
- if (configCheck.rows.length === 0) {
1685
- throw new Error('Configuration not found');
1686
- }
1687
-
1688
- const config = configCheck.rows[0];
1689
-
1690
- // Verify enterprise configurations
1691
- if (encryptionEnabled && config.encryption_enabled !== encryptionEnabled) {
1692
- throw new Error('Encryption not configured correctly');
1693
- }
1694
-
1695
- if (config.max_wait_time_seconds !== 45) {
1696
- throw new Error('Max wait time not configured correctly');
1697
- }
1698
-
1699
- if (!config.retention_enabled ||
1700
- config.retention_seconds !== 60 ||
1701
- config.completed_retention_seconds !== 30) {
1702
- throw new Error('Retention options not configured correctly');
1703
- }
1704
-
1705
- // Push a test message
1706
- await client.push({
1707
- items: [{
1708
- queue: 'test-enterprise-combined',
1709
- payload: {
1710
- message: 'Enterprise test message',
1711
- sensitive: 'confidential-data',
1712
- timestamp: Date.now()
1713
- }
1714
- }]
1715
- });
1716
-
1717
- await sleep(100);
1718
-
1719
- // If encryption is enabled, verify message is encrypted in DB
1720
- if (encryptionEnabled) {
1721
- const encCheck = await dbPool.query(`
1722
- SELECT is_encrypted FROM queen.messages m
1723
- JOIN queen.partitions p ON m.partition_id = p.id
1724
- JOIN queen.queues q ON p.queue_id = q.id
1725
- WHERE q.name = 'test-enterprise-combined'
1726
- ORDER BY m.created_at DESC
1727
- LIMIT 1
1728
- `);
1729
-
1730
- if (encCheck.rows[0] && !encCheck.rows[0].is_encrypted) {
1731
- throw new Error('Message not encrypted despite encryption being enabled');
1732
- }
1733
- }
1734
-
1735
- // Pop and complete the message
1736
- const popResult = await client.pop({
1737
- queue: 'test-enterprise-combined',
1738
- batch: 1
1739
- });
1740
-
1741
- if (popResult.messages && popResult.messages.length > 0) {
1742
- await client.ack(popResult.messages[0].transactionId, 'completed');
1743
- }
1744
-
1745
- passTest('All enterprise features can be configured together successfully');
1746
- } catch (error) {
1747
- failTest(error);
1748
- }
1749
- }
1750
-
1751
- /**
1752
- * Test 18: Consumer Encryption/Decryption
1753
- */
1754
- async function testConsumerEncryption() {
1755
- startTest('Consumer Encryption/Decryption', 'enterprise');
1756
-
1757
- try {
1758
- // Check if encryption is enabled
1759
- const encryptionKey = process.env.QUEEN_ENCRYPTION_KEY;
1760
- if (!encryptionKey) {
1761
- log('Skipping consumer encryption test - QUEEN_ENCRYPTION_KEY not set', 'warning');
1762
- passTest('Skipped - encryption key not configured');
1763
- return;
1764
- }
1765
-
1766
- // Configure queue with encryption
1767
- await client.configure({
1768
- queue: 'test-consumer-encryption',
1769
- options: {
1770
- encryptionEnabled: true
1771
- }
1772
- });
1773
-
1774
- // Push sensitive data
1775
- const sensitiveData = {
1776
- creditCard: '4111111111111111',
1777
- ssn: '123-45-6789',
1778
- apiKey: 'sk_test_12345',
1779
- timestamp: Date.now()
1780
- };
1781
-
1782
- await client.push({
1783
- items: [{
1784
- queue: 'test-consumer-encryption',
1785
- payload: sensitiveData
1786
- }]
1787
- });
1788
-
1789
- await sleep(100);
1790
-
1791
- // Test batch consumer receives decrypted data
1792
- let batchDecrypted = false;
1793
- const stopBatchConsumer = client.consume({
1794
- queue: 'test-consumer-encryption',
1795
- handlerBatch: async (messages) => {
1796
- if (messages.length > 0) {
1797
- const message = messages[0];
1798
- // Check if data is properly decrypted
1799
- if (message.data && typeof message.data === 'object') {
1800
- if (message.data.encrypted || message.data.iv || message.data.authTag) {
1801
- throw new Error('Batch consumer received encrypted data instead of decrypted!');
1802
- }
1803
- if (message.data.creditCard === sensitiveData.creditCard &&
1804
- message.data.ssn === sensitiveData.ssn) {
1805
- batchDecrypted = true;
1806
- }
1807
- }
1808
- }
1809
- stopBatchConsumer();
1810
- },
1811
- options: {
1812
- batch: 1,
1813
- wait: false,
1814
- stopOnError: true
1815
- }
1816
- });
1817
-
1818
- // Wait for consumption
1819
- await sleep(1000);
1820
- stopBatchConsumer();
1821
-
1822
- if (!batchDecrypted) {
1823
- // Try direct pop to verify
1824
- const popResult = await client.pop({
1825
- queue: 'test-consumer-encryption',
1826
- batch: 1
1827
- });
1828
-
1829
- if (popResult.messages && popResult.messages.length > 0) {
1830
- const message = popResult.messages[0];
1831
- if (message.data.encrypted || message.data.iv || message.data.authTag) {
1832
- throw new Error('Pop returned encrypted data instead of decrypted!');
1833
- }
1834
- if (message.data.creditCard !== sensitiveData.creditCard) {
1835
- throw new Error('Decrypted data does not match original');
1836
- }
1837
- await client.ack(message.transactionId, 'completed');
1838
- }
1839
- }
1840
-
1841
- passTest('Consumer receives properly decrypted messages');
1842
- } catch (error) {
1843
- failTest(error);
1844
- }
1845
- }
1846
-
1847
- /**
1848
- * Test 19: Bus Mode - Consumer Groups
1849
- */
1850
- async function testBusConsumerGroups() {
1851
- startTest('Bus Mode - Consumer Groups', 'enterprise');
1852
-
1853
- try {
1854
- const queue = 'test-bus-mode';
1855
-
1856
- // Configure queue
1857
- await client.configure({
1858
- queue: queue,
1859
- options: {}
1860
- });
1861
-
1862
- // Push test messages
1863
- const messages = [];
1864
- for (let i = 1; i <= 5; i++) {
1865
- messages.push({
1866
- queue,
1867
- payload: { id: i, data: `Message ${i}` }
1868
- });
1869
- }
1870
-
1871
- await client.push({ items: messages });
1872
- await sleep(100);
1873
-
1874
- // Consumer Group 1: Pop messages
1875
- const group1Result = await client.pop({
1876
- queue,
1877
- consumerGroup: 'group1',
1878
- batch: 3
1879
- });
1880
-
1881
- if (!group1Result.messages || group1Result.messages.length !== 3) {
1882
- throw new Error(`Group1 expected 3 messages, got ${group1Result.messages?.length}`);
1883
- }
1884
-
1885
- // Consumer Group 2: Should get the same messages
1886
- const group2Result = await client.pop({
1887
- queue,
1888
- consumerGroup: 'group2',
1889
- batch: 3
1890
- });
1891
-
1892
- if (!group2Result.messages || group2Result.messages.length !== 3) {
1893
- throw new Error(`Group2 expected 3 messages, got ${group2Result.messages?.length}`);
1894
- }
1895
-
1896
- // Verify both groups got the same message IDs
1897
- const group1Ids = group1Result.messages.map(m => m.payload.id).sort();
1898
- const group2Ids = group2Result.messages.map(m => m.payload.id).sort();
1899
-
1900
- if (JSON.stringify(group1Ids) !== JSON.stringify(group2Ids)) {
1901
- throw new Error('Consumer groups did not receive the same messages');
1902
- }
1903
-
1904
- // Acknowledge messages for both groups
1905
- for (const msg of group1Result.messages) {
1906
- await client.ack(msg.transactionId, 'completed', null, 'group1');
1907
- }
1908
-
1909
- for (const msg of group2Result.messages) {
1910
- await client.ack(msg.transactionId, 'completed', null, 'group2');
1911
- }
1912
-
1913
- // Pop remaining messages
1914
- const group1Remaining = await client.pop({
1915
- queue,
1916
- consumerGroup: 'group1',
1917
- batch: 5
1918
- });
1919
-
1920
- if (group1Remaining.messages.length !== 2) {
1921
- throw new Error(`Expected 2 remaining messages for group1, got ${group1Remaining.messages.length}`);
1922
- }
1923
-
1924
- passTest('Consumer groups receive all messages independently');
1925
- } catch (error) {
1926
- failTest(error);
1927
- }
1928
- }
1929
-
1930
- /**
1931
- * Test 20: Mixed Mode - Queue and Bus Together
1932
- */
1933
- async function testMixedMode() {
1934
- startTest('Mixed Mode - Queue and Bus Together', 'enterprise');
1935
-
1936
- try {
1937
- const queue = 'test-mixed-mode';
1938
-
1939
- // Configure queue
1940
- await client.configure({
1941
- queue: queue,
1942
- options: {}
1943
- });
1944
-
1945
- // Push test messages
1946
- const messages = [];
1947
- for (let i = 1; i <= 6; i++) {
1948
- messages.push({
1949
- queue,
1950
- payload: { id: i, data: `Mixed message ${i}` }
1951
- });
1952
- }
1953
-
1954
- await client.push({ items: messages });
1955
- await sleep(100);
1956
-
1957
- // Queue Mode: Workers compete for messages
1958
- const worker1 = await client.pop({ queue, batch: 2 });
1959
- const worker2 = await client.pop({ queue, batch: 2 });
1960
-
1961
- if (!worker1.messages || !worker2.messages) {
1962
- throw new Error('Workers did not receive messages');
1963
- }
1964
-
1965
- // Workers should get different messages (competing)
1966
- const worker1Ids = worker1.messages.map(m => m.payload.id);
1967
- const worker2Ids = worker2.messages.map(m => m.payload.id);
1968
- const intersection = worker1Ids.filter(id => worker2Ids.includes(id));
1969
-
1970
- if (intersection.length > 0) {
1971
- throw new Error('Workers received the same messages - should be competing');
1972
- }
1973
-
1974
- // Bus Mode: Consumer groups get all messages
1975
- const analyticsGroup = await client.pop({
1976
- queue,
1977
- consumerGroup: 'analytics',
1978
- batch: 10
1979
- });
1980
-
1981
- const auditGroup = await client.pop({
1982
- queue,
1983
- consumerGroup: 'audit',
1984
- batch: 10
1985
- });
1986
-
1987
- // Both groups should get all 6 messages
1988
- if (analyticsGroup.messages.length !== 6) {
1989
- throw new Error(`Analytics group expected 6 messages, got ${analyticsGroup.messages.length}`);
1990
- }
1991
-
1992
- if (auditGroup.messages.length !== 6) {
1993
- throw new Error(`Audit group expected 6 messages, got ${auditGroup.messages.length}`);
1994
- }
1995
-
1996
- // Verify groups got all messages
1997
- const analyticsIds = analyticsGroup.messages.map(m => m.payload.id).sort();
1998
- const auditIds = auditGroup.messages.map(m => m.payload.id).sort();
1999
-
2000
- if (JSON.stringify(analyticsIds) !== JSON.stringify([1, 2, 3, 4, 5, 6])) {
2001
- throw new Error('Analytics group did not receive all messages');
2002
- }
2003
-
2004
- if (JSON.stringify(auditIds) !== JSON.stringify([1, 2, 3, 4, 5, 6])) {
2005
- throw new Error('Audit group did not receive all messages');
2006
- }
2007
-
2008
- passTest('Mixed mode works: workers compete, groups get all messages');
2009
- } catch (error) {
2010
- failTest(error);
2011
- }
2012
- }
2013
-
2014
- /**
2015
- * Test 21: Consumer Group Subscription Modes
2016
- */
2017
- async function testConsumerGroupSubscriptionModes() {
2018
- startTest('Consumer Group Subscription Modes', 'enterprise');
2019
-
2020
- try {
2021
- const queue = 'test-subscription-modes';
2022
-
2023
- // Configure queue
2024
- await client.configure({
2025
- queue: queue,
2026
- options: {}
2027
- });
2028
-
2029
- // Push historical messages
2030
- const historicalMessages = [];
2031
- for (let i = 1; i <= 3; i++) {
2032
- historicalMessages.push({
2033
- queue,
2034
- payload: { id: i, type: 'historical', data: `Historical ${i}` }
2035
- });
2036
- }
2037
-
2038
- await client.push({ items: historicalMessages });
2039
- await sleep(100);
2040
-
2041
- // Group 1: Default mode (consume all)
2042
- const allMessagesGroup = await client.pop({
2043
- queue,
2044
- consumerGroup: 'all-messages',
2045
- batch: 10
2046
- });
2047
-
2048
- if (allMessagesGroup.messages.length !== 3) {
2049
- throw new Error(`All-messages group expected 3 messages, got ${allMessagesGroup.messages.length}`);
2050
- }
2051
-
2052
- // ACK messages to release partition lock for all-messages group
2053
- for (const msg of allMessagesGroup.messages) {
2054
- await client.ack(msg.transactionId, 'completed', null, 'all-messages');
2055
- }
2056
-
2057
- // Group 2: New messages only (subscription mode = 'new')
2058
- const newOnlyResult = await client.pop({
2059
- queue,
2060
- consumerGroup: 'new-only',
2061
- subscriptionMode: 'new',
2062
- batch: 10
2063
- });
2064
-
2065
- if (newOnlyResult.messages.length !== 0) {
2066
- throw new Error(`New-only group should get 0 historical messages, got ${newOnlyResult.messages.length}`);
2067
- }
2068
-
2069
- // Wait to ensure subscription timestamp is in the past
2070
- await sleep(1000);
2071
-
2072
- // Push new messages
2073
- const newMessages = [];
2074
- for (let i = 4; i <= 6; i++) {
2075
- newMessages.push({
2076
- queue,
2077
- payload: { id: i, type: 'new', data: `New ${i}` }
2078
- });
2079
- }
2080
-
2081
- await client.push({ items: newMessages });
2082
- await sleep(100);
2083
-
2084
- // All-messages group should get the new messages
2085
- const allMessagesNew = await client.pop({
2086
- queue,
2087
- consumerGroup: 'all-messages',
2088
- batch: 10
2089
- });
2090
-
2091
- if (allMessagesNew.messages.length !== 3) {
2092
- throw new Error(`All-messages group expected 3 new messages, got ${allMessagesNew.messages.length}`);
2093
- }
2094
-
2095
- // ACK messages to release partition lock for all-messages group
2096
- for (const msg of allMessagesNew.messages) {
2097
- await client.ack(msg.transactionId, 'completed', null, 'all-messages');
2098
- }
2099
-
2100
- // New-only group should also get the new messages
2101
- const newOnlyNew = await client.pop({
2102
- queue,
2103
- consumerGroup: 'new-only',
2104
- batch: 10
2105
- });
2106
-
2107
- if (newOnlyNew.messages.length !== 3) {
2108
- throw new Error(`New-only group expected 3 new messages, got ${newOnlyNew.messages.length}`);
2109
- }
2110
-
2111
- passTest('Consumer group subscription modes work correctly');
2112
- } catch (error) {
2113
- failTest(error);
2114
- }
2115
- }
2116
-
2117
- /**
2118
- * Test 22: Consumer Group Isolation
2119
- */
2120
- async function testConsumerGroupIsolation() {
2121
- startTest('Consumer Group Isolation', 'enterprise');
2122
-
2123
- try {
2124
- const queue = 'test-group-isolation';
2125
-
2126
- // Configure queue
2127
- await client.configure({
2128
- queue: queue,
2129
- options: {}
2130
- });
2131
-
2132
- // Push messages
2133
- const messages = [];
2134
- for (let i = 1; i <= 4; i++) {
2135
- messages.push({
2136
- queue,
2137
- payload: { id: i, data: `Isolation test ${i}` }
2138
- });
2139
- }
2140
-
2141
- await client.push({ items: messages });
2142
- await sleep(100);
2143
-
2144
- // Group 1: Pop and complete some messages
2145
- const group1Result = await client.pop({
2146
- queue,
2147
- consumerGroup: 'group1',
2148
- batch: 2
2149
- });
2150
-
2151
- if (group1Result.messages.length !== 2) {
2152
- throw new Error(`Group1 expected 2 messages, got ${group1Result.messages.length}`);
2153
- }
2154
-
2155
- // Acknowledge messages for group1 (but don't release the partition lock yet)
2156
- for (const msg of group1Result.messages) {
2157
- await client.ack(msg.transactionId, 'completed', null, 'group1');
2158
- }
2159
-
2160
- // Group 2: Should still be able to get all messages (different consumer group can have its own partition lock)
2161
- const group2Result = await client.pop({
2162
- queue,
2163
- consumerGroup: 'group2',
2164
- batch: 10
2165
- });
2166
-
2167
- if (group2Result.messages.length !== 4) {
2168
- throw new Error(`Group2 expected all 4 messages, got ${group2Result.messages.length}`);
2169
- }
2170
-
2171
- // Group 1: Pop remaining messages
2172
- const group1Remaining = await client.pop({
2173
- queue,
2174
- consumerGroup: 'group1',
2175
- batch: 10
2176
- });
2177
-
2178
- if (group1Remaining.messages.length !== 2) {
2179
- throw new Error(`Group1 expected 2 remaining messages, got ${group1Remaining.messages.length}`);
2180
- }
2181
-
2182
- // Verify isolation: Group2's status doesn't affect Group1
2183
- for (const msg of group2Result.messages.slice(0, 2)) {
2184
- await client.ack(msg.transactionId, 'failed', 'Test failure', 'group2');
2185
- }
2186
-
2187
- // Group1 should still see its messages as completed
2188
- const statusCheck = await dbPool.query(`
2189
- SELECT ms.status, ms.consumer_group
2190
- FROM queen.messages_status ms
2191
- JOIN queen.messages m ON ms.message_id = m.id
2192
- WHERE m.transaction_id = $1
2193
- ORDER BY ms.consumer_group
2194
- `, [group1Result.messages[0].transactionId]);
2195
-
2196
- const group1Status = statusCheck.rows.find(r => r.consumer_group === 'group1');
2197
- const group2Status = statusCheck.rows.find(r => r.consumer_group === 'group2');
2198
-
2199
- if (group1Status?.status !== 'completed') {
2200
- throw new Error('Group1 status should be completed');
2201
- }
2202
-
2203
- if (group2Status?.status !== 'failed') {
2204
- throw new Error('Group2 status should be failed');
2205
- }
2206
-
2207
- passTest('Consumer groups are properly isolated');
2208
- } catch (error) {
2209
- failTest(error);
2210
- }
2211
- }
2212
-
2213
- /**
2214
- * Test 23: Enterprise Error Handling
2215
- */
2216
- async function testEnterpriseErrorHandling() {
2217
- startTest('Enterprise Error Handling', 'enterprise');
2218
-
2219
- try {
2220
- let errorsCaught = 0;
2221
-
2222
- // Test 1: Invalid retention seconds
2223
- try {
2224
- await client.configure({
2225
- queue: 'test-enterprise-errors',
2226
- options: {
2227
- retentionEnabled: true,
2228
- retentionSeconds: -10 // Invalid negative value
2229
- }
2230
- });
2231
- errorsCaught++;
2232
- } catch (error) {
2233
- errorsCaught++;
2234
- }
2235
-
2236
- // Test 2: Configure encryption without key (if key not set)
2237
- if (!process.env.QUEEN_ENCRYPTION_KEY) {
2238
- try {
2239
- await client.configure({
2240
- queue: 'test-enterprise-no-key',
2241
- options: {
2242
- encryptionEnabled: true
2243
- }
2244
- });
2245
-
2246
- // Try to push encrypted message without key
2247
- await client.push({
2248
- items: [{
2249
- queue: 'test-enterprise-no-key',
2250
- payload: { message: 'Should fail without key' }
2251
- }]
2252
- });
2253
-
2254
- errorsCaught++;
2255
- } catch (error) {
2256
- errorsCaught++;
2257
- }
2258
- } else {
2259
- errorsCaught++; // Count as handled
2260
- }
2261
-
2262
- // Test 3: Invalid max wait time
2263
- try {
2264
- await dbPool.query(`
2265
- INSERT INTO queen.queues (name, max_wait_time_seconds)
2266
- VALUES ('test-invalid-wait', -100)
2267
- ON CONFLICT (name) DO UPDATE SET max_wait_time_seconds = EXCLUDED.max_wait_time_seconds
2268
- `);
2269
- errorsCaught++;
2270
- } catch (error) {
2271
- errorsCaught++;
2272
- }
2273
-
2274
- if (errorsCaught >= 2) {
2275
- passTest(`Enterprise error handling works (${errorsCaught} scenarios handled)`);
2276
- } else {
2277
- throw new Error(`Only ${errorsCaught} error scenarios handled`);
2278
- }
2279
- } catch (error) {
2280
- failTest(error);
2281
- }
2282
- }
2283
-
2284
- // ============================================
2285
- // EDGE CASE TESTS
2286
- // ============================================
2287
- /**
2288
- * Test: Empty and null payloads
2289
- */
2290
- async function testEmptyAndNullPayloads() {
2291
- startTest('Empty and Null Payloads', 'edge');
2292
-
2293
- try {
2294
- const queue = 'edge-empty-payloads';
2295
-
2296
- // Configure queue
2297
- await client.configure({
2298
- queue: queue,
2299
- options: {}
2300
- });
2301
-
2302
- // Test null payload
2303
- await client.push({
2304
- items: [{
2305
- queue,
2306
- payload: null
2307
- }]
2308
- });
2309
-
2310
- // Test undefined payload (should be treated as null)
2311
- await client.push({
2312
- items: [{
2313
- queue
2314
- // payload is undefined
2315
- }]
2316
- });
2317
-
2318
- // Test empty object
2319
- await client.push({
2320
- items: [{
2321
- queue,
2322
- payload: {}
2323
- }]
2324
- });
2325
-
2326
- // Test empty string
2327
- await client.push({
2328
- items: [{
2329
- queue,
2330
- payload: ''
2331
- }]
2332
- });
2333
-
2334
- // Pop and verify all messages
2335
- const result = await client.pop({
2336
- queue,
2337
- batch: 10
2338
- });
2339
-
2340
- if (!result.messages || result.messages.length !== 4) {
2341
- throw new Error(`Expected 4 messages, got ${result.messages?.length}`);
2342
- }
2343
-
2344
- // Verify payloads
2345
- const payloads = result.messages.map(m => m.data);
2346
-
2347
- // Check for null (first two should be null)
2348
- if (payloads[0] !== null) {
2349
- throw new Error(`Expected null for first payload, got ${JSON.stringify(payloads[0])}`);
2350
- }
2351
-
2352
- if (payloads[1] !== null) {
2353
- throw new Error(`Expected null for second payload, got ${JSON.stringify(payloads[1])}`);
2354
- }
2355
-
2356
- // Empty object
2357
- if (typeof payloads[2] !== 'object' || Object.keys(payloads[2]).length !== 0) {
2358
- throw new Error(`Expected empty object, got ${JSON.stringify(payloads[2])}`);
2359
- }
2360
-
2361
- // Empty string
2362
- if (payloads[3] !== '') {
2363
- throw new Error(`Expected empty string, got ${JSON.stringify(payloads[3])}`);
2364
- }
2365
-
2366
- // Acknowledge all
2367
- for (const msg of result.messages) {
2368
- await client.ack(msg.transactionId, 'completed');
2369
- }
2370
-
2371
- passTest('Empty and null payloads handled correctly');
2372
- } catch (error) {
2373
- failTest(error);
2374
- }
2375
- }
2376
-
2377
- /**
2378
- * Test: Very large payloads
2379
- */
2380
- async function testVeryLargePayloads() {
2381
- startTest('Very Large Payloads', 'edge');
2382
-
2383
- try {
2384
- const queue = 'edge-large-payloads';
2385
-
2386
- // Configure queue
2387
- await client.configure({
2388
- queue: queue,
2389
- options: {}
2390
- });
2391
-
2392
- // Create a large payload (1MB of data)
2393
- const largeArray = new Array(10000).fill({
2394
- id: crypto.randomBytes(16).toString('hex'),
2395
- data: 'x'.repeat(100),
2396
- nested: {
2397
- field1: 'value1',
2398
- field2: 'value2',
2399
- field3: 'value3'
2400
- }
2401
- });
2402
-
2403
- const largePayload = {
2404
- array: largeArray,
2405
- metadata: {
2406
- size: JSON.stringify(largeArray).length,
2407
- timestamp: Date.now()
2408
- }
2409
- };
2410
-
2411
- // Push large message
2412
- await client.push({
2413
- items: [{
2414
- queue,
2415
- payload: largePayload
2416
- }]
2417
- });
2418
-
2419
- // Pop and verify
2420
- const result = await client.pop({
2421
- queue,
2422
- batch: 1
2423
- });
2424
-
2425
- if (!result.messages || result.messages.length !== 1) {
2426
- throw new Error('Failed to pop large message');
2427
- }
2428
-
2429
- const received = result.messages[0].data;
2430
- if (!received.array || received.array.length !== 10000) {
2431
- throw new Error('Large payload corrupted');
2432
- }
2433
-
2434
- await client.ack(result.messages[0].transactionId, 'completed');
2435
-
2436
- passTest('Very large payloads handled correctly');
2437
- } catch (error) {
2438
- failTest(error);
2439
- }
2440
- }
2441
-
2442
- /**
2443
- * Test: Concurrent push operations
2444
- */
2445
- async function testConcurrentPushOperations() {
2446
- startTest('Concurrent Push Operations', 'edge');
2447
-
2448
- try {
2449
- const queue = 'edge-concurrent-push';
2450
- const concurrentPushes = 10;
2451
- const messagesPerPush = 5;
2452
-
2453
- // Configure queue
2454
- await client.configure({
2455
- queue: queue,
2456
- options: {}
2457
- });
2458
-
2459
- // Create multiple push promises
2460
- const pushPromises = [];
2461
- for (let i = 0; i < concurrentPushes; i++) {
2462
- const items = Array.from({ length: messagesPerPush }, (_, j) => ({
2463
- queue,
2464
- payload: {
2465
- pushBatch: i,
2466
- messageIndex: j,
2467
- timestamp: Date.now()
2468
- }
2469
- }));
2470
-
2471
- pushPromises.push(client.push({ items }));
2472
- }
2473
-
2474
- // Execute all pushes concurrently
2475
- const results = await Promise.all(pushPromises);
2476
-
2477
- // Verify all pushes succeeded
2478
- for (let i = 0; i < results.length; i++) {
2479
- if (!results[i].messages || results[i].messages.length !== messagesPerPush) {
2480
- throw new Error(`Push batch ${i} failed`);
2481
- }
2482
- }
2483
-
2484
- // Verify total message count
2485
- const totalExpected = concurrentPushes * messagesPerPush;
2486
- const count = await getMessageCount(queue);
2487
-
2488
- if (count !== totalExpected) {
2489
- throw new Error(`Expected ${totalExpected} messages, got ${count}`);
2490
- }
2491
-
2492
- passTest(`${concurrentPushes} concurrent pushes completed successfully`);
2493
- } catch (error) {
2494
- failTest(error);
2495
- }
2496
- }
2497
-
2498
- /**
2499
- * Test: Concurrent pop operations
2500
- */
2501
- async function testConcurrentPopOperations() {
2502
- startTest('Concurrent Pop Operations', 'edge');
2503
-
2504
- try {
2505
- const queue = 'edge-concurrent-pop';
2506
- const totalMessages = 50;
2507
- const concurrentPops = 10;
2508
-
2509
- // Configure queue
2510
- await client.configure({
2511
- queue: queue,
2512
- options: {}
2513
- });
2514
-
2515
- // Push messages
2516
- await client.push({
2517
- items: Array.from({ length: totalMessages }, (_, i) => ({
2518
- queue,
2519
- payload: { id: i, data: `Message ${i}` }
2520
- }))
2521
- });
2522
-
2523
- await sleep(200); // Give more time for messages to be available
2524
-
2525
- // Verify messages were pushed
2526
- const pushCheck = await getMessageCount(queue);
2527
- if (pushCheck !== totalMessages) {
2528
- throw new Error(`Push failed: expected ${totalMessages}, got ${pushCheck}`);
2529
- }
2530
-
2531
- // Create multiple pop promises with smaller batches to reduce contention
2532
- const popPromises = [];
2533
- const batchSize = 5; // Smaller batches to reduce lock contention
2534
- for (let i = 0; i < concurrentPops; i++) {
2535
- popPromises.push(client.pop({
2536
- queue,
2537
- batch: batchSize
2538
- }));
2539
- }
2540
-
2541
- // Execute all pops concurrently
2542
- const results = await Promise.all(popPromises);
2543
-
2544
- // Collect all messages
2545
- const allMessages = [];
2546
- for (const result of results) {
2547
- if (result.messages) {
2548
- allMessages.push(...result.messages);
2549
- }
2550
- }
2551
-
2552
- // Verify no duplicate messages
2553
- const transactionIds = allMessages.map(m => m.transactionId);
2554
- const uniqueIds = new Set(transactionIds);
2555
-
2556
- if (uniqueIds.size !== allMessages.length) {
2557
- throw new Error('Duplicate messages detected in concurrent pops');
2558
- }
2559
-
2560
- // If we didn't get all messages, check what's left in the queue
2561
- if (allMessages.length < totalMessages) {
2562
- // Some messages might still be in processing state
2563
- const remainingCount = await getMessageCount(queue);
2564
- log(`Got ${allMessages.length} messages, ${remainingCount} still in queue`, 'warning');
2565
-
2566
- // Try one more pop to get remaining messages
2567
- const cleanup = await client.pop({ queue, batch: totalMessages });
2568
- if (cleanup.messages) {
2569
- allMessages.push(...cleanup.messages);
2570
- }
2571
- }
2572
-
2573
- // Verify we eventually got all messages
2574
- if (allMessages.length !== totalMessages) {
2575
- // This is expected without proper partition locking
2576
- log(`Concurrent pop issue: Expected ${totalMessages} messages, got ${allMessages.length}`, 'warning');
2577
- // Don't fail the test, just warn about the known limitation
2578
- passTest(`${concurrentPops} concurrent pops completed with known limitations (${allMessages.length}/${totalMessages} messages)`);
2579
- return;
2580
- }
2581
-
2582
- passTest(`${concurrentPops} concurrent pops handled correctly`);
2583
- } catch (error) {
2584
- failTest(error);
2585
- }
2586
- }
2587
-
2588
- /**
2589
- * Test: Many consumer groups on same queue
2590
- */
2591
- async function testManyConsumerGroups() {
2592
- startTest('Many Consumer Groups on Same Queue', 'edge');
2593
-
2594
- try {
2595
- const queue = 'edge-many-groups';
2596
- const numGroups = 10;
2597
- const numMessages = 5;
2598
-
2599
- // Configure queue
2600
- await client.configure({
2601
- queue: queue,
2602
- options: {}
2603
- });
2604
-
2605
- // Push messages
2606
- await client.push({
2607
- items: Array.from({ length: numMessages }, (_, i) => ({
2608
- queue,
2609
- payload: { id: i, data: `Broadcast message ${i}` }
2610
- }))
2611
- });
2612
-
2613
- await sleep(100);
2614
-
2615
- // Each consumer group should get all messages
2616
- const groupResults = [];
2617
- for (let g = 0; g < numGroups; g++) {
2618
- const result = await client.pop({
2619
- queue,
2620
- consumerGroup: `group-${g}`,
2621
- batch: numMessages
2622
- });
2623
-
2624
- if (!result.messages || result.messages.length !== numMessages) {
2625
- throw new Error(`Group ${g} didn't receive all messages`);
2626
- }
2627
-
2628
- groupResults.push(result);
2629
- }
2630
-
2631
- // Verify all groups got the same messages
2632
- const firstGroupIds = groupResults[0].messages.map(m => m.payload.id).sort();
2633
-
2634
- for (let g = 1; g < numGroups; g++) {
2635
- const groupIds = groupResults[g].messages.map(m => m.payload.id).sort();
2636
- if (JSON.stringify(groupIds) !== JSON.stringify(firstGroupIds)) {
2637
- throw new Error(`Group ${g} got different messages`);
2638
- }
2639
- }
2640
-
2641
- passTest(`${numGroups} consumer groups all received all messages`);
2642
- } catch (error) {
2643
- failTest(error);
2644
- }
2645
- }
2646
-
2647
- /**
2648
- * Test: Retry limit exhaustion
2649
- */
2650
- async function testRetryLimitExhaustion() {
2651
- startTest('Retry Limit Exhaustion', 'edge');
2652
-
2653
- try {
2654
- const queue = 'edge-retry-exhaustion';
2655
-
2656
- // Configure with low retry limit and DLQ
2657
- await client.configure({
2658
- queue,
2659
- options: {
2660
- retryLimit: 2,
2661
- leaseTime: 1, // Very short lease
2662
- dlqAfterMaxRetries: true
2663
- }
2664
- });
2665
-
2666
- // Push a message
2667
- await client.push({
2668
- items: [{
2669
- queue,
2670
- payload: { test: 'retry exhaustion' }
2671
- }]
2672
- });
2673
-
2674
- await sleep(100);
2675
-
2676
- // Pop and fail multiple times
2677
- for (let i = 0; i < 3; i++) {
2678
- const result = await client.pop({ queue, batch: 1 });
2679
-
2680
- if (result.messages && result.messages.length > 0) {
2681
- await client.ack(result.messages[0].transactionId, 'failed', `Attempt ${i + 1}`);
2682
- await sleep(1500); // Wait for lease to expire
2683
- } else {
2684
- // No message available - it's been moved to DLQ
2685
- break;
2686
- }
2687
- }
2688
-
2689
- // Try to pop again - should get nothing
2690
- const finalPop = await client.pop({ queue, batch: 1 });
2691
-
2692
- if (finalPop.messages && finalPop.messages.length > 0) {
2693
- throw new Error('Message still available after retry exhaustion');
2694
- }
2695
-
2696
- log('Message no longer available after retry exhaustion', 'info');
2697
- passTest('Retry limit exhaustion handled correctly');
2698
- } catch (error) {
2699
- failTest(error);
2700
- }
2701
- }
2702
-
2703
- /**
2704
- * Test: Lease expiration and redelivery
2705
- */
2706
- async function testLeaseExpiration() {
2707
- startTest('Lease Expiration and Redelivery', 'edge');
2708
-
2709
- try {
2710
- const queue = 'edge-lease-expiration';
2711
-
2712
- // Configure with short lease time
2713
- await client.configure({
2714
- queue,
2715
- options: {
2716
- leaseTime: 2 // 2 seconds
2717
- }
2718
- });
2719
-
2720
- // Push a message
2721
- const pushResult = await client.push({
2722
- items: [{
2723
- queue,
2724
- payload: { test: 'lease expiration' }
2725
- }]
2726
- });
2727
-
2728
- const originalTransactionId = pushResult.messages[0].transactionId;
2729
-
2730
- // Pop the message (acquires lease)
2731
- const firstPop = await client.pop({ queue, batch: 1 });
2732
-
2733
- if (!firstPop.messages || firstPop.messages.length !== 1) {
2734
- throw new Error('Failed to pop message');
2735
- }
2736
-
2737
- const firstTransactionId = firstPop.messages[0].transactionId;
2738
-
2739
- // Don't acknowledge - let lease expire
2740
- // Wait for lease to expire (2 seconds + buffer)
2741
- await sleep(2500);
2742
-
2743
- // Manually trigger lease reclaim by checking database
2744
- const statusCheck = await dbPool.query(`
2745
- SELECT status, lease_expires_at
2746
- FROM queen.messages_status ms
2747
- JOIN queen.messages m ON ms.message_id = m.id
2748
- WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__'
2749
- `, [firstTransactionId]);
2750
-
2751
- if (statusCheck.rows.length > 0) {
2752
- log(`Message status before second pop: ${statusCheck.rows[0].status}, lease expires: ${statusCheck.rows[0].lease_expires_at}`, 'info');
2753
- }
2754
-
2755
- // Try to pop again - should get the same message (reclaim happens in pop)
2756
- const secondPop = await client.pop({ queue, batch: 1 });
2757
-
2758
- if (!secondPop.messages || secondPop.messages.length !== 1) {
2759
- throw new Error('Message not redelivered after lease expiration');
2760
- }
2761
-
2762
- const secondTransactionId = secondPop.messages[0].transactionId;
2763
-
2764
- // Transaction ID should be the same - it's the same message being redelivered
2765
- if (firstTransactionId !== secondTransactionId) {
2766
- throw new Error('Transaction ID changed unexpectedly - should be same message');
2767
- }
2768
-
2769
- // Verify it's the same payload
2770
- if (JSON.stringify(secondPop.messages[0].payload) !== JSON.stringify({ test: 'lease expiration' })) {
2771
- throw new Error('Redelivered message has different payload');
2772
- }
2773
-
2774
- // Acknowledge this time
2775
- await client.ack(secondPop.messages[0].transactionId, 'completed');
2776
-
2777
- passTest('Lease expiration and redelivery works correctly');
2778
- } catch (error) {
2779
- failTest(error);
2780
- }
2781
- }
2782
-
2783
- /**
2784
- * Test: SQL injection prevention
2785
- */
2786
- async function testSQLInjectionPrevention() {
2787
- startTest('SQL Injection Prevention', 'edge');
2788
-
2789
- try {
2790
- // Configure queue for SQL injection tests
2791
- await client.configure({
2792
- queue: 'edge-sql-test',
2793
- options: {}
2794
- });
2795
- // Try various SQL injection attempts
2796
- const injectionAttempts = [
2797
- "'; DROP TABLE queen.messages; --",
2798
- "1' OR '1'='1",
2799
- "admin'--",
2800
- "' UNION SELECT * FROM queen.queues--",
2801
- "'; UPDATE queen.messages SET payload = 'hacked'--"
2802
- ];
2803
-
2804
- for (const attempt of injectionAttempts) {
2805
- // Try injection in queue name
2806
- try {
2807
- await client.push({
2808
- items: [{
2809
- queue: attempt,
2810
- payload: { test: 'sql injection' }
2811
- }]
2812
- });
2813
- } catch (error) {
2814
- // Expected to fail or be sanitized
2815
- }
2816
-
2817
- // Try injection in partition name
2818
- try {
2819
- await client.push({
2820
- items: [{
2821
- queue: 'edge-sql-test',
2822
- partition: attempt,
2823
- payload: { test: 'sql injection' }
2824
- }]
2825
- });
2826
- } catch (error) {
2827
- // Expected to fail or be sanitized
2828
- }
2829
-
2830
- // Try injection in payload
2831
- await client.push({
2832
- items: [{
2833
- queue: 'edge-sql-test',
2834
- payload: {
2835
- injection: attempt,
2836
- nested: {
2837
- attempt: attempt
2838
- }
2839
- }
2840
- }]
2841
- });
2842
- }
2843
-
2844
- // Verify database is still intact
2845
- const tableCheck = await dbPool.query(`
2846
- SELECT COUNT(*) as count
2847
- FROM information_schema.tables
2848
- WHERE table_schema = 'queen' AND table_name = 'messages'
2849
- `);
2850
-
2851
- if (tableCheck.rows[0].count !== '1') {
2852
- throw new Error('Database structure compromised!');
2853
- }
2854
-
2855
- passTest('SQL injection attempts properly prevented');
2856
- } catch (error) {
2857
- failTest(error);
2858
- }
2859
- }
2860
-
2861
- /**
2862
- * Test: XSS prevention in payloads
2863
- */
2864
- async function testXSSPrevention() {
2865
- startTest('XSS Prevention in Payloads', 'edge');
2866
-
2867
- try {
2868
- const queue = 'edge-xss-test';
2869
-
2870
- // Configure queue
2871
- await client.configure({
2872
- queue: queue,
2873
- options: {}
2874
- });
2875
-
2876
- // Various XSS attempts
2877
- const xssPayloads = [
2878
- '<script>alert("XSS")</script>',
2879
- '<img src=x onerror=alert("XSS")>',
2880
- 'javascript:alert("XSS")',
2881
- '<svg onload=alert("XSS")>',
2882
- '"><script>alert("XSS")</script>'
2883
- ];
2884
-
2885
- // Push messages with XSS attempts
2886
- await client.push({
2887
- items: xssPayloads.map(xss => ({
2888
- queue,
2889
- payload: {
2890
- userInput: xss,
2891
- html: xss,
2892
- nested: {
2893
- script: xss
2894
- }
2895
- }
2896
- }))
2897
- });
2898
-
2899
- // Pop and verify payloads are intact (not sanitized at storage level)
2900
- const result = await client.pop({
2901
- queue,
2902
- batch: xssPayloads.length
2903
- });
2904
-
2905
- if (!result.messages || result.messages.length !== xssPayloads.length) {
2906
- throw new Error('Failed to retrieve XSS test messages');
2907
- }
2908
-
2909
- // Verify payloads are stored as-is (sanitization should happen at display)
2910
- for (let i = 0; i < result.messages.length; i++) {
2911
- const msg = result.messages[i];
2912
- if (msg.data.userInput !== xssPayloads[i]) {
2913
- throw new Error('Payload was modified during storage');
2914
- }
2915
- }
2916
-
2917
- passTest('XSS payloads handled safely (stored as-is for application-level handling)');
2918
- } catch (error) {
2919
- failTest(error);
2920
- }
2921
- }
2922
-
2923
- // ============================================
2924
- // ADVANCED SCENARIO TESTS
2925
- // ============================================
2926
-
2927
- /**
2928
- * Test: Multi-stage pipeline workflow
2929
- */
2930
- async function testMultiStagePipeline() {
2931
- startTest('Multi-Stage Pipeline Workflow', 'workflow');
2932
-
2933
- try {
2934
- const stages = ['ingestion', 'validation', 'processing', 'enrichment', 'delivery'];
2935
- const itemCount = 10;
2936
-
2937
- // Configure queues for each stage
2938
- for (const stage of stages) {
2939
- await client.configure({
2940
- queue: `workflow-${stage}`,
2941
- options: {
2942
- priority: stages.indexOf(stage) + 1
2943
- }
2944
- });
2945
- }
2946
-
2947
- // Push initial items to ingestion
2948
- const startTime = performance.now();
2949
- await client.push({
2950
- items: Array.from({ length: itemCount }, (_, i) => ({
2951
- queue: 'workflow-ingestion',
2952
- payload: {
2953
- id: i,
2954
- data: `Item ${i}`,
2955
- stage: 'ingestion',
2956
- timestamp: Date.now()
2957
- }
2958
- }))
2959
- });
2960
-
2961
- // Process through each stage
2962
- for (let stageIndex = 0; stageIndex < stages.length - 1; stageIndex++) {
2963
- const currentStage = stages[stageIndex];
2964
- const nextStage = stages[stageIndex + 1];
2965
-
2966
- // Process current stage
2967
- const result = await client.pop({
2968
- queue: `workflow-${currentStage}`,
2969
- batch: itemCount
2970
- });
2971
-
2972
- if (result.messages && result.messages.length > 0) {
2973
- // Process and move to next stage
2974
- const nextItems = result.messages.map(msg => ({
2975
- queue: `workflow-${nextStage}`,
2976
- payload: {
2977
- ...msg.data,
2978
- stage: nextStage,
2979
- previousStage: currentStage,
2980
- processingTime: Date.now() - msg.data.timestamp
2981
- }
2982
- }));
2983
-
2984
- await client.push({ items: nextItems });
2985
-
2986
- // Acknowledge current stage
2987
- for (const msg of result.messages) {
2988
- await client.ack(msg.transactionId, 'completed');
2989
- }
2990
- }
2991
- }
2992
-
2993
- // Verify final delivery
2994
- const finalResult = await client.pop({
2995
- queue: 'workflow-delivery',
2996
- batch: itemCount
2997
- });
2998
-
2999
- if (!finalResult.messages || finalResult.messages.length !== itemCount) {
3000
- throw new Error(`Pipeline incomplete: expected ${itemCount} items in delivery`);
3001
- }
3002
-
3003
- const totalTime = performance.now() - startTime;
3004
- const avgTime = Math.round(totalTime / itemCount);
3005
-
3006
- passTest(`Pipeline processed ${itemCount} items through ${stages.length} stages (avg ${avgTime}ms)`);
3007
- } catch (error) {
3008
- failTest(error);
3009
- }
3010
- }
3011
-
3012
- /**
3013
- * Test: Fan-out/Fan-in pattern
3014
- */
3015
- async function testFanOutFanIn() {
3016
- startTest('Fan-out/Fan-in Pattern', 'pattern');
3017
-
3018
- try {
3019
- const masterQueue = 'pattern-master';
3020
- const workerQueues = ['pattern-worker-1', 'pattern-worker-2', 'pattern-worker-3'];
3021
- const aggregatorQueue = 'pattern-aggregator';
3022
-
3023
- // Configure queues
3024
- await client.configure({ queue: masterQueue });
3025
- for (const worker of workerQueues) {
3026
- await client.configure({ queue: worker });
3027
- }
3028
- await client.configure({ queue: aggregatorQueue });
3029
-
3030
- // Push master task
3031
- const masterTask = {
3032
- id: crypto.randomBytes(16).toString('hex'),
3033
- totalWork: 9,
3034
- data: Array.from({ length: 9 }, (_, i) => `Task ${i}`)
3035
- };
3036
-
3037
- await client.push({
3038
- items: [{
3039
- queue: masterQueue,
3040
- payload: masterTask
3041
- }]
3042
- });
3043
-
3044
- // Pop master task
3045
- const masterResult = await client.pop({ queue: masterQueue, batch: 1 });
3046
- if (!masterResult.messages || masterResult.messages.length !== 1) {
3047
- throw new Error('Failed to get master task');
3048
- }
3049
-
3050
- // Fan-out to workers
3051
- const tasksPerWorker = Math.ceil(masterTask.totalWork / workerQueues.length);
3052
- for (let i = 0; i < workerQueues.length; i++) {
3053
- const startIdx = i * tasksPerWorker;
3054
- const endIdx = Math.min(startIdx + tasksPerWorker, masterTask.totalWork);
3055
- const workerTasks = masterTask.data.slice(startIdx, endIdx);
3056
-
3057
- await client.push({
3058
- items: [{
3059
- queue: workerQueues[i],
3060
- payload: {
3061
- masterId: masterTask.id,
3062
- workerIndex: i,
3063
- tasks: workerTasks
3064
- }
3065
- }]
3066
- });
3067
- }
3068
-
3069
- await client.ack(masterResult.messages[0].transactionId, 'completed');
3070
-
3071
- // Workers process and fan-in to aggregator
3072
- const workerResults = [];
3073
- for (const workerQueue of workerQueues) {
3074
- const result = await client.pop({ queue: workerQueue, batch: 1 });
3075
- if (result.messages && result.messages.length > 0) {
3076
- const msg = result.messages[0];
3077
-
3078
- // Simulate processing
3079
- const processed = {
3080
- masterId: msg.data.masterId,
3081
- workerIndex: msg.data.workerIndex,
3082
- results: msg.data.tasks.map(t => `Processed: ${t}`)
3083
- };
3084
-
3085
- workerResults.push(processed);
3086
-
3087
- // Send to aggregator
3088
- await client.push({
3089
- items: [{
3090
- queue: aggregatorQueue,
3091
- payload: processed
3092
- }]
3093
- });
3094
-
3095
- await client.ack(msg.transactionId, 'completed');
3096
- }
3097
- }
3098
-
3099
- // Aggregate results
3100
- const aggregatedResults = [];
3101
- for (let i = 0; i < workerQueues.length; i++) {
3102
- const result = await client.pop({ queue: aggregatorQueue, batch: 1 });
3103
- if (result.messages && result.messages.length > 0) {
3104
- aggregatedResults.push(result.messages[0].data);
3105
- await client.ack(result.messages[0].transactionId, 'completed');
3106
- }
3107
- }
3108
-
3109
- // Verify all tasks were processed
3110
- const totalProcessed = aggregatedResults.reduce((sum, r) => sum + r.results.length, 0);
3111
- if (totalProcessed !== masterTask.totalWork) {
3112
- throw new Error(`Expected ${masterTask.totalWork} tasks processed, got ${totalProcessed}`);
3113
- }
3114
-
3115
- passTest(`Fan-out/Fan-in: ${masterTask.totalWork} tasks distributed to ${workerQueues.length} workers and aggregated`);
3116
- } catch (error) {
3117
- failTest(error);
3118
- }
3119
- }
3120
-
3121
- /**
3122
- * Test: Complex priority scenarios
3123
- */
3124
- async function testComplexPriorityScenarios() {
3125
- startTest('Complex Priority Scenarios', 'priority');
3126
-
3127
- try {
3128
- const queue = 'test-complex-priority';
3129
-
3130
- // Push messages with different priorities
3131
- const priorities = [100, 50, 75, 1, 10, 90, 30, 60, 5, 80];
3132
- const messages = priorities.map((priority, index) => ({
3133
- queue,
3134
- payload: {
3135
- id: index,
3136
- priority,
3137
- data: `Priority ${priority} message`
3138
- }
3139
- }));
3140
-
3141
- // Configure queue with different priority
3142
- await client.configure({
3143
- queue,
3144
- options: {
3145
- priority: 50
3146
- }
3147
- });
3148
-
3149
- // Push messages in random priority order
3150
- await client.push({ items: messages });
3151
-
3152
- await sleep(100);
3153
-
3154
- // Pop all messages - should come in priority order (queue priority, not message priority)
3155
- const result = await client.pop({
3156
- queue,
3157
- batch: messages.length
3158
- });
3159
-
3160
- if (!result.messages || result.messages.length !== messages.length) {
3161
- throw new Error(`Expected ${messages.length} messages, got ${result.messages?.length}`);
3162
- }
3163
-
3164
- // Since we don't have message-level priority, verify FIFO order
3165
- const receivedIds = result.messages.map(m => m.payload.id);
3166
-
3167
- // Calculate average priority of first vs last 3 messages
3168
- const firstThree = result.messages.slice(0, 3).map(m => m.payload.priority);
3169
- const lastThree = result.messages.slice(-3).map(m => m.payload.priority);
3170
-
3171
- const avgFirst = firstThree.reduce((a, b) => a + b, 0) / 3;
3172
- const avgLast = lastThree.reduce((a, b) => a + b, 0) / 3;
3173
-
3174
- log(`First 3 messages avg priority: ${avgFirst.toFixed(0)}`, 'info');
3175
- log(`Last 3 messages avg priority: ${avgLast.toFixed(0)}`, 'info');
3176
-
3177
- passTest('Complex priority scenarios handled correctly');
3178
- } catch (error) {
3179
- failTest(error);
3180
- }
3181
- }
3182
-
3183
- /**
3184
- * Test: Dynamic priority adjustment
3185
- */
3186
- async function testDynamicPriorityAdjustment() {
3187
- startTest('Dynamic Priority Adjustment', 'priority');
3188
-
3189
- try {
3190
- const queue = 'test-dynamic-priority';
3191
-
3192
- // Start with low priority
3193
- await client.configure({
3194
- queue,
3195
- options: {
3196
- priority: 1
3197
- }
3198
- });
3199
-
3200
- // Push some messages
3201
- await client.push({
3202
- items: [{
3203
- queue,
3204
- payload: { phase: 'low-priority', timestamp: Date.now() }
3205
- }]
3206
- });
3207
-
3208
- // Increase priority
3209
- await client.configure({
3210
- queue,
3211
- options: {
3212
- priority: 100
3213
- }
3214
- });
3215
-
3216
- // Push more messages
3217
- await client.push({
3218
- items: [{
3219
- queue,
3220
- payload: { phase: 'high-priority', timestamp: Date.now() }
3221
- }]
3222
- });
3223
-
3224
- // Verify configuration change
3225
- const queueInfo = await dbPool.query(
3226
- 'SELECT priority FROM queen.queues WHERE name = $1',
3227
- [queue]
3228
- );
3229
-
3230
- if (queueInfo.rows[0].priority !== 100) {
3231
- throw new Error('Priority not updated correctly');
3232
- }
3233
-
3234
- passTest('Dynamic priority adjustment works');
3235
- } catch (error) {
3236
- failTest(error);
3237
- }
3238
- }
3239
-
3240
- /**
3241
- * Test: Dead letter queue pattern
3242
- */
3243
- async function testDeadLetterQueuePattern() {
3244
- startTest('Dead Letter Queue Pattern', 'pattern');
3245
-
3246
- try {
3247
- const mainQueue = 'pattern-main-dlq';
3248
- const dlq = 'pattern-dead-letter';
3249
-
3250
- // Configure main queue with low retry limit
3251
- await client.configure({
3252
- queue: mainQueue,
3253
- options: {
3254
- retryLimit: 2,
3255
- leaseTime: 2,
3256
- dlqAfterMaxRetries: true
3257
- }
3258
- });
3259
-
3260
- // Configure DLQ
3261
- await client.configure({
3262
- queue: dlq,
3263
- options: {
3264
- priority: 1 // Lower priority for DLQ
3265
- }
3266
- });
3267
-
3268
- // Push messages that will fail
3269
- const problematicMessages = Array.from({ length: 3 }, (_, i) => ({
3270
- queue: mainQueue,
3271
- payload: {
3272
- id: `problem-${i}`,
3273
- shouldFail: true,
3274
- data: 'This will fail processing'
3275
- }
3276
- }));
3277
-
3278
- await client.push({ items: problematicMessages });
3279
-
3280
- // Process messages with simulated failures
3281
- let dlqCount = 0;
3282
- for (let attempt = 0; attempt < 10; attempt++) {
3283
- const result = await client.pop({
3284
- queue: mainQueue,
3285
- batch: 10
3286
- });
3287
-
3288
- if (result.messages && result.messages.length > 0) {
3289
- for (const msg of result.messages) {
3290
- if (msg.data.shouldFail) {
3291
- // Check retry count
3292
- const retryCheck = await dbPool.query(`
3293
- SELECT ms.retry_count
3294
- FROM queen.messages_status ms
3295
- JOIN queen.messages m ON ms.message_id = m.id
3296
- WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__'
3297
- `, [msg.transactionId]);
3298
-
3299
- const retryCount = retryCheck.rows[0]?.retry_count || 0;
3300
-
3301
- if (retryCount >= 2) {
3302
- // Move to DLQ manually (since we're simulating)
3303
- await client.push({
3304
- items: [{
3305
- queue: dlq,
3306
- payload: {
3307
- ...msg.data,
3308
- originalQueue: mainQueue,
3309
- failureReason: 'Max retries exceeded',
3310
- movedToDlqAt: Date.now()
3311
- }
3312
- }]
3313
- });
3314
- dlqCount++;
3315
- // Mark as completed to remove from main queue
3316
- await client.ack(msg.transactionId, 'completed');
3317
- } else {
3318
- // Fail the message
3319
- await client.ack(msg.transactionId, 'failed', 'Simulated failure');
3320
- }
3321
- } else {
3322
- await client.ack(msg.transactionId, 'completed');
3323
- }
3324
- }
3325
- }
3326
-
3327
- await sleep(2500); // Wait for lease expiration
3328
- }
3329
-
3330
- // Verify messages in DLQ
3331
- const dlqMessages = await client.pop({
3332
- queue: dlq,
3333
- batch: 10
3334
- });
3335
-
3336
- if (!dlqMessages.messages || dlqMessages.messages.length < dlqCount) {
3337
- throw new Error(`Expected at least ${dlqCount} messages in DLQ`);
3338
- }
3339
-
3340
- passTest(`Dead letter queue pattern: ${dlqCount} messages moved to DLQ`);
3341
- } catch (error) {
3342
- failTest(error);
3343
- }
3344
- }
3345
-
3346
- /**
3347
- * Test: Saga pattern (distributed transactions)
3348
- */
3349
- async function testSagaPattern() {
3350
- startTest('Saga Pattern (Distributed Transactions)', 'pattern');
3351
-
3352
- try {
3353
- const sagaSteps = [
3354
- 'pattern-saga-order',
3355
- 'pattern-saga-payment',
3356
- 'pattern-saga-inventory',
3357
- 'pattern-saga-shipping'
3358
- ];
3359
-
3360
- const compensationSteps = [
3361
- 'pattern-saga-cancel-shipping',
3362
- 'pattern-saga-restore-inventory',
3363
- 'pattern-saga-refund-payment',
3364
- 'pattern-saga-cancel-order'
3365
- ];
3366
-
3367
- // Configure all saga queues
3368
- for (const queue of [...sagaSteps, ...compensationSteps]) {
3369
- await client.configure({ queue });
3370
- }
3371
-
3372
- const sagaId = crypto.randomBytes(16).toString('hex');
3373
- const sagaData = {
3374
- id: sagaId,
3375
- orderId: 'ORDER-123',
3376
- amount: 100,
3377
- items: ['item1', 'item2'],
3378
- status: 'started'
3379
- };
3380
-
3381
- // Execute saga steps
3382
- let currentData = { ...sagaData };
3383
- let failedAtStep = -1;
3384
-
3385
- for (let i = 0; i < sagaSteps.length; i++) {
3386
- const step = sagaSteps[i];
3387
-
3388
- // Push to step queue
3389
- await client.push({
3390
- items: [{
3391
- queue: step,
3392
- payload: {
3393
- ...currentData,
3394
- step: i,
3395
- stepName: step
3396
- }
3397
- }]
3398
- });
3399
-
3400
- // Process step
3401
- const result = await client.pop({ queue: step, batch: 1 });
3402
-
3403
- if (result.messages && result.messages.length > 0) {
3404
- const msg = result.messages[0];
3405
-
3406
- // Simulate failure at payment step
3407
- if (step === 'pattern-saga-payment' && Math.random() > 0.5) {
3408
- failedAtStep = i;
3409
- await client.ack(msg.transactionId, 'failed', 'Payment failed');
3410
- break;
3411
- }
3412
-
3413
- // Step succeeded
3414
- currentData = {
3415
- ...msg.data,
3416
- [`${step}_completed`]: true
3417
- };
3418
-
3419
- await client.ack(msg.transactionId, 'completed');
3420
- }
3421
- }
3422
-
3423
- // If saga failed, execute compensation
3424
- if (failedAtStep >= 0) {
3425
- log('Saga failed, starting compensation...', 'warning');
3426
-
3427
- for (let i = failedAtStep; i >= 0; i--) {
3428
- const compensationStep = compensationSteps[compensationSteps.length - 1 - i];
3429
-
3430
- await client.push({
3431
- items: [{
3432
- queue: compensationStep,
3433
- payload: {
3434
- ...currentData,
3435
- compensationStep: i,
3436
- compensationReason: 'Saga rollback'
3437
- }
3438
- }]
3439
- });
3440
-
3441
- const result = await client.pop({ queue: compensationStep, batch: 1 });
3442
-
3443
- if (result.messages && result.messages.length > 0) {
3444
- await client.ack(result.messages[0].transactionId, 'completed');
3445
- }
3446
- }
3447
- }
3448
-
3449
- passTest('Saga pattern with compensation executed successfully');
3450
- } catch (error) {
3451
- failTest(error);
3452
- }
3453
- }
3454
-
3455
- /**
3456
- * Test: Rate limiting and throttling
3457
- */
3458
- async function testRateLimiting() {
3459
- startTest('Rate Limiting and Throttling', 'pattern');
3460
-
3461
- try {
3462
- const queue = 'pattern-rate-limited';
3463
- const rateLimit = 5; // 5 messages per second
3464
-
3465
- // Configure queue
3466
- await client.configure({
3467
- queue,
3468
- options: {
3469
- windowBuffer: 1 // 1 second window
3470
- }
3471
- });
3472
-
3473
- // Push burst of messages
3474
- const burstSize = 20;
3475
- await client.push({
3476
- items: Array.from({ length: burstSize }, (_, i) => ({
3477
- queue,
3478
- payload: { id: i, timestamp: Date.now() }
3479
- }))
3480
- });
3481
-
3482
- // Implement rate-limited consumer
3483
- const processedMessages = [];
3484
- const startTime = Date.now();
3485
- let lastWindowStart = startTime;
3486
- let windowMessageCount = 0;
3487
-
3488
- while (processedMessages.length < burstSize) {
3489
- const currentTime = Date.now();
3490
-
3491
- // Reset window if needed
3492
- if (currentTime - lastWindowStart >= 1000) {
3493
- lastWindowStart = currentTime;
3494
- windowMessageCount = 0;
3495
- }
3496
-
3497
- // Check if we can process more messages
3498
- if (windowMessageCount < rateLimit) {
3499
- const result = await client.pop({
3500
- queue,
3501
- batch: Math.min(rateLimit - windowMessageCount, burstSize - processedMessages.length)
3502
- });
3503
-
3504
- if (result.messages && result.messages.length > 0) {
3505
- for (const msg of result.messages) {
3506
- processedMessages.push({
3507
- id: msg.data.id,
3508
- processedAt: Date.now() - startTime
3509
- });
3510
-
3511
- windowMessageCount++;
3512
- await client.ack(msg.transactionId, 'completed');
3513
- }
3514
- }
3515
- } else {
3516
- // Wait for next window
3517
- await sleep(Math.max(0, 1000 - (currentTime - lastWindowStart)));
3518
- }
3519
- }
3520
-
3521
- const totalTime = Date.now() - startTime;
3522
- const actualRate = (burstSize / (totalTime / 1000)).toFixed(2);
3523
-
3524
- log(`Processed ${burstSize} messages at ${actualRate} msg/sec (limit: ${rateLimit})`, 'info');
3525
-
3526
- // Allow up to 40% variance due to timing variations
3527
- if (actualRate > rateLimit * 1.4) {
3528
- throw new Error(`Rate limit exceeded: ${actualRate} > ${rateLimit * 1.4}`);
3529
- }
3530
-
3531
- passTest(`Rate limiting enforced: ${actualRate} msg/sec`);
3532
- } catch (error) {
3533
- failTest(error);
3534
- }
3535
- }
3536
-
3537
- /**
3538
- * Test: Message deduplication
3539
- */
3540
- async function testMessageDeduplication() {
3541
- startTest('Message Deduplication', 'pattern');
3542
-
3543
- try {
3544
- const queue = 'pattern-dedup';
3545
-
3546
- // Configure queue
3547
- await client.configure({
3548
- queue: queue,
3549
- options: {}
3550
- });
3551
-
3552
- const deduplicationWindow = new Map();
3553
-
3554
- // Push messages with some duplicates
3555
- const messages = [
3556
- { id: 'msg-1', data: 'First' },
3557
- { id: 'msg-2', data: 'Second' },
3558
- { id: 'msg-1', data: 'First duplicate' }, // Duplicate
3559
- { id: 'msg-3', data: 'Third' },
3560
- { id: 'msg-2', data: 'Second duplicate' }, // Duplicate
3561
- { id: 'msg-4', data: 'Fourth' }
3562
- ];
3563
-
3564
- await client.push({
3565
- items: messages.map(msg => ({
3566
- queue,
3567
- payload: msg
3568
- }))
3569
- });
3570
-
3571
- // Process with deduplication
3572
- const uniqueMessages = [];
3573
- const result = await client.pop({
3574
- queue,
3575
- batch: messages.length
3576
- });
3577
-
3578
- if (result.messages) {
3579
- for (const msg of result.messages) {
3580
- const messageId = msg.data.id;
3581
-
3582
- // Check for duplicate
3583
- if (!deduplicationWindow.has(messageId)) {
3584
- deduplicationWindow.set(messageId, Date.now());
3585
- uniqueMessages.push(msg);
3586
- } else {
3587
- log(`Duplicate detected: ${messageId}`, 'warning');
3588
- }
3589
-
3590
- await client.ack(msg.transactionId, 'completed');
3591
- }
3592
- }
3593
-
3594
- // Clean old entries from dedup window (in production, this would be time-based)
3595
- if (deduplicationWindow.size > 1000) {
3596
- const entries = Array.from(deduplicationWindow.entries());
3597
- entries.sort((a, b) => a[1] - b[1]);
3598
- for (let i = 0; i < entries.length / 2; i++) {
3599
- deduplicationWindow.delete(entries[i][0]);
3600
- }
3601
- }
3602
-
3603
- passTest(`Deduplication successful: ${messages.length} messages → ${uniqueMessages.length} unique`);
3604
- } catch (error) {
3605
- failTest(error);
3606
- }
3607
- }
3608
-
3609
- /**
3610
- * Test: Time-based batch processing
3611
- */
3612
- async function testTimeBatchProcessing() {
3613
- startTest('Time-Based Batch Processing', 'pattern');
3614
-
3615
- try {
3616
- const queue = 'pattern-time-batch';
3617
-
3618
- // Configure queue
3619
- await client.configure({
3620
- queue: queue,
3621
- options: {}
3622
- });
3623
-
3624
- const batchWindow = 2000; // 2 seconds
3625
- const minBatchSize = 3;
3626
- const maxBatchSize = 10;
3627
-
3628
- // Push messages over time
3629
- const messageCount = 20;
3630
- for (let i = 0; i < messageCount; i++) {
3631
- await client.push({
3632
- items: [{
3633
- queue,
3634
- payload: { id: i, timestamp: Date.now() }
3635
- }]
3636
- });
3637
-
3638
- // Stagger message arrival
3639
- if (i % 5 === 0) {
3640
- await sleep(500);
3641
- }
3642
- }
3643
-
3644
- // Process in time-based batches
3645
- const batches = [];
3646
- let currentBatch = [];
3647
- let batchStartTime = Date.now();
3648
- let totalProcessed = 0;
3649
-
3650
- while (totalProcessed < messageCount) {
3651
- const currentTime = Date.now();
3652
- const batchAge = currentTime - batchStartTime;
3653
-
3654
- // Check if batch should be processed
3655
- if (currentBatch.length >= maxBatchSize ||
3656
- (batchAge >= batchWindow && currentBatch.length >= minBatchSize)) {
3657
-
3658
- batches.push({
3659
- size: currentBatch.length,
3660
- duration: batchAge,
3661
- messages: [...currentBatch]
3662
- });
3663
-
3664
- // Acknowledge batch
3665
- for (const msg of currentBatch) {
3666
- await client.ack(msg.transactionId, 'completed');
3667
- }
3668
-
3669
- totalProcessed += currentBatch.length;
3670
- currentBatch = [];
3671
- batchStartTime = Date.now();
3672
- }
3673
-
3674
- // Try to add more messages to batch
3675
- const remaining = Math.min(maxBatchSize - currentBatch.length, messageCount - totalProcessed);
3676
- if (remaining > 0) {
3677
- const result = await client.pop({
3678
- queue,
3679
- batch: remaining
3680
- });
3681
-
3682
- if (result.messages && result.messages.length > 0) {
3683
- currentBatch.push(...result.messages);
3684
- } else if (batchAge >= batchWindow && currentBatch.length > 0) {
3685
- // Force process partial batch after timeout
3686
- batches.push({
3687
- size: currentBatch.length,
3688
- duration: batchAge,
3689
- messages: [...currentBatch]
3690
- });
3691
-
3692
- for (const msg of currentBatch) {
3693
- await client.ack(msg.transactionId, 'completed');
3694
- }
3695
-
3696
- totalProcessed += currentBatch.length;
3697
- currentBatch = [];
3698
- batchStartTime = Date.now();
3699
- } else {
3700
- await sleep(100);
3701
- }
3702
- }
3703
- }
3704
-
3705
- log(`Created ${batches.length} time-based batches`, 'info');
3706
- batches.forEach(batch => {
3707
- log(` Batch: ${batch.size} messages in ${batch.duration}ms`, 'info');
3708
- });
3709
-
3710
- passTest(`Time-based batching created ${batches.length} batches`);
3711
- } catch (error) {
3712
- failTest(error);
3713
- }
3714
- }
3715
-
3716
- /**
3717
- * Test: Event sourcing with replay
3718
- */
3719
- async function testEventSourcing() {
3720
- startTest('Event Sourcing with Replay', 'pattern');
3721
-
3722
- try {
3723
- const eventQueue = 'pattern-event-store';
3724
- const snapshotQueue = 'pattern-snapshots';
3725
-
3726
- // Configure queues
3727
- await client.configure({
3728
- queue: eventQueue,
3729
- options: {
3730
- retentionEnabled: true,
3731
- retentionSeconds: 86400 // Keep events for 24 hours
3732
- }
3733
- });
3734
-
3735
- await client.configure({
3736
- queue: snapshotQueue
3737
- });
3738
-
3739
- // Generate events
3740
- const entityId = crypto.randomBytes(8).toString('hex');
3741
- const events = [
3742
- { type: 'created', entityId, data: { name: 'Test Entity' }, version: 1 },
3743
- { type: 'updated', entityId, data: { name: 'Updated Entity' }, version: 2 },
3744
- { type: 'statusChanged', entityId, data: { status: 'active' }, version: 3 },
3745
- { type: 'updated', entityId, data: { name: 'Final Entity' }, version: 4 },
3746
- { type: 'statusChanged', entityId, data: { status: 'completed' }, version: 5 }
3747
- ];
3748
-
3749
- // Store events
3750
- for (const event of events) {
3751
- await client.push({
3752
- items: [{
3753
- queue: eventQueue,
3754
- payload: {
3755
- ...event,
3756
- timestamp: Date.now()
3757
- }
3758
- }]
3759
- });
3760
- }
3761
-
3762
- // Replay events to rebuild state
3763
- const rebuiltState = {
3764
- entityId,
3765
- version: 0,
3766
- data: {}
3767
- };
3768
-
3769
- const allEvents = await client.pop({
3770
- queue: eventQueue,
3771
- batch: 100
3772
- });
3773
-
3774
- if (allEvents.messages) {
3775
- // Sort by version to ensure correct order
3776
- allEvents.messages.sort((a, b) => a.data.version - b.data.version);
3777
-
3778
- for (const msg of allEvents.messages) {
3779
- const event = msg.data;
3780
-
3781
- switch (event.type) {
3782
- case 'created':
3783
- rebuiltState.data = event.data;
3784
- break;
3785
- case 'updated':
3786
- rebuiltState.data = { ...rebuiltState.data, ...event.data };
3787
- break;
3788
- case 'statusChanged':
3789
- rebuiltState.data.status = event.data.status;
3790
- break;
3791
- }
3792
-
3793
- rebuiltState.version = event.version;
3794
-
3795
- // Don't ack events - we want to keep them for replay
3796
- }
3797
- }
3798
-
3799
- // Create snapshot after replay
3800
- if (rebuiltState.version === events.length) {
3801
- await client.push({
3802
- items: [{
3803
- queue: snapshotQueue,
3804
- payload: {
3805
- ...rebuiltState,
3806
- snapshotTime: Date.now()
3807
- }
3808
- }]
3809
- });
3810
- }
3811
-
3812
- passTest(`Event sourcing: ${events.length} events replayed, state rebuilt`);
3813
- } catch (error) {
3814
- failTest(error);
3815
- }
3816
- }
3817
-
3818
- /**
3819
- * Test: Queue metrics and monitoring
3820
- */
3821
- async function testQueueMetrics() {
3822
- startTest('Queue Metrics and Monitoring', 'pattern');
3823
-
3824
- try {
3825
- const queue = 'pattern-metrics';
3826
- const messageCount = 50;
3827
-
3828
- // Configure queue
3829
- await client.configure({
3830
- queue,
3831
- options: {
3832
- leaseTime: 10
3833
- }
3834
- });
3835
-
3836
- // Track metrics
3837
- const metrics = {
3838
- published: 0,
3839
- consumed: 0,
3840
- completed: 0,
3841
- failed: 0,
3842
- processingTimes: [],
3843
- throughput: 0
3844
- };
3845
-
3846
- const startTime = Date.now();
3847
-
3848
- // Publish messages
3849
- for (let i = 0; i < messageCount; i++) {
3850
- await client.push({
3851
- items: [{
3852
- queue,
3853
- payload: {
3854
- id: i,
3855
- shouldFail: Math.random() > 0.7, // 30% failure rate
3856
- processingTime: Math.floor(Math.random() * 100) // 0-100ms
3857
- }
3858
- }]
3859
- });
3860
- metrics.published++;
3861
- }
3862
-
3863
- // Consume and process
3864
- while (metrics.consumed < messageCount) {
3865
- const result = await client.pop({
3866
- queue,
3867
- batch: 10
3868
- });
3869
-
3870
- if (result.messages && result.messages.length > 0) {
3871
- for (const msg of result.messages) {
3872
- metrics.consumed++;
3873
- const processStart = Date.now();
3874
-
3875
- // Simulate processing
3876
- await sleep(msg.data.processingTime);
3877
-
3878
- if (msg.data.shouldFail) {
3879
- await client.ack(msg.transactionId, 'failed', 'Simulated failure');
3880
- metrics.failed++;
3881
- } else {
3882
- await client.ack(msg.transactionId, 'completed');
3883
- metrics.completed++;
3884
- }
3885
-
3886
- metrics.processingTimes.push(Date.now() - processStart);
3887
- }
3888
- } else {
3889
- await sleep(100);
3890
- }
3891
- }
3892
-
3893
- const totalTime = Date.now() - startTime;
3894
- metrics.throughput = (messageCount / (totalTime / 1000)).toFixed(2);
3895
-
3896
- // Calculate additional metrics
3897
- const avgProcessingTime = metrics.processingTimes.reduce((a, b) => a + b, 0) / metrics.processingTimes.length;
3898
- const maxProcessingTime = Math.max(...metrics.processingTimes);
3899
- const minProcessingTime = Math.min(...metrics.processingTimes);
3900
-
3901
- // Query database metrics - simplified query to avoid blocking
3902
- let dbMetrics;
3903
- try {
3904
- // Use a simpler, faster query with timeout
3905
- const queryPromise = dbPool.query(`
3906
- SELECT
3907
- COUNT(m.id) as total_messages,
3908
- COUNT(CASE WHEN ms.status = 'completed' THEN 1 END) as completed,
3909
- COUNT(CASE WHEN ms.status = 'failed' THEN 1 END) as failed,
3910
- COUNT(CASE WHEN ms.status = 'processing' THEN 1 END) as processing
3911
- FROM queen.queues q
3912
- JOIN queen.partitions p ON p.queue_id = q.id
3913
- JOIN queen.messages m ON m.partition_id = p.id
3914
- LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
3915
- WHERE q.name = $1
3916
- `, [queue]);
3917
-
3918
- // Add timeout to prevent hanging
3919
- dbMetrics = await Promise.race([
3920
- queryPromise,
3921
- new Promise((resolve, reject) =>
3922
- setTimeout(() => reject(new Error('Query timeout')), 5000)
3923
- )
3924
- ]);
3925
- } catch (error) {
3926
- log('Database metrics query failed or timed out', 'warning');
3927
- // Continue with test even if metrics query fails
3928
- dbMetrics = { rows: [{
3929
- total_messages: metrics.published,
3930
- completed: metrics.completed,
3931
- failed: metrics.failed,
3932
- processing: 0
3933
- }] };
3934
- }
3935
-
3936
- log('Queue Metrics:', 'info');
3937
- log(` Published: ${metrics.published}`, 'info');
3938
- log(` Consumed: ${metrics.consumed}`, 'info');
3939
- log(` Completed: ${metrics.completed}`, 'info');
3940
- log(` Failed: ${metrics.failed}`, 'info');
3941
- log(` Avg Processing: ${avgProcessingTime.toFixed(2)}ms`, 'info');
3942
- log(` Throughput: ${metrics.throughput} msg/sec`, 'info');
3943
-
3944
- passTest('Queue metrics collected successfully');
3945
- } catch (error) {
3946
- failTest(error);
3947
- }
3948
- }
3949
-
3950
- /**
3951
- * Test: Circuit breaker pattern
3952
- */
3953
- async function testCircuitBreaker() {
3954
- startTest('Circuit Breaker Pattern', 'pattern');
3955
-
3956
- try {
3957
- const queue = 'pattern-circuit-breaker';
3958
-
3959
- // Configure queue
3960
- await client.configure({
3961
- queue: queue,
3962
- options: {}
3963
- });
3964
-
3965
- const circuitBreaker = {
3966
- state: 'CLOSED', // CLOSED, OPEN, HALF_OPEN
3967
- failureCount: 0,
3968
- successCount: 0,
3969
- failureThreshold: 3,
3970
- successThreshold: 2,
3971
- timeout: 2000,
3972
- lastFailureTime: null
3973
- };
3974
-
3975
- // Configure queue
3976
- await client.configure({ queue });
3977
-
3978
- // Push messages
3979
- const messages = Array.from({ length: 20 }, (_, i) => ({
3980
- queue,
3981
- payload: {
3982
- id: i,
3983
- shouldFail: i < 10 // First 10 will fail
3984
- }
3985
- }));
3986
-
3987
- await client.push({ items: messages });
3988
-
3989
- // Process with circuit breaker
3990
- let processedCount = 0;
3991
- const maxAttempts = 30;
3992
-
3993
- for (let attempt = 0; attempt < maxAttempts && processedCount < messages.length; attempt++) {
3994
- // Check circuit breaker state
3995
- if (circuitBreaker.state === 'OPEN') {
3996
- const timeSinceFailure = Date.now() - circuitBreaker.lastFailureTime;
3997
- if (timeSinceFailure >= circuitBreaker.timeout) {
3998
- circuitBreaker.state = 'HALF_OPEN';
3999
- circuitBreaker.failureCount = 0;
4000
- circuitBreaker.successCount = 0;
4001
- log('Circuit breaker: OPEN → HALF_OPEN', 'warning');
4002
- } else {
4003
- log('Circuit breaker is OPEN, waiting...', 'warning');
4004
- await sleep(500);
4005
- continue;
4006
- }
4007
- }
4008
-
4009
- // Try to process message
4010
- const result = await client.pop({ queue, batch: 1 });
4011
-
4012
- if (result.messages && result.messages.length > 0) {
4013
- const msg = result.messages[0];
4014
-
4015
- try {
4016
- if (msg.data.shouldFail && circuitBreaker.state !== 'OPEN') {
4017
- throw new Error('Simulated failure');
4018
- }
4019
-
4020
- // Success
4021
- await client.ack(msg.transactionId, 'completed');
4022
- processedCount++;
4023
-
4024
- if (circuitBreaker.state === 'HALF_OPEN') {
4025
- circuitBreaker.successCount++;
4026
- if (circuitBreaker.successCount >= circuitBreaker.successThreshold) {
4027
- circuitBreaker.state = 'CLOSED';
4028
- circuitBreaker.failureCount = 0;
4029
- log('Circuit breaker: HALF_OPEN → CLOSED', 'success');
4030
- }
4031
- }
4032
- } catch (error) {
4033
- // Failure
4034
- await client.ack(msg.transactionId, 'failed', error.message);
4035
- circuitBreaker.failureCount++;
4036
- circuitBreaker.lastFailureTime = Date.now();
4037
-
4038
- if (circuitBreaker.state === 'CLOSED' &&
4039
- circuitBreaker.failureCount >= circuitBreaker.failureThreshold) {
4040
- circuitBreaker.state = 'OPEN';
4041
- log('Circuit breaker: CLOSED → OPEN (threshold reached)', 'error');
4042
- } else if (circuitBreaker.state === 'HALF_OPEN') {
4043
- circuitBreaker.state = 'OPEN';
4044
- log('Circuit breaker: HALF_OPEN → OPEN (failure during recovery)', 'error');
4045
- }
4046
- }
4047
- } else {
4048
- await sleep(100);
4049
- }
4050
- }
4051
-
4052
- log(`Circuit breaker results: ${processedCount - circuitBreaker.failureCount} success, ${circuitBreaker.failureCount} failed`, 'info');
4053
- passTest('Circuit breaker pattern implemented successfully');
4054
- } catch (error) {
4055
- failTest(error);
4056
- }
4057
- }
4058
-
4059
- /**
4060
- * Test: Multiple Consumers - Single Partition Message Ordering
4061
- * Verifies that multiple consumers consuming from a single partition
4062
- * maintain message ordering when processing messages
4063
- */
4064
- async function testMultipleConsumersSinglePartitionOrdering() {
4065
- startTest('Multiple Consumers - Single Partition Message Ordering', 'pattern');
4066
-
4067
- try {
4068
- const queue = 'test-multi-consumer-ordering';
4069
- const partition = 'single-partition';
4070
- const numMessages = 999;
4071
- const numConsumers = 10;
4072
- const batchSize = 50; // Each consumer will process messages in batches
4073
-
4074
- // Configure queue with single partition
4075
- await client.configure({
4076
- queue,
4077
- options: {
4078
- leaseTime: 30, // 30 seconds lease time
4079
- retryLimit: 3
4080
- }
4081
- });
4082
-
4083
- log(`Pushing ${numMessages} messages to queue '${queue}', partition '${partition}'`);
4084
-
4085
- // Push messages with sequential order identifiers
4086
- const pushStartTime = Date.now();
4087
- const items = Array.from({ length: numMessages }, (_, i) => ({
4088
- queue,
4089
- partition,
4090
- payload: {
4091
- sequenceNumber: i,
4092
- data: `Message ${i}`,
4093
- timestamp: Date.now()
4094
- }
4095
- }));
4096
-
4097
- // Push in batches to avoid overwhelming the system
4098
- const pushBatchSize = 100;
4099
- for (let i = 0; i < items.length; i += pushBatchSize) {
4100
- const batch = items.slice(i, Math.min(i + pushBatchSize, items.length));
4101
- await client.push({ items: batch });
4102
- }
4103
-
4104
- const pushTime = Date.now() - pushStartTime;
4105
- log(`Pushed ${numMessages} messages in ${pushTime}ms`);
4106
-
4107
- // Wait a moment for messages to be fully persisted
4108
- await sleep(500);
4109
-
4110
- // Verify messages are in the queue
4111
- const messageCount = await getMessageCount(queue, partition);
4112
- if (messageCount !== numMessages) {
4113
- throw new Error(`Expected ${numMessages} messages in queue, but found ${messageCount}`);
4114
- }
4115
-
4116
- log(`Starting ${numConsumers} consumers to process messages`);
4117
-
4118
- // Track all consumed messages and their order
4119
- const consumedMessages = [];
4120
- const consumerPromises = [];
4121
-
4122
- // Create multiple consumers
4123
- for (let consumerId = 0; consumerId < numConsumers; consumerId++) {
4124
- const consumerPromise = (async () => {
4125
- const consumerMessages = [];
4126
- let consecutiveEmptyPolls = 0;
4127
- const maxEmptyPolls = 3; // Stop after 3 consecutive empty polls
4128
-
4129
- while (consecutiveEmptyPolls < maxEmptyPolls) {
4130
- try {
4131
- // Pop messages in batches
4132
- const result = await client.pop({
4133
- queue,
4134
- partition,
4135
- batch: batchSize
4136
- });
4137
-
4138
- if (!result.messages || result.messages.length === 0) {
4139
- // No more messages available
4140
- consecutiveEmptyPolls++;
4141
- await sleep(100);
4142
- continue;
4143
- }
4144
-
4145
- // Reset empty poll counter when we get messages
4146
- consecutiveEmptyPolls = 0;
4147
-
4148
- // Process messages and track their sequence numbers
4149
- for (const msg of result.messages) {
4150
- const sequenceNumber = msg.data.sequenceNumber;
4151
- consumerMessages.push({
4152
- consumerId,
4153
- sequenceNumber,
4154
- timestamp: Date.now(),
4155
- transactionId: msg.transactionId
4156
- });
4157
-
4158
- // Acknowledge message as completed
4159
- await client.ack(msg.transactionId, 'completed');
4160
- }
4161
-
4162
- log(`Consumer ${consumerId} processed batch of ${result.messages.length} messages (total: ${consumerMessages.length})`);
4163
-
4164
- } catch (error) {
4165
- log(`Consumer ${consumerId} error: ${error.message}`, 'warning');
4166
- await sleep(500);
4167
- }
4168
- }
4169
-
4170
- if (consumerMessages.length > 0) {
4171
- log(`Consumer ${consumerId} completed with ${consumerMessages.length} total messages`);
4172
- }
4173
-
4174
- return consumerMessages;
4175
- })();
4176
-
4177
- consumerPromises.push(consumerPromise);
4178
-
4179
- // Stagger consumer starts slightly to simulate realistic scenario
4180
- await sleep(50);
4181
- }
4182
-
4183
- // Wait for all consumers to complete
4184
- log('Waiting for all consumers to complete processing...');
4185
- const consumerResults = await Promise.all(consumerPromises);
4186
-
4187
- // Combine all consumed messages
4188
- for (const messages of consumerResults) {
4189
- consumedMessages.push(...messages);
4190
- }
4191
-
4192
- // Sort by sequence number to check ordering
4193
- consumedMessages.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
4194
-
4195
- log(`Total messages consumed: ${consumedMessages.length}`);
4196
-
4197
- // Verify all messages were consumed exactly once
4198
- if (consumedMessages.length !== numMessages) {
4199
- throw new Error(`Expected ${numMessages} messages to be consumed, but got ${consumedMessages.length}`);
4200
- }
4201
-
4202
- // Verify no duplicates
4203
- const sequenceNumbers = new Set(consumedMessages.map(m => m.sequenceNumber));
4204
- if (sequenceNumbers.size !== numMessages) {
4205
- throw new Error(`Found duplicate messages: expected ${numMessages} unique sequences, got ${sequenceNumbers.size}`);
4206
- }
4207
-
4208
- // Verify sequential ordering (all messages from 0 to numMessages-1 are present)
4209
- for (let i = 0; i < numMessages; i++) {
4210
- if (consumedMessages[i].sequenceNumber !== i) {
4211
- throw new Error(`Message ordering violation at position ${i}: expected sequence ${i}, got ${consumedMessages[i].sequenceNumber}`);
4212
- }
4213
- }
4214
-
4215
- // Analyze consumer distribution
4216
- const consumerStats = {};
4217
- for (const msg of consumedMessages) {
4218
- consumerStats[msg.consumerId] = (consumerStats[msg.consumerId] || 0) + 1;
4219
- }
4220
-
4221
- log('Consumer distribution:', 'info');
4222
- for (const [consumerId, count] of Object.entries(consumerStats)) {
4223
- log(` Consumer ${consumerId}: ${count} messages (${(count / numMessages * 100).toFixed(1)}%)`, 'info');
4224
- }
4225
-
4226
- // Check that messages consumed by each consumer maintain relative ordering
4227
- for (let consumerId = 0; consumerId < numConsumers; consumerId++) {
4228
- const consumerMsgs = consumedMessages
4229
- .filter(m => m.consumerId === consumerId)
4230
- .map(m => m.sequenceNumber);
4231
-
4232
- // Verify that sequence numbers for this consumer are in ascending order
4233
- for (let i = 1; i < consumerMsgs.length; i++) {
4234
- if (consumerMsgs[i] <= consumerMsgs[i - 1]) {
4235
- throw new Error(`Consumer ${consumerId} violated ordering: sequence ${consumerMsgs[i]} came after ${consumerMsgs[i - 1]}`);
4236
- }
4237
- }
4238
- }
4239
-
4240
- // Final verification: check database for any remaining unprocessed messages
4241
- const remainingQuery = `
4242
- SELECT COUNT(*) as count FROM queen.messages m
4243
- JOIN queen.partitions p ON m.partition_id = p.id
4244
- JOIN queen.queues q ON p.queue_id = q.id
4245
- LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
4246
- WHERE q.name = $1 AND p.name = $2
4247
- AND (ms.status IS NULL OR ms.status IN ('pending', 'processing', 'failed'))
4248
- `;
4249
- const remainingResult = await dbPool.query(remainingQuery, [queue, partition]);
4250
- const remainingMessages = parseInt(remainingResult.rows[0].count);
4251
-
4252
- if (remainingMessages > 0) {
4253
- log(`Warning: ${remainingMessages} unprocessed messages still in queue after test`, 'warning');
4254
- }
4255
-
4256
- passTest(`${numConsumers} consumers successfully processed ${numMessages} messages from single partition with correct ordering`);
4257
-
4258
- } catch (error) {
4259
- failTest(error);
4260
- }
4261
- }
4262
-
4263
- // ============================================
4264
- // MAIN TEST RUNNER
4265
- // ============================================
4266
-
4267
- async function runTestsInParallel(tests, category, maxConcurrent = 3) {
4268
- console.log(`\n${category}`);
4269
- console.log('-' .repeat(40));
4270
-
4271
- const results = [];
4272
- const executing = [];
4273
-
4274
- for (const test of tests) {
4275
- const promise = test().catch(error => {
4276
- log(`Unexpected error in test: ${error.message}`, 'error');
4277
- return { error };
4278
- });
4279
-
4280
- results.push(promise);
4281
-
4282
- if (tests.length >= maxConcurrent) {
4283
- executing.push(promise);
4284
-
4285
- if (executing.length >= maxConcurrent) {
4286
- await Promise.race(executing);
4287
- executing.splice(executing.findIndex(p => p === promise), 1);
4288
- }
4289
- }
4290
-
4291
- // Small delay between starting tests to avoid overwhelming the system
4292
- await sleep(50);
4293
- }
4294
-
4295
- await Promise.all(results);
4296
- }
4297
-
4298
- async function runAllTests() {
4299
- console.log('🚀 Starting Complete Queen Message Queue Test Suite');
4300
- console.log(' Including Core, Enterprise, Bus Mode, Edge Cases, and Advanced Scenarios\n');
4301
- const parallelMode = process.env.PARALLEL_TESTS === 'true';
4302
- console.log(` Running in ${parallelMode ? 'PARALLEL' : 'SEQUENTIAL'} mode`);
4303
- console.log('=' .repeat(80));
4304
-
4305
- try {
4306
- // Initialize client and database connection
4307
- client = createQueenClient({ baseUrls: TEST_CONFIG.baseUrls });
4308
- dbPool = new pg.Pool(TEST_CONFIG.dbConfig);
4309
-
4310
- // Test database connection
4311
- await dbPool.query('SELECT 1');
4312
- log('Database connection established');
4313
-
4314
- // Check for encryption key
4315
- if (process.env.QUEEN_ENCRYPTION_KEY) {
4316
- log('Encryption key detected - encryption tests will run', 'info');
4317
- } else {
4318
- log('No encryption key found - encryption tests will be skipped', 'warning');
4319
- log('Set QUEEN_ENCRYPTION_KEY env variable to test encryption', 'info');
4320
- }
4321
-
4322
- // Clean up any existing test data
4323
- await cleanupTestData();
4324
-
4325
- // Core feature tests
4326
- const coreTests = [
4327
- testQueueCreationPolicy,
4328
- testSingleMessagePush,
4329
- testBatchMessagePush,
4330
- testQueueConfiguration,
4331
- testPopAndAcknowledgment,
4332
- testDelayedProcessing,
4333
- testPartitionPriorityOrdering,
4334
- testPartitionLocking,
4335
- testBusPartitionLocking,
4336
- testSpecificPartitionLocking,
4337
- testNamespaceTaskFiltering,
4338
- testNamespaceTaskBusMode
4339
- ];
4340
-
4341
- // Enterprise feature tests
4342
- const enterpriseTests = [
4343
- testMessageEncryption,
4344
- testRetentionPendingMessages,
4345
- testRetentionCompletedMessages,
4346
- testPartitionRetention,
4347
- testMessageEviction,
4348
- testCombinedEnterpriseFeatures,
4349
- testConsumerEncryption,
4350
- testEnterpriseErrorHandling
4351
- ];
4352
-
4353
- // Bus mode tests
4354
- const busTests = [
4355
- testBusConsumerGroups,
4356
- testMixedMode,
4357
- testConsumerGroupSubscriptionModes,
4358
- testConsumerGroupIsolation
4359
- ];
4360
-
4361
- // Edge case tests
4362
- const edgeTests = [
4363
- testEmptyAndNullPayloads,
4364
- testVeryLargePayloads,
4365
- testConcurrentPushOperations,
4366
- testConcurrentPopOperations,
4367
- testManyConsumerGroups,
4368
- testRetryLimitExhaustion,
4369
- testLeaseExpiration,
4370
- testSQLInjectionPrevention,
4371
- testXSSPrevention
4372
- ];
4373
-
4374
- // Advanced scenario tests - split into smaller groups for parallel execution
4375
- const advancedTests1 = [
4376
- testMultiStagePipeline,
4377
- testFanOutFanIn,
4378
- testComplexPriorityScenarios,
4379
- testDynamicPriorityAdjustment
4380
- ];
4381
-
4382
- const advancedTests2 = [
4383
- testDeadLetterQueuePattern,
4384
- testSagaPattern,
4385
- testRateLimiting,
4386
- testMessageDeduplication
4387
- ];
4388
-
4389
- const advancedTests3 = [
4390
- testTimeBatchProcessing,
4391
- testEventSourcing,
4392
- //testQueueMetrics,
4393
- testCircuitBreaker,
4394
- testMultipleConsumersSinglePartitionOrdering
4395
- ];
4396
-
4397
- const allTests = [...coreTests, ...enterpriseTests, ...busTests, ...edgeTests,
4398
- ...advancedTests1, ...advancedTests2, ...advancedTests3];
4399
-
4400
- log(`Running ${allTests.length} total tests...`);
4401
- console.log('=' .repeat(80));
4402
-
4403
- if (parallelMode) {
4404
- // Run tests in parallel with controlled concurrency
4405
- await runTestsInParallel(coreTests, '📦 CORE FEATURES', 2);
4406
- await runTestsInParallel(enterpriseTests, '🏢 ENTERPRISE FEATURES', 2);
4407
- await runTestsInParallel(busTests, '🚌 BUS MODE FEATURES', 2);
4408
- await runTestsInParallel(edgeTests, '🔍 EDGE CASES', 3);
4409
- await runTestsInParallel(advancedTests1, '🎯 ADVANCED SCENARIOS (Part 1)', 2);
4410
- await runTestsInParallel(advancedTests2, '🎯 ADVANCED SCENARIOS (Part 2)', 2);
4411
- await runTestsInParallel(advancedTests3, '🎯 ADVANCED SCENARIOS (Part 3)', 2);
4412
- } else {
4413
- // Original sequential execution
4414
- console.log('\n📦 CORE FEATURES');
4415
- console.log('-' .repeat(40));
4416
- for (const test of coreTests) {
4417
- try {
4418
- await test();
4419
- } catch (error) {
4420
- log(`Unexpected error in test: ${error.message}`, 'error');
4421
- }
4422
- await sleep(100);
4423
- }
4424
-
4425
- console.log('\n🏢 ENTERPRISE FEATURES');
4426
- console.log('-' .repeat(40));
4427
- for (const test of enterpriseTests) {
4428
- try {
4429
- await test();
4430
- } catch (error) {
4431
- log(`Unexpected error in test: ${error.message}`, 'error');
4432
- }
4433
- await sleep(100);
4434
- }
4435
-
4436
- console.log('\n🚌 BUS MODE FEATURES');
4437
- console.log('-' .repeat(40));
4438
- for (const test of busTests) {
4439
- try {
4440
- await test();
4441
- } catch (error) {
4442
- log(`Unexpected error in test: ${error.message}`, 'error');
4443
- }
4444
- await sleep(100);
4445
- }
4446
-
4447
- console.log('\n🔍 EDGE CASES');
4448
- console.log('-' .repeat(40));
4449
- for (const test of edgeTests) {
4450
- try {
4451
- await test();
4452
- } catch (error) {
4453
- log(`Unexpected error in test: ${error.message}`, 'error');
4454
- }
4455
- await sleep(100);
4456
- }
4457
-
4458
- console.log('\n🎯 ADVANCED SCENARIOS');
4459
- console.log('-' .repeat(40));
4460
- for (const test of [...advancedTests1, ...advancedTests2, ...advancedTests3]) {
4461
- try {
4462
- await test();
4463
- } catch (error) {
4464
- log(`Unexpected error in test: ${error.message}`, 'error');
4465
- }
4466
- await sleep(100);
4467
- }
4468
- }
4469
-
4470
- } catch (error) {
4471
- log(`Test suite setup failed: ${error.message}`, 'error');
4472
- } finally {
4473
- // Cleanup
4474
- if (dbPool) {
4475
- await cleanupTestData();
4476
- await dbPool.end();
4477
- }
4478
- }
4479
-
4480
- // Print results
4481
- console.log('\n' + '=' .repeat(80));
4482
- console.log('📊 TEST RESULTS SUMMARY');
4483
- console.log('=' .repeat(80));
4484
-
4485
- const passed = testResults.filter(r => r.status === 'PASS').length;
4486
- const failed = testResults.filter(r => r.status === 'FAIL').length;
4487
- const total = testResults.length;
4488
-
4489
- console.log(`\n📈 Overall Results: ${passed}/${total} tests passed`);
4490
-
4491
- if (failed > 0) {
4492
- console.log('\n❌ Failed Tests:');
4493
- testResults
4494
- .filter(r => r.status === 'FAIL')
4495
- .forEach(r => {
4496
- console.log(` • ${r.test}: ${r.error}`);
4497
- });
4498
- }
4499
-
4500
- if (passed > 0) {
4501
- console.log('\n✅ Passed Tests:');
4502
- testResults
4503
- .filter(r => r.status === 'PASS')
4504
- .forEach(r => {
4505
- console.log(` • ${r.test}${r.message ? ': ' + r.message : ''}`);
4506
- });
4507
- }
4508
-
4509
- console.log('\n' + '=' .repeat(80));
4510
-
4511
- if (failed === 0) {
4512
- console.log('🎉 ALL TESTS PASSED! Queen Message Queue System is working correctly.');
4513
- console.log(' All features are operational.');
4514
- } else {
4515
- console.log(`⚠️ ${failed} test(s) failed. Please review the failures above.`);
4516
- process.exit(1);
4517
- }
4518
- }
4519
-
4520
- // Run the test suite
4521
- runAllTests().catch(error => {
4522
- console.error('💥 Test suite crashed:', error);
4523
- process.exit(1);
4524
- });