queen-mq 0.2.0 → 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +186 -763
- package/{src → client-js}/benchmark/consumer.js +6 -4
- package/{src → client-js}/benchmark/producer.js +9 -4
- package/{src → client-js}/client/client.js +277 -7
- package/{src → client-js}/client/index.js +1 -4
- package/{src → client-js}/client/utils/http.js +3 -2
- package/{src → client-js}/client/utils/retry.js +7 -1
- package/{src → client-js}/test/README.md +30 -9
- package/{src → client-js}/test/core-tests.js +7 -2
- package/{src → client-js}/test/edge-case-tests.js +2 -1
- package/client-js/test/human.js +162 -0
- package/client-js/test/partition-transaction-tests.js +482 -0
- package/client-js/test/qos0-tests.js +334 -0
- package/{src → client-js}/test/test-new.js +55 -0
- package/package.json +8 -17
- package/init-db.js +0 -20
- package/src/cluster-server.js +0 -242
- package/src/config.js +0 -229
- package/src/database/connection.js +0 -129
- package/src/database/poolManager.js +0 -199
- package/src/database/schema-v2.sql +0 -278
- package/src/managers/eventManager.js +0 -59
- package/src/managers/queueManagerOptimized.js +0 -1634
- package/src/managers/resourceCache.js +0 -96
- package/src/managers/systemEventManager.js +0 -132
- package/src/routes/ack.js +0 -26
- package/src/routes/configure.js +0 -46
- package/src/routes/messages.js +0 -368
- package/src/routes/pop.js +0 -69
- package/src/routes/push.js +0 -28
- package/src/routes/resources.js +0 -330
- package/src/routes/status.js +0 -1037
- package/src/server.js +0 -1633
- package/src/test/MIGRATION_ISSUES.md +0 -174
- package/src/test/test.js +0 -4524
- package/src/utils/streaming.js +0 -231
- package/src/webapp-dist/assets/Analytics-DJrE4Rz2.css +0 -1
- package/src/webapp-dist/assets/Analytics-HEMXKmfJ.js +0 -1
- package/src/webapp-dist/assets/ConfirmDialog-YEoCdE7x.js +0 -1
- package/src/webapp-dist/assets/ConsumerGroups-CC5VJywp.css +0 -1
- package/src/webapp-dist/assets/ConsumerGroups-D8Z20HhZ.js +0 -1
- package/src/webapp-dist/assets/Dashboard-BRFOz6it.css +0 -1
- package/src/webapp-dist/assets/Dashboard-Bpx9I32h.js +0 -1
- package/src/webapp-dist/assets/LoadingSpinner-BPttXcbs.js +0 -1
- package/src/webapp-dist/assets/Messages-BOfVHKma.css +0 -1
- package/src/webapp-dist/assets/Messages-EKURCbLl.js +0 -1
- package/src/webapp-dist/assets/QueueDetail-BXXA7xoZ.css +0 -1
- package/src/webapp-dist/assets/QueueDetail-Bmxg0KiA.js +0 -1
- package/src/webapp-dist/assets/Queues-CvrJkrZT.css +0 -1
- package/src/webapp-dist/assets/Queues-yUx9_3-9.js +0 -1
- package/src/webapp-dist/assets/StatusBadge-C1bBtXHh.css +0 -1
- package/src/webapp-dist/assets/StatusBadge-C6N84XmO.js +0 -1
- package/src/webapp-dist/assets/analytics-BapTekLM.js +0 -1
- package/src/webapp-dist/assets/colors-5z6Q_kUr.js +0 -18
- package/src/webapp-dist/assets/index-B7w2bKT6.css +0 -1
- package/src/webapp-dist/assets/index-CUSnwYTH.js +0 -31
- package/src/webapp-dist/assets/messages-CRV6E-xU.js +0 -1
- package/src/webapp-dist/assets/queen-logo-blue.svg +0 -210
- package/src/webapp-dist/assets/queen-logo-cyan.svg +0 -210
- package/src/webapp-dist/assets/queen-logo-indigo.svg +0 -210
- package/src/webapp-dist/assets/queen-logo-orange.svg +0 -210
- package/src/webapp-dist/assets/queen-logo-pink.svg +0 -210
- package/src/webapp-dist/assets/queen-logo-purple.svg +0 -210
- package/src/webapp-dist/assets/queen-logo-rose.svg +0 -239
- package/src/webapp-dist/assets/queen-logo.svg +0 -263
- package/src/webapp-dist/assets/queues-BdUbxfBC.js +0 -1
- package/src/webapp-dist/assets/resources-Bs9U-ceF.js +0 -1
- package/src/webapp-dist/index.html +0 -15
- package/src/websocket/wsServer.js +0 -228
- /package/{src → client-js}/benchmark/consumer_multi.js +0 -0
- /package/{src → client-js}/benchmark/producer_multi.js +0 -0
- /package/{src → client-js}/client/utils/loadBalancer.js +0 -0
- /package/{src → client-js}/services/encryptionService.js +0 -0
- /package/{src → client-js}/services/evictionService.js +0 -0
- /package/{src → client-js}/services/retentionService.js +0 -0
- /package/{src → client-js}/services/startupSync.js +0 -0
- /package/{src → client-js}/test/advanced-client-tests.js +0 -0
- /package/{src → client-js}/test/advanced-pattern-tests.js +0 -0
- /package/{src → client-js}/test/bus-mode-tests.js +0 -0
- /package/{src → client-js}/test/enterprise-tests.js +0 -0
- /package/{src → client-js}/test/partition-locking-tests.js +0 -0
- /package/{src → client-js}/test/utils.js +0 -0
- /package/{src → client-js}/test/window-buffer-test.js +0 -0
- /package/{src → client-js}/utils/logger.js +0 -0
- /package/{src → client-js}/utils/uuid.js +0 -0
package/src/test/test.js
DELETED
|
@@ -1,4524 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Comprehensive Test Suite for Queen Message Queue System
|
|
5
|
-
* Consolidated from all test files
|
|
6
|
-
*
|
|
7
|
-
* Test Categories:
|
|
8
|
-
* 1. Core Features
|
|
9
|
-
* 2. Enterprise Features
|
|
10
|
-
* 3. Bus Mode Features
|
|
11
|
-
* 4. Edge Cases
|
|
12
|
-
* 5. Advanced Scenarios & Patterns
|
|
13
|
-
*/
|
|
14
|
-
|
|
15
|
-
import { createQueenClient } from '../client/queenClient.js';
|
|
16
|
-
import pg from 'pg';
|
|
17
|
-
import crypto from 'crypto';
|
|
18
|
-
|
|
19
|
-
// Test configuration
|
|
20
|
-
const TEST_CONFIG = {
|
|
21
|
-
baseUrls: ['http://localhost:6632', 'http://localhost:6633'],
|
|
22
|
-
dbConfig: {
|
|
23
|
-
host: process.env.PG_HOST || 'localhost',
|
|
24
|
-
port: process.env.PG_PORT || 5432,
|
|
25
|
-
database: process.env.PG_DB || 'postgres',
|
|
26
|
-
user: process.env.PG_USER || 'postgres',
|
|
27
|
-
password: process.env.PG_PASSWORD || 'postgres'
|
|
28
|
-
}
|
|
29
|
-
};
|
|
30
|
-
|
|
31
|
-
// Global test state
|
|
32
|
-
let client;
|
|
33
|
-
let dbPool;
|
|
34
|
-
let testResults = [];
|
|
35
|
-
let currentTest = '';
|
|
36
|
-
|
|
37
|
-
// Test utilities
|
|
38
|
-
const log = (message, type = 'info') => {
|
|
39
|
-
const timestamp = new Date().toISOString().substring(11, 23);
|
|
40
|
-
const prefix = {
|
|
41
|
-
info: '📝',
|
|
42
|
-
success: '✅',
|
|
43
|
-
error: '❌',
|
|
44
|
-
warning: '⚠️',
|
|
45
|
-
test: '🧪',
|
|
46
|
-
enterprise: '🏢',
|
|
47
|
-
edge: '🔍',
|
|
48
|
-
pattern: '🎯',
|
|
49
|
-
workflow: '🔄',
|
|
50
|
-
priority: '⚡'
|
|
51
|
-
}[type] || '📝';
|
|
52
|
-
|
|
53
|
-
console.log(`[${timestamp}] ${prefix} ${message}`);
|
|
54
|
-
};
|
|
55
|
-
|
|
56
|
-
const startTest = (testName, category = 'test') => {
|
|
57
|
-
currentTest = testName;
|
|
58
|
-
log(`Starting: ${testName}`, category);
|
|
59
|
-
};
|
|
60
|
-
|
|
61
|
-
const passTest = (message = '') => {
|
|
62
|
-
const result = { test: currentTest, status: 'PASS', message };
|
|
63
|
-
testResults.push(result);
|
|
64
|
-
log(`PASS: ${currentTest} ${message}`, 'success');
|
|
65
|
-
};
|
|
66
|
-
|
|
67
|
-
const failTest = (error) => {
|
|
68
|
-
const result = { test: currentTest, status: 'FAIL', error: error.message };
|
|
69
|
-
testResults.push(result);
|
|
70
|
-
log(`FAIL: ${currentTest} - ${error.message}`, 'error');
|
|
71
|
-
};
|
|
72
|
-
|
|
73
|
-
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
74
|
-
|
|
75
|
-
// Database utilities
|
|
76
|
-
const cleanupTestData = async () => {
|
|
77
|
-
try {
|
|
78
|
-
await dbPool.query(`
|
|
79
|
-
DELETE FROM queen.messages
|
|
80
|
-
WHERE partition_id IN (
|
|
81
|
-
SELECT p.id FROM queen.partitions p
|
|
82
|
-
JOIN queen.queues q ON p.queue_id = q.id
|
|
83
|
-
WHERE q.name LIKE 'test-%' OR q.name LIKE 'edge-%' OR q.name LIKE 'pattern-%' OR q.name LIKE 'workflow-%'
|
|
84
|
-
)
|
|
85
|
-
`);
|
|
86
|
-
|
|
87
|
-
await dbPool.query(`
|
|
88
|
-
DELETE FROM queen.partitions
|
|
89
|
-
WHERE queue_id IN (
|
|
90
|
-
SELECT id FROM queen.queues WHERE name LIKE 'test-%' OR name LIKE 'edge-%' OR name LIKE 'pattern-%' OR name LIKE 'workflow-%'
|
|
91
|
-
)
|
|
92
|
-
`);
|
|
93
|
-
|
|
94
|
-
await dbPool.query(`DELETE FROM queen.queues WHERE name LIKE 'test-%' OR name LIKE 'edge-%' OR name LIKE 'pattern-%' OR name LIKE 'workflow-%'`);
|
|
95
|
-
|
|
96
|
-
// Also clean up retention history for test partitions
|
|
97
|
-
await dbPool.query(`
|
|
98
|
-
DELETE FROM queen.retention_history
|
|
99
|
-
WHERE partition_id NOT IN (SELECT id FROM queen.partitions)
|
|
100
|
-
`);
|
|
101
|
-
|
|
102
|
-
log('Test data cleaned up');
|
|
103
|
-
} catch (error) {
|
|
104
|
-
log(`Cleanup error: ${error.message}`, 'warning');
|
|
105
|
-
}
|
|
106
|
-
};
|
|
107
|
-
|
|
108
|
-
const getMessageCount = async (queueName, partitionName = null) => {
|
|
109
|
-
const query = partitionName
|
|
110
|
-
? `SELECT COUNT(*) as count FROM queen.messages m
|
|
111
|
-
JOIN queen.partitions p ON m.partition_id = p.id
|
|
112
|
-
JOIN queen.queues q ON p.queue_id = q.id
|
|
113
|
-
WHERE q.name = $1 AND p.name = $2`
|
|
114
|
-
: `SELECT COUNT(*) as count FROM queen.messages m
|
|
115
|
-
JOIN queen.partitions p ON m.partition_id = p.id
|
|
116
|
-
JOIN queen.queues q ON p.queue_id = q.id
|
|
117
|
-
WHERE q.name = $1`;
|
|
118
|
-
|
|
119
|
-
const params = partitionName ? [queueName, partitionName] : [queueName];
|
|
120
|
-
const result = await dbPool.query(query, params);
|
|
121
|
-
return parseInt(result.rows[0].count);
|
|
122
|
-
};
|
|
123
|
-
|
|
124
|
-
// Helper function to ensure queue exists
|
|
125
|
-
const ensureQueue = async (queueName, options = {}) => {
|
|
126
|
-
try {
|
|
127
|
-
await client.configure({
|
|
128
|
-
queue: queueName,
|
|
129
|
-
options: options
|
|
130
|
-
});
|
|
131
|
-
} catch (error) {
|
|
132
|
-
// Queue might already exist, that's okay
|
|
133
|
-
if (!error.message.includes('already exists')) {
|
|
134
|
-
log(`Warning: Failed to configure queue ${queueName}: ${error.message}`, 'warning');
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
};
|
|
138
|
-
|
|
139
|
-
// ============================================
|
|
140
|
-
// CORE FEATURE TESTS
|
|
141
|
-
// ============================================
|
|
142
|
-
|
|
143
|
-
/**
|
|
144
|
-
* Test 1: Queue Creation Policy
|
|
145
|
-
* Verifies that queues must be created via configure endpoint
|
|
146
|
-
*/
|
|
147
|
-
async function testQueueCreationPolicy() {
|
|
148
|
-
startTest('Queue Creation Policy');
|
|
149
|
-
|
|
150
|
-
try {
|
|
151
|
-
// Test 1: Push to non-existent queue should fail
|
|
152
|
-
const nonExistentQueue = 'queue-that-does-not-exist-' + Date.now();
|
|
153
|
-
try {
|
|
154
|
-
await client.push({
|
|
155
|
-
items: [{
|
|
156
|
-
queue: nonExistentQueue,
|
|
157
|
-
partition: 'Default',
|
|
158
|
-
payload: { test: 'should fail' }
|
|
159
|
-
}]
|
|
160
|
-
});
|
|
161
|
-
throw new Error('Push to non-existent queue should have failed');
|
|
162
|
-
} catch (error) {
|
|
163
|
-
if (!error.message.includes('does not exist')) {
|
|
164
|
-
throw error;
|
|
165
|
-
}
|
|
166
|
-
log('Push to non-existent queue correctly failed', 'success');
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
// Test 2: Create queue via configure
|
|
170
|
-
const testQueue = 'test-creation-policy-' + Date.now();
|
|
171
|
-
await client.configure({
|
|
172
|
-
queue: testQueue,
|
|
173
|
-
namespace: 'test',
|
|
174
|
-
task: 'creation-policy',
|
|
175
|
-
options: {
|
|
176
|
-
retryLimit: 3,
|
|
177
|
-
priority: 5
|
|
178
|
-
}
|
|
179
|
-
});
|
|
180
|
-
log('Queue created via configure endpoint', 'success');
|
|
181
|
-
|
|
182
|
-
// Test 3: Push to configured queue should succeed
|
|
183
|
-
await client.push({
|
|
184
|
-
items: [{
|
|
185
|
-
queue: testQueue,
|
|
186
|
-
partition: 'Default',
|
|
187
|
-
payload: { test: 'message 1' }
|
|
188
|
-
}]
|
|
189
|
-
});
|
|
190
|
-
log('Push to configured queue succeeded', 'success');
|
|
191
|
-
|
|
192
|
-
// Test 4: Push to new partition (should create on-demand)
|
|
193
|
-
await client.push({
|
|
194
|
-
items: [{
|
|
195
|
-
queue: testQueue,
|
|
196
|
-
partition: 'NewPartition',
|
|
197
|
-
payload: { test: 'message 2' }
|
|
198
|
-
}]
|
|
199
|
-
});
|
|
200
|
-
log('Partition created on-demand', 'success');
|
|
201
|
-
|
|
202
|
-
// Clean up
|
|
203
|
-
await client.queues.clear(testQueue);
|
|
204
|
-
|
|
205
|
-
passTest('Queue creation policy working correctly');
|
|
206
|
-
} catch (error) {
|
|
207
|
-
failTest(error);
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
/**
|
|
212
|
-
* Test 2: Single Message Push
|
|
213
|
-
*/
|
|
214
|
-
async function testSingleMessagePush() {
|
|
215
|
-
startTest('Single Message Push');
|
|
216
|
-
|
|
217
|
-
try {
|
|
218
|
-
// Configure queue first
|
|
219
|
-
await client.configure({
|
|
220
|
-
queue: 'test-single-push',
|
|
221
|
-
options: {}
|
|
222
|
-
});
|
|
223
|
-
|
|
224
|
-
const result = await client.push({
|
|
225
|
-
items: [{
|
|
226
|
-
queue: 'test-single-push',
|
|
227
|
-
partition: 'default',
|
|
228
|
-
payload: { message: 'Single test message', timestamp: Date.now() }
|
|
229
|
-
}]
|
|
230
|
-
});
|
|
231
|
-
|
|
232
|
-
if (!result.messages || result.messages.length !== 1) {
|
|
233
|
-
throw new Error('Expected 1 message in response');
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
const message = result.messages[0];
|
|
237
|
-
if (!message.id || !message.transactionId || message.status !== 'queued') {
|
|
238
|
-
throw new Error('Invalid message response format');
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
// Verify message was stored
|
|
242
|
-
const count = await getMessageCount('test-single-push');
|
|
243
|
-
if (count !== 1) {
|
|
244
|
-
throw new Error(`Expected 1 message in database, got ${count}`);
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
passTest('Single message pushed successfully');
|
|
248
|
-
} catch (error) {
|
|
249
|
-
failTest(error);
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
/**
|
|
254
|
-
* Test 3: Batch Message Push
|
|
255
|
-
*/
|
|
256
|
-
async function testBatchMessagePush() {
|
|
257
|
-
startTest('Batch Message Push');
|
|
258
|
-
|
|
259
|
-
try {
|
|
260
|
-
// Configure queue first
|
|
261
|
-
await client.configure({
|
|
262
|
-
queue: 'test-batch-push',
|
|
263
|
-
options: {}
|
|
264
|
-
});
|
|
265
|
-
|
|
266
|
-
const batchSize = 5;
|
|
267
|
-
const items = Array.from({ length: batchSize }, (_, i) => ({
|
|
268
|
-
queue: 'test-batch-push',
|
|
269
|
-
partition: 'batch-partition',
|
|
270
|
-
payload: { message: `Batch message ${i + 1}`, index: i + 1 }
|
|
271
|
-
}));
|
|
272
|
-
|
|
273
|
-
const result = await client.push({ items });
|
|
274
|
-
|
|
275
|
-
if (!result.messages || result.messages.length !== batchSize) {
|
|
276
|
-
throw new Error(`Expected ${batchSize} messages in response, got ${result.messages?.length}`);
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
// Verify all messages have unique transaction IDs
|
|
280
|
-
const transactionIds = result.messages.map(m => m.transactionId);
|
|
281
|
-
const uniqueIds = new Set(transactionIds);
|
|
282
|
-
if (uniqueIds.size !== batchSize) {
|
|
283
|
-
throw new Error('Transaction IDs are not unique');
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
// Verify messages were stored
|
|
287
|
-
const count = await getMessageCount('test-batch-push', 'batch-partition');
|
|
288
|
-
if (count !== batchSize) {
|
|
289
|
-
throw new Error(`Expected ${batchSize} messages in database, got ${count}`);
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
passTest(`Batch of ${batchSize} messages pushed successfully`);
|
|
293
|
-
} catch (error) {
|
|
294
|
-
failTest(error);
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
/**
|
|
299
|
-
* Test 3: Queue Configuration
|
|
300
|
-
*/
|
|
301
|
-
async function testQueueConfiguration() {
|
|
302
|
-
startTest('Queue Configuration');
|
|
303
|
-
|
|
304
|
-
try {
|
|
305
|
-
const configResult = await client.configure({
|
|
306
|
-
queue: 'test-config-queue',
|
|
307
|
-
options: {
|
|
308
|
-
leaseTime: 600,
|
|
309
|
-
retryLimit: 5,
|
|
310
|
-
priority: 8,
|
|
311
|
-
delayedProcessing: 2,
|
|
312
|
-
windowBuffer: 3
|
|
313
|
-
}
|
|
314
|
-
});
|
|
315
|
-
|
|
316
|
-
if (!configResult.configured) {
|
|
317
|
-
throw new Error('Configuration failed');
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
if (configResult.queue !== 'test-config-queue') {
|
|
321
|
-
throw new Error('Configuration response has wrong queue name');
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
// Verify configuration was applied by checking database
|
|
325
|
-
const result = await dbPool.query(`
|
|
326
|
-
SELECT lease_time, retry_limit, priority, delayed_processing, window_buffer
|
|
327
|
-
FROM queen.queues
|
|
328
|
-
WHERE name = 'test-config-queue'
|
|
329
|
-
`);
|
|
330
|
-
|
|
331
|
-
if (result.rows.length === 0) {
|
|
332
|
-
throw new Error('Configured queue not found in database');
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
const row = result.rows[0];
|
|
336
|
-
if (row.lease_time !== 600 || row.retry_limit !== 5) {
|
|
337
|
-
throw new Error('Configuration options not saved correctly');
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
passTest('Queue configuration applied successfully');
|
|
341
|
-
} catch (error) {
|
|
342
|
-
failTest(error);
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
/**
|
|
347
|
-
* Test 4: Pop and Acknowledgment
|
|
348
|
-
*/
|
|
349
|
-
async function testPopAndAcknowledgment() {
|
|
350
|
-
startTest('Pop and Acknowledgment');
|
|
351
|
-
|
|
352
|
-
try {
|
|
353
|
-
// Configure queue first
|
|
354
|
-
await client.configure({
|
|
355
|
-
queue: 'test-pop-ack',
|
|
356
|
-
options: {}
|
|
357
|
-
});
|
|
358
|
-
|
|
359
|
-
// Push test messages
|
|
360
|
-
const pushResult = await client.push({
|
|
361
|
-
items: [
|
|
362
|
-
{
|
|
363
|
-
queue: 'test-pop-ack',
|
|
364
|
-
partition: 'ack-partition',
|
|
365
|
-
payload: { message: 'Message to complete' }
|
|
366
|
-
},
|
|
367
|
-
{
|
|
368
|
-
queue: 'test-pop-ack',
|
|
369
|
-
partition: 'ack-partition',
|
|
370
|
-
payload: { message: 'Message to fail' }
|
|
371
|
-
}
|
|
372
|
-
]
|
|
373
|
-
});
|
|
374
|
-
|
|
375
|
-
await sleep(100);
|
|
376
|
-
|
|
377
|
-
// Pop messages
|
|
378
|
-
const popResult = await client.pop({
|
|
379
|
-
queue: 'test-pop-ack',
|
|
380
|
-
partition: 'ack-partition',
|
|
381
|
-
batch: 2
|
|
382
|
-
});
|
|
383
|
-
|
|
384
|
-
if (!popResult.messages || popResult.messages.length !== 2) {
|
|
385
|
-
throw new Error('Expected 2 messages from pop');
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
// Acknowledge first as completed
|
|
389
|
-
const completeResult = await client.ack(
|
|
390
|
-
popResult.messages[0].transactionId,
|
|
391
|
-
'completed'
|
|
392
|
-
);
|
|
393
|
-
|
|
394
|
-
if (!completeResult || !completeResult.acknowledgedAt) {
|
|
395
|
-
console.log('Complete ACK result:', completeResult);
|
|
396
|
-
throw new Error('Failed to acknowledge message as completed');
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
// Acknowledge second as failed
|
|
400
|
-
const failResult = await client.ack(
|
|
401
|
-
popResult.messages[1].transactionId,
|
|
402
|
-
'failed',
|
|
403
|
-
'Test failure'
|
|
404
|
-
);
|
|
405
|
-
|
|
406
|
-
if (!failResult || !failResult.acknowledgedAt) {
|
|
407
|
-
console.log('Fail ACK result:', failResult);
|
|
408
|
-
console.log('Transaction ID:', popResult.messages[1].transactionId);
|
|
409
|
-
throw new Error('Failed to acknowledge message as failed');
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
passTest('Pop and acknowledgment work correctly');
|
|
413
|
-
} catch (error) {
|
|
414
|
-
failTest(error);
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
/**
|
|
419
|
-
* Test 5: Delayed Processing
|
|
420
|
-
*/
|
|
421
|
-
async function testDelayedProcessing() {
|
|
422
|
-
startTest('Delayed Processing');
|
|
423
|
-
|
|
424
|
-
try {
|
|
425
|
-
// Configure queue with delayed processing
|
|
426
|
-
await client.configure({
|
|
427
|
-
queue: 'test-delayed-queue',
|
|
428
|
-
options: {
|
|
429
|
-
delayedProcessing: 2 // 2 seconds delay
|
|
430
|
-
}
|
|
431
|
-
});
|
|
432
|
-
|
|
433
|
-
const startTime = Date.now();
|
|
434
|
-
|
|
435
|
-
// Push message
|
|
436
|
-
await client.push({
|
|
437
|
-
items: [{
|
|
438
|
-
queue: 'test-delayed-queue',
|
|
439
|
-
payload: { message: 'Delayed message', sentAt: startTime }
|
|
440
|
-
}]
|
|
441
|
-
});
|
|
442
|
-
|
|
443
|
-
// Try to pop immediately (should get nothing)
|
|
444
|
-
const immediateResult = await client.pop({
|
|
445
|
-
queue: 'test-delayed-queue',
|
|
446
|
-
batch: 1
|
|
447
|
-
});
|
|
448
|
-
|
|
449
|
-
if (immediateResult.messages && immediateResult.messages.length > 0) {
|
|
450
|
-
throw new Error('Got message immediately when it should be delayed');
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
// Wait for delay period plus buffer
|
|
454
|
-
await sleep(2500);
|
|
455
|
-
|
|
456
|
-
// Try to pop again (should get the message now)
|
|
457
|
-
const delayedResult = await client.pop({
|
|
458
|
-
queue: 'test-delayed-queue',
|
|
459
|
-
batch: 1
|
|
460
|
-
});
|
|
461
|
-
|
|
462
|
-
if (!delayedResult.messages || delayedResult.messages.length !== 1) {
|
|
463
|
-
throw new Error('Did not get delayed message after delay period');
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
const processingDelay = Date.now() - startTime;
|
|
467
|
-
if (processingDelay < 2000) {
|
|
468
|
-
throw new Error(`Message processed too early: ${processingDelay}ms`);
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
await client.ack(delayedResult.messages[0].transactionId, 'completed');
|
|
472
|
-
|
|
473
|
-
passTest(`Delayed processing works correctly (${processingDelay}ms delay)`);
|
|
474
|
-
} catch (error) {
|
|
475
|
-
failTest(error);
|
|
476
|
-
}
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
/**
|
|
480
|
-
* Test 6: FIFO Ordering Within Partitions
|
|
481
|
-
*/
|
|
482
|
-
async function testPartitionPriorityOrdering() {
|
|
483
|
-
startTest('FIFO Ordering Within Partitions');
|
|
484
|
-
|
|
485
|
-
try {
|
|
486
|
-
// Configure queue
|
|
487
|
-
await client.configure({
|
|
488
|
-
queue: 'test-partition-priority',
|
|
489
|
-
options: {
|
|
490
|
-
priority: 0
|
|
491
|
-
}
|
|
492
|
-
});
|
|
493
|
-
|
|
494
|
-
const partitions = [
|
|
495
|
-
{ name: 'ultra-high', priority: 100 },
|
|
496
|
-
{ name: 'high', priority: 10 },
|
|
497
|
-
{ name: 'medium', priority: 5 },
|
|
498
|
-
{ name: 'low', priority: 1 }
|
|
499
|
-
];
|
|
500
|
-
|
|
501
|
-
// Create partitions
|
|
502
|
-
for (const partition of partitions) {
|
|
503
|
-
await dbPool.query(`
|
|
504
|
-
INSERT INTO queen.partitions (queue_id, name)
|
|
505
|
-
SELECT q.id, $1
|
|
506
|
-
FROM queen.queues q
|
|
507
|
-
WHERE q.name = 'test-partition-priority'
|
|
508
|
-
ON CONFLICT (queue_id, name) DO NOTHING
|
|
509
|
-
`, [partition.name]);
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
// Push messages to partitions
|
|
513
|
-
const messages = partitions.map(p => ({
|
|
514
|
-
queue: 'test-partition-priority',
|
|
515
|
-
partition: p.name,
|
|
516
|
-
payload: {
|
|
517
|
-
message: `${p.name} priority message`,
|
|
518
|
-
priority: p.priority
|
|
519
|
-
}
|
|
520
|
-
}));
|
|
521
|
-
|
|
522
|
-
await client.push({ items: messages });
|
|
523
|
-
await sleep(100);
|
|
524
|
-
|
|
525
|
-
// With partition locking, we need to pop from each partition separately
|
|
526
|
-
// or pop multiple times to get messages from different partitions
|
|
527
|
-
const allMessages = [];
|
|
528
|
-
|
|
529
|
-
// Pop messages multiple times to get from different partitions
|
|
530
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
531
|
-
const result = await client.pop({
|
|
532
|
-
queue: 'test-partition-priority',
|
|
533
|
-
batch: 1
|
|
534
|
-
});
|
|
535
|
-
|
|
536
|
-
if (result.messages && result.messages.length > 0) {
|
|
537
|
-
allMessages.push(...result.messages);
|
|
538
|
-
}
|
|
539
|
-
}
|
|
540
|
-
|
|
541
|
-
if (allMessages.length !== partitions.length) {
|
|
542
|
-
throw new Error(`Expected ${partitions.length} messages, got ${allMessages.length}`);
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
// Verify we got all messages
|
|
546
|
-
const receivedMessages = allMessages.map(m => m.payload.message);
|
|
547
|
-
const expectedMessages = messages.map(m => m.payload.message).sort();
|
|
548
|
-
const actualMessages = receivedMessages.sort();
|
|
549
|
-
|
|
550
|
-
if (JSON.stringify(expectedMessages) !== JSON.stringify(actualMessages)) {
|
|
551
|
-
throw new Error(`Messages don't match. Expected: ${expectedMessages}, Got: ${actualMessages}`);
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
// Acknowledge all messages
|
|
555
|
-
for (const message of allMessages) {
|
|
556
|
-
await client.ack(message.transactionId, 'completed');
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
passTest('FIFO ordering within partitions works correctly');
|
|
560
|
-
} catch (error) {
|
|
561
|
-
failTest(error);
|
|
562
|
-
}
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
// ============================================
|
|
566
|
-
// PARTITION LOCKING TESTS
|
|
567
|
-
// ============================================
|
|
568
|
-
|
|
569
|
-
/**
|
|
570
|
-
* Test 7: Partition Locking in Queue Mode
|
|
571
|
-
* Tests that partitions are properly locked when consumed
|
|
572
|
-
*/
|
|
573
|
-
async function testPartitionLocking() {
|
|
574
|
-
startTest('Partition Locking in Queue Mode');
|
|
575
|
-
|
|
576
|
-
const QUEUE_NAME = 'test-partition-lock';
|
|
577
|
-
const PARTITION1 = 'partition-A';
|
|
578
|
-
const PARTITION2 = 'partition-B';
|
|
579
|
-
|
|
580
|
-
try {
|
|
581
|
-
// Configure queue
|
|
582
|
-
await client.configure({
|
|
583
|
-
queue: QUEUE_NAME,
|
|
584
|
-
options: {}
|
|
585
|
-
});
|
|
586
|
-
|
|
587
|
-
// Setup: Push 3 messages to each partition
|
|
588
|
-
const messages = [];
|
|
589
|
-
|
|
590
|
-
for (let i = 1; i <= 3; i++) {
|
|
591
|
-
messages.push({
|
|
592
|
-
queue: QUEUE_NAME,
|
|
593
|
-
partition: PARTITION1,
|
|
594
|
-
payload: { data: `Message ${i} for ${PARTITION1}` },
|
|
595
|
-
transactionId: `txn-p1-${i}`
|
|
596
|
-
});
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
for (let i = 1; i <= 3; i++) {
|
|
600
|
-
messages.push({
|
|
601
|
-
queue: QUEUE_NAME,
|
|
602
|
-
partition: PARTITION2,
|
|
603
|
-
payload: { data: `Message ${i} for ${PARTITION2}` },
|
|
604
|
-
transactionId: `txn-p2-${i}`
|
|
605
|
-
});
|
|
606
|
-
}
|
|
607
|
-
|
|
608
|
-
// Push all messages
|
|
609
|
-
await client.push({ items: messages });
|
|
610
|
-
log(`Pushed ${messages.length} messages (3 to each partition)`);
|
|
611
|
-
|
|
612
|
-
// Consumer 1: Pop without specifying partition
|
|
613
|
-
log('Consumer 1: Popping messages (no partition specified)...');
|
|
614
|
-
const result1 = await client.pop({ queue: QUEUE_NAME, batch: 2 });
|
|
615
|
-
|
|
616
|
-
if (result1.messages.length === 0) {
|
|
617
|
-
throw new Error('Consumer 1 got no messages');
|
|
618
|
-
}
|
|
619
|
-
|
|
620
|
-
const partition1 = result1.messages[0].partition;
|
|
621
|
-
log(`Consumer 1: Got ${result1.messages.length} messages from partition: ${partition1}`);
|
|
622
|
-
|
|
623
|
-
// Consumer 2: Pop without specifying partition (should get different partition)
|
|
624
|
-
log('Consumer 2: Popping messages (no partition specified)...');
|
|
625
|
-
const result2 = await client.pop({ queue: QUEUE_NAME, batch: 2 });
|
|
626
|
-
|
|
627
|
-
if (result2.messages.length === 0) {
|
|
628
|
-
throw new Error('Consumer 2 got no messages');
|
|
629
|
-
}
|
|
630
|
-
|
|
631
|
-
const partition2 = result2.messages[0].partition;
|
|
632
|
-
log(`Consumer 2: Got ${result2.messages.length} messages from partition: ${partition2}`);
|
|
633
|
-
|
|
634
|
-
// Verify they got different partitions
|
|
635
|
-
if (partition1 === partition2) {
|
|
636
|
-
throw new Error('Both consumers got the same partition - partition locking not working!');
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
log(`SUCCESS: Consumers got different partitions (${partition1} vs ${partition2})`);
|
|
640
|
-
|
|
641
|
-
// Consumer 3: Try to pop again (should get no messages as both partitions are locked)
|
|
642
|
-
log('Consumer 3: Trying to pop (both partitions should be locked)...');
|
|
643
|
-
const result3 = await client.pop({ queue: QUEUE_NAME, batch: 2 });
|
|
644
|
-
|
|
645
|
-
if (result3.messages.length !== 0) {
|
|
646
|
-
throw new Error(`Consumer 3 got ${result3.messages.length} messages (should have been 0)`);
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
log('Consumer 3: Got no messages (correct - both partitions are locked)');
|
|
650
|
-
|
|
651
|
-
// Now ACK messages from Consumer 1 to release its partition
|
|
652
|
-
log('Consumer 1: ACKing messages to release partition...');
|
|
653
|
-
for (const msg of result1.messages) {
|
|
654
|
-
await client.ack(msg.transactionId, 'completed');
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
// Consumer 4: Should now be able to get remaining message from Consumer 1's partition
|
|
658
|
-
log('Consumer 4: Popping after Consumer 1 released...');
|
|
659
|
-
const result4 = await client.pop({ queue: QUEUE_NAME, batch: 2 });
|
|
660
|
-
|
|
661
|
-
if (result4.messages.length > 0) {
|
|
662
|
-
const partition4 = result4.messages[0].partition;
|
|
663
|
-
log(`Consumer 4: Got ${result4.messages.length} messages from partition: ${partition4}`);
|
|
664
|
-
|
|
665
|
-
// Should be from the same partition as Consumer 1 had
|
|
666
|
-
if (partition4 !== partition1) {
|
|
667
|
-
throw new Error('Consumer 4 did not get messages from released partition');
|
|
668
|
-
}
|
|
669
|
-
}
|
|
670
|
-
|
|
671
|
-
// Clean up: ACK all remaining messages
|
|
672
|
-
log('Cleaning up: ACKing all remaining messages...');
|
|
673
|
-
for (const msg of result2.messages) {
|
|
674
|
-
await client.ack(msg.transactionId, 'completed');
|
|
675
|
-
}
|
|
676
|
-
for (const msg of result4.messages) {
|
|
677
|
-
await client.ack(msg.transactionId, 'completed');
|
|
678
|
-
}
|
|
679
|
-
|
|
680
|
-
passTest('Partition locking works correctly');
|
|
681
|
-
} catch (error) {
|
|
682
|
-
failTest(error);
|
|
683
|
-
}
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
/**
|
|
687
|
-
* Test 8: Partition Locking in Bus Mode
|
|
688
|
-
* Tests that partitions are properly locked per consumer group
|
|
689
|
-
*/
|
|
690
|
-
async function testBusPartitionLocking() {
|
|
691
|
-
startTest('Partition Locking in Bus Mode');
|
|
692
|
-
|
|
693
|
-
const QUEUE_NAME = 'test-bus-partition-lock';
|
|
694
|
-
const PARTITION1 = 'partition-X';
|
|
695
|
-
const PARTITION2 = 'partition-Y';
|
|
696
|
-
const CONSUMER_GROUP1 = 'group-alpha';
|
|
697
|
-
const CONSUMER_GROUP2 = 'group-beta';
|
|
698
|
-
|
|
699
|
-
try {
|
|
700
|
-
// Configure queue
|
|
701
|
-
await client.configure({
|
|
702
|
-
queue: QUEUE_NAME,
|
|
703
|
-
options: {}
|
|
704
|
-
});
|
|
705
|
-
|
|
706
|
-
// Setup: Push 3 messages to each partition
|
|
707
|
-
const messages = [];
|
|
708
|
-
|
|
709
|
-
for (let i = 1; i <= 3; i++) {
|
|
710
|
-
messages.push({
|
|
711
|
-
queue: QUEUE_NAME,
|
|
712
|
-
partition: PARTITION1,
|
|
713
|
-
payload: { data: `Message ${i} for ${PARTITION1}` },
|
|
714
|
-
transactionId: `txn-px-${i}`
|
|
715
|
-
});
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
for (let i = 1; i <= 3; i++) {
|
|
719
|
-
messages.push({
|
|
720
|
-
queue: QUEUE_NAME,
|
|
721
|
-
partition: PARTITION2,
|
|
722
|
-
payload: { data: `Message ${i} for ${PARTITION2}` },
|
|
723
|
-
transactionId: `txn-py-${i}`
|
|
724
|
-
});
|
|
725
|
-
}
|
|
726
|
-
|
|
727
|
-
// Push all messages
|
|
728
|
-
await client.push({ items: messages });
|
|
729
|
-
log(`Pushed ${messages.length} messages (3 to each partition)`);
|
|
730
|
-
|
|
731
|
-
// Group Alpha, Consumer 1: Pop without specifying partition
|
|
732
|
-
log('Group Alpha, Consumer 1: Popping messages...');
|
|
733
|
-
const result1 = await client.pop({
|
|
734
|
-
queue: QUEUE_NAME,
|
|
735
|
-
batch: 2,
|
|
736
|
-
consumerGroup: CONSUMER_GROUP1
|
|
737
|
-
});
|
|
738
|
-
|
|
739
|
-
if (result1.messages.length === 0) {
|
|
740
|
-
throw new Error('Group Alpha Consumer 1 got no messages');
|
|
741
|
-
}
|
|
742
|
-
|
|
743
|
-
const partition1 = result1.messages[0].partition;
|
|
744
|
-
log(`Group Alpha Consumer 1: Got ${result1.messages.length} messages from partition: ${partition1}`);
|
|
745
|
-
|
|
746
|
-
// Group Alpha, Consumer 2: Pop without specifying partition (should get different partition if locking works)
|
|
747
|
-
log('Group Alpha, Consumer 2: Popping messages...');
|
|
748
|
-
const result2 = await client.pop({
|
|
749
|
-
queue: QUEUE_NAME,
|
|
750
|
-
batch: 2,
|
|
751
|
-
consumerGroup: CONSUMER_GROUP1
|
|
752
|
-
});
|
|
753
|
-
|
|
754
|
-
if (result2.messages.length === 0) {
|
|
755
|
-
log('Group Alpha Consumer 2 got no messages (partition locking might be working)');
|
|
756
|
-
} else {
|
|
757
|
-
const partition2 = result2.messages[0].partition;
|
|
758
|
-
log(`Group Alpha Consumer 2: Got ${result2.messages.length} messages from partition: ${partition2}`);
|
|
759
|
-
|
|
760
|
-
if (partition1 === partition2) {
|
|
761
|
-
log('WARNING: Both consumers in same group got same partition!');
|
|
762
|
-
log('This means partition locking is NOT implemented for bus mode.');
|
|
763
|
-
} else {
|
|
764
|
-
log('Consumers in same group got different partitions');
|
|
765
|
-
}
|
|
766
|
-
}
|
|
767
|
-
|
|
768
|
-
// Group Beta, Consumer 1: Should be able to get messages from any partition (different consumer group)
|
|
769
|
-
log('Group Beta, Consumer 1: Popping messages (different consumer group)...');
|
|
770
|
-
const result3 = await client.pop({
|
|
771
|
-
queue: QUEUE_NAME,
|
|
772
|
-
batch: 2,
|
|
773
|
-
consumerGroup: CONSUMER_GROUP2
|
|
774
|
-
});
|
|
775
|
-
|
|
776
|
-
if (result3.messages.length > 0) {
|
|
777
|
-
const partition3 = result3.messages[0].partition;
|
|
778
|
-
log(`Group Beta Consumer 1: Got ${result3.messages.length} messages from partition: ${partition3}`);
|
|
779
|
-
log('Different consumer groups can access same partitions independently (correct)');
|
|
780
|
-
}
|
|
781
|
-
|
|
782
|
-
// Clean up: ACK all messages
|
|
783
|
-
log('Cleaning up: ACKing all messages...');
|
|
784
|
-
for (const msg of result1.messages) {
|
|
785
|
-
await client.ack(msg.transactionId, 'completed', null, CONSUMER_GROUP1);
|
|
786
|
-
}
|
|
787
|
-
for (const msg of result2.messages) {
|
|
788
|
-
await client.ack(msg.transactionId, 'completed', null, CONSUMER_GROUP1);
|
|
789
|
-
}
|
|
790
|
-
for (const msg of result3.messages) {
|
|
791
|
-
await client.ack(msg.transactionId, 'completed', null, CONSUMER_GROUP2);
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
passTest('Bus mode partition locking test completed');
|
|
795
|
-
} catch (error) {
|
|
796
|
-
failTest(error);
|
|
797
|
-
}
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
/**
|
|
801
|
-
* Test 9: Specific Partition Request with Locking
|
|
802
|
-
* Tests that requesting a specific partition respects locking
|
|
803
|
-
*/
|
|
804
|
-
async function testSpecificPartitionLocking() {
|
|
805
|
-
startTest('Specific Partition Request with Locking');
|
|
806
|
-
|
|
807
|
-
const QUEUE_NAME = 'test-specific-partition';
|
|
808
|
-
const PARTITION_NAME = 'specific-partition';
|
|
809
|
-
const CONSUMER_GROUP = 'test-group';
|
|
810
|
-
|
|
811
|
-
try {
|
|
812
|
-
// Configure queue
|
|
813
|
-
await client.configure({
|
|
814
|
-
queue: QUEUE_NAME,
|
|
815
|
-
options: {}
|
|
816
|
-
});
|
|
817
|
-
|
|
818
|
-
// Part 1: Test Queue Mode
|
|
819
|
-
log('=== Testing Queue Mode with Specific Partition ===');
|
|
820
|
-
|
|
821
|
-
// Push test messages
|
|
822
|
-
const messages = [];
|
|
823
|
-
for (let i = 1; i <= 5; i++) {
|
|
824
|
-
messages.push({
|
|
825
|
-
queue: QUEUE_NAME,
|
|
826
|
-
partition: PARTITION_NAME,
|
|
827
|
-
payload: { data: `Message ${i}` },
|
|
828
|
-
transactionId: `txn-${i}`
|
|
829
|
-
});
|
|
830
|
-
}
|
|
831
|
-
|
|
832
|
-
await client.push({ items: messages });
|
|
833
|
-
log(`Pushed ${messages.length} messages to ${PARTITION_NAME}`);
|
|
834
|
-
|
|
835
|
-
// Consumer 1: Request specific partition
|
|
836
|
-
log('Consumer 1: Requesting specific partition...');
|
|
837
|
-
const result1 = await client.pop({
|
|
838
|
-
queue: QUEUE_NAME,
|
|
839
|
-
batch: 2,
|
|
840
|
-
partition: PARTITION_NAME
|
|
841
|
-
});
|
|
842
|
-
|
|
843
|
-
if (result1.messages.length === 0) {
|
|
844
|
-
throw new Error('Consumer 1 got no messages');
|
|
845
|
-
}
|
|
846
|
-
|
|
847
|
-
log(`Consumer 1: Got ${result1.messages.length} messages from partition: ${result1.messages[0].partition}`);
|
|
848
|
-
|
|
849
|
-
// Consumer 2: Try to request same specific partition (should get nothing due to lock)
|
|
850
|
-
log('Consumer 2: Requesting same specific partition...');
|
|
851
|
-
const result2 = await client.pop({
|
|
852
|
-
queue: QUEUE_NAME,
|
|
853
|
-
batch: 2,
|
|
854
|
-
partition: PARTITION_NAME
|
|
855
|
-
});
|
|
856
|
-
|
|
857
|
-
if (result2.messages.length !== 0) {
|
|
858
|
-
throw new Error(`Consumer 2 got ${result2.messages.length} messages (should have been 0 due to lock)`);
|
|
859
|
-
}
|
|
860
|
-
|
|
861
|
-
log('Consumer 2: Got no messages (correct - partition is locked by Consumer 1)');
|
|
862
|
-
|
|
863
|
-
// ACK messages from Consumer 1
|
|
864
|
-
log('Consumer 1: ACKing messages...');
|
|
865
|
-
for (const msg of result1.messages) {
|
|
866
|
-
await client.ack(msg.transactionId, 'completed');
|
|
867
|
-
}
|
|
868
|
-
|
|
869
|
-
// Consumer 3: Should now get messages from the specific partition
|
|
870
|
-
log('Consumer 3: Requesting specific partition after release...');
|
|
871
|
-
const result3 = await client.pop({
|
|
872
|
-
queue: QUEUE_NAME,
|
|
873
|
-
batch: 2,
|
|
874
|
-
partition: PARTITION_NAME
|
|
875
|
-
});
|
|
876
|
-
|
|
877
|
-
if (result3.messages.length === 0) {
|
|
878
|
-
throw new Error('Consumer 3 got no messages (should have gotten remaining messages)');
|
|
879
|
-
}
|
|
880
|
-
|
|
881
|
-
log(`Consumer 3: Got ${result3.messages.length} messages after lease release`);
|
|
882
|
-
|
|
883
|
-
// Clean up queue mode messages
|
|
884
|
-
for (const msg of result3.messages) {
|
|
885
|
-
await client.ack(msg.transactionId, 'completed');
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
// Part 2: Test Bus Mode
|
|
889
|
-
log('=== Testing Bus Mode with Specific Partition ===');
|
|
890
|
-
|
|
891
|
-
// Push fresh messages
|
|
892
|
-
const busMessages = [];
|
|
893
|
-
for (let i = 1; i <= 5; i++) {
|
|
894
|
-
busMessages.push({
|
|
895
|
-
queue: QUEUE_NAME,
|
|
896
|
-
partition: PARTITION_NAME,
|
|
897
|
-
payload: { data: `Bus Message ${i}` },
|
|
898
|
-
transactionId: `bus-txn-${i}`
|
|
899
|
-
});
|
|
900
|
-
}
|
|
901
|
-
|
|
902
|
-
await client.push({ items: busMessages });
|
|
903
|
-
|
|
904
|
-
// Group Alpha, Consumer 1: Request specific partition
|
|
905
|
-
log('Group Alpha, Consumer 1: Requesting specific partition...');
|
|
906
|
-
const busResult1 = await client.pop({
|
|
907
|
-
queue: QUEUE_NAME,
|
|
908
|
-
batch: 2,
|
|
909
|
-
partition: PARTITION_NAME,
|
|
910
|
-
consumerGroup: CONSUMER_GROUP
|
|
911
|
-
});
|
|
912
|
-
|
|
913
|
-
if (busResult1.messages.length === 0) {
|
|
914
|
-
throw new Error('Group Alpha Consumer 1 got no messages');
|
|
915
|
-
}
|
|
916
|
-
|
|
917
|
-
log(`Group Alpha Consumer 1: Got ${busResult1.messages.length} messages`);
|
|
918
|
-
|
|
919
|
-
// Group Alpha, Consumer 2: Try same specific partition (should be locked)
|
|
920
|
-
log('Group Alpha, Consumer 2: Requesting same specific partition...');
|
|
921
|
-
const busResult2 = await client.pop({
|
|
922
|
-
queue: QUEUE_NAME,
|
|
923
|
-
batch: 2,
|
|
924
|
-
partition: PARTITION_NAME,
|
|
925
|
-
consumerGroup: CONSUMER_GROUP
|
|
926
|
-
});
|
|
927
|
-
|
|
928
|
-
if (busResult2.messages.length !== 0) {
|
|
929
|
-
throw new Error(`Group Alpha Consumer 2: Got ${busResult2.messages.length} messages (should be 0)`);
|
|
930
|
-
}
|
|
931
|
-
|
|
932
|
-
log('Group Alpha Consumer 2: Got no messages (correct - partition locked by Consumer 1)');
|
|
933
|
-
|
|
934
|
-
// Different Group, Consumer 1: Should get messages (different consumer group)
|
|
935
|
-
log('Group Beta, Consumer 1: Requesting same partition (different group)...');
|
|
936
|
-
const busResult3 = await client.pop({
|
|
937
|
-
queue: QUEUE_NAME,
|
|
938
|
-
batch: 2,
|
|
939
|
-
partition: PARTITION_NAME,
|
|
940
|
-
consumerGroup: 'group-beta'
|
|
941
|
-
});
|
|
942
|
-
|
|
943
|
-
if (busResult3.messages.length === 0) {
|
|
944
|
-
throw new Error('Group Beta should be able to access partition (different group)');
|
|
945
|
-
}
|
|
946
|
-
|
|
947
|
-
log(`Group Beta Consumer 1: Got ${busResult3.messages.length} messages (different group works)`);
|
|
948
|
-
|
|
949
|
-
// Clean up
|
|
950
|
-
log('Cleaning up...');
|
|
951
|
-
for (const msg of busResult1.messages) {
|
|
952
|
-
await client.ack(msg.transactionId, 'completed', null, CONSUMER_GROUP);
|
|
953
|
-
}
|
|
954
|
-
for (const msg of busResult3.messages) {
|
|
955
|
-
await client.ack(msg.transactionId, 'completed', null, 'group-beta');
|
|
956
|
-
}
|
|
957
|
-
|
|
958
|
-
passTest('Specific partition locking works correctly');
|
|
959
|
-
} catch (error) {
|
|
960
|
-
failTest(error);
|
|
961
|
-
}
|
|
962
|
-
}
|
|
963
|
-
|
|
964
|
-
/**
|
|
965
|
-
* Test 10: Namespace/Task Filtering with Partition Locking
|
|
966
|
-
* Tests that namespace/task filtering properly locks partitions
|
|
967
|
-
*/
|
|
968
|
-
async function testNamespaceTaskFiltering() {
|
|
969
|
-
startTest('Namespace/Task Filtering with Partition Locking');
|
|
970
|
-
|
|
971
|
-
try {
|
|
972
|
-
// Setup: Create multiple queues with same namespace but different tasks
|
|
973
|
-
const namespace = 'test-namespace';
|
|
974
|
-
const queues = [
|
|
975
|
-
{ name: 'test-queue-task-1', task: 'process', partition: 'p1' },
|
|
976
|
-
{ name: 'test-queue-task-2', task: 'process', partition: 'p2' },
|
|
977
|
-
{ name: 'test-queue-task-3', task: 'analyze', partition: 'p3' }
|
|
978
|
-
];
|
|
979
|
-
|
|
980
|
-
// Configure queues with namespace and task
|
|
981
|
-
for (const q of queues) {
|
|
982
|
-
// Configure queue with namespace and task
|
|
983
|
-
await client.configure({
|
|
984
|
-
queue: q.name,
|
|
985
|
-
namespace: namespace,
|
|
986
|
-
task: q.task,
|
|
987
|
-
options: { leaseTime: 30 }
|
|
988
|
-
});
|
|
989
|
-
// Push messages to each queue
|
|
990
|
-
await client.push({
|
|
991
|
-
items: [
|
|
992
|
-
{
|
|
993
|
-
queue: q.name,
|
|
994
|
-
partition: q.partition,
|
|
995
|
-
payload: { data: `Message for ${q.name}` },
|
|
996
|
-
transactionId: `txn-${q.name}-1`
|
|
997
|
-
},
|
|
998
|
-
{
|
|
999
|
-
queue: q.name,
|
|
1000
|
-
partition: q.partition,
|
|
1001
|
-
payload: { data: `Message 2 for ${q.name}` },
|
|
1002
|
-
transactionId: `txn-${q.name}-2`
|
|
1003
|
-
}
|
|
1004
|
-
]
|
|
1005
|
-
});
|
|
1006
|
-
}
|
|
1007
|
-
|
|
1008
|
-
log('Pushed messages to 3 queues with namespace/task metadata');
|
|
1009
|
-
|
|
1010
|
-
// Test 1: Pop by namespace only (should get messages from all queues)
|
|
1011
|
-
log('Consumer 1: Popping by namespace only...');
|
|
1012
|
-
const result1 = await client.pop({
|
|
1013
|
-
namespace: namespace
|
|
1014
|
-
}, { batch: 3 });
|
|
1015
|
-
|
|
1016
|
-
if (result1.messages.length === 0) {
|
|
1017
|
-
throw new Error('Consumer 1 got no messages');
|
|
1018
|
-
}
|
|
1019
|
-
|
|
1020
|
-
log(`Consumer 1: Got ${result1.messages.length} messages from namespace ${namespace}`);
|
|
1021
|
-
const partitions1 = [...new Set(result1.messages.map(m => m.partition))];
|
|
1022
|
-
log(`Consumer 1: Locked partitions: ${partitions1.join(', ')}`);
|
|
1023
|
-
|
|
1024
|
-
// Test 2: Another consumer tries to pop from same namespace (should get from unlocked partitions only)
|
|
1025
|
-
log('Consumer 2: Trying to pop from same namespace...');
|
|
1026
|
-
const result2 = await client.pop({
|
|
1027
|
-
namespace: namespace
|
|
1028
|
-
}, { batch: 3 });
|
|
1029
|
-
|
|
1030
|
-
if (result2.messages.length > 0) {
|
|
1031
|
-
const partitions2 = [...new Set(result2.messages.map(m => m.partition))];
|
|
1032
|
-
log(`Consumer 2: Got ${result2.messages.length} messages from partitions: ${partitions2.join(', ')}`);
|
|
1033
|
-
|
|
1034
|
-
// Verify no overlap in partitions
|
|
1035
|
-
const overlap = partitions1.filter(p => partitions2.includes(p));
|
|
1036
|
-
if (overlap.length > 0) {
|
|
1037
|
-
throw new Error(`Partition locking failed! Both consumers got partition(s): ${overlap.join(', ')}`);
|
|
1038
|
-
}
|
|
1039
|
-
log('SUCCESS: No partition overlap between consumers');
|
|
1040
|
-
} else {
|
|
1041
|
-
log('Consumer 2: Got no messages (all partitions locked)');
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
// Test 3: Pop by specific task (should only get from matching queues)
|
|
1045
|
-
log('Consumer 3: Popping by specific task "analyze"...');
|
|
1046
|
-
const result3 = await client.pop({
|
|
1047
|
-
namespace: namespace,
|
|
1048
|
-
task: 'analyze'
|
|
1049
|
-
}, { batch: 2 });
|
|
1050
|
-
|
|
1051
|
-
if (result3.messages.length > 0) {
|
|
1052
|
-
log(`Consumer 3: Got ${result3.messages.length} messages for task "analyze"`);
|
|
1053
|
-
// Verify all messages are from the analyze task
|
|
1054
|
-
const allAnalyze = result3.messages.every(m => m.queue === 'test-queue-task-3');
|
|
1055
|
-
if (!allAnalyze) {
|
|
1056
|
-
throw new Error('Got messages from wrong task!');
|
|
1057
|
-
}
|
|
1058
|
-
}
|
|
1059
|
-
|
|
1060
|
-
// Clean up: ACK all messages
|
|
1061
|
-
for (const msg of [...result1.messages, ...result2.messages, ...result3.messages]) {
|
|
1062
|
-
await client.ack(msg.transactionId, 'completed');
|
|
1063
|
-
}
|
|
1064
|
-
|
|
1065
|
-
passTest('Namespace/Task filtering with partition locking works correctly');
|
|
1066
|
-
} catch (error) {
|
|
1067
|
-
failTest(error);
|
|
1068
|
-
}
|
|
1069
|
-
}
|
|
1070
|
-
|
|
1071
|
-
/**
|
|
1072
|
-
* Test 11: Namespace/Task Filtering in Bus Mode
|
|
1073
|
-
* Tests that namespace/task filtering works with consumer groups
|
|
1074
|
-
*/
|
|
1075
|
-
async function testNamespaceTaskBusMode() {
|
|
1076
|
-
startTest('Namespace/Task Filtering in Bus Mode');
|
|
1077
|
-
|
|
1078
|
-
try {
|
|
1079
|
-
const namespace = 'bus-namespace';
|
|
1080
|
-
const task = 'bus-task';
|
|
1081
|
-
const queues = [
|
|
1082
|
-
{ name: 'test-bus-queue-1', partition: 'bus-p1' },
|
|
1083
|
-
{ name: 'test-bus-queue-2', partition: 'bus-p2' }
|
|
1084
|
-
];
|
|
1085
|
-
|
|
1086
|
-
// Setup queues and push messages
|
|
1087
|
-
for (const q of queues) {
|
|
1088
|
-
// Configure queue with namespace and task
|
|
1089
|
-
await client.configure({
|
|
1090
|
-
queue: q.name,
|
|
1091
|
-
namespace: namespace,
|
|
1092
|
-
task: task,
|
|
1093
|
-
options: { leaseTime: 30 }
|
|
1094
|
-
});
|
|
1095
|
-
await client.push({
|
|
1096
|
-
items: [
|
|
1097
|
-
{
|
|
1098
|
-
queue: q.name,
|
|
1099
|
-
partition: q.partition,
|
|
1100
|
-
payload: { data: `Bus message for ${q.name}` },
|
|
1101
|
-
transactionId: `bus-txn-${q.name}-1`
|
|
1102
|
-
},
|
|
1103
|
-
{
|
|
1104
|
-
queue: q.name,
|
|
1105
|
-
partition: q.partition,
|
|
1106
|
-
payload: { data: `Bus message 2 for ${q.name}` },
|
|
1107
|
-
transactionId: `bus-txn-${q.name}-2`
|
|
1108
|
-
}
|
|
1109
|
-
]
|
|
1110
|
-
});
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
log('Pushed messages to 2 queues for bus mode testing');
|
|
1114
|
-
|
|
1115
|
-
// Test 1: Consumer Group A pops messages
|
|
1116
|
-
log('Group A, Consumer 1: Popping with namespace/task filter...');
|
|
1117
|
-
const resultA1 = await client.pop({
|
|
1118
|
-
namespace: namespace,
|
|
1119
|
-
task: task,
|
|
1120
|
-
consumerGroup: 'group-a'
|
|
1121
|
-
}, { batch: 2 });
|
|
1122
|
-
|
|
1123
|
-
if (resultA1.messages.length === 0) {
|
|
1124
|
-
throw new Error('Group A Consumer 1 got no messages');
|
|
1125
|
-
}
|
|
1126
|
-
|
|
1127
|
-
const partitionsA1 = [...new Set(resultA1.messages.map(m => m.partition))];
|
|
1128
|
-
log(`Group A Consumer 1: Got ${resultA1.messages.length} messages from partitions: ${partitionsA1.join(', ')}`);
|
|
1129
|
-
|
|
1130
|
-
// Test 2: Another consumer in Group A (should get from different partitions due to locking)
|
|
1131
|
-
log('Group A, Consumer 2: Trying same namespace/task filter...');
|
|
1132
|
-
const resultA2 = await client.pop({
|
|
1133
|
-
namespace: namespace,
|
|
1134
|
-
task: task,
|
|
1135
|
-
consumerGroup: 'group-a'
|
|
1136
|
-
}, { batch: 2 });
|
|
1137
|
-
|
|
1138
|
-
if (resultA2.messages.length > 0) {
|
|
1139
|
-
const partitionsA2 = [...new Set(resultA2.messages.map(m => m.partition))];
|
|
1140
|
-
log(`Group A Consumer 2: Got ${resultA2.messages.length} messages from partitions: ${partitionsA2.join(', ')}`);
|
|
1141
|
-
|
|
1142
|
-
// Check for partition overlap
|
|
1143
|
-
const overlapA = partitionsA1.filter(p => partitionsA2.includes(p));
|
|
1144
|
-
if (overlapA.length > 0) {
|
|
1145
|
-
log(`WARNING: Partition overlap in same group: ${overlapA.join(', ')}`);
|
|
1146
|
-
}
|
|
1147
|
-
} else {
|
|
1148
|
-
log('Group A Consumer 2: No messages (partitions locked by Consumer 1)');
|
|
1149
|
-
}
|
|
1150
|
-
|
|
1151
|
-
// Test 3: Consumer Group B (should be independent)
|
|
1152
|
-
log('Group B, Consumer 1: Popping with same namespace/task filter...');
|
|
1153
|
-
const resultB1 = await client.pop({
|
|
1154
|
-
namespace: namespace,
|
|
1155
|
-
task: task,
|
|
1156
|
-
consumerGroup: 'group-b'
|
|
1157
|
-
}, { batch: 2 });
|
|
1158
|
-
|
|
1159
|
-
if (resultB1.messages.length > 0) {
|
|
1160
|
-
const partitionsB1 = [...new Set(resultB1.messages.map(m => m.partition))];
|
|
1161
|
-
log(`Group B Consumer 1: Got ${resultB1.messages.length} messages from partitions: ${partitionsB1.join(', ')}`);
|
|
1162
|
-
log('SUCCESS: Different consumer groups can access same partitions independently');
|
|
1163
|
-
} else {
|
|
1164
|
-
throw new Error('Group B should be able to get messages (different consumer group)');
|
|
1165
|
-
}
|
|
1166
|
-
|
|
1167
|
-
// Clean up: ACK all messages
|
|
1168
|
-
for (const msg of resultA1.messages) {
|
|
1169
|
-
await client.ack(msg.transactionId, 'completed', null, 'group-a');
|
|
1170
|
-
}
|
|
1171
|
-
for (const msg of resultA2.messages) {
|
|
1172
|
-
await client.ack(msg.transactionId, 'completed', null, 'group-a');
|
|
1173
|
-
}
|
|
1174
|
-
for (const msg of resultB1.messages) {
|
|
1175
|
-
await client.ack(msg.transactionId, 'completed', null, 'group-b');
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
|
-
passTest('Namespace/Task filtering in bus mode works correctly');
|
|
1179
|
-
} catch (error) {
|
|
1180
|
-
failTest(error);
|
|
1181
|
-
}
|
|
1182
|
-
}
|
|
1183
|
-
|
|
1184
|
-
// ============================================
|
|
1185
|
-
// ENTERPRISE FEATURE TESTS
|
|
1186
|
-
// ============================================
|
|
1187
|
-
|
|
1188
|
-
/**
|
|
1189
|
-
* Test 12: Message Encryption
|
|
1190
|
-
*/
|
|
1191
|
-
async function testMessageEncryption() {
|
|
1192
|
-
startTest('Message Encryption', 'enterprise');
|
|
1193
|
-
|
|
1194
|
-
try {
|
|
1195
|
-
// Check if encryption is enabled
|
|
1196
|
-
const encryptionKey = process.env.QUEEN_ENCRYPTION_KEY;
|
|
1197
|
-
if (!encryptionKey) {
|
|
1198
|
-
log('Skipping encryption test - QUEEN_ENCRYPTION_KEY not set', 'warning');
|
|
1199
|
-
passTest('Skipped - encryption key not configured');
|
|
1200
|
-
return;
|
|
1201
|
-
}
|
|
1202
|
-
|
|
1203
|
-
// Configure queue with encryption
|
|
1204
|
-
await client.configure({
|
|
1205
|
-
queue: 'test-encryption-queue',
|
|
1206
|
-
options: {
|
|
1207
|
-
encryptionEnabled: true
|
|
1208
|
-
}
|
|
1209
|
-
});
|
|
1210
|
-
|
|
1211
|
-
// Verify encryption is enabled in database
|
|
1212
|
-
const queueResult = await dbPool.query(`
|
|
1213
|
-
SELECT encryption_enabled
|
|
1214
|
-
FROM queen.queues
|
|
1215
|
-
WHERE name = 'test-encryption-queue'
|
|
1216
|
-
`);
|
|
1217
|
-
|
|
1218
|
-
if (queueResult.rows.length === 0) {
|
|
1219
|
-
throw new Error('Queue not found in database after configuration');
|
|
1220
|
-
}
|
|
1221
|
-
|
|
1222
|
-
if (!queueResult.rows[0].encryption_enabled) {
|
|
1223
|
-
throw new Error(`Encryption not enabled on queue`);
|
|
1224
|
-
}
|
|
1225
|
-
|
|
1226
|
-
// Push sensitive data
|
|
1227
|
-
const sensitiveData = {
|
|
1228
|
-
creditCard: '4111111111111111',
|
|
1229
|
-
ssn: '123-45-6789',
|
|
1230
|
-
password: 'super-secret-password',
|
|
1231
|
-
timestamp: Date.now()
|
|
1232
|
-
};
|
|
1233
|
-
|
|
1234
|
-
const pushResult = await client.push({
|
|
1235
|
-
items: [{
|
|
1236
|
-
queue: 'test-encryption-queue',
|
|
1237
|
-
payload: sensitiveData
|
|
1238
|
-
}]
|
|
1239
|
-
});
|
|
1240
|
-
|
|
1241
|
-
if (!pushResult.messages || pushResult.messages.length !== 1) {
|
|
1242
|
-
throw new Error('Failed to push encrypted message');
|
|
1243
|
-
}
|
|
1244
|
-
|
|
1245
|
-
await sleep(100);
|
|
1246
|
-
|
|
1247
|
-
// Check that the message is encrypted in database
|
|
1248
|
-
const dbResult = await dbPool.query(`
|
|
1249
|
-
SELECT m.payload, m.is_encrypted
|
|
1250
|
-
FROM queen.messages m
|
|
1251
|
-
JOIN queen.partitions p ON m.partition_id = p.id
|
|
1252
|
-
JOIN queen.queues q ON p.queue_id = q.id
|
|
1253
|
-
WHERE q.name = 'test-encryption-queue'
|
|
1254
|
-
ORDER BY m.created_at DESC
|
|
1255
|
-
LIMIT 1
|
|
1256
|
-
`);
|
|
1257
|
-
|
|
1258
|
-
if (dbResult.rows.length === 0) {
|
|
1259
|
-
throw new Error('No message found in database after push');
|
|
1260
|
-
}
|
|
1261
|
-
|
|
1262
|
-
if (!dbResult.rows[0].is_encrypted) {
|
|
1263
|
-
throw new Error('Message not marked as encrypted in database');
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
|
-
const storedData = dbResult.rows[0].payload;
|
|
1267
|
-
|
|
1268
|
-
// Verify the stored data is encrypted
|
|
1269
|
-
if (!storedData.encrypted || !storedData.iv || !storedData.authTag) {
|
|
1270
|
-
throw new Error('Message not properly encrypted in database');
|
|
1271
|
-
}
|
|
1272
|
-
|
|
1273
|
-
// Verify the stored data doesn't contain plain text sensitive info
|
|
1274
|
-
const storedStr = JSON.stringify(storedData);
|
|
1275
|
-
if (storedStr.includes('4111111111111111') ||
|
|
1276
|
-
storedStr.includes('123-45-6789') ||
|
|
1277
|
-
storedStr.includes('super-secret-password')) {
|
|
1278
|
-
throw new Error('Sensitive data found in plain text - encryption failed!');
|
|
1279
|
-
}
|
|
1280
|
-
|
|
1281
|
-
// Pop the message (should be decrypted automatically)
|
|
1282
|
-
const popResult = await client.pop({
|
|
1283
|
-
queue: 'test-encryption-queue',
|
|
1284
|
-
batch: 1
|
|
1285
|
-
});
|
|
1286
|
-
|
|
1287
|
-
if (!popResult.messages || popResult.messages.length !== 1) {
|
|
1288
|
-
throw new Error('Failed to pop encrypted message');
|
|
1289
|
-
}
|
|
1290
|
-
|
|
1291
|
-
const decryptedData = popResult.messages[0].data;
|
|
1292
|
-
|
|
1293
|
-
// Verify decrypted data matches original
|
|
1294
|
-
if (decryptedData.creditCard !== sensitiveData.creditCard ||
|
|
1295
|
-
decryptedData.ssn !== sensitiveData.ssn ||
|
|
1296
|
-
decryptedData.password !== sensitiveData.password ||
|
|
1297
|
-
decryptedData.timestamp !== sensitiveData.timestamp) {
|
|
1298
|
-
throw new Error('Decrypted data does not match original');
|
|
1299
|
-
}
|
|
1300
|
-
|
|
1301
|
-
await client.ack(popResult.messages[0].transactionId, 'completed');
|
|
1302
|
-
|
|
1303
|
-
passTest('Message encryption/decryption works correctly');
|
|
1304
|
-
} catch (error) {
|
|
1305
|
-
failTest(error);
|
|
1306
|
-
}
|
|
1307
|
-
}
|
|
1308
|
-
|
|
1309
|
-
/**
|
|
1310
|
-
* Test 13: Retention Policy - Pending Messages
|
|
1311
|
-
*/
|
|
1312
|
-
async function testRetentionPendingMessages() {
|
|
1313
|
-
startTest('Retention Policy - Pending Messages', 'enterprise');
|
|
1314
|
-
|
|
1315
|
-
try {
|
|
1316
|
-
// Configure queue with retention for pending messages
|
|
1317
|
-
await client.configure({
|
|
1318
|
-
queue: 'test-retention-pending',
|
|
1319
|
-
options: {
|
|
1320
|
-
retentionEnabled: true,
|
|
1321
|
-
retentionSeconds: 3 // 3 seconds retention for pending messages
|
|
1322
|
-
}
|
|
1323
|
-
});
|
|
1324
|
-
|
|
1325
|
-
// Push messages
|
|
1326
|
-
await client.push({
|
|
1327
|
-
items: [
|
|
1328
|
-
{
|
|
1329
|
-
queue: 'test-retention-pending',
|
|
1330
|
-
payload: { message: 'Message to be retained', order: 1 }
|
|
1331
|
-
},
|
|
1332
|
-
{
|
|
1333
|
-
queue: 'test-retention-pending',
|
|
1334
|
-
payload: { message: 'Another message to be retained', order: 2 }
|
|
1335
|
-
}
|
|
1336
|
-
]
|
|
1337
|
-
});
|
|
1338
|
-
|
|
1339
|
-
// Verify messages are in database
|
|
1340
|
-
const initialCount = await getMessageCount('test-retention-pending', 'Default');
|
|
1341
|
-
if (initialCount !== 2) {
|
|
1342
|
-
throw new Error(`Expected 2 messages, got ${initialCount}`);
|
|
1343
|
-
}
|
|
1344
|
-
|
|
1345
|
-
// Wait for retention period
|
|
1346
|
-
log('Waiting for retention period (3 seconds)...', 'info');
|
|
1347
|
-
await sleep(3500);
|
|
1348
|
-
|
|
1349
|
-
// Trigger retention by attempting a pop
|
|
1350
|
-
await client.pop({
|
|
1351
|
-
queue: 'test-retention-pending',
|
|
1352
|
-
batch: 1
|
|
1353
|
-
});
|
|
1354
|
-
|
|
1355
|
-
// Check if messages were retained
|
|
1356
|
-
const finalCount = await getMessageCount('test-retention-pending', 'Default');
|
|
1357
|
-
|
|
1358
|
-
log(`Messages after retention period: ${finalCount} (from ${initialCount})`, 'info');
|
|
1359
|
-
|
|
1360
|
-
// Check retention history
|
|
1361
|
-
const retentionHistory = await dbPool.query(`
|
|
1362
|
-
SELECT rh.messages_deleted, rh.retention_type
|
|
1363
|
-
FROM queen.retention_history rh
|
|
1364
|
-
JOIN queen.partitions p ON rh.partition_id = p.id
|
|
1365
|
-
JOIN queen.queues q ON p.queue_id = q.id
|
|
1366
|
-
WHERE q.name = 'test-retention-pending'
|
|
1367
|
-
ORDER BY rh.executed_at DESC
|
|
1368
|
-
LIMIT 1
|
|
1369
|
-
`);
|
|
1370
|
-
|
|
1371
|
-
if (retentionHistory.rows.length > 0) {
|
|
1372
|
-
log(`Retention history: ${retentionHistory.rows[0].messages_deleted} messages deleted (${retentionHistory.rows[0].retention_type})`, 'info');
|
|
1373
|
-
}
|
|
1374
|
-
|
|
1375
|
-
passTest('Retention policy for pending messages configured successfully');
|
|
1376
|
-
} catch (error) {
|
|
1377
|
-
failTest(error);
|
|
1378
|
-
}
|
|
1379
|
-
}
|
|
1380
|
-
|
|
1381
|
-
/**
|
|
1382
|
-
* Test 14: Retention Policy - Completed Messages
|
|
1383
|
-
*/
|
|
1384
|
-
async function testRetentionCompletedMessages() {
|
|
1385
|
-
startTest('Retention Policy - Completed Messages', 'enterprise');
|
|
1386
|
-
|
|
1387
|
-
try {
|
|
1388
|
-
// Configure queue with retention for completed messages
|
|
1389
|
-
await client.configure({
|
|
1390
|
-
queue: 'test-retention-completed',
|
|
1391
|
-
options: {
|
|
1392
|
-
retentionEnabled: true,
|
|
1393
|
-
completedRetentionSeconds: 2 // 2 seconds retention for completed messages
|
|
1394
|
-
}
|
|
1395
|
-
});
|
|
1396
|
-
|
|
1397
|
-
// Push messages
|
|
1398
|
-
const pushResult = await client.push({
|
|
1399
|
-
items: [
|
|
1400
|
-
{
|
|
1401
|
-
queue: 'test-retention-completed',
|
|
1402
|
-
payload: { message: 'Message to complete and retain' }
|
|
1403
|
-
}
|
|
1404
|
-
]
|
|
1405
|
-
});
|
|
1406
|
-
|
|
1407
|
-
await sleep(100);
|
|
1408
|
-
|
|
1409
|
-
// Pop and complete the message
|
|
1410
|
-
const popResult = await client.pop({
|
|
1411
|
-
queue: 'test-retention-completed',
|
|
1412
|
-
batch: 1
|
|
1413
|
-
});
|
|
1414
|
-
|
|
1415
|
-
if (!popResult.messages || popResult.messages.length !== 1) {
|
|
1416
|
-
throw new Error('Failed to pop message for completion');
|
|
1417
|
-
}
|
|
1418
|
-
|
|
1419
|
-
await client.ack(popResult.messages[0].transactionId, 'completed');
|
|
1420
|
-
|
|
1421
|
-
// Verify message is marked as completed
|
|
1422
|
-
const statusCheck = await dbPool.query(`
|
|
1423
|
-
SELECT ms.status
|
|
1424
|
-
FROM queen.messages_status ms
|
|
1425
|
-
JOIN queen.messages m ON ms.message_id = m.id
|
|
1426
|
-
WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__'
|
|
1427
|
-
`, [popResult.messages[0].transactionId]);
|
|
1428
|
-
|
|
1429
|
-
if (!statusCheck.rows[0] || statusCheck.rows[0].status !== 'completed') {
|
|
1430
|
-
throw new Error('Message not marked as completed');
|
|
1431
|
-
}
|
|
1432
|
-
|
|
1433
|
-
// Wait for retention period
|
|
1434
|
-
log('Waiting for completed retention period (2 seconds)...', 'info');
|
|
1435
|
-
await sleep(2500);
|
|
1436
|
-
|
|
1437
|
-
// Trigger retention check
|
|
1438
|
-
await client.pop({
|
|
1439
|
-
queue: 'test-retention-completed',
|
|
1440
|
-
batch: 1
|
|
1441
|
-
});
|
|
1442
|
-
|
|
1443
|
-
// Check if completed message still exists
|
|
1444
|
-
const retainedCheck = await dbPool.query(`
|
|
1445
|
-
SELECT m.id, ms.status, ms.completed_at
|
|
1446
|
-
FROM queen.messages m
|
|
1447
|
-
LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
|
|
1448
|
-
WHERE m.transaction_id = $1
|
|
1449
|
-
`, [popResult.messages[0].transactionId]);
|
|
1450
|
-
|
|
1451
|
-
const messageExists = retainedCheck.rows.length > 0;
|
|
1452
|
-
const isCompleted = retainedCheck.rows[0]?.status === 'completed';
|
|
1453
|
-
|
|
1454
|
-
if (!messageExists) {
|
|
1455
|
-
throw new Error('Message was deleted, but should be retained with completed status');
|
|
1456
|
-
}
|
|
1457
|
-
|
|
1458
|
-
if (!isCompleted) {
|
|
1459
|
-
throw new Error('Message status is not completed');
|
|
1460
|
-
}
|
|
1461
|
-
|
|
1462
|
-
log(`Message retained with completed status`, 'info');
|
|
1463
|
-
passTest('Retention policy for completed messages works correctly');
|
|
1464
|
-
} catch (error) {
|
|
1465
|
-
failTest(error);
|
|
1466
|
-
}
|
|
1467
|
-
}
|
|
1468
|
-
|
|
1469
|
-
/**
|
|
1470
|
-
* Test 15: Partition Management
|
|
1471
|
-
*/
|
|
1472
|
-
async function testPartitionRetention() {
|
|
1473
|
-
startTest('Partition Management', 'enterprise');
|
|
1474
|
-
|
|
1475
|
-
try {
|
|
1476
|
-
// Configure queue
|
|
1477
|
-
await client.configure({
|
|
1478
|
-
queue: 'test-partition-management',
|
|
1479
|
-
options: {
|
|
1480
|
-
retentionEnabled: false
|
|
1481
|
-
}
|
|
1482
|
-
});
|
|
1483
|
-
|
|
1484
|
-
// Push a message to create Default partition
|
|
1485
|
-
await client.push({
|
|
1486
|
-
items: [{
|
|
1487
|
-
queue: 'test-partition-management',
|
|
1488
|
-
payload: { message: 'Test message' }
|
|
1489
|
-
}]
|
|
1490
|
-
});
|
|
1491
|
-
|
|
1492
|
-
// Verify partition exists
|
|
1493
|
-
const partitionCheck = await dbPool.query(`
|
|
1494
|
-
SELECT p.id, p.last_activity
|
|
1495
|
-
FROM queen.partitions p
|
|
1496
|
-
JOIN queen.queues q ON p.queue_id = q.id
|
|
1497
|
-
WHERE q.name = 'test-partition-management' AND p.name = 'Default'
|
|
1498
|
-
`);
|
|
1499
|
-
|
|
1500
|
-
if (partitionCheck.rows.length === 0) {
|
|
1501
|
-
throw new Error('Default partition not created');
|
|
1502
|
-
}
|
|
1503
|
-
|
|
1504
|
-
const partitionId = partitionCheck.rows[0].id;
|
|
1505
|
-
log(`Partition created with ID: ${partitionId}`, 'info');
|
|
1506
|
-
|
|
1507
|
-
// Push another message to update last_activity
|
|
1508
|
-
await client.push({
|
|
1509
|
-
items: [{
|
|
1510
|
-
queue: 'test-partition-management',
|
|
1511
|
-
payload: { message: 'Another message' }
|
|
1512
|
-
}]
|
|
1513
|
-
});
|
|
1514
|
-
|
|
1515
|
-
// Pop and complete the message
|
|
1516
|
-
const popResult = await client.pop({
|
|
1517
|
-
queue: 'test-partition-management',
|
|
1518
|
-
batch: 2
|
|
1519
|
-
});
|
|
1520
|
-
|
|
1521
|
-
if (popResult.messages && popResult.messages.length > 0) {
|
|
1522
|
-
await client.ack(popResult.messages[0].transactionId, 'completed');
|
|
1523
|
-
}
|
|
1524
|
-
|
|
1525
|
-
// Check partition still exists
|
|
1526
|
-
const stillExists = await dbPool.query(`
|
|
1527
|
-
SELECT COUNT(*) as count FROM queen.partitions WHERE id = $1
|
|
1528
|
-
`, [partitionId]);
|
|
1529
|
-
|
|
1530
|
-
if (stillExists.rows[0].count === '0') {
|
|
1531
|
-
throw new Error('Partition deleted too early');
|
|
1532
|
-
}
|
|
1533
|
-
|
|
1534
|
-
passTest('Partition management works correctly');
|
|
1535
|
-
} catch (error) {
|
|
1536
|
-
failTest(error);
|
|
1537
|
-
}
|
|
1538
|
-
}
|
|
1539
|
-
|
|
1540
|
-
/**
|
|
1541
|
-
* Test 16: Message Eviction
|
|
1542
|
-
*/
|
|
1543
|
-
async function testMessageEviction() {
|
|
1544
|
-
startTest('Message Eviction', 'enterprise');
|
|
1545
|
-
|
|
1546
|
-
try {
|
|
1547
|
-
// Configure queue with max wait time
|
|
1548
|
-
await client.configure({
|
|
1549
|
-
queue: 'test-eviction-queue',
|
|
1550
|
-
options: {
|
|
1551
|
-
maxWaitTimeSeconds: 3
|
|
1552
|
-
}
|
|
1553
|
-
});
|
|
1554
|
-
|
|
1555
|
-
// Push messages
|
|
1556
|
-
await client.push({
|
|
1557
|
-
items: [
|
|
1558
|
-
{
|
|
1559
|
-
queue: 'test-eviction-queue',
|
|
1560
|
-
partition: 'eviction-partition',
|
|
1561
|
-
payload: { message: 'Message to be evicted', timestamp: Date.now() }
|
|
1562
|
-
}
|
|
1563
|
-
]
|
|
1564
|
-
});
|
|
1565
|
-
|
|
1566
|
-
// Verify message exists
|
|
1567
|
-
const pendingCheck = await dbPool.query(`
|
|
1568
|
-
SELECT m.id, m.created_at, ms.status
|
|
1569
|
-
FROM queen.messages m
|
|
1570
|
-
JOIN queen.partitions p ON m.partition_id = p.id
|
|
1571
|
-
JOIN queen.queues q ON p.queue_id = q.id
|
|
1572
|
-
LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
|
|
1573
|
-
WHERE q.name = 'test-eviction-queue' AND p.name = 'eviction-partition'
|
|
1574
|
-
ORDER BY m.created_at DESC
|
|
1575
|
-
LIMIT 1
|
|
1576
|
-
`);
|
|
1577
|
-
|
|
1578
|
-
if (pendingCheck.rows.length === 0) {
|
|
1579
|
-
throw new Error('Message not found');
|
|
1580
|
-
}
|
|
1581
|
-
|
|
1582
|
-
const status = pendingCheck.rows[0].status || 'pending';
|
|
1583
|
-
if (status !== 'pending') {
|
|
1584
|
-
throw new Error(`Message not in pending status, got: ${status}`);
|
|
1585
|
-
}
|
|
1586
|
-
|
|
1587
|
-
// Wait for max wait time to expire
|
|
1588
|
-
log('Waiting for max wait time (3 seconds)...', 'info');
|
|
1589
|
-
await sleep(3500);
|
|
1590
|
-
|
|
1591
|
-
// Try to pop (should trigger eviction)
|
|
1592
|
-
const popResult = await client.pop({
|
|
1593
|
-
queue: 'test-eviction-queue',
|
|
1594
|
-
partition: 'eviction-partition',
|
|
1595
|
-
batch: 1
|
|
1596
|
-
});
|
|
1597
|
-
|
|
1598
|
-
// Check if message was evicted
|
|
1599
|
-
const evictionCheck = await dbPool.query(`
|
|
1600
|
-
SELECT m.id, ms.status, ms.error_message
|
|
1601
|
-
FROM queen.messages m
|
|
1602
|
-
JOIN queen.partitions p ON m.partition_id = p.id
|
|
1603
|
-
JOIN queen.queues q ON p.queue_id = q.id
|
|
1604
|
-
LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
|
|
1605
|
-
WHERE q.name = 'test-eviction-queue' AND p.name = 'eviction-partition'
|
|
1606
|
-
ORDER BY m.created_at DESC
|
|
1607
|
-
LIMIT 1
|
|
1608
|
-
`);
|
|
1609
|
-
|
|
1610
|
-
if (evictionCheck.rows.length > 0) {
|
|
1611
|
-
const status = evictionCheck.rows[0].status;
|
|
1612
|
-
const errorMsg = evictionCheck.rows[0].error_message;
|
|
1613
|
-
|
|
1614
|
-
if (status === 'evicted') {
|
|
1615
|
-
log(`Message evicted with reason: ${errorMsg}`, 'info');
|
|
1616
|
-
passTest('Message eviction works correctly');
|
|
1617
|
-
} else if (!status && (!popResult.messages || popResult.messages.length === 0)) {
|
|
1618
|
-
log('Message evicted (no status entry, no messages returned)', 'info');
|
|
1619
|
-
passTest('Message eviction works correctly');
|
|
1620
|
-
} else if ((status === 'pending' || !status) && (!popResult.messages || popResult.messages.length === 0)) {
|
|
1621
|
-
log('Message may be evicted by background service', 'info');
|
|
1622
|
-
passTest('Eviction configured successfully');
|
|
1623
|
-
} else {
|
|
1624
|
-
log(`Message status: ${status || 'no status entry'}`, 'warning');
|
|
1625
|
-
passTest('Eviction configuration accepted');
|
|
1626
|
-
}
|
|
1627
|
-
} else {
|
|
1628
|
-
passTest('Message processed or evicted');
|
|
1629
|
-
}
|
|
1630
|
-
} catch (error) {
|
|
1631
|
-
failTest(error);
|
|
1632
|
-
}
|
|
1633
|
-
}
|
|
1634
|
-
|
|
1635
|
-
/**
|
|
1636
|
-
* Test 17: Combined Enterprise Features
|
|
1637
|
-
*/
|
|
1638
|
-
async function testCombinedEnterpriseFeatures() {
|
|
1639
|
-
startTest('Combined Enterprise Features', 'enterprise');
|
|
1640
|
-
|
|
1641
|
-
try {
|
|
1642
|
-
const encryptionEnabled = !!process.env.QUEEN_ENCRYPTION_KEY;
|
|
1643
|
-
|
|
1644
|
-
// Configure queue with all enterprise features
|
|
1645
|
-
await client.configure({
|
|
1646
|
-
queue: 'test-enterprise-combined',
|
|
1647
|
-
options: {
|
|
1648
|
-
// Core options
|
|
1649
|
-
leaseTime: 300,
|
|
1650
|
-
retryLimit: 3,
|
|
1651
|
-
priority: 10,
|
|
1652
|
-
|
|
1653
|
-
// Enterprise options
|
|
1654
|
-
encryptionEnabled: encryptionEnabled,
|
|
1655
|
-
retentionEnabled: true,
|
|
1656
|
-
retentionSeconds: 60,
|
|
1657
|
-
completedRetentionSeconds: 30,
|
|
1658
|
-
partitionRetentionSeconds: 120
|
|
1659
|
-
}
|
|
1660
|
-
});
|
|
1661
|
-
|
|
1662
|
-
// Also set max wait time on the queue
|
|
1663
|
-
await dbPool.query(`
|
|
1664
|
-
UPDATE queen.queues
|
|
1665
|
-
SET max_wait_time_seconds = 45
|
|
1666
|
-
WHERE name = 'test-enterprise-combined'
|
|
1667
|
-
`);
|
|
1668
|
-
|
|
1669
|
-
// Verify all configurations
|
|
1670
|
-
const configCheck = await dbPool.query(`
|
|
1671
|
-
SELECT
|
|
1672
|
-
q.encryption_enabled,
|
|
1673
|
-
q.max_wait_time_seconds,
|
|
1674
|
-
q.retention_enabled,
|
|
1675
|
-
q.retention_seconds,
|
|
1676
|
-
q.completed_retention_seconds,
|
|
1677
|
-
q.lease_time,
|
|
1678
|
-
q.retry_limit,
|
|
1679
|
-
q.priority
|
|
1680
|
-
FROM queen.queues q
|
|
1681
|
-
WHERE q.name = 'test-enterprise-combined'
|
|
1682
|
-
`);
|
|
1683
|
-
|
|
1684
|
-
if (configCheck.rows.length === 0) {
|
|
1685
|
-
throw new Error('Configuration not found');
|
|
1686
|
-
}
|
|
1687
|
-
|
|
1688
|
-
const config = configCheck.rows[0];
|
|
1689
|
-
|
|
1690
|
-
// Verify enterprise configurations
|
|
1691
|
-
if (encryptionEnabled && config.encryption_enabled !== encryptionEnabled) {
|
|
1692
|
-
throw new Error('Encryption not configured correctly');
|
|
1693
|
-
}
|
|
1694
|
-
|
|
1695
|
-
if (config.max_wait_time_seconds !== 45) {
|
|
1696
|
-
throw new Error('Max wait time not configured correctly');
|
|
1697
|
-
}
|
|
1698
|
-
|
|
1699
|
-
if (!config.retention_enabled ||
|
|
1700
|
-
config.retention_seconds !== 60 ||
|
|
1701
|
-
config.completed_retention_seconds !== 30) {
|
|
1702
|
-
throw new Error('Retention options not configured correctly');
|
|
1703
|
-
}
|
|
1704
|
-
|
|
1705
|
-
// Push a test message
|
|
1706
|
-
await client.push({
|
|
1707
|
-
items: [{
|
|
1708
|
-
queue: 'test-enterprise-combined',
|
|
1709
|
-
payload: {
|
|
1710
|
-
message: 'Enterprise test message',
|
|
1711
|
-
sensitive: 'confidential-data',
|
|
1712
|
-
timestamp: Date.now()
|
|
1713
|
-
}
|
|
1714
|
-
}]
|
|
1715
|
-
});
|
|
1716
|
-
|
|
1717
|
-
await sleep(100);
|
|
1718
|
-
|
|
1719
|
-
// If encryption is enabled, verify message is encrypted in DB
|
|
1720
|
-
if (encryptionEnabled) {
|
|
1721
|
-
const encCheck = await dbPool.query(`
|
|
1722
|
-
SELECT is_encrypted FROM queen.messages m
|
|
1723
|
-
JOIN queen.partitions p ON m.partition_id = p.id
|
|
1724
|
-
JOIN queen.queues q ON p.queue_id = q.id
|
|
1725
|
-
WHERE q.name = 'test-enterprise-combined'
|
|
1726
|
-
ORDER BY m.created_at DESC
|
|
1727
|
-
LIMIT 1
|
|
1728
|
-
`);
|
|
1729
|
-
|
|
1730
|
-
if (encCheck.rows[0] && !encCheck.rows[0].is_encrypted) {
|
|
1731
|
-
throw new Error('Message not encrypted despite encryption being enabled');
|
|
1732
|
-
}
|
|
1733
|
-
}
|
|
1734
|
-
|
|
1735
|
-
// Pop and complete the message
|
|
1736
|
-
const popResult = await client.pop({
|
|
1737
|
-
queue: 'test-enterprise-combined',
|
|
1738
|
-
batch: 1
|
|
1739
|
-
});
|
|
1740
|
-
|
|
1741
|
-
if (popResult.messages && popResult.messages.length > 0) {
|
|
1742
|
-
await client.ack(popResult.messages[0].transactionId, 'completed');
|
|
1743
|
-
}
|
|
1744
|
-
|
|
1745
|
-
passTest('All enterprise features can be configured together successfully');
|
|
1746
|
-
} catch (error) {
|
|
1747
|
-
failTest(error);
|
|
1748
|
-
}
|
|
1749
|
-
}
|
|
1750
|
-
|
|
1751
|
-
/**
|
|
1752
|
-
* Test 18: Consumer Encryption/Decryption
|
|
1753
|
-
*/
|
|
1754
|
-
async function testConsumerEncryption() {
|
|
1755
|
-
startTest('Consumer Encryption/Decryption', 'enterprise');
|
|
1756
|
-
|
|
1757
|
-
try {
|
|
1758
|
-
// Check if encryption is enabled
|
|
1759
|
-
const encryptionKey = process.env.QUEEN_ENCRYPTION_KEY;
|
|
1760
|
-
if (!encryptionKey) {
|
|
1761
|
-
log('Skipping consumer encryption test - QUEEN_ENCRYPTION_KEY not set', 'warning');
|
|
1762
|
-
passTest('Skipped - encryption key not configured');
|
|
1763
|
-
return;
|
|
1764
|
-
}
|
|
1765
|
-
|
|
1766
|
-
// Configure queue with encryption
|
|
1767
|
-
await client.configure({
|
|
1768
|
-
queue: 'test-consumer-encryption',
|
|
1769
|
-
options: {
|
|
1770
|
-
encryptionEnabled: true
|
|
1771
|
-
}
|
|
1772
|
-
});
|
|
1773
|
-
|
|
1774
|
-
// Push sensitive data
|
|
1775
|
-
const sensitiveData = {
|
|
1776
|
-
creditCard: '4111111111111111',
|
|
1777
|
-
ssn: '123-45-6789',
|
|
1778
|
-
apiKey: 'sk_test_12345',
|
|
1779
|
-
timestamp: Date.now()
|
|
1780
|
-
};
|
|
1781
|
-
|
|
1782
|
-
await client.push({
|
|
1783
|
-
items: [{
|
|
1784
|
-
queue: 'test-consumer-encryption',
|
|
1785
|
-
payload: sensitiveData
|
|
1786
|
-
}]
|
|
1787
|
-
});
|
|
1788
|
-
|
|
1789
|
-
await sleep(100);
|
|
1790
|
-
|
|
1791
|
-
// Test batch consumer receives decrypted data
|
|
1792
|
-
let batchDecrypted = false;
|
|
1793
|
-
const stopBatchConsumer = client.consume({
|
|
1794
|
-
queue: 'test-consumer-encryption',
|
|
1795
|
-
handlerBatch: async (messages) => {
|
|
1796
|
-
if (messages.length > 0) {
|
|
1797
|
-
const message = messages[0];
|
|
1798
|
-
// Check if data is properly decrypted
|
|
1799
|
-
if (message.data && typeof message.data === 'object') {
|
|
1800
|
-
if (message.data.encrypted || message.data.iv || message.data.authTag) {
|
|
1801
|
-
throw new Error('Batch consumer received encrypted data instead of decrypted!');
|
|
1802
|
-
}
|
|
1803
|
-
if (message.data.creditCard === sensitiveData.creditCard &&
|
|
1804
|
-
message.data.ssn === sensitiveData.ssn) {
|
|
1805
|
-
batchDecrypted = true;
|
|
1806
|
-
}
|
|
1807
|
-
}
|
|
1808
|
-
}
|
|
1809
|
-
stopBatchConsumer();
|
|
1810
|
-
},
|
|
1811
|
-
options: {
|
|
1812
|
-
batch: 1,
|
|
1813
|
-
wait: false,
|
|
1814
|
-
stopOnError: true
|
|
1815
|
-
}
|
|
1816
|
-
});
|
|
1817
|
-
|
|
1818
|
-
// Wait for consumption
|
|
1819
|
-
await sleep(1000);
|
|
1820
|
-
stopBatchConsumer();
|
|
1821
|
-
|
|
1822
|
-
if (!batchDecrypted) {
|
|
1823
|
-
// Try direct pop to verify
|
|
1824
|
-
const popResult = await client.pop({
|
|
1825
|
-
queue: 'test-consumer-encryption',
|
|
1826
|
-
batch: 1
|
|
1827
|
-
});
|
|
1828
|
-
|
|
1829
|
-
if (popResult.messages && popResult.messages.length > 0) {
|
|
1830
|
-
const message = popResult.messages[0];
|
|
1831
|
-
if (message.data.encrypted || message.data.iv || message.data.authTag) {
|
|
1832
|
-
throw new Error('Pop returned encrypted data instead of decrypted!');
|
|
1833
|
-
}
|
|
1834
|
-
if (message.data.creditCard !== sensitiveData.creditCard) {
|
|
1835
|
-
throw new Error('Decrypted data does not match original');
|
|
1836
|
-
}
|
|
1837
|
-
await client.ack(message.transactionId, 'completed');
|
|
1838
|
-
}
|
|
1839
|
-
}
|
|
1840
|
-
|
|
1841
|
-
passTest('Consumer receives properly decrypted messages');
|
|
1842
|
-
} catch (error) {
|
|
1843
|
-
failTest(error);
|
|
1844
|
-
}
|
|
1845
|
-
}
|
|
1846
|
-
|
|
1847
|
-
/**
|
|
1848
|
-
* Test 19: Bus Mode - Consumer Groups
|
|
1849
|
-
*/
|
|
1850
|
-
async function testBusConsumerGroups() {
|
|
1851
|
-
startTest('Bus Mode - Consumer Groups', 'enterprise');
|
|
1852
|
-
|
|
1853
|
-
try {
|
|
1854
|
-
const queue = 'test-bus-mode';
|
|
1855
|
-
|
|
1856
|
-
// Configure queue
|
|
1857
|
-
await client.configure({
|
|
1858
|
-
queue: queue,
|
|
1859
|
-
options: {}
|
|
1860
|
-
});
|
|
1861
|
-
|
|
1862
|
-
// Push test messages
|
|
1863
|
-
const messages = [];
|
|
1864
|
-
for (let i = 1; i <= 5; i++) {
|
|
1865
|
-
messages.push({
|
|
1866
|
-
queue,
|
|
1867
|
-
payload: { id: i, data: `Message ${i}` }
|
|
1868
|
-
});
|
|
1869
|
-
}
|
|
1870
|
-
|
|
1871
|
-
await client.push({ items: messages });
|
|
1872
|
-
await sleep(100);
|
|
1873
|
-
|
|
1874
|
-
// Consumer Group 1: Pop messages
|
|
1875
|
-
const group1Result = await client.pop({
|
|
1876
|
-
queue,
|
|
1877
|
-
consumerGroup: 'group1',
|
|
1878
|
-
batch: 3
|
|
1879
|
-
});
|
|
1880
|
-
|
|
1881
|
-
if (!group1Result.messages || group1Result.messages.length !== 3) {
|
|
1882
|
-
throw new Error(`Group1 expected 3 messages, got ${group1Result.messages?.length}`);
|
|
1883
|
-
}
|
|
1884
|
-
|
|
1885
|
-
// Consumer Group 2: Should get the same messages
|
|
1886
|
-
const group2Result = await client.pop({
|
|
1887
|
-
queue,
|
|
1888
|
-
consumerGroup: 'group2',
|
|
1889
|
-
batch: 3
|
|
1890
|
-
});
|
|
1891
|
-
|
|
1892
|
-
if (!group2Result.messages || group2Result.messages.length !== 3) {
|
|
1893
|
-
throw new Error(`Group2 expected 3 messages, got ${group2Result.messages?.length}`);
|
|
1894
|
-
}
|
|
1895
|
-
|
|
1896
|
-
// Verify both groups got the same message IDs
|
|
1897
|
-
const group1Ids = group1Result.messages.map(m => m.payload.id).sort();
|
|
1898
|
-
const group2Ids = group2Result.messages.map(m => m.payload.id).sort();
|
|
1899
|
-
|
|
1900
|
-
if (JSON.stringify(group1Ids) !== JSON.stringify(group2Ids)) {
|
|
1901
|
-
throw new Error('Consumer groups did not receive the same messages');
|
|
1902
|
-
}
|
|
1903
|
-
|
|
1904
|
-
// Acknowledge messages for both groups
|
|
1905
|
-
for (const msg of group1Result.messages) {
|
|
1906
|
-
await client.ack(msg.transactionId, 'completed', null, 'group1');
|
|
1907
|
-
}
|
|
1908
|
-
|
|
1909
|
-
for (const msg of group2Result.messages) {
|
|
1910
|
-
await client.ack(msg.transactionId, 'completed', null, 'group2');
|
|
1911
|
-
}
|
|
1912
|
-
|
|
1913
|
-
// Pop remaining messages
|
|
1914
|
-
const group1Remaining = await client.pop({
|
|
1915
|
-
queue,
|
|
1916
|
-
consumerGroup: 'group1',
|
|
1917
|
-
batch: 5
|
|
1918
|
-
});
|
|
1919
|
-
|
|
1920
|
-
if (group1Remaining.messages.length !== 2) {
|
|
1921
|
-
throw new Error(`Expected 2 remaining messages for group1, got ${group1Remaining.messages.length}`);
|
|
1922
|
-
}
|
|
1923
|
-
|
|
1924
|
-
passTest('Consumer groups receive all messages independently');
|
|
1925
|
-
} catch (error) {
|
|
1926
|
-
failTest(error);
|
|
1927
|
-
}
|
|
1928
|
-
}
|
|
1929
|
-
|
|
1930
|
-
/**
|
|
1931
|
-
* Test 20: Mixed Mode - Queue and Bus Together
|
|
1932
|
-
*/
|
|
1933
|
-
async function testMixedMode() {
|
|
1934
|
-
startTest('Mixed Mode - Queue and Bus Together', 'enterprise');
|
|
1935
|
-
|
|
1936
|
-
try {
|
|
1937
|
-
const queue = 'test-mixed-mode';
|
|
1938
|
-
|
|
1939
|
-
// Configure queue
|
|
1940
|
-
await client.configure({
|
|
1941
|
-
queue: queue,
|
|
1942
|
-
options: {}
|
|
1943
|
-
});
|
|
1944
|
-
|
|
1945
|
-
// Push test messages
|
|
1946
|
-
const messages = [];
|
|
1947
|
-
for (let i = 1; i <= 6; i++) {
|
|
1948
|
-
messages.push({
|
|
1949
|
-
queue,
|
|
1950
|
-
payload: { id: i, data: `Mixed message ${i}` }
|
|
1951
|
-
});
|
|
1952
|
-
}
|
|
1953
|
-
|
|
1954
|
-
await client.push({ items: messages });
|
|
1955
|
-
await sleep(100);
|
|
1956
|
-
|
|
1957
|
-
// Queue Mode: Workers compete for messages
|
|
1958
|
-
const worker1 = await client.pop({ queue, batch: 2 });
|
|
1959
|
-
const worker2 = await client.pop({ queue, batch: 2 });
|
|
1960
|
-
|
|
1961
|
-
if (!worker1.messages || !worker2.messages) {
|
|
1962
|
-
throw new Error('Workers did not receive messages');
|
|
1963
|
-
}
|
|
1964
|
-
|
|
1965
|
-
// Workers should get different messages (competing)
|
|
1966
|
-
const worker1Ids = worker1.messages.map(m => m.payload.id);
|
|
1967
|
-
const worker2Ids = worker2.messages.map(m => m.payload.id);
|
|
1968
|
-
const intersection = worker1Ids.filter(id => worker2Ids.includes(id));
|
|
1969
|
-
|
|
1970
|
-
if (intersection.length > 0) {
|
|
1971
|
-
throw new Error('Workers received the same messages - should be competing');
|
|
1972
|
-
}
|
|
1973
|
-
|
|
1974
|
-
// Bus Mode: Consumer groups get all messages
|
|
1975
|
-
const analyticsGroup = await client.pop({
|
|
1976
|
-
queue,
|
|
1977
|
-
consumerGroup: 'analytics',
|
|
1978
|
-
batch: 10
|
|
1979
|
-
});
|
|
1980
|
-
|
|
1981
|
-
const auditGroup = await client.pop({
|
|
1982
|
-
queue,
|
|
1983
|
-
consumerGroup: 'audit',
|
|
1984
|
-
batch: 10
|
|
1985
|
-
});
|
|
1986
|
-
|
|
1987
|
-
// Both groups should get all 6 messages
|
|
1988
|
-
if (analyticsGroup.messages.length !== 6) {
|
|
1989
|
-
throw new Error(`Analytics group expected 6 messages, got ${analyticsGroup.messages.length}`);
|
|
1990
|
-
}
|
|
1991
|
-
|
|
1992
|
-
if (auditGroup.messages.length !== 6) {
|
|
1993
|
-
throw new Error(`Audit group expected 6 messages, got ${auditGroup.messages.length}`);
|
|
1994
|
-
}
|
|
1995
|
-
|
|
1996
|
-
// Verify groups got all messages
|
|
1997
|
-
const analyticsIds = analyticsGroup.messages.map(m => m.payload.id).sort();
|
|
1998
|
-
const auditIds = auditGroup.messages.map(m => m.payload.id).sort();
|
|
1999
|
-
|
|
2000
|
-
if (JSON.stringify(analyticsIds) !== JSON.stringify([1, 2, 3, 4, 5, 6])) {
|
|
2001
|
-
throw new Error('Analytics group did not receive all messages');
|
|
2002
|
-
}
|
|
2003
|
-
|
|
2004
|
-
if (JSON.stringify(auditIds) !== JSON.stringify([1, 2, 3, 4, 5, 6])) {
|
|
2005
|
-
throw new Error('Audit group did not receive all messages');
|
|
2006
|
-
}
|
|
2007
|
-
|
|
2008
|
-
passTest('Mixed mode works: workers compete, groups get all messages');
|
|
2009
|
-
} catch (error) {
|
|
2010
|
-
failTest(error);
|
|
2011
|
-
}
|
|
2012
|
-
}
|
|
2013
|
-
|
|
2014
|
-
/**
|
|
2015
|
-
* Test 21: Consumer Group Subscription Modes
|
|
2016
|
-
*/
|
|
2017
|
-
async function testConsumerGroupSubscriptionModes() {
|
|
2018
|
-
startTest('Consumer Group Subscription Modes', 'enterprise');
|
|
2019
|
-
|
|
2020
|
-
try {
|
|
2021
|
-
const queue = 'test-subscription-modes';
|
|
2022
|
-
|
|
2023
|
-
// Configure queue
|
|
2024
|
-
await client.configure({
|
|
2025
|
-
queue: queue,
|
|
2026
|
-
options: {}
|
|
2027
|
-
});
|
|
2028
|
-
|
|
2029
|
-
// Push historical messages
|
|
2030
|
-
const historicalMessages = [];
|
|
2031
|
-
for (let i = 1; i <= 3; i++) {
|
|
2032
|
-
historicalMessages.push({
|
|
2033
|
-
queue,
|
|
2034
|
-
payload: { id: i, type: 'historical', data: `Historical ${i}` }
|
|
2035
|
-
});
|
|
2036
|
-
}
|
|
2037
|
-
|
|
2038
|
-
await client.push({ items: historicalMessages });
|
|
2039
|
-
await sleep(100);
|
|
2040
|
-
|
|
2041
|
-
// Group 1: Default mode (consume all)
|
|
2042
|
-
const allMessagesGroup = await client.pop({
|
|
2043
|
-
queue,
|
|
2044
|
-
consumerGroup: 'all-messages',
|
|
2045
|
-
batch: 10
|
|
2046
|
-
});
|
|
2047
|
-
|
|
2048
|
-
if (allMessagesGroup.messages.length !== 3) {
|
|
2049
|
-
throw new Error(`All-messages group expected 3 messages, got ${allMessagesGroup.messages.length}`);
|
|
2050
|
-
}
|
|
2051
|
-
|
|
2052
|
-
// ACK messages to release partition lock for all-messages group
|
|
2053
|
-
for (const msg of allMessagesGroup.messages) {
|
|
2054
|
-
await client.ack(msg.transactionId, 'completed', null, 'all-messages');
|
|
2055
|
-
}
|
|
2056
|
-
|
|
2057
|
-
// Group 2: New messages only (subscription mode = 'new')
|
|
2058
|
-
const newOnlyResult = await client.pop({
|
|
2059
|
-
queue,
|
|
2060
|
-
consumerGroup: 'new-only',
|
|
2061
|
-
subscriptionMode: 'new',
|
|
2062
|
-
batch: 10
|
|
2063
|
-
});
|
|
2064
|
-
|
|
2065
|
-
if (newOnlyResult.messages.length !== 0) {
|
|
2066
|
-
throw new Error(`New-only group should get 0 historical messages, got ${newOnlyResult.messages.length}`);
|
|
2067
|
-
}
|
|
2068
|
-
|
|
2069
|
-
// Wait to ensure subscription timestamp is in the past
|
|
2070
|
-
await sleep(1000);
|
|
2071
|
-
|
|
2072
|
-
// Push new messages
|
|
2073
|
-
const newMessages = [];
|
|
2074
|
-
for (let i = 4; i <= 6; i++) {
|
|
2075
|
-
newMessages.push({
|
|
2076
|
-
queue,
|
|
2077
|
-
payload: { id: i, type: 'new', data: `New ${i}` }
|
|
2078
|
-
});
|
|
2079
|
-
}
|
|
2080
|
-
|
|
2081
|
-
await client.push({ items: newMessages });
|
|
2082
|
-
await sleep(100);
|
|
2083
|
-
|
|
2084
|
-
// All-messages group should get the new messages
|
|
2085
|
-
const allMessagesNew = await client.pop({
|
|
2086
|
-
queue,
|
|
2087
|
-
consumerGroup: 'all-messages',
|
|
2088
|
-
batch: 10
|
|
2089
|
-
});
|
|
2090
|
-
|
|
2091
|
-
if (allMessagesNew.messages.length !== 3) {
|
|
2092
|
-
throw new Error(`All-messages group expected 3 new messages, got ${allMessagesNew.messages.length}`);
|
|
2093
|
-
}
|
|
2094
|
-
|
|
2095
|
-
// ACK messages to release partition lock for all-messages group
|
|
2096
|
-
for (const msg of allMessagesNew.messages) {
|
|
2097
|
-
await client.ack(msg.transactionId, 'completed', null, 'all-messages');
|
|
2098
|
-
}
|
|
2099
|
-
|
|
2100
|
-
// New-only group should also get the new messages
|
|
2101
|
-
const newOnlyNew = await client.pop({
|
|
2102
|
-
queue,
|
|
2103
|
-
consumerGroup: 'new-only',
|
|
2104
|
-
batch: 10
|
|
2105
|
-
});
|
|
2106
|
-
|
|
2107
|
-
if (newOnlyNew.messages.length !== 3) {
|
|
2108
|
-
throw new Error(`New-only group expected 3 new messages, got ${newOnlyNew.messages.length}`);
|
|
2109
|
-
}
|
|
2110
|
-
|
|
2111
|
-
passTest('Consumer group subscription modes work correctly');
|
|
2112
|
-
} catch (error) {
|
|
2113
|
-
failTest(error);
|
|
2114
|
-
}
|
|
2115
|
-
}
|
|
2116
|
-
|
|
2117
|
-
/**
|
|
2118
|
-
* Test 22: Consumer Group Isolation
|
|
2119
|
-
*/
|
|
2120
|
-
async function testConsumerGroupIsolation() {
|
|
2121
|
-
startTest('Consumer Group Isolation', 'enterprise');
|
|
2122
|
-
|
|
2123
|
-
try {
|
|
2124
|
-
const queue = 'test-group-isolation';
|
|
2125
|
-
|
|
2126
|
-
// Configure queue
|
|
2127
|
-
await client.configure({
|
|
2128
|
-
queue: queue,
|
|
2129
|
-
options: {}
|
|
2130
|
-
});
|
|
2131
|
-
|
|
2132
|
-
// Push messages
|
|
2133
|
-
const messages = [];
|
|
2134
|
-
for (let i = 1; i <= 4; i++) {
|
|
2135
|
-
messages.push({
|
|
2136
|
-
queue,
|
|
2137
|
-
payload: { id: i, data: `Isolation test ${i}` }
|
|
2138
|
-
});
|
|
2139
|
-
}
|
|
2140
|
-
|
|
2141
|
-
await client.push({ items: messages });
|
|
2142
|
-
await sleep(100);
|
|
2143
|
-
|
|
2144
|
-
// Group 1: Pop and complete some messages
|
|
2145
|
-
const group1Result = await client.pop({
|
|
2146
|
-
queue,
|
|
2147
|
-
consumerGroup: 'group1',
|
|
2148
|
-
batch: 2
|
|
2149
|
-
});
|
|
2150
|
-
|
|
2151
|
-
if (group1Result.messages.length !== 2) {
|
|
2152
|
-
throw new Error(`Group1 expected 2 messages, got ${group1Result.messages.length}`);
|
|
2153
|
-
}
|
|
2154
|
-
|
|
2155
|
-
// Acknowledge messages for group1 (but don't release the partition lock yet)
|
|
2156
|
-
for (const msg of group1Result.messages) {
|
|
2157
|
-
await client.ack(msg.transactionId, 'completed', null, 'group1');
|
|
2158
|
-
}
|
|
2159
|
-
|
|
2160
|
-
// Group 2: Should still be able to get all messages (different consumer group can have its own partition lock)
|
|
2161
|
-
const group2Result = await client.pop({
|
|
2162
|
-
queue,
|
|
2163
|
-
consumerGroup: 'group2',
|
|
2164
|
-
batch: 10
|
|
2165
|
-
});
|
|
2166
|
-
|
|
2167
|
-
if (group2Result.messages.length !== 4) {
|
|
2168
|
-
throw new Error(`Group2 expected all 4 messages, got ${group2Result.messages.length}`);
|
|
2169
|
-
}
|
|
2170
|
-
|
|
2171
|
-
// Group 1: Pop remaining messages
|
|
2172
|
-
const group1Remaining = await client.pop({
|
|
2173
|
-
queue,
|
|
2174
|
-
consumerGroup: 'group1',
|
|
2175
|
-
batch: 10
|
|
2176
|
-
});
|
|
2177
|
-
|
|
2178
|
-
if (group1Remaining.messages.length !== 2) {
|
|
2179
|
-
throw new Error(`Group1 expected 2 remaining messages, got ${group1Remaining.messages.length}`);
|
|
2180
|
-
}
|
|
2181
|
-
|
|
2182
|
-
// Verify isolation: Group2's status doesn't affect Group1
|
|
2183
|
-
for (const msg of group2Result.messages.slice(0, 2)) {
|
|
2184
|
-
await client.ack(msg.transactionId, 'failed', 'Test failure', 'group2');
|
|
2185
|
-
}
|
|
2186
|
-
|
|
2187
|
-
// Group1 should still see its messages as completed
|
|
2188
|
-
const statusCheck = await dbPool.query(`
|
|
2189
|
-
SELECT ms.status, ms.consumer_group
|
|
2190
|
-
FROM queen.messages_status ms
|
|
2191
|
-
JOIN queen.messages m ON ms.message_id = m.id
|
|
2192
|
-
WHERE m.transaction_id = $1
|
|
2193
|
-
ORDER BY ms.consumer_group
|
|
2194
|
-
`, [group1Result.messages[0].transactionId]);
|
|
2195
|
-
|
|
2196
|
-
const group1Status = statusCheck.rows.find(r => r.consumer_group === 'group1');
|
|
2197
|
-
const group2Status = statusCheck.rows.find(r => r.consumer_group === 'group2');
|
|
2198
|
-
|
|
2199
|
-
if (group1Status?.status !== 'completed') {
|
|
2200
|
-
throw new Error('Group1 status should be completed');
|
|
2201
|
-
}
|
|
2202
|
-
|
|
2203
|
-
if (group2Status?.status !== 'failed') {
|
|
2204
|
-
throw new Error('Group2 status should be failed');
|
|
2205
|
-
}
|
|
2206
|
-
|
|
2207
|
-
passTest('Consumer groups are properly isolated');
|
|
2208
|
-
} catch (error) {
|
|
2209
|
-
failTest(error);
|
|
2210
|
-
}
|
|
2211
|
-
}
|
|
2212
|
-
|
|
2213
|
-
/**
|
|
2214
|
-
* Test 23: Enterprise Error Handling
|
|
2215
|
-
*/
|
|
2216
|
-
async function testEnterpriseErrorHandling() {
|
|
2217
|
-
startTest('Enterprise Error Handling', 'enterprise');
|
|
2218
|
-
|
|
2219
|
-
try {
|
|
2220
|
-
let errorsCaught = 0;
|
|
2221
|
-
|
|
2222
|
-
// Test 1: Invalid retention seconds
|
|
2223
|
-
try {
|
|
2224
|
-
await client.configure({
|
|
2225
|
-
queue: 'test-enterprise-errors',
|
|
2226
|
-
options: {
|
|
2227
|
-
retentionEnabled: true,
|
|
2228
|
-
retentionSeconds: -10 // Invalid negative value
|
|
2229
|
-
}
|
|
2230
|
-
});
|
|
2231
|
-
errorsCaught++;
|
|
2232
|
-
} catch (error) {
|
|
2233
|
-
errorsCaught++;
|
|
2234
|
-
}
|
|
2235
|
-
|
|
2236
|
-
// Test 2: Configure encryption without key (if key not set)
|
|
2237
|
-
if (!process.env.QUEEN_ENCRYPTION_KEY) {
|
|
2238
|
-
try {
|
|
2239
|
-
await client.configure({
|
|
2240
|
-
queue: 'test-enterprise-no-key',
|
|
2241
|
-
options: {
|
|
2242
|
-
encryptionEnabled: true
|
|
2243
|
-
}
|
|
2244
|
-
});
|
|
2245
|
-
|
|
2246
|
-
// Try to push encrypted message without key
|
|
2247
|
-
await client.push({
|
|
2248
|
-
items: [{
|
|
2249
|
-
queue: 'test-enterprise-no-key',
|
|
2250
|
-
payload: { message: 'Should fail without key' }
|
|
2251
|
-
}]
|
|
2252
|
-
});
|
|
2253
|
-
|
|
2254
|
-
errorsCaught++;
|
|
2255
|
-
} catch (error) {
|
|
2256
|
-
errorsCaught++;
|
|
2257
|
-
}
|
|
2258
|
-
} else {
|
|
2259
|
-
errorsCaught++; // Count as handled
|
|
2260
|
-
}
|
|
2261
|
-
|
|
2262
|
-
// Test 3: Invalid max wait time
|
|
2263
|
-
try {
|
|
2264
|
-
await dbPool.query(`
|
|
2265
|
-
INSERT INTO queen.queues (name, max_wait_time_seconds)
|
|
2266
|
-
VALUES ('test-invalid-wait', -100)
|
|
2267
|
-
ON CONFLICT (name) DO UPDATE SET max_wait_time_seconds = EXCLUDED.max_wait_time_seconds
|
|
2268
|
-
`);
|
|
2269
|
-
errorsCaught++;
|
|
2270
|
-
} catch (error) {
|
|
2271
|
-
errorsCaught++;
|
|
2272
|
-
}
|
|
2273
|
-
|
|
2274
|
-
if (errorsCaught >= 2) {
|
|
2275
|
-
passTest(`Enterprise error handling works (${errorsCaught} scenarios handled)`);
|
|
2276
|
-
} else {
|
|
2277
|
-
throw new Error(`Only ${errorsCaught} error scenarios handled`);
|
|
2278
|
-
}
|
|
2279
|
-
} catch (error) {
|
|
2280
|
-
failTest(error);
|
|
2281
|
-
}
|
|
2282
|
-
}
|
|
2283
|
-
|
|
2284
|
-
// ============================================
|
|
2285
|
-
// EDGE CASE TESTS
|
|
2286
|
-
// ============================================
|
|
2287
|
-
/**
|
|
2288
|
-
* Test: Empty and null payloads
|
|
2289
|
-
*/
|
|
2290
|
-
async function testEmptyAndNullPayloads() {
|
|
2291
|
-
startTest('Empty and Null Payloads', 'edge');
|
|
2292
|
-
|
|
2293
|
-
try {
|
|
2294
|
-
const queue = 'edge-empty-payloads';
|
|
2295
|
-
|
|
2296
|
-
// Configure queue
|
|
2297
|
-
await client.configure({
|
|
2298
|
-
queue: queue,
|
|
2299
|
-
options: {}
|
|
2300
|
-
});
|
|
2301
|
-
|
|
2302
|
-
// Test null payload
|
|
2303
|
-
await client.push({
|
|
2304
|
-
items: [{
|
|
2305
|
-
queue,
|
|
2306
|
-
payload: null
|
|
2307
|
-
}]
|
|
2308
|
-
});
|
|
2309
|
-
|
|
2310
|
-
// Test undefined payload (should be treated as null)
|
|
2311
|
-
await client.push({
|
|
2312
|
-
items: [{
|
|
2313
|
-
queue
|
|
2314
|
-
// payload is undefined
|
|
2315
|
-
}]
|
|
2316
|
-
});
|
|
2317
|
-
|
|
2318
|
-
// Test empty object
|
|
2319
|
-
await client.push({
|
|
2320
|
-
items: [{
|
|
2321
|
-
queue,
|
|
2322
|
-
payload: {}
|
|
2323
|
-
}]
|
|
2324
|
-
});
|
|
2325
|
-
|
|
2326
|
-
// Test empty string
|
|
2327
|
-
await client.push({
|
|
2328
|
-
items: [{
|
|
2329
|
-
queue,
|
|
2330
|
-
payload: ''
|
|
2331
|
-
}]
|
|
2332
|
-
});
|
|
2333
|
-
|
|
2334
|
-
// Pop and verify all messages
|
|
2335
|
-
const result = await client.pop({
|
|
2336
|
-
queue,
|
|
2337
|
-
batch: 10
|
|
2338
|
-
});
|
|
2339
|
-
|
|
2340
|
-
if (!result.messages || result.messages.length !== 4) {
|
|
2341
|
-
throw new Error(`Expected 4 messages, got ${result.messages?.length}`);
|
|
2342
|
-
}
|
|
2343
|
-
|
|
2344
|
-
// Verify payloads
|
|
2345
|
-
const payloads = result.messages.map(m => m.data);
|
|
2346
|
-
|
|
2347
|
-
// Check for null (first two should be null)
|
|
2348
|
-
if (payloads[0] !== null) {
|
|
2349
|
-
throw new Error(`Expected null for first payload, got ${JSON.stringify(payloads[0])}`);
|
|
2350
|
-
}
|
|
2351
|
-
|
|
2352
|
-
if (payloads[1] !== null) {
|
|
2353
|
-
throw new Error(`Expected null for second payload, got ${JSON.stringify(payloads[1])}`);
|
|
2354
|
-
}
|
|
2355
|
-
|
|
2356
|
-
// Empty object
|
|
2357
|
-
if (typeof payloads[2] !== 'object' || Object.keys(payloads[2]).length !== 0) {
|
|
2358
|
-
throw new Error(`Expected empty object, got ${JSON.stringify(payloads[2])}`);
|
|
2359
|
-
}
|
|
2360
|
-
|
|
2361
|
-
// Empty string
|
|
2362
|
-
if (payloads[3] !== '') {
|
|
2363
|
-
throw new Error(`Expected empty string, got ${JSON.stringify(payloads[3])}`);
|
|
2364
|
-
}
|
|
2365
|
-
|
|
2366
|
-
// Acknowledge all
|
|
2367
|
-
for (const msg of result.messages) {
|
|
2368
|
-
await client.ack(msg.transactionId, 'completed');
|
|
2369
|
-
}
|
|
2370
|
-
|
|
2371
|
-
passTest('Empty and null payloads handled correctly');
|
|
2372
|
-
} catch (error) {
|
|
2373
|
-
failTest(error);
|
|
2374
|
-
}
|
|
2375
|
-
}
|
|
2376
|
-
|
|
2377
|
-
/**
|
|
2378
|
-
* Test: Very large payloads
|
|
2379
|
-
*/
|
|
2380
|
-
async function testVeryLargePayloads() {
|
|
2381
|
-
startTest('Very Large Payloads', 'edge');
|
|
2382
|
-
|
|
2383
|
-
try {
|
|
2384
|
-
const queue = 'edge-large-payloads';
|
|
2385
|
-
|
|
2386
|
-
// Configure queue
|
|
2387
|
-
await client.configure({
|
|
2388
|
-
queue: queue,
|
|
2389
|
-
options: {}
|
|
2390
|
-
});
|
|
2391
|
-
|
|
2392
|
-
// Create a large payload (1MB of data)
|
|
2393
|
-
const largeArray = new Array(10000).fill({
|
|
2394
|
-
id: crypto.randomBytes(16).toString('hex'),
|
|
2395
|
-
data: 'x'.repeat(100),
|
|
2396
|
-
nested: {
|
|
2397
|
-
field1: 'value1',
|
|
2398
|
-
field2: 'value2',
|
|
2399
|
-
field3: 'value3'
|
|
2400
|
-
}
|
|
2401
|
-
});
|
|
2402
|
-
|
|
2403
|
-
const largePayload = {
|
|
2404
|
-
array: largeArray,
|
|
2405
|
-
metadata: {
|
|
2406
|
-
size: JSON.stringify(largeArray).length,
|
|
2407
|
-
timestamp: Date.now()
|
|
2408
|
-
}
|
|
2409
|
-
};
|
|
2410
|
-
|
|
2411
|
-
// Push large message
|
|
2412
|
-
await client.push({
|
|
2413
|
-
items: [{
|
|
2414
|
-
queue,
|
|
2415
|
-
payload: largePayload
|
|
2416
|
-
}]
|
|
2417
|
-
});
|
|
2418
|
-
|
|
2419
|
-
// Pop and verify
|
|
2420
|
-
const result = await client.pop({
|
|
2421
|
-
queue,
|
|
2422
|
-
batch: 1
|
|
2423
|
-
});
|
|
2424
|
-
|
|
2425
|
-
if (!result.messages || result.messages.length !== 1) {
|
|
2426
|
-
throw new Error('Failed to pop large message');
|
|
2427
|
-
}
|
|
2428
|
-
|
|
2429
|
-
const received = result.messages[0].data;
|
|
2430
|
-
if (!received.array || received.array.length !== 10000) {
|
|
2431
|
-
throw new Error('Large payload corrupted');
|
|
2432
|
-
}
|
|
2433
|
-
|
|
2434
|
-
await client.ack(result.messages[0].transactionId, 'completed');
|
|
2435
|
-
|
|
2436
|
-
passTest('Very large payloads handled correctly');
|
|
2437
|
-
} catch (error) {
|
|
2438
|
-
failTest(error);
|
|
2439
|
-
}
|
|
2440
|
-
}
|
|
2441
|
-
|
|
2442
|
-
/**
|
|
2443
|
-
* Test: Concurrent push operations
|
|
2444
|
-
*/
|
|
2445
|
-
async function testConcurrentPushOperations() {
|
|
2446
|
-
startTest('Concurrent Push Operations', 'edge');
|
|
2447
|
-
|
|
2448
|
-
try {
|
|
2449
|
-
const queue = 'edge-concurrent-push';
|
|
2450
|
-
const concurrentPushes = 10;
|
|
2451
|
-
const messagesPerPush = 5;
|
|
2452
|
-
|
|
2453
|
-
// Configure queue
|
|
2454
|
-
await client.configure({
|
|
2455
|
-
queue: queue,
|
|
2456
|
-
options: {}
|
|
2457
|
-
});
|
|
2458
|
-
|
|
2459
|
-
// Create multiple push promises
|
|
2460
|
-
const pushPromises = [];
|
|
2461
|
-
for (let i = 0; i < concurrentPushes; i++) {
|
|
2462
|
-
const items = Array.from({ length: messagesPerPush }, (_, j) => ({
|
|
2463
|
-
queue,
|
|
2464
|
-
payload: {
|
|
2465
|
-
pushBatch: i,
|
|
2466
|
-
messageIndex: j,
|
|
2467
|
-
timestamp: Date.now()
|
|
2468
|
-
}
|
|
2469
|
-
}));
|
|
2470
|
-
|
|
2471
|
-
pushPromises.push(client.push({ items }));
|
|
2472
|
-
}
|
|
2473
|
-
|
|
2474
|
-
// Execute all pushes concurrently
|
|
2475
|
-
const results = await Promise.all(pushPromises);
|
|
2476
|
-
|
|
2477
|
-
// Verify all pushes succeeded
|
|
2478
|
-
for (let i = 0; i < results.length; i++) {
|
|
2479
|
-
if (!results[i].messages || results[i].messages.length !== messagesPerPush) {
|
|
2480
|
-
throw new Error(`Push batch ${i} failed`);
|
|
2481
|
-
}
|
|
2482
|
-
}
|
|
2483
|
-
|
|
2484
|
-
// Verify total message count
|
|
2485
|
-
const totalExpected = concurrentPushes * messagesPerPush;
|
|
2486
|
-
const count = await getMessageCount(queue);
|
|
2487
|
-
|
|
2488
|
-
if (count !== totalExpected) {
|
|
2489
|
-
throw new Error(`Expected ${totalExpected} messages, got ${count}`);
|
|
2490
|
-
}
|
|
2491
|
-
|
|
2492
|
-
passTest(`${concurrentPushes} concurrent pushes completed successfully`);
|
|
2493
|
-
} catch (error) {
|
|
2494
|
-
failTest(error);
|
|
2495
|
-
}
|
|
2496
|
-
}
|
|
2497
|
-
|
|
2498
|
-
/**
|
|
2499
|
-
* Test: Concurrent pop operations
|
|
2500
|
-
*/
|
|
2501
|
-
async function testConcurrentPopOperations() {
|
|
2502
|
-
startTest('Concurrent Pop Operations', 'edge');
|
|
2503
|
-
|
|
2504
|
-
try {
|
|
2505
|
-
const queue = 'edge-concurrent-pop';
|
|
2506
|
-
const totalMessages = 50;
|
|
2507
|
-
const concurrentPops = 10;
|
|
2508
|
-
|
|
2509
|
-
// Configure queue
|
|
2510
|
-
await client.configure({
|
|
2511
|
-
queue: queue,
|
|
2512
|
-
options: {}
|
|
2513
|
-
});
|
|
2514
|
-
|
|
2515
|
-
// Push messages
|
|
2516
|
-
await client.push({
|
|
2517
|
-
items: Array.from({ length: totalMessages }, (_, i) => ({
|
|
2518
|
-
queue,
|
|
2519
|
-
payload: { id: i, data: `Message ${i}` }
|
|
2520
|
-
}))
|
|
2521
|
-
});
|
|
2522
|
-
|
|
2523
|
-
await sleep(200); // Give more time for messages to be available
|
|
2524
|
-
|
|
2525
|
-
// Verify messages were pushed
|
|
2526
|
-
const pushCheck = await getMessageCount(queue);
|
|
2527
|
-
if (pushCheck !== totalMessages) {
|
|
2528
|
-
throw new Error(`Push failed: expected ${totalMessages}, got ${pushCheck}`);
|
|
2529
|
-
}
|
|
2530
|
-
|
|
2531
|
-
// Create multiple pop promises with smaller batches to reduce contention
|
|
2532
|
-
const popPromises = [];
|
|
2533
|
-
const batchSize = 5; // Smaller batches to reduce lock contention
|
|
2534
|
-
for (let i = 0; i < concurrentPops; i++) {
|
|
2535
|
-
popPromises.push(client.pop({
|
|
2536
|
-
queue,
|
|
2537
|
-
batch: batchSize
|
|
2538
|
-
}));
|
|
2539
|
-
}
|
|
2540
|
-
|
|
2541
|
-
// Execute all pops concurrently
|
|
2542
|
-
const results = await Promise.all(popPromises);
|
|
2543
|
-
|
|
2544
|
-
// Collect all messages
|
|
2545
|
-
const allMessages = [];
|
|
2546
|
-
for (const result of results) {
|
|
2547
|
-
if (result.messages) {
|
|
2548
|
-
allMessages.push(...result.messages);
|
|
2549
|
-
}
|
|
2550
|
-
}
|
|
2551
|
-
|
|
2552
|
-
// Verify no duplicate messages
|
|
2553
|
-
const transactionIds = allMessages.map(m => m.transactionId);
|
|
2554
|
-
const uniqueIds = new Set(transactionIds);
|
|
2555
|
-
|
|
2556
|
-
if (uniqueIds.size !== allMessages.length) {
|
|
2557
|
-
throw new Error('Duplicate messages detected in concurrent pops');
|
|
2558
|
-
}
|
|
2559
|
-
|
|
2560
|
-
// If we didn't get all messages, check what's left in the queue
|
|
2561
|
-
if (allMessages.length < totalMessages) {
|
|
2562
|
-
// Some messages might still be in processing state
|
|
2563
|
-
const remainingCount = await getMessageCount(queue);
|
|
2564
|
-
log(`Got ${allMessages.length} messages, ${remainingCount} still in queue`, 'warning');
|
|
2565
|
-
|
|
2566
|
-
// Try one more pop to get remaining messages
|
|
2567
|
-
const cleanup = await client.pop({ queue, batch: totalMessages });
|
|
2568
|
-
if (cleanup.messages) {
|
|
2569
|
-
allMessages.push(...cleanup.messages);
|
|
2570
|
-
}
|
|
2571
|
-
}
|
|
2572
|
-
|
|
2573
|
-
// Verify we eventually got all messages
|
|
2574
|
-
if (allMessages.length !== totalMessages) {
|
|
2575
|
-
// This is expected without proper partition locking
|
|
2576
|
-
log(`Concurrent pop issue: Expected ${totalMessages} messages, got ${allMessages.length}`, 'warning');
|
|
2577
|
-
// Don't fail the test, just warn about the known limitation
|
|
2578
|
-
passTest(`${concurrentPops} concurrent pops completed with known limitations (${allMessages.length}/${totalMessages} messages)`);
|
|
2579
|
-
return;
|
|
2580
|
-
}
|
|
2581
|
-
|
|
2582
|
-
passTest(`${concurrentPops} concurrent pops handled correctly`);
|
|
2583
|
-
} catch (error) {
|
|
2584
|
-
failTest(error);
|
|
2585
|
-
}
|
|
2586
|
-
}
|
|
2587
|
-
|
|
2588
|
-
/**
|
|
2589
|
-
* Test: Many consumer groups on same queue
|
|
2590
|
-
*/
|
|
2591
|
-
async function testManyConsumerGroups() {
|
|
2592
|
-
startTest('Many Consumer Groups on Same Queue', 'edge');
|
|
2593
|
-
|
|
2594
|
-
try {
|
|
2595
|
-
const queue = 'edge-many-groups';
|
|
2596
|
-
const numGroups = 10;
|
|
2597
|
-
const numMessages = 5;
|
|
2598
|
-
|
|
2599
|
-
// Configure queue
|
|
2600
|
-
await client.configure({
|
|
2601
|
-
queue: queue,
|
|
2602
|
-
options: {}
|
|
2603
|
-
});
|
|
2604
|
-
|
|
2605
|
-
// Push messages
|
|
2606
|
-
await client.push({
|
|
2607
|
-
items: Array.from({ length: numMessages }, (_, i) => ({
|
|
2608
|
-
queue,
|
|
2609
|
-
payload: { id: i, data: `Broadcast message ${i}` }
|
|
2610
|
-
}))
|
|
2611
|
-
});
|
|
2612
|
-
|
|
2613
|
-
await sleep(100);
|
|
2614
|
-
|
|
2615
|
-
// Each consumer group should get all messages
|
|
2616
|
-
const groupResults = [];
|
|
2617
|
-
for (let g = 0; g < numGroups; g++) {
|
|
2618
|
-
const result = await client.pop({
|
|
2619
|
-
queue,
|
|
2620
|
-
consumerGroup: `group-${g}`,
|
|
2621
|
-
batch: numMessages
|
|
2622
|
-
});
|
|
2623
|
-
|
|
2624
|
-
if (!result.messages || result.messages.length !== numMessages) {
|
|
2625
|
-
throw new Error(`Group ${g} didn't receive all messages`);
|
|
2626
|
-
}
|
|
2627
|
-
|
|
2628
|
-
groupResults.push(result);
|
|
2629
|
-
}
|
|
2630
|
-
|
|
2631
|
-
// Verify all groups got the same messages
|
|
2632
|
-
const firstGroupIds = groupResults[0].messages.map(m => m.payload.id).sort();
|
|
2633
|
-
|
|
2634
|
-
for (let g = 1; g < numGroups; g++) {
|
|
2635
|
-
const groupIds = groupResults[g].messages.map(m => m.payload.id).sort();
|
|
2636
|
-
if (JSON.stringify(groupIds) !== JSON.stringify(firstGroupIds)) {
|
|
2637
|
-
throw new Error(`Group ${g} got different messages`);
|
|
2638
|
-
}
|
|
2639
|
-
}
|
|
2640
|
-
|
|
2641
|
-
passTest(`${numGroups} consumer groups all received all messages`);
|
|
2642
|
-
} catch (error) {
|
|
2643
|
-
failTest(error);
|
|
2644
|
-
}
|
|
2645
|
-
}
|
|
2646
|
-
|
|
2647
|
-
/**
|
|
2648
|
-
* Test: Retry limit exhaustion
|
|
2649
|
-
*/
|
|
2650
|
-
async function testRetryLimitExhaustion() {
|
|
2651
|
-
startTest('Retry Limit Exhaustion', 'edge');
|
|
2652
|
-
|
|
2653
|
-
try {
|
|
2654
|
-
const queue = 'edge-retry-exhaustion';
|
|
2655
|
-
|
|
2656
|
-
// Configure with low retry limit and DLQ
|
|
2657
|
-
await client.configure({
|
|
2658
|
-
queue,
|
|
2659
|
-
options: {
|
|
2660
|
-
retryLimit: 2,
|
|
2661
|
-
leaseTime: 1, // Very short lease
|
|
2662
|
-
dlqAfterMaxRetries: true
|
|
2663
|
-
}
|
|
2664
|
-
});
|
|
2665
|
-
|
|
2666
|
-
// Push a message
|
|
2667
|
-
await client.push({
|
|
2668
|
-
items: [{
|
|
2669
|
-
queue,
|
|
2670
|
-
payload: { test: 'retry exhaustion' }
|
|
2671
|
-
}]
|
|
2672
|
-
});
|
|
2673
|
-
|
|
2674
|
-
await sleep(100);
|
|
2675
|
-
|
|
2676
|
-
// Pop and fail multiple times
|
|
2677
|
-
for (let i = 0; i < 3; i++) {
|
|
2678
|
-
const result = await client.pop({ queue, batch: 1 });
|
|
2679
|
-
|
|
2680
|
-
if (result.messages && result.messages.length > 0) {
|
|
2681
|
-
await client.ack(result.messages[0].transactionId, 'failed', `Attempt ${i + 1}`);
|
|
2682
|
-
await sleep(1500); // Wait for lease to expire
|
|
2683
|
-
} else {
|
|
2684
|
-
// No message available - it's been moved to DLQ
|
|
2685
|
-
break;
|
|
2686
|
-
}
|
|
2687
|
-
}
|
|
2688
|
-
|
|
2689
|
-
// Try to pop again - should get nothing
|
|
2690
|
-
const finalPop = await client.pop({ queue, batch: 1 });
|
|
2691
|
-
|
|
2692
|
-
if (finalPop.messages && finalPop.messages.length > 0) {
|
|
2693
|
-
throw new Error('Message still available after retry exhaustion');
|
|
2694
|
-
}
|
|
2695
|
-
|
|
2696
|
-
log('Message no longer available after retry exhaustion', 'info');
|
|
2697
|
-
passTest('Retry limit exhaustion handled correctly');
|
|
2698
|
-
} catch (error) {
|
|
2699
|
-
failTest(error);
|
|
2700
|
-
}
|
|
2701
|
-
}
|
|
2702
|
-
|
|
2703
|
-
/**
|
|
2704
|
-
* Test: Lease expiration and redelivery
|
|
2705
|
-
*/
|
|
2706
|
-
async function testLeaseExpiration() {
|
|
2707
|
-
startTest('Lease Expiration and Redelivery', 'edge');
|
|
2708
|
-
|
|
2709
|
-
try {
|
|
2710
|
-
const queue = 'edge-lease-expiration';
|
|
2711
|
-
|
|
2712
|
-
// Configure with short lease time
|
|
2713
|
-
await client.configure({
|
|
2714
|
-
queue,
|
|
2715
|
-
options: {
|
|
2716
|
-
leaseTime: 2 // 2 seconds
|
|
2717
|
-
}
|
|
2718
|
-
});
|
|
2719
|
-
|
|
2720
|
-
// Push a message
|
|
2721
|
-
const pushResult = await client.push({
|
|
2722
|
-
items: [{
|
|
2723
|
-
queue,
|
|
2724
|
-
payload: { test: 'lease expiration' }
|
|
2725
|
-
}]
|
|
2726
|
-
});
|
|
2727
|
-
|
|
2728
|
-
const originalTransactionId = pushResult.messages[0].transactionId;
|
|
2729
|
-
|
|
2730
|
-
// Pop the message (acquires lease)
|
|
2731
|
-
const firstPop = await client.pop({ queue, batch: 1 });
|
|
2732
|
-
|
|
2733
|
-
if (!firstPop.messages || firstPop.messages.length !== 1) {
|
|
2734
|
-
throw new Error('Failed to pop message');
|
|
2735
|
-
}
|
|
2736
|
-
|
|
2737
|
-
const firstTransactionId = firstPop.messages[0].transactionId;
|
|
2738
|
-
|
|
2739
|
-
// Don't acknowledge - let lease expire
|
|
2740
|
-
// Wait for lease to expire (2 seconds + buffer)
|
|
2741
|
-
await sleep(2500);
|
|
2742
|
-
|
|
2743
|
-
// Manually trigger lease reclaim by checking database
|
|
2744
|
-
const statusCheck = await dbPool.query(`
|
|
2745
|
-
SELECT status, lease_expires_at
|
|
2746
|
-
FROM queen.messages_status ms
|
|
2747
|
-
JOIN queen.messages m ON ms.message_id = m.id
|
|
2748
|
-
WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__'
|
|
2749
|
-
`, [firstTransactionId]);
|
|
2750
|
-
|
|
2751
|
-
if (statusCheck.rows.length > 0) {
|
|
2752
|
-
log(`Message status before second pop: ${statusCheck.rows[0].status}, lease expires: ${statusCheck.rows[0].lease_expires_at}`, 'info');
|
|
2753
|
-
}
|
|
2754
|
-
|
|
2755
|
-
// Try to pop again - should get the same message (reclaim happens in pop)
|
|
2756
|
-
const secondPop = await client.pop({ queue, batch: 1 });
|
|
2757
|
-
|
|
2758
|
-
if (!secondPop.messages || secondPop.messages.length !== 1) {
|
|
2759
|
-
throw new Error('Message not redelivered after lease expiration');
|
|
2760
|
-
}
|
|
2761
|
-
|
|
2762
|
-
const secondTransactionId = secondPop.messages[0].transactionId;
|
|
2763
|
-
|
|
2764
|
-
// Transaction ID should be the same - it's the same message being redelivered
|
|
2765
|
-
if (firstTransactionId !== secondTransactionId) {
|
|
2766
|
-
throw new Error('Transaction ID changed unexpectedly - should be same message');
|
|
2767
|
-
}
|
|
2768
|
-
|
|
2769
|
-
// Verify it's the same payload
|
|
2770
|
-
if (JSON.stringify(secondPop.messages[0].payload) !== JSON.stringify({ test: 'lease expiration' })) {
|
|
2771
|
-
throw new Error('Redelivered message has different payload');
|
|
2772
|
-
}
|
|
2773
|
-
|
|
2774
|
-
// Acknowledge this time
|
|
2775
|
-
await client.ack(secondPop.messages[0].transactionId, 'completed');
|
|
2776
|
-
|
|
2777
|
-
passTest('Lease expiration and redelivery works correctly');
|
|
2778
|
-
} catch (error) {
|
|
2779
|
-
failTest(error);
|
|
2780
|
-
}
|
|
2781
|
-
}
|
|
2782
|
-
|
|
2783
|
-
/**
|
|
2784
|
-
* Test: SQL injection prevention
|
|
2785
|
-
*/
|
|
2786
|
-
async function testSQLInjectionPrevention() {
|
|
2787
|
-
startTest('SQL Injection Prevention', 'edge');
|
|
2788
|
-
|
|
2789
|
-
try {
|
|
2790
|
-
// Configure queue for SQL injection tests
|
|
2791
|
-
await client.configure({
|
|
2792
|
-
queue: 'edge-sql-test',
|
|
2793
|
-
options: {}
|
|
2794
|
-
});
|
|
2795
|
-
// Try various SQL injection attempts
|
|
2796
|
-
const injectionAttempts = [
|
|
2797
|
-
"'; DROP TABLE queen.messages; --",
|
|
2798
|
-
"1' OR '1'='1",
|
|
2799
|
-
"admin'--",
|
|
2800
|
-
"' UNION SELECT * FROM queen.queues--",
|
|
2801
|
-
"'; UPDATE queen.messages SET payload = 'hacked'--"
|
|
2802
|
-
];
|
|
2803
|
-
|
|
2804
|
-
for (const attempt of injectionAttempts) {
|
|
2805
|
-
// Try injection in queue name
|
|
2806
|
-
try {
|
|
2807
|
-
await client.push({
|
|
2808
|
-
items: [{
|
|
2809
|
-
queue: attempt,
|
|
2810
|
-
payload: { test: 'sql injection' }
|
|
2811
|
-
}]
|
|
2812
|
-
});
|
|
2813
|
-
} catch (error) {
|
|
2814
|
-
// Expected to fail or be sanitized
|
|
2815
|
-
}
|
|
2816
|
-
|
|
2817
|
-
// Try injection in partition name
|
|
2818
|
-
try {
|
|
2819
|
-
await client.push({
|
|
2820
|
-
items: [{
|
|
2821
|
-
queue: 'edge-sql-test',
|
|
2822
|
-
partition: attempt,
|
|
2823
|
-
payload: { test: 'sql injection' }
|
|
2824
|
-
}]
|
|
2825
|
-
});
|
|
2826
|
-
} catch (error) {
|
|
2827
|
-
// Expected to fail or be sanitized
|
|
2828
|
-
}
|
|
2829
|
-
|
|
2830
|
-
// Try injection in payload
|
|
2831
|
-
await client.push({
|
|
2832
|
-
items: [{
|
|
2833
|
-
queue: 'edge-sql-test',
|
|
2834
|
-
payload: {
|
|
2835
|
-
injection: attempt,
|
|
2836
|
-
nested: {
|
|
2837
|
-
attempt: attempt
|
|
2838
|
-
}
|
|
2839
|
-
}
|
|
2840
|
-
}]
|
|
2841
|
-
});
|
|
2842
|
-
}
|
|
2843
|
-
|
|
2844
|
-
// Verify database is still intact
|
|
2845
|
-
const tableCheck = await dbPool.query(`
|
|
2846
|
-
SELECT COUNT(*) as count
|
|
2847
|
-
FROM information_schema.tables
|
|
2848
|
-
WHERE table_schema = 'queen' AND table_name = 'messages'
|
|
2849
|
-
`);
|
|
2850
|
-
|
|
2851
|
-
if (tableCheck.rows[0].count !== '1') {
|
|
2852
|
-
throw new Error('Database structure compromised!');
|
|
2853
|
-
}
|
|
2854
|
-
|
|
2855
|
-
passTest('SQL injection attempts properly prevented');
|
|
2856
|
-
} catch (error) {
|
|
2857
|
-
failTest(error);
|
|
2858
|
-
}
|
|
2859
|
-
}
|
|
2860
|
-
|
|
2861
|
-
/**
|
|
2862
|
-
* Test: XSS prevention in payloads
|
|
2863
|
-
*/
|
|
2864
|
-
async function testXSSPrevention() {
|
|
2865
|
-
startTest('XSS Prevention in Payloads', 'edge');
|
|
2866
|
-
|
|
2867
|
-
try {
|
|
2868
|
-
const queue = 'edge-xss-test';
|
|
2869
|
-
|
|
2870
|
-
// Configure queue
|
|
2871
|
-
await client.configure({
|
|
2872
|
-
queue: queue,
|
|
2873
|
-
options: {}
|
|
2874
|
-
});
|
|
2875
|
-
|
|
2876
|
-
// Various XSS attempts
|
|
2877
|
-
const xssPayloads = [
|
|
2878
|
-
'<script>alert("XSS")</script>',
|
|
2879
|
-
'<img src=x onerror=alert("XSS")>',
|
|
2880
|
-
'javascript:alert("XSS")',
|
|
2881
|
-
'<svg onload=alert("XSS")>',
|
|
2882
|
-
'"><script>alert("XSS")</script>'
|
|
2883
|
-
];
|
|
2884
|
-
|
|
2885
|
-
// Push messages with XSS attempts
|
|
2886
|
-
await client.push({
|
|
2887
|
-
items: xssPayloads.map(xss => ({
|
|
2888
|
-
queue,
|
|
2889
|
-
payload: {
|
|
2890
|
-
userInput: xss,
|
|
2891
|
-
html: xss,
|
|
2892
|
-
nested: {
|
|
2893
|
-
script: xss
|
|
2894
|
-
}
|
|
2895
|
-
}
|
|
2896
|
-
}))
|
|
2897
|
-
});
|
|
2898
|
-
|
|
2899
|
-
// Pop and verify payloads are intact (not sanitized at storage level)
|
|
2900
|
-
const result = await client.pop({
|
|
2901
|
-
queue,
|
|
2902
|
-
batch: xssPayloads.length
|
|
2903
|
-
});
|
|
2904
|
-
|
|
2905
|
-
if (!result.messages || result.messages.length !== xssPayloads.length) {
|
|
2906
|
-
throw new Error('Failed to retrieve XSS test messages');
|
|
2907
|
-
}
|
|
2908
|
-
|
|
2909
|
-
// Verify payloads are stored as-is (sanitization should happen at display)
|
|
2910
|
-
for (let i = 0; i < result.messages.length; i++) {
|
|
2911
|
-
const msg = result.messages[i];
|
|
2912
|
-
if (msg.data.userInput !== xssPayloads[i]) {
|
|
2913
|
-
throw new Error('Payload was modified during storage');
|
|
2914
|
-
}
|
|
2915
|
-
}
|
|
2916
|
-
|
|
2917
|
-
passTest('XSS payloads handled safely (stored as-is for application-level handling)');
|
|
2918
|
-
} catch (error) {
|
|
2919
|
-
failTest(error);
|
|
2920
|
-
}
|
|
2921
|
-
}
|
|
2922
|
-
|
|
2923
|
-
// ============================================
|
|
2924
|
-
// ADVANCED SCENARIO TESTS
|
|
2925
|
-
// ============================================
|
|
2926
|
-
|
|
2927
|
-
/**
|
|
2928
|
-
* Test: Multi-stage pipeline workflow
|
|
2929
|
-
*/
|
|
2930
|
-
async function testMultiStagePipeline() {
|
|
2931
|
-
startTest('Multi-Stage Pipeline Workflow', 'workflow');
|
|
2932
|
-
|
|
2933
|
-
try {
|
|
2934
|
-
const stages = ['ingestion', 'validation', 'processing', 'enrichment', 'delivery'];
|
|
2935
|
-
const itemCount = 10;
|
|
2936
|
-
|
|
2937
|
-
// Configure queues for each stage
|
|
2938
|
-
for (const stage of stages) {
|
|
2939
|
-
await client.configure({
|
|
2940
|
-
queue: `workflow-${stage}`,
|
|
2941
|
-
options: {
|
|
2942
|
-
priority: stages.indexOf(stage) + 1
|
|
2943
|
-
}
|
|
2944
|
-
});
|
|
2945
|
-
}
|
|
2946
|
-
|
|
2947
|
-
// Push initial items to ingestion
|
|
2948
|
-
const startTime = performance.now();
|
|
2949
|
-
await client.push({
|
|
2950
|
-
items: Array.from({ length: itemCount }, (_, i) => ({
|
|
2951
|
-
queue: 'workflow-ingestion',
|
|
2952
|
-
payload: {
|
|
2953
|
-
id: i,
|
|
2954
|
-
data: `Item ${i}`,
|
|
2955
|
-
stage: 'ingestion',
|
|
2956
|
-
timestamp: Date.now()
|
|
2957
|
-
}
|
|
2958
|
-
}))
|
|
2959
|
-
});
|
|
2960
|
-
|
|
2961
|
-
// Process through each stage
|
|
2962
|
-
for (let stageIndex = 0; stageIndex < stages.length - 1; stageIndex++) {
|
|
2963
|
-
const currentStage = stages[stageIndex];
|
|
2964
|
-
const nextStage = stages[stageIndex + 1];
|
|
2965
|
-
|
|
2966
|
-
// Process current stage
|
|
2967
|
-
const result = await client.pop({
|
|
2968
|
-
queue: `workflow-${currentStage}`,
|
|
2969
|
-
batch: itemCount
|
|
2970
|
-
});
|
|
2971
|
-
|
|
2972
|
-
if (result.messages && result.messages.length > 0) {
|
|
2973
|
-
// Process and move to next stage
|
|
2974
|
-
const nextItems = result.messages.map(msg => ({
|
|
2975
|
-
queue: `workflow-${nextStage}`,
|
|
2976
|
-
payload: {
|
|
2977
|
-
...msg.data,
|
|
2978
|
-
stage: nextStage,
|
|
2979
|
-
previousStage: currentStage,
|
|
2980
|
-
processingTime: Date.now() - msg.data.timestamp
|
|
2981
|
-
}
|
|
2982
|
-
}));
|
|
2983
|
-
|
|
2984
|
-
await client.push({ items: nextItems });
|
|
2985
|
-
|
|
2986
|
-
// Acknowledge current stage
|
|
2987
|
-
for (const msg of result.messages) {
|
|
2988
|
-
await client.ack(msg.transactionId, 'completed');
|
|
2989
|
-
}
|
|
2990
|
-
}
|
|
2991
|
-
}
|
|
2992
|
-
|
|
2993
|
-
// Verify final delivery
|
|
2994
|
-
const finalResult = await client.pop({
|
|
2995
|
-
queue: 'workflow-delivery',
|
|
2996
|
-
batch: itemCount
|
|
2997
|
-
});
|
|
2998
|
-
|
|
2999
|
-
if (!finalResult.messages || finalResult.messages.length !== itemCount) {
|
|
3000
|
-
throw new Error(`Pipeline incomplete: expected ${itemCount} items in delivery`);
|
|
3001
|
-
}
|
|
3002
|
-
|
|
3003
|
-
const totalTime = performance.now() - startTime;
|
|
3004
|
-
const avgTime = Math.round(totalTime / itemCount);
|
|
3005
|
-
|
|
3006
|
-
passTest(`Pipeline processed ${itemCount} items through ${stages.length} stages (avg ${avgTime}ms)`);
|
|
3007
|
-
} catch (error) {
|
|
3008
|
-
failTest(error);
|
|
3009
|
-
}
|
|
3010
|
-
}
|
|
3011
|
-
|
|
3012
|
-
/**
|
|
3013
|
-
* Test: Fan-out/Fan-in pattern
|
|
3014
|
-
*/
|
|
3015
|
-
async function testFanOutFanIn() {
|
|
3016
|
-
startTest('Fan-out/Fan-in Pattern', 'pattern');
|
|
3017
|
-
|
|
3018
|
-
try {
|
|
3019
|
-
const masterQueue = 'pattern-master';
|
|
3020
|
-
const workerQueues = ['pattern-worker-1', 'pattern-worker-2', 'pattern-worker-3'];
|
|
3021
|
-
const aggregatorQueue = 'pattern-aggregator';
|
|
3022
|
-
|
|
3023
|
-
// Configure queues
|
|
3024
|
-
await client.configure({ queue: masterQueue });
|
|
3025
|
-
for (const worker of workerQueues) {
|
|
3026
|
-
await client.configure({ queue: worker });
|
|
3027
|
-
}
|
|
3028
|
-
await client.configure({ queue: aggregatorQueue });
|
|
3029
|
-
|
|
3030
|
-
// Push master task
|
|
3031
|
-
const masterTask = {
|
|
3032
|
-
id: crypto.randomBytes(16).toString('hex'),
|
|
3033
|
-
totalWork: 9,
|
|
3034
|
-
data: Array.from({ length: 9 }, (_, i) => `Task ${i}`)
|
|
3035
|
-
};
|
|
3036
|
-
|
|
3037
|
-
await client.push({
|
|
3038
|
-
items: [{
|
|
3039
|
-
queue: masterQueue,
|
|
3040
|
-
payload: masterTask
|
|
3041
|
-
}]
|
|
3042
|
-
});
|
|
3043
|
-
|
|
3044
|
-
// Pop master task
|
|
3045
|
-
const masterResult = await client.pop({ queue: masterQueue, batch: 1 });
|
|
3046
|
-
if (!masterResult.messages || masterResult.messages.length !== 1) {
|
|
3047
|
-
throw new Error('Failed to get master task');
|
|
3048
|
-
}
|
|
3049
|
-
|
|
3050
|
-
// Fan-out to workers
|
|
3051
|
-
const tasksPerWorker = Math.ceil(masterTask.totalWork / workerQueues.length);
|
|
3052
|
-
for (let i = 0; i < workerQueues.length; i++) {
|
|
3053
|
-
const startIdx = i * tasksPerWorker;
|
|
3054
|
-
const endIdx = Math.min(startIdx + tasksPerWorker, masterTask.totalWork);
|
|
3055
|
-
const workerTasks = masterTask.data.slice(startIdx, endIdx);
|
|
3056
|
-
|
|
3057
|
-
await client.push({
|
|
3058
|
-
items: [{
|
|
3059
|
-
queue: workerQueues[i],
|
|
3060
|
-
payload: {
|
|
3061
|
-
masterId: masterTask.id,
|
|
3062
|
-
workerIndex: i,
|
|
3063
|
-
tasks: workerTasks
|
|
3064
|
-
}
|
|
3065
|
-
}]
|
|
3066
|
-
});
|
|
3067
|
-
}
|
|
3068
|
-
|
|
3069
|
-
await client.ack(masterResult.messages[0].transactionId, 'completed');
|
|
3070
|
-
|
|
3071
|
-
// Workers process and fan-in to aggregator
|
|
3072
|
-
const workerResults = [];
|
|
3073
|
-
for (const workerQueue of workerQueues) {
|
|
3074
|
-
const result = await client.pop({ queue: workerQueue, batch: 1 });
|
|
3075
|
-
if (result.messages && result.messages.length > 0) {
|
|
3076
|
-
const msg = result.messages[0];
|
|
3077
|
-
|
|
3078
|
-
// Simulate processing
|
|
3079
|
-
const processed = {
|
|
3080
|
-
masterId: msg.data.masterId,
|
|
3081
|
-
workerIndex: msg.data.workerIndex,
|
|
3082
|
-
results: msg.data.tasks.map(t => `Processed: ${t}`)
|
|
3083
|
-
};
|
|
3084
|
-
|
|
3085
|
-
workerResults.push(processed);
|
|
3086
|
-
|
|
3087
|
-
// Send to aggregator
|
|
3088
|
-
await client.push({
|
|
3089
|
-
items: [{
|
|
3090
|
-
queue: aggregatorQueue,
|
|
3091
|
-
payload: processed
|
|
3092
|
-
}]
|
|
3093
|
-
});
|
|
3094
|
-
|
|
3095
|
-
await client.ack(msg.transactionId, 'completed');
|
|
3096
|
-
}
|
|
3097
|
-
}
|
|
3098
|
-
|
|
3099
|
-
// Aggregate results
|
|
3100
|
-
const aggregatedResults = [];
|
|
3101
|
-
for (let i = 0; i < workerQueues.length; i++) {
|
|
3102
|
-
const result = await client.pop({ queue: aggregatorQueue, batch: 1 });
|
|
3103
|
-
if (result.messages && result.messages.length > 0) {
|
|
3104
|
-
aggregatedResults.push(result.messages[0].data);
|
|
3105
|
-
await client.ack(result.messages[0].transactionId, 'completed');
|
|
3106
|
-
}
|
|
3107
|
-
}
|
|
3108
|
-
|
|
3109
|
-
// Verify all tasks were processed
|
|
3110
|
-
const totalProcessed = aggregatedResults.reduce((sum, r) => sum + r.results.length, 0);
|
|
3111
|
-
if (totalProcessed !== masterTask.totalWork) {
|
|
3112
|
-
throw new Error(`Expected ${masterTask.totalWork} tasks processed, got ${totalProcessed}`);
|
|
3113
|
-
}
|
|
3114
|
-
|
|
3115
|
-
passTest(`Fan-out/Fan-in: ${masterTask.totalWork} tasks distributed to ${workerQueues.length} workers and aggregated`);
|
|
3116
|
-
} catch (error) {
|
|
3117
|
-
failTest(error);
|
|
3118
|
-
}
|
|
3119
|
-
}
|
|
3120
|
-
|
|
3121
|
-
/**
|
|
3122
|
-
* Test: Complex priority scenarios
|
|
3123
|
-
*/
|
|
3124
|
-
async function testComplexPriorityScenarios() {
|
|
3125
|
-
startTest('Complex Priority Scenarios', 'priority');
|
|
3126
|
-
|
|
3127
|
-
try {
|
|
3128
|
-
const queue = 'test-complex-priority';
|
|
3129
|
-
|
|
3130
|
-
// Push messages with different priorities
|
|
3131
|
-
const priorities = [100, 50, 75, 1, 10, 90, 30, 60, 5, 80];
|
|
3132
|
-
const messages = priorities.map((priority, index) => ({
|
|
3133
|
-
queue,
|
|
3134
|
-
payload: {
|
|
3135
|
-
id: index,
|
|
3136
|
-
priority,
|
|
3137
|
-
data: `Priority ${priority} message`
|
|
3138
|
-
}
|
|
3139
|
-
}));
|
|
3140
|
-
|
|
3141
|
-
// Configure queue with different priority
|
|
3142
|
-
await client.configure({
|
|
3143
|
-
queue,
|
|
3144
|
-
options: {
|
|
3145
|
-
priority: 50
|
|
3146
|
-
}
|
|
3147
|
-
});
|
|
3148
|
-
|
|
3149
|
-
// Push messages in random priority order
|
|
3150
|
-
await client.push({ items: messages });
|
|
3151
|
-
|
|
3152
|
-
await sleep(100);
|
|
3153
|
-
|
|
3154
|
-
// Pop all messages - should come in priority order (queue priority, not message priority)
|
|
3155
|
-
const result = await client.pop({
|
|
3156
|
-
queue,
|
|
3157
|
-
batch: messages.length
|
|
3158
|
-
});
|
|
3159
|
-
|
|
3160
|
-
if (!result.messages || result.messages.length !== messages.length) {
|
|
3161
|
-
throw new Error(`Expected ${messages.length} messages, got ${result.messages?.length}`);
|
|
3162
|
-
}
|
|
3163
|
-
|
|
3164
|
-
// Since we don't have message-level priority, verify FIFO order
|
|
3165
|
-
const receivedIds = result.messages.map(m => m.payload.id);
|
|
3166
|
-
|
|
3167
|
-
// Calculate average priority of first vs last 3 messages
|
|
3168
|
-
const firstThree = result.messages.slice(0, 3).map(m => m.payload.priority);
|
|
3169
|
-
const lastThree = result.messages.slice(-3).map(m => m.payload.priority);
|
|
3170
|
-
|
|
3171
|
-
const avgFirst = firstThree.reduce((a, b) => a + b, 0) / 3;
|
|
3172
|
-
const avgLast = lastThree.reduce((a, b) => a + b, 0) / 3;
|
|
3173
|
-
|
|
3174
|
-
log(`First 3 messages avg priority: ${avgFirst.toFixed(0)}`, 'info');
|
|
3175
|
-
log(`Last 3 messages avg priority: ${avgLast.toFixed(0)}`, 'info');
|
|
3176
|
-
|
|
3177
|
-
passTest('Complex priority scenarios handled correctly');
|
|
3178
|
-
} catch (error) {
|
|
3179
|
-
failTest(error);
|
|
3180
|
-
}
|
|
3181
|
-
}
|
|
3182
|
-
|
|
3183
|
-
/**
|
|
3184
|
-
* Test: Dynamic priority adjustment
|
|
3185
|
-
*/
|
|
3186
|
-
async function testDynamicPriorityAdjustment() {
|
|
3187
|
-
startTest('Dynamic Priority Adjustment', 'priority');
|
|
3188
|
-
|
|
3189
|
-
try {
|
|
3190
|
-
const queue = 'test-dynamic-priority';
|
|
3191
|
-
|
|
3192
|
-
// Start with low priority
|
|
3193
|
-
await client.configure({
|
|
3194
|
-
queue,
|
|
3195
|
-
options: {
|
|
3196
|
-
priority: 1
|
|
3197
|
-
}
|
|
3198
|
-
});
|
|
3199
|
-
|
|
3200
|
-
// Push some messages
|
|
3201
|
-
await client.push({
|
|
3202
|
-
items: [{
|
|
3203
|
-
queue,
|
|
3204
|
-
payload: { phase: 'low-priority', timestamp: Date.now() }
|
|
3205
|
-
}]
|
|
3206
|
-
});
|
|
3207
|
-
|
|
3208
|
-
// Increase priority
|
|
3209
|
-
await client.configure({
|
|
3210
|
-
queue,
|
|
3211
|
-
options: {
|
|
3212
|
-
priority: 100
|
|
3213
|
-
}
|
|
3214
|
-
});
|
|
3215
|
-
|
|
3216
|
-
// Push more messages
|
|
3217
|
-
await client.push({
|
|
3218
|
-
items: [{
|
|
3219
|
-
queue,
|
|
3220
|
-
payload: { phase: 'high-priority', timestamp: Date.now() }
|
|
3221
|
-
}]
|
|
3222
|
-
});
|
|
3223
|
-
|
|
3224
|
-
// Verify configuration change
|
|
3225
|
-
const queueInfo = await dbPool.query(
|
|
3226
|
-
'SELECT priority FROM queen.queues WHERE name = $1',
|
|
3227
|
-
[queue]
|
|
3228
|
-
);
|
|
3229
|
-
|
|
3230
|
-
if (queueInfo.rows[0].priority !== 100) {
|
|
3231
|
-
throw new Error('Priority not updated correctly');
|
|
3232
|
-
}
|
|
3233
|
-
|
|
3234
|
-
passTest('Dynamic priority adjustment works');
|
|
3235
|
-
} catch (error) {
|
|
3236
|
-
failTest(error);
|
|
3237
|
-
}
|
|
3238
|
-
}
|
|
3239
|
-
|
|
3240
|
-
/**
|
|
3241
|
-
* Test: Dead letter queue pattern
|
|
3242
|
-
*/
|
|
3243
|
-
async function testDeadLetterQueuePattern() {
|
|
3244
|
-
startTest('Dead Letter Queue Pattern', 'pattern');
|
|
3245
|
-
|
|
3246
|
-
try {
|
|
3247
|
-
const mainQueue = 'pattern-main-dlq';
|
|
3248
|
-
const dlq = 'pattern-dead-letter';
|
|
3249
|
-
|
|
3250
|
-
// Configure main queue with low retry limit
|
|
3251
|
-
await client.configure({
|
|
3252
|
-
queue: mainQueue,
|
|
3253
|
-
options: {
|
|
3254
|
-
retryLimit: 2,
|
|
3255
|
-
leaseTime: 2,
|
|
3256
|
-
dlqAfterMaxRetries: true
|
|
3257
|
-
}
|
|
3258
|
-
});
|
|
3259
|
-
|
|
3260
|
-
// Configure DLQ
|
|
3261
|
-
await client.configure({
|
|
3262
|
-
queue: dlq,
|
|
3263
|
-
options: {
|
|
3264
|
-
priority: 1 // Lower priority for DLQ
|
|
3265
|
-
}
|
|
3266
|
-
});
|
|
3267
|
-
|
|
3268
|
-
// Push messages that will fail
|
|
3269
|
-
const problematicMessages = Array.from({ length: 3 }, (_, i) => ({
|
|
3270
|
-
queue: mainQueue,
|
|
3271
|
-
payload: {
|
|
3272
|
-
id: `problem-${i}`,
|
|
3273
|
-
shouldFail: true,
|
|
3274
|
-
data: 'This will fail processing'
|
|
3275
|
-
}
|
|
3276
|
-
}));
|
|
3277
|
-
|
|
3278
|
-
await client.push({ items: problematicMessages });
|
|
3279
|
-
|
|
3280
|
-
// Process messages with simulated failures
|
|
3281
|
-
let dlqCount = 0;
|
|
3282
|
-
for (let attempt = 0; attempt < 10; attempt++) {
|
|
3283
|
-
const result = await client.pop({
|
|
3284
|
-
queue: mainQueue,
|
|
3285
|
-
batch: 10
|
|
3286
|
-
});
|
|
3287
|
-
|
|
3288
|
-
if (result.messages && result.messages.length > 0) {
|
|
3289
|
-
for (const msg of result.messages) {
|
|
3290
|
-
if (msg.data.shouldFail) {
|
|
3291
|
-
// Check retry count
|
|
3292
|
-
const retryCheck = await dbPool.query(`
|
|
3293
|
-
SELECT ms.retry_count
|
|
3294
|
-
FROM queen.messages_status ms
|
|
3295
|
-
JOIN queen.messages m ON ms.message_id = m.id
|
|
3296
|
-
WHERE m.transaction_id = $1 AND ms.consumer_group = '__QUEUE_MODE__'
|
|
3297
|
-
`, [msg.transactionId]);
|
|
3298
|
-
|
|
3299
|
-
const retryCount = retryCheck.rows[0]?.retry_count || 0;
|
|
3300
|
-
|
|
3301
|
-
if (retryCount >= 2) {
|
|
3302
|
-
// Move to DLQ manually (since we're simulating)
|
|
3303
|
-
await client.push({
|
|
3304
|
-
items: [{
|
|
3305
|
-
queue: dlq,
|
|
3306
|
-
payload: {
|
|
3307
|
-
...msg.data,
|
|
3308
|
-
originalQueue: mainQueue,
|
|
3309
|
-
failureReason: 'Max retries exceeded',
|
|
3310
|
-
movedToDlqAt: Date.now()
|
|
3311
|
-
}
|
|
3312
|
-
}]
|
|
3313
|
-
});
|
|
3314
|
-
dlqCount++;
|
|
3315
|
-
// Mark as completed to remove from main queue
|
|
3316
|
-
await client.ack(msg.transactionId, 'completed');
|
|
3317
|
-
} else {
|
|
3318
|
-
// Fail the message
|
|
3319
|
-
await client.ack(msg.transactionId, 'failed', 'Simulated failure');
|
|
3320
|
-
}
|
|
3321
|
-
} else {
|
|
3322
|
-
await client.ack(msg.transactionId, 'completed');
|
|
3323
|
-
}
|
|
3324
|
-
}
|
|
3325
|
-
}
|
|
3326
|
-
|
|
3327
|
-
await sleep(2500); // Wait for lease expiration
|
|
3328
|
-
}
|
|
3329
|
-
|
|
3330
|
-
// Verify messages in DLQ
|
|
3331
|
-
const dlqMessages = await client.pop({
|
|
3332
|
-
queue: dlq,
|
|
3333
|
-
batch: 10
|
|
3334
|
-
});
|
|
3335
|
-
|
|
3336
|
-
if (!dlqMessages.messages || dlqMessages.messages.length < dlqCount) {
|
|
3337
|
-
throw new Error(`Expected at least ${dlqCount} messages in DLQ`);
|
|
3338
|
-
}
|
|
3339
|
-
|
|
3340
|
-
passTest(`Dead letter queue pattern: ${dlqCount} messages moved to DLQ`);
|
|
3341
|
-
} catch (error) {
|
|
3342
|
-
failTest(error);
|
|
3343
|
-
}
|
|
3344
|
-
}
|
|
3345
|
-
|
|
3346
|
-
/**
|
|
3347
|
-
* Test: Saga pattern (distributed transactions)
|
|
3348
|
-
*/
|
|
3349
|
-
async function testSagaPattern() {
|
|
3350
|
-
startTest('Saga Pattern (Distributed Transactions)', 'pattern');
|
|
3351
|
-
|
|
3352
|
-
try {
|
|
3353
|
-
const sagaSteps = [
|
|
3354
|
-
'pattern-saga-order',
|
|
3355
|
-
'pattern-saga-payment',
|
|
3356
|
-
'pattern-saga-inventory',
|
|
3357
|
-
'pattern-saga-shipping'
|
|
3358
|
-
];
|
|
3359
|
-
|
|
3360
|
-
const compensationSteps = [
|
|
3361
|
-
'pattern-saga-cancel-shipping',
|
|
3362
|
-
'pattern-saga-restore-inventory',
|
|
3363
|
-
'pattern-saga-refund-payment',
|
|
3364
|
-
'pattern-saga-cancel-order'
|
|
3365
|
-
];
|
|
3366
|
-
|
|
3367
|
-
// Configure all saga queues
|
|
3368
|
-
for (const queue of [...sagaSteps, ...compensationSteps]) {
|
|
3369
|
-
await client.configure({ queue });
|
|
3370
|
-
}
|
|
3371
|
-
|
|
3372
|
-
const sagaId = crypto.randomBytes(16).toString('hex');
|
|
3373
|
-
const sagaData = {
|
|
3374
|
-
id: sagaId,
|
|
3375
|
-
orderId: 'ORDER-123',
|
|
3376
|
-
amount: 100,
|
|
3377
|
-
items: ['item1', 'item2'],
|
|
3378
|
-
status: 'started'
|
|
3379
|
-
};
|
|
3380
|
-
|
|
3381
|
-
// Execute saga steps
|
|
3382
|
-
let currentData = { ...sagaData };
|
|
3383
|
-
let failedAtStep = -1;
|
|
3384
|
-
|
|
3385
|
-
for (let i = 0; i < sagaSteps.length; i++) {
|
|
3386
|
-
const step = sagaSteps[i];
|
|
3387
|
-
|
|
3388
|
-
// Push to step queue
|
|
3389
|
-
await client.push({
|
|
3390
|
-
items: [{
|
|
3391
|
-
queue: step,
|
|
3392
|
-
payload: {
|
|
3393
|
-
...currentData,
|
|
3394
|
-
step: i,
|
|
3395
|
-
stepName: step
|
|
3396
|
-
}
|
|
3397
|
-
}]
|
|
3398
|
-
});
|
|
3399
|
-
|
|
3400
|
-
// Process step
|
|
3401
|
-
const result = await client.pop({ queue: step, batch: 1 });
|
|
3402
|
-
|
|
3403
|
-
if (result.messages && result.messages.length > 0) {
|
|
3404
|
-
const msg = result.messages[0];
|
|
3405
|
-
|
|
3406
|
-
// Simulate failure at payment step
|
|
3407
|
-
if (step === 'pattern-saga-payment' && Math.random() > 0.5) {
|
|
3408
|
-
failedAtStep = i;
|
|
3409
|
-
await client.ack(msg.transactionId, 'failed', 'Payment failed');
|
|
3410
|
-
break;
|
|
3411
|
-
}
|
|
3412
|
-
|
|
3413
|
-
// Step succeeded
|
|
3414
|
-
currentData = {
|
|
3415
|
-
...msg.data,
|
|
3416
|
-
[`${step}_completed`]: true
|
|
3417
|
-
};
|
|
3418
|
-
|
|
3419
|
-
await client.ack(msg.transactionId, 'completed');
|
|
3420
|
-
}
|
|
3421
|
-
}
|
|
3422
|
-
|
|
3423
|
-
// If saga failed, execute compensation
|
|
3424
|
-
if (failedAtStep >= 0) {
|
|
3425
|
-
log('Saga failed, starting compensation...', 'warning');
|
|
3426
|
-
|
|
3427
|
-
for (let i = failedAtStep; i >= 0; i--) {
|
|
3428
|
-
const compensationStep = compensationSteps[compensationSteps.length - 1 - i];
|
|
3429
|
-
|
|
3430
|
-
await client.push({
|
|
3431
|
-
items: [{
|
|
3432
|
-
queue: compensationStep,
|
|
3433
|
-
payload: {
|
|
3434
|
-
...currentData,
|
|
3435
|
-
compensationStep: i,
|
|
3436
|
-
compensationReason: 'Saga rollback'
|
|
3437
|
-
}
|
|
3438
|
-
}]
|
|
3439
|
-
});
|
|
3440
|
-
|
|
3441
|
-
const result = await client.pop({ queue: compensationStep, batch: 1 });
|
|
3442
|
-
|
|
3443
|
-
if (result.messages && result.messages.length > 0) {
|
|
3444
|
-
await client.ack(result.messages[0].transactionId, 'completed');
|
|
3445
|
-
}
|
|
3446
|
-
}
|
|
3447
|
-
}
|
|
3448
|
-
|
|
3449
|
-
passTest('Saga pattern with compensation executed successfully');
|
|
3450
|
-
} catch (error) {
|
|
3451
|
-
failTest(error);
|
|
3452
|
-
}
|
|
3453
|
-
}
|
|
3454
|
-
|
|
3455
|
-
/**
|
|
3456
|
-
* Test: Rate limiting and throttling
|
|
3457
|
-
*/
|
|
3458
|
-
async function testRateLimiting() {
|
|
3459
|
-
startTest('Rate Limiting and Throttling', 'pattern');
|
|
3460
|
-
|
|
3461
|
-
try {
|
|
3462
|
-
const queue = 'pattern-rate-limited';
|
|
3463
|
-
const rateLimit = 5; // 5 messages per second
|
|
3464
|
-
|
|
3465
|
-
// Configure queue
|
|
3466
|
-
await client.configure({
|
|
3467
|
-
queue,
|
|
3468
|
-
options: {
|
|
3469
|
-
windowBuffer: 1 // 1 second window
|
|
3470
|
-
}
|
|
3471
|
-
});
|
|
3472
|
-
|
|
3473
|
-
// Push burst of messages
|
|
3474
|
-
const burstSize = 20;
|
|
3475
|
-
await client.push({
|
|
3476
|
-
items: Array.from({ length: burstSize }, (_, i) => ({
|
|
3477
|
-
queue,
|
|
3478
|
-
payload: { id: i, timestamp: Date.now() }
|
|
3479
|
-
}))
|
|
3480
|
-
});
|
|
3481
|
-
|
|
3482
|
-
// Implement rate-limited consumer
|
|
3483
|
-
const processedMessages = [];
|
|
3484
|
-
const startTime = Date.now();
|
|
3485
|
-
let lastWindowStart = startTime;
|
|
3486
|
-
let windowMessageCount = 0;
|
|
3487
|
-
|
|
3488
|
-
while (processedMessages.length < burstSize) {
|
|
3489
|
-
const currentTime = Date.now();
|
|
3490
|
-
|
|
3491
|
-
// Reset window if needed
|
|
3492
|
-
if (currentTime - lastWindowStart >= 1000) {
|
|
3493
|
-
lastWindowStart = currentTime;
|
|
3494
|
-
windowMessageCount = 0;
|
|
3495
|
-
}
|
|
3496
|
-
|
|
3497
|
-
// Check if we can process more messages
|
|
3498
|
-
if (windowMessageCount < rateLimit) {
|
|
3499
|
-
const result = await client.pop({
|
|
3500
|
-
queue,
|
|
3501
|
-
batch: Math.min(rateLimit - windowMessageCount, burstSize - processedMessages.length)
|
|
3502
|
-
});
|
|
3503
|
-
|
|
3504
|
-
if (result.messages && result.messages.length > 0) {
|
|
3505
|
-
for (const msg of result.messages) {
|
|
3506
|
-
processedMessages.push({
|
|
3507
|
-
id: msg.data.id,
|
|
3508
|
-
processedAt: Date.now() - startTime
|
|
3509
|
-
});
|
|
3510
|
-
|
|
3511
|
-
windowMessageCount++;
|
|
3512
|
-
await client.ack(msg.transactionId, 'completed');
|
|
3513
|
-
}
|
|
3514
|
-
}
|
|
3515
|
-
} else {
|
|
3516
|
-
// Wait for next window
|
|
3517
|
-
await sleep(Math.max(0, 1000 - (currentTime - lastWindowStart)));
|
|
3518
|
-
}
|
|
3519
|
-
}
|
|
3520
|
-
|
|
3521
|
-
const totalTime = Date.now() - startTime;
|
|
3522
|
-
const actualRate = (burstSize / (totalTime / 1000)).toFixed(2);
|
|
3523
|
-
|
|
3524
|
-
log(`Processed ${burstSize} messages at ${actualRate} msg/sec (limit: ${rateLimit})`, 'info');
|
|
3525
|
-
|
|
3526
|
-
// Allow up to 40% variance due to timing variations
|
|
3527
|
-
if (actualRate > rateLimit * 1.4) {
|
|
3528
|
-
throw new Error(`Rate limit exceeded: ${actualRate} > ${rateLimit * 1.4}`);
|
|
3529
|
-
}
|
|
3530
|
-
|
|
3531
|
-
passTest(`Rate limiting enforced: ${actualRate} msg/sec`);
|
|
3532
|
-
} catch (error) {
|
|
3533
|
-
failTest(error);
|
|
3534
|
-
}
|
|
3535
|
-
}
|
|
3536
|
-
|
|
3537
|
-
/**
|
|
3538
|
-
* Test: Message deduplication
|
|
3539
|
-
*/
|
|
3540
|
-
async function testMessageDeduplication() {
|
|
3541
|
-
startTest('Message Deduplication', 'pattern');
|
|
3542
|
-
|
|
3543
|
-
try {
|
|
3544
|
-
const queue = 'pattern-dedup';
|
|
3545
|
-
|
|
3546
|
-
// Configure queue
|
|
3547
|
-
await client.configure({
|
|
3548
|
-
queue: queue,
|
|
3549
|
-
options: {}
|
|
3550
|
-
});
|
|
3551
|
-
|
|
3552
|
-
const deduplicationWindow = new Map();
|
|
3553
|
-
|
|
3554
|
-
// Push messages with some duplicates
|
|
3555
|
-
const messages = [
|
|
3556
|
-
{ id: 'msg-1', data: 'First' },
|
|
3557
|
-
{ id: 'msg-2', data: 'Second' },
|
|
3558
|
-
{ id: 'msg-1', data: 'First duplicate' }, // Duplicate
|
|
3559
|
-
{ id: 'msg-3', data: 'Third' },
|
|
3560
|
-
{ id: 'msg-2', data: 'Second duplicate' }, // Duplicate
|
|
3561
|
-
{ id: 'msg-4', data: 'Fourth' }
|
|
3562
|
-
];
|
|
3563
|
-
|
|
3564
|
-
await client.push({
|
|
3565
|
-
items: messages.map(msg => ({
|
|
3566
|
-
queue,
|
|
3567
|
-
payload: msg
|
|
3568
|
-
}))
|
|
3569
|
-
});
|
|
3570
|
-
|
|
3571
|
-
// Process with deduplication
|
|
3572
|
-
const uniqueMessages = [];
|
|
3573
|
-
const result = await client.pop({
|
|
3574
|
-
queue,
|
|
3575
|
-
batch: messages.length
|
|
3576
|
-
});
|
|
3577
|
-
|
|
3578
|
-
if (result.messages) {
|
|
3579
|
-
for (const msg of result.messages) {
|
|
3580
|
-
const messageId = msg.data.id;
|
|
3581
|
-
|
|
3582
|
-
// Check for duplicate
|
|
3583
|
-
if (!deduplicationWindow.has(messageId)) {
|
|
3584
|
-
deduplicationWindow.set(messageId, Date.now());
|
|
3585
|
-
uniqueMessages.push(msg);
|
|
3586
|
-
} else {
|
|
3587
|
-
log(`Duplicate detected: ${messageId}`, 'warning');
|
|
3588
|
-
}
|
|
3589
|
-
|
|
3590
|
-
await client.ack(msg.transactionId, 'completed');
|
|
3591
|
-
}
|
|
3592
|
-
}
|
|
3593
|
-
|
|
3594
|
-
// Clean old entries from dedup window (in production, this would be time-based)
|
|
3595
|
-
if (deduplicationWindow.size > 1000) {
|
|
3596
|
-
const entries = Array.from(deduplicationWindow.entries());
|
|
3597
|
-
entries.sort((a, b) => a[1] - b[1]);
|
|
3598
|
-
for (let i = 0; i < entries.length / 2; i++) {
|
|
3599
|
-
deduplicationWindow.delete(entries[i][0]);
|
|
3600
|
-
}
|
|
3601
|
-
}
|
|
3602
|
-
|
|
3603
|
-
passTest(`Deduplication successful: ${messages.length} messages → ${uniqueMessages.length} unique`);
|
|
3604
|
-
} catch (error) {
|
|
3605
|
-
failTest(error);
|
|
3606
|
-
}
|
|
3607
|
-
}
|
|
3608
|
-
|
|
3609
|
-
/**
|
|
3610
|
-
* Test: Time-based batch processing
|
|
3611
|
-
*/
|
|
3612
|
-
async function testTimeBatchProcessing() {
|
|
3613
|
-
startTest('Time-Based Batch Processing', 'pattern');
|
|
3614
|
-
|
|
3615
|
-
try {
|
|
3616
|
-
const queue = 'pattern-time-batch';
|
|
3617
|
-
|
|
3618
|
-
// Configure queue
|
|
3619
|
-
await client.configure({
|
|
3620
|
-
queue: queue,
|
|
3621
|
-
options: {}
|
|
3622
|
-
});
|
|
3623
|
-
|
|
3624
|
-
const batchWindow = 2000; // 2 seconds
|
|
3625
|
-
const minBatchSize = 3;
|
|
3626
|
-
const maxBatchSize = 10;
|
|
3627
|
-
|
|
3628
|
-
// Push messages over time
|
|
3629
|
-
const messageCount = 20;
|
|
3630
|
-
for (let i = 0; i < messageCount; i++) {
|
|
3631
|
-
await client.push({
|
|
3632
|
-
items: [{
|
|
3633
|
-
queue,
|
|
3634
|
-
payload: { id: i, timestamp: Date.now() }
|
|
3635
|
-
}]
|
|
3636
|
-
});
|
|
3637
|
-
|
|
3638
|
-
// Stagger message arrival
|
|
3639
|
-
if (i % 5 === 0) {
|
|
3640
|
-
await sleep(500);
|
|
3641
|
-
}
|
|
3642
|
-
}
|
|
3643
|
-
|
|
3644
|
-
// Process in time-based batches
|
|
3645
|
-
const batches = [];
|
|
3646
|
-
let currentBatch = [];
|
|
3647
|
-
let batchStartTime = Date.now();
|
|
3648
|
-
let totalProcessed = 0;
|
|
3649
|
-
|
|
3650
|
-
while (totalProcessed < messageCount) {
|
|
3651
|
-
const currentTime = Date.now();
|
|
3652
|
-
const batchAge = currentTime - batchStartTime;
|
|
3653
|
-
|
|
3654
|
-
// Check if batch should be processed
|
|
3655
|
-
if (currentBatch.length >= maxBatchSize ||
|
|
3656
|
-
(batchAge >= batchWindow && currentBatch.length >= minBatchSize)) {
|
|
3657
|
-
|
|
3658
|
-
batches.push({
|
|
3659
|
-
size: currentBatch.length,
|
|
3660
|
-
duration: batchAge,
|
|
3661
|
-
messages: [...currentBatch]
|
|
3662
|
-
});
|
|
3663
|
-
|
|
3664
|
-
// Acknowledge batch
|
|
3665
|
-
for (const msg of currentBatch) {
|
|
3666
|
-
await client.ack(msg.transactionId, 'completed');
|
|
3667
|
-
}
|
|
3668
|
-
|
|
3669
|
-
totalProcessed += currentBatch.length;
|
|
3670
|
-
currentBatch = [];
|
|
3671
|
-
batchStartTime = Date.now();
|
|
3672
|
-
}
|
|
3673
|
-
|
|
3674
|
-
// Try to add more messages to batch
|
|
3675
|
-
const remaining = Math.min(maxBatchSize - currentBatch.length, messageCount - totalProcessed);
|
|
3676
|
-
if (remaining > 0) {
|
|
3677
|
-
const result = await client.pop({
|
|
3678
|
-
queue,
|
|
3679
|
-
batch: remaining
|
|
3680
|
-
});
|
|
3681
|
-
|
|
3682
|
-
if (result.messages && result.messages.length > 0) {
|
|
3683
|
-
currentBatch.push(...result.messages);
|
|
3684
|
-
} else if (batchAge >= batchWindow && currentBatch.length > 0) {
|
|
3685
|
-
// Force process partial batch after timeout
|
|
3686
|
-
batches.push({
|
|
3687
|
-
size: currentBatch.length,
|
|
3688
|
-
duration: batchAge,
|
|
3689
|
-
messages: [...currentBatch]
|
|
3690
|
-
});
|
|
3691
|
-
|
|
3692
|
-
for (const msg of currentBatch) {
|
|
3693
|
-
await client.ack(msg.transactionId, 'completed');
|
|
3694
|
-
}
|
|
3695
|
-
|
|
3696
|
-
totalProcessed += currentBatch.length;
|
|
3697
|
-
currentBatch = [];
|
|
3698
|
-
batchStartTime = Date.now();
|
|
3699
|
-
} else {
|
|
3700
|
-
await sleep(100);
|
|
3701
|
-
}
|
|
3702
|
-
}
|
|
3703
|
-
}
|
|
3704
|
-
|
|
3705
|
-
log(`Created ${batches.length} time-based batches`, 'info');
|
|
3706
|
-
batches.forEach(batch => {
|
|
3707
|
-
log(` Batch: ${batch.size} messages in ${batch.duration}ms`, 'info');
|
|
3708
|
-
});
|
|
3709
|
-
|
|
3710
|
-
passTest(`Time-based batching created ${batches.length} batches`);
|
|
3711
|
-
} catch (error) {
|
|
3712
|
-
failTest(error);
|
|
3713
|
-
}
|
|
3714
|
-
}
|
|
3715
|
-
|
|
3716
|
-
/**
|
|
3717
|
-
* Test: Event sourcing with replay
|
|
3718
|
-
*/
|
|
3719
|
-
async function testEventSourcing() {
|
|
3720
|
-
startTest('Event Sourcing with Replay', 'pattern');
|
|
3721
|
-
|
|
3722
|
-
try {
|
|
3723
|
-
const eventQueue = 'pattern-event-store';
|
|
3724
|
-
const snapshotQueue = 'pattern-snapshots';
|
|
3725
|
-
|
|
3726
|
-
// Configure queues
|
|
3727
|
-
await client.configure({
|
|
3728
|
-
queue: eventQueue,
|
|
3729
|
-
options: {
|
|
3730
|
-
retentionEnabled: true,
|
|
3731
|
-
retentionSeconds: 86400 // Keep events for 24 hours
|
|
3732
|
-
}
|
|
3733
|
-
});
|
|
3734
|
-
|
|
3735
|
-
await client.configure({
|
|
3736
|
-
queue: snapshotQueue
|
|
3737
|
-
});
|
|
3738
|
-
|
|
3739
|
-
// Generate events
|
|
3740
|
-
const entityId = crypto.randomBytes(8).toString('hex');
|
|
3741
|
-
const events = [
|
|
3742
|
-
{ type: 'created', entityId, data: { name: 'Test Entity' }, version: 1 },
|
|
3743
|
-
{ type: 'updated', entityId, data: { name: 'Updated Entity' }, version: 2 },
|
|
3744
|
-
{ type: 'statusChanged', entityId, data: { status: 'active' }, version: 3 },
|
|
3745
|
-
{ type: 'updated', entityId, data: { name: 'Final Entity' }, version: 4 },
|
|
3746
|
-
{ type: 'statusChanged', entityId, data: { status: 'completed' }, version: 5 }
|
|
3747
|
-
];
|
|
3748
|
-
|
|
3749
|
-
// Store events
|
|
3750
|
-
for (const event of events) {
|
|
3751
|
-
await client.push({
|
|
3752
|
-
items: [{
|
|
3753
|
-
queue: eventQueue,
|
|
3754
|
-
payload: {
|
|
3755
|
-
...event,
|
|
3756
|
-
timestamp: Date.now()
|
|
3757
|
-
}
|
|
3758
|
-
}]
|
|
3759
|
-
});
|
|
3760
|
-
}
|
|
3761
|
-
|
|
3762
|
-
// Replay events to rebuild state
|
|
3763
|
-
const rebuiltState = {
|
|
3764
|
-
entityId,
|
|
3765
|
-
version: 0,
|
|
3766
|
-
data: {}
|
|
3767
|
-
};
|
|
3768
|
-
|
|
3769
|
-
const allEvents = await client.pop({
|
|
3770
|
-
queue: eventQueue,
|
|
3771
|
-
batch: 100
|
|
3772
|
-
});
|
|
3773
|
-
|
|
3774
|
-
if (allEvents.messages) {
|
|
3775
|
-
// Sort by version to ensure correct order
|
|
3776
|
-
allEvents.messages.sort((a, b) => a.data.version - b.data.version);
|
|
3777
|
-
|
|
3778
|
-
for (const msg of allEvents.messages) {
|
|
3779
|
-
const event = msg.data;
|
|
3780
|
-
|
|
3781
|
-
switch (event.type) {
|
|
3782
|
-
case 'created':
|
|
3783
|
-
rebuiltState.data = event.data;
|
|
3784
|
-
break;
|
|
3785
|
-
case 'updated':
|
|
3786
|
-
rebuiltState.data = { ...rebuiltState.data, ...event.data };
|
|
3787
|
-
break;
|
|
3788
|
-
case 'statusChanged':
|
|
3789
|
-
rebuiltState.data.status = event.data.status;
|
|
3790
|
-
break;
|
|
3791
|
-
}
|
|
3792
|
-
|
|
3793
|
-
rebuiltState.version = event.version;
|
|
3794
|
-
|
|
3795
|
-
// Don't ack events - we want to keep them for replay
|
|
3796
|
-
}
|
|
3797
|
-
}
|
|
3798
|
-
|
|
3799
|
-
// Create snapshot after replay
|
|
3800
|
-
if (rebuiltState.version === events.length) {
|
|
3801
|
-
await client.push({
|
|
3802
|
-
items: [{
|
|
3803
|
-
queue: snapshotQueue,
|
|
3804
|
-
payload: {
|
|
3805
|
-
...rebuiltState,
|
|
3806
|
-
snapshotTime: Date.now()
|
|
3807
|
-
}
|
|
3808
|
-
}]
|
|
3809
|
-
});
|
|
3810
|
-
}
|
|
3811
|
-
|
|
3812
|
-
passTest(`Event sourcing: ${events.length} events replayed, state rebuilt`);
|
|
3813
|
-
} catch (error) {
|
|
3814
|
-
failTest(error);
|
|
3815
|
-
}
|
|
3816
|
-
}
|
|
3817
|
-
|
|
3818
|
-
/**
|
|
3819
|
-
* Test: Queue metrics and monitoring
|
|
3820
|
-
*/
|
|
3821
|
-
async function testQueueMetrics() {
|
|
3822
|
-
startTest('Queue Metrics and Monitoring', 'pattern');
|
|
3823
|
-
|
|
3824
|
-
try {
|
|
3825
|
-
const queue = 'pattern-metrics';
|
|
3826
|
-
const messageCount = 50;
|
|
3827
|
-
|
|
3828
|
-
// Configure queue
|
|
3829
|
-
await client.configure({
|
|
3830
|
-
queue,
|
|
3831
|
-
options: {
|
|
3832
|
-
leaseTime: 10
|
|
3833
|
-
}
|
|
3834
|
-
});
|
|
3835
|
-
|
|
3836
|
-
// Track metrics
|
|
3837
|
-
const metrics = {
|
|
3838
|
-
published: 0,
|
|
3839
|
-
consumed: 0,
|
|
3840
|
-
completed: 0,
|
|
3841
|
-
failed: 0,
|
|
3842
|
-
processingTimes: [],
|
|
3843
|
-
throughput: 0
|
|
3844
|
-
};
|
|
3845
|
-
|
|
3846
|
-
const startTime = Date.now();
|
|
3847
|
-
|
|
3848
|
-
// Publish messages
|
|
3849
|
-
for (let i = 0; i < messageCount; i++) {
|
|
3850
|
-
await client.push({
|
|
3851
|
-
items: [{
|
|
3852
|
-
queue,
|
|
3853
|
-
payload: {
|
|
3854
|
-
id: i,
|
|
3855
|
-
shouldFail: Math.random() > 0.7, // 30% failure rate
|
|
3856
|
-
processingTime: Math.floor(Math.random() * 100) // 0-100ms
|
|
3857
|
-
}
|
|
3858
|
-
}]
|
|
3859
|
-
});
|
|
3860
|
-
metrics.published++;
|
|
3861
|
-
}
|
|
3862
|
-
|
|
3863
|
-
// Consume and process
|
|
3864
|
-
while (metrics.consumed < messageCount) {
|
|
3865
|
-
const result = await client.pop({
|
|
3866
|
-
queue,
|
|
3867
|
-
batch: 10
|
|
3868
|
-
});
|
|
3869
|
-
|
|
3870
|
-
if (result.messages && result.messages.length > 0) {
|
|
3871
|
-
for (const msg of result.messages) {
|
|
3872
|
-
metrics.consumed++;
|
|
3873
|
-
const processStart = Date.now();
|
|
3874
|
-
|
|
3875
|
-
// Simulate processing
|
|
3876
|
-
await sleep(msg.data.processingTime);
|
|
3877
|
-
|
|
3878
|
-
if (msg.data.shouldFail) {
|
|
3879
|
-
await client.ack(msg.transactionId, 'failed', 'Simulated failure');
|
|
3880
|
-
metrics.failed++;
|
|
3881
|
-
} else {
|
|
3882
|
-
await client.ack(msg.transactionId, 'completed');
|
|
3883
|
-
metrics.completed++;
|
|
3884
|
-
}
|
|
3885
|
-
|
|
3886
|
-
metrics.processingTimes.push(Date.now() - processStart);
|
|
3887
|
-
}
|
|
3888
|
-
} else {
|
|
3889
|
-
await sleep(100);
|
|
3890
|
-
}
|
|
3891
|
-
}
|
|
3892
|
-
|
|
3893
|
-
const totalTime = Date.now() - startTime;
|
|
3894
|
-
metrics.throughput = (messageCount / (totalTime / 1000)).toFixed(2);
|
|
3895
|
-
|
|
3896
|
-
// Calculate additional metrics
|
|
3897
|
-
const avgProcessingTime = metrics.processingTimes.reduce((a, b) => a + b, 0) / metrics.processingTimes.length;
|
|
3898
|
-
const maxProcessingTime = Math.max(...metrics.processingTimes);
|
|
3899
|
-
const minProcessingTime = Math.min(...metrics.processingTimes);
|
|
3900
|
-
|
|
3901
|
-
// Query database metrics - simplified query to avoid blocking
|
|
3902
|
-
let dbMetrics;
|
|
3903
|
-
try {
|
|
3904
|
-
// Use a simpler, faster query with timeout
|
|
3905
|
-
const queryPromise = dbPool.query(`
|
|
3906
|
-
SELECT
|
|
3907
|
-
COUNT(m.id) as total_messages,
|
|
3908
|
-
COUNT(CASE WHEN ms.status = 'completed' THEN 1 END) as completed,
|
|
3909
|
-
COUNT(CASE WHEN ms.status = 'failed' THEN 1 END) as failed,
|
|
3910
|
-
COUNT(CASE WHEN ms.status = 'processing' THEN 1 END) as processing
|
|
3911
|
-
FROM queen.queues q
|
|
3912
|
-
JOIN queen.partitions p ON p.queue_id = q.id
|
|
3913
|
-
JOIN queen.messages m ON m.partition_id = p.id
|
|
3914
|
-
LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
|
|
3915
|
-
WHERE q.name = $1
|
|
3916
|
-
`, [queue]);
|
|
3917
|
-
|
|
3918
|
-
// Add timeout to prevent hanging
|
|
3919
|
-
dbMetrics = await Promise.race([
|
|
3920
|
-
queryPromise,
|
|
3921
|
-
new Promise((resolve, reject) =>
|
|
3922
|
-
setTimeout(() => reject(new Error('Query timeout')), 5000)
|
|
3923
|
-
)
|
|
3924
|
-
]);
|
|
3925
|
-
} catch (error) {
|
|
3926
|
-
log('Database metrics query failed or timed out', 'warning');
|
|
3927
|
-
// Continue with test even if metrics query fails
|
|
3928
|
-
dbMetrics = { rows: [{
|
|
3929
|
-
total_messages: metrics.published,
|
|
3930
|
-
completed: metrics.completed,
|
|
3931
|
-
failed: metrics.failed,
|
|
3932
|
-
processing: 0
|
|
3933
|
-
}] };
|
|
3934
|
-
}
|
|
3935
|
-
|
|
3936
|
-
log('Queue Metrics:', 'info');
|
|
3937
|
-
log(` Published: ${metrics.published}`, 'info');
|
|
3938
|
-
log(` Consumed: ${metrics.consumed}`, 'info');
|
|
3939
|
-
log(` Completed: ${metrics.completed}`, 'info');
|
|
3940
|
-
log(` Failed: ${metrics.failed}`, 'info');
|
|
3941
|
-
log(` Avg Processing: ${avgProcessingTime.toFixed(2)}ms`, 'info');
|
|
3942
|
-
log(` Throughput: ${metrics.throughput} msg/sec`, 'info');
|
|
3943
|
-
|
|
3944
|
-
passTest('Queue metrics collected successfully');
|
|
3945
|
-
} catch (error) {
|
|
3946
|
-
failTest(error);
|
|
3947
|
-
}
|
|
3948
|
-
}
|
|
3949
|
-
|
|
3950
|
-
/**
|
|
3951
|
-
* Test: Circuit breaker pattern
|
|
3952
|
-
*/
|
|
3953
|
-
async function testCircuitBreaker() {
|
|
3954
|
-
startTest('Circuit Breaker Pattern', 'pattern');
|
|
3955
|
-
|
|
3956
|
-
try {
|
|
3957
|
-
const queue = 'pattern-circuit-breaker';
|
|
3958
|
-
|
|
3959
|
-
// Configure queue
|
|
3960
|
-
await client.configure({
|
|
3961
|
-
queue: queue,
|
|
3962
|
-
options: {}
|
|
3963
|
-
});
|
|
3964
|
-
|
|
3965
|
-
const circuitBreaker = {
|
|
3966
|
-
state: 'CLOSED', // CLOSED, OPEN, HALF_OPEN
|
|
3967
|
-
failureCount: 0,
|
|
3968
|
-
successCount: 0,
|
|
3969
|
-
failureThreshold: 3,
|
|
3970
|
-
successThreshold: 2,
|
|
3971
|
-
timeout: 2000,
|
|
3972
|
-
lastFailureTime: null
|
|
3973
|
-
};
|
|
3974
|
-
|
|
3975
|
-
// Configure queue
|
|
3976
|
-
await client.configure({ queue });
|
|
3977
|
-
|
|
3978
|
-
// Push messages
|
|
3979
|
-
const messages = Array.from({ length: 20 }, (_, i) => ({
|
|
3980
|
-
queue,
|
|
3981
|
-
payload: {
|
|
3982
|
-
id: i,
|
|
3983
|
-
shouldFail: i < 10 // First 10 will fail
|
|
3984
|
-
}
|
|
3985
|
-
}));
|
|
3986
|
-
|
|
3987
|
-
await client.push({ items: messages });
|
|
3988
|
-
|
|
3989
|
-
// Process with circuit breaker
|
|
3990
|
-
let processedCount = 0;
|
|
3991
|
-
const maxAttempts = 30;
|
|
3992
|
-
|
|
3993
|
-
for (let attempt = 0; attempt < maxAttempts && processedCount < messages.length; attempt++) {
|
|
3994
|
-
// Check circuit breaker state
|
|
3995
|
-
if (circuitBreaker.state === 'OPEN') {
|
|
3996
|
-
const timeSinceFailure = Date.now() - circuitBreaker.lastFailureTime;
|
|
3997
|
-
if (timeSinceFailure >= circuitBreaker.timeout) {
|
|
3998
|
-
circuitBreaker.state = 'HALF_OPEN';
|
|
3999
|
-
circuitBreaker.failureCount = 0;
|
|
4000
|
-
circuitBreaker.successCount = 0;
|
|
4001
|
-
log('Circuit breaker: OPEN → HALF_OPEN', 'warning');
|
|
4002
|
-
} else {
|
|
4003
|
-
log('Circuit breaker is OPEN, waiting...', 'warning');
|
|
4004
|
-
await sleep(500);
|
|
4005
|
-
continue;
|
|
4006
|
-
}
|
|
4007
|
-
}
|
|
4008
|
-
|
|
4009
|
-
// Try to process message
|
|
4010
|
-
const result = await client.pop({ queue, batch: 1 });
|
|
4011
|
-
|
|
4012
|
-
if (result.messages && result.messages.length > 0) {
|
|
4013
|
-
const msg = result.messages[0];
|
|
4014
|
-
|
|
4015
|
-
try {
|
|
4016
|
-
if (msg.data.shouldFail && circuitBreaker.state !== 'OPEN') {
|
|
4017
|
-
throw new Error('Simulated failure');
|
|
4018
|
-
}
|
|
4019
|
-
|
|
4020
|
-
// Success
|
|
4021
|
-
await client.ack(msg.transactionId, 'completed');
|
|
4022
|
-
processedCount++;
|
|
4023
|
-
|
|
4024
|
-
if (circuitBreaker.state === 'HALF_OPEN') {
|
|
4025
|
-
circuitBreaker.successCount++;
|
|
4026
|
-
if (circuitBreaker.successCount >= circuitBreaker.successThreshold) {
|
|
4027
|
-
circuitBreaker.state = 'CLOSED';
|
|
4028
|
-
circuitBreaker.failureCount = 0;
|
|
4029
|
-
log('Circuit breaker: HALF_OPEN → CLOSED', 'success');
|
|
4030
|
-
}
|
|
4031
|
-
}
|
|
4032
|
-
} catch (error) {
|
|
4033
|
-
// Failure
|
|
4034
|
-
await client.ack(msg.transactionId, 'failed', error.message);
|
|
4035
|
-
circuitBreaker.failureCount++;
|
|
4036
|
-
circuitBreaker.lastFailureTime = Date.now();
|
|
4037
|
-
|
|
4038
|
-
if (circuitBreaker.state === 'CLOSED' &&
|
|
4039
|
-
circuitBreaker.failureCount >= circuitBreaker.failureThreshold) {
|
|
4040
|
-
circuitBreaker.state = 'OPEN';
|
|
4041
|
-
log('Circuit breaker: CLOSED → OPEN (threshold reached)', 'error');
|
|
4042
|
-
} else if (circuitBreaker.state === 'HALF_OPEN') {
|
|
4043
|
-
circuitBreaker.state = 'OPEN';
|
|
4044
|
-
log('Circuit breaker: HALF_OPEN → OPEN (failure during recovery)', 'error');
|
|
4045
|
-
}
|
|
4046
|
-
}
|
|
4047
|
-
} else {
|
|
4048
|
-
await sleep(100);
|
|
4049
|
-
}
|
|
4050
|
-
}
|
|
4051
|
-
|
|
4052
|
-
log(`Circuit breaker results: ${processedCount - circuitBreaker.failureCount} success, ${circuitBreaker.failureCount} failed`, 'info');
|
|
4053
|
-
passTest('Circuit breaker pattern implemented successfully');
|
|
4054
|
-
} catch (error) {
|
|
4055
|
-
failTest(error);
|
|
4056
|
-
}
|
|
4057
|
-
}
|
|
4058
|
-
|
|
4059
|
-
/**
|
|
4060
|
-
* Test: Multiple Consumers - Single Partition Message Ordering
|
|
4061
|
-
* Verifies that multiple consumers consuming from a single partition
|
|
4062
|
-
* maintain message ordering when processing messages
|
|
4063
|
-
*/
|
|
4064
|
-
async function testMultipleConsumersSinglePartitionOrdering() {
|
|
4065
|
-
startTest('Multiple Consumers - Single Partition Message Ordering', 'pattern');
|
|
4066
|
-
|
|
4067
|
-
try {
|
|
4068
|
-
const queue = 'test-multi-consumer-ordering';
|
|
4069
|
-
const partition = 'single-partition';
|
|
4070
|
-
const numMessages = 999;
|
|
4071
|
-
const numConsumers = 10;
|
|
4072
|
-
const batchSize = 50; // Each consumer will process messages in batches
|
|
4073
|
-
|
|
4074
|
-
// Configure queue with single partition
|
|
4075
|
-
await client.configure({
|
|
4076
|
-
queue,
|
|
4077
|
-
options: {
|
|
4078
|
-
leaseTime: 30, // 30 seconds lease time
|
|
4079
|
-
retryLimit: 3
|
|
4080
|
-
}
|
|
4081
|
-
});
|
|
4082
|
-
|
|
4083
|
-
log(`Pushing ${numMessages} messages to queue '${queue}', partition '${partition}'`);
|
|
4084
|
-
|
|
4085
|
-
// Push messages with sequential order identifiers
|
|
4086
|
-
const pushStartTime = Date.now();
|
|
4087
|
-
const items = Array.from({ length: numMessages }, (_, i) => ({
|
|
4088
|
-
queue,
|
|
4089
|
-
partition,
|
|
4090
|
-
payload: {
|
|
4091
|
-
sequenceNumber: i,
|
|
4092
|
-
data: `Message ${i}`,
|
|
4093
|
-
timestamp: Date.now()
|
|
4094
|
-
}
|
|
4095
|
-
}));
|
|
4096
|
-
|
|
4097
|
-
// Push in batches to avoid overwhelming the system
|
|
4098
|
-
const pushBatchSize = 100;
|
|
4099
|
-
for (let i = 0; i < items.length; i += pushBatchSize) {
|
|
4100
|
-
const batch = items.slice(i, Math.min(i + pushBatchSize, items.length));
|
|
4101
|
-
await client.push({ items: batch });
|
|
4102
|
-
}
|
|
4103
|
-
|
|
4104
|
-
const pushTime = Date.now() - pushStartTime;
|
|
4105
|
-
log(`Pushed ${numMessages} messages in ${pushTime}ms`);
|
|
4106
|
-
|
|
4107
|
-
// Wait a moment for messages to be fully persisted
|
|
4108
|
-
await sleep(500);
|
|
4109
|
-
|
|
4110
|
-
// Verify messages are in the queue
|
|
4111
|
-
const messageCount = await getMessageCount(queue, partition);
|
|
4112
|
-
if (messageCount !== numMessages) {
|
|
4113
|
-
throw new Error(`Expected ${numMessages} messages in queue, but found ${messageCount}`);
|
|
4114
|
-
}
|
|
4115
|
-
|
|
4116
|
-
log(`Starting ${numConsumers} consumers to process messages`);
|
|
4117
|
-
|
|
4118
|
-
// Track all consumed messages and their order
|
|
4119
|
-
const consumedMessages = [];
|
|
4120
|
-
const consumerPromises = [];
|
|
4121
|
-
|
|
4122
|
-
// Create multiple consumers
|
|
4123
|
-
for (let consumerId = 0; consumerId < numConsumers; consumerId++) {
|
|
4124
|
-
const consumerPromise = (async () => {
|
|
4125
|
-
const consumerMessages = [];
|
|
4126
|
-
let consecutiveEmptyPolls = 0;
|
|
4127
|
-
const maxEmptyPolls = 3; // Stop after 3 consecutive empty polls
|
|
4128
|
-
|
|
4129
|
-
while (consecutiveEmptyPolls < maxEmptyPolls) {
|
|
4130
|
-
try {
|
|
4131
|
-
// Pop messages in batches
|
|
4132
|
-
const result = await client.pop({
|
|
4133
|
-
queue,
|
|
4134
|
-
partition,
|
|
4135
|
-
batch: batchSize
|
|
4136
|
-
});
|
|
4137
|
-
|
|
4138
|
-
if (!result.messages || result.messages.length === 0) {
|
|
4139
|
-
// No more messages available
|
|
4140
|
-
consecutiveEmptyPolls++;
|
|
4141
|
-
await sleep(100);
|
|
4142
|
-
continue;
|
|
4143
|
-
}
|
|
4144
|
-
|
|
4145
|
-
// Reset empty poll counter when we get messages
|
|
4146
|
-
consecutiveEmptyPolls = 0;
|
|
4147
|
-
|
|
4148
|
-
// Process messages and track their sequence numbers
|
|
4149
|
-
for (const msg of result.messages) {
|
|
4150
|
-
const sequenceNumber = msg.data.sequenceNumber;
|
|
4151
|
-
consumerMessages.push({
|
|
4152
|
-
consumerId,
|
|
4153
|
-
sequenceNumber,
|
|
4154
|
-
timestamp: Date.now(),
|
|
4155
|
-
transactionId: msg.transactionId
|
|
4156
|
-
});
|
|
4157
|
-
|
|
4158
|
-
// Acknowledge message as completed
|
|
4159
|
-
await client.ack(msg.transactionId, 'completed');
|
|
4160
|
-
}
|
|
4161
|
-
|
|
4162
|
-
log(`Consumer ${consumerId} processed batch of ${result.messages.length} messages (total: ${consumerMessages.length})`);
|
|
4163
|
-
|
|
4164
|
-
} catch (error) {
|
|
4165
|
-
log(`Consumer ${consumerId} error: ${error.message}`, 'warning');
|
|
4166
|
-
await sleep(500);
|
|
4167
|
-
}
|
|
4168
|
-
}
|
|
4169
|
-
|
|
4170
|
-
if (consumerMessages.length > 0) {
|
|
4171
|
-
log(`Consumer ${consumerId} completed with ${consumerMessages.length} total messages`);
|
|
4172
|
-
}
|
|
4173
|
-
|
|
4174
|
-
return consumerMessages;
|
|
4175
|
-
})();
|
|
4176
|
-
|
|
4177
|
-
consumerPromises.push(consumerPromise);
|
|
4178
|
-
|
|
4179
|
-
// Stagger consumer starts slightly to simulate realistic scenario
|
|
4180
|
-
await sleep(50);
|
|
4181
|
-
}
|
|
4182
|
-
|
|
4183
|
-
// Wait for all consumers to complete
|
|
4184
|
-
log('Waiting for all consumers to complete processing...');
|
|
4185
|
-
const consumerResults = await Promise.all(consumerPromises);
|
|
4186
|
-
|
|
4187
|
-
// Combine all consumed messages
|
|
4188
|
-
for (const messages of consumerResults) {
|
|
4189
|
-
consumedMessages.push(...messages);
|
|
4190
|
-
}
|
|
4191
|
-
|
|
4192
|
-
// Sort by sequence number to check ordering
|
|
4193
|
-
consumedMessages.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
|
|
4194
|
-
|
|
4195
|
-
log(`Total messages consumed: ${consumedMessages.length}`);
|
|
4196
|
-
|
|
4197
|
-
// Verify all messages were consumed exactly once
|
|
4198
|
-
if (consumedMessages.length !== numMessages) {
|
|
4199
|
-
throw new Error(`Expected ${numMessages} messages to be consumed, but got ${consumedMessages.length}`);
|
|
4200
|
-
}
|
|
4201
|
-
|
|
4202
|
-
// Verify no duplicates
|
|
4203
|
-
const sequenceNumbers = new Set(consumedMessages.map(m => m.sequenceNumber));
|
|
4204
|
-
if (sequenceNumbers.size !== numMessages) {
|
|
4205
|
-
throw new Error(`Found duplicate messages: expected ${numMessages} unique sequences, got ${sequenceNumbers.size}`);
|
|
4206
|
-
}
|
|
4207
|
-
|
|
4208
|
-
// Verify sequential ordering (all messages from 0 to numMessages-1 are present)
|
|
4209
|
-
for (let i = 0; i < numMessages; i++) {
|
|
4210
|
-
if (consumedMessages[i].sequenceNumber !== i) {
|
|
4211
|
-
throw new Error(`Message ordering violation at position ${i}: expected sequence ${i}, got ${consumedMessages[i].sequenceNumber}`);
|
|
4212
|
-
}
|
|
4213
|
-
}
|
|
4214
|
-
|
|
4215
|
-
// Analyze consumer distribution
|
|
4216
|
-
const consumerStats = {};
|
|
4217
|
-
for (const msg of consumedMessages) {
|
|
4218
|
-
consumerStats[msg.consumerId] = (consumerStats[msg.consumerId] || 0) + 1;
|
|
4219
|
-
}
|
|
4220
|
-
|
|
4221
|
-
log('Consumer distribution:', 'info');
|
|
4222
|
-
for (const [consumerId, count] of Object.entries(consumerStats)) {
|
|
4223
|
-
log(` Consumer ${consumerId}: ${count} messages (${(count / numMessages * 100).toFixed(1)}%)`, 'info');
|
|
4224
|
-
}
|
|
4225
|
-
|
|
4226
|
-
// Check that messages consumed by each consumer maintain relative ordering
|
|
4227
|
-
for (let consumerId = 0; consumerId < numConsumers; consumerId++) {
|
|
4228
|
-
const consumerMsgs = consumedMessages
|
|
4229
|
-
.filter(m => m.consumerId === consumerId)
|
|
4230
|
-
.map(m => m.sequenceNumber);
|
|
4231
|
-
|
|
4232
|
-
// Verify that sequence numbers for this consumer are in ascending order
|
|
4233
|
-
for (let i = 1; i < consumerMsgs.length; i++) {
|
|
4234
|
-
if (consumerMsgs[i] <= consumerMsgs[i - 1]) {
|
|
4235
|
-
throw new Error(`Consumer ${consumerId} violated ordering: sequence ${consumerMsgs[i]} came after ${consumerMsgs[i - 1]}`);
|
|
4236
|
-
}
|
|
4237
|
-
}
|
|
4238
|
-
}
|
|
4239
|
-
|
|
4240
|
-
// Final verification: check database for any remaining unprocessed messages
|
|
4241
|
-
const remainingQuery = `
|
|
4242
|
-
SELECT COUNT(*) as count FROM queen.messages m
|
|
4243
|
-
JOIN queen.partitions p ON m.partition_id = p.id
|
|
4244
|
-
JOIN queen.queues q ON p.queue_id = q.id
|
|
4245
|
-
LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group = '__QUEUE_MODE__'
|
|
4246
|
-
WHERE q.name = $1 AND p.name = $2
|
|
4247
|
-
AND (ms.status IS NULL OR ms.status IN ('pending', 'processing', 'failed'))
|
|
4248
|
-
`;
|
|
4249
|
-
const remainingResult = await dbPool.query(remainingQuery, [queue, partition]);
|
|
4250
|
-
const remainingMessages = parseInt(remainingResult.rows[0].count);
|
|
4251
|
-
|
|
4252
|
-
if (remainingMessages > 0) {
|
|
4253
|
-
log(`Warning: ${remainingMessages} unprocessed messages still in queue after test`, 'warning');
|
|
4254
|
-
}
|
|
4255
|
-
|
|
4256
|
-
passTest(`${numConsumers} consumers successfully processed ${numMessages} messages from single partition with correct ordering`);
|
|
4257
|
-
|
|
4258
|
-
} catch (error) {
|
|
4259
|
-
failTest(error);
|
|
4260
|
-
}
|
|
4261
|
-
}
|
|
4262
|
-
|
|
4263
|
-
// ============================================
|
|
4264
|
-
// MAIN TEST RUNNER
|
|
4265
|
-
// ============================================
|
|
4266
|
-
|
|
4267
|
-
async function runTestsInParallel(tests, category, maxConcurrent = 3) {
|
|
4268
|
-
console.log(`\n${category}`);
|
|
4269
|
-
console.log('-' .repeat(40));
|
|
4270
|
-
|
|
4271
|
-
const results = [];
|
|
4272
|
-
const executing = [];
|
|
4273
|
-
|
|
4274
|
-
for (const test of tests) {
|
|
4275
|
-
const promise = test().catch(error => {
|
|
4276
|
-
log(`Unexpected error in test: ${error.message}`, 'error');
|
|
4277
|
-
return { error };
|
|
4278
|
-
});
|
|
4279
|
-
|
|
4280
|
-
results.push(promise);
|
|
4281
|
-
|
|
4282
|
-
if (tests.length >= maxConcurrent) {
|
|
4283
|
-
executing.push(promise);
|
|
4284
|
-
|
|
4285
|
-
if (executing.length >= maxConcurrent) {
|
|
4286
|
-
await Promise.race(executing);
|
|
4287
|
-
executing.splice(executing.findIndex(p => p === promise), 1);
|
|
4288
|
-
}
|
|
4289
|
-
}
|
|
4290
|
-
|
|
4291
|
-
// Small delay between starting tests to avoid overwhelming the system
|
|
4292
|
-
await sleep(50);
|
|
4293
|
-
}
|
|
4294
|
-
|
|
4295
|
-
await Promise.all(results);
|
|
4296
|
-
}
|
|
4297
|
-
|
|
4298
|
-
async function runAllTests() {
|
|
4299
|
-
console.log('🚀 Starting Complete Queen Message Queue Test Suite');
|
|
4300
|
-
console.log(' Including Core, Enterprise, Bus Mode, Edge Cases, and Advanced Scenarios\n');
|
|
4301
|
-
const parallelMode = process.env.PARALLEL_TESTS === 'true';
|
|
4302
|
-
console.log(` Running in ${parallelMode ? 'PARALLEL' : 'SEQUENTIAL'} mode`);
|
|
4303
|
-
console.log('=' .repeat(80));
|
|
4304
|
-
|
|
4305
|
-
try {
|
|
4306
|
-
// Initialize client and database connection
|
|
4307
|
-
client = createQueenClient({ baseUrls: TEST_CONFIG.baseUrls });
|
|
4308
|
-
dbPool = new pg.Pool(TEST_CONFIG.dbConfig);
|
|
4309
|
-
|
|
4310
|
-
// Test database connection
|
|
4311
|
-
await dbPool.query('SELECT 1');
|
|
4312
|
-
log('Database connection established');
|
|
4313
|
-
|
|
4314
|
-
// Check for encryption key
|
|
4315
|
-
if (process.env.QUEEN_ENCRYPTION_KEY) {
|
|
4316
|
-
log('Encryption key detected - encryption tests will run', 'info');
|
|
4317
|
-
} else {
|
|
4318
|
-
log('No encryption key found - encryption tests will be skipped', 'warning');
|
|
4319
|
-
log('Set QUEEN_ENCRYPTION_KEY env variable to test encryption', 'info');
|
|
4320
|
-
}
|
|
4321
|
-
|
|
4322
|
-
// Clean up any existing test data
|
|
4323
|
-
await cleanupTestData();
|
|
4324
|
-
|
|
4325
|
-
// Core feature tests
|
|
4326
|
-
const coreTests = [
|
|
4327
|
-
testQueueCreationPolicy,
|
|
4328
|
-
testSingleMessagePush,
|
|
4329
|
-
testBatchMessagePush,
|
|
4330
|
-
testQueueConfiguration,
|
|
4331
|
-
testPopAndAcknowledgment,
|
|
4332
|
-
testDelayedProcessing,
|
|
4333
|
-
testPartitionPriorityOrdering,
|
|
4334
|
-
testPartitionLocking,
|
|
4335
|
-
testBusPartitionLocking,
|
|
4336
|
-
testSpecificPartitionLocking,
|
|
4337
|
-
testNamespaceTaskFiltering,
|
|
4338
|
-
testNamespaceTaskBusMode
|
|
4339
|
-
];
|
|
4340
|
-
|
|
4341
|
-
// Enterprise feature tests
|
|
4342
|
-
const enterpriseTests = [
|
|
4343
|
-
testMessageEncryption,
|
|
4344
|
-
testRetentionPendingMessages,
|
|
4345
|
-
testRetentionCompletedMessages,
|
|
4346
|
-
testPartitionRetention,
|
|
4347
|
-
testMessageEviction,
|
|
4348
|
-
testCombinedEnterpriseFeatures,
|
|
4349
|
-
testConsumerEncryption,
|
|
4350
|
-
testEnterpriseErrorHandling
|
|
4351
|
-
];
|
|
4352
|
-
|
|
4353
|
-
// Bus mode tests
|
|
4354
|
-
const busTests = [
|
|
4355
|
-
testBusConsumerGroups,
|
|
4356
|
-
testMixedMode,
|
|
4357
|
-
testConsumerGroupSubscriptionModes,
|
|
4358
|
-
testConsumerGroupIsolation
|
|
4359
|
-
];
|
|
4360
|
-
|
|
4361
|
-
// Edge case tests
|
|
4362
|
-
const edgeTests = [
|
|
4363
|
-
testEmptyAndNullPayloads,
|
|
4364
|
-
testVeryLargePayloads,
|
|
4365
|
-
testConcurrentPushOperations,
|
|
4366
|
-
testConcurrentPopOperations,
|
|
4367
|
-
testManyConsumerGroups,
|
|
4368
|
-
testRetryLimitExhaustion,
|
|
4369
|
-
testLeaseExpiration,
|
|
4370
|
-
testSQLInjectionPrevention,
|
|
4371
|
-
testXSSPrevention
|
|
4372
|
-
];
|
|
4373
|
-
|
|
4374
|
-
// Advanced scenario tests - split into smaller groups for parallel execution
|
|
4375
|
-
const advancedTests1 = [
|
|
4376
|
-
testMultiStagePipeline,
|
|
4377
|
-
testFanOutFanIn,
|
|
4378
|
-
testComplexPriorityScenarios,
|
|
4379
|
-
testDynamicPriorityAdjustment
|
|
4380
|
-
];
|
|
4381
|
-
|
|
4382
|
-
const advancedTests2 = [
|
|
4383
|
-
testDeadLetterQueuePattern,
|
|
4384
|
-
testSagaPattern,
|
|
4385
|
-
testRateLimiting,
|
|
4386
|
-
testMessageDeduplication
|
|
4387
|
-
];
|
|
4388
|
-
|
|
4389
|
-
const advancedTests3 = [
|
|
4390
|
-
testTimeBatchProcessing,
|
|
4391
|
-
testEventSourcing,
|
|
4392
|
-
//testQueueMetrics,
|
|
4393
|
-
testCircuitBreaker,
|
|
4394
|
-
testMultipleConsumersSinglePartitionOrdering
|
|
4395
|
-
];
|
|
4396
|
-
|
|
4397
|
-
const allTests = [...coreTests, ...enterpriseTests, ...busTests, ...edgeTests,
|
|
4398
|
-
...advancedTests1, ...advancedTests2, ...advancedTests3];
|
|
4399
|
-
|
|
4400
|
-
log(`Running ${allTests.length} total tests...`);
|
|
4401
|
-
console.log('=' .repeat(80));
|
|
4402
|
-
|
|
4403
|
-
if (parallelMode) {
|
|
4404
|
-
// Run tests in parallel with controlled concurrency
|
|
4405
|
-
await runTestsInParallel(coreTests, '📦 CORE FEATURES', 2);
|
|
4406
|
-
await runTestsInParallel(enterpriseTests, '🏢 ENTERPRISE FEATURES', 2);
|
|
4407
|
-
await runTestsInParallel(busTests, '🚌 BUS MODE FEATURES', 2);
|
|
4408
|
-
await runTestsInParallel(edgeTests, '🔍 EDGE CASES', 3);
|
|
4409
|
-
await runTestsInParallel(advancedTests1, '🎯 ADVANCED SCENARIOS (Part 1)', 2);
|
|
4410
|
-
await runTestsInParallel(advancedTests2, '🎯 ADVANCED SCENARIOS (Part 2)', 2);
|
|
4411
|
-
await runTestsInParallel(advancedTests3, '🎯 ADVANCED SCENARIOS (Part 3)', 2);
|
|
4412
|
-
} else {
|
|
4413
|
-
// Original sequential execution
|
|
4414
|
-
console.log('\n📦 CORE FEATURES');
|
|
4415
|
-
console.log('-' .repeat(40));
|
|
4416
|
-
for (const test of coreTests) {
|
|
4417
|
-
try {
|
|
4418
|
-
await test();
|
|
4419
|
-
} catch (error) {
|
|
4420
|
-
log(`Unexpected error in test: ${error.message}`, 'error');
|
|
4421
|
-
}
|
|
4422
|
-
await sleep(100);
|
|
4423
|
-
}
|
|
4424
|
-
|
|
4425
|
-
console.log('\n🏢 ENTERPRISE FEATURES');
|
|
4426
|
-
console.log('-' .repeat(40));
|
|
4427
|
-
for (const test of enterpriseTests) {
|
|
4428
|
-
try {
|
|
4429
|
-
await test();
|
|
4430
|
-
} catch (error) {
|
|
4431
|
-
log(`Unexpected error in test: ${error.message}`, 'error');
|
|
4432
|
-
}
|
|
4433
|
-
await sleep(100);
|
|
4434
|
-
}
|
|
4435
|
-
|
|
4436
|
-
console.log('\n🚌 BUS MODE FEATURES');
|
|
4437
|
-
console.log('-' .repeat(40));
|
|
4438
|
-
for (const test of busTests) {
|
|
4439
|
-
try {
|
|
4440
|
-
await test();
|
|
4441
|
-
} catch (error) {
|
|
4442
|
-
log(`Unexpected error in test: ${error.message}`, 'error');
|
|
4443
|
-
}
|
|
4444
|
-
await sleep(100);
|
|
4445
|
-
}
|
|
4446
|
-
|
|
4447
|
-
console.log('\n🔍 EDGE CASES');
|
|
4448
|
-
console.log('-' .repeat(40));
|
|
4449
|
-
for (const test of edgeTests) {
|
|
4450
|
-
try {
|
|
4451
|
-
await test();
|
|
4452
|
-
} catch (error) {
|
|
4453
|
-
log(`Unexpected error in test: ${error.message}`, 'error');
|
|
4454
|
-
}
|
|
4455
|
-
await sleep(100);
|
|
4456
|
-
}
|
|
4457
|
-
|
|
4458
|
-
console.log('\n🎯 ADVANCED SCENARIOS');
|
|
4459
|
-
console.log('-' .repeat(40));
|
|
4460
|
-
for (const test of [...advancedTests1, ...advancedTests2, ...advancedTests3]) {
|
|
4461
|
-
try {
|
|
4462
|
-
await test();
|
|
4463
|
-
} catch (error) {
|
|
4464
|
-
log(`Unexpected error in test: ${error.message}`, 'error');
|
|
4465
|
-
}
|
|
4466
|
-
await sleep(100);
|
|
4467
|
-
}
|
|
4468
|
-
}
|
|
4469
|
-
|
|
4470
|
-
} catch (error) {
|
|
4471
|
-
log(`Test suite setup failed: ${error.message}`, 'error');
|
|
4472
|
-
} finally {
|
|
4473
|
-
// Cleanup
|
|
4474
|
-
if (dbPool) {
|
|
4475
|
-
await cleanupTestData();
|
|
4476
|
-
await dbPool.end();
|
|
4477
|
-
}
|
|
4478
|
-
}
|
|
4479
|
-
|
|
4480
|
-
// Print results
|
|
4481
|
-
console.log('\n' + '=' .repeat(80));
|
|
4482
|
-
console.log('📊 TEST RESULTS SUMMARY');
|
|
4483
|
-
console.log('=' .repeat(80));
|
|
4484
|
-
|
|
4485
|
-
const passed = testResults.filter(r => r.status === 'PASS').length;
|
|
4486
|
-
const failed = testResults.filter(r => r.status === 'FAIL').length;
|
|
4487
|
-
const total = testResults.length;
|
|
4488
|
-
|
|
4489
|
-
console.log(`\n📈 Overall Results: ${passed}/${total} tests passed`);
|
|
4490
|
-
|
|
4491
|
-
if (failed > 0) {
|
|
4492
|
-
console.log('\n❌ Failed Tests:');
|
|
4493
|
-
testResults
|
|
4494
|
-
.filter(r => r.status === 'FAIL')
|
|
4495
|
-
.forEach(r => {
|
|
4496
|
-
console.log(` • ${r.test}: ${r.error}`);
|
|
4497
|
-
});
|
|
4498
|
-
}
|
|
4499
|
-
|
|
4500
|
-
if (passed > 0) {
|
|
4501
|
-
console.log('\n✅ Passed Tests:');
|
|
4502
|
-
testResults
|
|
4503
|
-
.filter(r => r.status === 'PASS')
|
|
4504
|
-
.forEach(r => {
|
|
4505
|
-
console.log(` • ${r.test}${r.message ? ': ' + r.message : ''}`);
|
|
4506
|
-
});
|
|
4507
|
-
}
|
|
4508
|
-
|
|
4509
|
-
console.log('\n' + '=' .repeat(80));
|
|
4510
|
-
|
|
4511
|
-
if (failed === 0) {
|
|
4512
|
-
console.log('🎉 ALL TESTS PASSED! Queen Message Queue System is working correctly.');
|
|
4513
|
-
console.log(' All features are operational.');
|
|
4514
|
-
} else {
|
|
4515
|
-
console.log(`⚠️ ${failed} test(s) failed. Please review the failures above.`);
|
|
4516
|
-
process.exit(1);
|
|
4517
|
-
}
|
|
4518
|
-
}
|
|
4519
|
-
|
|
4520
|
-
// Run the test suite
|
|
4521
|
-
runAllTests().catch(error => {
|
|
4522
|
-
console.error('💥 Test suite crashed:', error);
|
|
4523
|
-
process.exit(1);
|
|
4524
|
-
});
|