queen-mq 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +212 -144
- package/client-js/client-v2/Queen.js +28 -0
- package/client-js/client-v2/README.md +6 -2
- package/client-js/client-v2/builders/QueueBuilder.js +7 -1
- package/client-js/client-v2/builders/TransactionBuilder.js +20 -4
- package/client-js/client-v2/stream/StreamBuilder.js +72 -0
- package/client-js/client-v2/stream/StreamConsumer.js +156 -0
- package/client-js/client-v2/stream/Window.js +140 -0
- package/client-js/test-v2/MAINTENANCE_TEST.md +148 -0
- package/client-js/test-v2/consume.js +1 -0
- package/client-js/test-v2/maintenance.js +261 -0
- package/client-js/test-v2/retention.js +69 -0
- package/client-js/test-v2/run.js +7 -1
- package/package.json +2 -2
- package/client-js/client/client.js +0 -1536
- package/client-js/client/index.js +0 -4
- package/client-js/client/utils/http.js +0 -173
- package/client-js/client/utils/loadBalancer.js +0 -152
- package/client-js/client/utils/retry.js +0 -41
- package/client-js/services/encryptionService.js +0 -82
- package/client-js/services/evictionService.js +0 -160
- package/client-js/services/retentionService.js +0 -162
- package/client-js/services/startupSync.js +0 -35
- package/client-js/test/README.md +0 -224
- package/client-js/test/advanced-client-tests.js +0 -761
- package/client-js/test/advanced-pattern-tests.js +0 -1137
- package/client-js/test/bus-mode-tests.js +0 -361
- package/client-js/test/core-tests.js +0 -457
- package/client-js/test/edge-case-tests.js +0 -562
- package/client-js/test/enterprise-tests.js +0 -637
- package/client-js/test/human.js +0 -162
- package/client-js/test/partition-locking-tests.js +0 -545
- package/client-js/test/partition-transaction-tests.js +0 -482
- package/client-js/test/qos0-tests.js +0 -334
- package/client-js/test/test-new.js +0 -370
- package/client-js/test/utils.js +0 -169
- package/client-js/test/window-buffer-test.js +0 -114
- package/client-js/utils/logger.js +0 -44
- package/client-js/utils/uuid.js +0 -5
package/client-js/test/human.js
DELETED
|
@@ -1,162 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Test written by an human
|
|
3
|
-
*/
|
|
4
|
-
|
|
5
|
-
import { startTest, passTest, failTest, sleep, log } from './utils.js';
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* Test: Pipeline API - Basic Processing
|
|
9
|
-
*/
|
|
10
|
-
export async function testCreateQueue(client) {
|
|
11
|
-
startTest('Create Queue', 'human');
|
|
12
|
-
|
|
13
|
-
try {
|
|
14
|
-
const queue = 'test-create-queue-hm';
|
|
15
|
-
await client.queue(queue, { leaseTime: 30 });
|
|
16
|
-
passTest('Queue created successfully');
|
|
17
|
-
} catch (error) {
|
|
18
|
-
failTest('Create Queue', error );
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export async function workerQueueTest(client) {
|
|
23
|
-
startTest('Worker Queue', 'human');
|
|
24
|
-
|
|
25
|
-
try {
|
|
26
|
-
const queue = 'test-worker-queue-hm';
|
|
27
|
-
const processedQueue = queue + '-processed';
|
|
28
|
-
await client.queueDelete(queue);
|
|
29
|
-
await client.queueDelete(processedQueue);
|
|
30
|
-
await client.queue(queue, { leaseTime: 30 });
|
|
31
|
-
await client.queue(processedQueue, { leaseTime: 30 });
|
|
32
|
-
const pushMessages = async (count) => {
|
|
33
|
-
let messages = [];
|
|
34
|
-
for (let i = 0; i < count; i++) {
|
|
35
|
-
messages.push({ id: i, data: `Message ${i}` });
|
|
36
|
-
}
|
|
37
|
-
await client.push(queue, messages);
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
// Test 1
|
|
42
|
-
await pushMessages(10);
|
|
43
|
-
|
|
44
|
-
let counter1 = 0;
|
|
45
|
-
await client
|
|
46
|
-
.pipeline(queue)
|
|
47
|
-
.take(10)
|
|
48
|
-
.process(async (message) => {
|
|
49
|
-
counter1++;
|
|
50
|
-
return { id: message.data.id, data: message.data.data };
|
|
51
|
-
})
|
|
52
|
-
.execute();
|
|
53
|
-
|
|
54
|
-
if (counter1 !== 10) {
|
|
55
|
-
throw new Error('Expected 10 messages to be processed, got ' + counter1);
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
// Test 2
|
|
59
|
-
await pushMessages(100);
|
|
60
|
-
|
|
61
|
-
let counter2 = 0;
|
|
62
|
-
await client
|
|
63
|
-
.pipeline(queue)
|
|
64
|
-
.take(10)
|
|
65
|
-
.processBatch(async (messages) => {
|
|
66
|
-
counter2 += messages.length;
|
|
67
|
-
return messages;
|
|
68
|
-
})
|
|
69
|
-
.repeat({ maxIterations: 1, continuous: false })
|
|
70
|
-
.execute();
|
|
71
|
-
|
|
72
|
-
if (counter2 !== 10) {
|
|
73
|
-
throw new Error('Expected 10 messages to be processed, got ' + counter2);
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
// Test 3
|
|
77
|
-
let counter3 = 0;
|
|
78
|
-
await client
|
|
79
|
-
.pipeline(queue)
|
|
80
|
-
.take(10)
|
|
81
|
-
.processBatch(async (messages) => {
|
|
82
|
-
counter3 += messages.length;
|
|
83
|
-
return messages;
|
|
84
|
-
})
|
|
85
|
-
.repeat({ maxIterations: 9, continuous: false })
|
|
86
|
-
.execute();
|
|
87
|
-
|
|
88
|
-
if (counter3 !== 90) {
|
|
89
|
-
throw new Error('Expected 90 messages to be processed, got ' + counter3);
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
await pushMessages(100);
|
|
93
|
-
|
|
94
|
-
await client
|
|
95
|
-
.pipeline(queue)
|
|
96
|
-
.take(10)
|
|
97
|
-
.processBatch(async (messages) => {
|
|
98
|
-
return messages;
|
|
99
|
-
})
|
|
100
|
-
.atomically((tx, originalMessages, processedMessages) => {
|
|
101
|
-
tx.ack(originalMessages);
|
|
102
|
-
})
|
|
103
|
-
.repeat({ maxIterations: 10, continuous: true })
|
|
104
|
-
.execute();
|
|
105
|
-
|
|
106
|
-
const messagesPopped = await client.takeSingleBatch(queue);
|
|
107
|
-
let counter4 = messagesPopped.length;
|
|
108
|
-
if (counter4 !== 0) {
|
|
109
|
-
throw new Error('Expected 0 message to be processed, got ' + counter4);
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
// Test transaction that works correctly
|
|
113
|
-
await pushMessages(100);
|
|
114
|
-
|
|
115
|
-
await client
|
|
116
|
-
.pipeline(queue)
|
|
117
|
-
.take(10)
|
|
118
|
-
.processBatch(async (messages) => {
|
|
119
|
-
return messages;
|
|
120
|
-
})
|
|
121
|
-
.atomically((tx, originalMessages, processedMessages) => {
|
|
122
|
-
tx.ack(originalMessages);
|
|
123
|
-
tx.push(processedQueue, processedMessages);
|
|
124
|
-
})
|
|
125
|
-
.repeat({ maxIterations: 10, continuous: true })
|
|
126
|
-
.execute();
|
|
127
|
-
|
|
128
|
-
const messagesPopped2 = [];
|
|
129
|
-
for await (const messages of client.takeBatch(processedQueue, { batch: 100 })) {
|
|
130
|
-
messagesPopped2.push(...messages);
|
|
131
|
-
}
|
|
132
|
-
if (messagesPopped2.length !== 100) {
|
|
133
|
-
throw new Error('Expected 100 message to be processed, got ' + counter5);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
// Test transaction that fails
|
|
137
|
-
|
|
138
|
-
await pushMessages(100);
|
|
139
|
-
|
|
140
|
-
try {
|
|
141
|
-
await client
|
|
142
|
-
.pipeline(queue)
|
|
143
|
-
.take(10)
|
|
144
|
-
.processBatch(async (messages) => {
|
|
145
|
-
return messages;
|
|
146
|
-
})
|
|
147
|
-
.atomically((tx, originalMessages, processedMessages) => {
|
|
148
|
-
originalMessages.map(x => x.leaseId = 'invalid-lease-id-12345');
|
|
149
|
-
tx.ack(originalMessages);
|
|
150
|
-
tx.push(processedQueue, processedMessages);
|
|
151
|
-
})
|
|
152
|
-
.repeat({ maxIterations: 10, continuous: true })
|
|
153
|
-
.execute();
|
|
154
|
-
failTest('Expected transaction to fail, got ' + error);
|
|
155
|
-
} catch (error) {}
|
|
156
|
-
|
|
157
|
-
passTest('Worker Queue test passed');
|
|
158
|
-
} catch (error) {
|
|
159
|
-
console.error(error);
|
|
160
|
-
failTest('Worker Queue test failed', error);
|
|
161
|
-
}
|
|
162
|
-
}
|
|
@@ -1,545 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Partition Locking Tests for Queen Message Queue
|
|
3
|
-
* Tests: Partition locking in queue mode, bus mode, specific partition requests, namespace/task filtering
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
import { startTest, passTest, failTest, sleep, dbPool, log } from './utils.js';
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Test: Partition Locking in Queue Mode
|
|
10
|
-
* Tests that partitions are properly locked when consumed
|
|
11
|
-
*/
|
|
12
|
-
export async function testPartitionLocking(client) {
|
|
13
|
-
startTest('Partition Locking in Queue Mode');
|
|
14
|
-
|
|
15
|
-
const QUEUE_NAME = 'test-partition-lock';
|
|
16
|
-
const PARTITION1 = 'partition-A';
|
|
17
|
-
const PARTITION2 = 'partition-B';
|
|
18
|
-
|
|
19
|
-
try {
|
|
20
|
-
// Configure queue
|
|
21
|
-
await client.queue(QUEUE_NAME, {});
|
|
22
|
-
|
|
23
|
-
// Push 3 messages to each partition
|
|
24
|
-
await client.push(`${QUEUE_NAME}/${PARTITION1}`, [
|
|
25
|
-
{ data: `Message 1 for ${PARTITION1}` },
|
|
26
|
-
{ data: `Message 2 for ${PARTITION1}` },
|
|
27
|
-
{ data: `Message 3 for ${PARTITION1}` }
|
|
28
|
-
]);
|
|
29
|
-
|
|
30
|
-
await client.push(`${QUEUE_NAME}/${PARTITION2}`, [
|
|
31
|
-
{ data: `Message 1 for ${PARTITION2}` },
|
|
32
|
-
{ data: `Message 2 for ${PARTITION2}` },
|
|
33
|
-
{ data: `Message 3 for ${PARTITION2}` }
|
|
34
|
-
]);
|
|
35
|
-
|
|
36
|
-
log(`Pushed 6 messages (3 to each partition)`);
|
|
37
|
-
|
|
38
|
-
// Consumer 1: Take messages (will lock a partition)
|
|
39
|
-
log('Consumer 1: Taking messages...');
|
|
40
|
-
const consumer1Messages = [];
|
|
41
|
-
for await (const msg of client.take(QUEUE_NAME, { limit: 2, batch: 2 })) {
|
|
42
|
-
consumer1Messages.push(msg);
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
if (consumer1Messages.length === 0) {
|
|
46
|
-
throw new Error('Consumer 1 got no messages');
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
const partition1 = consumer1Messages[0].partition;
|
|
50
|
-
log(`Consumer 1: Got ${consumer1Messages.length} messages from partition: ${partition1}`);
|
|
51
|
-
|
|
52
|
-
// Consumer 2: Take messages (should get different partition)
|
|
53
|
-
log('Consumer 2: Taking messages...');
|
|
54
|
-
const consumer2Messages = [];
|
|
55
|
-
for await (const msg of client.take(QUEUE_NAME, { limit: 2, batch: 2 })) {
|
|
56
|
-
consumer2Messages.push(msg);
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
if (consumer2Messages.length === 0) {
|
|
60
|
-
throw new Error('Consumer 2 got no messages');
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
const partition2 = consumer2Messages[0].partition;
|
|
64
|
-
log(`Consumer 2: Got ${consumer2Messages.length} messages from partition: ${partition2}`);
|
|
65
|
-
|
|
66
|
-
// Verify they got different partitions
|
|
67
|
-
if (partition1 === partition2) {
|
|
68
|
-
throw new Error('Both consumers got the same partition - partition locking not working!');
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
log(`SUCCESS: Consumers got different partitions (${partition1} vs ${partition2})`);
|
|
72
|
-
|
|
73
|
-
// Consumer 3: Try to take again (should get no messages as both partitions are locked)
|
|
74
|
-
log('Consumer 3: Trying to take (both partitions should be locked)...');
|
|
75
|
-
let consumer3GotMessages = false;
|
|
76
|
-
for await (const msg of client.take(QUEUE_NAME, { limit: 2, batch: 2 })) {
|
|
77
|
-
consumer3GotMessages = true;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
if (consumer3GotMessages) {
|
|
81
|
-
throw new Error('Consumer 3 got messages (should have been 0 - both partitions locked)');
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
log('Consumer 3: Got no messages (correct - both partitions are locked)');
|
|
85
|
-
|
|
86
|
-
// ACK messages from Consumer 1 to release its partition
|
|
87
|
-
log('Consumer 1: ACKing messages to release partition...');
|
|
88
|
-
for (const msg of consumer1Messages) {
|
|
89
|
-
await client.ack(msg);
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
// Consumer 4: Should now be able to get remaining message from Consumer 1's partition
|
|
93
|
-
log('Consumer 4: Taking after Consumer 1 released...');
|
|
94
|
-
const consumer4Messages = [];
|
|
95
|
-
for await (const msg of client.take(QUEUE_NAME, { limit: 2, batch: 2 })) {
|
|
96
|
-
consumer4Messages.push(msg);
|
|
97
|
-
await client.ack(msg);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
if (consumer4Messages.length > 0) {
|
|
101
|
-
const partition4 = consumer4Messages[0].partition;
|
|
102
|
-
log(`Consumer 4: Got ${consumer4Messages.length} messages from partition: ${partition4}`);
|
|
103
|
-
|
|
104
|
-
// Should be from the same partition as Consumer 1 had
|
|
105
|
-
if (partition4 !== partition1) {
|
|
106
|
-
throw new Error('Consumer 4 did not get messages from released partition');
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
// Clean up: ACK all remaining messages
|
|
111
|
-
for (const msg of consumer2Messages) {
|
|
112
|
-
await client.ack(msg);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
passTest('Partition locking works correctly');
|
|
116
|
-
} catch (error) {
|
|
117
|
-
failTest(error);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* Test: Partition Locking in Bus Mode
|
|
123
|
-
* Tests that partitions are properly locked per consumer group
|
|
124
|
-
*/
|
|
125
|
-
export async function testBusPartitionLocking(client) {
|
|
126
|
-
startTest('Partition Locking in Bus Mode');
|
|
127
|
-
|
|
128
|
-
const QUEUE_NAME = 'test-bus-partition-lock';
|
|
129
|
-
const PARTITION1 = 'partition-X';
|
|
130
|
-
const PARTITION2 = 'partition-Y';
|
|
131
|
-
const CONSUMER_GROUP1 = 'group-alpha';
|
|
132
|
-
const CONSUMER_GROUP2 = 'group-beta';
|
|
133
|
-
|
|
134
|
-
try {
|
|
135
|
-
// Configure queue
|
|
136
|
-
await client.queue(QUEUE_NAME, {});
|
|
137
|
-
|
|
138
|
-
// Push 3 messages to each partition
|
|
139
|
-
await client.push(`${QUEUE_NAME}/${PARTITION1}`, [
|
|
140
|
-
{ data: `Message 1 for ${PARTITION1}` },
|
|
141
|
-
{ data: `Message 2 for ${PARTITION1}` },
|
|
142
|
-
{ data: `Message 3 for ${PARTITION1}` }
|
|
143
|
-
]);
|
|
144
|
-
|
|
145
|
-
await client.push(`${QUEUE_NAME}/${PARTITION2}`, [
|
|
146
|
-
{ data: `Message 1 for ${PARTITION2}` },
|
|
147
|
-
{ data: `Message 2 for ${PARTITION2}` },
|
|
148
|
-
{ data: `Message 3 for ${PARTITION2}` }
|
|
149
|
-
]);
|
|
150
|
-
|
|
151
|
-
log(`Pushed 6 messages (3 to each partition)`);
|
|
152
|
-
|
|
153
|
-
// Group Alpha, Consumer 1: Take messages
|
|
154
|
-
log('Group Alpha, Consumer 1: Taking messages...');
|
|
155
|
-
const group1Consumer1Messages = [];
|
|
156
|
-
for await (const msg of client.take(`${QUEUE_NAME}@${CONSUMER_GROUP1}`, { limit: 2, batch: 2 })) {
|
|
157
|
-
group1Consumer1Messages.push(msg);
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
if (group1Consumer1Messages.length === 0) {
|
|
161
|
-
throw new Error('Group Alpha Consumer 1 got no messages');
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
const partition1 = group1Consumer1Messages[0].partition;
|
|
165
|
-
log(`Group Alpha Consumer 1: Got ${group1Consumer1Messages.length} messages from partition: ${partition1}`);
|
|
166
|
-
|
|
167
|
-
// Group Alpha, Consumer 2: Take messages (should get different partition if locking works)
|
|
168
|
-
log('Group Alpha, Consumer 2: Taking messages...');
|
|
169
|
-
const group1Consumer2Messages = [];
|
|
170
|
-
for await (const msg of client.take(`${QUEUE_NAME}@${CONSUMER_GROUP1}`, { limit: 2, batch: 2 })) {
|
|
171
|
-
group1Consumer2Messages.push(msg);
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
if (group1Consumer2Messages.length === 0) {
|
|
175
|
-
log('Group Alpha Consumer 2 got no messages (partition locking might be working)');
|
|
176
|
-
} else {
|
|
177
|
-
const partition2 = group1Consumer2Messages[0].partition;
|
|
178
|
-
log(`Group Alpha Consumer 2: Got ${group1Consumer2Messages.length} messages from partition: ${partition2}`);
|
|
179
|
-
|
|
180
|
-
if (partition1 === partition2) {
|
|
181
|
-
log('WARNING: Both consumers in same group got same partition!');
|
|
182
|
-
} else {
|
|
183
|
-
log('Consumers in same group got different partitions');
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
// Group Beta, Consumer 1: Should be able to get messages from any partition (different consumer group)
|
|
188
|
-
log('Group Beta, Consumer 1: Taking messages (different consumer group)...');
|
|
189
|
-
const group2Consumer1Messages = [];
|
|
190
|
-
for await (const msg of client.take(`${QUEUE_NAME}@${CONSUMER_GROUP2}`, { limit: 2, batch: 2 })) {
|
|
191
|
-
group2Consumer1Messages.push(msg);
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
if (group2Consumer1Messages.length > 0) {
|
|
195
|
-
const partition3 = group2Consumer1Messages[0].partition;
|
|
196
|
-
log(`Group Beta Consumer 1: Got ${group2Consumer1Messages.length} messages from partition: ${partition3}`);
|
|
197
|
-
log('Different consumer groups can access same partitions independently (correct)');
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
// Clean up: ACK all messages
|
|
201
|
-
for (const msg of group1Consumer1Messages) {
|
|
202
|
-
await client.ack(msg, true, { group: CONSUMER_GROUP1 });
|
|
203
|
-
}
|
|
204
|
-
for (const msg of group1Consumer2Messages) {
|
|
205
|
-
await client.ack(msg, true, { group: CONSUMER_GROUP1 });
|
|
206
|
-
}
|
|
207
|
-
for (const msg of group2Consumer1Messages) {
|
|
208
|
-
await client.ack(msg, true, { group: CONSUMER_GROUP2 });
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
passTest('Bus mode partition locking test completed');
|
|
212
|
-
} catch (error) {
|
|
213
|
-
failTest(error);
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
/**
|
|
218
|
-
* Test: Specific Partition Request with Locking
|
|
219
|
-
* Tests that requesting a specific partition respects locking
|
|
220
|
-
*/
|
|
221
|
-
export async function testSpecificPartitionLocking(client) {
|
|
222
|
-
startTest('Specific Partition Request with Locking');
|
|
223
|
-
|
|
224
|
-
const QUEUE_NAME = 'test-specific-partition';
|
|
225
|
-
const PARTITION_NAME = 'specific-partition';
|
|
226
|
-
const CONSUMER_GROUP = 'test-group';
|
|
227
|
-
|
|
228
|
-
try {
|
|
229
|
-
// Configure queue
|
|
230
|
-
await client.queue(QUEUE_NAME, {});
|
|
231
|
-
|
|
232
|
-
// Part 1: Test Queue Mode
|
|
233
|
-
log('=== Testing Queue Mode with Specific Partition ===');
|
|
234
|
-
|
|
235
|
-
// Push test messages
|
|
236
|
-
await client.push(`${QUEUE_NAME}/${PARTITION_NAME}`, [
|
|
237
|
-
{ data: 'Message 1' },
|
|
238
|
-
{ data: 'Message 2' },
|
|
239
|
-
{ data: 'Message 3' },
|
|
240
|
-
{ data: 'Message 4' },
|
|
241
|
-
{ data: 'Message 5' }
|
|
242
|
-
]);
|
|
243
|
-
|
|
244
|
-
log(`Pushed 5 messages to ${PARTITION_NAME}`);
|
|
245
|
-
|
|
246
|
-
// Consumer 1: Request specific partition
|
|
247
|
-
log('Consumer 1: Requesting specific partition...');
|
|
248
|
-
const consumer1Messages = [];
|
|
249
|
-
for await (const msg of client.take(`${QUEUE_NAME}/${PARTITION_NAME}`, { limit: 2, batch: 2 })) {
|
|
250
|
-
consumer1Messages.push(msg);
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
if (consumer1Messages.length === 0) {
|
|
254
|
-
throw new Error('Consumer 1 got no messages');
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
log(`Consumer 1: Got ${consumer1Messages.length} messages from partition: ${consumer1Messages[0].partition}`);
|
|
258
|
-
|
|
259
|
-
// Consumer 2: Try to request same specific partition (should get nothing due to lock)
|
|
260
|
-
log('Consumer 2: Requesting same specific partition...');
|
|
261
|
-
let consumer2GotMessages = false;
|
|
262
|
-
for await (const msg of client.take(`${QUEUE_NAME}/${PARTITION_NAME}`, { limit: 2, batch: 2 })) {
|
|
263
|
-
consumer2GotMessages = true;
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
if (consumer2GotMessages) {
|
|
267
|
-
throw new Error('Consumer 2 got messages (should have been 0 due to lock)');
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
log('Consumer 2: Got no messages (correct - partition is locked by Consumer 1)');
|
|
271
|
-
|
|
272
|
-
// ACK messages from Consumer 1
|
|
273
|
-
for (const msg of consumer1Messages) {
|
|
274
|
-
await client.ack(msg);
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
// Consumer 3: Should now get messages from the specific partition
|
|
278
|
-
log('Consumer 3: Requesting specific partition after release...');
|
|
279
|
-
const consumer3Messages = [];
|
|
280
|
-
for await (const msg of client.take(`${QUEUE_NAME}/${PARTITION_NAME}`, { limit: 2, batch: 2 })) {
|
|
281
|
-
consumer3Messages.push(msg);
|
|
282
|
-
await client.ack(msg);
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
if (consumer3Messages.length === 0) {
|
|
286
|
-
throw new Error('Consumer 3 got no messages (should have gotten remaining messages)');
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
log(`Consumer 3: Got ${consumer3Messages.length} messages after lease release`);
|
|
290
|
-
|
|
291
|
-
// Part 2: Test Bus Mode
|
|
292
|
-
log('=== Testing Bus Mode with Specific Partition ===');
|
|
293
|
-
|
|
294
|
-
// Push fresh messages
|
|
295
|
-
await client.push(`${QUEUE_NAME}/${PARTITION_NAME}`, [
|
|
296
|
-
{ data: 'Bus Message 1' },
|
|
297
|
-
{ data: 'Bus Message 2' },
|
|
298
|
-
{ data: 'Bus Message 3' },
|
|
299
|
-
{ data: 'Bus Message 4' },
|
|
300
|
-
{ data: 'Bus Message 5' }
|
|
301
|
-
]);
|
|
302
|
-
|
|
303
|
-
// Group Alpha, Consumer 1: Request specific partition
|
|
304
|
-
log('Group Alpha, Consumer 1: Requesting specific partition...');
|
|
305
|
-
const busGroup1Messages = [];
|
|
306
|
-
for await (const msg of client.take(`${QUEUE_NAME}/${PARTITION_NAME}@${CONSUMER_GROUP}`, { limit: 2, batch: 2 })) {
|
|
307
|
-
busGroup1Messages.push(msg);
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
if (busGroup1Messages.length === 0) {
|
|
311
|
-
throw new Error('Group Alpha Consumer 1 got no messages');
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
log(`Group Alpha Consumer 1: Got ${busGroup1Messages.length} messages`);
|
|
315
|
-
|
|
316
|
-
// Group Alpha, Consumer 2: Try same specific partition (should be locked)
|
|
317
|
-
log('Group Alpha, Consumer 2: Requesting same specific partition...');
|
|
318
|
-
let busGroup1Consumer2GotMessages = false;
|
|
319
|
-
for await (const msg of client.take(`${QUEUE_NAME}/${PARTITION_NAME}@${CONSUMER_GROUP}`, { limit: 2, batch: 2 })) {
|
|
320
|
-
busGroup1Consumer2GotMessages = true;
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
if (busGroup1Consumer2GotMessages) {
|
|
324
|
-
throw new Error('Group Alpha Consumer 2: Got messages (should be 0)');
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
log('Group Alpha Consumer 2: Got no messages (correct - partition locked by Consumer 1)');
|
|
328
|
-
|
|
329
|
-
// Different Group, Consumer 1: Should get messages (different consumer group)
|
|
330
|
-
log('Group Beta, Consumer 1: Requesting same partition (different group)...');
|
|
331
|
-
const busGroup2Messages = [];
|
|
332
|
-
for await (const msg of client.take(`${QUEUE_NAME}/${PARTITION_NAME}@group-beta`, { limit: 2, batch: 2 })) {
|
|
333
|
-
busGroup2Messages.push(msg);
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
if (busGroup2Messages.length === 0) {
|
|
337
|
-
throw new Error('Group Beta should be able to access partition (different group)');
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
log(`Group Beta Consumer 1: Got ${busGroup2Messages.length} messages (different group works)`);
|
|
341
|
-
|
|
342
|
-
// Clean up
|
|
343
|
-
for (const msg of busGroup1Messages) {
|
|
344
|
-
await client.ack(msg, true, { group: CONSUMER_GROUP });
|
|
345
|
-
}
|
|
346
|
-
for (const msg of busGroup2Messages) {
|
|
347
|
-
await client.ack(msg, true, { group: 'group-beta' });
|
|
348
|
-
}
|
|
349
|
-
|
|
350
|
-
passTest('Specific partition locking works correctly');
|
|
351
|
-
} catch (error) {
|
|
352
|
-
failTest(error);
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
/**
|
|
357
|
-
* Test: Namespace/Task Filtering with Partition Locking
|
|
358
|
-
* Tests that namespace/task filtering properly locks partitions
|
|
359
|
-
*/
|
|
360
|
-
export async function testNamespaceTaskFiltering(client) {
|
|
361
|
-
startTest('Namespace/Task Filtering with Partition Locking');
|
|
362
|
-
|
|
363
|
-
try {
|
|
364
|
-
const namespace = 'test-namespace';
|
|
365
|
-
const queues = [
|
|
366
|
-
{ name: 'test-queue-task-1', task: 'process', partition: 'p1' },
|
|
367
|
-
{ name: 'test-queue-task-2', task: 'process', partition: 'p2' },
|
|
368
|
-
{ name: 'test-queue-task-3', task: 'analyze', partition: 'p3' }
|
|
369
|
-
];
|
|
370
|
-
|
|
371
|
-
// Configure queues with namespace and task
|
|
372
|
-
for (const q of queues) {
|
|
373
|
-
await client.queue(q.name, { leaseTime: 30 }, {
|
|
374
|
-
namespace: namespace,
|
|
375
|
-
task: q.task
|
|
376
|
-
});
|
|
377
|
-
|
|
378
|
-
// Push messages to each queue
|
|
379
|
-
await client.push(`${q.name}/${q.partition}`, [
|
|
380
|
-
{ data: `Message for ${q.name}` },
|
|
381
|
-
{ data: `Message 2 for ${q.name}` }
|
|
382
|
-
]);
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
log('Pushed messages to 3 queues with namespace/task metadata');
|
|
386
|
-
|
|
387
|
-
// Test 1: Take by namespace only (should get messages from all queues)
|
|
388
|
-
log('Consumer 1: Taking by namespace only...');
|
|
389
|
-
const consumer1Messages = [];
|
|
390
|
-
for await (const msg of client.take('namespace:test-namespace', { limit: 3, batch: 3 })) {
|
|
391
|
-
consumer1Messages.push(msg);
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
if (consumer1Messages.length === 0) {
|
|
395
|
-
throw new Error('Consumer 1 got no messages');
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
log(`Consumer 1: Got ${consumer1Messages.length} messages from namespace ${namespace}`);
|
|
399
|
-
const partitions1 = [...new Set(consumer1Messages.map(m => m.partition))];
|
|
400
|
-
log(`Consumer 1: Locked partitions: ${partitions1.join(', ')}`);
|
|
401
|
-
|
|
402
|
-
// Test 2: Another consumer tries to take from same namespace (should get from unlocked partitions only)
|
|
403
|
-
log('Consumer 2: Trying to take from same namespace...');
|
|
404
|
-
const consumer2Messages = [];
|
|
405
|
-
for await (const msg of client.take('namespace:test-namespace', { limit: 3, batch: 3 })) {
|
|
406
|
-
consumer2Messages.push(msg);
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
if (consumer2Messages.length > 0) {
|
|
410
|
-
const partitions2 = [...new Set(consumer2Messages.map(m => m.partition))];
|
|
411
|
-
log(`Consumer 2: Got ${consumer2Messages.length} messages from partitions: ${partitions2.join(', ')}`);
|
|
412
|
-
|
|
413
|
-
// Verify no overlap in partitions
|
|
414
|
-
const overlap = partitions1.filter(p => partitions2.includes(p));
|
|
415
|
-
if (overlap.length > 0) {
|
|
416
|
-
throw new Error(`Partition locking failed! Both consumers got partition(s): ${overlap.join(', ')}`);
|
|
417
|
-
}
|
|
418
|
-
log('SUCCESS: No partition overlap between consumers');
|
|
419
|
-
} else {
|
|
420
|
-
log('Consumer 2: Got no messages (all partitions locked)');
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
// Test 3: Take by specific task (should only get from matching queues)
|
|
424
|
-
log('Consumer 3: Taking by specific task "analyze"...');
|
|
425
|
-
const consumer3Messages = [];
|
|
426
|
-
for await (const msg of client.take('namespace:test-namespace/task:analyze', { limit: 2 })) {
|
|
427
|
-
consumer3Messages.push(msg);
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
if (consumer3Messages.length > 0) {
|
|
431
|
-
log(`Consumer 3: Got ${consumer3Messages.length} messages for task "analyze"`);
|
|
432
|
-
// Verify all messages are from the analyze task
|
|
433
|
-
const allAnalyze = consumer3Messages.every(m => m.queue === 'test-queue-task-3');
|
|
434
|
-
if (!allAnalyze) {
|
|
435
|
-
throw new Error('Got messages from wrong task!');
|
|
436
|
-
}
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
// Clean up: ACK all messages
|
|
440
|
-
for (const msg of [...consumer1Messages, ...consumer2Messages, ...consumer3Messages]) {
|
|
441
|
-
await client.ack(msg);
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
passTest('Namespace/Task filtering with partition locking works correctly');
|
|
445
|
-
} catch (error) {
|
|
446
|
-
failTest(error);
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
/**
|
|
451
|
-
* Test: Namespace/Task Filtering in Bus Mode
|
|
452
|
-
* Tests that namespace/task filtering works with consumer groups
|
|
453
|
-
*/
|
|
454
|
-
export async function testNamespaceTaskBusMode(client) {
|
|
455
|
-
startTest('Namespace/Task Filtering in Bus Mode');
|
|
456
|
-
|
|
457
|
-
try {
|
|
458
|
-
const namespace = 'bus-namespace';
|
|
459
|
-
const task = 'bus-task';
|
|
460
|
-
const queues = [
|
|
461
|
-
{ name: 'test-bus-queue-1', partition: 'bus-p1' },
|
|
462
|
-
{ name: 'test-bus-queue-2', partition: 'bus-p2' }
|
|
463
|
-
];
|
|
464
|
-
|
|
465
|
-
// Setup queues and push messages
|
|
466
|
-
for (const q of queues) {
|
|
467
|
-
await client.queue(q.name, { leaseTime: 30 }, {
|
|
468
|
-
namespace: namespace,
|
|
469
|
-
task: task
|
|
470
|
-
});
|
|
471
|
-
|
|
472
|
-
await client.push(`${q.name}/${q.partition}`, [
|
|
473
|
-
{ data: `Bus message for ${q.name}` },
|
|
474
|
-
{ data: `Bus message 2 for ${q.name}` }
|
|
475
|
-
]);
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
log('Pushed messages to 2 queues for bus mode testing');
|
|
479
|
-
|
|
480
|
-
// Test 1: Consumer Group A takes messages
|
|
481
|
-
log('Group A, Consumer 1: Taking with namespace/task filter...');
|
|
482
|
-
const groupAMessages = [];
|
|
483
|
-
for await (const msg of client.take('namespace:bus-namespace/task:bus-task@group-a', { limit: 2, batch: 2 })) {
|
|
484
|
-
groupAMessages.push(msg);
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
if (groupAMessages.length === 0) {
|
|
488
|
-
throw new Error('Group A Consumer 1 got no messages');
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
const partitionsA1 = [...new Set(groupAMessages.map(m => m.partition))];
|
|
492
|
-
log(`Group A Consumer 1: Got ${groupAMessages.length} messages from partitions: ${partitionsA1.join(', ')}`);
|
|
493
|
-
|
|
494
|
-
// Test 2: Another consumer in Group A (should get from different partitions due to locking)
|
|
495
|
-
log('Group A, Consumer 2: Trying same namespace/task filter...');
|
|
496
|
-
const groupA2Messages = [];
|
|
497
|
-
for await (const msg of client.take('namespace:bus-namespace/task:bus-task@group-a', { limit: 2, batch: 2 })) {
|
|
498
|
-
groupA2Messages.push(msg);
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
if (groupA2Messages.length > 0) {
|
|
502
|
-
const partitionsA2 = [...new Set(groupA2Messages.map(m => m.partition))];
|
|
503
|
-
log(`Group A Consumer 2: Got ${groupA2Messages.length} messages from partitions: ${partitionsA2.join(', ')}`);
|
|
504
|
-
|
|
505
|
-
// Check for partition overlap
|
|
506
|
-
const overlapA = partitionsA1.filter(p => partitionsA2.includes(p));
|
|
507
|
-
if (overlapA.length > 0) {
|
|
508
|
-
log(`WARNING: Partition overlap in same group: ${overlapA.join(', ')}`);
|
|
509
|
-
}
|
|
510
|
-
} else {
|
|
511
|
-
log('Group A Consumer 2: No messages (partitions locked by Consumer 1)');
|
|
512
|
-
}
|
|
513
|
-
|
|
514
|
-
// Test 3: Consumer Group B (should be independent)
|
|
515
|
-
log('Group B, Consumer 1: Taking with same namespace/task filter...');
|
|
516
|
-
const groupBMessages = [];
|
|
517
|
-
for await (const msg of client.take('namespace:bus-namespace/task:bus-task@group-b', { limit: 2, batch: 2 })) {
|
|
518
|
-
groupBMessages.push(msg);
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
if (groupBMessages.length > 0) {
|
|
522
|
-
const partitionsB1 = [...new Set(groupBMessages.map(m => m.partition))];
|
|
523
|
-
log(`Group B Consumer 1: Got ${groupBMessages.length} messages from partitions: ${partitionsB1.join(', ')}`);
|
|
524
|
-
log('SUCCESS: Different consumer groups can access same partitions independently');
|
|
525
|
-
} else {
|
|
526
|
-
throw new Error('Group B should be able to get messages (different consumer group)');
|
|
527
|
-
}
|
|
528
|
-
|
|
529
|
-
// Clean up: ACK all messages
|
|
530
|
-
for (const msg of groupAMessages) {
|
|
531
|
-
await client.ack(msg, true, { group: 'group-a' });
|
|
532
|
-
}
|
|
533
|
-
for (const msg of groupA2Messages) {
|
|
534
|
-
await client.ack(msg, true, { group: 'group-a' });
|
|
535
|
-
}
|
|
536
|
-
for (const msg of groupBMessages) {
|
|
537
|
-
await client.ack(msg, true, { group: 'group-b' });
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
passTest('Namespace/Task filtering in bus mode works correctly');
|
|
541
|
-
} catch (error) {
|
|
542
|
-
failTest(error);
|
|
543
|
-
}
|
|
544
|
-
}
|
|
545
|
-
|