queen-mq 1.3.0 โ 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -18
- package/client-v2/Queen.js +45 -13
- package/client-v2/README.md +26 -11
- package/client-v2/admin/Admin.js +0 -47
- package/client-v2/builders/QueueBuilder.js +33 -10
- package/client-v2/builders/TimerBuilder.js +2 -2
- package/client-v2/builders/TransactionBuilder.js +35 -9
- package/client-v2/consumer/ConsumerManager.js +67 -46
- package/client-v2/ephemeral/Ephemeral.js +7 -9
- package/client-v2/kv/Kv.js +4 -4
- package/client-v2/kv/expiry.js +1 -1
- package/client-v2/streams/Stream.js +8 -3
- package/client-v2/streams/helpers/rateLimiter.js +4 -4
- package/client-v2/streams/operators/GateOperator.js +9 -2
- package/client-v2/streams/operators/ReduceOperator.js +2 -2
- package/client-v2/streams/operators/WindowSessionOperator.js +3 -3
- package/client-v2/streams/runtime/Runner.js +111 -54
- package/client-v2/streams/runtime/cycle.js +4 -4
- package/client-v2/streams/runtime/register.js +1 -1
- package/client-v2/utils/conflation.js +0 -6
- package/client-v2/utils/consumerGroup.js +54 -0
- package/package.json +5 -8
- package/test-v2/_kvtimers.js +12 -13
- package/test-v2/ackwindow.js +12 -192
- package/test-v2/bootstrap.js +2 -2
- package/test-v2/conflation-unit/conflationWire.test.js +0 -12
- package/test-v2/consume.js +41 -1
- package/test-v2/consumer-unit/handlerError.test.js +161 -0
- package/test-v2/docs.js +5 -4
- package/test-v2/http-unit/pushStatus.test.js +142 -0
- package/test-v2/http-unit/renew.test.js +96 -0
- package/test-v2/kv-unit/timerWire.test.js +1 -1
- package/test-v2/kv-unit/txnWire.test.js +101 -2
- package/test-v2/kv.js +6 -5
- package/test-v2/pop.js +28 -1
- package/test-v2/run.js +50 -114
- package/test-v2/runner-unit/fatalExit.test.js +64 -0
- package/test-v2/semantics.js +20 -34
- package/test-v2/stream/_helpers.js +10 -19
- package/test-v2/stream/cron.js +1 -1
- package/test-v2/stream/gate.js +54 -0
- package/test-v2/stream/index.js +2 -0
- package/test-v2/stream/tumbling.js +3 -3
- package/test-v2/streams-unit/ack.test.js +61 -0
- package/test-v2/streams-unit/cycle.test.js +1 -1
- package/test-v2/streams-unit/e2e.test.js +6 -10
- package/test-v2/streams-unit/gate.test.js +189 -0
- package/test-v2/timers.js +6 -5
- package/test-v2/transaction.js +169 -0
- package/test-v2/watermark.js +38 -176
- package/test-v2/maintenance.js +0 -277
package/test-v2/maintenance.js
DELETED
|
@@ -1,277 +0,0 @@
|
|
|
1
|
-
import { Queen } from '../client-v2/index.js';
|
|
2
|
-
import axios from 'axios';
|
|
3
|
-
|
|
4
|
-
export async function test_maintenance_mode(client = null) {
|
|
5
|
-
console.log('๐งช Testing Maintenance Mode with File Buffer\n');
|
|
6
|
-
|
|
7
|
-
const queen = client
|
|
8
|
-
|
|
9
|
-
// Default Queen port; override with QUEEN_SERVER_URL (same as run.js).
|
|
10
|
-
const API_URL = process.env.QUEEN_SERVER_URL || 'http://localhost:6632';
|
|
11
|
-
const QUEUE_NAME = 'test-maintenance-queue';
|
|
12
|
-
const MESSAGES_PER_SECOND = 10;
|
|
13
|
-
const MAINTENANCE_DURATION_SECONDS = 10;
|
|
14
|
-
|
|
15
|
-
let producedCount = 0;
|
|
16
|
-
let receivedCount = 0;
|
|
17
|
-
let consumerActive = false;
|
|
18
|
-
|
|
19
|
-
try {
|
|
20
|
-
// 0. Ensure maintenance mode is OFF before starting
|
|
21
|
-
console.log('๐งน Step 0: Ensuring clean state...');
|
|
22
|
-
await axios.post(`${API_URL}/api/v1/system/maintenance`, { enabled: false });
|
|
23
|
-
|
|
24
|
-
let initialStatus = await axios.get(`${API_URL}/api/v1/system/maintenance`);
|
|
25
|
-
console.log(` Initial status:`, initialStatus.data);
|
|
26
|
-
|
|
27
|
-
if (initialStatus.data.bufferedMessages > 0) {
|
|
28
|
-
console.log(` โ ๏ธ Found ${initialStatus.data.bufferedMessages} buffered messages from previous runs`);
|
|
29
|
-
console.log(' Waiting for buffer to drain (checking every 2 seconds)...');
|
|
30
|
-
|
|
31
|
-
// Wait for buffer to drain completely
|
|
32
|
-
let waitTime = 0;
|
|
33
|
-
const maxWait = 60; // Max 60 seconds
|
|
34
|
-
while (waitTime < maxWait) {
|
|
35
|
-
await sleep(2000);
|
|
36
|
-
waitTime += 2;
|
|
37
|
-
const status = await axios.get(`${API_URL}/api/v1/system/maintenance`);
|
|
38
|
-
console.log(` ${waitTime}s - Buffered: ${status.data.bufferedMessages}`);
|
|
39
|
-
|
|
40
|
-
if (status.data.bufferedMessages === 0) {
|
|
41
|
-
console.log(' โ
Buffer drained completely');
|
|
42
|
-
break;
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
if (waitTime >= maxWait) {
|
|
47
|
-
console.log(` โ ๏ธ Timeout waiting for buffer to drain. Continuing anyway...`);
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
console.log('');
|
|
51
|
-
|
|
52
|
-
// 1. Configure queue
|
|
53
|
-
console.log(`๐ Step 1: Configuring queue "${QUEUE_NAME}"...`);
|
|
54
|
-
await queen.queue(QUEUE_NAME).config({
|
|
55
|
-
leaseTime: 60,
|
|
56
|
-
}).create();
|
|
57
|
-
console.log('โ
Queue configured\n');
|
|
58
|
-
|
|
59
|
-
// 2. Start consumer
|
|
60
|
-
console.log('๐ฅ Step 2: Starting consumer...');
|
|
61
|
-
consumerActive = true;
|
|
62
|
-
|
|
63
|
-
const consumePromise = (async () => {
|
|
64
|
-
while (consumerActive) {
|
|
65
|
-
try {
|
|
66
|
-
// wait(true) long-polls on the server. This is important: pushes
|
|
67
|
-
// commit before partition_lookup is refreshed (PUSHPOPLOOKUPSOL
|
|
68
|
-
// is async), so a wait(false) pop can race past freshly-pushed
|
|
69
|
-
// messages and return empty. Long-poll re-runs the candidate
|
|
70
|
-
// scan internally and picks up messages once the lookup row
|
|
71
|
-
// commits โ typically within milliseconds.
|
|
72
|
-
// Use a short timeout (1s) so the consumer notices
|
|
73
|
-
// consumerActive=false promptly when the test winds down.
|
|
74
|
-
const messages = await queen.queue(QUEUE_NAME)
|
|
75
|
-
.batch(10)
|
|
76
|
-
.wait(true)
|
|
77
|
-
.timeoutMillis(1000)
|
|
78
|
-
.limit(1)
|
|
79
|
-
.each()
|
|
80
|
-
.pop();
|
|
81
|
-
|
|
82
|
-
if (messages && messages.length > 0) {
|
|
83
|
-
receivedCount += messages.length;
|
|
84
|
-
console.log(` ๐จ Received ${messages.length} messages (total: ${receivedCount})`);
|
|
85
|
-
|
|
86
|
-
// Acknowledge messages
|
|
87
|
-
await queen.ack(messages);
|
|
88
|
-
|
|
89
|
-
}
|
|
90
|
-
} catch (err) {
|
|
91
|
-
if (consumerActive) {
|
|
92
|
-
console.error(' โ Consumer error:', err.message);
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
await sleep(10);
|
|
96
|
-
}
|
|
97
|
-
})();
|
|
98
|
-
|
|
99
|
-
console.log('โ
Consumer started\n');
|
|
100
|
-
|
|
101
|
-
// 3. Start producer (10 messages/second)
|
|
102
|
-
console.log(`๐ค Step 3: Starting producer (${MESSAGES_PER_SECOND} msgs/sec)...`);
|
|
103
|
-
let producerRunning = true;
|
|
104
|
-
|
|
105
|
-
const producerLoop = async () => {
|
|
106
|
-
while (producerRunning) {
|
|
107
|
-
try {
|
|
108
|
-
// Push batch of 10 messages
|
|
109
|
-
const batch = [];
|
|
110
|
-
for (let i = 0; i < MESSAGES_PER_SECOND; i++) {
|
|
111
|
-
batch.push({
|
|
112
|
-
data: {
|
|
113
|
-
messageNumber: producedCount + i,
|
|
114
|
-
timestamp: Date.now()
|
|
115
|
-
}
|
|
116
|
-
});
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
// Push and wait for confirmation
|
|
120
|
-
await queen.queue(QUEUE_NAME).push(batch);
|
|
121
|
-
producedCount += batch.length; // Only increment after successful push
|
|
122
|
-
|
|
123
|
-
// Wait 1 second before next batch
|
|
124
|
-
await sleep(1000);
|
|
125
|
-
} catch (err) {
|
|
126
|
-
console.error(' โ Producer error:', err.message);
|
|
127
|
-
await sleep(1000); // Wait before retry
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
};
|
|
131
|
-
|
|
132
|
-
const producerPromise = producerLoop();
|
|
133
|
-
|
|
134
|
-
console.log('โ
Producer started\n');
|
|
135
|
-
|
|
136
|
-
// Wait 3 seconds for messages to flow normally
|
|
137
|
-
console.log('โฑ๏ธ Step 4: Waiting 3 seconds for normal message flow...');
|
|
138
|
-
await sleep(3000);
|
|
139
|
-
console.log(` Produced: ${producedCount}, Received: ${receivedCount}\n`);
|
|
140
|
-
|
|
141
|
-
const receivedBeforeMaintenance = receivedCount;
|
|
142
|
-
|
|
143
|
-
// 4. Enable maintenance mode
|
|
144
|
-
console.log('๐ง Step 5: Enabling MAINTENANCE MODE...');
|
|
145
|
-
const maintenanceResponse = await axios.post(`${API_URL}/api/v1/system/maintenance`, {
|
|
146
|
-
enabled: true
|
|
147
|
-
});
|
|
148
|
-
console.log(` Response:`, maintenanceResponse.data);
|
|
149
|
-
console.log('โ
Maintenance mode enabled\n');
|
|
150
|
-
|
|
151
|
-
// Wait 2 seconds and check that consumer stops receiving
|
|
152
|
-
console.log('โฑ๏ธ Step 6: Waiting 5 seconds - consumer should stop receiving...');
|
|
153
|
-
await sleep(5000);
|
|
154
|
-
const receivedDuringWait = receivedCount - receivedBeforeMaintenance;
|
|
155
|
-
console.log(` Messages received during maintenance: ${receivedDuringWait}`);
|
|
156
|
-
console.log(` Total: Produced=${producedCount}, Received=${receivedCount}\n`);
|
|
157
|
-
|
|
158
|
-
// 5. Keep producing during maintenance
|
|
159
|
-
console.log(`โฑ๏ธ Step 7: Producing during maintenance for ${MAINTENANCE_DURATION_SECONDS} seconds...`);
|
|
160
|
-
console.log(' (Messages should go to file buffer)\n');
|
|
161
|
-
await sleep(MAINTENANCE_DURATION_SECONDS * 1000);
|
|
162
|
-
|
|
163
|
-
const producedDuringMaintenance = producedCount;
|
|
164
|
-
const receivedAfterMaintenance = receivedCount;
|
|
165
|
-
const messagesBuffered = producedDuringMaintenance - receivedAfterMaintenance;
|
|
166
|
-
|
|
167
|
-
console.log(` Maintenance period complete:`);
|
|
168
|
-
console.log(` - Total produced: ${producedCount}`);
|
|
169
|
-
console.log(` - Total received: ${receivedCount}`);
|
|
170
|
-
console.log(` - Buffered (should be ~${MESSAGES_PER_SECOND * MAINTENANCE_DURATION_SECONDS}): ${messagesBuffered}\n`);
|
|
171
|
-
|
|
172
|
-
// 6. Check maintenance status
|
|
173
|
-
console.log('๐ Step 8: Checking maintenance status...');
|
|
174
|
-
const statusResponse = await axios.get(`${API_URL}/api/v1/system/maintenance`);
|
|
175
|
-
console.log(` Status:`, statusResponse.data);
|
|
176
|
-
console.log('');
|
|
177
|
-
|
|
178
|
-
// 7. Disable maintenance mode
|
|
179
|
-
console.log('โ
Step 9: Disabling MAINTENANCE MODE...');
|
|
180
|
-
const disableResponse = await axios.post(`${API_URL}/api/v1/system/maintenance`, {
|
|
181
|
-
enabled: false
|
|
182
|
-
});
|
|
183
|
-
console.log(` Response:`, disableResponse.data);
|
|
184
|
-
console.log(' (File buffer should start draining to database)\n');
|
|
185
|
-
|
|
186
|
-
// 8. Wait for messages to resume and stop producer
|
|
187
|
-
console.log('โฑ๏ธ Step 10: Waiting for messages to resume...');
|
|
188
|
-
await sleep(2000);
|
|
189
|
-
|
|
190
|
-
const receivedAfterResume = receivedCount;
|
|
191
|
-
const messagesResumed = receivedAfterResume - receivedAfterMaintenance;
|
|
192
|
-
console.log(` Messages received after resuming: ${messagesResumed}`);
|
|
193
|
-
console.log('โ
Messages are flowing again!\n');
|
|
194
|
-
|
|
195
|
-
// Stop producer
|
|
196
|
-
console.log('๐ Step 11: Stopping producer...');
|
|
197
|
-
producerRunning = false;
|
|
198
|
-
await producerPromise; // Wait for producer to actually stop
|
|
199
|
-
|
|
200
|
-
const finalProducedCount = producedCount;
|
|
201
|
-
console.log(` Final produced count: ${finalProducedCount}\n`);
|
|
202
|
-
|
|
203
|
-
// 9. Wait for drain to complete
|
|
204
|
-
console.log('โฑ๏ธ Step 12: Waiting 10 seconds for file buffer to drain...');
|
|
205
|
-
for (let i = 0; i < 10; i++) {
|
|
206
|
-
await sleep(1000);
|
|
207
|
-
console.log(` ${i + 1}s - Received: ${receivedCount}/${finalProducedCount} (${((receivedCount/finalProducedCount)*100).toFixed(1)}%)`);
|
|
208
|
-
|
|
209
|
-
// Check if we've received all messages
|
|
210
|
-
if (receivedCount >= finalProducedCount) {
|
|
211
|
-
console.log(' โ
All messages received!\n');
|
|
212
|
-
break;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
// 10. Stop consumer
|
|
217
|
-
console.log('๐ Step 13: Stopping consumer...');
|
|
218
|
-
consumerActive = false;
|
|
219
|
-
await consumePromise;
|
|
220
|
-
console.log('โ
Consumer stopped\n');
|
|
221
|
-
|
|
222
|
-
// 11. Final verification
|
|
223
|
-
console.log('๐ Final Verification:\n');
|
|
224
|
-
console.log(` Total Produced: ${finalProducedCount}`);
|
|
225
|
-
console.log(` Total Received: ${receivedCount}`);
|
|
226
|
-
console.log(` Difference: ${finalProducedCount - receivedCount}`);
|
|
227
|
-
|
|
228
|
-
if (receivedCount === finalProducedCount) {
|
|
229
|
-
console.log('\nโ
SUCCESS: All messages accounted for!');
|
|
230
|
-
console.log(' Maintenance mode works correctly with file buffer.\n');
|
|
231
|
-
|
|
232
|
-
return {
|
|
233
|
-
success: true,
|
|
234
|
-
message: `All ${finalProducedCount} messages accounted for during maintenance mode test`
|
|
235
|
-
};
|
|
236
|
-
} else if (receivedCount < finalProducedCount) {
|
|
237
|
-
const missing = finalProducedCount - receivedCount;
|
|
238
|
-
console.log(`\nโ ๏ธ WARNING: Missing ${missing} messages`);
|
|
239
|
-
console.log(' They may still be in the file buffer or queue.');
|
|
240
|
-
console.log(' Check /var/lib/queen/buffers/ for remaining files.\n');
|
|
241
|
-
|
|
242
|
-
// Check buffer status
|
|
243
|
-
const finalStatus = await axios.get(`${API_URL}/api/v1/system/maintenance`);
|
|
244
|
-
console.log(' Buffer status:', finalStatus.data);
|
|
245
|
-
|
|
246
|
-
return {
|
|
247
|
-
success: false,
|
|
248
|
-
message: `Missing ${missing} messages (produced: ${finalProducedCount}, received: ${receivedCount})`
|
|
249
|
-
};
|
|
250
|
-
} else {
|
|
251
|
-
console.log(`\nโ ERROR: Received more messages than produced! (${receivedCount} > ${finalProducedCount})`);
|
|
252
|
-
console.log(' This should not happen - possible duplicate issue.\n');
|
|
253
|
-
|
|
254
|
-
return {
|
|
255
|
-
success: false,
|
|
256
|
-
message: `Received more than produced (${receivedCount} > ${finalProducedCount})`
|
|
257
|
-
};
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
} catch (err) {
|
|
261
|
-
console.error('\nโ Test failed:', err.message);
|
|
262
|
-
console.error(err.stack);
|
|
263
|
-
|
|
264
|
-
// Clean up
|
|
265
|
-
consumerActive = false;
|
|
266
|
-
|
|
267
|
-
return { success: false, message: `Test threw error: ${err.message}` };
|
|
268
|
-
} finally {
|
|
269
|
-
// Clean up
|
|
270
|
-
consumerActive = false;
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
function sleep(ms) {
|
|
275
|
-
return new Promise(resolve => setTimeout(resolve, ms));
|
|
276
|
-
}
|
|
277
|
-
|