queen-mq 1.3.0 โ†’ 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +71 -18
  2. package/client-v2/Queen.js +45 -13
  3. package/client-v2/README.md +26 -11
  4. package/client-v2/admin/Admin.js +0 -47
  5. package/client-v2/builders/QueueBuilder.js +33 -10
  6. package/client-v2/builders/TimerBuilder.js +2 -2
  7. package/client-v2/builders/TransactionBuilder.js +35 -9
  8. package/client-v2/consumer/ConsumerManager.js +67 -46
  9. package/client-v2/ephemeral/Ephemeral.js +7 -9
  10. package/client-v2/kv/Kv.js +4 -4
  11. package/client-v2/kv/expiry.js +1 -1
  12. package/client-v2/streams/Stream.js +8 -3
  13. package/client-v2/streams/helpers/rateLimiter.js +4 -4
  14. package/client-v2/streams/operators/GateOperator.js +9 -2
  15. package/client-v2/streams/operators/ReduceOperator.js +2 -2
  16. package/client-v2/streams/operators/WindowSessionOperator.js +3 -3
  17. package/client-v2/streams/runtime/Runner.js +111 -54
  18. package/client-v2/streams/runtime/cycle.js +4 -4
  19. package/client-v2/streams/runtime/register.js +1 -1
  20. package/client-v2/utils/conflation.js +0 -6
  21. package/client-v2/utils/consumerGroup.js +54 -0
  22. package/package.json +5 -8
  23. package/test-v2/_kvtimers.js +12 -13
  24. package/test-v2/ackwindow.js +12 -192
  25. package/test-v2/bootstrap.js +2 -2
  26. package/test-v2/conflation-unit/conflationWire.test.js +0 -12
  27. package/test-v2/consume.js +41 -1
  28. package/test-v2/consumer-unit/handlerError.test.js +161 -0
  29. package/test-v2/docs.js +5 -4
  30. package/test-v2/http-unit/pushStatus.test.js +142 -0
  31. package/test-v2/http-unit/renew.test.js +96 -0
  32. package/test-v2/kv-unit/timerWire.test.js +1 -1
  33. package/test-v2/kv-unit/txnWire.test.js +101 -2
  34. package/test-v2/kv.js +6 -5
  35. package/test-v2/pop.js +28 -1
  36. package/test-v2/run.js +50 -114
  37. package/test-v2/runner-unit/fatalExit.test.js +64 -0
  38. package/test-v2/semantics.js +20 -34
  39. package/test-v2/stream/_helpers.js +10 -19
  40. package/test-v2/stream/cron.js +1 -1
  41. package/test-v2/stream/gate.js +54 -0
  42. package/test-v2/stream/index.js +2 -0
  43. package/test-v2/stream/tumbling.js +3 -3
  44. package/test-v2/streams-unit/ack.test.js +61 -0
  45. package/test-v2/streams-unit/cycle.test.js +1 -1
  46. package/test-v2/streams-unit/e2e.test.js +6 -10
  47. package/test-v2/streams-unit/gate.test.js +189 -0
  48. package/test-v2/timers.js +6 -5
  49. package/test-v2/transaction.js +169 -0
  50. package/test-v2/watermark.js +38 -176
  51. package/test-v2/maintenance.js +0 -277
@@ -1,277 +0,0 @@
1
- import { Queen } from '../client-v2/index.js';
2
- import axios from 'axios';
3
-
4
- export async function test_maintenance_mode(client = null) {
5
- console.log('๐Ÿงช Testing Maintenance Mode with File Buffer\n');
6
-
7
- const queen = client
8
-
9
- // Default Queen port; override with QUEEN_SERVER_URL (same as run.js).
10
- const API_URL = process.env.QUEEN_SERVER_URL || 'http://localhost:6632';
11
- const QUEUE_NAME = 'test-maintenance-queue';
12
- const MESSAGES_PER_SECOND = 10;
13
- const MAINTENANCE_DURATION_SECONDS = 10;
14
-
15
- let producedCount = 0;
16
- let receivedCount = 0;
17
- let consumerActive = false;
18
-
19
- try {
20
- // 0. Ensure maintenance mode is OFF before starting
21
- console.log('๐Ÿงน Step 0: Ensuring clean state...');
22
- await axios.post(`${API_URL}/api/v1/system/maintenance`, { enabled: false });
23
-
24
- let initialStatus = await axios.get(`${API_URL}/api/v1/system/maintenance`);
25
- console.log(` Initial status:`, initialStatus.data);
26
-
27
- if (initialStatus.data.bufferedMessages > 0) {
28
- console.log(` โš ๏ธ Found ${initialStatus.data.bufferedMessages} buffered messages from previous runs`);
29
- console.log(' Waiting for buffer to drain (checking every 2 seconds)...');
30
-
31
- // Wait for buffer to drain completely
32
- let waitTime = 0;
33
- const maxWait = 60; // Max 60 seconds
34
- while (waitTime < maxWait) {
35
- await sleep(2000);
36
- waitTime += 2;
37
- const status = await axios.get(`${API_URL}/api/v1/system/maintenance`);
38
- console.log(` ${waitTime}s - Buffered: ${status.data.bufferedMessages}`);
39
-
40
- if (status.data.bufferedMessages === 0) {
41
- console.log(' โœ… Buffer drained completely');
42
- break;
43
- }
44
- }
45
-
46
- if (waitTime >= maxWait) {
47
- console.log(` โš ๏ธ Timeout waiting for buffer to drain. Continuing anyway...`);
48
- }
49
- }
50
- console.log('');
51
-
52
- // 1. Configure queue
53
- console.log(`๐Ÿ“‹ Step 1: Configuring queue "${QUEUE_NAME}"...`);
54
- await queen.queue(QUEUE_NAME).config({
55
- leaseTime: 60,
56
- }).create();
57
- console.log('โœ… Queue configured\n');
58
-
59
- // 2. Start consumer
60
- console.log('๐Ÿ“ฅ Step 2: Starting consumer...');
61
- consumerActive = true;
62
-
63
- const consumePromise = (async () => {
64
- while (consumerActive) {
65
- try {
66
- // wait(true) long-polls on the server. This is important: pushes
67
- // commit before partition_lookup is refreshed (PUSHPOPLOOKUPSOL
68
- // is async), so a wait(false) pop can race past freshly-pushed
69
- // messages and return empty. Long-poll re-runs the candidate
70
- // scan internally and picks up messages once the lookup row
71
- // commits โ€” typically within milliseconds.
72
- // Use a short timeout (1s) so the consumer notices
73
- // consumerActive=false promptly when the test winds down.
74
- const messages = await queen.queue(QUEUE_NAME)
75
- .batch(10)
76
- .wait(true)
77
- .timeoutMillis(1000)
78
- .limit(1)
79
- .each()
80
- .pop();
81
-
82
- if (messages && messages.length > 0) {
83
- receivedCount += messages.length;
84
- console.log(` ๐Ÿ“จ Received ${messages.length} messages (total: ${receivedCount})`);
85
-
86
- // Acknowledge messages
87
- await queen.ack(messages);
88
-
89
- }
90
- } catch (err) {
91
- if (consumerActive) {
92
- console.error(' โŒ Consumer error:', err.message);
93
- }
94
- }
95
- await sleep(10);
96
- }
97
- })();
98
-
99
- console.log('โœ… Consumer started\n');
100
-
101
- // 3. Start producer (10 messages/second)
102
- console.log(`๐Ÿ“ค Step 3: Starting producer (${MESSAGES_PER_SECOND} msgs/sec)...`);
103
- let producerRunning = true;
104
-
105
- const producerLoop = async () => {
106
- while (producerRunning) {
107
- try {
108
- // Push batch of 10 messages
109
- const batch = [];
110
- for (let i = 0; i < MESSAGES_PER_SECOND; i++) {
111
- batch.push({
112
- data: {
113
- messageNumber: producedCount + i,
114
- timestamp: Date.now()
115
- }
116
- });
117
- }
118
-
119
- // Push and wait for confirmation
120
- await queen.queue(QUEUE_NAME).push(batch);
121
- producedCount += batch.length; // Only increment after successful push
122
-
123
- // Wait 1 second before next batch
124
- await sleep(1000);
125
- } catch (err) {
126
- console.error(' โŒ Producer error:', err.message);
127
- await sleep(1000); // Wait before retry
128
- }
129
- }
130
- };
131
-
132
- const producerPromise = producerLoop();
133
-
134
- console.log('โœ… Producer started\n');
135
-
136
- // Wait 3 seconds for messages to flow normally
137
- console.log('โฑ๏ธ Step 4: Waiting 3 seconds for normal message flow...');
138
- await sleep(3000);
139
- console.log(` Produced: ${producedCount}, Received: ${receivedCount}\n`);
140
-
141
- const receivedBeforeMaintenance = receivedCount;
142
-
143
- // 4. Enable maintenance mode
144
- console.log('๐Ÿ”ง Step 5: Enabling MAINTENANCE MODE...');
145
- const maintenanceResponse = await axios.post(`${API_URL}/api/v1/system/maintenance`, {
146
- enabled: true
147
- });
148
- console.log(` Response:`, maintenanceResponse.data);
149
- console.log('โœ… Maintenance mode enabled\n');
150
-
151
- // Wait 2 seconds and check that consumer stops receiving
152
- console.log('โฑ๏ธ Step 6: Waiting 5 seconds - consumer should stop receiving...');
153
- await sleep(5000);
154
- const receivedDuringWait = receivedCount - receivedBeforeMaintenance;
155
- console.log(` Messages received during maintenance: ${receivedDuringWait}`);
156
- console.log(` Total: Produced=${producedCount}, Received=${receivedCount}\n`);
157
-
158
- // 5. Keep producing during maintenance
159
- console.log(`โฑ๏ธ Step 7: Producing during maintenance for ${MAINTENANCE_DURATION_SECONDS} seconds...`);
160
- console.log(' (Messages should go to file buffer)\n');
161
- await sleep(MAINTENANCE_DURATION_SECONDS * 1000);
162
-
163
- const producedDuringMaintenance = producedCount;
164
- const receivedAfterMaintenance = receivedCount;
165
- const messagesBuffered = producedDuringMaintenance - receivedAfterMaintenance;
166
-
167
- console.log(` Maintenance period complete:`);
168
- console.log(` - Total produced: ${producedCount}`);
169
- console.log(` - Total received: ${receivedCount}`);
170
- console.log(` - Buffered (should be ~${MESSAGES_PER_SECOND * MAINTENANCE_DURATION_SECONDS}): ${messagesBuffered}\n`);
171
-
172
- // 6. Check maintenance status
173
- console.log('๐Ÿ“Š Step 8: Checking maintenance status...');
174
- const statusResponse = await axios.get(`${API_URL}/api/v1/system/maintenance`);
175
- console.log(` Status:`, statusResponse.data);
176
- console.log('');
177
-
178
- // 7. Disable maintenance mode
179
- console.log('โœ… Step 9: Disabling MAINTENANCE MODE...');
180
- const disableResponse = await axios.post(`${API_URL}/api/v1/system/maintenance`, {
181
- enabled: false
182
- });
183
- console.log(` Response:`, disableResponse.data);
184
- console.log(' (File buffer should start draining to database)\n');
185
-
186
- // 8. Wait for messages to resume and stop producer
187
- console.log('โฑ๏ธ Step 10: Waiting for messages to resume...');
188
- await sleep(2000);
189
-
190
- const receivedAfterResume = receivedCount;
191
- const messagesResumed = receivedAfterResume - receivedAfterMaintenance;
192
- console.log(` Messages received after resuming: ${messagesResumed}`);
193
- console.log('โœ… Messages are flowing again!\n');
194
-
195
- // Stop producer
196
- console.log('๐Ÿ›‘ Step 11: Stopping producer...');
197
- producerRunning = false;
198
- await producerPromise; // Wait for producer to actually stop
199
-
200
- const finalProducedCount = producedCount;
201
- console.log(` Final produced count: ${finalProducedCount}\n`);
202
-
203
- // 9. Wait for drain to complete
204
- console.log('โฑ๏ธ Step 12: Waiting 10 seconds for file buffer to drain...');
205
- for (let i = 0; i < 10; i++) {
206
- await sleep(1000);
207
- console.log(` ${i + 1}s - Received: ${receivedCount}/${finalProducedCount} (${((receivedCount/finalProducedCount)*100).toFixed(1)}%)`);
208
-
209
- // Check if we've received all messages
210
- if (receivedCount >= finalProducedCount) {
211
- console.log(' โœ… All messages received!\n');
212
- break;
213
- }
214
- }
215
-
216
- // 10. Stop consumer
217
- console.log('๐Ÿ›‘ Step 13: Stopping consumer...');
218
- consumerActive = false;
219
- await consumePromise;
220
- console.log('โœ… Consumer stopped\n');
221
-
222
- // 11. Final verification
223
- console.log('๐Ÿ“Š Final Verification:\n');
224
- console.log(` Total Produced: ${finalProducedCount}`);
225
- console.log(` Total Received: ${receivedCount}`);
226
- console.log(` Difference: ${finalProducedCount - receivedCount}`);
227
-
228
- if (receivedCount === finalProducedCount) {
229
- console.log('\nโœ… SUCCESS: All messages accounted for!');
230
- console.log(' Maintenance mode works correctly with file buffer.\n');
231
-
232
- return {
233
- success: true,
234
- message: `All ${finalProducedCount} messages accounted for during maintenance mode test`
235
- };
236
- } else if (receivedCount < finalProducedCount) {
237
- const missing = finalProducedCount - receivedCount;
238
- console.log(`\nโš ๏ธ WARNING: Missing ${missing} messages`);
239
- console.log(' They may still be in the file buffer or queue.');
240
- console.log(' Check /var/lib/queen/buffers/ for remaining files.\n');
241
-
242
- // Check buffer status
243
- const finalStatus = await axios.get(`${API_URL}/api/v1/system/maintenance`);
244
- console.log(' Buffer status:', finalStatus.data);
245
-
246
- return {
247
- success: false,
248
- message: `Missing ${missing} messages (produced: ${finalProducedCount}, received: ${receivedCount})`
249
- };
250
- } else {
251
- console.log(`\nโŒ ERROR: Received more messages than produced! (${receivedCount} > ${finalProducedCount})`);
252
- console.log(' This should not happen - possible duplicate issue.\n');
253
-
254
- return {
255
- success: false,
256
- message: `Received more than produced (${receivedCount} > ${finalProducedCount})`
257
- };
258
- }
259
-
260
- } catch (err) {
261
- console.error('\nโŒ Test failed:', err.message);
262
- console.error(err.stack);
263
-
264
- // Clean up
265
- consumerActive = false;
266
-
267
- return { success: false, message: `Test threw error: ${err.message}` };
268
- } finally {
269
- // Clean up
270
- consumerActive = false;
271
- }
272
- }
273
-
274
- function sleep(ms) {
275
- return new Promise(resolve => setTimeout(resolve, ms));
276
- }
277
-