queen-mq 2.0.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -5
- package/client-v2/Queen.js +2 -2
- package/client-v2/README.md +8 -2
- package/client-v2/admin/Admin.js +30 -8
- package/client-v2/builders/QueueBuilder.js +73 -24
- package/client-v2/consumer/ConsumerManager.js +12 -7
- package/client-v2/ephemeral/Ephemeral.js +40 -8
- package/client-v2/utils/defaults.js +3 -2
- package/package.json +2 -2
- package/test-v2/admin-unit/removedRoutes.test.js +63 -0
- package/test-v2/consumer-unit/nackScope.test.js +88 -0
- package/test-v2/ephemeral-unit/ephemeralWire.test.js +24 -1
- package/test-v2/pop-unit/popDefaults.test.js +236 -0
- package/test-v2/pop.js +68 -0
package/README.md
CHANGED
|
@@ -22,7 +22,7 @@ Queen MQ is a partitioned message queue broker that keeps its state in its own r
|
|
|
22
22
|
- **Consumer Groups** - Kafka-style consumer groups for scalability
|
|
23
23
|
- **Flexible Semantics** - Exactly-once, at-least-once, and at-most-once delivery
|
|
24
24
|
- **Transactions** - Atomic operations across push and ack
|
|
25
|
-
- **High Performance** —
|
|
25
|
+
- **High Performance** — 1M msg/s in and out of one queue of 10M partitions on three nodes ([benchmarks](https://queenmq.com/benchmarks/))
|
|
26
26
|
- **Subscription Modes** - Process from beginning, new messages only, or from timestamp
|
|
27
27
|
- **Dead Letter Queue** - Automatic failure handling and monitoring
|
|
28
28
|
- **Message Tracing** - Debug distributed workflows with trace timelines
|
|
@@ -192,8 +192,8 @@ Notes:
|
|
|
192
192
|
the partition cap; either way it is clamped to 64, so a conflating pop returns
|
|
193
193
|
at most 64 messages per round-trip whatever `batch` says.
|
|
194
194
|
- Refused with 400 by the broker without a `.group(...)`, and together with
|
|
195
|
-
`.
|
|
196
|
-
|
|
195
|
+
`.commitOnDelivery()` (a commit at delivery would turn the guarantee above
|
|
196
|
+
into at-most-once). `pop()` raises that 400 instead of returning `[]`.
|
|
197
197
|
- Requires broker **>= 1.1.0**. An older broker ignores the flag and would
|
|
198
198
|
quietly deliver the whole backlog, so the SDK raises
|
|
199
199
|
`conflation was requested but this broker did not apply it` on the first
|
|
@@ -281,8 +281,9 @@ await queen.queue('tasks')
|
|
|
281
281
|
|
|
282
282
|
**When the handler throws**, the consumer nacks what it was given (the message, or the whole batch)
|
|
283
283
|
and keeps consuming. The broker redelivers it, and moves it to the dead letter queue once the queue's
|
|
284
|
-
`retryLimit` is spent. With `.each()`, the
|
|
285
|
-
broker redelivers
|
|
284
|
+
`retryLimit` is spent. With `.each()`, the later messages of the failed message's partition are
|
|
285
|
+
skipped after a nack (the broker redelivers them too); the other partitions of the same pop are
|
|
286
|
+
still handled.
|
|
286
287
|
|
|
287
288
|
This is the same with `.autoAck(false)`. That setting hands the *success* path to your handler (it
|
|
288
289
|
acks), not the failure path: a handler that threw never got to settle its messages. A message your
|
|
@@ -794,8 +795,20 @@ const msgs = await queen.queue('q').batch(10).pop()
|
|
|
794
795
|
const msgs = await queen.queue('q').batch(10).wait(true).pop()
|
|
795
796
|
const msgs = await queen.queue('q').batch(200).partitions(50).pop() // multi-partition pop
|
|
796
797
|
const { messages, autopilot } = await queen.queue('q').popResult() // + what the broker chose
|
|
798
|
+
const msgs = await queen.queue('q').group('g').commitOnDelivery().pop() // committed at delivery, nothing to ack
|
|
797
799
|
```
|
|
798
800
|
|
|
801
|
+
A pop long-polls, waiting up to `timeoutMillis` (30 s) for a message, unless you call
|
|
802
|
+
`.wait(false)`. Its messages come back leased, and the ack is yours.
|
|
803
|
+
|
|
804
|
+
`.commitOnDelivery()` changes that for `pop()` and `popResult()`. The broker moves the group's
|
|
805
|
+
cursor past the messages as it hands them out: there is no lease (`leaseId` is empty) and nothing
|
|
806
|
+
to ack. This is at-most-once delivery: a crash after the pop loses the messages. The broker
|
|
807
|
+
refuses it together with `.conflation()` (400). `consume()` always leases its messages, so it
|
|
808
|
+
throws before any request when the builder has `.commitOnDelivery()`.
|
|
809
|
+
|
|
810
|
+
`.autoAck()` is `consume()`'s ack after your handler and has no effect on a pop.
|
|
811
|
+
|
|
799
812
|
### Consume
|
|
800
813
|
|
|
801
814
|
```javascript
|
|
@@ -950,6 +963,18 @@ await queen.close() // Flush buffers and close connections
|
|
|
950
963
|
the broker. These values are what comes back with `.autopilot(false)` or
|
|
951
964
|
`QUEEN_SDK_POP_AUTOPILOT=off`.
|
|
952
965
|
|
|
966
|
+
### Pop Defaults
|
|
967
|
+
|
|
968
|
+
```javascript
|
|
969
|
+
{
|
|
970
|
+
batch: 1, // autopilot OFF only, as for consume
|
|
971
|
+
wait: true, // long-polls; .wait(false) returns at once
|
|
972
|
+
timeoutMillis: 30000, // the long-poll limit
|
|
973
|
+
autoAck: false, // consume()'s ack after the handler; no effect on pop()
|
|
974
|
+
commitOnDelivery: false // leased; .commitOnDelivery() commits at delivery (at-most-once)
|
|
975
|
+
}
|
|
976
|
+
```
|
|
977
|
+
|
|
953
978
|
---
|
|
954
979
|
|
|
955
980
|
## Logging
|
package/client-v2/Queen.js
CHANGED
|
@@ -319,8 +319,8 @@ export class Queen {
|
|
|
319
319
|
* creates it, which is what makes thousands of short-lived req/reply
|
|
320
320
|
* inboxes cheap (§1.1).
|
|
321
321
|
* * delivery is at-least-once while the owning broker lives, at-most-once
|
|
322
|
-
* with `
|
|
323
|
-
* need idempotency.
|
|
322
|
+
* with `commitOnDelivery` (§1.3) -- NOT "at most once" as a class.
|
|
323
|
+
* Consumers still need idempotency.
|
|
324
324
|
* * consumption semantics are the pop's `group`, exactly as on durable
|
|
325
325
|
* queues (§1.5): same group competes, own group fans out, no group is
|
|
326
326
|
* queue mode. There is no queue-level mode to set.
|
package/client-v2/README.md
CHANGED
|
@@ -1662,6 +1662,11 @@ const msgs = await queen.queue('q').partition('p1').pop()
|
|
|
1662
1662
|
// capped at 200 total messages. All partitions share one leaseId.
|
|
1663
1663
|
// Each returned message carries its own partitionId / partition / leaseId.
|
|
1664
1664
|
const msgs = await queen.queue('q').batch(200).partitions(50).pop()
|
|
1665
|
+
|
|
1666
|
+
// Commit at delivery: the broker moves the group's cursor past the messages
|
|
1667
|
+
// as it hands them out. No lease, nothing to ack, at-most-once: a crash after
|
|
1668
|
+
// the pop loses them. pop() and popResult() only; consume() throws.
|
|
1669
|
+
const msgs = await queen.queue('q').group('g').commitOnDelivery().pop()
|
|
1665
1670
|
```
|
|
1666
1671
|
|
|
1667
1672
|
### Consume
|
|
@@ -1915,8 +1920,9 @@ await queen.close()
|
|
|
1915
1920
|
```javascript
|
|
1916
1921
|
{
|
|
1917
1922
|
batch: 1,
|
|
1918
|
-
wait:
|
|
1919
|
-
autoAck: false
|
|
1923
|
+
wait: true, // Long polling; .wait(false) returns at once
|
|
1924
|
+
autoAck: false, // consume()'s ack after the handler; no effect on pop()
|
|
1925
|
+
commitOnDelivery: false // Manual ack required; .commitOnDelivery() commits at delivery
|
|
1920
1926
|
}
|
|
1921
1927
|
```
|
|
1922
1928
|
|
package/client-v2/admin/Admin.js
CHANGED
|
@@ -101,15 +101,26 @@ export class Admin {
|
|
|
101
101
|
}
|
|
102
102
|
|
|
103
103
|
/**
|
|
104
|
-
*
|
|
104
|
+
* @deprecated The 2.x broker has no route that clears a queue, so this
|
|
105
|
+
* always rejects, before any request. Seek each consumer group
|
|
106
|
+
* to the end instead.
|
|
107
|
+
*
|
|
108
|
+
* The old DELETE /api/v1/queues/:name/clear answers 404 no_such_route on
|
|
109
|
+
* 2.x. A seek to the end moves one group's cursor past every queued message
|
|
110
|
+
* (and releases its live leases); the group of a pop without one is
|
|
111
|
+
* '__QUEUE_MODE__'. Deleting the queue is not a clear: it removes the queue
|
|
112
|
+
* itself, with its configuration.
|
|
105
113
|
* @param {string} name - Queue name
|
|
106
|
-
* @param {string} [partition] -
|
|
107
|
-
* @returns {Promise<
|
|
114
|
+
* @param {string} [partition] - Ignored
|
|
115
|
+
* @returns {Promise<never>}
|
|
108
116
|
*/
|
|
109
117
|
async clearQueue(name, partition = null) {
|
|
110
118
|
logger.log('Admin.clearQueue', { name, partition })
|
|
111
|
-
|
|
112
|
-
|
|
119
|
+
throw new Error(
|
|
120
|
+
'Queen 2.x has no route that clears a queue. Move each consumer group past what is queued ' +
|
|
121
|
+
`with queen.admin.seekConsumerGroup(group, '${name}', { toEnd: true }); ` +
|
|
122
|
+
"a pop without a group reads as the group '__QUEUE_MODE__'."
|
|
123
|
+
)
|
|
113
124
|
}
|
|
114
125
|
|
|
115
126
|
/**
|
|
@@ -172,14 +183,25 @@ export class Admin {
|
|
|
172
183
|
}
|
|
173
184
|
|
|
174
185
|
/**
|
|
175
|
-
*
|
|
186
|
+
* @deprecated The 2.x broker has no route that moves a message to the DLQ
|
|
187
|
+
* by its address, so this always rejects, before any request.
|
|
188
|
+
* Ack the leased message with the `dlq` status instead.
|
|
189
|
+
*
|
|
190
|
+
* The old POST /api/v1/messages/:partitionId/:transactionId/dlq answers 404
|
|
191
|
+
* no_such_route on 2.x. The broker files a dead letter when the consumer
|
|
192
|
+
* group acks the message with the `dlq` status while it holds the lease (an
|
|
193
|
+
* address alone cannot name the lease), or when a `failed` ack spends the
|
|
194
|
+
* queue's last retry.
|
|
176
195
|
* @param {string} partitionId - Partition ID
|
|
177
196
|
* @param {string} transactionId - Transaction ID
|
|
178
|
-
* @returns {Promise<
|
|
197
|
+
* @returns {Promise<never>}
|
|
179
198
|
*/
|
|
180
199
|
async moveMessageToDLQ(partitionId, transactionId) {
|
|
181
200
|
logger.log('Admin.moveMessageToDLQ', { partitionId, transactionId })
|
|
182
|
-
|
|
201
|
+
throw new Error(
|
|
202
|
+
'Queen 2.x has no route that moves a message to the DLQ by its address. Pop the message for ' +
|
|
203
|
+
"its consumer group and call queen.ack(message, 'dlq', { group }) while you hold its lease."
|
|
204
|
+
)
|
|
183
205
|
}
|
|
184
206
|
|
|
185
207
|
// ===========================
|
|
@@ -24,6 +24,12 @@ import * as logger from '../utils/logger.js'
|
|
|
24
24
|
// this client cannot read is not a message it can report as stored.
|
|
25
25
|
const ACCEPTED_PUSH_STATUSES = new Set(['queued', 'buffered'])
|
|
26
26
|
|
|
27
|
+
// What consume() throws when the builder carries commitOnDelivery(): the loop
|
|
28
|
+
// always pops leased and acks after the handler, so it cannot honour a commit
|
|
29
|
+
// at delivery, and silently leasing would not be what the caller asked for.
|
|
30
|
+
const COMMIT_ON_DELIVERY_NOT_FOR_CONSUME =
|
|
31
|
+
'commitOnDelivery() is a pop() option; consume() always leases its messages'
|
|
32
|
+
|
|
27
33
|
export class QueueBuilder {
|
|
28
34
|
#queen
|
|
29
35
|
#httpClient
|
|
@@ -45,8 +51,14 @@ export class QueueBuilder {
|
|
|
45
51
|
#batch = null
|
|
46
52
|
#limit = CONSUME_DEFAULTS.limit
|
|
47
53
|
#idleMillis = CONSUME_DEFAULTS.idleMillis
|
|
48
|
-
|
|
49
|
-
|
|
54
|
+
// autoAck and wait hold the USER's value, and null means the setter was never
|
|
55
|
+
// called: consume() applies CONSUME_DEFAULTS and pop() POP_DEFAULTS at
|
|
56
|
+
// emission time. autoAck is consume()'s alone; pop() never sends it.
|
|
57
|
+
#autoAck = null
|
|
58
|
+
#wait = null
|
|
59
|
+
// The pop's own option for the broker's commit at delivery: pop() and
|
|
60
|
+
// popResult() send it as autoAck=true, and consume() refuses it.
|
|
61
|
+
#commitOnDelivery = POP_DEFAULTS.commitOnDelivery
|
|
50
62
|
#timeoutMillis = CONSUME_DEFAULTS.timeoutMillis
|
|
51
63
|
#renewLease = CONSUME_DEFAULTS.renewLease
|
|
52
64
|
#renewLeaseIntervalMillis = CONSUME_DEFAULTS.renewLeaseIntervalMillis
|
|
@@ -301,6 +313,12 @@ export class QueueBuilder {
|
|
|
301
313
|
return this
|
|
302
314
|
}
|
|
303
315
|
|
|
316
|
+
/**
|
|
317
|
+
* consume(): ack each message after the handler returns (default true; the
|
|
318
|
+
* client acks, nothing is sent with the pop). No effect on pop(): it never
|
|
319
|
+
* reaches the wire. For the broker's at-most-once commit at delivery on a
|
|
320
|
+
* pop, use commitOnDelivery().
|
|
321
|
+
*/
|
|
304
322
|
autoAck(enabled) {
|
|
305
323
|
this.#autoAck = enabled
|
|
306
324
|
return this
|
|
@@ -344,8 +362,9 @@ export class QueueBuilder {
|
|
|
344
362
|
* that does not echo the flag rather than draining it silently.
|
|
345
363
|
*
|
|
346
364
|
* Refused by the broker (400) when combined with queue mode (no consumer
|
|
347
|
-
* group) or with
|
|
348
|
-
* "the newest state is definitely processed" guarantee into
|
|
365
|
+
* group) or with commitOnDelivery(), which commits at delivery and would
|
|
366
|
+
* turn the "the newest state is definitely processed" guarantee into
|
|
367
|
+
* at-most-once. pop() raises that 400 instead of returning [].
|
|
349
368
|
*/
|
|
350
369
|
conflation(enabled = true) {
|
|
351
370
|
this.#conflation = !!enabled
|
|
@@ -362,6 +381,10 @@ export class QueueBuilder {
|
|
|
362
381
|
// ===========================
|
|
363
382
|
|
|
364
383
|
consume(handler, options = {}) {
|
|
384
|
+
if (this.#commitOnDelivery) {
|
|
385
|
+
throw new Error(COMMIT_ON_DELIVERY_NOT_FOR_CONSUME)
|
|
386
|
+
}
|
|
387
|
+
|
|
365
388
|
const consumeOptions = {
|
|
366
389
|
queue: this.#queueName,
|
|
367
390
|
partition: this.#partition !== 'Default' ? this.#partition : null,
|
|
@@ -372,8 +395,8 @@ export class QueueBuilder {
|
|
|
372
395
|
batch: this.#batch,
|
|
373
396
|
limit: this.#limit,
|
|
374
397
|
idleMillis: this.#idleMillis,
|
|
375
|
-
autoAck: this.#autoAck,
|
|
376
|
-
wait: this.#wait,
|
|
398
|
+
autoAck: this.#autoAck ?? CONSUME_DEFAULTS.autoAck,
|
|
399
|
+
wait: this.#wait ?? CONSUME_DEFAULTS.wait,
|
|
377
400
|
timeoutMillis: this.#timeoutMillis,
|
|
378
401
|
renewLease: this.#renewLease,
|
|
379
402
|
renewLeaseIntervalMillis: this.#renewLeaseIntervalMillis,
|
|
@@ -397,6 +420,10 @@ export class QueueBuilder {
|
|
|
397
420
|
// Pop Methods
|
|
398
421
|
// ===========================
|
|
399
422
|
|
|
423
|
+
/**
|
|
424
|
+
* Long-poll: wait up to timeoutMillis for a message instead of returning
|
|
425
|
+
* empty at once. Default true for both pop() and consume().
|
|
426
|
+
*/
|
|
400
427
|
wait(enabled) {
|
|
401
428
|
this.#wait = enabled
|
|
402
429
|
return this
|
|
@@ -416,6 +443,24 @@ export class QueueBuilder {
|
|
|
416
443
|
return this
|
|
417
444
|
}
|
|
418
445
|
|
|
446
|
+
/**
|
|
447
|
+
* Commit the messages of pop() and popResult() at delivery.
|
|
448
|
+
*
|
|
449
|
+
* The broker moves the consumer group's cursor past the messages as it hands
|
|
450
|
+
* them out: no lease, nothing to ack (the messages come back with an empty
|
|
451
|
+
* leaseId). That is at-most-once: a crash after the pop loses them. The
|
|
452
|
+
* request carries autoAck=true, the parameter every 2.x broker reads.
|
|
453
|
+
*
|
|
454
|
+
* A pop option only: consume() always leases its messages, so it throws
|
|
455
|
+
* when this is set. The broker refuses it together with conflation() (400).
|
|
456
|
+
*
|
|
457
|
+
* @param {boolean} [enabled=true]
|
|
458
|
+
*/
|
|
459
|
+
commitOnDelivery(enabled = true) {
|
|
460
|
+
this.#commitOnDelivery = !!enabled
|
|
461
|
+
return this
|
|
462
|
+
}
|
|
463
|
+
|
|
419
464
|
/**
|
|
420
465
|
* Claim messages and report what the broker chose for this pop.
|
|
421
466
|
*
|
|
@@ -437,15 +482,17 @@ export class QueueBuilder {
|
|
|
437
482
|
}
|
|
438
483
|
|
|
439
484
|
async #popWithDecision() {
|
|
440
|
-
|
|
485
|
+
// For pop(), use POP defaults (not CONSUME defaults) for what the caller
|
|
486
|
+
// did not set. autoAck() is consume()'s ack after the handler and never
|
|
487
|
+
// reaches the wire; the broker's at-most-once autoAck travels only from
|
|
488
|
+
// commitOnDelivery(). Without it a pop comes back leased.
|
|
489
|
+
const effectiveWait = this.#wait ?? POP_DEFAULTS.wait
|
|
490
|
+
|
|
491
|
+
logger.log('QueueBuilder.pop', { queue: this.#queueName, partition: this.#partition, namespace: this.#namespace, task: this.#task, batch: this.#batch, wait: effectiveWait, group: this.#group })
|
|
441
492
|
|
|
442
493
|
try {
|
|
443
494
|
const path = this.#buildPopPath()
|
|
444
495
|
|
|
445
|
-
// For pop(), use POP defaults (not CONSUME defaults)
|
|
446
|
-
// Override autoAck to false unless explicitly set
|
|
447
|
-
const effectiveAutoAck = this.#autoAck !== CONSUME_DEFAULTS.autoAck ? this.#autoAck : POP_DEFAULTS.autoAck
|
|
448
|
-
|
|
449
496
|
// Batch, partitions and with them the autopilot flag. The RULE for which
|
|
450
497
|
// of the three travel lives in one place (utils/autopilot.js) because
|
|
451
498
|
// consume() builds its query string separately; only the PLACEMENT is
|
|
@@ -458,17 +505,18 @@ export class QueueBuilder {
|
|
|
458
505
|
autopilot: this.#autopilotEnabled()
|
|
459
506
|
})
|
|
460
507
|
|
|
461
|
-
// Build params with correct autoAck for pop
|
|
462
508
|
const params = new URLSearchParams()
|
|
463
509
|
if (sizing.autopilot) params.append('autopilot', 'true')
|
|
464
510
|
if (sizing.batch !== null) params.append('batch', sizing.batch)
|
|
465
|
-
params.append('wait',
|
|
511
|
+
params.append('wait', effectiveWait.toString())
|
|
466
512
|
params.append('timeout', this.#timeoutMillis.toString())
|
|
467
513
|
|
|
468
514
|
if (this.#group) params.append('consumerGroup', this.#group)
|
|
469
515
|
if (this.#namespace) params.append('namespace', this.#namespace)
|
|
470
516
|
if (this.#task) params.append('task', this.#task)
|
|
471
|
-
|
|
517
|
+
// Sent only when true, where earlier SDKs placed it: an absent autoAck
|
|
518
|
+
// is the broker's leased default.
|
|
519
|
+
if (this.#commitOnDelivery) params.append('autoAck', 'true')
|
|
472
520
|
if (this.#subscriptionMode) params.append('subscriptionMode', this.#subscriptionMode)
|
|
473
521
|
if (this.#subscriptionFrom) params.append('subscriptionFrom', this.#subscriptionFrom)
|
|
474
522
|
if (sizing.partitions !== null) params.append('partitions', sizing.partitions)
|
|
@@ -483,7 +531,7 @@ export class QueueBuilder {
|
|
|
483
531
|
|
|
484
532
|
// wait=true is a long-poll: on 429 it should back off and keep waiting
|
|
485
533
|
// rather than give up after a handful of tries (retryKind: 'pop').
|
|
486
|
-
const result = await this.#httpClient.get(`${path}?${params}`, this.#timeoutMillis + 5000, affinityKey,
|
|
534
|
+
const result = await this.#httpClient.get(`${path}?${params}`, this.#timeoutMillis + 5000, affinityKey, effectiveWait ? 'pop' : null)
|
|
487
535
|
|
|
488
536
|
// Degrade-loudly (PLAN_CONFLATION §4), BEFORE the empty-response return:
|
|
489
537
|
// an old broker's empty pop is a bodiless 204 (result === null), and that
|
|
@@ -517,8 +565,8 @@ export class QueueBuilder {
|
|
|
517
565
|
// messages right now"; for a declared conflation it would mean "your
|
|
518
566
|
// last-value policy is not in force and you will never be told", which is
|
|
519
567
|
// the silent failure the feature is not allowed to have (§4). Both the
|
|
520
|
-
// missing-echo error and the broker's 400 refusals (queue mode /
|
|
521
|
-
// are permanent config faults, so they raise.
|
|
568
|
+
// missing-echo error and the broker's 400 refusals (queue mode /
|
|
569
|
+
// commitOnDelivery) are permanent config faults, so they raise.
|
|
522
570
|
if (error.code === CONFLATION_UNSUPPORTED || (this.#conflation && error.status === 400)) {
|
|
523
571
|
logger.error('QueueBuilder.pop', { error: error.message, status: error.status, code: error.code, conflation: true })
|
|
524
572
|
throw error
|
|
@@ -550,17 +598,18 @@ export class QueueBuilder {
|
|
|
550
598
|
|
|
551
599
|
// NOTE: a second, DEAD copy of the pop parameter builder lived here and was
|
|
552
600
|
// deleted with the kv/timers work (PLAN_KV_TIMERS.md §10.4). pop() builds its
|
|
553
|
-
// own params inline, above, because it
|
|
554
|
-
//
|
|
555
|
-
// the method whose name says "build pop params" would have added
|
|
556
|
-
// copy nobody calls: the pop would keep working and the parameter
|
|
557
|
-
// simply never arrive, which reads as a server-side mystery and not as
|
|
558
|
-
// client bug.
|
|
601
|
+
// own params inline, above, because it uses the POP defaults and carries
|
|
602
|
+
// commitOnDelivery, which consume() refuses. Anyone adding a parameter by
|
|
603
|
+
// looking for the method whose name says "build pop params" would have added
|
|
604
|
+
// it to the copy nobody calls: the pop would keep working and the parameter
|
|
605
|
+
// would simply never arrive, which reads as a server-side mystery and not as
|
|
606
|
+
// a client bug.
|
|
559
607
|
//
|
|
560
608
|
// The pair that is still live and MUST be kept in sync is pop()'s inline
|
|
561
609
|
// params above and ConsumerManager#buildParams: every pop query parameter
|
|
562
610
|
// (subscriptionMode, subscriptionFrom, partitions, conflation, ...) has to be
|
|
563
|
-
// appended in BOTH, because pop() and consume() share no builder.
|
|
611
|
+
// appended in BOTH, because pop() and consume() share no builder. The one
|
|
612
|
+
// exception is commitOnDelivery's autoAck=true: consume() throws instead.
|
|
564
613
|
|
|
565
614
|
// ===========================
|
|
566
615
|
// Buffer Management Methods
|
|
@@ -211,20 +211,25 @@ export class ConsumerManager {
|
|
|
211
211
|
try {
|
|
212
212
|
// Process messages
|
|
213
213
|
if (each) {
|
|
214
|
-
// Process one at a time
|
|
214
|
+
// Process one at a time. A nack releases the failed message's
|
|
215
|
+
// partition and clamps that partition's cursor at it: the later
|
|
216
|
+
// messages of THAT partition will be redelivered, so handling them
|
|
217
|
+
// now would only produce duplicates and rejected acks. The other
|
|
218
|
+
// partitions of a multi-partition pop are still leased to this
|
|
219
|
+
// worker, so their messages are handled now, not after the lease.
|
|
220
|
+
const nackedPartitions = new Set()
|
|
215
221
|
for (const message of messages) {
|
|
216
222
|
if (signal && signal.aborted) break
|
|
217
223
|
|
|
224
|
+
const partition = message.partitionId ?? message.partition
|
|
225
|
+
if (nackedPartitions.has(partition)) continue
|
|
226
|
+
|
|
218
227
|
const ok = await this.#processMessage(message, handler, autoAck, group)
|
|
219
228
|
processedCount++
|
|
220
229
|
|
|
221
|
-
// A nack releases the lease and clamps the server cursor at the
|
|
222
|
-
// failed message: everything after it in this popped batch WILL
|
|
223
|
-
// be redelivered. Processing it now would only produce duplicates
|
|
224
|
-
// and rejected acks — abandon the rest of the batch.
|
|
225
230
|
if (!ok) {
|
|
226
|
-
|
|
227
|
-
|
|
231
|
+
nackedPartitions.add(partition)
|
|
232
|
+
logger.warn('ConsumerManager.worker', { workerId, status: 'partition-abandoned-after-nack', partition })
|
|
228
233
|
}
|
|
229
234
|
|
|
230
235
|
if (limit && processedCount >= limit) break
|
|
@@ -17,12 +17,13 @@
|
|
|
17
17
|
* history to have.
|
|
18
18
|
*
|
|
19
19
|
* DELIVERY IS NOT "AT MOST ONCE" (§1.3), and the docs must not say it is. The
|
|
20
|
-
* class picks what can be LOST; the ack mode picks the guarantee.
|
|
21
|
-
* advances the cursor at delivery and is at-most-once. The
|
|
22
|
-
* ack -- is at-least-once for as long as the owning broker
|
|
23
|
-
* an unacked message redelivers when its lease expires,
|
|
24
|
-
* incremented, until `retryLimit`, after which it is DROPPED
|
|
25
|
-
* DLQ, §9). Consumers still need idempotency, exactly as on
|
|
20
|
+
* class picks what can be LOST; the ack mode picks the guarantee.
|
|
21
|
+
* `commitOnDelivery` advances the cursor at delivery and is at-most-once. The
|
|
22
|
+
* default -- explicit ack -- is at-least-once for as long as the owning broker
|
|
23
|
+
* incarnation lives: an unacked message redelivers when its lease expires,
|
|
24
|
+
* with `attempts` incremented, until `retryLimit`, after which it is DROPPED
|
|
25
|
+
* and counted (no DLQ, §9). Consumers still need idempotency, exactly as on
|
|
26
|
+
* durable queues.
|
|
26
27
|
*
|
|
27
28
|
* CONSUMPTION SEMANTICS COME FROM THE GROUP, EXACTLY AS ON THE DURABLE ENGINE
|
|
28
29
|
* (§1.5). There is no queue-level mode to choose:
|
|
@@ -404,7 +405,23 @@ export class Ephemeral {
|
|
|
404
405
|
*
|
|
405
406
|
* `group` is the whole of the consumption semantics (§1.5): same group =
|
|
406
407
|
* competing consumers, own group = fan-out, no group = queue mode.
|
|
407
|
-
*
|
|
408
|
+
*
|
|
409
|
+
* `commitOnDelivery:true` commits at delivery: the broker moves the group's
|
|
410
|
+
* cursor past the messages as it hands them out, with no lease and nothing
|
|
411
|
+
* to ack. That is at-most-once: a crash after the pop loses them. It
|
|
412
|
+
* travels as `autoAck=true`, the parameter the broker reads.
|
|
413
|
+
*
|
|
414
|
+
* @param {string} queue
|
|
415
|
+
* @param {object} [opts]
|
|
416
|
+
* @param {string} [opts.partition]
|
|
417
|
+
* @param {number} [opts.batch]
|
|
418
|
+
* @param {boolean} [opts.wait]
|
|
419
|
+
* @param {number} [opts.timeout] Milliseconds; `timeoutMillis` is the same.
|
|
420
|
+
* @param {string} [opts.group]
|
|
421
|
+
* @param {boolean} [opts.commitOnDelivery] Commit at delivery (at-most-once).
|
|
422
|
+
* @param {boolean} [opts.autoAck] Deprecated: the old name of
|
|
423
|
+
* `commitOnDelivery`, still read with the same meaning. Pass one or the
|
|
424
|
+
* other, not both.
|
|
408
425
|
*/
|
|
409
426
|
async pop(queue, opts = {}) {
|
|
410
427
|
requireQueue(queue)
|
|
@@ -413,6 +430,7 @@ export class Ephemeral {
|
|
|
413
430
|
const group = opts.group ?? null
|
|
414
431
|
const wait = opts.wait === true
|
|
415
432
|
const timeoutMillis = resolveTimeout(opts)
|
|
433
|
+
const commitOnDelivery = resolveCommitOnDelivery(opts)
|
|
416
434
|
|
|
417
435
|
const params = new URLSearchParams({ queue })
|
|
418
436
|
if (partition !== null) params.append('partition', partition)
|
|
@@ -424,7 +442,7 @@ export class Ephemeral {
|
|
|
424
442
|
params.append('timeout', String(timeoutMillis))
|
|
425
443
|
}
|
|
426
444
|
if (group !== null) params.append('group', group)
|
|
427
|
-
if (
|
|
445
|
+
if (commitOnDelivery) params.append('autoAck', 'true')
|
|
428
446
|
|
|
429
447
|
logger.log('Ephemeral.pop', { queue, partition, group, batch: opts.batch ?? null, wait })
|
|
430
448
|
|
|
@@ -547,3 +565,17 @@ function resolveTimeout(opts) {
|
|
|
547
565
|
if (hasMillis) return opts.timeoutMillis
|
|
548
566
|
return DEFAULT_WAIT_TIMEOUT_MILLIS
|
|
549
567
|
}
|
|
568
|
+
|
|
569
|
+
/**
|
|
570
|
+
* The pop's commit at delivery, from `commitOnDelivery` or its deprecated
|
|
571
|
+
* alias `autoAck`. Both spellings at once are refused, as for the timeout: two
|
|
572
|
+
* values for one setting would have to be resolved by a rule nobody reads.
|
|
573
|
+
*/
|
|
574
|
+
function resolveCommitOnDelivery(opts) {
|
|
575
|
+
const hasName = opts.commitOnDelivery !== undefined && opts.commitOnDelivery !== null
|
|
576
|
+
const hasAlias = opts.autoAck !== undefined && opts.autoAck !== null
|
|
577
|
+
if (hasName && hasAlias) {
|
|
578
|
+
throw new Error('ephemeral: pass either `commitOnDelivery` or its deprecated alias `autoAck`, not both')
|
|
579
|
+
}
|
|
580
|
+
return (hasName ? opts.commitOnDelivery : opts.autoAck) === true
|
|
581
|
+
}
|
|
@@ -81,9 +81,10 @@ export const CONSUME_DEFAULTS = {
|
|
|
81
81
|
// As in CONSUME_DEFAULTS, batch is the autopilot-OFF default.
|
|
82
82
|
export const POP_DEFAULTS = {
|
|
83
83
|
batch: 1, // One message (autopilot off only)
|
|
84
|
-
wait:
|
|
84
|
+
wait: true, // Long polling, as every pop has done; .wait(false) returns at once
|
|
85
85
|
timeoutMillis: 30000, // 30 seconds if wait=true
|
|
86
|
-
autoAck: false
|
|
86
|
+
autoAck: false, // Never sent: autoAck() is consume()'s ack after the handler
|
|
87
|
+
commitOnDelivery: false // Leased; true sends autoAck=true, the broker's at-most-once commit at delivery
|
|
87
88
|
}
|
|
88
89
|
|
|
89
90
|
export const BUFFER_DEFAULTS = {
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "queen-mq",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Partitioned message queue on a replicated broker log — broker client + fluent streaming SDK (windows, joins, gates) in one package",
|
|
6
6
|
"main": "client-v2/index.js",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"test": "npm run test:unit && node test-v2/run.js human",
|
|
9
|
-
"test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js",
|
|
9
|
+
"test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/consumer-unit/nackScope.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js test-v2/pop-unit/popDefaults.test.js test-v2/admin-unit/removedRoutes.test.js",
|
|
10
10
|
"test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
|
|
11
11
|
"test:integration": "node test-v2/run.js human",
|
|
12
12
|
"test:streams": "node test-v2/run.js stream",
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Admin methods whose route the 2.x broker does not have.
|
|
3
|
+
*
|
|
4
|
+
* clearQueue() sent DELETE /api/v1/queues/:name/clear and moveMessageToDLQ()
|
|
5
|
+
* sent POST /api/v1/messages/:partitionId/:transactionId/dlq. Neither route is
|
|
6
|
+
* registered (server/src/rsm/facade/real/phase2/reads.rs: the messages family
|
|
7
|
+
* is GET, DELETE and POST .../retry; there is no /api/v1/queues/... family), so
|
|
8
|
+
* a 2.x broker answers both with 404 no_such_route, and the caller saw
|
|
9
|
+
* "not found". Both now throw before any request, naming the way that works.
|
|
10
|
+
*
|
|
11
|
+
* Same style as conflation-unit/conflationWire.test.js: a real node:http
|
|
12
|
+
* server records every request, so "before any request" is asserted on the
|
|
13
|
+
* socket, not on a mock.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { describe, it } from 'node:test'
|
|
17
|
+
import assert from 'node:assert/strict'
|
|
18
|
+
|
|
19
|
+
import { Queen } from '../../client-v2/index.js'
|
|
20
|
+
import { withPlanServer } from '../kv-unit/_planServer.js'
|
|
21
|
+
|
|
22
|
+
const notFound = { status: 404, body: { code: 'no_such_route', error: 'not found' } }
|
|
23
|
+
|
|
24
|
+
async function withQueen(run) {
|
|
25
|
+
await withPlanServer([], notFound, async (url, hits) => {
|
|
26
|
+
const queen = new Queen({ url, handleSignals: false })
|
|
27
|
+
try {
|
|
28
|
+
await run(queen, hits)
|
|
29
|
+
} finally {
|
|
30
|
+
await queen.close()
|
|
31
|
+
}
|
|
32
|
+
})
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
describe('Admin — routes the 2.x broker does not have', () => {
|
|
36
|
+
it('moveMessageToDLQ() throws before any request and names the dlq ack', async () => {
|
|
37
|
+
await withQueen(async (queen, hits) => {
|
|
38
|
+
await assert.rejects(
|
|
39
|
+
queen.admin.moveMessageToDLQ('7', 'tx-1'),
|
|
40
|
+
(err) => {
|
|
41
|
+
assert.match(err.message, /no route/)
|
|
42
|
+
assert.match(err.message, /queen\.ack\(message, 'dlq', \{ group \}\)/)
|
|
43
|
+
return true
|
|
44
|
+
}
|
|
45
|
+
)
|
|
46
|
+
assert.equal(hits.length, 0, 'no request was sent')
|
|
47
|
+
})
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
it('clearQueue() throws before any request and names the seek to the end', async () => {
|
|
51
|
+
await withQueen(async (queen, hits) => {
|
|
52
|
+
await assert.rejects(
|
|
53
|
+
queen.admin.clearQueue('orders', 'p1'),
|
|
54
|
+
(err) => {
|
|
55
|
+
assert.match(err.message, /no route/)
|
|
56
|
+
assert.match(err.message, /seekConsumerGroup\(group, 'orders', \{ toEnd: true \}\)/)
|
|
57
|
+
return true
|
|
58
|
+
}
|
|
59
|
+
)
|
|
60
|
+
assert.equal(hits.length, 0, 'no request was sent')
|
|
61
|
+
})
|
|
62
|
+
})
|
|
63
|
+
})
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* each(): a nack releases ONE partition. A multi-partition pop claims several
|
|
3
|
+
* partitions under one lease; when the handler fails a message, the nack
|
|
4
|
+
* releases that message's partition and clamps its cursor, so the later
|
|
5
|
+
* messages of THAT partition come back on the next pop. The other partitions
|
|
6
|
+
* are still leased to this worker: their messages must be handled now.
|
|
7
|
+
*
|
|
8
|
+
* Before: the loop abandoned the whole popped batch after a nack. The other
|
|
9
|
+
* partitions' messages stayed leased and came back only when the lease
|
|
10
|
+
* expired (found live 2026-10-06 against 2.0.1: B1 and B2 waited the whole
|
|
11
|
+
* 6 s lease after A1 failed).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { describe, it } from 'node:test'
|
|
15
|
+
import assert from 'node:assert/strict'
|
|
16
|
+
import { createServer } from 'node:http'
|
|
17
|
+
|
|
18
|
+
import { Queen } from '../../client-v2/index.js'
|
|
19
|
+
|
|
20
|
+
const GROUP = 'workers'
|
|
21
|
+
|
|
22
|
+
const message = (partition, n) => ({
|
|
23
|
+
id: `msg-${partition}${n}`,
|
|
24
|
+
transactionId: `tx-${partition}${n}`,
|
|
25
|
+
partitionId: `pid-${partition}`,
|
|
26
|
+
partition,
|
|
27
|
+
leaseId: 'lease-1',
|
|
28
|
+
consumerGroup: GROUP,
|
|
29
|
+
data: { tag: `${partition}${n}` },
|
|
30
|
+
createdAt: '2026-10-06T10:00:00.000Z'
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
async function withBroker(pops, run) {
|
|
34
|
+
const queue = [...pops]
|
|
35
|
+
const requests = []
|
|
36
|
+
const server = createServer((req, res) => {
|
|
37
|
+
let raw = ''
|
|
38
|
+
req.on('data', chunk => { raw += chunk })
|
|
39
|
+
req.on('end', () => {
|
|
40
|
+
const body = raw ? JSON.parse(raw) : null
|
|
41
|
+
const path = req.url.split('?')[0]
|
|
42
|
+
requests.push({ method: req.method, path, body })
|
|
43
|
+
if (req.method === 'GET' && path.startsWith('/api/v1/pop')) {
|
|
44
|
+
const batch = queue.shift()
|
|
45
|
+
if (!batch) { res.writeHead(204); res.end(); return }
|
|
46
|
+
res.writeHead(200, { 'Content-Type': 'application/json' })
|
|
47
|
+
res.end(JSON.stringify({ success: true, consumerGroup: GROUP, messages: batch }))
|
|
48
|
+
return
|
|
49
|
+
}
|
|
50
|
+
if (req.method === 'POST' && (path === '/api/v1/ack' || path === '/api/v1/ack/batch')) {
|
|
51
|
+
const acks = body.acknowledgments || [body]
|
|
52
|
+
res.writeHead(200, { 'Content-Type': 'application/json' })
|
|
53
|
+
res.end(JSON.stringify(acks.map((a, i) => ({ index: i, transactionId: a.transactionId, success: true, error: null }))))
|
|
54
|
+
return
|
|
55
|
+
}
|
|
56
|
+
res.writeHead(404, { 'Content-Type': 'application/json' })
|
|
57
|
+
res.end('{"error":"not found"}')
|
|
58
|
+
})
|
|
59
|
+
})
|
|
60
|
+
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
|
|
61
|
+
const queen = new Queen({ url: `http://127.0.0.1:${server.address().port}`, handleSignals: false })
|
|
62
|
+
try {
|
|
63
|
+
await run(queen, requests)
|
|
64
|
+
} finally {
|
|
65
|
+
await queen.close()
|
|
66
|
+
await new Promise(resolve => server.close(resolve))
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
describe('each(): a nack skips only its own partition', () => {
|
|
71
|
+
it('handles the other partitions of the pop after a failure', async () => {
|
|
72
|
+
const pop = [message('A', 1), message('A', 2), message('B', 1), message('B', 2)]
|
|
73
|
+
await withBroker([pop], async (queen, requests) => {
|
|
74
|
+
const handled = []
|
|
75
|
+
await queen.queue('orders').group(GROUP).each().batch(4).limit(3).idleMillis(500)
|
|
76
|
+
.consume(async (m) => {
|
|
77
|
+
handled.push(m.data.tag)
|
|
78
|
+
if (m.data.tag === 'A1') throw new Error('A1 fails')
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
assert.deepEqual(handled, ['A1', 'B1', 'B2'], 'A2 is skipped (it comes back after the nack), B is handled now')
|
|
82
|
+
const settled = requests.filter(r => r.path.startsWith('/api/v1/ack'))
|
|
83
|
+
.flatMap(r => r.body.acknowledgments || [r.body])
|
|
84
|
+
.map(a => `${a.transactionId}:${a.status}`)
|
|
85
|
+
assert.deepEqual(settled, ['tx-A1:failed', 'tx-B1:completed', 'tx-B2:completed'])
|
|
86
|
+
})
|
|
87
|
+
})
|
|
88
|
+
})
|
|
@@ -201,7 +201,7 @@ describe('ephemeral wire — pop', () => {
|
|
|
201
201
|
|
|
202
202
|
it('pop puts every declared parameter on the query string, in order', async () => {
|
|
203
203
|
await withEphemeral([popped(QUEUE, [])], async (eph, hits) => {
|
|
204
|
-
await eph.pop(QUEUE, { partition: 'room-7', batch: 10, wait: true, timeout: 2000, group: 'workers',
|
|
204
|
+
await eph.pop(QUEUE, { partition: 'room-7', batch: 10, wait: true, timeout: 2000, group: 'workers', commitOnDelivery: true })
|
|
205
205
|
assert.equal(
|
|
206
206
|
hits[0].url,
|
|
207
207
|
`/api/v1/ephemeral/pop?queue=${QUEUE}&partition=room-7&batch=10&wait=true&timeout=2000&group=workers&autoAck=true`
|
|
@@ -209,6 +209,29 @@ describe('ephemeral wire — pop', () => {
|
|
|
209
209
|
})
|
|
210
210
|
})
|
|
211
211
|
|
|
212
|
+
it('pop sends `commitOnDelivery` as autoAck=true, and nothing for false', async () => {
|
|
213
|
+
await withEphemeral([popped(QUEUE, []), popped(QUEUE, [])], async (eph, hits) => {
|
|
214
|
+
await eph.pop(QUEUE, { commitOnDelivery: true })
|
|
215
|
+
assert.equal(hits[0].url, `/api/v1/ephemeral/pop?queue=${QUEUE}&autoAck=true`)
|
|
216
|
+
|
|
217
|
+
await eph.pop(QUEUE, { commitOnDelivery: false })
|
|
218
|
+
assert.equal(hits[1].url, `/api/v1/ephemeral/pop?queue=${QUEUE}`)
|
|
219
|
+
})
|
|
220
|
+
})
|
|
221
|
+
|
|
222
|
+
it('pop still reads the deprecated `autoAck` as `commitOnDelivery`, and refuses both spellings', async () => {
|
|
223
|
+
await withEphemeral([popped(QUEUE, [])], async (eph, hits) => {
|
|
224
|
+
await eph.pop(QUEUE, { group: 'workers', autoAck: true })
|
|
225
|
+
assert.equal(hits[0].url, `/api/v1/ephemeral/pop?queue=${QUEUE}&group=workers&autoAck=true`)
|
|
226
|
+
|
|
227
|
+
await assert.rejects(
|
|
228
|
+
() => eph.pop(QUEUE, { commitOnDelivery: true, autoAck: true }),
|
|
229
|
+
/pass either `commitOnDelivery` or its deprecated alias `autoAck`, not both/
|
|
230
|
+
)
|
|
231
|
+
assert.equal(hits.length, 1)
|
|
232
|
+
})
|
|
233
|
+
})
|
|
234
|
+
|
|
212
235
|
it('pop sends an explicit timeout whenever it waits, and none when it does not', async () => {
|
|
213
236
|
// The HTTP deadline is set PAST the server's, so the broker's long poll
|
|
214
237
|
// always ends the request first. Letting the two defaults drift apart is
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* pop() and consume() share one builder but not their defaults.
|
|
3
|
+
*
|
|
4
|
+
* autoAck commitOnDelivery wait
|
|
5
|
+
* pop() never sent off; on sends autoAck true (POP_DEFAULTS)
|
|
6
|
+
* consume true, client-side refused true (CONSUME_DEFAULTS)
|
|
7
|
+
*
|
|
8
|
+
* autoAck() is consume()'s ack after the handler and never reaches the wire.
|
|
9
|
+
* The broker's at-most-once commit at delivery is the pop's own option,
|
|
10
|
+
* commitOnDelivery(), which sends autoAck=true; consume() always leases, so it
|
|
11
|
+
* refuses that option before any request.
|
|
12
|
+
* POP_DEFAULTS.wait said false while every pop long-polled; the long poll is
|
|
13
|
+
* what callers rely on, so it stays, and POP_DEFAULTS and the guide say so.
|
|
14
|
+
*
|
|
15
|
+
* The broker contract (server/src/handlers/data.rs, PopParams): autoAck=true
|
|
16
|
+
* commits the messages at delivery, with an empty leaseId; an absent autoAck
|
|
17
|
+
* is false. An absent wait is false too, but this SDK always sends wait.
|
|
18
|
+
*
|
|
19
|
+
* Same style as conflation-unit/conflationWire.test.js: a real node:http
|
|
20
|
+
* server playing a canned plan, real fetch, and every assertion is about the
|
|
21
|
+
* bytes that crossed the socket.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { describe, it } from 'node:test'
|
|
25
|
+
import assert from 'node:assert/strict'
|
|
26
|
+
|
|
27
|
+
import { Queen } from '../../client-v2/index.js'
|
|
28
|
+
import { withPlanServer, ok } from '../kv-unit/_planServer.js'
|
|
29
|
+
|
|
30
|
+
const QUEUE = 'q'
|
|
31
|
+
const GROUP = 'g'
|
|
32
|
+
|
|
33
|
+
const frame = (n = 1) => ({
|
|
34
|
+
transactionId: `txn-${n}`,
|
|
35
|
+
partitionId: `part-${n}`,
|
|
36
|
+
partition: 'Default',
|
|
37
|
+
payload: { n },
|
|
38
|
+
leaseId: 'lease-1',
|
|
39
|
+
consumerGroup: GROUP
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
const popBody = (frames = [frame()]) => ok({ messages: frames, partitionsClaimed: frames.length })
|
|
43
|
+
|
|
44
|
+
// Acks answer with one result per acknowledgment; the plan only needs enough
|
|
45
|
+
// of them for the consume() cases below.
|
|
46
|
+
const ackBody = ok([{ index: 0, transactionId: 'txn-1', success: true, error: null }])
|
|
47
|
+
|
|
48
|
+
function query(url) {
|
|
49
|
+
const i = url.indexOf('?')
|
|
50
|
+
return new URLSearchParams(i < 0 ? '' : url.slice(i + 1))
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const pops = (hits) => hits.filter(h => h.method === 'GET' && h.url.startsWith('/api/v1/pop'))
|
|
54
|
+
const acks = (hits) => hits.filter(h => h.method === 'POST' && h.url.startsWith('/api/v1/ack'))
|
|
55
|
+
const statusesOf = (hit) => (hit.body.acknowledgments || [hit.body]).map(a => a.status)
|
|
56
|
+
|
|
57
|
+
async function withQueen(plan, defaultResponse, run) {
|
|
58
|
+
await withPlanServer(plan, defaultResponse, async (url, hits) => {
|
|
59
|
+
const queen = new Queen({ url, handleSignals: false })
|
|
60
|
+
try {
|
|
61
|
+
await run(queen, hits)
|
|
62
|
+
} finally {
|
|
63
|
+
await queen.close()
|
|
64
|
+
}
|
|
65
|
+
})
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
describe('pop() — autoAck', () => {
|
|
69
|
+
// autoAck() is consume()'s ack after the handler and never reaches the
|
|
70
|
+
// wire: a pop comes back leased whatever autoAck() said. The broker's
|
|
71
|
+
// at-most-once autoAck is commitOnDelivery(), below.
|
|
72
|
+
for (const [label, build] of [
|
|
73
|
+
['autoAck() never called', (q) => q.group(GROUP)],
|
|
74
|
+
['autoAck(true)', (q) => q.group(GROUP).autoAck(true)],
|
|
75
|
+
['autoAck(false)', (q) => q.group(GROUP).autoAck(false)],
|
|
76
|
+
]) {
|
|
77
|
+
it(`sends no autoAck after ${label}`, async () => {
|
|
78
|
+
await withQueen([popBody(), popBody()], popBody(), async (queen, hits) => {
|
|
79
|
+
await build(queen.queue(QUEUE)).pop()
|
|
80
|
+
await build(queen.queue(QUEUE)).popResult()
|
|
81
|
+
|
|
82
|
+
for (const hit of pops(hits)) {
|
|
83
|
+
assert.equal(query(hit.url).has('autoAck'), false)
|
|
84
|
+
}
|
|
85
|
+
})
|
|
86
|
+
})
|
|
87
|
+
}
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
describe('pop() — commitOnDelivery', () => {
|
|
91
|
+
// The broker's commit at delivery, on the wire name every 2.x broker reads:
|
|
92
|
+
// the pop carries autoAck=true and nothing else changes.
|
|
93
|
+
it('pop() and popResult() send autoAck=true and nothing else', async () => {
|
|
94
|
+
await withQueen([popBody(), popBody(), popBody()], popBody(), async (queen, hits) => {
|
|
95
|
+
await queen.queue(QUEUE).group(GROUP).pop()
|
|
96
|
+
await queen.queue(QUEUE).group(GROUP).commitOnDelivery().pop()
|
|
97
|
+
await queen.queue(QUEUE).group(GROUP).commitOnDelivery(true).popResult()
|
|
98
|
+
|
|
99
|
+
const [plain, ...committed] = pops(hits)
|
|
100
|
+
assert.equal(committed.length, 2)
|
|
101
|
+
for (const hit of committed) {
|
|
102
|
+
const q = query(hit.url)
|
|
103
|
+
assert.equal(q.get('autoAck'), 'true')
|
|
104
|
+
q.delete('autoAck')
|
|
105
|
+
assert.deepEqual(Object.fromEntries(q), Object.fromEntries(query(plain.url)))
|
|
106
|
+
assert.equal(hit.url.split('?')[0], plain.url.split('?')[0])
|
|
107
|
+
}
|
|
108
|
+
})
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
it('sends no autoAck after commitOnDelivery(false)', async () => {
|
|
112
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
113
|
+
await queen.queue(QUEUE).group(GROUP).commitOnDelivery(true).commitOnDelivery(false).pop()
|
|
114
|
+
|
|
115
|
+
assert.equal(query(pops(hits)[0].url).has('autoAck'), false)
|
|
116
|
+
})
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
it('does not depend on autoAck(), which stays consume()\'s', async () => {
|
|
120
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
121
|
+
await queen.queue(QUEUE).group(GROUP).autoAck(false).commitOnDelivery().pop()
|
|
122
|
+
|
|
123
|
+
assert.equal(query(pops(hits)[0].url).get('autoAck'), 'true')
|
|
124
|
+
})
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
it('with conflation(), sends both and raises the broker\'s 400 instead of returning []', async () => {
|
|
128
|
+
// The broker refuses conflation together with a commit at delivery
|
|
129
|
+
// (server/src/handlers/data.rs, conflation_refusal).
|
|
130
|
+
const refusal = {
|
|
131
|
+
status: 400,
|
|
132
|
+
body: { success: false, error: 'conflation cannot be combined with autoAck', messages: [] }
|
|
133
|
+
}
|
|
134
|
+
await withQueen([refusal], refusal, async (queen, hits) => {
|
|
135
|
+
await assert.rejects(
|
|
136
|
+
() => queen.queue(QUEUE).group(GROUP).conflation().commitOnDelivery().pop(),
|
|
137
|
+
(err) => err.status === 400
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
const q = query(pops(hits)[0].url)
|
|
141
|
+
assert.equal(q.get('autoAck'), 'true')
|
|
142
|
+
assert.equal(q.get('conflation'), 'true')
|
|
143
|
+
})
|
|
144
|
+
})
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
describe('consume() refuses commitOnDelivery()', () => {
|
|
148
|
+
it('throws before any request', async () => {
|
|
149
|
+
await withQueen([], popBody(), async (queen, hits) => {
|
|
150
|
+
assert.throws(
|
|
151
|
+
() => queen.queue(QUEUE).group(GROUP).commitOnDelivery().consume(async () => {}),
|
|
152
|
+
{
|
|
153
|
+
message: 'commitOnDelivery() is a pop() option; consume() always leases its messages'
|
|
154
|
+
}
|
|
155
|
+
)
|
|
156
|
+
assert.equal(hits.length, 0)
|
|
157
|
+
})
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
it('consumes as before after commitOnDelivery(false)', async () => {
|
|
161
|
+
await withQueen([popBody(), ackBody], ackBody, async (queen, hits) => {
|
|
162
|
+
await queen.queue(QUEUE).group(GROUP).commitOnDelivery(false).limit(1).consume(async () => {})
|
|
163
|
+
|
|
164
|
+
assert.equal(query(pops(hits)[0].url).has('autoAck'), false)
|
|
165
|
+
assert.equal(acks(hits).length, 1)
|
|
166
|
+
})
|
|
167
|
+
})
|
|
168
|
+
})
|
|
169
|
+
|
|
170
|
+
describe('pop() — wait', () => {
|
|
171
|
+
it('long-polls when wait() was never called', async () => {
|
|
172
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
173
|
+
await queen.queue(QUEUE).group(GROUP).pop()
|
|
174
|
+
|
|
175
|
+
assert.equal(query(pops(hits)[0].url).get('wait'), 'true')
|
|
176
|
+
})
|
|
177
|
+
})
|
|
178
|
+
|
|
179
|
+
it('long-polls for timeoutMillis() when only that was called', async () => {
|
|
180
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
181
|
+
await queen.queue(QUEUE).timeoutMillis(2000).pop()
|
|
182
|
+
|
|
183
|
+
const q = query(pops(hits)[0].url)
|
|
184
|
+
assert.equal(q.get('wait'), 'true')
|
|
185
|
+
assert.equal(q.get('timeout'), '2000')
|
|
186
|
+
})
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
it('sends wait=false after wait(false)', async () => {
|
|
190
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
191
|
+
await queen.queue(QUEUE).wait(false).pop()
|
|
192
|
+
|
|
193
|
+
assert.equal(query(pops(hits)[0].url).get('wait'), 'false')
|
|
194
|
+
})
|
|
195
|
+
})
|
|
196
|
+
})
|
|
197
|
+
|
|
198
|
+
describe('consume() keeps its own defaults', () => {
|
|
199
|
+
it('long-polls and never sends autoAck, also after autoAck(true): it acks after the handler', async () => {
|
|
200
|
+
await withQueen([popBody(), ackBody], ackBody, async (queen, hits) => {
|
|
201
|
+
await queen.queue(QUEUE).group(GROUP).autoAck(true).limit(1)
|
|
202
|
+
.consume(async () => {})
|
|
203
|
+
|
|
204
|
+
const q = query(pops(hits)[0].url)
|
|
205
|
+
assert.equal(q.get('wait'), 'true')
|
|
206
|
+
assert.equal(q.has('autoAck'), false)
|
|
207
|
+
assert.equal(acks(hits).length, 1, 'the consumer acked after the handler returned')
|
|
208
|
+
assert.deepEqual(statusesOf(acks(hits)[0]), ['completed'])
|
|
209
|
+
})
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
it('acks after the handler when autoAck() was never called', async () => {
|
|
213
|
+
await withQueen([popBody(), ackBody], ackBody, async (queen, hits) => {
|
|
214
|
+
await queen.queue(QUEUE).group(GROUP).limit(1).consume(async () => {})
|
|
215
|
+
|
|
216
|
+
assert.equal(query(pops(hits)[0].url).get('wait'), 'true')
|
|
217
|
+
assert.equal(acks(hits).length, 1)
|
|
218
|
+
})
|
|
219
|
+
})
|
|
220
|
+
|
|
221
|
+
it('sends no ack after autoAck(false)', async () => {
|
|
222
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
223
|
+
await queen.queue(QUEUE).group(GROUP).autoAck(false).limit(1).consume(async () => {})
|
|
224
|
+
|
|
225
|
+
assert.equal(acks(hits).length, 0)
|
|
226
|
+
})
|
|
227
|
+
})
|
|
228
|
+
|
|
229
|
+
it('sends wait=false after wait(false)', async () => {
|
|
230
|
+
await withQueen([popBody(), ackBody], ackBody, async (queen, hits) => {
|
|
231
|
+
await queen.queue(QUEUE).group(GROUP).wait(false).limit(1).consume(async () => {})
|
|
232
|
+
|
|
233
|
+
assert.equal(query(pops(hits)[0].url).get('wait'), 'false')
|
|
234
|
+
})
|
|
235
|
+
})
|
|
236
|
+
})
|
package/test-v2/pop.js
CHANGED
|
@@ -361,3 +361,71 @@ export async function renewReportsAReleasedLease(client) {
|
|
|
361
361
|
released.success === false && released.newExpiresAt === null && typeof released.error === 'string'
|
|
362
362
|
return { success, message: `live lease: ${JSON.stringify(live)}; after the ack: ${JSON.stringify(released)}` }
|
|
363
363
|
}
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
// autoAck() is consume()'s ack after the handler and never reaches the wire: a
|
|
367
|
+
// pop after autoAck(true) is leased like any pop, and with a 1 s lease the
|
|
368
|
+
// message comes back. The broker's at-most-once autoAck is commitOnDelivery().
|
|
369
|
+
export async function popAutoAckStaysLeased(client) {
|
|
370
|
+
const queueName = 'test-queue-v2-pop-auto-ack'
|
|
371
|
+
const queue = await client.queue(queueName).config({ leaseTime: 1 }).create()
|
|
372
|
+
if (!queue.configured) {
|
|
373
|
+
return { success: false, message: 'Queue not created' }
|
|
374
|
+
}
|
|
375
|
+
await client.queue(queueName).push([{ data: { n: 1 } }])
|
|
376
|
+
|
|
377
|
+
const [message] = await client.queue(queueName).batch(1).wait(true).timeoutMillis(5000).autoAck(true).pop()
|
|
378
|
+
if (!message) {
|
|
379
|
+
return { success: false, message: 'Nothing popped' }
|
|
380
|
+
}
|
|
381
|
+
// Past the 1 s lease: a leased message is delivered again now.
|
|
382
|
+
await new Promise(resolve => setTimeout(resolve, 2500))
|
|
383
|
+
const again = await client.queue(queueName).batch(1).wait(true).timeoutMillis(3000).pop()
|
|
384
|
+
|
|
385
|
+
const success = typeof message.leaseId === 'string' && message.leaseId !== '' && again.length === 1
|
|
386
|
+
return { success, message: `leaseId ${JSON.stringify(message.leaseId)}, delivered again after the lease: ${again.length}` }
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// commitOnDelivery() sends autoAck=true: the broker moves the group's cursor
|
|
390
|
+
// past the message as it hands it out, so the pop has no lease and the message
|
|
391
|
+
// does not come back after the 1 s lease of the queue.
|
|
392
|
+
export async function popCommitOnDeliveryNotRedelivered(client) {
|
|
393
|
+
const queueName = 'test-queue-v2-pop-commit-on-delivery'
|
|
394
|
+
const queue = await client.queue(queueName).config({ leaseTime: 1 }).create()
|
|
395
|
+
if (!queue.configured) {
|
|
396
|
+
return { success: false, message: 'Queue not created' }
|
|
397
|
+
}
|
|
398
|
+
await client.queue(queueName).push([{ data: { n: 1 } }])
|
|
399
|
+
|
|
400
|
+
const [message] = await client.queue(queueName).batch(1).wait(true).timeoutMillis(5000).commitOnDelivery().pop()
|
|
401
|
+
if (!message) {
|
|
402
|
+
return { success: false, message: 'Nothing popped' }
|
|
403
|
+
}
|
|
404
|
+
// Past the 1 s lease: a leased message would be delivered again now.
|
|
405
|
+
await new Promise(resolve => setTimeout(resolve, 2500))
|
|
406
|
+
const again = await client.queue(queueName).batch(1).wait(true).timeoutMillis(1500).pop()
|
|
407
|
+
|
|
408
|
+
const success = !message.leaseId && again.length === 0
|
|
409
|
+
return { success, message: `leaseId ${JSON.stringify(message.leaseId)}, delivered again after the lease: ${again.length}` }
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
// pop() long-polls by default (POP_DEFAULTS.wait), and wait(false) returns at
|
|
413
|
+
// once.
|
|
414
|
+
export async function popLongPollsByDefault(client) {
|
|
415
|
+
const queueName = 'test-queue-v2-pop-wait-default'
|
|
416
|
+
const queue = await client.queue(queueName).create()
|
|
417
|
+
if (!queue.configured) {
|
|
418
|
+
return { success: false, message: 'Queue not created' }
|
|
419
|
+
}
|
|
420
|
+
let started = Date.now()
|
|
421
|
+
const waited = await client.queue(queueName).batch(1).timeoutMillis(1500).pop()
|
|
422
|
+
const waitedMillis = Date.now() - started
|
|
423
|
+
started = Date.now()
|
|
424
|
+
const atOnce = await client.queue(queueName).batch(1).wait(false).pop()
|
|
425
|
+
const atOnceMillis = Date.now() - started
|
|
426
|
+
|
|
427
|
+
return {
|
|
428
|
+
success: waited.length === 0 && waitedMillis >= 1000 && atOnce.length === 0 && atOnceMillis < 1000,
|
|
429
|
+
message: `empty pop: ${waitedMillis} ms by default (long poll of 1500 ms), ${atOnceMillis} ms with wait(false)`,
|
|
430
|
+
}
|
|
431
|
+
}
|