queen-mq 1.3.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -21
- package/client-v2/Queen.js +47 -15
- package/client-v2/README.md +34 -13
- package/client-v2/admin/Admin.js +30 -55
- package/client-v2/builders/QueueBuilder.js +106 -34
- package/client-v2/builders/TimerBuilder.js +2 -2
- package/client-v2/builders/TransactionBuilder.js +35 -9
- package/client-v2/consumer/ConsumerManager.js +79 -53
- package/client-v2/ephemeral/Ephemeral.js +47 -17
- package/client-v2/kv/Kv.js +4 -4
- package/client-v2/kv/expiry.js +1 -1
- package/client-v2/streams/Stream.js +8 -3
- package/client-v2/streams/helpers/rateLimiter.js +4 -4
- package/client-v2/streams/operators/GateOperator.js +9 -2
- package/client-v2/streams/operators/ReduceOperator.js +2 -2
- package/client-v2/streams/operators/WindowSessionOperator.js +3 -3
- package/client-v2/streams/runtime/Runner.js +111 -54
- package/client-v2/streams/runtime/cycle.js +4 -4
- package/client-v2/streams/runtime/register.js +1 -1
- package/client-v2/utils/conflation.js +0 -6
- package/client-v2/utils/consumerGroup.js +54 -0
- package/client-v2/utils/defaults.js +3 -2
- package/package.json +5 -8
- package/test-v2/_kvtimers.js +12 -13
- package/test-v2/ackwindow.js +12 -192
- package/test-v2/admin-unit/removedRoutes.test.js +63 -0
- package/test-v2/bootstrap.js +2 -2
- package/test-v2/conflation-unit/conflationWire.test.js +0 -12
- package/test-v2/consume.js +41 -1
- package/test-v2/consumer-unit/handlerError.test.js +161 -0
- package/test-v2/consumer-unit/nackScope.test.js +88 -0
- package/test-v2/docs.js +5 -4
- package/test-v2/ephemeral-unit/ephemeralWire.test.js +24 -1
- package/test-v2/http-unit/pushStatus.test.js +142 -0
- package/test-v2/http-unit/renew.test.js +96 -0
- package/test-v2/kv-unit/timerWire.test.js +1 -1
- package/test-v2/kv-unit/txnWire.test.js +101 -2
- package/test-v2/kv.js +6 -5
- package/test-v2/pop-unit/popDefaults.test.js +236 -0
- package/test-v2/pop.js +96 -1
- package/test-v2/run.js +50 -114
- package/test-v2/runner-unit/fatalExit.test.js +64 -0
- package/test-v2/semantics.js +20 -34
- package/test-v2/stream/_helpers.js +10 -19
- package/test-v2/stream/cron.js +1 -1
- package/test-v2/stream/gate.js +54 -0
- package/test-v2/stream/index.js +2 -0
- package/test-v2/stream/tumbling.js +3 -3
- package/test-v2/streams-unit/ack.test.js +61 -0
- package/test-v2/streams-unit/cycle.test.js +1 -1
- package/test-v2/streams-unit/e2e.test.js +6 -10
- package/test-v2/streams-unit/gate.test.js +189 -0
- package/test-v2/timers.js +6 -5
- package/test-v2/transaction.js +169 -0
- package/test-v2/watermark.js +38 -176
- package/test-v2/maintenance.js +0 -277
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
[](https://www.npmjs.com/package/queen-mq)
|
|
8
8
|
[](LICENSE.md)
|
|
9
|
-
[](https://nodejs.org/)
|
|
10
10
|
|
|
11
11
|
[Quick Start](#quick-start) • [Complete Guide](client-v2/README.md) • [Examples](#examples) • [API Reference](#api-reference)
|
|
12
12
|
|
|
@@ -16,13 +16,13 @@
|
|
|
16
16
|
|
|
17
17
|
## What is Queen MQ?
|
|
18
18
|
|
|
19
|
-
Queen MQ is a
|
|
19
|
+
Queen MQ is a partitioned message queue broker that keeps its state in its own replicated log, with a powerful feature set:
|
|
20
20
|
|
|
21
21
|
- **FIFO Partitions** - Unlimited ordered partitions within queues
|
|
22
22
|
- **Consumer Groups** - Kafka-style consumer groups for scalability
|
|
23
23
|
- **Flexible Semantics** - Exactly-once, at-least-once, and at-most-once delivery
|
|
24
24
|
- **Transactions** - Atomic operations across push and ack
|
|
25
|
-
- **High Performance** —
|
|
25
|
+
- **High Performance** — 1M msg/s in and out of one queue of 10M partitions on three nodes ([benchmarks](https://queenmq.com/benchmarks/))
|
|
26
26
|
- **Subscription Modes** - Process from beginning, new messages only, or from timestamp
|
|
27
27
|
- **Dead Letter Queue** - Automatic failure handling and monitoring
|
|
28
28
|
- **Message Tracing** - Debug distributed workflows with trace timelines
|
|
@@ -40,7 +40,7 @@ This client provides a fluent, promise-based API for Node.js applications.
|
|
|
40
40
|
npm install queen-mq
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
-
**Requirements:** Node.js
|
|
43
|
+
**Requirements:** Node.js 24+
|
|
44
44
|
|
|
45
45
|
---
|
|
46
46
|
|
|
@@ -133,20 +133,21 @@ await queen.queue('emails')
|
|
|
133
133
|
|
|
134
134
|
### Subscription Modes
|
|
135
135
|
|
|
136
|
-
Control whether consumer groups process historical messages
|
|
136
|
+
Control whether consumer groups process historical messages. A group's mode is fixed the first
|
|
137
|
+
time the group pops a queue; unset, it is the broker's `DEFAULT_SUBSCRIPTION_MODE`, which is `new`.
|
|
137
138
|
|
|
138
139
|
```javascript
|
|
139
|
-
// Default:
|
|
140
|
-
await queen.queue('events')
|
|
141
|
-
.group('batch-analytics')
|
|
142
|
-
.consume(async (message) => { /* all messages */ })
|
|
143
|
-
|
|
144
|
-
// Skip history, only new messages
|
|
140
|
+
// Default ('new'): start where the group first pops, skip what is already there
|
|
145
141
|
await queen.queue('events')
|
|
146
142
|
.group('realtime-monitor')
|
|
147
|
-
.subscriptionMode('new')
|
|
148
143
|
.consume(async (message) => { /* new only */ })
|
|
149
144
|
|
|
145
|
+
// Process ALL messages, including the backlog
|
|
146
|
+
await queen.queue('events')
|
|
147
|
+
.group('batch-analytics')
|
|
148
|
+
.subscriptionMode('all')
|
|
149
|
+
.consume(async (message) => { /* all messages */ })
|
|
150
|
+
|
|
150
151
|
// Start from specific timestamp
|
|
151
152
|
await queen.queue('events')
|
|
152
153
|
.group('replay')
|
|
@@ -191,8 +192,8 @@ Notes:
|
|
|
191
192
|
the partition cap; either way it is clamped to 64, so a conflating pop returns
|
|
192
193
|
at most 64 messages per round-trip whatever `batch` says.
|
|
193
194
|
- Refused with 400 by the broker without a `.group(...)`, and together with
|
|
194
|
-
`.
|
|
195
|
-
|
|
195
|
+
`.commitOnDelivery()` (a commit at delivery would turn the guarantee above
|
|
196
|
+
into at-most-once). `pop()` raises that 400 instead of returning `[]`.
|
|
196
197
|
- Requires broker **>= 1.1.0**. An older broker ignores the flag and would
|
|
197
198
|
quietly deliver the whole backlog, so the SDK raises
|
|
198
199
|
`conflation was requested but this broker did not apply it` on the first
|
|
@@ -278,6 +279,52 @@ await queen.queue('tasks')
|
|
|
278
279
|
})
|
|
279
280
|
```
|
|
280
281
|
|
|
282
|
+
**When the handler throws**, the consumer nacks what it was given (the message, or the whole batch)
|
|
283
|
+
and keeps consuming. The broker redelivers it, and moves it to the dead letter queue once the queue's
|
|
284
|
+
`retryLimit` is spent. With `.each()`, the later messages of the failed message's partition are
|
|
285
|
+
skipped after a nack (the broker redelivers them too); the other partitions of the same pop are
|
|
286
|
+
still handled.
|
|
287
|
+
|
|
288
|
+
This is the same with `.autoAck(false)`. That setting hands the *success* path to your handler (it
|
|
289
|
+
acks), not the failure path: a handler that threw never got to settle its messages. A message your
|
|
290
|
+
handler acked before throwing is not affected: it is already settled, so the broker refuses
|
|
291
|
+
its nack.
|
|
292
|
+
|
|
293
|
+
To handle failures yourself, add `.onError(async (message, error) => { ... })`. The error then never
|
|
294
|
+
reaches the consumer, nothing is nacked for you, and the message is yours to ack, nack, or leave
|
|
295
|
+
until its lease expires. To stop consuming on an error, abort the `signal` you passed to
|
|
296
|
+
`consume(handler, { signal })` from inside `onError`.
|
|
297
|
+
|
|
298
|
+
```javascript
|
|
299
|
+
// autoAck(false): the handler acks. A throw before the ack is nacked and retried.
|
|
300
|
+
await queen.queue('tasks')
|
|
301
|
+
.group('workers')
|
|
302
|
+
.autoAck(false)
|
|
303
|
+
.each() // one message per call; without it the handler gets the popped array
|
|
304
|
+
.consume(async (message) => {
|
|
305
|
+
await processTask(message.data)
|
|
306
|
+
await queen.ack(message, true)
|
|
307
|
+
})
|
|
308
|
+
|
|
309
|
+
// Your own failure policy: nack, then stop consuming.
|
|
310
|
+
const stop = new AbortController()
|
|
311
|
+
await queen.queue('tasks')
|
|
312
|
+
.group('workers')
|
|
313
|
+
.autoAck(false)
|
|
314
|
+
.each()
|
|
315
|
+
.consume(async (message) => {
|
|
316
|
+
await processTask(message.data)
|
|
317
|
+
await queen.ack(message, true)
|
|
318
|
+
}, { signal: stop.signal })
|
|
319
|
+
.onError(async (message, error) => {
|
|
320
|
+
await queen.ack(message, false, { error: error.message })
|
|
321
|
+
stop.abort()
|
|
322
|
+
})
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
Earlier versions stopped the consumer on a throw under `.autoAck(false)`: `consume()` rejected after
|
|
326
|
+
the first failure, and the message stayed leased until its lease expired.
|
|
327
|
+
|
|
281
328
|
### Pop Messages (On-Demand Processing)
|
|
282
329
|
|
|
283
330
|
```javascript
|
|
@@ -494,9 +541,9 @@ Both surfaces are **always there**. There is nothing to enable: kv and timers ar
|
|
|
494
541
|
broker the way push and pop are, on every cell that runs it. There is no capability to probe and
|
|
495
542
|
no 404 that means "this cell does not have the feature" — a 404 from these routes is a bug.
|
|
496
543
|
|
|
497
|
-
What an operator can still do is **pause** them, with the runtime kill
|
|
498
|
-
|
|
499
|
-
|
|
544
|
+
What an operator can still do is **pause** them, with the broker's runtime kill switches
|
|
545
|
+
(`kv_enabled`, `timers_schedule_enabled`, `timers_fire_enabled`) — a lever pulled live during an
|
|
546
|
+
incident and expected to be pulled back.
|
|
500
547
|
A paused surface answers `503` with `Retry-After` and `error: 'kv_disabled'` / `'timers_disabled'`,
|
|
501
548
|
which this client retries like any other 5xx. Inside a transaction it is a `403` on the `kv` or
|
|
502
549
|
`timers` rider instead, so a bundle holding messages does not spin forever on a paused cell.
|
|
@@ -581,7 +628,7 @@ the retry budget.
|
|
|
581
628
|
|
|
582
629
|
Durations that can be sub-second are in **milliseconds** (`delayMs`), the ones that cannot are in
|
|
583
630
|
**seconds** (`ttlSeconds`). Only relative delays exist, because there is one clock and it is the
|
|
584
|
-
|
|
631
|
+
broker's. A delay in the past is legal and fires on the first cycle.
|
|
585
632
|
|
|
586
633
|
`deliverAt` is **"not before"**, never "exactly at".
|
|
587
634
|
|
|
@@ -748,8 +795,20 @@ const msgs = await queen.queue('q').batch(10).pop()
|
|
|
748
795
|
const msgs = await queen.queue('q').batch(10).wait(true).pop()
|
|
749
796
|
const msgs = await queen.queue('q').batch(200).partitions(50).pop() // multi-partition pop
|
|
750
797
|
const { messages, autopilot } = await queen.queue('q').popResult() // + what the broker chose
|
|
798
|
+
const msgs = await queen.queue('q').group('g').commitOnDelivery().pop() // committed at delivery, nothing to ack
|
|
751
799
|
```
|
|
752
800
|
|
|
801
|
+
A pop long-polls, waiting up to `timeoutMillis` (30 s) for a message, unless you call
|
|
802
|
+
`.wait(false)`. Its messages come back leased, and the ack is yours.
|
|
803
|
+
|
|
804
|
+
`.commitOnDelivery()` changes that for `pop()` and `popResult()`. The broker moves the group's
|
|
805
|
+
cursor past the messages as it hands them out: there is no lease (`leaseId` is empty) and nothing
|
|
806
|
+
to ack. This is at-most-once delivery: a crash after the pop loses the messages. The broker
|
|
807
|
+
refuses it together with `.conflation()` (400). `consume()` always leases its messages, so it
|
|
808
|
+
throws before any request when the builder has `.commitOnDelivery()`.
|
|
809
|
+
|
|
810
|
+
`.autoAck()` is `consume()`'s ack after your handler and has no effect on a pop.
|
|
811
|
+
|
|
753
812
|
### Consume
|
|
754
813
|
|
|
755
814
|
```javascript
|
|
@@ -768,6 +827,11 @@ await queen.ack(message, false, { error: 'reason' })
|
|
|
768
827
|
await queen.ack([msg1, msg2], true) // Batch ack
|
|
769
828
|
```
|
|
770
829
|
|
|
830
|
+
An ack is judged against the lease of the consumer group the message was popped under. `queen.ack()`
|
|
831
|
+
and `transaction().ack()` take that group from the message (`message.consumerGroup`, which every pop
|
|
832
|
+
returns) unless you name one: `{ group }` on `queen.ack()`, `{ consumerGroup }` or `{ group }` on a
|
|
833
|
+
transaction. A batch ack carries a single group, so `queen.ack()` throws on a batch that mixes groups.
|
|
834
|
+
|
|
771
835
|
### Transactions
|
|
772
836
|
|
|
773
837
|
```javascript
|
|
@@ -816,11 +880,14 @@ await queen.timer(q).list({ limit: 50, after }) // {rows, trunc
|
|
|
816
880
|
### Lease Renewal
|
|
817
881
|
|
|
818
882
|
```javascript
|
|
819
|
-
await queen.renew(message)
|
|
820
|
-
await queen.renew([msg1, msg2, msg3])
|
|
883
|
+
await queen.renew(message) // {leaseId, success, newExpiresAt, renewed}
|
|
884
|
+
await queen.renew([msg1, msg2, msg3]) // one result per distinct lease
|
|
821
885
|
await queen.queue('q').renewLease(true, 60000).consume(async (msg) => { /* auto-renew */ })
|
|
822
886
|
```
|
|
823
887
|
|
|
888
|
+
`success: false` (with an `error`) means nothing was renewed: the lease expired, an ack or nack
|
|
889
|
+
already released it, or it never existed. The messages it covered may already be redelivered.
|
|
890
|
+
|
|
824
891
|
### Buffering
|
|
825
892
|
|
|
826
893
|
```javascript
|
|
@@ -896,6 +963,18 @@ await queen.close() // Flush buffers and close connections
|
|
|
896
963
|
the broker. These values are what comes back with `.autopilot(false)` or
|
|
897
964
|
`QUEEN_SDK_POP_AUTOPILOT=off`.
|
|
898
965
|
|
|
966
|
+
### Pop Defaults
|
|
967
|
+
|
|
968
|
+
```javascript
|
|
969
|
+
{
|
|
970
|
+
batch: 1, // autopilot OFF only, as for consume
|
|
971
|
+
wait: true, // long-polls; .wait(false) returns at once
|
|
972
|
+
timeoutMillis: 30000, // the long-poll limit
|
|
973
|
+
autoAck: false, // consume()'s ack after the handler; no effect on pop()
|
|
974
|
+
commitOnDelivery: false // leased; .commitOnDelivery() commits at delivery (at-most-once)
|
|
975
|
+
}
|
|
976
|
+
```
|
|
977
|
+
|
|
899
978
|
---
|
|
900
979
|
|
|
901
980
|
## Logging
|
|
@@ -970,4 +1049,3 @@ const messages: Message<OrderData>[] = await queen.queue('orders').pop()
|
|
|
970
1049
|
## License
|
|
971
1050
|
|
|
972
1051
|
Apache 2.0 - See [LICENSE.md](../LICENSE.md)
|
|
973
|
-
|
package/client-v2/Queen.js
CHANGED
|
@@ -15,6 +15,7 @@ import { StreamBuilder } from './stream/StreamBuilder.js'
|
|
|
15
15
|
import { StreamConsumer } from './stream/StreamConsumer.js'
|
|
16
16
|
import { Admin } from './admin/Admin.js'
|
|
17
17
|
import { popAutopilotDisabledByEnv } from './utils/autopilot.js'
|
|
18
|
+
import { consumerGroupOf, sharedConsumerGroupOf } from './utils/consumerGroup.js'
|
|
18
19
|
import { CLIENT_DEFAULTS } from './utils/defaults.js'
|
|
19
20
|
import { validateUrl, validateUrls } from './utils/validation.js'
|
|
20
21
|
import * as logger from './utils/logger.js'
|
|
@@ -28,9 +29,9 @@ const CLOSE_FLUSH_DEADLINE_MILLIS = 30000
|
|
|
28
29
|
// Both /api/v1/ack and /api/v1/ack/batch respond with a top-level JSON array,
|
|
29
30
|
// one item per acknowledgment in request order:
|
|
30
31
|
// [{index, transactionId, success, error, queueName, partitionName, leaseReleased, dlq}]
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
//
|
|
32
|
+
// A rejected ack/nack (e.g. "Invalid or expired lease") still arrives as HTTP
|
|
33
|
+
// 200 with success=false on the item, so the per-item flag is the only signal
|
|
34
|
+
// that the broker accepted it.
|
|
34
35
|
|
|
35
36
|
function normalizeAckItem(item, index) {
|
|
36
37
|
if (item === null || typeof item !== 'object') {
|
|
@@ -61,6 +62,30 @@ function parseAckResults(result, expected) {
|
|
|
61
62
|
throw new Error('Unexpected ack response format: missing per-item result array')
|
|
62
63
|
}
|
|
63
64
|
|
|
65
|
+
// POST /api/v1/lease/:leaseId/extend answers HTTP 200 whether or not anything
|
|
66
|
+
// was renewed, so `success` in the body is the only signal:
|
|
67
|
+
// 1.x and 2.x brokers: {leaseId, success, renewed, newExpiresAt, expiresAt, lease_expires_at}
|
|
68
|
+
// the C++ broker: [{index, leaseId, success, error, expiresAt}]
|
|
69
|
+
// success:false means the lease is gone (expired, released by an ack or nack,
|
|
70
|
+
// or never existed): nothing was extended, and the messages it covered can
|
|
71
|
+
// already be on their way to another consumer.
|
|
72
|
+
function parseRenewResult(result, leaseId) {
|
|
73
|
+
const item = Array.isArray(result) ? result[0] : result
|
|
74
|
+
if (item === null || typeof item !== 'object') {
|
|
75
|
+
return { leaseId, success: false, newExpiresAt: null, error: 'Unexpected lease renewal response' }
|
|
76
|
+
}
|
|
77
|
+
const newExpiresAt = item.newExpiresAt ?? item.expiresAt ?? item.lease_expires_at ?? null
|
|
78
|
+
const success = typeof item.success === 'boolean' ? item.success : newExpiresAt !== null
|
|
79
|
+
const outcome = { leaseId, success, newExpiresAt }
|
|
80
|
+
if (typeof item.renewed === 'number') outcome.renewed = item.renewed
|
|
81
|
+
if (!success) {
|
|
82
|
+
outcome.error = typeof item.error === 'string' && item.error.length > 0
|
|
83
|
+
? item.error
|
|
84
|
+
: 'Lease not renewed: it expired, was released by an ack or nack, or does not exist'
|
|
85
|
+
}
|
|
86
|
+
return outcome
|
|
87
|
+
}
|
|
88
|
+
|
|
64
89
|
export class Queen {
|
|
65
90
|
#httpClient
|
|
66
91
|
#bufferManager
|
|
@@ -294,8 +319,8 @@ export class Queen {
|
|
|
294
319
|
* creates it, which is what makes thousands of short-lived req/reply
|
|
295
320
|
* inboxes cheap (§1.1).
|
|
296
321
|
* * delivery is at-least-once while the owning broker lives, at-most-once
|
|
297
|
-
* with `
|
|
298
|
-
* need idempotency.
|
|
322
|
+
* with `commitOnDelivery` (§1.3) -- NOT "at most once" as a class.
|
|
323
|
+
* Consumers still need idempotency.
|
|
299
324
|
* * consumption semantics are the pop's `group`, exactly as on durable
|
|
300
325
|
* queues (§1.5): same group competes, own group fans out, no group is
|
|
301
326
|
* queue mode. There is no queue-level mode to set.
|
|
@@ -438,11 +463,15 @@ export class Queen {
|
|
|
438
463
|
})
|
|
439
464
|
}
|
|
440
465
|
|
|
466
|
+
// The group the messages were popped under unless the caller names one:
|
|
467
|
+
// the lease every ack has to match belongs to it (utils/consumerGroup.js).
|
|
468
|
+
const consumerGroup = context.group || sharedConsumerGroupOf(message)
|
|
469
|
+
|
|
441
470
|
// Call batch ack endpoint
|
|
442
471
|
try {
|
|
443
472
|
const result = await this.#httpClient.post('/api/v1/ack/batch', {
|
|
444
473
|
acknowledgments,
|
|
445
|
-
consumerGroup
|
|
474
|
+
consumerGroup
|
|
446
475
|
})
|
|
447
476
|
|
|
448
477
|
const results = parseAckResults(result, acknowledgments.length)
|
|
@@ -487,7 +516,9 @@ export class Queen {
|
|
|
487
516
|
partitionId,
|
|
488
517
|
status: statusStr,
|
|
489
518
|
error: context.error || null,
|
|
490
|
-
|
|
519
|
+
// The group the message was popped under unless the caller names one
|
|
520
|
+
// (utils/consumerGroup.js).
|
|
521
|
+
consumerGroup: context.group || consumerGroupOf(message)
|
|
491
522
|
}
|
|
492
523
|
|
|
493
524
|
if (leaseId) body.leaseId = leaseId
|
|
@@ -529,8 +560,8 @@ export class Queen {
|
|
|
529
560
|
}
|
|
530
561
|
|
|
531
562
|
// Dedupe: with v4 multi-partition pop, all messages in one batch share
|
|
532
|
-
// the same leaseId (one
|
|
533
|
-
//
|
|
563
|
+
// the same leaseId (one renew call extends the lease of every claimed
|
|
564
|
+
// partition). Without this, callers passing the full
|
|
534
565
|
// messages array would issue N redundant identical HTTP calls.
|
|
535
566
|
leaseIds = [...new Set(leaseIds)]
|
|
536
567
|
|
|
@@ -545,12 +576,13 @@ export class Queen {
|
|
|
545
576
|
for (const leaseId of leaseIds) {
|
|
546
577
|
try {
|
|
547
578
|
const result = await this.#httpClient.post(`/api/v1/lease/${leaseId}/extend`, {})
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
}
|
|
553
|
-
|
|
579
|
+
const outcome = parseRenewResult(result, leaseId)
|
|
580
|
+
results.push(outcome)
|
|
581
|
+
if (outcome.success) {
|
|
582
|
+
logger.log('Queen.renew', { leaseId, success: true, renewed: outcome.renewed })
|
|
583
|
+
} else {
|
|
584
|
+
logger.error('Queen.renew', { leaseId, error: outcome.error })
|
|
585
|
+
}
|
|
554
586
|
} catch (error) {
|
|
555
587
|
results.push({ leaseId, success: false, error: error.message })
|
|
556
588
|
logger.error('Queen.renew', { leaseId, error: error.message })
|
package/client-v2/README.md
CHANGED
|
@@ -60,7 +60,7 @@ When connecting to multiple Queen servers, you can choose how requests are distr
|
|
|
60
60
|
|
|
61
61
|
#### Affinity Mode (Recommended for Production)
|
|
62
62
|
|
|
63
|
-
Uses consistent hashing with virtual nodes to route consumer groups to the same backend server.
|
|
63
|
+
Uses consistent hashing with virtual nodes to route consumer groups to the same backend server.
|
|
64
64
|
|
|
65
65
|
```javascript
|
|
66
66
|
const queen = new Queen({
|
|
@@ -444,28 +444,31 @@ When a consumer group first subscribes to a queue, should it process **all histo
|
|
|
444
444
|
- Join a stream at a specific point in time
|
|
445
445
|
- Skip historical data for new analytics consumers
|
|
446
446
|
|
|
447
|
-
### Default Behavior (
|
|
447
|
+
### Default Behavior (New Messages)
|
|
448
448
|
|
|
449
|
-
By default, consumer
|
|
449
|
+
By default, a consumer group starts where it first pops the queue and skips the messages that were
|
|
450
|
+
already there. That default is the broker's `DEFAULT_SUBSCRIPTION_MODE`, which is `new`:
|
|
450
451
|
|
|
451
452
|
```javascript
|
|
452
|
-
// This consumer group gets
|
|
453
|
+
// This consumer group gets only messages that arrive after it first pops
|
|
453
454
|
await queen
|
|
454
455
|
.queue('events')
|
|
455
456
|
.group('new-analytics')
|
|
457
|
+
.each()
|
|
456
458
|
.consume(async (message) => {
|
|
457
459
|
console.log('Processing:', message.data)
|
|
458
460
|
})
|
|
459
461
|
```
|
|
460
462
|
|
|
461
|
-
|
|
463
|
+
To process the backlog too, ask for it with `.subscriptionMode('all')`, or start the broker with
|
|
464
|
+
`DEFAULT_SUBSCRIPTION_MODE=all` to make that the default for every new group:
|
|
462
465
|
```bash
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
./bin/queen-server
|
|
466
|
+
export DEFAULT_SUBSCRIPTION_MODE="all"
|
|
467
|
+
./bin/queen
|
|
466
468
|
```
|
|
467
469
|
|
|
468
|
-
|
|
470
|
+
The mode is stored with the group the first time it pops; changing the default later does not move
|
|
471
|
+
an existing group.
|
|
469
472
|
|
|
470
473
|
### Subscription Mode: 'new'
|
|
471
474
|
|
|
@@ -1004,8 +1007,9 @@ const message = messages[0]
|
|
|
1004
1007
|
|
|
1005
1008
|
// Start long processing
|
|
1006
1009
|
const timer = setInterval(async () => {
|
|
1007
|
-
await queen.renew(message) // Extend lease
|
|
1008
|
-
console.log('Lease renewed
|
|
1010
|
+
const { success, newExpiresAt, error } = await queen.renew(message) // Extend lease
|
|
1011
|
+
if (success) console.log('Lease renewed until', newExpiresAt)
|
|
1012
|
+
else console.warn('Lease lost:', error) // expired or already released: it will be redelivered
|
|
1009
1013
|
}, 30000) // Every 30 seconds
|
|
1010
1014
|
|
|
1011
1015
|
try {
|
|
@@ -1432,6 +1436,16 @@ await queen.queue('reports').consume(async (msg) => {
|
|
|
1432
1436
|
|
|
1433
1437
|
Sometimes you need more control over what happens when messages succeed or fail.
|
|
1434
1438
|
|
|
1439
|
+
### What a Throw Does
|
|
1440
|
+
|
|
1441
|
+
Without callbacks, a handler that throws gets what it was given nacked (the message with `.each()`,
|
|
1442
|
+
otherwise the whole popped batch), and the consumer keeps going. The broker redelivers it, and moves
|
|
1443
|
+
it to the DLQ once the queue's `retryLimit` is spent. That holds with `.autoAck(false)` too: there
|
|
1444
|
+
your handler acks on success, but a handler that threw never got to settle anything.
|
|
1445
|
+
|
|
1446
|
+
With `.onError()` the failure is yours: nothing is nacked on your behalf. Ack, nack or DLQ the
|
|
1447
|
+
message in the callback, or it stays leased until its lease expires.
|
|
1448
|
+
|
|
1435
1449
|
### Success Callback
|
|
1436
1450
|
|
|
1437
1451
|
```javascript
|
|
@@ -1457,6 +1471,7 @@ await queen
|
|
|
1457
1471
|
.onError(async (message, error) => {
|
|
1458
1472
|
console.error('Failed:', error.message)
|
|
1459
1473
|
// Log to external service, send alert, etc.
|
|
1474
|
+
await queen.ack(message, false) // with onError, nothing is nacked for you
|
|
1460
1475
|
})
|
|
1461
1476
|
```
|
|
1462
1477
|
|
|
@@ -1647,6 +1662,11 @@ const msgs = await queen.queue('q').partition('p1').pop()
|
|
|
1647
1662
|
// capped at 200 total messages. All partitions share one leaseId.
|
|
1648
1663
|
// Each returned message carries its own partitionId / partition / leaseId.
|
|
1649
1664
|
const msgs = await queen.queue('q').batch(200).partitions(50).pop()
|
|
1665
|
+
|
|
1666
|
+
// Commit at delivery: the broker moves the group's cursor past the messages
|
|
1667
|
+
// as it hands them out. No lease, nothing to ack, at-most-once: a crash after
|
|
1668
|
+
// the pop loses them. pop() and popResult() only; consume() throws.
|
|
1669
|
+
const msgs = await queen.queue('q').group('g').commitOnDelivery().pop()
|
|
1650
1670
|
```
|
|
1651
1671
|
|
|
1652
1672
|
### Consume
|
|
@@ -1900,8 +1920,9 @@ await queen.close()
|
|
|
1900
1920
|
```javascript
|
|
1901
1921
|
{
|
|
1902
1922
|
batch: 1,
|
|
1903
|
-
wait:
|
|
1904
|
-
autoAck: false
|
|
1923
|
+
wait: true, // Long polling; .wait(false) returns at once
|
|
1924
|
+
autoAck: false, // consume()'s ack after the handler; no effect on pop()
|
|
1925
|
+
commitOnDelivery: false // Manual ack required; .commitOnDelivery() commits at delivery
|
|
1905
1926
|
}
|
|
1906
1927
|
```
|
|
1907
1928
|
|
package/client-v2/admin/Admin.js
CHANGED
|
@@ -101,15 +101,26 @@ export class Admin {
|
|
|
101
101
|
}
|
|
102
102
|
|
|
103
103
|
/**
|
|
104
|
-
*
|
|
104
|
+
* @deprecated The 2.x broker has no route that clears a queue, so this
|
|
105
|
+
* always rejects, before any request. Seek each consumer group
|
|
106
|
+
* to the end instead.
|
|
107
|
+
*
|
|
108
|
+
* The old DELETE /api/v1/queues/:name/clear answers 404 no_such_route on
|
|
109
|
+
* 2.x. A seek to the end moves one group's cursor past every queued message
|
|
110
|
+
* (and releases its live leases); the group of a pop without one is
|
|
111
|
+
* '__QUEUE_MODE__'. Deleting the queue is not a clear: it removes the queue
|
|
112
|
+
* itself, with its configuration.
|
|
105
113
|
* @param {string} name - Queue name
|
|
106
|
-
* @param {string} [partition] -
|
|
107
|
-
* @returns {Promise<
|
|
114
|
+
* @param {string} [partition] - Ignored
|
|
115
|
+
* @returns {Promise<never>}
|
|
108
116
|
*/
|
|
109
117
|
async clearQueue(name, partition = null) {
|
|
110
118
|
logger.log('Admin.clearQueue', { name, partition })
|
|
111
|
-
|
|
112
|
-
|
|
119
|
+
throw new Error(
|
|
120
|
+
'Queen 2.x has no route that clears a queue. Move each consumer group past what is queued ' +
|
|
121
|
+
`with queen.admin.seekConsumerGroup(group, '${name}', { toEnd: true }); ` +
|
|
122
|
+
"a pop without a group reads as the group '__QUEUE_MODE__'."
|
|
123
|
+
)
|
|
113
124
|
}
|
|
114
125
|
|
|
115
126
|
/**
|
|
@@ -172,14 +183,25 @@ export class Admin {
|
|
|
172
183
|
}
|
|
173
184
|
|
|
174
185
|
/**
|
|
175
|
-
*
|
|
186
|
+
* @deprecated The 2.x broker has no route that moves a message to the DLQ
|
|
187
|
+
* by its address, so this always rejects, before any request.
|
|
188
|
+
* Ack the leased message with the `dlq` status instead.
|
|
189
|
+
*
|
|
190
|
+
* The old POST /api/v1/messages/:partitionId/:transactionId/dlq answers 404
|
|
191
|
+
* no_such_route on 2.x. The broker files a dead letter when the consumer
|
|
192
|
+
* group acks the message with the `dlq` status while it holds the lease (an
|
|
193
|
+
* address alone cannot name the lease), or when a `failed` ack spends the
|
|
194
|
+
* queue's last retry.
|
|
176
195
|
* @param {string} partitionId - Partition ID
|
|
177
196
|
* @param {string} transactionId - Transaction ID
|
|
178
|
-
* @returns {Promise<
|
|
197
|
+
* @returns {Promise<never>}
|
|
179
198
|
*/
|
|
180
199
|
async moveMessageToDLQ(partitionId, transactionId) {
|
|
181
200
|
logger.log('Admin.moveMessageToDLQ', { partitionId, transactionId })
|
|
182
|
-
|
|
201
|
+
throw new Error(
|
|
202
|
+
'Queen 2.x has no route that moves a message to the DLQ by its address. Pop the message for ' +
|
|
203
|
+
"its consumer group and call queen.ack(message, 'dlq', { group }) while you hold its lease."
|
|
204
|
+
)
|
|
183
205
|
}
|
|
184
206
|
|
|
185
207
|
// ===========================
|
|
@@ -359,44 +381,6 @@ export class Admin {
|
|
|
359
381
|
return this.#httpClient.get('/metrics')
|
|
360
382
|
}
|
|
361
383
|
|
|
362
|
-
/**
|
|
363
|
-
* Get push maintenance mode status
|
|
364
|
-
* @returns {Promise<object>}
|
|
365
|
-
*/
|
|
366
|
-
async getMaintenanceMode() {
|
|
367
|
-
logger.log('Admin.getMaintenanceMode', {})
|
|
368
|
-
return this.#httpClient.get('/api/v1/system/maintenance')
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
/**
|
|
372
|
-
* Set push maintenance mode
|
|
373
|
-
* @param {boolean} enabled - Enable or disable maintenance mode
|
|
374
|
-
* @returns {Promise<object>}
|
|
375
|
-
*/
|
|
376
|
-
async setMaintenanceMode(enabled) {
|
|
377
|
-
logger.log('Admin.setMaintenanceMode', { enabled })
|
|
378
|
-
return this.#httpClient.post('/api/v1/system/maintenance', { enabled })
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
/**
|
|
382
|
-
* Get pop maintenance mode status
|
|
383
|
-
* @returns {Promise<object>}
|
|
384
|
-
*/
|
|
385
|
-
async getPopMaintenanceMode() {
|
|
386
|
-
logger.log('Admin.getPopMaintenanceMode', {})
|
|
387
|
-
return this.#httpClient.get('/api/v1/system/maintenance/pop')
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
/**
|
|
391
|
-
* Set pop maintenance mode
|
|
392
|
-
* @param {boolean} enabled - Enable or disable pop maintenance mode
|
|
393
|
-
* @returns {Promise<object>}
|
|
394
|
-
*/
|
|
395
|
-
async setPopMaintenanceMode(enabled) {
|
|
396
|
-
logger.log('Admin.setPopMaintenanceMode', { enabled })
|
|
397
|
-
return this.#httpClient.post('/api/v1/system/maintenance/pop', { enabled })
|
|
398
|
-
}
|
|
399
|
-
|
|
400
384
|
/**
|
|
401
385
|
* Get system metrics (CPU, memory, connections, etc.)
|
|
402
386
|
* @param {object} params - Query parameters (from, to, etc.)
|
|
@@ -419,15 +403,6 @@ export class Admin {
|
|
|
419
403
|
return this.#httpClient.get(`/api/v1/analytics/worker-metrics${queryString}`)
|
|
420
404
|
}
|
|
421
405
|
|
|
422
|
-
/**
|
|
423
|
-
* Get PostgreSQL statistics
|
|
424
|
-
* @returns {Promise<object>}
|
|
425
|
-
*/
|
|
426
|
-
async getPostgresStats() {
|
|
427
|
-
logger.log('Admin.getPostgresStats', {})
|
|
428
|
-
return this.#httpClient.get('/api/v1/analytics/postgres-stats')
|
|
429
|
-
}
|
|
430
|
-
|
|
431
406
|
// ===========================
|
|
432
407
|
// Helper Methods
|
|
433
408
|
// ===========================
|