queen-mq 1.0.6 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -0
- package/client-v2/Queen.js +50 -0
- package/client-v2/admin/Admin.js +15 -1
- package/client-v2/buffer/BufferManager.js +18 -6
- package/client-v2/buffer/MessageBuffer.js +17 -1
- package/client-v2/buffer/sinks.js +89 -0
- package/client-v2/builders/QueueBuilder.js +71 -2
- package/client-v2/consumer/ConsumerManager.js +39 -4
- package/client-v2/ephemeral/Ephemeral.js +551 -0
- package/client-v2/index.js +12 -0
- package/client-v2/streams/Stream.js +3 -0
- package/client-v2/streams/runtime/Runner.js +18 -0
- package/client-v2/utils/conflation.js +118 -0
- package/client-v2/utils/defaults.js +8 -1
- package/package.json +3 -3
- package/test-v2/conflation-unit/conflationWire.test.js +398 -0
- package/test-v2/ephemeral-unit/_planServer.js +73 -0
- package/test-v2/ephemeral-unit/durableSinkPin.test.js +112 -0
- package/test-v2/ephemeral-unit/ephemeralBuffer.test.js +305 -0
- package/test-v2/ephemeral-unit/ephemeralWire.test.js +398 -0
package/README.md
CHANGED
|
@@ -154,6 +154,53 @@ await queen.queue('events')
|
|
|
154
154
|
.consume(async (message) => { /* from timestamp */ })
|
|
155
155
|
```
|
|
156
156
|
|
|
157
|
+
### Conflation (Last-Value Delivery)
|
|
158
|
+
|
|
159
|
+
For command-style queues where one partition is one logical task key — "recompute
|
|
160
|
+
customer 42", "this entity is dirty" — only the newest pending message matters.
|
|
161
|
+
`.conflation(true)` makes a pop of a partition deliver exactly one message, the
|
|
162
|
+
newest visible one, and commit past everything it skipped:
|
|
163
|
+
|
|
164
|
+
```javascript
|
|
165
|
+
// A backlog of 4 000 recompute requests across 12 entities becomes
|
|
166
|
+
// 12 handler calls, each with the freshest input.
|
|
167
|
+
await queen.queue('recompute')
|
|
168
|
+
.group('workers')
|
|
169
|
+
.conflation(true)
|
|
170
|
+
.partitions(64)
|
|
171
|
+
.consume(async (message) => {
|
|
172
|
+
await recompute(message.data.entityId)
|
|
173
|
+
})
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
The guarantee: **after the last push to a partition, at least one handler run
|
|
177
|
+
starts after that push committed.** Nothing is deleted — conflation is a delivery
|
|
178
|
+
policy, not compaction; retention still governs what is stored, and a
|
|
179
|
+
non-conflating group on the same queue still sees every message.
|
|
180
|
+
|
|
181
|
+
Notes:
|
|
182
|
+
|
|
183
|
+
- It is a property of the **consumer group**, fixed when that group first
|
|
184
|
+
registers on the queue. A later consumer declaring the opposite does not flip
|
|
185
|
+
it — the stored value wins, that consumer keeps working, and the SDK warns
|
|
186
|
+
once per (queue, group).
|
|
187
|
+
- Skipping is per **partition**, so partitioning is the key: one partition = one
|
|
188
|
+
logical key is the contract this workload has to hold up.
|
|
189
|
+
- A conflating pop returns at most one message per partition, so **partitions**
|
|
190
|
+
size the round-trip, not `batch`. Left unset, the broker uses `.batch(N)` as
|
|
191
|
+
the partition cap; either way it is clamped to 64, so a conflating pop returns
|
|
192
|
+
at most 64 messages per round-trip whatever `batch` says.
|
|
193
|
+
- Refused with 400 by the broker without a `.group(...)`, and together with
|
|
194
|
+
`.autoAck(true)` (auto-ack commits at delivery, which would turn the guarantee
|
|
195
|
+
above into at-most-once).
|
|
196
|
+
- Requires broker **>= 1.1.0**. An older broker ignores the flag and would
|
|
197
|
+
quietly deliver the whole backlog, so the SDK raises
|
|
198
|
+
`conflation was requested but this broker did not apply it` on the first
|
|
199
|
+
response that does not echo it — before any message is processed.
|
|
200
|
+
- `admin.getQueueDepth(queue, group)` reports `effectivePending` (handler calls
|
|
201
|
+
still owed) next to `pending` (log positions still to retire). For a
|
|
202
|
+
conflating group `pending: 4000000, effectivePending: 12` is healthy.
|
|
203
|
+
|
|
157
204
|
---
|
|
158
205
|
|
|
159
206
|
## Connection Options
|
package/client-v2/Queen.js
CHANGED
|
@@ -10,6 +10,7 @@ import { QueueBuilder } from './builders/QueueBuilder.js'
|
|
|
10
10
|
import { TransactionBuilder } from './builders/TransactionBuilder.js'
|
|
11
11
|
import { TimerBuilder } from './builders/TimerBuilder.js'
|
|
12
12
|
import { Kv } from './kv/Kv.js'
|
|
13
|
+
import { Ephemeral } from './ephemeral/Ephemeral.js'
|
|
13
14
|
import { StreamBuilder } from './stream/StreamBuilder.js'
|
|
14
15
|
import { StreamConsumer } from './stream/StreamConsumer.js'
|
|
15
16
|
import { Admin } from './admin/Admin.js'
|
|
@@ -66,6 +67,7 @@ export class Queen {
|
|
|
66
67
|
#shutdownHandlers = []
|
|
67
68
|
#admin = null
|
|
68
69
|
#kv = null
|
|
70
|
+
#ephemeral = null
|
|
69
71
|
|
|
70
72
|
constructor(config = {}) {
|
|
71
73
|
// Configure custom logger before anything else.
|
|
@@ -253,6 +255,54 @@ export class Queen {
|
|
|
253
255
|
return this.#kv
|
|
254
256
|
}
|
|
255
257
|
|
|
258
|
+
// ===========================
|
|
259
|
+
// Ephemeral API Entry Point
|
|
260
|
+
// ===========================
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* RAM-class queues: `/api/v1/ephemeral/*` (EPHEMERAL_QUEUES.md §1, §4).
|
|
264
|
+
*
|
|
265
|
+
* await queen.ephemeral.push('inbox:7', [{ hello: 'world' }])
|
|
266
|
+
* const { messages } = await queen.ephemeral.pop('inbox:7', { wait: true })
|
|
267
|
+
* await queen.ephemeral.ack('inbox:7', messages, { group: 'workers' })
|
|
268
|
+
*
|
|
269
|
+
* A different STORAGE CLASS, not a different API style. What changes:
|
|
270
|
+
*
|
|
271
|
+
* * CONTENTS SURVIVE NOTHING (§1.2) -- restart, crash, deploy, or the
|
|
272
|
+
* ownership move a membership change causes. Treat a failover like a
|
|
273
|
+
* Redis restart. A declared queue's OPTIONS are durable; it comes back
|
|
274
|
+
* configured and EMPTY.
|
|
275
|
+
* * a queue does not have to exist: the first push or pop that names one
|
|
276
|
+
* creates it, which is what makes thousands of short-lived req/reply
|
|
277
|
+
* inboxes cheap (§1.1).
|
|
278
|
+
* * delivery is at-least-once while the owning broker lives, at-most-once
|
|
279
|
+
* with `autoAck` (§1.3) -- NOT "at most once" as a class. Consumers still
|
|
280
|
+
* need idempotency.
|
|
281
|
+
* * consumption semantics are the pop's `group`, exactly as on durable
|
|
282
|
+
* queues (§1.5): same group competes, own group fans out, no group is
|
|
283
|
+
* queue mode. There is no queue-level mode to set.
|
|
284
|
+
* * there is no replay, no subscriptionMode, no DLQ, no transactions -- the
|
|
285
|
+
* verbs are absent because the concepts have no referent (§9).
|
|
286
|
+
*
|
|
287
|
+
* `push(..., {buffered})` shares the durable buffer machinery, so
|
|
288
|
+
* `queen.close()` drains it on the same deadline (§4.1).
|
|
289
|
+
*
|
|
290
|
+
* Requires broker/proxy >= 1.1; an older one 404s the whole family and every
|
|
291
|
+
* verb here maps that to `.code === EPHEMERAL_UNSUPPORTED`. Not to be
|
|
292
|
+
* confused with the OTHER 404: `depth` on a queue that does not exist raises
|
|
293
|
+
* `.code === EPHEMERAL_QUEUE_NOT_FOUND`, which is a missing queue and not a
|
|
294
|
+
* missing feature.
|
|
295
|
+
*
|
|
296
|
+
* Lazily initialized, singleton, like `admin` and `kv`.
|
|
297
|
+
* @returns {Ephemeral}
|
|
298
|
+
*/
|
|
299
|
+
get ephemeral() {
|
|
300
|
+
if (!this.#ephemeral) {
|
|
301
|
+
this.#ephemeral = new Ephemeral(this.#httpClient, this.#bufferManager)
|
|
302
|
+
}
|
|
303
|
+
return this.#ephemeral
|
|
304
|
+
}
|
|
305
|
+
|
|
256
306
|
// ===========================
|
|
257
307
|
// Timers API Entry Point
|
|
258
308
|
// ===========================
|
package/client-v2/admin/Admin.js
CHANGED
|
@@ -71,11 +71,25 @@ export class Admin {
|
|
|
71
71
|
/**
|
|
72
72
|
* Per-partition backlog for a queue — the cheap sibling of getQueue:
|
|
73
73
|
* watermark arithmetic only, no segments, no timestamps. Shape:
|
|
74
|
-
* {queue, group, pending,
|
|
74
|
+
* {queue, group, pending, partitionsPending, conflation, effectivePending,
|
|
75
|
+
* partitions: [{partition, pending}]}.
|
|
75
76
|
* Omitting group gives queue-level pending under the same worst-cursor
|
|
76
77
|
* precedence the dashboard publishes; a named group is that group's own
|
|
77
78
|
* backlog per partition. Requires broker >= 1.0.4 — an older broker
|
|
78
79
|
* answers 404 no_such_route, so fall back to getQueue there.
|
|
80
|
+
*
|
|
81
|
+
* The three fields added in 1.1.0 (PLAN_CONFLATION §2.5/§5.3):
|
|
82
|
+
* - `partitionsPending`: how many partitions owe work (pending > 0). Useful
|
|
83
|
+
* for every group; it is what queenctl used to compute client-side.
|
|
84
|
+
* - `conflation`: the group's stored last-value delivery policy.
|
|
85
|
+
* - `effectivePending`: WORK depth — handler invocations still owed. For a
|
|
86
|
+
* conflating group that is `partitionsPending` (one call per partition,
|
|
87
|
+
* newest message only); otherwise it equals `pending`.
|
|
88
|
+
*
|
|
89
|
+
* Read them together: for a conflating group `pending` is LOG depth (log
|
|
90
|
+
* positions still to retire), so `pending: 4000000, effectivePending: 12` is
|
|
91
|
+
* healthy — the same two numbers on a non-conflating group are an incident.
|
|
92
|
+
* Absent on brokers older than 1.1.0.
|
|
79
93
|
* @param {string} name - Queue name
|
|
80
94
|
* @param {string|null} [group] - Consumer group (optional)
|
|
81
95
|
* @returns {Promise<object>}
|
|
@@ -19,6 +19,14 @@
|
|
|
19
19
|
* path that has a SIGTERM grace period to respect. When the deadline expires
|
|
20
20
|
* the messages are still in the buffer -- the error says how many -- so the
|
|
21
21
|
* failure is loud rather than silent.
|
|
22
|
+
*
|
|
23
|
+
* WHERE a batch goes is the buffer's DESTINATION (buffer/sinks.js), not
|
|
24
|
+
* something this loop knows: durable pushes and ephemeral pushes are two routes
|
|
25
|
+
* with two body shapes and exactly one set of ordering, backpressure and retry
|
|
26
|
+
* semantics, so the drain is parametrized instead of copied. Addresses are
|
|
27
|
+
* namespaced per family (`eph:` prefix), so the two never share a buffer, a
|
|
28
|
+
* drain, or a retry queue. A buffer created without a destination drains to the
|
|
29
|
+
* durable push exactly as before.
|
|
22
30
|
*/
|
|
23
31
|
|
|
24
32
|
import { MessageBuffer } from './MessageBuffer.js'
|
|
@@ -45,9 +53,12 @@ export class BufferManager {
|
|
|
45
53
|
* @param {string} queueAddress
|
|
46
54
|
* @param {object} formattedMessage
|
|
47
55
|
* @param {object} bufferOptions
|
|
48
|
-
* @param {{ signal?: AbortSignal }} [opts]
|
|
56
|
+
* @param {{ signal?: AbortSignal, destination?: object }} [opts] - `destination`
|
|
57
|
+
* is `{ sink, queue, partition }` (buffer/sinks.js) and is read ONLY when
|
|
58
|
+
* this address's buffer is created: an address belongs to one queue of one
|
|
59
|
+
* storage class, so its route cannot change under an in-flight retry.
|
|
49
60
|
*/
|
|
50
|
-
async addMessage(queueAddress, formattedMessage, bufferOptions, { signal } = {}) {
|
|
61
|
+
async addMessage(queueAddress, formattedMessage, bufferOptions, { signal, destination = null } = {}) {
|
|
51
62
|
// A push after cleanup() would otherwise create a fresh buffer that nothing
|
|
52
63
|
// will ever flush -- messages accepted into a client that is already shut
|
|
53
64
|
// down, which is the same false success the bound exists to remove.
|
|
@@ -60,8 +71,8 @@ export class BufferManager {
|
|
|
60
71
|
// maxSize is derived from the messageCount this caller asked for, and
|
|
61
72
|
// merging defaults here first would hide the difference between "not set"
|
|
62
73
|
// and "set to the default".
|
|
63
|
-
const created = new MessageBuffer(queueAddress, bufferOptions, (addr) => { this.#startDrain(addr) })
|
|
64
|
-
logger.log('BufferManager.createBuffer', { queueAddress, options: created.options })
|
|
74
|
+
const created = new MessageBuffer(queueAddress, bufferOptions, (addr) => { this.#startDrain(addr) }, destination)
|
|
75
|
+
logger.log('BufferManager.createBuffer', { queueAddress, options: created.options, sink: created.destination.sink.name })
|
|
65
76
|
this.#buffers.set(queueAddress, created)
|
|
66
77
|
}
|
|
67
78
|
|
|
@@ -111,6 +122,7 @@ export class BufferManager {
|
|
|
111
122
|
|
|
112
123
|
async #drain(queueAddress, buffer, ctl) {
|
|
113
124
|
const { messageCount, retryDelayMillis } = buffer.options
|
|
125
|
+
const { sink, queue, partition } = buffer.destination
|
|
114
126
|
|
|
115
127
|
try {
|
|
116
128
|
while (buffer.messageCount > 0 && !buffer.isStopped) {
|
|
@@ -118,8 +130,8 @@ export class BufferManager {
|
|
|
118
130
|
if (batch.length === 0) break
|
|
119
131
|
|
|
120
132
|
try {
|
|
121
|
-
const result = await this.#httpClient.post(
|
|
122
|
-
logger.debug('BufferManager.drain', { queueAddress, sent: batch.length, serverResponse: result ? `${result.length || 'N/A'} items` : 'null' })
|
|
133
|
+
const result = await this.#httpClient.post(sink.path, sink.format(queue, partition, batch))
|
|
134
|
+
logger.debug('BufferManager.drain', { queueAddress, sink: sink.name, sent: batch.length, serverResponse: result ? `${result.length || 'N/A'} items` : 'null' })
|
|
123
135
|
|
|
124
136
|
this.#flushCount++
|
|
125
137
|
// Capacity freed. Woken here rather than at takeBatch, because a
|
|
@@ -34,14 +34,24 @@
|
|
|
34
34
|
* Buffered messages still live only in this process's memory. A crash, or a
|
|
35
35
|
* `process.exit()` that skips `close()`, loses them -- buffering belongs on
|
|
36
36
|
* telemetry-shaped traffic, not on anything that must not be lost.
|
|
37
|
+
*
|
|
38
|
+
* A buffer also carries its DESTINATION (buffer/sinks.js): the route its
|
|
39
|
+
* batches are posted to and the shape they are posted in. It is fixed at
|
|
40
|
+
* creation and never changes, because it is a property of the address -- one
|
|
41
|
+
* address is one queue of one storage class -- and because a drain that could
|
|
42
|
+
* change route mid-retry would post a re-queued batch somewhere its earlier
|
|
43
|
+
* attempt did not go. Absent, it is the durable push, which is what every
|
|
44
|
+
* caller that predates ephemeral queues gets without knowing sinks exist.
|
|
37
45
|
*/
|
|
38
46
|
|
|
39
47
|
import { BUFFER_DEFAULTS } from '../utils/defaults.js'
|
|
48
|
+
import { DURABLE_DESTINATION } from './sinks.js'
|
|
40
49
|
|
|
41
50
|
export class MessageBuffer {
|
|
42
51
|
#queueAddress
|
|
43
52
|
#messages = []
|
|
44
53
|
#options
|
|
54
|
+
#destination
|
|
45
55
|
#flushCallback
|
|
46
56
|
#timer = null
|
|
47
57
|
#firstMessageTime = null
|
|
@@ -58,10 +68,11 @@ export class MessageBuffer {
|
|
|
58
68
|
#parked = 0
|
|
59
69
|
#stopWaiters = []
|
|
60
70
|
|
|
61
|
-
constructor(queueAddress, options, flushCallback) {
|
|
71
|
+
constructor(queueAddress, options, flushCallback, destination = null) {
|
|
62
72
|
this.#queueAddress = queueAddress
|
|
63
73
|
this.#options = MessageBuffer.normalizeOptions(options)
|
|
64
74
|
this.#flushCallback = flushCallback
|
|
75
|
+
this.#destination = destination || DURABLE_DESTINATION
|
|
65
76
|
}
|
|
66
77
|
|
|
67
78
|
/**
|
|
@@ -298,6 +309,11 @@ export class MessageBuffer {
|
|
|
298
309
|
return this.#options
|
|
299
310
|
}
|
|
300
311
|
|
|
312
|
+
/** `{ sink, queue, partition }` -- where this buffer's batches are posted. */
|
|
313
|
+
get destination() {
|
|
314
|
+
return this.#destination
|
|
315
|
+
}
|
|
316
|
+
|
|
301
317
|
get isFlushing() {
|
|
302
318
|
return this.#flushing
|
|
303
319
|
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drain sinks: WHERE a buffered batch goes, and in WHAT shape.
|
|
3
|
+
*
|
|
4
|
+
* The buffer machinery -- blocking backpressure at `maxSize`, one drain loop
|
|
5
|
+
* per address, a failed batch put back at the FRONT and retried until it lands
|
|
6
|
+
* or a flush deadline expires -- is about ordering, occupancy and loss. None of
|
|
7
|
+
* that is durable-specific, and none of it is worth writing twice. So the drain
|
|
8
|
+
* takes a SINK instead of a hardcoded POST:
|
|
9
|
+
*
|
|
10
|
+
* { path, format(queue, partition, batch) -> body }
|
|
11
|
+
*
|
|
12
|
+
* `format` receives the queue and partition because the two storage classes
|
|
13
|
+
* disagree about where that identity lives on the wire, and that disagreement
|
|
14
|
+
* is the entire reason this parameter exists:
|
|
15
|
+
*
|
|
16
|
+
* * the DURABLE push wire repeats `{queue, partition}` on EVERY item, so the
|
|
17
|
+
* envelope is just `{items}` and the sink ignores both arguments;
|
|
18
|
+
* * the EPHEMERAL push wire hoists them to the envelope --
|
|
19
|
+
* `{queue, partition?, messages:[{payload}...]}` -- so the batch elements
|
|
20
|
+
* carry nothing but their payload.
|
|
21
|
+
*
|
|
22
|
+
* DURABLE_SINK IS TODAY'S REQUEST, BYTE FOR BYTE. It is the default for a
|
|
23
|
+
* buffer created without a destination, which is every caller that existed
|
|
24
|
+
* before ephemeral queues did, and test-v2/ephemeral-unit/durableSinkPin.test.js
|
|
25
|
+
* exists for no other reason than to fail if that ever stops being true.
|
|
26
|
+
*
|
|
27
|
+
* ADDRESSES ARE NAMESPACED. A buffer address is the key of the one-buffer-one-
|
|
28
|
+
* drain map, so an ephemeral `orders` and a durable `orders` must not hash to
|
|
29
|
+
* the same entry -- they are unrelated objects (EPHEMERAL_QUEUES.md §10 Q8) and
|
|
30
|
+
* a shared buffer would post one family's messages to the other family's route.
|
|
31
|
+
* The `eph:` prefix is the same namespacing the broker applies to its own queue
|
|
32
|
+
* keys (§3.2), for the same reason.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
/** The durable push wire: identity per item, envelope carries only the batch. */
|
|
36
|
+
export const DURABLE_SINK = {
|
|
37
|
+
name: 'durable',
|
|
38
|
+
path: '/api/v1/push',
|
|
39
|
+
format(_queue, _partition, batch) {
|
|
40
|
+
return { items: batch }
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** The ephemeral push wire (EPHEMERAL_QUEUES.md §3.1): identity on the envelope. */
|
|
45
|
+
export const EPHEMERAL_SINK = {
|
|
46
|
+
name: 'ephemeral',
|
|
47
|
+
path: '/api/v1/ephemeral/push',
|
|
48
|
+
format(queue, partition, batch) {
|
|
49
|
+
const body = { queue }
|
|
50
|
+
// Omitted, never defaulted client-side: which partition an ephemeral push
|
|
51
|
+
// without one lands on is the broker's rule, and inventing a 'Default' here
|
|
52
|
+
// would take that decision away from it in a way the caller never asked for.
|
|
53
|
+
if (partition !== null && partition !== undefined) body.partition = partition
|
|
54
|
+
body.messages = batch
|
|
55
|
+
return body
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* What a buffer drains into: the sink, plus the identity that sink formats for.
|
|
61
|
+
* The default is the durable push, so a buffer created without one behaves
|
|
62
|
+
* exactly as buffers did before sinks existed.
|
|
63
|
+
*/
|
|
64
|
+
export const DURABLE_DESTINATION = { sink: DURABLE_SINK, queue: null, partition: null }
|
|
65
|
+
|
|
66
|
+
/** The ephemeral counterpart, bound to one (queue, partition). */
|
|
67
|
+
export function ephemeralDestination(queue, partition = null) {
|
|
68
|
+
return { sink: EPHEMERAL_SINK, queue, partition }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The durable buffer address, unchanged: `queue/partition`. Kept here next to
|
|
73
|
+
* its ephemeral sibling so the two can be compared at a glance.
|
|
74
|
+
*/
|
|
75
|
+
export function durableAddress(queue, partition) {
|
|
76
|
+
return `${queue}/${partition}`
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* The ephemeral buffer address: `eph:queue/partition`, or `eph:queue` when the
|
|
81
|
+
* caller named no partition (which is a different destination from any named
|
|
82
|
+
* one, because the broker picks, and a buffer must not merge the two).
|
|
83
|
+
*
|
|
84
|
+
* Same ambiguity as the durable address -- a queue named `a/b` collides with
|
|
85
|
+
* (`a`, `b`) -- inherited deliberately rather than fixed on one side only.
|
|
86
|
+
*/
|
|
87
|
+
export function ephemeralAddress(queue, partition = null) {
|
|
88
|
+
return partition === null || partition === undefined ? `eph:${queue}` : `eph:${queue}/${partition}`
|
|
89
|
+
}
|
|
@@ -10,7 +10,9 @@ export const generateUUID = () => {
|
|
|
10
10
|
|
|
11
11
|
//import { generateUUID } from '../../utils/uuid.js'
|
|
12
12
|
import { isValidUUID } from '../utils/validation.js'
|
|
13
|
+
import { durableAddress } from '../buffer/sinks.js'
|
|
13
14
|
import { QUEUE_DEFAULTS, CONSUME_DEFAULTS, POP_DEFAULTS } from '../utils/defaults.js'
|
|
15
|
+
import { checkConflationResponse, CONFLATION_UNSUPPORTED } from '../utils/conflation.js'
|
|
14
16
|
import * as logger from '../utils/logger.js'
|
|
15
17
|
|
|
16
18
|
export class QueueBuilder {
|
|
@@ -36,6 +38,7 @@ export class QueueBuilder {
|
|
|
36
38
|
#renewLeaseIntervalMillis = CONSUME_DEFAULTS.renewLeaseIntervalMillis
|
|
37
39
|
#subscriptionMode = CONSUME_DEFAULTS.subscriptionMode
|
|
38
40
|
#subscriptionFrom = CONSUME_DEFAULTS.subscriptionFrom
|
|
41
|
+
#conflation = CONSUME_DEFAULTS.conflation
|
|
39
42
|
#each = false
|
|
40
43
|
#maxPartitions = 1
|
|
41
44
|
|
|
@@ -265,6 +268,34 @@ export class QueueBuilder {
|
|
|
265
268
|
return this
|
|
266
269
|
}
|
|
267
270
|
|
|
271
|
+
/**
|
|
272
|
+
* Last-value delivery for this consumer group (PLAN_CONFLATION §1.1).
|
|
273
|
+
*
|
|
274
|
+
* A pop of a partition delivers exactly ONE message — the newest visible one
|
|
275
|
+
* — and commits past everything it skipped. For command-style queues where
|
|
276
|
+
* one partition is one logical task key ("recompute entity X"), a consumer
|
|
277
|
+
* behind a backlog then does the work once with the latest input instead of
|
|
278
|
+
* replaying every stale intermediate.
|
|
279
|
+
*
|
|
280
|
+
* It is a property of the GROUP, not of the call: it is persisted when the
|
|
281
|
+
* group first registers on the queue, and from then on the stored value wins
|
|
282
|
+
* for every consumer of that group. Declaring the opposite later does not
|
|
283
|
+
* flip it — the SDK warns once and keeps working. Default off; a group
|
|
284
|
+
* created without it behaves exactly as before.
|
|
285
|
+
*
|
|
286
|
+
* Requires broker >= 1.1.0. An older broker ignores the parameter and would
|
|
287
|
+
* quietly deliver the whole backlog, so the SDK raises on the first response
|
|
288
|
+
* that does not echo the flag rather than draining it silently.
|
|
289
|
+
*
|
|
290
|
+
* Refused by the broker (400) when combined with queue mode (no consumer
|
|
291
|
+
* group) or with autoAck, which commits at delivery and would turn the
|
|
292
|
+
* "the newest state is definitely processed" guarantee into at-most-once.
|
|
293
|
+
*/
|
|
294
|
+
conflation(enabled = true) {
|
|
295
|
+
this.#conflation = !!enabled
|
|
296
|
+
return this
|
|
297
|
+
}
|
|
298
|
+
|
|
268
299
|
each() {
|
|
269
300
|
this.#each = true
|
|
270
301
|
return this
|
|
@@ -292,6 +323,7 @@ export class QueueBuilder {
|
|
|
292
323
|
renewLeaseIntervalMillis: this.#renewLeaseIntervalMillis,
|
|
293
324
|
subscriptionMode: this.#subscriptionMode,
|
|
294
325
|
subscriptionFrom: this.#subscriptionFrom,
|
|
326
|
+
conflation: this.#conflation,
|
|
295
327
|
each: this.#each,
|
|
296
328
|
maxPartitions: this.#maxPartitions,
|
|
297
329
|
signal: options.signal
|
|
@@ -347,6 +379,11 @@ export class QueueBuilder {
|
|
|
347
379
|
if (this.#subscriptionMode) params.append('subscriptionMode', this.#subscriptionMode)
|
|
348
380
|
if (this.#subscriptionFrom) params.append('subscriptionFrom', this.#subscriptionFrom)
|
|
349
381
|
if (this.#maxPartitions > 1) params.append('partitions', this.#maxPartitions.toString())
|
|
382
|
+
// Conflation (PLAN_CONFLATION §3.1): sent ONLY when true, so an
|
|
383
|
+
// undeclared pop is byte-identical to today. NOTE: this is the pop
|
|
384
|
+
// builder; consume() builds its params in ConsumerManager#buildParams —
|
|
385
|
+
// see the comment below #buildPopPath about exactly this hazard.
|
|
386
|
+
if (this.#conflation) params.append('conflation', 'true')
|
|
350
387
|
|
|
351
388
|
// Generate affinity key for consistent routing to same backend
|
|
352
389
|
const affinityKey = this.#getAffinityKey()
|
|
@@ -355,6 +392,19 @@ export class QueueBuilder {
|
|
|
355
392
|
// rather than give up after a handful of tries (retryKind: 'pop').
|
|
356
393
|
const result = await this.#httpClient.get(`${path}?${params}`, this.#timeoutMillis + 5000, affinityKey, this.#wait ? 'pop' : null)
|
|
357
394
|
|
|
395
|
+
// Degrade-loudly (PLAN_CONFLATION §4), BEFORE the empty-response return:
|
|
396
|
+
// an old broker's empty pop is a bodiless 204 (result === null), and that
|
|
397
|
+
// is precisely the first thing a consumer on an idle queue sees. Also
|
|
398
|
+
// where a declaration conflict is warned about, exactly once.
|
|
399
|
+
if (this.#conflation) {
|
|
400
|
+
checkConflationResponse(result, {
|
|
401
|
+
queue: this.#queueName,
|
|
402
|
+
namespace: this.#namespace,
|
|
403
|
+
task: this.#task,
|
|
404
|
+
group: this.#group
|
|
405
|
+
})
|
|
406
|
+
}
|
|
407
|
+
|
|
358
408
|
if (!result || !result.messages) {
|
|
359
409
|
logger.log('QueueBuilder.pop', { status: 'no-messages' })
|
|
360
410
|
return []
|
|
@@ -364,6 +414,17 @@ export class QueueBuilder {
|
|
|
364
414
|
logger.log('QueueBuilder.pop', { status: 'success', count: messages.length })
|
|
365
415
|
return messages
|
|
366
416
|
} catch (error) {
|
|
417
|
+
// Conflation is the one thing this method does NOT swallow. The
|
|
418
|
+
// swallow-to-[] contract exists for transport faults, where [] means "no
|
|
419
|
+
// messages right now"; for a declared conflation it would mean "your
|
|
420
|
+
// last-value policy is not in force and you will never be told", which is
|
|
421
|
+
// the silent failure the feature is not allowed to have (§4). Both the
|
|
422
|
+
// missing-echo error and the broker's 400 refusals (queue mode / autoAck)
|
|
423
|
+
// are permanent config faults, so they raise.
|
|
424
|
+
if (error.code === CONFLATION_UNSUPPORTED || (this.#conflation && error.status === 400)) {
|
|
425
|
+
logger.error('QueueBuilder.pop', { error: error.message, status: error.status, code: error.code, conflation: true })
|
|
426
|
+
throw error
|
|
427
|
+
}
|
|
367
428
|
// Return empty array on error instead of throwing. This also covers a
|
|
368
429
|
// 429 whose retry429 policy was exhausted (bounded pop, or an explicit
|
|
369
430
|
// maxAttempts override) and a terminal 403 (e.g. cluster_suspended) --
|
|
@@ -397,6 +458,11 @@ export class QueueBuilder {
|
|
|
397
458
|
// copy nobody calls: the pop would keep working and the parameter would
|
|
398
459
|
// simply never arrive, which reads as a server-side mystery and not as a
|
|
399
460
|
// client bug.
|
|
461
|
+
//
|
|
462
|
+
// The pair that is still live and MUST be kept in sync is pop()'s inline
|
|
463
|
+
// params above and ConsumerManager#buildParams: every pop query parameter
|
|
464
|
+
// (subscriptionMode, subscriptionFrom, partitions, conflation, ...) has to be
|
|
465
|
+
// appended in BOTH, because pop() and consume() share no builder.
|
|
400
466
|
|
|
401
467
|
// ===========================
|
|
402
468
|
// Buffer Management Methods
|
|
@@ -406,7 +472,7 @@ export class QueueBuilder {
|
|
|
406
472
|
if (!this.#queueName) {
|
|
407
473
|
throw new Error('Queue name is required for buffer flush')
|
|
408
474
|
}
|
|
409
|
-
const queueAddress =
|
|
475
|
+
const queueAddress = durableAddress(this.#queueName, this.#partition)
|
|
410
476
|
logger.log('QueueBuilder.flushBuffer', { queueAddress })
|
|
411
477
|
await this.#bufferManager.flushBuffer(queueAddress)
|
|
412
478
|
}
|
|
@@ -643,7 +709,10 @@ class PushBuilder {
|
|
|
643
709
|
// off without awaiting would report success for messages the buffer never
|
|
644
710
|
// accepted -- the exact failure this bound exists to remove.
|
|
645
711
|
if (this.#bufferOptions) {
|
|
646
|
-
|
|
712
|
+
// No destination: the durable push is the default sink, so this address's
|
|
713
|
+
// buffer drains to POST /api/v1/push with a `{items}` body exactly as it
|
|
714
|
+
// did before sinks existed (buffer/sinks.js).
|
|
715
|
+
const queueAddress = durableAddress(this.#queueName, this.#partition)
|
|
647
716
|
const accepted = []
|
|
648
717
|
|
|
649
718
|
try {
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import * as logger from '../utils/logger.js'
|
|
6
|
+
import { checkConflationResponse, CONFLATION_UNSUPPORTED } from '../utils/conflation.js'
|
|
6
7
|
|
|
7
8
|
export class ConsumerManager {
|
|
8
9
|
#httpClient
|
|
@@ -47,6 +48,7 @@ export class ConsumerManager {
|
|
|
47
48
|
renewLeaseIntervalMillis,
|
|
48
49
|
subscriptionMode,
|
|
49
50
|
subscriptionFrom,
|
|
51
|
+
conflation,
|
|
50
52
|
each,
|
|
51
53
|
maxPartitions,
|
|
52
54
|
signal
|
|
@@ -68,7 +70,7 @@ export class ConsumerManager {
|
|
|
68
70
|
|
|
69
71
|
// Build the path and params for pop requests
|
|
70
72
|
const path = this.#buildPath(queue, partition, namespace, task)
|
|
71
|
-
const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions)
|
|
73
|
+
const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions, conflation)
|
|
72
74
|
|
|
73
75
|
// Generate affinity key for consistent routing to same backend
|
|
74
76
|
const affinityKey = this.#getAffinityKey(queue, partition, namespace, task, group)
|
|
@@ -88,7 +90,12 @@ export class ConsumerManager {
|
|
|
88
90
|
each,
|
|
89
91
|
signal,
|
|
90
92
|
group, // Pass consumer group to workers
|
|
91
|
-
affinityKey // Pass affinity key to workers
|
|
93
|
+
affinityKey, // Pass affinity key to workers
|
|
94
|
+
// Conflation was REQUESTED by this consumer: the worker has to check
|
|
95
|
+
// every response for the broker's echo (PLAN_CONFLATION §4) and needs
|
|
96
|
+
// the pop target to key the once-per-(queue,group) conflict warning.
|
|
97
|
+
conflation,
|
|
98
|
+
conflationCtx: { queue, namespace, task, group }
|
|
92
99
|
}))
|
|
93
100
|
}
|
|
94
101
|
|
|
@@ -113,7 +120,9 @@ export class ConsumerManager {
|
|
|
113
120
|
each,
|
|
114
121
|
signal,
|
|
115
122
|
group,
|
|
116
|
-
affinityKey
|
|
123
|
+
affinityKey,
|
|
124
|
+
conflation,
|
|
125
|
+
conflationCtx
|
|
117
126
|
} = options
|
|
118
127
|
|
|
119
128
|
logger.log('ConsumerManager.worker', { workerId, status: 'started', limit, idleMillis })
|
|
@@ -150,6 +159,16 @@ export class ConsumerManager {
|
|
|
150
159
|
const clientTimeout = wait ? timeoutMillis + 5000 : timeoutMillis
|
|
151
160
|
const result = await this.#httpClient.get(`${path}?${baseParams}`, clientTimeout, affinityKey, wait ? 'pop' : null)
|
|
152
161
|
|
|
162
|
+
// Degrade-loudly (PLAN_CONFLATION §4). Checked BEFORE the empty-response
|
|
163
|
+
// branch on purpose: a pre-1.1.0 broker answers an empty pop with a
|
|
164
|
+
// bodiless 204 (result === null), which is the first thing a consumer
|
|
165
|
+
// on an idle queue sees — and the whole point is to raise before a
|
|
166
|
+
// single message of a backlog is processed one-by-one. Throwing here
|
|
167
|
+
// leaves the loop through the catch below, which stops this worker.
|
|
168
|
+
if (conflation) {
|
|
169
|
+
checkConflationResponse(result, conflationCtx)
|
|
170
|
+
}
|
|
171
|
+
|
|
153
172
|
// Handle empty response
|
|
154
173
|
if (!result || !result.messages || result.messages.length === 0) {
|
|
155
174
|
if (wait) {
|
|
@@ -219,6 +238,17 @@ export class ConsumerManager {
|
|
|
219
238
|
}
|
|
220
239
|
|
|
221
240
|
} catch (error) {
|
|
241
|
+
// Conflation faults are terminal and are classified FIRST, ahead of the
|
|
242
|
+
// message-substring heuristics below: a consumer that asked for
|
|
243
|
+
// last-value delivery and is not getting it must stop, not retry
|
|
244
|
+
// (PLAN_CONFLATION §4). The broker's 400 refusals (queue mode /
|
|
245
|
+
// autoAck) are permanent config faults and stop the loop for the same
|
|
246
|
+
// reason — retrying them forever would be the silent version.
|
|
247
|
+
if (error.code === CONFLATION_UNSUPPORTED || (conflation && error.status === 400)) {
|
|
248
|
+
logger.error('ConsumerManager.worker', { workerId, status: 'conflation-unavailable', code: error.code, httpStatus: error.status, error: error.message })
|
|
249
|
+
throw error
|
|
250
|
+
}
|
|
251
|
+
|
|
222
252
|
// Check if this is a timeout error (expected for long polling)
|
|
223
253
|
const isTimeoutError = error.name === 'AbortError' ||
|
|
224
254
|
error.message?.includes('timeout')
|
|
@@ -442,7 +472,7 @@ export class ConsumerManager {
|
|
|
442
472
|
throw new Error('Must specify queue, namespace, or task')
|
|
443
473
|
}
|
|
444
474
|
|
|
445
|
-
#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions) {
|
|
475
|
+
#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions, conflation) {
|
|
446
476
|
const params = new URLSearchParams({
|
|
447
477
|
batch: batch.toString(),
|
|
448
478
|
wait: wait.toString(),
|
|
@@ -456,6 +486,11 @@ export class ConsumerManager {
|
|
|
456
486
|
if (task) params.append('task', task)
|
|
457
487
|
// v4 multi-partition pop: drain up to N sparse partitions per call.
|
|
458
488
|
if (maxPartitions && maxPartitions > 1) params.append('partitions', maxPartitions.toString())
|
|
489
|
+
// Conflation (PLAN_CONFLATION §3.1): last-value delivery for this group.
|
|
490
|
+
// Sent ONLY when true, so a consumer that never declares it puts no new
|
|
491
|
+
// bytes on the wire. THIS IS THE SECOND PARAMETER BUILDER — the pop() one
|
|
492
|
+
// lives inline in QueueBuilder.pop and must gain every parameter too.
|
|
493
|
+
if (conflation) params.append('conflation', 'true')
|
|
459
494
|
// NEVER send autoAck for consume - client always manages acking
|
|
460
495
|
// autoAck is only for pop() where server auto-acks immediately
|
|
461
496
|
|