queen-mq 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -4
- package/client-v2/Queen.js +18 -0
- package/client-v2/builders/QueueBuilder.js +104 -14
- package/client-v2/consumer/ConsumerManager.js +42 -10
- package/client-v2/streams/runtime/Runner.js +8 -3
- package/client-v2/utils/autopilot.js +168 -0
- package/client-v2/utils/defaults.js +9 -3
- package/package.json +3 -3
- package/test-v2/autopilot-unit/autopilotWire.test.js +410 -0
package/README.md
CHANGED
|
@@ -333,12 +333,64 @@ per-tenant work queues, per-device telemetry). Reduces network round-trips
|
|
|
333
333
|
from O(P) to O(P / N) while preserving per-partition FIFO ordering.
|
|
334
334
|
|
|
335
335
|
**When not to use:** few partitions, or each one busy enough to fill
|
|
336
|
-
`batch(B)` on its own.
|
|
337
|
-
|
|
336
|
+
`batch(B)` on its own. Leaving `.partitions()` unset hands the sweep width to
|
|
337
|
+
the broker (see Pop Autopilot below); `.partitions(1)` pins the legacy
|
|
338
|
+
single-partition behaviour.
|
|
338
339
|
|
|
339
340
|
`.partitions(N)` only applies to **wildcard** pops; specifying
|
|
340
341
|
`.partition('name')` ignores the cap.
|
|
341
342
|
|
|
343
|
+
### Pop Autopilot (Let the Broker Size the Pop)
|
|
344
|
+
|
|
345
|
+
Since 1.2, `batch` and `partitions` that you do **not** set are chosen by the
|
|
346
|
+
broker, per pop, from state the client cannot see: how many partitions of the
|
|
347
|
+
group are ready, how old their oldest ready message is, how fast messages are
|
|
348
|
+
arriving. The knobs you *do* set are never touched.
|
|
349
|
+
|
|
350
|
+
```javascript
|
|
351
|
+
// Both knobs are the broker's: it picks the sweep width and the budget.
|
|
352
|
+
await queen.queue('events').group('workers')
|
|
353
|
+
.consume(async (msgs) => { /* ... */ })
|
|
354
|
+
|
|
355
|
+
// One knob pinned, one delegated: this consumer stays on one partition
|
|
356
|
+
// forever, and the broker sizes the batch for it.
|
|
357
|
+
await queen.queue('events').group('workers').partitions(1)
|
|
358
|
+
.consume(async (msgs) => { /* ... */ })
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
The request carries `autopilot=true` and simply omits the delegated knobs.
|
|
362
|
+
Setting both leaves nothing to decide, so nothing changes on the wire at all.
|
|
363
|
+
|
|
364
|
+
**Two ways to switch it off**, both restoring the previous client-side
|
|
365
|
+
defaults (batch 1, partitions 1) byte for byte:
|
|
366
|
+
|
|
367
|
+
```javascript
|
|
368
|
+
await queen.queue('events').autopilot(false).consume(handler)
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
```bash
|
|
372
|
+
QUEEN_SDK_POP_AUTOPILOT=off # whole process, read once at client creation
|
|
373
|
+
```
|
|
374
|
+
|
|
375
|
+
**What the broker chose** rides back on the response and is there for the
|
|
376
|
+
reading, along with an optional pacing hint the consume loop honours in place
|
|
377
|
+
of its own delay between empty polls:
|
|
378
|
+
|
|
379
|
+
```javascript
|
|
380
|
+
const { messages, autopilot } = await queen.queue('events').group('workers').popResult()
|
|
381
|
+
if (autopilot) {
|
|
382
|
+
console.log(`${autopilot.partitions} partitions, batch ${autopilot.batch}, ` +
|
|
383
|
+
`poll again in ${autopilot.waitMillis}ms`)
|
|
384
|
+
}
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
**Requires broker >= 1.2.** An older broker ignores the parameter, so the
|
|
388
|
+
omitted knobs take *its* defaults (batch 200, partitions 1) instead of the old
|
|
389
|
+
client-side ones. That is a sizing difference and nothing else — no message is
|
|
390
|
+
lost, reordered or duplicated — so unlike conflation it degrades silently and
|
|
391
|
+
on purpose. Pin the values explicitly, or turn autopilot off, if you need the
|
|
392
|
+
old numbers against an old broker.
|
|
393
|
+
|
|
342
394
|
### Transactions (Atomic Operations)
|
|
343
395
|
|
|
344
396
|
```javascript
|
|
@@ -691,10 +743,11 @@ await queen.queue('q').buffer({ messageCount: 100, timeMillis: 1000 }).push([...
|
|
|
691
743
|
### Pop
|
|
692
744
|
|
|
693
745
|
```javascript
|
|
694
|
-
const msgs = await queen.queue('q').pop()
|
|
746
|
+
const msgs = await queen.queue('q').pop() // broker-sized (see Pop Autopilot)
|
|
695
747
|
const msgs = await queen.queue('q').batch(10).pop()
|
|
696
748
|
const msgs = await queen.queue('q').batch(10).wait(true).pop()
|
|
697
749
|
const msgs = await queen.queue('q').batch(200).partitions(50).pop() // multi-partition pop
|
|
750
|
+
const { messages, autopilot } = await queen.queue('q').popResult() // + what the broker chose
|
|
698
751
|
```
|
|
699
752
|
|
|
700
753
|
### Consume
|
|
@@ -828,7 +881,8 @@ await queen.close() // Flush buffers and close connections
|
|
|
828
881
|
```javascript
|
|
829
882
|
{
|
|
830
883
|
concurrency: 1,
|
|
831
|
-
batch: 1,
|
|
884
|
+
batch: 1, // autopilot OFF only -- unset means the broker sizes it
|
|
885
|
+
partitions: 1, // autopilot OFF only -- unset means the broker sizes it
|
|
832
886
|
autoAck: true,
|
|
833
887
|
wait: true, // Long polling
|
|
834
888
|
timeoutMillis: 30000,
|
|
@@ -837,6 +891,11 @@ await queen.close() // Flush buffers and close connections
|
|
|
837
891
|
}
|
|
838
892
|
```
|
|
839
893
|
|
|
894
|
+
`batch` and `partitions` are the **autopilot-off** defaults: with autopilot on
|
|
895
|
+
(the default) a knob you never set is not defaulted at all, it is delegated to
|
|
896
|
+
the broker. These values are what comes back with `.autopilot(false)` or
|
|
897
|
+
`QUEEN_SDK_POP_AUTOPILOT=off`.
|
|
898
|
+
|
|
840
899
|
---
|
|
841
900
|
|
|
842
901
|
## Logging
|
package/client-v2/Queen.js
CHANGED
|
@@ -14,6 +14,7 @@ import { Ephemeral } from './ephemeral/Ephemeral.js'
|
|
|
14
14
|
import { StreamBuilder } from './stream/StreamBuilder.js'
|
|
15
15
|
import { StreamConsumer } from './stream/StreamConsumer.js'
|
|
16
16
|
import { Admin } from './admin/Admin.js'
|
|
17
|
+
import { popAutopilotDisabledByEnv } from './utils/autopilot.js'
|
|
17
18
|
import { CLIENT_DEFAULTS } from './utils/defaults.js'
|
|
18
19
|
import { validateUrl, validateUrls } from './utils/validation.js'
|
|
19
20
|
import * as logger from './utils/logger.js'
|
|
@@ -68,6 +69,11 @@ export class Queen {
|
|
|
68
69
|
#admin = null
|
|
69
70
|
#kv = null
|
|
70
71
|
#ephemeral = null
|
|
72
|
+
// Process-wide kill switch for pop autopilot, read from
|
|
73
|
+
// QUEEN_SDK_POP_AUTOPILOT once here rather than on every pop: it is a
|
|
74
|
+
// deployment-level rollback, and re-reading it per request would let a
|
|
75
|
+
// running process change wire shape halfway through.
|
|
76
|
+
#autopilotOff = false
|
|
71
77
|
|
|
72
78
|
constructor(config = {}) {
|
|
73
79
|
// Configure custom logger before anything else.
|
|
@@ -83,6 +89,9 @@ export class Queen {
|
|
|
83
89
|
// Normalize config
|
|
84
90
|
this.#config = this.#normalizeConfig(config)
|
|
85
91
|
|
|
92
|
+
// Pop autopilot: on unless the environment rolls it back (utils/autopilot.js).
|
|
93
|
+
this.#autopilotOff = popAutopilotDisabledByEnv()
|
|
94
|
+
|
|
86
95
|
// Create HTTP client
|
|
87
96
|
this.#httpClient = this.#createHttpClient()
|
|
88
97
|
|
|
@@ -97,6 +106,15 @@ export class Queen {
|
|
|
97
106
|
logger.log('Queen.constructor', { status: 'initialized', urls: this.#config.urls.length, handleSignals: this.#config.handleSignals })
|
|
98
107
|
}
|
|
99
108
|
|
|
109
|
+
/**
|
|
110
|
+
* Whether pop autopilot is off for this client because the environment asked
|
|
111
|
+
* (QUEEN_SDK_POP_AUTOPILOT). Read by the builders; a per-call
|
|
112
|
+
* `.autopilot(...)` still outranks it.
|
|
113
|
+
*/
|
|
114
|
+
get autopilotOff() {
|
|
115
|
+
return this.#autopilotOff
|
|
116
|
+
}
|
|
117
|
+
|
|
100
118
|
#normalizeConfig(config) {
|
|
101
119
|
// Handle different input formats
|
|
102
120
|
if (typeof config === 'string') {
|
|
@@ -13,6 +13,7 @@ import { isValidUUID } from '../utils/validation.js'
|
|
|
13
13
|
import { durableAddress } from '../buffer/sinks.js'
|
|
14
14
|
import { QUEUE_DEFAULTS, CONSUME_DEFAULTS, POP_DEFAULTS } from '../utils/defaults.js'
|
|
15
15
|
import { checkConflationResponse, CONFLATION_UNSUPPORTED } from '../utils/conflation.js'
|
|
16
|
+
import { popSizing, parseAutopilotDecision } from '../utils/autopilot.js'
|
|
16
17
|
import * as logger from '../utils/logger.js'
|
|
17
18
|
|
|
18
19
|
export class QueueBuilder {
|
|
@@ -28,7 +29,12 @@ export class QueueBuilder {
|
|
|
28
29
|
|
|
29
30
|
// Consume options
|
|
30
31
|
#concurrency = CONSUME_DEFAULTS.concurrency
|
|
31
|
-
|
|
32
|
+
// batch / maxPartitions hold the USER's value, and null means the setter was
|
|
33
|
+
// never called -- which is the dimension pop autopilot gets to choose. The
|
|
34
|
+
// client-side defaults are applied at emission time (utils/autopilot.js), not
|
|
35
|
+
// here, because filling them in here would erase the difference between
|
|
36
|
+
// "never called batch()" and "called batch(1)".
|
|
37
|
+
#batch = null
|
|
32
38
|
#limit = CONSUME_DEFAULTS.limit
|
|
33
39
|
#idleMillis = CONSUME_DEFAULTS.idleMillis
|
|
34
40
|
#autoAck = CONSUME_DEFAULTS.autoAck
|
|
@@ -40,7 +46,10 @@ export class QueueBuilder {
|
|
|
40
46
|
#subscriptionFrom = CONSUME_DEFAULTS.subscriptionFrom
|
|
41
47
|
#conflation = CONSUME_DEFAULTS.conflation
|
|
42
48
|
#each = false
|
|
43
|
-
#maxPartitions =
|
|
49
|
+
#maxPartitions = null
|
|
50
|
+
// Per-call override for pop autopilot: null = the client default (on unless
|
|
51
|
+
// QUEEN_SDK_POP_AUTOPILOT turned it off).
|
|
52
|
+
#autopilot = null
|
|
44
53
|
|
|
45
54
|
// Buffer options
|
|
46
55
|
#bufferOptions = null
|
|
@@ -214,8 +223,15 @@ export class QueueBuilder {
|
|
|
214
223
|
return this
|
|
215
224
|
}
|
|
216
225
|
|
|
226
|
+
/**
|
|
227
|
+
* Pin the message budget for one pop. Leave it unset and the broker sizes it
|
|
228
|
+
* (see `autopilot`), where it used to mean the client-side default of 1.
|
|
229
|
+
*
|
|
230
|
+
* `batch(0)` is not "a batch of zero" and never was: it is the absence of an
|
|
231
|
+
* opinion, so it reads as unset.
|
|
232
|
+
*/
|
|
217
233
|
batch(size) {
|
|
218
|
-
this.#batch =
|
|
234
|
+
this.#batch = size > 0 ? size : null
|
|
219
235
|
return this
|
|
220
236
|
}
|
|
221
237
|
|
|
@@ -228,13 +244,45 @@ export class QueueBuilder {
|
|
|
228
244
|
* partitions, in a single network round-trip. All N share one leaseId
|
|
229
245
|
* (renewing once extends them all).
|
|
230
246
|
*
|
|
231
|
-
*
|
|
247
|
+
* Leave it unset and the broker chooses the sweep width (see `autopilot`);
|
|
248
|
+
* `partitions(1)` pins the legacy single-partition behaviour, which is a
|
|
249
|
+
* decision the broker is told about and never overrides.
|
|
232
250
|
*/
|
|
233
251
|
partitions(n) {
|
|
234
|
-
this.#maxPartitions =
|
|
252
|
+
this.#maxPartitions = n > 0 ? n : null
|
|
253
|
+
return this
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Turn broker-side pop sizing on or off for this builder.
|
|
258
|
+
*
|
|
259
|
+
* On (the default) the broker chooses `batch` and `partitions` for the pops
|
|
260
|
+
* of this builder. Even then, a `batch` or `partitions` set explicitly
|
|
261
|
+
* travels on the wire as it always did and is never second-guessed: autopilot
|
|
262
|
+
* only ever fills the knobs left unset.
|
|
263
|
+
*
|
|
264
|
+
* `autopilot(false)` restores this SDK's pre-1.2 behaviour byte for byte: the
|
|
265
|
+
* client-side defaults come back (batch 1, partitions 1) and no autopilot
|
|
266
|
+
* parameter is sent. QUEEN_SDK_POP_AUTOPILOT=off does the same for a whole
|
|
267
|
+
* process; an explicit call here outranks the environment in both directions.
|
|
268
|
+
*
|
|
269
|
+
* Setting BOTH batch and partitions leaves autopilot nothing to decide, so no
|
|
270
|
+
* autopilot parameter is sent in that case either, whatever this flag says.
|
|
271
|
+
*/
|
|
272
|
+
autopilot(enabled = true) {
|
|
273
|
+
this.#autopilot = !!enabled
|
|
235
274
|
return this
|
|
236
275
|
}
|
|
237
276
|
|
|
277
|
+
/**
|
|
278
|
+
* This builder's resolved autopilot decision: its own flag when set,
|
|
279
|
+
* otherwise the client-wide default settled in the Queen constructor.
|
|
280
|
+
*/
|
|
281
|
+
#autopilotEnabled() {
|
|
282
|
+
if (this.#autopilot !== null) return this.#autopilot
|
|
283
|
+
return !this.#queen || !this.#queen.autopilotOff
|
|
284
|
+
}
|
|
285
|
+
|
|
238
286
|
limit(count) {
|
|
239
287
|
this.#limit = count
|
|
240
288
|
return this
|
|
@@ -326,6 +374,11 @@ export class QueueBuilder {
|
|
|
326
374
|
conflation: this.#conflation,
|
|
327
375
|
each: this.#each,
|
|
328
376
|
maxPartitions: this.#maxPartitions,
|
|
377
|
+
// Resolved here so ConsumerManager sees a decision and not a null. batch
|
|
378
|
+
// and maxPartitions keep their null when autopilot is on, and that null
|
|
379
|
+
// has to survive all the way to #buildParams: it is the ONLY record that
|
|
380
|
+
// the user said nothing about that dimension.
|
|
381
|
+
autopilot: this.#autopilotEnabled(),
|
|
329
382
|
signal: options.signal
|
|
330
383
|
}
|
|
331
384
|
|
|
@@ -355,7 +408,27 @@ export class QueueBuilder {
|
|
|
355
408
|
return this
|
|
356
409
|
}
|
|
357
410
|
|
|
411
|
+
/**
|
|
412
|
+
* Claim messages and report what the broker chose for this pop.
|
|
413
|
+
*
|
|
414
|
+
* Same call as `pop()` — this is the shape that also carries the additive
|
|
415
|
+
* `autopilot` echo, which is null when this pop did not engage autopilot or
|
|
416
|
+
* the broker is older than 1.2.
|
|
417
|
+
*
|
|
418
|
+
* const { messages, autopilot } = await client.queue('events').group('w').popResult()
|
|
419
|
+
* if (autopilot) console.log(autopilot.partitions, autopilot.batch, autopilot.waitMillis)
|
|
420
|
+
*
|
|
421
|
+
* @returns {Promise<{messages: object[], autopilot: {partitions: number, batch: number, waitMillis: number}|null}>}
|
|
422
|
+
*/
|
|
423
|
+
async popResult() {
|
|
424
|
+
return this.#popWithDecision()
|
|
425
|
+
}
|
|
426
|
+
|
|
358
427
|
async pop() {
|
|
428
|
+
return (await this.#popWithDecision()).messages
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
async #popWithDecision() {
|
|
359
432
|
logger.log('QueueBuilder.pop', { queue: this.#queueName, partition: this.#partition, namespace: this.#namespace, task: this.#task, batch: this.#batch, wait: this.#wait, group: this.#group })
|
|
360
433
|
|
|
361
434
|
try {
|
|
@@ -365,20 +438,32 @@ export class QueueBuilder {
|
|
|
365
438
|
// Override autoAck to false unless explicitly set
|
|
366
439
|
const effectiveAutoAck = this.#autoAck !== CONSUME_DEFAULTS.autoAck ? this.#autoAck : POP_DEFAULTS.autoAck
|
|
367
440
|
|
|
368
|
-
//
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
441
|
+
// Batch, partitions and with them the autopilot flag. The RULE for which
|
|
442
|
+
// of the three travel lives in one place (utils/autopilot.js) because
|
|
443
|
+
// consume() builds its query string separately; only the PLACEMENT is
|
|
444
|
+
// here, and it is the pre-autopilot placement so an autopilot-off request
|
|
445
|
+
// is byte-identical to the one this SDK used to send.
|
|
446
|
+
const sizing = popSizing({
|
|
447
|
+
batch: this.#batch,
|
|
448
|
+
maxPartitions: this.#maxPartitions,
|
|
449
|
+
fallbackBatch: POP_DEFAULTS.batch,
|
|
450
|
+
autopilot: this.#autopilotEnabled()
|
|
373
451
|
})
|
|
374
452
|
|
|
453
|
+
// Build params with correct autoAck for pop
|
|
454
|
+
const params = new URLSearchParams()
|
|
455
|
+
if (sizing.autopilot) params.append('autopilot', 'true')
|
|
456
|
+
if (sizing.batch !== null) params.append('batch', sizing.batch)
|
|
457
|
+
params.append('wait', this.#wait.toString())
|
|
458
|
+
params.append('timeout', this.#timeoutMillis.toString())
|
|
459
|
+
|
|
375
460
|
if (this.#group) params.append('consumerGroup', this.#group)
|
|
376
461
|
if (this.#namespace) params.append('namespace', this.#namespace)
|
|
377
462
|
if (this.#task) params.append('task', this.#task)
|
|
378
463
|
if (effectiveAutoAck) params.append('autoAck', 'true')
|
|
379
464
|
if (this.#subscriptionMode) params.append('subscriptionMode', this.#subscriptionMode)
|
|
380
465
|
if (this.#subscriptionFrom) params.append('subscriptionFrom', this.#subscriptionFrom)
|
|
381
|
-
if (
|
|
466
|
+
if (sizing.partitions !== null) params.append('partitions', sizing.partitions)
|
|
382
467
|
// Conflation (PLAN_CONFLATION §3.1): sent ONLY when true, so an
|
|
383
468
|
// undeclared pop is byte-identical to today. NOTE: this is the pop
|
|
384
469
|
// builder; consume() builds its params in ConsumerManager#buildParams —
|
|
@@ -405,14 +490,19 @@ export class QueueBuilder {
|
|
|
405
490
|
})
|
|
406
491
|
}
|
|
407
492
|
|
|
493
|
+
// The broker's own account of how it sized this pop, when the request
|
|
494
|
+
// engaged autopilot and the answer had a body to carry it (a bodiless 204
|
|
495
|
+
// cannot, so an empty short pop reports null).
|
|
496
|
+
const autopilot = parseAutopilotDecision(result)
|
|
497
|
+
|
|
408
498
|
if (!result || !result.messages) {
|
|
409
499
|
logger.log('QueueBuilder.pop', { status: 'no-messages' })
|
|
410
|
-
return []
|
|
500
|
+
return { messages: [], autopilot }
|
|
411
501
|
}
|
|
412
502
|
|
|
413
503
|
const messages = result.messages.filter(msg => msg != null)
|
|
414
504
|
logger.log('QueueBuilder.pop', { status: 'success', count: messages.length })
|
|
415
|
-
return messages
|
|
505
|
+
return { messages, autopilot }
|
|
416
506
|
} catch (error) {
|
|
417
507
|
// Conflation is the one thing this method does NOT swallow. The
|
|
418
508
|
// swallow-to-[] contract exists for transport faults, where [] means "no
|
|
@@ -431,7 +521,7 @@ export class QueueBuilder {
|
|
|
431
521
|
// both are logged with their `.code` rather than raising, matching this
|
|
432
522
|
// method's existing swallow-to-[] contract.
|
|
433
523
|
logger.error('QueueBuilder.pop', { error: error.message, status: error.status, code: error.code })
|
|
434
|
-
return []
|
|
524
|
+
return { messages: [], autopilot: null }
|
|
435
525
|
}
|
|
436
526
|
}
|
|
437
527
|
|
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
|
|
5
5
|
import * as logger from '../utils/logger.js'
|
|
6
6
|
import { checkConflationResponse, CONFLATION_UNSUPPORTED } from '../utils/conflation.js'
|
|
7
|
+
import { popSizing, parseAutopilotDecision, emptyPollDelayMillis } from '../utils/autopilot.js'
|
|
8
|
+
import { CONSUME_DEFAULTS } from '../utils/defaults.js'
|
|
7
9
|
|
|
8
10
|
export class ConsumerManager {
|
|
9
11
|
#httpClient
|
|
@@ -51,6 +53,7 @@ export class ConsumerManager {
|
|
|
51
53
|
conflation,
|
|
52
54
|
each,
|
|
53
55
|
maxPartitions,
|
|
56
|
+
autopilot,
|
|
54
57
|
signal
|
|
55
58
|
} = options
|
|
56
59
|
|
|
@@ -70,7 +73,7 @@ export class ConsumerManager {
|
|
|
70
73
|
|
|
71
74
|
// Build the path and params for pop requests
|
|
72
75
|
const path = this.#buildPath(queue, partition, namespace, task)
|
|
73
|
-
const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions, conflation)
|
|
76
|
+
const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions, conflation, this.#autopilotEnabled(autopilot))
|
|
74
77
|
|
|
75
78
|
// Generate affinity key for consistent routing to same backend
|
|
76
79
|
const affinityKey = this.#getAffinityKey(queue, partition, namespace, task, group)
|
|
@@ -174,8 +177,11 @@ export class ConsumerManager {
|
|
|
174
177
|
if (wait) {
|
|
175
178
|
continue // Long polling timeout, retry
|
|
176
179
|
} else {
|
|
177
|
-
// Short delay before retry
|
|
178
|
-
|
|
180
|
+
// Short delay before retry -- the broker's advised pacing when this
|
|
181
|
+
// pop engaged autopilot and the broker had an opinion (it knows the
|
|
182
|
+
// arrival rate on this queue and this client does not), otherwise
|
|
183
|
+
// the historical 100ms.
|
|
184
|
+
await new Promise(resolve => setTimeout(resolve, emptyPollDelayMillis(parseAutopilotDecision(result))))
|
|
179
185
|
continue
|
|
180
186
|
}
|
|
181
187
|
}
|
|
@@ -472,20 +478,46 @@ export class ConsumerManager {
|
|
|
472
478
|
throw new Error('Must specify queue, namespace, or task')
|
|
473
479
|
}
|
|
474
480
|
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
481
|
+
/**
|
|
482
|
+
* The autopilot decision for one consume: the caller's explicit option if
|
|
483
|
+
* there is one, otherwise the client-wide default settled in the Queen
|
|
484
|
+
* constructor. The builder path has already resolved it; the undefined case
|
|
485
|
+
* is for callers that drive ConsumerManager with options of their own.
|
|
486
|
+
*/
|
|
487
|
+
#autopilotEnabled(autopilot) {
|
|
488
|
+
if (typeof autopilot === 'boolean') return autopilot
|
|
489
|
+
return !this.#queen || !this.#queen.autopilotOff
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions, conflation, autopilot = true) {
|
|
493
|
+
// Batch, partitions and with them the autopilot flag. null/0 means the user
|
|
494
|
+
// set nothing (QueueBuilder leaves it that way on purpose), which is the
|
|
495
|
+
// dimension the broker gets to choose. THE RULE lives in one place
|
|
496
|
+
// (utils/autopilot.js) precisely because this is the SECOND parameter
|
|
497
|
+
// builder; only the placement of the keys is here, and it is the
|
|
498
|
+
// pre-autopilot placement so an autopilot-off request is byte-identical.
|
|
499
|
+
const sizing = popSizing({
|
|
500
|
+
batch,
|
|
501
|
+
maxPartitions,
|
|
502
|
+
fallbackBatch: CONSUME_DEFAULTS.batch,
|
|
503
|
+
autopilot
|
|
480
504
|
})
|
|
481
505
|
|
|
506
|
+
const params = new URLSearchParams()
|
|
507
|
+
if (sizing.autopilot) params.append('autopilot', 'true')
|
|
508
|
+
if (sizing.batch !== null) params.append('batch', sizing.batch)
|
|
509
|
+
params.append('wait', wait.toString())
|
|
510
|
+
params.append('timeout', timeoutMillis.toString()) // Server expects 'timeout', not 'timeoutMillis'
|
|
511
|
+
|
|
482
512
|
if (group) params.append('consumerGroup', group)
|
|
483
513
|
if (subscriptionMode) params.append('subscriptionMode', subscriptionMode)
|
|
484
514
|
if (subscriptionFrom) params.append('subscriptionFrom', subscriptionFrom)
|
|
485
515
|
if (namespace) params.append('namespace', namespace)
|
|
486
516
|
if (task) params.append('task', task)
|
|
487
|
-
// v4 multi-partition pop: drain up to N sparse partitions per call.
|
|
488
|
-
|
|
517
|
+
// v4 multi-partition pop: drain up to N sparse partitions per call. Under
|
|
518
|
+
// autopilot a pinned width travels even when it is 1, because 1 is then a
|
|
519
|
+
// decision and not the absence of one.
|
|
520
|
+
if (sizing.partitions !== null) params.append('partitions', sizing.partitions)
|
|
489
521
|
// Conflation (PLAN_CONFLATION §3.1): last-value delivery for this group.
|
|
490
522
|
// Sent ONLY when true, so a consumer that never declares it puts no new
|
|
491
523
|
// bytes on the wire. THIS IS THE SECOND PARAMETER BUILDER — the pop() one
|
|
@@ -240,9 +240,14 @@ export class Runner {
|
|
|
240
240
|
.timeoutMillis(this.maxWaitMillis)
|
|
241
241
|
.group(this.consumerGroup)
|
|
242
242
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
243
|
+
// Unconditional, and it has to be: the runtime has already defaulted
|
|
244
|
+
// maxPartitions (to 4), so this is always a decision the streams layer
|
|
245
|
+
// made. Skipping the call for maxPartitions === 1 used to be a harmless
|
|
246
|
+
// optimisation -- 1 was what an omitted `partitions` meant on the wire --
|
|
247
|
+
// but with pop autopilot an omitted `partitions` means "broker, you
|
|
248
|
+
// choose", which would widen a query that explicitly asked for one
|
|
249
|
+
// partition per pop.
|
|
250
|
+
qb = qb.partitions(this.maxPartitions)
|
|
246
251
|
if (this.subscriptionMode) qb = qb.subscriptionMode(this.subscriptionMode)
|
|
247
252
|
if (this.subscriptionFrom) qb = qb.subscriptionFrom(this.subscriptionFrom)
|
|
248
253
|
if (this.conflation) qb = qb.conflation(true)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pop autopilot, client side.
|
|
3
|
+
*
|
|
4
|
+
* The broker owns a controller that sizes a pop from state this client cannot
|
|
5
|
+
* see: how many partitions of the (queue, group) are ready, how old their
|
|
6
|
+
* oldest ready message is, at what rate messages are arriving. Two knobs are
|
|
7
|
+
* under its control — `partitions` (the sweep width) and `batch` (the message
|
|
8
|
+
* budget for the sweep).
|
|
9
|
+
*
|
|
10
|
+
* THE RULE, and it is the only one: an explicit user value is sacred. Autopilot
|
|
11
|
+
* applies ONLY to the knobs the user left unset, and it applies to them one by
|
|
12
|
+
* one. A consumer that pins `partitions(1)` and says nothing about batch keeps
|
|
13
|
+
* its single-partition claim forever and lets the broker size the batch; the
|
|
14
|
+
* pinned dimension is never "adjusted", not even towards a value the controller
|
|
15
|
+
* would consider better.
|
|
16
|
+
*
|
|
17
|
+
* The wire shape follows the conflation precedent (see conflation.js): a client
|
|
18
|
+
* that is not engaging autopilot sends the byte-identical request it sent
|
|
19
|
+
* before this feature existed.
|
|
20
|
+
*
|
|
21
|
+
* autopilot=true emitted ONLY when at least one of the two knobs is
|
|
22
|
+
* being left to the broker. Never as autopilot=false.
|
|
23
|
+
* partitions / batch OMITTED for the dimensions the broker is choosing,
|
|
24
|
+
* sent exactly as before for the ones the user set.
|
|
25
|
+
*
|
|
26
|
+
* WHAT AN OLD BROKER DOES, and why there is no capability check here. A broker
|
|
27
|
+
* older than 1.2 ignores unknown query params: the request succeeds, and the
|
|
28
|
+
* two omitted knobs fall back to the SERVER-side defaults (batch 200,
|
|
29
|
+
* partitions 1) instead of the old client-side ones. That is a sizing
|
|
30
|
+
* difference, not a correctness one — nothing is lost, misordered or delivered
|
|
31
|
+
* twice — so unlike conflation (which silently hands a last-value consumer a
|
|
32
|
+
* whole backlog, hence CONFLATION_UNSUPPORTED) this degrades quietly and on
|
|
33
|
+
* purpose. Callers who need the old numbers against an old broker set them
|
|
34
|
+
* explicitly, or turn autopilot off.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The environment variable that disables pop autopilot for a whole process:
|
|
39
|
+
* QUEEN_SDK_POP_AUTOPILOT=off restores the client-side defaults this SDK
|
|
40
|
+
* applied before autopilot existed, byte for byte. It is read once, in the
|
|
41
|
+
* Queen constructor, so a single deployment can be rolled back without touching
|
|
42
|
+
* code.
|
|
43
|
+
*
|
|
44
|
+
* "off", "false", "0", "no" and "disabled" all disable it (case-insensitive,
|
|
45
|
+
* surrounding space ignored). Every other value, including the empty one,
|
|
46
|
+
* leaves autopilot on.
|
|
47
|
+
*/
|
|
48
|
+
export const ENV_POP_AUTOPILOT = 'QUEEN_SDK_POP_AUTOPILOT'
|
|
49
|
+
|
|
50
|
+
const DISABLING_VALUES = new Set(['off', 'false', '0', 'no', 'disabled'])
|
|
51
|
+
|
|
52
|
+
/** Whether ENV_POP_AUTOPILOT asks for the pre-autopilot behavior. */
|
|
53
|
+
export function popAutopilotDisabledByEnv() {
|
|
54
|
+
// `process` is absent in a browser bundle; there the variable cannot be set
|
|
55
|
+
// and autopilot is simply on.
|
|
56
|
+
const raw = typeof process !== 'undefined' && process.env ? process.env[ENV_POP_AUTOPILOT] : undefined
|
|
57
|
+
return DISABLING_VALUES.has(String(raw ?? '').trim().toLowerCase())
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The batch/partitions/autopilot decision for one pop — the values that travel
|
|
62
|
+
* and the ones that do not.
|
|
63
|
+
*
|
|
64
|
+
* IT EXISTS SO THERE IS EXACTLY ONE COPY OF THE EMISSION RULE. pop() and
|
|
65
|
+
* consume() build their query strings separately (QueueBuilder.pop's inline
|
|
66
|
+
* params vs ConsumerManager#buildParams) and the two have drifted before — the
|
|
67
|
+
* standing comment in QueueBuilder.js is there because of it. A rule with three
|
|
68
|
+
* branches and a per-dimension carve-out is precisely the kind that gets copied
|
|
69
|
+
* wrong, so both builders call this and then only PLACE what it returns; where
|
|
70
|
+
* each key sits in the query string stays with the builder, because the
|
|
71
|
+
* pre-autopilot key order is part of what "byte-identical" means here.
|
|
72
|
+
*
|
|
73
|
+
* @param {object} opts
|
|
74
|
+
* @param {number|null} opts.batch - the USER's batch. null/0/undefined means
|
|
75
|
+
* unset: the dimension the broker gets to choose. Neither builder may
|
|
76
|
+
* substitute a default before calling this.
|
|
77
|
+
* @param {number|null} opts.maxPartitions - the USER's sweep width, same
|
|
78
|
+
* convention.
|
|
79
|
+
* @param {number} opts.fallbackBatch - client-side default applied to an unset
|
|
80
|
+
* batch when autopilot is NOT engaged.
|
|
81
|
+
* @param {boolean} opts.autopilot - the resolved decision for this call.
|
|
82
|
+
* @returns {{autopilot: boolean, batch: string|null, partitions: string|null}}
|
|
83
|
+
* Strings are ready to append; null means "this key does not travel".
|
|
84
|
+
*/
|
|
85
|
+
export function popSizing({ batch, maxPartitions, fallbackBatch, autopilot }) {
|
|
86
|
+
const batchSet = typeof batch === 'number' && batch > 0
|
|
87
|
+
const partitionsSet = typeof maxPartitions === 'number' && maxPartitions > 0
|
|
88
|
+
|
|
89
|
+
// Note the case that looks like an omission and is not: when the user set
|
|
90
|
+
// BOTH knobs there is nothing left for the controller to decide, so
|
|
91
|
+
// autopilot=true is NOT emitted and the request is byte-identical to the one
|
|
92
|
+
// this SDK sent before autopilot existed. Sending the flag anyway would be
|
|
93
|
+
// harmless on the broker and dishonest in a packet capture.
|
|
94
|
+
if (autopilot && !(batchSet && partitionsSet)) {
|
|
95
|
+
return {
|
|
96
|
+
autopilot: true,
|
|
97
|
+
batch: batchSet ? String(batch) : null,
|
|
98
|
+
partitions: partitionsSet ? String(maxPartitions) : null
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return {
|
|
103
|
+
autopilot: false,
|
|
104
|
+
batch: String(batchSet ? batch : fallbackBatch),
|
|
105
|
+
// The legacy gate: partitions travels only above 1, because 1 IS the
|
|
106
|
+
// server-side default and a v4-era client never sent it.
|
|
107
|
+
partitions: partitionsSet && maxPartitions > 1 ? String(maxPartitions) : null
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* What the broker chose for one pop, echoed back in the response under
|
|
113
|
+
* "autopilot" when the request engaged autopilot. It is additive: a broker that
|
|
114
|
+
* does not send it, or a pop that never asked, yields null everywhere it is
|
|
115
|
+
* exposed.
|
|
116
|
+
*
|
|
117
|
+
* Reading it is optional — the messages are already sized by it — but it is the
|
|
118
|
+
* only way to see the controller working from the client side, and the only
|
|
119
|
+
* input to the empty-poll pacing below.
|
|
120
|
+
*
|
|
121
|
+
* Unknown keys inside it are ignored, and an unknown-shaped value is treated as
|
|
122
|
+
* absent rather than as an error: this field is the broker telling the client
|
|
123
|
+
* what it did, and a client that refuses to run because a newer broker grew a
|
|
124
|
+
* fourth number would be a self-inflicted outage.
|
|
125
|
+
*
|
|
126
|
+
* @param {object|null} result - parsed pop response (null for a 204)
|
|
127
|
+
* @returns {{partitions: number, batch: number, waitMillis: number}|null}
|
|
128
|
+
*/
|
|
129
|
+
export function parseAutopilotDecision(result) {
|
|
130
|
+
const raw = result && typeof result === 'object' ? result.autopilot : null
|
|
131
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null
|
|
132
|
+
|
|
133
|
+
const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : 0)
|
|
134
|
+
return {
|
|
135
|
+
/** Sweep width the broker used for this pop. */
|
|
136
|
+
partitions: num(raw.partitions),
|
|
137
|
+
/** Message budget the broker used for this pop. */
|
|
138
|
+
batch: num(raw.batch),
|
|
139
|
+
/**
|
|
140
|
+
* The broker's advice on how long to wait before polling again (wire name:
|
|
141
|
+
* waitMs). Present only when the broker has an opinion, and it is advice,
|
|
142
|
+
* not a lease: the consume loop honors it for the sleep it was already
|
|
143
|
+
* taking between empty non-waiting pops, nothing more. 0 = no advice.
|
|
144
|
+
*/
|
|
145
|
+
waitMillis: num(raw.waitMs)
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* The sleep the consume loop has always taken between two empty pops that are
|
|
151
|
+
* NOT long-polling. A waiting pop already blocks on the broker, so it never
|
|
152
|
+
* reaches here.
|
|
153
|
+
*/
|
|
154
|
+
export const EMPTY_POLL_BACKOFF_MILLIS = 100
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* How long to wait after an empty pop: the broker's advice when it gave one,
|
|
158
|
+
* the historical constant otherwise.
|
|
159
|
+
*
|
|
160
|
+
* The advice is honored as given, without a ceiling of this client's invention.
|
|
161
|
+
* The sleep it feeds is raced against the caller's abort signal by the loop, so
|
|
162
|
+
* even an absurd value cannot outlive a cancellation — which is the only
|
|
163
|
+
* property that has to hold locally.
|
|
164
|
+
*/
|
|
165
|
+
export function emptyPollDelayMillis(decision) {
|
|
166
|
+
if (decision && decision.waitMillis > 0) return decision.waitMillis
|
|
167
|
+
return EMPTY_POLL_BACKOFF_MILLIS
|
|
168
|
+
}
|
|
@@ -51,9 +51,14 @@ export const QUEUE_DEFAULTS = {
|
|
|
51
51
|
encryptionEnabled: false // No encryption by default
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
+
// batch and maxPartitions here are the AUTOPILOT-OFF defaults. With autopilot on
|
|
55
|
+
// (the default) a knob the caller never set is not defaulted at all -- it is
|
|
56
|
+
// omitted from the pop so the broker sizes it (see utils/autopilot.js). These
|
|
57
|
+
// values are what comes back with QueueBuilder.autopilot(false), or with
|
|
58
|
+
// QUEEN_SDK_POP_AUTOPILOT=off for a whole process.
|
|
54
59
|
export const CONSUME_DEFAULTS = {
|
|
55
60
|
concurrency: 1, // Single worker
|
|
56
|
-
batch: 1, // One message at a time
|
|
61
|
+
batch: 1, // One message at a time (autopilot off only)
|
|
57
62
|
autoAck: true, // Client-side auto-ack (NOT sent to server)
|
|
58
63
|
wait: true, // Long polling enabled
|
|
59
64
|
timeoutMillis: 30000, // 30 seconds long poll timeout
|
|
@@ -63,7 +68,7 @@ export const CONSUME_DEFAULTS = {
|
|
|
63
68
|
renewLeaseIntervalMillis: null, // Auto-renewal interval when enabled
|
|
64
69
|
subscriptionMode: null, // No subscription mode (standard queue mode)
|
|
65
70
|
subscriptionFrom: null, // No subscription start point
|
|
66
|
-
maxPartitions: 1, // v4 multi-partition pop cap (
|
|
71
|
+
maxPartitions: 1, // v4 multi-partition pop cap (autopilot off only)
|
|
67
72
|
// Last-value delivery for this consumer group (PLAN_CONFLATION §1.1): a pop
|
|
68
73
|
// of a partition delivers only the NEWEST visible message and commits past
|
|
69
74
|
// the ones it skipped. Off by default, and only ever SENT when true, so a
|
|
@@ -73,8 +78,9 @@ export const CONSUME_DEFAULTS = {
|
|
|
73
78
|
conflation: false
|
|
74
79
|
}
|
|
75
80
|
|
|
81
|
+
// As in CONSUME_DEFAULTS, batch is the autopilot-OFF default.
|
|
76
82
|
export const POP_DEFAULTS = {
|
|
77
|
-
batch: 1, // One message
|
|
83
|
+
batch: 1, // One message (autopilot off only)
|
|
78
84
|
wait: false, // No long polling (immediate return)
|
|
79
85
|
timeoutMillis: 30000, // 30 seconds if wait=true
|
|
80
86
|
autoAck: false // Server-side auto-ack (false = manual ack required)
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "queen-mq",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Partitioned message queue on PostgreSQL — broker client + fluent streaming SDK (windows, joins, gates) in one package",
|
|
6
6
|
"main": "client-v2/index.js",
|
|
7
7
|
"scripts": {
|
|
8
|
-
"test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js && node test-v2/run.js human",
|
|
9
|
-
"test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js",
|
|
8
|
+
"test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js && node test-v2/run.js human",
|
|
9
|
+
"test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js",
|
|
10
10
|
"test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
|
|
11
11
|
"test:integration": "node test-v2/run.js human",
|
|
12
12
|
"test:streams": "node test-v2/run.js stream",
|
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pop autopilot, client side.
|
|
3
|
+
*
|
|
4
|
+
* The four things a client can be wrong about here, and why each is asserted
|
|
5
|
+
* against the WHOLE query string rather than against one parameter:
|
|
6
|
+
*
|
|
7
|
+
* 1. BOTH BUILDERS MUST AGREE. pop() and consume() assemble their query
|
|
8
|
+
* strings separately (QueueBuilder.pop's inline params vs
|
|
9
|
+
* ConsumerManager#buildParams) — the hazard the standing comment in
|
|
10
|
+
* QueueBuilder.js already names. Every case below is run through both, and
|
|
11
|
+
* both are compared to the same expected string, so a rule implemented in
|
|
12
|
+
* one and not the other cannot pass.
|
|
13
|
+
*
|
|
14
|
+
* 2. NOT ENGAGING AUTOPILOT MUST BE BYTE-IDENTICAL TO THE OLD SDK. The escape
|
|
15
|
+
* hatch is only worth having if it is exact, and "exact" is not something a
|
|
16
|
+
* test of one parameter can show: a stray autopilot=true, or a batch that
|
|
17
|
+
* stopped being emitted, is a different request. Hence full-string equality
|
|
18
|
+
* including the parameters this feature never touches, and including their
|
|
19
|
+
* ORDER — this SDK appends rather than sorting, so order is part of the
|
|
20
|
+
* bytes.
|
|
21
|
+
*
|
|
22
|
+
* 3. AN EXPLICIT VALUE IS SACRED, PER DIMENSION. partitions(1) and "never
|
|
23
|
+
* called partitions" both used to reach the wire as nothing at all; they are
|
|
24
|
+
* now different requests, and the pinned one must survive autopilot.
|
|
25
|
+
*
|
|
26
|
+
* 4. THE ADDITIVE RESPONSE FIELD MUST NOT BE LOAD-BEARING. A broker that does
|
|
27
|
+
* not send it, sends it half-filled, or sends it with fields this SDK has
|
|
28
|
+
* never heard of, all have to work.
|
|
29
|
+
*
|
|
30
|
+
* Same style as conflation-unit/conflationWire.test.js: a real node:http server
|
|
31
|
+
* playing a canned plan, real fetch, real JSON, and every assertion is about
|
|
32
|
+
* bytes that actually crossed a socket.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { describe, it, afterEach } from 'node:test'
|
|
36
|
+
import assert from 'node:assert/strict'
|
|
37
|
+
|
|
38
|
+
import { Queen } from '../../client-v2/index.js'
|
|
39
|
+
import { Runner } from '../../client-v2/streams/runtime/Runner.js'
|
|
40
|
+
import {
|
|
41
|
+
ENV_POP_AUTOPILOT,
|
|
42
|
+
popAutopilotDisabledByEnv,
|
|
43
|
+
popSizing,
|
|
44
|
+
parseAutopilotDecision,
|
|
45
|
+
emptyPollDelayMillis,
|
|
46
|
+
EMPTY_POLL_BACKOFF_MILLIS
|
|
47
|
+
} from '../../client-v2/utils/autopilot.js'
|
|
48
|
+
import { withPlanServer, ok } from '../kv-unit/_planServer.js'
|
|
49
|
+
|
|
50
|
+
const QUEUE = 'q'
|
|
51
|
+
const GROUP = 'g'
|
|
52
|
+
|
|
53
|
+
/** One delivered frame, shaped like a real pop response element. */
|
|
54
|
+
function frame(n = 1) {
|
|
55
|
+
return {
|
|
56
|
+
transactionId: `txn-${n}`,
|
|
57
|
+
partitionId: `part-${n}`,
|
|
58
|
+
partition: 'Default',
|
|
59
|
+
payload: { n },
|
|
60
|
+
leaseId: 'lease-1',
|
|
61
|
+
consumerGroup: GROUP
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** A 200 pop body; `extra` carries the additive autopilot echo under test. */
|
|
66
|
+
function popBody(extra = {}, frames = [frame()]) {
|
|
67
|
+
return ok({ messages: frames, partitionsClaimed: frames.length, ...extra })
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Query string of a recorded hit, exactly as it crossed the socket. */
|
|
71
|
+
function rawQuery(url) {
|
|
72
|
+
const i = url.indexOf('?')
|
|
73
|
+
return i < 0 ? '' : url.slice(i + 1)
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Only the pop hits — consume also acks, and the ack is not under test here. */
|
|
77
|
+
function popHits(hits) {
|
|
78
|
+
return hits.filter(h => h.url.startsWith('/api/v1/pop'))
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function withQueen(plan, defaultResponse, run) {
|
|
82
|
+
await withPlanServer(plan, defaultResponse, async (url, hits) => {
|
|
83
|
+
const queen = new Queen({ url, handleSignals: false })
|
|
84
|
+
try {
|
|
85
|
+
await run(queen, hits)
|
|
86
|
+
} finally {
|
|
87
|
+
await queen.close()
|
|
88
|
+
}
|
|
89
|
+
})
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
afterEach(() => { delete process.env[ENV_POP_AUTOPILOT] })
|
|
93
|
+
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
// 1. Param assembly, from both builders.
|
|
96
|
+
// ---------------------------------------------------------------------------
|
|
97
|
+
|
|
98
|
+
// The shared spine of every case: a named queue and group, no long poll,
|
|
99
|
+
// default timeout. Everything that varies below is sizing.
|
|
100
|
+
const GROUP_AND_TAIL = `wait=false&timeout=30000&consumerGroup=${GROUP}`
|
|
101
|
+
|
|
102
|
+
const CASES = [
|
|
103
|
+
{
|
|
104
|
+
// (a) nothing set: both knobs go to the broker, neither travels.
|
|
105
|
+
name: 'nothing set',
|
|
106
|
+
build: qb => qb,
|
|
107
|
+
want: `autopilot=true&${GROUP_AND_TAIL}`
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
// (b) partitions pinned, batch left to the broker.
|
|
111
|
+
name: 'partitions only',
|
|
112
|
+
build: qb => qb.partitions(4),
|
|
113
|
+
want: `autopilot=true&${GROUP_AND_TAIL}&partitions=4`
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
// (b') the pin that used to be indistinguishable from unset. partitions(1)
|
|
117
|
+
// is a decision — hold this consumer to one partition — and the broker has
|
|
118
|
+
// to be told, or autopilot would widen it.
|
|
119
|
+
name: 'partitions pinned to one',
|
|
120
|
+
build: qb => qb.partitions(1),
|
|
121
|
+
want: `autopilot=true&${GROUP_AND_TAIL}&partitions=1`
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
// (c) batch pinned, sweep width left to the broker.
|
|
125
|
+
name: 'batch only',
|
|
126
|
+
build: qb => qb.batch(50),
|
|
127
|
+
want: `autopilot=true&batch=50&${GROUP_AND_TAIL}`
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
// (d) both set: nothing left to decide, so no autopilot parameter and the
|
|
131
|
+
// exact request the pre-autopilot SDK sent.
|
|
132
|
+
name: 'both set',
|
|
133
|
+
build: qb => qb.batch(50).partitions(4),
|
|
134
|
+
want: `batch=50&${GROUP_AND_TAIL}&partitions=4`
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
// (d') both set with partitions at 1: still byte-identical to the old SDK,
|
|
138
|
+
// which never emitted partitions=1.
|
|
139
|
+
name: 'both set, partitions one',
|
|
140
|
+
build: qb => qb.batch(50).partitions(1),
|
|
141
|
+
want: `batch=50&${GROUP_AND_TAIL}`
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
// (e) escape hatch, nothing set: the client-side defaults are back.
|
|
145
|
+
name: 'autopilot off, nothing set',
|
|
146
|
+
build: qb => qb.autopilot(false),
|
|
147
|
+
want: `batch=1&${GROUP_AND_TAIL}`
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
// (e') escape hatch with a pin: partitions=1 stays off the wire, exactly as
|
|
151
|
+
// before autopilot existed.
|
|
152
|
+
name: 'autopilot off, partitions pinned to one',
|
|
153
|
+
build: qb => qb.autopilot(false).partitions(1),
|
|
154
|
+
want: `batch=1&${GROUP_AND_TAIL}`
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
name: 'autopilot off, both set',
|
|
158
|
+
build: qb => qb.autopilot(false).batch(50).partitions(4),
|
|
159
|
+
want: `batch=50&${GROUP_AND_TAIL}&partitions=4`
|
|
160
|
+
},
|
|
161
|
+
{
|
|
162
|
+
// autopilot(true) is the default, spelled out. It must not change anything,
|
|
163
|
+
// including for a caller who set both knobs.
|
|
164
|
+
name: 'autopilot explicitly on, both set',
|
|
165
|
+
build: qb => qb.autopilot(true).batch(50).partitions(4),
|
|
166
|
+
want: `batch=50&${GROUP_AND_TAIL}&partitions=4`
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
// batch(0) is not "a batch of zero" and never was: it is the absence of an
|
|
170
|
+
// opinion, which now means the broker decides.
|
|
171
|
+
name: 'batch zero is unset',
|
|
172
|
+
build: qb => qb.batch(0),
|
|
173
|
+
want: `autopilot=true&${GROUP_AND_TAIL}`
|
|
174
|
+
}
|
|
175
|
+
]
|
|
176
|
+
|
|
177
|
+
describe('pop autopilot — param assembly (pop)', () => {
|
|
178
|
+
for (const tc of CASES) {
|
|
179
|
+
it(tc.name, async () => {
|
|
180
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
181
|
+
await tc.build(queen.queue(QUEUE).group(GROUP).wait(false)).pop()
|
|
182
|
+
|
|
183
|
+
assert.equal(popHits(hits).length, 1)
|
|
184
|
+
assert.equal(rawQuery(popHits(hits)[0].url), tc.want)
|
|
185
|
+
})
|
|
186
|
+
})
|
|
187
|
+
}
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
describe('pop autopilot — param assembly (consume)', () => {
|
|
191
|
+
for (const tc of CASES) {
|
|
192
|
+
it(tc.name, async () => {
|
|
193
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
194
|
+
let handled = 0
|
|
195
|
+
await tc.build(queen.queue(QUEUE).group(GROUP).wait(false))
|
|
196
|
+
.limit(1)
|
|
197
|
+
.consume(() => { handled++ })
|
|
198
|
+
|
|
199
|
+
assert.equal(handled, 1)
|
|
200
|
+
const pops = popHits(hits)
|
|
201
|
+
assert.ok(pops.length >= 1, 'consume made no pop request')
|
|
202
|
+
assert.equal(rawQuery(pops[0].url), tc.want)
|
|
203
|
+
})
|
|
204
|
+
})
|
|
205
|
+
}
|
|
206
|
+
})
|
|
207
|
+
|
|
208
|
+
// ---------------------------------------------------------------------------
|
|
209
|
+
// 2. The process-wide rollback.
|
|
210
|
+
// ---------------------------------------------------------------------------
|
|
211
|
+
|
|
212
|
+
describe('pop autopilot — QUEEN_SDK_POP_AUTOPILOT', () => {
|
|
213
|
+
it('a client built while the variable is set sends the pre-autopilot request', async () => {
|
|
214
|
+
process.env[ENV_POP_AUTOPILOT] = 'off'
|
|
215
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
216
|
+
await queen.queue(QUEUE).group(GROUP).wait(false).pop()
|
|
217
|
+
|
|
218
|
+
assert.equal(rawQuery(popHits(hits)[0].url), `batch=1&${GROUP_AND_TAIL}`)
|
|
219
|
+
})
|
|
220
|
+
})
|
|
221
|
+
|
|
222
|
+
it('is read ONCE, at construction — changing it later does not move a live client', async () => {
|
|
223
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
224
|
+
process.env[ENV_POP_AUTOPILOT] = 'off'
|
|
225
|
+
await queen.queue(QUEUE).group(GROUP).wait(false).pop()
|
|
226
|
+
|
|
227
|
+
assert.equal(rawQuery(popHits(hits)[0].url), `autopilot=true&${GROUP_AND_TAIL}`)
|
|
228
|
+
})
|
|
229
|
+
})
|
|
230
|
+
|
|
231
|
+
it('an explicit .autopilot(true) outranks the environment', async () => {
|
|
232
|
+
process.env[ENV_POP_AUTOPILOT] = 'off'
|
|
233
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
234
|
+
await queen.queue(QUEUE).group(GROUP).wait(false).autopilot(true).pop()
|
|
235
|
+
|
|
236
|
+
assert.equal(rawQuery(popHits(hits)[0].url), `autopilot=true&${GROUP_AND_TAIL}`)
|
|
237
|
+
})
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
it('accepts the whole vocabulary, and nothing else', () => {
|
|
241
|
+
for (const v of ['off', 'OFF', ' off ', 'false', '0', 'no', 'disabled']) {
|
|
242
|
+
process.env[ENV_POP_AUTOPILOT] = v
|
|
243
|
+
assert.equal(popAutopilotDisabledByEnv(), true, `${v} should disable autopilot`)
|
|
244
|
+
}
|
|
245
|
+
for (const v of ['', 'on', 'true', '1', 'yes', 'nonsense']) {
|
|
246
|
+
process.env[ENV_POP_AUTOPILOT] = v
|
|
247
|
+
assert.equal(popAutopilotDisabledByEnv(), false, `${v} should leave autopilot on`)
|
|
248
|
+
}
|
|
249
|
+
delete process.env[ENV_POP_AUTOPILOT]
|
|
250
|
+
assert.equal(popAutopilotDisabledByEnv(), false, 'unset leaves autopilot on')
|
|
251
|
+
})
|
|
252
|
+
})
|
|
253
|
+
|
|
254
|
+
// ---------------------------------------------------------------------------
|
|
255
|
+
// 3. The additive response field.
|
|
256
|
+
// ---------------------------------------------------------------------------
|
|
257
|
+
|
|
258
|
+
describe('pop autopilot — the echo', () => {
|
|
259
|
+
it('parses what the broker chose and ignores what it did not', () => {
|
|
260
|
+
assert.equal(parseAutopilotDecision(null), null)
|
|
261
|
+
assert.equal(parseAutopilotDecision({ messages: [] }), null, 'absent')
|
|
262
|
+
assert.equal(parseAutopilotDecision({ autopilot: null }), null, 'null')
|
|
263
|
+
assert.equal(parseAutopilotDecision({ autopilot: true }), null, 'not an object')
|
|
264
|
+
assert.equal(parseAutopilotDecision({ autopilot: [] }), null, 'an array is not an object')
|
|
265
|
+
|
|
266
|
+
assert.deepEqual(
|
|
267
|
+
parseAutopilotDecision({ autopilot: { partitions: 8, batch: 200, waitMs: 25 } }),
|
|
268
|
+
{ partitions: 8, batch: 200, waitMillis: 25 }
|
|
269
|
+
)
|
|
270
|
+
// waitMs is optional: the broker sends it only when it has an opinion.
|
|
271
|
+
assert.deepEqual(
|
|
272
|
+
parseAutopilotDecision({ autopilot: { partitions: 4, batch: 64 } }),
|
|
273
|
+
{ partitions: 4, batch: 64, waitMillis: 0 }
|
|
274
|
+
)
|
|
275
|
+
// Forward compatibility: a newer broker growing a field must not cost this
|
|
276
|
+
// client the fields it does understand.
|
|
277
|
+
assert.deepEqual(
|
|
278
|
+
parseAutopilotDecision({
|
|
279
|
+
autopilot: { partitions: 2, batch: 10, waitMs: 5, reason: 'ready_age', confidence: 0.9 }
|
|
280
|
+
}),
|
|
281
|
+
{ partitions: 2, batch: 10, waitMillis: 5 }
|
|
282
|
+
)
|
|
283
|
+
// A field of the wrong type is dropped, not fatal.
|
|
284
|
+
assert.deepEqual(
|
|
285
|
+
parseAutopilotDecision({ autopilot: { partitions: 'eight', batch: 10 } }),
|
|
286
|
+
{ partitions: 0, batch: 10, waitMillis: 0 }
|
|
287
|
+
)
|
|
288
|
+
})
|
|
289
|
+
|
|
290
|
+
it('popResult() hands the caller what the broker chose', async () => {
|
|
291
|
+
const body = popBody({ autopilot: { partitions: 8, batch: 200, waitMs: 25 } })
|
|
292
|
+
await withQueen([body], body, async (queen) => {
|
|
293
|
+
const res = await queen.queue(QUEUE).group(GROUP).wait(false).popResult()
|
|
294
|
+
|
|
295
|
+
assert.equal(res.messages.length, 1)
|
|
296
|
+
assert.deepEqual(res.autopilot, { partitions: 8, batch: 200, waitMillis: 25 })
|
|
297
|
+
})
|
|
298
|
+
})
|
|
299
|
+
|
|
300
|
+
it('popResult() reports null when the broker said nothing — a 1.1 broker, say', async () => {
|
|
301
|
+
await withQueen([popBody()], popBody(), async (queen) => {
|
|
302
|
+
const res = await queen.queue(QUEUE).group(GROUP).wait(false).popResult()
|
|
303
|
+
|
|
304
|
+
assert.equal(res.messages.length, 1)
|
|
305
|
+
assert.equal(res.autopilot, null)
|
|
306
|
+
})
|
|
307
|
+
})
|
|
308
|
+
|
|
309
|
+
it('pop() still returns a bare array of messages', async () => {
|
|
310
|
+
const body = popBody({ autopilot: { partitions: 8, batch: 200 } })
|
|
311
|
+
await withQueen([body], body, async (queen) => {
|
|
312
|
+
const messages = await queen.queue(QUEUE).group(GROUP).wait(false).pop()
|
|
313
|
+
|
|
314
|
+
assert.ok(Array.isArray(messages))
|
|
315
|
+
assert.equal(messages.length, 1)
|
|
316
|
+
})
|
|
317
|
+
})
|
|
318
|
+
})
|
|
319
|
+
|
|
320
|
+
// ---------------------------------------------------------------------------
|
|
321
|
+
// 4. Empty-poll pacing.
|
|
322
|
+
// ---------------------------------------------------------------------------
|
|
323
|
+
|
|
324
|
+
describe('pop autopilot — empty-poll pacing', () => {
|
|
325
|
+
it('honours the broker advice, and falls back to the historical delay', () => {
|
|
326
|
+
assert.equal(emptyPollDelayMillis(null), EMPTY_POLL_BACKOFF_MILLIS)
|
|
327
|
+
assert.equal(emptyPollDelayMillis({ partitions: 1, batch: 1, waitMillis: 0 }), EMPTY_POLL_BACKOFF_MILLIS)
|
|
328
|
+
assert.equal(emptyPollDelayMillis({ partitions: 1, batch: 1, waitMillis: 250 }), 250)
|
|
329
|
+
})
|
|
330
|
+
})
|
|
331
|
+
|
|
332
|
+
// ---------------------------------------------------------------------------
|
|
333
|
+
// 5. The rule itself, in isolation.
|
|
334
|
+
// ---------------------------------------------------------------------------
|
|
335
|
+
|
|
336
|
+
describe('pop autopilot — popSizing', () => {
|
|
337
|
+
it('leaves a pinned dimension alone and delegates only the unset one', () => {
|
|
338
|
+
assert.deepEqual(
|
|
339
|
+
popSizing({ batch: null, maxPartitions: null, fallbackBatch: 1, autopilot: true }),
|
|
340
|
+
{ autopilot: true, batch: null, partitions: null }
|
|
341
|
+
)
|
|
342
|
+
assert.deepEqual(
|
|
343
|
+
popSizing({ batch: 50, maxPartitions: null, fallbackBatch: 1, autopilot: true }),
|
|
344
|
+
{ autopilot: true, batch: '50', partitions: null }
|
|
345
|
+
)
|
|
346
|
+
assert.deepEqual(
|
|
347
|
+
popSizing({ batch: null, maxPartitions: 1, fallbackBatch: 1, autopilot: true }),
|
|
348
|
+
{ autopilot: true, batch: null, partitions: '1' }
|
|
349
|
+
)
|
|
350
|
+
// Both set: nothing to decide, so the flag does not travel either.
|
|
351
|
+
assert.deepEqual(
|
|
352
|
+
popSizing({ batch: 50, maxPartitions: 4, fallbackBatch: 1, autopilot: true }),
|
|
353
|
+
{ autopilot: false, batch: '50', partitions: '4' }
|
|
354
|
+
)
|
|
355
|
+
// Off: the client-side default comes back and partitions keeps its >1 gate.
|
|
356
|
+
assert.deepEqual(
|
|
357
|
+
popSizing({ batch: null, maxPartitions: null, fallbackBatch: 1, autopilot: false }),
|
|
358
|
+
{ autopilot: false, batch: '1', partitions: null }
|
|
359
|
+
)
|
|
360
|
+
assert.deepEqual(
|
|
361
|
+
popSizing({ batch: null, maxPartitions: 1, fallbackBatch: 1, autopilot: false }),
|
|
362
|
+
{ autopilot: false, batch: '1', partitions: null }
|
|
363
|
+
)
|
|
364
|
+
})
|
|
365
|
+
})
|
|
366
|
+
|
|
367
|
+
// ---------------------------------------------------------------------------
|
|
368
|
+
// 6. The streams runtime pins its width.
|
|
369
|
+
// ---------------------------------------------------------------------------
|
|
370
|
+
|
|
371
|
+
describe('pop autopilot — streams runtime', () => {
|
|
372
|
+
/** A QueueBuilder stand-in that records the chain the runner builds. */
|
|
373
|
+
function recordingSource(calls) {
|
|
374
|
+
const qb = {
|
|
375
|
+
batch(v) { calls.push(['batch', v]); return qb },
|
|
376
|
+
wait(v) { calls.push(['wait', v]); return qb },
|
|
377
|
+
timeoutMillis(v) { calls.push(['timeoutMillis', v]); return qb },
|
|
378
|
+
group(v) { calls.push(['group', v]); return qb },
|
|
379
|
+
partitions(v) { calls.push(['partitions', v]); return qb },
|
|
380
|
+
subscriptionMode(v) { calls.push(['subscriptionMode', v]); return qb },
|
|
381
|
+
subscriptionFrom(v) { calls.push(['subscriptionFrom', v]); return qb },
|
|
382
|
+
conflation(v) { calls.push(['conflation', v]); return qb },
|
|
383
|
+
async pop() { return [] }
|
|
384
|
+
}
|
|
385
|
+
return qb
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
it('pins maxPartitions even at 1, so autopilot cannot widen a stream cycle', async () => {
|
|
389
|
+
const calls = []
|
|
390
|
+
const runner = Object.create(Runner.prototype)
|
|
391
|
+
Object.assign(runner, {
|
|
392
|
+
stream: { source: recordingSource(calls) },
|
|
393
|
+
batchSize: 100,
|
|
394
|
+
maxWaitMillis: 1000,
|
|
395
|
+
consumerGroup: 'stream-g',
|
|
396
|
+
maxPartitions: 1,
|
|
397
|
+
subscriptionMode: null,
|
|
398
|
+
subscriptionFrom: null,
|
|
399
|
+
conflation: false
|
|
400
|
+
})
|
|
401
|
+
|
|
402
|
+
await runner._popMessages()
|
|
403
|
+
|
|
404
|
+
assert.deepEqual(
|
|
405
|
+
calls.filter(([k]) => k === 'partitions'),
|
|
406
|
+
[['partitions', 1]],
|
|
407
|
+
'the streams layer always decided its own width; it must keep saying so'
|
|
408
|
+
)
|
|
409
|
+
})
|
|
410
|
+
})
|