queen-mq 1.0.6 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -0
- package/client-v2/Queen.js +50 -0
- package/client-v2/admin/Admin.js +15 -1
- package/client-v2/buffer/BufferManager.js +18 -6
- package/client-v2/buffer/MessageBuffer.js +17 -1
- package/client-v2/buffer/sinks.js +89 -0
- package/client-v2/builders/QueueBuilder.js +71 -2
- package/client-v2/consumer/ConsumerManager.js +39 -4
- package/client-v2/ephemeral/Ephemeral.js +551 -0
- package/client-v2/index.js +12 -0
- package/client-v2/streams/Stream.js +3 -0
- package/client-v2/streams/runtime/Runner.js +18 -0
- package/client-v2/utils/conflation.js +118 -0
- package/client-v2/utils/defaults.js +8 -1
- package/package.json +3 -3
- package/test-v2/conflation-unit/conflationWire.test.js +398 -0
- package/test-v2/ephemeral-unit/_planServer.js +73 -0
- package/test-v2/ephemeral-unit/durableSinkPin.test.js +112 -0
- package/test-v2/ephemeral-unit/ephemeralBuffer.test.js +305 -0
- package/test-v2/ephemeral-unit/ephemeralWire.test.js +398 -0
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conflation response contract (PLAN_CONFLATION.md §3.3, §4).
|
|
3
|
+
*
|
|
4
|
+
* Conflation — last-value delivery for a consumer group — is declared as a pop
|
|
5
|
+
* query parameter and answered in the pop RESPONSE. Two response keys carry the
|
|
6
|
+
* whole client-side contract, and both are handled here so that pop() and the
|
|
7
|
+
* consume loop cannot drift apart:
|
|
8
|
+
*
|
|
9
|
+
* "conflation": true the broker actually applied last-value delivery.
|
|
10
|
+
* Rides EVERY conflating response, empty ones
|
|
11
|
+
* included — a conflating pop answers 200 with a
|
|
12
|
+
* body instead of a bodiless 204 precisely so the
|
|
13
|
+
* echo has somewhere to sit.
|
|
14
|
+
* "conflationConflict": true this consumer declared a policy the group does
|
|
15
|
+
* not have. The STORED group setting wins (§3.3);
|
|
16
|
+
* the consumer keeps working and warns once.
|
|
17
|
+
*
|
|
18
|
+
* DEGRADE LOUDLY (§4). No SDK negotiates a version with the broker. A 1.1.0
|
|
19
|
+
* client against an older broker sends `conflation=true`, the broker ignores the
|
|
20
|
+
* unknown query parameter, and the consumer silently processes the entire
|
|
21
|
+
* backlog message by message — the exact silent failure the feature must not
|
|
22
|
+
* have. So: requested conflation + no echo => raise, on the FIRST response,
|
|
23
|
+
* before a single message is handled. `null` (an old broker's 204 on an empty
|
|
24
|
+
* pop) is the most important case to raise on, not the least: it is what an idle
|
|
25
|
+
* queue looks like, and it is the first thing a fresh consumer sees.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import * as logger from './logger.js'
|
|
29
|
+
|
|
30
|
+
/** `error.code` on the degrade-loudly error, so callers can branch without string-matching. */
|
|
31
|
+
export const CONFLATION_UNSUPPORTED = 'conflation_unsupported'
|
|
32
|
+
|
|
33
|
+
/** The message the plan fixes for every SDK (§4). Keep it identical across clients. */
|
|
34
|
+
export const CONFLATION_UNSUPPORTED_MESSAGE =
|
|
35
|
+
'conflation was requested but this broker did not apply it — requires broker >= 1.1.0'
|
|
36
|
+
|
|
37
|
+
// Once per (queue, group) PER PROCESS, per §3.3: a mismatched fleet during a
|
|
38
|
+
// rolling deploy would otherwise emit one warning per pop, forever.
|
|
39
|
+
const warnedConflicts = new Set()
|
|
40
|
+
|
|
41
|
+
// A (queue, group) whose conflict the broker has ALREADY told us about. Proof
|
|
42
|
+
// that this broker understands the parameter, so a later response without the
|
|
43
|
+
// echo for that same pair is the same known conflict and not an old broker.
|
|
44
|
+
//
|
|
45
|
+
// It is belt-and-braces now rather than the load-bearing part it once was: a
|
|
46
|
+
// 1.1.0 broker keeps the 200 and the body whenever the answer has anything to
|
|
47
|
+
// say about conflation (`pop_status`), so an EMPTY conflicting pop carries
|
|
48
|
+
// `conflationConflict` too and the check above catches it on the first round
|
|
49
|
+
// trip. Keeping the memo costs nothing and still covers a first contact whose
|
|
50
|
+
// answer somehow arrives without either key.
|
|
51
|
+
const knownConflicts = new Set()
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Identity of the thing a policy hangs on: the queue (or namespace/task pop
|
|
55
|
+
* target) and the consumer group. Deliberately NOT the affinity key, which
|
|
56
|
+
* includes the partition — the policy is per (queue, group).
|
|
57
|
+
*/
|
|
58
|
+
export function conflationKey({ queue, namespace, task, group }) {
|
|
59
|
+
const target = queue || `${namespace || '*'}/${task || '*'}`
|
|
60
|
+
return `${target}|${group || '__QUEUE_MODE__'}`
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Inspect one pop response on behalf of a consumer that ASKED for conflation.
|
|
65
|
+
* Call it for every response, including empty ones — that is where the old
|
|
66
|
+
* broker shows itself first.
|
|
67
|
+
*
|
|
68
|
+
* @param {object|null} result - parsed pop response (null for a 204)
|
|
69
|
+
* @param {object} ctx - { queue, namespace, task, group }
|
|
70
|
+
* @throws {Error} with `.code === CONFLATION_UNSUPPORTED` when the broker did
|
|
71
|
+
* not apply conflation and did not explain why.
|
|
72
|
+
*/
|
|
73
|
+
export function checkConflationResponse(result, ctx) {
|
|
74
|
+
const key = conflationKey(ctx)
|
|
75
|
+
|
|
76
|
+
if (result && result.conflationConflict === true) {
|
|
77
|
+
knownConflicts.add(key)
|
|
78
|
+
if (!warnedConflicts.has(key)) {
|
|
79
|
+
warnedConflicts.add(key)
|
|
80
|
+
logger.warn('Conflation.conflict', {
|
|
81
|
+
queue: ctx.queue || null,
|
|
82
|
+
namespace: ctx.namespace || null,
|
|
83
|
+
task: ctx.task || null,
|
|
84
|
+
group: ctx.group || null,
|
|
85
|
+
message:
|
|
86
|
+
'this consumer declared conflation but the consumer group is registered without it — ' +
|
|
87
|
+
'the STORED group setting wins and this consumer is receiving normal batches. ' +
|
|
88
|
+
'Delete/recreate the group to change its policy.'
|
|
89
|
+
})
|
|
90
|
+
}
|
|
91
|
+
return
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (result && result.conflation === true) return
|
|
95
|
+
|
|
96
|
+
// Pop maintenance: the broker refused the pop before it reached the claim
|
|
97
|
+
// path, so there is no policy to echo and nothing to conclude from the
|
|
98
|
+
// absence of one. Reading it as "old broker" would stop every conflating
|
|
99
|
+
// consumer in the fleet the moment an operator pauses pops.
|
|
100
|
+
if (result && result.paused === true) return
|
|
101
|
+
|
|
102
|
+
// The group is known to disagree with us; this response is that same
|
|
103
|
+
// conflict, seen through a body that could not carry the flag.
|
|
104
|
+
if (knownConflicts.has(key)) return
|
|
105
|
+
|
|
106
|
+
const error = new Error(CONFLATION_UNSUPPORTED_MESSAGE)
|
|
107
|
+
error.code = CONFLATION_UNSUPPORTED
|
|
108
|
+
throw error
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Test hook: the warn-once and known-conflict registries are process-wide by
|
|
113
|
+
* design, which makes them sticky across tests in one runner process.
|
|
114
|
+
*/
|
|
115
|
+
export function resetConflationWarnings() {
|
|
116
|
+
warnedConflicts.clear()
|
|
117
|
+
knownConflicts.clear()
|
|
118
|
+
}
|
|
@@ -63,7 +63,14 @@ export const CONSUME_DEFAULTS = {
|
|
|
63
63
|
renewLeaseIntervalMillis: null, // Auto-renewal interval when enabled
|
|
64
64
|
subscriptionMode: null, // No subscription mode (standard queue mode)
|
|
65
65
|
subscriptionFrom: null, // No subscription start point
|
|
66
|
-
maxPartitions: 1
|
|
66
|
+
maxPartitions: 1, // v4 multi-partition pop cap (1 = legacy single-partition)
|
|
67
|
+
// Last-value delivery for this consumer group (PLAN_CONFLATION §1.1): a pop
|
|
68
|
+
// of a partition delivers only the NEWEST visible message and commits past
|
|
69
|
+
// the ones it skipped. Off by default, and only ever SENT when true, so a
|
|
70
|
+
// client that never touches it is byte-identical on the wire. Requires
|
|
71
|
+
// broker >= 1.1.0 — an older one ignores the parameter, which the SDK
|
|
72
|
+
// detects from the missing response echo and raises on (§4).
|
|
73
|
+
conflation: false
|
|
67
74
|
}
|
|
68
75
|
|
|
69
76
|
export const POP_DEFAULTS = {
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "queen-mq",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Partitioned message queue on PostgreSQL — broker client + fluent streaming SDK (windows, joins, gates) in one package",
|
|
6
6
|
"main": "client-v2/index.js",
|
|
7
7
|
"scripts": {
|
|
8
|
-
"test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js && node test-v2/run.js human",
|
|
9
|
-
"test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js",
|
|
8
|
+
"test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js && node test-v2/run.js human",
|
|
9
|
+
"test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js",
|
|
10
10
|
"test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
|
|
11
11
|
"test:integration": "node test-v2/run.js human",
|
|
12
12
|
"test:streams": "node test-v2/run.js stream",
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conflation: wire contract + degrade-loudly (PLAN_CONFLATION.md §3.1, §3.3, §4).
|
|
3
|
+
*
|
|
4
|
+
* Three things are pinned here, and each of them is a defect that ships
|
|
5
|
+
* silently if it is not:
|
|
6
|
+
*
|
|
7
|
+
* 1. `conflation` reaches the wire from BOTH entry points. pop() and
|
|
8
|
+
* consume() build their query strings in two SEPARATE places
|
|
9
|
+
* (QueueBuilder.pop's inline params and ConsumerManager#buildParams) —
|
|
10
|
+
* the standing comment at QueueBuilder.js documents a bug of exactly this
|
|
11
|
+
* shape, where the parameter was added to one builder and the other kept
|
|
12
|
+
* working while the option "did nothing". Both directions are asserted,
|
|
13
|
+
* including the negative: an undeclared conflation must not add the key at
|
|
14
|
+
* all, because the broker's response bytes are pinned byte-for-byte for
|
|
15
|
+
* flag-off deployments.
|
|
16
|
+
*
|
|
17
|
+
* 2. Degrade-loudly (§4). No SDK negotiates a version, so a new client against
|
|
18
|
+
* a pre-1.1.0 broker sends `conflation=true`, the broker ignores the unknown
|
|
19
|
+
* query parameter, and the consumer quietly drains the whole backlog one
|
|
20
|
+
* message at a time. The detection is response-driven: a conflating broker
|
|
21
|
+
* echoes `"conflation":true` on EVERY response, empty ones included (it
|
|
22
|
+
* answers 200-with-body instead of 204 precisely so the echo has somewhere
|
|
23
|
+
* to ride), so the first round trip is enough. An old broker's empty pop is
|
|
24
|
+
* a bodiless 204 => `null` here, which is why null must raise too.
|
|
25
|
+
*
|
|
26
|
+
* 3. Declaration conflict (§3.3). The STORED group policy wins; the consumer
|
|
27
|
+
* that disagreed keeps working and warns exactly ONCE per (queue, group)
|
|
28
|
+
* per process. Rejecting instead would take down the correct half of a
|
|
29
|
+
* rolling deploy (plan Q3), and warning per response would flood.
|
|
30
|
+
*
|
|
31
|
+
* Same style as http-unit/retry429.test.js and kv-unit/kvWire.test.js: a real
|
|
32
|
+
* node:http server playing a canned plan, real fetch, real JSON. No mocking
|
|
33
|
+
* framework, and every assertion is about bytes that actually crossed a socket.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
import { describe, it, beforeEach, afterEach } from 'node:test'
|
|
37
|
+
import assert from 'node:assert/strict'
|
|
38
|
+
|
|
39
|
+
import { Queen } from '../../client-v2/index.js'
|
|
40
|
+
import { Runner } from '../../client-v2/streams/runtime/Runner.js'
|
|
41
|
+
import * as logger from '../../client-v2/utils/logger.js'
|
|
42
|
+
import {
|
|
43
|
+
CONFLATION_UNSUPPORTED,
|
|
44
|
+
resetConflationWarnings
|
|
45
|
+
} from '../../client-v2/utils/conflation.js'
|
|
46
|
+
import { withPlanServer, ok } from '../kv-unit/_planServer.js'
|
|
47
|
+
|
|
48
|
+
const QUEUE = 'test-conflation'
|
|
49
|
+
const GROUP = 'workers'
|
|
50
|
+
|
|
51
|
+
/** One delivered frame, shaped like a real pop response element. */
|
|
52
|
+
function frame(n = 1) {
|
|
53
|
+
return {
|
|
54
|
+
transactionId: `txn-${n}`,
|
|
55
|
+
partitionId: `part-${n}`,
|
|
56
|
+
partition: 'Default',
|
|
57
|
+
payload: { n },
|
|
58
|
+
leaseId: 'lease-1',
|
|
59
|
+
consumerGroup: GROUP
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** A 200 pop body; `extra` carries the conflation echo keys under test. */
|
|
64
|
+
function popBody(extra = {}, frames = [frame()]) {
|
|
65
|
+
return ok({ messages: frames, partitionsClaimed: frames.length, ...extra })
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** A pre-1.1.0 broker's empty pop: bodiless 204 (HttpClient turns it into null). */
|
|
69
|
+
const emptyNoContent = { status: 204 }
|
|
70
|
+
|
|
71
|
+
/** Query parameters of a recorded hit. */
|
|
72
|
+
function query(url) {
|
|
73
|
+
const i = url.indexOf('?')
|
|
74
|
+
return new URLSearchParams(i < 0 ? '' : url.slice(i + 1))
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Path (without query) of a recorded hit. */
|
|
78
|
+
function path(url) {
|
|
79
|
+
const i = url.indexOf('?')
|
|
80
|
+
return i < 0 ? url : url.slice(0, i)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async function withQueen(plan, defaultResponse, run) {
|
|
84
|
+
await withPlanServer(plan, defaultResponse, async (url, hits) => {
|
|
85
|
+
const queen = new Queen({ url, handleSignals: false })
|
|
86
|
+
try {
|
|
87
|
+
await run(queen, hits)
|
|
88
|
+
} finally {
|
|
89
|
+
await queen.close()
|
|
90
|
+
}
|
|
91
|
+
})
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// The conflict warning is deliberately once-per-process state, so every test
|
|
95
|
+
// starts from a clean registry.
|
|
96
|
+
beforeEach(() => { resetConflationWarnings() })
|
|
97
|
+
afterEach(() => { resetConflationWarnings(); logger.configure(null) })
|
|
98
|
+
|
|
99
|
+
// ---------------------------------------------------------------------------
|
|
100
|
+
// 1. The option reaches the wire — from pop() AND from consume().
|
|
101
|
+
// ---------------------------------------------------------------------------
|
|
102
|
+
|
|
103
|
+
describe('conflation — wire (pop)', () => {
|
|
104
|
+
it('pop() sends conflation=true when the builder declares it', async () => {
|
|
105
|
+
await withQueen([popBody({ conflation: true })], popBody({ conflation: true }), async (queen, hits) => {
|
|
106
|
+
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
107
|
+
|
|
108
|
+
assert.equal(messages.length, 1)
|
|
109
|
+
assert.equal(hits.length, 1)
|
|
110
|
+
assert.equal(path(hits[0].url), `/api/v1/pop/queue/${QUEUE}`)
|
|
111
|
+
assert.equal(query(hits[0].url).get('conflation'), 'true')
|
|
112
|
+
assert.equal(query(hits[0].url).get('consumerGroup'), GROUP)
|
|
113
|
+
})
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
it('pop() omits the parameter entirely when conflation is not declared', async () => {
|
|
117
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
118
|
+
await queen.queue(QUEUE).group(GROUP).pop()
|
|
119
|
+
|
|
120
|
+
assert.equal(hits.length, 1)
|
|
121
|
+
assert.equal(
|
|
122
|
+
query(hits[0].url).has('conflation'),
|
|
123
|
+
false,
|
|
124
|
+
'a flag-off pop must be byte-identical to today: no conflation key at all'
|
|
125
|
+
)
|
|
126
|
+
})
|
|
127
|
+
})
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
describe('conflation — wire (consume)', () => {
|
|
131
|
+
it('consume() sends conflation=true through ConsumerManager#buildParams', async () => {
|
|
132
|
+
await withQueen([popBody({ conflation: true })], popBody({ conflation: true }), async (queen, hits) => {
|
|
133
|
+
let handled = 0
|
|
134
|
+
await queen.queue(QUEUE)
|
|
135
|
+
.group(GROUP)
|
|
136
|
+
.conflation(true)
|
|
137
|
+
.wait(false)
|
|
138
|
+
.autoAck(false)
|
|
139
|
+
.limit(1)
|
|
140
|
+
.consume(async () => { handled++ })
|
|
141
|
+
|
|
142
|
+
assert.equal(handled, 1)
|
|
143
|
+
assert.ok(hits.length >= 1)
|
|
144
|
+
assert.equal(path(hits[0].url), `/api/v1/pop/queue/${QUEUE}`)
|
|
145
|
+
assert.equal(
|
|
146
|
+
query(hits[0].url).get('conflation'),
|
|
147
|
+
'true',
|
|
148
|
+
'consume() builds its params in a DIFFERENT place than pop() — both must carry it'
|
|
149
|
+
)
|
|
150
|
+
})
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
it('consume() omits the parameter entirely when conflation is not declared', async () => {
|
|
154
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
155
|
+
await queen.queue(QUEUE)
|
|
156
|
+
.group(GROUP)
|
|
157
|
+
.wait(false)
|
|
158
|
+
.autoAck(false)
|
|
159
|
+
.limit(1)
|
|
160
|
+
.consume(async () => {})
|
|
161
|
+
|
|
162
|
+
assert.ok(hits.length >= 1)
|
|
163
|
+
assert.equal(query(hits[0].url).has('conflation'), false)
|
|
164
|
+
})
|
|
165
|
+
})
|
|
166
|
+
})
|
|
167
|
+
|
|
168
|
+
// ---------------------------------------------------------------------------
|
|
169
|
+
// 2. Degrade-loudly (§4): an old broker must never be silent.
|
|
170
|
+
// ---------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
describe('conflation — degrade loudly against a pre-1.1.0 broker', () => {
|
|
173
|
+
it('pop() raises when the response carries messages but no conflation echo', async () => {
|
|
174
|
+
await withQueen([popBody()], popBody(), async (queen) => {
|
|
175
|
+
await assert.rejects(
|
|
176
|
+
() => queen.queue(QUEUE).group(GROUP).conflation(true).pop(),
|
|
177
|
+
(err) => {
|
|
178
|
+
assert.equal(err.code, CONFLATION_UNSUPPORTED)
|
|
179
|
+
assert.match(err.message, /requires broker >= 1\.1\.0/)
|
|
180
|
+
return true
|
|
181
|
+
}
|
|
182
|
+
)
|
|
183
|
+
})
|
|
184
|
+
})
|
|
185
|
+
|
|
186
|
+
it('pop() raises on the FIRST empty pop, which an old broker answers 204', async () => {
|
|
187
|
+
await withQueen([emptyNoContent], emptyNoContent, async (queen, hits) => {
|
|
188
|
+
await assert.rejects(
|
|
189
|
+
() => queen.queue(QUEUE).group(GROUP).conflation(true).pop(),
|
|
190
|
+
(err) => {
|
|
191
|
+
assert.equal(err.code, CONFLATION_UNSUPPORTED)
|
|
192
|
+
return true
|
|
193
|
+
}
|
|
194
|
+
)
|
|
195
|
+
assert.equal(hits.length, 1, 'the error fires on the first round trip, before any message')
|
|
196
|
+
})
|
|
197
|
+
})
|
|
198
|
+
|
|
199
|
+
it('consume() stops the loop and rejects before the handler ever runs', async () => {
|
|
200
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
201
|
+
let handled = 0
|
|
202
|
+
// consume() returns a thenable ConsumeBuilder, not a Promise: await it
|
|
203
|
+
// inside the arrow so assert.rejects sees a real promise.
|
|
204
|
+
await assert.rejects(
|
|
205
|
+
async () => {
|
|
206
|
+
await queen.queue(QUEUE)
|
|
207
|
+
.group(GROUP)
|
|
208
|
+
.conflation(true)
|
|
209
|
+
.wait(false)
|
|
210
|
+
.autoAck(false)
|
|
211
|
+
.consume(async () => { handled++ })
|
|
212
|
+
},
|
|
213
|
+
(err) => {
|
|
214
|
+
assert.equal(err.code, CONFLATION_UNSUPPORTED)
|
|
215
|
+
assert.match(err.message, /requires broker >= 1\.1\.0/)
|
|
216
|
+
return true
|
|
217
|
+
}
|
|
218
|
+
)
|
|
219
|
+
assert.equal(handled, 0, 'not one message may be processed under a silent degrade')
|
|
220
|
+
assert.equal(hits.length, 1, 'the loop stops instead of draining the backlog')
|
|
221
|
+
})
|
|
222
|
+
})
|
|
223
|
+
|
|
224
|
+
it('leaves a conflation-less response alone when conflation was never requested', async () => {
|
|
225
|
+
await withQueen([popBody()], popBody(), async (queen) => {
|
|
226
|
+
const messages = await queen.queue(QUEUE).group(GROUP).pop()
|
|
227
|
+
assert.equal(messages.length, 1, 'today\'s behaviour is untouched when the flag is off')
|
|
228
|
+
})
|
|
229
|
+
})
|
|
230
|
+
})
|
|
231
|
+
|
|
232
|
+
// ---------------------------------------------------------------------------
|
|
233
|
+
// 3. Declaration conflict (§3.3): the stored policy wins, loudly but once.
|
|
234
|
+
// ---------------------------------------------------------------------------
|
|
235
|
+
|
|
236
|
+
/** Collect every warning the SDK emits while `run` executes. */
|
|
237
|
+
async function captureWarnings(run) {
|
|
238
|
+
const warnings = []
|
|
239
|
+
logger.configure({
|
|
240
|
+
info() {},
|
|
241
|
+
warn(entry) { warnings.push(typeof entry === 'string' ? entry : JSON.stringify(entry)) },
|
|
242
|
+
error() {}
|
|
243
|
+
})
|
|
244
|
+
try {
|
|
245
|
+
await run()
|
|
246
|
+
} finally {
|
|
247
|
+
logger.configure(null)
|
|
248
|
+
}
|
|
249
|
+
return warnings
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
describe('conflation — declaration conflict', () => {
|
|
253
|
+
it('warns exactly once per (queue, group) and keeps delivering messages', async () => {
|
|
254
|
+
const conflicted = popBody({ conflationConflict: true })
|
|
255
|
+
await withQueen([conflicted, conflicted, conflicted], conflicted, async (queen, hits) => {
|
|
256
|
+
const warnings = await captureWarnings(async () => {
|
|
257
|
+
for (let i = 0; i < 3; i++) {
|
|
258
|
+
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
259
|
+
assert.equal(messages.length, 1, 'a conflict never costs the consumer its messages')
|
|
260
|
+
}
|
|
261
|
+
})
|
|
262
|
+
|
|
263
|
+
assert.equal(hits.length, 3)
|
|
264
|
+
const conflictWarnings = warnings.filter(w => w.includes('Conflation.conflict'))
|
|
265
|
+
assert.equal(
|
|
266
|
+
conflictWarnings.length,
|
|
267
|
+
1,
|
|
268
|
+
`exactly one warning per (queue,group) per process, got: ${JSON.stringify(conflictWarnings)}`
|
|
269
|
+
)
|
|
270
|
+
assert.match(conflictWarnings[0], new RegExp(QUEUE))
|
|
271
|
+
assert.match(conflictWarnings[0], new RegExp(GROUP))
|
|
272
|
+
})
|
|
273
|
+
})
|
|
274
|
+
|
|
275
|
+
it('warns again for a different group on the same queue', async () => {
|
|
276
|
+
const conflicted = popBody({ conflationConflict: true })
|
|
277
|
+
await withQueen([conflicted, conflicted], conflicted, async (queen) => {
|
|
278
|
+
const warnings = await captureWarnings(async () => {
|
|
279
|
+
await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
280
|
+
await queen.queue(QUEUE).group('audit').conflation(true).pop()
|
|
281
|
+
})
|
|
282
|
+
|
|
283
|
+
const conflictWarnings = warnings.filter(w => w.includes('Conflation.conflict'))
|
|
284
|
+
assert.equal(conflictWarnings.length, 2, 'the warn-once key is (queue, group), not queue')
|
|
285
|
+
})
|
|
286
|
+
})
|
|
287
|
+
|
|
288
|
+
it('a conflict is never a degrade error — the broker plainly understood the flag', async () => {
|
|
289
|
+
const conflicted = popBody({ conflationConflict: true })
|
|
290
|
+
await withQueen([conflicted], conflicted, async (queen) => {
|
|
291
|
+
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
292
|
+
assert.equal(messages.length, 1)
|
|
293
|
+
})
|
|
294
|
+
})
|
|
295
|
+
|
|
296
|
+
it('an EMPTY conflicting pop is answered 200-with-body and must not raise', async () => {
|
|
297
|
+
// The steady state of a disagreeing consumer, and the case the SDK cannot
|
|
298
|
+
// rescue on its own: this is a consumer's FIRST poll of an idle queue, so
|
|
299
|
+
// the knownConflicts memo below is empty and only the response can say what
|
|
300
|
+
// is going on. The broker keeps the 200 whenever the answer has anything to
|
|
301
|
+
// say about conflation (`pop_status`); if it ever regressed to a bodiless
|
|
302
|
+
// 204 here, every conflicting consumer would exit on its first poll.
|
|
303
|
+
const emptyConflict = ok({ messages: [], partitionsClaimed: 0, conflationConflict: true })
|
|
304
|
+
await withQueen([emptyConflict], emptyConflict, async (queen, hits) => {
|
|
305
|
+
const warnings = await captureWarnings(async () => {
|
|
306
|
+
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
307
|
+
assert.equal(messages.length, 0)
|
|
308
|
+
})
|
|
309
|
+
assert.equal(hits.length, 1)
|
|
310
|
+
assert.equal(warnings.filter(w => w.includes('Conflation.conflict')).length, 1)
|
|
311
|
+
})
|
|
312
|
+
})
|
|
313
|
+
|
|
314
|
+
it('pop maintenance is not a version skew', async () => {
|
|
315
|
+
// `{"messages":[],"paused":true}` means an operator turned pops off. The
|
|
316
|
+
// request never reached the claim path, so there is no echo to expect —
|
|
317
|
+
// raising here would stop every conflating consumer in the fleet on a
|
|
318
|
+
// routine operator action.
|
|
319
|
+
const paused = ok({ messages: [], paused: true })
|
|
320
|
+
await withQueen([paused], paused, async (queen) => {
|
|
321
|
+
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
322
|
+
assert.equal(messages.length, 0)
|
|
323
|
+
})
|
|
324
|
+
})
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
// ---------------------------------------------------------------------------
|
|
328
|
+
// 4. The streams runtime pops through the same QueueBuilder — so the option has
|
|
329
|
+
// to be threaded there too, or `run({ conflation: true })` is a no-op.
|
|
330
|
+
// ---------------------------------------------------------------------------
|
|
331
|
+
|
|
332
|
+
/** A QueueBuilder-shaped source that records the chain the Runner builds. */
|
|
333
|
+
function recordingSource(calls) {
|
|
334
|
+
const qb = {
|
|
335
|
+
batch(v) { calls.push(['batch', v]); return qb },
|
|
336
|
+
wait(v) { calls.push(['wait', v]); return qb },
|
|
337
|
+
timeoutMillis(v) { calls.push(['timeoutMillis', v]); return qb },
|
|
338
|
+
group(v) { calls.push(['group', v]); return qb },
|
|
339
|
+
partitions(v) { calls.push(['partitions', v]); return qb },
|
|
340
|
+
subscriptionMode(v) { calls.push(['subscriptionMode', v]); return qb },
|
|
341
|
+
subscriptionFrom(v) { calls.push(['subscriptionFrom', v]); return qb },
|
|
342
|
+
conflation(v) { calls.push(['conflation', v]); return qb },
|
|
343
|
+
async pop() { calls.push(['pop']); return [] }
|
|
344
|
+
}
|
|
345
|
+
return qb
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function runnerWith(opts, calls) {
|
|
349
|
+
return new Runner({
|
|
350
|
+
url: 'http://127.0.0.1:1',
|
|
351
|
+
queryId: 'test-conflation-query',
|
|
352
|
+
stream: { source: recordingSource(calls), stages: {}, config_hash: 'hash' },
|
|
353
|
+
...opts
|
|
354
|
+
})
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
describe('conflation — streams runtime', () => {
|
|
358
|
+
it('Stream.run({ conflation: true }) declares it on the source pop', async () => {
|
|
359
|
+
const calls = []
|
|
360
|
+
await runnerWith({ conflation: true }, calls)._popMessages()
|
|
361
|
+
assert.deepEqual(calls.filter(c => c[0] === 'conflation'), [['conflation', true]])
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
it('leaves the source pop untouched when the option is absent', async () => {
|
|
365
|
+
const calls = []
|
|
366
|
+
await runnerWith({}, calls)._popMessages()
|
|
367
|
+
assert.deepEqual(calls.filter(c => c[0] === 'conflation'), [])
|
|
368
|
+
})
|
|
369
|
+
})
|
|
370
|
+
|
|
371
|
+
// ---------------------------------------------------------------------------
|
|
372
|
+
// 5. Depth: the new fields must survive the admin call (§2.5/§5.3).
|
|
373
|
+
// ---------------------------------------------------------------------------
|
|
374
|
+
|
|
375
|
+
describe('conflation — queue depth', () => {
|
|
376
|
+
it('getQueueDepth surfaces partitionsPending / conflation / effectivePending', async () => {
|
|
377
|
+
const depth = ok({
|
|
378
|
+
queue: QUEUE,
|
|
379
|
+
group: GROUP,
|
|
380
|
+
pending: 4000000,
|
|
381
|
+
partitionsPending: 12,
|
|
382
|
+
conflation: true,
|
|
383
|
+
effectivePending: 12,
|
|
384
|
+
partitions: [{ partition: 'a', pending: 333333 }]
|
|
385
|
+
})
|
|
386
|
+
await withQueen([depth], depth, async (queen, hits) => {
|
|
387
|
+
const got = await queen.admin.getQueueDepth(QUEUE, GROUP)
|
|
388
|
+
|
|
389
|
+
assert.equal(path(hits[0].url), `/api/v1/resources/queues/${QUEUE}/depth`)
|
|
390
|
+
assert.equal(query(hits[0].url).get('group'), GROUP)
|
|
391
|
+
// Log depth vs work depth: 4M positions to retire, 12 handler calls owed.
|
|
392
|
+
assert.equal(got.pending, 4000000)
|
|
393
|
+
assert.equal(got.partitionsPending, 12)
|
|
394
|
+
assert.equal(got.conflation, true)
|
|
395
|
+
assert.equal(got.effectivePending, 12)
|
|
396
|
+
})
|
|
397
|
+
})
|
|
398
|
+
})
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The scripted plan server, plus the ephemeral envelopes these tests answer
|
|
3
|
+
* with.
|
|
4
|
+
*
|
|
5
|
+
* The server itself is kv-unit's -- a real node:http socket driven by a canned
|
|
6
|
+
* list of responses, recording method, url, raw bytes and parsed body for every
|
|
7
|
+
* request. It is IMPORTED rather than copied for the same reason
|
|
8
|
+
* conflation-unit imports it: two plan servers would drift, and the day one of
|
|
9
|
+
* them stops recording `raw` is the day a byte-identity pin silently becomes a
|
|
10
|
+
* shape check.
|
|
11
|
+
*
|
|
12
|
+
* What is local here is the vocabulary: the ephemeral routes have their own
|
|
13
|
+
* envelopes (§3.1), and a test that spells `{queue, messages}` out by hand at
|
|
14
|
+
* every call site hides the wire behind its own noise.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { Queen } from '../../client-v2/index.js'
|
|
18
|
+
import { withPlanServer, ok } from '../kv-unit/_planServer.js'
|
|
19
|
+
|
|
20
|
+
export { withPlanServer, ok }
|
|
21
|
+
|
|
22
|
+
/** `POST /api/v1/ephemeral/push` -> 201 `{pushed}` (all-or-nothing per request). */
|
|
23
|
+
export function pushed(count) {
|
|
24
|
+
return { status: 201, body: { pushed: count } }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** `GET /api/v1/ephemeral/pop` -> 200 `{queue, messages}`; empty array on timeout. */
|
|
28
|
+
export function popped(queue, messages = []) {
|
|
29
|
+
return ok({ queue, messages })
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** One delivered frame, shaped like a real ephemeral pop element (§3.1). */
|
|
33
|
+
export function frame(n = 1, extra = {}) {
|
|
34
|
+
return { id: `e:beef:Default:${n}`, partition: 'Default', payload: { n }, attempts: 0, ...extra }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** `POST /api/v1/ephemeral/ack` -> 200 `{results:[{id, outcome}]}`. */
|
|
38
|
+
export function acked(...results) {
|
|
39
|
+
return ok({ results })
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* What a broker or proxy older than 1.1 answers on every route of this family:
|
|
44
|
+
* the broker because the routes do not exist, the proxy because an unknown API
|
|
45
|
+
* path is `route_blocked` and fails closed.
|
|
46
|
+
*/
|
|
47
|
+
export const OLD_BROKER = { status: 404, body: { error: 'not_found' } }
|
|
48
|
+
export const OLD_PROXY = { status: 404, body: { error: 'route_blocked', code: 'route_blocked' } }
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The OTHER 404, and the reason the mapping has to read the body: a broker that
|
|
52
|
+
* fully supports the family, answering `depth` about a queue that is not there.
|
|
53
|
+
*/
|
|
54
|
+
export const QUEUE_NOT_FOUND = {
|
|
55
|
+
status: 404,
|
|
56
|
+
body: { error: 'ephemeral queue not found', code: 'ephemeral_queue_not_found' }
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Run `fn(ephemeral, hits, queen)` against a plan server, closing the client
|
|
61
|
+
* afterwards. `handleSignals:false` because a test process must not have its
|
|
62
|
+
* SIGINT handler replaced by every client it builds.
|
|
63
|
+
*/
|
|
64
|
+
export async function withEphemeral(plan, run, clientOptions = {}) {
|
|
65
|
+
await withPlanServer(plan, ok({ ok: true }), async (url, hits) => {
|
|
66
|
+
const queen = new Queen({ url, handleSignals: false, ...clientOptions })
|
|
67
|
+
try {
|
|
68
|
+
await run(queen.ephemeral, hits, queen)
|
|
69
|
+
} finally {
|
|
70
|
+
await queen.close()
|
|
71
|
+
}
|
|
72
|
+
})
|
|
73
|
+
}
|