queen-mq 0.16.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client-v2/Queen.js +71 -15
- package/client-v2/README.md +37 -0
- package/client-v2/builders/QueueBuilder.js +10 -4
- package/client-v2/builders/TransactionBuilder.js +14 -2
- package/client-v2/consumer/ConsumerManager.js +64 -12
- package/client-v2/http/HttpClient.js +382 -32
- package/client-v2/streams/runtime/Runner.js +45 -0
- package/client-v2/utils/defaults.js +20 -1
- package/package.json +4 -3
- package/test-v2/ackwindow.js +258 -0
- package/test-v2/auth.js +65 -149
- package/test-v2/consume.js +2 -0
- package/test-v2/http-unit/hostHeader.test.js +411 -0
- package/test-v2/http-unit/retry429.test.js +314 -0
- package/test-v2/load.js +37 -41
- package/test-v2/maintenance.js +2 -2
- package/test-v2/pop.js +2 -0
- package/test-v2/push.js +29 -35
- package/test-v2/run.js +37 -5
- package/test-v2/semantics.js +792 -0
- package/test-v2/stream/_helpers.js +1 -1
- package/test-v2/stream/tumbling.js +10 -6
- package/test-v2/streams-unit/ack.test.js +203 -0
- package/test-v2/streams-unit/e2e.test.js +4 -1
- package/test-v2/transaction.js +2 -0
- package/test-v2/watermark.js +78 -62
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ack-window honesty tests (2026-07-30).
|
|
3
|
+
*
|
|
4
|
+
* queen.log_ack_by_hash_v1 resolves txn hashes through the queen.log_txns
|
|
5
|
+
* sidecar, which is purged on its own clock (GREATEST(dedup_window,
|
|
6
|
+
* completed_retention, 900s)). A hash that cannot be resolved — purged row,
|
|
7
|
+
* or a transactionId that never existed — is correctly NOT acked (the cursor
|
|
8
|
+
* stops, frames redeliver: redelivery over loss). The BUG these tests pin:
|
|
9
|
+
* the broker reported those items as success=true (they appear in neither
|
|
10
|
+
* noopHashes nor staleHashes), so the client believed the ack landed while
|
|
11
|
+
* the cursor never moved — a silent redelivery livelock, and a nack/dlq in
|
|
12
|
+
* that state could never dead-letter its poison message.
|
|
13
|
+
*
|
|
14
|
+
* Expected contract (post-fix): unresolvable items come back success=false
|
|
15
|
+
* with an explicit "unresolvable" error, and the cursor is untouched.
|
|
16
|
+
*
|
|
17
|
+
* Companion test: emptyPartitionCursorSealsAfterRetention pins the empty-
|
|
18
|
+
* partition cursor seal in log_pop_v1 (the Rust port of the C++
|
|
19
|
+
* pop_unified_batch_v4 starvation fix): a partition whose segments were all
|
|
20
|
+
* removed by retention must stop being "phantom pending" after one pop.
|
|
21
|
+
*
|
|
22
|
+
* Run: node run.js ackUnknownTxnMustFail
|
|
23
|
+
* node run.js ackAfterHashPurgeMustFailExplicitly
|
|
24
|
+
* node run.js emptyPartitionCursorSealsAfterRetention
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { dbPool } from './run.js'
|
|
28
|
+
|
|
29
|
+
const TENANT = '00000000-0000-0000-0000-000000000001'
|
|
30
|
+
|
|
31
|
+
function sleep(ms) { return new Promise(r => setTimeout(r, ms)) }
|
|
32
|
+
|
|
33
|
+
async function popRetry(client, queue, { batch = 3, group = null, partition = null, tries = 30 } = {}) {
|
|
34
|
+
for (let i = 0; i < tries; i++) {
|
|
35
|
+
let b = client.queue(queue).batch(batch).wait(false)
|
|
36
|
+
if (group) b = b.group(group)
|
|
37
|
+
if (partition) b = b.partition(partition)
|
|
38
|
+
const msgs = await b.pop()
|
|
39
|
+
if (msgs && msgs.length > 0) return msgs
|
|
40
|
+
await sleep(150)
|
|
41
|
+
}
|
|
42
|
+
return []
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async function partitionRow(queue, partition = 'Default') {
|
|
46
|
+
const r = await dbPool.query(
|
|
47
|
+
`SELECT p.id::text AS id, p.last_offset::bigint AS last_offset, p.log_start::bigint AS log_start
|
|
48
|
+
FROM queen.log_partitions p
|
|
49
|
+
JOIN queen.queues lq ON lq.id = p.queue_id
|
|
50
|
+
WHERE lq.name = $1 AND p.name = $2 AND lq.tenant_id = $3::uuid`,
|
|
51
|
+
[queue, partition, TENANT])
|
|
52
|
+
return r.rows[0] || null
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function committedOf(partitionId, group) {
|
|
56
|
+
const r = await dbPool.query(
|
|
57
|
+
`SELECT c.committed::bigint AS committed FROM queen.log_consumers c
|
|
58
|
+
WHERE c.partition_id = $1::uuid AND c.consumer_group = $2`,
|
|
59
|
+
[partitionId, group])
|
|
60
|
+
return r.rows.length ? Number(r.rows[0].committed) : null
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function segmentCount(partitionId) {
|
|
64
|
+
const r = await dbPool.query(
|
|
65
|
+
`SELECT count(*)::int AS n FROM queen.log_segments WHERE partition_id = $1::uuid`,
|
|
66
|
+
[partitionId])
|
|
67
|
+
return r.rows[0].n
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Test A (wire-only): acking a transactionId that never existed must fail
|
|
72
|
+
* explicitly, for both a completed ack and a failed nack. Today the broker
|
|
73
|
+
* answers success=true for both.
|
|
74
|
+
*/
|
|
75
|
+
export async function ackUnknownTxnMustFail(client) {
|
|
76
|
+
const queue = 'test-queue-v2-ackwindow-unknown'
|
|
77
|
+
await client.queue(queue).create()
|
|
78
|
+
await client.queue(queue).partition('Default').push([
|
|
79
|
+
{ data: { n: 1 }, transactionId: `${queue}-tx-1` },
|
|
80
|
+
{ data: { n: 2 }, transactionId: `${queue}-tx-2` },
|
|
81
|
+
{ data: { n: 3 }, transactionId: `${queue}-tx-3` },
|
|
82
|
+
])
|
|
83
|
+
|
|
84
|
+
const msgs = await popRetry(client, queue, { batch: 3 })
|
|
85
|
+
if (msgs.length !== 3) return { success: false, message: `expected 3 messages, got ${msgs.length}` }
|
|
86
|
+
|
|
87
|
+
const ghost = {
|
|
88
|
+
transactionId: `${queue}-ghost-never-pushed`,
|
|
89
|
+
partitionId: msgs[0].partitionId,
|
|
90
|
+
leaseId: msgs[0].leaseId,
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Completed ack of a nonexistent txn: must NOT be reported as success.
|
|
94
|
+
const r1 = await client.ack([ghost], true)
|
|
95
|
+
const item1 = (r1.results || [])[0] || {}
|
|
96
|
+
if (r1.success !== false || item1.success !== false) {
|
|
97
|
+
return { success: false, message: `BUG: completed ack of a never-pushed txn reported success (overall=${r1.success}, item=${item1.success})` }
|
|
98
|
+
}
|
|
99
|
+
if (!/unresolv/i.test(item1.error || '')) {
|
|
100
|
+
return { success: false, message: `unknown-txn ack rejected with wrong error: '${item1.error}'` }
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Failed nack of a nonexistent txn: must NOT be swallowed as success
|
|
104
|
+
// (pre-fix this was the worst variant: the nack vanished — no retry
|
|
105
|
+
// charge, no DLQ — while the client was told ok).
|
|
106
|
+
const r2 = await client.ack([{ ...ghost }], false)
|
|
107
|
+
const item2 = (r2.results || [])[0] || {}
|
|
108
|
+
if (r2.success !== false || item2.success !== false) {
|
|
109
|
+
return { success: false, message: `BUG: failed nack of a never-pushed txn reported success (overall=${r2.success}, item=${item2.success})` }
|
|
110
|
+
}
|
|
111
|
+
if (!/unresolv/i.test(item2.error || '')) {
|
|
112
|
+
return { success: false, message: `unknown-txn nack rejected with wrong error: '${item2.error}'` }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// The real batch must still be ackable (the lease survived the rejected calls).
|
|
116
|
+
const r3 = await client.ack(msgs, true)
|
|
117
|
+
if (r3.success !== true) {
|
|
118
|
+
return { success: false, message: `real batch ack failed after ghost-ack rejections: ${r3.error}` }
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
return { success: true, message: 'unknown-txn ack and nack both rejected explicitly; real batch still ackable' }
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Test B (with SQL): the real-world shape — messages still delivered (segments
|
|
126
|
+
* intact) but their log_txns hash rows purged. Per-message acks must fail
|
|
127
|
+
* explicitly and the cursor must not move. Pre-fix: every ack reported
|
|
128
|
+
* success=true while committed stayed put (silent livelock).
|
|
129
|
+
*/
|
|
130
|
+
export async function ackAfterHashPurgeMustFailExplicitly(client) {
|
|
131
|
+
const queue = 'test-queue-v2-ackwindow-purged'
|
|
132
|
+
await client.queue(queue).create()
|
|
133
|
+
const txns = [1, 2, 3, 4, 5].map(n => `${queue}-tx-${n}`)
|
|
134
|
+
await client.queue(queue).partition('Default').push(
|
|
135
|
+
txns.map((t, i) => ({ data: { n: i }, transactionId: t })))
|
|
136
|
+
|
|
137
|
+
// Wait until the segments are visible, then purge the hash sidecar —
|
|
138
|
+
// exactly what log_txns_purge_step_v1 does after the window expires
|
|
139
|
+
// (the 900s floor makes the real purge untestable in-suite).
|
|
140
|
+
const part = await partitionRow(queue)
|
|
141
|
+
if (!part) return { success: false, message: 'partition row not found' }
|
|
142
|
+
for (let i = 0; i < 30 && (await segmentCount(part.id)) === 0; i++) await sleep(100)
|
|
143
|
+
|
|
144
|
+
const del = await dbPool.query(
|
|
145
|
+
`DELETE FROM queen.log_txns WHERE partition_id = $1::uuid`, [part.id])
|
|
146
|
+
if (del.rowCount === 0) return { success: false, message: 'no log_txns rows to purge — test setup broken' }
|
|
147
|
+
|
|
148
|
+
// Segments untouched: the messages still DELIVER.
|
|
149
|
+
const msgs = await popRetry(client, queue, { batch: 5 })
|
|
150
|
+
if (msgs.length !== 5) return { success: false, message: `expected 5 delivered messages after purge, got ${msgs.length}` }
|
|
151
|
+
|
|
152
|
+
// Per-message completed ack (the JS per-message consumer shape): the hash
|
|
153
|
+
// cannot resolve, so the honest answer is an explicit failure.
|
|
154
|
+
const r1 = await client.ack([msgs[0]], true)
|
|
155
|
+
const item1 = (r1.results || [])[0] || {}
|
|
156
|
+
if (item1.success !== false || !/unresolv/i.test(item1.error || '')) {
|
|
157
|
+
return { success: false, message: `BUG: ack with purged hash answered success=${item1.success}, error='${item1.error}'` }
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Nack of a purged-hash message: same — and pre-fix this also meant the
|
|
161
|
+
// poison could never reach the DLQ.
|
|
162
|
+
const r2 = await client.ack([msgs[1]], false)
|
|
163
|
+
const item2 = (r2.results || [])[0] || {}
|
|
164
|
+
if (item2.success !== false || !/unresolv/i.test(item2.error || '')) {
|
|
165
|
+
return { success: false, message: `BUG: nack with purged hash answered success=${item2.success}, error='${item2.error}'` }
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// The cursor must not have moved (redelivery over loss — that part of the
|
|
169
|
+
// contract was always right; only the reporting lied).
|
|
170
|
+
const committed = await committedOf(part.id, '__QUEUE_MODE__')
|
|
171
|
+
if (committed !== -1) {
|
|
172
|
+
return { success: false, message: `cursor moved on unresolvable acks: committed=${committed}, expected -1` }
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return { success: true, message: 'purged-hash ack and nack rejected explicitly, cursor untouched, messages still deliverable' }
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Test C (bug 2, with SQL): empty-partition cursor seal. A group consumes
|
|
180
|
+
* part of a partition, retention then deletes ALL segments (time rule deletes
|
|
181
|
+
* unconsumed data too): last_offset > committed with zero segments = a
|
|
182
|
+
* phantom-pending row that used to stay a wildcard candidate forever and
|
|
183
|
+
* pinned the empty-scan watermark at epoch. After the fix, one empty pop
|
|
184
|
+
* seals committed to the partition's last_offset. A delayed-processing queue
|
|
185
|
+
* pins the guard: segments EXIST but are deferred → no seal.
|
|
186
|
+
*/
|
|
187
|
+
export async function emptyPartitionCursorSealsAfterRetention(client) {
|
|
188
|
+
const queue = 'test-queue-v2-ackwindow-seal'
|
|
189
|
+
const group = 'g-seal'
|
|
190
|
+
await client.queue(queue).config({ retentionEnabled: true, retentionSeconds: 1, leaseTime: 30 }).create()
|
|
191
|
+
await client.queue(queue).partition('Default').push(
|
|
192
|
+
[0, 1, 2, 3, 4, 5].map(n => ({ data: { n }, transactionId: `${queue}-tx-${n}` })))
|
|
193
|
+
|
|
194
|
+
// Consume + ack only the first two: committed lands at 1, the rest stays.
|
|
195
|
+
const first = await popRetry(client, queue, { batch: 2, group })
|
|
196
|
+
if (first.length !== 2) return { success: false, message: `expected 2 messages, got ${first.length}` }
|
|
197
|
+
const ackr = await client.ack(first, true, { group })
|
|
198
|
+
if (ackr.success !== true) return { success: false, message: `ack failed: ${ackr.error}` }
|
|
199
|
+
|
|
200
|
+
const part = await partitionRow(queue)
|
|
201
|
+
if (!part) return { success: false, message: 'partition row not found' }
|
|
202
|
+
|
|
203
|
+
// Retention (interval 2s in the harness, cutoff 1s) must delete ALL
|
|
204
|
+
// segments — the time rule ignores cursors.
|
|
205
|
+
let segs = -1
|
|
206
|
+
for (let i = 0; i < 60; i++) {
|
|
207
|
+
segs = await segmentCount(part.id)
|
|
208
|
+
if (segs === 0) break
|
|
209
|
+
await sleep(500)
|
|
210
|
+
}
|
|
211
|
+
if (segs !== 0) return { success: false, message: `retention never emptied the partition (segments=${segs})` }
|
|
212
|
+
|
|
213
|
+
const before = await committedOf(part.id, group)
|
|
214
|
+
const rowNow = await partitionRow(queue)
|
|
215
|
+
if (before === null) return { success: false, message: 'consumer row missing before the empty pop' }
|
|
216
|
+
if (before >= Number(rowNow.last_offset)) {
|
|
217
|
+
return { success: false, message: `setup broken: no phantom span (committed=${before}, last_offset=${rowNow.last_offset})` }
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// ONE empty pop on the emptied partition must seal the cursor.
|
|
221
|
+
const empty = await client.queue(queue).partition('Default').group(group).batch(1).wait(false).pop()
|
|
222
|
+
if (empty && empty.length > 0) return { success: false, message: `expected an empty pop, got ${empty.length} messages` }
|
|
223
|
+
|
|
224
|
+
const after = await committedOf(part.id, group)
|
|
225
|
+
if (after !== Number(rowNow.last_offset)) {
|
|
226
|
+
return { success: false, message: `BUG: empty pop did not seal the cursor (committed=${after}, want last_offset=${rowNow.last_offset}) — partition stays phantom-pending forever` }
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
// Guard: delayed messages are NOT sealed over. Segments exist but are
|
|
230
|
+
// deferred; the cursor must stay put so they deliver when eligible.
|
|
231
|
+
const dqueue = 'test-queue-v2-ackwindow-seal-delayed'
|
|
232
|
+
await client.queue(dqueue).config({ delayedProcessing: 30, leaseTime: 30 }).create()
|
|
233
|
+
await client.queue(dqueue).partition('Default').push([
|
|
234
|
+
{ data: { n: 0 }, transactionId: `${dqueue}-tx-0` },
|
|
235
|
+
{ data: { n: 1 }, transactionId: `${dqueue}-tx-1` },
|
|
236
|
+
])
|
|
237
|
+
// Give the fusion flush a moment so the segments exist before the pop.
|
|
238
|
+
const dpart0 = await (async () => {
|
|
239
|
+
for (let i = 0; i < 30; i++) {
|
|
240
|
+
const p = await partitionRow(dqueue)
|
|
241
|
+
if (p && (await segmentCount(p.id)) > 0) return p
|
|
242
|
+
await sleep(100)
|
|
243
|
+
}
|
|
244
|
+
return null
|
|
245
|
+
})()
|
|
246
|
+
if (!dpart0) return { success: false, message: 'delayed queue segments never appeared' }
|
|
247
|
+
|
|
248
|
+
const dempty = await client.queue(dqueue).partition('Default').group(group).batch(1).wait(false).pop()
|
|
249
|
+
if (dempty && dempty.length > 0) return { success: false, message: 'delayed message delivered early?' }
|
|
250
|
+
const dcommitted = await committedOf(dpart0.id, group)
|
|
251
|
+
const dsegs = await segmentCount(dpart0.id)
|
|
252
|
+
if (dsegs === 0) return { success: false, message: 'delayed queue segments vanished' }
|
|
253
|
+
if (dcommitted !== -1 && dcommitted !== null) {
|
|
254
|
+
return { success: false, message: `BUG: seal fired over deferred-but-present segments (committed=${dcommitted}) — delayed messages would be skipped` }
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
return { success: true, message: 'empty partition sealed after one pop; deferred segments not sealed over' }
|
|
258
|
+
}
|
package/test-v2/auth.js
CHANGED
|
@@ -19,9 +19,8 @@
|
|
|
19
19
|
// unattended in either server configuration.
|
|
20
20
|
|
|
21
21
|
import crypto from 'crypto'
|
|
22
|
-
import { dbPool } from './run.js'
|
|
23
22
|
|
|
24
|
-
const SERVER_URL = process.env.QUEEN_URL || 'http://localhost:6632'
|
|
23
|
+
const SERVER_URL = process.env.QUEEN_URL || process.env.QUEEN_SERVER_URL || 'http://localhost:6632'
|
|
25
24
|
const JWT_SECRET = process.env.JWT_SECRET || ''
|
|
26
25
|
const JWT_ENABLED = JWT_SECRET.length > 0
|
|
27
26
|
|
|
@@ -50,23 +49,6 @@ function makeToken(sub, role = 'read-write') {
|
|
|
50
49
|
)
|
|
51
50
|
}
|
|
52
51
|
|
|
53
|
-
// ---------------------------------------------------------------------------
|
|
54
|
-
// Ground-truth helper: read producer_sub directly from Postgres, bypassing
|
|
55
|
-
// the API so we can tell the difference between a server that actually stores
|
|
56
|
-
// the value and one that just echoes the client's input.
|
|
57
|
-
// ---------------------------------------------------------------------------
|
|
58
|
-
async function getStoredProducerSub(queueName, transactionId) {
|
|
59
|
-
const res = await dbPool.query(
|
|
60
|
-
`SELECT m.producer_sub
|
|
61
|
-
FROM queen.messages m
|
|
62
|
-
JOIN queen.partitions p ON p.id = m.partition_id
|
|
63
|
-
JOIN queen.queues q ON q.id = p.queue_id
|
|
64
|
-
WHERE q.name = $1 AND m.transaction_id = $2`,
|
|
65
|
-
[queueName, transactionId]
|
|
66
|
-
)
|
|
67
|
-
return res.rows.length > 0 ? res.rows[0].producer_sub : null
|
|
68
|
-
}
|
|
69
|
-
|
|
70
52
|
// Raw HTTP push so we can control the Authorization header and inject
|
|
71
53
|
// arbitrary extra fields (like a forged `producerSub`) per-test.
|
|
72
54
|
async function httpPush({ queue, partition = 'Default', transactionId, data, bearerToken, extraItemFields = {} }) {
|
|
@@ -110,132 +92,51 @@ async function httpConfigure(queue, bearerToken) {
|
|
|
110
92
|
}
|
|
111
93
|
}
|
|
112
94
|
|
|
113
|
-
//
|
|
114
|
-
//
|
|
115
|
-
//
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
)
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
const
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
partition_name: 'Default',
|
|
129
|
-
consumer_group: '__QUEUE_MODE__',
|
|
130
|
-
lease_seconds: 60,
|
|
131
|
-
worker_id: `test-worker-${Date.now()}`,
|
|
132
|
-
batch_size: 10,
|
|
133
|
-
sub_mode: 'all',
|
|
134
|
-
sub_from: ''
|
|
135
|
-
}]
|
|
136
|
-
const res = await dbPool.query(
|
|
137
|
-
'SELECT queen.pop_specific_batch($1::jsonb) as out',
|
|
138
|
-
[JSON.stringify(reqs)]
|
|
139
|
-
)
|
|
140
|
-
const out = res.rows[0].out
|
|
141
|
-
return out[0]?.result?.messages || []
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
// ===========================================================================
|
|
145
|
-
// TEST 1: Stored procedure round-trip (SQL-only, no HTTP, no JWT)
|
|
146
|
-
// ---------------------------------------------------------------------------
|
|
147
|
-
// Exercises the schema + SP contract end-to-end purely via SQL. If this fails,
|
|
148
|
-
// the migration or 001_push.sql / 002_pop_unified.sql is wrong. Runs in any
|
|
149
|
-
// server configuration because it talks to Postgres directly.
|
|
150
|
-
// ===========================================================================
|
|
151
|
-
export async function producerSubRoundTripViaStoredProcedure(client) {
|
|
152
|
-
const q = 'test-auth-sp-roundtrip'
|
|
153
|
-
const tx = `tx-sp-${Date.now()}`
|
|
154
|
-
|
|
155
|
-
await callPushMessagesV3([{
|
|
156
|
-
queue: q,
|
|
157
|
-
partition: 'Default',
|
|
158
|
-
transactionId: tx,
|
|
159
|
-
payload: { x: 1 },
|
|
160
|
-
producerSub: 'sp-test-sub'
|
|
161
|
-
}])
|
|
162
|
-
|
|
163
|
-
const stored = await getStoredProducerSub(q, tx)
|
|
164
|
-
if (stored !== 'sp-test-sub') {
|
|
165
|
-
return { success: false, message: `Expected producer_sub='sp-test-sub' in DB, got ${JSON.stringify(stored)}` }
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
const popped = await callPopSpecificBatch(q)
|
|
169
|
-
const target = popped.find(m => m.transactionId === tx)
|
|
170
|
-
if (!target) {
|
|
171
|
-
return { success: false, message: `Expected to find tx ${tx} in pop result` }
|
|
172
|
-
}
|
|
173
|
-
if (target.producerSub !== 'sp-test-sub') {
|
|
174
|
-
return { success: false, message: `Pop returned producerSub=${JSON.stringify(target.producerSub)}, expected 'sp-test-sub'` }
|
|
95
|
+
// Black-box observation for the HTTP push tests: pop the message back through
|
|
96
|
+
// the API and read its `producerSub`. The segments engine stores messages in
|
|
97
|
+
// queen.seg_segments (never queen.messages), so a seg-broker push is only
|
|
98
|
+
// observable via pop — the seg-native ground truth. Optional bearer so the
|
|
99
|
+
// JWT-enabled pops are authorized.
|
|
100
|
+
async function httpPop({ queue, bearerToken, batch = 20, waitMs = 5000 }) {
|
|
101
|
+
const headers = {}
|
|
102
|
+
if (bearerToken) headers['Authorization'] = `Bearer ${bearerToken}`
|
|
103
|
+
const url = `${SERVER_URL}/api/v1/pop/queue/${encodeURIComponent(queue)}`
|
|
104
|
+
+ `?batch=${batch}&wait=true&timeout=${waitMs}&autoAck=true`
|
|
105
|
+
const res = await fetch(url, { method: 'GET', headers })
|
|
106
|
+
if (res.status === 204) return []
|
|
107
|
+
const text = await res.text()
|
|
108
|
+
if (!res.ok) {
|
|
109
|
+
throw new Error(`pop failed ${res.status}: ${text}`)
|
|
175
110
|
}
|
|
176
|
-
|
|
177
|
-
return
|
|
111
|
+
const body = text ? JSON.parse(text) : {}
|
|
112
|
+
return (body.messages || []).filter(m => m != null)
|
|
178
113
|
}
|
|
179
114
|
|
|
180
|
-
//
|
|
181
|
-
//
|
|
182
|
-
//
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
queue: q,
|
|
191
|
-
partition: 'Default',
|
|
192
|
-
transactionId: tx,
|
|
193
|
-
payload: { x: 1 }
|
|
194
|
-
// no producerSub
|
|
195
|
-
}])
|
|
196
|
-
|
|
197
|
-
const stored = await getStoredProducerSub(q, tx)
|
|
198
|
-
if (stored !== null) {
|
|
199
|
-
return { success: false, message: `Expected NULL producer_sub, got ${JSON.stringify(stored)}` }
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
const popped = await callPopSpecificBatch(q)
|
|
203
|
-
const target = popped.find(m => m.transactionId === tx)
|
|
204
|
-
if (!target) {
|
|
205
|
-
return { success: false, message: `Expected to find tx ${tx} in pop result` }
|
|
206
|
-
}
|
|
207
|
-
if (target.producerSub != null) {
|
|
208
|
-
return { success: false, message: `Pop returned producerSub=${JSON.stringify(target.producerSub)}, expected null` }
|
|
115
|
+
// Pop until we observe the message with the given transactionId; returns its
|
|
116
|
+
// producerSub (a seg push is immediately visible via fire-on-idle, but retry a
|
|
117
|
+
// few times to be robust to any push→visibility race).
|
|
118
|
+
async function poppedProducerSub(queue, transactionId, bearerToken) {
|
|
119
|
+
const deadline = Date.now() + 6000
|
|
120
|
+
while (Date.now() < deadline) {
|
|
121
|
+
const msgs = await httpPop({ queue, bearerToken })
|
|
122
|
+
const hit = msgs.find(m => m.transactionId === transactionId)
|
|
123
|
+
if (hit) return { found: true, producerSub: hit.producerSub }
|
|
124
|
+
await new Promise(r => setTimeout(r, 150))
|
|
209
125
|
}
|
|
210
|
-
|
|
211
|
-
return { success: true, message: 'Omitting producerSub yields NULL in DB and null in pop response' }
|
|
126
|
+
return { found: false, producerSub: undefined }
|
|
212
127
|
}
|
|
213
128
|
|
|
214
129
|
// ===========================================================================
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
//
|
|
130
|
+
// (Retired) SP-level producer_sub round-trip / NULL / empty-string tests.
|
|
131
|
+
// These drove the ROWS engine directly (queen.push_messages_v3 +
|
|
132
|
+
// queen.pop_specific_batch + a queen.messages read) to verify the SP's
|
|
133
|
+
// producer_sub NULLIF handling. The rows engine is retired (segments-only), so
|
|
134
|
+
// they were removed — the segments engine stores payloads in queen.seg_segments
|
|
135
|
+
// and stamps producer_sub in the broker, never in queen.messages. The shipping
|
|
136
|
+
// producer_sub behavior (server-stamped, body-ignored without auth, spoof-proof
|
|
137
|
+
// under JWT) is covered black-box through the HTTP push -> pop path by the tests
|
|
138
|
+
// below.
|
|
219
139
|
// ===========================================================================
|
|
220
|
-
export async function producerSubEmptyStringStoredAsNull(client) {
|
|
221
|
-
const q = 'test-auth-sp-empty'
|
|
222
|
-
const tx = `tx-sp-empty-${Date.now()}`
|
|
223
|
-
|
|
224
|
-
await callPushMessagesV3([{
|
|
225
|
-
queue: q,
|
|
226
|
-
partition: 'Default',
|
|
227
|
-
transactionId: tx,
|
|
228
|
-
payload: { x: 1 },
|
|
229
|
-
producerSub: ''
|
|
230
|
-
}])
|
|
231
|
-
|
|
232
|
-
const stored = await getStoredProducerSub(q, tx)
|
|
233
|
-
if (stored !== null) {
|
|
234
|
-
return { success: false, message: `Expected NULL for empty-string producerSub, got ${JSON.stringify(stored)}` }
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
return { success: true, message: 'Empty-string producerSub is stored as NULL' }
|
|
238
|
-
}
|
|
239
140
|
|
|
240
141
|
// ===========================================================================
|
|
241
142
|
// TEST 4: HTTP push without JWT => producer_sub is NULL
|
|
@@ -265,12 +166,17 @@ export async function producerSubIgnoredFromBodyWithoutAuth(client) {
|
|
|
265
166
|
extraItemFields: { producerSub: 'attacker-no-jwt' }
|
|
266
167
|
})
|
|
267
168
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
169
|
+
// Black-box: observe the pushed message via pop. producerSub must be null
|
|
170
|
+
// (the field is not client-settable, and there is no JWT sub to stamp).
|
|
171
|
+
const { found, producerSub } = await poppedProducerSub(q, tx)
|
|
172
|
+
if (!found) {
|
|
173
|
+
return { success: false, message: `Pushed message ${tx} not observed via pop` }
|
|
174
|
+
}
|
|
175
|
+
if (producerSub !== null) {
|
|
176
|
+
return { success: false, message: `Expected producerSub null (auth disabled), got ${JSON.stringify(producerSub)} - client was able to set it!` }
|
|
271
177
|
}
|
|
272
178
|
|
|
273
|
-
return { success: true, message: 'Body-supplied producerSub ignored when auth disabled;
|
|
179
|
+
return { success: true, message: 'Body-supplied producerSub ignored when auth disabled; pop shows null' }
|
|
274
180
|
}
|
|
275
181
|
|
|
276
182
|
// ===========================================================================
|
|
@@ -297,12 +203,17 @@ export async function producerSubStampedFromJwt(client) {
|
|
|
297
203
|
bearerToken: token
|
|
298
204
|
})
|
|
299
205
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
206
|
+
// Black-box: pop the message back and read producerSub (the seg-native
|
|
207
|
+
// ground truth). It must equal the authenticated JWT sub.
|
|
208
|
+
const { found, producerSub } = await poppedProducerSub(q, tx, token)
|
|
209
|
+
if (!found) {
|
|
210
|
+
return { success: false, message: `Pushed message ${tx} not observed via pop` }
|
|
211
|
+
}
|
|
212
|
+
if (producerSub !== 'alice-producer') {
|
|
213
|
+
return { success: false, message: `Expected producerSub='alice-producer', got ${JSON.stringify(producerSub)}` }
|
|
303
214
|
}
|
|
304
215
|
|
|
305
|
-
return { success: true, message: 'producer_sub stamped from authenticated JWT sub claim' }
|
|
216
|
+
return { success: true, message: 'producer_sub stamped from authenticated JWT sub claim (observed via pop)' }
|
|
306
217
|
}
|
|
307
218
|
|
|
308
219
|
// ===========================================================================
|
|
@@ -328,13 +239,18 @@ export async function producerSubSpoofingIgnoredWithJwt(client) {
|
|
|
328
239
|
extraItemFields: { producerSub: 'attacker' }
|
|
329
240
|
})
|
|
330
241
|
|
|
331
|
-
|
|
332
|
-
|
|
242
|
+
// Black-box: pop the message back; the body-supplied producerSub must be
|
|
243
|
+
// ignored and the observed value must be the validated JWT sub.
|
|
244
|
+
const { found, producerSub } = await poppedProducerSub(q, tx, token)
|
|
245
|
+
if (!found) {
|
|
246
|
+
return { success: false, message: `Pushed message ${tx} not observed via pop` }
|
|
247
|
+
}
|
|
248
|
+
if (producerSub !== 'legit-producer') {
|
|
333
249
|
return {
|
|
334
250
|
success: false,
|
|
335
|
-
message: `Impersonation not prevented:
|
|
251
|
+
message: `Impersonation not prevented: observed producerSub=${JSON.stringify(producerSub)}, expected 'legit-producer'`
|
|
336
252
|
}
|
|
337
253
|
}
|
|
338
254
|
|
|
339
|
-
return { success: true, message: 'Body-supplied producerSub ignored;
|
|
255
|
+
return { success: true, message: 'Body-supplied producerSub ignored; observed sub is from validated JWT' }
|
|
340
256
|
}
|
package/test-v2/consume.js
CHANGED
|
@@ -10,6 +10,7 @@ export async function testConsumer(client) {
|
|
|
10
10
|
|
|
11
11
|
let msgToReturn = null
|
|
12
12
|
|
|
13
|
+
// docs:start(js-consume)
|
|
13
14
|
await client
|
|
14
15
|
.queue('test-queue-v2-consume')
|
|
15
16
|
.batch(1)
|
|
@@ -18,6 +19,7 @@ export async function testConsumer(client) {
|
|
|
18
19
|
.consume(async msg => {
|
|
19
20
|
msgToReturn = msg
|
|
20
21
|
})
|
|
22
|
+
// docs:end
|
|
21
23
|
|
|
22
24
|
return { success: msgToReturn !== null, message: 'Consumer test completed successfully' }
|
|
23
25
|
}
|