queen-mq 0.12.4 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/client-v2/Queen.js +6 -0
- package/client-v2/README.md +12 -1
- package/client-v2/builders/QueueBuilder.js +33 -0
- package/client-v2/consumer/ConsumerManager.js +5 -2
- package/client-v2/utils/defaults.js +2 -1
- package/package.json +1 -1
- package/test-v2/auth.js +340 -0
- package/test-v2/consume.js +96 -0
- package/test-v2/maintenance.js +16 -2
- package/test-v2/pop.js +225 -3
- package/test-v2/push.js +287 -3
- package/test-v2/run.js +5 -1
- package/test-v2/watermark.js +507 -0
package/README.md
CHANGED
|
@@ -250,6 +250,47 @@ for (const message of messages) {
|
|
|
250
250
|
}
|
|
251
251
|
```
|
|
252
252
|
|
|
253
|
+
### Multi-Partition Pop (Drain Many Partitions Per Call)
|
|
254
|
+
|
|
255
|
+
```javascript
|
|
256
|
+
// One round-trip drains up to 200 messages spread across up to 50 partitions.
|
|
257
|
+
// batch(200) is the GLOBAL cap on total messages; partitions(50) is the
|
|
258
|
+
// hard cap on partitions claimed. All claimed partitions share one leaseId
|
|
259
|
+
// — a single renew() call extends every partition's lease atomically.
|
|
260
|
+
const messages = await queen.queue('events')
|
|
261
|
+
.batch(200)
|
|
262
|
+
.partitions(50)
|
|
263
|
+
.wait(true)
|
|
264
|
+
.pop()
|
|
265
|
+
|
|
266
|
+
// Each message carries its own partition info (per-message partitionId,
|
|
267
|
+
// partition name, leaseId, consumerGroup) — ACK and renew always work
|
|
268
|
+
// message-by-message regardless of how many partitions the batch spans.
|
|
269
|
+
for (const m of messages) {
|
|
270
|
+
console.log(`from ${m.partition}:`, m.data)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Same builder works on .consume() for long-running workers
|
|
274
|
+
await queen.queue('events')
|
|
275
|
+
.batch(100)
|
|
276
|
+
.partitions(8)
|
|
277
|
+
.consume(async (msgs) => {
|
|
278
|
+
for (const m of msgs) await process(m.data)
|
|
279
|
+
})
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
**When to use:** queues with many partitions where each partition only has
|
|
283
|
+
a handful of new messages per polling interval (per-customer event streams,
|
|
284
|
+
per-tenant work queues, per-device telemetry). Reduces network round-trips
|
|
285
|
+
from O(P) to O(P / N) while preserving per-partition FIFO ordering.
|
|
286
|
+
|
|
287
|
+
**When not to use:** few partitions, or each one busy enough to fill
|
|
288
|
+
`batch(B)` on its own. Default is `partitions(1)` which preserves the
|
|
289
|
+
legacy single-partition behaviour.
|
|
290
|
+
|
|
291
|
+
`.partitions(N)` only applies to **wildcard** pops; specifying
|
|
292
|
+
`.partition('name')` ignores the cap.
|
|
293
|
+
|
|
253
294
|
### Transactions (Atomic Operations)
|
|
254
295
|
|
|
255
296
|
```javascript
|
|
@@ -451,6 +492,7 @@ await queen.queue('q').buffer({ messageCount: 100, timeMillis: 1000 }).push([...
|
|
|
451
492
|
const msgs = await queen.queue('q').pop()
|
|
452
493
|
const msgs = await queen.queue('q').batch(10).pop()
|
|
453
494
|
const msgs = await queen.queue('q').batch(10).wait(true).pop()
|
|
495
|
+
const msgs = await queen.queue('q').batch(200).partitions(50).pop() // multi-partition pop
|
|
454
496
|
```
|
|
455
497
|
|
|
456
498
|
### Consume
|
package/client-v2/Queen.js
CHANGED
|
@@ -350,6 +350,12 @@ export class Queen {
|
|
|
350
350
|
}
|
|
351
351
|
}
|
|
352
352
|
|
|
353
|
+
// Dedupe: with v4 multi-partition pop, all messages in one batch share
|
|
354
|
+
// the same leaseId (one renew_lease_v2 call extends every claimed
|
|
355
|
+
// partition_consumers row). Without this, callers passing the full
|
|
356
|
+
// messages array would issue N redundant identical HTTP calls.
|
|
357
|
+
leaseIds = [...new Set(leaseIds)]
|
|
358
|
+
|
|
353
359
|
if (leaseIds.length === 0) {
|
|
354
360
|
logger.warn('Queen.renew', 'No valid lease IDs found for renewal')
|
|
355
361
|
return { success: false, error: 'No valid lease IDs found for renewal' }
|
package/client-v2/README.md
CHANGED
|
@@ -1605,6 +1605,11 @@ const msgs = await queen.queue('q').batch(10).wait(true).pop()
|
|
|
1605
1605
|
|
|
1606
1606
|
// Pop from partition
|
|
1607
1607
|
const msgs = await queen.queue('q').partition('p1').pop()
|
|
1608
|
+
|
|
1609
|
+
// Multi-partition pop: drain up to 50 partitions in one round-trip,
|
|
1610
|
+
// capped at 200 total messages. All partitions share one leaseId.
|
|
1611
|
+
// Each returned message carries its own partitionId / partition / leaseId.
|
|
1612
|
+
const msgs = await queen.queue('q').batch(200).partitions(50).pop()
|
|
1608
1613
|
```
|
|
1609
1614
|
|
|
1610
1615
|
### Consume
|
|
@@ -1625,6 +1630,12 @@ await queen.queue('q').concurrency(5).consume(async (msg) => { /* 5 parallel wor
|
|
|
1625
1630
|
// Consume from partition
|
|
1626
1631
|
await queen.queue('q').partition('p1').consume(async (msg) => { /* process */ })
|
|
1627
1632
|
|
|
1633
|
+
// Multi-partition consume: drain up to 8 partitions per poll.
|
|
1634
|
+
// batch(100) is a global cap on total messages across all claimed partitions.
|
|
1635
|
+
await queen.queue('q').batch(100).partitions(8).consume(async (msgs) => {
|
|
1636
|
+
for (const m of msgs) await process(m.data) // m.partitionId / m.partition baked in
|
|
1637
|
+
})
|
|
1638
|
+
|
|
1628
1639
|
// Consume with consumer group
|
|
1629
1640
|
await queen.queue('q').group('my-group').consume(async (msg) => { /* process */ })
|
|
1630
1641
|
|
|
@@ -1863,7 +1874,7 @@ Example log output:
|
|
|
1863
1874
|
Here's a complete example showing many features together:
|
|
1864
1875
|
|
|
1865
1876
|
```javascript
|
|
1866
|
-
import { Queen } from './
|
|
1877
|
+
import { Queen } from './index.js'
|
|
1867
1878
|
|
|
1868
1879
|
const queen = new Queen('http://localhost:6632')
|
|
1869
1880
|
|
|
@@ -37,6 +37,7 @@ export class QueueBuilder {
|
|
|
37
37
|
#subscriptionMode = CONSUME_DEFAULTS.subscriptionMode
|
|
38
38
|
#subscriptionFrom = CONSUME_DEFAULTS.subscriptionFrom
|
|
39
39
|
#each = false
|
|
40
|
+
#maxPartitions = 1
|
|
40
41
|
|
|
41
42
|
// Buffer options
|
|
42
43
|
#bufferOptions = null
|
|
@@ -190,6 +191,22 @@ export class QueueBuilder {
|
|
|
190
191
|
return this
|
|
191
192
|
}
|
|
192
193
|
|
|
194
|
+
/**
|
|
195
|
+
* Pop messages from up to N partitions in a single call (v4 multi-partition pop).
|
|
196
|
+
*
|
|
197
|
+
* Use this to drain many sparsely-loaded partitions efficiently. With
|
|
198
|
+
* `partitions(N)`, the global `batch(B)` budget is shared across all
|
|
199
|
+
* claimed partitions: at most B total messages, drawn from up to N
|
|
200
|
+
* partitions, in a single network round-trip. All N share one leaseId
|
|
201
|
+
* (renewing once extends them all).
|
|
202
|
+
*
|
|
203
|
+
* Default 1 = legacy single-partition behavior.
|
|
204
|
+
*/
|
|
205
|
+
partitions(n) {
|
|
206
|
+
this.#maxPartitions = Math.max(1, n)
|
|
207
|
+
return this
|
|
208
|
+
}
|
|
209
|
+
|
|
193
210
|
limit(count) {
|
|
194
211
|
this.#limit = count
|
|
195
212
|
return this
|
|
@@ -251,6 +268,7 @@ export class QueueBuilder {
|
|
|
251
268
|
subscriptionMode: this.#subscriptionMode,
|
|
252
269
|
subscriptionFrom: this.#subscriptionFrom,
|
|
253
270
|
each: this.#each,
|
|
271
|
+
maxPartitions: this.#maxPartitions,
|
|
254
272
|
signal: options.signal
|
|
255
273
|
}
|
|
256
274
|
|
|
@@ -266,6 +284,20 @@ export class QueueBuilder {
|
|
|
266
284
|
return this
|
|
267
285
|
}
|
|
268
286
|
|
|
287
|
+
/**
|
|
288
|
+
* Set the long-poll timeout in milliseconds. Only meaningful when wait(true)
|
|
289
|
+
* is also set. Default is CONSUME_DEFAULTS.timeoutMillis (30000ms).
|
|
290
|
+
*
|
|
291
|
+
* Use a short timeout in tight consumer loops where you want the long-poll
|
|
292
|
+
* to release the request frequently (e.g. so a shutdown flag is observed
|
|
293
|
+
* promptly). The HTTP layer adds 5s of slack so the server-side timeout
|
|
294
|
+
* always fires first.
|
|
295
|
+
*/
|
|
296
|
+
timeoutMillis(millis) {
|
|
297
|
+
this.#timeoutMillis = Math.max(1, millis)
|
|
298
|
+
return this
|
|
299
|
+
}
|
|
300
|
+
|
|
269
301
|
async pop() {
|
|
270
302
|
logger.log('QueueBuilder.pop', { queue: this.#queueName, partition: this.#partition, namespace: this.#namespace, task: this.#task, batch: this.#batch, wait: this.#wait, group: this.#group })
|
|
271
303
|
|
|
@@ -289,6 +321,7 @@ export class QueueBuilder {
|
|
|
289
321
|
if (effectiveAutoAck) params.append('autoAck', 'true')
|
|
290
322
|
if (this.#subscriptionMode) params.append('subscriptionMode', this.#subscriptionMode)
|
|
291
323
|
if (this.#subscriptionFrom) params.append('subscriptionFrom', this.#subscriptionFrom)
|
|
324
|
+
if (this.#maxPartitions > 1) params.append('partitions', this.#maxPartitions.toString())
|
|
292
325
|
|
|
293
326
|
// Generate affinity key for consistent routing to same backend
|
|
294
327
|
const affinityKey = this.#getAffinityKey()
|
|
@@ -48,6 +48,7 @@ export class ConsumerManager {
|
|
|
48
48
|
subscriptionMode,
|
|
49
49
|
subscriptionFrom,
|
|
50
50
|
each,
|
|
51
|
+
maxPartitions,
|
|
51
52
|
signal
|
|
52
53
|
} = options
|
|
53
54
|
|
|
@@ -67,7 +68,7 @@ export class ConsumerManager {
|
|
|
67
68
|
|
|
68
69
|
// Build the path and params for pop requests
|
|
69
70
|
const path = this.#buildPath(queue, partition, namespace, task)
|
|
70
|
-
const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck)
|
|
71
|
+
const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions)
|
|
71
72
|
|
|
72
73
|
// Generate affinity key for consistent routing to same backend
|
|
73
74
|
const affinityKey = this.#getAffinityKey(queue, partition, namespace, task, group)
|
|
@@ -389,7 +390,7 @@ export class ConsumerManager {
|
|
|
389
390
|
throw new Error('Must specify queue, namespace, or task')
|
|
390
391
|
}
|
|
391
392
|
|
|
392
|
-
#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck) {
|
|
393
|
+
#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions) {
|
|
393
394
|
const params = new URLSearchParams({
|
|
394
395
|
batch: batch.toString(),
|
|
395
396
|
wait: wait.toString(),
|
|
@@ -401,6 +402,8 @@ export class ConsumerManager {
|
|
|
401
402
|
if (subscriptionFrom) params.append('subscriptionFrom', subscriptionFrom)
|
|
402
403
|
if (namespace) params.append('namespace', namespace)
|
|
403
404
|
if (task) params.append('task', task)
|
|
405
|
+
// v4 multi-partition pop: drain up to N sparse partitions per call.
|
|
406
|
+
if (maxPartitions && maxPartitions > 1) params.append('partitions', maxPartitions.toString())
|
|
404
407
|
// NEVER send autoAck for consume - client always manages acking
|
|
405
408
|
// autoAck is only for pop() where server auto-acks immediately
|
|
406
409
|
|
|
@@ -43,7 +43,8 @@ export const CONSUME_DEFAULTS = {
|
|
|
43
43
|
renewLease: false, // No auto-renewal
|
|
44
44
|
renewLeaseIntervalMillis: null, // Auto-renewal interval when enabled
|
|
45
45
|
subscriptionMode: null, // No subscription mode (standard queue mode)
|
|
46
|
-
subscriptionFrom: null
|
|
46
|
+
subscriptionFrom: null, // No subscription start point
|
|
47
|
+
maxPartitions: 1 // v4 multi-partition pop cap (1 = legacy single-partition)
|
|
47
48
|
}
|
|
48
49
|
|
|
49
50
|
export const POP_DEFAULTS = {
|
package/package.json
CHANGED
package/test-v2/auth.js
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
// Tests for server-stamped producer identity (issue #23, feature A).
|
|
2
|
+
//
|
|
3
|
+
// The goal of feature A is: when JWT auth is enabled, the server attaches the
|
|
4
|
+
// authenticated client's `sub` claim to every pushed message, and a client
|
|
5
|
+
// cannot spoof another producer's identity by supplying `producerSub` in the
|
|
6
|
+
// request body.
|
|
7
|
+
//
|
|
8
|
+
// Three environments are exercised here:
|
|
9
|
+
//
|
|
10
|
+
// 1. No auth (default dev setup): `producer_sub` must be NULL; clients that
|
|
11
|
+
// send `producerSub` in the body must still get NULL (the field is not
|
|
12
|
+
// client-settable).
|
|
13
|
+
// 2. JWT auth enabled (set JWT_SECRET env to match the server's
|
|
14
|
+
// JWT_SECRET): `producer_sub` must equal the `sub` claim.
|
|
15
|
+
// 3. JWT auth enabled + spoofing attempt: body-supplied `producerSub` must
|
|
16
|
+
// be ignored; stored value is the JWT `sub`.
|
|
17
|
+
//
|
|
18
|
+
// Tests gate themselves on the `JWT_SECRET` env var so the suite can run
|
|
19
|
+
// unattended in either server configuration.
|
|
20
|
+
|
|
21
|
+
import crypto from 'crypto'
|
|
22
|
+
import { dbPool } from './run.js'
|
|
23
|
+
|
|
24
|
+
const SERVER_URL = process.env.QUEEN_URL || 'http://localhost:6632'
|
|
25
|
+
const JWT_SECRET = process.env.JWT_SECRET || ''
|
|
26
|
+
const JWT_ENABLED = JWT_SECRET.length > 0
|
|
27
|
+
|
|
28
|
+
// ---------------------------------------------------------------------------
|
|
29
|
+
// Minimal HS256 JWT signer (avoids pulling a dependency just for tests)
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
function b64url(buf) {
|
|
32
|
+
return Buffer.from(buf).toString('base64')
|
|
33
|
+
.replace(/=/g, '').replace(/\+/g, '-').replace(/\//g, '_')
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function signHs256Jwt(payload, secret) {
|
|
37
|
+
const header = { alg: 'HS256', typ: 'JWT' }
|
|
38
|
+
const encHeader = b64url(JSON.stringify(header))
|
|
39
|
+
const encPayload = b64url(JSON.stringify(payload))
|
|
40
|
+
const signingInput = `${encHeader}.${encPayload}`
|
|
41
|
+
const sig = crypto.createHmac('sha256', secret).update(signingInput).digest()
|
|
42
|
+
return `${signingInput}.${b64url(sig)}`
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function makeToken(sub, role = 'read-write') {
|
|
46
|
+
const now = Math.floor(Date.now() / 1000)
|
|
47
|
+
return signHs256Jwt(
|
|
48
|
+
{ sub, username: sub, role, iat: now, exp: now + 3600 },
|
|
49
|
+
JWT_SECRET
|
|
50
|
+
)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
// Ground-truth helper: read producer_sub directly from Postgres, bypassing
|
|
55
|
+
// the API so we can tell the difference between a server that actually stores
|
|
56
|
+
// the value and one that just echoes the client's input.
|
|
57
|
+
// ---------------------------------------------------------------------------
|
|
58
|
+
async function getStoredProducerSub(queueName, transactionId) {
|
|
59
|
+
const res = await dbPool.query(
|
|
60
|
+
`SELECT m.producer_sub
|
|
61
|
+
FROM queen.messages m
|
|
62
|
+
JOIN queen.partitions p ON p.id = m.partition_id
|
|
63
|
+
JOIN queen.queues q ON q.id = p.queue_id
|
|
64
|
+
WHERE q.name = $1 AND m.transaction_id = $2`,
|
|
65
|
+
[queueName, transactionId]
|
|
66
|
+
)
|
|
67
|
+
return res.rows.length > 0 ? res.rows[0].producer_sub : null
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Raw HTTP push so we can control the Authorization header and inject
|
|
71
|
+
// arbitrary extra fields (like a forged `producerSub`) per-test.
|
|
72
|
+
async function httpPush({ queue, partition = 'Default', transactionId, data, bearerToken, extraItemFields = {} }) {
|
|
73
|
+
const headers = { 'Content-Type': 'application/json' }
|
|
74
|
+
if (bearerToken) headers['Authorization'] = `Bearer ${bearerToken}`
|
|
75
|
+
|
|
76
|
+
const body = {
|
|
77
|
+
items: [{
|
|
78
|
+
queue,
|
|
79
|
+
partition,
|
|
80
|
+
transactionId,
|
|
81
|
+
payload: data,
|
|
82
|
+
...extraItemFields
|
|
83
|
+
}]
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const res = await fetch(`${SERVER_URL}/api/v1/push`, {
|
|
87
|
+
method: 'POST',
|
|
88
|
+
headers,
|
|
89
|
+
body: JSON.stringify(body)
|
|
90
|
+
})
|
|
91
|
+
const text = await res.text()
|
|
92
|
+
if (!res.ok) {
|
|
93
|
+
throw new Error(`push failed ${res.status}: ${text}`)
|
|
94
|
+
}
|
|
95
|
+
return text ? JSON.parse(text) : null
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Raw HTTP configure (queue creation) with optional bearer.
|
|
99
|
+
async function httpConfigure(queue, bearerToken) {
|
|
100
|
+
const headers = { 'Content-Type': 'application/json' }
|
|
101
|
+
if (bearerToken) headers['Authorization'] = `Bearer ${bearerToken}`
|
|
102
|
+
const res = await fetch(`${SERVER_URL}/api/v1/configure`, {
|
|
103
|
+
method: 'POST',
|
|
104
|
+
headers,
|
|
105
|
+
body: JSON.stringify({ queue, options: {} })
|
|
106
|
+
})
|
|
107
|
+
if (!res.ok) {
|
|
108
|
+
const text = await res.text()
|
|
109
|
+
throw new Error(`configure failed ${res.status}: ${text}`)
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Direct SP call via dbPool — bypasses HTTP and JWT entirely. Used by the
|
|
114
|
+
// SP-level tests below so they work regardless of whether the server has
|
|
115
|
+
// auth enabled or not.
|
|
116
|
+
async function callPushMessagesV3(items) {
|
|
117
|
+
await dbPool.query(
|
|
118
|
+
'SELECT queen.push_messages_v3($1::jsonb)',
|
|
119
|
+
[JSON.stringify(items)]
|
|
120
|
+
)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Direct SP pop. Returns the `messages` array for the first (only) partition.
|
|
124
|
+
async function callPopSpecificBatch(queueName) {
|
|
125
|
+
const reqs = [{
|
|
126
|
+
idx: 0,
|
|
127
|
+
queue_name: queueName,
|
|
128
|
+
partition_name: 'Default',
|
|
129
|
+
consumer_group: '__QUEUE_MODE__',
|
|
130
|
+
lease_seconds: 60,
|
|
131
|
+
worker_id: `test-worker-${Date.now()}`,
|
|
132
|
+
batch_size: 10,
|
|
133
|
+
sub_mode: 'all',
|
|
134
|
+
sub_from: ''
|
|
135
|
+
}]
|
|
136
|
+
const res = await dbPool.query(
|
|
137
|
+
'SELECT queen.pop_specific_batch($1::jsonb) as out',
|
|
138
|
+
[JSON.stringify(reqs)]
|
|
139
|
+
)
|
|
140
|
+
const out = res.rows[0].out
|
|
141
|
+
return out[0]?.result?.messages || []
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// ===========================================================================
|
|
145
|
+
// TEST 1: Stored procedure round-trip (SQL-only, no HTTP, no JWT)
|
|
146
|
+
// ---------------------------------------------------------------------------
|
|
147
|
+
// Exercises the schema + SP contract end-to-end purely via SQL. If this fails,
|
|
148
|
+
// the migration or 001_push.sql / 002_pop_unified.sql is wrong. Runs in any
|
|
149
|
+
// server configuration because it talks to Postgres directly.
|
|
150
|
+
// ===========================================================================
|
|
151
|
+
export async function producerSubRoundTripViaStoredProcedure(client) {
|
|
152
|
+
const q = 'test-auth-sp-roundtrip'
|
|
153
|
+
const tx = `tx-sp-${Date.now()}`
|
|
154
|
+
|
|
155
|
+
await callPushMessagesV3([{
|
|
156
|
+
queue: q,
|
|
157
|
+
partition: 'Default',
|
|
158
|
+
transactionId: tx,
|
|
159
|
+
payload: { x: 1 },
|
|
160
|
+
producerSub: 'sp-test-sub'
|
|
161
|
+
}])
|
|
162
|
+
|
|
163
|
+
const stored = await getStoredProducerSub(q, tx)
|
|
164
|
+
if (stored !== 'sp-test-sub') {
|
|
165
|
+
return { success: false, message: `Expected producer_sub='sp-test-sub' in DB, got ${JSON.stringify(stored)}` }
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const popped = await callPopSpecificBatch(q)
|
|
169
|
+
const target = popped.find(m => m.transactionId === tx)
|
|
170
|
+
if (!target) {
|
|
171
|
+
return { success: false, message: `Expected to find tx ${tx} in pop result` }
|
|
172
|
+
}
|
|
173
|
+
if (target.producerSub !== 'sp-test-sub') {
|
|
174
|
+
return { success: false, message: `Pop returned producerSub=${JSON.stringify(target.producerSub)}, expected 'sp-test-sub'` }
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
return { success: true, message: 'producer_sub round-trips through push_messages_v3 and pop exposes it' }
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// ===========================================================================
|
|
181
|
+
// TEST 2: SP with no producerSub => producer_sub is NULL
|
|
182
|
+
// ---------------------------------------------------------------------------
|
|
183
|
+
// Guards against an accidental "default to empty string" regression in the SP.
|
|
184
|
+
// ===========================================================================
|
|
185
|
+
export async function producerSubNullWhenNotProvided(client) {
|
|
186
|
+
const q = 'test-auth-sp-null'
|
|
187
|
+
const tx = `tx-sp-null-${Date.now()}`
|
|
188
|
+
|
|
189
|
+
await callPushMessagesV3([{
|
|
190
|
+
queue: q,
|
|
191
|
+
partition: 'Default',
|
|
192
|
+
transactionId: tx,
|
|
193
|
+
payload: { x: 1 }
|
|
194
|
+
// no producerSub
|
|
195
|
+
}])
|
|
196
|
+
|
|
197
|
+
const stored = await getStoredProducerSub(q, tx)
|
|
198
|
+
if (stored !== null) {
|
|
199
|
+
return { success: false, message: `Expected NULL producer_sub, got ${JSON.stringify(stored)}` }
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const popped = await callPopSpecificBatch(q)
|
|
203
|
+
const target = popped.find(m => m.transactionId === tx)
|
|
204
|
+
if (!target) {
|
|
205
|
+
return { success: false, message: `Expected to find tx ${tx} in pop result` }
|
|
206
|
+
}
|
|
207
|
+
if (target.producerSub != null) {
|
|
208
|
+
return { success: false, message: `Pop returned producerSub=${JSON.stringify(target.producerSub)}, expected null` }
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
return { success: true, message: 'Omitting producerSub yields NULL in DB and null in pop response' }
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// ===========================================================================
|
|
215
|
+
// TEST 3: SP treats empty string producerSub as NULL
|
|
216
|
+
// ---------------------------------------------------------------------------
|
|
217
|
+
// The SP uses NULLIF(..., '') to avoid storing a literal empty string when a
|
|
218
|
+
// client accidentally sends "". Verifies that invariant.
|
|
219
|
+
// ===========================================================================
|
|
220
|
+
export async function producerSubEmptyStringStoredAsNull(client) {
|
|
221
|
+
const q = 'test-auth-sp-empty'
|
|
222
|
+
const tx = `tx-sp-empty-${Date.now()}`
|
|
223
|
+
|
|
224
|
+
await callPushMessagesV3([{
|
|
225
|
+
queue: q,
|
|
226
|
+
partition: 'Default',
|
|
227
|
+
transactionId: tx,
|
|
228
|
+
payload: { x: 1 },
|
|
229
|
+
producerSub: ''
|
|
230
|
+
}])
|
|
231
|
+
|
|
232
|
+
const stored = await getStoredProducerSub(q, tx)
|
|
233
|
+
if (stored !== null) {
|
|
234
|
+
return { success: false, message: `Expected NULL for empty-string producerSub, got ${JSON.stringify(stored)}` }
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
return { success: true, message: 'Empty-string producerSub is stored as NULL' }
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// ===========================================================================
|
|
241
|
+
// TEST 4: HTTP push without JWT => producer_sub is NULL
|
|
242
|
+
// ---------------------------------------------------------------------------
|
|
243
|
+
// Runs only when the server is NOT configured with JWT (typical dev setup).
|
|
244
|
+
// Also checks that a client sending `producerSub` in the push body does NOT
|
|
245
|
+
// end up with it persisted (the field is not client-settable under any mode).
|
|
246
|
+
// ===========================================================================
|
|
247
|
+
export async function producerSubIgnoredFromBodyWithoutAuth(client) {
|
|
248
|
+
if (JWT_ENABLED) {
|
|
249
|
+
return { success: true, message: 'Skipped (JWT_SECRET is set - run producerSubStampedFromJwt instead)' }
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
const q = 'test-auth-http-no-jwt'
|
|
253
|
+
const tx = `tx-noauth-${Date.now()}`
|
|
254
|
+
|
|
255
|
+
const queue = await client.queue(q).create()
|
|
256
|
+
if (!queue.configured) {
|
|
257
|
+
return { success: false, message: 'Queue not created' }
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// Spoofing attempt against an unauthenticated server.
|
|
261
|
+
await httpPush({
|
|
262
|
+
queue: q,
|
|
263
|
+
transactionId: tx,
|
|
264
|
+
data: { hello: 'world' },
|
|
265
|
+
extraItemFields: { producerSub: 'attacker-no-jwt' }
|
|
266
|
+
})
|
|
267
|
+
|
|
268
|
+
const stored = await getStoredProducerSub(q, tx)
|
|
269
|
+
if (stored !== null) {
|
|
270
|
+
return { success: false, message: `Expected NULL producer_sub (auth disabled), got ${JSON.stringify(stored)} - client was able to set it!` }
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
return { success: true, message: 'Body-supplied producerSub ignored when auth disabled; stored as NULL' }
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// ===========================================================================
|
|
277
|
+
// TEST 5: HTTP push WITH JWT => producer_sub = sub claim
|
|
278
|
+
// ---------------------------------------------------------------------------
|
|
279
|
+
// Requires:
|
|
280
|
+
// - Server started with JWT_ENABLED=true JWT_ALGORITHM=HS256 JWT_SECRET=<x>
|
|
281
|
+
// - Test run with JWT_SECRET=<x>
|
|
282
|
+
// ===========================================================================
|
|
283
|
+
export async function producerSubStampedFromJwt(client) {
|
|
284
|
+
if (!JWT_ENABLED) {
|
|
285
|
+
return { success: true, message: 'Skipped (set JWT_SECRET env var matching server to run)' }
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
const q = 'test-auth-http-jwt-stamp'
|
|
289
|
+
const tx = `tx-jwt-${Date.now()}`
|
|
290
|
+
const token = makeToken('alice-producer')
|
|
291
|
+
|
|
292
|
+
await httpConfigure(q, token)
|
|
293
|
+
await httpPush({
|
|
294
|
+
queue: q,
|
|
295
|
+
transactionId: tx,
|
|
296
|
+
data: { hello: 'world' },
|
|
297
|
+
bearerToken: token
|
|
298
|
+
})
|
|
299
|
+
|
|
300
|
+
const stored = await getStoredProducerSub(q, tx)
|
|
301
|
+
if (stored !== 'alice-producer') {
|
|
302
|
+
return { success: false, message: `Expected producer_sub='alice-producer', got ${JSON.stringify(stored)}` }
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
return { success: true, message: 'producer_sub stamped from authenticated JWT sub claim' }
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// ===========================================================================
|
|
309
|
+
// TEST 6: Spoofing producerSub in body is ignored even with valid JWT
|
|
310
|
+
// ---------------------------------------------------------------------------
|
|
311
|
+
// This is the core anti-impersonation invariant of issue #23.
|
|
312
|
+
// ===========================================================================
|
|
313
|
+
export async function producerSubSpoofingIgnoredWithJwt(client) {
|
|
314
|
+
if (!JWT_ENABLED) {
|
|
315
|
+
return { success: true, message: 'Skipped (set JWT_SECRET env var matching server to run)' }
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const q = 'test-auth-http-jwt-spoof'
|
|
319
|
+
const tx = `tx-spoof-${Date.now()}`
|
|
320
|
+
const token = makeToken('legit-producer')
|
|
321
|
+
|
|
322
|
+
await httpConfigure(q, token)
|
|
323
|
+
await httpPush({
|
|
324
|
+
queue: q,
|
|
325
|
+
transactionId: tx,
|
|
326
|
+
data: { hello: 'world' },
|
|
327
|
+
bearerToken: token,
|
|
328
|
+
extraItemFields: { producerSub: 'attacker' }
|
|
329
|
+
})
|
|
330
|
+
|
|
331
|
+
const stored = await getStoredProducerSub(q, tx)
|
|
332
|
+
if (stored !== 'legit-producer') {
|
|
333
|
+
return {
|
|
334
|
+
success: false,
|
|
335
|
+
message: `Impersonation not prevented: stored producer_sub=${JSON.stringify(stored)}, expected 'legit-producer'`
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
return { success: true, message: 'Body-supplied producerSub ignored; stored sub is from validated JWT' }
|
|
340
|
+
}
|
package/test-v2/consume.js
CHANGED
|
@@ -612,4 +612,100 @@ export async function autoRenewLease(client) {
|
|
|
612
612
|
})
|
|
613
613
|
|
|
614
614
|
return { success: shouldFailHasFailed === false && shouldPassHasPass === true, message: 'Auto renew lease test completed successfully' }
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
// ============================================================================
|
|
619
|
+
// v4 Multi-Partition Consume Tests
|
|
620
|
+
// ============================================================================
|
|
621
|
+
//
|
|
622
|
+
// Exercises the .partitions(N) builder method against the high-level
|
|
623
|
+
// consume() API (which calls pop_unified_batch_v4 under the hood).
|
|
624
|
+
|
|
625
|
+
export async function testConsumerMultiPartition(client) {
|
|
626
|
+
const queueName = 'test-queue-v2-v4-consume-multi'
|
|
627
|
+
const queue = await client.queue(queueName).create()
|
|
628
|
+
if (!queue.configured) {
|
|
629
|
+
return { success: false, message: 'Queue not created' }
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
// Seed 3 partitions × 4 messages each = 12 total.
|
|
633
|
+
for (let p = 0; p < 3; p++) {
|
|
634
|
+
const payloads = []
|
|
635
|
+
for (let m = 0; m < 4; m++) payloads.push({ data: { p, m } })
|
|
636
|
+
await client.queue(queueName).partition(`p${p}`).push(payloads)
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
// Allow the post-push partition_lookup follow-up to settle.
|
|
640
|
+
await new Promise(r => setTimeout(r, 500))
|
|
641
|
+
|
|
642
|
+
let received = []
|
|
643
|
+
|
|
644
|
+
// .partitions(3).batch(100) → up to 100 msgs from up to 3 partitions
|
|
645
|
+
// in a single pop. .limit(1) ends the consume after one batch.
|
|
646
|
+
await client
|
|
647
|
+
.queue(queueName)
|
|
648
|
+
.batch(100)
|
|
649
|
+
.partitions(3)
|
|
650
|
+
.wait(false)
|
|
651
|
+
.limit(1)
|
|
652
|
+
.consume(async msgs => {
|
|
653
|
+
received = msgs
|
|
654
|
+
})
|
|
655
|
+
|
|
656
|
+
if (received.length !== 12) {
|
|
657
|
+
return { success: false, message: `Expected 12 messages across 3 partitions, got ${received.length}` }
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
const distinctPartitionIds = new Set(received.map(m => m.partitionId))
|
|
661
|
+
const distinctLeaseIds = new Set(received.map(m => m.leaseId).filter(Boolean))
|
|
662
|
+
|
|
663
|
+
if (distinctPartitionIds.size !== 3) {
|
|
664
|
+
return { success: false, message: `Expected 3 distinct partitionIds, got ${distinctPartitionIds.size}` }
|
|
665
|
+
}
|
|
666
|
+
// consume() uses client-side auto-ack which means leaseId may be
|
|
667
|
+
// empty if server-side autoAck is used. Accept either: shared lease
|
|
668
|
+
// id (1 distinct value) or all-empty (0).
|
|
669
|
+
if (distinctLeaseIds.size > 1) {
|
|
670
|
+
return { success: false, message: `All messages must share at most one leaseId, got ${distinctLeaseIds.size}` }
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
return { success: true, message: 'Consumer with .partitions(3) drained 3 partitions in one batch' }
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
export async function testConsumerMultiPartitionGlobalCap(client) {
|
|
677
|
+
const queueName = 'test-queue-v2-v4-consume-cap'
|
|
678
|
+
const queue = await client.queue(queueName).create()
|
|
679
|
+
if (!queue.configured) {
|
|
680
|
+
return { success: false, message: 'Queue not created' }
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
// Seed 5 partitions × 50 messages each = 250 total available.
|
|
684
|
+
for (let p = 0; p < 5; p++) {
|
|
685
|
+
const payloads = []
|
|
686
|
+
for (let m = 0; m < 50; m++) payloads.push({ data: { p, m } })
|
|
687
|
+
await client.queue(queueName).partition(`p${p}`).push(payloads)
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
await new Promise(r => setTimeout(r, 500))
|
|
691
|
+
|
|
692
|
+
let received = []
|
|
693
|
+
|
|
694
|
+
// batch(20) is the GLOBAL cap. With partitions(5) we can claim up to
|
|
695
|
+
// 5 partitions, but the total returned is bounded at 20 messages.
|
|
696
|
+
await client
|
|
697
|
+
.queue(queueName)
|
|
698
|
+
.batch(20)
|
|
699
|
+
.partitions(5)
|
|
700
|
+
.wait(false)
|
|
701
|
+
.limit(1)
|
|
702
|
+
.consume(async msgs => {
|
|
703
|
+
received = msgs
|
|
704
|
+
})
|
|
705
|
+
|
|
706
|
+
if (received.length !== 20) {
|
|
707
|
+
return { success: false, message: `batch(20) must be a hard global cap, got ${received.length}` }
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
return { success: true, message: 'Global batch cap respected across multi-partition consume' }
|
|
615
711
|
}
|