queen-mq 0.12.1 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client-v2/Queen.js +5 -3
- package/client-v2/http/HttpClient.js +7 -2
- package/client-v2/utils/defaults.js +2 -1
- package/package.json +1 -1
- package/test-v2/bootstrap.js +316 -0
- package/test-v2/run.js +4 -15
- package/test-v2/AI_TEST_SUMMARY.md +0 -226
- package/test-v2/GETTING_STARTED.md +0 -181
- package/test-v2/MAINTENANCE_TEST.md +0 -148
- package/test-v2/README_SUBSCRIPTION_TESTS.md +0 -201
- package/test-v2/ai_buffering.js +0 -194
- package/test-v2/ai_error_handling.js +0 -223
- package/test-v2/ai_lease_renewal.js +0 -206
- package/test-v2/ai_mixed_scenarios.js +0 -278
- package/test-v2/ai_priority.js +0 -169
- package/test-v2/ai_resources.js +0 -217
- package/test-v2/ai_ttl_retention.js +0 -170
package/client-v2/Queen.js
CHANGED
|
@@ -77,7 +77,7 @@ export class Queen {
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
#createHttpClient() {
|
|
80
|
-
const { urls, timeoutMillis, retryAttempts, retryDelayMillis, loadBalancingStrategy, affinityHashRing, healthRetryAfterMillis, enableFailover, bearerToken } = this.#config
|
|
80
|
+
const { urls, timeoutMillis, retryAttempts, retryDelayMillis, loadBalancingStrategy, affinityHashRing, healthRetryAfterMillis, enableFailover, bearerToken, headers } = this.#config
|
|
81
81
|
|
|
82
82
|
if (urls.length === 1) {
|
|
83
83
|
// Single server
|
|
@@ -86,7 +86,8 @@ export class Queen {
|
|
|
86
86
|
timeoutMillis,
|
|
87
87
|
retryAttempts,
|
|
88
88
|
retryDelayMillis,
|
|
89
|
-
bearerToken
|
|
89
|
+
bearerToken,
|
|
90
|
+
headers
|
|
90
91
|
})
|
|
91
92
|
}
|
|
92
93
|
|
|
@@ -101,7 +102,8 @@ export class Queen {
|
|
|
101
102
|
retryAttempts,
|
|
102
103
|
retryDelayMillis,
|
|
103
104
|
enableFailover,
|
|
104
|
-
bearerToken
|
|
105
|
+
bearerToken,
|
|
106
|
+
headers
|
|
105
107
|
})
|
|
106
108
|
}
|
|
107
109
|
|
|
@@ -12,6 +12,7 @@ export class HttpClient {
|
|
|
12
12
|
#retryDelayMillis
|
|
13
13
|
#enableFailover
|
|
14
14
|
#bearerToken
|
|
15
|
+
#headers
|
|
15
16
|
|
|
16
17
|
constructor(options = {}) {
|
|
17
18
|
const {
|
|
@@ -21,7 +22,8 @@ export class HttpClient {
|
|
|
21
22
|
retryAttempts = 3,
|
|
22
23
|
retryDelayMillis = 1000,
|
|
23
24
|
enableFailover = true,
|
|
24
|
-
bearerToken = null
|
|
25
|
+
bearerToken = null,
|
|
26
|
+
headers = {}
|
|
25
27
|
} = options
|
|
26
28
|
|
|
27
29
|
this.#baseUrl = baseUrl
|
|
@@ -31,6 +33,7 @@ export class HttpClient {
|
|
|
31
33
|
this.#retryDelayMillis = retryDelayMillis
|
|
32
34
|
this.#enableFailover = enableFailover
|
|
33
35
|
this.#bearerToken = bearerToken
|
|
36
|
+
this.#headers = headers || {}
|
|
34
37
|
|
|
35
38
|
logger.log('HttpClient.constructor', {
|
|
36
39
|
hasLoadBalancer: !!loadBalancer,
|
|
@@ -38,7 +41,8 @@ export class HttpClient {
|
|
|
38
41
|
timeoutMillis,
|
|
39
42
|
retryAttempts,
|
|
40
43
|
enableFailover,
|
|
41
|
-
hasAuth: !!bearerToken
|
|
44
|
+
hasAuth: !!bearerToken,
|
|
45
|
+
customHeaders: Object.keys(this.#headers).length
|
|
42
46
|
})
|
|
43
47
|
}
|
|
44
48
|
|
|
@@ -54,6 +58,7 @@ export class HttpClient {
|
|
|
54
58
|
if (this.#bearerToken) {
|
|
55
59
|
headers['Authorization'] = `Bearer ${this.#bearerToken}`
|
|
56
60
|
}
|
|
61
|
+
Object.assign(headers, this.#headers)
|
|
57
62
|
|
|
58
63
|
const options = {
|
|
59
64
|
method,
|
|
@@ -14,7 +14,8 @@ export const CLIENT_DEFAULTS = {
|
|
|
14
14
|
affinityHashRing: 128, // Number of virtual nodes per server for affinity strategy
|
|
15
15
|
enableFailover: true, // Auto-failover to other servers
|
|
16
16
|
healthRetryAfterMillis: 5000, // Retry unhealthy backends after 5 seconds
|
|
17
|
-
bearerToken: null
|
|
17
|
+
bearerToken: null, // Bearer token for proxy authentication
|
|
18
|
+
headers: {} // Custom headers to include in every request
|
|
18
19
|
}
|
|
19
20
|
|
|
20
21
|
export const QUEUE_DEFAULTS = {
|
package/package.json
CHANGED
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Consumer Group Bootstrap Tests
|
|
3
|
+
*
|
|
4
|
+
* Tests the partition_consumers bootstrap optimization:
|
|
5
|
+
* When a new consumer group registers (via subscription mode), all existing
|
|
6
|
+
* partition_consumers rows should be pre-seeded in bulk, avoiding slow
|
|
7
|
+
* one-at-a-time discovery.
|
|
8
|
+
*
|
|
9
|
+
* Test scenarios:
|
|
10
|
+
* 1. testCgBootstrapNew: CG with subscriptionMode('new')
|
|
11
|
+
* 2. testCgBootstrapTimestamp: CG with subscriptionFrom(timestamp)
|
|
12
|
+
*
|
|
13
|
+
* Both push 5000 partitions, register a CG (triggering bootstrap), then
|
|
14
|
+
* consume new messages and verify all partitions were discovered fast.
|
|
15
|
+
*
|
|
16
|
+
* Run: node run.js testCgBootstrapNew
|
|
17
|
+
* node run.js testCgBootstrapTimestamp
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const PARTITION_COUNT = 5000
|
|
21
|
+
const QUEUE_NAME_NEW = 'test-queue-v2-bootstrap-new'
|
|
22
|
+
const QUEUE_NAME_TS = 'test-queue-v2-bootstrap-ts'
|
|
23
|
+
|
|
24
|
+
async function pushToPartitions(client, queueName, partitionCount, prefix = '') {
|
|
25
|
+
const PARALLEL_BATCH = 50
|
|
26
|
+
let pushed = 0
|
|
27
|
+
for (let i = 0; i < partitionCount; i += PARALLEL_BATCH) {
|
|
28
|
+
const batch = []
|
|
29
|
+
for (let j = i; j < Math.min(i + PARALLEL_BATCH, partitionCount); j++) {
|
|
30
|
+
batch.push(
|
|
31
|
+
client
|
|
32
|
+
.queue(queueName)
|
|
33
|
+
.partition(`p-${j}`)
|
|
34
|
+
.push([{ data: { id: j, prefix } }])
|
|
35
|
+
)
|
|
36
|
+
}
|
|
37
|
+
await Promise.all(batch)
|
|
38
|
+
pushed += batch.length
|
|
39
|
+
}
|
|
40
|
+
return pushed
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Test bootstrap with subscriptionMode('new')
|
|
45
|
+
* - Push historical messages to 5000 partitions
|
|
46
|
+
* - Register CG with mode 'new' (triggers bootstrap)
|
|
47
|
+
* - Push new messages to all partitions
|
|
48
|
+
* - Consume with CG and verify all new messages are received fast
|
|
49
|
+
*/
|
|
50
|
+
export async function testCgBootstrapNew(client) {
|
|
51
|
+
// Cleanup
|
|
52
|
+
try {
|
|
53
|
+
await client.deleteConsumerGroup('bootstrap-cg-new', true)
|
|
54
|
+
} catch (e) {
|
|
55
|
+
// Ignore
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// 1. Create queue
|
|
59
|
+
const queue = await client.queue(QUEUE_NAME_NEW).create()
|
|
60
|
+
if (!queue.configured) {
|
|
61
|
+
return { success: false, message: 'Queue not created' }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// 2. Push 1 message to each of 5000 partitions (historical messages)
|
|
65
|
+
console.log(` Pushing to ${PARTITION_COUNT} partitions...`)
|
|
66
|
+
const pushStart = Date.now()
|
|
67
|
+
await pushToPartitions(client, QUEUE_NAME_NEW, PARTITION_COUNT, 'historical')
|
|
68
|
+
const pushTime = Date.now() - pushStart
|
|
69
|
+
console.log(` Push complete: ${PARTITION_COUNT} partitions in ${pushTime}ms`)
|
|
70
|
+
|
|
71
|
+
// 3. Consume all messages in queue mode (no CG) - baseline
|
|
72
|
+
console.log(` Consuming in queue mode (no CG)...`)
|
|
73
|
+
const queueModeStart = Date.now()
|
|
74
|
+
let queueModeCount = 0
|
|
75
|
+
|
|
76
|
+
await client
|
|
77
|
+
.queue(QUEUE_NAME_NEW)
|
|
78
|
+
.concurrency(10)
|
|
79
|
+
.batch(1)
|
|
80
|
+
.wait(false)
|
|
81
|
+
.idleMillis(5000)
|
|
82
|
+
.consume(async msgs => {
|
|
83
|
+
queueModeCount += msgs.length
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
const queueModeTime = Date.now() - queueModeStart
|
|
87
|
+
console.log(` Queue mode: consumed ${queueModeCount} messages in ${queueModeTime}ms`)
|
|
88
|
+
|
|
89
|
+
if (queueModeCount !== PARTITION_COUNT) {
|
|
90
|
+
return {
|
|
91
|
+
success: false,
|
|
92
|
+
message: `Queue mode consumed ${queueModeCount}/${PARTITION_COUNT} messages in ${queueModeTime}ms`
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// 4. Register a new CG with 'new' mode
|
|
97
|
+
// The first pop triggers metadata registration + bootstrap (seeds all partition_consumers)
|
|
98
|
+
console.log(` Registering CG 'bootstrap-cg-new' with mode 'new'...`)
|
|
99
|
+
const registerStart = Date.now()
|
|
100
|
+
|
|
101
|
+
const firstPop = await client
|
|
102
|
+
.queue(QUEUE_NAME_NEW)
|
|
103
|
+
.group('bootstrap-cg-new')
|
|
104
|
+
.subscriptionMode('new')
|
|
105
|
+
.batch(1)
|
|
106
|
+
.wait(false)
|
|
107
|
+
.pop()
|
|
108
|
+
|
|
109
|
+
const registerTime = Date.now() - registerStart
|
|
110
|
+
console.log(` CG registered + bootstrap in ${registerTime}ms (first pop returned ${firstPop.length} msgs)`)
|
|
111
|
+
|
|
112
|
+
// First pop should return 0 messages (mode 'new' skips historical)
|
|
113
|
+
if (firstPop.length !== 0) {
|
|
114
|
+
return {
|
|
115
|
+
success: false,
|
|
116
|
+
message: `CG with mode 'new' should skip historical messages, got ${firstPop.length}`
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// 5. Push NEW messages to all 5000 partitions
|
|
121
|
+
console.log(` Pushing new messages to ${PARTITION_COUNT} partitions...`)
|
|
122
|
+
const pushNewStart = Date.now()
|
|
123
|
+
await pushToPartitions(client, QUEUE_NAME_NEW, PARTITION_COUNT, 'new')
|
|
124
|
+
const pushNewTime = Date.now() - pushNewStart
|
|
125
|
+
console.log(` Push new complete: ${PARTITION_COUNT} partitions in ${pushNewTime}ms`)
|
|
126
|
+
|
|
127
|
+
// 6. Consume all new messages with the CG
|
|
128
|
+
// If bootstrap worked, all partitions are already known -> fast consumption
|
|
129
|
+
// If bootstrap didn't work, it would discover one-at-a-time -> very slow
|
|
130
|
+
console.log(` Consuming with CG 'bootstrap-cg-new'...`)
|
|
131
|
+
const cgConsumeStart = Date.now()
|
|
132
|
+
let cgConsumeCount = 0
|
|
133
|
+
|
|
134
|
+
await client
|
|
135
|
+
.queue(QUEUE_NAME_NEW)
|
|
136
|
+
.group('bootstrap-cg-new')
|
|
137
|
+
.subscriptionMode('new')
|
|
138
|
+
.concurrency(10)
|
|
139
|
+
.batch(1)
|
|
140
|
+
.wait(false)
|
|
141
|
+
.idleMillis(5000)
|
|
142
|
+
.consume(async msgs => {
|
|
143
|
+
cgConsumeCount += msgs.length
|
|
144
|
+
})
|
|
145
|
+
|
|
146
|
+
const cgConsumeTime = Date.now() - cgConsumeStart
|
|
147
|
+
console.log(` CG consume: ${cgConsumeCount} messages in ${cgConsumeTime}ms`)
|
|
148
|
+
|
|
149
|
+
// 7. Drain new messages in queue mode too (cleanup)
|
|
150
|
+
console.log(` Draining new messages in queue mode...`)
|
|
151
|
+
let queueDrainCount = 0
|
|
152
|
+
await client
|
|
153
|
+
.queue(QUEUE_NAME_NEW)
|
|
154
|
+
.concurrency(10)
|
|
155
|
+
.batch(1)
|
|
156
|
+
.wait(false)
|
|
157
|
+
.idleMillis(5000)
|
|
158
|
+
.consume(async msgs => {
|
|
159
|
+
queueDrainCount += msgs.length
|
|
160
|
+
})
|
|
161
|
+
console.log(` Queue mode drain: ${queueDrainCount} messages`)
|
|
162
|
+
|
|
163
|
+
// 8. Results
|
|
164
|
+
const allConsumed = cgConsumeCount === PARTITION_COUNT
|
|
165
|
+
const bootstrapFast = registerTime < 10000
|
|
166
|
+
|
|
167
|
+
console.log(` ---- Results (subscriptionMode='new') ----`)
|
|
168
|
+
console.log(` Partitions: ${PARTITION_COUNT}`)
|
|
169
|
+
console.log(` Push (historical): ${pushTime}ms`)
|
|
170
|
+
console.log(` Queue mode consume: ${queueModeTime}ms (${queueModeCount} msgs)`)
|
|
171
|
+
console.log(` CG registration: ${registerTime}ms (includes bootstrap)`)
|
|
172
|
+
console.log(` Push (new): ${pushNewTime}ms`)
|
|
173
|
+
console.log(` CG consume: ${cgConsumeTime}ms (${cgConsumeCount} msgs)`)
|
|
174
|
+
console.log(` Bootstrap fast: ${bootstrapFast ? 'YES' : 'NO'} (${registerTime}ms < 10000ms)`)
|
|
175
|
+
|
|
176
|
+
if (!allConsumed) {
|
|
177
|
+
return {
|
|
178
|
+
success: false,
|
|
179
|
+
message: `CG consumed ${cgConsumeCount}/${PARTITION_COUNT} in ${cgConsumeTime}ms (bootstrap: ${registerTime}ms)`
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
return {
|
|
184
|
+
success: true,
|
|
185
|
+
message: `[new] ${PARTITION_COUNT} partitions: push=${pushTime}ms, queue_mode=${queueModeTime}ms, bootstrap=${registerTime}ms, cg_consume=${cgConsumeTime}ms`
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Test bootstrap with subscriptionFrom(timestamp)
|
|
191
|
+
* - Push historical messages to 5000 partitions
|
|
192
|
+
* - Record a cutoff timestamp
|
|
193
|
+
* - Register CG with subscriptionFrom(cutoff) (triggers bootstrap via 'timestamp' path)
|
|
194
|
+
* - Push new messages to all partitions
|
|
195
|
+
* - Consume with CG and verify all new messages are received fast
|
|
196
|
+
*/
|
|
197
|
+
export async function testCgBootstrapTimestamp(client) {
|
|
198
|
+
// Cleanup
|
|
199
|
+
try {
|
|
200
|
+
await client.deleteConsumerGroup('bootstrap-cg-ts', true)
|
|
201
|
+
} catch (e) {
|
|
202
|
+
// Ignore
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// 1. Create queue
|
|
206
|
+
const queue = await client.queue(QUEUE_NAME_TS).create()
|
|
207
|
+
if (!queue.configured) {
|
|
208
|
+
return { success: false, message: 'Queue not created' }
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// 2. Push 1 message to each of 5000 partitions (historical messages)
|
|
212
|
+
console.log(` Pushing to ${PARTITION_COUNT} partitions...`)
|
|
213
|
+
const pushStart = Date.now()
|
|
214
|
+
await pushToPartitions(client, QUEUE_NAME_TS, PARTITION_COUNT, 'historical')
|
|
215
|
+
const pushTime = Date.now() - pushStart
|
|
216
|
+
console.log(` Push complete: ${PARTITION_COUNT} partitions in ${pushTime}ms`)
|
|
217
|
+
|
|
218
|
+
// 3. Record cutoff timestamp (after historical, before new messages)
|
|
219
|
+
// Wait before capturing the cutoff to handle clock skew between the
|
|
220
|
+
// client (Date.now) and PostgreSQL (NOW()). Without this, the last
|
|
221
|
+
// push batch may have a PG created_at AFTER the client-side cutoff.
|
|
222
|
+
await new Promise(resolve => setTimeout(resolve, 2000))
|
|
223
|
+
const cutoffTimestamp = new Date().toISOString()
|
|
224
|
+
console.log(` Cutoff timestamp: ${cutoffTimestamp}`)
|
|
225
|
+
|
|
226
|
+
// 4. Register a new CG with subscriptionFrom(timestamp)
|
|
227
|
+
// The first pop triggers metadata registration + bootstrap (seeds all partition_consumers)
|
|
228
|
+
console.log(` Registering CG 'bootstrap-cg-ts' with subscriptionFrom('${cutoffTimestamp}')...`)
|
|
229
|
+
const registerStart = Date.now()
|
|
230
|
+
|
|
231
|
+
const firstPop = await client
|
|
232
|
+
.queue(QUEUE_NAME_TS)
|
|
233
|
+
.group('bootstrap-cg-ts')
|
|
234
|
+
.subscriptionFrom(cutoffTimestamp)
|
|
235
|
+
.batch(1)
|
|
236
|
+
.wait(false)
|
|
237
|
+
.pop()
|
|
238
|
+
|
|
239
|
+
const registerTime = Date.now() - registerStart
|
|
240
|
+
console.log(` CG registered + bootstrap in ${registerTime}ms (first pop returned ${firstPop.length} msgs)`)
|
|
241
|
+
|
|
242
|
+
// First pop should return 0 messages (all messages are before the cutoff)
|
|
243
|
+
if (firstPop.length !== 0) {
|
|
244
|
+
return {
|
|
245
|
+
success: false,
|
|
246
|
+
message: `CG with subscriptionFrom should skip historical messages, got ${firstPop.length}`
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// 5. Push NEW messages to all 5000 partitions (after cutoff)
|
|
251
|
+
console.log(` Pushing new messages to ${PARTITION_COUNT} partitions...`)
|
|
252
|
+
const pushNewStart = Date.now()
|
|
253
|
+
await pushToPartitions(client, QUEUE_NAME_TS, PARTITION_COUNT, 'new')
|
|
254
|
+
const pushNewTime = Date.now() - pushNewStart
|
|
255
|
+
console.log(` Push new complete: ${PARTITION_COUNT} partitions in ${pushNewTime}ms`)
|
|
256
|
+
|
|
257
|
+
// 6. Consume all new messages with the CG
|
|
258
|
+
// If bootstrap worked, all partitions are already known -> fast consumption
|
|
259
|
+
// If bootstrap didn't work, it would discover one-at-a-time -> very slow
|
|
260
|
+
console.log(` Consuming with CG 'bootstrap-cg-ts'...`)
|
|
261
|
+
const cgConsumeStart = Date.now()
|
|
262
|
+
let cgConsumeCount = 0
|
|
263
|
+
|
|
264
|
+
await client
|
|
265
|
+
.queue(QUEUE_NAME_TS)
|
|
266
|
+
.group('bootstrap-cg-ts')
|
|
267
|
+
.subscriptionFrom(cutoffTimestamp)
|
|
268
|
+
.concurrency(10)
|
|
269
|
+
.batch(1)
|
|
270
|
+
.wait(false)
|
|
271
|
+
.idleMillis(5000)
|
|
272
|
+
.consume(async msgs => {
|
|
273
|
+
cgConsumeCount += msgs.length
|
|
274
|
+
})
|
|
275
|
+
|
|
276
|
+
const cgConsumeTime = Date.now() - cgConsumeStart
|
|
277
|
+
console.log(` CG consume: ${cgConsumeCount} messages in ${cgConsumeTime}ms`)
|
|
278
|
+
|
|
279
|
+
// 7. Drain new messages in queue mode too (cleanup)
|
|
280
|
+
console.log(` Draining new messages in queue mode...`)
|
|
281
|
+
let queueDrainCount = 0
|
|
282
|
+
await client
|
|
283
|
+
.queue(QUEUE_NAME_TS)
|
|
284
|
+
.concurrency(10)
|
|
285
|
+
.batch(1)
|
|
286
|
+
.wait(false)
|
|
287
|
+
.idleMillis(5000)
|
|
288
|
+
.consume(async msgs => {
|
|
289
|
+
queueDrainCount += msgs.length
|
|
290
|
+
})
|
|
291
|
+
console.log(` Queue mode drain: ${queueDrainCount} messages`)
|
|
292
|
+
|
|
293
|
+
// 8. Results
|
|
294
|
+
const allConsumed = cgConsumeCount === PARTITION_COUNT
|
|
295
|
+
const bootstrapFast = registerTime < 10000
|
|
296
|
+
|
|
297
|
+
console.log(` ---- Results (subscriptionFrom=timestamp) ----`)
|
|
298
|
+
console.log(` Partitions: ${PARTITION_COUNT}`)
|
|
299
|
+
console.log(` Push (historical): ${pushTime}ms`)
|
|
300
|
+
console.log(` CG registration: ${registerTime}ms (includes bootstrap)`)
|
|
301
|
+
console.log(` Push (new): ${pushNewTime}ms`)
|
|
302
|
+
console.log(` CG consume: ${cgConsumeTime}ms (${cgConsumeCount} msgs)`)
|
|
303
|
+
console.log(` Bootstrap fast: ${bootstrapFast ? 'YES' : 'NO'} (${registerTime}ms < 10000ms)`)
|
|
304
|
+
|
|
305
|
+
if (!allConsumed) {
|
|
306
|
+
return {
|
|
307
|
+
success: false,
|
|
308
|
+
message: `CG consumed ${cgConsumeCount}/${PARTITION_COUNT} in ${cgConsumeTime}ms (bootstrap: ${registerTime}ms)`
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
return {
|
|
313
|
+
success: true,
|
|
314
|
+
message: `[timestamp] ${PARTITION_COUNT} partitions: push=${pushTime}ms, bootstrap=${registerTime}ms, cg_consume=${cgConsumeTime}ms`
|
|
315
|
+
}
|
|
316
|
+
}
|
package/test-v2/run.js
CHANGED
|
@@ -11,13 +11,7 @@ import * as transactionTests from './transaction.js'
|
|
|
11
11
|
import * as subscriptionTests from './subscription.js'
|
|
12
12
|
import * as maintenanceTests from './maintenance.js'
|
|
13
13
|
import * as retentionTests from './retention.js'
|
|
14
|
-
import * as
|
|
15
|
-
import * as leaseRenewalTests from './ai_lease_renewal.js'
|
|
16
|
-
import * as resourcesTests from './ai_resources.js'
|
|
17
|
-
import * as bufferingTests from './ai_buffering.js'
|
|
18
|
-
import * as priorityTests from './ai_priority.js'
|
|
19
|
-
import * as ttlRetentionTests from './ai_ttl_retention.js'
|
|
20
|
-
import * as mixedScenariosTests from './ai_mixed_scenarios.js'
|
|
14
|
+
import * as bootstrapTests from './bootstrap.js'
|
|
21
15
|
import { LoadBalancer } from '../client-v2/http/LoadBalancer.js';
|
|
22
16
|
|
|
23
17
|
|
|
@@ -118,17 +112,12 @@ async function main() {
|
|
|
118
112
|
transactionTests,
|
|
119
113
|
subscriptionTests,
|
|
120
114
|
retentionTests,
|
|
121
|
-
maintenanceTests
|
|
115
|
+
maintenanceTests,
|
|
116
|
+
bootstrapTests
|
|
122
117
|
]
|
|
123
118
|
|
|
124
119
|
const aiTests = [
|
|
125
|
-
|
|
126
|
-
leaseRenewalTests,
|
|
127
|
-
resourcesTests,
|
|
128
|
-
bufferingTests,
|
|
129
|
-
priorityTests,
|
|
130
|
-
ttlRetentionTests,
|
|
131
|
-
mixedScenariosTests
|
|
120
|
+
|
|
132
121
|
]
|
|
133
122
|
|
|
134
123
|
const allTests = [...humanTests, ...aiTests]
|
|
@@ -1,226 +0,0 @@
|
|
|
1
|
-
# AI-Generated Test Coverage Summary
|
|
2
|
-
|
|
3
|
-
This document describes the additional test files created to improve test coverage for the Queen Message Queue client v2.
|
|
4
|
-
|
|
5
|
-
## New Test Files Created
|
|
6
|
-
|
|
7
|
-
### 1. `ai_error_handling.js` - Error Handling & Edge Cases
|
|
8
|
-
|
|
9
|
-
Tests various error conditions and edge cases to ensure the system handles failures gracefully.
|
|
10
|
-
|
|
11
|
-
**Tests:**
|
|
12
|
-
- `testInvalidQueueName` - Empty queue name validation
|
|
13
|
-
- `testInvalidConfiguration` - Invalid config values (negative lease times, etc.)
|
|
14
|
-
- `testInvalidMessageFormat` - Messages with incorrect structure
|
|
15
|
-
- `testPushEmptyArray` - Pushing empty message arrays
|
|
16
|
-
- `testAckWithoutPartitionId` - Acknowledgment without required partitionId
|
|
17
|
-
- `testPopFromNonExistentQueue` - Handling non-existent queues
|
|
18
|
-
- `testMultiplePushesWithSameTransactionId` - Duplicate detection across multiple pushes
|
|
19
|
-
- `testAckExpiredLease` - Attempting to ack after lease expiration
|
|
20
|
-
- `testBatchAckMixedResults` - Batch acknowledgment with some expired leases
|
|
21
|
-
- `testVeryLargePayload` - Testing system limits with very large messages
|
|
22
|
-
|
|
23
|
-
**Purpose:** Ensure robust error handling and proper validation of inputs.
|
|
24
|
-
|
|
25
|
-
---
|
|
26
|
-
|
|
27
|
-
### 2. `ai_lease_renewal.js` - Lease Extension Management
|
|
28
|
-
|
|
29
|
-
Tests manual lease renewal functionality to keep messages locked during long-running operations.
|
|
30
|
-
|
|
31
|
-
**Tests:**
|
|
32
|
-
- `testManualLeaseRenewal` - Basic lease renewal with `client.renew()`
|
|
33
|
-
- `testBatchLeaseRenewal` - Renewing multiple message leases at once
|
|
34
|
-
- `testRenewalWithLeaseId` - Renewing using just the leaseId string
|
|
35
|
-
- `testRenewalOfExpiredLease` - Attempting to renew an already-expired lease
|
|
36
|
-
- `testMultipleRenewals` - Renewing the same message multiple times
|
|
37
|
-
|
|
38
|
-
**Purpose:** Validate that lease renewal prevents messages from being re-delivered during long processing.
|
|
39
|
-
|
|
40
|
-
---
|
|
41
|
-
|
|
42
|
-
### 3. `ai_resources.js` - Resource & Status APIs
|
|
43
|
-
|
|
44
|
-
Tests the administrative and monitoring APIs for querying system state.
|
|
45
|
-
|
|
46
|
-
**Tests:**
|
|
47
|
-
- `testListQueues` - List all queues in the system
|
|
48
|
-
- `testGetQueueDetails` - Get details for a specific queue
|
|
49
|
-
- `testGetNamespaces` - List all namespaces
|
|
50
|
-
- `testGetTasks` - List all tasks
|
|
51
|
-
- `testGetMessages` - Query messages with filters
|
|
52
|
-
- `testSystemOverview` - Get system-wide statistics
|
|
53
|
-
- `testHealthCheck` - Health check endpoint
|
|
54
|
-
- `testDeleteQueue` - Delete a queue and verify removal
|
|
55
|
-
|
|
56
|
-
**Purpose:** Ensure monitoring and administrative APIs work correctly.
|
|
57
|
-
|
|
58
|
-
---
|
|
59
|
-
|
|
60
|
-
### 4. `ai_buffering.js` - Client-Side Buffering Management
|
|
61
|
-
|
|
62
|
-
Tests client-side message buffering features for high-throughput scenarios.
|
|
63
|
-
|
|
64
|
-
**Tests:**
|
|
65
|
-
- `testBufferStatistics` - Getting buffer statistics via `getBufferStats()`
|
|
66
|
-
- `testManualBufferFlush` - Manual flush with `flushAllBuffers()`
|
|
67
|
-
- `testBufferTimeThreshold` - Time-based buffer flushing
|
|
68
|
-
- `testBufferCountThreshold` - Count-based buffer flushing
|
|
69
|
-
- `testMultipleBuffers` - Multiple queues with independent buffers
|
|
70
|
-
- `testBufferWithPartition` - Buffering with specific partitions
|
|
71
|
-
|
|
72
|
-
**Purpose:** Validate client-side buffering behavior for batch optimization.
|
|
73
|
-
|
|
74
|
-
---
|
|
75
|
-
|
|
76
|
-
### 5. `ai_priority.js` - Priority Queue Testing
|
|
77
|
-
|
|
78
|
-
Tests message priority handling to ensure high-priority messages are processed first.
|
|
79
|
-
|
|
80
|
-
**Tests:**
|
|
81
|
-
- `testBasicPriority` - Queues with different priority levels
|
|
82
|
-
- `testPriorityWithNamespace` - Priority ordering within a namespace
|
|
83
|
-
- `testPriorityWithTask` - Priority ordering within a task
|
|
84
|
-
- `testDynamicPriorityChange` - Reconfiguring queue priority
|
|
85
|
-
|
|
86
|
-
**Purpose:** Ensure priority-based message processing works correctly.
|
|
87
|
-
|
|
88
|
-
---
|
|
89
|
-
|
|
90
|
-
### 6. `ai_ttl_retention.js` - TTL and Retention Policies
|
|
91
|
-
|
|
92
|
-
Tests message time-to-live and retention features.
|
|
93
|
-
|
|
94
|
-
**Tests:**
|
|
95
|
-
- `testMessageTTL` - Messages expiring based on TTL configuration
|
|
96
|
-
- `testRetentionEnabled` - Retention of pending messages
|
|
97
|
-
- `testCompletedRetention` - Retention of completed messages
|
|
98
|
-
- `testNoRetention` - Immediate deletion without retention
|
|
99
|
-
- `testMaxWaitTimeDLQ` - Messages moving to DLQ based on max wait time
|
|
100
|
-
|
|
101
|
-
**Purpose:** Validate message lifecycle and retention policies.
|
|
102
|
-
|
|
103
|
-
---
|
|
104
|
-
|
|
105
|
-
### 7. `ai_mixed_scenarios.js` - Complex Integration Tests
|
|
106
|
-
|
|
107
|
-
Tests complex scenarios combining multiple features.
|
|
108
|
-
|
|
109
|
-
**Tests:**
|
|
110
|
-
- `testMultipleQueuesSimultaneous` - Multiple queues with different configurations
|
|
111
|
-
- `testCrossQueueWorkflow` - 3-stage processing pipeline across queues
|
|
112
|
-
- `testPartitionWithConsumerGroupAndRetries` - Complex partition + group + retry scenario
|
|
113
|
-
- `testBufferedPushWithTransactionConsume` - Buffering combined with transactional consumption
|
|
114
|
-
- `testNamespaceWithMultipleQueues` - Multiple queues in same namespace
|
|
115
|
-
- `testEncryptedWithPartitionAndGroup` - Encryption + partition + consumer group
|
|
116
|
-
- `testHighConcurrencyMixedOperations` - High concurrency with mixed operations
|
|
117
|
-
|
|
118
|
-
**Purpose:** Validate that multiple features work correctly when combined.
|
|
119
|
-
|
|
120
|
-
---
|
|
121
|
-
|
|
122
|
-
## Test Coverage Summary
|
|
123
|
-
|
|
124
|
-
### Before AI Tests
|
|
125
|
-
- ✅ Queue creation/deletion/configuration
|
|
126
|
-
- ✅ Basic push/pop operations
|
|
127
|
-
- ✅ Consumer operations
|
|
128
|
-
- ✅ Transactions
|
|
129
|
-
- ✅ DLQ functionality
|
|
130
|
-
- ✅ Load testing
|
|
131
|
-
|
|
132
|
-
### After AI Tests (New Coverage)
|
|
133
|
-
- ✅ **Error handling and edge cases**
|
|
134
|
-
- ✅ **Lease renewal API**
|
|
135
|
-
- ✅ **Resource/Status APIs**
|
|
136
|
-
- ✅ **Client-side buffering management**
|
|
137
|
-
- ✅ **Priority queue behavior**
|
|
138
|
-
- ✅ **TTL and retention policies**
|
|
139
|
-
- ✅ **Complex multi-feature scenarios**
|
|
140
|
-
|
|
141
|
-
## Running the Tests
|
|
142
|
-
|
|
143
|
-
### Run Only AI-Generated Tests (45 tests)
|
|
144
|
-
```bash
|
|
145
|
-
cd /Users/alice/Work/queen/client-js/test-v2
|
|
146
|
-
node run.js ai
|
|
147
|
-
```
|
|
148
|
-
|
|
149
|
-
### Run Only Human-Written Tests (49 tests)
|
|
150
|
-
```bash
|
|
151
|
-
node run.js human
|
|
152
|
-
```
|
|
153
|
-
|
|
154
|
-
### Run All Tests (94 tests)
|
|
155
|
-
```bash
|
|
156
|
-
node run.js
|
|
157
|
-
# or
|
|
158
|
-
node run.js all
|
|
159
|
-
```
|
|
160
|
-
|
|
161
|
-
### Run Specific Test
|
|
162
|
-
```bash
|
|
163
|
-
# AI test examples
|
|
164
|
-
node run.js testManualLeaseRenewal
|
|
165
|
-
node run.js testInvalidQueueName
|
|
166
|
-
node run.js testListQueues
|
|
167
|
-
|
|
168
|
-
# Human test examples
|
|
169
|
-
node run.js pushMessage
|
|
170
|
-
node run.js testConsumer
|
|
171
|
-
node run.js transactionBasicPushAck
|
|
172
|
-
```
|
|
173
|
-
|
|
174
|
-
### List All Available Tests
|
|
175
|
-
```bash
|
|
176
|
-
# Show help with categorized test list
|
|
177
|
-
node run.js help
|
|
178
|
-
```
|
|
179
|
-
|
|
180
|
-
## Total Test Count
|
|
181
|
-
|
|
182
|
-
| Category | Original Tests | AI-Generated Tests | Total |
|
|
183
|
-
|----------|---------------|-------------------|-------|
|
|
184
|
-
| Queue Operations | 3 | 0 | 3 |
|
|
185
|
-
| Push Operations | 13 | 0 | 13 |
|
|
186
|
-
| Pop Operations | 5 | 0 | 5 |
|
|
187
|
-
| Consumer Operations | 12 | 0 | 12 |
|
|
188
|
-
| Load Tests | 3 | 0 | 3 |
|
|
189
|
-
| DLQ Tests | 1 | 0 | 1 |
|
|
190
|
-
| Transaction Tests | 11 | 0 | 11 |
|
|
191
|
-
| Complete Workflow | 1 | 0 | 1 |
|
|
192
|
-
| **Error Handling** | **0** | **10** | **10** |
|
|
193
|
-
| **Lease Renewal** | **0** | **5** | **5** |
|
|
194
|
-
| **Resources/Status** | **0** | **8** | **8** |
|
|
195
|
-
| **Buffering** | **0** | **6** | **6** |
|
|
196
|
-
| **Priority** | **0** | **4** | **4** |
|
|
197
|
-
| **TTL/Retention** | **0** | **5** | **5** |
|
|
198
|
-
| **Mixed Scenarios** | **0** | **7** | **7** |
|
|
199
|
-
| **TOTAL** | **49** | **45** | **94** |
|
|
200
|
-
|
|
201
|
-
## Notable Test Features
|
|
202
|
-
|
|
203
|
-
1. **Comprehensive Error Coverage**: Tests handle various failure modes including network errors, invalid inputs, and expired leases.
|
|
204
|
-
|
|
205
|
-
2. **Real-World Scenarios**: Mixed scenario tests simulate actual production use cases like multi-stage pipelines and cross-queue workflows.
|
|
206
|
-
|
|
207
|
-
3. **Administrative Testing**: Resource API tests enable validation of monitoring and management features.
|
|
208
|
-
|
|
209
|
-
4. **Performance Features**: Buffering and priority tests ensure high-throughput and prioritization work correctly.
|
|
210
|
-
|
|
211
|
-
5. **Data Lifecycle**: TTL and retention tests validate complete message lifecycle management.
|
|
212
|
-
|
|
213
|
-
## Test Dependencies
|
|
214
|
-
|
|
215
|
-
All tests require:
|
|
216
|
-
- Queen server running on `http://localhost:6632`
|
|
217
|
-
- PostgreSQL database accessible
|
|
218
|
-
- Node.js with ES modules support
|
|
219
|
-
|
|
220
|
-
## Notes
|
|
221
|
-
|
|
222
|
-
- Tests prefixed with `ai_*` are AI-generated
|
|
223
|
-
- All tests are non-destructive and use isolated test queues
|
|
224
|
-
- Tests clean up after themselves where possible
|
|
225
|
-
- Some tests intentionally trigger errors to validate error handling
|
|
226
|
-
|