queen-mq 1.2.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +71 -18
  2. package/client-v2/Queen.js +45 -13
  3. package/client-v2/README.md +26 -11
  4. package/client-v2/admin/Admin.js +0 -47
  5. package/client-v2/builders/QueueBuilder.js +33 -10
  6. package/client-v2/builders/TimerBuilder.js +2 -2
  7. package/client-v2/builders/TransactionBuilder.js +35 -9
  8. package/client-v2/consumer/ConsumerManager.js +67 -46
  9. package/client-v2/ephemeral/Ephemeral.js +7 -9
  10. package/client-v2/kv/Kv.js +4 -4
  11. package/client-v2/kv/expiry.js +1 -1
  12. package/client-v2/streams/Stream.js +8 -3
  13. package/client-v2/streams/helpers/rateLimiter.js +4 -4
  14. package/client-v2/streams/operators/GateOperator.js +9 -2
  15. package/client-v2/streams/operators/ReduceOperator.js +2 -2
  16. package/client-v2/streams/operators/WindowSessionOperator.js +3 -3
  17. package/client-v2/streams/runtime/Runner.js +111 -54
  18. package/client-v2/streams/runtime/cycle.js +4 -4
  19. package/client-v2/streams/runtime/register.js +1 -1
  20. package/client-v2/utils/conflation.js +0 -6
  21. package/client-v2/utils/consumerGroup.js +54 -0
  22. package/package.json +5 -8
  23. package/test-v2/_kvtimers.js +12 -13
  24. package/test-v2/ackwindow.js +12 -192
  25. package/test-v2/bootstrap.js +2 -2
  26. package/test-v2/conflation-unit/conflationWire.test.js +0 -12
  27. package/test-v2/consume.js +41 -1
  28. package/test-v2/consumer-unit/handlerError.test.js +161 -0
  29. package/test-v2/docs.js +5 -4
  30. package/test-v2/http-unit/pushStatus.test.js +142 -0
  31. package/test-v2/http-unit/renew.test.js +96 -0
  32. package/test-v2/kv-unit/timerWire.test.js +1 -1
  33. package/test-v2/kv-unit/txnWire.test.js +101 -2
  34. package/test-v2/kv.js +6 -5
  35. package/test-v2/pop.js +28 -1
  36. package/test-v2/retention.js +28 -7
  37. package/test-v2/run.js +50 -114
  38. package/test-v2/runner-unit/fatalExit.test.js +64 -0
  39. package/test-v2/semantics.js +20 -34
  40. package/test-v2/stream/_helpers.js +10 -19
  41. package/test-v2/stream/cron.js +1 -1
  42. package/test-v2/stream/gate.js +54 -0
  43. package/test-v2/stream/index.js +2 -0
  44. package/test-v2/stream/tumbling.js +13 -9
  45. package/test-v2/streams-unit/ack.test.js +61 -0
  46. package/test-v2/streams-unit/cycle.test.js +1 -1
  47. package/test-v2/streams-unit/e2e.test.js +6 -10
  48. package/test-v2/streams-unit/gate.test.js +189 -0
  49. package/test-v2/timers.js +6 -5
  50. package/test-v2/transaction.js +169 -0
  51. package/test-v2/watermark.js +38 -176
  52. package/test-v2/maintenance.js +0 -277
package/test-v2/run.js CHANGED
@@ -1,4 +1,3 @@
1
- import pg from 'pg';
2
1
  import { Queen } from '../client-v2/index.js'
3
2
  import * as queueTests from './queue.js'
4
3
  import * as pushTests from './push.js'
@@ -9,7 +8,6 @@ import * as dlqTests from './dlq.js'
9
8
  import * as completeTests from './complete.js'
10
9
  import * as transactionTests from './transaction.js'
11
10
  import * as subscriptionTests from './subscription.js'
12
- import * as maintenanceTests from './maintenance.js'
13
11
  import * as retentionTests from './retention.js'
14
12
  import * as bootstrapTests from './bootstrap.js'
15
13
  import * as loggerTests from './logger.js'
@@ -28,46 +26,43 @@ import { LoadBalancer } from '../client-v2/http/LoadBalancer.js';
28
26
 
29
27
  export const TEST_CONFIG_SINGLE = {
30
28
  baseUrls: [process.env.QUEEN_SERVER_URL || 'http://localhost:6632'],
31
- loadBalancingStrategy: 'affinity',
32
- dbConfig: {
33
- host: process.env.PG_HOST || 'localhost',
34
- port: process.env.PG_PORT || 5432,
35
- database: process.env.PG_DB || 'postgres',
36
- user: process.env.PG_USER || 'postgres',
37
- password: process.env.PG_PASSWORD || 'postgres'
38
- }
29
+ loadBalancingStrategy: 'affinity'
39
30
  };
40
31
 
41
32
  export const TEST_CONFIG_MULTIPLE = {
42
33
  baseUrls: ['http://localhost:6632','http://localhost:6633'],
43
- loadBalancingStrategy: 'round-robin',
44
- dbConfig: {
45
- host: process.env.PG_HOST || 'localhost',
46
- port: process.env.PG_PORT || 5432,
47
- database: process.env.PG_DB || 'postgres',
48
- user: process.env.PG_USER || 'postgres',
49
- password: process.env.PG_PASSWORD || 'postgres'
50
- }
34
+ loadBalancingStrategy: 'round-robin'
51
35
  };
52
36
 
53
37
  export const TEST_CONFIG = process.env.TEST_CONFIG === 'multiple' ? TEST_CONFIG_MULTIPLE : TEST_CONFIG_SINGLE;
54
38
  console.log('TEST_CONFIG:', TEST_CONFIG);
55
39
 
56
40
  // Global test state
57
- export let dbPool;
58
-
59
- // Initialize database pool
60
- export async function initDb() {
61
- dbPool = new pg.Pool(TEST_CONFIG.dbConfig);
62
- await dbPool.query('SELECT 1');
63
- return dbPool;
64
- }
65
-
66
- // Close database pool
67
- export async function closeDb() {
68
- if (dbPool) {
69
- await dbPool.end();
70
- }
41
+ let activeClient;
42
+
43
+ // The suite talks to the broker through its public HTTP API only: there is no
44
+ // database and no cleanup step. test/run.sh gives every lane a FRESH broker
45
+ // (its data volume is created empty and destroyed with `down -v`), so the
46
+ // suite starts on an empty store. A second run against the same broker is not
47
+ // expected to be green: fixed-name fixtures (docs.js's fixed transactionId,
48
+ // the kv and timer suites) would find the first run's state.
49
+ //
50
+ // The preflight makes a missing or unhealthy broker fail the run up front,
51
+ // with the reason, instead of as a wall of per-test connection errors.
52
+ async function preflightBroker(baseUrl) {
53
+ const url = `${baseUrl}/health`
54
+ let res
55
+ try {
56
+ res = await fetch(url, { signal: AbortSignal.timeout(5000) })
57
+ } catch (e) {
58
+ throw new Error(`broker preflight ${url}: ${e.cause?.message || e.message}`)
59
+ }
60
+ // Drain the body so the connection is released either way.
61
+ await res.arrayBuffer().catch(() => {})
62
+ if (!res.ok) {
63
+ throw new Error(`broker preflight ${url}: HTTP ${res.status}`)
64
+ }
65
+ log(true, `Broker preflight ${url}: HTTP ${res.status}`)
71
66
  }
72
67
 
73
68
  function log (success, ...args) {
@@ -92,78 +87,13 @@ function printResults() {
92
87
  console.log('='.repeat(80))
93
88
  }
94
89
 
95
- export const cleanupTestData = async () => {
96
- // All the LIKE patterns test queues use. The three exact names are the
97
- // documentation queues (test-v2/docs.js): purging them here is what lets
98
- // the published dedup snippet keep a fixed transactionId across runs.
99
- const patterns = ['test-%', 'edge-%', 'pattern-%', 'workflow-%', 'orders', 'payments', 'invoices'];
100
- try {
101
- // Drop streaming queries first (CASCADE removes their state rows).
102
- // Safe even when queen_streams isn't installed yet — we swallow the
103
- // error if the schema doesn't exist.
104
- try {
105
- await dbPool.query(`DELETE FROM queen_streams.queries WHERE name LIKE 'test-%'`);
106
- } catch (e) {
107
- // queen_streams schema not installed — ignore.
108
- }
109
-
110
- // Queue identity is now the queen.queues id (log_queues was merged
111
- // away): log_partitions, consumer_watermarks, consumer_groups_metadata
112
- // and queue_lag_metrics all cascade from the queues row. Only log_txns
113
- // and log_dlq have NO foreign key by design, so they get an explicit
114
- // purge keyed via log_partitions first. Without it, every suite run
115
- // inherits the previous run's messages and dedup window entries —
116
- // fixed-transactionId tests report 'duplicate' on their FIRST push.
117
- try {
118
- await dbPool.query(`
119
- WITH parts AS (
120
- SELECT lp.id FROM queen.log_partitions lp
121
- JOIN queen.queues q ON q.id = lp.queue_id
122
- WHERE q.name LIKE ANY($1::text[])
123
- ),
124
- d1 AS (DELETE FROM queen.log_txns WHERE partition_id IN (SELECT id FROM parts)),
125
- d2 AS (DELETE FROM queen.log_dlq WHERE partition_id IN (SELECT id FROM parts))
126
- SELECT 1`, [patterns]);
127
- } catch (e) {
128
- // Log-engine schema not installed (rows-only server) — ignore.
129
- }
130
-
131
- await dbPool.query(`DELETE FROM queen.queues WHERE name LIKE ANY($1::text[])`, [patterns]);
132
-
133
- // KV keys and pending timers (PLAN_KV_TIMERS.md §10.4). NOT cosmetic:
134
- // without this purge a putIfAbsent test is green on its first run and red
135
- // forever after, an incr test accumulates between runs, and a timer left
136
- // pending by an earlier run fires into a later one and shows up as a
137
- // phantom message in an unrelated test. Neither table has a foreign key
138
- // to queen.queues -- log_timers is keyed by NAMES on purpose -- so the
139
- // queue delete above does not reach them.
140
- //
141
- // Both are deleted across every tenant: a test rig may run with
142
- // QUEEN_TENANCY_HEADER on, and the rows to purge are identified by the
143
- // test naming convention, never by tenant.
144
- //
145
- // These two used to be wrapped in a swallowing try/catch, on the grounds
146
- // that a broker booted with the kv/timer flags off had never applied
147
- // 024_kv.sql / 025_timers.sql. There are no such flags: schema.rs applies
148
- // both on every boot, so a missing `queen.kv` or `queen.log_timers` is a
149
- // broken rig and must be loud. Swallowing it would leave the purge silently
150
- // undone, which is exactly the failure the purge exists to prevent -- a
151
- // putIfAbsent test green on its first run and red forever after.
152
- await dbPool.query(`DELETE FROM queen.kv WHERE namespace LIKE ANY($1::text[])`, [patterns]);
153
- await dbPool.query(`DELETE FROM queen.log_timers WHERE queue LIKE ANY($1::text[])`, [patterns]);
154
-
155
- log(true, 'Test data cleaned up (rows + segments + kv + timers)');
156
- } catch (error) {
157
- log(false, `Cleanup error: ${error.message}`);
158
- }
159
- };
160
-
161
90
  async function main() {
162
91
  const client = new Queen({
163
92
  urls: TEST_CONFIG.baseUrls,
164
93
  loadBalancingStrategy: TEST_CONFIG.loadBalancingStrategy
165
94
  })
166
- await initDb()
95
+ activeClient = client
96
+ await preflightBroker(TEST_CONFIG.baseUrls[0])
167
97
 
168
98
  // Separate human and AI tests
169
99
  const humanTests = [
@@ -177,7 +107,6 @@ async function main() {
177
107
  transactionTests,
178
108
  subscriptionTests,
179
109
  retentionTests,
180
- maintenanceTests,
181
110
  bootstrapTests,
182
111
  loggerTests,
183
112
  watermarkTests,
@@ -194,7 +123,7 @@ async function main() {
194
123
  ]
195
124
 
196
125
  // Streaming tests (queen-streams). Run via `node run.js stream`.
197
- // Require Queen v0.2+ with the queen_streams schema applied.
126
+ // Require a broker that serves the /streams/v1 routes.
198
127
  const streamGroupTests = [streamTests]
199
128
 
200
129
  const allTests = [...humanTests, ...aiTests, ...streamGroupTests]
@@ -240,8 +169,7 @@ async function main() {
240
169
  humanTestFunctions.forEach(t => console.log(` - ${t.name}`))
241
170
  console.log('\n🌊 queen-streams tests:')
242
171
  streamTestFunctions.forEach(t => console.log(` - ${t.name}`))
243
- await closeDb()
244
- process.exit(1)
172
+ return 1
245
173
  }
246
174
  testsToRun = [testFunc]
247
175
  mode = 'single'
@@ -250,9 +178,6 @@ async function main() {
250
178
  log(true, `Running all tests (${allTestFunctions.length} tests)...`)
251
179
  }
252
180
 
253
- // Cleanup test data
254
- await cleanupTestData()
255
-
256
181
  for (const test of testsToRun) {
257
182
  try {
258
183
  console.log('Running test:', test.name)
@@ -279,17 +204,28 @@ async function main() {
279
204
  console.log('\n💡 Tip: Run "node run.js" to test all tests')
280
205
  }
281
206
 
282
- //await cleanupTestData()
283
- await closeDb()
284
- // Queen.close() destroys the per-client HTTP agent, releasing keep-alive
285
- // sockets so the loop can drain; the explicit exit code is for CI.
286
- await client.close()
287
207
  const failedCount = testResults.filter(x => !x.success).length
288
- process.exit(failedCount > 0 ? 1 : 0)
208
+ return failedCount > 0 ? 1 : 0
289
209
  }
290
210
 
211
+ let exitCode = 1
291
212
  try {
292
- await main()
213
+ exitCode = await main()
293
214
  } catch (error) {
294
215
  log(false, 'Main error:', error.message)
295
- }
216
+ } finally {
217
+ // Always release the client. Previously, an init failure skipped this
218
+ // teardown and the outer catch returned a successful process status,
219
+ // allowing a broken integration lane to appear green.
220
+ const cleanupResults = await Promise.allSettled([
221
+ activeClient ? activeClient.close() : Promise.resolve()
222
+ ])
223
+ for (const result of cleanupResults) {
224
+ if (result.status === 'rejected') {
225
+ exitCode = 1
226
+ log(false, 'Cleanup error:', result.reason?.message || String(result.reason))
227
+ }
228
+ }
229
+ }
230
+
231
+ process.exit(exitCode)
@@ -0,0 +1,64 @@
1
+ import assert from 'node:assert/strict'
2
+ import { spawn } from 'node:child_process'
3
+ import net from 'node:net'
4
+ import { fileURLToPath } from 'node:url'
5
+ import path from 'node:path'
6
+ import { describe, it } from 'node:test'
7
+
8
+ const suiteRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
9
+
10
+ // A loopback port with nothing listening on it: bind an ephemeral port, then
11
+ // release it. Not port 1: fetch() rejects the Fetch-spec "bad ports" (1 is one
12
+ // of them) before it ever connects, so the preflight would fail with "bad port"
13
+ // and never exercise a refused connection.
14
+ function closedLoopbackPort() {
15
+ return new Promise((resolve, reject) => {
16
+ const server = net.createServer()
17
+ server.once('error', reject)
18
+ server.listen(0, '127.0.0.1', () => {
19
+ const { port } = server.address()
20
+ server.close(error => (error ? reject(error) : resolve(port)))
21
+ })
22
+ })
23
+ }
24
+
25
+ function runWithUnreachableBroker(brokerUrl) {
26
+ return new Promise((resolve, reject) => {
27
+ const env = { ...process.env, QUEEN_SERVER_URL: brokerUrl }
28
+ // TEST_CONFIG=multiple would point run.js at its fixed localhost URLs
29
+ // instead of QUEEN_SERVER_URL.
30
+ delete env.TEST_CONFIG
31
+ const child = spawn(process.execPath, ['test-v2/run.js', 'human'], {
32
+ cwd: suiteRoot,
33
+ env,
34
+ stdio: ['ignore', 'pipe', 'pipe']
35
+ })
36
+
37
+ let output = ''
38
+ const deadline = setTimeout(() => {
39
+ child.kill('SIGKILL')
40
+ reject(new Error(`integration runner did not exit within 5s:\n${output}`))
41
+ }, 5000)
42
+ child.stdout.on('data', chunk => { output += chunk })
43
+ child.stderr.on('data', chunk => { output += chunk })
44
+ child.once('error', error => {
45
+ clearTimeout(deadline)
46
+ reject(error)
47
+ })
48
+ child.once('close', (code, signal) => {
49
+ clearTimeout(deadline)
50
+ resolve({ code, signal, output })
51
+ })
52
+ })
53
+ }
54
+
55
+ describe('integration runner lifecycle', () => {
56
+ it('returns a failure status when the broker preflight fails', async () => {
57
+ const port = await closedLoopbackPort()
58
+ const result = await runWithUnreachableBroker(`http://127.0.0.1:${port}`)
59
+
60
+ assert.equal(result.signal, null)
61
+ assert.equal(result.code, 1, result.output)
62
+ assert.match(result.output, /Main error: broker preflight .*ECONNREFUSED/)
63
+ })
64
+ })
@@ -16,9 +16,9 @@
16
16
  * AFTER the group registered (durable subscription record).
17
17
  * 8. Queue-mode pops never skip backlog, even when carrying subscriptionMode.
18
18
  * 9. Empty polls while another worker holds the lease never strand backlog
19
- * (wildcard watermark guard).
20
- * 10. Push-only (never-configured) queues get a queen.queues row: visible in
21
- * resources, discoverable by namespace pop.
19
+ * (an empty answer must not hide the partition once the lease lapses).
20
+ * 10. Push-only (never-configured) queues are registered by the push: visible
21
+ * in resources, discoverable by namespace pop.
22
22
  * 11. GET /api/v1/messages/:pid/:txn keeps the full v0.16.0 shape (including
23
23
  * consumerGroups[].name — not .group).
24
24
  * 12. Transactions honor `retry` and `dlq` ack statuses (not collapsed to a
@@ -39,7 +39,7 @@
39
39
  * Run: node run.js <testName> e.g. node run.js implicitAckCompletesBatch
40
40
  */
41
41
 
42
- import { dbPool, TEST_CONFIG } from './run.js'
42
+ import { TEST_CONFIG } from './run.js'
43
43
 
44
44
  // Lazy: run.js imports this module before TEST_CONFIG is initialized (TDZ),
45
45
  // so the base URL must be resolved at call time, not module-eval time.
@@ -369,7 +369,7 @@ export async function queueModePopNeverSkipsBacklog(client) {
369
369
  }
370
370
 
371
371
  // ============================================================================
372
- // 9. Watermark: empty polls during a held lease must not strand backlog
372
+ // 9. Empty polls during a held lease must not strand backlog
373
373
  // ============================================================================
374
374
  export async function leasedBacklogNotStrandedByEmptyPolls(client) {
375
375
  const queue = uniq('watermark-lease')
@@ -385,34 +385,26 @@ export async function leasedBacklogNotStrandedByEmptyPolls(client) {
385
385
  if (a.length !== 1) return { success: false, message: 'Worker A did not get the message' }
386
386
 
387
387
  // Worker B (same group) polls empty repeatedly while A holds the lease.
388
- // A buggy engine advances the empty-scan watermark here even though the
389
- // backlog is only invisible because of A's lease.
388
+ // The backlog is invisible here ONLY because of A's lease: a broker that
389
+ // remembers these empty answers as "nothing to deliver" keeps hiding the
390
+ // partition after the lease lapses.
390
391
  for (let i = 0; i < 3; i++) {
391
392
  await client.queue(queue).group(group).batch(1).wait(false).pop()
392
393
  await sleep(100)
393
394
  }
394
395
 
395
- // Defeat the 2-minute candidate-filter grace so a wrongly-advanced
396
- // watermark actually hides the partition (same trick as watermark.js).
397
- // Queue identity is now the queen.queues id (log_queues merged away).
398
- await dbPool.query(`
399
- UPDATE queen.log_partitions p
400
- SET last_write_at = last_write_at - interval '10 minutes'
401
- FROM queen.queues q
402
- WHERE p.queue_id = q.id AND q.name = $1
403
- `, [queue])
404
-
405
- // Wait for A's lease to lapse. NO new push arrives.
396
+ // Wait for A's lease to lapse. NO new push arrives, so nothing but the
397
+ // lease expiry can make the message deliverable again.
406
398
  await sleep(3500)
407
399
 
408
400
  const b = await popRetry(client, queue, { group, tries: 10 })
409
401
  if (b.length !== 1) {
410
402
  return {
411
403
  success: false,
412
- message: 'Backlog stranded: empty polls during a held lease advanced the watermark past unconsumed data'
404
+ message: 'Backlog stranded: after empty polls during a held lease, the message never came back once the lease lapsed'
413
405
  }
414
406
  }
415
- return { success: true, message: 'Backlog survived lease churn + empty polls (watermark guard holds)' }
407
+ return { success: true, message: 'Backlog survived lease churn + empty polls' }
416
408
  }
417
409
 
418
410
  // ============================================================================
@@ -433,7 +425,7 @@ export async function pushOnlyQueueIsDiscoverable(client) {
433
425
  const list = Array.isArray(body) ? body : (body.queues || [])
434
426
  const entry = list.find(q => (q.name || q.queue) === queue)
435
427
  if (!entry) {
436
- return { success: false, message: 'Push-only queue missing from /api/v1/resources/queues (no queen.queues row?)' }
428
+ return { success: false, message: 'Push-only queue missing from /api/v1/resources/queues (never registered by the push?)' }
437
429
  }
438
430
  const entryNs = entry.namespace ?? entry.ns
439
431
  if (entryNs !== ns) {
@@ -455,19 +447,13 @@ export async function pushOnlyQueueIsDiscoverable(client) {
455
447
  if (!found) await sleep(200)
456
448
  }
457
449
 
458
- // Cleanup: queue identity is now the queen.queues id, and deleting the
459
- // queues row cascades partitions/watermarks/metadata/lag-metrics. Only
460
- // log_txns/log_dlq are FK-less by design → explicit purge via partitions.
461
- await dbPool.query(`
462
- WITH parts AS (
463
- SELECT lp.id FROM queen.log_partitions lp
464
- JOIN queen.queues q ON q.id = lp.queue_id
465
- WHERE q.name = $1
466
- ),
467
- d1 AS (DELETE FROM queen.log_txns WHERE partition_id IN (SELECT id FROM parts)),
468
- d2 AS (DELETE FROM queen.log_dlq WHERE partition_id IN (SELECT id FROM parts))
469
- SELECT 1`, [queue])
470
- await dbPool.query(`DELETE FROM queen.queues WHERE name = $1`, [queue])
450
+ // Best-effort cleanup through the public API. The lane's broker is thrown
451
+ // away after the run anyway, so a failed delete is not this test's verdict.
452
+ try {
453
+ await client.queue(queue).delete()
454
+ } catch (e) {
455
+ // Ignore
456
+ }
471
457
 
472
458
  if (!found) {
473
459
  return { success: false, message: 'Namespace discovery pop never found the push-only queue' }
@@ -1,13 +1,12 @@
1
1
  /**
2
2
  * Shared utilities for streaming tests.
3
3
  *
4
- * Tests in test-v2/stream/* are run live against a Queen instance and a
5
- * Postgres database. Each test:
4
+ * Tests in test-v2/stream/* are run live against a Queen broker. Each test:
6
5
  * - generates a unique queue/query name to avoid collisions across runs
7
6
  * - sets up source + sink queues
8
7
  * - runs a Stream pipeline for a bounded time
9
8
  * - asserts on the resulting sink-queue contents and/or runner metrics
10
- * - stops the stream and lets the global cleanup hook drop the rows
9
+ * - stops the stream (the lane's broker is thrown away after the run)
11
10
  *
12
11
  * Test-name uniqueness is achieved with the test function's name + a
13
12
  * monotonic counter so the same suite can be re-run without state bleed.
@@ -20,7 +19,7 @@ let _nameCounter = 0
20
19
 
21
20
  /**
22
21
  * Build a unique queue/query name scoped to a test. The prefix is always
23
- * "test-stream-" so the global cleanup query in run.js deletes it.
22
+ * "test-stream-", the suite's test-name convention.
24
23
  *
25
24
  * @param {string} testName - e.g. fn.name
26
25
  * @param {string} suffix - 'src' | 'sink' | 'query' | etc.
@@ -89,12 +88,12 @@ export async function pushSpread(client, queueName, items) {
89
88
  * the broker default of 'new' its cursor would be seeded at the sink's tail
90
89
  * and the emits under test would be invisible.
91
90
  *
92
- * NOTE on batch size: we use batch=1 so that each ack triggers
93
- * `acked_count >= batch_size` in queen.partition_consumers and the lease
94
- * is released between pops. With a larger batch the lease would stay
95
- * held until acked_count reached the full batch_size (often never on
96
- * intermittent emits), and subsequent pops would skip the partition as
97
- * "leased by another worker." Worth the round-trip overhead in tests.
91
+ * NOTE on batch size: we use batch=1 so that each ack reaches the end of
92
+ * the leased batch and the lease is released between pops. With a larger
93
+ * batch the lease would stay held until the acks reached the end of the
94
+ * full batch (often never on intermittent emits), and subsequent pops would
95
+ * skip the partition as "leased by another worker." Worth the round-trip
96
+ * overhead in tests.
98
97
  */
99
98
  export async function drainSink(client, queueName, { timeoutMs = 5000, group } = {}) {
100
99
  const cg = group || `drain-${Date.now()}-${Math.floor(Math.random() * 1e6)}`
@@ -110,7 +109,7 @@ export async function drainSink(client, queueName, { timeoutMs = 5000, group } =
110
109
  .pop()
111
110
  if (!popped || popped.length === 0) break
112
111
  for (const m of popped) out.push(m)
113
- // Pass the consumer group so the SP validates the lease against OUR
112
+ // Pass the consumer group so the broker validates the lease against OUR
114
113
  // group, not the default __QUEUE_MODE__. queen-mq uses `context.group`
115
114
  // (NOT `consumerGroup`) for the ack-context shape.
116
115
  await client.ack(popped, true, { group: cg })
@@ -187,11 +186,3 @@ export async function runStreamFor(streamFactory, runMs) {
187
186
  await handle.stop()
188
187
  return handle
189
188
  }
190
-
191
- /**
192
- * Helpful guard for tests that need a clean queen_streams.queries row for
193
- * a given query name. Drops it (state cascades) before the test runs.
194
- */
195
- export async function dropStreamQuery(dbPool, queryName) {
196
- await dbPool.query(`DELETE FROM queen_streams.queries WHERE name = $1`, [queryName])
197
- }
@@ -3,7 +3,7 @@
3
3
  * 'second' | 'minute' | 'hour' | 'day' | 'week'
4
4
  *
5
5
  * Boundaries are anchored to UTC. Day = midnight UTC; week = Monday 00:00
6
- * UTC. Tests here exercise live behaviour (in-PG state) for the second
6
+ * UTC. Tests here exercise live behaviour (broker-side state) for the second
7
7
  * granularity (fast enough to test in seconds rather than hours).
8
8
  */
9
9
 
@@ -0,0 +1,54 @@
1
+ /**
2
+ * .gate() against a live broker: the partial ack counts SOURCE MESSAGES.
3
+ *
4
+ * The gate's partial ack is an offset commit -- the broker advances
5
+ * `ack.count` messages from the head of the leased batch and keeps the lease
6
+ * -- and the runner used to count the envelopes the gate saw instead. With a
7
+ * .flatMap() in front, a denied message was acked together with its
8
+ * predecessor and never came back (2026-10-02, 2.0.0-beta.6: 4 of 6 sink
9
+ * items, message 2 lost). The unit tests pin the cycle body
10
+ * (streams-unit/gate.test.js); this one pins the broker's side of it.
11
+ */
12
+
13
+ import { Stream } from '../../client-v2/index.js'
14
+ import { STREAMS_URL, mkName, drainUntil, expect, summarise } from './_helpers.js'
15
+
16
+ export async function streamGateFlatMapPartialAck(client) {
17
+ const src = mkName('streamGateFlatMapPartialAck', 'src')
18
+ const sink = mkName('streamGateFlatMapPartialAck', 'sink')
19
+ const queryId = mkName('streamGateFlatMapPartialAck', 'q')
20
+
21
+ // A short lease: the denied tail comes back when it expires.
22
+ await client.queue(src).config({ leaseTime: 2 }).create()
23
+ await client.queue(sink).create()
24
+ await client.queue(src).partition('tenant-1').push([{ data: { n: 1 } }, { data: { n: 2 } }, { data: { n: 3 } }])
25
+
26
+ // Every message becomes two values; message 2 is denied the first time it
27
+ // is seen and allowed after its redelivery.
28
+ let deniedOnce = false
29
+ const handle = await Stream
30
+ .from(client.queue(src))
31
+ .flatMap(m => [m.data, m.data])
32
+ .gate((v) => {
33
+ if (v.n === 2 && !deniedOnce) {
34
+ deniedOnce = true
35
+ return false
36
+ }
37
+ return true
38
+ })
39
+ .to(client.queue(sink))
40
+ .run({ queryId, url: STREAMS_URL, batchSize: 10, maxPartitions: 1, reset: true, subscriptionMode: 'all' })
41
+
42
+ const drained = await drainUntil(client, sink, { until: out => out.length >= 6, timeoutMs: 15000 })
43
+ await handle.stop()
44
+
45
+ const count = (n) => drained.filter(m => m.data.n === n).length
46
+ const checks = [
47
+ expect(deniedOnce, '===', true, 'the gate denied message 2 once'),
48
+ expect(count(1), '===', 2, 'message 1 on the sink'),
49
+ expect(count(2), '===', 2, 'message 2 on the sink after its redelivery'),
50
+ expect(count(3), '===', 2, 'message 3 on the sink'),
51
+ expect(handle.metrics().errorsTotal, '===', 0, 'no errors')
52
+ ]
53
+ return summarise('streamGateFlatMapPartialAck', checks)
54
+ }
@@ -12,6 +12,7 @@
12
12
  * recovery.js — config_hash mismatch + reset + mid-stream resume
13
13
  * throughput.js — multi-partition / many-window throughput
14
14
  * combined.js — full pipelines + concurrent streams
15
+ * gate.js — .gate() partial acks against the broker
15
16
  */
16
17
 
17
18
  export * from './operators.js'
@@ -23,3 +24,4 @@ export * from './eventTime.js'
23
24
  export * from './recovery.js'
24
25
  export * from './throughput.js'
25
26
  export * from './combined.js'
27
+ export * from './gate.js'
@@ -149,12 +149,16 @@ export async function tumblingAggregateAllStats(client) {
149
149
  await client.queue(src).create()
150
150
  await client.queue(sink).create()
151
151
 
152
- // Push 5 values into the SAME window (all within 1 second), then idle
153
- // flush will close it. Sum = 50, count = 5, avg = 10, min = 2, max = 30.
154
- const values = [10, 5, 30, 2, 3]
155
- for (const v of values) {
156
- await client.queue(src).partition('p').push([{ data: { v } }])
157
- }
152
+ // One batched push = one segment = one timestamp, so the five values cannot
153
+ // straddle a 3-second window boundary. Pushing them "within 1 second" of each
154
+ // other does NOT put them in one window: the bucket is absolute-aligned
155
+ // (floor(ts / 3000) * 3000), so two pushes milliseconds apart still split when
156
+ // a boundary falls between them, and the assertion below reads only the FIRST
157
+ // emit. Sum = 50, count = 5, avg = 10, min = 2, max = 30.
158
+ await client.queue(src).partition('p').push([
159
+ { data: { v: 10 } }, { data: { v: 5 } }, { data: { v: 30 } },
160
+ { data: { v: 2 } }, { data: { v: 3 } },
161
+ ])
158
162
 
159
163
  const handle = await Stream
160
164
  .from(client.queue(src))
@@ -290,9 +294,9 @@ export async function tumblingForeachCtxHasWindowAndPartition(client) {
290
294
  const queryId = mkName('tumblingForeachCtxHasWindowAndPartition', 'q')
291
295
 
292
296
  await client.queue(src).create()
293
- for (let i = 0; i < 3; i++) {
294
- await client.queue(src).partition('myKey').push([{ data: { v: i } }])
295
- }
297
+ // One push, so all three get the same broker createdAt: three pushes can
298
+ // straddle a second and split across two 1 s windows (count 1, then 2).
299
+ await client.queue(src).partition('myKey').push([0, 1, 2].map(v => ({ data: { v } })))
296
300
 
297
301
  const captured = []
298
302
  const handle = await Stream
@@ -201,3 +201,64 @@ describe('Queen.ack batch (/api/v1/ack/batch)', () => {
201
201
  )
202
202
  })
203
203
  })
204
+
205
+ // The consumer group of an ack. /ack and /ack/batch judge an ack that names no
206
+ // group in queue mode, so a message popped by a group and acked without one
207
+ // came back `invalid or expired lease` (found 2026-10-02 against
208
+ // 2.0.0-beta.6). Every pop answers each message with its `consumerGroup`.
209
+ describe('Queen.ack — the group the message was popped under', () => {
210
+ const okFor = (body) => (body.acknowledgments || [body]).map((a, i) => ackResultItem(i, a.transactionId, true))
211
+ const popped = (n, group) => ({ transactionId: `tx-${n}`, partitionId: 'p-1', leaseId: 'lease-1', consumerGroup: group })
212
+
213
+ it('single: takes consumerGroup from the message when the caller names none', async () => {
214
+ await withAckServer(okFor, async (client, requests) => {
215
+ await client.ack(popped(1, 'workers'), true)
216
+ assert.equal(requests[0].body.consumerGroup, 'workers')
217
+ })
218
+ })
219
+
220
+ it('single: a queue-mode or hand-built message still sends consumerGroup null', async () => {
221
+ await withAckServer(okFor, async (client, requests) => {
222
+ await client.ack(popped(1, '__QUEUE_MODE__'), true)
223
+ await client.ack(MSG, true)
224
+ assert.equal(requests[0].body.consumerGroup, null)
225
+ assert.equal(requests[1].body.consumerGroup, null)
226
+ })
227
+ })
228
+
229
+ it('single: an explicit { group } wins', async () => {
230
+ await withAckServer(okFor, async (client, requests) => {
231
+ await client.ack(popped(1, 'workers'), false, { group: 'auditors' })
232
+ assert.equal(requests[0].body.consumerGroup, 'auditors')
233
+ })
234
+ })
235
+
236
+ it('batch: the group the messages share', async () => {
237
+ await withAckServer(okFor, async (client, requests) => {
238
+ const result = await client.ack([popped(1, 'workers'), popped(2, 'workers')], true)
239
+ assert.equal(result.success, true)
240
+ assert.equal(requests[0].path, '/api/v1/ack/batch')
241
+ assert.equal(requests[0].body.consumerGroup, 'workers')
242
+ })
243
+ })
244
+
245
+ it('batch: queue-mode messages still send consumerGroup null', async () => {
246
+ await withAckServer(okFor, async (client, requests) => {
247
+ await client.ack([popped(1, '__QUEUE_MODE__'), popped(2, '__QUEUE_MODE__')], true)
248
+ assert.equal(requests[0].body.consumerGroup, null)
249
+ })
250
+ })
251
+
252
+ it('batch: messages from two groups are refused before anything is sent', async () => {
253
+ await withAckServer(okFor, async (client, requests) => {
254
+ await assert.rejects(
255
+ () => client.ack([popped(1, 'a'), popped(2, 'b')], true),
256
+ /different consumer groups in one call \(a, b\)/
257
+ )
258
+ assert.equal(requests.length, 0)
259
+ // ... unless the caller names the group, which is then theirs to get right.
260
+ await client.ack([popped(1, 'a'), popped(2, 'b')], true, { group: 'a' })
261
+ assert.equal(requests[0].body.consumerGroup, 'a')
262
+ })
263
+ })
264
+ })