queen-mq 2.0.3 → 2.0.4

package/README.md

@@ -1079,3 +1079,25 @@ const messages: Message<OrderData>[] = await queen.queue('orders').pop()
 ## License
 
 Apache 2.0 - See [LICENSE.md](../LICENSE.md)
+
+### Optional consumer supervision
+
+```js
+await queen.queue('orders').group('billing')
+  .supervision({ group: 'billing-production' })
+  .concurrency(4).each().consume(async message => { /* process message */ })
+```
+
+Supervision defaults to off; `.supervision(false)` disables it. The group names
+an application/deployment in the dashboard, independently of the consumer group.
+Each consume invocation publishes its own instance into the broker's
+`queen-supervisor` KV namespace every 10 seconds (30-second heartbeat timeout,
+60-second TTL), plus a final stopped observation. The credential needs KV write
+access. Publication is serialized and best effort with a two-second deadline.
+
+The Supervisors page supporting `queen.consumer.status/v1` shows live async
+consumer loops, busy handlers, successful/failed handler calls and progress times.
+A batch is one handler call; completion does not imply ACK success. No payloads or
+error text are published. Event-loop starvation can stop heartbeats. Reporting
+does not restart processes or tasks, change ACK/lease policies, or enable remote
+control. With reporting off there is no additional timer or network traffic.

package/client-v2/builders/QueueBuilder.js

@@ -42,6 +42,7 @@ export class QueueBuilder {
   #config = {}
 
   // Consume options
+  #supervision = false
   #concurrency = CONSUME_DEFAULTS.concurrency
   // batch / maxPartitions hold the USER's value, and null means the setter was
   // never called -- which is the dimension pop autopilot gets to choose. The
@@ -238,6 +239,12 @@ export class QueueBuilder {
     return this
   }
 
+  /** Opt-in consumer observations in the broker dashboard; false by default. */
+  supervision(config = false) {
+    this.#supervision = config
+    return this
+  }
+
   concurrency(count) {
     this.#concurrency = Math.max(1, count)
     return this
@@ -410,6 +417,7 @@ export class QueueBuilder {
       // has to survive all the way to #buildParams: it is the ONLY record that
       // the user said nothing about that dimension.
       autopilot: this.#autopilotEnabled(),
+      supervision: options.supervision ?? this.#supervision,
       signal: options.signal
     }
 

package/client-v2/consumer/ConsumerManager.js

@@ -3,6 +3,7 @@
  */
 
 import * as logger from '../utils/logger.js'
+import { Supervision } from './Supervision.js'
 import { checkConflationResponse, CONFLATION_UNSUPPORTED } from '../utils/conflation.js'
 import { popSizing, parseAutopilotDecision, emptyPollDelayMillis } from '../utils/autopilot.js'
 import { CONSUME_DEFAULTS } from '../utils/defaults.js'
@@ -92,9 +93,13 @@ export class ConsumerManager {
     // Generate affinity key for consistent routing to same backend
     const affinityKey = this.#getAffinityKey(queue, partition, namespace, task, group)
 
+    const supervision = options.supervision ? new Supervision(this.#httpClient, options.supervision, options) : null
+    if (supervision) handler = supervision.wrap(handler)
+
     // Start workers
     const workers = []
     for (let i = 0; i < concurrency; i++) {
+      if (supervision) supervision.running++
       workers.push(this.#worker(i, handler, path, baseParams, {
         batch,
         limit,
@@ -113,9 +118,12 @@ export class ConsumerManager {
         // the pop target to key the once-per-(queue,group) conflict warning.
         conflation,
         conflationCtx: { queue, namespace, task, group }
+      }).finally(async () => {
+        if (supervision && --supervision.running === 0) await supervision.stop()
       }))
     }
 
+    supervision?.start()
     logger.log('ConsumerManager.start', { status: 'workers-started', count: concurrency })
 
     // Wait for all workers to complete

package/client-v2/consumer/Supervision.js

@@ -0,0 +1,78 @@
+import { randomBytes } from 'node:crypto'
+import { hostname } from 'node:os'
+import * as logger from '../utils/logger.js'
+
+// One opt-in reporter per consume invocation. Handler calls are not ACKs.
+export class Supervision {
+  constructor(http, config, options) {
+    if (!config || typeof config !== 'object' || typeof config.group !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,254}$/.test(config.group) || config.group === 'coordination' || /[\r\n]/.test(config.group)) throw new Error('supervision.group must be a valid application/deployment name')
+    if (!Number.isInteger(options.concurrency) || options.concurrency < 1 || options.concurrency > 4096) throw new Error('supervision requires concurrency between 1 and 4096')
+    this.http = http
+    this.options = options
+    this.group = config.group
+    this.id = randomBytes(16).toString('hex')
+    this.started = Date.now()
+    this.monotonic = performance.now()
+    this.running = this.completed = this.failed = this.sequence = 0
+    this.last = this.pending = this.timer = null
+    this.active = new Map()
+    this.state = 'running'
+    this.warned = false
+  }
+  wrap(handler) {
+    return async (...args) => {
+      const id = this.sequence++
+      this.active.set(id, performance.now())
+      try { const result = await handler(...args); this.completed++; return result }
+      catch (error) { this.failed++; throw error }
+      finally { this.active.delete(id); this.last = Math.floor(Date.now() / 1000) }
+    }
+  }
+  start() {
+    this.publish()
+    this.timer = setInterval(() => this.publish(), 10_000)
+    this.timer.unref()
+  }
+  document() {
+    const now = performance.now()
+    return {
+      schema: 'queen.consumer.status/v1', instance_id: this.id, engine: 'js', execution_model: 'async-tasks',
+      hostname: hostname(), pid: process.pid, state: this.state, updated_at_epoch: Math.floor(Date.now() / 1000),
+      started_at_epoch: Math.floor(this.started / 1000), uptime_seconds: Math.floor((now - this.monotonic) / 1000),
+      configuration: { heartbeat_timeout: 30 },
+      pool_status: [{ name: 'consumer', queue: this.options.queue || null, namespace: this.options.namespace || null,
+        task: this.options.task || null, consumer_group: this.options.group || '__QUEUE_MODE__',
+        desired: this.options.concurrency, running: this.running, busy: this.active.size,
+        completed: this.completed, failed: this.failed, last_completed_at_epoch: this.last,
+        oldest_inflight_seconds: this.active.size ? Math.floor((now - Math.min(...this.active.values())) / 1000) : null }],
+    }
+  }
+  publish() {
+    if (this.pending) return this.pending
+    this.pending = this.send().finally(() => { this.pending = null })
+    return this.pending
+  }
+  async send() {
+    try {
+      const bytes = Buffer.from(JSON.stringify(this.document()))
+      if (bytes.length > 45_000) throw new Error('Consumer status exceeds one chunk')
+      const write = randomBytes(16).toString('hex'), slot = `${this.group}/${this.id}`
+      const ops = [
+        { op: 'put', ns: 'queen-supervisor', key: `${slot}/head`, value: { format: 'queen.supervisor.remote-status/v1', write, chunks: 1, bytes: bytes.length }, ttlSeconds: 60 },
+        { op: 'put', ns: 'queen-supervisor', key: `${slot}/chunk/0000`, value: { write, index: 0, data: bytes.toString('base64') }, ttlSeconds: 60 },
+      ]
+      const result = await this.http.post('/api/v1/kv', { operations: ops }, 2000, null, null, AbortSignal.timeout(2000))
+      if (!Array.isArray(result?.results) || result.results.length !== 2 || result.results.some(r => r.applied !== true)) throw new Error('Consumer status publication was not applied')
+      this.warned = false
+    } catch {
+      if (!this.warned) logger.warn('Consumer.supervision', 'Status publication failed; consumption continues')
+      this.warned = true
+    }
+  }
+  async stop() {
+    clearInterval(this.timer)
+    if (this.pending) await this.pending
+    this.state = 'stopped'
+    await this.publish()
+  }
+}

package/client-v2/http/HttpClient.js

@@ -611,8 +611,8 @@ export class HttpClient {
     return this.#requestWithFailover('GET', path, null, requestTimeoutMillis, affinityKey, retryKind, signal)
   }
 
-  async post(path, body = null, requestTimeoutMillis = null, affinityKey = null, retryKind = null) {
-    return this.#requestWithFailover('POST', path, body, requestTimeoutMillis, affinityKey, retryKind)
+  async post(path, body = null, requestTimeoutMillis = null, affinityKey = null, retryKind = null, signal = null) {
+    return this.#requestWithFailover('POST', path, body, requestTimeoutMillis, affinityKey, retryKind, signal)
   }
 
   async put(path, body = null, requestTimeoutMillis = null, affinityKey = null, retryKind = null) {

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "queen-mq",
-  "version": "2.0.3",
+  "version": "2.0.4",
   "type": "module",
   "description": "Partitioned message queue on a replicated broker log — broker client + fluent streaming SDK (windows, joins, gates) in one package",
   "main": "client-v2/index.js",
   "scripts": {
     "test": "npm run test:unit && node test-v2/run.js human",
-    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/consumer-unit/nackScope.test.js test-v2/consumer-unit/stopOnAbort.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js test-v2/pop-unit/popDefaults.test.js test-v2/admin-unit/removedRoutes.test.js",
+    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/consumer-unit/nackScope.test.js test-v2/consumer-unit/stopOnAbort.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js test-v2/pop-unit/popDefaults.test.js test-v2/admin-unit/removedRoutes.test.js test-v2/consumer-unit/supervision.test.js",
     "test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
     "test:integration": "node test-v2/run.js human",
     "test:streams": "node test-v2/run.js stream",

package/test-v2/consumer-unit/supervision.test.js

@@ -0,0 +1,97 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { ConsumerManager } from '../../client-v2/consumer/ConsumerManager.js'
+import { Supervision } from '../../client-v2/consumer/Supervision.js'
+
+const options = { queue: 'orders', concurrency: 2, timeoutMillis: 30000, limit: 1, batch: 1, each: true, autoAck: true, wait: false }
+const decode = body => {
+  assert.equal(body.operations.length, 2)
+  const [head, chunk] = body.operations
+  assert.equal(head.ns, 'queen-supervisor')
+  assert.equal(head.ttlSeconds, 60)
+  assert.equal(chunk.ttlSeconds, 60)
+  assert.equal(head.value.write, chunk.value.write)
+  const bytes = Buffer.from(chunk.value.data, 'base64')
+  assert.equal(bytes.length, head.value.bytes)
+  return JSON.parse(bytes)
+}
+function rig(fail = false) {
+  const docs = [], acks = []
+  const http = { get: async () => ({ messages: [{ transactionId: 't', partitionId: 'p', data: { secret: 42 } }] }),
+    post: async (path, body) => { assert.equal(path, '/api/v1/kv'); docs.push(decode(body)); if (fail) throw new Error('offline'); return { results: [{ applied: true }, { applied: true }] } } }
+  const queen = { ack: async (_, success) => { acks.push(success); return { success: true } } }
+  return { docs, acks, http, manager: new ConsumerManager(http, queen) }
+}
+test('default and explicit off create no publications and preserve acknowledgement', async () => {
+  for (const supervision of [undefined, false]) {
+    const r = rig(); await r.manager.start(async () => {}, { ...options, supervision })
+    assert.equal(r.docs.length, 0); assert.deepEqual(r.acks, [true, true])
+  }
+})
+test('enabled consumers count actual exits, handler failures and final state without changing ACKs', async () => {
+  const r = rig()
+  let n = 0
+  await r.manager.start(async () => { if (++n === 1) throw new Error('private error') }, { ...options, supervision: { group: 'billing-production' } })
+  const last = r.docs.at(-1)
+  assert.equal(last.state, 'stopped'); assert.equal(last.pool_status[0].running, 0)
+  assert.equal(last.pool_status[0].busy, 0); assert.equal(last.pool_status[0].completed, 1)
+  assert.equal(last.pool_status[0].failed, 1); assert.deepEqual(r.acks.sort(), [false, true])
+  assert.equal(JSON.stringify(r.docs).includes('private error'), false)
+  assert.equal(JSON.stringify(r.docs).includes('secret'), false)
+})
+test('publication errors do not affect consumption; instances are unique', async () => {
+  const r = rig(true)
+  await r.manager.start(async () => {}, { ...options, supervision: { group: 'billing' } })
+  const first = r.docs[0].instance_id
+  await r.manager.start(async () => {}, { ...options, supervision: { group: 'billing' } })
+  assert.notEqual(first, r.docs.at(-1).instance_id); assert.equal(r.acks.length, 4)
+})
+test('busy handlers remain observable and publication is serialized', async () => {
+  const r = rig(); const reporter = new Supervision(r.http, { group: 'billing' }, options)
+  let release
+  const work = reporter.wrap(() => new Promise(resolve => { release = resolve }))()
+  reporter.running = 1
+  await Promise.all([reporter.publish(), reporter.publish()])
+  assert.equal(r.docs.length, 1); assert.equal(r.docs[0].pool_status[0].busy, 1)
+  assert.equal(r.docs[0].pool_status[0].completed, 0)
+  release(); await work
+  assert.equal(reporter.document().pool_status[0].completed, 1)
+  assert.equal(reporter.document().pool_status[0].oldest_inflight_seconds, null)
+})
+test('invalid opt-in groups fail before polling', async () => {
+  for (const group of ['', 'coordination', 'a/b', 'billing\n', undefined]) {
+    const r = rig(); await assert.rejects(r.manager.start(async () => {}, { ...options, supervision: { group } }), /supervision.group/)
+    assert.equal(r.docs.length, 0)
+  }
+})
+
+test('real HTTP publishing preserves authentication and has a total deadline', async () => {
+  const { createServer } = await import('node:http')
+  const { Queen } = await import('../../client-v2/index.js')
+  let kvCalls = 0, polls = 0
+  const docs = []
+  const server = createServer((req, res) => {
+    if (req.url === '/api/v1/kv') {
+      kvCalls++
+      assert.equal(req.headers.authorization, 'Bearer test-token')
+      let bytes = ''
+      req.on('data', chunk => { bytes += chunk })
+      req.on('end', () => { docs.push(decode(JSON.parse(bytes))) })
+      return // Deliberately never answer. Consumption and shutdown stay bounded.
+    }
+    polls++
+    res.setHeader('Content-Type', 'application/json')
+    res.end(JSON.stringify({ messages: [{ transactionId: 't', partitionId: 'p', data: {} }] }))
+  })
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+  const queen = new Queen({ url: `http://127.0.0.1:${server.address().port}`, bearerToken: 'test-token', handleSignals: false })
+  const start = performance.now()
+  try {
+    await queen.queue('orders').supervision({ group: 'wire' }).wait(false).autoAck(false).limit(1).consume(async () => {})
+    assert.equal(polls, 1); assert.equal(kvCalls, 2)
+    assert.equal(docs.at(-1).state, 'stopped')
+    assert.ok(performance.now() - start < 6000, 'two publications are bounded by two seconds each')
+  } finally {
+    await queen.close(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve))
+  }
+})