queen-mq 0.12.1 → 0.12.3

package/client-v2/Queen.js

@@ -34,10 +34,12 @@ export class Queen {
     // Create buffer manager
     this.#bufferManager = new BufferManager(this.#httpClient)
 
-    // Setup graceful shutdown
-    this.#setupGracefulShutdown()
+    // Setup graceful shutdown (opt-out via handleSignals: false)
+    if (this.#config.handleSignals) {
+      this.#setupGracefulShutdown()
+    }
     
-    logger.log('Queen.constructor', { status: 'initialized', urls: this.#config.urls.length })
+    logger.log('Queen.constructor', { status: 'initialized', urls: this.#config.urls.length, handleSignals: this.#config.handleSignals })
   }
 
   #normalizeConfig(config) {
@@ -77,7 +79,7 @@ export class Queen {
   }
 
   #createHttpClient() {
-    const { urls, timeoutMillis, retryAttempts, retryDelayMillis, loadBalancingStrategy, affinityHashRing, healthRetryAfterMillis, enableFailover, bearerToken } = this.#config
+    const { urls, timeoutMillis, retryAttempts, retryDelayMillis, loadBalancingStrategy, affinityHashRing, healthRetryAfterMillis, enableFailover, bearerToken, headers } = this.#config
 
     if (urls.length === 1) {
       // Single server
@@ -86,7 +88,8 @@ export class Queen {
         timeoutMillis,
         retryAttempts,
         retryDelayMillis,
-        bearerToken
+        bearerToken,
+        headers
       })
     }
 
@@ -101,7 +104,8 @@ export class Queen {
       retryAttempts,
       retryDelayMillis,
       enableFailover,
-      bearerToken
+      bearerToken,
+      headers
     })
   }
 
@@ -452,6 +456,28 @@ export class Queen {
   // Graceful Shutdown
   // ===========================
 
+  /**
+   * Register SIGINT/SIGTERM handlers so the client flushes buffers before exit.
+   * Called automatically unless handleSignals is set to false.
+   */
+  enableGracefulShutdown() {
+    if (this.#shutdownHandlers.length > 0) return this
+    this.#setupGracefulShutdown()
+    return this
+  }
+
+  /**
+   * Remove previously registered SIGINT/SIGTERM handlers.
+   * Use this when Queen is embedded in a larger application that manages its own lifecycle.
+   */
+  disableGracefulShutdown() {
+    for (const cleanup of this.#shutdownHandlers) {
+      cleanup()
+    }
+    this.#shutdownHandlers = []
+    return this
+  }
+
   async close() {
     logger.log('Queen.close', 'Starting shutdown')
     console.log('Closing Queen client...')

package/client-v2/http/HttpClient.js

@@ -12,6 +12,7 @@ export class HttpClient {
   #retryDelayMillis
   #enableFailover
   #bearerToken
+  #headers
 
   constructor(options = {}) {
     const {
@@ -21,7 +22,8 @@ export class HttpClient {
       retryAttempts = 3,
       retryDelayMillis = 1000,
       enableFailover = true,
-      bearerToken = null
+      bearerToken = null,
+      headers = {}
     } = options
 
     this.#baseUrl = baseUrl
@@ -31,6 +33,7 @@ export class HttpClient {
     this.#retryDelayMillis = retryDelayMillis
     this.#enableFailover = enableFailover
     this.#bearerToken = bearerToken
+    this.#headers = headers || {}
     
     logger.log('HttpClient.constructor', { 
       hasLoadBalancer: !!loadBalancer, 
@@ -38,7 +41,8 @@ export class HttpClient {
       timeoutMillis, 
       retryAttempts,
       enableFailover,
-      hasAuth: !!bearerToken
+      hasAuth: !!bearerToken,
+      customHeaders: Object.keys(this.#headers).length
     })
   }
 
@@ -54,6 +58,7 @@ export class HttpClient {
       if (this.#bearerToken) {
         headers['Authorization'] = `Bearer ${this.#bearerToken}`
       }
+      Object.assign(headers, this.#headers)
 
       const options = {
         method,

package/client-v2/utils/defaults.js

@@ -14,7 +14,9 @@ export const CLIENT_DEFAULTS = {
   affinityHashRing: 128,               // Number of virtual nodes per server for affinity strategy
   enableFailover: true,                // Auto-failover to other servers
   healthRetryAfterMillis: 5000,        // Retry unhealthy backends after 5 seconds
-  bearerToken: null                    // Bearer token for proxy authentication
+  bearerToken: null,                   // Bearer token for proxy authentication
+  headers: {},                         // Custom headers to include in every request
+  handleSignals: true                  // Register SIGINT/SIGTERM handlers (disable when used as a library)
 }
 
 export const QUEUE_DEFAULTS = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "queen-mq",
-  "version": "0.12.1",
+  "version": "0.12.3",
   "type": "module",
   "description": "High-performance C++ message queue backed by PostgreSQL",
   "main": "client-v2/index.js",

package/test-v2/bootstrap.js

@@ -0,0 +1,316 @@
+/**
+ * Consumer Group Bootstrap Tests
+ * 
+ * Tests the partition_consumers bootstrap optimization:
+ * When a new consumer group registers (via subscription mode), all existing
+ * partition_consumers rows should be pre-seeded in bulk, avoiding slow
+ * one-at-a-time discovery.
+ * 
+ * Test scenarios:
+ * 1. testCgBootstrapNew: CG with subscriptionMode('new')
+ * 2. testCgBootstrapTimestamp: CG with subscriptionFrom(timestamp)
+ * 
+ * Both push 5000 partitions, register a CG (triggering bootstrap), then
+ * consume new messages and verify all partitions were discovered fast.
+ * 
+ * Run: node run.js testCgBootstrapNew
+ *      node run.js testCgBootstrapTimestamp
+ */
+
+const PARTITION_COUNT = 5000
+const QUEUE_NAME_NEW = 'test-queue-v2-bootstrap-new'
+const QUEUE_NAME_TS = 'test-queue-v2-bootstrap-ts'
+
+async function pushToPartitions(client, queueName, partitionCount, prefix = '') {
+    const PARALLEL_BATCH = 50
+    let pushed = 0
+    for (let i = 0; i < partitionCount; i += PARALLEL_BATCH) {
+        const batch = []
+        for (let j = i; j < Math.min(i + PARALLEL_BATCH, partitionCount); j++) {
+            batch.push(
+                client
+                    .queue(queueName)
+                    .partition(`p-${j}`)
+                    .push([{ data: { id: j, prefix } }])
+            )
+        }
+        await Promise.all(batch)
+        pushed += batch.length
+    }
+    return pushed
+}
+
+/**
+ * Test bootstrap with subscriptionMode('new')
+ * - Push historical messages to 5000 partitions
+ * - Register CG with mode 'new' (triggers bootstrap)
+ * - Push new messages to all partitions
+ * - Consume with CG and verify all new messages are received fast
+ */
+export async function testCgBootstrapNew(client) {
+    // Cleanup
+    try {
+        await client.deleteConsumerGroup('bootstrap-cg-new', true)
+    } catch (e) {
+        // Ignore
+    }
+
+    // 1. Create queue
+    const queue = await client.queue(QUEUE_NAME_NEW).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // 2. Push 1 message to each of 5000 partitions (historical messages)
+    console.log(`  Pushing to ${PARTITION_COUNT} partitions...`)
+    const pushStart = Date.now()
+    await pushToPartitions(client, QUEUE_NAME_NEW, PARTITION_COUNT, 'historical')
+    const pushTime = Date.now() - pushStart
+    console.log(`  Push complete: ${PARTITION_COUNT} partitions in ${pushTime}ms`)
+
+    // 3. Consume all messages in queue mode (no CG) - baseline
+    console.log(`  Consuming in queue mode (no CG)...`)
+    const queueModeStart = Date.now()
+    let queueModeCount = 0
+
+    await client
+        .queue(QUEUE_NAME_NEW)
+        .concurrency(10)
+        .batch(1)
+        .wait(false)
+        .idleMillis(5000)
+        .consume(async msgs => {
+            queueModeCount += msgs.length
+        })
+
+    const queueModeTime = Date.now() - queueModeStart
+    console.log(`  Queue mode: consumed ${queueModeCount} messages in ${queueModeTime}ms`)
+
+    if (queueModeCount !== PARTITION_COUNT) {
+        return {
+            success: false,
+            message: `Queue mode consumed ${queueModeCount}/${PARTITION_COUNT} messages in ${queueModeTime}ms`
+        }
+    }
+
+    // 4. Register a new CG with 'new' mode
+    //    The first pop triggers metadata registration + bootstrap (seeds all partition_consumers)
+    console.log(`  Registering CG 'bootstrap-cg-new' with mode 'new'...`)
+    const registerStart = Date.now()
+
+    const firstPop = await client
+        .queue(QUEUE_NAME_NEW)
+        .group('bootstrap-cg-new')
+        .subscriptionMode('new')
+        .batch(1)
+        .wait(false)
+        .pop()
+
+    const registerTime = Date.now() - registerStart
+    console.log(`  CG registered + bootstrap in ${registerTime}ms (first pop returned ${firstPop.length} msgs)`)
+
+    // First pop should return 0 messages (mode 'new' skips historical)
+    if (firstPop.length !== 0) {
+        return {
+            success: false,
+            message: `CG with mode 'new' should skip historical messages, got ${firstPop.length}`
+        }
+    }
+
+    // 5. Push NEW messages to all 5000 partitions
+    console.log(`  Pushing new messages to ${PARTITION_COUNT} partitions...`)
+    const pushNewStart = Date.now()
+    await pushToPartitions(client, QUEUE_NAME_NEW, PARTITION_COUNT, 'new')
+    const pushNewTime = Date.now() - pushNewStart
+    console.log(`  Push new complete: ${PARTITION_COUNT} partitions in ${pushNewTime}ms`)
+
+    // 6. Consume all new messages with the CG
+    //    If bootstrap worked, all partitions are already known -> fast consumption
+    //    If bootstrap didn't work, it would discover one-at-a-time -> very slow
+    console.log(`  Consuming with CG 'bootstrap-cg-new'...`)
+    const cgConsumeStart = Date.now()
+    let cgConsumeCount = 0
+
+    await client
+        .queue(QUEUE_NAME_NEW)
+        .group('bootstrap-cg-new')
+        .subscriptionMode('new')
+        .concurrency(10)
+        .batch(1)
+        .wait(false)
+        .idleMillis(5000)
+        .consume(async msgs => {
+            cgConsumeCount += msgs.length
+        })
+
+    const cgConsumeTime = Date.now() - cgConsumeStart
+    console.log(`  CG consume: ${cgConsumeCount} messages in ${cgConsumeTime}ms`)
+
+    // 7. Drain new messages in queue mode too (cleanup)
+    console.log(`  Draining new messages in queue mode...`)
+    let queueDrainCount = 0
+    await client
+        .queue(QUEUE_NAME_NEW)
+        .concurrency(10)
+        .batch(1)
+        .wait(false)
+        .idleMillis(5000)
+        .consume(async msgs => {
+            queueDrainCount += msgs.length
+        })
+    console.log(`  Queue mode drain: ${queueDrainCount} messages`)
+
+    // 8. Results
+    const allConsumed = cgConsumeCount === PARTITION_COUNT
+    const bootstrapFast = registerTime < 10000
+
+    console.log(`  ---- Results (subscriptionMode='new') ----`)
+    console.log(`  Partitions:          ${PARTITION_COUNT}`)
+    console.log(`  Push (historical):   ${pushTime}ms`)
+    console.log(`  Queue mode consume:  ${queueModeTime}ms (${queueModeCount} msgs)`)
+    console.log(`  CG registration:     ${registerTime}ms (includes bootstrap)`)
+    console.log(`  Push (new):          ${pushNewTime}ms`)
+    console.log(`  CG consume:          ${cgConsumeTime}ms (${cgConsumeCount} msgs)`)
+    console.log(`  Bootstrap fast:      ${bootstrapFast ? 'YES' : 'NO'} (${registerTime}ms < 10000ms)`)
+
+    if (!allConsumed) {
+        return {
+            success: false,
+            message: `CG consumed ${cgConsumeCount}/${PARTITION_COUNT} in ${cgConsumeTime}ms (bootstrap: ${registerTime}ms)`
+        }
+    }
+
+    return {
+        success: true,
+        message: `[new] ${PARTITION_COUNT} partitions: push=${pushTime}ms, queue_mode=${queueModeTime}ms, bootstrap=${registerTime}ms, cg_consume=${cgConsumeTime}ms`
+    }
+}
+
+/**
+ * Test bootstrap with subscriptionFrom(timestamp)
+ * - Push historical messages to 5000 partitions
+ * - Record a cutoff timestamp
+ * - Register CG with subscriptionFrom(cutoff) (triggers bootstrap via 'timestamp' path)
+ * - Push new messages to all partitions
+ * - Consume with CG and verify all new messages are received fast
+ */
+export async function testCgBootstrapTimestamp(client) {
+    // Cleanup
+    try {
+        await client.deleteConsumerGroup('bootstrap-cg-ts', true)
+    } catch (e) {
+        // Ignore
+    }
+
+    // 1. Create queue
+    const queue = await client.queue(QUEUE_NAME_TS).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // 2. Push 1 message to each of 5000 partitions (historical messages)
+    console.log(`  Pushing to ${PARTITION_COUNT} partitions...`)
+    const pushStart = Date.now()
+    await pushToPartitions(client, QUEUE_NAME_TS, PARTITION_COUNT, 'historical')
+    const pushTime = Date.now() - pushStart
+    console.log(`  Push complete: ${PARTITION_COUNT} partitions in ${pushTime}ms`)
+
+    // 3. Record cutoff timestamp (after historical, before new messages)
+    //    Wait before capturing the cutoff to handle clock skew between the
+    //    client (Date.now) and PostgreSQL (NOW()). Without this, the last
+    //    push batch may have a PG created_at AFTER the client-side cutoff.
+    await new Promise(resolve => setTimeout(resolve, 2000))
+    const cutoffTimestamp = new Date().toISOString()
+    console.log(`  Cutoff timestamp: ${cutoffTimestamp}`)
+
+    // 4. Register a new CG with subscriptionFrom(timestamp)
+    //    The first pop triggers metadata registration + bootstrap (seeds all partition_consumers)
+    console.log(`  Registering CG 'bootstrap-cg-ts' with subscriptionFrom('${cutoffTimestamp}')...`)
+    const registerStart = Date.now()
+
+    const firstPop = await client
+        .queue(QUEUE_NAME_TS)
+        .group('bootstrap-cg-ts')
+        .subscriptionFrom(cutoffTimestamp)
+        .batch(1)
+        .wait(false)
+        .pop()
+
+    const registerTime = Date.now() - registerStart
+    console.log(`  CG registered + bootstrap in ${registerTime}ms (first pop returned ${firstPop.length} msgs)`)
+
+    // First pop should return 0 messages (all messages are before the cutoff)
+    if (firstPop.length !== 0) {
+        return {
+            success: false,
+            message: `CG with subscriptionFrom should skip historical messages, got ${firstPop.length}`
+        }
+    }
+
+    // 5. Push NEW messages to all 5000 partitions (after cutoff)
+    console.log(`  Pushing new messages to ${PARTITION_COUNT} partitions...`)
+    const pushNewStart = Date.now()
+    await pushToPartitions(client, QUEUE_NAME_TS, PARTITION_COUNT, 'new')
+    const pushNewTime = Date.now() - pushNewStart
+    console.log(`  Push new complete: ${PARTITION_COUNT} partitions in ${pushNewTime}ms`)
+
+    // 6. Consume all new messages with the CG
+    //    If bootstrap worked, all partitions are already known -> fast consumption
+    //    If bootstrap didn't work, it would discover one-at-a-time -> very slow
+    console.log(`  Consuming with CG 'bootstrap-cg-ts'...`)
+    const cgConsumeStart = Date.now()
+    let cgConsumeCount = 0
+
+    await client
+        .queue(QUEUE_NAME_TS)
+        .group('bootstrap-cg-ts')
+        .subscriptionFrom(cutoffTimestamp)
+        .concurrency(10)
+        .batch(1)
+        .wait(false)
+        .idleMillis(5000)
+        .consume(async msgs => {
+            cgConsumeCount += msgs.length
+        })
+
+    const cgConsumeTime = Date.now() - cgConsumeStart
+    console.log(`  CG consume: ${cgConsumeCount} messages in ${cgConsumeTime}ms`)
+
+    // 7. Drain new messages in queue mode too (cleanup)
+    console.log(`  Draining new messages in queue mode...`)
+    let queueDrainCount = 0
+    await client
+        .queue(QUEUE_NAME_TS)
+        .concurrency(10)
+        .batch(1)
+        .wait(false)
+        .idleMillis(5000)
+        .consume(async msgs => {
+            queueDrainCount += msgs.length
+        })
+    console.log(`  Queue mode drain: ${queueDrainCount} messages`)
+
+    // 8. Results
+    const allConsumed = cgConsumeCount === PARTITION_COUNT
+    const bootstrapFast = registerTime < 10000
+
+    console.log(`  ---- Results (subscriptionFrom=timestamp) ----`)
+    console.log(`  Partitions:          ${PARTITION_COUNT}`)
+    console.log(`  Push (historical):   ${pushTime}ms`)
+    console.log(`  CG registration:     ${registerTime}ms (includes bootstrap)`)
+    console.log(`  Push (new):          ${pushNewTime}ms`)
+    console.log(`  CG consume:          ${cgConsumeTime}ms (${cgConsumeCount} msgs)`)
+    console.log(`  Bootstrap fast:      ${bootstrapFast ? 'YES' : 'NO'} (${registerTime}ms < 10000ms)`)
+
+    if (!allConsumed) {
+        return {
+            success: false,
+            message: `CG consumed ${cgConsumeCount}/${PARTITION_COUNT} in ${cgConsumeTime}ms (bootstrap: ${registerTime}ms)`
+        }
+    }
+
+    return {
+        success: true,
+        message: `[timestamp] ${PARTITION_COUNT} partitions: push=${pushTime}ms, bootstrap=${registerTime}ms, cg_consume=${cgConsumeTime}ms`
+    }
+}

package/test-v2/run.js

@@ -11,13 +11,7 @@ import * as transactionTests from './transaction.js'
 import * as subscriptionTests from './subscription.js'
 import * as maintenanceTests from './maintenance.js'
 import * as retentionTests from './retention.js'
-import * as errorHandlingTests from './ai_error_handling.js'
-import * as leaseRenewalTests from './ai_lease_renewal.js'
-import * as resourcesTests from './ai_resources.js'
-import * as bufferingTests from './ai_buffering.js'
-import * as priorityTests from './ai_priority.js'
-import * as ttlRetentionTests from './ai_ttl_retention.js'
-import * as mixedScenariosTests from './ai_mixed_scenarios.js'
+import * as bootstrapTests from './bootstrap.js'
 import { LoadBalancer } from '../client-v2/http/LoadBalancer.js';
 
 
@@ -118,17 +112,12 @@ async function main() {
         transactionTests,
         subscriptionTests,
         retentionTests,
-        maintenanceTests
+        maintenanceTests,
+        bootstrapTests
     ]
     
     const aiTests = [
-        errorHandlingTests,
-        leaseRenewalTests,
-        resourcesTests,
-        bufferingTests,
-        priorityTests,
-        ttlRetentionTests,
-        mixedScenariosTests
+
     ]
 
     const allTests = [...humanTests, ...aiTests]