queen-mq 0.6.4 → 0.12.0

package/README.md

@@ -614,7 +614,7 @@ const messages: Message<OrderData>[] = await queen.queue('orders').pop()
 - **[Complete V2 Guide](client-v2/README.md)** - Full tutorial with all features (94 test examples)
 - **[HTTP API Reference](https://github.com/smartpricing/queen/blob/master/server/API.md)** - Raw HTTP endpoints
 - **[Server Guide](https://github.com/smartpricing/queen/blob/master/server/README.md)** - Server setup and configuration
-- **[Architecture Guide](https://github.com/smartpricing/queen/blob/master/docs/ARCHITECTURE.md)** - Deep dive into internals
+- **[Architecture Guide](https://github.com/smartpricing/queen/blob/master/documentation/ARCHITECTURE.md)** - Deep dive into internals
 
 ---
 

package/client-v2/Queen.js

@@ -10,6 +10,7 @@ import { QueueBuilder } from './builders/QueueBuilder.js'
 import { TransactionBuilder } from './builders/TransactionBuilder.js'
 import { StreamBuilder } from './stream/StreamBuilder.js'
 import { StreamConsumer } from './stream/StreamConsumer.js'
+import { Admin } from './admin/Admin.js'
 import { CLIENT_DEFAULTS } from './utils/defaults.js'
 import { validateUrl, validateUrls } from './utils/validation.js'
 import * as logger from './utils/logger.js'
@@ -19,6 +20,7 @@ export class Queen {
   #bufferManager
   #config
   #shutdownHandlers = []
+  #admin = null
 
   constructor(config = {}) {
     logger.log('Queen.constructor', { config: typeof config === 'object' && !Array.isArray(config) ? { ...config, urls: config.urls?.length || 0 } : { type: typeof config } })
@@ -75,7 +77,7 @@ export class Queen {
   }
 
   #createHttpClient() {
-    const { urls, timeoutMillis, retryAttempts, retryDelayMillis, loadBalancingStrategy, enableFailover } = this.#config
+    const { urls, timeoutMillis, retryAttempts, retryDelayMillis, loadBalancingStrategy, affinityHashRing, healthRetryAfterMillis, enableFailover, bearerToken } = this.#config
 
     if (urls.length === 1) {
       // Single server
@@ -83,22 +85,32 @@ export class Queen {
         baseUrl: urls[0],
         timeoutMillis,
         retryAttempts,
-        retryDelayMillis
+        retryDelayMillis,
+        bearerToken
       })
     }
 
     // Multiple servers with load balancing
-    const loadBalancer = new LoadBalancer(urls, loadBalancingStrategy)
+    const loadBalancer = new LoadBalancer(urls, loadBalancingStrategy, {
+      affinityHashRing,
+      healthRetryAfterMillis
+    })
     return new HttpClient({
       loadBalancer,
       timeoutMillis,
       retryAttempts,
       retryDelayMillis,
-      enableFailover
+      enableFailover,
+      bearerToken
     })
   }
 
   #setupGracefulShutdown() {
+    // Skip signal handlers in browser environment
+    if (typeof process === 'undefined' || typeof process.on !== 'function') {
+      return
+    }
+
     let signalReceivedCount = 0;
     const shutdown = async (signal) => {
       console.log(`\nReceived ${signal}, shutting down gracefully...`)
@@ -137,6 +149,21 @@ export class Queen {
     return new QueueBuilder(this, this.#httpClient, this.#bufferManager, name)
   }
 
+  // ===========================
+  // Admin API Entry Point
+  // ===========================
+
+  /**
+   * Get the Admin API for administrative and observability operations
+   * @returns {Admin} Admin API instance (lazily initialized, singleton)
+   */
+  get admin() {
+    if (!this.#admin) {
+      this.#admin = new Admin(this.#httpClient)
+    }
+    return this.#admin
+  }
+
   // ===========================
   // Transaction API
   // ===========================

package/client-v2/README.md

@@ -43,16 +43,67 @@ const queen = new Queen(['http://server1:6632', 'http://server2:6632'])
 
 // Or with full configuration
 const queen = new Queen({
-  urls: ['http://server1:6632', 'http://server2:6632'],
+  urls: ['http://server1:6632', 'http://server2:6632', 'http://server3:6632'],
   timeoutMillis: 30000,
   retryAttempts: 3,
-  loadBalancingStrategy: 'round-robin', // or 'session'
+  loadBalancingStrategy: 'affinity',  // 'affinity', 'round-robin', or 'session'
+  affinityHashRing: 150,               // Virtual nodes per server (for affinity mode)
   enableFailover: true
 })
 ```
 
 That's it! You're connected. Now let's do something fun. 🎉
 
+### Load Balancing Strategies
+
+When connecting to multiple Queen servers, you can choose how requests are distributed:
+
+#### Affinity Mode (Recommended for Production)
+
+Uses consistent hashing with virtual nodes to route consumer groups to the same backend server. This optimizes database queries by consolidating poll intentions.
+
+```javascript
+const queen = new Queen({
+  urls: ['http://server1:6632', 'http://server2:6632', 'http://server3:6632'],
+  loadBalancingStrategy: 'affinity',
+  affinityHashRing: 150  // Virtual nodes per server (default: 150)
+})
+```
+
+**Benefits:**
+- ✅ Same consumer group always routes to same server
+- ✅ Poll intentions consolidated → optimized DB queries
+- ✅ Graceful failover (only ~33% of keys move if server fails)
+- ✅ Works great with 3-server HA setup
+
+**How it works:**
+1. Each consumer group generates an affinity key: `queue:partition:consumerGroup`
+2. Key is hashed and mapped to a virtual node on the ring
+3. Virtual node maps to a real backend server
+4. Same key always routes to same server
+
+#### Round-Robin Mode
+
+Cycles through servers in order. Simple but doesn't optimize for poll intention consolidation.
+
+```javascript
+const queen = new Queen({
+  urls: ['http://server1:6632', 'http://server2:6632'],
+  loadBalancingStrategy: 'round-robin'
+})
+```
+
+#### Session Mode
+
+Sticky sessions - each client instance sticks to one server.
+
+```javascript
+const queen = new Queen({
+  urls: ['http://server1:6632', 'http://server2:6632'],
+  loadBalancingStrategy: 'session'
+})
+```
+
 ---
 
 ## Part 1: Hello Queue!

package/client-v2/admin/Admin.js

@@ -0,0 +1,414 @@
+/**
+ * Admin API for Queen - Administrative and observability endpoints
+ * These APIs are typically used by dashboards and admin tools, not regular applications
+ */
+
+import * as logger from '../utils/logger.js'
+
+export class Admin {
+  #httpClient
+
+  constructor(httpClient) {
+    this.#httpClient = httpClient
+  }
+
+  // ===========================
+  // Resources API
+  // ===========================
+
+  /**
+   * Get system overview with queue counts, message stats, etc.
+   * @returns {Promise<object>}
+   */
+  async getOverview() {
+    logger.log('Admin.getOverview', {})
+    return this.#httpClient.get('/api/v1/resources/overview')
+  }
+
+  /**
+   * Get all namespaces
+   * @returns {Promise<object>}
+   */
+  async getNamespaces() {
+    logger.log('Admin.getNamespaces', {})
+    return this.#httpClient.get('/api/v1/resources/namespaces')
+  }
+
+  /**
+   * Get all tasks
+   * @returns {Promise<object>}
+   */
+  async getTasks() {
+    logger.log('Admin.getTasks', {})
+    return this.#httpClient.get('/api/v1/resources/tasks')
+  }
+
+  // ===========================
+  // Queues API
+  // ===========================
+
+  /**
+   * List all queues
+   * @param {object} params - Query parameters (limit, offset, search, etc.)
+   * @returns {Promise<object>}
+   */
+  async listQueues(params = {}) {
+    logger.log('Admin.listQueues', { params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/resources/queues${queryString}`)
+  }
+
+  /**
+   * Get queue details
+   * @param {string} name - Queue name
+   * @returns {Promise<object>}
+   */
+  async getQueue(name) {
+    logger.log('Admin.getQueue', { name })
+    return this.#httpClient.get(`/api/v1/resources/queues/${encodeURIComponent(name)}`)
+  }
+
+  /**
+   * Clear all messages from a queue
+   * @param {string} name - Queue name
+   * @param {string} [partition] - Optional partition to clear
+   * @returns {Promise<object>}
+   */
+  async clearQueue(name, partition = null) {
+    logger.log('Admin.clearQueue', { name, partition })
+    const queryString = partition ? `?partition=${encodeURIComponent(partition)}` : ''
+    return this.#httpClient.delete(`/api/v1/queues/${encodeURIComponent(name)}/clear${queryString}`)
+  }
+
+  /**
+   * Get all partitions
+   * @param {object} params - Query parameters (queue, limit, etc.)
+   * @returns {Promise<object>}
+   */
+  async getPartitions(params = {}) {
+    logger.log('Admin.getPartitions', { params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/resources/partitions${queryString}`)
+  }
+
+  // ===========================
+  // Messages API
+  // ===========================
+
+  /**
+   * List messages with filters
+   * @param {object} params - Query parameters (queue, partition, status, limit, offset, etc.)
+   * @returns {Promise<object>}
+   */
+  async listMessages(params = {}) {
+    logger.log('Admin.listMessages', { params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/messages${queryString}`)
+  }
+
+  /**
+   * Get a specific message
+   * @param {string} partitionId - Partition ID
+   * @param {string} transactionId - Transaction ID
+   * @returns {Promise<object>}
+   */
+  async getMessage(partitionId, transactionId) {
+    logger.log('Admin.getMessage', { partitionId, transactionId })
+    return this.#httpClient.get(`/api/v1/messages/${partitionId}/${transactionId}`)
+  }
+
+  /**
+   * Delete a specific message
+   * @param {string} partitionId - Partition ID
+   * @param {string} transactionId - Transaction ID
+   * @returns {Promise<object>}
+   */
+  async deleteMessage(partitionId, transactionId) {
+    logger.log('Admin.deleteMessage', { partitionId, transactionId })
+    return this.#httpClient.delete(`/api/v1/messages/${partitionId}/${transactionId}`)
+  }
+
+  /**
+   * Retry a failed message
+   * @param {string} partitionId - Partition ID
+   * @param {string} transactionId - Transaction ID
+   * @returns {Promise<object>}
+   */
+  async retryMessage(partitionId, transactionId) {
+    logger.log('Admin.retryMessage', { partitionId, transactionId })
+    return this.#httpClient.post(`/api/v1/messages/${partitionId}/${transactionId}/retry`, {})
+  }
+
+  /**
+   * Move a message to the Dead Letter Queue
+   * @param {string} partitionId - Partition ID
+   * @param {string} transactionId - Transaction ID
+   * @returns {Promise<object>}
+   */
+  async moveMessageToDLQ(partitionId, transactionId) {
+    logger.log('Admin.moveMessageToDLQ', { partitionId, transactionId })
+    return this.#httpClient.post(`/api/v1/messages/${partitionId}/${transactionId}/dlq`, {})
+  }
+
+  // ===========================
+  // Traces API
+  // ===========================
+
+  /**
+   * Get traces by name
+   * @param {string} traceName - Trace name to search for
+   * @param {object} params - Query parameters (limit, offset, from, to, etc.)
+   * @returns {Promise<object>}
+   */
+  async getTracesByName(traceName, params = {}) {
+    logger.log('Admin.getTracesByName', { traceName, params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/traces/by-name/${encodeURIComponent(traceName)}${queryString}`)
+  }
+
+  /**
+   * Get available trace names
+   * @param {object} params - Query parameters (limit, search, etc.)
+   * @returns {Promise<object>}
+   */
+  async getTraceNames(params = {}) {
+    logger.log('Admin.getTraceNames', { params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/traces/names${queryString}`)
+  }
+
+  /**
+   * Get traces for a specific message
+   * @param {string} partitionId - Partition ID
+   * @param {string} transactionId - Transaction ID
+   * @returns {Promise<object>}
+   */
+  async getTracesForMessage(partitionId, transactionId) {
+    logger.log('Admin.getTracesForMessage', { partitionId, transactionId })
+    return this.#httpClient.get(`/api/v1/traces/${partitionId}/${transactionId}`)
+  }
+
+  // ===========================
+  // Analytics/Status API
+  // ===========================
+
+  /**
+   * Get system status
+   * @param {object} params - Query parameters
+   * @returns {Promise<object>}
+   */
+  async getStatus(params = {}) {
+    logger.log('Admin.getStatus', { params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/status${queryString}`)
+  }
+
+  /**
+   * Get queue statistics
+   * @param {object} params - Query parameters (limit, offset, etc.)
+   * @returns {Promise<object>}
+   */
+  async getQueueStats(params = {}) {
+    logger.log('Admin.getQueueStats', { params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/status/queues${queryString}`)
+  }
+
+  /**
+   * Get detailed statistics for a specific queue
+   * @param {string} name - Queue name
+   * @param {object} params - Query parameters
+   * @returns {Promise<object>}
+   */
+  async getQueueDetail(name, params = {}) {
+    logger.log('Admin.getQueueDetail', { name, params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/status/queues/${encodeURIComponent(name)}${queryString}`)
+  }
+
+  /**
+   * Get analytics data
+   * @param {object} params - Query parameters (from, to, interval, etc.)
+   * @returns {Promise<object>}
+   */
+  async getAnalytics(params = {}) {
+    logger.log('Admin.getAnalytics', { params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/status/analytics${queryString}`)
+  }
+
+  // ===========================
+  // Consumer Groups API
+  // ===========================
+
+  /**
+   * List all consumer groups
+   * @returns {Promise<object>}
+   */
+  async listConsumerGroups() {
+    logger.log('Admin.listConsumerGroups', {})
+    return this.#httpClient.get('/api/v1/consumer-groups')
+  }
+
+  /**
+   * Refresh consumer group statistics
+   * @returns {Promise<object>}
+   */
+  async refreshConsumerStats() {
+    logger.log('Admin.refreshConsumerStats', {})
+    return this.#httpClient.post('/api/v1/stats/refresh', {})
+  }
+
+  /**
+   * Get consumer group details
+   * @param {string} name - Consumer group name
+   * @returns {Promise<object>}
+   */
+  async getConsumerGroup(name) {
+    logger.log('Admin.getConsumerGroup', { name })
+    return this.#httpClient.get(`/api/v1/consumer-groups/${encodeURIComponent(name)}`)
+  }
+
+  /**
+   * Get lagging consumer groups
+   * @param {number} [minLagSeconds=60] - Minimum lag in seconds to be considered lagging
+   * @returns {Promise<object>}
+   */
+  async getLaggingConsumers(minLagSeconds = 60) {
+    logger.log('Admin.getLaggingConsumers', { minLagSeconds })
+    return this.#httpClient.get(`/api/v1/consumer-groups/lagging?minLagSeconds=${minLagSeconds}`)
+  }
+
+  /**
+   * Delete a consumer group for a specific queue
+   * @param {string} consumerGroup - Consumer group name
+   * @param {string} queueName - Queue name
+   * @param {boolean} [deleteMetadata=true] - Whether to delete subscription metadata
+   * @returns {Promise<object>}
+   */
+  async deleteConsumerGroupForQueue(consumerGroup, queueName, deleteMetadata = true) {
+    logger.log('Admin.deleteConsumerGroupForQueue', { consumerGroup, queueName, deleteMetadata })
+    const url = `/api/v1/consumer-groups/${encodeURIComponent(consumerGroup)}/queues/${encodeURIComponent(queueName)}?deleteMetadata=${deleteMetadata}`
+    return this.#httpClient.delete(url)
+  }
+
+  /**
+   * Seek consumer group offset for a queue
+   * @param {string} consumerGroup - Consumer group name
+   * @param {string} queueName - Queue name
+   * @param {object} options - Seek options (timestamp, position, etc.)
+   * @returns {Promise<object>}
+   */
+  async seekConsumerGroup(consumerGroup, queueName, options = {}) {
+    logger.log('Admin.seekConsumerGroup', { consumerGroup, queueName, options })
+    const url = `/api/v1/consumer-groups/${encodeURIComponent(consumerGroup)}/queues/${encodeURIComponent(queueName)}/seek`
+    return this.#httpClient.post(url, options)
+  }
+
+  // ===========================
+  // System API
+  // ===========================
+
+  /**
+   * Health check
+   * @returns {Promise<object>}
+   */
+  async health() {
+    logger.log('Admin.health', {})
+    return this.#httpClient.get('/health')
+  }
+
+  /**
+   * Get Prometheus metrics
+   * @returns {Promise<string>} Raw metrics text
+   */
+  async metrics() {
+    logger.log('Admin.metrics', {})
+    return this.#httpClient.get('/metrics')
+  }
+
+  /**
+   * Get push maintenance mode status
+   * @returns {Promise<object>}
+   */
+  async getMaintenanceMode() {
+    logger.log('Admin.getMaintenanceMode', {})
+    return this.#httpClient.get('/api/v1/system/maintenance')
+  }
+
+  /**
+   * Set push maintenance mode
+   * @param {boolean} enabled - Enable or disable maintenance mode
+   * @returns {Promise<object>}
+   */
+  async setMaintenanceMode(enabled) {
+    logger.log('Admin.setMaintenanceMode', { enabled })
+    return this.#httpClient.post('/api/v1/system/maintenance', { enabled })
+  }
+
+  /**
+   * Get pop maintenance mode status
+   * @returns {Promise<object>}
+   */
+  async getPopMaintenanceMode() {
+    logger.log('Admin.getPopMaintenanceMode', {})
+    return this.#httpClient.get('/api/v1/system/maintenance/pop')
+  }
+
+  /**
+   * Set pop maintenance mode
+   * @param {boolean} enabled - Enable or disable pop maintenance mode
+   * @returns {Promise<object>}
+   */
+  async setPopMaintenanceMode(enabled) {
+    logger.log('Admin.setPopMaintenanceMode', { enabled })
+    return this.#httpClient.post('/api/v1/system/maintenance/pop', { enabled })
+  }
+
+  /**
+   * Get system metrics (CPU, memory, connections, etc.)
+   * @param {object} params - Query parameters (from, to, etc.)
+   * @returns {Promise<object>}
+   */
+  async getSystemMetrics(params = {}) {
+    logger.log('Admin.getSystemMetrics', { params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/analytics/system-metrics${queryString}`)
+  }
+
+  /**
+   * Get worker metrics
+   * @param {object} params - Query parameters
+   * @returns {Promise<object>}
+   */
+  async getWorkerMetrics(params = {}) {
+    logger.log('Admin.getWorkerMetrics', { params })
+    const queryString = this.#buildQueryString(params)
+    return this.#httpClient.get(`/api/v1/analytics/worker-metrics${queryString}`)
+  }
+
+  /**
+   * Get PostgreSQL statistics
+   * @returns {Promise<object>}
+   */
+  async getPostgresStats() {
+    logger.log('Admin.getPostgresStats', {})
+    return this.#httpClient.get('/api/v1/analytics/postgres-stats')
+  }
+
+  // ===========================
+  // Helper Methods
+  // ===========================
+
+  #buildQueryString(params) {
+    const searchParams = new URLSearchParams()
+    for (const [key, value] of Object.entries(params)) {
+      if (value !== undefined && value !== null) {
+        searchParams.append(key, String(value))
+      }
+    }
+    const queryString = searchParams.toString()
+    return queryString ? `?${queryString}` : ''
+  }
+}
+

package/client-v2/admin/index.js

@@ -0,0 +1,6 @@
+/**
+ * Admin module exports
+ */
+
+export { Admin } from './Admin.js'
+

package/client-v2/builders/QueueBuilder.js

@@ -48,6 +48,26 @@ export class QueueBuilder {
     this.#queueName = queueName
   }
 
+  // ===========================
+  // Affinity Key Generation
+  // ===========================
+
+  /**
+   * Generate affinity key for consistent routing
+   * Matches server's PollIntention::grouping_key() format
+   * Format: queue:partition:consumerGroup or namespace:task:consumerGroup
+   */
+  #getAffinityKey() {
+    if (this.#queueName) {
+      // Queue-based routing: queue:partition:consumerGroup
+      return `${this.#queueName}:${this.#partition || '*'}:${this.#group || '__QUEUE_MODE__'}`
+    } else if (this.#namespace || this.#task) {
+      // Namespace/task-based routing: namespace:task:consumerGroup
+      return `${this.#namespace || '*'}:${this.#task || '*'}:${this.#group || '__QUEUE_MODE__'}`
+    }
+    return null
+  }
+
   // ===========================
   // Queue Configuration Methods
   // ===========================
@@ -270,7 +290,10 @@ export class QueueBuilder {
       if (this.#subscriptionMode) params.append('subscriptionMode', this.#subscriptionMode)
       if (this.#subscriptionFrom) params.append('subscriptionFrom', this.#subscriptionFrom)
 
-      const result = await this.#httpClient.get(`${path}?${params}`, this.#timeoutMillis + 5000)
+      // Generate affinity key for consistent routing to same backend
+      const affinityKey = this.#getAffinityKey()
+      
+      const result = await this.#httpClient.get(`${path}?${params}`, this.#timeoutMillis + 5000, affinityKey)
 
       if (!result || !result.messages) {
         logger.log('QueueBuilder.pop', { status: 'no-messages' })

package/client-v2/consumer/ConsumerManager.js

@@ -13,6 +13,22 @@ export class ConsumerManager {
     this.#queen = queen
   }
 
+  /**
+   * Generate affinity key for consistent routing
+   * Matches server's PollIntention::grouping_key() format
+   * Format: queue:partition:consumerGroup or namespace:task:consumerGroup
+   */
+  #getAffinityKey(queue, partition, namespace, task, group) {
+    if (queue) {
+      // Queue-based routing: queue:partition:consumerGroup
+      return `${queue}:${partition || '*'}:${group || '__QUEUE_MODE__'}`
+    } else if (namespace || task) {
+      // Namespace/task-based routing: namespace:task:consumerGroup
+      return `${namespace || '*'}:${task || '*'}:${group || '__QUEUE_MODE__'}`
+    }
+    return null
+  }
+
   async start(handler, options) {
     const {
       queue,
@@ -52,6 +68,9 @@ export class ConsumerManager {
     // Build the path and params for pop requests
     const path = this.#buildPath(queue, partition, namespace, task)
     const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck)
+    
+    // Generate affinity key for consistent routing to same backend
+    const affinityKey = this.#getAffinityKey(queue, partition, namespace, task, group)
 
     // Start workers
     const workers = []
@@ -67,7 +86,8 @@ export class ConsumerManager {
         renewLeaseIntervalMillis,
         each,
         signal,
-        group  // Pass consumer group to workers
+        group,  // Pass consumer group to workers
+        affinityKey  // Pass affinity key to workers
       }))
     }
 
@@ -91,7 +111,8 @@ export class ConsumerManager {
       renewLeaseIntervalMillis,
       each,
       signal,
-      group
+      group,
+      affinityKey
     } = options
 
     logger.log('ConsumerManager.worker', { workerId, status: 'started', limit, idleMillis })
@@ -122,9 +143,9 @@ export class ConsumerManager {
       }
 
       try {
-        // Pop messages
+        // Pop messages with affinity key for consistent routing
         const clientTimeout = wait ? timeoutMillis + 5000 : timeoutMillis
-        const result = await this.#httpClient.get(`${path}?${baseParams}`, clientTimeout)
+        const result = await this.#httpClient.get(`${path}?${baseParams}`, clientTimeout, affinityKey)
 
         // Handle empty response
         if (!result || !result.messages || result.messages.length === 0) {