queen-mq 0.2.23 → 0.3.1

package/client-js/client-v2/builders/TransactionBuilder.js

@@ -0,0 +1,110 @@
+/**
+ * Transaction builder for atomic operations
+ */
+
+import * as logger from '../utils/logger.js'
+
+export class TransactionBuilder {
+  #httpClient
+  #operations = []
+  #requiredLeases = []
+
+  constructor(httpClient) {
+    this.#httpClient = httpClient
+  }
+
+  ack(messages, status = 'completed') {
+    const msgs = Array.isArray(messages) ? messages : [messages]
+    
+    logger.log('TransactionBuilder.ack', { count: msgs.length, status })
+
+    msgs.forEach(msg => {
+      const transactionId = typeof msg === 'string' ? msg : (msg.transactionId || msg.id)
+      const partitionId = typeof msg === 'object' ? msg.partitionId : null
+      const leaseId = typeof msg === 'object' ? msg.leaseId : null
+
+      if (!transactionId) {
+        throw new Error('Message must have transactionId or id property')
+      }
+
+      // CRITICAL: partitionId is now MANDATORY to prevent acking wrong message
+      if (!partitionId) {
+        throw new Error('Message must have partitionId property to ensure message uniqueness')
+      }
+
+      this.#operations.push({
+        type: 'ack',
+        transactionId,
+        partitionId,
+        status
+      })
+
+      if (leaseId) {
+        this.#requiredLeases.push(leaseId)
+      }
+    })
+
+    return this
+  }
+
+  queue(queueName) {
+    // Return a sub-builder for push operations
+    return {
+      push: (items) => {
+        const itemArray = Array.isArray(items) ? items : [items]
+        
+        logger.log('TransactionBuilder.queue.push', { queue: queueName, count: itemArray.length })
+
+        this.#operations.push({
+          type: 'push',
+          items: itemArray.map(item => {
+            // Check if property exists, not just truthy (to support null values)
+            let payloadValue
+            if ('data' in item) {
+              payloadValue = item.data
+            } else if ('payload' in item) {
+              payloadValue = item.payload
+            } else {
+              payloadValue = item
+            }
+
+            return {
+              queue: queueName,
+              payload: payloadValue
+            }
+          })
+        })
+
+        return this
+      }
+    }
+  }
+
+  async commit() {
+    if (this.#operations.length === 0) {
+      logger.error('TransactionBuilder.commit', 'No operations to commit')
+      throw new Error('Transaction has no operations to commit')
+    }
+
+    logger.log('TransactionBuilder.commit', { operationCount: this.#operations.length, requiredLeases: this.#requiredLeases.length })
+
+    try {
+      const result = await this.#httpClient.post('/api/v1/transaction', {
+        operations: this.#operations,
+        requiredLeases: [...new Set(this.#requiredLeases)] // Unique leases
+      })
+
+      if (!result.success) {
+        logger.error('TransactionBuilder.commit', { error: result.error })
+        throw new Error(result.error || 'Transaction failed')
+      }
+
+      logger.log('TransactionBuilder.commit', { status: 'success' })
+      return result
+    } catch (error) {
+      logger.error('TransactionBuilder.commit', { error: error.message })
+      throw error
+    }
+  }
+}
+

package/client-js/client-v2/consumer/ConsumerManager.js

@@ -0,0 +1,390 @@
+/**
+ * Consumer manager for handling concurrent workers
+ */
+
+import * as logger from '../utils/logger.js'
+
+export class ConsumerManager {
+  #httpClient
+  #queen
+
+  constructor(httpClient, queen) {
+    this.#httpClient = httpClient
+    this.#queen = queen
+  }
+
+  async start(handler, options) {
+    const {
+      queue,
+      partition,
+      namespace,
+      task,
+      group,
+      concurrency,
+      batch,
+      limit,
+      idleMillis,
+      autoAck,
+      wait,
+      timeoutMillis,
+      renewLease,
+      renewLeaseIntervalMillis,
+      subscriptionMode,
+      subscriptionFrom,
+      each,
+      signal
+    } = options
+
+    logger.log('ConsumerManager.start', { 
+      queue, 
+      partition, 
+      namespace, 
+      task, 
+      group, 
+      concurrency, 
+      batch, 
+      limit, 
+      autoAck,
+      wait,
+      each
+    })
+
+    // Build the path and params for pop requests
+    const path = this.#buildPath(queue, partition, namespace, task)
+    const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck)
+
+    // Start workers
+    const workers = []
+    for (let i = 0; i < concurrency; i++) {
+      workers.push(this.#worker(i, handler, path, baseParams, {
+        batch,
+        limit,
+        idleMillis,
+        autoAck,
+        wait,
+        timeoutMillis,
+        renewLease,
+        renewLeaseIntervalMillis,
+        each,
+        signal,
+        group  // Pass consumer group to workers
+      }))
+    }
+
+    logger.log('ConsumerManager.start', { status: 'workers-started', count: concurrency })
+
+    // Wait for all workers to complete
+    await Promise.all(workers)
+    
+    logger.log('ConsumerManager.start', { status: 'completed' })
+  }
+
+  async #worker(workerId, handler, path, baseParams, options) {
+    const {
+      batch,
+      limit,
+      idleMillis,
+      autoAck,
+      wait,
+      timeoutMillis,
+      renewLease,
+      renewLeaseIntervalMillis,
+      each,
+      signal,
+      group
+    } = options
+
+    logger.log('ConsumerManager.worker', { workerId, status: 'started', limit, idleMillis })
+    
+    let processedCount = 0
+    let lastMessageTime = idleMillis ? Date.now() : null
+
+    while (true) {
+      // Check abort signal
+      if (signal && signal.aborted) {
+        logger.log('ConsumerManager.worker', { workerId, status: 'aborted', processedCount })
+        break
+      }
+
+      // Check limit
+      if (limit && processedCount >= limit) {
+        logger.log('ConsumerManager.worker', { workerId, status: 'limit-reached', processedCount, limit })
+        break
+      }
+
+      // Check idle timeout
+      if (idleMillis && lastMessageTime) {
+        const idleTime = Date.now() - lastMessageTime
+        if (idleTime >= idleMillis) {
+          logger.log('ConsumerManager.worker', { workerId, status: 'idle-timeout', processedCount, idleTime })
+          break
+        }
+      }
+
+      try {
+        // Pop messages
+        const clientTimeout = wait ? timeoutMillis + 5000 : timeoutMillis
+        const result = await this.#httpClient.get(`${path}?${baseParams}`, clientTimeout)
+
+        // Handle empty response
+        if (!result || !result.messages || result.messages.length === 0) {
+          if (wait) {
+            continue // Long polling timeout, retry
+          } else {
+            // Short delay before retry
+            await new Promise(resolve => setTimeout(resolve, 100))
+            continue
+          }
+        }
+
+        const messages = result.messages.filter(msg => msg != null)
+
+        if (messages.length === 0) {
+          continue
+        }
+
+        logger.log('ConsumerManager.worker', { workerId, status: 'messages-received', count: messages.length })
+
+        // Enhance messages with trace() method
+        this.#enhanceMessagesWithTrace(messages, group)
+
+        // Update last message time
+        if (idleMillis) {
+          lastMessageTime = Date.now()
+        }
+
+        // Set up lease renewal if enabled
+        let renewalTimer = null
+        if (renewLease && renewLeaseIntervalMillis) {
+          renewalTimer = this.#setupLeaseRenewal(messages, renewLeaseIntervalMillis)
+        }
+
+        try {
+          // Process messages
+          if (each) {
+            // Process one at a time
+            for (const message of messages) {
+              if (signal && signal.aborted) break
+
+              await this.#processMessage(message, handler, autoAck, group)
+              processedCount++
+
+              if (limit && processedCount >= limit) break
+            }
+          } else {
+            // Process as batch
+            await this.#processBatch(messages, handler, autoAck, group)
+            processedCount += messages.length
+          }
+          
+          logger.log('ConsumerManager.worker', { workerId, status: 'messages-processed', count: messages.length, total: processedCount })
+        } finally {
+          // Clear renewal timer
+          if (renewalTimer) {
+            clearInterval(renewalTimer)
+          }
+        }
+
+      } catch (error) {
+        // Check if this is a timeout error (expected for long polling)
+        const isTimeoutError = error.name === 'AbortError' ||
+                              error.message?.includes('timeout')
+
+        if (isTimeoutError && wait) {
+          continue // Retry on timeout
+        }
+
+        // Check if network error
+        const isNetworkError = error.message?.includes('fetch failed') ||
+                              error.message?.includes('ECONNREFUSED') ||
+                              error.code === 'ECONNREFUSED'
+
+        if (isNetworkError) {
+          logger.warn('ConsumerManager.worker', { workerId, error: 'network', message: error.message })
+          console.warn(`Worker ${workerId}: Network error - ${error.message}`)
+          // Wait before retry
+          await new Promise(resolve => setTimeout(resolve, 1000))
+          continue
+        }
+
+        // Other errors - rethrow
+        logger.error('ConsumerManager.worker', { workerId, error: error.message })
+        throw error
+      }
+    }
+    
+    logger.log('ConsumerManager.worker', { workerId, status: 'stopped', processedCount })
+  }
+
+  async #processMessage(message, handler, autoAck, group) {
+    try {
+      await handler(message)
+
+      // Auto-ack on success if enabled
+      if (autoAck) {
+        const context = group ? { group } : {}
+        await this.#queen.ack(message, true, context)
+        logger.log('ConsumerManager.processMessage', { transactionId: message.transactionId, status: 'acked' })
+      }
+    } catch (error) {
+      // Auto-nack on error if enabled
+      if (autoAck) {
+        const context = group ? { group } : {}
+        await this.#queen.ack(message, false, context)
+        logger.error('ConsumerManager.processMessage', { transactionId: message.transactionId, error: error.message, status: 'nacked' })
+        // Don't rethrow when autoAck is enabled - NACK was already sent
+        // This allows the consumer to continue and retry
+        return
+      }
+      logger.error('ConsumerManager.processMessage', { transactionId: message.transactionId, error: error.message })
+      throw error
+    }
+  }
+
+  async #processBatch(messages, handler, autoAck, group) {
+    try {
+      await handler(messages)
+
+      // Auto-ack on success if enabled
+      if (autoAck) {
+        const context = group ? { group } : {}
+        await this.#queen.ack(messages, true, context)
+        logger.log('ConsumerManager.processBatch', { count: messages.length, status: 'acked' })
+      }
+    } catch (error) {
+      // Auto-nack on error if enabled
+      if (autoAck) {
+        const context = group ? { group } : {}
+        await this.#queen.ack(messages, false, context)
+        logger.error('ConsumerManager.processBatch', { count: messages.length, error: error.message, status: 'nacked' })
+        // Don't rethrow when autoAck is enabled - NACK was already sent
+        // This allows the consumer to continue and retry
+        return
+      }
+      logger.error('ConsumerManager.processBatch', { count: messages.length, error: error.message })
+      throw error
+    }
+  }
+
+  #setupLeaseRenewal(messages, intervalMillis) {
+    const leaseIds = messages.map(m => m.leaseId).filter(id => id != null)
+
+    if (leaseIds.length === 0) return null
+
+    return setInterval(async () => {
+      try {
+        await this.#queen.renew(messages)
+      } catch (error) {
+        console.error('Lease renewal failed:', error)
+      }
+    }, intervalMillis)
+  }
+
+  #enhanceMessagesWithTrace(messages, group) {
+    const httpClient = this.#httpClient
+    const consumerGroup = group || '__QUEUE_MODE__'
+    
+    for (const message of messages) {
+      /**
+       * Record a trace event for this message
+       * @param {object} traceConfig - Configuration object
+       * @param {string|string[]} [traceConfig.traceName] - Single name or array of names for categorization
+       * @param {string} [traceConfig.eventType='info'] - Event type (info, error, step, processing, etc.)
+       * @param {object} traceConfig.data - User data to store with the trace
+       * @returns {Promise<object>} Result with success status
+       * 
+       * IMPORTANT: This method will NEVER crash - errors are logged but don't throw
+       * 
+       * @example
+       * await msg.trace({
+       *   traceName: ['tenant-acme', 'chat-room-123'],
+       *   eventType: 'info',
+       *   data: { text: 'Started processing', orderId: 123 }
+       * });
+       */
+      message.trace = async (traceConfig) => {
+        try {
+          // Validate required structure
+          if (typeof traceConfig !== 'object' || !traceConfig.data) {
+            logger.warn('ConsumerManager.trace', { 
+              error: 'Invalid trace config: requires { data: {...} }',
+              transactionId: message.transactionId 
+            })
+            return { success: false, error: 'Invalid trace config: requires { data: {...} }' }
+          }
+          
+          // Normalize traceName to array
+          let traceNames = null
+          if (traceConfig.traceName) {
+            if (Array.isArray(traceConfig.traceName)) {
+              traceNames = traceConfig.traceName.filter(n => typeof n === 'string' && n.length > 0)
+              if (traceNames.length === 0) traceNames = null
+            } else if (typeof traceConfig.traceName === 'string') {
+              traceNames = [traceConfig.traceName]
+            }
+          }
+          
+          const response = await httpClient.post('/api/v1/traces', {
+            transactionId: message.transactionId,
+            partitionId: message.partitionId,
+            consumerGroup: consumerGroup,
+            traceNames: traceNames,
+            eventType: traceConfig.eventType || 'info',
+            data: traceConfig.data
+          })
+          
+          logger.log('ConsumerManager.trace', { 
+            transactionId: message.transactionId, 
+            success: true,
+            traceNames: traceNames 
+          })
+          return { success: true, ...response }
+        } catch (error) {
+          // CRITICAL: NEVER CRASH - just log and return gracefully
+          logger.error('ConsumerManager.trace', {
+            transactionId: message.transactionId,
+            error: error.message,
+            phase: 'trace-failed'
+          })
+          console.warn(`[TRACE FAILED] ${message.transactionId}: ${error.message}`)
+          
+          return { success: false, error: error.message }
+        }
+      }
+    }
+  }
+
+  #buildPath(queue, partition, namespace, task) {
+    if (queue) {
+      if (partition) {
+        return `/api/v1/pop/queue/${queue}/partition/${partition}`
+      }
+      return `/api/v1/pop/queue/${queue}`
+    }
+
+    if (namespace || task) {
+      return '/api/v1/pop'
+    }
+
+    throw new Error('Must specify queue, namespace, or task')
+  }
+
+  #buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck) {
+    const params = new URLSearchParams({
+      batch: batch.toString(),
+      wait: wait.toString(),
+      timeout: timeoutMillis.toString()  // Server expects 'timeout', not 'timeoutMillis'
+    })
+
+    if (group) params.append('consumerGroup', group)
+    if (subscriptionMode) params.append('subscriptionMode', subscriptionMode)
+    if (subscriptionFrom) params.append('subscriptionFrom', subscriptionFrom)
+    if (namespace) params.append('namespace', namespace)
+    if (task) params.append('task', task)
+    // NEVER send autoAck for consume - client always manages acking
+    // autoAck is only for pop() where server auto-acks immediately
+
+    return params
+  }
+}
+

package/client-js/client-v2/http/HttpClient.js

@@ -0,0 +1,215 @@
+/**
+ * HTTP client with retry, load balancing, and failover support
+ */
+
+import * as logger from '../utils/logger.js'
+
+export class HttpClient {
+  #baseUrl
+  #loadBalancer
+  #timeoutMillis
+  #retryAttempts
+  #retryDelayMillis
+  #enableFailover
+
+  constructor(options = {}) {
+    const {
+      baseUrl = null,
+      loadBalancer = null,
+      timeoutMillis = 30000,
+      retryAttempts = 3,
+      retryDelayMillis = 1000,
+      enableFailover = true
+    } = options
+
+    this.#baseUrl = baseUrl
+    this.#loadBalancer = loadBalancer
+    this.#timeoutMillis = timeoutMillis
+    this.#retryAttempts = retryAttempts
+    this.#retryDelayMillis = retryDelayMillis
+    this.#enableFailover = enableFailover
+    
+    logger.log('HttpClient.constructor', { 
+      hasLoadBalancer: !!loadBalancer, 
+      baseUrl: baseUrl || 'load-balanced', 
+      timeoutMillis, 
+      retryAttempts,
+      enableFailover 
+    })
+  }
+
+  async #executeRequest(url, method, body = null, requestTimeoutMillis = null) {
+    const effectiveTimeout = requestTimeoutMillis || this.#timeoutMillis
+    logger.log('HttpClient.request', { method, url, hasBody: !!body, timeout: effectiveTimeout })
+    
+    const controller = new AbortController()
+    const timeoutId = setTimeout(() => controller.abort(), effectiveTimeout)
+
+    try {
+      const options = {
+        method,
+        signal: controller.signal,
+        headers: { 'Content-Type': 'application/json' }
+      }
+
+      if (body) {
+        options.body = JSON.stringify(body)
+      }
+
+      const response = await fetch(url, options)
+      
+      logger.log('HttpClient.response', { method, url, status: response.status })
+
+      // Handle 204 No Content
+      if (response.status === 204) {
+        return null
+      }
+
+      // Handle errors
+      if (!response.ok) {
+        const error = new Error(`HTTP ${response.status}: ${response.statusText}`)
+        error.status = response.status
+
+        try {
+          const text = await response.text()
+          if (text) {
+            const body = JSON.parse(text)
+            error.message = body.error || error.message
+          }
+        } catch (e) {
+          // Ignore JSON parse errors
+        }
+
+        logger.error('HttpClient.request', { method, url, status: error.status, error: error.message })
+        throw error
+      }
+
+      // Parse successful response
+      const contentType = response.headers.get('content-type')
+      const contentLength = response.headers.get('content-length')
+
+      if (!contentType || !contentType.includes('application/json') || contentLength === '0') {
+        const text = await response.text()
+        if (!text || text.length === 0) {
+          return null
+        }
+        try {
+          return JSON.parse(text)
+        } catch (e) {
+          return null
+        }
+      }
+
+      return response.json()
+
+    } catch (error) {
+      if (error.name === 'AbortError') {
+        const timeoutError = new Error(`Request timeout after ${effectiveTimeout}ms`)
+        timeoutError.name = 'AbortError'
+        timeoutError.timeout = effectiveTimeout
+        logger.error('HttpClient.request', { method, url, error: 'timeout', timeout: effectiveTimeout })
+        throw timeoutError
+      }
+      logger.error('HttpClient.request', { method, url, error: error.message })
+      throw error
+    } finally {
+      clearTimeout(timeoutId)
+    }
+  }
+
+  async #requestWithRetry(method, path, body = null, requestTimeoutMillis = null) {
+    let lastError = null
+
+    for (let attempt = 0; attempt < this.#retryAttempts; attempt++) {
+      try {
+        const url = this.#getUrl() + path
+        return await this.#executeRequest(url, method, body, requestTimeoutMillis)
+      } catch (error) {
+        lastError = error
+
+        // Don't retry on client errors (4xx)
+        if (error.status && error.status >= 400 && error.status < 500) {
+          throw error
+        }
+
+        // Wait before retry (except on last attempt)
+        if (attempt < this.#retryAttempts - 1) {
+          const delay = this.#retryDelayMillis * Math.pow(2, attempt)
+          logger.warn('HttpClient.retry', { method, path, attempt: attempt + 1, delay, error: error.message })
+          await new Promise(resolve => setTimeout(resolve, delay))
+        }
+      }
+    }
+
+    logger.error('HttpClient.retry', { method, path, error: 'Max retries exceeded', attempts: this.#retryAttempts })
+    throw lastError
+  }
+
+  async #requestWithFailover(method, path, body = null, requestTimeoutMillis = null) {
+    if (!this.#loadBalancer || !this.#enableFailover) {
+      return this.#requestWithRetry(method, path, body, requestTimeoutMillis)
+    }
+
+    const urls = this.#loadBalancer.getAllUrls()
+    const attemptedUrls = new Set()
+    let lastError = null
+
+    logger.log('HttpClient.failover', { method, path, totalServers: urls.length })
+
+    for (let i = 0; i < urls.length; i++) {
+      const url = this.#loadBalancer.getNextUrl()
+
+      if (attemptedUrls.has(url)) {
+        continue
+      }
+
+      attemptedUrls.add(url)
+
+      try {
+        return await this.#executeRequest(url + path, method, body, requestTimeoutMillis)
+      } catch (error) {
+        lastError = error
+        logger.warn('HttpClient.failover', { url, method, path, error: error.message })
+        console.warn(`Request failed for ${url}: ${method} ${path} - ${error.message}`)
+
+        // Don't retry on client errors (4xx)
+        if (error.status && error.status >= 400 && error.status < 500) {
+          throw error
+        }
+
+        // Continue to next server for server errors or network issues
+      }
+    }
+
+    logger.error('HttpClient.failover', { method, path, error: 'All servers failed', attempted: attemptedUrls.size })
+    throw lastError || new Error('All servers failed')
+  }
+
+  #getUrl() {
+    if (this.#loadBalancer) {
+      return this.#loadBalancer.getNextUrl()
+    }
+    return this.#baseUrl
+  }
+
+  async get(path, requestTimeoutMillis = null) {
+    return this.#requestWithFailover('GET', path, null, requestTimeoutMillis)
+  }
+
+  async post(path, body = null, requestTimeoutMillis = null) {
+    return this.#requestWithFailover('POST', path, body, requestTimeoutMillis)
+  }
+
+  async put(path, body = null, requestTimeoutMillis = null) {
+    return this.#requestWithFailover('PUT', path, body, requestTimeoutMillis)
+  }
+
+  async delete(path, requestTimeoutMillis = null) {
+    return this.#requestWithFailover('DELETE', path, null, requestTimeoutMillis)
+  }
+
+  getLoadBalancer() {
+    return this.#loadBalancer
+  }
+}
+