queen-mq 0.12.3 → 0.13.0

package/client-v2/LOGGING.md

@@ -2,9 +2,64 @@
 
 ## Overview
 
-The Queen Client V2 includes comprehensive operation logging that captures every significant action performed by the client. Logging is **disabled by default** and can be enabled via the `QUEEN_CLIENT_LOG` environment variable.
+The Queen Client V2 includes comprehensive operation logging that captures every significant action performed by the client. You can either:
 
-## Enabling Logging
+1. **Inject a custom logger** (pino, winston, bunyan, etc.) via the `logger` config option -- recommended for production.
+2. **Use the built-in console logger** gated by the `QUEEN_CLIENT_LOG` environment variable -- handy for quick debugging.
+
+## Custom Logger (Recommended)
+
+Pass any logger instance that implements `info()`, `warn()`, and `error()` methods. When a custom logger is configured, it is always active (level filtering is controlled by your logger, not by `QUEEN_CLIENT_LOG`).
+
+### Pino
+
+```javascript
+import pino from 'pino'
+import { Queen } from '@punkish/queen'
+
+const queen = new Queen({
+  urls: ['http://localhost:6632'],
+  logger: pino()
+})
+```
+
+### Winston
+
+```javascript
+import winston from 'winston'
+import { Queen } from '@punkish/queen'
+
+const winstonLogger = winston.createLogger({
+  level: 'info',
+  transports: [new winston.transports.Console()]
+})
+
+const queen = new Queen({
+  urls: ['http://localhost:6632'],
+  logger: winstonLogger
+})
+```
+
+### Custom implementation
+
+Any object with `info`, `warn`, and `error` methods works:
+
+```javascript
+const queen = new Queen({
+  urls: ['http://localhost:6632'],
+  logger: {
+    info: (msg) => myLogSink('INFO', msg),
+    warn: (msg) => myLogSink('WARN', msg),
+    error: (msg) => myLogSink('ERROR', msg)
+  }
+})
+```
+
+The `debug()` method is optional. If your logger provides it, verbose internal details (buffer flush progress, server responses, etc.) will be emitted at `debug` level. Otherwise they fall back to `info`.
+
+## Built-in Console Logger
+
+If no custom logger is provided, the built-in console logger can be enabled via environment variable:
 
 ```bash
 export QUEEN_CLIENT_LOG=true
@@ -158,11 +213,13 @@ pm2 restart my-app
 
 ## Performance Impact
 
-- **Disabled (default)**: Zero performance impact - all logging calls are no-ops
-- **Enabled**: Minimal impact - logging is asynchronous and uses `console.log/warn/error`
+- **No logger configured + `QUEEN_CLIENT_LOG` unset (default)**: Zero performance impact - all logging calls are no-ops
+- **Built-in console logger enabled**: Minimal impact
+- **Custom logger**: Depends on the logger implementation; pino in async mode has negligible overhead
 
 ## Log Levels
 
+- **DEBUG**: Verbose operational details (buffer flush progress, extracted message counts, server responses)
 - **INFO**: Normal operations (most logs)
 - **WARN**: Recoverable issues (retries, failover, network errors)
 - **ERROR**: Operation failures (push failed, ack failed, etc.)
@@ -191,31 +248,7 @@ QUEEN_CLIENT_LOG=true node app.js 2>&1 | grep "pop"
 
 ## Integration with Log Aggregation
 
-The structured JSON format makes it easy to integrate with log aggregation tools:
-
-### Winston
-```javascript
-import winston from 'winston'
-
-// Redirect console.log to Winston
-console.log = winston.info
-console.error = winston.error
-console.warn = winston.warn
-```
-
-### Pino
-```javascript
-import pino from 'pino'
-const logger = pino()
-
-console.log = (msg) => logger.info(msg)
-console.error = (msg) => logger.error(msg)
-console.warn = (msg) => logger.warn(msg)
-```
-
-### Datadog, CloudWatch, etc.
-
-The ISO 8601 timestamps and JSON format are compatible with most log aggregation services.
+When using a custom logger, integration with log aggregation services (Datadog, CloudWatch, Elastic, etc.) depends on your logger's configuration, not on Queen. For example, pino outputs NDJSON by default which is compatible with most log aggregation pipelines out of the box.
 
 ## Best Practices
 
@@ -236,5 +269,5 @@ The logging system is implemented in `utils/logger.js` and imported by all major
 - `QueueBuilder.js` - Queue operations
 - `TransactionBuilder.js` - Atomic transactions
 
-All logging calls check the `QUEEN_CLIENT_LOG` environment variable and are no-ops when disabled, ensuring zero performance impact by default.
+When a custom logger is configured via `new Queen({ logger })`, all internal logging is routed through it. When no custom logger is provided, logging falls back to the built-in console logger gated by `QUEEN_CLIENT_LOG`.
 

package/client-v2/Queen.js

@@ -23,6 +23,11 @@ export class Queen {
   #admin = null
 
   constructor(config = {}) {
+    // Configure custom logger before anything else
+    if (config && typeof config === 'object' && !Array.isArray(config) && config.logger) {
+      logger.configure(config.logger)
+    }
+
     logger.log('Queen.constructor', { config: typeof config === 'object' && !Array.isArray(config) ? { ...config, urls: config.urls?.length || 0 } : { type: typeof config } })
     
     // Normalize config
@@ -117,17 +122,17 @@ export class Queen {
 
     let signalReceivedCount = 0;
     const shutdown = async (signal) => {
-      console.log(`\nReceived ${signal}, shutting down gracefully...`)
+      logger.log('Queen.shutdown', { signal })
       try {
         signalReceivedCount++;
         if (signalReceivedCount > 1) {
-          console.log('Received multiple shutdown signals, exiting immediately')
+          logger.warn('Queen.shutdown', 'Received multiple shutdown signals, exiting immediately')
           process.exit(1)
         }
         await this.close()
         process.exit(0)
       } catch (error) {
-        console.error('Error during shutdown:', error)
+        logger.error('Queen.shutdown', { error: error.message })
         process.exit(1)
       }
     }
@@ -480,16 +485,13 @@ export class Queen {
 
   async close() {
     logger.log('Queen.close', 'Starting shutdown')
-    console.log('Closing Queen client...')
 
     // Flush all buffers
     try {
       await this.#bufferManager.flushAllBuffers()
       logger.log('Queen.close', 'All buffers flushed')
-      console.log('All buffers flushed')
     } catch (error) {
       logger.error('Queen.close', { error: error.message, phase: 'buffer-flush' })
-      console.warn('Error flushing buffers:', error)
     }
 
     // Cleanup buffer manager
@@ -502,7 +504,6 @@ export class Queen {
     this.#shutdownHandlers = []
 
     logger.log('Queen.close', 'Client closed successfully')
-    console.log('Queen client closed')
   }
 }
 

package/client-v2/buffer/BufferManager.js

@@ -36,13 +36,11 @@ export class BufferManager {
   async #flushBuffer(queueAddress) {
     const buffer = this.#buffers.get(queueAddress)
     if (!buffer || buffer.messageCount === 0) {
-      logger.log('BufferManager.flushBuffer', { queueAddress, status: 'empty' })
-      console.log(`No buffer or empty buffer for ${queueAddress}`)
+      logger.debug('BufferManager.flushBuffer', { queueAddress, status: 'empty' })
       return
     }
 
     logger.log('BufferManager.flushBuffer', { queueAddress, messageCount: buffer.messageCount })
-    console.log(`Flushing ${buffer.messageCount} messages for ${queueAddress}`)
     buffer.setFlushing(true)
 
     // Create a promise for this flush and track it
@@ -50,13 +48,13 @@ export class BufferManager {
       try {
         const messages = buffer.extractMessages()
 
-        console.log(`Extracted ${messages.length} messages, sending to server...`)
+        logger.debug('BufferManager.flushBuffer', { queueAddress, extracted: messages.length })
 
         if (messages.length === 0) return
 
         // Send to server
         const result = await this.#httpClient.post('/api/v1/push', { items: messages })
-        console.log(`Server responded:`, result ? `${result.length || 'N/A'} items` : 'null')
+        logger.debug('BufferManager.flushBuffer', { queueAddress, serverResponse: result ? `${result.length || 'N/A'} items` : 'null' })
 
         this.#flushCount++
         logger.log('BufferManager.flushBuffer', { queueAddress, status: 'success', messagesSent: messages.length })
@@ -66,7 +64,6 @@ export class BufferManager {
 
       } catch (error) {
         logger.error('BufferManager.flushBuffer', { queueAddress, error: error.message })
-        console.error(`Flush error for ${queueAddress}:`, error.message)
         buffer.setFlushing(false)
         throw error
       } finally {
@@ -94,13 +91,13 @@ export class BufferManager {
       try {
         const messages = buffer.extractMessages(batchSize)
 
-        console.log(`Extracted ${messages.length} messages, sending to server...`)
+        logger.debug('BufferManager.flushBufferBatch', { queueAddress, extracted: messages.length })
 
         if (messages.length === 0) return
 
         // Send to server
         const result = await this.#httpClient.post('/api/v1/push', { items: messages })
-        console.log(`Server responded:`, result ? `${result.length || 'N/A'} items` : 'null')
+        logger.debug('BufferManager.flushBufferBatch', { queueAddress, serverResponse: result ? `${result.length || 'N/A'} items` : 'null' })
 
         this.#flushCount++
 
@@ -112,7 +109,7 @@ export class BufferManager {
         }
 
       } catch (error) {
-        console.error(`Flush error for ${queueAddress}:`, error.message)
+        logger.error('BufferManager.flushBufferBatch', { queueAddress, error: error.message })
         buffer.setFlushing(false)
         throw error
       } finally {
@@ -129,14 +126,10 @@ export class BufferManager {
 
   async flushBuffer(queueAddress) {
     logger.log('BufferManager.flushBuffer', { queueAddress, activeBuffers: this.#buffers.size, pendingFlushes: this.#pendingFlushes.size })
-    console.log(`flushBuffer called for address: ${queueAddress}`)
-    console.log(`Active buffers:`, Array.from(this.#buffers.keys()))
-    console.log(`Pending flushes:`, this.#pendingFlushes.size)
     
     const buffer = this.#buffers.get(queueAddress)
     if (!buffer) {
-      logger.log('BufferManager.flushBuffer', { queueAddress, status: 'not-found' })
-      console.log(`No buffer found for ${queueAddress}`)
+      logger.debug('BufferManager.flushBuffer', { queueAddress, status: 'not-found' })
       await this.#waitForPendingFlushes()
       return
     }
@@ -149,22 +142,20 @@ export class BufferManager {
     
     // Flush all messages in batches
     while (buffer.messageCount > 0) {
-      console.log(`Flushing batch of up to ${batchSize} messages (${buffer.messageCount} remaining)`)
+      logger.debug('BufferManager.flushBuffer', { queueAddress, batchSize, remaining: buffer.messageCount })
       await this.#flushBufferBatch(queueAddress, batchSize)
     }
     
     // Wait for all pending flushes to complete
     await this.#waitForPendingFlushes()
     
-    console.log(`flushBuffer completed for ${queueAddress}`)
+    logger.debug('BufferManager.flushBuffer', { queueAddress, status: 'completed' })
   }
 
   async flushAllBuffers() {
+    // Get all queue addresses that have buffers
     const queueAddresses = Array.from(this.#buffers.keys())
     logger.log('BufferManager.flushAllBuffers', { bufferCount: queueAddresses.length, pendingFlushes: this.#pendingFlushes.size })
-    console.log(`flushAllBuffers called, pending flushes: ${this.#pendingFlushes.size}`)
-    
-    // Get all queue addresses that have buffers
     
     // Flush each buffer in batches
     for (const queueAddress of queueAddresses) {
@@ -172,15 +163,14 @@ export class BufferManager {
     }
     
     logger.log('BufferManager.flushAllBuffers', { status: 'completed' })
-    console.log(`flushAllBuffers completed`)
   }
 
   async #waitForPendingFlushes() {
     if (this.#pendingFlushes.size === 0) return
     
-    console.log(`Waiting for ${this.#pendingFlushes.size} pending flushes...`)
+    logger.debug('BufferManager.waitForPendingFlushes', { count: this.#pendingFlushes.size })
     await Promise.all(Array.from(this.#pendingFlushes))
-    console.log(`All pending flushes completed`)
+    logger.debug('BufferManager.waitForPendingFlushes', { status: 'completed' })
   }
 
   getStats() {
@@ -212,4 +202,3 @@ export class BufferManager {
     this.#buffers.clear()
   }
 }
-

package/client-v2/builders/QueueBuilder.js

@@ -306,7 +306,6 @@ export class QueueBuilder {
     } catch (error) {
       // Return empty array on error instead of throwing
       logger.error('QueueBuilder.pop', { error: error.message })
-      console.warn('Pop failed:', error.message)
       return []
     }
   }
@@ -745,7 +744,6 @@ class DLQBuilder {
       return result || { messages: [], total: 0 }
     } catch (error) {
       logger.error('DLQBuilder.get', { error: error.message })
-      console.warn('DLQ query failed:', error.message)
       return { messages: [], total: 0 }
     }
   }

package/client-v2/consumer/ConsumerManager.js

@@ -222,7 +222,6 @@ export class ConsumerManager {
 
         if (isNetworkError) {
           logger.warn('ConsumerManager.worker', { workerId, error: 'network', message: error.message })
-          console.warn(`Worker ${workerId}: Network error - ${error.message}`)
           // Wait before retry
           await new Promise(resolve => setTimeout(resolve, 1000))
           continue
@@ -296,7 +295,7 @@ export class ConsumerManager {
       try {
         await this.#queen.renew(messages)
       } catch (error) {
-        console.error('Lease renewal failed:', error)
+        logger.error('ConsumerManager.leaseRenewal', { error: error.message })
       }
     }, intervalMillis)
   }
@@ -367,7 +366,7 @@ export class ConsumerManager {
             error: error.message,
             phase: 'trace-failed'
           })
-          console.warn(`[TRACE FAILED] ${message.transactionId}: ${error.message}`)
+          logger.warn('ConsumerManager.trace', { transactionId: message.transactionId, error: error.message })
           
           return { success: false, error: error.message }
         }

package/client-v2/http/HttpClient.js

@@ -196,7 +196,6 @@ export class HttpClient {
         }
         
         logger.warn('HttpClient.failover', { url, method, path, error: error.message })
-        console.warn(`Request failed for ${url}: ${method} ${path} - ${error.message}`)
 
         // Don't retry on client errors (4xx)
         if (error.status && error.status >= 400 && error.status < 500) {

package/client-v2/utils/defaults.js

@@ -16,7 +16,8 @@ export const CLIENT_DEFAULTS = {
   healthRetryAfterMillis: 5000,        // Retry unhealthy backends after 5 seconds
   bearerToken: null,                   // Bearer token for proxy authentication
   headers: {},                         // Custom headers to include in every request
-  handleSignals: true                  // Register SIGINT/SIGTERM handlers (disable when used as a library)
+  handleSignals: true,                 // Register SIGINT/SIGTERM handlers (disable when used as a library)
+  logger: null                         // Custom logger instance (must implement info/warn/error)
 }
 
 export const QUEUE_DEFAULTS = {

package/client-v2/utils/logger.js

@@ -1,7 +1,10 @@
 /**
  * Logger utility for Queen Client v2
- * Controlled by QUEEN_CLIENT_LOG environment variable (Node.js)
- * or window.QUEEN_CLIENT_LOG (Browser)
+ * 
+ * Supports pluggable logger backends (pino, winston, bunyan, etc.).
+ * When no custom logger is configured, falls back to console-based logging
+ * gated by the QUEEN_CLIENT_LOG environment variable (Node.js)
+ * or window.QUEEN_CLIENT_LOG (Browser).
  */
 
 const LOG_ENABLED = (() => {
@@ -16,9 +19,8 @@ const LOG_ENABLED = (() => {
   return false
 })()
 
-/**
- * Get formatted timestamp
- */
+let customLogger = null
+
 function getTimestamp() {
   return new Date().toISOString()
 }
@@ -33,25 +35,56 @@ function formatLog(operation, details, level = 'INFO') {
 }
 
 /**
- * Log an operation
+ * Configure a custom logger backend.
+ * The logger must implement: info(msg), warn(msg), error(msg).
+ * debug(msg) is optional and falls back to info(msg) if missing.
+ * When a custom logger is set, it is always active (no env var gating).
+ * @param {object} logger - Logger instance (e.g. pino(), winston.createLogger())
  */
+export function configure(logger) {
+  if (logger && typeof logger.info !== 'function') {
+    throw new Error('Custom logger must implement info(), warn(), and error() methods')
+  }
+  customLogger = logger
+}
+
 export function log(operation, details) {
+  if (customLogger) {
+    const detailsStr = typeof details === 'object' ? JSON.stringify(details) : details
+    customLogger.info(`[${operation}] ${detailsStr}`)
+    return
+  }
   if (!LOG_ENABLED) return
   console.log(formatLog(operation, details))
 }
 
-/**
- * Log a warning
- */
+export function debug(operation, details) {
+  if (customLogger) {
+    const detailsStr = typeof details === 'object' ? JSON.stringify(details) : details
+    const fn = customLogger.debug || customLogger.info
+    fn.call(customLogger, `[${operation}] ${detailsStr}`)
+    return
+  }
+  if (!LOG_ENABLED) return
+  console.log(formatLog(operation, details, 'DEBUG'))
+}
+
 export function warn(operation, details) {
+  if (customLogger) {
+    const detailsStr = typeof details === 'object' ? JSON.stringify(details) : details
+    customLogger.warn(`[${operation}] ${detailsStr}`)
+    return
+  }
   if (!LOG_ENABLED) return
   console.warn(formatLog(operation, details, 'WARN'))
 }
 
-/**
- * Log an error
- */
 export function error(operation, details) {
+  if (customLogger) {
+    const detailsStr = typeof details === 'object' ? JSON.stringify(details) : details
+    customLogger.error(`[${operation}] ${detailsStr}`)
+    return
+  }
   if (!LOG_ENABLED) return
   console.error(formatLog(operation, details, 'ERROR'))
 }
@@ -60,6 +93,5 @@ export function error(operation, details) {
  * Check if logging is enabled
  */
 export function isEnabled() {
-  return LOG_ENABLED
+  return customLogger != null || LOG_ENABLED
 }
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "queen-mq",
-  "version": "0.12.3",
+  "version": "0.13.0",
   "type": "module",
   "description": "High-performance C++ message queue backed by PostgreSQL",
   "main": "client-v2/index.js",