queen-mq 0.3.0 → 0.4.0

package/client-js/client/index.js

@@ -1,4 +0,0 @@
-export { LoadBalancingStrategy } from './utils/loadBalancer.js';
-
-// New minimalist interface
-export { Queen } from './client.js';

package/client-js/client/utils/http.js

@@ -1,173 +0,0 @@
-export const httpRequest = async (url, options = {}) => {
-  //console.log('httpRequest', url, options);
-  const response = await fetch(url, {
-    ...options,
-    headers: {
-      'Content-Type': 'application/json',
-      ...options.headers
-    }
-  });
-  //console.log('response', response);
-  // Handle 204 No Content
-  if (response.status === 204) {
-    return null; // No content
-  }
-  
-  // Handle errors
-  if (!response.ok) {
-    const error = new Error(`HTTP ${response.status}: ${response.statusText}`);
-    error.status = response.status;
-    
-    try {
-      const text = await response.text();
-      if (text) {
-        const body = JSON.parse(text);
-        error.message = body.error || error.message;
-      }
-    } catch (e) {
-      // Ignore JSON parse errors
-    }
-    
-    throw error;
-  }
-  
-  // Handle successful responses
-  const contentType = response.headers.get('content-type');
-  const contentLength = response.headers.get('content-length');
-  
-  // Check if there's actually content to parse
-  if (!contentType || !contentType.includes('application/json') || contentLength === '0') {
-    const text = await response.text();
-    if (!text || text.length === 0) {
-      return null; // Empty response
-    }
-    try {
-      return JSON.parse(text);
-    } catch (e) {
-      console.warn('Failed to parse response as JSON:', text);
-      return null;
-    }
-  }
-  
-  return response.json();
-};
-
-export const createHttpClient = ({ baseUrl, timeout = 30000 }) => {
-  const request = (method, path, body = null, requestTimeout = null) => {
-    const effectiveTimeout = requestTimeout || timeout;
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), effectiveTimeout);
-    
-    const options = {
-      method,
-      signal: controller.signal
-    };
-    
-    if (body) {
-      options.body = JSON.stringify(body);
-    }
-    
-    return httpRequest(`${baseUrl}${path}`, options)
-      .catch(error => {
-        // Enhance abort errors with more context
-        if (error.name === 'AbortError') {
-          const timeoutError = new Error(`Request timeout after ${effectiveTimeout}ms`);
-          timeoutError.name = 'AbortError';
-          timeoutError.timeout = effectiveTimeout;
-          throw timeoutError;
-        }
-        throw error;
-      })
-      .finally(() => clearTimeout(timeoutId));
-  };
-  
-  return {
-    get: (path, requestTimeout) => request('GET', path, null, requestTimeout),
-    post: (path, body, requestTimeout) => request('POST', path, body, requestTimeout),
-    put: (path, body, requestTimeout) => request('PUT', path, body, requestTimeout),
-    delete: (path, requestTimeout) => request('DELETE', path, null, requestTimeout)
-  };
-};
-
-/**
- * Create an HTTP client with load balancing support
- */
-export const createLoadBalancedHttpClient = ({ baseUrls, loadBalancer, timeout = 30000, enableFailover = true }) => {
-  const request = async (method, path, body = null, requestTimeout = null) => {
-    const effectiveTimeout = requestTimeout || timeout;
-    
-    const executeRequest = async (baseUrl) => {
-      const controller = new AbortController();
-      const timeoutId = setTimeout(() => controller.abort(), effectiveTimeout);
-      
-      const options = {
-        method,
-        signal: controller.signal
-      };
-      
-      if (body) {
-        options.body = JSON.stringify(body);
-      }
-      
-      try {
-        return await httpRequest(`${baseUrl}${path}`, options);
-      } catch (error) {
-        // Enhance abort errors with more context
-        if (error.name === 'AbortError') {
-          const timeoutError = new Error(`Request timeout after ${effectiveTimeout}ms`);
-          timeoutError.name = 'AbortError';
-          timeoutError.timeout = effectiveTimeout;
-          throw timeoutError;
-        }
-        throw error;
-      } finally {
-        clearTimeout(timeoutId);
-      }
-    };
-    
-    if (enableFailover) {
-      // Try with failover to different servers
-      const attemptedUrls = new Set();
-      const urls = loadBalancer.getAllUrls();
-      let lastError = null;
-      
-      for (let attempt = 0; attempt < urls.length; attempt++) {
-        const url = loadBalancer.getNextUrl();
-        
-        if (attemptedUrls.has(url)) {
-          continue;
-        }
-        
-        attemptedUrls.add(url);
-        
-        try {
-          return await executeRequest(url);
-        } catch (error) {
-          lastError = error;
-          console.warn(`Request failed for ${url}: ${method} ${path} - ${error.message}`);
-          
-          // Don't retry on client errors (4xx)
-          if (error.status && error.status >= 400 && error.status < 500) {
-            throw error;
-          }
-          
-          // Continue to next server for server errors or network issues
-        }
-      }
-      
-      throw lastError || new Error('All servers failed');
-    } else {
-      // No failover, just use the next URL from load balancer
-      const baseUrl = loadBalancer.getNextUrl();
-      return executeRequest(baseUrl);
-    }
-  };
-  
-  return {
-    get: (path, requestTimeout) => request('GET', path, null, requestTimeout),
-    post: (path, body, requestTimeout) => request('POST', path, body, requestTimeout),
-    put: (path, body, requestTimeout) => request('PUT', path, body, requestTimeout),
-    delete: (path, requestTimeout) => request('DELETE', path, null, requestTimeout),
-    getLoadBalancer: () => loadBalancer
-  };
-};

package/client-js/client/utils/loadBalancer.js

@@ -1,152 +0,0 @@
-/**
- * Load balancer utility for distributing requests across multiple servers
- */
-
-export const LoadBalancingStrategy = {
-  ROUND_ROBIN: 'ROUND_ROBIN',
-  SESSION: 'SESSION'
-};
-
-/**
- * Create a load balancer for distributing requests across multiple base URLs
- */
-export const createLoadBalancer = (urls, strategy = LoadBalancingStrategy.ROUND_ROBIN) => {
-  if (!Array.isArray(urls) || urls.length === 0) {
-    throw new Error('URLs must be a non-empty array');
-  }
-  
-  // Normalize URLs (remove trailing slashes)
-  const normalizedUrls = urls.map(url => url.replace(/\/$/, ''));
-  
-  // Round robin state
-  let currentIndex = 0;
-  
-  // Session affinity state - map of session ID to URL index
-  const sessionMap = new Map();
-  
-  // Generate a session ID for the current client instance
-  const sessionId = generateSessionId();
-  
-  /**
-   * Get the next URL based on the configured strategy
-   */
-  const getNextUrl = (options = {}) => {
-    const { sessionKey = sessionId } = options;
-    
-    switch (strategy) {
-      case LoadBalancingStrategy.ROUND_ROBIN:
-        // Round robin: cycle through URLs
-        const url = normalizedUrls[currentIndex];
-        currentIndex = (currentIndex + 1) % normalizedUrls.length;
-        return url;
-        
-      case LoadBalancingStrategy.SESSION:
-        // Session affinity: stick to the same server per session
-        if (!sessionMap.has(sessionKey)) {
-          // Assign a server to this session (using round robin for initial assignment)
-          const assignedIndex = currentIndex;
-          sessionMap.set(sessionKey, assignedIndex);
-          currentIndex = (currentIndex + 1) % normalizedUrls.length;
-        }
-        return normalizedUrls[sessionMap.get(sessionKey)];
-        
-      default:
-        throw new Error(`Unknown load balancing strategy: ${strategy}`);
-    }
-  };
-  
-  /**
-   * Get all configured URLs
-   */
-  const getAllUrls = () => [...normalizedUrls];
-  
-  /**
-   * Get current strategy
-   */
-  const getStrategy = () => strategy;
-  
-  /**
-   * Reset the load balancer state
-   */
-  const reset = () => {
-    currentIndex = 0;
-    sessionMap.clear();
-  };
-  
-  /**
-   * Get statistics about URL usage (for debugging)
-   */
-  const getStats = () => {
-    return {
-      strategy,
-      urls: normalizedUrls,
-      currentIndex,
-      sessionCount: sessionMap.size,
-      sessions: Array.from(sessionMap.entries()).map(([key, index]) => ({
-        sessionKey: key,
-        url: normalizedUrls[index]
-      }))
-    };
-  };
-  
-  return {
-    getNextUrl,
-    getAllUrls,
-    getStrategy,
-    reset,
-    getStats
-  };
-};
-
-/**
- * Generate a unique session ID
- */
-const generateSessionId = () => {
-  return `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-};
-
-/**
- * Create a failover wrapper that tries different servers on failure
- */
-export const createFailoverWrapper = (loadBalancer, maxRetries = 3) => {
-  const attemptedUrls = new Set();
-  
-  const executeWithFailover = async (requestFn, options = {}) => {
-    const urls = loadBalancer.getAllUrls();
-    let lastError = null;
-    
-    // Try each URL up to maxRetries times total
-    for (let attempt = 0; attempt < Math.min(maxRetries, urls.length); attempt++) {
-      const url = loadBalancer.getNextUrl(options);
-      
-      // Skip if we've already tried this URL in this request
-      if (attemptedUrls.has(url)) {
-        continue;
-      }
-      
-      attemptedUrls.add(url);
-      
-      try {
-        const result = await requestFn(url);
-        attemptedUrls.clear(); // Clear for next request
-        return result;
-      } catch (error) {
-        lastError = error;
-        console.warn(`Request failed for ${url}, attempting next server...`, error.message);
-        
-        // If it's a client error (4xx), don't retry on other servers
-        if (error.status && error.status >= 400 && error.status < 500) {
-          attemptedUrls.clear();
-          throw error;
-        }
-      }
-    }
-    
-    attemptedUrls.clear();
-    throw lastError || new Error('All servers failed');
-  };
-  
-  return {
-    executeWithFailover
-  };
-};

package/client-js/client/utils/retry.js

@@ -1,41 +0,0 @@
-const config = {
-  CLIENT: {
-    DEFAULT_RETRY_ATTEMPTS: 3,
-    DEFAULT_RETRY_DELAY: 1000,
-    DEFAULT_RETRY_BACKOFF: 2 
-  }
-}
-
-export const withRetry = async (fn, attempts = config.CLIENT.DEFAULT_RETRY_ATTEMPTS, delay = config.CLIENT.DEFAULT_RETRY_DELAY, backoff = config.CLIENT.DEFAULT_RETRY_BACKOFF) => {
-  let lastError;
-  
-  for (let i = 0; i < attempts; i++) {
-    try {
-      return await fn();
-    } catch (error) {
-      lastError = error;
-      
-      // Don't retry on client errors (4xx)
-      if (error.status && error.status >= 400 && error.status < 500) {
-        throw error;
-      }
-      
-      if (i < attempts - 1) {
-        const waitTime = delay * Math.pow(backoff, i);
-        await new Promise(resolve => setTimeout(resolve, waitTime));
-      }
-    }
-  }
-  
-  throw lastError;
-};
-
-export const createRetryableFunction = (fn, options = {}) => {
-  const {
-    attempts = config.CLIENT.DEFAULT_RETRY_ATTEMPTS,
-    delay = config.CLIENT.DEFAULT_RETRY_DELAY,
-    backoff = config.CLIENT.DEFAULT_RETRY_BACKOFF
-  } = options;
-  
-  return (...args) => withRetry(() => fn(...args), attempts, delay, backoff);
-};

package/client-js/services/encryptionService.js

@@ -1,82 +0,0 @@
-import crypto from 'crypto';
-import { log } from '../utils/logger.js';
-
-const ALGORITHM = 'aes-256-gcm';
-const KEY_ENV = 'QUEEN_ENCRYPTION_KEY';
-
-// Get and validate the encryption key
-const getKey = () => {
-  const keyHex = process.env[KEY_ENV];
-  if (!keyHex) return null;
-  
-  try {
-    const key = Buffer.from(keyHex, 'hex');
-    if (key.length !== 32) {
-      log(`ERROR: ${KEY_ENV} must be 32 bytes (64 hex characters)`);
-      return null;
-    }
-    return key;
-  } catch (error) {
-    log(`ERROR: Invalid ${KEY_ENV} format:`, error.message);
-    return null;
-  }
-};
-
-// Encrypt a payload
-export const encryptPayload = async (payload) => {
-  const key = getKey();
-  if (!key) throw new Error('Encryption key not configured');
-  
-  const iv = crypto.randomBytes(16);
-  const cipher = crypto.createCipheriv(ALGORITHM, key, iv);
-  
-  const payloadStr = JSON.stringify(payload);
-  const encrypted = Buffer.concat([
-    cipher.update(payloadStr, 'utf8'),
-    cipher.final()
-  ]);
-  
-  return {
-    encrypted: encrypted.toString('base64'),
-    iv: iv.toString('base64'),
-    authTag: cipher.getAuthTag().toString('base64')
-  };
-};
-
-// Decrypt a payload
-export const decryptPayload = async (encryptedData) => {
-  const key = getKey();
-  if (!key) throw new Error('Encryption key not configured');
-  
-  const decipher = crypto.createDecipheriv(
-    ALGORITHM,
-    key,
-    Buffer.from(encryptedData.iv, 'base64')
-  );
-  
-  decipher.setAuthTag(Buffer.from(encryptedData.authTag, 'base64'));
-  
-  const decrypted = Buffer.concat([
-    decipher.update(Buffer.from(encryptedData.encrypted, 'base64')),
-    decipher.final()
-  ]);
-  
-  return JSON.parse(decrypted.toString('utf8'));
-};
-
-// Check if encryption is available
-export const isEncryptionEnabled = () => {
-  const key = getKey();
-  return key !== null;
-};
-
-// Initialize and log status
-export const initEncryption = () => {
-  const enabled = isEncryptionEnabled();
-  if (enabled) {
-    log('✅ Encryption service initialized');
-  } else {
-    log('⚠️  Encryption service disabled (QUEEN_ENCRYPTION_KEY not set)');
-  }
-  return enabled;
-};

package/client-js/services/evictionService.js

@@ -1,160 +0,0 @@
-// Functional eviction service for stale messages
-import { log, LogTypes } from '../utils/logger.js';
-import config from '../config.js';
-
-const EVICTION_INTERVAL = config.JOBS.EVICTION_INTERVAL;
-
-// Evict messages that exceeded max wait time for a queue
-// Note: Eviction is already handled by POP query filtering old messages
-// This service optionally moves them to DLQ for visibility
-const evictMessages = async (client, queueName, maxWaitTimeSeconds) => {
-  if (!maxWaitTimeSeconds || maxWaitTimeSeconds <= 0) return 0;
-  
-  try {
-    // Find old messages that haven't been consumed and move to DLQ
-    // Use a WHERE EXISTS check to ensure message still exists before inserting into DLQ
-    const result = await client.query(`
-      WITH queue_partitions AS (
-        SELECT p.id, p.queue_id
-        FROM queen.partitions p
-        JOIN queen.queues q ON p.queue_id = q.id
-        WHERE q.name = $1
-      ),
-      old_unconsumed_messages AS (
-        SELECT m.id, m.partition_id, m.created_at
-        FROM queen.messages m
-        JOIN queue_partitions qp ON m.partition_id = qp.id
-        LEFT JOIN queen.partition_consumers pc ON pc.partition_id = m.partition_id
-          AND pc.consumer_group = '__QUEUE_MODE__'
-        WHERE m.created_at < NOW() - INTERVAL '1 second' * $2
-          -- Message has not been consumed yet
-          AND ((m.created_at, m.id) > (COALESCE(pc.last_consumed_created_at, '1970-01-01'::timestamptz),
-                                         COALESCE(pc.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid))
-               OR pc.last_consumed_id IS NULL)
-      )
-      INSERT INTO queen.dead_letter_queue (
-          message_id, 
-          partition_id, 
-          consumer_group, 
-          error_message, 
-          original_created_at
-      )
-      SELECT 
-          om.id,
-          om.partition_id,
-          '__QUEUE_MODE__',
-          'Message exceeded maximum wait time',
-          om.created_at
-      FROM old_unconsumed_messages om
-      WHERE EXISTS (
-          SELECT 1 FROM queen.messages m WHERE m.id = om.id
-      )
-      ON CONFLICT DO NOTHING
-      RETURNING message_id
-    `, [queueName, maxWaitTimeSeconds]);
-    
-    return result.rowCount || 0;
-  } catch (error) {
-    // Catch foreign key violations (messages deleted between SELECT and INSERT)
-    if (error.code === '23503') {
-      // Foreign key constraint violation - message was deleted
-      // This is a harmless race condition, just log it
-      log(`${LogTypes.EVICTION} | Queue: ${queueName} | Warning: Message deleted during eviction (race condition)`);
-      return 0;
-    }
-    // Re-throw other errors
-    throw error;
-  }
-};
-
-// Evict messages during pop operation (inline eviction)
-export const evictOnPop = async (client, queueName) => {
-  const queueResult = await client.query(
-    'SELECT max_wait_time_seconds FROM queen.queues WHERE name = $1',
-    [queueName]
-  );
-  
-  if (!queueResult.rows[0]) return 0;
-  
-  const maxWaitTimeSeconds = queueResult.rows[0].max_wait_time_seconds;
-  const evictedCount = await evictMessages(client, queueName, maxWaitTimeSeconds);
-  
-  if (evictedCount > 0) {
-    log(`${LogTypes.EVICTION} | Queue: ${queueName} | Count: ${evictedCount} | MaxWaitTime: ${maxWaitTimeSeconds}s`);
-    
-    // Log to retention history (reusing the table)
-    await client.query(`
-      INSERT INTO queen.retention_history (messages_deleted, retention_type)
-      VALUES ($1, 'evicted')
-    `, [evictedCount]);
-  }
-  
-  return evictedCount;
-};
-
-// Background eviction for all queues
-const performEviction = async (pool) => {
-  const client = await pool.connect();
-  
-  try {
-    // Get all queues with eviction configured
-    const queuesResult = await client.query(`
-      SELECT name, max_wait_time_seconds
-      FROM queen.queues
-      WHERE max_wait_time_seconds > 0
-    `);
-    
-    let totalEvicted = 0;
-    for (const queue of queuesResult.rows) {
-      const evicted = await evictMessages(
-        client, 
-        queue.name, 
-        queue.max_wait_time_seconds
-      );
-      
-      if (evicted > 0) {
-        log(`${LogTypes.EVICTION} | Queue: ${queue.name} | Count: ${evicted} | MaxWaitTime: ${queue.max_wait_time_seconds}s`);
-        totalEvicted += evicted;
-      }
-    }
-    
-    if (totalEvicted > 0) {
-      await client.query(`
-        INSERT INTO queen.retention_history (messages_deleted, retention_type)
-        VALUES ($1, 'evicted')
-      `, [totalEvicted]);
-    }
-    
-    return totalEvicted;
-  } catch (error) {
-    log('Eviction error:', error);
-    return 0;
-  } finally {
-    client.release();
-  }
-};
-
-// Start the eviction job (optional background job)
-export const startEvictionJob = (pool, eventManager) => {
-  const intervalId = setInterval(async () => {
-    try {
-      const evicted = await performEviction(pool);
-      if (evicted > 0 && eventManager) {
-        eventManager.emit('messages:evicted', { count: evicted });
-      }
-    } catch (error) {
-      log('Eviction job error:', error);
-    }
-  }, EVICTION_INTERVAL);
-  
-  log(`⏰ Eviction job started (interval: ${EVICTION_INTERVAL}ms)`);
-  
-  // Return cleanup function
-  return () => {
-    clearInterval(intervalId);
-    log('Eviction job stopped');
-  };
-};
-
-// Export for manual execution
-export const runEviction = performEviction;