queen-mq 0.2.0 → 0.2.22

package/src/routes/pop.js

@@ -1,69 +0,0 @@
-import config from '../config.js';
-
-export const createPopRoute = (queueManager, eventManager) => {
-  return async (scope, options = {}) => {
-    const { 
-      wait = false, 
-      timeout = config.QUEUE.DEFAULT_TIMEOUT, 
-      batch = config.QUEUE.DEFAULT_BATCH_SIZE,
-      consumerGroup = scope.consumerGroup || null,
-      subscriptionMode = scope.subscriptionMode || null,
-      subscriptionFrom = scope.subscriptionFrom || null
-    } = options;
-    const maxTimeout = config.QUEUE.MAX_TIMEOUT;
-    const effectiveTimeout = Math.min(timeout, maxTimeout);
-    
-    // ALWAYS USE UNIFIED POP - handles all 3 modes (direct, queue-level, filtered)
-    let result = await queueManager.uniquePop(scope, { 
-      batch, 
-      wait: false, 
-      timeout: effectiveTimeout, 
-      subscriptionMode, 
-      subscriptionFrom 
-    });
-    
-    if (result.messages.length > 0 || !wait) {
-      return result;
-    }
-    
-    // Long polling with event-driven optimization and exponential backoff
-    const queuePath = scope.partition 
-      ? `${scope.queue}/${scope.partition}`
-      : scope.queue;
-    
-    const startTime = Date.now();
-    const initialPollInterval = Math.min(config.QUEUE.POLL_INTERVAL, effectiveTimeout / 10);
-    const maxPollInterval = config.QUEUE.MAX_POLL_INTERVAL || 2000; // Default 2 seconds
-    const backoffThreshold = config.QUEUE.BACKOFF_THRESHOLD || 5; // Retries before backoff
-    const backoffMultiplier = config.QUEUE.BACKOFF_MULTIPLIER || 2;
-    
-    let currentPollInterval = initialPollInterval;
-    let consecutiveEmptyPolls = 0;
-    
-    while (Date.now() - startTime < effectiveTimeout) {
-      // Wait for message notification or current poll interval
-      const notification = await Promise.race([
-        eventManager.waitForMessage(queuePath, currentPollInterval),
-        new Promise(resolve => setTimeout(() => resolve(null), currentPollInterval))
-      ]);
-      
-      // Try to get messages whether notified or not
-      result = await queueManager.uniquePop(scope, { batch, wait: false, subscriptionMode, subscriptionFrom });
-      if (result.messages.length > 0) {
-        return result;
-      }
-      
-      // Implement exponential backoff after threshold
-      consecutiveEmptyPolls++;
-      if (consecutiveEmptyPolls >= backoffThreshold) {
-        currentPollInterval = Math.min(
-          currentPollInterval * backoffMultiplier,
-          maxPollInterval
-        );
-      }
-    }
-    
-    // Timeout reached, no messages
-    return { messages: [] };
-  };
-};

package/src/routes/push.js

@@ -1,28 +0,0 @@
-export const createPushRoute = (queueManager) => {
-  return async (body) => {
-    const { items = [], config = {} } = body;
-    
-    if (!items || items.length === 0) {
-      throw new Error('No items to push');
-    }
-    
-    // Validate items
-    for (const item of items) {
-      if (!item.queue) {
-        throw new Error('Each item must have a queue');
-      }
-      // Allow missing, null or undefined payloads - they'll be stored as null in JSONB
-      // If payload is not provided, set it to null
-      if (!('payload' in item)) {
-        item.payload = null;
-      }
-      // partition is optional, defaults to "Default"
-    }
-    
-    const results = await queueManager.pushMessages(items);
-    
-    return {
-      messages: results
-    };
-  };
-};

package/src/routes/resources.js

@@ -1,330 +0,0 @@
-import { EventTypes } from '../managers/systemEventManager.js';
-
-export const createResourcesRoutes = (pool, systemEventManager) => {
-  
-  // Get all queues with their partitions
-  const getQueues = async (filters = {}) => {
-    const { namespace, task } = filters;
-    
-    let query = `
-      SELECT DISTINCT
-        q.id as queue_id,
-        q.name as queue_name,
-        q.namespace,
-        q.task,
-        q.created_at as queue_created_at,
-        COUNT(DISTINCT p.id) as partition_count,
-        COUNT(DISTINCT m.id) as message_count,
-        COALESCE(SUM(DISTINCT pc.pending_estimate), 0) as pending_count,
-        COUNT(DISTINCT CASE WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN p.id END) as processing_count
-      FROM queen.queues q
-      LEFT JOIN queen.partitions p ON p.queue_id = q.id
-      LEFT JOIN queen.messages m ON m.partition_id = p.id
-      LEFT JOIN queen.partition_consumers pc ON pc.partition_id = p.id AND pc.consumer_group = '__QUEUE_MODE__'
-      WHERE 1=1
-    `;
-    
-    const params = [];
-    if (namespace) {
-      params.push(namespace);
-      query += ` AND q.namespace = $${params.length}`;
-    }
-    if (task) {
-      params.push(task);
-      query += ` AND q.task = $${params.length}`;
-    }
-    
-    query += ` GROUP BY q.id, q.name, q.namespace, q.task, q.created_at
-               ORDER BY q.created_at DESC`;
-    
-    const result = await pool.query(query, params);
-    
-    return result.rows.map(row => ({
-      id: row.queue_id,
-      name: row.queue_name,
-      namespace: row.namespace,
-      task: row.task,
-      createdAt: row.queue_created_at,
-      partitions: parseInt(row.partition_count),
-      messages: {
-        total: parseInt(row.message_count),
-        pending: parseInt(row.pending_count),
-        processing: parseInt(row.processing_count)
-      }
-    }));
-  };
-  
-  // Get single queue with detailed partition information
-  const getQueue = async (queueName) => {
-    // Get queue info
-    const queueResult = await pool.query(`
-      SELECT 
-        q.id,
-        q.name,
-        q.namespace,
-        q.task,
-        q.created_at
-      FROM queen.queues q
-      WHERE q.name = $1
-    `, [queueName]);
-    
-    if (queueResult.rows.length === 0) {
-      throw new Error('Queue not found');
-    }
-    
-    const queue = queueResult.rows[0];
-    
-    // Get partitions with stats using partition_consumers
-    const partitionsResult = await pool.query(`
-      SELECT 
-        p.id,
-        p.name,
-        p.created_at,
-        COUNT(DISTINCT m.id) as message_count,
-        COALESCE(pc.pending_estimate, 0) as pending,
-        CASE WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN 1 ELSE 0 END as processing,
-        COALESCE(pc.total_messages_consumed, 0) as completed,
-        0 as failed,
-        (SELECT COUNT(*) FROM queen.dead_letter_queue dlq 
-         WHERE dlq.partition_id = p.id 
-         AND (dlq.consumer_group IS NULL OR dlq.consumer_group = '__QUEUE_MODE__')) as dead_letter,
-        MIN(m.created_at) as oldest_message,
-        MAX(m.created_at) as newest_message
-      FROM queen.partitions p
-      LEFT JOIN queen.messages m ON m.partition_id = p.id
-      LEFT JOIN queen.partition_consumers pc ON pc.partition_id = p.id AND pc.consumer_group = '__QUEUE_MODE__'
-      WHERE p.queue_id = $1
-      GROUP BY p.id, p.name, p.created_at, pc.pending_estimate, pc.lease_expires_at, pc.total_messages_consumed
-      ORDER BY p.name
-    `, [queue.id]);
-    
-    const partitions = partitionsResult.rows.map(row => ({
-      id: row.id,
-      name: row.name,
-      createdAt: row.created_at,
-      stats: {
-        total: parseInt(row.message_count),
-        pending: parseInt(row.pending),
-        processing: parseInt(row.processing),
-        completed: parseInt(row.completed),
-        failed: parseInt(row.failed),
-        deadLetter: parseInt(row.dead_letter)
-      },
-      oldestMessage: row.oldest_message,
-      newestMessage: row.newest_message
-    }));
-    
-    // Calculate totals
-    const totals = partitions.reduce((acc, p) => ({
-      total: acc.total + p.stats.total,
-      pending: acc.pending + p.stats.pending,
-      processing: acc.processing + p.stats.processing,
-      completed: acc.completed + p.stats.completed,
-      failed: acc.failed + p.stats.failed,
-      deadLetter: acc.deadLetter + p.stats.deadLetter
-    }), {
-      total: 0,
-      pending: 0,
-      processing: 0,
-      completed: 0,
-      failed: 0,
-      deadLetter: 0
-    });
-    
-    return {
-      id: queue.id,
-      name: queue.name,
-      namespace: queue.namespace,
-      task: queue.task,
-      createdAt: queue.created_at,
-      partitions,
-      totals
-    };
-  };
-  
-  // Get all partitions across all queues
-  const getPartitions = async (filters = {}) => {
-    const { queue, minDepth } = filters;
-    
-    let query = `
-      SELECT 
-        p.id,
-        p.name as partition_name,
-        p.created_at,
-        q.name as queue_name,
-        q.namespace,
-        q.task,
-        q.priority,
-        COALESCE(pc.pending_estimate, 0) as pending,
-        CASE WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN 1 ELSE 0 END as processing,
-        COUNT(DISTINCT m.id) as total
-      FROM queen.partitions p
-      JOIN queen.queues q ON q.id = p.queue_id
-      LEFT JOIN queen.messages m ON m.partition_id = p.id
-      LEFT JOIN queen.partition_consumers pc ON pc.partition_id = p.id AND pc.consumer_group = '__QUEUE_MODE__'
-      WHERE 1=1
-    `;
-    
-    const params = [];
-    if (queue) {
-      params.push(queue);
-      query += ` AND q.name = $${params.length}`;
-    }
-    
-    query += ` GROUP BY p.id, p.name, p.created_at, 
-                        q.name, q.namespace, q.task, q.priority, pc.pending_estimate, pc.lease_expires_at`;
-    
-    if (minDepth) {
-      params.push(minDepth);
-      query += ` HAVING COALESCE(pc.pending_estimate, 0) >= $${params.length}`;
-    }
-    
-    query += ` ORDER BY q.name, q.priority DESC, p.name`;
-    
-    const result = await pool.query(query, params);
-    
-    return result.rows.map(row => ({
-      id: row.id,
-      name: row.partition_name,
-      queue: row.queue_name,
-      namespace: row.namespace,
-      task: row.task,
-      queuePriority: row.priority, // Priority is now at queue level
-      createdAt: row.created_at,
-      depth: parseInt(row.pending),
-      processing: parseInt(row.processing),
-      total: parseInt(row.total)
-    }));
-  };
-  
-  // Get namespace/task groupings
-  const getNamespaces = async () => {
-    const result = await pool.query(`
-      SELECT DISTINCT
-        q.namespace,
-        COUNT(DISTINCT q.id) as queue_count,
-        COUNT(DISTINCT p.id) as partition_count,
-        COUNT(DISTINCT m.id) as message_count,
-        COALESCE(SUM(pc.pending_estimate), 0) as pending_count
-      FROM queen.queues q
-      LEFT JOIN queen.partitions p ON p.queue_id = q.id
-      LEFT JOIN queen.messages m ON m.partition_id = p.id
-      LEFT JOIN queen.partition_consumers pc ON pc.partition_id = p.id AND pc.consumer_group = '__QUEUE_MODE__'
-      WHERE q.namespace IS NOT NULL
-      GROUP BY q.namespace
-      ORDER BY q.namespace
-    `);
-    
-    return result.rows.map(row => ({
-      namespace: row.namespace,
-      queues: parseInt(row.queue_count),
-      partitions: parseInt(row.partition_count),
-      messages: {
-        total: parseInt(row.message_count),
-        pending: parseInt(row.pending_count)
-      }
-    }));
-  };
-  
-  const getTasks = async () => {
-    const result = await pool.query(`
-      SELECT DISTINCT
-        q.task,
-        COUNT(DISTINCT q.id) as queue_count,
-        COUNT(DISTINCT p.id) as partition_count,
-        COUNT(DISTINCT m.id) as message_count,
-        COALESCE(SUM(pc.pending_estimate), 0) as pending_count
-      FROM queen.queues q
-      LEFT JOIN queen.partitions p ON p.queue_id = q.id
-      LEFT JOIN queen.messages m ON m.partition_id = p.id
-      LEFT JOIN queen.partition_consumers pc ON pc.partition_id = p.id AND pc.consumer_group = '__QUEUE_MODE__'
-      WHERE q.task IS NOT NULL
-      GROUP BY q.task
-      ORDER BY q.task
-    `);
-    
-    return result.rows.map(row => ({
-      task: row.task,
-      queues: parseInt(row.queue_count),
-      partitions: parseInt(row.partition_count),
-      messages: {
-        total: parseInt(row.message_count),
-        pending: parseInt(row.pending_count)
-      }
-    }));
-  };
-  
-  // Get system overview
-  const getSystemOverview = async () => {
-    const overview = await pool.query(`
-      SELECT
-        (SELECT COUNT(*) FROM queen.queues) as total_queues,
-        (SELECT COUNT(*) FROM queen.partitions) as total_partitions,
-        (SELECT COUNT(*) FROM queen.messages) as total_messages,
-        (SELECT COALESCE(SUM(pending_estimate), 0)::integer FROM queen.partition_consumers 
-         WHERE consumer_group = '__QUEUE_MODE__') as pending_messages,
-        (SELECT COUNT(*) FROM queen.partition_consumers 
-         WHERE consumer_group = '__QUEUE_MODE__' 
-         AND lease_expires_at IS NOT NULL 
-         AND lease_expires_at > NOW()) as processing_messages,
-        (SELECT COALESCE(SUM(total_messages_consumed), 0)::integer FROM queen.partition_consumers 
-         WHERE consumer_group = '__QUEUE_MODE__') as completed_messages,
-        0 as failed_messages,
-        (SELECT COUNT(*) FROM queen.dead_letter_queue WHERE consumer_group IS NULL OR consumer_group = '__QUEUE_MODE__') as dead_letter_messages,
-        (SELECT COUNT(DISTINCT namespace) FROM queen.queues WHERE namespace IS NOT NULL) as namespaces,
-        (SELECT COUNT(DISTINCT task) FROM queen.queues WHERE task IS NOT NULL) as tasks
-    `);
-    
-    const row = overview.rows[0];
-    
-    return {
-      queues: parseInt(row.total_queues),
-      partitions: parseInt(row.total_partitions),
-      namespaces: parseInt(row.namespaces),
-      tasks: parseInt(row.tasks),
-      messages: {
-        total: parseInt(row.total_messages),
-        pending: parseInt(row.pending_messages),
-        processing: parseInt(row.processing_messages),
-        completed: parseInt(row.completed_messages),
-        failed: parseInt(row.failed_messages),
-        deadLetter: parseInt(row.dead_letter_messages)
-      },
-      timestamp: new Date().toISOString()
-    };
-  };
-  
-  // Delete a queue (cascades to partitions, messages, etc.)
-  // Idempotent: returns success even if queue doesn't exist
-  const deleteQueue = async (queueName) => {
-    const result = await pool.query(
-      `DELETE FROM queen.queues WHERE name = $1 RETURNING id, name`,
-      [queueName]
-    );
-    
-    // Emit system event for cache invalidation across nodes
-    if (result.rows.length > 0 && systemEventManager) {
-      await systemEventManager.emit(EventTypes.QUEUE_DELETED, {
-        entityId: queueName,
-        queueId: result.rows[0].id
-      });
-    }
-    
-    return {
-      deleted: true,
-      queue: queueName,
-      existed: result.rows.length > 0,
-      id: result.rows.length > 0 ? result.rows[0].id : null
-    };
-  };
-  
-  return {
-    getQueues,
-    getQueue,
-    getPartitions,
-    getNamespaces,
-    getTasks,
-    getSystemOverview,
-    deleteQueue
-  };
-};