queen-mq 0.1.0

package/src/services/encryptionService.js

@@ -0,0 +1,82 @@
+import crypto from 'crypto';
+import { log } from '../utils/logger.js';
+
+const ALGORITHM = 'aes-256-gcm';
+const KEY_ENV = 'QUEEN_ENCRYPTION_KEY';
+
+// Get and validate the encryption key
+const getKey = () => {
+  const keyHex = process.env[KEY_ENV];
+  if (!keyHex) return null;
+  
+  try {
+    const key = Buffer.from(keyHex, 'hex');
+    if (key.length !== 32) {
+      log(`ERROR: ${KEY_ENV} must be 32 bytes (64 hex characters)`);
+      return null;
+    }
+    return key;
+  } catch (error) {
+    log(`ERROR: Invalid ${KEY_ENV} format:`, error.message);
+    return null;
+  }
+};
+
+// Encrypt a payload
+export const encryptPayload = async (payload) => {
+  const key = getKey();
+  if (!key) throw new Error('Encryption key not configured');
+  
+  const iv = crypto.randomBytes(16);
+  const cipher = crypto.createCipheriv(ALGORITHM, key, iv);
+  
+  const payloadStr = JSON.stringify(payload);
+  const encrypted = Buffer.concat([
+    cipher.update(payloadStr, 'utf8'),
+    cipher.final()
+  ]);
+  
+  return {
+    encrypted: encrypted.toString('base64'),
+    iv: iv.toString('base64'),
+    authTag: cipher.getAuthTag().toString('base64')
+  };
+};
+
+// Decrypt a payload
+export const decryptPayload = async (encryptedData) => {
+  const key = getKey();
+  if (!key) throw new Error('Encryption key not configured');
+  
+  const decipher = crypto.createDecipheriv(
+    ALGORITHM,
+    key,
+    Buffer.from(encryptedData.iv, 'base64')
+  );
+  
+  decipher.setAuthTag(Buffer.from(encryptedData.authTag, 'base64'));
+  
+  const decrypted = Buffer.concat([
+    decipher.update(Buffer.from(encryptedData.encrypted, 'base64')),
+    decipher.final()
+  ]);
+  
+  return JSON.parse(decrypted.toString('utf8'));
+};
+
+// Check if encryption is available
+export const isEncryptionEnabled = () => {
+  const key = getKey();
+  return key !== null;
+};
+
+// Initialize and log status
+export const initEncryption = () => {
+  const enabled = isEncryptionEnabled();
+  if (enabled) {
+    log('✅ Encryption service initialized');
+  } else {
+    log('⚠️  Encryption service disabled (QUEEN_ENCRYPTION_KEY not set)');
+  }
+  return enabled;
+};

package/src/services/evictionService.js

@@ -0,0 +1,131 @@
+// Functional eviction service for stale messages
+import { log, LogTypes } from '../utils/logger.js';
+import config from '../config.js';
+
+const EVICTION_INTERVAL = config.JOBS.EVICTION_INTERVAL;
+
+// Evict messages that exceeded max wait time for a queue
+const evictMessages = async (client, queueName, maxWaitTimeSeconds) => {
+  if (!maxWaitTimeSeconds || maxWaitTimeSeconds <= 0) return 0;
+  
+  // For the new schema, we need to update or create status entries for evicted messages
+  const result = await client.query(`
+    WITH queue_partitions AS (
+      SELECT p.id
+      FROM queen.partitions p
+      JOIN queen.queues q ON p.queue_id = q.id
+      WHERE q.name = $1
+    ),
+    eligible_messages AS (
+      SELECT m.id
+      FROM queen.messages m
+      JOIN queue_partitions qp ON m.partition_id = qp.id
+      LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group IS NULL
+      WHERE m.created_at < NOW() - INTERVAL '1 second' * $2
+        AND (ms.status = 'pending' OR ms.status IS NULL)
+    )
+    INSERT INTO queen.messages_status (message_id, consumer_group, status, completed_at, error_message)
+    SELECT id, NULL, 'evicted', NOW(), 'Message exceeded maximum wait time'
+    FROM eligible_messages
+    ON CONFLICT (message_id, consumer_group) 
+    DO UPDATE SET 
+      status = 'evicted',
+      completed_at = NOW(),
+      error_message = 'Message exceeded maximum wait time'
+    RETURNING message_id
+  `, [queueName, maxWaitTimeSeconds]);
+  
+  return result.rowCount || 0;
+};
+
+// Evict messages during pop operation (inline eviction)
+export const evictOnPop = async (client, queueName) => {
+  const queueResult = await client.query(
+    'SELECT max_wait_time_seconds FROM queen.queues WHERE name = $1',
+    [queueName]
+  );
+  
+  if (!queueResult.rows[0]) return 0;
+  
+  const maxWaitTimeSeconds = queueResult.rows[0].max_wait_time_seconds;
+  const evictedCount = await evictMessages(client, queueName, maxWaitTimeSeconds);
+  
+  if (evictedCount > 0) {
+    log(`${LogTypes.EVICTION} | Queue: ${queueName} | Count: ${evictedCount} | MaxWaitTime: ${maxWaitTimeSeconds}s`);
+    
+    // Log to retention history (reusing the table)
+    await client.query(`
+      INSERT INTO queen.retention_history (messages_deleted, retention_type)
+      VALUES ($1, 'evicted')
+    `, [evictedCount]);
+  }
+  
+  return evictedCount;
+};
+
+// Background eviction for all queues
+const performEviction = async (pool) => {
+  const client = await pool.connect();
+  
+  try {
+    // Get all queues with eviction configured
+    const queuesResult = await client.query(`
+      SELECT name, max_wait_time_seconds
+      FROM queen.queues
+      WHERE max_wait_time_seconds > 0
+    `);
+    
+    let totalEvicted = 0;
+    for (const queue of queuesResult.rows) {
+      const evicted = await evictMessages(
+        client, 
+        queue.name, 
+        queue.max_wait_time_seconds
+      );
+      
+      if (evicted > 0) {
+        log(`${LogTypes.EVICTION} | Queue: ${queue.name} | Count: ${evicted} | MaxWaitTime: ${queue.max_wait_time_seconds}s`);
+        totalEvicted += evicted;
+      }
+    }
+    
+    if (totalEvicted > 0) {
+      await client.query(`
+        INSERT INTO queen.retention_history (messages_deleted, retention_type)
+        VALUES ($1, 'evicted')
+      `, [totalEvicted]);
+    }
+    
+    return totalEvicted;
+  } catch (error) {
+    log('Eviction error:', error);
+    return 0;
+  } finally {
+    client.release();
+  }
+};
+
+// Start the eviction job (optional background job)
+export const startEvictionJob = (pool, eventManager) => {
+  const intervalId = setInterval(async () => {
+    try {
+      const evicted = await performEviction(pool);
+      if (evicted > 0 && eventManager) {
+        eventManager.emit('messages:evicted', { count: evicted });
+      }
+    } catch (error) {
+      log('Eviction job error:', error);
+    }
+  }, EVICTION_INTERVAL);
+  
+  log(`⏰ Eviction job started (interval: ${EVICTION_INTERVAL}ms)`);
+  
+  // Return cleanup function
+  return () => {
+    clearInterval(intervalId);
+    log('Eviction job stopped');
+  };
+};
+
+// Export for manual execution
+export const runEviction = performEviction;

package/src/services/retentionService.js

@@ -0,0 +1,129 @@
+// Functional retention service for message cleanup
+import { log, LogTypes } from '../utils/logger.js';
+import config from '../config.js';
+
+const RETENTION_INTERVAL = config.JOBS.RETENTION_INTERVAL;
+
+// Apply retention policy to a single queue
+const retentionQueue = async (client, queue) => {
+  const { 
+    retention_seconds: retentionSeconds = 0,
+    completed_retention_seconds: completedRetentionSeconds = 0,
+    retention_enabled: retentionEnabled = false 
+  } = queue;
+  
+  if (!retentionEnabled) return 0;
+  
+  let totalDeleted = 0;
+  
+  // Delete old pending messages from all partitions in this queue
+  if (retentionSeconds > 0) {
+    const result = await client.query(`
+      DELETE FROM queen.messages
+      WHERE partition_id IN (
+        SELECT id FROM queen.partitions WHERE queue_id = $1
+      )
+        AND status = 'pending'
+        AND created_at < NOW() - INTERVAL '1 second' * $2
+      RETURNING id
+    `, [queue.id, retentionSeconds]);
+    
+    totalDeleted += result.rowCount || 0;
+  }
+  
+  // Delete completed/failed/evicted messages from all partitions in this queue
+  if (completedRetentionSeconds > 0) {
+    const result = await client.query(`
+      DELETE FROM queen.messages
+      WHERE partition_id IN (
+        SELECT id FROM queen.partitions WHERE queue_id = $1
+      )
+        AND status IN ('completed', 'failed', 'evicted')
+        AND COALESCE(completed_at, failed_at, created_at) < NOW() - INTERVAL '1 second' * $2
+      RETURNING id
+    `, [queue.id, completedRetentionSeconds]);
+    
+    totalDeleted += result.rowCount || 0;
+  }
+  
+  // Log retention if messages were deleted
+  if (totalDeleted > 0) {
+    // Note: We might need to update retention_history table to use queue_id instead
+    log(`${LogTypes.RETENTION} | Queue: ${queue.name} | Count: ${totalDeleted} | RetentionSeconds: ${retentionSeconds} | CompletedRetentionSeconds: ${completedRetentionSeconds}`);
+  }
+  
+  return totalDeleted;
+};
+
+// Clean up empty partitions (no longer based on partition-level config)
+const cleanupEmptyPartitions = async (client) => {
+  // For now, we keep Default partitions and only clean up truly empty non-default partitions
+  // that haven't had activity for a long time (e.g., 7 days)
+  const result = await client.query(`
+    DELETE FROM queen.partitions p
+    WHERE p.name != 'Default'
+      AND NOT EXISTS (
+        SELECT 1 FROM queen.messages m WHERE m.partition_id = p.id
+      )
+      AND p.last_activity < NOW() - INTERVAL '7 days'
+    RETURNING id, name
+  `);
+  
+  if (result.rowCount > 0) {
+    log(`🗑️  Deleted ${result.rowCount} empty partitions`);
+  }
+  
+  return result.rowCount;
+};
+
+// Main retention function
+const performRetention = async (pool) => {
+  const client = await pool.connect();
+  
+  try {
+    // Get queues with retention enabled
+    const queuesResult = await client.query(`
+      SELECT q.id, q.name, q.retention_enabled, q.retention_seconds, 
+             q.completed_retention_seconds
+      FROM queen.queues q
+      WHERE q.retention_enabled = true
+    `);
+    
+    let totalDeleted = 0;
+    for (const queue of queuesResult.rows) {
+      totalDeleted += await retentionQueue(client, queue);
+    }
+    
+    // Cleanup empty partitions
+    await cleanupEmptyPartitions(client);
+    
+    return totalDeleted;
+  } catch (error) {
+    log('Retention error:', error);
+    return 0;
+  } finally {
+    client.release();
+  }
+};
+
+// Start the retention job
+export const startRetentionJob = (pool) => {
+  const intervalId = setInterval(async () => {
+    try {
+      await performRetention(pool);
+    } catch (error) {
+      log('Retention job error:', error);
+    }
+  }, RETENTION_INTERVAL);
+  
+  log(`♻️  Retention job started (interval: ${RETENTION_INTERVAL}ms)`);
+  
+  // Return cleanup function
+  return () => {
+    clearInterval(intervalId);
+    log('Retention job stopped');
+  };
+};
+
+// Export for manual execution
+export const runRetention = performRetention;

package/src/services/startupSync.js

@@ -0,0 +1,35 @@
+import { SYSTEM_QUEUE } from '../managers/systemEventManager.js';
+
+export async function syncSystemEvents(client, eventManager, consumerGroup) {
+  console.log('Synchronizing system events...');
+  
+  let processed = 0;
+  let hasMore = true;
+  
+  while (hasMore) {
+    // Pop and process system events in batches using the server's consumer group
+    // This ensures we only process events we haven't seen before
+    const result = await client.pop({
+      queue: SYSTEM_QUEUE,
+      consumerGroup: consumerGroup,  // Use server's unique consumer group
+      batch: 100,
+      wait: false
+    });
+    
+    if (!result.messages || result.messages.length === 0) {
+      hasMore = false;
+      break;
+    }
+    
+    for (const message of result.messages) {
+      await eventManager.processSystemEvent(message.payload);
+      await client.ack(message.transactionId, 'completed', null, consumerGroup);
+      processed++;
+    }
+  }
+  
+  if (processed > 0) {
+    console.log(`Synchronized ${processed} system events`);
+  }
+  return processed;
+}