queen-mq 0.1.0

package/src/managers/resourceCache.js

@@ -0,0 +1,96 @@
+import { EventTypes } from './systemEventManager.js';
+
+// In-memory cache for resource existence to avoid repeated DB lookups
+export const createResourceCache = () => {
+  const cache = new Map();
+  // Make TTL configurable - set to 0 to disable caching in multi-server setups
+  const TTL = process.env.QUEEN_CACHE_TTL ? parseInt(process.env.QUEEN_CACHE_TTL) : 60000; // Default 1 minute
+  
+  const getCacheKey = (queue, partition) => `${queue}:${partition || 'Default'}`;
+  
+  const checkResource = (queue, partition) => {
+    // If TTL is 0, caching is disabled
+    if (TTL === 0) return null;
+    
+    const key = getCacheKey(queue, partition);
+    const cached = cache.get(key);
+    
+    if (cached && Date.now() - cached.timestamp < TTL) {
+      return cached.data;
+    }
+    
+    return null; // Not in cache
+  };
+  
+  const cacheResource = (queue, partition, data) => {
+    // If TTL is 0, caching is disabled
+    if (TTL === 0) return;
+    
+    const key = getCacheKey(queue, partition);
+    cache.set(key, { data, timestamp: Date.now() });
+  };
+  
+  const invalidate = (queue, partition) => {
+    if (queue && partition) {
+      cache.delete(getCacheKey(queue, partition));
+    } else if (queue) {
+      // Clear all partitions for this queue
+      for (const key of cache.keys()) {
+        if (key.startsWith(`${queue}:`)) {
+          cache.delete(key);
+        }
+      }
+    } else {
+      cache.clear(); // Clear all if no specific resource
+    }
+  };
+  
+  const invalidateQueue = (queue) => {
+    // Clear all partitions for this queue when queue config changes
+    for (const key of cache.keys()) {
+      if (key.startsWith(`${queue}:`)) {
+        cache.delete(key);
+      }
+    }
+  };
+  
+  // Cleanup old entries periodically (only if caching is enabled)
+  if (TTL > 0) {
+    setInterval(() => {
+      const now = Date.now();
+      for (const [key, value] of cache.entries()) {
+        if (now - value.timestamp > TTL) {
+          cache.delete(key);
+        }
+      }
+    }, Math.min(TTL, 60000)); // Check at most once per minute
+  }
+  
+  // Register event handlers for cache invalidation
+  const registerEventHandlers = (eventManager) => {
+    // Invalidate cache on queue changes
+    eventManager.on(EventTypes.QUEUE_UPDATED, (event) => {
+      invalidateQueue(event.entityId);
+    });
+    
+    eventManager.on(EventTypes.QUEUE_DELETED, (event) => {
+      invalidateQueue(event.entityId);
+    });
+    
+    eventManager.on(EventTypes.PARTITION_CREATED, (event) => {
+      invalidate(event.queueName, event.entityId);
+    });
+    
+    eventManager.on(EventTypes.PARTITION_DELETED, (event) => {
+      invalidate(event.queueName, event.entityId);
+    });
+  };
+  
+  return {
+    checkResource,
+    cacheResource,
+    invalidate,
+    invalidateQueue,
+    registerEventHandlers
+  };
+};

package/src/managers/systemEventManager.js

@@ -0,0 +1,127 @@
+import { generateUUID } from '../utils/uuid.js';
+
+export const SYSTEM_QUEUE = '__system_events__';
+
+export const EventTypes = {
+  // Queue events
+  QUEUE_CREATED: 'queue.created',
+  QUEUE_UPDATED: 'queue.updated',
+  QUEUE_DELETED: 'queue.deleted',
+  
+  // Partition events
+  PARTITION_CREATED: 'partition.created',
+  PARTITION_DELETED: 'partition.deleted',
+  
+  // Future events
+  CONSUMER_GROUP_CREATED: 'consumer_group.created',
+  CONSUMER_GROUP_UPDATED: 'consumer_group.updated'
+};
+
+export class SystemEventManager {
+  constructor(pool, serverInstanceId) {
+    this.pool = pool;
+    this.serverInstanceId = serverInstanceId;
+    this.handlers = new Map();
+    this.eventQueue = [];
+    this.batchTimer = null;
+  }
+  
+  // Register event handlers
+  on(eventType, handler) {
+    if (!this.handlers.has(eventType)) {
+      this.handlers.set(eventType, []);
+    }
+    this.handlers.get(eventType).push(handler);
+  }
+  
+  // Emit event locally and to system queue
+  async emit(eventType, data) {
+    const event = {
+      id: generateUUID(),
+      eventType,
+      ...data,
+      timestamp: Date.now(),
+      sourceServer: this.serverInstanceId,
+      version: 1
+    };
+    
+    // Handle locally first (immediate consistency for originating server)
+    await this.handleEventLocally(event);
+    
+    // Queue for batch publishing
+    this.eventQueue.push(event);
+    this.scheduleBatch();
+  }
+  
+  // Process incoming event from system queue
+  async processSystemEvent(event) {
+    // Skip own events (already processed locally)
+    if (event.sourceServer === this.serverInstanceId) {
+      return;
+    }
+    
+    await this.handleEventLocally(event);
+  }
+  
+  async handleEventLocally(event) {
+    const handlers = this.handlers.get(event.eventType) || [];
+    for (const handler of handlers) {
+      try {
+        await handler(event);
+      } catch (error) {
+        console.error(`Handler failed for ${event.eventType}:`, error);
+      }
+    }
+  }
+  
+  scheduleBatch() {
+    if (this.batchTimer) return;
+    
+    this.batchTimer = setTimeout(() => {
+      this.publishBatch();
+    }, 10); // 10ms batching
+  }
+  
+  async publishBatch() {
+    if (this.eventQueue.length === 0) return;
+    
+    const events = [...this.eventQueue];
+    this.eventQueue = [];
+    this.batchTimer = null;
+    
+    try {
+      // Direct database insert to avoid circular dependency
+      const client = await this.pool.connect();
+      try {
+        for (const event of events) {
+          await this.insertSystemEvent(client, event);
+        }
+      } finally {
+        client.release();
+      }
+    } catch (error) {
+      console.error('Failed to publish system events:', error);
+    }
+  }
+  
+  async insertSystemEvent(client, event) {
+    // Direct insert bypassing queue manager to avoid circular dependency
+    await client.query(`
+      INSERT INTO queen.messages (
+        id, transaction_id, partition_id, payload, 
+        created_at, trace_id
+      )
+      SELECT 
+        $1, $2, p.id, $3, NOW(), $4
+      FROM queen.partitions p
+      JOIN queen.queues q ON p.queue_id = q.id
+      WHERE q.name = $5 AND p.name = 'Default'
+    `, [
+      generateUUID(),
+      generateUUID(),
+      JSON.stringify(event),
+      event.id,
+      SYSTEM_QUEUE
+    ]);
+  }
+}

package/src/routes/ack.js

@@ -0,0 +1,26 @@
+export const createAckRoute = (queueManager) => {
+  return async (body) => {
+    const { transactionId, status, error, consumerGroup } = body;
+    
+    if (!transactionId) {
+      throw new Error('transactionId is required');
+    }
+    
+    if (!status || !['completed', 'failed'].includes(status)) {
+      throw new Error('status must be "completed" or "failed"');
+    }
+    
+    const result = await queueManager.acknowledgeMessage(transactionId, status, error, consumerGroup);
+    
+    if (!result) {
+      throw new Error('Message not found or not in processing state');
+    }
+    
+    return {
+      transactionId: result.transaction_id,
+      status: result.status,
+      consumerGroup: consumerGroup || null,
+      acknowledgedAt: new Date().toISOString()
+    };
+  };
+};