queen-mq 0.1.0 → 0.1.2

package/CACHE.md

@@ -1,519 +0,0 @@
-# Queen Internal Event Propagation System
-
-## Overview
-A general-purpose internal event system using Queen's own infrastructure to propagate system events (configuration changes, cache invalidations, etc.) across multiple server instances.
-
-## Architecture
-
-### Event Flow
-```
-┌─────────────────┐     ┌─────────────────┐     ┌─────────────────┐
-│   Server #1     │     │   Server #2     │     │   Server #3     │
-│                 │     │                 │     │                 │
-│ Event Producer  │     │ Event Consumer  │     │ Event Consumer  │
-│ Event Consumer  │     │                 │     │                 │
-└────────┬────────┘     └────────┬────────┘     └────────┬────────┘
-         │                       │                       │
-         └───────────┬───────────┴───────────────────────┘
-                     ▼
-         ┌───────────────────────────┐
-         │    __system_events__      │
-         │         Queue              │
-         └───────────────────────────┘
-```
-
-### Event Structure
-```javascript
-{
-  eventType: 'queue.updated',  // queue.created, queue.updated, queue.deleted
-  entityType: 'queue',         // queue, partition, consumer_group
-  entityId: 'user-tasks',      // queue name or other identifier
-  changes: {                   // What changed
-    retryLimit: { old: 3, new: 5 },
-    priority: { old: 0, new: 10 }
-  },
-  timestamp: 1234567890,
-  sourceServer: 'server_1234',
-  version: 1                   // Event schema version
-}
-```
-
-## Implementation
-
-### Phase 1: Core Event System
-
-#### 1.1 System Event Manager
-**File**: `src/managers/systemEventManager.js`
-```javascript
-import { generateUUID } from '../utils/uuid.js';
-
-export const SYSTEM_QUEUE = '__system_events__';
-
-export const EventTypes = {
-  // Queue events
-  QUEUE_CREATED: 'queue.created',
-  QUEUE_UPDATED: 'queue.updated',
-  QUEUE_DELETED: 'queue.deleted',
-  
-  // Partition events
-  PARTITION_CREATED: 'partition.created',
-  PARTITION_DELETED: 'partition.deleted',
-  
-  // Future events
-  CONSUMER_GROUP_CREATED: 'consumer_group.created',
-  CONSUMER_GROUP_UPDATED: 'consumer_group.updated'
-};
-
-export class SystemEventManager {
-  constructor(pool, serverInstanceId) {
-    this.pool = pool;
-    this.serverInstanceId = serverInstanceId;
-    this.handlers = new Map();
-    this.eventQueue = [];
-    this.batchTimer = null;
-  }
-  
-  // Register event handlers
-  on(eventType, handler) {
-    if (!this.handlers.has(eventType)) {
-      this.handlers.set(eventType, []);
-    }
-    this.handlers.get(eventType).push(handler);
-  }
-  
-  // Emit event locally and to system queue
-  async emit(eventType, data) {
-    const event = {
-      id: generateUUID(),
-      eventType,
-      ...data,
-      timestamp: Date.now(),
-      sourceServer: this.serverInstanceId,
-      version: 1
-    };
-    
-    // Handle locally first (immediate consistency for originating server)
-    await this.handleEventLocally(event);
-    
-    // Queue for batch publishing
-    this.eventQueue.push(event);
-    this.scheduleBatch();
-  }
-  
-  // Process incoming event from system queue
-  async processSystemEvent(event) {
-    // Skip own events (already processed locally)
-    if (event.sourceServer === this.serverInstanceId) {
-      return;
-    }
-    
-    await this.handleEventLocally(event);
-  }
-  
-  async handleEventLocally(event) {
-    const handlers = this.handlers.get(event.eventType) || [];
-    for (const handler of handlers) {
-      try {
-        await handler(event);
-      } catch (error) {
-        console.error(`Handler failed for ${event.eventType}:`, error);
-      }
-    }
-  }
-  
-  scheduleBatch() {
-    if (this.batchTimer) return;
-    
-    this.batchTimer = setTimeout(() => {
-      this.publishBatch();
-    }, 10); // 10ms batching
-  }
-  
-  async publishBatch() {
-    if (this.eventQueue.length === 0) return;
-    
-    const events = [...this.eventQueue];
-    this.eventQueue = [];
-    this.batchTimer = null;
-    
-    try {
-      // Direct database insert to avoid circular dependency
-      const client = await this.pool.connect();
-      try {
-        for (const event of events) {
-          await this.insertSystemEvent(client, event);
-        }
-      } finally {
-        client.release();
-      }
-    } catch (error) {
-      console.error('Failed to publish system events:', error);
-    }
-  }
-  
-  async insertSystemEvent(client, event) {
-    // Direct insert bypassing queue manager to avoid circular dependency
-    await client.query(`
-      INSERT INTO queen.messages (
-        id, transaction_id, partition_id, payload, 
-        created_at, trace_id
-      )
-      SELECT 
-        $1, $2, p.id, $3, NOW(), $4
-      FROM queen.partitions p
-      JOIN queen.queues q ON p.queue_id = q.id
-      WHERE q.name = $5 AND p.name = 'Default'
-    `, [
-      generateUUID(),
-      generateUUID(),
-      JSON.stringify(event),
-      event.id,
-      SYSTEM_QUEUE
-    ]);
-  }
-}
-```
-
-#### 1.2 Server Startup Synchronization
-**File**: `src/services/startupSync.js`
-```javascript
-export async function syncSystemEvents(client, eventManager) {
-  console.log('Synchronizing system events...');
-  
-  let processed = 0;
-  let hasMore = true;
-  
-  while (hasMore) {
-    // Pop and process system events in batches
-    const result = await client.pop({
-      queue: SYSTEM_QUEUE,
-      batch: 100,
-      wait: false
-    });
-    
-    if (!result.messages || result.messages.length === 0) {
-      hasMore = false;
-      break;
-    }
-    
-    for (const message of result.messages) {
-      await eventManager.processSystemEvent(message.payload);
-      await client.ack(message.transactionId, 'completed');
-      processed++;
-    }
-  }
-  
-  console.log(`Synchronized ${processed} system events`);
-  return processed;
-}
-```
-
-#### 1.3 Resource Cache Integration
-**File**: `src/managers/resourceCache.js` (additions)
-```javascript
-export const createResourceCache = () => {
-  // ... existing code ...
-  
-  // Register event handlers
-  const registerEventHandlers = (eventManager) => {
-    // Invalidate cache on queue changes
-    eventManager.on(EventTypes.QUEUE_UPDATED, (event) => {
-      invalidateQueue(event.entityId);
-    });
-    
-    eventManager.on(EventTypes.QUEUE_DELETED, (event) => {
-      invalidateQueue(event.entityId);
-    });
-    
-    eventManager.on(EventTypes.PARTITION_CREATED, (event) => {
-      invalidate(event.queueName, event.entityId);
-    });
-    
-    eventManager.on(EventTypes.PARTITION_DELETED, (event) => {
-      invalidate(event.queueName, event.entityId);
-    });
-  };
-  
-  return {
-    // ... existing methods ...
-    registerEventHandlers
-  };
-};
-```
-
-### Phase 2: Queue Manager Integration
-
-#### 2.1 Emit Events on Configuration Changes
-**File**: `src/managers/queueManagerOptimized.js` (modifications)
-```javascript
-const configureQueue = async (queueName, options = {}, namespace = null, task = null) => {
-  return withTransaction(pool, async (client) => {
-    // Check if queue exists (for update vs create detection)
-    const existingQueue = await client.query(
-      'SELECT * FROM queen.queues WHERE name = $1',
-      [queueName]
-    );
-    const isUpdate = existingQueue.rows.length > 0;
-    
-    // ... existing configuration code ...
-    
-    // Detect what changed
-    const changes = {};
-    if (isUpdate) {
-      const old = existingQueue.rows[0];
-      for (const [key, value] of Object.entries(options)) {
-        const dbKey = optionMappings[key];
-        if (old[dbKey] !== value) {
-          changes[key] = { old: old[dbKey], new: value };
-        }
-      }
-    }
-    
-    // Emit appropriate event
-    await eventManager.emit(
-      isUpdate ? EventTypes.QUEUE_UPDATED : EventTypes.QUEUE_CREATED,
-      {
-        entityType: 'queue',
-        entityId: queueName,
-        changes: isUpdate ? changes : options,
-        namespace,
-        task
-      }
-    );
-    
-    // Local cache invalidation (immediate)
-    resourceCache.invalidateQueue(queueName);
-    
-    return result;
-  });
-};
-```
-
-### Phase 3: Server Integration
-
-#### 3.1 Server Initialization
-**File**: `src/server.js` (modifications)
-```javascript
-import { SystemEventManager, SYSTEM_QUEUE } from './managers/systemEventManager.js';
-import { syncSystemEvents } from './services/startupSync.js';
-
-// Generate unique server instance ID
-const SERVER_INSTANCE_ID = `server_${process.pid}_${Date.now()}`;
-
-// Initialize system event manager
-const systemEventManager = new SystemEventManager(pool, SERVER_INSTANCE_ID);
-
-// Register cache event handlers
-resourceCache.registerEventHandlers(systemEventManager);
-
-// Initialize system queue
-async function initializeSystemQueue() {
-  try {
-    // Direct database operation to create system queue
-    await pool.query(`
-      INSERT INTO queen.queues (name, ttl, priority, max_queue_size)
-      VALUES ($1, 300, 100, 10000)
-      ON CONFLICT (name) DO UPDATE SET
-        ttl = EXCLUDED.ttl,
-        priority = EXCLUDED.priority,
-        max_queue_size = EXCLUDED.max_queue_size
-    `, [SYSTEM_QUEUE]);
-    
-    await pool.query(`
-      INSERT INTO queen.partitions (queue_id, name)
-      SELECT id, 'Default' FROM queen.queues WHERE name = $1
-      ON CONFLICT (queue_id, name) DO NOTHING
-    `, [SYSTEM_QUEUE]);
-  } catch (error) {
-    console.error('Failed to initialize system queue:', error);
-    throw error;
-  }
-}
-
-// Startup sequence
-async function startServer() {
-  console.log('Starting Queen server...');
-  
-  // 1. Initialize database connection
-  await testDatabaseConnection();
-  
-  // 2. Initialize system queue
-  await initializeSystemQueue();
-  
-  // 3. Create temporary client for synchronization
-  const syncClient = createQueenClient({ 
-    baseUrl: `http://localhost:${PORT}` 
-  });
-  
-  // 4. Synchronize system events (catch up on missed events)
-  await syncSystemEvents(syncClient, systemEventManager);
-  
-  // 5. Start consuming system events
-  const systemEventConsumer = syncClient.consume({
-    queue: SYSTEM_QUEUE,
-    consumerGroup: SERVER_INSTANCE_ID,
-    handler: async (message) => {
-      await systemEventManager.processSystemEvent(message.payload);
-    },
-    options: {
-      batch: 10,
-      wait: true
-    }
-  });
-  
-  // 6. Start HTTP server
-  app.listen(PORT, '0.0.0.0', (listenSocket) => {
-    if (listenSocket) {
-      console.log(`Queen server running on port ${PORT}`);
-      console.log(`Server Instance ID: ${SERVER_INSTANCE_ID}`);
-    }
-  });
-}
-
-// Graceful shutdown
-process.on('SIGTERM', async () => {
-  console.log('Shutting down...');
-  systemEventConsumer?.stop();
-  await pool.end();
-  process.exit(0);
-});
-```
-
-#### 3.2 Bypass Cache for System Queue
-**File**: `src/managers/queueManagerOptimized.js` (modification)
-```javascript
-const ensureResources = async (client, queueName, partitionName = 'Default', namespace = null, task = null) => {
-  // CRITICAL: System queues bypass cache completely
-  if (queueName.startsWith('__system_')) {
-    // Direct database query for system queues
-    const result = await client.query(`
-      SELECT 
-        q.id as queue_id,
-        q.name as queue_name,
-        p.id as partition_id,
-        q.*
-      FROM queen.queues q
-      JOIN queen.partitions p ON p.queue_id = q.id
-      WHERE q.name = $1 AND p.name = $2
-    `, [queueName, partitionName]);
-    
-    if (result.rows.length === 0) {
-      throw new Error(`System queue ${queueName} not found`);
-    }
-    
-    const row = result.rows[0];
-    return {
-      queueId: row.queue_id,
-      queueName: row.queue_name,
-      partitionId: row.partition_id,
-      queueConfig: {
-        leaseTime: row.lease_time,
-        retryLimit: row.retry_limit,
-        // ... other config fields
-      },
-      encryptionEnabled: false,  // System queues never encrypted
-      maxWaitTimeSeconds: row.max_wait_time_seconds
-    };
-  }
-  
-  // ... existing cache logic for regular queues ...
-};
-```
-
-## Configuration
-
-### Environment Variables
-```bash
-# System event configuration
-QUEEN_SYSTEM_EVENTS_ENABLED=true         # Enable system event propagation
-QUEEN_SYSTEM_EVENTS_BATCH_MS=10          # Batching window for events
-QUEEN_SYSTEM_EVENTS_SYNC_TIMEOUT=30000   # Timeout for startup synchronization
-
-# Cache configuration (unchanged)
-QUEEN_CACHE_TTL=60000                    # Cache TTL when events are enabled
-```
-
-## Monitoring
-
-### Key Metrics
-```javascript
-// System event metrics
-{
-  systemEventsPublished: counter,
-  systemEventsProcessed: counter,
-  systemEventLag: gauge,
-  startupSyncDuration: histogram,
-  eventProcessingErrors: counter
-}
-```
-
-### Health Checks
-```javascript
-// Add to health endpoint
-async function checkSystemEventHealth() {
-  const result = await pool.query(`
-    SELECT COUNT(*) as pending
-    FROM queen.messages m
-    JOIN queen.partitions p ON m.partition_id = p.id
-    JOIN queen.queues q ON p.queue_id = q.id
-    WHERE q.name = $1
-      AND m.status = 'pending'
-  `, [SYSTEM_QUEUE]);
-  
-  return {
-    systemEventQueueDepth: result.rows[0].pending,
-    healthy: result.rows[0].pending < 1000
-  };
-}
-```
-
-## Migration Plan
-
-### Step 1: Deploy with Feature Flag Off
-```bash
-QUEEN_SYSTEM_EVENTS_ENABLED=false
-```
-
-### Step 2: Enable and Monitor
-```bash
-QUEEN_SYSTEM_EVENTS_ENABLED=true
-QUEEN_CACHE_TTL=60000  # Re-enable caching
-```
-
-### Step 3: Optimize
-- Increase cache TTL once confidence is established
-- Tune batching windows based on event volume
-
-## Rollback Plan
-
-### Immediate Rollback
-```bash
-# Disable system events
-QUEEN_SYSTEM_EVENTS_ENABLED=false
-QUEEN_CACHE_TTL=0  # Disable cache
-```
-
-### Gradual Rollback
-```bash
-# Keep events but reduce cache TTL
-QUEEN_SYSTEM_EVENTS_ENABLED=true
-QUEEN_CACHE_TTL=5000  # 5 second cache
-```
-
-## Future Extensions
-
-The event system can be extended for:
-- **Audit Logging**: Track all configuration changes
-- **Metrics Collection**: Aggregate queue statistics
-- **Distributed Locks**: Coordinate operations across servers
-- **Leader Election**: Designate primary server for certain operations
-- **Schema Migrations**: Coordinate database updates
-- **Feature Flags**: Propagate feature flag changes
-
-## Success Criteria
-
-- [ ] System events published for all queue operations
-- [ ] All servers process events and update caches
-- [ ] Startup synchronization completes in < 5 seconds
-- [ ] No circular dependencies with system queue
-- [ ] Cache consistency across all servers within 100ms