queen-mq 0.1.0

package/docs/multi-server-cache-solutions.md

@@ -0,0 +1,185 @@
+# Multi-Server Cache Consistency Solutions
+
+## Problem
+When running multiple Queen servers, each server maintains its own in-memory resource cache with a 1-minute TTL. Configuration changes on one server are not reflected on other servers until the cache expires.
+
+## Solutions
+
+### 1. **Disable Cache in Multi-Server Mode** (Quick Fix)
+Set cache TTL to 0 or very short duration when running multiple servers.
+
+```javascript
+// In src/managers/resourceCache.js
+const TTL = process.env.QUEEN_CACHE_TTL ? parseInt(process.env.QUEEN_CACHE_TTL) : 60000;
+```
+
+Run servers with:
+```bash
+QUEEN_CACHE_TTL=0 PORT=6632 node src/server.js
+QUEEN_CACHE_TTL=0 PORT=6633 node src/server.js
+```
+
+### 2. **Cache Invalidation via Database Triggers** (Recommended)
+Add a `cache_invalidation` table and use database triggers to notify servers of changes.
+
+```sql
+-- Add to schema
+CREATE TABLE queen.cache_invalidations (
+    id SERIAL PRIMARY KEY,
+    resource_type VARCHAR(50) NOT NULL,
+    resource_key VARCHAR(255) NOT NULL,
+    invalidated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX idx_cache_invalidations_time ON queen.cache_invalidations(invalidated_at);
+
+-- Trigger on queue updates
+CREATE OR REPLACE FUNCTION queen.notify_cache_invalidation()
+RETURNS TRIGGER AS $$
+BEGIN
+    INSERT INTO queen.cache_invalidations (resource_type, resource_key)
+    VALUES ('queue', NEW.name);
+    
+    PERFORM pg_notify('cache_invalidation', json_build_object(
+        'type', 'queue',
+        'key', NEW.name
+    )::text);
+    
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER queue_cache_invalidation
+AFTER UPDATE ON queen.queues
+FOR EACH ROW
+EXECUTE FUNCTION queen.notify_cache_invalidation();
+```
+
+### 3. **Redis-Based Distributed Cache** (Best for Scale)
+Replace in-memory cache with Redis for shared caching across servers.
+
+```javascript
+// src/managers/redisResourceCache.js
+import Redis from 'ioredis';
+
+export const createRedisResourceCache = (redisConfig) => {
+  const redis = new Redis(redisConfig);
+  const TTL = 60; // seconds
+  
+  const getCacheKey = (queue, partition) => 
+    `queen:resource:${queue}:${partition || 'Default'}`;
+  
+  const checkResource = async (queue, partition) => {
+    const key = getCacheKey(queue, partition);
+    const cached = await redis.get(key);
+    return cached ? JSON.parse(cached) : null;
+  };
+  
+  const cacheResource = async (queue, partition, data) => {
+    const key = getCacheKey(queue, partition);
+    await redis.setex(key, TTL, JSON.stringify(data));
+  };
+  
+  const invalidate = async (queue, partition) => {
+    if (queue && partition) {
+      await redis.del(getCacheKey(queue, partition));
+    } else if (queue) {
+      const keys = await redis.keys(`queen:resource:${queue}:*`);
+      if (keys.length > 0) {
+        await redis.del(...keys);
+      }
+    } else {
+      const keys = await redis.keys('queen:resource:*');
+      if (keys.length > 0) {
+        await redis.del(...keys);
+      }
+    }
+  };
+  
+  return {
+    checkResource,
+    cacheResource,
+    invalidate,
+    invalidateQueue: (queue) => invalidate(queue)
+  };
+};
+```
+
+### 4. **Polling-Based Cache Invalidation** (Simple)
+Each server periodically checks for configuration changes.
+
+```javascript
+// Add to resourceCache.js
+const pollForChanges = async (pool) => {
+  const result = await pool.query(`
+    SELECT name, updated_at 
+    FROM queen.queues 
+    WHERE updated_at > $1
+  `, [lastCheckTime]);
+  
+  for (const row of result.rows) {
+    invalidateQueue(row.name);
+  }
+  
+  lastCheckTime = new Date();
+};
+
+// Poll every 5 seconds
+setInterval(() => pollForChanges(pool), 5000);
+```
+
+### 5. **Event-Based Invalidation via Message Queue** 
+Use the Queen queue itself to propagate cache invalidation events.
+
+```javascript
+// When configuration changes on one server
+await client.push({
+  items: [{
+    queue: '_system_cache_invalidation',
+    partition: 'Default',
+    payload: {
+      action: 'invalidate',
+      resourceType: 'queue',
+      resourceKey: queueName,
+      timestamp: Date.now()
+    }
+  }]
+});
+
+// All servers consume from this queue
+client.consume({
+  queue: '_system_cache_invalidation',
+  handler: async (message) => {
+    const { resourceType, resourceKey } = message.payload;
+    if (resourceType === 'queue') {
+      resourceCache.invalidateQueue(resourceKey);
+    }
+  }
+});
+```
+
+## Recommended Approach
+
+For your immediate testing needs, use **Solution 1** (disable cache):
+```bash
+# Start servers without cache
+QUEEN_CACHE_TTL=0 PORT=6632 node src/server.js
+QUEEN_CACHE_TTL=0 PORT=6633 node src/server.js
+```
+
+For production multi-server deployment:
+- **Small scale (2-3 servers)**: Solution 2 (Database triggers with LISTEN/NOTIFY)
+- **Medium scale (4-10 servers)**: Solution 3 (Redis cache)
+- **Large scale (10+ servers)**: Solution 3 (Redis) + Solution 5 (Event-based)
+
+## Testing Cache Behavior
+
+After implementing a solution, run:
+```bash
+node examples/test-cache-multi-server.js
+```
+
+The test should show:
+- Both servers returning the same messages ✅
+- Configuration changes immediately visible on both servers ✅
+- No cache inconsistency warnings ✅

package/docs/performance-tuning.md

@@ -0,0 +1,222 @@
+# Performance Tuning Guide for Queen
+
+## Database Connection Pool Optimization
+
+When running Queen under high load, you may encounter database connection pool exhaustion errors:
+```
+Error: timeout exceeded when trying to connect
+```
+
+This happens when the rate of incoming requests exceeds the available database connections.
+
+## Solution 1: Increase Connection Pool Size
+
+Set these environment variables before starting the server:
+
+```bash
+# Increase the connection pool size (default is 20)
+export DB_POOL_SIZE=50
+
+# Increase connection timeout (default is 2000ms)
+export DB_CONNECTION_TIMEOUT=5000
+
+# Start the server
+npm start
+```
+
+## Solution 2: Optimize PostgreSQL Settings
+
+Edit your `postgresql.conf`:
+
+```conf
+# Increase maximum connections
+max_connections = 200
+
+# Connection pooling
+shared_preload_libraries = 'pg_stat_statements'
+
+# Memory settings for better performance
+shared_buffers = 256MB
+effective_cache_size = 1GB
+work_mem = 4MB
+maintenance_work_mem = 64MB
+```
+
+## Solution 3: Use Batch Inserts (Database Optimization)
+
+The current implementation processes messages individually within a transaction. For better performance under high load, consider using batch inserts.
+
+### Current Implementation (Sequential)
+```javascript
+// Process each item individually
+for (const item of items) {
+  // Individual INSERT for each message
+}
+```
+
+### Optimized Implementation (Batch)
+```sql
+-- Use single INSERT with multiple VALUES
+INSERT INTO queen.messages (transaction_id, queue_id, payload, status)
+VALUES 
+  ($1, $2, $3, 'pending'),
+  ($4, $5, $6, 'pending'),
+  ...
+RETURNING id, transaction_id;
+```
+
+## Solution 4: Implement Connection Pooling with PgBouncer
+
+For production environments with very high load, use PgBouncer as a connection pooler:
+
+1. Install PgBouncer:
+```bash
+# macOS
+brew install pgbouncer
+
+# Linux
+apt-get install pgbouncer
+```
+
+2. Configure PgBouncer (`/etc/pgbouncer/pgbouncer.ini`):
+```ini
+[databases]
+queen = host=localhost port=5432 dbname=postgres
+
+[pgbouncer]
+listen_port = 6432
+listen_addr = *
+auth_type = md5
+auth_file = /etc/pgbouncer/userlist.txt
+pool_mode = transaction
+max_client_conn = 1000
+default_pool_size = 50
+```
+
+3. Update Queen to connect through PgBouncer:
+```bash
+export PG_PORT=6432  # PgBouncer port
+npm start
+```
+
+## Recommended Production Settings
+
+For a production environment handling high message throughput:
+
+### Environment Variables
+```bash
+# Database
+DB_POOL_SIZE=50
+DB_CONNECTION_TIMEOUT=5000
+DB_IDLE_TIMEOUT=30000
+
+# Application
+NODE_ENV=production
+NODE_OPTIONS="--max-old-space-size=4096"
+
+# Monitoring
+LOG_LEVEL=warn
+ENABLE_METRICS=true
+```
+
+### Example High-Load Configuration
+
+For sustained load of 1000+ messages/second:
+
+```bash
+# Start server with optimized settings
+DB_POOL_SIZE=100 \
+DB_CONNECTION_TIMEOUT=10000 \
+NODE_OPTIONS="--max-old-space-size=8192" \
+npm start
+```
+
+## Testing Under Load
+
+### Reasonable Load Test
+```bash
+# 10 messages per batch, every 100ms = ~100 msg/sec
+BATCH_SIZE=10 INTERVAL=100 node examples/continuous-producer.js
+```
+
+### Medium Load Test
+```bash
+# 50 messages per batch, every 500ms = ~100 msg/sec
+BATCH_SIZE=50 INTERVAL=500 node examples/continuous-producer.js
+```
+
+### High Load Test (requires tuning)
+```bash
+# First, increase DB pool
+export DB_POOL_SIZE=100
+
+# Then run high load producer
+# 100 messages per batch, every 1000ms = ~100 msg/sec
+BATCH_SIZE=100 INTERVAL=1000 node examples/continuous-producer.js
+```
+
+## Monitoring Performance
+
+Monitor these metrics to identify bottlenecks:
+
+1. **Database Connections**
+```sql
+-- Check active connections
+SELECT count(*) FROM pg_stat_activity;
+
+-- Check connections by state
+SELECT state, count(*) 
+FROM pg_stat_activity 
+GROUP BY state;
+```
+
+2. **Connection Pool Status**
+```javascript
+// Add monitoring endpoint to server
+app.get('/metrics/pool', (req, res) => {
+  res.json({
+    totalCount: pool.totalCount,
+    idleCount: pool.idleCount,
+    waitingCount: pool.waitingCount
+  });
+});
+```
+
+3. **Message Throughput**
+```bash
+# Monitor message insertion rate
+watch -n 1 "psql -c 'SELECT count(*) FROM queen.messages WHERE created_at > NOW() - INTERVAL '\''1 minute'\'';'"
+```
+
+## Best Practices
+
+1. **Start Conservative**: Begin with lower message rates and gradually increase
+2. **Monitor Resources**: Watch CPU, memory, and database connections
+3. **Use Batching**: Process messages in reasonable batches (10-100 per batch)
+4. **Implement Backpressure**: Slow down producers when consumers can't keep up
+5. **Scale Horizontally**: Run multiple Queen instances behind a load balancer
+6. **Use Read Replicas**: For analytics and read-heavy operations
+
+## Troubleshooting
+
+### "timeout exceeded when trying to connect"
+- Increase `DB_POOL_SIZE`
+- Increase `DB_CONNECTION_TIMEOUT`
+- Check PostgreSQL `max_connections`
+- Consider using PgBouncer
+
+### "Push request aborted"
+- Client timeout is too short
+- Increase client timeout in producer
+- Reduce batch size
+
+### High Memory Usage
+- Reduce batch sizes
+- Increase Node.js heap size with `--max-old-space-size`
+- Implement message pagination
+
+### Slow Message Processing
+- Add database indexes
+- Use batch operations
+- Implement caching for frequently accessed data
+- Consider partitioning large tables

package/examples/bus-mode.js

@@ -0,0 +1,239 @@
+#!/usr/bin/env node
+
+/**
+ * Bus Mode Example for Queen V3
+ * 
+ * This example demonstrates the new bus/pub-sub functionality where
+ * multiple consumer groups can receive ALL messages independently,
+ * similar to Kafka's consumer groups.
+ * 
+ * Each consumer group maintains its own progress through the message stream.
+ */
+
+import { createQueenClient } from '../src/client/index.js';
+
+async function main() {
+  const client = createQueenClient({
+    baseUrl: process.env.QUEEN_URL || 'http://localhost:6632',
+    timeout: 35000
+  });
+
+  const queue = 'events';
+  
+  console.log('🚀 Starting Bus Mode Example');
+  console.log('📦 Queue:', queue);
+  console.log('');
+  console.log('This example will:');
+  console.log('1. Push some test messages to the queue');
+  console.log('2. Start 3 different consumer groups');
+  console.log('3. Show how each group receives ALL messages independently');
+  console.log('');
+  
+  // First, push some test messages
+  console.log('📤 Pushing test messages...');
+  const testMessages = [];
+  for (let i = 1; i <= 5; i++) {
+    testMessages.push({
+      queue,
+      payload: {
+        id: i,
+        type: 'test-event',
+        timestamp: new Date().toISOString(),
+        data: `Message ${i}`
+      }
+    });
+  }
+  
+  await client.push({ items: testMessages });
+  console.log(`✅ Pushed ${testMessages.length} messages\n`);
+  
+  // Track messages received by each consumer group
+  const received = {
+    analytics: [],
+    notifications: [],
+    audit: []
+  };
+  
+  // Consumer Group 1: Analytics Service
+  console.log('🔵 Starting Analytics Consumer Group...');
+  const analyticsConsumer = client.consume({
+    queue,
+    consumerGroup: 'analytics-service',
+    handler: async (message) => {
+      console.log(`  [Analytics] Received message ${message.data.id}: ${message.data.data}`);
+      received.analytics.push(message.data.id);
+      
+      // Simulate processing
+      await new Promise(resolve => setTimeout(resolve, 100));
+    },
+    options: {
+      batch: 2,
+      wait: false  // Don't wait for new messages in this example
+    }
+  });
+  
+  // Consumer Group 2: Notification Service
+  console.log('🟢 Starting Notifications Consumer Group...');
+  const notificationConsumer = client.consume({
+    queue,
+    consumerGroup: 'notification-service',
+    handler: async (message) => {
+      console.log(`  [Notifications] Received message ${message.data.id}: ${message.data.data}`);
+      received.notifications.push(message.data.id);
+      
+      // Simulate processing
+      await new Promise(resolve => setTimeout(resolve, 150));
+    },
+    options: {
+      batch: 1,
+      wait: false
+    }
+  });
+  
+  // Consumer Group 3: Audit Log Service
+  console.log('🟡 Starting Audit Log Consumer Group...');
+  const auditConsumer = client.consume({
+    queue,
+    consumerGroup: 'audit-log',
+    handler: async (message) => {
+      console.log(`  [Audit] Received message ${message.data.id}: ${message.data.data}`);
+      received.audit.push(message.data.id);
+      
+      // Simulate processing
+      await new Promise(resolve => setTimeout(resolve, 50));
+    },
+    options: {
+      batch: 3,
+      wait: false
+    }
+  });
+  
+  // Wait for all messages to be processed
+  console.log('\n⏳ Processing messages...\n');
+  await new Promise(resolve => setTimeout(resolve, 3000));
+  
+  // Stop all consumers
+  analyticsConsumer();
+  notificationConsumer();
+  auditConsumer();
+  
+  // Display results
+  console.log('\n📊 Results:');
+  console.log('═══════════════════════════════════════════');
+  console.log(`Analytics Service received: [${received.analytics.sort().join(', ')}]`);
+  console.log(`Notification Service received: [${received.notifications.sort().join(', ')}]`);
+  console.log(`Audit Log Service received: [${received.audit.sort().join(', ')}]`);
+  console.log('');
+  
+  // Verify all groups received all messages
+  const allReceived = 
+    received.analytics.length === testMessages.length &&
+    received.notifications.length === testMessages.length &&
+    received.audit.length === testMessages.length;
+  
+  if (allReceived) {
+    console.log('✅ SUCCESS: All consumer groups received ALL messages!');
+    console.log('   This demonstrates bus/pub-sub mode where each consumer group');
+    console.log('   gets its own copy of every message, independently.');
+  } else {
+    console.log('⚠️  Some messages were not received by all groups.');
+    console.log('   This might be due to timing. Try running again.');
+  }
+  
+  console.log('\n🎯 Key Differences from Queue Mode:');
+  console.log('  • In Queue Mode: Messages are consumed by ONE worker');
+  console.log('  • In Bus Mode: Messages are consumed by ALL consumer groups');
+  console.log('  • Each group maintains its own progress/offset');
+  console.log('  • Perfect for: analytics, notifications, audit logs, etc.');
+}
+
+// Demonstrate subscription modes
+async function demonstrateSubscriptionModes() {
+  const client = createQueenClient({
+    baseUrl: process.env.QUEEN_URL || 'http://localhost:6632'
+  });
+  
+  const queue = 'subscription-test';
+  
+  console.log('\n\n📅 Demonstrating Subscription Modes');
+  console.log('════════════════════════════════════════════');
+  
+  // Push some historical messages
+  console.log('📤 Pushing historical messages...');
+  for (let i = 1; i <= 3; i++) {
+    await client.push({
+      items: [{
+        queue,
+        payload: { id: i, type: 'historical', data: `Historical message ${i}` }
+      }]
+    });
+  }
+  
+  console.log('✅ Historical messages pushed\n');
+  
+  // Consumer 1: Consume all messages (default)
+  console.log('🔵 Consumer Group "all-messages" (default mode):');
+  const allMessages = [];
+  const consumer1 = client.consume({
+    queue,
+    consumerGroup: 'all-messages',
+    handler: async (msg) => {
+      console.log(`  Received: ${msg.data.type} - ${msg.data.data}`);
+      allMessages.push(msg.data.id);
+    },
+    options: { wait: false }
+  });
+  
+  await new Promise(resolve => setTimeout(resolve, 1000));
+  consumer1();
+  
+  // Consumer 2: Only new messages from subscription time
+  console.log('\n🟢 Consumer Group "new-only" (subscriptionMode: "new"):');
+  const newOnly = [];
+  const consumer2 = client.consume({
+    queue,
+    consumerGroup: 'new-only',
+    handler: async (msg) => {
+      console.log(`  Received: ${msg.data.type} - ${msg.data.data}`);
+      newOnly.push(msg.data.id);
+    },
+    options: { 
+      wait: false,
+      subscriptionMode: 'new'
+    }
+  });
+  
+  // Push new messages after subscription
+  console.log('\n📤 Pushing new messages...');
+  for (let i = 4; i <= 6; i++) {
+    await client.push({
+      items: [{
+        queue,
+        payload: { id: i, type: 'new', data: `New message ${i}` }
+      }]
+    });
+  }
+  
+  await new Promise(resolve => setTimeout(resolve, 1000));
+  consumer2();
+  
+  // Display results
+  console.log('\n📊 Subscription Mode Results:');
+  console.log('════════════════════════════════════');
+  console.log(`"all-messages" group received: [${allMessages.sort().join(', ')}] (all 6 messages)`);
+  console.log(`"new-only" group received: [${newOnly.sort().join(', ')}] (only new messages)`);
+  console.log('');
+  console.log('✅ This demonstrates how consumer groups can choose when to start consuming!');
+}
+
+// Run the examples
+main()
+  .then(() => demonstrateSubscriptionModes())
+  .then(() => {
+    console.log('\n✨ Bus mode examples completed!');
+    process.exit(0);
+  })
+  .catch(error => {
+    console.error('❌ Error:', error);
+    process.exit(1);
+  });