queen-mq 0.3.1 → 0.4.0

package/client-js/client-v2/stream/StreamConsumer.js

@@ -0,0 +1,156 @@
+import { Window } from './Window.js';
+
+/**
+ * StreamConsumer - Manages consuming windows from a stream
+ */
+export class StreamConsumer {
+  constructor(httpClient, queen, streamName, consumerGroup) {
+    this.httpClient = httpClient;
+    this.queen = queen;
+    this.streamName = streamName;
+    this.consumerGroup = consumerGroup;
+    this.pollTimeout = 30000;        // 30s long poll
+    this.leaseRenewInterval = 20000; // 20s renewal (before 60s timeout)
+    this.running = false;
+  }
+
+  /**
+   * Start processing windows with the provided callback
+   * Runs in an infinite loop until stopped
+   * @param {Function} callback - async function(window) to process each window
+   */
+  async process(callback) {
+    this.running = true;
+    
+    
+    while (this.running) {
+      let window = null;
+      
+      try {
+        window = await this.pollWindow();
+
+        if (!window) {
+          // 204 No Content - no window available
+          continue;
+        }
+
+        await this.executeCallback(window, callback);
+
+      } catch (err) {
+        // Backoff on error
+        await new Promise(r => setTimeout(r, 1000));
+      }
+    }
+  }
+
+  /**
+   * Stop the processing loop
+   */
+  stop() {
+    this.running = false;
+  }
+
+  /**
+   * Poll for a window (blocking call)
+   * @returns {Promise<Window|null>} Window or null if no content
+   */
+  async pollWindow() {
+    try {
+      const response = await this.httpClient.post('/api/v1/stream/poll', {
+        streamName: this.streamName,
+        consumerGroup: this.consumerGroup,
+        timeout: this.pollTimeout
+      });
+      
+      // Handle 204 No Content (HttpClient returns null for 204)
+      if (!response) {
+        return null;
+      }
+      
+      // HttpClient returns the JSON body directly (e.g., {"window": {...}})
+      if (!response.window) {
+        return null;
+      }
+      
+      return new Window(response.window);
+      
+    } catch (err) {
+      // HttpClient throws on errors, returns null on 204
+      // So any error here is a real network/server error
+      throw err;
+    }
+  }
+
+  /**
+   * Execute the callback with lease renewal
+   * @param {Window} window - The window to process
+   * @param {Function} callback - User callback
+   */
+  async executeCallback(window, callback) {
+    let leaseTimer = null;
+    let leaseExpired = false;
+    
+    try {
+      // Start lease renewal timer
+      leaseTimer = setInterval(async () => {
+        try {
+          await this.httpClient.post('/api/v1/stream/renew-lease', { 
+            leaseId: window.leaseId,
+            extend_ms: this.leaseRenewInterval + 10000  // Extend by renewal interval + buffer
+          });
+        } catch (e) {
+          leaseExpired = true;
+        }
+      }, this.leaseRenewInterval);
+
+      // Execute user callback
+      await callback(window);
+      
+      // ACK if lease hasn't expired
+      if (!leaseExpired) {
+        await this.httpClient.post('/api/v1/stream/ack', {
+          windowId: window.id,
+          leaseId: window.leaseId,
+          success: true
+        });
+      }
+
+    } catch (err) {
+      
+      // NACK if lease hasn't expired
+      if (!leaseExpired) {
+        try {
+          await this.httpClient.post('/api/v1/stream/ack', {
+            windowId: window.id,
+            leaseId: window.leaseId,
+            success: false  // NACK
+          });
+        } catch (nackErr) {
+          
+        }
+      }
+      
+      // Re-throw to trigger backoff
+      throw err;
+      
+    } finally {
+      // Stop lease renewal
+      if (leaseTimer) {
+        clearInterval(leaseTimer);
+      }
+    }
+  }
+
+  /**
+   * Seek to a specific timestamp
+   * @param {string} timestamp - ISO timestamp to seek to
+   */
+  async seek(timestamp) {
+    return this.httpClient.post('/api/v1/stream/seek', {
+      streamName: this.streamName,
+      consumerGroup: this.consumerGroup,
+      timestamp
+    });
+  }
+}
+

package/client-js/client-v2/stream/Window.js

@@ -0,0 +1,140 @@
+/**
+ * Utility function to get nested property value from object using dot notation
+ */
+const getPath = (obj, path) => 
+  path.split('.').reduce((o, k) => (o && o[k] !== undefined ? o[k] : null), obj);
+
+/**
+ * Window - Represents a time window of messages with utility methods
+ */
+export class Window {
+  constructor(rawWindow) {
+    // Copy all properties from raw window
+    Object.assign(this, rawWindow);
+    
+    // Store immutable original messages
+    this.allMessages = Object.freeze(rawWindow.messages || []);
+    
+    // Working copy for transformations
+    this.messages = [...this.allMessages];
+  }
+
+  /**
+   * Filter messages based on a predicate function
+   * @param {Function} filterFn - Predicate function (msg) => boolean
+   * @returns {Window} this for chaining
+   */
+  filter(filterFn) {
+    this.messages = this.messages.filter(filterFn);
+    return this;
+  }
+
+  /**
+   * Group messages by a key path (dot notation supported)
+   * @param {string} keyPath - Path to key in message data (e.g., 'data.userId')
+   * @returns {Object} Object with keys as group names and values as arrays of messages
+   */
+  groupBy(keyPath) {
+    const groups = {};
+    for (const msg of this.messages) {
+      const key = getPath(msg, keyPath) || 'null_key';
+      if (!groups[key]) {
+        groups[key] = [];
+      }
+      groups[key].push(msg);
+    }
+    return groups;
+  }
+
+  /**
+   * Aggregate messages using various aggregation functions
+   * @param {Object} config - Aggregation configuration
+   * @param {boolean} config.count - Count messages
+   * @param {string[]} config.sum - Array of paths to sum
+   * @param {string[]} config.avg - Array of paths to average
+   * @param {string[]} config.min - Array of paths to find minimum
+   * @param {string[]} config.max - Array of paths to find maximum
+   * @returns {Object} Aggregation results
+   */
+  aggregate(config = {}) {
+    const results = {};
+
+    // Count
+    if (config.count) {
+      results.count = this.messages.length;
+    }
+
+    // Sum
+    if (config.sum) {
+      results.sum = {};
+      for (const path of config.sum) {
+        results.sum[path] = this.messages.reduce((total, msg) => {
+          const val = getPath(msg, path);
+          return total + (typeof val === 'number' ? val : 0);
+        }, 0);
+      }
+    }
+
+    // Average
+    if (config.avg) {
+      results.avg = {};
+      for (const path of config.avg) {
+        const sum = this.messages.reduce((total, msg) => {
+          const val = getPath(msg, path);
+          return total + (typeof val === 'number' ? val : 0);
+        }, 0);
+        results.avg[path] = this.messages.length > 0 ? sum / this.messages.length : 0;
+      }
+    }
+
+    // Min
+    if (config.min) {
+      results.min = {};
+      for (const path of config.min) {
+        const values = this.messages
+          .map(msg => getPath(msg, path))
+          .filter(val => typeof val === 'number');
+        results.min[path] = values.length > 0 ? Math.min(...values) : null;
+      }
+    }
+
+    // Max
+    if (config.max) {
+      results.max = {};
+      for (const path of config.max) {
+        const values = this.messages
+          .map(msg => getPath(msg, path))
+          .filter(val => typeof val === 'number');
+        results.max[path] = values.length > 0 ? Math.max(...values) : null;
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Reset working messages to the original frozen set
+   * @returns {Window} this for chaining
+   */
+  reset() {
+    this.messages = [...this.allMessages];
+    return this;
+  }
+
+  /**
+   * Get count of messages in working set
+   * @returns {number}
+   */
+  size() {
+    return this.messages.length;
+  }
+
+  /**
+   * Get count of original messages
+   * @returns {number}
+   */
+  originalSize() {
+    return this.allMessages.length;
+  }
+}
+

package/client-js/test-v2/MAINTENANCE_TEST.md

@@ -0,0 +1,148 @@
+# Maintenance Mode Test
+
+## Overview
+
+Comprehensive test for the maintenance mode feature with file buffer.
+
+## Test Scenario
+
+1. **Start consumer** - Begins consuming messages from queue
+2. **Start producer** - Produces 10 messages/second
+3. **Enable maintenance** - Triggers maintenance mode via API
+4. **Verify buffering** - Consumer stops receiving (messages go to file buffer)
+5. **Continue producing** - Producer keeps running for 10 seconds during maintenance
+6. **Disable maintenance** - Deactivates maintenance mode via API
+7. **Verify drain** - Consumer resumes receiving messages
+8. **Stop producer** - Halts message production
+9. **Wait for completion** - Waits 10 seconds for file buffer to drain
+10. **Verify counts** - Ensures all produced messages were received
+
+## Running the Test
+
+### Standalone
+
+```bash
+cd /Users/alice/Work/queen/client-js/test-v2
+node maintenance.js
+```
+
+### Via Test Runner
+
+```bash
+# Run just this test
+node run.js test_maintenance_mode
+
+# Run all human tests (includes this one)
+node run.js human
+
+# Run all tests
+node run.js
+```
+
+## Expected Output
+
+```
+🧪 Testing Maintenance Mode with File Buffer
+
+📋 Step 1: Configuring queue "test-maintenance-queue"...
+✅ Queue configured
+
+📥 Step 2: Starting consumer...
+✅ Consumer started
+
+📤 Step 3: Starting producer (10 msgs/sec)...
+✅ Producer started
+
+⏱️  Step 4: Waiting 3 seconds for normal message flow...
+  📨 Received 10 messages (total: 10)
+  📨 Received 10 messages (total: 20)
+  📨 Received 10 messages (total: 30)
+   Produced: 30, Received: 30
+
+🔧 Step 5: Enabling MAINTENANCE MODE...
+   Response: { maintenanceMode: true, bufferedMessages: 0, ... }
+✅ Maintenance mode enabled
+
+⏱️  Step 6: Waiting 2 seconds - consumer should stop receiving...
+   Messages received during maintenance: 0
+   Total: Produced=50, Received=30
+
+⏱️  Step 7: Producing during maintenance for 10 seconds...
+   (Messages should go to file buffer)
+
+   Maintenance period complete:
+   - Total produced: 150
+   - Total received: 30
+   - Buffered (should be ~100): 120
+
+📊 Step 8: Checking maintenance status...
+   Status: { maintenanceMode: true, bufferedMessages: 120, ... }
+
+✅ Step 9: Disabling MAINTENANCE MODE...
+   Response: { maintenanceMode: false, bufferedMessages: 120, ... }
+   (File buffer should start draining to database)
+
+⏱️  Step 10: Waiting for messages to resume...
+  📨 Received 10 messages (total: 40)
+   Messages received after resuming: 10
+✅ Messages are flowing again!
+
+🛑 Step 11: Stopping producer...
+   Final produced count: 150
+
+⏱️  Step 12: Waiting 10 seconds for file buffer to drain...
+   1s - Received: 50/150 (33.3%)
+  📨 Received 10 messages (total: 60)
+   2s - Received: 70/150 (46.7%)
+  📨 Received 10 messages (total: 80)
+   3s - Received: 90/150 (60.0%)
+  📨 Received 10 messages (total: 100)
+   4s - Received: 110/150 (73.3%)
+  📨 Received 10 messages (total: 120)
+   5s - Received: 130/150 (86.7%)
+  📨 Received 10 messages (total: 140)
+   6s - Received: 150/150 (100.0%)
+   ✅ All messages received!
+
+🛑 Step 13: Stopping consumer...
+✅ Consumer stopped
+
+📊 Final Verification:
+
+   Total Produced: 150
+   Total Received: 150
+   Difference: 0
+
+✅ SUCCESS: All messages accounted for!
+   Maintenance mode works correctly with file buffer.
+
+👋 Queen client closed
+
+✅ All tests passed!
+```
+
+## What It Tests
+
+✅ **Maintenance mode activation** - API responds correctly
+✅ **Message buffering** - Messages go to file buffer (consumer stops receiving)
+✅ **Continued production** - Can push messages during maintenance
+✅ **Maintenance mode deactivation** - API responds correctly
+✅ **Automatic drain** - File buffer drains to database
+✅ **Message ordering** - All messages received in order (FIFO)
+✅ **No message loss** - Produced count == Received count
+✅ **Multi-instance support** - Uses database for state (works across restarts)
+
+## Success Criteria
+
+- All produced messages are eventually received
+- Consumer stops receiving during maintenance
+- Consumer resumes receiving after maintenance is disabled
+- No duplicates or lost messages
+- Drain completes within 10 seconds
+
+## Notes
+
+- Uses `axios` for direct API calls (maintenance endpoints)
+- Uses Queen client for queue operations (push/pop/ack)
+- Test takes ~25 seconds total (3s warmup + 10s maintenance + 10s drain)
+

package/client-js/test-v2/consume.js

@@ -190,6 +190,7 @@ export async function testConsumerOrdering(client) {
         await client
         .queue('test-queue-v2-consume-batch')
         .push([{ data: { id: i } }])
+        console.log(new Date().toISOString(), 'Pushed message:', i)
     }