queen-mq 0.3.1 → 0.6.3

package/client-js/client-v2/README.md

@@ -28,8 +28,12 @@ Welcome to Queen client! This is your friendly guide to mastering message queues
 
 First, install and import:
 
+```sh
+npm install queen-mq
+```
+
 ```javascript
-import { Queen } from './client-js/client-v2/index.js'
+import { Queen } from 'queen-mq'
 
 // Connect to your Queen server
 const queen = new Queen('http://localhost:6632')
@@ -366,12 +370,21 @@ await queen
   })
 ```
 
+**Server Default:** The server can be configured to change this default behavior:
+```bash
+# Make all new consumer groups skip history by default
+export DEFAULT_SUBSCRIPTION_MODE="new"
+./bin/queen-server
+```
+
+When `DEFAULT_SUBSCRIPTION_MODE="new"` is set, new consumer groups automatically skip historical messages unless you explicitly override with `.subscriptionMode('all')`.
+
 ### Subscription Mode: 'new'
 
-Skip all historical messages and only process messages that arrive **after** subscription:
+Skip historical messages and process messages that arrive **near** subscription time:
 
 ```javascript
-// Only process NEW messages, skip historical backlog
+// Process recent messages (not historical backlog)
 await queen
   .queue('events')
   .group('realtime-monitor')
@@ -382,9 +395,33 @@ await queen
 ```
 
 **What happens:**
-1. Consumer subscribes at `T0`
-2. All messages before `T0` are skipped
-3. Only messages arriving after `T0` are processed
+1. Consumer makes first pop at `T0` (e.g., 10:00:00)
+2. Server records `subscription_timestamp = T0` in metadata table
+3. Only messages with `created_at > T0` are processed
+4. All historical messages are skipped
+
+**How it ensures consistency across partitions:**
+
+Queen tracks subscription time separately from partition-level cursors:
+
+```javascript
+// Timeline:
+10:00:00 - First pop() call
+         → Metadata recorded: subscription_timestamp = 10:00:00
+
+// Consumer processes partition P1 for 10 minutes
+
+10:10:00 - New partition P2 is created, messages arrive
+10:15:00 - Consumer discovers P2 via pop()
+         → Uses ORIGINAL subscription_timestamp (10:00:00)
+         → Messages from 10:10:00 are captured! ✓
+```
+
+**Key benefits:**
+- ✅ **Consistent**: All partitions use the same subscription timestamp
+- ✅ **No skipping**: New partitions discovered later are processed correctly
+- ✅ **True NEW semantics**: Only messages after first pop request
+- ✅ **Works with wildcards**: Namespace/task filters maintain subscription time
 
 ### Subscription Mode: 'new-only'
 
@@ -493,11 +530,18 @@ await queen
 - Subsequent consumers in the same group inherit the same position
 - To change subscription mode, use a different group name
 
+⏰ **NEW mode subscription tracking:**
+- NEW mode tracks when the consumer group **first subscribes** (first pop request)
+- This subscription timestamp is used consistently across all partitions
+- Ensures new partitions discovered later don't skip messages
+- Stored in `consumer_groups_metadata` table on the server
+
 💡 **Best Practices:**
-- Use `'new'` for real-time monitoring and alerting
-- Use default (all) for batch processing and analytics
-- Use timestamps for replay/debugging scenarios
+- Use `'new'` for real-time monitoring (skip historical backlog)
+- Use default (all) for batch processing and full history replay
+- Use timestamps for precise replay/debugging scenarios
 - Name groups descriptively based on their subscription mode
+- Be aware that "NEW" means messages after the **first pop request**, not the first message arrival
 
 ---
 
@@ -1656,6 +1700,19 @@ const dlq = await queen.queue('q').dlq('consumer-group').limit(10).get()
 const dlq = await queen.queue('q').dlq().from('2025-01-01').to('2025-01-31').get()
 ```
 
+### Consumer Group Management
+
+```javascript
+// Delete a consumer group (including metadata)
+await queen.deleteConsumerGroup('my-group')
+
+// Delete consumer group but keep subscription metadata
+await queen.deleteConsumerGroup('my-group', false)
+
+// Update subscription timestamp
+await queen.updateConsumerGroupTimestamp('my-group', '2025-11-10T10:00:00Z')
+```
+
 ### Shutdown
 
 ```javascript
@@ -1874,7 +1931,7 @@ process.on('SIGINT', async () => {
 You now know everything about Queen v2! 🎉
 
 **Additional resources:**
-- [API Documentation](../../API.md) - Complete API reference
+- [API Documentation](../../server/API.md) - Complete API reference
 - [Test Examples](../test-v2/) - 94 working test cases
 - [Architecture Guide](../../docs/) - Deep dive into Queen's internals
 

package/client-js/client-v2/builders/QueueBuilder.js

@@ -2,7 +2,13 @@
  * Queue builder for fluent API
  */
 
-import { generateUUID } from '../../utils/uuid.js'
+import { v7 as uuidv7 } from 'uuid';
+
+export const generateUUID = () => {
+  return uuidv7();
+};
+
+//import { generateUUID } from '../../utils/uuid.js'
 import { isValidUUID } from '../utils/validation.js'
 import { QUEUE_DEFAULTS, CONSUME_DEFAULTS, POP_DEFAULTS } from '../utils/defaults.js'
 import * as logger from '../utils/logger.js'

package/client-js/client-v2/builders/TransactionBuilder.js

@@ -48,12 +48,19 @@ export class TransactionBuilder {
   }
 
   queue(queueName) {
-    // Return a sub-builder for push operations
-    return {
+    // Return a sub-builder for push operations with partition support
+    let partition = null
+    
+    const subBuilder = {
+      partition: (partitionKey) => {
+        partition = partitionKey
+        return subBuilder
+      },
+      
       push: (items) => {
         const itemArray = Array.isArray(items) ? items : [items]
         
-        logger.log('TransactionBuilder.queue.push', { queue: queueName, count: itemArray.length })
+        logger.log('TransactionBuilder.queue.push', { queue: queueName, partition, count: itemArray.length })
 
         this.#operations.push({
           type: 'push',
@@ -68,16 +75,25 @@ export class TransactionBuilder {
               payloadValue = item
             }
 
-            return {
+            const result = {
               queue: queueName,
               payload: payloadValue
             }
+            
+            // Add partition if set
+            if (partition !== null) {
+              result.partition = partition
+            }
+
+            return result
           })
         })
 
         return this
       }
     }
+    
+    return subBuilder
   }
 
   async commit() {

package/client-js/client-v2/stream/StreamBuilder.js

@@ -0,0 +1,72 @@
+/**
+ * StreamBuilder - Fluent API for defining streams
+ */
+export class StreamBuilder {
+  constructor(httpClient, queen, name, namespace) {
+    this.httpClient = httpClient;
+    this.queen = queen;
+    this.config = {
+      name,
+      namespace,
+      source_queue_names: [],
+      partitioned: false,
+      window_type: 'tumbling',
+      window_duration_ms: 60000,       // 1 minute default
+      window_grace_period_ms: 30000,   // 30 seconds default
+      window_lease_timeout_ms: 60000   // 1 minute default
+    };
+  }
+
+  /**
+   * Set the source queues for this stream
+   * @param {string[]} queueNames - Array of queue names
+   */
+  sources(queueNames = []) {
+    this.config.source_queue_names = queueNames;
+    return this;
+  }
+  
+  /**
+   * Enable partitioned processing (group by partition_id)
+   */
+  partitioned() {
+    this.config.partitioned = true;
+    return this;
+  }
+
+  /**
+   * Configure tumbling time window
+   * @param {number} seconds - Window duration in seconds
+   */
+  tumblingTime(seconds) {
+    this.config.window_type = 'tumbling';
+    this.config.window_duration_ms = seconds * 1000;
+    return this;
+  }
+  
+  /**
+   * Configure grace period for late-arriving messages
+   * @param {number} seconds - Grace period in seconds
+   */
+  gracePeriod(seconds) {
+    this.config.window_grace_period_ms = seconds * 1000;
+    return this;
+  }
+  
+  /**
+   * Configure lease timeout
+   * @param {number} seconds - Lease timeout in seconds
+   */
+  leaseTimeout(seconds) {
+    this.config.window_lease_timeout_ms = seconds * 1000;
+    return this;
+  }
+
+  /**
+   * Define/create the stream on the server
+   */
+  async define() {
+    return this.httpClient.post('/api/v1/stream/define', this.config);
+  }
+}
+

package/client-js/client-v2/stream/StreamConsumer.js

@@ -0,0 +1,156 @@
+import { Window } from './Window.js';
+
+/**
+ * StreamConsumer - Manages consuming windows from a stream
+ */
+export class StreamConsumer {
+  constructor(httpClient, queen, streamName, consumerGroup) {
+    this.httpClient = httpClient;
+    this.queen = queen;
+    this.streamName = streamName;
+    this.consumerGroup = consumerGroup;
+    this.pollTimeout = 30000;        // 30s long poll
+    this.leaseRenewInterval = 20000; // 20s renewal (before 60s timeout)
+    this.running = false;
+  }
+
+  /**
+   * Start processing windows with the provided callback
+   * Runs in an infinite loop until stopped
+   * @param {Function} callback - async function(window) to process each window
+   */
+  async process(callback) {
+    this.running = true;
+    
+    
+    while (this.running) {
+      let window = null;
+      
+      try {
+        window = await this.pollWindow();
+
+        if (!window) {
+          // 204 No Content - no window available
+          continue;
+        }
+
+        await this.executeCallback(window, callback);
+
+      } catch (err) {
+        // Backoff on error
+        await new Promise(r => setTimeout(r, 1000));
+      }
+    }
+  }
+
+  /**
+   * Stop the processing loop
+   */
+  stop() {
+    this.running = false;
+  }
+
+  /**
+   * Poll for a window (blocking call)
+   * @returns {Promise<Window|null>} Window or null if no content
+   */
+  async pollWindow() {
+    try {
+      const response = await this.httpClient.post('/api/v1/stream/poll', {
+        streamName: this.streamName,
+        consumerGroup: this.consumerGroup,
+        timeout: this.pollTimeout
+      });
+      
+      // Handle 204 No Content (HttpClient returns null for 204)
+      if (!response) {
+        return null;
+      }
+      
+      // HttpClient returns the JSON body directly (e.g., {"window": {...}})
+      if (!response.window) {
+        return null;
+      }
+      
+      return new Window(response.window);
+      
+    } catch (err) {
+      // HttpClient throws on errors, returns null on 204
+      // So any error here is a real network/server error
+      throw err;
+    }
+  }
+
+  /**
+   * Execute the callback with lease renewal
+   * @param {Window} window - The window to process
+   * @param {Function} callback - User callback
+   */
+  async executeCallback(window, callback) {
+    let leaseTimer = null;
+    let leaseExpired = false;
+    
+    try {
+      // Start lease renewal timer
+      leaseTimer = setInterval(async () => {
+        try {
+          await this.httpClient.post('/api/v1/stream/renew-lease', { 
+            leaseId: window.leaseId,
+            extend_ms: this.leaseRenewInterval + 10000  // Extend by renewal interval + buffer
+          });
+        } catch (e) {
+          leaseExpired = true;
+        }
+      }, this.leaseRenewInterval);
+
+      // Execute user callback
+      await callback(window);
+      
+      // ACK if lease hasn't expired
+      if (!leaseExpired) {
+        await this.httpClient.post('/api/v1/stream/ack', {
+          windowId: window.id,
+          leaseId: window.leaseId,
+          success: true
+        });
+      }
+
+    } catch (err) {
+      
+      // NACK if lease hasn't expired
+      if (!leaseExpired) {
+        try {
+          await this.httpClient.post('/api/v1/stream/ack', {
+            windowId: window.id,
+            leaseId: window.leaseId,
+            success: false  // NACK
+          });
+        } catch (nackErr) {
+          
+        }
+      }
+      
+      // Re-throw to trigger backoff
+      throw err;
+      
+    } finally {
+      // Stop lease renewal
+      if (leaseTimer) {
+        clearInterval(leaseTimer);
+      }
+    }
+  }
+
+  /**
+   * Seek to a specific timestamp
+   * @param {string} timestamp - ISO timestamp to seek to
+   */
+  async seek(timestamp) {
+    return this.httpClient.post('/api/v1/stream/seek', {
+      streamName: this.streamName,
+      consumerGroup: this.consumerGroup,
+      timestamp
+    });
+  }
+}
+

package/client-js/client-v2/stream/Window.js

@@ -0,0 +1,140 @@
+/**
+ * Utility function to get nested property value from object using dot notation
+ */
+const getPath = (obj, path) => 
+  path.split('.').reduce((o, k) => (o && o[k] !== undefined ? o[k] : null), obj);
+
+/**
+ * Window - Represents a time window of messages with utility methods
+ */
+export class Window {
+  constructor(rawWindow) {
+    // Copy all properties from raw window
+    Object.assign(this, rawWindow);
+    
+    // Store immutable original messages
+    this.allMessages = Object.freeze(rawWindow.messages || []);
+    
+    // Working copy for transformations
+    this.messages = [...this.allMessages];
+  }
+
+  /**
+   * Filter messages based on a predicate function
+   * @param {Function} filterFn - Predicate function (msg) => boolean
+   * @returns {Window} this for chaining
+   */
+  filter(filterFn) {
+    this.messages = this.messages.filter(filterFn);
+    return this;
+  }
+
+  /**
+   * Group messages by a key path (dot notation supported)
+   * @param {string} keyPath - Path to key in message data (e.g., 'data.userId')
+   * @returns {Object} Object with keys as group names and values as arrays of messages
+   */
+  groupBy(keyPath) {
+    const groups = {};
+    for (const msg of this.messages) {
+      const key = getPath(msg, keyPath) || 'null_key';
+      if (!groups[key]) {
+        groups[key] = [];
+      }
+      groups[key].push(msg);
+    }
+    return groups;
+  }
+
+  /**
+   * Aggregate messages using various aggregation functions
+   * @param {Object} config - Aggregation configuration
+   * @param {boolean} config.count - Count messages
+   * @param {string[]} config.sum - Array of paths to sum
+   * @param {string[]} config.avg - Array of paths to average
+   * @param {string[]} config.min - Array of paths to find minimum
+   * @param {string[]} config.max - Array of paths to find maximum
+   * @returns {Object} Aggregation results
+   */
+  aggregate(config = {}) {
+    const results = {};
+
+    // Count
+    if (config.count) {
+      results.count = this.messages.length;
+    }
+
+    // Sum
+    if (config.sum) {
+      results.sum = {};
+      for (const path of config.sum) {
+        results.sum[path] = this.messages.reduce((total, msg) => {
+          const val = getPath(msg, path);
+          return total + (typeof val === 'number' ? val : 0);
+        }, 0);
+      }
+    }
+
+    // Average
+    if (config.avg) {
+      results.avg = {};
+      for (const path of config.avg) {
+        const sum = this.messages.reduce((total, msg) => {
+          const val = getPath(msg, path);
+          return total + (typeof val === 'number' ? val : 0);
+        }, 0);
+        results.avg[path] = this.messages.length > 0 ? sum / this.messages.length : 0;
+      }
+    }
+
+    // Min
+    if (config.min) {
+      results.min = {};
+      for (const path of config.min) {
+        const values = this.messages
+          .map(msg => getPath(msg, path))
+          .filter(val => typeof val === 'number');
+        results.min[path] = values.length > 0 ? Math.min(...values) : null;
+      }
+    }
+
+    // Max
+    if (config.max) {
+      results.max = {};
+      for (const path of config.max) {
+        const values = this.messages
+          .map(msg => getPath(msg, path))
+          .filter(val => typeof val === 'number');
+        results.max[path] = values.length > 0 ? Math.max(...values) : null;
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Reset working messages to the original frozen set
+   * @returns {Window} this for chaining
+   */
+  reset() {
+    this.messages = [...this.allMessages];
+    return this;
+  }
+
+  /**
+   * Get count of messages in working set
+   * @returns {number}
+   */
+  size() {
+    return this.messages.length;
+  }
+
+  /**
+   * Get count of original messages
+   * @returns {number}
+   */
+  originalSize() {
+    return this.allMessages.length;
+  }
+}
+

package/client-js/test-v2/GETTING_STARTED.md

@@ -119,6 +119,33 @@ Make sure:
 3. ✅ Environment variables are set (if needed)
 4. ✅ No production data in test database
 
+## Testing with Different Server Configurations
+
+### Standard Configuration (Default)
+
+Run server with default settings:
+```bash
+./bin/queen-server
+node test-v2/run.js
+```
+
+**Expected:** Consumer groups without explicit `.subscriptionMode()` process all historical messages.
+
+### With DEFAULT_SUBSCRIPTION_MODE="new"
+
+Run server with "new" as default:
+```bash
+DEFAULT_SUBSCRIPTION_MODE="new" ./bin/queen-server
+node test-v2/run.js
+```
+
+**Expected:** 
+- Tests with explicit `.subscriptionMode('new')` work the same
+- Tests without explicit mode will skip historical messages
+- `subscriptionModeServerDefault` test detects and reports server default
+
+**All tests pass with both configurations!** The tests are designed to be agnostic to server defaults.
+
 ## Test Philosophy
 
 The AI-generated tests follow these principles: