queen-mq 0.4.0 → 0.6.4

package/{client-js/client-v2 → client-v2}/Queen.js

@@ -357,6 +357,44 @@ export class Queen {
     return stats
   }
 
+  // ===========================
+  // Consumer Group Management
+  // ===========================
+
+  /**
+   * Delete a consumer group and optionally its subscription metadata
+   * @param {string} consumerGroup - Consumer group name
+   * @param {boolean} deleteMetadata - Whether to delete subscription metadata (default: true)
+   * @returns {Promise<object>}
+   */
+  async deleteConsumerGroup(consumerGroup, deleteMetadata = true) {
+    logger.log('Queen.deleteConsumerGroup', { consumerGroup, deleteMetadata })
+    
+    const url = `/api/v1/consumer-groups/${encodeURIComponent(consumerGroup)}?deleteMetadata=${deleteMetadata}`
+    const response = await this.#httpClient.delete(url)
+    
+    logger.log('Queen.deleteConsumerGroup', { success: true, consumerGroup })
+    return response
+  }
+
+  /**
+   * Update subscription timestamp for a consumer group
+   * @param {string} consumerGroup - Consumer group name
+   * @param {string} timestamp - New subscription timestamp (ISO 8601)
+   * @returns {Promise<object>}
+   */
+  async updateConsumerGroupTimestamp(consumerGroup, timestamp) {
+    logger.log('Queen.updateConsumerGroupTimestamp', { consumerGroup, timestamp })
+    
+    const url = `/api/v1/consumer-groups/${encodeURIComponent(consumerGroup)}/subscription`
+    const response = await this.#httpClient.post(url, {
+      subscriptionTimestamp: timestamp
+    })
+    
+    logger.log('Queen.updateConsumerGroupTimestamp', { success: true, consumerGroup })
+    return response
+  }
+
   // ===========================
   // Streaming API
   // ===========================

package/{client-js/client-v2 → client-v2}/README.md

@@ -370,12 +370,21 @@ await queen
   })
 ```
 
+**Server Default:** The server can be configured to change this default behavior:
+```bash
+# Make all new consumer groups skip history by default
+export DEFAULT_SUBSCRIPTION_MODE="new"
+./bin/queen-server
+```
+
+When `DEFAULT_SUBSCRIPTION_MODE="new"` is set, new consumer groups automatically skip historical messages unless you explicitly override with `.subscriptionMode('all')`.
+
 ### Subscription Mode: 'new'
 
-Skip all historical messages and only process messages that arrive **after** subscription:
+Skip historical messages and process messages that arrive **near** subscription time:
 
 ```javascript
-// Only process NEW messages, skip historical backlog
+// Process recent messages (not historical backlog)
 await queen
   .queue('events')
   .group('realtime-monitor')
@@ -386,9 +395,33 @@ await queen
 ```
 
 **What happens:**
-1. Consumer subscribes at `T0`
-2. All messages before `T0` are skipped
-3. Only messages arriving after `T0` are processed
+1. Consumer makes first pop at `T0` (e.g., 10:00:00)
+2. Server records `subscription_timestamp = T0` in metadata table
+3. Only messages with `created_at > T0` are processed
+4. All historical messages are skipped
+
+**How it ensures consistency across partitions:**
+
+Queen tracks subscription time separately from partition-level cursors:
+
+```javascript
+// Timeline:
+10:00:00 - First pop() call
+         → Metadata recorded: subscription_timestamp = 10:00:00
+
+// Consumer processes partition P1 for 10 minutes
+
+10:10:00 - New partition P2 is created, messages arrive
+10:15:00 - Consumer discovers P2 via pop()
+         → Uses ORIGINAL subscription_timestamp (10:00:00)
+         → Messages from 10:10:00 are captured! ✓
+```
+
+**Key benefits:**
+- ✅ **Consistent**: All partitions use the same subscription timestamp
+- ✅ **No skipping**: New partitions discovered later are processed correctly
+- ✅ **True NEW semantics**: Only messages after first pop request
+- ✅ **Works with wildcards**: Namespace/task filters maintain subscription time
 
 ### Subscription Mode: 'new-only'
 
@@ -497,11 +530,18 @@ await queen
 - Subsequent consumers in the same group inherit the same position
 - To change subscription mode, use a different group name
 
+⏰ **NEW mode subscription tracking:**
+- NEW mode tracks when the consumer group **first subscribes** (first pop request)
+- This subscription timestamp is used consistently across all partitions
+- Ensures new partitions discovered later don't skip messages
+- Stored in `consumer_groups_metadata` table on the server
+
 💡 **Best Practices:**
-- Use `'new'` for real-time monitoring and alerting
-- Use default (all) for batch processing and analytics
-- Use timestamps for replay/debugging scenarios
+- Use `'new'` for real-time monitoring (skip historical backlog)
+- Use default (all) for batch processing and full history replay
+- Use timestamps for precise replay/debugging scenarios
 - Name groups descriptively based on their subscription mode
+- Be aware that "NEW" means messages after the **first pop request**, not the first message arrival
 
 ---
 
@@ -1660,6 +1700,19 @@ const dlq = await queen.queue('q').dlq('consumer-group').limit(10).get()
 const dlq = await queen.queue('q').dlq().from('2025-01-01').to('2025-01-31').get()
 ```
 
+### Consumer Group Management
+
+```javascript
+// Delete a consumer group (including metadata)
+await queen.deleteConsumerGroup('my-group')
+
+// Delete consumer group but keep subscription metadata
+await queen.deleteConsumerGroup('my-group', false)
+
+// Update subscription timestamp
+await queen.updateConsumerGroupTimestamp('my-group', '2025-11-10T10:00:00Z')
+```
+
 ### Shutdown
 
 ```javascript

package/package.json

@@ -1,18 +1,15 @@
 {
   "name": "queen-mq",
-  "version": "0.4.0",
+  "version": "0.6.4",
   "type": "module",
   "description": "High-performance C++ message queue backed by PostgreSQL",
-  "main": "client-js/client-v2/index.js",
+  "main": "client-v2/index.js",
   "scripts": {
-    "start": "./server/bin/queen-server",
-    "build:webapp": "cd webapp && npm install && npm run build",
-    "publish:public": "npm publish --access public",
-    "test": "node client-js/test/test-new.js"
+    "test": "node test-v2/run.js human"
   },
   "files": [
-    "client-js/",
-    "LICENSE.md",
+    "client-v2/**/*", 
+    "test-v2/**/*",
     "README.md"
   ],
   "dependencies": {

package/{client-js/test-v2 → test-v2}/GETTING_STARTED.md

@@ -119,6 +119,33 @@ Make sure:
 3. ✅ Environment variables are set (if needed)
 4. ✅ No production data in test database
 
+## Testing with Different Server Configurations
+
+### Standard Configuration (Default)
+
+Run server with default settings:
+```bash
+./bin/queen-server
+node test-v2/run.js
+```
+
+**Expected:** Consumer groups without explicit `.subscriptionMode()` process all historical messages.
+
+### With DEFAULT_SUBSCRIPTION_MODE="new"
+
+Run server with "new" as default:
+```bash
+DEFAULT_SUBSCRIPTION_MODE="new" ./bin/queen-server
+node test-v2/run.js
+```
+
+**Expected:** 
+- Tests with explicit `.subscriptionMode('new')` work the same
+- Tests without explicit mode will skip historical messages
+- `subscriptionModeServerDefault` test detects and reports server default
+
+**All tests pass with both configurations!** The tests are designed to be agnostic to server defaults.
+
 ## Test Philosophy
 
 The AI-generated tests follow these principles:

package/test-v2/README_SUBSCRIPTION_TESTS.md

@@ -0,0 +1,201 @@
+# Subscription Mode Tests
+
+Tests for consumer group subscription modes, compatible with any server `DEFAULT_SUBSCRIPTION_MODE` configuration.
+
+## Test Overview
+
+| Test Function | Description | Explicit Mode Used |
+|--------------|-------------|-------------------|
+| `subscriptionModeNew` | Validates `.subscriptionMode('new')` skips historical messages | Yes (`new`) |
+| `subscriptionModeNewOnly` | Validates alias `.subscriptionMode('new-only')` | Yes (`new-only`) |
+| `subscriptionFromNow` | Validates `.subscriptionFrom('now')` | Yes (`now`) |
+| `subscriptionFromTimestamp` | Validates timestamp-based subscription | Yes (timestamp) |
+| `subscriptionModeAll` | Tests default behavior (depends on server config) | Mixed |
+| `subscriptionModeServerDefault` | **NEW:** Detects and validates server default | Mixed |
+
+## Running the Tests
+
+### Run all subscription tests:
+```bash
+cd client-js/test-v2
+node run.js subscription
+```
+
+### Run specific subscription test:
+```bash
+node run.js subscriptionModeNew
+node run.js subscriptionModeServerDefault
+```
+
+## Server Configuration Compatibility
+
+These tests work with **any** server `DEFAULT_SUBSCRIPTION_MODE` configuration:
+
+### Configuration 1: Standard (Default)
+
+```bash
+# Start server without default
+./bin/queen-server
+
+# Run tests
+node run.js subscription
+```
+
+**Expected Behavior:**
+- Consumer groups without explicit mode process all historical messages
+- Consumer groups with `.subscriptionMode('new')` skip history
+- `subscriptionModeServerDefault` reports: "all (or empty string)"
+
+### Configuration 2: With DEFAULT_SUBSCRIPTION_MODE="new"
+
+```bash
+# Start server with "new" as default
+DEFAULT_SUBSCRIPTION_MODE="new" ./bin/queen-server
+
+# Run tests
+node run.js subscription
+```
+
+**Expected Behavior:**
+- Consumer groups without explicit mode skip historical messages
+- Consumer groups with `.subscriptionMode('new')` skip history (same)
+- `subscriptionModeServerDefault` reports: "new"
+
+**✅ All tests pass with both configurations!**
+
+## Test Details
+
+### subscriptionModeNew
+
+Tests the explicit `.subscriptionMode('new')` behavior:
+
+1. Pushes 5 historical messages
+2. Creates two consumer groups:
+   - One without explicit mode (behavior depends on server)
+   - One with `.subscriptionMode('new')` (always skips)
+3. Verifies the `new` mode group gets 0 historical messages
+4. Pushes 3 new messages
+5. Verifies both groups get the new messages
+
+**Key Assertion:** `.subscriptionMode('new')` always skips historical messages.
+
+### subscriptionModeServerDefault (NEW)
+
+Detects and validates the server's default subscription mode:
+
+1. Pushes historical messages
+2. Creates consumer group WITHOUT explicit subscription mode
+3. Detects server default based on behavior:
+   - Got historical messages → server default is "all"
+   - Got 0 messages → server default is "new"
+4. Validates explicit `.subscriptionMode('new')` still works
+5. Verifies new messages are received by both groups
+
+**Key Assertion:** Server default affects groups without explicit mode, but explicit modes always work.
+
+### subscriptionFromTimestamp
+
+Tests timestamp-based subscription:
+
+1. Pushes "first batch" of messages
+2. Records cutoff timestamp
+3. Pushes "second batch" of messages
+4. Creates consumer group with `.subscriptionFrom(cutoffTimestamp)`
+5. Verifies only messages after timestamp are received
+
+**Note:** Timestamp precision may vary based on database clock.
+
+## Common Issues
+
+### Issue: Test fails with "expected X, got Y"
+
+**Cause:** Server has different default than test expects.
+
+**Solution:** Check server configuration:
+```bash
+# Check if DEFAULT_SUBSCRIPTION_MODE is set
+echo $DEFAULT_SUBSCRIPTION_MODE
+
+# Check server logs on startup
+LOG_LEVEL=debug ./bin/queen-server | grep "default subscription"
+```
+
+### Issue: Consumer group exists from previous run
+
+**Cause:** Consumer groups persist between test runs.
+
+**Solution:** The test runner cleans up test data automatically:
+```javascript
+await dbPool.query(`DELETE FROM queen.queues WHERE name LIKE 'test-%'`)
+```
+
+Or manually:
+```sql
+DELETE FROM queen.partition_consumers 
+WHERE consumer_group LIKE 'group-%';
+```
+
+### Issue: Timing-related test failures
+
+**Cause:** Messages not fully persisted before consumption.
+
+**Solution:** Tests include appropriate delays. If still failing, increase delays:
+```javascript
+await new Promise(resolve => setTimeout(resolve, 500))  // Increase if needed
+```
+
+## Adding New Subscription Tests
+
+When adding new subscription mode tests:
+
+1. **Use unique queue/group names** to avoid conflicts
+2. **Be explicit about subscription modes** in test assertions
+3. **Document expected behavior** for both server configurations
+4. **Use descriptive group names** like `group-explicit-new` vs `group-default`
+
+Example:
+```javascript
+export async function testNewFeature(client) {
+  // Setup
+  await client.queue('test-new-feature').create()
+  
+  // Test explicit behavior (works with any server default)
+  const result = await client
+    .queue('test-new-feature')
+    .group('group-explicit-new')
+    .subscriptionMode('new')  // Explicit!
+    .pop()
+  
+  // Assert based on explicit mode, not server default
+  // ...
+}
+```
+
+## Debugging Failed Tests
+
+Enable detailed logging:
+
+```bash
+# Client logging
+QUEEN_CLIENT_LOG=true node run.js subscriptionModeNew
+
+# Server logging
+LOG_LEVEL=debug ./bin/queen-server
+```
+
+Look for these log lines:
+```
+Consumer group 'group-xxx' exists: false, has subscription options: true
+Subscription mode 'new' - starting from latest message: <uuid>
+Applying default subscription mode 'new' for consumer group 'group-yyy'
+```
+
+## Summary
+
+- ✅ Tests work with any `DEFAULT_SUBSCRIPTION_MODE` configuration
+- ✅ Explicit subscription modes are always tested and validated
+- ✅ Server default behavior is detected and reported
+- ✅ All tests are documented and maintainable
+
+Run the tests and see them pass! 🎉
+

package/{client-js/test-v2 → test-v2}/consume.js

@@ -371,6 +371,7 @@ export async function consumerGroup(client) {
     
     await client
     .queue('test-queue-v2-consume-group')
+    .subscriptionMode('from_beginning')
     .group('test-group-01')
     .batch(messagesToPush)
     .limit(1)
@@ -381,6 +382,7 @@ export async function consumerGroup(client) {
 
     await client
     .queue('test-queue-v2-consume-group')
+    .subscriptionMode('from_beginning')
     .group('test-group-02')
     .batch(messagesToPush)
     .limit(1)
@@ -414,6 +416,7 @@ export async function consumerGroupWithPartition(client) {
     await client
     .queue('test-queue-v2-consume-group-with-partition')
     .partition('test-partition-01')
+    .subscriptionMode('from_beginning')
     .group('test-group-01')
     .batch(messagesToPush)
     .limit(1)
@@ -425,6 +428,7 @@ export async function consumerGroupWithPartition(client) {
     await client
     .queue('test-queue-v2-consume-group-with-partition')
     .partition('test-partition-01')
+    .subscriptionMode('from_beginning')
     .group('test-group-02')
     .batch(messagesToPush)
     .limit(1)
@@ -462,6 +466,7 @@ export async function manualAck(client) {
     await client
     .queue('test-queue-v2-manual-ack')
     .concurrency(10)
+    .subscriptionMode('from_beginning')
     .batch(1000)
     .wait(false)
     .limit(1)
@@ -514,6 +519,7 @@ export async function retries(client) {
     await client
     .queue('test-queue-v2-retries')
     .concurrency(1)
+    .subscriptionMode('from_beginning')
     .batch(100)
     .wait(false)
     .limit(300)  // Allow up to 300 messages (3 batches of 100)
@@ -554,6 +560,7 @@ export async function retriesConsumerGroup(client) {
     .queue('test-queue-v2-retries-consumer-group')
     .group('test-group-01')
     .concurrency(1)
+    .subscriptionMode('from_beginning')
     .batch(100)
     .wait(false)
     .limit(300)  // Allow up to 300 messages (3 batches of 100)
@@ -574,6 +581,7 @@ export async function retriesConsumerGroup(client) {
     .queue('test-queue-v2-retries-consumer-group')
     .group('test-group-02')
     .concurrency(1)
+    .subscriptionMode('from_beginning')
     .batch(100)
     .wait(false)
     .limit(100)  // Allow up to 300 messages (3 batches of 100)
@@ -607,6 +615,7 @@ export async function autoRenewLease(client) {
     .queue('test-queue-v2-auto-renew-lease')
     .batch(1)
     .wait(false)
+    .subscriptionMode('from_beginning')
     .limit(1)
     .each()
     .consume(async msg => {
@@ -633,6 +642,7 @@ export async function autoRenewLease(client) {
     await client
     .queue('test-queue-v2-auto-renew-lease')
     .batch(1)
+    .subscriptionMode('from_beginning')
     .wait(false)
     .limit(1)
     .renewLease(true, 1000)

package/{client-js/test-v2 → test-v2}/dlq.js

@@ -25,7 +25,7 @@ export async function testDLQ(client) {
         .queue(queueName)
         .batch(1)
         .wait(false)
-        .limit(1)  // Process up to 2 messages (original + retry)
+        .limit(2)  // Process up to 2 messages (original + retry)
         .each()
         .consume(async msg => {
             // Always fail to trigger DLQ

package/{client-js/test-v2 → test-v2}/load.js

@@ -129,6 +129,7 @@ export async function testLoadConsumerGroup(client) {
     await client
     .queue('test-queue-v2-load-consumer-group')
     .group('test-consumer-group-a')
+    .subscriptionMode('from_beginning')
     .concurrency(10)
     .batch(10000)
     .wait(false)
@@ -153,6 +154,7 @@ export async function testLoadConsumerGroup(client) {
     await client
     .queue('test-queue-v2-load-consumer-group')
     .group('test-consumer-group-b')
+    .subscriptionMode('from_beginning')
     .concurrency(10)
     .batch(10000)
     .wait(false)