queen-mq 0.3.1 → 0.6.3

package/client-js/test-v2/MAINTENANCE_TEST.md

@@ -0,0 +1,148 @@
+# Maintenance Mode Test
+
+## Overview
+
+Comprehensive test for the maintenance mode feature with file buffer.
+
+## Test Scenario
+
+1. **Start consumer** - Begins consuming messages from queue
+2. **Start producer** - Produces 10 messages/second
+3. **Enable maintenance** - Triggers maintenance mode via API
+4. **Verify buffering** - Consumer stops receiving (messages go to file buffer)
+5. **Continue producing** - Producer keeps running for 10 seconds during maintenance
+6. **Disable maintenance** - Deactivates maintenance mode via API
+7. **Verify drain** - Consumer resumes receiving messages
+8. **Stop producer** - Halts message production
+9. **Wait for completion** - Waits 10 seconds for file buffer to drain
+10. **Verify counts** - Ensures all produced messages were received
+
+## Running the Test
+
+### Standalone
+
+```bash
+cd /Users/alice/Work/queen/client-js/test-v2
+node maintenance.js
+```
+
+### Via Test Runner
+
+```bash
+# Run just this test
+node run.js test_maintenance_mode
+
+# Run all human tests (includes this one)
+node run.js human
+
+# Run all tests
+node run.js
+```
+
+## Expected Output
+
+```
+🧪 Testing Maintenance Mode with File Buffer
+
+📋 Step 1: Configuring queue "test-maintenance-queue"...
+✅ Queue configured
+
+📥 Step 2: Starting consumer...
+✅ Consumer started
+
+📤 Step 3: Starting producer (10 msgs/sec)...
+✅ Producer started
+
+⏱️  Step 4: Waiting 3 seconds for normal message flow...
+  📨 Received 10 messages (total: 10)
+  📨 Received 10 messages (total: 20)
+  📨 Received 10 messages (total: 30)
+   Produced: 30, Received: 30
+
+🔧 Step 5: Enabling MAINTENANCE MODE...
+   Response: { maintenanceMode: true, bufferedMessages: 0, ... }
+✅ Maintenance mode enabled
+
+⏱️  Step 6: Waiting 2 seconds - consumer should stop receiving...
+   Messages received during maintenance: 0
+   Total: Produced=50, Received=30
+
+⏱️  Step 7: Producing during maintenance for 10 seconds...
+   (Messages should go to file buffer)
+
+   Maintenance period complete:
+   - Total produced: 150
+   - Total received: 30
+   - Buffered (should be ~100): 120
+
+📊 Step 8: Checking maintenance status...
+   Status: { maintenanceMode: true, bufferedMessages: 120, ... }
+
+✅ Step 9: Disabling MAINTENANCE MODE...
+   Response: { maintenanceMode: false, bufferedMessages: 120, ... }
+   (File buffer should start draining to database)
+
+⏱️  Step 10: Waiting for messages to resume...
+  📨 Received 10 messages (total: 40)
+   Messages received after resuming: 10
+✅ Messages are flowing again!
+
+🛑 Step 11: Stopping producer...
+   Final produced count: 150
+
+⏱️  Step 12: Waiting 10 seconds for file buffer to drain...
+   1s - Received: 50/150 (33.3%)
+  📨 Received 10 messages (total: 60)
+   2s - Received: 70/150 (46.7%)
+  📨 Received 10 messages (total: 80)
+   3s - Received: 90/150 (60.0%)
+  📨 Received 10 messages (total: 100)
+   4s - Received: 110/150 (73.3%)
+  📨 Received 10 messages (total: 120)
+   5s - Received: 130/150 (86.7%)
+  📨 Received 10 messages (total: 140)
+   6s - Received: 150/150 (100.0%)
+   ✅ All messages received!
+
+🛑 Step 13: Stopping consumer...
+✅ Consumer stopped
+
+📊 Final Verification:
+
+   Total Produced: 150
+   Total Received: 150
+   Difference: 0
+
+✅ SUCCESS: All messages accounted for!
+   Maintenance mode works correctly with file buffer.
+
+👋 Queen client closed
+
+✅ All tests passed!
+```
+
+## What It Tests
+
+✅ **Maintenance mode activation** - API responds correctly
+✅ **Message buffering** - Messages go to file buffer (consumer stops receiving)
+✅ **Continued production** - Can push messages during maintenance
+✅ **Maintenance mode deactivation** - API responds correctly
+✅ **Automatic drain** - File buffer drains to database
+✅ **Message ordering** - All messages received in order (FIFO)
+✅ **No message loss** - Produced count == Received count
+✅ **Multi-instance support** - Uses database for state (works across restarts)
+
+## Success Criteria
+
+- All produced messages are eventually received
+- Consumer stops receiving during maintenance
+- Consumer resumes receiving after maintenance is disabled
+- No duplicates or lost messages
+- Drain completes within 10 seconds
+
+## Notes
+
+- Uses `axios` for direct API calls (maintenance endpoints)
+- Uses Queen client for queue operations (push/pop/ack)
+- Test takes ~25 seconds total (3s warmup + 10s maintenance + 10s drain)
+

package/client-js/test-v2/README_SUBSCRIPTION_TESTS.md

@@ -0,0 +1,201 @@
+# Subscription Mode Tests
+
+Tests for consumer group subscription modes, compatible with any server `DEFAULT_SUBSCRIPTION_MODE` configuration.
+
+## Test Overview
+
+| Test Function | Description | Explicit Mode Used |
+|--------------|-------------|-------------------|
+| `subscriptionModeNew` | Validates `.subscriptionMode('new')` skips historical messages | Yes (`new`) |
+| `subscriptionModeNewOnly` | Validates alias `.subscriptionMode('new-only')` | Yes (`new-only`) |
+| `subscriptionFromNow` | Validates `.subscriptionFrom('now')` | Yes (`now`) |
+| `subscriptionFromTimestamp` | Validates timestamp-based subscription | Yes (timestamp) |
+| `subscriptionModeAll` | Tests default behavior (depends on server config) | Mixed |
+| `subscriptionModeServerDefault` | **NEW:** Detects and validates server default | Mixed |
+
+## Running the Tests
+
+### Run all subscription tests:
+```bash
+cd client-js/test-v2
+node run.js subscription
+```
+
+### Run specific subscription test:
+```bash
+node run.js subscriptionModeNew
+node run.js subscriptionModeServerDefault
+```
+
+## Server Configuration Compatibility
+
+These tests work with **any** server `DEFAULT_SUBSCRIPTION_MODE` configuration:
+
+### Configuration 1: Standard (Default)
+
+```bash
+# Start server without default
+./bin/queen-server
+
+# Run tests
+node run.js subscription
+```
+
+**Expected Behavior:**
+- Consumer groups without explicit mode process all historical messages
+- Consumer groups with `.subscriptionMode('new')` skip history
+- `subscriptionModeServerDefault` reports: "all (or empty string)"
+
+### Configuration 2: With DEFAULT_SUBSCRIPTION_MODE="new"
+
+```bash
+# Start server with "new" as default
+DEFAULT_SUBSCRIPTION_MODE="new" ./bin/queen-server
+
+# Run tests
+node run.js subscription
+```
+
+**Expected Behavior:**
+- Consumer groups without explicit mode skip historical messages
+- Consumer groups with `.subscriptionMode('new')` skip history (same)
+- `subscriptionModeServerDefault` reports: "new"
+
+**✅ All tests pass with both configurations!**
+
+## Test Details
+
+### subscriptionModeNew
+
+Tests the explicit `.subscriptionMode('new')` behavior:
+
+1. Pushes 5 historical messages
+2. Creates two consumer groups:
+   - One without explicit mode (behavior depends on server)
+   - One with `.subscriptionMode('new')` (always skips)
+3. Verifies the `new` mode group gets 0 historical messages
+4. Pushes 3 new messages
+5. Verifies both groups get the new messages
+
+**Key Assertion:** `.subscriptionMode('new')` always skips historical messages.
+
+### subscriptionModeServerDefault (NEW)
+
+Detects and validates the server's default subscription mode:
+
+1. Pushes historical messages
+2. Creates consumer group WITHOUT explicit subscription mode
+3. Detects server default based on behavior:
+   - Got historical messages → server default is "all"
+   - Got 0 messages → server default is "new"
+4. Validates explicit `.subscriptionMode('new')` still works
+5. Verifies new messages are received by both groups
+
+**Key Assertion:** Server default affects groups without explicit mode, but explicit modes always work.
+
+### subscriptionFromTimestamp
+
+Tests timestamp-based subscription:
+
+1. Pushes "first batch" of messages
+2. Records cutoff timestamp
+3. Pushes "second batch" of messages
+4. Creates consumer group with `.subscriptionFrom(cutoffTimestamp)`
+5. Verifies only messages after timestamp are received
+
+**Note:** Timestamp precision may vary based on database clock.
+
+## Common Issues
+
+### Issue: Test fails with "expected X, got Y"
+
+**Cause:** Server has different default than test expects.
+
+**Solution:** Check server configuration:
+```bash
+# Check if DEFAULT_SUBSCRIPTION_MODE is set
+echo $DEFAULT_SUBSCRIPTION_MODE
+
+# Check server logs on startup
+LOG_LEVEL=debug ./bin/queen-server | grep "default subscription"
+```
+
+### Issue: Consumer group exists from previous run
+
+**Cause:** Consumer groups persist between test runs.
+
+**Solution:** The test runner cleans up test data automatically:
+```javascript
+await dbPool.query(`DELETE FROM queen.queues WHERE name LIKE 'test-%'`)
+```
+
+Or manually:
+```sql
+DELETE FROM queen.partition_consumers 
+WHERE consumer_group LIKE 'group-%';
+```
+
+### Issue: Timing-related test failures
+
+**Cause:** Messages not fully persisted before consumption.
+
+**Solution:** Tests include appropriate delays. If still failing, increase delays:
+```javascript
+await new Promise(resolve => setTimeout(resolve, 500))  // Increase if needed
+```
+
+## Adding New Subscription Tests
+
+When adding new subscription mode tests:
+
+1. **Use unique queue/group names** to avoid conflicts
+2. **Be explicit about subscription modes** in test assertions
+3. **Document expected behavior** for both server configurations
+4. **Use descriptive group names** like `group-explicit-new` vs `group-default`
+
+Example:
+```javascript
+export async function testNewFeature(client) {
+  // Setup
+  await client.queue('test-new-feature').create()
+  
+  // Test explicit behavior (works with any server default)
+  const result = await client
+    .queue('test-new-feature')
+    .group('group-explicit-new')
+    .subscriptionMode('new')  // Explicit!
+    .pop()
+  
+  // Assert based on explicit mode, not server default
+  // ...
+}
+```
+
+## Debugging Failed Tests
+
+Enable detailed logging:
+
+```bash
+# Client logging
+QUEEN_CLIENT_LOG=true node run.js subscriptionModeNew
+
+# Server logging
+LOG_LEVEL=debug ./bin/queen-server
+```
+
+Look for these log lines:
+```
+Consumer group 'group-xxx' exists: false, has subscription options: true
+Subscription mode 'new' - starting from latest message: <uuid>
+Applying default subscription mode 'new' for consumer group 'group-yyy'
+```
+
+## Summary
+
+- ✅ Tests work with any `DEFAULT_SUBSCRIPTION_MODE` configuration
+- ✅ Explicit subscription modes are always tested and validated
+- ✅ Server default behavior is detected and reported
+- ✅ All tests are documented and maintainable
+
+Run the tests and see them pass! 🎉
+

package/client-js/test-v2/consume.js

@@ -190,6 +190,7 @@ export async function testConsumerOrdering(client) {
         await client
         .queue('test-queue-v2-consume-batch')
         .push([{ data: { id: i } }])
+        console.log(new Date().toISOString(), 'Pushed message:', i)
     }
 
     
@@ -370,6 +371,7 @@ export async function consumerGroup(client) {
     
     await client
     .queue('test-queue-v2-consume-group')
+    .subscriptionMode('from_beginning')
     .group('test-group-01')
     .batch(messagesToPush)
     .limit(1)
@@ -380,6 +382,7 @@ export async function consumerGroup(client) {
 
     await client
     .queue('test-queue-v2-consume-group')
+    .subscriptionMode('from_beginning')
     .group('test-group-02')
     .batch(messagesToPush)
     .limit(1)
@@ -413,6 +416,7 @@ export async function consumerGroupWithPartition(client) {
     await client
     .queue('test-queue-v2-consume-group-with-partition')
     .partition('test-partition-01')
+    .subscriptionMode('from_beginning')
     .group('test-group-01')
     .batch(messagesToPush)
     .limit(1)
@@ -424,6 +428,7 @@ export async function consumerGroupWithPartition(client) {
     await client
     .queue('test-queue-v2-consume-group-with-partition')
     .partition('test-partition-01')
+    .subscriptionMode('from_beginning')
     .group('test-group-02')
     .batch(messagesToPush)
     .limit(1)
@@ -461,6 +466,7 @@ export async function manualAck(client) {
     await client
     .queue('test-queue-v2-manual-ack')
     .concurrency(10)
+    .subscriptionMode('from_beginning')
     .batch(1000)
     .wait(false)
     .limit(1)
@@ -513,6 +519,7 @@ export async function retries(client) {
     await client
     .queue('test-queue-v2-retries')
     .concurrency(1)
+    .subscriptionMode('from_beginning')
     .batch(100)
     .wait(false)
     .limit(300)  // Allow up to 300 messages (3 batches of 100)
@@ -553,6 +560,7 @@ export async function retriesConsumerGroup(client) {
     .queue('test-queue-v2-retries-consumer-group')
     .group('test-group-01')
     .concurrency(1)
+    .subscriptionMode('from_beginning')
     .batch(100)
     .wait(false)
     .limit(300)  // Allow up to 300 messages (3 batches of 100)
@@ -573,6 +581,7 @@ export async function retriesConsumerGroup(client) {
     .queue('test-queue-v2-retries-consumer-group')
     .group('test-group-02')
     .concurrency(1)
+    .subscriptionMode('from_beginning')
     .batch(100)
     .wait(false)
     .limit(100)  // Allow up to 300 messages (3 batches of 100)
@@ -606,6 +615,7 @@ export async function autoRenewLease(client) {
     .queue('test-queue-v2-auto-renew-lease')
     .batch(1)
     .wait(false)
+    .subscriptionMode('from_beginning')
     .limit(1)
     .each()
     .consume(async msg => {
@@ -632,6 +642,7 @@ export async function autoRenewLease(client) {
     await client
     .queue('test-queue-v2-auto-renew-lease')
     .batch(1)
+    .subscriptionMode('from_beginning')
     .wait(false)
     .limit(1)
     .renewLease(true, 1000)

package/client-js/test-v2/dlq.js

@@ -25,7 +25,7 @@ export async function testDLQ(client) {
         .queue(queueName)
         .batch(1)
         .wait(false)
-        .limit(1)  // Process up to 2 messages (original + retry)
+        .limit(2)  // Process up to 2 messages (original + retry)
         .each()
         .consume(async msg => {
             // Always fail to trigger DLQ

package/client-js/test-v2/load.js

@@ -129,6 +129,7 @@ export async function testLoadConsumerGroup(client) {
     await client
     .queue('test-queue-v2-load-consumer-group')
     .group('test-consumer-group-a')
+    .subscriptionMode('from_beginning')
     .concurrency(10)
     .batch(10000)
     .wait(false)
@@ -153,6 +154,7 @@ export async function testLoadConsumerGroup(client) {
     await client
     .queue('test-queue-v2-load-consumer-group')
     .group('test-consumer-group-b')
+    .subscriptionMode('from_beginning')
     .concurrency(10)
     .batch(10000)
     .wait(false)