queen-mq 1.3.0 → 2.0.2

package/test-v2/watermark.js

@@ -1,90 +1,36 @@
 /**
- * Seek / delete re-consumption tests (behaviour, not bookkeeping)
+ * Seek / delete re-consumption tests.
  *
- * These tests verify the BEHAVIOUR a user cares about:
+ * These pin the BEHAVIOUR a user cares about, and nothing else:
  *   1. Seeking a consumer group backwards re-exposes the old messages, and the
  *      group re-consumes them from the start within a short bound.
  *   2. Deleting a consumer group (all queues, or for one queue) lets a group of
  *      the same name consume every message again from the start.
  *
- * WHY THIS WAS REWRITTEN (2026-07-24)
- * -----------------------------------
- * The consumer_watermarks table is a LEGACY-path optimisation: it records when a
- * consumer last found a queue empty so an indexed empty-scan can be skipped. The
- * original tests asserted the *row* was created after an empty poll and then aged
- * it to exercise the 2-minute buffer:
- *     pl.updated_at >= (v_watermark - interval '2 minutes')
- *
- * With the wildcard hot-list ON (QUEEN_HOTLIST=1, the default) empty polls are
- * answered from the in-memory candidate ring and that bookkeeping row is NEVER
- * written — by design, not a bug: the ring, not the watermark, is the discovery
- * source. So asserting the row exists fails on the hot-list path even though
- * re-consumption works perfectly.
- *
- * The behaviour is identical on both engines, so these tests now assert the
- * behaviour directly and only exercise the watermark-age scenario on the LEGACY
- * path — detected at runtime by whether the empty poll actually wrote the row
- * (no env flag needed). On the hot-list path the broker's seek/delete handlers
- * re-seed / invalidate the ring so the reconsume is immediate; the idle-bounded
- * consume below (idleMillis 3000) is what proves "within a short bound".
+ * Each test first drains the queue and polls it empty, so the group has seen
+ * "nothing to deliver" before the seek or the delete. "Within a short bound"
+ * is the idle-bounded consume (idleMillis 3000) that follows: a seek or a
+ * delete that left stale per-group state behind makes that consume go idle
+ * before every message came back, and the count comes up short.
  *
  * Run: node run.js seekBackwardsAllowsReconsume
  *      node run.js deleteConsumerGroupAllowsReconsume
  *      node run.js deleteConsumerGroupForQueueAllowsReconsume
  */
 
-import { dbPool } from './run.js'
-
 const QUEUE_NAME_SEEK = 'test-queue-v2-watermark-seek'
 const QUEUE_NAME_DELETE = 'test-queue-v2-watermark-delete'
 
-/**
- * Helper: Move the watermark FORWARD in time to simulate that partitions
- * are "old" relative to the watermark.
- *
- * The filter is: pl.updated_at >= (v_watermark - interval '2 minutes')
- *
- * If watermark = NOW + 10 minutes, then:
- *   filter = pl.updated_at >= (NOW + 10 - 2) = pl.updated_at >= NOW + 8
- *
- * Partitions updated at NOW will fail this filter (NOW < NOW + 8)
- */
-async function advanceWatermark(queueName, consumerGroup, minutesForward = 10) {
-    const result = await dbPool.query(`
-        UPDATE queen.consumer_watermarks w
-        SET last_empty_scan_at = NOW() + interval '${minutesForward} minutes',
-            updated_at = NOW() + interval '${minutesForward} minutes'
-        FROM queen.queues q
-        WHERE w.queue_id = q.id AND q.name = $1 AND w.consumer_group = $2
-        RETURNING w.*
-    `, [queueName, consumerGroup])
-    return result.rows[0]
-}
-
-/**
- * Helper: Check if watermark exists for a queue/consumer_group
- */
-async function getWatermark(queueName, consumerGroup) {
-    const result = await dbPool.query(`
-        SELECT w.* FROM queen.consumer_watermarks w
-        JOIN queen.queues q ON q.id = w.queue_id
-        WHERE q.name = $1 AND w.consumer_group = $2
-    `, [queueName, consumerGroup])
-    return result.rows[0]
-}
-
 /**
  * Test: Seeking backwards should allow re-consumption of messages
  *
  * Steps:
  * 1. Push messages to queue
- * 2. Consume all messages with a consumer group (watermark advances)
- * 3. Artificially age the watermark (simulate 10 minutes passing)
- * 4. Seek the consumer group backwards to before the messages
- * 5. Try to consume again - should get all messages
+ * 2. Consume all messages with a consumer group, then poll it empty
+ * 3. Seek the consumer group backwards to before the messages
+ * 4. Try to consume again - should get all messages
  *
- * Expected: All messages are re-consumed after seek
- * Actual (bug): Watermark blocks re-consumption, 0 messages returned
+ * Expected: All messages are re-consumed after seek, within the idle window
  */
 export async function seekBackwardsAllowsReconsume(client) {
     const consumerGroup = 'watermark-seek-test-cg'
@@ -92,11 +38,6 @@ export async function seekBackwardsAllowsReconsume(client) {
     // Cleanup
     try {
         await client.deleteConsumerGroup(consumerGroup, true)
-        // Also clean up any orphaned watermark from previous runs
-        await dbPool.query(
-            'DELETE FROM queen.consumer_watermarks WHERE consumer_group = $1',
-            [consumerGroup]
-        )
     } catch (e) {
         // Ignore
     }
@@ -147,7 +88,7 @@ export async function seekBackwardsAllowsReconsume(client) {
         }
     }
 
-    // 5. Verify queue is empty for this consumer (triggers watermark advancement)
+    // 5. Verify queue is empty for this consumer (the group has now polled empty)
     const emptyCheck = await client
         .queue(QUEUE_NAME_SEEK)
         .group(consumerGroup)
@@ -162,24 +103,9 @@ export async function seekBackwardsAllowsReconsume(client) {
         }
     }
 
-    console.log(`  Verified queue is empty (watermark should be set)`)
-
-    // 6. Detect which discovery path we are on. On the LEGACY path the empty poll
-    //    wrote a consumer_watermarks row; on the hot-list path it did not (empty
-    //    polls are served from the ring — by design). We do NOT assert the row: it
-    //    is a path-specific bookkeeping detail, not the behaviour under test.
-    const watermarkBefore = await getWatermark(QUEUE_NAME_SEEK, consumerGroup)
-    if (watermarkBefore) {
-        // Legacy path: age the watermark 10 minutes into the future so the old
-        // partitions fall outside the 2-minute buffer — the original repro that
-        // proves seek must reset the watermark.
-        const advancedWatermark = await advanceWatermark(QUEUE_NAME_SEEK, consumerGroup, 10)
-        console.log(`  [legacy] Watermark row exists; advanced to ${advancedWatermark.last_empty_scan_at} (10m future)`)
-    } else {
-        console.log(`  [hotlist] No watermark row (empty polls served from the ring) — skipping the watermark-age step`)
-    }
+    console.log(`  Verified queue is empty`)
 
-    // 8. Seek backwards to BEFORE the messages were pushed
+    // 6. Seek backwards to BEFORE the messages were pushed
     console.log(`  Seeking to timestamp: ${timestampBeforeMessages}`)
 
     const seekResult = await client.admin.seekConsumerGroup(
@@ -190,18 +116,10 @@ export async function seekBackwardsAllowsReconsume(client) {
 
     console.log(`  Seek result: ${JSON.stringify(seekResult)}`)
 
-    // 9. Check if watermark was reset (it should be, but currently isn't - that's the bug)
-    const watermarkAfterSeek = await getWatermark(QUEUE_NAME_SEEK, consumerGroup)
-    if (watermarkAfterSeek) {
-        console.log(`  WARNING: Watermark still exists after seek: ${watermarkAfterSeek.last_empty_scan_at}`)
-    } else {
-        console.log(`  Watermark was properly deleted after seek`)
-    }
-
     // Small delay after seek
     await new Promise(resolve => setTimeout(resolve, 500))
 
-    // 10. Try to consume again - should get all messages back
+    // 7. Try to consume again - should get all messages back
     let secondConsumeCount = 0
     await client
         .queue(QUEUE_NAME_SEEK)
@@ -216,17 +134,13 @@ export async function seekBackwardsAllowsReconsume(client) {
 
     console.log(`  Second consume (after seek): ${secondConsumeCount} messages`)
 
-    // 11. Verify we got all messages back within the idle-bounded consume above.
-    //     A shortfall means the backward seek did not re-expose the messages
-    //     promptly: on the legacy path the consumer_watermarks row was not reset
-    //     (still at ${watermarkAfterSeek?.last_empty_scan_at}); on the hot-list path
-    //     the ring was not re-seeded, so reconsume would only resume at the ≤30s
-    //     periodic floor instead of immediately.
+    // 8. Verify we got all messages back within the idle-bounded consume above.
+    //    A shortfall means the backward seek did not re-expose the messages
+    //    promptly.
     if (secondConsumeCount !== messageCount) {
         return {
             success: false,
-            message: `After seek backwards, should re-consume ${messageCount} messages within the idle window, but got ${secondConsumeCount}. ` +
-                     `(legacy watermark after seek: ${watermarkAfterSeek?.last_empty_scan_at})`
+            message: `After seek backwards, should re-consume ${messageCount} messages within the idle window, but got ${secondConsumeCount}`
         }
     }
 
@@ -241,14 +155,12 @@ export async function seekBackwardsAllowsReconsume(client) {
  *
  * Steps:
  * 1. Push messages to queue
- * 2. Consume all messages with a consumer group (watermark advances)
- * 3. Artificially age the watermark (simulate 10 minutes passing)
- * 4. Delete the consumer group
- * 5. Create a new consumer group with the SAME name
- * 6. Try to consume - should get all messages
+ * 2. Consume all messages with a consumer group, then poll it empty
+ * 3. Delete the consumer group
+ * 4. Create a new consumer group with the SAME name
+ * 5. Try to consume - should get all messages
  *
  * Expected: All messages are consumed by the "new" consumer group
- * Actual (bug): Orphaned watermark blocks consumption, 0 messages returned
  */
 export async function deleteConsumerGroupAllowsReconsume(client) {
     const consumerGroup = 'watermark-delete-test-cg'
@@ -256,11 +168,6 @@ export async function deleteConsumerGroupAllowsReconsume(client) {
     // Cleanup
     try {
         await client.deleteConsumerGroup(consumerGroup, true)
-        // Also clean up any orphaned watermark from previous runs
-        await dbPool.query(
-            'DELETE FROM queen.consumer_watermarks WHERE consumer_group = $1',
-            [consumerGroup]
-        )
     } catch (e) {
         // Ignore
     }
@@ -305,7 +212,7 @@ export async function deleteConsumerGroupAllowsReconsume(client) {
         }
     }
 
-    // 4. Verify queue is empty for this consumer (triggers watermark advancement)
+    // 4. Verify queue is empty for this consumer (the group has now polled empty)
     const emptyCheck = await client
         .queue(QUEUE_NAME_DELETE)
         .group(consumerGroup)
@@ -320,40 +227,20 @@ export async function deleteConsumerGroupAllowsReconsume(client) {
         }
     }
 
-    console.log(`  Verified queue is empty (watermark should be set)`)
-
-    // 5. Detect the discovery path (see seekBackwardsAllowsReconsume). The
-    //    watermark row is a legacy-path artefact, not the behaviour under test, so
-    //    we do not assert it — we only exercise the watermark-age scenario when the
-    //    row was actually written (legacy path).
-    const watermarkBefore = await getWatermark(QUEUE_NAME_DELETE, consumerGroup)
-    if (watermarkBefore) {
-        const advancedWatermark = await advanceWatermark(QUEUE_NAME_DELETE, consumerGroup, 10)
-        console.log(`  [legacy] Watermark row exists; advanced to ${advancedWatermark.last_empty_scan_at} (10m future)`)
-    } else {
-        console.log(`  [hotlist] No watermark row (empty polls served from the ring) — skipping the watermark-age step`)
-    }
+    console.log(`  Verified queue is empty`)
 
-    // 7. Delete the consumer group
+    // 5. Delete the consumer group
     console.log(`  Deleting consumer group: ${consumerGroup}`)
 
     const deleteResult = await client.deleteConsumerGroup(consumerGroup, true)
     console.log(`  Delete result: ${JSON.stringify(deleteResult)}`)
 
-    // 8. Check if watermark was deleted (it should be, but currently isn't - that's the bug)
-    const watermarkAfterDelete = await getWatermark(QUEUE_NAME_DELETE, consumerGroup)
-    if (watermarkAfterDelete) {
-        console.log(`  WARNING: Orphaned watermark still exists: ${watermarkAfterDelete.last_empty_scan_at}`)
-    } else {
-        console.log(`  Watermark was properly deleted`)
-    }
-
     // Small delay after delete
     await new Promise(resolve => setTimeout(resolve, 500))
 
-    // 9. Create a "new" consumer group with the same name and try to consume
-    //    Since partition_consumers was deleted, this should start fresh
-    //    But if consumer_watermarks wasn't deleted, it will skip all messages
+    // 6. Create a "new" consumer group with the same name and try to consume.
+    //    The delete removed the group's cursors and metadata, so it must start
+    //    fresh; any per-group state that survived would skip the messages.
     let secondConsumeCount = 0
     await client
         .queue(QUEUE_NAME_DELETE)
@@ -369,16 +256,14 @@ export async function deleteConsumerGroupAllowsReconsume(client) {
 
     console.log(`  Second consume (after delete + recreate): ${secondConsumeCount} messages`)
 
-    // 10. Verify we got all messages within the idle-bounded consume above. A
-    //     shortfall means the delete left stale discovery state: a legacy orphaned
-    //     watermark (at ${watermarkAfterDelete?.last_empty_scan_at}), or, on the
-    //     hot-list path, a surviving group ring / stale seeded-groups cache that
-    //     suppressed the from-the-start reconsume.
+    // 7. Verify we got all messages within the idle-bounded consume above. A
+    //    shortfall means the delete left stale per-group state behind that
+    //    suppressed the from-the-start reconsume.
     if (secondConsumeCount !== messageCount) {
         return {
             success: false,
             message: `After deleting CG, new CG with same name should consume ${messageCount} messages within the idle window, ` +
-                     `but got ${secondConsumeCount}. (legacy watermark after delete: ${watermarkAfterDelete?.last_empty_scan_at})`
+                     `but got ${secondConsumeCount}`
         }
     }
 
@@ -400,11 +285,6 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
     // Cleanup
     try {
         await client.admin.deleteConsumerGroupForQueue(consumerGroup, queueName, true)
-        // Also clean up any orphaned watermark from previous runs
-        await dbPool.query(
-            'DELETE FROM queen.consumer_watermarks w USING queen.queues q WHERE w.queue_id = q.id AND q.name = $1 AND w.consumer_group = $2',
-            [queueName, consumerGroup]
-        )
     } catch (e) {
         // Ignore
     }
@@ -449,7 +329,7 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
         }
     }
 
-    // 4. Verify queue is empty (triggers watermark)
+    // 4. Verify queue is empty (the group has now polled empty)
     const emptyCheck = await client
         .queue(queueName)
         .group(consumerGroup)
@@ -464,35 +344,17 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
         }
     }
 
-    console.log(`  Verified queue is empty (watermark should be set)`)
-
-    // 5. Detect the discovery path (see seekBackwardsAllowsReconsume). The
-    //    watermark row is a legacy-path artefact, not the behaviour under test.
-    const watermarkBefore = await getWatermark(queueName, consumerGroup)
-    if (watermarkBefore) {
-        const advancedWatermark = await advanceWatermark(queueName, consumerGroup, 10)
-        console.log(`  [legacy] Watermark row exists; advanced to ${advancedWatermark.last_empty_scan_at} (10m future)`)
-    } else {
-        console.log(`  [hotlist] No watermark row (empty polls served from the ring) — skipping the watermark-age step`)
-    }
+    console.log(`  Verified queue is empty`)
 
-    // 7. Delete consumer group for this specific queue
+    // 5. Delete consumer group for this specific queue
     console.log(`  Deleting consumer group for queue: ${consumerGroup} / ${queueName}`)
 
     const deleteResult = await client.admin.deleteConsumerGroupForQueue(consumerGroup, queueName, true)
     console.log(`  Delete result: ${JSON.stringify(deleteResult)}`)
 
-    // 8. Check if watermark was deleted
-    const watermarkAfterDelete = await getWatermark(queueName, consumerGroup)
-    if (watermarkAfterDelete) {
-        console.log(`  WARNING: Orphaned watermark still exists: ${watermarkAfterDelete.last_empty_scan_at}`)
-    } else {
-        console.log(`  Watermark was properly deleted`)
-    }
-
     await new Promise(resolve => setTimeout(resolve, 500))
 
-    // 9. Try to consume again
+    // 6. Try to consume again
     let secondConsumeCount = 0
     await client
         .queue(queueName)
@@ -512,7 +374,7 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
         return {
             success: false,
             message: `After deleting CG for queue, should consume ${messageCount} messages within the idle window, ` +
-                     `but got ${secondConsumeCount}. (legacy watermark after delete: ${watermarkAfterDelete?.last_empty_scan_at})`
+                     `but got ${secondConsumeCount}`
         }
     }
 

package/test-v2/maintenance.js

@@ -1,277 +0,0 @@
-import { Queen } from '../client-v2/index.js';
-import axios from 'axios';
-
-export async function test_maintenance_mode(client = null) {
-  console.log('🧪 Testing Maintenance Mode with File Buffer\n');
-  
-  const queen = client
-  
-  // Default Queen port; override with QUEEN_SERVER_URL (same as run.js).
-  const API_URL = process.env.QUEEN_SERVER_URL || 'http://localhost:6632';
-  const QUEUE_NAME = 'test-maintenance-queue';
-  const MESSAGES_PER_SECOND = 10;
-  const MAINTENANCE_DURATION_SECONDS = 10;
-  
-  let producedCount = 0;
-  let receivedCount = 0;
-  let consumerActive = false;
-  
-  try {
-    // 0. Ensure maintenance mode is OFF before starting
-    console.log('🧹 Step 0: Ensuring clean state...');
-    await axios.post(`${API_URL}/api/v1/system/maintenance`, { enabled: false });
-    
-    let initialStatus = await axios.get(`${API_URL}/api/v1/system/maintenance`);
-    console.log(`   Initial status:`, initialStatus.data);
-    
-    if (initialStatus.data.bufferedMessages > 0) {
-      console.log(`   ⚠️  Found ${initialStatus.data.bufferedMessages} buffered messages from previous runs`);
-      console.log('   Waiting for buffer to drain (checking every 2 seconds)...');
-      
-      // Wait for buffer to drain completely
-      let waitTime = 0;
-      const maxWait = 60; // Max 60 seconds
-      while (waitTime < maxWait) {
-        await sleep(2000);
-        waitTime += 2;
-        const status = await axios.get(`${API_URL}/api/v1/system/maintenance`);
-        console.log(`   ${waitTime}s - Buffered: ${status.data.bufferedMessages}`);
-        
-        if (status.data.bufferedMessages === 0) {
-          console.log('   ✅ Buffer drained completely');
-          break;
-        }
-      }
-      
-      if (waitTime >= maxWait) {
-        console.log(`   ⚠️  Timeout waiting for buffer to drain. Continuing anyway...`);
-      }
-    }
-    console.log('');
-    
-    // 1. Configure queue
-    console.log(`📋 Step 1: Configuring queue "${QUEUE_NAME}"...`);
-    await queen.queue(QUEUE_NAME).config({
-      leaseTime: 60,
-    }).create();
-    console.log('✅ Queue configured\n');
-    
-    // 2. Start consumer
-    console.log('📥 Step 2: Starting consumer...');
-    consumerActive = true;
-    
-    const consumePromise = (async () => {
-      while (consumerActive) {
-        try {
-          // wait(true) long-polls on the server. This is important: pushes
-          // commit before partition_lookup is refreshed (PUSHPOPLOOKUPSOL
-          // is async), so a wait(false) pop can race past freshly-pushed
-          // messages and return empty. Long-poll re-runs the candidate
-          // scan internally and picks up messages once the lookup row
-          // commits — typically within milliseconds.
-          // Use a short timeout (1s) so the consumer notices
-          // consumerActive=false promptly when the test winds down.
-          const messages = await queen.queue(QUEUE_NAME)
-            .batch(10)
-            .wait(true)
-            .timeoutMillis(1000)
-            .limit(1)
-            .each()
-            .pop();
-
-          if (messages && messages.length > 0) {
-            receivedCount += messages.length;
-            console.log(`  📨 Received ${messages.length} messages (total: ${receivedCount})`);
-            
-            // Acknowledge messages
-            await queen.ack(messages);
-            
-          }
-        } catch (err) {
-          if (consumerActive) {
-            console.error('  ❌ Consumer error:', err.message);
-          }
-        }
-        await sleep(10);
-      }
-    })();
-    
-    console.log('✅ Consumer started\n');
-    
-    // 3. Start producer (10 messages/second)
-    console.log(`📤 Step 3: Starting producer (${MESSAGES_PER_SECOND} msgs/sec)...`);
-    let producerRunning = true;
-    
-    const producerLoop = async () => {
-      while (producerRunning) {
-        try {
-          // Push batch of 10 messages
-          const batch = [];
-          for (let i = 0; i < MESSAGES_PER_SECOND; i++) {
-            batch.push({
-              data: { 
-                messageNumber: producedCount + i,
-                timestamp: Date.now()
-              }
-            });
-          }
-          
-          // Push and wait for confirmation
-          await queen.queue(QUEUE_NAME).push(batch);
-          producedCount += batch.length;  // Only increment after successful push
-          
-          // Wait 1 second before next batch
-          await sleep(1000);
-        } catch (err) {
-          console.error('  ❌ Producer error:', err.message);
-          await sleep(1000); // Wait before retry
-        }
-      }
-    };
-    
-    const producerPromise = producerLoop();
-    
-    console.log('✅ Producer started\n');
-    
-    // Wait 3 seconds for messages to flow normally
-    console.log('⏱️  Step 4: Waiting 3 seconds for normal message flow...');
-    await sleep(3000);
-    console.log(`   Produced: ${producedCount}, Received: ${receivedCount}\n`);
-    
-    const receivedBeforeMaintenance = receivedCount;
-    
-    // 4. Enable maintenance mode
-    console.log('🔧 Step 5: Enabling MAINTENANCE MODE...');
-    const maintenanceResponse = await axios.post(`${API_URL}/api/v1/system/maintenance`, {
-      enabled: true
-    });
-    console.log(`   Response:`, maintenanceResponse.data);
-    console.log('✅ Maintenance mode enabled\n');
-    
-    // Wait 2 seconds and check that consumer stops receiving
-    console.log('⏱️  Step 6: Waiting 5 seconds - consumer should stop receiving...');
-    await sleep(5000);
-    const receivedDuringWait = receivedCount - receivedBeforeMaintenance;
-    console.log(`   Messages received during maintenance: ${receivedDuringWait}`);
-    console.log(`   Total: Produced=${producedCount}, Received=${receivedCount}\n`);
-    
-    // 5. Keep producing during maintenance
-    console.log(`⏱️  Step 7: Producing during maintenance for ${MAINTENANCE_DURATION_SECONDS} seconds...`);
-    console.log('   (Messages should go to file buffer)\n');
-    await sleep(MAINTENANCE_DURATION_SECONDS * 1000);
-    
-    const producedDuringMaintenance = producedCount;
-    const receivedAfterMaintenance = receivedCount;
-    const messagesBuffered = producedDuringMaintenance - receivedAfterMaintenance;
-    
-    console.log(`   Maintenance period complete:`);
-    console.log(`   - Total produced: ${producedCount}`);
-    console.log(`   - Total received: ${receivedCount}`);
-    console.log(`   - Buffered (should be ~${MESSAGES_PER_SECOND * MAINTENANCE_DURATION_SECONDS}): ${messagesBuffered}\n`);
-    
-    // 6. Check maintenance status
-    console.log('📊 Step 8: Checking maintenance status...');
-    const statusResponse = await axios.get(`${API_URL}/api/v1/system/maintenance`);
-    console.log(`   Status:`, statusResponse.data);
-    console.log('');
-    
-    // 7. Disable maintenance mode
-    console.log('✅ Step 9: Disabling MAINTENANCE MODE...');
-    const disableResponse = await axios.post(`${API_URL}/api/v1/system/maintenance`, {
-      enabled: false
-    });
-    console.log(`   Response:`, disableResponse.data);
-    console.log('   (File buffer should start draining to database)\n');
-    
-    // 8. Wait for messages to resume and stop producer
-    console.log('⏱️  Step 10: Waiting for messages to resume...');
-    await sleep(2000);
-    
-    const receivedAfterResume = receivedCount;
-    const messagesResumed = receivedAfterResume - receivedAfterMaintenance;
-    console.log(`   Messages received after resuming: ${messagesResumed}`);
-    console.log('✅ Messages are flowing again!\n');
-    
-    // Stop producer
-    console.log('🛑 Step 11: Stopping producer...');
-    producerRunning = false;
-    await producerPromise;  // Wait for producer to actually stop
-    
-    const finalProducedCount = producedCount;
-    console.log(`   Final produced count: ${finalProducedCount}\n`);
-    
-    // 9. Wait for drain to complete
-    console.log('⏱️  Step 12: Waiting 10 seconds for file buffer to drain...');
-    for (let i = 0; i < 10; i++) {
-      await sleep(1000);
-      console.log(`   ${i + 1}s - Received: ${receivedCount}/${finalProducedCount} (${((receivedCount/finalProducedCount)*100).toFixed(1)}%)`);
-      
-      // Check if we've received all messages
-      if (receivedCount >= finalProducedCount) {
-        console.log('   ✅ All messages received!\n');
-        break;
-      }
-    }
-    
-    // 10. Stop consumer
-    console.log('🛑 Step 13: Stopping consumer...');
-    consumerActive = false;
-    await consumePromise;
-    console.log('✅ Consumer stopped\n');
-    
-    // 11. Final verification
-    console.log('📊 Final Verification:\n');
-    console.log(`   Total Produced: ${finalProducedCount}`);
-    console.log(`   Total Received: ${receivedCount}`);
-    console.log(`   Difference: ${finalProducedCount - receivedCount}`);
-    
-    if (receivedCount === finalProducedCount) {
-      console.log('\n✅ SUCCESS: All messages accounted for!');
-      console.log('   Maintenance mode works correctly with file buffer.\n');
-      
-      return { 
-        success: true, 
-        message: `All ${finalProducedCount} messages accounted for during maintenance mode test` 
-      };
-    } else if (receivedCount < finalProducedCount) {
-      const missing = finalProducedCount - receivedCount;
-      console.log(`\n⚠️  WARNING: Missing ${missing} messages`);
-      console.log('   They may still be in the file buffer or queue.');
-      console.log('   Check /var/lib/queen/buffers/ for remaining files.\n');
-      
-      // Check buffer status
-      const finalStatus = await axios.get(`${API_URL}/api/v1/system/maintenance`);
-      console.log('   Buffer status:', finalStatus.data);
-      
-      return { 
-        success: false, 
-        message: `Missing ${missing} messages (produced: ${finalProducedCount}, received: ${receivedCount})` 
-      };
-    } else {
-      console.log(`\n❌ ERROR: Received more messages than produced! (${receivedCount} > ${finalProducedCount})`);
-      console.log('   This should not happen - possible duplicate issue.\n');
-      
-      return { 
-        success: false, 
-        message: `Received more than produced (${receivedCount} > ${finalProducedCount})` 
-      };
-    }
-    
-  } catch (err) {
-    console.error('\n❌ Test failed:', err.message);
-    console.error(err.stack);
-    
-    // Clean up
-    consumerActive = false;
-    
-    return { success: false, message: `Test threw error: ${err.message}` };
-  } finally {
-    // Clean up
-    consumerActive = false;
-  }
-}
-
-function sleep(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-