queen-mq 0.1.0 → 0.1.2

package/src/services/evictionService.js

@@ -5,37 +5,66 @@ import config from '../config.js';
 const EVICTION_INTERVAL = config.JOBS.EVICTION_INTERVAL;
 
 // Evict messages that exceeded max wait time for a queue
+// Note: Eviction is already handled by POP query filtering old messages
+// This service optionally moves them to DLQ for visibility
 const evictMessages = async (client, queueName, maxWaitTimeSeconds) => {
   if (!maxWaitTimeSeconds || maxWaitTimeSeconds <= 0) return 0;
   
-  // For the new schema, we need to update or create status entries for evicted messages
-  const result = await client.query(`
-    WITH queue_partitions AS (
-      SELECT p.id
-      FROM queen.partitions p
-      JOIN queen.queues q ON p.queue_id = q.id
-      WHERE q.name = $1
-    ),
-    eligible_messages AS (
-      SELECT m.id
-      FROM queen.messages m
-      JOIN queue_partitions qp ON m.partition_id = qp.id
-      LEFT JOIN queen.messages_status ms ON m.id = ms.message_id AND ms.consumer_group IS NULL
-      WHERE m.created_at < NOW() - INTERVAL '1 second' * $2
-        AND (ms.status = 'pending' OR ms.status IS NULL)
-    )
-    INSERT INTO queen.messages_status (message_id, consumer_group, status, completed_at, error_message)
-    SELECT id, NULL, 'evicted', NOW(), 'Message exceeded maximum wait time'
-    FROM eligible_messages
-    ON CONFLICT (message_id, consumer_group) 
-    DO UPDATE SET 
-      status = 'evicted',
-      completed_at = NOW(),
-      error_message = 'Message exceeded maximum wait time'
-    RETURNING message_id
-  `, [queueName, maxWaitTimeSeconds]);
-  
-  return result.rowCount || 0;
+  try {
+    // Find old messages that haven't been consumed and move to DLQ
+    // Use a WHERE EXISTS check to ensure message still exists before inserting into DLQ
+    const result = await client.query(`
+      WITH queue_partitions AS (
+        SELECT p.id, p.queue_id
+        FROM queen.partitions p
+        JOIN queen.queues q ON p.queue_id = q.id
+        WHERE q.name = $1
+      ),
+      old_unconsumed_messages AS (
+        SELECT m.id, m.partition_id, m.created_at
+        FROM queen.messages m
+        JOIN queue_partitions qp ON m.partition_id = qp.id
+        LEFT JOIN queen.partition_consumers pc ON pc.partition_id = m.partition_id
+          AND pc.consumer_group = '__QUEUE_MODE__'
+        WHERE m.created_at < NOW() - INTERVAL '1 second' * $2
+          -- Message has not been consumed yet
+          AND ((m.created_at, m.id) > (COALESCE(pc.last_consumed_created_at, '1970-01-01'::timestamptz),
+                                         COALESCE(pc.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid))
+               OR pc.last_consumed_id IS NULL)
+      )
+      INSERT INTO queen.dead_letter_queue (
+          message_id, 
+          partition_id, 
+          consumer_group, 
+          error_message, 
+          original_created_at
+      )
+      SELECT 
+          om.id,
+          om.partition_id,
+          '__QUEUE_MODE__',
+          'Message exceeded maximum wait time',
+          om.created_at
+      FROM old_unconsumed_messages om
+      WHERE EXISTS (
+          SELECT 1 FROM queen.messages m WHERE m.id = om.id
+      )
+      ON CONFLICT DO NOTHING
+      RETURNING message_id
+    `, [queueName, maxWaitTimeSeconds]);
+    
+    return result.rowCount || 0;
+  } catch (error) {
+    // Catch foreign key violations (messages deleted between SELECT and INSERT)
+    if (error.code === '23503') {
+      // Foreign key constraint violation - message was deleted
+      // This is a harmless race condition, just log it
+      log(`${LogTypes.EVICTION} | Queue: ${queueName} | Warning: Message deleted during eviction (race condition)`);
+      return 0;
+    }
+    // Re-throw other errors
+    throw error;
+  }
 };
 
 // Evict messages during pop operation (inline eviction)

package/src/services/retentionService.js

@@ -16,30 +16,44 @@ const retentionQueue = async (client, queue) => {
   
   let totalDeleted = 0;
   
-  // Delete old pending messages from all partitions in this queue
+  // Delete old unconsumed messages from all partitions in this queue
   if (retentionSeconds > 0) {
     const result = await client.query(`
-      DELETE FROM queen.messages
-      WHERE partition_id IN (
+      DELETE FROM queen.messages m
+      WHERE m.partition_id IN (
         SELECT id FROM queen.partitions WHERE queue_id = $1
       )
-        AND status = 'pending'
-        AND created_at < NOW() - INTERVAL '1 second' * $2
+      AND m.created_at < NOW() - INTERVAL '1 second' * $2
+      AND NOT EXISTS (
+          -- Message has been consumed by at least one consumer
+          SELECT 1 
+          FROM queen.partition_consumers pc
+          WHERE pc.partition_id = m.partition_id
+            AND (m.created_at, m.id) <= (pc.last_consumed_created_at, pc.last_consumed_id)
+      )
       RETURNING id
     `, [queue.id, retentionSeconds]);
     
     totalDeleted += result.rowCount || 0;
   }
   
-  // Delete completed/failed/evicted messages from all partitions in this queue
+  // Delete old consumed messages from all partitions in this queue
+  // (messages consumed by ALL consumer groups on that partition)
   if (completedRetentionSeconds > 0) {
     const result = await client.query(`
-      DELETE FROM queen.messages
-      WHERE partition_id IN (
+      DELETE FROM queen.messages m
+      WHERE m.partition_id IN (
         SELECT id FROM queen.partitions WHERE queue_id = $1
       )
-        AND status IN ('completed', 'failed', 'evicted')
-        AND COALESCE(completed_at, failed_at, created_at) < NOW() - INTERVAL '1 second' * $2
+      AND m.created_at < NOW() - INTERVAL '1 second' * $2
+      AND NOT EXISTS (
+          -- No consumer group that hasn't consumed this message yet
+          SELECT 1 
+          FROM queen.partition_consumers pc
+          WHERE pc.partition_id = m.partition_id
+            AND ((m.created_at, m.id) > (pc.last_consumed_created_at, pc.last_consumed_id)
+                 OR pc.last_consumed_id IS NULL)
+      )
       RETURNING id
     `, [queue.id, completedRetentionSeconds]);
     
@@ -48,7 +62,6 @@ const retentionQueue = async (client, queue) => {
   
   // Log retention if messages were deleted
   if (totalDeleted > 0) {
-    // Note: We might need to update retention_history table to use queue_id instead
     log(`${LogTypes.RETENTION} | Queue: ${queue.name} | Count: ${totalDeleted} | RetentionSeconds: ${retentionSeconds} | CompletedRetentionSeconds: ${completedRetentionSeconds}`);
   }
   
@@ -76,6 +89,23 @@ const cleanupEmptyPartitions = async (client) => {
   return result.rowCount;
 };
 
+// Clean up old metrics data (messages_consumed table)
+const cleanupOldMetrics = async (client) => {
+  const retentionDays = config.JOBS.METRICS_RETENTION_DAYS;
+  
+  // Delete metrics data older than the retention period
+  const result = await client.query(`
+    DELETE FROM queen.messages_consumed
+    WHERE acked_at < NOW() - INTERVAL '1 day' * $1
+  `, [retentionDays]);
+  
+  if (result.rowCount > 0) {
+    log(`📊 Deleted ${result.rowCount} old metrics records (older than ${retentionDays} days)`);
+  }
+  
+  return result.rowCount;
+};
+
 // Main retention function
 const performRetention = async (pool) => {
   const client = await pool.connect();
@@ -97,6 +127,9 @@ const performRetention = async (pool) => {
     // Cleanup empty partitions
     await cleanupEmptyPartitions(client);
     
+    // Cleanup old metrics data
+    await cleanupOldMetrics(client);
+    
     return totalDeleted;
   } catch (error) {
     log('Retention error:', error);

package/src/test/MIGRATION_ISSUES.md

@@ -0,0 +1,174 @@
+# Test Migration Issues and Fixes
+
+## Summary
+
+When migrating tests from the old `queenClient` interface to the new minimalist `Queen` client interface, we discovered client bugs that needed fixing. The system is transactional - ACKs immediately release partition locks when awaited.
+
+## Client Bugs Fixed
+
+### 1. Null Payload Handling in `push()`
+**Issue**: `typeof null === 'object'` is `true` in JavaScript, causing the code to try to access properties on `null` payloads.
+
+**Location**: `src/client/client.js` line 153
+
+**Fix**:
+```javascript
+// Before
+const isMessageObject = typeof item === 'object' && 
+  (item._payload || item._transactionId || item._traceId);
+
+// After  
+const isMessageObject = typeof item === 'object' && item !== null &&
+  (item._payload || item._transactionId || item._traceId);
+```
+
+**Why**: Tests correctly validate edge cases including null payloads. The client must handle them gracefully.
+
+### 2. Null Handling in `ack()`
+**Issue**: Same null-checking issue when extracting transaction ID from message object.
+
+**Location**: `src/client/client.js` line 330
+
+**Fix**:
+```javascript
+// Before
+const transactionId = typeof message === 'object' ? 
+  (message.transactionId || message.id) : 
+  message;
+
+// After
+const transactionId = typeof message === 'object' && message !== null ? 
+  (message.transactionId || message.id) : 
+  message;
+```
+
+### 3. Consumer Group Parsing for Namespace/Task Addresses
+**Issue**: The `#parseAddress()` method didn't extract consumer groups from namespace/task patterns like `namespace:billing@group-a`.
+
+**Location**: `src/client/client.js` line 77-101
+
+**Fix**: Extract `@group` portion before parsing namespace/task segments:
+```javascript
+#parseAddress(address) {
+  if (address.includes('namespace:') || address.includes('task:')) {
+    const parts = {};
+    
+    // Extract consumer group if present (after @)
+    let workingAddress = address;
+    const atIndex = address.lastIndexOf('@');
+    if (atIndex > 0) {
+      parts.consumerGroup = address.substring(atIndex + 1);
+      workingAddress = address.substring(0, atIndex);
+    }
+    
+    const segments = workingAddress.split('/');
+    for (const segment of segments) {
+      if (segment.startsWith('namespace:')) {
+        parts.namespace = segment.substring(10);
+      } else if (segment.startsWith('task:')) {
+        parts.task = segment.substring(5);
+      }
+    }
+    return parts;
+  }
+  // ...
+}
+```
+
+### 4. Defensive Filtering of Null Messages
+**Issue**: Server might return null values in the `messages` array response.
+
+**Location**: `src/client/client.js` in `take()` method
+
+**Fix**: Filter out null/undefined messages when yielding:
+```javascript
+// Yield each message (filter out any null/undefined values)
+for (const message of result.messages) {
+  if (message) { // Skip null/undefined messages
+    yield message;
+    count++;
+    if (limit && count >= limit) return;
+  }
+}
+```
+
+## Test Fixes
+
+### 1. Consumer Group Isolation Test
+**Problem**: Test was ACKing all messages as completed for both groups, but the test expects group2 to have some failed messages to verify isolation.
+
+**Solution**: Actually fail the first 2 messages for group2:
+```javascript
+// Fail first 2 messages for group2, complete the rest
+for (let i = 0; i < group2Messages.length; i++) {
+  if (i < 2) {
+    await client.ack(group2Messages[i], false, { 
+      error: 'Test failure', 
+      group: 'group2' 
+    });
+  } else {
+    await client.ack(group2Messages[i], true, { group: 'group2' });
+  }
+}
+```
+
+### 2. Empty and Null Payloads Test
+**Problem**: If server returns null message objects in the array, the test would crash.
+
+**Solution**: Add defensive check (though client now filters these):
+```javascript
+for await (const msg of client.take(queue, { limit: 10 })) {
+  if (!msg) {
+    throw new Error('Received null/undefined message from take()');
+  }
+  messages.push(msg);
+  await client.ack(msg);
+}
+```
+
+## Key Learnings
+
+### System is Transactional ✅
+
+The Queen system is **transactional**. When you `await client.ack(msg)`:
+- The ACK completes atomically
+- The partition lock is released immediately
+- No "timing issues" or delays needed
+
+### Correct Pattern
+```javascript
+// ✅ This works correctly:
+for await (const msg of client.take(queue, { limit: 10 })) {
+  await client.ack(msg); // Partition unlocked when this completes
+}
+
+// Next consumer can immediately access the partition:
+for await (const msg of client.take(queue, { limit: 10 })) {
+  // Gets messages right away
+}
+```
+
+### Edge Case Handling
+
+1. **Null payloads are valid**: Tests should validate that the system handles `null`, `undefined`, empty objects, and empty strings as payloads.
+
+2. **Client must be defensive**: Check for `typeof x === 'object' && x !== null` when you need to access properties.
+
+3. **Server responses should be filtered**: If server returns null messages in an array, filter them out client-side.
+
+### Test Design Principles
+
+1. **Test real edge cases**: null, undefined, empty strings, large payloads, etc.
+
+2. **Don't add artificial delays**: The system is transactional. If you need a delay, something is wrong.
+
+3. **Verify isolation**: When testing consumer groups, actually test that different groups can have different statuses for the same messages.
+
+4. **Use appropriate batch sizes**: Match batch size to expected message count for efficient testing.
+
+## Final Results
+
+After fixing client bugs and test issues:
+- 33/33 tests should pass
+- No timing sensitivity
+- Clean, transactional behavior throughout

package/src/test/README.md

@@ -0,0 +1,203 @@
+# Queen Message Queue Test Suite
+
+This directory contains the comprehensive test suite for the Queen Message Queue system, using the new minimalist `Queen` client interface.
+
+## Test Structure
+
+The test suite is organized into focused, modular files:
+
+### Core Files
+
+- **`test-new.js`** - Main test runner that orchestrates all tests
+- **`utils.js`** - Shared utilities (logging, database helpers, result tracking)
+
+### Test Categories
+
+#### 1. Core Features (`core-tests.js`)
+- Queue creation policy
+- Single/batch message push
+- Queue configuration
+- Take and acknowledgment
+- Delayed processing
+- FIFO ordering within partitions
+
+#### 2. Partition Locking (`partition-locking-tests.js`)
+- Partition locking in queue mode
+- Partition locking in bus mode
+- Specific partition requests with locking
+- Namespace/task filtering with partition locking
+
+#### 3. Enterprise Features (`enterprise-tests.js`)
+- Message encryption/decryption
+- Retention policies (pending & completed messages)
+- Message eviction
+- Combined enterprise features
+- Enterprise error handling
+
+#### 4. Bus Mode Features (`bus-mode-tests.js`)
+- Consumer groups
+- Mixed mode (queue + bus)
+- Subscription modes (all vs new messages)
+- Consumer group isolation
+
+#### 5. Edge Cases (`edge-case-tests.js`)
+- Empty and null payloads
+- Very large payloads
+- Concurrent push/take operations
+- Retry limit exhaustion
+- Lease expiration and redelivery
+- SQL injection prevention
+- XSS prevention
+
+#### 6. Advanced Patterns (`advanced-pattern-tests.js`)
+- Multi-stage pipeline workflow
+- Fan-out/fan-in pattern
+- Dead letter queue pattern
+- Circuit breaker pattern
+- Message deduplication
+
+## New vs Old Interface
+
+### Old Interface (test.js)
+```javascript
+import { createQueenClient } from '../client/queenClient.js';
+
+const client = createQueenClient({ baseUrls: [...] });
+
+// Configure
+await client.configure({ queue: 'myqueue', options: {} });
+
+// Push
+await client.push({ items: [{ queue: 'myqueue', partition: 'Default', payload: {...} }] });
+
+// Pop
+const result = await client.pop({ queue: 'myqueue', batch: 10 });
+
+// Ack
+await client.ack(transactionId, 'completed', null, consumerGroup);
+```
+
+### New Interface (test-new.js)
+```javascript
+import { Queen } from '../client/client.js';
+
+const client = new Queen({ baseUrls: [...] });
+
+// Configure
+await client.queue('myqueue', {}, { namespace, task });
+
+// Push
+await client.push('myqueue/partition', payload);
+
+// Take (async iterator)
+for await (const msg of client.take('myqueue/partition@group', { limit: 10 })) {
+  // Process message
+  await client.ack(msg, true, { group: 'mygroup' });
+}
+```
+
+## Key Differences
+
+1. **Address Format**: The new interface uses a unified address format:
+   - `"queue"` - Simple queue
+   - `"queue/partition"` - Specific partition
+   - `"queue@group"` - Consumer group
+   - `"queue/partition@group"` - Partition + group
+   - `"namespace:name"` - Namespace filter
+   - `"task:name"` - Task filter
+   - `"namespace:billing/task:process"` - Combined filters
+
+2. **Async Iterator**: `pop()` → `take()` using async iteration
+   ```javascript
+   for await (const msg of client.take(address, options)) {
+     // Handle message
+   }
+   ```
+
+3. **Simplified Methods**: 4 core methods instead of many:
+   - `client.queue(name, options, metadata)` - Configure
+   - `client.push(address, payload, options)` - Send
+   - `client.take(address, options)` - Receive (async iterator)
+   - `client.ack(message, status, context)` - Acknowledge
+
+4. **Message Properties**: Can be in payload or options:
+   ```javascript
+   // In options
+   await client.push('queue', { data }, { transactionId: 'txn-1' });
+   
+   // In payload
+   await client.push('queue', { data, transactionId: 'txn-1' });
+   ```
+
+## Running Tests
+
+```bash
+# Make sure Queen servers are running first
+# And database is accessible
+
+cd /Users/alice/Work/queen
+
+# Run all tests
+nvm use 22 && node src/test/test-new.js
+
+# Run specific test category
+node src/test/test-new.js core        # Core features only
+node src/test/test-new.js partition   # Partition locking tests only
+node src/test/test-new.js enterprise  # Enterprise features only
+node src/test/test-new.js bus         # Bus mode tests only
+node src/test/test-new.js edge        # Edge cases only
+node src/test/test-new.js advanced    # Advanced patterns only
+
+# Show help
+node src/test/test-new.js help
+```
+
+### Available Test Categories
+
+- **`core`** - Core features (queue creation, push, take, ack, delayed processing, FIFO)
+- **`partition`** (or `locking`) - Partition locking in queue and bus modes
+- **`enterprise`** - Enterprise features (encryption, retention, eviction)
+- **`bus`** - Bus mode features (consumer groups, mixed mode, subscription modes, isolation)
+- **`edge`** - Edge cases (null payloads, large payloads, concurrency, SQL injection, XSS)
+- **`advanced`** (or `pattern`) - Advanced patterns (pipeline, fan-out/fan-in, DLQ, circuit breaker, deduplication)
+
+## Test Configuration
+
+Tests use environment variables for configuration:
+- `PG_HOST` - PostgreSQL host (default: localhost)
+- `PG_PORT` - PostgreSQL port (default: 5432)
+- `PG_DB` - Database name (default: postgres)
+- `PG_USER` - Database user (default: postgres)
+- `PG_PASSWORD` - Database password (default: postgres)
+- `QUEEN_ENCRYPTION_KEY` - Optional encryption key for encryption tests
+
+## Adding New Tests
+
+1. Create a test function in the appropriate category file:
+   ```javascript
+   export async function testMyFeature(client) {
+     startTest('My Feature Test', 'category');
+     try {
+       // Test code here
+       passTest('Feature works correctly');
+     } catch (error) {
+       failTest(error);
+     }
+   }
+   ```
+
+2. Import and add to `test-new.js`:
+   ```javascript
+   import { testMyFeature } from './category-tests.js';
+   
+   // In runTests():
+   await runTest(() => testMyFeature(client));
+   ```
+
+## Notes
+
+- All tests automatically clean up test data before and after running
+- Tests use the `test-*`, `edge-*`, `pattern-*`, and `workflow-*` queue name prefixes
+- The test suite supports both single-server and multi-server configurations
+- Some enterprise tests (encryption, retention) are skipped if not configured
+