queen-mq 0.3.0 → 0.4.0

package/client-js/services/retentionService.js

@@ -1,162 +0,0 @@
-// Functional retention service for message cleanup
-import { log, LogTypes } from '../utils/logger.js';
-import config from '../config.js';
-
-const RETENTION_INTERVAL = config.JOBS.RETENTION_INTERVAL;
-
-// Apply retention policy to a single queue
-const retentionQueue = async (client, queue) => {
-  const { 
-    retention_seconds: retentionSeconds = 0,
-    completed_retention_seconds: completedRetentionSeconds = 0,
-    retention_enabled: retentionEnabled = false 
-  } = queue;
-  
-  if (!retentionEnabled) return 0;
-  
-  let totalDeleted = 0;
-  
-  // Delete old unconsumed messages from all partitions in this queue
-  if (retentionSeconds > 0) {
-    const result = await client.query(`
-      DELETE FROM queen.messages m
-      WHERE m.partition_id IN (
-        SELECT id FROM queen.partitions WHERE queue_id = $1
-      )
-      AND m.created_at < NOW() - INTERVAL '1 second' * $2
-      AND NOT EXISTS (
-          -- Message has been consumed by at least one consumer
-          SELECT 1 
-          FROM queen.partition_consumers pc
-          WHERE pc.partition_id = m.partition_id
-            AND (m.created_at, m.id) <= (pc.last_consumed_created_at, pc.last_consumed_id)
-      )
-      RETURNING id
-    `, [queue.id, retentionSeconds]);
-    
-    totalDeleted += result.rowCount || 0;
-  }
-  
-  // Delete old consumed messages from all partitions in this queue
-  // (messages consumed by ALL consumer groups on that partition)
-  if (completedRetentionSeconds > 0) {
-    const result = await client.query(`
-      DELETE FROM queen.messages m
-      WHERE m.partition_id IN (
-        SELECT id FROM queen.partitions WHERE queue_id = $1
-      )
-      AND m.created_at < NOW() - INTERVAL '1 second' * $2
-      AND NOT EXISTS (
-          -- No consumer group that hasn't consumed this message yet
-          SELECT 1 
-          FROM queen.partition_consumers pc
-          WHERE pc.partition_id = m.partition_id
-            AND ((m.created_at, m.id) > (pc.last_consumed_created_at, pc.last_consumed_id)
-                 OR pc.last_consumed_id IS NULL)
-      )
-      RETURNING id
-    `, [queue.id, completedRetentionSeconds]);
-    
-    totalDeleted += result.rowCount || 0;
-  }
-  
-  // Log retention if messages were deleted
-  if (totalDeleted > 0) {
-    log(`${LogTypes.RETENTION} | Queue: ${queue.name} | Count: ${totalDeleted} | RetentionSeconds: ${retentionSeconds} | CompletedRetentionSeconds: ${completedRetentionSeconds}`);
-  }
-  
-  return totalDeleted;
-};
-
-// Clean up empty partitions (no longer based on partition-level config)
-const cleanupEmptyPartitions = async (client) => {
-  // For now, we keep Default partitions and only clean up truly empty non-default partitions
-  // that haven't had activity for a long time (e.g., 7 days)
-  const result = await client.query(`
-    DELETE FROM queen.partitions p
-    WHERE p.name != 'Default'
-      AND NOT EXISTS (
-        SELECT 1 FROM queen.messages m WHERE m.partition_id = p.id
-      )
-      AND p.last_activity < NOW() - INTERVAL '7 days'
-    RETURNING id, name
-  `);
-  
-  if (result.rowCount > 0) {
-    log(`🗑️  Deleted ${result.rowCount} empty partitions`);
-  }
-  
-  return result.rowCount;
-};
-
-// Clean up old metrics data (messages_consumed table)
-const cleanupOldMetrics = async (client) => {
-  const retentionDays = config.JOBS.METRICS_RETENTION_DAYS;
-  
-  // Delete metrics data older than the retention period
-  const result = await client.query(`
-    DELETE FROM queen.messages_consumed
-    WHERE acked_at < NOW() - INTERVAL '1 day' * $1
-  `, [retentionDays]);
-  
-  if (result.rowCount > 0) {
-    log(`📊 Deleted ${result.rowCount} old metrics records (older than ${retentionDays} days)`);
-  }
-  
-  return result.rowCount;
-};
-
-// Main retention function
-const performRetention = async (pool) => {
-  const client = await pool.connect();
-  
-  try {
-    // Get queues with retention enabled
-    const queuesResult = await client.query(`
-      SELECT q.id, q.name, q.retention_enabled, q.retention_seconds, 
-             q.completed_retention_seconds
-      FROM queen.queues q
-      WHERE q.retention_enabled = true
-    `);
-    
-    let totalDeleted = 0;
-    for (const queue of queuesResult.rows) {
-      totalDeleted += await retentionQueue(client, queue);
-    }
-    
-    // Cleanup empty partitions
-    await cleanupEmptyPartitions(client);
-    
-    // Cleanup old metrics data
-    await cleanupOldMetrics(client);
-    
-    return totalDeleted;
-  } catch (error) {
-    log('Retention error:', error);
-    return 0;
-  } finally {
-    client.release();
-  }
-};
-
-// Start the retention job
-export const startRetentionJob = (pool) => {
-  const intervalId = setInterval(async () => {
-    try {
-      await performRetention(pool);
-    } catch (error) {
-      log('Retention job error:', error);
-    }
-  }, RETENTION_INTERVAL);
-  
-  log(`♻️  Retention job started (interval: ${RETENTION_INTERVAL}ms)`);
-  
-  // Return cleanup function
-  return () => {
-    clearInterval(intervalId);
-    log('Retention job stopped');
-  };
-};
-
-// Export for manual execution
-export const runRetention = performRetention;

package/client-js/services/startupSync.js

@@ -1,35 +0,0 @@
-import { SYSTEM_QUEUE } from '../managers/systemEventManager.js';
-
-export async function syncSystemEvents(client, eventManager, consumerGroup) {
-  console.log('Synchronizing system events...');
-  
-  let processed = 0;
-  let hasMore = true;
-  
-  while (hasMore) {
-    // Pop and process system events in batches using the server's consumer group
-    // This ensures we only process events we haven't seen before
-    const result = await client.pop({
-      queue: SYSTEM_QUEUE,
-      consumerGroup: consumerGroup,  // Use server's unique consumer group
-      batch: 100,
-      wait: false
-    });
-    
-    if (!result.messages || result.messages.length === 0) {
-      hasMore = false;
-      break;
-    }
-    
-    for (const message of result.messages) {
-      await eventManager.processSystemEvent(message.data);
-      await client.ack(message.transactionId, 'completed', null, consumerGroup);
-      processed++;
-    }
-  }
-  
-  if (processed > 0) {
-    console.log(`Synchronized ${processed} system events`);
-  }
-  return processed;
-}

package/client-js/test/README.md

@@ -1,224 +0,0 @@
-# Queen Message Queue Test Suite
-
-This directory contains the comprehensive test suite for the Queen Message Queue system, using the new minimalist `Queen` client interface.
-
-## Test Structure
-
-The test suite is organized into focused, modular files:
-
-### Core Files
-
-- **`test-new.js`** - Main test runner that orchestrates all tests
-- **`utils.js`** - Shared utilities (logging, database helpers, result tracking)
-
-### Test Categories
-
-#### 1. Core Features (`core-tests.js`)
-- Queue creation policy
-- Single/batch message push
-- Queue configuration
-- Take and acknowledgment
-- Delayed processing
-- FIFO ordering within partitions
-
-#### 2. Partition Locking (`partition-locking-tests.js`)
-- Partition locking in queue mode
-- Partition locking in bus mode
-- Specific partition requests with locking
-- Namespace/task filtering with partition locking
-
-#### 3. Enterprise Features (`enterprise-tests.js`)
-- Message encryption/decryption
-- Retention policies (pending & completed messages)
-- Message eviction
-- Combined enterprise features
-- Enterprise error handling
-
-#### 4. Bus Mode Features (`bus-mode-tests.js`)
-- Consumer groups
-- Mixed mode (queue + bus)
-- Subscription modes (all vs new messages)
-- Consumer group isolation
-
-#### 5. Edge Cases (`edge-case-tests.js` & `partition-transaction-tests.js`)
-- Empty and null payloads
-- Very large payloads
-- Concurrent push/take operations
-- Retry limit exhaustion
-- Lease expiration and redelivery
-- SQL injection prevention
-- XSS prevention
-- **Partition-scoped transaction_id handling** (`partition-transaction-tests.js`):
-  - Duplicate transaction IDs across partitions with correct ACK targeting
-  - Batch ACK with duplicate transaction IDs
-  - DLQ operations with duplicate transaction IDs
-  - Analytics API with duplicate transaction IDs
-
-#### 6. Advanced Patterns (`advanced-pattern-tests.js`)
-- Multi-stage pipeline workflow
-- Fan-out/fan-in pattern
-- Dead letter queue pattern
-- Circuit breaker pattern
-- Message deduplication
-
-## New vs Old Interface
-
-### Old Interface (test.js)
-```javascript
-import { createQueenClient } from '../client/queenClient.js';
-
-const client = createQueenClient({ baseUrls: [...] });
-
-// Configure
-await client.configure({ queue: 'myqueue', options: {} });
-
-// Push
-await client.push({ items: [{ queue: 'myqueue', partition: 'Default', payload: {...} }] });
-
-// Pop
-const result = await client.pop({ queue: 'myqueue', batch: 10 });
-
-// Ack
-await client.ack(transactionId, 'completed', null, consumerGroup);
-```
-
-### New Interface (test-new.js)
-```javascript
-import { Queen } from '../client/client.js';
-
-const client = new Queen({ baseUrls: [...] });
-
-// Configure
-await client.queue('myqueue', {}, { namespace, task });
-
-// Push
-await client.push('myqueue/partition', payload);
-
-// Take (async iterator)
-for await (const msg of client.take('myqueue/partition@group', { limit: 10 })) {
-  // Process message
-  await client.ack(msg, true, { group: 'mygroup' });
-}
-```
-
-## Key Differences
-
-1. **Address Format**: The new interface uses a unified address format:
-   - `"queue"` - Simple queue
-   - `"queue/partition"` - Specific partition
-   - `"queue@group"` - Consumer group
-   - `"queue/partition@group"` - Partition + group
-   - `"namespace:name"` - Namespace filter
-   - `"task:name"` - Task filter
-   - `"namespace:billing/task:process"` - Combined filters
-
-2. **Async Iterator**: `pop()` → `take()` using async iteration
-   ```javascript
-   for await (const msg of client.take(address, options)) {
-     // Handle message
-   }
-   ```
-
-3. **Simplified Methods**: 4 core methods instead of many:
-   - `client.queue(name, options, metadata)` - Configure
-   - `client.push(address, payload, options)` - Send
-   - `client.take(address, options)` - Receive (async iterator)
-   - `client.ack(message, status, context)` - Acknowledge
-
-4. **Message Properties**: Can be in payload or options:
-   ```javascript
-   // In options
-   await client.push('queue', { data }, { transactionId: 'txn-1' });
-   
-   // In payload
-   await client.push('queue', { data, transactionId: 'txn-1' });
-   ```
-
-## Running Tests
-
-```bash
-# Make sure Queen servers are running first
-# And database is accessible
-
-cd /Users/alice/Work/queen
-
-# Run all tests
-nvm use 22 && node client-js/test/test-new.js
-
-# Run specific test category
-node client-js/test/test-new.js core        # Core features only
-node client-js/test/test-new.js partition   # Partition locking tests only
-node client-js/test/test-new.js enterprise  # Enterprise features only
-node client-js/test/test-new.js bus         # Bus mode tests only
-node client-js/test/test-new.js edge        # Edge cases (includes partition-scoped txn_id tests)
-node client-js/test/test-new.js advanced    # Advanced patterns only
-
-# Show help
-node client-js/test/test-new.js help
-```
-
-### Testing Partition-Scoped Transaction ID Fix
-
-The `edge` test category includes critical tests for partition-scoped transaction_id handling:
-
-```bash
-# Run all edge case tests (includes partition transaction_id tests)
-node client-js/test/test-new.js edge
-```
-
-These tests verify that:
-- ✅ The same `transaction_id` can exist in multiple partitions
-- ✅ ACK operations correctly target messages using BOTH `partition_id` AND `transaction_id`
-- ✅ Batch ACK operations scope correctly by partition
-- ✅ DLQ operations only affect the intended partition
-- ✅ Analytics API (`GET /api/v1/messages/:partitionId/:transactionId`) returns the correct message
-
-### Available Test Categories
-
-- **`core`** - Core features (queue creation, push, take, ack, delayed processing, FIFO)
-- **`partition`** (or `locking`) - Partition locking in queue and bus modes
-- **`enterprise`** - Enterprise features (encryption, retention, eviction)
-- **`bus`** - Bus mode features (consumer groups, mixed mode, subscription modes, isolation)
-- **`edge`** - Edge cases (null payloads, large payloads, concurrency, SQL injection, XSS)
-- **`advanced`** (or `pattern`) - Advanced patterns (pipeline, fan-out/fan-in, DLQ, circuit breaker, deduplication)
-
-## Test Configuration
-
-Tests use environment variables for configuration:
-- `PG_HOST` - PostgreSQL host (default: localhost)
-- `PG_PORT` - PostgreSQL port (default: 5432)
-- `PG_DB` - Database name (default: postgres)
-- `PG_USER` - Database user (default: postgres)
-- `PG_PASSWORD` - Database password (default: postgres)
-- `QUEEN_ENCRYPTION_KEY` - Optional encryption key for encryption tests
-
-## Adding New Tests
-
-1. Create a test function in the appropriate category file:
-   ```javascript
-   export async function testMyFeature(client) {
-     startTest('My Feature Test', 'category');
-     try {
-       // Test code here
-       passTest('Feature works correctly');
-     } catch (error) {
-       failTest(error);
-     }
-   }
-   ```
-
-2. Import and add to `test-new.js`:
-   ```javascript
-   import { testMyFeature } from './category-tests.js';
-   
-   // In runTests():
-   await runTest(() => testMyFeature(client));
-   ```
-
-## Notes
-
-- All tests automatically clean up test data before and after running
-- Tests use the `test-*`, `edge-*`, `pattern-*`, and `workflow-*` queue name prefixes
-- The test suite supports both single-server and multi-server configurations
-- Some enterprise tests (encryption, retention) are skipped if not configured
-