queen-mq 0.2.0 → 0.2.22

package/{src → client-js}/benchmark/consumer.js

@@ -3,7 +3,8 @@ import fs from 'fs';
 
 const QUEUE_NAME = 'benchmark-queue-001';
 const NUMBER_OF_CONSUMERS = 10;
-const BATCH_SIZE = 10;
+const STARTING_PARTITION = parseInt(process.argv[2]) || 0;
+const BATCH_SIZE = 10000;
 const CONSUME_MODE = 'partition';
 
 // Global metrics tracking
@@ -21,7 +22,7 @@ const metrics = {
 
 async function consumer(consumerId, partition) {
   const q = new Queen({
-    baseUrls: ['http://localhost:6632'],
+    baseUrls: [ 'http://localhost:6632'],
     timeout: 30000
   });
   
@@ -49,7 +50,7 @@ async function consumer(consumerId, partition) {
   try {
     for await (const messages of q.takeBatch(target, { //`namespace:benchmark`, 
       wait: true,
-      timeout: 15000,
+      timeout: 30000,
       batch: BATCH_SIZE,
       idleTimeout: 5000
     })) {
@@ -126,8 +127,9 @@ metrics.startTime = Date.now();
 
 // Launch all consumers
 const consumerPromises = [];
+//
 for (let i = 0; i < NUMBER_OF_CONSUMERS; i++) {
-  consumerPromises.push(consumer(i, i));
+  consumerPromises.push(consumer(i, i ));
 }
 
 // Wait for all consumers to complete

package/{src → client-js}/benchmark/producer.js

@@ -1,9 +1,9 @@
 import { Queen } from '../client/client.js'; 
 import fs from 'fs';
 
-const TOTAL_MESSAGES = 100000;
+const TOTAL_MESSAGES = 1000000;
 const PARTITIONS = 10; 
-const MAX_BATCH_SIZE = 1000;  // Reduced from 10000
+const MAX_BATCH_SIZE = 100;  // Reduced from 10000
 const PUSH_PARALLEL = 10;      // Reduced from 10 to avoid lock contention
 const QUEUE_NAME = 'benchmark-queue-001';
 
@@ -12,11 +12,16 @@ const q = new Queen({
   timeout: 120000
 });
 
-//await q.queueDelete(QUEUE_NAME);
-await q.queue(QUEUE_NAME, {
+
+try {
+  await q.queueDelete(QUEUE_NAME);
+  await q.queue(QUEUE_NAME, {
   leaseTime: 10,
   retryLimit: 3
 }, { namespace: 'benchmark' });
+} catch (e) {
+  console.error(e);
+}
 
 
 

package/{src → client-js}/client/client.js

@@ -1,6 +1,7 @@
 import { createHttpClient, createLoadBalancedHttpClient } from './utils/http.js';
 import { withRetry } from './utils/retry.js';
 import { createLoadBalancer, LoadBalancingStrategy } from './utils/loadBalancer.js';
+import { generateUUID } from '../utils/uuid.js';
 
 /**
  * Minimalist Queen Message Queue Client
@@ -16,6 +17,7 @@ export class Queen {
   #http;
   #loadBalancer;
   #connected = false;
+  #bufferManager;
   
   constructor(config = {}) {
     this.#config = {
@@ -27,6 +29,7 @@ export class Queen {
       retryDelay: 1000,
       ...config
     };
+    this.#bufferManager = new BufferManager(this);
   }
   
   /**
@@ -146,7 +149,7 @@ export class Queen {
    * Push messages to a queue
    * @param {string} address - Queue address (e.g., "myqueue" or "myqueue/partition")
    * @param {Object|Array} payload - Single message or array of messages
-   * @param {Object} options - Optional message properties { transactionId, traceId }
+   * @param {Object} options - Optional message properties { transactionId, traceId, buffer }
    */
   async push(address, payload, options = {}) {
     await this.#ensureConnected();
@@ -167,7 +170,7 @@ export class Queen {
           queue,
           partition,
           payload: item._payload || item,
-          transactionId: item._transactionId || item.transactionId
+          transactionId: item._transactionId || item.transactionId || generateUUID()  // Always generate if missing
         };
         
         // Include traceId if provided and valid UUID
@@ -184,7 +187,9 @@ export class Queen {
         queue,
         partition,
         payload: item,
-        transactionId: options.transactionId || (item && typeof item === 'object' ? item.transactionId : undefined)
+        transactionId: options.transactionId || 
+                      (item && typeof item === 'object' ? item.transactionId : undefined) ||
+                      generateUUID()  // Always generate if missing
       };
       
       // Include traceId if provided and valid UUID
@@ -196,10 +201,26 @@ export class Queen {
       return result;
     });
     
+    // Check if client-side buffering is enabled
+    if (options.buffer && typeof options.buffer === 'object') {
+      return this.#pushBuffered(address, formattedItems, options.buffer);
+    }
+    
+    // Build request body for immediate push
+    const requestBody = { items: formattedItems };
+    
+    // Add server-side buffering flag if specified
+    if (options.buffer === true || options.qos0 === true) {
+      requestBody.qos0 = true;  // Server still uses 'qos0' internally
+    }
+    
+    // Legacy support: bufferMs/bufferMax are aliases for buffer
+    if (options.bufferMs !== undefined || options.bufferMax !== undefined) {
+      requestBody.qos0 = true;
+    }
+    
     const result = await withRetry(
-      () => this.#http.post('/api/v1/push', { 
-        items: formattedItems 
-      }),
+      () => this.#http.post('/api/v1/push', requestBody),
       this.#config.retryAttempts,
       this.#config.retryDelay
     );
@@ -211,6 +232,25 @@ export class Queen {
     return result;
   }
   
+  /**
+   * Push messages using client-side buffering
+   * @private
+   */
+  #pushBuffered(address, formattedItems, bufferOptions) {
+    // Validate buffer options
+    if (!bufferOptions.size || !bufferOptions.time) {
+      throw new Error('Buffer options must include both size and time');
+    }
+    
+    // Add each formatted message to the buffer
+    for (const item of formattedItems) {
+      this.#bufferManager.addMessage(address, item, bufferOptions);
+    }
+    
+    // Return immediately (non-blocking)
+    return { buffered: true, count: formattedItems.length };
+  }
+  
   /**
    * Internal method that yields batches of messages (arrays)
    * @private
@@ -266,6 +306,9 @@ export class Queen {
       if (subscriptionMode) params.append('subscriptionMode', subscriptionMode);
       if (subscriptionFrom) params.append('subscriptionFrom', subscriptionFrom);
       
+      // Add auto-ack option if specified
+      if (options.autoAck) params.append('autoAck', 'true');
+      
       // Determine the endpoint based on parameters
       if (queue) {
         if (partition && partition !== 'Default') {
@@ -668,10 +711,45 @@ export class Queen {
     return result;
   }
   
+  /**
+   * Manually flush a specific buffer
+   * @param {string} address - Queue address to flush
+   * @returns {Promise<void>}
+   */
+  async flushBuffer(address) {
+    return this.#bufferManager.flushBuffer(address);
+  }
+  
+  /**
+   * Manually flush all buffers
+   * @returns {Promise<void>}
+   */
+  async flushAllBuffers() {
+    return this.#bufferManager.flushAllBuffers();
+  }
+  
+  /**
+   * Get buffer statistics
+   * @returns {Object} Buffer statistics
+   */
+  getBufferStats() {
+    return this.#bufferManager.getStats();
+  }
+  
   /**
    * Close the client connection
    */
   async close() {
+    // Flush all buffers before closing
+    try {
+      await this.#bufferManager.flushAllBuffers();
+    } catch (error) {
+      console.warn('Error flushing buffers during close:', error);
+    }
+    
+    // Cleanup buffer manager
+    this.#bufferManager.cleanup();
+    
     this.#connected = false;
     this.#http = null;
     this.#loadBalancer = null;
@@ -702,6 +780,194 @@ export class Queen {
   get _http() {
     return this.#http;
   }
+  
+  get _config() {
+    return this.#config;
+  }
+}
+
+/**
+ * Message Buffer for client-side buffering
+ */
+class MessageBuffer {
+  constructor(address, options, flushCallback) {
+    this.address = address;
+    this.messages = [];
+    this.options = options;
+    this.flushCallback = flushCallback;
+    this.timer = null;
+    this.firstMessageTime = null;
+    this.flushing = false;
+  }
+  
+  add(formattedMessage) {
+    // Set first message time if this is the first message
+    if (this.messages.length === 0) {
+      this.firstMessageTime = Date.now();
+      this.#startTimer();
+    }
+    
+    this.messages.push(formattedMessage);
+    
+    // Check if we should flush based on size
+    if (this.messages.length >= this.options.size) {
+      this.#triggerFlush();
+    }
+  }
+  
+  #startTimer() {
+    if (this.timer) return; // Timer already running
+    
+    this.timer = setTimeout(() => {
+      this.#triggerFlush();
+    }, this.options.time);
+  }
+  
+  #triggerFlush() {
+    if (this.flushing || this.messages.length === 0) return;
+    
+    // Clear timer
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+    
+    // Trigger flush via callback
+    this.flushCallback(this.address);
+  }
+  
+  extractMessages() {
+    const messages = [...this.messages];
+    this.messages = [];
+    this.firstMessageTime = null;
+    this.flushing = false;
+    
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+    
+    return messages;
+  }
+  
+  cleanup() {
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+    this.messages = [];
+    this.firstMessageTime = null;
+    this.flushing = false;
+  }
+}
+
+/**
+ * Buffer Manager for client-side message buffering
+ */
+class BufferManager {
+  constructor(client) {
+    this.client = client;
+    this.buffers = new Map(); // address -> MessageBuffer
+    this.flushCount = 0;
+  }
+  
+  addMessage(address, formattedMessage, bufferOptions) {
+    const bufferKey = address;
+    
+    if (!this.buffers.has(bufferKey)) {
+      this.buffers.set(bufferKey, new MessageBuffer(
+        address, 
+        bufferOptions, 
+        (addr) => this.#flushBuffer(addr)
+      ));
+    }
+    
+    const buffer = this.buffers.get(bufferKey);
+    buffer.add(formattedMessage);
+  }
+  
+  async #flushBuffer(address) {
+    const buffer = this.buffers.get(address);
+    if (!buffer || buffer.flushing || buffer.messages.length === 0) {
+      return;
+    }
+    
+    buffer.flushing = true;
+    
+    try {
+      const messages = buffer.extractMessages();
+      
+      if (messages.length === 0) return;
+      
+      // Call the original push logic with the buffered messages
+      const requestBody = { items: messages };
+      
+      await withRetry(
+        () => this.client._http.post('/api/v1/push', requestBody),
+        this.client._config.retryAttempts,
+        this.client._config.retryDelay
+      );
+      
+      this.flushCount++;
+      
+      // Call onFlush callback if provided
+      if (buffer.options.onFlush) {
+        buffer.options.onFlush(address, messages.length);
+      }
+      
+      // Remove empty buffer
+      this.buffers.delete(address);
+      
+    } catch (error) {
+      buffer.flushing = false;
+      
+      // Call onError callback if provided
+      if (buffer.options.onError) {
+        buffer.options.onError(address, error);
+      }
+      
+      throw error;
+    }
+  }
+  
+  async flushBuffer(address) {
+    return this.#flushBuffer(address);
+  }
+  
+  async flushAllBuffers() {
+    const flushPromises = [];
+    for (const address of this.buffers.keys()) {
+      flushPromises.push(this.#flushBuffer(address));
+    }
+    await Promise.all(flushPromises);
+  }
+  
+  getStats() {
+    let totalBufferedMessages = 0;
+    let oldestBufferAge = 0;
+    
+    for (const buffer of this.buffers.values()) {
+      totalBufferedMessages += buffer.messages.length;
+      if (buffer.firstMessageTime) {
+        const age = Date.now() - buffer.firstMessageTime;
+        oldestBufferAge = Math.max(oldestBufferAge, age);
+      }
+    }
+    
+    return {
+      activeBuffers: this.buffers.size,
+      totalBufferedMessages,
+      oldestBufferAge,
+      flushesPerformed: this.flushCount
+    };
+  }
+  
+  cleanup() {
+    for (const buffer of this.buffers.values()) {
+      buffer.cleanup();
+    }
+    this.buffers.clear();
+  }
 }
 
 /**
@@ -978,7 +1244,11 @@ class PipelineBuilder {
           await tx.commit();
         } else {
           // Default: just ACK the messages
-          await this.#client.ack(messages, true);
+          // Extract consumer group from queue address if present (format: queue@group)
+          const atIndex = this.#queue.lastIndexOf('@');
+          const consumerGroup = atIndex > 0 ? this.#queue.substring(atIndex + 1) : null;
+          const context = consumerGroup ? { group: consumerGroup } : {};
+          await this.#client.ack(messages, true, context);
         }
         
         return {

package/{src → client-js}/client/index.js

@@ -1,7 +1,4 @@
 export { LoadBalancingStrategy } from './utils/loadBalancer.js';
 
 // New minimalist interface
-export { Queen } from './client.js';
-
-// Server
-export { QueenServer } from '../server.js';
+export { Queen } from './client.js';

package/{src → client-js}/client/utils/http.js

@@ -1,4 +1,5 @@
 export const httpRequest = async (url, options = {}) => {
+  //console.log('httpRequest', url, options);
   const response = await fetch(url, {
     ...options,
     headers: {
@@ -6,7 +7,7 @@ export const httpRequest = async (url, options = {}) => {
       ...options.headers
     }
   });
-  
+  //console.log('response', response);
   // Handle 204 No Content
   if (response.status === 204) {
     return null; // No content
@@ -143,7 +144,7 @@ export const createLoadBalancedHttpClient = ({ baseUrls, loadBalancer, timeout =
           return await executeRequest(url);
         } catch (error) {
           lastError = error;
-          console.warn(`Request failed for ${url}: ${error.message}`);
+          console.warn(`Request failed for ${url}: ${method} ${path} - ${error.message}`);
           
           // Don't retry on client errors (4xx)
           if (error.status && error.status >= 400 && error.status < 500) {

package/{src → client-js}/client/utils/retry.js

@@ -1,4 +1,10 @@
-import config from '../../config.js';
+const config = {
+  CLIENT: {
+    DEFAULT_RETRY_ATTEMPTS: 3,
+    DEFAULT_RETRY_DELAY: 1000,
+    DEFAULT_RETRY_BACKOFF: 2 
+  }
+}
 
 export const withRetry = async (fn, attempts = config.CLIENT.DEFAULT_RETRY_ATTEMPTS, delay = config.CLIENT.DEFAULT_RETRY_DELAY, backoff = config.CLIENT.DEFAULT_RETRY_BACKOFF) => {
   let lastError;

package/{src → client-js}/test/README.md

@@ -40,7 +40,7 @@ The test suite is organized into focused, modular files:
 - Subscription modes (all vs new messages)
 - Consumer group isolation
 
-#### 5. Edge Cases (`edge-case-tests.js`)
+#### 5. Edge Cases (`edge-case-tests.js` & `partition-transaction-tests.js`)
 - Empty and null payloads
 - Very large payloads
 - Concurrent push/take operations
@@ -48,6 +48,11 @@ The test suite is organized into focused, modular files:
 - Lease expiration and redelivery
 - SQL injection prevention
 - XSS prevention
+- **Partition-scoped transaction_id handling** (`partition-transaction-tests.js`):
+  - Duplicate transaction IDs across partitions with correct ACK targeting
+  - Batch ACK with duplicate transaction IDs
+  - DLQ operations with duplicate transaction IDs
+  - Analytics API with duplicate transaction IDs
 
 #### 6. Advanced Patterns (`advanced-pattern-tests.js`)
 - Multi-stage pipeline workflow
@@ -138,20 +143,36 @@ for await (const msg of client.take('myqueue/partition@group', { limit: 10 })) {
 cd /Users/alice/Work/queen
 
 # Run all tests
-nvm use 22 && node src/test/test-new.js
+nvm use 22 && node client-js/test/test-new.js
 
 # Run specific test category
-node src/test/test-new.js core        # Core features only
-node src/test/test-new.js partition   # Partition locking tests only
-node src/test/test-new.js enterprise  # Enterprise features only
-node src/test/test-new.js bus         # Bus mode tests only
-node src/test/test-new.js edge        # Edge cases only
-node src/test/test-new.js advanced    # Advanced patterns only
+node client-js/test/test-new.js core        # Core features only
+node client-js/test/test-new.js partition   # Partition locking tests only
+node client-js/test/test-new.js enterprise  # Enterprise features only
+node client-js/test/test-new.js bus         # Bus mode tests only
+node client-js/test/test-new.js edge        # Edge cases (includes partition-scoped txn_id tests)
+node client-js/test/test-new.js advanced    # Advanced patterns only
 
 # Show help
-node src/test/test-new.js help
+node client-js/test/test-new.js help
 ```
 
+### Testing Partition-Scoped Transaction ID Fix
+
+The `edge` test category includes critical tests for partition-scoped transaction_id handling:
+
+```bash
+# Run all edge case tests (includes partition transaction_id tests)
+node client-js/test/test-new.js edge
+```
+
+These tests verify that:
+- ✅ The same `transaction_id` can exist in multiple partitions
+- ✅ ACK operations correctly target messages using BOTH `partition_id` AND `transaction_id`
+- ✅ Batch ACK operations scope correctly by partition
+- ✅ DLQ operations only affect the intended partition
+- ✅ Analytics API (`GET /api/v1/messages/:partitionId/:transactionId`) returns the correct message
+
 ### Available Test Categories
 
 - **`core`** - Core features (queue creation, push, take, ack, delayed processing, FIFO)

package/{src → client-js}/test/core-tests.js

@@ -113,19 +113,23 @@ export async function testBatchMessagePush(client) {
     
     // Push batch
     await client.push(`${queue}/batch-partition`, messages);
-    
+
     // Verify messages were stored
     const count = await getMessageCount(queue, 'batch-partition');
+
     if (count !== batchSize) {
       throw new Error(`Expected ${batchSize} messages in database, got ${count}`);
     }
-    
+
     // Take all messages
     const received = [];
     for await (const msg of client.take(`${queue}/batch-partition`, { limit: batchSize })) {
       received.push(msg);
+
       await client.ack(msg);
     }
+
+
     
     if (received.length !== batchSize) {
       throw new Error(`Expected ${batchSize} messages, got ${received.length}`);
@@ -233,6 +237,7 @@ export async function testDelayedProcessing(client) {
     await client.queue(queue, {
       delayedProcessing: 2 // 2 seconds delay
     });
+
     
     const startTime = Date.now();
     

package/{src → client-js}/test/edge-case-tests.js

@@ -105,10 +105,11 @@ export async function testVeryLargePayloads(client) {
     
     // Take and verify
     let received = null;
-    for await (const msg of client.take(queue, { limit: 1 })) {
+    for await (const msg of client.take(queue, { limit: 1, timeout: 20000 })) {
       received = msg.data;
       await client.ack(msg);
     }
+
     
     if (!received || !received.array || received.array.length !== 10000) {
       throw new Error('Large payload corrupted');