queen-mq 0.2.0 → 0.2.11

package/{src → client-js}/benchmark/consumer.js

@@ -3,7 +3,8 @@ import fs from 'fs';
 
 const QUEUE_NAME = 'benchmark-queue-001';
 const NUMBER_OF_CONSUMERS = 10;
-const BATCH_SIZE = 10;
+const STARTING_PARTITION = parseInt(process.argv[2]) || 0;
+const BATCH_SIZE = 10000;
 const CONSUME_MODE = 'partition';
 
 // Global metrics tracking
@@ -21,7 +22,7 @@ const metrics = {
 
 async function consumer(consumerId, partition) {
   const q = new Queen({
-    baseUrls: ['http://localhost:6632'],
+    baseUrls: [ 'http://localhost:6632'],
     timeout: 30000
   });
   
@@ -126,8 +127,9 @@ metrics.startTime = Date.now();
 
 // Launch all consumers
 const consumerPromises = [];
+//
 for (let i = 0; i < NUMBER_OF_CONSUMERS; i++) {
-  consumerPromises.push(consumer(i, i));
+  consumerPromises.push(consumer(i, i ));
 }
 
 // Wait for all consumers to complete

package/{src → client-js}/benchmark/producer.js

@@ -1,9 +1,9 @@
 import { Queen } from '../client/client.js'; 
 import fs from 'fs';
 
-const TOTAL_MESSAGES = 100000;
+const TOTAL_MESSAGES = 1000000;
 const PARTITIONS = 10; 
-const MAX_BATCH_SIZE = 1000;  // Reduced from 10000
+const MAX_BATCH_SIZE = 10000;  // Reduced from 10000
 const PUSH_PARALLEL = 10;      // Reduced from 10 to avoid lock contention
 const QUEUE_NAME = 'benchmark-queue-001';
 
@@ -12,11 +12,16 @@ const q = new Queen({
   timeout: 120000
 });
 
-//await q.queueDelete(QUEUE_NAME);
-await q.queue(QUEUE_NAME, {
+
+try {
+  await q.queueDelete(QUEUE_NAME);
+  await q.queue(QUEUE_NAME, {
   leaseTime: 10,
   retryLimit: 3
 }, { namespace: 'benchmark' });
+} catch (e) {
+  console.error(e);
+}
 
 
 

package/{src → client-js}/client/client.js

@@ -1,6 +1,7 @@
 import { createHttpClient, createLoadBalancedHttpClient } from './utils/http.js';
 import { withRetry } from './utils/retry.js';
 import { createLoadBalancer, LoadBalancingStrategy } from './utils/loadBalancer.js';
+import { generateUUID } from '../utils/uuid.js';
 
 /**
  * Minimalist Queen Message Queue Client
@@ -16,6 +17,7 @@ export class Queen {
   #http;
   #loadBalancer;
   #connected = false;
+  #bufferManager;
   
   constructor(config = {}) {
     this.#config = {
@@ -27,6 +29,7 @@ export class Queen {
       retryDelay: 1000,
       ...config
     };
+    this.#bufferManager = new BufferManager(this);
   }
   
   /**
@@ -146,7 +149,7 @@ export class Queen {
    * Push messages to a queue
    * @param {string} address - Queue address (e.g., "myqueue" or "myqueue/partition")
    * @param {Object|Array} payload - Single message or array of messages
-   * @param {Object} options - Optional message properties { transactionId, traceId }
+   * @param {Object} options - Optional message properties { transactionId, traceId, buffer }
    */
   async push(address, payload, options = {}) {
     await this.#ensureConnected();
@@ -167,7 +170,7 @@ export class Queen {
           queue,
           partition,
           payload: item._payload || item,
-          transactionId: item._transactionId || item.transactionId
+          transactionId: item._transactionId || item.transactionId || generateUUID()  // Always generate if missing
         };
         
         // Include traceId if provided and valid UUID
@@ -184,7 +187,9 @@ export class Queen {
         queue,
         partition,
         payload: item,
-        transactionId: options.transactionId || (item && typeof item === 'object' ? item.transactionId : undefined)
+        transactionId: options.transactionId || 
+                      (item && typeof item === 'object' ? item.transactionId : undefined) ||
+                      generateUUID()  // Always generate if missing
       };
       
       // Include traceId if provided and valid UUID
@@ -196,10 +201,26 @@ export class Queen {
       return result;
     });
     
+    // Check if client-side buffering is enabled
+    if (options.buffer && typeof options.buffer === 'object') {
+      return this.#pushBuffered(address, formattedItems, options.buffer);
+    }
+    
+    // Build request body for immediate push
+    const requestBody = { items: formattedItems };
+    
+    // Add server-side buffering flag if specified
+    if (options.buffer === true || options.qos0 === true) {
+      requestBody.qos0 = true;  // Server still uses 'qos0' internally
+    }
+    
+    // Legacy support: bufferMs/bufferMax are aliases for buffer
+    if (options.bufferMs !== undefined || options.bufferMax !== undefined) {
+      requestBody.qos0 = true;
+    }
+    
     const result = await withRetry(
-      () => this.#http.post('/api/v1/push', { 
-        items: formattedItems 
-      }),
+      () => this.#http.post('/api/v1/push', requestBody),
       this.#config.retryAttempts,
       this.#config.retryDelay
     );
@@ -211,6 +232,25 @@ export class Queen {
     return result;
   }
   
+  /**
+   * Push messages using client-side buffering
+   * @private
+   */
+  #pushBuffered(address, formattedItems, bufferOptions) {
+    // Validate buffer options
+    if (!bufferOptions.size || !bufferOptions.time) {
+      throw new Error('Buffer options must include both size and time');
+    }
+    
+    // Add each formatted message to the buffer
+    for (const item of formattedItems) {
+      this.#bufferManager.addMessage(address, item, bufferOptions);
+    }
+    
+    // Return immediately (non-blocking)
+    return { buffered: true, count: formattedItems.length };
+  }
+  
   /**
    * Internal method that yields batches of messages (arrays)
    * @private
@@ -266,6 +306,9 @@ export class Queen {
       if (subscriptionMode) params.append('subscriptionMode', subscriptionMode);
       if (subscriptionFrom) params.append('subscriptionFrom', subscriptionFrom);
       
+      // Add auto-ack option if specified
+      if (options.autoAck) params.append('autoAck', 'true');
+      
       // Determine the endpoint based on parameters
       if (queue) {
         if (partition && partition !== 'Default') {
@@ -668,10 +711,45 @@ export class Queen {
     return result;
   }
   
+  /**
+   * Manually flush a specific buffer
+   * @param {string} address - Queue address to flush
+   * @returns {Promise<void>}
+   */
+  async flushBuffer(address) {
+    return this.#bufferManager.flushBuffer(address);
+  }
+  
+  /**
+   * Manually flush all buffers
+   * @returns {Promise<void>}
+   */
+  async flushAllBuffers() {
+    return this.#bufferManager.flushAllBuffers();
+  }
+  
+  /**
+   * Get buffer statistics
+   * @returns {Object} Buffer statistics
+   */
+  getBufferStats() {
+    return this.#bufferManager.getStats();
+  }
+  
   /**
    * Close the client connection
    */
   async close() {
+    // Flush all buffers before closing
+    try {
+      await this.#bufferManager.flushAllBuffers();
+    } catch (error) {
+      console.warn('Error flushing buffers during close:', error);
+    }
+    
+    // Cleanup buffer manager
+    this.#bufferManager.cleanup();
+    
     this.#connected = false;
     this.#http = null;
     this.#loadBalancer = null;
@@ -702,6 +780,194 @@ export class Queen {
   get _http() {
     return this.#http;
   }
+  
+  get _config() {
+    return this.#config;
+  }
+}
+
+/**
+ * Message Buffer for client-side buffering
+ */
+class MessageBuffer {
+  constructor(address, options, flushCallback) {
+    this.address = address;
+    this.messages = [];
+    this.options = options;
+    this.flushCallback = flushCallback;
+    this.timer = null;
+    this.firstMessageTime = null;
+    this.flushing = false;
+  }
+  
+  add(formattedMessage) {
+    // Set first message time if this is the first message
+    if (this.messages.length === 0) {
+      this.firstMessageTime = Date.now();
+      this.#startTimer();
+    }
+    
+    this.messages.push(formattedMessage);
+    
+    // Check if we should flush based on size
+    if (this.messages.length >= this.options.size) {
+      this.#triggerFlush();
+    }
+  }
+  
+  #startTimer() {
+    if (this.timer) return; // Timer already running
+    
+    this.timer = setTimeout(() => {
+      this.#triggerFlush();
+    }, this.options.time);
+  }
+  
+  #triggerFlush() {
+    if (this.flushing || this.messages.length === 0) return;
+    
+    // Clear timer
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+    
+    // Trigger flush via callback
+    this.flushCallback(this.address);
+  }
+  
+  extractMessages() {
+    const messages = [...this.messages];
+    this.messages = [];
+    this.firstMessageTime = null;
+    this.flushing = false;
+    
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+    
+    return messages;
+  }
+  
+  cleanup() {
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+    this.messages = [];
+    this.firstMessageTime = null;
+    this.flushing = false;
+  }
+}
+
+/**
+ * Buffer Manager for client-side message buffering
+ */
+class BufferManager {
+  constructor(client) {
+    this.client = client;
+    this.buffers = new Map(); // address -> MessageBuffer
+    this.flushCount = 0;
+  }
+  
+  addMessage(address, formattedMessage, bufferOptions) {
+    const bufferKey = address;
+    
+    if (!this.buffers.has(bufferKey)) {
+      this.buffers.set(bufferKey, new MessageBuffer(
+        address, 
+        bufferOptions, 
+        (addr) => this.#flushBuffer(addr)
+      ));
+    }
+    
+    const buffer = this.buffers.get(bufferKey);
+    buffer.add(formattedMessage);
+  }
+  
+  async #flushBuffer(address) {
+    const buffer = this.buffers.get(address);
+    if (!buffer || buffer.flushing || buffer.messages.length === 0) {
+      return;
+    }
+    
+    buffer.flushing = true;
+    
+    try {
+      const messages = buffer.extractMessages();
+      
+      if (messages.length === 0) return;
+      
+      // Call the original push logic with the buffered messages
+      const requestBody = { items: messages };
+      
+      await withRetry(
+        () => this.client._http.post('/api/v1/push', requestBody),
+        this.client._config.retryAttempts,
+        this.client._config.retryDelay
+      );
+      
+      this.flushCount++;
+      
+      // Call onFlush callback if provided
+      if (buffer.options.onFlush) {
+        buffer.options.onFlush(address, messages.length);
+      }
+      
+      // Remove empty buffer
+      this.buffers.delete(address);
+      
+    } catch (error) {
+      buffer.flushing = false;
+      
+      // Call onError callback if provided
+      if (buffer.options.onError) {
+        buffer.options.onError(address, error);
+      }
+      
+      throw error;
+    }
+  }
+  
+  async flushBuffer(address) {
+    return this.#flushBuffer(address);
+  }
+  
+  async flushAllBuffers() {
+    const flushPromises = [];
+    for (const address of this.buffers.keys()) {
+      flushPromises.push(this.#flushBuffer(address));
+    }
+    await Promise.all(flushPromises);
+  }
+  
+  getStats() {
+    let totalBufferedMessages = 0;
+    let oldestBufferAge = 0;
+    
+    for (const buffer of this.buffers.values()) {
+      totalBufferedMessages += buffer.messages.length;
+      if (buffer.firstMessageTime) {
+        const age = Date.now() - buffer.firstMessageTime;
+        oldestBufferAge = Math.max(oldestBufferAge, age);
+      }
+    }
+    
+    return {
+      activeBuffers: this.buffers.size,
+      totalBufferedMessages,
+      oldestBufferAge,
+      flushesPerformed: this.flushCount
+    };
+  }
+  
+  cleanup() {
+    for (const buffer of this.buffers.values()) {
+      buffer.cleanup();
+    }
+    this.buffers.clear();
+  }
 }
 
 /**

package/{src → client-js}/client/index.js

@@ -1,7 +1,4 @@
 export { LoadBalancingStrategy } from './utils/loadBalancer.js';
 
 // New minimalist interface
-export { Queen } from './client.js';
-
-// Server
-export { QueenServer } from '../server.js';
+export { Queen } from './client.js';

package/{src → client-js}/client/utils/http.js

@@ -1,4 +1,5 @@
 export const httpRequest = async (url, options = {}) => {
+  //console.log('httpRequest', url, options);
   const response = await fetch(url, {
     ...options,
     headers: {
@@ -6,7 +7,7 @@ export const httpRequest = async (url, options = {}) => {
       ...options.headers
     }
   });
-  
+  //console.log('response', response);
   // Handle 204 No Content
   if (response.status === 204) {
     return null; // No content
@@ -143,7 +144,7 @@ export const createLoadBalancedHttpClient = ({ baseUrls, loadBalancer, timeout =
           return await executeRequest(url);
         } catch (error) {
           lastError = error;
-          console.warn(`Request failed for ${url}: ${error.message}`);
+          console.warn(`Request failed for ${url}: ${method} ${path} - ${error.message}`);
           
           // Don't retry on client errors (4xx)
           if (error.status && error.status >= 400 && error.status < 500) {

package/{src → client-js}/client/utils/retry.js

@@ -1,4 +1,10 @@
-import config from '../../config.js';
+const config = {
+  CLIENT: {
+    DEFAULT_RETRY_ATTEMPTS: 3,
+    DEFAULT_RETRY_DELAY: 1000,
+    DEFAULT_RETRY_BACKOFF: 2 
+  }
+}
 
 export const withRetry = async (fn, attempts = config.CLIENT.DEFAULT_RETRY_ATTEMPTS, delay = config.CLIENT.DEFAULT_RETRY_DELAY, backoff = config.CLIENT.DEFAULT_RETRY_BACKOFF) => {
   let lastError;

package/{src → client-js}/test/core-tests.js

@@ -113,19 +113,23 @@ export async function testBatchMessagePush(client) {
     
     // Push batch
     await client.push(`${queue}/batch-partition`, messages);
-    
+
     // Verify messages were stored
     const count = await getMessageCount(queue, 'batch-partition');
+
     if (count !== batchSize) {
       throw new Error(`Expected ${batchSize} messages in database, got ${count}`);
     }
-    
+
     // Take all messages
     const received = [];
     for await (const msg of client.take(`${queue}/batch-partition`, { limit: batchSize })) {
       received.push(msg);
+
       await client.ack(msg);
     }
+
+
     
     if (received.length !== batchSize) {
       throw new Error(`Expected ${batchSize} messages, got ${received.length}`);
@@ -233,6 +237,7 @@ export async function testDelayedProcessing(client) {
     await client.queue(queue, {
       delayedProcessing: 2 // 2 seconds delay
     });
+
     
     const startTime = Date.now();
     

package/{src → client-js}/test/edge-case-tests.js

@@ -105,10 +105,11 @@ export async function testVeryLargePayloads(client) {
     
     // Take and verify
     let received = null;
-    for await (const msg of client.take(queue, { limit: 1 })) {
+    for await (const msg of client.take(queue, { limit: 1, timeout: 20000 })) {
       received = msg.data;
       await client.ack(msg);
     }
+
     
     if (!received || !received.array || received.array.length !== 10000) {
       throw new Error('Large payload corrupted');

package/client-js/test/human.js

@@ -0,0 +1,162 @@
+/**
+ * Test written by an human
+ */
+
+import { startTest, passTest, failTest, sleep, log } from './utils.js';
+
+/**
+ * Test: Pipeline API - Basic Processing
+ */
+export async function testCreateQueue(client) {
+    startTest('Create Queue', 'human');
+
+    try {
+        const queue = 'test-create-queue-hm';
+        await client.queue(queue, { leaseTime: 30 });
+        passTest('Queue created successfully');
+    } catch (error) {
+        failTest('Create Queue', error  );
+    }
+}
+
+export async function workerQueueTest(client) {
+    startTest('Worker Queue', 'human');
+
+    try {
+        const queue = 'test-worker-queue-hm';
+        const processedQueue = queue + '-processed';
+        await client.queueDelete(queue);
+        await client.queueDelete(processedQueue); 
+        await client.queue(queue, { leaseTime: 30 });
+        await client.queue(processedQueue, { leaseTime: 30 });
+        const pushMessages = async (count) => {
+            let messages = [];
+            for (let i = 0; i < count; i++) {
+                messages.push({ id: i, data: `Message ${i}` });
+            }
+            await client.push(queue, messages); 
+        }
+
+
+        // Test 1
+        await pushMessages(10);
+
+        let counter1 = 0;
+        await client
+        .pipeline(queue)
+        .take(10)
+        .process(async (message) => {
+            counter1++;
+            return { id: message.data.id, data: message.data.data };
+        })
+        .execute();
+
+        if (counter1 !== 10) {
+            throw new Error('Expected 10 messages to be processed, got ' + counter1);
+        }
+
+        // Test 2
+        await pushMessages(100); 
+
+        let counter2 = 0;
+        await client 
+        .pipeline(queue)
+        .take(10)
+        .processBatch(async (messages) => {
+            counter2 += messages.length;
+            return messages;
+        })
+        .repeat({ maxIterations: 1, continuous: false })
+        .execute();
+
+        if (counter2 !== 10) {
+            throw new Error('Expected 10 messages to be processed, got ' + counter2);
+        }
+
+        // Test 3
+        let counter3 = 0;
+        await client 
+        .pipeline(queue)
+        .take(10)
+        .processBatch(async (messages) => {
+            counter3 += messages.length;
+            return messages;
+        })
+        .repeat({ maxIterations: 9, continuous: false })
+        .execute();        
+
+        if (counter3 !== 90) {
+            throw new Error('Expected 90 messages to be processed, got ' + counter3);
+        }
+
+        await pushMessages(100);
+
+        await client
+        .pipeline(queue)
+        .take(10)
+        .processBatch(async (messages) => {
+            return messages;
+        })
+        .atomically((tx, originalMessages, processedMessages) => {
+            tx.ack(originalMessages);
+        })
+        .repeat({ maxIterations: 10, continuous: true })
+        .execute();
+
+        const messagesPopped = await client.takeSingleBatch(queue);
+        let counter4 = messagesPopped.length;
+        if (counter4 !== 0) {
+            throw new Error('Expected 0 message to be processed, got ' + counter4);
+        }
+
+        // Test transaction that works correctly
+        await pushMessages(100);
+
+        await client
+        .pipeline(queue)
+        .take(10)
+        .processBatch(async (messages) => {
+            return messages;
+        })
+        .atomically((tx, originalMessages, processedMessages) => {
+            tx.ack(originalMessages);
+            tx.push(processedQueue, processedMessages);
+        })
+        .repeat({ maxIterations: 10, continuous: true })
+        .execute();
+
+        const messagesPopped2 = [];
+        for await (const messages of client.takeBatch(processedQueue, { batch: 100 })) {
+            messagesPopped2.push(...messages);
+        }
+        if (messagesPopped2.length !== 100) {
+            throw new Error('Expected 100 message to be processed, got ' + counter5);
+        }
+
+        // Test transaction that fails
+        
+        await pushMessages(100);
+
+        try {
+        await client
+        .pipeline(queue)
+        .take(10)
+        .processBatch(async (messages) => {
+            return messages;
+        })
+        .atomically((tx, originalMessages, processedMessages) => {
+            originalMessages.map(x => x.leaseId = 'invalid-lease-id-12345');
+            tx.ack(originalMessages);
+            tx.push(processedQueue, processedMessages);
+        })
+        .repeat({ maxIterations: 10, continuous: true })
+        .execute();
+        failTest('Expected transaction to fail, got ' + error);
+        } catch (error) {}
+
+        passTest('Worker Queue test passed');
+    } catch (error) {
+        console.error(error);
+        failTest('Worker Queue test failed', error);
+    }
+}