queen-mq 0.13.0 → 0.14.1

package/client-v2/Queen.js

@@ -350,6 +350,12 @@ export class Queen {
       }
     }
 
+    // Dedupe: with v4 multi-partition pop, all messages in one batch share
+    // the same leaseId (one renew_lease_v2 call extends every claimed
+    // partition_consumers row). Without this, callers passing the full
+    // messages array would issue N redundant identical HTTP calls.
+    leaseIds = [...new Set(leaseIds)]
+
     if (leaseIds.length === 0) {
       logger.warn('Queen.renew', 'No valid lease IDs found for renewal')
       return { success: false, error: 'No valid lease IDs found for renewal' }

package/client-v2/builders/QueueBuilder.js

@@ -37,6 +37,7 @@ export class QueueBuilder {
   #subscriptionMode = CONSUME_DEFAULTS.subscriptionMode
   #subscriptionFrom = CONSUME_DEFAULTS.subscriptionFrom
   #each = false
+  #maxPartitions = 1
 
   // Buffer options
   #bufferOptions = null
@@ -190,6 +191,22 @@ export class QueueBuilder {
     return this
   }
 
+  /**
+   * Pop messages from up to N partitions in a single call (v4 multi-partition pop).
+   *
+   * Use this to drain many sparsely-loaded partitions efficiently. With
+   * `partitions(N)`, the global `batch(B)` budget is shared across all
+   * claimed partitions: at most B total messages, drawn from up to N
+   * partitions, in a single network round-trip. All N share one leaseId
+   * (renewing once extends them all).
+   *
+   * Default 1 = legacy single-partition behavior.
+   */
+  partitions(n) {
+    this.#maxPartitions = Math.max(1, n)
+    return this
+  }
+
   limit(count) {
     this.#limit = count
     return this
@@ -251,6 +268,7 @@ export class QueueBuilder {
       subscriptionMode: this.#subscriptionMode,
       subscriptionFrom: this.#subscriptionFrom,
       each: this.#each,
+      maxPartitions: this.#maxPartitions,
       signal: options.signal
     }
 
@@ -266,6 +284,20 @@ export class QueueBuilder {
     return this
   }
 
+  /**
+   * Set the long-poll timeout in milliseconds. Only meaningful when wait(true)
+   * is also set. Default is CONSUME_DEFAULTS.timeoutMillis (30000ms).
+   *
+   * Use a short timeout in tight consumer loops where you want the long-poll
+   * to release the request frequently (e.g. so a shutdown flag is observed
+   * promptly). The HTTP layer adds 5s of slack so the server-side timeout
+   * always fires first.
+   */
+  timeoutMillis(millis) {
+    this.#timeoutMillis = Math.max(1, millis)
+    return this
+  }
+
   async pop() {
     logger.log('QueueBuilder.pop', { queue: this.#queueName, partition: this.#partition, namespace: this.#namespace, task: this.#task, batch: this.#batch, wait: this.#wait, group: this.#group })
     
@@ -289,6 +321,7 @@ export class QueueBuilder {
       if (effectiveAutoAck) params.append('autoAck', 'true')
       if (this.#subscriptionMode) params.append('subscriptionMode', this.#subscriptionMode)
       if (this.#subscriptionFrom) params.append('subscriptionFrom', this.#subscriptionFrom)
+      if (this.#maxPartitions > 1) params.append('partitions', this.#maxPartitions.toString())
 
       // Generate affinity key for consistent routing to same backend
       const affinityKey = this.#getAffinityKey()

package/client-v2/consumer/ConsumerManager.js

@@ -48,6 +48,7 @@ export class ConsumerManager {
       subscriptionMode,
       subscriptionFrom,
       each,
+      maxPartitions,
       signal
     } = options
 
@@ -67,7 +68,7 @@ export class ConsumerManager {
 
     // Build the path and params for pop requests
     const path = this.#buildPath(queue, partition, namespace, task)
-    const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck)
+    const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions)
     
     // Generate affinity key for consistent routing to same backend
     const affinityKey = this.#getAffinityKey(queue, partition, namespace, task, group)
@@ -389,7 +390,7 @@ export class ConsumerManager {
     throw new Error('Must specify queue, namespace, or task')
   }
 
-  #buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck) {
+  #buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions) {
     const params = new URLSearchParams({
       batch: batch.toString(),
       wait: wait.toString(),
@@ -401,6 +402,8 @@ export class ConsumerManager {
     if (subscriptionFrom) params.append('subscriptionFrom', subscriptionFrom)
     if (namespace) params.append('namespace', namespace)
     if (task) params.append('task', task)
+    // v4 multi-partition pop: drain up to N sparse partitions per call.
+    if (maxPartitions && maxPartitions > 1) params.append('partitions', maxPartitions.toString())
     // NEVER send autoAck for consume - client always manages acking
     // autoAck is only for pop() where server auto-acks immediately
 

package/client-v2/utils/defaults.js

@@ -43,7 +43,8 @@ export const CONSUME_DEFAULTS = {
   renewLease: false,                   // No auto-renewal
   renewLeaseIntervalMillis: null,      // Auto-renewal interval when enabled
   subscriptionMode: null,              // No subscription mode (standard queue mode)
-  subscriptionFrom: null               // No subscription start point
+  subscriptionFrom: null,              // No subscription start point
+  maxPartitions: 1                     // v4 multi-partition pop cap (1 = legacy single-partition)
 }
 
 export const POP_DEFAULTS = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "queen-mq",
-  "version": "0.13.0",
+  "version": "0.14.0001",
   "type": "module",
   "description": "High-performance C++ message queue backed by PostgreSQL",
   "main": "client-v2/index.js",

package/test-v2/consume.js

@@ -612,4 +612,100 @@ export async function autoRenewLease(client) {
     })    
 
     return { success: shouldFailHasFailed === false && shouldPassHasPass === true, message: 'Auto renew lease test completed successfully' }
+}
+
+
+// ============================================================================
+// v4 Multi-Partition Consume Tests
+// ============================================================================
+//
+// Exercises the .partitions(N) builder method against the high-level
+// consume() API (which calls pop_unified_batch_v4 under the hood).
+
+export async function testConsumerMultiPartition(client) {
+    const queueName = 'test-queue-v2-v4-consume-multi'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // Seed 3 partitions × 4 messages each = 12 total.
+    for (let p = 0; p < 3; p++) {
+        const payloads = []
+        for (let m = 0; m < 4; m++) payloads.push({ data: { p, m } })
+        await client.queue(queueName).partition(`p${p}`).push(payloads)
+    }
+
+    // Allow the post-push partition_lookup follow-up to settle.
+    await new Promise(r => setTimeout(r, 500))
+
+    let received = []
+
+    // .partitions(3).batch(100) → up to 100 msgs from up to 3 partitions
+    // in a single pop. .limit(1) ends the consume after one batch.
+    await client
+    .queue(queueName)
+    .batch(100)
+    .partitions(3)
+    .wait(false)
+    .limit(1)
+    .consume(async msgs => {
+        received = msgs
+    })
+
+    if (received.length !== 12) {
+        return { success: false, message: `Expected 12 messages across 3 partitions, got ${received.length}` }
+    }
+
+    const distinctPartitionIds = new Set(received.map(m => m.partitionId))
+    const distinctLeaseIds = new Set(received.map(m => m.leaseId).filter(Boolean))
+
+    if (distinctPartitionIds.size !== 3) {
+        return { success: false, message: `Expected 3 distinct partitionIds, got ${distinctPartitionIds.size}` }
+    }
+    // consume() uses client-side auto-ack which means leaseId may be
+    // empty if server-side autoAck is used. Accept either: shared lease
+    // id (1 distinct value) or all-empty (0).
+    if (distinctLeaseIds.size > 1) {
+        return { success: false, message: `All messages must share at most one leaseId, got ${distinctLeaseIds.size}` }
+    }
+
+    return { success: true, message: 'Consumer with .partitions(3) drained 3 partitions in one batch' }
+}
+
+export async function testConsumerMultiPartitionGlobalCap(client) {
+    const queueName = 'test-queue-v2-v4-consume-cap'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // Seed 5 partitions × 50 messages each = 250 total available.
+    for (let p = 0; p < 5; p++) {
+        const payloads = []
+        for (let m = 0; m < 50; m++) payloads.push({ data: { p, m } })
+        await client.queue(queueName).partition(`p${p}`).push(payloads)
+    }
+
+    await new Promise(r => setTimeout(r, 500))
+
+    let received = []
+
+    // batch(20) is the GLOBAL cap. With partitions(5) we can claim up to
+    // 5 partitions, but the total returned is bounded at 20 messages.
+    await client
+    .queue(queueName)
+    .batch(20)
+    .partitions(5)
+    .wait(false)
+    .limit(1)
+    .consume(async msgs => {
+        received = msgs
+    })
+
+    if (received.length !== 20) {
+        return { success: false, message: `batch(20) must be a hard global cap, got ${received.length}` }
+    }
+
+    return { success: true, message: 'Global batch cap respected across multi-partition consume' }
 }

package/test-v2/maintenance.js

@@ -63,8 +63,22 @@ export async function test_maintenance_mode(client = null) {
     const consumePromise = (async () => {
       while (consumerActive) {
         try {
-          const messages = await queen.queue(QUEUE_NAME).batch(10).wait(false).limit(1).each().pop();
-          
+          // wait(true) long-polls on the server. This is important: pushes
+          // commit before partition_lookup is refreshed (PUSHPOPLOOKUPSOL
+          // is async), so a wait(false) pop can race past freshly-pushed
+          // messages and return empty. Long-poll re-runs the candidate
+          // scan internally and picks up messages once the lookup row
+          // commits — typically within milliseconds.
+          // Use a short timeout (1s) so the consumer notices
+          // consumerActive=false promptly when the test winds down.
+          const messages = await queen.queue(QUEUE_NAME)
+            .batch(10)
+            .wait(true)
+            .timeoutMillis(1000)
+            .limit(1)
+            .each()
+            .pop();
+
           if (messages && messages.length > 0) {
             receivedCount += messages.length;
             console.log(`  📨 Received ${messages.length} messages (total: ${receivedCount})`);

package/test-v2/pop.js

@@ -20,10 +20,15 @@ export async function popNonEmptyQueue(client) {
     .queue('test-queue-v2-pop-non-empty')
     .push([{ data: { message: 'Hello, world!' } }])
 
+    // wait(true) rides out the PUSHPOPLOOKUPSOL race: partition_lookup is
+    // updated asynchronously after push commits, so an immediate
+    // wait(false) wildcard pop can race past the not-yet-committed lookup
+    // row. Long-poll re-runs the candidate scan and picks up the row
+    // once it commits (typically within ms).
     const res = await client
     .queue('test-queue-v2-pop-non-empty')
     .batch(1)
-    .wait(false)
+    .wait(true)
     .pop()
     return { success: res.length === 1 }
 }
@@ -58,10 +63,11 @@ export async function popWithAck(client) {
     .push([{ data: { message: 'Hello, world!' } }]) 
 
 
+    // wait(true): ride out the PUSHPOPLOOKUPSOL race (see popNonEmptyQueue).
     const res = await client
     .queue('test-queue-v2-pop-with-ack')
     .batch(1)
-    .wait(false)
+    .wait(true)
     .pop()
 
     const resAck = await client.ack(res[0])
@@ -88,10 +94,11 @@ export async function popWithAckReconsume(client) {
     .queue('test-queue-v2-pop-with-ack-reconsume')
     .push([{ data: { message: 'Hello, world!' } }])
 
+    // wait(true): ride out the PUSHPOPLOOKUPSOL race (see popNonEmptyQueue).
     const res = await client
     .queue('test-queue-v2-pop-with-ack-reconsume')
     .batch(1)
-    .wait(false)
+    .wait(true)
     .pop()
 
     if (res.length !== 1) {
@@ -112,3 +119,218 @@ export async function popWithAckReconsume(client) {
     return { success: false, message: `Expected 1 message, got ${res2.length}` }
 }
 
+// ============================================================================
+// v4 Multi-Partition Pop Tests
+// ============================================================================
+//
+// Exercises the .partitions(N) builder method end-to-end against the
+// pop_unified_batch_v4 procedure:
+//   * Server drains up to N partitions per call
+//   * Each message carries its own partitionId / leaseId / partition
+//   * batch(B) is a global cap on TOTAL messages across all partitions
+//   * Default (no .partitions() call) preserves single-partition behaviour
+
+export async function popV4MultiPartitionBasic(client) {
+    const queueName = 'test-queue-v2-v4-pop-basic'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // Seed 3 partitions × 2 messages each = 6 total.
+    for (let p = 0; p < 3; p++) {
+        await client
+        .queue(queueName)
+        .partition(`p${p}`)
+        .push([
+            { data: { p, m: 0 } },
+            { data: { p, m: 1 } },
+        ])
+    }
+
+    // Allow the post-push partition_lookup follow-up (PUSHPOPLOOKUPSOL)
+    // to settle before the wildcard scan.
+    await new Promise(r => setTimeout(r, 500))
+
+    const res = await client
+    .queue(queueName)
+    .batch(100)
+    .partitions(3)
+    .wait(false)
+    .pop()
+
+    if (res.length !== 6) {
+        return { success: false, message: `Expected 6 messages across 3 partitions, got ${res.length}` }
+    }
+
+    // Each message must carry its own partitionId; partitionIds must
+    // span all 3 partitions; leaseIds must collapse to a single value.
+    const distinctPartitionIds = new Set(res.map(m => m.partitionId))
+    const distinctLeaseIds = new Set(res.map(m => m.leaseId).filter(Boolean))
+
+    if (distinctPartitionIds.size !== 3) {
+        return { success: false, message: `Expected 3 distinct partitionIds, got ${distinctPartitionIds.size}` }
+    }
+    if (distinctLeaseIds.size !== 1) {
+        return { success: false, message: `All messages must share a single leaseId, got ${distinctLeaseIds.size} distinct` }
+    }
+
+    return { success: true, message: 'Multi-partition pop returned messages spanning 3 partitions with shared leaseId' }
+}
+
+export async function popV4GlobalCap(client) {
+    const queueName = 'test-queue-v2-v4-pop-cap'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // Seed 5 partitions × 100 messages each = 500 total available.
+    for (let p = 0; p < 5; p++) {
+        const payloads = []
+        for (let m = 0; m < 100; m++) payloads.push({ data: { p, m } })
+        await client.queue(queueName).partition(`p${p}`).push(payloads)
+    }
+
+    await new Promise(r => setTimeout(r, 500))
+
+    // batch=10 is the GLOBAL cap, partitions=5 lets us span partitions.
+    // We must receive EXACTLY 10 messages, not 5*10 or 5*100.
+    const res = await client
+    .queue(queueName)
+    .batch(10)
+    .partitions(5)
+    .wait(false)
+    .pop()
+
+    if (res.length !== 10) {
+        return { success: false, message: `batch(10) must be a hard global cap, got ${res.length}` }
+    }
+
+    return { success: true, message: 'Global batch cap respected across multi-partition pop' }
+}
+
+export async function popV4DefaultOne(client) {
+    const queueName = 'test-queue-v2-v4-pop-default'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // Seed 4 partitions × 5 messages each.
+    for (let p = 0; p < 4; p++) {
+        const payloads = []
+        for (let m = 0; m < 5; m++) payloads.push({ data: { p, m } })
+        await client.queue(queueName).partition(`p${p}`).push(payloads)
+    }
+
+    await new Promise(r => setTimeout(r, 500))
+
+    // No .partitions() → max_partitions defaults to 1 → single partition.
+    const res = await client
+    .queue(queueName)
+    .batch(100)
+    .wait(false)
+    .pop()
+
+    if (res.length !== 5) {
+        return { success: false, message: `Default max_partitions=1 should drain only one partition (5 msgs), got ${res.length}` }
+    }
+
+    const distinctPartitionIds = new Set(res.map(m => m.partitionId))
+    if (distinctPartitionIds.size !== 1) {
+        return { success: false, message: `All messages should come from a single partition, got ${distinctPartitionIds.size}` }
+    }
+
+    return { success: true, message: 'Default pop preserves legacy single-partition behaviour' }
+}
+
+export async function popV4MultiPartitionAck(client) {
+    const queueName = 'test-queue-v2-v4-pop-ack'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // Seed 2 partitions × 3 messages each.
+    for (let p = 0; p < 2; p++) {
+        await client.queue(queueName).partition(`p${p}`).push([
+            { data: { p, m: 0 } },
+            { data: { p, m: 1 } },
+            { data: { p, m: 2 } },
+        ])
+    }
+
+    await new Promise(r => setTimeout(r, 500))
+
+    const res = await client
+    .queue(queueName)
+    .batch(100)
+    .partitions(2)
+    .wait(false)
+    .pop()
+
+    if (res.length !== 6) {
+        return { success: false, message: `Expected 6 messages across 2 partitions, got ${res.length}` }
+    }
+
+    // ACK each message individually using its own per-message partitionId
+    // and the shared leaseId. The v4 procedure has set each partition's
+    // batch_size to its actual contribution (3), so once 3 ACKs land for
+    // a given partition_id, that partition's lease auto-releases.
+    const ackResult = await client.ack(res, true)
+    if (!ackResult.success) {
+        return { success: false, message: `Batch ACK failed: ${ackResult.error || 'unknown'}` }
+    }
+
+    return { success: true, message: 'Per-message ACK across multi-partition pop succeeded' }
+}
+
+export async function popV4SharedLeaseRenew(client) {
+    const queueName = 'test-queue-v2-v4-pop-renew'
+    const queue = await client
+    .queue(queueName)
+    .config({ leaseTime: 60 })
+    .create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // Seed 3 partitions × 1 message each.
+    for (let p = 0; p < 3; p++) {
+        await client.queue(queueName).partition(`p${p}`).push([{ data: { p } }])
+    }
+
+    await new Promise(r => setTimeout(r, 500))
+
+    const res = await client
+    .queue(queueName)
+    .batch(100)
+    .partitions(3)
+    .wait(false)
+    .pop()
+
+    if (res.length !== 3) {
+        return { success: false, message: `Expected 3 messages from 3 partitions, got ${res.length}` }
+    }
+
+    // All 3 messages share one leaseId. Passing res to renew() exercises
+    // both the dedup logic AND the server-side multi-row lease extension
+    // (renew_lease_v2 filters on worker_id alone, which matches all 3
+    // partition_consumers rows).
+    const renewResult = await client.renew(res)
+    // renew() returns array when input is array. After dedup, it's a
+    // single result for the shared leaseId.
+    if (!Array.isArray(renewResult) || renewResult.length !== 1) {
+        return { success: false, message: `Expected dedup to collapse to 1 renew call, got ${Array.isArray(renewResult) ? renewResult.length : 'non-array'}` }
+    }
+    if (!renewResult[0].success) {
+        return { success: false, message: `Renew failed: ${renewResult[0].error}` }
+    }
+
+    // Cleanup: release leases by acking everything.
+    await client.ack(res, true)
+
+    return { success: true, message: 'Shared leaseId renewed once and dedup confirmed' }
+}
+

package/test-v2/push.js

@@ -221,10 +221,15 @@ export async function pushLargePayload(client) {
     .queue('test-queue-large-payload')
     .push([{ data: largePayload }])
 
+    // wait(true) rides out the PUSHPOPLOOKUPSOL race: partition_lookup is
+    // updated asynchronously after push commits, so an immediate
+    // wait(false) wildcard pop can race past the not-yet-committed lookup
+    // row. Long-poll re-runs the candidate scan and picks up the row
+    // once it commits (typically within ms).
     const received = await client
     .queue('test-queue-large-payload')
     .batch(1)
-    .wait(false)
+    .wait(true)
     .pop()
     
     if (!received || !received[0].data || !received[0].data.array || received[0].data.array.length !== 10000) {
@@ -243,10 +248,11 @@ export async function pushNullPayload(client) {
     .queue('test-queue-null-payload')
     .push([{ data: null }])
 
+    // wait(true): ride out the PUSHPOPLOOKUPSOL race (see pushLargePayload).
     const received = await client
     .queue('test-queue-null-payload')
     .batch(1)
-    .wait(false)
+    .wait(true)
     .pop()
 
     if (!received || received[0].data !== null) {
@@ -265,10 +271,11 @@ export async function pushEmptyPayload(client) {
     .queue('test-queue-empty-payload')
     .push([{ data: {} }])
 
+    // wait(true): ride out the PUSHPOPLOOKUPSOL race (see pushLargePayload).
     const received = await client
     .queue('test-queue-empty-payload')
     .batch(1)
-    .wait(false)
+    .wait(true)
     .pop()
     
     if (!received || Object.keys(received[0].data).length !== 0) {
@@ -300,7 +307,8 @@ export async function testSimplePushRoundTrip(client) {
     if (!res[0] || res[0].status !== 'queued') {
         return { success: false, message: `Expected queued, got ${JSON.stringify(res[0])}` }
     }
-    const popped = await client.queue(queueName).batch(1).wait(false).pop()
+    // wait(true): ride out the PUSHPOPLOOKUPSOL race (see pushLargePayload).
+    const popped = await client.queue(queueName).batch(1).wait(true).pop()
     if (!popped || popped.length !== 1 || popped[0].data.n !== 1) {
         return { success: false, message: 'Round-trip pop failed' }
     }
@@ -316,8 +324,9 @@ export async function testPushAutoCreatesQueueAndPartition(client) {
     if (!res[0] || res[0].status !== 'queued') {
         return { success: false, message: `Expected queued, got ${JSON.stringify(res[0])}` }
     }
+    // wait(true): ride out the PUSHPOPLOOKUPSOL race (see pushLargePayload).
     const popped = await client.queue(queueName).partition(partitionName)
-        .batch(1).wait(false).pop()
+        .batch(1).wait(true).pop()
     if (!popped || popped.length !== 1) {
         return { success: false, message: 'Auto-created partition did not receive the message' }
     }