queen-mq 2.1.0 → 2.3.0

package/README.md

@@ -229,7 +229,11 @@ const queen = new Queen({
   timeoutMillis: 30000,
   retryAttempts: 3,
   loadBalancingStrategy: 'affinity',  // or 'round-robin', 'session'
-  enableFailover: true
+  enableFailover: true,
+  // Opt-in wire forms, off by default (see Binary Push, Push Compression and Framed Pop)
+  binaryPush: false,
+  pushCompression: false,
+  framedPop: false
 })
 ```
 
@@ -515,6 +519,54 @@ rate, and a broker outage shows up as slow pushes with bounded memory rather
 than as messages that quietly disappeared. `close()` flushes with a 30 second
 deadline and logs how many messages were left unsent if it expires.
 
+### Binary Push, Push Compression and Framed Pop
+
+Three opt-in forms of the push, pop and transaction requests. They change how
+the messages travel between the client and the broker and nothing else: the
+calls, their results, deduplication and the delivery guarantees stay the same.
+All three are off by default, and with them off the client sends and reads the
+JSON every broker knows.
+
+```javascript
+const queen = new Queen({
+  url: 'http://localhost:6632',
+  binaryPush: true,       // a push travels as a binary body
+  pushCompression: true,  // with binaryPush: framed and compressed here
+  framedPop: true         // a pop returns the stored records, unpacked here
+})
+```
+
+| Option | What it does |
+| --- | --- |
+| `binaryPush` | A push, direct or buffered, sends its items as one length-prefixed binary body instead of a JSON document. The answer is the same list, one entry per item. |
+| `pushCompression` | Needs `binaryPush`. The client frames each partition's batch the way the broker stores it, mints the message ids (UUIDv7) and compresses the batch with zstd, once. The broker checks the frames and, when it has nothing to add to them, stores those bytes without compressing them again. A transaction that pushes sends its pushes the same way, with the rest of the bundle as JSON in front of them. |
+| `framedPop` | `pop()` and `consume()` ask for the stored records (`frames=true`) and unpack them here into the same message objects, instead of a JSON answer that the broker renders message by message. |
+
+Turn them on when the broker's CPU or the network limits a high message rate:
+the framing, the compression and the unpacking then run in the clients, and the
+batches cross the network compressed. zstd is the one of Node.js (`node:zlib`),
+so there is nothing to install.
+
+**What stays JSON**, with no action from you:
+
+- A push with a `traceId` on one of its items.
+- A transaction with no push, or with a `traceId` on one of its pushes.
+- A pop the broker cannot answer in frames: a pop of a conflating group, and
+  every pop on a broker that runs with an encryption key
+  (`QUEEN_ENCRYPTION_KEY`). The client reads the JSON answer as always.
+- Acks, KV, timers, locks and the admin calls. Ephemeral queues have their own
+  routes and do not change either.
+
+A transaction's `results` come back in the order you added the operations, as
+with JSON. The broker answers a binary transaction with the acks first and the
+pushes after them, and the client puts them back.
+
+**Requires broker 2.3.0 or later** for `binaryPush` and `pushCompression`.
+A broker up to 2.2.0 answers 400 to the binary bodies and the client does not
+fall back to JSON, so turn the two on after every broker of the cluster is
+upgraded. `framedPop` is safe against any broker: an older one ignores the
+parameter and answers JSON.
+
 ### Dead Letter Queue
 
 ```javascript
@@ -997,7 +1049,10 @@ await queen.close()  // Flush buffers and close connections
   loadBalancingStrategy: 'affinity',   // 'affinity' | 'round-robin' | 'session'
   affinityHashRing: 128,
   enableFailover: true,
-  healthRetryAfterMillis: 5000
+  healthRetryAfterMillis: 5000,
+  binaryPush: false,                   // a push as a binary body
+  pushCompression: false,              // with binaryPush: frames compressed in the client
+  framedPop: false                     // a pop as the stored records
 }
 ```
 

package/client-v2/Queen.js

@@ -179,7 +179,7 @@ export class Queen {
   }
 
   #createHttpClient() {
-    const { urls, timeoutMillis, retryAttempts, retryDelayMillis, loadBalancingStrategy, affinityHashRing, healthRetryAfterMillis, enableFailover, bearerToken, headers, hostHeader, retry429 } = this.#config
+    const { urls, timeoutMillis, retryAttempts, retryDelayMillis, loadBalancingStrategy, affinityHashRing, healthRetryAfterMillis, enableFailover, bearerToken, headers, hostHeader, retry429, binaryPush, pushCompression, framedPop } = this.#config
 
     if (urls.length === 1) {
       // Single server
@@ -191,7 +191,10 @@ export class Queen {
         bearerToken,
         headers,
         hostHeader,
-        retry429
+        retry429,
+        binaryPush,
+        pushCompression,
+        framedPop
       })
     }
 
@@ -209,7 +212,10 @@ export class Queen {
       bearerToken,
       headers,
       hostHeader,
-      retry429
+      retry429,
+      binaryPush,
+      pushCompression,
+      framedPop
     })
   }
 

package/client-v2/README.md

@@ -20,6 +20,7 @@ Welcome to Queen client! This is your friendly guide to mastering message queues
 - [Part 12: Message Tracing - Debug Your Workflows](#part-12-message-tracing---debug-your-workflows)
 - [Part 13: Callbacks & Error Handling](#part-13-callbacks--error-handling)
 - [Part 14: Graceful Shutdown](#part-14-graceful-shutdown)
+- [Part 15: Binary Push, Compression & Framed Pop](#part-15-binary-push-compression--framed-pop)
 - [Cheat Sheet](#cheat-sheet)
 
 ---
@@ -1607,6 +1608,67 @@ await queen.close()
 
 ---
 
+## Part 15: Binary Push, Compression & Framed Pop
+
+Everything so far travels as JSON. For a high message rate, the client has three
+opt-in forms of the push, pop and transaction requests. They change how the
+messages travel and nothing else: your calls, their results, deduplication and
+the delivery guarantees stay the same. All three are off by default.
+
+```javascript
+const queen = new Queen({
+  url: 'http://localhost:6632',
+  binaryPush: true,       // a push travels as a binary body
+  pushCompression: true,  // with binaryPush: framed and compressed here
+  framedPop: true         // a pop returns the stored records, unpacked here
+})
+
+// The rest of your code does not change.
+await queen.queue('events').partition('device-7').push([{ data: { n: 1 } }])
+await queen.queue('events').group('workers').consume(async (message) => {
+  console.log(message.data)
+})
+```
+
+### What Each Option Does
+
+| Option | What it does |
+| --- | --- |
+| `binaryPush` | A push, direct or buffered, sends its items as one length-prefixed binary body instead of a JSON document. The answer is the same list, one entry per item. |
+| `pushCompression` | Needs `binaryPush`. The client frames each partition's batch the way the broker stores it, mints the message ids (UUIDv7) and compresses the batch with zstd, once. The broker checks the frames and, when it has nothing to add to them, stores those bytes without compressing them again. A transaction that pushes sends its pushes the same way, with the rest of the bundle as JSON in front of them. |
+| `framedPop` | `pop()` and `consume()` ask for the stored records (`frames=true`) and unpack them here into the same message objects, instead of a JSON answer that the broker renders message by message. |
+
+Turn them on when the broker's CPU or the network limits a high message rate:
+the framing, the compression and the unpacking then run in the clients, and the
+batches cross the network compressed. zstd is the one of Node.js (`node:zlib`),
+so there is nothing to install.
+
+### What Stays JSON
+
+The client falls back by itself in these cases:
+
+- A push with a `traceId` on one of its items.
+- A transaction with no push, or with a `traceId` on one of its pushes.
+- A pop the broker cannot answer in frames: a pop of a conflating group, and
+  every pop on a broker that runs with an encryption key
+  (`QUEEN_ENCRYPTION_KEY`). The client reads the JSON answer as always.
+- Acks, KV, timers, locks and the admin calls. Ephemeral queues have their own
+  routes and do not change either.
+
+A transaction's `results` come back in the order you added the operations, as
+with JSON. The broker answers a binary transaction with the acks first and the
+pushes after them, and the client puts them back.
+
+### Which Broker
+
+`binaryPush` and `pushCompression` need broker 2.3.0 or later. A broker up
+to 2.2.0 answers 400 to the binary bodies and the client does not fall back to
+JSON, so turn the two on after every broker of the cluster is upgraded.
+`framedPop` is safe against any broker: an older one ignores the parameter and
+answers JSON.
+
+---
+
 ## Cheat Sheet
 
 ### Connection
@@ -1883,7 +1945,10 @@ await queen.close()
   loadBalancingStrategy: 'affinity',     // 'affinity' | 'round-robin' | 'session'
   affinityHashRing: 128,
   enableFailover: true,
-  healthRetryAfterMillis: 5000
+  healthRetryAfterMillis: 5000,
+  binaryPush: false,                     // a push as a binary body
+  pushCompression: false,                // with binaryPush: frames compressed in the client
+  framedPop: false                       // a pop as the stored records
 }
 ```
 

package/client-v2/http/HttpClient.js

@@ -3,6 +3,8 @@
  */
 
 import * as logger from '../utils/logger.js'
+import { binaryPushBody, transactionFrames, restoreResultOrder, PUSH_BINARY_CONTENT_TYPE } from './pushBinary.js'
+import { decodePopFrames, isPopFrames } from './popFrames.js'
 
 /**
  * The error a request ends with when its caller aborted it (a consumer being
@@ -142,6 +144,12 @@ export class HttpClient {
   // carries the same virtual Host.
   #hostOverride = null
   #pinnedDispatchers = new Map()
+  // The binary push body and the framed pop answer (pushBinary.js,
+  // popFrames.js). All three off by default: the client then sends and reads
+  // the JSON every broker knows.
+  #binaryPush
+  #pushCompression
+  #framedPop
 
   constructor(options = {}) {
     const {
@@ -169,7 +177,21 @@ export class HttpClient {
       // unbounded (retry forever, paced by backoff) unless maxAttempts is
       // explicitly set here, in which case it applies to both.
       // See PLAN_QUEEN_PROXY_CLOUD.md §4/§9 (client 429 backoff, blocker B4).
-      retry429 = {}
+      retry429 = {},
+      // A push to /api/v1/push goes out as the binary push body instead of
+      // JSON. It needs a broker that accepts that body. Items with a trace id
+      // stay JSON.
+      binaryPush = false,
+      // With binaryPush: the messages are framed as the broker stores them and
+      // each partition's batch is zstd-compressed once, here; the broker keeps
+      // those bytes as they are. The client then mints the message ids. A
+      // transaction that pushes sends its pushes the same way (the binary
+      // transaction body), and the rest of the bundle as JSON in front of them.
+      pushCompression = false,
+      // Pops ask the broker for the messages as the stored records
+      // (frames=true) and unpack them here. A broker that does not know the
+      // parameter answers JSON, which is read as always.
+      framedPop = false
     } = options
 
     this.#baseUrl = baseUrl
@@ -180,6 +202,9 @@ export class HttpClient {
     this.#enableFailover = enableFailover
     this.#bearerToken = bearerToken
     this.#retry429 = retry429 || {}
+    this.#binaryPush = binaryPush === true
+    this.#pushCompression = pushCompression === true
+    this.#framedPop = framedPop === true
 
     // A Host in `headers` can never go out over fetch(); rather than let it be
     // dropped, map it onto the supported mechanism and say so out loud.
@@ -408,11 +433,15 @@ export class HttpClient {
         }
       }
 
-      if (body) {
+      if (body instanceof Uint8Array) {
+        // Already encoded (the binary push body): sent as it is.
+        headers['Content-Type'] = PUSH_BINARY_CONTENT_TYPE
+        options.body = body
+      } else if (body) {
         options.body = JSON.stringify(body)
       }
 
-      const response = await fetch(requestUrl, options)
+      let response = await fetch(requestUrl, options)
 
       // The broker granted the leases of a pop when it sent these headers. From
       // here the caller's abort no longer cuts the read: the body says which
@@ -460,6 +489,19 @@ export class HttpClient {
         throw error
       }
 
+      // A pop that asked for frames: its first bytes say which answer came
+      // back, whatever a proxy made of the Content-Type on the way. The framed
+      // one is unpacked into the object the JSON answer parses to, so its
+      // readers do not tell the two apart; a JSON one (a broker that does not
+      // know the parameter) goes on to be read as always.
+      if (this.#framedPop && isPop(method, url)) {
+        const bytes = new Uint8Array(await response.arrayBuffer())
+        if (isPopFrames(bytes)) {
+          return await decodePopFrames(bytes)
+        }
+        response = new Response(bytes, { status: response.status, headers: response.headers })
+      }
+
       // Parse successful response
       const contentType = response.headers.get('content-type')
       const contentLength = response.headers.get('content-length')
@@ -608,10 +650,32 @@ export class HttpClient {
   // `signal`: aborting it closes the request, which then rejects with an
   // error carrying `aborted: true` -- never retried and never failed over.
   async get(path, requestTimeoutMillis = null, affinityKey = null, retryKind = null, signal = null) {
+    if (this.#framedPop && path.startsWith('/api/v1/pop')) {
+      path += `${path.includes('?') ? '&' : '?'}frames=true`
+    }
     return this.#requestWithFailover('GET', path, null, requestTimeoutMillis, affinityKey, retryKind, signal)
   }
 
   async post(path, body = null, requestTimeoutMillis = null, affinityKey = null, retryKind = null, signal = null) {
+    if (this.#binaryPush && path === '/api/v1/push' && body && Array.isArray(body.items)) {
+      // Encoded once, before any retry: every attempt sends the same bytes
+      // (with pushCompression, the same message ids). null = an item the
+      // binary body does not carry, and the push stays JSON.
+      body = (await binaryPushBody(body.items, this.#pushCompression)) ?? body
+    }
+    if (this.#binaryPush && this.#pushCompression && path === '/api/v1/transaction') {
+      // The pushes of a bundle as frames built and compressed here, the rest
+      // of it as the JSON head in front of them. null = no push, or one the
+      // binary body does not carry: the bundle stays JSON.
+      const framed = await transactionFrames(body)
+      if (framed) {
+        const answer = await this.#requestWithFailover('POST', path, framed.body, requestTimeoutMillis, affinityKey, retryKind, signal)
+        // The broker answers the head's operations before the pushes: the
+        // caller gets the results in the order of its own operations.
+        restoreResultOrder(answer, framed.order)
+        return answer
+      }
+    }
     return this.#requestWithFailover('POST', path, body, requestTimeoutMillis, affinityKey, retryKind, signal)
   }
 

package/client-v2/http/popFrames.js

@@ -0,0 +1,211 @@
+/**
+ * The framed pop answer: what a broker sends for a pop with `frames=true`.
+ * The messages come as the stored records, each still the zstd frame the
+ * producer (or the broker) made, and are unpacked here.
+ *
+ *   "QPF1" | u8 1 | u32 len, the answer's own keys as one JSON object
+ *   u16 len, queue | u16 len, leaseId | u16 len, consumerGroup | u32 claims
+ *   per claim:  u64 partitionId | u16 len, partition | u64 start | u64 end | u32 deliveryAttempt | u32 records
+ *   per record: u64 base_offset | i64 created_at_us | u32 count | u8 codec (0 raw, 1 zstd) | u32 len | bytes
+ *
+ * All integers little-endian. A record's bytes are `count` frames:
+ *
+ *   u32 body_len | u8 flags | u8[16] message id | [u8[16] trace id, flag 1]
+ *   | u16 len, transactionId | [u16 len, producer, flag 2] | payload
+ *
+ * Message i of a record has offset base_offset+i and belongs to the delivery
+ * when that lies in the claim's start..=end. The JSON object in front holds what
+ * the JSON answer has beside its messages: the conflation echo and the
+ * autopilot's choice.
+ */
+
+import { uuidText } from './pushBinary.js'
+import { zstdDecompress } from './zstd.js'
+
+export const POP_FRAMES_CONTENT_TYPE = 'application/x-queen-pop'
+
+const MAGIC = 'QPF1'
+const FLAG_TRACE = 1
+const FLAG_PRODUCER = 2
+const FLAG_ENCRYPTED = 4
+
+const utf8 = new TextDecoder()
+
+function damaged(what) {
+  return new Error(`framed pop answer: ${what}`)
+}
+
+/** Reads little-endian fields off a byte array; a read past the end throws. */
+class In {
+  constructor(bytes, short = 'truncated') {
+    this.bytes = bytes
+    this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+    this.at = 0
+    this.short = short
+  }
+
+  take(length) {
+    if (this.bytes.length - this.at < length) throw damaged(this.short)
+    const slice = this.bytes.subarray(this.at, this.at + length)
+    this.at += length
+    return slice
+  }
+
+  u8() {
+    return this.take(1)[0]
+  }
+
+  u16() {
+    this.take(2)
+    return this.view.getUint16(this.at - 2, true)
+  }
+
+  u32() {
+    this.take(4)
+    return this.view.getUint32(this.at - 4, true)
+  }
+
+  u64() {
+    this.take(8)
+    return this.view.getBigUint64(this.at - 8, true)
+  }
+
+  i64() {
+    this.take(8)
+    return this.view.getBigInt64(this.at - 8, true)
+  }
+
+  text() {
+    return utf8.decode(this.take(this.u16()))
+  }
+
+  rest() {
+    return this.bytes.subarray(this.at)
+  }
+}
+
+/** Whether a pop answer's bytes are the framed one. */
+export function isPopFrames(bytes) {
+  if (!bytes || bytes.length < MAGIC.length + 1) return false
+  for (let i = 0; i < MAGIC.length; i++) {
+    if (bytes[i] !== MAGIC.charCodeAt(i)) return false
+  }
+  return true
+}
+
+/**
+ * Unpack a framed pop answer into the object the JSON answer parses to:
+ *
+ *   { success, queue, partition, partitionId, leaseId, consumerGroup,
+ *     messages: [{ id, transactionId, traceId, data, producerSub, createdAt,
+ *                  partitionId, partition, leaseId, consumerGroup,
+ *                  deliveryAttempt, offset }],
+ *     partitionsClaimed, ...the answer's own keys }
+ *
+ * The messages are in delivery order. Everything that reads a pop answer
+ * (the conflation check, the autopilot's choice, the consumer loop) reads
+ * this object as it reads the JSON one.
+ *
+ * @param {Uint8Array} bytes
+ * @returns {Promise<object>}
+ */
+export async function decodePopFrames(bytes) {
+  if (!isPopFrames(bytes)) throw damaged('not a framed pop answer')
+  const input = new In(bytes)
+  input.take(MAGIC.length)
+  const version = input.u8()
+  let own = null
+  if (version === 1) {
+    const text = utf8.decode(input.take(input.u32()))
+    if (text.length > 0) {
+      try {
+        own = JSON.parse(text)
+      } catch (error) {
+        throw damaged(`its own keys: ${error.message}`)
+      }
+    }
+  } else if (version !== 0) {
+    throw damaged(`unknown version ${version}`)
+  }
+
+  const queue = input.text()
+  const leaseId = input.text()
+  const consumerGroup = input.text()
+  const claims = input.u32()
+  const messages = []
+  let firstPartition = ''
+  let firstPartitionId = ''
+
+  for (let c = 0; c < claims; c++) {
+    const partitionId = input.u64().toString()
+    const partition = input.text()
+    const start = input.u64()
+    const end = input.u64()
+    const deliveryAttempt = input.u32()
+    const records = input.u32()
+    if (c === 0) {
+      firstPartition = partition
+      firstPartitionId = partitionId
+    }
+    for (let r = 0; r < records; r++) {
+      const base = input.u64()
+      const createdAt = new Date(Number(input.i64() / 1000n)).toISOString()
+      const count = input.u32()
+      const codec = input.u8()
+      let raw = input.take(input.u32())
+      if (codec === 1) {
+        try {
+          raw = await zstdDecompress(raw)
+        } catch (error) {
+          throw damaged(`zstd: ${error.message}`)
+        }
+      } else if (codec !== 0) {
+        throw damaged(`unknown codec ${codec}`)
+      }
+      const frames = new In(raw, `a record is shorter than its ${count} frames`)
+      for (let i = 0; i < count; i++) {
+        const frame = new In(frames.take(frames.u32()), 'a frame is malformed')
+        const flags = frame.u8()
+        const id = frame.take(16)
+        const trace = flags & FLAG_TRACE ? frame.take(16) : null
+        const transactionId = frame.text()
+        const producerSub = flags & FLAG_PRODUCER ? frame.text() : null
+        if (flags & FLAG_ENCRYPTED) throw damaged('a frame is encrypted')
+        const offset = base + BigInt(i)
+        if (offset < start || offset > end) continue
+        const payload = frame.rest()
+        messages.push({
+          id: uuidText(id),
+          transactionId,
+          traceId: trace ? uuidText(trace) : null,
+          data: payload.length === 0 ? null : JSON.parse(utf8.decode(payload)),
+          producerSub,
+          createdAt,
+          partitionId,
+          partition,
+          leaseId,
+          consumerGroup,
+          deliveryAttempt,
+          offset: Number(offset)
+        })
+      }
+    }
+  }
+
+  const answer = {
+    success: true,
+    queue,
+    partition: firstPartition,
+    partitionId: firstPartitionId,
+    leaseId,
+    consumerGroup,
+    messages,
+    partitionsClaimed: claims
+  }
+  if (own && typeof own === 'object' && !Array.isArray(own)) {
+    // `frames` only says the answer is this one; the JSON answer has no such key.
+    const { frames: _frames, ...keys } = own
+    Object.assign(answer, keys)
+  }
+  return answer
+}