queen-mq 2.1.0 → 2.6.0

package/test-v2/reads.js

@@ -0,0 +1,109 @@
+/**
+ * Reads by offset (queue.fetch) and partition discovery (queue.changedSince)
+ * against a live broker.
+ *
+ * Run: node run.js fetchReadsEveryRecordByOffset
+ *      node run.js changedSinceListsTheWrittenPartitions
+ */
+
+const QUEUE_FETCH = 'test-queue-v2-reads-fetch'
+const QUEUE_CHANGED = 'test-queue-v2-reads-changed'
+
+export async function fetchReadsEveryRecordByOffset(client) {
+  // 1,200 partitions: more than one fetch call (1000 entries each).
+  const partitions = Array.from({ length: 1200 }, (_, i) => `entity-${i}`)
+  const perPartition = 3
+  for (let n = 0; n < perPartition; n++) {
+    const items = partitions.map(p => ({ partition: p, transactionId: `${p}-${n}`, data: { p, n } }))
+    for (let i = 0; i < items.length; i += 500) {
+      const chunk = items.slice(i, i + 500)
+      // One push per partition group keeps the per-partition order of n.
+      await Promise.all(chunk.map(x => client.queue(QUEUE_FETCH).partition(x.partition).push([{ transactionId: x.transactionId, data: x.data }])))
+    }
+  }
+
+  const read = await client.queue(QUEUE_FETCH).fetch(partitions)
+  if (read.size !== partitions.length) {
+    return { success: false, message: `fetch answered ${read.size} partitions for ${partitions.length}` }
+  }
+  for (const p of partitions) {
+    const part = read.get(p)
+    const got = part.records.map(r => `${r.offset}:${r.payload.p}:${r.payload.n}:${r.transactionId}`)
+    const want = Array.from({ length: perPartition }, (_, n) => `${n}:${p}:${n}:${p}-${n}`)
+    if (got.join(',') !== want.join(',')) {
+      return { success: false, message: `${p}: read ${got.join(',')}, expected ${want.join(',')}` }
+    }
+    if (part.highWatermark !== perPartition || part.nextOffset !== perPartition || part.logStartOffset !== 0) {
+      return { success: false, message: `${p}: bounds ${JSON.stringify(part)}` }
+    }
+  }
+
+  // A later read from nextOffset answers only what came after.
+  await client.queue(QUEUE_FETCH).partition('entity-7').push([{ transactionId: 'entity-7-late', data: { late: true } }])
+  const later = await client.queue(QUEUE_FETCH).fetch(['entity-7', 'entity-8'], {
+    offsets: { 'entity-7': read.get('entity-7').nextOffset, 'entity-8': read.get('entity-8').nextOffset }
+  })
+  const late = later.get('entity-7').records
+  if (late.length !== 1 || late[0].offset !== perPartition || late[0].payload.late !== true) {
+    return { success: false, message: `read from nextOffset answered ${JSON.stringify(late)}` }
+  }
+  if (later.get('entity-8').records.length !== 0) {
+    return { success: false, message: 'a partition with nothing new answered records' }
+  }
+
+  // The reads took no lease and moved no cursor: a pop still gets the message.
+  const popped = await client.queue(QUEUE_FETCH).partition('entity-9').batch(10).wait(false).pop()
+  if (popped.length !== perPartition) {
+    return { success: false, message: `a pop after the fetch got ${popped.length} messages, expected ${perPartition}` }
+  }
+
+  const missing = await client.queue('test-queue-v2-reads-does-not-exist').fetch(['x'])
+  if (missing.get('x').records.length !== 0) {
+    return { success: false, message: 'a queue that does not exist answered records' }
+  }
+  return { success: true, message: `read ${partitions.length} partitions by offset` }
+}
+
+export async function changedSinceListsTheWrittenPartitions(client) {
+  const before = new Date(Date.now() - 60_000)
+  const names = Array.from({ length: 1100 }, (_, i) => `c-${i}`)
+  for (let i = 0; i < names.length; i += 100) {
+    await Promise.all(names.slice(i, i + 100).map(p => client.queue(QUEUE_CHANGED).partition(p).push([{ data: { p } }])))
+  }
+
+  // 1,100 partitions: two pages.
+  const first = await client.queue(QUEUE_CHANGED).changedSince(before)
+  const listed = new Set(first.partitions.map(p => p.name))
+  const missing = names.filter(n => !listed.has(n))
+  if (missing.length > 0) {
+    return { success: false, message: `changedSince missed ${missing.length} partitions, e.g. ${missing.slice(0, 3)}` }
+  }
+  if (first.partitions.length !== names.length || typeof first.safeTime !== 'string') {
+    return { success: false, message: `changedSince answered ${first.partitions.length} partitions, safeTime ${first.safeTime}` }
+  }
+  const one = first.partitions.find(p => p.name === 'c-5')
+  if (one.lastOffset !== 0 || one.logStart !== 0 || typeof one.lastWriteAt !== 'string' || typeof one.id !== 'string') {
+    return { success: false, message: `partition fields ${JSON.stringify(one)}` }
+  }
+
+  // From the safeTime of that pass, a write to one partition lists it, and
+  // nothing written before that safeTime comes back.
+  await client.queue(QUEUE_CHANGED).partition('c-42').push([{ data: { again: true } }])
+  const next = await client.queue(QUEUE_CHANGED).changedSince(first.safeTime)
+  if (!next.partitions.some(p => p.name === 'c-42' && p.lastOffset === 1)) {
+    return { success: false, message: `c-42 not listed after its write: ${JSON.stringify(next.partitions.slice(0, 5))}` }
+  }
+  if (next.partitions.some(p => p.lastWriteAt < first.safeTime)) {
+    return { success: false, message: 'a partition written before safeTime was listed' }
+  }
+
+  const all = await client.queue(QUEUE_CHANGED).changedSince()
+  if (all.partitions.length !== names.length) {
+    return { success: false, message: `changedSince() without since answered ${all.partitions.length}` }
+  }
+  const none = await client.queue('test-queue-v2-reads-does-not-exist').changedSince(before)
+  if (none.partitions.length !== 0) {
+    return { success: false, message: 'a queue that does not exist listed partitions' }
+  }
+  return { success: true, message: `listed ${names.length} partitions over two pages` }
+}

package/test-v2/run.js

@@ -20,6 +20,7 @@ import * as timerTests from './timers.js'
 import * as lockTests from './locks.js'
 import * as streamTests from './stream/index.js'
 import * as docsTests from './docs.js'
+import * as readsTests from './reads.js'
 import { LoadBalancer } from '../client-v2/http/LoadBalancer.js';
 
 
@@ -117,6 +118,7 @@ async function main() {
         kvTests,
         timerTests,
         lockTests,
+        readsTests,
         docsTests
     ]
     

package/test-v2/wire-unit/_wireServer.js

@@ -0,0 +1,47 @@
+/**
+ * A real node:http server for the binary wire forms: it records every request
+ * with its body as bytes (kv-unit/_planServer.js reads bodies as text, which a
+ * binary push body does not survive) and answers what `answer(hit)` returns:
+ * `{ status, contentType, body }`, body a Buffer or a string.
+ */
+
+import { createServer } from 'node:http'
+
+export function withWireServer(answer, run) {
+  const hits = []
+
+  const server = createServer((req, res) => {
+    const chunks = []
+    req.on('data', chunk => chunks.push(chunk))
+    req.on('end', () => {
+      const hit = { method: req.method, url: req.url, headers: req.headers, raw: Buffer.concat(chunks) }
+      hits.push(hit)
+      const { status = 200, contentType = 'application/json', body = '' } = answer(hit) || {}
+      if (status === 204) {
+        res.writeHead(204)
+        res.end()
+        return
+      }
+      res.writeHead(status, { 'Content-Type': contentType })
+      res.end(body)
+    })
+  })
+
+  return new Promise((resolve, reject) => {
+    server.listen(0, '127.0.0.1', () => {
+      const { port } = server.address()
+      const teardown = () => new Promise(r => server.close(r))
+      Promise.resolve()
+        .then(() => run(`http://127.0.0.1:${port}`, hits))
+        .then(
+          async (value) => { await teardown(); resolve(value) },
+          async (err) => { await teardown(); reject(err) }
+        )
+    })
+  })
+}
+
+export function query(url) {
+  const i = url.indexOf('?')
+  return new URLSearchParams(i < 0 ? '' : url.slice(i + 1))
+}

package/test-v2/wire-unit/popFrames.test.js

@@ -0,0 +1,264 @@
+/**
+ * The framed pop answer (client-v2/http/popFrames.js), against hand-built
+ * answers and a server that plays them. What is at stake: a consumer that
+ * sets framedPop must be handed exactly the messages the JSON answer gives it,
+ * and one that does not set it must send the request every released broker
+ * knows.
+ */
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { zstdCompressSync } from 'node:zlib'
+
+import { Queen, CONFLATION_UNSUPPORTED } from '../../client-v2/index.js'
+import { decodePopFrames, isPopFrames } from '../../client-v2/http/popFrames.js'
+import { withWireServer, query } from './_wireServer.js'
+
+const u16 = (n) => { const b = Buffer.alloc(2); b.writeUInt16LE(n); return b }
+const u32 = (n) => { const b = Buffer.alloc(4); b.writeUInt32LE(n); return b }
+const u64 = (n) => { const b = Buffer.alloc(8); b.writeBigUInt64LE(BigInt(n)); return b }
+const text = (s) => Buffer.concat([u16(Buffer.byteLength(s)), Buffer.from(s)])
+
+/** One stored frame: u32 body_len | flags | id | [trace] | txn | [producer] | payload. */
+function storedFrame(id, txn, payload, { producer = null, trace = null } = {}) {
+  const idBytes = Buffer.alloc(16)
+  idBytes[0] = id
+  idBytes[6] = 0x70
+  idBytes[8] = 0x80
+  const body = Buffer.concat([
+    Buffer.of((trace ? 1 : 0) | (producer ? 2 : 0)),
+    idBytes,
+    trace ? Buffer.from(trace.replaceAll('-', ''), 'hex') : Buffer.alloc(0),
+    text(txn),
+    producer ? text(producer) : Buffer.alloc(0),
+    Buffer.from(payload)
+  ])
+  return Buffer.concat([u32(body.length), body])
+}
+
+/**
+ * A framed pop answer of one claim on partition 7 ("p7") of "orders", lease
+ * "lease-1": one record at base offset 10 holding `frames`, created at
+ * 2023-11-14T22:13:20Z, of which the claim covers start..=end.
+ */
+function framedAnswer(own, start, end, compress, frames) {
+  let raw = Buffer.concat(frames)
+  if (compress) raw = zstdCompressSync(raw)
+  return Buffer.concat([
+    Buffer.from('QPF1'), Buffer.of(1), u32(Buffer.byteLength(own)), Buffer.from(own),
+    text('orders'), text('lease-1'), text('__QUEUE_MODE__'),
+    u32(1), // claims
+    u64(7), text('p7'), u64(start), u64(end),
+    u32(3), // delivery attempt
+    u32(1), // records
+    u64(10), u64(1_700_000_000_000_000n), u32(frames.length), Buffer.of(compress ? 1 : 0), u32(raw.length), raw
+  ])
+}
+
+const twoFrames = () => [storedFrame(1, 't-10', '{"n":10}'), storedFrame(2, 't-11', '{"n":11,"s":"x"}', { producer: 'svc' })]
+
+// The JSON answer of the same two messages, as the broker renders it.
+const TWO_MESSAGES_JSON = '{"success":true,"queue":"orders","partition":"p7","partitionId":"7","leaseId":"lease-1","consumerGroup":"__QUEUE_MODE__","messages":[' +
+  '{"id":"01000000-0000-7000-8000-000000000000","transactionId":"t-10","traceId":null,"data":{"n":10},"producerSub":null,"createdAt":"2023-11-14T22:13:20.000Z","partitionId":"7","partition":"p7","leaseId":"lease-1","consumerGroup":"__QUEUE_MODE__","deliveryAttempt":3,"offset":10},' +
+  '{"id":"02000000-0000-7000-8000-000000000000","transactionId":"t-11","traceId":null,"data":{"n":11,"s":"x"},"producerSub":"svc","createdAt":"2023-11-14T22:13:20.000Z","partitionId":"7","partition":"p7","leaseId":"lease-1","consumerGroup":"__QUEUE_MODE__","deliveryAttempt":3,"offset":11}' +
+  '],"partitionsClaimed":1}'
+
+const json = (body) => ({ body })
+const framed = (body) => ({ contentType: 'application/x-queen-pop', body })
+const pops = (hits) => hits.filter(h => h.method === 'GET' && h.url.startsWith('/api/v1/pop'))
+
+async function withQueen(config, answer, run) {
+  await withWireServer(answer, async (url, hits) => {
+    const queen = new Queen({ url, handleSignals: false, retryAttempts: 1, ...config })
+    try {
+      await run(queen, hits)
+    } finally {
+      await queen.close()
+    }
+  })
+}
+
+describe('decodePopFrames', () => {
+  for (const compress of [false, true]) {
+    it(`unpacks ${compress ? 'a zstd' : 'a raw'} record into the JSON answer's object`, async () => {
+      const got = await decodePopFrames(framedAnswer('{"frames":true}', 10, 11, compress, twoFrames()))
+
+      assert.deepEqual(got, JSON.parse(TWO_MESSAGES_JSON))
+      assert.deepEqual(Object.keys(got.messages[0]), Object.keys(JSON.parse(TWO_MESSAGES_JSON).messages[0]))
+    })
+
+    it(`keeps only the messages inside the claim (${compress ? 'zstd' : 'raw'})`, async () => {
+      const got = await decodePopFrames(framedAnswer('{"frames":true}', 11, 11, compress, twoFrames()))
+
+      assert.equal(got.messages.length, 1)
+      assert.equal(got.messages[0].offset, 11)
+      assert.equal(got.messages[0].transactionId, 't-11')
+      assert.equal(got.messages[0].producerSub, 'svc')
+      assert.equal(got.messages[0].id, '02000000-0000-7000-8000-000000000000')
+    })
+  }
+
+  it('reads a trace id, an empty payload as null, and the answer without its own keys', async () => {
+    const trace = '0190aaaa-0000-4000-8000-000000000001'
+    const answer = framedAnswer('', 10, 11, false, [storedFrame(1, 't-10', '', { trace }), storedFrame(2, 't-11', '[1,"due"]')])
+    const got = await decodePopFrames(answer)
+
+    assert.equal(got.messages[0].traceId, trace)
+    assert.equal(got.messages[0].data, null)
+    assert.deepEqual(got.messages[1].data, [1, 'due'])
+
+    // Version 0 has no JSON object in front of the names.
+    const v0 = Buffer.concat([Buffer.from('QPF1'), Buffer.of(0), answer.subarray(9)])
+    assert.deepEqual(await decodePopFrames(v0), got)
+  })
+
+  it('adds the answer\'s own keys, without `frames`', async () => {
+    const own = '{"frames":true,"conflation":true,"autopilot":{"partitions":4,"batch":50}}'
+    const got = await decodePopFrames(framedAnswer(own, 10, 11, false, twoFrames()))
+
+    assert.equal('frames' in got, false)
+    assert.equal(got.conflation, true)
+    assert.deepEqual(got.autopilot, { partitions: 4, batch: 50 })
+  })
+
+  it('rejects damage', async () => {
+    const answer = framedAnswer('{"frames":true}', 10, 11, false, twoFrames())
+
+    assert.equal(isPopFrames(answer), true)
+    assert.equal(isPopFrames(Buffer.from(TWO_MESSAGES_JSON)), false)
+    await assert.rejects(decodePopFrames(answer.subarray(0, answer.length - 3)), /framed pop answer/)
+    await assert.rejects(decodePopFrames(Buffer.from(TWO_MESSAGES_JSON)), /not a framed pop answer/)
+
+    const future = Buffer.from(answer)
+    future[4] = 9
+    await assert.rejects(decodePopFrames(future), /unknown version 9/)
+
+    // A record that declares more frames than it holds.
+    const short = Buffer.from(answer)
+    const frames = twoFrames()
+    short.writeUInt32LE(3, short.length - frames[0].length - frames[1].length - 9)
+    await assert.rejects(decodePopFrames(short), /a record is shorter than its 3 frames/)
+
+    const badCodec = Buffer.from(answer)
+    badCodec[badCodec.length - frames[0].length - frames[1].length - 5] = 7
+    await assert.rejects(decodePopFrames(badCodec), /unknown codec 7/)
+  })
+})
+
+describe('pop — the default', () => {
+  it('sends no frames parameter', async () => {
+    await withQueen({}, () => json(TWO_MESSAGES_JSON), async (queen, hits) => {
+      const messages = await queen.queue('orders').batch(2).wait(false).pop()
+
+      assert.equal(messages.length, 2)
+      assert.equal(query(pops(hits)[0].url).has('frames'), false)
+    })
+  })
+})
+
+describe('pop — framedPop', () => {
+  for (const compress of [false, true]) {
+    it(`gives the messages of the JSON answer (${compress ? 'zstd' : 'raw'} record)`, async () => {
+      let want = null
+      await withQueen({}, () => json(TWO_MESSAGES_JSON), async (queen) => {
+        want = await queen.queue('orders').batch(2).wait(false).pop()
+      })
+
+      const answer = framedAnswer('{"frames":true}', 10, 11, compress, twoFrames())
+      await withQueen({ framedPop: true }, () => framed(answer), async (queen, hits) => {
+        const got = await queen.queue('orders').batch(2).wait(false).pop()
+
+        assert.equal(query(pops(hits)[0].url).get('frames'), 'true')
+        assert.equal(got.length, 2)
+        assert.deepEqual(got, want)
+      })
+    })
+  }
+
+  it('asks for frames on the partition and the namespace routes too', async () => {
+    await withQueen({ framedPop: true }, () => ({ status: 204 }), async (queen, hits) => {
+      await queen.queue('orders').partition('p7').wait(false).pop()
+      await queen.queue().namespace('ns').wait(false).pop()
+
+      const [byPartition, byNamespace] = pops(hits)
+      assert.ok(byPartition.url.startsWith('/api/v1/pop/queue/orders/partition/p7?'))
+      assert.equal(query(byPartition.url).get('frames'), 'true')
+      assert.ok(byNamespace.url.startsWith('/api/v1/pop?'))
+      assert.equal(query(byNamespace.url).get('frames'), 'true')
+    })
+  })
+
+  it('knows a framed answer by its first bytes, whatever its content type', async () => {
+    const answer = framedAnswer('{"frames":true}', 10, 11, false, twoFrames())
+    for (const contentType of ['application/octet-stream', 'application/json']) {
+      await withQueen({ framedPop: true }, () => ({ contentType, body: answer }), async (queen) => {
+        const got = await queen.queue('orders').batch(2).wait(false).pop()
+
+        assert.deepEqual(got.map(m => m.transactionId), ['t-10', 't-11'], contentType)
+      })
+    }
+  })
+
+  it('reads a JSON answer as always: a broker that does not know the parameter', async () => {
+    await withQueen({ framedPop: true }, () => json(TWO_MESSAGES_JSON), async (queen) => {
+      const got = await queen.queue('orders').batch(2).wait(false).pop()
+
+      assert.equal(got.length, 2)
+      assert.equal(got[1].transactionId, 't-11')
+      assert.equal(got[1].data.s, 'x')
+    })
+  })
+
+  it('reads an empty pop', async () => {
+    await withQueen({ framedPop: true }, () => ({ status: 204 }), async (queen) => {
+      assert.deepEqual(await queen.queue('orders').wait(false).pop(), [])
+    })
+  })
+
+  it('carries the autopilot\'s choice in the answer\'s own keys', async () => {
+    const own = '{"frames":true,"autopilot":{"partitions":4,"batch":50}}'
+    await withQueen({ framedPop: true }, () => framed(framedAnswer(own, 10, 11, false, twoFrames())), async (queen) => {
+      const { messages, autopilot } = await queen.queue('orders').wait(false).popResult()
+
+      assert.equal(messages.length, 2)
+      assert.deepEqual(autopilot, { partitions: 4, batch: 50, waitMillis: 0 })
+    })
+  })
+
+  it('carries the conflation echo, and still raises when the broker sent none', async () => {
+    const echoed = framedAnswer('{"frames":true,"conflation":true}', 10, 11, false, twoFrames())
+    await withQueen({ framedPop: true }, () => framed(echoed), async (queen) => {
+      const got = await queen.queue('orders').group('g').conflation(true).wait(false).pop()
+      assert.equal(got.length, 2)
+    })
+
+    const silent = framedAnswer('{"frames":true}', 10, 11, false, twoFrames())
+    await withQueen({ framedPop: true }, () => framed(silent), async (queen) => {
+      await assert.rejects(
+        queen.queue('orders').group('g2').conflation(true).wait(false).pop(),
+        { code: CONFLATION_UNSUPPORTED }
+      )
+    })
+  })
+})
+
+describe('consume — framedPop', () => {
+  // The consumer loop on framed answers: the handler gets every message of
+  // the claim, in order, and the loop stops at its limit.
+  it('hands the handler the messages of the claim, in order', async () => {
+    const answer = framedAnswer('{"frames":true}', 10, 11, true, twoFrames())
+    await withQueen({ framedPop: true }, () => framed(answer), async (queen, hits) => {
+      const seen = []
+      await queen.queue('orders').batch(2).wait(false).autoAck(false).each().limit(2)
+        .consume(async (message) => {
+          seen.push(`${message.transactionId}@${message.partitionId}/${message.leaseId}`)
+        })
+
+      assert.deepEqual(seen, ['t-10@7/lease-1', 't-11@7/lease-1'])
+      assert.ok(pops(hits).length >= 1)
+      for (const hit of pops(hits)) {
+        assert.equal(query(hit.url).get('frames'), 'true')
+      }
+    })
+  })
+})

package/test-v2/wire-unit/pushBinary.test.js

@@ -0,0 +1,235 @@
+/**
+ * The binary push body (client-v2/http/pushBinary.js), asserted against a
+ * server that records the bytes. What is at stake is the default: a client
+ * that does not set binaryPush must send the JSON request every released
+ * broker knows.
+ */
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { zstdDecompressSync } from 'node:zlib'
+
+import { Queen } from '../../client-v2/index.js'
+import { binaryPushBody, encodePushBinary, encodePushFrames } from '../../client-v2/http/pushBinary.js'
+import { withWireServer } from './_wireServer.js'
+
+const TXN = '0190aaaa-0000-7000-8000-000000000001'
+// The client sends a trace id only when it is a version-4 UUID.
+const TRACE = '0190aaaa-0000-4000-8000-000000000001'
+
+const queued = (n = 1) => ({ body: JSON.stringify(Array.from({ length: n }, () => ({ status: 'queued' }))) })
+const pushes = (hits) => hits.filter(h => h.method === 'POST' && h.url === '/api/v1/push')
+
+async function withQueen(config, answer, run) {
+  await withWireServer(answer, async (url, hits) => {
+    const queen = new Queen({ url, handleSignals: false, retryAttempts: 1, ...config })
+    try {
+      await run(queen, hits)
+    } finally {
+      await queen.close()
+    }
+  })
+}
+
+/** Reads the fields of a binary push body in order. */
+function reader(bytes) {
+  let at = 0
+  const r = {
+    u16() { at += 2; return bytes.readUInt16LE(at - 2) },
+    u32() { at += 4; return bytes.readUInt32LE(at - 4) },
+    take(n) { at += n; return bytes.subarray(at - n, at) },
+    text16() { return r.take(r.u16()).toString() },
+    text32() { return r.take(r.u32()).toString() },
+    left() { return bytes.length - at }
+  }
+  return r
+}
+
+/** The groups of a plain binary push body (after magic and flags). */
+function plainGroups(bytes, withTxn) {
+  const r = reader(bytes)
+  const groups = []
+  for (let g = r.u32(); g > 0; g--) {
+    const group = { queue: r.text16(), partition: r.text16(), items: [] }
+    for (let n = r.u32(); n > 0; n--) {
+      const txn = withTxn ? r.text16() : null
+      group.items.push({ txn, payload: r.text32() })
+    }
+    groups.push(group)
+  }
+  assert.equal(r.left(), 0, 'bytes left after the last payload')
+  return groups
+}
+
+/** The groups of a framed binary push body, each with its frames unpacked. */
+function framedGroups(bytes) {
+  const r = reader(bytes)
+  const groups = []
+  for (let g = r.u32(); g > 0; g--) {
+    const group = { queue: r.text16(), partition: r.text16(), count: r.u32(), frames: [] }
+    const f = reader(zstdDecompressSync(r.take(r.u32())))
+    while (f.left() > 0) {
+      const body = reader(f.take(f.u32()))
+      const flags = body.take(1)[0]
+      const id = Buffer.from(body.take(16))
+      group.frames.push({ flags, id, txn: body.text16(), payload: body.take(body.left()).toString() })
+    }
+    groups.push(group)
+  }
+  assert.equal(r.left(), 0, 'bytes left after the last group')
+  return groups
+}
+
+describe('push — the default', () => {
+  it('is the JSON request', async () => {
+    await withQueen({}, () => queued(), async (queen, hits) => {
+      const res = await queen.queue('orders').partition('p1').push({ data: { a: 1 } })
+
+      assert.equal(res[0].status, 'queued')
+      const hit = pushes(hits)[0]
+      assert.equal(hit.headers['content-type'], 'application/json')
+      const body = JSON.parse(hit.raw.toString())
+      assert.deepEqual(Object.keys(body), ['items'])
+      assert.deepEqual(body.items[0].payload, { a: 1 })
+    })
+  })
+})
+
+describe('push — binaryPush', () => {
+  it('sends the binary body, each item with its transaction id', async () => {
+    await withQueen({ binaryPush: true }, () => queued(), async (queen, hits) => {
+      const res = await queen.queue('orders').partition('p1').push({ data: { a: 1 }, transactionId: TXN })
+
+      assert.equal(res[0].status, 'queued')
+      const hit = pushes(hits)[0]
+      assert.equal(hit.headers['content-type'], 'application/x-queen-push')
+      assert.equal(hit.raw.subarray(0, 5).toString('latin1'), 'QPB1\x04')
+      assert.deepEqual(plainGroups(hit.raw.subarray(5), true), [
+        { queue: 'orders', partition: 'p1', items: [{ txn: TXN, payload: '{"a":1}' }] }
+      ])
+    })
+  })
+
+  it('keeps the order of the items: consecutive items of one partition share a group', async () => {
+    const item = (partition, payload) => ({ queue: 'q', partition, payload, transactionId: `t${payload}` })
+    const bytes = Buffer.from(await binaryPushBody([item('a', 1), item('a', 2), item('b', 3), item('a', 4)], false))
+
+    assert.equal(bytes.subarray(0, 5).toString('latin1'), 'QPB1\x04')
+    assert.deepEqual(
+      plainGroups(bytes.subarray(5), true).map(g => [g.partition, g.items.map(i => i.txn + '=' + i.payload)]),
+      [['a', ['t1=1', 't2=2']], ['b', ['t3=3']], ['a', ['t4=4']]]
+    )
+  })
+
+  it('carries payloads that are not objects, and non-ASCII text', async () => {
+    await withQueen({ binaryPush: true }, () => queued(5), async (queen, hits) => {
+      await queen.queue('ordini').partition('città').push([
+        { data: 'testo "con" virgolette\n' }, { data: [1, 2] }, { data: 7 }, { data: null }, { data: { è: '€' } }
+      ])
+
+      const [group] = plainGroups(pushes(hits)[0].raw.subarray(5), true)
+      assert.equal(group.partition, 'città')
+      assert.deepEqual(group.items.map(i => JSON.parse(i.payload)), ['testo "con" virgolette\n', [1, 2], 7, null, { è: '€' }])
+    })
+  })
+
+  it('leaves an item with a trace id to the JSON request', async () => {
+    await withQueen({ binaryPush: true }, () => queued(), async (queen, hits) => {
+      await queen.queue('orders').push({ data: { a: 1 }, traceId: TRACE })
+
+      const hit = pushes(hits)[0]
+      assert.equal(hit.headers['content-type'], 'application/json')
+      assert.equal(JSON.parse(hit.raw.toString()).items[0].traceId, TRACE)
+    })
+  })
+
+  it('sends a buffered push as the binary body too', async () => {
+    const { flags, body } = await bodyOf(async (queen) => {
+      await queen.queue('orders').partition('p1').buffer({ messageCount: 10, timeMillis: 60000 }).push({ data: { a: 1 } })
+      await queen.flushAllBuffers()
+    })
+    assert.equal(flags, 4)
+    assert.equal(body[0].items[0].payload, '{"a":1}')
+  })
+
+  /** Runs `use` on a binaryPush client and returns the one push it sent, decoded. */
+  async function bodyOf(use) {
+    let sent = null
+    await withQueen({ binaryPush: true }, () => queued(2), async (queen, hits) => {
+      await use(queen)
+      assert.equal(pushes(hits).length, 1)
+      sent = pushes(hits)[0]
+    })
+    assert.equal(sent.raw.subarray(0, 4).toString('latin1'), 'QPB1')
+    return { flags: sent.raw[4], body: plainGroups(sent.raw.subarray(5), true) }
+  }
+})
+
+describe('push — binaryPush with pushCompression', () => {
+  it('sends each partition as one zstd frame of stored frames, ids minted here', async () => {
+    await withQueen({ binaryPush: true, pushCompression: true }, () => queued(3), async (queen, hits) => {
+      await queen.queue('orders').partition('p1').push([{ data: { a: 1 }, transactionId: TXN }, { data: { a: 2 } }, { data: 'x' }])
+
+      const hit = pushes(hits)[0]
+      assert.equal(hit.headers['content-type'], 'application/x-queen-push')
+      assert.equal(hit.raw.subarray(0, 5).toString('latin1'), 'QPB1\x08')
+      const [group, ...rest] = framedGroups(hit.raw.subarray(5))
+      assert.equal(rest.length, 0)
+      assert.equal(group.queue, 'orders')
+      assert.equal(group.partition, 'p1')
+      assert.equal(group.count, 3)
+      assert.deepEqual(group.frames.map(f => f.payload), ['{"a":1}', '{"a":2}', '"x"'])
+      assert.equal(group.frames[0].txn, TXN)
+      for (const frame of group.frames) {
+        assert.equal(frame.flags, 0)
+        assert.equal(frame.id[6] >> 4, 7, 'the message id is a version-7 UUID')
+        assert.equal(frame.id[8] & 0xc0, 0x80)
+        assert.equal(frame.txn.length, 36)
+      }
+      assert.equal(new Set(group.frames.map(f => f.id.toString('hex'))).size, 3)
+    })
+  })
+
+  it('gives a message without a transaction id the text of its id', async () => {
+    const bytes = Buffer.from(await encodePushFrames([{ queue: 'q', partition: 'p', payloads: ['1'] }]))
+    const [{ frames: [frame] }] = framedGroups(bytes.subarray(5))
+    const hex = frame.id.toString('hex')
+    assert.equal(frame.txn, `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`)
+  })
+
+  it('writes the content size in each zstd frame: the broker asks for it', async () => {
+    const bytes = Buffer.from(await encodePushFrames([{ queue: 'q', partition: 'p', payloads: ['{"a":1}'] }]))
+    const r = reader(bytes.subarray(5))
+    r.u32(); r.text16(); r.text16(); r.u32()
+    const frame = r.take(r.u32())
+    assert.equal(frame.readUInt32LE(0), 0xfd2fb528, 'a zstd frame')
+    // Frame header descriptor: the single-segment bit, or a content-size
+    // field of 2 bytes or more, says the frame carries its content size.
+    assert.ok((frame[4] & 0x20) !== 0 || (frame[4] >> 6) !== 0)
+  })
+})
+
+describe('encodePushBinary — flags', () => {
+  const batches = [{ queue: 'q', partition: 'p', payloads: ['1'] }]
+
+  it('asks for the counts answer with flag 2', async () => {
+    const bytes = Buffer.from(await encodePushBinary(batches, { counts: true }))
+    assert.equal(bytes.subarray(0, 5).toString('latin1'), 'QPB1\x02')
+    assert.equal(Buffer.from(await encodePushFrames(batches, { counts: true }))[4], 8 | 2)
+  })
+
+  it('compresses the whole body with flag 1, its raw length in front', async () => {
+    const plain = Buffer.from(await encodePushBinary(batches))
+    const packed = Buffer.from(await encodePushBinary(batches, { compress: true }))
+    assert.equal(packed.subarray(0, 5).toString('latin1'), 'QPB1\x01')
+    assert.equal(packed.readUInt32LE(5), plain.length - 5)
+    assert.deepEqual(zstdDecompressSync(packed.subarray(9)), plain.subarray(5))
+  })
+
+  it('refuses transaction ids that do not match the payloads', async () => {
+    await assert.rejects(
+      encodePushBinary([{ queue: 'q', partition: 'p', payloads: ['1', '2'], transactionIds: ['t'] }]),
+      { message: '1 transaction ids for 2 payloads' }
+    )
+  })
+})