queen-mq 2.1.0 → 2.6.0

package/client-v2/reads/offsetReads.js

@@ -0,0 +1,252 @@
+/**
+ * Reads by offset and partition discovery, as two queue methods:
+ *
+ *   const read = await queen.queue('orders').fetch(['customer-42', 'customer-7'])
+ *   read.get('customer-42').records   // [{ offset, transactionId, payload, ts }, ...]
+ *
+ *   const { partitions, safeTime } = await queen.queue('orders').changedSince(since)
+ *
+ * `fetch` is `POST /api/v1/fetch` and `changedSince` is
+ * `POST /api/v1/partitions/changed`. Neither takes a lease, moves a cursor or
+ * needs an ack: two readers asking the same question get the same answer, and
+ * no consumer group notices them. Each is served from the answering node's own
+ * applied state.
+ *
+ * Both do the loops a caller of the raw routes has to write:
+ *
+ * - fetch asks for at most 1000 partitions per call, and asks again from one
+ *   past the last record until each partition reaches the high watermark its
+ *   first answer reported. A partition whose start was deleted by retention is
+ *   read from its oldest kept record (an offset below `logStartOffset` answers
+ *   OFFSET_OUT_OF_RANGE and no records, so a loop that does not restart there
+ *   silently reads nothing).
+ * - changedSince follows the page cursor (`next`) until the broker says the
+ *   pass is over, and answers the smallest `safeTime` of its pages.
+ */
+
+// The broker refuses more than 1024 entries in one call; 1000 leaves room.
+const ENTRIES_PER_CALL = 1000
+
+// Bytes of payload one call may carry: each entry's budget and the ceiling of
+// the whole call. 8 MiB is the most the broker reads for one entry.
+const DEFAULT_MAX_BYTES = 8 * 1024 * 1024
+
+// Partitions per discovery page: the broker's default and its ceiling.
+const CHANGED_PAGE = 1000
+
+const OFFSET_OUT_OF_RANGE = 'OFFSET_OUT_OF_RANGE'
+const UNKNOWN_TOPIC_OR_PARTITION = 'UNKNOWN_TOPIC_OR_PARTITION'
+
+// Every call of one read goes to the same node: a follower answers from what it
+// has applied, so two nodes can disagree about where a partition ends.
+const readAffinity = (queue) => `${queue}:*:__READ__`
+
+function partitionNames(partitions) {
+  const list = typeof partitions === 'string' ? [partitions] : partitions
+  if (!Array.isArray(list)) {
+    throw new Error('fetch: partitions must be a partition name or an array of names')
+  }
+  const names = []
+  const seen = new Set()
+  for (const p of list) {
+    const name = typeof p === 'number' && Number.isInteger(p) ? String(p) : p
+    if (typeof name !== 'string' || name.length === 0) {
+      throw new Error(`fetch: a partition name must be a non-empty string, got ${JSON.stringify(p)}`)
+    }
+    if (!seen.has(name)) {
+      seen.add(name)
+      names.push(name)
+    }
+  }
+  return names
+}
+
+function startOffsets(offsets) {
+  if (offsets == null) return new Map()
+  const entries = offsets instanceof Map ? [...offsets.entries()] : Object.entries(offsets)
+  const out = new Map()
+  for (const [name, offset] of entries) {
+    if (!Number.isSafeInteger(offset) || offset < 0) {
+      throw new Error(`fetch: offsets['${name}'] must be a non-negative integer, got ${JSON.stringify(offset)}`)
+    }
+    out.set(String(name), offset)
+  }
+  return out
+}
+
+function positiveInteger(value, what) {
+  if (!Number.isSafeInteger(value) || value < 1) {
+    throw new Error(`${what} must be a positive integer, got ${JSON.stringify(value)}`)
+  }
+  return value
+}
+
+/**
+ * Every record of `partitions`, read by offset: from each partition's offset in
+ * `options.offsets` (default 0, the oldest record still kept) up to where the
+ * partition ended when it was first read.
+ *
+ * Answers a Map, in the order the partitions were named, of
+ * `{ partition, records, highWatermark, logStartOffset, nextOffset }`:
+ * - `records`: `{ offset, transactionId, payload, ts }` in offset order. `ts` is
+ *   the time the push that wrote the record was stored.
+ * - `highWatermark`: the next offset the partition will assign;
+ *   `logStartOffset`: the oldest one retention kept.
+ * - `nextOffset`: where to start next time to read only what is new.
+ *
+ * A partition nobody has written, or of a queue that does not exist, answers no
+ * records. So does an offset past the end of the partition (`nextOffset` above
+ * `highWatermark` says so: the partition was deleted and written again).
+ */
+export async function fetchPartitions(httpClient, queue, partitions, options = {}) {
+  if (!queue) throw new Error('Queue name is required for fetch')
+  const names = partitionNames(partitions)
+  const offsets = startOffsets(options.offsets)
+  const maxBytes = positiveInteger(options.maxBytes ?? DEFAULT_MAX_BYTES, 'fetch: maxBytes')
+
+  const state = new Map()
+  for (const name of names) {
+    state.set(name, {
+      partition: name,
+      records: [],
+      highWatermark: 0,
+      logStartOffset: 0,
+      nextOffset: offsets.get(name) ?? 0,
+      // The high watermark of the first answer: the read ends there.
+      end: null
+    })
+  }
+
+  let pending = names
+  while (pending.length > 0) {
+    const again = []
+    for (let i = 0; i < pending.length; i += ENTRIES_PER_CALL) {
+      const chunk = pending.slice(i, i + ENTRIES_PER_CALL)
+      const body = {
+        entries: chunk.map(name => ({
+          queue,
+          partition: name,
+          offset: state.get(name).nextOffset,
+          maxBytes
+        })),
+        maxBytes
+      }
+      const answer = await httpClient.post('/api/v1/fetch', body, null, readAffinity(queue))
+      const entries = answer?.entries
+      if (!Array.isArray(entries) || entries.length !== chunk.length) {
+        throw new Error(`fetch ${queue}: the broker answered ${Array.isArray(entries) ? entries.length : 'no'} entries for ${chunk.length} asked`)
+      }
+
+      let delivered = 0
+      // Entries that came back empty while short of their end: the call's byte
+      // ceiling was spent by the entries before them.
+      const starved = []
+      chunk.forEach((name, j) => {
+        const entry = entries[j]
+        const s = state.get(name)
+        s.highWatermark = entry.highWatermark ?? s.highWatermark
+        s.logStartOffset = entry.logStartOffset ?? s.logStartOffset
+        if (entry.error) {
+          if (entry.error === OFFSET_OUT_OF_RANGE && s.nextOffset < s.logStartOffset) {
+            // Retention deleted what was asked for: go on from the oldest kept.
+            s.nextOffset = s.logStartOffset
+            if (s.end === null) s.end = s.highWatermark
+            if (s.nextOffset < s.end) again.push(name)
+            return
+          }
+          if (entry.error === OFFSET_OUT_OF_RANGE || entry.error === UNKNOWN_TOPIC_OR_PARTITION) {
+            return
+          }
+          throw new Error(`fetch ${queue}/${name}: the broker answered ${entry.error}`)
+        }
+        if (s.end === null) s.end = s.highWatermark
+        const records = Array.isArray(entry.records) ? entry.records : []
+        if (records.length > 0) {
+          delivered += records.length
+          for (const r of records) {
+            s.records.push({ offset: r.offset, transactionId: r.transactionId, payload: r.payload, ts: r.ts })
+          }
+          s.nextOffset = records[records.length - 1].offset + 1
+          if (s.nextOffset < s.end) again.push(name)
+        } else if (s.nextOffset < s.end) {
+          starved.push(name)
+        }
+      })
+      // An empty entry is asked again only when its call read something: a
+      // call that read nothing at all would answer the same next time.
+      if (delivered > 0) again.push(...starved)
+    }
+    pending = again
+  }
+
+  const out = new Map()
+  for (const name of names) {
+    const { partition, records, highWatermark, logStartOffset, nextOffset } = state.get(name)
+    out.set(name, { partition, records, highWatermark, logStartOffset, nextOffset })
+  }
+  return out
+}
+
+function sinceIso(since) {
+  if (since == null) return null
+  if (since instanceof Date) {
+    if (Number.isNaN(since.getTime())) throw new Error('changedSince: since is an invalid Date')
+    return since.toISOString()
+  }
+  if (typeof since === 'number') {
+    const d = new Date(since)
+    if (Number.isNaN(d.getTime())) throw new Error(`changedSince: since ${since} is not a time`)
+    return d.toISOString()
+  }
+  if (typeof since === 'string' && since.length > 0) return since
+  throw new Error(`changedSince: since must be a Date, epoch milliseconds or an ISO-8601 string, got ${JSON.stringify(since)}`)
+}
+
+/**
+ * The partitions of the queue written at or after `since`, in the order they
+ * were created: `{ partitions, safeTime }`. Each partition is
+ * `{ name, id, lastOffset, logStart, lastWriteAt }`; `lastOffset` is the offset
+ * of its last record (-1 when none).
+ *
+ * `since` is a Date, epoch milliseconds or an ISO-8601 string; null or absent
+ * lists every partition of the queue. A queue that does not exist has none.
+ *
+ * `safeTime` is the broker's time up to which every write is counted in this
+ * answer: pass it as the next `since` to see only what changed after this
+ * call. It comes from the broker's clock, never compare it with the local one.
+ */
+export async function partitionsChangedSince(httpClient, queue, since) {
+  if (!queue) throw new Error('Queue name is required for changedSince')
+  const from = sinceIso(since)
+  const partitions = []
+  let safeTime = null
+  let after = null
+  for (;;) {
+    const entry = { queue, limit: CHANGED_PAGE }
+    if (from !== null) entry.since = from
+    if (after !== null) entry.after = after
+    const answer = await httpClient.post('/api/v1/partitions/changed', { entries: [entry] }, null, readAffinity(queue))
+    // Pages served by different nodes: the smallest is the one that holds for
+    // all of them. The broker spells every one the same way, so the strings
+    // compare in time order.
+    if (typeof answer?.safeTime === 'string' && (safeTime === null || answer.safeTime < safeTime)) {
+      safeTime = answer.safeTime
+    }
+    const result = answer?.entries?.[0]
+    if (!result) throw new Error(`changedSince ${queue}: the broker answered no entry`)
+    if (result.error === UNKNOWN_TOPIC_OR_PARTITION) break
+    if (result.error) throw new Error(`changedSince ${queue}: the broker answered ${result.error}`)
+    for (const p of result.partitions || []) {
+      partitions.push({
+        name: p.name,
+        id: p.id ?? null,
+        lastOffset: p.lastOffset,
+        logStart: p.logStart,
+        lastWriteAt: p.lastWriteAt
+      })
+    }
+    if (!result.next) break
+    after = result.next
+  }
+  return { partitions, safeTime }
+}

package/client-v2/utils/defaults.js

@@ -36,7 +36,23 @@ export const CLIENT_DEFAULTS = {
   // retries unboundedly (paced by backoff) unless maxAttempts is set here, in
   // which case it applies to both. baseMs (500) / capMs (30000) size the
   // exponential backoff used when the server doesn't send Retry-After.
-  retry429: undefined
+  retry429: undefined,
+  // The three below need a broker that has the binary push body and the framed
+  // pop answer. Off, the client sends and reads the JSON every broker knows.
+  //
+  // binaryPush: a push goes out as the binary push body (http/pushBinary.js)
+  // instead of JSON. Items with a trace id stay JSON.
+  binaryPush: false,
+  // pushCompression (with binaryPush): the client frames the messages as the
+  // broker stores them, mints their ids and zstd-compresses each partition's
+  // batch once; the broker keeps those bytes as they are. A transaction that
+  // pushes sends its pushes the same way, and the rest of the bundle as JSON
+  // in front of them.
+  pushCompression: false,
+  // framedPop: pops ask for the messages as the stored records (frames=true)
+  // and unpack them here (http/popFrames.js). A broker that does not know the
+  // parameter answers JSON, which is read as always.
+  framedPop: false
 }
 
 export const QUEUE_DEFAULTS = {

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "queen-mq",
-  "version": "2.1.0",
+  "version": "2.6.0",
   "type": "module",
   "description": "Partitioned message queue on a replicated broker log — broker client + fluent streaming SDK (windows, joins, gates) in one package",
   "main": "client-v2/index.js",
   "scripts": {
     "test": "npm run test:unit && node test-v2/run.js human",
-    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/consumer-unit/nackScope.test.js test-v2/consumer-unit/stopOnAbort.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/kv-unit/locksWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js test-v2/pop-unit/popDefaults.test.js test-v2/admin-unit/removedRoutes.test.js test-v2/consumer-unit/supervision.test.js",
+    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/consumer-unit/nackScope.test.js test-v2/consumer-unit/stopOnAbort.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/kv-unit/locksWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js test-v2/pop-unit/popDefaults.test.js test-v2/admin-unit/removedRoutes.test.js test-v2/consumer-unit/supervision.test.js test-v2/wire-unit/pushBinary.test.js test-v2/wire-unit/popFrames.test.js test-v2/wire-unit/transactionFrames.test.js test-v2/reads-unit/offsetReads.test.js",
     "test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
     "test:integration": "node test-v2/run.js human",
     "test:streams": "node test-v2/run.js stream",

package/test-v2/reads-unit/offsetReads.test.js

@@ -0,0 +1,380 @@
+/**
+ * queue.fetch() and queue.changedSince() against a fake broker that answers
+ * POST /api/v1/fetch and POST /api/v1/partitions/changed the way
+ * server/src/rsm/facade/real/phase2/reads.rs does: entries answered in order,
+ * each entry's byte budget and the whole call's ceiling (the first record of an
+ * entry read whatever it weighs), a record cap per entry, OFFSET_OUT_OF_RANGE
+ * below logStartOffset and above the high watermark, UNKNOWN_TOPIC_OR_PARTITION
+ * for a queue that does not exist, discovery pages with a `next` cursor exactly
+ * when a page is full, BAD_CURSOR for a cursor it did not issue.
+ *
+ * A real node:http server records every request, so the bodies the client sends
+ * are asserted on the socket.
+ */
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { createServer } from 'node:http'
+
+import { Queen } from '../../client-v2/index.js'
+
+const MiB = 1024 * 1024
+
+class FakeBroker {
+  constructor({ recordsPerEntry = 10_000 } = {}) {
+    this.queues = new Map() // queue -> Map(partition -> { records, logStart, created })
+    this.recordsPerEntry = recordsPerEntry
+    this.hits = []
+    this.safeTimes = []      // answered in turn by partitions/changed, then the last one
+    this.afterFetch = null   // (broker) => void, run after every fetch answer
+    this.created = 0
+  }
+
+  queue(name) {
+    if (!this.queues.has(name)) this.queues.set(name, new Map())
+    return this.queues.get(name)
+  }
+
+  write(queue, partition, payloads, { lastWriteAt = '2026-10-20T10:00:00.000000Z' } = {}) {
+    const q = this.queue(queue)
+    if (!q.has(partition)) q.set(partition, { records: [], logStart: 0, created: this.created++, lastWriteAt })
+    const p = q.get(partition)
+    for (const payload of payloads) {
+      const offset = p.records.length === 0 ? p.logStart : p.records[p.records.length - 1].offset + 1
+      p.records.push({ offset, transactionId: `${partition}-${offset}`, payload, ts: '2026-10-20T10:00:00.000000Z' })
+    }
+    p.lastWriteAt = lastWriteAt
+  }
+
+  /** Retention: everything below `offset` is gone. */
+  trim(queue, partition, offset) {
+    const p = this.queue(queue).get(partition)
+    p.records = p.records.filter(r => r.offset >= offset)
+    p.logStart = offset
+  }
+
+  fetch(body) {
+    if (body.entries.length > 1024) return [400, { error: 'too many entries' }]
+    const ceiling = body.maxBytes ?? null
+    let used = 0
+    const entries = body.entries.map(e => {
+      const out = { queue: e.queue, partition: e.partition ?? 'Default', records: [], highWatermark: 0, logStartOffset: 0 }
+      const q = this.queues.get(e.queue)
+      if (!q) return { ...out, error: 'UNKNOWN_TOPIC_OR_PARTITION' }
+      const p = q.get(out.partition)
+      if (!p) return out
+      const last = p.records.length ? p.records[p.records.length - 1].offset : p.logStart - 1
+      out.highWatermark = last + 1
+      out.logStartOffset = p.logStart
+      if (e.offset < p.logStart || e.offset > out.highWatermark) return { ...out, error: 'OFFSET_OUT_OF_RANGE' }
+      if (ceiling !== null && used >= ceiling) return out
+      const budget = Math.min(e.maxBytes ?? MiB, ceiling === null ? Infinity : ceiling - used)
+      let spent = 0
+      for (const r of p.records) {
+        if (r.offset < e.offset) continue
+        if (out.records.length >= this.recordsPerEntry) break
+        const weight = Math.max(JSON.stringify(r.payload).length, 1)
+        if (out.records.length > 0 && spent + weight > budget) break
+        spent += weight
+        out.records.push(r)
+      }
+      used += spent
+      return out
+    })
+    return [200, { entries }]
+  }
+
+  changed(body) {
+    if (body.entries.length > 1024) return [400, { error: 'too many entries' }]
+    const safeTime = this.safeTimes.length > 1 ? this.safeTimes.shift() : (this.safeTimes[0] ?? '2026-10-20T12:00:00.000000Z')
+    const entries = body.entries.map(e => {
+      const q = this.queues.get(e.queue)
+      if (!q) return { queue: e.queue, error: 'UNKNOWN_TOPIC_OR_PARTITION' }
+      let from = 0
+      if (e.after) {
+        const m = /^p\|(\d+)$/.exec(e.after)
+        if (!m) return { queue: e.queue, error: 'BAD_CURSOR' }
+        from = Number(m[1]) + 1
+      }
+      const limit = Math.min(Math.max(e.limit ?? 1000, 1), 1000)
+      const all = [...q.entries()].sort((a, b) => a[1].created - b[1].created)
+      const partitions = []
+      let last = null
+      for (const [name, p] of all) {
+        if (p.created < from) continue
+        if (e.since && p.lastWriteAt < e.since) continue
+        const lastOffset = p.records.length ? p.records[p.records.length - 1].offset : p.logStart - 1
+        partitions.push({ name, id: `id-${name}`, lastOffset, logStart: p.logStart, lastWriteAt: p.lastWriteAt })
+        last = p.created
+        if (partitions.length >= limit) break
+      }
+      return { queue: e.queue, partitions, next: partitions.length >= limit ? `p|${last}` : null }
+    })
+    return [200, { safeTime, safeTimeDegraded: false, entries }]
+  }
+
+  async run(test) {
+    const server = createServer((req, res) => {
+      let raw = ''
+      req.on('data', c => { raw += c })
+      req.on('end', () => {
+        const body = raw ? JSON.parse(raw) : null
+        this.hits.push({ method: req.method, url: req.url, body })
+        let answer
+        if (req.method === 'POST' && req.url === '/api/v1/fetch') {
+          answer = this.fetch(body)
+          this.afterFetch?.(this)
+        } else if (req.method === 'POST' && req.url === '/api/v1/partitions/changed') {
+          answer = this.changed(body)
+        } else {
+          answer = [404, { error: 'not found' }]
+        }
+        res.writeHead(answer[0], { 'Content-Type': 'application/json' })
+        res.end(JSON.stringify(answer[1]))
+      })
+    })
+    await new Promise(r => server.listen(0, '127.0.0.1', r))
+    const queen = new Queen({ url: `http://127.0.0.1:${server.address().port}`, handleSignals: false })
+    try {
+      await test(queen)
+    } finally {
+      await queen.close()
+      await new Promise(r => server.close(r))
+    }
+  }
+
+  fetches() {
+    return this.hits.filter(h => h.url === '/api/v1/fetch')
+  }
+}
+
+const payloads = (n, from = 0) => Array.from({ length: n }, (_, i) => ({ n: from + i }))
+const offsetsOf = (part) => part.records.map(r => r.offset)
+
+describe('queue.fetch()', () => {
+  it('reads every record of each partition, from offset 0, in one call when they fit', async () => {
+    const broker = new FakeBroker()
+    broker.write('tf.events', 'trace-1', payloads(3))
+    broker.write('tf.events', 'trace-2', [{ a: 1 }])
+    await broker.run(async (queen) => {
+      const read = await queen.queue('tf.events').fetch(['trace-1', 'trace-2'])
+      assert.deepEqual([...read.keys()], ['trace-1', 'trace-2'])
+      const t1 = read.get('trace-1')
+      assert.deepEqual(t1.records.map(r => r.payload), payloads(3))
+      assert.deepEqual(t1.records[0], { offset: 0, transactionId: 'trace-1-0', payload: { n: 0 }, ts: '2026-10-20T10:00:00.000000Z' })
+      assert.equal(t1.partition, 'trace-1')
+      assert.equal(t1.highWatermark, 3)
+      assert.equal(t1.logStartOffset, 0)
+      assert.equal(t1.nextOffset, 3)
+      assert.deepEqual(read.get('trace-2').records.map(r => r.payload), [{ a: 1 }])
+
+      const fetches = broker.fetches()
+      assert.equal(fetches.length, 1)
+      assert.deepEqual(fetches[0].body, {
+        entries: [
+          { queue: 'tf.events', partition: 'trace-1', offset: 0, maxBytes: 8 * MiB },
+          { queue: 'tf.events', partition: 'trace-2', offset: 0, maxBytes: 8 * MiB }
+        ],
+        maxBytes: 8 * MiB
+      })
+    })
+  })
+
+  it('asks again from one past the last record until the high watermark of the first answer', async () => {
+    const broker = new FakeBroker({ recordsPerEntry: 3 })
+    broker.write('q', 'p', payloads(10))
+    await broker.run(async (queen) => {
+      const read = await queen.queue('q').fetch('p')
+      assert.deepEqual(offsetsOf(read.get('p')), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+      assert.deepEqual(broker.fetches().map(h => h.body.entries[0].offset), [0, 3, 6, 9])
+      assert.equal(read.get('p').nextOffset, 10)
+    })
+  })
+
+  it('does not chase a partition written during the read', async () => {
+    const broker = new FakeBroker({ recordsPerEntry: 2 })
+    broker.write('q', 'p', payloads(4))
+    broker.afterFetch = (b) => b.write('q', 'p', [{ late: true }])
+    await broker.run(async (queen) => {
+      const read = await queen.queue('q').fetch(['p'])
+      // The first answer said the partition ended at 4: offsets 0..3 are read,
+      // and the call stops there although the partition kept growing.
+      assert.deepEqual(offsetsOf(read.get('p')), [0, 1, 2, 3])
+      assert.equal(broker.fetches().length, 2)
+      assert.equal(read.get('p').nextOffset, 4)
+    })
+  })
+
+  it('asks for at most 1000 partitions per call', async () => {
+    const broker = new FakeBroker()
+    const names = Array.from({ length: 2500 }, (_, i) => `p-${i}`)
+    for (const name of names) broker.write('q', name, [{ name }])
+    await broker.run(async (queen) => {
+      const read = await queen.queue('q').fetch(names)
+      assert.deepEqual(broker.fetches().map(h => h.body.entries.length), [1000, 1000, 500])
+      assert.equal(read.size, 2500)
+      for (const name of names) assert.deepEqual(read.get(name).records.map(r => r.payload), [{ name }])
+    })
+  })
+
+  it('asks again for the partitions the call ceiling left empty', async () => {
+    const broker = new FakeBroker()
+    const fat = 'x'.repeat(100)
+    for (const name of ['a', 'b', 'c']) broker.write('q', name, [fat, fat])
+    await broker.run(async (queen) => {
+      // Each record weighs 102 bytes: one call carries two of them at most.
+      const read = await queen.queue('q').fetch(['a', 'b', 'c'], { maxBytes: 250 })
+      for (const name of ['a', 'b', 'c']) assert.deepEqual(offsetsOf(read.get(name)), [0, 1])
+      assert.ok(broker.fetches().length > 1)
+      assert.ok(broker.fetches().every(h => h.body.maxBytes === 250))
+    })
+  })
+
+  it('reads from logStartOffset when retention deleted the start', async () => {
+    const broker = new FakeBroker()
+    broker.write('q', 'p', payloads(10))
+    broker.trim('q', 'p', 6)
+    await broker.run(async (queen) => {
+      const read = await queen.queue('q').fetch(['p'])
+      assert.deepEqual(offsetsOf(read.get('p')), [6, 7, 8, 9])
+      assert.equal(read.get('p').logStartOffset, 6)
+      assert.deepEqual(broker.fetches().map(h => h.body.entries[0].offset), [0, 6])
+    })
+  })
+
+  it('starts each partition at its offset in options.offsets', async () => {
+    const broker = new FakeBroker()
+    broker.write('q', 'a', payloads(5))
+    broker.write('q', 'b', payloads(5))
+    await broker.run(async (queen) => {
+      const read = await queen.queue('q').fetch(['a', 'b'], { offsets: { a: 3 } })
+      assert.deepEqual(offsetsOf(read.get('a')), [3, 4])
+      assert.deepEqual(offsetsOf(read.get('b')), [0, 1, 2, 3, 4])
+      const again = await queen.queue('q').fetch(['a'], { offsets: new Map([['a', read.get('a').nextOffset]]) })
+      assert.deepEqual(again.get('a').records, [])
+      assert.equal(again.get('a').nextOffset, 5)
+    })
+  })
+
+  it('answers no records for a missing queue, an unwritten partition and an offset past the end', async () => {
+    const broker = new FakeBroker()
+    broker.write('q', 'p', payloads(2))
+    await broker.run(async (queen) => {
+      const missing = await queen.queue('nope').fetch(['p'])
+      assert.deepEqual(missing.get('p').records, [])
+      const read = await queen.queue('q').fetch(['never', 'p'], { offsets: { p: 7 } })
+      assert.deepEqual(read.get('never').records, [])
+      assert.deepEqual(read.get('p').records, [])
+      assert.equal(read.get('p').nextOffset, 7)
+      assert.equal(read.get('p').highWatermark, 2)
+    })
+  })
+
+  it('throws on an error marker it does not know', async () => {
+    const broker = new FakeBroker()
+    broker.write('q', 'p', payloads(1))
+    broker.fetch = () => [200, { entries: [{ queue: 'q', partition: 'p', records: [], highWatermark: 1, logStartOffset: 0, error: 'SOMETHING_NEW' }] }]
+    await broker.run(async (queen) => {
+      await assert.rejects(queen.queue('q').fetch(['p']), /SOMETHING_NEW/)
+    })
+  })
+
+  it('stops when a call reads nothing although the partition is not at its end', async () => {
+    const broker = new FakeBroker()
+    broker.fetch = () => [200, { entries: [{ queue: 'q', partition: 'p', records: [], highWatermark: 5, logStartOffset: 0 }] }]
+    await broker.run(async (queen) => {
+      const read = await queen.queue('q').fetch(['p'])
+      assert.deepEqual(read.get('p').records, [])
+      assert.equal(broker.fetches().length, 1)
+    })
+  })
+
+  it('asks once for a partition named twice and refuses a bad name or offset before any call', async () => {
+    const broker = new FakeBroker()
+    broker.write('q', 'p', payloads(1))
+    await broker.run(async (queen) => {
+      const read = await queen.queue('q').fetch(['p', 'p'])
+      assert.equal(read.size, 1)
+      assert.equal(broker.fetches()[0].body.entries.length, 1)
+      await assert.rejects(queen.queue('q').fetch(['']), /non-empty string/)
+      await assert.rejects(queen.queue('q').fetch(['p'], { offsets: { p: -1 } }), /non-negative integer/)
+      await assert.rejects(queen.queue('q').fetch(['p'], { maxBytes: 0 }), /positive integer/)
+      assert.equal(broker.fetches().length, 1)
+    })
+  })
+})
+
+describe('queue.changedSince()', () => {
+  it('follows the page cursor to the end of the pass, in creation order', async () => {
+    const broker = new FakeBroker()
+    for (let i = 0; i < 2300; i++) broker.write('tf.events', `t-${i}`, [{ i }])
+    await broker.run(async (queen) => {
+      const { partitions, safeTime } = await queen.queue('tf.events').changedSince('2026-10-20T09:00:00Z')
+      assert.equal(partitions.length, 2300)
+      assert.equal(partitions[0].name, 't-0')
+      assert.equal(partitions[2299].name, 't-2299')
+      assert.deepEqual(partitions[0], { name: 't-0', id: 'id-t-0', lastOffset: 0, logStart: 0, lastWriteAt: '2026-10-20T10:00:00.000000Z' })
+      assert.equal(safeTime, '2026-10-20T12:00:00.000000Z')
+      const bodies = broker.hits.map(h => h.body.entries[0])
+      assert.deepEqual(bodies.map(b => b.after ?? null), [null, 'p|999', 'p|1999'])
+      assert.ok(bodies.every(b => b.since === '2026-10-20T09:00:00Z' && b.limit === 1000 && b.queue === 'tf.events'))
+    })
+  })
+
+  it('lists only the partitions written at or after since', async () => {
+    const broker = new FakeBroker()
+    broker.write('q', 'old', [1], { lastWriteAt: '2026-10-20T08:00:00.000000Z' })
+    broker.write('q', 'new', [1], { lastWriteAt: '2026-10-20T10:30:00.000000Z' })
+    await broker.run(async (queen) => {
+      const since = new Date('2026-10-20T10:00:00Z')
+      const { partitions } = await queen.queue('q').changedSince(since)
+      assert.deepEqual(partitions.map(p => p.name), ['new'])
+      assert.equal(broker.hits[0].body.entries[0].since, '2026-10-20T10:00:00.000Z')
+      await queen.queue('q').changedSince(since.getTime())
+      assert.equal(broker.hits[1].body.entries[0].since, '2026-10-20T10:00:00.000Z')
+    })
+  })
+
+  it('without since lists every partition, and sends no since', async () => {
+    const broker = new FakeBroker()
+    broker.write('q', 'a', [1], { lastWriteAt: '2020-01-01T00:00:00.000000Z' })
+    await broker.run(async (queen) => {
+      const { partitions } = await queen.queue('q').changedSince()
+      assert.deepEqual(partitions.map(p => p.name), ['a'])
+      assert.equal('since' in broker.hits[0].body.entries[0], false)
+    })
+  })
+
+  it('answers the smallest safeTime of its pages', async () => {
+    const broker = new FakeBroker()
+    for (let i = 0; i < 1500; i++) broker.write('q', `p-${i}`, [1])
+    broker.safeTimes = ['2026-10-20T12:00:05.000000Z', '2026-10-20T12:00:01.000000Z']
+    await broker.run(async (queen) => {
+      const { safeTime } = await queen.queue('q').changedSince(null)
+      assert.equal(safeTime, '2026-10-20T12:00:01.000000Z')
+    })
+  })
+
+  it('answers no partitions for a queue that does not exist, and throws on BAD_CURSOR', async () => {
+    const broker = new FakeBroker()
+    await broker.run(async (queen) => {
+      const { partitions, safeTime } = await queen.queue('ghost').changedSince('2026-10-20T00:00:00Z')
+      assert.deepEqual(partitions, [])
+      assert.equal(safeTime, '2026-10-20T12:00:00.000000Z')
+    })
+    const bad = new FakeBroker()
+    bad.changed = () => [200, { safeTime: '2026-10-20T12:00:00.000000Z', safeTimeDegraded: false, entries: [{ queue: 'q', error: 'BAD_CURSOR' }] }]
+    await bad.run(async (queen) => {
+      await assert.rejects(queen.queue('q').changedSince(null), /BAD_CURSOR/)
+    })
+  })
+
+  it('refuses a since that is not a time before any call', async () => {
+    const broker = new FakeBroker()
+    await broker.run(async (queen) => {
+      await assert.rejects(queen.queue('q').changedSince(new Date('nope')), /invalid Date/)
+      await assert.rejects(queen.queue('q').changedSince({}), /ISO-8601/)
+      assert.equal(broker.hits.length, 0)
+    })
+  })
+})