queen-mq 0.16.0 → 1.0.3

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "queen-mq",
-  "version": "0.16.0",
+  "version": "1.0.3",
   "type": "module",
   "description": "Partitioned message queue on PostgreSQL — broker client + fluent streaming SDK (windows, joins, gates) in one package",
   "main": "client-v2/index.js",
   "scripts": {
-    "test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js && node test-v2/run.js human",
-    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js",
+    "test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js && node test-v2/run.js human",
+    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js",
     "test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
     "test:integration": "node test-v2/run.js human",
     "test:streams": "node test-v2/run.js stream",
@@ -20,6 +20,7 @@
   "dependencies": {
     "axios": "^1.12.2",
     "pg": "^8.16.3",
+    "undici": "^6.21.0",
     "uuid": "^13.0.0"
   },
   "engines": {
@@ -27,16 +28,24 @@
   },
   "author": "Smartness",
   "license": "Apache-2.0",
+  "homepage": "https://queenmq.com",
   "repository": {
     "type": "git",
     "url": "https://github.com/queen-mq/queen"
   },
+  "bugs": {
+    "url": "https://github.com/queen-mq/queen/issues"
+  },
   "keywords": [
     "message-queue",
     "queue",
+    "postgres",
+    "postgresql",
     "broker",
     "message-queue-system",
     "fifo",
+    "consumer-groups",
+    "dead-letter-queue",
     "streaming",
     "stream-processing",
     "rate-limiter",

package/test-v2/_kvtimers.js

@@ -0,0 +1,71 @@
+/**
+ * Shared helpers for the kv and timers integration suites.
+ *
+ * Deliberately NOT registered in run.js: that runner treats every export of a
+ * registered module as a test function, so a helper exported from kv.js would
+ * be executed as a test and reported as one.
+ *
+ * WHY THERE IS NO AVAILABILITY PROBE HERE ANY MORE. There used to be one, plus
+ * a `skipped()` verdict, because `QUEEN_KV_ENABLED` and `QUEEN_TIMERS_ENABLED`
+ * were boot flags that defaulted to false: with them off the routes were not
+ * registered and every call here answered 404, so a red suite was the DEFAULT
+ * configuration's own fault and the skip existed to stop that training everyone
+ * to ignore the colour.
+ *
+ * Those flags are gone. Kv and timers are not features, they are the broker:
+ * there is no `QUEEN_PUSH_ENABLED` either. Every cell that runs this binary has
+ * both surfaces, so these suites RUN, and a 404 from any of them is a bug in
+ * the broker, not a configuration to detect. A test that skips is a test that
+ * says nothing, and that was only tolerable while the 404 was legitimate.
+ *
+ * What still exists is the operator's RUNTIME kill switch (`kv_enabled`,
+ * `timers_schedule_enabled`, `timers_fire_enabled` in `queen.system_state`) --
+ * the maintenance-mode lever, pulled live during an incident. It answers 503
+ * with `Retry-After` on the kv/timer routes and 403 on a `kv`/`timers` rider
+ * inside a transaction, never 404. If a run here ever goes red against that,
+ * the switch is down on the rig and somebody pulled it; it is not something
+ * these tests should paper over.
+ */
+
+export const KV_NS = 'test-kv'
+export const TIMER_QUEUE = 'test-timers'
+
+export const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+
+/**
+ * Poll a queue until `match` accepts a message, or the deadline passes.
+ *
+ * WHY THIS IS A POLL AND NOT A LONG POLL, MEASURED ON A REAL RIG. A timer's
+ * delivery is committed by the sweeper INSIDE PostgreSQL
+ * (`log_timers_fire_v1` = DELETE + push in one transaction), so it does not
+ * pass through the broker's push handler and the destination partition is not
+ * marked ready in that broker's hot list. A consumer therefore sees the
+ * message only at the next hot-list reseed: on a default broker
+ * (QUEEN_HOTLIST_RESEED_MS = 30000) a timer scheduled for +300 ms was measured
+ * arriving at +30.7 s, and a 20 s long poll timed out with the message already
+ * committed in the log.
+ *
+ * That is a broker property, not a client one, and it is why these tests
+ * assert ARRIVAL and never latency: `deliverAt` is "not before", never
+ * "exactly at". The deadline here is generous on purpose -- it is sized
+ * against the reseed period, not against the delay under test.
+ */
+export async function popUntil(client, queueName, match, timeoutMs = 60000) {
+  const deadline = Date.now() + timeoutMs
+  for (;;) {
+    const messages = await client.queue(queueName).batch(10).wait(false).pop()
+    const hit = messages.filter(match)
+    if (hit.length > 0) return hit
+    // Anything else in this queue is a stranger -- a delivery left behind by an
+    // earlier run of this same test. It is ACKED rather than ignored, and that
+    // is not tidiness: queue-mode delivery is ordered, so an unacked stranger
+    // holds its lease and blocks the cursor, and the message under test would
+    // never be reached inside any deadline. This is the shape of the failure
+    // that a "wait longer" would never have fixed.
+    for (const stranger of messages) {
+      await client.ack(stranger)
+    }
+    if (Date.now() > deadline) return []
+    await sleep(500)
+  }
+}

package/test-v2/ackwindow.js

@@ -0,0 +1,265 @@
+/**
+ * Ack-window honesty tests (2026-07-30).
+ *
+ * queen.log_ack_by_hash_v1 resolves txn hashes through the queen.log_txns
+ * sidecar, which is purged on its own clock (GREATEST(dedup_window,
+ * completed_retention, 900s)). A hash that cannot be resolved — purged row,
+ * or a transactionId that never existed — is correctly NOT acked (the cursor
+ * stops, frames redeliver: redelivery over loss). The BUG these tests pin:
+ * the broker reported those items as success=true (they appear in neither
+ * noopHashes nor staleHashes), so the client believed the ack landed while
+ * the cursor never moved — a silent redelivery livelock, and a nack/dlq in
+ * that state could never dead-letter its poison message.
+ *
+ * Expected contract (post-fix): unresolvable items come back success=false
+ * with an explicit "unresolvable" error, and the cursor is untouched.
+ *
+ * Companion test: emptyPartitionCursorSealsAfterRetention pins the empty-
+ * partition cursor seal in log_pop_v1 (the Rust port of the C++
+ * pop_unified_batch_v4 starvation fix): a partition whose segments were all
+ * removed by retention must stop being "phantom pending" after one pop.
+ *
+ * Run: node run.js ackUnknownTxnMustFail
+ *      node run.js ackAfterHashPurgeMustFailExplicitly
+ *      node run.js emptyPartitionCursorSealsAfterRetention
+ */
+
+import { dbPool } from './run.js'
+
+const TENANT = '00000000-0000-0000-0000-000000000001'
+
+function sleep(ms) { return new Promise(r => setTimeout(r, ms)) }
+
+async function popRetry(client, queue, { batch = 3, group = null, partition = null, tries = 30, mode = null } = {}) {
+    for (let i = 0; i < tries; i++) {
+        let b = client.queue(queue).batch(batch).wait(false)
+        if (group) b = b.group(group)
+        if (partition) b = b.partition(partition)
+        if (mode) b = b.subscriptionMode(mode)
+        const msgs = await b.pop()
+        if (msgs && msgs.length > 0) return msgs
+        await sleep(150)
+    }
+    return []
+}
+
+async function partitionRow(queue, partition = 'Default') {
+    const r = await dbPool.query(
+        `SELECT p.id::text AS id, p.last_offset::bigint AS last_offset, p.log_start::bigint AS log_start
+         FROM queen.log_partitions p
+         JOIN queen.queues lq ON lq.id = p.queue_id
+         WHERE lq.name = $1 AND p.name = $2 AND lq.tenant_id = $3::uuid`,
+        [queue, partition, TENANT])
+    return r.rows[0] || null
+}
+
+async function committedOf(partitionId, group) {
+    const r = await dbPool.query(
+        `SELECT c.committed::bigint AS committed FROM queen.log_consumers c
+         WHERE c.partition_id = $1::uuid AND c.consumer_group = $2`,
+        [partitionId, group])
+    return r.rows.length ? Number(r.rows[0].committed) : null
+}
+
+async function segmentCount(partitionId) {
+    const r = await dbPool.query(
+        `SELECT count(*)::int AS n FROM queen.log_segments WHERE partition_id = $1::uuid`,
+        [partitionId])
+    return r.rows[0].n
+}
+
+/**
+ * Test A (wire-only): acking a transactionId that never existed must fail
+ * explicitly, for both a completed ack and a failed nack. Today the broker
+ * answers success=true for both.
+ */
+export async function ackUnknownTxnMustFail(client) {
+    const queue = 'test-queue-v2-ackwindow-unknown'
+    await client.queue(queue).create()
+    await client.queue(queue).partition('Default').push([
+        { data: { n: 1 }, transactionId: `${queue}-tx-1` },
+        { data: { n: 2 }, transactionId: `${queue}-tx-2` },
+        { data: { n: 3 }, transactionId: `${queue}-tx-3` },
+    ])
+
+    const msgs = await popRetry(client, queue, { batch: 3 })
+    if (msgs.length !== 3) return { success: false, message: `expected 3 messages, got ${msgs.length}` }
+
+    const ghost = {
+        transactionId: `${queue}-ghost-never-pushed`,
+        partitionId: msgs[0].partitionId,
+        leaseId: msgs[0].leaseId,
+    }
+
+    // Completed ack of a nonexistent txn: must NOT be reported as success.
+    const r1 = await client.ack([ghost], true)
+    const item1 = (r1.results || [])[0] || {}
+    if (r1.success !== false || item1.success !== false) {
+        return { success: false, message: `BUG: completed ack of a never-pushed txn reported success (overall=${r1.success}, item=${item1.success})` }
+    }
+    if (!/unresolv/i.test(item1.error || '')) {
+        return { success: false, message: `unknown-txn ack rejected with wrong error: '${item1.error}'` }
+    }
+
+    // Failed nack of a nonexistent txn: must NOT be swallowed as success
+    // (pre-fix this was the worst variant: the nack vanished — no retry
+    // charge, no DLQ — while the client was told ok).
+    const r2 = await client.ack([{ ...ghost }], false)
+    const item2 = (r2.results || [])[0] || {}
+    if (r2.success !== false || item2.success !== false) {
+        return { success: false, message: `BUG: failed nack of a never-pushed txn reported success (overall=${r2.success}, item=${item2.success})` }
+    }
+    if (!/unresolv/i.test(item2.error || '')) {
+        return { success: false, message: `unknown-txn nack rejected with wrong error: '${item2.error}'` }
+    }
+
+    // The real batch must still be ackable (the lease survived the rejected calls).
+    const r3 = await client.ack(msgs, true)
+    if (r3.success !== true) {
+        return { success: false, message: `real batch ack failed after ghost-ack rejections: ${r3.error}` }
+    }
+
+    return { success: true, message: 'unknown-txn ack and nack both rejected explicitly; real batch still ackable' }
+}
+
+/**
+ * Test B (with SQL): the real-world shape — messages still delivered (segments
+ * intact) but their log_txns hash rows purged. Per-message acks must fail
+ * explicitly and the cursor must not move. Pre-fix: every ack reported
+ * success=true while committed stayed put (silent livelock).
+ */
+export async function ackAfterHashPurgeMustFailExplicitly(client) {
+    const queue = 'test-queue-v2-ackwindow-purged'
+    await client.queue(queue).create()
+    const txns = [1, 2, 3, 4, 5].map(n => `${queue}-tx-${n}`)
+    await client.queue(queue).partition('Default').push(
+        txns.map((t, i) => ({ data: { n: i }, transactionId: t })))
+
+    // Wait until the segments are visible, then purge the hash sidecar —
+    // exactly what log_txns_purge_step_v1 does after the window expires
+    // (the 900s floor makes the real purge untestable in-suite).
+    const part = await partitionRow(queue)
+    if (!part) return { success: false, message: 'partition row not found' }
+    for (let i = 0; i < 30 && (await segmentCount(part.id)) === 0; i++) await sleep(100)
+
+    const del = await dbPool.query(
+        `DELETE FROM queen.log_txns WHERE partition_id = $1::uuid`, [part.id])
+    if (del.rowCount === 0) return { success: false, message: 'no log_txns rows to purge — test setup broken' }
+
+    // Segments untouched: the messages still DELIVER.
+    const msgs = await popRetry(client, queue, { batch: 5 })
+    if (msgs.length !== 5) return { success: false, message: `expected 5 delivered messages after purge, got ${msgs.length}` }
+
+    // Per-message completed ack (the JS per-message consumer shape): the hash
+    // cannot resolve, so the honest answer is an explicit failure.
+    const r1 = await client.ack([msgs[0]], true)
+    const item1 = (r1.results || [])[0] || {}
+    if (item1.success !== false || !/unresolv/i.test(item1.error || '')) {
+        return { success: false, message: `BUG: ack with purged hash answered success=${item1.success}, error='${item1.error}'` }
+    }
+
+    // Nack of a purged-hash message: same — and pre-fix this also meant the
+    // poison could never reach the DLQ.
+    const r2 = await client.ack([msgs[1]], false)
+    const item2 = (r2.results || [])[0] || {}
+    if (item2.success !== false || !/unresolv/i.test(item2.error || '')) {
+        return { success: false, message: `BUG: nack with purged hash answered success=${item2.success}, error='${item2.error}'` }
+    }
+
+    // The cursor must not have moved (redelivery over loss — that part of the
+    // contract was always right; only the reporting lied).
+    const committed = await committedOf(part.id, '__QUEUE_MODE__')
+    if (committed !== -1) {
+        return { success: false, message: `cursor moved on unresolvable acks: committed=${committed}, expected -1` }
+    }
+
+    return { success: true, message: 'purged-hash ack and nack rejected explicitly, cursor untouched, messages still deliverable' }
+}
+
+/**
+ * Test C (bug 2, with SQL): empty-partition cursor seal. A group consumes
+ * part of a partition, retention then deletes ALL segments (time rule deletes
+ * unconsumed data too): last_offset > committed with zero segments = a
+ * phantom-pending row that used to stay a wildcard candidate forever and
+ * pinned the empty-scan watermark at epoch. After the fix, one empty pop
+ * seals committed to the partition's last_offset. A delayed-processing queue
+ * pins the guard: segments EXIST but are deferred → no seal.
+ */
+export async function emptyPartitionCursorSealsAfterRetention(client) {
+    const queue = 'test-queue-v2-ackwindow-seal'
+    const group = 'g-seal'
+    await client.queue(queue).config({ retentionEnabled: true, retentionSeconds: 1, leaseTime: 30 }).create()
+    await client.queue(queue).partition('Default').push(
+        [0, 1, 2, 3, 4, 5].map(n => ({ data: { n }, transactionId: `${queue}-tx-${n}` })))
+
+    // Consume + ack only the first two: committed lands at 1, the rest stays.
+    // First contact for g-seal, and the 6 messages are already pushed: the
+    // group has to ask for the backlog it is about to half-consume.
+    const first = await popRetry(client, queue, { batch: 2, group, mode: 'all' })
+    if (first.length !== 2) return { success: false, message: `expected 2 messages, got ${first.length}` }
+    const ackr = await client.ack(first, true, { group })
+    if (ackr.success !== true) return { success: false, message: `ack failed: ${ackr.error}` }
+
+    const part = await partitionRow(queue)
+    if (!part) return { success: false, message: 'partition row not found' }
+
+    // Retention (interval 2s in the harness, cutoff 1s) must delete ALL
+    // segments — the time rule ignores cursors.
+    let segs = -1
+    for (let i = 0; i < 60; i++) {
+        segs = await segmentCount(part.id)
+        if (segs === 0) break
+        await sleep(500)
+    }
+    if (segs !== 0) return { success: false, message: `retention never emptied the partition (segments=${segs})` }
+
+    const before = await committedOf(part.id, group)
+    const rowNow = await partitionRow(queue)
+    if (before === null) return { success: false, message: 'consumer row missing before the empty pop' }
+    if (before >= Number(rowNow.last_offset)) {
+        return { success: false, message: `setup broken: no phantom span (committed=${before}, last_offset=${rowNow.last_offset})` }
+    }
+
+    // ONE empty pop on the emptied partition must seal the cursor.
+    const empty = await client.queue(queue).partition('Default').group(group).batch(1).wait(false).pop()
+    if (empty && empty.length > 0) return { success: false, message: `expected an empty pop, got ${empty.length} messages` }
+
+    const after = await committedOf(part.id, group)
+    if (after !== Number(rowNow.last_offset)) {
+        return { success: false, message: `BUG: empty pop did not seal the cursor (committed=${after}, want last_offset=${rowNow.last_offset}) — partition stays phantom-pending forever` }
+    }
+
+    // Guard: delayed messages are NOT sealed over. Segments exist but are
+    // deferred; the cursor must stay put so they deliver when eligible.
+    const dqueue = 'test-queue-v2-ackwindow-seal-delayed'
+    await client.queue(dqueue).config({ delayedProcessing: 30, leaseTime: 30 }).create()
+    await client.queue(dqueue).partition('Default').push([
+        { data: { n: 0 }, transactionId: `${dqueue}-tx-0` },
+        { data: { n: 1 }, transactionId: `${dqueue}-tx-1` },
+    ])
+    // Give the fusion flush a moment so the segments exist before the pop.
+    const dpart0 = await (async () => {
+        for (let i = 0; i < 30; i++) {
+            const p = await partitionRow(dqueue)
+            if (p && (await segmentCount(p.id)) > 0) return p
+            await sleep(100)
+        }
+        return null
+    })()
+    if (!dpart0) return { success: false, message: 'delayed queue segments never appeared' }
+
+    // First contact for g-seal on THIS queue too (metadata is queue-scoped), and
+    // the 2 delayed messages are already pushed: under the default 'new' the
+    // cursor would seed at last_offset and the guard below could not tell a seed
+    // from a seal. 'all' seeds at -1, so a non -1 cursor can only be the seal.
+    const dempty = await client.queue(dqueue).partition('Default').group(group).subscriptionMode('all').batch(1).wait(false).pop()
+    if (dempty && dempty.length > 0) return { success: false, message: 'delayed message delivered early?' }
+    const dcommitted = await committedOf(dpart0.id, group)
+    const dsegs = await segmentCount(dpart0.id)
+    if (dsegs === 0) return { success: false, message: 'delayed queue segments vanished' }
+    if (dcommitted !== -1 && dcommitted !== null) {
+        return { success: false, message: `BUG: seal fired over deferred-but-present segments (committed=${dcommitted}) — delayed messages would be skipped` }
+    }
+
+    return { success: true, message: 'empty partition sealed after one pop; deferred segments not sealed over' }
+}

package/test-v2/auth.js

@@ -19,9 +19,8 @@
 // unattended in either server configuration.
 
 import crypto from 'crypto'
-import { dbPool } from './run.js'
 
-const SERVER_URL = process.env.QUEEN_URL || 'http://localhost:6632'
+const SERVER_URL = process.env.QUEEN_URL || process.env.QUEEN_SERVER_URL || 'http://localhost:6632'
 const JWT_SECRET = process.env.JWT_SECRET || ''
 const JWT_ENABLED = JWT_SECRET.length > 0
 
@@ -50,23 +49,6 @@ function makeToken(sub, role = 'read-write') {
     )
 }
 
-// ---------------------------------------------------------------------------
-// Ground-truth helper: read producer_sub directly from Postgres, bypassing
-// the API so we can tell the difference between a server that actually stores
-// the value and one that just echoes the client's input.
-// ---------------------------------------------------------------------------
-async function getStoredProducerSub(queueName, transactionId) {
-    const res = await dbPool.query(
-        `SELECT m.producer_sub
-         FROM queen.messages m
-         JOIN queen.partitions p ON p.id = m.partition_id
-         JOIN queen.queues q ON q.id = p.queue_id
-         WHERE q.name = $1 AND m.transaction_id = $2`,
-        [queueName, transactionId]
-    )
-    return res.rows.length > 0 ? res.rows[0].producer_sub : null
-}
-
 // Raw HTTP push so we can control the Authorization header and inject
 // arbitrary extra fields (like a forged `producerSub`) per-test.
 async function httpPush({ queue, partition = 'Default', transactionId, data, bearerToken, extraItemFields = {} }) {
@@ -110,132 +92,51 @@ async function httpConfigure(queue, bearerToken) {
     }
 }
 
-// Direct SP call via dbPool — bypasses HTTP and JWT entirely. Used by the
-// SP-level tests below so they work regardless of whether the server has
-// auth enabled or not.
-async function callPushMessagesV3(items) {
-    await dbPool.query(
-        'SELECT queen.push_messages_v3($1::jsonb)',
-        [JSON.stringify(items)]
-    )
-}
-
-// Direct SP pop. Returns the `messages` array for the first (only) partition.
-async function callPopSpecificBatch(queueName) {
-    const reqs = [{
-        idx: 0,
-        queue_name: queueName,
-        partition_name: 'Default',
-        consumer_group: '__QUEUE_MODE__',
-        lease_seconds: 60,
-        worker_id: `test-worker-${Date.now()}`,
-        batch_size: 10,
-        sub_mode: 'all',
-        sub_from: ''
-    }]
-    const res = await dbPool.query(
-        'SELECT queen.pop_specific_batch($1::jsonb) as out',
-        [JSON.stringify(reqs)]
-    )
-    const out = res.rows[0].out
-    return out[0]?.result?.messages || []
-}
-
-// ===========================================================================
-// TEST 1: Stored procedure round-trip (SQL-only, no HTTP, no JWT)
-// ---------------------------------------------------------------------------
-// Exercises the schema + SP contract end-to-end purely via SQL. If this fails,
-// the migration or 001_push.sql / 002_pop_unified.sql is wrong. Runs in any
-// server configuration because it talks to Postgres directly.
-// ===========================================================================
-export async function producerSubRoundTripViaStoredProcedure(client) {
-    const q = 'test-auth-sp-roundtrip'
-    const tx = `tx-sp-${Date.now()}`
-
-    await callPushMessagesV3([{
-        queue: q,
-        partition: 'Default',
-        transactionId: tx,
-        payload: { x: 1 },
-        producerSub: 'sp-test-sub'
-    }])
-
-    const stored = await getStoredProducerSub(q, tx)
-    if (stored !== 'sp-test-sub') {
-        return { success: false, message: `Expected producer_sub='sp-test-sub' in DB, got ${JSON.stringify(stored)}` }
-    }
-
-    const popped = await callPopSpecificBatch(q)
-    const target = popped.find(m => m.transactionId === tx)
-    if (!target) {
-        return { success: false, message: `Expected to find tx ${tx} in pop result` }
-    }
-    if (target.producerSub !== 'sp-test-sub') {
-        return { success: false, message: `Pop returned producerSub=${JSON.stringify(target.producerSub)}, expected 'sp-test-sub'` }
+// Black-box observation for the HTTP push tests: pop the message back through
+// the API and read its `producerSub`. The segments engine stores messages in
+// queen.seg_segments (never queen.messages), so a seg-broker push is only
+// observable via pop — the seg-native ground truth. Optional bearer so the
+// JWT-enabled pops are authorized.
+async function httpPop({ queue, bearerToken, batch = 20, waitMs = 5000 }) {
+    const headers = {}
+    if (bearerToken) headers['Authorization'] = `Bearer ${bearerToken}`
+    const url = `${SERVER_URL}/api/v1/pop/queue/${encodeURIComponent(queue)}`
+        + `?batch=${batch}&wait=true&timeout=${waitMs}&autoAck=true`
+    const res = await fetch(url, { method: 'GET', headers })
+    if (res.status === 204) return []
+    const text = await res.text()
+    if (!res.ok) {
+        throw new Error(`pop failed ${res.status}: ${text}`)
     }
-
-    return { success: true, message: 'producer_sub round-trips through push_messages_v3 and pop exposes it' }
+    const body = text ? JSON.parse(text) : {}
+    return (body.messages || []).filter(m => m != null)
 }
 
-// ===========================================================================
-// TEST 2: SP with no producerSub => producer_sub is NULL
-// ---------------------------------------------------------------------------
-// Guards against an accidental "default to empty string" regression in the SP.
-// ===========================================================================
-export async function producerSubNullWhenNotProvided(client) {
-    const q = 'test-auth-sp-null'
-    const tx = `tx-sp-null-${Date.now()}`
-
-    await callPushMessagesV3([{
-        queue: q,
-        partition: 'Default',
-        transactionId: tx,
-        payload: { x: 1 }
-        // no producerSub
-    }])
-
-    const stored = await getStoredProducerSub(q, tx)
-    if (stored !== null) {
-        return { success: false, message: `Expected NULL producer_sub, got ${JSON.stringify(stored)}` }
-    }
-
-    const popped = await callPopSpecificBatch(q)
-    const target = popped.find(m => m.transactionId === tx)
-    if (!target) {
-        return { success: false, message: `Expected to find tx ${tx} in pop result` }
-    }
-    if (target.producerSub != null) {
-        return { success: false, message: `Pop returned producerSub=${JSON.stringify(target.producerSub)}, expected null` }
+// Pop until we observe the message with the given transactionId; returns its
+// producerSub (a seg push is immediately visible via fire-on-idle, but retry a
+// few times to be robust to any push→visibility race).
+async function poppedProducerSub(queue, transactionId, bearerToken) {
+    const deadline = Date.now() + 6000
+    while (Date.now() < deadline) {
+        const msgs = await httpPop({ queue, bearerToken })
+        const hit = msgs.find(m => m.transactionId === transactionId)
+        if (hit) return { found: true, producerSub: hit.producerSub }
+        await new Promise(r => setTimeout(r, 150))
     }
-
-    return { success: true, message: 'Omitting producerSub yields NULL in DB and null in pop response' }
+    return { found: false, producerSub: undefined }
 }
 
 // ===========================================================================
-// TEST 3: SP treats empty string producerSub as NULL
-// ---------------------------------------------------------------------------
-// The SP uses NULLIF(..., '') to avoid storing a literal empty string when a
-// client accidentally sends "". Verifies that invariant.
+// (Retired) SP-level producer_sub round-trip / NULL / empty-string tests.
+// These drove the ROWS engine directly (queen.push_messages_v3 +
+// queen.pop_specific_batch + a queen.messages read) to verify the SP's
+// producer_sub NULLIF handling. The rows engine is retired (segments-only), so
+// they were removed — the segments engine stores payloads in queen.seg_segments
+// and stamps producer_sub in the broker, never in queen.messages. The shipping
+// producer_sub behavior (server-stamped, body-ignored without auth, spoof-proof
+// under JWT) is covered black-box through the HTTP push -> pop path by the tests
+// below.
 // ===========================================================================
-export async function producerSubEmptyStringStoredAsNull(client) {
-    const q = 'test-auth-sp-empty'
-    const tx = `tx-sp-empty-${Date.now()}`
-
-    await callPushMessagesV3([{
-        queue: q,
-        partition: 'Default',
-        transactionId: tx,
-        payload: { x: 1 },
-        producerSub: ''
-    }])
-
-    const stored = await getStoredProducerSub(q, tx)
-    if (stored !== null) {
-        return { success: false, message: `Expected NULL for empty-string producerSub, got ${JSON.stringify(stored)}` }
-    }
-
-    return { success: true, message: 'Empty-string producerSub is stored as NULL' }
-}
 
 // ===========================================================================
 // TEST 4: HTTP push without JWT => producer_sub is NULL
@@ -265,12 +166,17 @@ export async function producerSubIgnoredFromBodyWithoutAuth(client) {
         extraItemFields: { producerSub: 'attacker-no-jwt' }
     })
 
-    const stored = await getStoredProducerSub(q, tx)
-    if (stored !== null) {
-        return { success: false, message: `Expected NULL producer_sub (auth disabled), got ${JSON.stringify(stored)} - client was able to set it!` }
+    // Black-box: observe the pushed message via pop. producerSub must be null
+    // (the field is not client-settable, and there is no JWT sub to stamp).
+    const { found, producerSub } = await poppedProducerSub(q, tx)
+    if (!found) {
+        return { success: false, message: `Pushed message ${tx} not observed via pop` }
+    }
+    if (producerSub !== null) {
+        return { success: false, message: `Expected producerSub null (auth disabled), got ${JSON.stringify(producerSub)} - client was able to set it!` }
     }
 
-    return { success: true, message: 'Body-supplied producerSub ignored when auth disabled; stored as NULL' }
+    return { success: true, message: 'Body-supplied producerSub ignored when auth disabled; pop shows null' }
 }
 
 // ===========================================================================
@@ -297,12 +203,17 @@ export async function producerSubStampedFromJwt(client) {
         bearerToken: token
     })
 
-    const stored = await getStoredProducerSub(q, tx)
-    if (stored !== 'alice-producer') {
-        return { success: false, message: `Expected producer_sub='alice-producer', got ${JSON.stringify(stored)}` }
+    // Black-box: pop the message back and read producerSub (the seg-native
+    // ground truth). It must equal the authenticated JWT sub.
+    const { found, producerSub } = await poppedProducerSub(q, tx, token)
+    if (!found) {
+        return { success: false, message: `Pushed message ${tx} not observed via pop` }
+    }
+    if (producerSub !== 'alice-producer') {
+        return { success: false, message: `Expected producerSub='alice-producer', got ${JSON.stringify(producerSub)}` }
     }
 
-    return { success: true, message: 'producer_sub stamped from authenticated JWT sub claim' }
+    return { success: true, message: 'producer_sub stamped from authenticated JWT sub claim (observed via pop)' }
 }
 
 // ===========================================================================
@@ -328,13 +239,18 @@ export async function producerSubSpoofingIgnoredWithJwt(client) {
         extraItemFields: { producerSub: 'attacker' }
     })
 
-    const stored = await getStoredProducerSub(q, tx)
-    if (stored !== 'legit-producer') {
+    // Black-box: pop the message back; the body-supplied producerSub must be
+    // ignored and the observed value must be the validated JWT sub.
+    const { found, producerSub } = await poppedProducerSub(q, tx, token)
+    if (!found) {
+        return { success: false, message: `Pushed message ${tx} not observed via pop` }
+    }
+    if (producerSub !== 'legit-producer') {
         return {
             success: false,
-            message: `Impersonation not prevented: stored producer_sub=${JSON.stringify(stored)}, expected 'legit-producer'`
+            message: `Impersonation not prevented: observed producerSub=${JSON.stringify(producerSub)}, expected 'legit-producer'`
         }
     }
 
-    return { success: true, message: 'Body-supplied producerSub ignored; stored sub is from validated JWT' }
+    return { success: true, message: 'Body-supplied producerSub ignored; observed sub is from validated JWT' }
 }