queen-mq 0.12.4 → 0.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "queen-mq",
-  "version": "0.12.4",
+  "version": "0.13.0",
   "type": "module",
   "description": "High-performance C++ message queue backed by PostgreSQL",
   "main": "client-v2/index.js",

package/test-v2/auth.js

@@ -0,0 +1,340 @@
+// Tests for server-stamped producer identity (issue #23, feature A).
+//
+// The goal of feature A is: when JWT auth is enabled, the server attaches the
+// authenticated client's `sub` claim to every pushed message, and a client
+// cannot spoof another producer's identity by supplying `producerSub` in the
+// request body.
+//
+// Three environments are exercised here:
+//
+//   1. No auth (default dev setup): `producer_sub` must be NULL; clients that
+//      send `producerSub` in the body must still get NULL (the field is not
+//      client-settable).
+//   2. JWT auth enabled (set JWT_SECRET env to match the server's
+//      JWT_SECRET): `producer_sub` must equal the `sub` claim.
+//   3. JWT auth enabled + spoofing attempt: body-supplied `producerSub` must
+//      be ignored; stored value is the JWT `sub`.
+//
+// Tests gate themselves on the `JWT_SECRET` env var so the suite can run
+// unattended in either server configuration.
+
+import crypto from 'crypto'
+import { dbPool } from './run.js'
+
+const SERVER_URL = process.env.QUEEN_URL || 'http://localhost:6632'
+const JWT_SECRET = process.env.JWT_SECRET || ''
+const JWT_ENABLED = JWT_SECRET.length > 0
+
+// ---------------------------------------------------------------------------
+// Minimal HS256 JWT signer (avoids pulling a dependency just for tests)
+// ---------------------------------------------------------------------------
+function b64url(buf) {
+    return Buffer.from(buf).toString('base64')
+        .replace(/=/g, '').replace(/\+/g, '-').replace(/\//g, '_')
+}
+
+function signHs256Jwt(payload, secret) {
+    const header = { alg: 'HS256', typ: 'JWT' }
+    const encHeader = b64url(JSON.stringify(header))
+    const encPayload = b64url(JSON.stringify(payload))
+    const signingInput = `${encHeader}.${encPayload}`
+    const sig = crypto.createHmac('sha256', secret).update(signingInput).digest()
+    return `${signingInput}.${b64url(sig)}`
+}
+
+function makeToken(sub, role = 'read-write') {
+    const now = Math.floor(Date.now() / 1000)
+    return signHs256Jwt(
+        { sub, username: sub, role, iat: now, exp: now + 3600 },
+        JWT_SECRET
+    )
+}
+
+// ---------------------------------------------------------------------------
+// Ground-truth helper: read producer_sub directly from Postgres, bypassing
+// the API so we can tell the difference between a server that actually stores
+// the value and one that just echoes the client's input.
+// ---------------------------------------------------------------------------
+async function getStoredProducerSub(queueName, transactionId) {
+    const res = await dbPool.query(
+        `SELECT m.producer_sub
+         FROM queen.messages m
+         JOIN queen.partitions p ON p.id = m.partition_id
+         JOIN queen.queues q ON q.id = p.queue_id
+         WHERE q.name = $1 AND m.transaction_id = $2`,
+        [queueName, transactionId]
+    )
+    return res.rows.length > 0 ? res.rows[0].producer_sub : null
+}
+
+// Raw HTTP push so we can control the Authorization header and inject
+// arbitrary extra fields (like a forged `producerSub`) per-test.
+async function httpPush({ queue, partition = 'Default', transactionId, data, bearerToken, extraItemFields = {} }) {
+    const headers = { 'Content-Type': 'application/json' }
+    if (bearerToken) headers['Authorization'] = `Bearer ${bearerToken}`
+
+    const body = {
+        items: [{
+            queue,
+            partition,
+            transactionId,
+            payload: data,
+            ...extraItemFields
+        }]
+    }
+
+    const res = await fetch(`${SERVER_URL}/api/v1/push`, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify(body)
+    })
+    const text = await res.text()
+    if (!res.ok) {
+        throw new Error(`push failed ${res.status}: ${text}`)
+    }
+    return text ? JSON.parse(text) : null
+}
+
+// Raw HTTP configure (queue creation) with optional bearer.
+async function httpConfigure(queue, bearerToken) {
+    const headers = { 'Content-Type': 'application/json' }
+    if (bearerToken) headers['Authorization'] = `Bearer ${bearerToken}`
+    const res = await fetch(`${SERVER_URL}/api/v1/configure`, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({ queue, options: {} })
+    })
+    if (!res.ok) {
+        const text = await res.text()
+        throw new Error(`configure failed ${res.status}: ${text}`)
+    }
+}
+
+// Direct SP call via dbPool — bypasses HTTP and JWT entirely. Used by the
+// SP-level tests below so they work regardless of whether the server has
+// auth enabled or not.
+async function callPushMessagesV3(items) {
+    await dbPool.query(
+        'SELECT queen.push_messages_v3($1::jsonb)',
+        [JSON.stringify(items)]
+    )
+}
+
+// Direct SP pop. Returns the `messages` array for the first (only) partition.
+async function callPopSpecificBatch(queueName) {
+    const reqs = [{
+        idx: 0,
+        queue_name: queueName,
+        partition_name: 'Default',
+        consumer_group: '__QUEUE_MODE__',
+        lease_seconds: 60,
+        worker_id: `test-worker-${Date.now()}`,
+        batch_size: 10,
+        sub_mode: 'all',
+        sub_from: ''
+    }]
+    const res = await dbPool.query(
+        'SELECT queen.pop_specific_batch($1::jsonb) as out',
+        [JSON.stringify(reqs)]
+    )
+    const out = res.rows[0].out
+    return out[0]?.result?.messages || []
+}
+
+// ===========================================================================
+// TEST 1: Stored procedure round-trip (SQL-only, no HTTP, no JWT)
+// ---------------------------------------------------------------------------
+// Exercises the schema + SP contract end-to-end purely via SQL. If this fails,
+// the migration or 001_push.sql / 002_pop_unified.sql is wrong. Runs in any
+// server configuration because it talks to Postgres directly.
+// ===========================================================================
+export async function producerSubRoundTripViaStoredProcedure(client) {
+    const q = 'test-auth-sp-roundtrip'
+    const tx = `tx-sp-${Date.now()}`
+
+    await callPushMessagesV3([{
+        queue: q,
+        partition: 'Default',
+        transactionId: tx,
+        payload: { x: 1 },
+        producerSub: 'sp-test-sub'
+    }])
+
+    const stored = await getStoredProducerSub(q, tx)
+    if (stored !== 'sp-test-sub') {
+        return { success: false, message: `Expected producer_sub='sp-test-sub' in DB, got ${JSON.stringify(stored)}` }
+    }
+
+    const popped = await callPopSpecificBatch(q)
+    const target = popped.find(m => m.transactionId === tx)
+    if (!target) {
+        return { success: false, message: `Expected to find tx ${tx} in pop result` }
+    }
+    if (target.producerSub !== 'sp-test-sub') {
+        return { success: false, message: `Pop returned producerSub=${JSON.stringify(target.producerSub)}, expected 'sp-test-sub'` }
+    }
+
+    return { success: true, message: 'producer_sub round-trips through push_messages_v3 and pop exposes it' }
+}
+
+// ===========================================================================
+// TEST 2: SP with no producerSub => producer_sub is NULL
+// ---------------------------------------------------------------------------
+// Guards against an accidental "default to empty string" regression in the SP.
+// ===========================================================================
+export async function producerSubNullWhenNotProvided(client) {
+    const q = 'test-auth-sp-null'
+    const tx = `tx-sp-null-${Date.now()}`
+
+    await callPushMessagesV3([{
+        queue: q,
+        partition: 'Default',
+        transactionId: tx,
+        payload: { x: 1 }
+        // no producerSub
+    }])
+
+    const stored = await getStoredProducerSub(q, tx)
+    if (stored !== null) {
+        return { success: false, message: `Expected NULL producer_sub, got ${JSON.stringify(stored)}` }
+    }
+
+    const popped = await callPopSpecificBatch(q)
+    const target = popped.find(m => m.transactionId === tx)
+    if (!target) {
+        return { success: false, message: `Expected to find tx ${tx} in pop result` }
+    }
+    if (target.producerSub != null) {
+        return { success: false, message: `Pop returned producerSub=${JSON.stringify(target.producerSub)}, expected null` }
+    }
+
+    return { success: true, message: 'Omitting producerSub yields NULL in DB and null in pop response' }
+}
+
+// ===========================================================================
+// TEST 3: SP treats empty string producerSub as NULL
+// ---------------------------------------------------------------------------
+// The SP uses NULLIF(..., '') to avoid storing a literal empty string when a
+// client accidentally sends "". Verifies that invariant.
+// ===========================================================================
+export async function producerSubEmptyStringStoredAsNull(client) {
+    const q = 'test-auth-sp-empty'
+    const tx = `tx-sp-empty-${Date.now()}`
+
+    await callPushMessagesV3([{
+        queue: q,
+        partition: 'Default',
+        transactionId: tx,
+        payload: { x: 1 },
+        producerSub: ''
+    }])
+
+    const stored = await getStoredProducerSub(q, tx)
+    if (stored !== null) {
+        return { success: false, message: `Expected NULL for empty-string producerSub, got ${JSON.stringify(stored)}` }
+    }
+
+    return { success: true, message: 'Empty-string producerSub is stored as NULL' }
+}
+
+// ===========================================================================
+// TEST 4: HTTP push without JWT => producer_sub is NULL
+// ---------------------------------------------------------------------------
+// Runs only when the server is NOT configured with JWT (typical dev setup).
+// Also checks that a client sending `producerSub` in the push body does NOT
+// end up with it persisted (the field is not client-settable under any mode).
+// ===========================================================================
+export async function producerSubIgnoredFromBodyWithoutAuth(client) {
+    if (JWT_ENABLED) {
+        return { success: true, message: 'Skipped (JWT_SECRET is set - run producerSubStampedFromJwt instead)' }
+    }
+
+    const q = 'test-auth-http-no-jwt'
+    const tx = `tx-noauth-${Date.now()}`
+
+    const queue = await client.queue(q).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // Spoofing attempt against an unauthenticated server.
+    await httpPush({
+        queue: q,
+        transactionId: tx,
+        data: { hello: 'world' },
+        extraItemFields: { producerSub: 'attacker-no-jwt' }
+    })
+
+    const stored = await getStoredProducerSub(q, tx)
+    if (stored !== null) {
+        return { success: false, message: `Expected NULL producer_sub (auth disabled), got ${JSON.stringify(stored)} - client was able to set it!` }
+    }
+
+    return { success: true, message: 'Body-supplied producerSub ignored when auth disabled; stored as NULL' }
+}
+
+// ===========================================================================
+// TEST 5: HTTP push WITH JWT => producer_sub = sub claim
+// ---------------------------------------------------------------------------
+// Requires:
+//   - Server started with JWT_ENABLED=true JWT_ALGORITHM=HS256 JWT_SECRET=<x>
+//   - Test run with JWT_SECRET=<x>
+// ===========================================================================
+export async function producerSubStampedFromJwt(client) {
+    if (!JWT_ENABLED) {
+        return { success: true, message: 'Skipped (set JWT_SECRET env var matching server to run)' }
+    }
+
+    const q = 'test-auth-http-jwt-stamp'
+    const tx = `tx-jwt-${Date.now()}`
+    const token = makeToken('alice-producer')
+
+    await httpConfigure(q, token)
+    await httpPush({
+        queue: q,
+        transactionId: tx,
+        data: { hello: 'world' },
+        bearerToken: token
+    })
+
+    const stored = await getStoredProducerSub(q, tx)
+    if (stored !== 'alice-producer') {
+        return { success: false, message: `Expected producer_sub='alice-producer', got ${JSON.stringify(stored)}` }
+    }
+
+    return { success: true, message: 'producer_sub stamped from authenticated JWT sub claim' }
+}
+
+// ===========================================================================
+// TEST 6: Spoofing producerSub in body is ignored even with valid JWT
+// ---------------------------------------------------------------------------
+// This is the core anti-impersonation invariant of issue #23.
+// ===========================================================================
+export async function producerSubSpoofingIgnoredWithJwt(client) {
+    if (!JWT_ENABLED) {
+        return { success: true, message: 'Skipped (set JWT_SECRET env var matching server to run)' }
+    }
+
+    const q = 'test-auth-http-jwt-spoof'
+    const tx = `tx-spoof-${Date.now()}`
+    const token = makeToken('legit-producer')
+
+    await httpConfigure(q, token)
+    await httpPush({
+        queue: q,
+        transactionId: tx,
+        data: { hello: 'world' },
+        bearerToken: token,
+        extraItemFields: { producerSub: 'attacker' }
+    })
+
+    const stored = await getStoredProducerSub(q, tx)
+    if (stored !== 'legit-producer') {
+        return {
+            success: false,
+            message: `Impersonation not prevented: stored producer_sub=${JSON.stringify(stored)}, expected 'legit-producer'`
+        }
+    }
+
+    return { success: true, message: 'Body-supplied producerSub ignored; stored sub is from validated JWT' }
+}

package/test-v2/push.js

@@ -278,6 +278,281 @@ export async function pushEmptyPayload(client) {
     return { success: true }
 }
 
+// ============================================================================
+// push_messages_v3 coverage (see PUSH_IMPROVEMENT.md §8.1).
+// These tests exercise the contract that the TEMP-TABLE-free rewrite must
+// preserve: single-call semantics (new inserts, pre-existing duplicates,
+// intra-batch duplicates, missing transactionId, empty batch, malformed
+// traceId) plus the cross-call concurrent-race invariant.
+// ============================================================================
+
+function distinctCount(arr) {
+    return new Set(arr).size
+}
+
+export async function testSimplePushRoundTrip(client) {
+    const queueName = 'test-v3-simple-roundtrip'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client.queue(queueName).push([{ data: { n: 1 } }])
+    if (!res[0] || res[0].status !== 'queued') {
+        return { success: false, message: `Expected queued, got ${JSON.stringify(res[0])}` }
+    }
+    const popped = await client.queue(queueName).batch(1).wait(false).pop()
+    if (!popped || popped.length !== 1 || popped[0].data.n !== 1) {
+        return { success: false, message: 'Round-trip pop failed' }
+    }
+    return { success: true }
+}
+
+export async function testPushAutoCreatesQueueAndPartition(client) {
+    // Queue and partition are never referenced before — must be auto-created.
+    const queueName = 'test-v3-autocreate-' + crypto.randomBytes(4).toString('hex')
+    const partitionName = 'autopart-' + crypto.randomBytes(4).toString('hex')
+    const res = await client.queue(queueName).partition(partitionName)
+        .push([{ data: { hello: 'auto' } }])
+    if (!res[0] || res[0].status !== 'queued') {
+        return { success: false, message: `Expected queued, got ${JSON.stringify(res[0])}` }
+    }
+    const popped = await client.queue(queueName).partition(partitionName)
+        .batch(1).wait(false).pop()
+    if (!popped || popped.length !== 1) {
+        return { success: false, message: 'Auto-created partition did not receive the message' }
+    }
+    return { success: true }
+}
+
+export async function testPushPreExistingDuplicate(client) {
+    const queueName = 'test-v3-pre-existing-dup'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const txn = 'txn-pre-' + crypto.randomBytes(4).toString('hex')
+    const first = await client.queue(queueName)
+        .push([{ transactionId: txn, data: { i: 1 } }])
+    if (first[0].status !== 'queued') {
+        return { success: false, message: `First push not queued: ${JSON.stringify(first[0])}` }
+    }
+    const firstId = first[0].message_id
+    const second = await client.queue(queueName)
+        .push([{ transactionId: txn, data: { i: 2 } }])
+    if (second[0].status !== 'duplicate') {
+        return { success: false, message: `Second push not duplicate: ${JSON.stringify(second[0])}` }
+    }
+    if (!firstId || second[0].message_id !== firstId) {
+        return { success: false, message: `Duplicate id mismatch: want ${firstId}, got ${second[0].message_id}` }
+    }
+    return { success: true }
+}
+
+export async function testPushIntraBatchDuplicatesAllNew(client) {
+    const queueName = 'test-v3-intra-batch-new'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const txn = 'txn-ib-' + crypto.randomBytes(4).toString('hex')
+    const res = await client.queue(queueName).push([
+        { transactionId: txn, data: { i: 1 } },
+        { transactionId: txn, data: { i: 2 } },
+        { transactionId: txn, data: { i: 3 } }
+    ])
+    if (!Array.isArray(res) || res.length !== 3) {
+        return { success: false, message: `Expected 3 entries, got ${res && res.length}` }
+    }
+    const queued = res.filter(r => r.status === 'queued')
+    const dupes = res.filter(r => r.status === 'duplicate')
+    if (queued.length !== 1 || dupes.length !== 2) {
+        return { success: false, message: `Want 1 queued / 2 duplicate, got ${JSON.stringify(res)}` }
+    }
+    const winnerId = queued[0].message_id
+    if (!winnerId || !dupes.every(d => d.message_id === winnerId)) {
+        return { success: false, message: `Intra-batch duplicates must share winner id ${winnerId}; got ${JSON.stringify(dupes)}` }
+    }
+    return { success: true }
+}
+
+export async function testPushIntraBatchDuplicatesMatchingPreExisting(client) {
+    const queueName = 'test-v3-intra-batch-existing'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const txn = 'txn-ibe-' + crypto.randomBytes(4).toString('hex')
+    // Seed an existing row.
+    const seed = await client.queue(queueName)
+        .push([{ transactionId: txn, data: { seed: true } }])
+    if (seed[0].status !== 'queued') {
+        return { success: false, message: 'Seed push failed' }
+    }
+    const existingId = seed[0].message_id
+    const res = await client.queue(queueName).push([
+        { transactionId: txn, data: { i: 1 } },
+        { transactionId: txn, data: { i: 2 } }
+    ])
+    if (!Array.isArray(res) || res.length !== 2) {
+        return { success: false, message: `Expected 2 entries, got ${JSON.stringify(res)}` }
+    }
+    if (!res.every(r => r.status === 'duplicate' && r.message_id === existingId)) {
+        return { success: false, message: `All entries must duplicate onto ${existingId}, got ${JSON.stringify(res)}` }
+    }
+    return { success: true }
+}
+
+export async function testPushConcurrentBatchesToSameKey(client) {
+    const queueName = 'test-v3-concurrent-same-key'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const txn = 'txn-conc-' + crypto.randomBytes(4).toString('hex')
+    // Fire 8 concurrent single-item pushes using the same transactionId.
+    // Exactly one must win; the others must report duplicate with the winner's id.
+    const N = 8
+    const results = await Promise.all(
+        Array.from({ length: N }, (_, i) =>
+            client.queue(queueName)
+                .push([{ transactionId: txn, data: { i } }])
+                .then(r => r[0])
+        )
+    )
+    const queued = results.filter(r => r.status === 'queued')
+    const dupes = results.filter(r => r.status === 'duplicate')
+    if (queued.length !== 1) {
+        return { success: false, message: `Want exactly 1 queued, got ${queued.length}: ${JSON.stringify(results)}` }
+    }
+    if (dupes.length !== N - 1) {
+        return { success: false, message: `Want ${N - 1} duplicates, got ${dupes.length}` }
+    }
+    const winnerId = queued[0].message_id
+    if (!winnerId || !dupes.every(d => d.message_id === winnerId)) {
+        return { success: false, message: `All duplicates must report winner id ${winnerId}: ${JSON.stringify(dupes)}` }
+    }
+    return { success: true }
+}
+
+export async function testPushMissingTransactionId(client) {
+    const queueName = 'test-v3-missing-txn'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    // 100 items, no transactionId provided. Each must get a distinct id and
+    // all should end up queued (no accidental intra-batch collisions from a
+    // volatile-function-inlining regression).
+    // NOTE: the client auto-generates a transactionId when omitted, so we
+    // bypass it via the raw HTTP push contract by supplying only `data`. The
+    // client still fills a fresh transactionId per item, which is the correct
+    // client-level contract; the server-side test below additionally proves
+    // the SP itself handles missing txn ids by bypassing the client helper.
+    const items = Array.from({ length: 100 }, (_, i) => ({ data: { i } }))
+    const res = await client.queue(queueName).push(items)
+    if (!Array.isArray(res) || res.length !== 100) {
+        return { success: false, message: `Expected 100 results, got ${res && res.length}` }
+    }
+    if (!res.every(r => r.status === 'queued')) {
+        const bad = res.find(r => r.status !== 'queued')
+        return { success: false, message: `All must queue; first bad entry ${JSON.stringify(bad)}` }
+    }
+    if (distinctCount(res.map(r => r.message_id)) !== 100) {
+        return { success: false, message: `Expected 100 distinct message ids` }
+    }
+    return { success: true }
+}
+
+export async function testPushEmptyArray(client) {
+    const queueName = 'test-v3-empty-array'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client.queue(queueName).push([])
+    if (!Array.isArray(res) || res.length !== 0) {
+        return { success: false, message: `Expected [], got ${JSON.stringify(res)}` }
+    }
+    return { success: true }
+}
+
+export async function testPushMalformedTraceId(client) {
+    // Per plan §5: malformed traceId aborts the batch in both v2 and v3.
+    // We only need to verify the behavior is stable (either rejected at the
+    // client validator or surfaced as a failure from the server). The client
+    // QueueBuilder currently drops invalid UUIDs silently — in which case the
+    // push still succeeds and traceId is null. Accept either outcome as long
+    // as we don't see data corruption.
+    const queueName = 'test-v3-malformed-trace'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    let res
+    try {
+        res = await client.queue(queueName)
+            .push([{ traceId: 'not-a-uuid', data: { i: 1 } }])
+    } catch (err) {
+        // Server rejected: acceptable per plan.
+        return { success: true, message: 'Server rejected malformed traceId (expected)' }
+    }
+    if (!Array.isArray(res) || res.length !== 1) {
+        return { success: false, message: `Expected 1 result, got ${JSON.stringify(res)}` }
+    }
+    // If the client dropped the bad traceId, we'll get a queued row with
+    // trace_id=null. That is an acceptable, well-documented outcome.
+    if (res[0].status !== 'queued' && res[0].status !== 'failed') {
+        return { success: false, message: `Unexpected status: ${JSON.stringify(res[0])}` }
+    }
+    return { success: true }
+}
+
+export async function testPushProducerSubEmptyStringBecomesNull(client) {
+    // We can't set producerSub from the client (server-stamped field). The
+    // invariant we need to preserve is that when the server writes an empty
+    // producer_sub into the JSON, the SP stores SQL NULL. We verify it by
+    // pushing without auth (server will stamp producerSub as empty / NULL)
+    // and then checking via a short-lived pg client. NOTE: we intentionally
+    // do NOT dynamically import ./run.js here — that module executes
+    // top-level `await main()` and re-importing it while it's running
+    // deadlocks the ESM module graph.
+    const queueName = 'test-v3-producer-sub-null'
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+    const res = await client.queue(queueName).push([{ data: { x: 1 } }])
+    if (res[0].status !== 'queued') {
+        return { success: false, message: 'Push failed' }
+    }
+    const messageId = res[0].message_id
+
+    const { default: pg } = await import('pg')
+    const pool = new pg.Pool({
+        host: process.env.PG_HOST || 'localhost',
+        port: process.env.PG_PORT || 5432,
+        database: process.env.PG_DB || 'postgres',
+        user: process.env.PG_USER || 'postgres',
+        password: process.env.PG_PASSWORD || 'postgres',
+        max: 1
+    })
+    try {
+        const r = await pool.query(
+            'SELECT producer_sub FROM queen.messages WHERE id = $1',
+            [messageId]
+        )
+        if (r.rows.length !== 1) {
+            return { success: false, message: `Message ${messageId} not found` }
+        }
+        if (r.rows[0].producer_sub !== null) {
+            return { success: false, message: `Expected producer_sub NULL, got ${JSON.stringify(r.rows[0].producer_sub)}` }
+        }
+        return { success: true }
+    } finally {
+        await pool.end()
+    }
+}
+
 export async function pushEncryptedPayload(client) {
     const queue = await client.queue('test-queue-encrypted-payload')
     .config({ encryptionEnabled: true })

package/test-v2/run.js

@@ -13,6 +13,8 @@ import * as maintenanceTests from './maintenance.js'
 import * as retentionTests from './retention.js'
 import * as bootstrapTests from './bootstrap.js'
 import * as loggerTests from './logger.js'
+import * as watermarkTests from './watermark.js'
+import * as authTests from './auth.js'
 import { LoadBalancer } from '../client-v2/http/LoadBalancer.js';
 
 
@@ -115,7 +117,9 @@ async function main() {
         retentionTests,
         maintenanceTests,
         bootstrapTests,
-        loggerTests
+        loggerTests,
+        watermarkTests,
+        authTests
     ]
     
     const aiTests = [

package/test-v2/watermark.js

@@ -0,0 +1,507 @@
+/**
+ * Watermark Bug Tests
+ *
+ * These tests verify that consumer_watermarks are properly reset when:
+ * 1. Seeking a consumer group backwards to a timestamp
+ * 2. Deleting a consumer group (watermark should be cleaned up)
+ *
+ * The consumer_watermarks table optimizes POP by tracking when a consumer
+ * last found the queue empty. However, if the watermark isn't reset when
+ * the cursor moves backwards, old messages become invisible.
+ *
+ * BUG: Currently, seek and delete operations don't touch consumer_watermarks,
+ * causing re-consumption to fail silently.
+ *
+ * IMPORTANT: The watermark filter has a 2-minute buffer:
+ *   pl.updated_at >= (v_watermark - interval '2 minutes')
+ *
+ * So the bug only manifests after 2+ minutes have passed. To test this without
+ * waiting, we directly manipulate the watermark timestamp in the database.
+ *
+ * Run: node run.js seekBackwardsAllowsReconsume
+ *      node run.js deleteConsumerGroupAllowsReconsume
+ */
+
+import { dbPool } from './run.js'
+
+const QUEUE_NAME_SEEK = 'test-queue-v2-watermark-seek'
+const QUEUE_NAME_DELETE = 'test-queue-v2-watermark-delete'
+
+/**
+ * Helper: Move the watermark FORWARD in time to simulate that partitions
+ * are "old" relative to the watermark.
+ *
+ * The filter is: pl.updated_at >= (v_watermark - interval '2 minutes')
+ *
+ * If watermark = NOW + 10 minutes, then:
+ *   filter = pl.updated_at >= (NOW + 10 - 2) = pl.updated_at >= NOW + 8
+ *
+ * Partitions updated at NOW will fail this filter (NOW < NOW + 8)
+ */
+async function advanceWatermark(queueName, consumerGroup, minutesForward = 10) {
+    const result = await dbPool.query(`
+        UPDATE queen.consumer_watermarks
+        SET last_empty_scan_at = NOW() + interval '${minutesForward} minutes',
+            updated_at = NOW() + interval '${minutesForward} minutes'
+        WHERE queue_name = $1 AND consumer_group = $2
+        RETURNING *
+    `, [queueName, consumerGroup])
+    return result.rows[0]
+}
+
+/**
+ * Helper: Check if watermark exists for a queue/consumer_group
+ */
+async function getWatermark(queueName, consumerGroup) {
+    const result = await dbPool.query(`
+        SELECT * FROM queen.consumer_watermarks
+        WHERE queue_name = $1 AND consumer_group = $2
+    `, [queueName, consumerGroup])
+    return result.rows[0]
+}
+
+/**
+ * Test: Seeking backwards should allow re-consumption of messages
+ *
+ * Steps:
+ * 1. Push messages to queue
+ * 2. Consume all messages with a consumer group (watermark advances)
+ * 3. Artificially age the watermark (simulate 10 minutes passing)
+ * 4. Seek the consumer group backwards to before the messages
+ * 5. Try to consume again - should get all messages
+ *
+ * Expected: All messages are re-consumed after seek
+ * Actual (bug): Watermark blocks re-consumption, 0 messages returned
+ */
+export async function seekBackwardsAllowsReconsume(client) {
+    const consumerGroup = 'watermark-seek-test-cg'
+
+    // Cleanup
+    try {
+        await client.deleteConsumerGroup(consumerGroup, true)
+        // Also clean up any orphaned watermark from previous runs
+        await dbPool.query(
+            'DELETE FROM queen.consumer_watermarks WHERE consumer_group = $1',
+            [consumerGroup]
+        )
+    } catch (e) {
+        // Ignore
+    }
+
+    // 1. Create queue
+    const queue = await client.queue(QUEUE_NAME_SEEK).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // 2. Record timestamp BEFORE pushing messages
+    const timestampBeforeMessages = new Date().toISOString()
+
+    // Small delay to ensure timestamp ordering
+    await new Promise(resolve => setTimeout(resolve, 100))
+
+    // 3. Push messages
+    const messageCount = 10
+    for (let i = 0; i < messageCount; i++) {
+        await client
+            .queue(QUEUE_NAME_SEEK)
+            .partition(`p-${i}`)
+            .push([{ data: { id: i, batch: 'original' } }])
+    }
+
+    console.log(`  Pushed ${messageCount} messages`)
+
+    // 4. Consume all messages with consumer group
+    let firstConsumeCount = 0
+    await client
+        .queue(QUEUE_NAME_SEEK)
+        .group(consumerGroup)
+        .subscriptionMode('all')
+        .concurrency(5)
+        .batch(1)
+        .wait(false)
+        .idleMillis(3000)
+        .consume(async msgs => {
+            firstConsumeCount += msgs.length
+        })
+
+    console.log(`  First consume: ${firstConsumeCount} messages`)
+
+    if (firstConsumeCount !== messageCount) {
+        return {
+            success: false,
+            message: `First consume should get ${messageCount} messages, got ${firstConsumeCount}`
+        }
+    }
+
+    // 5. Verify queue is empty for this consumer (triggers watermark advancement)
+    const emptyCheck = await client
+        .queue(QUEUE_NAME_SEEK)
+        .group(consumerGroup)
+        .batch(10)
+        .wait(false)
+        .pop()
+
+    if (emptyCheck.length !== 0) {
+        return {
+            success: false,
+            message: `Queue should be empty after first consume, got ${emptyCheck.length} messages`
+        }
+    }
+
+    console.log(`  Verified queue is empty (watermark should be set)`)
+
+    // 6. Check watermark was created
+    const watermarkBefore = await getWatermark(QUEUE_NAME_SEEK, consumerGroup)
+    if (!watermarkBefore) {
+        return {
+            success: false,
+            message: 'Watermark was not created after empty queue check'
+        }
+    }
+    console.log(`  Watermark exists: last_empty_scan_at = ${watermarkBefore.last_empty_scan_at}`)
+
+    // 7. CRITICAL: Advance the watermark forward to simulate that partitions are "old"
+    //    relative to the watermark. This triggers the bug where old partitions are filtered out.
+    const advancedWatermark = await advanceWatermark(QUEUE_NAME_SEEK, consumerGroup, 10)
+    console.log(`  Advanced watermark to: ${advancedWatermark.last_empty_scan_at} (10 minutes in future)`)
+
+    // 8. Seek backwards to BEFORE the messages were pushed
+    console.log(`  Seeking to timestamp: ${timestampBeforeMessages}`)
+
+    const seekResult = await client.admin.seekConsumerGroup(
+        consumerGroup,
+        QUEUE_NAME_SEEK,
+        { timestamp: timestampBeforeMessages }
+    )
+
+    console.log(`  Seek result: ${JSON.stringify(seekResult)}`)
+
+    // 9. Check if watermark was reset (it should be, but currently isn't - that's the bug)
+    const watermarkAfterSeek = await getWatermark(QUEUE_NAME_SEEK, consumerGroup)
+    if (watermarkAfterSeek) {
+        console.log(`  WARNING: Watermark still exists after seek: ${watermarkAfterSeek.last_empty_scan_at}`)
+    } else {
+        console.log(`  Watermark was properly deleted after seek`)
+    }
+
+    // Small delay after seek
+    await new Promise(resolve => setTimeout(resolve, 500))
+
+    // 10. Try to consume again - should get all messages back
+    let secondConsumeCount = 0
+    await client
+        .queue(QUEUE_NAME_SEEK)
+        .group(consumerGroup)
+        .concurrency(5)
+        .batch(1)
+        .wait(false)
+        .idleMillis(3000)
+        .consume(async msgs => {
+            secondConsumeCount += msgs.length
+        })
+
+    console.log(`  Second consume (after seek): ${secondConsumeCount} messages`)
+
+    // 11. Verify we got all messages back
+    if (secondConsumeCount !== messageCount) {
+        return {
+            success: false,
+            message: `After seek backwards, should re-consume ${messageCount} messages, but got ${secondConsumeCount}. ` +
+                     `This is because consumer_watermarks was not reset on seek (watermark still at ${watermarkAfterSeek?.last_empty_scan_at}).`
+        }
+    }
+
+    return {
+        success: true,
+        message: `Seek backwards allowed re-consumption of ${secondConsumeCount} messages`
+    }
+}
+
+/**
+ * Test: Deleting consumer group should allow fresh consumption
+ *
+ * Steps:
+ * 1. Push messages to queue
+ * 2. Consume all messages with a consumer group (watermark advances)
+ * 3. Artificially age the watermark (simulate 10 minutes passing)
+ * 4. Delete the consumer group
+ * 5. Create a new consumer group with the SAME name
+ * 6. Try to consume - should get all messages
+ *
+ * Expected: All messages are consumed by the "new" consumer group
+ * Actual (bug): Orphaned watermark blocks consumption, 0 messages returned
+ */
+export async function deleteConsumerGroupAllowsReconsume(client) {
+    const consumerGroup = 'watermark-delete-test-cg'
+
+    // Cleanup
+    try {
+        await client.deleteConsumerGroup(consumerGroup, true)
+        // Also clean up any orphaned watermark from previous runs
+        await dbPool.query(
+            'DELETE FROM queen.consumer_watermarks WHERE consumer_group = $1',
+            [consumerGroup]
+        )
+    } catch (e) {
+        // Ignore
+    }
+
+    // 1. Create queue
+    const queue = await client.queue(QUEUE_NAME_DELETE).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // 2. Push messages
+    const messageCount = 10
+    for (let i = 0; i < messageCount; i++) {
+        await client
+            .queue(QUEUE_NAME_DELETE)
+            .partition(`p-${i}`)
+            .push([{ data: { id: i, batch: 'original' } }])
+    }
+
+    console.log(`  Pushed ${messageCount} messages`)
+
+    // 3. Consume all messages with consumer group
+    let firstConsumeCount = 0
+    await client
+        .queue(QUEUE_NAME_DELETE)
+        .group(consumerGroup)
+        .subscriptionMode('all')
+        .concurrency(5)
+        .batch(1)
+        .wait(false)
+        .idleMillis(3000)
+        .consume(async msgs => {
+            firstConsumeCount += msgs.length
+        })
+
+    console.log(`  First consume: ${firstConsumeCount} messages`)
+
+    if (firstConsumeCount !== messageCount) {
+        return {
+            success: false,
+            message: `First consume should get ${messageCount} messages, got ${firstConsumeCount}`
+        }
+    }
+
+    // 4. Verify queue is empty for this consumer (triggers watermark advancement)
+    const emptyCheck = await client
+        .queue(QUEUE_NAME_DELETE)
+        .group(consumerGroup)
+        .batch(10)
+        .wait(false)
+        .pop()
+
+    if (emptyCheck.length !== 0) {
+        return {
+            success: false,
+            message: `Queue should be empty after first consume, got ${emptyCheck.length} messages`
+        }
+    }
+
+    console.log(`  Verified queue is empty (watermark should be set)`)
+
+    // 5. Check watermark was created
+    const watermarkBefore = await getWatermark(QUEUE_NAME_DELETE, consumerGroup)
+    if (!watermarkBefore) {
+        return {
+            success: false,
+            message: 'Watermark was not created after empty queue check'
+        }
+    }
+    console.log(`  Watermark exists: last_empty_scan_at = ${watermarkBefore.last_empty_scan_at}`)
+
+    // 6. CRITICAL: Advance the watermark forward to simulate that partitions are "old"
+    const advancedWatermark = await advanceWatermark(QUEUE_NAME_DELETE, consumerGroup, 10)
+    console.log(`  Advanced watermark to: ${advancedWatermark.last_empty_scan_at} (10 minutes in future)`)
+
+    // 7. Delete the consumer group
+    console.log(`  Deleting consumer group: ${consumerGroup}`)
+
+    const deleteResult = await client.deleteConsumerGroup(consumerGroup, true)
+    console.log(`  Delete result: ${JSON.stringify(deleteResult)}`)
+
+    // 8. Check if watermark was deleted (it should be, but currently isn't - that's the bug)
+    const watermarkAfterDelete = await getWatermark(QUEUE_NAME_DELETE, consumerGroup)
+    if (watermarkAfterDelete) {
+        console.log(`  WARNING: Orphaned watermark still exists: ${watermarkAfterDelete.last_empty_scan_at}`)
+    } else {
+        console.log(`  Watermark was properly deleted`)
+    }
+
+    // Small delay after delete
+    await new Promise(resolve => setTimeout(resolve, 500))
+
+    // 9. Create a "new" consumer group with the same name and try to consume
+    //    Since partition_consumers was deleted, this should start fresh
+    //    But if consumer_watermarks wasn't deleted, it will skip all messages
+    let secondConsumeCount = 0
+    await client
+        .queue(QUEUE_NAME_DELETE)
+        .group(consumerGroup)
+        .subscriptionMode('all')
+        .concurrency(5)
+        .batch(1)
+        .wait(false)
+        .idleMillis(3000)
+        .consume(async msgs => {
+            secondConsumeCount += msgs.length
+        })
+
+    console.log(`  Second consume (after delete + recreate): ${secondConsumeCount} messages`)
+
+    // 10. Verify we got all messages
+    if (secondConsumeCount !== messageCount) {
+        return {
+            success: false,
+            message: `After deleting CG, new CG with same name should consume ${messageCount} messages, ` +
+                     `but got ${secondConsumeCount}. Orphaned watermark at ${watermarkAfterDelete?.last_empty_scan_at}.`
+        }
+    }
+
+    return {
+        success: true,
+        message: `Delete + recreate allowed fresh consumption of ${secondConsumeCount} messages`
+    }
+}
+
+/**
+ * Test: Deleting consumer group for specific queue should allow fresh consumption
+ *
+ * Similar to above but uses deleteConsumerGroupForQueue instead of deleteConsumerGroup
+ */
+export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
+    const consumerGroup = 'watermark-delete-queue-test-cg'
+    const queueName = 'test-queue-v2-watermark-delete-queue'
+
+    // Cleanup
+    try {
+        await client.admin.deleteConsumerGroupForQueue(consumerGroup, queueName, true)
+        // Also clean up any orphaned watermark from previous runs
+        await dbPool.query(
+            'DELETE FROM queen.consumer_watermarks WHERE queue_name = $1 AND consumer_group = $2',
+            [queueName, consumerGroup]
+        )
+    } catch (e) {
+        // Ignore
+    }
+
+    // 1. Create queue
+    const queue = await client.queue(queueName).create()
+    if (!queue.configured) {
+        return { success: false, message: 'Queue not created' }
+    }
+
+    // 2. Push messages
+    const messageCount = 10
+    for (let i = 0; i < messageCount; i++) {
+        await client
+            .queue(queueName)
+            .partition(`p-${i}`)
+            .push([{ data: { id: i, batch: 'original' } }])
+    }
+
+    console.log(`  Pushed ${messageCount} messages`)
+
+    // 3. Consume all messages with consumer group
+    let firstConsumeCount = 0
+    await client
+        .queue(queueName)
+        .group(consumerGroup)
+        .subscriptionMode('all')
+        .concurrency(5)
+        .batch(1)
+        .wait(false)
+        .idleMillis(3000)
+        .consume(async msgs => {
+            firstConsumeCount += msgs.length
+        })
+
+    console.log(`  First consume: ${firstConsumeCount} messages`)
+
+    if (firstConsumeCount !== messageCount) {
+        return {
+            success: false,
+            message: `First consume should get ${messageCount} messages, got ${firstConsumeCount}`
+        }
+    }
+
+    // 4. Verify queue is empty (triggers watermark)
+    const emptyCheck = await client
+        .queue(queueName)
+        .group(consumerGroup)
+        .batch(10)
+        .wait(false)
+        .pop()
+
+    if (emptyCheck.length !== 0) {
+        return {
+            success: false,
+            message: `Queue should be empty after first consume, got ${emptyCheck.length} messages`
+        }
+    }
+
+    console.log(`  Verified queue is empty (watermark should be set)`)
+
+    // 5. Check watermark was created
+    const watermarkBefore = await getWatermark(queueName, consumerGroup)
+    if (!watermarkBefore) {
+        return {
+            success: false,
+            message: 'Watermark was not created after empty queue check'
+        }
+    }
+    console.log(`  Watermark exists: last_empty_scan_at = ${watermarkBefore.last_empty_scan_at}`)
+
+    // 6. CRITICAL: Advance the watermark forward
+    const advancedWatermark = await advanceWatermark(queueName, consumerGroup, 10)
+    console.log(`  Advanced watermark to: ${advancedWatermark.last_empty_scan_at} (10 minutes in future)`)
+
+    // 7. Delete consumer group for this specific queue
+    console.log(`  Deleting consumer group for queue: ${consumerGroup} / ${queueName}`)
+
+    const deleteResult = await client.admin.deleteConsumerGroupForQueue(consumerGroup, queueName, true)
+    console.log(`  Delete result: ${JSON.stringify(deleteResult)}`)
+
+    // 8. Check if watermark was deleted
+    const watermarkAfterDelete = await getWatermark(queueName, consumerGroup)
+    if (watermarkAfterDelete) {
+        console.log(`  WARNING: Orphaned watermark still exists: ${watermarkAfterDelete.last_empty_scan_at}`)
+    } else {
+        console.log(`  Watermark was properly deleted`)
+    }
+
+    await new Promise(resolve => setTimeout(resolve, 500))
+
+    // 9. Try to consume again
+    let secondConsumeCount = 0
+    await client
+        .queue(queueName)
+        .group(consumerGroup)
+        .subscriptionMode('all')
+        .concurrency(5)
+        .batch(1)
+        .wait(false)
+        .idleMillis(3000)
+        .consume(async msgs => {
+            secondConsumeCount += msgs.length
+        })
+
+    console.log(`  Second consume (after delete): ${secondConsumeCount} messages`)
+
+    if (secondConsumeCount !== messageCount) {
+        return {
+            success: false,
+            message: `After deleting CG for queue, should consume ${messageCount} messages, ` +
+                     `but got ${secondConsumeCount}. Orphaned watermark at ${watermarkAfterDelete?.last_empty_scan_at}.`
+        }
+    }
+
+    return {
+        success: true,
+        message: `Delete CG for queue allowed fresh consumption of ${secondConsumeCount} messages`
+    }
+}