queen-mq 0.16.0 → 1.0.3

package/test-v2/docs.js

@@ -0,0 +1,204 @@
+/**
+ * The tests behind the published examples.
+ *
+ * Every marked region in this file is rendered on queenmq.com: the homepage,
+ * the quickstart, the concept pages and the HTTP reference pull these exact
+ * lines through webdoc/scripts/gen-snippets.mjs. They are tests written to
+ * read as documentation: real queue names, a partition key that means
+ * something, and no test scaffolding inside a marked region. Assertions stay
+ * outside the markers.
+ *
+ * After editing a marked region, regenerate the partials with
+ * `pnpm --dir webdoc gen` or the docs CI check fails on drift. The queues used
+ * here (orders, payments, invoices) are wiped by cleanupTestData in run.js
+ * before every suite run, exactly like the test-% queues.
+ */
+
+export async function docsProduceAndConsume(client) {
+    // docs:start(js-push)
+    const res = await client
+      .queue('orders')
+      .partition('customer-42')
+      .push([{ data: { orderId: 9137, amount: 99.5 } }])
+    // docs:end
+    if (res[0].status !== 'queued') {
+        return { success: false, message: `Push not queued: ${JSON.stringify(res[0])}` }
+    }
+
+    // The consume loop acks each message when the callback resolves, so when
+    // the awaited chain returns, the billing group's cursor has moved.
+    // docs:start(js-consume)
+    await client
+      .queue('orders')
+      .group('billing')
+      .subscriptionMode('all')
+      .limit(1)
+      .each()
+      .consume(async (message) => {
+        console.log(message.data)
+      })
+    // docs:end
+
+    const drained = await client
+        .queue('orders')
+        .group('billing')
+        .batch(1)
+        .wait(false)
+        .pop()
+    if (drained.length !== 0) {
+        return { success: false, message: 'Billing group cursor did not advance after consume' }
+    }
+
+    // A raw pop on another cursor still sees the message: groups are fan-out.
+    // docs:start(js-pop)
+    const messages = await client
+      .queue('orders')
+      .batch(10)
+      .wait(true)
+      .pop()
+    // docs:end
+    if (messages.length !== 1 || messages[0].data.orderId !== 9137) {
+        return { success: false, message: `Fan-out pop expected the order back, got ${JSON.stringify(messages)}` }
+    }
+
+    return { success: true, message: 'Push, consume and fan-out pop all verified' }
+}
+
+export async function docsDeduplication(client) {
+    // The fixed transactionId below survives reruns because cleanupTestData
+    // purges log_txns for these queues before the suite starts.
+    // docs:start(js-push-dedup)
+    const first = await client
+      .queue('payments')
+      .partition('customer-42')
+      .push([{ transactionId: 'order-9137-paid', data: { orderId: 9137, amount: 99.5 } }])
+
+    const retry = await client
+      .queue('payments')
+      .partition('customer-42')
+      .push([{ transactionId: 'order-9137-paid', data: { orderId: 9137, amount: 99.5 } }])
+    // retry[0].status is 'duplicate': the second push wrote nothing
+    // and answers with the first message's id.
+    // docs:end
+    if (first[0].status !== 'queued') {
+        return { success: false, message: `First push not queued: ${JSON.stringify(first[0])}` }
+    }
+    if (retry[0].status !== 'duplicate') {
+        return { success: false, message: `Retry not deduplicated: ${JSON.stringify(retry[0])}` }
+    }
+    if (retry[0].message_id !== first[0].message_id) {
+        return { success: false, message: 'Duplicate did not answer with the original message id' }
+    }
+    return { success: true, message: 'Second push with the same transactionId wrote nothing' }
+}
+
+export async function docsReplayAndSeek(client) {
+    const seeded = await client.queue('orders').partition('customer-91').push([
+        { data: { orderId: 5001, amount: 12 } },
+        { data: { orderId: 5002, amount: 24 } },
+        { data: { orderId: 5003, amount: 36 } },
+    ])
+    if (!seeded.every(r => r.status === 'queued')) {
+        return { success: false, message: 'Seed pushes failed' }
+    }
+
+    // docs:start(js-replay)
+    // A fresh group with subscriptionMode('all') reads the whole retained
+    // lane from the beginning, without touching any other group's cursor.
+    const replayed = await client
+      .queue('orders')
+      .partition('customer-91')
+      .group('audit')
+      .subscriptionMode('all')
+      .batch(100)
+      .wait(true)
+      .pop()
+    // docs:end
+    const ids = replayed.map(m => m.data.orderId)
+    if (ids.length !== 3 || ids[0] !== 5001 || ids[1] !== 5002 || ids[2] !== 5003) {
+        return { success: false, message: `Replay expected [5001,5002,5003] in order, got ${JSON.stringify(ids)}` }
+    }
+
+    // docs:start(js-seek)
+    // Move the audit group's cursor back one hour. The seek also releases
+    // any live lease, so an in-flight batch is abandoned, not acked.
+    await client.admin.seekConsumerGroup('audit', 'orders', {
+      timestamp: new Date(Date.now() - 3600 * 1000).toISOString(),
+    })
+    // docs:end
+
+    const again = await client
+        .queue('orders')
+        .partition('customer-91')
+        .group('audit')
+        .batch(100)
+        .wait(true)
+        .pop()
+    const againIds = again.map(m => m.data.orderId)
+    if (againIds.length !== 3 || againIds[0] !== 5001) {
+        return { success: false, message: `Post-seek pop expected the lane again, got ${JSON.stringify(againIds)}` }
+    }
+    return { success: true, message: 'Replay from the beginning and seek-back both verified' }
+}
+
+export async function docsTransactionalHandoff(client) {
+    const seeded = await client
+        .queue('orders')
+        .partition('customer-77')
+        .push([{ data: { orderId: 4102, amount: 18 } }])
+    if (seeded[0].status !== 'queued') {
+        return { success: false, message: 'Seed push failed' }
+    }
+
+    // docs:start(js-transaction)
+    await client
+      .queue('orders')
+      .group('invoicing')
+      .subscriptionMode('all')
+      .each()
+      .autoAck(false) // the acknowledgement belongs to the transaction, not to the loop
+      .limit(1)
+      .idleMillis(5000)
+      .consume(async (message) => {
+        // commit() throws when the broker rejects the bundle, so reaching the
+        // line after it means the ack and the push are both durable.
+        await client
+          .transaction()
+          .queue('invoices')
+          .push([{ data: { orderId: message.data.orderId, invoiced: true } }])
+          .ack(message, 'completed', { consumerGroup: 'invoicing' })
+          .commit()
+      })
+    // docs:end
+
+    // The loop takes whichever order is oldest for this group, and other docs
+    // tests push to the same queue, so identity is checked against the orders
+    // that actually exist rather than against a hardcoded id. A throwaway group
+    // reads them without disturbing the invoicing cursor.
+    const everyOrder = await client
+        .queue('orders')
+        .group(`docs-audit-${Date.now()}`)
+        .subscriptionMode('all')
+        .batch(50)
+        .partitions(20)
+        .wait(false)
+        .pop()
+    const orderIds = everyOrder.map(m => m.data.orderId)
+
+    const invoiced = await client
+        .queue('invoices')
+        .batch(1)
+        .partitions(10)
+        .wait(true)
+        .pop()
+    if (invoiced.length !== 1 || !orderIds.includes(invoiced[0].data.orderId)) {
+        return {
+            success: false,
+            message: `Invoice not committed for a real order: ${JSON.stringify(invoiced)} against ${JSON.stringify(orderIds)}`,
+        }
+    }
+    if (invoiced[0].data.invoiced !== true) {
+        return { success: false, message: 'Invoice payload corrupted' }
+    }
+    return { success: true, message: 'Ack and next-stage push committed together' }
+}

package/test-v2/http-unit/hostHeader.test.js

@@ -0,0 +1,411 @@
+/**
+ * Host-routed proxy support: selecting a tenant cluster at a queen_proxy
+ * deployment (PLAN_QUEEN_PROXY_CLOUD.md §2 -- the proxy resolves the cluster
+ * from the Host header's first DNS label, proxy/src/cache.rs
+ * slug_from_host).
+ *
+ * The defect these tests lock down: `fetch()` silently drops a caller-supplied
+ * `Host` header (WHATWG forbidden header name), so a JS user pointing at a
+ * shared address either got 421 cluster_unknown or -- worse, on a cell with a
+ * default cluster -- was routed to somebody ELSE's cluster with no signal.
+ * `hostHeader` advertises the cluster host while the socket stays pinned to the
+ * configured address, and a `headers: { Host }` is mapped onto it out loud.
+ *
+ * Same style as retry429.test.js: real node:http servers, no mocking framework.
+ * Every assertion here is about what actually arrives on the wire (req.headers.host).
+ */
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { createServer } from 'node:http'
+
+import { Queen } from '../../client-v2/index.js'
+import { HttpClient } from '../../client-v2/http/HttpClient.js'
+import { LoadBalancer } from '../../client-v2/http/LoadBalancer.js'
+
+/**
+ * Spin up a real HTTP server that records the Host header (and the rest) of
+ * every request it receives, then answers with `respond(hit)`.
+ */
+function withHostServer(run, respond = () => ({ status: 200, body: { ok: true } })) {
+  const hits = []
+  const server = createServer((req, res) => {
+    let raw = ''
+    req.on('data', chunk => { raw += chunk })
+    req.on('end', () => {
+      const hit = { method: req.method, url: req.url, host: req.headers.host, headers: req.headers }
+      hits.push(hit)
+      const descriptor = respond(hit)
+      res.writeHead(descriptor.status ?? 200, { 'Content-Type': 'application/json' })
+      res.end(JSON.stringify(descriptor.body ?? { ok: true }))
+    })
+  })
+
+  return new Promise((resolve, reject) => {
+    server.listen(0, '127.0.0.1', () => {
+      const { port } = server.address()
+      const teardown = () => new Promise(r => server.close(r))
+      Promise.resolve()
+        .then(() => run({ url: `http://127.0.0.1:${port}`, port, hits }))
+        .then(
+          async (value) => { await teardown(); resolve(value) },
+          async (err) => { await teardown(); reject(err) }
+        )
+    })
+  })
+}
+
+/** Capture console.warn for the duration of `fn` (the trap warning is not
+ *  routed through the QUEEN_CLIENT_LOG-gated logger on purpose). */
+async function captureWarnings(fn) {
+  const original = console.warn
+  const warnings = []
+  console.warn = (...args) => { warnings.push(args.join(' ')) }
+  try {
+    return { value: await fn(), warnings }
+  } finally {
+    console.warn = original
+  }
+}
+
+// ---------------------------------------------------------------------------
+// The wire: what Host actually goes out.
+// ---------------------------------------------------------------------------
+
+describe('HttpClient — hostHeader on the wire', () => {
+  it('sends the address as Host when no hostHeader is configured (unchanged default)', async () => {
+    await withHostServer(async ({ url, port, hits }) => {
+      const client = new HttpClient({ baseUrl: url })
+      try {
+        await client.get('/x')
+        assert.equal(hits.length, 1)
+        assert.equal(hits[0].host, `127.0.0.1:${port}`)
+      } finally {
+        await client.destroy()
+      }
+    })
+  })
+
+  it('sends the configured hostHeader verbatim while still connecting to baseUrl', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      const client = new HttpClient({ baseUrl: url, hostHeader: 'acme.eu1.queenmq.cloud' })
+      try {
+        const result = await client.get('/x')
+        assert.deepEqual(result, { ok: true }, 'the request must still reach the configured address')
+        assert.equal(hits[0].host, 'acme.eu1.queenmq.cloud')
+      } finally {
+        await client.destroy()
+      }
+    })
+  })
+
+  it('advertises a bare slug (what queen_proxy turns into a cluster) without appending the real port', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      const client = new HttpClient({ baseUrl: url, hostHeader: 'sdka' })
+      try {
+        await client.get('/x')
+        assert.equal(hits[0].host, 'sdka', 'slug_from_host takes the first label — no :port noise')
+      } finally {
+        await client.destroy()
+      }
+    })
+  })
+
+  it('keeps an explicit port in the advertised Host, and never resolves that name in DNS', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      // `acme.invalid:6711` cannot resolve and 6711 is not our listener: the
+      // request can only arrive if the socket is pinned to baseUrl.
+      const client = new HttpClient({ baseUrl: url, hostHeader: 'acme.invalid:6711' })
+      try {
+        await client.get('/x')
+        assert.equal(hits[0].host, 'acme.invalid:6711')
+      } finally {
+        await client.destroy()
+      }
+    })
+  })
+
+  it('applies to every verb and keeps the path/body intact', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      const client = new HttpClient({ baseUrl: url, hostHeader: 'acme' })
+      try {
+        await client.post('/api/v1/push', { items: [1] })
+        await client.delete('/api/v1/queues/q1')
+        assert.deepEqual(hits.map(h => h.host), ['acme', 'acme'])
+        assert.deepEqual(hits.map(h => h.url), ['/api/v1/push', '/api/v1/queues/q1'])
+      } finally {
+        await client.destroy()
+      }
+    })
+  })
+
+  it('shares one pinned agent across concurrent first requests', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      const client = new HttpClient({ baseUrl: url, hostHeader: 'acme' })
+      try {
+        // All five race the (async) agent creation; each must still get the
+        // advertised Host, and destroy() must be able to release every socket.
+        const results = await Promise.all(Array.from({ length: 5 }, () => client.get('/x')))
+        assert.equal(results.length, 5)
+        assert.equal(hits.length, 5)
+        for (const hit of hits) assert.equal(hit.host, 'acme')
+      } finally {
+        await client.destroy()
+      }
+    })
+  })
+
+  it('leaves other custom headers (and the bearer token) untouched', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      const client = new HttpClient({
+        baseUrl: url,
+        bearerToken: 'key-a',
+        hostHeader: 'acme',
+        headers: { 'X-Trace': 'abc' }
+      })
+      try {
+        await client.get('/x')
+        assert.equal(hits[0].host, 'acme')
+        assert.equal(hits[0].headers['x-trace'], 'abc')
+        assert.equal(hits[0].headers['authorization'], 'Bearer key-a')
+      } finally {
+        await client.destroy()
+      }
+    })
+  })
+})
+
+// ---------------------------------------------------------------------------
+// The trap: `headers: { Host }` must never be silently dropped.
+// ---------------------------------------------------------------------------
+
+describe('HttpClient — headers: { Host } is mapped, never silently ignored', () => {
+  it('maps headers.Host onto hostHeader and warns on the console once', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      const { warnings } = await captureWarnings(async () => {
+        // Distinct value per test: the warn-once dedupe is keyed by value.
+        const client = new HttpClient({ baseUrl: url, headers: { Host: 'mapped-a.example.test' } })
+        try {
+          await client.get('/x')
+        } finally {
+          await client.destroy()
+        }
+      })
+      assert.equal(hits[0].host, 'mapped-a.example.test', 'the Host the caller asked for must reach the wire')
+      assert.equal(warnings.length, 1, 'exactly one warning for the first occurrence')
+      assert.match(warnings[0], /forbidden header name/)
+      assert.match(warnings[0], /hostHeader/)
+    })
+  })
+
+  it('warns only once for a repeated identical misconfiguration', async () => {
+    await withHostServer(async ({ url }) => {
+      const { warnings } = await captureWarnings(async () => {
+        for (let i = 0; i < 3; i++) {
+          const client = new HttpClient({ baseUrl: url, headers: { Host: 'mapped-b.example.test' } })
+          await client.get('/x')
+          await client.destroy()
+        }
+      })
+      assert.equal(warnings.length, 1, 'one warning per distinct offending value, not per client')
+    })
+  })
+
+  it('accepts a lower-case `host` key too (header names are case-insensitive)', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      await captureWarnings(async () => {
+        const client = new HttpClient({ baseUrl: url, headers: { host: 'mapped-c.example.test' } })
+        try {
+          await client.get('/x')
+        } finally {
+          await client.destroy()
+        }
+      })
+      assert.equal(hits[0].host, 'mapped-c.example.test')
+    })
+  })
+
+  it('lets an explicit hostHeader win over a conflicting headers.Host, and says so', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      const { warnings } = await captureWarnings(async () => {
+        const client = new HttpClient({
+          baseUrl: url,
+          hostHeader: 'explicit-d.example.test',
+          headers: { Host: 'conflicting-d.example.test' }
+        })
+        try {
+          await client.get('/x')
+        } finally {
+          await client.destroy()
+        }
+      })
+      assert.equal(hits[0].host, 'explicit-d.example.test')
+      assert.equal(warnings.length, 1)
+      assert.match(warnings[0], /is ignored because hostHeader/)
+    })
+  })
+
+  it('does not warn when headers.Host merely repeats hostHeader', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      const { warnings } = await captureWarnings(async () => {
+        const client = new HttpClient({
+          baseUrl: url,
+          hostHeader: 'same-e.example.test',
+          headers: { Host: 'same-e.example.test' }
+        })
+        try {
+          await client.get('/x')
+        } finally {
+          await client.destroy()
+        }
+      })
+      assert.equal(hits[0].host, 'same-e.example.test')
+      assert.equal(warnings.length, 0)
+    })
+  })
+})
+
+// ---------------------------------------------------------------------------
+// Validation + failure modes: a bad or unhonorable Host must fail loudly.
+// ---------------------------------------------------------------------------
+
+describe('HttpClient — hostHeader validation', () => {
+  it('rejects a URL-shaped hostHeader at construction', () => {
+    assert.throws(
+      () => new HttpClient({ baseUrl: 'http://127.0.0.1:1', hostHeader: 'http://acme.example.test' }),
+      /bare authority/
+    )
+  })
+
+  it('rejects a hostHeader with a path or whitespace', () => {
+    assert.throws(() => new HttpClient({ baseUrl: 'http://127.0.0.1:1', hostHeader: 'acme/foo' }), /bare authority/)
+    assert.throws(() => new HttpClient({ baseUrl: 'http://127.0.0.1:1', hostHeader: 'acme host' }), /bare authority/)
+  })
+
+  it('rejects an empty hostHeader', () => {
+    assert.throws(() => new HttpClient({ baseUrl: 'http://127.0.0.1:1', hostHeader: '   ' }), /non-empty string/)
+  })
+
+  it('treats hostHeader: null as "no override" (default behaviour)', async () => {
+    await withHostServer(async ({ url, port, hits }) => {
+      const client = new HttpClient({ baseUrl: url, hostHeader: null })
+      try {
+        await client.get('/x')
+        assert.equal(hits[0].host, `127.0.0.1:${port}`)
+      } finally {
+        await client.destroy()
+      }
+    })
+  })
+
+  it('refuses to send (instead of leaking the address as Host) once the client is destroyed', async () => {
+    await withHostServer(async ({ url, hits }) => {
+      const client = new HttpClient({ baseUrl: url, hostHeader: 'acme', retryAttempts: 1 })
+      await client.get('/x')
+      await client.destroy()
+      await assert.rejects(client.get('/x'), /cannot be honored|Refusing to send/)
+      assert.equal(hits.length, 1, 'the post-destroy request must not reach the server with the wrong Host')
+    })
+  })
+})
+
+// ---------------------------------------------------------------------------
+// Load balancing: one virtual Host, several real backends.
+// ---------------------------------------------------------------------------
+
+describe('HttpClient — hostHeader with multiple backends', () => {
+  it('advertises the same Host to every backend while dialing each one', async () => {
+    await withHostServer(async ({ url: urlA, hits: hitsA }) => {
+      await withHostServer(async ({ url: urlB, hits: hitsB }) => {
+        const loadBalancer = new LoadBalancer([urlA, urlB], 'round-robin')
+        const client = new HttpClient({ loadBalancer, hostHeader: 'acme.eu1.queenmq.cloud' })
+        try {
+          for (let i = 0; i < 4; i++) await client.get('/x')
+          assert.ok(hitsA.length > 0 && hitsB.length > 0, `both backends must be dialed (${hitsA.length}/${hitsB.length})`)
+          for (const hit of [...hitsA, ...hitsB]) {
+            assert.equal(hit.host, 'acme.eu1.queenmq.cloud')
+          }
+        } finally {
+          await client.destroy()
+        }
+      })
+    })
+  })
+
+  it('keeps failover working: a 500 on one backend still lands on the other with the right Host', async () => {
+    await withHostServer(
+      async ({ url: urlA, hits: hitsA }) => {
+        await withHostServer(async ({ url: urlB, hits: hitsB }) => {
+          const loadBalancer = new LoadBalancer([urlA, urlB], 'round-robin')
+          const client = new HttpClient({ loadBalancer, hostHeader: 'acme', retryAttempts: 1 })
+          try {
+            const result = await client.get('/x')
+            assert.deepEqual(result, { ok: true })
+            assert.ok(hitsA.length > 0, 'the failing backend was tried')
+            assert.ok(hitsB.length > 0, 'and the request failed over to the healthy one')
+            for (const hit of [...hitsA, ...hitsB]) assert.equal(hit.host, 'acme')
+          } finally {
+            await client.destroy()
+          }
+        })
+      },
+      () => ({ status: 500, body: { error: 'boom' } })
+    )
+  })
+})
+
+// ---------------------------------------------------------------------------
+// Queen public API wiring.
+// ---------------------------------------------------------------------------
+
+describe('Queen — hostHeader config plumbing', () => {
+  it('push() through a Queen configured with hostHeader carries it on the wire', async () => {
+    await withHostServer(
+      async ({ url, hits }) => {
+        const queen = new Queen({ url, hostHeader: 'sdkb', bearerToken: 'key-b', handleSignals: false })
+        try {
+          const result = await queen.queue('q1').push({ hello: 'world' })
+          assert.equal(result[0].status, 'queued')
+          assert.equal(hits[0].host, 'sdkb')
+          assert.equal(hits[0].headers['authorization'], 'Bearer key-b')
+        } finally {
+          await queen.close()
+        }
+      },
+      () => ({ status: 200, body: [{ status: 'queued', transactionId: 'tx-1' }] })
+    )
+  })
+
+  it('a Queen built with headers: { Host } routes to that cluster (Go/Python parity)', async () => {
+    await withHostServer(
+      async ({ url, hits }) => {
+        const { value: result } = await captureWarnings(async () => {
+          const queen = new Queen({ url, headers: { Host: 'parity-f.example.test' }, handleSignals: false })
+          try {
+            return await queen.queue('q1').push({ hello: 'world' })
+          } finally {
+            await queen.close()
+          }
+        })
+        assert.equal(result[0].status, 'queued')
+        assert.equal(hits[0].host, 'parity-f.example.test')
+      },
+      () => ({ status: 200, body: [{ status: 'queued', transactionId: 'tx-1' }] })
+    )
+  })
+
+  it('a Queen without hostHeader is byte-for-byte unchanged (Host = the address)', async () => {
+    await withHostServer(
+      async ({ url, port, hits }) => {
+        const queen = new Queen({ url, handleSignals: false })
+        try {
+          await queen.queue('q1').push({ hello: 'world' })
+          assert.equal(hits[0].host, `127.0.0.1:${port}`)
+        } finally {
+          await queen.close()
+        }
+      },
+      () => ({ status: 200, body: [{ status: 'queued', transactionId: 'tx-1' }] })
+    )
+  })
+})