queen-mq 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -0
- package/client-v2/Queen.js +54 -0
- package/client-v2/README.md +32 -0
- package/client-v2/builders/QueueBuilder.js +8 -16
- package/client-v2/builders/TimerBuilder.js +262 -0
- package/client-v2/builders/TransactionBuilder.js +185 -8
- package/client-v2/kv/Kv.js +432 -0
- package/client-v2/kv/expiry.js +148 -0
- package/package.json +11 -3
- package/test-v2/_kvtimers.js +71 -0
- package/test-v2/ackwindow.js +10 -3
- package/test-v2/consume.js +0 -2
- package/test-v2/docs.js +204 -0
- package/test-v2/http-unit/retry429.test.js +6 -1
- package/test-v2/kv-unit/_planServer.js +65 -0
- package/test-v2/kv-unit/kvWire.test.js +377 -0
- package/test-v2/kv-unit/timerWire.test.js +177 -0
- package/test-v2/kv-unit/txnWire.test.js +222 -0
- package/test-v2/kv.js +273 -0
- package/test-v2/pop.js +0 -2
- package/test-v2/push.js +0 -4
- package/test-v2/run.js +34 -4
- package/test-v2/semantics.js +16 -7
- package/test-v2/stream/_helpers.js +7 -0
- package/test-v2/stream/combined.js +4 -3
- package/test-v2/stream/cron.js +1 -1
- package/test-v2/stream/eventTime.js +8 -5
- package/test-v2/stream/operators.js +5 -5
- package/test-v2/stream/recovery.js +4 -1
- package/test-v2/stream/session.js +3 -3
- package/test-v2/stream/sliding.js +1 -1
- package/test-v2/stream/throughput.js +3 -2
- package/test-v2/stream/tumbling.js +6 -6
- package/test-v2/timers.js +209 -0
- package/test-v2/transaction.js +4 -3
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Expiry sugar for the KV surface, and the ONE duration parser this SDK has.
|
|
3
|
+
*
|
|
4
|
+
* PLAN_KV_TIMERS.md §20.1 ratified `ttlSeconds` as the wire's unit: the server
|
|
5
|
+
* speaks seconds everywhere (`dedup_window_seconds`, `lease_seconds`,
|
|
6
|
+
* `retention_seconds`), so the product keeps ONE duration convention and the
|
|
7
|
+
* SDKs do the converting. `ttlMillis` does not exist in any client and must not
|
|
8
|
+
* be added -- it would reintroduce, through the service entrance, the double
|
|
9
|
+
* convention that decision removed.
|
|
10
|
+
*
|
|
11
|
+
* §20.6 is the other half of the rule, and it is what this file's parser
|
|
12
|
+
* serves twice: DURATIONS THAT CAN BE SUB-SECOND ARE IN MILLISECONDS, THE ONES
|
|
13
|
+
* THAT CANNOT ARE IN SECONDS. A 250 ms retry backoff is a real and central use
|
|
14
|
+
* of timers, so their wire is `delayMs`; a sub-second TTL is not a real use for
|
|
15
|
+
* anybody, so the KV wire is `ttlSeconds`. The parser below returns
|
|
16
|
+
* MILLISECONDS -- the finer of the two -- and each caller converts.
|
|
17
|
+
*
|
|
18
|
+
* NOT EXPORTED FROM THE BARREL, deliberately (§10.4). The client's stated
|
|
19
|
+
* convention is "a number with the unit in its name", and this is its first
|
|
20
|
+
* string duration. Confining the parser to this file is what keeps it from
|
|
21
|
+
* becoming a general-purpose utility that then has to accept every format
|
|
22
|
+
* anybody ever wrote.
|
|
23
|
+
*
|
|
24
|
+
* ROUNDING IS ALWAYS UP. A TTL rounded down can expire a marker BEFORE the
|
|
25
|
+
* window it was supposed to cover, which is the failure this feature exists to
|
|
26
|
+
* prevent; rounding up costs at most a second of retention.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
const UNIT_MS = {
|
|
30
|
+
ms: 1,
|
|
31
|
+
s: 1000,
|
|
32
|
+
m: 60 * 1000,
|
|
33
|
+
h: 60 * 60 * 1000,
|
|
34
|
+
d: 24 * 60 * 60 * 1000
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// One or more <number><unit> pairs: '30s', '1h30m', '250ms'. Anchored, so a
|
|
38
|
+
// typo is an error and never a silent partial parse ('1hour' does not become
|
|
39
|
+
// one hour).
|
|
40
|
+
const DURATION_RE = /^(\d+(?:\.\d+)?)(ms|s|m|h|d)$/
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Parse a duration STRING into milliseconds. Numbers are refused on purpose:
|
|
44
|
+
* `{ ttl: 5000 }` cannot be read as seconds or milliseconds without guessing,
|
|
45
|
+
* and guessing here would be a factor-of-1000 bug in somebody's retention.
|
|
46
|
+
* The numeric spellings are the ones with the unit in the name --
|
|
47
|
+
* `ttlSeconds` and `delayMs`.
|
|
48
|
+
*/
|
|
49
|
+
export function parseDurationMs(value) {
|
|
50
|
+
if (typeof value !== 'string' || value.trim() === '') {
|
|
51
|
+
throw new Error(
|
|
52
|
+
`duration must be a string with a unit, e.g. '250ms', '30s', '15m', '24h', '7d' — got ${JSON.stringify(value)}. ` +
|
|
53
|
+
'For a plain number use ttlSeconds (KV) or delayMs (timers), which carry their unit in the name.'
|
|
54
|
+
)
|
|
55
|
+
}
|
|
56
|
+
const parts = value.trim().toLowerCase().match(/\d+(?:\.\d+)?[a-z]+/g)
|
|
57
|
+
const joined = parts ? parts.join('') : ''
|
|
58
|
+
if (!parts || joined !== value.trim().toLowerCase()) {
|
|
59
|
+
throw new Error(`invalid duration '${value}': expected <number><unit> parts, e.g. '30s' or '1h30m'`)
|
|
60
|
+
}
|
|
61
|
+
let ms = 0
|
|
62
|
+
for (const part of parts) {
|
|
63
|
+
const m = DURATION_RE.exec(part)
|
|
64
|
+
if (!m) {
|
|
65
|
+
throw new Error(`invalid duration '${value}': unknown unit in '${part}' (use ms, s, m, h, d)`)
|
|
66
|
+
}
|
|
67
|
+
ms += Number(m[1]) * UNIT_MS[m[2]]
|
|
68
|
+
}
|
|
69
|
+
if (!(ms > 0)) {
|
|
70
|
+
throw new Error(`invalid duration '${value}': must be greater than zero`)
|
|
71
|
+
}
|
|
72
|
+
return ms
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Milliseconds since the epoch for a Date, an ISO string or an epoch number. */
|
|
76
|
+
function instantMs(value) {
|
|
77
|
+
if (value instanceof Date) {
|
|
78
|
+
const t = value.getTime()
|
|
79
|
+
if (Number.isNaN(t)) throw new Error('until: invalid Date')
|
|
80
|
+
return t
|
|
81
|
+
}
|
|
82
|
+
if (typeof value === 'number' && Number.isFinite(value)) return value
|
|
83
|
+
if (typeof value === 'string') {
|
|
84
|
+
const t = Date.parse(value)
|
|
85
|
+
if (Number.isNaN(t)) throw new Error(`until: '${value}' is not a parseable date`)
|
|
86
|
+
return t
|
|
87
|
+
}
|
|
88
|
+
throw new Error(`until must be a Date, an ISO 8601 string or an epoch-milliseconds number — got ${typeof value}`)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Resolve the expiry fields of one KV write into the CANONICAL wire shape.
|
|
93
|
+
*
|
|
94
|
+
* Returns `{ ttlSeconds }`, `{ forever: true }`, or `{}`.
|
|
95
|
+
*
|
|
96
|
+
* The empty answer is deliberate and is the reason this function validates so
|
|
97
|
+
* little: §5.1's rule -- exactly one of `ttlSeconds` and `forever:true`, zero
|
|
98
|
+
* or two being the same error -- lives in `kv_apply_v1`, so that all seven
|
|
99
|
+
* clients AND the embedded broker (which never passes through an HTTP handler)
|
|
100
|
+
* inherit it without a line of their own. Re-implementing it here would give
|
|
101
|
+
* the product two places that can disagree about when a key dies. What IS
|
|
102
|
+
* checked here is only what the server cannot see, because it is sugar that
|
|
103
|
+
* never reaches it: two spellings of the same field, a `forever` that is not
|
|
104
|
+
* `true`, and an `until` already in the past.
|
|
105
|
+
*
|
|
106
|
+
* `nowMs` is a parameter so the conversion happens at SEND time. In a
|
|
107
|
+
* transaction the ops are built when the caller writes them and sent at
|
|
108
|
+
* `commit()`, and an `until` frozen at build time would ship a TTL that is
|
|
109
|
+
* already stale by however long the bundle took to assemble.
|
|
110
|
+
*/
|
|
111
|
+
export function resolveExpiry(opts = {}, nowMs = Date.now()) {
|
|
112
|
+
const spellings = ['ttlSeconds', 'ttl', 'until'].filter(k => opts[k] !== undefined)
|
|
113
|
+
if (spellings.length > 1) {
|
|
114
|
+
throw new Error(
|
|
115
|
+
`expiry declared twice (${spellings.join(' and ')}): ttlSeconds is the canonical field, ` +
|
|
116
|
+
'`ttl` is a duration string and `until` is an instant — pick one'
|
|
117
|
+
)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const out = {}
|
|
121
|
+
|
|
122
|
+
if (opts.forever !== undefined) {
|
|
123
|
+
if (opts.forever !== true) {
|
|
124
|
+
throw new Error('forever must be exactly true; there is no forever:false — omit it and declare a TTL')
|
|
125
|
+
}
|
|
126
|
+
out.forever = true
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (opts.ttlSeconds !== undefined) {
|
|
130
|
+
out.ttlSeconds = opts.ttlSeconds
|
|
131
|
+
} else if (opts.ttl !== undefined) {
|
|
132
|
+
out.ttlSeconds = Math.ceil(parseDurationMs(opts.ttl) / 1000)
|
|
133
|
+
} else if (opts.until !== undefined) {
|
|
134
|
+
const deltaMs = instantMs(opts.until) - nowMs
|
|
135
|
+
const seconds = Math.ceil(deltaMs / 1000)
|
|
136
|
+
if (seconds <= 0) {
|
|
137
|
+
throw new Error(
|
|
138
|
+
`until is already in the past (${Math.round(deltaMs)}ms from now): a key written with a dead ` +
|
|
139
|
+
'TTL would be refused by the broker, and a key written with none would be immortal'
|
|
140
|
+
)
|
|
141
|
+
}
|
|
142
|
+
out.ttlSeconds = seconds
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Both declared: forwarded as-is on purpose, so the broker gives its own
|
|
146
|
+
// `kv_expiry_not_specified` verdict and there is one rule, in one place.
|
|
147
|
+
return out
|
|
148
|
+
}
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "queen-mq",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Partitioned message queue on PostgreSQL — broker client + fluent streaming SDK (windows, joins, gates) in one package",
|
|
6
6
|
"main": "client-v2/index.js",
|
|
7
7
|
"scripts": {
|
|
8
|
-
"test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js && node test-v2/run.js human",
|
|
9
|
-
"test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js",
|
|
8
|
+
"test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js && node test-v2/run.js human",
|
|
9
|
+
"test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js",
|
|
10
10
|
"test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
|
|
11
11
|
"test:integration": "node test-v2/run.js human",
|
|
12
12
|
"test:streams": "node test-v2/run.js stream",
|
|
@@ -28,16 +28,24 @@
|
|
|
28
28
|
},
|
|
29
29
|
"author": "Smartness",
|
|
30
30
|
"license": "Apache-2.0",
|
|
31
|
+
"homepage": "https://queenmq.com",
|
|
31
32
|
"repository": {
|
|
32
33
|
"type": "git",
|
|
33
34
|
"url": "https://github.com/queen-mq/queen"
|
|
34
35
|
},
|
|
36
|
+
"bugs": {
|
|
37
|
+
"url": "https://github.com/queen-mq/queen/issues"
|
|
38
|
+
},
|
|
35
39
|
"keywords": [
|
|
36
40
|
"message-queue",
|
|
37
41
|
"queue",
|
|
42
|
+
"postgres",
|
|
43
|
+
"postgresql",
|
|
38
44
|
"broker",
|
|
39
45
|
"message-queue-system",
|
|
40
46
|
"fifo",
|
|
47
|
+
"consumer-groups",
|
|
48
|
+
"dead-letter-queue",
|
|
41
49
|
"streaming",
|
|
42
50
|
"stream-processing",
|
|
43
51
|
"rate-limiter",
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared helpers for the kv and timers integration suites.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately NOT registered in run.js: that runner treats every export of a
|
|
5
|
+
* registered module as a test function, so a helper exported from kv.js would
|
|
6
|
+
* be executed as a test and reported as one.
|
|
7
|
+
*
|
|
8
|
+
* WHY THERE IS NO AVAILABILITY PROBE HERE ANY MORE. There used to be one, plus
|
|
9
|
+
* a `skipped()` verdict, because `QUEEN_KV_ENABLED` and `QUEEN_TIMERS_ENABLED`
|
|
10
|
+
* were boot flags that defaulted to false: with them off the routes were not
|
|
11
|
+
* registered and every call here answered 404, so a red suite was the DEFAULT
|
|
12
|
+
* configuration's own fault and the skip existed to stop that training everyone
|
|
13
|
+
* to ignore the colour.
|
|
14
|
+
*
|
|
15
|
+
* Those flags are gone. Kv and timers are not features, they are the broker:
|
|
16
|
+
* there is no `QUEEN_PUSH_ENABLED` either. Every cell that runs this binary has
|
|
17
|
+
* both surfaces, so these suites RUN, and a 404 from any of them is a bug in
|
|
18
|
+
* the broker, not a configuration to detect. A test that skips is a test that
|
|
19
|
+
* says nothing, and that was only tolerable while the 404 was legitimate.
|
|
20
|
+
*
|
|
21
|
+
* What still exists is the operator's RUNTIME kill switch (`kv_enabled`,
|
|
22
|
+
* `timers_schedule_enabled`, `timers_fire_enabled` in `queen.system_state`) --
|
|
23
|
+
* the maintenance-mode lever, pulled live during an incident. It answers 503
|
|
24
|
+
* with `Retry-After` on the kv/timer routes and 403 on a `kv`/`timers` rider
|
|
25
|
+
* inside a transaction, never 404. If a run here ever goes red against that,
|
|
26
|
+
* the switch is down on the rig and somebody pulled it; it is not something
|
|
27
|
+
* these tests should paper over.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
export const KV_NS = 'test-kv'
|
|
31
|
+
export const TIMER_QUEUE = 'test-timers'
|
|
32
|
+
|
|
33
|
+
export const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Poll a queue until `match` accepts a message, or the deadline passes.
|
|
37
|
+
*
|
|
38
|
+
* WHY THIS IS A POLL AND NOT A LONG POLL, MEASURED ON A REAL RIG. A timer's
|
|
39
|
+
* delivery is committed by the sweeper INSIDE PostgreSQL
|
|
40
|
+
* (`log_timers_fire_v1` = DELETE + push in one transaction), so it does not
|
|
41
|
+
* pass through the broker's push handler and the destination partition is not
|
|
42
|
+
* marked ready in that broker's hot list. A consumer therefore sees the
|
|
43
|
+
* message only at the next hot-list reseed: on a default broker
|
|
44
|
+
* (QUEEN_HOTLIST_RESEED_MS = 30000) a timer scheduled for +300 ms was measured
|
|
45
|
+
* arriving at +30.7 s, and a 20 s long poll timed out with the message already
|
|
46
|
+
* committed in the log.
|
|
47
|
+
*
|
|
48
|
+
* That is a broker property, not a client one, and it is why these tests
|
|
49
|
+
* assert ARRIVAL and never latency: `deliverAt` is "not before", never
|
|
50
|
+
* "exactly at". The deadline here is generous on purpose -- it is sized
|
|
51
|
+
* against the reseed period, not against the delay under test.
|
|
52
|
+
*/
|
|
53
|
+
export async function popUntil(client, queueName, match, timeoutMs = 60000) {
|
|
54
|
+
const deadline = Date.now() + timeoutMs
|
|
55
|
+
for (;;) {
|
|
56
|
+
const messages = await client.queue(queueName).batch(10).wait(false).pop()
|
|
57
|
+
const hit = messages.filter(match)
|
|
58
|
+
if (hit.length > 0) return hit
|
|
59
|
+
// Anything else in this queue is a stranger -- a delivery left behind by an
|
|
60
|
+
// earlier run of this same test. It is ACKED rather than ignored, and that
|
|
61
|
+
// is not tidiness: queue-mode delivery is ordered, so an unacked stranger
|
|
62
|
+
// holds its lease and blocks the cursor, and the message under test would
|
|
63
|
+
// never be reached inside any deadline. This is the shape of the failure
|
|
64
|
+
// that a "wait longer" would never have fixed.
|
|
65
|
+
for (const stranger of messages) {
|
|
66
|
+
await client.ack(stranger)
|
|
67
|
+
}
|
|
68
|
+
if (Date.now() > deadline) return []
|
|
69
|
+
await sleep(500)
|
|
70
|
+
}
|
|
71
|
+
}
|
package/test-v2/ackwindow.js
CHANGED
|
@@ -30,11 +30,12 @@ const TENANT = '00000000-0000-0000-0000-000000000001'
|
|
|
30
30
|
|
|
31
31
|
function sleep(ms) { return new Promise(r => setTimeout(r, ms)) }
|
|
32
32
|
|
|
33
|
-
async function popRetry(client, queue, { batch = 3, group = null, partition = null, tries = 30 } = {}) {
|
|
33
|
+
async function popRetry(client, queue, { batch = 3, group = null, partition = null, tries = 30, mode = null } = {}) {
|
|
34
34
|
for (let i = 0; i < tries; i++) {
|
|
35
35
|
let b = client.queue(queue).batch(batch).wait(false)
|
|
36
36
|
if (group) b = b.group(group)
|
|
37
37
|
if (partition) b = b.partition(partition)
|
|
38
|
+
if (mode) b = b.subscriptionMode(mode)
|
|
38
39
|
const msgs = await b.pop()
|
|
39
40
|
if (msgs && msgs.length > 0) return msgs
|
|
40
41
|
await sleep(150)
|
|
@@ -192,7 +193,9 @@ export async function emptyPartitionCursorSealsAfterRetention(client) {
|
|
|
192
193
|
[0, 1, 2, 3, 4, 5].map(n => ({ data: { n }, transactionId: `${queue}-tx-${n}` })))
|
|
193
194
|
|
|
194
195
|
// Consume + ack only the first two: committed lands at 1, the rest stays.
|
|
195
|
-
|
|
196
|
+
// First contact for g-seal, and the 6 messages are already pushed: the
|
|
197
|
+
// group has to ask for the backlog it is about to half-consume.
|
|
198
|
+
const first = await popRetry(client, queue, { batch: 2, group, mode: 'all' })
|
|
196
199
|
if (first.length !== 2) return { success: false, message: `expected 2 messages, got ${first.length}` }
|
|
197
200
|
const ackr = await client.ack(first, true, { group })
|
|
198
201
|
if (ackr.success !== true) return { success: false, message: `ack failed: ${ackr.error}` }
|
|
@@ -245,7 +248,11 @@ export async function emptyPartitionCursorSealsAfterRetention(client) {
|
|
|
245
248
|
})()
|
|
246
249
|
if (!dpart0) return { success: false, message: 'delayed queue segments never appeared' }
|
|
247
250
|
|
|
248
|
-
|
|
251
|
+
// First contact for g-seal on THIS queue too (metadata is queue-scoped), and
|
|
252
|
+
// the 2 delayed messages are already pushed: under the default 'new' the
|
|
253
|
+
// cursor would seed at last_offset and the guard below could not tell a seed
|
|
254
|
+
// from a seal. 'all' seeds at -1, so a non -1 cursor can only be the seal.
|
|
255
|
+
const dempty = await client.queue(dqueue).partition('Default').group(group).subscriptionMode('all').batch(1).wait(false).pop()
|
|
249
256
|
if (dempty && dempty.length > 0) return { success: false, message: 'delayed message delivered early?' }
|
|
250
257
|
const dcommitted = await committedOf(dpart0.id, group)
|
|
251
258
|
const dsegs = await segmentCount(dpart0.id)
|
package/test-v2/consume.js
CHANGED
|
@@ -10,7 +10,6 @@ export async function testConsumer(client) {
|
|
|
10
10
|
|
|
11
11
|
let msgToReturn = null
|
|
12
12
|
|
|
13
|
-
// docs:start(js-consume)
|
|
14
13
|
await client
|
|
15
14
|
.queue('test-queue-v2-consume')
|
|
16
15
|
.batch(1)
|
|
@@ -19,7 +18,6 @@ export async function testConsumer(client) {
|
|
|
19
18
|
.consume(async msg => {
|
|
20
19
|
msgToReturn = msg
|
|
21
20
|
})
|
|
22
|
-
// docs:end
|
|
23
21
|
|
|
24
22
|
return { success: msgToReturn !== null, message: 'Consumer test completed successfully' }
|
|
25
23
|
}
|
package/test-v2/docs.js
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The tests behind the published examples.
|
|
3
|
+
*
|
|
4
|
+
* Every marked region in this file is rendered on queenmq.com: the homepage,
|
|
5
|
+
* the quickstart, the concept pages and the HTTP reference pull these exact
|
|
6
|
+
* lines through webdoc/scripts/gen-snippets.mjs. They are tests written to
|
|
7
|
+
* read as documentation: real queue names, a partition key that means
|
|
8
|
+
* something, and no test scaffolding inside a marked region. Assertions stay
|
|
9
|
+
* outside the markers.
|
|
10
|
+
*
|
|
11
|
+
* After editing a marked region, regenerate the partials with
|
|
12
|
+
* `pnpm --dir webdoc gen` or the docs CI check fails on drift. The queues used
|
|
13
|
+
* here (orders, payments, invoices) are wiped by cleanupTestData in run.js
|
|
14
|
+
* before every suite run, exactly like the test-% queues.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
export async function docsProduceAndConsume(client) {
|
|
18
|
+
// docs:start(js-push)
|
|
19
|
+
const res = await client
|
|
20
|
+
.queue('orders')
|
|
21
|
+
.partition('customer-42')
|
|
22
|
+
.push([{ data: { orderId: 9137, amount: 99.5 } }])
|
|
23
|
+
// docs:end
|
|
24
|
+
if (res[0].status !== 'queued') {
|
|
25
|
+
return { success: false, message: `Push not queued: ${JSON.stringify(res[0])}` }
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// The consume loop acks each message when the callback resolves, so when
|
|
29
|
+
// the awaited chain returns, the billing group's cursor has moved.
|
|
30
|
+
// docs:start(js-consume)
|
|
31
|
+
await client
|
|
32
|
+
.queue('orders')
|
|
33
|
+
.group('billing')
|
|
34
|
+
.subscriptionMode('all')
|
|
35
|
+
.limit(1)
|
|
36
|
+
.each()
|
|
37
|
+
.consume(async (message) => {
|
|
38
|
+
console.log(message.data)
|
|
39
|
+
})
|
|
40
|
+
// docs:end
|
|
41
|
+
|
|
42
|
+
const drained = await client
|
|
43
|
+
.queue('orders')
|
|
44
|
+
.group('billing')
|
|
45
|
+
.batch(1)
|
|
46
|
+
.wait(false)
|
|
47
|
+
.pop()
|
|
48
|
+
if (drained.length !== 0) {
|
|
49
|
+
return { success: false, message: 'Billing group cursor did not advance after consume' }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// A raw pop on another cursor still sees the message: groups are fan-out.
|
|
53
|
+
// docs:start(js-pop)
|
|
54
|
+
const messages = await client
|
|
55
|
+
.queue('orders')
|
|
56
|
+
.batch(10)
|
|
57
|
+
.wait(true)
|
|
58
|
+
.pop()
|
|
59
|
+
// docs:end
|
|
60
|
+
if (messages.length !== 1 || messages[0].data.orderId !== 9137) {
|
|
61
|
+
return { success: false, message: `Fan-out pop expected the order back, got ${JSON.stringify(messages)}` }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return { success: true, message: 'Push, consume and fan-out pop all verified' }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export async function docsDeduplication(client) {
|
|
68
|
+
// The fixed transactionId below survives reruns because cleanupTestData
|
|
69
|
+
// purges log_txns for these queues before the suite starts.
|
|
70
|
+
// docs:start(js-push-dedup)
|
|
71
|
+
const first = await client
|
|
72
|
+
.queue('payments')
|
|
73
|
+
.partition('customer-42')
|
|
74
|
+
.push([{ transactionId: 'order-9137-paid', data: { orderId: 9137, amount: 99.5 } }])
|
|
75
|
+
|
|
76
|
+
const retry = await client
|
|
77
|
+
.queue('payments')
|
|
78
|
+
.partition('customer-42')
|
|
79
|
+
.push([{ transactionId: 'order-9137-paid', data: { orderId: 9137, amount: 99.5 } }])
|
|
80
|
+
// retry[0].status is 'duplicate': the second push wrote nothing
|
|
81
|
+
// and answers with the first message's id.
|
|
82
|
+
// docs:end
|
|
83
|
+
if (first[0].status !== 'queued') {
|
|
84
|
+
return { success: false, message: `First push not queued: ${JSON.stringify(first[0])}` }
|
|
85
|
+
}
|
|
86
|
+
if (retry[0].status !== 'duplicate') {
|
|
87
|
+
return { success: false, message: `Retry not deduplicated: ${JSON.stringify(retry[0])}` }
|
|
88
|
+
}
|
|
89
|
+
if (retry[0].message_id !== first[0].message_id) {
|
|
90
|
+
return { success: false, message: 'Duplicate did not answer with the original message id' }
|
|
91
|
+
}
|
|
92
|
+
return { success: true, message: 'Second push with the same transactionId wrote nothing' }
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export async function docsReplayAndSeek(client) {
|
|
96
|
+
const seeded = await client.queue('orders').partition('customer-91').push([
|
|
97
|
+
{ data: { orderId: 5001, amount: 12 } },
|
|
98
|
+
{ data: { orderId: 5002, amount: 24 } },
|
|
99
|
+
{ data: { orderId: 5003, amount: 36 } },
|
|
100
|
+
])
|
|
101
|
+
if (!seeded.every(r => r.status === 'queued')) {
|
|
102
|
+
return { success: false, message: 'Seed pushes failed' }
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// docs:start(js-replay)
|
|
106
|
+
// A fresh group with subscriptionMode('all') reads the whole retained
|
|
107
|
+
// lane from the beginning, without touching any other group's cursor.
|
|
108
|
+
const replayed = await client
|
|
109
|
+
.queue('orders')
|
|
110
|
+
.partition('customer-91')
|
|
111
|
+
.group('audit')
|
|
112
|
+
.subscriptionMode('all')
|
|
113
|
+
.batch(100)
|
|
114
|
+
.wait(true)
|
|
115
|
+
.pop()
|
|
116
|
+
// docs:end
|
|
117
|
+
const ids = replayed.map(m => m.data.orderId)
|
|
118
|
+
if (ids.length !== 3 || ids[0] !== 5001 || ids[1] !== 5002 || ids[2] !== 5003) {
|
|
119
|
+
return { success: false, message: `Replay expected [5001,5002,5003] in order, got ${JSON.stringify(ids)}` }
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// docs:start(js-seek)
|
|
123
|
+
// Move the audit group's cursor back one hour. The seek also releases
|
|
124
|
+
// any live lease, so an in-flight batch is abandoned, not acked.
|
|
125
|
+
await client.admin.seekConsumerGroup('audit', 'orders', {
|
|
126
|
+
timestamp: new Date(Date.now() - 3600 * 1000).toISOString(),
|
|
127
|
+
})
|
|
128
|
+
// docs:end
|
|
129
|
+
|
|
130
|
+
const again = await client
|
|
131
|
+
.queue('orders')
|
|
132
|
+
.partition('customer-91')
|
|
133
|
+
.group('audit')
|
|
134
|
+
.batch(100)
|
|
135
|
+
.wait(true)
|
|
136
|
+
.pop()
|
|
137
|
+
const againIds = again.map(m => m.data.orderId)
|
|
138
|
+
if (againIds.length !== 3 || againIds[0] !== 5001) {
|
|
139
|
+
return { success: false, message: `Post-seek pop expected the lane again, got ${JSON.stringify(againIds)}` }
|
|
140
|
+
}
|
|
141
|
+
return { success: true, message: 'Replay from the beginning and seek-back both verified' }
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export async function docsTransactionalHandoff(client) {
|
|
145
|
+
const seeded = await client
|
|
146
|
+
.queue('orders')
|
|
147
|
+
.partition('customer-77')
|
|
148
|
+
.push([{ data: { orderId: 4102, amount: 18 } }])
|
|
149
|
+
if (seeded[0].status !== 'queued') {
|
|
150
|
+
return { success: false, message: 'Seed push failed' }
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// docs:start(js-transaction)
|
|
154
|
+
await client
|
|
155
|
+
.queue('orders')
|
|
156
|
+
.group('invoicing')
|
|
157
|
+
.subscriptionMode('all')
|
|
158
|
+
.each()
|
|
159
|
+
.autoAck(false) // the acknowledgement belongs to the transaction, not to the loop
|
|
160
|
+
.limit(1)
|
|
161
|
+
.idleMillis(5000)
|
|
162
|
+
.consume(async (message) => {
|
|
163
|
+
// commit() throws when the broker rejects the bundle, so reaching the
|
|
164
|
+
// line after it means the ack and the push are both durable.
|
|
165
|
+
await client
|
|
166
|
+
.transaction()
|
|
167
|
+
.queue('invoices')
|
|
168
|
+
.push([{ data: { orderId: message.data.orderId, invoiced: true } }])
|
|
169
|
+
.ack(message, 'completed', { consumerGroup: 'invoicing' })
|
|
170
|
+
.commit()
|
|
171
|
+
})
|
|
172
|
+
// docs:end
|
|
173
|
+
|
|
174
|
+
// The loop takes whichever order is oldest for this group, and other docs
|
|
175
|
+
// tests push to the same queue, so identity is checked against the orders
|
|
176
|
+
// that actually exist rather than against a hardcoded id. A throwaway group
|
|
177
|
+
// reads them without disturbing the invoicing cursor.
|
|
178
|
+
const everyOrder = await client
|
|
179
|
+
.queue('orders')
|
|
180
|
+
.group(`docs-audit-${Date.now()}`)
|
|
181
|
+
.subscriptionMode('all')
|
|
182
|
+
.batch(50)
|
|
183
|
+
.partitions(20)
|
|
184
|
+
.wait(false)
|
|
185
|
+
.pop()
|
|
186
|
+
const orderIds = everyOrder.map(m => m.data.orderId)
|
|
187
|
+
|
|
188
|
+
const invoiced = await client
|
|
189
|
+
.queue('invoices')
|
|
190
|
+
.batch(1)
|
|
191
|
+
.partitions(10)
|
|
192
|
+
.wait(true)
|
|
193
|
+
.pop()
|
|
194
|
+
if (invoiced.length !== 1 || !orderIds.includes(invoiced[0].data.orderId)) {
|
|
195
|
+
return {
|
|
196
|
+
success: false,
|
|
197
|
+
message: `Invoice not committed for a real order: ${JSON.stringify(invoiced)} against ${JSON.stringify(orderIds)}`,
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
if (invoiced[0].data.invoiced !== true) {
|
|
201
|
+
return { success: false, message: 'Invoice payload corrupted' }
|
|
202
|
+
}
|
|
203
|
+
return { success: true, message: 'Ack and next-stage push committed together' }
|
|
204
|
+
}
|
|
@@ -95,7 +95,12 @@ describe('HttpClient — 429 retry policy', () => {
|
|
|
95
95
|
it('falls back to exponential backoff when Retry-After is absent, and the gap grows', async () => {
|
|
96
96
|
const plan = repeat(rateLimited(), 2)
|
|
97
97
|
await withPlanServer(plan, { status: 200, body: { ok: true } }, async (url, hits) => {
|
|
98
|
-
|
|
98
|
+
// baseMs has to be well above the event loop's scheduling noise. At 20ms
|
|
99
|
+
// a loaded runner added ~27ms to BOTH gaps, which collapses their ratio
|
|
100
|
+
// towards 1 (observed: 47ms then 54ms) and failed a test about growth
|
|
101
|
+
// that was working perfectly. The delay is additive, so the fix is a base
|
|
102
|
+
// big enough to dominate it, not a looser threshold.
|
|
103
|
+
const client = new HttpClient({ baseUrl: url, retry429: { baseMs: 100, capMs: 2000 } })
|
|
99
104
|
try {
|
|
100
105
|
const result = await client.get('/x')
|
|
101
106
|
assert.deepEqual(result, { ok: true })
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A real node:http server driven by a canned response plan, recording every
|
|
3
|
+
* request it receives (method, url, parsed body).
|
|
4
|
+
*
|
|
5
|
+
* Same shape as test-v2/http-unit/retry429.test.js's withPlanServer and
|
|
6
|
+
* streams-unit/fakeServer.js -- no mocking framework, a real socket, real
|
|
7
|
+
* fetch, real JSON. The difference is that these tests assert the REQUEST,
|
|
8
|
+
* not the retry behaviour: the exact JSON body of every KV and timer
|
|
9
|
+
* operation is the contract towards the broker, and a plan server is the only
|
|
10
|
+
* place that contract can be pinned without a database.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { createServer } from 'node:http'
|
|
14
|
+
|
|
15
|
+
export function withPlanServer(plan, defaultResponse, run) {
|
|
16
|
+
const hits = []
|
|
17
|
+
let index = 0
|
|
18
|
+
|
|
19
|
+
const server = createServer((req, res) => {
|
|
20
|
+
let raw = ''
|
|
21
|
+
req.on('data', chunk => { raw += chunk })
|
|
22
|
+
req.on('end', () => {
|
|
23
|
+
let body = null
|
|
24
|
+
if (raw.length > 0) {
|
|
25
|
+
try {
|
|
26
|
+
body = JSON.parse(raw)
|
|
27
|
+
} catch {
|
|
28
|
+
body = { __unparseable: raw }
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
hits.push({ method: req.method, url: req.url, raw, body })
|
|
32
|
+
|
|
33
|
+
const descriptor = index < plan.length ? plan[index] : defaultResponse
|
|
34
|
+
index++
|
|
35
|
+
|
|
36
|
+
const status = descriptor.status ?? 200
|
|
37
|
+
const headers = { 'Content-Type': 'application/json', ...(descriptor.headers || {}) }
|
|
38
|
+
res.writeHead(status, headers)
|
|
39
|
+
res.end(JSON.stringify(descriptor.body ?? { ok: true }))
|
|
40
|
+
})
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
return new Promise((resolve, reject) => {
|
|
44
|
+
server.listen(0, '127.0.0.1', () => {
|
|
45
|
+
const { port } = server.address()
|
|
46
|
+
const teardown = () => new Promise(r => server.close(r))
|
|
47
|
+
Promise.resolve()
|
|
48
|
+
.then(() => run(`http://127.0.0.1:${port}`, hits))
|
|
49
|
+
.then(
|
|
50
|
+
async (value) => { await teardown(); resolve(value) },
|
|
51
|
+
async (err) => { await teardown(); reject(err) }
|
|
52
|
+
)
|
|
53
|
+
})
|
|
54
|
+
})
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** One canned 200 with this JSON body. */
|
|
58
|
+
export function ok(body) {
|
|
59
|
+
return { status: 200, body }
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** The KV batch envelope: {"results":[...]}, index-aligned to the input. */
|
|
63
|
+
export function kvResults(...results) {
|
|
64
|
+
return ok({ results })
|
|
65
|
+
}
|