queen-mq 1.0.0 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -0
- package/client-v2/Queen.js +54 -0
- package/client-v2/README.md +32 -0
- package/client-v2/admin/Admin.js +18 -0
- package/client-v2/builders/QueueBuilder.js +8 -16
- package/client-v2/builders/TimerBuilder.js +262 -0
- package/client-v2/builders/TransactionBuilder.js +185 -8
- package/client-v2/kv/Kv.js +432 -0
- package/client-v2/kv/expiry.js +148 -0
- package/package.json +11 -3
- package/test-v2/_kvtimers.js +71 -0
- package/test-v2/ackwindow.js +10 -3
- package/test-v2/consume.js +0 -2
- package/test-v2/docs.js +204 -0
- package/test-v2/http-unit/retry429.test.js +6 -1
- package/test-v2/kv-unit/_planServer.js +65 -0
- package/test-v2/kv-unit/kvWire.test.js +377 -0
- package/test-v2/kv-unit/timerWire.test.js +177 -0
- package/test-v2/kv-unit/txnWire.test.js +222 -0
- package/test-v2/kv.js +273 -0
- package/test-v2/pop.js +0 -2
- package/test-v2/push.js +0 -4
- package/test-v2/run.js +34 -4
- package/test-v2/semantics.js +16 -7
- package/test-v2/stream/_helpers.js +7 -0
- package/test-v2/stream/combined.js +4 -3
- package/test-v2/stream/cron.js +1 -1
- package/test-v2/stream/eventTime.js +8 -5
- package/test-v2/stream/operators.js +5 -5
- package/test-v2/stream/recovery.js +4 -1
- package/test-v2/stream/session.js +3 -3
- package/test-v2/stream/sliding.js +1 -1
- package/test-v2/stream/throughput.js +3 -2
- package/test-v2/stream/tumbling.js +6 -6
- package/test-v2/timers.js +209 -0
- package/test-v2/transaction.js +4 -3
package/README.md
CHANGED
|
@@ -28,6 +28,7 @@ Queen MQ is a PostgreSQL-backed message queue system with a powerful feature set
|
|
|
28
28
|
- **Message Tracing** - Debug distributed workflows with trace timelines
|
|
29
29
|
- **Client-Side Buffering** - 10x-100x throughput boost for high-volume pushes
|
|
30
30
|
- **Real-time Streaming** - Windowed aggregation and processing
|
|
31
|
+
- **Key/Value State and Timers** - Transactional state and scheduled messages
|
|
31
32
|
|
|
32
33
|
This client provides a fluent, promise-based API for Node.js applications.
|
|
33
34
|
|
|
@@ -373,6 +374,145 @@ await queen.queue('orders').consume(async (msg) => {
|
|
|
373
374
|
|
|
374
375
|
---
|
|
375
376
|
|
|
377
|
+
## Key/Value State and Timers
|
|
378
|
+
|
|
379
|
+
Both surfaces are **always there**. There is nothing to enable: kv and timers are part of the
|
|
380
|
+
broker the way push and pop are, on every cell that runs it. There is no capability to probe and
|
|
381
|
+
no 404 that means "this cell does not have the feature" — a 404 from these routes is a bug.
|
|
382
|
+
|
|
383
|
+
What an operator can still do is **pause** them, with the runtime kill switch in
|
|
384
|
+
`queen.system_state` (`kv_enabled`, `timers_schedule_enabled`, `timers_fire_enabled`) — the same
|
|
385
|
+
class of lever as maintenance mode, pulled live during an incident and expected to be pulled back.
|
|
386
|
+
A paused surface answers `503` with `Retry-After` and `error: 'kv_disabled'` / `'timers_disabled'`,
|
|
387
|
+
which this client retries like any other 5xx. Inside a transaction it is a `403` on the `kv` or
|
|
388
|
+
`timers` rider instead, so a bundle holding messages does not spin forever on a paused cell.
|
|
389
|
+
|
|
390
|
+
```javascript
|
|
391
|
+
try {
|
|
392
|
+
await queen.kv.put('orders', 'order:9f1', { state: 'held' }, { ttl: '60s' })
|
|
393
|
+
} catch (e) {
|
|
394
|
+
if (e.code === 'kv_disabled') { /* paused by an operator; it will come back */ }
|
|
395
|
+
throw e
|
|
396
|
+
}
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
Branch on `e.code`, never on the message. And write that branch as "temporarily paused", not as
|
|
400
|
+
"this deployment lacks KV": handling the refusal is right, treating it as a configuration to check
|
|
401
|
+
before you use the surface is not.
|
|
402
|
+
|
|
403
|
+
### Key/Value
|
|
404
|
+
|
|
405
|
+
```javascript
|
|
406
|
+
// An expiry is MANDATORY on every write: exactly one of ttlSeconds (or the
|
|
407
|
+
// sugar ttl / until) and forever: true. A put never inherits the previous TTL.
|
|
408
|
+
await queen.kv.put('orders', 'order:9f1', { state: 'held' }, { ttl: '60s' })
|
|
409
|
+
|
|
410
|
+
const row = await queen.kv.get('orders', 'order:9f1')
|
|
411
|
+
if (row.found) console.log(row.value, row.version) // found is separate: null is a legal value
|
|
412
|
+
|
|
413
|
+
// "Did I win?" in one call. This is the idempotency marker.
|
|
414
|
+
const { won, value } = await queen.kv.once('dedup', eventId, { ttl: '24h' })
|
|
415
|
+
if (!won) return // somebody already did this
|
|
416
|
+
|
|
417
|
+
// Optimistic lock. expect: 0 means "must not exist"; expect: N is a pure
|
|
418
|
+
// update that creates nothing when it matches no row.
|
|
419
|
+
const res = await queen.kv.put('orders', 'order:9f1', { state: 'shipped' },
|
|
420
|
+
{ ttl: '60s', expect: row.version })
|
|
421
|
+
if (!res.applied) console.log(res.reason) // 'version' | 'exists' | 'absent' | 'limit' | 'type'
|
|
422
|
+
|
|
423
|
+
// Rate limiting without a CAS loop. With max, `applied` IS the admission
|
|
424
|
+
// decision: nothing saturates, nothing truncates, a refusal spends no budget.
|
|
425
|
+
const hit = await queen.kv.incr('quota', `${customer}:${hour}`, 1, { max: 1000, ttl: '1h' })
|
|
426
|
+
if (!hit.applied) throw new TooManyRequests()
|
|
427
|
+
|
|
428
|
+
for await (const r of queen.kv.listAll('saga', 'order:9f1:')) { /* follows nextAfter */ }
|
|
429
|
+
```
|
|
430
|
+
|
|
431
|
+
Seven operations: `get`, `getMany`, `getPrefix`, `put`, `putIfAbsent`, `delete`, `incr`, plus the
|
|
432
|
+
two conveniences this client owns, `once` and `listAll`.
|
|
433
|
+
|
|
434
|
+
**Every write returns an OBJECT, and objects are always truthy.** `if (await queen.kv.delete(ns,
|
|
435
|
+
key))` is always taken and is a bug. Read `.applied`, or `.won` on `once`, or `.found` on a read.
|
|
436
|
+
That holds for all five writes, and it is the one trap this language cannot defend against
|
|
437
|
+
structurally.
|
|
438
|
+
|
|
439
|
+
**A write that did not apply is not an error.** `applied: false` answers HTTP 200 with the current
|
|
440
|
+
value and version, so the loser needs no second round trip.
|
|
441
|
+
|
|
442
|
+
**Read-modify-write across two calls is safe only when the KV key derives from the partition key.**
|
|
443
|
+
Otherwise the lanes do not serialise it for you: use `incr`, or carry `expect`.
|
|
444
|
+
|
|
445
|
+
**`putIfAbsent` plus a TTL is not a distributed lock.** A lock that expires is not revoked: the old
|
|
446
|
+
holder keeps working, it simply no longer has the row. Carry your `version` as `expect` on every
|
|
447
|
+
later write so a lapsed holder fails with `reason: 'version'` instead of overwriting the new one.
|
|
448
|
+
|
|
449
|
+
### Timers
|
|
450
|
+
|
|
451
|
+
```javascript
|
|
452
|
+
// Fire no earlier than 30 minutes from now, into a real queue, through the log.
|
|
453
|
+
const res = await queen.timer('reminders')
|
|
454
|
+
.key(`order:${orderId}`)
|
|
455
|
+
.delay('30m') // or .delayMs(250)
|
|
456
|
+
.payload({ orderId })
|
|
457
|
+
.schedule() // status: 'scheduled' | 'rescheduled' | 'too_late'
|
|
458
|
+
|
|
459
|
+
await queen.timer('reminders').key(`order:${orderId}`).peek()
|
|
460
|
+
await queen.timer('reminders').list({ limit: 50 })
|
|
461
|
+
await queen.timer('reminders').key(`order:${orderId}`).cancel()
|
|
462
|
+
```
|
|
463
|
+
|
|
464
|
+
Scheduling the same `(queue, timerKey)` again is the same upsert, so a retry after a client crash
|
|
465
|
+
is safe by construction and `status` says which it was. A reschedule mints a new `txn` and resets
|
|
466
|
+
the retry budget.
|
|
467
|
+
|
|
468
|
+
Durations that can be sub-second are in **milliseconds** (`delayMs`), the ones that cannot are in
|
|
469
|
+
**seconds** (`ttlSeconds`). Only relative delays exist, because there is one clock and it is the
|
|
470
|
+
database's. A delay in the past is legal and fires on the first cycle.
|
|
471
|
+
|
|
472
|
+
`deliverAt` is **"not before"**, never "exactly at".
|
|
473
|
+
|
|
474
|
+
**`absent` means "no longer pending" and may mean ALREADY DELIVERED.** There is no tombstone: a
|
|
475
|
+
fired timer has no row left, so `absent` carries `ok: false` and the answer echoes the `txn` so the
|
|
476
|
+
authority, the log, can be consulted without a second API. A saga that cancels a compensation timer
|
|
477
|
+
must have the compensating consumer re-check the saga's KV state before compensating, because the
|
|
478
|
+
cancel may have arrived 5 ms after the fire.
|
|
479
|
+
|
|
480
|
+
Use `queen.timer(q).key(k).cancel()` rather than a cancel inside a bundle when the cancel must land
|
|
481
|
+
regardless: it takes the DELETE route, the one a quota is forbidden to block. A tenant that cannot
|
|
482
|
+
cancel keeps producing messages it cannot stop.
|
|
483
|
+
|
|
484
|
+
### Inside a transaction
|
|
485
|
+
|
|
486
|
+
The transaction is the **primary fence**; `expect` is only the secondary assertion. A state write
|
|
487
|
+
that shares the transaction with its ack is undone when an expired lease makes the ack fail, which
|
|
488
|
+
a compare-and-set cannot do.
|
|
489
|
+
|
|
490
|
+
```javascript
|
|
491
|
+
const result = await queen.transaction()
|
|
492
|
+
.ack(message)
|
|
493
|
+
.queue('emails').push([{ data: mail }])
|
|
494
|
+
.once('sent', message.transactionId, { ttl: '24h' }) // the gate
|
|
495
|
+
.timer('reminders').key(orderId).delay('24h').payload({ orderId }).schedule()
|
|
496
|
+
.commit()
|
|
497
|
+
|
|
498
|
+
if (result.success === false) {
|
|
499
|
+
// RETURNED, not thrown: a lost gate is the expected outcome of a legitimate
|
|
500
|
+
// redelivery, so it stays out of your retry policy and your error metrics.
|
|
501
|
+
result.reason // 'kv_precondition'
|
|
502
|
+
result.failedIndex, result.kvReason, result.version, result.value
|
|
503
|
+
return
|
|
504
|
+
}
|
|
505
|
+
```
|
|
506
|
+
|
|
507
|
+
`once` is `putIfAbsent` with `required: true`, and `required` is what makes it a gate: without it a
|
|
508
|
+
lost precondition is only a verdict in the results, and the push and the ack still go through.
|
|
509
|
+
|
|
510
|
+
`kv.getPrefix` is not available inside a transaction and throws here rather than at the broker: its
|
|
511
|
+
cost is not bounded by the caller. `get` and `getMany` are allowed, because they are. Everything
|
|
512
|
+
other than the lost precondition still throws.
|
|
513
|
+
|
|
514
|
+
---
|
|
515
|
+
|
|
376
516
|
## Examples
|
|
377
517
|
|
|
378
518
|
### Complete Pipeline with Consumer Groups
|
|
@@ -521,6 +661,41 @@ await queen.transaction()
|
|
|
521
661
|
.queue('output')
|
|
522
662
|
.push([{ data: { result: 'processed' } }])
|
|
523
663
|
.commit()
|
|
664
|
+
|
|
665
|
+
// Riders. commit() RETURNS on a lost gate, and throws on everything else.
|
|
666
|
+
await queen.transaction()
|
|
667
|
+
.ack(message)
|
|
668
|
+
.kv.put('saga', sagaId, { step: 'charged' }, { ttl: '24h' })
|
|
669
|
+
.once('dedup', message.transactionId, { ttl: '24h' })
|
|
670
|
+
.timer('reminders').key(orderId).delay('30m').payload({ orderId }).schedule()
|
|
671
|
+
.commit()
|
|
672
|
+
```
|
|
673
|
+
|
|
674
|
+
### Key/Value
|
|
675
|
+
|
|
676
|
+
```javascript
|
|
677
|
+
await queen.kv.get(ns, key) // {found, key, value, version, expiresAt, updatedAt}
|
|
678
|
+
await queen.kv.getMany(ns, [k1, k2]) // {rows, missing, truncated}
|
|
679
|
+
await queen.kv.getPrefix(ns, prefix, { limit: 100, after, keysOnly })
|
|
680
|
+
await queen.kv.put(ns, key, value, { ttl: '24h', expect, required })
|
|
681
|
+
await queen.kv.putIfAbsent(ns, key, value, { ttlSeconds: 86400 })
|
|
682
|
+
await queen.kv.delete(ns, key, { expect })
|
|
683
|
+
await queen.kv.incr(ns, key, 1, { max: 1000, min, ttl: '1h' })
|
|
684
|
+
await queen.kv.once(ns, key, { ttl: '24h' }) // {won, value, version, result}
|
|
685
|
+
for await (const row of queen.kv.listAll(ns, prefix)) { }
|
|
686
|
+
```
|
|
687
|
+
|
|
688
|
+
### Timers
|
|
689
|
+
|
|
690
|
+
```javascript
|
|
691
|
+
// Builder steps: .key(timerKey) required, .delayMs(250) or .delay('30m') required,
|
|
692
|
+
// .payload(anyJsonOrBuffer) required to schedule, .partition(name) optional
|
|
693
|
+
// (defaults to 'Default'), .txn(transactionId) optional (minted when absent).
|
|
694
|
+
|
|
695
|
+
await queen.timer(q).key(k).delay('30m').payload(p).schedule() // {ok, status, txn, messageId, deliverAt}
|
|
696
|
+
await queen.timer(q).key(k).cancel() // {ok, status, txn}
|
|
697
|
+
await queen.timer(q).key(k).peek() // {found, ...}
|
|
698
|
+
await queen.timer(q).list({ limit: 50, after }) // {rows, truncated, nextAfter}
|
|
524
699
|
```
|
|
525
700
|
|
|
526
701
|
### Lease Renewal
|
package/client-v2/Queen.js
CHANGED
|
@@ -8,6 +8,8 @@ import { LoadBalancer } from './http/LoadBalancer.js'
|
|
|
8
8
|
import { BufferManager } from './buffer/BufferManager.js'
|
|
9
9
|
import { QueueBuilder } from './builders/QueueBuilder.js'
|
|
10
10
|
import { TransactionBuilder } from './builders/TransactionBuilder.js'
|
|
11
|
+
import { TimerBuilder } from './builders/TimerBuilder.js'
|
|
12
|
+
import { Kv } from './kv/Kv.js'
|
|
11
13
|
import { StreamBuilder } from './stream/StreamBuilder.js'
|
|
12
14
|
import { StreamConsumer } from './stream/StreamConsumer.js'
|
|
13
15
|
import { Admin } from './admin/Admin.js'
|
|
@@ -57,6 +59,7 @@ export class Queen {
|
|
|
57
59
|
#config
|
|
58
60
|
#shutdownHandlers = []
|
|
59
61
|
#admin = null
|
|
62
|
+
#kv = null
|
|
60
63
|
|
|
61
64
|
constructor(config = {}) {
|
|
62
65
|
// Configure custom logger before anything else.
|
|
@@ -216,6 +219,57 @@ export class Queen {
|
|
|
216
219
|
return this.#admin
|
|
217
220
|
}
|
|
218
221
|
|
|
222
|
+
// ===========================
|
|
223
|
+
// KV API Entry Point
|
|
224
|
+
// ===========================
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Transactional key/value state, alongside the log
|
|
228
|
+
* (PLAN_KV_TIMERS.md §5).
|
|
229
|
+
*
|
|
230
|
+
* const { won } = await queen.kv.once('idem', orderId, { ttl: '24h' })
|
|
231
|
+
* const row = await queen.kv.get('saga', sagaId) // {found, value, version, ...}
|
|
232
|
+
*
|
|
233
|
+
* Two things to carry into every use of it:
|
|
234
|
+
* * every write returns an OBJECT, and objects are always truthy --
|
|
235
|
+
* `if (await queen.kv.delete(ns, key))` is always true. Read `.applied`.
|
|
236
|
+
* * an expiry is MANDATORY on every write: exactly one of `ttlSeconds`
|
|
237
|
+
* (or the sugar `ttl` / `until`) and `forever: true`. There is no
|
|
238
|
+
* default, because a default is how a marker becomes immortal.
|
|
239
|
+
*
|
|
240
|
+
* Lazily initialized, singleton, like `admin`.
|
|
241
|
+
* @returns {Kv}
|
|
242
|
+
*/
|
|
243
|
+
get kv() {
|
|
244
|
+
if (!this.#kv) {
|
|
245
|
+
this.#kv = new Kv(this.#httpClient)
|
|
246
|
+
}
|
|
247
|
+
return this.#kv
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ===========================
|
|
251
|
+
// Timers API Entry Point
|
|
252
|
+
// ===========================
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Scheduled messages for one queue (PLAN_KV_TIMERS.md §4).
|
|
256
|
+
*
|
|
257
|
+
* await queen.timer('orders').key(orderId).delay('30m')
|
|
258
|
+
* .payload({ orderId }).schedule()
|
|
259
|
+
* await queen.timer('orders').key(orderId).cancel()
|
|
260
|
+
*
|
|
261
|
+
* `deliverAt` is "not before", never "exactly at". A cancel that answers
|
|
262
|
+
* `absent` means "no longer pending" and MAY MEAN ALREADY DELIVERED -- there
|
|
263
|
+
* is no tombstone, and the authority is the log.
|
|
264
|
+
*
|
|
265
|
+
* @param {string} queueName - destination queue (mandatory)
|
|
266
|
+
* @returns {TimerBuilder}
|
|
267
|
+
*/
|
|
268
|
+
timer(queueName) {
|
|
269
|
+
logger.log('Queen.timer', { queue: queueName })
|
|
270
|
+
return new TimerBuilder(this.#httpClient, queueName)
|
|
271
|
+
}
|
|
272
|
+
|
|
219
273
|
// ===========================
|
|
220
274
|
// Transaction API
|
|
221
275
|
// ===========================
|
package/client-v2/README.md
CHANGED
|
@@ -1730,6 +1730,38 @@ await queen
|
|
|
1730
1730
|
.commit()
|
|
1731
1731
|
```
|
|
1732
1732
|
|
|
1733
|
+
### Key/Value and Timers
|
|
1734
|
+
|
|
1735
|
+
Always present on every broker — no flag, nothing to probe. An operator's runtime kill switch can
|
|
1736
|
+
pause them (`503` + `Retry-After`, `error: 'kv_disabled'` / `'timers_disabled'`; `403` on a rider
|
|
1737
|
+
inside a transaction), which is a pause and not an absence. Full treatment in the
|
|
1738
|
+
[client README](../README.md#keyvalue-state-and-timers).
|
|
1739
|
+
|
|
1740
|
+
```javascript
|
|
1741
|
+
// Every write states its lifetime: exactly one of ttl/ttlSeconds/until and forever.
|
|
1742
|
+
await queen.kv.put('orders', 'order:9f1', { state: 'held' }, { ttl: '60s' })
|
|
1743
|
+
const row = await queen.kv.get('orders', 'order:9f1') // {found, value, version, ...}
|
|
1744
|
+
|
|
1745
|
+
// Writes return an OBJECT, always truthy. Read .applied, never the result itself.
|
|
1746
|
+
const res = await queen.kv.delete('orders', 'order:9f1')
|
|
1747
|
+
if (res.applied) { }
|
|
1748
|
+
|
|
1749
|
+
const { won } = await queen.kv.once('dedup', eventId, { ttl: '24h' })
|
|
1750
|
+
const hit = await queen.kv.incr('quota', key, 1, { max: 1000, ttl: '1h' }) // applied IS admission
|
|
1751
|
+
|
|
1752
|
+
await queen.timer('reminders').key(orderId).delay('30m').payload({ orderId }).schedule()
|
|
1753
|
+
await queen.timer('reminders').key(orderId).cancel() // 'absent' may mean already delivered
|
|
1754
|
+
|
|
1755
|
+
// The gate: marker, push and ack commit or roll back together.
|
|
1756
|
+
const out = await queen
|
|
1757
|
+
.transaction()
|
|
1758
|
+
.ack(message)
|
|
1759
|
+
.queue('emails').push([{ data: mail }])
|
|
1760
|
+
.once('sent', message.transactionId, { ttl: '24h' })
|
|
1761
|
+
.commit()
|
|
1762
|
+
if (out.success === false) return // returned, not thrown: a redelivery already handled
|
|
1763
|
+
```
|
|
1764
|
+
|
|
1733
1765
|
### Lease Renewal
|
|
1734
1766
|
|
|
1735
1767
|
```javascript
|
package/client-v2/admin/Admin.js
CHANGED
|
@@ -68,6 +68,24 @@ export class Admin {
|
|
|
68
68
|
return this.#httpClient.get(`/api/v1/resources/queues/${encodeURIComponent(name)}`)
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
+
/**
|
|
72
|
+
* Per-partition backlog for a queue — the cheap sibling of getQueue:
|
|
73
|
+
* watermark arithmetic only, no segments, no timestamps. Shape:
|
|
74
|
+
* {queue, group, pending, partitions: [{partition, pending}]}.
|
|
75
|
+
* Omitting group gives queue-level pending under the same worst-cursor
|
|
76
|
+
* precedence the dashboard publishes; a named group is that group's own
|
|
77
|
+
* backlog per partition. Requires broker >= 1.0.4 — an older broker
|
|
78
|
+
* answers 404 no_such_route, so fall back to getQueue there.
|
|
79
|
+
* @param {string} name - Queue name
|
|
80
|
+
* @param {string|null} [group] - Consumer group (optional)
|
|
81
|
+
* @returns {Promise<object>}
|
|
82
|
+
*/
|
|
83
|
+
async getQueueDepth(name, group = null) {
|
|
84
|
+
logger.log('Admin.getQueueDepth', { name, group })
|
|
85
|
+
const queryString = group ? `?group=${encodeURIComponent(group)}` : ''
|
|
86
|
+
return this.#httpClient.get(`/api/v1/resources/queues/${encodeURIComponent(name)}/depth${queryString}`)
|
|
87
|
+
}
|
|
88
|
+
|
|
71
89
|
/**
|
|
72
90
|
* Clear all messages from a queue
|
|
73
91
|
* @param {string} name - Queue name
|
|
@@ -374,22 +374,14 @@ export class QueueBuilder {
|
|
|
374
374
|
throw new Error('Must specify queue, namespace, or task for pop operation')
|
|
375
375
|
}
|
|
376
376
|
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
if (this.#namespace) params.append('namespace', this.#namespace)
|
|
386
|
-
if (this.#task) params.append('task', this.#task)
|
|
387
|
-
if (this.#autoAck) params.append('autoAck', 'true')
|
|
388
|
-
if (this.#subscriptionMode) params.append('subscriptionMode', this.#subscriptionMode)
|
|
389
|
-
if (this.#subscriptionFrom) params.append('subscriptionFrom', this.#subscriptionFrom)
|
|
390
|
-
|
|
391
|
-
return params
|
|
392
|
-
}
|
|
377
|
+
// NOTE: a second, DEAD copy of the pop parameter builder lived here and was
|
|
378
|
+
// deleted with the kv/timers work (PLAN_KV_TIMERS.md §10.4). pop() builds its
|
|
379
|
+
// own params inline, above, because it has to override autoAck with the POP
|
|
380
|
+
// defaults; the dead copy did not. Anyone adding a parameter by looking for
|
|
381
|
+
// the method whose name says "build pop params" would have added it to the
|
|
382
|
+
// copy nobody calls: the pop would keep working and the parameter would
|
|
383
|
+
// simply never arrive, which reads as a server-side mystery and not as a
|
|
384
|
+
// client bug.
|
|
393
385
|
|
|
394
386
|
// ===========================
|
|
395
387
|
// Buffer Management Methods
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Timers (PLAN_KV_TIMERS.md §4, §8.1, §9.6).
|
|
3
|
+
*
|
|
4
|
+
* A timer is a message you promise now and the broker delivers later, into a
|
|
5
|
+
* real queue, through the real log. Four terminals, all EXPLICIT -- nothing on
|
|
6
|
+
* this builder does anything until one of them is called:
|
|
7
|
+
*
|
|
8
|
+
* await queen.timer('orders').key('order-9f1').delay('30s')
|
|
9
|
+
* .payload({ orderId: '9f1' }).schedule()
|
|
10
|
+
* await queen.timer('orders').key('order-9f1').cancel()
|
|
11
|
+
* await queen.timer('orders').key('order-9f1').peek()
|
|
12
|
+
* await queen.timer('orders').list({ limit: 50 })
|
|
13
|
+
*
|
|
14
|
+
* THE FOUR THINGS THIS FILE EXISTS TO GET RIGHT
|
|
15
|
+
*
|
|
16
|
+
* 1. CANCEL USES ITS OWN ROUTE, AND THAT IS NOT A DETAIL (§9.6).
|
|
17
|
+
* `DELETE /api/v1/timers/:queue/*timerKey` is the one route a proxy is
|
|
18
|
+
* forbidden to block; a cancel sent inside `POST /api/v1/timers` inherits
|
|
19
|
+
* the schedule's authorization instead. Since the FIRE never switches
|
|
20
|
+
* itself off, a tenant that can no longer cancel keeps producing messages it
|
|
21
|
+
* cannot stop -- a block there produces the exact opposite of its purpose.
|
|
22
|
+
* So `cancel()` on this builder always uses the DELETE route. (Inside a
|
|
23
|
+
* transaction it necessarily rides the bundle's array and inherits the
|
|
24
|
+
* bundle's fate; see TransactionBuilder, which says so where it happens.)
|
|
25
|
+
*
|
|
26
|
+
* 2. ONLY RELATIVE DURATIONS, IN MILLISECONDS (§4.2, §20.6). `delayMs`, never
|
|
27
|
+
* `delaySeconds` and never an absolute instant: `deliver_at` is computed in
|
|
28
|
+
* Postgres, so there is ONE clock and no broker's skew can enter. The rule
|
|
29
|
+
* of the product is "durations that can be sub-second are in milliseconds,
|
|
30
|
+
* the ones that cannot are in seconds" -- a 250 ms retry backoff is a real
|
|
31
|
+
* and central use of timers, which is why this wire is the millisecond one.
|
|
32
|
+
* A delay in the past is LEGAL and fires on the first cycle.
|
|
33
|
+
*
|
|
34
|
+
* 3. `deliverAt` IS "NOT BEFORE", NEVER "EXACTLY AT". The floor on this stack
|
|
35
|
+
* is a single hop (p50 ~10 ms, fsync ~4 ms) plus one sweep cycle.
|
|
36
|
+
*
|
|
37
|
+
* 4. THERE IS NO TOMBSTONE (§4.4). Once a timer has fired its row is gone, so a
|
|
38
|
+
* later cancel answers `absent` -- **which MAY mean it was already
|
|
39
|
+
* delivered**. `absent` carries `ok:false` for exactly that reason, and the
|
|
40
|
+
* response echoes the `txn` so the authority (the log, in the destination
|
|
41
|
+
* queue) can be consulted without a second API. Any saga that cancels a
|
|
42
|
+
* compensation timer must have the compensating consumer check the saga's
|
|
43
|
+
* KV state before compensating: otherwise "the timer went out 5 ms before
|
|
44
|
+
* the cancel" unwinds a booking that was already shipped.
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
import * as logger from '../utils/logger.js'
|
|
48
|
+
import { generateUUID } from './QueueBuilder.js'
|
|
49
|
+
import { parseDurationMs } from '../kv/expiry.js'
|
|
50
|
+
|
|
51
|
+
/** JSON payloads are encoded for the caller; raw bytes are passed through. */
|
|
52
|
+
function encodePayload(payload) {
|
|
53
|
+
if (payload === undefined) return undefined
|
|
54
|
+
if (typeof Buffer !== 'undefined' && Buffer.isBuffer(payload)) {
|
|
55
|
+
return payload.toString('base64')
|
|
56
|
+
}
|
|
57
|
+
if (payload instanceof Uint8Array) {
|
|
58
|
+
return Buffer.from(payload).toString('base64')
|
|
59
|
+
}
|
|
60
|
+
return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64')
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export class TimerBuilder {
|
|
64
|
+
#httpClient
|
|
65
|
+
#queue
|
|
66
|
+
#sink
|
|
67
|
+
#timerKey = null
|
|
68
|
+
#partition = null
|
|
69
|
+
#delayMs = null
|
|
70
|
+
#payloadB64 = undefined
|
|
71
|
+
#txn = null
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* `sink`, when present, receives the finished op instead of the network:
|
|
75
|
+
* that is how the same builder serves `queen.timer(...)` and
|
|
76
|
+
* `tx.timer(...)`, so the wire shape of a timer op is written once.
|
|
77
|
+
*/
|
|
78
|
+
constructor(httpClient, queue, sink = null) {
|
|
79
|
+
if (typeof queue !== 'string' || queue.length === 0) {
|
|
80
|
+
throw new Error('timer: a queue name is required — a timer without a queue has nowhere to be delivered')
|
|
81
|
+
}
|
|
82
|
+
this.#httpClient = httpClient
|
|
83
|
+
this.#queue = queue
|
|
84
|
+
this.#sink = sink
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** The timer's identity inside its queue. `(queue, timerKey)` is the primary key. */
|
|
88
|
+
key(timerKey) {
|
|
89
|
+
if (typeof timerKey !== 'string' || timerKey.length === 0) {
|
|
90
|
+
throw new Error('timer: timerKey must be a non-empty string')
|
|
91
|
+
}
|
|
92
|
+
this.#timerKey = timerKey
|
|
93
|
+
return this
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Destination partition of the delivered message. Defaults to 'Default'. */
|
|
97
|
+
partition(name) {
|
|
98
|
+
this.#partition = name
|
|
99
|
+
return this
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Fire no earlier than this many milliseconds from now. Negative is legal: it fires on the first cycle. */
|
|
103
|
+
delayMs(ms) {
|
|
104
|
+
if (typeof ms !== 'number' || !Number.isFinite(ms)) {
|
|
105
|
+
throw new Error(`timer: delayMs must be a finite number of milliseconds, got ${JSON.stringify(ms)}`)
|
|
106
|
+
}
|
|
107
|
+
this.#delayMs = Math.round(ms)
|
|
108
|
+
return this
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** The same delay as a duration string: '250ms', '30s', '2h'. Converted to delayMs here. */
|
|
112
|
+
delay(duration) {
|
|
113
|
+
this.#delayMs = Math.round(parseDurationMs(duration))
|
|
114
|
+
return this
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** The message body. Any JSON value, or a Buffer/Uint8Array for raw bytes. */
|
|
118
|
+
payload(payload) {
|
|
119
|
+
this.#payloadB64 = encodePayload(payload)
|
|
120
|
+
return this
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* The transaction id the delivered message will carry. Minted when absent.
|
|
125
|
+
*
|
|
126
|
+
* §20.2: every reschedule overwrites it, because a rescheduled timer is a
|
|
127
|
+
* NEW message -- and the corollary that must travel with it is that there is
|
|
128
|
+
* no dedup net on the fire at all, so rescheduling or republishing a timer
|
|
129
|
+
* that has already gone out produces a second message in the log and nothing
|
|
130
|
+
* stops it.
|
|
131
|
+
*/
|
|
132
|
+
txn(transactionId) {
|
|
133
|
+
this.#txn = transactionId
|
|
134
|
+
return this
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// ------------------------------------------------------------ op shapes
|
|
138
|
+
|
|
139
|
+
#scheduleOp() {
|
|
140
|
+
if (!this.#timerKey) throw new Error('timer: key(...) is required to schedule')
|
|
141
|
+
if (this.#delayMs === null) {
|
|
142
|
+
throw new Error('timer: delayMs(...) or delay(...) is required — an absolute instant is not expressible on this wire')
|
|
143
|
+
}
|
|
144
|
+
if (this.#payloadB64 === undefined) throw new Error('timer: payload(...) is required to schedule')
|
|
145
|
+
const op = {
|
|
146
|
+
op: 'schedule',
|
|
147
|
+
queue: this.#queue,
|
|
148
|
+
timerKey: this.#timerKey
|
|
149
|
+
}
|
|
150
|
+
if (this.#partition !== null && this.#partition !== undefined) op.partition = this.#partition
|
|
151
|
+
op.delayMs = this.#delayMs
|
|
152
|
+
op.txn = this.#txn || generateUUID()
|
|
153
|
+
op.payload = this.#payloadB64
|
|
154
|
+
return op
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
#cancelOp() {
|
|
158
|
+
if (!this.#timerKey) throw new Error('timer: key(...) is required to cancel')
|
|
159
|
+
const op = { op: 'cancel', queue: this.#queue, timerKey: this.#timerKey }
|
|
160
|
+
if (this.#txn) op.txn = this.#txn
|
|
161
|
+
return op
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
#path(suffix = '') {
|
|
165
|
+
return `/api/v1/timers/${encodeURIComponent(this.#queue)}${suffix}`
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
#keyPath() {
|
|
169
|
+
return this.#path(`/${encodeURIComponent(this.#timerKey)}`)
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
#single(body, what) {
|
|
173
|
+
if (!body || typeof body !== 'object' || !Array.isArray(body.results)) {
|
|
174
|
+
throw new Error(`timer: unexpected ${what} response envelope — expected {"results":[...]}`)
|
|
175
|
+
}
|
|
176
|
+
if (body.results.length !== 1) {
|
|
177
|
+
throw new Error(`timer: got ${body.results.length} results for 1 operation`)
|
|
178
|
+
}
|
|
179
|
+
return body.results[0]
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ------------------------------------------------------------ terminals
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Schedule (or reschedule) this timer. An UPSERT on `(queue, timerKey)`, so
|
|
186
|
+
* a retry after a client crash is safe by construction; the answer says
|
|
187
|
+
* which it was, `status: 'scheduled' | 'rescheduled'`.
|
|
188
|
+
*
|
|
189
|
+
* A reschedule resets `attempts` and clears `last_error`: a rescheduled
|
|
190
|
+
* timer is a new timer under an old name, and a freshly corrected payload
|
|
191
|
+
* must not inherit the budget consumed by the one that was poisoning it.
|
|
192
|
+
*
|
|
193
|
+
* `too_late` (with `ok:false`) means the timer is already claimed by a
|
|
194
|
+
* broker that is about to commit it. Bounded by the sweeper lease. The
|
|
195
|
+
* remedy is a new key, or waiting for the delivery and acting on the message.
|
|
196
|
+
*/
|
|
197
|
+
// NOT `async`, deliberately: with a sink this terminal is the transaction's
|
|
198
|
+
// own chaining call and must hand back the TransactionBuilder itself, not a
|
|
199
|
+
// promise of it, or `.timer(q)...schedule().commit()` stops being a chain.
|
|
200
|
+
// The network path returns the promise, so `await ...schedule()` is
|
|
201
|
+
// unchanged for the standalone caller.
|
|
202
|
+
schedule() {
|
|
203
|
+
if (this.#sink) return this.#sink(this.#scheduleOp())
|
|
204
|
+
return this.#scheduleRemote()
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// The op is built INSIDE the async function so that a shape error on the
|
|
208
|
+
// network path is a rejected promise like every other failure of this
|
|
209
|
+
// client, and not a synchronous throw that `.catch()` would miss. On the
|
|
210
|
+
// transaction path it necessarily stays synchronous: there is no promise
|
|
211
|
+
// there to reject.
|
|
212
|
+
async #scheduleRemote() {
|
|
213
|
+
const op = this.#scheduleOp()
|
|
214
|
+
logger.log('TimerBuilder.schedule', { queue: this.#queue, timerKey: op.timerKey, delayMs: op.delayMs })
|
|
215
|
+
const body = await this.#httpClient.post('/api/v1/timers', { operations: [op] })
|
|
216
|
+
return this.#single(body, 'schedule')
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Cancel this timer, through the route that is never blockable (§9.6).
|
|
221
|
+
*
|
|
222
|
+
* Idempotent: `absent` answers `ok:false` and MAY MEAN ALREADY DELIVERED --
|
|
223
|
+
* there is no tombstone. Pass `.txn(...)` and it comes back in the answer,
|
|
224
|
+
* so the log can settle the question.
|
|
225
|
+
*/
|
|
226
|
+
cancel() {
|
|
227
|
+
if (this.#sink) return this.#sink(this.#cancelOp())
|
|
228
|
+
return this.#cancelRemote()
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
async #cancelRemote() {
|
|
232
|
+
const op = this.#cancelOp()
|
|
233
|
+
logger.log('TimerBuilder.cancel', { queue: this.#queue, timerKey: op.timerKey })
|
|
234
|
+
const query = op.txn ? `?txn=${encodeURIComponent(op.txn)}` : ''
|
|
235
|
+
return this.#httpClient.delete(`${this.#keyPath()}${query}`)
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Read one pending timer, payload included. A miss is `{found:false}` with HTTP 200, never a 404. */
|
|
239
|
+
async peek() {
|
|
240
|
+
if (this.#sink) throw new Error('timer: peek is a read, not a transaction operation — use queen.timer(...).peek()')
|
|
241
|
+
if (!this.#timerKey) throw new Error('timer: key(...) is required to peek')
|
|
242
|
+
logger.log('TimerBuilder.peek', { queue: this.#queue, timerKey: this.#timerKey })
|
|
243
|
+
return this.#httpClient.get(this.#keyPath())
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* List the pending timers of this queue: `{rows, truncated, nextAfter}`.
|
|
248
|
+
*
|
|
249
|
+
* The queue is mandatory and that is why it is a path segment rather than a
|
|
250
|
+
* filter: a tenant-wide list would be a scan that an end user of the
|
|
251
|
+
* customer could trigger. `after` is an exclusive keyset cursor.
|
|
252
|
+
*/
|
|
253
|
+
async list(opts = {}) {
|
|
254
|
+
if (this.#sink) throw new Error('timer: list is a read, not a transaction operation — use queen.timer(...).list()')
|
|
255
|
+
const params = new URLSearchParams()
|
|
256
|
+
if (opts.after !== undefined && opts.after !== null && opts.after !== '') params.append('after', opts.after)
|
|
257
|
+
if (opts.limit !== undefined && opts.limit !== null) params.append('limit', String(opts.limit))
|
|
258
|
+
const query = params.toString()
|
|
259
|
+
logger.log('TimerBuilder.list', { queue: this.#queue, after: opts.after, limit: opts.limit })
|
|
260
|
+
return this.#httpClient.get(this.#path(query ? `?${query}` : ''))
|
|
261
|
+
}
|
|
262
|
+
}
|