queen-mq 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -0
- package/client-v2/Queen.js +54 -0
- package/client-v2/README.md +32 -0
- package/client-v2/builders/QueueBuilder.js +8 -16
- package/client-v2/builders/TimerBuilder.js +262 -0
- package/client-v2/builders/TransactionBuilder.js +185 -8
- package/client-v2/kv/Kv.js +432 -0
- package/client-v2/kv/expiry.js +148 -0
- package/package.json +11 -3
- package/test-v2/_kvtimers.js +71 -0
- package/test-v2/ackwindow.js +10 -3
- package/test-v2/consume.js +0 -2
- package/test-v2/docs.js +204 -0
- package/test-v2/http-unit/retry429.test.js +6 -1
- package/test-v2/kv-unit/_planServer.js +65 -0
- package/test-v2/kv-unit/kvWire.test.js +377 -0
- package/test-v2/kv-unit/timerWire.test.js +177 -0
- package/test-v2/kv-unit/txnWire.test.js +222 -0
- package/test-v2/kv.js +273 -0
- package/test-v2/pop.js +0 -2
- package/test-v2/push.js +0 -4
- package/test-v2/run.js +34 -4
- package/test-v2/semantics.js +16 -7
- package/test-v2/stream/_helpers.js +7 -0
- package/test-v2/stream/combined.js +4 -3
- package/test-v2/stream/cron.js +1 -1
- package/test-v2/stream/eventTime.js +8 -5
- package/test-v2/stream/operators.js +5 -5
- package/test-v2/stream/recovery.js +4 -1
- package/test-v2/stream/session.js +3 -3
- package/test-v2/stream/sliding.js +1 -1
- package/test-v2/stream/throughput.js +3 -2
- package/test-v2/stream/tumbling.js +6 -6
- package/test-v2/timers.js +209 -0
- package/test-v2/transaction.js +4 -3
|
@@ -1,15 +1,46 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Transaction builder for atomic operations
|
|
3
|
+
*
|
|
4
|
+
* KV AND TIMER RIDERS (PLAN_KV_TIMERS.md §6.3, §8.2, §10.4)
|
|
5
|
+
*
|
|
6
|
+
* `kv` and `timers` are TOP-LEVEL fields of the request body, beside
|
|
7
|
+
* `operations` and NEVER elements of it. That is a wire decision, not a style
|
|
8
|
+
* one, and the reason is a silent failure in another language that this shape
|
|
9
|
+
* has to respect because one broker parses one wire for all seven clients: two
|
|
10
|
+
* Go struct fields carrying the same JSON key at the same level are BOTH
|
|
11
|
+
* DROPPED by encoding/json, with no error and no warning. A bundle would go
|
|
12
|
+
* out with zero kv operations, the broker would commit a transaction with no
|
|
13
|
+
* gate, and the `putIfAbsent` the bundle existed for would never have
|
|
14
|
+
* happened. An op that still arrives as `{"type":"kv"}` inside `operations`
|
|
15
|
+
* gets a named 400 from the broker, which is the best failure available.
|
|
16
|
+
*
|
|
17
|
+
* A bundle that carries neither array produces exactly the body it produces
|
|
18
|
+
* today -- no `kv: []`, no `timers: null` -- so nothing changes for anyone who
|
|
19
|
+
* does not use the feature, including brokers that predate it.
|
|
20
|
+
*
|
|
21
|
+
* WHY THE RIDERS ARE WORTH THE PARAGRAPH: the transaction is the PRIMARY
|
|
22
|
+
* fence, and `expect` only the secondary assertion. A KV write that shares the
|
|
23
|
+
* transaction with the ack is undone when an expired lease makes the ack fail
|
|
24
|
+
* -- something a CAS cannot do, because an `expect` on a still-matching
|
|
25
|
+
* version succeeds even from a zombie worker.
|
|
3
26
|
*/
|
|
4
27
|
|
|
5
28
|
import * as logger from '../utils/logger.js'
|
|
6
29
|
import { generateUUID } from './QueueBuilder.js'
|
|
7
30
|
import { isValidUUID } from '../utils/validation.js'
|
|
31
|
+
import { kvOp, materializeKvOp } from '../kv/Kv.js'
|
|
32
|
+
import { TimerBuilder } from './TimerBuilder.js'
|
|
8
33
|
|
|
9
34
|
export class TransactionBuilder {
|
|
10
35
|
#httpClient
|
|
11
36
|
#operations = []
|
|
12
37
|
#requiredLeases = []
|
|
38
|
+
// Unresolved on purpose: `until` is an instant, and freezing it when the op
|
|
39
|
+
// was queued would ship a TTL already stale by however long the bundle took
|
|
40
|
+
// to assemble. Materialized in commit(), which is send time.
|
|
41
|
+
#kvEntries = []
|
|
42
|
+
#timerOps = []
|
|
43
|
+
#kvApi = null
|
|
13
44
|
|
|
14
45
|
constructor(httpClient) {
|
|
15
46
|
this.#httpClient = httpClient
|
|
@@ -115,23 +146,169 @@ export class TransactionBuilder {
|
|
|
115
146
|
return subBuilder
|
|
116
147
|
}
|
|
117
148
|
|
|
149
|
+
// ===========================
|
|
150
|
+
// KV rider (§5, §8.2)
|
|
151
|
+
// ===========================
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* KV operations that commit with this transaction.
|
|
155
|
+
*
|
|
156
|
+
* await queen.transaction()
|
|
157
|
+
* .ack(msg)
|
|
158
|
+
* .kv.put('saga', sagaId, { step: 'charged' }, { ttl: '24h' })
|
|
159
|
+
* .commit()
|
|
160
|
+
*
|
|
161
|
+
* Every method returns the TRANSACTION, so the chain keeps reading like one.
|
|
162
|
+
* Same op shapes as `queen.kv`, built by the same code, so the two surfaces
|
|
163
|
+
* cannot drift.
|
|
164
|
+
*
|
|
165
|
+
* `getPrefix` is deliberately absent from this wire and throws here rather
|
|
166
|
+
* than at the broker: it is unbounded read work inside the transaction that
|
|
167
|
+
* holds the outermost lock space and, downstream, the partition ones. `get`
|
|
168
|
+
* and `getMany` are allowed because the caller fixes their cost -- the
|
|
169
|
+
* boundary is COST, not the kind of operation.
|
|
170
|
+
*/
|
|
171
|
+
get kv() {
|
|
172
|
+
if (this.#kvApi) return this.#kvApi
|
|
173
|
+
const add = (entry) => {
|
|
174
|
+
this.#kvEntries.push(entry)
|
|
175
|
+
return this
|
|
176
|
+
}
|
|
177
|
+
this.#kvApi = {
|
|
178
|
+
get: (ns, key) => add(kvOp.get(ns, key)),
|
|
179
|
+
getMany: (ns, keys) => add(kvOp.getMany(ns, keys)),
|
|
180
|
+
put: (ns, key, value, opts = {}) => add(kvOp.put(ns, key, value, opts)),
|
|
181
|
+
putIfAbsent: (ns, key, value, opts = {}) => add(kvOp.putIfAbsent(ns, key, value, opts)),
|
|
182
|
+
delete: (ns, key, opts = {}) => add(kvOp.delete(ns, key, opts)),
|
|
183
|
+
incr: (ns, key, delta = 1, opts = {}) => add(kvOp.incr(ns, key, delta, opts)),
|
|
184
|
+
getPrefix: () => {
|
|
185
|
+
throw new Error(
|
|
186
|
+
'kv: getPrefix is not available inside a transaction — unbounded read work under the outermost ' +
|
|
187
|
+
'lock space. Use POST /api/v1/kv (queen.kv.getPrefix / queen.kv.listAll) outside the bundle.'
|
|
188
|
+
)
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return this.#kvApi
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* The gate, which is the reason this feature exists: do the bundle at most
|
|
196
|
+
* once.
|
|
197
|
+
*
|
|
198
|
+
* const res = await queen.transaction()
|
|
199
|
+
* .ack(msg)
|
|
200
|
+
* .queue('emails').push([{ data: mail }])
|
|
201
|
+
* .once('test-idem', orderId, { ttl: '24h' })
|
|
202
|
+
* .commit()
|
|
203
|
+
* if (res.success === false) return // a redelivery: already done
|
|
204
|
+
*
|
|
205
|
+
* `putIfAbsent` with `required:true`, so a marker that already exists ABORTS
|
|
206
|
+
* the transaction: the push and the ack roll back together with it. That is
|
|
207
|
+
* what makes "the email is sent exactly once" a property of the database
|
|
208
|
+
* rather than a hope about redelivery.
|
|
209
|
+
*
|
|
210
|
+
* The verdict is RETURNED by `commit()`, not thrown -- see there.
|
|
211
|
+
*/
|
|
212
|
+
once(ns, key, opts = {}) {
|
|
213
|
+
const value = opts.value !== undefined ? opts.value : true
|
|
214
|
+
const required = opts.required === false ? {} : { required: true }
|
|
215
|
+
return this.kv.putIfAbsent(ns, key, value, { ...opts, ...required })
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// ===========================
|
|
219
|
+
// Timer rider (§4, §9.6)
|
|
220
|
+
// ===========================
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Schedule or cancel a timer as part of this transaction.
|
|
224
|
+
*
|
|
225
|
+
* await queen.transaction()
|
|
226
|
+
* .ack(msg)
|
|
227
|
+
* .timer('reminders').key(orderId).delay('24h').payload({ orderId }).schedule()
|
|
228
|
+
* .commit()
|
|
229
|
+
*
|
|
230
|
+
* `.schedule()` and `.cancel()` are the terminals; they add the op and hand
|
|
231
|
+
* the transaction back. `peek` and `list` are reads and are not transaction
|
|
232
|
+
* operations.
|
|
233
|
+
*
|
|
234
|
+
* ONE ASYMMETRY TO KNOW ABOUT (§9.6): a cancel sent this way rides the
|
|
235
|
+
* bundle and shares its fate, including its authorization. The cancel that
|
|
236
|
+
* is guaranteed never to be blocked is the standalone
|
|
237
|
+
* `queen.timer(q).key(k).cancel()`, on its own DELETE route. Cancel inside a
|
|
238
|
+
* bundle when you need atomicity with an ack; cancel outside it when you
|
|
239
|
+
* need the cancel to land no matter what.
|
|
240
|
+
*/
|
|
241
|
+
timer(queueName) {
|
|
242
|
+
return new TimerBuilder(null, queueName, (op) => {
|
|
243
|
+
this.#timerOps.push(op)
|
|
244
|
+
return this
|
|
245
|
+
})
|
|
246
|
+
}
|
|
247
|
+
|
|
118
248
|
async commit() {
|
|
119
|
-
|
|
249
|
+
const riderCount = this.#kvEntries.length + this.#timerOps.length
|
|
250
|
+
if (this.#operations.length === 0 && riderCount === 0) {
|
|
120
251
|
logger.error('TransactionBuilder.commit', 'No operations to commit')
|
|
121
252
|
throw new Error('Transaction has no operations to commit')
|
|
122
253
|
}
|
|
123
254
|
|
|
124
|
-
logger.log('TransactionBuilder.commit', {
|
|
255
|
+
logger.log('TransactionBuilder.commit', {
|
|
256
|
+
operationCount: this.#operations.length,
|
|
257
|
+
requiredLeases: this.#requiredLeases.length,
|
|
258
|
+
kv: this.#kvEntries.length,
|
|
259
|
+
timers: this.#timerOps.length
|
|
260
|
+
})
|
|
261
|
+
|
|
262
|
+
// Byte-identity when the riders are absent (§6.3): the keys are added only
|
|
263
|
+
// when there is something in them, so a bundle that uses neither feature
|
|
264
|
+
// produces exactly the body it produced before this feature existed.
|
|
265
|
+
const body = {
|
|
266
|
+
operations: this.#operations,
|
|
267
|
+
requiredLeases: [...new Set(this.#requiredLeases)] // Unique leases
|
|
268
|
+
}
|
|
269
|
+
if (this.#kvEntries.length > 0) {
|
|
270
|
+
const now = Date.now()
|
|
271
|
+
body.kv = this.#kvEntries.map(e => materializeKvOp(e, now))
|
|
272
|
+
}
|
|
273
|
+
if (this.#timerOps.length > 0) {
|
|
274
|
+
body.timers = this.#timerOps
|
|
275
|
+
}
|
|
125
276
|
|
|
126
277
|
try {
|
|
127
|
-
const result = await this.#httpClient.post('/api/v1/transaction',
|
|
128
|
-
operations: this.#operations,
|
|
129
|
-
requiredLeases: [...new Set(this.#requiredLeases)] // Unique leases
|
|
130
|
-
})
|
|
278
|
+
const result = await this.#httpClient.post('/api/v1/transaction', body)
|
|
131
279
|
|
|
132
280
|
if (!result.success) {
|
|
133
|
-
|
|
134
|
-
|
|
281
|
+
// THE ONE OUTCOME THAT RETURNS INSTEAD OF THROWING (§8.3).
|
|
282
|
+
//
|
|
283
|
+
// A lost `required` KV precondition is not a failure: it is the
|
|
284
|
+
// EXPECTED outcome of every legitimate redelivery -- the idempotency
|
|
285
|
+
// marker doing its job -- and it arrives as HTTP 200 with
|
|
286
|
+
// `success:false, reason:'kv_precondition'` precisely so that it stays
|
|
287
|
+
// out of retry policies and error metrics. Throwing it would put the
|
|
288
|
+
// single most frequent outcome of this product inside every caller's
|
|
289
|
+
// catch block, where the natural reflex is to retry, which is the one
|
|
290
|
+
// thing that must not happen.
|
|
291
|
+
//
|
|
292
|
+
// The body carries `failedIndex` (in the FLAT index space of
|
|
293
|
+
// `results[]`), `kvReason`, `version` and `value`, so the caller can
|
|
294
|
+
// see who won without a second round trip.
|
|
295
|
+
if (result.reason === 'kv_precondition') {
|
|
296
|
+
logger.log('TransactionBuilder.commit', {
|
|
297
|
+
status: 'kv_precondition',
|
|
298
|
+
failedIndex: result.failedIndex,
|
|
299
|
+
kvReason: result.kvReason
|
|
300
|
+
})
|
|
301
|
+
return result
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
logger.error('TransactionBuilder.commit', { error: result.error, reason: result.reason })
|
|
305
|
+
const error = new Error(result.error || 'Transaction failed')
|
|
306
|
+
// The closed-taxonomy code travels ON the error, so no caller ever has
|
|
307
|
+
// to match the message: bad_request | duplicate | ack_rejected |
|
|
308
|
+
// timer_horizon_exceeded | payload_too_large | misaligned | db_error.
|
|
309
|
+
if (result.reason) error.reason = result.reason
|
|
310
|
+
error.result = result
|
|
311
|
+
throw error
|
|
135
312
|
}
|
|
136
313
|
|
|
137
314
|
logger.log('TransactionBuilder.commit', { status: 'success' })
|
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The KV surface (PLAN_KV_TIMERS.md §5, §8.1).
|
|
3
|
+
*
|
|
4
|
+
* Seven operations, five code paths: get, getMany, getPrefix, put,
|
|
5
|
+
* putIfAbsent (an alias that desugars to put with expect:0 inside the stored
|
|
6
|
+
* procedure), delete, incr. Plus two conveniences this client owns: `once`,
|
|
7
|
+
* which is the idempotency marker written the way people actually reach for
|
|
8
|
+
* it, and `listAll`, which walks the keyset cursor of getPrefix.
|
|
9
|
+
*
|
|
10
|
+
* EVERYTHING GOES THROUGH `POST /api/v1/kv`, INCLUDING THE SINGLE-KEY READS.
|
|
11
|
+
* The path routes (`GET|PUT|DELETE /api/v1/kv/:ns/*key`) exist and are correct,
|
|
12
|
+
* but they are sugar "for the three cases people write by hand" (§8.1). An SDK
|
|
13
|
+
* is not one of them, and the batch route buys three things a path route
|
|
14
|
+
* cannot: it is the ONLY surface that accepts `incr` and `getPrefix`, so one
|
|
15
|
+
* transport serves all seven ops; it never has to percent-encode a key into a
|
|
16
|
+
* URL; and it keeps keys out of access logs, proxy samples and tracing spans,
|
|
17
|
+
* which is the same reasoning §5.5 applies to prefixes and which does not stop
|
|
18
|
+
* being true for a key.
|
|
19
|
+
*
|
|
20
|
+
* THE STATUS-CODE RULE YOU WILL NOTICE FIRST (§8.1): the HTTP status describes
|
|
21
|
+
* the outcome of the CALL, never the verdict of the predicate. An absent key, a
|
|
22
|
+
* lost putIfAbsent race, a delete that hit nothing -- all 200, with an explicit
|
|
23
|
+
* field in the body. `applied:false` is the single most frequent outcome of
|
|
24
|
+
* this product and it is not an error.
|
|
25
|
+
*
|
|
26
|
+
* WHICH LEADS TO THE ONE TRAP THIS LANGUAGE CANNOT DEFEND AGAINST
|
|
27
|
+
* STRUCTURALLY (§10.4): **every write returns an OBJECT, and objects are
|
|
28
|
+
* always truthy**.
|
|
29
|
+
*
|
|
30
|
+
* if (await kv.delete(ns, key)) { ... } // ALWAYS TAKEN. A bug.
|
|
31
|
+
* const r = await kv.delete(ns, key)
|
|
32
|
+
* if (r.applied) { ... } // the field is the answer
|
|
33
|
+
*
|
|
34
|
+
* That holds for all five writes -- put, putIfAbsent, delete, incr, once (whose
|
|
35
|
+
* field is `won`).
|
|
36
|
+
*
|
|
37
|
+
* AND THE ONE ABOUT EXPIRY (§5.7): a key that has expired is never returned and
|
|
38
|
+
* never counts as existing, even before the sweeper has pruned it. A `put`
|
|
39
|
+
* does NOT inherit the previous TTL -- that is not expressible, because a put
|
|
40
|
+
* that silently inherited an expiry is the fastest way to make a marker
|
|
41
|
+
* immortal.
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
import * as logger from '../utils/logger.js'
|
|
45
|
+
import { resolveExpiry } from './expiry.js'
|
|
46
|
+
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
// Op builders. Shared with TransactionBuilder, which puts the very same
|
|
49
|
+
// objects in the `kv` array of a bundle -- one definition of the wire shape,
|
|
50
|
+
// so the standalone route and the transaction rider can never drift.
|
|
51
|
+
//
|
|
52
|
+
// An entry is `{ base, expiry }`: `base` is the finished op minus its expiry,
|
|
53
|
+
// `expiry` is the caller's sugar, unresolved. `materializeKvOp` resolves it,
|
|
54
|
+
// and it is called at SEND time -- immediately here, at commit() there --
|
|
55
|
+
// because `until` is an instant and freezing it when the op was queued would
|
|
56
|
+
// ship a TTL that is already stale.
|
|
57
|
+
// ---------------------------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
function requireName(value, what) {
|
|
60
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
61
|
+
throw new Error(`kv: ${what} must be a non-empty string, got ${JSON.stringify(value)}`)
|
|
62
|
+
}
|
|
63
|
+
return value
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* §5.3: an explicitly undefined or null `expect` is a CLIENT-SIDE BUG, never a
|
|
68
|
+
* silent downgrade to upsert. The caller wrote the word `expect`, so they
|
|
69
|
+
* declared the intention to fence; sending the op without it would perform the
|
|
70
|
+
* unconditional write the fence existed to prevent.
|
|
71
|
+
*/
|
|
72
|
+
function withExpect(base, opts) {
|
|
73
|
+
if ('expect' in opts) {
|
|
74
|
+
if (opts.expect === undefined || opts.expect === null) {
|
|
75
|
+
throw new Error(
|
|
76
|
+
'kv: `expect` was written but has no value. An absent expect is not an upsert — it is a bug in the ' +
|
|
77
|
+
'caller: drop the field to mean "unconditional", or pass 0 to mean "must not exist".'
|
|
78
|
+
)
|
|
79
|
+
}
|
|
80
|
+
base.expect = opts.expect
|
|
81
|
+
}
|
|
82
|
+
return base
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function withRequired(base, opts) {
|
|
86
|
+
if (opts.required === true) base.required = true
|
|
87
|
+
return base
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export const kvOp = {
|
|
91
|
+
get(ns, key) {
|
|
92
|
+
return { base: { op: 'get', ns: requireName(ns, 'ns'), key: requireName(key, 'key') }, expiry: null }
|
|
93
|
+
},
|
|
94
|
+
|
|
95
|
+
getMany(ns, keys) {
|
|
96
|
+
requireName(ns, 'ns')
|
|
97
|
+
if (!Array.isArray(keys) || keys.length === 0) {
|
|
98
|
+
throw new Error('kv: getMany needs a non-empty array of keys')
|
|
99
|
+
}
|
|
100
|
+
keys.forEach(k => requireName(k, 'key'))
|
|
101
|
+
return { base: { op: 'getMany', ns, keys: [...keys] }, expiry: null }
|
|
102
|
+
},
|
|
103
|
+
|
|
104
|
+
getPrefix(ns, prefix, opts = {}) {
|
|
105
|
+
requireName(ns, 'ns')
|
|
106
|
+
// A namespace is not a table to enumerate: the empty prefix is the declared
|
|
107
|
+
// boundary, not an oversight (§5.5).
|
|
108
|
+
requireName(prefix, 'prefix')
|
|
109
|
+
const base = { op: 'getPrefix', ns, prefix }
|
|
110
|
+
if (opts.limit !== undefined) base.limit = opts.limit
|
|
111
|
+
if (opts.after !== undefined && opts.after !== null) base.after = opts.after
|
|
112
|
+
if (opts.keysOnly === true) base.keysOnly = true
|
|
113
|
+
return { base, expiry: null }
|
|
114
|
+
},
|
|
115
|
+
|
|
116
|
+
put(ns, key, value, opts = {}) {
|
|
117
|
+
requireName(ns, 'ns')
|
|
118
|
+
requireName(key, 'key')
|
|
119
|
+
const base = { op: 'put', ns, key, value }
|
|
120
|
+
return { base: withRequired(withExpect(base, opts), opts), expiry: opts }
|
|
121
|
+
},
|
|
122
|
+
|
|
123
|
+
putIfAbsent(ns, key, value, opts = {}) {
|
|
124
|
+
requireName(ns, 'ns')
|
|
125
|
+
requireName(key, 'key')
|
|
126
|
+
if ('expect' in opts) {
|
|
127
|
+
throw new Error('kv: putIfAbsent IS expect:0 — passing a different expect is a contradiction the broker rejects')
|
|
128
|
+
}
|
|
129
|
+
const base = { op: 'putIfAbsent', ns, key, value }
|
|
130
|
+
return { base: withRequired(base, opts), expiry: opts }
|
|
131
|
+
},
|
|
132
|
+
|
|
133
|
+
delete(ns, key, opts = {}) {
|
|
134
|
+
requireName(ns, 'ns')
|
|
135
|
+
requireName(key, 'key')
|
|
136
|
+
const base = { op: 'delete', ns, key }
|
|
137
|
+
return { base: withRequired(withExpect(base, opts), opts), expiry: null }
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
incr(ns, key, delta = 1, opts = {}) {
|
|
141
|
+
requireName(ns, 'ns')
|
|
142
|
+
requireName(key, 'key')
|
|
143
|
+
if ('expect' in opts) {
|
|
144
|
+
// §5.4: incr is the way OUT of CAS. A precondition on it would
|
|
145
|
+
// reintroduce the very loop it exists to remove.
|
|
146
|
+
throw new Error('kv: incr takes no expect — it is the escape from the CAS loop, not another CAS')
|
|
147
|
+
}
|
|
148
|
+
if (typeof delta !== 'number' || !Number.isFinite(delta)) {
|
|
149
|
+
throw new Error(`kv: incr needs a finite numeric delta, got ${JSON.stringify(delta)}`)
|
|
150
|
+
}
|
|
151
|
+
const base = { op: 'incr', ns, key, delta }
|
|
152
|
+
if (opts.min !== undefined) base.min = opts.min
|
|
153
|
+
if (opts.max !== undefined) base.max = opts.max
|
|
154
|
+
return { base: withRequired(base, opts), expiry: opts }
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Finish one op: merge the resolved expiry into the base. */
|
|
159
|
+
export function materializeKvOp(entry, nowMs = Date.now()) {
|
|
160
|
+
if (!entry.expiry) return { ...entry.base }
|
|
161
|
+
return { ...entry.base, ...resolveExpiry(entry.expiry, nowMs) }
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// ---------------------------------------------------------------------------
|
|
165
|
+
// Response handling.
|
|
166
|
+
// ---------------------------------------------------------------------------
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* The KV routes put the CODE in `error` (§9.5's closed taxonomy) and have no
|
|
170
|
+
* separate `code` field, while the proxy's own refusals do. HttpClient maps
|
|
171
|
+
* `body.error` onto `.message`, so on these routes the message IS the code.
|
|
172
|
+
* Copying it into `.code` -- without ever overwriting a proxy-set one -- is
|
|
173
|
+
* what lets callers branch on `err.code` here exactly as they do on a proxy
|
|
174
|
+
* 403, and never on prose, which is forbidden everywhere in this codebase.
|
|
175
|
+
*/
|
|
176
|
+
function decorate(error) {
|
|
177
|
+
if (error && !error.code && typeof error.message === 'string') {
|
|
178
|
+
error.code = error.message
|
|
179
|
+
}
|
|
180
|
+
return error
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** A lost `required` gate arrives as HTTP 200 with this envelope (§8.3). */
|
|
184
|
+
function isPrecondition(body) {
|
|
185
|
+
return body && typeof body === 'object' && body.ok === false && body.reason === 'kv_precondition'
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Render a precondition verdict as a WriteResult, so that `applied` remains
|
|
190
|
+
* THE field to read whichever way the verdict arrived. `precondition:true`
|
|
191
|
+
* says the transaction was rolled back rather than merely refused, and
|
|
192
|
+
* `kvReason` is preserved under the name `reason` the other path uses.
|
|
193
|
+
*/
|
|
194
|
+
function preconditionResult(body, op, key) {
|
|
195
|
+
return {
|
|
196
|
+
op,
|
|
197
|
+
key,
|
|
198
|
+
applied: false,
|
|
199
|
+
precondition: true,
|
|
200
|
+
reason: body.kvReason ?? null,
|
|
201
|
+
value: body.value,
|
|
202
|
+
version: body.version,
|
|
203
|
+
failedIndex: body.failedIndex
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* §5.4: past 2^53 `JSON.parse` loses precision SILENTLY, and `incr` runs on
|
|
209
|
+
* `numeric` server-side while `version` is a BIGINT. A rate limiter reading a
|
|
210
|
+
* counter that is quietly wrong is worse than one that fails, so this raises.
|
|
211
|
+
*/
|
|
212
|
+
function assertSafeNumber(value, what) {
|
|
213
|
+
if (typeof value === 'number' && Math.abs(value) > Number.MAX_SAFE_INTEGER) {
|
|
214
|
+
throw new Error(
|
|
215
|
+
`kv: ${what} is ${value}, beyond 2^53 — JSON.parse has already lost precision on it. ` +
|
|
216
|
+
'Refusing to hand back a number that is silently wrong.'
|
|
217
|
+
)
|
|
218
|
+
}
|
|
219
|
+
return value
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export class Kv {
|
|
223
|
+
#httpClient
|
|
224
|
+
|
|
225
|
+
constructor(httpClient) {
|
|
226
|
+
this.#httpClient = httpClient
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* The one path to the broker. Returns either `{ results }` (index-aligned to
|
|
231
|
+
* the ops, §6.4) or `{ precondition }` for the 200-with-`ok:false` verdict.
|
|
232
|
+
*/
|
|
233
|
+
async #apply(entries) {
|
|
234
|
+
const now = Date.now()
|
|
235
|
+
const operations = entries.map(e => materializeKvOp(e, now))
|
|
236
|
+
logger.log('Kv.apply', { count: operations.length, ops: operations.map(o => o.op) })
|
|
237
|
+
|
|
238
|
+
let body
|
|
239
|
+
try {
|
|
240
|
+
body = await this.#httpClient.post('/api/v1/kv', { operations })
|
|
241
|
+
} catch (error) {
|
|
242
|
+
logger.error('Kv.apply', { error: error.message, status: error.status, code: error.code })
|
|
243
|
+
throw decorate(error)
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (isPrecondition(body)) return { precondition: body }
|
|
247
|
+
|
|
248
|
+
const results = body && Array.isArray(body.results) ? body.results : null
|
|
249
|
+
if (!results) {
|
|
250
|
+
throw new Error('kv: unexpected response envelope — expected {"results":[...]}')
|
|
251
|
+
}
|
|
252
|
+
// The stored procedure guarantees one result per op and raises rather than
|
|
253
|
+
// returning a short array (§6.4). A short one here means something between
|
|
254
|
+
// the two rewrote the answer, and attributing result i to op j is how a
|
|
255
|
+
// caller ends up trusting the wrong verdict.
|
|
256
|
+
if (results.length !== operations.length) {
|
|
257
|
+
throw new Error(`kv: got ${results.length} results for ${operations.length} operations`)
|
|
258
|
+
}
|
|
259
|
+
return { results }
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async #one(entry) {
|
|
263
|
+
const { results, precondition } = await this.#apply([entry])
|
|
264
|
+
if (precondition) return { precondition }
|
|
265
|
+
return { result: results[0] }
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** A read result, unwrapped. Reads have no precondition to lose. */
|
|
269
|
+
async #read(entry) {
|
|
270
|
+
const { result, precondition } = await this.#one(entry)
|
|
271
|
+
if (precondition) {
|
|
272
|
+
// Unreachable by construction (`required` is a write-only escalation),
|
|
273
|
+
// and therefore exactly the kind of thing that must not silently return
|
|
274
|
+
// `undefined` if the broker's envelope ever changes shape.
|
|
275
|
+
throw new Error(`kv: ${entry.base.op} came back as a precondition verdict, which a read cannot lose`)
|
|
276
|
+
}
|
|
277
|
+
return result
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/** A write result, with the precondition envelope folded into the same shape. */
|
|
281
|
+
async #write(entry) {
|
|
282
|
+
const { result, precondition } = await this.#one(entry)
|
|
283
|
+
if (precondition) return preconditionResult(precondition, entry.base.op, entry.base.key)
|
|
284
|
+
if (typeof result.applied !== 'boolean') {
|
|
285
|
+
throw new Error(`kv: ${entry.base.op} result carries no \`applied\` field; refusing to guess the verdict`)
|
|
286
|
+
}
|
|
287
|
+
return result
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// -------------------------------------------------------------- reads
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* One key. Returns the ROW, not the value:
|
|
294
|
+
* `{found, key, value?, version?, expiresAt?, updatedAt?}`.
|
|
295
|
+
*
|
|
296
|
+
* `found` is separate from `value` because `null` is a legal JSONB value
|
|
297
|
+
* (§5.5): `{found:true, value:null}` and `{found:false}` are different
|
|
298
|
+
* things, and an SDK that returned "the value or null" would collapse them.
|
|
299
|
+
*/
|
|
300
|
+
async get(ns, key) {
|
|
301
|
+
return this.#read(kvOp.get(ns, key))
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Many keys of one namespace. Returns `{rows, missing, truncated}`.
|
|
306
|
+
*
|
|
307
|
+
* `missing` is EXPLICIT: absence is a datum, not a hole the caller computes
|
|
308
|
+
* by difference. `truncated` means the byte budget cut the page -- those
|
|
309
|
+
* keys are in neither list, because calling them absent would be a lie.
|
|
310
|
+
*/
|
|
311
|
+
async getMany(ns, keys) {
|
|
312
|
+
return this.#read(kvOp.getMany(ns, keys))
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* One page of a prefix scan: `{rows, truncated, nextAfter}`.
|
|
317
|
+
*
|
|
318
|
+
* `limit` is CLAMPED by the broker (default 100, ceiling 1000) and never
|
|
319
|
+
* rejected, plus a byte ceiling on the aggregate, with `truncated` telling
|
|
320
|
+
* the truth about both. `after` is an exclusive keyset cursor, not an
|
|
321
|
+
* offset.
|
|
322
|
+
*
|
|
323
|
+
* EACH PAGE IS ITS OWN SNAPSHOT. It is not an instant of the namespace: with
|
|
324
|
+
* `after` it can miss a key inserted behind the cursor. Good for compacting
|
|
325
|
+
* state, not for an exact count.
|
|
326
|
+
*/
|
|
327
|
+
async getPrefix(ns, prefix, opts = {}) {
|
|
328
|
+
return this.#read(kvOp.getPrefix(ns, prefix, opts))
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Every row under a prefix, one page at a time.
|
|
333
|
+
*
|
|
334
|
+
* An ASYNC GENERATOR, not an array: the page count is not knowable in
|
|
335
|
+
* advance, and a method that quietly buffered a namespace into memory would
|
|
336
|
+
* be the one thing this API is careful not to offer.
|
|
337
|
+
*
|
|
338
|
+
* for await (const row of kv.listAll(ns, 'quota:acme:')) { ... }
|
|
339
|
+
*
|
|
340
|
+
* It inherits getPrefix's snapshot caveat, one page at a time.
|
|
341
|
+
*/
|
|
342
|
+
async *listAll(ns, prefix, opts = {}) {
|
|
343
|
+
let after = opts.after
|
|
344
|
+
for (;;) {
|
|
345
|
+
const page = await this.getPrefix(ns, prefix, { ...opts, after })
|
|
346
|
+
for (const row of page.rows || []) yield row
|
|
347
|
+
if (!page.truncated) return
|
|
348
|
+
if (!page.nextAfter) return
|
|
349
|
+
after = page.nextAfter
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// ------------------------------------------------------------- writes
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* Write a value. Exactly one of `ttlSeconds` (or its sugar `ttl` / `until`)
|
|
357
|
+
* and `forever:true` is required -- the broker owns that rule so all seven
|
|
358
|
+
* clients inherit it.
|
|
359
|
+
*
|
|
360
|
+
* `expect` is the optimistic lock: `0` means "must not exist" (and wins even
|
|
361
|
+
* against an expired row not yet pruned), `N > 0` is a pure UPDATE that
|
|
362
|
+
* creates NOTHING when it matches no row. The version handed to a loser is
|
|
363
|
+
* ADVISORY -- never reuse it blindly as a fencing token.
|
|
364
|
+
*
|
|
365
|
+
* `required:true` escalates a lost precondition into a rolled-back
|
|
366
|
+
* transaction; on this route that comes back as a WriteResult with
|
|
367
|
+
* `precondition:true`, and inside a bundle it is what makes `commit()`
|
|
368
|
+
* return the verdict.
|
|
369
|
+
*
|
|
370
|
+
* Returns a WriteResult: `{applied, op, key, value, version, reason?}`.
|
|
371
|
+
* ALWAYS TRUTHY -- read `applied`.
|
|
372
|
+
*/
|
|
373
|
+
async put(ns, key, value, opts = {}) {
|
|
374
|
+
return this.#write(kvOp.put(ns, key, value, opts))
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/** put with `expect:0`, under the name of the thing. Same WriteResult, and the loser gets the winner's value. */
|
|
378
|
+
async putIfAbsent(ns, key, value, opts = {}) {
|
|
379
|
+
return this.#write(kvOp.putIfAbsent(ns, key, value, opts))
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Delete a key, optionally fenced with `expect`.
|
|
384
|
+
*
|
|
385
|
+
* Returns a WriteResult. `if (await kv.delete(...))` is ALWAYS TRUE and is a
|
|
386
|
+
* bug: read `.applied`.
|
|
387
|
+
*/
|
|
388
|
+
async delete(ns, key, opts = {}) {
|
|
389
|
+
return this.#write(kvOp.delete(ns, key, opts))
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Add `delta` to a numeric key, atomically and without a CAS loop.
|
|
394
|
+
*
|
|
395
|
+
* With `max`, **`applied` IS the admission decision**: an increment that
|
|
396
|
+
* would break the ceiling does not apply, does not saturate and does not
|
|
397
|
+
* truncate -- it comes back `applied:false, reason:'limit'` with the CURRENT
|
|
398
|
+
* value. Comparing client-side after incrementing means the request that
|
|
399
|
+
* broke the ceiling has already consumed budget.
|
|
400
|
+
*
|
|
401
|
+
* The TTL is CREATE-ONLY: a live row keeps its expiry, so a fixed-window
|
|
402
|
+
* limiter actually closes its window. An expired row counts as zero and
|
|
403
|
+
* starts a new window, which is what makes the limiter one call.
|
|
404
|
+
*/
|
|
405
|
+
async incr(ns, key, delta = 1, opts = {}) {
|
|
406
|
+
const res = await this.#write(kvOp.incr(ns, key, delta, opts))
|
|
407
|
+
assertSafeNumber(res.value, `the counter at ${key}`)
|
|
408
|
+
return res
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* The idempotency marker, written the way it is actually used:
|
|
413
|
+
*
|
|
414
|
+
* const { won } = await kv.once('test-idem', orderId, { ttl: '24h' })
|
|
415
|
+
* if (!won) return // somebody already did this
|
|
416
|
+
*
|
|
417
|
+
* `putIfAbsent` with a default value of `true`. Returns `{won, value,
|
|
418
|
+
* version, result}` -- `value` being the WINNER's value, so the loser never
|
|
419
|
+
* needs a second round trip.
|
|
420
|
+
*
|
|
421
|
+
* And the sentence that has to travel with it: **putIfAbsent plus a TTL is
|
|
422
|
+
* not a distributed lock** (§5.7). A lock that expires is not revoked; the
|
|
423
|
+
* old holder keeps working, it simply no longer has the row. The defence is
|
|
424
|
+
* fencing -- carry the `version` as `expect` on every later write -- and
|
|
425
|
+
* that limits the damage rather than removing it.
|
|
426
|
+
*/
|
|
427
|
+
async once(ns, key, opts = {}) {
|
|
428
|
+
const value = opts.value !== undefined ? opts.value : true
|
|
429
|
+
const res = await this.putIfAbsent(ns, key, value, opts)
|
|
430
|
+
return { won: res.applied === true, value: res.value, version: res.version, result: res }
|
|
431
|
+
}
|
|
432
|
+
}
|