relay-dsh-plugin-events 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/README.zh.md +5 -3
- package/SPEC.md +48 -4
- package/contracts/index.mjs +19 -0
- package/docs/acceptance-scenarios.md +24 -2
- package/docs/test-review.md +90 -0
- package/lib/client.js +1975 -94
- package/lib/client.js.map +1 -1
- package/lib/host-plugin.js +1389 -58
- package/lib/host-plugin.js.map +1 -1
- package/lib/typert.host.js +53 -2
- package/lib/typert.host.js.map +1 -1
- package/package.json +1 -1
package/lib/host-plugin.js
CHANGED
|
@@ -75,10 +75,18 @@ function validateMonitorProvider(provider) {
|
|
|
75
75
|
for (const method of ["prepare", "checkMonitor"]) if (typeof provider[method] !== "function") throw new TypeError(`monitor provider requires ${method}()`);
|
|
76
76
|
return provider;
|
|
77
77
|
}
|
|
78
|
+
function validateBoundEventSourceProvider(provider) {
|
|
79
|
+
if (!provider || typeof provider !== "object") throw new TypeError("bound Event source provider is required");
|
|
80
|
+
if (!/^[a-z][a-z0-9._-]{0,63}$/u.test(provider.id ?? "")) throw new TypeError("bound Event source provider requires a lowercase stable id");
|
|
81
|
+
if (!Array.isArray(provider.sources) || provider.sources.length === 0) throw new TypeError("bound Event source provider requires at least one source");
|
|
82
|
+
for (const source of provider.sources) if (typeof source !== "string" || source.length === 0 || source.length > 128) throw new TypeError("bound Event source provider has an invalid source");
|
|
83
|
+
if (new Set(provider.sources).size !== provider.sources.length) throw new TypeError("bound Event source provider sources must be unique");
|
|
84
|
+
return provider;
|
|
85
|
+
}
|
|
78
86
|
//#endregion
|
|
79
87
|
//#region src/runtime/runtime.mjs
|
|
80
88
|
var RelayRuntime = class {
|
|
81
|
-
constructor({ store, router, inbox, monitorRegistrar = null, workerId = `relay-dispatcher-${randomUUID()}`, leaseMs = 6e4, maxDispatchBatches = 100 }) {
|
|
89
|
+
constructor({ store, router, inbox, monitorRegistrar = null, workerId = `relay-dispatcher-${randomUUID()}`, leaseMs = 6e4, maxDispatchBatches = 100, routingFailureLimit = 3, deliveryFailureLimit = 5, deliveryRetryBaseMs = 1e3 }) {
|
|
82
90
|
assert.ok(store, "store is required");
|
|
83
91
|
assert.equal(typeof router?.route, "function", "router.route is required");
|
|
84
92
|
assert.equal(typeof inbox?.deliver, "function", "inbox.deliver is required");
|
|
@@ -90,16 +98,22 @@ var RelayRuntime = class {
|
|
|
90
98
|
this.workerId = workerId;
|
|
91
99
|
this.leaseMs = leaseMs;
|
|
92
100
|
this.maxDispatchBatches = maxDispatchBatches;
|
|
101
|
+
assert.ok(Number.isSafeInteger(routingFailureLimit) && routingFailureLimit > 0, "routingFailureLimit must be positive");
|
|
102
|
+
this.routingFailureLimit = routingFailureLimit;
|
|
103
|
+
assert.ok(Number.isSafeInteger(deliveryFailureLimit) && deliveryFailureLimit > 0, "deliveryFailureLimit must be positive");
|
|
104
|
+
assert.ok(Number.isSafeInteger(deliveryRetryBaseMs) && deliveryRetryBaseMs > 0, "deliveryRetryBaseMs must be positive");
|
|
105
|
+
this.deliveryFailureLimit = deliveryFailureLimit;
|
|
106
|
+
this.deliveryRetryBaseMs = deliveryRetryBaseMs;
|
|
93
107
|
}
|
|
94
108
|
async registerWaits({ sessionId, taskSummary, context = {}, waits, monitors = [], monitorRearms = [] }) {
|
|
95
|
-
validateWaitRegistration({
|
|
109
|
+
waits = validateWaitRegistration({
|
|
96
110
|
sessionId,
|
|
97
111
|
taskSummary,
|
|
98
112
|
context,
|
|
99
113
|
waits,
|
|
100
114
|
monitors,
|
|
101
115
|
monitorRearms
|
|
102
|
-
});
|
|
116
|
+
}).waits;
|
|
103
117
|
let preparedMonitors = monitors;
|
|
104
118
|
if (monitors.length > 0) {
|
|
105
119
|
assert.ok(this.monitorRegistrar, "monitor proposals require a monitorRegistrar");
|
|
@@ -124,8 +138,54 @@ var RelayRuntime = class {
|
|
|
124
138
|
return this.store.listWaitRegistrations();
|
|
125
139
|
}
|
|
126
140
|
async handleEvent(eventInput) {
|
|
141
|
+
return this.handleUnboundEvent(eventInput, { allowCorrelation: false });
|
|
142
|
+
}
|
|
143
|
+
async handleTrustedEvent(eventInput, { providerId } = {}) {
|
|
144
|
+
assert.equal(typeof providerId, "string", "trusted Event provider id is required");
|
|
145
|
+
return this.handleUnboundEvent(eventInput, { allowCorrelation: true });
|
|
146
|
+
}
|
|
147
|
+
async handleTrustedDismissal(eventInput, { providerId, summary = "The trusted Connector dismissed an unsupported Event." } = {}) {
|
|
148
|
+
validateEventInput(eventInput);
|
|
149
|
+
assert.equal(typeof providerId, "string", "trusted Event provider id is required");
|
|
150
|
+
assert.equal(typeof summary, "string", "trusted dismissal summary is required");
|
|
151
|
+
assert.ok(summary.length > 0 && summary.length <= 2e3, "trusted dismissal summary is invalid");
|
|
152
|
+
const ingestion = this.store.ingestEvent(eventInput, { allowCorrelation: true });
|
|
153
|
+
const eventId = ingestion.event.event_id;
|
|
154
|
+
let currentEvent = this.store.inspectEvent(eventId);
|
|
155
|
+
if (currentEvent.state === "received" || currentEvent.state === "routing") {
|
|
156
|
+
const snapshot = this.store.beginRouting(eventId);
|
|
157
|
+
if (!snapshot.alreadyRouted) {
|
|
158
|
+
const decision = {
|
|
159
|
+
disposition: "dismiss",
|
|
160
|
+
actionable: false,
|
|
161
|
+
deliveries: [],
|
|
162
|
+
evidence: [`trusted provider ${providerId} classified the Event as unsupported`],
|
|
163
|
+
summary
|
|
164
|
+
};
|
|
165
|
+
validateRoutingDecision({
|
|
166
|
+
decision,
|
|
167
|
+
sessions: snapshot.sessions,
|
|
168
|
+
label: `trusted dismissal ${eventId}`
|
|
169
|
+
});
|
|
170
|
+
this.store.recordRoutingAttempt({
|
|
171
|
+
eventId,
|
|
172
|
+
router: `dismiss:${providerId}`,
|
|
173
|
+
output: decision
|
|
174
|
+
});
|
|
175
|
+
this.store.commitRouting(snapshot, decision);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
currentEvent = this.store.inspectEvent(eventId);
|
|
179
|
+
return {
|
|
180
|
+
duplicate: ingestion.duplicate,
|
|
181
|
+
event: currentEvent,
|
|
182
|
+
registrations: [],
|
|
183
|
+
dispatchResults: []
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
async handleUnboundEvent(eventInput, { allowCorrelation }) {
|
|
127
187
|
validateEventInput(eventInput);
|
|
128
|
-
const ingestion = this.store.ingestEvent(eventInput);
|
|
188
|
+
const ingestion = this.store.ingestEvent(eventInput, { allowCorrelation });
|
|
129
189
|
const eventId = ingestion.event.event_id;
|
|
130
190
|
let currentEvent = this.store.inspectEvent(eventId);
|
|
131
191
|
let sessionIds = [];
|
|
@@ -140,6 +200,41 @@ var RelayRuntime = class {
|
|
|
140
200
|
dispatchResults
|
|
141
201
|
};
|
|
142
202
|
}
|
|
203
|
+
async handleBoundEvent(eventInput, binding, { providerId } = {}) {
|
|
204
|
+
validateEventInput(eventInput);
|
|
205
|
+
validateTrustedBinding(binding);
|
|
206
|
+
assert.equal(typeof providerId, "string", "bound Event provider id is required");
|
|
207
|
+
const ingestion = this.store.ingestEvent(eventInput, { allowCorrelation: true });
|
|
208
|
+
const eventId = ingestion.event.event_id;
|
|
209
|
+
let currentEvent = this.store.inspectEvent(eventId);
|
|
210
|
+
let sessionIds = [];
|
|
211
|
+
if (currentEvent.state === "received" || currentEvent.state === "routing") {
|
|
212
|
+
const snapshot = this.store.beginRouting(eventId);
|
|
213
|
+
if (snapshot.alreadyRouted) sessionIds = snapshot.result.sessionIds;
|
|
214
|
+
else {
|
|
215
|
+
const decision = boundRoutingDecision(snapshot.sessions, binding, providerId);
|
|
216
|
+
validateRoutingDecision({
|
|
217
|
+
decision,
|
|
218
|
+
sessions: snapshot.sessions,
|
|
219
|
+
label: `bound event ${eventId}`
|
|
220
|
+
});
|
|
221
|
+
this.store.recordRoutingAttempt({
|
|
222
|
+
eventId,
|
|
223
|
+
router: `bound:${providerId}`,
|
|
224
|
+
output: decision
|
|
225
|
+
});
|
|
226
|
+
sessionIds = this.store.commitRouting(snapshot, decision).sessionIds;
|
|
227
|
+
}
|
|
228
|
+
} else if (currentEvent.state === "dispatched") sessionIds = [...new Set(currentEvent.deliveries.filter((delivery) => delivery.state === "queued").map((delivery) => delivery.session_id))];
|
|
229
|
+
const dispatchResults = await Promise.all(sessionIds.map((sessionId) => this.dispatchSession(sessionId)));
|
|
230
|
+
currentEvent = this.store.inspectEvent(eventId);
|
|
231
|
+
return {
|
|
232
|
+
duplicate: ingestion.duplicate,
|
|
233
|
+
event: currentEvent,
|
|
234
|
+
registrations: sessionIds.map((sessionId) => this.store.inspectWaitRegistration(sessionId)),
|
|
235
|
+
dispatchResults
|
|
236
|
+
};
|
|
237
|
+
}
|
|
143
238
|
async routeEvent(eventId) {
|
|
144
239
|
const snapshot = this.store.beginRouting(eventId);
|
|
145
240
|
if (snapshot.alreadyRouted) return snapshot.result;
|
|
@@ -172,6 +267,16 @@ var RelayRuntime = class {
|
|
|
172
267
|
output: routed?.decision ?? routed ?? null,
|
|
173
268
|
error: error.stack ?? error.message
|
|
174
269
|
});
|
|
270
|
+
if (this.store.countRoutingAttempts(eventId) >= this.routingFailureLimit) {
|
|
271
|
+
const terminal = {
|
|
272
|
+
disposition: "escalate",
|
|
273
|
+
actionable: true,
|
|
274
|
+
deliveries: [],
|
|
275
|
+
evidence: [`Router ${this.router.name ?? "anonymous-router"} exhausted its failure budget.`],
|
|
276
|
+
summary: "Relay could not safely route this actionable Event."
|
|
277
|
+
};
|
|
278
|
+
return this.store.commitRouting(snapshot, terminal);
|
|
279
|
+
}
|
|
175
280
|
throw error;
|
|
176
281
|
}
|
|
177
282
|
}
|
|
@@ -195,12 +300,17 @@ var RelayRuntime = class {
|
|
|
195
300
|
registration = this.store.completeDispatch(sessionId, started.activation.activation_id, this.workerId);
|
|
196
301
|
activationIds.push(started.activation.activation_id);
|
|
197
302
|
} catch (error) {
|
|
198
|
-
this.store.failDispatch(sessionId, started.activation.activation_id, this.workerId, error.stack ?? error.message
|
|
303
|
+
const failure = this.store.failDispatch(sessionId, started.activation.activation_id, this.workerId, error.stack ?? error.message, {
|
|
304
|
+
failureLimit: this.deliveryFailureLimit,
|
|
305
|
+
baseDelayMs: this.deliveryRetryBaseMs
|
|
306
|
+
});
|
|
199
307
|
return {
|
|
200
|
-
status:
|
|
308
|
+
status: failure.status,
|
|
201
309
|
activationId: started.activation.activation_id,
|
|
202
310
|
activationIds,
|
|
203
|
-
error: error.stack ?? error.message
|
|
311
|
+
error: error.stack ?? error.message,
|
|
312
|
+
eventIds: failure.eventIds,
|
|
313
|
+
activation: failure.activation
|
|
204
314
|
};
|
|
205
315
|
}
|
|
206
316
|
}
|
|
@@ -223,6 +333,10 @@ function validateWaitRegistration({ sessionId, taskSummary, context, waits, moni
|
|
|
223
333
|
assert.ok(Array.isArray(wait.entities), "wait entities must be an array");
|
|
224
334
|
assert.equal(typeof wait.prior_exchange, "string", "wait prior_exchange is required");
|
|
225
335
|
}
|
|
336
|
+
const normalizedWaits = waits.map((wait) => ({
|
|
337
|
+
...wait,
|
|
338
|
+
continuation: normalizeContinuation(wait.continuation)
|
|
339
|
+
}));
|
|
226
340
|
assert.ok(Array.isArray(monitors), "monitors must be an array");
|
|
227
341
|
const monitorIds = monitors.map((monitor) => monitor.monitor_id);
|
|
228
342
|
assert.equal(new Set(monitorIds).size, monitorIds.length, "monitor IDs must be unique");
|
|
@@ -240,16 +354,93 @@ function validateWaitRegistration({ sessionId, taskSummary, context, waits, moni
|
|
|
240
354
|
sessionId,
|
|
241
355
|
taskSummary,
|
|
242
356
|
context,
|
|
243
|
-
waits,
|
|
357
|
+
waits: normalizedWaits,
|
|
244
358
|
monitors,
|
|
245
359
|
monitorRearms
|
|
246
360
|
};
|
|
247
361
|
}
|
|
362
|
+
function normalizeContinuation(value) {
|
|
363
|
+
const input = value ?? {};
|
|
364
|
+
assert.ok(input && typeof input === "object" && !Array.isArray(input), "wait continuation must be an object");
|
|
365
|
+
const version = input.version ?? 1;
|
|
366
|
+
assert.equal(version, 1, "wait continuation version must be 1");
|
|
367
|
+
const continuation = {
|
|
368
|
+
version,
|
|
369
|
+
next_action: boundedString(input.next_action ?? "", "continuation next_action", 8e3),
|
|
370
|
+
success_condition: boundedString(input.success_condition ?? "", "continuation success_condition", 4e3),
|
|
371
|
+
constraints: boundedStringArray(input.constraints ?? [], "continuation constraints", 32, 2e3),
|
|
372
|
+
artifacts: normalizeArtifacts(input.artifacts ?? []),
|
|
373
|
+
on_failure: boundedString(input.on_failure ?? "", "continuation on_failure", 4e3),
|
|
374
|
+
on_timeout: boundedString(input.on_timeout ?? "", "continuation on_timeout", 4e3)
|
|
375
|
+
};
|
|
376
|
+
assert.ok(JSON.stringify(continuation).length <= 32e3, "wait continuation exceeds 32000 characters");
|
|
377
|
+
return continuation;
|
|
378
|
+
}
|
|
379
|
+
function normalizeArtifacts(value) {
|
|
380
|
+
assert.ok(Array.isArray(value), "continuation artifacts must be an array");
|
|
381
|
+
assert.ok(value.length <= 32, "continuation artifacts exceeds 32 items");
|
|
382
|
+
return value.map((artifact, index) => {
|
|
383
|
+
assert.ok(artifact && typeof artifact === "object" && !Array.isArray(artifact), `continuation artifact ${index} must be an object`);
|
|
384
|
+
const normalized = {
|
|
385
|
+
kind: boundedString(artifact.kind, `continuation artifact ${index} kind`, 64, true),
|
|
386
|
+
id: boundedString(artifact.id, `continuation artifact ${index} id`, 2048, true)
|
|
387
|
+
};
|
|
388
|
+
if (artifact.label != null) normalized.label = boundedString(artifact.label, `continuation artifact ${index} label`, 512);
|
|
389
|
+
if (artifact.url != null) normalized.url = boundedString(artifact.url, `continuation artifact ${index} url`, 4096);
|
|
390
|
+
return normalized;
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
function boundedStringArray(value, label, maxItems, maxLength) {
|
|
394
|
+
assert.ok(Array.isArray(value), `${label} must be an array`);
|
|
395
|
+
assert.ok(value.length <= maxItems, `${label} exceeds ${maxItems} items`);
|
|
396
|
+
return value.map((item, index) => boundedString(item, `${label} item ${index}`, maxLength));
|
|
397
|
+
}
|
|
398
|
+
function boundedString(value, label, maxLength, required = false) {
|
|
399
|
+
assert.equal(typeof value, "string", `${label} must be a string`);
|
|
400
|
+
if (required) assert.ok(value.length > 0, `${label} is required`);
|
|
401
|
+
assert.ok(value.length <= maxLength, `${label} exceeds ${maxLength} characters`);
|
|
402
|
+
return value;
|
|
403
|
+
}
|
|
248
404
|
function validateEventInput(input) {
|
|
249
405
|
assert.ok(input && typeof input === "object", "event input is required");
|
|
250
406
|
assert.equal(typeof input.source, "string", "event source is required");
|
|
251
407
|
assert.equal(typeof input.fingerprint, "string", "event fingerprint is required");
|
|
252
408
|
}
|
|
409
|
+
function validateTrustedBinding(binding) {
|
|
410
|
+
assert.ok(binding && typeof binding === "object" && !Array.isArray(binding), "trusted binding is required");
|
|
411
|
+
assert.equal(typeof binding.session_id, "string", "trusted binding session_id is required");
|
|
412
|
+
assert.equal(typeof binding.wait_id, "string", "trusted binding wait_id is required");
|
|
413
|
+
if (binding.wait_version != null) assert.ok(Number.isSafeInteger(binding.wait_version) && binding.wait_version >= 0, "trusted binding wait_version is invalid");
|
|
414
|
+
if (binding.source_subject != null) {
|
|
415
|
+
assert.equal(typeof binding.source_subject, "string", "trusted binding source_subject must be a string");
|
|
416
|
+
assert.ok(binding.source_subject.length <= 2048, "trusted binding source_subject exceeds 2048 characters");
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
function boundRoutingDecision(sessions, binding, providerId) {
|
|
420
|
+
const session = sessions.find((candidate) => candidate.session_id === binding.session_id);
|
|
421
|
+
const wait = session?.waits.find((candidate) => candidate.wait_id === binding.wait_id);
|
|
422
|
+
const reason = !session ? `bound Session ${binding.session_id} is not routable` : !wait ? `bound Wait ${binding.wait_id} does not belong to active Session ${binding.session_id}` : wait.status !== "active" ? `bound Wait ${binding.wait_id} is ${wait.status}` : binding.wait_version != null && wait.version !== binding.wait_version ? `bound Wait ${binding.wait_id} version changed` : null;
|
|
423
|
+
if (reason) return {
|
|
424
|
+
disposition: "escalate",
|
|
425
|
+
actionable: true,
|
|
426
|
+
deliveries: [],
|
|
427
|
+
evidence: [`trusted provider ${providerId}: ${reason}`],
|
|
428
|
+
summary: "A trusted Event binding is stale or invalid."
|
|
429
|
+
};
|
|
430
|
+
const subject = binding.source_subject ? ` for ${binding.source_subject}` : "";
|
|
431
|
+
return {
|
|
432
|
+
disposition: "deliver",
|
|
433
|
+
actionable: true,
|
|
434
|
+
deliveries: [{
|
|
435
|
+
session_id: session.session_id,
|
|
436
|
+
wait_ids: [wait.wait_id],
|
|
437
|
+
relation: `validated binding from ${providerId}${subject}`,
|
|
438
|
+
confidence: 1
|
|
439
|
+
}],
|
|
440
|
+
evidence: [`trusted provider ${providerId} bound the Event to Wait ${wait.wait_id}`],
|
|
441
|
+
summary: `Deliver the trusted bound Event to Session ${session.session_id}.`
|
|
442
|
+
};
|
|
443
|
+
}
|
|
253
444
|
//#endregion
|
|
254
445
|
//#region src/runtime/schema.mjs
|
|
255
446
|
const SCHEMA_SQL = `
|
|
@@ -289,6 +480,10 @@ const SCHEMA_SQL = `
|
|
|
289
480
|
lease_expires_at TEXT,
|
|
290
481
|
accepted_at TEXT,
|
|
291
482
|
last_error TEXT,
|
|
483
|
+
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
484
|
+
next_attempt_at TEXT,
|
|
485
|
+
terminal_reason_code TEXT,
|
|
486
|
+
terminal_at TEXT,
|
|
292
487
|
committed_at TEXT,
|
|
293
488
|
created_at TEXT NOT NULL,
|
|
294
489
|
updated_at TEXT NOT NULL
|
|
@@ -339,6 +534,7 @@ const SCHEMA_SQL = `
|
|
|
339
534
|
source TEXT NOT NULL,
|
|
340
535
|
source_event_id TEXT,
|
|
341
536
|
fingerprint TEXT NOT NULL,
|
|
537
|
+
correlation_key TEXT,
|
|
342
538
|
payload_json TEXT NOT NULL,
|
|
343
539
|
state TEXT NOT NULL CHECK (state IN (
|
|
344
540
|
'received', 'routing', 'dispatched', 'resolved'
|
|
@@ -355,6 +551,10 @@ const SCHEMA_SQL = `
|
|
|
355
551
|
CREATE UNIQUE INDEX IF NOT EXISTS events_source_fingerprint
|
|
356
552
|
ON events(source, fingerprint);
|
|
357
553
|
|
|
554
|
+
CREATE UNIQUE INDEX IF NOT EXISTS events_trusted_correlation
|
|
555
|
+
ON events(correlation_key)
|
|
556
|
+
WHERE correlation_key IS NOT NULL;
|
|
557
|
+
|
|
358
558
|
CREATE TABLE IF NOT EXISTS routing_attempts (
|
|
359
559
|
id TEXT PRIMARY KEY,
|
|
360
560
|
event_id TEXT NOT NULL REFERENCES events(id),
|
|
@@ -394,6 +594,8 @@ const SCHEMA_SQL = `
|
|
|
394
594
|
CREATE TABLE IF NOT EXISTS delivery_waits (
|
|
395
595
|
delivery_id TEXT NOT NULL REFERENCES deliveries(id),
|
|
396
596
|
wait_id TEXT NOT NULL REFERENCES waits(id),
|
|
597
|
+
ordinal INTEGER NOT NULL DEFAULT 0,
|
|
598
|
+
wait_snapshot_json TEXT,
|
|
397
599
|
PRIMARY KEY (delivery_id, wait_id)
|
|
398
600
|
) STRICT;
|
|
399
601
|
|
|
@@ -420,6 +622,11 @@ const SCHEMA_SQL = `
|
|
|
420
622
|
capabilities_json TEXT NOT NULL,
|
|
421
623
|
next_check_at TEXT,
|
|
422
624
|
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
625
|
+
paused INTEGER NOT NULL DEFAULT 0 CHECK (paused IN (0, 1)),
|
|
626
|
+
terminal_reason_code TEXT,
|
|
627
|
+
terminal_reason_detail TEXT,
|
|
628
|
+
terminal_actor TEXT,
|
|
629
|
+
terminal_at TEXT,
|
|
423
630
|
version INTEGER NOT NULL DEFAULT 0,
|
|
424
631
|
lease_owner TEXT,
|
|
425
632
|
lease_expires_at TEXT,
|
|
@@ -478,6 +685,17 @@ const SCHEMA_SQL = `
|
|
|
478
685
|
|
|
479
686
|
CREATE INDEX IF NOT EXISTS monitor_checks_monitor_time
|
|
480
687
|
ON monitor_checks(monitor_id, started_at);
|
|
688
|
+
|
|
689
|
+
CREATE TABLE IF NOT EXISTS notification_outcomes (
|
|
690
|
+
event_id TEXT PRIMARY KEY REFERENCES events(id),
|
|
691
|
+
provider TEXT,
|
|
692
|
+
state TEXT NOT NULL CHECK (state IN ('delivered', 'unavailable', 'failed')),
|
|
693
|
+
error_class TEXT,
|
|
694
|
+
receipt_id TEXT,
|
|
695
|
+
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
696
|
+
attempted_at TEXT NOT NULL,
|
|
697
|
+
updated_at TEXT NOT NULL
|
|
698
|
+
) STRICT;
|
|
481
699
|
`;
|
|
482
700
|
//#endregion
|
|
483
701
|
//#region src/runtime/store.mjs
|
|
@@ -491,12 +709,12 @@ var RelayStore = class {
|
|
|
491
709
|
this.database.exec("PRAGMA journal_mode = WAL");
|
|
492
710
|
this.database.exec(SCHEMA_SQL);
|
|
493
711
|
const schemaVersion = this.database.prepare("SELECT MAX(version) AS version FROM relay_schema").get().version;
|
|
494
|
-
if (schemaVersion == null) this.database.prepare("INSERT INTO relay_schema (version, applied_at) VALUES (?, ?)").run(
|
|
712
|
+
if (schemaVersion == null) this.database.prepare("INSERT INTO relay_schema (version, applied_at) VALUES (?, ?)").run(10, this.now());
|
|
495
713
|
else {
|
|
496
|
-
assert.ok(schemaVersion <=
|
|
497
|
-
if (schemaVersion <
|
|
714
|
+
assert.ok(schemaVersion <= 10, `unsupported schema version ${schemaVersion}`);
|
|
715
|
+
if (schemaVersion < 10) {
|
|
498
716
|
this.migrateSchema(schemaVersion);
|
|
499
|
-
this.database.prepare("INSERT INTO relay_schema (version, applied_at) VALUES (?, ?)").run(
|
|
717
|
+
this.database.prepare("INSERT INTO relay_schema (version, applied_at) VALUES (?, ?)").run(10, this.now());
|
|
500
718
|
}
|
|
501
719
|
}
|
|
502
720
|
this.database.prepare("INSERT OR IGNORE INTO runtime_counters (name, value) VALUES ('routing_epoch', 0)").run();
|
|
@@ -555,6 +773,51 @@ var RelayStore = class {
|
|
|
555
773
|
`);
|
|
556
774
|
});
|
|
557
775
|
}
|
|
776
|
+
if (fromVersion < 5) {
|
|
777
|
+
const columns = this.database.prepare("PRAGMA table_info(delivery_waits)").all();
|
|
778
|
+
const names = new Set(columns.map((column) => column.name));
|
|
779
|
+
if (!names.has("ordinal")) this.database.exec("ALTER TABLE delivery_waits ADD COLUMN ordinal INTEGER NOT NULL DEFAULT 0");
|
|
780
|
+
if (!names.has("wait_snapshot_json")) this.database.exec("ALTER TABLE delivery_waits ADD COLUMN wait_snapshot_json TEXT");
|
|
781
|
+
}
|
|
782
|
+
if (fromVersion < 6) {
|
|
783
|
+
const columns = this.database.prepare("PRAGMA table_info(monitors)").all();
|
|
784
|
+
const names = new Set(columns.map((column) => column.name));
|
|
785
|
+
for (const [name, type] of [
|
|
786
|
+
["paused", "INTEGER NOT NULL DEFAULT 0 CHECK (paused IN (0, 1))"],
|
|
787
|
+
["terminal_reason_code", "TEXT"],
|
|
788
|
+
["terminal_reason_detail", "TEXT"],
|
|
789
|
+
["terminal_actor", "TEXT"],
|
|
790
|
+
["terminal_at", "TEXT"]
|
|
791
|
+
]) if (!names.has(name)) this.database.exec(`ALTER TABLE monitors ADD COLUMN ${name} ${type}`);
|
|
792
|
+
}
|
|
793
|
+
if (fromVersion < 7) this.database.exec(`
|
|
794
|
+
CREATE TABLE IF NOT EXISTS notification_outcomes (
|
|
795
|
+
event_id TEXT PRIMARY KEY REFERENCES events(id),
|
|
796
|
+
provider TEXT,
|
|
797
|
+
state TEXT NOT NULL CHECK (state IN ('delivered', 'unavailable', 'failed')),
|
|
798
|
+
error_class TEXT,
|
|
799
|
+
attempted_at TEXT NOT NULL,
|
|
800
|
+
updated_at TEXT NOT NULL
|
|
801
|
+
) STRICT;
|
|
802
|
+
`);
|
|
803
|
+
if (fromVersion < 8) {
|
|
804
|
+
if (!this.database.prepare("PRAGMA table_info(events)").all().some((column) => column.name === "correlation_key")) this.database.exec("ALTER TABLE events ADD COLUMN correlation_key TEXT");
|
|
805
|
+
this.database.exec("CREATE UNIQUE INDEX IF NOT EXISTS events_trusted_correlation ON events(correlation_key) WHERE correlation_key IS NOT NULL");
|
|
806
|
+
}
|
|
807
|
+
if (fromVersion < 9) {
|
|
808
|
+
const columns = new Set(this.database.prepare("PRAGMA table_info(activations)").all().map((column) => column.name));
|
|
809
|
+
for (const [name, type] of [
|
|
810
|
+
["attempt_count", "INTEGER NOT NULL DEFAULT 0"],
|
|
811
|
+
["next_attempt_at", "TEXT"],
|
|
812
|
+
["terminal_reason_code", "TEXT"],
|
|
813
|
+
["terminal_at", "TEXT"]
|
|
814
|
+
]) if (!columns.has(name)) this.database.exec(`ALTER TABLE activations ADD COLUMN ${name} ${type}`);
|
|
815
|
+
}
|
|
816
|
+
if (fromVersion < 10) {
|
|
817
|
+
const columns = new Set(this.database.prepare("PRAGMA table_info(notification_outcomes)").all().map((column) => column.name));
|
|
818
|
+
if (!columns.has("receipt_id")) this.database.exec("ALTER TABLE notification_outcomes ADD COLUMN receipt_id TEXT");
|
|
819
|
+
if (!columns.has("attempt_count")) this.database.exec("ALTER TABLE notification_outcomes ADD COLUMN attempt_count INTEGER NOT NULL DEFAULT 0");
|
|
820
|
+
}
|
|
558
821
|
}
|
|
559
822
|
close() {
|
|
560
823
|
this.database.close();
|
|
@@ -627,15 +890,72 @@ var RelayStore = class {
|
|
|
627
890
|
});
|
|
628
891
|
}
|
|
629
892
|
listWaitRegistrations() {
|
|
630
|
-
return this.
|
|
631
|
-
...hydrateWaitRegistration(row, this.getWaits(row.id)),
|
|
632
|
-
monitors: this.getMonitors(row.id)
|
|
633
|
-
})).filter((registration) => registration.waits.some((wait) => wait.status === "active" || wait.status === "claimed") || registration.monitors.some((monitor) => (/* @__PURE__ */ new Set([
|
|
893
|
+
return this.listAllWaitRegistrations().filter((registration) => registration.waits.some((wait) => wait.status === "active" || wait.status === "claimed") || registration.monitors.some((monitor) => (/* @__PURE__ */ new Set([
|
|
634
894
|
"active",
|
|
895
|
+
"paused",
|
|
635
896
|
"triggered",
|
|
636
897
|
"degraded"
|
|
637
898
|
])).has(monitor.state)));
|
|
638
899
|
}
|
|
900
|
+
listAllWaitRegistrations() {
|
|
901
|
+
return this.database.prepare("SELECT * FROM sessions ORDER BY updated_at DESC, id").all().map((row) => ({
|
|
902
|
+
...hydrateWaitRegistration(row, this.getWaits(row.id)),
|
|
903
|
+
monitors: this.getMonitors(row.id)
|
|
904
|
+
}));
|
|
905
|
+
}
|
|
906
|
+
listEvents(limit = 100) {
|
|
907
|
+
return this.listEventsPage({ limit }).items;
|
|
908
|
+
}
|
|
909
|
+
listEventsPage({ limit = 20, cursor = null } = {}) {
|
|
910
|
+
assert.ok(Number.isSafeInteger(limit) && limit > 0 && limit <= 100, "Event history limit is invalid");
|
|
911
|
+
const boundary = cursor == null ? null : decodeHistoryCursor(cursor);
|
|
912
|
+
const rows = boundary == null ? this.database.prepare(`
|
|
913
|
+
SELECT id, received_at FROM events
|
|
914
|
+
ORDER BY received_at DESC, id DESC LIMIT ?
|
|
915
|
+
`).all(limit + 1) : this.database.prepare(`
|
|
916
|
+
SELECT id, received_at FROM events
|
|
917
|
+
WHERE received_at < ? OR (received_at = ? AND id < ?)
|
|
918
|
+
ORDER BY received_at DESC, id DESC LIMIT ?
|
|
919
|
+
`).all(boundary.received_at, boundary.received_at, boundary.id, limit + 1);
|
|
920
|
+
const hasMore = rows.length > limit;
|
|
921
|
+
const pageRows = rows.slice(0, limit);
|
|
922
|
+
const last = pageRows.at(-1);
|
|
923
|
+
return {
|
|
924
|
+
items: pageRows.map((row) => this.inspectEvent(row.id)),
|
|
925
|
+
next_cursor: hasMore && last ? encodeHistoryCursor(last) : null,
|
|
926
|
+
total: this.database.prepare("SELECT COUNT(*) AS count FROM events").get().count
|
|
927
|
+
};
|
|
928
|
+
}
|
|
929
|
+
getNotificationOutcome(eventId) {
|
|
930
|
+
const row = this.database.prepare("SELECT * FROM notification_outcomes WHERE event_id = ?").get(eventId);
|
|
931
|
+
return row ? {
|
|
932
|
+
event_id: row.event_id,
|
|
933
|
+
provider: row.provider,
|
|
934
|
+
state: row.state,
|
|
935
|
+
error_class: row.error_class,
|
|
936
|
+
receipt_id: row.receipt_id,
|
|
937
|
+
attempt_count: row.attempt_count,
|
|
938
|
+
attempted_at: row.attempted_at,
|
|
939
|
+
updated_at: row.updated_at
|
|
940
|
+
} : null;
|
|
941
|
+
}
|
|
942
|
+
recordNotificationOutcome(eventId, { provider = null, state, errorClass = null, receiptId = null }) {
|
|
943
|
+
assert.ok((/* @__PURE__ */ new Set([
|
|
944
|
+
"delivered",
|
|
945
|
+
"unavailable",
|
|
946
|
+
"failed"
|
|
947
|
+
])).has(state), "notification state is invalid");
|
|
948
|
+
this.requireEventRow(eventId);
|
|
949
|
+
const timestamp = this.now();
|
|
950
|
+
this.database.prepare(`
|
|
951
|
+
INSERT INTO notification_outcomes (event_id, provider, state, error_class, receipt_id, attempt_count, attempted_at, updated_at)
|
|
952
|
+
VALUES (?, ?, ?, ?, ?, 1, ?, ?)
|
|
953
|
+
ON CONFLICT(event_id) DO UPDATE SET provider=excluded.provider, state=excluded.state,
|
|
954
|
+
error_class=excluded.error_class, receipt_id=excluded.receipt_id,
|
|
955
|
+
attempt_count=notification_outcomes.attempt_count + 1, updated_at=excluded.updated_at
|
|
956
|
+
`).run(eventId, provider, state, errorClass, receiptId, timestamp, timestamp);
|
|
957
|
+
return this.getNotificationOutcome(eventId);
|
|
958
|
+
}
|
|
639
959
|
listQueuedDeliverySessionIds() {
|
|
640
960
|
return this.database.prepare(`
|
|
641
961
|
SELECT DISTINCT session_id
|
|
@@ -644,6 +964,18 @@ var RelayStore = class {
|
|
|
644
964
|
ORDER BY session_id
|
|
645
965
|
`).all().map((row) => row.session_id);
|
|
646
966
|
}
|
|
967
|
+
listRoutableEventIds(limit = 100) {
|
|
968
|
+
return this.database.prepare(`
|
|
969
|
+
SELECT id
|
|
970
|
+
FROM events
|
|
971
|
+
WHERE state IN ('received', 'routing')
|
|
972
|
+
ORDER BY received_at, id
|
|
973
|
+
LIMIT ?
|
|
974
|
+
`).all(limit).map((row) => row.id);
|
|
975
|
+
}
|
|
976
|
+
countRoutingAttempts(eventId) {
|
|
977
|
+
return this.database.prepare("SELECT COUNT(*) AS count FROM routing_attempts WHERE event_id = ?").get(eventId).count;
|
|
978
|
+
}
|
|
647
979
|
getActivation(activationId) {
|
|
648
980
|
const row = this.database.prepare("SELECT * FROM activations WHERE id = ?").get(activationId);
|
|
649
981
|
return row ? hydrateActivation(row) : null;
|
|
@@ -655,6 +987,10 @@ var RelayStore = class {
|
|
|
655
987
|
const now = toIso(nowDate);
|
|
656
988
|
let activationRow = this.database.prepare("SELECT * FROM activations WHERE session_id = ? AND state = 'active'").get(sessionId);
|
|
657
989
|
if (activationRow?.lease_owner && activationRow.lease_expires_at > now) return { status: "busy" };
|
|
990
|
+
if (activationRow?.next_attempt_at && activationRow.next_attempt_at > now) return {
|
|
991
|
+
status: "retry_scheduled",
|
|
992
|
+
activation: hydrateActivation(activationRow)
|
|
993
|
+
};
|
|
658
994
|
let deliveryRows;
|
|
659
995
|
if (activationRow) deliveryRows = decodeJson(activationRow.delivery_ids_json).map((deliveryId) => {
|
|
660
996
|
const row = this.database.prepare("SELECT * FROM deliveries WHERE id = ?").get(deliveryId);
|
|
@@ -680,7 +1016,8 @@ var RelayStore = class {
|
|
|
680
1016
|
const leaseExpiresAt = new Date(nowDate.getTime() + leaseMs).toISOString();
|
|
681
1017
|
this.database.prepare(`
|
|
682
1018
|
UPDATE activations
|
|
683
|
-
SET lease_owner = ?, lease_expires_at = ?, last_error = NULL,
|
|
1019
|
+
SET lease_owner = ?, lease_expires_at = ?, last_error = NULL,
|
|
1020
|
+
next_attempt_at = NULL, updated_at = ?
|
|
684
1021
|
WHERE id = ? AND state = 'active'
|
|
685
1022
|
`).run(owner, leaseExpiresAt, now, activationRow.id);
|
|
686
1023
|
return {
|
|
@@ -740,18 +1077,78 @@ var RelayStore = class {
|
|
|
740
1077
|
return this.inspectWaitRegistration(sessionId);
|
|
741
1078
|
});
|
|
742
1079
|
}
|
|
743
|
-
failDispatch(sessionId, activationId, owner, error) {
|
|
1080
|
+
failDispatch(sessionId, activationId, owner, error, { failureLimit = 5, baseDelayMs = 1e3 } = {}) {
|
|
744
1081
|
return this.transaction(() => {
|
|
745
1082
|
const activation = this.database.prepare("SELECT * FROM activations WHERE id = ?").get(activationId);
|
|
746
1083
|
assert.ok(activation, `activation ${activationId} does not exist`);
|
|
747
1084
|
assert.equal(activation.session_id, sessionId, `activation ${activationId} has wrong session`);
|
|
748
1085
|
assert.equal(activation.state, "active", `activation ${activationId} is not active`);
|
|
749
1086
|
assert.equal(activation.lease_owner, owner, `worker does not own activation ${activationId}`);
|
|
750
|
-
|
|
1087
|
+
assert.ok(Number.isSafeInteger(failureLimit) && failureLimit > 0, "delivery failure limit must be positive");
|
|
1088
|
+
assert.ok(Number.isSafeInteger(baseDelayMs) && baseDelayMs > 0, "delivery retry base delay must be positive");
|
|
1089
|
+
const attemptCount = (activation.attempt_count ?? 0) + 1;
|
|
1090
|
+
const timestamp = this.now();
|
|
1091
|
+
if (attemptCount < failureLimit) {
|
|
1092
|
+
const delay = Math.min(baseDelayMs * 2 ** (attemptCount - 1), 36e5);
|
|
1093
|
+
const nextAttemptAt = new Date(this.clock().getTime() + delay).toISOString();
|
|
1094
|
+
this.database.prepare(`
|
|
751
1095
|
UPDATE activations
|
|
752
|
-
SET lease_owner = NULL, lease_expires_at = NULL, last_error = ?,
|
|
1096
|
+
SET lease_owner = NULL, lease_expires_at = NULL, last_error = ?,
|
|
1097
|
+
attempt_count = ?, next_attempt_at = ?, updated_at = ?
|
|
753
1098
|
WHERE id = ?
|
|
754
|
-
`).run(error,
|
|
1099
|
+
`).run(error, attemptCount, nextAttemptAt, timestamp, activationId);
|
|
1100
|
+
return {
|
|
1101
|
+
status: "retry",
|
|
1102
|
+
activation: this.getActivation(activationId),
|
|
1103
|
+
eventIds: []
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
const deliveryIds = decodeJson(activation.delivery_ids_json);
|
|
1107
|
+
const eventIds = /* @__PURE__ */ new Set();
|
|
1108
|
+
for (const deliveryId of deliveryIds) {
|
|
1109
|
+
const delivery = this.database.prepare("SELECT * FROM deliveries WHERE id = ?").get(deliveryId);
|
|
1110
|
+
assert.ok(delivery, `delivery ${deliveryId} does not exist`);
|
|
1111
|
+
eventIds.add(delivery.event_id);
|
|
1112
|
+
this.database.prepare("UPDATE deliveries SET state = 'failed', updated_at = ? WHERE id = ?").run(timestamp, deliveryId);
|
|
1113
|
+
}
|
|
1114
|
+
for (const eventId of eventIds) if (this.database.prepare("SELECT COUNT(*) AS count FROM deliveries WHERE event_id = ? AND state IN ('queued', 'running')").get(eventId).count === 0) this.database.prepare("UPDATE events SET state = 'resolved', version = version + 1, updated_at = ? WHERE id = ?").run(timestamp, eventId);
|
|
1115
|
+
this.database.prepare(`
|
|
1116
|
+
UPDATE activations
|
|
1117
|
+
SET state = 'committed', lease_owner = NULL, lease_expires_at = NULL,
|
|
1118
|
+
last_error = ?, attempt_count = ?, next_attempt_at = NULL,
|
|
1119
|
+
terminal_reason_code = 'delivery_retry_exhausted', terminal_at = ?,
|
|
1120
|
+
committed_at = ?, updated_at = ?
|
|
1121
|
+
WHERE id = ?
|
|
1122
|
+
`).run(error, attemptCount, timestamp, timestamp, timestamp, activationId);
|
|
1123
|
+
return {
|
|
1124
|
+
status: "failed",
|
|
1125
|
+
activation: this.getActivation(activationId),
|
|
1126
|
+
eventIds: [...eventIds]
|
|
1127
|
+
};
|
|
1128
|
+
});
|
|
1129
|
+
}
|
|
1130
|
+
retryActivation(activationId) {
|
|
1131
|
+
return this.transaction(() => {
|
|
1132
|
+
const row = this.database.prepare("SELECT * FROM activations WHERE id = ?").get(activationId);
|
|
1133
|
+
assert.ok(row, `activation ${activationId} does not exist`);
|
|
1134
|
+
assert.equal(row.state, "committed", `activation ${activationId} is not terminal`);
|
|
1135
|
+
assert.equal(row.terminal_reason_code, "delivery_retry_exhausted", `activation ${activationId} is not retryable`);
|
|
1136
|
+
const timestamp = this.now();
|
|
1137
|
+
const deliveryIds = decodeJson(row.delivery_ids_json);
|
|
1138
|
+
for (const deliveryId of deliveryIds) {
|
|
1139
|
+
const delivery = this.database.prepare("SELECT * FROM deliveries WHERE id = ?").get(deliveryId);
|
|
1140
|
+
assert.ok(delivery, `delivery ${deliveryId} does not exist`);
|
|
1141
|
+
assert.equal(delivery.state, "failed", `delivery ${deliveryId} is not failed`);
|
|
1142
|
+
this.database.prepare("UPDATE deliveries SET state = 'queued', updated_at = ? WHERE id = ?").run(timestamp, deliveryId);
|
|
1143
|
+
this.database.prepare("UPDATE events SET state = 'dispatched', version = version + 1, updated_at = ? WHERE id = ?").run(timestamp, delivery.event_id);
|
|
1144
|
+
}
|
|
1145
|
+
this.database.prepare(`
|
|
1146
|
+
UPDATE activations
|
|
1147
|
+
SET state = 'active', attempt_count = 0, next_attempt_at = NULL,
|
|
1148
|
+
terminal_reason_code = NULL, terminal_at = NULL, committed_at = NULL,
|
|
1149
|
+
last_error = NULL, updated_at = ?
|
|
1150
|
+
WHERE id = ?
|
|
1151
|
+
`).run(timestamp, activationId);
|
|
755
1152
|
return this.getActivation(activationId);
|
|
756
1153
|
});
|
|
757
1154
|
}
|
|
@@ -762,6 +1159,145 @@ var RelayStore = class {
|
|
|
762
1159
|
const row = this.database.prepare("SELECT * FROM monitors WHERE id = ?").get(monitorId);
|
|
763
1160
|
return row ? this.hydrateMonitor(row) : null;
|
|
764
1161
|
}
|
|
1162
|
+
pauseMonitor(monitorId, { expectedVersion } = {}) {
|
|
1163
|
+
return this.transaction(() => {
|
|
1164
|
+
const row = this.requireMonitorRow(monitorId);
|
|
1165
|
+
if (expectedVersion != null) assert.equal(row.version, expectedVersion, `monitor ${monitorId} version changed`);
|
|
1166
|
+
assert.ok((/* @__PURE__ */ new Set(["active", "degraded"])).has(row.state), `monitor ${monitorId} cannot be paused from ${row.state}`);
|
|
1167
|
+
assert.equal(Boolean(row.paused), false, `monitor ${monitorId} is already paused`);
|
|
1168
|
+
assert.ok(!row.lease_owner || row.lease_expires_at <= this.now(), `monitor ${monitorId} is busy`);
|
|
1169
|
+
this.database.prepare(`
|
|
1170
|
+
UPDATE monitors
|
|
1171
|
+
SET paused = 1, next_check_at = NULL, lease_owner = NULL,
|
|
1172
|
+
lease_expires_at = NULL, version = version + 1, updated_at = ?
|
|
1173
|
+
WHERE id = ?
|
|
1174
|
+
`).run(this.now(), monitorId);
|
|
1175
|
+
return this.inspectMonitor(monitorId);
|
|
1176
|
+
});
|
|
1177
|
+
}
|
|
1178
|
+
resumeMonitor(monitorId, { expectedVersion } = {}) {
|
|
1179
|
+
return this.transaction(() => {
|
|
1180
|
+
const row = this.requireMonitorRow(monitorId);
|
|
1181
|
+
if (expectedVersion != null) assert.equal(row.version, expectedVersion, `monitor ${monitorId} version changed`);
|
|
1182
|
+
assert.equal(Boolean(row.paused), true, `monitor ${monitorId} is not paused`);
|
|
1183
|
+
assert.ok((/* @__PURE__ */ new Set(["active", "degraded"])).has(row.state), `monitor ${monitorId} cannot resume from ${row.state}`);
|
|
1184
|
+
const timestamp = this.now();
|
|
1185
|
+
this.database.prepare(`
|
|
1186
|
+
UPDATE monitors
|
|
1187
|
+
SET paused = 0, next_check_at = ?, version = version + 1, updated_at = ?
|
|
1188
|
+
WHERE id = ?
|
|
1189
|
+
`).run(nextMonitorCheckAt(decodeJson(row.schedule_json), this.clock()), timestamp, monitorId);
|
|
1190
|
+
return this.inspectMonitor(monitorId);
|
|
1191
|
+
});
|
|
1192
|
+
}
|
|
1193
|
+
updateMonitorCadence(monitorId, intervalSeconds, { expectedVersion } = {}) {
|
|
1194
|
+
assert.ok(Number.isSafeInteger(intervalSeconds) && intervalSeconds >= 1 && intervalSeconds <= 86400, "monitor interval_seconds must be a whole number from 1 to 86400");
|
|
1195
|
+
return this.transaction(() => {
|
|
1196
|
+
const row = this.requireMonitorRow(monitorId);
|
|
1197
|
+
if (expectedVersion != null) assert.equal(row.version, expectedVersion, `monitor ${monitorId} version changed`);
|
|
1198
|
+
assert.ok((/* @__PURE__ */ new Set(["active", "degraded"])).has(row.state), `monitor ${monitorId} cannot be updated from ${row.state}`);
|
|
1199
|
+
const schedule = {
|
|
1200
|
+
...decodeJson(row.schedule_json),
|
|
1201
|
+
interval_seconds: intervalSeconds
|
|
1202
|
+
};
|
|
1203
|
+
const timestamp = this.now();
|
|
1204
|
+
this.database.prepare(`
|
|
1205
|
+
UPDATE monitors
|
|
1206
|
+
SET schedule_json = ?, next_check_at = ?, version = version + 1, updated_at = ?
|
|
1207
|
+
WHERE id = ?
|
|
1208
|
+
`).run(encodeJson(schedule), row.paused ? null : nextMonitorCheckAt(schedule, this.clock()), timestamp, monitorId);
|
|
1209
|
+
return this.inspectMonitor(monitorId);
|
|
1210
|
+
});
|
|
1211
|
+
}
|
|
1212
|
+
rebaselineMonitor(monitorId, prepared, { expectedVersion } = {}) {
|
|
1213
|
+
assert.ok(prepared?.baseline_observation && typeof prepared.baseline_observation === "object", "monitor baseline_observation is required");
|
|
1214
|
+
return this.transaction(() => {
|
|
1215
|
+
const row = this.requireMonitorRow(monitorId);
|
|
1216
|
+
if (expectedVersion != null) assert.equal(row.version, expectedVersion, `monitor ${monitorId} version changed`);
|
|
1217
|
+
assert.ok((/* @__PURE__ */ new Set(["active", "degraded"])).has(row.state), `monitor ${monitorId} cannot be updated from ${row.state}`);
|
|
1218
|
+
assert.ok(!row.lease_owner || row.lease_expires_at <= this.now(), `monitor ${monitorId} is busy`);
|
|
1219
|
+
assert.equal(prepared.monitor_id, monitorId, "monitor identity cannot change during rebaseline");
|
|
1220
|
+
assert.equal(prepared.wait_id, row.wait_id, "monitor wait cannot change during rebaseline");
|
|
1221
|
+
const schedule = {
|
|
1222
|
+
interval_seconds: prepared.schedule?.interval_seconds ?? decodeJson(row.schedule_json).interval_seconds,
|
|
1223
|
+
jitter_seconds: prepared.schedule?.jitter_seconds ?? decodeJson(row.schedule_json).jitter_seconds ?? 0
|
|
1224
|
+
};
|
|
1225
|
+
assert.ok(Number.isSafeInteger(schedule.interval_seconds) && schedule.interval_seconds >= 1 && schedule.interval_seconds <= 86400, "monitor interval_seconds must be a whole number from 1 to 86400");
|
|
1226
|
+
assert.ok(Number.isSafeInteger(schedule.jitter_seconds) && schedule.jitter_seconds >= 0 && schedule.jitter_seconds <= Math.min(schedule.interval_seconds, 3600), "monitor jitter_seconds must be a bounded whole number no greater than interval_seconds");
|
|
1227
|
+
const retry = {
|
|
1228
|
+
degraded_after: prepared.retry?.degraded_after ?? decodeJson(row.retry_json).degraded_after,
|
|
1229
|
+
fail_after: prepared.retry?.fail_after ?? decodeJson(row.retry_json).fail_after,
|
|
1230
|
+
backoff_seconds: prepared.retry?.backoff_seconds ?? decodeJson(row.retry_json).backoff_seconds ?? []
|
|
1231
|
+
};
|
|
1232
|
+
assert.ok(Number.isSafeInteger(retry.degraded_after) && retry.degraded_after >= 1 && retry.degraded_after <= 100, "monitor degraded_after must be a whole number from 1 to 100");
|
|
1233
|
+
assert.ok(Number.isSafeInteger(retry.fail_after) && retry.fail_after >= retry.degraded_after && retry.fail_after <= 100, "monitor fail_after must follow degraded_after and be at most 100");
|
|
1234
|
+
assert.ok(Array.isArray(retry.backoff_seconds) && retry.backoff_seconds.length <= 20 && retry.backoff_seconds.every((value) => Number.isSafeInteger(value) && value >= 1 && value <= 86400), "monitor backoff_seconds must contain at most 20 whole-second delays from 1 to 86400");
|
|
1235
|
+
const manifest = {
|
|
1236
|
+
observer: prepared.observer ?? (prepared.detector?.kind === "deadline_reached" ? { provider: "clock" } : null),
|
|
1237
|
+
detector: prepared.detector,
|
|
1238
|
+
schedule,
|
|
1239
|
+
retry,
|
|
1240
|
+
capabilities: prepared.capabilities ?? {},
|
|
1241
|
+
artifact: prepared.artifact ?? { kind: "fixture" }
|
|
1242
|
+
};
|
|
1243
|
+
const versionId = this.idFactory();
|
|
1244
|
+
const timestamp = this.now();
|
|
1245
|
+
const artifactHash = prepared.artifact?.version_sha256 ?? prepared.artifact?.sha256 ?? hashJson(manifest);
|
|
1246
|
+
const existingVersion = this.database.prepare(`
|
|
1247
|
+
SELECT id, manifest_json FROM monitor_versions WHERE monitor_id = ? AND artifact_hash = ?
|
|
1248
|
+
`).get(monitorId, artifactHash);
|
|
1249
|
+
const activeVersionId = existingVersion?.id ?? versionId;
|
|
1250
|
+
if (existingVersion) assert.deepEqual(decodeJson(existingVersion.manifest_json), manifest, `monitor ${monitorId} artifact hash conflicts with different version content`);
|
|
1251
|
+
else this.database.prepare(`
|
|
1252
|
+
INSERT INTO monitor_versions (id, monitor_id, artifact_hash, manifest_json, created_by_run_id, created_at)
|
|
1253
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
1254
|
+
`).run(versionId, monitorId, artifactHash, encodeJson(manifest), prepared.created_by_run_id ?? null, timestamp);
|
|
1255
|
+
const checkId = this.idFactory();
|
|
1256
|
+
this.database.prepare(`
|
|
1257
|
+
INSERT INTO monitor_checks (id, monitor_id, version_id, kind, state, started_at, finished_at)
|
|
1258
|
+
VALUES (?, ?, ?, 'baseline', 'succeeded', ?, ?)
|
|
1259
|
+
`).run(checkId, monitorId, activeVersionId, timestamp, timestamp);
|
|
1260
|
+
const sequence = this.database.prepare("SELECT COALESCE(MAX(sequence), -1) + 1 AS sequence FROM observations WHERE monitor_id = ?").get(monitorId).sequence;
|
|
1261
|
+
this.database.prepare(`
|
|
1262
|
+
INSERT INTO observations (id, check_id, monitor_id, sequence, state_hash, data_json, observed_at)
|
|
1263
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1264
|
+
`).run(this.idFactory(), checkId, monitorId, sequence, hashJson(prepared.baseline_observation), encodeJson(prepared.baseline_observation), timestamp);
|
|
1265
|
+
this.database.prepare(`
|
|
1266
|
+
UPDATE monitors
|
|
1267
|
+
SET active_version_id = ?, detector_json = ?, schedule_json = ?, retry_json = ?,
|
|
1268
|
+
capabilities_json = ?, next_check_at = ?, consecutive_failures = 0,
|
|
1269
|
+
state = 'active', version = version + 1, updated_at = ?
|
|
1270
|
+
WHERE id = ?
|
|
1271
|
+
`).run(activeVersionId, encodeJson(prepared.detector), encodeJson(schedule), encodeJson(retry), encodeJson(prepared.capabilities ?? {}), row.paused ? null : nextMonitorCheckAt(schedule, this.clock()), timestamp, monitorId);
|
|
1272
|
+
return this.inspectMonitor(monitorId);
|
|
1273
|
+
});
|
|
1274
|
+
}
|
|
1275
|
+
stopMonitor(monitorId, { expectedVersion, actor = "user", reasonCode = "stopped_by_user", detail = "" } = {}) {
|
|
1276
|
+
assert.ok(typeof actor === "string" && actor.length > 0 && actor.length <= 256, "monitor stop actor is invalid");
|
|
1277
|
+
assert.ok(typeof reasonCode === "string" && /^[a-z][a-z0-9._-]{0,127}$/u.test(reasonCode), "monitor stop reason code is invalid");
|
|
1278
|
+
assert.ok(typeof detail === "string" && detail.length <= 2e3, "monitor stop detail is invalid");
|
|
1279
|
+
return this.transaction(() => {
|
|
1280
|
+
const row = this.requireMonitorRow(monitorId);
|
|
1281
|
+
if (expectedVersion != null) assert.equal(row.version, expectedVersion, `monitor ${monitorId} version changed`);
|
|
1282
|
+
assert.ok(!(/* @__PURE__ */ new Set([
|
|
1283
|
+
"completed",
|
|
1284
|
+
"failed",
|
|
1285
|
+
"expired",
|
|
1286
|
+
"cancelled"
|
|
1287
|
+
])).has(row.state), `monitor ${monitorId} is already terminal`);
|
|
1288
|
+
assert.ok(!row.lease_owner || row.lease_expires_at <= this.now(), `monitor ${monitorId} is busy`);
|
|
1289
|
+
const timestamp = this.now();
|
|
1290
|
+
this.database.prepare(`
|
|
1291
|
+
UPDATE monitors
|
|
1292
|
+
SET state = 'cancelled', paused = 0, next_check_at = NULL,
|
|
1293
|
+
lease_owner = NULL, lease_expires_at = NULL,
|
|
1294
|
+
terminal_reason_code = ?, terminal_reason_detail = ?, terminal_actor = ?, terminal_at = ?,
|
|
1295
|
+
version = version + 1, updated_at = ?
|
|
1296
|
+
WHERE id = ?
|
|
1297
|
+
`).run(reasonCode, detail, actor, timestamp, timestamp, monitorId);
|
|
1298
|
+
return this.inspectMonitor(monitorId);
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
765
1301
|
getMonitors(sessionId) {
|
|
766
1302
|
return this.database.prepare("SELECT * FROM monitors WHERE session_id = ? ORDER BY created_at, id").all(sessionId).map((row) => this.hydrateMonitor(row));
|
|
767
1303
|
}
|
|
@@ -780,9 +1316,12 @@ var RelayStore = class {
|
|
|
780
1316
|
triggers
|
|
781
1317
|
};
|
|
782
1318
|
}
|
|
783
|
-
ingestEvent(input) {
|
|
1319
|
+
ingestEvent(input, { allowCorrelation = false } = {}) {
|
|
784
1320
|
return this.transaction(() => {
|
|
785
|
-
const
|
|
1321
|
+
const correlationKey = allowCorrelation && typeof input.correlation_key === "string" && input.correlation_key.length > 0 ? input.correlation_key : null;
|
|
1322
|
+
if (correlationKey != null) assert.ok(correlationKey.length <= 1024, "trusted Event correlation key is too long");
|
|
1323
|
+
const storedInput = correlationKey == null ? Object.fromEntries(Object.entries(input).filter(([key]) => key !== "correlation_key")) : input;
|
|
1324
|
+
const existing = this.findDuplicateEvent(storedInput, correlationKey);
|
|
786
1325
|
if (existing) return {
|
|
787
1326
|
event: hydrateEvent(existing),
|
|
788
1327
|
duplicate: true
|
|
@@ -791,10 +1330,10 @@ var RelayStore = class {
|
|
|
791
1330
|
const timestamp = this.now();
|
|
792
1331
|
this.database.prepare(`
|
|
793
1332
|
INSERT INTO events (
|
|
794
|
-
id, source, source_event_id, fingerprint, payload_json,
|
|
1333
|
+
id, source, source_event_id, fingerprint, correlation_key, payload_json,
|
|
795
1334
|
state, version, received_at, updated_at
|
|
796
|
-
) VALUES (?, ?, ?, ?, ?, 'received', 0, ?, ?)
|
|
797
|
-
`).run(eventId, input.source, input.source_event_id ?? null, input.fingerprint, encodeJson(
|
|
1335
|
+
) VALUES (?, ?, ?, ?, ?, ?, 'received', 0, ?, ?)
|
|
1336
|
+
`).run(eventId, input.source, input.source_event_id ?? null, input.fingerprint, correlationKey, encodeJson(storedInput), timestamp, timestamp);
|
|
798
1337
|
return {
|
|
799
1338
|
event: this.getEvent(eventId),
|
|
800
1339
|
duplicate: false
|
|
@@ -811,11 +1350,20 @@ var RelayStore = class {
|
|
|
811
1350
|
const decisionRow = this.database.prepare("SELECT * FROM routing_decisions WHERE event_id = ?").get(eventId);
|
|
812
1351
|
const deliveries = this.database.prepare("SELECT * FROM deliveries WHERE event_id = ? ORDER BY created_at, id").all(eventId).map((row) => this.hydrateDelivery(row));
|
|
813
1352
|
const routingAttempts = this.database.prepare("SELECT * FROM routing_attempts WHERE event_id = ? ORDER BY created_at, id").all(eventId).map(hydrateRoutingAttempt);
|
|
1353
|
+
const activations = this.database.prepare(`
|
|
1354
|
+
SELECT DISTINCT a.*
|
|
1355
|
+
FROM activations a, json_each(a.delivery_ids_json) ids
|
|
1356
|
+
JOIN deliveries d ON d.id = ids.value
|
|
1357
|
+
WHERE d.event_id = ?
|
|
1358
|
+
ORDER BY a.created_at, a.id
|
|
1359
|
+
`).all(eventId).map(hydrateActivation);
|
|
814
1360
|
return {
|
|
815
1361
|
...event,
|
|
816
1362
|
decision: decisionRow ? hydrateDecision(decisionRow) : null,
|
|
817
1363
|
deliveries,
|
|
818
|
-
routing_attempts: routingAttempts
|
|
1364
|
+
routing_attempts: routingAttempts,
|
|
1365
|
+
activations,
|
|
1366
|
+
notification: this.getNotificationOutcome(eventId)
|
|
819
1367
|
};
|
|
820
1368
|
}
|
|
821
1369
|
beginRouting(eventId) {
|
|
@@ -886,14 +1434,18 @@ var RelayStore = class {
|
|
|
886
1434
|
});
|
|
887
1435
|
deliveryIds.push(deliveryId);
|
|
888
1436
|
sessionIds.push(delivery.session_id);
|
|
889
|
-
for (const waitId of delivery.wait_ids) {
|
|
1437
|
+
for (const [ordinal, waitId] of delivery.wait_ids.entries()) {
|
|
890
1438
|
const snapshotWait = snapshotSession.waits.find((wait) => wait.wait_id === waitId);
|
|
891
1439
|
assert.ok(snapshotWait, `unknown snapshot wait ${waitId}`);
|
|
892
1440
|
const waitRow = this.database.prepare("SELECT * FROM waits WHERE id = ?").get(waitId);
|
|
893
1441
|
assert.ok(waitRow, `wait ${waitId} no longer exists`);
|
|
894
1442
|
assert.equal(waitRow.status, "active", `wait ${waitId} is not active`);
|
|
895
1443
|
assert.equal(waitRow.version, snapshotWait.version, `wait ${waitId} version changed`);
|
|
896
|
-
this.database.prepare(
|
|
1444
|
+
this.database.prepare(`
|
|
1445
|
+
INSERT INTO delivery_waits (
|
|
1446
|
+
delivery_id, wait_id, ordinal, wait_snapshot_json
|
|
1447
|
+
) VALUES (?, ?, ?, ?)
|
|
1448
|
+
`).run(deliveryId, waitId, ordinal, encodeJson(snapshotWait));
|
|
897
1449
|
this.database.prepare(`
|
|
898
1450
|
UPDATE waits
|
|
899
1451
|
SET status = 'claimed', version = version + 1, updated_at = ?
|
|
@@ -931,15 +1483,57 @@ var RelayStore = class {
|
|
|
931
1483
|
return this.database.prepare(`
|
|
932
1484
|
SELECT * FROM monitors
|
|
933
1485
|
WHERE state IN ('active', 'degraded')
|
|
1486
|
+
AND paused = 0
|
|
934
1487
|
AND next_check_at IS NOT NULL
|
|
935
1488
|
AND next_check_at <= ?
|
|
936
1489
|
ORDER BY next_check_at, id
|
|
937
1490
|
LIMIT ?
|
|
938
1491
|
`).all(toIso(at), limit).map((row) => this.hydrateMonitor(row));
|
|
939
1492
|
}
|
|
1493
|
+
cleanupRetention({ terminalBefore, limit = 1e3 } = {}) {
|
|
1494
|
+
const before = toIso(terminalBefore);
|
|
1495
|
+
assert.ok(Number.isSafeInteger(limit) && limit >= 1 && limit <= 1e4, "retention cleanup limit must be from 1 to 10000");
|
|
1496
|
+
return this.transaction(() => {
|
|
1497
|
+
const rows = this.database.prepare(`
|
|
1498
|
+
SELECT id FROM events
|
|
1499
|
+
WHERE state = 'resolved' AND updated_at < ?
|
|
1500
|
+
AND json_extract(payload_json, '$.__relay_retention_redacted') IS NOT 1
|
|
1501
|
+
ORDER BY updated_at, id
|
|
1502
|
+
LIMIT ?
|
|
1503
|
+
`).all(before, limit);
|
|
1504
|
+
const timestamp = this.now();
|
|
1505
|
+
for (const row of rows) {
|
|
1506
|
+
this.database.prepare(`
|
|
1507
|
+
UPDATE events
|
|
1508
|
+
SET payload_json = ?, version = version + 1, updated_at = ?
|
|
1509
|
+
WHERE id = ? AND state = 'resolved'
|
|
1510
|
+
`).run(encodeJson({
|
|
1511
|
+
__relay_retention_redacted: 1,
|
|
1512
|
+
redacted_at: timestamp
|
|
1513
|
+
}), timestamp, row.id);
|
|
1514
|
+
this.database.prepare(`
|
|
1515
|
+
UPDATE routing_attempts SET output_json = NULL, error = NULL
|
|
1516
|
+
WHERE event_id = ?
|
|
1517
|
+
`).run(row.id);
|
|
1518
|
+
}
|
|
1519
|
+
return {
|
|
1520
|
+
terminal_before: before,
|
|
1521
|
+
scanned: rows.length,
|
|
1522
|
+
redacted_events: rows.length,
|
|
1523
|
+
retained_events: this.database.prepare("SELECT COUNT(*) AS count FROM events").get().count,
|
|
1524
|
+
retained_decisions: this.database.prepare("SELECT COUNT(*) AS count FROM routing_decisions").get().count,
|
|
1525
|
+
retained_deliveries: this.database.prepare("SELECT COUNT(*) AS count FROM deliveries").get().count,
|
|
1526
|
+
retained_activations: this.database.prepare("SELECT COUNT(*) AS count FROM activations").get().count
|
|
1527
|
+
};
|
|
1528
|
+
});
|
|
1529
|
+
}
|
|
940
1530
|
beginMonitorCheck(monitorId, owner, leaseMs, { force = false } = {}) {
|
|
941
1531
|
return this.transaction(() => {
|
|
942
1532
|
let row = this.requireMonitorRow(monitorId);
|
|
1533
|
+
if (row.paused) return {
|
|
1534
|
+
status: "paused",
|
|
1535
|
+
monitor: this.hydrateMonitor(row)
|
|
1536
|
+
};
|
|
943
1537
|
if (!(/* @__PURE__ */ new Set(["active", "degraded"])).has(row.state)) return {
|
|
944
1538
|
status: "inactive",
|
|
945
1539
|
monitor: this.hydrateMonitor(row)
|
|
@@ -983,6 +1577,45 @@ var RelayStore = class {
|
|
|
983
1577
|
assert.ok(Array.isArray(proposedEvents), "proposedEvents must be an array");
|
|
984
1578
|
assert.ok(proposedEvents.length <= 1, "initial Monitor slice emits at most one Event per check");
|
|
985
1579
|
return this.transaction(() => {
|
|
1580
|
+
const current = this.requireMonitorRow(snapshot.monitor.monitor_id);
|
|
1581
|
+
const proposal = proposedEvents[0];
|
|
1582
|
+
const correlationKey = typeof proposal?.correlation_key === "string" && proposal.correlation_key.length > 0 ? proposal.correlation_key : null;
|
|
1583
|
+
const correlated = correlationKey == null ? null : this.database.prepare("SELECT id FROM events WHERE correlation_key = ?").get(correlationKey);
|
|
1584
|
+
if ((current.lease_owner !== owner || current.version !== snapshot.monitor.version || current.active_version_id !== snapshot.monitor.active_version_id) && correlated) {
|
|
1585
|
+
const check = this.database.prepare("SELECT * FROM monitor_checks WHERE id = ?").get(snapshot.check_id);
|
|
1586
|
+
assert.ok(check, `monitor check ${snapshot.check_id} does not exist`);
|
|
1587
|
+
assert.equal(check.state, "running", `monitor check ${snapshot.check_id} is not running`);
|
|
1588
|
+
const timestamp = this.now();
|
|
1589
|
+
const sequence = (this.latestObservationRow(current.id)?.sequence ?? -1) + 1;
|
|
1590
|
+
this.database.prepare(`
|
|
1591
|
+
INSERT INTO observations (id, check_id, monitor_id, sequence, state_hash, data_json, observed_at)
|
|
1592
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1593
|
+
`).run(this.idFactory(), snapshot.check_id, current.id, sequence, hashJson(observation), encodeJson(observation), timestamp);
|
|
1594
|
+
if (!this.database.prepare("SELECT event_id FROM monitor_triggers WHERE monitor_id = ? AND trigger_key = ?").get(current.id, proposal.key)) this.insertBoundMonitorEvent({
|
|
1595
|
+
monitorRow: current,
|
|
1596
|
+
checkId: snapshot.check_id,
|
|
1597
|
+
proposal,
|
|
1598
|
+
claimWait: false,
|
|
1599
|
+
timestamp
|
|
1600
|
+
});
|
|
1601
|
+
const nextState = current.lifecycle === "one_shot" ? "completed" : "triggered";
|
|
1602
|
+
this.database.prepare(`
|
|
1603
|
+
UPDATE monitors
|
|
1604
|
+
SET state = ?, paused = 0, next_check_at = NULL, consecutive_failures = 0,
|
|
1605
|
+
lease_owner = NULL, lease_expires_at = NULL,
|
|
1606
|
+
terminal_reason_code = NULL, terminal_reason_detail = NULL,
|
|
1607
|
+
terminal_actor = NULL, terminal_at = NULL,
|
|
1608
|
+
version = version + 1, updated_at = ?
|
|
1609
|
+
WHERE id = ?
|
|
1610
|
+
`).run(nextState, timestamp, current.id);
|
|
1611
|
+
this.database.prepare("UPDATE monitor_checks SET state = 'succeeded', finished_at = ? WHERE id = ? AND state = 'running'").run(timestamp, snapshot.check_id);
|
|
1612
|
+
return {
|
|
1613
|
+
status: "converged",
|
|
1614
|
+
monitor: this.inspectMonitor(current.id),
|
|
1615
|
+
eventIds: [correlated.id],
|
|
1616
|
+
sessionIds: []
|
|
1617
|
+
};
|
|
1618
|
+
}
|
|
986
1619
|
const row = this.requireMonitorCommit(snapshot, owner);
|
|
987
1620
|
const timestamp = this.now();
|
|
988
1621
|
const sequence = (this.latestObservationRow(row.id)?.sequence ?? -1) + 1;
|
|
@@ -1082,6 +1715,43 @@ var RelayStore = class {
|
|
|
1082
1715
|
};
|
|
1083
1716
|
});
|
|
1084
1717
|
}
|
|
1718
|
+
expireMonitorCheck(snapshot, owner) {
|
|
1719
|
+
return this.transaction(() => {
|
|
1720
|
+
const row = this.requireMonitorCommit(snapshot, owner);
|
|
1721
|
+
const timestamp = this.now();
|
|
1722
|
+
this.database.prepare(`
|
|
1723
|
+
UPDATE monitor_checks
|
|
1724
|
+
SET state = 'failed', error_class = 'bundle_expired',
|
|
1725
|
+
error = 'custom Monitor Bundle expired', finished_at = ?
|
|
1726
|
+
WHERE id = ? AND state = 'running'
|
|
1727
|
+
`).run(timestamp, snapshot.check_id);
|
|
1728
|
+
this.database.prepare(`
|
|
1729
|
+
UPDATE monitors
|
|
1730
|
+
SET state = 'expired', paused = 0, next_check_at = NULL,
|
|
1731
|
+
consecutive_failures = 0, lease_owner = NULL, lease_expires_at = NULL,
|
|
1732
|
+
terminal_reason_code = 'bundle_expired',
|
|
1733
|
+
terminal_reason_detail = 'The custom Monitor Bundle reached its declared expiry.',
|
|
1734
|
+
terminal_actor = 'relay.monitors', terminal_at = ?,
|
|
1735
|
+
version = version + 1, updated_at = ?
|
|
1736
|
+
WHERE id = ?
|
|
1737
|
+
`).run(timestamp, timestamp, row.id);
|
|
1738
|
+
this.database.prepare(`
|
|
1739
|
+
UPDATE waits SET status = 'cancelled', version = version + 1, updated_at = ?
|
|
1740
|
+
WHERE id = ? AND status = 'active'
|
|
1741
|
+
`).run(timestamp, row.wait_id);
|
|
1742
|
+
if (!this.database.prepare("SELECT 1 FROM waits WHERE session_id = ? AND status IN ('active', 'claimed') LIMIT 1").get(row.session_id)) this.database.prepare(`
|
|
1743
|
+
UPDATE sessions SET state = 'created', lease_owner = NULL, lease_expires_at = NULL,
|
|
1744
|
+
version = version + 1, updated_at = ? WHERE id = ?
|
|
1745
|
+
`).run(timestamp, row.session_id);
|
|
1746
|
+
this.bumpRoutingEpoch();
|
|
1747
|
+
return {
|
|
1748
|
+
status: "expired",
|
|
1749
|
+
monitor: this.inspectMonitor(row.id),
|
|
1750
|
+
eventIds: [],
|
|
1751
|
+
sessionIds: []
|
|
1752
|
+
};
|
|
1753
|
+
});
|
|
1754
|
+
}
|
|
1085
1755
|
getRoutingEpoch() {
|
|
1086
1756
|
return this.database.prepare("SELECT value FROM runtime_counters WHERE name = 'routing_epoch'").get().value;
|
|
1087
1757
|
}
|
|
@@ -1117,7 +1787,8 @@ var RelayStore = class {
|
|
|
1117
1787
|
caused_by: wait.caused_by,
|
|
1118
1788
|
actors: wait.actors ?? [],
|
|
1119
1789
|
entities: wait.entities ?? [],
|
|
1120
|
-
prior_exchange: wait.prior_exchange
|
|
1790
|
+
prior_exchange: wait.prior_exchange,
|
|
1791
|
+
continuation: wait.continuation
|
|
1121
1792
|
};
|
|
1122
1793
|
this.database.prepare(`
|
|
1123
1794
|
INSERT INTO waits (
|
|
@@ -1144,14 +1815,16 @@ var RelayStore = class {
|
|
|
1144
1815
|
interval_seconds: monitor.schedule?.interval_seconds ?? 60,
|
|
1145
1816
|
jitter_seconds: monitor.schedule?.jitter_seconds ?? 0
|
|
1146
1817
|
};
|
|
1147
|
-
assert.ok(schedule.interval_seconds
|
|
1818
|
+
assert.ok(Number.isSafeInteger(schedule.interval_seconds) && schedule.interval_seconds >= 1 && schedule.interval_seconds <= 86400, "monitor interval_seconds must be a whole number from 1 to 86400");
|
|
1819
|
+
assert.ok(Number.isSafeInteger(schedule.jitter_seconds) && schedule.jitter_seconds >= 0 && schedule.jitter_seconds <= Math.min(schedule.interval_seconds, 3600), "monitor jitter_seconds must be a bounded whole number no greater than interval_seconds");
|
|
1148
1820
|
const retry = {
|
|
1149
1821
|
degraded_after: monitor.retry?.degraded_after ?? 1,
|
|
1150
1822
|
fail_after: monitor.retry?.fail_after ?? 3,
|
|
1151
1823
|
backoff_seconds: monitor.retry?.backoff_seconds ?? []
|
|
1152
1824
|
};
|
|
1153
|
-
assert.ok(retry.degraded_after
|
|
1154
|
-
assert.ok(retry.fail_after >= retry.degraded_after, "monitor fail_after must follow degraded_after");
|
|
1825
|
+
assert.ok(Number.isSafeInteger(retry.degraded_after) && retry.degraded_after >= 1 && retry.degraded_after <= 100, "monitor degraded_after must be a whole number from 1 to 100");
|
|
1826
|
+
assert.ok(Number.isSafeInteger(retry.fail_after) && retry.fail_after >= retry.degraded_after && retry.fail_after <= 100, "monitor fail_after must follow degraded_after and be at most 100");
|
|
1827
|
+
assert.ok(Array.isArray(retry.backoff_seconds) && retry.backoff_seconds.length <= 20 && retry.backoff_seconds.every((value) => Number.isSafeInteger(value) && value >= 1 && value <= 86400), "monitor backoff_seconds must contain at most 20 whole-second delays from 1 to 86400");
|
|
1155
1828
|
const versionId = this.idFactory();
|
|
1156
1829
|
const manifest = {
|
|
1157
1830
|
observer: monitor.observer ?? (monitor.detector.kind === "deadline_reached" ? { provider: "clock" } : null),
|
|
@@ -1161,7 +1834,7 @@ var RelayStore = class {
|
|
|
1161
1834
|
capabilities: monitor.capabilities ?? {},
|
|
1162
1835
|
artifact: monitor.artifact ?? { kind: "fixture" }
|
|
1163
1836
|
};
|
|
1164
|
-
const artifactHash = monitor.artifact?.sha256 ?? hashJson(manifest);
|
|
1837
|
+
const artifactHash = monitor.artifact?.version_sha256 ?? monitor.artifact?.sha256 ?? hashJson(manifest);
|
|
1165
1838
|
this.database.prepare(`
|
|
1166
1839
|
INSERT INTO monitors (
|
|
1167
1840
|
id, session_id, wait_id, state, lifecycle, fire_on_initial_match,
|
|
@@ -1239,14 +1912,25 @@ var RelayStore = class {
|
|
|
1239
1912
|
return deliveryId;
|
|
1240
1913
|
}
|
|
1241
1914
|
hydrateDelivery(row, includeEvent = false) {
|
|
1242
|
-
const
|
|
1915
|
+
const matchedWaitRows = this.database.prepare(`
|
|
1916
|
+
SELECT dw.wait_id, dw.ordinal, dw.wait_snapshot_json, w.*
|
|
1917
|
+
FROM delivery_waits dw
|
|
1918
|
+
JOIN waits w ON w.id = dw.wait_id
|
|
1919
|
+
WHERE dw.delivery_id = ?
|
|
1920
|
+
ORDER BY dw.ordinal, dw.wait_id
|
|
1921
|
+
`).all(row.id);
|
|
1922
|
+
const waitIds = matchedWaitRows.map((item) => item.wait_id);
|
|
1923
|
+
const matchedWaits = matchedWaitRows.map((item) => item.wait_snapshot_json == null ? hydrateWait(item) : decodeJson(item.wait_snapshot_json));
|
|
1924
|
+
const decision = this.database.prepare("SELECT evidence_json FROM routing_decisions WHERE event_id = ?").get(row.event_id);
|
|
1243
1925
|
const delivery = {
|
|
1244
1926
|
delivery_id: row.id,
|
|
1245
1927
|
event_id: row.event_id,
|
|
1246
1928
|
session_id: row.session_id,
|
|
1247
1929
|
state: row.state,
|
|
1248
1930
|
wait_ids: waitIds,
|
|
1931
|
+
matched_waits: matchedWaits,
|
|
1249
1932
|
relation: row.relation,
|
|
1933
|
+
routing_evidence: decision ? decodeJson(decision.evidence_json) : [],
|
|
1250
1934
|
confidence: row.confidence,
|
|
1251
1935
|
created_at: row.created_at,
|
|
1252
1936
|
updated_at: row.updated_at
|
|
@@ -1254,10 +1938,17 @@ var RelayStore = class {
|
|
|
1254
1938
|
if (includeEvent) delivery.event = this.getEvent(row.event_id)?.payload ?? null;
|
|
1255
1939
|
return delivery;
|
|
1256
1940
|
}
|
|
1257
|
-
findDuplicateEvent(input) {
|
|
1941
|
+
findDuplicateEvent(input, correlationKey = null) {
|
|
1258
1942
|
if (input.source_event_id != null) {
|
|
1259
1943
|
const bySourceId = this.database.prepare("SELECT * FROM events WHERE source = ? AND source_event_id = ?").get(input.source, input.source_event_id);
|
|
1260
|
-
if (bySourceId)
|
|
1944
|
+
if (bySourceId) {
|
|
1945
|
+
assert.equal(bySourceId.fingerprint, input.fingerprint, `source Event identity ${input.source}/${input.source_event_id} was reused with conflicting content`);
|
|
1946
|
+
return bySourceId;
|
|
1947
|
+
}
|
|
1948
|
+
}
|
|
1949
|
+
if (correlationKey != null) {
|
|
1950
|
+
const correlated = this.database.prepare("SELECT * FROM events WHERE correlation_key = ?").get(correlationKey);
|
|
1951
|
+
if (correlated) return correlated;
|
|
1261
1952
|
}
|
|
1262
1953
|
return this.database.prepare("SELECT * FROM events WHERE source = ? AND fingerprint = ?").get(input.source, input.fingerprint);
|
|
1263
1954
|
}
|
|
@@ -1292,12 +1983,14 @@ var RelayStore = class {
|
|
|
1292
1983
|
hydrateMonitor(row) {
|
|
1293
1984
|
const versionRow = this.database.prepare("SELECT * FROM monitor_versions WHERE id = ?").get(row.active_version_id);
|
|
1294
1985
|
const observationRow = this.latestObservationRow(row.id);
|
|
1986
|
+
const checkRow = this.database.prepare("SELECT * FROM monitor_checks WHERE monitor_id = ? ORDER BY started_at DESC, id DESC LIMIT 1").get(row.id);
|
|
1987
|
+
const triggerRow = this.database.prepare("SELECT * FROM monitor_triggers WHERE monitor_id = ? ORDER BY created_at DESC, id DESC LIMIT 1").get(row.id);
|
|
1295
1988
|
const manifest = decodeJson(versionRow?.manifest_json) ?? {};
|
|
1296
1989
|
return {
|
|
1297
1990
|
monitor_id: row.id,
|
|
1298
1991
|
session_id: row.session_id,
|
|
1299
1992
|
wait_id: row.wait_id,
|
|
1300
|
-
state: row.state,
|
|
1993
|
+
state: row.paused ? "paused" : row.state,
|
|
1301
1994
|
lifecycle: row.lifecycle,
|
|
1302
1995
|
fire_on_initial_match: Boolean(row.fire_on_initial_match),
|
|
1303
1996
|
active_version_id: row.active_version_id,
|
|
@@ -1310,10 +2003,18 @@ var RelayStore = class {
|
|
|
1310
2003
|
capabilities: decodeJson(row.capabilities_json),
|
|
1311
2004
|
next_check_at: row.next_check_at,
|
|
1312
2005
|
consecutive_failures: row.consecutive_failures,
|
|
2006
|
+
terminal_reason: row.terminal_reason_code == null ? null : {
|
|
2007
|
+
code: row.terminal_reason_code,
|
|
2008
|
+
detail: row.terminal_reason_detail ?? "",
|
|
2009
|
+
actor: row.terminal_actor,
|
|
2010
|
+
at: row.terminal_at
|
|
2011
|
+
},
|
|
1313
2012
|
version: row.version,
|
|
1314
2013
|
lease_owner: row.lease_owner,
|
|
1315
2014
|
lease_expires_at: row.lease_expires_at,
|
|
2015
|
+
last_check: checkRow ? hydrateMonitorCheck(checkRow) : null,
|
|
1316
2016
|
last_observation: observationRow ? hydrateObservation(observationRow) : null,
|
|
2017
|
+
last_trigger: triggerRow ? hydrateMonitorTrigger(triggerRow) : null,
|
|
1317
2018
|
created_at: row.created_at,
|
|
1318
2019
|
updated_at: row.updated_at
|
|
1319
2020
|
};
|
|
@@ -1323,6 +2024,18 @@ var RelayStore = class {
|
|
|
1323
2024
|
assert.equal(typeof proposal.key, "string", "monitor trigger key is required");
|
|
1324
2025
|
const existing = this.database.prepare("SELECT event_id FROM monitor_triggers WHERE monitor_id = ? AND trigger_key = ?").get(monitorRow.id, proposal.key);
|
|
1325
2026
|
if (existing) return existing.event_id;
|
|
2027
|
+
const correlationKey = typeof proposal.correlation_key === "string" && proposal.correlation_key.length > 0 ? proposal.correlation_key : null;
|
|
2028
|
+
if (correlationKey != null) {
|
|
2029
|
+
assert.ok(correlationKey.length <= 1024, "Monitor correlation key is too long");
|
|
2030
|
+
const correlated = this.database.prepare("SELECT id FROM events WHERE correlation_key = ?").get(correlationKey);
|
|
2031
|
+
if (correlated) {
|
|
2032
|
+
this.database.prepare(`
|
|
2033
|
+
INSERT INTO monitor_triggers (id, monitor_id, check_id, trigger_key, event_id, created_at)
|
|
2034
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
2035
|
+
`).run(this.idFactory(), monitorRow.id, checkId, proposal.key, correlated.id, timestamp);
|
|
2036
|
+
return correlated.id;
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
1326
2039
|
const waitRow = this.database.prepare("SELECT * FROM waits WHERE id = ?").get(monitorRow.wait_id);
|
|
1327
2040
|
assert.ok(waitRow, `monitor wait ${monitorRow.wait_id} does not exist`);
|
|
1328
2041
|
assert.equal(waitRow.session_id, monitorRow.session_id, "monitor binding owner changed");
|
|
@@ -1337,6 +2050,7 @@ var RelayStore = class {
|
|
|
1337
2050
|
monitor_id: monitorRow.id,
|
|
1338
2051
|
trigger_key: proposal.key
|
|
1339
2052
|
}),
|
|
2053
|
+
...correlationKey ? { correlation_key: correlationKey } : {},
|
|
1340
2054
|
channel: "monitor",
|
|
1341
2055
|
type: proposal.type,
|
|
1342
2056
|
monitor_id: monitorRow.id,
|
|
@@ -1345,10 +2059,10 @@ var RelayStore = class {
|
|
|
1345
2059
|
};
|
|
1346
2060
|
this.database.prepare(`
|
|
1347
2061
|
INSERT INTO events (
|
|
1348
|
-
id, source, source_event_id, fingerprint, payload_json,
|
|
2062
|
+
id, source, source_event_id, fingerprint, correlation_key, payload_json,
|
|
1349
2063
|
state, version, received_at, updated_at
|
|
1350
|
-
) VALUES (?, 'relay-monitor', ?, ?, ?, 'dispatched', 1, ?, ?)
|
|
1351
|
-
`).run(eventId, sourceEventId, payload.fingerprint, encodeJson(payload), timestamp, timestamp);
|
|
2064
|
+
) VALUES (?, 'relay-monitor', ?, ?, ?, ?, 'dispatched', 1, ?, ?)
|
|
2065
|
+
`).run(eventId, sourceEventId, payload.fingerprint, correlationKey, encodeJson(payload), timestamp, timestamp);
|
|
1352
2066
|
const waitIds = claimWait ? [monitorRow.wait_id] : [];
|
|
1353
2067
|
const relation = claimWait ? `bound Monitor ${monitorRow.id} detected ${proposal.type}` : `bound Monitor ${monitorRow.id} failed`;
|
|
1354
2068
|
const decision = {
|
|
@@ -1377,7 +2091,11 @@ var RelayStore = class {
|
|
|
1377
2091
|
timestamp
|
|
1378
2092
|
});
|
|
1379
2093
|
if (claimWait) {
|
|
1380
|
-
this.database.prepare(
|
|
2094
|
+
this.database.prepare(`
|
|
2095
|
+
INSERT INTO delivery_waits (
|
|
2096
|
+
delivery_id, wait_id, ordinal, wait_snapshot_json
|
|
2097
|
+
) VALUES (?, ?, 0, ?)
|
|
2098
|
+
`).run(deliveryId, monitorRow.wait_id, encodeJson(hydrateWait(waitRow)));
|
|
1381
2099
|
this.database.prepare(`
|
|
1382
2100
|
UPDATE waits
|
|
1383
2101
|
SET status = 'claimed', version = version + 1, updated_at = ?
|
|
@@ -1440,6 +2158,7 @@ function hydrateEvent(row) {
|
|
|
1440
2158
|
source: row.source,
|
|
1441
2159
|
source_event_id: row.source_event_id,
|
|
1442
2160
|
fingerprint: row.fingerprint,
|
|
2161
|
+
correlation_key: row.correlation_key,
|
|
1443
2162
|
payload: decodeJson(row.payload_json),
|
|
1444
2163
|
state: row.state,
|
|
1445
2164
|
version: row.version,
|
|
@@ -1470,6 +2189,10 @@ function hydrateActivation(row) {
|
|
|
1470
2189
|
lease_expires_at: row.lease_expires_at ?? null,
|
|
1471
2190
|
accepted_at: row.accepted_at ?? null,
|
|
1472
2191
|
last_error: row.last_error ?? null,
|
|
2192
|
+
attempt_count: row.attempt_count ?? 0,
|
|
2193
|
+
next_attempt_at: row.next_attempt_at ?? null,
|
|
2194
|
+
terminal_reason_code: row.terminal_reason_code ?? null,
|
|
2195
|
+
terminal_at: row.terminal_at ?? null,
|
|
1473
2196
|
committed_at: row.committed_at,
|
|
1474
2197
|
created_at: row.created_at,
|
|
1475
2198
|
updated_at: row.updated_at
|
|
@@ -1531,6 +2254,25 @@ function encodeJson(value) {
|
|
|
1531
2254
|
function decodeJson(value) {
|
|
1532
2255
|
return value == null ? null : JSON.parse(value);
|
|
1533
2256
|
}
|
|
2257
|
+
function encodeHistoryCursor(row) {
|
|
2258
|
+
return Buffer.from(JSON.stringify({
|
|
2259
|
+
received_at: row.received_at,
|
|
2260
|
+
id: row.id
|
|
2261
|
+
}), "utf8").toString("base64url");
|
|
2262
|
+
}
|
|
2263
|
+
function decodeHistoryCursor(value) {
|
|
2264
|
+
assert.ok(typeof value === "string" && value.length > 0 && value.length <= 2048, "Event history cursor is invalid");
|
|
2265
|
+
let parsed;
|
|
2266
|
+
try {
|
|
2267
|
+
parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
2268
|
+
} catch {
|
|
2269
|
+
assert.fail("Event history cursor is invalid");
|
|
2270
|
+
}
|
|
2271
|
+
assert.ok(parsed && typeof parsed === "object" && Object.keys(parsed).length === 2, "Event history cursor is invalid");
|
|
2272
|
+
assert.ok(typeof parsed.id === "string" && parsed.id.length > 0 && parsed.id.length <= 512, "Event history cursor is invalid");
|
|
2273
|
+
assert.ok(typeof parsed.received_at === "string" && parsed.received_at.length <= 64 && !Number.isNaN(Date.parse(parsed.received_at)), "Event history cursor is invalid");
|
|
2274
|
+
return parsed;
|
|
2275
|
+
}
|
|
1534
2276
|
function toIso(value) {
|
|
1535
2277
|
return value instanceof Date ? value.toISOString() : new Date(value).toISOString();
|
|
1536
2278
|
}
|
|
@@ -1538,7 +2280,7 @@ function toIso(value) {
|
|
|
1538
2280
|
//#region events-service.js
|
|
1539
2281
|
var RelayEventsService = class extends Service {
|
|
1540
2282
|
apiVersion = 1;
|
|
1541
|
-
constructor(ctx, { databasePath, inbox, dispatchPollIntervalMs = 1e3, clock, idFactory } = {}) {
|
|
2283
|
+
constructor(ctx, { databasePath, inbox, dispatchPollIntervalMs = 1e3, routingFailureLimit = 3, deliveryFailureLimit = 5, deliveryRetryBaseMs = 1e3, globalEventsPerMinute = 600, globalConcurrentEvents = 32, clock, idFactory } = {}) {
|
|
1542
2284
|
super(ctx, "relayEvents");
|
|
1543
2285
|
assert.equal(typeof inbox?.deliver, "function", "Events requires inbox.deliver()");
|
|
1544
2286
|
const resolvedPath = resolveDatabasePath(databasePath);
|
|
@@ -1550,7 +2292,16 @@ var RelayEventsService = class extends Service {
|
|
|
1550
2292
|
this.router = createExactEventRouter();
|
|
1551
2293
|
this.routerProvider = null;
|
|
1552
2294
|
this.monitorProvider = null;
|
|
2295
|
+
this.boundEventSources = /* @__PURE__ */ new Map();
|
|
2296
|
+
this.notificationProvider = null;
|
|
2297
|
+
this.connectorProviders = /* @__PURE__ */ new Map();
|
|
2298
|
+
this.bundleCatalogProvider = null;
|
|
1553
2299
|
this.operations = new OperationGate();
|
|
2300
|
+
this.admission = new EventAdmissionGate({
|
|
2301
|
+
eventsPerMinute: globalEventsPerMinute,
|
|
2302
|
+
concurrentEvents: globalConcurrentEvents,
|
|
2303
|
+
clock: () => (clock ? clock() : /* @__PURE__ */ new Date()).getTime()
|
|
2304
|
+
});
|
|
1554
2305
|
const service = this;
|
|
1555
2306
|
this.runtime = new RelayRuntime({
|
|
1556
2307
|
store: this.store,
|
|
@@ -1565,7 +2316,10 @@ var RelayEventsService = class extends Service {
|
|
|
1565
2316
|
},
|
|
1566
2317
|
inbox,
|
|
1567
2318
|
monitorRegistrar: { prepare: (input) => this.prepareMonitors(input) },
|
|
1568
|
-
workerId: "relay-events-dispatcher"
|
|
2319
|
+
workerId: "relay-events-dispatcher",
|
|
2320
|
+
routingFailureLimit,
|
|
2321
|
+
deliveryFailureLimit,
|
|
2322
|
+
deliveryRetryBaseMs
|
|
1569
2323
|
});
|
|
1570
2324
|
this.stopped = false;
|
|
1571
2325
|
this.dispatchTimer = null;
|
|
@@ -1600,6 +2354,76 @@ var RelayEventsService = class extends Service {
|
|
|
1600
2354
|
if (this.monitorProvider === provider) this.monitorProvider = null;
|
|
1601
2355
|
};
|
|
1602
2356
|
}
|
|
2357
|
+
registerBoundEventSource(provider) {
|
|
2358
|
+
if (this.stopped) throw new Error("Relay Events is shutting down");
|
|
2359
|
+
validateBoundEventSourceProvider(provider);
|
|
2360
|
+
if (this.boundEventSources.has(provider.id)) throw new Error(`bound Event source provider ${provider.id} is already registered`);
|
|
2361
|
+
const sources = new Set(provider.sources);
|
|
2362
|
+
for (const active of this.boundEventSources.values()) for (const source of sources) if (active.sources.has(source)) throw new Error(`bound Event source ${source} is already registered by ${active.id}`);
|
|
2363
|
+
const registration = {
|
|
2364
|
+
id: provider.id,
|
|
2365
|
+
sources,
|
|
2366
|
+
active: true
|
|
2367
|
+
};
|
|
2368
|
+
this.boundEventSources.set(provider.id, registration);
|
|
2369
|
+
return {
|
|
2370
|
+
id: provider.id,
|
|
2371
|
+
handleEvent: ({ event, binding }) => {
|
|
2372
|
+
if (!registration.active || this.stopped) throw new Error(`bound Event source provider ${provider.id} is not active`);
|
|
2373
|
+
if (!sources.has(event?.source)) throw new Error(`bound Event source provider ${provider.id} cannot ingest ${event?.source ?? "unknown"}`);
|
|
2374
|
+
return this.admission.run(() => this.operations.run(async () => this.finalizeResult(binding == null ? await this.runtime.handleTrustedEvent(event, { providerId: provider.id }) : await this.runtime.handleBoundEvent(event, binding, { providerId: provider.id }))));
|
|
2375
|
+
},
|
|
2376
|
+
dismissEvent: ({ event, summary }) => {
|
|
2377
|
+
if (!registration.active || this.stopped) throw new Error(`bound Event source provider ${provider.id} is not active`);
|
|
2378
|
+
if (!sources.has(event?.source)) throw new Error(`bound Event source provider ${provider.id} cannot ingest ${event?.source ?? "unknown"}`);
|
|
2379
|
+
return this.admission.run(() => this.operations.run(async () => this.finalizeResult(await this.runtime.handleTrustedDismissal(event, {
|
|
2380
|
+
providerId: provider.id,
|
|
2381
|
+
summary
|
|
2382
|
+
}))));
|
|
2383
|
+
},
|
|
2384
|
+
dispose: () => {
|
|
2385
|
+
if (!registration.active) return;
|
|
2386
|
+
registration.active = false;
|
|
2387
|
+
if (this.boundEventSources.get(provider.id) === registration) this.boundEventSources.delete(provider.id);
|
|
2388
|
+
}
|
|
2389
|
+
};
|
|
2390
|
+
}
|
|
2391
|
+
registerNotificationProvider(provider) {
|
|
2392
|
+
if (this.stopped) throw new Error("Relay Events is shutting down");
|
|
2393
|
+
if (!provider || typeof provider !== "object" || !/^[a-z][a-z0-9._-]{0,63}$/u.test(provider.id ?? "") || typeof provider.notify !== "function") throw new TypeError("notification provider requires a lowercase stable id and notify()");
|
|
2394
|
+
if (this.notificationProvider) throw new Error(`notification provider ${this.notificationProvider.id} is already registered`);
|
|
2395
|
+
this.notificationProvider = provider;
|
|
2396
|
+
let active = true;
|
|
2397
|
+
return () => {
|
|
2398
|
+
if (!active) return;
|
|
2399
|
+
active = false;
|
|
2400
|
+
if (this.notificationProvider === provider) this.notificationProvider = null;
|
|
2401
|
+
};
|
|
2402
|
+
}
|
|
2403
|
+
registerConnectorProvider(provider) {
|
|
2404
|
+
if (this.stopped) throw new Error("Relay Events is shutting down");
|
|
2405
|
+
if (!provider || !/^[a-z][a-z0-9._-]{0,63}$/u.test(provider.id ?? "") || typeof provider.inspect !== "function" || typeof provider.execute !== "function") throw new TypeError("connector provider requires a lowercase stable id, inspect(), and execute()");
|
|
2406
|
+
if (this.connectorProviders.has(provider.id)) throw new Error(`connector provider ${provider.id} is already registered`);
|
|
2407
|
+
this.connectorProviders.set(provider.id, provider);
|
|
2408
|
+
let active = true;
|
|
2409
|
+
return () => {
|
|
2410
|
+
if (!active) return;
|
|
2411
|
+
active = false;
|
|
2412
|
+
if (this.connectorProviders.get(provider.id) === provider) this.connectorProviders.delete(provider.id);
|
|
2413
|
+
};
|
|
2414
|
+
}
|
|
2415
|
+
registerBundleCatalogProvider(provider) {
|
|
2416
|
+
if (this.stopped) throw new Error("Relay Events is shutting down");
|
|
2417
|
+
if (!provider || !/^[a-z][a-z0-9._-]{0,63}$/u.test(provider.id ?? "") || typeof provider.list !== "function") throw new TypeError("Bundle catalog provider requires a lowercase stable id and list()");
|
|
2418
|
+
if (this.bundleCatalogProvider) throw new Error(`Bundle catalog provider ${this.bundleCatalogProvider.id} is already registered`);
|
|
2419
|
+
this.bundleCatalogProvider = provider;
|
|
2420
|
+
let active = true;
|
|
2421
|
+
return () => {
|
|
2422
|
+
if (!active) return;
|
|
2423
|
+
active = false;
|
|
2424
|
+
if (this.bundleCatalogProvider === provider) this.bundleCatalogProvider = null;
|
|
2425
|
+
};
|
|
2426
|
+
}
|
|
1603
2427
|
registerWaits(input) {
|
|
1604
2428
|
return this.operations.run(() => this.runtime.registerWaits(input));
|
|
1605
2429
|
}
|
|
@@ -1609,8 +2433,116 @@ var RelayEventsService = class extends Service {
|
|
|
1609
2433
|
listWaits() {
|
|
1610
2434
|
return this.operations.run(() => this.runtime.listWaits());
|
|
1611
2435
|
}
|
|
2436
|
+
managementSnapshot({ eventCursor = null, eventLimit = 20, bundleCursor = null, bundleLimit = 20, locale = "en-US" } = {}) {
|
|
2437
|
+
return this.operations.run(async () => {
|
|
2438
|
+
assert.ok(Number.isSafeInteger(bundleLimit) && bundleLimit > 0 && bundleLimit <= 100, "Bundle catalog limit is invalid");
|
|
2439
|
+
const eventPage = this.store.listEventsPage({
|
|
2440
|
+
cursor: eventCursor,
|
|
2441
|
+
limit: eventLimit
|
|
2442
|
+
});
|
|
2443
|
+
const providerBundleTypes = this.bundleCatalogProvider ? await this.bundleCatalogProvider.list({ locale: locale === "zh-CN" ? "zh-CN" : "en-US" }) : [];
|
|
2444
|
+
if (!Array.isArray(providerBundleTypes)) throw new TypeError("Bundle catalog provider returned an invalid list");
|
|
2445
|
+
const allBundleTypes = [...providerBundleTypes];
|
|
2446
|
+
allBundleTypes.sort((left, right) => bundleCatalogKey(left).localeCompare(bundleCatalogKey(right), "en"));
|
|
2447
|
+
const after = bundleCursor == null ? null : decodeBundleCatalogCursor(bundleCursor);
|
|
2448
|
+
const remainingBundleTypes = after == null ? allBundleTypes : allBundleTypes.filter((entry) => bundleCatalogKey(entry) > after);
|
|
2449
|
+
const bundleTypes = remainingBundleTypes.slice(0, bundleLimit);
|
|
2450
|
+
return {
|
|
2451
|
+
registrations: this.store.listAllWaitRegistrations(),
|
|
2452
|
+
bundle_types: bundleTypes,
|
|
2453
|
+
bundle_page: {
|
|
2454
|
+
next_cursor: remainingBundleTypes.length > bundleLimit && bundleTypes.length > 0 ? encodeBundleCatalogCursor(bundleCatalogKey(bundleTypes.at(-1))) : null,
|
|
2455
|
+
total: allBundleTypes.length,
|
|
2456
|
+
limit: bundleLimit
|
|
2457
|
+
},
|
|
2458
|
+
events: eventPage.items,
|
|
2459
|
+
event_page: {
|
|
2460
|
+
next_cursor: eventPage.next_cursor,
|
|
2461
|
+
total: eventPage.total,
|
|
2462
|
+
limit: eventLimit
|
|
2463
|
+
},
|
|
2464
|
+
connectors: await Promise.all([...this.connectorProviders.values()].map(async (provider) => ({
|
|
2465
|
+
id: provider.id,
|
|
2466
|
+
...await provider.inspect()
|
|
2467
|
+
})))
|
|
2468
|
+
};
|
|
2469
|
+
});
|
|
2470
|
+
}
|
|
2471
|
+
cleanupRetention(options) {
|
|
2472
|
+
return this.operations.run(() => this.store.cleanupRetention(options));
|
|
2473
|
+
}
|
|
2474
|
+
executeConnectorAction(connectorId, action, input = {}) {
|
|
2475
|
+
return this.operations.run(async () => {
|
|
2476
|
+
const provider = this.connectorProviders.get(connectorId);
|
|
2477
|
+
assert.ok(provider, `connector provider ${connectorId} is not available`);
|
|
2478
|
+
await provider.execute(action, input);
|
|
2479
|
+
return { connector: {
|
|
2480
|
+
id: provider.id,
|
|
2481
|
+
...await provider.inspect()
|
|
2482
|
+
} };
|
|
2483
|
+
});
|
|
2484
|
+
}
|
|
2485
|
+
inspectMonitor(monitorId) {
|
|
2486
|
+
return this.operations.run(() => this.store.inspectMonitor(monitorId));
|
|
2487
|
+
}
|
|
2488
|
+
pauseMonitor(monitorId, options) {
|
|
2489
|
+
return this.operations.run(() => this.store.pauseMonitor(monitorId, options));
|
|
2490
|
+
}
|
|
2491
|
+
resumeMonitor(monitorId, options) {
|
|
2492
|
+
return this.operations.run(() => this.store.resumeMonitor(monitorId, options));
|
|
2493
|
+
}
|
|
2494
|
+
updateMonitorCadence(monitorId, intervalSeconds, options) {
|
|
2495
|
+
return this.operations.run(() => this.store.updateMonitorCadence(monitorId, intervalSeconds, options));
|
|
2496
|
+
}
|
|
2497
|
+
rebaselineMonitor(monitorId, proposal, options = {}) {
|
|
2498
|
+
return this.operations.run(async () => {
|
|
2499
|
+
const current = this.store.inspectMonitor(monitorId);
|
|
2500
|
+
assert.ok(current, `monitor ${monitorId} does not exist`);
|
|
2501
|
+
if (!this.monitorProvider) throw new Error("Relay Monitors plugin is not installed");
|
|
2502
|
+
const wait = this.store.getWaits(current.session_id).find((candidate) => candidate.wait_id === current.wait_id);
|
|
2503
|
+
assert.ok(wait, `monitor wait ${current.wait_id} does not exist`);
|
|
2504
|
+
const [prepared] = await this.monitorProvider.prepare({
|
|
2505
|
+
waits: [wait],
|
|
2506
|
+
monitors: [{
|
|
2507
|
+
monitor_id: monitorId,
|
|
2508
|
+
wait_id: current.wait_id,
|
|
2509
|
+
lifecycle: current.lifecycle,
|
|
2510
|
+
observer: proposal.observer ?? current.observer,
|
|
2511
|
+
artifact: proposal.artifact ?? current.artifact,
|
|
2512
|
+
detector: proposal.detector ?? current.detector,
|
|
2513
|
+
schedule: proposal.schedule ?? current.schedule,
|
|
2514
|
+
retry: proposal.retry ?? current.retry,
|
|
2515
|
+
capabilities: proposal.capabilities ?? current.capabilities
|
|
2516
|
+
}]
|
|
2517
|
+
});
|
|
2518
|
+
return this.store.rebaselineMonitor(monitorId, prepared, options);
|
|
2519
|
+
});
|
|
2520
|
+
}
|
|
2521
|
+
stopMonitor(monitorId, options) {
|
|
2522
|
+
return this.operations.run(() => this.store.stopMonitor(monitorId, options));
|
|
2523
|
+
}
|
|
2524
|
+
retryActivation(activationId) {
|
|
2525
|
+
return this.operations.run(async () => {
|
|
2526
|
+
const activation = this.store.retryActivation(activationId);
|
|
2527
|
+
const result = await this.runtime.dispatchSession(activation.session_id);
|
|
2528
|
+
return {
|
|
2529
|
+
activation: this.store.getActivation(activationId),
|
|
2530
|
+
result
|
|
2531
|
+
};
|
|
2532
|
+
});
|
|
2533
|
+
}
|
|
2534
|
+
retryNotification(eventId) {
|
|
2535
|
+
return this.operations.run(async () => {
|
|
2536
|
+
const event = this.store.inspectEvent(eventId);
|
|
2537
|
+
assert.ok(event, `event ${eventId} does not exist`);
|
|
2538
|
+
assert.ok(event.notification && (/* @__PURE__ */ new Set(["failed", "unavailable"])).has(event.notification.state), `notification for event ${eventId} is not retryable`);
|
|
2539
|
+
if (event.decision?.disposition === "escalate") return this.notifyEscalation(eventId, { force: true });
|
|
2540
|
+
if (event.deliveries?.some((delivery) => delivery.state === "failed")) return this.notifyTerminalFailure(eventId, { force: true });
|
|
2541
|
+
throw new Error(`event ${eventId} has no retryable terminal notification`);
|
|
2542
|
+
});
|
|
2543
|
+
}
|
|
1612
2544
|
handleEvent(input) {
|
|
1613
|
-
return this.operations.run(() => this.runtime.handleEvent(input));
|
|
2545
|
+
return this.admission.run(() => this.operations.run(async () => this.finalizeResult(await this.runtime.handleEvent(input))));
|
|
1614
2546
|
}
|
|
1615
2547
|
dispatchSession(sessionId) {
|
|
1616
2548
|
return this.operations.run(() => this.runtime.dispatchSession(sessionId));
|
|
@@ -1628,6 +2560,9 @@ var RelayEventsService = class extends Service {
|
|
|
1628
2560
|
failMonitorCheck(...args) {
|
|
1629
2561
|
return this.operations.run(() => this.store.failMonitorCheck(...args));
|
|
1630
2562
|
}
|
|
2563
|
+
expireMonitorCheck(...args) {
|
|
2564
|
+
return this.operations.run(() => this.store.expireMonitorCheck(...args));
|
|
2565
|
+
}
|
|
1631
2566
|
abandonMonitorCheck(...args) {
|
|
1632
2567
|
return this.operations.run(() => this.store.abandonMonitorCheck(...args));
|
|
1633
2568
|
}
|
|
@@ -1641,12 +2576,96 @@ var RelayEventsService = class extends Service {
|
|
|
1641
2576
|
}
|
|
1642
2577
|
async recoverQueuedDeliveries() {
|
|
1643
2578
|
const sessionIds = this.store.listQueuedDeliverySessionIds();
|
|
1644
|
-
|
|
2579
|
+
const results = await Promise.all(sessionIds.map((sessionId) => this.runtime.dispatchSession(sessionId)));
|
|
2580
|
+
for (const result of results) for (const eventId of result.eventIds ?? []) await this.notifyTerminalFailure(eventId);
|
|
2581
|
+
return results;
|
|
2582
|
+
}
|
|
2583
|
+
async recoverPendingWork() {
|
|
2584
|
+
for (const eventId of this.store.listRoutableEventIds()) try {
|
|
2585
|
+
await this.runtime.routeEvent(eventId);
|
|
2586
|
+
await this.notifyEscalation(eventId);
|
|
2587
|
+
} catch (error) {
|
|
2588
|
+
this.ctx.logger?.warn?.(`Relay routing recovery failed for ${eventId}: ${error?.message ?? error}`);
|
|
2589
|
+
}
|
|
2590
|
+
return this.recoverQueuedDeliveries();
|
|
2591
|
+
}
|
|
2592
|
+
async finalizeResult(result) {
|
|
2593
|
+
await this.notifyEscalation(result.event.event_id);
|
|
2594
|
+
if (result.event.deliveries?.some((delivery) => delivery.state === "failed")) await this.notifyTerminalFailure(result.event.event_id);
|
|
2595
|
+
result.event = this.store.inspectEvent(result.event.event_id);
|
|
2596
|
+
return result;
|
|
2597
|
+
}
|
|
2598
|
+
async notifyEscalation(eventId, { force = false } = {}) {
|
|
2599
|
+
const event = this.store.inspectEvent(eventId);
|
|
2600
|
+
if (event?.decision?.disposition !== "escalate") return null;
|
|
2601
|
+
const existing = event.notification;
|
|
2602
|
+
if (existing && !force) return existing;
|
|
2603
|
+
const provider = this.notificationProvider;
|
|
2604
|
+
if (!provider) return this.store.recordNotificationOutcome(eventId, { state: "unavailable" });
|
|
2605
|
+
try {
|
|
2606
|
+
const receipt = await provider.notify({
|
|
2607
|
+
event: {
|
|
2608
|
+
event_id: event.event_id,
|
|
2609
|
+
source: event.source,
|
|
2610
|
+
type: event.payload?.type ?? null
|
|
2611
|
+
},
|
|
2612
|
+
decision: {
|
|
2613
|
+
disposition: "escalate",
|
|
2614
|
+
summary: event.decision.summary,
|
|
2615
|
+
evidence: event.decision.evidence
|
|
2616
|
+
}
|
|
2617
|
+
});
|
|
2618
|
+
return this.store.recordNotificationOutcome(eventId, {
|
|
2619
|
+
provider: provider.id,
|
|
2620
|
+
state: "delivered",
|
|
2621
|
+
receiptId: notificationReceiptId(receipt)
|
|
2622
|
+
});
|
|
2623
|
+
} catch (error) {
|
|
2624
|
+
const errorClass = typeof error?.errorClass === "string" ? error.errorClass.slice(0, 128) : "notification_failed";
|
|
2625
|
+
return this.store.recordNotificationOutcome(eventId, {
|
|
2626
|
+
provider: provider.id,
|
|
2627
|
+
state: "failed",
|
|
2628
|
+
errorClass
|
|
2629
|
+
});
|
|
2630
|
+
}
|
|
2631
|
+
}
|
|
2632
|
+
async notifyTerminalFailure(eventId, { force = false } = {}) {
|
|
2633
|
+
const event = this.store.inspectEvent(eventId);
|
|
2634
|
+
if (!event?.deliveries?.some((delivery) => delivery.state === "failed")) return null;
|
|
2635
|
+
if (event.notification && !force) return event.notification;
|
|
2636
|
+
const provider = this.notificationProvider;
|
|
2637
|
+
if (!provider) return this.store.recordNotificationOutcome(eventId, { state: "unavailable" });
|
|
2638
|
+
try {
|
|
2639
|
+
const receipt = await provider.notify({
|
|
2640
|
+
event: {
|
|
2641
|
+
event_id: event.event_id,
|
|
2642
|
+
source: event.source,
|
|
2643
|
+
type: event.payload?.type ?? null
|
|
2644
|
+
},
|
|
2645
|
+
decision: {
|
|
2646
|
+
disposition: "escalate",
|
|
2647
|
+
summary: "Relay exhausted its Delivery retry budget.",
|
|
2648
|
+
evidence: ["delivery_retry_exhausted"]
|
|
2649
|
+
}
|
|
2650
|
+
});
|
|
2651
|
+
return this.store.recordNotificationOutcome(eventId, {
|
|
2652
|
+
provider: provider.id,
|
|
2653
|
+
state: "delivered",
|
|
2654
|
+
receiptId: notificationReceiptId(receipt)
|
|
2655
|
+
});
|
|
2656
|
+
} catch (error) {
|
|
2657
|
+
const errorClass = typeof error?.errorClass === "string" ? error.errorClass.slice(0, 128) : "notification_failed";
|
|
2658
|
+
return this.store.recordNotificationOutcome(eventId, {
|
|
2659
|
+
provider: provider.id,
|
|
2660
|
+
state: "failed",
|
|
2661
|
+
errorClass
|
|
2662
|
+
});
|
|
2663
|
+
}
|
|
1645
2664
|
}
|
|
1646
2665
|
scheduleRecovery(delay = this.dispatchPollIntervalMs) {
|
|
1647
2666
|
if (this.stopped) return;
|
|
1648
2667
|
this.dispatchTimer = setTimeout(() => {
|
|
1649
|
-
this.operations.run(() => this.
|
|
2668
|
+
this.operations.run(() => this.recoverPendingWork()).catch((error) => {
|
|
1650
2669
|
this.ctx.logger?.error?.(`Relay delivery recovery failed: ${error?.stack ?? error}`);
|
|
1651
2670
|
}).finally(() => this.scheduleRecovery());
|
|
1652
2671
|
}, delay);
|
|
@@ -1656,9 +2675,35 @@ var RelayEventsService = class extends Service {
|
|
|
1656
2675
|
this.stopped = true;
|
|
1657
2676
|
if (this.dispatchTimer) clearTimeout(this.dispatchTimer);
|
|
1658
2677
|
await this.operations.stop();
|
|
2678
|
+
for (const registration of this.boundEventSources.values()) registration.active = false;
|
|
2679
|
+
this.boundEventSources.clear();
|
|
2680
|
+
this.notificationProvider = null;
|
|
2681
|
+
this.connectorProviders.clear();
|
|
1659
2682
|
this.store.close();
|
|
1660
2683
|
}
|
|
1661
2684
|
};
|
|
2685
|
+
function bundleCatalogKey(entry) {
|
|
2686
|
+
return `${entry?.type_id ?? ""}\u0000${String(entry?.bundle_version ?? 0).padStart(12, "0")}\u0000${entry?.artifact_hash ?? ""}`;
|
|
2687
|
+
}
|
|
2688
|
+
function encodeBundleCatalogCursor(after) {
|
|
2689
|
+
return Buffer.from(JSON.stringify({
|
|
2690
|
+
v: 1,
|
|
2691
|
+
after
|
|
2692
|
+
}), "utf8").toString("base64url");
|
|
2693
|
+
}
|
|
2694
|
+
function decodeBundleCatalogCursor(cursor) {
|
|
2695
|
+
try {
|
|
2696
|
+
const value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
|
|
2697
|
+
if (value?.v !== 1 || typeof value.after !== "string" || value.after.length > 1e3) throw new Error();
|
|
2698
|
+
return value.after;
|
|
2699
|
+
} catch {
|
|
2700
|
+
throw new TypeError("Bundle catalog cursor is invalid");
|
|
2701
|
+
}
|
|
2702
|
+
}
|
|
2703
|
+
function notificationReceiptId(receipt) {
|
|
2704
|
+
const value = typeof receipt === "string" ? receipt : receipt?.receipt_id ?? receipt?.id;
|
|
2705
|
+
return typeof value === "string" && value.trim() ? value.trim().slice(0, 256) : null;
|
|
2706
|
+
}
|
|
1662
2707
|
var OperationGate = class {
|
|
1663
2708
|
accepting = true;
|
|
1664
2709
|
inFlight = /* @__PURE__ */ new Set();
|
|
@@ -1676,6 +2721,45 @@ var OperationGate = class {
|
|
|
1676
2721
|
await Promise.allSettled([...this.inFlight]);
|
|
1677
2722
|
}
|
|
1678
2723
|
};
|
|
2724
|
+
var EventAdmissionGate = class {
|
|
2725
|
+
constructor({ eventsPerMinute, concurrentEvents, clock }) {
|
|
2726
|
+
assert.ok(Number.isSafeInteger(eventsPerMinute) && eventsPerMinute > 0 && eventsPerMinute <= 1e6, "globalEventsPerMinute is invalid");
|
|
2727
|
+
assert.ok(Number.isSafeInteger(concurrentEvents) && concurrentEvents > 0 && concurrentEvents <= 1e4, "globalConcurrentEvents is invalid");
|
|
2728
|
+
this.eventsPerMinute = eventsPerMinute;
|
|
2729
|
+
this.concurrentEvents = concurrentEvents;
|
|
2730
|
+
this.clock = clock;
|
|
2731
|
+
this.window = Math.floor(clock() / 6e4);
|
|
2732
|
+
this.used = 0;
|
|
2733
|
+
this.active = 0;
|
|
2734
|
+
}
|
|
2735
|
+
run(operation) {
|
|
2736
|
+
const current = Math.floor(this.clock() / 6e4);
|
|
2737
|
+
if (current !== this.window) {
|
|
2738
|
+
this.window = current;
|
|
2739
|
+
this.used = 0;
|
|
2740
|
+
}
|
|
2741
|
+
if (this.used >= this.eventsPerMinute) throw admissionError("global_rate_limited", 429);
|
|
2742
|
+
if (this.active >= this.concurrentEvents) throw admissionError("global_concurrency_limited", 503);
|
|
2743
|
+
this.used += 1;
|
|
2744
|
+
this.active += 1;
|
|
2745
|
+
let result;
|
|
2746
|
+
try {
|
|
2747
|
+
result = operation();
|
|
2748
|
+
} catch (error) {
|
|
2749
|
+
this.active -= 1;
|
|
2750
|
+
throw error;
|
|
2751
|
+
}
|
|
2752
|
+
return Promise.resolve(result).finally(() => {
|
|
2753
|
+
this.active -= 1;
|
|
2754
|
+
});
|
|
2755
|
+
}
|
|
2756
|
+
};
|
|
2757
|
+
function admissionError(errorClass, statusCode) {
|
|
2758
|
+
const error = /* @__PURE__ */ new Error(errorClass === "global_rate_limited" ? "Relay global Event rate limit was reached" : "Relay global Event concurrency limit was reached");
|
|
2759
|
+
error.errorClass = errorClass;
|
|
2760
|
+
error.statusCode = statusCode;
|
|
2761
|
+
return error;
|
|
2762
|
+
}
|
|
1679
2763
|
function createExactEventRouter() {
|
|
1680
2764
|
return {
|
|
1681
2765
|
id: "relay.exact-event-type",
|
|
@@ -1693,8 +2777,15 @@ function createExactEventRouter() {
|
|
|
1693
2777
|
evidence: eventType ? [`No active wait expects ${eventType}.`] : ["Event has no type."],
|
|
1694
2778
|
summary: "No exact Relay wait matched the event."
|
|
1695
2779
|
};
|
|
1696
|
-
const
|
|
1697
|
-
|
|
2780
|
+
const matchedSessionIds = new Set(matches.map(({ session }) => session.session_id));
|
|
2781
|
+
if (matchedSessionIds.size > 1 && matches.some(({ wait }) => wait.exclusive)) return {
|
|
2782
|
+
disposition: "escalate",
|
|
2783
|
+
actionable: true,
|
|
2784
|
+
deliveries: [],
|
|
2785
|
+
evidence: [`Event type ${eventType} matches conflicting exclusive waits in ${matchedSessionIds.size} sessions.`],
|
|
2786
|
+
summary: `Cannot safely choose one owner for ${eventType}.`
|
|
2787
|
+
};
|
|
2788
|
+
const selected = matches;
|
|
1698
2789
|
const deliveries = /* @__PURE__ */ new Map();
|
|
1699
2790
|
for (const { session, wait } of selected) {
|
|
1700
2791
|
const delivery = deliveries.get(session.session_id) ?? {
|
|
@@ -1726,10 +2817,11 @@ function positiveInteger$1(value, fallback) {
|
|
|
1726
2817
|
}
|
|
1727
2818
|
//#endregion
|
|
1728
2819
|
//#region agent-bridge.js
|
|
1729
|
-
function installRelayAgentBridge(ctx, { sessionId, registerWaits, cancelWaits }) {
|
|
2820
|
+
function installRelayAgentBridge(ctx, { sessionId, registerWaits, cancelWaits, manageMonitor }) {
|
|
1730
2821
|
if (!sessionId) throw new Error("Relay bridge requires the current DSH session id");
|
|
1731
2822
|
if (typeof registerWaits !== "function") throw new Error("registerWaits callback is required");
|
|
1732
2823
|
if (typeof cancelWaits !== "function") throw new Error("cancelWaits callback is required");
|
|
2824
|
+
if (typeof manageMonitor !== "function") throw new Error("manageMonitor callback is required");
|
|
1733
2825
|
const unregisterWaits = ctx.tools.register(defineTool({
|
|
1734
2826
|
name: "relay_register_waits",
|
|
1735
2827
|
description: "Ask Relay to watch external conditions for this conversation, with optional bound Monitors.",
|
|
@@ -1778,6 +2870,43 @@ function installRelayAgentBridge(ctx, { sessionId, registerWaits, cancelWaits })
|
|
|
1778
2870
|
prior_exchange: {
|
|
1779
2871
|
type: "string",
|
|
1780
2872
|
required: true
|
|
2873
|
+
},
|
|
2874
|
+
continuation: {
|
|
2875
|
+
type: "object",
|
|
2876
|
+
additionalProperties: false,
|
|
2877
|
+
properties: {
|
|
2878
|
+
version: {
|
|
2879
|
+
type: "number",
|
|
2880
|
+
enum: [1]
|
|
2881
|
+
},
|
|
2882
|
+
next_action: { type: "string" },
|
|
2883
|
+
success_condition: { type: "string" },
|
|
2884
|
+
constraints: {
|
|
2885
|
+
type: "array",
|
|
2886
|
+
items: { type: "string" }
|
|
2887
|
+
},
|
|
2888
|
+
artifacts: {
|
|
2889
|
+
type: "array",
|
|
2890
|
+
items: {
|
|
2891
|
+
type: "object",
|
|
2892
|
+
additionalProperties: false,
|
|
2893
|
+
properties: {
|
|
2894
|
+
kind: {
|
|
2895
|
+
type: "string",
|
|
2896
|
+
required: true
|
|
2897
|
+
},
|
|
2898
|
+
id: {
|
|
2899
|
+
type: "string",
|
|
2900
|
+
required: true
|
|
2901
|
+
},
|
|
2902
|
+
label: { type: "string" },
|
|
2903
|
+
url: { type: "string" }
|
|
2904
|
+
}
|
|
2905
|
+
}
|
|
2906
|
+
},
|
|
2907
|
+
on_failure: { type: "string" },
|
|
2908
|
+
on_timeout: { type: "string" }
|
|
2909
|
+
}
|
|
1781
2910
|
}
|
|
1782
2911
|
}
|
|
1783
2912
|
}
|
|
@@ -1892,7 +3021,113 @@ function installRelayAgentBridge(ctx, { sessionId, registerWaits, cancelWaits })
|
|
|
1892
3021
|
};
|
|
1893
3022
|
}
|
|
1894
3023
|
}));
|
|
3024
|
+
const unregisterMonitor = ctx.tools.register(defineTool({
|
|
3025
|
+
name: "relay_manage_monitor",
|
|
3026
|
+
description: "Inspect or control one Relay Monitor owned by this conversation.",
|
|
3027
|
+
parameters: {
|
|
3028
|
+
monitor_id: {
|
|
3029
|
+
type: "string",
|
|
3030
|
+
required: true
|
|
3031
|
+
},
|
|
3032
|
+
action: {
|
|
3033
|
+
type: "string",
|
|
3034
|
+
required: true,
|
|
3035
|
+
enum: [
|
|
3036
|
+
"inspect",
|
|
3037
|
+
"pause",
|
|
3038
|
+
"resume",
|
|
3039
|
+
"run_now",
|
|
3040
|
+
"update_cadence",
|
|
3041
|
+
"update_target",
|
|
3042
|
+
"stop"
|
|
3043
|
+
]
|
|
3044
|
+
},
|
|
3045
|
+
expected_version: { type: "integer" },
|
|
3046
|
+
interval_seconds: { type: "integer" },
|
|
3047
|
+
reason_code: { type: "string" },
|
|
3048
|
+
detail: { type: "string" },
|
|
3049
|
+
observer: {
|
|
3050
|
+
type: "object",
|
|
3051
|
+
additionalProperties: false,
|
|
3052
|
+
properties: { provider: {
|
|
3053
|
+
type: "string",
|
|
3054
|
+
required: true
|
|
3055
|
+
} }
|
|
3056
|
+
},
|
|
3057
|
+
artifact: {
|
|
3058
|
+
type: "object",
|
|
3059
|
+
additionalProperties: true,
|
|
3060
|
+
properties: {}
|
|
3061
|
+
},
|
|
3062
|
+
detector: {
|
|
3063
|
+
type: "object",
|
|
3064
|
+
additionalProperties: true,
|
|
3065
|
+
properties: {}
|
|
3066
|
+
},
|
|
3067
|
+
capabilities: {
|
|
3068
|
+
type: "object",
|
|
3069
|
+
additionalProperties: true,
|
|
3070
|
+
properties: {}
|
|
3071
|
+
}
|
|
3072
|
+
},
|
|
3073
|
+
output: {
|
|
3074
|
+
schema: {
|
|
3075
|
+
type: "object",
|
|
3076
|
+
additionalProperties: false,
|
|
3077
|
+
properties: {
|
|
3078
|
+
applied: {
|
|
3079
|
+
type: "boolean",
|
|
3080
|
+
required: true
|
|
3081
|
+
},
|
|
3082
|
+
sessionId: {
|
|
3083
|
+
type: "string",
|
|
3084
|
+
required: true
|
|
3085
|
+
},
|
|
3086
|
+
monitorId: {
|
|
3087
|
+
type: "string",
|
|
3088
|
+
required: true
|
|
3089
|
+
},
|
|
3090
|
+
action: {
|
|
3091
|
+
type: "string",
|
|
3092
|
+
required: true
|
|
3093
|
+
},
|
|
3094
|
+
result: {
|
|
3095
|
+
type: "object",
|
|
3096
|
+
additionalProperties: true,
|
|
3097
|
+
properties: {},
|
|
3098
|
+
required: true
|
|
3099
|
+
}
|
|
3100
|
+
}
|
|
3101
|
+
},
|
|
3102
|
+
render: (_args, value) => [{
|
|
3103
|
+
type: "text",
|
|
3104
|
+
text: JSON.stringify(value)
|
|
3105
|
+
}]
|
|
3106
|
+
},
|
|
3107
|
+
async execute(args) {
|
|
3108
|
+
const result = await manageMonitor(sessionId, {
|
|
3109
|
+
monitorId: args.monitor_id,
|
|
3110
|
+
action: args.action,
|
|
3111
|
+
...args.expected_version === void 0 ? {} : { expectedVersion: args.expected_version },
|
|
3112
|
+
...args.interval_seconds === void 0 ? {} : { intervalSeconds: args.interval_seconds },
|
|
3113
|
+
...args.reason_code === void 0 ? {} : { reasonCode: args.reason_code },
|
|
3114
|
+
...args.detail === void 0 ? {} : { detail: args.detail },
|
|
3115
|
+
...args.observer === void 0 ? {} : { observer: args.observer },
|
|
3116
|
+
...args.artifact === void 0 ? {} : { artifact: args.artifact },
|
|
3117
|
+
...args.detector === void 0 ? {} : { detector: args.detector },
|
|
3118
|
+
...args.capabilities === void 0 ? {} : { capabilities: args.capabilities }
|
|
3119
|
+
});
|
|
3120
|
+
return {
|
|
3121
|
+
applied: true,
|
|
3122
|
+
sessionId,
|
|
3123
|
+
monitorId: args.monitor_id,
|
|
3124
|
+
action: args.action,
|
|
3125
|
+
result
|
|
3126
|
+
};
|
|
3127
|
+
}
|
|
3128
|
+
}));
|
|
1895
3129
|
return () => {
|
|
3130
|
+
unregisterMonitor?.();
|
|
1896
3131
|
unregisterCancel?.();
|
|
1897
3132
|
unregisterWaits?.();
|
|
1898
3133
|
};
|
|
@@ -1948,6 +3183,10 @@ function createRelayEventHandler({ relayRuntime, token, maxBodyBytes = 1048576 }
|
|
|
1948
3183
|
writeJson(response, 403, { error: "forbidden" });
|
|
1949
3184
|
return;
|
|
1950
3185
|
}
|
|
3186
|
+
if (String(request.headers?.["content-encoding"] ?? "identity").trim().toLowerCase() !== "identity") {
|
|
3187
|
+
writeJson(response, 415, { error: "unsupported_content_encoding" });
|
|
3188
|
+
return;
|
|
3189
|
+
}
|
|
1951
3190
|
try {
|
|
1952
3191
|
const event = normalizeEvent(await readJson(request, maxBodyBytes));
|
|
1953
3192
|
const result = await relayRuntime.handleEvent(event);
|
|
@@ -1964,14 +3203,21 @@ function createRelayEventHandler({ relayRuntime, token, maxBodyBytes = 1048576 }
|
|
|
1964
3203
|
}))
|
|
1965
3204
|
});
|
|
1966
3205
|
} catch (error) {
|
|
1967
|
-
const status = error instanceof EventIngressError ? error.statusCode : 500;
|
|
3206
|
+
const status = error instanceof EventIngressError ? error.statusCode : Number.isInteger(error?.statusCode) ? error.statusCode : 500;
|
|
3207
|
+
const publicCode = status === 413 ? "payload_too_large" : status === 429 ? "rate_limited" : status === 503 && error?.errorClass === "global_concurrency_limited" ? "temporarily_overloaded" : status < 500 ? "invalid_event" : "event_delivery_failed";
|
|
1968
3208
|
writeJson(response, status, {
|
|
1969
|
-
error:
|
|
1970
|
-
message:
|
|
3209
|
+
error: publicCode,
|
|
3210
|
+
message: publicErrorMessage(publicCode, error)
|
|
1971
3211
|
});
|
|
1972
3212
|
}
|
|
1973
3213
|
};
|
|
1974
3214
|
}
|
|
3215
|
+
function publicErrorMessage(code, error) {
|
|
3216
|
+
if (code === "rate_limited") return "Relay event admission rate limit exceeded";
|
|
3217
|
+
if (code === "temporarily_overloaded") return "Relay event admission is temporarily overloaded";
|
|
3218
|
+
if (code === "event_delivery_failed") return "Relay could not accept the event";
|
|
3219
|
+
return error?.message ?? String(error);
|
|
3220
|
+
}
|
|
1975
3221
|
function normalizeEvent(body) {
|
|
1976
3222
|
if (!body || typeof body !== "object" || Array.isArray(body)) throw new EventIngressError(400, "event body must be a JSON object");
|
|
1977
3223
|
const type = requiredString(body.type, "type");
|
|
@@ -1997,11 +3243,27 @@ async function readJson(request, maxBodyBytes) {
|
|
|
1997
3243
|
chunks.push(buffer);
|
|
1998
3244
|
}
|
|
1999
3245
|
if (size === 0) throw new EventIngressError(400, "event body is empty");
|
|
3246
|
+
let value;
|
|
2000
3247
|
try {
|
|
2001
|
-
|
|
3248
|
+
value = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
2002
3249
|
} catch {
|
|
2003
3250
|
throw new EventIngressError(400, "event body is not valid JSON");
|
|
2004
3251
|
}
|
|
3252
|
+
validateComplexity(value);
|
|
3253
|
+
return value;
|
|
3254
|
+
}
|
|
3255
|
+
function validateComplexity(value) {
|
|
3256
|
+
let keys = 0;
|
|
3257
|
+
const visit = (node, depth) => {
|
|
3258
|
+
if (depth > 32) throw new EventIngressError(413, "event body nesting is too deep");
|
|
3259
|
+
if (!node || typeof node !== "object") return;
|
|
3260
|
+
for (const child of Object.values(node)) {
|
|
3261
|
+
keys += 1;
|
|
3262
|
+
if (keys > 1e4) throw new EventIngressError(413, "event body has too many fields");
|
|
3263
|
+
visit(child, depth + 1);
|
|
3264
|
+
}
|
|
3265
|
+
};
|
|
3266
|
+
visit(value, 0);
|
|
2005
3267
|
}
|
|
2006
3268
|
function authorized(request, token) {
|
|
2007
3269
|
if (isLoopback(request.socket?.remoteAddress)) return true;
|
|
@@ -2038,6 +3300,16 @@ var EventIngressError = class extends Error {
|
|
|
2038
3300
|
}
|
|
2039
3301
|
};
|
|
2040
3302
|
//#endregion
|
|
3303
|
+
//#region dsh-compat.mjs
|
|
3304
|
+
function sessionEvents(session, from = 0) {
|
|
3305
|
+
if (!Number.isSafeInteger(from) || from < 0) throw new RangeError("session event offset must be a non-negative integer");
|
|
3306
|
+
const snapshot = Reflect.get(session, "snapshotEvents");
|
|
3307
|
+
if (typeof snapshot === "function") return from === 0 ? Reflect.apply(snapshot, session, []) : Reflect.apply(snapshot, session, [from]);
|
|
3308
|
+
const events = Reflect.get(session, "events");
|
|
3309
|
+
if (!Array.isArray(events)) throw new TypeError("DSH Session exposes neither snapshotEvents() nor events");
|
|
3310
|
+
return from === 0 ? events : events.slice(from);
|
|
3311
|
+
}
|
|
3312
|
+
//#endregion
|
|
2041
3313
|
//#region inbox-adapter.js
|
|
2042
3314
|
var DshInboxAdapter = class {
|
|
2043
3315
|
constructor({ resolveAgent, awaitDurable, maxInputChars = 1e5, debug = false }) {
|
|
@@ -2057,7 +3329,7 @@ var DshInboxAdapter = class {
|
|
|
2057
3329
|
deliveries
|
|
2058
3330
|
});
|
|
2059
3331
|
this.log(`enqueue ${activationId} into ${sessionId}`);
|
|
2060
|
-
if (!(resolved.agent.session
|
|
3332
|
+
if (!sessionEvents(resolved.agent.session).some((event) => {
|
|
2061
3333
|
return (event.type === "agent/inbox/spliced" ? event.data.inserted ?? [] : event.type === "user/message" ? [event.data] : []).some((message) => message.id === activationId && message.source?.kind === "plugin" && message.source.plugin === "relay");
|
|
2062
3334
|
})) resolved.agent.followup({
|
|
2063
3335
|
...createUserMessage({
|
|
@@ -2091,7 +3363,9 @@ var DshInboxAdapter = class {
|
|
|
2091
3363
|
delivery_id: delivery.delivery_id,
|
|
2092
3364
|
event_id: delivery.event_id,
|
|
2093
3365
|
wait_ids: delivery.wait_ids,
|
|
3366
|
+
matched_waits: delivery.matched_waits ?? [],
|
|
2094
3367
|
relation: delivery.relation,
|
|
3368
|
+
routing_evidence: delivery.routing_evidence ?? [],
|
|
2095
3369
|
event: delivery.event
|
|
2096
3370
|
}))
|
|
2097
3371
|
};
|
|
@@ -2116,8 +3390,8 @@ var RelayManagementGateway = class extends TypertRemoteService {
|
|
|
2116
3390
|
super(ctx, "relayManagement");
|
|
2117
3391
|
this.relayEvents = relayEvents;
|
|
2118
3392
|
}
|
|
2119
|
-
list() {
|
|
2120
|
-
return
|
|
3393
|
+
list(options = {}) {
|
|
3394
|
+
return this.relayEvents.managementSnapshot(options);
|
|
2121
3395
|
}
|
|
2122
3396
|
cancel(sessionId) {
|
|
2123
3397
|
return { registration: this.relayEvents.cancelWaits(sessionId) };
|
|
@@ -2128,6 +3402,35 @@ var RelayManagementGateway = class extends TypertRemoteService {
|
|
|
2128
3402
|
registrations: this.relayEvents.listWaits()
|
|
2129
3403
|
};
|
|
2130
3404
|
}
|
|
3405
|
+
inspectMonitor(monitorId) {
|
|
3406
|
+
return { monitor: this.relayEvents.inspectMonitor(monitorId) };
|
|
3407
|
+
}
|
|
3408
|
+
pauseMonitor(monitorId, expectedVersion) {
|
|
3409
|
+
return { monitor: this.relayEvents.pauseMonitor(monitorId, { expectedVersion }) };
|
|
3410
|
+
}
|
|
3411
|
+
resumeMonitor(monitorId, expectedVersion) {
|
|
3412
|
+
return { monitor: this.relayEvents.resumeMonitor(monitorId, { expectedVersion }) };
|
|
3413
|
+
}
|
|
3414
|
+
updateMonitorCadence(monitorId, intervalSeconds, expectedVersion) {
|
|
3415
|
+
return { monitor: this.relayEvents.updateMonitorCadence(monitorId, intervalSeconds, { expectedVersion }) };
|
|
3416
|
+
}
|
|
3417
|
+
stopMonitor(monitorId, expectedVersion, reasonCode, detail) {
|
|
3418
|
+
return { monitor: this.relayEvents.stopMonitor(monitorId, {
|
|
3419
|
+
expectedVersion,
|
|
3420
|
+
actor: "management-ui",
|
|
3421
|
+
reasonCode,
|
|
3422
|
+
detail
|
|
3423
|
+
}) };
|
|
3424
|
+
}
|
|
3425
|
+
retryActivation(activationId) {
|
|
3426
|
+
return this.relayEvents.retryActivation(activationId);
|
|
3427
|
+
}
|
|
3428
|
+
retryNotification(eventId) {
|
|
3429
|
+
return { notification: this.relayEvents.retryNotification(eventId) };
|
|
3430
|
+
}
|
|
3431
|
+
connectorAction(connectorId, action, input) {
|
|
3432
|
+
return this.relayEvents.executeConnectorAction(connectorId, action, input);
|
|
3433
|
+
}
|
|
2131
3434
|
};
|
|
2132
3435
|
//#endregion
|
|
2133
3436
|
//#region host-plugin.js
|
|
@@ -2151,6 +3454,11 @@ async function apply(ctx, config = {}) {
|
|
|
2151
3454
|
const events = new RelayEventsService(ctx, {
|
|
2152
3455
|
databasePath: config.databasePath,
|
|
2153
3456
|
dispatchPollIntervalMs: config.dispatchPollIntervalMs,
|
|
3457
|
+
routingFailureLimit: config.routingFailureLimit,
|
|
3458
|
+
deliveryFailureLimit: config.deliveryFailureLimit,
|
|
3459
|
+
deliveryRetryBaseMs: config.deliveryRetryBaseMs,
|
|
3460
|
+
globalEventsPerMinute: config.globalEventsPerMinute,
|
|
3461
|
+
globalConcurrentEvents: config.globalConcurrentEvents,
|
|
2154
3462
|
inbox
|
|
2155
3463
|
});
|
|
2156
3464
|
ctx.effect(() => () => events.stop(), "relayEvents.stop()");
|
|
@@ -2172,12 +3480,35 @@ async function apply(ctx, config = {}) {
|
|
|
2172
3480
|
ctx.effect(() => installRelayAgentBridge(agent.ctx, {
|
|
2173
3481
|
sessionId: agent.id,
|
|
2174
3482
|
registerWaits: (input) => events.registerWaits(input),
|
|
2175
|
-
cancelWaits: (sessionId) => events.cancelWaits(sessionId)
|
|
3483
|
+
cancelWaits: (sessionId) => events.cancelWaits(sessionId),
|
|
3484
|
+
manageMonitor: (sessionId, input) => manageOwnedMonitor(events, sessionId, input)
|
|
2176
3485
|
}), "relay events tools");
|
|
2177
3486
|
};
|
|
2178
3487
|
ctx.effect(() => ctx.on("agent/created", ({ agent }) => attach(agent)), "relay events agent bridge");
|
|
2179
3488
|
for (const agent of ctx.agents.roots()) attach(agent);
|
|
2180
3489
|
}
|
|
3490
|
+
async function manageOwnedMonitor(events, sessionId, input) {
|
|
3491
|
+
const monitor = events.inspectMonitor(input.monitorId);
|
|
3492
|
+
if (!monitor || monitor.session_id !== sessionId) throw new Error(`monitor ${input.monitorId} does not belong to this Session`);
|
|
3493
|
+
if (input.action === "inspect") return monitor;
|
|
3494
|
+
if (input.action === "run_now") return events.checkMonitor(input.monitorId, { force: true });
|
|
3495
|
+
if (input.action === "pause") return events.pauseMonitor(input.monitorId, { expectedVersion: input.expectedVersion });
|
|
3496
|
+
if (input.action === "resume") return events.resumeMonitor(input.monitorId, { expectedVersion: input.expectedVersion });
|
|
3497
|
+
if (input.action === "update_cadence") return events.updateMonitorCadence(input.monitorId, input.intervalSeconds, { expectedVersion: input.expectedVersion });
|
|
3498
|
+
if (input.action === "update_target") return events.rebaselineMonitor(input.monitorId, {
|
|
3499
|
+
observer: input.observer,
|
|
3500
|
+
artifact: input.artifact,
|
|
3501
|
+
detector: input.detector,
|
|
3502
|
+
capabilities: input.capabilities
|
|
3503
|
+
}, { expectedVersion: input.expectedVersion });
|
|
3504
|
+
if (input.action === "stop") return events.stopMonitor(input.monitorId, {
|
|
3505
|
+
expectedVersion: input.expectedVersion,
|
|
3506
|
+
actor: `session:${sessionId}`,
|
|
3507
|
+
reasonCode: input.reasonCode ?? "stopped_by_agent",
|
|
3508
|
+
detail: input.detail ?? ""
|
|
3509
|
+
});
|
|
3510
|
+
throw new Error(`unsupported monitor action ${input.action}`);
|
|
3511
|
+
}
|
|
2181
3512
|
function createSharedAgentLookup(ctx) {
|
|
2182
3513
|
const lookup = ctx.typert.lookups.get("agent");
|
|
2183
3514
|
if (!lookup) throw new Error("Relay requires DSH's configured shared Agent lookup");
|