solid-objects 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +110 -0
- package/MIT-LICENSE +19 -0
- package/README.md +958 -0
- package/dist/actor.d.ts +81 -0
- package/dist/actor.d.ts.map +1 -0
- package/dist/actor.js +141 -0
- package/dist/actor.js.map +1 -0
- package/dist/application-database.d.ts +3 -0
- package/dist/application-database.d.ts.map +1 -0
- package/dist/application-database.js +59 -0
- package/dist/application-database.js.map +1 -0
- package/dist/broadcast-worker.d.ts +18 -0
- package/dist/broadcast-worker.d.ts.map +1 -0
- package/dist/broadcast-worker.js +74 -0
- package/dist/broadcast-worker.js.map +1 -0
- package/dist/browser/components.d.ts +64 -0
- package/dist/browser/components.d.ts.map +1 -0
- package/dist/browser/components.js +294 -0
- package/dist/browser/components.js.map +1 -0
- package/dist/browser/index.d.ts +46 -0
- package/dist/browser/index.d.ts.map +1 -0
- package/dist/browser/index.js +207 -0
- package/dist/browser/index.js.map +1 -0
- package/dist/cli.d.ts +9 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +216 -0
- package/dist/cli.js.map +1 -0
- package/dist/configuration.d.ts +96 -0
- package/dist/configuration.d.ts.map +1 -0
- package/dist/configuration.js +169 -0
- package/dist/configuration.js.map +1 -0
- package/dist/context.d.ts +20 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +24 -0
- package/dist/context.js.map +1 -0
- package/dist/database/deadline.d.ts +9 -0
- package/dist/database/deadline.d.ts.map +1 -0
- package/dist/database/deadline.js +49 -0
- package/dist/database/deadline.js.map +1 -0
- package/dist/database/mysql.d.ts +21 -0
- package/dist/database/mysql.d.ts.map +1 -0
- package/dist/database/mysql.js +192 -0
- package/dist/database/mysql.js.map +1 -0
- package/dist/database/postgresql-sql.d.ts +2 -0
- package/dist/database/postgresql-sql.d.ts.map +1 -0
- package/dist/database/postgresql-sql.js +102 -0
- package/dist/database/postgresql-sql.js.map +1 -0
- package/dist/database/postgresql.d.ts +31 -0
- package/dist/database/postgresql.d.ts.map +1 -0
- package/dist/database/postgresql.js +172 -0
- package/dist/database/postgresql.js.map +1 -0
- package/dist/database/sqlite.d.ts +27 -0
- package/dist/database/sqlite.d.ts.map +1 -0
- package/dist/database/sqlite.js +190 -0
- package/dist/database/sqlite.js.map +1 -0
- package/dist/database/transaction-context.d.ts +3 -0
- package/dist/database/transaction-context.d.ts.map +1 -0
- package/dist/database/transaction-context.js +19 -0
- package/dist/database/transaction-context.js.map +1 -0
- package/dist/database/types.d.ts +20 -0
- package/dist/database/types.d.ts.map +1 -0
- package/dist/database/types.js +2 -0
- package/dist/database/types.js.map +1 -0
- package/dist/dead-letters.d.ts +23 -0
- package/dist/dead-letters.d.ts.map +1 -0
- package/dist/dead-letters.js +13 -0
- package/dist/dead-letters.js.map +1 -0
- package/dist/default-runtime.d.ts +5 -0
- package/dist/default-runtime.d.ts.map +1 -0
- package/dist/default-runtime.js +14 -0
- package/dist/default-runtime.js.map +1 -0
- package/dist/definition.d.ts +32 -0
- package/dist/definition.d.ts.map +1 -0
- package/dist/definition.js +173 -0
- package/dist/definition.js.map +1 -0
- package/dist/doctor.d.ts +32 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +500 -0
- package/dist/doctor.js.map +1 -0
- package/dist/effect-worker.d.ts +18 -0
- package/dist/effect-worker.d.ts.map +1 -0
- package/dist/effect-worker.js +74 -0
- package/dist/effect-worker.js.map +1 -0
- package/dist/errors.d.ts +145 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +120 -0
- package/dist/errors.js.map +1 -0
- package/dist/executable.d.ts +3 -0
- package/dist/executable.d.ts.map +1 -0
- package/dist/executable.js +12 -0
- package/dist/executable.js.map +1 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +1 -0
- package/dist/maintenance-scheduler.d.ts +17 -0
- package/dist/maintenance-scheduler.d.ts.map +1 -0
- package/dist/maintenance-scheduler.js +41 -0
- package/dist/maintenance-scheduler.js.map +1 -0
- package/dist/process-administration.d.ts +30 -0
- package/dist/process-administration.d.ts.map +1 -0
- package/dist/process-administration.js +13 -0
- package/dist/process-administration.js.map +1 -0
- package/dist/realtime.d.ts +33 -0
- package/dist/realtime.d.ts.map +1 -0
- package/dist/realtime.js +238 -0
- package/dist/realtime.js.map +1 -0
- package/dist/reconciliation.d.ts +41 -0
- package/dist/reconciliation.d.ts.map +1 -0
- package/dist/reconciliation.js +19 -0
- package/dist/reconciliation.js.map +1 -0
- package/dist/records.d.ts +137 -0
- package/dist/records.d.ts.map +1 -0
- package/dist/records.js +2 -0
- package/dist/records.js.map +1 -0
- package/dist/reference.d.ts +97 -0
- package/dist/reference.d.ts.map +1 -0
- package/dist/reference.js +157 -0
- package/dist/reference.js.map +1 -0
- package/dist/reminder-administration.d.ts +35 -0
- package/dist/reminder-administration.d.ts.map +1 -0
- package/dist/reminder-administration.js +13 -0
- package/dist/reminder-administration.js.map +1 -0
- package/dist/reminder-scheduler.d.ts +18 -0
- package/dist/reminder-scheduler.d.ts.map +1 -0
- package/dist/reminder-scheduler.js +88 -0
- package/dist/reminder-scheduler.js.map +1 -0
- package/dist/repository.d.ts +142 -0
- package/dist/repository.d.ts.map +1 -0
- package/dist/repository.js +1237 -0
- package/dist/repository.js.map +1 -0
- package/dist/retention.d.ts +17 -0
- package/dist/retention.d.ts.map +1 -0
- package/dist/retention.js +13 -0
- package/dist/retention.js.map +1 -0
- package/dist/runtime.d.ts +140 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +1737 -0
- package/dist/runtime.js.map +1 -0
- package/dist/schema.d.ts +8 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +260 -0
- package/dist/schema.js.map +1 -0
- package/dist/serialization.d.ts +11 -0
- package/dist/serialization.d.ts.map +1 -0
- package/dist/serialization.js +82 -0
- package/dist/serialization.js.map +1 -0
- package/dist/test-helper.d.ts +15 -0
- package/dist/test-helper.d.ts.map +1 -0
- package/dist/test-helper.js +69 -0
- package/dist/test-helper.js.map +1 -0
- package/dist/types.d.ts +71 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +2 -0
- package/dist/version.js.map +1 -0
- package/dist/wake-up/postgresql.d.ts +42 -0
- package/dist/wake-up/postgresql.d.ts.map +1 -0
- package/dist/wake-up/postgresql.js +234 -0
- package/dist/wake-up/postgresql.js.map +1 -0
- package/dist/wake-up/redis.d.ts +35 -0
- package/dist/wake-up/redis.d.ts.map +1 -0
- package/dist/wake-up/redis.js +151 -0
- package/dist/wake-up/redis.js.map +1 -0
- package/dist/wake-up.d.ts +24 -0
- package/dist/wake-up.d.ts.map +1 -0
- package/dist/wake-up.js +66 -0
- package/dist/wake-up.js.map +1 -0
- package/dist/worker.d.ts +36 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +281 -0
- package/dist/worker.js.map +1 -0
- package/docs/api.md +197 -0
- package/docs/architecture.md +113 -0
- package/docs/authorization.md +31 -0
- package/docs/browser-protocol.md +93 -0
- package/docs/configuration.md +168 -0
- package/docs/correctness.md +53 -0
- package/docs/errors-and-recovery.md +89 -0
- package/docs/operations.md +170 -0
- package/docs/parity.md +94 -0
- package/docs/releasing.md +45 -0
- package/docs/state-and-lifecycle.md +162 -0
- package/package.json +100 -0
|
@@ -0,0 +1,1237 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { hostname } from "node:os";
|
|
3
|
+
import { ActorDestroyed, IdempotencyConflict, LostActivation, MailboxFull, ReminderNotPaused, UnknownDeadLetter, UnknownReminder, } from "./errors.js";
|
|
4
|
+
import { jsonObject, normalizeJson } from "./serialization.js";
|
|
5
|
+
import { VERSION } from "./version.js";
|
|
6
|
+
export class Repository {
|
|
7
|
+
settings;
|
|
8
|
+
constructor(settings) {
|
|
9
|
+
this.settings = settings;
|
|
10
|
+
}
|
|
11
|
+
table(name) {
|
|
12
|
+
return `${this.settings.tableNamePrefix}${name}`;
|
|
13
|
+
}
|
|
14
|
+
async registerProcess(processId, kind) {
|
|
15
|
+
await this.settings.database.transaction(async (connection) => {
|
|
16
|
+
const now = await connection.nowMilliseconds();
|
|
17
|
+
await connection.run(`INSERT INTO ${this.table("processes")}
|
|
18
|
+
(id, kind, hostname, host_process_id, metadata, started_at_ms, heartbeat_at_ms,
|
|
19
|
+
shutdown_state)
|
|
20
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 'running')
|
|
21
|
+
ON CONFLICT(id) DO UPDATE SET heartbeat_at_ms = excluded.heartbeat_at_ms,
|
|
22
|
+
hostname = excluded.hostname, host_process_id = excluded.host_process_id,
|
|
23
|
+
metadata = excluded.metadata, shutdown_requested_at_ms = NULL,
|
|
24
|
+
stopped_at_ms = NULL, shutdown_state = 'running'`, [
|
|
25
|
+
processId,
|
|
26
|
+
kind,
|
|
27
|
+
hostname(),
|
|
28
|
+
process.pid,
|
|
29
|
+
JSON.stringify({ solidObjectsVersion: VERSION, nodeVersion: process.version }),
|
|
30
|
+
now,
|
|
31
|
+
now,
|
|
32
|
+
]);
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
async startDrainingProcess(processId) {
|
|
36
|
+
await this.settings.database.transaction(async (connection) => {
|
|
37
|
+
const now = await connection.nowMilliseconds();
|
|
38
|
+
await connection.run(`UPDATE ${this.table("processes")}
|
|
39
|
+
SET shutdown_state = 'draining', shutdown_requested_at_ms = ?
|
|
40
|
+
WHERE id = ? AND shutdown_state = 'running'`, [now, processId]);
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
async stopProcess(processId) {
|
|
44
|
+
await this.settings.database.transaction(async (connection) => {
|
|
45
|
+
const now = await connection.nowMilliseconds();
|
|
46
|
+
await this.releaseProcessOwnership({ connection, processId, now });
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
async heartbeatProcess(processId) {
|
|
50
|
+
await this.settings.database.transaction(async (connection) => {
|
|
51
|
+
const now = await connection.nowMilliseconds();
|
|
52
|
+
await connection.run(`UPDATE ${this.table("processes")} SET heartbeat_at_ms = ?
|
|
53
|
+
WHERE id = ? AND shutdown_state = 'running'`, [now, processId]);
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
async listProcesses() {
|
|
57
|
+
return this.settings.database.connection(async (connection) => {
|
|
58
|
+
const now = await connection.nowMilliseconds();
|
|
59
|
+
const rows = await connection.all(`SELECT * FROM ${this.table("processes")} ORDER BY kind, started_at_ms, id`);
|
|
60
|
+
return {
|
|
61
|
+
rows,
|
|
62
|
+
staleAtMilliseconds: now - this.settings.processAliveThresholdMilliseconds,
|
|
63
|
+
};
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
async cleanupStaleProcesses() {
|
|
67
|
+
return this.settings.database.transaction(async (connection) => {
|
|
68
|
+
const now = await connection.nowMilliseconds();
|
|
69
|
+
const staleAt = now - this.settings.processAliveThresholdMilliseconds;
|
|
70
|
+
const processes = await connection.all(`SELECT id FROM ${this.table("processes")}
|
|
71
|
+
WHERE shutdown_state <> 'stopped' AND heartbeat_at_ms <= ?
|
|
72
|
+
ORDER BY id`, [staleAt]);
|
|
73
|
+
for (const process of processes) {
|
|
74
|
+
await this.releaseProcessOwnership({ connection, processId: process.id, now });
|
|
75
|
+
}
|
|
76
|
+
return processes.length;
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
async releaseProcessOwnership(options) {
|
|
80
|
+
const { connection, processId, now } = options;
|
|
81
|
+
const claims = await connection.all(`SELECT message_id, instance_id, sequence
|
|
82
|
+
FROM ${this.table("claimed_messages")} WHERE process_id = ?`, [processId]);
|
|
83
|
+
for (const claim of claims) {
|
|
84
|
+
await connection.run(`DELETE FROM ${this.table("claimed_messages")} WHERE message_id = ?`, [
|
|
85
|
+
claim.message_id,
|
|
86
|
+
]);
|
|
87
|
+
await connection.run(`INSERT INTO ${this.table("ready_messages")}
|
|
88
|
+
(message_id, instance_id, sequence, available_at_ms)
|
|
89
|
+
VALUES (?, ?, ?, ?) ON CONFLICT(message_id) DO NOTHING`, [claim.message_id, claim.instance_id, claim.sequence, now]);
|
|
90
|
+
}
|
|
91
|
+
await connection.run(`UPDATE ${this.table("instances")}
|
|
92
|
+
SET activation_owner_id = NULL, activation_token = NULL,
|
|
93
|
+
activation_expires_at_ms = NULL, updated_at_ms = ?
|
|
94
|
+
WHERE activation_owner_id = ?`, [now, processId]);
|
|
95
|
+
await connection.run(`UPDATE ${this.table("effects")} SET status = 'pending', claimed_by = NULL
|
|
96
|
+
WHERE status = 'processing' AND claimed_by = ?`, [processId]);
|
|
97
|
+
await connection.run(`UPDATE ${this.table("reminders")} SET claimed_by = NULL, claimed_at_ms = NULL
|
|
98
|
+
WHERE claimed_by = ?`, [processId]);
|
|
99
|
+
await connection.run(`UPDATE ${this.table("broadcasts")} SET status = 'pending', claimed_by = NULL
|
|
100
|
+
WHERE status = 'processing' AND claimed_by = ?`, [processId]);
|
|
101
|
+
await connection.run(`UPDATE ${this.table("processes")}
|
|
102
|
+
SET shutdown_state = 'stopped', stopped_at_ms = ?, heartbeat_at_ms = ?
|
|
103
|
+
WHERE id = ? AND shutdown_state <> 'stopped'`, [now, now, processId]);
|
|
104
|
+
}
|
|
105
|
+
async enqueue(input) {
|
|
106
|
+
for (let attempt = 1; attempt <= 8; attempt += 1) {
|
|
107
|
+
try {
|
|
108
|
+
return await this.settings.database.transaction(async (connection) => this.enqueueInTransaction(connection, input));
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
if (!retryableMySQLDeadlock({ database: this.settings.database, error, attempt })) {
|
|
112
|
+
throw error;
|
|
113
|
+
}
|
|
114
|
+
await transactionRetryBackoff(attempt);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
throw new Error("enqueue retries exhausted");
|
|
118
|
+
}
|
|
119
|
+
async enqueueInTransaction(connection, input) {
|
|
120
|
+
const now = await connection.nowMilliseconds();
|
|
121
|
+
let instance = await this.findInstance({
|
|
122
|
+
connection,
|
|
123
|
+
actorType: input.actorType,
|
|
124
|
+
actorId: input.actorId,
|
|
125
|
+
...(this.settings.database.family === "postgresql" ? { lock: "update" } : {}),
|
|
126
|
+
});
|
|
127
|
+
if (!instance) {
|
|
128
|
+
const instanceId = randomUUID();
|
|
129
|
+
await connection.run(`INSERT INTO ${this.table("instances")}
|
|
130
|
+
(id, actor_type, actor_id, state, state_version, created_at_ms, updated_at_ms)
|
|
131
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
132
|
+
ON CONFLICT(actor_type, actor_id) DO NOTHING`, [
|
|
133
|
+
instanceId,
|
|
134
|
+
input.actorType,
|
|
135
|
+
input.actorId,
|
|
136
|
+
JSON.stringify(input.initialState ?? {}),
|
|
137
|
+
input.stateVersion ?? 1,
|
|
138
|
+
now,
|
|
139
|
+
now,
|
|
140
|
+
]);
|
|
141
|
+
}
|
|
142
|
+
if (this.settings.database.family === "mysql" || !instance) {
|
|
143
|
+
instance = await this.findInstance({
|
|
144
|
+
connection,
|
|
145
|
+
actorType: input.actorType,
|
|
146
|
+
actorId: input.actorId,
|
|
147
|
+
...(this.settings.database.family === "sqlite" ? {} : { lock: "update" }),
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
if (!instance)
|
|
151
|
+
throw new ActorDestroyed("actor disappeared during enqueue");
|
|
152
|
+
if (input.idempotencyKey !== undefined) {
|
|
153
|
+
const existing = await connection.get(`SELECT * FROM ${this.table("messages")} WHERE actor_type = ? AND actor_id = ? AND idempotency_key = ?`, [input.actorType, input.actorId, input.idempotencyKey]);
|
|
154
|
+
if (existing) {
|
|
155
|
+
if (existing.operation !== input.operation ||
|
|
156
|
+
existing.delivery_mode !== input.deliveryMode ||
|
|
157
|
+
existing.arguments !== JSON.stringify(input.arguments)) {
|
|
158
|
+
throw new IdempotencyConflict("idempotency key identifies a different invocation");
|
|
159
|
+
}
|
|
160
|
+
return existing;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
const live = await connection.get(`SELECT COUNT(*) AS count FROM (
|
|
164
|
+
SELECT message_id FROM ${this.table("ready_messages")} WHERE instance_id = ?
|
|
165
|
+
UNION ALL
|
|
166
|
+
SELECT message_id FROM ${this.table("claimed_messages")} WHERE instance_id = ?
|
|
167
|
+
) AS live_messages`, [instance.id, instance.id]);
|
|
168
|
+
if (Number(live?.count ?? 0) >= this.settings.maxMailboxLength) {
|
|
169
|
+
throw new MailboxFull(`mailbox is full for ${input.actorType}(${JSON.stringify(input.actorId)})`);
|
|
170
|
+
}
|
|
171
|
+
const sequence = BigInt(instance.next_message_sequence);
|
|
172
|
+
const messageId = randomUUID();
|
|
173
|
+
const requestId = randomUUID();
|
|
174
|
+
await connection.run(`UPDATE ${this.table("instances")} SET next_message_sequence = next_message_sequence + 1,
|
|
175
|
+
updated_at_ms = ? WHERE id = ?`, [now, instance.id]);
|
|
176
|
+
await connection.run(`INSERT INTO ${this.table("messages")}
|
|
177
|
+
(id, request_id, idempotency_key, instance_id, actor_type, actor_id, sequence, operation,
|
|
178
|
+
delivery_mode, arguments, max_attempts, created_at_ms, updated_at_ms)
|
|
179
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
180
|
+
messageId,
|
|
181
|
+
requestId,
|
|
182
|
+
input.idempotencyKey ?? null,
|
|
183
|
+
instance.id,
|
|
184
|
+
input.actorType,
|
|
185
|
+
input.actorId,
|
|
186
|
+
sequence,
|
|
187
|
+
input.operation,
|
|
188
|
+
input.deliveryMode,
|
|
189
|
+
JSON.stringify(input.arguments),
|
|
190
|
+
this.settings.maxAttempts,
|
|
191
|
+
now,
|
|
192
|
+
now,
|
|
193
|
+
]);
|
|
194
|
+
await connection.run(`INSERT INTO ${this.table("ready_messages")}(message_id, instance_id, sequence, available_at_ms)
|
|
195
|
+
VALUES (?, ?, ?, ?)`, [messageId, instance.id, sequence, input.availableAtMilliseconds ?? now]);
|
|
196
|
+
const message = await connection.get(`SELECT * FROM ${this.table("messages")} WHERE id = ?`, [messageId]);
|
|
197
|
+
if (!message)
|
|
198
|
+
throw new Error("enqueued message could not be read");
|
|
199
|
+
return message;
|
|
200
|
+
}
|
|
201
|
+
async claim(processId, options = {}) {
|
|
202
|
+
return this.settings.database.transaction(async (connection) => {
|
|
203
|
+
const now = await connection.nowMilliseconds();
|
|
204
|
+
await this.recoverExpiredClaims(connection, now);
|
|
205
|
+
const instanceCondition = options.activation !== undefined || options.instanceId !== undefined
|
|
206
|
+
? "AND ready.instance_id = ?"
|
|
207
|
+
: "";
|
|
208
|
+
const leaseCondition = options.activation
|
|
209
|
+
? `AND instances.activation_owner_id = ? AND instances.activation_token = ?
|
|
210
|
+
AND instances.activation_generation = ? AND instances.activation_expires_at_ms > ?`
|
|
211
|
+
: "AND (instances.activation_owner_id IS NULL OR instances.activation_expires_at_ms <= ?)";
|
|
212
|
+
const parameters = [now];
|
|
213
|
+
if (options.activation) {
|
|
214
|
+
parameters.push(options.activation.instanceId, options.activation.processId, options.activation.activationToken, options.activation.activationGeneration, now);
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
if (options.instanceId !== undefined)
|
|
218
|
+
parameters.push(options.instanceId);
|
|
219
|
+
parameters.push(now);
|
|
220
|
+
}
|
|
221
|
+
const candidates = await connection.all(`SELECT ready.message_id, ready.instance_id
|
|
222
|
+
FROM ${this.table("ready_messages")} ready
|
|
223
|
+
JOIN ${this.table("instances")} instances ON instances.id = ready.instance_id
|
|
224
|
+
WHERE ready.available_at_ms <= ? AND instances.paused = 0
|
|
225
|
+
${instanceCondition}
|
|
226
|
+
AND NOT EXISTS (
|
|
227
|
+
SELECT 1 FROM ${this.table("claimed_messages")} claimed
|
|
228
|
+
WHERE claimed.instance_id = ready.instance_id AND claimed.sequence < ready.sequence
|
|
229
|
+
)
|
|
230
|
+
AND NOT EXISTS (
|
|
231
|
+
SELECT 1 FROM ${this.table("ready_messages")} earlier
|
|
232
|
+
WHERE earlier.instance_id = ready.instance_id AND earlier.sequence < ready.sequence
|
|
233
|
+
)
|
|
234
|
+
${leaseCondition}
|
|
235
|
+
ORDER BY ready.available_at_ms, ready.sequence, ready.message_id
|
|
236
|
+
LIMIT ?`, [
|
|
237
|
+
...parameters,
|
|
238
|
+
options.activation || options.instanceId ? 1 : this.settings.claimScanLimit,
|
|
239
|
+
]);
|
|
240
|
+
for (const candidate of candidates) {
|
|
241
|
+
const token = options.activation?.activationToken ?? randomUUID();
|
|
242
|
+
if (options.activation) {
|
|
243
|
+
return this.claimCandidate({ connection, candidate, processId, token, now });
|
|
244
|
+
}
|
|
245
|
+
const lease = await connection.run(`UPDATE ${this.table("instances")}
|
|
246
|
+
SET activation_owner_id = ?, activation_token = ?, activation_generation = activation_generation + 1,
|
|
247
|
+
activation_expires_at_ms = ?, updated_at_ms = ?
|
|
248
|
+
WHERE id = ? AND (activation_owner_id IS NULL OR activation_expires_at_ms <= ?)`, [
|
|
249
|
+
processId,
|
|
250
|
+
token,
|
|
251
|
+
now + this.settings.leaseDurationMilliseconds,
|
|
252
|
+
now,
|
|
253
|
+
candidate.instance_id,
|
|
254
|
+
now,
|
|
255
|
+
]);
|
|
256
|
+
if (lease.changes !== 1)
|
|
257
|
+
continue;
|
|
258
|
+
return this.claimCandidate({ connection, candidate, processId, token, now });
|
|
259
|
+
}
|
|
260
|
+
return undefined;
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
async claimCandidate(options) {
|
|
264
|
+
const { connection, candidate, processId, token, now } = options;
|
|
265
|
+
const instance = await connection.get(`SELECT * FROM ${this.table("instances")} WHERE id = ?`, [candidate.instance_id]);
|
|
266
|
+
const message = await connection.get(`SELECT * FROM ${this.table("messages")} WHERE id = ?`, [candidate.message_id]);
|
|
267
|
+
if (!instance || !message)
|
|
268
|
+
return undefined;
|
|
269
|
+
const removed = await connection.run(`DELETE FROM ${this.table("ready_messages")} WHERE message_id = ?`, [message.id]);
|
|
270
|
+
if (removed.changes !== 1)
|
|
271
|
+
return undefined;
|
|
272
|
+
await connection.run(`INSERT INTO ${this.table("claimed_messages")}
|
|
273
|
+
(message_id, instance_id, sequence, process_id, activation_token, activation_generation, claimed_at_ms)
|
|
274
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`, [
|
|
275
|
+
message.id,
|
|
276
|
+
instance.id,
|
|
277
|
+
message.sequence,
|
|
278
|
+
processId,
|
|
279
|
+
token,
|
|
280
|
+
instance.activation_generation,
|
|
281
|
+
now,
|
|
282
|
+
]);
|
|
283
|
+
await connection.run(`UPDATE ${this.table("messages")} SET attempt_count = attempt_count + 1, updated_at_ms = ? WHERE id = ?`, [now, message.id]);
|
|
284
|
+
message.attempt_count = Number(message.attempt_count) + 1;
|
|
285
|
+
return {
|
|
286
|
+
instance,
|
|
287
|
+
message,
|
|
288
|
+
processId,
|
|
289
|
+
activationToken: token,
|
|
290
|
+
activationGeneration: BigInt(instance.activation_generation),
|
|
291
|
+
nowMilliseconds: now,
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
async renewTurn(turn) {
|
|
295
|
+
await this.settings.database.transaction(async (connection) => {
|
|
296
|
+
const now = await connection.nowMilliseconds();
|
|
297
|
+
const renewed = await connection.run(`UPDATE ${this.table("instances")}
|
|
298
|
+
SET activation_expires_at_ms = ?, updated_at_ms = ?
|
|
299
|
+
WHERE id = ? AND activation_owner_id = ? AND activation_token = ?
|
|
300
|
+
AND activation_generation = ? AND activation_expires_at_ms > ?
|
|
301
|
+
AND EXISTS (
|
|
302
|
+
SELECT 1 FROM ${this.table("claimed_messages")} claimed
|
|
303
|
+
WHERE claimed.message_id = ? AND claimed.instance_id = ?
|
|
304
|
+
AND claimed.process_id = ? AND claimed.activation_token = ?
|
|
305
|
+
AND claimed.activation_generation = ?
|
|
306
|
+
)`, [
|
|
307
|
+
now + this.settings.leaseDurationMilliseconds,
|
|
308
|
+
now,
|
|
309
|
+
turn.instance.id,
|
|
310
|
+
turn.processId,
|
|
311
|
+
turn.activationToken,
|
|
312
|
+
turn.activationGeneration,
|
|
313
|
+
now,
|
|
314
|
+
turn.message.id,
|
|
315
|
+
turn.instance.id,
|
|
316
|
+
turn.processId,
|
|
317
|
+
turn.activationToken,
|
|
318
|
+
turn.activationGeneration,
|
|
319
|
+
]);
|
|
320
|
+
if (renewed.changes !== 1)
|
|
321
|
+
throw new LostActivation("activation lease could not be renewed");
|
|
322
|
+
await connection.run(`UPDATE ${this.table("processes")} SET heartbeat_at_ms = ? WHERE id = ?`, [now, turn.processId]);
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
async releaseFailedTurnSetup(turn) {
|
|
326
|
+
await this.settings.database.transaction(async (connection) => {
|
|
327
|
+
const now = await connection.nowMilliseconds();
|
|
328
|
+
const released = await connection.run(`UPDATE ${this.table("instances")}
|
|
329
|
+
SET activation_owner_id = NULL, activation_token = NULL,
|
|
330
|
+
activation_expires_at_ms = NULL, updated_at_ms = ?
|
|
331
|
+
WHERE id = ? AND activation_owner_id = ? AND activation_token = ?
|
|
332
|
+
AND activation_generation = ?`, [now, turn.instance.id, turn.processId, turn.activationToken, turn.activationGeneration]);
|
|
333
|
+
if (released.changes !== 1)
|
|
334
|
+
throw new LostActivation("activation fence no longer matches");
|
|
335
|
+
const removed = await connection.run(`DELETE FROM ${this.table("claimed_messages")}
|
|
336
|
+
WHERE message_id = ? AND instance_id = ? AND process_id = ? AND activation_token = ?
|
|
337
|
+
AND activation_generation = ?`, [
|
|
338
|
+
turn.message.id,
|
|
339
|
+
turn.instance.id,
|
|
340
|
+
turn.processId,
|
|
341
|
+
turn.activationToken,
|
|
342
|
+
turn.activationGeneration,
|
|
343
|
+
]);
|
|
344
|
+
if (removed.changes !== 1)
|
|
345
|
+
throw new LostActivation("message claim no longer matches");
|
|
346
|
+
const restored = await connection.run(`INSERT INTO ${this.table("ready_messages")}
|
|
347
|
+
(message_id, instance_id, sequence, available_at_ms)
|
|
348
|
+
VALUES (?, ?, ?, ?) ON CONFLICT(message_id) DO NOTHING`, [turn.message.id, turn.instance.id, turn.message.sequence, now]);
|
|
349
|
+
if (restored.changes !== 1)
|
|
350
|
+
throw new LostActivation("ready membership could not be restored");
|
|
351
|
+
await connection.run(`UPDATE ${this.table("messages")}
|
|
352
|
+
SET attempt_count = attempt_count - 1, updated_at_ms = ? WHERE id = ?`, [now, turn.message.id]);
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
async renewActivation(activation) {
|
|
356
|
+
await this.settings.database.transaction(async (connection) => {
|
|
357
|
+
const now = await connection.nowMilliseconds();
|
|
358
|
+
const renewed = await connection.run(`UPDATE ${this.table("instances")}
|
|
359
|
+
SET activation_expires_at_ms = ?, updated_at_ms = ?
|
|
360
|
+
WHERE id = ? AND activation_owner_id = ? AND activation_token = ?
|
|
361
|
+
AND activation_generation = ? AND activation_expires_at_ms > ?`, [
|
|
362
|
+
now + this.settings.leaseDurationMilliseconds,
|
|
363
|
+
now,
|
|
364
|
+
activation.instanceId,
|
|
365
|
+
activation.processId,
|
|
366
|
+
activation.activationToken,
|
|
367
|
+
activation.activationGeneration,
|
|
368
|
+
now,
|
|
369
|
+
]);
|
|
370
|
+
if (renewed.changes !== 1)
|
|
371
|
+
throw new LostActivation("activation lease could not be renewed");
|
|
372
|
+
await connection.run(`UPDATE ${this.table("processes")} SET heartbeat_at_ms = ? WHERE id = ?`, [now, activation.processId]);
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
async releaseActivation(activation) {
|
|
376
|
+
await this.settings.database.transaction(async (connection) => {
|
|
377
|
+
const now = await connection.nowMilliseconds();
|
|
378
|
+
await connection.run(`UPDATE ${this.table("instances")}
|
|
379
|
+
SET activation_owner_id = NULL, activation_token = NULL,
|
|
380
|
+
activation_expires_at_ms = NULL, updated_at_ms = ?
|
|
381
|
+
WHERE id = ? AND activation_owner_id = ? AND activation_token = ?
|
|
382
|
+
AND activation_generation = ?`, [
|
|
383
|
+
now,
|
|
384
|
+
activation.instanceId,
|
|
385
|
+
activation.processId,
|
|
386
|
+
activation.activationToken,
|
|
387
|
+
activation.activationGeneration,
|
|
388
|
+
]);
|
|
389
|
+
});
|
|
390
|
+
}
|
|
391
|
+
async yieldReadyMessages(instanceId) {
|
|
392
|
+
return this.settings.database.transaction(async (connection) => {
|
|
393
|
+
const now = await connection.nowMilliseconds();
|
|
394
|
+
const yielded = await connection.run(`UPDATE ${this.table("ready_messages")} SET available_at_ms = ?
|
|
395
|
+
WHERE instance_id = ? AND available_at_ms <= ?`, [now, instanceId, now]);
|
|
396
|
+
return yielded.changes;
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
async complete(turn, input) {
|
|
400
|
+
const reminderReplacements = [];
|
|
401
|
+
await this.settings.database.transaction(async (connection) => {
|
|
402
|
+
const now = await this.assertFence(connection, turn);
|
|
403
|
+
for (const intent of input.intents.commitActions) {
|
|
404
|
+
await input.executeCommitAction(intent, connection);
|
|
405
|
+
}
|
|
406
|
+
await connection.run(`UPDATE ${this.table("instances")} SET state = ?, state_version = ?, state_revision = ?,
|
|
407
|
+
updated_at_ms = ? WHERE id = ?`, [
|
|
408
|
+
JSON.stringify(input.state),
|
|
409
|
+
input.stateVersion,
|
|
410
|
+
turn.message.sequence,
|
|
411
|
+
now,
|
|
412
|
+
turn.instance.id,
|
|
413
|
+
]);
|
|
414
|
+
await connection.run(`UPDATE ${this.table("messages")} SET result = ?, completed_at_ms = ?, updated_at_ms = ? WHERE id = ?`, [JSON.stringify(input.result), now, now, turn.message.id]);
|
|
415
|
+
await connection.run(`DELETE FROM ${this.table("claimed_messages")} WHERE message_id = ?`, [
|
|
416
|
+
turn.message.id,
|
|
417
|
+
]);
|
|
418
|
+
for (const effect of input.intents.effects) {
|
|
419
|
+
await connection.run(`INSERT INTO ${this.table("effects")}
|
|
420
|
+
(id, message_id, instance_id, name, arguments, success_operation, failure_operation,
|
|
421
|
+
status, max_attempts, available_at_ms)
|
|
422
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)`, [
|
|
423
|
+
randomUUID(),
|
|
424
|
+
turn.message.id,
|
|
425
|
+
turn.instance.id,
|
|
426
|
+
effect.name,
|
|
427
|
+
JSON.stringify(effect.arguments),
|
|
428
|
+
effect.successOperation ?? null,
|
|
429
|
+
effect.failureOperation ?? null,
|
|
430
|
+
this.settings.maxAttempts,
|
|
431
|
+
now,
|
|
432
|
+
]);
|
|
433
|
+
}
|
|
434
|
+
for (const reminder of input.intents.reminders) {
|
|
435
|
+
const existing = await connection.get(`SELECT id, run_at_ms FROM ${this.table("reminders")}
|
|
436
|
+
WHERE instance_id = ? AND operation = ?`, [turn.instance.id, reminder.operation]);
|
|
437
|
+
if (existing && Number(existing.run_at_ms) !== reminder.atMilliseconds) {
|
|
438
|
+
reminderReplacements.push({
|
|
439
|
+
reminderId: existing.id,
|
|
440
|
+
operation: reminder.operation,
|
|
441
|
+
previousRunAtMilliseconds: Number(existing.run_at_ms),
|
|
442
|
+
nextRunAtMilliseconds: reminder.atMilliseconds,
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
await connection.run(`INSERT INTO ${this.table("reminders")}
|
|
446
|
+
(id, instance_id, operation, run_at_ms, arguments, interval_ms, missed_policy, status)
|
|
447
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 'scheduled')
|
|
448
|
+
ON CONFLICT(instance_id, operation) DO UPDATE SET run_at_ms = excluded.run_at_ms,
|
|
449
|
+
arguments = excluded.arguments, interval_ms = excluded.interval_ms,
|
|
450
|
+
missed_policy = excluded.missed_policy, status = 'scheduled', error = NULL`, [
|
|
451
|
+
randomUUID(),
|
|
452
|
+
turn.instance.id,
|
|
453
|
+
reminder.operation,
|
|
454
|
+
reminder.atMilliseconds,
|
|
455
|
+
JSON.stringify(reminder.arguments),
|
|
456
|
+
reminder.intervalMilliseconds ?? null,
|
|
457
|
+
reminder.missedPolicy,
|
|
458
|
+
]);
|
|
459
|
+
}
|
|
460
|
+
for (const outbound of input.intents.outboundMessages) {
|
|
461
|
+
await this.enqueueInTransaction(connection, {
|
|
462
|
+
actorType: outbound.actorType,
|
|
463
|
+
actorId: outbound.actorId,
|
|
464
|
+
operation: outbound.operation,
|
|
465
|
+
deliveryMode: "internal",
|
|
466
|
+
arguments: outbound.arguments,
|
|
467
|
+
...(outbound.availableAtMilliseconds === undefined
|
|
468
|
+
? {}
|
|
469
|
+
: { availableAtMilliseconds: outbound.availableAtMilliseconds }),
|
|
470
|
+
...(outbound.idempotencyKey === undefined
|
|
471
|
+
? {}
|
|
472
|
+
: { idempotencyKey: outbound.idempotencyKey }),
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
if (input.broadcastObservables !== undefined) {
|
|
476
|
+
await connection.run(`INSERT INTO ${this.table("broadcasts")}
|
|
477
|
+
(id, message_id, instance_id, actor_type, actor_id, state_revision, observables,
|
|
478
|
+
status, available_at_ms)
|
|
479
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 'pending', ?)`, [
|
|
480
|
+
randomUUID(),
|
|
481
|
+
turn.message.id,
|
|
482
|
+
turn.instance.id,
|
|
483
|
+
turn.message.actor_type,
|
|
484
|
+
turn.message.actor_id,
|
|
485
|
+
turn.message.sequence,
|
|
486
|
+
JSON.stringify(input.broadcastObservables),
|
|
487
|
+
now,
|
|
488
|
+
]);
|
|
489
|
+
}
|
|
490
|
+
});
|
|
491
|
+
return { reminderReplacements };
|
|
492
|
+
}
|
|
493
|
+
async listReminders(options) {
|
|
494
|
+
const conditions = [];
|
|
495
|
+
const parameters = [];
|
|
496
|
+
if (options.actorType !== undefined) {
|
|
497
|
+
conditions.push("instances.actor_type = ?");
|
|
498
|
+
parameters.push(options.actorType);
|
|
499
|
+
}
|
|
500
|
+
if (options.status !== undefined) {
|
|
501
|
+
conditions.push("reminders.status = ?");
|
|
502
|
+
parameters.push(options.status);
|
|
503
|
+
}
|
|
504
|
+
if (options.cursor !== undefined) {
|
|
505
|
+
conditions.push("reminders.id > ?");
|
|
506
|
+
parameters.push(options.cursor);
|
|
507
|
+
}
|
|
508
|
+
parameters.push(options.limit + 1);
|
|
509
|
+
const where = conditions.length === 0 ? "" : `WHERE ${conditions.join(" AND ")}`;
|
|
510
|
+
return this.settings.database.connection((connection) => connection.all(`SELECT reminders.*, instances.actor_type, instances.actor_id
|
|
511
|
+
FROM ${this.table("reminders")} reminders
|
|
512
|
+
JOIN ${this.table("instances")} instances ON instances.id = reminders.instance_id
|
|
513
|
+
${where} ORDER BY reminders.id LIMIT ?`, parameters));
|
|
514
|
+
}
|
|
515
|
+
async resumeReminder(options) {
|
|
516
|
+
return this.settings.database.transaction(async (connection) => {
|
|
517
|
+
const reminder = await this.findReminderInConnection(connection, options.id);
|
|
518
|
+
if (!reminder)
|
|
519
|
+
throw new UnknownReminder(`unknown reminder ${options.id}`);
|
|
520
|
+
if (reminder.status === "completed") {
|
|
521
|
+
throw new ReminderNotPaused(`completed reminder ${options.id} cannot be resumed`);
|
|
522
|
+
}
|
|
523
|
+
if (reminder.status === "scheduled")
|
|
524
|
+
return { reminder, resumed: false };
|
|
525
|
+
const runAt = options.runAtMilliseconds ?? (await connection.nowMilliseconds());
|
|
526
|
+
await connection.run(`UPDATE ${this.table("reminders")}
|
|
527
|
+
SET status = 'scheduled', run_at_ms = ?, error = NULL, claimed_by = NULL, claimed_at_ms = NULL
|
|
528
|
+
WHERE id = ? AND status = 'paused'`, [runAt, options.id]);
|
|
529
|
+
const resumed = await this.findReminderInConnection(connection, options.id);
|
|
530
|
+
if (!resumed)
|
|
531
|
+
throw new UnknownReminder(`unknown reminder ${options.id}`);
|
|
532
|
+
return { reminder: resumed, resumed: true };
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
async findReminderInConnection(connection, id) {
|
|
536
|
+
return connection.get(`SELECT reminders.*, instances.actor_type, instances.actor_id
|
|
537
|
+
FROM ${this.table("reminders")} reminders
|
|
538
|
+
JOIN ${this.table("instances")} instances ON instances.id = reminders.instance_id
|
|
539
|
+
WHERE reminders.id = ?`, [id]);
|
|
540
|
+
}
|
|
541
|
+
async reject(turn, rejection) {
|
|
542
|
+
await this.settings.database.transaction(async (connection) => {
|
|
543
|
+
const now = await this.assertFence(connection, turn);
|
|
544
|
+
await connection.run(`UPDATE ${this.table("messages")} SET rejection = ?, completed_at_ms = ?, updated_at_ms = ? WHERE id = ?`, [JSON.stringify(rejection), now, now, turn.message.id]);
|
|
545
|
+
await this.releaseClaim({ connection, turn });
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
async fail(turn, options) {
|
|
549
|
+
return this.settings.database.transaction(async (connection) => {
|
|
550
|
+
const now = await this.assertFence(connection, turn);
|
|
551
|
+
const errorRecord = safeError(options.error);
|
|
552
|
+
const exhausted = !options.retryable ||
|
|
553
|
+
Number(turn.message.attempt_count) >= Number(turn.message.max_attempts);
|
|
554
|
+
if (exhausted) {
|
|
555
|
+
await connection.run(`UPDATE ${this.table("messages")} SET error = ?, completed_at_ms = ?, updated_at_ms = ? WHERE id = ?`, [JSON.stringify(errorRecord), now, now, turn.message.id]);
|
|
556
|
+
await connection.run(`INSERT INTO ${this.table("dead_letters")}
|
|
557
|
+
(id, message_id, instance_id, attempts, error, created_at_ms) VALUES (?, ?, ?, ?, ?, ?)`, [
|
|
558
|
+
randomUUID(),
|
|
559
|
+
turn.message.id,
|
|
560
|
+
turn.instance.id,
|
|
561
|
+
turn.message.attempt_count,
|
|
562
|
+
JSON.stringify(errorRecord),
|
|
563
|
+
now,
|
|
564
|
+
]);
|
|
565
|
+
await this.releaseClaim({ connection, turn });
|
|
566
|
+
return "dead";
|
|
567
|
+
}
|
|
568
|
+
const availableAt = now + this.settings.retryDelayMilliseconds(Number(turn.message.attempt_count));
|
|
569
|
+
await connection.run(`UPDATE ${this.table("messages")} SET error = ?, updated_at_ms = ? WHERE id = ?`, [JSON.stringify(errorRecord), now, turn.message.id]);
|
|
570
|
+
await connection.run(`DELETE FROM ${this.table("claimed_messages")} WHERE message_id = ?`, [
|
|
571
|
+
turn.message.id,
|
|
572
|
+
]);
|
|
573
|
+
await connection.run(`INSERT INTO ${this.table("ready_messages")}(message_id, instance_id, sequence, available_at_ms)
|
|
574
|
+
VALUES (?, ?, ?, ?)`, [turn.message.id, turn.instance.id, turn.message.sequence, availableAt]);
|
|
575
|
+
return "retrying";
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
async findMessage(id) {
|
|
579
|
+
return this.settings.database.connection((connection) => connection.get(`SELECT * FROM ${this.table("messages")} WHERE id = ?`, [id]));
|
|
580
|
+
}
|
|
581
|
+
async messageStatus(id) {
|
|
582
|
+
return this.settings.database.connection(async (connection) => {
|
|
583
|
+
const message = await connection.get(`SELECT * FROM ${this.table("messages")} WHERE id = ?`, [id]);
|
|
584
|
+
if (!message)
|
|
585
|
+
return "unknown";
|
|
586
|
+
if (message.rejection !== null)
|
|
587
|
+
return "rejected";
|
|
588
|
+
if (message.completed_at_ms !== null) {
|
|
589
|
+
const dead = await connection.get(`SELECT 1 AS found FROM ${this.table("dead_letters")} WHERE message_id = ?`, [id]);
|
|
590
|
+
return dead ? "dead" : "completed";
|
|
591
|
+
}
|
|
592
|
+
const claimed = await connection.get(`SELECT 1 AS found FROM ${this.table("claimed_messages")} WHERE message_id = ?`, [id]);
|
|
593
|
+
if (claimed)
|
|
594
|
+
return "claimed";
|
|
595
|
+
const ready = await connection.get(`SELECT 1 AS found FROM ${this.table("ready_messages")} WHERE message_id = ?`, [id]);
|
|
596
|
+
return ready ? "ready" : "unknown";
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
async syncDiagnostics(messageId) {
|
|
600
|
+
return this.settings.database.connection(async (connection) => {
|
|
601
|
+
const message = await connection.get(`SELECT * FROM ${this.table("messages")} WHERE id = ?`, [messageId]);
|
|
602
|
+
if (!message)
|
|
603
|
+
return undefined;
|
|
604
|
+
const instance = await connection.get(`SELECT * FROM ${this.table("instances")} WHERE id = ?`, [message.instance_id]);
|
|
605
|
+
if (!instance)
|
|
606
|
+
return undefined;
|
|
607
|
+
const process = instance.activation_owner_id
|
|
608
|
+
? await connection.get(`SELECT * FROM ${this.table("processes")} WHERE id = ?`, [instance.activation_owner_id])
|
|
609
|
+
: undefined;
|
|
610
|
+
const claimed = await connection.get(`SELECT 1 AS found FROM ${this.table("claimed_messages")} WHERE message_id = ?`, [message.id]);
|
|
611
|
+
const ready = await connection.get(`SELECT available_at_ms FROM ${this.table("ready_messages")} WHERE message_id = ?`, [message.id]);
|
|
612
|
+
const dead = message.completed_at_ms === null
|
|
613
|
+
? undefined
|
|
614
|
+
: await connection.get(`SELECT 1 AS found FROM ${this.table("dead_letters")} WHERE message_id = ?`, [message.id]);
|
|
615
|
+
const blocker = await connection.get(`SELECT messages.*,
|
|
616
|
+
CASE WHEN claimed.message_id IS NOT NULL THEN 'claimed' ELSE 'ready' END AS membership_status
|
|
617
|
+
FROM ${this.table("messages")} messages
|
|
618
|
+
LEFT JOIN ${this.table("ready_messages")} ready ON ready.message_id = messages.id
|
|
619
|
+
LEFT JOIN ${this.table("claimed_messages")} claimed ON claimed.message_id = messages.id
|
|
620
|
+
WHERE messages.instance_id = ? AND messages.sequence < ?
|
|
621
|
+
AND (ready.message_id IS NOT NULL OR claimed.message_id IS NOT NULL)
|
|
622
|
+
ORDER BY messages.sequence LIMIT 1`, [message.instance_id, message.sequence]);
|
|
623
|
+
const status = message.rejection
|
|
624
|
+
? "rejected"
|
|
625
|
+
: message.completed_at_ms !== null
|
|
626
|
+
? dead
|
|
627
|
+
? "dead"
|
|
628
|
+
: "completed"
|
|
629
|
+
: claimed
|
|
630
|
+
? "claimed"
|
|
631
|
+
: ready
|
|
632
|
+
? "ready"
|
|
633
|
+
: "unknown";
|
|
634
|
+
return {
|
|
635
|
+
message,
|
|
636
|
+
instance,
|
|
637
|
+
process,
|
|
638
|
+
status,
|
|
639
|
+
readyAvailableAtMilliseconds: ready === undefined ? undefined : Number(ready.available_at_ms),
|
|
640
|
+
blocker,
|
|
641
|
+
nowMilliseconds: await connection.nowMilliseconds(),
|
|
642
|
+
};
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
async listDeadLetters() {
|
|
646
|
+
return this.settings.database.connection((connection) => connection.all(`SELECT dead_letters.*, messages.actor_type, messages.actor_id, messages.operation,
|
|
647
|
+
messages.delivery_mode, messages.arguments
|
|
648
|
+
FROM ${this.table("dead_letters")} dead_letters
|
|
649
|
+
JOIN ${this.table("messages")} messages ON messages.id = dead_letters.message_id
|
|
650
|
+
ORDER BY dead_letters.created_at_ms DESC, dead_letters.id DESC`));
|
|
651
|
+
}
|
|
652
|
+
async findDeadLetter(id) {
|
|
653
|
+
return this.settings.database.connection((connection) => this.findDeadLetterInConnection(connection, id));
|
|
654
|
+
}
|
|
655
|
+
async retryDeadLetter(options) {
|
|
656
|
+
return this.settings.database.transaction(async (connection) => {
|
|
657
|
+
const deadLetter = await this.findDeadLetterInConnection(connection, options.id);
|
|
658
|
+
if (!deadLetter)
|
|
659
|
+
throw new UnknownDeadLetter(`unknown dead letter ${options.id}`);
|
|
660
|
+
if (deadLetter.retried_message_id) {
|
|
661
|
+
const retriedMessage = await connection.get(`SELECT * FROM ${this.table("messages")} WHERE id = ?`, [deadLetter.retried_message_id]);
|
|
662
|
+
if (!retriedMessage) {
|
|
663
|
+
throw new Error(`retried message ${deadLetter.retried_message_id} is missing`);
|
|
664
|
+
}
|
|
665
|
+
return retriedMessage;
|
|
666
|
+
}
|
|
667
|
+
const retriedMessage = await this.enqueueInTransaction(connection, {
|
|
668
|
+
actorType: deadLetter.actor_type,
|
|
669
|
+
actorId: deadLetter.actor_id,
|
|
670
|
+
operation: deadLetter.operation,
|
|
671
|
+
deliveryMode: deadLetter.delivery_mode,
|
|
672
|
+
arguments: jsonObject(JSON.parse(deadLetter.arguments)),
|
|
673
|
+
initialState: options.initialState,
|
|
674
|
+
stateVersion: options.stateVersion,
|
|
675
|
+
idempotencyKey: `dead-letter:${deadLetter.id}`,
|
|
676
|
+
});
|
|
677
|
+
await connection.run(`UPDATE ${this.table("dead_letters")} SET retried_message_id = ? WHERE id = ?`, [retriedMessage.id, deadLetter.id]);
|
|
678
|
+
return retriedMessage;
|
|
679
|
+
});
|
|
680
|
+
}
|
|
681
|
+
async findInstanceByIdentity(actorType, actorId) {
|
|
682
|
+
return this.settings.database.connection((connection) => this.findInstance({ connection, actorType, actorId }));
|
|
683
|
+
}
|
|
684
|
+
async activeInstances(options) {
|
|
685
|
+
const conditions = ["paused = 0"];
|
|
686
|
+
const parameters = [];
|
|
687
|
+
if (options.actorType !== undefined) {
|
|
688
|
+
conditions.push("actor_type = ?");
|
|
689
|
+
parameters.push(options.actorType);
|
|
690
|
+
}
|
|
691
|
+
if (options.cursor !== undefined) {
|
|
692
|
+
conditions.push("id > ?");
|
|
693
|
+
parameters.push(options.cursor);
|
|
694
|
+
}
|
|
695
|
+
parameters.push(options.limit + 1);
|
|
696
|
+
return this.settings.database.connection((connection) => connection.all(`SELECT * FROM ${this.table("instances")}
|
|
697
|
+
WHERE ${conditions.join(" AND ")}
|
|
698
|
+
ORDER BY id LIMIT ?`, parameters));
|
|
699
|
+
}
|
|
700
|
+
async instancesWithoutPendingWork(options) {
|
|
701
|
+
return this.settings.database.connection(async (connection) => {
|
|
702
|
+
const now = await connection.nowMilliseconds();
|
|
703
|
+
const conditions = [
|
|
704
|
+
"instances.paused = 0",
|
|
705
|
+
"instances.updated_at_ms <= ?",
|
|
706
|
+
`NOT EXISTS (
|
|
707
|
+
SELECT 1 FROM ${this.table("ready_messages")} ready
|
|
708
|
+
WHERE ready.instance_id = instances.id
|
|
709
|
+
)`,
|
|
710
|
+
`NOT EXISTS (
|
|
711
|
+
SELECT 1 FROM ${this.table("claimed_messages")} claimed
|
|
712
|
+
WHERE claimed.instance_id = instances.id
|
|
713
|
+
)`,
|
|
714
|
+
`NOT EXISTS (
|
|
715
|
+
SELECT 1 FROM ${this.table("reminders")} reminders
|
|
716
|
+
WHERE reminders.instance_id = instances.id AND reminders.status = 'scheduled'
|
|
717
|
+
)`,
|
|
718
|
+
];
|
|
719
|
+
const parameters = [now - options.quietForMilliseconds];
|
|
720
|
+
if (options.actorType !== undefined) {
|
|
721
|
+
conditions.push("instances.actor_type = ?");
|
|
722
|
+
parameters.push(options.actorType);
|
|
723
|
+
}
|
|
724
|
+
if (options.cursor !== undefined) {
|
|
725
|
+
conditions.push("instances.id > ?");
|
|
726
|
+
parameters.push(options.cursor);
|
|
727
|
+
}
|
|
728
|
+
parameters.push(options.limit + 1);
|
|
729
|
+
return connection.all(`SELECT instances.* FROM ${this.table("instances")} instances
|
|
730
|
+
WHERE ${conditions.join(" AND ")}
|
|
731
|
+
ORDER BY instances.id LIMIT ?`, parameters);
|
|
732
|
+
});
|
|
733
|
+
}
|
|
734
|
+
async instanceStatesFor(options) {
|
|
735
|
+
if (options.actorIds.length === 0)
|
|
736
|
+
return [];
|
|
737
|
+
return this.settings.database.connection((connection) => connection.all(`SELECT * FROM ${this.table("instances")}
|
|
738
|
+
WHERE actor_type = ?
|
|
739
|
+
AND actor_id IN (${parameterList(options.actorIds.length)})`, [options.actorType, ...options.actorIds]));
|
|
740
|
+
}
|
|
741
|
+
async orphanedInstances(options) {
|
|
742
|
+
const conditions = ["actor_type = ?"];
|
|
743
|
+
const parameters = [options.actorType];
|
|
744
|
+
if (options.ownerIds.length > 0) {
|
|
745
|
+
conditions.push(`actor_id NOT IN (${parameterList(options.ownerIds.length)})`);
|
|
746
|
+
parameters.push(...options.ownerIds);
|
|
747
|
+
}
|
|
748
|
+
if (options.cursor !== undefined) {
|
|
749
|
+
conditions.push("id > ?");
|
|
750
|
+
parameters.push(options.cursor);
|
|
751
|
+
}
|
|
752
|
+
parameters.push(options.limit + 1);
|
|
753
|
+
return this.settings.database.connection((connection) => connection.all(`SELECT * FROM ${this.table("instances")}
|
|
754
|
+
WHERE ${conditions.join(" AND ")}
|
|
755
|
+
ORDER BY id LIMIT ?`, parameters));
|
|
756
|
+
}
|
|
757
|
+
async previewRetention(target) {
|
|
758
|
+
return this.settings.database.connection(async (connection) => {
|
|
759
|
+
const now = await connection.nowMilliseconds();
|
|
760
|
+
const predicate = this.retentionPredicate({ target, now });
|
|
761
|
+
const row = await connection.get(`SELECT COUNT(*) AS count FROM ${this.table(target)} WHERE ${predicate.sql}`, predicate.parameters);
|
|
762
|
+
return Number(row?.count ?? 0);
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
async pruneRetention(target) {
|
|
766
|
+
let count = 0;
|
|
767
|
+
while (true) {
|
|
768
|
+
const deleted = await this.settings.database.transaction(async (connection) => {
|
|
769
|
+
const now = await connection.nowMilliseconds();
|
|
770
|
+
const predicate = this.retentionPredicate({ target, now });
|
|
771
|
+
const candidates = await connection.all(`SELECT id FROM ${this.table(target)} WHERE ${predicate.sql}
|
|
772
|
+
ORDER BY id LIMIT ?`, [...predicate.parameters, this.settings.pruneBatchSize]);
|
|
773
|
+
if (candidates.length === 0)
|
|
774
|
+
return 0;
|
|
775
|
+
const candidateIds = candidates.map(({ id }) => id);
|
|
776
|
+
const result = await connection.run(`DELETE FROM ${this.table(target)}
|
|
777
|
+
WHERE id IN (${parameterList(candidateIds.length)})
|
|
778
|
+
AND ${predicate.sql}`, [...candidateIds, ...predicate.parameters]);
|
|
779
|
+
return result.changes;
|
|
780
|
+
});
|
|
781
|
+
if (deleted === 0)
|
|
782
|
+
return count;
|
|
783
|
+
count += deleted;
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
retentionPredicate(options) {
|
|
787
|
+
if (options.target === "messages")
|
|
788
|
+
return this.messageRetentionPredicate(options.now);
|
|
789
|
+
if (options.target === "instances")
|
|
790
|
+
return this.instanceRetentionPredicate(options.now);
|
|
791
|
+
return this.processRetentionPredicate(options.now);
|
|
792
|
+
}
|
|
793
|
+
messageRetentionPredicate(now) {
|
|
794
|
+
const table = this.table("messages");
|
|
795
|
+
const retention = retentionPolicy({
|
|
796
|
+
table,
|
|
797
|
+
timestampColumn: "completed_at_ms",
|
|
798
|
+
defaultRetention: this.settings.messageRetentionMilliseconds,
|
|
799
|
+
overrides: this.settings.messageRetentionByActorType,
|
|
800
|
+
now,
|
|
801
|
+
});
|
|
802
|
+
return {
|
|
803
|
+
sql: `${retention.sql}
|
|
804
|
+
AND ${table}.completed_at_ms IS NOT NULL
|
|
805
|
+
AND NOT EXISTS (
|
|
806
|
+
SELECT 1 FROM ${this.table("ready_messages")} ready
|
|
807
|
+
WHERE ready.message_id = ${table}.id
|
|
808
|
+
)
|
|
809
|
+
AND NOT EXISTS (
|
|
810
|
+
SELECT 1 FROM ${this.table("claimed_messages")} claimed
|
|
811
|
+
WHERE claimed.message_id = ${table}.id
|
|
812
|
+
)
|
|
813
|
+
AND NOT EXISTS (
|
|
814
|
+
SELECT 1 FROM ${this.table("dead_letters")} dead
|
|
815
|
+
WHERE dead.message_id = ${table}.id OR dead.retried_message_id = ${table}.id
|
|
816
|
+
)
|
|
817
|
+
AND NOT EXISTS (
|
|
818
|
+
SELECT 1 FROM ${this.table("effects")} effects
|
|
819
|
+
WHERE effects.message_id = ${table}.id AND effects.status != 'completed'
|
|
820
|
+
)
|
|
821
|
+
AND NOT EXISTS (
|
|
822
|
+
SELECT 1 FROM ${this.table("broadcasts")} broadcasts
|
|
823
|
+
WHERE broadcasts.message_id = ${table}.id AND broadcasts.status != 'delivered'
|
|
824
|
+
)`,
|
|
825
|
+
parameters: retention.parameters,
|
|
826
|
+
};
|
|
827
|
+
}
|
|
828
|
+
instanceRetentionPredicate(now) {
|
|
829
|
+
const table = this.table("instances");
|
|
830
|
+
const retention = retentionPolicy({
|
|
831
|
+
table,
|
|
832
|
+
timestampColumn: "updated_at_ms",
|
|
833
|
+
overrides: this.settings.instanceRetentionByActorType,
|
|
834
|
+
now,
|
|
835
|
+
});
|
|
836
|
+
return {
|
|
837
|
+
sql: `${retention.sql}
|
|
838
|
+
AND ${table}.activation_owner_id IS NULL
|
|
839
|
+
AND ${table}.paused = 0
|
|
840
|
+
AND NOT EXISTS (
|
|
841
|
+
SELECT 1 FROM ${this.table("ready_messages")} ready
|
|
842
|
+
WHERE ready.instance_id = ${table}.id
|
|
843
|
+
)
|
|
844
|
+
AND NOT EXISTS (
|
|
845
|
+
SELECT 1 FROM ${this.table("claimed_messages")} claimed
|
|
846
|
+
WHERE claimed.instance_id = ${table}.id
|
|
847
|
+
)
|
|
848
|
+
AND NOT EXISTS (
|
|
849
|
+
SELECT 1 FROM ${this.table("reminders")} reminders
|
|
850
|
+
WHERE reminders.instance_id = ${table}.id AND reminders.status = 'scheduled'
|
|
851
|
+
)
|
|
852
|
+
AND NOT EXISTS (
|
|
853
|
+
SELECT 1 FROM ${this.table("effects")} effects
|
|
854
|
+
WHERE effects.instance_id = ${table}.id AND effects.status != 'completed'
|
|
855
|
+
)
|
|
856
|
+
AND NOT EXISTS (
|
|
857
|
+
SELECT 1 FROM ${this.table("broadcasts")} broadcasts
|
|
858
|
+
WHERE broadcasts.instance_id = ${table}.id AND broadcasts.status != 'delivered'
|
|
859
|
+
)
|
|
860
|
+
AND NOT EXISTS (
|
|
861
|
+
SELECT 1 FROM ${this.table("dead_letters")} dead
|
|
862
|
+
WHERE dead.instance_id = ${table}.id
|
|
863
|
+
)`,
|
|
864
|
+
parameters: retention.parameters,
|
|
865
|
+
};
|
|
866
|
+
}
|
|
867
|
+
processRetentionPredicate(now) {
|
|
868
|
+
const table = this.table("processes");
|
|
869
|
+
return {
|
|
870
|
+
sql: `${table}.shutdown_state = 'stopped'
|
|
871
|
+
AND ${table}.stopped_at_ms IS NOT NULL
|
|
872
|
+
AND ${table}.stopped_at_ms < ?
|
|
873
|
+
AND NOT EXISTS (
|
|
874
|
+
SELECT 1 FROM ${this.table("claimed_messages")} claimed
|
|
875
|
+
WHERE claimed.process_id = ${table}.id
|
|
876
|
+
)
|
|
877
|
+
AND NOT EXISTS (
|
|
878
|
+
SELECT 1 FROM ${this.table("instances")} instances
|
|
879
|
+
WHERE instances.activation_owner_id = ${table}.id
|
|
880
|
+
)`,
|
|
881
|
+
parameters: [now - this.settings.processRetentionMilliseconds],
|
|
882
|
+
};
|
|
883
|
+
}
|
|
884
|
+
async findDeadLetterInConnection(connection, id) {
|
|
885
|
+
return connection.get(`SELECT dead_letters.*, messages.actor_type, messages.actor_id, messages.operation,
|
|
886
|
+
messages.delivery_mode, messages.arguments
|
|
887
|
+
FROM ${this.table("dead_letters")} dead_letters
|
|
888
|
+
JOIN ${this.table("messages")} messages ON messages.id = dead_letters.message_id
|
|
889
|
+
WHERE dead_letters.id = ?`, [id]);
|
|
890
|
+
}
|
|
891
|
+
async destroy(actorType, actorId) {
|
|
892
|
+
return this.settings.database.transaction(async (connection) => {
|
|
893
|
+
const result = await connection.run(`DELETE FROM ${this.table("instances")} WHERE actor_type = ? AND actor_id = ?`, [actorType, actorId]);
|
|
894
|
+
return result.changes === 1;
|
|
895
|
+
});
|
|
896
|
+
}
|
|
897
|
+
async resetForTesting() {
|
|
898
|
+
const tables = [
|
|
899
|
+
"dead_letters",
|
|
900
|
+
"claimed_messages",
|
|
901
|
+
"ready_messages",
|
|
902
|
+
"broadcasts",
|
|
903
|
+
"effects",
|
|
904
|
+
"reminders",
|
|
905
|
+
"messages",
|
|
906
|
+
"instances",
|
|
907
|
+
"processes",
|
|
908
|
+
];
|
|
909
|
+
await this.settings.database.transaction(async (connection) => {
|
|
910
|
+
for (const table of tables)
|
|
911
|
+
await connection.run(`DELETE FROM ${this.table(table)}`);
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
async claimEffect(processId) {
|
|
915
|
+
return this.settings.database.transaction(async (connection) => {
|
|
916
|
+
const now = await connection.nowMilliseconds();
|
|
917
|
+
const staleAt = now - this.settings.processAliveThresholdMilliseconds;
|
|
918
|
+
await connection.run(`UPDATE ${this.table("effects")} SET status = 'pending', claimed_by = NULL
|
|
919
|
+
WHERE status = 'processing' AND (
|
|
920
|
+
claimed_by IS NULL OR NOT EXISTS (
|
|
921
|
+
SELECT 1 FROM ${this.table("processes")} processes
|
|
922
|
+
WHERE processes.id = ${this.table("effects")}.claimed_by
|
|
923
|
+
AND processes.shutdown_state = 'running' AND processes.heartbeat_at_ms > ?
|
|
924
|
+
)
|
|
925
|
+
)`, [staleAt]);
|
|
926
|
+
const effect = await connection.get(`SELECT effects.*, instances.actor_type, instances.actor_id
|
|
927
|
+
FROM ${this.table("effects")} effects
|
|
928
|
+
JOIN ${this.table("instances")} instances ON instances.id = effects.instance_id
|
|
929
|
+
WHERE effects.status = 'pending' AND effects.available_at_ms <= ?
|
|
930
|
+
ORDER BY effects.available_at_ms, effects.id
|
|
931
|
+
LIMIT 1`, [now]);
|
|
932
|
+
if (!effect)
|
|
933
|
+
return undefined;
|
|
934
|
+
const claimed = await connection.run(`UPDATE ${this.table("effects")}
|
|
935
|
+
SET status = 'processing', claimed_by = ?, attempt_count = attempt_count + 1
|
|
936
|
+
WHERE id = ? AND status = 'pending'`, [processId, effect.id]);
|
|
937
|
+
if (claimed.changes !== 1)
|
|
938
|
+
return undefined;
|
|
939
|
+
effect.status = "processing";
|
|
940
|
+
effect.claimed_by = processId;
|
|
941
|
+
effect.attempt_count = Number(effect.attempt_count) + 1;
|
|
942
|
+
return effect;
|
|
943
|
+
});
|
|
944
|
+
}
|
|
945
|
+
async completeEffect(effect, result) {
|
|
946
|
+
await this.settings.database.transaction(async (connection) => {
|
|
947
|
+
const now = await connection.nowMilliseconds();
|
|
948
|
+
const updated = await connection.run(`UPDATE ${this.table("effects")}
|
|
949
|
+
SET status = 'completed', result = ?, error = NULL, claimed_by = NULL
|
|
950
|
+
WHERE id = ? AND status = 'processing' AND claimed_by = ?`, [JSON.stringify(result), effect.id, effect.claimed_by]);
|
|
951
|
+
if (updated.changes !== 1)
|
|
952
|
+
throw new LostActivation("effect claim no longer matches");
|
|
953
|
+
if (!effect.success_operation)
|
|
954
|
+
return;
|
|
955
|
+
await this.enqueueInTransaction(connection, {
|
|
956
|
+
actorType: effect.actor_type,
|
|
957
|
+
actorId: effect.actor_id,
|
|
958
|
+
operation: effect.success_operation,
|
|
959
|
+
deliveryMode: "internal",
|
|
960
|
+
arguments: jsonObject({ effectId: effect.id, result }),
|
|
961
|
+
idempotencyKey: `effect:${effect.id}:success`,
|
|
962
|
+
});
|
|
963
|
+
void now;
|
|
964
|
+
});
|
|
965
|
+
}
|
|
966
|
+
async failEffect(options) {
|
|
967
|
+
const { effect, error, retryable } = options;
|
|
968
|
+
await this.settings.database.transaction(async (connection) => {
|
|
969
|
+
const now = await connection.nowMilliseconds();
|
|
970
|
+
const errorRecord = safeError(error);
|
|
971
|
+
const exhausted = !retryable || Number(effect.attempt_count) >= Number(effect.max_attempts);
|
|
972
|
+
if (!exhausted) {
|
|
973
|
+
const availableAt = now + this.settings.retryDelayMilliseconds(Number(effect.attempt_count));
|
|
974
|
+
const updated = await connection.run(`UPDATE ${this.table("effects")}
|
|
975
|
+
SET status = 'pending', error = ?, available_at_ms = ?, claimed_by = NULL
|
|
976
|
+
WHERE id = ? AND status = 'processing' AND claimed_by = ?`, [JSON.stringify(errorRecord), availableAt, effect.id, effect.claimed_by]);
|
|
977
|
+
if (updated.changes !== 1)
|
|
978
|
+
throw new LostActivation("effect claim no longer matches");
|
|
979
|
+
return;
|
|
980
|
+
}
|
|
981
|
+
const updated = await connection.run(`UPDATE ${this.table("effects")}
|
|
982
|
+
SET status = 'dead', error = ?, claimed_by = NULL
|
|
983
|
+
WHERE id = ? AND status = 'processing' AND claimed_by = ?`, [JSON.stringify(errorRecord), effect.id, effect.claimed_by]);
|
|
984
|
+
if (updated.changes !== 1)
|
|
985
|
+
throw new LostActivation("effect claim no longer matches");
|
|
986
|
+
if (!effect.failure_operation)
|
|
987
|
+
return;
|
|
988
|
+
await this.enqueueInTransaction(connection, {
|
|
989
|
+
actorType: effect.actor_type,
|
|
990
|
+
actorId: effect.actor_id,
|
|
991
|
+
operation: effect.failure_operation,
|
|
992
|
+
deliveryMode: "internal",
|
|
993
|
+
arguments: jsonObject({ effectId: effect.id, error: errorRecord }),
|
|
994
|
+
idempotencyKey: `effect:${effect.id}:failure`,
|
|
995
|
+
});
|
|
996
|
+
});
|
|
997
|
+
}
|
|
998
|
+
async claimReminder(processId) {
|
|
999
|
+
return this.settings.database.transaction(async (connection) => {
|
|
1000
|
+
const now = await connection.nowMilliseconds();
|
|
1001
|
+
const staleAt = now - this.settings.processAliveThresholdMilliseconds;
|
|
1002
|
+
await connection.run(`UPDATE ${this.table("reminders")} SET claimed_by = NULL, claimed_at_ms = NULL
|
|
1003
|
+
WHERE claimed_by IS NOT NULL AND (
|
|
1004
|
+
claimed_at_ms <= ? OR NOT EXISTS (
|
|
1005
|
+
SELECT 1 FROM ${this.table("processes")} processes
|
|
1006
|
+
WHERE processes.id = ${this.table("reminders")}.claimed_by
|
|
1007
|
+
AND processes.shutdown_state = 'running' AND processes.heartbeat_at_ms > ?
|
|
1008
|
+
)
|
|
1009
|
+
)`, [staleAt, staleAt]);
|
|
1010
|
+
const reminder = await connection.get(`SELECT reminders.*, instances.actor_type, instances.actor_id
|
|
1011
|
+
FROM ${this.table("reminders")} reminders
|
|
1012
|
+
JOIN ${this.table("instances")} instances ON instances.id = reminders.instance_id
|
|
1013
|
+
WHERE reminders.status = 'scheduled' AND reminders.run_at_ms <= ?
|
|
1014
|
+
AND reminders.claimed_by IS NULL
|
|
1015
|
+
ORDER BY reminders.run_at_ms, reminders.id
|
|
1016
|
+
LIMIT 1`, [now]);
|
|
1017
|
+
if (!reminder)
|
|
1018
|
+
return undefined;
|
|
1019
|
+
const claimed = await connection.run(`UPDATE ${this.table("reminders")} SET claimed_by = ?, claimed_at_ms = ?
|
|
1020
|
+
WHERE id = ? AND status = 'scheduled' AND claimed_by IS NULL`, [processId, now, reminder.id]);
|
|
1021
|
+
if (claimed.changes !== 1)
|
|
1022
|
+
return undefined;
|
|
1023
|
+
reminder.claimed_by = processId;
|
|
1024
|
+
reminder.claimed_at_ms = now;
|
|
1025
|
+
return reminder;
|
|
1026
|
+
});
|
|
1027
|
+
}
|
|
1028
|
+
async enqueueReminder(reminder) {
|
|
1029
|
+
await this.settings.database.transaction(async (connection) => {
|
|
1030
|
+
const now = await connection.nowMilliseconds();
|
|
1031
|
+
const claimed = await connection.get(`SELECT reminders.*, instances.actor_type, instances.actor_id
|
|
1032
|
+
FROM ${this.table("reminders")} reminders
|
|
1033
|
+
JOIN ${this.table("instances")} instances ON instances.id = reminders.instance_id
|
|
1034
|
+
WHERE reminders.id = ? AND reminders.status = 'scheduled' AND reminders.claimed_by = ?`, [reminder.id, reminder.claimed_by]);
|
|
1035
|
+
if (!claimed)
|
|
1036
|
+
throw new LostActivation("reminder claim no longer matches");
|
|
1037
|
+
await this.enqueueInTransaction(connection, {
|
|
1038
|
+
actorType: claimed.actor_type,
|
|
1039
|
+
actorId: claimed.actor_id,
|
|
1040
|
+
operation: claimed.operation,
|
|
1041
|
+
deliveryMode: "internal",
|
|
1042
|
+
arguments: jsonObject(JSON.parse(claimed.arguments)),
|
|
1043
|
+
idempotencyKey: `reminder:${claimed.id}:${claimed.occurrence}`,
|
|
1044
|
+
});
|
|
1045
|
+
const interval = claimed.interval_ms === null ? undefined : Number(claimed.interval_ms);
|
|
1046
|
+
const nextRun = interval === undefined
|
|
1047
|
+
? Number(claimed.run_at_ms)
|
|
1048
|
+
: nextReminderRun({
|
|
1049
|
+
previousRun: Number(claimed.run_at_ms),
|
|
1050
|
+
interval,
|
|
1051
|
+
missedPolicy: claimed.missed_policy,
|
|
1052
|
+
now,
|
|
1053
|
+
});
|
|
1054
|
+
await connection.run(`UPDATE ${this.table("reminders")}
|
|
1055
|
+
SET status = ?, occurrence = occurrence + 1, run_at_ms = ?, claimed_by = NULL, claimed_at_ms = NULL
|
|
1056
|
+
WHERE id = ? AND claimed_by = ?`, [
|
|
1057
|
+
interval === undefined ? "completed" : "scheduled",
|
|
1058
|
+
nextRun,
|
|
1059
|
+
claimed.id,
|
|
1060
|
+
claimed.claimed_by,
|
|
1061
|
+
]);
|
|
1062
|
+
});
|
|
1063
|
+
}
|
|
1064
|
+
async releaseReminder(reminder) {
|
|
1065
|
+
await this.settings.database.transaction(async (connection) => {
|
|
1066
|
+
await connection.run(`UPDATE ${this.table("reminders")} SET claimed_by = NULL, claimed_at_ms = NULL
|
|
1067
|
+
WHERE id = ? AND claimed_by = ?`, [reminder.id, reminder.claimed_by]);
|
|
1068
|
+
});
|
|
1069
|
+
}
|
|
1070
|
+
async failReminder(reminder, error) {
|
|
1071
|
+
await this.settings.database.transaction(async (connection) => {
|
|
1072
|
+
const updated = await connection.run(`UPDATE ${this.table("reminders")}
|
|
1073
|
+
SET status = 'paused', claimed_by = NULL, claimed_at_ms = NULL, error = ?
|
|
1074
|
+
WHERE id = ? AND status = 'scheduled' AND claimed_by = ?`, [JSON.stringify(safeError(error)), reminder.id, reminder.claimed_by]);
|
|
1075
|
+
if (updated.changes !== 1)
|
|
1076
|
+
throw new LostActivation("reminder claim no longer matches");
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
1079
|
+
async claimBroadcast(processId) {
|
|
1080
|
+
return this.settings.database.transaction(async (connection) => {
|
|
1081
|
+
const now = await connection.nowMilliseconds();
|
|
1082
|
+
const staleAt = now - this.settings.processAliveThresholdMilliseconds;
|
|
1083
|
+
await connection.run(`UPDATE ${this.table("broadcasts")} SET status = 'pending', claimed_by = NULL
|
|
1084
|
+
WHERE status = 'processing' AND (
|
|
1085
|
+
claimed_by IS NULL OR NOT EXISTS (
|
|
1086
|
+
SELECT 1 FROM ${this.table("processes")} processes
|
|
1087
|
+
WHERE processes.id = ${this.table("broadcasts")}.claimed_by
|
|
1088
|
+
AND processes.shutdown_state = 'running' AND processes.heartbeat_at_ms > ?
|
|
1089
|
+
)
|
|
1090
|
+
)`, [staleAt]);
|
|
1091
|
+
const broadcast = await connection.get(`SELECT broadcasts.* FROM ${this.table("broadcasts")} broadcasts
|
|
1092
|
+
WHERE broadcasts.status = 'pending' AND broadcasts.available_at_ms <= ?
|
|
1093
|
+
AND NOT EXISTS (
|
|
1094
|
+
SELECT 1 FROM ${this.table("broadcasts")} earlier
|
|
1095
|
+
WHERE earlier.instance_id = broadcasts.instance_id
|
|
1096
|
+
AND earlier.state_revision < broadcasts.state_revision
|
|
1097
|
+
AND earlier.status IN ('pending', 'processing')
|
|
1098
|
+
)
|
|
1099
|
+
ORDER BY broadcasts.available_at_ms, broadcasts.id
|
|
1100
|
+
LIMIT 1`, [now]);
|
|
1101
|
+
if (!broadcast)
|
|
1102
|
+
return undefined;
|
|
1103
|
+
const claimed = await connection.run(`UPDATE ${this.table("broadcasts")}
|
|
1104
|
+
SET status = 'processing', claimed_by = ?, attempt_count = attempt_count + 1
|
|
1105
|
+
WHERE id = ? AND status = 'pending'`, [processId, broadcast.id]);
|
|
1106
|
+
if (claimed.changes !== 1)
|
|
1107
|
+
return undefined;
|
|
1108
|
+
broadcast.status = "processing";
|
|
1109
|
+
broadcast.claimed_by = processId;
|
|
1110
|
+
broadcast.attempt_count = Number(broadcast.attempt_count) + 1;
|
|
1111
|
+
return broadcast;
|
|
1112
|
+
});
|
|
1113
|
+
}
|
|
1114
|
+
async completeBroadcast(broadcast) {
|
|
1115
|
+
await this.settings.database.transaction(async (connection) => {
|
|
1116
|
+
const updated = await connection.run(`UPDATE ${this.table("broadcasts")}
|
|
1117
|
+
SET status = 'delivered', claimed_by = NULL, error = NULL
|
|
1118
|
+
WHERE id = ? AND status = 'processing' AND claimed_by = ?`, [broadcast.id, broadcast.claimed_by]);
|
|
1119
|
+
if (updated.changes !== 1)
|
|
1120
|
+
throw new LostActivation("broadcast claim no longer matches");
|
|
1121
|
+
});
|
|
1122
|
+
}
|
|
1123
|
+
async failBroadcast(broadcast, error) {
|
|
1124
|
+
await this.settings.database.transaction(async (connection) => {
|
|
1125
|
+
const now = await connection.nowMilliseconds();
|
|
1126
|
+
const exhausted = Number(broadcast.attempt_count) >= this.settings.maxAttempts;
|
|
1127
|
+
const status = exhausted ? "dead" : "pending";
|
|
1128
|
+
const availableAt = exhausted
|
|
1129
|
+
? Number(broadcast.available_at_ms)
|
|
1130
|
+
: now + this.settings.retryDelayMilliseconds(Number(broadcast.attempt_count));
|
|
1131
|
+
const updated = await connection.run(`UPDATE ${this.table("broadcasts")}
|
|
1132
|
+
SET status = ?, error = ?, available_at_ms = ?, claimed_by = NULL
|
|
1133
|
+
WHERE id = ? AND status = 'processing' AND claimed_by = ?`, [status, JSON.stringify(safeError(error)), availableAt, broadcast.id, broadcast.claimed_by]);
|
|
1134
|
+
if (updated.changes !== 1)
|
|
1135
|
+
throw new LostActivation("broadcast claim no longer matches");
|
|
1136
|
+
});
|
|
1137
|
+
}
|
|
1138
|
+
findInstance(options) {
|
|
1139
|
+
const { connection, actorType, actorId } = options;
|
|
1140
|
+
return connection.get(`SELECT * FROM ${this.table("instances")} WHERE actor_type = ? AND actor_id = ?${options.lock === "update" ? " FOR UPDATE" : ""}`, [actorType, actorId]);
|
|
1141
|
+
}
|
|
1142
|
+
async recoverExpiredClaims(connection, now) {
|
|
1143
|
+
const expired = await connection.all(`SELECT claimed.message_id, claimed.instance_id, claimed.sequence
|
|
1144
|
+
FROM ${this.table("claimed_messages")} claimed
|
|
1145
|
+
JOIN ${this.table("instances")} instances ON instances.id = claimed.instance_id
|
|
1146
|
+
WHERE instances.activation_expires_at_ms <= ?`, [now]);
|
|
1147
|
+
for (const claim of expired) {
|
|
1148
|
+
await connection.run(`DELETE FROM ${this.table("claimed_messages")} WHERE message_id = ?`, [
|
|
1149
|
+
claim.message_id,
|
|
1150
|
+
]);
|
|
1151
|
+
await connection.run(`INSERT INTO ${this.table("ready_messages")}(message_id, instance_id, sequence, available_at_ms)
|
|
1152
|
+
VALUES (?, ?, ?, ?) ON CONFLICT(message_id) DO NOTHING`, [claim.message_id, claim.instance_id, claim.sequence, now]);
|
|
1153
|
+
await connection.run(`UPDATE ${this.table("instances")}
|
|
1154
|
+
SET activation_owner_id = NULL, activation_token = NULL, activation_expires_at_ms = NULL,
|
|
1155
|
+
updated_at_ms = ? WHERE id = ?`, [now, claim.instance_id]);
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
async assertFence(connection, turn) {
|
|
1159
|
+
const now = await connection.nowMilliseconds();
|
|
1160
|
+
const fence = await connection.get(`SELECT 1 AS found FROM ${this.table("instances")} instances
|
|
1161
|
+
JOIN ${this.table("claimed_messages")} claimed ON claimed.instance_id = instances.id
|
|
1162
|
+
WHERE instances.id = ? AND instances.activation_owner_id = ? AND instances.activation_token = ?
|
|
1163
|
+
AND instances.activation_generation = ? AND instances.activation_expires_at_ms > ?
|
|
1164
|
+
AND claimed.message_id = ? AND claimed.process_id = ? AND claimed.activation_token = ?
|
|
1165
|
+
AND claimed.activation_generation = ?`, [
|
|
1166
|
+
turn.instance.id,
|
|
1167
|
+
turn.processId,
|
|
1168
|
+
turn.activationToken,
|
|
1169
|
+
turn.activationGeneration,
|
|
1170
|
+
now,
|
|
1171
|
+
turn.message.id,
|
|
1172
|
+
turn.processId,
|
|
1173
|
+
turn.activationToken,
|
|
1174
|
+
turn.activationGeneration,
|
|
1175
|
+
]);
|
|
1176
|
+
if (!fence)
|
|
1177
|
+
throw new LostActivation("activation fence no longer matches");
|
|
1178
|
+
return now;
|
|
1179
|
+
}
|
|
1180
|
+
async releaseClaim(options) {
|
|
1181
|
+
const { connection, turn } = options;
|
|
1182
|
+
await connection.run(`DELETE FROM ${this.table("claimed_messages")} WHERE message_id = ?`, [
|
|
1183
|
+
turn.message.id,
|
|
1184
|
+
]);
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
function nextReminderRun(options) {
|
|
1188
|
+
const { previousRun, interval, missedPolicy, now } = options;
|
|
1189
|
+
if (missedPolicy === "all")
|
|
1190
|
+
return previousRun + interval;
|
|
1191
|
+
if (previousRun > now)
|
|
1192
|
+
return previousRun;
|
|
1193
|
+
return previousRun + (Math.floor((now - previousRun) / interval) + 1) * interval;
|
|
1194
|
+
}
|
|
1195
|
+
function safeError(error) {
|
|
1196
|
+
if (error instanceof Error) {
|
|
1197
|
+
return jsonObject({ name: error.name, message: error.message });
|
|
1198
|
+
}
|
|
1199
|
+
return jsonObject({ name: "Error", message: String(normalizeJson(error)) });
|
|
1200
|
+
}
|
|
1201
|
+
function retentionPolicy(options) {
|
|
1202
|
+
const entries = Object.entries(options.overrides);
|
|
1203
|
+
const conditions = [];
|
|
1204
|
+
const parameters = [];
|
|
1205
|
+
for (const [actorType, retention] of entries) {
|
|
1206
|
+
conditions.push(`(${options.table}.actor_type = ? AND ${options.table}.${options.timestampColumn} < ?)`);
|
|
1207
|
+
parameters.push(actorType, options.now - retention);
|
|
1208
|
+
}
|
|
1209
|
+
if (options.defaultRetention !== undefined) {
|
|
1210
|
+
const exclusion = entries.length
|
|
1211
|
+
? `${options.table}.actor_type NOT IN (${parameterList(entries.length)}) AND `
|
|
1212
|
+
: "";
|
|
1213
|
+
conditions.push(`(${exclusion}${options.table}.${options.timestampColumn} < ?)`);
|
|
1214
|
+
if (entries.length)
|
|
1215
|
+
parameters.push(...entries.map(([actorType]) => actorType));
|
|
1216
|
+
parameters.push(options.now - options.defaultRetention);
|
|
1217
|
+
}
|
|
1218
|
+
return {
|
|
1219
|
+
sql: conditions.length === 0 ? "0 = 1" : `(${conditions.join(" OR ")})`,
|
|
1220
|
+
parameters,
|
|
1221
|
+
};
|
|
1222
|
+
}
|
|
1223
|
+
function parameterList(length) {
|
|
1224
|
+
return Array.from({ length }, () => "?").join(", ");
|
|
1225
|
+
}
|
|
1226
|
+
function retryableMySQLDeadlock(options) {
|
|
1227
|
+
if (options.database.family !== "mysql" || options.attempt >= 8)
|
|
1228
|
+
return false;
|
|
1229
|
+
if (typeof options.error !== "object" || options.error === null)
|
|
1230
|
+
return false;
|
|
1231
|
+
return options.error.code === "ER_LOCK_DEADLOCK";
|
|
1232
|
+
}
|
|
1233
|
+
async function transactionRetryBackoff(attempt) {
|
|
1234
|
+
const delay = 2 ** attempt + Math.floor(Math.random() * 5);
|
|
1235
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1236
|
+
}
|
|
1237
|
+
//# sourceMappingURL=repository.js.map
|