rastack 0.0.16 → 0.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43874 -43013
- package/dist/.prettierrc +7 -7
- package/dist/compile/analyze.js +19 -11
- package/dist/compile/index.js +26 -17
- package/dist/compile/manifest.js +2 -3
- package/dist/compile/openapi.js +5 -8
- package/dist/compile/program.js +19 -10
- package/dist/csv-schema.js +5 -6
- package/dist/define/index.js +2 -2
- package/dist/entity-generation/delete-method.js +8 -10
- package/dist/entity-generation/form.js +80 -82
- package/dist/entity-generation/get-method.js +35 -36
- package/dist/entity-generation/imports.js +24 -20
- package/dist/entity-generation/sync-method.d.ts +1 -1
- package/dist/entity-generation/sync-method.js +45 -46
- package/dist/entity-generation/update-method.js +9 -10
- package/dist/rad-wasm-build.d.ts +1 -1
- package/dist/rad-wasm-build.js +15 -6
- package/dist/rad.js +9 -2
- package/dist/schema/camel-to-pastel.js +1 -2
- package/dist/schema/capitalise-first-letter.js +1 -2
- package/dist/schema/extract-response.js +3 -4
- package/dist/schema/fetch-schema.js +4 -16
- package/dist/schema/remove-non-model-paths.js +1 -2
- package/dist/schema/to-camel-case.js +1 -2
- package/dist/schema/to-pastel-case.js +1 -2
- package/dist/schema-convert.js +22 -33
- package/dist/schema-entities.js +177 -189
- package/dist/schema-fetch.js +46 -63
- package/dist/schema-full.js +23 -36
- package/dist/schema-index.js +16 -27
- package/dist/schema-params.js +11 -22
- package/dist/seed.js +95 -97
- package/hooks/form/form.ts +207 -207
- package/hooks/form/index.ts +8 -8
- package/hooks/form/interfaces.ts +217 -217
- package/hooks/form/structure.ts +39 -39
- package/hooks/form/validate-schema.ts +49 -49
- package/hooks/index.ts +3 -3
- package/hooks/query/api.ts +42 -42
- package/hooks/query/delete.ts +45 -45
- package/hooks/query/fetch.ts +48 -48
- package/hooks/query/index.ts +21 -21
- package/hooks/query/interfaces.ts +111 -111
- package/hooks/query/list.ts +286 -286
- package/hooks/query/update.ts +88 -88
- package/hooks/query/url.ts +31 -31
- package/hooks/real-time/index.ts +1 -1
- package/hooks/real-time/pusher.ts +43 -43
- package/jest.config.cjs +6 -6
- package/package.json +57 -57
- package/provider/index.ts +13 -13
- package/provider/provider.tsx +187 -187
- package/provider/types.ts +114 -114
- package/provider/warehouse.ts +167 -167
- package/provider/wasm.ts +124 -124
- package/runtime.ts +3 -3
- package/src/.prettierrc +7 -7
- package/src/compile/analyze.ts +224 -224
- package/src/compile/index.ts +86 -86
- package/src/compile/manifest.ts +10 -10
- package/src/compile/model.ts +69 -69
- package/src/compile/openapi.ts +266 -266
- package/src/compile/program.ts +40 -40
- package/src/csv-schema.ts +187 -187
- package/src/define/index.ts +162 -162
- package/src/entity-generation/delete-method.ts +37 -37
- package/src/entity-generation/form.ts +169 -169
- package/src/entity-generation/get-method.ts +80 -80
- package/src/entity-generation/imports.ts +110 -110
- package/src/entity-generation/sync-method.ts +222 -222
- package/src/entity-generation/update-method.ts +38 -38
- package/src/index.ts +77 -77
- package/src/rad-compile.ts +88 -88
- package/src/rad-wasm-build.ts +102 -102
- package/src/rad.ts +101 -101
- package/src/scan.ts +89 -89
- package/src/schema/camel-to-pastel.ts +3 -3
- package/src/schema/capitalise-first-letter.ts +3 -3
- package/src/schema/extract-response.ts +38 -38
- package/src/schema/fetch-schema.ts +12 -12
- package/src/schema/remove-non-model-paths.ts +13 -13
- package/src/schema/to-camel-case.ts +7 -7
- package/src/schema/to-pastel-case.ts +6 -6
- package/src/schema-convert.ts +61 -61
- package/src/schema-entities.ts +401 -401
- package/src/schema-fetch.ts +82 -82
- package/src/schema-full.ts +38 -38
- package/src/schema-index.ts +29 -29
- package/src/schema-params.ts +116 -116
- package/src/seed.ts +297 -297
- package/sync/engine.ts +392 -392
- package/sync/hooks.ts +450 -450
- package/sync/index.ts +8 -8
- package/sync/persistence.ts +237 -237
- package/sync/provider.tsx +91 -91
- package/sync/registry.ts +36 -36
- package/sync/store.ts +126 -126
- package/sync/transactions.ts +300 -300
- package/sync/types.ts +94 -94
- package/test/compile.spec.ts +192 -192
- package/test/csv-schema.spec.ts +143 -143
- package/test/schema-entities.spec.ts +688 -688
- package/tsconfig.json +24 -24
- package/types.ts +18 -18
package/sync/engine.ts
CHANGED
|
@@ -1,392 +1,392 @@
|
|
|
1
|
-
import { AxiosInstance } from "axios";
|
|
2
|
-
import { createPersistence, SyncPersistence } from "./persistence";
|
|
3
|
-
import { getEntity, getRegisteredEntities } from "./registry";
|
|
4
|
-
import { SyncStore } from "./store";
|
|
5
|
-
import { TransactionQueue } from "./transactions";
|
|
6
|
-
import {
|
|
7
|
-
BootstrapResponse,
|
|
8
|
-
ChangesResponse,
|
|
9
|
-
EngineStatus,
|
|
10
|
-
EntityDescriptor,
|
|
11
|
-
EntityKey,
|
|
12
|
-
entityKey,
|
|
13
|
-
SyncActionPacket,
|
|
14
|
-
SyncEngineConfig,
|
|
15
|
-
} from "./types";
|
|
16
|
-
|
|
17
|
-
const CURSOR_META_KEY = "lastSyncId";
|
|
18
|
-
const SCHEMA_META_KEY = "schemaHash";
|
|
19
|
-
const READY_META_KEY = "readyEntities";
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* The sync engine. Read path: local hydrate -> delta catch-up -> full
|
|
23
|
-
* bootstrap (only when there is no usable local state), then a long-poll
|
|
24
|
-
* `/changes` loop for the app's lifetime. Lazy entities are excluded from
|
|
25
|
-
* the initial bootstrap and partial-bootstrapped on first use. Write path:
|
|
26
|
-
* the TransactionQueue replays optimistic writes through the generated REST
|
|
27
|
-
* endpoints; deltas arriving here rebase and resolve those transactions.
|
|
28
|
-
* A server schema-hash change or a cursor past the server's change-log
|
|
29
|
-
* retention (HTTP 410) drops local state and re-bootstraps.
|
|
30
|
-
*/
|
|
31
|
-
export class SyncEngine {
|
|
32
|
-
readonly store: SyncStore;
|
|
33
|
-
readonly queue: TransactionQueue;
|
|
34
|
-
readonly persistence: SyncPersistence;
|
|
35
|
-
|
|
36
|
-
readonly api: AxiosInstance;
|
|
37
|
-
private syncBasePath: string;
|
|
38
|
-
private wait: number;
|
|
39
|
-
private onError?: (error: unknown) => void;
|
|
40
|
-
|
|
41
|
-
private status: EngineStatus = "idle";
|
|
42
|
-
private statusListeners = new Set<() => void>();
|
|
43
|
-
private running = false;
|
|
44
|
-
private lastSyncId: number | null = null;
|
|
45
|
-
private schemaHash: string | undefined;
|
|
46
|
-
private readyKeys = new Set<EntityKey>();
|
|
47
|
-
private inflightBootstraps = new Map<EntityKey, Promise<void>>();
|
|
48
|
-
private pendingEnsures = new Set<EntityKey>();
|
|
49
|
-
private applyLock: Promise<void> = Promise.resolve();
|
|
50
|
-
private pollController: AbortController | null = null;
|
|
51
|
-
private onOnline = () => this.wake();
|
|
52
|
-
|
|
53
|
-
constructor(api: AxiosInstance, config: SyncEngineConfig = {}) {
|
|
54
|
-
this.api = api;
|
|
55
|
-
this.syncBasePath = config.syncBasePath || "/api/sync/v1";
|
|
56
|
-
this.wait = config.wait ?? 25;
|
|
57
|
-
this.onError = config.onError;
|
|
58
|
-
this.store = new SyncStore();
|
|
59
|
-
this.persistence = createPersistence(
|
|
60
|
-
config.persist !== false,
|
|
61
|
-
config.databaseName || "rad-sync",
|
|
62
|
-
);
|
|
63
|
-
this.queue = new TransactionQueue({
|
|
64
|
-
api,
|
|
65
|
-
persistence: this.persistence,
|
|
66
|
-
store: this.store,
|
|
67
|
-
getEntity: (key) => getEntity(key),
|
|
68
|
-
onError: config.onError
|
|
69
|
-
? (error) => config.onError!(error)
|
|
70
|
-
: undefined,
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
getStatus(): EngineStatus {
|
|
75
|
-
return this.status;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
subscribeStatus(listener: () => void): () => void {
|
|
79
|
-
this.statusListeners.add(listener);
|
|
80
|
-
return () => {
|
|
81
|
-
this.statusListeners.delete(listener);
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
async start(): Promise<void> {
|
|
86
|
-
if (this.running) {
|
|
87
|
-
return;
|
|
88
|
-
}
|
|
89
|
-
this.running = true;
|
|
90
|
-
if (typeof window !== "undefined") {
|
|
91
|
-
window.addEventListener("online", this.onOnline);
|
|
92
|
-
}
|
|
93
|
-
try {
|
|
94
|
-
this.setStatus("hydrating");
|
|
95
|
-
const cursor = await this.persistence.getMeta(CURSOR_META_KEY);
|
|
96
|
-
this.schemaHash = await this.persistence.getMeta(SCHEMA_META_KEY);
|
|
97
|
-
const readyList: EntityKey[] =
|
|
98
|
-
(await this.persistence.getMeta(READY_META_KEY)) || [];
|
|
99
|
-
this.readyKeys = new Set(readyList);
|
|
100
|
-
if (typeof cursor === "number") {
|
|
101
|
-
this.lastSyncId = cursor;
|
|
102
|
-
await this.hydrateFromLocal();
|
|
103
|
-
}
|
|
104
|
-
await this.queue.resume();
|
|
105
|
-
|
|
106
|
-
const instant = this.entities().filter((entity) => !entity.lazy);
|
|
107
|
-
const missing = instant
|
|
108
|
-
.map(entityKey)
|
|
109
|
-
.filter((key) => !this.readyKeys.has(key));
|
|
110
|
-
if (this.lastSyncId === null || missing.length > 0) {
|
|
111
|
-
this.setStatus("bootstrapping");
|
|
112
|
-
await this.bootstrapEntities(
|
|
113
|
-
this.lastSyncId === null ? instant.map(entityKey) : missing,
|
|
114
|
-
);
|
|
115
|
-
}
|
|
116
|
-
if (this.pendingEnsures.size > 0) {
|
|
117
|
-
const ensures = Array.from(this.pendingEnsures);
|
|
118
|
-
this.pendingEnsures.clear();
|
|
119
|
-
await this.bootstrapEntities(
|
|
120
|
-
ensures.filter((key) => !this.readyKeys.has(key)),
|
|
121
|
-
);
|
|
122
|
-
}
|
|
123
|
-
void this.changesLoop();
|
|
124
|
-
} catch (error) {
|
|
125
|
-
this.setStatus("error");
|
|
126
|
-
this.onError?.(error);
|
|
127
|
-
throw error;
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
stop(): void {
|
|
132
|
-
this.running = false;
|
|
133
|
-
this.pollController?.abort();
|
|
134
|
-
if (typeof window !== "undefined") {
|
|
135
|
-
window.removeEventListener("online", this.onOnline);
|
|
136
|
-
}
|
|
137
|
-
this.setStatus("idle");
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Kick the engine: cancels the in-flight long poll so a fresh `/changes`
|
|
142
|
-
* request fires immediately, and retries any queued writes. This is the
|
|
143
|
-
* hook a push transport (Pusher, API Gateway WebSocket) calls — push only
|
|
144
|
-
* wakes the client, the HTTP protocol stays the source of truth.
|
|
145
|
-
*/
|
|
146
|
-
wake(): void {
|
|
147
|
-
this.queue.wake();
|
|
148
|
-
this.pollController?.abort();
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
/**
|
|
152
|
-
* Make sure an entity's data is (being) loaded. No-op once it's ready;
|
|
153
|
-
* for lazy entities this performs the on-demand partial bootstrap. Read
|
|
154
|
-
* hooks call this on mount, so lazy data loads the first time a component
|
|
155
|
-
* asks for it (and late-registered entities still get synced).
|
|
156
|
-
*/
|
|
157
|
-
ensureEntity(key: EntityKey): Promise<void> {
|
|
158
|
-
if (this.readyKeys.has(key) || !getEntity(key)) {
|
|
159
|
-
return Promise.resolve();
|
|
160
|
-
}
|
|
161
|
-
if (!this.running) {
|
|
162
|
-
// Requested before start() — remembered and loaded right after.
|
|
163
|
-
this.pendingEnsures.add(key);
|
|
164
|
-
return Promise.resolve();
|
|
165
|
-
}
|
|
166
|
-
return this.bootstrapEntities([key]);
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
isEntityReady(key: EntityKey): boolean {
|
|
170
|
-
return this.readyKeys.has(key);
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
/** Drop local state and re-sync from scratch. */
|
|
174
|
-
async reset(): Promise<void> {
|
|
175
|
-
this.lastSyncId = null;
|
|
176
|
-
this.schemaHash = undefined;
|
|
177
|
-
this.readyKeys = new Set();
|
|
178
|
-
this.inflightBootstraps.clear();
|
|
179
|
-
await this.persistence.clear();
|
|
180
|
-
this.store.clear();
|
|
181
|
-
this.setStatus("bootstrapping");
|
|
182
|
-
// Instant entities reload now; lazy ones re-fetch on next use.
|
|
183
|
-
await this.bootstrapEntities(
|
|
184
|
-
this.entities()
|
|
185
|
-
.filter((entity) => !entity.lazy)
|
|
186
|
-
.map(entityKey),
|
|
187
|
-
);
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
private setStatus(status: EngineStatus): void {
|
|
191
|
-
if (this.status !== status) {
|
|
192
|
-
this.status = status;
|
|
193
|
-
this.statusListeners.forEach((listener) => listener());
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
private entities(): EntityDescriptor[] {
|
|
198
|
-
return getRegisteredEntities();
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
private async hydrateFromLocal(): Promise<void> {
|
|
202
|
-
for (const entity of this.entities()) {
|
|
203
|
-
const key = entityKey(entity);
|
|
204
|
-
if (!this.readyKeys.has(key)) {
|
|
205
|
-
continue; // never bootstrapped (e.g. lazy and untouched) — no local truth
|
|
206
|
-
}
|
|
207
|
-
const rows = await this.persistence.getRecords(key);
|
|
208
|
-
this.store.hydrate(key, rows, entity.idField || "id");
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
/** Bootstrap (or partial-bootstrap) the given entities from the server. */
|
|
213
|
-
private bootstrapEntities(keys: EntityKey[]): Promise<void> {
|
|
214
|
-
const wanted = keys.filter(
|
|
215
|
-
(key) => !this.readyKeys.has(key) && !this.inflightBootstraps.has(key),
|
|
216
|
-
);
|
|
217
|
-
if (wanted.length === 0) {
|
|
218
|
-
const inflight = keys
|
|
219
|
-
.map((key) => this.inflightBootstraps.get(key))
|
|
220
|
-
.filter(Boolean) as Promise<void>[];
|
|
221
|
-
return Promise.all(inflight).then(() => undefined);
|
|
222
|
-
}
|
|
223
|
-
const run = this.fetchBootstrap(wanted).finally(() => {
|
|
224
|
-
wanted.forEach((key) => this.inflightBootstraps.delete(key));
|
|
225
|
-
});
|
|
226
|
-
wanted.forEach((key) => this.inflightBootstraps.set(key, run));
|
|
227
|
-
return run;
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
private async fetchBootstrap(keys: EntityKey[]): Promise<void> {
|
|
231
|
-
if (keys.length === 0) {
|
|
232
|
-
return;
|
|
233
|
-
}
|
|
234
|
-
const response = await this.api.get<BootstrapResponse>(
|
|
235
|
-
`${this.syncBasePath}/bootstrap/`,
|
|
236
|
-
{ params: { models: keys.join(",") } },
|
|
237
|
-
);
|
|
238
|
-
const payload = response.data;
|
|
239
|
-
for (const key of keys) {
|
|
240
|
-
const entity = getEntity(key);
|
|
241
|
-
if (!entity) {
|
|
242
|
-
continue;
|
|
243
|
-
}
|
|
244
|
-
const idField = entity.idField || "id";
|
|
245
|
-
const rows = payload.models[key] || [];
|
|
246
|
-
await this.persistence.replaceRecords(
|
|
247
|
-
key,
|
|
248
|
-
rows.map((row) => ({ id: String(row[idField]), data: row })),
|
|
249
|
-
);
|
|
250
|
-
this.store.hydrate(key, rows, idField);
|
|
251
|
-
this.readyKeys.add(key);
|
|
252
|
-
}
|
|
253
|
-
await this.persistence.setMeta(READY_META_KEY, Array.from(this.readyKeys));
|
|
254
|
-
|
|
255
|
-
// The cursor only ever advances: a partial bootstrap snapshot is at
|
|
256
|
-
// least as new as any cursor we already hold.
|
|
257
|
-
if (this.lastSyncId === null || payload.last_sync_id > this.lastSyncId) {
|
|
258
|
-
this.lastSyncId = payload.last_sync_id;
|
|
259
|
-
await this.persistence.setMeta(CURSOR_META_KEY, this.lastSyncId);
|
|
260
|
-
}
|
|
261
|
-
if (payload.schema_hash) {
|
|
262
|
-
this.schemaHash = payload.schema_hash;
|
|
263
|
-
await this.persistence.setMeta(SCHEMA_META_KEY, this.schemaHash);
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
private async changesLoop(): Promise<void> {
|
|
268
|
-
let backoff = 1000;
|
|
269
|
-
while (this.running) {
|
|
270
|
-
this.pollController = new AbortController();
|
|
271
|
-
try {
|
|
272
|
-
const response = await this.api.get<ChangesResponse>(
|
|
273
|
-
`${this.syncBasePath}/changes/`,
|
|
274
|
-
{
|
|
275
|
-
params: { since: this.lastSyncId ?? 0, wait: this.wait },
|
|
276
|
-
signal: this.pollController.signal,
|
|
277
|
-
},
|
|
278
|
-
);
|
|
279
|
-
const changed = await this.checkSchemaHash(
|
|
280
|
-
response.data.schema_hash,
|
|
281
|
-
);
|
|
282
|
-
if (!changed) {
|
|
283
|
-
await this.applyChanges(response.data);
|
|
284
|
-
}
|
|
285
|
-
this.setStatus("live");
|
|
286
|
-
backoff = 1000;
|
|
287
|
-
} catch (error: any) {
|
|
288
|
-
if (!this.running) {
|
|
289
|
-
break;
|
|
290
|
-
}
|
|
291
|
-
if (this.pollController.signal.aborted) {
|
|
292
|
-
// wake() cancelled the poll on purpose — go straight around.
|
|
293
|
-
continue;
|
|
294
|
-
}
|
|
295
|
-
const status = error?.response?.status;
|
|
296
|
-
if (status === 410) {
|
|
297
|
-
// Cursor fell out of the server's change-log retention.
|
|
298
|
-
try {
|
|
299
|
-
await this.reset();
|
|
300
|
-
continue;
|
|
301
|
-
} catch (resetError) {
|
|
302
|
-
this.onError?.(resetError);
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
this.setStatus("offline");
|
|
306
|
-
this.onError?.(error);
|
|
307
|
-
await new Promise((resolve) => setTimeout(resolve, backoff));
|
|
308
|
-
backoff = Math.min(backoff * 2, 30000);
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
/**
|
|
314
|
-
* Returns true (after resetting) when the server's schema hash no longer
|
|
315
|
-
* matches the one local state was built against.
|
|
316
|
-
*/
|
|
317
|
-
private async checkSchemaHash(hash: string | undefined): Promise<boolean> {
|
|
318
|
-
if (!hash) {
|
|
319
|
-
return false;
|
|
320
|
-
}
|
|
321
|
-
if (this.schemaHash === undefined) {
|
|
322
|
-
this.schemaHash = hash;
|
|
323
|
-
await this.persistence.setMeta(SCHEMA_META_KEY, hash);
|
|
324
|
-
return false;
|
|
325
|
-
}
|
|
326
|
-
if (this.schemaHash === hash) {
|
|
327
|
-
return false;
|
|
328
|
-
}
|
|
329
|
-
await this.reset();
|
|
330
|
-
return true;
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
private applyChanges(payload: ChangesResponse): Promise<void> {
|
|
334
|
-
// Serialize applications so a wake-triggered poll can never interleave
|
|
335
|
-
// with one still being written.
|
|
336
|
-
this.applyLock = this.applyLock.then(() =>
|
|
337
|
-
this.applyChangesExclusive(payload),
|
|
338
|
-
);
|
|
339
|
-
return this.applyLock;
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
private async applyChangesExclusive(payload: ChangesResponse): Promise<void> {
|
|
343
|
-
for (const action of payload.actions) {
|
|
344
|
-
await this.applyAction(action);
|
|
345
|
-
}
|
|
346
|
-
if (payload.last_sync_id > (this.lastSyncId ?? 0)) {
|
|
347
|
-
this.lastSyncId = payload.last_sync_id;
|
|
348
|
-
await this.persistence.setMeta(CURSOR_META_KEY, this.lastSyncId);
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
private async applyAction(action: SyncActionPacket): Promise<void> {
|
|
353
|
-
const entity = getEntity(action.model);
|
|
354
|
-
if (!entity) {
|
|
355
|
-
return; // an entity this bundle doesn't use
|
|
356
|
-
}
|
|
357
|
-
const key = action.model;
|
|
358
|
-
if (entity.lazy && !this.readyKeys.has(key)) {
|
|
359
|
-
return; // not loaded yet — the on-demand bootstrap will include it
|
|
360
|
-
}
|
|
361
|
-
const idField = entity.idField || "id";
|
|
362
|
-
const id = String(action.id);
|
|
363
|
-
const deleted =
|
|
364
|
-
action.op === "D" || (action.data && action.data.deleted_at != null);
|
|
365
|
-
|
|
366
|
-
if (deleted) {
|
|
367
|
-
await this.persistence.deleteRecord(key, id);
|
|
368
|
-
this.store.remove(key, id);
|
|
369
|
-
} else if (action.data) {
|
|
370
|
-
await this.persistence.putRecords(key, [{ id, data: action.data }]);
|
|
371
|
-
const pending = this.queue.pendingFor(key, id);
|
|
372
|
-
if (pending.some((txn) => txn.op === "delete")) {
|
|
373
|
-
// Locally deleted, not yet confirmed: keep it gone from the UI.
|
|
374
|
-
} else if (pending.length > 0) {
|
|
375
|
-
// Rebase: server row underneath, unconfirmed local edits on top
|
|
376
|
-
// (last-writer-wins — the local write supersedes on its next send).
|
|
377
|
-
const overlay = pending.reduce(
|
|
378
|
-
(acc, txn) => ({ ...acc, ...(txn.payload || {}) }),
|
|
379
|
-
{} as Record<string, any>,
|
|
380
|
-
);
|
|
381
|
-
this.store.upsert(key, id, {
|
|
382
|
-
...action.data,
|
|
383
|
-
...overlay,
|
|
384
|
-
[idField]: action.data[idField],
|
|
385
|
-
});
|
|
386
|
-
} else {
|
|
387
|
-
this.store.upsert(key, id, action.data);
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
await this.queue.resolveForRow(key, id);
|
|
391
|
-
}
|
|
392
|
-
}
|
|
1
|
+
import { AxiosInstance } from "axios";
|
|
2
|
+
import { createPersistence, SyncPersistence } from "./persistence";
|
|
3
|
+
import { getEntity, getRegisteredEntities } from "./registry";
|
|
4
|
+
import { SyncStore } from "./store";
|
|
5
|
+
import { TransactionQueue } from "./transactions";
|
|
6
|
+
import {
|
|
7
|
+
BootstrapResponse,
|
|
8
|
+
ChangesResponse,
|
|
9
|
+
EngineStatus,
|
|
10
|
+
EntityDescriptor,
|
|
11
|
+
EntityKey,
|
|
12
|
+
entityKey,
|
|
13
|
+
SyncActionPacket,
|
|
14
|
+
SyncEngineConfig,
|
|
15
|
+
} from "./types";
|
|
16
|
+
|
|
17
|
+
const CURSOR_META_KEY = "lastSyncId";
|
|
18
|
+
const SCHEMA_META_KEY = "schemaHash";
|
|
19
|
+
const READY_META_KEY = "readyEntities";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* The sync engine. Read path: local hydrate -> delta catch-up -> full
|
|
23
|
+
* bootstrap (only when there is no usable local state), then a long-poll
|
|
24
|
+
* `/changes` loop for the app's lifetime. Lazy entities are excluded from
|
|
25
|
+
* the initial bootstrap and partial-bootstrapped on first use. Write path:
|
|
26
|
+
* the TransactionQueue replays optimistic writes through the generated REST
|
|
27
|
+
* endpoints; deltas arriving here rebase and resolve those transactions.
|
|
28
|
+
* A server schema-hash change or a cursor past the server's change-log
|
|
29
|
+
* retention (HTTP 410) drops local state and re-bootstraps.
|
|
30
|
+
*/
|
|
31
|
+
export class SyncEngine {
|
|
32
|
+
readonly store: SyncStore;
|
|
33
|
+
readonly queue: TransactionQueue;
|
|
34
|
+
readonly persistence: SyncPersistence;
|
|
35
|
+
|
|
36
|
+
readonly api: AxiosInstance;
|
|
37
|
+
private syncBasePath: string;
|
|
38
|
+
private wait: number;
|
|
39
|
+
private onError?: (error: unknown) => void;
|
|
40
|
+
|
|
41
|
+
private status: EngineStatus = "idle";
|
|
42
|
+
private statusListeners = new Set<() => void>();
|
|
43
|
+
private running = false;
|
|
44
|
+
private lastSyncId: number | null = null;
|
|
45
|
+
private schemaHash: string | undefined;
|
|
46
|
+
private readyKeys = new Set<EntityKey>();
|
|
47
|
+
private inflightBootstraps = new Map<EntityKey, Promise<void>>();
|
|
48
|
+
private pendingEnsures = new Set<EntityKey>();
|
|
49
|
+
private applyLock: Promise<void> = Promise.resolve();
|
|
50
|
+
private pollController: AbortController | null = null;
|
|
51
|
+
private onOnline = () => this.wake();
|
|
52
|
+
|
|
53
|
+
constructor(api: AxiosInstance, config: SyncEngineConfig = {}) {
|
|
54
|
+
this.api = api;
|
|
55
|
+
this.syncBasePath = config.syncBasePath || "/api/sync/v1";
|
|
56
|
+
this.wait = config.wait ?? 25;
|
|
57
|
+
this.onError = config.onError;
|
|
58
|
+
this.store = new SyncStore();
|
|
59
|
+
this.persistence = createPersistence(
|
|
60
|
+
config.persist !== false,
|
|
61
|
+
config.databaseName || "rad-sync",
|
|
62
|
+
);
|
|
63
|
+
this.queue = new TransactionQueue({
|
|
64
|
+
api,
|
|
65
|
+
persistence: this.persistence,
|
|
66
|
+
store: this.store,
|
|
67
|
+
getEntity: (key) => getEntity(key),
|
|
68
|
+
onError: config.onError
|
|
69
|
+
? (error) => config.onError!(error)
|
|
70
|
+
: undefined,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
getStatus(): EngineStatus {
|
|
75
|
+
return this.status;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
subscribeStatus(listener: () => void): () => void {
|
|
79
|
+
this.statusListeners.add(listener);
|
|
80
|
+
return () => {
|
|
81
|
+
this.statusListeners.delete(listener);
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async start(): Promise<void> {
|
|
86
|
+
if (this.running) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
this.running = true;
|
|
90
|
+
if (typeof window !== "undefined") {
|
|
91
|
+
window.addEventListener("online", this.onOnline);
|
|
92
|
+
}
|
|
93
|
+
try {
|
|
94
|
+
this.setStatus("hydrating");
|
|
95
|
+
const cursor = await this.persistence.getMeta(CURSOR_META_KEY);
|
|
96
|
+
this.schemaHash = await this.persistence.getMeta(SCHEMA_META_KEY);
|
|
97
|
+
const readyList: EntityKey[] =
|
|
98
|
+
(await this.persistence.getMeta(READY_META_KEY)) || [];
|
|
99
|
+
this.readyKeys = new Set(readyList);
|
|
100
|
+
if (typeof cursor === "number") {
|
|
101
|
+
this.lastSyncId = cursor;
|
|
102
|
+
await this.hydrateFromLocal();
|
|
103
|
+
}
|
|
104
|
+
await this.queue.resume();
|
|
105
|
+
|
|
106
|
+
const instant = this.entities().filter((entity) => !entity.lazy);
|
|
107
|
+
const missing = instant
|
|
108
|
+
.map(entityKey)
|
|
109
|
+
.filter((key) => !this.readyKeys.has(key));
|
|
110
|
+
if (this.lastSyncId === null || missing.length > 0) {
|
|
111
|
+
this.setStatus("bootstrapping");
|
|
112
|
+
await this.bootstrapEntities(
|
|
113
|
+
this.lastSyncId === null ? instant.map(entityKey) : missing,
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
if (this.pendingEnsures.size > 0) {
|
|
117
|
+
const ensures = Array.from(this.pendingEnsures);
|
|
118
|
+
this.pendingEnsures.clear();
|
|
119
|
+
await this.bootstrapEntities(
|
|
120
|
+
ensures.filter((key) => !this.readyKeys.has(key)),
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
void this.changesLoop();
|
|
124
|
+
} catch (error) {
|
|
125
|
+
this.setStatus("error");
|
|
126
|
+
this.onError?.(error);
|
|
127
|
+
throw error;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
stop(): void {
|
|
132
|
+
this.running = false;
|
|
133
|
+
this.pollController?.abort();
|
|
134
|
+
if (typeof window !== "undefined") {
|
|
135
|
+
window.removeEventListener("online", this.onOnline);
|
|
136
|
+
}
|
|
137
|
+
this.setStatus("idle");
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Kick the engine: cancels the in-flight long poll so a fresh `/changes`
|
|
142
|
+
* request fires immediately, and retries any queued writes. This is the
|
|
143
|
+
* hook a push transport (Pusher, API Gateway WebSocket) calls — push only
|
|
144
|
+
* wakes the client, the HTTP protocol stays the source of truth.
|
|
145
|
+
*/
|
|
146
|
+
wake(): void {
|
|
147
|
+
this.queue.wake();
|
|
148
|
+
this.pollController?.abort();
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Make sure an entity's data is (being) loaded. No-op once it's ready;
|
|
153
|
+
* for lazy entities this performs the on-demand partial bootstrap. Read
|
|
154
|
+
* hooks call this on mount, so lazy data loads the first time a component
|
|
155
|
+
* asks for it (and late-registered entities still get synced).
|
|
156
|
+
*/
|
|
157
|
+
ensureEntity(key: EntityKey): Promise<void> {
|
|
158
|
+
if (this.readyKeys.has(key) || !getEntity(key)) {
|
|
159
|
+
return Promise.resolve();
|
|
160
|
+
}
|
|
161
|
+
if (!this.running) {
|
|
162
|
+
// Requested before start() — remembered and loaded right after.
|
|
163
|
+
this.pendingEnsures.add(key);
|
|
164
|
+
return Promise.resolve();
|
|
165
|
+
}
|
|
166
|
+
return this.bootstrapEntities([key]);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
isEntityReady(key: EntityKey): boolean {
|
|
170
|
+
return this.readyKeys.has(key);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** Drop local state and re-sync from scratch. */
|
|
174
|
+
async reset(): Promise<void> {
|
|
175
|
+
this.lastSyncId = null;
|
|
176
|
+
this.schemaHash = undefined;
|
|
177
|
+
this.readyKeys = new Set();
|
|
178
|
+
this.inflightBootstraps.clear();
|
|
179
|
+
await this.persistence.clear();
|
|
180
|
+
this.store.clear();
|
|
181
|
+
this.setStatus("bootstrapping");
|
|
182
|
+
// Instant entities reload now; lazy ones re-fetch on next use.
|
|
183
|
+
await this.bootstrapEntities(
|
|
184
|
+
this.entities()
|
|
185
|
+
.filter((entity) => !entity.lazy)
|
|
186
|
+
.map(entityKey),
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private setStatus(status: EngineStatus): void {
|
|
191
|
+
if (this.status !== status) {
|
|
192
|
+
this.status = status;
|
|
193
|
+
this.statusListeners.forEach((listener) => listener());
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
private entities(): EntityDescriptor[] {
|
|
198
|
+
return getRegisteredEntities();
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
private async hydrateFromLocal(): Promise<void> {
|
|
202
|
+
for (const entity of this.entities()) {
|
|
203
|
+
const key = entityKey(entity);
|
|
204
|
+
if (!this.readyKeys.has(key)) {
|
|
205
|
+
continue; // never bootstrapped (e.g. lazy and untouched) — no local truth
|
|
206
|
+
}
|
|
207
|
+
const rows = await this.persistence.getRecords(key);
|
|
208
|
+
this.store.hydrate(key, rows, entity.idField || "id");
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** Bootstrap (or partial-bootstrap) the given entities from the server. */
|
|
213
|
+
private bootstrapEntities(keys: EntityKey[]): Promise<void> {
|
|
214
|
+
const wanted = keys.filter(
|
|
215
|
+
(key) => !this.readyKeys.has(key) && !this.inflightBootstraps.has(key),
|
|
216
|
+
);
|
|
217
|
+
if (wanted.length === 0) {
|
|
218
|
+
const inflight = keys
|
|
219
|
+
.map((key) => this.inflightBootstraps.get(key))
|
|
220
|
+
.filter(Boolean) as Promise<void>[];
|
|
221
|
+
return Promise.all(inflight).then(() => undefined);
|
|
222
|
+
}
|
|
223
|
+
const run = this.fetchBootstrap(wanted).finally(() => {
|
|
224
|
+
wanted.forEach((key) => this.inflightBootstraps.delete(key));
|
|
225
|
+
});
|
|
226
|
+
wanted.forEach((key) => this.inflightBootstraps.set(key, run));
|
|
227
|
+
return run;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
private async fetchBootstrap(keys: EntityKey[]): Promise<void> {
|
|
231
|
+
if (keys.length === 0) {
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
const response = await this.api.get<BootstrapResponse>(
|
|
235
|
+
`${this.syncBasePath}/bootstrap/`,
|
|
236
|
+
{ params: { models: keys.join(",") } },
|
|
237
|
+
);
|
|
238
|
+
const payload = response.data;
|
|
239
|
+
for (const key of keys) {
|
|
240
|
+
const entity = getEntity(key);
|
|
241
|
+
if (!entity) {
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
const idField = entity.idField || "id";
|
|
245
|
+
const rows = payload.models[key] || [];
|
|
246
|
+
await this.persistence.replaceRecords(
|
|
247
|
+
key,
|
|
248
|
+
rows.map((row) => ({ id: String(row[idField]), data: row })),
|
|
249
|
+
);
|
|
250
|
+
this.store.hydrate(key, rows, idField);
|
|
251
|
+
this.readyKeys.add(key);
|
|
252
|
+
}
|
|
253
|
+
await this.persistence.setMeta(READY_META_KEY, Array.from(this.readyKeys));
|
|
254
|
+
|
|
255
|
+
// The cursor only ever advances: a partial bootstrap snapshot is at
|
|
256
|
+
// least as new as any cursor we already hold.
|
|
257
|
+
if (this.lastSyncId === null || payload.last_sync_id > this.lastSyncId) {
|
|
258
|
+
this.lastSyncId = payload.last_sync_id;
|
|
259
|
+
await this.persistence.setMeta(CURSOR_META_KEY, this.lastSyncId);
|
|
260
|
+
}
|
|
261
|
+
if (payload.schema_hash) {
|
|
262
|
+
this.schemaHash = payload.schema_hash;
|
|
263
|
+
await this.persistence.setMeta(SCHEMA_META_KEY, this.schemaHash);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
private async changesLoop(): Promise<void> {
|
|
268
|
+
let backoff = 1000;
|
|
269
|
+
while (this.running) {
|
|
270
|
+
this.pollController = new AbortController();
|
|
271
|
+
try {
|
|
272
|
+
const response = await this.api.get<ChangesResponse>(
|
|
273
|
+
`${this.syncBasePath}/changes/`,
|
|
274
|
+
{
|
|
275
|
+
params: { since: this.lastSyncId ?? 0, wait: this.wait },
|
|
276
|
+
signal: this.pollController.signal,
|
|
277
|
+
},
|
|
278
|
+
);
|
|
279
|
+
const changed = await this.checkSchemaHash(
|
|
280
|
+
response.data.schema_hash,
|
|
281
|
+
);
|
|
282
|
+
if (!changed) {
|
|
283
|
+
await this.applyChanges(response.data);
|
|
284
|
+
}
|
|
285
|
+
this.setStatus("live");
|
|
286
|
+
backoff = 1000;
|
|
287
|
+
} catch (error: any) {
|
|
288
|
+
if (!this.running) {
|
|
289
|
+
break;
|
|
290
|
+
}
|
|
291
|
+
if (this.pollController.signal.aborted) {
|
|
292
|
+
// wake() cancelled the poll on purpose — go straight around.
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
const status = error?.response?.status;
|
|
296
|
+
if (status === 410) {
|
|
297
|
+
// Cursor fell out of the server's change-log retention.
|
|
298
|
+
try {
|
|
299
|
+
await this.reset();
|
|
300
|
+
continue;
|
|
301
|
+
} catch (resetError) {
|
|
302
|
+
this.onError?.(resetError);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
this.setStatus("offline");
|
|
306
|
+
this.onError?.(error);
|
|
307
|
+
await new Promise((resolve) => setTimeout(resolve, backoff));
|
|
308
|
+
backoff = Math.min(backoff * 2, 30000);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Returns true (after resetting) when the server's schema hash no longer
|
|
315
|
+
* matches the one local state was built against.
|
|
316
|
+
*/
|
|
317
|
+
private async checkSchemaHash(hash: string | undefined): Promise<boolean> {
|
|
318
|
+
if (!hash) {
|
|
319
|
+
return false;
|
|
320
|
+
}
|
|
321
|
+
if (this.schemaHash === undefined) {
|
|
322
|
+
this.schemaHash = hash;
|
|
323
|
+
await this.persistence.setMeta(SCHEMA_META_KEY, hash);
|
|
324
|
+
return false;
|
|
325
|
+
}
|
|
326
|
+
if (this.schemaHash === hash) {
|
|
327
|
+
return false;
|
|
328
|
+
}
|
|
329
|
+
await this.reset();
|
|
330
|
+
return true;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
private applyChanges(payload: ChangesResponse): Promise<void> {
|
|
334
|
+
// Serialize applications so a wake-triggered poll can never interleave
|
|
335
|
+
// with one still being written.
|
|
336
|
+
this.applyLock = this.applyLock.then(() =>
|
|
337
|
+
this.applyChangesExclusive(payload),
|
|
338
|
+
);
|
|
339
|
+
return this.applyLock;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
private async applyChangesExclusive(payload: ChangesResponse): Promise<void> {
|
|
343
|
+
for (const action of payload.actions) {
|
|
344
|
+
await this.applyAction(action);
|
|
345
|
+
}
|
|
346
|
+
if (payload.last_sync_id > (this.lastSyncId ?? 0)) {
|
|
347
|
+
this.lastSyncId = payload.last_sync_id;
|
|
348
|
+
await this.persistence.setMeta(CURSOR_META_KEY, this.lastSyncId);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
private async applyAction(action: SyncActionPacket): Promise<void> {
|
|
353
|
+
const entity = getEntity(action.model);
|
|
354
|
+
if (!entity) {
|
|
355
|
+
return; // an entity this bundle doesn't use
|
|
356
|
+
}
|
|
357
|
+
const key = action.model;
|
|
358
|
+
if (entity.lazy && !this.readyKeys.has(key)) {
|
|
359
|
+
return; // not loaded yet — the on-demand bootstrap will include it
|
|
360
|
+
}
|
|
361
|
+
const idField = entity.idField || "id";
|
|
362
|
+
const id = String(action.id);
|
|
363
|
+
const deleted =
|
|
364
|
+
action.op === "D" || (action.data && action.data.deleted_at != null);
|
|
365
|
+
|
|
366
|
+
if (deleted) {
|
|
367
|
+
await this.persistence.deleteRecord(key, id);
|
|
368
|
+
this.store.remove(key, id);
|
|
369
|
+
} else if (action.data) {
|
|
370
|
+
await this.persistence.putRecords(key, [{ id, data: action.data }]);
|
|
371
|
+
const pending = this.queue.pendingFor(key, id);
|
|
372
|
+
if (pending.some((txn) => txn.op === "delete")) {
|
|
373
|
+
// Locally deleted, not yet confirmed: keep it gone from the UI.
|
|
374
|
+
} else if (pending.length > 0) {
|
|
375
|
+
// Rebase: server row underneath, unconfirmed local edits on top
|
|
376
|
+
// (last-writer-wins — the local write supersedes on its next send).
|
|
377
|
+
const overlay = pending.reduce(
|
|
378
|
+
(acc, txn) => ({ ...acc, ...(txn.payload || {}) }),
|
|
379
|
+
{} as Record<string, any>,
|
|
380
|
+
);
|
|
381
|
+
this.store.upsert(key, id, {
|
|
382
|
+
...action.data,
|
|
383
|
+
...overlay,
|
|
384
|
+
[idField]: action.data[idField],
|
|
385
|
+
});
|
|
386
|
+
} else {
|
|
387
|
+
this.store.upsert(key, id, action.data);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
await this.queue.resolveForRow(key, id);
|
|
391
|
+
}
|
|
392
|
+
}
|