react-native-cloud-sync 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/README.md +116 -58
  2. package/ios/CloudSyncError.swift +21 -0
  3. package/ios/CloudSyncImpl.swift +515 -95
  4. package/ios/RNCloudSync.mm +73 -1
  5. package/jest-mock.js +1 -0
  6. package/lib/commonjs/hooks.js +309 -3
  7. package/lib/commonjs/hooks.js.map +1 -1
  8. package/lib/commonjs/internal/autoFlush.js +10 -1
  9. package/lib/commonjs/internal/autoFlush.js.map +1 -1
  10. package/lib/commonjs/internal/base64.js +25 -5
  11. package/lib/commonjs/internal/base64.js.map +1 -1
  12. package/lib/commonjs/internal/cloudKitRest.js +91 -15
  13. package/lib/commonjs/internal/cloudKitRest.js.map +1 -1
  14. package/lib/commonjs/internal/googleDriveRest.js +304 -38
  15. package/lib/commonjs/internal/googleDriveRest.js.map +1 -1
  16. package/lib/commonjs/internal/keys.js +34 -17
  17. package/lib/commonjs/internal/keys.js.map +1 -1
  18. package/lib/commonjs/internal/records.js +48 -0
  19. package/lib/commonjs/internal/records.js.map +1 -0
  20. package/lib/commonjs/providers/cloudKit.js +85 -17
  21. package/lib/commonjs/providers/cloudKit.js.map +1 -1
  22. package/lib/commonjs/providers/cloudKitEncrypted.js +30 -7
  23. package/lib/commonjs/providers/cloudKitEncrypted.js.map +1 -1
  24. package/lib/commonjs/providers/googleDrive.js +37 -2
  25. package/lib/commonjs/providers/googleDrive.js.map +1 -1
  26. package/lib/commonjs/providers/icloudDocuments.js +27 -0
  27. package/lib/commonjs/providers/icloudDocuments.js.map +1 -1
  28. package/lib/commonjs/providers/icloudKV.js +13 -3
  29. package/lib/commonjs/providers/icloudKV.js.map +1 -1
  30. package/lib/commonjs/providers/icloudKV.web.js +8 -1
  31. package/lib/commonjs/providers/icloudKV.web.js.map +1 -1
  32. package/lib/commonjs/providers/memory.js +10 -0
  33. package/lib/commonjs/providers/memory.js.map +1 -1
  34. package/lib/commonjs/resolvers.js +27 -13
  35. package/lib/commonjs/resolvers.js.map +1 -1
  36. package/lib/commonjs/specs/NativeRNCloudSync.js +3 -0
  37. package/lib/commonjs/specs/NativeRNCloudSync.js.map +1 -1
  38. package/lib/commonjs/store.js +360 -173
  39. package/lib/commonjs/store.js.map +1 -1
  40. package/lib/commonjs/types.js.map +1 -1
  41. package/lib/module/hooks.js +307 -3
  42. package/lib/module/hooks.js.map +1 -1
  43. package/lib/module/index.js +2 -2
  44. package/lib/module/internal/autoFlush.js +10 -1
  45. package/lib/module/internal/autoFlush.js.map +1 -1
  46. package/lib/module/internal/base64.js +25 -5
  47. package/lib/module/internal/base64.js.map +1 -1
  48. package/lib/module/internal/cloudKitRest.js +89 -13
  49. package/lib/module/internal/cloudKitRest.js.map +1 -1
  50. package/lib/module/internal/googleDriveRest.js +304 -38
  51. package/lib/module/internal/googleDriveRest.js.map +1 -1
  52. package/lib/module/internal/keys.js +33 -17
  53. package/lib/module/internal/keys.js.map +1 -1
  54. package/lib/module/internal/records.js +41 -0
  55. package/lib/module/internal/records.js.map +1 -0
  56. package/lib/module/providers/cloudKit.js +86 -18
  57. package/lib/module/providers/cloudKit.js.map +1 -1
  58. package/lib/module/providers/cloudKitEncrypted.js +31 -8
  59. package/lib/module/providers/cloudKitEncrypted.js.map +1 -1
  60. package/lib/module/providers/googleDrive.js +37 -2
  61. package/lib/module/providers/googleDrive.js.map +1 -1
  62. package/lib/module/providers/icloudDocuments.js +28 -1
  63. package/lib/module/providers/icloudDocuments.js.map +1 -1
  64. package/lib/module/providers/icloudKV.js +13 -3
  65. package/lib/module/providers/icloudKV.js.map +1 -1
  66. package/lib/module/providers/icloudKV.web.js +8 -1
  67. package/lib/module/providers/icloudKV.web.js.map +1 -1
  68. package/lib/module/providers/memory.js +10 -0
  69. package/lib/module/providers/memory.js.map +1 -1
  70. package/lib/module/resolvers.js +27 -12
  71. package/lib/module/resolvers.js.map +1 -1
  72. package/lib/module/specs/NativeRNCloudSync.js +4 -0
  73. package/lib/module/specs/NativeRNCloudSync.js.map +1 -1
  74. package/lib/module/store.js +360 -173
  75. package/lib/module/store.js.map +1 -1
  76. package/lib/module/types.js.map +1 -1
  77. package/lib/typescript/commonjs/errors.d.ts +16 -16
  78. package/lib/typescript/commonjs/errors.d.ts.map +1 -1
  79. package/lib/typescript/commonjs/hooks.d.ts +68 -0
  80. package/lib/typescript/commonjs/hooks.d.ts.map +1 -1
  81. package/lib/typescript/commonjs/index.d.ts +2 -2
  82. package/lib/typescript/commonjs/internal/autoFlush.d.ts.map +1 -1
  83. package/lib/typescript/commonjs/internal/base64.d.ts.map +1 -1
  84. package/lib/typescript/commonjs/internal/cloudKitRest.d.ts +20 -0
  85. package/lib/typescript/commonjs/internal/cloudKitRest.d.ts.map +1 -1
  86. package/lib/typescript/commonjs/internal/googleDriveRest.d.ts +19 -0
  87. package/lib/typescript/commonjs/internal/googleDriveRest.d.ts.map +1 -1
  88. package/lib/typescript/commonjs/internal/keys.d.ts +18 -3
  89. package/lib/typescript/commonjs/internal/keys.d.ts.map +1 -1
  90. package/lib/typescript/commonjs/internal/records.d.ts +24 -0
  91. package/lib/typescript/commonjs/internal/records.d.ts.map +1 -0
  92. package/lib/typescript/commonjs/providers/cloudKit.d.ts +1 -1
  93. package/lib/typescript/commonjs/providers/cloudKit.d.ts.map +1 -1
  94. package/lib/typescript/commonjs/providers/cloudKitBackup.d.ts +1 -1
  95. package/lib/typescript/commonjs/providers/cloudKitBackup.d.ts.map +1 -1
  96. package/lib/typescript/commonjs/providers/cloudKitEncrypted.d.ts.map +1 -1
  97. package/lib/typescript/commonjs/providers/googleDrive.d.ts.map +1 -1
  98. package/lib/typescript/commonjs/providers/googleDriveFiles.d.ts.map +1 -1
  99. package/lib/typescript/commonjs/providers/icloudDocuments.d.ts +1 -1
  100. package/lib/typescript/commonjs/providers/icloudDocuments.d.ts.map +1 -1
  101. package/lib/typescript/commonjs/providers/icloudKV.d.ts.map +1 -1
  102. package/lib/typescript/commonjs/providers/icloudKV.web.d.ts.map +1 -1
  103. package/lib/typescript/commonjs/providers/memory.d.ts.map +1 -1
  104. package/lib/typescript/commonjs/resolvers.d.ts +6 -3
  105. package/lib/typescript/commonjs/resolvers.d.ts.map +1 -1
  106. package/lib/typescript/commonjs/specs/NativeRNCloudSync.d.ts +12 -3
  107. package/lib/typescript/commonjs/specs/NativeRNCloudSync.d.ts.map +1 -1
  108. package/lib/typescript/commonjs/store.d.ts.map +1 -1
  109. package/lib/typescript/commonjs/types.d.ts +31 -3
  110. package/lib/typescript/commonjs/types.d.ts.map +1 -1
  111. package/lib/typescript/module/errors.d.ts +16 -16
  112. package/lib/typescript/module/errors.d.ts.map +1 -1
  113. package/lib/typescript/module/hooks.d.ts +68 -0
  114. package/lib/typescript/module/hooks.d.ts.map +1 -1
  115. package/lib/typescript/module/index.d.ts +2 -2
  116. package/lib/typescript/module/internal/autoFlush.d.ts.map +1 -1
  117. package/lib/typescript/module/internal/base64.d.ts.map +1 -1
  118. package/lib/typescript/module/internal/cloudKitRest.d.ts +20 -0
  119. package/lib/typescript/module/internal/cloudKitRest.d.ts.map +1 -1
  120. package/lib/typescript/module/internal/googleDriveRest.d.ts +19 -0
  121. package/lib/typescript/module/internal/googleDriveRest.d.ts.map +1 -1
  122. package/lib/typescript/module/internal/keys.d.ts +18 -3
  123. package/lib/typescript/module/internal/keys.d.ts.map +1 -1
  124. package/lib/typescript/module/internal/records.d.ts +24 -0
  125. package/lib/typescript/module/internal/records.d.ts.map +1 -0
  126. package/lib/typescript/module/providers/cloudKit.d.ts +1 -1
  127. package/lib/typescript/module/providers/cloudKit.d.ts.map +1 -1
  128. package/lib/typescript/module/providers/cloudKitBackup.d.ts +1 -1
  129. package/lib/typescript/module/providers/cloudKitBackup.d.ts.map +1 -1
  130. package/lib/typescript/module/providers/cloudKitEncrypted.d.ts.map +1 -1
  131. package/lib/typescript/module/providers/googleDrive.d.ts.map +1 -1
  132. package/lib/typescript/module/providers/googleDriveFiles.d.ts.map +1 -1
  133. package/lib/typescript/module/providers/icloudDocuments.d.ts +1 -1
  134. package/lib/typescript/module/providers/icloudDocuments.d.ts.map +1 -1
  135. package/lib/typescript/module/providers/icloudKV.d.ts.map +1 -1
  136. package/lib/typescript/module/providers/icloudKV.web.d.ts.map +1 -1
  137. package/lib/typescript/module/providers/memory.d.ts.map +1 -1
  138. package/lib/typescript/module/resolvers.d.ts +6 -3
  139. package/lib/typescript/module/resolvers.d.ts.map +1 -1
  140. package/lib/typescript/module/specs/NativeRNCloudSync.d.ts +12 -3
  141. package/lib/typescript/module/specs/NativeRNCloudSync.d.ts.map +1 -1
  142. package/lib/typescript/module/store.d.ts.map +1 -1
  143. package/lib/typescript/module/types.d.ts +31 -3
  144. package/lib/typescript/module/types.d.ts.map +1 -1
  145. package/package.json +36 -23
  146. package/plugin/build/index.js +45 -13
  147. package/react-native-cloud-sync.podspec +10 -3
  148. package/src/hooks.ts +357 -3
  149. package/src/index.ts +2 -2
  150. package/src/internal/autoFlush.ts +10 -1
  151. package/src/internal/base64.ts +26 -5
  152. package/src/internal/cloudKitRest.ts +103 -14
  153. package/src/internal/googleDriveRest.ts +338 -37
  154. package/src/internal/keys.ts +42 -16
  155. package/src/internal/records.ts +38 -0
  156. package/src/providers/cloudKit.ts +96 -20
  157. package/src/providers/cloudKitEncrypted.ts +37 -8
  158. package/src/providers/googleDrive.ts +38 -2
  159. package/src/providers/icloudDocuments.ts +40 -2
  160. package/src/providers/icloudKV.ts +13 -3
  161. package/src/providers/icloudKV.web.ts +8 -2
  162. package/src/providers/memory.ts +8 -0
  163. package/src/resolvers.ts +28 -13
  164. package/src/specs/NativeRNCloudSync.ts +16 -2
  165. package/src/store.ts +433 -190
  166. package/src/types.ts +32 -3
@@ -8,6 +8,7 @@ import { byteLength } from "./internal/bytes.js";
8
8
  import { assertKey } from "./internal/keys.js";
9
9
  import { withTimeout } from "./internal/timeout.js";
10
10
  import { attachAutoFlush } from "./internal/autoFlush.js";
11
+ import { clampRetryAfterMs, setOwn } from "./internal/records.js";
11
12
  import { CloudSyncError, ErrorCode, isRetryable, normalizeError } from "./errors.js";
12
13
  import { DEFAULT_TIERING } from "./types.js";
13
14
 
@@ -27,20 +28,16 @@ import { DEFAULT_TIERING } from "./types.js";
27
28
  const OUTBOX_KEY = 'rncs.outbox.v1';
28
29
 
29
30
  /**
30
- * Identity of a queued write: one entry per (provider, key).
31
+ * Providers whose queued writes are never handed to `outboxStorage`.
31
32
  *
32
- * A single helper rather than a template literal at each site, because the two
33
- * places that build it have to agree exactly - when they did not, a flush
34
- * failed to recognise the entries it had just processed and wrote every one of
35
- * them straight back, so a drained queue never shrank.
36
- *
37
- * The separator is an escaped NUL, which cannot appear in a provider name or a
38
- * key, so the two halves can never run together and make two different pairs
39
- * collide.
33
+ * `cloudKitEncrypted` encrypts on device, inside the native layer, so the value
34
+ * the store hands it is the app's plaintext. Persisting a failed write to the
35
+ * durable outbox - typically an unencrypted MMKV or AsyncStorage file - would
36
+ * leave exactly the data the app chose end-to-end encryption for sitting on
37
+ * disk in the clear, for up to `outboxMaxAgeMs`. Those entries are queued in
38
+ * memory instead: retried for as long as the process lives, lost on restart.
40
39
  */
41
- function outboxId(provider, key) {
42
- return `${provider}\u0000${key}`;
43
- }
40
+ const MEMORY_ONLY_OUTBOX_PROVIDERS = new Set(['cloudKitEncrypted']);
44
41
  const DEFAULT_OUTBOX_MAX_ENTRIES = 1000;
45
42
  const DEFAULT_OUTBOX_MAX_ATTEMPTS = 12;
46
43
  const DEFAULT_OUTBOX_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
@@ -159,18 +156,44 @@ export function createCloudStore(options) {
159
156
 
160
157
  // ---------------------------------------------------------------- outbox
161
158
 
159
+ /** Queued writes that must not reach durable storage - see {@link MEMORY_ONLY_OUTBOX_PROVIDERS}. */
160
+ let volatileOutbox = [];
162
161
  function readOutbox() {
162
+ return [...readDurableOutbox(), ...volatileOutbox];
163
+ }
164
+ function readDurableOutbox() {
163
165
  const raw = storage.getString(OUTBOX_KEY);
164
166
  if (raw == null) return [];
165
167
  try {
166
168
  const parsed = JSON.parse(raw);
167
- return Array.isArray(parsed) ? parsed : [];
169
+ // Entry by entry: one malformed record in a shared storage file must not
170
+ // throw on every later write path that reads the queue.
171
+ return Array.isArray(parsed) ? parsed.filter(isOutboxEntry).filter(e => !MEMORY_ONLY_OUTBOX_PROVIDERS.has(e.provider)) : [];
168
172
  } catch {
169
173
  return [];
170
174
  }
171
175
  }
176
+
177
+ /**
178
+ * Bumped on every rewrite, so a flush can tell cheaply whether the queue
179
+ * changed under it while it was awaiting the network.
180
+ */
181
+ let outboxVersion = 0;
172
182
  function writeOutbox(entries) {
173
- storage.set(OUTBOX_KEY, JSON.stringify(entries));
183
+ outboxVersion += 1;
184
+ volatileOutbox = entries.filter(e => MEMORY_ONLY_OUTBOX_PROVIDERS.has(e.provider));
185
+ storage.set(OUTBOX_KEY, JSON.stringify(entries.filter(e => !MEMORY_ONLY_OUTBOX_PROVIDERS.has(e.provider))));
186
+ }
187
+
188
+ // An earlier version persisted these entries like any other. Move them into
189
+ // memory and rewrite the durable copy without them, so plaintext a previous
190
+ // run left on disk does not outlive this one.
191
+ try {
192
+ const raw = storage.getString(OUTBOX_KEY);
193
+ const parsed = raw == null ? null : JSON.parse(raw);
194
+ if (Array.isArray(parsed) && parsed.some(e => isOutboxEntry(e) && MEMORY_ONLY_OUTBOX_PROVIDERS.has(e.provider))) writeOutbox(parsed.filter(isOutboxEntry));
195
+ } catch {
196
+ // Unreadable storage is handled - as an empty queue - by readOutbox.
174
197
  }
175
198
  function reportDropped(entry, reason, error) {
176
199
  options.onDropped?.({
@@ -224,6 +247,47 @@ export function createCloudStore(options) {
224
247
  if (kept.length !== entries.length) writeOutbox(kept);
225
248
  }
226
249
 
250
+ // ------------------------------------------------------- per-key ordering
251
+
252
+ /**
253
+ * Serialises every operation on one (provider, key) pair.
254
+ *
255
+ * Without it a flush could be re-sending a queued v1 while a newer direct
256
+ * `setItem(k, v2)` ran alongside it, and whichever request the server saw
257
+ * last won - so v1 could land on top of v2. Holding the pair for the whole
258
+ * of "send, then update the outbox" makes the two strictly ordered: either
259
+ * the flush sends v1 first and v2 lands after it, or v2 lands first, drops
260
+ * the queued v1, and the flush finds nothing left to send.
261
+ *
262
+ * A batch takes every pair it touches at once. Waiters only ever wait on
263
+ * holders registered before them, so this cannot deadlock - but nothing
264
+ * holding a pair may try to take the same pair again.
265
+ *
266
+ * With `timeoutMs`, a timed-out request releases the pair while the request
267
+ * itself may still be in flight; a promise cannot be cancelled.
268
+ */
269
+ const keyLocks = new Map();
270
+ async function withKeys(provider, keys, fn) {
271
+ const ids = [...new Set(keys)].map(k => `${provider}\u0000${k}`);
272
+ const before = [];
273
+ for (const id of ids) {
274
+ const held = keyLocks.get(id);
275
+ if (held != null) before.push(held);
276
+ }
277
+ let release;
278
+ const mine = new Promise(resolve => {
279
+ release = resolve;
280
+ });
281
+ for (const id of ids) keyLocks.set(id, mine);
282
+ try {
283
+ if (before.length > 0) await Promise.all(before);
284
+ return await fn();
285
+ } finally {
286
+ release();
287
+ for (const id of ids) if (keyLocks.get(id) === mine) keyLocks.delete(id);
288
+ }
289
+ }
290
+
227
291
  // ------------------------------------------------------------ size rules
228
292
 
229
293
  /**
@@ -236,7 +300,8 @@ export function createCloudStore(options) {
236
300
  function limitFor(provider) {
237
301
  if (tiering == null) return null;
238
302
  if (provider.name === 'icloudKV') return tiering.kvMaxBytes;
239
- if (provider.name === 'cloudKit') return tiering.recordMaxBytes;
303
+ // Encrypted records are still CKRecords, under the same 1 MB ceiling.
304
+ if (provider.name === 'cloudKit' || provider.name === 'cloudKitEncrypted') return tiering.recordMaxBytes;
240
305
  return null;
241
306
  }
242
307
  function canHold(provider, value) {
@@ -284,6 +349,27 @@ export function createCloudStore(options) {
284
349
  for (const key of keys) out.push(await run(p, () => p.getItem(key)));
285
350
  return out;
286
351
  }
352
+
353
+ /**
354
+ * A batched read for a resolver, with server timestamps when the resolver
355
+ * uses them - so `multiGet` resolves exactly as `getItem` does. Stays one
356
+ * request per provider when the resolver ignores timestamps, or when the
357
+ * provider can batch them.
358
+ */
359
+ async function readBatchForResolve(p, keys, withMeta) {
360
+ if (withMeta) {
361
+ if (p.multiGetWithMeta != null) return await run(p, () => p.multiGetWithMeta(keys));
362
+ if (p.getItemWithMeta != null) {
363
+ const out = [];
364
+ for (const key of keys) out.push(await readOne(p, key));
365
+ return out;
366
+ }
367
+ }
368
+ const values = await readBatch(p, keys);
369
+ return values.map(value => value == null ? null : {
370
+ value
371
+ });
372
+ }
287
373
  async function writeBatch(p, entries) {
288
374
  if (entries.length === 0) return;
289
375
  if (p.multiSet != null) {
@@ -303,21 +389,50 @@ export function createCloudStore(options) {
303
389
 
304
390
  // ---------------------------------------------------------------- writes
305
391
 
306
- /** Single-destination write, queueing a retryable failure. */
392
+ /**
393
+ * Single-destination write, queueing a retryable failure. Resolves true when
394
+ * the value landed, false when it was queued instead.
395
+ */
307
396
  async function writeOne(target, key, value) {
308
- try {
309
- await run(target, () => target.setItem(key, value));
310
- // This write supersedes anything queued for the same destination.
311
- dequeue(key, target.name);
312
- } catch (e) {
313
- const err = normalizeError(e, target.name);
314
- if (useOutbox && isRetryable(err)) {
315
- enqueue(key, value, target.name, err.code);
397
+ return await withKeys(target.name, [key], async () => {
398
+ try {
399
+ await run(target, () => target.setItem(key, value));
400
+ // This write supersedes anything queued for the same destination.
401
+ dequeue(key, target.name);
402
+ return true;
403
+ } catch (e) {
404
+ const err = normalizeError(e, target.name);
405
+ if (useOutbox && isRetryable(err)) {
406
+ enqueue(key, value, target.name, err.code);
407
+ options.onError?.(err);
408
+ return false;
409
+ }
410
+ throw err;
411
+ }
412
+ });
413
+ }
414
+
415
+ /**
416
+ * One destination's share of a multi-destination operation, under that
417
+ * destination's lock. Resolves 'done' or 'queued', or the fatal error -
418
+ * already reported through `onError` - for the caller to weigh.
419
+ */
420
+ async function attempt(target, keys, send, queued) {
421
+ return await withKeys(target.name, keys, async () => {
422
+ try {
423
+ await run(target, send);
424
+ dequeueMany(keys, target.name);
425
+ return 'done';
426
+ } catch (e) {
427
+ const err = normalizeError(e, target.name);
316
428
  options.onError?.(err);
317
- return;
429
+ if (useOutbox && isRetryable(err)) {
430
+ keys.forEach((key, i) => enqueue(key, queued[i], target.name, err.code));
431
+ return 'queued';
432
+ }
433
+ return err;
318
434
  }
319
- throw err;
320
- }
435
+ });
321
436
  }
322
437
 
323
438
  /**
@@ -333,14 +448,16 @@ export function createCloudStore(options) {
333
448
  async function evictWhereTooLarge(targets, key, value) {
334
449
  for (const target of targets) {
335
450
  if (canHold(target, value)) continue;
336
- try {
337
- await run(target, () => target.removeItem(key));
338
- dequeue(key, target.name);
339
- } catch (e) {
340
- // Best effort. A stale copy left behind is reported, not fatal - the
341
- // value itself did land somewhere.
342
- options.onError?.(normalizeError(e, target.name));
343
- }
451
+ await withKeys(target.name, [key], async () => {
452
+ try {
453
+ await run(target, () => target.removeItem(key));
454
+ dequeue(key, target.name);
455
+ } catch (e) {
456
+ // Best effort. A stale copy left behind is reported, not fatal - the
457
+ // value itself did land somewhere.
458
+ options.onError?.(normalizeError(e, target.name));
459
+ }
460
+ });
344
461
  }
345
462
  }
346
463
 
@@ -370,19 +487,9 @@ export function createCloudStore(options) {
370
487
  }
371
488
  let stored = 0;
372
489
  let fatal = null;
373
- for (const target of eligible) try {
374
- await run(target, () => target.setItem(key, value));
375
- dequeue(key, target.name);
376
- stored += 1;
377
- } catch (e) {
378
- const err = normalizeError(e, target.name);
379
- if (useOutbox && isRetryable(err)) {
380
- enqueue(key, value, target.name, err.code);
381
- options.onError?.(err);
382
- continue;
383
- }
384
- fatal ??= err;
385
- options.onError?.(err);
490
+ for (const target of eligible) {
491
+ const outcome = await attempt(target, [key], () => target.setItem(key, value), [value]);
492
+ if (outcome === 'done') stored += 1;else if (outcome !== 'queued') fatal ??= outcome;
386
493
  }
387
494
 
388
495
  // Only once the value has a home somewhere: dropping the old copy first
@@ -402,19 +509,9 @@ export function createCloudStore(options) {
402
509
  if (targets.length === 0) throw noneAvailable();
403
510
  let removed = 0;
404
511
  let fatal = null;
405
- for (const target of targets) try {
406
- await run(target, () => target.removeItem(key));
407
- dequeue(key, target.name);
408
- removed += 1;
409
- } catch (e) {
410
- const err = normalizeError(e, target.name);
411
- if (useOutbox && isRetryable(err)) {
412
- enqueue(key, null, target.name, err.code);
413
- options.onError?.(err);
414
- continue;
415
- }
416
- fatal ??= err;
417
- options.onError?.(err);
512
+ for (const target of targets) {
513
+ const outcome = await attempt(target, [key], () => target.removeItem(key), [null]);
514
+ if (outcome === 'done') removed += 1;else if (outcome !== 'queued') fatal ??= outcome;
418
515
  }
419
516
  if (removed === 0 && fatal != null) throw fatal;
420
517
  }
@@ -425,27 +522,59 @@ export function createCloudStore(options) {
425
522
  *
426
523
  * A provider the winner no longer fits in has its stale copy removed instead
427
524
  * of being silently left holding it - see {@link evictWhereTooLarge}.
525
+ *
526
+ * The outbox is consulted per provider, under that provider's lock for the
527
+ * key. A queued write there is this device's own pending value, so it is
528
+ * weighed as one more candidate:
529
+ *
530
+ * - if the resolver still prefers the winner, the queued write is older
531
+ * news - it is dropped once the repair lands, or the next flush would
532
+ * put it back on top of the repaired value;
533
+ * - if the resolver prefers the queued value, that provider is left alone
534
+ * and the outbox delivers it;
535
+ * - a queued delete is left to land, since repairing would resurrect a
536
+ * value the user removed.
428
537
  */
429
- async function repair(key, winner, candidates, reachable) {
538
+ async function repair(resolve, key, winner, candidates, reachable) {
430
539
  const byProvider = new Map(candidates.map(c => [c.provider, c.value]));
431
540
  const stored = await encode(key, winner);
432
- for (const [name, provider] of reachable) {
433
- if (byProvider.get(name) === winner) continue;
434
- if (!canHold(provider, stored)) {
435
- if (!byProvider.has(name)) continue;
436
- try {
541
+ for (const [name, provider] of reachable) await withKeys(name, [key], async () => {
542
+ try {
543
+ const queued = readOutbox().find(e => e.provider === name && e.key === key);
544
+ if (queued != null) {
545
+ if (queued.value == null) return;
546
+ const pending = await decode(key, queued.value);
547
+ const pick = resolve([...candidates, {
548
+ provider: name,
549
+ value: pending,
550
+ modifiedAt: queued.enqueuedAt
551
+ }]);
552
+ if (pick !== winner) return;
553
+ }
554
+ if (byProvider.get(name) !== winner) if (canHold(provider, stored)) {
555
+ await run(provider, () => provider.setItem(key, stored));
556
+ } else {
557
+ if (!byProvider.has(name)) return;
437
558
  await run(provider, () => provider.removeItem(key));
438
- } catch (e) {
439
- options.onError?.(normalizeError(e, name));
440
559
  }
441
- continue;
442
- }
443
- try {
444
- await run(provider, () => provider.setItem(key, stored));
560
+ if (queued != null) dequeue(key, name);
445
561
  } catch (e) {
446
562
  options.onError?.(normalizeError(e, name));
447
563
  }
448
- }
564
+ });
565
+ }
566
+
567
+ /** Resolves one key's candidates and, if configured, starts read repair. */
568
+ function settle(resolve, key, candidates, reachable) {
569
+ const winner = resolve(candidates);
570
+ if (winner == null) return null;
571
+
572
+ // Best-effort convergence. Never fails the read - the caller already has
573
+ // the right answer, and a failed repair only costs another resolve later.
574
+ if (repairOnRead) void repair(resolve, key, winner, candidates, reachable).catch(e => {
575
+ options.onError?.(normalizeError(e));
576
+ });
577
+ return winner;
449
578
  }
450
579
 
451
580
  // ---------------------------------------------------------------- events
@@ -612,16 +741,12 @@ export function createCloudStore(options) {
612
741
  reachedAny
613
742
  };
614
743
  }
615
- const winner = resolveValue(candidates);
744
+ const winner = settle(resolveValue, key, candidates, reachable);
616
745
  if (winner == null) return {
617
746
  found: false,
618
747
  value: null,
619
748
  reachedAny
620
749
  };
621
-
622
- // Best-effort convergence. Never fails the read - the caller already has
623
- // the right answer, and a failed repair only costs another resolve later.
624
- if (repairOnRead) void repair(key, winner, candidates, reachable);
625
750
  return {
626
751
  found: true,
627
752
  value: winner,
@@ -645,19 +770,25 @@ export function createCloudStore(options) {
645
770
  checkKey(key);
646
771
  const stored = await encode(key, value);
647
772
  if (writeMode === 'failover') {
648
- const target = await providerForSize(stored, await primary());
649
- await writeOne(target, key, stored);
773
+ const preferred = await primary();
774
+ const target = await providerForSize(stored, preferred);
775
+ const landed = await writeOne(target, key, stored);
776
+ // Tiering routed the value past the preferred provider, which may still
777
+ // hold an older, smaller copy - and reads reach that one first. Without
778
+ // this, a value that grows past `kvMaxBytes` reads back stale forever.
779
+ if (landed && target !== preferred) await evictWhereTooLarge([preferred], key, stored);
650
780
  return;
651
781
  }
652
782
  await writeMany(await allAvailable(), key, stored);
653
783
  },
654
784
  removeItem: async key => {
655
785
  checkKey(key);
656
- // A mirrored write must be a mirrored delete. Removing from only the
657
- // preferred provider would leave a copy behind that reads then fall
658
- // through to and resurrect.
659
- const targets = writeMode === 'mirror' ? await allAvailable() : [await primary()];
660
- await removeMany(targets, key);
786
+ // A delete has to reach every provider reads fall through to, in both
787
+ // modes. Mirror wrote everywhere; failover may have written to a later
788
+ // provider too - because the preferred one was down at the time, or
789
+ // because tiering routed a large value past it. Removing from only the
790
+ // preferred provider leaves that copy behind for reads to resurrect.
791
+ await removeMany(await allAvailable(), key);
661
792
  },
662
793
  getAllKeys: async () => {
663
794
  const seen = new Set();
@@ -682,21 +813,30 @@ export function createCloudStore(options) {
682
813
  // batched request rather than one per key.
683
814
  if (resolveValue != null) {
684
815
  const perKey = new Map(keys.map(k => [k, []]));
816
+ const reachable = new Map();
817
+ // Same candidates `getItem` would build, timestamps included, so the
818
+ // two reads cannot resolve the same key differently.
819
+ const withMeta = resolveValue.usesModifiedAt !== false;
685
820
  let reachedAny = false;
686
821
  for (const name of options.providers) {
687
822
  const p = resolveProvider(name);
688
823
  try {
689
824
  if (!(await isAvailable(p))) continue;
690
- const values = await readBatch(p, keys);
691
- reachedAny = true;
825
+ const hits = await readBatchForResolve(p, keys, withMeta);
826
+ const found = [];
692
827
  for (let i = 0; i < keys.length; i += 1) {
693
- const raw = values[i];
694
- if (raw == null) continue;
695
- perKey.get(keys[i]).push({
828
+ const hit = hits[i];
829
+ if (hit == null) continue;
830
+ found.push([keys[i], {
696
831
  provider: p.name,
697
- value: await decode(keys[i], raw)
698
- });
832
+ value: await decode(keys[i], hit.value),
833
+ modifiedAt: hit.modifiedAt
834
+ }]);
699
835
  }
836
+ // Only a provider read in full is a repair target - see readAcross.
837
+ reachable.set(p.name, p);
838
+ reachedAny = true;
839
+ for (const [key, candidate] of found) perKey.get(key).push(candidate);
700
840
  } catch {
701
841
  // Same rule as `getAllKeys`: one provider failing to answer must not
702
842
  // erase the answers the others gave.
@@ -705,7 +845,7 @@ export function createCloudStore(options) {
705
845
  if (!reachedAny) throw noneAvailable();
706
846
  for (const key of keys) {
707
847
  const candidates = perKey.get(key);
708
- answers.set(key, candidates.length === 0 ? null : resolveValue(candidates));
848
+ answers.set(key, candidates.length === 0 ? null : settle(resolveValue, key, candidates, reachable));
709
849
  }
710
850
  return keys.map(k => [k, answers.get(k) ?? null]);
711
851
  }
@@ -756,17 +896,13 @@ export function createCloudStore(options) {
756
896
  for (const {
757
897
  provider,
758
898
  items
759
- } of byTarget.values()) try {
760
- await writeBatch(provider, items);
761
- dequeueMany(items.map(([k]) => k), provider.name);
762
- } catch (e) {
763
- const err = normalizeError(e, provider.name);
764
- if (useOutbox && isRetryable(err)) {
765
- for (const [key, value] of items) enqueue(key, value, provider.name, err.code);
766
- options.onError?.(err);
767
- continue;
768
- }
769
- throw err;
899
+ } of byTarget.values()) {
900
+ const outcome = await attempt(provider, items.map(([k]) => k), () => writeBatch(provider, items), items.map(([, v]) => v));
901
+ if (outcome === 'queued') continue;
902
+ if (outcome !== 'done') throw outcome;
903
+ // Same stale-copy rule as `setItem`: values routed past the
904
+ // preferred provider must not leave an older copy there.
905
+ if (provider !== preferred) for (const [key, value] of items) await evictWhereTooLarge([preferred], key, value);
770
906
  }
771
907
  return;
772
908
  }
@@ -777,20 +913,8 @@ export function createCloudStore(options) {
777
913
  for (const target of targets) {
778
914
  const fits = encoded.filter(([, value]) => canHold(target, value));
779
915
  if (fits.length === 0) continue;
780
- try {
781
- await writeBatch(target, fits);
782
- dequeueMany(fits.map(([k]) => k), target.name);
783
- anyStored = true;
784
- } catch (e) {
785
- const err = normalizeError(e, target.name);
786
- if (useOutbox && isRetryable(err)) {
787
- for (const [key, value] of fits) enqueue(key, value, target.name, err.code);
788
- options.onError?.(err);
789
- continue;
790
- }
791
- fatal ??= err;
792
- options.onError?.(err);
793
- }
916
+ const outcome = await attempt(target, fits.map(([k]) => k), () => writeBatch(target, fits), fits.map(([, v]) => v));
917
+ if (outcome === 'done') anyStored = true;else if (outcome !== 'queued') fatal ??= outcome;
794
918
  }
795
919
  if (anyStored) for (const [key, value] of encoded) await evictWhereTooLarge(targets, key, value);
796
920
  if (!anyStored && fatal != null) throw fatal;
@@ -804,23 +928,15 @@ export function createCloudStore(options) {
804
928
  multiRemove: async keys => {
805
929
  for (const key of keys) checkKey(key);
806
930
  if (keys.length === 0) return;
807
- const targets = writeMode === 'mirror' ? await allAvailable() : [await primary()];
931
+
932
+ // Every available provider, in both modes - see `removeItem`.
933
+ const targets = await allAvailable();
808
934
  if (targets.length === 0) throw noneAvailable();
809
935
  let removedAnywhere = false;
810
936
  let fatal = null;
811
- for (const target of targets) try {
812
- await removeBatch(target, keys);
813
- dequeueMany(keys, target.name);
814
- removedAnywhere = true;
815
- } catch (e) {
816
- const err = normalizeError(e, target.name);
817
- if (useOutbox && isRetryable(err)) {
818
- for (const key of keys) enqueue(key, null, target.name, err.code);
819
- options.onError?.(err);
820
- continue;
821
- }
822
- fatal ??= err;
823
- options.onError?.(err);
937
+ for (const target of targets) {
938
+ const outcome = await attempt(target, keys, () => removeBatch(target, keys), keys.map(() => null));
939
+ if (outcome === 'done') removedAnywhere = true;else if (outcome !== 'queued') fatal ??= outcome;
824
940
  }
825
941
  if (!removedAnywhere && fatal != null) throw fatal;
826
942
  },
@@ -838,7 +954,9 @@ export function createCloudStore(options) {
838
954
  const keys = await store.getAllKeys();
839
955
  if (keys.length === 0) return {};
840
956
  const out = {};
841
- for (const [key, value] of await store.multiGet(keys)) if (value != null) out[key] = value;
957
+ // Keys come from provider listings; a file named `__proto__` must land
958
+ // as an entry, not be swallowed by the prototype setter.
959
+ for (const [key, value] of await store.multiGet(keys)) if (value != null) setOwn(out, key, value);
842
960
  return out;
843
961
  },
844
962
  getQuota: async () => {
@@ -875,16 +993,19 @@ export function createCloudStore(options) {
875
993
  // on the way out only to re-encode on the way in would be wasted work -
876
994
  // and, for a codec with a random nonce, would rewrite every byte.
877
995
  for (const key of keys) {
996
+ // Which end failed, so the error names the provider that raised it.
997
+ let side = from;
878
998
  try {
879
999
  const value = await run(src, () => src.getItem(key));
880
1000
  if (value == null) {
881
1001
  skipped.push(key);
882
1002
  } else {
883
- await run(dst, () => dst.setItem(key, value));
1003
+ side = to;
1004
+ await withKeys(to, [key], () => run(dst, () => dst.setItem(key, value)));
884
1005
  copied.push(key);
885
1006
  }
886
1007
  } catch (e) {
887
- const err = normalizeError(e, from);
1008
+ const err = normalizeError(e, side);
888
1009
  failed.push({
889
1010
  key,
890
1011
  error: err
@@ -969,10 +1090,26 @@ export function createCloudStore(options) {
969
1090
  dropped: []
970
1091
  };
971
1092
  const now = Date.now();
1093
+ // Each kept entry alongside the snapshot entry it came from, so the final
1094
+ // merge can check that the original is still the one queued.
972
1095
  const keep = [];
973
1096
  const dropped = [];
974
- const handled = new Set();
975
1097
  let drained = 0;
1098
+
1099
+ // The queue can change while this flush awaits the network: a newer write
1100
+ // supersedes an entry (dequeue or replace), `discardPendingWrites` removes
1101
+ // one, or an account switch empties it. An entry that is no longer queued
1102
+ // must not be sent - after an account switch that would write the previous
1103
+ // user's data into the new account - nor written back at the end.
1104
+ let seenVersion = outboxVersion;
1105
+ let current = entries;
1106
+ function stillQueued(entry) {
1107
+ if (outboxVersion !== seenVersion) {
1108
+ current = readOutbox();
1109
+ seenVersion = outboxVersion;
1110
+ }
1111
+ return current.some(e => sameEntry(e, entry));
1112
+ }
976
1113
  function drop(entry, reason, error) {
977
1114
  dropped.push({
978
1115
  entry,
@@ -981,14 +1118,28 @@ export function createCloudStore(options) {
981
1118
  });
982
1119
  reportDropped(entry, reason, error);
983
1120
  }
1121
+
1122
+ /**
1123
+ * Takes a delivered entry out of storage straight away, rather than at the
1124
+ * final merge, so nothing consulting the outbox meanwhile - read repair,
1125
+ * `pendingWrites()` - mistakes it for a write still waiting.
1126
+ */
1127
+ function removeExact(entry) {
1128
+ const queued = readOutbox();
1129
+ const rest = queued.filter(e => !sameEntry(e, entry));
1130
+ if (rest.length !== queued.length) writeOutbox(rest);
1131
+ }
984
1132
  for (const entry of entries) {
985
- handled.add(outboxId(entry.provider, entry.key));
1133
+ if (!stillQueued(entry)) continue;
986
1134
  if (now - entry.enqueuedAt > maxAgeMs) {
987
1135
  drop(entry, 'expired');
988
1136
  continue;
989
1137
  }
990
1138
  if (entry.nextAttemptAt > now) {
991
- keep.push(entry);
1139
+ keep.push({
1140
+ entry,
1141
+ from: entry
1142
+ });
992
1143
  continue;
993
1144
  }
994
1145
  let p;
@@ -999,56 +1150,92 @@ export function createCloudStore(options) {
999
1150
  drop(entry, 'notRetryable', normalizeError(e, entry.provider));
1000
1151
  continue;
1001
1152
  }
1002
- try {
1003
- if (entry.value == null) await run(p, () => p.removeItem(entry.key));else await run(p, () => p.setItem(entry.key, entry.value));
1004
- drained += 1;
1005
- } catch (e) {
1006
- const err = normalizeError(e, entry.provider);
1007
- options.onError?.(err);
1008
-
1009
- // A write only enters the queue for a retryable reason. If retrying it
1010
- // now fails for one the user must act on - quota, signed out, a payload
1011
- // the store will never accept - re-queueing turns it into a poison
1012
- // entry that retries forever, never drains, and never surfaces.
1013
- if (!isRetryable(err)) {
1014
- drop(entry, 'notRetryable', err);
1015
- continue;
1016
- }
1017
- const attempts = entry.attempts + 1;
1018
- if (attempts >= maxAttempts) {
1019
- drop({
1020
- ...entry,
1021
- attempts,
1022
- lastErrorCode: err.code
1023
- }, 'tooManyAttempts', err);
1024
- continue;
1153
+ const provider = p;
1154
+ // Under the pair's lock, so a direct write to the same key cannot run
1155
+ // alongside this re-send - see `withKeys`. Checked again once the lock
1156
+ // is held: a direct write that got there first has dropped this entry.
1157
+ await withKeys(entry.provider, [entry.key], async () => {
1158
+ if (!stillQueued(entry)) return;
1159
+ try {
1160
+ if (entry.value == null) await run(provider, () => provider.removeItem(entry.key));else await run(provider, () => provider.setItem(entry.key, entry.value));
1161
+ drained += 1;
1162
+ removeExact(entry);
1163
+ } catch (e) {
1164
+ retryLater(entry, normalizeError(e, entry.provider));
1025
1165
  }
1166
+ });
1167
+ }
1168
+ function retryLater(entry, err) {
1169
+ options.onError?.(err);
1026
1170
 
1027
- // Exponential backoff, honouring a server-supplied retry hint when
1028
- // there is one (CloudKit sends `retryAfter`, Drive `Retry-After`).
1029
- const backoff = err.retryAfterMs ?? Math.min(2 ** attempts * 1000, 5 * 60_000);
1030
- keep.push({
1171
+ // A write only enters the queue for a retryable reason. If retrying it
1172
+ // now fails for one the user must act on - quota, signed out, a payload
1173
+ // the store will never accept - re-queueing turns it into a poison
1174
+ // entry that retries forever, never drains, and never surfaces.
1175
+ if (!isRetryable(err)) {
1176
+ drop(entry, 'notRetryable', err);
1177
+ return;
1178
+ }
1179
+ const attempts = entry.attempts + 1;
1180
+ if (attempts >= maxAttempts) {
1181
+ drop({
1031
1182
  ...entry,
1032
1183
  attempts,
1033
- nextAttemptAt: Date.now() + backoff,
1034
1184
  lastErrorCode: err.code
1035
- });
1185
+ }, 'tooManyAttempts', err);
1186
+ return;
1036
1187
  }
1188
+
1189
+ // Exponential backoff, honouring a server-supplied retry hint when
1190
+ // there is one (CloudKit sends `retryAfter`, Drive `Retry-After`).
1191
+ // Clamped again here: a third-party provider can set any number.
1192
+ const backoff = clampRetryAfterMs(err.retryAfterMs) ?? Math.min(2 ** attempts * 1000, 5 * 60_000);
1193
+ keep.push({
1194
+ entry: {
1195
+ ...entry,
1196
+ attempts,
1197
+ nextAttemptAt: Date.now() + backoff,
1198
+ lastErrorCode: err.code
1199
+ },
1200
+ from: entry
1201
+ });
1037
1202
  }
1038
1203
 
1039
1204
  // Merge rather than overwrite. A `setItem` that failed while this flush was
1040
- // awaiting the network enqueued itself into storage; writing the snapshot
1041
- // back wholesale would silently discard it.
1042
- const arrived = readOutbox().filter(e => !handled.has(outboxId(e.provider, e.key)));
1043
- writeOutbox([...keep, ...arrived]);
1205
+ // awaiting the network enqueued itself into storage - possibly replacing
1206
+ // an entry for the same key with a newer value - and writing the snapshot
1207
+ // back wholesale would silently discard it. Equally, a kept entry whose
1208
+ // original was removed meanwhile (superseded, discarded, account switch)
1209
+ // must stay removed.
1210
+ const latest = readOutbox();
1211
+ const kept = keep.filter(({
1212
+ from
1213
+ }) => latest.some(e => sameEntry(e, from))).map(({
1214
+ entry
1215
+ }) => entry);
1216
+ const arrived = latest.filter(e => !entries.some(s => sameEntry(s, e)));
1217
+ writeOutbox([...kept, ...arrived]);
1044
1218
  return {
1045
1219
  drained,
1046
- remaining: keep.length + arrived.length,
1220
+ remaining: kept.length + arrived.length,
1047
1221
  dropped
1048
1222
  };
1049
1223
  }
1050
1224
  return store;
1051
1225
  }
1226
+
1227
+ /**
1228
+ * The same queued write, not merely one for the same key: a newer write to the
1229
+ * key replaces the entry with a fresh `enqueuedAt` and value.
1230
+ */
1231
+ function sameEntry(a, b) {
1232
+ return a.provider === b.provider && a.key === b.key && a.enqueuedAt === b.enqueuedAt && a.value === b.value;
1233
+ }
1234
+ function isOutboxEntry(e) {
1235
+ if (e == null || typeof e !== 'object') return false;
1236
+ const o = e;
1237
+ return typeof o.key === 'string' && typeof o.provider === 'string' && (o.value === null || typeof o.value === 'string') && typeof o.attempts === 'number' && typeof o.nextAttemptAt === 'number' && typeof o.enqueuedAt === 'number';
1238
+ }
1052
1239
  function createMemoryOutboxStorage() {
1053
1240
  const map = new Map();
1054
1241
  return {