react-native-cloud-sync 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/README.md +18 -0
  2. package/ios/CloudSyncError.swift +21 -0
  3. package/ios/CloudSyncImpl.swift +515 -95
  4. package/ios/RNCloudSync.mm +73 -1
  5. package/jest-mock.js +1 -0
  6. package/lib/commonjs/hooks.js +73 -9
  7. package/lib/commonjs/hooks.js.map +1 -1
  8. package/lib/commonjs/internal/autoFlush.js +10 -1
  9. package/lib/commonjs/internal/autoFlush.js.map +1 -1
  10. package/lib/commonjs/internal/base64.js +25 -5
  11. package/lib/commonjs/internal/base64.js.map +1 -1
  12. package/lib/commonjs/internal/cloudKitRest.js +91 -15
  13. package/lib/commonjs/internal/cloudKitRest.js.map +1 -1
  14. package/lib/commonjs/internal/googleDriveRest.js +304 -38
  15. package/lib/commonjs/internal/googleDriveRest.js.map +1 -1
  16. package/lib/commonjs/internal/keys.js +34 -17
  17. package/lib/commonjs/internal/keys.js.map +1 -1
  18. package/lib/commonjs/internal/records.js +48 -0
  19. package/lib/commonjs/internal/records.js.map +1 -0
  20. package/lib/commonjs/providers/cloudKit.js +85 -17
  21. package/lib/commonjs/providers/cloudKit.js.map +1 -1
  22. package/lib/commonjs/providers/cloudKitEncrypted.js +30 -7
  23. package/lib/commonjs/providers/cloudKitEncrypted.js.map +1 -1
  24. package/lib/commonjs/providers/googleDrive.js +37 -2
  25. package/lib/commonjs/providers/googleDrive.js.map +1 -1
  26. package/lib/commonjs/providers/icloudDocuments.js +27 -0
  27. package/lib/commonjs/providers/icloudDocuments.js.map +1 -1
  28. package/lib/commonjs/providers/icloudKV.js +13 -3
  29. package/lib/commonjs/providers/icloudKV.js.map +1 -1
  30. package/lib/commonjs/providers/icloudKV.web.js +8 -1
  31. package/lib/commonjs/providers/icloudKV.web.js.map +1 -1
  32. package/lib/commonjs/providers/memory.js +10 -0
  33. package/lib/commonjs/providers/memory.js.map +1 -1
  34. package/lib/commonjs/resolvers.js +27 -13
  35. package/lib/commonjs/resolvers.js.map +1 -1
  36. package/lib/commonjs/specs/NativeRNCloudSync.js +3 -0
  37. package/lib/commonjs/specs/NativeRNCloudSync.js.map +1 -1
  38. package/lib/commonjs/store.js +360 -173
  39. package/lib/commonjs/store.js.map +1 -1
  40. package/lib/commonjs/types.js.map +1 -1
  41. package/lib/module/hooks.js +73 -9
  42. package/lib/module/hooks.js.map +1 -1
  43. package/lib/module/internal/autoFlush.js +10 -1
  44. package/lib/module/internal/autoFlush.js.map +1 -1
  45. package/lib/module/internal/base64.js +25 -5
  46. package/lib/module/internal/base64.js.map +1 -1
  47. package/lib/module/internal/cloudKitRest.js +89 -13
  48. package/lib/module/internal/cloudKitRest.js.map +1 -1
  49. package/lib/module/internal/googleDriveRest.js +304 -38
  50. package/lib/module/internal/googleDriveRest.js.map +1 -1
  51. package/lib/module/internal/keys.js +33 -17
  52. package/lib/module/internal/keys.js.map +1 -1
  53. package/lib/module/internal/records.js +41 -0
  54. package/lib/module/internal/records.js.map +1 -0
  55. package/lib/module/providers/cloudKit.js +86 -18
  56. package/lib/module/providers/cloudKit.js.map +1 -1
  57. package/lib/module/providers/cloudKitEncrypted.js +31 -8
  58. package/lib/module/providers/cloudKitEncrypted.js.map +1 -1
  59. package/lib/module/providers/googleDrive.js +37 -2
  60. package/lib/module/providers/googleDrive.js.map +1 -1
  61. package/lib/module/providers/icloudDocuments.js +28 -1
  62. package/lib/module/providers/icloudDocuments.js.map +1 -1
  63. package/lib/module/providers/icloudKV.js +13 -3
  64. package/lib/module/providers/icloudKV.js.map +1 -1
  65. package/lib/module/providers/icloudKV.web.js +8 -1
  66. package/lib/module/providers/icloudKV.web.js.map +1 -1
  67. package/lib/module/providers/memory.js +10 -0
  68. package/lib/module/providers/memory.js.map +1 -1
  69. package/lib/module/resolvers.js +27 -12
  70. package/lib/module/resolvers.js.map +1 -1
  71. package/lib/module/specs/NativeRNCloudSync.js +4 -0
  72. package/lib/module/specs/NativeRNCloudSync.js.map +1 -1
  73. package/lib/module/store.js +360 -173
  74. package/lib/module/store.js.map +1 -1
  75. package/lib/module/types.js.map +1 -1
  76. package/lib/typescript/commonjs/errors.d.ts +16 -16
  77. package/lib/typescript/commonjs/errors.d.ts.map +1 -1
  78. package/lib/typescript/commonjs/hooks.d.ts.map +1 -1
  79. package/lib/typescript/commonjs/internal/autoFlush.d.ts.map +1 -1
  80. package/lib/typescript/commonjs/internal/base64.d.ts.map +1 -1
  81. package/lib/typescript/commonjs/internal/cloudKitRest.d.ts +20 -0
  82. package/lib/typescript/commonjs/internal/cloudKitRest.d.ts.map +1 -1
  83. package/lib/typescript/commonjs/internal/googleDriveRest.d.ts +19 -0
  84. package/lib/typescript/commonjs/internal/googleDriveRest.d.ts.map +1 -1
  85. package/lib/typescript/commonjs/internal/keys.d.ts +18 -3
  86. package/lib/typescript/commonjs/internal/keys.d.ts.map +1 -1
  87. package/lib/typescript/commonjs/internal/records.d.ts +24 -0
  88. package/lib/typescript/commonjs/internal/records.d.ts.map +1 -0
  89. package/lib/typescript/commonjs/providers/cloudKit.d.ts +1 -1
  90. package/lib/typescript/commonjs/providers/cloudKit.d.ts.map +1 -1
  91. package/lib/typescript/commonjs/providers/cloudKitBackup.d.ts +1 -1
  92. package/lib/typescript/commonjs/providers/cloudKitBackup.d.ts.map +1 -1
  93. package/lib/typescript/commonjs/providers/cloudKitEncrypted.d.ts.map +1 -1
  94. package/lib/typescript/commonjs/providers/googleDrive.d.ts.map +1 -1
  95. package/lib/typescript/commonjs/providers/googleDriveFiles.d.ts.map +1 -1
  96. package/lib/typescript/commonjs/providers/icloudDocuments.d.ts +1 -1
  97. package/lib/typescript/commonjs/providers/icloudDocuments.d.ts.map +1 -1
  98. package/lib/typescript/commonjs/providers/icloudKV.d.ts.map +1 -1
  99. package/lib/typescript/commonjs/providers/icloudKV.web.d.ts.map +1 -1
  100. package/lib/typescript/commonjs/providers/memory.d.ts.map +1 -1
  101. package/lib/typescript/commonjs/resolvers.d.ts +6 -3
  102. package/lib/typescript/commonjs/resolvers.d.ts.map +1 -1
  103. package/lib/typescript/commonjs/specs/NativeRNCloudSync.d.ts +12 -3
  104. package/lib/typescript/commonjs/specs/NativeRNCloudSync.d.ts.map +1 -1
  105. package/lib/typescript/commonjs/store.d.ts.map +1 -1
  106. package/lib/typescript/commonjs/types.d.ts +31 -3
  107. package/lib/typescript/commonjs/types.d.ts.map +1 -1
  108. package/lib/typescript/module/errors.d.ts +16 -16
  109. package/lib/typescript/module/errors.d.ts.map +1 -1
  110. package/lib/typescript/module/hooks.d.ts.map +1 -1
  111. package/lib/typescript/module/internal/autoFlush.d.ts.map +1 -1
  112. package/lib/typescript/module/internal/base64.d.ts.map +1 -1
  113. package/lib/typescript/module/internal/cloudKitRest.d.ts +20 -0
  114. package/lib/typescript/module/internal/cloudKitRest.d.ts.map +1 -1
  115. package/lib/typescript/module/internal/googleDriveRest.d.ts +19 -0
  116. package/lib/typescript/module/internal/googleDriveRest.d.ts.map +1 -1
  117. package/lib/typescript/module/internal/keys.d.ts +18 -3
  118. package/lib/typescript/module/internal/keys.d.ts.map +1 -1
  119. package/lib/typescript/module/internal/records.d.ts +24 -0
  120. package/lib/typescript/module/internal/records.d.ts.map +1 -0
  121. package/lib/typescript/module/providers/cloudKit.d.ts +1 -1
  122. package/lib/typescript/module/providers/cloudKit.d.ts.map +1 -1
  123. package/lib/typescript/module/providers/cloudKitBackup.d.ts +1 -1
  124. package/lib/typescript/module/providers/cloudKitBackup.d.ts.map +1 -1
  125. package/lib/typescript/module/providers/cloudKitEncrypted.d.ts.map +1 -1
  126. package/lib/typescript/module/providers/googleDrive.d.ts.map +1 -1
  127. package/lib/typescript/module/providers/googleDriveFiles.d.ts.map +1 -1
  128. package/lib/typescript/module/providers/icloudDocuments.d.ts +1 -1
  129. package/lib/typescript/module/providers/icloudDocuments.d.ts.map +1 -1
  130. package/lib/typescript/module/providers/icloudKV.d.ts.map +1 -1
  131. package/lib/typescript/module/providers/icloudKV.web.d.ts.map +1 -1
  132. package/lib/typescript/module/providers/memory.d.ts.map +1 -1
  133. package/lib/typescript/module/resolvers.d.ts +6 -3
  134. package/lib/typescript/module/resolvers.d.ts.map +1 -1
  135. package/lib/typescript/module/specs/NativeRNCloudSync.d.ts +12 -3
  136. package/lib/typescript/module/specs/NativeRNCloudSync.d.ts.map +1 -1
  137. package/lib/typescript/module/store.d.ts.map +1 -1
  138. package/lib/typescript/module/types.d.ts +31 -3
  139. package/lib/typescript/module/types.d.ts.map +1 -1
  140. package/package.json +35 -25
  141. package/plugin/build/index.js +45 -13
  142. package/react-native-cloud-sync.podspec +10 -3
  143. package/src/hooks.ts +69 -9
  144. package/src/internal/autoFlush.ts +10 -1
  145. package/src/internal/base64.ts +26 -5
  146. package/src/internal/cloudKitRest.ts +103 -14
  147. package/src/internal/googleDriveRest.ts +338 -37
  148. package/src/internal/keys.ts +42 -16
  149. package/src/internal/records.ts +38 -0
  150. package/src/providers/cloudKit.ts +96 -20
  151. package/src/providers/cloudKitEncrypted.ts +37 -8
  152. package/src/providers/googleDrive.ts +38 -2
  153. package/src/providers/icloudDocuments.ts +40 -2
  154. package/src/providers/icloudKV.ts +13 -3
  155. package/src/providers/icloudKV.web.ts +8 -2
  156. package/src/providers/memory.ts +8 -0
  157. package/src/resolvers.ts +28 -13
  158. package/src/specs/NativeRNCloudSync.ts +16 -2
  159. package/src/store.ts +433 -190
  160. package/src/types.ts +32 -3
package/src/store.ts CHANGED
@@ -6,6 +6,7 @@ import { byteLength } from './internal/bytes'
6
6
  import { assertKey } from './internal/keys'
7
7
  import { withTimeout } from './internal/timeout'
8
8
  import { attachAutoFlush } from './internal/autoFlush'
9
+ import { clampRetryAfterMs, setOwn } from './internal/records'
9
10
  import { CloudSyncError, ErrorCode, isRetryable, normalizeError } from './errors'
10
11
  import {
11
12
  DEFAULT_TIERING,
@@ -20,6 +21,7 @@ import {
20
21
  type QuotaInfo,
21
22
  type RemoteChangeEvent,
22
23
  type ResolveCandidate,
24
+ type ResolveFn,
23
25
  type TieringConfig,
24
26
  type Unsubscribe,
25
27
  type ValueCodec,
@@ -45,20 +47,16 @@ export interface OutboxStorage {
45
47
  const OUTBOX_KEY = 'rncs.outbox.v1'
46
48
 
47
49
  /**
48
- * Identity of a queued write: one entry per (provider, key).
50
+ * Providers whose queued writes are never handed to `outboxStorage`.
49
51
  *
50
- * A single helper rather than a template literal at each site, because the two
51
- * places that build it have to agree exactly - when they did not, a flush
52
- * failed to recognise the entries it had just processed and wrote every one of
53
- * them straight back, so a drained queue never shrank.
54
- *
55
- * The separator is an escaped NUL, which cannot appear in a provider name or a
56
- * key, so the two halves can never run together and make two different pairs
57
- * collide.
52
+ * `cloudKitEncrypted` encrypts on device, inside the native layer, so the value
53
+ * the store hands it is the app's plaintext. Persisting a failed write to the
54
+ * durable outbox - typically an unencrypted MMKV or AsyncStorage file - would
55
+ * leave exactly the data the app chose end-to-end encryption for sitting on
56
+ * disk in the clear, for up to `outboxMaxAgeMs`. Those entries are queued in
57
+ * memory instead: retried for as long as the process lives, lost on restart.
58
58
  */
59
- function outboxId(provider: ProviderName, key: string): string {
60
- return `${provider}\u0000${key}`
61
- }
59
+ const MEMORY_ONLY_OUTBOX_PROVIDERS: ReadonlySet<string> = new Set(['cloudKitEncrypted'])
62
60
 
63
61
  const DEFAULT_OUTBOX_MAX_ENTRIES = 1000
64
62
  const DEFAULT_OUTBOX_MAX_ATTEMPTS = 12
@@ -306,20 +304,55 @@ export function createCloudStore(
306
304
 
307
305
  // ---------------------------------------------------------------- outbox
308
306
 
307
+ /** Queued writes that must not reach durable storage - see {@link MEMORY_ONLY_OUTBOX_PROVIDERS}. */
308
+ let volatileOutbox: OutboxEntry[] = []
309
+
309
310
  function readOutbox(): OutboxEntry[] {
311
+ return [...readDurableOutbox(), ...volatileOutbox]
312
+ }
313
+
314
+ function readDurableOutbox(): OutboxEntry[] {
310
315
  const raw = storage.getString(OUTBOX_KEY)
311
316
  if (raw == null) return []
312
317
  try {
313
318
  const parsed: unknown = JSON.parse(raw)
314
- return Array.isArray(parsed) ? (parsed as OutboxEntry[]) : []
319
+ // Entry by entry: one malformed record in a shared storage file must not
320
+ // throw on every later write path that reads the queue.
321
+ return Array.isArray(parsed)
322
+ ? parsed.filter(isOutboxEntry).filter(e => !MEMORY_ONLY_OUTBOX_PROVIDERS.has(e.provider))
323
+ : []
315
324
  }
316
325
  catch {
317
326
  return []
318
327
  }
319
328
  }
320
329
 
330
+ /**
331
+ * Bumped on every rewrite, so a flush can tell cheaply whether the queue
332
+ * changed under it while it was awaiting the network.
333
+ */
334
+ let outboxVersion = 0
335
+
321
336
  function writeOutbox(entries: OutboxEntry[]): void {
322
- storage.set(OUTBOX_KEY, JSON.stringify(entries))
337
+ outboxVersion += 1
338
+ volatileOutbox = entries.filter(e => MEMORY_ONLY_OUTBOX_PROVIDERS.has(e.provider))
339
+ storage.set(
340
+ OUTBOX_KEY,
341
+ JSON.stringify(entries.filter(e => !MEMORY_ONLY_OUTBOX_PROVIDERS.has(e.provider)))
342
+ )
343
+ }
344
+
345
+ // An earlier version persisted these entries like any other. Move them into
346
+ // memory and rewrite the durable copy without them, so plaintext a previous
347
+ // run left on disk does not outlive this one.
348
+ try {
349
+ const raw = storage.getString(OUTBOX_KEY)
350
+ const parsed: unknown = raw == null ? null : JSON.parse(raw)
351
+ if (Array.isArray(parsed) && parsed.some(e => isOutboxEntry(e) && MEMORY_ONLY_OUTBOX_PROVIDERS.has(e.provider)))
352
+ writeOutbox(parsed.filter(isOutboxEntry))
353
+ }
354
+ catch {
355
+ // Unreadable storage is handled - as an empty queue - by readOutbox.
323
356
  }
324
357
 
325
358
  function reportDropped(entry: OutboxEntry, reason: DropReason, error?: CloudSyncError): void {
@@ -378,6 +411,55 @@ export function createCloudStore(
378
411
  if (kept.length !== entries.length) writeOutbox(kept)
379
412
  }
380
413
 
414
+ // ------------------------------------------------------- per-key ordering
415
+
416
+ /**
417
+ * Serialises every operation on one (provider, key) pair.
418
+ *
419
+ * Without it a flush could be re-sending a queued v1 while a newer direct
420
+ * `setItem(k, v2)` ran alongside it, and whichever request the server saw
421
+ * last won - so v1 could land on top of v2. Holding the pair for the whole
422
+ * of "send, then update the outbox" makes the two strictly ordered: either
423
+ * the flush sends v1 first and v2 lands after it, or v2 lands first, drops
424
+ * the queued v1, and the flush finds nothing left to send.
425
+ *
426
+ * A batch takes every pair it touches at once. Waiters only ever wait on
427
+ * holders registered before them, so this cannot deadlock - but nothing
428
+ * holding a pair may try to take the same pair again.
429
+ *
430
+ * With `timeoutMs`, a timed-out request releases the pair while the request
431
+ * itself may still be in flight; a promise cannot be cancelled.
432
+ */
433
+ const keyLocks = new Map<string, Promise<void>>()
434
+
435
+ async function withKeys<T>(
436
+ provider: ProviderName,
437
+ keys: string[],
438
+ fn: () => Promise<T>
439
+ ): Promise<T> {
440
+ const ids = [...new Set(keys)].map(k => `${provider}\u0000${k}`)
441
+ const before: Promise<void>[] = []
442
+ for (const id of ids) {
443
+ const held = keyLocks.get(id)
444
+ if (held != null) before.push(held)
445
+ }
446
+
447
+ let release!: () => void
448
+ const mine = new Promise<void>((resolve) => {
449
+ release = resolve
450
+ })
451
+ for (const id of ids) keyLocks.set(id, mine)
452
+
453
+ try {
454
+ if (before.length > 0) await Promise.all(before)
455
+ return await fn()
456
+ }
457
+ finally {
458
+ release()
459
+ for (const id of ids) if (keyLocks.get(id) === mine) keyLocks.delete(id)
460
+ }
461
+ }
462
+
381
463
  // ------------------------------------------------------------ size rules
382
464
 
383
465
  /**
@@ -390,7 +472,9 @@ export function createCloudStore(
390
472
  function limitFor(provider: CloudProvider): number | null {
391
473
  if (tiering == null) return null
392
474
  if (provider.name === 'icloudKV') return tiering.kvMaxBytes
393
- if (provider.name === 'cloudKit') return tiering.recordMaxBytes
475
+ // Encrypted records are still CKRecords, under the same 1 MB ceiling.
476
+ if (provider.name === 'cloudKit' || provider.name === 'cloudKitEncrypted')
477
+ return tiering.recordMaxBytes
394
478
  return null
395
479
  }
396
480
 
@@ -447,6 +531,29 @@ export function createCloudStore(
447
531
  return out
448
532
  }
449
533
 
534
+ /**
535
+ * A batched read for a resolver, with server timestamps when the resolver
536
+ * uses them - so `multiGet` resolves exactly as `getItem` does. Stays one
537
+ * request per provider when the resolver ignores timestamps, or when the
538
+ * provider can batch them.
539
+ */
540
+ async function readBatchForResolve(
541
+ p: CloudProvider,
542
+ keys: string[],
543
+ withMeta: boolean
544
+ ): Promise<(ItemWithMeta | null)[]> {
545
+ if (withMeta) {
546
+ if (p.multiGetWithMeta != null) return await run(p, () => p.multiGetWithMeta!(keys))
547
+ if (p.getItemWithMeta != null) {
548
+ const out: (ItemWithMeta | null)[] = []
549
+ for (const key of keys) out.push(await readOne(p, key))
550
+ return out
551
+ }
552
+ }
553
+ const values = await readBatch(p, keys)
554
+ return values.map(value => (value == null ? null : { value }))
555
+ }
556
+
450
557
  async function writeBatch(p: CloudProvider, entries: [string, string][]): Promise<void> {
451
558
  if (entries.length === 0) return
452
559
  if (p.multiSet != null) {
@@ -467,22 +574,57 @@ export function createCloudStore(
467
574
 
468
575
  // ---------------------------------------------------------------- writes
469
576
 
470
- /** Single-destination write, queueing a retryable failure. */
471
- async function writeOne(target: CloudProvider, key: string, value: string): Promise<void> {
472
- try {
473
- await run(target, () => target.setItem(key, value))
474
- // This write supersedes anything queued for the same destination.
475
- dequeue(key, target.name)
476
- }
477
- catch (e) {
478
- const err = normalizeError(e, target.name)
479
- if (useOutbox && isRetryable(err)) {
480
- enqueue(key, value, target.name, err.code)
577
+ /**
578
+ * Single-destination write, queueing a retryable failure. Resolves true when
579
+ * the value landed, false when it was queued instead.
580
+ */
581
+ async function writeOne(target: CloudProvider, key: string, value: string): Promise<boolean> {
582
+ return await withKeys(target.name, [key], async () => {
583
+ try {
584
+ await run(target, () => target.setItem(key, value))
585
+ // This write supersedes anything queued for the same destination.
586
+ dequeue(key, target.name)
587
+ return true
588
+ }
589
+ catch (e) {
590
+ const err = normalizeError(e, target.name)
591
+ if (useOutbox && isRetryable(err)) {
592
+ enqueue(key, value, target.name, err.code)
593
+ options.onError?.(err)
594
+ return false
595
+ }
596
+ throw err
597
+ }
598
+ })
599
+ }
600
+
601
+ /**
602
+ * One destination's share of a multi-destination operation, under that
603
+ * destination's lock. Resolves 'done' or 'queued', or the fatal error -
604
+ * already reported through `onError` - for the caller to weigh.
605
+ */
606
+ async function attempt(
607
+ target: CloudProvider,
608
+ keys: string[],
609
+ send: () => Promise<void>,
610
+ queued: (string | null)[]
611
+ ): Promise<'done' | 'queued' | CloudSyncError> {
612
+ return await withKeys(target.name, keys, async () => {
613
+ try {
614
+ await run(target, send)
615
+ dequeueMany(keys, target.name)
616
+ return 'done' as const
617
+ }
618
+ catch (e) {
619
+ const err = normalizeError(e, target.name)
481
620
  options.onError?.(err)
482
- return
621
+ if (useOutbox && isRetryable(err)) {
622
+ keys.forEach((key, i) => enqueue(key, queued[i], target.name, err.code))
623
+ return 'queued' as const
624
+ }
625
+ return err
483
626
  }
484
- throw err
485
- }
627
+ })
486
628
  }
487
629
 
488
630
  /**
@@ -502,15 +644,17 @@ export function createCloudStore(
502
644
  ): Promise<void> {
503
645
  for (const target of targets) {
504
646
  if (canHold(target, value)) continue
505
- try {
506
- await run(target, () => target.removeItem(key))
507
- dequeue(key, target.name)
508
- }
509
- catch (e) {
510
- // Best effort. A stale copy left behind is reported, not fatal - the
511
- // value itself did land somewhere.
512
- options.onError?.(normalizeError(e, target.name))
513
- }
647
+ await withKeys(target.name, [key], async () => {
648
+ try {
649
+ await run(target, () => target.removeItem(key))
650
+ dequeue(key, target.name)
651
+ }
652
+ catch (e) {
653
+ // Best effort. A stale copy left behind is reported, not fatal - the
654
+ // value itself did land somewhere.
655
+ options.onError?.(normalizeError(e, target.name))
656
+ }
657
+ })
514
658
  }
515
659
  }
516
660
 
@@ -547,22 +691,11 @@ export function createCloudStore(
547
691
  let stored = 0
548
692
  let fatal: CloudSyncError | null = null
549
693
 
550
- for (const target of eligible)
551
- try {
552
- await run(target, () => target.setItem(key, value))
553
- dequeue(key, target.name)
554
- stored += 1
555
- }
556
- catch (e) {
557
- const err = normalizeError(e, target.name)
558
- if (useOutbox && isRetryable(err)) {
559
- enqueue(key, value, target.name, err.code)
560
- options.onError?.(err)
561
- continue
562
- }
563
- fatal ??= err
564
- options.onError?.(err)
565
- }
694
+ for (const target of eligible) {
695
+ const outcome = await attempt(target, [key], () => target.setItem(key, value), [value])
696
+ if (outcome === 'done') stored += 1
697
+ else if (outcome !== 'queued') fatal ??= outcome
698
+ }
566
699
 
567
700
  // Only once the value has a home somewhere: dropping the old copy first
568
701
  // would open a window in which the key exists nowhere.
@@ -583,22 +716,11 @@ export function createCloudStore(
583
716
  let removed = 0
584
717
  let fatal: CloudSyncError | null = null
585
718
 
586
- for (const target of targets)
587
- try {
588
- await run(target, () => target.removeItem(key))
589
- dequeue(key, target.name)
590
- removed += 1
591
- }
592
- catch (e) {
593
- const err = normalizeError(e, target.name)
594
- if (useOutbox && isRetryable(err)) {
595
- enqueue(key, null, target.name, err.code)
596
- options.onError?.(err)
597
- continue
598
- }
599
- fatal ??= err
600
- options.onError?.(err)
601
- }
719
+ for (const target of targets) {
720
+ const outcome = await attempt(target, [key], () => target.removeItem(key), [null])
721
+ if (outcome === 'done') removed += 1
722
+ else if (outcome !== 'queued') fatal ??= outcome
723
+ }
602
724
 
603
725
  if (removed === 0 && fatal != null) throw fatal
604
726
  }
@@ -609,8 +731,21 @@ export function createCloudStore(
609
731
  *
610
732
  * A provider the winner no longer fits in has its stale copy removed instead
611
733
  * of being silently left holding it - see {@link evictWhereTooLarge}.
734
+ *
735
+ * The outbox is consulted per provider, under that provider's lock for the
736
+ * key. A queued write there is this device's own pending value, so it is
737
+ * weighed as one more candidate:
738
+ *
739
+ * - if the resolver still prefers the winner, the queued write is older
740
+ * news - it is dropped once the repair lands, or the next flush would
741
+ * put it back on top of the repaired value;
742
+ * - if the resolver prefers the queued value, that provider is left alone
743
+ * and the outbox delivers it;
744
+ * - a queued delete is left to land, since repairing would resurrect a
745
+ * value the user removed.
612
746
  */
613
747
  async function repair(
748
+ resolve: ResolveFn,
614
749
  key: string,
615
750
  winner: string,
616
751
  candidates: ResolveCandidate[],
@@ -619,27 +754,55 @@ export function createCloudStore(
619
754
  const byProvider = new Map(candidates.map(c => [c.provider, c.value]))
620
755
  const stored = await encode(key, winner)
621
756
 
622
- for (const [name, provider] of reachable) {
623
- if (byProvider.get(name) === winner) continue
624
-
625
- if (!canHold(provider, stored)) {
626
- if (!byProvider.has(name)) continue
757
+ for (const [name, provider] of reachable)
758
+ await withKeys(name, [key], async () => {
627
759
  try {
628
- await run(provider, () => provider.removeItem(key))
760
+ const queued = readOutbox().find(e => e.provider === name && e.key === key)
761
+ if (queued != null) {
762
+ if (queued.value == null) return
763
+ const pending = await decode(key, queued.value)
764
+ const pick = resolve([
765
+ ...candidates,
766
+ { provider: name, value: pending, modifiedAt: queued.enqueuedAt },
767
+ ])
768
+ if (pick !== winner) return
769
+ }
770
+
771
+ if (byProvider.get(name) !== winner)
772
+ if (canHold(provider, stored)) {
773
+ await run(provider, () => provider.setItem(key, stored))
774
+ }
775
+ else {
776
+ if (!byProvider.has(name)) return
777
+ await run(provider, () => provider.removeItem(key))
778
+ }
779
+
780
+ if (queued != null) dequeue(key, name)
629
781
  }
630
782
  catch (e) {
631
783
  options.onError?.(normalizeError(e, name))
632
784
  }
633
- continue
634
- }
785
+ })
786
+ }
635
787
 
636
- try {
637
- await run(provider, () => provider.setItem(key, stored))
638
- }
639
- catch (e) {
640
- options.onError?.(normalizeError(e, name))
641
- }
642
- }
788
+ /** Resolves one key's candidates and, if configured, starts read repair. */
789
+ function settle(
790
+ resolve: ResolveFn,
791
+ key: string,
792
+ candidates: ResolveCandidate[],
793
+ reachable: Map<ProviderName, CloudProvider>
794
+ ): string | null {
795
+ const winner = resolve(candidates)
796
+ if (winner == null) return null
797
+
798
+ // Best-effort convergence. Never fails the read - the caller already has
799
+ // the right answer, and a failed repair only costs another resolve later.
800
+ if (repairOnRead)
801
+ void repair(resolve, key, winner, candidates, reachable).catch((e: unknown) => {
802
+ options.onError?.(normalizeError(e))
803
+ })
804
+
805
+ return winner
643
806
  }
644
807
 
645
808
  // ---------------------------------------------------------------- events
@@ -813,13 +976,8 @@ export function createCloudStore(
813
976
  return { found: false, value: null, reachedAny }
814
977
  }
815
978
 
816
- const winner = resolveValue(candidates)
979
+ const winner = settle(resolveValue, key, candidates, reachable)
817
980
  if (winner == null) return { found: false, value: null, reachedAny }
818
-
819
- // Best-effort convergence. Never fails the read - the caller already has
820
- // the right answer, and a failed repair only costs another resolve later.
821
- if (repairOnRead) void repair(key, winner, candidates, reachable)
822
-
823
981
  return { found: true, value: winner, reachedAny }
824
982
  }
825
983
 
@@ -841,8 +999,13 @@ export function createCloudStore(
841
999
  const stored = await encode(key, value)
842
1000
 
843
1001
  if (writeMode === 'failover') {
844
- const target = await providerForSize(stored, await primary())
845
- await writeOne(target, key, stored)
1002
+ const preferred = await primary()
1003
+ const target = await providerForSize(stored, preferred)
1004
+ const landed = await writeOne(target, key, stored)
1005
+ // Tiering routed the value past the preferred provider, which may still
1006
+ // hold an older, smaller copy - and reads reach that one first. Without
1007
+ // this, a value that grows past `kvMaxBytes` reads back stale forever.
1008
+ if (landed && target !== preferred) await evictWhereTooLarge([preferred], key, stored)
846
1009
  return
847
1010
  }
848
1011
  await writeMany(await allAvailable(), key, stored)
@@ -850,11 +1013,12 @@ export function createCloudStore(
850
1013
 
851
1014
  removeItem: async (key: string) => {
852
1015
  checkKey(key)
853
- // A mirrored write must be a mirrored delete. Removing from only the
854
- // preferred provider would leave a copy behind that reads then fall
855
- // through to and resurrect.
856
- const targets = writeMode === 'mirror' ? await allAvailable() : [await primary()]
857
- await removeMany(targets, key)
1016
+ // A delete has to reach every provider reads fall through to, in both
1017
+ // modes. Mirror wrote everywhere; failover may have written to a later
1018
+ // provider too - because the preferred one was down at the time, or
1019
+ // because tiering routed a large value past it. Removing from only the
1020
+ // preferred provider leaves that copy behind for reads to resurrect.
1021
+ await removeMany(await allAvailable(), key)
858
1022
  },
859
1023
 
860
1024
  getAllKeys: async () => {
@@ -883,19 +1047,31 @@ export function createCloudStore(
883
1047
  // batched request rather than one per key.
884
1048
  if (resolveValue != null) {
885
1049
  const perKey = new Map<string, ResolveCandidate[]>(keys.map(k => [k, []]))
1050
+ const reachable = new Map<ProviderName, CloudProvider>()
1051
+ // Same candidates `getItem` would build, timestamps included, so the
1052
+ // two reads cannot resolve the same key differently.
1053
+ const withMeta = resolveValue.usesModifiedAt !== false
886
1054
  let reachedAny = false
887
1055
 
888
1056
  for (const name of options.providers) {
889
1057
  const p = resolveProvider(name)
890
1058
  try {
891
1059
  if (!(await isAvailable(p))) continue
892
- const values = await readBatch(p, keys)
893
- reachedAny = true
1060
+ const hits = await readBatchForResolve(p, keys, withMeta)
1061
+ const found: [string, ResolveCandidate][] = []
894
1062
  for (let i = 0; i < keys.length; i += 1) {
895
- const raw = values[i]
896
- if (raw == null) continue
897
- perKey.get(keys[i])!.push({ provider: p.name, value: await decode(keys[i], raw) })
1063
+ const hit = hits[i]
1064
+ if (hit == null) continue
1065
+ found.push([keys[i], {
1066
+ provider: p.name,
1067
+ value: await decode(keys[i], hit.value),
1068
+ modifiedAt: hit.modifiedAt,
1069
+ }])
898
1070
  }
1071
+ // Only a provider read in full is a repair target - see readAcross.
1072
+ reachable.set(p.name, p)
1073
+ reachedAny = true
1074
+ for (const [key, candidate] of found) perKey.get(key)!.push(candidate)
899
1075
  }
900
1076
  catch {
901
1077
  // Same rule as `getAllKeys`: one provider failing to answer must not
@@ -906,7 +1082,10 @@ export function createCloudStore(
906
1082
  if (!reachedAny) throw noneAvailable()
907
1083
  for (const key of keys) {
908
1084
  const candidates = perKey.get(key)!
909
- answers.set(key, candidates.length === 0 ? null : resolveValue(candidates))
1085
+ answers.set(
1086
+ key,
1087
+ candidates.length === 0 ? null : settle(resolveValue, key, candidates, reachable)
1088
+ )
910
1089
  }
911
1090
  return keys.map(k => [k, answers.get(k) ?? null] as [string, string | null])
912
1091
  }
@@ -961,20 +1140,20 @@ export function createCloudStore(
961
1140
  byTarget.set(target.name, bucket)
962
1141
  }
963
1142
 
964
- for (const { provider, items } of byTarget.values())
965
- try {
966
- await writeBatch(provider, items)
967
- dequeueMany(items.map(([k]) => k), provider.name)
968
- }
969
- catch (e) {
970
- const err = normalizeError(e, provider.name)
971
- if (useOutbox && isRetryable(err)) {
972
- for (const [key, value] of items) enqueue(key, value, provider.name, err.code)
973
- options.onError?.(err)
974
- continue
975
- }
976
- throw err
977
- }
1143
+ for (const { provider, items } of byTarget.values()) {
1144
+ const outcome = await attempt(
1145
+ provider,
1146
+ items.map(([k]) => k),
1147
+ () => writeBatch(provider, items),
1148
+ items.map(([, v]) => v)
1149
+ )
1150
+ if (outcome === 'queued') continue
1151
+ if (outcome !== 'done') throw outcome
1152
+ // Same stale-copy rule as `setItem`: values routed past the
1153
+ // preferred provider must not leave an older copy there.
1154
+ if (provider !== preferred)
1155
+ for (const [key, value] of items) await evictWhereTooLarge([preferred], key, value)
1156
+ }
978
1157
  return
979
1158
  }
980
1159
 
@@ -987,21 +1166,14 @@ export function createCloudStore(
987
1166
  for (const target of targets) {
988
1167
  const fits = encoded.filter(([, value]) => canHold(target, value))
989
1168
  if (fits.length === 0) continue
990
- try {
991
- await writeBatch(target, fits)
992
- dequeueMany(fits.map(([k]) => k), target.name)
993
- anyStored = true
994
- }
995
- catch (e) {
996
- const err = normalizeError(e, target.name)
997
- if (useOutbox && isRetryable(err)) {
998
- for (const [key, value] of fits) enqueue(key, value, target.name, err.code)
999
- options.onError?.(err)
1000
- continue
1001
- }
1002
- fatal ??= err
1003
- options.onError?.(err)
1004
- }
1169
+ const outcome = await attempt(
1170
+ target,
1171
+ fits.map(([k]) => k),
1172
+ () => writeBatch(target, fits),
1173
+ fits.map(([, v]) => v)
1174
+ )
1175
+ if (outcome === 'done') anyStored = true
1176
+ else if (outcome !== 'queued') fatal ??= outcome
1005
1177
  }
1006
1178
 
1007
1179
  if (anyStored)
@@ -1022,28 +1194,23 @@ export function createCloudStore(
1022
1194
  for (const key of keys) checkKey(key)
1023
1195
  if (keys.length === 0) return
1024
1196
 
1025
- const targets = writeMode === 'mirror' ? await allAvailable() : [await primary()]
1197
+ // Every available provider, in both modes - see `removeItem`.
1198
+ const targets = await allAvailable()
1026
1199
  if (targets.length === 0) throw noneAvailable()
1027
1200
 
1028
1201
  let removedAnywhere = false
1029
1202
  let fatal: CloudSyncError | null = null
1030
1203
 
1031
- for (const target of targets)
1032
- try {
1033
- await removeBatch(target, keys)
1034
- dequeueMany(keys, target.name)
1035
- removedAnywhere = true
1036
- }
1037
- catch (e) {
1038
- const err = normalizeError(e, target.name)
1039
- if (useOutbox && isRetryable(err)) {
1040
- for (const key of keys) enqueue(key, null, target.name, err.code)
1041
- options.onError?.(err)
1042
- continue
1043
- }
1044
- fatal ??= err
1045
- options.onError?.(err)
1046
- }
1204
+ for (const target of targets) {
1205
+ const outcome = await attempt(
1206
+ target,
1207
+ keys,
1208
+ () => removeBatch(target, keys),
1209
+ keys.map(() => null)
1210
+ )
1211
+ if (outcome === 'done') removedAnywhere = true
1212
+ else if (outcome !== 'queued') fatal ??= outcome
1213
+ }
1047
1214
 
1048
1215
  if (!removedAnywhere && fatal != null) throw fatal
1049
1216
  },
@@ -1060,8 +1227,10 @@ export function createCloudStore(
1060
1227
  if (keys.length === 0) return {}
1061
1228
 
1062
1229
  const out: Record<string, string> = {}
1230
+ // Keys come from provider listings; a file named `__proto__` must land
1231
+ // as an entry, not be swallowed by the prototype setter.
1063
1232
  for (const [key, value] of await store.multiGet(keys))
1064
- if (value != null) out[key] = value
1233
+ if (value != null) setOwn(out, key, value)
1065
1234
 
1066
1235
  return out
1067
1236
  },
@@ -1097,18 +1266,21 @@ export function createCloudStore(
1097
1266
  // on the way out only to re-encode on the way in would be wasted work -
1098
1267
  // and, for a codec with a random nonce, would rewrite every byte.
1099
1268
  for (const key of keys) {
1269
+ // Which end failed, so the error names the provider that raised it.
1270
+ let side: ProviderName = from
1100
1271
  try {
1101
1272
  const value = await run(src, () => src.getItem(key))
1102
1273
  if (value == null) {
1103
1274
  skipped.push(key)
1104
1275
  }
1105
1276
  else {
1106
- await run(dst, () => dst.setItem(key, value))
1277
+ side = to
1278
+ await withKeys(to, [key], () => run(dst, () => dst.setItem(key, value)))
1107
1279
  copied.push(key)
1108
1280
  }
1109
1281
  }
1110
1282
  catch (e) {
1111
- const err = normalizeError(e, from)
1283
+ const err = normalizeError(e, side)
1112
1284
  failed.push({ key, error: err })
1113
1285
  options.onError?.(err)
1114
1286
  // Aborting on the first bad key leaves the user half migrated with no
@@ -1199,25 +1371,52 @@ export function createCloudStore(
1199
1371
  if (entries.length === 0) return { drained: 0, remaining: 0, dropped: [] }
1200
1372
 
1201
1373
  const now = Date.now()
1202
- const keep: OutboxEntry[] = []
1374
+ // Each kept entry alongside the snapshot entry it came from, so the final
1375
+ // merge can check that the original is still the one queued.
1376
+ const keep: { entry: OutboxEntry; from: OutboxEntry }[] = []
1203
1377
  const dropped: DroppedWrite[] = []
1204
- const handled = new Set<string>()
1205
1378
  let drained = 0
1206
1379
 
1380
+ // The queue can change while this flush awaits the network: a newer write
1381
+ // supersedes an entry (dequeue or replace), `discardPendingWrites` removes
1382
+ // one, or an account switch empties it. An entry that is no longer queued
1383
+ // must not be sent - after an account switch that would write the previous
1384
+ // user's data into the new account - nor written back at the end.
1385
+ let seenVersion = outboxVersion
1386
+ let current = entries
1387
+ function stillQueued(entry: OutboxEntry): boolean {
1388
+ if (outboxVersion !== seenVersion) {
1389
+ current = readOutbox()
1390
+ seenVersion = outboxVersion
1391
+ }
1392
+ return current.some(e => sameEntry(e, entry))
1393
+ }
1394
+
1207
1395
  function drop(entry: OutboxEntry, reason: DropReason, error?: CloudSyncError): void {
1208
1396
  dropped.push({ entry, reason, error })
1209
1397
  reportDropped(entry, reason, error)
1210
1398
  }
1211
1399
 
1400
+ /**
1401
+ * Takes a delivered entry out of storage straight away, rather than at the
1402
+ * final merge, so nothing consulting the outbox meanwhile - read repair,
1403
+ * `pendingWrites()` - mistakes it for a write still waiting.
1404
+ */
1405
+ function removeExact(entry: OutboxEntry): void {
1406
+ const queued = readOutbox()
1407
+ const rest = queued.filter(e => !sameEntry(e, entry))
1408
+ if (rest.length !== queued.length) writeOutbox(rest)
1409
+ }
1410
+
1212
1411
  for (const entry of entries) {
1213
- handled.add(outboxId(entry.provider, entry.key))
1412
+ if (!stillQueued(entry)) continue
1214
1413
 
1215
1414
  if (now - entry.enqueuedAt > maxAgeMs) {
1216
1415
  drop(entry, 'expired')
1217
1416
  continue
1218
1417
  }
1219
1418
  if (entry.nextAttemptAt > now) {
1220
- keep.push(entry)
1419
+ keep.push({ entry, from: entry })
1221
1420
  continue
1222
1421
  }
1223
1422
 
@@ -1231,54 +1430,98 @@ export function createCloudStore(
1231
1430
  continue
1232
1431
  }
1233
1432
 
1234
- try {
1235
- if (entry.value == null) await run(p, () => p.removeItem(entry.key))
1236
- else await run(p, () => p.setItem(entry.key, entry.value!))
1237
- drained += 1
1238
- }
1239
- catch (e) {
1240
- const err = normalizeError(e, entry.provider)
1241
- options.onError?.(err)
1242
-
1243
- // A write only enters the queue for a retryable reason. If retrying it
1244
- // now fails for one the user must act on - quota, signed out, a payload
1245
- // the store will never accept - re-queueing turns it into a poison
1246
- // entry that retries forever, never drains, and never surfaces.
1247
- if (!isRetryable(err)) {
1248
- drop(entry, 'notRetryable', err)
1249
- continue
1433
+ const provider = p
1434
+ // Under the pair's lock, so a direct write to the same key cannot run
1435
+ // alongside this re-send - see `withKeys`. Checked again once the lock
1436
+ // is held: a direct write that got there first has dropped this entry.
1437
+ await withKeys(entry.provider, [entry.key], async () => {
1438
+ if (!stillQueued(entry)) return
1439
+ try {
1440
+ if (entry.value == null) await run(provider, () => provider.removeItem(entry.key))
1441
+ else await run(provider, () => provider.setItem(entry.key, entry.value!))
1442
+ drained += 1
1443
+ removeExact(entry)
1250
1444
  }
1251
-
1252
- const attempts = entry.attempts + 1
1253
- if (attempts >= maxAttempts) {
1254
- drop({ ...entry, attempts, lastErrorCode: err.code }, 'tooManyAttempts', err)
1255
- continue
1445
+ catch (e) {
1446
+ retryLater(entry, normalizeError(e, entry.provider))
1256
1447
  }
1448
+ })
1449
+ }
1450
+
1451
+ function retryLater(entry: OutboxEntry, err: CloudSyncError): void {
1452
+ options.onError?.(err)
1453
+
1454
+ // A write only enters the queue for a retryable reason. If retrying it
1455
+ // now fails for one the user must act on - quota, signed out, a payload
1456
+ // the store will never accept - re-queueing turns it into a poison
1457
+ // entry that retries forever, never drains, and never surfaces.
1458
+ if (!isRetryable(err)) {
1459
+ drop(entry, 'notRetryable', err)
1460
+ return
1461
+ }
1257
1462
 
1258
- // Exponential backoff, honouring a server-supplied retry hint when
1259
- // there is one (CloudKit sends `retryAfter`, Drive `Retry-After`).
1260
- const backoff = err.retryAfterMs ?? Math.min(2 ** attempts * 1000, 5 * 60_000)
1261
- keep.push({
1463
+ const attempts = entry.attempts + 1
1464
+ if (attempts >= maxAttempts) {
1465
+ drop({ ...entry, attempts, lastErrorCode: err.code }, 'tooManyAttempts', err)
1466
+ return
1467
+ }
1468
+
1469
+ // Exponential backoff, honouring a server-supplied retry hint when
1470
+ // there is one (CloudKit sends `retryAfter`, Drive `Retry-After`).
1471
+ // Clamped again here: a third-party provider can set any number.
1472
+ const backoff = clampRetryAfterMs(err.retryAfterMs) ?? Math.min(2 ** attempts * 1000, 5 * 60_000)
1473
+ keep.push({
1474
+ entry: {
1262
1475
  ...entry,
1263
1476
  attempts,
1264
1477
  nextAttemptAt: Date.now() + backoff,
1265
1478
  lastErrorCode: err.code,
1266
- })
1267
- }
1479
+ },
1480
+ from: entry,
1481
+ })
1268
1482
  }
1269
1483
 
1270
1484
  // Merge rather than overwrite. A `setItem` that failed while this flush was
1271
- // awaiting the network enqueued itself into storage; writing the snapshot
1272
- // back wholesale would silently discard it.
1273
- const arrived = readOutbox().filter(e => !handled.has(outboxId(e.provider, e.key)))
1274
- writeOutbox([...keep, ...arrived])
1275
-
1276
- return { drained, remaining: keep.length + arrived.length, dropped }
1485
+ // awaiting the network enqueued itself into storage - possibly replacing
1486
+ // an entry for the same key with a newer value - and writing the snapshot
1487
+ // back wholesale would silently discard it. Equally, a kept entry whose
1488
+ // original was removed meanwhile (superseded, discarded, account switch)
1489
+ // must stay removed.
1490
+ const latest = readOutbox()
1491
+ const kept = keep
1492
+ .filter(({ from }) => latest.some(e => sameEntry(e, from)))
1493
+ .map(({ entry }) => entry)
1494
+ const arrived = latest.filter(e => !entries.some(s => sameEntry(s, e)))
1495
+ writeOutbox([...kept, ...arrived])
1496
+
1497
+ return { drained, remaining: kept.length + arrived.length, dropped }
1277
1498
  }
1278
1499
 
1279
1500
  return store
1280
1501
  }
1281
1502
 
1503
+ /**
1504
+ * The same queued write, not merely one for the same key: a newer write to the
1505
+ * key replaces the entry with a fresh `enqueuedAt` and value.
1506
+ */
1507
+ function sameEntry(a: OutboxEntry, b: OutboxEntry): boolean {
1508
+ return a.provider === b.provider
1509
+ && a.key === b.key
1510
+ && a.enqueuedAt === b.enqueuedAt
1511
+ && a.value === b.value
1512
+ }
1513
+
1514
+ function isOutboxEntry(e: unknown): e is OutboxEntry {
1515
+ if (e == null || typeof e !== 'object') return false
1516
+ const o = e as Record<string, unknown>
1517
+ return typeof o.key === 'string'
1518
+ && typeof o.provider === 'string'
1519
+ && (o.value === null || typeof o.value === 'string')
1520
+ && typeof o.attempts === 'number'
1521
+ && typeof o.nextAttemptAt === 'number'
1522
+ && typeof o.enqueuedAt === 'number'
1523
+ }
1524
+
1282
1525
  function createMemoryOutboxStorage(): OutboxStorage {
1283
1526
  const map = new Map<string, string>()
1284
1527
  return {