zen-fs-config 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DESIGN.md +197 -17
- package/dist/index.d.mts +10 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +69 -4
- package/dist/index.mjs +69 -4
- package/package.json +1 -1
package/DESIGN.md
CHANGED
|
@@ -704,31 +704,206 @@ createConfigRepo('my-app', options?)
|
|
|
704
704
|
│
|
|
705
705
|
├─ 5. If options.backendInfo provided:
|
|
706
706
|
│ ├─ Generate replica ID (options.primaryBackendId or auto)
|
|
707
|
-
│
|
|
707
|
+
│ ├─ Dedup check: same type + options (stable key) already registered?
|
|
708
|
+
│ └─ Write descriptor to .meta/backends/{replicaId}.json (if not duplicate)
|
|
708
709
|
│
|
|
709
710
|
├─ 6. Read all backend descriptors from .meta/backends/
|
|
711
|
+
│ └─ Dedup: remove duplicates (same type + options, different ID)
|
|
712
|
+
│ ├─ Delete duplicate files on ALL replicas directly
|
|
713
|
+
│ └─ Create tombstone + delete local file
|
|
710
714
|
│
|
|
711
|
-
├─ 7. Determine nodeId (explicit
|
|
715
|
+
├─ 7. Determine nodeId (explicit parameter > auto-generated)
|
|
712
716
|
│
|
|
713
|
-
├─ 8. Create ConfigRepo
|
|
717
|
+
├─ 8. Create final ConfigRepo instance (primary = 'local-idb')
|
|
714
718
|
│
|
|
715
|
-
├─ 9. setupSync: for each backend
|
|
716
|
-
│ ├─ Create backend instance
|
|
717
|
-
│ ├─ Create SyncPair(IndexedDB
|
|
718
|
-
│
|
|
719
|
+
├─ 9. setupSync: for each replica backend:
|
|
720
|
+
│ ├─ Create backend instance (e.g., Gitee, RemoteStorage)
|
|
721
|
+
│ ├─ Create SyncPair(IndexedDB ↔ replica, bi-directional)
|
|
722
|
+
│ ├─ Register conflict handler
|
|
723
|
+
│ └─ NOTE: Does NOT call watch() yet (see §11.4 for why)
|
|
719
724
|
│
|
|
720
|
-
├─ 10. Load
|
|
721
|
-
│ For each referenced data-sync group:
|
|
722
|
-
│ ├─ Create data-sync backends (merge account fields if accountBackendId set)
|
|
723
|
-
│ ├─ Create SyncPair for each data backend
|
|
724
|
-
│ └─ syncEngine.watch(pairId)
|
|
725
|
+
├─ 10. Load config cache from IndexedDB (fast, local-only)
|
|
725
726
|
│
|
|
726
|
-
├─ 11.
|
|
727
|
+
├─ 11. initialSyncAndDedup() — only if replicas exist:
|
|
728
|
+
│ ├─ unwatchAll() — safety: clear any stale snapshots
|
|
729
|
+
│ ├─ syncAll() — full bidirectional sync (no cached snapshot
|
|
730
|
+
│ │ → every file is compared, remote-only files
|
|
731
|
+
│ │ are pulled to local)
|
|
732
|
+
│ ├─ readAllBackendDescriptors() — dedup duplicates pulled from remote
|
|
733
|
+
│ │ ├─ Delete dup files on ALL replicas directly
|
|
734
|
+
│ │ └─ Create tombstones for deduped descriptors
|
|
735
|
+
│ ├─ processTombstones() — delete deduped files on all replicas
|
|
736
|
+
│ └─ watchAll() — start monitoring for future changes
|
|
737
|
+
│ (snapshots now reflect the fully synced state)
|
|
727
738
|
│
|
|
728
|
-
|
|
739
|
+
├─ 12. syncMetaToReplicas() — background push of .meta/ changes
|
|
740
|
+
│ (watchers already running, this just speeds up initial propagation)
|
|
741
|
+
│
|
|
742
|
+
└─ 13. Return ConfigRepo instance
|
|
743
|
+
```
|
|
744
|
+
|
|
745
|
+
### 11.2 Why "Sync Before Watch" (Critical Design Decision)
|
|
746
|
+
|
|
747
|
+
The sync engine (`zen-fs-sync`) uses **snapshot-based change detection**. When `watch()` is called on a SyncPair, it triggers `buildInitialSnapshots()` which:
|
|
748
|
+
|
|
749
|
+
1. Builds a snapshot of the source (IndexedDB) — walks all files, records `path`, `size`, `mtimeMs`
|
|
750
|
+
2. Builds a snapshot of the target (remote backend) — same process
|
|
751
|
+
3. **Merges** both snapshots into one map (`source ∪ target`)
|
|
752
|
+
4. Caches this merged snapshot as `sourceSnapshots`
|
|
753
|
+
|
|
754
|
+
On the next `syncAll()`, `syncBidirectional()` compares the current merged snapshot with the cached one. If all paths, sizes, and mtimes match → **"unchanged" → skip sync entirely**.
|
|
755
|
+
|
|
756
|
+
**The problem**: `buildInitialSnapshots()` only *reads* file metadata — it does NOT copy any files. So if the remote has files that the local doesn't (e.g., duplicate backend descriptors written by another node), the merged snapshot includes them from the remote side. The subsequent sync sees "the merged snapshot already has this file" and skips — the file is never actually copied to local, and local-only dedup logic never runs.
|
|
757
|
+
|
|
758
|
+
**The fix**: Always perform a full `syncAll()` **before** `watch()`. With no cached snapshot, `syncBidirectional()` does a complete comparison and copies all missing files. After sync completes, `watch()` builds snapshots from the now-consistent state.
|
|
759
|
+
|
|
760
|
+
This pattern is applied in three places:
|
|
761
|
+
- `createConfigRepo()` → `initialSyncAndDedup()` (sync → dedup → watch)
|
|
762
|
+
- `addBackend()` → `syncMetaToReplicas()` then `watch()` (sync → watch)
|
|
763
|
+
- `AppDataGroupImpl.connect()` → `syncAll()` then `watchAll()` (sync → watch)
|
|
764
|
+
|
|
765
|
+
### 11.3 `flush()` — Manual Sync Trigger
|
|
766
|
+
|
|
767
|
+
```
|
|
768
|
+
flush()
|
|
769
|
+
│
|
|
770
|
+
├─ 1. processTombstones()
|
|
771
|
+
│ For each tombstone in /.meta/.deleted/:
|
|
772
|
+
│ ├─ Delete the actual file on primary (in case re-created)
|
|
773
|
+
│ ├─ Delete the actual file on ALL replicas
|
|
774
|
+
│ └─ Delete version sidecars on all replicas
|
|
775
|
+
│
|
|
776
|
+
├─ 2. syncAll()
|
|
777
|
+
│ For each SyncPair (IndexedDB ↔ replica):
|
|
778
|
+
│ ├─ Build current snapshots of both sides
|
|
779
|
+
│ ├─ Compare with cached snapshot (if any)
|
|
780
|
+
│ ├─ Detect changes: Created / Modified / Deleted
|
|
781
|
+
│ ├─ Resolve conflicts (source-wins strategy)
|
|
782
|
+
│ └─ Copy files in both directions as needed
|
|
783
|
+
│
|
|
784
|
+
├─ 3. readAllBackendDescriptors() — post-sync dedup
|
|
785
|
+
│ Sync may have pulled duplicate backend descriptors from remote.
|
|
786
|
+
│ Re-run dedup to catch and remove them.
|
|
787
|
+
│ ├─ Delete dup files on ALL replicas directly
|
|
788
|
+
│ └─ Create tombstones for deduped descriptors
|
|
789
|
+
│
|
|
790
|
+
├─ 4. processTombstones() — process any new tombstones from step 3
|
|
791
|
+
│
|
|
792
|
+
├─ 5. updateTombstoneConfirmations()
|
|
793
|
+
│ Mark each tombstone as confirmed by all replica backends
|
|
794
|
+
│
|
|
795
|
+
├─ 6. gcTombstones()
|
|
796
|
+
│ Remove tombstones confirmed by ALL backends in the topology
|
|
797
|
+
│
|
|
798
|
+
└─ Return SyncResult[] (one per sync pair)
|
|
799
|
+
```
|
|
800
|
+
|
|
801
|
+
### 11.4 Tombstone-Based Deletion Propagation
|
|
802
|
+
|
|
803
|
+
When a file is deleted via `deleteFile(path)`:
|
|
804
|
+
|
|
805
|
+
```
|
|
806
|
+
deleteFile('/.meta/backends/old-backend.json')
|
|
807
|
+
│
|
|
808
|
+
├─ 1. Write tombstone: /.meta/.deleted/++meta__backends__old-backend++json.json
|
|
809
|
+
│ { path, deletedAt, deletedBy, confirmedBy: [primaryBackendId] }
|
|
810
|
+
│
|
|
811
|
+
├─ 2. Delete the actual file on primary (IndexedDB)
|
|
812
|
+
│
|
|
813
|
+
└─ 3. Delete version sidecar (.old-backend.json.version) on primary
|
|
814
|
+
```
|
|
815
|
+
|
|
816
|
+
On the next `processTombstones()` (called by `flush()` or `initialSyncAndDedup()`):
|
|
817
|
+
|
|
818
|
+
```
|
|
819
|
+
For each tombstone:
|
|
820
|
+
├─ Delete file on primary (in case sync re-created it)
|
|
821
|
+
├─ Delete file on ALL replicas
|
|
822
|
+
├─ Delete version sidecar on ALL replicas
|
|
823
|
+
└─ Tombstone file itself is synced to replicas via syncAll()
|
|
824
|
+
→ Late-joining replicas see the tombstone and delete the file
|
|
729
825
|
```
|
|
730
826
|
|
|
731
|
-
|
|
827
|
+
**Why tombstones?** Without them, bi-directional sync treats a deleted local file as "missing → needs to be copied from remote". The tombstone explicitly signals "this file was intentionally deleted" so all replicas honor the deletion. Tombstones are garbage-collected after all backends confirm receipt.
|
|
828
|
+
|
|
829
|
+
### 11.5 Backend Deduplication
|
|
830
|
+
|
|
831
|
+
When `readAllBackendDescriptors()` detects two backends with the same `type` + `options` (using stable key ordering) but different IDs:
|
|
832
|
+
|
|
833
|
+
```
|
|
834
|
+
Detected: rs-1 and rs-2 have identical type + options
|
|
835
|
+
│
|
|
836
|
+
├─ 1. Keep the one with the earliest mtime (created first)
|
|
837
|
+
│
|
|
838
|
+
├─ 2. For each duplicate:
|
|
839
|
+
│ ├─ Delete descriptor file on ALL replicas directly
|
|
840
|
+
│ │ (prevents sync from pulling it back)
|
|
841
|
+
│ ├─ Delete version sidecar on ALL replicas
|
|
842
|
+
│ └─ Create tombstone + delete local file
|
|
843
|
+
│
|
|
844
|
+
└─ 3. Return deduplicated list (duplicates removed)
|
|
845
|
+
```
|
|
846
|
+
|
|
847
|
+
The stable key function (`backendDedupKey`) sorts object keys recursively, so `{ token: 'a', owner: 'b' }` and `{ owner: 'b', token: 'a' }` produce the same key and are correctly detected as duplicates.
|
|
848
|
+
|
|
849
|
+
### 11.6 Dynamic Backend Management
|
|
850
|
+
|
|
851
|
+
**`addBackend(id, type, options)`**:
|
|
852
|
+
|
|
853
|
+
```
|
|
854
|
+
├─ 1. Dedup check: reject if same type+options already registered
|
|
855
|
+
├─ 2. Create backend instance
|
|
856
|
+
├─ 3. Write descriptor to .meta/backends/{id}.json
|
|
857
|
+
├─ 4. Create SyncPair (IndexedDB ↔ new replica, bi-directional)
|
|
858
|
+
├─ 5. syncMetaToReplicas() — full sync FIRST (pull + push)
|
|
859
|
+
└─ 6. watch(pairId) — start monitoring AFTER sync completes
|
|
860
|
+
```
|
|
861
|
+
|
|
862
|
+
**`removeBackend(id)`**:
|
|
863
|
+
|
|
864
|
+
```
|
|
865
|
+
├─ 1. Delete descriptor file on the remote backend DIRECTLY
|
|
866
|
+
│ (must happen before removing sync pair — otherwise can't reach remote)
|
|
867
|
+
├─ 2. Delete version sidecar on remote
|
|
868
|
+
├─ 3. Create tombstone + delete local descriptor file
|
|
869
|
+
├─ 4. Remove sync pair (stops watching + disposes)
|
|
870
|
+
├─ 5. Remove from replicaBackends map
|
|
871
|
+
├─ 6. Dispose backend instance
|
|
872
|
+
├─ 7. processTombstones() — propagate deletion to remaining replicas
|
|
873
|
+
└─ 8. flush() — sync + GC tombstones
|
|
874
|
+
```
|
|
875
|
+
|
|
876
|
+
### 11.7 Watch Mode (Auto-Sync)
|
|
877
|
+
|
|
878
|
+
After initialization, each SyncPair runs in **watch mode** with hybrid change detection:
|
|
879
|
+
|
|
880
|
+
```
|
|
881
|
+
watch() triggers:
|
|
882
|
+
│
|
|
883
|
+
├─ 1. Register onChange callbacks (if backend supports it)
|
|
884
|
+
│ Local backends (IndexedDB) push change notifications
|
|
885
|
+
│ → triggers debounced sync (default 300ms)
|
|
886
|
+
│
|
|
887
|
+
├─ 2. buildInitialSnapshots()
|
|
888
|
+
│ ├─ BiDirectional: merge source + target snapshots
|
|
889
|
+
│ └─ OneWay: snapshot source only
|
|
890
|
+
│
|
|
891
|
+
└─ 3. Start poll timers (if backend supports shouldSync)
|
|
892
|
+
├─ Remote backends poll shouldSync() every pollIntervalMs (default 30min)
|
|
893
|
+
└─ Fallback: if no onChange and no shouldSync, poll every interval
|
|
894
|
+
```
|
|
895
|
+
|
|
896
|
+
**State guard**: If `unwatch()` is called during `buildInitialSnapshots()` (which is async), the snapshots are discarded — they won't be cached. This prevents stale snapshots from causing sync skips.
|
|
897
|
+
|
|
898
|
+
**Snapshot comparison** in `syncBidirectional()`:
|
|
899
|
+
1. Build current snapshots of both sides
|
|
900
|
+
2. Merge into `currentMerged = source ∪ target`
|
|
901
|
+
3. Compare with cached `sourceSnapshots`:
|
|
902
|
+
- If all paths + mtimes + sizes match → "unchanged" → skip
|
|
903
|
+
- Otherwise → proceed with full diff and file operations
|
|
904
|
+
4. Cache `currentMerged` for next comparison
|
|
905
|
+
|
|
906
|
+
### 11.8 Standalone Data-Sync Group (`createDataSyncGroup`)
|
|
732
907
|
|
|
733
908
|
```
|
|
734
909
|
createDataSyncGroup('my-app', options?)
|
|
@@ -743,11 +918,16 @@ createDataSyncGroup('my-app', options?)
|
|
|
743
918
|
├─ 3. Create IndexedDB as local primary (for offline access)
|
|
744
919
|
│
|
|
745
920
|
├─ 4. Setup sync: IndexedDB ↔ each data backend (bi-directional)
|
|
921
|
+
│ NOTE: Does NOT watch yet — sync first
|
|
922
|
+
│
|
|
923
|
+
├─ 5. syncAll() — pull data from remote backends
|
|
924
|
+
│
|
|
925
|
+
├─ 6. watchAll() — start monitoring AFTER sync completes
|
|
746
926
|
│
|
|
747
|
-
└─
|
|
927
|
+
└─ 7. Return DataSyncGroup handle with direct fs access
|
|
748
928
|
```
|
|
749
929
|
|
|
750
|
-
### 11.
|
|
930
|
+
### 11.9 Unified Entry Point (`connect`)
|
|
751
931
|
|
|
752
932
|
`createConfigRepo` and `createDataSyncGroup` are lower-level factory functions. The recommended entry point is `connect`, which auto-detects the group type and dispatches to the appropriate factory:
|
|
753
933
|
|
package/dist/index.d.mts
CHANGED
|
@@ -465,6 +465,8 @@ declare class ConfigRepo implements IConfigRepo {
|
|
|
465
465
|
constructor(appId: string, nodeId: string, primaryBackendId: string, cachedFS: MinimalAsyncFS, serializer: PathAwareSerializer, onConflict?: (conflict: ConflictInfo) => Promise<unknown | null>, pollIntervalMs?: number);
|
|
466
466
|
/** Full path to this node's directory on the primary backend. */
|
|
467
467
|
get nodePath(): string;
|
|
468
|
+
/** Number of replica backends registered (excludes the local primary). */
|
|
469
|
+
get replicaCount(): number;
|
|
468
470
|
load(rawConfig?: string): Promise<void>;
|
|
469
471
|
getConfig<T = unknown>(path: string): T;
|
|
470
472
|
setConfig(path: string, data: unknown): void;
|
|
@@ -489,6 +491,14 @@ declare class ConfigRepo implements IConfigRepo {
|
|
|
489
491
|
* This prevents bi-directional sync from copying the file back.
|
|
490
492
|
*/
|
|
491
493
|
private processTombstones;
|
|
494
|
+
/** Public wrapper for processTombstones — used by createConfigRepo. */
|
|
495
|
+
processTombstonesPublic(): Promise<void>;
|
|
496
|
+
/**
|
|
497
|
+
* Perform a full sync + dedup cycle without the watch snapshot cache.
|
|
498
|
+
* Used by createConfigRepo to pull remote-only files (like duplicate
|
|
499
|
+
* backend descriptors) that watch()'s initial snapshot would skip.
|
|
500
|
+
*/
|
|
501
|
+
initialSyncAndDedup(): Promise<void>;
|
|
492
502
|
/**
|
|
493
503
|
* After sync: mark each tombstone as confirmed by all replica backends.
|
|
494
504
|
*/
|
package/dist/index.d.ts
CHANGED
|
@@ -465,6 +465,8 @@ declare class ConfigRepo implements IConfigRepo {
|
|
|
465
465
|
constructor(appId: string, nodeId: string, primaryBackendId: string, cachedFS: MinimalAsyncFS, serializer: PathAwareSerializer, onConflict?: (conflict: ConflictInfo) => Promise<unknown | null>, pollIntervalMs?: number);
|
|
466
466
|
/** Full path to this node's directory on the primary backend. */
|
|
467
467
|
get nodePath(): string;
|
|
468
|
+
/** Number of replica backends registered (excludes the local primary). */
|
|
469
|
+
get replicaCount(): number;
|
|
468
470
|
load(rawConfig?: string): Promise<void>;
|
|
469
471
|
getConfig<T = unknown>(path: string): T;
|
|
470
472
|
setConfig(path: string, data: unknown): void;
|
|
@@ -489,6 +491,14 @@ declare class ConfigRepo implements IConfigRepo {
|
|
|
489
491
|
* This prevents bi-directional sync from copying the file back.
|
|
490
492
|
*/
|
|
491
493
|
private processTombstones;
|
|
494
|
+
/** Public wrapper for processTombstones — used by createConfigRepo. */
|
|
495
|
+
processTombstonesPublic(): Promise<void>;
|
|
496
|
+
/**
|
|
497
|
+
* Perform a full sync + dedup cycle without the watch snapshot cache.
|
|
498
|
+
* Used by createConfigRepo to pull remote-only files (like duplicate
|
|
499
|
+
* backend descriptors) that watch()'s initial snapshot would skip.
|
|
500
|
+
*/
|
|
501
|
+
initialSyncAndDedup(): Promise<void>;
|
|
492
502
|
/**
|
|
493
503
|
* After sync: mark each tombstone as confirmed by all replica backends.
|
|
494
504
|
*/
|
package/dist/index.js
CHANGED
|
@@ -585,6 +585,10 @@ var ConfigRepo = class {
|
|
|
585
585
|
get nodePath() {
|
|
586
586
|
return `/nodes/${this.nodeId}`;
|
|
587
587
|
}
|
|
588
|
+
/** Number of replica backends registered (excludes the local primary). */
|
|
589
|
+
get replicaCount() {
|
|
590
|
+
return this.replicaBackends.size;
|
|
591
|
+
}
|
|
588
592
|
// -----------------------------------------------------------------------
|
|
589
593
|
// IConfigRepo — Load
|
|
590
594
|
// -----------------------------------------------------------------------
|
|
@@ -723,6 +727,8 @@ var ConfigRepo = class {
|
|
|
723
727
|
this.assertNotDisposed();
|
|
724
728
|
await this.processTombstones();
|
|
725
729
|
const resultsMap = await this.syncEngine.syncAll();
|
|
730
|
+
await this.readAllBackendDescriptors();
|
|
731
|
+
await this.processTombstones();
|
|
726
732
|
await this.updateTombstoneConfirmations();
|
|
727
733
|
await this.gcTombstones();
|
|
728
734
|
return Array.from(resultsMap.values());
|
|
@@ -811,6 +817,22 @@ var ConfigRepo = class {
|
|
|
811
817
|
}
|
|
812
818
|
}
|
|
813
819
|
}
|
|
820
|
+
/** Public wrapper for processTombstones — used by createConfigRepo. */
|
|
821
|
+
async processTombstonesPublic() {
|
|
822
|
+
await this.processTombstones();
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* Perform a full sync + dedup cycle without the watch snapshot cache.
|
|
826
|
+
* Used by createConfigRepo to pull remote-only files (like duplicate
|
|
827
|
+
* backend descriptors) that watch()'s initial snapshot would skip.
|
|
828
|
+
*/
|
|
829
|
+
async initialSyncAndDedup() {
|
|
830
|
+
this.syncEngine.unwatchAll();
|
|
831
|
+
await this.syncEngine.syncAll();
|
|
832
|
+
await this.readAllBackendDescriptors();
|
|
833
|
+
await this.processTombstones();
|
|
834
|
+
this.syncEngine.watchAll();
|
|
835
|
+
}
|
|
814
836
|
/**
|
|
815
837
|
* After sync: mark each tombstone as confirmed by all replica backends.
|
|
816
838
|
*/
|
|
@@ -988,7 +1010,6 @@ var ConfigRepo = class {
|
|
|
988
1010
|
this.handleConflict(event);
|
|
989
1011
|
};
|
|
990
1012
|
this.syncEngine.on(pair.pairId, "conflict", conflictHandler);
|
|
991
|
-
this.syncEngine.watch(pair.pairId);
|
|
992
1013
|
console.log(`[ConfigRepo] Replica ${desc.id} created, sync pair=${pair.pairId}`);
|
|
993
1014
|
} catch (err) {
|
|
994
1015
|
console.error(`[ConfigRepo] Failed to create replica ${desc.id} (${desc.type}):`, err);
|
|
@@ -1158,6 +1179,7 @@ var ConfigRepo = class {
|
|
|
1158
1179
|
try {
|
|
1159
1180
|
const entries = await this.cachedFS.readdir(BACKENDS_DIR);
|
|
1160
1181
|
const items = [];
|
|
1182
|
+
const corruptFiles = [];
|
|
1161
1183
|
for (const entry of entries) {
|
|
1162
1184
|
if (!entry.endsWith(".json")) continue;
|
|
1163
1185
|
const filePath = `${BACKENDS_DIR}/${entry}`;
|
|
@@ -1172,8 +1194,37 @@ var ConfigRepo = class {
|
|
|
1172
1194
|
} catch {
|
|
1173
1195
|
}
|
|
1174
1196
|
items.push({ desc, mtime });
|
|
1197
|
+
} else {
|
|
1198
|
+
console.warn(`[ConfigRepo] Backend descriptor ${entry} is missing id/type fields, marking for cleanup`);
|
|
1199
|
+
corruptFiles.push(filePath);
|
|
1175
1200
|
}
|
|
1201
|
+
} catch (parseErr) {
|
|
1202
|
+
console.warn(`[ConfigRepo] Backend descriptor ${entry} has corrupted JSON: ${parseErr}. Marking for cleanup.`);
|
|
1203
|
+
corruptFiles.push(filePath);
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
for (const corruptPath of corruptFiles) {
|
|
1207
|
+
for (const [, replica] of this.replicaBackends) {
|
|
1208
|
+
try {
|
|
1209
|
+
await replica.instance.unlink(corruptPath);
|
|
1210
|
+
} catch {
|
|
1211
|
+
}
|
|
1212
|
+
try {
|
|
1213
|
+
await replica.instance.unlink(versionPathFor(corruptPath));
|
|
1214
|
+
} catch {
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
try {
|
|
1218
|
+
await this.deleteFile(corruptPath);
|
|
1176
1219
|
} catch {
|
|
1220
|
+
try {
|
|
1221
|
+
await this.cachedFS.unlink(corruptPath);
|
|
1222
|
+
} catch {
|
|
1223
|
+
}
|
|
1224
|
+
try {
|
|
1225
|
+
await this.cachedFS.unlink(versionPathFor(corruptPath));
|
|
1226
|
+
} catch {
|
|
1227
|
+
}
|
|
1177
1228
|
}
|
|
1178
1229
|
}
|
|
1179
1230
|
const seen = /* @__PURE__ */ new Map();
|
|
@@ -1198,6 +1249,16 @@ var ConfigRepo = class {
|
|
|
1198
1249
|
);
|
|
1199
1250
|
for (const dupId of duplicates) {
|
|
1200
1251
|
const descPath = this.backendFilePath(dupId);
|
|
1252
|
+
for (const [replicaId, replica] of this.replicaBackends) {
|
|
1253
|
+
try {
|
|
1254
|
+
await replica.instance.unlink(descPath);
|
|
1255
|
+
} catch {
|
|
1256
|
+
}
|
|
1257
|
+
try {
|
|
1258
|
+
await replica.instance.unlink(versionPathFor(descPath));
|
|
1259
|
+
} catch {
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1201
1262
|
try {
|
|
1202
1263
|
await this.deleteFile(descPath);
|
|
1203
1264
|
} catch {
|
|
@@ -1312,9 +1373,9 @@ var ConfigRepo = class {
|
|
|
1312
1373
|
this.handleConflict(event);
|
|
1313
1374
|
};
|
|
1314
1375
|
this.syncEngine.on(pair.pairId, "conflict", conflictHandler);
|
|
1315
|
-
this.syncEngine.watch(pair.pairId);
|
|
1316
1376
|
console.log(`[ConfigRepo] addBackend: ${id} (${type}) added, sync pair=${pair.pairId}`);
|
|
1317
1377
|
await this.syncMetaToReplicas();
|
|
1378
|
+
this.syncEngine.watch(pair.pairId);
|
|
1318
1379
|
}
|
|
1319
1380
|
async removeBackend(id) {
|
|
1320
1381
|
this.assertNotDisposed();
|
|
@@ -1559,7 +1620,6 @@ var AppDataGroupImpl = class {
|
|
|
1559
1620
|
"/"
|
|
1560
1621
|
);
|
|
1561
1622
|
this.dataBackends.set(desc.id, { instance, syncable, pairId: pair.pairId, desc });
|
|
1562
|
-
this.syncEngine.watch(pair.pairId);
|
|
1563
1623
|
console.log(`[AppDataGroup:${this.groupId}] backend ${desc.id} (${desc.type}) connected, pair=${pair.pairId}`);
|
|
1564
1624
|
} catch (err) {
|
|
1565
1625
|
console.error(`[AppDataGroup:${this.groupId}] Failed to create backend ${desc.id} (${desc.type}):`, err);
|
|
@@ -1570,6 +1630,7 @@ var AppDataGroupImpl = class {
|
|
|
1570
1630
|
} catch (err) {
|
|
1571
1631
|
console.warn(`[AppDataGroup:${this.groupId}] Initial sync failed:`, err);
|
|
1572
1632
|
}
|
|
1633
|
+
this.syncEngine.watchAll();
|
|
1573
1634
|
}
|
|
1574
1635
|
getSyncStatuses() {
|
|
1575
1636
|
return this.syncEngine.getStatusAll();
|
|
@@ -1599,13 +1660,13 @@ var AppDataGroupImpl = class {
|
|
|
1599
1660
|
);
|
|
1600
1661
|
const desc = { id, type, options, description };
|
|
1601
1662
|
this.dataBackends.set(id, { instance, syncable, pairId: pair.pairId, desc });
|
|
1602
|
-
this.syncEngine.watch(pair.pairId);
|
|
1603
1663
|
console.log(`[AppDataGroup:${this.groupId}] addBackend: ${id} (${type}) connected, pair=${pair.pairId}`);
|
|
1604
1664
|
try {
|
|
1605
1665
|
await this.syncEngine.sync(pair.pairId);
|
|
1606
1666
|
} catch (err) {
|
|
1607
1667
|
console.warn(`[AppDataGroup:${this.groupId}] addBackend: initial sync failed for ${id}:`, err);
|
|
1608
1668
|
}
|
|
1669
|
+
this.syncEngine.watch(pair.pairId);
|
|
1609
1670
|
}
|
|
1610
1671
|
async removeBackend(id) {
|
|
1611
1672
|
if (this.disposed) throw new Error("DataSyncGroup has been disposed");
|
|
@@ -1739,6 +1800,10 @@ async function createConfigRepo(appId, options = {}) {
|
|
|
1739
1800
|
);
|
|
1740
1801
|
await repo.setupSync(allBackends, LOCAL_IDB_BACKEND_ID, options.syncPollIntervalMs);
|
|
1741
1802
|
await repo.load();
|
|
1803
|
+
if (repo.replicaCount > 0) {
|
|
1804
|
+
console.log("[createConfigRepo] Initial sync + dedup cycle...");
|
|
1805
|
+
await repo.initialSyncAndDedup();
|
|
1806
|
+
}
|
|
1742
1807
|
repo.syncMetaToReplicas().catch((err) => {
|
|
1743
1808
|
console.error("[createConfigRepo] background syncMetaToReplicas failed:", err);
|
|
1744
1809
|
});
|
package/dist/index.mjs
CHANGED
|
@@ -528,6 +528,10 @@ var ConfigRepo = class {
|
|
|
528
528
|
get nodePath() {
|
|
529
529
|
return `/nodes/${this.nodeId}`;
|
|
530
530
|
}
|
|
531
|
+
/** Number of replica backends registered (excludes the local primary). */
|
|
532
|
+
get replicaCount() {
|
|
533
|
+
return this.replicaBackends.size;
|
|
534
|
+
}
|
|
531
535
|
// -----------------------------------------------------------------------
|
|
532
536
|
// IConfigRepo — Load
|
|
533
537
|
// -----------------------------------------------------------------------
|
|
@@ -666,6 +670,8 @@ var ConfigRepo = class {
|
|
|
666
670
|
this.assertNotDisposed();
|
|
667
671
|
await this.processTombstones();
|
|
668
672
|
const resultsMap = await this.syncEngine.syncAll();
|
|
673
|
+
await this.readAllBackendDescriptors();
|
|
674
|
+
await this.processTombstones();
|
|
669
675
|
await this.updateTombstoneConfirmations();
|
|
670
676
|
await this.gcTombstones();
|
|
671
677
|
return Array.from(resultsMap.values());
|
|
@@ -754,6 +760,22 @@ var ConfigRepo = class {
|
|
|
754
760
|
}
|
|
755
761
|
}
|
|
756
762
|
}
|
|
763
|
+
/** Public wrapper for processTombstones — used by createConfigRepo. */
|
|
764
|
+
async processTombstonesPublic() {
|
|
765
|
+
await this.processTombstones();
|
|
766
|
+
}
|
|
767
|
+
/**
|
|
768
|
+
* Perform a full sync + dedup cycle without the watch snapshot cache.
|
|
769
|
+
* Used by createConfigRepo to pull remote-only files (like duplicate
|
|
770
|
+
* backend descriptors) that watch()'s initial snapshot would skip.
|
|
771
|
+
*/
|
|
772
|
+
async initialSyncAndDedup() {
|
|
773
|
+
this.syncEngine.unwatchAll();
|
|
774
|
+
await this.syncEngine.syncAll();
|
|
775
|
+
await this.readAllBackendDescriptors();
|
|
776
|
+
await this.processTombstones();
|
|
777
|
+
this.syncEngine.watchAll();
|
|
778
|
+
}
|
|
757
779
|
/**
|
|
758
780
|
* After sync: mark each tombstone as confirmed by all replica backends.
|
|
759
781
|
*/
|
|
@@ -931,7 +953,6 @@ var ConfigRepo = class {
|
|
|
931
953
|
this.handleConflict(event);
|
|
932
954
|
};
|
|
933
955
|
this.syncEngine.on(pair.pairId, "conflict", conflictHandler);
|
|
934
|
-
this.syncEngine.watch(pair.pairId);
|
|
935
956
|
console.log(`[ConfigRepo] Replica ${desc.id} created, sync pair=${pair.pairId}`);
|
|
936
957
|
} catch (err) {
|
|
937
958
|
console.error(`[ConfigRepo] Failed to create replica ${desc.id} (${desc.type}):`, err);
|
|
@@ -1101,6 +1122,7 @@ var ConfigRepo = class {
|
|
|
1101
1122
|
try {
|
|
1102
1123
|
const entries = await this.cachedFS.readdir(BACKENDS_DIR);
|
|
1103
1124
|
const items = [];
|
|
1125
|
+
const corruptFiles = [];
|
|
1104
1126
|
for (const entry of entries) {
|
|
1105
1127
|
if (!entry.endsWith(".json")) continue;
|
|
1106
1128
|
const filePath = `${BACKENDS_DIR}/${entry}`;
|
|
@@ -1115,8 +1137,37 @@ var ConfigRepo = class {
|
|
|
1115
1137
|
} catch {
|
|
1116
1138
|
}
|
|
1117
1139
|
items.push({ desc, mtime });
|
|
1140
|
+
} else {
|
|
1141
|
+
console.warn(`[ConfigRepo] Backend descriptor ${entry} is missing id/type fields, marking for cleanup`);
|
|
1142
|
+
corruptFiles.push(filePath);
|
|
1118
1143
|
}
|
|
1144
|
+
} catch (parseErr) {
|
|
1145
|
+
console.warn(`[ConfigRepo] Backend descriptor ${entry} has corrupted JSON: ${parseErr}. Marking for cleanup.`);
|
|
1146
|
+
corruptFiles.push(filePath);
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
for (const corruptPath of corruptFiles) {
|
|
1150
|
+
for (const [, replica] of this.replicaBackends) {
|
|
1151
|
+
try {
|
|
1152
|
+
await replica.instance.unlink(corruptPath);
|
|
1153
|
+
} catch {
|
|
1154
|
+
}
|
|
1155
|
+
try {
|
|
1156
|
+
await replica.instance.unlink(versionPathFor(corruptPath));
|
|
1157
|
+
} catch {
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
try {
|
|
1161
|
+
await this.deleteFile(corruptPath);
|
|
1119
1162
|
} catch {
|
|
1163
|
+
try {
|
|
1164
|
+
await this.cachedFS.unlink(corruptPath);
|
|
1165
|
+
} catch {
|
|
1166
|
+
}
|
|
1167
|
+
try {
|
|
1168
|
+
await this.cachedFS.unlink(versionPathFor(corruptPath));
|
|
1169
|
+
} catch {
|
|
1170
|
+
}
|
|
1120
1171
|
}
|
|
1121
1172
|
}
|
|
1122
1173
|
const seen = /* @__PURE__ */ new Map();
|
|
@@ -1141,6 +1192,16 @@ var ConfigRepo = class {
|
|
|
1141
1192
|
);
|
|
1142
1193
|
for (const dupId of duplicates) {
|
|
1143
1194
|
const descPath = this.backendFilePath(dupId);
|
|
1195
|
+
for (const [replicaId, replica] of this.replicaBackends) {
|
|
1196
|
+
try {
|
|
1197
|
+
await replica.instance.unlink(descPath);
|
|
1198
|
+
} catch {
|
|
1199
|
+
}
|
|
1200
|
+
try {
|
|
1201
|
+
await replica.instance.unlink(versionPathFor(descPath));
|
|
1202
|
+
} catch {
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1144
1205
|
try {
|
|
1145
1206
|
await this.deleteFile(descPath);
|
|
1146
1207
|
} catch {
|
|
@@ -1255,9 +1316,9 @@ var ConfigRepo = class {
|
|
|
1255
1316
|
this.handleConflict(event);
|
|
1256
1317
|
};
|
|
1257
1318
|
this.syncEngine.on(pair.pairId, "conflict", conflictHandler);
|
|
1258
|
-
this.syncEngine.watch(pair.pairId);
|
|
1259
1319
|
console.log(`[ConfigRepo] addBackend: ${id} (${type}) added, sync pair=${pair.pairId}`);
|
|
1260
1320
|
await this.syncMetaToReplicas();
|
|
1321
|
+
this.syncEngine.watch(pair.pairId);
|
|
1261
1322
|
}
|
|
1262
1323
|
async removeBackend(id) {
|
|
1263
1324
|
this.assertNotDisposed();
|
|
@@ -1502,7 +1563,6 @@ var AppDataGroupImpl = class {
|
|
|
1502
1563
|
"/"
|
|
1503
1564
|
);
|
|
1504
1565
|
this.dataBackends.set(desc.id, { instance, syncable, pairId: pair.pairId, desc });
|
|
1505
|
-
this.syncEngine.watch(pair.pairId);
|
|
1506
1566
|
console.log(`[AppDataGroup:${this.groupId}] backend ${desc.id} (${desc.type}) connected, pair=${pair.pairId}`);
|
|
1507
1567
|
} catch (err) {
|
|
1508
1568
|
console.error(`[AppDataGroup:${this.groupId}] Failed to create backend ${desc.id} (${desc.type}):`, err);
|
|
@@ -1513,6 +1573,7 @@ var AppDataGroupImpl = class {
|
|
|
1513
1573
|
} catch (err) {
|
|
1514
1574
|
console.warn(`[AppDataGroup:${this.groupId}] Initial sync failed:`, err);
|
|
1515
1575
|
}
|
|
1576
|
+
this.syncEngine.watchAll();
|
|
1516
1577
|
}
|
|
1517
1578
|
getSyncStatuses() {
|
|
1518
1579
|
return this.syncEngine.getStatusAll();
|
|
@@ -1542,13 +1603,13 @@ var AppDataGroupImpl = class {
|
|
|
1542
1603
|
);
|
|
1543
1604
|
const desc = { id, type, options, description };
|
|
1544
1605
|
this.dataBackends.set(id, { instance, syncable, pairId: pair.pairId, desc });
|
|
1545
|
-
this.syncEngine.watch(pair.pairId);
|
|
1546
1606
|
console.log(`[AppDataGroup:${this.groupId}] addBackend: ${id} (${type}) connected, pair=${pair.pairId}`);
|
|
1547
1607
|
try {
|
|
1548
1608
|
await this.syncEngine.sync(pair.pairId);
|
|
1549
1609
|
} catch (err) {
|
|
1550
1610
|
console.warn(`[AppDataGroup:${this.groupId}] addBackend: initial sync failed for ${id}:`, err);
|
|
1551
1611
|
}
|
|
1612
|
+
this.syncEngine.watch(pair.pairId);
|
|
1552
1613
|
}
|
|
1553
1614
|
async removeBackend(id) {
|
|
1554
1615
|
if (this.disposed) throw new Error("DataSyncGroup has been disposed");
|
|
@@ -1682,6 +1743,10 @@ async function createConfigRepo(appId, options = {}) {
|
|
|
1682
1743
|
);
|
|
1683
1744
|
await repo.setupSync(allBackends, LOCAL_IDB_BACKEND_ID, options.syncPollIntervalMs);
|
|
1684
1745
|
await repo.load();
|
|
1746
|
+
if (repo.replicaCount > 0) {
|
|
1747
|
+
console.log("[createConfigRepo] Initial sync + dedup cycle...");
|
|
1748
|
+
await repo.initialSyncAndDedup();
|
|
1749
|
+
}
|
|
1685
1750
|
repo.syncMetaToReplicas().catch((err) => {
|
|
1686
1751
|
console.error("[createConfigRepo] background syncMetaToReplicas failed:", err);
|
|
1687
1752
|
});
|