sliftutils 1.7.108 → 1.7.109

package/index.d.ts

@@ -4820,7 +4820,7 @@ declare module "sliftutils/storage/remoteStorage/storeSync" {
         waitForRequiredScans(): Promise<void>;
         /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
         rescanBase(): Promise<void>;
-        /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
+        /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. Only one round per SOURCE runs at a time (cache keys the serializer by source index). */
         private syncSource;
         /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
         boundaryScanRemote(source: IArchives, config: {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.108",
+    "version": "1.7.109",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/remoteStorage/storeSync.d.ts

@@ -37,7 +37,7 @@ export declare class StoreSync {
     waitForRequiredScans(): Promise<void>;
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     rescanBase(): Promise<void>;
-    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. Only one round per SOURCE runs at a time (cache keys the serializer by source index). */
     private syncSource;
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
     boundaryScanRemote(source: IArchives, config: {

package/storage/remoteStorage/storeSync.ts

@@ -1,4 +1,5 @@
-import { runInfinitePoll, delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { runInfinitePoll, delay, runInfinitePollCallAtStart, runInSerial } from "socket-function/src/batching";
+import { cache } from "socket-function/src/caching";
 import { timeInMinute, sort, promiseObj } from "socket-function/src/misc";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import {
@@ -219,11 +220,11 @@ export class StoreSync {
 
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     public async rescanBase(): Promise<void> {
-        await this.syncSource(0);
+        await this.syncSource(0)();
     }
 
-    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
-    private async syncSource(sourceIndex: number, push?: "push"): Promise<void> {
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. Only one round per SOURCE runs at a time (cache keys the serializer by source index). */
+    private syncSource = cache((sourceIndex: number) => runInSerial(async (push?: "push"): Promise<void> => {
         let listing = await this.pullSource(sourceIndex);
         let state = this.states[sourceIndex];
         state.scanComplete = true;
@@ -231,7 +232,7 @@ export class StoreSync {
         if (push && !this.store.stopped.stop && !state.stopped.stop) {
             await this.pushSource(sourceIndex, listing);
         }
-    }
+    }));
 
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
     public async boundaryScanRemote(source: IArchives, config: { since: number; route?: [number, number] }): Promise<void> {
@@ -304,12 +305,15 @@ export class StoreSync {
             }
         }
         if (this.store.stopped.stop || state.stopped.stop) return;
-        let pollInterval = sourceObj.sourceConfig?.type === "remote" ? FULL_RESCAN_INTERVAL : FULL_RESCAN_NON_REMOTE_INTERVAL;
-        await runInfinitePollCallAtStart(pollInterval, async () => {
+        // Hourly for our own disk (no sourceConfig - a cheap local walk, and how a sibling process's writes into the shared folder are found) and for remote peers (index-backed, cheap listings); the slow interval is only for sources where a full listing is genuinely expensive (backblaze)
+        let pollInterval = (!sourceObj.sourceConfig || sourceObj.sourceConfig.type === "remote") && FULL_RESCAN_INTERVAL || FULL_RESCAN_NON_REMOTE_INTERVAL;
+        // Both loops below run one at a time: a change-poll tick landing mid full round would otherwise start a second copy pass over the same pending list, downloading everything twice
+        let serial = runInSerial(async (fnc: () => Promise<void>) => await fnc());
+        await runInfinitePollCallAtStart(pollInterval, () => serial(async () => {
             while (!this.store.stopped.stop && !state.stopped.stop) {
                 try {
-                    await this.syncSource(sourceIndex, "push");
-                    if (!noFullSync()) await this.copySourceFiles(sourceIndex);
+                    await this.syncSource(sourceIndex)("push");
+                    if (!noFullSync()) await this.copySourceFiles(sourceIndex)();
                 } catch (e) {
                     logSyncEvent({ event: "scanFailed", store: this.store.folder, source: source.getDebugName(), retryInMs: SCAN_RETRY_DELAY, error: String((e as Error).stack ?? e).slice(0, 2000) });
                     await delay(SCAN_RETRY_DELAY);
@@ -319,16 +323,16 @@ export class StoreSync {
                 state.initialScan.resolve(undefined);
                 break;
             }
-        }, state.stopped);
+        }), state.stopped);
         if (state.supportsChangesAfter) {
-            runInfinitePoll(CHANGES_POLL_INTERVAL, async () => {
+            runInfinitePoll(CHANGES_POLL_INTERVAL, () => serial(async () => {
                 // A scan that has not succeeded yet is retried HERE, before anything else - polling changes and copying against a never-scanned source would run on arbitrarily stale information
                 if (!state.scanSucceeded) {
-                    await this.syncSource(sourceIndex, "push");
+                    await this.syncSource(sourceIndex)("push");
                 }
                 await this.pollChanges(sourceIndex);
-                if (!noFullSync()) await this.copySourceFiles(sourceIndex);
-            }, state.stopped);
+                if (!noFullSync()) await this.copySourceFiles(sourceIndex)();
+            }), state.stopped);
         }
     }
 
@@ -522,8 +526,8 @@ export class StoreSync {
         state.changesAfterTime = pollStart - CHANGES_POLL_OVERLAP;
     }
 
-    // Downloads the files a source currently holds onto our own base source (the local disk), preserving their modified times — so a newer local write always wins. Skipped for noFullSync sources (fronting a large database without copying it); reads still down-cache lazily.
-    private async copySourceFiles(sourceIndex: number): Promise<void> {
+    // Downloads the files a source currently holds onto our own base source (the local disk), preserving their modified times — so a newer local write always wins. Skipped for noFullSync sources (fronting a large database without copying it); reads still down-cache lazily. Only one pass per SOURCE runs at a time (cache keys the serializer by source index) - two passes over the same pending list would download everything twice.
+    private copySourceFiles = cache((sourceIndex: number) => runInSerial(async (): Promise<void> => {
         if (sourceIndex === 0) return;
         let { source } = this.store.sources[sourceIndex];
         let state = this.states[sourceIndex];
@@ -648,7 +652,7 @@ export class StoreSync {
             }
             logSyncEvent({ event: "fullSyncFinish", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, durationMs: Date.now() - activity.startTime, totalFiles: pending.length, totalBytes, processedFiles: activity.doneFiles, copiedFiles, copiedBytes, missingOnSource, failed, aborted });
         }
-    }
+    }));
 
     // ── maintenance ──