sliftutils 1.7.107 → 1.7.109

package/storage/remoteStorage/storageLogs.d.ts

@@ -2,7 +2,7 @@
 /// <reference types="node" />
 import { LogFileInfo } from "../StreamingLogs";
 export declare const LOGS_FOLDER_NAME = "logs";
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Also printed to the console. */
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Stream-only - one entry per write is exactly what the console does NOT need. */
 export declare function logMutation(entry: {
     op: string;
     account: string;

package/storage/remoteStorage/storageLogs.ts

@@ -66,9 +66,9 @@ function write(kind: string, entry: { [key: string]: unknown }, alsoConsole: boo
     logs.log({ kind, time: Date.now(), ...baseFields(), ...entry });
 }
 
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Also printed to the console. */
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Stream-only - one entry per write is exactly what the console does NOT need. */
 export function logMutation(entry: { op: string; account: string; bucketName: string; store?: string; path: string; toPath?: string; size?: number; writeTime?: number; callerId?: string; internal?: boolean }): void {
-    write("mutation", entry, true);
+    write("mutation", entry, false);
 }
 
 /** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. Also printed to the console. */

package/storage/remoteStorage/storeSync.d.ts

@@ -37,14 +37,14 @@ export declare class StoreSync {
     waitForRequiredScans(): Promise<void>;
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     rescanBase(): Promise<void>;
-    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. Only one round per SOURCE runs at a time (cache keys the serializer by source index). */
     private syncSource;
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
     boundaryScanRemote(source: IArchives, config: {
         since: number;
         route?: [number, number];
     }): Promise<void>;
-    private runSourceSync;
+    private startSourceSyncLoops;
     private pullSource;
     private pushSource;
     private updateScanIndex;

package/storage/remoteStorage/storeSync.ts

@@ -1,4 +1,5 @@
-import { runInfinitePoll, delay } from "socket-function/src/batching";
+import { runInfinitePoll, delay, runInfinitePollCallAtStart, runInSerial } from "socket-function/src/batching";
+import { cache } from "socket-function/src/caching";
 import { timeInMinute, sort, promiseObj } from "socket-function/src/misc";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import {
@@ -21,7 +22,7 @@ const CHANGES_POLL_INTERVAL = 1000 * 60;
 const CONFIG_POLL_INTERVAL = 1000 * 60 * 5;
 // Full metadata rescans. supportsChangesAfter is the heuristic for "one of our own storage servers": their index-backed listings are cheap, so hourly is fine. Everything else (backblaze, plain disk) pays the full listing cost, so it rescans much less often.
 const FULL_RESCAN_INTERVAL = 1000 * 60 * 60;
-const FULL_RESCAN_UNINDEXED_INTERVAL = 1000 * 60 * 60 * 6;
+const FULL_RESCAN_NON_REMOTE_INTERVAL = 1000 * 60 * 60 * 6;
 // Change polls re-request this much overlap, so clock skew between us and a source can't drop changes
 const CHANGES_POLL_OVERLAP = timeInMinute;
 const SCAN_RETRY_DELAY = 1000 * 30;
@@ -97,7 +98,7 @@ export class StoreSync {
     public start(): void {
         for (let i = 0; i < this.store.sources.length; i++) {
             if (!this.isLive(i)) continue;
-            void this.runSourceSync(i);
+            void this.startSourceSyncLoops(i);
         }
         runInfinitePoll(CONFIG_POLL_INTERVAL, () => this.pollRoutingConfig(), this.store.stopped);
         runInfinitePoll(TOMBSTONE_CLEANUP_INTERVAL, () => this.cleanupTombstones(), this.store.stopped);
@@ -147,7 +148,7 @@ export class StoreSync {
     public addSource(slot: number): void {
         this.states[slot] = newSourceState();
         if (this.store.syncStarted) {
-            void this.runSourceSync(slot);
+            void this.startSourceSyncLoops(slot);
         }
     }
 
@@ -219,11 +220,11 @@ export class StoreSync {
 
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     public async rescanBase(): Promise<void> {
-        await this.syncSource(0);
+        await this.syncSource(0)();
     }
 
-    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
-    private async syncSource(sourceIndex: number, push?: "push"): Promise<void> {
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. Only one round per SOURCE runs at a time (cache keys the serializer by source index). */
+    private syncSource = cache((sourceIndex: number) => runInSerial(async (push?: "push"): Promise<void> => {
         let listing = await this.pullSource(sourceIndex);
         let state = this.states[sourceIndex];
         state.scanComplete = true;
@@ -231,7 +232,7 @@ export class StoreSync {
         if (push && !this.store.stopped.stop && !state.stopped.stop) {
             await this.pushSource(sourceIndex, listing);
         }
-    }
+    }));
 
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
     public async boundaryScanRemote(source: IArchives, config: { since: number; route?: [number, number] }): Promise<void> {
@@ -255,7 +256,7 @@ export class StoreSync {
                 tally.tombstone++;
                 continue;
             }
-            let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: file.path, size: file.size, writeTime: file.createTime, forceSetImmutable: true, noChecks: true, internal: true });
+            let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: file.path, forceSetImmutable: true, noChecks: true, internal: true });
             if (!copied) {
                 console.warn(`Boundary scan could not copy ${file.path} from ${source.getDebugName()} (store ${this.store.folder}): its change feed listed it (${file.size} bytes, writeTime ${new Date(file.createTime).toISOString()}) but the read found nothing`);
                 continue;
@@ -278,9 +279,10 @@ export class StoreSync {
 
     // ── per-source loops ──
 
-    private async runSourceSync(sourceIndex: number): Promise<void> {
+    private async startSourceSyncLoops(sourceIndex: number): Promise<void> {
         await this.store.registerSlot(sourceIndex);
-        let { source } = this.store.sources[sourceIndex];
+        let sourceObj = this.store.sources[sourceIndex];
+        let source = sourceObj.source;
         let state = this.states[sourceIndex];
         // If a slot ever gets TWO of these, its loops are doubled - every poll and full sync runs twice
         logSyncEvent({ event: "sourceSyncStart", store: this.store.folder, source: source.getDebugName(), sourceIndex, url: this.store.sources[sourceIndex].url, intermediate: this.store.sources[sourceIndex].intermediate });
@@ -296,38 +298,42 @@ export class StoreSync {
             try {
                 let config = await source.getConfig();
                 state.supportsChangesAfter = !!config.supportsChangesAfter;
-                await this.syncSource(sourceIndex, "push");
                 break;
             } catch (e) {
                 logSyncEvent({ event: "initialScanFailed", store: this.store.folder, source: source.getDebugName(), retryInMs: SCAN_RETRY_DELAY, error: String((e as Error).stack ?? e).slice(0, 2000) });
                 await delay(SCAN_RETRY_DELAY);
             }
         }
-        // The loop can also exit by the stop tokens, and nothing may wait forever on a stopped source's initial scan
-        state.scanComplete = true;
-        state.initialScan.resolve(undefined);
         if (this.store.stopped.stop || state.stopped.stop) return;
-        if (!noFullSync()) {
-            try {
-                await this.copySourceFiles(sourceIndex);
-            } catch (e) {
-                console.error(`Copying files from sync source ${source.getDebugName()} failed:`, e);
+        // Hourly for our own disk (no sourceConfig - a cheap local walk, and how a sibling process's writes into the shared folder are found) and for remote peers (index-backed, cheap listings); the slow interval is only for sources where a full listing is genuinely expensive (backblaze)
+        let pollInterval = (!sourceObj.sourceConfig || sourceObj.sourceConfig.type === "remote") && FULL_RESCAN_INTERVAL || FULL_RESCAN_NON_REMOTE_INTERVAL;
+        // Both loops below run one at a time: a change-poll tick landing mid full round would otherwise start a second copy pass over the same pending list, downloading everything twice
+        let serial = runInSerial(async (fnc: () => Promise<void>) => await fnc());
+        await runInfinitePollCallAtStart(pollInterval, () => serial(async () => {
+            while (!this.store.stopped.stop && !state.stopped.stop) {
+                try {
+                    await this.syncSource(sourceIndex)("push");
+                    if (!noFullSync()) await this.copySourceFiles(sourceIndex)();
+                } catch (e) {
+                    logSyncEvent({ event: "scanFailed", store: this.store.folder, source: source.getDebugName(), retryInMs: SCAN_RETRY_DELAY, error: String((e as Error).stack ?? e).slice(0, 2000) });
+                    await delay(SCAN_RETRY_DELAY);
+                    continue;
+                }
+                state.scanComplete = true;
+                state.initialScan.resolve(undefined);
+                break;
             }
-        }
+        }), state.stopped);
         if (state.supportsChangesAfter) {
-            runInfinitePoll(CHANGES_POLL_INTERVAL, async () => {
+            runInfinitePoll(CHANGES_POLL_INTERVAL, () => serial(async () => {
                 // A scan that has not succeeded yet is retried HERE, before anything else - polling changes and copying against a never-scanned source would run on arbitrarily stale information
                 if (!state.scanSucceeded) {
-                    await this.syncSource(sourceIndex, "push");
+                    await this.syncSource(sourceIndex)("push");
                 }
                 await this.pollChanges(sourceIndex);
-                if (!noFullSync()) await this.copySourceFiles(sourceIndex);
-            }, state.stopped);
+                if (!noFullSync()) await this.copySourceFiles(sourceIndex)();
+            }), state.stopped);
         }
-        runInfinitePoll(FULL_RESCAN_UNINDEXED_INTERVAL, async () => {
-            await this.syncSource(sourceIndex, "push");
-            if (!noFullSync()) await this.copySourceFiles(sourceIndex);
-        }, state.stopped);
     }
 
     // The PULL direction: a full metadata scan (size, writeTime, path) of one source, applied to our index. Returns the source's listing (path -> write time), which pushSource uses for the opposite direction.
@@ -383,13 +389,22 @@ export class StoreSync {
             tally[this.updateScanIndex(sourceIndex, file)]++;
         }
         state.scannedCount = files.length;
-        // Entries this source was the holder of, but that its listing did not mention, are forgotten - we were wrong about where they are, which is not the same as them having been deleted. Entries changed after the scan started are kept: the listing may simply predate them. Tombstones are not walked here at all, because they are not files a listing could vouch for.
+        // Entries this source was the holder of, but that its listing did not mention: our own disk takes over as holder when it has a copy, and only with no local copy either is the entry forgotten - we were wrong about where the file is, which is not the same as it having been deleted. Entries changed after the scan started are kept: the listing may simply predate them. Tombstones are not walked here at all, because they are not files a listing could vouch for.
         let removedFromIndex = 0;
+        let repointedToLocal = 0;
         let missingOnSource = 0;
         let scannedSourcesListIndex = this.store.sourcesListIndexOfSlot(sourceIndex);
         for (let [key, entry] of this.store.indexEntries()) {
             if (seen.has(key)) continue;
             if (entry.sourcesListIndex === scannedSourcesListIndex && entry.changedAt < scanStart) {
+                if (sourceIndex !== 0) {
+                    let local = await this.store.sources[0].source.getInfo(key);
+                    if (local) {
+                        this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                        repointedToLocal++;
+                        continue;
+                    }
+                }
                 this.store.purgeIndexEntry(key);
                 removedFromIndex++;
                 continue;
@@ -399,7 +414,7 @@ export class StoreSync {
             if (!routeContains(route, getRoute(key))) continue;
             missingOnSource++;
         }
-        logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, unchanged: tally.unchanged, outsideRoute: tally.filtered, missingOnSource, removedFromIndex });
+        logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, unchanged: tally.unchanged, outsideRoute: tally.filtered, missingOnSource, repointedToLocal, removedFromIndex });
         state.changesAfterTime = Math.max(state.changesAfterTime, scanStart - CHANGES_POLL_OVERLAP);
         state.scanSucceeded = true;
         return seen;
@@ -445,7 +460,7 @@ export class StoreSync {
                 }
                 let holder = await this.store.getEntryHolder(entry);
                 if (!holder) continue;
-                let copied = await copyArchiveFile({ from: holder, to: source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
+                let copied = await copyArchiveFile({ from: holder, to: source, path: key, forceSetImmutable: true, noChecks: true, internal: true });
                 if (!copied) continue;
                 this.store.noteSyncTransfer("sync set", key, copied.size);
                 pushed++;
@@ -511,8 +526,8 @@ export class StoreSync {
         state.changesAfterTime = pollStart - CHANGES_POLL_OVERLAP;
     }
 
-    // Downloads the files a source currently holds onto our own base source (the local disk), preserving their modified times — so a newer local write always wins. Skipped for noFullSync sources (fronting a large database without copying it); reads still down-cache lazily.
-    private async copySourceFiles(sourceIndex: number): Promise<void> {
+    // Downloads the files a source currently holds onto our own base source (the local disk), preserving their modified times — so a newer local write always wins. Skipped for noFullSync sources (fronting a large database without copying it); reads still down-cache lazily. Only one pass per SOURCE runs at a time (cache keys the serializer by source index) - two passes over the same pending list would download everything twice.
+    private copySourceFiles = cache((sourceIndex: number) => runInSerial(async (): Promise<void> => {
         if (sourceIndex === 0) return;
         let { source } = this.store.sources[sourceIndex];
         let state = this.states[sourceIndex];
@@ -562,6 +577,23 @@ export class StoreSync {
         let copiedBytes = 0;
         let missingOnSource = 0;
         let aborted = false;
+        // The source cannot give us the file (its read had nothing, or it errored). Retrying the same entry forever is not an answer: if our own disk holds a copy, IT becomes the holder; with no local copy either, the entry is purged - forgotten, not deleted, exactly like a read that comes up empty everywhere - and the next full pull re-finds it if it exists anywhere.
+        let resolveUnavailable = async (event: string, key: string, entry: IndexEntry, extra: { [key: string]: unknown }) => {
+            let base = { event, store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, ...extra };
+            let local = await this.store.sources[0].source.getInfo(key);
+            if (local) {
+                this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                logSyncEvent({ ...base, resolution: "repointed to our local copy", localSize: local.size });
+                return;
+            }
+            if (this.store.currentWriteTime(key) <= entry.writeTime) {
+                this.store.purgeIndexEntry(key);
+                logSyncEvent({ ...base, resolution: "purged - neither the source nor our disk has it, so it does not exist" });
+                return;
+            }
+            // A newer write landed while we were checking - the entry is no longer the one we found unavailable, so it is left alone
+            logSyncEvent({ ...base, resolution: "kept - a newer write appeared while checking", currentWriteTime: this.store.currentWriteTime(key) });
+        };
         try {
             let nextIndex = 0;
             let consecutiveFailures = 0;
@@ -571,7 +603,7 @@ export class StoreSync {
                     if (index >= pending.length) return;
                     let { key, entry } = pending[index];
                     try {
-                        let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
+                        let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: key, forceSetImmutable: true, noChecks: true, internal: true });
                         if (copied) {
                             copiedFiles++;
                             copiedBytes += copied.size;
@@ -582,25 +614,18 @@ export class StoreSync {
                             }
                         } else {
                             missingOnSource++;
-                            // The source's listing vouched for the file but its read has nothing - and asking again forever is not an answer. If our own disk holds a copy, IT is the real holder, so the entry repoints there; if neither side has bytes, the entry was simply wrong, and it is purged - forgotten, not deleted, exactly like a read that comes up empty everywhere (a scan re-finds the file if it exists anywhere else).
-                            let local = await this.store.sources[0].source.getInfo(key);
-                            if (local) {
-                                this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
-                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "repointed to our local copy", localSize: local.size });
-                            } else if (this.store.currentWriteTime(key) <= entry.writeTime) {
-                                this.store.purgeIndexEntry(key);
-                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "purged - neither the source nor our disk has it, so it does not exist" });
-                            } else {
-                                // A newer write landed while we were checking - the entry is no longer the one we found missing, so it is left alone
-                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "kept - a newer write appeared while checking", currentWriteTime: this.store.currentWriteTime(key) });
-                            }
+                            await resolveUnavailable("fullSyncMissingOnSource", key, entry, {});
                         }
                         consecutiveFailures = 0;
                     } catch (e) {
-                        // One failing file must not abort the sync: it stays on its source (reads still reach it there), and the next sync pass retries it
+                        // A failing file resolves exactly like a missing one (the source cannot give it to us either way), so it never wedges the pass on retries forever - and the next full pull re-finds it if the source recovers
                         failed++;
                         consecutiveFailures++;
-                        logSyncEvent({ event: "fullSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, size: entry.size, error: String((e as Error).stack ?? e).slice(0, 2000) });
+                        try {
+                            await resolveUnavailable("fullSyncCopyFailed", key, entry, { error: String((e as Error).stack ?? e).slice(0, 2000) });
+                        } catch (resolveError) {
+                            logSyncEvent({ event: "fullSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, error: String((e as Error).stack ?? e).slice(0, 2000), resolveError: String((resolveError as Error).stack ?? resolveError).slice(0, 2000) });
+                        }
                         if (consecutiveFailures >= SYNC_MAX_CONSECUTIVE_FAILURES) {
                             aborted = true;
                             return;
@@ -627,7 +652,7 @@ export class StoreSync {
             }
             logSyncEvent({ event: "fullSyncFinish", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, durationMs: Date.now() - activity.startTime, totalFiles: pending.length, totalBytes, processedFiles: activity.doneFiles, copiedFiles, copiedBytes, missingOnSource, failed, aborted });
         }
-    }
+    }));
 
     // ── maintenance ──