sliftutils 1.7.107 → 1.7.108

package/storage/remoteStorage/storageLogs.d.ts

@@ -2,7 +2,7 @@
 /// <reference types="node" />
 import { LogFileInfo } from "../StreamingLogs";
 export declare const LOGS_FOLDER_NAME = "logs";
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Also printed to the console. */
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Stream-only - one entry per write is exactly what the console does NOT need. */
 export declare function logMutation(entry: {
     op: string;
     account: string;

package/storage/remoteStorage/storageLogs.ts

@@ -66,9 +66,9 @@ function write(kind: string, entry: { [key: string]: unknown }, alsoConsole: boo
     logs.log({ kind, time: Date.now(), ...baseFields(), ...entry });
 }
 
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Also printed to the console. */
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Stream-only - one entry per write is exactly what the console does NOT need. */
 export function logMutation(entry: { op: string; account: string; bucketName: string; store?: string; path: string; toPath?: string; size?: number; writeTime?: number; callerId?: string; internal?: boolean }): void {
-    write("mutation", entry, true);
+    write("mutation", entry, false);
 }
 
 /** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. Also printed to the console. */

package/storage/remoteStorage/storeSync.d.ts

@@ -44,7 +44,7 @@ export declare class StoreSync {
         since: number;
         route?: [number, number];
     }): Promise<void>;
-    private runSourceSync;
+    private startSourceSyncLoops;
     private pullSource;
     private pushSource;
     private updateScanIndex;

package/storage/remoteStorage/storeSync.ts

@@ -1,4 +1,4 @@
-import { runInfinitePoll, delay } from "socket-function/src/batching";
+import { runInfinitePoll, delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { timeInMinute, sort, promiseObj } from "socket-function/src/misc";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import {
@@ -21,7 +21,7 @@ const CHANGES_POLL_INTERVAL = 1000 * 60;
 const CONFIG_POLL_INTERVAL = 1000 * 60 * 5;
 // Full metadata rescans. supportsChangesAfter is the heuristic for "one of our own storage servers": their index-backed listings are cheap, so hourly is fine. Everything else (backblaze, plain disk) pays the full listing cost, so it rescans much less often.
 const FULL_RESCAN_INTERVAL = 1000 * 60 * 60;
-const FULL_RESCAN_UNINDEXED_INTERVAL = 1000 * 60 * 60 * 6;
+const FULL_RESCAN_NON_REMOTE_INTERVAL = 1000 * 60 * 60 * 6;
 // Change polls re-request this much overlap, so clock skew between us and a source can't drop changes
 const CHANGES_POLL_OVERLAP = timeInMinute;
 const SCAN_RETRY_DELAY = 1000 * 30;
@@ -97,7 +97,7 @@ export class StoreSync {
     public start(): void {
         for (let i = 0; i < this.store.sources.length; i++) {
             if (!this.isLive(i)) continue;
-            void this.runSourceSync(i);
+            void this.startSourceSyncLoops(i);
         }
         runInfinitePoll(CONFIG_POLL_INTERVAL, () => this.pollRoutingConfig(), this.store.stopped);
         runInfinitePoll(TOMBSTONE_CLEANUP_INTERVAL, () => this.cleanupTombstones(), this.store.stopped);
@@ -147,7 +147,7 @@ export class StoreSync {
     public addSource(slot: number): void {
         this.states[slot] = newSourceState();
         if (this.store.syncStarted) {
-            void this.runSourceSync(slot);
+            void this.startSourceSyncLoops(slot);
         }
     }
 
@@ -255,7 +255,7 @@ export class StoreSync {
                 tally.tombstone++;
                 continue;
             }
-            let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: file.path, size: file.size, writeTime: file.createTime, forceSetImmutable: true, noChecks: true, internal: true });
+            let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: file.path, forceSetImmutable: true, noChecks: true, internal: true });
             if (!copied) {
                 console.warn(`Boundary scan could not copy ${file.path} from ${source.getDebugName()} (store ${this.store.folder}): its change feed listed it (${file.size} bytes, writeTime ${new Date(file.createTime).toISOString()}) but the read found nothing`);
                 continue;
@@ -278,9 +278,10 @@ export class StoreSync {
 
     // ── per-source loops ──
 
-    private async runSourceSync(sourceIndex: number): Promise<void> {
+    private async startSourceSyncLoops(sourceIndex: number): Promise<void> {
         await this.store.registerSlot(sourceIndex);
-        let { source } = this.store.sources[sourceIndex];
+        let sourceObj = this.store.sources[sourceIndex];
+        let source = sourceObj.source;
         let state = this.states[sourceIndex];
         // If a slot ever gets TWO of these, its loops are doubled - every poll and full sync runs twice
         logSyncEvent({ event: "sourceSyncStart", store: this.store.folder, source: source.getDebugName(), sourceIndex, url: this.store.sources[sourceIndex].url, intermediate: this.store.sources[sourceIndex].intermediate });
@@ -296,24 +297,29 @@ export class StoreSync {
             try {
                 let config = await source.getConfig();
                 state.supportsChangesAfter = !!config.supportsChangesAfter;
-                await this.syncSource(sourceIndex, "push");
                 break;
             } catch (e) {
                 logSyncEvent({ event: "initialScanFailed", store: this.store.folder, source: source.getDebugName(), retryInMs: SCAN_RETRY_DELAY, error: String((e as Error).stack ?? e).slice(0, 2000) });
                 await delay(SCAN_RETRY_DELAY);
             }
         }
-        // The loop can also exit by the stop tokens, and nothing may wait forever on a stopped source's initial scan
-        state.scanComplete = true;
-        state.initialScan.resolve(undefined);
         if (this.store.stopped.stop || state.stopped.stop) return;
-        if (!noFullSync()) {
-            try {
-                await this.copySourceFiles(sourceIndex);
-            } catch (e) {
-                console.error(`Copying files from sync source ${source.getDebugName()} failed:`, e);
+        let pollInterval = sourceObj.sourceConfig?.type === "remote" ? FULL_RESCAN_INTERVAL : FULL_RESCAN_NON_REMOTE_INTERVAL;
+        await runInfinitePollCallAtStart(pollInterval, async () => {
+            while (!this.store.stopped.stop && !state.stopped.stop) {
+                try {
+                    await this.syncSource(sourceIndex, "push");
+                    if (!noFullSync()) await this.copySourceFiles(sourceIndex);
+                } catch (e) {
+                    logSyncEvent({ event: "scanFailed", store: this.store.folder, source: source.getDebugName(), retryInMs: SCAN_RETRY_DELAY, error: String((e as Error).stack ?? e).slice(0, 2000) });
+                    await delay(SCAN_RETRY_DELAY);
+                    continue;
+                }
+                state.scanComplete = true;
+                state.initialScan.resolve(undefined);
+                break;
             }
-        }
+        }, state.stopped);
         if (state.supportsChangesAfter) {
             runInfinitePoll(CHANGES_POLL_INTERVAL, async () => {
                 // A scan that has not succeeded yet is retried HERE, before anything else - polling changes and copying against a never-scanned source would run on arbitrarily stale information
@@ -324,10 +330,6 @@ export class StoreSync {
                 if (!noFullSync()) await this.copySourceFiles(sourceIndex);
             }, state.stopped);
         }
-        runInfinitePoll(FULL_RESCAN_UNINDEXED_INTERVAL, async () => {
-            await this.syncSource(sourceIndex, "push");
-            if (!noFullSync()) await this.copySourceFiles(sourceIndex);
-        }, state.stopped);
     }
 
     // The PULL direction: a full metadata scan (size, writeTime, path) of one source, applied to our index. Returns the source's listing (path -> write time), which pushSource uses for the opposite direction.
@@ -383,13 +385,22 @@ export class StoreSync {
             tally[this.updateScanIndex(sourceIndex, file)]++;
         }
         state.scannedCount = files.length;
-        // Entries this source was the holder of, but that its listing did not mention, are forgotten - we were wrong about where they are, which is not the same as them having been deleted. Entries changed after the scan started are kept: the listing may simply predate them. Tombstones are not walked here at all, because they are not files a listing could vouch for.
+        // Entries this source was the holder of, but that its listing did not mention: our own disk takes over as holder when it has a copy, and only with no local copy either is the entry forgotten - we were wrong about where the file is, which is not the same as it having been deleted. Entries changed after the scan started are kept: the listing may simply predate them. Tombstones are not walked here at all, because they are not files a listing could vouch for.
         let removedFromIndex = 0;
+        let repointedToLocal = 0;
         let missingOnSource = 0;
         let scannedSourcesListIndex = this.store.sourcesListIndexOfSlot(sourceIndex);
         for (let [key, entry] of this.store.indexEntries()) {
             if (seen.has(key)) continue;
             if (entry.sourcesListIndex === scannedSourcesListIndex && entry.changedAt < scanStart) {
+                if (sourceIndex !== 0) {
+                    let local = await this.store.sources[0].source.getInfo(key);
+                    if (local) {
+                        this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                        repointedToLocal++;
+                        continue;
+                    }
+                }
                 this.store.purgeIndexEntry(key);
                 removedFromIndex++;
                 continue;
@@ -399,7 +410,7 @@ export class StoreSync {
             if (!routeContains(route, getRoute(key))) continue;
             missingOnSource++;
         }
-        logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, unchanged: tally.unchanged, outsideRoute: tally.filtered, missingOnSource, removedFromIndex });
+        logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, unchanged: tally.unchanged, outsideRoute: tally.filtered, missingOnSource, repointedToLocal, removedFromIndex });
         state.changesAfterTime = Math.max(state.changesAfterTime, scanStart - CHANGES_POLL_OVERLAP);
         state.scanSucceeded = true;
         return seen;
@@ -445,7 +456,7 @@ export class StoreSync {
                 }
                 let holder = await this.store.getEntryHolder(entry);
                 if (!holder) continue;
-                let copied = await copyArchiveFile({ from: holder, to: source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
+                let copied = await copyArchiveFile({ from: holder, to: source, path: key, forceSetImmutable: true, noChecks: true, internal: true });
                 if (!copied) continue;
                 this.store.noteSyncTransfer("sync set", key, copied.size);
                 pushed++;
@@ -562,6 +573,23 @@ export class StoreSync {
         let copiedBytes = 0;
         let missingOnSource = 0;
         let aborted = false;
+        // The source cannot give us the file (its read had nothing, or it errored). Retrying the same entry forever is not an answer: if our own disk holds a copy, IT becomes the holder; with no local copy either, the entry is purged - forgotten, not deleted, exactly like a read that comes up empty everywhere - and the next full pull re-finds it if it exists anywhere.
+        let resolveUnavailable = async (event: string, key: string, entry: IndexEntry, extra: { [key: string]: unknown }) => {
+            let base = { event, store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, ...extra };
+            let local = await this.store.sources[0].source.getInfo(key);
+            if (local) {
+                this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                logSyncEvent({ ...base, resolution: "repointed to our local copy", localSize: local.size });
+                return;
+            }
+            if (this.store.currentWriteTime(key) <= entry.writeTime) {
+                this.store.purgeIndexEntry(key);
+                logSyncEvent({ ...base, resolution: "purged - neither the source nor our disk has it, so it does not exist" });
+                return;
+            }
+            // A newer write landed while we were checking - the entry is no longer the one we found unavailable, so it is left alone
+            logSyncEvent({ ...base, resolution: "kept - a newer write appeared while checking", currentWriteTime: this.store.currentWriteTime(key) });
+        };
         try {
             let nextIndex = 0;
             let consecutiveFailures = 0;
@@ -571,7 +599,7 @@ export class StoreSync {
                     if (index >= pending.length) return;
                     let { key, entry } = pending[index];
                     try {
-                        let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
+                        let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: key, forceSetImmutable: true, noChecks: true, internal: true });
                         if (copied) {
                             copiedFiles++;
                             copiedBytes += copied.size;
@@ -582,25 +610,18 @@ export class StoreSync {
                             }
                         } else {
                             missingOnSource++;
-                            // The source's listing vouched for the file but its read has nothing - and asking again forever is not an answer. If our own disk holds a copy, IT is the real holder, so the entry repoints there; if neither side has bytes, the entry was simply wrong, and it is purged - forgotten, not deleted, exactly like a read that comes up empty everywhere (a scan re-finds the file if it exists anywhere else).
-                            let local = await this.store.sources[0].source.getInfo(key);
-                            if (local) {
-                                this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
-                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "repointed to our local copy", localSize: local.size });
-                            } else if (this.store.currentWriteTime(key) <= entry.writeTime) {
-                                this.store.purgeIndexEntry(key);
-                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "purged - neither the source nor our disk has it, so it does not exist" });
-                            } else {
-                                // A newer write landed while we were checking - the entry is no longer the one we found missing, so it is left alone
-                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "kept - a newer write appeared while checking", currentWriteTime: this.store.currentWriteTime(key) });
-                            }
+                            await resolveUnavailable("fullSyncMissingOnSource", key, entry, {});
                         }
                         consecutiveFailures = 0;
                     } catch (e) {
-                        // One failing file must not abort the sync: it stays on its source (reads still reach it there), and the next sync pass retries it
+                        // A failing file resolves exactly like a missing one (the source cannot give it to us either way), so it never wedges the pass on retries forever - and the next full pull re-finds it if the source recovers
                         failed++;
                         consecutiveFailures++;
-                        logSyncEvent({ event: "fullSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, size: entry.size, error: String((e as Error).stack ?? e).slice(0, 2000) });
+                        try {
+                            await resolveUnavailable("fullSyncCopyFailed", key, entry, { error: String((e as Error).stack ?? e).slice(0, 2000) });
+                        } catch (resolveError) {
+                            logSyncEvent({ event: "fullSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, error: String((e as Error).stack ?? e).slice(0, 2000), resolveError: String((resolveError as Error).stack ?? resolveError).slice(0, 2000) });
+                        }
                         if (consecutiveFailures >= SYNC_MAX_CONSECUTIVE_FAILURES) {
                             aborted = true;
                             return;