sliftutils 1.7.106 → 1.7.108

package/storage/remoteStorage/storageLogs.d.ts

@@ -2,7 +2,7 @@
 /// <reference types="node" />
 import { LogFileInfo } from "../StreamingLogs";
 export declare const LOGS_FOLDER_NAME = "logs";
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Also printed to the console. */
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Stream-only - one entry per write is exactly what the console does NOT need. */
 export declare function logMutation(entry: {
     op: string;
     account: string;

package/storage/remoteStorage/storageLogs.ts

@@ -9,7 +9,10 @@ import { getOwnThreadId } from "../../misc/https/certs";
 
 export const LOGS_FOLDER_NAME = "logs";
 
-const getLogs = lazy((): StreamingLogs | undefined => {
+// NOT lazy: a failure to resolve the folder (the server config may simply not be SET yet this early in startup) must be retried on the next log call, not cached as "no logs ever"
+let logsInstance: StreamingLogs | undefined;
+function getLogs(): StreamingLogs | undefined {
+    if (logsInstance) return logsInstance;
     let folder: string;
     try {
         folder = path.join(getStorageFolder(), LOGS_FOLDER_NAME);
@@ -17,8 +20,9 @@ const getLogs = lazy((): StreamingLogs | undefined => {
         return undefined;
     }
     hookErrorLogging();
-    return new StreamingLogs({ folder, threadId: getThreadIdSafe() });
-});
+    logsInstance = new StreamingLogs({ folder, threadId: getThreadIdSafe() });
+    return logsInstance;
+}
 
 function firstExternalIPv4(): string | undefined {
     for (let addresses of Object.values(os.networkInterfaces())) {
@@ -29,22 +33,28 @@ function firstExternalIPv4(): string | undefined {
     return undefined;
 }
 
+// Memoized on SUCCESS only: the thread cert may not be loaded yet when the first entries are written, and that must not permanently strip the thread id from every later entry
+let cachedThreadId: string | undefined;
 function getThreadIdSafe(): string | undefined {
+    if (cachedThreadId) return cachedThreadId;
     try {
-        return getOwnThreadId(getStorageServerConfigOptional()?.rootDomain || "");
-    } catch {
-        return undefined;
-    }
+        cachedThreadId = getOwnThreadId(getStorageServerConfigOptional()?.rootDomain || "");
+    } catch { }
+    return cachedThreadId;
 }
 
-// Computed once: none of it changes over a process's lifetime
-const baseFields = lazy(() => ({
-    pid: process.pid,
-    threadId: getThreadIdSafe(),
-    entryPoint: process.argv[1],
-    ip: firstExternalIPv4(),
-    domain: getStorageServerConfigOptional()?.domain,
-}));
+const cachedIp = lazy(() => firstExternalIPv4());
+
+// threadId and domain are re-resolved until known (early entries may predate the config/certs); the rest never changes
+function baseFields() {
+    return {
+        pid: process.pid,
+        threadId: getThreadIdSafe(),
+        entryPoint: process.argv[1],
+        ip: cachedIp(),
+        domain: getStorageServerConfigOptional()?.domain,
+    };
+}
 
 // Everything logged here goes BOTH to the storage log stream and the console - one call, never two. The console line carries only the entry (the base fields are constant per process, so they would just be noise there); the stream gets everything.
 function write(kind: string, entry: { [key: string]: unknown }, alsoConsole: boolean): void {
@@ -56,9 +66,9 @@ function write(kind: string, entry: { [key: string]: unknown }, alsoConsole: boo
     logs.log({ kind, time: Date.now(), ...baseFields(), ...entry });
 }
 
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Also printed to the console. */
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Stream-only - one entry per write is exactly what the console does NOT need. */
 export function logMutation(entry: { op: string; account: string; bucketName: string; store?: string; path: string; toPath?: string; size?: number; writeTime?: number; callerId?: string; internal?: boolean }): void {
-    write("mutation", entry, true);
+    write("mutation", entry, false);
 }
 
 /** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. Also printed to the console. */

package/storage/remoteStorage/storeSync.d.ts

@@ -37,14 +37,16 @@ export declare class StoreSync {
     waitForRequiredScans(): Promise<void>;
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     rescanBase(): Promise<void>;
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
+    private syncSource;
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
     boundaryScanRemote(source: IArchives, config: {
         since: number;
         route?: [number, number];
     }): Promise<void>;
-    private runSourceSync;
-    private scanSource;
-    private reconcileSource;
+    private startSourceSyncLoops;
+    private pullSource;
+    private pushSource;
     private updateScanIndex;
     private pollChanges;
     private copySourceFiles;

package/storage/remoteStorage/storeSync.ts

@@ -1,4 +1,4 @@
-import { runInfinitePoll, delay } from "socket-function/src/batching";
+import { runInfinitePoll, delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { timeInMinute, sort, promiseObj } from "socket-function/src/misc";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import {
@@ -21,7 +21,7 @@ const CHANGES_POLL_INTERVAL = 1000 * 60;
 const CONFIG_POLL_INTERVAL = 1000 * 60 * 5;
 // Full metadata rescans. supportsChangesAfter is the heuristic for "one of our own storage servers": their index-backed listings are cheap, so hourly is fine. Everything else (backblaze, plain disk) pays the full listing cost, so it rescans much less often.
 const FULL_RESCAN_INTERVAL = 1000 * 60 * 60;
-const FULL_RESCAN_UNINDEXED_INTERVAL = 1000 * 60 * 60 * 6;
+const FULL_RESCAN_NON_REMOTE_INTERVAL = 1000 * 60 * 60 * 6;
 // Change polls re-request this much overlap, so clock skew between us and a source can't drop changes
 const CHANGES_POLL_OVERLAP = timeInMinute;
 const SCAN_RETRY_DELAY = 1000 * 30;
@@ -48,6 +48,8 @@ type SourceState = {
     supportsChangesAfter: boolean;
     initialScan: ReturnType<typeof promiseObj>;
     scanComplete: boolean;
+    // Whether a scan of this source has ever actually SUCCEEDED - what full syncs gate on: without one, the index's view of this source can be arbitrarily stale (e.g. it was restarting when we first tried), and syncing from stale information churns forever
+    scanSucceeded: boolean;
     // Files seen in this source's scans / change polls so far
     scannedCount: number;
     // Watermark for getChangesAfter2 polls
@@ -63,6 +65,7 @@ function newSourceState(): SourceState {
         supportsChangesAfter: false,
         initialScan: promiseObj(),
         scanComplete: false,
+        scanSucceeded: false,
         scannedCount: 0,
         changesAfterTime: 0,
         stopped: { stop: false },
@@ -94,7 +97,7 @@ export class StoreSync {
     public start(): void {
         for (let i = 0; i < this.store.sources.length; i++) {
             if (!this.isLive(i)) continue;
-            void this.runSourceSync(i);
+            void this.startSourceSyncLoops(i);
         }
         runInfinitePoll(CONFIG_POLL_INTERVAL, () => this.pollRoutingConfig(), this.store.stopped);
         runInfinitePoll(TOMBSTONE_CLEANUP_INTERVAL, () => this.cleanupTombstones(), this.store.stopped);
@@ -144,7 +147,7 @@ export class StoreSync {
     public addSource(slot: number): void {
         this.states[slot] = newSourceState();
         if (this.store.syncStarted) {
-            void this.runSourceSync(slot);
+            void this.startSourceSyncLoops(slot);
         }
     }
 
@@ -216,7 +219,18 @@ export class StoreSync {
 
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     public async rescanBase(): Promise<void> {
-        await this.scanSource(0);
+        await this.syncSource(0);
+    }
+
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
+    private async syncSource(sourceIndex: number, push?: "push"): Promise<void> {
+        let listing = await this.pullSource(sourceIndex);
+        let state = this.states[sourceIndex];
+        state.scanComplete = true;
+        state.initialScan.resolve(undefined);
+        if (push && !this.store.stopped.stop && !state.stopped.stop) {
+            await this.pushSource(sourceIndex, listing);
+        }
     }
 
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
@@ -241,7 +255,7 @@ export class StoreSync {
                 tally.tombstone++;
                 continue;
             }
-            let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: file.path, size: file.size, writeTime: file.createTime, forceSetImmutable: true, noChecks: true, internal: true });
+            let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: file.path, forceSetImmutable: true, noChecks: true, internal: true });
             if (!copied) {
                 console.warn(`Boundary scan could not copy ${file.path} from ${source.getDebugName()} (store ${this.store.folder}): its change feed listed it (${file.size} bytes, writeTime ${new Date(file.createTime).toISOString()}) but the read found nothing`);
                 continue;
@@ -264,15 +278,15 @@ export class StoreSync {
 
     // ── per-source loops ──
 
-    private async runSourceSync(sourceIndex: number): Promise<void> {
+    private async startSourceSyncLoops(sourceIndex: number): Promise<void> {
         await this.store.registerSlot(sourceIndex);
-        let { source } = this.store.sources[sourceIndex];
+        let sourceObj = this.store.sources[sourceIndex];
+        let source = sourceObj.source;
         let state = this.states[sourceIndex];
         // If a slot ever gets TWO of these, its loops are doubled - every poll and full sync runs twice
         logSyncEvent({ event: "sourceSyncStart", store: this.store.folder, source: source.getDebugName(), sourceIndex, url: this.store.sources[sourceIndex].url, intermediate: this.store.sources[sourceIndex].intermediate });
         // Read live for every pass, not captured - the store's source list can change while loops run
         let noFullSync = () => this.store.sources[sourceIndex].noFullSync;
-        let listing: Map<string, number> | undefined;
         // An intermediate is a deploy switchover's temporary alternate PORT onto a source we already have: the same bucket, reachable another way for a few minutes. Scanning it would list exactly what scanning that source lists, against a port that is about to disappear - so it is never scanned, and the source it was split out of covers it for as long as it exists and after it is gone.
         if (this.store.sources[sourceIndex].intermediate) {
             state.scanComplete = true;
@@ -283,47 +297,43 @@ export class StoreSync {
             try {
                 let config = await source.getConfig();
                 state.supportsChangesAfter = !!config.supportsChangesAfter;
-                listing = await this.scanSource(sourceIndex);
                 break;
             } catch (e) {
-                console.error(`Initial scan of sync source ${source.getDebugName()} failed, retrying:`, e);
+                logSyncEvent({ event: "initialScanFailed", store: this.store.folder, source: source.getDebugName(), retryInMs: SCAN_RETRY_DELAY, error: String((e as Error).stack ?? e).slice(0, 2000) });
                 await delay(SCAN_RETRY_DELAY);
             }
         }
-        state.scanComplete = true;
-        state.initialScan.resolve(undefined);
         if (this.store.stopped.stop || state.stopped.stop) return;
-        if (listing) {
-            await this.reconcileSource(sourceIndex, listing);
-        }
-        if (!noFullSync()) {
-            try {
-                await this.copySourceFiles(sourceIndex);
-            } catch (e) {
-                console.error(`Copying files from sync source ${source.getDebugName()} failed:`, e);
+        let pollInterval = sourceObj.sourceConfig?.type === "remote" ? FULL_RESCAN_INTERVAL : FULL_RESCAN_NON_REMOTE_INTERVAL;
+        await runInfinitePollCallAtStart(pollInterval, async () => {
+            while (!this.store.stopped.stop && !state.stopped.stop) {
+                try {
+                    await this.syncSource(sourceIndex, "push");
+                    if (!noFullSync()) await this.copySourceFiles(sourceIndex);
+                } catch (e) {
+                    logSyncEvent({ event: "scanFailed", store: this.store.folder, source: source.getDebugName(), retryInMs: SCAN_RETRY_DELAY, error: String((e as Error).stack ?? e).slice(0, 2000) });
+                    await delay(SCAN_RETRY_DELAY);
+                    continue;
+                }
+                state.scanComplete = true;
+                state.initialScan.resolve(undefined);
+                break;
             }
-        }
+        }, state.stopped);
         if (state.supportsChangesAfter) {
             runInfinitePoll(CHANGES_POLL_INTERVAL, async () => {
+                // A scan that has not succeeded yet is retried HERE, before anything else - polling changes and copying against a never-scanned source would run on arbitrarily stale information
+                if (!state.scanSucceeded) {
+                    await this.syncSource(sourceIndex, "push");
+                }
                 await this.pollChanges(sourceIndex);
                 if (!noFullSync()) await this.copySourceFiles(sourceIndex);
             }, state.stopped);
-            // Change polls only show what the source HAS, never what it's missing, so pushes run on the full-rescan cadence (findInfo on an index-backed source is cheap)
-            runInfinitePoll(FULL_RESCAN_INTERVAL, async () => {
-                let files = await source.findInfo("");
-                await this.reconcileSource(sourceIndex, new Map(files.map(x => [x.path, x.createTime])));
-            }, state.stopped);
-        } else {
-            runInfinitePoll(FULL_RESCAN_UNINDEXED_INTERVAL, async () => {
-                let rescan = await this.scanSource(sourceIndex);
-                await this.reconcileSource(sourceIndex, rescan);
-                if (!noFullSync()) await this.copySourceFiles(sourceIndex);
-            }, state.stopped);
         }
     }
 
-    // Full metadata scan (size, writeTime, path) of one source, applied to the index. Returns the source's listing (path -> write time), which reconcileSource uses for the push direction.
-    private async scanSource(sourceIndex: number): Promise<Map<string, number>> {
+    // The PULL direction: a full metadata scan (size, writeTime, path) of one source, applied to our index. Returns the source's listing (path -> write time), which pushSource uses for the opposite direction.
+    private async pullSource(sourceIndex: number): Promise<Map<string, number>> {
         let { source, route } = this.store.sources[sourceIndex];
         let state = this.states[sourceIndex];
         let scanStart = Date.now();
@@ -375,13 +385,22 @@ export class StoreSync {
             tally[this.updateScanIndex(sourceIndex, file)]++;
         }
         state.scannedCount = files.length;
-        // Entries this source was the holder of, but that its listing did not mention, are forgotten - we were wrong about where they are, which is not the same as them having been deleted. Entries changed after the scan started are kept: the listing may simply predate them. Tombstones are not walked here at all, because they are not files a listing could vouch for.
+        // Entries this source was the holder of, but that its listing did not mention: our own disk takes over as holder when it has a copy, and only with no local copy either is the entry forgotten - we were wrong about where the file is, which is not the same as it having been deleted. Entries changed after the scan started are kept: the listing may simply predate them. Tombstones are not walked here at all, because they are not files a listing could vouch for.
         let removedFromIndex = 0;
+        let repointedToLocal = 0;
         let missingOnSource = 0;
         let scannedSourcesListIndex = this.store.sourcesListIndexOfSlot(sourceIndex);
         for (let [key, entry] of this.store.indexEntries()) {
             if (seen.has(key)) continue;
             if (entry.sourcesListIndex === scannedSourcesListIndex && entry.changedAt < scanStart) {
+                if (sourceIndex !== 0) {
+                    let local = await this.store.sources[0].source.getInfo(key);
+                    if (local) {
+                        this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                        repointedToLocal++;
+                        continue;
+                    }
+                }
                 this.store.purgeIndexEntry(key);
                 removedFromIndex++;
                 continue;
@@ -391,13 +410,14 @@ export class StoreSync {
             if (!routeContains(route, getRoute(key))) continue;
             missingOnSource++;
         }
-        logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, unchanged: tally.unchanged, outsideRoute: tally.filtered, missingOnSource, removedFromIndex });
+        logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, unchanged: tally.unchanged, outsideRoute: tally.filtered, missingOnSource, repointedToLocal, removedFromIndex });
         state.changesAfterTime = Math.max(state.changesAfterTime, scanStart - CHANGES_POLL_OVERLAP);
+        state.scanSucceeded = true;
         return seen;
     }
 
     // The push direction of synchronization: everything we know that the source is missing (or holds an older copy of) is written to it — including deletions, as tombstone writes. This is what heals a source whose background writes failed (e.g. it was down): the next scan sees what's missing and re-sends it. A failing file is skipped, not fatal (immutable targets are handled by forceSetImmutable, and one unreadable value must not stop the rest of the pass) - only a run of consecutive failures (the source itself is down) aborts until the next scan cycle.
-    private async reconcileSource(sourceIndex: number, listing: Map<string, number>): Promise<void> {
+    private async pushSource(sourceIndex: number, listing: Map<string, number>): Promise<void> {
         let { source, validWindows, route } = this.store.sources[sourceIndex];
         let state = this.states[sourceIndex];
         let acceptsWrites = windowsAcceptWrites(validWindows);
@@ -436,7 +456,7 @@ export class StoreSync {
                 }
                 let holder = await this.store.getEntryHolder(entry);
                 if (!holder) continue;
-                let copied = await copyArchiveFile({ from: holder, to: source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
+                let copied = await copyArchiveFile({ from: holder, to: source, path: key, forceSetImmutable: true, noChecks: true, internal: true });
                 if (!copied) continue;
                 this.store.noteSyncTransfer("sync set", key, copied.size);
                 pushed++;
@@ -458,9 +478,7 @@ export class StoreSync {
         if (failed) {
             console.error(`Reconciling sync source ${source.getDebugName()} (store ${this.store.folder}): pushed ${pushed} files, ${failed} failed${aborted && ` before aborting the pass (${consecutiveFailures} consecutive failures - the source looks down; the next scan cycle retries)` || ""}. First errors: ${errors.join(" | ")}`);
         }
-        if (pushed || failed) {
-            logSyncEvent({ event: "reconcileFinish", store: this.store.folder, source: source.getDebugName(), pushed, failed, aborted });
-        }
+        logSyncEvent({ event: "reconcileFinish", store: this.store.folder, source: source.getDebugName(), pushed, failed, aborted });
     }
 
     private updateScanIndex(sourceIndex: number, file: ArchiveFileInfo): ScanOutcome {
@@ -555,36 +573,54 @@ export class StoreSync {
         let copiedBytes = 0;
         let missingOnSource = 0;
         let aborted = false;
+        // The source cannot give us the file (its read had nothing, or it errored). Retrying the same entry forever is not an answer: if our own disk holds a copy, IT becomes the holder; with no local copy either, the entry is purged - forgotten, not deleted, exactly like a read that comes up empty everywhere - and the next full pull re-finds it if it exists anywhere.
+        let resolveUnavailable = async (event: string, key: string, entry: IndexEntry, extra: { [key: string]: unknown }) => {
+            let base = { event, store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, ...extra };
+            let local = await this.store.sources[0].source.getInfo(key);
+            if (local) {
+                this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                logSyncEvent({ ...base, resolution: "repointed to our local copy", localSize: local.size });
+                return;
+            }
+            if (this.store.currentWriteTime(key) <= entry.writeTime) {
+                this.store.purgeIndexEntry(key);
+                logSyncEvent({ ...base, resolution: "purged - neither the source nor our disk has it, so it does not exist" });
+                return;
+            }
+            // A newer write landed while we were checking - the entry is no longer the one we found unavailable, so it is left alone
+            logSyncEvent({ ...base, resolution: "kept - a newer write appeared while checking", currentWriteTime: this.store.currentWriteTime(key) });
+        };
         try {
             let nextIndex = 0;
             let consecutiveFailures = 0;
-            let errors: string[] = [];
             let copyWorker = async () => {
                 while (!aborted && !this.store.stopped.stop && !state.stopped.stop) {
                     let index = nextIndex++;
                     if (index >= pending.length) return;
                     let { key, entry } = pending[index];
                     try {
-                        let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
+                        let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: key, forceSetImmutable: true, noChecks: true, internal: true });
                         if (copied) {
                             copiedFiles++;
                             copiedBytes += copied.size;
                             this.store.noteSyncTransfer("sync get", key, copied.size);
-                            // The copy carries the source's write time, and the index commits it under the normal ordering rule (>= the current time wins) - setIndexEntry refusing means a NEWER write landed while we copied, which must be said, not swallowed
+                            // The copy carries the source's write time, and the index commits it under the normal ordering rule (>= the current time wins) - it refusing means a NEWER write landed while we copied, which must be said, not swallowed
                             if (!this.store.setIndexEntry(key, { writeTime: copied.writeTime, size: copied.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) })) {
-                                console.warn(`Full sync copied ${key} from ${source.getDebugName()} (store ${this.store.folder}) with writeTime ${copied.writeTime} (${new Date(copied.writeTime).toISOString()}), but the index now has ${this.store.currentWriteTime(key)} (${new Date(this.store.currentWriteTime(key)).toISOString()}) - something wrote it while we were syncing; keeping the newer entry`);
+                                logSyncEvent({ event: "fullSyncCommitSuperseded", store: this.store.folder, source: source.getDebugName(), path: key, copiedWriteTime: copied.writeTime, currentWriteTime: this.store.currentWriteTime(key) });
                             }
                         } else {
                             missingOnSource++;
-                            console.warn(`Full sync could not copy ${key} from ${source.getDebugName()} (store ${this.store.folder}): the source answered but no longer has it (expected ${entry.size} bytes, writeTime ${new Date(entry.writeTime).toISOString()})`);
+                            await resolveUnavailable("fullSyncMissingOnSource", key, entry, {});
                         }
                         consecutiveFailures = 0;
                     } catch (e) {
-                        // One failing file must not abort the sync: it stays on its source (reads still reach it there), and the next sync pass retries it
+                        // A failing file resolves exactly like a missing one (the source cannot give it to us either way), so it never wedges the pass on retries forever - and the next full pull re-finds it if the source recovers
                         failed++;
                         consecutiveFailures++;
-                        if (errors.length < SYNC_ERROR_LOG_LIMIT) {
-                            errors.push(`${key}: ${(e as Error).stack ?? e}`);
+                        try {
+                            await resolveUnavailable("fullSyncCopyFailed", key, entry, { error: String((e as Error).stack ?? e).slice(0, 2000) });
+                        } catch (resolveError) {
+                            logSyncEvent({ event: "fullSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, error: String((e as Error).stack ?? e).slice(0, 2000), resolveError: String((resolveError as Error).stack ?? resolveError).slice(0, 2000) });
                         }
                         if (consecutiveFailures >= SYNC_MAX_CONSECUTIVE_FAILURES) {
                             aborted = true;
@@ -602,9 +638,6 @@ export class StoreSync {
                 workers.push(copyWorker());
             }
             await Promise.all(workers);
-            if (failed) {
-                console.error(`Full sync from ${source.getDebugName()} (store ${this.store.folder}): ${failed} of ${pending.length} files failed to copy${aborted && ` before aborting the pass (${SYNC_MAX_CONSECUTIVE_FAILURES} consecutive failures - the source looks down; the next sync pass retries)` || " (they stay on their source, and the next sync pass retries them)"}. First errors: ${errors.join(" | ")}`);
-            }
         } finally {
             clearInterval(progressTimer);
             clearInterval(slowErrorTimer);