sliftutils 1.7.106 → 1.7.107

package/storage/remoteStorage/storageLogs.ts

@@ -9,7 +9,10 @@ import { getOwnThreadId } from "../../misc/https/certs";
 
 export const LOGS_FOLDER_NAME = "logs";
 
-const getLogs = lazy((): StreamingLogs | undefined => {
+// NOT lazy: a failure to resolve the folder (the server config may simply not be SET yet this early in startup) must be retried on the next log call, not cached as "no logs ever"
+let logsInstance: StreamingLogs | undefined;
+function getLogs(): StreamingLogs | undefined {
+    if (logsInstance) return logsInstance;
     let folder: string;
     try {
         folder = path.join(getStorageFolder(), LOGS_FOLDER_NAME);
@@ -17,8 +20,9 @@ const getLogs = lazy((): StreamingLogs | undefined => {
         return undefined;
     }
     hookErrorLogging();
-    return new StreamingLogs({ folder, threadId: getThreadIdSafe() });
-});
+    logsInstance = new StreamingLogs({ folder, threadId: getThreadIdSafe() });
+    return logsInstance;
+}
 
 function firstExternalIPv4(): string | undefined {
     for (let addresses of Object.values(os.networkInterfaces())) {
@@ -29,22 +33,28 @@ function firstExternalIPv4(): string | undefined {
     return undefined;
 }
 
+// Memoized on SUCCESS only: the thread cert may not be loaded yet when the first entries are written, and that must not permanently strip the thread id from every later entry
+let cachedThreadId: string | undefined;
 function getThreadIdSafe(): string | undefined {
+    if (cachedThreadId) return cachedThreadId;
     try {
-        return getOwnThreadId(getStorageServerConfigOptional()?.rootDomain || "");
-    } catch {
-        return undefined;
-    }
+        cachedThreadId = getOwnThreadId(getStorageServerConfigOptional()?.rootDomain || "");
+    } catch { }
+    return cachedThreadId;
 }
 
-// Computed once: none of it changes over a process's lifetime
-const baseFields = lazy(() => ({
-    pid: process.pid,
-    threadId: getThreadIdSafe(),
-    entryPoint: process.argv[1],
-    ip: firstExternalIPv4(),
-    domain: getStorageServerConfigOptional()?.domain,
-}));
+const cachedIp = lazy(() => firstExternalIPv4());
+
+// threadId and domain are re-resolved until known (early entries may predate the config/certs); the rest never changes
+function baseFields() {
+    return {
+        pid: process.pid,
+        threadId: getThreadIdSafe(),
+        entryPoint: process.argv[1],
+        ip: cachedIp(),
+        domain: getStorageServerConfigOptional()?.domain,
+    };
+}
 
 // Everything logged here goes BOTH to the storage log stream and the console - one call, never two. The console line carries only the entry (the base fields are constant per process, so they would just be noise there); the stream gets everything.
 function write(kind: string, entry: { [key: string]: unknown }, alsoConsole: boolean): void {

package/storage/remoteStorage/storeSync.d.ts

@@ -37,14 +37,16 @@ export declare class StoreSync {
     waitForRequiredScans(): Promise<void>;
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     rescanBase(): Promise<void>;
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
+    private syncSource;
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
     boundaryScanRemote(source: IArchives, config: {
         since: number;
         route?: [number, number];
     }): Promise<void>;
     private runSourceSync;
-    private scanSource;
-    private reconcileSource;
+    private pullSource;
+    private pushSource;
     private updateScanIndex;
     private pollChanges;
     private copySourceFiles;

package/storage/remoteStorage/storeSync.ts

@@ -48,6 +48,8 @@ type SourceState = {
     supportsChangesAfter: boolean;
     initialScan: ReturnType<typeof promiseObj>;
     scanComplete: boolean;
+    // Whether a scan of this source has ever actually SUCCEEDED - what full syncs gate on: without one, the index's view of this source can be arbitrarily stale (e.g. it was restarting when we first tried), and syncing from stale information churns forever
+    scanSucceeded: boolean;
     // Files seen in this source's scans / change polls so far
     scannedCount: number;
     // Watermark for getChangesAfter2 polls
@@ -63,6 +65,7 @@ function newSourceState(): SourceState {
         supportsChangesAfter: false,
         initialScan: promiseObj(),
         scanComplete: false,
+        scanSucceeded: false,
         scannedCount: 0,
         changesAfterTime: 0,
         stopped: { stop: false },
@@ -216,7 +219,18 @@ export class StoreSync {
 
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     public async rescanBase(): Promise<void> {
-        await this.scanSource(0);
+        await this.syncSource(0);
+    }
+
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
+    private async syncSource(sourceIndex: number, push?: "push"): Promise<void> {
+        let listing = await this.pullSource(sourceIndex);
+        let state = this.states[sourceIndex];
+        state.scanComplete = true;
+        state.initialScan.resolve(undefined);
+        if (push && !this.store.stopped.stop && !state.stopped.stop) {
+            await this.pushSource(sourceIndex, listing);
+        }
     }
 
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
@@ -272,7 +286,6 @@ export class StoreSync {
         logSyncEvent({ event: "sourceSyncStart", store: this.store.folder, source: source.getDebugName(), sourceIndex, url: this.store.sources[sourceIndex].url, intermediate: this.store.sources[sourceIndex].intermediate });
         // Read live for every pass, not captured - the store's source list can change while loops run
         let noFullSync = () => this.store.sources[sourceIndex].noFullSync;
-        let listing: Map<string, number> | undefined;
         // An intermediate is a deploy switchover's temporary alternate PORT onto a source we already have: the same bucket, reachable another way for a few minutes. Scanning it would list exactly what scanning that source lists, against a port that is about to disappear - so it is never scanned, and the source it was split out of covers it for as long as it exists and after it is gone.
         if (this.store.sources[sourceIndex].intermediate) {
             state.scanComplete = true;
@@ -283,19 +296,17 @@ export class StoreSync {
             try {
                 let config = await source.getConfig();
                 state.supportsChangesAfter = !!config.supportsChangesAfter;
-                listing = await this.scanSource(sourceIndex);
+                await this.syncSource(sourceIndex, "push");
                 break;
             } catch (e) {
-                console.error(`Initial scan of sync source ${source.getDebugName()} failed, retrying:`, e);
+                logSyncEvent({ event: "initialScanFailed", store: this.store.folder, source: source.getDebugName(), retryInMs: SCAN_RETRY_DELAY, error: String((e as Error).stack ?? e).slice(0, 2000) });
                 await delay(SCAN_RETRY_DELAY);
             }
         }
+        // The loop can also exit by the stop tokens, and nothing may wait forever on a stopped source's initial scan
         state.scanComplete = true;
         state.initialScan.resolve(undefined);
         if (this.store.stopped.stop || state.stopped.stop) return;
-        if (listing) {
-            await this.reconcileSource(sourceIndex, listing);
-        }
         if (!noFullSync()) {
             try {
                 await this.copySourceFiles(sourceIndex);
@@ -305,25 +316,22 @@ export class StoreSync {
         }
         if (state.supportsChangesAfter) {
             runInfinitePoll(CHANGES_POLL_INTERVAL, async () => {
+                // A scan that has not succeeded yet is retried HERE, before anything else - polling changes and copying against a never-scanned source would run on arbitrarily stale information
+                if (!state.scanSucceeded) {
+                    await this.syncSource(sourceIndex, "push");
+                }
                 await this.pollChanges(sourceIndex);
                 if (!noFullSync()) await this.copySourceFiles(sourceIndex);
             }, state.stopped);
-            // Change polls only show what the source HAS, never what it's missing, so pushes run on the full-rescan cadence (findInfo on an index-backed source is cheap)
-            runInfinitePoll(FULL_RESCAN_INTERVAL, async () => {
-                let files = await source.findInfo("");
-                await this.reconcileSource(sourceIndex, new Map(files.map(x => [x.path, x.createTime])));
-            }, state.stopped);
-        } else {
-            runInfinitePoll(FULL_RESCAN_UNINDEXED_INTERVAL, async () => {
-                let rescan = await this.scanSource(sourceIndex);
-                await this.reconcileSource(sourceIndex, rescan);
-                if (!noFullSync()) await this.copySourceFiles(sourceIndex);
-            }, state.stopped);
         }
+        runInfinitePoll(FULL_RESCAN_UNINDEXED_INTERVAL, async () => {
+            await this.syncSource(sourceIndex, "push");
+            if (!noFullSync()) await this.copySourceFiles(sourceIndex);
+        }, state.stopped);
     }
 
-    // Full metadata scan (size, writeTime, path) of one source, applied to the index. Returns the source's listing (path -> write time), which reconcileSource uses for the push direction.
-    private async scanSource(sourceIndex: number): Promise<Map<string, number>> {
+    // The PULL direction: a full metadata scan (size, writeTime, path) of one source, applied to our index. Returns the source's listing (path -> write time), which pushSource uses for the opposite direction.
+    private async pullSource(sourceIndex: number): Promise<Map<string, number>> {
         let { source, route } = this.store.sources[sourceIndex];
         let state = this.states[sourceIndex];
         let scanStart = Date.now();
@@ -393,11 +401,12 @@ export class StoreSync {
         }
         logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, unchanged: tally.unchanged, outsideRoute: tally.filtered, missingOnSource, removedFromIndex });
         state.changesAfterTime = Math.max(state.changesAfterTime, scanStart - CHANGES_POLL_OVERLAP);
+        state.scanSucceeded = true;
         return seen;
     }
 
     // The push direction of synchronization: everything we know that the source is missing (or holds an older copy of) is written to it — including deletions, as tombstone writes. This is what heals a source whose background writes failed (e.g. it was down): the next scan sees what's missing and re-sends it. A failing file is skipped, not fatal (immutable targets are handled by forceSetImmutable, and one unreadable value must not stop the rest of the pass) - only a run of consecutive failures (the source itself is down) aborts until the next scan cycle.
-    private async reconcileSource(sourceIndex: number, listing: Map<string, number>): Promise<void> {
+    private async pushSource(sourceIndex: number, listing: Map<string, number>): Promise<void> {
         let { source, validWindows, route } = this.store.sources[sourceIndex];
         let state = this.states[sourceIndex];
         let acceptsWrites = windowsAcceptWrites(validWindows);
@@ -458,9 +467,7 @@ export class StoreSync {
         if (failed) {
             console.error(`Reconciling sync source ${source.getDebugName()} (store ${this.store.folder}): pushed ${pushed} files, ${failed} failed${aborted && ` before aborting the pass (${consecutiveFailures} consecutive failures - the source looks down; the next scan cycle retries)` || ""}. First errors: ${errors.join(" | ")}`);
         }
-        if (pushed || failed) {
-            logSyncEvent({ event: "reconcileFinish", store: this.store.folder, source: source.getDebugName(), pushed, failed, aborted });
-        }
+        logSyncEvent({ event: "reconcileFinish", store: this.store.folder, source: source.getDebugName(), pushed, failed, aborted });
     }
 
     private updateScanIndex(sourceIndex: number, file: ArchiveFileInfo): ScanOutcome {
@@ -558,7 +565,6 @@ export class StoreSync {
         try {
             let nextIndex = 0;
             let consecutiveFailures = 0;
-            let errors: string[] = [];
             let copyWorker = async () => {
                 while (!aborted && !this.store.stopped.stop && !state.stopped.stop) {
                     let index = nextIndex++;
@@ -570,22 +576,31 @@ export class StoreSync {
                             copiedFiles++;
                             copiedBytes += copied.size;
                             this.store.noteSyncTransfer("sync get", key, copied.size);
-                            // The copy carries the source's write time, and the index commits it under the normal ordering rule (>= the current time wins) - setIndexEntry refusing means a NEWER write landed while we copied, which must be said, not swallowed
+                            // The copy carries the source's write time, and the index commits it under the normal ordering rule (>= the current time wins) - it refusing means a NEWER write landed while we copied, which must be said, not swallowed
                             if (!this.store.setIndexEntry(key, { writeTime: copied.writeTime, size: copied.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) })) {
-                                console.warn(`Full sync copied ${key} from ${source.getDebugName()} (store ${this.store.folder}) with writeTime ${copied.writeTime} (${new Date(copied.writeTime).toISOString()}), but the index now has ${this.store.currentWriteTime(key)} (${new Date(this.store.currentWriteTime(key)).toISOString()}) - something wrote it while we were syncing; keeping the newer entry`);
+                                logSyncEvent({ event: "fullSyncCommitSuperseded", store: this.store.folder, source: source.getDebugName(), path: key, copiedWriteTime: copied.writeTime, currentWriteTime: this.store.currentWriteTime(key) });
                             }
                         } else {
                             missingOnSource++;
-                            console.warn(`Full sync could not copy ${key} from ${source.getDebugName()} (store ${this.store.folder}): the source answered but no longer has it (expected ${entry.size} bytes, writeTime ${new Date(entry.writeTime).toISOString()})`);
+                            // The source's listing vouched for the file but its read has nothing - and asking again forever is not an answer. If our own disk holds a copy, IT is the real holder, so the entry repoints there; if neither side has bytes, the entry was simply wrong, and it is purged - forgotten, not deleted, exactly like a read that comes up empty everywhere (a scan re-finds the file if it exists anywhere else).
+                            let local = await this.store.sources[0].source.getInfo(key);
+                            if (local) {
+                                this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "repointed to our local copy", localSize: local.size });
+                            } else if (this.store.currentWriteTime(key) <= entry.writeTime) {
+                                this.store.purgeIndexEntry(key);
+                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "purged - neither the source nor our disk has it, so it does not exist" });
+                            } else {
+                                // A newer write landed while we were checking - the entry is no longer the one we found missing, so it is left alone
+                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "kept - a newer write appeared while checking", currentWriteTime: this.store.currentWriteTime(key) });
+                            }
                         }
                         consecutiveFailures = 0;
                     } catch (e) {
                         // One failing file must not abort the sync: it stays on its source (reads still reach it there), and the next sync pass retries it
                         failed++;
                         consecutiveFailures++;
-                        if (errors.length < SYNC_ERROR_LOG_LIMIT) {
-                            errors.push(`${key}: ${(e as Error).stack ?? e}`);
-                        }
+                        logSyncEvent({ event: "fullSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, size: entry.size, error: String((e as Error).stack ?? e).slice(0, 2000) });
                         if (consecutiveFailures >= SYNC_MAX_CONSECUTIVE_FAILURES) {
                             aborted = true;
                             return;
@@ -602,9 +617,6 @@ export class StoreSync {
                 workers.push(copyWorker());
             }
             await Promise.all(workers);
-            if (failed) {
-                console.error(`Full sync from ${source.getDebugName()} (store ${this.store.folder}): ${failed} of ${pending.length} files failed to copy${aborted && ` before aborting the pass (${SYNC_MAX_CONSECUTIVE_FAILURES} consecutive failures - the source looks down; the next sync pass retries)` || " (they stay on their source, and the next sync pass retries them)"}. First errors: ${errors.join(" | ")}`);
-            }
         } finally {
             clearInterval(progressTimer);
             clearInterval(slowErrorTimer);