sliftutils 1.7.109 → 1.7.111

package/storage/remoteStorage/storageLogs.d.ts

@@ -2,12 +2,13 @@
 /// <reference types="node" />
 import { LogFileInfo } from "../StreamingLogs";
 export declare const LOGS_FOLDER_NAME = "logs";
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Stream-only - one entry per write is exactly what the console does NOT need. */
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Logged DELIBERATELY at two layers: the controller (which knows the account/bucket and the caller) AND BlobStore itself (which knows the folder, and sees the writes that never pass through the controller) - the redundancy is the point, because a write that only one layer saw is exactly the kind of masked issue these logs exist to expose. Stream-only - one entry per write is exactly what the console does NOT need. */
 export declare function logMutation(entry: {
     op: string;
-    account: string;
-    bucketName: string;
+    account?: string;
+    bucketName?: string;
     store?: string;
+    folder?: string;
     path: string;
     toPath?: string;
     size?: number;
@@ -24,6 +25,8 @@ export declare function logSyncEvent(entry: {
 }): void;
 /** One console.error is all an error takes (console.error and console.warn are HOOKED to feed the stream) - this just guarantees the hook is installed first, for very-early callers. */
 export declare function logStorageError(message: string): void;
+/** logStorageError at warn level: guarantees the console.warn hook is installed before warning, so warnings from before the first logged mutation still reach the stream. */
+export declare function logStorageWarn(message: string): void;
 /** The log files this server holds - see StreamingLogs.listFiles. Empty on processes with no storage folder. */
 export declare function listStorageLogFiles(): Promise<LogFileInfo[]>;
 /** One log file's bytes, always LZ4-compressed - see StreamingLogs.readFileCompressed (decode with decodeLogFile). */

package/storage/remoteStorage/storageLogs.ts

@@ -66,8 +66,8 @@ function write(kind: string, entry: { [key: string]: unknown }, alsoConsole: boo
     logs.log({ kind, time: Date.now(), ...baseFields(), ...entry });
 }
 
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Stream-only - one entry per write is exactly what the console does NOT need. */
-export function logMutation(entry: { op: string; account: string; bucketName: string; store?: string; path: string; toPath?: string; size?: number; writeTime?: number; callerId?: string; internal?: boolean }): void {
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Logged DELIBERATELY at two layers: the controller (which knows the account/bucket and the caller) AND BlobStore itself (which knows the folder, and sees the writes that never pass through the controller) - the redundancy is the point, because a write that only one layer saw is exactly the kind of masked issue these logs exist to expose. Stream-only - one entry per write is exactly what the console does NOT need. */
+export function logMutation(entry: { op: string; account?: string; bucketName?: string; store?: string; folder?: string; path: string; toPath?: string; size?: number; writeTime?: number; callerId?: string; internal?: boolean }): void {
     write("mutation", entry, false);
 }
 
@@ -82,6 +82,12 @@ export function logStorageError(message: string): void {
     console.error(message);
 }
 
+/** logStorageError at warn level: guarantees the console.warn hook is installed before warning, so warnings from before the first logged mutation still reach the stream. */
+export function logStorageWarn(message: string): void {
+    getLogs();
+    console.warn(message);
+}
+
 // Everything console.errored or console.warned also lands in the log stream (with a reentrancy guard: a failure INSIDE logging must not log itself forever)
 let writingHooked = false;
 const hookErrorLogging = lazy(() => {

package/storage/remoteStorage/storeSync.d.ts

@@ -44,6 +44,7 @@ export declare class StoreSync {
         since: number;
         route?: [number, number];
     }): Promise<void>;
+    private windowsAllowScanning;
     private startSourceSyncLoops;
     private pullSource;
     private pushSource;

package/storage/remoteStorage/storeSync.ts

@@ -44,6 +44,8 @@ const SYNC_FAILURE_DELAY = 1000 * 15;
 // A full listing that comes back EMPTY - or under half of what we know the source holds - is treated as the other end being briefly broken, not as the truth: it is retried this many times, this far apart, before being believed. Believing a wrongly-shrunken listing purges every index entry the source held, which is how sync progress "goes backwards".
 const SUSPICIOUS_SCAN_RETRIES = 3;
 const SUSPICIOUS_SCAN_RETRY_DELAY = 1000 * 60;
+// A source whose every window ended this long ago (and has none in the future) can never receive writes again - the window gates them, deletions included. Scanning such a source can only re-ingest history it will never be told to delete, which after tombstone expiry IS resurrection - so it is not scanned at all. The grace covers handoff stragglers around a window's end.
+const SCAN_STALE_WINDOW_GRACE = 1000 * 60 * 60;
 
 type SourceState = {
     supportsChangesAfter: boolean;
@@ -258,6 +260,14 @@ export class StoreSync {
             }
             let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: file.path, forceSetImmutable: true, noChecks: true, internal: true });
             if (!copied) {
+                // Undefined is two cases (see copyArchiveFile): our own disk already holding something newer is the boundary scan working as intended (our writes since the boundary outrank the neighbor's), only a genuinely unreadable file is worth a warning
+                let local = await this.store.sources[0].source.getInfo(file.path);
+                if (local && local.writeTime > file.createTime) {
+                    logSyncEvent({ event: "boundaryScanLocalNewer", store: this.store.folder, source: source.getDebugName(), path: file.path, theirWriteTime: new Date(file.createTime).toISOString(), theirSize: file.size, localWriteTime: new Date(local.writeTime).toISOString(), localSize: local.size });
+                    this.store.setIndexEntry(file.path, { writeTime: local.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                    tally.unchanged++;
+                    continue;
+                }
                 console.warn(`Boundary scan could not copy ${file.path} from ${source.getDebugName()} (store ${this.store.folder}): its change feed listed it (${file.size} bytes, writeTime ${new Date(file.createTime).toISOString()}) but the read found nothing`);
                 continue;
             }
@@ -279,6 +289,12 @@ export class StoreSync {
 
     // ── per-source loops ──
 
+    // Slot 0 is exempt: our own disk answers for every window this store EVER held, and scanning it is what keeps the index honest about them. Read live each call - updateSources moves the windows on running slots, so a source goes stale (or comes back) while its loops run.
+    private windowsAllowScanning(sourceIndex: number): boolean {
+        if (sourceIndex === 0) return true;
+        return this.store.sources[sourceIndex].validWindows.some(w => w[1] > Date.now() - SCAN_STALE_WINDOW_GRACE);
+    }
+
     private async startSourceSyncLoops(sourceIndex: number): Promise<void> {
         await this.store.registerSlot(sourceIndex);
         let sourceObj = this.store.sources[sourceIndex];
@@ -294,7 +310,23 @@ export class StoreSync {
             state.initialScan.resolve(undefined);
             return;
         }
-        while (!this.store.stopped.stop && !state.stopped.stop) {
+        // Checked per tick, never decided once: the windows move on running slots (see updateSources), so a source goes stale mid-life and can come back if a config extends them. Logged only on each transition into staleness.
+        let loggedStale = false;
+        let skipStale = () => {
+            if (this.windowsAllowScanning(sourceIndex)) {
+                loggedStale = false;
+                return false;
+            }
+            state.scanComplete = true;
+            state.initialScan.resolve(undefined);
+            if (!loggedStale) {
+                loggedStale = true;
+                logSyncEvent({ event: "scanSkippedStaleWindows", store: this.store.folder, source: source.getDebugName(), validWindows: this.store.sources[sourceIndex].validWindows.map(w => [new Date(w[0]).toISOString(), new Date(w[1]).toISOString()]), graceMs: SCAN_STALE_WINDOW_GRACE });
+            }
+            return true;
+        };
+        // An already-stale source skips getConfig entirely: its endpoint is often long gone, and retrying that every 30s forever is noise about a source we would not scan anyway (if its windows are later extended, the full-round poll below picks it up - just without change polling until a restart)
+        while (!skipStale() && !this.store.stopped.stop && !state.stopped.stop) {
             try {
                 let config = await source.getConfig();
                 state.supportsChangesAfter = !!config.supportsChangesAfter;
@@ -310,6 +342,7 @@ export class StoreSync {
         // Both loops below run one at a time: a change-poll tick landing mid full round would otherwise start a second copy pass over the same pending list, downloading everything twice
         let serial = runInSerial(async (fnc: () => Promise<void>) => await fnc());
         await runInfinitePollCallAtStart(pollInterval, () => serial(async () => {
+            if (skipStale()) return;
             while (!this.store.stopped.stop && !state.stopped.stop) {
                 try {
                     await this.syncSource(sourceIndex)("push");
@@ -326,6 +359,7 @@ export class StoreSync {
         }), state.stopped);
         if (state.supportsChangesAfter) {
             runInfinitePoll(CHANGES_POLL_INTERVAL, () => serial(async () => {
+                if (skipStale()) return;
                 // A scan that has not succeeded yet is retried HERE, before anything else - polling changes and copying against a never-scanned source would run on arbitrarily stale information
                 if (!state.scanSucceeded) {
                     await this.syncSource(sourceIndex)("push");
@@ -461,7 +495,17 @@ export class StoreSync {
                 let holder = await this.store.getEntryHolder(entry);
                 if (!holder) continue;
                 let copied = await copyArchiveFile({ from: holder, to: source, path: key, forceSetImmutable: true, noChecks: true, internal: true });
-                if (!copied) continue;
+                if (!copied) {
+                    // Undefined is two cases (see copyArchiveFile). The source having a NEWER file than our index (the listing we pushed from was stale) is adopted exactly the way the pull direction adopts a listing entry - and pushing our stale copy stops. Otherwise the HOLDER could not produce the file, which the next round's pull re-resolves; either way, silence here was the bug.
+                    let theirs = await source.getInfo(key);
+                    if (theirs && theirs.writeTime > writeTime) {
+                        this.store.setIndexEntry(key, { writeTime: theirs.writeTime, size: theirs.size, sourcesListIndex: targetSourcesListIndex });
+                        logSyncEvent({ event: "pushFoundNewerOnSource", store: this.store.folder, source: source.getDebugName(), path: key, ourWriteTime: new Date(writeTime).toISOString(), theirWriteTime: new Date(theirs.writeTime).toISOString(), ourSize: entry.size, theirSize: theirs.size });
+                        continue;
+                    }
+                    logSyncEvent({ event: "pushCopyUnavailable", store: this.store.folder, source: source.getDebugName(), path: key, holder: holder.getDebugName(), expectedSize: entry.size, expectedWriteTime: new Date(writeTime).toISOString(), sourceHas: theirs && `${theirs.size} bytes at ${new Date(theirs.writeTime).toISOString()}` || "nothing" });
+                    continue;
+                }
                 this.store.noteSyncTransfer("sync set", key, copied.size);
                 pushed++;
                 consecutiveFailures = 0;
@@ -550,11 +594,11 @@ export class StoreSync {
             totalBytes,
         };
         this.activities.add(activity);
-        logSyncEvent({ event: "fullSyncStart", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, files: pending.length, bytes: totalBytes });
+        logSyncEvent({ event: "deltaSyncStart", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, files: pending.length, bytes: totalBytes });
         let progressLogged = false;
         let logProgress = () => {
             progressLogged = true;
-            console.log(magenta(`Full sync from ${source.getDebugName()} (store ${this.store.folder}): ${activity.doneFiles}/${pending.length} files (${((activity.doneFiles || 0) / pending.length * 100).toFixed(1)}%), ${formatNumber(activity.doneBytes || 0)}B/${formatNumber(totalBytes)}B (${(totalBytes && (activity.doneBytes || 0) / totalBytes * 100 || 100).toFixed(1)}%)`));
+            console.log(magenta(`Delta sync from ${source.getDebugName()} (store ${this.store.folder}): ${activity.doneFiles}/${pending.length} files (${((activity.doneFiles || 0) / pending.length * 100).toFixed(1)}%), ${formatNumber(activity.doneBytes || 0)}B/${formatNumber(totalBytes)}B (${(totalBytes && (activity.doneBytes || 0) / totalBytes * 100 || 100).toFixed(1)}%)`));
         };
         let progressTimer = setInterval(logProgress, SYNC_PROGRESS_LOG_INTERVAL);
         (progressTimer as { unref?: () => void }).unref?.();
@@ -569,7 +613,7 @@ export class StoreSync {
                 let remainingMs = remainingBytes / bytesPerSecond * 1000;
                 etaText = `${formatTime(remainingMs)} remaining, completing around ${new Date(Date.now() + remainingMs).toISOString()}`;
             }
-            console.warn(`Full sync from ${source.getDebugName()} (store ${this.store.folder}) has been running for ${formatTime(elapsed)}: ${doneFiles}/${pending.length} files (${(doneFiles / pending.length * 100).toFixed(1)}%), ${formatNumber(doneBytes)}B/${formatNumber(totalBytes)}B (${(totalBytes && doneBytes / totalBytes * 100 || 100).toFixed(1)}%), ${formatNumber(bytesPerSecond)}B/s. Estimated ${etaText}.`);
+            console.warn(`Delta sync from ${source.getDebugName()} (store ${this.store.folder}) has been running for ${formatTime(elapsed)}: ${doneFiles}/${pending.length} files (${(doneFiles / pending.length * 100).toFixed(1)}%), ${formatNumber(doneBytes)}B/${formatNumber(totalBytes)}B (${(totalBytes && doneBytes / totalBytes * 100 || 100).toFixed(1)}%), ${formatNumber(bytesPerSecond)}B/s. Estimated ${etaText}.`);
         }, FULL_SYNC_SLOW_ERROR_INTERVAL);
         (slowErrorTimer as { unref?: () => void }).unref?.();
         let failed = 0;
@@ -582,8 +626,9 @@ export class StoreSync {
             let base = { event, store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, ...extra };
             let local = await this.store.sources[0].source.getInfo(key);
             if (local) {
-                this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
-                logSyncEvent({ ...base, resolution: "repointed to our local copy", localSize: local.size });
+                // The local copy can be NEWER than the entry (which is exactly why copyArchiveFile refuses to overwrite it - see its destination check), so the repoint keeps the newer of the two times rather than rolling the index back
+                this.store.setIndexEntry(key, { writeTime: Math.max(entry.writeTime, local.writeTime), size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                logSyncEvent({ ...base, resolution: "repointed to our local copy", localSize: local.size, localWriteTime: new Date(local.writeTime).toISOString() });
                 return;
             }
             if (this.store.currentWriteTime(key) <= entry.writeTime) {
@@ -610,11 +655,11 @@ export class StoreSync {
                             this.store.noteSyncTransfer("sync get", key, copied.size);
                             // The copy carries the source's write time, and the index commits it under the normal ordering rule (>= the current time wins) - it refusing means a NEWER write landed while we copied, which must be said, not swallowed
                             if (!this.store.setIndexEntry(key, { writeTime: copied.writeTime, size: copied.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) })) {
-                                logSyncEvent({ event: "fullSyncCommitSuperseded", store: this.store.folder, source: source.getDebugName(), path: key, copiedWriteTime: copied.writeTime, currentWriteTime: this.store.currentWriteTime(key) });
+                                logSyncEvent({ event: "deltaSyncCommitSuperseded", store: this.store.folder, source: source.getDebugName(), path: key, copiedWriteTime: copied.writeTime, currentWriteTime: this.store.currentWriteTime(key) });
                             }
                         } else {
                             missingOnSource++;
-                            await resolveUnavailable("fullSyncMissingOnSource", key, entry, {});
+                            await resolveUnavailable("deltaSyncMissingOnSource", key, entry, {});
                         }
                         consecutiveFailures = 0;
                     } catch (e) {
@@ -622,9 +667,9 @@ export class StoreSync {
                         failed++;
                         consecutiveFailures++;
                         try {
-                            await resolveUnavailable("fullSyncCopyFailed", key, entry, { error: String((e as Error).stack ?? e).slice(0, 2000) });
+                            await resolveUnavailable("deltaSyncCopyFailed", key, entry, { error: String((e as Error).stack ?? e).slice(0, 2000) });
                         } catch (resolveError) {
-                            logSyncEvent({ event: "fullSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, error: String((e as Error).stack ?? e).slice(0, 2000), resolveError: String((resolveError as Error).stack ?? resolveError).slice(0, 2000) });
+                            logSyncEvent({ event: "deltaSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, error: String((e as Error).stack ?? e).slice(0, 2000), resolveError: String((resolveError as Error).stack ?? resolveError).slice(0, 2000) });
                         }
                         if (consecutiveFailures >= SYNC_MAX_CONSECUTIVE_FAILURES) {
                             aborted = true;
@@ -650,7 +695,7 @@ export class StoreSync {
             if (progressLogged) {
                 logProgress();
             }
-            logSyncEvent({ event: "fullSyncFinish", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, durationMs: Date.now() - activity.startTime, totalFiles: pending.length, totalBytes, processedFiles: activity.doneFiles, copiedFiles, copiedBytes, missingOnSource, failed, aborted });
+            logSyncEvent({ event: "deltaSyncFinish", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, durationMs: Date.now() - activity.startTime, totalFiles: pending.length, totalBytes, processedFiles: activity.doneFiles, copiedFiles, copiedBytes, missingOnSource, failed, aborted });
         }
     }));