sliftutils 1.7.108 → 1.7.110

package/storage/remoteStorage/storageLogs.d.ts

@@ -2,12 +2,13 @@
 /// <reference types="node" />
 import { LogFileInfo } from "../StreamingLogs";
 export declare const LOGS_FOLDER_NAME = "logs";
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Stream-only - one entry per write is exactly what the console does NOT need. */
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Logged DELIBERATELY at two layers: the controller (which knows the account/bucket and the caller) AND BlobStore itself (which knows the folder, and sees the writes that never pass through the controller) - the redundancy is the point, because a write that only one layer saw is exactly the kind of masked issue these logs exist to expose. Stream-only - one entry per write is exactly what the console does NOT need. */
 export declare function logMutation(entry: {
     op: string;
-    account: string;
-    bucketName: string;
+    account?: string;
+    bucketName?: string;
     store?: string;
+    folder?: string;
     path: string;
     toPath?: string;
     size?: number;
@@ -24,6 +25,8 @@ export declare function logSyncEvent(entry: {
 }): void;
 /** One console.error is all an error takes (console.error and console.warn are HOOKED to feed the stream) - this just guarantees the hook is installed first, for very-early callers. */
 export declare function logStorageError(message: string): void;
+/** logStorageError at warn level: guarantees the console.warn hook is installed before warning, so warnings from before the first logged mutation still reach the stream. */
+export declare function logStorageWarn(message: string): void;
 /** The log files this server holds - see StreamingLogs.listFiles. Empty on processes with no storage folder. */
 export declare function listStorageLogFiles(): Promise<LogFileInfo[]>;
 /** One log file's bytes, always LZ4-compressed - see StreamingLogs.readFileCompressed (decode with decodeLogFile). */

package/storage/remoteStorage/storageLogs.ts

@@ -66,8 +66,8 @@ function write(kind: string, entry: { [key: string]: unknown }, alsoConsole: boo
     logs.log({ kind, time: Date.now(), ...baseFields(), ...entry });
 }
 
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Stream-only - one entry per write is exactly what the console does NOT need. */
-export function logMutation(entry: { op: string; account: string; bucketName: string; store?: string; path: string; toPath?: string; size?: number; writeTime?: number; callerId?: string; internal?: boolean }): void {
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Logged DELIBERATELY at two layers: the controller (which knows the account/bucket and the caller) AND BlobStore itself (which knows the folder, and sees the writes that never pass through the controller) - the redundancy is the point, because a write that only one layer saw is exactly the kind of masked issue these logs exist to expose. Stream-only - one entry per write is exactly what the console does NOT need. */
+export function logMutation(entry: { op: string; account?: string; bucketName?: string; store?: string; folder?: string; path: string; toPath?: string; size?: number; writeTime?: number; callerId?: string; internal?: boolean }): void {
     write("mutation", entry, false);
 }
 
@@ -82,6 +82,12 @@ export function logStorageError(message: string): void {
     console.error(message);
 }
 
+/** logStorageError at warn level: guarantees the console.warn hook is installed before warning, so warnings from before the first logged mutation still reach the stream. */
+export function logStorageWarn(message: string): void {
+    getLogs();
+    console.warn(message);
+}
+
 // Everything console.errored or console.warned also lands in the log stream (with a reentrancy guard: a failure INSIDE logging must not log itself forever)
 let writingHooked = false;
 const hookErrorLogging = lazy(() => {

package/storage/remoteStorage/storeSync.d.ts

@@ -37,7 +37,7 @@ export declare class StoreSync {
     waitForRequiredScans(): Promise<void>;
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     rescanBase(): Promise<void>;
-    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. Only one round per SOURCE runs at a time (cache keys the serializer by source index). */
     private syncSource;
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
     boundaryScanRemote(source: IArchives, config: {

package/storage/remoteStorage/storeSync.ts

@@ -1,4 +1,5 @@
-import { runInfinitePoll, delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { runInfinitePoll, delay, runInfinitePollCallAtStart, runInSerial } from "socket-function/src/batching";
+import { cache } from "socket-function/src/caching";
 import { timeInMinute, sort, promiseObj } from "socket-function/src/misc";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import {
@@ -219,11 +220,11 @@ export class StoreSync {
 
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     public async rescanBase(): Promise<void> {
-        await this.syncSource(0);
+        await this.syncSource(0)();
     }
 
-    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
-    private async syncSource(sourceIndex: number, push?: "push"): Promise<void> {
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. Only one round per SOURCE runs at a time (cache keys the serializer by source index). */
+    private syncSource = cache((sourceIndex: number) => runInSerial(async (push?: "push"): Promise<void> => {
         let listing = await this.pullSource(sourceIndex);
         let state = this.states[sourceIndex];
         state.scanComplete = true;
@@ -231,7 +232,7 @@ export class StoreSync {
         if (push && !this.store.stopped.stop && !state.stopped.stop) {
             await this.pushSource(sourceIndex, listing);
         }
-    }
+    }));
 
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
     public async boundaryScanRemote(source: IArchives, config: { since: number; route?: [number, number] }): Promise<void> {
@@ -257,6 +258,14 @@ export class StoreSync {
             }
             let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: file.path, forceSetImmutable: true, noChecks: true, internal: true });
             if (!copied) {
+                // Undefined is two cases (see copyArchiveFile): our own disk already holding something newer is the boundary scan working as intended (our writes since the boundary outrank the neighbor's), only a genuinely unreadable file is worth a warning
+                let local = await this.store.sources[0].source.getInfo(file.path);
+                if (local && local.writeTime > file.createTime) {
+                    logSyncEvent({ event: "boundaryScanLocalNewer", store: this.store.folder, source: source.getDebugName(), path: file.path, theirWriteTime: new Date(file.createTime).toISOString(), theirSize: file.size, localWriteTime: new Date(local.writeTime).toISOString(), localSize: local.size });
+                    this.store.setIndexEntry(file.path, { writeTime: local.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                    tally.unchanged++;
+                    continue;
+                }
                 console.warn(`Boundary scan could not copy ${file.path} from ${source.getDebugName()} (store ${this.store.folder}): its change feed listed it (${file.size} bytes, writeTime ${new Date(file.createTime).toISOString()}) but the read found nothing`);
                 continue;
             }
@@ -304,12 +313,15 @@ export class StoreSync {
             }
         }
         if (this.store.stopped.stop || state.stopped.stop) return;
-        let pollInterval = sourceObj.sourceConfig?.type === "remote" ? FULL_RESCAN_INTERVAL : FULL_RESCAN_NON_REMOTE_INTERVAL;
-        await runInfinitePollCallAtStart(pollInterval, async () => {
+        // Hourly for our own disk (no sourceConfig - a cheap local walk, and how a sibling process's writes into the shared folder are found) and for remote peers (index-backed, cheap listings); the slow interval is only for sources where a full listing is genuinely expensive (backblaze)
+        let pollInterval = (!sourceObj.sourceConfig || sourceObj.sourceConfig.type === "remote") && FULL_RESCAN_INTERVAL || FULL_RESCAN_NON_REMOTE_INTERVAL;
+        // Both loops below run one at a time: a change-poll tick landing mid full round would otherwise start a second copy pass over the same pending list, downloading everything twice
+        let serial = runInSerial(async (fnc: () => Promise<void>) => await fnc());
+        await runInfinitePollCallAtStart(pollInterval, () => serial(async () => {
             while (!this.store.stopped.stop && !state.stopped.stop) {
                 try {
-                    await this.syncSource(sourceIndex, "push");
-                    if (!noFullSync()) await this.copySourceFiles(sourceIndex);
+                    await this.syncSource(sourceIndex)("push");
+                    if (!noFullSync()) await this.copySourceFiles(sourceIndex)();
                 } catch (e) {
                     logSyncEvent({ event: "scanFailed", store: this.store.folder, source: source.getDebugName(), retryInMs: SCAN_RETRY_DELAY, error: String((e as Error).stack ?? e).slice(0, 2000) });
                     await delay(SCAN_RETRY_DELAY);
@@ -319,16 +331,16 @@ export class StoreSync {
                 state.initialScan.resolve(undefined);
                 break;
             }
-        }, state.stopped);
+        }), state.stopped);
         if (state.supportsChangesAfter) {
-            runInfinitePoll(CHANGES_POLL_INTERVAL, async () => {
+            runInfinitePoll(CHANGES_POLL_INTERVAL, () => serial(async () => {
                 // A scan that has not succeeded yet is retried HERE, before anything else - polling changes and copying against a never-scanned source would run on arbitrarily stale information
                 if (!state.scanSucceeded) {
-                    await this.syncSource(sourceIndex, "push");
+                    await this.syncSource(sourceIndex)("push");
                 }
                 await this.pollChanges(sourceIndex);
-                if (!noFullSync()) await this.copySourceFiles(sourceIndex);
-            }, state.stopped);
+                if (!noFullSync()) await this.copySourceFiles(sourceIndex)();
+            }), state.stopped);
         }
     }
 
@@ -457,7 +469,17 @@ export class StoreSync {
                 let holder = await this.store.getEntryHolder(entry);
                 if (!holder) continue;
                 let copied = await copyArchiveFile({ from: holder, to: source, path: key, forceSetImmutable: true, noChecks: true, internal: true });
-                if (!copied) continue;
+                if (!copied) {
+                    // Undefined is two cases (see copyArchiveFile). The source having a NEWER file than our index (the listing we pushed from was stale) is adopted exactly the way the pull direction adopts a listing entry - and pushing our stale copy stops. Otherwise the HOLDER could not produce the file, which the next round's pull re-resolves; either way, silence here was the bug.
+                    let theirs = await source.getInfo(key);
+                    if (theirs && theirs.writeTime > writeTime) {
+                        this.store.setIndexEntry(key, { writeTime: theirs.writeTime, size: theirs.size, sourcesListIndex: targetSourcesListIndex });
+                        logSyncEvent({ event: "pushFoundNewerOnSource", store: this.store.folder, source: source.getDebugName(), path: key, ourWriteTime: new Date(writeTime).toISOString(), theirWriteTime: new Date(theirs.writeTime).toISOString(), ourSize: entry.size, theirSize: theirs.size });
+                        continue;
+                    }
+                    logSyncEvent({ event: "pushCopyUnavailable", store: this.store.folder, source: source.getDebugName(), path: key, holder: holder.getDebugName(), expectedSize: entry.size, expectedWriteTime: new Date(writeTime).toISOString(), sourceHas: theirs && `${theirs.size} bytes at ${new Date(theirs.writeTime).toISOString()}` || "nothing" });
+                    continue;
+                }
                 this.store.noteSyncTransfer("sync set", key, copied.size);
                 pushed++;
                 consecutiveFailures = 0;
@@ -522,8 +544,8 @@ export class StoreSync {
         state.changesAfterTime = pollStart - CHANGES_POLL_OVERLAP;
     }
 
-    // Downloads the files a source currently holds onto our own base source (the local disk), preserving their modified times — so a newer local write always wins. Skipped for noFullSync sources (fronting a large database without copying it); reads still down-cache lazily.
-    private async copySourceFiles(sourceIndex: number): Promise<void> {
+    // Downloads the files a source currently holds onto our own base source (the local disk), preserving their modified times — so a newer local write always wins. Skipped for noFullSync sources (fronting a large database without copying it); reads still down-cache lazily. Only one pass per SOURCE runs at a time (cache keys the serializer by source index) - two passes over the same pending list would download everything twice.
+    private copySourceFiles = cache((sourceIndex: number) => runInSerial(async (): Promise<void> => {
         if (sourceIndex === 0) return;
         let { source } = this.store.sources[sourceIndex];
         let state = this.states[sourceIndex];
@@ -546,11 +568,11 @@ export class StoreSync {
             totalBytes,
         };
         this.activities.add(activity);
-        logSyncEvent({ event: "fullSyncStart", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, files: pending.length, bytes: totalBytes });
+        logSyncEvent({ event: "deltaSyncStart", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, files: pending.length, bytes: totalBytes });
         let progressLogged = false;
         let logProgress = () => {
             progressLogged = true;
-            console.log(magenta(`Full sync from ${source.getDebugName()} (store ${this.store.folder}): ${activity.doneFiles}/${pending.length} files (${((activity.doneFiles || 0) / pending.length * 100).toFixed(1)}%), ${formatNumber(activity.doneBytes || 0)}B/${formatNumber(totalBytes)}B (${(totalBytes && (activity.doneBytes || 0) / totalBytes * 100 || 100).toFixed(1)}%)`));
+            console.log(magenta(`Delta sync from ${source.getDebugName()} (store ${this.store.folder}): ${activity.doneFiles}/${pending.length} files (${((activity.doneFiles || 0) / pending.length * 100).toFixed(1)}%), ${formatNumber(activity.doneBytes || 0)}B/${formatNumber(totalBytes)}B (${(totalBytes && (activity.doneBytes || 0) / totalBytes * 100 || 100).toFixed(1)}%)`));
         };
         let progressTimer = setInterval(logProgress, SYNC_PROGRESS_LOG_INTERVAL);
         (progressTimer as { unref?: () => void }).unref?.();
@@ -565,7 +587,7 @@ export class StoreSync {
                 let remainingMs = remainingBytes / bytesPerSecond * 1000;
                 etaText = `${formatTime(remainingMs)} remaining, completing around ${new Date(Date.now() + remainingMs).toISOString()}`;
             }
-            console.warn(`Full sync from ${source.getDebugName()} (store ${this.store.folder}) has been running for ${formatTime(elapsed)}: ${doneFiles}/${pending.length} files (${(doneFiles / pending.length * 100).toFixed(1)}%), ${formatNumber(doneBytes)}B/${formatNumber(totalBytes)}B (${(totalBytes && doneBytes / totalBytes * 100 || 100).toFixed(1)}%), ${formatNumber(bytesPerSecond)}B/s. Estimated ${etaText}.`);
+            console.warn(`Delta sync from ${source.getDebugName()} (store ${this.store.folder}) has been running for ${formatTime(elapsed)}: ${doneFiles}/${pending.length} files (${(doneFiles / pending.length * 100).toFixed(1)}%), ${formatNumber(doneBytes)}B/${formatNumber(totalBytes)}B (${(totalBytes && doneBytes / totalBytes * 100 || 100).toFixed(1)}%), ${formatNumber(bytesPerSecond)}B/s. Estimated ${etaText}.`);
         }, FULL_SYNC_SLOW_ERROR_INTERVAL);
         (slowErrorTimer as { unref?: () => void }).unref?.();
         let failed = 0;
@@ -578,8 +600,9 @@ export class StoreSync {
             let base = { event, store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, ...extra };
             let local = await this.store.sources[0].source.getInfo(key);
             if (local) {
-                this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
-                logSyncEvent({ ...base, resolution: "repointed to our local copy", localSize: local.size });
+                // The local copy can be NEWER than the entry (which is exactly why copyArchiveFile refuses to overwrite it - see its destination check), so the repoint keeps the newer of the two times rather than rolling the index back
+                this.store.setIndexEntry(key, { writeTime: Math.max(entry.writeTime, local.writeTime), size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                logSyncEvent({ ...base, resolution: "repointed to our local copy", localSize: local.size, localWriteTime: new Date(local.writeTime).toISOString() });
                 return;
             }
             if (this.store.currentWriteTime(key) <= entry.writeTime) {
@@ -606,11 +629,11 @@ export class StoreSync {
                             this.store.noteSyncTransfer("sync get", key, copied.size);
                             // The copy carries the source's write time, and the index commits it under the normal ordering rule (>= the current time wins) - it refusing means a NEWER write landed while we copied, which must be said, not swallowed
                             if (!this.store.setIndexEntry(key, { writeTime: copied.writeTime, size: copied.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) })) {
-                                logSyncEvent({ event: "fullSyncCommitSuperseded", store: this.store.folder, source: source.getDebugName(), path: key, copiedWriteTime: copied.writeTime, currentWriteTime: this.store.currentWriteTime(key) });
+                                logSyncEvent({ event: "deltaSyncCommitSuperseded", store: this.store.folder, source: source.getDebugName(), path: key, copiedWriteTime: copied.writeTime, currentWriteTime: this.store.currentWriteTime(key) });
                             }
                         } else {
                             missingOnSource++;
-                            await resolveUnavailable("fullSyncMissingOnSource", key, entry, {});
+                            await resolveUnavailable("deltaSyncMissingOnSource", key, entry, {});
                         }
                         consecutiveFailures = 0;
                     } catch (e) {
@@ -618,9 +641,9 @@ export class StoreSync {
                         failed++;
                         consecutiveFailures++;
                         try {
-                            await resolveUnavailable("fullSyncCopyFailed", key, entry, { error: String((e as Error).stack ?? e).slice(0, 2000) });
+                            await resolveUnavailable("deltaSyncCopyFailed", key, entry, { error: String((e as Error).stack ?? e).slice(0, 2000) });
                         } catch (resolveError) {
-                            logSyncEvent({ event: "fullSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, error: String((e as Error).stack ?? e).slice(0, 2000), resolveError: String((resolveError as Error).stack ?? resolveError).slice(0, 2000) });
+                            logSyncEvent({ event: "deltaSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, error: String((e as Error).stack ?? e).slice(0, 2000), resolveError: String((resolveError as Error).stack ?? resolveError).slice(0, 2000) });
                         }
                         if (consecutiveFailures >= SYNC_MAX_CONSECUTIVE_FAILURES) {
                             aborted = true;
@@ -646,9 +669,9 @@ export class StoreSync {
             if (progressLogged) {
                 logProgress();
             }
-            logSyncEvent({ event: "fullSyncFinish", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, durationMs: Date.now() - activity.startTime, totalFiles: pending.length, totalBytes, processedFiles: activity.doneFiles, copiedFiles, copiedBytes, missingOnSource, failed, aborted });
+            logSyncEvent({ event: "deltaSyncFinish", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, durationMs: Date.now() - activity.startTime, totalFiles: pending.length, totalBytes, processedFiles: activity.doneFiles, copiedFiles, copiedBytes, missingOnSource, failed, aborted });
         }
-    }
+    }));
 
     // ── maintenance ──