sliftutils 1.7.105 → 1.7.107

package/storage/remoteStorage/dist/validation.ts.cache

@@ -1,4 +1,4 @@
-"use strict"; var console = (function shimConsole() {             let baseConsole = globalThis.console;             if (typeof window !== "undefined")                 return baseConsole;             let name = __filename.replaceAll("\\", "/").split("/").pop();             let dirName = __dirname.replaceAll("\\", "/").split("/").pop();             let context = {                 __FILE__: __filename,                 __DIR__: __dirname,                 __NAME__: name,                 __DIRNAME__: dirName             };             let newConsole = {                 ...baseConsole             };             for (let shim of ["log", "info", "warn", "error"]) {                 newConsole[shim] = (...args) => {                     args.push(context);                     baseConsole[shim](...args);                 };             }             newConsole.__context = context;             return newConsole;         })(); "use strict";
+"use strict";
 Object.defineProperty(exports, "__esModule", { value: true , configurable: true});
 //exports.assertValidArgs = exports.assertValidPath = exports.assertValidSourceName = exports.assertValidName = void 0;
 function assertValidName(value, kind) {
@@ -47,4 +47,4 @@ function assertValidArgs(target, key, descriptor) {
 }
 exports.assertValidArgs = assertValidArgs;
 //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidmFsaWRhdGlvbi5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbInZhbGlkYXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBQUEsU0FBZ0IsZUFBZSxDQUFDLEtBQWEsRUFBRSxJQUFZO0lBQ3ZELElBQUksQ0FBQyxlQUFlLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDL0IsTUFBTSxJQUFJLEtBQUssQ0FBQyxXQUFXLElBQUksSUFBSSxJQUFJLENBQUMsU0FBUyxDQUFDLEtBQUssQ0FBQywrREFBK0QsQ0FBQyxDQUFDO0lBQzdILENBQUM7QUFDTCxDQUFDO0FBSkQsMENBSUM7QUFFRCx1VEFBdVQ7QUFDdlQsU0FBZ0IscUJBQXFCLENBQUMsS0FBYTtJQUMvQyxJQUFJLENBQUMsZ0JBQWdCLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDaEMsTUFBTSxJQUFJLEtBQUssQ0FBQyx1QkFBdUIsSUFBSSxDQUFDLFNBQVMsQ0FBQyxLQUFLLENBQUMsbUVBQW1FLENBQUMsQ0FBQztJQUNySSxDQUFDO0lBQ0QsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDdEIsTUFBTSxJQUFJLEtBQUssQ0FBQyx1QkFBdUIsSUFBSSxDQUFDLFNBQVMsQ0FBQyxLQUFLLENBQUMsb0VBQW9FLENBQUMsQ0FBQztJQUN0SSxDQUFDO0FBQ0wsQ0FBQztBQVBELHNEQU9DO0FBRUQsU0FBZ0IsZUFBZSxDQUFDLElBQVk7SUFDeEMsSUFBSSxNQUFNLENBQUMsSUFBSSxDQUFDLElBQUksRUFBRSxNQUFNLENBQUMsQ0FBQyxNQUFNLEdBQUcsSUFBSSxFQUFFLENBQUM7UUFDMUMsTUFBTSxJQUFJLEtBQUssQ0FBQyxrQkFBa0IsSUFBSSxDQUFDLE1BQU0sNkJBQTZCLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUNwRyxDQUFDO0lBQ0QsSUFBSSxDQUFDLElBQUksSUFBSSxJQUFJLENBQUMsVUFBVSxDQUFDLEdBQUcsQ0FBQyxJQUFJLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLElBQUksSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsSUFBSSxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxJQUFJLElBQUksQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQztRQUM3SCxNQUFNLElBQUksS0FBSyxDQUFDLGdCQUFnQixJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxDQUFDLDhGQUE4RixDQUFDLENBQUM7SUFDdEssQ0FBQztJQUNELElBQUksSUFBSSxDQUFDLEtBQUssQ0FBQyxHQUFHLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxJQUFJLEtBQUssR0FBRyxJQUFJLElBQUksS0FBSyxJQUFJLENBQUMsRUFBRSxDQUFDO1FBQzlELE1BQU0sSUFBSSxLQUFLLENBQUMsZ0JBQWdCLElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDLEVBQUUsR0FBRyxDQUFDLENBQUMseUNBQXlDLENBQUMsQ0FBQztJQUNqSCxDQUFDO0FBQ0wsQ0FBQztBQVZELDBDQVVDO0FBRUQsNFdBQTRXO0FBQzVXLFNBQWdCLGVBQWUsQ0FBQyxNQUFlLEVBQUUsR0FBVyxFQUFFLFVBQThCO0lBQ3hGLElBQUksUUFBUSxHQUFHLFVBQVUsQ0FBQyxLQUF3QyxDQUFDO0lBQ25FLFVBQVUsQ0FBQyxLQUFLLEdBQUcsVUFBVSxHQUFHLElBQWU7UUFDM0MsSUFBSSxNQUFNLEdBQUcsSUFBSSxDQUFDLENBQUMsQ0FBeUUsQ0FBQztRQUM3RixJQUFJLE1BQU0sSUFBSSxPQUFPLE1BQU0sS0FBSyxRQUFRLEVBQUUsQ0FBQztZQUN2QyxJQUFJLFNBQVMsSUFBSSxNQUFNO2dCQUFFLGVBQWUsQ0FBQyxNQUFNLENBQUMsT0FBaUIsRUFBRSxTQUFTLENBQUMsQ0FBQztZQUM5RSxJQUFJLFlBQVksSUFBSSxNQUFNO2dCQUFFLGVBQWUsQ0FBQyxNQUFNLENBQUMsVUFBb0IsRUFBRSxhQUFhLENBQUMsQ0FBQztZQUN4RixJQUFJLE1BQU0sSUFBSSxNQUFNO2dCQUFFLGVBQWUsQ0FBQyxNQUFNLENBQUMsSUFBYyxDQUFDLENBQUM7UUFDakUsQ0FBQztRQUNELE9BQU8sUUFBUSxDQUFDLEtBQUssQ0FBQyxJQUFJLEVBQUUsSUFBSSxDQUFDLENBQUM7SUFDdEMsQ0FBQyxDQUFDO0FBQ04sQ0FBQztBQVhELDBDQVdDIiwic291cmNlc0NvbnRlbnQiOlsiZXhwb3J0IGZ1bmN0aW9uIGFzc2VydFZhbGlkTmFtZSh2YWx1ZTogc3RyaW5nLCBraW5kOiBzdHJpbmcpOiB2b2lkIHtcbiAgICBpZiAoIS9eW1xcdy1dezEsNjR9JC8udGVzdCh2YWx1ZSkpIHtcbiAgICAgICAgdGhyb3cgbmV3IEVycm9yKGBJbnZhbGlkICR7a2luZH0gJHtKU09OLnN0cmluZ2lmeSh2YWx1ZSl9LCBleHBlY3RlZCAxLTY0IGNoYXJhY3RlcnMgb2YgbGV0dGVycy9udW1iZXJzL3VuZGVyc2NvcmUvZGFzaGApO1xuICAgIH1cbn1cblxuLyoqIEEgc3RvcmUncyBuYW1lIChzZWUgQ29tbW9uQ29uZmlnLm5hbWUpLCB3aGljaCBhbHNvIGFsbG93cyBkb3RzIC0gYSBuYW1lIGlzIG9mdGVuIGEgaG9zdCBvciBhIHZlcnNpb24sIGFuZCBib3RoIHJlYWQgd3Jvbmcgd2l0aG91dCB0aGVtLiBJdCBpcyBvbmUgcGF0aCBzZWdtZW50IG9mIHRoZSBzdG9yZSdzIGZvbGRlciwgc28gdGhlIHR3byBuYW1lcyB0aGF0IHdvdWxkIG1lYW4gYSBkaWZmZXJlbnQgZm9sZGVyIGVudGlyZWx5IGFyZSByZWplY3RlZDogZXZlcnl0aGluZyBlbHNlIGNvbnRhaW5pbmcgZG90cyBpcyBqdXN0IGEgbmFtZS4gKi9cbmV4cG9ydCBmdW5jdGlvbiBhc3NlcnRWYWxpZFNvdXJjZU5hbWUodmFsdWU6IHN0cmluZyk6IHZvaWQge1xuICAgIGlmICghL15bXFx3Li1dezEsNjR9JC8udGVzdCh2YWx1ZSkpIHtcbiAgICAgICAgdGhyb3cgbmV3IEVycm9yKGBJbnZhbGlkIHNvdXJjZSBuYW1lICR7SlNPTi5zdHJpbmdpZnkodmFsdWUpfSwgZXhwZWN0ZWQgMS02NCBjaGFyYWN0ZXJzIG9mIGxldHRlcnMvbnVtYmVycy91bmRlcnNjb3JlL2Rhc2gvZG90YCk7XG4gICAgfVxuICAgIGlmICgvXlxcLiskLy50ZXN0KHZhbHVlKSkge1xuICAgICAgICB0aHJvdyBuZXcgRXJyb3IoYEludmFsaWQgc291cmNlIG5hbWUgJHtKU09OLnN0cmluZ2lmeSh2YWx1ZSl9OiBhIG5hbWUgb2Ygb25seSBkb3RzIHJlZmVycyB0byBhIGRpcmVjdG9yeSByYXRoZXIgdGhhbiBuYW1pbmcgb25lYCk7XG4gICAgfVxufVxuXG5leHBvcnQgZnVuY3Rpb24gYXNzZXJ0VmFsaWRQYXRoKHBhdGg6IHN0cmluZyk6IHZvaWQge1xuICAgIGlmIChCdWZmZXIuZnJvbShwYXRoLCBcInV0ZjhcIikubGVuZ3RoID4gMTAwMCkge1xuICAgICAgICB0aHJvdyBuZXcgRXJyb3IoYFBhdGggdG9vIGxvbmc6ICR7cGF0aC5sZW5ndGh9IGNoYXJhY3RlcnMgPiAxMDAwLiBQYXRoOiAke3BhdGguc2xpY2UoMCwgMjAwKX1gKTtcbiAgICB9XG4gICAgaWYgKCFwYXRoIHx8IHBhdGguc3RhcnRzV2l0aChcIi9cIikgfHwgcGF0aC5lbmRzV2l0aChcIi9cIikgfHwgcGF0aC5pbmNsdWRlcyhcIi8vXCIpIHx8IHBhdGguaW5jbHVkZXMoXCJcXFxcXCIpIHx8IHBhdGguaW5jbHVkZXMoXCJcXHgwMFwiKSkge1xuICAgICAgICB0aHJvdyBuZXcgRXJyb3IoYEludmFsaWQgcGF0aCAke0pTT04uc3RyaW5naWZ5KHBhdGguc2xpY2UoMCwgMjAwKSl9LCBwYXRocyBjYW5ub3QgYmUgZW1wdHksIHN0YXJ0IG9yIGVuZCB3aXRoIC8sIG9yIGNvbnRhaW4gLy8sIGJhY2tzbGFzaGVzLCBvciBudWxsIGNoYXJhY3RlcnNgKTtcbiAgICB9XG4gICAgaWYgKHBhdGguc3BsaXQoXCIvXCIpLnNvbWUocGFydCA9PiBwYXJ0ID09PSBcIi5cIiB8fCBwYXJ0ID09PSBcIi4uXCIpKSB7XG4gICAgICAgIHRocm93IG5ldyBFcnJvcihgSW52YWxpZCBwYXRoICR7SlNPTi5zdHJpbmdpZnkocGF0aC5zbGljZSgwLCAyMDApKX0sIHBhdGhzIGNhbm5vdCBjb250YWluIC4gb3IgLi4gc2VnbWVudHNgKTtcbiAgICB9XG59XG5cbi8qKiBNZXRob2QgZGVjb3JhdG9yOiB2YWxpZGF0ZXMgdGhlIHdlbGwta25vd24gZmllbGRzIG9mIHRoZSBtZXRob2QncyBzaW5nbGUgY29uZmlnLW9iamVjdCBhcmd1bWVudCAtIGFjY291bnQvYnVja2V0TmFtZSBhcyBuYW1lcywgcGF0aCBhcyBhIHBhdGggLSBiZWZvcmUgdGhlIG1ldGhvZCBydW5zLiBGaWVsZHMgdGhlIGNvbmZpZyBkb2Vzbid0IGhhdmUgYXJlIHNraXBwZWQsIHNvIGl0IGFwcGxpZXMgdG8gZXZlcnkgQVBJIG1ldGhvZCB1bmlmb3JtbHkuIHByZWZpeCBpcyBkZWxpYmVyYXRlbHkgTk9UIHZhbGlkYXRlZDogcHJlZml4ZXMgbWF5IGJlIGVtcHR5IG9yIGVuZCB3aXRoIFwiL1wiLCBib3RoIGludmFsaWQgZm9yIHBhdGhzLiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGFzc2VydFZhbGlkQXJncyh0YXJnZXQ6IHVua25vd24sIGtleTogc3RyaW5nLCBkZXNjcmlwdG9yOiBQcm9wZXJ0eURlc2NyaXB0b3IpOiB2b2lkIHtcbiAgICBsZXQgb3JpZ2luYWwgPSBkZXNjcmlwdG9yLnZhbHVlIGFzICguLi5hcmdzOiB1bmtub3duW10pID0+IHVua25vd247XG4gICAgZGVzY3JpcHRvci52YWx1ZSA9IGZ1bmN0aW9uICguLi5hcmdzOiB1bmtub3duW10pOiB1bmtub3duIHtcbiAgICAgICAgbGV0IGNvbmZpZyA9IGFyZ3NbMF0gYXMgeyBhY2NvdW50Pzogc3RyaW5nOyBidWNrZXROYW1lPzogc3RyaW5nOyBwYXRoPzogc3RyaW5nIH0gfCB1bmRlZmluZWQ7XG4gICAgICAgIGlmIChjb25maWcgJiYgdHlwZW9mIGNvbmZpZyA9PT0gXCJvYmplY3RcIikge1xuICAgICAgICAgICAgaWYgKFwiYWNjb3VudFwiIGluIGNvbmZpZykgYXNzZXJ0VmFsaWROYW1lKGNvbmZpZy5hY2NvdW50IGFzIHN0cmluZywgXCJhY2NvdW50XCIpO1xuICAgICAgICAgICAgaWYgKFwiYnVja2V0TmFtZVwiIGluIGNvbmZpZykgYXNzZXJ0VmFsaWROYW1lKGNvbmZpZy5idWNrZXROYW1lIGFzIHN0cmluZywgXCJidWNrZXQgbmFtZVwiKTtcbiAgICAgICAgICAgIGlmIChcInBhdGhcIiBpbiBjb25maWcpIGFzc2VydFZhbGlkUGF0aChjb25maWcucGF0aCBhcyBzdHJpbmcpO1xuICAgICAgICB9XG4gICAgICAgIHJldHVybiBvcmlnaW5hbC5hcHBseSh0aGlzLCBhcmdzKTtcbiAgICB9O1xufVxuIl19
- /* _JS_SOURCE_HASH = "3c47af61d817d76651ac4b91cc2d83a30a45619b8072517ce152b34b6075b1d8"; */
+ /* _JS_SOURCE_HASH = "52f73d0ef7e2f65848acc68373c50e34bd39c017c3ff908220fc66a6f3d0296f"; */

package/storage/remoteStorage/storageLogs.d.ts

@@ -2,7 +2,7 @@
 /// <reference types="node" />
 import { LogFileInfo } from "../StreamingLogs";
 export declare const LOGS_FOLDER_NAME = "logs";
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. */
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Also printed to the console. */
 export declare function logMutation(entry: {
     op: string;
     account: string;
@@ -15,13 +15,14 @@ export declare function logMutation(entry: {
     callerId?: string;
     internal?: boolean;
 }): void;
-/** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. */
+/** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. Also printed to the console. */
 export declare function logSyncEvent(entry: {
     event: string;
     store: string;
     source?: string;
     [key: string]: unknown;
 }): void;
+/** One console.error is all an error takes (console.error and console.warn are HOOKED to feed the stream) - this just guarantees the hook is installed first, for very-early callers. */
 export declare function logStorageError(message: string): void;
 /** The log files this server holds - see StreamingLogs.listFiles. Empty on processes with no storage folder. */
 export declare function listStorageLogFiles(): Promise<LogFileInfo[]>;

package/storage/remoteStorage/storageLogs.ts

@@ -9,22 +9,20 @@ import { getOwnThreadId } from "../../misc/https/certs";
 
 export const LOGS_FOLDER_NAME = "logs";
 
-const getLogs = lazy((): StreamingLogs | undefined => {
+// NOT lazy: a failure to resolve the folder (the server config may simply not be SET yet this early in startup) must be retried on the next log call, not cached as "no logs ever"
+let logsInstance: StreamingLogs | undefined;
+function getLogs(): StreamingLogs | undefined {
+    if (logsInstance) return logsInstance;
     let folder: string;
     try {
         folder = path.join(getStorageFolder(), LOGS_FOLDER_NAME);
     } catch {
         return undefined;
     }
-    let threadId: string | undefined;
-    try {
-        threadId = getOwnThreadId(getStorageServerConfigOptional()?.rootDomain || "");
-    } catch {
-        threadId = undefined;
-    }
     hookErrorLogging();
-    return new StreamingLogs({ folder, threadId });
-});
+    logsInstance = new StreamingLogs({ folder, threadId: getThreadIdSafe() });
+    return logsInstance;
+}
 
 function firstExternalIPv4(): string | undefined {
     for (let addresses of Object.values(os.networkInterfaces())) {
@@ -35,48 +33,71 @@ function firstExternalIPv4(): string | undefined {
     return undefined;
 }
 
-// Computed once: none of it changes over a process's lifetime
-const baseFields = lazy(() => ({
-    pid: process.pid,
-    entryPoint: process.argv[1],
-    ip: firstExternalIPv4(),
-    domain: getStorageServerConfigOptional()?.domain,
-}));
+// Memoized on SUCCESS only: the thread cert may not be loaded yet when the first entries are written, and that must not permanently strip the thread id from every later entry
+let cachedThreadId: string | undefined;
+function getThreadIdSafe(): string | undefined {
+    if (cachedThreadId) return cachedThreadId;
+    try {
+        cachedThreadId = getOwnThreadId(getStorageServerConfigOptional()?.rootDomain || "");
+    } catch { }
+    return cachedThreadId;
+}
+
+const cachedIp = lazy(() => firstExternalIPv4());
 
-function write(kind: string, entry: { [key: string]: unknown }): void {
+// threadId and domain are re-resolved until known (early entries may predate the config/certs); the rest never changes
+function baseFields() {
+    return {
+        pid: process.pid,
+        threadId: getThreadIdSafe(),
+        entryPoint: process.argv[1],
+        ip: cachedIp(),
+        domain: getStorageServerConfigOptional()?.domain,
+    };
+}
+
+// Everything logged here goes BOTH to the storage log stream and the console - one call, never two. The console line carries only the entry (the base fields are constant per process, so they would just be noise there); the stream gets everything.
+function write(kind: string, entry: { [key: string]: unknown }, alsoConsole: boolean): void {
+    if (alsoConsole) {
+        console.log(`[${kind}] ${JSON.stringify(entry)}`);
+    }
     let logs = getLogs();
     if (!logs) return;
     logs.log({ kind, time: Date.now(), ...baseFields(), ...entry });
 }
 
-/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. */
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. Also printed to the console. */
 export function logMutation(entry: { op: string; account: string; bucketName: string; store?: string; path: string; toPath?: string; size?: number; writeTime?: number; callerId?: string; internal?: boolean }): void {
-    write("mutation", entry);
+    write("mutation", entry, true);
 }
 
-/** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. */
+/** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. Also printed to the console. */
 export function logSyncEvent(entry: { event: string; store: string; source?: string; [key: string]: unknown }): void {
-    write("sync", entry);
+    write("sync", entry, true);
 }
 
+/** One console.error is all an error takes (console.error and console.warn are HOOKED to feed the stream) - this just guarantees the hook is installed first, for very-early callers. */
 export function logStorageError(message: string): void {
-    write("error", { message });
+    getLogs();
+    console.error(message);
 }
 
-// Everything console.errored also lands in the log stream (with a reentrancy guard: a failure INSIDE logging must not log itself forever)
-let writingError = false;
+// Everything console.errored or console.warned also lands in the log stream (with a reentrancy guard: a failure INSIDE logging must not log itself forever)
+let writingHooked = false;
 const hookErrorLogging = lazy(() => {
-    let original = console.error.bind(console);
-    console.error = (...args: unknown[]) => {
-        original(...args);
-        if (writingError) return;
-        writingError = true;
-        try {
-            logStorageError(args.map(x => typeof x === "string" && x || ((x as Error)?.stack ?? JSON.stringify(x))).join(" ").slice(0, 5000));
-        } catch { } finally {
-            writingError = false;
-        }
-    };
+    for (let level of ["error", "warn"] as const) {
+        let original = console[level].bind(console);
+        console[level] = (...args: unknown[]) => {
+            original(...args);
+            if (writingHooked) return;
+            writingHooked = true;
+            try {
+                write(level, { message: args.map(x => typeof x === "string" && x || ((x as Error)?.stack ?? JSON.stringify(x))).join(" ").slice(0, 5000) }, false);
+            } catch { } finally {
+                writingHooked = false;
+            }
+        };
+    }
 });
 
 /** The log files this server holds - see StreamingLogs.listFiles. Empty on processes with no storage folder. */

package/storage/remoteStorage/storeSync.d.ts

@@ -37,14 +37,16 @@ export declare class StoreSync {
     waitForRequiredScans(): Promise<void>;
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     rescanBase(): Promise<void>;
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
+    private syncSource;
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
     boundaryScanRemote(source: IArchives, config: {
         since: number;
         route?: [number, number];
     }): Promise<void>;
     private runSourceSync;
-    private scanSource;
-    private reconcileSource;
+    private pullSource;
+    private pushSource;
     private updateScanIndex;
     private pollChanges;
     private copySourceFiles;

package/storage/remoteStorage/storeSync.ts

@@ -48,6 +48,8 @@ type SourceState = {
     supportsChangesAfter: boolean;
     initialScan: ReturnType<typeof promiseObj>;
     scanComplete: boolean;
+    // Whether a scan of this source has ever actually SUCCEEDED - what full syncs gate on: without one, the index's view of this source can be arbitrarily stale (e.g. it was restarting when we first tried), and syncing from stale information churns forever
+    scanSucceeded: boolean;
     // Files seen in this source's scans / change polls so far
     scannedCount: number;
     // Watermark for getChangesAfter2 polls
@@ -63,6 +65,7 @@ function newSourceState(): SourceState {
         supportsChangesAfter: false,
         initialScan: promiseObj(),
         scanComplete: false,
+        scanSucceeded: false,
         scannedCount: 0,
         changesAfterTime: 0,
         stopped: { stop: false },
@@ -75,10 +78,6 @@ type ScanTally = Record<ScanOutcome, number>;
 function newScanTally(): ScanTally {
     return { filtered: 0, new: 0, updated: 0, tombstone: 0, unchanged: 0 };
 }
-function formatScanTally(tally: ScanTally, total: number): string {
-    let pct = (n: number) => `${Math.round(n / Math.max(total, 1) * 1000) / 10}%`;
-    return `${tally.new} new paths (${pct(tally.new)}), ${tally.updated} newer writes (${pct(tally.updated)}), ${tally.tombstone} deletions (${pct(tally.tombstone)}), ${tally.unchanged} unchanged (${pct(tally.unchanged)}), ${tally.filtered} outside route (${pct(tally.filtered)})`;
-}
 
 export class StoreSync {
     // Per source slot, in step with the store's sources array. Slots are never spliced out - a removed one is marked dead - so a slot number held by a running loop always means the same source.
@@ -220,13 +219,24 @@ export class StoreSync {
 
     /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
     public async rescanBase(): Promise<void> {
-        await this.scanSource(0);
+        await this.syncSource(0);
+    }
+
+    /** One synchronization round of a source: the PULL direction always (its listing, applied to our index), and with "push" the push direction too (what our index says the source is missing, written to it). Push is an argument rather than a separate call because it cannot run without the pull's listing - the index alone cannot say what the source already holds. Listings unblock (initialScan) between the halves, so they never wait behind a push. */
+    private async syncSource(sourceIndex: number, push?: "push"): Promise<void> {
+        let listing = await this.pullSource(sourceIndex);
+        let state = this.states[sourceIndex];
+        state.scanComplete = true;
+        state.initialScan.resolve(undefined);
+        if (push && !this.store.stopped.stop && !state.stopped.stop) {
+            await this.pushSource(sourceIndex, listing);
+        }
     }
 
     /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
     public async boundaryScanRemote(source: IArchives, config: { since: number; route?: [number, number] }): Promise<void> {
         let scanStart = Date.now();
-        console.log(`Boundary scan of ${source.getDebugName()} starting: changes since ${new Date(config.since).toISOString()}, route ${JSON.stringify(config.route || FULL_ROUTE)} (store ${this.store.folder})`);
+        logSyncEvent({ event: "boundaryScanStart", store: this.store.folder, source: source.getDebugName(), since: config.since, route: config.route || FULL_ROUTE });
         let changes = await source.getChangesAfter2({ time: config.since, routes: config.route && [config.route] || undefined });
         let tally = newScanTally();
         for (let file of changes) {
@@ -246,7 +256,10 @@ export class StoreSync {
                 continue;
             }
             let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: file.path, size: file.size, writeTime: file.createTime, forceSetImmutable: true, noChecks: true, internal: true });
-            if (!copied) continue;
+            if (!copied) {
+                console.warn(`Boundary scan could not copy ${file.path} from ${source.getDebugName()} (store ${this.store.folder}): its change feed listed it (${file.size} bytes, writeTime ${new Date(file.createTime).toISOString()}) but the read found nothing`);
+                continue;
+            }
             this.store.noteSyncTransfer("sync get", file.path, copied.size);
             if (copied.size === 0) {
                 this.store.setIndexDeleted(file.path, copied.writeTime);
@@ -260,8 +273,7 @@ export class StoreSync {
                 tally.new++;
             }
         }
-        console.log(`Boundary scan of ${source.getDebugName()} finished in ${Math.round((Date.now() - scanStart) / 1000)}s (store ${this.store.folder}): ${changes.length} changes: ${formatScanTally(tally, changes.length)}`);
-        logSyncEvent({ event: "boundaryScanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, since: config.since, changes: changes.length, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone });
+        logSyncEvent({ event: "boundaryScanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, since: config.since, changes: changes.length, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, unchanged: tally.unchanged });
     }
 
     // ── per-source loops ──
@@ -270,9 +282,10 @@ export class StoreSync {
         await this.store.registerSlot(sourceIndex);
         let { source } = this.store.sources[sourceIndex];
         let state = this.states[sourceIndex];
+        // If a slot ever gets TWO of these, its loops are doubled - every poll and full sync runs twice
+        logSyncEvent({ event: "sourceSyncStart", store: this.store.folder, source: source.getDebugName(), sourceIndex, url: this.store.sources[sourceIndex].url, intermediate: this.store.sources[sourceIndex].intermediate });
         // Read live for every pass, not captured - the store's source list can change while loops run
         let noFullSync = () => this.store.sources[sourceIndex].noFullSync;
-        let listing: Map<string, number> | undefined;
         // An intermediate is a deploy switchover's temporary alternate PORT onto a source we already have: the same bucket, reachable another way for a few minutes. Scanning it would list exactly what scanning that source lists, against a port that is about to disappear - so it is never scanned, and the source it was split out of covers it for as long as it exists and after it is gone.
         if (this.store.sources[sourceIndex].intermediate) {
             state.scanComplete = true;
@@ -283,19 +296,17 @@ export class StoreSync {
             try {
                 let config = await source.getConfig();
                 state.supportsChangesAfter = !!config.supportsChangesAfter;
-                listing = await this.scanSource(sourceIndex);
+                await this.syncSource(sourceIndex, "push");
                 break;
             } catch (e) {
-                console.error(`Initial scan of sync source ${source.getDebugName()} failed, retrying:`, e);
+                logSyncEvent({ event: "initialScanFailed", store: this.store.folder, source: source.getDebugName(), retryInMs: SCAN_RETRY_DELAY, error: String((e as Error).stack ?? e).slice(0, 2000) });
                 await delay(SCAN_RETRY_DELAY);
             }
         }
+        // The loop can also exit by the stop tokens, and nothing may wait forever on a stopped source's initial scan
         state.scanComplete = true;
         state.initialScan.resolve(undefined);
         if (this.store.stopped.stop || state.stopped.stop) return;
-        if (listing) {
-            await this.reconcileSource(sourceIndex, listing);
-        }
         if (!noFullSync()) {
             try {
                 await this.copySourceFiles(sourceIndex);
@@ -305,31 +316,27 @@ export class StoreSync {
         }
         if (state.supportsChangesAfter) {
             runInfinitePoll(CHANGES_POLL_INTERVAL, async () => {
+                // A scan that has not succeeded yet is retried HERE, before anything else - polling changes and copying against a never-scanned source would run on arbitrarily stale information
+                if (!state.scanSucceeded) {
+                    await this.syncSource(sourceIndex, "push");
+                }
                 await this.pollChanges(sourceIndex);
                 if (!noFullSync()) await this.copySourceFiles(sourceIndex);
             }, state.stopped);
-            // Change polls only show what the source HAS, never what it's missing, so pushes run on the full-rescan cadence (findInfo on an index-backed source is cheap)
-            runInfinitePoll(FULL_RESCAN_INTERVAL, async () => {
-                let files = await source.findInfo("");
-                await this.reconcileSource(sourceIndex, new Map(files.map(x => [x.path, x.createTime])));
-            }, state.stopped);
-        } else {
-            runInfinitePoll(FULL_RESCAN_UNINDEXED_INTERVAL, async () => {
-                let rescan = await this.scanSource(sourceIndex);
-                await this.reconcileSource(sourceIndex, rescan);
-                if (!noFullSync()) await this.copySourceFiles(sourceIndex);
-            }, state.stopped);
         }
+        runInfinitePoll(FULL_RESCAN_UNINDEXED_INTERVAL, async () => {
+            await this.syncSource(sourceIndex, "push");
+            if (!noFullSync()) await this.copySourceFiles(sourceIndex);
+        }, state.stopped);
     }
 
-    // Full metadata scan (size, writeTime, path) of one source, applied to the index. Returns the source's listing (path -> write time), which reconcileSource uses for the push direction.
-    private async scanSource(sourceIndex: number): Promise<Map<string, number>> {
+    // The PULL direction: a full metadata scan (size, writeTime, path) of one source, applied to our index. Returns the source's listing (path -> write time), which pushSource uses for the opposite direction.
+    private async pullSource(sourceIndex: number): Promise<Map<string, number>> {
         let { source, route } = this.store.sources[sourceIndex];
         let state = this.states[sourceIndex];
         let scanStart = Date.now();
         let activity: SyncActivity = { type: "metadataScan", sourceDebugName: source.getDebugName(), startTime: scanStart };
         this.activities.add(activity);
-        console.log(`Metadata scan of ${source.getDebugName()} starting (store ${this.store.folder})`);
         logSyncEvent({ event: "scanStart", store: this.store.folder, source: source.getDebugName() });
         let progressTimer = setInterval(() => {
             console.log(`Metadata scan of ${source.getDebugName()} still running (${Math.round((Date.now() - scanStart) / 1000)}s, store ${this.store.folder})`);
@@ -392,17 +399,14 @@ export class StoreSync {
             if (!routeContains(route, getRoute(key))) continue;
             missingOnSource++;
         }
-        // Percentages are of the union of both sides (our index + their listing), so every count has a stable denominator
-        let union = indexSizeBefore + newPaths;
-        let pct = (n: number) => `${Math.round(n / Math.max(union, 1) * 1000) / 10}%`;
-        console.log(`Metadata scan of ${source.getDebugName()} finished in ${Math.round((Date.now() - scanStart) / 1000)}s (store ${this.store.folder}): ${files.length} listed vs ${indexSizeBefore} indexed (union ${union}): ${formatScanTally(tally, union)}, ${missingOnSource} in index but missing on source (${pct(missingOnSource)}), ${removedFromIndex} removed from index (${pct(removedFromIndex)})`);
-        logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, missingOnSource, removedFromIndex });
+        logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, unchanged: tally.unchanged, outsideRoute: tally.filtered, missingOnSource, removedFromIndex });
         state.changesAfterTime = Math.max(state.changesAfterTime, scanStart - CHANGES_POLL_OVERLAP);
+        state.scanSucceeded = true;
         return seen;
     }
 
     // The push direction of synchronization: everything we know that the source is missing (or holds an older copy of) is written to it — including deletions, as tombstone writes. This is what heals a source whose background writes failed (e.g. it was down): the next scan sees what's missing and re-sends it. A failing file is skipped, not fatal (immutable targets are handled by forceSetImmutable, and one unreadable value must not stop the rest of the pass) - only a run of consecutive failures (the source itself is down) aborts until the next scan cycle.
-    private async reconcileSource(sourceIndex: number, listing: Map<string, number>): Promise<void> {
+    private async pushSource(sourceIndex: number, listing: Map<string, number>): Promise<void> {
         let { source, validWindows, route } = this.store.sources[sourceIndex];
         let state = this.states[sourceIndex];
         let acceptsWrites = windowsAcceptWrites(validWindows);
@@ -462,12 +466,8 @@ export class StoreSync {
         }
         if (failed) {
             console.error(`Reconciling sync source ${source.getDebugName()} (store ${this.store.folder}): pushed ${pushed} files, ${failed} failed${aborted && ` before aborting the pass (${consecutiveFailures} consecutive failures - the source looks down; the next scan cycle retries)` || ""}. First errors: ${errors.join(" | ")}`);
-        } else if (pushed) {
-            console.log(`Reconciled sync source ${source.getDebugName()} (store ${this.store.folder}): pushed ${pushed} files it was missing or held older copies of`);
-        }
-        if (pushed || failed) {
-            logSyncEvent({ event: "reconcileFinish", store: this.store.folder, source: source.getDebugName(), pushed, failed, aborted });
         }
+        logSyncEvent({ event: "reconcileFinish", store: this.store.folder, source: source.getDebugName(), pushed, failed, aborted });
     }
 
     private updateScanIndex(sourceIndex: number, file: ArchiveFileInfo): ScanOutcome {
@@ -503,9 +503,9 @@ export class StoreSync {
         for (let file of changes) {
             tally[this.updateScanIndex(sourceIndex, file)]++;
         }
-        // Polls run constantly, so only the ones that actually changed the index get a line
+        // Polls run constantly, so only the ones that actually changed the index get an entry
         if (tally.new || tally.updated || tally.tombstone) {
-            console.log(`Changes poll of ${source.getDebugName()} (store ${this.store.folder}): ${changes.length} changes: ${formatScanTally(tally, changes.length)}`);
+            logSyncEvent({ event: "changesPoll", store: this.store.folder, source: source.getDebugName(), changes: changes.length, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, unchanged: tally.unchanged });
         }
         state.scannedCount += changes.length;
         state.changesAfterTime = pollStart - CHANGES_POLL_OVERLAP;
@@ -535,7 +535,7 @@ export class StoreSync {
             totalBytes,
         };
         this.activities.add(activity);
-        logSyncEvent({ event: "fullSyncStart", store: this.store.folder, source: source.getDebugName(), files: pending.length, bytes: totalBytes });
+        logSyncEvent({ event: "fullSyncStart", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, files: pending.length, bytes: totalBytes });
         let progressLogged = false;
         let logProgress = () => {
             progressLogged = true;
@@ -557,12 +557,14 @@ export class StoreSync {
             console.warn(`Full sync from ${source.getDebugName()} (store ${this.store.folder}) has been running for ${formatTime(elapsed)}: ${doneFiles}/${pending.length} files (${(doneFiles / pending.length * 100).toFixed(1)}%), ${formatNumber(doneBytes)}B/${formatNumber(totalBytes)}B (${(totalBytes && doneBytes / totalBytes * 100 || 100).toFixed(1)}%), ${formatNumber(bytesPerSecond)}B/s. Estimated ${etaText}.`);
         }, FULL_SYNC_SLOW_ERROR_INTERVAL);
         (slowErrorTimer as { unref?: () => void }).unref?.();
+        let failed = 0;
+        let copiedFiles = 0;
+        let copiedBytes = 0;
+        let missingOnSource = 0;
+        let aborted = false;
         try {
             let nextIndex = 0;
-            let failed = 0;
             let consecutiveFailures = 0;
-            let errors: string[] = [];
-            let aborted = false;
             let copyWorker = async () => {
                 while (!aborted && !this.store.stopped.stop && !state.stopped.stop) {
                     let index = nextIndex++;
@@ -571,10 +573,26 @@ export class StoreSync {
                     try {
                         let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
                         if (copied) {
+                            copiedFiles++;
+                            copiedBytes += copied.size;
                             this.store.noteSyncTransfer("sync get", key, copied.size);
-                            // Only move the entry's source if it wasn't changed while we copied
-                            if (this.entryUnchanged(key, entry)) {
-                                this.store.setIndexEntry(key, { writeTime: copied.writeTime, size: copied.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                            // The copy carries the source's write time, and the index commits it under the normal ordering rule (>= the current time wins) - it refusing means a NEWER write landed while we copied, which must be said, not swallowed
+                            if (!this.store.setIndexEntry(key, { writeTime: copied.writeTime, size: copied.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) })) {
+                                logSyncEvent({ event: "fullSyncCommitSuperseded", store: this.store.folder, source: source.getDebugName(), path: key, copiedWriteTime: copied.writeTime, currentWriteTime: this.store.currentWriteTime(key) });
+                            }
+                        } else {
+                            missingOnSource++;
+                            // The source's listing vouched for the file but its read has nothing - and asking again forever is not an answer. If our own disk holds a copy, IT is the real holder, so the entry repoints there; if neither side has bytes, the entry was simply wrong, and it is purged - forgotten, not deleted, exactly like a read that comes up empty everywhere (a scan re-finds the file if it exists anywhere else).
+                            let local = await this.store.sources[0].source.getInfo(key);
+                            if (local) {
+                                this.store.setIndexEntry(key, { writeTime: entry.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "repointed to our local copy", localSize: local.size });
+                            } else if (this.store.currentWriteTime(key) <= entry.writeTime) {
+                                this.store.purgeIndexEntry(key);
+                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "purged - neither the source nor our disk has it, so it does not exist" });
+                            } else {
+                                // A newer write landed while we were checking - the entry is no longer the one we found missing, so it is left alone
+                                logSyncEvent({ event: "fullSyncMissingOnSource", store: this.store.folder, source: source.getDebugName(), path: key, expectedSize: entry.size, expectedWriteTime: entry.writeTime, resolution: "kept - a newer write appeared while checking", currentWriteTime: this.store.currentWriteTime(key) });
                             }
                         }
                         consecutiveFailures = 0;
@@ -582,9 +600,7 @@ export class StoreSync {
                         // One failing file must not abort the sync: it stays on its source (reads still reach it there), and the next sync pass retries it
                         failed++;
                         consecutiveFailures++;
-                        if (errors.length < SYNC_ERROR_LOG_LIMIT) {
-                            errors.push(`${key}: ${(e as Error).stack ?? e}`);
-                        }
+                        logSyncEvent({ event: "fullSyncCopyFailed", store: this.store.folder, source: source.getDebugName(), path: key, size: entry.size, error: String((e as Error).stack ?? e).slice(0, 2000) });
                         if (consecutiveFailures >= SYNC_MAX_CONSECUTIVE_FAILURES) {
                             aborted = true;
                             return;
@@ -601,9 +617,6 @@ export class StoreSync {
                 workers.push(copyWorker());
             }
             await Promise.all(workers);
-            if (failed) {
-                console.error(`Full sync from ${source.getDebugName()} (store ${this.store.folder}): ${failed} of ${pending.length} files failed to copy${aborted && ` before aborting the pass (${SYNC_MAX_CONSECUTIVE_FAILURES} consecutive failures - the source looks down; the next sync pass retries)` || " (they stay on their source, and the next sync pass retries them)"}. First errors: ${errors.join(" | ")}`);
-            }
         } finally {
             clearInterval(progressTimer);
             clearInterval(slowErrorTimer);
@@ -612,7 +625,7 @@ export class StoreSync {
             if (progressLogged) {
                 logProgress();
             }
-            logSyncEvent({ event: "fullSyncFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - activity.startTime, copiedFiles: activity.doneFiles, copiedBytes: activity.doneBytes, totalFiles: pending.length, totalBytes });
+            logSyncEvent({ event: "fullSyncFinish", store: this.store.folder, source: source.getDebugName(), diffBased: state.supportsChangesAfter, durationMs: Date.now() - activity.startTime, totalFiles: pending.length, totalBytes, processedFiles: activity.doneFiles, copiedFiles, copiedBytes, missingOnSource, failed, aborted });
         }
     }