sliftutils 1.7.99 → 1.7.100

package/storage/remoteStorage/storageController.d.ts

@@ -195,6 +195,7 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
     uploadPart: (config: {
         uploadId: string;
         data: Buffer;
+        offset?: number;
     }) => Promise<void>;
     finishLargeFile: (config: {
         uploadId: string;

package/storage/remoteStorage/storageController.ts

@@ -391,12 +391,13 @@ class RemoteStorageControllerBase {
         largeUploadInfo.set(id, { ...write, account: config.account, bucketName: config.bucketName, storeName: config.sourceConfig.name });
         return id;
     }
-    async uploadPart(config: { uploadId: string; data: Buffer }): Promise<void> {
+    /** offset makes the part write positional (see ArchivesDisk.appendLargeUpload), which is what lets the client RETRY a failed part - a re-sent part lands on the same bytes instead of appending twice. */
+    async uploadPart(config: { uploadId: string; data: Buffer; offset?: number }): Promise<void> {
         assertWritesAllowed();
         let info = largeUploadInfo.get(config.uploadId);
         if (!info) throw new Error(`Unknown large upload ${config.uploadId}`);
         trackAccess({ account: info.account, operation: "uploadPart", path: `${info.bucketName}/${info.path}`, size: config.data.length });
-        await getStore(info.account, info.bucketName, info.storeName).appendLargeUpload({ id: config.uploadId, data: Buffer.from(config.data) });
+        await getStore(info.account, info.bucketName, info.storeName).appendLargeUpload({ id: config.uploadId, data: Buffer.from(config.data), offset: config.offset });
     }
     async finishLargeFile(config: { uploadId: string }): Promise<void> {
         assertWritesAllowed();

package/storage/remoteStorage/storeSync.ts

@@ -9,6 +9,7 @@ import { copyArchiveFile } from "../archiveHelpers";
 import { ArchivesBackblaze } from "../backblaze";
 import { ROUTING_FILE, getRoute, routeContains, parseRoutingData, getConfigVersion } from "./remoteConfig";
 import type { BlobStore, IndexEntry } from "./blobStore";
+import { magenta } from "socket-function/src/formatting/logColors";
 
 // Everything that keeps a store's index in agreement with its sources, plus the two maintenance loops that follow from holding an index (evicting a bounded disk cache, expiring tombstones). The store owns the sources and decides where writes go; this decides what is scanned, pulled, pushed, evicted, and forgotten - so the write path stays readable without the several hundred lines of synchronization machinery interleaved into it.
 
@@ -32,9 +33,14 @@ const DISK_LIMIT_CHECK_INTERVAL = 1000 * 60;
 const FULL_SYNC_PARALLEL = 8;
 // A full sync running longer than this is console.errored (and again every interval after), so a sync that will take days is loud instead of a quiet console.log every minute
 const FULL_SYNC_SLOW_ERROR_INTERVAL = 1000 * 60 * 60;
-// A reconcile pass skips failing files (one bad value must not stop the rest), but this many failures in a row means the target itself is down, so the pass aborts until the next scan cycle
-const RECONCILE_MAX_CONSECUTIVE_FAILURES = 5;
-const RECONCILE_ERROR_LOG_LIMIT = 3;
+// Reconcile passes AND full syncs skip failing files (one bad value must not stop the rest - a source failing one request in a thousand could otherwise never finish a sync), but this many failures in a row means the other side itself is down, so the pass aborts until the next cycle
+const SYNC_MAX_CONSECUTIVE_FAILURES = 5;
+const SYNC_ERROR_LOG_LIMIT = 3;
+// Every failed file waits this long before the pass continues, so a down network burns through the consecutive-failure allowance over a minute-plus instead of in an instant
+const SYNC_FAILURE_DELAY = 1000 * 15;
+// A full listing that comes back EMPTY - or under half of what we know the source holds - is treated as the other end being briefly broken, not as the truth: it is retried this many times, this far apart, before being believed. Believing a wrongly-shrunken listing purges every index entry the source held, which is how sync progress "goes backwards".
+const SUSPICIOUS_SCAN_RETRIES = 3;
+const SUSPICIOUS_SCAN_RETRY_DELAY = 1000 * 60;
 
 type SourceState = {
     supportsChangesAfter: boolean;
@@ -326,7 +332,26 @@ export class StoreSync {
         // The listing request deliberately takes no time or route filters: our slowest sources (backblaze) support neither, so filtering would happen after the full fetch anyway - little benefit, more room for desynchronization. And if a full listing ever becomes too big to send over the network, it is also too big for the receiving process to hold in memory - the fix is more routing shards (each storing and sending less), not filtering.
         let files: ArchiveFileInfo[];
         try {
-            files = await source.findInfo("");
+            let attempts = 0;
+            while (true) {
+                files = await source.findInfo("");
+                if (this.store.stopped.stop || state.stopped.stop) break;
+                // What we believe the source has: whichever is larger of its previous listing and the index entries naming it as holder. Files do not just vanish - a listing far below this means the other end is briefly broken, and believing it would purge everything it held.
+                let held = 0;
+                let scannedSourcesListIndex = this.store.sourcesListIndexOfSlot(sourceIndex);
+                for (let [, entry] of this.store.indexEntries()) {
+                    if (entry.sourcesListIndex === scannedSourcesListIndex) held++;
+                }
+                let expected = Math.max(state.scannedCount, held);
+                if (!(expected > 0 && files.length < expected / 2)) break;
+                if (attempts >= SUSPICIOUS_SCAN_RETRIES) {
+                    console.warn(`Metadata scan of ${source.getDebugName()} (store ${this.store.folder}) STILL lists ${files.length} files where ~${expected} were expected, after ${attempts} retries - accepting the listing as the truth`);
+                    break;
+                }
+                attempts++;
+                console.warn(`Metadata scan of ${source.getDebugName()} (store ${this.store.folder}) listed ${files.length} files, but ~${expected} were expected (previous listing ${state.scannedCount}, index entries it holds ${held}) - the other end is probably briefly broken; asking again in ${SUSPICIOUS_SCAN_RETRY_DELAY / 1000}s (retry ${attempts} of ${SUSPICIOUS_SCAN_RETRIES})`);
+                await delay(SUSPICIOUS_SCAN_RETRY_DELAY);
+            }
         } finally {
             clearInterval(progressTimer);
             this.activities.delete(activity);
@@ -415,13 +440,15 @@ export class StoreSync {
             } catch (e) {
                 failed++;
                 consecutiveFailures++;
-                if (errors.length < RECONCILE_ERROR_LOG_LIMIT) {
+                if (errors.length < SYNC_ERROR_LOG_LIMIT) {
                     errors.push(`${key}: ${(e as Error).stack ?? e}`);
                 }
-                if (consecutiveFailures >= RECONCILE_MAX_CONSECUTIVE_FAILURES) {
+                if (consecutiveFailures >= SYNC_MAX_CONSECUTIVE_FAILURES) {
                     aborted = true;
                     break;
                 }
+                // A pause per failure, so a down network takes minutes to burn through the consecutive-failure allowance instead of an instant
+                await delay(SYNC_FAILURE_DELAY);
             }
         }
         if (failed) {
@@ -499,7 +526,7 @@ export class StoreSync {
         let progressLogged = false;
         let logProgress = () => {
             progressLogged = true;
-            console.log(`Full sync from ${source.getDebugName()} (store ${this.store.folder}): ${activity.doneFiles}/${pending.length} files (${((activity.doneFiles || 0) / pending.length * 100).toFixed(1)}%), ${formatNumber(activity.doneBytes || 0)}B/${formatNumber(totalBytes)}B (${(totalBytes && (activity.doneBytes || 0) / totalBytes * 100 || 100).toFixed(1)}%)`);
+            console.log(magenta(`Full sync from ${source.getDebugName()} (store ${this.store.folder}): ${activity.doneFiles}/${pending.length} files (${((activity.doneFiles || 0) / pending.length * 100).toFixed(1)}%), ${formatNumber(activity.doneBytes || 0)}B/${formatNumber(totalBytes)}B (${(totalBytes && (activity.doneBytes || 0) / totalBytes * 100 || 100).toFixed(1)}%)`));
         };
         let progressTimer = setInterval(logProgress, SYNC_PROGRESS_LOG_INTERVAL);
         (progressTimer as { unref?: () => void }).unref?.();
@@ -519,18 +546,37 @@ export class StoreSync {
         (slowErrorTimer as { unref?: () => void }).unref?.();
         try {
             let nextIndex = 0;
-            let failed = false;
+            let failed = 0;
+            let consecutiveFailures = 0;
+            let errors: string[] = [];
+            let aborted = false;
             let copyWorker = async () => {
-                while (!failed && !this.store.stopped.stop && !state.stopped.stop) {
+                while (!aborted && !this.store.stopped.stop && !state.stopped.stop) {
                     let index = nextIndex++;
                     if (index >= pending.length) return;
                     let { key, entry } = pending[index];
-                    let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
-                    if (copied) {
-                        // Only move the entry's source if it wasn't changed while we copied
-                        if (this.entryUnchanged(key, entry)) {
-                            this.store.setIndexEntry(key, { writeTime: copied.writeTime, size: copied.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                    try {
+                        let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
+                        if (copied) {
+                            // Only move the entry's source if it wasn't changed while we copied
+                            if (this.entryUnchanged(key, entry)) {
+                                this.store.setIndexEntry(key, { writeTime: copied.writeTime, size: copied.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
+                            }
+                        }
+                        consecutiveFailures = 0;
+                    } catch (e) {
+                        // One failing file must not abort the sync: it stays on its source (reads still reach it there), and the next sync pass retries it
+                        failed++;
+                        consecutiveFailures++;
+                        if (errors.length < SYNC_ERROR_LOG_LIMIT) {
+                            errors.push(`${key}: ${(e as Error).stack ?? e}`);
                         }
+                        if (consecutiveFailures >= SYNC_MAX_CONSECUTIVE_FAILURES) {
+                            aborted = true;
+                            return;
+                        }
+                        // A pause per failure, so a down network takes minutes to burn through the consecutive-failure allowance instead of an instant
+                        await delay(SYNC_FAILURE_DELAY);
                     }
                     activity.doneFiles = (activity.doneFiles || 0) + 1;
                     activity.doneBytes = (activity.doneBytes || 0) + entry.size;
@@ -538,13 +584,12 @@ export class StoreSync {
             };
             let workers: Promise<void>[] = [];
             for (let i = 0; i < Math.min(FULL_SYNC_PARALLEL, pending.length); i++) {
-                workers.push(copyWorker().catch((e: Error) => {
-                    // Stop the other workers pulling new files, then surface the error
-                    failed = true;
-                    throw e;
-                }));
+                workers.push(copyWorker());
             }
             await Promise.all(workers);
+            if (failed) {
+                console.error(`Full sync from ${source.getDebugName()} (store ${this.store.folder}): ${failed} of ${pending.length} files failed to copy${aborted && ` before aborting the pass (${SYNC_MAX_CONSECUTIVE_FAILURES} consecutive failures - the source looks down; the next sync pass retries)` || " (they stay on their source, and the next sync pass retries them)"}. First errors: ${errors.join(" | ")}`);
+            }
         } finally {
             clearInterval(progressTimer);
             clearInterval(slowErrorTimer);