sliftutils 1.7.102 → 1.7.103

package/storage/remoteStorage/blobStore.ts

@@ -22,6 +22,16 @@ export const WINDOW_END_FLUSH_MARGIN = timeInMinute * 5;
 const WRONG_TARGET_LOG_THROTTLE = 60 * 1000;
 // Marks an upload id that never reached the disk, so it can never collide with one ArchivesDisk handed out
 const DISCARDED_UPLOAD_PREFIX = "discarded_";
+// set refuses empty buffers, and an undelete carries no data - this single ignored byte satisfies the wire
+const UNDELETE_PLACEHOLDER = Buffer.from([1]);
+
+// Deletion history: deleted files keep their bytes on disk (marked in the index) until the history outgrows max(HISTORY_MIN_BYTES, live bytes * history factor) - see StoreSync.enforceHistoryLimit
+export const HISTORY_MIN_BYTES = 10 * 1024 * 1024 * 1024;
+const HISTORY_FACTOR = 1;
+/** The multiple of a store's live bytes its deletion history may grow to. Async so it can later become dynamic and user-configurable; for now it is a constant. */
+export async function getHistoryFactor(): Promise<number> {
+    return HISTORY_FACTOR;
+}
 
 
 /** What we store about a file. Its times are not in here: the index keeps those for every key, deleted ones included (see TransactionFile). */
@@ -274,7 +284,7 @@ export class BlobStore {
         await this.index.flush();
     }
 
-    public async get2(config: { path: string; range?: { start: number; end: number }; internal?: boolean; includeTombstones?: boolean }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
+    public async get2(config: { path: string; range?: { start: number; end: number }; internal?: boolean; includeTombstones?: boolean; includeMarked?: boolean }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         if (config.internal) {
             return await this.getInternal2(config);
         }
@@ -283,6 +293,12 @@ export class BlobStore {
         let range = config.range;
         // Not in the index means it does not exist here: the index IS the answer, and scanning heals it on its own schedule. An entry whose holder is no longer in the source list is still valid - getEntryHolder resolves the persisted URL directly, and get2's fallback loop covers a holder that is gone entirely.
         let entry = this.getIndexEntry(key);
+        // A marked deletion still has its bytes, and the kept value says where - so an includeMarked read is a normal read through the marked entry (it just must not RE-INDEX anything: see the markedRead guards below)
+        let markedRead = false;
+        if (!entry && config.includeMarked) {
+            entry = this.getMarkedEntry(key);
+            markedRead = !!entry;
+        }
         if (!entry) {
             if (!config.includeTombstones) return undefined;
             // A deletion has no bytes - the tombstone IS the answer, so a flag-caller gets its time with empty data
@@ -310,8 +326,8 @@ export class BlobStore {
             }
         }
         if (result) {
-            // Ranged reads can't populate a cache (they're partial)
-            if (this.slotForSourcesListIndex(entry.sourcesListIndex) !== 0 && !range) {
+            // Ranged reads can't populate a cache (they're partial), and a marked read must not re-index anything - caching would resurrect the deleted key as live
+            if (!markedRead && this.slotForSourcesListIndex(entry.sourcesListIndex) !== 0 && !range) {
                 await this.cacheRead(key, result);
             }
             return result;
@@ -336,7 +352,10 @@ export class BlobStore {
                 continue;
             }
             if (!fallback) continue;
-            await this.cacheRead(key, fallback);
+            // A marked read must not re-index anything - caching would resurrect the deleted key as live
+            if (!markedRead) {
+                await this.cacheRead(key, fallback);
+            }
             let data = fallback.data;
             if (range) {
                 data = data.subarray(Math.min(range.start, data.length), Math.min(range.end, data.length));
@@ -344,12 +363,14 @@ export class BlobStore {
             return { data, writeTime: fallback.writeTime, size: fallback.size };
         }
         if (holderError) throw holderError;
-        // The holder answered "not there" and no other source has it either: the entry was stale. Forgotten rather than deleted - nothing happened to this file, we were simply wrong about holding it, and saying otherwise would push a deletion out to everyone else.
-        this.purgeIndexEntry(key);
+        // The holder answered "not there" and no other source has it either: the entry was stale. Forgotten rather than deleted - nothing happened to this file, we were simply wrong about holding it, and saying otherwise would push a deletion out to everyone else. (Never for a marked read: purging would erase the deletion's tombstone.)
+        if (!markedRead) {
+            this.purgeIndexEntry(key);
+        }
         return undefined;
     }
 
-    public async set(config: { path: string; data: Buffer; lastModified?: number; forceSetImmutable?: boolean; internal?: boolean }): Promise<void> {
+    public async set(config: { path: string; data: Buffer; lastModified?: number; forceSetImmutable?: boolean; internal?: boolean; undelete?: boolean }): Promise<void> {
         let { path: key, data } = config;
         if (!data.length) {
             throw new Error(`set was called with an empty buffer for ${JSON.stringify(key)} (store ${this.folder}): an empty file IS a deletion in this system and would read back as missing - call del instead`);
@@ -363,6 +384,13 @@ export class BlobStore {
         } else {
             this.assertWriteTarget(key, route, config.lastModified);
         }
+        if (config.undelete) {
+            if (key === ROUTING_FILE) {
+                throw new Error(`The routing config ${JSON.stringify(ROUTING_FILE)} cannot be undeleted (it cannot be deleted in the first place)`);
+            }
+            await this.undeleteKey(key, writeTime, config.internal);
+            return;
+        }
         if (key !== ROUTING_FILE && this.storeConfig.all().length) {
             if (config.forceSetImmutable) {
                 if (!config.lastModified) {
@@ -447,6 +475,13 @@ export class BlobStore {
             if (!key.startsWith(prefix)) continue;
             infos.set(key, { path: key, createTime: entry.writeTime, size: entry.size });
         }
+        if (config.includeMarked) {
+            // A key is live OR marked, never both, so this cannot collide with the loop above
+            for (let [key, entry] of this.markedEntries()) {
+                if (!key.startsWith(prefix)) continue;
+                infos.set(key, { path: key, createTime: entry.writeTime, size: entry.size });
+            }
+        }
         let files = applyFindInfoShape(Array.from(infos.values()), prefix, { shallow: config.shallow, type: config.type });
         sort(files, x => x.path);
         return files;
@@ -480,6 +515,7 @@ export class BlobStore {
     /** The index's totals plus any in-progress background synchronization. */
     public getSyncProgress(): {
         index: { fileCount: number; byteCount: number };
+        marked: { fileCount: number; byteCount: number; oldestDeleteTime?: number };
         sources: { debugName: string; fileCount: number; byteCount: number }[];
         readerDiskLimit?: number;
         syncing: SyncActivity[];
@@ -487,6 +523,7 @@ export class BlobStore {
         let totals = this.namedIndexTotals();
         return {
             index: { fileCount: totals.fileCount, byteCount: totals.byteCount },
+            marked: this.markedTotals(),
             sources: totals.sources,
             readerDiskLimit: this.readerDiskLimit,
             syncing: this.sync.getActivities(),
@@ -762,6 +799,71 @@ export class BlobStore {
         }
     }
 
+    /** A file MARKED for deletion: its kept index value plus when it was deleted. Undefined when the key is live, never existed, or its history was already dropped. */
+    public getMarkedEntry(key: string): (IndexEntry & { deleteTime: number }) | undefined {
+        let tombstone = this.index.getDeleted(key);
+        if (!tombstone || tombstone.value === undefined) return undefined;
+        return { ...tombstone.value, writeTime: tombstone.valueTime || tombstone.time, changedAt: tombstone.changedAt, deleteTime: tombstone.time };
+    }
+
+    /** Every file marked for deletion - the deletion history, walked by retention and by includeMarked listings. */
+    public *markedEntries(): IterableIterator<[string, IndexEntry & { deleteTime: number }]> {
+        for (let [key, tombstone] of this.index.deletedEntries()) {
+            if (tombstone.value === undefined) continue;
+            yield [key, { ...tombstone.value, writeTime: tombstone.valueTime || tombstone.time, changedAt: tombstone.changedAt, deleteTime: tombstone.time }];
+        }
+    }
+
+    /** The deletion history's totals: how many marked files, their bytes, and the delete time of the OLDEST one - which is how far back the history reaches. */
+    public markedTotals(): { fileCount: number; byteCount: number; oldestDeleteTime?: number } {
+        let fileCount = 0;
+        let byteCount = 0;
+        let oldestDeleteTime: number | undefined;
+        for (let [, entry] of this.markedEntries()) {
+            fileCount++;
+            byteCount += entry.size;
+            if (oldestDeleteTime === undefined || entry.deleteTime < oldestDeleteTime) {
+                oldestDeleteTime = entry.deleteTime;
+            }
+        }
+        return { fileCount, byteCount, oldestDeleteTime };
+    }
+
+    /** Physically removes a marked file's bytes from our disk and drops its kept value, leaving a plain tombstone that ages out normally - retention calling time on the oldest history. */
+    public async dropMarkedHistory(key: string): Promise<void> {
+        await this.ownDisk.del(key);
+        this.index.dropValue(key);
+    }
+
+    /** See SetConfig.undelete: flips a marked deletion back to live (fresh write time, so the restore outranks the deletion everywhere it propagated) - the bytes never left the disk, so reads just work again. Internal restores are a peer's propagation and tolerate having nothing to restore (this node may never have held the file); a caller's restore throws instead. */
+    private async undeleteKey(key: string, writeTime: number, internal: boolean | undefined): Promise<void> {
+        let marked = this.getMarkedEntry(key);
+        if (!this.index.unmark(key, writeTime)) {
+            // Already live: undelete is idempotent (retries and peer propagation both re-send it)
+            if (this.getIndexEntry(key)) return;
+            if (internal) {
+                console.log(`Undelete of ${JSON.stringify(key)} (store ${this.folder}) has nothing to restore here - this node may never have held the file`);
+                return;
+            }
+            throw new Error(`Cannot undelete ${JSON.stringify(key)} (store ${this.folder}): it has no marked deletion to restore - it was never deleted here${this.getDeletedEntry(key) && ", or its deletion history was already dropped (a plain tombstone remains, but the bytes are gone)" || ""}`);
+        }
+        console.log(`Undeleted ${JSON.stringify(key)} (store ${this.folder}): the index entry (${marked?.size} bytes, deleted ${marked && new Date(marked.deleteTime).toISOString()}) is live again as of ${new Date(writeTime).toISOString()} - the bytes never left the disk`);
+        this.config?.onIndexChanged?.(key);
+        if (internal) return;
+        // Peers marked their own copies when the deletion propagated, so the restore propagates the same way. Straight past any write delay - a buffered undelete could be read back as its placeholder byte.
+        let route = getRoute(key);
+        for (let i of this.getWritableSources()) {
+            if (i === 0) continue;
+            if (!routeContains(this.sources[i].route, route)) continue;
+            // Only our own servers understand the flag - a raw source (backblaze) would store the placeholder byte as the file. Their older copy is re-pushed by reconciliation instead.
+            if (this.sources[i].sourceConfig?.type !== "remote") continue;
+            let push = unwrapDelayed(this.sources[i].source).set(key, UNDELETE_PLACEHOLDER, { lastModified: writeTime, undelete: true, noChecks: true, internal: true });
+            void push.catch((e: Error) => {
+                console.error(`Background undelete of ${key} on sync source ${this.sources[i].source.getDebugName()} failed (its scan of us re-finds the file anyway): ${e.stack ?? e}`);
+            });
+        }
+    }
+
     /** How many files we hold, deletions excluded. */
     public indexSize(): number {
         return this.index.size;
@@ -922,8 +1024,7 @@ export class BlobStore {
         assertValidLastModified(config.lastModified);
         if (config.lastModified < await this.currentWriteTime(key)) return;
         if (data.length === 0) {
-            // A deletion stores nothing on our own source - the tombstone is the whole of it
-            await this.sources[0].source.del(key);
+            // The bytes stay on our disk as deletion history (see writeToSources) - the marked index entry is the whole of the deletion
             this.setIndexDeleted(key, config.lastModified);
             return;
         }
@@ -935,6 +1036,8 @@ export class BlobStore {
     private async cacheRead(key: string, result: { data: Buffer; writeTime: number }): Promise<void> {
         await this.sources[0].source.set(key, result.data, { lastModified: result.writeTime, forceSetImmutable: true, noChecks: true });
         this.setIndexEntry(key, { writeTime: result.writeTime, size: result.data.length, sourcesListIndex: this.sourcesListIndexOfSlot(0) });
+        // A down-cache pulls the bytes off a source exactly like synchronization does, so it counts (and logs) the same way
+        this.noteSyncTransfer("sync get", key, result.data.length);
     }
 
     // The shared engine of set and del: an empty buffer is exactly a deletion here, which is why the empty-buffer rejection lives in set (the public API), not in this machinery
@@ -985,8 +1088,7 @@ export class BlobStore {
         }
         // Only our own (first) source blocks the write. Downstream sources are written in the background: a down downstream source must not fail or stall writes, and reconciliation re-sends anything they missed once they come back.
         if (data.length === 0) {
-            // A deletion stores nothing on our own source - the tombstone is the whole of it
-            await this.sources[first].source.del(key);
+            // The bytes STAY on our disk, as deletion history: the index marks the key deleted (keeping its value - see TransactionFile.delete), reads stop finding it, and the retention pass physically removes the oldest history once it outgrows its budget (see StoreSync.enforceHistoryLimit). The marked index entry is the whole of the local deletion.
             this.setIndexDeleted(key, writeTime);
         } else {
             await this.sources[first].source.set(key, data, { lastModified: writeTime, noChecks: true });
@@ -994,6 +1096,10 @@ export class BlobStore {
         }
         if (isRouting) return;
         let route = getRoute(key);
+        // Deletions did not consume the first slot (nothing is written anywhere locally), so it receives the propagated deletion like the rest - except slot 0, our own disk, whose bytes ARE the history being kept
+        if (data.length === 0 && first !== 0) {
+            writable.unshift(first);
+        }
         for (let i of writable) {
             if (!routeContains(this.sources[i].route, route)) continue;
             // Deletions travel as del carrying the original write time (never as empty sets - set rejects empty buffers). Backblaze materializes such dels as real empty files, so its listings still show the deletion for other stores to scan in as a tombstone.

package/storage/remoteStorage/createArchives.d.ts

@@ -1,7 +1,8 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, RemoteConfig, RemoteConfigBase, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SetConfig, SetLargeFileConfig } from "../IArchives";
+import { IArchives, RemoteConfig, RemoteConfigBase, SourceConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SetConfig, SetLargeFileConfig } from "../IArchives";
 import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
+import { LogFileInfo } from "../StreamingLogs";
 /** The address, port, account, and bucket name a bucket routing URL addresses. Throws when the URL isn't a hosted bucket routing URL (https://host:port/file/<account>/<bucketName>/storage/storagerouting.json). */
 export { parseHostedUrl, parseBackblazeUrl, getBucketBaseUrl } from "./remoteConfig";
 /** A client for ONE source - see storeSources.ts. Re-exported here because a chain is built out of them. */
@@ -26,6 +27,9 @@ export declare class ArchivesChain implements IArchives {
         machineId: string;
         ip: string;
     } | undefined>;
+    /** The sources that can serve a file right now, in dispatch order - the first is the write node, the one a plain read asks first. Each entry's url is what GetConfig.sourceUrl / GetInfoConfig.sourceUrl accept, so listing these and then reading with sourceUrl compares the copies the sources actually hold. */
+    getFileSources(fileName: string): Promise<SourceConfig[]>;
+    private runOnSource;
     get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
     /** get2, but trying sources in latency order (fastest first) instead of config order. While this is much faster, it might miss immediate writes: the write node is no longer tried first, so a lagging replica may answer with a slightly older value. Exclusive with noFallbacks (which only considers one source - the write node - so there is no order to speed up); passing both throws. */
     getFast(fileName: string, config?: GetConfig): Promise<{
@@ -67,6 +71,8 @@ export declare class ArchivesChain implements IArchives {
     set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
     private setRoutingConfig;
     del(fileName: string, config?: DelConfig): Promise<void>;
+    /** See IArchives.undelete: restores a file marked for deletion, dispatched to the write node as SetConfig.undelete (the write node propagates the restore to its peers itself). */
+    undelete(fileName: string): Promise<void>;
     /** See IArchives.move. When one node is the write target for BOTH paths, that node moves the file itself - the bytes never come through us - with the same wrong-window/route re-resolution as any write. When the paths route to different shards no single node holds both, so the move degrades to a copy through us plus a delete, CONFIRMED at the destination before the source is touched. No smart timeout on the node-side move: it can be a big file's worth of node-side work, which the upload-sized deadlines would misjudge. */
     move(config: MoveFileConfig): Promise<void>;
     private getVariableShardTargets;
@@ -117,6 +123,39 @@ export declare function clearServerWriteStats(config: {
 }): Promise<{
     clearedBuckets: number;
 }>;
+/** The operation-log files ONE storage server holds (every server logs only its own operations - see listAllServerLogFiles for the whole fleet). The names carry pid/thread/time-range/entry-count metadata; see LogFileInfo. */
+export declare function listServerLogFiles(config: {
+    url: string;
+    account: string;
+}): Promise<LogFileInfo[]>;
+/** Every server's log files at once, one entry per url - a server that cannot answer reports its error instead of failing the rest. */
+export declare function listAllServerLogFiles(config: {
+    urls: string[];
+    account: string;
+}): Promise<{
+    url: string;
+    files?: LogFileInfo[];
+    error?: string;
+}[]>;
+/** Downloads the named log files off one server and decodes them into the logged objects (the wire always carries them LZ4-compressed - live files are compressed in memory server-side). */
+export declare function getServerLogs(config: {
+    url: string;
+    account: string;
+    names: string[];
+}): Promise<{
+    name: string;
+    entries: unknown[];
+}[]>;
+/** getServerLogs, but SEARCHED instead of fully decoded: the raw JSON text is substring-matched (every search string must appear in a statement - see createLogSearcher), and only the matching statements are decoded into objects. Far cheaper than decoding whole files to look for one path or caller. */
+export declare function searchServerLogs(config: {
+    url: string;
+    account: string;
+    names: string[];
+    searches: string[];
+}): Promise<{
+    name: string;
+    entries: unknown[];
+}[]>;
 export declare function getBucketInfo(config: {
     url: string;
 }): Promise<ArchivesConfig>;

package/storage/remoteStorage/createArchives.ts

@@ -18,6 +18,7 @@ import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
 import { RemoteStorageController, STORAGE_NOT_AUTHENTICATED } from "./storageController";
 import { SourceWrapper } from "./sourceWrapper";
 import { ChainState, ChainStateManager } from "./chainStartup";
+import { LogFileInfo, decodeLogFile, createLogSearcher } from "../StreamingLogs";
 import { formatTime } from "socket-function/src/formatting/format";
 
 // How many extra full passes over the sources fallback dispatch makes when EVERY source in a pass failed, before the operation throws (see GetConfig.retries) - most requests shouldn't fail at all, so a couple of blanket retries buys availability for almost nothing
@@ -331,12 +332,33 @@ export class ArchivesChain implements IArchives {
         return undefined;
     }
 
+    /** The sources that can serve a file right now, in dispatch order - the first is the write node, the one a plain read asks first. Each entry's url is what GetConfig.sourceUrl / GetInfoConfig.sourceUrl accept, so listing these and then reading with sourceUrl compares the copies the sources actually hold. */
+    public async getFileSources(fileName: string): Promise<SourceConfig[]> {
+        let state = await this.state.getState();
+        let route = getRoute(fileName);
+        return state.sources.filter(x => routeContains(x.config.route, route) && configWindowCurrent(x.config)).map(x => x.config);
+    }
+
+    // The one exact source a sourceUrl read means, no fallback of any kind - asking for a SPECIFIC source's copy and getting another's would defeat the point
+    private async runOnSource<T>(sourceUrl: string, run: (archives: IArchives) => Promise<T>): Promise<T> {
+        let state = await this.state.getState();
+        let source = state.sources.find(x => x.config.url === sourceUrl);
+        if (!source) {
+            throw new Error(`No configured source has the url ${JSON.stringify(sourceUrl)} for ${this.getDebugName()} (sources: ${state.sources.map(x => x.config.url).join(", ")})`);
+        }
+        return await source.read(run);
+    }
+
     public async get(fileName: string, config?: GetConfig): Promise<Buffer | undefined> {
         let result = await this.get2(fileName, config);
         return result && result.data || undefined;
     }
     /** get2, but trying sources in latency order (fastest first) instead of config order. While this is much faster, it might miss immediate writes: the write node is no longer tried first, so a lagging replica may answer with a slightly older value. Exclusive with noFallbacks (which only considers one source - the write node - so there is no order to speed up); passing both throws. */
     public async getFast(fileName: string, config?: GetConfig): Promise<{ data: Buffer; writeTime: number; size: number; url: string } | { data?: undefined; writeTime?: undefined; size?: undefined; url: string }> {
+        if (config?.sourceUrl) {
+            // A specific source leaves nothing for the latency ordering to decide
+            return await this.get2(fileName, config);
+        }
         return await this.request({ route: getRoute(fileName), noFallbacks: config?.noFallbacks, retries: config?.retries, fast: true, timeout: { path: fileName } }, async (archives, url) => {
             let result = await archives.get2(fileName, config);
             // Empty data is a tombstone, not content - see get2
@@ -346,6 +368,14 @@ export class ArchivesChain implements IArchives {
     }
     /** Always resolves with a url - the authority that answered. A value that doesn't exist is still an answer FROM a server, so it comes back as { url } with no data (never plain undefined); errors from every source throw instead. */
     public async get2(fileName: string, config?: GetConfig): Promise<{ data: Buffer; writeTime: number; size: number; url: string } | { data?: undefined; writeTime?: undefined; size?: undefined; url: string }> {
+        const sourceUrl = config?.sourceUrl;
+        if (sourceUrl) {
+            return await this.runOnSource(sourceUrl, async archives => {
+                let result = await archives.get2(fileName, config);
+                if (!result || !result.data || !result.data.length && !config?.includeTombstones && !(config?.range && result.size)) return { url: sourceUrl };
+                return { data: result.data, writeTime: result.writeTime, size: result.size, url: sourceUrl };
+            });
+        }
         return await this.request({ route: getRoute(fileName), noFallbacks: config?.noFallbacks, retries: config?.retries, timeout: { path: fileName } }, async (archives, url) => {
             let result = await archives.get2(fileName, config);
             // Empty data is a tombstone, not content (unless the caller asked for tombstones) - a ranged read of a REAL file can legitimately be empty though (range past EOF), which the total size distinguishes
@@ -354,6 +384,13 @@ export class ArchivesChain implements IArchives {
         });
     }
     public async getInfo(fileName: string, config?: GetInfoConfig): Promise<{ writeTime: number; size: number; url: string } | undefined> {
+        const sourceUrl = config?.sourceUrl;
+        if (sourceUrl) {
+            return await this.runOnSource(sourceUrl, async archives => {
+                let result = await archives.getInfo(fileName, config);
+                return result && { ...result, url: sourceUrl } || undefined;
+            });
+        }
         return await this.request({ route: getRoute(fileName), noFallbacks: config?.noFallbacks, retries: config?.retries }, async (archives, url) => {
             let result = await archives.getInfo(fileName, config);
             return result && { ...result, url } || undefined;
@@ -561,6 +598,13 @@ export class ArchivesChain implements IArchives {
         await this.request({ write: true, fallbacks: config?.fallbacks, retries: config?.retries, route: getRoute(fileName), timeout: { uploadBytes: 0, label: `Deletion of ${JSON.stringify(fileName)}` } }, archives => archives.del(fileName, config));
     }
 
+    /** See IArchives.undelete: restores a file marked for deletion, dispatched to the write node as SetConfig.undelete (the write node propagates the restore to its peers itself). */
+    public async undelete(fileName: string): Promise<void> {
+        // set refuses empty buffers, and an undelete carries no data - the byte is ignored
+        let placeholder = Buffer.from([1]);
+        await this.request({ write: true, route: getRoute(fileName), timeout: { uploadBytes: placeholder.length, label: `Undelete of ${JSON.stringify(fileName)}` } }, archives => archives.set(fileName, placeholder, { undelete: true }));
+    }
+
     /** See IArchives.move. When one node is the write target for BOTH paths, that node moves the file itself - the bytes never come through us - with the same wrong-window/route re-resolution as any write. When the paths route to different shards no single node holds both, so the move degrades to a copy through us plus a delete, CONFIRMED at the destination before the source is touched. No smart timeout on the node-side move: it can be a big file's worth of node-side work, which the upload-sized deadlines would misjudge. */
     public async move(config: MoveFileConfig): Promise<void> {
         if (config.fromPath === config.toPath) return;
@@ -809,6 +853,46 @@ export async function clearServerWriteStats(config: { url: string; account: stri
     return await callServer(config.url, controller => controller.clearWriteStats({ account: config.account }));
 }
 
+/** The operation-log files ONE storage server holds (every server logs only its own operations - see listAllServerLogFiles for the whole fleet). The names carry pid/thread/time-range/entry-count metadata; see LogFileInfo. */
+export async function listServerLogFiles(config: { url: string; account: string }): Promise<LogFileInfo[]> {
+    return await callServer(config.url, controller => controller.listLogFiles({ account: config.account }));
+}
+
+/** Every server's log files at once, one entry per url - a server that cannot answer reports its error instead of failing the rest. */
+export async function listAllServerLogFiles(config: { urls: string[]; account: string }): Promise<{ url: string; files?: LogFileInfo[]; error?: string }[]> {
+    return await Promise.all(config.urls.map(async url => {
+        try {
+            return { url, files: await listServerLogFiles({ url, account: config.account }) };
+        } catch (e) {
+            return { url, error: String((e as Error).stack ?? e) };
+        }
+    }));
+}
+
+/** Downloads the named log files off one server and decodes them into the logged objects (the wire always carries them LZ4-compressed - live files are compressed in memory server-side). */
+export async function getServerLogs(config: { url: string; account: string; names: string[] }): Promise<{ name: string; entries: unknown[] }[]> {
+    return await callServer(config.url, async controller => {
+        let results: { name: string; entries: unknown[] }[] = [];
+        for (let name of config.names) {
+            let data = await controller.getLogFile({ account: config.account, name });
+            results.push({ name, entries: decodeLogFile(Buffer.from(data)) });
+        }
+        return results;
+    });
+}
+
+/** getServerLogs, but SEARCHED instead of fully decoded: the raw JSON text is substring-matched (every search string must appear in a statement - see createLogSearcher), and only the matching statements are decoded into objects. Far cheaper than decoding whole files to look for one path or caller. */
+export async function searchServerLogs(config: { url: string; account: string; names: string[]; searches: string[] }): Promise<{ name: string; entries: unknown[] }[]> {
+    return await callServer(config.url, async controller => {
+        let results: { name: string; entries: unknown[] }[] = [];
+        for (let name of config.names) {
+            let data = await controller.getLogFile({ account: config.account, name });
+            results.push({ name, entries: createLogSearcher(Buffer.from(data))(config.searches) });
+        }
+        return results;
+    });
+}
+
 export async function getBucketInfo(config: { url: string }): Promise<ArchivesConfig> {
     // getConfig is bucket-level (no store selection), so the fabricated sourceConfig never has to match anything
     let remote = new ArchivesRemote({ url: config.url, waitForAccess: false, sourceConfig: normalizeSource(config.url) });