sliftutils 1.7.65 → 1.7.67

package/storage/remoteStorage/sourceWrapper.ts

@@ -6,7 +6,7 @@ import { ROUTING_FILE, getBucketBaseUrl, parseHostedUrl, parseBackblazeUrl, pars
 import { ArchivesRemote } from "./ArchivesRemote";
 import { ArchivesUrl } from "./ArchivesUrl";
 import { ArchivesBackblaze } from "../backblaze";
-import { getStorageServerConfigOptional, getLocalArchives } from "./storageServerState";
+import { getLocalArchives, isOwnAddress } from "./storageServerState";
 
 export const RETRY_START_DELAY = 2 * 1000;
 export const RETRY_MAX_DELAY = 5 * 60 * 1000;
@@ -75,8 +75,7 @@ export class SourceWrapper {
             return wrapper;
         }
         let parsed = parseHostedUrl(config.url);
-        let server = isNode() && getStorageServerConfigOptional() || undefined;
-        if (server && parsed.address === server.domain && parsed.port === server.port) {
+        if (isNode() && isOwnAddress(parsed.address, parsed.port)) {
             // A bucket hosted by our own process - use it directly instead of calling ourselves
             wrapper.api = getLocalArchives(parsed.account, parsed.bucketName);
             return wrapper;

package/storage/remoteStorage/storageController.d.ts

@@ -65,14 +65,14 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
     get2: (account: string, bucketName: string, path: string, range?: {
         start: number;
         end: number;
-    }) => Promise<{
+    }, internal?: boolean) => Promise<{
         data: Buffer;
         writeTime: number;
         size: number;
     } | undefined>;
-    set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean) => Promise<void>;
+    set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean, internal?: boolean) => Promise<void>;
     del: (account: string, bucketName: string, path: string) => Promise<void>;
-    getInfo: (account: string, bucketName: string, path: string) => Promise<{
+    getInfo: (account: string, bucketName: string, path: string, includeTombstones?: boolean) => Promise<{
         writeTime: number;
         size: number;
     } | undefined>;

package/storage/remoteStorage/storageController.ts

@@ -289,27 +289,30 @@ class RemoteStorageControllerBase {
         let result = await this.get2(account, bucketName, path, range);
         return result && result.data || undefined;
     }
-    async get2(account: string, bucketName: string, path: string, range?: { start: number; end: number }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
+    async get2(account: string, bucketName: string, path: string, range?: { start: number; end: number }, internal?: boolean): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         assertValidPath(path);
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return undefined;
+        if (internal && bucket.store.getInternal2) {
+            return await bucket.store.getInternal2(path, { range });
+        }
         return await bucket.store.get2(path, { range });
     }
-    async set(account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean): Promise<void> {
+    async set(account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean, internal?: boolean): Promise<void> {
         assertValidName(bucketName, "bucket name");
         assertValidPath(path);
-        await writeBucketFile(account, bucketName, path, Buffer.from(data), { lastModified, forceSetImmutable });
+        await writeBucketFile(account, bucketName, path, Buffer.from(data), { lastModified, forceSetImmutable, internal });
     }
     async del(account: string, bucketName: string, path: string): Promise<void> {
         assertValidName(bucketName, "bucket name");
         assertValidPath(path);
         await deleteBucketFile(account, bucketName, path);
     }
-    async getInfo(account: string, bucketName: string, path: string): Promise<{ writeTime: number; size: number } | undefined> {
+    async getInfo(account: string, bucketName: string, path: string, includeTombstones?: boolean): Promise<{ writeTime: number; size: number } | undefined> {
         assertValidPath(path);
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return undefined;
-        return await bucket.store.getInfo(path);
+        return await bucket.store.getInfo(path, { includeTombstones });
     }
     async findInfo(account: string, bucketName: string, prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
         let bucket = await getBucket(account, bucketName);

package/storage/remoteStorage/storageServerState.d.ts

@@ -46,6 +46,8 @@ export type LoadedBucket = {
 };
 export declare function addExtraListenPort(port: number): void;
 export declare function removeExtraListenPort(port: number): void;
+/** Whether address:port is this server process. The ONE self test - findSelfIndexes, createApiArchives, and SourceWrapper all consult it, so "is this me" cannot disagree between the routing plan and connection building: a URL that is us on an extra listen port must never become a network client to ourselves, which is how infinite self-request loops form. */
+export declare function isOwnAddress(address: string, port: number): boolean;
 /** A cached IArchives for a persisted source identity: a routing URL (hosted/backblaze) or a disk folder path - the form BlobStore's sources list stores. Configuration (valid windows, routes) decides WHEN a source should be used; for reading bytes the index says a source holds, the URL alone is enough - even for sources no longer in any config. */
 export declare function resolveSourceArchives(url: string): IArchives;
 /** Our role in a bucket's routing config, summarized across ALL currently-valid self entries. Stored instead of a single representative HostedConfig, so nothing can accidentally use one entry's route or flags where the union is required - the standard config has the same URL twice: a routed write-shard entry plus an unrouted read-everything entry. */
@@ -63,6 +65,7 @@ export declare function assertMutable(bucket: LoadedBucket, filePath: string, wr
 export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
     lastModified?: number;
     forceSetImmutable?: boolean;
+    internal?: boolean;
 }): Promise<void>;
 export declare function getBucketConfig(bucket: LoadedBucket): ArchivesConfig;
 /** Which buckets this process currently has loaded - what a deploy successor asks its predecessor for, so it activates exactly the buckets that are actually in use. */

package/storage/remoteStorage/storageServerState.ts

@@ -10,7 +10,7 @@ import { ArchivesDisk } from "../ArchivesDisk";
 import { BlobStore, IBucketStore, DEFAULT_FAST_WRITE_DELAY, WINDOW_END_FLUSH_MARGIN } from "./blobStore";
 import {
     RemoteConfig, HostedConfig, BackblazeConfig, IArchives, ArchivesSource, ArchiveFileInfo, ArchivesConfig,
-    ArchivesSyncStatus, ChangesAfterConfig, GetConfig, SetConfig,
+    ArchivesSyncStatus, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig,
     STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
 } from "../IArchives";
 import { ROUTING_FILE, parseRoutingData, serializeRemoteConfig, parseHostedUrl, replaceHostedUrlPort, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection, normalizeSource } from "./remoteConfig";
@@ -224,15 +224,22 @@ export function addExtraListenPort(port: number): void {
 export function removeExtraListenPort(port: number): void {
     extraListenPorts.delete(port);
 }
+/** Whether address:port is this server process. The ONE self test - findSelfIndexes, createApiArchives, and SourceWrapper all consult it, so "is this me" cannot disagree between the routing plan and connection building: a URL that is us on an extra listen port must never become a network client to ourselves, which is how infinite self-request loops form. */
+export function isOwnAddress(address: string, port: number): boolean {
+    let config = getStorageServerConfigOptional();
+    if (!config) return false;
+    if (address !== config.domain) return false;
+    return port === config.port || extraListenPorts.has(port);
+}
+
 function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: string): number[] {
-    let { domain, port } = getStorageServerConfig();
     let indexes: number[] = [];
     for (let i = 0; i < routing.sources.length; i++) {
         let source = routing.sources[i];
         if (typeof source === "string" || source.type !== "remote") continue;
         let parsed = parseHostedUrl(source.url);
-        if (parsed.address !== domain || parsed.account !== account || parsed.bucketName !== bucketName) continue;
-        if (parsed.port === port || extraListenPorts.has(parsed.port)) {
+        if (parsed.account !== account || parsed.bucketName !== bucketName) continue;
+        if (isOwnAddress(parsed.address, parsed.port)) {
             indexes.push(i);
         }
     }
@@ -737,7 +744,7 @@ async function writeRoutingConfig(account: string, bucketName: string, data: Buf
     await scheduleRoutingReload(account, bucketName);
 }
 
-export async function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: { lastModified?: number; forceSetImmutable?: boolean }): Promise<void> {
+export async function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: { lastModified?: number; forceSetImmutable?: boolean; internal?: boolean }): Promise<void> {
     assertWritesAllowed();
     if (filePath === ROUTING_FILE) {
         let key = `${account}/${bucketName}`;
@@ -772,6 +779,23 @@ export async function writeBucketFile(account: string, bucketName: string, fileP
     } else {
         await assertMutable(loaded, filePath, writeTime);
     }
+    if (config?.internal) {
+        if (!config.lastModified) {
+            throw new Error(`Internal writes must carry lastModified (they are synchronization pushes, ordered by their write time), writing ${JSON.stringify(filePath)} to bucket ${account}/${bucketName}`);
+        }
+        // See SetConfig.internal: the stamp must land inside SOME window+route this server is configured for (any window, including past ones - synchronization moves old data), so a confused peer can't stuff data onto a node that was never meant to hold it
+        let covered = loaded.selfEntries.some(x => writeTime >= x.validWindow[0] && writeTime < x.validWindow[1] && routeContains(x.route, route));
+        if (!covered) {
+            throw new Error(`Internal write of ${JSON.stringify(filePath)} to bucket ${account}/${bucketName} rejected: writeTime ${writeTime} (${new Date(writeTime).toISOString()}) at route ${route} is outside every window/route this server is configured for: ${JSON.stringify(loaded.selfEntries.map(x => ({ validWindow: x.validWindow, route: x.route || FULL_ROUTE })))}`);
+        }
+        if (loaded.store.setInternal) {
+            await loaded.store.setInternal(filePath, data, { lastModified: writeTime });
+            return;
+        }
+        // rawDisk stores: the normal set already IS the disk
+        await loaded.store.set(filePath, data, { lastModified: writeTime });
+        return;
+    }
     await loaded.store.set(filePath, data, { ...getWriteConfig(loaded, writeTime, route), lastModified: writeTime });
 }
 
@@ -1112,6 +1136,9 @@ class ArchivesLocalBucket implements IArchives {
     public async get2(fileName: string, config?: GetConfig): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         let bucket = await this.getBucket();
         if (!bucket) return undefined;
+        if (config?.internal && bucket.store.getInternal2) {
+            return await bucket.store.getInternal2(fileName, config);
+        }
         return await bucket.store.get2(fileName, config);
     }
     public async set(fileName: string, data: Buffer, config?: SetConfig): Promise<string> {
@@ -1143,10 +1170,10 @@ class ArchivesLocalBucket implements IArchives {
             throw e;
         }
     }
-    public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
+    public async getInfo(fileName: string, config?: GetInfoConfig): Promise<{ writeTime: number; size: number } | undefined> {
         let bucket = await this.getBucket();
         if (!bucket) return undefined;
-        return await bucket.store.getInfo(fileName);
+        return await bucket.store.getInfo(fileName, config);
     }
     public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
         let bucket = await this.getBucket();