sliftutils 1.7.9 → 1.7.10

package/storage/remoteStorage/storageController.d.ts

@@ -1,8 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { ArchiveFileInfo } from "../IArchives";
-import type { BlobStore } from "./blobStore";
-import type { IStorage } from "../IStorage";
+import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
 export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
 export declare const STORAGE_NOT_AUTHENTICATED = "REMOTE_STORAGE_NOT_AUTHENTICATED_cf2f7b1e";
@@ -25,33 +23,13 @@ export type TrustRecord = {
     ip: string;
     time: number;
 };
-export type BucketConfig = {
-    public?: boolean;
-    fast?: boolean;
-    writeDelay?: number;
-};
 export type AccessState = {
     machineId: string;
     ip: string;
     hasAccess: boolean;
-    listAccessCommand: string;
     grantAccessCommand?: string;
-    machines?: (AccessRequest & {
-        trusted: boolean;
-    })[];
-};
-export type StorageServerState = {
-    domain: string;
-    port: number;
-    rootDomain: string;
-    sshTarget: string;
-    serverCommand: string;
-    blobStore: BlobStore;
-    trust: IStorage<TrustRecord>;
-    requests: IStorage<AccessRequest[]>;
-    buckets: IStorage<BucketConfig>;
+    trustedMachines?: TrustRecord[];
 };
-export declare function setStorageServerState(state: StorageServerState): void;
 export declare const RemoteStorageController: import("socket-function/SocketFunctionTypes").SocketRegistered<{
     authenticate: (token: AuthToken) => Promise<{
         machineId: string;
@@ -64,14 +42,22 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         grantAccessCommand: string;
     }>;
     getAccessState: (account: string) => Promise<AccessState>;
+    listRequestsForIP: (account: string, ip: string) => Promise<AccessRequest[]>;
+    grantAccess: (requestId: string) => Promise<TrustRecord>;
     adminListRequests: (ip: string) => Promise<AccessRequest[]>;
     adminGrantAccess: (requestId: string) => Promise<TrustRecord>;
-    ensureBucket: (account: string, bucketName: string, config: BucketConfig) => Promise<void>;
     get: (account: string, bucketName: string, path: string, range?: {
         start: number;
         end: number;
     }) => Promise<Buffer | undefined>;
-    set: (account: string, bucketName: string, path: string, data: Buffer) => Promise<void>;
+    get2: (account: string, bucketName: string, path: string, range?: {
+        start: number;
+        end: number;
+    }) => Promise<{
+        data: Buffer;
+        writeTime: number;
+    } | undefined>;
+    set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number) => Promise<void>;
     del: (account: string, bucketName: string, path: string) => Promise<void>;
     getInfo: (account: string, bucketName: string, path: string) => Promise<{
         writeTime: number;
@@ -81,9 +67,15 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         shallow?: boolean;
         type?: "files" | "folders";
     }) => Promise<ArchiveFileInfo[]>;
+    getChangesAfter: (account: string, bucketName: string, time: number) => Promise<ArchiveFileInfo[]>;
+    getArchivesConfig: (account: string, bucketName: string) => Promise<ArchivesConfig>;
+    getSyncStatus: (account: string, bucketName: string) => Promise<ArchivesSyncStatus>;
     startLargeFile: (account: string, bucketName: string, path: string) => Promise<string>;
     uploadPart: (uploadId: string, data: Buffer) => Promise<void>;
     finishLargeFile: (uploadId: string) => Promise<void>;
     cancelLargeFile: (uploadId: string) => Promise<void>;
-    getPublicFile: (account: string, bucketName: string, path: string) => Promise<Buffer>;
+    httpEntry: (config?: {
+        requireCalls?: string[];
+        cacheTime?: number;
+    }) => Promise<Buffer>;
 }>;

package/storage/remoteStorage/storageController.ts

@@ -2,18 +2,27 @@ module.allowclient = true;
 
 import { SocketFunction } from "socket-function/SocketFunction";
 import { getNodeIdIP } from "socket-function/src/nodeCache";
-import { setHTTPResultHeaders } from "socket-function/src/callHTTPHandler";
+import { setHTTPResultHeaders, getCurrentHTTPRequest } from "socket-function/src/callHTTPHandler";
+import { performLocalCall } from "socket-function/src/callManager";
+import { RequireController } from "socket-function/require/RequireController";
 import { timeInMinute } from "socket-function/src/misc";
 import { getCommonName, getPublicIdentifier, getOwnMachineId, verify, verifyMachineIdForPublicKey } from "../../misc/https/certs";
-import { ArchiveFileInfo } from "../IArchives";
-import type { BlobStore, WriteConfig } from "./blobStore";
-import type { IStorage } from "../IStorage";
+import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+import { ROUTING_FILE } from "./remoteConfig";
+import {
+    getStorageServerConfig, getTrust, getRequests, getLoadedBucket, writeBucketFile,
+    deleteBucketFile, assertWritesAllowed, assertMutable, LoadedBucket,
+} from "./storageServerState";
 
 // The remote storage server's API. Authentication uses certs.ts machine identities: a client
 // proves it owns its machine key by signing a timestamped token (bound to this server, so tokens
 // can't be replayed elsewhere), and the server then trusts that connection as that machineId.
 // Access to an account is granted to specific machineIds, via a command line command run on the
 // storage machine (see storageServer.ts).
+//
+// There is no bucket-creation API: a bucket exists iff its routing config (ROUTING_FILE) exists,
+// and writing that file creates or reconfigures the bucket (see storageServerState.ts). Reads of
+// nonexistent buckets return undefined / empty, same as reads of nonexistent files.
 
 export const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
@@ -43,47 +52,20 @@ export type TrustRecord = {
     ip: string;
     time: number;
 };
-export type BucketConfig = {
-    public?: boolean;
-    fast?: boolean;
-    writeDelay?: number;
-};
 export type AccessState = {
     machineId: string;
     ip: string;
     hasAccess: boolean;
-    // Single ssh commands, runnable from anywhere, that run the admin CLI on the storage machine
-    // (shown when the caller has no access). grantAccessCommand grants the caller's own request.
-    listAccessCommand: string;
+    // A single ssh command, runnable from anywhere, that runs the grantAccess CLI on the storage
+    // machine to grant the caller's own pending request. Only set when the caller has a pending
+    // request (so an already-trusted caller has no need for it).
     grantAccessCommand?: string;
-    // Only provided when the caller has access
-    machines?: (AccessRequest & { trusted: boolean })[];
-};
-
-export type StorageServerState = {
-    domain: string;
-    port: number;
-    rootDomain: string;
-    // user@externalIp of the storage machine, for generated ssh commands
-    sshTarget: string;
-    // Absolute command that runs storageServer.ts on the storage machine (admin args appended)
-    serverCommand: string;
-    blobStore: BlobStore;
-    trust: IStorage<TrustRecord>;
-    requests: IStorage<AccessRequest[]>;
-    buckets: IStorage<BucketConfig>;
+    // Only the machines that ALREADY have access. Pending requests are NEVER listed here — showing
+    // them would let a trusted user accidentally approve a random request. Callers see pending
+    // requests only by explicitly typing an IP into listRequestsForIP.
+    trustedMachines?: TrustRecord[];
 };
 
-let serverState: StorageServerState | undefined;
-export function setStorageServerState(state: StorageServerState) {
-    serverState = state;
-}
-function getState(): StorageServerState {
-    let state = serverState;
-    if (!state) throw new Error(`Storage server is not initialized (this API only works on the storage server)`);
-    return state;
-}
-
 // callerNodeId -> authenticated machineId. Connections are long-lived websockets, so a session
 // lasts until the connection drops (clients re-authenticate on reconnect).
 const sessions = new Map<string, string>();
@@ -107,6 +89,10 @@ function assertValidPath(path: string) {
     if (!path || path.startsWith("/") || path.endsWith("/") || path.includes("//") || path.includes("\\") || path.includes("\x00")) {
         throw new Error(`Invalid path ${JSON.stringify(path.slice(0, 200))}, paths cannot be empty, start or end with /, or contain //, backslashes, or null characters`);
     }
+    // Paths are one-to-one with files on disk, so . and .. segments would escape the store folder
+    if (path.split("/").some(part => part === "." || part === "..")) {
+        throw new Error(`Invalid path ${JSON.stringify(path.slice(0, 200))}, paths cannot contain . or .. segments`);
+    }
 }
 
 function getCallerMachineId(): string {
@@ -121,8 +107,7 @@ function getCallerIP(): string {
     return getNodeIdIP(SocketFunction.getCaller().nodeId);
 }
 function isAdmin(machineId: string): boolean {
-    let state = getState();
-    return machineId === getOwnMachineId(state.rootDomain);
+    return machineId === getOwnMachineId(getStorageServerConfig().rootDomain);
 }
 function requireAdmin(): string {
     let machineId = getCallerMachineId();
@@ -133,46 +118,35 @@ function requireAdmin(): string {
 }
 async function requireAccess(account: string): Promise<string> {
     assertValidName(account, "account");
-    let state = getState();
     let machineId = getCallerMachineId();
     if (isAdmin(machineId)) return machineId;
-    let trusted = await state.trust.get(`${account}|${machineId}`);
+    let trust = await getTrust();
+    let trusted = await trust.get(`${account}|${machineId}`);
     if (!trusted) {
-        throw new Error(`${STORAGE_ACCESS_DENIED} Machine ${machineId} has no access to account ${JSON.stringify(account)}. Visit https://${state.domain}:${state.port}/${account} for access instructions.`);
+        let { domain, port } = getStorageServerConfig();
+        throw new Error(`${STORAGE_ACCESS_DENIED} Machine ${machineId} has no access to account ${JSON.stringify(account)}. Visit https://${domain}:${port}/${account} for access instructions.`);
     }
     return machineId;
 }
 
 // A single command, runnable from anywhere, that sshes into the storage machine and runs the
-// admin CLI there
-function getAdminCommand(args: string): string {
-    let state = getState();
-    return `ssh ${state.sshTarget} '${state.serverCommand} ${args}'`;
-}
-function getListAccessCommand(ip: string): string {
-    return getAdminCommand(`--listAccess ${ip}`);
-}
+// grantAccess CLI there
 function getGrantAccessCommand(requestId: string): string {
-    return getAdminCommand(`--grantAccess ${requestId}`);
+    let { sshTarget, serverCommand } = getStorageServerConfig();
+    return `ssh ${sshTarget} '${serverCommand} --requestId ${requestId}'`;
 }
 
-async function getBucketWriteConfig(account: string, bucketName: string): Promise<WriteConfig> {
-    let state = getState();
-    let config = await state.buckets.get(`${account}/${bucketName}`);
-    return { fast: config?.fast, writeDelay: config?.writeDelay };
-}
-function fileKey(account: string, bucketName: string, path: string): string {
+async function getBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined> {
     assertValidName(account, "account");
     assertValidName(bucketName, "bucket name");
-    assertValidPath(path);
-    return `${account}/${bucketName}/${path}`;
+    return await getLoadedBucket(account, bucketName);
 }
 
 class RemoteStorageControllerBase {
     // Proves the caller owns the machine key for the machineId in its certificate. The signature
     // must be fresh and bound to this server, so it cannot be replayed to (or from) other servers.
     async authenticate(token: AuthToken): Promise<{ machineId: string; ip: string }> {
-        let state = getState();
+        let { domain, port } = getStorageServerConfig();
         let caller = SocketFunction.getCaller();
         if (Math.abs(Date.now() - token.time) > AUTH_TIME_WINDOW) {
             throw new Error(`Auth token time is too far from the server time (token ${token.time}, server ${Date.now()}, allowed drift ${AUTH_TIME_WINDOW}ms)`);
@@ -180,7 +154,7 @@ class RemoteStorageControllerBase {
         verify(token.certPem, token.signature, {
             purpose: STORAGE_AUTH_PURPOSE,
             time: token.time,
-            server: `${state.domain}:${state.port}`,
+            server: `${domain}:${port}`,
         });
         let machineId = getCommonName(token.certPem).split(".")[0];
         if (!verifyMachineIdForPublicKey({ machineId, publicKey: getPublicIdentifier(token.certPem) })) {
@@ -199,10 +173,10 @@ class RemoteStorageControllerBase {
     // IP, so the storage machine's admin can list them with --listAccess <ip> and grant one.
     async requestAccess(account: string): Promise<{ machineId: string; ip: string; requestId: string; grantAccessCommand: string }> {
         assertValidName(account, "account");
-        let state = getState();
         let machineId = getCallerMachineId();
         let ip = getCallerIP();
-        let requests = await state.requests.get(ip) || [];
+        let requestsStorage = await getRequests();
+        let requests = await requestsStorage.get(ip) || [];
         let existing = requests.find(x => x.account === account && x.machineId === machineId);
         if (existing) {
             existing.time = Date.now();
@@ -217,63 +191,87 @@ class RemoteStorageControllerBase {
             requests.push(existing);
         }
         while (requests.length > MAX_REQUESTS_PER_IP) requests.shift();
-        await state.requests.set(ip, requests);
+        await requestsStorage.set(ip, requests);
         return { machineId, ip, requestId: existing.requestId, grantAccessCommand: getGrantAccessCommand(existing.requestId) };
     }
 
     async getAccessState(account: string): Promise<AccessState> {
         assertValidName(account, "account");
-        let state = getState();
         let machineId = getCallerMachineId();
         let ip = getCallerIP();
-        let hasAccess = isAdmin(machineId) || !!await state.trust.get(`${account}|${machineId}`);
-        let result: AccessState = { machineId, ip, hasAccess, listAccessCommand: getListAccessCommand(ip) };
+        let trust = await getTrust();
+        let hasAccess = isAdmin(machineId) || !!await trust.get(`${account}|${machineId}`);
+        let result: AccessState = { machineId, ip, hasAccess };
         if (!hasAccess) {
-            let ownRequest = (await state.requests.get(ip) || []).find(x => x.account === account && x.machineId === machineId);
+            let requests = await getRequests();
+            let ownRequest = (await requests.get(ip) || []).find(x => x.account === account && x.machineId === machineId);
             if (ownRequest) {
                 result.grantAccessCommand = getGrantAccessCommand(ownRequest.requestId);
             }
             return result;
         }
 
-        let machines = new Map<string, AccessRequest & { trusted: boolean }>();
-        for (let requestIp of await state.requests.getKeys()) {
-            for (let request of await state.requests.get(requestIp) || []) {
-                if (request.account !== account) continue;
-                machines.set(request.machineId, { ...request, trusted: false });
-            }
-        }
-        for (let key of await state.trust.getKeys()) {
+        let trustedMachines: TrustRecord[] = [];
+        for (let key of await trust.getKeys()) {
             if (!key.startsWith(`${account}|`)) continue;
-            let record = await state.trust.get(key);
-            if (!record) continue;
-            let existing = machines.get(record.machineId);
-            machines.set(record.machineId, {
-                requestId: existing?.requestId || "",
-                account,
-                machineId: record.machineId,
-                ip: record.ip,
-                time: record.time,
-                trusted: true,
-            });
+            let record = await trust.get(key);
+            if (record) trustedMachines.push(record);
         }
-        result.machines = Array.from(machines.values());
+        result.trustedMachines = trustedMachines;
         return result;
     }
 
+    // Callable by any machine that has access to `account`. Returns pending access requests for the
+    // account that come from EXACTLY `ip`. Callers must type in an IP explicitly — the server never
+    // volunteers a list of requesting IPs, so a trusted user can't accidentally approve a random
+    // request from a machine they didn't mean to trust.
+    async listRequestsForIP(account: string, ip: string): Promise<AccessRequest[]> {
+        await requireAccess(account);
+        let requests = await getRequests();
+        return (await requests.get(ip) || []).filter(x => x.account === account);
+    }
+
+    // Callable by any machine that has access to the request's account (or by the storage-machine
+    // admin). Grants the requested access; the caller must supply the specific requestId, which they
+    // only get by explicitly looking up requests for a specific IP.
+    async grantAccess(requestId: string): Promise<TrustRecord> {
+        // Must capture in the synchronous phase — SocketFunction.getCaller() only works before any await.
+        let callerMachineId = getCallerMachineId();
+        let trust = await getTrust();
+        let requests = await getRequests();
+        for (let ip of await requests.getKeys()) {
+            for (let request of await requests.get(ip) || []) {
+                if (request.requestId !== requestId) continue;
+                if (!isAdmin(callerMachineId) && !await trust.get(`${request.account}|${callerMachineId}`)) {
+                    throw new Error(`${STORAGE_ACCESS_DENIED} Machine ${callerMachineId} has no access to account ${JSON.stringify(request.account)}`);
+                }
+                let record: TrustRecord = {
+                    account: request.account,
+                    machineId: request.machineId,
+                    ip: request.ip,
+                    time: Date.now(),
+                };
+                await trust.set(`${request.account}|${request.machineId}`, record);
+                return record;
+            }
+        }
+        throw new Error(`No access request found with id ${JSON.stringify(requestId)}. It may have already been granted or expired.`);
+    }
+
     // Admin (must be run from the storage machine itself, which shares the server's machineId).
     // Only returns requests for the given IP, so you cannot accidentally grant a request from an
     // IP you didn't explicitly type in.
     async adminListRequests(ip: string): Promise<AccessRequest[]> {
         requireAdmin();
-        let state = getState();
-        return await state.requests.get(ip) || [];
+        let requests = await getRequests();
+        return await requests.get(ip) || [];
     }
     async adminGrantAccess(requestId: string): Promise<TrustRecord> {
         requireAdmin();
-        let state = getState();
-        for (let ip of await state.requests.getKeys()) {
-            for (let request of await state.requests.get(ip) || []) {
+        let trust = await getTrust();
+        let requests = await getRequests();
+        for (let ip of await requests.getKeys()) {
+            for (let request of await requests.get(ip) || []) {
                 if (request.requestId !== requestId) continue;
                 let record: TrustRecord = {
                     account: request.account,
@@ -281,93 +279,159 @@ class RemoteStorageControllerBase {
                     ip: request.ip,
                     time: Date.now(),
                 };
-                await state.trust.set(`${request.account}|${request.machineId}`, record);
+                await trust.set(`${request.account}|${request.machineId}`, record);
                 return record;
             }
         }
-        throw new Error(`No access request found with id ${JSON.stringify(requestId)}. Run --listAccess <ip> to see request ids.`);
-    }
-
-    async ensureBucket(account: string, bucketName: string, config: BucketConfig): Promise<void> {
-        await requireAccess(account);
-        assertValidName(bucketName, "bucket name");
-        let state = getState();
-        let key = `${account}/${bucketName}`;
-        let existing = await state.buckets.get(key);
-        if (existing && JSON.stringify(existing) === JSON.stringify(config)) return;
-        await state.buckets.set(key, config);
+        throw new Error(`No access request found with id ${JSON.stringify(requestId)}. It may have already been granted or expired.`);
     }
 
     async get(account: string, bucketName: string, path: string, range?: { start: number; end: number }): Promise<Buffer | undefined> {
+        let result = await this.get2(account, bucketName, path, range);
+        return result && result.data || undefined;
+    }
+    async get2(account: string, bucketName: string, path: string, range?: { start: number; end: number }): Promise<{ data: Buffer; writeTime: number } | undefined> {
         await requireAccess(account);
-        return await getState().blobStore.get(fileKey(account, bucketName, path), range);
+        assertValidPath(path);
+        let bucket = await getBucket(account, bucketName);
+        if (!bucket) return undefined;
+        return await bucket.store.get2(path, { range });
     }
-    async set(account: string, bucketName: string, path: string, data: Buffer): Promise<void> {
+    async set(account: string, bucketName: string, path: string, data: Buffer, lastModified?: number): Promise<void> {
         await requireAccess(account);
-        let writeConfig = await getBucketWriteConfig(account, bucketName);
-        await getState().blobStore.set(fileKey(account, bucketName, path), Buffer.from(data), writeConfig);
+        assertValidName(bucketName, "bucket name");
+        assertValidPath(path);
+        // Handles bucket creation (writes of ROUTING_FILE), reconfiguration, fast mode, and
+        // immutability — see storageServerState.ts
+        await writeBucketFile(account, bucketName, path, Buffer.from(data), { lastModified });
     }
     async del(account: string, bucketName: string, path: string): Promise<void> {
         await requireAccess(account);
-        let writeConfig = await getBucketWriteConfig(account, bucketName);
-        await getState().blobStore.del(fileKey(account, bucketName, path), writeConfig);
+        assertValidName(bucketName, "bucket name");
+        assertValidPath(path);
+        await deleteBucketFile(account, bucketName, path);
     }
     async getInfo(account: string, bucketName: string, path: string): Promise<{ writeTime: number; size: number } | undefined> {
         await requireAccess(account);
-        return await getState().blobStore.getInfo(fileKey(account, bucketName, path));
+        assertValidPath(path);
+        let bucket = await getBucket(account, bucketName);
+        if (!bucket) return undefined;
+        return await bucket.store.getInfo(path);
     }
     async findInfo(account: string, bucketName: string, prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
         await requireAccess(account);
-        assertValidName(bucketName, "bucket name");
-        let bucketRoot = `${account}/${bucketName}/`;
-        let infos = await getState().blobStore.findInfo(bucketRoot + prefix, config);
-        return infos.map(info => ({ ...info, path: info.path.slice(bucketRoot.length) }));
+        let bucket = await getBucket(account, bucketName);
+        if (!bucket) return [];
+        return await bucket.store.findInfo(prefix, config);
+    }
+    // Fast (served from the store's BulkDatabase2 index, not a scan) — see IArchives.getChangesAfter
+    async getChangesAfter(account: string, bucketName: string, time: number): Promise<ArchiveFileInfo[]> {
+        await requireAccess(account);
+        let bucket = await getBucket(account, bucketName);
+        if (!bucket) return [];
+        if (!bucket.store.getChangesAfter) {
+            throw new Error(`Bucket ${account}/${bucketName} does not support getChangesAfter (rawDisk buckets have no index)`);
+        }
+        return await bucket.store.getChangesAfter(time);
+    }
+    async getArchivesConfig(account: string, bucketName: string): Promise<ArchivesConfig> {
+        await requireAccess(account);
+        let bucket = await getBucket(account, bucketName);
+        // Missing buckets say true, matching what they become once created (the default store type)
+        return { supportsChangesAfter: !bucket || !!bucket.store.getChangesAfter };
+    }
+    async getSyncStatus(account: string, bucketName: string): Promise<ArchivesSyncStatus> {
+        await requireAccess(account);
+        let bucket = await getBucket(account, bucketName);
+        if (!bucket) return { allScansComplete: true, indexSize: 0, sources: [] };
+        if (!bucket.store.getSyncStatus) {
+            throw new Error(`Bucket ${account}/${bucketName} does not support getSyncStatus (rawDisk buckets have no synchronization)`);
+        }
+        return await bucket.store.getSyncStatus();
     }
 
     async startLargeFile(account: string, bucketName: string, path: string): Promise<string> {
+        assertWritesAllowed();
         await requireAccess(account);
         // Validates now, so the upload doesn't fail at the end
-        fileKey(account, bucketName, path);
-        let id = await getState().blobStore.startLargeUpload();
+        assertValidPath(path);
+        let bucket = await getBucket(account, bucketName);
+        if (!bucket) {
+            throw new Error(`Bucket ${account}/${bucketName} does not exist. Write its routing config to ${JSON.stringify(ROUTING_FILE)} to create it.`);
+        }
+        await assertMutable(bucket, path);
+        let id = await bucket.store.startLargeUpload();
         largeUploadInfo.set(id, { account, bucketName, path });
         return id;
     }
     async uploadPart(uploadId: string, data: Buffer): Promise<void> {
+        assertWritesAllowed();
         let info = largeUploadInfo.get(uploadId);
         if (!info) throw new Error(`Unknown large upload ${uploadId}`);
         await requireAccess(info.account);
-        await getState().blobStore.appendLargeUpload(uploadId, Buffer.from(data));
+        let bucket = await getBucket(info.account, info.bucketName);
+        if (!bucket) throw new Error(`Bucket ${info.account}/${info.bucketName} no longer exists`);
+        await bucket.store.appendLargeUpload(uploadId, Buffer.from(data));
     }
     async finishLargeFile(uploadId: string): Promise<void> {
+        assertWritesAllowed();
         let info = largeUploadInfo.get(uploadId);
         if (!info) throw new Error(`Unknown large upload ${uploadId}`);
         await requireAccess(info.account);
         largeUploadInfo.delete(uploadId);
-        await getState().blobStore.finishLargeUpload(uploadId, fileKey(info.account, info.bucketName, info.path));
+        let bucket = await getBucket(info.account, info.bucketName);
+        if (!bucket) throw new Error(`Bucket ${info.account}/${info.bucketName} no longer exists`);
+        await bucket.store.finishLargeUpload(uploadId, info.path);
     }
     async cancelLargeFile(uploadId: string): Promise<void> {
         let info = largeUploadInfo.get(uploadId);
         if (!info) return;
         await requireAccess(info.account);
         largeUploadInfo.delete(uploadId);
-        await getState().blobStore.cancelLargeUpload(uploadId);
+        let bucket = await getBucket(info.account, info.bucketName);
+        if (!bucket) return;
+        await bucket.store.cancelLargeUpload(uploadId);
     }
 
-    // Serves files from public buckets over plain HTTP GET (see IArchives getURL). No
-    // authentication, which is what public means (private buckets are API-access only).
-    async getPublicFile(account: string, bucketName: string, path: string): Promise<Buffer> {
-        let state = getState();
-        let bucket = await state.buckets.get(`${account}/${bucketName}`);
-        if (!bucket?.public) {
-            throw new Error(`Bucket ${account}/${bucketName} is not public`);
+    // The server's single default HTTP route. /file/<account>/<bucketName>/<path> serves files from
+    // public buckets over plain GETs (see IArchives.getURL); every other path serves the access
+    // page (via RequireController.requireHTML — the path is the account name, see accessPage.tsx).
+    async httpEntry(config?: { requireCalls?: string[]; cacheTime?: number }): Promise<Buffer> {
+        // Both are keyed by the current call and must be captured synchronously, before any await
+        let caller = SocketFunction.getCaller();
+        let request = getCurrentHTTPRequest();
+        let pathname = new URL(request?.url || "/", "https://localhost").pathname;
+        if (!pathname.startsWith("/file/")) {
+            return await performLocalCall({
+                caller,
+                call: { nodeId: caller.nodeId, classGuid: RequireController._classGuid, functionName: "requireHTML", args: [config] },
+            }) as Buffer;
+        }
+        let parts = pathname.split("/").filter(x => x).map(decodeURIComponent);
+        let account = parts[1];
+        let bucketName = parts[2];
+        let filePath = parts.slice(3).join("/");
+        if (!account || !bucketName || !filePath) {
+            return setHTTPResultHeaders(Buffer.from(""), { status: "404" });
+        }
+        assertValidName(account, "account");
+        assertValidName(bucketName, "bucket name");
+        assertValidPath(filePath);
+        let bucket = await getLoadedBucket(account, bucketName);
+        if (!bucket) {
+            return setHTTPResultHeaders(Buffer.from(""), { status: "404" });
+        }
+        if (!bucket.self?.public) {
+            throw new Error(`Bucket ${account}/${bucketName} is not public, so its files cannot be read over plain URLs`);
         }
-        let data = await state.blobStore.get(fileKey(account, bucketName, path));
-        if (!data) {
-            throw new Error(`File not found: ${path} in ${account}/${bucketName}`);
+        let result = await bucket.store.get2(filePath);
+        if (!result) {
+            return setHTTPResultHeaders(Buffer.from(""), { status: "404" });
         }
-        let ext = path.split(".").pop() || "";
-        return setHTTPResultHeaders(data, {
+        let ext = filePath.split(".").pop() || "";
+        return setHTTPResultHeaders(result.data, {
             "Content-Type": CONTENT_TYPES[ext.toLowerCase()] || "application/octet-stream",
+            "Last-Modified": new Date(result.writeTime).toUTCString(),
         });
     }
 }
@@ -381,18 +445,23 @@ export const RemoteStorageController = SocketFunction.register(
         authenticate: {},
         requestAccess: {},
         getAccessState: {},
+        listRequestsForIP: {},
+        grantAccess: {},
         adminListRequests: {},
         adminGrantAccess: {},
-        ensureBucket: {},
         get: {},
+        get2: {},
         set: {},
         del: {},
         getInfo: {},
         findInfo: {},
+        getChangesAfter: {},
+        getArchivesConfig: {},
+        getSyncStatus: {},
         startLargeFile: {},
         uploadPart: {},
         finishLargeFile: {},
         cancelLargeFile: {},
-        getPublicFile: {},
+        httpEntry: {},
     })
 );

package/storage/remoteStorage/storageServer.d.ts

@@ -1 +1,12 @@
 import "./accessPage";
+export type HostStorageServerConfig = {
+    url: string;
+    folder: string;
+    cloudflareApiToken?: {
+        key: string;
+    } | {
+        path: string;
+    };
+    lowSpaceThresholdBytes?: number;
+};
+export declare function hostStorageServer(config: HostStorageServerConfig): Promise<void>;