sliftutils 1.7.10 → 1.7.11

package/storage/remoteStorage/storageController.ts

@@ -1,13 +1,14 @@
 module.allowclient = true;
 
 import { SocketFunction } from "socket-function/SocketFunction";
+import { SocketFunctionHook } from "socket-function/SocketFunctionTypes";
 import { getNodeIdIP } from "socket-function/src/nodeCache";
 import { setHTTPResultHeaders, getCurrentHTTPRequest } from "socket-function/src/callHTTPHandler";
 import { performLocalCall } from "socket-function/src/callManager";
 import { RequireController } from "socket-function/require/RequireController";
 import { timeInMinute } from "socket-function/src/misc";
 import { getCommonName, getPublicIdentifier, getOwnMachineId, verify, verifyMachineIdForPublicKey } from "../../misc/https/certs";
-import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, IMMUTABLE_CACHE_TIME } from "../IArchives";
 import { ROUTING_FILE } from "./remoteConfig";
 import {
     getStorageServerConfig, getTrust, getRequests, getLoadedBucket, writeBucketFile,
@@ -143,6 +144,9 @@ async function getBucket(account: string, bucketName: string): Promise<LoadedBuc
 }
 
 class RemoteStorageControllerBase {
+    // Latency measurement (see SourceWrapper's pinging); no auth, it measures the transport
+    async ping(): Promise<void> { }
+
     // Proves the caller owns the machine key for the machineId in its certificate. The signature
     // must be fresh and bound to this server, so it cannot be replayed to (or from) other servers.
     async authenticate(token: AuthToken): Promise<{ machineId: string; ip: string }> {
@@ -226,7 +230,6 @@ class RemoteStorageControllerBase {
     // volunteers a list of requesting IPs, so a trusted user can't accidentally approve a random
     // request from a machine they didn't mean to trust.
     async listRequestsForIP(account: string, ip: string): Promise<AccessRequest[]> {
-        await requireAccess(account);
         let requests = await getRequests();
         return (await requests.get(ip) || []).filter(x => x.account === account);
     }
@@ -262,12 +265,10 @@ class RemoteStorageControllerBase {
     // Only returns requests for the given IP, so you cannot accidentally grant a request from an
     // IP you didn't explicitly type in.
     async adminListRequests(ip: string): Promise<AccessRequest[]> {
-        requireAdmin();
         let requests = await getRequests();
         return await requests.get(ip) || [];
     }
     async adminGrantAccess(requestId: string): Promise<TrustRecord> {
-        requireAdmin();
         let trust = await getTrust();
         let requests = await getRequests();
         for (let ip of await requests.getKeys()) {
@@ -290,15 +291,13 @@ class RemoteStorageControllerBase {
         let result = await this.get2(account, bucketName, path, range);
         return result && result.data || undefined;
     }
-    async get2(account: string, bucketName: string, path: string, range?: { start: number; end: number }): Promise<{ data: Buffer; writeTime: number } | undefined> {
-        await requireAccess(account);
+    async get2(account: string, bucketName: string, path: string, range?: { start: number; end: number }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         assertValidPath(path);
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return undefined;
         return await bucket.store.get2(path, { range });
     }
     async set(account: string, bucketName: string, path: string, data: Buffer, lastModified?: number): Promise<void> {
-        await requireAccess(account);
         assertValidName(bucketName, "bucket name");
         assertValidPath(path);
         // Handles bucket creation (writes of ROUTING_FILE), reconfiguration, fast mode, and
@@ -306,27 +305,23 @@ class RemoteStorageControllerBase {
         await writeBucketFile(account, bucketName, path, Buffer.from(data), { lastModified });
     }
     async del(account: string, bucketName: string, path: string): Promise<void> {
-        await requireAccess(account);
         assertValidName(bucketName, "bucket name");
         assertValidPath(path);
         await deleteBucketFile(account, bucketName, path);
     }
     async getInfo(account: string, bucketName: string, path: string): Promise<{ writeTime: number; size: number } | undefined> {
-        await requireAccess(account);
         assertValidPath(path);
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return undefined;
         return await bucket.store.getInfo(path);
     }
     async findInfo(account: string, bucketName: string, prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
-        await requireAccess(account);
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return [];
         return await bucket.store.findInfo(prefix, config);
     }
     // Fast (served from the store's BulkDatabase2 index, not a scan) — see IArchives.getChangesAfter
     async getChangesAfter(account: string, bucketName: string, time: number): Promise<ArchiveFileInfo[]> {
-        await requireAccess(account);
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return [];
         if (!bucket.store.getChangesAfter) {
@@ -335,13 +330,26 @@ class RemoteStorageControllerBase {
         return await bucket.store.getChangesAfter(time);
     }
     async getArchivesConfig(account: string, bucketName: string): Promise<ArchivesConfig> {
-        await requireAccess(account);
         let bucket = await getBucket(account, bucketName);
         // Missing buckets say true, matching what they become once created (the default store type)
-        return { supportsChangesAfter: !bucket || !!bucket.store.getChangesAfter };
+        let progress = bucket?.store.getSyncProgress?.();
+        return {
+            supportsChangesAfter: !bucket || !!bucket.store.getChangesAfter,
+            remoteConfig: bucket?.routing,
+            index: progress?.index,
+            indexSources: progress?.sources,
+            readerDiskLimit: progress?.readerDiskLimit,
+            syncing: progress?.syncing,
+        };
+    }
+    /** Walks the whole index for exact totals (overall and per holding source) - more expensive
+     *  than the maintained counters that getArchivesConfig returns, but immune to counter drift. */
+    async getIndexInfo(account: string, bucketName: string): Promise<{ fileCount: number; byteCount: number; sources: { debugName: string; fileCount: number; byteCount: number }[] } | undefined> {
+        let bucket = await getBucket(account, bucketName);
+        if (!bucket || !bucket.store.computeIndexTotals) return undefined;
+        return await bucket.store.computeIndexTotals();
     }
     async getSyncStatus(account: string, bucketName: string): Promise<ArchivesSyncStatus> {
-        await requireAccess(account);
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return { allScansComplete: true, indexSize: 0, sources: [] };
         if (!bucket.store.getSyncStatus) {
@@ -352,14 +360,13 @@ class RemoteStorageControllerBase {
 
     async startLargeFile(account: string, bucketName: string, path: string): Promise<string> {
         assertWritesAllowed();
-        await requireAccess(account);
         // Validates now, so the upload doesn't fail at the end
         assertValidPath(path);
         let bucket = await getBucket(account, bucketName);
         if (!bucket) {
             throw new Error(`Bucket ${account}/${bucketName} does not exist. Write its routing config to ${JSON.stringify(ROUTING_FILE)} to create it.`);
         }
-        await assertMutable(bucket, path);
+        await assertMutable(bucket, path, Date.now());
         let id = await bucket.store.startLargeUpload();
         largeUploadInfo.set(id, { account, bucketName, path });
         return id;
@@ -368,7 +375,6 @@ class RemoteStorageControllerBase {
         assertWritesAllowed();
         let info = largeUploadInfo.get(uploadId);
         if (!info) throw new Error(`Unknown large upload ${uploadId}`);
-        await requireAccess(info.account);
         let bucket = await getBucket(info.account, info.bucketName);
         if (!bucket) throw new Error(`Bucket ${info.account}/${info.bucketName} no longer exists`);
         await bucket.store.appendLargeUpload(uploadId, Buffer.from(data));
@@ -377,7 +383,6 @@ class RemoteStorageControllerBase {
         assertWritesAllowed();
         let info = largeUploadInfo.get(uploadId);
         if (!info) throw new Error(`Unknown large upload ${uploadId}`);
-        await requireAccess(info.account);
         largeUploadInfo.delete(uploadId);
         let bucket = await getBucket(info.account, info.bucketName);
         if (!bucket) throw new Error(`Bucket ${info.account}/${info.bucketName} no longer exists`);
@@ -386,7 +391,6 @@ class RemoteStorageControllerBase {
     async cancelLargeFile(uploadId: string): Promise<void> {
         let info = largeUploadInfo.get(uploadId);
         if (!info) return;
-        await requireAccess(info.account);
         largeUploadInfo.delete(uploadId);
         let bucket = await getBucket(info.account, info.bucketName);
         if (!bucket) return;
@@ -424,44 +428,109 @@ class RemoteStorageControllerBase {
         if (!bucket.self?.public) {
             throw new Error(`Bucket ${account}/${bucketName} is not public, so its files cannot be read over plain URLs`);
         }
-        let result = await bucket.store.get2(filePath);
-        if (!result) {
+        // The index answers existence + write time + size without touching any data, so
+        // If-Modified-Since and range validation cost nothing
+        let info = await bucket.store.getInfo(filePath);
+        if (!info || !info.size) {
             return setHTTPResultHeaders(Buffer.from(""), { status: "404" });
         }
         let ext = filePath.split(".").pop() || "";
-        return setHTTPResultHeaders(result.data, {
+        let headers: { [header: string]: string } = {
             "Content-Type": CONTENT_TYPES[ext.toLowerCase()] || "application/octet-stream",
-            "Last-Modified": new Date(result.writeTime).toUTCString(),
-        });
+            "Last-Modified": new Date(info.writeTime).toUTCString(),
+            "Accept-Ranges": "bytes",
+        };
+        if (bucket.self?.immutable) {
+            headers["Cache-Control"] = `max-age=${IMMUTABLE_CACHE_TIME / 1000}`;
+        }
+        let ifModifiedSince = request?.headers["if-modified-since"];
+        if (typeof ifModifiedSince === "string") {
+            let since = new Date(ifModifiedSince).getTime();
+            // Last-Modified is served at 1 second resolution, so compare at that resolution
+            if (since && Math.floor(info.writeTime / 1000) * 1000 <= since) {
+                return setHTTPResultHeaders(Buffer.from(""), { ...headers, status: "304" });
+            }
+        }
+        let range: { start: number; end: number } | undefined;
+        let rangeHeader = request?.headers["range"];
+        if (typeof rangeHeader === "string") {
+            // Single-range form only (bytes=start-end / start- / -suffix); anything else serves the full file
+            let match = /^bytes=(\d*)-(\d*)$/.exec(rangeHeader.trim());
+            if (match && (match[1] || match[2])) {
+                let start: number;
+                let endInclusive = info.size - 1;
+                if (!match[1]) {
+                    start = Math.max(0, info.size - +match[2]);
+                } else {
+                    start = +match[1];
+                    if (match[2]) {
+                        endInclusive = Math.min(+match[2], info.size - 1);
+                    }
+                }
+                if (start >= info.size || start > endInclusive) {
+                    return setHTTPResultHeaders(Buffer.from(""), { ...headers, "Content-Range": `bytes */${info.size}`, status: "416" });
+                }
+                range = { start, end: endInclusive + 1 };
+            }
+        }
+        let result = await bucket.store.get2(filePath, { range });
+        if (!result) {
+            return setHTTPResultHeaders(Buffer.from(""), { status: "404" });
+        }
+        if (range) {
+            return setHTTPResultHeaders(result.data, { ...headers, "Content-Range": `bytes ${range.start}-${range.end - 1}/${info.size}`, status: "206" });
+        }
+        return setHTTPResultHeaders(result.data, headers);
     }
 }
 
 const largeUploadInfo = new Map<string, { account: string; bucketName: string; path: string }>();
 
+// Access checks run as hooks on the register shape below, keyed off the call's arguments, so the
+// method bodies don't each repeat them
+const accountAccess: SocketFunctionHook = async (context) => {
+    await requireAccess(String(context.call.args[0]));
+};
+const uploadAccess: SocketFunctionHook = async (context) => {
+    let info = largeUploadInfo.get(String(context.call.args[0]));
+    // Unknown upload ids are handled by the methods themselves (throw / no-op)
+    if (!info) return;
+    await requireAccess(info.account);
+};
+const adminAccess: SocketFunctionHook = async () => {
+    requireAdmin();
+};
+
 export const RemoteStorageController = SocketFunction.register(
     REMOTE_STORAGE_CLASS_GUID,
     new RemoteStorageControllerBase(),
     () => ({
+        ping: {},
         authenticate: {},
         requestAccess: {},
         getAccessState: {},
-        listRequestsForIP: {},
+        listRequestsForIP: { hooks: [accountAccess] },
         grantAccess: {},
-        adminListRequests: {},
-        adminGrantAccess: {},
-        get: {},
-        get2: {},
-        set: {},
-        del: {},
-        getInfo: {},
-        findInfo: {},
-        getChangesAfter: {},
-        getArchivesConfig: {},
-        getSyncStatus: {},
-        startLargeFile: {},
-        uploadPart: {},
-        finishLargeFile: {},
-        cancelLargeFile: {},
+        adminListRequests: { hooks: [adminAccess] },
+        adminGrantAccess: { hooks: [adminAccess] },
+        get: { hooks: [accountAccess] },
+        get2: { hooks: [accountAccess] },
+        set: { hooks: [accountAccess] },
+        del: { hooks: [accountAccess] },
+        getInfo: { hooks: [accountAccess] },
+        findInfo: { hooks: [accountAccess] },
+        getChangesAfter: { hooks: [accountAccess] },
+        getArchivesConfig: { hooks: [accountAccess] },
+        getIndexInfo: { hooks: [accountAccess] },
+        getSyncStatus: { hooks: [accountAccess] },
+        startLargeFile: { hooks: [accountAccess] },
+        uploadPart: { hooks: [uploadAccess] },
+        finishLargeFile: { hooks: [uploadAccess] },
+        cancelLargeFile: { hooks: [uploadAccess] },
         httpEntry: {},
+    }),
+    () => ({
+        noClientHooks: true,
+        noDefaultHooks: true,
     })
 );

package/storage/remoteStorage/storageServer.d.ts

@@ -2,11 +2,6 @@ import "./accessPage";
 export type HostStorageServerConfig = {
     url: string;
     folder: string;
-    cloudflareApiToken?: {
-        key: string;
-    } | {
-        path: string;
-    };
     lowSpaceThresholdBytes?: number;
 };
 export declare function hostStorageServer(config: HostStorageServerConfig): Promise<void>;

package/storage/remoteStorage/storageServer.ts

@@ -5,7 +5,6 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { getExternalIP } from "socket-function/src/networking";
 import { RequireController } from "socket-function/require/RequireController";
 import { hostServer } from "../../misc/https/hostServer";
-import { getSecret } from "../../misc/getSecret";
 import { RemoteStorageController } from "./storageController";
 import { setStorageServerConfig, setWritesRejectedReason } from "./storageServerState";
 import { parseStorageUrl } from "./ArchivesRemote";
@@ -23,14 +22,13 @@ const HARD_REJECT_FRACTION = 0.1;
 // grantAccess.js bootstrap (next to this file) is what the access page's shown SSH command points at.
 
 export type HostStorageServerConfig = {
-    // Full URL of this storage server, e.g. "https://storage.example.com:4444". The domain and
-    // port are extracted from it (bucket routing URLs clients use look like
-    // https://storage.example.com:4444/file/<account>/<bucketName>/storage/storagerouting.json).
+    // Full URL of this storage server, e.g. "https://1-2-3-4.example.com:4444". The subdomain must
+    // be an ip domain - this machine's external IP with dashes, or 127-0-0-1 for local testing
+    // (see the validation in hostStorageServer). The domain and port are extracted from it (bucket
+    // routing URLs clients use look like
+    // https://1-2-3-4.example.com:4444/file/<account>/<bucketName>/storage/storagerouting.json).
     url: string;
     folder: string;
-    // Set hostServer.ts:HostServerConfig:cloudflareApiToken. Defaults to getSecret("cloudflare.json")
-    // (~/cloudflare.json, { key: string }).
-    cloudflareApiToken?: { key: string } | { path: string };
     // When free space on the folder's drive drops below this many bytes, the server console.errors
     // every 15 minutes. Below 10% of it, the server also rejects write operations (creating files,
     // large uploads, new buckets) — reads, findInfo, and deletes still work so the user can free
@@ -79,12 +77,22 @@ function getGrantAccessCliPath(): string {
 export async function hostStorageServer(config: HostStorageServerConfig): Promise<void> {
     let { url, folder } = config;
     let { address: domain, port } = parseStorageUrl(url);
+    let rootDomain = domain.split(".").slice(-2).join(".");
+    let externalIP = (await getExternalIP()).trim();
+    // The subdomain must be an ip domain: the domain's A record points at exactly one machine, so a
+    // dynamic name would let the same script run on two servers and silently fight over it. Encoding
+    // the IP makes that mistake fail loudly - the wrong machine's domain just doesn't match.
+    let allowedDomains = [`127-0-0-1.${rootDomain}`, `${externalIP.replaceAll(".", "-")}.${rootDomain}`];
+    if (!allowedDomains.includes(domain)) {
+        throw new Error(`The storage server domain is based on the machine's IP (the subdomain is the IP with dots replaced by dashes). Expected ${allowedDomains.join(" or ")}, was ${domain}. Your external IP is ${externalIP}.`);
+    }
+    await fsp.mkdir(folder, { recursive: true });
     let lowSpaceThreshold = config.lowSpaceThresholdBytes ?? DEFAULT_LOW_SPACE_THRESHOLD_BYTES;
     setStorageServerConfig({
         domain,
         port,
-        rootDomain: domain.split(".").slice(-2).join("."),
-        sshTarget: `${os.userInfo().username}@${await getExternalIP()}`,
+        rootDomain,
+        sshTarget: `${os.userInfo().username}@${externalIP}`,
         serverCommand: `node ${getGrantAccessCliPath()} --url ${url}`,
         folder: path.resolve(folder),
     });
@@ -113,7 +121,6 @@ export async function hostStorageServer(config: HostStorageServerConfig): Promis
     await hostServer({
         domain,
         port,
-        cloudflareApiToken: config.cloudflareApiToken || { key: await getSecret("cloudflare.json.key") },
         setDNSRecord: true,
     });
 }

package/storage/remoteStorage/storageServerCli.ts

@@ -16,8 +16,6 @@ async function main() {
     if (!url) throw new Error(`--url is required (ex: --url https://storage.example.com:4444)`);
     let folder = getArg("folder");
     if (!folder) throw new Error(`--folder is required (the folder all data is stored in)`);
-    // Optional: hostStorageServer falls back to ~/cloudflare.json
-    let cloudflareApiToken = getArg("cloudflareApiToken");
     let lowSpaceThresholdBytes: number | undefined;
     let lowSpaceThreshold = getArg("lowSpaceThreshold");
     if (lowSpaceThreshold) {
@@ -30,7 +28,6 @@ async function main() {
     await hostStorageServer({
         url,
         folder,
-        cloudflareApiToken: cloudflareApiToken && { path: cloudflareApiToken } || undefined,
         lowSpaceThresholdBytes,
     });
 }

package/storage/remoteStorage/storageServerState.d.ts

@@ -20,16 +20,22 @@ export declare function getWritesRejectedReason(): string | undefined;
 export declare function assertWritesAllowed(): void;
 export declare function getTrust(): Promise<IStorage<TrustRecord>>;
 export declare function getRequests(): Promise<IStorage<AccessRequest[]>>;
+/** Makes machineIds the complete trust list for the account: machines not in the list lose access, machines already trusted keep their existing record, and missing ones are added. */
+export declare function setTrustedMachines(config: {
+    account: string;
+    machineIds: string[];
+}): Promise<void>;
 export type LoadedBucket = {
     account: string;
     bucketName: string;
     routing: RemoteConfig;
     routingJSON: string;
+    selfEntries: HostedConfig[];
     self: HostedConfig | undefined;
     store: IBucketStore;
 };
 export declare function getLoadedBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined>;
-export declare function assertMutable(bucket: LoadedBucket, filePath: string): Promise<void>;
+export declare function assertMutable(bucket: LoadedBucket, filePath: string, writeTime: number): Promise<void>;
 export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
     lastModified?: number;
 }): Promise<void>;