sliftutils 1.7.9 → 1.7.11

package/storage/remoteStorage/storageServer.ts

@@ -1,132 +1,126 @@
-process.env.NODE_ENV = "production";
 import os from "os";
 import path from "path";
+import fsp from "fs/promises";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { getExternalIP } from "socket-function/src/networking";
 import { RequireController } from "socket-function/require/RequireController";
 import { hostServer } from "../../misc/https/hostServer";
-import { getFileStorageNested2 } from "../FileFolderAPI";
-import { TransactionStorage } from "../TransactionStorage";
-import { JSONStorage } from "../JSONStorage";
-import { BlobStore } from "./blobStore";
-import {
-    RemoteStorageController, setStorageServerState,
-    AccessRequest, TrustRecord, BucketConfig,
-} from "./storageController";
-import { authenticateStorage } from "./ArchivesRemote";
+import { RemoteStorageController } from "./storageController";
+import { setStorageServerConfig, setWritesRejectedReason } from "./storageServerState";
+import { parseStorageUrl } from "./ArchivesRemote";
 // Import browser code, so it is allowed to be required by the client
 import "./accessPage";
 
-// The remote storage server. Run modes:
-//   Host the server:
-//     typenode storage/remoteStorage/storageServer.ts --domain storage.example.com --port 4444
-//       --folder /storage/data --cloudflareApiTokenPath ~/example.com.key
-//   Admin commands (run on the storage machine itself, against the running server):
-//     --listAccess <ip>            lists pending access requests from that IP (with requestIds)
-//     --grantAccess <requestId>    trusts the machine from that request for the requested account
+const DEFAULT_LOW_SPACE_THRESHOLD_BYTES = 25 * 1024 ** 3;
+const DISK_SPACE_CHECK_INTERVAL_MS = 15 * 60 * 1000;
+// Below this fraction of the warn threshold, we start rejecting writes so the server itself doesn't
+// tip the machine into instability. Reads/deletes still work so users can free space.
+const HARD_REJECT_FRACTION = 0.1;
 
-process.on("unhandledRejection", (error) => {
-    console.error("Unhandled promise rejection:", error);
-});
-process.on("uncaughtException", (error) => {
-    console.error("Uncaught exception:", error);
-});
+// The remote storage server, as a library function: consumers call hostStorageServer() from their
+// own process to start hosting (or use the storageserver bin, see storageServerCli.ts). The
+// grantAccess.js bootstrap (next to this file) is what the access page's shown SSH command points at.
 
-// The absolute command that runs this script (with the given args) on this machine, usable from
-// any working directory (ex: over ssh)
-function getServerCommand(args: string): string {
-    return `${process.execPath} ${require.resolve("typenode/bootstrap.js")} ${__filename} ${args}`;
-}
+export type HostStorageServerConfig = {
+    // Full URL of this storage server, e.g. "https://1-2-3-4.example.com:4444". The subdomain must
+    // be an ip domain - this machine's external IP with dashes, or 127-0-0-1 for local testing
+    // (see the validation in hostStorageServer). The domain and port are extracted from it (bucket
+    // routing URLs clients use look like
+    // https://1-2-3-4.example.com:4444/file/<account>/<bucketName>/storage/storagerouting.json).
+    url: string;
+    folder: string;
+    // When free space on the folder's drive drops below this many bytes, the server console.errors
+    // every 15 minutes. Below 10% of it, the server also rejects write operations (creating files,
+    // large uploads, new buckets) — reads, findInfo, and deletes still work so the user can free
+    // space. Default 25 GiB.
+    lowSpaceThresholdBytes?: number;
+};
 
-function getArg(name: string): string | undefined {
-    let index = process.argv.indexOf(`--${name}`);
-    if (index < 0) return undefined;
-    let value = process.argv[index + 1];
-    if (!value || value.startsWith("--")) {
-        throw new Error(`Missing value for --${name}`);
-    }
-    return value;
+function formatBytes(bytes: number): string {
+    return `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
 }
 
-async function runAdminCommand(config: { domain: string; port: number; listAccess?: string; grantAccess?: string }) {
-    let nodeId = SocketFunction.connect({ address: config.domain, port: config.port });
-    let controller = RemoteStorageController.nodes[nodeId];
-    await authenticateStorage({ address: config.domain, port: config.port, nodeId });
-    if (config.listAccess) {
-        let requests = await controller.adminListRequests(config.listAccess);
-        if (!requests.length) {
-            console.log(`No access requests from ${config.listAccess}`);
-            return;
-        }
-        console.log(`Access requests from ${config.listAccess}:`);
-        for (let request of requests) {
-            console.log(`  --grantAccess ${request.requestId}  (account ${request.account}, machine ${request.machineId}, requested ${new Date(request.time).toISOString()})`);
-        }
-        console.log(`Grant one with: ${getServerCommand(`--domain ${config.domain} --port ${config.port} --grantAccess <requestId>`)}`);
+async function checkDiskSpace(config: { folder: string; threshold: number }): Promise<void> {
+    let { folder, threshold } = config;
+    let stats = await fsp.statfs(folder);
+    let free = Number(stats.bavail) * Number(stats.bsize);
+    let hardLimit = threshold * HARD_REJECT_FRACTION;
+    if (free >= threshold) {
+        setWritesRejectedReason(undefined);
         return;
     }
-    if (config.grantAccess) {
-        let record = await controller.adminGrantAccess(config.grantAccess);
-        console.log(`Granted machine ${record.machineId} access to account ${JSON.stringify(record.account)}`);
+    let under = threshold - free;
+    let rejecting = free < hardLimit;
+    console.error(
+        `Storage folder ${folder} is low on disk: ${formatBytes(free)} free`
+        + ` (warn threshold ${formatBytes(threshold)}, ${formatBytes(under)} under;`
+        + ` hard-reject threshold ${formatBytes(hardLimit)}${rejecting ? ", REACHED — write ops now rejected" : ""}).`
+    );
+    if (rejecting) {
+        setWritesRejectedReason(
+            `Storage server is out of disk space: only ${formatBytes(free)} free on ${folder}`
+            + ` (hard-reject threshold ${formatBytes(hardLimit)}, warn threshold ${formatBytes(threshold)}).`
+            + ` Write operations (create/append/new bucket) are rejected; reads, findInfo, and deletes still work — please free space.`
+        );
+    } else {
+        setWritesRejectedReason(undefined);
     }
 }
 
-async function main() {
-    let domain = getArg("domain");
-    if (!domain) throw new Error(`--domain is required (ex: --domain storage.example.com)`);
-    let port = +(getArg("port") || 443);
+// Full path to the grantAccess CLI bootstrap that lives next to this file. The SSH command shown on
+// the access page invokes it via `node <path> ...` (found through __dirname so consumers don't have
+// to know where our source lives).
+function getGrantAccessCliPath(): string {
+    return path.join(__dirname, "grantAccess.js");
+}
 
-    let listAccess = getArg("listAccess");
-    let grantAccess = getArg("grantAccess");
-    if (listAccess || grantAccess) {
-        await runAdminCommand({ domain, port, listAccess, grantAccess });
-        process.exit(0);
+export async function hostStorageServer(config: HostStorageServerConfig): Promise<void> {
+    let { url, folder } = config;
+    let { address: domain, port } = parseStorageUrl(url);
+    let rootDomain = domain.split(".").slice(-2).join(".");
+    let externalIP = (await getExternalIP()).trim();
+    // The subdomain must be an ip domain: the domain's A record points at exactly one machine, so a
+    // dynamic name would let the same script run on two servers and silently fight over it. Encoding
+    // the IP makes that mistake fail loudly - the wrong machine's domain just doesn't match.
+    let allowedDomains = [`127-0-0-1.${rootDomain}`, `${externalIP.replaceAll(".", "-")}.${rootDomain}`];
+    if (!allowedDomains.includes(domain)) {
+        throw new Error(`The storage server domain is based on the machine's IP (the subdomain is the IP with dots replaced by dashes). Expected ${allowedDomains.join(" or ")}, was ${domain}. Your external IP is ${externalIP}.`);
     }
-
-    let folder = getArg("folder");
-    if (!folder) throw new Error(`--folder is required (where the storage server keeps its data)`);
-    let cloudflareApiTokenPath = getArg("cloudflareApiTokenPath");
-    if (!cloudflareApiTokenPath) throw new Error(`--cloudflareApiTokenPath is required (path to a Cloudflare API token file, for HTTPS certs)`);
-
-    let root = await getFileStorageNested2(path.resolve(folder));
-    let system = await root.folder.getStorage("system");
-    let trust = new JSONStorage<TrustRecord>(new TransactionStorage(await system.folder.getStorage("trust"), "storageTrust"));
-    let requests = new JSONStorage<AccessRequest[]>(new TransactionStorage(await system.folder.getStorage("requests"), "storageRequests"));
-    let buckets = new JSONStorage<BucketConfig>(new TransactionStorage(await system.folder.getStorage("buckets"), "storageBuckets"));
-
-    setStorageServerState({
+    await fsp.mkdir(folder, { recursive: true });
+    let lowSpaceThreshold = config.lowSpaceThresholdBytes ?? DEFAULT_LOW_SPACE_THRESHOLD_BYTES;
+    setStorageServerConfig({
         domain,
         port,
-        rootDomain: domain.split(".").slice(-2).join("."),
-        sshTarget: `${os.userInfo().username}@${await getExternalIP()}`,
-        serverCommand: getServerCommand(`--domain ${domain} --port ${port}`),
-        blobStore: new BlobStore(path.resolve(folder)),
-        trust,
-        requests,
-        buckets,
+        rootDomain,
+        sshTarget: `${os.userInfo().username}@${externalIP}`,
+        serverCommand: `node ${getGrantAccessCliPath()} --url ${url}`,
+        folder: path.resolve(folder),
     });
 
     RequireController.allowAllNodeModules();
     SocketFunction.expose(RequireController);
     SocketFunction.expose(RemoteStorageController);
-    // No static roots, so the access page HTML is served at every path (the path is the account
-    // name, see accessPage.tsx)
+    // Every HTTP path goes through httpEntry: /file/<account>/<bucketName>/... serves public
+    // bucket files, everything else serves the access page (the path is the account name, see
+    // accessPage.tsx).
     // A full URL, so the page resolves modules from the origin root even when served at
-    // /accountName (a relative require would resolve inside the account path)
-    SocketFunction.setDefaultHTTPCall(RequireController, "requireHTML", {
+    // /accountName (a relative require would resolve inside the account path).
+    SocketFunction.setDefaultHTTPCall(RemoteStorageController, "httpEntry", {
         requireCalls: [`https://${domain}:${port}/./storage/remoteStorage/accessPage.tsx`],
     });
 
+    // Initial check so a server starting under-limit immediately rejects writes; then keep checking
+    // every 15 minutes so recovery (freed disk space) is picked up automatically.
+    await checkDiskSpace({ folder, threshold: lowSpaceThreshold });
+    let interval = setInterval(() => {
+        void checkDiskSpace({ folder, threshold: lowSpaceThreshold })
+            .catch(e => console.error(`Disk space check failed for ${folder}:`, e));
+    }, DISK_SPACE_CHECK_INTERVAL_MS);
+    (interval as { unref?: () => void }).unref?.();
+
     await hostServer({
         domain,
         port,
-        cloudflareApiTokenPath,
         setDNSRecord: true,
     });
-    console.log(`Storage server running at https://${domain}:${port}`);
 }
-
-main().catch(e => {
-    console.error(e);
-    process.exit(1);
-});

package/storage/remoteStorage/storageServerCli.d.ts

@@ -0,0 +1 @@
+export {};

package/storage/remoteStorage/storageServerCli.ts

@@ -0,0 +1,38 @@
+process.env.NODE_ENV = "production";
+import { hostStorageServer } from "./storageServer";
+import { getArg } from "./cliArgs";
+
+// Hosts a storage server from the command line (via the storageserver bin, see package.json).
+
+process.on("unhandledRejection", (error) => {
+    console.error("Unhandled promise rejection:", error);
+});
+process.on("uncaughtException", (error) => {
+    console.error("Uncaught exception:", error);
+});
+
+async function main() {
+    let url = getArg("url");
+    if (!url) throw new Error(`--url is required (ex: --url https://storage.example.com:4444)`);
+    let folder = getArg("folder");
+    if (!folder) throw new Error(`--folder is required (the folder all data is stored in)`);
+    let lowSpaceThresholdBytes: number | undefined;
+    let lowSpaceThreshold = getArg("lowSpaceThreshold");
+    if (lowSpaceThreshold) {
+        lowSpaceThresholdBytes = +lowSpaceThreshold;
+        if (isNaN(lowSpaceThresholdBytes)) {
+            throw new Error(`--lowSpaceThreshold must be a number of bytes, was ${JSON.stringify(lowSpaceThreshold)}`);
+        }
+    }
+
+    await hostStorageServer({
+        url,
+        folder,
+        lowSpaceThresholdBytes,
+    });
+}
+
+main().catch(e => {
+    console.error(e);
+    process.exit(1);
+});

package/storage/remoteStorage/storageServerState.d.ts

@@ -0,0 +1,43 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { IBucketStore } from "./blobStore";
+import { RemoteConfig, HostedConfig, IArchives } from "../IArchives";
+import type { IStorage } from "../IStorage";
+import type { AccessRequest, TrustRecord } from "./storageController";
+export type StorageServerConfig = {
+    domain: string;
+    port: number;
+    rootDomain: string;
+    sshTarget: string;
+    serverCommand: string;
+    folder: string;
+};
+export declare function setStorageServerConfig(value: StorageServerConfig): void;
+export declare function getStorageServerConfig(): StorageServerConfig;
+export declare function getStorageServerConfigOptional(): StorageServerConfig | undefined;
+export declare function setWritesRejectedReason(reason: string | undefined): void;
+export declare function getWritesRejectedReason(): string | undefined;
+export declare function assertWritesAllowed(): void;
+export declare function getTrust(): Promise<IStorage<TrustRecord>>;
+export declare function getRequests(): Promise<IStorage<AccessRequest[]>>;
+/** Makes machineIds the complete trust list for the account: machines not in the list lose access, machines already trusted keep their existing record, and missing ones are added. */
+export declare function setTrustedMachines(config: {
+    account: string;
+    machineIds: string[];
+}): Promise<void>;
+export type LoadedBucket = {
+    account: string;
+    bucketName: string;
+    routing: RemoteConfig;
+    routingJSON: string;
+    selfEntries: HostedConfig[];
+    self: HostedConfig | undefined;
+    store: IBucketStore;
+};
+export declare function getLoadedBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined>;
+export declare function assertMutable(bucket: LoadedBucket, filePath: string, writeTime: number): Promise<void>;
+export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
+    lastModified?: number;
+}): Promise<void>;
+export declare function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void>;
+export declare function getLocalArchives(account: string, bucketName: string): IArchives;