sliftutils 1.7.4 → 1.7.6

package/storage/remoteStorage/ArchivesRemote.ts

@@ -0,0 +1,195 @@
+module.allowclient = true;
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { timeInMinute } from "socket-function/src/misc";
+import { delay } from "socket-function/src/batching";
+import { getIdentityCA, loadIdentityCA, sign } from "../../misc/https/certs";
+import { IArchives, ArchiveFileInfo } from "../IArchives";
+import {
+    RemoteStorageController, REMOTE_STORAGE_CLASS_GUID, STORAGE_AUTH_PURPOSE,
+    STORAGE_NOT_AUTHENTICATED, STORAGE_ACCESS_DENIED,
+} from "./storageController";
+
+// A bucket on our remote storage server (storageServer.ts), used like ArchivesBackblaze. Works in
+// Node.js and the browser. Authenticates with this machine's certs.ts identity; if the account
+// hasn't trusted this machine yet it requests access, waits, and logs instructions every minute
+// (calls block until access is granted).
+
+const ACCESS_RETRY_DELAY = 1000 * 30;
+const LARGE_FILE_PART_SIZE = 8 * 1024 * 1024;
+
+export type ArchivesRemoteConfig = {
+    address: string;
+    port: number;
+    account: string;
+    bucketName: string;
+    // Public buckets are served over plain HTTPS GETs (getURL). Private buckets are API-access only.
+    public?: boolean;
+    // Fast mode: the server acknowledges writes once they are in memory, flushing to disk after
+    // writeDelay (default 5 minutes) and coalescing writes to the same file. A server crash loses
+    // writes that haven't flushed yet.
+    fast?: boolean;
+    writeDelay?: number;
+};
+
+export function buildPublicFileURL(config: { address: string; port: number; account: string; bucketName: string; path: string }): string {
+    let args = encodeURIComponent(JSON.stringify([config.account, config.bucketName, config.path]));
+    return `https://${config.address}:${config.port}/?classGuid=${REMOTE_STORAGE_CLASS_GUID}&functionName=getPublicFile&args=${args}`;
+}
+
+// Authenticates a connection to a storage server with this machine's certs.ts identity
+export async function authenticateStorage(config: { address: string; port: number; nodeId: string }): Promise<{ machineId: string; ip: string }> {
+    // hostServer nodeIds are machine-specific, so connections by domain must target "any server
+    // at this address" (which is how browsers always connect)
+    SocketFunction.ENABLE_CLIENT_MODE = true;
+    let rootDomain = config.address.split(".").slice(-2).join(".");
+    await loadIdentityCA(rootDomain);
+    let ca = getIdentityCA(rootDomain);
+    let time = Date.now();
+    let signature = sign({ key: ca.key }, {
+        purpose: STORAGE_AUTH_PURPOSE,
+        time,
+        server: `${config.address}:${config.port}`,
+    });
+    return await RemoteStorageController.nodes[config.nodeId].authenticate({ certPem: ca.cert.toString(), time, signature });
+}
+
+export class ArchivesRemote implements IArchives {
+    constructor(private config: ArchivesRemoteConfig) {
+        // hostServer nodeIds are machine-specific, so connections by domain must target "any
+        // server at this address" (which is how browsers always connect)
+        SocketFunction.ENABLE_CLIENT_MODE = true;
+    }
+
+    private nodeId = SocketFunction.connect({ address: this.config.address, port: this.config.port });
+    private controller = RemoteStorageController.nodes[this.nodeId];
+    private setupDone = false;
+    private lastDeniedLog = 0;
+
+    public getDebugName() {
+        return `remoteStorage/${this.config.address}:${this.config.port}/${this.config.account}/${this.config.bucketName}`;
+    }
+
+    private async authenticate(): Promise<void> {
+        await authenticateStorage({ address: this.config.address, port: this.config.port, nodeId: this.nodeId });
+    }
+
+    // Runs a call, authenticating (and re-authenticating after reconnects) as needed. Unlike
+    // call(), does NOT wait for account access.
+    private async callAuthed<T>(fnc: () => Promise<T>): Promise<T> {
+        try {
+            return await fnc();
+        } catch (e: any) {
+            if (!String(e.stack || e).includes(STORAGE_NOT_AUTHENTICATED)) throw e;
+            await this.authenticate();
+            return await fnc();
+        }
+    }
+
+    // Returns undefined if this machine has access to the account. Otherwise puts in an access
+    // request and returns the link to the page where access can be granted.
+    public async waitingForAccess(): Promise<string | undefined> {
+        let state = await this.callAuthed(() => this.controller.getAccessState(this.config.account));
+        if (state.hasAccess) return undefined;
+        await this.callAuthed(() => this.controller.requestAccess(this.config.account));
+        return `https://${this.config.address}:${this.config.port}/${this.config.account}`;
+    }
+
+    private async onAccessDenied(): Promise<void> {
+        let requested = await this.callAuthed(() => this.controller.requestAccess(this.config.account));
+        if (Date.now() - this.lastDeniedLog > timeInMinute) {
+            this.lastDeniedLog = Date.now();
+            console.log(`No access to storage account ${JSON.stringify(this.config.account)} on ${this.config.address}:${this.config.port} (our machine ${requested.machineId}, ip ${requested.ip}). Waiting for access to be granted. See https://${this.config.address}:${this.config.port}/${this.config.account} - or grant it with: ${requested.grantAccessCommand}`);
+        }
+        await delay(ACCESS_RETRY_DELAY);
+    }
+
+    private async ensureSetup(): Promise<void> {
+        if (this.setupDone) return;
+        await this.controller.ensureBucket(this.config.account, this.config.bucketName, {
+            public: this.config.public,
+            fast: this.config.fast,
+            writeDelay: this.config.writeDelay,
+        });
+        this.setupDone = true;
+    }
+
+    // Runs a call, authenticating (and re-authenticating after reconnects) and waiting for account
+    // access as needed.
+    private async call<T>(fnc: () => Promise<T>): Promise<T> {
+        while (true) {
+            try {
+                await this.ensureSetup();
+                return await fnc();
+            } catch (e: any) {
+                let message = String(e.stack || e);
+                if (message.includes(STORAGE_NOT_AUTHENTICATED)) {
+                    this.setupDone = false;
+                    await this.authenticate();
+                    continue;
+                }
+                if (message.includes(STORAGE_ACCESS_DENIED)) {
+                    this.setupDone = false;
+                    await this.onAccessDenied();
+                    continue;
+                }
+                throw e;
+            }
+        }
+    }
+
+    public async get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
+        let result = await this.call(() => this.controller.get(this.config.account, this.config.bucketName, fileName, config?.range));
+        return result && Buffer.from(result) || undefined;
+    }
+    public async set(fileName: string, data: Buffer): Promise<void> {
+        await this.call(() => this.controller.set(this.config.account, this.config.bucketName, fileName, data));
+    }
+    public async del(fileName: string): Promise<void> {
+        await this.call(() => this.controller.del(this.config.account, this.config.bucketName, fileName));
+    }
+    public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
+        return await this.call(() => this.controller.getInfo(this.config.account, this.config.bucketName, fileName));
+    }
+    public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
+        return await this.call(() => this.controller.findInfo(this.config.account, this.config.bucketName, prefix, config));
+    }
+    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
+        return (await this.findInfo(prefix, config)).map(x => x.path);
+    }
+
+    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
+        // Ensure we're authenticated with access BEFORE consuming any data (the stream cannot be
+        // rewound, so we can't use the retry loop around the actual upload)
+        await this.call(() => this.controller.getInfo(this.config.account, this.config.bucketName, config.path));
+        let uploadId = await this.controller.startLargeFile(this.config.account, this.config.bucketName, config.path);
+        try {
+            while (true) {
+                let data = await config.getNextData();
+                if (!data) break;
+                for (let offset = 0; offset < data.length; offset += LARGE_FILE_PART_SIZE) {
+                    await this.controller.uploadPart(uploadId, data.subarray(offset, offset + LARGE_FILE_PART_SIZE));
+                }
+            }
+            await this.controller.finishLargeFile(uploadId);
+        } catch (e) {
+            try {
+                await this.controller.cancelLargeFile(uploadId);
+            } catch { }
+            throw e;
+        }
+    }
+
+    public async getURL(path: string): Promise<string> {
+        if (!this.config.public) {
+            throw new Error(`getURL only works on public buckets (private buckets are API-access only). Bucket: ${this.getDebugName()}`);
+        }
+        return buildPublicFileURL({
+            address: this.config.address,
+            port: this.config.port,
+            account: this.config.account,
+            bucketName: this.config.bucketName,
+            path,
+        });
+    }
+}

package/storage/remoteStorage/accessPage.d.ts

@@ -0,0 +1 @@
+export {};

package/storage/remoteStorage/accessPage.tsx

@@ -0,0 +1,133 @@
+module.allowclient = true;
+
+import preact from "preact";
+import { observable } from "mobx";
+import { observer } from "../../render-utils/observer";
+import { isNode } from "socket-function/src/misc";
+import { css } from "typesafecss";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { RemoteStorageController, AccessState } from "./storageController";
+import { authenticateStorage } from "./ArchivesRemote";
+
+// The storage server's access page. Visit https://<storageDomain>:<port>/<accountName> to request
+// access to that account for this browser's machine identity. Shows the command to run on the
+// storage machine to grant it, and once granted, the machines with (or requesting) access.
+
+const REFRESH_INTERVAL = 1000 * 15;
+const COPIED_RESET_DELAY = 2000;
+
+@observer
+class CopyableCommand extends preact.Component<{ command: string }> {
+    synced = observable({
+        copied: false,
+    });
+
+    render() {
+        let command = this.props.command;
+        return <div className={css.hbox(8).alignItems("flex-start")}>
+            <button onClick={async () => {
+                await navigator.clipboard.writeText(command);
+                this.synced.copied = true;
+                setTimeout(() => { this.synced.copied = false; }, COPIED_RESET_DELAY);
+            }}>
+                {this.synced.copied && "Copied!" || "Copy"}
+            </button>
+            <code className={css.fontFamily("monospace").whiteSpace("pre-wrap")
+                .hsl(0, 0, 92).pad2(8, 4).borderRadius(4)
+            }>
+                {command}
+            </code>
+        </div>;
+    }
+}
+
+@observer
+class AccessPage extends preact.Component {
+    synced = observable({
+        account: "",
+        state: undefined as AccessState | undefined,
+        error: "",
+    });
+
+    componentDidMount() {
+        let account = decodeURIComponent(location.pathname.split("/")[1] || "");
+        this.synced.account = account;
+        if (!account) return;
+        void (async () => {
+            while (true) {
+                try {
+                    await this.refresh(account);
+                } catch (e) {
+                    this.synced.error = String(e);
+                }
+                if (this.synced.state?.hasAccess) break;
+                await new Promise(resolve => setTimeout(resolve, REFRESH_INTERVAL));
+            }
+        })();
+    }
+
+    private authenticated = false;
+    private async refresh(account: string) {
+        let address = location.hostname;
+        let port = +location.port || 443;
+        let nodeId = SocketFunction.connect({ address, port });
+        let controller = RemoteStorageController.nodes[nodeId];
+        if (!this.authenticated) {
+            await authenticateStorage({ address, port, nodeId });
+            this.authenticated = true;
+        }
+        await controller.requestAccess(account);
+        this.synced.state = await controller.getAccessState(account);
+        this.synced.error = "";
+    }
+
+    render() {
+        let { account, state, error } = this.synced;
+        if (!account) {
+            return <div className={css.vbox(8).pad2(16)}>
+                <div>Remote storage server.</div>
+                <div>Visit /(account name) to request access to an account for this browser.</div>
+            </div>;
+        }
+        return <div className={css.vbox(12).pad2(16)}>
+            <div>Storage account: {account}</div>
+            {error && <div>Error: {error}</div>}
+            {!state && !error && <div>Requesting access...</div>}
+            {state && !state.hasAccess && <div className={css.vbox(8)}>
+                <div>This machine ({state.machineId}, ip {state.ip}) does NOT have access yet.</div>
+                <div>An access request has been made. To grant it, run this single command (it sshes into the storage machine):</div>
+                <CopyableCommand command={state.grantAccessCommand || state.listAccessCommand} />
+                <div>This page rechecks every {REFRESH_INTERVAL / 1000} seconds.</div>
+            </div>}
+            {state && state.hasAccess && <div className={css.vbox(8)}>
+                <div>This machine ({state.machineId}, ip {state.ip}) has access.</div>
+                <div>Machines with or requesting access:</div>
+                <table>
+                    <thead>
+                        <tr>
+                            <th>Machine</th>
+                            <th>IP</th>
+                            <th>Time</th>
+                            <th>Status</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {(state.machines || []).map(machine => <tr key={machine.machineId}>
+                            <td className={css.pad2(8, 2)}>{machine.machineId}</td>
+                            <td className={css.pad2(8, 2)}>{machine.ip}</td>
+                            <td className={css.pad2(8, 2)}>{new Date(machine.time).toLocaleString()}</td>
+                            <td className={css.pad2(8, 2)}>{machine.trusted && "has access" || "requesting access"}</td>
+                        </tr>)}
+                    </tbody>
+                </table>
+            </div>}
+        </div>;
+    }
+}
+
+async function main() {
+    if (isNode()) return;
+    preact.render(<AccessPage />, document.body);
+}
+
+main().catch(console.error);

package/storage/remoteStorage/blobStore.d.ts

@@ -0,0 +1,56 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { ArchiveFileInfo } from "../IArchives";
+export declare const DEFAULT_FAST_WRITE_DELAY: number;
+export type WriteConfig = {
+    fast?: boolean;
+    writeDelay?: number;
+};
+export declare class BlobStore {
+    private folder;
+    constructor(folder: string);
+    private memCache;
+    private overlay;
+    private writeQueue;
+    private openBlobs;
+    private largeUploads;
+    private nextLargeUploadId;
+    private currentBlobNumber;
+    private currentBlobOffset;
+    private currentBlobFd;
+    private index;
+    private deadBytes;
+    private blobsDir;
+    init: {
+        (): Promise<void>;
+        reset(): void;
+        set(newValue: Promise<void>): void;
+    };
+    private blobName;
+    private blobPath;
+    private getBlobHandle;
+    private closeBlobHandle;
+    private addDeadBytes;
+    private appendData;
+    private setIndexEntry;
+    set(key: string, data: Buffer, config?: WriteConfig): Promise<void>;
+    del(key: string, config?: WriteConfig): Promise<void>;
+    get(key: string, range?: {
+        start: number;
+        end: number;
+    }): Promise<Buffer | undefined>;
+    getInfo(key: string): Promise<{
+        writeTime: number;
+        size: number;
+    } | undefined>;
+    findInfo(prefix: string, config?: {
+        shallow?: boolean;
+        type?: "files" | "folders";
+    }): Promise<ArchiveFileInfo[]>;
+    startLargeUpload(): Promise<string>;
+    appendLargeUpload(id: string, data: Buffer): Promise<void>;
+    finishLargeUpload(id: string, key: string): Promise<void>;
+    cancelLargeUpload(id: string): Promise<void>;
+    private flushOverlay;
+    private compact;
+}