sliftutils 1.7.8 → 1.7.10

package/storage/remoteStorage/cliArgs.d.ts

@@ -0,0 +1 @@
+export declare function getArg(name: string): string | undefined;

package/storage/remoteStorage/cliArgs.ts

@@ -0,0 +1,9 @@
+export function getArg(name: string): string | undefined {
+    let index = process.argv.indexOf(`--${name}`);
+    if (index < 0) return undefined;
+    let value = process.argv[index + 1];
+    if (!value || value.startsWith("--")) {
+        throw new Error(`Missing value for --${name}`);
+    }
+    return value;
+}

package/storage/remoteStorage/createArchives.d.ts

@@ -0,0 +1,64 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+export declare function createApiArchives(source: HostedConfig | BackblazeConfig): IArchives;
+export declare class ArchivesChain implements IArchives {
+    private normalized;
+    private adopted;
+    private sourcesPromise;
+    constructor(config: RemoteConfig | RemoteConfigBase);
+    getDebugName(): string;
+    private getSourceConfigs;
+    private getSources;
+    private init;
+    private ensureRouting;
+    private readFromSource;
+    private read;
+    private getAccessHelp;
+    private write;
+    waitingForAccess(): Promise<{
+        link: string;
+        machineId: string;
+        ip: string;
+    } | undefined>;
+    get(fileName: string, config?: {
+        range?: {
+            start: number;
+            end: number;
+        };
+    }): Promise<Buffer | undefined>;
+    get2(fileName: string, config?: {
+        range?: {
+            start: number;
+            end: number;
+        };
+    }): Promise<{
+        data: Buffer;
+        writeTime: number;
+    } | undefined>;
+    getInfo(fileName: string): Promise<{
+        writeTime: number;
+        size: number;
+    } | undefined>;
+    find(prefix: string, config?: {
+        shallow?: boolean;
+        type: "files" | "folders";
+    }): Promise<string[]>;
+    findInfo(prefix: string, config?: {
+        shallow?: boolean;
+        type: "files" | "folders";
+    }): Promise<ArchiveFileInfo[]>;
+    getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+    getSyncStatus(): Promise<ArchivesSyncStatus>;
+    getConfig(): Promise<ArchivesConfig>;
+    set(fileName: string, data: Buffer, config?: {
+        lastModified?: number;
+    }): Promise<void>;
+    del(fileName: string): Promise<void>;
+    setLargeFile(config: {
+        path: string;
+        getNextData(): Promise<Buffer | undefined>;
+    }): Promise<void>;
+    getURL(path: string): Promise<string>;
+}
+export declare function createArchives(config: RemoteConfig | RemoteConfigBase): ArchivesChain;

package/storage/remoteStorage/createArchives.ts

@@ -0,0 +1,395 @@
+module.allowclient = true;
+
+import fs from "fs";
+import os from "os";
+import { isNode } from "socket-function/src/misc";
+import {
+    IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig,
+    ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus,
+} from "../IArchives";
+import {
+    ROUTING_FILE, getConfigVersion, getBucketBaseUrl, parseHostedUrl,
+    normalizeRemoteConfig, normalizeSource, parseRoutingData, serializeRemoteConfig,
+} from "./remoteConfig";
+import { ArchivesRemote } from "./ArchivesRemote";
+import { ArchivesUrl } from "./ArchivesUrl";
+import { ArchivesBackblaze } from "../backblaze";
+import { getStorageServerConfigOptional, getLocalArchives } from "./storageServerState";
+import { STORAGE_ACCESS_DENIED } from "./storageController";
+
+// Turns a RemoteConfig into a usable IArchives (createArchives). Setup is fully lazy: on the first
+// call we read the routing file (ROUTING_FILE) from every source, adopt the newest version found
+// (exactly ONE level of indirection - we never fetch routing configs from newly adopted sources),
+// and write the adopted config to every source that is missing it (which CREATES not-yet-existing
+// buckets) or holds an older version.
+
+const DOWN_RETRY_DELAY = 30 * 1000;
+const ENSURE_RETRY_DELAY = 30 * 1000;
+
+// The direct API IArchives for one source, with no URL-form fallback and no chaining. Used by the
+// storage server for its synchronization sources: hosted sources block waiting for account access
+// (logging instructions) when access hasn't been granted yet.
+export function createApiArchives(source: HostedConfig | BackblazeConfig): IArchives {
+    if (source.type === "backblaze") {
+        return new ArchivesBackblaze({ bucketName: source.bucketName, public: source.public, immutable: source.immutable });
+    }
+    let parsed = parseHostedUrl(source.url);
+    let server = isNode() && getStorageServerConfigOptional() || undefined;
+    if (server && parsed.address === server.domain && parsed.port === server.port) {
+        // This source is a bucket hosted by our own process - use it directly instead of calling
+        // ourselves over HTTPS
+        return getLocalArchives(parsed.account, parsed.bucketName);
+    }
+    return new ArchivesRemote({ url: source.url, accountName: source.accountName });
+}
+
+function hasBackblazeCreds(): boolean {
+    return isNode() && fs.existsSync(os.homedir() + "/backblaze.json");
+}
+
+type ChainSource = {
+    config: HostedConfig | BackblazeConfig;
+    // Direct API access. Missing when we can only use the public-URL form (backblaze without
+    // credentials, or backblaze in the browser).
+    api?: IArchives;
+    // Public-URL fallback, used when the API denies us access. Never created when the config
+    // explicitly says public: false (then we don't even hit the URL).
+    url?: ArchivesUrl;
+    // Why writes to this source can never work (collected into the error when a write finds no
+    // source that accepts it)
+    writeBlocked?: string;
+    // After a source errors we stop trying it until this time, so one down source doesn't stall
+    // every call
+    downUntil: number;
+    // Whether we confirmed this source holds an up-to-date routing file. Writing that file creates
+    // the bucket, so this also tracks which buckets we've initialized.
+    routingEnsured: boolean;
+    lastEnsureAttempt: number;
+};
+
+function buildChainSource(config: HostedConfig | BackblazeConfig): ChainSource {
+    let source: ChainSource = { config, downUntil: 0, routingEnsured: false, lastEnsureAttempt: 0 };
+    if (config.type === "backblaze") {
+        if (hasBackblazeCreds()) {
+            source.api = new ArchivesBackblaze({ bucketName: config.bucketName, public: config.public, immutable: config.immutable });
+        } else if (isNode()) {
+            source.writeBlocked = `No backblaze credentials for ${config.url}. Create ~/backblaze.json with { "applicationKeyId": ..., "applicationKey": ... } to enable API access.`;
+        } else {
+            source.writeBlocked = `Browsers cannot write to backblaze (bucket ${config.url})`;
+        }
+        if (!source.api && config.public !== false) {
+            source.url = new ArchivesUrl(getBucketBaseUrl(config.url));
+        }
+        return source;
+    }
+    let parsed = parseHostedUrl(config.url);
+    let server = isNode() && getStorageServerConfigOptional() || undefined;
+    if (server && parsed.address === server.domain && parsed.port === server.port) {
+        // A bucket hosted by our own process - use it directly instead of calling ourselves
+        source.api = getLocalArchives(parsed.account, parsed.bucketName);
+        return source;
+    }
+    source.api = new ArchivesRemote({ url: config.url, accountName: config.accountName, waitForAccess: false });
+    if (config.public !== false) {
+        source.url = new ArchivesUrl(getBucketBaseUrl(config.url));
+    }
+    return source;
+}
+
+export class ArchivesChain implements IArchives {
+    private normalized: RemoteConfig;
+    // The config we operate on: ours, or a newer-versioned one adopted from a source's routing file
+    private adopted: RemoteConfig;
+    private sourcesPromise: Promise<ChainSource[]> | undefined;
+
+    constructor(config: RemoteConfig | RemoteConfigBase) {
+        this.normalized = normalizeRemoteConfig(config);
+        this.adopted = this.normalized;
+    }
+
+    public getDebugName() {
+        let urls = this.adopted.sources.map(x => typeof x === "string" && x || (x as HostedConfig | BackblazeConfig).url);
+        return `chain/${urls.join(",")}`;
+    }
+
+    private getSourceConfigs(config: RemoteConfig): (HostedConfig | BackblazeConfig)[] {
+        return config.sources.map(normalizeSource);
+    }
+
+    private getSources(): Promise<ChainSource[]> {
+        if (!this.sourcesPromise) {
+            let promise = this.init();
+            this.sourcesPromise = promise;
+            // A failed init isn't cached, so a source coming up later fixes us
+            promise.catch(() => {
+                if (this.sourcesPromise === promise) {
+                    this.sourcesPromise = undefined;
+                }
+            });
+        }
+        return this.sourcesPromise;
+    }
+
+    private async init(): Promise<ChainSource[]> {
+        let sources = this.getSourceConfigs(this.normalized).map(buildChainSource);
+        let fetched = await Promise.all(sources.map(async source => {
+            try {
+                let data = await this.readFromSource(source, archives => archives.get(ROUTING_FILE));
+                return data && parseRoutingData(data) || undefined;
+            } catch {
+                // Down or access denied - ensureRouting retries reconciliation for this source later
+                return undefined;
+            }
+        }));
+        let best = this.normalized;
+        for (let config of fetched) {
+            if (config && getConfigVersion(config) > getConfigVersion(best)) {
+                best = config;
+            }
+        }
+        this.adopted = best;
+        if (best !== this.normalized) {
+            // Exactly one level of indirection: the adopted config's sources are used directly,
+            // and we never fetch routing configs from THEM to adopt something newer again
+            sources = this.getSourceConfigs(best).map(buildChainSource);
+            await Promise.all(sources.map(source => this.ensureRouting(source)));
+            return sources;
+        }
+        for (let i = 0; i < sources.length; i++) {
+            let config = fetched[i];
+            if (config && getConfigVersion(config) >= getConfigVersion(best)) {
+                sources[i].routingEnsured = true;
+            }
+        }
+        await Promise.all(sources.map((source, i) => this.ensureRouting(source, { known: { existing: fetched[i] } })));
+        return sources;
+    }
+
+    // Makes sure a source holds our adopted routing config, creating the bucket when it doesn't
+    // exist yet (a bucket exists iff its routing file does) and upgrading older versions. Failures
+    // (no access, source down) are swallowed - we retry periodically, and actual writes surface
+    // the real error.
+    private async ensureRouting(source: ChainSource, config?: { known?: { existing: RemoteConfig | undefined }; force?: boolean }): Promise<void> {
+        if (source.routingEnsured) return;
+        if (!config?.force && source.lastEnsureAttempt !== 0 && Date.now() - source.lastEnsureAttempt < ENSURE_RETRY_DELAY) return;
+        source.lastEnsureAttempt = Date.now();
+        let known = config?.known;
+        try {
+            let existing = known && known.existing;
+            if (!known) {
+                let data = await this.readFromSource(source, archives => archives.get(ROUTING_FILE));
+                existing = data && parseRoutingData(data) || undefined;
+            }
+            if (existing && getConfigVersion(existing) >= getConfigVersion(this.adopted)) {
+                source.routingEnsured = true;
+                return;
+            }
+            if (!source.api) return;
+            await source.api.set(ROUTING_FILE, serializeRemoteConfig(this.adopted));
+            source.routingEnsured = true;
+        } catch { }
+    }
+
+    // Reads from one source, API first, falling back to the public-URL form when the API denies us
+    // access. Access-denied does NOT mark the source down (access can be granted at any moment);
+    // genuine failures do.
+    private async readFromSource<T>(source: ChainSource, run: (archives: IArchives) => Promise<T>): Promise<T> {
+        if (source.api) {
+            try {
+                return await run(source.api);
+            } catch (e) {
+                let denied = String((e as Error).stack || e).includes(STORAGE_ACCESS_DENIED);
+                if (!denied) {
+                    source.downUntil = Date.now() + DOWN_RETRY_DELAY;
+                    throw e;
+                }
+                if (!source.url) throw e;
+            }
+        }
+        if (!source.url) {
+            throw new Error(`Source ${source.config.url} has no API access and no public URL form (public: false)`);
+        }
+        try {
+            return await run(source.url);
+        } catch (e) {
+            source.downUntil = Date.now() + DOWN_RETRY_DELAY;
+            throw e;
+        }
+    }
+
+    // Tries each source in turn, skipping recently-down ones. A source that answers - even with
+    // undefined / empty - is authoritative: sources are synchronized copies of the same bucket, so
+    // a miss on one is a miss everywhere (we don't scan the rest).
+    private async read<T>(config: { apiOnly?: boolean }, run: (archives: IArchives) => Promise<T>): Promise<T> {
+        let sources = await this.getSources();
+        let errors: string[] = [];
+        for (let source of sources) {
+            if (source.downUntil > Date.now()) {
+                errors.push(`${source.config.url} is down, retrying after ${new Date(source.downUntil).toISOString()}`);
+                continue;
+            }
+            // Opportunistic (even on reads): create the bucket / upgrade its routing where needed
+            void this.ensureRouting(source);
+            if (config.apiOnly) {
+                let api = source.api;
+                if (!api) {
+                    errors.push(`${source.config.url} has URL-only access, which cannot serve this operation`);
+                    continue;
+                }
+                try {
+                    return await run(api);
+                } catch (e) {
+                    if (!String((e as Error).stack || e).includes(STORAGE_ACCESS_DENIED)) {
+                        source.downUntil = Date.now() + DOWN_RETRY_DELAY;
+                    }
+                    errors.push(String(e));
+                    continue;
+                }
+            }
+            try {
+                return await this.readFromSource(source, run);
+            } catch (e) {
+                errors.push(String(e));
+            }
+        }
+        throw new Error(`All sources failed for ${this.getDebugName()}: ${errors.join(" | ")}`);
+    }
+
+    private getAccessHelp(source: ChainSource): string {
+        if (source.config.type === "remote") {
+            let parsed = parseHostedUrl(source.config.url);
+            let account = source.config.accountName || parsed.account;
+            return `No write access to account ${JSON.stringify(account)} on ${parsed.address}:${parsed.port}. Visit https://${parsed.address}:${parsed.port}/${account} to grant this machine access.`;
+        }
+        return `Write to ${source.config.url} was denied (check the ~/backblaze.json credentials)`;
+    }
+
+    // Writes go to the first source that accepts them - the storage servers synchronize the
+    // sources between themselves. If no source accepts the write, the error explains how to get
+    // write access (the access page link, or the backblaze secret to create).
+    private async write(run: (archives: IArchives) => Promise<void>): Promise<void> {
+        let sources = await this.getSources();
+        let errors: string[] = [];
+        for (let source of sources) {
+            if (source.config.syncOptions?.noWriteBack) continue;
+            if (source.writeBlocked) {
+                errors.push(source.writeBlocked);
+                continue;
+            }
+            let api = source.api;
+            if (!api) continue;
+            if (source.downUntil > Date.now()) {
+                errors.push(`${source.config.url} is down, retrying after ${new Date(source.downUntil).toISOString()}`);
+                continue;
+            }
+            // The write needs the bucket to exist, so this one is awaited (and never throttled)
+            await this.ensureRouting(source, { force: true });
+            try {
+                return await run(api);
+            } catch (e) {
+                if (String((e as Error).stack || e).includes(STORAGE_ACCESS_DENIED)) {
+                    errors.push(this.getAccessHelp(source));
+                } else {
+                    source.downUntil = Date.now() + DOWN_RETRY_DELAY;
+                    errors.push(String(e));
+                }
+            }
+        }
+        throw new Error(`Write failed on every source of ${this.getDebugName()}: ${errors.join(" | ") || "no sources accept writes"}`);
+    }
+
+    // The access-page link (plus our machineId/ip, for the approver to match the request) for the
+    // first hosted source that hasn't granted us access. undefined when we have access everywhere
+    // we can have it. Also registers the access request server-side.
+    public async waitingForAccess(): Promise<{ link: string; machineId: string; ip: string } | undefined> {
+        let sources = await this.getSources();
+        for (let source of sources) {
+            if (source.api instanceof ArchivesRemote) {
+                let waiting = await source.api.waitingForAccess();
+                if (waiting) return waiting;
+            }
+        }
+        return undefined;
+    }
+
+    public async get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
+        let result = await this.get2(fileName, config);
+        return result && result.data || undefined;
+    }
+    public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined> {
+        return await this.read({}, archives => archives.get2(fileName, config));
+    }
+    public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
+        return await this.read({}, archives => archives.getInfo(fileName));
+    }
+    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
+        return await this.read({ apiOnly: true }, archives => archives.find(prefix, config));
+    }
+    public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
+        return await this.read({ apiOnly: true }, archives => archives.findInfo(prefix, config));
+    }
+    public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
+        return await this.read({ apiOnly: true }, async archives => {
+            if (!archives.getChangesAfter) {
+                throw new Error(`${archives.getDebugName()} does not support getChangesAfter`);
+            }
+            return await archives.getChangesAfter(time);
+        });
+    }
+    public async getSyncStatus(): Promise<ArchivesSyncStatus> {
+        return await this.read({ apiOnly: true }, async archives => {
+            if (!archives.getSyncStatus) {
+                throw new Error(`${archives.getDebugName()} does not support getSyncStatus`);
+            }
+            return await archives.getSyncStatus();
+        });
+    }
+    public async getConfig(): Promise<ArchivesConfig> {
+        let sources = await this.getSources();
+        if (!sources.some(x => x.api)) return {};
+        return await this.read({ apiOnly: true }, archives => archives.getConfig());
+    }
+
+    public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
+        await this.write(archives => archives.set(fileName, data, config));
+    }
+    public async del(fileName: string): Promise<void> {
+        await this.write(archives => archives.del(fileName));
+    }
+    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
+        let sources = await this.getSources();
+        let errors: string[] = [];
+        for (let source of sources) {
+            if (source.config.syncOptions?.noWriteBack) continue;
+            if (source.writeBlocked) {
+                errors.push(source.writeBlocked);
+                continue;
+            }
+            if (!source.api) continue;
+            if (source.downUntil > Date.now()) {
+                errors.push(`${source.config.url} is down, retrying after ${new Date(source.downUntil).toISOString()}`);
+                continue;
+            }
+            await this.ensureRouting(source, { force: true });
+            // The data stream cannot be rewound, so there is no failover to later sources here
+            return await source.api.setLargeFile(config);
+        }
+        throw new Error(`No writable source for setLargeFile on ${this.getDebugName()}: ${errors.join(" | ") || "no sources accept writes"}`);
+    }
+
+    public async getURL(path: string): Promise<string> {
+        let sources = await this.getSources();
+        for (let source of sources) {
+            if (source.config.public === false) continue;
+            if (source.url) return await source.url.getURL(path);
+            if (source.api) return await source.api.getURL(path);
+        }
+        throw new Error(`No public source to build a URL from for ${this.getDebugName()} (every source has public: false)`);
+    }
+}
+
+// The IArchives for a RemoteConfig (or a single source - a routing URL string works). Fully lazy:
+// nothing is contacted until the first call.
+export function createArchives(config: RemoteConfig | RemoteConfigBase): ArchivesChain {
+    return new ArchivesChain(config);
+}

package/storage/remoteStorage/grantAccess.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+"use strict";
+require("typenode");
+require("./grantAccessCli");

package/storage/remoteStorage/grantAccessCli.d.ts

@@ -0,0 +1 @@
+export {};

package/storage/remoteStorage/grantAccessCli.ts

@@ -0,0 +1,27 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { RemoteStorageController } from "./storageController";
+import { authenticateStorage, parseStorageUrl } from "./ArchivesRemote";
+import { getArg } from "./cliArgs";
+
+// The grantAccess CLI. Invoked via the sibling grantAccess.js bootstrap (which loads typenode and
+// then requires this file). Grants a specific access request by its requestId; must be run on the
+// storage machine itself (its certs.ts identity is what the server trusts as admin).
+
+async function main() {
+    let url = getArg("url");
+    if (!url) throw new Error(`--url is required (ex: --url https://storage.example.com:4444)`);
+    let requestId = getArg("requestId");
+    if (!requestId) throw new Error(`--requestId is required (the request id shown on the access page)`);
+
+    let { address, port } = parseStorageUrl(url);
+    let nodeId = SocketFunction.connect({ address, port });
+    await authenticateStorage({ address, port, nodeId });
+    let record = await RemoteStorageController.nodes[nodeId].adminGrantAccess(requestId);
+    console.log(`Granted machine ${record.machineId} access to account ${JSON.stringify(record.account)}`);
+    process.exit(0);
+}
+
+main().catch(e => {
+    console.error(e);
+    process.exit(1);
+});

package/storage/remoteStorage/remoteConfig.d.ts

@@ -0,0 +1,21 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig } from "../IArchives";
+export declare const ROUTING_FILE = "storage/storagerouting.json";
+export declare function getConfigVersion(config: RemoteConfig): number;
+/** Strips the routing-file suffix, leaving the bucket's public base URL (file paths append to it). */
+export declare function getBucketBaseUrl(url: string): string;
+export declare function buildFileUrl(baseUrl: string, filePath: string): string;
+export declare function parseHostedUrl(url: string): {
+    address: string;
+    port: number;
+    account: string;
+    bucketName: string;
+};
+export declare function parseBackblazeUrl(url: string): {
+    bucketName: string;
+};
+export declare function normalizeSource(source: RemoteConfigBase): HostedConfig | BackblazeConfig;
+export declare function normalizeRemoteConfig(config: RemoteConfig | RemoteConfigBase): RemoteConfig;
+export declare function parseRoutingData(data: Buffer): RemoteConfig;
+export declare function serializeRemoteConfig(config: RemoteConfig): Buffer;

package/storage/remoteStorage/remoteConfig.ts

@@ -0,0 +1,94 @@
+module.allowclient = true;
+
+import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig } from "../IArchives";
+
+// Parsing / normalization of RemoteConfig (see IArchives.ts). Every bucket stores its own
+// configuration (a RemoteConfig) inside itself, at ROUTING_FILE. Writing that file creates the
+// bucket / reconfigures it (see storageServerState.ts); clients reconcile it by version (see
+// createArchives.ts).
+
+export const ROUTING_FILE = "storage/storagerouting.json";
+const ROUTING_SUFFIX = "/" + ROUTING_FILE;
+
+// A missing version counts as -1, so any explicitly versioned config beats an unversioned one
+export function getConfigVersion(config: RemoteConfig): number {
+    return config.version ?? -1;
+}
+
+/** Strips the routing-file suffix, leaving the bucket's public base URL (file paths append to it). */
+export function getBucketBaseUrl(url: string): string {
+    if (!url.endsWith(ROUTING_SUFFIX)) {
+        throw new Error(`Expected a bucket routing URL ending with ${JSON.stringify(ROUTING_SUFFIX)}, was ${JSON.stringify(url)}`);
+    }
+    return url.slice(0, -ROUTING_SUFFIX.length);
+}
+
+export function buildFileUrl(baseUrl: string, filePath: string): string {
+    return baseUrl + "/" + filePath.split("/").map(encodeURIComponent).join("/");
+}
+
+// Ex: https://storage2.vidgridweb.com:4445/file/exampleaccount/examplebucket/storage/storagerouting.json
+export function parseHostedUrl(url: string): { address: string; port: number; account: string; bucketName: string } {
+    let base = getBucketBaseUrl(url);
+    let u = new URL(base);
+    if (u.protocol !== "https:") {
+        throw new Error(`Storage URL must use https, got ${JSON.stringify(u.protocol)} in ${JSON.stringify(url)}`);
+    }
+    let parts = u.pathname.split("/").filter(x => x);
+    if (parts.length !== 3 || parts[0] !== "file") {
+        throw new Error(`Expected a hosted bucket URL like https://host:port/file/<account>/<bucketName>${ROUTING_SUFFIX}, was ${JSON.stringify(url)}`);
+    }
+    return { address: u.hostname, port: +u.port || 443, account: decodeURIComponent(parts[1]), bucketName: decodeURIComponent(parts[2]) };
+}
+
+// Ex: https://f002.backblazeb2.com/file/querysubtest-com-public-immutable/storage/storagerouting.json
+export function parseBackblazeUrl(url: string): { bucketName: string } {
+    let base = getBucketBaseUrl(url);
+    let u = new URL(base);
+    let parts = u.pathname.split("/").filter(x => x);
+    if (parts.length !== 2 || parts[0] !== "file") {
+        throw new Error(`Expected a backblaze bucket URL like https://f002.backblazeb2.com/file/<bucketName>${ROUTING_SUFFIX}, was ${JSON.stringify(url)}`);
+    }
+    return { bucketName: decodeURIComponent(parts[1]) };
+}
+
+export function normalizeSource(source: RemoteConfigBase): HostedConfig | BackblazeConfig {
+    if (typeof source !== "string") {
+        if (source.type === "remote") {
+            // Throws if the URL is malformed, so bad configs are rejected before they're stored
+            parseHostedUrl(source.url);
+        }
+        return source;
+    }
+    let hostname = new URL(source).hostname;
+    if (hostname.endsWith(".backblazeb2.com")) {
+        return { type: "backblaze", url: source, bucketName: parseBackblazeUrl(source).bucketName };
+    }
+    parseHostedUrl(source);
+    return { type: "remote", url: source };
+}
+
+export function normalizeRemoteConfig(config: RemoteConfig | RemoteConfigBase): RemoteConfig {
+    if (typeof config !== "string" && "sources" in config) {
+        return { version: config.version, sources: config.sources.map(normalizeSource) };
+    }
+    return { sources: [normalizeSource(config)] };
+}
+
+export function parseRoutingData(data: Buffer): RemoteConfig {
+    let text = data.toString();
+    let parsed: RemoteConfig;
+    try {
+        parsed = JSON.parse(text) as RemoteConfig;
+    } catch (e) {
+        throw new Error(`Routing config is not valid JSON (${String(e)}). Data: ${text.slice(0, 500)}`);
+    }
+    if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.sources)) {
+        throw new Error(`Routing config must be { version?, sources: [...] }, was ${text.slice(0, 500)}`);
+    }
+    return normalizeRemoteConfig(parsed);
+}
+
+export function serializeRemoteConfig(config: RemoteConfig): Buffer {
+    return Buffer.from(JSON.stringify(config, undefined, 4));
+}