sliftutils 1.4.12 → 1.4.14

package/index.d.ts

@@ -1014,6 +1014,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
             column: string;
             byteSize: number;
         }[]>;
+        getKeyStats(): Promise<{
+            rawKeys: number;
+            finalKeys: number;
+            wastedKeys: number;
+            duplication: number;
+            readers: number;
+        }>;
         getReaderInfo(): Promise<{
             rowCount: number;
             totalBytes: number;
@@ -1364,6 +1371,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
     export type DirectoryWrapper = {
         readonly kind: "directory";
         readonly name: string;
+        readonly fullPath?: string;
         removeEntry(key: string, options?: {
             recursive?: boolean;
         }): Promise<void>;
@@ -1403,6 +1411,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
         constructor(rootPath: string);
         readonly kind: "directory";
         get name(): string;
+        get fullPath(): string;
         entries(): AsyncIterableIterator<[string, FileWrapper | DirectoryWrapper]>;
         removeEntry(key: string, options?: {
             recursive?: boolean;
@@ -1850,6 +1859,7 @@ declare module "sliftutils/storage/remoteFileStorage" {
     export type RemoteOptions = {
         chunkBytes?: number;
         cacheBytes?: number;
+        maxFetchBytes?: number;
         latencyMs?: number;
         stats?: {
             requestCount: number;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.12",
+    "version": "1.4.14",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -117,6 +117,13 @@ export declare class BulkDatabaseBase<T extends {
         column: string;
         byteSize: number;
     }[]>;
+    getKeyStats(): Promise<{
+        rawKeys: number;
+        finalKeys: number;
+        wastedKeys: number;
+        duplication: number;
+        readers: number;
+    }>;
     getReaderInfo(): Promise<{
         rowCount: number;
         totalBytes: number;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -637,7 +637,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
             syncBroadcastSeal(this.name);
             this.streamFileName = undefined;
             const { bulkFiles, streamFiles } = await this.listFiles();
-            if (bulkFiles.length + streamFiles.length >= 1) await this.mergeFileSet(bulkFiles, streamFiles);
+            // compact() merges every file on disk, so nothing older survives outside it — tombstones for
+            // fully-deleted keys can be dropped rather than carried into a fresh carry stream.
+            if (bulkFiles.length + streamFiles.length >= 1) await this.mergeFileSet(bulkFiles, streamFiles, true);
         } finally {
             releaseMergeLock(this.name, writerId);
         }
@@ -662,7 +664,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const selStream = streamFiles.filter(f =>
             f.timestamp <= timeHi && f.timestamp + bulkDatabase2Timing.streamSealAgeMs >= timeLo);
         if (selBulk.length + selStream.length < 2) return;
-        await this.mergeFileSet(selBulk, selStream);
+        // timeLo <= 0 reaches the start of time: every older file overlaps [timeLo, timeHi] and is in the
+        // merge, so no older set survives outside it and surviving tombstones can be dropped.
+        await this.mergeFileSet(selBulk, selStream, timeLo <= 0);
     }
 
     // Throws MissingFileError (not a generic error) when the file is gone, so callers can distinguish a
@@ -804,7 +808,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // merge removes them), never a gap. A bulk file is deleted only if we actually read it; a stream file
     // only if it's aged out (its writer has switched files) or — when cross-tab sync sealed it — its size
     // didn't change while we read it. Returns whether it produced anything.
-    private async mergeFileSet(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[]): Promise<boolean> {
+    // `includesOldest` means this merge consumes every file at or before its time range — there is no
+    // file before it on disk. A surviving tombstone only exists to suppress an OLDER set in some file
+    // outside the merge; if nothing older exists, that older set can't exist either, so the tombstone has
+    // nothing left to suppress and we drop it instead of carrying it forward. (A full compact and a
+    // merge that reaches time 0 are the cases where this holds.)
+    private async mergeFileSet(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false): Promise<boolean> {
         const storage = await this.storage();
         const timestamp = nextFileTime();
         const now = Date.now();
@@ -837,7 +846,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
                 newNames.push(name);
             }
         }
-        if (deletes.size) {
+        // Carry surviving tombstones forward only if older files exist outside this merge that they still
+        // need to suppress; when this merge includes the oldest data there's nothing older to suppress.
+        const carriedDeletes = includesOldest ? 0 : deletes.size;
+        if (carriedDeletes) {
             const carryName = `stream_${Date.now()}_${Math.random().toString(36).slice(2, 10)}${STREAM_EXTENSION}`;
             await storage.set(carryName, frameDeletes([...deletes].map(([key, time]) => ({ time, key }))));
         }
@@ -849,7 +861,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         }
 
         this.resetReader();
-        return newNames.length > 0 || deletes.size > 0;
+        return newNames.length > 0 || carriedDeletes > 0;
     }
 
     // A stream file is safe to delete iff no writer will ever append to it again: it's aged past the seal
@@ -1177,6 +1189,25 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return reader.columns;
     }
 
+    // Raw vs. resolved key counts: how much duplicate/stale key data is sitting on disk that a compact()
+    // would collapse. rawKeys counts every key-slot across all loaded files — each set and each delete
+    // tombstone (a key written into N files counts N times); finalKeys is the number of live resolved
+    // keys (after newest-write-wins and tombstones). Both come straight from the already-loaded reader, so
+    // this is ~free. wastedKeys = rawKeys - finalKeys; duplication = rawKeys / finalKeys (well above 1 ⇒
+    // fragmented, compaction would shrink it).
+    public async getKeyStats(): Promise<{ rawKeys: number; finalKeys: number; wastedKeys: number; duplication: number; readers: number }> {
+        const reader = await this.reader();
+        const rawKeys = reader.rawKeyCount;
+        const finalKeys = reader.keys.length;
+        return {
+            rawKeys,
+            finalKeys,
+            wastedKeys: rawKeys - finalKeys,
+            duplication: finalKeys ? rawKeys / finalKeys : 0,
+            readers: reader.readerCount,
+        };
+    }
+
     public async getReaderInfo() {
         let reader = await this.reader();
         return {
@@ -1212,6 +1243,11 @@ type ResolvedReader = {
     rowCount: number;
     totalBytes: number;
     keys: string[];
+    // Total key-slots across every loaded reader — every set AND every delete tombstone (a key stored in
+    // N files counts N times) — and how many readers there are. Already in memory after the join, so
+    // getKeyStats is ~free; rawKeyCount vs keys.length is how much duplication a compaction would collapse.
+    rawKeyCount: number;
+    readerCount: number;
     columns: { column: string; byteSize: number }[];
     getColumn: (column: string) => Promise<{ key: string; value: unknown; time: number }[]>;
     getSingleField: (key: string, column: string) => Promise<{ value: unknown; time: number } | undefined>;
@@ -1256,6 +1292,8 @@ async function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): Promise<R
         totalBytes: databases.reduce((acc, db) => acc + db.totalBytes, 0),
         rowCount: keys.length,
         keys,
+        rawKeyCount: databases.reduce((acc, db) => acc + db.keyTimes.size + (db.deleteTimes?.size ?? 0), 0),
+        readerCount: databases.length,
         columns,
         async getColumn(column) {
             const perReader = await Promise.all(databases.map(async db => {

package/storage/FileFolderAPI.d.ts

@@ -50,6 +50,7 @@ export type FileWrapper = {
 export type DirectoryWrapper = {
     readonly kind: "directory";
     readonly name: string;
+    readonly fullPath?: string;
     removeEntry(key: string, options?: {
         recursive?: boolean;
     }): Promise<void>;
@@ -89,6 +90,7 @@ export declare class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
     constructor(rootPath: string);
     readonly kind: "directory";
     get name(): string;
+    get fullPath(): string;
     entries(): AsyncIterableIterator<[string, FileWrapper | DirectoryWrapper]>;
     removeEntry(key: string, options?: {
         recursive?: boolean;

package/storage/FileFolderAPI.tsx

@@ -60,6 +60,9 @@ export type FileWrapper = {
 export type DirectoryWrapper = {
     readonly kind: "directory";
     readonly name: string;
+    // Full path from the storage root, for diagnostics/logging (the native handle doesn't expose paths,
+    // so it's optional). e.g. "bulkDatabases2/myCollection".
+    readonly fullPath?: string;
     removeEntry(key: string, options?: { recursive?: boolean }): Promise<void>;
     getFileHandle(key: string, options?: { create?: boolean }): Promise<FileWrapper>;
     getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper>;
@@ -291,6 +294,7 @@ export class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
     }
     readonly kind = "directory" as const;
     get name() { return path.basename(this.rootPath); }
+    get fullPath() { return this.rootPath; }
     entries() { return this[Symbol.asyncIterator](); }
 
     async removeEntry(key: string, options?: { recursive?: boolean }) {
@@ -574,9 +578,11 @@ async function readWithRetry<T>(label: string, key: string, read: () => Promise<
 }
 
 function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
+    // Log the full path (root + filename) when available, so a failing file is identifiable.
+    const pathOf = (key: string) => handle.fullPath ? handle.fullPath + "/" + key : key;
     return {
         async getInfo(key: string) {
-            return readWithRetry("getInfo", key, async () => {
+            return readWithRetry("getInfo", pathOf(key), async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
                 return {
@@ -586,7 +592,7 @@ function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
             });
         },
         async get(key: string): Promise<Buffer | undefined> {
-            return readWithRetry("get", key, async () => {
+            return readWithRetry("get", pathOf(key), async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
                 const arrayBuffer = await fileContent.arrayBuffer();
@@ -600,7 +606,7 @@ function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
         },
 
         async getRange(key: string, config: { start: number; end: number }): Promise<Buffer | undefined> {
-            return readWithRetry("getRange", key, async () => {
+            return readWithRetry("getRange", pathOf(key), async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
                 const clampedStart = Math.min(Math.max(config.start, 0), fileContent.size);

package/storage/remoteFileServer.ts

@@ -7,6 +7,7 @@ import os from "os";
 import { execFileSync } from "child_process";
 import { getExternalIP } from "socket-function/src/networking";
 import { forwardPort } from "socket-function/src/forwardPort";
+import { WebSocketServer, WebSocket } from "ws";
 
 // Remote file server for sliftutils getRemoteFileStorage / BulkDatabase2. Serves one folder on disk over
 // self-signed HTTPS, authenticated with an auto-generated 6-word password (sent as
@@ -400,6 +401,85 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
         }
     });
 
+    // WebSocket transport: same operations as the HTTP handler, but binary-framed and multiplexed over one
+    // socket so large concurrent reads don't pay per-request HTTP overhead. Frame: [u32 headerLen LE][header
+    // JSON][body bytes]. Request header {id, op, path, start?, end?, password?}; response {id, status, error?}.
+    const EMPTY = Buffer.alloc(0);
+    const wss = new WebSocketServer({ server });
+    wss.on("connection", (ws: WebSocket, req: IncomingMessage) => {
+        const ip = clientIp(req);
+        let authed = false;
+        ws.on("message", (data: Buffer) => {
+            let id = 0;
+            const reply = (status: number, extra: object, body?: Buffer) => {
+                const h = Buffer.from(JSON.stringify(Object.assign({ id, status }, extra)), "utf8");
+                const len = Buffer.alloc(4);
+                len.writeUInt32LE(h.length, 0);
+                ws.send(Buffer.concat([len, h, body || EMPTY]));
+            };
+            void (async () => {
+                let header: any;
+                let body: Buffer;
+                try {
+                    const headerLen = data.readUInt32LE(0);
+                    header = JSON.parse(data.subarray(4, 4 + headerLen).toString("utf8"));
+                    body = data.subarray(4 + headerLen);
+                } catch { return; }
+                id = header.id || 0;
+                try {
+                    if (header.op === "auth") {
+                        authed = timingSafeEqualStr(normalizePassword(header.password || ""), normPassword);
+                        return reply(authed ? 200 : 401, {});
+                    }
+                    if (!authed) return reply(401, { error: "unauthorized" });
+                    const rel = String(header.path || "");
+                    const full = safeResolve(root, rel);
+                    if (full === undefined) return reply(403, { error: "path escapes root" });
+                    if (header.op === "info") {
+                        let st: fs.Stats;
+                        try { st = fs.statSync(full); } catch { return reply(404, {}); }
+                        return reply(200, {}, Buffer.from(JSON.stringify({ size: st.size, lastModified: st.mtimeMs, dir: st.isDirectory() })));
+                    }
+                    if (header.op === "list") {
+                        let entries: fs.Dirent[];
+                        try { entries = fs.readdirSync(full, { withFileTypes: true }); }
+                        catch (e) { return (e as NodeJS.ErrnoException).code === "ENOENT" ? reply(200, {}, Buffer.from("[]")) : reply(500, { error: (e as Error).message }); }
+                        return reply(200, {}, Buffer.from(JSON.stringify(entries.filter(d => d.isFile() || d.isDirectory()).map(d => ({ name: d.name, dir: d.isDirectory() })))));
+                    }
+                    if (header.op === "read") {
+                        let st: fs.Stats;
+                        try { st = fs.statSync(full); } catch { return reply(404, {}); }
+                        const start = Math.min(Math.max(Number(header.start) || 0, 0), st.size);
+                        const end = header.end != null ? Math.min(Math.max(Number(header.end), start), st.size) : st.size;
+                        recordAccess(ip, "read", rel, end - start);
+                        if (end === start) return reply(200, {}, EMPTY);
+                        const fh = await fs.promises.open(full, "r");
+                        try {
+                            const buf = Buffer.allocUnsafe(end - start);
+                            const { bytesRead } = await fh.read(buf, 0, end - start, start);
+                            reply(200, {}, bytesRead === buf.length ? buf : buf.subarray(0, bytesRead));
+                        } finally { await fh.close(); }
+                        return;
+                    }
+                    if (header.op === "append" || header.op === "set") {
+                        fs.mkdirSync(path.dirname(full), { recursive: true });
+                        if (header.op === "append") fs.appendFileSync(full, body);
+                        else fs.writeFileSync(full, body);
+                        recordAccess(ip, "write", rel, body.length);
+                        return reply(200, {});
+                    }
+                    if (header.op === "remove") {
+                        try { fs.rmSync(full, { recursive: true, force: true }); } catch (e) { return reply(500, { error: (e as Error).message }); }
+                        return reply(200, {});
+                    }
+                    return reply(404, { error: "unknown op" });
+                } catch (e) {
+                    try { reply(500, { error: String((e as Error)?.message || e) }); } catch { /* socket gone */ }
+                }
+            })();
+        });
+    });
+
     return new Promise<RemoteFileServerHandle>((resolve, reject) => {
         server.on("error", reject);
         server.listen(port, host, () => {
@@ -409,7 +489,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
                 port: actualPort,
                 password,
                 url: `https://localhost:${actualPort}`,
-                close: () => new Promise<void>(r => { if (flushTimer) clearInterval(flushTimer); server.close(() => r()); }),
+                close: () => new Promise<void>(r => { if (flushTimer) clearInterval(flushTimer); for (const c of wss.clients) c.terminate(); wss.close(); server.close(() => r()); }),
             });
         });
     });

package/storage/remoteFileStorage.d.ts

@@ -2,6 +2,7 @@ import type { DirectoryWrapper } from "./FileFolderAPI";
 export type RemoteOptions = {
     chunkBytes?: number;
     cacheBytes?: number;
+    maxFetchBytes?: number;
     latencyMs?: number;
     stats?: {
         requestCount: number;

package/storage/remoteFileStorage.ts

@@ -1,5 +1,4 @@
 import { isNode } from "typesafecss";
-import https from "https";
 import type { DirectoryWrapper, FileWrapper } from "./FileFolderAPI";
 
 // A remote server (remoteFileServer.js) exposed as a DirectoryWrapper — the SAME interface as the
@@ -17,6 +16,7 @@ const DEFAULT_CACHE_BYTES = 128 * 1024 * 1024;
 export type RemoteOptions = {
     chunkBytes?: number;
     cacheBytes?: number;
+    maxFetchBytes?: number;      // cap per read request; large reads split into this many bytes each
     latencyMs?: number;          // artificial per-request delay, for simulating network latency in tests
     stats?: { requestCount: number; bytesFetched: number };
 };
@@ -35,46 +35,185 @@ type Stat = { size: number; lastModified: number; dir: boolean };
 // file, not one per block. Short enough that an append by another writer shows up promptly; our own
 // writes invalidate it immediately.
 const INFO_TTL_MS = 2000;
+// We keep at most this many bytes of read/write requests in flight at once (the rest queue), so a big
+// read doesn't fire hundreds of MB of requests at the server simultaneously.
+const MAX_INFLIGHT_BYTES = 64 * 1024 * 1024;
+// A single read request fetches at most this much; large reads are split into this many concurrent
+// requests (bounded by MAX_INFLIGHT_BYTES).
+const DEFAULT_MAX_FETCH_BYTES = 4 * 1024 * 1024;
+const STATS_LOG_INTERVAL_MS = 10 * 1000;
+const RECV_WINDOW_MS = 60 * 1000;
 
+function fmtBytes(n: number): string {
+    if (n < 1024) return n + "B";
+    if (n < 1024 * 1024) return (n / 1024).toFixed(1) + "KB";
+    if (n < 1024 * 1024 * 1024) return (n / 1024 / 1024).toFixed(1) + "MB";
+    return (n / 1024 / 1024 / 1024).toFixed(2) + "GB";
+}
+
+// Minimal WebSocket surface common to the browser's WebSocket and the `ws` package.
+type WSLike = {
+    readyState: number;
+    binaryType: string;
+    send(data: Uint8Array): void;
+    close(): void;
+    onopen: (() => void) | null;
+    onmessage: ((ev: { data: ArrayBuffer | Buffer }) => void) | null;
+    onclose: (() => void) | null;
+    onerror: ((e: unknown) => void) | null;
+};
+function makeWebSocket(url: string): WSLike {
+    if (isNode()) {
+        // `ws` is a Node dep; the eval hides it from the browser bundler (this branch never runs there).
+        const WS = (eval("require") as NodeRequire)("ws");
+        return new WS(url, { rejectUnauthorized: false }) as WSLike; // self-signed cert
+    }
+    return new WebSocket(url) as unknown as WSLike;
+}
+
+// Binary frame: [u32 headerLen LE][header JSON][body bytes].
+function encodeFrame(header: object, body?: Buffer): Buffer {
+    const h = Buffer.from(JSON.stringify(header), "utf8");
+    const len = Buffer.alloc(4);
+    len.writeUInt32LE(h.length, 0);
+    return Buffer.concat([len, h, body || EMPTY]);
+}
+function decodeFrame(buf: Buffer): { header: any; body: Buffer } {
+    const headerLen = buf.readUInt32LE(0);
+    const header = JSON.parse(buf.subarray(4, 4 + headerLen).toString("utf8"));
+    return { header, body: buf.subarray(4 + headerLen) };
+}
+
+type QueuedRequest = { id: number; frame: Buffer; bytes: number; resolve: (r: { status: number; body: Buffer }) => void; reject: (e: Error) => void };
+
+// One WebSocket to a remote server, multiplexing all ops over it. Requests are queued and only sent
+// while < MAX_INFLIGHT_BYTES are outstanding; every 10s (when there's traffic) it logs the queue depth
+// and throughput.
 class Connection {
-    private agent = isNode() ? new https.Agent({ rejectUnauthorized: false }) : undefined;
     private cache: RangeCache;
     private infoCache = new Map<string, { stat: Stat | undefined; at: number }>();
+
+    private ws: WSLike | undefined;
+    private connecting: Promise<void> | undefined;
+    private nextId = 1;
+    private pending = new Map<number, { resolve: (r: { status: number; body: Buffer }) => void; reject: (e: Error) => void; bytes: number }>();
+    private queue: QueuedRequest[] = [];
+    private inFlightBytes = 0;
+
+    private receivedTotal = 0;
+    private recvWindow: { t: number; n: number }[] = [];
+    private wsUrl: string;
+    private host: string;
+    private statsTimer: ReturnType<typeof setInterval>;
+
     constructor(public url: string, public password: string, private opts: RemoteOptions) {
-        this.cache = new RangeCache(opts.chunkBytes || DEFAULT_CHUNK_BYTES, opts.cacheBytes || DEFAULT_CACHE_BYTES);
         this.url = url.replace(/\/+$/, "");
+        this.wsUrl = this.url.replace(/^http/, "ws"); // https→wss, http→ws
+        this.host = (() => { try { return new URL(this.url).host; } catch { return this.url; } })();
+        this.cache = new RangeCache(opts.chunkBytes || DEFAULT_CHUNK_BYTES, opts.cacheBytes || DEFAULT_CACHE_BYTES, opts.maxFetchBytes || DEFAULT_MAX_FETCH_BYTES);
+        this.statsTimer = setInterval(() => this.logStats(), STATS_LOG_INTERVAL_MS);
+        (this.statsTimer as { unref?: () => void }).unref?.();
     }
 
-    // One HTTP request. Node uses the https module (self-signed cert → verification disabled); the
-    // browser uses fetch (the user has accepted the cert). Returns the raw status + body bytes.
-    async request(method: string, op: string, params: Record<string, string>, body?: Buffer): Promise<{ status: number; body: Buffer }> {
-        if (this.opts.latencyMs) await sleep(this.opts.latencyMs);
-        if (this.opts.stats) this.opts.stats.requestCount++;
-        const qs = new URLSearchParams(params).toString();
-        const fullUrl = this.url + op + (qs ? "?" + qs : "");
-        const headers: Record<string, string> = { authorization: "Bearer " + this.password };
-        if (body) headers["content-type"] = "application/octet-stream";
-        if (isNode()) {
-            return await new Promise((resolve, reject) => {
-                const u = new URL(fullUrl);
-                const req = https.request({ hostname: u.hostname, port: u.port, path: u.pathname + u.search, method, headers, agent: this.agent }, res => {
-                    const chunks: Buffer[] = [];
-                    res.on("data", d => chunks.push(d as Buffer));
-                    res.on("end", () => resolve({ status: res.statusCode || 0, body: Buffer.concat(chunks) }));
-                });
-                req.on("error", reject);
-                if (body) req.write(body);
-                req.end();
-            });
+    close() {
+        clearInterval(this.statsTimer);
+        try { this.ws?.close(); } catch { /* */ }
+        this.ws = undefined;
+        this.connecting = undefined;
+    }
+
+    private logStats() {
+        const now = Date.now();
+        this.recvWindow = this.recvWindow.filter(e => now - e.t < RECV_WINDOW_MS);
+        if (this.inFlightBytes === 0 && this.queue.length === 0) return; // only when there's traffic
+        const recv60 = this.recvWindow.reduce((a, e) => a + e.n, 0);
+        const queuedBytes = this.queue.reduce((a, q) => a + q.bytes, 0);
+        console.log(`[remote ${this.host}] outstanding ${fmtBytes(this.inFlightBytes)} (${this.pending.size} req), queued ${fmtBytes(queuedBytes)} (${this.queue.length} req), recv/60s ${fmtBytes(recv60)}, recv total ${fmtBytes(this.receivedTotal)}`);
+    }
+
+    private ensureConnected(): Promise<void> {
+        if (this.ws && this.ws.readyState === 1) return Promise.resolve();
+        if (this.connecting) return this.connecting;
+        this.connecting = new Promise<void>((resolve, reject) => {
+            let authed = false;
+            const ws = makeWebSocket(this.wsUrl);
+            ws.binaryType = "arraybuffer";
+            this.ws = ws;
+            ws.onopen = () => ws.send(encodeFrame({ id: 0, op: "auth", password: this.password }));
+            ws.onmessage = (ev) => {
+                const buf = Buffer.isBuffer(ev.data) ? ev.data as Buffer : Buffer.from(ev.data as ArrayBuffer);
+                const { header } = decodeFrame(buf);
+                if (!authed && header.id === 0) {
+                    if (header.status === 200) { authed = true; resolve(); }
+                    else { reject(new Error("remote: authentication failed")); try { ws.close(); } catch { /* */ } }
+                    return;
+                }
+                this.onMessage(buf);
+            };
+            ws.onerror = () => { if (!authed) reject(new Error("remote: websocket connection error")); };
+            ws.onclose = () => {
+                if (!authed) reject(new Error("remote: connection closed before authenticating"));
+                this.handleDisconnect();
+            };
+        });
+        // Let a failed connection be retried next time.
+        this.connecting.catch(() => { this.ws = undefined; this.connecting = undefined; });
+        return this.connecting;
+    }
+
+    private onMessage(buf: Buffer) {
+        const { header, body } = decodeFrame(buf);
+        const p = this.pending.get(header.id);
+        if (!p) return;
+        this.pending.delete(header.id);
+        this.inFlightBytes -= p.bytes;
+        this.receivedTotal += body.length;
+        if (this.opts.stats) this.opts.stats.bytesFetched += body.length;
+        this.recvWindow.push({ t: Date.now(), n: body.length });
+        p.resolve({ status: header.status, body });
+        this.drain();
+    }
+
+    private handleDisconnect() {
+        const err = new Error("remote: websocket disconnected");
+        for (const p of this.pending.values()) p.reject(err);
+        for (const q of this.queue) q.reject(err);
+        this.pending.clear();
+        this.queue = [];
+        this.inFlightBytes = 0;
+        this.ws = undefined;
+        this.connecting = undefined;
+    }
+
+    private drain() {
+        while (this.queue.length && this.ws && this.ws.readyState === 1) {
+            const next = this.queue[0];
+            if (this.inFlightBytes > 0 && this.inFlightBytes + next.bytes > MAX_INFLIGHT_BYTES) break; // hold the queue
+            this.queue.shift();
+            this.inFlightBytes += next.bytes;
+            this.pending.set(next.id, { resolve: next.resolve, reject: next.reject, bytes: next.bytes });
+            if (this.opts.stats) this.opts.stats.requestCount++;
+            this.ws.send(next.frame);
         }
-        const res = await fetch(fullUrl, { method, headers, body: body ? new Uint8Array(body) : undefined });
-        return { status: res.status, body: Buffer.from(await res.arrayBuffer()) };
+    }
+
+    // Sends one request over the WebSocket (queued + throttled). `bytes` is the expected payload size,
+    // used for the in-flight cap.
+    private async request(op: string, params: Record<string, unknown>, body: Buffer | undefined, bytes: number): Promise<{ status: number; body: Buffer }> {
+        if (this.opts.latencyMs) await sleep(this.opts.latencyMs);
+        await this.ensureConnected();
+        const id = this.nextId++;
+        const frame = encodeFrame({ id, op, ...params }, body);
+        return new Promise((resolve, reject) => {
+            this.queue.push({ id, frame, bytes: Math.max(bytes, body ? body.length : 0, 1), resolve, reject });
+            this.drain();
+        });
     }
 
     async stat(path: string): Promise<Stat | undefined> {
         const cached = this.infoCache.get(path);
         if (cached && Date.now() - cached.at < INFO_TTL_MS) return cached.stat;
-        const r = await this.request("GET", "/info", { path });
+        const r = await this.request("info", { path }, undefined, 256);
         let stat: Stat | undefined;
         if (r.status === 404) stat = undefined;
         else if (r.status !== 200) throw new Error(`remote info failed (${r.status})`);
@@ -83,7 +222,7 @@ class Connection {
         return stat;
     }
     async list(path: string): Promise<{ name: string; dir: boolean }[]> {
-        const r = await this.request("GET", "/list", { path });
+        const r = await this.request("list", { path }, undefined, 4096);
         if (r.status !== 200) throw new Error(`remote list failed (${r.status})`);
         return JSON.parse(r.body.toString("utf8"));
     }
@@ -91,26 +230,24 @@ class Connection {
         return (await this.cache.read(this, path, start, end)) ?? EMPTY;
     }
     async readServer(path: string, start: number, end: number): Promise<Buffer | undefined> {
-        const r = await this.request("GET", "/read", { path, start: String(start), end: String(end) });
+        const r = await this.request("read", { path, start, end }, undefined, end - start);
         if (r.status === 404) return undefined;
         if (r.status !== 200) throw new Error(`remote read failed (${r.status})`);
-        if (this.opts.stats) this.opts.stats.bytesFetched += r.body.length;
         return r.body;
     }
     async append(path: string, body: Buffer): Promise<void> {
-        const r = await this.request("PUT", "/append", { path }, body);
+        const r = await this.request("append", { path }, body, body.length);
         if (r.status !== 200) throw new Error(`remote append failed (${r.status})`);
-        // Append-only keeps existing bytes, so cached chunks stay valid; just the size changed.
-        this.infoCache.delete(path);
+        this.infoCache.delete(path); // append-only keeps existing bytes; only the size changed
     }
     async set(path: string, body: Buffer): Promise<void> {
-        const r = await this.request("PUT", "/set", { path }, body);
+        const r = await this.request("set", { path }, body, body.length);
         if (r.status !== 200) throw new Error(`remote set failed (${r.status})`);
         this.cache.invalidate(path);
         this.infoCache.delete(path);
     }
     async remove(path: string): Promise<void> {
-        const r = await this.request("DELETE", "/remove", { path });
+        const r = await this.request("remove", { path }, undefined, 256);
         if (r.status !== 200) throw new Error(`remote remove failed (${r.status})`);
         this.cache.invalidate(path);
         this.infoCache.delete(path);
@@ -123,7 +260,7 @@ class Connection {
 class RangeCache {
     private chunks = new Map<string, Buffer>(); // path + "
" + chunkIndex, insertion-ordered (LRU)
     private bytes = 0;
-    constructor(private chunkBytes: number, private budget: number) { }
+    constructor(private chunkBytes: number, private budget: number, private maxFetchBytes: number) { }
     private key(path: string, c: number) { return path + "
" + c; }
     private peek(path: string, c: number): Buffer | undefined {
         const k = this.key(path, c);
@@ -152,24 +289,37 @@ class RangeCache {
     async read(conn: Connection, path: string, start: number, end: number): Promise<Buffer | undefined> {
         if (end <= start) return EMPTY;
         const CHUNK = this.chunkBytes;
+        const maxRunChunks = Math.max(1, Math.floor(this.maxFetchBytes / CHUNK));
         const firstChunk = Math.floor(start / CHUNK);
         const lastChunk = Math.floor((end - 1) / CHUNK);
-        let fetchFrom = -1, fetchTo = -1;
+        // Collect the chunks we don't have deeply enough into bounded contiguous runs (each <= maxFetchBytes).
+        const runs: { from: number; to: number }[] = [];
+        let runFrom = -1;
         for (let c = firstChunk; c <= lastChunk; c++) {
             const cStart = c * CHUNK;
             const needEnd = Math.min(end, cStart + CHUNK) - cStart;
             const have = this.peek(path, c);
-            if (!have || have.length < needEnd) { if (fetchFrom < 0) fetchFrom = c; fetchTo = c; }
+            if (!have || have.length < needEnd) {
+                if (runFrom < 0) runFrom = c;
+                if (c - runFrom + 1 >= maxRunChunks) { runs.push({ from: runFrom, to: c }); runFrom = -1; }
+            } else if (runFrom >= 0) {
+                runs.push({ from: runFrom, to: c - 1 });
+                runFrom = -1;
+            }
         }
-        if (fetchFrom >= 0) {
-            const bytes = await conn.readServer(path, fetchFrom * CHUNK, (fetchTo + 1) * CHUNK);
-            if (bytes === undefined) return undefined;
-            for (let c = fetchFrom; c <= fetchTo; c++) {
-                const off = (c - fetchFrom) * CHUNK;
+        if (runFrom >= 0) runs.push({ from: runFrom, to: lastChunk });
+        // Fetch all runs concurrently; the connection's queue caps total in-flight bytes.
+        let missingFile = false;
+        await Promise.all(runs.map(async run => {
+            const bytes = await conn.readServer(path, run.from * CHUNK, (run.to + 1) * CHUNK);
+            if (bytes === undefined) { missingFile = true; return; }
+            for (let c = run.from; c <= run.to; c++) {
+                const off = (c - run.from) * CHUNK;
                 if (off >= bytes.length) break;
                 this.store(path, c, bytes.subarray(off, Math.min(off + CHUNK, bytes.length)));
             }
-        }
+        }));
+        if (missingFile) return undefined;
         const parts: Buffer[] = [];
         for (let c = firstChunk; c <= lastChunk; c++) {
             const cStart = c * CHUNK;
@@ -228,6 +378,7 @@ class RemoteDirectoryWrapper implements DirectoryWrapper {
     // native API — e.g. recursive walks using `handle.entries()` — works the same over the network.
     readonly kind = "directory" as const;
     get name() { return baseName(this.dirPath); }
+    get fullPath() { return this.dirPath; }
     async removeEntry(key: string): Promise<void> {
         await this.conn.remove(joinPath(this.dirPath, key));
     }
@@ -271,17 +422,19 @@ export function getRemoteDirectoryHandle(url: string, password: string, options:
 
 export type RemoteConnectResult = { status: "ok" } | { status: "unauthorized" } | { status: "unreachable"; error: string };
 
-// Verifies a server is reachable and the password works — by actually listing the root. Distinguishes
-// "connected" / "wrong password" / "couldn't reach it" (the last usually meaning the self-signed cert
-// isn't trusted yet in the browser, since fetch failures carry no detail).
+// Verifies a server is reachable and the password works — by opening the WebSocket, authenticating, and
+// listing the root. Distinguishes "connected" / "wrong password" / "couldn't reach it" (the last usually
+// meaning the self-signed cert isn't trusted yet in the browser, since the socket just fails to open).
 export async function testRemoteConnection(url: string, password: string, options: RemoteOptions = {}): Promise<RemoteConnectResult> {
     const conn = new Connection(url, password, options);
     try {
-        const r = await conn.request("GET", "/list", { path: "" });
-        if (r.status === 200) return { status: "ok" };
-        if (r.status === 401) return { status: "unauthorized" };
-        return { status: "unreachable", error: `server returned ${r.status}` };
+        await conn.list("");
+        return { status: "ok" };
     } catch (e) {
-        return { status: "unreachable", error: (e as Error)?.message || String(e) };
+        const msg = (e as Error)?.message || String(e);
+        if (/auth/i.test(msg)) return { status: "unauthorized" };
+        return { status: "unreachable", error: msg };
+    } finally {
+        conn.close();
     }
 }