sliftutils 1.1.41 → 1.1.43

package/storage/BulkDatabase2/syncWorker.ts

@@ -0,0 +1,65 @@
+import { isNode } from "typesafecss";
+
+// SharedWorker that synchronizes BulkDatabase2 writes between tabs of the same origin. It does NOT
+// persist anything (each tab still writes to disk itself); it just relays writes to other tabs and
+// buffers the recent ones so a freshly-opened tab can catch writes that haven't been flushed to disk
+// yet. Per collection it keeps the latest write per key, pruned to a short recency window (older
+// writes are already on disk, so a new tab gets them from there).
+
+type RemoteWrite = { key: string; time: number; deleted?: boolean; value?: unknown };
+type Collection = { ports: Set<MessagePort>; recent: Map<string, RemoteWrite> };
+
+const RECENT_WINDOW_MS = 60_000;
+const collections = new Map<string, Collection>();
+
+function getCollection(name: string): Collection {
+    let c = collections.get(name);
+    if (!c) {
+        c = { ports: new Set(), recent: new Map() };
+        collections.set(name, c);
+    }
+    return c;
+}
+
+function prune(c: Collection) {
+    let cutoff = Date.now() - RECENT_WINDOW_MS;
+    for (let [key, write] of c.recent) {
+        if (write.time < cutoff) c.recent.delete(key);
+    }
+}
+
+function post(port: MessagePort, message: unknown) {
+    try {
+        port.postMessage(message);
+    } catch {
+        // Port closed (tab gone); it'll be dropped on the next failed send.
+    }
+}
+
+function main() {
+    if (isNode()) return;
+    (self as unknown as { onconnect: (e: MessageEvent) => void }).onconnect = (event: MessageEvent) => {
+        let port = (event as unknown as { ports: MessagePort[] }).ports[0];
+        port.onmessage = (ev: MessageEvent) => {
+            let msg = ev.data as { type: string; collection: string } & RemoteWrite;
+            let c = getCollection(msg.collection);
+            if (msg.type === "hello") {
+                c.ports.add(port);
+                prune(c);
+                port.postMessage({ type: "recent", collection: msg.collection, writes: [...c.recent.values()] });
+            } else if (msg.type === "write") {
+                prune(c);
+                let existing = c.recent.get(msg.key);
+                if (!existing || msg.time > existing.time) {
+                    c.recent.set(msg.key, { key: msg.key, time: msg.time, deleted: msg.deleted, value: msg.value });
+                }
+                for (let other of c.ports) {
+                    if (other !== port) post(other, msg);
+                }
+            }
+        };
+        port.start();
+    };
+}
+
+main();

package/storage/FileFolderAPI.d.ts

@@ -70,6 +70,52 @@ type DirectoryWrapper = {
     ]>;
 };
 export declare function setFileAPIKey(key: string): void;
+export declare class NodeJSFileHandleWrapper implements FileWrapper {
+    private filePath;
+    constructor(filePath: string);
+    getFile(): Promise<{
+        size: number;
+        lastModified: number;
+        arrayBuffer: () => Promise<ArrayBuffer>;
+        slice: (start: number, end: number) => {
+            arrayBuffer: () => Promise<ArrayBuffer>;
+        };
+    }>;
+    createWritable(config?: {
+        keepExistingData?: boolean;
+    }): Promise<{
+        seek: (offset: number) => Promise<void>;
+        write: (value: Buffer) => Promise<void>;
+        close: () => Promise<void>;
+    }>;
+}
+export declare class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
+    private rootPath;
+    constructor(rootPath: string);
+    removeEntry(key: string, options?: {
+        recursive?: boolean;
+    }): Promise<void>;
+    getFileHandle(key: string, options?: {
+        create?: boolean;
+    }): Promise<FileWrapper>;
+    getDirectoryHandle(key: string, options?: {
+        create?: boolean;
+    }): Promise<DirectoryWrapper>;
+    [Symbol.asyncIterator](): AsyncIterableIterator<[
+        string,
+        {
+            kind: "file";
+            name: string;
+            getFile(): Promise<FileWrapper>;
+        } | {
+            kind: "directory";
+            name: string;
+            getDirectoryHandle(key: string, options?: {
+                create?: boolean;
+            }): Promise<DirectoryWrapper>;
+        }
+    ]>;
+}
 export declare const getDirectoryHandle: {
     (): Promise<DirectoryWrapper>;
     reset(): void;
@@ -98,4 +144,6 @@ export type NestedFileStorage = {
 export type FileStorage = IStorageRaw & {
     folder: NestedFileStorage;
 };
+export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
+export declare function tryToLoadPointer(pointer: string): Promise<DirectoryWrapper | undefined>;
 export {};

package/storage/FileFolderAPI.tsx

@@ -96,7 +96,7 @@ class DirectoryPrompter extends preact.Component {
     }
 }
 
-class NodeJSFileHandleWrapper implements FileWrapper {
+export class NodeJSFileHandleWrapper implements FileWrapper {
     constructor(private filePath: string) {
     }
 
@@ -159,7 +159,7 @@ class NodeJSFileHandleWrapper implements FileWrapper {
     }
 }
 
-class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
+export class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
     constructor(private rootPath: string) {
     }
 
@@ -241,6 +241,7 @@ class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
 
 
 // NOTE: Blocks until the user provides a directory
+// NOTE: If you want to override the location, you can explicitly call getDirectoryHandle.set with your own directory wrapper. Which if your server side can be the Node.js directory handle wrapper, or if your client side it can just be a pointer from tryToLoadPointer/fileSystemPointer.ts.
 export const getDirectoryHandle = lazy(async function getDirectoryHandle(): Promise<DirectoryWrapper> {
     if (isNode()) {
         return new NodeJSDirectoryHandleWrapper(path.resolve("./data/"));
@@ -434,40 +435,78 @@ async function fixedGetFileHandle(config: {
     return await config.handle.getFileHandle(config.key, { create: true });
 }
 
+// A file that genuinely does not exist. We must NOT retry these, otherwise reading many missing
+// files (a common pattern) gets catastrophically slow.
+function isMissingError(error: unknown): boolean {
+    let name = (error as { name?: string })?.name;
+    let code = (error as { code?: string })?.code;
+    return name === "NotFoundError" || code === "ENOENT";
+}
+
+const MAX_READ_RETRIES = 6;
+
+// The browser File System Access API can transiently fail reads under heavy load (it appears to
+// throttle when a page issues many reads at once). Those failures surface as errors OTHER than
+// "not found", so we retry them with backoff. A real missing file (NotFoundError / ENOENT) returns
+// undefined immediately with no retry.
+async function readWithRetry<T>(label: string, key: string, read: () => Promise<T>): Promise<T | undefined> {
+    let backoff = 25;
+    for (let attempt = 0; ; attempt++) {
+        try {
+            return await read();
+        } catch (error) {
+            if (isMissingError(error)) return undefined;
+            let name = (error as { name?: string })?.name || "Error";
+            let message = (error as { message?: string })?.message || String(error);
+            if (attempt >= MAX_READ_RETRIES) {
+                console.warn(`${label} gave up on ${JSON.stringify(key)} after ${attempt} retries (${name}): ${message.slice(0, 200)}`);
+                return undefined;
+            }
+            console.warn(`${label} retrying ${JSON.stringify(key)} (attempt ${attempt + 1}, ${name}): ${message.slice(0, 200)}`);
+            await new Promise(resolve => setTimeout(resolve, backoff));
+            backoff = Math.min(backoff * 2, 1000);
+        }
+    }
+}
+
 function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
     return {
         async getInfo(key: string) {
-            try {
+            return readWithRetry("getInfo", key, async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
                 return {
                     size: fileContent.size,
                     lastModified: fileContent.lastModified,
                 };
-            } catch (error) {
-                return undefined;
-            }
+            });
         },
         async get(key: string): Promise<Buffer | undefined> {
-            try {
+            return readWithRetry("get", key, async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
                 const arrayBuffer = await fileContent.arrayBuffer();
+                // Under load the FS Access API can resolve with a truncated buffer and no error.
+                //  Treat a short read as transient so readWithRetry retries it.
+                if (arrayBuffer.byteLength !== fileContent.size) {
+                    throw Object.assign(new Error(`Short read: got ${arrayBuffer.byteLength} of ${fileContent.size} bytes`), { name: "ShortReadError" });
+                }
                 return Buffer.from(arrayBuffer);
-            } catch (error) {
-                return undefined;
-            }
+            });
         },
 
         async getRange(key: string, config: { start: number; end: number }): Promise<Buffer | undefined> {
-            try {
+            return readWithRetry("getRange", key, async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
+                const clampedStart = Math.min(Math.max(config.start, 0), fileContent.size);
+                const clampedEnd = Math.min(Math.max(config.end, clampedStart), fileContent.size);
                 const arrayBuffer = await fileContent.slice(config.start, config.end).arrayBuffer();
+                if (arrayBuffer.byteLength !== clampedEnd - clampedStart) {
+                    throw Object.assign(new Error(`Short range read: got ${arrayBuffer.byteLength} of ${clampedEnd - clampedStart} bytes`), { name: "ShortReadError" });
+                }
                 return Buffer.from(arrayBuffer);
-            } catch (error) {
-                return undefined;
-            }
+            });
         },
 
         async append(key: string, value: Buffer): Promise<void> {
@@ -496,10 +535,18 @@ function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
 
         async getKeys(includeFolders: boolean = false): Promise<string[]> {
             const keys: string[] = [];
-            for await (const [name, entry] of handle) {
-                if (entry.kind === "file" || includeFolders) {
-                    keys.push(entry.name);
+            try {
+                for await (const [name, entry] of handle) {
+                    if (entry.kind === "file" || includeFolders) {
+                        keys.push(entry.name);
+                    }
                 }
+            } catch (error) {
+                let name = (error as { name?: string })?.name || "Error";
+                let message = (error as { message?: string })?.message || String(error);
+                // A failure mid-iteration would silently truncate the listing, so surface it loudly.
+                console.error(`getKeys directory iteration failed after ${keys.length} entries (${name}): ${message.slice(0, 300)}`);
+                throw error;
             }
             return keys;
         },
@@ -544,14 +591,14 @@ function wrapHandleNested(handle: DirectoryWrapper): NestedFileStorage {
     };
 }
 
-function wrapHandle(handle: DirectoryWrapper): FileStorage {
+export function wrapHandle(handle: DirectoryWrapper): FileStorage {
     return {
         ...wrapHandleFiles(handle),
         folder: wrapHandleNested(handle),
     };
 }
 
-async function tryToLoadPointer(pointer: string) {
+export async function tryToLoadPointer(pointer: string) {
     let result = await getFileSystemPointer({ pointer });
     if (!result) return;
     let handle = await result?.onUserActivation();

package/yarn.lock

@@ -1940,10 +1940,10 @@ slash@^3.0.0:
   resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
   integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
 
-socket-function@^1.1.30:
-  version "1.1.30"
-  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.1.30.tgz#adeb9614996e727518e2abdc7a1aa90544d669cd"
-  integrity sha512-EcqNpM8i0icBr2iU80aovXI05aUBrB4P4e9U7EQyju68TGAMagdA4OQXi+V1MHUipH/3nIdw3UAVYv+kFwZwnQ==
+socket-function@^1.1.38:
+  version "1.1.38"
+  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.1.38.tgz#8923a1db19480d5828d9bde9975eabac505698ce"
+  integrity sha512-LJFeVUaXT8D8egyKr7fZajq7ctmLUN58Uih6GHx3/pfXV3pDFfJS0+XJRqsk0udn0II1ZqozEz242khs4fLCKQ==
   dependencies:
     "@types/pako" "^2.0.3"
     "@types/ws" "^8.5.3"