sliftutils 1.2.38 → 1.3.0

package/storage/BulkDatabase2/syncClient.ts

@@ -0,0 +1,81 @@
+import { isNode } from "typesafecss";
+
+// Browser-side cross-tab write sync for BulkDatabase2, over BroadcastChannel (one channel per
+// collection, same-origin tabs). It does NOT persist anything — each tab writes to disk itself; this
+// just relays live writes to other open tabs and, when a tab starts up, asks peers for writes they've
+// made recently that may not be on disk yet. No-op in Node / where BroadcastChannel is unavailable, so
+// BulkDatabase2 can call these unconditionally. This is an optional feature, so there is no fallback.
+
+export type RemoteWrite = { key: string; time: number; deleted?: boolean; value?: unknown };
+
+// Writes older than this are assumed already flushed to disk (a freshly-opened tab gets those by
+// reading disk), so a tab only replays writes newer than this when a peer says hello.
+const RECENT_WINDOW_MS = 60_000;
+
+type Channel = {
+    bc: BroadcastChannel;
+    subscribers: ((write: RemoteWrite) => void)[];
+    // This tab's own recent writes, kept so it can answer another tab's "hello".
+    recent: RemoteWrite[];
+};
+
+const channels = new Map<string, Channel>();
+
+export function isSyncSupported(): boolean {
+    return !isNode() && typeof BroadcastChannel !== "undefined";
+}
+
+function pruneRecent(channel: Channel) {
+    const cutoff = Date.now() - RECENT_WINDOW_MS;
+    channel.recent = channel.recent.filter(w => w.time >= cutoff);
+}
+
+function deliver(channel: Channel, write: RemoteWrite) {
+    for (const sub of channel.subscribers) sub(write);
+}
+
+function ensure(collection: string): Channel | undefined {
+    if (!isSyncSupported()) return undefined;
+    let channel = channels.get(collection);
+    if (channel) return channel;
+    // BroadcastChannel never delivers a message back to the instance that sent it, so a tab never
+    // hears its own writes — only the other open tabs do.
+    const bc = new BroadcastChannel(`bulkDatabase2:${collection}`);
+    const created: Channel = { bc, subscribers: [], recent: [] };
+    bc.onmessage = (event: MessageEvent) => {
+        const msg = event.data as { type: string; write?: RemoteWrite; writes?: RemoteWrite[] };
+        if (!msg) return;
+        if (msg.type === "write" && msg.write) {
+            deliver(created, msg.write);
+        } else if (msg.type === "recent" && msg.writes) {
+            for (const write of msg.writes) deliver(created, write);
+        } else if (msg.type === "hello") {
+            // Another tab just started; replay our recent writes so it doesn't miss any. The reply goes
+            // to every tab, but peers that already have a write ignore it (its timestamp isn't newer
+            // than what they hold), so the redundant broadcast is harmless.
+            pruneRecent(created);
+            if (created.recent.length) created.bc.postMessage({ type: "recent", writes: created.recent });
+        }
+    };
+    channels.set(collection, created);
+    return created;
+}
+
+// Subscribe to remote writes for a collection. Recent writes from already-open tabs arrive through the
+// same onWrite callback (as the reply to our hello), so the returned array is always empty — it's kept
+// only for API compatibility with callers that await it.
+export function connect(collection: string, onWrite: (write: RemoteWrite) => void): Promise<RemoteWrite[]> {
+    const channel = ensure(collection);
+    if (!channel) return Promise.resolve([]);
+    channel.subscribers.push(onWrite);
+    channel.bc.postMessage({ type: "hello" });
+    return Promise.resolve([]);
+}
+
+export function broadcast(collection: string, write: RemoteWrite): void {
+    const channel = ensure(collection);
+    if (!channel) return;
+    channel.recent.push(write);
+    pruneRecent(channel);
+    channel.bc.postMessage({ type: "write", write });
+}

package/storage/DiskCollection.d.ts

@@ -6,6 +6,8 @@ import { TransactionStorage } from "./TransactionStorage";
 export declare class DiskCollection<T> implements IStorageSync<T> {
     private collectionName;
     private config?;
+    static getForceNoPrompt(): boolean;
+    static setForceNoPrompt(forceNoPrompt: boolean): void;
     constructor(collectionName: string, config?: {
         writeDelay?: number | undefined;
         cbor?: boolean | undefined;

package/storage/DiskCollection.ts

@@ -11,7 +11,23 @@ import { PrivateFileSystemStorage } from "./PrivateFileSystemStorage";
 import { isInChromeExtension, isInChromeExtensionBackground } from "../misc/environment";
 import { CBORStorage } from "./CBORStorage";
 
+const FORCE_NO_PROMPT_KEY = "DiskCollection_forceNoPrompt";
+
 export class DiskCollection<T> implements IStorageSync<T> {
+    // Globally forces every DiskCollection into noPrompt mode (browser only), overriding per-collection config.
+    public static getForceNoPrompt(): boolean {
+        if (isNode()) return false;
+        return !!localStorage.getItem(FORCE_NO_PROMPT_KEY);
+    }
+    public static setForceNoPrompt(forceNoPrompt: boolean) {
+        if (isNode()) return;
+        if (forceNoPrompt) {
+            localStorage.setItem(FORCE_NO_PROMPT_KEY, "1");
+        } else {
+            localStorage.removeItem(FORCE_NO_PROMPT_KEY);
+        }
+    }
+
     constructor(
         private collectionName: string,
         private config?: {
@@ -29,7 +45,7 @@ export class DiskCollection<T> implements IStorageSync<T> {
         // If a Chrome extension, just return null. 
         if (isInChromeExtensionBackground()) return null as any;
         let curCollection: IStorageRaw;
-        if (this.config?.noPrompt && !isNode()) {
+        if (!isNode() && (this.config?.noPrompt || DiskCollection.getForceNoPrompt())) {
             curCollection = await new PrivateFileSystemStorage(`collections/${this.collectionName}`);
         } else {
             let fileStorage = await getFileStorage();

package/storage/FileFolderAPI.d.ts

@@ -70,6 +70,52 @@ type DirectoryWrapper = {
     ]>;
 };
 export declare function setFileAPIKey(key: string): void;
+export declare class NodeJSFileHandleWrapper implements FileWrapper {
+    private filePath;
+    constructor(filePath: string);
+    getFile(): Promise<{
+        size: number;
+        lastModified: number;
+        arrayBuffer: () => Promise<ArrayBuffer>;
+        slice: (start: number, end: number) => {
+            arrayBuffer: () => Promise<ArrayBuffer>;
+        };
+    }>;
+    createWritable(config?: {
+        keepExistingData?: boolean;
+    }): Promise<{
+        seek: (offset: number) => Promise<void>;
+        write: (value: Buffer) => Promise<void>;
+        close: () => Promise<void>;
+    }>;
+}
+export declare class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
+    private rootPath;
+    constructor(rootPath: string);
+    removeEntry(key: string, options?: {
+        recursive?: boolean;
+    }): Promise<void>;
+    getFileHandle(key: string, options?: {
+        create?: boolean;
+    }): Promise<FileWrapper>;
+    getDirectoryHandle(key: string, options?: {
+        create?: boolean;
+    }): Promise<DirectoryWrapper>;
+    [Symbol.asyncIterator](): AsyncIterableIterator<[
+        string,
+        {
+            kind: "file";
+            name: string;
+            getFile(): Promise<FileWrapper>;
+        } | {
+            kind: "directory";
+            name: string;
+            getDirectoryHandle(key: string, options?: {
+                create?: boolean;
+            }): Promise<DirectoryWrapper>;
+        }
+    ]>;
+}
 export declare const getDirectoryHandle: {
     (): Promise<DirectoryWrapper>;
     reset(): void;
@@ -83,6 +129,14 @@ export declare const getFileStorageNested: {
     getAllKeys(): string[];
     get(key: string): Promise<FileStorage> | undefined;
 };
+export declare const getFileStorageNested2: {
+    (key: string): Promise<FileStorage>;
+    clear(key: string): void;
+    clearAll(): void;
+    forceSet(key: string, value: Promise<FileStorage>): void;
+    getAllKeys(): string[];
+    get(key: string): Promise<FileStorage> | undefined;
+};
 export declare const getFileStorage: {
     (): Promise<FileStorage>;
     reset(): void;
@@ -98,4 +152,6 @@ export type NestedFileStorage = {
 export type FileStorage = IStorageRaw & {
     folder: NestedFileStorage;
 };
+export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
+export declare function tryToLoadPointer(pointer: string): Promise<DirectoryWrapper | undefined>;
 export {};

package/storage/FileFolderAPI.tsx

@@ -96,7 +96,7 @@ class DirectoryPrompter extends preact.Component {
     }
 }
 
-class NodeJSFileHandleWrapper implements FileWrapper {
+export class NodeJSFileHandleWrapper implements FileWrapper {
     constructor(private filePath: string) {
     }
 
@@ -159,7 +159,7 @@ class NodeJSFileHandleWrapper implements FileWrapper {
     }
 }
 
-class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
+export class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
     constructor(private rootPath: string) {
     }
 
@@ -241,6 +241,7 @@ class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
 
 
 // NOTE: Blocks until the user provides a directory
+// NOTE: If you want to override the location, you can explicitly call getDirectoryHandle.set with your own directory wrapper. Which if your server side can be the Node.js directory handle wrapper, or if your client side it can just be a pointer from tryToLoadPointer/fileSystemPointer.ts.
 export const getDirectoryHandle = lazy(async function getDirectoryHandle(): Promise<DirectoryWrapper> {
     if (isNode()) {
         return new NodeJSDirectoryHandleWrapper(path.resolve("./data/"));
@@ -378,6 +379,33 @@ export const getFileStorageNested = cache(async function getFileStorage(path: st
     }
     return wrapHandle(base);
 });
+// Supports if the user selects the folder that contains the data folder or the data folder directly. If pathStr is an absolute path (and we're in nodejs) we use it directly.
+export const getFileStorageNested2 = cache(async function getFileStorage(pathStr: string): Promise<FileStorage> {
+    let base: DirectoryWrapper;
+    pathStr = pathStr.replaceAll("\\", "/");
+    if (isNode()) {
+        if (path.isAbsolute(pathStr)) {
+            return wrapHandle(new NodeJSDirectoryHandleWrapper(pathStr));
+        }
+        base = new NodeJSDirectoryHandleWrapper(path.resolve("./data/"));
+    } else {
+        base = await getDirectoryHandle();
+        let dirs: string[] = [];
+        for await (const [name, entry] of base) {
+            if (entry.kind === "directory") {
+                dirs.push(name);
+            }
+        }
+        if (dirs.includes(".git") || dirs.includes("data")) {
+            base = await base.getDirectoryHandle("data", { create: true });
+        }
+    }
+    for (let part of pathStr.split("/")) {
+        if (!part) continue;
+        base = await base.getDirectoryHandle(part, { create: true });
+    }
+    return wrapHandle(base);
+});
 export const getFileStorage = lazy(async function getFileStorage(): Promise<FileStorage> {
     if (USE_INDEXED_DB) {
         return await getFileStorageIndexDB();
@@ -434,40 +462,78 @@ async function fixedGetFileHandle(config: {
     return await config.handle.getFileHandle(config.key, { create: true });
 }
 
+// A file that genuinely does not exist. We must NOT retry these, otherwise reading many missing
+// files (a common pattern) gets catastrophically slow.
+function isMissingError(error: unknown): boolean {
+    let name = (error as { name?: string })?.name;
+    let code = (error as { code?: string })?.code;
+    return name === "NotFoundError" || code === "ENOENT";
+}
+
+const MAX_READ_RETRIES = 6;
+
+// The browser File System Access API can transiently fail reads under heavy load (it appears to
+// throttle when a page issues many reads at once). Those failures surface as errors OTHER than
+// "not found", so we retry them with backoff. A real missing file (NotFoundError / ENOENT) returns
+// undefined immediately with no retry.
+async function readWithRetry<T>(label: string, key: string, read: () => Promise<T>): Promise<T | undefined> {
+    let backoff = 25;
+    for (let attempt = 0; ; attempt++) {
+        try {
+            return await read();
+        } catch (error) {
+            if (isMissingError(error)) return undefined;
+            let name = (error as { name?: string })?.name || "Error";
+            let message = (error as { message?: string })?.message || String(error);
+            if (attempt >= MAX_READ_RETRIES) {
+                console.warn(`${label} gave up on ${JSON.stringify(key)} after ${attempt} retries (${name}): ${message.slice(0, 200)}`);
+                return undefined;
+            }
+            console.warn(`${label} retrying ${JSON.stringify(key)} (attempt ${attempt + 1}, ${name}): ${message.slice(0, 200)}`);
+            await new Promise(resolve => setTimeout(resolve, backoff));
+            backoff = Math.min(backoff * 2, 1000);
+        }
+    }
+}
+
 function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
     return {
         async getInfo(key: string) {
-            try {
+            return readWithRetry("getInfo", key, async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
                 return {
                     size: fileContent.size,
                     lastModified: fileContent.lastModified,
                 };
-            } catch (error) {
-                return undefined;
-            }
+            });
         },
         async get(key: string): Promise<Buffer | undefined> {
-            try {
+            return readWithRetry("get", key, async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
                 const arrayBuffer = await fileContent.arrayBuffer();
+                // Under load the FS Access API can resolve with a truncated buffer and no error.
+                //  Treat a short read as transient so readWithRetry retries it.
+                if (arrayBuffer.byteLength !== fileContent.size) {
+                    throw Object.assign(new Error(`Short read: got ${arrayBuffer.byteLength} of ${fileContent.size} bytes`), { name: "ShortReadError" });
+                }
                 return Buffer.from(arrayBuffer);
-            } catch (error) {
-                return undefined;
-            }
+            });
         },
 
         async getRange(key: string, config: { start: number; end: number }): Promise<Buffer | undefined> {
-            try {
+            return readWithRetry("getRange", key, async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
+                const clampedStart = Math.min(Math.max(config.start, 0), fileContent.size);
+                const clampedEnd = Math.min(Math.max(config.end, clampedStart), fileContent.size);
                 const arrayBuffer = await fileContent.slice(config.start, config.end).arrayBuffer();
+                if (arrayBuffer.byteLength !== clampedEnd - clampedStart) {
+                    throw Object.assign(new Error(`Short range read: got ${arrayBuffer.byteLength} of ${clampedEnd - clampedStart} bytes`), { name: "ShortReadError" });
+                }
                 return Buffer.from(arrayBuffer);
-            } catch (error) {
-                return undefined;
-            }
+            });
         },
 
         async append(key: string, value: Buffer): Promise<void> {
@@ -496,10 +562,18 @@ function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
 
         async getKeys(includeFolders: boolean = false): Promise<string[]> {
             const keys: string[] = [];
-            for await (const [name, entry] of handle) {
-                if (entry.kind === "file" || includeFolders) {
-                    keys.push(entry.name);
+            try {
+                for await (const [name, entry] of handle) {
+                    if (entry.kind === "file" || includeFolders) {
+                        keys.push(entry.name);
+                    }
                 }
+            } catch (error) {
+                let name = (error as { name?: string })?.name || "Error";
+                let message = (error as { message?: string })?.message || String(error);
+                // A failure mid-iteration would silently truncate the listing, so surface it loudly.
+                console.error(`getKeys directory iteration failed after ${keys.length} entries (${name}): ${message.slice(0, 300)}`);
+                throw error;
             }
             return keys;
         },
@@ -544,14 +618,14 @@ function wrapHandleNested(handle: DirectoryWrapper): NestedFileStorage {
     };
 }
 
-function wrapHandle(handle: DirectoryWrapper): FileStorage {
+export function wrapHandle(handle: DirectoryWrapper): FileStorage {
     return {
         ...wrapHandleFiles(handle),
         folder: wrapHandleNested(handle),
     };
 }
 
-async function tryToLoadPointer(pointer: string) {
+export async function tryToLoadPointer(pointer: string) {
     let result = await getFileSystemPointer({ pointer });
     if (!result) return;
     let handle = await result?.onUserActivation();

package/yarn.lock

@@ -1940,10 +1940,10 @@ slash@^3.0.0:
   resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
   integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
 
-socket-function@^1.1.30:
-  version "1.1.30"
-  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.1.30.tgz#adeb9614996e727518e2abdc7a1aa90544d669cd"
-  integrity sha512-EcqNpM8i0icBr2iU80aovXI05aUBrB4P4e9U7EQyju68TGAMagdA4OQXi+V1MHUipH/3nIdw3UAVYv+kFwZwnQ==
+socket-function@^1.1.42:
+  version "1.1.42"
+  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.1.42.tgz#79694906fe37c77da720f0fcc46a812021349062"
+  integrity sha512-+nhKi73LNv8m99w/sShaQbxQbUA1qgO7QvP0PX7MdV9SvhAbnOY9O4siaqTfqtJy5cMuX8Xnbfz/pptvvPDTaA==
   dependencies:
     "@types/pako" "^2.0.3"
     "@types/ws" "^8.5.3"