sliftutils 1.1.40 → 1.1.42

package/index.d.ts

@@ -55,6 +55,7 @@ declare module "sliftutils/misc/getSecret" {
         getAllKeys(): string[];
         get(key: string): Promise<string> | undefined;
     };
+    export declare function setSecret(key: string, value: string): void;
 
 }
 
@@ -986,6 +987,52 @@ declare module "sliftutils/storage/FileFolderAPI" {
         ]>;
     };
     export declare function setFileAPIKey(key: string): void;
+    export declare class NodeJSFileHandleWrapper implements FileWrapper {
+        private filePath;
+        constructor(filePath: string);
+        getFile(): Promise<{
+            size: number;
+            lastModified: number;
+            arrayBuffer: () => Promise<ArrayBuffer>;
+            slice: (start: number, end: number) => {
+                arrayBuffer: () => Promise<ArrayBuffer>;
+            };
+        }>;
+        createWritable(config?: {
+            keepExistingData?: boolean;
+        }): Promise<{
+            seek: (offset: number) => Promise<void>;
+            write: (value: Buffer) => Promise<void>;
+            close: () => Promise<void>;
+        }>;
+    }
+    export declare class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
+        private rootPath;
+        constructor(rootPath: string);
+        removeEntry(key: string, options?: {
+            recursive?: boolean;
+        }): Promise<void>;
+        getFileHandle(key: string, options?: {
+            create?: boolean;
+        }): Promise<FileWrapper>;
+        getDirectoryHandle(key: string, options?: {
+            create?: boolean;
+        }): Promise<DirectoryWrapper>;
+        [Symbol.asyncIterator](): AsyncIterableIterator<[
+            string,
+            {
+                kind: "file";
+                name: string;
+                getFile(): Promise<FileWrapper>;
+            } | {
+                kind: "directory";
+                name: string;
+                getDirectoryHandle(key: string, options?: {
+                    create?: boolean;
+                }): Promise<DirectoryWrapper>;
+            }
+        ]>;
+    }
     export declare const getDirectoryHandle: {
         (): Promise<DirectoryWrapper>;
         reset(): void;
@@ -1014,6 +1061,8 @@ declare module "sliftutils/storage/FileFolderAPI" {
     export type FileStorage = IStorageRaw & {
         folder: NestedFileStorage;
     };
+    export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
+    export declare function tryToLoadPointer(pointer: string): Promise<DirectoryWrapper | undefined>;
     export {};
 
 }

package/misc/getSecret.d.ts

@@ -7,3 +7,4 @@ export declare const getSecret: {
     getAllKeys(): string[];
     get(key: string): Promise<string> | undefined;
 };
+export declare function setSecret(key: string, value: string): void;

package/misc/getSecret.ts

@@ -96,3 +96,10 @@ export const getSecret = cache(async function getSecret(key: string): Promise<st
         });
     });
 });
+
+export function setSecret(key: string, value: string) {
+    if (isNode()) {
+        throw new Error("setSecret is only supported in the browser. We don't want to break the user's file system with setSecret in NodeJS.");
+    }
+    localStorage.setItem(getStorageKey(key), value);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.1.40",
+    "version": "1.1.42",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/FileFolderAPI.d.ts

@@ -70,6 +70,52 @@ type DirectoryWrapper = {
     ]>;
 };
 export declare function setFileAPIKey(key: string): void;
+export declare class NodeJSFileHandleWrapper implements FileWrapper {
+    private filePath;
+    constructor(filePath: string);
+    getFile(): Promise<{
+        size: number;
+        lastModified: number;
+        arrayBuffer: () => Promise<ArrayBuffer>;
+        slice: (start: number, end: number) => {
+            arrayBuffer: () => Promise<ArrayBuffer>;
+        };
+    }>;
+    createWritable(config?: {
+        keepExistingData?: boolean;
+    }): Promise<{
+        seek: (offset: number) => Promise<void>;
+        write: (value: Buffer) => Promise<void>;
+        close: () => Promise<void>;
+    }>;
+}
+export declare class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
+    private rootPath;
+    constructor(rootPath: string);
+    removeEntry(key: string, options?: {
+        recursive?: boolean;
+    }): Promise<void>;
+    getFileHandle(key: string, options?: {
+        create?: boolean;
+    }): Promise<FileWrapper>;
+    getDirectoryHandle(key: string, options?: {
+        create?: boolean;
+    }): Promise<DirectoryWrapper>;
+    [Symbol.asyncIterator](): AsyncIterableIterator<[
+        string,
+        {
+            kind: "file";
+            name: string;
+            getFile(): Promise<FileWrapper>;
+        } | {
+            kind: "directory";
+            name: string;
+            getDirectoryHandle(key: string, options?: {
+                create?: boolean;
+            }): Promise<DirectoryWrapper>;
+        }
+    ]>;
+}
 export declare const getDirectoryHandle: {
     (): Promise<DirectoryWrapper>;
     reset(): void;
@@ -98,4 +144,6 @@ export type NestedFileStorage = {
 export type FileStorage = IStorageRaw & {
     folder: NestedFileStorage;
 };
+export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
+export declare function tryToLoadPointer(pointer: string): Promise<DirectoryWrapper | undefined>;
 export {};

package/storage/FileFolderAPI.tsx

@@ -96,7 +96,7 @@ class DirectoryPrompter extends preact.Component {
     }
 }
 
-class NodeJSFileHandleWrapper implements FileWrapper {
+export class NodeJSFileHandleWrapper implements FileWrapper {
     constructor(private filePath: string) {
     }
 
@@ -159,7 +159,7 @@ class NodeJSFileHandleWrapper implements FileWrapper {
     }
 }
 
-class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
+export class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
     constructor(private rootPath: string) {
     }
 
@@ -241,6 +241,7 @@ class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
 
 
 // NOTE: Blocks until the user provides a directory
+// NOTE: If you want to override the location, you can explicitly call getDirectoryHandle.set with your own directory wrapper. Which if your server side can be the Node.js directory handle wrapper, or if your client side it can just be a pointer from tryToLoadPointer/fileSystemPointer.ts.
 export const getDirectoryHandle = lazy(async function getDirectoryHandle(): Promise<DirectoryWrapper> {
     if (isNode()) {
         return new NodeJSDirectoryHandleWrapper(path.resolve("./data/"));
@@ -434,40 +435,78 @@ async function fixedGetFileHandle(config: {
     return await config.handle.getFileHandle(config.key, { create: true });
 }
 
+// A file that genuinely does not exist. We must NOT retry these, otherwise reading many missing
+// files (a common pattern) gets catastrophically slow.
+function isMissingError(error: unknown): boolean {
+    let name = (error as { name?: string })?.name;
+    let code = (error as { code?: string })?.code;
+    return name === "NotFoundError" || code === "ENOENT";
+}
+
+const MAX_READ_RETRIES = 6;
+
+// The browser File System Access API can transiently fail reads under heavy load (it appears to
+// throttle when a page issues many reads at once). Those failures surface as errors OTHER than
+// "not found", so we retry them with backoff. A real missing file (NotFoundError / ENOENT) returns
+// undefined immediately with no retry.
+async function readWithRetry<T>(label: string, key: string, read: () => Promise<T>): Promise<T | undefined> {
+    let backoff = 25;
+    for (let attempt = 0; ; attempt++) {
+        try {
+            return await read();
+        } catch (error) {
+            if (isMissingError(error)) return undefined;
+            let name = (error as { name?: string })?.name || "Error";
+            let message = (error as { message?: string })?.message || String(error);
+            if (attempt >= MAX_READ_RETRIES) {
+                console.warn(`${label} gave up on ${JSON.stringify(key)} after ${attempt} retries (${name}): ${message.slice(0, 200)}`);
+                return undefined;
+            }
+            console.warn(`${label} retrying ${JSON.stringify(key)} (attempt ${attempt + 1}, ${name}): ${message.slice(0, 200)}`);
+            await new Promise(resolve => setTimeout(resolve, backoff));
+            backoff = Math.min(backoff * 2, 1000);
+        }
+    }
+}
+
 function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
     return {
         async getInfo(key: string) {
-            try {
+            return readWithRetry("getInfo", key, async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
                 return {
                     size: fileContent.size,
                     lastModified: fileContent.lastModified,
                 };
-            } catch (error) {
-                return undefined;
-            }
+            });
         },
         async get(key: string): Promise<Buffer | undefined> {
-            try {
+            return readWithRetry("get", key, async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
                 const arrayBuffer = await fileContent.arrayBuffer();
+                // Under load the FS Access API can resolve with a truncated buffer and no error.
+                //  Treat a short read as transient so readWithRetry retries it.
+                if (arrayBuffer.byteLength !== fileContent.size) {
+                    throw Object.assign(new Error(`Short read: got ${arrayBuffer.byteLength} of ${fileContent.size} bytes`), { name: "ShortReadError" });
+                }
                 return Buffer.from(arrayBuffer);
-            } catch (error) {
-                return undefined;
-            }
+            });
         },
 
         async getRange(key: string, config: { start: number; end: number }): Promise<Buffer | undefined> {
-            try {
+            return readWithRetry("getRange", key, async () => {
                 const file = await handle.getFileHandle(key);
                 const fileContent = await file.getFile();
+                const clampedStart = Math.min(Math.max(config.start, 0), fileContent.size);
+                const clampedEnd = Math.min(Math.max(config.end, clampedStart), fileContent.size);
                 const arrayBuffer = await fileContent.slice(config.start, config.end).arrayBuffer();
+                if (arrayBuffer.byteLength !== clampedEnd - clampedStart) {
+                    throw Object.assign(new Error(`Short range read: got ${arrayBuffer.byteLength} of ${clampedEnd - clampedStart} bytes`), { name: "ShortReadError" });
+                }
                 return Buffer.from(arrayBuffer);
-            } catch (error) {
-                return undefined;
-            }
+            });
         },
 
         async append(key: string, value: Buffer): Promise<void> {
@@ -496,10 +535,18 @@ function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
 
         async getKeys(includeFolders: boolean = false): Promise<string[]> {
             const keys: string[] = [];
-            for await (const [name, entry] of handle) {
-                if (entry.kind === "file" || includeFolders) {
-                    keys.push(entry.name);
+            try {
+                for await (const [name, entry] of handle) {
+                    if (entry.kind === "file" || includeFolders) {
+                        keys.push(entry.name);
+                    }
                 }
+            } catch (error) {
+                let name = (error as { name?: string })?.name || "Error";
+                let message = (error as { message?: string })?.message || String(error);
+                // A failure mid-iteration would silently truncate the listing, so surface it loudly.
+                console.error(`getKeys directory iteration failed after ${keys.length} entries (${name}): ${message.slice(0, 300)}`);
+                throw error;
             }
             return keys;
         },
@@ -544,14 +591,14 @@ function wrapHandleNested(handle: DirectoryWrapper): NestedFileStorage {
     };
 }
 
-function wrapHandle(handle: DirectoryWrapper): FileStorage {
+export function wrapHandle(handle: DirectoryWrapper): FileStorage {
     return {
         ...wrapHandleFiles(handle),
         folder: wrapHandleNested(handle),
     };
 }
 
-async function tryToLoadPointer(pointer: string) {
+export async function tryToLoadPointer(pointer: string) {
     let result = await getFileSystemPointer({ pointer });
     if (!result) return;
     let handle = await result?.onUserActivation();