sliftutils 1.1.4 → 1.1.41

package/storage/FileFolderAPI.tsx

@@ -40,6 +40,9 @@ type FileWrapper = {
         size: number;
         lastModified: number;
         arrayBuffer(): Promise<ArrayBuffer>;
+        // Matches Blob.slice (which the native File object provides), so the browser
+        //  implementation works vanilla. End is exclusive, both clamped to the file size.
+        slice(start: number, end: number): { arrayBuffer(): Promise<ArrayBuffer> };
     }>;
     createWritable(config?: { keepExistingData?: boolean }): Promise<{
         seek(offset: number): Promise<void>;
@@ -82,7 +85,7 @@ class DirectoryPrompter extends preact.Component {
         return (
             <div className={
                 css.position("fixed").pos(0, 0).size("100vw", "100vh")
-                    .zIndex(1)
+                    .zIndex(2147483647)
                     .background("white")
                     .center
                     .fontSize(40)
@@ -99,13 +102,29 @@ class NodeJSFileHandleWrapper implements FileWrapper {
 
     async getFile() {
         const stats = await fs.promises.stat(this.filePath);
+        const filePath = this.filePath;
         return {
             size: stats.size,
             lastModified: stats.mtimeMs,
             arrayBuffer: async () => {
-                const buffer = await fs.promises.readFile(this.filePath);
+                const buffer = await fs.promises.readFile(filePath);
                 return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
-            }
+            },
+            slice: (start: number, end: number) => ({
+                arrayBuffer: async () => {
+                    const clampedStart = Math.min(Math.max(start, 0), stats.size);
+                    const clampedEnd = Math.min(Math.max(end, clampedStart), stats.size);
+                    const length = clampedEnd - clampedStart;
+                    const fileHandle = await fs.promises.open(filePath, "r");
+                    try {
+                        const buffer = Buffer.alloc(length);
+                        await fileHandle.read(buffer, 0, length, clampedStart);
+                        return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+                    } finally {
+                        await fileHandle.close();
+                    }
+                }
+            })
         };
     }
 
@@ -251,30 +270,56 @@ export const getDirectoryHandle = lazy(async function getDirectoryHandle(): Prom
                     doneLoad = true;
                     // Show UI to get user to click and retry
                     let retryCallback: (success: boolean) => void;
-                    let retryPromise = new Promise<boolean>(resolve => {
+                    let retryReject: (err: Error) => void;
+                    let retryPromise = new Promise<boolean>((resolve, reject) => {
                         retryCallback = resolve;
+                        retryReject = reject;
                     });
                     displayData.ui = (
-                        <button
-                            className={css.fontSize(40).pad2(80, 40)}
-                            onClick={async () => {
-                                displayData.ui = "Loading...";
-                                try {
-                                    const retryHandle = await tryToLoadPointer(storedId);
-                                    if (retryHandle) {
-                                        handle = retryHandle;
-                                        retryCallback(true);
-                                    } else {
+                        <div className={css.vbox(20).center}>
+                            <button
+                                className={css.fontSize(40).pad2(80, 40)}
+                                onClick={async () => {
+                                    displayData.ui = "Loading...";
+                                    try {
+                                        const retryHandle = await tryToLoadPointer(storedId);
+                                        if (retryHandle) {
+                                            handle = retryHandle;
+                                            retryCallback(true);
+                                        } else {
+                                            retryCallback(false);
+                                        }
+                                    } catch (retryError) {
+                                        console.error("Retry failed:", retryError);
                                         retryCallback(false);
                                     }
-                                } catch (retryError) {
-                                    console.error("Retry failed:", retryError);
-                                    retryCallback(false);
-                                }
-                            }}
-                        >
-                            Click to restore file system access
-                        </button>
+                                }}
+                            >
+                                Click to restore file system access
+                            </button>
+                            <button
+                                className={css.fontSize(40).pad2(80, 40)}
+                                onClick={async () => {
+                                    console.log("Waiting for user to give permission");
+                                    const pickedHandle = await window.showDirectoryPicker();
+                                    await pickedHandle.requestPermission({ mode: "readwrite" });
+                                    let newStoredId = await storeFileSystemPointer({ mode: "readwrite", handle: pickedHandle });
+                                    localStorage.setItem(getFileAPIKey(), newStoredId);
+                                    handle = pickedHandle as any;
+                                    retryCallback(true);
+                                }}
+                            >
+                                Pick Data Directory
+                            </button>
+                            <button
+                                className={css.fontSize(40).pad2(80, 40)}
+                                onClick={() => {
+                                    retryReject(new Error("User dismissed file system access prompt"));
+                                }}
+                            >
+                                Dismiss
+                            </button>
+                        </div>
                     );
                     const success = await retryPromise;
                     if (handle) {
@@ -288,23 +333,35 @@ export const getDirectoryHandle = lazy(async function getDirectoryHandle(): Prom
             }
         }
         let fileCallback: (handle: DirectoryWrapper) => void;
-        let promise = new Promise<DirectoryWrapper>(resolve => {
+        let fileReject: (err: Error) => void;
+        let promise = new Promise<DirectoryWrapper>((resolve, reject) => {
             fileCallback = resolve;
+            fileReject = reject;
         });
         displayData.ui = (
-            <button
-                className={css.fontSize(40).pad2(80, 40)}
-                onClick={async () => {
-                    console.log("Waiting for user to give permission");
-                    const handle = await window.showDirectoryPicker();
-                    await handle.requestPermission({ mode: "readwrite" });
-                    let storedId = await storeFileSystemPointer({ mode: "readwrite", handle });
-                    localStorage.setItem(getFileAPIKey(), storedId);
-                    fileCallback(handle as any);
-                }}
-            >
-                Pick Data Directory
-            </button>
+            <div className={css.vbox(20).center}>
+                <button
+                    className={css.fontSize(40).pad2(80, 40)}
+                    onClick={async () => {
+                        console.log("Waiting for user to give permission");
+                        const handle = await window.showDirectoryPicker();
+                        await handle.requestPermission({ mode: "readwrite" });
+                        let storedId = await storeFileSystemPointer({ mode: "readwrite", handle });
+                        localStorage.setItem(getFileAPIKey(), storedId);
+                        fileCallback(handle as any);
+                    }}
+                >
+                    Pick Data Directory
+                </button>
+                <button
+                    className={css.fontSize(40).pad2(80, 40)}
+                    onClick={() => {
+                        fileReject(new Error("User dismissed file system access prompt"));
+                    }}
+                >
+                    Dismiss
+                </button>
+            </div>
         );
         return await promise;
     } finally {
@@ -402,6 +459,17 @@ function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
             }
         },
 
+        async getRange(key: string, config: { start: number; end: number }): Promise<Buffer | undefined> {
+            try {
+                const file = await handle.getFileHandle(key);
+                const fileContent = await file.getFile();
+                const arrayBuffer = await fileContent.slice(config.start, config.end).arrayBuffer();
+                return Buffer.from(arrayBuffer);
+            } catch (error) {
+                return undefined;
+            }
+        },
+
         async append(key: string, value: Buffer): Promise<void> {
             await appendQueue(key)(async () => {
                 // NOTE: Interesting point. Chrome doesn't optimize this to be an append, and instead

package/storage/IStorage.d.ts

@@ -27,6 +27,10 @@ export type IStorage<T> = {
 };
 export type IStorageRaw = {
     get(key: string): Promise<Buffer | undefined>;
+    getRange(key: string, config: {
+        start: number;
+        end: number;
+    }): Promise<Buffer | undefined>;
     append(key: string, value: Buffer): Promise<void>;
     set(key: string, value: Buffer): Promise<void>;
     remove(key: string): Promise<void>;

package/storage/IStorage.ts

@@ -29,6 +29,9 @@ export type IStorage<T> = {
 //  (/ makes a folder). And there are even more rules, such as lengths per folder, etc, etc.
 export type IStorageRaw = {
     get(key: string): Promise<Buffer | undefined>;
+    // Reads bytes in the range [start, end) (end is exclusive, clamped to the file size).
+    //  Returns undefined if the file doesn't exist.
+    getRange(key: string, config: { start: number; end: number }): Promise<Buffer | undefined>;
     // May or may not be efficient in the underlying storage
     append(key: string, value: Buffer): Promise<void>;
     set(key: string, value: Buffer): Promise<void>;

package/storage/IndexedDBFileFolderAPI.ts

@@ -44,6 +44,12 @@ class VirtualFileStorage implements FileStorage {
         return badBuffer;
     }
 
+    async getRange(key: string, config: { start: number; end: number }): Promise<Buffer | undefined> {
+        const fullBuffer = await this.get(key);
+        if (!fullBuffer) return undefined;
+        return fullBuffer.subarray(config.start, config.end);
+    }
+
     async append(key: string, value: Buffer): Promise<void> {
         const store = this.getStore("readwrite");
         const fullPath = this.id + key;

package/storage/PrivateFileSystemStorage.d.ts

@@ -11,6 +11,10 @@ export declare class PrivateFileSystemStorage implements IStorageRaw {
     private getFileHandle;
     private fileExists;
     get(key: string): Promise<Buffer | undefined>;
+    getRange(key: string, config: {
+        start: number;
+        end: number;
+    }): Promise<Buffer | undefined>;
     set(key: string, value: Buffer): Promise<void>;
     append(key: string, value: Buffer): Promise<void>;
     remove(key: string): Promise<void>;

package/storage/PrivateFileSystemStorage.ts

@@ -114,6 +114,17 @@ export class PrivateFileSystemStorage implements IStorageRaw {
         }
     }
 
+    public async getRange(key: string, config: { start: number; end: number }): Promise<Buffer | undefined> {
+        const fileHandle = await this.getFileHandle(key, false);
+        if (!fileHandle) {
+            return undefined;
+        }
+
+        const file = await fileHandle.getFile();
+        const arrayBuffer = await file.slice(config.start, config.end).arrayBuffer();
+        return Buffer.from(arrayBuffer);
+    }
+
     public async set(key: string, value: Buffer): Promise<void> {
         try {
             const fileHandle = await this.getFileHandle(key, true);

package/storage/StorageObservable.d.ts

@@ -1,4 +1,7 @@
 import { IStorage, IStorageSync } from "./IStorage";
+export declare const storagePendingAccesses: {
+    value: number;
+};
 export declare class StorageSync<T> implements IStorageSync<T> {
     storage: IStorage<T>;
     private config?;
@@ -38,3 +41,4 @@ export declare class StorageSync<T> implements IStorageSync<T> {
     reloadKey(key: string): void;
     reset(): Promise<void>;
 }
+export declare function waitUntilNextLoad(): Promise<void>;

package/storage/StorageObservable.ts

@@ -1,8 +1,11 @@
 import { observable } from "mobx";
 import { deepFreezeObject, freezeObject, isDefined } from "../misc/types";
 import { IStorage, IStorageSync } from "./IStorage";
+import { PromiseObj } from "socket-function/src/misc";
 
-// NOTE: At around 500K values (depending on their size to some degree), this will take about 2 minutes to load. But once it does it will be fast. So... keep that in mind. I recommend not exceeding 100K
+export const storagePendingAccesses = { value: 0 };
+
+// NOTE: At around 500K values (depending on their size to some degree), this will take about 2 minutes to load. But once it does it will be fast. So... keep that in mind. I recommend not exceeding 100K.
 export class StorageSync<T> implements IStorageSync<T> {
     cached = observable.map<string, T | undefined>(undefined, { deep: false });
     infoCached = observable.map<string, { size: number; lastModified: number } | undefined>(undefined, { deep: false });
@@ -85,6 +88,7 @@ export class StorageSync<T> implements IStorageSync<T> {
     public async getPromise(key: string): Promise<T | undefined> {
         let value = this.cached.get(key);
         if (value === undefined) {
+            storagePendingAccesses.value++;
             value = await this.storage.get(key);
             if (value !== undefined && this.cached.get(key) === undefined) {
                 if (this.config?.freeze === "shallow") {
@@ -94,15 +98,19 @@ export class StorageSync<T> implements IStorageSync<T> {
                 }
                 this.cached.set(key, value);
             }
+            triggerLoad();
         }
         return value;
     }
     private pendingGetKeys: Promise<string[]> | undefined;
     public async getKeysPromise(): Promise<string[]> {
+        if (this.loadedKeys) {
+            return Array.from(this.keys);
+        }
+        storagePendingAccesses.value++;
         if (this.pendingGetKeys) {
             return this.pendingGetKeys;
         }
-        if (this.loadedKeys) return Array.from(this.keys);
         this.loadedKeys = true;
         this.pendingGetKeys = this.storage.getKeys();
         void this.pendingGetKeys.finally(() => {
@@ -113,6 +121,7 @@ export class StorageSync<T> implements IStorageSync<T> {
             this.keys = new Set(keys);
             this.synced.keySeqNum++;
         }
+        triggerLoad();
         return Array.from(this.keys);
     }
 
@@ -140,4 +149,20 @@ export class StorageSync<T> implements IStorageSync<T> {
         this.synced.keySeqNum++;
         await this.storage.reset();
     }
+}
+
+let waitUntilNextLoadWatchers: PromiseObj<void>[] = [];
+export function waitUntilNextLoad(): Promise<void> {
+    let promise = new PromiseObj<void>();
+    waitUntilNextLoadWatchers.push(promise);
+    return promise.promise;
+}
+function triggerLoad() {
+    let watchers = waitUntilNextLoadWatchers;
+    waitUntilNextLoadWatchers = [];
+    void Promise.resolve().finally(() => {
+        for (let watcher of watchers) {
+            watcher.resolve();
+        }
+    });
 }

package/storage/StorageObservableAsync.d.ts

@@ -0,0 +1,2 @@
+/** Reruns the code until all StorageSyncs accessed have loaded their values. Not efficient,although will usually be O(values accessed), just due to how loading works (it won't be quadratic). */
+export declare function rerunCodeUntilAllLoaded<T>(code: () => T): Promise<T>;

package/storage/StorageObservableAsync.ts

@@ -0,0 +1,20 @@
+import { storagePendingAccesses, waitUntilNextLoad } from "./StorageObservable";
+
+/** Reruns the code until all StorageSyncs accessed have loaded their values. Not efficient,although will usually be O(values accessed), just due to how loading works (it won't be quadratic). */
+export async function rerunCodeUntilAllLoaded<T>(code: () => T): Promise<T> {
+    while (true) {
+        let beforeAccesses = storagePendingAccesses.value;
+        try {
+            let result = await code();
+            if (storagePendingAccesses.value === beforeAccesses) {
+                return result;
+            }
+        } catch (error) {
+            if (storagePendingAccesses.value === beforeAccesses) {
+                throw error;
+            }
+        }
+        console.log(`Rerunning synchronous check as loading starting while evaluating code. Function name`, code);
+        await waitUntilNextLoad();
+    }
+}

package/storage/backblaze.d.ts

@@ -0,0 +1,97 @@
+/// <reference types="node" />
+/// <reference types="node" />
+export declare class ArchivesBackblaze {
+    private config;
+    constructor(config: {
+        bucketName: string;
+        public?: boolean;
+        immutable?: boolean;
+        cacheTime?: number;
+    });
+    private bucketName;
+    private bucketId;
+    private logging;
+    enableLogging(): void;
+    private log;
+    getDebugName(): string;
+    private getBucketAPI;
+    private last503Reset;
+    private apiRetryLogic;
+    get(fileName: string, config?: {
+        range?: {
+            start: number;
+            end: number;
+        };
+        retryCount?: number;
+    }): Promise<Buffer | undefined>;
+    set(fileName: string, data: Buffer): Promise<void>;
+    del(fileName: string): Promise<void>;
+    setLargeFile(config: {
+        path: string;
+        getNextData(): Promise<Buffer | undefined>;
+    }): Promise<void>;
+    getInfo(fileName: string): Promise<{
+        writeTime: number;
+        size: number;
+    } | undefined>;
+    find(prefix: string, config?: {
+        shallow?: boolean;
+        type: "files" | "folders";
+    }): Promise<string[]>;
+    findInfo(prefix: string, config?: {
+        shallow?: boolean;
+        type: "files" | "folders";
+    }): Promise<{
+        path: string;
+        createTime: number;
+        size: number;
+    }[]>;
+    assertPathValid(path: string): Promise<void>;
+    getURL(path: string): Promise<string>;
+    getDownloadAuthorization(config: {
+        fileNamePrefix?: string;
+        validDurationInSeconds: number;
+        b2ContentDisposition?: string;
+        b2ContentLanguage?: string;
+        b2Expires?: string;
+        b2CacheControl?: string;
+        b2ContentEncoding?: string;
+        b2ContentType?: string;
+    }): Promise<{
+        bucketId: string;
+        fileNamePrefix: string;
+        authorizationToken: string;
+    }>;
+}
+export declare const getArchivesBackblaze: {
+    (key: string): ArchivesBackblaze;
+    clear(key: string): void;
+    clearAll(): void;
+    forceSet(key: string, value: ArchivesBackblaze): void;
+    getAllKeys(): string[];
+    get(key: string): ArchivesBackblaze | undefined;
+};
+export declare const getArchivesBackblazePrivateImmutable: {
+    (key: string): ArchivesBackblaze;
+    clear(key: string): void;
+    clearAll(): void;
+    forceSet(key: string, value: ArchivesBackblaze): void;
+    getAllKeys(): string[];
+    get(key: string): ArchivesBackblaze | undefined;
+};
+export declare const getArchivesBackblazePublicImmutable: {
+    (key: string): ArchivesBackblaze;
+    clear(key: string): void;
+    clearAll(): void;
+    forceSet(key: string, value: ArchivesBackblaze): void;
+    getAllKeys(): string[];
+    get(key: string): ArchivesBackblaze | undefined;
+};
+export declare const getArchivesBackblazePublic: {
+    (key: string): ArchivesBackblaze;
+    clear(key: string): void;
+    clearAll(): void;
+    forceSet(key: string, value: ArchivesBackblaze): void;
+    getAllKeys(): string[];
+    get(key: string): ArchivesBackblaze | undefined;
+};