sliftutils 0.1.1

package/storage/IStorage.ts

@@ -0,0 +1,40 @@
+// TODO: Create a IStorageKeyEscaped interface, which uses escaping so that any keys
+//      are allowed. At the moment if you use keys which the underlying storage (ex,
+//      the file system) doesn't support, it will just throw.
+
+export type IStorageSync<T> = {
+    get(key: string): T | undefined;
+    set(key: string, value: T): void;
+    remove(key: string): void;
+    getKeys(): string[];
+    getValues(): T[];
+    getEntries(): [string, T][];
+    getInfo(key: string): { size: number; lastModified: number } | undefined;
+    reset(): Promise<void>;
+};
+export type IStorage<T> = {
+    get(key: string): Promise<T | undefined>;
+    set(key: string, value: T): Promise<void>;
+    remove(key: string): Promise<void>;
+    getKeys(): Promise<string[]>;
+    getInfo(key: string): Promise<undefined | {
+        size: number;
+        lastModified: number;
+    }>;
+    reset(): Promise<void>;
+};
+// NOTE: In the file system some characters are disallowed, and some characters do special things
+//  (/ makes a folder). And there are even more rules, such as lengths per folder, etc, etc.
+export type IStorageRaw = {
+    get(key: string): Promise<Buffer | undefined>;
+    // May or may not be efficient in the underlying storage
+    append(key: string, value: Buffer): Promise<void>;
+    set(key: string, value: Buffer): Promise<void>;
+    remove(key: string): Promise<void>;
+    getKeys(): Promise<string[]>;
+    getInfo(key: string): Promise<undefined | {
+        size: number;
+        lastModified: number;
+    }>;
+    reset(): Promise<void>;
+};

package/storage/IndexedDBFileFolderAPI.ts

@@ -0,0 +1,170 @@
+import { lazy } from "socket-function/src/caching";
+import { IStorageRaw } from "./IStorage";
+import { FileStorage } from "./FileFolderAPI";
+
+const DB_NAME = "FileStorage";
+const STORE_NAME = "files";
+const DB_VERSION = 1;
+
+interface FileRecord {
+    data: Buffer;
+    lastModified: number;
+}
+
+class VirtualFileStorage implements FileStorage {
+    private db: IDBDatabase;
+
+    constructor(
+        db: IDBDatabase,
+        public readonly id: string
+    ) {
+        if (!db) debugger;
+        this.db = db;
+    }
+
+    private getStore(mode: IDBTransactionMode = "readonly") {
+        const transaction = this.db.transaction(STORE_NAME, mode);
+        return transaction.objectStore(STORE_NAME);
+    }
+
+    private request<T>(request: IDBRequest<T>): Promise<T> {
+        return new Promise((resolve, reject) => {
+            request.onsuccess = () => resolve(request.result);
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+
+    // IStorageRaw implementation
+    async get(key: string): Promise<Buffer | undefined> {
+        const store = this.getStore();
+        const result = await this.request<FileRecord | undefined>(store.get(this.id + key));
+        let badBuffer = result?.data;
+        if (badBuffer) badBuffer = Buffer.from(badBuffer);
+        return badBuffer;
+    }
+
+    async append(key: string, value: Buffer): Promise<void> {
+        const store = this.getStore("readwrite");
+        const fullPath = this.id + key;
+        const existing = await this.request<FileRecord | undefined>(store.get(fullPath));
+
+        const newRecord: FileRecord = {
+            data: existing
+                ? Buffer.concat([existing.data, value])
+                : value,
+            lastModified: Date.now()
+        };
+
+        await this.request(store.put(newRecord, fullPath));
+    }
+
+    async set(key: string, value: Buffer): Promise<void> {
+        const store = this.getStore("readwrite");
+        const record: FileRecord = {
+            data: value,
+            lastModified: Date.now()
+        };
+        await this.request(store.put(record, this.id + key));
+    }
+
+    async remove(key: string): Promise<void> {
+        const store = this.getStore("readwrite");
+        await this.request(store.delete(this.id + key));
+    }
+
+    private async getKeysWithPrefix(prefix: string): Promise<string[]> {
+        const store = this.getStore();
+        const range = IDBKeyRange.bound(prefix, prefix + "\uffff", false, true);
+
+        return new Promise((resolve, reject) => {
+            const keys: string[] = [];
+            const request = store.openCursor(range);
+
+            request.onerror = () => reject(request.error);
+            request.onsuccess = () => {
+                const cursor = request.result;
+                if (cursor) {
+                    let newKey = cursor.key as string;
+                    newKey = newKey.slice(this.id.length);
+                    keys.push(newKey);
+                    cursor.continue();
+                } else {
+                    resolve(keys);
+                }
+            };
+        });
+    }
+
+    async getKeys(): Promise<string[]> {
+        let keys = await this.getKeysWithPrefix(this.id);
+        return keys.filter(x => !x.includes("/"));
+    }
+
+    async getInfo(key: string): Promise<{ size: number; lastModified: number; } | undefined> {
+        const store = this.getStore();
+        const result = await this.request<FileRecord | undefined>(store.get(this.id + key));
+
+        if (!result) return undefined;
+
+        return {
+            size: result.data.length,
+            lastModified: result.lastModified
+        };
+    }
+
+    async reset(): Promise<void> {
+        let keys = await this.getKeysWithPrefix(this.id);
+        for (let key of keys) {
+            await this.remove(key);
+        }
+    }
+
+    // NestedFileStorage implementation
+    folder = {
+        hasKey: async (key: string): Promise<boolean> => {
+            const folderPath = this.id + key + "/";
+            const keys = await this.getKeysWithPrefix(folderPath);
+            return keys.length > 0;
+        },
+
+        getStorage: async (key: string): Promise<FileStorage> => {
+            const newPath = this.id + key + "/";
+            return new VirtualFileStorage(this.db, newPath);
+        },
+
+        removeStorage: async (key: string): Promise<void> => {
+            let nested = new VirtualFileStorage(this.db, this.id + key + "/");
+            await nested.reset();
+        },
+
+        getKeys: async (): Promise<string[]> => {
+            let keys = await this.getKeysWithPrefix(this.id);
+            let folderKeys = new Set<string>();
+            for (let key of keys) {
+                if (!key.includes("/")) continue;
+                let parts = key.split("/");
+                folderKeys.add(parts[0]);
+            }
+            return Array.from(folderKeys);
+        }
+    };
+}
+
+export const getFileStorageIndexDB = lazy(async (): Promise<FileStorage> => {
+    const db = await new Promise<IDBDatabase>((resolve, reject) => {
+        const request = indexedDB.open(DB_NAME, DB_VERSION);
+
+        request.onerror = () => reject(request.error);
+        request.onsuccess = () => resolve(request.result);
+
+        request.onupgradeneeded = (event) => {
+            const db = (event.target as IDBOpenDBRequest).result;
+            if (!db.objectStoreNames.contains(STORE_NAME)) {
+                db.createObjectStore(STORE_NAME);
+            }
+        };
+    });
+
+    return new VirtualFileStorage(db, "/");
+});

package/storage/JSONStorage.ts

@@ -0,0 +1,35 @@
+import { IStorage } from "./IStorage";
+
+export class JSONStorage<T> implements IStorage<T> {
+    constructor(private storage: IStorage<Buffer>) { }
+    public async get(key: string): Promise<T | undefined> {
+        let buffer = await this.storage.get(key);
+        if (buffer === undefined) {
+            return undefined;
+        }
+        try {
+            return JSON.parse(buffer.toString());
+        } catch (e) {
+            console.warn(`Failed to parse JSON for key: ${key}`, buffer.toString(), e);
+        }
+    }
+    public async set(key: string, value: T): Promise<void> {
+        await this.storage.set(key, Buffer.from(JSON.stringify(value)));
+    }
+    public async remove(key: string): Promise<void> {
+        await this.storage.remove(key);
+    }
+    public async getKeys(): Promise<string[]> {
+        return await this.storage.getKeys();
+    }
+    public async getInfo(key: string) {
+        return await this.storage.getInfo(key);
+    }
+
+
+
+
+    public async reset() {
+        await this.storage.reset();
+    }
+}

package/storage/PendingManager.tsx

@@ -0,0 +1,63 @@
+import { throttleFunction } from "socket-function/src/misc";
+import { observable } from "mobx";
+import preact from "preact";
+import { css } from "typesafecss";
+import { observer } from "../web/observer";
+import { formatTime } from "socket-function/src/formatting/format";
+
+let watchState = observable({
+    pending: {} as { [group: string]: string }
+});
+let pendingLastSets = new Map<string, number>();
+
+let pendingCache = new Map<string, string>();
+
+// "" clears the pending value
+export function setPending(group: string, message: string) {
+    pendingCache.set(group, message);
+    void setPendingBase();
+}
+
+export function hasPending(): boolean {
+    return Object.keys(watchState.pending).length > 0;
+}
+
+// NOTE: This not only prevents render overload, but also means any pending that are < this
+//  delay don't show up (which is useful to reduce unnecessary pending messages).
+const setPendingBase = throttleFunction(500, function setPendingBase() {
+    for (let [group, message] of pendingCache) {
+
+        if (!message) {
+            let lastSet = pendingLastSets.get(group);
+            if (lastSet) {
+                let duration = Date.now() - lastSet;
+                if (duration > 500) {
+                    console.log(`Finished slow task after ${formatTime(duration)}: ${JSON.stringify(group)}, last is ${JSON.stringify(watchState.pending[group])}`);
+                }
+                pendingLastSets.delete(group);
+            }
+            delete watchState.pending[group];
+        } else {
+            //console.log("setPending", group, message);
+            if (!(group in watchState.pending)) {
+                pendingLastSets.set(group, Date.now());
+            }
+            watchState.pending[group] = message;
+        }
+    }
+    pendingCache.clear();
+});
+
+@observer
+export class PendingDisplay extends preact.Component {
+    render() {
+        // Single line, giving equal space, and ellipsis for overflow
+        return <div className={css.hbox(10)}>
+            {Object.keys(watchState.pending).map(group => (
+                <div className={css.center.textOverflow("ellipsis").border("1px solid black").pad2(6, 2)}>
+                    {group}: {watchState.pending[group]}
+                </div>
+            ))}
+        </div>;
+    }
+}

package/storage/PendingStorage.ts

@@ -0,0 +1,47 @@
+import { throttleFunction } from "socket-function/src/misc";
+import { IStorage } from "./IStorage";
+import { setPending } from "./PendingManager";
+
+export class PendingStorage<T> implements IStorage<T> {
+    pending = new Map<string, number>();
+    constructor(
+        private pendingGroup: string,
+        private storage: IStorage<T>,
+    ) { }
+    public async get(key: string): Promise<T | undefined> {
+        return this.watchPending("get", this.storage.get(key));
+    }
+    public async set(key: string, value: T): Promise<void> {
+        return this.watchPending("set", this.storage.set(key, value));
+    }
+    public async remove(key: string): Promise<void> {
+        return this.watchPending("remove", this.storage.remove(key));
+    }
+    public async getKeys(): Promise<string[]> {
+        return this.watchPending("getKeys", this.storage.getKeys());
+    }
+    public async getInfo(key: string) {
+        return this.watchPending("getInfo", this.storage.getInfo(key));
+    }
+
+    private watchPending<T>(type: string, promise: Promise<T>): Promise<T> {
+        this.pending.set(type, (this.pending.get(type) || 0) + 1);
+        void this.updatePending();
+        void promise.finally(() => {
+            this.pending.set(type, (this.pending.get(type) || 0) - 1);
+            if (this.pending.get(type) === 0) {
+                this.pending.delete(type);
+            }
+            void this.updatePending();
+        });
+        return promise;
+    }
+    private updatePending = throttleFunction(100, () => {
+        let text = Array.from(this.pending.entries()).map(([key, value]) => `${key}: ${value}`).join(", ");
+        setPending(this.pendingGroup, text);
+    });
+
+    public async reset() {
+        return this.storage.reset();
+    }
+}

package/storage/PrivateFileSystemStorage.ts

@@ -0,0 +1,192 @@
+import { IStorageRaw } from "./IStorage";
+
+export class PrivateFileSystemStorage implements IStorageRaw {
+    private rootHandle: FileSystemDirectoryHandle | undefined;
+
+    constructor(private path: string) { }
+
+    private async ensureInitialized(): Promise<FileSystemDirectoryHandle> {
+        if (!this.rootHandle) {
+            if (!navigator.storage?.getDirectory) {
+                throw new Error("Private File System Access API not supported in this browser");
+            }
+            this.rootHandle = await navigator.storage.getDirectory();
+        }
+        return this.rootHandle;
+    }
+
+    private async directoryExists(): Promise<boolean> {
+        try {
+            const root = await this.ensureInitialized();
+
+            if (!this.path || this.path === "" || this.path === "/") {
+                return true; // Root always exists
+            }
+
+            const pathParts = this.path.split("/").filter(part => part.length > 0);
+            let currentHandle = root;
+
+            for (const part of pathParts) {
+                try {
+                    currentHandle = await currentHandle.getDirectoryHandle(part);
+                } catch (error) {
+                    return false;
+                }
+            }
+
+            return true;
+        } catch (error) {
+            return false;
+        }
+    }
+
+    private async getDirectoryHandle(createPath: boolean = true): Promise<FileSystemDirectoryHandle> {
+        const root = await this.ensureInitialized();
+
+        if (!this.path || this.path === "" || this.path === "/") {
+            return root;
+        }
+
+        const pathParts = this.path.split("/").filter(part => part.length > 0);
+        let currentHandle = root;
+
+        for (const part of pathParts) {
+            try {
+                currentHandle = await currentHandle.getDirectoryHandle(part, { create: createPath });
+            } catch (error) {
+                if (!createPath) {
+                    throw new Error(`Directory not found: ${this.path}`);
+                }
+                throw error;
+            }
+        }
+
+        return currentHandle;
+    }
+
+    private async getFileHandle(key: string, create: boolean = false): Promise<FileSystemFileHandle | undefined> {
+        try {
+            const dirHandle = await this.getDirectoryHandle(create);
+            return await dirHandle.getFileHandle(key, { create });
+        } catch (error) {
+            return undefined;
+        }
+    }
+
+    private async fileExists(key: string): Promise<boolean> {
+        try {
+            // First check if directory exists
+            if (!(await this.directoryExists())) {
+                return false;
+            }
+
+            const dirHandle = await this.getDirectoryHandle(false);
+            await dirHandle.getFileHandle(key);
+            return true;
+        } catch (error) {
+            return false;
+        }
+    }
+
+    public async get(key: string): Promise<Buffer | undefined> {
+        // Check if file exists first to avoid unnecessary errors
+        if (!(await this.fileExists(key))) {
+            return undefined;
+        }
+
+        try {
+            const fileHandle = await this.getFileHandle(key, false);
+            if (!fileHandle) {
+                return undefined;
+            }
+
+            const file = await fileHandle.getFile();
+            const arrayBuffer = await file.arrayBuffer();
+            return Buffer.from(arrayBuffer);
+        } catch (error) {
+            console.warn(`Error reading file ${key}:`, error);
+            return undefined;
+        }
+    }
+
+    public async set(key: string, value: Buffer): Promise<void> {
+        try {
+            const fileHandle = await this.getFileHandle(key, true);
+            if (!fileHandle) {
+                throw new Error(`Failed to create file handle for key: ${key}`);
+            }
+
+            const writable = await fileHandle.createWritable();
+            await writable.write(value);
+            await writable.close();
+        } catch (error) {
+            throw new Error(`Failed to write file ${key}: ${error}`);
+        }
+    }
+
+    public async append(key: string, value: Buffer): Promise<void> {
+        const existingContent = await this.get(key);
+        let newContent: Buffer;
+        if (existingContent) {
+            newContent = Buffer.concat([existingContent, value]);
+        } else {
+            newContent = value;
+        }
+        await this.set(key, newContent);
+    }
+
+    public async remove(key: string): Promise<void> {
+        // Check if file exists first to avoid unnecessary errors
+        if (!(await this.fileExists(key))) {
+            return; // File doesn't exist, nothing to remove
+        }
+
+        try {
+            const dirHandle = await this.getDirectoryHandle(false);
+            await dirHandle.removeEntry(key);
+        } catch { }
+    }
+
+    public async getKeys(): Promise<string[]> {
+        // Check if directory exists first to avoid unnecessary errors
+        if (!(await this.directoryExists())) {
+            return [];
+        }
+
+        try {
+            const dirHandle = await this.getDirectoryHandle(false);
+            const keys: string[] = [];
+
+            // Use the async iterator protocol for FileSystemDirectoryHandle
+            for await (const [name, handle] of (dirHandle as any).entries()) {
+                if (handle.kind === "file") {
+                    keys.push(name);
+                }
+            }
+
+            return keys.sort();
+        } catch (error) {
+            if (error instanceof Error && error.message.includes("Directory not found")) {
+                return [];
+            }
+            throw new Error(`Failed to list files: ${error}`);
+        }
+    }
+    public async getInfo(key: string): Promise<undefined | {
+        size: number;
+        lastModified: number;
+    }> {
+        let fileHandle = await this.getFileHandle(key, false);
+        if (!fileHandle) {
+            return undefined;
+        }
+        let file = await fileHandle.getFile();
+        return {
+            size: file.size,
+            lastModified: file.lastModified,
+        };
+    }
+    public async reset(): Promise<void> {
+        throw new Error("Not implemented");
+    }
+}

package/storage/StorageObservable.ts

@@ -0,0 +1,122 @@
+import { observable } from "mobx";
+import { isDefined } from "../misc/types";
+import { IStorage, IStorageSync } from "./IStorage";
+
+// NOTE: At around 500K values (depending on their size to some degree), this will take about 2 minutes to load. But once it does it will be fast. So... keep that in mind. I recommend not exceeding 100K
+export class StorageSync<T> implements IStorageSync<T> {
+    cached = observable.map<string, T | undefined>(undefined, { deep: false });
+    infoCached = observable.map<string, { size: number; lastModified: number } | undefined>(undefined, { deep: false });
+    keys = new Set<string>();
+    synced = observable({
+        keySeqNum: 0,
+    }, undefined, { deep: false });
+
+    constructor(public storage: IStorage<T>) { }
+
+    public get(key: string): T | undefined {
+        if (!this.cached.has(key)) {
+            this.cached.set(key, undefined);
+            void this.getPromise(key);
+        }
+        if (this.cached.get(key) === undefined) {
+            this.synced.keySeqNum;
+        }
+        return this.cached.get(key);
+    }
+    public set(key: string, value: T): void {
+        if (!this.keys.has(key)) {
+            this.keys.add(key);
+            this.synced.keySeqNum++;
+        }
+        this.cached.set(key, value);
+        void this.storage.set(key, value);
+    }
+    public remove(key: string): void {
+        if (this.keys.has(key)) {
+            this.keys.delete(key);
+            this.synced.keySeqNum++;
+        }
+        this.cached.delete(key);
+        void this.storage.remove(key);
+    }
+    private loadedKeys = false;
+    public getKeys(): string[] {
+        void this.getKeysPromise();
+        this.synced.keySeqNum;
+        return Array.from(this.keys);
+    }
+
+    public getInfo(key: string): { size: number; lastModified: number } | undefined {
+        if (!this.infoCached.has(key)) {
+            this.infoCached.set(key, { size: 0, lastModified: 0 });
+            void this.storage.getInfo(key).then(info => {
+                this.infoCached.set(key, info);
+            });
+        }
+        return this.infoCached.get(key);
+    }
+
+    public getValues(): T[] {
+        let keys = this.getKeys();
+        return keys.map(key => this.get(key)).filter(isDefined);
+    }
+    public getEntries(): [string, T][] {
+        let keys = this.getKeys();
+        return keys.map(key => [key, this.get(key)]).filter(([_, value]) => isDefined(value)) as [string, T][];
+    }
+
+
+    public async getPromise(key: string): Promise<T | undefined> {
+        let value = this.cached.get(key);
+        if (value === undefined) {
+            value = await this.storage.get(key);
+            if (this.cached.get(key) === undefined) {
+                this.cached.set(key, value);
+            }
+        }
+        return value;
+    }
+    private pendingGetKeys: Promise<string[]> | undefined;
+    public async getKeysPromise(): Promise<string[]> {
+        if (this.pendingGetKeys) {
+            return this.pendingGetKeys;
+        }
+        if (this.loadedKeys) return Array.from(this.keys);
+        this.loadedKeys = true;
+        this.pendingGetKeys = this.storage.getKeys();
+        void this.pendingGetKeys.finally(() => {
+            this.pendingGetKeys = undefined;
+        });
+        let keys = await this.pendingGetKeys;
+        if (keys.length > 0) {
+            this.keys = new Set(keys);
+            this.synced.keySeqNum++;
+        }
+        return Array.from(this.keys);
+    }
+
+    public reload() {
+        this.loadedKeys = false;
+        this.synced.keySeqNum++;
+        this.cached.clear();
+        this.infoCached.clear();
+        this.keys.clear();
+    }
+    public reloadKeys() {
+        this.loadedKeys = false;
+        this.synced.keySeqNum++;
+    }
+    public reloadKey(key: string) {
+        this.cached.delete(key);
+        this.infoCached.delete(key);
+        this.keys.delete(key);
+    }
+
+    public async reset() {
+        this.cached.clear();
+        this.infoCached.clear();
+        this.keys.clear();
+        this.synced.keySeqNum++;
+        await this.storage.reset();
+    }
+}