sliftutils 0.1.1

package/storage/DiskCollection.ts

@@ -0,0 +1,272 @@
+import { isNode } from "typesafecss";
+import { DelayedStorage } from "./DelayedStorage";
+import { FileStorage, getFileStorage, getFileStorageNested } from "./FileFolderAPI";
+import { IStorage, IStorageSync } from "./IStorage";
+import { JSONStorage } from "./JSONStorage";
+import { StorageSync } from "./StorageObservable";
+import { TransactionStorage } from "./TransactionStorage";
+import { PendingStorage } from "./PendingStorage";
+import { isDefined } from "../misc/types";
+import { PrivateFileSystemStorage } from "./PrivateFileSystemStorage";
+import { isInChromeExtension } from "../misc/environment";
+
+export class DiskCollection<T> implements IStorageSync<T> {
+    constructor(
+        private collectionName: string,
+        private writeDelay?: number,
+    ) {
+    }
+    public transactionStorage: TransactionStorage | undefined;
+    async initStorage(): Promise<IStorage<T>> {
+        // If a Chrome extension, just return null. 
+        if (isInChromeExtension()) return null as any;
+        let fileStorage = await getFileStorage();
+        let collections = await fileStorage.folder.getStorage("collections");
+        let curCollection = await collections.folder.getStorage(this.collectionName);
+        let baseStorage = new TransactionStorage(curCollection, this.collectionName, this.writeDelay);
+        this.transactionStorage = baseStorage;
+        return new JSONStorage<T>(baseStorage);
+    }
+    public baseStorage = this.initStorage();
+    private synced = new StorageSync(
+        new PendingStorage(`Collection (${this.collectionName})`,
+            new DelayedStorage<T>(this.baseStorage)
+        )
+    );
+
+    public get(key: string): T | undefined {
+        return this.synced.get(key);
+    }
+    public async getPromise(key: string): Promise<T | undefined> {
+        let base = await this.baseStorage;
+        return base.get(key);
+    }
+    public set(key: string, value: T): void {
+        this.synced.set(key, value);
+    }
+    public remove(key: string): void {
+        this.synced.remove(key);
+    }
+    public getKeys(): string[] {
+        return this.synced.getKeys();
+    }
+    public getKeysPromise(): Promise<string[]> {
+        return this.synced.getKeysPromise();
+    }
+
+    public getEntries(): [string, T][] {
+        let keys = this.getKeys();
+        return keys.map(key => [key, this.get(key)]).filter(([_, value]) => isDefined(value)) as [string, T][];
+    }
+    public getValues(): T[] {
+        let keys = this.getKeys();
+        return keys.map(key => this.get(key)).filter(isDefined);
+    }
+    public async getValuesPromise(): Promise<T[]> {
+        let keys = await this.getKeysPromise();
+        let values: T[] = [];
+        for (let key of keys) {
+            let value = await this.getPromise(key);
+            if (isDefined(value)) {
+                values.push(value);
+            }
+        }
+        return values;
+    }
+    public getInfo(key: string) {
+        return this.synced.getInfo(key);
+    }
+
+    public async reset() {
+        await this.synced.reset();
+    }
+}
+
+export class DiskCollectionBrowser<T> implements IStorageSync<T> {
+    constructor(
+        private collectionName: string,
+        private writeDelay?: number,
+    ) {
+    }
+    public transactionStorage: TransactionStorage | undefined;
+    async initStorage(): Promise<IStorage<T>> {
+        if (isNode()) return undefined as any;
+        let curCollection = await new PrivateFileSystemStorage(`collections/${this.collectionName}`);
+        let baseStorage = new TransactionStorage(curCollection, this.collectionName);
+        return new JSONStorage<T>(baseStorage);
+    }
+    public baseStorage = this.initStorage();
+    private synced = new StorageSync(
+        new PendingStorage(`Collection (${this.collectionName})`,
+            new DelayedStorage<T>(this.baseStorage)
+        )
+    );
+
+    public get(key: string): T | undefined {
+        return this.synced.get(key);
+    }
+    public async getPromise(key: string): Promise<T | undefined> {
+        let base = await this.baseStorage;
+        return base.get(key);
+    }
+    public set(key: string, value: T): void {
+        this.synced.set(key, value);
+    }
+    public remove(key: string): void {
+        this.synced.remove(key);
+    }
+    public getKeys(): string[] {
+        return this.synced.getKeys();
+    }
+    public getKeysPromise(): Promise<string[]> {
+        return this.synced.getKeysPromise();
+    }
+
+    public getEntries(): [string, T][] {
+        let keys = this.getKeys();
+        return keys.map(key => [key, this.get(key)]).filter(([_, value]) => isDefined(value)) as [string, T][];
+    }
+    public getValues(): T[] {
+        let keys = this.getKeys();
+        return keys.map(key => this.get(key)).filter(isDefined);
+    }
+    public async getValuesPromise(): Promise<T[]> {
+        let keys = await this.getKeysPromise();
+        let values: T[] = [];
+        for (let key of keys) {
+            let value = await this.getPromise(key);
+            if (isDefined(value)) {
+                values.push(value);
+            }
+        }
+        return values;
+    }
+    public getInfo(key: string) {
+        return this.synced.getInfo(key);
+    }
+
+    public async reset() {
+        await this.synced.reset();
+    }
+}
+
+export class DiskCollectionPromise<T> implements IStorage<T> {
+    constructor(
+        private collectionName: string,
+        private writeDelay?: number,
+    ) { }
+    async initStorage(): Promise<IStorage<T>> {
+        let fileStorage = await getFileStorage();
+        let collections = await fileStorage.folder.getStorage("collections");
+        let curCollection = await collections.folder.getStorage(this.collectionName);
+        let baseStorage = new TransactionStorage(curCollection, this.collectionName, this.writeDelay);
+        return new JSONStorage<T>(baseStorage);
+    }
+    private synced = (
+        new PendingStorage(`Collection (${this.collectionName})`,
+            new DelayedStorage<T>(this.initStorage())
+        )
+    );
+
+    public async get(key: string): Promise<T | undefined> {
+        return await this.synced.get(key);
+    }
+    public async set(key: string, value: T): Promise<void> {
+        await this.synced.set(key, value);
+    }
+    public async remove(key: string): Promise<void> {
+        await this.synced.remove(key);
+    }
+    public async getKeys(): Promise<string[]> {
+        return await this.synced.getKeys();
+    }
+    public async getInfo(key: string) {
+        return await this.synced.getInfo(key);
+    }
+
+    public async reset() {
+        await this.synced.reset();
+    }
+}
+
+export class DiskCollectionRaw implements IStorage<Buffer> {
+    constructor(private collectionName: string) { }
+    async initStorage(): Promise<IStorage<Buffer>> {
+        let fileStorage = await getFileStorage();
+        let collections = await fileStorage.folder.getStorage("collections");
+        let baseStorage = await collections.folder.getStorage(this.collectionName);
+        return baseStorage;
+    }
+    private synced = (
+        new PendingStorage(`Collection (${this.collectionName})`,
+            new DelayedStorage(this.initStorage())
+        )
+    );
+
+    public async get(key: string): Promise<Buffer | undefined> {
+        return await this.synced.get(key);
+    }
+    public async set(key: string, value: Buffer): Promise<void> {
+        await this.synced.set(key, value);
+    }
+    public async remove(key: string): Promise<void> {
+        await this.synced.remove(key);
+    }
+    public async getKeys(): Promise<string[]> {
+        return await this.synced.getKeys();
+    }
+    public async getInfo(key: string) {
+        return await this.synced.getInfo(key);
+    }
+
+    public async reset() {
+        await this.synced.reset();
+    }
+}
+
+export class DiskCollectionRawBrowser {
+    constructor(private collectionName: string) { }
+    async initStorage(): Promise<IStorage<Buffer>> {
+        return await new PrivateFileSystemStorage(`collections/${this.collectionName}`);
+    }
+    private synced = new StorageSync(
+        new PendingStorage(`Collection (${this.collectionName})`,
+            new DelayedStorage(this.initStorage())
+        )
+    );
+
+    public get(key: string): Buffer | undefined {
+        return this.synced.get(key);
+    }
+
+    public async getPromise(key: string): Promise<Buffer | undefined> {
+        return await this.synced.get(key);
+    }
+    public set(key: string, value: Buffer) {
+        this.synced.set(key, value);
+    }
+    public async getKeys(): Promise<string[]> {
+        return await this.synced.getKeys();
+    }
+    public async getInfo(key: string) {
+        return await this.synced.getInfo(key);
+    }
+
+    public async reset() {
+        await this.synced.reset();
+    }
+}
+
+export function newFileStorageBufferSyncer(folder = "") {
+    let base = new PendingStorage(`FileStorageBufferSyncer`,
+        new DelayedStorage(getFileStorageNested(folder))
+    );
+    return new StorageSync(base);
+}
+
+export function newFileStorageJSONSyncer<T>(folder = "") {
+    let base = new PendingStorage(`FileStorageJSONSyncer`,
+        new DelayedStorage(getFileStorageNested(folder))
+    );
+    return new StorageSync(new JSONStorage<T>(base));
+}

package/storage/FileFolderAPI.tsx

@@ -0,0 +1,427 @@
+import preact from "preact";
+import { getFileSystemPointer, storeFileSystemPointer } from "./fileSystemPointer";
+import { observable } from "mobx";
+import { observer } from "../web/observer";
+import { cache, lazy } from "socket-function/src/caching";
+import { css, isNode } from "typesafecss";
+import { IStorageRaw } from "./IStorage";
+import { runInSerial } from "socket-function/src/batching";
+import { getFileStorageIndexDB } from "./IndexedDBFileFolderAPI";
+import fs from "fs";
+import path from "path";
+
+// NOTE: IndexedDB is required for iOS, at least. We MIGHT want to make
+//  this a user supported toggle too, so they can choose during runtime if they want it.
+// DO NOT enable this is isNode
+const USE_INDEXED_DB = false;
+
+type FileWrapper = {
+    getFile(): Promise<{
+        size: number;
+        lastModified: number;
+        arrayBuffer(): Promise<ArrayBuffer>;
+    }>;
+    createWritable(config?: { keepExistingData?: boolean }): Promise<{
+        seek(offset: number): Promise<void>;
+        write(value: Buffer): Promise<void>;
+        close(): Promise<void>;
+    }>;
+};
+type DirectoryWrapper = {
+    removeEntry(key: string, options?: { recursive?: boolean }): Promise<void>;
+    getFileHandle(key: string, options?: { create?: boolean }): Promise<FileWrapper>;
+    getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper>;
+    [Symbol.asyncIterator](): AsyncIterableIterator<[string, {
+        kind: "file";
+        name: string;
+        getFile(): Promise<FileWrapper>;
+    } | {
+        kind: "directory";
+        name: string;
+        getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper>;
+    }]>;
+};
+
+let displayData = observable({
+    ui: undefined as undefined | preact.ComponentChildren,
+}, undefined, { deep: false });
+
+const storageKey = "syncFileSystemCamera3";
+
+@observer
+class DirectoryPrompter extends preact.Component {
+    render() {
+        if (!displayData.ui) return undefined;
+        return (
+            <div className={
+                css.position("fixed").pos(0, 0).size("100vw", "100vh")
+                    .zIndex(1)
+                    .background("white")
+                    .center
+                    .fontSize(40)
+            }>
+                {displayData.ui}
+            </div>
+        );
+    }
+}
+
+class NodeJSFileHandleWrapper implements FileWrapper {
+    constructor(private filePath: string) {
+    }
+
+    async getFile() {
+        const stats = await fs.promises.stat(this.filePath);
+        return {
+            size: stats.size,
+            lastModified: stats.mtimeMs,
+            arrayBuffer: async () => {
+                const buffer = await fs.promises.readFile(this.filePath);
+                return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+            }
+        };
+    }
+
+    async createWritable(config?: { keepExistingData?: boolean }) {
+        let fileHandle: fs.promises.FileHandle;
+        const flags = config?.keepExistingData ? "r+" : "w";
+
+        // Ensure the directory exists
+        await fs.promises.mkdir(path.dirname(this.filePath), { recursive: true });
+
+        // Open or create the file
+        if (config?.keepExistingData && await fs.promises.access(this.filePath).then(() => true).catch(() => false)) {
+            fileHandle = await fs.promises.open(this.filePath, flags);
+        } else {
+            fileHandle = await fs.promises.open(this.filePath, "w");
+        }
+
+        let position = 0;
+
+        return {
+            seek: async (offset: number) => {
+                position = offset;
+            },
+            write: async (value: Buffer) => {
+                await fileHandle.write(value, 0, value.length, position);
+                position += value.length;
+            },
+            close: async () => {
+                await fileHandle.close();
+            }
+        };
+    }
+}
+
+class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
+    constructor(private rootPath: string) {
+    }
+
+    async removeEntry(key: string, options?: { recursive?: boolean }) {
+        const entryPath = path.join(this.rootPath, key);
+        if (options?.recursive) {
+            await fs.promises.rm(entryPath, { recursive: true, force: true });
+        } else {
+            await fs.promises.unlink(entryPath);
+        }
+    }
+
+    async getFileHandle(key: string, options?: { create?: boolean }): Promise<FileWrapper> {
+        const filePath = path.join(this.rootPath, key);
+
+        const exists = await fs.promises.access(filePath).then(() => true).catch(() => false);
+
+        if (!exists && options?.create) {
+            // Ensure the directory exists
+            await fs.promises.mkdir(path.dirname(filePath), { recursive: true });
+            // Create the file
+            await fs.promises.writeFile(filePath, Buffer.alloc(0));
+        } else if (!exists) {
+            throw new Error(`File not found: ${filePath}`);
+        }
+
+        return new NodeJSFileHandleWrapper(filePath);
+    }
+
+    async getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper> {
+        const dirPath = path.join(this.rootPath, key);
+
+        if (options?.create) {
+            await fs.promises.mkdir(dirPath, { recursive: true });
+        } else {
+            const exists = await fs.promises.access(dirPath).then(() => true).catch(() => false);
+            if (!exists) {
+                throw new Error(`Directory not found: ${dirPath}`);
+            }
+        }
+
+        return new NodeJSDirectoryHandleWrapper(dirPath);
+    }
+
+    async *[Symbol.asyncIterator](): AsyncIterableIterator<[string, {
+        kind: "file";
+        name: string;
+        getFile(): Promise<FileWrapper>;
+    } | {
+        kind: "directory";
+        name: string;
+        getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper>;
+    }]> {
+        // Ensure directory exists
+        await fs.promises.mkdir(this.rootPath, { recursive: true });
+
+        const entries = await fs.promises.readdir(this.rootPath, { withFileTypes: true });
+
+        for (const entry of entries) {
+            if (entry.isFile()) {
+                yield [entry.name, {
+                    kind: "file",
+                    name: entry.name,
+                    getFile: async () => new NodeJSFileHandleWrapper(path.join(this.rootPath, entry.name))
+                }];
+            } else if (entry.isDirectory()) {
+                const dirPath = path.join(this.rootPath, entry.name);
+                yield [entry.name, {
+                    kind: "directory",
+                    name: entry.name,
+                    getDirectoryHandle: async (key: string, options?: { create?: boolean }) => {
+                        return new NodeJSDirectoryHandleWrapper(dirPath).getDirectoryHandle(key, options);
+                    }
+                }];
+            }
+        }
+    }
+}
+
+
+// NOTE: Blocks until the user provides a directory
+export const getDirectoryHandle = lazy(async function getDirectoryHandle(): Promise<DirectoryWrapper> {
+    if (isNode()) {
+        return new NodeJSDirectoryHandleWrapper(path.resolve("./data/"));
+    }
+    let root = document.createElement("div");
+    document.body.appendChild(root);
+    preact.render(<DirectoryPrompter />, root);
+    try {
+
+        let handle: DirectoryWrapper | undefined;
+
+        let storedId = localStorage.getItem(storageKey);
+        if (storedId) {
+            let doneLoad = false;
+            setTimeout(() => {
+                if (doneLoad) return;
+                console.log("Waiting for user to click");
+                displayData.ui = "Click anywhere to allow file system access";
+            }, 500);
+            try {
+                handle = await tryToLoadPointer(storedId);
+            } catch { }
+            doneLoad = true;
+            if (handle) {
+                return handle;
+            }
+        }
+        let fileCallback: (handle: DirectoryWrapper) => void;
+        let promise = new Promise<DirectoryWrapper>(resolve => {
+            fileCallback = resolve;
+        });
+        displayData.ui = (
+            <button
+                className={css.fontSize(40).pad2(80, 40)}
+                onClick={async () => {
+                    console.log("Waiting for user to give permission");
+                    const handle = await window.showDirectoryPicker();
+                    await handle.requestPermission({ mode: "readwrite" });
+                    let storedId = await storeFileSystemPointer({ mode: "readwrite", handle });
+                    localStorage.setItem(storageKey, storedId);
+                    fileCallback(handle as any);
+                }}
+            >
+                Pick Media Directory
+            </button>
+        );
+        return await promise;
+    } finally {
+        preact.render(null, root);
+        root.remove();
+    }
+});
+
+export const getFileStorageNested = cache(async function getFileStorage(path: string): Promise<FileStorage> {
+    let base = await getDirectoryHandle();
+    for (let part of path.split("/")) {
+        if (!part) continue;
+        base = await base.getDirectoryHandle(part, { create: true });
+    }
+    return wrapHandle(base);
+});
+export const getFileStorage = lazy(async function getFileStorage(): Promise<FileStorage> {
+    if (USE_INDEXED_DB) {
+        return await getFileStorageIndexDB();
+    }
+
+    let handle = await getDirectoryHandle();
+    return wrapHandle(handle);
+});
+export function resetStorageLocation() {
+    localStorage.removeItem(storageKey);
+    window.location.reload();
+}
+
+export type NestedFileStorage = {
+    hasKey(key: string): Promise<boolean>;
+    getStorage(key: string): Promise<FileStorage>;
+    removeStorage(key: string): Promise<void>;
+    getKeys(): Promise<string[]>;
+};
+
+export type FileStorage = IStorageRaw & {
+    folder: NestedFileStorage;
+};
+
+let appendQueue = cache((key: string) => {
+    return runInSerial((fnc: () => Promise<void>) => fnc());
+});
+
+
+async function fixedGetFileHandle(config: {
+    handle: DirectoryWrapper;
+    key: string;
+    create: true;
+}): Promise<FileWrapper>;
+async function fixedGetFileHandle(config: {
+    handle: DirectoryWrapper;
+    key: string;
+    create?: boolean;
+}): Promise<FileWrapper | undefined>;
+async function fixedGetFileHandle(config: {
+    handle: DirectoryWrapper;
+    key: string;
+    create?: boolean;
+}): Promise<FileWrapper | undefined> {
+    if (config.key.includes("/")) {
+        throw new Error(`Cannot use folders directly in file system read / writes. Use a wrapper which handles the folder navigation. Path was ${JSON.stringify(config.key)}`);
+    }
+    // ALWAYS try without create, because the sshfs-win sucks and doesn't support `create: true`? Wtf...
+    try {
+        return await config.handle.getFileHandle(config.key);
+    } catch {
+        if (!config.create) return undefined;
+    }
+    return await config.handle.getFileHandle(config.key, { create: true });
+}
+
+function wrapHandleFiles(handle: DirectoryWrapper): IStorageRaw {
+    return {
+        async getInfo(key: string) {
+            try {
+                const file = await handle.getFileHandle(key);
+                const fileContent = await file.getFile();
+                return {
+                    size: fileContent.size,
+                    lastModified: fileContent.lastModified,
+                };
+            } catch (error) {
+                return undefined;
+            }
+        },
+        async get(key: string): Promise<Buffer | undefined> {
+            try {
+                const file = await handle.getFileHandle(key);
+                const fileContent = await file.getFile();
+                const arrayBuffer = await fileContent.arrayBuffer();
+                return Buffer.from(arrayBuffer);
+            } catch (error) {
+                return undefined;
+            }
+        },
+
+        async append(key: string, value: Buffer): Promise<void> {
+            await appendQueue(key)(async () => {
+                // NOTE: Interesting point. Chrome doesn't optimize this to be an append, and instead
+                //  rewrites the entire file.
+                const file = await fixedGetFileHandle({ handle, key, create: true });
+                const writable = await file.createWritable({ keepExistingData: true });
+                let offset = (await file.getFile()).size;
+                await writable.seek(offset);
+                await writable.write(value);
+                await writable.close();
+            });
+        },
+
+        async set(key: string, value: Buffer): Promise<void> {
+            const file = await fixedGetFileHandle({ handle, key, create: true });
+            const writable = await file.createWritable();
+            await writable.write(value);
+            await writable.close();
+        },
+
+        async remove(key: string): Promise<void> {
+            await handle.removeEntry(key);
+        },
+
+        async getKeys(): Promise<string[]> {
+            const keys: string[] = [];
+            for await (const [name, entry] of handle) {
+                if (entry.kind === "file") {
+                    keys.push(entry.name);
+                }
+            }
+            return keys;
+        },
+
+        async reset() {
+            for await (const [name, entry] of handle) {
+                await handle.removeEntry(entry.name, { recursive: true });
+            }
+        },
+    };
+}
+
+function wrapHandleNested(handle: DirectoryWrapper): NestedFileStorage {
+    return {
+        async hasKey(key: string): Promise<boolean> {
+            try {
+                await handle.getDirectoryHandle(key);
+                return true;
+            } catch (error) {
+                return false;
+            }
+        },
+
+        async getStorage(key: string): Promise<FileStorage> {
+            const subDirectory = await handle.getDirectoryHandle(key, { create: true });
+            return wrapHandle(subDirectory);
+        },
+
+        async removeStorage(key: string): Promise<void> {
+            await handle.removeEntry(key, { recursive: true });
+        },
+
+        async getKeys(): Promise<string[]> {
+            const keys: string[] = [];
+            for await (const [name, entry] of handle) {
+                if (entry.kind === "directory") {
+                    keys.push(entry.name);
+                }
+            }
+            return keys;
+        },
+    };
+}
+
+function wrapHandle(handle: DirectoryWrapper): FileStorage {
+    return {
+        ...wrapHandleFiles(handle),
+        folder: wrapHandleNested(handle),
+    };
+}
+
+async function tryToLoadPointer(pointer: string) {
+    let result = await getFileSystemPointer({ pointer });
+    if (!result) return;
+    let handle = await result?.onUserActivation();
+    if (!handle) return;
+    return handle as any as DirectoryWrapper;
+}