querysub 0.591.0 → 0.593.0

package/src/-a-archives/archivesLimitedCache.ts

@@ -1,307 +0,0 @@
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { Archives } from "./archives";
-import { cache } from "socket-function/src/caching";
-import { measureFnc } from "socket-function/src/profiling/measure";
-import { batchFunction, runInfinitePoll } from "socket-function/src/batching";
-import { timeInHour } from "socket-function/src/misc";
-
-interface FileInfo {
-    writeTime: number;
-    accessTime: number;
-    size: number;
-}
-
-interface IndexData {
-    files: Record<string, FileInfo>;
-}
-
-class ArchivesLimitedCache {
-    private baseArchives: Archives;
-    private maxFiles: number;
-    private maxSize: number;
-    private cache = new Map<string, FileInfo>();
-    private initialized = false;
-    private readonly indexPath = ".cache-index.json";
-
-    constructor(baseArchives: Archives, config: { maxFiles: number; maxSize: number }) {
-        this.baseArchives = baseArchives;
-        this.maxFiles = config.maxFiles;
-        this.maxSize = config.maxSize;
-        this.initOptionalMethods();
-        this.initPeriodicSync();
-    }
-
-    public getDebugName(): string {
-        return `limitedCache(${this.maxFiles}files,${Math.round(this.maxSize / (1024 * 1024))}MB)/${this.baseArchives.getDebugName()}`;
-    }
-
-    @measureFnc
-    private async rebuildCacheFromFiles(): Promise<void> {
-        const allFiles = await this.baseArchives.findInfo("", { type: "files" });
-        const newCache = new Map<string, FileInfo>();
-
-        // Build map of actual files (excluding index file)
-        for (const file of allFiles) {
-            if (file.path === this.indexPath) continue;
-
-            newCache.set(file.path, {
-                writeTime: file.createTime,
-                accessTime: this.cache.get(file.path)?.accessTime ?? file.createTime,
-                size: file.size,
-            });
-        }
-
-        // Update cache with actual files
-        this.cache.clear();
-        for (const [path, info] of newCache.entries()) {
-            this.cache.set(path, info);
-        }
-    }
-
-    @measureFnc
-    private async ensureInitialized(): Promise<void> {
-        if (this.initialized) return;
-
-        // First try to load from index file
-        await this.loadIndex();
-
-        // If index is empty or missing, fall back to scanning all files
-        if (this.cache.size === 0) {
-            console.log("Index file missing or empty, scanning all files...");
-            await this.rebuildCacheFromFiles();
-            // Save the newly built index
-            await this.saveIndex();
-        }
-
-        this.initialized = true;
-
-        // Cleanup if we're already over limits
-        await this.applyLimits();
-    }
-
-    private async applyLimits(): Promise<void> {
-        const files = Array.from(this.cache.entries()).map(([path, info]) => ({
-            path,
-            ...info,
-        }));
-
-        const totalSize = files.reduce((sum, file) => sum + file.size, 0);
-        const totalFiles = files.length;
-
-        if (totalFiles <= this.maxFiles && totalSize <= this.maxSize) {
-            return; // No cleanup needed
-        }
-
-        // Sort by access time (oldest first)
-        files.sort((a, b) => a.accessTime - b.accessTime);
-
-        let currentSize = totalSize;
-        let currentFiles = totalFiles;
-
-        // Remove files until we're under both limits
-        for (const file of files) {
-            if (currentFiles <= this.maxFiles && currentSize <= this.maxSize) {
-                break;
-            }
-
-            console.warn(`Deleting file ${file.path} to maintain limits. Currently have ${formatNumber(currentFiles)} files and ${formatNumber(currentSize)}B`);
-            try {
-                await this.baseArchives.del(file.path);
-                this.cache.delete(file.path);
-                currentSize -= file.size;
-                currentFiles--;
-            } catch (error) {
-                // File might already be deleted, continue
-                console.warn(`Failed to delete file ${file.path}:`, error);
-                this.cache.delete(file.path); // Remove from cache anyway
-                currentSize -= file.size;
-                currentFiles--;
-            }
-        }
-
-        // Trigger index flush if any files were deleted
-        if (currentFiles < totalFiles) {
-            this.triggerIndexFlush();
-        }
-    }
-
-    private updateAccessTime(path: string): void {
-        const info = this.cache.get(path);
-        if (info) {
-            info.accessTime = Date.now();
-            this.triggerIndexFlush();
-        }
-    }
-
-    public async get(path: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
-        await this.ensureInitialized();
-        this.updateAccessTime(path);
-        return this.baseArchives.get(path, config);
-    }
-
-    public async set(path: string, data: Buffer): Promise<void> {
-        await this.ensureInitialized();
-        // Limit before, not after, so we don't remove the file we just added
-        await this.applyLimits();
-        await this.baseArchives.set(path, data);
-        const now = Date.now();
-        this.cache.set(path, {
-            writeTime: now,
-            accessTime: now,
-            size: data.length,
-        });
-        this.triggerIndexFlush();
-    }
-
-    public async append(path: string, data: Buffer): Promise<void> {
-        await this.ensureInitialized();
-        await this.baseArchives.append(path, data);
-        // Update the cache info - increment size and update times
-        const info = this.cache.get(path);
-        const now = Date.now();
-        if (info) {
-            info.size += data.length;
-            info.writeTime = now;
-            info.accessTime = now;
-        } else {
-            // File didn't exist in cache, add it
-            this.cache.set(path, {
-                writeTime: now,
-                accessTime: now,
-                size: data.length,
-            });
-        }
-        this.triggerIndexFlush();
-    }
-
-    public async del(path: string): Promise<void> {
-        await this.ensureInitialized();
-        await this.baseArchives.del(path);
-        this.cache.delete(path);
-        this.triggerIndexFlush();
-    }
-
-    public async getInfo(path: string): Promise<{ writeTime: number; size: number; } | undefined> {
-        await this.ensureInitialized();
-        this.updateAccessTime(path);
-        return this.baseArchives.getInfo(path);
-    }
-
-    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
-        throw new Error(`setLargeFile in archivesLimitedCache is not supported`);
-    }
-
-    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
-        await this.ensureInitialized();
-        return this.baseArchives.find(prefix, config);
-    }
-
-    public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
-        await this.ensureInitialized();
-        return this.baseArchives.findInfo(prefix, config);
-    }
-
-    public async move(config: { path: string; target: Archives; targetPath: string }): Promise<void> {
-        throw new Error(`Move in archivesLimitedCache is not supported`);
-    }
-
-    public async copy(config: { path: string; target: Archives; targetPath: string }): Promise<void> {
-        throw new Error(`Copy in archivesLimitedCache is not supported`);
-    }
-
-    public enableLogging(): void {
-        this.baseArchives.enableLogging();
-    }
-
-    public async assertPathValid(path: string): Promise<void> {
-        await this.baseArchives.assertPathValid(path);
-    }
-
-    public getBaseArchives?: () => { parentPath: string; archives: Archives; } | undefined = () => {
-        const base = this.baseArchives.getBaseArchives?.();
-        if (base) {
-            return base;
-        }
-        return { parentPath: "", archives: this.baseArchives };
-    };
-
-    public getURL?: (path: string) => Promise<string>;
-    public getDownloadAuthorization?: (config: { validDurationInSeconds: number }) => Promise<{ bucketId: string; fileNamePrefix: string; authorizationToken: string; }>;
-
-    private batchedFlushIndex = batchFunction(
-        { delay: 2000 }, // Flush index changes every 2 seconds
-        async () => {
-            // We don't actually need the batched values, just flush the index
-            await this.saveIndex();
-        }
-    );
-
-    private async loadIndex(): Promise<void> {
-        try {
-            const indexBuffer = await this.baseArchives.get(this.indexPath);
-            if (!indexBuffer) {
-                // No existing index, will need to build it
-                return;
-            }
-
-            const indexData: IndexData = JSON.parse(indexBuffer.toString("utf8"));
-
-            // Load the cache from the index
-            this.cache.clear();
-            for (const [path, info] of Object.entries(indexData.files)) {
-                this.cache.set(path, info);
-            }
-
-            console.log(`Loaded index with ${this.cache.size} files`);
-        } catch (error) {
-            console.warn(`Failed to load index file:`, error);
-            // Continue without index, will rebuild
-        }
-    }
-
-    private async saveIndex(): Promise<void> {
-        try {
-            const indexData: IndexData = {
-                files: Object.fromEntries(this.cache.entries())
-            };
-
-            const indexBuffer = Buffer.from(JSON.stringify(indexData), "utf8");
-            await this.baseArchives.set(this.indexPath, indexBuffer);
-        } catch (error) {
-            console.warn(`Failed to save index file:`, error);
-        }
-    }
-
-    private triggerIndexFlush(): void {
-        void this.batchedFlushIndex(undefined);
-    }
-
-    private initPeriodicSync(): void {
-        // Sync index with actual files every hour to prevent drift
-        runInfinitePoll(timeInHour, async () => {
-            await this.syncIndexWithFiles();
-        });
-    }
-
-    private async syncIndexWithFiles(): Promise<void> {
-        try {
-            console.log("Syncing index with actual files...");
-            await this.rebuildCacheFromFiles();
-
-            // Save updated index
-            await this.saveIndex();
-            console.log(`Index synced with ${this.cache.size} actual files`);
-        } catch (error) {
-            console.warn("Failed to sync index with files:", error);
-        }
-    }
-
-    private initOptionalMethods(): void {
-        this.getURL = this.baseArchives.getURL;
-        this.getDownloadAuthorization = this.baseArchives.getDownloadAuthorization;
-    }
-}
-
-export function createArchivesLimitedCache(baseArchives: Archives, config: { maxFiles: number; maxSize: number }): Archives {
-    return new ArchivesLimitedCache(baseArchives, config);
-}

package/src/-a-archives/archivesPrivateFileSystem.ts

@@ -1,326 +0,0 @@
-/// <reference path="../src.d.ts" />
-
-import { measureFnc } from "socket-function/src/profiling/measure";
-import { blue } from "socket-function/src/formatting/logColors";
-import { Archives } from "./archives";
-import { cache, lazy } from "socket-function/src/caching";
-import { delay } from "socket-function/src/batching";
-import { isDefined } from "../misc";
-import { formatTime } from "socket-function/src/formatting/format";
-
-let getRootDirectory = lazy(async () => {
-    await navigator.storage.persist();
-    return navigator.storage.getDirectory();
-});
-
-class ArchivesPrivateFileSystem {
-    public LAG = 0;
-    private rootDir: string;
-
-    private logging = false;
-    public enableLogging() {
-        this.logging = true;
-    }
-    private log(text: string) {
-        if (!this.logging) return;
-        console.log(text);
-    }
-
-    constructor(rootDir: string) {
-        this.rootDir = rootDir;
-        // Ensure rootDir ends with "/" for consistency
-        if (!this.rootDir.endsWith("/")) {
-            this.rootDir = this.rootDir + "/";
-        }
-    }
-
-    public getDebugName() {
-        return "privateFS/" + this.rootDir;
-    }
-
-    private async getOrCreateDirectory(path: string): Promise<FileSystemDirectoryHandle> {
-        const root = await getRootDirectory();
-        const pathParts = (this.rootDir + path).split("/").filter(part => part.length > 0);
-
-        let currentDir = root;
-        for (const part of pathParts) {
-            currentDir = await currentDir.getDirectoryHandle(part, { create: true });
-        }
-        return currentDir;
-    }
-
-    private async ensureDirsExist(path: string) {
-        const pathParts = path.split("/");
-        // Create directories up to the parent of the file (don't include the file itself)
-        if (pathParts.length > 1) {
-            const dirPath = pathParts.slice(0, -1).join("/");
-            await this.getOrCreateDirectory(dirPath);
-        }
-    }
-
-    @measureFnc
-    public async set(fileName: string, data: Buffer): Promise<void> {
-        this.log(blue(`Setting file ${fileName} = ${data.length} bytes`));
-
-        await this.ensureDirsExist(fileName);
-
-        const pathParts = fileName.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        const directory = await this.getOrCreateDirectory(dirPath);
-        const fileHandle = await directory.getFileHandle(filename, { create: true });
-        const writable = await fileHandle.createWritable();
-        await writable.write(data);
-        await writable.close();
-    }
-
-    @measureFnc
-    public async append(fileName: string, data: Buffer): Promise<void> {
-        this.log(blue(`Appending to file ${fileName} += ${data.length} bytes`));
-
-        await this.ensureDirsExist(fileName);
-
-        const pathParts = fileName.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        const directory = await this.getOrCreateDirectory(dirPath);
-        const fileHandle = await directory.getFileHandle(filename, { create: true });
-
-        // Get current file to find the size for appending at the end
-        const file = await fileHandle.getFile();
-        const currentSize = file.size;
-
-        const writable = await fileHandle.createWritable({ keepExistingData: true });
-        await writable.write({ type: "write", position: currentSize, data });
-        await writable.close();
-    }
-
-    @measureFnc
-    public async del(fileName: string): Promise<void> {
-        this.log(blue(`Deleting file ${fileName}`));
-
-        const pathParts = fileName.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        try {
-            const directory = await this.getOrCreateDirectory(dirPath);
-            await directory.removeEntry(filename);
-        } catch (e: any) {
-            // File might not exist, which is fine for delete operations
-            if (e.name !== "NotFoundError") {
-                throw e;
-            }
-        }
-    }
-
-    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
-        let { path } = config;
-        await this.ensureDirsExist(path);
-
-        const pathParts = path.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        const directory = await this.getOrCreateDirectory(dirPath);
-        const fileHandle = await directory.getFileHandle(filename, { create: true });
-        const writable = await fileHandle.createWritable();
-
-        try {
-            while (true) {
-                let data = await config.getNextData();
-                if (!data?.length) break;
-                await writable.write(data);
-            }
-        } finally {
-            await writable.close();
-        }
-    }
-
-    @measureFnc
-    public async get(fileName: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
-        this.log(blue(`Start read file ${fileName}`));
-
-        const pathParts = fileName.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        try {
-            const directory = await this.getOrCreateDirectory(dirPath);
-            const fileHandle = await directory.getFileHandle(filename, { create: false });
-            const file = await fileHandle.getFile();
-
-            if (config?.range) {
-                const start = config.range.start;
-                const end = config.range.end;
-                const slice = file.slice(start, end);
-                const arrayBuffer = await slice.arrayBuffer();
-                return Buffer.from(arrayBuffer);
-            } else {
-                const arrayBuffer = await file.arrayBuffer();
-                return Buffer.from(arrayBuffer);
-            }
-        } catch (e: any) {
-            if (e.name === "NotFoundError") {
-                return undefined;
-            }
-            throw e;
-        }
-    }
-
-    @measureFnc
-    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
-        this.log(blue(`findFileNames ${prefix}`));
-
-        let fileNames: string[] = [];
-        let folderNames: string[] = [];
-
-        async function readDir(directory: FileSystemDirectoryHandle, currentPath: string) {
-            for await (const [name, handle] of directory) {
-                const fullPath = currentPath + (currentPath ? "/" : "") + name;
-
-                if (handle.kind === "directory") {
-                    folderNames.push(fullPath);
-                    if (!config?.shallow) {
-                        await readDir(handle, fullPath);
-                    }
-                } else {
-                    fileNames.push(fullPath);
-                }
-            }
-        }
-
-        try {
-            let readDirTime = Date.now();
-            // Start from the root directory of our namespace
-            const rootDirectory = await this.getOrCreateDirectory("");
-            await readDir(rootDirectory, "");
-            console.log(`readDir took ${formatTime(Date.now() - readDirTime)}`);
-
-            let results = config?.type === "folders" ? folderNames : fileNames;
-            results = results.filter(name => name.startsWith(prefix));
-
-            if (config?.shallow) {
-                let targetDepth = prefix.split("/").length;
-                results = results.filter(name => name.split("/").length === targetDepth);
-            }
-
-            return results;
-        } catch (e: any) {
-            if (e.name === "NotFoundError") {
-                return [];
-            }
-            throw e;
-        }
-    }
-
-    @measureFnc
-    public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
-        let files = await this.find(prefix, config);
-        return (await Promise.all(files.map(async file => {
-            try {
-                const info = await this.getInfo(file);
-                if (!info) return undefined;
-                return { path: file, createTime: info.writeTime, size: info.size };
-            } catch {
-                return undefined;
-            }
-        }))).filter(isDefined);
-    }
-
-    @measureFnc
-    public async getInfo(pathInput: string): Promise<{
-        writeTime: number;
-        size: number;
-    } | undefined> {
-        const pathParts = pathInput.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        try {
-            const directory = await this.getOrCreateDirectory(dirPath);
-            const fileHandle = await directory.getFileHandle(filename, { create: false });
-            const file = await fileHandle.getFile();
-
-            return {
-                writeTime: file.lastModified,
-                size: file.size,
-            };
-        } catch (e: any) {
-            if (e.name === "NotFoundError") {
-                return undefined;
-            }
-            throw e;
-        }
-    }
-
-    public async assertPathValid(path: string) {
-        // Private file system has fewer restrictions than regular file system
-        // Most characters are allowed, so we just check for obviously problematic patterns
-        if (path.includes("..")) {
-            throw new Error(`Invalid path contains '..': ${path}`);
-        }
-        if (path.startsWith("/")) {
-            throw new Error(`Invalid path starts with '/': ${path}`);
-        }
-    }
-
-    @measureFnc
-    public async move(config: {
-        path: string;
-        target: Archives;
-        targetPath: string;
-    }) {
-        let { path, target, targetPath } = config;
-
-        // Unwrap any nested archives
-        while (true) {
-            let targetUnwrapped = target.getBaseArchives?.();
-            if (!targetUnwrapped) break;
-            target = targetUnwrapped.archives;
-            targetPath = targetUnwrapped.parentPath + targetPath;
-        }
-
-        // For moves, we can optimize if both source and target are private file system
-        if (target instanceof ArchivesPrivateFileSystem) {
-            // Private File System API doesn't have a native move operation
-            // So we fall back to copy + delete
-            await this.copy({ path, target, targetPath });
-            await this.del(path);
-        } else {
-            // Moving to a different archive type - copy then delete
-            let data = await this.get(path);
-            if (!data) throw new Error(`File not found to move: ${path}`);
-            await target.set(targetPath, data);
-            await this.del(path);
-        }
-    }
-
-    @measureFnc
-    public async copy(config: {
-        path: string;
-        target: Archives;
-        targetPath: string;
-    }) {
-        let { path, target, targetPath } = config;
-
-        // Unwrap any nested archives
-        while (true) {
-            let targetUnwrapped = target.getBaseArchives?.();
-            if (!targetUnwrapped) break;
-            target = targetUnwrapped.archives;
-            targetPath = targetUnwrapped.parentPath + targetPath;
-        }
-
-        // Get the data and set it in the target
-        let data = await this.get(path);
-        if (!data) throw new Error(`File not found to copy: ${path}`);
-        await target.set(targetPath, data);
-    }
-}
-
-export const getArchivesPrivateFileSystem = cache((rootDir: string): Archives => {
-    return new ArchivesPrivateFileSystem(rootDir);
-});

package/src/-a-archives/copyLocalToBackblaze.ts

@@ -1,24 +0,0 @@
-import { formatNumber } from "socket-function/src/formatting/format";
-import { getDomain } from "../config";
-import { getArchivesBackblaze } from "./archivesBackBlaze";
-import { getArchivesLocal } from "./archivesDisk";
-
-async function main() {
-    const domain = getDomain();
-    let local = getArchivesLocal(domain);
-    let remote = getArchivesBackblaze(domain);
-
-    let files = await local.find("");
-    let allRemoteFiles = new Set(await remote.find(""));
-    for (let file of files) {
-        if (allRemoteFiles.has(file)) continue;
-        let buffer = await local.get(file);
-        if (!buffer) continue;
-        console.log(`Copying ${file} size ${formatNumber(buffer.length)}B`);
-        await remote.set(file, buffer);
-    }
-}
-main().catch(console.error).finally(() => process.exit(0));
-
-// archives = getArchivesLocal(domain);
-// archives = getArchivesBackblaze(domain);