querysub 0.590.0 → 0.592.0

package/src/-a-archives/archivesPrivateFileSystem.ts

@@ -1,326 +0,0 @@
-/// <reference path="../src.d.ts" />
-
-import { measureFnc } from "socket-function/src/profiling/measure";
-import { blue } from "socket-function/src/formatting/logColors";
-import { Archives } from "./archives";
-import { cache, lazy } from "socket-function/src/caching";
-import { delay } from "socket-function/src/batching";
-import { isDefined } from "../misc";
-import { formatTime } from "socket-function/src/formatting/format";
-
-let getRootDirectory = lazy(async () => {
-    await navigator.storage.persist();
-    return navigator.storage.getDirectory();
-});
-
-class ArchivesPrivateFileSystem {
-    public LAG = 0;
-    private rootDir: string;
-
-    private logging = false;
-    public enableLogging() {
-        this.logging = true;
-    }
-    private log(text: string) {
-        if (!this.logging) return;
-        console.log(text);
-    }
-
-    constructor(rootDir: string) {
-        this.rootDir = rootDir;
-        // Ensure rootDir ends with "/" for consistency
-        if (!this.rootDir.endsWith("/")) {
-            this.rootDir = this.rootDir + "/";
-        }
-    }
-
-    public getDebugName() {
-        return "privateFS/" + this.rootDir;
-    }
-
-    private async getOrCreateDirectory(path: string): Promise<FileSystemDirectoryHandle> {
-        const root = await getRootDirectory();
-        const pathParts = (this.rootDir + path).split("/").filter(part => part.length > 0);
-
-        let currentDir = root;
-        for (const part of pathParts) {
-            currentDir = await currentDir.getDirectoryHandle(part, { create: true });
-        }
-        return currentDir;
-    }
-
-    private async ensureDirsExist(path: string) {
-        const pathParts = path.split("/");
-        // Create directories up to the parent of the file (don't include the file itself)
-        if (pathParts.length > 1) {
-            const dirPath = pathParts.slice(0, -1).join("/");
-            await this.getOrCreateDirectory(dirPath);
-        }
-    }
-
-    @measureFnc
-    public async set(fileName: string, data: Buffer): Promise<void> {
-        this.log(blue(`Setting file ${fileName} = ${data.length} bytes`));
-
-        await this.ensureDirsExist(fileName);
-
-        const pathParts = fileName.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        const directory = await this.getOrCreateDirectory(dirPath);
-        const fileHandle = await directory.getFileHandle(filename, { create: true });
-        const writable = await fileHandle.createWritable();
-        await writable.write(data);
-        await writable.close();
-    }
-
-    @measureFnc
-    public async append(fileName: string, data: Buffer): Promise<void> {
-        this.log(blue(`Appending to file ${fileName} += ${data.length} bytes`));
-
-        await this.ensureDirsExist(fileName);
-
-        const pathParts = fileName.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        const directory = await this.getOrCreateDirectory(dirPath);
-        const fileHandle = await directory.getFileHandle(filename, { create: true });
-
-        // Get current file to find the size for appending at the end
-        const file = await fileHandle.getFile();
-        const currentSize = file.size;
-
-        const writable = await fileHandle.createWritable({ keepExistingData: true });
-        await writable.write({ type: "write", position: currentSize, data });
-        await writable.close();
-    }
-
-    @measureFnc
-    public async del(fileName: string): Promise<void> {
-        this.log(blue(`Deleting file ${fileName}`));
-
-        const pathParts = fileName.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        try {
-            const directory = await this.getOrCreateDirectory(dirPath);
-            await directory.removeEntry(filename);
-        } catch (e: any) {
-            // File might not exist, which is fine for delete operations
-            if (e.name !== "NotFoundError") {
-                throw e;
-            }
-        }
-    }
-
-    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
-        let { path } = config;
-        await this.ensureDirsExist(path);
-
-        const pathParts = path.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        const directory = await this.getOrCreateDirectory(dirPath);
-        const fileHandle = await directory.getFileHandle(filename, { create: true });
-        const writable = await fileHandle.createWritable();
-
-        try {
-            while (true) {
-                let data = await config.getNextData();
-                if (!data?.length) break;
-                await writable.write(data);
-            }
-        } finally {
-            await writable.close();
-        }
-    }
-
-    @measureFnc
-    public async get(fileName: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
-        this.log(blue(`Start read file ${fileName}`));
-
-        const pathParts = fileName.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        try {
-            const directory = await this.getOrCreateDirectory(dirPath);
-            const fileHandle = await directory.getFileHandle(filename, { create: false });
-            const file = await fileHandle.getFile();
-
-            if (config?.range) {
-                const start = config.range.start;
-                const end = config.range.end;
-                const slice = file.slice(start, end);
-                const arrayBuffer = await slice.arrayBuffer();
-                return Buffer.from(arrayBuffer);
-            } else {
-                const arrayBuffer = await file.arrayBuffer();
-                return Buffer.from(arrayBuffer);
-            }
-        } catch (e: any) {
-            if (e.name === "NotFoundError") {
-                return undefined;
-            }
-            throw e;
-        }
-    }
-
-    @measureFnc
-    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
-        this.log(blue(`findFileNames ${prefix}`));
-
-        let fileNames: string[] = [];
-        let folderNames: string[] = [];
-
-        async function readDir(directory: FileSystemDirectoryHandle, currentPath: string) {
-            for await (const [name, handle] of directory) {
-                const fullPath = currentPath + (currentPath ? "/" : "") + name;
-
-                if (handle.kind === "directory") {
-                    folderNames.push(fullPath);
-                    if (!config?.shallow) {
-                        await readDir(handle, fullPath);
-                    }
-                } else {
-                    fileNames.push(fullPath);
-                }
-            }
-        }
-
-        try {
-            let readDirTime = Date.now();
-            // Start from the root directory of our namespace
-            const rootDirectory = await this.getOrCreateDirectory("");
-            await readDir(rootDirectory, "");
-            console.log(`readDir took ${formatTime(Date.now() - readDirTime)}`);
-
-            let results = config?.type === "folders" ? folderNames : fileNames;
-            results = results.filter(name => name.startsWith(prefix));
-
-            if (config?.shallow) {
-                let targetDepth = prefix.split("/").length;
-                results = results.filter(name => name.split("/").length === targetDepth);
-            }
-
-            return results;
-        } catch (e: any) {
-            if (e.name === "NotFoundError") {
-                return [];
-            }
-            throw e;
-        }
-    }
-
-    @measureFnc
-    public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
-        let files = await this.find(prefix, config);
-        return (await Promise.all(files.map(async file => {
-            try {
-                const info = await this.getInfo(file);
-                if (!info) return undefined;
-                return { path: file, createTime: info.writeTime, size: info.size };
-            } catch {
-                return undefined;
-            }
-        }))).filter(isDefined);
-    }
-
-    @measureFnc
-    public async getInfo(pathInput: string): Promise<{
-        writeTime: number;
-        size: number;
-    } | undefined> {
-        const pathParts = pathInput.split("/");
-        const filename = pathParts[pathParts.length - 1];
-        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
-
-        try {
-            const directory = await this.getOrCreateDirectory(dirPath);
-            const fileHandle = await directory.getFileHandle(filename, { create: false });
-            const file = await fileHandle.getFile();
-
-            return {
-                writeTime: file.lastModified,
-                size: file.size,
-            };
-        } catch (e: any) {
-            if (e.name === "NotFoundError") {
-                return undefined;
-            }
-            throw e;
-        }
-    }
-
-    public async assertPathValid(path: string) {
-        // Private file system has fewer restrictions than regular file system
-        // Most characters are allowed, so we just check for obviously problematic patterns
-        if (path.includes("..")) {
-            throw new Error(`Invalid path contains '..': ${path}`);
-        }
-        if (path.startsWith("/")) {
-            throw new Error(`Invalid path starts with '/': ${path}`);
-        }
-    }
-
-    @measureFnc
-    public async move(config: {
-        path: string;
-        target: Archives;
-        targetPath: string;
-    }) {
-        let { path, target, targetPath } = config;
-
-        // Unwrap any nested archives
-        while (true) {
-            let targetUnwrapped = target.getBaseArchives?.();
-            if (!targetUnwrapped) break;
-            target = targetUnwrapped.archives;
-            targetPath = targetUnwrapped.parentPath + targetPath;
-        }
-
-        // For moves, we can optimize if both source and target are private file system
-        if (target instanceof ArchivesPrivateFileSystem) {
-            // Private File System API doesn't have a native move operation
-            // So we fall back to copy + delete
-            await this.copy({ path, target, targetPath });
-            await this.del(path);
-        } else {
-            // Moving to a different archive type - copy then delete
-            let data = await this.get(path);
-            if (!data) throw new Error(`File not found to move: ${path}`);
-            await target.set(targetPath, data);
-            await this.del(path);
-        }
-    }
-
-    @measureFnc
-    public async copy(config: {
-        path: string;
-        target: Archives;
-        targetPath: string;
-    }) {
-        let { path, target, targetPath } = config;
-
-        // Unwrap any nested archives
-        while (true) {
-            let targetUnwrapped = target.getBaseArchives?.();
-            if (!targetUnwrapped) break;
-            target = targetUnwrapped.archives;
-            targetPath = targetUnwrapped.parentPath + targetPath;
-        }
-
-        // Get the data and set it in the target
-        let data = await this.get(path);
-        if (!data) throw new Error(`File not found to copy: ${path}`);
-        await target.set(targetPath, data);
-    }
-}
-
-export const getArchivesPrivateFileSystem = cache((rootDir: string): Archives => {
-    return new ArchivesPrivateFileSystem(rootDir);
-});

package/src/-a-archives/copyLocalToBackblaze.ts

@@ -1,24 +0,0 @@
-import { formatNumber } from "socket-function/src/formatting/format";
-import { getDomain } from "../config";
-import { getArchivesBackblaze } from "./archivesBackBlaze";
-import { getArchivesLocal } from "./archivesDisk";
-
-async function main() {
-    const domain = getDomain();
-    let local = getArchivesLocal(domain);
-    let remote = getArchivesBackblaze(domain);
-
-    let files = await local.find("");
-    let allRemoteFiles = new Set(await remote.find(""));
-    for (let file of files) {
-        if (allRemoteFiles.has(file)) continue;
-        let buffer = await local.get(file);
-        if (!buffer) continue;
-        console.log(`Copying ${file} size ${formatNumber(buffer.length)}B`);
-        await remote.set(file, buffer);
-    }
-}
-main().catch(console.error).finally(() => process.exit(0));
-
-// archives = getArchivesLocal(domain);
-// archives = getArchivesBackblaze(domain);

package/src/-b-authorities/cdnAuthority.ts

@@ -1,53 +0,0 @@
-import { Archives } from "../-a-archives/archives";
-import { ArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
-import { cloudflareGETCall, cloudflarePOSTCall } from "sliftutils/misc/https/cloudflareHelpers";
-import { setRecord, getZoneId } from "./dnsAuthority";
-import debugbreak from "debugbreak";
-
-// servicestest.querysub.com/servicesIndex.json
-
-export async function hostArchives(config: {
-    archives: Archives;
-    // Ex, "archives"
-    subdomain: string;
-    // Ex, "example.com"
-    domain: string;
-}): Promise<{
-    getURL: (path: string) => Promise<string>;
-}> {
-    let { archives, subdomain, domain } = config;
-
-    let base = archives.getBaseArchives?.();
-    let parentPath = base?.parentPath ?? "";
-    archives = base?.archives ?? archives;
-    if (parentPath && !parentPath.endsWith("/")) {
-        parentPath += "/";
-    }
-
-    if (!(archives instanceof ArchivesBackblaze)) {
-        throw new Error(`hostArchives not implemented for ${archives.constructor.name}, only "ArchivesBackblaze" is implemented at the moment`);
-    }
-
-    let archiveT = archives as ArchivesBackblaze;
-    let baseURL = await archiveT.getURL!("");
-    // Remove the trailing slash if it exists
-    if (baseURL.endsWith("/")) {
-        baseURL = baseURL.slice(0, -1);
-    }
-    let backblazeHost = new URL(baseURL).hostname;
-
-    // HACK: Assuming cloudflare is still our DNS provider. If it isn't... this won't work.
-    await setRecord("CNAME", subdomain + "." + domain, backblazeHost, "proxied");
-
-    async function getURL(path: string) {
-        if (path.startsWith("/")) {
-            path = path.slice(1);
-        }
-        let targetPath = await archiveT.getURL!(parentPath + path);
-        let url = new URL(targetPath);
-        url.hostname = subdomain + "." + domain;
-        return url.toString();
-    }
-
-    return { getURL };
-}