querysub 0.590.0 → 0.592.0

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -1,8 +1,6 @@
 import { lazy } from "socket-function/src/caching";
-import { nestArchives } from "../../../-a-archives/archives";
-import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
+import { getArchives2 } from "../../../-a-archives/archives2";
 import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
-import { getDomain } from "../../../config";
 import { runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { timeInMinute } from "socket-function/src/misc";
 import { SocketFunction } from "socket-function/SocketFunction";
@@ -56,7 +54,7 @@ export function getVariables(entry: LifeCycleEntry): { key: string, title?: stri
     return variables;
 }
 
-const lifeCycles = archiveJSONT<LifeCycle>(() => nestArchives("logs/life-cycles/", getArchivesBackblaze(getDomain())));
+const lifeCycles = archiveJSONT<LifeCycle>(() => getArchives2("logs/life-cycles/"));
 let lifeCyclesCache: LifeCycle[] = [];
 
 let ensureWatching = lazy(async () => {

package/src/diagnostics/watchdog.ts

@@ -38,10 +38,6 @@ export type MeasureSummary = {
     startTime: number;
     endTime: number;
     profiledTime: number;
-    entries: {
-        name: string;
-        ownTime: number;
-    }[];
 };
 
 let measureChunks: MeasureChunk[] = [];
@@ -78,7 +74,6 @@ export function getLastMeasureSummary(): MeasureSummary | undefined {
         startTime: chunk.startTime,
         endTime: chunk.endTime,
         profiledTime: chunk.profiledTime,
-        entries: chunk.entries.map(x => ({ name: x.name, ownTime: x.ownTime.sum })),
     };
 }
 export function getMeasureBreakdown(range: { startTime: number; endTime: number }): MeasureChunk | undefined {

package/src/-a-archives/archivesBackBlaze.ts

@@ -1,156 +0,0 @@
-import { cache } from "socket-function/src/caching";
-import { Archives } from "./archives";
-import fs from "fs";
-import { isNode, timeInMinute } from "socket-function/src/misc";
-import { isLogBackblaze } from "../config";
-import { getBackblazePath } from "../misc/appPaths";
-import { ArchivesBackblaze as ArchivesBackblazeBase } from "sliftutils/storage/backblaze";
-import type { IArchives, ArchivesConfig } from "sliftutils/storage/IArchives";
-
-export function hasBackblazePermissions() {
-    return isNode() && fs.existsSync(getBackblazePath());
-}
-
-// The implementation lives in sliftutils (it reads credentials via getSecret, which our appSecrets.ts serves from getBackblazePath). We forward to an inner instance instead of subclassing, because our Archives interface's move/copy take a querysub Archives target (with getBaseArchives unwrapping) which is type-incompatible with the base class's IArchives signatures.
-export class ArchivesBackblaze {
-    private archives: ArchivesBackblazeBase;
-
-    public constructor(config: {
-        bucketName: string;
-        public?: boolean;
-        immutable?: boolean;
-        cacheTime?: number;
-        allowedOrigins?: string[];
-    }) {
-        this.archives = new ArchivesBackblazeBase(config);
-        if (isLogBackblaze()) {
-            this.archives.enableLogging();
-        }
-    }
-
-    public enableLogging(): void {
-        this.archives.enableLogging();
-    }
-    public getDebugName(): string {
-        return this.archives.getDebugName();
-    }
-    public async hasWriteAccess(): Promise<boolean> {
-        return this.archives.hasWriteAccess();
-    }
-    public async getConfig(): Promise<ArchivesConfig> {
-        return this.archives.getConfig();
-    }
-    public async get(fileName: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
-        return this.archives.get(fileName, config);
-    }
-    public async get2(fileName: string, config?: { range?: { start: number; end: number; } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
-        return this.archives.get2(fileName, config);
-    }
-    public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
-        return this.archives.set(fileName, data, config);
-    }
-    public async append(fileName: string, data: Buffer): Promise<void> {
-        throw new Error(`ArchivesBackblaze does not support append. Use set instead.`);
-    }
-    public async del(fileName: string): Promise<void> {
-        return this.archives.del(fileName);
-    }
-    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
-        return this.archives.setLargeFile(config);
-    }
-    public async getInfo(fileName: string): Promise<{ writeTime: number; size: number; } | undefined> {
-        return this.archives.getInfo(fileName);
-    }
-    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
-        return this.archives.find(prefix, config);
-    }
-    public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
-        return this.archives.findInfo(prefix, config);
-    }
-    public async assertPathValid(path: string): Promise<void> {
-        return this.archives.assertPathValid(path);
-    }
-
-    public async move(config: {
-        path: string;
-        target: Archives;
-        targetPath: string;
-        copyInstead?: boolean;
-    }): Promise<void> {
-        let { target, targetPath } = config;
-        let base = target.getBaseArchives?.();
-        if (base) {
-            target = base.archives;
-            targetPath = base.parentPath + targetPath;
-        }
-        // Hand the inner instance across so the base implementation's bucket-to-bucket copyFile fast path (an instanceof check on its own class) still triggers.
-        let innerTarget: IArchives = target instanceof ArchivesBackblaze
-            ? target.archives
-            : target as unknown as IArchives;
-        return this.archives.move({
-            path: config.path,
-            target: innerTarget as any,
-            targetPath,
-            copyInstead: config.copyInstead,
-        });
-    }
-
-    public async copy(config: {
-        path: string;
-        target: Archives;
-        targetPath: string;
-    }): Promise<void> {
-        return this.move({ ...config, copyInstead: true });
-    }
-
-    public async getURL(path: string): Promise<string> {
-        return this.archives.getURL(path);
-    }
-
-    public async getDownloadAuthorization(config: {
-        fileNamePrefix?: string;
-        validDurationInSeconds: number;
-        b2ContentDisposition?: string;
-        b2ContentLanguage?: string;
-        b2Expires?: string;
-        b2CacheControl?: string;
-        b2ContentEncoding?: string;
-        b2ContentType?: string;
-    }): Promise<{
-        bucketId: string;
-        fileNamePrefix: string;
-        authorizationToken: string;
-    }> {
-        return this.archives.getDownloadAuthorization(config);
-    }
-}
-
-export const getArchivesBackblaze = cache((domain: string) => {
-    return new ArchivesBackblaze({ bucketName: domain });
-});
-export const getArchivesBackblazePrivateImmutable = cache((domain: string) => {
-    return new ArchivesBackblaze({
-        bucketName: domain + "-private-immutable",
-        immutable: true
-    });
-});
-export const getArchivesBackblazePublicImmutable = cache((domain: string) => {
-    return new ArchivesBackblaze({
-        bucketName: domain + "-public-immutable",
-        public: true,
-        immutable: true,
-        allowedOrigins: [`https://${domain}`, `https://127-0-0-1.${domain}:7007`],
-    });
-});
-
-// NOTE: Cache by a minute. This might be a bad idea, but... usually whole reason for public is
-//  for cloudflare caching (as otherwise we can just access it through a server), or for large files
-//  (which should be cached anyways, and probably even use immutable caching).
-export const getArchivesBackblazePublic = cache((domain: string) => {
-    return new ArchivesBackblaze({
-        bucketName: domain + "-public",
-        public: true,
-        cacheTime: timeInMinute,
-        allowedOrigins: [`https://${domain}`, `https://127-0-0-1.${domain}:7007`],
-    });
-});

package/src/-a-archives/archivesCborT.ts

@@ -1,52 +0,0 @@
-import { lazy } from "socket-function/src/caching";
-import { Archives } from "./archives";
-import { decodeCborx, encodeCborx } from "../misc/cloneHelpers";
-import { ArchiveT } from "./archivesJSONT";
-
-export function archiveCborT<T>(archives: () => Archives): ArchiveT<T> {
-    archives = lazy(archives);
-    async function get(key: string) {
-        let buffer = await archives().get(key);
-        if (!buffer) return undefined;
-        return decodeCborx(buffer) as T;
-    }
-    async function set(key: string, value: T) {
-        await archives().set(key, encodeCborx(value));
-    }
-    async function deleteFnc(key: string) {
-        await archives().del(key);
-    }
-    async function keys() {
-        return (await archives().find("")).map(value => value.toString());
-    }
-    async function values() {
-        let keysArray = await keys();
-        let results: T[] = [];
-        await Promise.all(keysArray.map(async key => {
-            let value = await get(key);
-            if (value) {
-                results.push(value);
-            }
-        }));
-        return results;
-    }
-    async function entries(): Promise<[string, T][]> {
-        let keysArray = await keys();
-        let results: [string, T][] = [];
-        await Promise.all(keysArray.map(async key => {
-            let value = await get(key);
-            if (value) {
-                results.push([key, value]);
-            }
-        }));
-        return results;
-    }
-    async function* asyncIterator(): AsyncIterator<[string, T]> {
-        for (let [key, value] of await entries()) {
-            yield [key, value];
-        }
-    }
-    return {
-        get, set, delete: deleteFnc, keys, values, entries, [Symbol.asyncIterator]: asyncIterator
-    };
-}

package/src/-a-archives/archivesLimitedCache.ts

@@ -1,307 +0,0 @@
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { Archives } from "./archives";
-import { cache } from "socket-function/src/caching";
-import { measureFnc } from "socket-function/src/profiling/measure";
-import { batchFunction, runInfinitePoll } from "socket-function/src/batching";
-import { timeInHour } from "socket-function/src/misc";
-
-interface FileInfo {
-    writeTime: number;
-    accessTime: number;
-    size: number;
-}
-
-interface IndexData {
-    files: Record<string, FileInfo>;
-}
-
-class ArchivesLimitedCache {
-    private baseArchives: Archives;
-    private maxFiles: number;
-    private maxSize: number;
-    private cache = new Map<string, FileInfo>();
-    private initialized = false;
-    private readonly indexPath = ".cache-index.json";
-
-    constructor(baseArchives: Archives, config: { maxFiles: number; maxSize: number }) {
-        this.baseArchives = baseArchives;
-        this.maxFiles = config.maxFiles;
-        this.maxSize = config.maxSize;
-        this.initOptionalMethods();
-        this.initPeriodicSync();
-    }
-
-    public getDebugName(): string {
-        return `limitedCache(${this.maxFiles}files,${Math.round(this.maxSize / (1024 * 1024))}MB)/${this.baseArchives.getDebugName()}`;
-    }
-
-    @measureFnc
-    private async rebuildCacheFromFiles(): Promise<void> {
-        const allFiles = await this.baseArchives.findInfo("", { type: "files" });
-        const newCache = new Map<string, FileInfo>();
-
-        // Build map of actual files (excluding index file)
-        for (const file of allFiles) {
-            if (file.path === this.indexPath) continue;
-
-            newCache.set(file.path, {
-                writeTime: file.createTime,
-                accessTime: this.cache.get(file.path)?.accessTime ?? file.createTime,
-                size: file.size,
-            });
-        }
-
-        // Update cache with actual files
-        this.cache.clear();
-        for (const [path, info] of newCache.entries()) {
-            this.cache.set(path, info);
-        }
-    }
-
-    @measureFnc
-    private async ensureInitialized(): Promise<void> {
-        if (this.initialized) return;
-
-        // First try to load from index file
-        await this.loadIndex();
-
-        // If index is empty or missing, fall back to scanning all files
-        if (this.cache.size === 0) {
-            console.log("Index file missing or empty, scanning all files...");
-            await this.rebuildCacheFromFiles();
-            // Save the newly built index
-            await this.saveIndex();
-        }
-
-        this.initialized = true;
-
-        // Cleanup if we're already over limits
-        await this.applyLimits();
-    }
-
-    private async applyLimits(): Promise<void> {
-        const files = Array.from(this.cache.entries()).map(([path, info]) => ({
-            path,
-            ...info,
-        }));
-
-        const totalSize = files.reduce((sum, file) => sum + file.size, 0);
-        const totalFiles = files.length;
-
-        if (totalFiles <= this.maxFiles && totalSize <= this.maxSize) {
-            return; // No cleanup needed
-        }
-
-        // Sort by access time (oldest first)
-        files.sort((a, b) => a.accessTime - b.accessTime);
-
-        let currentSize = totalSize;
-        let currentFiles = totalFiles;
-
-        // Remove files until we're under both limits
-        for (const file of files) {
-            if (currentFiles <= this.maxFiles && currentSize <= this.maxSize) {
-                break;
-            }
-
-            console.warn(`Deleting file ${file.path} to maintain limits. Currently have ${formatNumber(currentFiles)} files and ${formatNumber(currentSize)}B`);
-            try {
-                await this.baseArchives.del(file.path);
-                this.cache.delete(file.path);
-                currentSize -= file.size;
-                currentFiles--;
-            } catch (error) {
-                // File might already be deleted, continue
-                console.warn(`Failed to delete file ${file.path}:`, error);
-                this.cache.delete(file.path); // Remove from cache anyway
-                currentSize -= file.size;
-                currentFiles--;
-            }
-        }
-
-        // Trigger index flush if any files were deleted
-        if (currentFiles < totalFiles) {
-            this.triggerIndexFlush();
-        }
-    }
-
-    private updateAccessTime(path: string): void {
-        const info = this.cache.get(path);
-        if (info) {
-            info.accessTime = Date.now();
-            this.triggerIndexFlush();
-        }
-    }
-
-    public async get(path: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
-        await this.ensureInitialized();
-        this.updateAccessTime(path);
-        return this.baseArchives.get(path, config);
-    }
-
-    public async set(path: string, data: Buffer): Promise<void> {
-        await this.ensureInitialized();
-        // Limit before, not after, so we don't remove the file we just added
-        await this.applyLimits();
-        await this.baseArchives.set(path, data);
-        const now = Date.now();
-        this.cache.set(path, {
-            writeTime: now,
-            accessTime: now,
-            size: data.length,
-        });
-        this.triggerIndexFlush();
-    }
-
-    public async append(path: string, data: Buffer): Promise<void> {
-        await this.ensureInitialized();
-        await this.baseArchives.append(path, data);
-        // Update the cache info - increment size and update times
-        const info = this.cache.get(path);
-        const now = Date.now();
-        if (info) {
-            info.size += data.length;
-            info.writeTime = now;
-            info.accessTime = now;
-        } else {
-            // File didn't exist in cache, add it
-            this.cache.set(path, {
-                writeTime: now,
-                accessTime: now,
-                size: data.length,
-            });
-        }
-        this.triggerIndexFlush();
-    }
-
-    public async del(path: string): Promise<void> {
-        await this.ensureInitialized();
-        await this.baseArchives.del(path);
-        this.cache.delete(path);
-        this.triggerIndexFlush();
-    }
-
-    public async getInfo(path: string): Promise<{ writeTime: number; size: number; } | undefined> {
-        await this.ensureInitialized();
-        this.updateAccessTime(path);
-        return this.baseArchives.getInfo(path);
-    }
-
-    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
-        throw new Error(`setLargeFile in archivesLimitedCache is not supported`);
-    }
-
-    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
-        await this.ensureInitialized();
-        return this.baseArchives.find(prefix, config);
-    }
-
-    public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
-        await this.ensureInitialized();
-        return this.baseArchives.findInfo(prefix, config);
-    }
-
-    public async move(config: { path: string; target: Archives; targetPath: string }): Promise<void> {
-        throw new Error(`Move in archivesLimitedCache is not supported`);
-    }
-
-    public async copy(config: { path: string; target: Archives; targetPath: string }): Promise<void> {
-        throw new Error(`Copy in archivesLimitedCache is not supported`);
-    }
-
-    public enableLogging(): void {
-        this.baseArchives.enableLogging();
-    }
-
-    public async assertPathValid(path: string): Promise<void> {
-        await this.baseArchives.assertPathValid(path);
-    }
-
-    public getBaseArchives?: () => { parentPath: string; archives: Archives; } | undefined = () => {
-        const base = this.baseArchives.getBaseArchives?.();
-        if (base) {
-            return base;
-        }
-        return { parentPath: "", archives: this.baseArchives };
-    };
-
-    public getURL?: (path: string) => Promise<string>;
-    public getDownloadAuthorization?: (config: { validDurationInSeconds: number }) => Promise<{ bucketId: string; fileNamePrefix: string; authorizationToken: string; }>;
-
-    private batchedFlushIndex = batchFunction(
-        { delay: 2000 }, // Flush index changes every 2 seconds
-        async () => {
-            // We don't actually need the batched values, just flush the index
-            await this.saveIndex();
-        }
-    );
-
-    private async loadIndex(): Promise<void> {
-        try {
-            const indexBuffer = await this.baseArchives.get(this.indexPath);
-            if (!indexBuffer) {
-                // No existing index, will need to build it
-                return;
-            }
-
-            const indexData: IndexData = JSON.parse(indexBuffer.toString("utf8"));
-
-            // Load the cache from the index
-            this.cache.clear();
-            for (const [path, info] of Object.entries(indexData.files)) {
-                this.cache.set(path, info);
-            }
-
-            console.log(`Loaded index with ${this.cache.size} files`);
-        } catch (error) {
-            console.warn(`Failed to load index file:`, error);
-            // Continue without index, will rebuild
-        }
-    }
-
-    private async saveIndex(): Promise<void> {
-        try {
-            const indexData: IndexData = {
-                files: Object.fromEntries(this.cache.entries())
-            };
-
-            const indexBuffer = Buffer.from(JSON.stringify(indexData), "utf8");
-            await this.baseArchives.set(this.indexPath, indexBuffer);
-        } catch (error) {
-            console.warn(`Failed to save index file:`, error);
-        }
-    }
-
-    private triggerIndexFlush(): void {
-        void this.batchedFlushIndex(undefined);
-    }
-
-    private initPeriodicSync(): void {
-        // Sync index with actual files every hour to prevent drift
-        runInfinitePoll(timeInHour, async () => {
-            await this.syncIndexWithFiles();
-        });
-    }
-
-    private async syncIndexWithFiles(): Promise<void> {
-        try {
-            console.log("Syncing index with actual files...");
-            await this.rebuildCacheFromFiles();
-
-            // Save updated index
-            await this.saveIndex();
-            console.log(`Index synced with ${this.cache.size} actual files`);
-        } catch (error) {
-            console.warn("Failed to sync index with files:", error);
-        }
-    }
-
-    private initOptionalMethods(): void {
-        this.getURL = this.baseArchives.getURL;
-        this.getDownloadAuthorization = this.baseArchives.getDownloadAuthorization;
-    }
-}
-
-export function createArchivesLimitedCache(baseArchives: Archives, config: { maxFiles: number; maxSize: number }): Archives {
-    return new ArchivesLimitedCache(baseArchives, config);
-}