querysub 0.355.0 → 0.357.0

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -0,0 +1,598 @@
+import { lazy } from "socket-function/src/caching";
+import { Archives, nestArchives } from "../../../-a-archives/archives";
+import { deepCloneJSON, keyByArray, nextId, sort, timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { BufferIndex } from "./BufferIndex";
+import { delay, runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { Reader, SearchParams } from "./BufferIndexHelpers";
+import { getDomain, isPublic } from "../../../config";
+import { getArchivesLocal } from "../../../-a-archives/archivesDisk";
+import { getArchivesBackblaze, getArchivesBackblazePrivateImmutable } from "../../../-a-archives/archivesBackBlaze";
+import { getOwnThreadId } from "../../../-a-auth/certs";
+import { ArchivesMemoryCacheStats, createArchivesMemoryCache } from "../../../-a-archives/archivesMemoryCache";
+import { registerShutdownHandler } from "../../periodic";
+import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
+import { isNode } from "typesafecss";
+import { getOwnMachineId } from "../../../-f-node-discovery/NodeDiscovery";
+import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
+import { LogStreamer } from "./LogStreamer";
+import { moveLogsToPublic } from "./moveIndexLogsToPublic";
+import { MAX_SINGLE_FILE_DATA, MAX_COUNT_PER_FILE, DISK_FLUSH_INTERVAL, PUBLIC_MOVE_THRESHOLD, MOVING_TIMEOUT } from "./BufferIndexLogsOptimizationConstants";
+import { FindProgressTracker } from "./FindProgressTracker";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { assertIsManagementUser } from "../../managementPages";
+import { ignoreErrors } from "../../../errors";
+
+export type TimeFilePathWithSize = TimeFilePath & {
+    size: number;
+    sourceName: string;
+};
+
+export type IndexedLogResults = {
+    matchCount: number;
+
+    // NOTE: A lot of the metadata won't be accurate if multiple searches happen at the same time. However, for debugging, it should be sufficient. 
+    reads: {
+        cached: boolean;
+        remote: boolean;
+        count: number;
+        size: number;
+
+        totalSize: number;
+        totalCount: number;
+    }[];
+
+    localFilesSearched: number;
+    backblazeFilesSearched: number;
+
+    totalBlockCount: number;
+    blockCheckedCount: number;
+    blocksCheckedCompressedSize: number;
+    blocksCheckedDecompressedSize: number;
+    blockErrors: string[];
+
+    fileErrors: string[];
+
+    indexesSearched: number;
+    indexSize: number;
+
+    timeToFirstMatch: number;
+    fileFindTime: number;
+    indexSearchTime: number;
+    blockSearchTime: number;
+
+    totalSearchTime: number;
+};
+
+let logByName = new Map<string, IndexedLogs<unknown>>();
+
+
+export class IndexedLogs<T> {
+    public constructor(public config: {
+        name: string,
+        maxSingleFileData?: number;
+        maxCountPerFile?: number;
+        forceUsePublicLogs?: boolean;
+        getTime: (result: T) => number | undefined;
+    }) {
+        logByName.set(this.config.name, this as any);
+    }
+
+
+    private static shouldRunLoop = false;
+    public static runLogMoveLoop() {
+        IndexedLogs.shouldRunLoop = true;
+        for (let indexedLogs of logByName.values()) {
+            indexedLogs.runLogMoverLoop();
+        }
+    }
+
+    private findCallbacks = new Map<string, (match: T) => void>();
+    public async clientFind(config: {
+        params: SearchParams;
+        onResult: (match: T) => void;
+    }): Promise<IndexedLogResults> {
+        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
+        let findId = nextId();
+        let callback = (match: T) => {
+            config.onResult(match);
+        };
+        this.findCallbacks.set(findId, callback);
+        try {
+            return await controller.find({
+                findId,
+                indexedLogsName: this.config.name,
+                params: config.params,
+            });
+        } finally {
+            // There's some trailing time after the controller call finishes when the results will be trickling back to us. 
+            setTimeout(() => {
+                this.findCallbacks.delete(findId);
+            }, timeInMinute * 30);
+        }
+    }
+    public async clientGetPaths(config: {
+        startTime: number;
+        endTime: number;
+        only?: "local" | "public";
+    }): Promise<TimeFilePathWithSize[]> {
+        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
+        return await controller.getPaths({
+            indexedLogsName: this.config.name,
+            startTime: config.startTime,
+            endTime: config.endTime,
+            only: config.only,
+        });
+    }
+    public onFindResult(config: {
+        findId: string;
+        result: unknown;
+    }) {
+        let callback = this.findCallbacks.get(config.findId);
+        if (!callback) return;
+        callback(config.result as T);
+    }
+
+    public async clientForceMoveLogsToPublic() {
+        await this.moveLogsToPublic(true);
+    }
+
+    private localLogsStats = {
+        cachedReads: 0,
+        uncachedReads: 0,
+        cachedReadSize: 0,
+        uncachedReadSize: 0,
+        totalCacheSize: 0,
+        totalCacheCount: 0,
+    };
+    private backblazeLogsStats = {
+        cachedReads: 0,
+        uncachedReads: 0,
+        cachedReadSize: 0,
+        uncachedReadSize: 0,
+        totalCacheSize: 0,
+        totalCacheCount: 0,
+    };
+
+    private getLocalLogs = lazy((): Archives => {
+        let baseDisk = getArchivesLocal(getDomain());
+        let archives = nestArchives("indexed-logs/" + this.config.name, baseDisk);
+        archives = createArchivesMemoryCache(archives, {
+            maxSize: 1024 * 1024 * 512,
+            maxCount: 1000 * 100,
+            stats: this.localLogsStats
+        });
+        return archives;
+    });
+    private getPublicLogs = lazy((): Archives => {
+        let basePublic: Archives = getArchivesLocal(getDomain());
+        let extraReadSize = 1024;
+        if (this.config.forceUsePublicLogs || isPublic()) {
+            basePublic = getArchivesBackblaze(getDomain());
+            // The latency on Backblaze is so high that if we're going to do a read, we might as well read a lot. As in, minimum 2s latency. The bandwidth is good though.
+            extraReadSize = 1024 * 1024 * 10;
+        }
+        let archives = nestArchives("final-indexed-logs/" + this.config.name, basePublic);
+        archives = createArchivesMemoryCache(archives, {
+            maxSize: 1024 * 1024 * 1024 * 2,
+            maxCount: 1000 * 500,
+            fullyImmutable: true,
+            extraReadSize,
+            stats: this.backblazeLogsStats,
+        });
+        return archives;
+    });
+
+    private getTimeBlock(time: number): { startTime: number, endTime: number } {
+        let startTime = Math.floor(time / timeInHour) * timeInHour;
+        let endTime = startTime + timeInHour;
+        return { startTime, endTime };
+    }
+
+    private runLogMoverLoop = lazy(() => {
+        runInfinitePoll(timeInMinute * 5, async () => {
+            await this.moveLogsToPublic();
+        });
+    });
+
+    private currentLogStream: LogStreamer<T> | undefined;
+    private forceFlushStream: (() => Promise<void>) | undefined;
+
+    private getCurrentLogStream(): LogStreamer<T> {
+        if (!this.currentLogStream) {
+            let { startTime, endTime } = this.getTimeBlock(Date.now());
+            let path = new TimeFileTree(this.getLocalLogs()).getNewPendingPath({ startTime, endTime });
+            let indexPath = this.getIndexPath(path);
+            let streamer = BufferIndex.createStreamer();
+            let currentLogStreamSize = 0;
+            let currentLogStreamCount = 0;
+            let newStreamer = async () => {
+                let result = streamer.close();
+                await this.getLocalLogs().append(path, result.data);
+                await this.getLocalLogs().append(indexPath, result.index);
+
+                path = new TimeFileTree(this.getLocalLogs()).getNewPendingPath({ startTime, endTime });
+                indexPath = this.getIndexPath(path);
+                streamer = BufferIndex.createStreamer();
+                currentLogStreamSize = 0;
+                currentLogStreamCount = 0;
+            };
+            this.forceFlushStream = async () => {
+                // FORCE the stream to finish and write to disk, by making it too large
+                currentLogStreamSize = Number.MAX_SAFE_INTEGER;
+                currentLogStreamCount = Number.MAX_SAFE_INTEGER;
+                await this.currentLogStream?.flushNow?.();
+
+            };
+            let writeBuffers = async (buffers: Buffer[]) => {
+                let maxSize = this.config.maxSingleFileData || MAX_SINGLE_FILE_DATA;
+                let maxCount = this.config.maxCountPerFile || MAX_COUNT_PER_FILE;
+
+                let bufferGroups = this.splitBuffersIntoGroups(buffers, maxSize, maxCount, currentLogStreamSize, currentLogStreamCount);
+
+                for (let i = 0; i < bufferGroups.length; i++) {
+                    let group = bufferGroups[i];
+
+                    if (i > 0) {
+                        await newStreamer();
+                    }
+
+                    let result = streamer.add(group);
+                    await this.getLocalLogs().append(path, result.data);
+                    if (result.index) {
+                        await this.getLocalLogs().append(indexPath, result.index);
+                    }
+
+                    currentLogStreamSize += result.data.length;
+                    currentLogStreamCount += group.length;
+                }
+            };
+
+            this.currentLogStream = new LogStreamer<T>(DISK_FLUSH_INTERVAL, writeBuffers);
+        }
+        return this.currentLogStream;
+    }
+
+    private splitBuffersIntoGroups(buffers: Buffer[], maxSize: number, maxCount: number, currentLogStreamSize: number, currentLogStreamCount: number): Buffer[][] {
+        let groups: Buffer[][] = [];
+        let currentGroup: Buffer[] = [];
+        let currentGroupSize = 0;
+        let remainingSize = maxSize - currentLogStreamSize;
+        let remainingCount = maxCount - currentLogStreamCount;
+
+        for (let buffer of buffers) {
+            if (
+                // Allow empty at the start, so we create a new streamer
+                (currentGroup.length > 0 || groups.length === 0)
+                &&
+                (currentGroupSize + buffer.length > remainingSize || currentGroup.length >= remainingCount)
+            ) {
+                groups.push(currentGroup);
+                currentGroup = [];
+                currentGroupSize = 0;
+                remainingSize = maxSize;
+                remainingCount = maxCount;
+            }
+
+            currentGroup.push(buffer);
+            currentGroupSize += buffer.length;
+        }
+
+        if (currentGroup.length > 0) {
+            groups.push(currentGroup);
+        }
+
+        return groups;
+    }
+
+    private getIndexPath(dataPath: string): string {
+        return dataPath + ".index";
+    }
+
+    public append(datum: T) {
+        this.getCurrentLogStream().append(datum);
+        if (IndexedLogs.shouldRunLoop) {
+            this.runLogMoverLoop();
+        }
+    }
+
+    public async getPaths(config: {
+        startTime: number;
+        endTime: number;
+        only?: "local" | "public";
+    }): Promise<TimeFilePathWithSize[]> {
+        let localLogs = this.getLocalLogs();
+        let backblazeLogs = this.getPublicLogs();
+
+        let paths: TimeFilePath[] = [];
+
+        if (config.only !== "public") {
+            let localPaths = await new TimeFileTree(localLogs).findAllPaths({
+                startTime: config.startTime,
+                endTime: config.endTime,
+            });
+            paths.push(...localPaths);
+        }
+
+        if (config.only !== "local") {
+            let backblazePaths = await new TimeFileTree(backblazeLogs).findAllPaths({
+                startTime: config.startTime,
+                endTime: config.endTime,
+            });
+            paths.push(...backblazePaths);
+        }
+
+        // Bake in the size to ensure more cache hits (without this, every single access will always result in a file/remote call).
+        let pathsWithSize = await Promise.all(paths.map(async (path): Promise<TimeFilePathWithSize> => {
+            let archives = path.logCount !== undefined ? backblazeLogs : localLogs;
+            let info = await archives.getInfo(path.fullPath);
+            let size = info?.size || 0;
+            return { ...path, size, sourceName: this.config.name };
+        }));
+
+        return pathsWithSize;
+    }
+
+    public async find(config: {
+        params: SearchParams;
+        onResult: (match: T) => void;
+    }): Promise<IndexedLogResults> {
+        let startTime = Date.now();
+        let localLogs = this.getLocalLogs();
+        let backblazeLogs = this.getPublicLogs();
+
+        let fileFindTime = Date.now();
+        let paths = config.params.pathOverrides ?? await this.getPaths({
+            startTime: config.params.startTime,
+            endTime: config.params.endTime,
+            only: config.params.only,
+        });
+        paths = paths.filter(x => x.sourceName === this.config.name);
+        fileFindTime = Date.now() - fileFindTime;
+
+        // Newest first
+        sort(paths, x => - x.startTime);
+
+        let localPaths = paths.filter(x => x.logCount === undefined);
+        let backblazePaths = paths.filter(x => x.logCount !== undefined);
+
+        let initialLocalStats = deepCloneJSON(this.localLogsStats);
+        let initialBackblazeStats = deepCloneJSON(this.backblazeLogsStats);
+        let stats = {
+            blockCheckedCount: 0,
+            blocksCheckedCompressedSize: 0,
+            blocksCheckedDecompressedSize: 0,
+            totalBlockCount: 0,
+            indexSearchTime: 0,
+            blockSearchTime: 0,
+            indexesSearched: 0,
+            indexSize: 0,
+            matchCount: 0,
+            timeToFirstMatch: -1,
+        };
+        let fileErrors: string[] = [];
+        let blockErrors: string[] = [];
+
+        let progressTracker = new FindProgressTracker<T>({
+            params: config.params,
+            deserialize: (buffer: Buffer) => LogStreamer.deserialize<T>(buffer),
+            getTime: this.config.getTime,
+            onResult: (match: T) => {
+                if (stats.timeToFirstMatch < 0) {
+                    stats.timeToFirstMatch = Date.now() - startTime;
+                }
+                stats.matchCount++;
+                config.onResult(match);
+            },
+        });
+
+
+        for (let path of paths) {
+            if (!progressTracker.isSourceRelevant(path)) continue;
+
+            let archives = path.logCount !== undefined ? backblazeLogs : localLogs;
+            try {
+                let readIndexTime = Date.now();
+                let index = await archives.get(this.getIndexPath(path.fullPath)) || Buffer.alloc(0);
+                readIndexTime = Date.now() - readIndexTime;
+                stats.indexSearchTime += readIndexTime;
+                let dataReader: Reader = {
+                    getLength: async () => path.size,
+                    read: async (offset, length) => {
+                        let endOffset = Math.min(offset + length, path.size);
+                        if (offset >= path.size) return Buffer.alloc(0);
+                        let data = await archives.get(path.fullPath, { range: { start: offset, end: endOffset } });
+                        if (!data) return Buffer.alloc(0);
+                        return data;
+                    },
+                };
+                let totalSearchTime = Date.now();
+                let findObj = await BufferIndex.find({
+                    index,
+                    dataReader,
+                    params: {
+                        ...config.params,
+                        limit: config.params.limit - stats.matchCount,
+                    },
+                    keepIterating: () => progressTracker.isSourceRelevant(path),
+                    onResult: (match: Buffer) => {
+                        if (stats.matchCount >= config.params.limit) return;
+                        progressTracker.addResult(match, path);
+                    },
+                });
+                blockErrors.push(...findObj.blocksWithErrors);
+                totalSearchTime = Date.now() - totalSearchTime;
+                stats.indexSearchTime += totalSearchTime - findObj.blockSearchTime;
+                stats.blockSearchTime += findObj.blockSearchTime;
+                stats.indexesSearched += findObj.indexCount;
+                stats.indexSize += findObj.indexSize;
+                stats.blockCheckedCount += findObj.blocksChecked;
+                stats.blocksCheckedCompressedSize += findObj.blocksCheckedCompressedSize;
+                stats.blocksCheckedDecompressedSize += findObj.blocksCheckedDecompressedSize;
+                stats.totalBlockCount += findObj.totalBlockCount;
+
+                if (stats.matchCount >= config.params.limit) break;
+            } catch (e: any) {
+                fileErrors.push(String(e?.stack || e));
+                console.warn(`Error in reading log file ${path.fullPath} logs: ${e}`);
+            }
+        }
+
+        if (stats.timeToFirstMatch < 0) {
+            stats.timeToFirstMatch = Date.now() - startTime;
+        }
+
+
+        let reads: { cached: boolean; remote: boolean; count: number; size: number; totalSize: number; totalCount: number; }[] = [];
+        function addStatsRead(initial: ArchivesMemoryCacheStats, final: ArchivesMemoryCacheStats, remote: boolean) {
+            reads.push({
+                cached: false,
+                remote,
+                count: final.uncachedReads - initial.uncachedReads,
+                size: final.uncachedReadSize - initial.uncachedReadSize,
+                totalSize: final.totalCacheSize,
+                totalCount: final.totalCacheCount,
+            });
+            reads.push({
+                cached: true,
+                remote,
+                count: final.cachedReads - initial.cachedReads,
+                size: final.cachedReadSize - initial.cachedReadSize,
+                totalSize: final.totalCacheSize,
+                totalCount: final.totalCacheCount,
+            });
+        }
+
+        addStatsRead(initialLocalStats, this.localLogsStats, false);
+        addStatsRead(initialBackblazeStats, this.backblazeLogsStats, true);
+
+
+        return {
+            reads,
+            localFilesSearched: localPaths.length,
+            backblazeFilesSearched: backblazePaths.length,
+            fileFindTime,
+            fileErrors,
+            blockErrors,
+            totalSearchTime: Date.now() - startTime,
+            ...stats,
+        };
+    }
+
+    public async moveLogsToPublic(forceAll = false) {
+        if (forceAll) {
+            await this.currentLogStream?.flushNow?.();
+        }
+
+        await moveLogsToPublic({
+            forceAll,
+            localLogs: this.getLocalLogs(),
+            publicLogs: this.getPublicLogs(),
+            publicMoveThreshold: PUBLIC_MOVE_THRESHOLD,
+            maxSingleFileData: this.config.maxSingleFileData || MAX_SINGLE_FILE_DATA,
+            movingTimeout: MOVING_TIMEOUT,
+            getIndexPath: (path: string) => this.getIndexPath(path),
+        });
+    }
+
+
+
+    // TEST functions
+    public async TEST_flushNow() {
+        await this.forceFlushStream?.();
+    }
+    public async TEST_deleteAllLocalLogs() {
+        let localLogs = this.getLocalLogs();
+        await new TimeFileTree(localLogs).deleteAll();
+    }
+    public async TEST_deleteAllBackblazeLogs() {
+        let backblazeLogs = this.getPublicLogs();
+        await new TimeFileTree(backblazeLogs).deleteAll();
+    }
+    public async TEST_deleteAllLogs() {
+        await this.TEST_deleteAllLocalLogs();
+        await this.TEST_deleteAllBackblazeLogs();
+
+        let localLogs = this.getLocalLogs();
+        // Also delete all .moving files
+        let movingFiles = await localLogs.find("", { shallow: true, type: "files" });
+        movingFiles = movingFiles.filter(x => x.endsWith(".moving"));
+        for (let movingFile of movingFiles) {
+            await localLogs.del(movingFile);
+        }
+    }
+}
+
+class IndexedLogClient {
+    public async onFind(config: {
+        findId: string;
+        indexedLogsName: string;
+        result: unknown;
+    }) {
+        if (isNode()) return;
+        let indexedLogs = logByName.get(config.indexedLogsName);
+        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        indexedLogs.onFindResult({
+            findId: config.findId,
+            result: config.result,
+        });
+    }
+}
+class IndexedLogShim {
+    public async find(config: {
+        findId: string;
+        indexedLogsName: string;
+        params: SearchParams;
+    }): Promise<IndexedLogResults> {
+        let caller = SocketFunction.getCaller();
+        let indexedLogs = logByName.get(config.indexedLogsName);
+        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        return indexedLogs.find({
+            params: config.params,
+            onResult: (match: unknown) => {
+                ignoreErrors(IndexedLogClientController.nodes[caller.nodeId].onFind({
+                    findId: config.findId,
+                    indexedLogsName: config.indexedLogsName,
+                    result: match,
+                }));
+            },
+        });
+    }
+    public async getPaths(config: {
+        indexedLogsName: string;
+        startTime: number;
+        endTime: number;
+        only?: "local" | "public";
+    }): Promise<TimeFilePathWithSize[]> {
+        let indexedLogs = logByName.get(config.indexedLogsName);
+        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        return indexedLogs.getPaths({
+            startTime: config.startTime,
+            endTime: config.endTime,
+            only: config.only,
+        });
+    }
+}
+
+const IndexedLogShimController = SocketFunction.register(
+    "IndexedLogShim-019c87b7-73ca-72ec-91b3-2d45ebb616cd",
+    new IndexedLogShim(),
+    () => ({
+        find: {
+            hooks: [assertIsManagementUser]
+        },
+        getPaths: {
+            hooks: [assertIsManagementUser]
+        },
+        clientForceMoveLogsToPublic: {
+            hooks: [assertIsManagementUser]
+        }
+    })
+);
+
+const IndexedLogClientController = SocketFunction.register(
+    "IndexedLogClient-019c87b9-1c6d-72ed-8bc9-d52451e2c1b9",
+    new IndexedLogClient(),
+    () => ({
+        onFind: {}
+    })
+);

package/src/diagnostics/logs/IndexedLogs/LogStreamer.ts

@@ -0,0 +1,47 @@
+import { runInfinitePoll, runInSerial } from "socket-function/src/batching";
+import { measureBlock } from "socket-function/src/profiling/measure";
+import { isNode } from "typesafecss";
+import { registerShutdownHandler } from "../../periodic";
+
+export class LogStreamer<T> {
+    public constructor(diskFlushInterval: number, public writeBuffers: (buffer: Buffer[]) => Promise<void>) {
+        registerShutdownHandler(async () => {
+            await this.flushNow();
+        });
+        runInfinitePoll(diskFlushInterval, async () => {
+            await this.flushNow();
+        });
+    }
+
+    private pendingWriteQueue: T[] = [];
+
+    private insideAppend = false;
+    public append(datum: T) {
+        if (!isNode()) return;
+        if (this.insideAppend) return;
+        this.insideAppend = true;
+        try {
+            this.pendingWriteQueue.push(datum);
+        } finally {
+            this.insideAppend = false;
+        }
+    }
+    insideFlush = false;
+    public flushNow = runInSerial(async () => {
+        this.insideFlush = true;
+        try {
+            // Remember to add timing inside of this to the serialization parts?
+            let writeParts = measureBlock(() => {
+                return this.pendingWriteQueue.map(x => Buffer.from(JSON.stringify(x)));
+            }, `LogStreamer|serialize log data`);
+            this.pendingWriteQueue = [];
+            await this.writeBuffers(writeParts);
+        } finally {
+            this.insideFlush = false;
+        }
+    });
+    public static deserialize<T>(data: Buffer): T {
+        return JSON.parse(data.toString());
+    }
+}
+