querysub 0.356.0 → 0.358.0

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -0,0 +1,685 @@
+import { lazy } from "socket-function/src/caching";
+import { Archives, nestArchives } from "../../../-a-archives/archives";
+import { deepCloneJSON, keyByArray, nextId, sort, timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { BufferIndex } from "./BufferIndex";
+import { delay, runInParallel, runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { IndexedLogResults, Reader, SearchParams, addReadToResults, createEmptyIndexedLogResults, INDEX_EXTENSION } from "./BufferIndexHelpers";
+import { getDomain, isPublic } from "../../../config";
+import { getArchivesLocal } from "../../../-a-archives/archivesDisk";
+import { getArchivesBackblaze, getArchivesBackblazePrivateImmutable } from "../../../-a-archives/archivesBackBlaze";
+import { getOwnThreadId } from "../../../-a-auth/certs";
+import { ArchivesMemoryCacheStats, createArchivesMemoryCache } from "../../../-a-archives/archivesMemoryCache";
+import { registerShutdownHandler } from "../../periodic";
+import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
+import { isNode } from "typesafecss";
+import { getOwnMachineId } from "../../../-f-node-discovery/NodeDiscovery";
+import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
+import { LogStreamer } from "./LogStreamer";
+import { moveLogsToPublic } from "./moveIndexLogsToPublic";
+import { MAX_SINGLE_FILE_DATA, MAX_COUNT_PER_FILE, DISK_FLUSH_INTERVAL, PUBLIC_MOVE_THRESHOLD, MOVING_TIMEOUT } from "./BufferIndexLogsOptimizationConstants";
+import { FindProgressTracker } from "./FindProgressTracker";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { assertIsManagementUser } from "../../managementPages";
+import { ignoreErrors } from "../../../errors";
+import { blue } from "socket-function/src/formatting/logColors";
+import { LimitGroup } from "../../../functional/limitProcessing";
+
+export type TimeFilePathWithSize = TimeFilePath & {
+    size: number;
+    indexSize: number;
+    sourceName: string;
+};
+
+let loggerByName = new Map<string, IndexedLogs<unknown>>();
+
+
+export class IndexedLogs<T> {
+    public constructor(public config: {
+        name: string,
+        maxSingleFileData?: number;
+        maxCountPerFile?: number;
+        forceUsePublicLogs?: boolean;
+        getTime: (result: T) => number | undefined;
+    }) {
+        loggerByName.set(this.config.name, this as any);
+    }
+
+
+    private static shouldRunLoop = false;
+    public static runLogMoveLoop() {
+        IndexedLogs.shouldRunLoop = true;
+        for (let indexedLogs of loggerByName.values()) {
+            indexedLogs.runLogMoverLoop();
+        }
+    }
+
+    private findCallbacks = new Map<string, (match: T) => void>();
+    private resultsCallbacks = new Map<string, (results: IndexedLogResults) => void>();
+    public async clientFind(config: {
+        params: SearchParams;
+        onResult: (match: T) => void;
+        onResults?: (results: IndexedLogResults) => void;
+    }): Promise<IndexedLogResults> {
+        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
+        let findId = nextId();
+        let callback = (match: T) => {
+            config.onResult(match);
+        };
+        this.findCallbacks.set(findId, callback);
+        if (config.onResults) {
+            this.resultsCallbacks.set(findId, config.onResults);
+        }
+        try {
+            return await controller.find({
+                findId,
+                indexedLogsName: this.config.name,
+                params: config.params,
+            });
+        } finally {
+            // There's some trailing time after the controller call finishes when the results will be trickling back to us. 
+            setTimeout(() => {
+                this.findCallbacks.delete(findId);
+                this.resultsCallbacks.delete(findId);
+            }, timeInMinute * 30);
+        }
+    }
+    public clientCancelAllCallbacks() {
+        this.findCallbacks.clear();
+        this.resultsCallbacks.clear();
+    }
+    public async clientGetPaths(config: {
+        startTime: number;
+        endTime: number;
+        only?: "local" | "public";
+    }): Promise<TimeFilePathWithSize[]> {
+        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
+        return await controller.getPaths({
+            indexedLogsName: this.config.name,
+            startTime: config.startTime,
+            endTime: config.endTime,
+            only: config.only,
+        });
+    }
+    public onFindResult(config: {
+        findId: string;
+        result: unknown;
+    }) {
+        let callback = this.findCallbacks.get(config.findId);
+        if (!callback) throw new Error(`Find callback ${config.findId} not found`);
+        callback(config.result as T);
+    }
+    public async onResults(config: {
+        findId: string;
+        results: IndexedLogResults;
+    }): Promise<boolean> {
+        let callback = this.resultsCallbacks.get(config.findId);
+        if (!callback) throw new Error(`Results callback ${config.findId} not found`);
+        callback(config.results);
+        return true;
+    }
+
+    public async clientForceMoveLogsToPublic() {
+        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
+        await controller.forceMoveLogsToPublic({
+            indexedLogsName: this.config.name,
+        });
+    }
+
+    private localLogsStats = {
+        cachedReads: 0,
+        uncachedReads: 0,
+        cachedReadSize: 0,
+        uncachedReadSize: 0,
+        totalCacheSize: 0,
+        totalCacheCount: 0,
+    };
+    private backblazeLogsStats = {
+        cachedReads: 0,
+        uncachedReads: 0,
+        cachedReadSize: 0,
+        uncachedReadSize: 0,
+        totalCacheSize: 0,
+        totalCacheCount: 0,
+    };
+
+    private fileSizeCache = new Map<string, number>();
+
+    private getLocalLogs = lazy((): Archives => {
+        let baseDisk = getArchivesLocal(getDomain());
+        let archives = nestArchives("indexed-logs/" + this.config.name, baseDisk);
+        archives = createArchivesMemoryCache(archives, {
+            maxSize: 1024 * 1024 * 512,
+            maxCount: 1000 * 100,
+            stats: this.localLogsStats,
+            sizeCache: this.fileSizeCache,
+            // Local disk reads are fast, but even local reads can have high latency
+            extraReadSize: 1024 * 1024 * 1,
+        });
+        return archives;
+    });
+    private getPublicLogs = lazy((): Archives => {
+        let basePublic: Archives = getArchivesLocal(getDomain());
+        // NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever. 
+        let extraReadSize = 1024 * 1024 * 10;
+        if (this.config.forceUsePublicLogs || isPublic()) {
+            basePublic = getArchivesBackblaze(getDomain());
+            // NOTE: While the latency to back plays is high, now we're reading in parallel, so it shouldn't be as big of an issue. 
+            extraReadSize = 1024 * 1024 * 1;
+        }
+        let archives = nestArchives("final-indexed-logs/" + this.config.name, basePublic);
+        archives = createArchivesMemoryCache(archives, {
+            maxSize: 1024 * 1024 * 1024 * 2,
+            maxCount: 1000 * 500,
+            fullyImmutable: true,
+            extraReadSize,
+            stats: this.backblazeLogsStats,
+            sizeCache: this.fileSizeCache,
+        });
+        return archives;
+    });
+
+    private getTimeBlock(time: number): { startTime: number, endTime: number } {
+        let startTime = Math.floor(time / timeInHour) * timeInHour;
+        let endTime = startTime + timeInHour;
+        return { startTime, endTime };
+    }
+
+    private runLogMoverLoop = lazy(() => {
+        runInfinitePoll(timeInMinute * 5, async () => {
+            await this.moveLogsToPublic();
+        });
+    });
+
+    private currentLogStream: LogStreamer<T> | undefined;
+    private forceFlushStream: (() => Promise<void>) | undefined;
+
+    private getCurrentLogStream(): LogStreamer<T> {
+
+        if (!this.currentLogStream) {
+            let { startTime, endTime } = this.getTimeBlock(Date.now());
+            let path = new TimeFileTree(this.getLocalLogs()).getNewPendingPath({ startTime, endTime });
+            let streamer = BufferIndex.createStreamer();
+            let currentLogStreamSize = 0;
+            let currentLogStreamCount = 0;
+            let newStreamer = async () => {
+                let result = streamer.close();
+                await this.getLocalLogs().append(path, result.data);
+                await this.getLocalLogs().append(path + INDEX_EXTENSION, result.index);
+
+                path = new TimeFileTree(this.getLocalLogs()).getNewPendingPath({ startTime, endTime });
+                streamer = BufferIndex.createStreamer();
+                currentLogStreamSize = 0;
+                currentLogStreamCount = 0;
+            };
+            this.forceFlushStream = async () => {
+                // FORCE the stream to finish and write to disk, by making it too large
+                currentLogStreamSize = Number.MAX_SAFE_INTEGER;
+                currentLogStreamCount = Number.MAX_SAFE_INTEGER;
+                await this.currentLogStream?.flushNow?.();
+
+            };
+            let writeBuffers = async (buffers: Buffer[]) => {
+                if (Date.now() > endTime) {
+                    let timeBlockObj = this.getTimeBlock(Date.now());
+                    startTime = timeBlockObj.startTime;
+                    endTime = timeBlockObj.endTime;
+                    path = new TimeFileTree(this.getLocalLogs()).getNewPendingPath(timeBlockObj);
+                    await newStreamer();
+                }
+
+                let maxSize = this.config.maxSingleFileData || MAX_SINGLE_FILE_DATA;
+                let maxCount = this.config.maxCountPerFile || MAX_COUNT_PER_FILE;
+
+                let bufferGroups = this.splitBuffersIntoGroups(buffers, maxSize, maxCount, currentLogStreamSize, currentLogStreamCount);
+
+                for (let i = 0; i < bufferGroups.length; i++) {
+                    let group = bufferGroups[i];
+
+                    if (i > 0) {
+                        await newStreamer();
+                    }
+
+                    let result = streamer.add(group);
+                    await this.getLocalLogs().append(path, result.data);
+                    if (result.index) {
+                        await this.getLocalLogs().append(path + INDEX_EXTENSION, result.index);
+                    }
+
+                    currentLogStreamSize += result.data.length;
+                    currentLogStreamCount += group.length;
+                }
+            };
+
+            this.currentLogStream = new LogStreamer<T>(DISK_FLUSH_INTERVAL, writeBuffers);
+        }
+        return this.currentLogStream;
+    }
+
+    private splitBuffersIntoGroups(buffers: Buffer[], maxSize: number, maxCount: number, currentLogStreamSize: number, currentLogStreamCount: number): Buffer[][] {
+        let groups: Buffer[][] = [];
+        let currentGroup: Buffer[] = [];
+        let currentGroupSize = 0;
+        let remainingSize = maxSize - currentLogStreamSize;
+        let remainingCount = maxCount - currentLogStreamCount;
+
+        for (let buffer of buffers) {
+            if (
+                // Allow empty at the start, so we create a new streamer
+                (currentGroup.length > 0 || groups.length === 0)
+                &&
+                (currentGroupSize + buffer.length > remainingSize || currentGroup.length >= remainingCount)
+            ) {
+                groups.push(currentGroup);
+                currentGroup = [];
+                currentGroupSize = 0;
+                remainingSize = maxSize;
+                remainingCount = maxCount;
+            }
+
+            currentGroup.push(buffer);
+            currentGroupSize += buffer.length;
+        }
+
+        if (currentGroup.length > 0) {
+            groups.push(currentGroup);
+        }
+
+        return groups;
+    }
+
+    public append(datum: T) {
+        this.getCurrentLogStream().append(datum);
+        if (IndexedLogs.shouldRunLoop) {
+            this.runLogMoverLoop();
+        }
+    }
+
+    @measureFnc
+    public async getPaths(config: {
+        startTime: number;
+        endTime: number;
+        only?: "local" | "public";
+    }): Promise<TimeFilePathWithSize[]> {
+        let localLogs = this.getLocalLogs();
+        let backblazeLogs = this.getPublicLogs();
+
+        let paths: TimeFilePath[] = [];
+
+        if (config.only !== "public") {
+            let localPaths = await new TimeFileTree(localLogs).findAllPaths({
+                startTime: config.startTime,
+                endTime: config.endTime,
+            });
+            paths.push(...localPaths);
+        }
+
+        if (config.only !== "local") {
+            let backblazePaths = await new TimeFileTree(backblazeLogs).findAllPaths({
+                startTime: config.startTime,
+                endTime: config.endTime,
+            });
+            paths.push(...backblazePaths);
+        }
+
+        // Bake in the size to ensure more cache hits (without this, every single access will always result in a file/remote call).
+        let pathsWithSize = await Promise.all(paths.map(async (path): Promise<TimeFilePathWithSize> => {
+            let archives = path.logCount !== undefined ? backblazeLogs : localLogs;
+            let info = await archives.getInfo(path.fullPath);
+            let size = info?.size || 0;
+            this.fileSizeCache.set(path.fullPath, size);
+            let indexPath = path.fullPath + INDEX_EXTENSION;
+            let indexInfo = await archives.getInfo(indexPath);
+            let indexSize = indexInfo?.size || 0;
+            this.fileSizeCache.set(indexPath, indexSize);
+            return { ...path, size, indexSize, sourceName: this.config.name };
+        }));
+
+        return pathsWithSize;
+    }
+
+    @measureFnc
+    public async find(config: {
+        params: SearchParams;
+        onResult: (match: T) => void;
+        onResults?: (results: IndexedLogResults) => Promise<boolean>;
+    }): Promise<IndexedLogResults> {
+        let startTime = Date.now();
+        let interval: NodeJS.Timeout | undefined;
+
+        let results: IndexedLogResults = createEmptyIndexedLogResults();
+        results.limitGroup = new LimitGroup({
+            maxTimePerBeforeWait: 500,
+            waitTime: 250,
+        });
+
+        let initialLocalStats = deepCloneJSON(this.localLogsStats);
+        let initialBackblazeStats = deepCloneJSON(this.backblazeLogsStats);
+
+        let updateResultsStats = () => {
+            results.reads = [];
+
+            function addStatsRead(initial: ArchivesMemoryCacheStats, final: ArchivesMemoryCacheStats, remote: boolean) {
+                let uncached = addReadToResults(results, {
+                    cached: false,
+                    remote,
+                    count: final.uncachedReads - initial.uncachedReads,
+                    size: final.uncachedReadSize - initial.uncachedReadSize,
+                });
+                uncached.totalSize = final.totalCacheSize;
+                uncached.totalCount = final.totalCacheCount;
+
+                let cached = addReadToResults(results, {
+                    cached: true,
+                    remote,
+                    count: final.cachedReads - initial.cachedReads,
+                    size: final.cachedReadSize - initial.cachedReadSize,
+                });
+                cached.totalSize = final.totalCacheSize;
+                cached.totalCount = final.totalCacheCount;
+            }
+
+            addStatsRead(initialLocalStats, this.localLogsStats, false);
+            addStatsRead(initialBackblazeStats, this.backblazeLogsStats, true);
+
+            results.totalSearchTime = Date.now() - startTime;
+        };
+
+        const onResultsCallback = config.onResults;
+        if (onResultsCallback) {
+            interval = setInterval(async () => {
+                updateResultsStats();
+                let shouldContinue = await onResultsCallback(deepCloneJSON(results));
+                if (!shouldContinue) {
+                    if (!results.cancel) {
+                        console.log(blue(`Cancelled search on ${this.config.name}`));
+                    }
+                    results.cancel = true;
+                }
+            }, timeInSecond);
+        }
+
+        try {
+            await this.findBase({
+                params: config.params,
+                onResult: config.onResult,
+                results,
+            });
+            updateResultsStats();
+            return results;
+        } finally {
+            if (interval) {
+                clearInterval(interval);
+            }
+        }
+    }
+
+    private async findBase(config: {
+        params: SearchParams;
+        onResult: (match: T) => void;
+        results: IndexedLogResults;
+    }): Promise<void> {
+        let startTime = Date.now();
+        let localLogs = this.getLocalLogs();
+        let backblazeLogs = this.getPublicLogs();
+
+        let fileFindTime = Date.now();
+        let paths = config.params.pathOverrides ?? await this.getPaths({
+            startTime: config.params.startTime,
+            endTime: config.params.endTime,
+            only: config.params.only,
+        });
+        paths = paths.filter(x => x.sourceName === this.config.name);
+        for (let path of paths) {
+            this.fileSizeCache.set(path.fullPath, path.size);
+            this.fileSizeCache.set(path.fullPath + INDEX_EXTENSION, path.indexSize);
+        }
+        fileFindTime = Date.now() - fileFindTime;
+
+        // Newest first
+        sort(paths, x => - x.startTime);
+
+        let results = config.results;
+        let localPaths = paths.filter(x => x.logCount === undefined);
+        let remotePaths = paths.filter(x => x.logCount !== undefined);
+        results.totalLocalFiles = localPaths.length;
+        results.totalBackblazeFiles = remotePaths.length;
+        results.fileFindTime = fileFindTime;
+
+        let progressTracker = new FindProgressTracker<T>({
+            params: config.params,
+            deserialize: (buffer: Buffer) => LogStreamer.deserialize<T>(buffer),
+            getTime: this.config.getTime,
+            onResult: (match: T) => {
+                if (results.timeToFirstMatch < 0) {
+                    results.timeToFirstMatch = Date.now() - startTime;
+                }
+                results.matchCount++;
+                config.onResult(match);
+            },
+        });
+
+
+        const searchPath = async (path: TimeFilePathWithSize) => {
+            if (!progressTracker.isSourceRelevant(path)) return;
+            // Wait, so we don't lock up the main thread?
+            await delay(0);
+
+            let remote = path.logCount !== undefined;
+            let archives = remote ? backblazeLogs : localLogs;
+            if (remote) {
+                results.backblazeFilesSearched++;
+            } else {
+                results.localFilesSearched++;
+            }
+            try {
+                let readIndexTime = Date.now();
+                let index = await archives.get(
+                    path.fullPath + INDEX_EXTENSION,
+                    { range: { start: 0, end: path.indexSize } }
+                ) || Buffer.alloc(0);
+                readIndexTime = Date.now() - readIndexTime;
+                results.indexSearchTime += readIndexTime;
+                let dataReader: Reader = {
+                    getLength: async () => path.size,
+                    read: async (offset, length) => {
+                        let endOffset = Math.min(offset + length, path.size);
+                        if (offset >= path.size) return Buffer.alloc(0);
+                        let data = await archives.get(path.fullPath, { range: { start: offset, end: endOffset } });
+                        if (!data) return Buffer.alloc(0);
+                        return data;
+                    },
+                };
+                let totalSearchTime = Date.now();
+                let blockSearchTimeBefore = results.blockSearchTime;
+                await BufferIndex.find({
+                    index,
+                    dataReader,
+                    params: {
+                        ...config.params,
+                        limit: config.params.limit - results.matchCount,
+                    },
+                    keepIterating: () => !results.cancel && progressTracker.isSourceRelevant(path),
+                    onResult: (match: Buffer) => {
+                        if (results.matchCount >= config.params.limit) return;
+                        progressTracker.addResult(match, path);
+                    },
+                    results,
+                });
+                totalSearchTime = Date.now() - totalSearchTime;
+                let blockSearchTimeAdded = results.blockSearchTime - blockSearchTimeBefore;
+                results.indexSearchTime += totalSearchTime - blockSearchTimeAdded;
+
+            } catch (e: any) {
+                results.fileErrors.push({ error: String(e?.stack || e), path: path.fullPath });
+                console.warn(`Error in reading log file ${path.fullPath} logs: ${e}`);
+            }
+        };
+        let localDone = (async () => {
+            for (let path of localPaths) {
+                await searchPath(path);
+            }
+        })();
+        // No parallel count when running locally, as running it in parallel makes timing more difficult. 
+        let parallelCount = isPublic() ? 32 : 1;
+        let remoteParallel = runInParallel({ parallelCount }, searchPath);
+        await Promise.all(remotePaths.map(remoteParallel));
+        await localDone;
+
+        if (results.timeToFirstMatch < 0) {
+            results.timeToFirstMatch = Date.now() - startTime;
+        }
+    }
+
+    public async moveLogsToPublic(forceAll = false) {
+        if (forceAll) {
+            await this.currentLogStream?.flushNow?.();
+        }
+
+        await moveLogsToPublic({
+            forceAll,
+            localLogs: this.getLocalLogs(),
+            publicLogs: this.getPublicLogs(),
+            publicMoveThreshold: PUBLIC_MOVE_THRESHOLD,
+            maxSingleFileData: this.config.maxSingleFileData || MAX_SINGLE_FILE_DATA,
+            movingTimeout: MOVING_TIMEOUT,
+            indexExtension: INDEX_EXTENSION,
+        });
+    }
+
+
+
+    // TEST functions
+    public async TEST_flushNow() {
+        await this.forceFlushStream?.();
+    }
+    public async TEST_deleteAllLocalLogs() {
+        let localLogs = this.getLocalLogs();
+        await new TimeFileTree(localLogs).deleteAll();
+    }
+    public async TEST_deleteAllBackblazeLogs() {
+        let backblazeLogs = this.getPublicLogs();
+        await new TimeFileTree(backblazeLogs).deleteAll();
+    }
+    public async TEST_deleteAllLogs() {
+        await this.TEST_deleteAllLocalLogs();
+        await this.TEST_deleteAllBackblazeLogs();
+
+        let localLogs = this.getLocalLogs();
+        // Also delete all .moving files
+        let movingFiles = await localLogs.find("", { shallow: true, type: "files" });
+        movingFiles = movingFiles.filter(x => x.endsWith(".moving"));
+        for (let movingFile of movingFiles) {
+            await localLogs.del(movingFile);
+        }
+    }
+}
+
+class IndexedLogClient {
+    public async onFind(config: {
+        findId: string;
+        indexedLogsName: string;
+        result: unknown;
+    }) {
+        if (isNode()) return;
+        let indexedLogs = loggerByName.get(config.indexedLogsName);
+        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        indexedLogs.onFindResult({
+            findId: config.findId,
+            result: config.result,
+        });
+    }
+    public async onResults(config: {
+        findId: string;
+        indexedLogsName: string;
+        results: IndexedLogResults;
+    }): Promise<boolean> {
+        if (isNode()) return true;
+        let indexedLogs = loggerByName.get(config.indexedLogsName);
+        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        return await indexedLogs.onResults({
+            findId: config.findId,
+            results: config.results,
+        });
+    }
+}
+class IndexedLogShim {
+    public async find(config: {
+        findId: string;
+        indexedLogsName: string;
+        params: SearchParams;
+    }): Promise<IndexedLogResults> {
+        let caller = SocketFunction.getCaller();
+        let indexedLogs = loggerByName.get(config.indexedLogsName);
+        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+
+        return indexedLogs.find({
+            params: config.params,
+            onResult: (match: unknown) => {
+                ignoreErrors(IndexedLogClientController.nodes[caller.nodeId].onFind({
+                    findId: config.findId,
+                    indexedLogsName: config.indexedLogsName,
+                    result: match,
+                }));
+            },
+            onResults: async (results: IndexedLogResults) => {
+                try {
+                    await IndexedLogClientController.nodes[caller.nodeId].onResults({
+                        findId: config.findId,
+                        indexedLogsName: config.indexedLogsName,
+                        results: results,
+                    });
+                    return true;
+                } catch (e) {
+                    return false;
+                }
+            },
+        });
+    }
+    public async getPaths(config: {
+        indexedLogsName: string;
+        startTime: number;
+        endTime: number;
+        only?: "local" | "public";
+    }): Promise<TimeFilePathWithSize[]> {
+        let indexedLogs = loggerByName.get(config.indexedLogsName);
+        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        return indexedLogs.getPaths({
+            startTime: config.startTime,
+            endTime: config.endTime,
+            only: config.only,
+        });
+    }
+
+    public async forceMoveLogsToPublic(config: {
+        indexedLogsName: string;
+    }) {
+        let indexedLogs = loggerByName.get(config.indexedLogsName);
+        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        await indexedLogs.moveLogsToPublic(true);
+    }
+}
+
+const IndexedLogShimController = SocketFunction.register(
+    "IndexedLogShim-019c87b7-73ca-72ec-91b3-2d45ebb616cd",
+    new IndexedLogShim(),
+    () => ({
+        find: {
+            hooks: [assertIsManagementUser]
+        },
+        getPaths: {
+            hooks: [assertIsManagementUser]
+        },
+        forceMoveLogsToPublic: {
+            hooks: [assertIsManagementUser]
+        }
+    })
+);
+
+const IndexedLogClientController = SocketFunction.register(
+    "IndexedLogClient-019c87b9-1c6d-72ed-8bc9-d52451e2c1b9",
+    new IndexedLogClient(),
+    () => ({
+        onFind: {},
+        onResults: {}
+    })
+);