querysub 0.357.0 → 0.358.0

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -2,8 +2,8 @@ import { lazy } from "socket-function/src/caching";
 import { Archives, nestArchives } from "../../../-a-archives/archives";
 import { deepCloneJSON, keyByArray, nextId, sort, timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { BufferIndex } from "./BufferIndex";
-import { delay, runInSerial, runInfinitePoll } from "socket-function/src/batching";
-import { Reader, SearchParams } from "./BufferIndexHelpers";
+import { delay, runInParallel, runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { IndexedLogResults, Reader, SearchParams, addReadToResults, createEmptyIndexedLogResults, INDEX_EXTENSION } from "./BufferIndexHelpers";
 import { getDomain, isPublic } from "../../../config";
 import { getArchivesLocal } from "../../../-a-archives/archivesDisk";
 import { getArchivesBackblaze, getArchivesBackblazePrivateImmutable } from "../../../-a-archives/archivesBackBlaze";
@@ -21,49 +21,16 @@ import { FindProgressTracker } from "./FindProgressTracker";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { assertIsManagementUser } from "../../managementPages";
 import { ignoreErrors } from "../../../errors";
+import { blue } from "socket-function/src/formatting/logColors";
+import { LimitGroup } from "../../../functional/limitProcessing";
 
 export type TimeFilePathWithSize = TimeFilePath & {
     size: number;
-    sourceName: string;
-};
-
-export type IndexedLogResults = {
-    matchCount: number;
-
-    // NOTE: A lot of the metadata won't be accurate if multiple searches happen at the same time. However, for debugging, it should be sufficient. 
-    reads: {
-        cached: boolean;
-        remote: boolean;
-        count: number;
-        size: number;
-
-        totalSize: number;
-        totalCount: number;
-    }[];
-
-    localFilesSearched: number;
-    backblazeFilesSearched: number;
-
-    totalBlockCount: number;
-    blockCheckedCount: number;
-    blocksCheckedCompressedSize: number;
-    blocksCheckedDecompressedSize: number;
-    blockErrors: string[];
-
-    fileErrors: string[];
-
-    indexesSearched: number;
     indexSize: number;
-
-    timeToFirstMatch: number;
-    fileFindTime: number;
-    indexSearchTime: number;
-    blockSearchTime: number;
-
-    totalSearchTime: number;
+    sourceName: string;
 };
 
-let logByName = new Map<string, IndexedLogs<unknown>>();
+let loggerByName = new Map<string, IndexedLogs<unknown>>();
 
 
 export class IndexedLogs<T> {
@@ -74,22 +41,24 @@ export class IndexedLogs<T> {
         forceUsePublicLogs?: boolean;
         getTime: (result: T) => number | undefined;
     }) {
-        logByName.set(this.config.name, this as any);
+        loggerByName.set(this.config.name, this as any);
     }
 
 
     private static shouldRunLoop = false;
     public static runLogMoveLoop() {
         IndexedLogs.shouldRunLoop = true;
-        for (let indexedLogs of logByName.values()) {
+        for (let indexedLogs of loggerByName.values()) {
             indexedLogs.runLogMoverLoop();
         }
     }
 
     private findCallbacks = new Map<string, (match: T) => void>();
+    private resultsCallbacks = new Map<string, (results: IndexedLogResults) => void>();
     public async clientFind(config: {
         params: SearchParams;
         onResult: (match: T) => void;
+        onResults?: (results: IndexedLogResults) => void;
     }): Promise<IndexedLogResults> {
         let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
         let findId = nextId();
@@ -97,6 +66,9 @@ export class IndexedLogs<T> {
             config.onResult(match);
         };
         this.findCallbacks.set(findId, callback);
+        if (config.onResults) {
+            this.resultsCallbacks.set(findId, config.onResults);
+        }
         try {
             return await controller.find({
                 findId,
@@ -107,9 +79,14 @@ export class IndexedLogs<T> {
             // There's some trailing time after the controller call finishes when the results will be trickling back to us. 
             setTimeout(() => {
                 this.findCallbacks.delete(findId);
+                this.resultsCallbacks.delete(findId);
             }, timeInMinute * 30);
         }
     }
+    public clientCancelAllCallbacks() {
+        this.findCallbacks.clear();
+        this.resultsCallbacks.clear();
+    }
     public async clientGetPaths(config: {
         startTime: number;
         endTime: number;
@@ -128,12 +105,24 @@ export class IndexedLogs<T> {
         result: unknown;
     }) {
         let callback = this.findCallbacks.get(config.findId);
-        if (!callback) return;
+        if (!callback) throw new Error(`Find callback ${config.findId} not found`);
         callback(config.result as T);
     }
+    public async onResults(config: {
+        findId: string;
+        results: IndexedLogResults;
+    }): Promise<boolean> {
+        let callback = this.resultsCallbacks.get(config.findId);
+        if (!callback) throw new Error(`Results callback ${config.findId} not found`);
+        callback(config.results);
+        return true;
+    }
 
     public async clientForceMoveLogsToPublic() {
-        await this.moveLogsToPublic(true);
+        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
+        await controller.forceMoveLogsToPublic({
+            indexedLogsName: this.config.name,
+        });
     }
 
     private localLogsStats = {
@@ -153,23 +142,29 @@ export class IndexedLogs<T> {
         totalCacheCount: 0,
     };
 
+    private fileSizeCache = new Map<string, number>();
+
     private getLocalLogs = lazy((): Archives => {
         let baseDisk = getArchivesLocal(getDomain());
         let archives = nestArchives("indexed-logs/" + this.config.name, baseDisk);
         archives = createArchivesMemoryCache(archives, {
             maxSize: 1024 * 1024 * 512,
             maxCount: 1000 * 100,
-            stats: this.localLogsStats
+            stats: this.localLogsStats,
+            sizeCache: this.fileSizeCache,
+            // Local disk reads are fast, but even local reads can have high latency
+            extraReadSize: 1024 * 1024 * 1,
         });
         return archives;
     });
     private getPublicLogs = lazy((): Archives => {
         let basePublic: Archives = getArchivesLocal(getDomain());
-        let extraReadSize = 1024;
+        // NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever. 
+        let extraReadSize = 1024 * 1024 * 10;
         if (this.config.forceUsePublicLogs || isPublic()) {
             basePublic = getArchivesBackblaze(getDomain());
-            // The latency on Backblaze is so high that if we're going to do a read, we might as well read a lot. As in, minimum 2s latency. The bandwidth is good though.
-            extraReadSize = 1024 * 1024 * 10;
+            // NOTE: While the latency to back plays is high, now we're reading in parallel, so it shouldn't be as big of an issue. 
+            extraReadSize = 1024 * 1024 * 1;
         }
         let archives = nestArchives("final-indexed-logs/" + this.config.name, basePublic);
         archives = createArchivesMemoryCache(archives, {
@@ -178,6 +173,7 @@ export class IndexedLogs<T> {
             fullyImmutable: true,
             extraReadSize,
             stats: this.backblazeLogsStats,
+            sizeCache: this.fileSizeCache,
         });
         return archives;
     });
@@ -198,20 +194,19 @@ export class IndexedLogs<T> {
     private forceFlushStream: (() => Promise<void>) | undefined;
 
     private getCurrentLogStream(): LogStreamer<T> {
+
         if (!this.currentLogStream) {
             let { startTime, endTime } = this.getTimeBlock(Date.now());
             let path = new TimeFileTree(this.getLocalLogs()).getNewPendingPath({ startTime, endTime });
-            let indexPath = this.getIndexPath(path);
             let streamer = BufferIndex.createStreamer();
             let currentLogStreamSize = 0;
             let currentLogStreamCount = 0;
             let newStreamer = async () => {
                 let result = streamer.close();
                 await this.getLocalLogs().append(path, result.data);
-                await this.getLocalLogs().append(indexPath, result.index);
+                await this.getLocalLogs().append(path + INDEX_EXTENSION, result.index);
 
                 path = new TimeFileTree(this.getLocalLogs()).getNewPendingPath({ startTime, endTime });
-                indexPath = this.getIndexPath(path);
                 streamer = BufferIndex.createStreamer();
                 currentLogStreamSize = 0;
                 currentLogStreamCount = 0;
@@ -224,6 +219,14 @@ export class IndexedLogs<T> {
 
             };
             let writeBuffers = async (buffers: Buffer[]) => {
+                if (Date.now() > endTime) {
+                    let timeBlockObj = this.getTimeBlock(Date.now());
+                    startTime = timeBlockObj.startTime;
+                    endTime = timeBlockObj.endTime;
+                    path = new TimeFileTree(this.getLocalLogs()).getNewPendingPath(timeBlockObj);
+                    await newStreamer();
+                }
+
                 let maxSize = this.config.maxSingleFileData || MAX_SINGLE_FILE_DATA;
                 let maxCount = this.config.maxCountPerFile || MAX_COUNT_PER_FILE;
 
@@ -239,7 +242,7 @@ export class IndexedLogs<T> {
                     let result = streamer.add(group);
                     await this.getLocalLogs().append(path, result.data);
                     if (result.index) {
-                        await this.getLocalLogs().append(indexPath, result.index);
+                        await this.getLocalLogs().append(path + INDEX_EXTENSION, result.index);
                     }
 
                     currentLogStreamSize += result.data.length;
@@ -284,10 +287,6 @@ export class IndexedLogs<T> {
         return groups;
     }
 
-    private getIndexPath(dataPath: string): string {
-        return dataPath + ".index";
-    }
-
     public append(datum: T) {
         this.getCurrentLogStream().append(datum);
         if (IndexedLogs.shouldRunLoop) {
@@ -295,6 +294,7 @@ export class IndexedLogs<T> {
         }
     }
 
+    @measureFnc
     public async getPaths(config: {
         startTime: number;
         endTime: number;
@@ -326,16 +326,98 @@ export class IndexedLogs<T> {
             let archives = path.logCount !== undefined ? backblazeLogs : localLogs;
             let info = await archives.getInfo(path.fullPath);
             let size = info?.size || 0;
-            return { ...path, size, sourceName: this.config.name };
+            this.fileSizeCache.set(path.fullPath, size);
+            let indexPath = path.fullPath + INDEX_EXTENSION;
+            let indexInfo = await archives.getInfo(indexPath);
+            let indexSize = indexInfo?.size || 0;
+            this.fileSizeCache.set(indexPath, indexSize);
+            return { ...path, size, indexSize, sourceName: this.config.name };
         }));
 
         return pathsWithSize;
     }
 
+    @measureFnc
     public async find(config: {
         params: SearchParams;
         onResult: (match: T) => void;
+        onResults?: (results: IndexedLogResults) => Promise<boolean>;
     }): Promise<IndexedLogResults> {
+        let startTime = Date.now();
+        let interval: NodeJS.Timeout | undefined;
+
+        let results: IndexedLogResults = createEmptyIndexedLogResults();
+        results.limitGroup = new LimitGroup({
+            maxTimePerBeforeWait: 500,
+            waitTime: 250,
+        });
+
+        let initialLocalStats = deepCloneJSON(this.localLogsStats);
+        let initialBackblazeStats = deepCloneJSON(this.backblazeLogsStats);
+
+        let updateResultsStats = () => {
+            results.reads = [];
+
+            function addStatsRead(initial: ArchivesMemoryCacheStats, final: ArchivesMemoryCacheStats, remote: boolean) {
+                let uncached = addReadToResults(results, {
+                    cached: false,
+                    remote,
+                    count: final.uncachedReads - initial.uncachedReads,
+                    size: final.uncachedReadSize - initial.uncachedReadSize,
+                });
+                uncached.totalSize = final.totalCacheSize;
+                uncached.totalCount = final.totalCacheCount;
+
+                let cached = addReadToResults(results, {
+                    cached: true,
+                    remote,
+                    count: final.cachedReads - initial.cachedReads,
+                    size: final.cachedReadSize - initial.cachedReadSize,
+                });
+                cached.totalSize = final.totalCacheSize;
+                cached.totalCount = final.totalCacheCount;
+            }
+
+            addStatsRead(initialLocalStats, this.localLogsStats, false);
+            addStatsRead(initialBackblazeStats, this.backblazeLogsStats, true);
+
+            results.totalSearchTime = Date.now() - startTime;
+        };
+
+        const onResultsCallback = config.onResults;
+        if (onResultsCallback) {
+            interval = setInterval(async () => {
+                updateResultsStats();
+                let shouldContinue = await onResultsCallback(deepCloneJSON(results));
+                if (!shouldContinue) {
+                    if (!results.cancel) {
+                        console.log(blue(`Cancelled search on ${this.config.name}`));
+                    }
+                    results.cancel = true;
+                }
+            }, timeInSecond);
+        }
+
+        try {
+            await this.findBase({
+                params: config.params,
+                onResult: config.onResult,
+                results,
+            });
+            updateResultsStats();
+            return results;
+        } finally {
+            if (interval) {
+                clearInterval(interval);
+            }
+        }
+    }
+
+    private async findBase(config: {
+        params: SearchParams;
+        onResult: (match: T) => void;
+        results: IndexedLogResults;
+    }): Promise<void> {
         let startTime = Date.now();
         let localLogs = this.getLocalLogs();
         let backblazeLogs = this.getPublicLogs();
@@ -347,54 +429,56 @@ export class IndexedLogs<T> {
             only: config.params.only,
         });
         paths = paths.filter(x => x.sourceName === this.config.name);
+        for (let path of paths) {
+            this.fileSizeCache.set(path.fullPath, path.size);
+            this.fileSizeCache.set(path.fullPath + INDEX_EXTENSION, path.indexSize);
+        }
         fileFindTime = Date.now() - fileFindTime;
 
         // Newest first
         sort(paths, x => - x.startTime);
 
+        let results = config.results;
         let localPaths = paths.filter(x => x.logCount === undefined);
-        let backblazePaths = paths.filter(x => x.logCount !== undefined);
-
-        let initialLocalStats = deepCloneJSON(this.localLogsStats);
-        let initialBackblazeStats = deepCloneJSON(this.backblazeLogsStats);
-        let stats = {
-            blockCheckedCount: 0,
-            blocksCheckedCompressedSize: 0,
-            blocksCheckedDecompressedSize: 0,
-            totalBlockCount: 0,
-            indexSearchTime: 0,
-            blockSearchTime: 0,
-            indexesSearched: 0,
-            indexSize: 0,
-            matchCount: 0,
-            timeToFirstMatch: -1,
-        };
-        let fileErrors: string[] = [];
-        let blockErrors: string[] = [];
+        let remotePaths = paths.filter(x => x.logCount !== undefined);
+        results.totalLocalFiles = localPaths.length;
+        results.totalBackblazeFiles = remotePaths.length;
+        results.fileFindTime = fileFindTime;
 
         let progressTracker = new FindProgressTracker<T>({
             params: config.params,
             deserialize: (buffer: Buffer) => LogStreamer.deserialize<T>(buffer),
             getTime: this.config.getTime,
             onResult: (match: T) => {
-                if (stats.timeToFirstMatch < 0) {
-                    stats.timeToFirstMatch = Date.now() - startTime;
+                if (results.timeToFirstMatch < 0) {
+                    results.timeToFirstMatch = Date.now() - startTime;
                 }
-                stats.matchCount++;
+                results.matchCount++;
                 config.onResult(match);
             },
         });
 
 
-        for (let path of paths) {
-            if (!progressTracker.isSourceRelevant(path)) continue;
+        const searchPath = async (path: TimeFilePathWithSize) => {
+            if (!progressTracker.isSourceRelevant(path)) return;
+            // Wait, so we don't lock up the main thread?
+            await delay(0);
 
-            let archives = path.logCount !== undefined ? backblazeLogs : localLogs;
+            let remote = path.logCount !== undefined;
+            let archives = remote ? backblazeLogs : localLogs;
+            if (remote) {
+                results.backblazeFilesSearched++;
+            } else {
+                results.localFilesSearched++;
+            }
             try {
                 let readIndexTime = Date.now();
-                let index = await archives.get(this.getIndexPath(path.fullPath)) || Buffer.alloc(0);
+                let index = await archives.get(
+                    path.fullPath + INDEX_EXTENSION,
+                    { range: { start: 0, end: path.indexSize } }
+                ) || Buffer.alloc(0);
                 readIndexTime = Date.now() - readIndexTime;
-                stats.indexSearchTime += readIndexTime;
+                results.indexSearchTime += readIndexTime;
                 let dataReader: Reader = {
                     getLength: async () => path.size,
                     read: async (offset, length) => {
@@ -406,76 +490,44 @@ export class IndexedLogs<T> {
                     },
                 };
                 let totalSearchTime = Date.now();
-                let findObj = await BufferIndex.find({
+                let blockSearchTimeBefore = results.blockSearchTime;
+                await BufferIndex.find({
                     index,
                     dataReader,
                     params: {
                         ...config.params,
-                        limit: config.params.limit - stats.matchCount,
+                        limit: config.params.limit - results.matchCount,
                     },
-                    keepIterating: () => progressTracker.isSourceRelevant(path),
+                    keepIterating: () => !results.cancel && progressTracker.isSourceRelevant(path),
                     onResult: (match: Buffer) => {
-                        if (stats.matchCount >= config.params.limit) return;
+                        if (results.matchCount >= config.params.limit) return;
                         progressTracker.addResult(match, path);
                     },
+                    results,
                 });
-                blockErrors.push(...findObj.blocksWithErrors);
                 totalSearchTime = Date.now() - totalSearchTime;
-                stats.indexSearchTime += totalSearchTime - findObj.blockSearchTime;
-                stats.blockSearchTime += findObj.blockSearchTime;
-                stats.indexesSearched += findObj.indexCount;
-                stats.indexSize += findObj.indexSize;
-                stats.blockCheckedCount += findObj.blocksChecked;
-                stats.blocksCheckedCompressedSize += findObj.blocksCheckedCompressedSize;
-                stats.blocksCheckedDecompressedSize += findObj.blocksCheckedDecompressedSize;
-                stats.totalBlockCount += findObj.totalBlockCount;
-
-                if (stats.matchCount >= config.params.limit) break;
+                let blockSearchTimeAdded = results.blockSearchTime - blockSearchTimeBefore;
+                results.indexSearchTime += totalSearchTime - blockSearchTimeAdded;
+
             } catch (e: any) {
-                fileErrors.push(String(e?.stack || e));
+                results.fileErrors.push({ error: String(e?.stack || e), path: path.fullPath });
                 console.warn(`Error in reading log file ${path.fullPath} logs: ${e}`);
             }
-        }
-
-        if (stats.timeToFirstMatch < 0) {
-            stats.timeToFirstMatch = Date.now() - startTime;
-        }
-
-
-        let reads: { cached: boolean; remote: boolean; count: number; size: number; totalSize: number; totalCount: number; }[] = [];
-        function addStatsRead(initial: ArchivesMemoryCacheStats, final: ArchivesMemoryCacheStats, remote: boolean) {
-            reads.push({
-                cached: false,
-                remote,
-                count: final.uncachedReads - initial.uncachedReads,
-                size: final.uncachedReadSize - initial.uncachedReadSize,
-                totalSize: final.totalCacheSize,
-                totalCount: final.totalCacheCount,
-            });
-            reads.push({
-                cached: true,
-                remote,
-                count: final.cachedReads - initial.cachedReads,
-                size: final.cachedReadSize - initial.cachedReadSize,
-                totalSize: final.totalCacheSize,
-                totalCount: final.totalCacheCount,
-            });
-        }
-
-        addStatsRead(initialLocalStats, this.localLogsStats, false);
-        addStatsRead(initialBackblazeStats, this.backblazeLogsStats, true);
-
-
-        return {
-            reads,
-            localFilesSearched: localPaths.length,
-            backblazeFilesSearched: backblazePaths.length,
-            fileFindTime,
-            fileErrors,
-            blockErrors,
-            totalSearchTime: Date.now() - startTime,
-            ...stats,
         };
+        let localDone = (async () => {
+            for (let path of localPaths) {
+                await searchPath(path);
+            }
+        })();
+        // No parallel count when running locally, as running it in parallel makes timing more difficult. 
+        let parallelCount = isPublic() ? 32 : 1;
+        let remoteParallel = runInParallel({ parallelCount }, searchPath);
+        await Promise.all(remotePaths.map(remoteParallel));
+        await localDone;
+
+        if (results.timeToFirstMatch < 0) {
+            results.timeToFirstMatch = Date.now() - startTime;
+        }
     }
 
     public async moveLogsToPublic(forceAll = false) {
@@ -490,7 +542,7 @@ export class IndexedLogs<T> {
             publicMoveThreshold: PUBLIC_MOVE_THRESHOLD,
             maxSingleFileData: this.config.maxSingleFileData || MAX_SINGLE_FILE_DATA,
             movingTimeout: MOVING_TIMEOUT,
-            getIndexPath: (path: string) => this.getIndexPath(path),
+            indexExtension: INDEX_EXTENSION,
         });
     }
 
@@ -529,13 +581,26 @@ class IndexedLogClient {
         result: unknown;
     }) {
         if (isNode()) return;
-        let indexedLogs = logByName.get(config.indexedLogsName);
+        let indexedLogs = loggerByName.get(config.indexedLogsName);
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
         indexedLogs.onFindResult({
             findId: config.findId,
             result: config.result,
         });
     }
+    public async onResults(config: {
+        findId: string;
+        indexedLogsName: string;
+        results: IndexedLogResults;
+    }): Promise<boolean> {
+        if (isNode()) return true;
+        let indexedLogs = loggerByName.get(config.indexedLogsName);
+        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        return await indexedLogs.onResults({
+            findId: config.findId,
+            results: config.results,
+        });
+    }
 }
 class IndexedLogShim {
     public async find(config: {
@@ -544,8 +609,9 @@ class IndexedLogShim {
         params: SearchParams;
     }): Promise<IndexedLogResults> {
         let caller = SocketFunction.getCaller();
-        let indexedLogs = logByName.get(config.indexedLogsName);
+        let indexedLogs = loggerByName.get(config.indexedLogsName);
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+
         return indexedLogs.find({
             params: config.params,
             onResult: (match: unknown) => {
@@ -555,6 +621,18 @@ class IndexedLogShim {
                     result: match,
                 }));
             },
+            onResults: async (results: IndexedLogResults) => {
+                try {
+                    await IndexedLogClientController.nodes[caller.nodeId].onResults({
+                        findId: config.findId,
+                        indexedLogsName: config.indexedLogsName,
+                        results: results,
+                    });
+                    return true;
+                } catch (e) {
+                    return false;
+                }
+            },
         });
     }
     public async getPaths(config: {
@@ -563,7 +641,7 @@ class IndexedLogShim {
         endTime: number;
         only?: "local" | "public";
     }): Promise<TimeFilePathWithSize[]> {
-        let indexedLogs = logByName.get(config.indexedLogsName);
+        let indexedLogs = loggerByName.get(config.indexedLogsName);
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
         return indexedLogs.getPaths({
             startTime: config.startTime,
@@ -571,6 +649,14 @@ class IndexedLogShim {
             only: config.only,
         });
     }
+
+    public async forceMoveLogsToPublic(config: {
+        indexedLogsName: string;
+    }) {
+        let indexedLogs = loggerByName.get(config.indexedLogsName);
+        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        await indexedLogs.moveLogsToPublic(true);
+    }
 }
 
 const IndexedLogShimController = SocketFunction.register(
@@ -583,7 +669,7 @@ const IndexedLogShimController = SocketFunction.register(
         getPaths: {
             hooks: [assertIsManagementUser]
         },
-        clientForceMoveLogsToPublic: {
+        forceMoveLogsToPublic: {
             hooks: [assertIsManagementUser]
         }
     })
@@ -593,6 +679,7 @@ const IndexedLogClientController = SocketFunction.register(
     "IndexedLogClient-019c87b9-1c6d-72ed-8bc9-d52451e2c1b9",
     new IndexedLogClient(),
     () => ({
-        onFind: {}
+        onFind: {},
+        onResults: {}
     })
 );