querysub 0.406.0 → 0.408.0

package/src/test/satTest.ts

@@ -1,193 +0,0 @@
-import debugbreak from "debugbreak";
-import { configRootDiscoveryLocation } from "../-f-node-discovery/NodeDiscovery";
-import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { Querysub } from "../4-querysub/QuerysubController";
-configRootDiscoveryLocation({ domain: "querysub.com", port: 5050 });
-
-import { Benchmark } from "../diagnostics/benchmark";
-import { createBenchmarkFunctions, sat_functions } from "../diagnostics/satSchema";
-import { runSaturationTests, DriftValues, runSingleSaturationTest } from "../diagnostics/runSaturationTest";
-import { PermissionsCheck } from "../4-querysub/permissions";
-import { appendToPathStr, getPathIndex, getPathStr, getPathSuffix, rootPathStr, trimPathStrToDepth } from "../path";
-import { runCodeWithDatabase } from "../2-proxy/pathDatabaseProxyBase";
-import { createPathValueProxy } from "../2-proxy/pathValueProxy";
-import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { measureCode, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
-import { DEPTH_TO_DATA, DOMAIN_INDEX, MODULE_INDEX, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
-import { cache } from "socket-function/src/caching";
-import { timeInHour, timeInMinute } from "socket-function/src/misc";
-import { delay } from "socket-function/src/batching";
-import { parseHeapSnapshot, processAndSummarizeHeapsnapshot, writeHeapSnapshot } from "../heapDumps";
-import { formatNumber } from "socket-function/src/formatting/format";
-import fs from "fs";
-
-
-// A stress test with a single watcher
-/*
-    (watchdog at 12/31/2023) Profiled 684ms (logged at 2023-12-31T07:05:41.871Z)
-    ClientWatcher().setWatches(async)           34.47% (  236ms = 85 * 2.73ms   +  114 * 33.2us )
-    findFunctionsToCall                         16.12% (  110ms =      85 *  1.3ms )
-    PathWatcher().watchPath                     15.25% (  104ms = 85 * 1.22ms   +  114 * 8.68us )
-    watchLatest|getSingleReadNode                7.23% ( 49.5ms = 85 *  570us   +  114 * 9.35us )
-    PathWatcher().triggerLatestWatcher           5.95% ( 40.7ms =     169 *  241us )
-
-    (watchdog at 12/31/2023) Profiled 1.21s (logged at 2023-12-31T07:23:31.501Z)
-    findFunctionsToCall                         21.14% (  256ms = 5 * 42.1ms   +  19 * 2.39ms )
-    PathWatcher().triggerLatestWatcher          15.39% (  186ms =     919 *  203us )
-    PathValueCommitter().commitValues           12.14% (  147ms =     898 *  164us )
-    RemoteWatcher().watchLatest(async)          11.21% (  136ms =     922 *  147us )
-    AuthorityPathValueStorage().ingestValues     6.22% ( 75.3ms = 52 *  696us   +  869 * 45us )
-*/
-//  The proxyWatcher is taking most of the time, specifically due to the fact that it does not natively
-//      support delta based functions.
-
-async function stressTest() {
-    let targetRatePerSecond = 200;
-    let startTime = Date.now();
-
-    for (let i = 0; i < 300 * 300; i++) {
-        sat_functions.setValues([i]);
-
-        while (true) {
-            let rate = i / (Date.now() - startTime) * 1000;
-            if (rate < targetRatePerSecond) break;
-            await delay(16);
-        }
-    }
-
-}
-stressTest().catch(e => console.error(e)).finally(() => process.exit());
-
-async function timeTest() {
-    await Querysub.hostService("test");
-
-    await proxyWatcher.commitFunction({
-        watchFunction() {
-            const check = new PermissionsCheck({ callerMachineId: "", callerIP: "" });
-            let allowed = check.checkPermissions(getPathStr(["querysub.com", "PathFunctionRunner", "test", "Data", "test", "1"]));
-            console.log({ allowed });
-        },
-    });
-
-    // ~1.62us, so... if we can optimize this code... that would be good...
-    //  AND, it doesn't even emit any specific paths, so... we can probably cache it a lot...
-    let cache = new Map<string, { domainName: string; moduleId: string; pathPrefix: string; rootKey: string }>();
-    function splitParts(path: string): { domainName: string; moduleId: string; pathPrefix: string; rootKey: string } {
-        let pathPrefix = trimPathStrToDepth(path, DEPTH_TO_DATA);
-        let result = cache.get(pathPrefix);
-        if (!result) {
-            const domainName = getPathIndex(path, DOMAIN_INDEX) || "";
-            const moduleId = getPathIndex(path, MODULE_INDEX) || "";
-            let rootKey = getPathIndex(path, DEPTH_TO_DATA - 1) || "";
-            result = { domainName, moduleId, pathPrefix, rootKey };
-            cache.set(pathPrefix, result);
-        }
-        return result;
-    }
-
-    const simulateCheckPermissions = measureWrap(function simulateCheckPermissions(path: string) {
-        return trimPathStrToDepth(path, DEPTH_TO_DATA);
-
-        // let obj = splitParts(path);
-        // if (!obj.domainName) return { permissionsPath: trimPathStrToDepth(path, DOMAIN_INDEX + 1), allowed: false };
-        // if (!obj.moduleId) return { permissionsPath: trimPathStrToDepth(path, MODULE_INDEX + 1), allowed: false };
-        // if (!obj.rootKey) return { permissionsPath: trimPathStrToDepth(path, DEPTH_TO_DATA), allowed: false };
-
-        // let pathInModule = getPathSuffix(path, DEPTH_TO_DATA - 1);
-        // if (!pathInModule) return { permissionsPath: path, allowed: false };
-        // if (pathInModule === rootPathStr) return { permissionsPath: path, allowed: false };
-
-        // return obj;
-    });
-
-
-    await measureCode(async function permissionsCheckTest() {
-        await proxyWatcher.commitFunction({
-            watchFunction() {
-                console.log(`Start perf test`);
-
-                const check = new PermissionsCheck({ callerMachineId: "", callerIP: "127.0.0.1" });
-                let paths: string[] = [];
-                for (let i = 0; i < 1000 * 100; i++) {
-                    paths.push(getPathStr(["querysub.com", "PathFunctionRunner", "test", "Results", i.toString()]));
-                    //paths.push(getPathStr(["querysub.com", "PathFunctionRunner", "test", "Data", i.toString()]));
-                }
-                for (let path of paths) {
-                    check.checkPermissions(path);
-                }
-            },
-        });
-    });
-}
-//timeTest().catch(e => console.error(e)).finally(() => process.exit());
-
-export async function main() {
-    // let values = new DriftValues({ count: 9, modulus: 50 * 50, driftFraction: 0.1 });
-    // for (let i = 0; i < 10; i++) {
-    //     let calls = Array(100).fill(0).map(() =>
-    //         Benchmark.waitForCallToFinish(sat_functions.setValues(values.getNextValues()))
-    //     );
-    //     await Promise.all(calls);
-    // }
-
-
-    // await runSingleSaturationTest({
-    //     duration: 5000,
-    //     saturation: 100,
-    //     writeSpec: new DriftValues({ count: 9, modulus: 50 * 50, driftFraction: 0.1 }),
-    //     functions: createBenchmarkFunctions(),
-    // });
-
-    await runSaturationTests({
-        durationPer: 1000,
-        //writeSpec: new DriftValues({ count: 77, modulus: INDEX_EXTENT, driftFraction: 0.1 }),
-        //writeSpec: new DriftValues({ count: 10, modulus: INDEX_EXTENT, driftFraction: 0.1 }),
-        //writeSpec: new DriftValues({ count: 9, modulus: 50 * 50, driftFraction: 0.1 }),
-        writeSpec: new DriftValues({ count: 1, modulus: 50 * 50, driftFraction: 0.1 }),
-        functions: createBenchmarkFunctions(),
-    });
-}
-//main().catch(e => console.error(e)).finally(() => process.exit());
-
-
-/* 1s test time, 0 watchers (more watchers is slower)
-    new DriftValues({ count: 9, modulus: 50 * 50, driftFraction: 0.1 })
-Suggestions
-    Min latency          60.7/s     latency = 16.5ms (24.68% saturation)       (15 count)
-*** Min latency 2X       259/s      latency = 30.9ms (799.51% saturation)      (264 count)
-    100ms latency        348/s      latency = 91.6ms (3190.90% saturation)     (352 count)
-*/
-
-/*  1s test time
-    new DriftValues({ count: 1, modulus: 50 * 50, driftFraction: 0.1 })
-Suggestions
-    Min latency          42.6/s     latency = 23.4ms (25.45% saturation)       (11 count)
-*** Min latency 2X       303/s      latency = 52.7ms (1596.21% saturation)     (304 count)
-    100ms latency        368/s      latency = 86.8ms (3191.19% saturation)     (384 count)
-*/
-
-
-/* Same test on MongoDB
-    1s test time
-    writeSpec: new DriftValues({ count: 1, modulus: 50 * 50, driftFraction: 0.1 }),
-Suggestions
-    Min latency          214/s      latency = 4.68ms (24.53% saturation)       (53 count)
-*** Min latency 2X       4467/s     latency = 7.14ms (3190.04% saturation)     (4485 count)
-    100ms latency        4768/s     latency = 104ms  (49459.71% saturation)    (4983 count)
-    !!!! ERROR !!!!: 534 indexes were wrong, 1.80% of operations were missed
-
-    So mongo has about 1/4 the latency and 15X the throughput.
-        - Although, only 10X the throughput if we oversaturate our server and allow our latency to jump to 76.6ms.
-            WHICH, is fair, because the clientside predicts all the values, so the perceived latency will still usually be 0ms.
-        - ALSO, technically our perceived latency will be 0ms, at least for writes, due to client side prediction.
-            - ALSO, on any reasonable network the latency will be MUCH higher than 7ms, or 26ms, due to the network latency.
-                Except due to clientside prediction, Querysub will again, have 0ms perceived latency, so... our latency is
-                    actually MUCH better.
-        - The throughput different is promising, considering how we use javascript (and worse, Proxies) for everything. Also,
-            we are bottlenecked on function evaluation, which technically evaluates arbitrary application functions, which makes
-            it significantly more powerful than what mongodb support.
-            - The overwhelming majority of our time is spent in function evaluation (~80%), which is quite isolated, so
-                optimization should be possible (ex, rewriting FunctionRunner to run in another language, to run
-                specially application optimized functions in that other language).
-*/
-