querysub 0.2.0

package/src/diagnostics/runSaturationTest.ts

@@ -0,0 +1,416 @@
+import { delay } from "socket-function/src/batching";
+import { formatTime, formatNumber } from "socket-function/src/formatting/format";
+import { blue, green, red, yellow } from "socket-function/src/formatting/logColors";
+import { createStatsValue, addToStatsValue, StatsValue, getStatsTop } from "socket-function/src/profiling/stats";
+import { percent, formatStats } from "socket-function/src/profiling/statsFormat";
+import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { timeoutToError, logErrors } from "../errors";
+import { PromiseObj } from "../promise";
+import { Benchmark } from "./benchmark";
+
+export class DriftValues {
+    constructor(public readonly config: {
+        count: number;
+        modulus: number;
+        driftFraction: number;
+    }) { }
+    seqNum = 0;
+    public getNextValues(): number[] {
+        const { count, modulus, driftFraction } = this.config;
+        let curSeqNum = this.seqNum++;
+        let drift = Math.ceil(driftFraction * count);
+        let startValue = curSeqNum * drift % modulus;
+        return Array(count).fill(0).map((_, i) => (startValue + i) % modulus);
+    }
+}
+
+export interface BenchmarkFunctions {
+    setValues(indexes: number[], value?: number): Promise<void>;
+    getValues(indexes: number[]): Promise<number[]>;
+}
+
+export async function runSingleSaturationTest(config: {
+    duration: number;
+    saturation: number;
+    writeSpec: DriftValues;
+    predictedValues?: number[];
+    functions: BenchmarkFunctions;
+}) {
+    const { duration, saturation, writeSpec, predictedValues, functions } = config;
+
+    let finalEndTime = Date.now() + duration;
+    let done = new PromiseObj();
+
+    let baseStartTime = Date.now();
+    let totalOverlappedWaitTime = 0;
+    let errorCount = 0;
+
+    let lastActiveStartTime = 0;
+    let totalActiveTime = 0;
+
+    let runCount = 0;
+
+    let latencyStats = createStatsValue();
+
+    let currentParallelCount = 0;
+    async function spawnCall() {
+        if (currentParallelCount === 0) {
+            lastActiveStartTime = Date.now();
+        }
+        currentParallelCount++;
+        let time = Date.now();
+        let errored = true;
+        try {
+            let indexes = writeSpec.getNextValues();
+            if (predictedValues) {
+                for (let i of indexes) {
+                    predictedValues[i]++;
+                }
+            }
+            await timeoutToError(30 * 1000, functions.setValues(indexes), () => new Error(`Call timed out after 30 seconds`));
+            errored = false;
+        } finally {
+            let latency = Date.now() - time;
+            totalOverlappedWaitTime += latency;
+            addToStatsValue(latencyStats, latency);
+            if (errored) {
+                errorCount++;
+            }
+            runCount++;
+            currentParallelCount--;
+            if (currentParallelCount === 0) {
+                totalActiveTime += Date.now() - lastActiveStartTime;
+            }
+            trySpawnMore();
+        }
+    }
+    function getTargetParallelCount() {
+        let currentElapsedTime = Date.now() - baseStartTime;
+        let currentOverallSaturation = totalOverlappedWaitTime / currentElapsedTime;
+
+        // TODO: Use higher or lower saturation values if we are way off, BUT, only after the time is advanced enough,
+        //  and only between 2X and 0.5X.
+
+        // Round down if we are oversatured, round up if we are undersaturated
+        if (currentOverallSaturation > saturation) {
+            return Math.floor(saturation);
+        } else {
+            return Math.ceil(saturation);
+        }
+    }
+
+    function trySpawnMore() {
+        if (Date.now() >= finalEndTime) {
+            if (currentParallelCount === 0) {
+                done.resolve();
+            }
+            return;
+        }
+        let parallelCount = getTargetParallelCount();
+        // If we want to run 0 values, then just wait a bit, and check again
+        if (parallelCount === 0) {
+            delay(10).finally(trySpawnMore);
+        }
+        while (currentParallelCount < parallelCount) {
+            logErrors(spawnCall());
+        }
+    }
+
+    trySpawnMore();
+
+    await done.promise;
+
+    let effectiveRate = runCount / totalActiveTime * 1000;
+
+    return {
+        effectiveRate,
+        achievedSaturation: totalOverlappedWaitTime / (Date.now() - baseStartTime),
+        latencyStats,
+        runCount,
+        errorCount,
+    };
+}
+
+export async function runSaturationTests(config: {
+    durationPer: number;
+    writeSpec: DriftValues;
+    functions: BenchmarkFunctions;
+}) {
+    const { functions, durationPer, writeSpec } = config;
+
+    function shouldStopIterating(config: {
+        rateIncrease: number;
+        latencyIncrease: number;
+        saturation: number;
+        latencyAverage: number;
+    }): boolean {
+        const { rateIncrease, latencyIncrease, saturation, latencyAverage } = config;
+        if (rateIncrease < 1.1 && saturation >= 1) {
+            console.log(`Stopping due to low rate increase ${rateIncrease}`);
+            return true;
+        }
+        if (saturation >= 5 && rateIncrease / latencyIncrease < 0.1) {
+            console.log(`Stopping due to low rate/latency increase ${rateIncrease / latencyIncrease}`);
+            return true;
+        }
+        if (latencyAverage >= 500) {
+            console.log(`Stopping due to high latency ${latencyAverage}`);
+            return true;
+        }
+        return false;
+    }
+
+    let results: string[] = [];
+    function log(text = "") {
+        results.push(text);
+        console.log(text);
+    }
+
+    // Run a test to warm things up
+    let initialResult = await runSingleSaturationTest({ duration: config.durationPer, saturation: 0.5, writeSpec, functions });
+
+    // Let our warmup settle
+    await delay(initialResult.latencyStats.sum * 0.5);
+
+    // Reset all of the values, both to fill up the server reservations, and so we can easily predict what
+    //  the final values should be
+    console.log(`Start value fill`);
+    let fillTime = Date.now();
+    const batchCount = 1000;
+    for (let i = 0; i < writeSpec.config.modulus; i += batchCount) {
+        let currentCount = Math.min(batchCount, writeSpec.config.modulus - i);
+        await functions.setValues(Array(currentCount).fill(0).map((_, a) => i + a), 0);
+    }
+    fillTime = Date.now() - fillTime;
+    log(`Finished value fill in ${formatTime(fillTime)}`);
+    await delay(fillTime * 0.5 + 250);
+
+    type Result = {
+        actualSaturation: number;
+        effectiveRate: number;
+        latencyStats: StatsValue;
+    };
+    let allResults: Result[] = [];
+
+    let predictedValues = Array(writeSpec.config.modulus).fill(0);
+
+    let saturation = 1 / 16;
+    let lastRate = 0;
+    let lastLatency = Number.POSITIVE_INFINITY;
+    let stopCount = 0;
+    let totalErrorCount = 0;
+    while (true) {
+        console.log(`Start saturation ${percent(saturation)}`);
+        let actualTime = Date.now();
+        let result = await runSingleSaturationTest({
+            duration: durationPer,
+            saturation,
+            writeSpec,
+            predictedValues,
+            functions,
+        });
+
+        actualTime = Date.now() - actualTime;
+
+        allResults.push({
+            actualSaturation: result.achievedSaturation,
+            effectiveRate: result.effectiveRate,
+            latencyStats: result.latencyStats,
+        });
+
+        log(`In ${formatTime(actualTime).padEnd(8, " ")} target saturation ${percent(saturation).padEnd(8, " ")} achieved saturation ${percent(result.achievedSaturation).padEnd(8, " ")} effective rate ${(formatNumber(result.effectiveRate) + "/s").padEnd(12, " ")} latency ${formatStats(result.latencyStats, { noColor: true })}`);
+        if (result.errorCount > 0) {
+            log(`    ERROR: ${result.errorCount} errors!`);
+            totalErrorCount += result.errorCount;
+        }
+        // Wait to let the server cleanup / recover
+        console.log(`Waiting for ${formatTime(actualTime * 0.5)}`);
+        await delay(actualTime * 0.5);
+        console.log(`End saturation ${percent(saturation)}`);
+
+        // Always stop if we are over 3X the target time
+        if (actualTime / durationPer > 2) {
+            console.log(`Stopping due to very high test time ${formatTime(actualTime)}`);
+            break;
+        }
+
+        let rateIncrease = result.effectiveRate / lastRate;
+        let latencyIncrease = result.latencyStats.sum / lastLatency;
+        lastRate = result.effectiveRate;
+        lastLatency = result.latencyStats.sum;
+        if (shouldStopIterating({
+            rateIncrease,
+            latencyIncrease,
+            saturation: saturation,
+            latencyAverage: result.latencyStats.sum / result.latencyStats.count
+        })) {
+            stopCount++;
+            if (stopCount >= 3) {
+                break;
+            }
+        } else {
+            stopCount = 0;
+        }
+
+        saturation *= 2;
+    }
+
+    let afterResult = await runSingleSaturationTest({ duration: config.durationPer, saturation: 0.5, writeSpec, predictedValues, functions });
+
+    let initialAvgLatency = initialResult.latencyStats.sum / initialResult.latencyStats.count;
+    let afterAvgLatency = afterResult.latencyStats.sum / afterResult.latencyStats.count;
+
+    let latencyIncrease = afterAvgLatency / initialAvgLatency;
+    if (latencyIncrease > 1.2) {
+        console.log(red(`WARNING: Latency increased by ${percent(latencyIncrease - 1)}!`));
+    }
+
+
+    log("");
+    log(`Rate chart (op/s)`);
+    logChart(allResults.map(result => ({
+        x: result.actualSaturation,
+        y: result.effectiveRate,
+        xStr: percent(result.actualSaturation),
+        yStr: formatNumber(result.effectiveRate) + "/s",
+    })));
+
+    log("");
+    log(`Latency chart (average)`);
+    logChart(allResults.map(result => ({
+        x: result.actualSaturation,
+        y: result.latencyStats.sum / result.latencyStats.count,
+        xStr: percent(result.actualSaturation),
+        yStr: formatTime(result.latencyStats.sum / result.latencyStats.count),
+    })));
+
+
+    log("");
+    log(`Suggestions`);
+
+    let maxRate = allResults.reduce((a, b) => Math.max(a, b.effectiveRate), 0);
+    let minLatency = allResults[2].latencyStats ?? allResults[0].latencyStats;
+
+    function findClosest(dist: (other: Result) => number): Result {
+        let closest: Result | undefined;
+        let closestDist = Number.POSITIVE_INFINITY;
+        for (let result of allResults) {
+            let currentDist = Math.abs(dist(result));
+            if (currentDist < closestDist) {
+                closest = result;
+                closestDist = currentDist;
+            }
+        }
+        return closest!;
+    }
+
+    const maxRateObj = findClosest(r => r.effectiveRate - maxRate * 0.8);
+    const heavySat = findClosest(r => r.actualSaturation - maxRateObj.actualSaturation * 0.5);
+    const minLatencyObj = findClosest(r => r.latencyStats.sum / r.latencyStats.count - minLatency.sum / minLatency.count);
+    const minLatency2XObj = findClosest(r => r.latencyStats.sum / r.latencyStats.count - minLatency.sum / minLatency.count * 2);
+
+    const normalLatency = findClosest(r => r.latencyStats.sum / r.latencyStats.count - 100);
+    const highLatency = findClosest(r => r.latencyStats.sum / r.latencyStats.count - 500);
+
+    function getSatObj(title: string, obj: Result) {
+        let output = "";
+        output += (title.padEnd(20, " ")) + " ";
+        output += (`${formatNumber(obj.effectiveRate)}/s`.padEnd(10, " ")) + " ";
+
+        let top = getStatsTop(obj.latencyStats);
+        if (top.topHeavy) {
+            output += `latency = ${formatStats(obj.latencyStats, { noColor: true, noSum: true })}`.padEnd(16, " ") + " ";
+        } else {
+            output += `latency = ${formatTime(obj.latencyStats.sum / obj.latencyStats.count)}`.padEnd(16, " ") + " ";
+        }
+        output += (`(${percent(obj.actualSaturation)} saturation)`.padEnd(25, " ")) + " ";
+        output += (`(${formatNumber(obj.latencyStats.count)} count)`.padEnd(12, " ")) + " ";
+
+        return output;
+    }
+
+    if (maxRateObj.effectiveRate > highLatency.effectiveRate * 1.2) {
+        log(getSatObj("Max rate", maxRateObj));
+    }
+    if (heavySat.effectiveRate > highLatency.effectiveRate * 1.2) {
+        log(getSatObj("Heavy saturation", heavySat));
+    }
+    log(getSatObj("Min latency", minLatencyObj));
+    if (highLatency.effectiveRate > normalLatency.effectiveRate * 1.2) {
+        log(blue(getSatObj("500ms latency", highLatency)));
+    }
+    if (minLatency2XObj.effectiveRate > minLatencyObj.effectiveRate * 1.2) {
+        log(yellow(getSatObj("Min latency 2X", minLatency2XObj)));
+    }
+    if (
+        normalLatency.effectiveRate > minLatencyObj.effectiveRate * 1.2
+        && (normalLatency.effectiveRate > minLatency2XObj.effectiveRate * 1.2)
+    ) {
+        log(green(getSatObj("100ms latency", normalLatency)));
+    }
+
+
+    let realValues: number[] = await functions.getValues(predictedValues.map((_, i) => i));
+
+    let wrongIndexes = 0;
+    let wrongSum = 0;
+    for (let i = 0; i < predictedValues.length; i++) {
+        if (predictedValues[i] !== realValues[i]) {
+            wrongIndexes++;
+            wrongSum += Math.abs(predictedValues[i] - realValues[i]);
+        }
+    }
+    let wrongFraction = wrongSum / predictedValues.reduce((a, b) => a + b, 0);
+    if (wrongIndexes > 0) {
+        log(`!!!! ERROR !!!!: ${formatNumber(wrongIndexes)} indexes were wrong, ${percent(wrongFraction)} of operations were missed`);
+    }
+    if (totalErrorCount > 0) {
+        log(`!!!! ERROR !!!!: ${formatNumber(totalErrorCount)} errors were encountered`);
+    }
+
+
+    console.log();
+    console.log("Results");
+    for (let result of results) {
+        console.log(result);
+    }
+
+    return results;
+
+    function logChart(values: { x: number; y: number; xStr: string; yStr: string; }[]) {
+        const barWidth = 4;
+        let columnWidth = values.length * barWidth;
+        let columnHeight = 12;
+        function rightJustifyLog(text: string) {
+            let padding = " ".repeat(Math.max(columnWidth - text.length, 0));
+            log(padding + text);
+        }
+        function sidesLog(left: string, right: string) {
+            let padding = " ".repeat(Math.max(columnWidth - left.length - right.length, 0));
+            log(left + padding + right);
+        }
+        rightJustifyLog(values.at(-1)!.yStr);
+        log();
+        let maxY = values.reduce((a, b) => Math.max(a, b.y), 0);
+        let minY = values.reduce((a, b) => Math.min(a, b.y), Number.POSITIVE_INFINITY);
+        minY = 0;
+        let grid = Array(columnHeight).fill(0).map(() => Array(columnWidth).fill(" "));
+        let xIndex = 0;
+        for (let value of values) {
+            let barHeight = Math.floor((value.y - minY) / (maxY - minY) * columnHeight);
+            for (let y = 0; y < barHeight; y++) {
+                for (let xOff = 0; xOff < barWidth; xOff++) {
+                    grid[y][xIndex + xOff] = "█";
+                }
+            }
+            xIndex += barWidth;
+        }
+        grid.reverse();
+        for (let row of grid) {
+            log(row.join(""));
+        }
+        rightJustifyLog(values.at(0)!.yStr);
+        sidesLog(values.at(0)!.xStr, values.at(-1)!.xStr);
+        log();
+    }
+}

package/src/diagnostics/satSchema.ts

@@ -0,0 +1,64 @@
+import { red } from "socket-function/src/formatting/logColors";
+import { Querysub } from "../4-querysub/Querysub";
+import { delay } from "socket-function/src/batching";
+import { atomic, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { Benchmark } from "./benchmark";
+import { BenchmarkFunctions } from "./runSaturationTest";
+import debugbreak from "debugbreak";
+
+let { data, functions } = Querysub.createSchema<{
+    values: {
+        [index: number]: number;
+    };
+}>()({
+    domainName: "querysub.com",
+    functions: {
+        setValues(indexes: number[], value?: number) {
+            for (let index of indexes) {
+                let newValue = value ?? (data().values[index] + 1);
+                data().values[index] = newValue;
+            }
+        }
+    },
+    module,
+    moduleId: "saturationTest",
+    functionMetadata: {},
+    permissions: {
+        PERMISSIONS: isRegisteredUserCheck
+    }
+});
+
+function isRegisteredUserCheck(): boolean {
+    let allowed = Querysub.getCallerIP() === "127.0.0.1";
+    if (!allowed && Querysub.isAllSynced()) {
+        console.log(red(`Permission denied to IP ${JSON.stringify(Querysub.getCallerIP())}`));
+    }
+    return allowed;
+}
+
+export const sat_data = data;
+export const sat_functions = functions;
+
+
+export function createBenchmarkFunctions(): BenchmarkFunctions {
+    return {
+        async setValues(indexes: number[], value?: number) {
+            let call = sat_functions.setValues(indexes, value);
+            await Benchmark.waitForCallToFinish(call);
+        },
+        async getValues(indexes: number[]): Promise<number[]> {
+            let realValues: number[] = [];
+            const watcher = proxyWatcher.createWatcher({
+                watchFunction() {
+                    for (let i = 0; i < indexes.length; i++) {
+                        let index = indexes[i];
+                        realValues[i] = atomic(sat_data().values[index]) ?? 0;
+                    }
+                },
+            });
+            await delay(10 * 1000);
+            watcher.dispose();
+            return realValues;
+        },
+    };
+}

package/src/diagnostics/trackResources.ts

@@ -0,0 +1,82 @@
+import { runInfinitePoll } from "socket-function/src/batching";
+import { lazy } from "socket-function/src/caching";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { blue } from "socket-function/src/formatting/logColors";
+import { isNode } from "socket-function/src/misc";
+import { registerPeriodic } from "./periodic";
+import { registerMeasureInfo } from "socket-function/src/profiling/measure";
+import { logNodeStateStats } from "../5-diagnostics/nodeMetadata";
+
+let resources: {
+    name: string;
+    getCount: () => number;
+}[] = [];
+
+export function registerResource<T extends Set<unknown> | Map<unknown, unknown> | unknown[]>(name: string, data: T): T {
+    let getCount: () => number;
+    if (Array.isArray(data)) {
+        getCount = () => data.length;
+    } else {
+        getCount = () => data.size;
+    }
+    resources.push({ name, getCount });
+    return data;
+}
+export function registerMapArrayResource<T extends Map<unknown, unknown[]>>(name: string, data: T): T {
+    let getCount = () => {
+        let sum = 0;
+        for (let value of data.values()) {
+            sum += value.length;
+        }
+        return sum;
+    };
+    resources.push({ name, getCount });
+    return data;
+}
+export function registerDynamicResource(name: string, getCount: () => number) {
+    resources.push({ name, getCount });
+}
+
+function getHeapSize() {
+    if (isNode()) {
+        let usage = process.memoryUsage();
+        // Not really the heap size, but it is more what we are looking for
+        return usage.rss;
+    } else {
+        return (window.performance as any).memory.totalJSHeapSize;
+    }
+}
+function getUsedHeapSize() {
+    if (isNode()) {
+        let usage = process.memoryUsage();
+        return usage.heapUsed + usage.arrayBuffers;
+    } else {
+        if (!window.performance) return 0;
+        return (window.performance as any).memory.usedJSHeapSize;
+    }
+}
+
+function logResourcesNow() {
+    let resourcesWithCounts = resources.map(resource => ({ ...resource, count: resource.getCount() }));
+    resourcesWithCounts.sort((a, b) => b.count - a.count);
+    if (!isNode()) {
+        console.log(`Memory usage ${blue(formatNumber(getUsedHeapSize()) + "B")}/${blue(formatNumber(getHeapSize()) + "B")} at ${new Date().toISOString()}`);
+        for (let resource of resourcesWithCounts) {
+            if (resource.count === 0) continue;
+            console.log(`  ${resource.name}: ${formatNumber(resource.count)}`);
+        }
+    } else {
+        logNodeStateStats("Heap", formatNumber)(getUsedHeapSize());
+        logNodeStateStats("All Memory", formatNumber)(getHeapSize());
+        for (let resource of resourcesWithCounts) {
+            if (resource.count === 0) continue;
+            logNodeStateStats(resource.name, formatNumber)(resource.count);
+        }
+    }
+}
+
+registerMeasureInfo(() => {
+    return formatNumber(getUsedHeapSize()) + "B";
+});
+
+registerPeriodic(logResourcesNow);

package/src/diagnostics/watchdog.ts

@@ -0,0 +1,55 @@
+import { runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { FormattedMeasureTable, logMeasureTable, startMeasure } from "socket-function/src/profiling/measure";
+import { logErrors } from "../errors";
+import { isNode } from "socket-function/src/misc";
+import debugbreak from "debugbreak";
+import { registerPeriodic } from "./periodic";
+import { getOwnMachineId } from "../-a-auth/certs";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { logDisk } from "./logs/diskLogger";
+
+
+let measureObj = startMeasure();
+function logProfileMeasuresTimingsNow() {
+    let profile = measureObj.finish();
+    measureObj = startMeasure();
+    function diskLogMeasureObj(table: FormattedMeasureTable | undefined) {
+        if (!table) return;
+        logDisk(table.title, table.entries);
+    }
+    diskLogMeasureObj(logMeasureTable(profile, {
+        name: `watchdog at ${new Date().toLocaleString()}`,
+        // NOTE: Much higher min log times, now that we are combining logs.
+        minTimeToLog: 250,
+        thresholdInTable: 0,
+        setTitle: true
+    }));
+    diskLogMeasureObj(logMeasureTable(profile, {
+        name: `watchdog at ${new Date().toLocaleString()}`,
+        mergeDepth: 1,
+        minTimeToLog: 250,
+    }));
+    // if (isNode()) {
+    //     if (SocketFunction.mountedNodeId) {
+    //         console.log("Mounted as " + SocketFunction.mountedNodeId);
+    //     } else {
+    //         console.log("Not mounted");
+    //     }
+    // }
+}
+(globalThis as any).logProfileMeasuresNow = logProfileMeasuresTimingsNow;
+
+(globalThis as any).logUnfiltered = function logUnfiltered(depth = 2) {
+    let profile = measureObj.finish();
+    measureObj = startMeasure();
+    logMeasureTable(profile, {
+        name: `all logs at ${new Date().toLocaleString()}`,
+        mergeDepth: depth,
+        minTimeToLog: 0,
+        maxTableEntries: 10000000,
+        thresholdInTable: 0
+    });
+};
+
+
+registerPeriodic(logProfileMeasuresTimingsNow);