querysub 0.2.0

package/src/heapDumps.ts

@@ -0,0 +1,666 @@
+// See https://github.com/nodejs/node/blob/main/deps/v8/src/profiler/heap-snapshot-generator.cc
+
+import v8 from "v8";
+import { red, yellow, blue } from "socket-function/src/formatting/logColors";
+import { logErrors } from "./errors";
+import { formatNumber } from "socket-function/src/formatting/format";
+import fs from "fs";
+import { delay } from "socket-function/src/batching";
+import { getSubFolder } from "./fs";
+
+// Made to be called by stepping in via the debugger, ex:
+//  require("D:/repos/shard/src/heapDumps.ts").writeHeapSnapshot(undefined, "0_afterboot")
+//  (If no one imports this it won't be available until someone requires it, so you always have to require it...)
+
+(async function pasteToDump() {
+    const v8 = require("v8");
+    const fs = require("fs");
+    // @ts-ignore
+    let parts = [];
+    let stream = v8.getHeapSnapshot();
+    // @ts-ignore
+    stream.on("data", chunk => parts.push(chunk));
+    await new Promise(resolve => stream.on("end", resolve));
+    // @ts-ignore
+    let buffer = Buffer.concat(parts);
+    fs.writeFileSync("./heapdump.heapsnapshot", buffer);
+})().catch(e => console.error(e)).finally(() => console.log("done"));
+
+export async function writeHeapSnapshot(writeDir = getSubFolder("heapdumps"), name: string): Promise<Buffer> {
+    global.gc?.();
+    let parts: Buffer[] = [];
+    let stream = v8.getHeapSnapshot();
+    stream.on("data", chunk => parts.push(chunk));
+    await new Promise(resolve => stream.on("end", resolve));
+    let buffer = Buffer.concat(parts);
+    if (writeDir) {
+        let size = getHeapsnapshotMemoryUsage(buffer);
+        let filename = `heapdump-${name}-pid-${process.pid}-time-${new Date().toISOString().replace(/:/g, "-")}-size-${formatNumber(size)}B.heapsnapshot`;
+        if (!writeDir.endsWith("/")) {
+            writeDir += "/";
+        }
+        await fs.promises.writeFile(writeDir + filename, buffer);
+    }
+    return buffer;
+}
+
+const progressCalls = 100;
+const exportsGlobal = true;
+
+export type HeapSummary = {
+    name: string;
+    total: number;
+    summary: {
+        [key: string]: {
+            key: string;
+            sum: number;
+            count: number;
+            firstId: number;
+        };
+    };
+};
+
+export interface HeapsnapshotFormat {
+    snapshot: {
+        meta: {
+            edge_fields: ("type" | "name_or_index" | "to_node")[];
+            location_fields: ("object_index" | "script_id" | "line" | "column")[];
+            node_fields: ("type" | "name" | "id" | "self_size" | "edge_count" | "trace_node_id" | "detachedness")[];
+
+            edge_types: (string | string[])[];
+            node_types: (string | string[])[];
+        };
+
+        node_count: number;
+        edge_count: number;
+        trace_function_count: number;
+    };
+
+    edges: number[];
+    locations: number[];
+    nodes: number[];
+
+    strings: string[];
+
+    name: string;
+}
+
+/** TIMING: 38ms for a 90MB file, so effectively free (this code doesn't even have to read the entire buffer to get the memory usage). */
+export function getHeapsnapshotMemoryUsage(buffer: Buffer): number {
+    let sum = 0;
+
+    let index = 0;
+    const newLine = "\n".charCodeAt(0);
+    const end = "]".charCodeAt(0);
+    const comma = ",".charCodeAt(0);
+
+    // Skip until the second line
+    while (buffer[index] !== 10) {
+        index++;
+    }
+    index++;
+
+    let metadata = JSON.parse(buffer.toString("utf8", 0, index - 2) + "}") as HeapsnapshotFormat;
+
+    index += `"nodes":`.length;
+
+    let nodeSelfSizeIndex = metadata.snapshot.meta.node_fields.indexOf("self_size");
+    let selfSizes: number[] = [];
+
+    while (true) {
+        let firstCh = buffer[index];
+        if (firstCh === end) break;
+        index++;
+        let skippedCommas = 0;
+        while (skippedCommas < nodeSelfSizeIndex) {
+            if (buffer[index] === comma) {
+                skippedCommas++;
+            }
+            index++;
+        }
+
+        let nextComma = index;
+        while (buffer[nextComma] !== comma && buffer[nextComma] !== newLine) {
+            nextComma++;
+        }
+
+        let selfSize = 0;
+        for (let i = index; i < nextComma; i++) {
+            selfSize = selfSize * 10 + (buffer[i] - 48);
+        }
+        selfSizes.push(selfSize);
+        sum += selfSize;
+
+        // Skip until next line
+        while (buffer[index] !== newLine) {
+            index++;
+        }
+        index++;
+    }
+
+    return sum;
+}
+
+export async function parseHeapSnapshot(name: string, buffer: Buffer, setProgress: (fraction: number, stage: string) => void): Promise<HeapsnapshotFormat> {
+    let index = 0;
+    function parseNextLine(): string {
+        let start = index;
+        while (true) {
+            if (index >= buffer.length || buffer[index] === 10) {
+                let line = buffer.toString("utf8", start, index);
+                index++;
+                return line;
+            }
+            index++;
+        }
+    }
+
+    let metadataLine = parseNextLine();
+    let metadata = JSON.parse(metadataLine.slice(0, -1) + "}") as HeapsnapshotFormat;
+    metadata.name = name;
+
+    let nextProgressIndex = 0;
+    while (index < buffer.length) {
+
+        let startIndex = index;
+        let line = parseNextLine();
+        let field = line.split(":")[0];
+        index = startIndex + field.length + ":".length;
+
+        field = JSON.parse(field);
+
+        if (field === "strings") {
+            let strings = metadata.strings = [] as string[];
+            while (index < buffer.length) {
+                let line = parseNextLine();
+                if (line === "]," || line === "[],") break;
+                if (line[0] === "[") {
+                    line = line.slice(1, -1);
+                } else if (line.endsWith("]}")) {
+                    line = line.slice(0, -2);
+                } else {
+                    line = line.slice(0, -1);
+                }
+                strings.push(JSON.parse(line));
+
+                if (index >= nextProgressIndex) {
+                    setProgress(index / buffer.length, "buffer => numbers");
+                    // Wait for progress to render
+                    await delay(0);
+                    nextProgressIndex = index + Math.floor(buffer.length / progressCalls);
+                }
+            }
+        } else {
+            let numbers: number[] = [];
+            if (field === "nodes" || field === "edges") {
+                let count = metadata.snapshot[field.slice(0, -1) + "_count" as "node_count"];
+                let fields = metadata.snapshot.meta[field.slice(0, -1) + "_fields" as "node_fields"];
+                let numberIndex = 0;
+                numbers = new Int32Array(count * fields.length) as any;
+                while (index < buffer.length) {
+                    let line = parseNextLine();
+                    if (line === "]," || line === "[],") break;
+                    let parsed = JSON.parse("[" + line.slice(1) + "]");
+                    for (let parsedPart of parsed) {
+                        numbers[numberIndex++] = parsedPart;
+                    }
+
+                    if (index >= nextProgressIndex) {
+                        setProgress(index / buffer.length, "buffer => numbers");
+                        // Wait for progress to render
+                        await delay(0);
+                        nextProgressIndex = index + Math.floor(buffer.length / progressCalls);
+                    }
+                }
+            } else {
+                while (index < buffer.length) {
+                    let line = parseNextLine();
+                    if (line === "]," || line === "[],") break;
+                    let parsed = JSON.parse("[" + line.slice(1) + "]");
+                    for (let parsedPart of parsed) {
+                        numbers.push(parsedPart);
+                    }
+
+                    if (index >= nextProgressIndex) {
+                        setProgress(index / buffer.length, "buffer => numbers");
+                        // Wait for progress to render
+                        await delay(0);
+                        nextProgressIndex = index + Math.floor(buffer.length / progressCalls);
+                    }
+                }
+            }
+
+            metadata[field as "edges"] = numbers;
+        }
+    }
+    setProgress(1, "buffer => numbers");
+
+    return metadata;
+}
+
+export async function getHeapsnapshotIds(snapshot: HeapsnapshotFormat) {
+    let ids = new Set<number>();
+    const nodeFields = snapshot.snapshot.meta.node_fields.length;
+    const nodeIdIndex = snapshot.snapshot.meta.node_fields.indexOf("id");
+    const nodes = snapshot.nodes;
+
+    for (let i = 0; i < nodes.length; i += nodeFields) {
+        ids.add(nodes[i + nodeIdIndex]);
+    }
+    return ids;
+}
+
+export async function processAndSummarizeHeapsnapshot(
+    snapshot: HeapsnapshotFormat,
+    setProgress: (fraction: number, stage: string) => void,
+    config?: {
+        diff?: {
+            baseIds: Set<number>;
+            // NOTE: "removed" just requires reversing the snapshots and then doing "new"
+            type: "new" | "kept";
+        };
+        filter?: { filter: string; depth: number };
+        maxDepth?: number;
+        includesFilter?: string,
+    }
+): Promise<HeapSummary> {
+    const filter = config?.filter ?? { filter: "", depth: 0 };
+    const diff = config?.diff;
+    const maxDepth = config?.maxDepth ?? 20;
+    const includesFilter = config?.includesFilter;
+
+    const nodeFields = snapshot.snapshot.meta.node_fields.length;
+    const edgeFields = snapshot.snapshot.meta.edge_fields.length;
+
+    const nodeTypeIndex = snapshot.snapshot.meta.node_fields.indexOf("type");
+    let nodeTypes = snapshot.snapshot.meta.node_types[nodeTypeIndex] as string[];
+    const nodeTypeString = nodeTypes.indexOf("string");
+    const nodeNameIndex = snapshot.snapshot.meta.node_fields.indexOf("name");
+    const nodeEdgeCountIndex = snapshot.snapshot.meta.node_fields.indexOf("edge_count");
+    const nodeSelfSizeIndex = snapshot.snapshot.meta.node_fields.indexOf("self_size");
+    const nodeIdIndex = snapshot.snapshot.meta.node_fields.indexOf("id");
+    const nodes = snapshot.nodes;
+    const strings = snapshot.strings;
+
+    const edges = snapshot.edges;
+    const edgeToNodeIndex = snapshot.snapshot.meta.edge_fields.indexOf("to_node");
+    const edgeTypeIndex = snapshot.snapshot.meta.edge_fields.indexOf("type");
+    const edgeNameOrIndexIndex = snapshot.snapshot.meta.edge_fields.indexOf("name_or_index");
+
+    const edgeTypeWeak = (snapshot.snapshot.meta.edge_types[edgeTypeIndex] as string[]).indexOf("weak");
+    const edgeTypeHidden = (snapshot.snapshot.meta.edge_types[edgeTypeIndex] as string[]).indexOf("hidden");
+    const edgeTypeClosure = (snapshot.snapshot.meta.edge_types[edgeTypeIndex] as string[]).indexOf("closure");
+    const edgeTypeContext = (snapshot.snapshot.meta.edge_types[edgeTypeIndex] as string[]).indexOf("context");
+    const edgeTypeProperty = (snapshot.snapshot.meta.edge_types[edgeTypeIndex] as string[]).indexOf("property");
+
+    const locations = snapshot.locations;
+    const locationFields = snapshot.snapshot.meta.location_fields.length;
+    const locationObjectIndex = snapshot.snapshot.meta.location_fields.indexOf("object_index");
+    const locationScriptIdIndex = snapshot.snapshot.meta.location_fields.indexOf("script_id");
+
+    let nodeIndexToScriptId = new Map<number, number>();
+    for (let i = 0; i < locations.length; i += locationFields) {
+        let objectIndex = locations[i + locationObjectIndex];
+        let scriptId = locations[i + locationScriptIdIndex];
+        nodeIndexToScriptId.set(objectIndex, scriptId);
+    }
+
+    // nodeIndex => [edgeIndex, nodeIndex, ...]
+    let keptAliveBy = new Map<number, number[]>();
+    {
+        let nextProgress = 0;
+        let edgeIndex = 0;
+        for (let nodeIndex = 0; nodeIndex < nodes.length; nodeIndex += nodeFields) {
+            let edgeCount = nodes[nodeIndex + nodeEdgeCountIndex];
+
+            for (let j = 0; j < edgeCount; j++) {
+                let toNodeIndex = edges[edgeIndex + edgeToNodeIndex];
+
+                let toNodeKeptAlive = keptAliveBy.get(toNodeIndex);
+                if (!toNodeKeptAlive) {
+                    toNodeKeptAlive = [];
+                    keptAliveBy.set(toNodeIndex, toNodeKeptAlive);
+                }
+                toNodeKeptAlive.push(edgeIndex);
+                toNodeKeptAlive.push(nodeIndex);
+
+                edgeIndex += edgeFields;
+            }
+            if (nodeIndex >= nextProgress) {
+                setProgress(nodeIndex / nodes.length, "Inverting references");
+                nextProgress = nodeIndex + Math.floor(nodes.length / progressCalls);
+                await delay(0);
+            }
+        }
+    }
+
+
+    function getNodeDebugPath(nodeIndex: number, takeNonRootPath = false): string[] {
+        let visitedNodes = new Set<number>();
+        let path: string[] = [];
+        const intermediateDepthLimit = Math.max(100, maxDepth);
+        addNode(nodeIndex);
+        if (path.length === 0 && !takeNonRootPath) {
+            return getNodeDebugPath(nodeIndex, true);
+        }
+        if (path.length > maxDepth) {
+            path = path.slice(0, maxDepth);
+            path.push(red("(further path cut off)"));
+        }
+        if (path.length === 0) {
+            return [red("No path to exports/global")];
+        }
+        let type = nodeTypes[nodes[nodeIndex + nodeTypeIndex]];
+        path.unshift(`(type=${type})`);
+        return path;
+        function addNode(node: number): boolean {
+            if (path.length > intermediateDepthLimit) {
+                return true;
+            }
+            if (visitedNodes.has(node)) return false;
+            visitedNodes.add(node);
+
+            function ellipsize(text: string, limit: number) {
+                if (text.length <= limit) return text;
+                return text.slice(0, limit - 3) + "...";
+            }
+
+            const name = strings[nodes[node + nodeNameIndex]];
+
+            let previousPathLength = path.length;
+            if (nodes[node + nodeTypeIndex] === nodeTypeString) {
+                path.push(`string`);
+            } else {
+                path.push(`${JSON.stringify(ellipsize(name, 50))}`);
+            }
+
+            let location = nodeIndexToScriptId.get(node);
+            if (location !== undefined) {
+                path.push(`(scriptId=${location})`);
+            }
+
+            if (name === "global") {
+                // If it is a global we only really need to know what the global property is called
+                path = path.slice(-3 - filter?.depth);
+                return true;
+            }
+            if (name === "(GC roots)") {
+                return true;
+            }
+            // Find the first node keeping it alive (not weak references, as those certainly aren't causing any leaks!)
+            let keptAliveArray = keptAliveBy.get(node);
+            let anyNotWeak = false;
+            if (keptAliveArray) {
+                let forcedEdge: number | undefined;
+                {
+                    for (let i = 0; i < keptAliveArray.length; i += 2) {
+                        let edgeIndex = keptAliveArray[i];
+                        let fromNodeIndex = keptAliveArray[i + 1];
+
+                        let edgeType = edges[edgeIndex + edgeTypeIndex];
+                        if (edgeType === edgeTypeWeak) continue;
+                        const name = strings[edges[edgeIndex + edgeNameOrIndexIndex]];
+                        const toName = strings[nodes[fromNodeIndex + nodeNameIndex]];
+
+                        if (edgeType === edgeTypeProperty && name === "exports" && exportsGlobal) {
+                            forcedEdge = i;
+                            break;
+                        } else if (toName === "global" || toName === "(GC roots)") {
+                            forcedEdge = i;
+                            break;
+                        }
+                    }
+                }
+
+                for (let i = forcedEdge ?? 0; i < keptAliveArray.length; i += 2) {
+                    let edgeIndex = keptAliveArray[i];
+                    let fromNodeIndex = keptAliveArray[i + 1];
+
+                    let edgeType = edges[edgeIndex + edgeTypeIndex];
+
+                    if (edgeType === edgeTypeWeak) continue;
+                    // Not sure what hidden edges are... but maybe best to skip them
+                    //  (as that seems to be what devtools does).
+                    if (edgeType === edgeTypeHidden) continue;
+                    anyNotWeak = true;
+
+                    // https://github.com/nodejs/node/blob/main/deps/v8/src/profiler/heap-snapshot-generator.cc#L212
+                    //  - For "element" and "hidden" name is actually an index (not a string)
+                    const name = strings[edges[edgeIndex + edgeNameOrIndexIndex]];
+                    if (
+                        edgeType === edgeTypeClosure
+                        || edgeType === edgeTypeContext
+                    ) {
+                        path.push(yellow(`${name}`));
+                    } else if (
+                        edgeType === edgeTypeProperty
+                        //|| edge.type === "element"
+                    ) {
+                        path.push(blue(`[${name}]`));
+                        // Exports are kept alive by default, so we don't need to show a random referenced
+                        //  location and pretend that is the reason we are keeping the export alive.
+                        if (name === "exports" && exportsGlobal) {
+                            // If it is an export we only really need to know what the exported property is called
+                            path = path.slice(-3 - filter?.depth);
+                            return true;
+                        }
+                    }
+                    if (addNode(fromNodeIndex)) {
+                        return true;
+                    }
+                }
+            }
+            if (takeNonRootPath) {
+                if (!keptAliveArray?.length) {
+                    path.push(yellow("(no references)"));
+                    return true;
+                }
+                if (!anyNotWeak) {
+                    path.push(yellow("(weak)"));
+                    return true;
+                }
+            }
+            // This is the case where all references were cyclic references
+            //      (We have to undo the path we added as well)
+            path = path.slice(0, previousPathLength);
+            return false;
+        }
+    }
+
+    let nodeGroupings: Map<string, number[]> = new Map();
+
+    let diffFilter = (id: number) => true;
+    if (diff?.type === "new") {
+        diffFilter = (id: number) => !diff.baseIds.has(id);
+    } else if (diff?.type === "kept") {
+        diffFilter = (id: number) => diff.baseIds.has(id);
+    }
+
+    let total = 0;
+    let suffixFilter = config?.filter?.filter ?? "";
+    let nextProgress = 0;
+    for (let i = 0; i < nodes.length; i += nodeFields) {
+        if (i >= nextProgress) {
+            setProgress(i / nodes.length, "Grouping nodes");
+            nextProgress = i + Math.floor(nodes.length / progressCalls);
+            await delay(0);
+        }
+        let node = i;
+
+        let id = nodes[node + nodeIdIndex];
+        if (!diffFilter(id)) {
+            continue;
+        }
+
+        let key = getNodeDebugPath(node).join(".");
+        if (includesFilter && !key.includes(includesFilter)) continue;
+
+        total += nodes[node + nodeSelfSizeIndex];
+
+        if (suffixFilter && !key.endsWith(suffixFilter)) continue;
+        let group = nodeGroupings.get(key);
+        if (group === undefined) {
+            group = [];
+            nodeGroupings.set(key, group);
+        }
+        group.push(node);
+    }
+
+    let summary: HeapSummary["summary"] = {};
+    for (let [key, nodesInGroup] of nodeGroupings) {
+        let sum = 0;
+        for (let node of nodesInGroup) {
+            sum += nodes[node + nodeSelfSizeIndex];
+        }
+        summary[key] = {
+            key,
+            sum,
+            count: nodesInGroup.length,
+            firstId: nodes[nodesInGroup[0] + nodeIdIndex],
+        };
+    }
+
+    return {
+        name: snapshot.name,
+        summary,
+        total,
+    };
+}
+
+export async function debugAllocation(snapshot: HeapsnapshotFormat, id: number, setProgress: (fraction: number, stage: string) => void) {
+    type LocationType = {
+        object_index: number;
+        script_id: number;
+        line: number;
+        column: number;
+    };
+    type NodeType = {
+        type: string;
+        name: string;
+        id: number;
+        self_size: number;
+        edge_count: number;
+        trace_node_id: string;
+        detachedness: number;
+        // Everything that this node is keeping alive
+        keepingAlive: EdgeType[];
+        keptAliveBy: { edge: EdgeType; node: NodeType; }[];
+        location: LocationType | undefined;
+    };
+    type EdgeType = {
+        type: string;
+        name_or_index: number | string;
+        to_node: NodeType;
+    };
+
+    async function dumpObjects(
+        data: number[],
+        fields: string[],
+        types: (string | string[])[],
+        strings: string[],
+        setProgress: (fraction: number) => void
+    ) {
+        let output: unknown[] = [];
+        let fieldCount = fields.length;
+        let nextProgress = 0;
+        for (let i = 0; i < data.length; i += fieldCount) {
+            let obj: any = {};
+            for (let j = 0; j < fieldCount; j++) {
+                let value = data[i + j] as any;
+                let type = types[j];
+                if (type === "string") {
+                    value = strings[value];
+                } else if (Array.isArray(type)) {
+                    value = type[value];
+                }
+                obj[fields[j]] = value;
+            }
+            output.push(obj);
+            if (i >= nextProgress) {
+                setProgress(i / data.length);
+                nextProgress = i + Math.floor(data.length / progressCalls);
+                await delay(0);
+            }
+        }
+        return output;
+    }
+
+    let metadata = snapshot.snapshot.meta;
+    let strings = snapshot.strings;
+    let locations = await dumpObjects(snapshot.locations, metadata.location_fields, ["number", "number", "number", "number"], strings,
+        progress => setProgress(progress, "numbers => locations")
+    ) as LocationType[];
+    let nodes = await dumpObjects(snapshot.nodes, metadata.node_fields, metadata.node_types, strings,
+        progress => setProgress(progress, "numbers => nodes")
+    ) as NodeType[];
+    let edges = await dumpObjects(snapshot.edges, metadata.edge_fields, metadata.edge_types, strings,
+        progress => setProgress(progress, "numbers => edges")
+    ) as EdgeType[];
+
+    setProgress(0, "initing ref arrays");
+    await delay(0);
+
+    for (let i = 0; i < nodes.length; i++) {
+        nodes[i].keepingAlive = [];
+        nodes[i].keptAliveBy = [];
+    }
+    setProgress(1, "initing ref arrays");
+    await delay(0);
+
+    // Update object references
+    for (let i = 0; i < edges.length; i++) {
+        let edge = edges[i];
+        edge.to_node = nodes[(edge.to_node as any as number) / metadata.node_fields.length];
+
+        // https://github.com/nodejs/node/blob/main/deps/v8/src/profiler/heap-snapshot-generator.cc#L212
+        if (edge.type === "element" || edge.type === "hidden") {
+            // Then it is an (node?) index, which we should do something with
+        } else {
+            edge.name_or_index = strings[edge.name_or_index as any as number];
+        }
+    }
+    setProgress(1, "settings edge objects");
+    await delay(0);
+
+    let edgeIndex = 0;
+    for (let i = 0; i < nodes.length; i++) {
+        let node = nodes[i];
+        for (let j = 0; j < node.edge_count; j++) {
+            let edge = edges[edgeIndex];
+            node.keepingAlive.push(edge);
+            edge.to_node.keptAliveBy.push({ edge, node, });
+            edgeIndex++;
+        }
+    }
+    setProgress(1, "populating ref arrays");
+    await delay(0);
+
+    for (let i = 0; i < locations.length; i++) {
+        let location = locations[i];
+        let node = nodes[location.object_index / metadata.node_fields.length];
+        node.location = location;
+    }
+    setProgress(1, "settings locations");
+    await delay(0);
+
+    let node = nodes.find(x => x.id === id);
+    let specialNodes: unknown[] = [];
+    //node = nodes.find(x => x.id === 8727);
+    //specialNodes = nodes.filter(x => x.location?.script_id === 693);
+    logErrors(stepIntoNode(node, specialNodes));
+
+    return nodes;
+}
+
+async function stepIntoNode(node: unknown, nodes: unknown[]) {
+    console.log("Garbage collect NOW!");
+    // Wait... so hopefully garbage collection occurs, so only node and nodes
+    //  it references stay alive.
+    await delay(5000);
+    await delay(5000);
+    await delay(5000);
+    await delay(5000);
+    debugger;
+    console.log({ node });
+    console.log({ nodes });
+}

package/src/https.ts

@@ -0,0 +1,2 @@
+import { httpsRequest } from "socket-function/src/https";
+export { httpsRequest };

package/src/inject.ts

@@ -0,0 +1 @@
+import "./diagnostics/logs/injectFileLocationToConsole";