sliftutils 1.7.65 → 1.7.67

package/test.ts

@@ -1,22 +1,42 @@
 import * as fs from "fs";
 import * as path from "path";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { sort } from "socket-function/src/misc";
 import { TreeSummary } from "./treeSummary";
 
 const SUMMARY_COUNT = 30;
+const TIMING_RUNS = 20;
+const SCAN_DIRS = [
+    "D:/repos/qs-cyoa",
+    "D:/repos/querysub",
+    "D:/repos/socket-function",
+    "D:/repos/sliftutils",
+    "D:/repos/inference-weights",
+];
+
+// Matches the gitignored "test*.json" pattern, so the cache never gets tracked.
+const CACHE_FILE = "D:/repos/sliftutils/test-files-cache.json";
 
 type FileInfo = { path: string; size: number };
 type FileSummary = { count: number; totalSize: number };
 
-async function collectFiles(dir: string, basePath: string, output: FileInfo[]) {
+async function collectFiles(dirs: string[], output: FileInfo[]) {
+    for (let dir of dirs) {
+        let basePath = dir.endsWith("/") && dir || dir + "/";
+        await collectDir(dir, basePath, output);
+    }
+}
+
+async function collectDir(dir: string, basePath: string, output: FileInfo[]) {
     let entries = await fs.promises.readdir(dir, { withFileTypes: true });
     for (let entry of entries) {
         let fullPath = path.join(dir, entry.name);
-        let relativePath = basePath + entry.name;
+        let entryPath = basePath + entry.name;
         if (entry.isDirectory()) {
-            await collectFiles(fullPath, relativePath + "/", output);
+            await collectDir(fullPath, entryPath + "/", output);
         } else if (entry.isFile()) {
             let stats = await fs.promises.stat(fullPath);
-            output.push({ path: relativePath, size: stats.size });
+            output.push({ path: entryPath, size: stats.size });
         }
     }
 }
@@ -29,26 +49,123 @@ function summarize(files: FileInfo[], weightBySize: boolean) {
             summary.count++;
             summary.totalSize += file.size;
         },
+        mergeSummaries: (target, source) => {
+            target.count += source.count;
+            target.totalSize += source.totalSize;
+        },
         getWeight: summary => {
             if (weightBySize) return summary.totalSize;
             return summary.count;
         },
+        expectedOutputCount: SUMMARY_COUNT,
     });
+    let addStart = performance.now();
     for (let file of files) {
         tree.add(file);
     }
+    let addTime = performance.now() - addStart;
+    let queryStart = performance.now();
+    let summaries = tree.getSummaries(SUMMARY_COUNT);
+    for (let i = 1; i < TIMING_RUNS; i++) {
+        summaries = tree.getSummaries(SUMMARY_COUNT);
+    }
+    let queryTime = (performance.now() - queryStart) / TIMING_RUNS;
     let label = weightBySize && "size" || "count";
     console.log(`\n=== weighted by ${label}, ${SUMMARY_COUNT} entries ===`);
-    for (let entry of tree.getSummaries(SUMMARY_COUNT)) {
-        let sizeKB = (entry.summary.totalSize / 1024).toFixed(0);
-        console.log(`${entry.path.padEnd(60)} files=${String(entry.summary.count).padStart(6)} size=${sizeKB.padStart(9)}KB`);
+    console.log(`add: ${formatTime(addTime)} total for ${formatNumber(files.length)} files, tracking ${formatNumber(tree.getTrackedNodeCount())} nodes, getSummaries: ${formatTime(queryTime)} avg over ${TIMING_RUNS} runs`);
+    let totalWeight = 0;
+    for (let entry of summaries) {
+        totalWeight += entry.weight;
+    }
+    sort(summaries, entry => -entry.weight);
+    // Verify each entry against the raw file list: assign every file to its most specific displayed entry (exact paths beat prefixes, longer prefixes beat shorter), which is the partition the entries claim to represent.
+    let actuals = summaries.map(() => ({ count: 0, totalSize: 0 }));
+    let unmatchedCount = 0;
+    let unmatchedSize = 0;
+    for (let file of files) {
+        let bestIndex = -1;
+        let bestLen = -1;
+        for (let i = 0; i < summaries.length; i++) {
+            let entryPath = summaries[i].path;
+            if (entryPath.endsWith("*")) {
+                let prefix = entryPath.slice(0, -1);
+                if (prefix.length > bestLen && file.path.startsWith(prefix)) {
+                    bestIndex = i;
+                    bestLen = prefix.length;
+                }
+            } else if (file.path === entryPath && entryPath.length + 1 > bestLen) {
+                bestIndex = i;
+                bestLen = entryPath.length + 1;
+            }
+        }
+        if (bestIndex === -1) {
+            unmatchedCount++;
+            unmatchedSize += file.size;
+            continue;
+        }
+        actuals[bestIndex].count++;
+        actuals[bestIndex].totalSize += file.size;
+    }
+    let shownPercent = 0;
+    let specificPercent = 0;
+    for (let i = 0; i < summaries.length; i++) {
+        let entry = summaries[i];
+        let percent = entry.weight / totalWeight * 100;
+        shownPercent += percent;
+        if (entry.kind === "self" || entry.kind === "subtree") {
+            specificPercent += percent;
+        }
+        let actual = actuals[i];
+        let diff = "";
+        if (actual.count !== entry.summary.count || actual.totalSize !== entry.summary.totalSize) {
+            diff = `  DIFF actual files=${formatNumber(actual.count)} size=${formatNumber(actual.totalSize)}B`;
+        }
+        console.log(`${percent.toFixed(1).padStart(5)}% ${entry.path.padEnd(60)} files=${formatNumber(entry.summary.count).padStart(6)} size=${(formatNumber(entry.summary.totalSize) + "B").padStart(8)}${diff}`);
+    }
+    if (unmatchedCount > 0) {
+        console.log(`UNMATCHED: ${formatNumber(unmatchedCount)} files (${formatNumber(unmatchedSize)}B) matched no displayed entry`);
+    }
+    console.log(`top ${summaries.length} entries cover ${shownPercent.toFixed(1)}% (specific paths ${specificPercent.toFixed(1)}%, combined leftovers ${(shownPercent - specificPercent).toFixed(1)}%)`);
+    // Entries are a strict partition (each file in exactly one entry), so these must match the real totals exactly.
+    let entryCount = 0;
+    let entrySize = 0;
+    for (let entry of summaries) {
+        entryCount += entry.summary.count;
+        entrySize += entry.summary.totalSize;
     }
+    let actualSize = 0;
+    for (let file of files) {
+        actualSize += file.size;
+    }
+    console.log(`entry totals: files=${formatNumber(entryCount)} (actual ${formatNumber(files.length)}), size=${formatNumber(entrySize)}B (actual ${formatNumber(actualSize)}B)`);
 }
 
-async function main() {
+async function getFiles(): Promise<FileInfo[]> {
+    try {
+        let cached = JSON.parse(await fs.promises.readFile(CACHE_FILE, "utf8")) as { dirs: string[]; files: FileInfo[] };
+        if (JSON.stringify(cached.dirs) === JSON.stringify(SCAN_DIRS)) {
+            console.log(`loaded ${formatNumber(cached.files.length)} files from cache ${CACHE_FILE}`);
+            return cached.files;
+        }
+        console.log(`cache dirs changed (cached ${JSON.stringify(cached.dirs)}, now ${JSON.stringify(SCAN_DIRS)}), rescanning`);
+    } catch (err) {
+        if ((err as { code?: string }).code === "ENOENT") {
+            console.log(`no cache file at ${CACHE_FILE}, scanning`);
+        } else {
+            console.log(`cache read failed, rescanning:`, (err as Error).stack ?? err);
+        }
+    }
     let files: FileInfo[] = [];
-    await collectFiles(process.cwd(), "", files);
-    console.log(`collected ${files.length} files from ${process.cwd()}`);
+    let scanStart = performance.now();
+    await collectFiles(SCAN_DIRS, files);
+    await fs.promises.writeFile(CACHE_FILE, JSON.stringify({ dirs: SCAN_DIRS, files }));
+    console.log(`scanned ${formatNumber(files.length)} files in ${formatTime(performance.now() - scanStart)}, cached to ${CACHE_FILE}`);
+    return files;
+}
+
+async function main() {
+    let files = await getFiles();
+    console.log(`collected ${formatNumber(files.length)} files from ${SCAN_DIRS.join(", ")}`);
     summarize(files, true);
     summarize(files, false);
 }

package/treeSummary.ts

@@ -1,48 +1,96 @@
 import { sort } from "socket-function/src/misc";
 
+const DEFAULT_OUTPUT_COUNT = 100;
+const TRACKED_NODES_PER_OUTPUT = 100;
+const MIN_TRACKED_NODES = 1_000;
+const MAX_TRACKED_NODES = 10_000;
+const PRUNE_TARGET_DIVISOR = 2;
+const PRUNE_SEARCH_ITERATIONS = 20;
+
 type TreeNode<S> = {
+    // The path fragment this node covers, i.e. the edge from its parent. Children are keyed by the first character of their label.
+    label: string;
     children: Map<string, TreeNode<S>>;
     // Aggregate of every value whose path passes through (or ends at) this node.
     summary: S;
     // Aggregate of only the values whose path ends exactly at this node.
     selfSummary?: S;
+    // Aggregate of values that were below this node before pruning collapsed their branches. Their path detail is gone, but their data is preserved.
+    truncatedSummary?: S;
 };
 
-type SummaryEntry<S> = {
+export type SummaryEntry<S> = {
+    // Ends with "*" unless the entry is a single exact path that was heavy enough to stand on its own.
     path: string;
+    // "self" = values whose path is exactly path. "subtree" = every value whose path starts with path. "group" = values under path kept combined (light sibling branches, chain leftovers, or both) because none was heavy enough to deserve its own entry. "truncated" = values under path whose detail was pruned away.
+    kind: "self" | "subtree" | "group" | "truncated";
     summary: S;
     weight: number;
 };
 
-type PendingEntry<S> = SummaryEntry<S> & {
-    node: TreeNode<S>;
-    expandable: boolean;
+type PendingEntry<S> = {
+    kind: "self" | "subtree" | "group" | "truncated";
+    path: string;
+    node?: TreeNode<S>;
+    // Where node actually sits when path compression descended a single-child chain past carried residues; equals path when no residues were carried.
+    nodePath?: string;
+    members?: Map<string, TreeNode<S>>;
+    // Self/truncated summaries that sit between path and node (for "subtree"), or the loose summaries of a combined entry (for "group"/"truncated").
+    extras?: S[];
+    weight: number;
+};
+
+type NodeStats = {
+    size: number;
+    weight: number;
 };
 
 export class TreeSummary<T, S> {
     private root: TreeNode<S>;
+    private nodeCount = 1;
+    private maxTrackedNodes: number;
 
     constructor(private config: {
         getPath: (value: T) => string;
         createSummary: () => S;
         addToSummary: (value: T, summary: S) => void;
+        mergeSummaries: (target: S, source: S) => void;
         getWeight: (summary: S) => number;
+        // Sizes the internal tree, which keeps roughly TRACKED_NODES_PER_OUTPUT times this many nodes so output granularity decisions have detail to work with.
+        expectedOutputCount?: number;
     }) {
-        this.root = { children: new Map(), summary: config.createSummary() };
+        this.root = { label: "", children: new Map(), summary: config.createSummary() };
+        let outputCount = config.expectedOutputCount || DEFAULT_OUTPUT_COUNT;
+        this.maxTrackedNodes = Math.min(Math.max(outputCount * TRACKED_NODES_PER_OUTPUT, MIN_TRACKED_NODES), MAX_TRACKED_NODES);
     }
 
     public add(value: T) {
         let path = this.config.getPath(value);
         let node = this.root;
         this.config.addToSummary(value, node.summary);
-        for (let char of path) {
-            let child = node.children.get(char);
+        let pos = 0;
+        while (pos < path.length) {
+            let child = node.children.get(path[pos]);
             if (!child) {
-                child = { children: new Map(), summary: this.config.createSummary() };
-                node.children.set(char, child);
+                child = { label: path.slice(pos), children: new Map(), summary: this.config.createSummary() };
+                node.children.set(path[pos], child);
+                this.nodeCount++;
+                this.config.addToSummary(value, child.summary);
+                node = child;
+                break;
+            }
+            let label = child.label;
+            let limit = Math.min(label.length, path.length - pos);
+            let common = 1;
+            while (common < limit && label.charCodeAt(common) === path.charCodeAt(pos + common)) {
+                common++;
+            }
+            if (common < label.length) {
+                this.splitNode(child, common);
             }
             this.config.addToSummary(value, child.summary);
             node = child;
+            pos += common;
         }
         let selfSummary = node.selfSummary;
         if (!selfSummary) {
@@ -50,78 +98,307 @@ export class TreeSummary<T, S> {
             node.selfSummary = selfSummary;
         }
         this.config.addToSummary(value, selfSummary);
+        if (this.nodeCount > this.maxTrackedNodes) {
+            this.prune();
+        }
     }
 
-    // Returns at most maxCount entries. Starts with the root as a single entry, then repeatedly expands the heaviest expandable entry into its children, so heavy prefixes get more depth and light ones stay collapsed.
+    // Splits a node's edge label at labelPos, inserting a new parent covering the prefix. The prefix node's summary is a merge-clone of the original aggregate, since it covers the exact same set of values.
+    private splitNode(node: TreeNode<S>, labelPos: number) {
+        let suffix: TreeNode<S> = {
+            label: node.label.slice(labelPos),
+            children: node.children,
+            summary: node.summary,
+            selfSummary: node.selfSummary,
+            truncatedSummary: node.truncatedSummary,
+        };
+        let prefixSummary = this.config.createSummary();
+        this.config.mergeSummaries(prefixSummary, suffix.summary);
+        node.label = node.label.slice(0, labelPos);
+        node.summary = prefixSummary;
+        node.selfSummary = undefined;
+        node.truncatedSummary = undefined;
+        node.children = new Map([[suffix.label[0], suffix]]);
+        this.nodeCount++;
+    }
+
+    public getTrackedNodeCount(): number {
+        return this.nodeCount;
+    }
+
+    // Starts fully collapsed (one entry for the whole tree) and repeatedly refines the heaviest entry, one split at a time. A split pulls the heaviest branch out as its own entry and leaves the remaining siblings combined in a group entry, so entries are never wasted on light branches. This recomputes from the live per-node summaries, so it always reflects everything added so far.
     public getSummaries(maxCount: number): SummaryEntry<S>[] {
         if (maxCount <= 0) {
             throw new Error(`maxCount must be positive, was ${maxCount}`);
         }
-        if (!this.root.selfSummary && this.root.children.size === 0) {
+        if (!this.root.selfSummary && !this.root.truncatedSummary && this.root.children.size === 0) {
             return [];
         }
-        let rootStart = this.compressChain(this.root, "");
-        let entries: PendingEntry<S>[] = [this.makeEntry(rootStart.node, rootStart.path)];
-        while (true) {
-            let best: PendingEntry<S> | undefined;
-            for (let entry of entries) {
-                if (!entry.expandable) continue;
-                // Expanding replaces 1 entry with this many.
-                let expandCount = entry.node.children.size + (entry.node.selfSummary && 1 || 0);
-                if (entries.length - 1 + expandCount > maxCount) {
-                    entry.expandable = false;
-                    continue;
-                }
-                if (!best || entry.weight > best.weight) {
-                    best = entry;
+        let entries: PendingEntry<S>[] = [this.subtreeEntry(this.root, "")];
+        while (entries.length < maxCount) {
+            let bestIndex = -1;
+            for (let i = 0; i < entries.length; i++) {
+                if (!this.isRefinable(entries[i])) continue;
+                if (bestIndex === -1 || entries[i].weight > entries[bestIndex].weight) {
+                    bestIndex = i;
                 }
             }
-            if (!best) break;
-            let expanded = this.expandNode(best.node, best.path);
-            entries.splice(entries.indexOf(best), 1, ...expanded);
+            if (bestIndex === -1) break;
+            let refined = this.refine(entries[bestIndex]);
+            entries.splice(bestIndex, 1, ...refined);
         }
-        let output = entries.map(entry => ({ path: entry.path, summary: entry.summary, weight: entry.weight }));
+        let output = entries.map(entry => {
+            let path = entry.path;
+            if (entry.kind !== "self") {
+                path += "*";
+            }
+            return {
+                path,
+                kind: entry.kind,
+                summary: this.buildOutputSummary(entry),
+                weight: entry.weight,
+            };
+        });
         sort(output, entry => entry.path);
         return output;
     }
 
-    private makeEntry(node: TreeNode<S>, path: string): PendingEntry<S> {
+    private isRefinable(entry: PendingEntry<S>): boolean {
+        if (entry.kind === "self" || entry.kind === "truncated") return false;
+        if (entry.kind === "group") {
+            let members = entry.members;
+            if (!members || members.size === 0) return false;
+            return members.size + (entry.extras && entry.extras.length || 0) >= 2;
+        }
+        let node = entry.node;
+        if (!node) return false;
+        if (entry.extras && entry.extras.length > 0) return true;
+        let parts = node.children.size + (node.selfSummary && 1 || 0) + (node.truncatedSummary && 1 || 0);
+        return parts >= 2;
+    }
+
+    // Replaces one entry with exactly two, so each refinement costs exactly one output slot.
+    private refine(entry: PendingEntry<S>): PendingEntry<S>[] {
+        if (entry.kind === "group") {
+            let members = entry.members;
+            if (!members) {
+                throw new Error(`Group entry missing members at ${entry.path}`);
+            }
+            return this.splitParts(members, entry.extras || [], entry.path);
+        }
+        let node = entry.node;
+        if (entry.kind !== "subtree" || !node) {
+            throw new Error(`Entry is not refinable at ${entry.path}`);
+        }
+        let extras = entry.extras;
+        if (extras && extras.length > 0) {
+            // Split off the chain residues as one combined entry, so a long chain of pruning leftovers costs a single slot instead of one per character.
+            let weight = 0;
+            for (let extra of extras) {
+                weight += this.config.getWeight(extra);
+            }
+            return [
+                { kind: "group", path: entry.path, extras, weight },
+                this.subtreeEntry(node, entry.nodePath || entry.path),
+            ];
+        }
+        // Children live below the node, so their paths must extend nodePath, never the (possibly shorter) display path.
+        let nodePath = entry.nodePath || entry.path;
+        let truncatedList: S[] = [];
+        if (node.truncatedSummary) {
+            truncatedList.push(node.truncatedSummary);
+        }
+        let selfSummary = node.selfSummary;
+        if (selfSummary) {
+            return [
+                { kind: "self", path: nodePath, node, weight: this.config.getWeight(selfSummary) },
+                this.entryForParts(node.children, truncatedList, nodePath),
+            ];
+        }
+        return this.splitParts(node.children, truncatedList, nodePath);
+    }
+
+    private splitParts(children: Map<string, TreeNode<S>>, extras: S[], path: string): PendingEntry<S>[] {
+        let heaviestChar = "";
+        let heaviest: TreeNode<S> | undefined;
+        for (let [char, child] of children) {
+            if (!heaviest || this.config.getWeight(child.summary) > this.config.getWeight(heaviest.summary)) {
+                heaviestChar = char;
+                heaviest = child;
+            }
+        }
+        if (!heaviest) {
+            throw new Error(`Cannot split empty child set at ${path}`);
+        }
+        let rest = new Map(children);
+        rest.delete(heaviestChar);
+        return [
+            this.subtreeEntry(heaviest, path + heaviest.label),
+            this.entryForParts(rest, extras, path),
+        ];
+    }
+
+    private entryForParts(children: Map<string, TreeNode<S>>, extras: S[], path: string): PendingEntry<S> {
+        if (children.size === 0) {
+            if (extras.length === 0) {
+                throw new Error(`No parts to make an entry from at ${path}`);
+            }
+            let weight = 0;
+            for (let extra of extras) {
+                weight += this.config.getWeight(extra);
+            }
+            return { kind: "truncated", path, extras, weight };
+        }
+        if (children.size === 1 && extras.length === 0) {
+            for (let child of children.values()) {
+                return this.subtreeEntry(child, path + child.label);
+            }
+        }
+        let weight = 0;
+        for (let extra of extras) {
+            weight += this.config.getWeight(extra);
+        }
+        for (let child of children.values()) {
+            weight += this.config.getWeight(child.summary);
+        }
+        return { kind: "group", path, members: children, extras, weight };
+    }
+
+    private subtreeEntry(node: TreeNode<S>, path: string): PendingEntry<S> {
+        // Follow single-child chains so paths are maximal shared prefixes instead of single characters. Self/truncated summaries found along the chain are carried on the entry instead of stopping the descent, so pruning leftovers scattered along a chain end up combined instead of each forcing an entry.
+        let carried: S[] = [];
+        let carriedWeight = 0;
+        let nodePath = path;
+        while (node.children.size === 1) {
+            let selfSummary = node.selfSummary;
+            if (selfSummary) {
+                carried.push(selfSummary);
+                carriedWeight += this.config.getWeight(selfSummary);
+            }
+            let truncated = node.truncatedSummary;
+            if (truncated) {
+                carried.push(truncated);
+                carriedWeight += this.config.getWeight(truncated);
+            }
+            for (let child of node.children.values()) {
+                nodePath += child.label;
+                node = child;
+            }
+        }
+        // With nothing carried the entry is just the deep node, so display the full descended prefix.
+        if (carried.length === 0) {
+            path = nodePath;
+        }
         return {
+            kind: "subtree",
             path,
             node,
-            summary: node.summary,
-            weight: this.config.getWeight(node.summary),
-            expandable: node.children.size > 0,
+            nodePath,
+            extras: carried,
+            weight: this.config.getWeight(node.summary) + carriedWeight,
         };
     }
 
-    private expandNode(node: TreeNode<S>, path: string): PendingEntry<S>[] {
-        let result: PendingEntry<S>[] = [];
-        let selfSummary = node.selfSummary;
-        if (selfSummary) {
-            result.push({
-                path,
-                node,
-                summary: selfSummary,
-                weight: this.config.getWeight(selfSummary),
-                expandable: false,
-            });
+    private buildOutputSummary(entry: PendingEntry<S>): S {
+        let result = this.config.createSummary();
+        if (entry.kind === "group" || entry.kind === "truncated") {
+            for (let extra of entry.extras || []) {
+                this.config.mergeSummaries(result, extra);
+            }
+            if (entry.members) {
+                for (let member of entry.members.values()) {
+                    this.config.mergeSummaries(result, member.summary);
+                }
+            }
+            return result;
+        }
+        let node = entry.node;
+        if (!node) {
+            throw new Error(`Entry missing node at ${entry.path}`);
+        }
+        if (entry.kind === "self") {
+            let selfSummary = node.selfSummary;
+            if (!selfSummary) {
+                throw new Error(`Self entry missing selfSummary at ${entry.path}`);
+            }
+            this.config.mergeSummaries(result, selfSummary);
+            return result;
         }
-        for (let [char, child] of node.children) {
-            let compressed = this.compressChain(child, path + char);
-            result.push(this.makeEntry(compressed.node, compressed.path));
+        for (let extra of entry.extras || []) {
+            this.config.mergeSummaries(result, extra);
         }
+        this.config.mergeSummaries(result, node.summary);
         return result;
     }
 
-    // Follows single-child chains with no terminal values, so entry paths are maximal shared prefixes instead of single characters.
-    private compressChain(node: TreeNode<S>, path: string): { node: TreeNode<S>; path: string } {
-        while (!node.selfSummary && node.children.size === 1) {
-            for (let [char, child] of node.children) {
-                path += char;
-                node = child;
+    // Collapses the lightest subtrees until the node count is back under the target. Collapsing a node merges everything below it into its truncatedSummary, keeping the data but dropping the path detail. If those prefixes get heavy again later, new adds rebuild branches there and the next prune takes out whatever stayed light instead.
+    private prune() {
+        let target = Math.floor(this.maxTrackedNodes / PRUNE_TARGET_DIVISOR);
+        let need = this.nodeCount - target;
+        if (need <= 0) return;
+        let stats = new Map<TreeNode<S>, NodeStats>();
+        let measure = (node: TreeNode<S>): NodeStats => {
+            let size = 1;
+            for (let child of node.children.values()) {
+                size += measure(child).size;
+            }
+            let result = { size, weight: this.config.getWeight(node.summary) };
+            stats.set(node, result);
+            return result;
+        };
+        let rootStats = measure(this.root);
+        let getStats = (node: TreeNode<S>): NodeStats => {
+            let stat = stats.get(node);
+            if (!stat) {
+                throw new Error(`Missing node stats during prune`);
+            }
+            return stat;
+        };
+        // Collapsing every maximal subtree whose weight is under a limit frees a monotonically increasing number of nodes, so binary search for the smallest limit that frees enough.
+        let freedAt = (limit: number): number => {
+            let freed = 0;
+            let visit = (node: TreeNode<S>) => {
+                if (node.children.size === 0) return;
+                if (getStats(node).weight <= limit) {
+                    freed += getStats(node).size - 1;
+                    return;
+                }
+                for (let child of node.children.values()) {
+                    visit(child);
+                }
+            };
+            visit(this.root);
+            return freed;
+        };
+        let low = 0;
+        let high = rootStats.weight;
+        for (let i = 0; i < PRUNE_SEARCH_ITERATIONS; i++) {
+            let mid = (low + high) / 2;
+            if (freedAt(mid) >= need) {
+                high = mid;
+            } else {
+                low = mid;
             }
         }
-        return { node, path };
+        let collapseVisit = (node: TreeNode<S>) => {
+            if (node.children.size === 0) return;
+            if (getStats(node).weight <= high) {
+                let truncated = node.truncatedSummary;
+                if (!truncated) {
+                    truncated = this.config.createSummary();
+                    node.truncatedSummary = truncated;
+                }
+                for (let child of node.children.values()) {
+                    this.config.mergeSummaries(truncated, child.summary);
+                }
+                this.nodeCount -= getStats(node).size - 1;
+                node.children.clear();
+                return;
+            }
+            for (let child of node.children.values()) {
+                collapseVisit(child);
+            }
+        };
+        collapseVisit(this.root);
     }
 }