querysub 0.356.0 → 0.357.0

package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts

@@ -0,0 +1,146 @@
+import { formatPercent, formatNumber } from "socket-function/src/formatting/format";
+import { red } from "socket-function/src/formatting/logColors";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { Unit } from "./BufferIndexHelpers";
+
+export class UnitSet {
+    // Hash-based set for checking unit membership (no positions stored)
+    // Similar to the hash table approach in UnitRefList.encode, but only stores presence
+
+    @measureFnc
+    static encode(blocks: Buffer[][]): Buffer {
+        const MAX_FILL_RATIO = 0.65;
+
+        // First pass: count total units
+        let totalUnits = 0;
+        for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
+            const block = blocks[blockIndex];
+            for (let bufferIndex = 0; bufferIndex < block.length; bufferIndex++) {
+                const buffer = block[bufferIndex];
+                if (buffer.length >= 4) {
+                    totalUnits += buffer.length - 3;
+                }
+            }
+        }
+
+        if (totalUnits === 0) {
+            return Buffer.from(new Uint32Array([0]).buffer);
+        }
+
+        function tryEncodeWithRatio(estimatedUniqueRatio: number): { hashTable: Uint32Array; collisions: number; totalInserts: number; } | undefined {
+            const HASH_FACTOR = 2;
+            const estimatedUnique = Math.max(Math.ceil(totalUnits / estimatedUniqueRatio), 16);
+            const desiredCapacity = estimatedUnique * HASH_FACTOR;
+            const hashTableCount = Math.pow(2, Math.ceil(Math.log2(desiredCapacity)));
+
+            const maxUniqueCount = Math.floor(hashTableCount * MAX_FILL_RATIO);
+
+            // Each entry is just a single uint32 (the unit value, 0 means empty)
+            const hashTable = new Uint32Array(hashTableCount);
+
+            function getHashIndex(unit: number): number {
+                const hash = Math.imul(unit, 2654435761);
+                return (hash >>> 16) & (hashTableCount - 1);
+            }
+
+            function getNextIndex(index: number): number {
+                return (index + 1) & (hashTableCount - 1);
+            }
+
+            let uniqueCount = 0;
+            let collisions = 0;
+            let totalInserts = 0;
+
+            // Iterate and add to hash table directly (no first pass)
+            for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
+                const block = blocks[blockIndex];
+                for (let bufferIndex = 0; bufferIndex < block.length; bufferIndex++) {
+                    const buffer = block[bufferIndex];
+                    // Extract units directly
+                    for (let i = 0; i <= buffer.length - 4; i++) {
+                        const unit = buffer.readUint32LE(i);
+                        if (!unit) continue;
+
+                        totalInserts++;
+                        let index = getHashIndex(unit);
+                        // Linear probing
+                        while (true) {
+                            if (hashTable[index] === 0) {
+                                // Empty slot - insert
+                                hashTable[index] = unit;
+                                uniqueCount++;
+                                if (uniqueCount > maxUniqueCount) {
+                                    return undefined; // Failed, exceeded fill ratio
+                                }
+                                break;
+                            }
+                            if (hashTable[index] === unit) {
+                                // Already present
+                                break;
+                            }
+                            // Collision, probe next
+                            collisions++;
+                            index = getNextIndex(index);
+                        }
+                    }
+                }
+            }
+
+            return { hashTable, collisions, totalInserts };
+        }
+
+        // Try with increasingly conservative ratios until we succeed
+        let ratio = 1000;
+        let result: ReturnType<typeof tryEncodeWithRatio>;
+        while (true) {
+            result = tryEncodeWithRatio(ratio);
+            if (result) break;
+            ratio /= 2;
+        }
+
+        const collisionRate = result.totalInserts > 0 ? result.collisions / result.totalInserts : 0;
+
+        if (collisionRate > 0.5) {
+            console.warn(red(`WARNING: UnitSet collision rate is over 50%: ${formatPercent(collisionRate)}`));
+        }
+
+        // Store capacity as first uint32, followed by the hash table
+        const output = new Uint32Array(1 + result.hashTable.length);
+        output[0] = result.hashTable.length;
+        output.set(result.hashTable, 1);
+
+        return Buffer.from(output.buffer);
+    }
+
+    @measureFnc
+    static has(data: Buffer, unit: Unit): boolean {
+        const hashTableCount = data.readUInt32LE(0);
+
+        if (hashTableCount === 0) return false;
+
+        function getHashIndex(unit: number): number {
+            const hash = Math.imul(unit, 2654435761);
+            return (hash >>> 16) & (hashTableCount - 1);
+        }
+
+        function getNextIndex(index: number): number {
+            return (index + 1) & (hashTableCount - 1);
+        }
+
+        let index = getHashIndex(unit);
+        const maxProbes = hashTableCount;
+
+        for (let probes = 0; probes < maxProbes; probes++) {
+            const value = data.readUInt32LE((1 + index) * 4);
+            if (value === 0) return false; // Empty slot, unit not present
+            if (value === unit) return true; // Found it
+            index = getNextIndex(index);
+        }
+
+        return false; // Shouldn't reach here if hash table isn't full
+    }
+}
+
+
+
+

package/src/diagnostics/logs/IndexedLogs/FilePathSelector.tsx

@@ -0,0 +1,408 @@
+import { formatDateTime, formatNumber } from "socket-function/src/formatting/format";
+import { css } from "typesafecss";
+import { t } from "../../../2-proxy/schema2";
+import { qreact } from "../../../4-dom/qreact";
+import { Button } from "../../../library-components/Button";
+import { TimeFilePathWithSize } from "./IndexedLogs";
+import { keyByArray } from "socket-function/src/misc";
+import { MachineThreadInfo } from "../../MachineThreadInfo";
+
+export type FilePathsByThread = {
+    threadId: string;
+    files: TimeFilePathWithSize[];
+    totalSize: number;
+    totalLogCount: number;
+};
+
+export type FilePathsByMachine = {
+    machineId: string;
+    threads: FilePathsByThread[];
+    totalSize: number;
+    totalLogCount: number;
+};
+
+
+export class FilePathSelector extends qreact.Component<{
+    paths: TimeFilePathWithSize[];
+    onChange: (paths: TimeFilePathWithSize[]) => void;
+}> {
+    state = t.state({
+        showFullScreenModal: t.boolean,
+    });
+
+    getGroupedByMachine(): FilePathsByMachine[] {
+        let byMachine = keyByArray(this.props.paths, (path) => path.machineId || "unknown");
+
+        let result: FilePathsByMachine[] = [];
+        for (let [machineId, files] of byMachine) {
+            let byThread = keyByArray(files, (file) => file.threadId || "unknown");
+
+            let threads: FilePathsByThread[] = [];
+            let totalSize = 0;
+            let totalLogCount = 0;
+
+            for (let [threadId, threadFiles] of byThread) {
+                let threadSize = 0;
+                let threadLogCount = 0;
+                for (let file of threadFiles) {
+                    threadSize += file.size;
+                    threadLogCount += file.logCount || 0;
+                }
+                threads.push({ threadId, files: threadFiles, totalSize: threadSize, totalLogCount: threadLogCount });
+                totalSize += threadSize;
+                totalLogCount += threadLogCount;
+            }
+
+            result.push({ machineId, threads, totalSize, totalLogCount });
+        }
+
+        return result;
+    }
+
+    getPendingCount(): number {
+        return this.props.paths.filter(p => p.threadId !== undefined).length;
+    }
+
+    getUniqueThreadCount(): number {
+        let uniqueThreads = new Set<string>();
+        for (let path of this.props.paths) {
+            if (path.threadId !== undefined) {
+                uniqueThreads.add(path.threadId);
+            }
+        }
+        return uniqueThreads.size;
+    }
+
+    getTotals() {
+        let totalSize = 0;
+        let totalLogCount = 0;
+        for (let path of this.props.paths) {
+            totalSize += path.size;
+            totalLogCount += path.logCount || 0;
+        }
+        return { totalSize, totalLogCount, totalPending: this.getPendingCount(), uniqueThreads: this.getUniqueThreadCount() };
+    }
+
+    formatBytes(bytes: number): string {
+        return formatNumber(bytes) + "B";
+    }
+
+    render() {
+        let grouped = this.getGroupedByMachine();
+        let totals = this.getTotals();
+
+        return (
+            <div className={css.vbox(10)}>
+                <div
+                    className={css.hbox(10).pad2(10).bord(1, { h: 0, s: 0, l: 80 }).hsl(240, 20, 95).hslhover(240, 30, 90).cursor("pointer")}
+                    onClick={() => this.state.showFullScreenModal = true}
+                >
+                    <div>Machines: {formatNumber(grouped.length)} |</div>
+                    <div>Threads: {formatNumber(totals.uniqueThreads)} |</div>
+                    <div>Files: {formatNumber(this.props.paths.length)} |</div>
+                    <div>Pending: {formatNumber(totals.totalPending)} |</div>
+                    <div>Size: {this.formatBytes(totals.totalSize)} |</div>
+                    <div>Log Count: {formatNumber(totals.totalLogCount)}</div>
+                </div>
+
+                {this.state.showFullScreenModal && (
+                    <div
+                        className={css.fixed.pos(0, 0).size("100%", "100%").hsla(0, 0, 0, 0.5).vbox(0).zIndex(1000)}
+                        onClick={() => this.state.showFullScreenModal = false}
+                    >
+                        <FilePathSelectorModal
+                            allPaths={this.props.paths}
+                            grouped={grouped}
+                            onSave={(selectedPaths) => {
+                                this.props.onChange(selectedPaths);
+                                this.state.showFullScreenModal = false;
+                            }}
+                            onCancel={() => this.state.showFullScreenModal = false}
+                            formatBytes={(bytes) => this.formatBytes(bytes)}
+                        />
+                    </div>
+                )}
+            </div>
+        );
+    }
+}
+
+
+export class FilePathSelectorModal extends qreact.Component<{
+    allPaths: TimeFilePathWithSize[];
+    grouped: FilePathsByMachine[];
+    onSave: (selectedPaths: TimeFilePathWithSize[]) => void;
+    onCancel: () => void;
+    formatBytes: (bytes: number) => string;
+}> {
+    state = t.state({
+        expandedMachines: t.lookup({
+            expanded: t.boolean
+        }),
+        expandedThreads: t.lookup({
+            expanded: t.boolean
+        }),
+        selectedPaths: t.atomic<Set<string>>(new Set()),
+    });
+
+    componentDidMount(): void {
+        this.initializeSelectedPaths();
+        this.initializeExpandedMachines();
+    }
+
+    initializeSelectedPaths() {
+        let selected = new Set<string>();
+        for (let path of this.props.allPaths) {
+            selected.add(path.fullPath);
+        }
+        this.state.selectedPaths = selected;
+    }
+
+    initializeExpandedMachines() {
+        for (let machine of this.props.grouped) {
+            this.state.expandedMachines[machine.machineId] = { expanded: true };
+        }
+    }
+
+    toggleMachine(machineId: string) {
+        let machine = this.props.grouped.find(m => m.machineId === machineId);
+        if (!machine) return;
+
+        let allFiles = machine.threads.flatMap(t => t.files);
+        let allSelected = allFiles.every(f => this.state.selectedPaths.has(f.fullPath));
+        let newSelected = new Set(this.state.selectedPaths);
+
+        if (allSelected) {
+            for (let file of allFiles) {
+                newSelected.delete(file.fullPath);
+            }
+        } else {
+            for (let file of allFiles) {
+                newSelected.add(file.fullPath);
+            }
+        }
+
+        this.state.selectedPaths = newSelected;
+    }
+
+    toggleThread(machineId: string, threadId: string) {
+        let machine = this.props.grouped.find(m => m.machineId === machineId);
+        if (!machine) return;
+
+        let thread = machine.threads.find(t => t.threadId === threadId);
+        if (!thread) return;
+
+        let allSelected = thread.files.every(f => this.state.selectedPaths.has(f.fullPath));
+        let newSelected = new Set(this.state.selectedPaths);
+
+        if (allSelected) {
+            for (let file of thread.files) {
+                newSelected.delete(file.fullPath);
+            }
+        } else {
+            for (let file of thread.files) {
+                newSelected.add(file.fullPath);
+            }
+        }
+
+        this.state.selectedPaths = newSelected;
+    }
+
+    toggleFile(fullPath: string) {
+        let newSelected = new Set(this.state.selectedPaths);
+        if (newSelected.has(fullPath)) {
+            newSelected.delete(fullPath);
+        } else {
+            newSelected.add(fullPath);
+        }
+        this.state.selectedPaths = newSelected;
+    }
+
+    save() {
+        let selectedPaths = this.props.allPaths.filter(p => this.state.selectedPaths.has(p.fullPath));
+        this.props.onSave(selectedPaths);
+    }
+
+    getSelectedSummary() {
+        let selectedFiles = this.props.allPaths.filter(p => this.state.selectedPaths.has(p.fullPath));
+        let totalSize = 0;
+        let totalLogCount = 0;
+        let uniqueMachines = new Set<string>();
+        let uniqueThreads = new Set<string>();
+
+        for (let file of selectedFiles) {
+            totalSize += file.size;
+            totalLogCount += file.logCount || 0;
+            if (file.machineId) {
+                uniqueMachines.add(file.machineId);
+            }
+            if (file.threadId) {
+                uniqueThreads.add(file.threadId);
+            }
+        }
+
+        return {
+            fileCount: selectedFiles.length,
+            machineCount: uniqueMachines.size,
+            threadCount: uniqueThreads.size,
+            totalSize,
+            totalLogCount
+        };
+    }
+
+    renderFile(file: TimeFilePathWithSize, machineId: string) {
+        let isSelected = this.state.selectedPaths.has(file.fullPath);
+        return (
+            <tr key={file.fullPath}>
+                <td className={css.pad2(2)}>
+                    <Button
+                        onClick={() => this.toggleFile(file.fullPath)}
+                        className={css.minWidth(100).pad2(5, 2)}
+                        hue={isSelected ? 120 : undefined}
+                    >
+                        {isSelected ? "Selected" : "Not Selected"}
+                    </Button>
+                </td>
+                <td className={css.pad2(2)}>{this.props.formatBytes(file.size)}</td>
+                <td className={css.pad2(2)}>{file.logCount !== undefined ? formatNumber(file.logCount) : ""}</td>
+                <td className={css.pad2(2)}>{file.dedupe}</td>
+                <td className={css.pad2(2)}>{formatDateTime(file.startTime)}</td>
+            </tr>
+        );
+    }
+
+    render() {
+        let summary = this.getSelectedSummary();
+
+        return (
+            <div
+                className={css.vbox(10).pad2(10).hsl(0, 0, 100).overflowAuto.width("75vw").height("75vh").marginAuto}
+                onClick={(e) => e.stopPropagation()}
+            >
+                <div className={css.hbox(10)}>
+                    <Button onClick={() => this.state.selectedPaths = new Set()}>
+                        Unselect All
+                    </Button>
+                    <Button onClick={() => this.state.selectedPaths = new Set(this.props.allPaths.map(p => p.fullPath))}>
+                        Select All
+                    </Button>
+                    <div>Machines: {formatNumber(summary.machineCount)}</div>
+                    <div>Threads: {formatNumber(summary.threadCount)}</div>
+                    <div>Files: {formatNumber(summary.fileCount)}</div>
+                    <div>Size: {this.props.formatBytes(summary.totalSize)}</div>
+                    <div>Logs: {formatNumber(summary.totalLogCount)}</div>
+                </div>
+
+                {this.props.grouped.map((machine) => {
+                    let isMachineExpanded = machine.machineId in this.state.expandedMachines;
+                    let allFiles = machine.threads.flatMap(t => t.files);
+                    let allSelected = allFiles.every(f => this.state.selectedPaths.has(f.fullPath));
+                    let someSelected = allFiles.some(f => this.state.selectedPaths.has(f.fullPath));
+                    let totalFileCount = allFiles.length;
+
+                    return (
+                        <div key={machine.machineId} className={css.vbox(0).bord(1, { h: 0, s: 0, l: 80 })}>
+                            <div
+                                className={css.hbox(5).button}
+                                onMouseDown={() => {
+                                    if (isMachineExpanded) {
+                                        delete this.state.expandedMachines[machine.machineId];
+                                    } else {
+                                        this.state.expandedMachines[machine.machineId] = { expanded: true };
+                                    }
+                                }}
+                            >
+                                <div>
+                                    {isMachineExpanded ? "▼" : "▶"}
+                                </div>
+                                <Button
+                                    onClick={(e) => {
+                                        e.stopPropagation();
+                                        this.toggleMachine(machine.machineId);
+                                    }}
+                                    onMouseDown={(e) => e.stopPropagation()}
+                                    className={css.minWidth(100)}
+                                    hue={allSelected ? 120 : undefined}
+                                >
+                                    {allSelected ? "Selected" : someSelected ? "Partial" : "Not Selected"}
+                                </Button>
+                                <MachineThreadInfo machineId={machine.machineId} />
+                                <div>Threads: {formatNumber(machine.threads.length)}</div>
+                                <div className={css.minWidth(80)}>Files: {formatNumber(totalFileCount)}</div>
+                                <div className={css.minWidth(100)}>Size: {this.props.formatBytes(machine.totalSize)}</div>
+                                <div className={css.minWidth(100)}>Logs: {formatNumber(machine.totalLogCount)}</div>
+                            </div>
+
+                            {isMachineExpanded && machine.threads.map((thread) => {
+                                let threadKey = `${machine.machineId}:${thread.threadId}`;
+                                let isThreadExpanded = threadKey in this.state.expandedThreads;
+                                let allThreadSelected = thread.files.every(f => this.state.selectedPaths.has(f.fullPath));
+                                let someThreadSelected = thread.files.some(f => this.state.selectedPaths.has(f.fullPath));
+
+                                return (
+                                    <div key={threadKey} className={css.vbox(0).bord(1, { h: 0, s: 0, l: 85 }).marginLeft(10)}>
+                                        <div
+                                            className={css.hbox(5).button}
+                                            onMouseDown={() => {
+                                                if (isThreadExpanded) {
+                                                    delete this.state.expandedThreads[threadKey];
+                                                } else {
+                                                    this.state.expandedThreads[threadKey] = { expanded: true };
+                                                }
+                                            }}
+                                        >
+                                            <div>
+                                                {isThreadExpanded ? "▼" : "▶"}
+                                            </div>
+                                            <Button
+                                                onClick={(e) => {
+                                                    e.stopPropagation();
+                                                    this.toggleThread(machine.machineId, thread.threadId);
+                                                }}
+                                                onMouseDown={(e) => e.stopPropagation()}
+                                                className={css.minWidth(100)}
+                                                hue={allThreadSelected ? 120 : undefined}
+                                            >
+                                                {allThreadSelected ? "Selected" : someThreadSelected ? "Partial" : "Not Selected"}
+                                            </Button>
+                                            <div className={css.minWidth(80)}>Files: {formatNumber(thread.files.length)}</div>
+                                            <div className={css.minWidth(100)}>Size: {this.props.formatBytes(thread.totalSize)}</div>
+                                            <div className={css.minWidth(100)}>Logs: {formatNumber(thread.totalLogCount)}</div>
+                                            <MachineThreadInfo machineId={machine.machineId} threadId={thread.threadId} onlyShowThread />
+                                        </div>
+
+                                        {isThreadExpanded && (
+                                            <table className={css.width("100%")}>
+                                                <thead>
+                                                    <tr>
+                                                        <th className={css.pad2(2).textAlign("left")}>Selection</th>
+                                                        <th className={css.pad2(2).textAlign("left")}>Size</th>
+                                                        <th className={css.pad2(2).textAlign("left")}>Logs</th>
+                                                        <th className={css.pad2(2).textAlign("left")}>Dedupe</th>
+                                                        <th className={css.pad2(2).textAlign("left")}>Start Time</th>
+                                                    </tr>
+                                                </thead>
+                                                <tbody>
+                                                    {thread.files.map((file) => this.renderFile(file, machine.machineId))}
+                                                </tbody>
+                                            </table>
+                                        )}
+                                    </div>
+                                );
+                            })}
+                        </div>
+                    );
+                })}
+
+                <div className={css.hbox(10)}>
+                    <Button onClick={() => this.save()} hue={120}>
+                        Save
+                    </Button>
+                    <Button onClick={() => this.props.onCancel()}>
+                        Cancel
+                    </Button>
+                </div>
+            </div>
+        );
+    }
+}

package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts

@@ -0,0 +1,45 @@
+import { binarySearchBasic2 } from "socket-function/src/misc";
+import { SearchParams } from "./BufferIndexHelpers";
+
+export class FindProgressTracker<T> {
+    public constructor(private config: {
+        params: SearchParams;
+        deserialize: (result: Buffer) => T;
+        getTime: (result: T) => number | undefined;
+        onResult: (result: T) => void;
+    }) { }
+
+
+    private results: { time: number; result: T }[] = [];
+
+    public addResult(buffer: Buffer, source: { startTime: number; endTime: number; }): boolean {
+        let result = this.config.deserialize(buffer);
+        let time = this.config.getTime(result) ?? source.endTime;
+
+        if (time < this.config.params.startTime) return false;
+        if (time >= this.config.params.endTime) return false;
+
+        if (this.results.length >= this.config.params.limit && time < this.results[0].time) {
+            return false;
+        }
+
+        let newObj = { time, result };
+        let index = binarySearchBasic2(this.results, x => x.time, newObj);
+        if (index < 0) index = ~index;
+        this.results.splice(index, 0, newObj);
+
+        if (this.results.length > this.config.params.limit) {
+            this.results = this.results.slice(-this.config.params.limit);
+        }
+
+        this.config.onResult(result);
+        return true;
+    }
+
+    public isSourceRelevant(source: { startTime: number; endTime: number; }): boolean {
+        if (this.results.length >= this.config.params.limit && source.endTime < this.results[0].time) {
+            return false;
+        }
+        return source.startTime <= this.config.params.endTime && source.endTime >= this.config.params.startTime;
+    }
+}