querysub 0.356.0 → 0.358.0

package/src/diagnostics/logs/IndexedLogs/FilePathSelector.tsx

@@ -0,0 +1,569 @@
+import { formatDate, formatDateTime, formatNumber } from "socket-function/src/formatting/format";
+import { css } from "typesafecss";
+import { t } from "../../../2-proxy/schema2";
+import { qreact } from "../../../4-dom/qreact";
+import { Button } from "../../../library-components/Button";
+import { TimeFilePathWithSize } from "./IndexedLogs";
+import { keyBy, keyByArray } from "socket-function/src/misc";
+import { MachineThreadInfo } from "../../MachineThreadInfo";
+
+export type FilePathsByThread = {
+    threadId: string;
+    files: TimeFilePathWithSize[];
+    totalSize: number;
+    totalLogCount: number;
+    pendingCount: number;
+    errorCount: number;
+};
+
+export type FilePathsByMachine = {
+    machineId: string;
+    threads: FilePathsByThread[];
+    totalSize: number;
+    totalLogCount: number;
+    pendingCount: number;
+    errorCount: number;
+};
+
+
+export class FilePathSelector extends qreact.Component<{
+    paths: TimeFilePathWithSize[];
+    onChange: (paths: TimeFilePathWithSize[]) => void;
+    fileErrors?: { error: string; path: string; }[];
+}> {
+    state = t.state({
+        showFullScreenModal: t.boolean,
+    });
+
+    getGroupedByMachine(): FilePathsByMachine[] {
+        let byMachine = keyByArray(this.props.paths, (path) => path.machineId || "unknown");
+        let errorsByPath = keyBy(this.props.fileErrors || [], x => x.path);
+
+        let result: FilePathsByMachine[] = [];
+        for (let [machineId, files] of byMachine) {
+            let byThread = keyByArray(files, (file) => file.threadId || "unknown");
+
+            let threads: FilePathsByThread[] = [];
+            let totalSize = 0;
+            let totalLogCount = 0;
+            let machinePendingCount = 0;
+            let machineErrorCount = 0;
+
+            for (let [threadId, threadFiles] of byThread) {
+                let threadSize = 0;
+                let threadLogCount = 0;
+                let threadPendingCount = 0;
+                let threadErrorCount = 0;
+                for (let file of threadFiles) {
+                    threadSize += file.size;
+                    threadLogCount += file.logCount || 0;
+                    if (file.logCount === undefined) {
+                        threadPendingCount++;
+                    }
+                    if (errorsByPath.get(file.fullPath)) {
+                        threadErrorCount++;
+                    }
+                }
+                threads.push({ threadId, files: threadFiles, totalSize: threadSize, totalLogCount: threadLogCount, pendingCount: threadPendingCount, errorCount: threadErrorCount });
+                totalSize += threadSize;
+                totalLogCount += threadLogCount;
+                machinePendingCount += threadPendingCount;
+                machineErrorCount += threadErrorCount;
+            }
+
+            result.push({ machineId, threads, totalSize, totalLogCount, pendingCount: machinePendingCount, errorCount: machineErrorCount });
+        }
+
+        return result;
+    }
+
+    getPendingCount(): number {
+        return this.props.paths.filter(p => p.logCount === undefined).length;
+    }
+
+    getUniqueThreadCount(): number {
+        let uniqueThreads = new Set<string>();
+        for (let path of this.props.paths) {
+            if (path.threadId !== undefined) {
+                uniqueThreads.add(path.threadId);
+            }
+        }
+        return uniqueThreads.size;
+    }
+
+    getTotals() {
+        let totalSize = 0;
+        let totalLogCount = 0;
+        let minTime = Infinity;
+        let maxTime = -Infinity;
+        for (let path of this.props.paths) {
+            totalSize += path.size;
+            totalLogCount += path.logCount || 0;
+            if (path.startTime < minTime) {
+                minTime = path.startTime;
+            }
+            if (path.startTime > maxTime) {
+                maxTime = path.startTime;
+            }
+        }
+        const errorCount = this.props.fileErrors?.length || 0;
+        return {
+            totalSize,
+            totalLogCount,
+            totalPending: this.getPendingCount(),
+            uniqueThreads: this.getUniqueThreadCount(),
+            minTime: minTime === Infinity ? undefined : minTime,
+            maxTime: maxTime === -Infinity ? undefined : maxTime,
+            errorCount
+        };
+    }
+
+    formatBytes(bytes: number): string {
+        return formatNumber(bytes) + "B";
+    }
+
+    render() {
+        let grouped = this.getGroupedByMachine();
+        let totals = this.getTotals();
+
+        return (
+            <div className={css.vbox(10)}>
+                <div
+                    className={css.hbox(10).pad2(10).bord(1, { h: 0, s: 0, l: 80 }).hsl(240, 20, 95).hslhover(240, 30, 90).cursor("pointer").wrap}
+                    onClick={() => this.state.showFullScreenModal = true}
+                >
+                    <Button hue={220}>Edit</Button>
+                    <div>|</div>
+                    <div>Machines: {formatNumber(grouped.length)}</div>
+                    <div>|</div>
+                    <div>Threads: {formatNumber(totals.uniqueThreads)}</div>
+                    <div>|</div>
+                    <div>Files: {formatNumber(this.props.paths.length)}</div>
+                    <div>|</div>
+                    <div>Pending: {formatNumber(totals.totalPending)}</div>
+                    <div>|</div>
+                    <div>Size: {this.formatBytes(totals.totalSize)}</div>
+                    <div>|</div>
+                    <div>Log Count: {formatNumber(totals.totalLogCount)}</div>
+                    {totals.errorCount > 0 && (
+                        <>
+                            <div>|</div>
+                            <div className={css.colorhsl(0, 80, 40)}>Errors: {formatNumber(totals.errorCount)}</div>
+                        </>
+                    )}
+                    {totals.minTime !== undefined && totals.maxTime !== undefined && (
+                        <>
+                            <div>|</div>
+                            <div>{formatDateTime(totals.minTime)} - {formatDateTime(totals.maxTime)}</div>
+                        </>
+                    )}
+                </div>
+
+                {this.state.showFullScreenModal && (
+                    <div
+                        className={css.fixed.pos(0, 0).size("100%", "100%").hsla(0, 0, 0, 0.5).vbox(0).zIndex(1000)}
+                        onClick={() => this.state.showFullScreenModal = false}
+                    >
+                        <FilePathSelectorModal
+                            allPaths={this.props.paths}
+                            grouped={grouped}
+                            onSave={(selectedPaths) => {
+                                this.props.onChange(selectedPaths);
+                                this.state.showFullScreenModal = false;
+                            }}
+                            onCancel={() => this.state.showFullScreenModal = false}
+                            formatBytes={(bytes) => this.formatBytes(bytes)}
+                            fileErrors={this.props.fileErrors}
+                        />
+                    </div>
+                )}
+            </div>
+        );
+    }
+}
+
+
+export class FilePathSelectorModal extends qreact.Component<{
+    allPaths: TimeFilePathWithSize[];
+    grouped: FilePathsByMachine[];
+    onSave: (selectedPaths: TimeFilePathWithSize[]) => void;
+    onCancel: () => void;
+    formatBytes: (bytes: number) => string;
+    fileErrors?: { error: string; path: string; }[];
+}> {
+    state = t.state({
+        expandedMachines: t.lookup({
+            expanded: t.boolean
+        }),
+        expandedThreads: t.lookup({
+            expanded: t.boolean
+        }),
+        selectedPaths: t.atomic<Set<string>>(new Set()),
+    });
+
+    componentDidMount(): void {
+        this.initializeSelectedPaths();
+        this.initializeExpandedMachines();
+    }
+
+    initializeSelectedPaths() {
+        let selected = new Set<string>();
+        for (let path of this.props.allPaths) {
+            selected.add(path.fullPath);
+        }
+        this.state.selectedPaths = selected;
+    }
+
+    initializeExpandedMachines() {
+        for (let machine of this.props.grouped) {
+            this.state.expandedMachines[machine.machineId] = { expanded: true };
+        }
+    }
+
+    toggleMachine(machineId: string) {
+        let machine = this.props.grouped.find(m => m.machineId === machineId);
+        if (!machine) return;
+
+        let allFiles = machine.threads.flatMap(t => t.files);
+        let allSelected = allFiles.every(f => this.state.selectedPaths.has(f.fullPath));
+        let newSelected = new Set(this.state.selectedPaths);
+
+        if (allSelected) {
+            for (let file of allFiles) {
+                newSelected.delete(file.fullPath);
+            }
+        } else {
+            for (let file of allFiles) {
+                newSelected.add(file.fullPath);
+            }
+        }
+
+        this.state.selectedPaths = newSelected;
+    }
+
+    toggleThread(machineId: string, threadId: string) {
+        let machine = this.props.grouped.find(m => m.machineId === machineId);
+        if (!machine) return;
+
+        let thread = machine.threads.find(t => t.threadId === threadId);
+        if (!thread) return;
+
+        let allSelected = thread.files.every(f => this.state.selectedPaths.has(f.fullPath));
+        let newSelected = new Set(this.state.selectedPaths);
+
+        if (allSelected) {
+            for (let file of thread.files) {
+                newSelected.delete(file.fullPath);
+            }
+        } else {
+            for (let file of thread.files) {
+                newSelected.add(file.fullPath);
+            }
+        }
+
+        this.state.selectedPaths = newSelected;
+    }
+
+    toggleFile(fullPath: string) {
+        let newSelected = new Set(this.state.selectedPaths);
+        if (newSelected.has(fullPath)) {
+            newSelected.delete(fullPath);
+        } else {
+            newSelected.add(fullPath);
+        }
+        this.state.selectedPaths = newSelected;
+    }
+
+    save() {
+        let selectedPaths = this.props.allPaths.filter(p => this.state.selectedPaths.has(p.fullPath));
+        this.props.onSave(selectedPaths);
+    }
+
+    expandAll() {
+        for (let machine of this.props.grouped) {
+            this.state.expandedMachines[machine.machineId] = { expanded: true };
+            for (let thread of machine.threads) {
+                let threadKey = `${machine.machineId}:${thread.threadId}`;
+                this.state.expandedThreads[threadKey] = { expanded: true };
+            }
+        }
+    }
+
+    collapseAll() {
+        for (let machineId in this.state.expandedMachines) {
+            delete this.state.expandedMachines[machineId];
+        }
+        for (let threadKey in this.state.expandedThreads) {
+            delete this.state.expandedThreads[threadKey];
+        }
+    }
+
+    getSelectedSummary() {
+        let selectedFiles = this.props.allPaths.filter(p => this.state.selectedPaths.has(p.fullPath));
+        let errorsByPath = keyBy(this.props.fileErrors || [], x => x.path);
+        let totalSize = 0;
+        let totalLogCount = 0;
+        let pendingCount = 0;
+        let errorCount = 0;
+        let uniqueMachines = new Set<string>();
+        let uniqueThreads = new Set<string>();
+
+        for (let file of selectedFiles) {
+            totalSize += file.size;
+            totalLogCount += file.logCount || 0;
+            if (file.logCount === undefined) {
+                pendingCount++;
+            }
+            if (file.machineId) {
+                uniqueMachines.add(file.machineId);
+            }
+            if (file.threadId) {
+                uniqueThreads.add(file.threadId);
+            }
+            if (errorsByPath.get(file.fullPath)) {
+                errorCount++;
+            }
+        }
+
+        return {
+            fileCount: selectedFiles.length,
+            machineCount: uniqueMachines.size,
+            threadCount: uniqueThreads.size,
+            totalSize,
+            totalLogCount,
+            pendingCount,
+            errorCount
+        };
+    }
+
+    getSelectedSummaryBySource(): Map<string, { fileCount: number; size: number; logCount: number; pendingCount: number; errorCount: number; }> {
+        let selectedFiles = this.props.allPaths.filter(p => this.state.selectedPaths.has(p.fullPath));
+        let errorsByPath = keyBy(this.props.fileErrors || [], x => x.path);
+        let bySource = new Map<string, { fileCount: number; size: number; logCount: number; pendingCount: number; errorCount: number; }>();
+        for (let file of selectedFiles) {
+            let sourceName = file.sourceName || "unknown";
+            let existing = bySource.get(sourceName);
+            if (!existing) {
+                existing = { fileCount: 0, size: 0, logCount: 0, pendingCount: 0, errorCount: 0 };
+                bySource.set(sourceName, existing);
+            }
+            existing.fileCount++;
+            existing.size += file.size;
+            existing.logCount += file.logCount || 0;
+            if (file.logCount === undefined) {
+                existing.pendingCount++;
+            }
+            if (errorsByPath.get(file.fullPath)) {
+                existing.errorCount++;
+            }
+        }
+        return bySource;
+    }
+
+    renderFile(file: TimeFilePathWithSize, machineId: string, errorsByPath: Map<string, { error: string; path: string; }>) {
+        let isSelected = this.state.selectedPaths.has(file.fullPath);
+        let isPending = file.logCount === undefined;
+        const errorObj = errorsByPath.get(file.fullPath);
+        const error = errorObj?.error;
+        let rowClassName = error && css.hsl(0, 30, 95) || isPending && css.hsl(40, 30, 95) || undefined;
+        return (
+            <tr key={file.fullPath} className={rowClassName}>
+                <td className={css.pad2(2)}>
+                    <Button
+                        onClick={() => this.toggleFile(file.fullPath)}
+                        className={css.minWidth(100).pad2(5, 2)}
+                        hue={isSelected ? 120 : undefined}
+                    >
+                        {isSelected ? "Selected" : "Not Selected"}
+                    </Button>
+                </td>
+                <td className={css.pad2(2)}>{this.props.formatBytes(file.size)}</td>
+                <td className={css.pad2(2)}>{file.logCount !== undefined ? formatNumber(file.logCount) : <span className={css.colorhsl(40, 60, 40)}>pending</span>}</td>
+                <td className={css.pad2(2)}>{file.sourceName}</td>
+                <td className={css.pad2(2)}>{file.dedupe}</td>
+                <td className={css.pad2(2)}>{formatDateTime(file.startTime)}</td>
+                <td className={css.pad2(2)}>
+                    {error && (
+                        <div className={css.colorhsl(0, 80, 40).fontWeight(600)} title={error}>ERROR</div>
+                    )}
+                </td>
+            </tr>
+        );
+    }
+
+    render() {
+        let summary = this.getSelectedSummary();
+        let bySource = this.getSelectedSummaryBySource();
+        let errorsByPath = keyBy(this.props.fileErrors || [], x => x.path);
+
+        return (
+            <div
+                className={css.vbox(10).pad2(10).hsl(0, 0, 100).overflowAuto.width("75vw").height("75vh").marginAuto}
+                onClick={(e) => e.stopPropagation()}
+            >
+                <div className={css.hbox(10).alignItems("start")}>
+                    <Button onClick={() => this.state.selectedPaths = new Set()}>
+                        Unselect All
+                    </Button>
+                    <Button onClick={() => this.state.selectedPaths = new Set(this.props.allPaths.map(p => p.fullPath))}>
+                        Select All
+                    </Button>
+                    <Button onClick={() => this.expandAll()}>
+                        Expand All
+                    </Button>
+                    <Button onClick={() => this.collapseAll()}>
+                        Collapse All
+                    </Button>
+
+                    <div className={css.vbox(5)}>
+                        <div className={css.hbox(10)}>
+                            <div className={css.minWidth(120)}>Machines: {formatNumber(summary.machineCount)}</div>
+                            <div className={css.minWidth(110)}>Threads: {formatNumber(summary.threadCount)}</div>
+                            <div className={css.minWidth(140)}>Files: {formatNumber(summary.fileCount)} ({formatNumber(summary.pendingCount)} pending)</div>
+                            <div className={css.minWidth(120)}>Size: {this.props.formatBytes(summary.totalSize)}</div>
+                            <div className={css.minWidth(100)}>Logs: {formatNumber(summary.totalLogCount)}</div>
+                            {summary.errorCount > 0 && (
+                                <div className={css.minWidth(100) + css.colorhsl(0, 80, 40)}>Errors: {formatNumber(summary.errorCount)}</div>
+                            )}
+                        </div>
+
+                        {bySource.size > 0 && (
+                            <div className={css.vbox(5)}>
+                                {Array.from(bySource.entries()).map(([sourceName, stats]) => (
+                                    <div key={sourceName} className={css.hbox(10)}>
+                                        <div className={css.minWidth(240)}>{sourceName}:</div>
+                                        <div className={css.minWidth(140)}>Files: {formatNumber(stats.fileCount)} ({formatNumber(stats.pendingCount)} pending)</div>
+                                        <div className={css.minWidth(120)}>Size: {this.props.formatBytes(stats.size)}</div>
+                                        <div className={css.minWidth(100)}>Logs: {formatNumber(stats.logCount)}</div>
+                                        {stats.errorCount > 0 && (
+                                            <div className={css.minWidth(100) + css.colorhsl(0, 80, 40)}>Errors: {formatNumber(stats.errorCount)}</div>
+                                        )}
+                                    </div>
+                                ))}
+                            </div>
+                        )}
+                    </div>
+                </div>
+
+
+                {this.props.grouped.map((machine) => {
+                    let isMachineExpanded = machine.machineId in this.state.expandedMachines;
+                    let allFiles = machine.threads.flatMap(t => t.files);
+                    let allSelected = allFiles.every(f => this.state.selectedPaths.has(f.fullPath));
+                    let someSelected = allFiles.some(f => this.state.selectedPaths.has(f.fullPath));
+                    let totalFileCount = allFiles.length;
+
+                    return (
+                        <div key={machine.machineId} className={css.vbox(0).bord(1, { h: 0, s: 0, l: 80 })}>
+                            <div
+                                className={css.hbox(5).button}
+                                onMouseDown={() => {
+                                    if (isMachineExpanded) {
+                                        delete this.state.expandedMachines[machine.machineId];
+                                    } else {
+                                        this.state.expandedMachines[machine.machineId] = { expanded: true };
+                                    }
+                                }}
+                            >
+                                <div>
+                                    {isMachineExpanded ? "▼" : "▶"}
+                                </div>
+                                <Button
+                                    onClick={(e) => {
+                                        e.stopPropagation();
+                                        this.toggleMachine(machine.machineId);
+                                    }}
+                                    onMouseDown={(e) => e.stopPropagation()}
+                                    className={css.minWidth(100)}
+                                    hue={allSelected ? 120 : undefined}
+                                >
+                                    {allSelected ? "Selected" : someSelected ? "Partial" : "Not Selected"}
+                                </Button>
+                                <MachineThreadInfo machineId={machine.machineId} />
+                                <div>Threads: {formatNumber(machine.threads.length)}</div>
+                                <div className={css.minWidth(120)}>Files: {formatNumber(totalFileCount)} ({formatNumber(machine.pendingCount)} pending)</div>
+                                <div className={css.minWidth(100)}>Size: {this.props.formatBytes(machine.totalSize)}</div>
+                                <div className={css.minWidth(100)}>Logs: {formatNumber(machine.totalLogCount)}</div>
+                                {machine.errorCount > 0 && (
+                                    <div className={css.minWidth(100) + css.colorhsl(0, 80, 40)}>Errors: {formatNumber(machine.errorCount)}</div>
+                                )}
+                            </div>
+
+                            {isMachineExpanded && machine.threads.map((thread) => {
+                                let threadKey = `${machine.machineId}:${thread.threadId}`;
+                                let isThreadExpanded = threadKey in this.state.expandedThreads;
+                                let allThreadSelected = thread.files.every(f => this.state.selectedPaths.has(f.fullPath));
+                                let someThreadSelected = thread.files.some(f => this.state.selectedPaths.has(f.fullPath));
+
+                                return (
+                                    <div key={threadKey} className={css.vbox(0).bord(1, { h: 0, s: 0, l: 85 }).marginLeft(10)}>
+                                        <div
+                                            className={css.hbox(5).button}
+                                            onMouseDown={() => {
+                                                if (isThreadExpanded) {
+                                                    delete this.state.expandedThreads[threadKey];
+                                                } else {
+                                                    this.state.expandedThreads[threadKey] = { expanded: true };
+                                                }
+                                            }}
+                                        >
+                                            <div>
+                                                {isThreadExpanded ? "▼" : "▶"}
+                                            </div>
+                                            <Button
+                                                onClick={(e) => {
+                                                    e.stopPropagation();
+                                                    this.toggleThread(machine.machineId, thread.threadId);
+                                                }}
+                                                onMouseDown={(e) => e.stopPropagation()}
+                                                className={css.minWidth(100)}
+                                                hue={allThreadSelected ? 120 : undefined}
+                                            >
+                                                {allThreadSelected ? "Selected" : someThreadSelected ? "Partial" : "Not Selected"}
+                                            </Button>
+                                            <div className={css.minWidth(120)}>Files: {formatNumber(thread.files.length)} ({formatNumber(thread.pendingCount)} pending)</div>
+                                            <div className={css.minWidth(100)}>Size: {this.props.formatBytes(thread.totalSize)}</div>
+                                            <div className={css.minWidth(100)}>Logs: {formatNumber(thread.totalLogCount)}</div>
+                                            {thread.errorCount > 0 && (
+                                                <div className={css.minWidth(100) + css.colorhsl(0, 80, 40)}>Errors: {formatNumber(thread.errorCount)}</div>
+                                            )}
+                                            <MachineThreadInfo machineId={machine.machineId} threadId={thread.threadId} onlyShowThread />
+                                        </div>
+
+                                        {isThreadExpanded && (
+                                            <table className={css.width("100%")}>
+                                                <thead>
+                                                    <tr>
+                                                        <th className={css.pad2(2).textAlign("left")}>Selection</th>
+                                                        <th className={css.pad2(2).textAlign("left")}>Size</th>
+                                                        <th className={css.pad2(2).textAlign("left")}>Logs</th>
+                                                        <th className={css.pad2(2).textAlign("left")}>Source</th>
+                                                        <th className={css.pad2(2).textAlign("left")}>Dedupe</th>
+                                                        <th className={css.pad2(2).textAlign("left")}>Start Time</th>
+                                                        <th className={css.pad2(2).textAlign("left")}>Status</th>
+                                                    </tr>
+                                                </thead>
+                                                <tbody>
+                                                    {thread.files.map((file) => this.renderFile(file, machine.machineId, errorsByPath))}
+                                                </tbody>
+                                            </table>
+                                        )}
+                                    </div>
+                                );
+                            })}
+                        </div>
+                    );
+                })}
+
+                <div className={css.hbox(10)}>
+                    <Button onClick={() => this.save()} hue={120}>
+                        Save
+                    </Button>
+                    <Button onClick={() => this.props.onCancel()}>
+                        Cancel
+                    </Button>
+                </div>
+            </div>
+        );
+    }
+}

package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts

@@ -0,0 +1,45 @@
+import { binarySearchBasic2 } from "socket-function/src/misc";
+import { SearchParams } from "./BufferIndexHelpers";
+
+export class FindProgressTracker<T> {
+    public constructor(private config: {
+        params: SearchParams;
+        deserialize: (result: Buffer) => T;
+        getTime: (result: T) => number | undefined;
+        onResult: (result: T) => void;
+    }) { }
+
+
+    private results: { time: number; result: T }[] = [];
+
+    public addResult(buffer: Buffer, source: { startTime: number; endTime: number; }): boolean {
+        let result = this.config.deserialize(buffer);
+        let time = this.config.getTime(result) ?? source.endTime;
+
+        if (time < this.config.params.startTime) return false;
+        if (time >= this.config.params.endTime) return false;
+
+        if (this.results.length >= this.config.params.limit && time < this.results[0].time) {
+            return false;
+        }
+
+        let newObj = { time, result };
+        let index = binarySearchBasic2(this.results, x => x.time, newObj);
+        if (index < 0) index = ~index;
+        this.results.splice(index, 0, newObj);
+
+        if (this.results.length > this.config.params.limit) {
+            this.results = this.results.slice(-this.config.params.limit);
+        }
+
+        this.config.onResult(result);
+        return true;
+    }
+
+    public isSourceRelevant(source: { startTime: number; endTime: number; }): boolean {
+        if (this.results.length >= this.config.params.limit && source.endTime < this.results[0].time) {
+            return false;
+        }
+        return source.startTime <= this.config.params.endTime && source.endTime >= this.config.params.startTime;
+    }
+}