querysub 0.45.0 → 0.50.0

package/src/diagnostics/logs/DiskLoggerPage.tsx

@@ -68,6 +68,7 @@ let endTimeURL = new URLParam("endTime", undefined as number | undefined);
 
 let inspectTimeURL = new URLParam("inspectTime", "");
 
+
 export class DiskLoggerPage extends qreact.Component {
     state = t.state({
         downloadFilesHash: t.string,
@@ -118,14 +119,14 @@ export class DiskLoggerPage extends qreact.Component {
         let startTime = Number(startTimeURL.value) || (Date.now() - timeInDay);
         let endTime = Number(endTimeURL.value) || (Date.now() + timeInDay);
         const controller = diskLoggerController(getBrowserUrlNode());
-        let files: (LogFile & { nodeId: string })[] = [];
+        let allFiles: (LogFile & { nodeId: string })[] = [];
         let loadingCount = 0;
 
         for (let nodeId of selectedNodesIds) {
             try {
                 let newFiles = controller.getRemoteLogFiles(nodeId);
                 if (newFiles) {
-                    files.push(...newFiles.map(x => ({ ...x, nodeId })));
+                    allFiles.push(...newFiles.map(x => ({ ...x, nodeId })));
                 } else {
                     loadingCount++;
                 }
@@ -133,19 +134,29 @@ export class DiskLoggerPage extends qreact.Component {
                 console.log("Error reading files", nodeId, e);
             }
         }
-        let allFiles = files;
-        files = files.filter(file => file.startTime <= endTime && file.endTime >= startTime);
+        //files = files.filter(file => file.startTime <= endTime && file.endTime >= startTime);
+        // File to fix some broken files
+        allFiles = allFiles.filter(file => file.startTime >= +new Date("1980"));
         let buffers: Buffer[] = [];
 
         let firstTime = Number.MAX_SAFE_INTEGER;
         let lastTime = Number.MIN_SAFE_INTEGER;
 
-        let filesHash = JSON.stringify(files.map(x => x.path));
+        let filesHash = JSON.stringify(allFiles.map(x => x.path));
+        let fileSizeSum = 0;
+        let fileCount = 0;
+
+        let selectedFiles: (LogFile & { nodeId: string })[] = [];
 
-        for (let file of files) {
+        for (let file of allFiles) {
             try {
                 if (file.startTime < firstTime) firstTime = file.startTime;
                 if (file.endTime > lastTime) lastTime = file.endTime;
+                if (file.startTime > endTime) continue;
+                if (file.endTime < startTime) continue;
+                selectedFiles.push(file);
+                fileCount++;
+                fileSizeSum += file.size;
                 if (filesHash !== this.state.downloadFilesHash) {
                     loadingCount++;
                     continue;
@@ -164,7 +175,7 @@ export class DiskLoggerPage extends qreact.Component {
         if (firstTime === Number.MAX_SAFE_INTEGER) firstTime = startTime;
         if (lastTime === Number.MIN_SAFE_INTEGER) lastTime = endTime;
 
-        let logs: LogObj[] = loadingCount ? [] : parseLogBufferCached(JSON.stringify(files), buffers);
+        let logs: LogObj[] = loadingCount ? [] : parseLogBufferCached(JSON.stringify(allFiles), buffers);
 
         const defaultFields = ["time", "type", "actions", "param0"];
         let selectedValues = selectedFields.value.split("|").filter(x => x);
@@ -292,11 +303,11 @@ export class DiskLoggerPage extends qreact.Component {
                     options={nodeIds.map(({ nodeId, entryPoint }) => ({ value: nodeId, label: `${nodeId} (${entryPoint.replaceAll("\\", "/").split("/").pop()})` }))}
                 />
                 <div className={css.fontSize(28).boldStyle}>
-                    {selectedNodesIds.length} nodes, {files.length} files, filtered logs {formatNumber(filteredCount)} / {formatNumber(totalCount)}
+                    {selectedNodesIds.length} nodes, {fileCount}/{allFiles.length} files, filtered logs {formatNumber(filteredCount)} / {formatNumber(totalCount)}
                 </div>
                 <ShowMore maxHeight={40}>
                     <div class={css.vbox(4)}>
-                        {files.map(file =>
+                        {selectedFiles.map(file =>
                             <div class={css.hbox(4)}>
                                 <span>{formatNumber(file.size)}B</span>
                                 <span>{formatDateTime(file.startTime)} - {formatDateTime(file.endTime)}</span>
@@ -309,9 +320,11 @@ export class DiskLoggerPage extends qreact.Component {
                     <button onClick={() => {
                         this.state.downloadFilesHash = filesHash;
                     }}>
-                        Download Files
+                        Download Files ({formatNumber(fileSizeSum)}B)
                     </button>
-                </div>}
+                </div>
+                    || loadingCount > 0 && <h1 className={css.hsl(210, 75, 75).pad2(10, 2)}>Downloading {loadingCount} file(s)</h1>
+                }
                 {!inspectTime && <>
                     <TimeRangeSelector
                         start={startTimeURL} end={endTimeURL}
@@ -367,7 +380,6 @@ export class DiskLoggerPage extends qreact.Component {
                         number
                     />
                 )}
-                {loadingCount > 0 && <h1>Loading {loadingCount} files</h1>}
                 <Table {...table} initialLimit={30} />
                 {!!inspectTime && (
                     <InputLabel
@@ -398,6 +410,7 @@ const parseLogBufferCached = measureWrap(function parseLogBufferCached(hash: str
         cached.lastAccess = Date.now();
         return cached.logs;
     }
+    console.log(`Parsing buffers ${buffers.length}`);
     let logs = buffers.map(x => parseLogBuffer(x)).flat();
     let size = buffers.map(x => x.length).reduce((a, b) => a + b, 0);
     remainingCacheSize -= size;
@@ -411,6 +424,7 @@ const parseLogBufferCached = measureWrap(function parseLogBufferCached(hash: str
         logs,
         lastAccess: Date.now()
     });
+    console.log(`Parsed buffers ${buffers.length} into logs ${logs.length}`);
     return logs;
 });
 
@@ -430,6 +444,7 @@ const processLogs = cacheShallowConfigArgEqual((config: {
     filteredCount: number;
 } => {
     return measureBlock(function processLogs() {
+        console.log(`Processing logs ${config.logs.length}`);
         let { logs, order, filter, startTime, endTime, inspectTime, selectedFields } = config;
         logs = logs.slice();
         let totalCount = logs.length;
@@ -543,6 +558,7 @@ const processLogs = cacheShallowConfigArgEqual((config: {
             table.columns[key] = {};
         }
 
+        console.log(`Processed logs ${logs.length}`);
         return { table, allFields: fields, totalCount, filteredCount };
     });
 }, 10);

package/src/diagnostics/logs/diskLogger.ts

@@ -126,9 +126,12 @@ export async function getLogBuffer(path: string): Promise<Buffer | undefined> {
     return buffer;
 }
 export function parseLogBuffer(buffer: Buffer): LogObj[] {
+    let time = Date.now();
+    console.log(`Parsing buffer ${buffer.length}`);
     let pos = 0;
     let logs: LogObj[] = [];
     const newLine = "\n".charCodeAt(0);
+    let lastLoggedPos = 0;
     while (pos < buffer.length) {
         let end = buffer.indexOf(newLine, pos);
         if (end === -1) {
@@ -139,7 +142,13 @@ export function parseLogBuffer(buffer: Buffer): LogObj[] {
             logs.push(JSON.parse(line));
         } catch { }
         pos = end + 1;
+        if (pos - lastLoggedPos > 1_000_000) {
+            console.log(`Parsed ${pos} of ${buffer.length}`);
+            lastLoggedPos = pos;
+        }
     }
+    time = Date.now() - time;
+    console.log(`Parsed ${pos} of ${buffer.length} ${formatNumber(time)}`);
     return logs;
 }
 

package/src/diagnostics/trackResources.ts

@@ -5,7 +5,7 @@ import { blue } from "socket-function/src/formatting/logColors";
 import { isNode } from "socket-function/src/misc";
 import { registerPeriodic } from "./periodic";
 import { registerMeasureInfo } from "socket-function/src/profiling/measure";
-import { logNodeStateStats } from "../5-diagnostics/nodeMetadata";
+import { logNodeStateStats } from "../-0-hooks/hooks";
 
 let resources: {
     name: string;
@@ -80,7 +80,7 @@ function logResourcesNow() {
     } else {
         for (let resource of resourcesWithCounts) {
             if (resource.count === 0) continue;
-            logNodeStateStats(resource.name, formatNumber)(resource.count);
+            logNodeStateStats(resource.name, formatNumber, resource.count);
         }
     }
 }

package/src/diagnostics/watchdog.ts

@@ -7,23 +7,25 @@ import { registerPeriodic } from "./periodic";
 import { getOwnMachineId } from "../-a-auth/certs";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { logDisk } from "./logs/diskLogger";
-import { addStatPeriodic, addStatSumPeriodic, addTimeProfileDistribution, registerNodeMetadata } from "../5-diagnostics/nodeMetadata";
 import { formatPercent, formatTime } from "socket-function/src/formatting/format";
+import { pathWatcher } from "../0-path-value-core/pathValueCore";
+import { addStatPeriodic, addStatSumPeriodic, addTimeProfileDistribution, registerNodeMetadata } from "../-0-hooks/hooks";
 
 let lastProfile: MeasureProfile | undefined;
 
-addStatPeriodic(
-    "% Profiled",
-    () => {
+addStatPeriodic({
+    title: "% Profiled",
+    getValue: () => {
         if (!lastProfile) return 0;
         let entries = Object.values(lastProfile.entries);
         if (entries.length === 0) return 0;
         const timeProfiled = lastProfile.endTime - lastProfile.startTime;
+
         let timeActive = entries.map(x => x.ownTime.sum).reduce((a, b) => a + b, 0);
         return timeActive / timeProfiled;
     },
-    formatPercent,
-);
+    format: formatPercent,
+});
 registerNodeMetadata({
     columnName: "Top Profiled",
     getValue() {
@@ -31,6 +33,7 @@ registerNodeMetadata({
         let entries = Object.values(lastProfile.entries);
         if (entries.length === 0) return "";
         const timeProfiled = lastProfile.endTime - lastProfile.startTime;
+
         let rootSums = new Map<string, number>();
         for (let entry of entries) {
             let name = entry.name.split("|")[0];
@@ -42,8 +45,16 @@ registerNodeMetadata({
     },
 });
 addTimeProfileDistribution("Network Calls", () => SocketFunction.harvestCallTimes());
-addStatPeriodic("Pending Net Calls", () => SocketFunction.getPendingCallCount());
-addStatSumPeriodic("Failed Net Calls", () => SocketFunction.harvestFailedCallCount());
+addStatPeriodic({
+    title: "Pending Net Calls",
+    getValue: () => SocketFunction.getPendingCallCount(),
+});
+addStatSumPeriodic({
+    title: "Failed Net Calls",
+    getValue: () => SocketFunction.harvestFailedCallCount(),
+});
+addTimeProfileDistribution("Syncing", () => pathWatcher.debug_harvestSyncTimes());
+
 
 function endEllipsis(str: string, maxLength: number) {
     if (str.length <= maxLength) return str;

package/src/library-components/TimeRangeSelector.tsx

@@ -159,7 +159,7 @@ export let incrementUps: { [key in IncrementType]: IncrementType } = {
     minute2: "hour",
     hour: "day",
     hour6: "day",
-    day: "week",
+    day: "month",
     week: "month",
     week2: "month",
     month: "year",