querysub 0.42.0 → 0.44.0

package/src/diagnostics/logs/diskLogger.ts

@@ -1,5 +1,5 @@
 
-import { batchFunction, runInfinitePoll } from "socket-function/src/batching";
+import { batchFunction, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { nextId, timeInDay, timeInHour } from "socket-function/src/misc";
 import { getStorageDir, getStorageFolder } from "../../fs";
 import fs from "fs";
@@ -8,6 +8,8 @@ import { SizeLimiter } from "../SizeLimiter";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { isNode } from "typesafecss";
 import { logGitHashes } from "./logGitHashes";
+import { Zip } from "../../zip";
+import { formatNumber } from "socket-function/src/formatting/format";
 
 if (isNode()) {
     // Delayed setup, as we depend on diskLogger early, and we don't want to force high level
@@ -105,6 +107,7 @@ export type LogFile = {
 export async function getLogFiles(): Promise<LogFile[]> {
     let files = await fs.promises.readdir(folder);
     let paths = files.map(file => folder + file);
+    paths = paths.filter(x => x.endsWith(".log") || x.endsWith(".zip"));
     let objs = paths.map(decodeLogFileName);
     for (let obj of objs) {
         try {
@@ -214,7 +217,8 @@ function packageLogObj(args: unknown[]): LogObj {
 
 function decodeLogFileName(path: string): LogFile {
     let name = path.split("/").pop()!;
-    let [start, end] = name.split(".log")[0].split("-").map(Number);
+    let withoutExt = name.split(".").slice(0, -1).join(".");
+    let [start, end] = withoutExt.split("-").map(Number);
     return {
         startTime: start,
         endTime: end,
@@ -231,7 +235,16 @@ const logBase = batchFunction({ delay: 0 }, async function logBase(logObjList: L
 
     let byLogPath = new Map<string, LogObj[]>();
     for (let log of logs) {
-        let logFile = getLogFileName(log);
+        function createLogFileName(logObj: LogObj): LogFile {
+            let start = Math.floor(logObj.time / LOG_FILE_DURATION) * LOG_FILE_DURATION;
+            let startTime = start;
+            let endTime = start + LOG_FILE_DURATION;
+            let name = startTime + "-" + endTime + ".log";
+            let path = folder + name;
+            let logFile: LogFile = { startTime, endTime, name, path, size: 0, };
+            return logFile;
+        }
+        let logFile = createLogFileName(log);
         let list = byLogPath.get(logFile.path);
         if (!list) {
             list = [];
@@ -268,18 +281,9 @@ const logBase = batchFunction({ delay: 0 }, async function logBase(logObjList: L
     }
 });
 
-function getLogFileName(logObj: LogObj): LogFile {
-    let start = Math.floor(logObj.time / LOG_FILE_DURATION) * LOG_FILE_DURATION;
-    let startTime = start;
-    let endTime = start + LOG_FILE_DURATION;
-    let name = startTime + "-" + endTime + ".log";
-    let path = folder + name;
-    let logFile: LogFile = { startTime, endTime, name, path, size: 0, };
-    return logFile;
-}
 
 if (isNode()) {
-    runInfinitePoll(timeInHour, async function logClearOldLogs() {
+    runInfinitePoll(timeInHour, async function compressLogs() {
         // Maintain our size restrictions
         let logFiles = await fs.promises.readdir(folder);
         let objs: { time: number; bytes: number; path: string; }[] = [];
@@ -293,6 +297,33 @@ if (isNode()) {
             await fs.promises.unlink(file.path);
         }
     });
+    // Wait a random time, so we hopefully don't synchronize with any other services on this machine
+    runInfinitePoll(timeInHour + (1 + Math.random()), async function compressOldLogs() {
+        let logFiles = await fs.promises.readdir(folder);
+        let compressTime = Date.now() - LOG_FILE_DURATION * 2;
+        let filesCompressed = 0;
+        for (let file of logFiles) {
+            if (!file.endsWith(".log")) continue;
+            if (filesCompressed === 0) {
+                console.log("Compressing old logs");
+            }
+            filesCompressed++;
+            let path = folder + file;
+            if (decodeLogFileName(path).endTime > compressTime) continue;
+            let basePath = path.split(".").slice(0, -1).join(".");
+            let buffer = await fs.promises.readFile(path);
+            let beforeSize = buffer.length;
+            buffer = await Zip.gzip(buffer);
+            console.log(`Compressed ${file} from ${formatNumber(beforeSize)}B to ${formatNumber(buffer.length)}B`);
+            let tempPath = basePath + Math.random() + ".temp";
+            await fs.promises.writeFile(tempPath, buffer);
+            await fs.promises.rename(tempPath, basePath + ".zip");
+            await fs.promises.unlink(path);
+        }
+        if (filesCompressed > 0) {
+            console.log(`Compressed ${filesCompressed} old log files`);
+        }
+    });
 }
 
 /*

package/src/diagnostics/periodic.ts

@@ -1,5 +1,5 @@
 import { runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { isNode } from "socket-function/src/misc";
+import { isNode, timeInMinute } from "socket-function/src/misc";
 import { logErrors } from "../errors";
 import debugbreak from "debugbreak";
 import { nodeDiscoveryShutdown } from "../-f-node-discovery/NodeDiscovery";
@@ -22,7 +22,7 @@ function logAll() {
     }
 }
 
-logErrors(runInfinitePollCallAtStart(1000 * 60, logAll));
+logErrors(runInfinitePollCallAtStart(timeInMinute * 5, logAll));
 
 async function shutdown() {
     const { authorityStorage } = await import("../0-path-value-core/pathValueCore");

package/src/diagnostics/trackResources.ts

@@ -64,6 +64,10 @@ function getBufferUsage() {
     }
 }
 
+registerDynamicResource("Heap", getUsedHeapSize);
+registerDynamicResource("Buffers", getBufferUsage);
+registerDynamicResource("All Memory", getHeapSize);
+
 function logResourcesNow() {
     let resourcesWithCounts = resources.map(resource => ({ ...resource, count: resource.getCount() }));
     resourcesWithCounts.sort((a, b) => b.count - a.count);
@@ -74,9 +78,6 @@ function logResourcesNow() {
             console.log(`  ${resource.name}: ${formatNumber(resource.count)}`);
         }
     } else {
-        logNodeStateStats("Heap", formatNumber)(getUsedHeapSize());
-        logNodeStateStats("Buffers", formatNumber)(getBufferUsage());
-        logNodeStateStats("All Memory", formatNumber)(getHeapSize());
         for (let resource of resourcesWithCounts) {
             if (resource.count === 0) continue;
             logNodeStateStats(resource.name, formatNumber)(resource.count);

package/src/diagnostics/watchdog.ts

@@ -7,20 +7,30 @@ import { registerPeriodic } from "./periodic";
 import { getOwnMachineId } from "../-a-auth/certs";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { logDisk } from "./logs/diskLogger";
-import { registerNodeMetadata } from "../5-diagnostics/nodeMetadata";
-import { formatPercent } from "socket-function/src/formatting/format";
+import { addStatPeriodic, addStatSumPeriodic, addTimeProfileDistribution, registerNodeMetadata } from "../5-diagnostics/nodeMetadata";
+import { formatPercent, formatTime } from "socket-function/src/formatting/format";
 
 let lastProfile: MeasureProfile | undefined;
 
+addStatPeriodic(
+    "% Profiled",
+    () => {
+        if (!lastProfile) return 0;
+        let entries = Object.values(lastProfile.entries);
+        if (entries.length === 0) return 0;
+        const timeProfiled = lastProfile.endTime - lastProfile.startTime;
+        let timeActive = entries.map(x => x.ownTime.sum).reduce((a, b) => a + b, 0);
+        return timeActive / timeProfiled;
+    },
+    formatPercent,
+);
 registerNodeMetadata({
-    columnName: "Est Recent % Usage",
+    columnName: "Top Profiled",
     getValue() {
         if (!lastProfile) return "";
         let entries = Object.values(lastProfile.entries);
         if (entries.length === 0) return "";
         const timeProfiled = lastProfile.endTime - lastProfile.startTime;
-        let timeActive = entries.map(x => x.ownTime.sum).reduce((a, b) => a + b, 0);
-        let frac = timeActive / timeProfiled;
         let rootSums = new Map<string, number>();
         for (let entry of entries) {
             let name = entry.name.split("|")[0];
@@ -28,9 +38,13 @@ registerNodeMetadata({
             rootSums.set(name, sum + entry.ownTime.sum);
         }
         let most = sort(Array.from(rootSums.entries()), x => -x[1])[0];
-        return `${formatPercent(frac)} (${formatPercent(most[1] / timeProfiled)} ${endEllipsis(most[0], 26)})`;
+        return `${endEllipsis(most[0], 26)} = ${formatPercent(most[1] / timeProfiled)}`;
     },
 });
+addTimeProfileDistribution("Network Calls", () => SocketFunction.harvestCallTimes());
+addStatPeriodic("Pending Net Calls", () => SocketFunction.getPendingCallCount());
+addStatSumPeriodic("Failed Net Calls", () => SocketFunction.harvestFailedCallCount());
+
 function endEllipsis(str: string, maxLength: number) {
     if (str.length <= maxLength) return str;
     return "..." + str.slice(-(maxLength - 3));

package/src/library-components/Input.tsx

@@ -153,7 +153,7 @@ export class Input extends qreact.Component<InputProps> {
                     this.lastChecked = e.currentTarget.checked;
                 }
                 let callback = props.onInput || props.onChange;
-                callback?.(e);
+                callback?.(e as any);
             },
             onChange: e => {
                 if (

package/src/zip.ts

@@ -61,6 +61,15 @@ export class Zip {
         }
     }
 
+    @measureFnc
+    public static gunzipSync(buffer: Buffer): Buffer {
+        if (isNode()) {
+            return Buffer.from(zlib.gunzipSync(buffer));
+        } else {
+            return Buffer.from(pako.inflate(buffer));
+        }
+    }
+
     @measureFnc
     public static async gunzipBatch(buffers: Buffer[]): Promise<Buffer[]> {
         let time = Date.now();