querysub 0.42.0 → 0.43.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.42.0",
+    "version": "0.43.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/diagnostics/logs/DiskLoggerPage.tsx

@@ -31,6 +31,7 @@ import { decodeNodeId, encodeNodeId, getMachineId } from "../../-a-auth/certs";
 import { Button } from "../../library-components/Button";
 import { TimeRangeSelector } from "../../library-components/TimeRangeSelector";
 import { BrowserLargeFileCache, cacheCalls } from "./BrowserLargeFileCache";
+import { Zip } from "../../zip";
 
 // TODO: Realtime log mode, by reading from the previous length forward, to add buffers
 //  to what we already read.
@@ -141,6 +142,7 @@ export class DiskLoggerPage extends qreact.Component {
                 if (file.endTime > lastTime) lastTime = file.endTime;
                 let buffer = controller.getRemoteLogBuffer(file.nodeId, file.path);
                 if (buffer) {
+                    buffer = Zip.gunzipSync(buffer);
                     buffers.push(buffer);
                 } else {
                     loadingCount++;
@@ -542,7 +544,14 @@ class DiskLoggerControllerBase {
         return await getLogFiles();
     }
     public async getLogBuffer(path: string): Promise<Buffer | undefined> {
-        return await getLogBuffer(path);
+        // Always compress it. If it is in progress it can't be compressed on disk, but
+        //  the compression ratio is so high it's worth it to compress it dynamically now,
+        //  to speed up the network transfer.
+        let buffer = await getLogBuffer(path);
+        if (buffer && path.endsWith(".log")) {
+            buffer = await Zip.gzip(buffer);
+        }
+        return buffer;
     }
 }
 
@@ -552,14 +561,11 @@ export const DiskLoggerController = SocketFunction.register(
     () => ({
         getRemoteLogFiles: {},
         getRemoteLogBuffer: {
-            compress: true,
             clientHooks: [cacheCalls]
         },
 
         getLogFiles: {},
-        getLogBuffer: {
-            compress: true,
-        },
+        getLogBuffer: {},
     }),
     () => ({
         hooks: [assertIsManagementUser],

package/src/diagnostics/logs/diskLogger.ts

@@ -1,5 +1,5 @@
 
-import { batchFunction, runInfinitePoll } from "socket-function/src/batching";
+import { batchFunction, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { nextId, timeInDay, timeInHour } from "socket-function/src/misc";
 import { getStorageDir, getStorageFolder } from "../../fs";
 import fs from "fs";
@@ -8,6 +8,7 @@ import { SizeLimiter } from "../SizeLimiter";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { isNode } from "typesafecss";
 import { logGitHashes } from "./logGitHashes";
+import { Zip } from "../../zip";
 
 if (isNode()) {
     // Delayed setup, as we depend on diskLogger early, and we don't want to force high level
@@ -105,6 +106,7 @@ export type LogFile = {
 export async function getLogFiles(): Promise<LogFile[]> {
     let files = await fs.promises.readdir(folder);
     let paths = files.map(file => folder + file);
+    paths = paths.filter(x => x.endsWith(".log") || x.endsWith(".zip"));
     let objs = paths.map(decodeLogFileName);
     for (let obj of objs) {
         try {
@@ -214,7 +216,8 @@ function packageLogObj(args: unknown[]): LogObj {
 
 function decodeLogFileName(path: string): LogFile {
     let name = path.split("/").pop()!;
-    let [start, end] = name.split(".log")[0].split("-").map(Number);
+    let withoutExt = name.split(".").slice(0, -1).join(".");
+    let [start, end] = withoutExt.split("-").map(Number);
     return {
         startTime: start,
         endTime: end,
@@ -231,7 +234,16 @@ const logBase = batchFunction({ delay: 0 }, async function logBase(logObjList: L
 
     let byLogPath = new Map<string, LogObj[]>();
     for (let log of logs) {
-        let logFile = getLogFileName(log);
+        function createLogFileName(logObj: LogObj): LogFile {
+            let start = Math.floor(logObj.time / LOG_FILE_DURATION) * LOG_FILE_DURATION;
+            let startTime = start;
+            let endTime = start + LOG_FILE_DURATION;
+            let name = startTime + "-" + endTime + ".log";
+            let path = folder + name;
+            let logFile: LogFile = { startTime, endTime, name, path, size: 0, };
+            return logFile;
+        }
+        let logFile = createLogFileName(log);
         let list = byLogPath.get(logFile.path);
         if (!list) {
             list = [];
@@ -268,18 +280,9 @@ const logBase = batchFunction({ delay: 0 }, async function logBase(logObjList: L
     }
 });
 
-function getLogFileName(logObj: LogObj): LogFile {
-    let start = Math.floor(logObj.time / LOG_FILE_DURATION) * LOG_FILE_DURATION;
-    let startTime = start;
-    let endTime = start + LOG_FILE_DURATION;
-    let name = startTime + "-" + endTime + ".log";
-    let path = folder + name;
-    let logFile: LogFile = { startTime, endTime, name, path, size: 0, };
-    return logFile;
-}
 
 if (isNode()) {
-    runInfinitePoll(timeInHour, async function logClearOldLogs() {
+    runInfinitePoll(timeInHour, async function compressLogs() {
         // Maintain our size restrictions
         let logFiles = await fs.promises.readdir(folder);
         let objs: { time: number; bytes: number; path: string; }[] = [];
@@ -293,6 +296,23 @@ if (isNode()) {
             await fs.promises.unlink(file.path);
         }
     });
+    // Wait a random time, so we hopefully don't synchronize with any other services on this machine
+    void runInfinitePollCallAtStart(timeInHour + (1 + Math.random()), async function compressOldLogs() {
+        let logFiles = await fs.promises.readdir(folder);
+        let compressTime = Date.now() - LOG_FILE_DURATION * 2;
+        for (let file of logFiles) {
+            if (!file.endsWith(".log")) continue;
+            let path = folder + file;
+            if (decodeLogFileName(path).endTime > compressTime) continue;
+            let basePath = path.split(".").slice(0, -1).join(".");
+            let buffer = await fs.promises.readFile(path);
+            buffer = await Zip.gzip(buffer);
+            let tempPath = basePath + Math.random() + ".temp";
+            await fs.promises.writeFile(tempPath, buffer);
+            await fs.promises.rename(tempPath, basePath + ".zip");
+            await fs.promises.unlink(path);
+        }
+    });
 }
 
 /*

package/src/zip.ts

@@ -61,6 +61,14 @@ export class Zip {
         }
     }
 
+    public static gunzipSync(buffer: Buffer): Buffer {
+        if (isNode()) {
+            return Buffer.from(zlib.gunzipSync(buffer));
+        } else {
+            return Buffer.from(pako.inflate(buffer));
+        }
+    }
+
     @measureFnc
     public static async gunzipBatch(buffers: Buffer[]): Promise<Buffer[]> {
         let time = Date.now();