querysub 0.41.0 → 0.43.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.41.0",
+    "version": "0.43.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/diagnostics/logs/BrowserLargeFileCache.ts

@@ -0,0 +1,64 @@
+import { SocketFunctionClientHook } from "socket-function/SocketFunctionTypes";
+import { cache, lazy } from "socket-function/src/caching";
+import { decodeCborx, encodeCborx } from "../../misc/cloneHelpers";
+import { sha256 } from "js-sha256";
+import { errorToUndefined } from "../../errors";
+import { isNode } from "typesafecss";
+
+let getRootDirectory = lazy(async () => {
+    await navigator.storage.persist();
+    return navigator.storage.getDirectory();
+});
+
+export const getBrowserLargeFileCache = cache((name: string) => new BrowserLargeFileCache(name));
+
+export class BrowserLargeFileCache {
+    constructor(private name: string) { }
+
+    private getDir = lazy(async () => {
+        let root = await getRootDirectory();
+        return await root.getDirectoryHandle(this.name, { create: true });
+    });
+
+    public async set(key: string, value: Buffer) {
+        let dir = await this.getDir();
+        let file = await dir.getFileHandle(key, { create: true });
+        let writable = await file.createWritable();
+        await writable.write(value);
+        await writable.close();
+    }
+    public async get(key: string): Promise<Buffer | undefined> {
+        let dir = await this.getDir();
+        try {
+            let file = await dir.getFileHandle(key, { create: false });
+            let readable = await file.getFile();
+            return Buffer.from(await readable.arrayBuffer());
+        } catch {
+            return undefined;
+        }
+    }
+}
+
+/** Cache key = [args, functionName, classGuid]
+ *      - Not nodeId, as that change so frequently, so caching based on server is not usually useful.
+ *          - If you want to cache based on nodeId, pass it as an unused arg.
+ * 
+ *  Uses a file per key, and never cleans them up. So... basically, this sucks, but, it should
+ *      work for a few specific functions with large and consistent values.
+ */
+export const cacheCalls: SocketFunctionClientHook = async config => {
+    if (isNode()) return;
+
+    let { args, functionName, classGuid } = config.call;
+    let bucket = sha256(JSON.stringify({ functionName, classGuid }));
+    let cache = getBrowserLargeFileCache(bucket);
+    let key = sha256(encodeCborx(args));
+    let cachedValue = await cache.get(key);
+    if (cachedValue) {
+        config.overrideResult = decodeCborx(cachedValue);
+        return;
+    }
+    config.onResult.push(async (result) => {
+        await errorToUndefined(cache.set(key, encodeCborx(result)));
+    });
+};

package/src/diagnostics/logs/DiskLoggerPage.tsx

@@ -10,7 +10,7 @@ import { getSyncedController } from "../../library-components/SyncedController";
 import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
 import { formatDateTime, formatNiceDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { Anchor } from "../../library-components/ATag";
-import { cache, cacheLimited, cacheShallowConfigArgEqual } from "socket-function/src/caching";
+import { cache, cacheLimited, cacheShallowConfigArgEqual, lazy } from "socket-function/src/caching";
 import { InputLabel, InputLabelURL } from "../../library-components/InputLabel";
 import { DropdownSelector } from "../../library-components/DropdownSelector";
 import { Table, TableType } from "../../5-diagnostics/Table";
@@ -30,6 +30,8 @@ import { getNodeIdLocation } from "socket-function/src/nodeCache";
 import { decodeNodeId, encodeNodeId, getMachineId } from "../../-a-auth/certs";
 import { Button } from "../../library-components/Button";
 import { TimeRangeSelector } from "../../library-components/TimeRangeSelector";
+import { BrowserLargeFileCache, cacheCalls } from "./BrowserLargeFileCache";
+import { Zip } from "../../zip";
 
 // TODO: Realtime log mode, by reading from the previous length forward, to add buffers
 //  to what we already read.
@@ -140,6 +142,7 @@ export class DiskLoggerPage extends qreact.Component {
                 if (file.endTime > lastTime) lastTime = file.endTime;
                 let buffer = controller.getRemoteLogBuffer(file.nodeId, file.path);
                 if (buffer) {
+                    buffer = Zip.gunzipSync(buffer);
                     buffers.push(buffer);
                 } else {
                     loadingCount++;
@@ -541,7 +544,14 @@ class DiskLoggerControllerBase {
         return await getLogFiles();
     }
     public async getLogBuffer(path: string): Promise<Buffer | undefined> {
-        return await getLogBuffer(path);
+        // Always compress it. If it is in progress it can't be compressed on disk, but
+        //  the compression ratio is so high it's worth it to compress it dynamically now,
+        //  to speed up the network transfer.
+        let buffer = await getLogBuffer(path);
+        if (buffer && path.endsWith(".log")) {
+            buffer = await Zip.gzip(buffer);
+        }
+        return buffer;
     }
 }
 
@@ -550,7 +560,9 @@ export const DiskLoggerController = SocketFunction.register(
     new DiskLoggerControllerBase(),
     () => ({
         getRemoteLogFiles: {},
-        getRemoteLogBuffer: {},
+        getRemoteLogBuffer: {
+            clientHooks: [cacheCalls]
+        },
 
         getLogFiles: {},
         getLogBuffer: {},

package/src/diagnostics/logs/diskLogger.ts

@@ -1,5 +1,5 @@
 
-import { batchFunction, runInfinitePoll } from "socket-function/src/batching";
+import { batchFunction, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { nextId, timeInDay, timeInHour } from "socket-function/src/misc";
 import { getStorageDir, getStorageFolder } from "../../fs";
 import fs from "fs";
@@ -8,6 +8,7 @@ import { SizeLimiter } from "../SizeLimiter";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { isNode } from "typesafecss";
 import { logGitHashes } from "./logGitHashes";
+import { Zip } from "../../zip";
 
 if (isNode()) {
     // Delayed setup, as we depend on diskLogger early, and we don't want to force high level
@@ -105,6 +106,7 @@ export type LogFile = {
 export async function getLogFiles(): Promise<LogFile[]> {
     let files = await fs.promises.readdir(folder);
     let paths = files.map(file => folder + file);
+    paths = paths.filter(x => x.endsWith(".log") || x.endsWith(".zip"));
     let objs = paths.map(decodeLogFileName);
     for (let obj of objs) {
         try {
@@ -214,7 +216,8 @@ function packageLogObj(args: unknown[]): LogObj {
 
 function decodeLogFileName(path: string): LogFile {
     let name = path.split("/").pop()!;
-    let [start, end] = name.split(".log")[0].split("-").map(Number);
+    let withoutExt = name.split(".").slice(0, -1).join(".");
+    let [start, end] = withoutExt.split("-").map(Number);
     return {
         startTime: start,
         endTime: end,
@@ -231,7 +234,16 @@ const logBase = batchFunction({ delay: 0 }, async function logBase(logObjList: L
 
     let byLogPath = new Map<string, LogObj[]>();
     for (let log of logs) {
-        let logFile = getLogFileName(log);
+        function createLogFileName(logObj: LogObj): LogFile {
+            let start = Math.floor(logObj.time / LOG_FILE_DURATION) * LOG_FILE_DURATION;
+            let startTime = start;
+            let endTime = start + LOG_FILE_DURATION;
+            let name = startTime + "-" + endTime + ".log";
+            let path = folder + name;
+            let logFile: LogFile = { startTime, endTime, name, path, size: 0, };
+            return logFile;
+        }
+        let logFile = createLogFileName(log);
         let list = byLogPath.get(logFile.path);
         if (!list) {
             list = [];
@@ -268,18 +280,9 @@ const logBase = batchFunction({ delay: 0 }, async function logBase(logObjList: L
     }
 });
 
-function getLogFileName(logObj: LogObj): LogFile {
-    let start = Math.floor(logObj.time / LOG_FILE_DURATION) * LOG_FILE_DURATION;
-    let startTime = start;
-    let endTime = start + LOG_FILE_DURATION;
-    let name = startTime + "-" + endTime + ".log";
-    let path = folder + name;
-    let logFile: LogFile = { startTime, endTime, name, path, size: 0, };
-    return logFile;
-}
 
 if (isNode()) {
-    runInfinitePoll(timeInHour, async function logClearOldLogs() {
+    runInfinitePoll(timeInHour, async function compressLogs() {
         // Maintain our size restrictions
         let logFiles = await fs.promises.readdir(folder);
         let objs: { time: number; bytes: number; path: string; }[] = [];
@@ -293,6 +296,23 @@ if (isNode()) {
             await fs.promises.unlink(file.path);
         }
     });
+    // Wait a random time, so we hopefully don't synchronize with any other services on this machine
+    void runInfinitePollCallAtStart(timeInHour + (1 + Math.random()), async function compressOldLogs() {
+        let logFiles = await fs.promises.readdir(folder);
+        let compressTime = Date.now() - LOG_FILE_DURATION * 2;
+        for (let file of logFiles) {
+            if (!file.endsWith(".log")) continue;
+            let path = folder + file;
+            if (decodeLogFileName(path).endTime > compressTime) continue;
+            let basePath = path.split(".").slice(0, -1).join(".");
+            let buffer = await fs.promises.readFile(path);
+            buffer = await Zip.gzip(buffer);
+            let tempPath = basePath + Math.random() + ".temp";
+            await fs.promises.writeFile(tempPath, buffer);
+            await fs.promises.rename(tempPath, basePath + ".zip");
+            await fs.promises.unlink(path);
+        }
+    });
 }
 
 /*

package/src/zip.ts

@@ -61,6 +61,14 @@ export class Zip {
         }
     }
 
+    public static gunzipSync(buffer: Buffer): Buffer {
+        if (isNode()) {
+            return Buffer.from(zlib.gunzipSync(buffer));
+        } else {
+            return Buffer.from(pako.inflate(buffer));
+        }
+    }
+
     @measureFnc
     public static async gunzipBatch(buffers: Buffer[]): Promise<Buffer[]> {
         let time = Date.now();