querysub 0.356.0 → 0.358.0

package/.cursorrules

@@ -4,6 +4,12 @@ Always directly set the state (this.state.x = 1), and never use setState.
 
 Unless schema values use atomic or type they will always be returned as a value (as a proxy). Use "in" to check if a value is in a t.lookup.
 
+Use double quotes.
+
+When running a command in the current project, don't "cd" to it. It's redundant, and won't work.
+
+Don't use redundant comments. If it's a single line and the function name says the same thing that the comment is going to say, you don't need the comment.
+
 NEVER EVER pass state to qreact.Component as a template parameter. It should ALSO be declared like so (inside the class):
     state = t.state({
         num: t.number,
@@ -20,9 +26,12 @@ NEVER EVER pass state to qreact.Component as a template parameter. It should ALS
 
     t.lookup objects cannot be set with `this.state.someLookup = {}`. You can add keys or remove keys, and keys are added implicitly (`this.state.someLookup["someKey"] = { x: 1, y: 1 }`, always works, the key is automatically added if it doesn't exist). Removal uses delete `delete this.state.someLookup["someKey"]`.
 
+Follow the rule of minimum scoping. If something can be a local variable, it should be a local variable, it shouldn't be a class variable. If it can be a class variable, it should be a class variable, not a state variable. Etc, etc. Keep it simple, reference things closer, and write less code.
 
 Try not to use "null", and instead always use "undefined".
 
+Never try to add dynamic pluralization in the UI, just use an s. If you add dynamic pluralization, if the code ever gets localized, all of your changes have to be undone, and you just made localization much harder.
+
 Never use the ternary operator. Instead, do this: "x ? y : z" => "x && y || z".
 
 If are inside an async Event Handlers, you need to use... Querysub.onCommitFinished(() => ...). and put the async code inside the callback. Then when you set state, you need to put the state setting code inside of Querysub.commit(() => ...).

package/bin/movelogs.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/diagnostics/logs/IndexedLogs/moveLogsEntry");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.356.0",
+  "version": "0.358.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -12,11 +12,12 @@
     "servershardedtest": "yarn server --authority ./pathremain.json & yarn server --authority ./patha.json & yarn server --authority ./pathb.json & yarn server --authority ./pathc.json & yarn server --authority ./pathd.json",
     "type": "yarn tsc --noEmit",
     "depend": "yarn --silent depcruise src --include-only \"^src\" --config --output-type dot | dot -T svg > dependency-graph.svg",
-    "test": "yarn typenode ./test.ts",
-    "test3": "yarn typenode ./src/test/test.tsx --local",
-    "test2": "yarn typenode ./src/4-dom/qreactTest.tsx --local",
+    "t": "yarn typenode ./src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts",
+    "test-wat": "yarn typenode ./src/wat/watCompiler.ts",
     "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx",
-    "error-email": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx"
+    "error-email": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx",
+    "build-native": "cd src/diagnostics/logs/IndexedLogs && node-gyp rebuild",
+    "postinstall": "node scripts/postinstall.js"
   },
   "bin": {
     "deploy": "./bin/deploy.js",
@@ -32,12 +33,14 @@
     "gc-watch-public": "./bin/gc-watch-public.js",
     "join": "./bin/join.js",
     "join-public": "./bin/join-public.js",
+    "movelogs": "./bin/movelogs.js",
     "addsuperuser": "./bin/addsuperuser.js",
     "error-email": "./bin/error-email.js",
     "error-im": "./bin/error-im.js"
   },
   "dependencies": {
     "@types/fs-ext": "^2.0.3",
+    "@types/lz4": "^0.6.4",
     "@types/node-forge": "^1.3.1",
     "@types/pako": "^2.0.3",
     "@types/yargs": "^15.0.5",
@@ -48,9 +51,13 @@
     "fs-ext": "^2.0.0",
     "js-sha256": "https://github.com/sliftist/js-sha256",
     "js-sha512": "^0.9.0",
+    "lz4": "^0.6.5",
+    "lz4-wasm-nodejs": "^0.9.2",
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
-    "socket-function": "^1.0.4",
+    "peggy": "^5.0.6",
+    "querysub": "^0.357.0",
+    "socket-function": "^1.1.2",
     "terser": "^5.31.0",
     "typesafecss": "^0.28.0",
     "yaml": "^2.5.0",

package/scripts/postinstall.js

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+
+// Only run build-native if we're being installed as a dependency in someone else's project
+// Not when we're the root project adding new dependencies
+const path = require("path");
+const { execSync } = require("child_process");
+
+// Check if we're in node_modules (installed as a dependency)
+const isInstalledAsDependency = __dirname.includes("node_modules");
+
+if (isInstalledAsDependency) {
+    console.log("Running native build for querysub...");
+    try {
+        execSync("node-gyp rebuild", { 
+            cwd: path.join(__dirname, "..", "src", "diagnostics", "logs", "IndexedLogs"),
+            stdio: "inherit" 
+        });
+    } catch (err) {
+        console.log("Native build failed (this is okay, continuing...)");
+    }
+} else {
+    console.log("Skipping native build (running in development)");
+}

package/src/-a-archives/archiveCache.ts

@@ -1,5 +1,5 @@
 import { getStorageDir, getSubFolder } from "../fs";
-import { Archives } from "./archives";
+import { Archives, createArchivesOverride } from "./archives";
 import fs from "fs";
 
 import { list, nextId, timeInHour, timeInMinute } from "socket-function/src/misc";
@@ -492,9 +492,8 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
     function debugGetPath(key: string) {
         return getArchiveCachePath(archives, key);
     }
-    return {
+    return createArchivesOverride(archives, {
         [cacheArchivesSymbol]: true,
-        getDebugName: () => archives.getDebugName(),
         debugGetPath,
 
         get: async (fileName: string, config) => {
@@ -530,8 +529,13 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
             let metrics = await getDiskMetrics();
             const tempPath = cacheArchives2 + nextId() + TEMP_SUFFIX;
             await fs.promises.writeFile(tempPath, data);
-            await metrics.addCacheFile(archives, fileName, tempPath);
             await archives.set(fileName, data);
+            await metrics.addCacheFile(archives, fileName, tempPath);
+        },
+        append: async (fileName: string, data: Buffer) => {
+            let metrics = await getDiskMetrics();
+            await archives.append(fileName, data);
+            await metrics.delCacheFile(archives, fileName);
         },
         setLargeFile,
         del: async (fileName: string) => {
@@ -539,10 +543,6 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
             await metrics.delCacheFile(archives, fileName);
             await archives.del(fileName);
         },
-        getInfo: (fileName: string) => archives.getInfo(fileName),
-        find: async (prefix: string, config) => archives.find(prefix, config),
-        findInfo: async (prefix: string, config) => archives.findInfo(prefix, config),
-        enableLogging: () => archives.enableLogging(),
         move: async (config) => {
             // Delete the file from ourselves, even if it is a self move, as not having
             //  values is in the cache is the safest thing, but having extra values
@@ -556,11 +556,9 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
             }
             await archives.move(config);
         },
-        copy: config => archives.copy(config),
-        assertPathValid: (path: string) => archives.assertPathValid(path),
         getBaseArchives: () => archives.getBaseArchives?.() ?? ({ archives: archives, parentPath: "" }),
 
-        async lockRegion(code) {
+        async lockRegion<T>(code: (fncs: LockFncs) => Promise<T>) {
             let locked: string[] = [];
             let fncs = {
                 getPathAndLock: createGetPathAndLock(locked),
@@ -576,7 +574,7 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
                 }
             }
         },
-    };
+    });
 }
 
 

package/src/-a-archives/archives.ts

@@ -21,11 +21,14 @@ export interface Archives {
     set(path: string, data: Buffer): Promise<void>;
     del(path: string): Promise<void>;
 
+    append(path: string, data: Buffer): Promise<void>;
+
     getInfo(path: string): Promise<{ writeTime: number; size: number; } | undefined>;
 
     setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void>;
 
     // For example findFileNames("ips/")
+    /** Returns to full paths. Returns paths WILL NOT end with / */
     find(
         prefix: string,
         config?: {
@@ -76,6 +79,31 @@ export interface Archives {
     }>;
 }
 
+export function createArchivesOverride<T extends Partial<Archives>>(
+    baseArchives: Archives,
+    overrideArchives: T
+): T & Archives {
+    return {
+        getDebugName: () => baseArchives.getDebugName(),
+        get: (fileName: string, config) => baseArchives.get(fileName, config),
+        set: (fileName: string, data: Buffer) => baseArchives.set(fileName, data),
+        append: (fileName: string, data: Buffer) => baseArchives.append(fileName, data),
+        del: (fileName: string) => baseArchives.del(fileName),
+        getInfo: (fileName: string) => baseArchives.getInfo(fileName),
+        setLargeFile: (config) => baseArchives.setLargeFile(config),
+        find: async (prefix: string, config) => baseArchives.find(prefix, config),
+        findInfo: async (prefix: string, config) => baseArchives.findInfo(prefix, config),
+        enableLogging: () => baseArchives.enableLogging(),
+        move: (config) => baseArchives.move(config),
+        copy: (config) => baseArchives.copy(config),
+        assertPathValid: (path: string) => baseArchives.assertPathValid(path),
+        getBaseArchives: () => baseArchives.getBaseArchives?.() ?? ({ archives: baseArchives, parentPath: "" }),
+        getURL: baseArchives.getURL ? (filePath: string) => baseArchives.getURL!(filePath) : undefined,
+        getDownloadAuthorization: baseArchives.getDownloadAuthorization ? (config: { validDurationInSeconds: number }) => baseArchives.getDownloadAuthorization!(config) : undefined,
+        ...overrideArchives,
+    };
+}
+
 export function nestArchives(path: string, archives: Archives): Archives {
     if (!path) return archives;
     if (!path.endsWith("/")) {
@@ -92,6 +120,7 @@ export function nestArchives(path: string, archives: Archives): Archives {
         get: (fileName: string, config) => archives.get(path + stripFilePrefix(fileName), config),
         setLargeFile: (config) => archives.setLargeFile({ ...config, path: path + stripFilePrefix(config.path) }),
         set: (fileName: string, data: Buffer) => archives.set(path + stripFilePrefix(fileName), data),
+        append: (fileName: string, data: Buffer) => archives.append(path + stripFilePrefix(fileName), data),
         del: (fileName: string) => archives.del(path + stripFilePrefix(fileName)),
         getInfo: (fileName: string) => archives.getInfo(path + stripFilePrefix(fileName)),
         find: async (prefix: string, config) => archives.find(path + prefix, config).then(x => x.map(x => x.slice(path.length))),

package/src/-a-archives/archivesBackBlaze.ts

@@ -238,7 +238,7 @@ const getAPI = lazy(async () => {
         };
         action: string;
         uploadTimestamp: number;
-    }>("b2_get_file_info", "POST");
+    }>("b2_get_file_info", "POST", "noAccountId");
 
     const listFileNames = createB2Function<{
         bucketId: string;
@@ -594,10 +594,41 @@ export class ArchivesBackblaze {
             };
             setTimeout(downloadPoll, 5000);
             let result = await this.apiRetryLogic(async (api) => {
+                let range = config?.range;
+                if (range) {
+                    let fileInfo = await this.getInfo(fileName);
+                    if (!fileInfo) throw new Error(`File ${fileName} not found`);
+                    let rangeStart = range.start;
+                    let rangeEnd = Math.min(range.end, fileInfo.size);
+                    // NOTE: I think if we request nothing, it confuses Backblaze and ends up giving us the entire file.
+                    if (rangeEnd <= rangeStart) return Buffer.alloc(0);
+                    let result = await api.downloadFileByName({
+                        bucketName: this.bucketName,
+                        fileName,
+                        range: { start: rangeStart, end: rangeEnd },
+                    });
+                    if (result.length !== rangeEnd - rangeStart) {
+                        let afterLength = await this.getInfo(fileName);
+                        if (afterLength && afterLength.size >= fileInfo.size) {
+                            console.error(`Backblaze range download return the correct number of bytes. Tried to get ${rangeStart}-${rangeEnd}, but received ${rangeStart}-${rangeStart + result.length}. For file: ${fileName}`);
+                            // I'm not sure if it's a bug that where we get extra data if we try to read beyond the end of the file, or if the bug is due to some kind of lag that will resolve itself if we wait a little bit. 
+                            setTimeout(async () => {
+                                let resultAgain = await api.downloadFileByName({
+                                    bucketName: this.bucketName,
+                                    fileName,
+                                    range: { start: rangeStart, end: rangeEnd },
+                                });
+                                devDebugbreak();
+                                let didResultFixItSelf = resultAgain.length === rangeEnd - rangeStart;
+
+                                console.log({ didResultFixItSelf }, resultAgain);
+                            }, timeInMinute * 2);
+                        }
+                    }
+                }
                 return await api.downloadFileByName({
                     bucketName: this.bucketName,
                     fileName,
-                    range: config?.range
                 });
             });
             let timeStr = formatTime(Date.now() - time);
@@ -631,6 +662,14 @@ export class ArchivesBackblaze {
         }
 
     }
+    public async append(fileName: string, data: Buffer): Promise<void> {
+        throw new Error(`ArchivesBackblaze does not support append. Use set instead.`);
+        // this.log(`backblaze append (${formatNumber(data.length)}B) ${fileName}`);
+        // // Backblaze doesn't have native append, so we need to get, concatenate, and set
+        // let existing = await this.get(fileName);
+        // let newData = existing ? Buffer.concat([existing, data]) : data;
+        // await this.set(fileName, newData);
+    }
     public async del(fileName: string): Promise<void> {
         this.log(`backblaze delete ${fileName}`);
         try {
@@ -796,17 +835,26 @@ export class ArchivesBackblaze {
 
     public async getInfo(fileName: string): Promise<{ writeTime: number; size: number; } | undefined> {
         return await this.apiRetryLogic(async (api) => {
-            let info = await api.listFileNames({ bucketId: this.bucketId, prefix: fileName, });
-            let file = info.files.find(x => x.fileName === fileName);
-            if (!file) {
-                this.log(`Backblaze file not exists ${fileName}`);
-                return undefined;
+            try {
+                // NOTE: Apparently, there's no other way to do this, as the file name does not equal the file ID, and git file info requires the file ID. 
+                let info = await api.listFileNames({ bucketId: this.bucketId, prefix: fileName, maxFileCount: 1 });
+                let file = info.files.find(x => x.fileName === fileName && x.action === "upload");
+                if (!file) {
+                    this.log(`Backblaze file not exists ${fileName}`);
+                    return undefined;
+                }
+                this.log(`Backblaze file exists ${fileName}`);
+                return {
+                    writeTime: file.uploadTimestamp,
+                    size: file.contentLength,
+                };
+            } catch (e: any) {
+                if (e.stack.includes(`file_not_found`)) {
+                    this.log(`Backblaze file not exists ${fileName}`);
+                    return undefined;
+                }
+                throw e;
             }
-            this.log(`Backblaze file exists ${fileName}`);
-            return {
-                writeTime: file.uploadTimestamp,
-                size: file.contentLength,
-            };
         });
     }
 

package/src/-a-archives/archivesDisk.ts

@@ -65,13 +65,17 @@ class ArchivesDisk {
         // Don't create the drive (and also only add up to the last path, via slicing (0, i)
         for (let i = 1; i < fileNameParts.length; i++) {
             let dir = this.LOCAL_ARCHIVE_FOLDER + fileNameParts.slice(0, i).join("/");
-            if (!await fsExistsAsync(dir)) {
-                await fs.promises.mkdir(dir);
-            }
+            try {
+                if (!await fsExistsAsync(dir)) {
+                    await fs.promises.mkdir(dir, {
+                        recursive: true,
+                    });
+                }
+            } catch { }
         }
     }
 
-    @measureFnc
+    // We don't time the file system writes as it's almost never blocking, it's not stopping our processing. And if it is stopping our processing, then we actually want to know the parent function which is calling it, and then we want it to just void it instead of awaiting it (And the only way to know the parent caller is to not time these functions).
     public async set(fileName: string, data: Buffer): Promise<void> {
         await this.init();
 
@@ -84,7 +88,18 @@ class ArchivesDisk {
 
         await fs.promises.writeFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
     }
-    @measureFnc
+    public async append(fileName: string, data: Buffer): Promise<void> {
+        await this.init();
+
+        this.log(blue(`Appending to file ${fileName} += ${data.length} bytes`));
+        if (storageDisabled) return;
+        fileName = escapeFileName(fileName);
+        await this.simulateLag();
+
+        await this.ensureDirsExist(fileName);
+
+        await fs.promises.appendFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
+    }
     public async del(fileName: string): Promise<void> {
         await this.init();
         this.log(blue(`Deleting file ${fileName}`));
@@ -93,10 +108,18 @@ class ArchivesDisk {
         await this.simulateLag();
         if (!await fsExistsAsync(this.LOCAL_ARCHIVE_FOLDER + fileName)) return;
         try {
-            await fs.promises.unlink(this.LOCAL_ARCHIVE_FOLDER + fileName);
+            await fs.promises.rm(this.LOCAL_ARCHIVE_FOLDER + fileName, { recursive: true });
+        } catch { }
+        try {
+            let dir = fileName;
+            while (dir.length > 0) {
+                dir = dir.replaceAll("\\", "/").split("/").slice(0, -1).join("/");
+                if (dir.endsWith(":")) break;
+                let files = await fs.promises.readdir(this.LOCAL_ARCHIVE_FOLDER + dir);
+                if (files.length > 0) break;
+                await fs.promises.rmdir(this.LOCAL_ARCHIVE_FOLDER + dir);
+            }
         } catch { }
-        let dir = fileName.replaceAll("\\", "/").split("/").slice(0, -1).join("/");
-        await this.gcDir(dir);
     }
 
     public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
@@ -163,7 +186,8 @@ class ArchivesDisk {
      *      it all at once).
      * @fsErrorRetryCount Retries count for file system error (not user errors). Ex, too many open files, etc.
      */
-    @measureFnc
+    // NOTE: I've commented out the gate measuring so we can determine what is taking the time. We might want to remove all the timing from this file, as if we find any timing this file is slow, it doesn't help us optimize at all, because we then need to figure out what is calling it. 
+    //@measureFnc
     public async get(fileNameInput: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
         await this.init();
         this.log(blue(`Start read file ${fileNameInput}`));
@@ -191,7 +215,8 @@ class ArchivesDisk {
                 throw new Error(`File modified changed while reading file ${fileName}`);
             }
 
-            return buffer.slice(0, read.bytesRead);
+            // Make sure we only take as much data as we want in case we are given extra data. 
+            return buffer.slice(0, end - start);
         } catch (e: any) {
             if (e.code === "EMFILE") {
                 if (retryCount > 0) {
@@ -221,7 +246,7 @@ class ArchivesDisk {
         await this.simulateLag();
         let fileNames: string[] = [];
         let folderNames: string[] = [];
-        async function readDir(dir: string) {
+        async function readDir(dir: string, depthToRead: number) {
             if (!await fsExistsAsync(dir)) {
                 return;
             }
@@ -236,7 +261,7 @@ class ArchivesDisk {
                     try {
                         if (fileObj.isDirectory()) {
                             folderNames.push(dir + fileName);
-                            await readDir(dir + fileName + "/");
+                            await readDir(dir + fileName + "/", depthToRead - 1);
                         } else {
                             fileNames.push(dir + fileName);
                         }
@@ -250,7 +275,8 @@ class ArchivesDisk {
         if (pathParts.length > 1) {
             rootDir = pathParts.slice(0, -1).join("/") + "/";
         }
-        await readDir(this.LOCAL_ARCHIVE_FOLDER + rootDir);
+        // We don't know if the prefix they gave us is the prefix of folders or if it is a folder itself. So we have to cut off the end and then read to a depth of 2 to make this work.
+        await readDir(this.LOCAL_ARCHIVE_FOLDER + rootDir, config?.shallow ? 2 : Number.MAX_SAFE_INTEGER);
 
         let results = config?.type === "folders" ? folderNames : fileNames;
 

package/src/-a-archives/archivesLimitedCache.ts

@@ -153,6 +153,27 @@ class ArchivesLimitedCache {
         this.triggerIndexFlush();
     }
 
+    public async append(path: string, data: Buffer): Promise<void> {
+        await this.ensureInitialized();
+        await this.baseArchives.append(path, data);
+        // Update the cache info - increment size and update times
+        const info = this.cache.get(path);
+        const now = Date.now();
+        if (info) {
+            info.size += data.length;
+            info.writeTime = now;
+            info.accessTime = now;
+        } else {
+            // File didn't exist in cache, add it
+            this.cache.set(path, {
+                writeTime: now,
+                accessTime: now,
+                size: data.length,
+            });
+        }
+        this.triggerIndexFlush();
+    }
+
     public async del(path: string): Promise<void> {
         await this.ensureInitialized();
         await this.baseArchives.del(path);