querysub 0.356.0 → 0.357.0

package/.cursorrules

@@ -4,6 +4,12 @@ Always directly set the state (this.state.x = 1), and never use setState.
 
 Unless schema values use atomic or type they will always be returned as a value (as a proxy). Use "in" to check if a value is in a t.lookup.
 
+Use double quotes.
+
+When running a command in the current project, don't "cd" to it. It's redundant, and won't work.
+
+Don't use redundant comments. If it's a single line and the function name says the same thing that the comment is going to say, you don't need the comment.
+
 NEVER EVER pass state to qreact.Component as a template parameter. It should ALSO be declared like so (inside the class):
     state = t.state({
         num: t.number,
@@ -23,6 +29,8 @@ NEVER EVER pass state to qreact.Component as a template parameter. It should ALS
 
 Try not to use "null", and instead always use "undefined".
 
+Never try to add dynamic pluralization in the UI, just use an s. If you add dynamic pluralization, if the code ever gets localized, all of your changes have to be undone, and you just made localization much harder.
+
 Never use the ternary operator. Instead, do this: "x ? y : z" => "x && y || z".
 
 If are inside an async Event Handlers, you need to use... Querysub.onCommitFinished(() => ...). and put the async code inside the callback. Then when you set state, you need to put the state setting code inside of Querysub.commit(() => ...).

package/bin/movelogs.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/diagnostics/logs/IndexedLogs/moveLogsEntry");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.356.0",
+  "version": "0.357.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -12,11 +12,12 @@
     "servershardedtest": "yarn server --authority ./pathremain.json & yarn server --authority ./patha.json & yarn server --authority ./pathb.json & yarn server --authority ./pathc.json & yarn server --authority ./pathd.json",
     "type": "yarn tsc --noEmit",
     "depend": "yarn --silent depcruise src --include-only \"^src\" --config --output-type dot | dot -T svg > dependency-graph.svg",
-    "test": "yarn typenode ./test.ts",
-    "test3": "yarn typenode ./src/test/test.tsx --local",
-    "test2": "yarn typenode ./src/4-dom/qreactTest.tsx --local",
+    "t": "yarn typenode ./src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts",
+    "test-wat": "yarn typenode ./src/wat/watCompiler.ts",
     "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx",
-    "error-email": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx"
+    "error-email": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx",
+    "build-native": "cd src/diagnostics/logs/IndexedLogs && node-gyp rebuild",
+    "postinstall": "node scripts/postinstall.js"
   },
   "bin": {
     "deploy": "./bin/deploy.js",
@@ -32,12 +33,14 @@
     "gc-watch-public": "./bin/gc-watch-public.js",
     "join": "./bin/join.js",
     "join-public": "./bin/join-public.js",
+    "movelogs": "./bin/movelogs.js",
     "addsuperuser": "./bin/addsuperuser.js",
     "error-email": "./bin/error-email.js",
     "error-im": "./bin/error-im.js"
   },
   "dependencies": {
     "@types/fs-ext": "^2.0.3",
+    "@types/lz4": "^0.6.4",
     "@types/node-forge": "^1.3.1",
     "@types/pako": "^2.0.3",
     "@types/yargs": "^15.0.5",
@@ -48,9 +51,12 @@
     "fs-ext": "^2.0.0",
     "js-sha256": "https://github.com/sliftist/js-sha256",
     "js-sha512": "^0.9.0",
+    "lz4": "^0.6.5",
+    "lz4-wasm-nodejs": "^0.9.2",
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
-    "socket-function": "^1.0.4",
+    "peggy": "^5.0.6",
+    "socket-function": "^1.1.2",
     "terser": "^5.31.0",
     "typesafecss": "^0.28.0",
     "yaml": "^2.5.0",

package/scripts/postinstall.js

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+
+// Only run build-native if we're being installed as a dependency in someone else's project
+// Not when we're the root project adding new dependencies
+const path = require("path");
+const { execSync } = require("child_process");
+
+// Check if we're in node_modules (installed as a dependency)
+const isInstalledAsDependency = __dirname.includes("node_modules");
+
+if (isInstalledAsDependency) {
+    console.log("Running native build for querysub...");
+    try {
+        execSync("node-gyp rebuild", { 
+            cwd: path.join(__dirname, "..", "src", "diagnostics", "logs", "IndexedLogs"),
+            stdio: "inherit" 
+        });
+    } catch (err) {
+        console.log("Native build failed (this is okay, continuing...)");
+    }
+} else {
+    console.log("Skipping native build (running in development)");
+}

package/src/-a-archives/archiveCache.ts

@@ -1,5 +1,5 @@
 import { getStorageDir, getSubFolder } from "../fs";
-import { Archives } from "./archives";
+import { Archives, createArchivesOverride } from "./archives";
 import fs from "fs";
 
 import { list, nextId, timeInHour, timeInMinute } from "socket-function/src/misc";
@@ -492,9 +492,8 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
     function debugGetPath(key: string) {
         return getArchiveCachePath(archives, key);
     }
-    return {
+    return createArchivesOverride(archives, {
         [cacheArchivesSymbol]: true,
-        getDebugName: () => archives.getDebugName(),
         debugGetPath,
 
         get: async (fileName: string, config) => {
@@ -530,8 +529,13 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
             let metrics = await getDiskMetrics();
             const tempPath = cacheArchives2 + nextId() + TEMP_SUFFIX;
             await fs.promises.writeFile(tempPath, data);
-            await metrics.addCacheFile(archives, fileName, tempPath);
             await archives.set(fileName, data);
+            await metrics.addCacheFile(archives, fileName, tempPath);
+        },
+        append: async (fileName: string, data: Buffer) => {
+            let metrics = await getDiskMetrics();
+            await archives.append(fileName, data);
+            await metrics.delCacheFile(archives, fileName);
         },
         setLargeFile,
         del: async (fileName: string) => {
@@ -539,10 +543,6 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
             await metrics.delCacheFile(archives, fileName);
             await archives.del(fileName);
         },
-        getInfo: (fileName: string) => archives.getInfo(fileName),
-        find: async (prefix: string, config) => archives.find(prefix, config),
-        findInfo: async (prefix: string, config) => archives.findInfo(prefix, config),
-        enableLogging: () => archives.enableLogging(),
         move: async (config) => {
             // Delete the file from ourselves, even if it is a self move, as not having
             //  values is in the cache is the safest thing, but having extra values
@@ -556,11 +556,9 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
             }
             await archives.move(config);
         },
-        copy: config => archives.copy(config),
-        assertPathValid: (path: string) => archives.assertPathValid(path),
         getBaseArchives: () => archives.getBaseArchives?.() ?? ({ archives: archives, parentPath: "" }),
 
-        async lockRegion(code) {
+        async lockRegion<T>(code: (fncs: LockFncs) => Promise<T>) {
             let locked: string[] = [];
             let fncs = {
                 getPathAndLock: createGetPathAndLock(locked),
@@ -576,7 +574,7 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
                 }
             }
         },
-    };
+    });
 }
 
 

package/src/-a-archives/archives.ts

@@ -21,11 +21,14 @@ export interface Archives {
     set(path: string, data: Buffer): Promise<void>;
     del(path: string): Promise<void>;
 
+    append(path: string, data: Buffer): Promise<void>;
+
     getInfo(path: string): Promise<{ writeTime: number; size: number; } | undefined>;
 
     setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void>;
 
     // For example findFileNames("ips/")
+    /** Returns to full paths. Returns paths WILL NOT end with / */
     find(
         prefix: string,
         config?: {
@@ -76,6 +79,31 @@ export interface Archives {
     }>;
 }
 
+export function createArchivesOverride<T extends Partial<Archives>>(
+    baseArchives: Archives,
+    overrideArchives: T
+): T & Archives {
+    return {
+        getDebugName: () => baseArchives.getDebugName(),
+        get: (fileName: string, config) => baseArchives.get(fileName, config),
+        set: (fileName: string, data: Buffer) => baseArchives.set(fileName, data),
+        append: (fileName: string, data: Buffer) => baseArchives.append(fileName, data),
+        del: (fileName: string) => baseArchives.del(fileName),
+        getInfo: (fileName: string) => baseArchives.getInfo(fileName),
+        setLargeFile: (config) => baseArchives.setLargeFile(config),
+        find: async (prefix: string, config) => baseArchives.find(prefix, config),
+        findInfo: async (prefix: string, config) => baseArchives.findInfo(prefix, config),
+        enableLogging: () => baseArchives.enableLogging(),
+        move: (config) => baseArchives.move(config),
+        copy: (config) => baseArchives.copy(config),
+        assertPathValid: (path: string) => baseArchives.assertPathValid(path),
+        getBaseArchives: () => baseArchives.getBaseArchives?.() ?? ({ archives: baseArchives, parentPath: "" }),
+        getURL: baseArchives.getURL ? (filePath: string) => baseArchives.getURL!(filePath) : undefined,
+        getDownloadAuthorization: baseArchives.getDownloadAuthorization ? (config: { validDurationInSeconds: number }) => baseArchives.getDownloadAuthorization!(config) : undefined,
+        ...overrideArchives,
+    };
+}
+
 export function nestArchives(path: string, archives: Archives): Archives {
     if (!path) return archives;
     if (!path.endsWith("/")) {
@@ -92,6 +120,7 @@ export function nestArchives(path: string, archives: Archives): Archives {
         get: (fileName: string, config) => archives.get(path + stripFilePrefix(fileName), config),
         setLargeFile: (config) => archives.setLargeFile({ ...config, path: path + stripFilePrefix(config.path) }),
         set: (fileName: string, data: Buffer) => archives.set(path + stripFilePrefix(fileName), data),
+        append: (fileName: string, data: Buffer) => archives.append(path + stripFilePrefix(fileName), data),
         del: (fileName: string) => archives.del(path + stripFilePrefix(fileName)),
         getInfo: (fileName: string) => archives.getInfo(path + stripFilePrefix(fileName)),
         find: async (prefix: string, config) => archives.find(path + prefix, config).then(x => x.map(x => x.slice(path.length))),

package/src/-a-archives/archivesBackBlaze.ts

@@ -238,7 +238,7 @@ const getAPI = lazy(async () => {
         };
         action: string;
         uploadTimestamp: number;
-    }>("b2_get_file_info", "POST");
+    }>("b2_get_file_info", "POST", "noAccountId");
 
     const listFileNames = createB2Function<{
         bucketId: string;
@@ -594,10 +594,41 @@ export class ArchivesBackblaze {
             };
             setTimeout(downloadPoll, 5000);
             let result = await this.apiRetryLogic(async (api) => {
+                let range = config?.range;
+                if (range) {
+                    let fileInfo = await this.getInfo(fileName);
+                    if (!fileInfo) throw new Error(`File ${fileName} not found`);
+                    let rangeStart = range.start;
+                    let rangeEnd = Math.min(range.end, fileInfo.size);
+                    // NOTE: I think if we request nothing, it confuses Backblaze and ends up giving us the entire file.
+                    if (rangeEnd <= rangeStart) return Buffer.alloc(0);
+                    let result = await api.downloadFileByName({
+                        bucketName: this.bucketName,
+                        fileName,
+                        range: { start: rangeStart, end: rangeEnd },
+                    });
+                    if (result.length !== rangeEnd - rangeStart) {
+                        let afterLength = await this.getInfo(fileName);
+                        if (afterLength && afterLength.size >= fileInfo.size) {
+                            console.error(`Backblaze range download return the correct number of bytes. Tried to get ${rangeStart}-${rangeEnd}, but received ${rangeStart}-${rangeStart + result.length}. For file: ${fileName}`);
+                            // I'm not sure if it's a bug that where we get extra data if we try to read beyond the end of the file, or if the bug is due to some kind of lag that will resolve itself if we wait a little bit. 
+                            setTimeout(async () => {
+                                let resultAgain = await api.downloadFileByName({
+                                    bucketName: this.bucketName,
+                                    fileName,
+                                    range: { start: rangeStart, end: rangeEnd },
+                                });
+                                devDebugbreak();
+                                let didResultFixItSelf = resultAgain.length === rangeEnd - rangeStart;
+
+                                console.log({ didResultFixItSelf }, resultAgain);
+                            }, timeInMinute * 2);
+                        }
+                    }
+                }
                 return await api.downloadFileByName({
                     bucketName: this.bucketName,
                     fileName,
-                    range: config?.range
                 });
             });
             let timeStr = formatTime(Date.now() - time);
@@ -631,6 +662,14 @@ export class ArchivesBackblaze {
         }
 
     }
+    public async append(fileName: string, data: Buffer): Promise<void> {
+        throw new Error(`ArchivesBackblaze does not support append. Use set instead.`);
+        // this.log(`backblaze append (${formatNumber(data.length)}B) ${fileName}`);
+        // // Backblaze doesn't have native append, so we need to get, concatenate, and set
+        // let existing = await this.get(fileName);
+        // let newData = existing ? Buffer.concat([existing, data]) : data;
+        // await this.set(fileName, newData);
+    }
     public async del(fileName: string): Promise<void> {
         this.log(`backblaze delete ${fileName}`);
         try {
@@ -796,17 +835,26 @@ export class ArchivesBackblaze {
 
     public async getInfo(fileName: string): Promise<{ writeTime: number; size: number; } | undefined> {
         return await this.apiRetryLogic(async (api) => {
-            let info = await api.listFileNames({ bucketId: this.bucketId, prefix: fileName, });
-            let file = info.files.find(x => x.fileName === fileName);
-            if (!file) {
-                this.log(`Backblaze file not exists ${fileName}`);
-                return undefined;
+            try {
+                // NOTE: Apparently, there's no other way to do this, as the file name does not equal the file ID, and git file info requires the file ID. 
+                let info = await api.listFileNames({ bucketId: this.bucketId, prefix: fileName, maxFileCount: 1 });
+                let file = info.files.find(x => x.fileName === fileName && x.action === "upload");
+                if (!file) {
+                    this.log(`Backblaze file not exists ${fileName}`);
+                    return undefined;
+                }
+                this.log(`Backblaze file exists ${fileName}`);
+                return {
+                    writeTime: file.uploadTimestamp,
+                    size: file.contentLength,
+                };
+            } catch (e: any) {
+                if (e.stack.includes(`file_not_found`)) {
+                    this.log(`Backblaze file not exists ${fileName}`);
+                    return undefined;
+                }
+                throw e;
             }
-            this.log(`Backblaze file exists ${fileName}`);
-            return {
-                writeTime: file.uploadTimestamp,
-                size: file.contentLength,
-            };
         });
     }
 

package/src/-a-archives/archivesDisk.ts

@@ -65,9 +65,13 @@ class ArchivesDisk {
         // Don't create the drive (and also only add up to the last path, via slicing (0, i)
         for (let i = 1; i < fileNameParts.length; i++) {
             let dir = this.LOCAL_ARCHIVE_FOLDER + fileNameParts.slice(0, i).join("/");
-            if (!await fsExistsAsync(dir)) {
-                await fs.promises.mkdir(dir);
-            }
+            try {
+                if (!await fsExistsAsync(dir)) {
+                    await fs.promises.mkdir(dir, {
+                        recursive: true,
+                    });
+                }
+            } catch { }
         }
     }
 
@@ -85,6 +89,19 @@ class ArchivesDisk {
         await fs.promises.writeFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
     }
     @measureFnc
+    public async append(fileName: string, data: Buffer): Promise<void> {
+        await this.init();
+
+        this.log(blue(`Appending to file ${fileName} += ${data.length} bytes`));
+        if (storageDisabled) return;
+        fileName = escapeFileName(fileName);
+        await this.simulateLag();
+
+        await this.ensureDirsExist(fileName);
+
+        await fs.promises.appendFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
+    }
+    @measureFnc
     public async del(fileName: string): Promise<void> {
         await this.init();
         this.log(blue(`Deleting file ${fileName}`));
@@ -93,7 +110,7 @@ class ArchivesDisk {
         await this.simulateLag();
         if (!await fsExistsAsync(this.LOCAL_ARCHIVE_FOLDER + fileName)) return;
         try {
-            await fs.promises.unlink(this.LOCAL_ARCHIVE_FOLDER + fileName);
+            await fs.promises.rm(this.LOCAL_ARCHIVE_FOLDER + fileName, { recursive: true });
         } catch { }
         let dir = fileName.replaceAll("\\", "/").split("/").slice(0, -1).join("/");
         await this.gcDir(dir);
@@ -191,7 +208,8 @@ class ArchivesDisk {
                 throw new Error(`File modified changed while reading file ${fileName}`);
             }
 
-            return buffer.slice(0, read.bytesRead);
+            // Make sure we only take as much data as we want in case we are given extra data. 
+            return buffer.slice(0, end - start);
         } catch (e: any) {
             if (e.code === "EMFILE") {
                 if (retryCount > 0) {
@@ -221,7 +239,7 @@ class ArchivesDisk {
         await this.simulateLag();
         let fileNames: string[] = [];
         let folderNames: string[] = [];
-        async function readDir(dir: string) {
+        async function readDir(dir: string, depthToRead: number) {
             if (!await fsExistsAsync(dir)) {
                 return;
             }
@@ -236,7 +254,7 @@ class ArchivesDisk {
                     try {
                         if (fileObj.isDirectory()) {
                             folderNames.push(dir + fileName);
-                            await readDir(dir + fileName + "/");
+                            await readDir(dir + fileName + "/", depthToRead - 1);
                         } else {
                             fileNames.push(dir + fileName);
                         }
@@ -250,7 +268,8 @@ class ArchivesDisk {
         if (pathParts.length > 1) {
             rootDir = pathParts.slice(0, -1).join("/") + "/";
         }
-        await readDir(this.LOCAL_ARCHIVE_FOLDER + rootDir);
+        // We don't know if the prefix they gave us is the prefix of folders or if it is a folder itself. So we have to cut off the end and then read to a depth of 2 to make this work.
+        await readDir(this.LOCAL_ARCHIVE_FOLDER + rootDir, config?.shallow ? 2 : Number.MAX_SAFE_INTEGER);
 
         let results = config?.type === "folders" ? folderNames : fileNames;
 

package/src/-a-archives/archivesLimitedCache.ts

@@ -153,6 +153,27 @@ class ArchivesLimitedCache {
         this.triggerIndexFlush();
     }
 
+    public async append(path: string, data: Buffer): Promise<void> {
+        await this.ensureInitialized();
+        await this.baseArchives.append(path, data);
+        // Update the cache info - increment size and update times
+        const info = this.cache.get(path);
+        const now = Date.now();
+        if (info) {
+            info.size += data.length;
+            info.writeTime = now;
+            info.accessTime = now;
+        } else {
+            // File didn't exist in cache, add it
+            this.cache.set(path, {
+                writeTime: now,
+                accessTime: now,
+                size: data.length,
+            });
+        }
+        this.triggerIndexFlush();
+    }
+
     public async del(path: string): Promise<void> {
         await this.ensureInitialized();
         await this.baseArchives.del(path);