querysub 0.356.0 → 0.357.0

package/src/-a-archives/archivesMemoryCache.ts

@@ -0,0 +1,350 @@
+import { measureWrap } from "socket-function/src/profiling/measure";
+import { Archives, createArchivesOverride } from "./archives";
+
+interface CacheEntry {
+    path: string;
+    start: number;
+    end: number;
+    data: Buffer;
+    size: number;
+    lastAccess: number;
+}
+
+export type ArchivesMemoryCacheStats = {
+    cachedReads: number;
+    uncachedReads: number;
+    cachedReadSize: number;
+    uncachedReadSize: number;
+    totalCacheSize: number;
+    totalCacheCount: number;
+};
+
+let cacheDisabled = false;
+export function testDisableCache() {
+    cacheDisabled = true;
+}
+
+/** Creates an in-memory cache for the archives. We expect that the values will only have values appended to them or be deleted. Also, if a value is deleted, we won't erase our in-memory cache, and so we will still allow reading from deleted values. */
+export function createArchivesMemoryCache(
+    archives: Archives,
+    config?: {
+        extraReadSize?: number;
+        maxSize?: number;
+        maxCount?: number;
+        // If the files are guaranteed to be immutable, then all reads, including reading entire file and reading any range, can be cached. 
+        //  - Otherwise, we can't cache when the entire file is read, as there might be further appends after it. 
+        //  - Also, it means if there's a range read that happens, if that range read doesn't return enough bytes, for example, we try to read from 0 to 100, but we only get 50 bytes back, we can only cache the 0 to 50, and then if they read from 0 to 100 again, we can use the cache, but we have to also read 50 to 100 from the source, because the file might have gotten larger. 
+        fullyImmutable?: boolean;
+        stats?: ArchivesMemoryCacheStats;
+    }
+): Archives {
+    let {
+        maxSize = 1024 * 1024 * 1024 * 4,
+        maxCount = 1000 * 1000,
+        fullyImmutable = false,
+        extraReadSize = 1024
+    } = config ?? {};
+
+    // Cache structure: Map from path to sorted array of ranges
+    let cacheByPath = new Map<string, CacheEntry[]>();
+    // LRU tracking: all entries sorted by last access time
+    let lruArray: CacheEntry[] = [];
+    let totalSize = 0;
+
+    const alignRange = measureWrap(function alignRange(start: number, end: number): { start: number; end: number; } {
+        if (extraReadSize <= 1) {
+            return { start, end };
+        }
+        return {
+            start: Math.floor(start / extraReadSize) * extraReadSize,
+            end: Math.ceil(end / extraReadSize) * extraReadSize
+        };
+    });
+
+    const updateAccess = measureWrap(function updateAccess(entry: CacheEntry) {
+        entry.lastAccess = Date.now();
+        // Move to end of LRU array (most recent)
+        let index = lruArray.indexOf(entry);
+        if (index !== -1) {
+            lruArray.splice(index, 1);
+            lruArray.push(entry);
+        }
+    });
+
+    const evictOldest = measureWrap(function evictOldest() {
+        while ((totalSize > maxSize || lruArray.length > maxCount) && lruArray.length > 0) {
+            let oldest = lruArray.shift();
+            if (oldest) {
+                totalSize -= oldest.size;
+                // Remove from path map
+                let pathEntries = cacheByPath.get(oldest.path);
+                if (pathEntries) {
+                    let index = pathEntries.indexOf(oldest);
+                    if (index !== -1) {
+                        pathEntries.splice(index, 1);
+                    }
+                    if (pathEntries.length === 0) {
+                        cacheByPath.delete(oldest.path);
+                    }
+                }
+            }
+        }
+    });
+
+    const removeEntry = measureWrap(function removeEntry(entry: CacheEntry) {
+        totalSize -= entry.size;
+
+        // Remove from path map
+        let pathEntries = cacheByPath.get(entry.path);
+        if (pathEntries) {
+            let index = pathEntries.indexOf(entry);
+            if (index !== -1) {
+                pathEntries.splice(index, 1);
+            }
+        }
+
+        // Remove from LRU array
+        let lruIndex = lruArray.indexOf(entry);
+        if (lruIndex !== -1) {
+            lruArray.splice(lruIndex, 1);
+        }
+    });
+
+    const addEntry = measureWrap(function addEntry(path: string, start: number, end: number, data: Buffer) {
+        let entry: CacheEntry = {
+            path,
+            start,
+            end,
+            data,
+            size: data.length,
+            lastAccess: Date.now()
+        };
+
+        // Add to path map (keeping sorted by start)
+        let pathEntries = cacheByPath.get(path);
+        if (!pathEntries) {
+            pathEntries = [];
+            cacheByPath.set(path, pathEntries);
+        }
+
+        // Insert sorted by start position
+        let insertIndex = 0;
+        while (insertIndex < pathEntries.length && pathEntries[insertIndex].start < start) {
+            insertIndex++;
+        }
+        pathEntries.splice(insertIndex, 0, entry);
+
+        // Add to LRU array
+        lruArray.push(entry);
+        totalSize += entry.size;
+
+        evictOldest();
+    });
+
+    const findOverlappingEntries = measureWrap(function findOverlappingEntries(path: string, start: number, end: number): CacheEntry[] {
+        let pathEntries = cacheByPath.get(path);
+        if (!pathEntries) {
+            return [];
+        }
+
+        let overlapping: CacheEntry[] = [];
+        for (let entry of pathEntries) {
+            // Check if ranges overlap: entry overlaps if it ends after start and starts before end
+            if (entry.end > start && entry.start < end) {
+                overlapping.push(entry);
+            }
+        }
+        return overlapping;
+    });
+
+    const merge = measureWrap(function merge(path: string) {
+        let pathEntries = cacheByPath.get(path);
+        if (!pathEntries || pathEntries.length <= 1) {
+            return;
+        }
+
+        // Entries are already sorted by start
+        // Merge any overlapping or adjacent entries
+        for (let i = 0; i < pathEntries.length - 1; i++) {
+            let current = pathEntries[i];
+            let next = pathEntries[i + 1];
+
+            // Check if current and next are overlapping or adjacent
+            if (current.end >= next.start) {
+                // Calculate how much data from next extends beyond current
+                let newEnd = Math.max(current.end, next.end);
+                let bytesToCopy = newEnd - current.end;
+
+                if (bytesToCopy > 0) {
+                    // Extend current's data with data from next
+                    let newData = Buffer.alloc(newEnd - current.start);
+                    current.data.copy(newData, 0);
+
+                    // Copy the part of next that extends beyond current
+                    let nextOffset = current.end - next.start;
+                    next.data.copy(newData, current.end - current.start, nextOffset, nextOffset + bytesToCopy);
+
+                    current.data = newData;
+                    current.end = newEnd;
+                    current.size = newData.length;
+                    totalSize += bytesToCopy;
+                }
+
+                // Remove next entry
+                removeEntry(next);
+
+                // Check current against the new next
+                i--;
+            }
+        }
+    });
+
+    async function cachedGet(path: string, getConfig?: {
+        range?: { start: number; end: number; };
+        retryCount?: number;
+        fastRead?: boolean;
+    }): Promise<Buffer | undefined> {
+        if (cacheDisabled) {
+            return await archives.get(path, getConfig);
+        }
+
+        let range = getConfig?.range;
+
+        // If no range specified, read entire file
+        if (!range) {
+            // If not fully immutable, don't cache entire file reads (might have appends)
+            if (!fullyImmutable) {
+                return await archives.get(path, getConfig);
+            }
+
+            // Get file info to determine size, then treat as range read
+            let info = await archives.getInfo(path);
+            if (!info) {
+                return undefined;
+            }
+
+            // Convert to range read from 0 to file size
+            range = { start: 0, end: info.size };
+        }
+
+        // Handle range reads
+        let { start, end } = range;
+
+        // If we have something which satisfies the range request, we should use this first. Otherwise, every single request requires doing an API call, which is extremely slow. 
+        {
+            for (let entry of cacheByPath.get(path) || []) {
+                if (entry.start <= start && entry.end >= end) {
+                    return entry.data.slice(start - entry.start, end - entry.start);
+                }
+            }
+        }
+
+        let size = await archives.getInfo(path);
+        if (!size) {
+            return undefined;
+        }
+
+        // Track if we had to perform any read from the underlying archives
+        let didRead = false;
+        {
+            let aligned = alignRange(start, end);
+
+            let readStart = aligned.start;
+            let readEnd = Math.min(aligned.end, size.size);
+
+            let overlapping = findOverlappingEntries(path, start, end);
+
+            // Check if we can trim the beginning
+            for (let entry of overlapping) {
+                if (entry.start <= readStart && entry.end > readStart) {
+                    // We have cached data covering the beginning
+                    readStart = Math.max(readStart, entry.end);
+                }
+            }
+
+            let haveEnd = false;
+
+            // Check if we can trim the end
+            for (let entry of overlapping) {
+                if (entry.start < readEnd && entry.end >= readEnd) {
+                    // We have cached data covering the end
+                    readEnd = Math.min(readEnd, entry.start);
+                    haveEnd = true;
+                }
+            }
+
+
+            // If after trimming we have nothing to read, we have complete coverage
+            if (readStart < readEnd) {
+                // Read the single contiguous range
+                let data = await archives.get(path, {
+                    ...getConfig,
+                    range: { start: readStart, end: readEnd }
+                });
+
+                if (!data) {
+                    return undefined;
+                }
+
+                didRead = true;
+
+                // Cache the data we read (it's immutable even if the file can have future appends)
+                readEnd = readStart + data.length;
+                // If they tried to read too far then we reduce the end value, otherwise we'll be looking for something which we didn't satisfy and we'll get an error. 
+                if (!haveEnd) {
+                    end = Math.min(end, readEnd);
+                }
+                addEntry(path, readStart, readEnd, data);
+            }
+        }
+
+        // Always merge after potentially adding new data
+        merge(path);
+
+        // After merging, find the single entry that covers our aligned range
+        let pathEntries = cacheByPath.get(path);
+        if (!pathEntries) {
+            throw new Error(`Expected cached entries for path ${path} after merge`);
+        }
+
+        let coveringEntry: CacheEntry | undefined = undefined;
+        for (let entry of pathEntries) {
+            if (entry.start <= start && entry.end >= end) {
+                if (coveringEntry) {
+                    throw new Error(`Multiple entries cover range ${start}-${end} for path ${path}`);
+                }
+                coveringEntry = entry;
+            }
+        }
+
+        if (!coveringEntry) {
+            throw new Error(`No entry covers range ${start}-${end} for path ${path}`);
+        }
+
+        updateAccess(coveringEntry);
+
+        // Extract the requested range
+        let offsetStart = start - coveringEntry.start;
+        let offsetEnd = offsetStart + (end - start);
+        let result = coveringEntry.data.slice(offsetStart, offsetEnd);
+
+        // Update statistics at the end when we know the final result
+        if (config?.stats) {
+            if (didRead) {
+                config.stats.uncachedReads++;
+                config.stats.uncachedReadSize += result.length;
+            } else {
+                config.stats.cachedReads++;
+                config.stats.cachedReadSize += result.length;
+            }
+            config.stats.totalCacheSize = totalSize;
+            config.stats.totalCacheCount = lruArray.length;
+        }
+
+        return result;
+    }
+
+    return createArchivesOverride(archives, {
+        get: cachedGet
+    });
+}

package/src/-a-archives/archivesPrivateFileSystem.ts

@@ -75,6 +75,28 @@ class ArchivesPrivateFileSystem {
         await writable.close();
     }
 
+    @measureFnc
+    public async append(fileName: string, data: Buffer): Promise<void> {
+        this.log(blue(`Appending to file ${fileName} += ${data.length} bytes`));
+
+        await this.ensureDirsExist(fileName);
+
+        const pathParts = fileName.split("/");
+        const filename = pathParts[pathParts.length - 1];
+        const dirPath = pathParts.length > 1 ? pathParts.slice(0, -1).join("/") : "";
+
+        const directory = await this.getOrCreateDirectory(dirPath);
+        const fileHandle = await directory.getFileHandle(filename, { create: true });
+
+        // Get current file to find the size for appending at the end
+        const file = await fileHandle.getFile();
+        const currentSize = file.size;
+
+        const writable = await fileHandle.createWritable({ keepExistingData: true });
+        await writable.write({ type: "write", position: currentSize, data });
+        await writable.close();
+    }
+
     @measureFnc
     public async del(fileName: string): Promise<void> {
         this.log(blue(`Deleting file ${fileName}`));

package/src/-g-core-values/NodeCapabilities.ts

@@ -19,6 +19,9 @@ import { hackDevtoolsWebsocketForward } from "./oneTimeForward";
 import { getOwnMachineId, decodeNodeId, decodeNodeIdAssert } from "../-a-auth/certs";
 import { sort } from "socket-function/src/misc";
 import { getPathStr2 } from "../path";
+setImmediate(() => {
+    import("../diagnostics/MachineThreadInfo");
+});
 
 let loadTime = Date.now();
 

package/src/0-path-value-core/auditLogs.ts

@@ -1,9 +1,10 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
-import { isDevDebugbreak } from "../config";
+import { isDevDebugbreak, isDiskAudit } from "../config";
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { QueueLimited } from "socket-function/src/misc";
 import { isNode } from "typesafecss";
+import { logDisk } from "../diagnostics/logs/diskLogger";
 
 export type DebugLog = {
     type: string;
@@ -56,6 +57,9 @@ function debugLogBase(type: string, values: { [key: string]: unknown }) {
         disableAuditLogging();
         return;
     }
+    if (isDiskAudit()) {
+        logDisk("log", type, values, { "diskAudit": true });
+    }
     let newEntry: DebugLog = { type, time: Date.now(), values };
     logHistory.push(newEntry);
 };

package/src/0-path-value-core/pathValueCore.ts

@@ -17,7 +17,6 @@ import { binarySearchIndex, isNode, isNodeTrue, last, promiseObj, sort, timeInHo
 import { isNodeTrusted, isTrusted, isTrustedByNode } from "../-d-trust/NetworkTrust2";
 import { AuthorityPath, LOCAL_DOMAIN, LOCAL_DOMAIN_PATH, pathValueAuthority2 } from "./NodePathAuthorities";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
-import { isNoNetwork } from "../config";
 import { formatTime } from "socket-function/src/formatting/format";
 import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { getNodeIdDomain, getNodeIdDomainMaybeUndefined, getNodeIdIP } from "socket-function/src/nodeCache";
@@ -406,12 +405,6 @@ export function getCompressNetwork() {
     return getCompressNetworkBase();
 }
 
-let getCompressDiskBase = () => false;
-export const registerGetCompressDisk = (fnc: () => boolean) => { getCompressDiskBase = fnc; };
-export function getCompressDisk() {
-    return getCompressDiskBase();
-}
-
 
 const filterChildPathsBase = measureWrap(
     function filterChildPathsBase(parentPath: string, packedSuffix: string, paths: Set<string>): Set<string> {
@@ -1260,6 +1253,13 @@ class PathWatcher {
                 for (let path of newParentsWatched) {
                     auditLog("new local WATCH PARENT", { path });
                 }
+            } else {
+                for (let path of newPathsWatched) {
+                    auditLog("new non-local WATCH", { path, watcher: config.callback });
+                }
+                for (let path of newParentsWatched) {
+                    auditLog("new non-local WATCH PARENT", { path, watcher: config.callback });
+                }
             }
             logDisk("log", `New PathValue watches`, {
                 newPathsWatched: newPathsWatched.size,

package/src/4-dom/qreact.tsx

@@ -1223,6 +1223,7 @@ class QRenderClass {
                         //  break our render function.
                         void Promise.resolve().finally(() => {
                             logErrors(proxyWatcher.commitFunction({
+                                debugName: getDebugName("ref"),
                                 canWrite: true,
                                 baseFunction: ref,
                                 watchFunction() {

package/src/4-querysub/Querysub.ts

@@ -19,7 +19,7 @@ import { cache, cacheLimited, lazy } from "socket-function/src/caching";
 import { getOwnMachineId, getOwnThreadId, getThreadKeyCert, verifyMachineIdForPublicKey } from "../-a-auth/certs";
 import { getHostedIP, getSNICerts, publishMachineARecords } from "../-e-certs/EdgeCertController";
 import { LOCAL_DOMAIN, nodePathAuthority } from "../0-path-value-core/NodePathAuthorities";
-import { debugCoreMode, registerGetCompressNetwork, encodeParentFilter, registerGetCompressDisk, authorityStorage } from "../0-path-value-core/pathValueCore";
+import { debugCoreMode, registerGetCompressNetwork, encodeParentFilter, authorityStorage } from "../0-path-value-core/pathValueCore";
 import { clientWatcher, ClientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { SyncWatcher, proxyWatcher, specialObjectWriteValue, isSynced, PathValueProxyWatcher, atomic, doAtomicWrites, noAtomicSchema, undeleteFromLookup, registerSchemaPrefix, WatcherOptions, doProxyOptions } from "../2-proxy/PathValueProxyWatcher";
 import { isInProxyDatabase, rawSchema } from "../2-proxy/pathDatabaseProxyBase";
@@ -202,9 +202,6 @@ export class Querysub {
     public static MAX_FUTURE_CALL_TIME = 10_000;
 
 
-    /** Compression makes serialization about 2X slower, but reduces the size by about 2X (more or less if your
-     *      data size is dominated by value size instead of key size). */
-    public static COMPRESS_DISK = false;
     public static COMPRESS_NETWORK = true;
 
     public static now = getSyncedTime;
@@ -1297,7 +1294,6 @@ setImmediate(async () => {
 });
 
 registerGetCompressNetwork(() => Querysub.COMPRESS_NETWORK);
-registerGetCompressDisk(() => Querysub.COMPRESS_DISK);
 
 (globalThis as any).Querysub = Querysub;
 

package/src/config.ts

@@ -22,6 +22,7 @@ let yargObj = parseArgsFactory()
     // TODO: The bootstrapper is a single file. Maybe we shouldn't run the entire service just for that. Although... maybe it's fine, as services are light?
     .option("bootstraponly", { type: "boolean", desc: "Don't register as an edge node, so we serve the bootstrap files, but we don't need up to date code because we are not used for endpoints or the UI." })
     .option("notifyemails", { type: "array", desc: "The emails to notify when errors occur." })
+    .option("diskaudit", { type: "boolean", desc: "Track all audit logs to disk. This might end up writing A LOT of data." })
     .argv
     ;
 type QuerysubConfig = {
@@ -74,6 +75,10 @@ export function isRecovery() {
     return yargObj.recovery;
 }
 
+export function isDiskAudit() {
+    return !!yargObj.diskaudit;
+}
+
 export function devDebugbreak() {
     if (!isNode()) {
         debugger;