querysub 0.394.0 → 0.395.0

package/.cursorrules

@@ -8,6 +8,8 @@ Use double quotes.
 
 When running a command in the current project, don't "cd" to it. It's redundant, and won't work.
 
+Use the tool calls in order to modify files. Don't try to run terminal commands to modify files.
+
 Don't use redundant comments. If it's a single line and the function name says the same thing that the comment is going to say, you don't need the comment.
 
 When you're inline passing a lambda as an argument, never specify the type. The type should always be inferred. Specifying the type is bad, and it's hard coding, and we don't like hard coding. 
@@ -32,10 +34,16 @@ Follow the rule of minimum scoping. If something can be a local variable, it sho
 
 Try not to use "null", and instead always use "undefined".
 
+Use square brackets for arrays, not a generic.
+    USE: T[]
+    NEVER: Array<T>
+
 Never try to add dynamic pluralization in the UI, just use an s. If you add dynamic pluralization, if the code ever gets localized, all of your changes have to be undone, and you just made localization much harder.
 
 Never use the ternary operator. Instead, do this: "x ? y : z" => "x && y || z".
 
+Never use <h2>, <h3>, etc, they have bad styling.
+
 If are inside an async Event Handlers, you need to use... Querysub.onCommitFinished(() => ...). and put the async code inside the callback. Then when you set state, you need to put the state setting code inside of Querysub.commit(() => ...).
 
 The site automatically builds and hotreloads. Just save the files. DO NOT try to run any build scripts, because it automatically builds on save. DO NOT RUN "npm run build". DO NOT TRY TO RUN UNIT TESTS. DO NOT TRY TO RUN TESTS.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.394.0",
+  "version": "0.395.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-a-archives/archivesJSONT.ts

@@ -13,6 +13,13 @@ export type ArchiveT<T> = {
 
 export function archiveJSONT<T>(archives: () => Archives): ArchiveT<T> {
     archives = lazy(archives);
+
+    let valuesCache = new Map<string, {
+        createTime: number;
+        size: number;
+        value: Buffer;
+    }>();
+
     async function get(key: string) {
         let buffer = await archives().get(key);
         if (!buffer) return undefined;
@@ -28,15 +35,71 @@ export function archiveJSONT<T>(archives: () => Archives): ArchiveT<T> {
         return (await archives().find("")).map(value => value.toString());
     }
     async function values() {
-        let keysArray = await keys();
-        let results: T[] = [];
-        await Promise.all(keysArray.map(async key => {
-            let value = await get(key);
-            if (value) {
-                results.push(value);
+        let infos = await archives().findInfo("");
+
+        let needsUpdate = false;
+        let currentKeys = new Set(infos.map(info => info.path));
+        let cachedKeys = new Set(valuesCache.keys());
+
+        if (currentKeys.size !== cachedKeys.size) {
+            needsUpdate = true;
+        } else {
+            for (let info of infos) {
+                let cached = valuesCache.get(info.path);
+                if (!cached || cached.createTime !== info.createTime || cached.size !== info.size) {
+                    needsUpdate = true;
+                    break;
+                }
             }
-        }));
-        return results;
+        }
+
+        if (needsUpdate) {
+            let maxRetries = 10;
+            for (let attempt = 0; attempt < maxRetries; attempt++) {
+                infos = await archives().findInfo("");
+
+                valuesCache.clear();
+                let allFound = true;
+
+                await Promise.all(infos.map(async info => {
+                    let buffer = await archives().get(info.path);
+                    if (!buffer) {
+                        allFound = false;
+                    } else {
+                        valuesCache.set(info.path, {
+                            createTime: info.createTime,
+                            size: info.size,
+                            value: buffer
+                        });
+                    }
+                }));
+
+                if (!allFound) continue;
+
+                let newInfos = await archives().findInfo("");
+                if (newInfos.length !== infos.length) continue;
+                function anyChanged() {
+                    for (let i = 0; i < newInfos.length; i++) {
+                        if (
+                            newInfos[i].path !== infos[i].path ||
+                            newInfos[i].createTime !== infos[i].createTime ||
+                            newInfos[i].size !== infos[i].size
+                        ) {
+                            return true;
+                        }
+                    }
+                }
+                if (!anyChanged()) break;
+            }
+
+            if (needsUpdate && valuesCache.size === 0) {
+                throw new Error(`Failed to get consistent snapshot of values after ${maxRetries} attempts`);
+            }
+        }
+
+        return Array.from(valuesCache.values()).map(cached =>
+            JSON.parse(cached.value.toString()) as T
+        );
     }
     async function entries(): Promise<[string, T][]> {
         let keysArray = await keys();

package/src/0-path-value-core/pathValueCore.ts

@@ -29,6 +29,7 @@ import { PromiseObj } from "../promise";
 import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { auditLog, isDebugLogEnabled } from "./auditLogs";
 import { logDisk } from "../diagnostics/logs/diskLogger";
+import { isDiskAudit } from "../config";
 
 
 let yargObj = isNodeTrue() && yargs(process.argv)
@@ -1341,6 +1342,12 @@ class PathWatcher {
             if (!obj) continue;
             obj.watchers.delete(callback);
 
+            if (isOwnNodeId(callback)) {
+                auditLog("local UNWATCH PARENT", { path });
+            } else {
+                auditLog("non-local UNWATCH PARENT", { path, watcher: callback });
+            }
+
             if (obj.watchers.size === 0) {
                 watchersObj.delete(path);
                 this.parentWatchers.delete(path);
@@ -1372,6 +1379,13 @@ class PathWatcher {
             let watchers = this.watchers.get(path);
             if (!watchers) continue;
             watchers.watchers.delete(callback);
+
+            if (isOwnNodeId(callback)) {
+                auditLog("local UNWATCH", { path });
+            } else {
+                auditLog("non-local UNWATCH", { path, watcher: callback });
+            }
+
             if (watchers.watchers.size === 0) {
                 this.watchers.delete(path);
 
@@ -1555,6 +1569,11 @@ class PathWatcher {
                 callback(changes, parentPaths ?? []);
             }
         } else {
+            if (isDiskAudit()) {
+                for (let change of changes) {
+                    auditLog("non-local TRIGGER", { path: change.path, time: change.time.time, watcher });
+                }
+            }
             if (!isCoreQuiet) {
                 console.log(`(${Date.now()}) Sending values to client: ${changes.length} (${watcher})`);
             }
@@ -1570,7 +1589,7 @@ class PathWatcher {
                 if (isDebugLogEnabled()) {
                     for (let pathValue of changes) {
 
-                        auditLog("SEND VALUE", { path: pathValue.path, time: pathValue.time.time, nodeId: debugNodeId(watcher), transparent: pathValue.isTransparent, canGC: pathValue.canGCValue });
+                        auditLog("SEND VALUE", { path: pathValue.path, time: pathValue.time.time, watcher, nodeId: debugNodeId(watcher), transparent: pathValue.isTransparent, canGC: pathValue.canGCValue });
                     }
                 }
                 await PathValueController.nodes[watcher].forwardWrites(

package/src/5-diagnostics/GenericFormat.tsx

@@ -44,7 +44,7 @@ let formatters: { [formatter in StringFormatters]: (value: unknown) => preact.Co
     number: (value) => d(value, formatNumber(Number(value))),
     percent: (value) => d(value, formatPercent(Number(value))),
     timeSpan: (value) => d(value, formatTime(Number(value))),
-    date: (value) => d(value, <span title={formatDateTimeDetailed(Number(value))}>{formatVeryNiceDateTime(Number(value))}</span>),
+    date: (value) => d(value, <span title={formatDateTimeDetailed(Number(value))}>{formatDateTime(Number(value))}</span>),
     error: (value) => d(value, <span class={errorMessage}>{String(value)}</span>),
     toSpaceCase: (value) => d(value, toSpaceCase(String(value))),
     "<Selector>": (value) => d(value, <Selector {...JSON.parse(String(value).slice("<Selector>".length))} />),

package/src/deployManager/components/MachinesListPage.tsx

@@ -92,11 +92,10 @@ export class MachinesListPage extends qreact.Component {
                                             this.state.isDeleteMode = false;
                                             this.state.isDeleting = false;
                                         });
-                                    } catch (error) {
+                                    } finally {
                                         Querysub.commit(() => {
                                             this.state.isDeleting = false;
                                         });
-                                        alert(`Failed to delete machines: ${error instanceof Error ? error.message : String(error)}`);
                                     }
                                 });
                             }}>

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -334,8 +334,13 @@ export class BufferIndex {
         results.totalBlockCount += indexEntries.length;
         results.localBlockCount += indexEntries.length;
 
-        // Iterate newest-first so the caller gets the most recent matches first.
-        for (let i = indexEntries.length - 1; i >= 0; i--) {
+        // Iterate based on search direction
+        const iterateForward = !!params.searchFromStart;
+        const startIdx = iterateForward ? 0 : indexEntries.length - 1;
+        const endIdx = iterateForward ? indexEntries.length : -1;
+        const step = iterateForward ? 1 : -1;
+
+        for (let i = startIdx; iterateForward ? i < endIdx : i > endIdx; i += step) {
             if (matchCount >= params.limit || !config.keepIterating()) break;
             await config.results.limitGroup?.wait();
             const blockIndex = i;
@@ -380,7 +385,11 @@ export class BufferIndex {
                 results.localBlockCheckedCount++;
 
                 // Scan all buffers in this block
-                for (let bufferIndex = buffers.length - 1; bufferIndex >= 0; bufferIndex--) {
+                const bufferStartIdx = iterateForward ? 0 : buffers.length - 1;
+                const bufferEndIdx = iterateForward ? buffers.length : -1;
+                const bufferStep = iterateForward ? 1 : -1;
+
+                for (let bufferIndex = bufferStartIdx; iterateForward ? bufferIndex < bufferEndIdx : bufferIndex > bufferEndIdx; bufferIndex += bufferStep) {
                     if (matchCount >= params.limit || !config.keepIterating()) break;
                     await config.results.limitGroup?.wait();
 

package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts

@@ -4,7 +4,7 @@ import { formatNumber, formatPercent } from "socket-function/src/formatting/form
 import { red } from "socket-function/src/formatting/logColors";
 import { MaybePromise } from "socket-function/src/types";
 import { TimeFilePathWithSize } from "./IndexedLogs";
-import { LimitGroup } from "../../../functional/limitProcessing";
+import { ThrottleGroup } from "../../../functional/throttleProcessing";
 
 export const INDEX_EXTENSION = ".index";
 
@@ -17,6 +17,7 @@ export type SearchParams = {
     pathOverrides?: TimeFilePathWithSize[];
     only?: "local" | "public";
     forceReadProduction?: boolean;
+    searchFromStart?: boolean;
 };
 
 export type Unit = number;
@@ -130,6 +131,8 @@ export type IndexedLogResults = {
     totalBackblazeFiles: number;
     localFilesSearched: number;
     backblazeFilesSearched: number;
+    totalBackblazeLogs: number;
+    backblazeLogsSearched: number;
 
     totalBlockCount: number;
     blockCheckedCount: number;
@@ -158,11 +161,11 @@ export type IndexedLogResults = {
     totalSearchTime: number;
 
     cancel?: boolean;
-    limitGroup?: LimitGroup;
+    limitGroup?: ThrottleGroup;
 };
 export function createEmptyIndexedLogResults(): IndexedLogResults {
     return {
-        matchCount: 0, reads: [], totalLocalFiles: 0, totalBackblazeFiles: 0, localFilesSearched: 0, backblazeFilesSearched: 0, totalBlockCount: 0, blockCheckedCount: 0, remoteBlockCount: 0, localBlockCount: 0, remoteBlockCheckedCount: 0, localBlockCheckedCount: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, blockErrors: [], fileErrors: [], remoteIndexesSearched: 0, remoteIndexSize: 0, localIndexesSearched: 0, localIndexSize: 0, timeToFirstMatch: -1, fileFindTime: 0, indexSearchTime: 0, blockSearchTime: 0, totalSearchTime: 0, cancel: undefined, limitGroup: undefined,
+        matchCount: 0, reads: [], totalLocalFiles: 0, totalBackblazeFiles: 0, localFilesSearched: 0, backblazeFilesSearched: 0, totalBackblazeLogs: 0, backblazeLogsSearched: 0, totalBlockCount: 0, blockCheckedCount: 0, remoteBlockCount: 0, localBlockCount: 0, remoteBlockCheckedCount: 0, localBlockCheckedCount: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, blockErrors: [], fileErrors: [], remoteIndexesSearched: 0, remoteIndexSize: 0, localIndexesSearched: 0, localIndexSize: 0, timeToFirstMatch: -1, fileFindTime: 0, indexSearchTime: 0, blockSearchTime: 0, totalSearchTime: 0, cancel: undefined, limitGroup: undefined,
     };
 }
 
@@ -202,6 +205,8 @@ export function mergeIndexedLogResults(existing: IndexedLogResults, incoming: In
         reads: Array.from(readsByKey.values()),
         localFilesSearched: existing.localFilesSearched + incoming.localFilesSearched,
         backblazeFilesSearched: existing.backblazeFilesSearched + incoming.backblazeFilesSearched,
+        totalBackblazeLogs: existing.totalBackblazeLogs + incoming.totalBackblazeLogs,
+        backblazeLogsSearched: existing.backblazeLogsSearched + incoming.backblazeLogsSearched,
         totalBlockCount: existing.totalBlockCount + incoming.totalBlockCount,
         blockCheckedCount: existing.blockCheckedCount + incoming.blockCheckedCount,
         blocksCheckedCompressedSize: existing.blocksCheckedCompressedSize + incoming.blocksCheckedCompressedSize,

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -381,7 +381,7 @@ export class BufferUnitIndex {
             }
         }, `buildHashTableWithLinkedLists`);
 
-        console.log(`Hash table filled slots: ${filledSlots} / ${hashTableCapacity} (${formatPercent(filledSlots / hashTableCapacity)} utilization)`);
+        console.log(`Hash table filled slots: ${filledSlots} / ${hashTableCapacity} (${formatPercent(filledSlots / hashTableCapacity)} utilization, higher means hash function is good, and no probing is used, so 100% is fine)`);
 
         // Step 5: Flatten linked lists and encode to binary
         // Build data section and hash table by traversing linked lists
@@ -532,13 +532,19 @@ export class BufferUnitIndex {
 
             const searchBlock = async (blockIndex: number) => {
                 if (!candidateBlocksSet.has(blockIndex)) return;
-                // This is kind of a weird thing. Basically, because we search in parallel, we might search out of order. So we can only look at the counts before or at us, as if we match a whole bunch after us, but we should still keep going as our matches are going to take precedence.
+                // Check if we should stop iterating based on match counts and direction
                 let stopIterating = () => {
-                    let countBefore = 0;
-                    for (let i = 0; i <= blockIndex; i++) {
-                        countBefore += matchCounts[i];
+                    let relevantCount = 0;
+                    if (params.searchFromStart) {
+                        for (let i = 0; i <= blockIndex; i++) {
+                            relevantCount += matchCounts[i];
+                        }
+                    } else {
+                        for (let i = blockIndex; i < blockCount; i++) {
+                            relevantCount += matchCounts[i];
+                        }
                     }
-                    return countBefore >= params.limit || !keepIterating();
+                    return relevantCount >= params.limit || !keepIterating();
                 };
                 if (stopIterating()) return;
 
@@ -561,7 +567,12 @@ export class BufferUnitIndex {
                     let bufferCount = await this.getBufferCountFromBlock(blockReader);
 
                     // Check each buffer for a match
-                    for (let i = 0; i < bufferCount; i++) {
+                    const iterateForward = !!params.searchFromStart;
+                    const startIdx = iterateForward ? 0 : bufferCount - 1;
+                    const endIdx = iterateForward ? bufferCount : -1;
+                    const step = iterateForward ? 1 : -1;
+
+                    for (let i = startIdx; iterateForward ? i < endIdx : i > endIdx; i += step) {
                         if (stopIterating()) break;
                         await results.limitGroup?.wait();
 
@@ -582,9 +593,13 @@ export class BufferUnitIndex {
                 { parallelCount: BufferUnitIndexParallelSearchCount },
                 searchBlock
             );
-            // Search first first, as moveLogsToPublic should have made it so this is the newest.
+            // Sort blocks by search direction
             let searchOrder = Array.from(candidateBlocksSet);
-            sort(searchOrder, x => x);
+            if (params.searchFromStart) {
+                sort(searchOrder, x => x);
+            } else {
+                sort(searchOrder, x => -x);
+            }
             await Promise.all(searchOrder.map(runSearchBlock));
 
         }, `searchBlocks`);

package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts

@@ -112,7 +112,6 @@ export class UnitSet {
         return Buffer.from(output.buffer);
     }
 
-    @measureFnc
     static has(data: Buffer, unit: Unit): boolean {
         const hashTableCount = data.readUInt32LE(0);
 

package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts

@@ -19,8 +19,14 @@ export class FindProgressTracker<T> {
         if (time < this.config.params.startTime) return false;
         if (time >= this.config.params.endTime) return false;
 
-        if (this.results.length >= this.config.params.limit && time < this.results[0].time) {
-            return false;
+        if (this.config.params.searchFromStart) {
+            if (this.results.length >= this.config.params.limit && time > this.results[this.results.length - 1].time) {
+                return false;
+            }
+        } else {
+            if (this.results.length >= this.config.params.limit && time < this.results[0].time) {
+                return false;
+            }
         }
 
         let newObj = { time, result };
@@ -29,7 +35,11 @@ export class FindProgressTracker<T> {
         this.results.splice(index, 0, newObj);
 
         if (this.results.length > this.config.params.limit) {
-            this.results = this.results.slice(-this.config.params.limit);
+            if (this.config.params.searchFromStart) {
+                this.results = this.results.slice(0, this.config.params.limit);
+            } else {
+                this.results = this.results.slice(-this.config.params.limit);
+            }
         }
 
         this.config.onResult(result);
@@ -37,8 +47,14 @@ export class FindProgressTracker<T> {
     }
 
     public isSourceRelevant(source: { startTime: number; endTime: number; }): boolean {
-        if (this.results.length >= this.config.params.limit && source.endTime < this.results[0].time) {
-            return false;
+        if (this.config.params.searchFromStart) {
+            if (this.results.length >= this.config.params.limit && source.startTime > this.results[this.results.length - 1].time) {
+                return false;
+            }
+        } else {
+            if (this.results.length >= this.config.params.limit && source.endTime < this.results[0].time) {
+                return false;
+            }
         }
         return source.startTime <= this.config.params.endTime && source.endTime >= this.config.params.startTime;
     }

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -22,7 +22,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { assertIsManagementUser } from "../../managementPages";
 import { ignoreErrors } from "../../../errors";
 import { blue } from "socket-function/src/formatting/logColors";
-import { LimitGroup } from "../../../functional/limitProcessing";
+import { ThrottleGroup } from "../../../functional/throttleProcessing";
 import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
 import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
 import { getLoggers2Async } from "../diskLogger";
@@ -427,7 +427,7 @@ export class IndexedLogs<T> {
         allResults.set("", results);
 
 
-        results.limitGroup = new LimitGroup({
+        results.limitGroup = new ThrottleGroup({
             maxTimePerBeforeWait: 500,
             waitTime: 250,
         });
@@ -518,14 +518,19 @@ export class IndexedLogs<T> {
         }
         fileFindTime = Date.now() - fileFindTime;
 
-        // Newest first
-        sort(paths, x => - x.startTime);
+        // Sort by time based on search direction
+        if (config.params.searchFromStart) {
+            sort(paths, x => x.startTime);
+        } else {
+            sort(paths, x => - x.startTime);
+        }
 
         let results = config.results;
         let localPaths = paths.filter(x => x.logCount === undefined);
         let remotePaths = paths.filter(x => x.logCount !== undefined);
         results.totalLocalFiles = localPaths.length;
         results.totalBackblazeFiles = remotePaths.length;
+        results.totalBackblazeLogs = remotePaths.reduce((sum, x) => sum + (x.logCount ?? 0), 0);
         results.fileFindTime = fileFindTime;
 
         let progressTracker = new FindProgressTracker<T>({
@@ -551,6 +556,7 @@ export class IndexedLogs<T> {
             let archives = remote ? backblazeLogs : localLogs;
             if (remote) {
                 results.backblazeFilesSearched++;
+                results.backblazeLogsSearched += path.logCount ?? 0;
             } else {
                 results.localFilesSearched++;
             }