querysub 0.375.0 → 0.376.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.375.0",
+  "version": "0.376.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -10,7 +10,7 @@ import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profi
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { magenta, yellow } from "socket-function/src/formatting/logColors";
 import { Unit, getAllUnits, Reader, createOffsetReader, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
-import { createMatchesPattern, getSearchUnits } from "./bufferMatcher";
+import { createMatchesPattern, getSearchUnits } from "./bufferSearchFindMatcher";
 import { UnitSet } from "./BufferUnitSet";
 import { BufferUnitIndex } from "./BufferUnitIndex";
 import { BufferListStreamer } from "./BufferListStreamer";
@@ -178,13 +178,13 @@ export class BufferIndex {
 
     // IMPORTANT! The input data values should be sorted from newest to oldest. 
     @measureFnc
-    public static encodeAll(config: {
+    public static async encodeAll(config: {
         data: Buffer[];
-    }): {
+    }): Promise<{
         data: Buffer;
         index: Buffer;
-    } {
-        let obj = BufferUnitIndex.encode(config.data, {
+    }> {
+        let obj = await BufferUnitIndex.encode(config.data, {
             type: BULK_TYPE,
         });
         return {
@@ -453,6 +453,11 @@ export class BufferIndex {
             results.remoteIndexesSearched += 1;
             results.remoteIndexSize += index.length;
         } else {
+            let dataLength = await dataReader.getLength();
+            if (dataLength < 32) {
+                // If it's a small file, nothing of value will be lost by ignoring it. And almost all of our errors are coming from extremely small files. 
+                return;
+            }
             throw new Error(`Unknown type in index file: ${type}`);
         }
     }

package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.cpp

@@ -3,6 +3,20 @@
 #include <string.h>
 #include <algorithm>
 
+static uint32_t RemapUnit(uint32_t unit) {
+    uint8_t byte0 = (unit >> 0) & 0xFF;
+    uint8_t byte1 = (unit >> 8) & 0xFF;
+    uint8_t byte2 = (unit >> 16) & 0xFF;
+    uint8_t byte3 = (unit >> 24) & 0xFF;
+
+    if (byte0 >= 65 && byte0 <= 90) byte0 += 32;
+    if (byte1 >= 65 && byte1 <= 90) byte1 += 32;
+    if (byte2 >= 65 && byte2 <= 90) byte2 += 32;
+    if (byte3 >= 65 && byte3 <= 90) byte3 += 32;
+
+    return byte0 | (byte1 << 8) | (byte2 << 16) | (byte3 << 24);
+}
+
 static napi_value PopulateUnits(napi_env env, napi_callback_info info) {
     size_t argc = 2;
     napi_value args[2];
@@ -75,6 +89,8 @@ static napi_value PopulateUnits(napi_env env, napi_callback_info info) {
                                (bufferData[i+2] << 16) | 
                                (bufferData[i+3] << 24);
                 
+                unit = RemapUnit(unit);
+                
                 if (unit == 0) continue;
                 
                 unitsArray[currentOffset] = unit;
@@ -184,6 +200,8 @@ static napi_value BuildHashTable(napi_env env, napi_callback_info info) {
             // This reads 4 bytes starting at position i
             uint32_t unit = *(uint32_t*)(blockData + i);
             
+            unit = RemapUnit(unit);
+            
             uint32_t maskedHash = GetMaskedHash(unit, mask);
             uint32_t currentHead = hashTable[maskedHash];
             
@@ -291,6 +309,8 @@ static napi_value EstimateUniqueUnits(napi_env env, napi_callback_info info) {
             // Read unit directly by casting pointer (little-endian architecture)
             uint32_t unit = *(uint32_t*)(bufferData + i);
             
+            unit = RemapUnit(unit);
+            
             // Get masked hash with 0xFFFF mask (16-bit)
             uint32_t maskedHash = GetMaskedHash(unit, 0xFFFF);
             

package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts

@@ -57,6 +57,33 @@ export function createOffsetReader(reader: Reader, offset: number): Reader {
     };
 }
 
+export function remapUnit(unit: Unit): Unit {
+    let byte0 = (unit >> 0) & 0xFF;
+    let byte1 = (unit >> 8) & 0xFF;
+    let byte2 = (unit >> 16) & 0xFF;
+    let byte3 = (unit >> 24) & 0xFF;
+
+    if (byte0 >= 65 && byte0 <= 90) byte0 += 32;
+    if (byte1 >= 65 && byte1 <= 90) byte1 += 32;
+    if (byte2 >= 65 && byte2 <= 90) byte2 += 32;
+    if (byte3 >= 65 && byte3 <= 90) byte3 += 32;
+
+    return (byte0 | (byte1 << 8) | (byte2 << 16) | (byte3 << 24)) >>> 0;
+}
+export function remapSearchBuffer(buffer: Buffer) {
+    let newBuffer = Buffer.alloc(buffer.length);
+    buffer.copy(newBuffer);
+    buffer = newBuffer;
+    for (let i = 0; i < buffer.length; i++) {
+        let byte = buffer[i];
+        if (byte >= 65 && byte <= 90) {
+            buffer[i] = byte + 32;
+        } else {
+            buffer[i] = byte;
+        }
+    }
+    return buffer;
+}
 
 export const getAllUnits = measureWrap(function getAllUnits(config: {
     buffer: Buffer;
@@ -71,7 +98,7 @@ export const getAllUnits = measureWrap(function getAllUnits(config: {
     const result: UnitRef[] = [];
     // For each byte position, create a unit from that position and the next 3 bytes
     for (let i = 0; i <= buffer.length - 4; i++) {
-        const unit = buffer.readUint32LE(i);
+        const unit = remapUnit(buffer.readUint32LE(i));
         if (!unit) continue;
 
         result.push({

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -3,7 +3,7 @@
 import { LZ4 } from "../../../storage/LZ4";
 import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
 import { Zip } from "../../../zip";
-import { BufferReader, Reader, SearchParams, IndexedLogResults, Unit } from "./BufferIndexHelpers";
+import { BufferReader, Reader, SearchParams, IndexedLogResults, Unit, remapUnit } from "./BufferIndexHelpers";
 import { formatNumber, formatPercent } from "socket-function/src/formatting/format";
 import { lazy } from "socket-function/src/caching";
 import { list, sort } from "socket-function/src/misc";
@@ -11,23 +11,64 @@ import { testDisableCache } from "../../../-a-archives/archivesMemoryCache";
 import { devDebugbreak } from "../../../config";
 import { BufferUnitIndexParallelSearchCount, DEFAULT_BLOCK_SIZE, DEFAULT_TARGET_UNITS_PER_BUCKET } from "./BufferIndexLogsOptimizationConstants";
 import { runInParallel } from "socket-function/src/batching";
-import { createMatchesPattern } from "./bufferMatcher";
+import { createMatchesPattern } from "./bufferSearchFindMatcher";
+import * as path from "path";
+import * as fs from "fs";
+import { runPromise } from "socket-function/src/runPromise";
+import * as crypto from "crypto";
 
 const USE_COMPRESSION = true;
 
+const getCppAddon = lazy(async () => {
+    if (typeof process === "undefined" || !process.versions || !process.versions.node) {
+        return undefined;
+    }
 
-// Lazy-load the C++ addon (only loaded when actually needed, not in client-side code)
-let getCppAddon = lazy(() => {
-    try {
-        // Check if we're in Node.js environment
-        if (typeof process === "undefined" || !process.versions || !process.versions.node) {
+    const checkNeedsBuild = (): boolean => {
+        const cppFilePath = path.join(__dirname, "BufferIndexCPP.cpp");
+        const distPath = path.join(__dirname, "dist");
+        const versionFilePath = path.join(distPath, "BufferIndexCPP.version");
+        const buildPath = path.join(__dirname, "build");
+
+        if (!fs.existsSync(buildPath)) {
+            return true;
+        }
+        if (!fs.existsSync(versionFilePath)) {
+            return true;
+        }
+        try {
+            const cppFileContent = fs.readFileSync(cppFilePath, "utf-8");
+            const currentHash = crypto.createHash("sha256").update(cppFileContent).digest("hex");
+            const storedHash = fs.readFileSync(versionFilePath, "utf-8").trim();
+            if (storedHash !== currentHash) {
+                return true;
+            }
+        } catch (e) {
+            return true;
+        }
+        return false;
+    };
+
+    if (checkNeedsBuild()) {
+        try {
+            await runPromise("node-gyp rebuild");
+            const cppFilePath = path.join(__dirname, "BufferIndexCPP.cpp");
+            const distPath = path.join(__dirname, "dist");
+            const versionFilePath = path.join(distPath, "BufferIndexCPP.version");
+            if (!fs.existsSync(distPath)) {
+                fs.mkdirSync(distPath, { recursive: true });
+            }
+            const cppFileContent = fs.readFileSync(cppFilePath, "utf-8");
+            const currentHash = crypto.createHash("sha256").update(cppFileContent).digest("hex");
+            fs.writeFileSync(versionFilePath, currentHash);
+        } catch (e) {
             return undefined;
         }
-        // Try to require the C++ addon
-        const addon = require("./BufferIndexCPP") as typeof import("./BufferIndexCPP");
-        return addon;
+    }
+
+    try {
+        return require("./BufferIndexCPP") as typeof import("./BufferIndexCPP");
     } catch (e) {
-        console.warn("C++ addon not available, falling back to TypeScript implementation", e);
         return undefined;
     }
 });
@@ -107,14 +148,14 @@ export class BufferUnitIndex {
 
     // IMPORTANT! The input data values should be sorted from newest to oldest. 
     @measureFnc
-    public static encode(allData: Buffer[], options: {
+    public static async encode(allData: Buffer[], options: {
         type: number;
         blockSize?: number;
         targetUnitsPerBucket?: number;
-    }): {
+    }): Promise<{
         blocks: Buffer;
         index: Buffer;
-    } {
+    }> {
         const type = options.type;
         const BLOCK_SIZE = options.blockSize ?? DEFAULT_BLOCK_SIZE;
         const TARGET_UNITS_PER_BUCKET = options.targetUnitsPerBucket ?? DEFAULT_TARGET_UNITS_PER_BUCKET;
@@ -199,8 +240,8 @@ export class BufferUnitIndex {
             return Buffer.concat([blocksHeaderBuffer, ...compressedBlocks]);
         }, `createFinalBlocksBuffer`);
 
-        const estimatedUniqueUnits = measureBlock(() => {
-            const cppAddon = getCppAddon();
+        const estimatedUniqueUnits = await measureBlock(async () => {
+            const cppAddon = await getCppAddon();
             if (cppAddon) {
                 // Use C++ addon for performance
                 return cppAddon.estimateUniqueUnits(allData);
@@ -213,7 +254,7 @@ export class BufferUnitIndex {
 
                 for (const buffer of allData) {
                     for (let i = 0; i <= buffer.length - 4; i++) {
-                        const unit = buffer.readUint32LE(i);
+                        const unit = remapUnit(buffer.readUint32LE(i));
                         const maskedHash = getMaskedHash(unit, 0xFFFF);
 
                         if (seenMaskedHashes[maskedHash] === 0) {
@@ -251,8 +292,8 @@ export class BufferUnitIndex {
 
 
         // Step 4: Build hash table with linked lists using Uint32Arrays for performance
-        const { hashTable, linkedListData, nodeCount, filledSlots } = measureBlock(() => {
-            const cppAddon = getCppAddon();
+        const { hashTable, linkedListData, nodeCount, filledSlots } = await measureBlock(async () => {
+            const cppAddon = await getCppAddon();
             if (cppAddon) {
                 // Use C++ addon for performance
                 return cppAddon.buildHashTable(blocksUncompressed, hashTableCapacity, mask);
@@ -298,7 +339,7 @@ export class BufferUnitIndex {
 
                     // Iterate through all units and add them directly
                     for (let i = 0; i <= blockUncompressed.length - 4; i++) {
-                        const unit = blockUncompressed.readUint32LE(i);
+                        const unit = remapUnit(blockUncompressed.readUint32LE(i));
                         const maskedHash = getMaskedHash(unit, mask);
 
                         const currentHead = hashTable[maskedHash];

package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts

@@ -1,7 +1,7 @@
 import { formatPercent, formatNumber } from "socket-function/src/formatting/format";
 import { red } from "socket-function/src/formatting/logColors";
 import { measureFnc } from "socket-function/src/profiling/measure";
-import { Unit } from "./BufferIndexHelpers";
+import { Unit, remapUnit } from "./BufferIndexHelpers";
 
 export class UnitSet {
     // Hash-based set for checking unit membership (no positions stored)
@@ -58,7 +58,7 @@ export class UnitSet {
                     const buffer = block[bufferIndex];
                     // Extract units directly
                     for (let i = 0; i <= buffer.length - 4; i++) {
-                        const unit = buffer.readUint32LE(i);
+                        const unit = remapUnit(buffer.readUint32LE(i));
                         if (!unit) continue;
 
                         totalInserts++;

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -283,8 +283,8 @@ export class LogViewer3 extends qreact.Component {
                     forceReadProduction: readProductionLogsURL.value,
                 },
                 onResult: (match: LogDatum) => {
-                    console.log("onResult", match);
                     results.push(match);
+                    sort(results, x => -x.time);
                     void updateResults();
                 },
                 onResults: (loggerStats: IndexedLogResults) => {
@@ -433,6 +433,52 @@ export class LogViewer3 extends qreact.Component {
         );
     }
 
+    renderForceMoveLogs() {
+        return <><Button
+            onClick={async () => {
+                Querysub.commitLocal(() => {
+                    this.state.forceMoveStartTime = Date.now();
+                    this.state.forceMoveEndTime = undefined;
+                });
+
+                let forceMoveInterval = setInterval(() => {
+                    this.forceUpdate();
+                }, 100);
+
+                try {
+                    let loggers = await getLoggers2Async();
+                    for (let logger of Object.values(loggers)) {
+                        await logger.clientForceMoveLogsToPublic();
+                    }
+                    await this.loadPaths();
+                } finally {
+                    if (forceMoveInterval) {
+                        clearInterval(forceMoveInterval);
+                    }
+
+                    Querysub.commitLocal(() => {
+                        this.state.forceMoveEndTime = Date.now();
+                    });
+                }
+            }}
+            hue={180}
+        >
+            Force Move Logs to Public
+        </Button>
+            {(() => {
+                if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime === undefined) {
+                    const elapsed = Date.now() - this.state.forceMoveStartTime;
+                    return <div>Moving logs... {formatTime(elapsed)}</div>;
+                }
+                if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime !== undefined) {
+                    const duration = this.state.forceMoveEndTime - this.state.forceMoveStartTime;
+                    return <div>Moved logs in {formatTime(duration)}</div>;
+                }
+                return undefined;
+            })()}
+        </>;
+    }
+
     renderPendingLogWarnings() {
         const paths = this.getPaths();
         const now = Date.now();
@@ -499,48 +545,7 @@ export class LogViewer3 extends qreact.Component {
                                 <span>is not moving logs to remote storage. {formatNumber(warning.count)} files ({formatNumber(warning.totalSize)}B) are {formatTime(now - warning.oldestTime)} old</span>
                             </div>
                         ))}
-                        {!isPublic() && !readProductionLogsURL.value && <Button
-                            onClick={async () => {
-                                Querysub.commitLocal(() => {
-                                    this.state.forceMoveStartTime = Date.now();
-                                    this.state.forceMoveEndTime = undefined;
-                                });
-
-                                let forceMoveInterval = setInterval(() => {
-                                    this.forceUpdate();
-                                }, 100);
-
-                                try {
-                                    let loggers = await getLoggers2Async();
-                                    for (let logger of Object.values(loggers)) {
-                                        await logger.clientForceMoveLogsToPublic();
-                                    }
-                                    await this.loadPaths();
-                                } finally {
-                                    if (forceMoveInterval) {
-                                        clearInterval(forceMoveInterval);
-                                    }
-
-                                    Querysub.commitLocal(() => {
-                                        this.state.forceMoveEndTime = Date.now();
-                                    });
-                                }
-                            }}
-                            hue={180}
-                        >
-                            Force Move Logs to Public
-                        </Button>}
-                        {(() => {
-                            if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime === undefined) {
-                                const elapsed = Date.now() - this.state.forceMoveStartTime;
-                                return <div>Moving logs... {formatTime(elapsed)}</div>;
-                            }
-                            if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime !== undefined) {
-                                const duration = this.state.forceMoveEndTime - this.state.forceMoveStartTime;
-                                return <div>Moved logs in {formatTime(duration)}</div>;
-                            }
-                            return undefined;
-                        })()}
+                        {!isPublic() && !readProductionLogsURL.value && this.renderForceMoveLogs()}
                     </>
                 )}
             </div>
@@ -709,6 +714,7 @@ export class LogViewer3 extends qreact.Component {
                         number
                         url={limitURL}
                     />
+                    {!isPublic() && this.renderForceMoveLogs()}
                 </div>
 
                 <div className={css.hbox(10)}>

package/src/diagnostics/logs/IndexedLogs/{bufferMatcher.ts → bufferSearchFindMatcher.ts}

@@ -1,5 +1,5 @@
 import { measureWrap } from "socket-function/src/profiling/measure";
-import { Unit, getAllUnits } from "./BufferIndexHelpers";
+import { Unit, getAllUnits, remapSearchBuffer } from "./BufferIndexHelpers";
 
 const WILD_CARD_BYTE = 42;
 const WILD_CARD_CHAR = "*";
@@ -54,6 +54,8 @@ function parseMatchStructure(pattern: Buffer): MatchStructure {
 }
 // inside is and, outside is outer
 export function getSearchUnits(pattern: Buffer, disableSpecialCharacters: boolean): Unit[][] {
+    pattern = remapSearchBuffer(pattern);
+
     if (disableSpecialCharacters || !pattern.includes(SEARCH_OR_BYTE) && !pattern.includes(SEARCH_AND_BYTE) && !pattern.includes(WILD_CARD_BYTE)) {
         return [getAllUnits({ buffer: pattern, bufferIndex: 0, block: 0 }).map(u => u.unit)];
     }
@@ -107,9 +109,11 @@ function getSimplerStructure(structure: MatchStructure): Buffer[][] {
 // side must appear in order somewhere within buffer.
 // Returns a function that matches buffers against the pre-processed pattern.
 export function createMatchesPattern(pattern: Buffer, disableSpecialCharacters: boolean): (buffer: Buffer) => boolean {
+    pattern = remapSearchBuffer(pattern);
 
     if (disableSpecialCharacters || !pattern.includes(SEARCH_OR_BYTE) && !pattern.includes(SEARCH_AND_BYTE) && !pattern.includes(WILD_CARD_BYTE)) {
         return measureWrap(function matchesPattern(buffer: Buffer): boolean {
+            buffer = remapSearchBuffer(buffer);
             return buffer.indexOf(pattern) !== -1;
         }, "BufferIndex|matchesPatternExact");
     }
@@ -120,6 +124,7 @@ export function createMatchesPattern(pattern: Buffer, disableSpecialCharacters:
     let hasAnyWildcard = pattern.includes(WILD_CARD_BYTE);
 
     return measureWrap(function matchesPattern(buffer: Buffer): boolean {
+        buffer = remapSearchBuffer(buffer);
         // 1) First, use simplerStructure to exclude any false cases
         // simplerStructure is Buffer[][] where outer array is OR, inner array is AND
         let anyOrClausePassed = false;

package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts

@@ -147,7 +147,7 @@ export async function moveLogsToPublic(config: {
         }));
         if (buffers.length === 0) continue;
 
-        console.log(magenta(`Merging and moving ${group.length} log files backblaze`));
+        console.log(magenta(`Merging and moving ${group.length} log files backblaze (${blue(config.loggerName)})`));
 
         let allBuffers: Buffer[][] = [];
 
@@ -188,7 +188,7 @@ export async function moveLogsToPublic(config: {
             for (let buffer of group) {
                 uncompressedSize += buffer.length;
             }
-            let obj = BufferIndex.encodeAll({
+            let obj = await BufferIndex.encodeAll({
                 data: group,
             });
             encoded.push({
@@ -221,7 +221,7 @@ export async function moveLogsToPublic(config: {
             await localLogs.del(path.fullPath + indexExtension);
         }));
 
-        console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`));
+        console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
     }
 
     // Clean up orphaned index files (index files without corresponding data files)

package/src/diagnostics/logs/errorNotifications2/errorNotifications2.ts

@@ -1,3 +1,9 @@
 
 //todonext
-// FIRST, finish up our logs stuff, which is just about done...
+
+// single service, no history search, discord messaging, suppression and suppression page
+//  - Storing suppression history just in memory? (although suppression values, obviously, on disk)
+//  - receive all errors, and service will do suppression
+
+// Start with page itself showing errors and suppressing them, for debugging/testing
+// - Ai to automatically create suppression searches, and combine existing ones, etc

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -24,7 +24,6 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 2) Create lot of remote server logs
     - Via our refresh loop
 
-
 3) Verify search is still fast, and works correctly!
     - Try to get a rare value in that is in the logs, but is hard to find
     Good rare search. Checks for sending emails, which we do very rarely. If we pack in 10GB of logs after this, it will really stress our indexing system.
@@ -32,13 +31,6 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
     https://127-0-0-1.querysubtest.com:7007/?hot&readPublicLogs&showingmanagement&machineview=services&managementpage=LogViewer3&searchText=Sending%20email&selectedFields=%7B%22path%22%3Atrue%7D
     
 
-    
-Remove old error digest/notification code
-    - We don't need email digests anymore. I think discord messaging is enough
-Remove all old LogViewer/FastArchiveAppendable code
-    - Make sure to find all links and update them as well
-
-
 
 Rewrite error notification code
     - Single service, which talks to all machines