querysub 0.374.0 → 0.376.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.374.0",
+  "version": "0.376.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -34,9 +34,7 @@
     "join": "./bin/join.js",
     "join-public": "./bin/join-public.js",
     "movelogs": "./bin/movelogs.js",
-    "addsuperuser": "./bin/addsuperuser.js",
-    "error-email": "./bin/error-email.js",
-    "error-im": "./bin/error-im.js"
+    "addsuperuser": "./bin/addsuperuser.js"
   },
   "dependencies": {
     "@types/fs-ext": "^2.0.3",

package/src/deployManager/components/MachineDetailPage.tsx

@@ -7,10 +7,10 @@ import { formatNumber, formatVeryNiceDateTime } from "socket-function/src/format
 import { sort } from "socket-function/src/misc";
 import { ATag, Anchor } from "../../library-components/ATag";
 import { ShowMore } from "../../library-components/ShowMore";
-import { filterParam } from "../../diagnostics/logs/FastArchiveViewer";
 import { managementPageURL } from "../../diagnostics/managementPages";
 import { t } from "../../2-proxy/schema2";
 import { Querysub } from "../../4-querysub/QuerysubController";
+import { getLogViewerParams } from "../../diagnostics/logs/IndexedLogs/LogViewerParams";
 
 export class MachineDetailPage extends qreact.Component {
     render() {
@@ -105,10 +105,7 @@ export class MachineDetailPage extends qreact.Component {
                     <div>
                         Apply Node ID: {machine.applyNodeId}
                     </div>
-                    <ATag values={[
-                        managementPageURL.getOverride("LogViewer2"),
-                        filterParam.getOverride(`__machineId = ${machineInfo.machineId}`),
-                    ]}>
+                    <ATag values={getLogViewerParams({ __machineId: machineInfo.machineId })}>
                         Machine Logs
                     </ATag>
                 </div>

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -20,8 +20,8 @@ import { PrimitiveDisplay } from "../../diagnostics/logs/ObjectDisplay";
 import { parseAnsiColors, rgbToHsl } from "../../diagnostics/logs/ansiFormat";
 import { RenderGitRefInfo, UpdateServiceButtons, bigEmoji, buttonStyle } from "./deployButtons";
 import { TypedConfigEditor } from "../../library-components/TypedConfigEditor";
-import { filterParam } from "../../diagnostics/logs/FastArchiveViewer";
 import { managementPageURL } from "../../diagnostics/managementPages";
+import { getLogViewerParams } from "../../diagnostics/logs/IndexedLogs/LogViewerParams";
 
 
 
@@ -401,10 +401,7 @@ export class ServiceDetailPage extends qreact.Component {
                                     {isWatching ? "Stop Watching Output" : "Watch Screen Output"}
                                 </div>
 
-                                <ATag values={[
-                                    managementPageURL.getOverride("LogViewer2"),
-                                    filterParam.getOverride(`__machineId = ${machineId}`),
-                                ]}>
+                                <ATag values={getLogViewerParams({ __machineId: machineId })}>
                                     Machine Logs
                                 </ATag>
                             </div>

package/src/deployManager/machineApplyMainCode.ts

@@ -866,6 +866,13 @@ export async function machineApplyMain() {
         if (await quickIsOutdated()) {
             console.log(magenta("Likely outdated, resyncing now"));
             await resyncServices();
+        } else {
+            let config = await machineInfos.get(getOwnMachineId());
+            if (config) {
+                config.heartbeat = Date.now();
+                console.log(magenta(`Updating heartbeat for ${getOwnMachineId()} to ${config.heartbeat}`));
+                await machineInfos.set(getOwnMachineId(), config);
+            }
         }
     });
 

package/src/diagnostics/NodeViewer.tsx

@@ -36,8 +36,8 @@ import { SocketRegistered } from "socket-function/SocketFunctionTypes";
 import { ATag } from "../library-components/ATag";
 import { getSyncedController } from "../library-components/SyncedController";
 import child_process from "child_process";
-import { filterParam } from "./logs/FastArchiveViewer";
 import { getHTTPSKeyCert } from "../-e-certs/certAuthority";
+import { getLogViewerParams } from "./logs/IndexedLogs/LogViewerParams";
 
 
 type NodeData = {
@@ -296,10 +296,9 @@ export class NodeViewer extends qreact.Component {
                                         if (str.startsWith("http")) return formatValue(obj);
 
                                         return (
-                                            <ATag values={[
-                                                managementPageURL.getOverride("LogViewer2"),
-                                                filterParam.getOverride(`__nodeId = ${str}`),
-                                            ]}>
+                                            <ATag values={getLogViewerParams({
+                                                __nodeId: str,
+                                            })}>
                                                 Logs
                                             </ATag>
                                         );

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -10,7 +10,7 @@ import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profi
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { magenta, yellow } from "socket-function/src/formatting/logColors";
 import { Unit, getAllUnits, Reader, createOffsetReader, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
-import { createMatchesPattern, getSearchUnits } from "./bufferMatcher";
+import { createMatchesPattern, getSearchUnits } from "./bufferSearchFindMatcher";
 import { UnitSet } from "./BufferUnitSet";
 import { BufferUnitIndex } from "./BufferUnitIndex";
 import { BufferListStreamer } from "./BufferListStreamer";
@@ -178,13 +178,13 @@ export class BufferIndex {
 
     // IMPORTANT! The input data values should be sorted from newest to oldest. 
     @measureFnc
-    public static encodeAll(config: {
+    public static async encodeAll(config: {
         data: Buffer[];
-    }): {
+    }): Promise<{
         data: Buffer;
         index: Buffer;
-    } {
-        let obj = BufferUnitIndex.encode(config.data, {
+    }> {
+        let obj = await BufferUnitIndex.encode(config.data, {
             type: BULK_TYPE,
         });
         return {
@@ -453,6 +453,11 @@ export class BufferIndex {
             results.remoteIndexesSearched += 1;
             results.remoteIndexSize += index.length;
         } else {
+            let dataLength = await dataReader.getLength();
+            if (dataLength < 32) {
+                // If it's a small file, nothing of value will be lost by ignoring it. And almost all of our errors are coming from extremely small files. 
+                return;
+            }
             throw new Error(`Unknown type in index file: ${type}`);
         }
     }

package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.cpp

@@ -3,6 +3,20 @@
 #include <string.h>
 #include <algorithm>
 
+static uint32_t RemapUnit(uint32_t unit) {
+    uint8_t byte0 = (unit >> 0) & 0xFF;
+    uint8_t byte1 = (unit >> 8) & 0xFF;
+    uint8_t byte2 = (unit >> 16) & 0xFF;
+    uint8_t byte3 = (unit >> 24) & 0xFF;
+
+    if (byte0 >= 65 && byte0 <= 90) byte0 += 32;
+    if (byte1 >= 65 && byte1 <= 90) byte1 += 32;
+    if (byte2 >= 65 && byte2 <= 90) byte2 += 32;
+    if (byte3 >= 65 && byte3 <= 90) byte3 += 32;
+
+    return byte0 | (byte1 << 8) | (byte2 << 16) | (byte3 << 24);
+}
+
 static napi_value PopulateUnits(napi_env env, napi_callback_info info) {
     size_t argc = 2;
     napi_value args[2];
@@ -75,6 +89,8 @@ static napi_value PopulateUnits(napi_env env, napi_callback_info info) {
                                (bufferData[i+2] << 16) | 
                                (bufferData[i+3] << 24);
                 
+                unit = RemapUnit(unit);
+                
                 if (unit == 0) continue;
                 
                 unitsArray[currentOffset] = unit;
@@ -184,6 +200,8 @@ static napi_value BuildHashTable(napi_env env, napi_callback_info info) {
             // This reads 4 bytes starting at position i
             uint32_t unit = *(uint32_t*)(blockData + i);
             
+            unit = RemapUnit(unit);
+            
             uint32_t maskedHash = GetMaskedHash(unit, mask);
             uint32_t currentHead = hashTable[maskedHash];
             
@@ -291,6 +309,8 @@ static napi_value EstimateUniqueUnits(napi_env env, napi_callback_info info) {
             // Read unit directly by casting pointer (little-endian architecture)
             uint32_t unit = *(uint32_t*)(bufferData + i);
             
+            unit = RemapUnit(unit);
+            
             // Get masked hash with 0xFFFF mask (16-bit)
             uint32_t maskedHash = GetMaskedHash(unit, 0xFFFF);
             

package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts

@@ -16,7 +16,7 @@ export type SearchParams = {
     findBuffer: Buffer;
     pathOverrides?: TimeFilePathWithSize[];
     only?: "local" | "public";
-    forceReadPublic?: boolean;
+    forceReadProduction?: boolean;
 };
 
 export type Unit = number;
@@ -57,6 +57,33 @@ export function createOffsetReader(reader: Reader, offset: number): Reader {
     };
 }
 
+export function remapUnit(unit: Unit): Unit {
+    let byte0 = (unit >> 0) & 0xFF;
+    let byte1 = (unit >> 8) & 0xFF;
+    let byte2 = (unit >> 16) & 0xFF;
+    let byte3 = (unit >> 24) & 0xFF;
+
+    if (byte0 >= 65 && byte0 <= 90) byte0 += 32;
+    if (byte1 >= 65 && byte1 <= 90) byte1 += 32;
+    if (byte2 >= 65 && byte2 <= 90) byte2 += 32;
+    if (byte3 >= 65 && byte3 <= 90) byte3 += 32;
+
+    return (byte0 | (byte1 << 8) | (byte2 << 16) | (byte3 << 24)) >>> 0;
+}
+export function remapSearchBuffer(buffer: Buffer) {
+    let newBuffer = Buffer.alloc(buffer.length);
+    buffer.copy(newBuffer);
+    buffer = newBuffer;
+    for (let i = 0; i < buffer.length; i++) {
+        let byte = buffer[i];
+        if (byte >= 65 && byte <= 90) {
+            buffer[i] = byte + 32;
+        } else {
+            buffer[i] = byte;
+        }
+    }
+    return buffer;
+}
 
 export const getAllUnits = measureWrap(function getAllUnits(config: {
     buffer: Buffer;
@@ -71,7 +98,7 @@ export const getAllUnits = measureWrap(function getAllUnits(config: {
     const result: UnitRef[] = [];
     // For each byte position, create a unit from that position and the next 3 bytes
     for (let i = 0; i <= buffer.length - 4; i++) {
-        const unit = buffer.readUint32LE(i);
+        const unit = remapUnit(buffer.readUint32LE(i));
         if (!unit) continue;
 
         result.push({

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -3,7 +3,7 @@
 import { LZ4 } from "../../../storage/LZ4";
 import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
 import { Zip } from "../../../zip";
-import { BufferReader, Reader, SearchParams, IndexedLogResults, Unit } from "./BufferIndexHelpers";
+import { BufferReader, Reader, SearchParams, IndexedLogResults, Unit, remapUnit } from "./BufferIndexHelpers";
 import { formatNumber, formatPercent } from "socket-function/src/formatting/format";
 import { lazy } from "socket-function/src/caching";
 import { list, sort } from "socket-function/src/misc";
@@ -11,23 +11,64 @@ import { testDisableCache } from "../../../-a-archives/archivesMemoryCache";
 import { devDebugbreak } from "../../../config";
 import { BufferUnitIndexParallelSearchCount, DEFAULT_BLOCK_SIZE, DEFAULT_TARGET_UNITS_PER_BUCKET } from "./BufferIndexLogsOptimizationConstants";
 import { runInParallel } from "socket-function/src/batching";
-import { createMatchesPattern } from "./bufferMatcher";
+import { createMatchesPattern } from "./bufferSearchFindMatcher";
+import * as path from "path";
+import * as fs from "fs";
+import { runPromise } from "socket-function/src/runPromise";
+import * as crypto from "crypto";
 
 const USE_COMPRESSION = true;
 
+const getCppAddon = lazy(async () => {
+    if (typeof process === "undefined" || !process.versions || !process.versions.node) {
+        return undefined;
+    }
 
-// Lazy-load the C++ addon (only loaded when actually needed, not in client-side code)
-let getCppAddon = lazy(() => {
-    try {
-        // Check if we're in Node.js environment
-        if (typeof process === "undefined" || !process.versions || !process.versions.node) {
+    const checkNeedsBuild = (): boolean => {
+        const cppFilePath = path.join(__dirname, "BufferIndexCPP.cpp");
+        const distPath = path.join(__dirname, "dist");
+        const versionFilePath = path.join(distPath, "BufferIndexCPP.version");
+        const buildPath = path.join(__dirname, "build");
+
+        if (!fs.existsSync(buildPath)) {
+            return true;
+        }
+        if (!fs.existsSync(versionFilePath)) {
+            return true;
+        }
+        try {
+            const cppFileContent = fs.readFileSync(cppFilePath, "utf-8");
+            const currentHash = crypto.createHash("sha256").update(cppFileContent).digest("hex");
+            const storedHash = fs.readFileSync(versionFilePath, "utf-8").trim();
+            if (storedHash !== currentHash) {
+                return true;
+            }
+        } catch (e) {
+            return true;
+        }
+        return false;
+    };
+
+    if (checkNeedsBuild()) {
+        try {
+            await runPromise("node-gyp rebuild");
+            const cppFilePath = path.join(__dirname, "BufferIndexCPP.cpp");
+            const distPath = path.join(__dirname, "dist");
+            const versionFilePath = path.join(distPath, "BufferIndexCPP.version");
+            if (!fs.existsSync(distPath)) {
+                fs.mkdirSync(distPath, { recursive: true });
+            }
+            const cppFileContent = fs.readFileSync(cppFilePath, "utf-8");
+            const currentHash = crypto.createHash("sha256").update(cppFileContent).digest("hex");
+            fs.writeFileSync(versionFilePath, currentHash);
+        } catch (e) {
             return undefined;
         }
-        // Try to require the C++ addon
-        const addon = require("./BufferIndexCPP") as typeof import("./BufferIndexCPP");
-        return addon;
+    }
+
+    try {
+        return require("./BufferIndexCPP") as typeof import("./BufferIndexCPP");
     } catch (e) {
-        console.warn("C++ addon not available, falling back to TypeScript implementation", e);
         return undefined;
     }
 });
@@ -107,14 +148,14 @@ export class BufferUnitIndex {
 
     // IMPORTANT! The input data values should be sorted from newest to oldest. 
     @measureFnc
-    public static encode(allData: Buffer[], options: {
+    public static async encode(allData: Buffer[], options: {
         type: number;
         blockSize?: number;
         targetUnitsPerBucket?: number;
-    }): {
+    }): Promise<{
         blocks: Buffer;
         index: Buffer;
-    } {
+    }> {
         const type = options.type;
         const BLOCK_SIZE = options.blockSize ?? DEFAULT_BLOCK_SIZE;
         const TARGET_UNITS_PER_BUCKET = options.targetUnitsPerBucket ?? DEFAULT_TARGET_UNITS_PER_BUCKET;
@@ -199,8 +240,8 @@ export class BufferUnitIndex {
             return Buffer.concat([blocksHeaderBuffer, ...compressedBlocks]);
         }, `createFinalBlocksBuffer`);
 
-        const estimatedUniqueUnits = measureBlock(() => {
-            const cppAddon = getCppAddon();
+        const estimatedUniqueUnits = await measureBlock(async () => {
+            const cppAddon = await getCppAddon();
             if (cppAddon) {
                 // Use C++ addon for performance
                 return cppAddon.estimateUniqueUnits(allData);
@@ -213,7 +254,7 @@ export class BufferUnitIndex {
 
                 for (const buffer of allData) {
                     for (let i = 0; i <= buffer.length - 4; i++) {
-                        const unit = buffer.readUint32LE(i);
+                        const unit = remapUnit(buffer.readUint32LE(i));
                         const maskedHash = getMaskedHash(unit, 0xFFFF);
 
                         if (seenMaskedHashes[maskedHash] === 0) {
@@ -251,8 +292,8 @@ export class BufferUnitIndex {
 
 
         // Step 4: Build hash table with linked lists using Uint32Arrays for performance
-        const { hashTable, linkedListData, nodeCount, filledSlots } = measureBlock(() => {
-            const cppAddon = getCppAddon();
+        const { hashTable, linkedListData, nodeCount, filledSlots } = await measureBlock(async () => {
+            const cppAddon = await getCppAddon();
             if (cppAddon) {
                 // Use C++ addon for performance
                 return cppAddon.buildHashTable(blocksUncompressed, hashTableCapacity, mask);
@@ -298,7 +339,7 @@ export class BufferUnitIndex {
 
                     // Iterate through all units and add them directly
                     for (let i = 0; i <= blockUncompressed.length - 4; i++) {
-                        const unit = blockUncompressed.readUint32LE(i);
+                        const unit = remapUnit(blockUncompressed.readUint32LE(i));
                         const maskedHash = getMaskedHash(unit, mask);
 
                         const currentHead = hashTable[maskedHash];

package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts

@@ -1,7 +1,7 @@
 import { formatPercent, formatNumber } from "socket-function/src/formatting/format";
 import { red } from "socket-function/src/formatting/logColors";
 import { measureFnc } from "socket-function/src/profiling/measure";
-import { Unit } from "./BufferIndexHelpers";
+import { Unit, remapUnit } from "./BufferIndexHelpers";
 
 export class UnitSet {
     // Hash-based set for checking unit membership (no positions stored)
@@ -58,7 +58,7 @@ export class UnitSet {
                     const buffer = block[bufferIndex];
                     // Extract units directly
                     for (let i = 0; i <= buffer.length - 4; i++) {
-                        const unit = buffer.readUint32LE(i);
+                        const unit = remapUnit(buffer.readUint32LE(i));
                         if (!unit) continue;
 
                         totalInserts++;

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -272,10 +272,10 @@ export class IndexedLogs<T> {
         startTime: number;
         endTime: number;
         only?: "local" | "public";
-        forceReadPublic?: boolean;
+        forceReadProduction?: boolean;
     }): Promise<TimeFilePathWithSize[]> {
         let finalPaths: TimeFilePathWithSize[] = [];
-        if (config.forceReadPublic && !isPublic()) {
+        if (config.forceReadProduction && !isPublic()) {
             let machineNodes = await this.getMachineNodes();
             if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
             return await IndexedLogShimController.nodes[machineNodes[0]].getPaths({
@@ -352,7 +352,7 @@ export class IndexedLogs<T> {
         onResults?: (results: IndexedLogResults) => Promise<boolean>;
     }): Promise<IndexedLogResults> {
 
-        if (config.params.forceReadPublic && !isPublic()) {
+        if (config.params.forceReadProduction && !isPublic()) {
             let machineNodes = await this.getMachineNodes();
             if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
             return await this.clientFind({
@@ -649,7 +649,7 @@ export class IndexedLogs<T> {
         startTime: number;
         endTime: number;
         only?: "local" | "public";
-        forceReadPublic?: boolean;
+        forceReadProduction?: boolean;
 
     }): Promise<TimeFilePathWithSize[]> {
         let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
@@ -658,7 +658,7 @@ export class IndexedLogs<T> {
             startTime: config.startTime,
             endTime: config.endTime,
             only: config.only,
-            forceReadPublic: config.forceReadPublic,
+            forceReadProduction: config.forceReadProduction,
         });
     }
     public onFindResult(config: {
@@ -778,7 +778,7 @@ class IndexedLogShim {
         startTime: number;
         endTime: number;
         only?: "local" | "public";
-        forceReadPublic?: boolean;
+        forceReadProduction?: boolean;
     }): Promise<TimeFilePathWithSize[]> {
         let indexedLogs = loggerByName.get(config.indexedLogsName);
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
@@ -786,7 +786,7 @@ class IndexedLogShim {
             startTime: config.startTime,
             endTime: config.endTime,
             only: config.only,
-            forceReadPublic: config.forceReadPublic,
+            forceReadProduction: config.forceReadProduction,
         });
     }