querysub 0.440.0 → 0.442.0

package/src/4-querysub/Querysub.ts

@@ -32,7 +32,7 @@ import { inlineNestedCalls, syncSchema } from "../3-path-functions/syncSchema";
 import type { identityStorageKey, IdentityStorageType } from "../-a-auth/certs";
 
 import { ExternalRenderClass, qreact } from "../4-dom/qreact";
-import { configRootDiscoveryLocation, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { configRootDiscoveryLocation, getOwnNodeId, onNodeBroadcasted, onNodeDiscoveryReady } from "../-f-node-discovery/NodeDiscovery";
 import { pathValueCommitter } from "../0-path-value-core/PathValueCommitter";
 import debugbreak from "debugbreak";
 import { registerDynamicResource } from "../diagnostics/trackResources";
@@ -183,6 +183,9 @@ export class Querysub {
     public static AUDIT_PREDICTIONS = true;
     public static SIMULATE_LAG = 0;
 
+    public static registerAliveChecker = <T>(config: AliveChecker<T>) => registerAliveChecker(config);
+    public static registerGarbageCollection = <T>(config: AliveChecker<T>) => registerAliveChecker(config);
+
     /** Delay used when functions are specified as delayCommit */
     public static DELAY_COMMIT_DELAY = 1000 * 3;
 
@@ -287,11 +290,16 @@ export class Querysub {
      *          isn't presently necessary in most serverside scripts.
      */
     @measureFnc
-    public static async optionalStartupWait() {
+    public static async optionalStartupWait(addTime?: (name: string) => void) {
+        // Used by authorityLookup, pulled out here so timing is more clear.
+        await onNodeDiscoveryReady();
+        addTime?.("onNodeDiscoveryReady");
         await authorityLookup.startSyncing();
+        addTime?.("authorityLookup.startSyncing");
         await measureBlock(async () => {
             await waitForFirstTimeSync();
         }, "waitForFirstTimeSync");
+        addTime?.("waitForFirstTimeSync");
     }
 
     public static createWatcher(watcher: (obj: SyncWatcher) => void, options?: Partial<WatcherOptions<unknown>>): {
@@ -1046,18 +1054,30 @@ export class Querysub {
         if (isClient()) {
             throw new Error(`--client processes cannot host a service. Either stop passing --client and keep the process on the network and trusted, or stop calling hostServer and call Querysub.configRootDiscoveryLocation instead.`);
         }
+        let times: {
+            name: string;
+            duration: number
+        }[] = [];
+        let time = Date.now();
+        function addTime(name: string) {
+            times.push({ name, duration: Date.now() - time });
+            time = Date.now();
+        }
 
         this.socketFunctionInit();
-        await SocketFunction.mount({
+        let mountPromise = SocketFunction.mount({
             public: isPublic(),
             port,
             ...await getThreadKeyCert(),
         });
 
-        await publishMachineARecords();
+        let publishPromise = publishMachineARecords();
+        await Promise.all([mountPromise, publishPromise]);
+        addTime("mount & publish a records");
+
+        await Querysub.optionalStartupWait(addTime);
 
-        await Querysub.optionalStartupWait();
-        console.log(magenta(`Started hosting service ${name}`));
+        console.log(magenta(`Started hosting service ${name}`) + ` | ${times.map(t => `${blue(t.name)}: ${green(formatTime(t.duration))}`).join(" | ")}`);
     }
 
     /** Syncs keys AND values (as we won't return a key for a value that is undefined). */
@@ -1330,7 +1350,7 @@ import "../diagnostics/trackResources";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { getCountPerPaint } from "../functional/onNextPaint";
 import { addEpsilons } from "../bits";
-import { blue, magenta } from "socket-function/src/formatting/logColors";
+import { blue, green, magenta } from "socket-function/src/formatting/logColors";
 import { MachineController } from "../deployManager/machineController";
 import { getRecords, setRecord } from "../-b-authorities/dnsAuthority";
 import { testTCPIsListening } from "socket-function/src/networking";
@@ -1339,4 +1359,5 @@ import { onAllPredictionsFinished } from "../-0-hooks/hooks";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathRouter";
 import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
 import { encodeParentFilter } from "../0-path-value-core/hackedPackedPathParentFiltering";
+import { AliveChecker, registerAliveChecker } from "../2-proxy/garbageCollection";
 

package/src/4-querysub/QuerysubController.ts

@@ -40,6 +40,7 @@ import { getDomain, isBootstrapOnly } from "../config";
 import { flushPredictionQueueBase, runInPredictionQueue, syncHasPendingPredictionsBase } from "./predictionQueue";
 import { PathRouter } from "../0-path-value-core/PathRouter";
 import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
+import { PathValueArchives } from "../0-path-value-core/pathValueArchives";
 
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("fncfilter", { type: "string", default: "", desc: `Sets the filterable state for function calls, causing them to target specific FunctionRunners. If no FunctionRunner matches, all functions will fail to run. For example: "devtestserver" will match a FunctionRunner that uses the "devtestserver" filter. Merges with the existing filterable state if a client sets it explicitly.` })
@@ -490,7 +491,7 @@ export class QuerysubControllerBase {
 
             if (removedPaths.size > 0) {
                 let [removedParentPaths, removedJustPaths] = splitParentPaths(removedPaths);
-                pathWatcher.unwatchPath({ callback: callerId, paths: removedJustPaths, parentPaths: removedParentPaths });
+                pathWatcher.unwatchPath({ callback: callerId, paths: removedJustPaths, parentPaths: removedParentPaths, reason: "QuerysubController.watch removed paths (permissions)" });
             }
             if (allowedPaths.size > 0) {
                 let [newParentPathsAllowed, newPathsAllowed] = splitParentPaths(allowedPaths);
@@ -531,7 +532,7 @@ export class QuerysubControllerBase {
                 delPermissionsPath(appendToPathStr(path, ""), true);
             }
         }
-        pathWatcher.unwatchPath({ callback: callerId, ...config });
+        pathWatcher.unwatchPath({ callback: callerId, ...config, reason: "QuerysubController.unwatch" });
     }
 
     // NOTE: Calls are going to be temporary and random. Any user can use any call ID, so technically you could clobber other users' call IDs, or your our. There wouldn't really be any benefit. Nothing would really happen if you do that, so I don't believe these need to be kept secret. I think if you know someone else's call ID you might be able to read that data, but also it's securely random, so you're not going to be able to guess the call ID.
@@ -686,6 +687,10 @@ export class QuerysubControllerBase {
         }
         return result;
     }
+
+    public async debugGetValuePathCount(): Promise<number> {
+        return PathValueArchives.getValuePathCount();
+    }
 }
 
 export const QuerysubController = SocketFunction.register(
@@ -696,11 +701,13 @@ export const QuerysubController = SocketFunction.register(
         watch: { compress: true, },
         unwatch: { compress: true, },
         addCall: { compress: true, },
+        // NOTE: Most of these debug functions are pretty innocuous. A lot of it is actually already exposed, and other parts of it is fine. It's fine for the user to know what node a path is on. It's fine for them to know how many value paths there are.
         debugGetPathNodeIds: {},
         debugGetNodeSpecs: {},
         debugGetSingleReadNode: {},
         getModulePath: {},
         getDevFunctionSpecFromCall: {},
+        debugGetValuePathCount: {},
     }),
     () => ({
 

package/src/archiveapps/archiveJoinEntry.ts

@@ -110,7 +110,7 @@ async function runGenesisJoinIteration(config?: { force?: boolean }) {
                 });
             }
 
-            console.log(magenta(`Joining ${formatNumber(usedFiles.length)} files with ${formatNumber(allCombinedValues.length)} values in ${formatNumber(totalSize)} bytes. Original count: ${formatNumber(originalCount)}, under path count: ${formatNumber(underPathCount)}, within time range count: ${formatNumber(withinTimeRangeCount)}`));
+            console.log(magenta(`Joining ${formatNumber(usedFiles.length)} => ${transaction.createFiles.length} files with ${formatNumber(allCombinedValues.length)} values in ${formatNumber(totalSize)} bytes. Original count: ${formatNumber(originalCount)}, under path count: ${formatNumber(underPathCount)}, within time range count: ${formatNumber(withinTimeRangeCount)}`));
 
             return [transaction];
         });

package/src/diagnostics/ValuePathWarning.tsx

@@ -0,0 +1,68 @@
+import { qreact } from "../4-dom/qreact";
+import { css } from "../4-dom/css";
+import { Querysub, QuerysubController, querysubNodeId } from "../4-querysub/QuerysubController";
+import { isCurrentUserSuperUser } from "../user-implementation/userData";
+import { t } from "../2-proxy/schema2";
+
+module.hotreload = true;
+
+export class ValuePathWarning extends qreact.Component {
+    state = t.state({
+        count: t.number
+    });
+    componentDidMount() {
+        Querysub.onCommitFinished(async () => {
+            let nodeId = await querysubNodeId();
+            if (!nodeId) return;
+            let count = await QuerysubController.nodes[nodeId].debugGetValuePathCount();
+            Querysub.localCommit(() => this.state.count = count);
+        });
+    }
+    renderBase() {
+        if (!isCurrentUserSuperUser()) return undefined;
+        let count = this.state.count;
+        if (!count) return undefined;
+
+        if (count <= 150) {
+            return <div className={css.hbox(4)}>
+                <span>📄</span>
+                <span>{count}</span>
+            </div>;
+        }
+
+        let bgClass = "";
+        let flashing = false;
+        if (count > 500) {
+            bgClass = css.hsl(0, 80, 60);
+            flashing = true;
+        } else if (count > 300) {
+            bgClass = css.hsl(0, 80, 60);
+        } else {
+            bgClass = css.hsl(50, 100, 40);
+        }
+
+        let animClassName = "ValuePathWarning-flash";
+        return <div className={css.hbox(4).pad2(6, 2) + " " + bgClass + (flashing ? " " + animClassName : "")}>
+            <span>📄</span>
+            <span>{count}</span>
+            <span>warning: high number of files, join is likely not running</span>
+            {flashing && <style>{`
+                @keyframes ${animClassName}-anim {
+                    0%, 100% { background-color: hsl(0, 80%, 60%); }
+                    50% { background-color: hsl(0, 80%, 30%); }
+                }
+                .${animClassName} {
+                    animation: ${animClassName}-anim 0.6s infinite;
+                }
+            `}</style>}
+        </div>;
+    }
+    render() {
+        try {
+            return this.renderBase();
+        } catch (error) {
+            console.error("Error in rendering ValuePathWarning:", error);
+            return undefined;
+        }
+    }
+}

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -9,7 +9,7 @@ import { cacheArgsEqual, cacheLimited, cacheWeak, lazy } from "socket-function/s
 import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { magenta, yellow } from "socket-function/src/formatting/logColors";
-import { Unit, getAllUnits, Reader, createOffsetReader, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
+import { Unit, getAllUnits, Reader, BufferReader, createOffsetReader, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
 import { createMatchesPattern, getSearchUnits } from "./bufferSearchFindMatcher";
 import { UnitSet } from "./BufferUnitSet";
 import { BufferUnitIndex } from "./BufferUnitIndex";
@@ -409,6 +409,118 @@ export class BufferIndex {
         results.blockSearchTime += Date.now() - blockSearchTimeStart;
     }
 
+    // Returns the block indices in `index` that could contain a match for `query`.
+    // Mirrors the AND/OR scan inside findLocal and BufferUnitIndex.find but does
+    // not read or scan data — only the index. Used by MCPIndexedLogs to split
+    // search into "find candidate blocks" then "scan those blocks".
+    @measureFnc
+    public static async findMatchingBlocks(config: {
+        index: Buffer;
+        dataReader: Reader;
+        query: Buffer;
+        disableWildCards?: boolean;
+        results: IndexedLogResults;
+    }): Promise<number[]> {
+        let { index, dataReader, query, results } = config;
+        let allSearchUnits = getSearchUnits(query, !!config.disableWildCards);
+        if (allSearchUnits.length === 0) return [];
+
+        let type = index[0];
+        if (!type) {
+            type = (await dataReader.read(0, 1))?.[0];
+        }
+
+        if (type === STREAM_TYPE) {
+            if (index.length === 0) {
+                index = await BufferIndex.rebuildLocalIndexFromData(dataReader);
+                if (index.length === 0) return [];
+            }
+            index = await BufferIndex.fixPartialIndex({ index, dataReader, results });
+
+            let decoded = decodeTypeHeader(index);
+            if (!decoded) return [];
+            let indexEntries = await indexStreamerType.getAllBlocks(decoded.data);
+
+            let matching: number[] = [];
+            for (let i = 0; i < indexEntries.length; i++) {
+                let blockIndexData = indexEntries[i];
+                for (let or of allSearchUnits) {
+                    let hasAllUnits = true;
+                    for (let unit of or) {
+                        if (!UnitSet.has(blockIndexData, unit)) {
+                            hasAllUnits = false;
+                            break;
+                        }
+                    }
+                    if (hasAllUnits) {
+                        matching.push(i);
+                        break;
+                    }
+                }
+            }
+            return matching;
+        } else if (type === BULK_TYPE) {
+            let candidateSet = new Set<number>();
+            for (let or of allSearchUnits) {
+                let blocks = BufferUnitIndex.findBlocks({ units: or, index });
+                for (let b of blocks) candidateSet.add(b);
+            }
+            let result = Array.from(candidateSet);
+            sort(result, x => x);
+            return result;
+        }
+        return [];
+    }
+
+    // Loads and returns just the buffers in a single block, by index.
+    // For STREAM_TYPE, blockIndex matches the index entry order produced by
+    // findMatchingBlocks. For BULK_TYPE, blockIndex matches BufferUnitIndex's
+    // internal block ordering (also what findMatchingBlocks returns).
+    @measureFnc
+    public static async getBlockBuffers(config: {
+        index: Buffer;
+        dataReader: Reader;
+        blockIndex: number;
+    }): Promise<Buffer[]> {
+        let { index, dataReader, blockIndex } = config;
+
+        let type = index[0];
+        if (!type) {
+            type = (await dataReader.read(0, 1))?.[0];
+        }
+
+        if (type === STREAM_TYPE) {
+            let headerBuf = await dataReader.read(0, 5);
+            if (headerBuf.length < 5) return [];
+            let headerSize = headerBuf.readInt32LE(1);
+            let totalHeaderSize = 1 + 4 + headerSize;
+            let dataWithoutHeaderReader = createOffsetReader(dataReader, totalHeaderSize);
+
+            let blocks = await dataStreamerType.getBlockRange({
+                reader: dataWithoutHeaderReader,
+                startIndex: blockIndex,
+                endIndex: blockIndex + 1,
+            });
+            if (blocks.length === 0) return [];
+            try {
+                let decompressed = CompressedStream.decode(blocks[0]);
+                return await blockStreamerType.getAllBlocks(decompressed);
+            } catch (e) {
+                return [];
+            }
+        } else if (type === BULK_TYPE) {
+            let obj = await BufferUnitIndex.getBlock(dataReader, blockIndex);
+            let blockReader = new BufferReader(obj.block);
+            let bufferCount = await BufferUnitIndex.getBufferCountFromBlock(blockReader);
+            let result: Buffer[] = [];
+            for (let i = 0; i < bufferCount; i++) {
+                result.push(await BufferUnitIndex.getBufferFromBlock(blockReader, i));
+            }
+            return result;
+        }
+        return [];
+    }
+
     @measureFnc
     public static async find(config: {
         index: Buffer;

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -611,7 +611,7 @@ export class BufferUnitIndex {
     }
 
     @measureFnc
-    private static findBlocks(config: {
+    public static findBlocks(config: {
         units: number[];
         index: Buffer;
     }): number[] {
@@ -679,12 +679,12 @@ export class BufferUnitIndex {
     }
 
 
-    private static async getBlockCount(reader: Reader): Promise<number> {
+    public static async getBlockCount(reader: Reader): Promise<number> {
         const headerBuffer = await reader.read(4, 8);
         return headerBuffer.readUInt32LE(0);
     }
 
-    private static async getBlock(reader: Reader, blockIndex: number, debugOffsets?: {
+    public static async getBlock(reader: Reader, blockIndex: number, debugOffsets?: {
         startOffset: number;
         endOffset: number;
     }): Promise<{
@@ -715,7 +715,7 @@ export class BufferUnitIndex {
         return (await reader.read(0, 4)).readUInt32LE(0);
     }
 
-    private static async getBufferFromBlock(reader: Reader, bufferIndex: number): Promise<Buffer> {
+    public static async getBufferFromBlock(reader: Reader, bufferIndex: number): Promise<Buffer> {
         let startOffset = (await reader.read(4 + bufferIndex * 4, 4)).readUInt32LE(0);
         let endOffset = (await reader.read(4 + (bufferIndex + 1) * 4, 4)).readUInt32LE(0);
         let buffer = await reader.read(startOffset, endOffset - startOffset);

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -1,4 +1,4 @@
-import { lazy } from "socket-function/src/caching";
+import { cache, lazy } from "socket-function/src/caching";
 import { Archives, nestArchives } from "../../../-a-archives/archives";
 import { deepCloneJSON, keyByArray, nextId, sort, timeInHour, timeInMinute, timeInSecond, timeoutToUndefinedSilent } from "socket-function/src/misc";
 import { BufferIndex } from "./BufferIndex";
@@ -27,6 +27,7 @@ import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
 import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
 import { getLoggers2Async } from "../diskLogger";
 import { watchAllValues } from "../errorNotifications2/logWatcher";
+import { wrapArchivesWithCache } from "../../../-a-archives/archiveCache";
 
 // Ensure it's available so that the controller is listening on all servers
 watchAllValues;
@@ -113,6 +114,23 @@ export class IndexedLogs<T> {
         });
         return archives;
     };
+    public debugGetCachedLogs = cache((config: {
+        type: "local" | "public";
+    }) => {
+        let usePublic = config.type === "public";
+        let archives = usePublic ? getArchivesBackblaze(getDomain()) : getArchivesHome(getDomain());
+        archives = nestArchives("final-indexed-logs/" + this.config.name, archives);
+        archives = wrapArchivesWithCache(archives);
+        archives = createArchivesMemoryCache(archives, {
+            maxSize: 1024 * 1024 * 1024 * 12,
+            maxCount: 1000 * 500,
+            fullyImmutable: true,
+        });
+        return archives;
+    });
+    public debugIsPublic() {
+        return isPublic();
+    }
     private getPublicLogs = lazy((): Archives => {
         return this.getPublicLogsBase(isPublic());
     });
@@ -756,14 +774,20 @@ export class IndexedLogs<T> {
     }
 }
 
+
+function getLogByName(name: string): IndexedLogs<unknown> {
+    let indexedLogs = loggerByName.get(name);
+    if (!indexedLogs) throw new Error(`Indexed logs ${name} not found, have ${Array.from(loggerByName.keys()).join(" | ")}`);
+    return indexedLogs;
+}
 class IndexedLogClient {
+
     public async onFind(config: {
         findId: string;
         indexedLogsName: string;
         result: unknown;
     }) {
-        let indexedLogs = loggerByName.get(config.indexedLogsName);
-        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        let indexedLogs = getLogByName(config.indexedLogsName);
         indexedLogs.onFindResult({
             findId: config.findId,
             result: config.result,
@@ -774,8 +798,7 @@ class IndexedLogClient {
         indexedLogsName: string;
         results: IndexedLogResults;
     }): Promise<boolean> {
-        let indexedLogs = loggerByName.get(config.indexedLogsName);
-        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        let indexedLogs = getLogByName(config.indexedLogsName);
         return await indexedLogs.onResults({
             findId: config.findId,
             results: config.results,
@@ -789,8 +812,7 @@ class IndexedLogShim {
         params: SearchParams;
     }): Promise<IndexedLogResults> {
         let caller = SocketFunction.getCaller();
-        let indexedLogs = loggerByName.get(config.indexedLogsName);
-        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        let indexedLogs = getLogByName(config.indexedLogsName);
 
         return indexedLogs.find({
             params: config.params,
@@ -822,8 +844,7 @@ class IndexedLogShim {
         only?: "local" | "public";
         forceReadProduction?: boolean;
     }): Promise<TimeFilePathWithSize[]> {
-        let indexedLogs = loggerByName.get(config.indexedLogsName);
-        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        let indexedLogs = getLogByName(config.indexedLogsName);
         return indexedLogs.getPaths({
             startTime: config.startTime,
             endTime: config.endTime,
@@ -835,17 +856,30 @@ class IndexedLogShim {
     public async forceMoveLogsToPublic(config: {
         indexedLogsName: string;
     }) {
-        let indexedLogs = loggerByName.get(config.indexedLogsName);
-        if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
+        let indexedLogs = getLogByName(config.indexedLogsName);
         await indexedLogs.moveLogsToPublic(true);
     }
 
     public async hasLogger(name: string) {
         return loggerByName.has(name);
     }
+
+    public async hasPendingInRange(config: {
+        indexedLogsName: string;
+        startTime: number;
+        endTime: number;
+    }): Promise<boolean> {
+        let logs = getLogByName(config.indexedLogsName);
+        let paths = await logs.getPaths({
+            startTime: config.startTime,
+            endTime: config.endTime,
+            only: "local",
+        });
+        return paths.some(p => p.logCount === undefined);
+    }
 }
 
-const IndexedLogShimController = SocketFunction.register(
+export const IndexedLogShimController = SocketFunction.register(
     "IndexedLogShim-019c87b7-73ca-72ec-91b3-2d45ebb616cd",
     new IndexedLogShim(),
     () => ({
@@ -860,6 +894,9 @@ const IndexedLogShimController = SocketFunction.register(
         },
         hasLogger: {
             hooks: [assertIsManagementUser]
+        },
+        hasPendingInRange: {
+            hooks: [assertIsManagementUser]
         }
     })
 );