querysub 0.460.0 → 0.462.0

package/test.ts

@@ -1,197 +1,56 @@
 import { chdir } from "process";
 chdir("D:/repos/qs-cyoa/");
 
-import { isPublic, setIsPublic } from "./src/config";
-setIsPublic(true);
+import { setIsPublic } from "./src/config";
+setIsPublic(false);
 
 import "./inject";
 import { Querysub } from "./src/4-querysub/Querysub";
 import { getLoggers2Async, LogDatum } from "./src/diagnostics/logs/diskLogger";
-import { IndexedLogs } from "./src/diagnostics/logs/IndexedLogs/IndexedLogs";
 import { SearchParams } from "./src/diagnostics/logs/IndexedLogs/BufferIndexHelpers";
-import { formatDateTimeDetailed, formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { formatDateTimeDetailed, formatTime } from "socket-function/src/formatting/format";
 
-// Pulled verbatim from the URLs the user shared.
 const START_TIME = 1779598800000;
 const END_TIME = 1779604200000;
 const LIMIT = 1600;
-const SEARCH_BROAD = `wvupofthbgq & "__threadId":"1f72e0ea774fcc81"`;
-const SEARCH_NARROW = `wvupofthbgq & "__threadId":"1f72e0ea774fcc81" & new`;
+const SEARCH = `wvupofthbgq & "__threadId":"1f72e0ea774fcc81"`;
+const NODE_ID = "46456fa53a7392cf.a794fbcf7b104c68.querysubtest.com:44173";
 
-type Emit = {
-    time: number;
-    logger: string;
-    datum: LogDatum;
-};
-
-// Stable identity for cross-query comparison. `time` alone isn't unique (many
-// entries share the same ms), so we fold in threadId + entry text.
-function emitKey(e: Emit): string {
-    return `${e.time}|${e.datum.__threadId ?? ""}|${e.datum.__entry ?? ""}|${e.datum.param0 ?? ""}`;
-}
+async function main() {
+    await Querysub.hostService("test");
 
-async function runQuery(label: string, searchText: string, limit: number = LIMIT): Promise<Emit[]> {
-    console.log(`\n=== ${label}: ${JSON.stringify(searchText)} (limit=${limit}) ===`);
     let loggers = await getLoggers2Async();
-    let allLoggers: { name: string; logger: IndexedLogs<LogDatum> }[] = [
-        { name: "info", logger: loggers.infoLogs },
-    ];
+    let infoLogs = loggers.infoLogs;
 
     let params: SearchParams = {
         startTime: START_TIME,
         endTime: END_TIME,
-        limit,
-        findBuffer: Buffer.from(searchText, "utf8"),
+        limit: LIMIT,
+        findBuffer: Buffer.from(SEARCH, "utf8"),
         searchFromStart: true,
-        only: "public",
+        forceReadProduction: true,
     };
 
-    let allEmits: Emit[] = [];
-    let queryStart = Date.now();
-
-    await Promise.all(allLoggers.map(async ({ name, logger }) => {
-        let perLoggerEmits: Emit[] = [];
-        let loggerStart = Date.now();
-        let result = await logger.find({
-            params,
-            onResult: (match: LogDatum) => {
-                perLoggerEmits.push({ time: match.time, logger: name, datum: match });
-            },
-        });
-
-        let earliest = perLoggerEmits.length > 0 ? Math.min(...perLoggerEmits.map(e => e.time)) : undefined;
-        let latest = perLoggerEmits.length > 0 ? Math.max(...perLoggerEmits.map(e => e.time)) : undefined;
-        console.log(
-            `  [${name}] emits=${perLoggerEmits.length} ` +
-            `matchCount=${result.matchCount} ` +
-            `blocksChecked=${result.blockCheckedCount}/${result.totalBlockCount} ` +
-            `filesScanned=${result.backblazeFilesSearched}/${result.totalBackblazeFiles} ` +
-            `earliest=${earliest !== undefined ? formatDateTimeDetailed(earliest) : "—"} ` +
-            `latest=${latest !== undefined ? formatDateTimeDetailed(latest) : "—"} ` +
-            `time=${formatTime(Date.now() - loggerStart)}`
-        );
-        allEmits.push(...perLoggerEmits);
-    }));
-
-    console.log(`  total emits=${allEmits.length} in ${formatTime(Date.now() - queryStart)}`);
-
-    // Sort + trim to limit (mirroring the client-side display).
-    allEmits.sort((a, b) => a.time - b.time);
-    if (allEmits.length > limit) allEmits.length = limit;
-
+    let emits: LogDatum[] = [];
+    let start = Date.now();
+    let result = await infoLogs.clientFind({
+        params,
+        nodeId: NODE_ID,
+        onResult: (match) => {
+            emits.push(match);
+        },
+    });
+    let elapsed = Date.now() - start;
+    let earliest = emits.length > 0 ? Math.min(...emits.map(e => e.time)) : undefined;
+    let latest = emits.length > 0 ? Math.max(...emits.map(e => e.time)) : undefined;
     console.log(
-        `  kept top-${allEmits.length} ` +
-        `earliest=${allEmits.length > 0 ? formatDateTimeDetailed(allEmits[0].time) : "—"} ` +
-        `latest=${allEmits.length > 0 ? formatDateTimeDetailed(allEmits[allEmits.length - 1].time) : "—"}`
+        `emits=${emits.length} matchCount=${result?.matchCount} ` +
+        `blocksChecked=${result?.blockCheckedCount}/${result?.totalBlockCount} ` +
+        `filesScanned=${result?.backblazeFilesSearched}/${result?.totalBackblazeFiles} ` +
+        `earliest=${earliest !== undefined ? formatDateTimeDetailed(earliest) : "—"} ` +
+        `latest=${latest !== undefined ? formatDateTimeDetailed(latest) : "—"} ` +
+        `time=${formatTime(elapsed)}`
     );
-    return allEmits;
-}
-
-async function main() {
-    await Querysub.hostService("test");
-
-    // Dump every info file in range — declared startTime / endTime — so we
-    // can spot files whose declared range disagrees with the entries inside.
-    let loggers = await getLoggers2Async();
-    let paths = await loggers.infoLogs.getPaths({
-        startTime: START_TIME,
-        endTime: END_TIME,
-        only: "public",
-    });
-    paths.sort((a, b) => a.startTime - b.startTime);
-    console.log(`\n=== INFO FILES IN RANGE (${paths.length}) — declared ranges ===`);
-    for (let p of paths) {
-        console.log(`  start=${formatDateTimeDetailed(p.startTime)} end=${formatDateTimeDetailed(p.endTime)} logCount=${p.logCount ?? "?"} ${p.fullPath}`);
-    }
-
-    let broad = await runQuery("BROAD", SEARCH_BROAD);
-    let narrow = await runQuery("NARROW", SEARCH_NARROW);
-    // Sanity: run broad again with a huge limit. If those 4 entries appear
-    // here but not in the limit=1600 run, the per-file `stopIterating` cap is
-    // skipping the block that contains them. If they're still missing, the
-    // bug is upstream (index pre-filter or block scanner missing them).
-    let broadHuge = await runQuery("BROAD_HUGE", SEARCH_BROAD, 1_000_000);
-
-    // The diagnostic: narrow ⊂ broad by definition. So every narrow result
-    // whose time falls inside broad's kept window MUST appear in broad. If any
-    // are missing, the broad scan dropped them — that's the bug.
-    if (broad.length === 0 || narrow.length === 0) {
-        console.log(`\nSkipping comparison: broad=${broad.length} narrow=${narrow.length}`);
-        return;
-    }
-
-    let broadKeys = new Set(broad.map(emitKey));
-    let broadCutoff = broad[broad.length - 1].time;
-    let broadEarliest = broad[0].time;
-
-    console.log(`\n=== COMPARE ===`);
-    console.log(`broad window: [${formatDateTimeDetailed(broadEarliest)}, ${formatDateTimeDetailed(broadCutoff)}]`);
-
-    let narrowInWindow = narrow.filter(n => n.time <= broadCutoff);
-    let missing = narrowInWindow.filter(n => !broadKeys.has(emitKey(n)));
-
-    console.log(`narrow total: ${narrow.length}`);
-    console.log(`narrow within broad window (<= broad cutoff): ${narrowInWindow.length}`);
-    console.log(`narrow missing from broad kept top-K: ${missing.length}`);
-
-    if (missing.length > 0) {
-        console.log(`\nFirst ${Math.min(20, missing.length)} missing entries (these prove broad dropped them):`);
-        for (let m of missing.slice(0, 20)) {
-            console.log(
-                `  time=${formatDateTimeDetailed(m.time)} ` +
-                `logger=${m.logger} ` +
-                `entry=${(m.datum.__entry ?? "").slice(0, 80)} ` +
-                `param0=${String(m.datum.param0 ?? "").slice(0, 80)}`
-            );
-        }
-
-        // Group missing by logger so we can tell which scan dropped them.
-        let byLogger = new Map<string, number>();
-        for (let m of missing) byLogger.set(m.logger, (byLogger.get(m.logger) ?? 0) + 1);
-        console.log(`\nMissing by logger:`);
-        for (let [k, v] of byLogger) console.log(`  ${k}: ${formatNumber(v)}`);
-    } else {
-        console.log(`\nNo missing entries — broad correctly contains all narrow results in its window.`);
-    }
-
-    // For each missing entry, identify the file whose declared range *should*
-    // cover its time, and find the file whose declared range *doesn't* cover
-    // it (the bug indicator).
-    if (missing.length > 0) {
-        let missingTimes = missing.map(m => m.time);
-        let minMissing = Math.min(...missingTimes);
-        let maxMissing = Math.max(...missingTimes);
-        console.log(`\nMissing entry times span: [${formatDateTimeDetailed(minMissing)}, ${formatDateTimeDetailed(maxMissing)}]`);
-        console.log(`Files whose declared range overlaps [${formatDateTimeDetailed(minMissing)}, ${formatDateTimeDetailed(maxMissing)}]:`);
-        for (let p of paths) {
-            if (p.endTime < minMissing || p.startTime > maxMissing) continue;
-            console.log(`  start=${formatDateTimeDetailed(p.startTime)} end=${formatDateTimeDetailed(p.endTime)} ${p.fullPath}`);
-        }
-        // Also: any file whose declared startTime > broad cutoff (and so would
-        // be isSourceRelevant-pruned) — these are candidates for the bug.
-        console.log(`Files whose declared startTime > broad cutoff (${formatDateTimeDetailed(broadCutoff)}) — these would be pruned by isSourceRelevant once broad fills:`);
-        for (let p of paths) {
-            if (p.startTime > broadCutoff) {
-                let overlap = p.startTime <= maxMissing && p.endTime >= minMissing ? "  <- OVERLAPS MISSING" : "";
-                console.log(`  start=${formatDateTimeDetailed(p.startTime)} end=${formatDateTimeDetailed(p.endTime)} ${p.fullPath}${overlap}`);
-            }
-        }
-    }
-
-    // Check whether broad-with-huge-limit catches the missing entries.
-    let broadHugeKeys = new Set(broadHuge.map(emitKey));
-    let stillMissingFromHuge = narrow.filter(n => !broadHugeKeys.has(emitKey(n)));
-    console.log(`\n=== BROAD_HUGE check ===`);
-    console.log(`narrow missing from broad_huge: ${stillMissingFromHuge.length}`);
-    if (stillMissingFromHuge.length > 0) {
-        console.log(`First ${Math.min(20, stillMissingFromHuge.length)} still-missing:`);
-        for (let m of stillMissingFromHuge.slice(0, 20)) {
-            console.log(`  time=${formatDateTimeDetailed(m.time)} param0=${String(m.datum.param0 ?? "").slice(0, 60)}`);
-        }
-        console.log(`-> bug is NOT (only) the per-file cap; the scanner / index pre-filter is dropping entries even without the cap.`);
-    } else {
-        console.log(`-> all narrow entries are in broad_huge; the per-file stopIterating cap is the culprit for the limit=1600 miss.`);
-    }
 }
 
 main().catch(e => console.error((e as Error).stack ?? e))

package/src/diagnostics/AuditLogPage.tsx

@@ -1,147 +0,0 @@
-import { SocketFunction } from "socket-function/SocketFunction";
-import { qreact } from "../4-dom/qreact";
-import { assertIsManagementUser } from "./managementPages";
-import { css } from "typesafecss";
-import { getSyncedController } from "../library-components/SyncedController";
-import { DebugLog, DebugLogController, getLogHistoryEquals, getLogHistoryIncludes } from "../0-path-value-core/auditLogs";
-import { remoteWatcher } from "../1-path-client/RemoteWatcher";
-import { getBrowserUrlNode, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { nextId, sort } from "socket-function/src/misc";
-import { t } from "../2-proxy/schema2";
-import { InputLabelURL } from "../library-components/InputLabel";
-import { URLParam } from "../library-components/URLParam";
-import { Table } from "../5-diagnostics/Table";
-import { ObjectDisplay } from "./logs/ObjectDisplay";
-import { PathRouter } from "../0-path-value-core/PathRouter";
-
-/**
-TODO:
-    1) Support watching multiple paths at once
-    2) Show extra metadata for the paths
-        - Current value (raw, as in, don't sync it)
-        - If it is synced
-        - The authority of it
-        - An expander which shows a table of the full history of values, with valid state, etc
-*/
-
-type DebugLogWithSource = DebugLog & {
-    sourceType: "local" | "remote";
-    sourceNodeId: string;
-}
-
-let path = new URLParam("path", "");
-let includeDescendants = new URLParam("includeDescendants", false);
-export class AuditLogPage extends qreact.Component {
-    loadId = nextId();
-    state = t.state({
-        loadedId: t.string
-    });
-    render() {
-        let logs: DebugLogWithSource[] = [];
-        let loaded = this.loadId === this.state.loadedId;
-        if (loaded) {
-            let obj = auditLogController(getBrowserUrlNode()).getPathHistory({
-                path: path.value,
-                includeDescendants: includeDescendants.value,
-            });
-            if (obj) {
-                logs = obj.fullLogs;
-            }
-        }
-        return (
-            <div class={css.pad2(10).vbox(10).fillWidth}>
-                <InputLabelURL
-                    label="Path"
-                    url={path}
-                    fillWidth
-                    onChange={() => this.loadId = nextId()}
-                />
-                <InputLabelURL
-                    label="Include Descendants"
-                    url={includeDescendants}
-                    checkbox
-                    onChange={() => this.loadId = nextId()}
-                />
-                {!loaded && <button onClick={() => this.state.loadedId = this.loadId}>Load</button>}
-                <Table
-                    rows={logs}
-                    columns={{
-                        type: {},
-                        time: {},
-                        sourceType: {},
-                        sourceNodeId: {},
-                        values: {
-                            formatter(value) {
-                                return <ObjectDisplay value={value} />;
-                            },
-                        },
-                    }}
-                />
-            </div>
-        );
-    }
-}
-
-
-class AuditLogControllerBase {
-    public async getPathHistory(config: {
-        path: string;
-        includeDescendants?: boolean;
-    }) {
-        let { path, includeDescendants } = config;
-        let authorityNodeId = remoteWatcher.getExistingWatchRemoteNodeId(path);
-        if (!authorityNodeId) {
-            authorityNodeId = await PathRouter.getReadyAuthority(path)?.nodeId;
-        }
-        let logs: DebugLog[] = [];
-        let remoteLogs: DebugLog[] = [];
-        if (includeDescendants) {
-            // Also look for the parent, so we can capture keys() syncs as well
-            //  (this should give us child paths too, because that's how includes works)
-            logs = getLogHistoryIncludes(path);
-            if (authorityNodeId) {
-                remoteLogs = await DebugLogController.nodes[authorityNodeId].getLogHistoryEquals(path);
-            }
-        } else {
-            logs = getLogHistoryEquals(path);
-            if (authorityNodeId) {
-                remoteLogs = await DebugLogController.nodes[authorityNodeId].getLogHistoryEquals(path);
-            }
-        }
-
-        let fullLogs: DebugLogWithSource[] = [];
-        let ownNodeId = getOwnNodeId();
-        for (let log of logs) {
-            fullLogs.push({
-                ...log,
-                sourceType: "local",
-                sourceNodeId: ownNodeId,
-            });
-        }
-        if (authorityNodeId) {
-            for (let log of remoteLogs) {
-                fullLogs.push({
-                    ...log,
-                    sourceType: "remote",
-                    sourceNodeId: authorityNodeId,
-                });
-            }
-        }
-        sort(fullLogs, x => x.time);
-        return {
-            fullLogs,
-        };
-    }
-}
-
-export const AuditLogController = SocketFunction.register(
-    "AuditLogController-bf798faf-c803-4add-ac2c-99d0f44e753a",
-    new AuditLogControllerBase(),
-    () => ({
-        getPathHistory: {},
-    }),
-    () => ({
-        hooks: [assertIsManagementUser],
-    })
-);
-export const auditLogController = getSyncedController(AuditLogController);

package/src/diagnostics/NodeConnectionsPage.tsx

@@ -1,167 +0,0 @@
-import { SocketFunction } from "socket-function/SocketFunction";
-import { qreact } from "../4-dom/qreact";
-import { assertIsManagementUser } from "./managementPages";
-import { css } from "typesafecss";
-import { getSyncedController } from "../library-components/SyncedController";
-import { getAllNodeIds, getBrowserUrlNode } from "../-f-node-discovery/NodeDiscovery";
-import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
-import { debugNodeId } from "../-c-identity/IdentityController";
-import { fastHash } from "../misc/hash";
-import { NodeCapabilitiesController } from "../-g-core-values/NodeCapabilities";
-import { sort } from "socket-function/src/misc";
-
-
-
-function getColorForNodeId(friendlyNodeId: string): string {
-    let hash = fastHash(friendlyNodeId);
-    let hue = hash % 360;
-    return `hsl(${hue}, 70%, 60%)`;
-}
-
-export class NodeConnectionsPage extends qreact.Component {
-    render() {
-        let browserNode = getBrowserUrlNode();
-        let controller = nodeConnectionsController(browserNode);
-
-        let nodeIds = controller.getAllNodeIds();
-        if (!nodeIds) {
-            return <div class={css.pad2(10)}>Loading...</div>;
-        }
-
-        let connectableNodes: Array<{ nodeId: string; friendlyNodeId: string; entryPoint: string | undefined; connections: Array<{ nodeId: string; friendlyNodeId: string }> }> = [];
-        let nonConnectableNodes: string[] = [];
-
-        for (let nodeId of nodeIds) {
-            try {
-                let result = controller.getNodeConnections_forBrowser(nodeId);
-                if (result) {
-                    let friendlyNodeId = debugNodeId(nodeId);
-                    let entryPoint: string | undefined;
-                    try {
-                        entryPoint = controller.getEntryPoint_forBrowser(nodeId);
-                    } catch (e) {
-                        entryPoint = undefined;
-                    }
-                    connectableNodes.push({
-                        nodeId,
-                        friendlyNodeId,
-                        entryPoint,
-                        connections: result.connections,
-                    });
-                }
-            } catch (e) {
-                nonConnectableNodes.push(nodeId);
-            }
-        }
-
-        return (
-            <div class={css.pad2(10).vbox(20).fillWidth}>
-                <h1>Connectable Nodes</h1>
-                <div class={css.hbox(20).wrap.fillWidth}>
-                    {connectableNodes.map(node => {
-                        let color = getColorForNodeId(node.friendlyNodeId);
-                        return (
-                            <div class={css.vbox(5).pad2(10).maxHeight(400).overflowAuto.bord(1, { h: 0, s: 0, l: 80 })}>
-                                <div class={css.vbox(3)}>
-                                    {node.entryPoint && (
-                                        <div class={css.fontSize(14).fontWeight("bold")}>
-                                            {node.entryPoint}
-                                        </div>
-                                    )}
-                                    <div class={css.hbox(5)}>
-                                        <span class={css.background(color).pad2(4, 2).fontWeight("bold")}>
-                                            {node.friendlyNodeId}
-                                        </span>
-                                        <span class={css.color("gray").fontSize(12)}>
-                                            ({node.nodeId})
-                                        </span>
-                                    </div>
-                                </div>
-                                <div class={css.vbox(3).paddingLeft(20)}>
-                                    {node.connections.length === 0 && (
-                                        <div class={css.color("gray")}>No connections</div>
-                                    )}
-                                    {node.connections.map(conn => {
-                                        let connColor = getColorForNodeId(conn.friendlyNodeId);
-                                        return (
-                                            <div class={css.hbox(5)}>
-                                                <span>→ Connected to:</span>
-                                                <span class={css.background(connColor).pad2(4, 2).fontWeight("bold")}>
-                                                    {conn.friendlyNodeId}
-                                                </span>
-                                                <span class={css.color("gray").fontSize(12)}>
-                                                    ({conn.nodeId})
-                                                </span>
-                                            </div>
-                                        );
-                                    })}
-                                </div>
-                            </div>
-                        );
-                    })}
-                </div>
-
-                <div class={css.vbox(10).fillWidth}>
-                    <h1>Non-Connectable Nodes</h1>
-                    <div class={css.vbox(5).pad2(10).maxHeight(600).overflowAuto.bord(1, { h: 0, s: 0, l: 80 })}>
-                        {nonConnectableNodes.map(nodeId => {
-                            let friendlyNodeId = debugNodeId(nodeId);
-                            return (
-                                <div class={css.hbox(5)}>
-                                    <span class={css.background("lightgray").pad2(4, 2).fontWeight("bold")}>
-                                        {friendlyNodeId}
-                                    </span>
-                                    <span class={css.color("gray").fontSize(12)}>
-                                        ({nodeId})
-                                    </span>
-                                    <span class={css.color("gray")}>- Can't connect</span>
-                                </div>
-                            );
-                        })}
-                    </div>
-                </div>
-            </div>
-        );
-    }
-}
-
-class NodeConnectionsControllerBase {
-    public async getNodeConnections() {
-        let factories = debugGetAllCallFactories();
-        let connections = factories.filter(x => x.isConnected).map(factory => ({
-            nodeId: factory.nodeId,
-            friendlyNodeId: debugNodeId(factory.nodeId),
-        }));
-        return { connections };
-    }
-
-    public async getNodeConnections_forBrowser(nodeId: string) {
-        let result = await NodeConnectionsController.nodes[nodeId].getNodeConnections();
-        sort(result.connections, x => x.friendlyNodeId);
-        return result;
-    }
-
-    public async getEntryPoint_forBrowser(nodeId: string) {
-        return (await NodeCapabilitiesController.nodes[nodeId].getMetadata()).entryPoint;
-    }
-
-    public async getAllNodeIds() {
-        return await getAllNodeIds();
-    }
-}
-
-export const NodeConnectionsController = SocketFunction.register(
-    "NodeConnectionsController-8f3e2a1b-4c5d-4f2e-9a3b-7c8d1e2f3a4b",
-    new NodeConnectionsControllerBase(),
-    () => ({
-        getNodeConnections: {},
-        getNodeConnections_forBrowser: {},
-        getEntryPoint_forBrowser: {},
-        getAllNodeIds: {},
-    }),
-    () => ({
-        hooks: [assertIsManagementUser],
-    })
-);
-
-export const nodeConnectionsController = getSyncedController(NodeConnectionsController);