querysub 0.356.0 → 0.357.0

package/src/diagnostics/MachineThreadInfo.tsx

@@ -0,0 +1,235 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { NodeCapabilitiesController } from "../-g-core-values/NodeCapabilities";
+import { qreact } from "../4-dom/qreact";
+import { getSyncedController } from "../library-components/SyncedController";
+import { isManagementUser } from "../-0-hooks/hooks";
+import { assertIsManagementUser } from "./managementPages";
+import { getAllNodeIds } from "../-f-node-discovery/NodeDiscovery";
+import { getNodeIdDomain, getNodeIdLocation, getNodeIdDomainMaybeUndefined } from "socket-function/src/nodeCache";
+import { assertIsNetworkTrusted } from "../-d-trust/NetworkTrust2";
+import { runPromise } from "../functional/runCommand";
+import { errorToUndefinedSilent } from "../errors";
+import { decodeNodeId } from "../-a-auth/certs";
+import { keyBy, keyByArray, timeInSecond, timeoutToUndefined, timeoutToUndefinedSilent } from "socket-function/src/misc";
+import dns from "dns";
+import { isDefined } from "../misc";
+import { getDebuggerUrl } from "./listenOnDebugger";
+import { Button } from "../library-components/Button";
+import { Querysub } from "../4-querysub/QuerysubController";
+import { NodeViewerController } from "./NodeViewer";
+import { showFullscreenModal } from "../5-diagnostics/FullscreenModal";
+import { css } from "../4-dom/css";
+import { formatVeryNiceDateTime } from "socket-function/src/formatting/format";
+
+module.hotreload = true;
+module.noserverhotreload = false;
+
+type NodeSpecialInfo = {
+    nodeId: string;
+    machineId: string;
+    threadId: string;
+    ip: string;
+    port: number;
+    hostname: string;
+    entrypoint: string;
+    aliveTime: number;
+    inspectURL: string;
+};
+
+export class MachineThreadInfo extends qreact.Component<{
+    machineId: string;
+    threadId?: string;
+    onlyShowThread?: boolean;
+}> {
+    static renderInProgress = true;
+
+    handleAttach = async (threadInfo: NodeSpecialInfo | undefined) => {
+        Querysub.onCommitFinished(async () => {
+            const url = await NodeViewerController.nodes[SocketFunction.getBrowserNodeId()].getExternalInspectURL(threadInfo?.nodeId || "");
+            window.open(url, "_blank");
+        });
+    };
+
+    handleRefresh = async () => {
+        Querysub.onCommitFinished(async () => {
+            MachineThreadInfoController(SocketFunction.browserNodeId()).getAllInfo.refresh();
+        });
+    };
+
+    showExpandedView = (machineInfo: { hostname: string; ip: string; port: number; } | undefined, threadInfo: NodeSpecialInfo | undefined) => {
+        showFullscreenModal({
+            content: <div className={css.vbox(16).pad2(24)}>
+                <div className={css.vbox(12)}>
+                    <h2>Machine & Thread Info</h2>
+
+                    <div className={css.vbox(8)}>
+                        <div className={css.vbox(4)}>
+                            <strong>Machine ID</strong>
+                            <div>{this.props.machineId}</div>
+                        </div>
+
+                        {this.props.threadId && (
+                            <div className={css.vbox(4)}>
+                                <strong>Thread ID</strong>
+                                <div>{this.props.threadId}</div>
+                            </div>
+                        )}
+
+                        {machineInfo && (
+                            <>
+                                <div className={css.vbox(4)}>
+                                    <strong>Hostname</strong>
+                                    <div>{machineInfo.hostname}</div>
+                                </div>
+
+                                <div className={css.vbox(4)}>
+                                    <strong>IP:Port</strong>
+                                    <div>{machineInfo.ip}:{machineInfo.port}</div>
+                                </div>
+                            </>
+                        )}
+
+                        {threadInfo && (
+                            <>
+                                <div className={css.vbox(4)}>
+                                    <strong>Entry Point</strong>
+                                    <div>{threadInfo.entrypoint}</div>
+                                </div>
+
+                                <div className={css.vbox(4)}>
+                                    <strong>Start Time</strong>
+                                    <div>{formatVeryNiceDateTime(threadInfo.aliveTime)}</div>
+                                </div>
+
+                                {threadInfo.inspectURL && (
+                                    <div className={css.vbox(4)}>
+                                        <strong>Inspector URL</strong>
+                                        <a href={threadInfo.inspectURL} target="_blank">{threadInfo.inspectURL}</a>
+                                    </div>
+                                )}
+                            </>
+                        )}
+                    </div>
+
+                    <div className={css.hbox(12)}>
+                        <Button onClick={() => this.handleAttach(threadInfo)}>
+                            Attach Debugger
+                        </Button>
+
+                        <Button onClick={this.handleRefresh}>
+                            Refresh
+                        </Button>
+
+                        {threadInfo?.nodeId && (
+                            <Button onClick={() => {
+                                window.location.hash = `#/diagnostics/logs/${threadInfo.nodeId}`;
+                            }}>
+                                View Logs
+                            </Button>
+                        )}
+                    </div>
+                </div>
+            </div>
+        });
+    };
+
+    render() {
+        const info = MachineThreadInfoController(SocketFunction.browserNodeId()).getAllInfo();
+        // NOTE: Only these fields can be used from the machine info. Otherwise, you're just accessing our random thread. But for these fields, they're the same for everything with the same machine ID. So they're safe to access, even if our thread no longer exists.
+        const machineInfo = info?.byMachine.get(this.props.machineId) as undefined | {
+            hostname: string;
+            ip: string;
+            port: number;
+        };
+        const threadInfo = info?.byThread.get(this.props.threadId || "");
+
+        return (
+            <div
+                className={css.button.hbox(4).pad2(4).hsla(0, 0, 100, 0.5)}
+                onClick={(e) => {
+                    e.stopPropagation();
+                    this.showExpandedView(machineInfo, threadInfo);
+                }}
+            >
+                {!this.props.onlyShowThread && <span title={this.props.machineId}>
+                    {machineInfo?.hostname || this.props.machineId}
+                </span>}
+                {!this.props.onlyShowThread && this.props.threadId && " | "}
+                {this.props.threadId && <span title={this.props.threadId}>
+                    {/* TODO: Create a mapping from entrypoints to nice names. The full folder path will likely be too large, but just the filename is too similar (it's server.ts both both the PathValueServer and the application server). */}
+                    {threadInfo?.entrypoint || this.props.threadId}
+                </span>}
+            </div>
+        );
+    }
+}
+
+const aliveTime = Date.now();
+
+class MachineThreadInfoBase {
+    public async getAllInfo(): Promise<{
+        byMachine: Map<string, NodeSpecialInfo>;
+        byThread: Map<string, NodeSpecialInfo>;
+    }> {
+        let nodeIds = await getAllNodeIds();
+        let specialInfos = await Promise.all(nodeIds.map(async (nodeId): Promise<NodeSpecialInfo | undefined> => {
+            const nodeParts = decodeNodeId(nodeId);
+            if (!nodeParts) return undefined;
+
+            const location = getNodeIdLocation(nodeId);
+            if (!location) return undefined;
+
+            const domain = getNodeIdDomainMaybeUndefined(nodeId);
+            if (!domain) return undefined;
+
+            let ip = "";
+            const ipResult = await dns.promises.lookup(domain);
+            if (ipResult) {
+                ip = ipResult.address;
+            }
+
+            const nodeInfo = await MachineThreadInfoController(nodeId).getNodeInfo.promise();
+
+            return {
+                nodeId: nodeId,
+                machineId: nodeParts.machineId,
+                threadId: nodeParts.threadId,
+                ip: ip,
+                port: location.port,
+                hostname: nodeInfo?.hostname || "",
+                entrypoint: nodeInfo?.entrypoint || "",
+                aliveTime: nodeInfo?.aliveTime || 0,
+                inspectURL: nodeInfo?.inspectURL || "",
+            };
+        }).map(x => timeoutToUndefinedSilent(timeInSecond * 5, x)));
+
+        const validInfos = specialInfos.filter(isDefined);
+
+        return {
+            byMachine: keyBy(validInfos, info => info.machineId),
+            byThread: keyBy(validInfos, info => info.threadId),
+        };
+    }
+
+    // This function is only intended to be called between nodes
+    public async getNodeInfo() {
+        const hostname = await errorToUndefinedSilent(runPromise("hostname"));
+        return {
+            hostname: hostname?.trim() || "",
+            entrypoint: process.argv[1] || "",
+            aliveTime: aliveTime,
+            inspectURL: await getDebuggerUrl(),
+        };
+    }
+
+}
+const MachineThreadInfoController = getSyncedController(SocketFunction.register(
+    "MachineThreadInfoController-019c88cb-a16f-7219-9a5e-08919bd1475b",
+    new MachineThreadInfoBase(),
+    () => ({
+        getAllInfo: { hooks: [assertIsManagementUser] },
+        getNodeInfo: { hooks: [assertIsNetworkTrusted] },
+    }),
+    () => ({
+    })
+));

package/src/diagnostics/NodeViewer.tsx

@@ -22,7 +22,7 @@ import { encodeFormattedSelector, errorMessage, formatValue, toSpaceCase } from
 import { ValueAuditController } from "../../src/5-diagnostics/memoryValueAudit";
 import { getExternalIP } from "../../src/misc/networking";
 import dns from "dns";
-import { getNodeIdDomain, getNodeIdIP } from "socket-function/src/nodeCache";
+import { getNodeIdDomain, getNodeIdDomainMaybeUndefined, getNodeIdIP } from "socket-function/src/nodeCache";
 import ws from "ws";
 import https from "https";
 import debugbreak from "debugbreak";
@@ -344,7 +344,8 @@ class NodeViewerControllerBase {
     }
 
     public async getNodeIP(nodeId: string) {
-        let domain = nodeId.split(":")[0];
+        let domain = getNodeIdDomainMaybeUndefined(nodeId);
+        if (!domain) return undefined;
         // Resolve the domain to an IP
         let result = await dns.promises.lookup(domain);
         return result.address;

package/src/diagnostics/logs/FastArchiveAppendable.ts

@@ -12,7 +12,7 @@ import fs from "fs";
 import { MaybePromise, canHaveChildren } from "socket-function/src/types";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { cache, lazy } from "socket-function/src/caching";
-import { getArchives, nestArchives } from "../../-a-archives/archives";
+import { Archives, getArchives, nestArchives } from "../../-a-archives/archives";
 import { Zip } from "../../zip";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { assertIsManagementUser } from "../managementPages";
@@ -26,7 +26,7 @@ import { getPathIndex, getPathStr2 } from "../../path";
 import { onNextPaint } from "../../functional/onNextPaint";
 import { getArchivesBackblazePrivateImmutable, getArchivesBackblazePublicImmutable } from "../../-a-archives/archivesBackBlaze";
 import { httpsRequest } from "socket-function/src/https";
-import { getDomain } from "../../config";
+import { getDomain, isPublic } from "../../config";
 import { getIPDomain } from "../../-e-certs/EdgeCertController";
 import { getArchivesPrivateFileSystem } from "../../-a-archives/archivesPrivateFileSystem";
 import { createArchivesLimitedCache } from "../../-a-archives/archivesLimitedCache";
@@ -36,6 +36,7 @@ import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { FileMetadata, FastArchiveAppendableControllerBase, FastArchiveAppendableController, getFileMetadataHash } from "./FastArchiveController";
 import { fsExistsAsync } from "../../fs";
 import { ScanFnc } from "./FastArchiveViewer";
+import { getArchivesLocal } from "../../-a-archives/archivesDisk";
 
 // NOTE: In a single command line micro-test it looks like we can write about 40K writes of 500 per once, when using 10X parallel, on a fairly potato server. We should probably batch though, and only do 1X parallel.
 /*
@@ -59,7 +60,7 @@ const UNCOMPRESSED_LOG_FILE_WARN_THRESHOLD = 1024 * 1024 * 512;
 const UNCOMPRESSED_LOG_FILE_STOP_THRESHOLD = 1024 * 1024 * 1024 * 2;
 
 // Add a large wait, due to day light saving time, or whatever
-const UPLOAD_THRESHOLD = timeInHour * 3;
+const UPLOAD_THRESHOLD = timeInHour * 1.5;
 const DEAD_TIMEOUT = timeInHour * 6;
 const DELETE_TIMEOUT = timeInHour * 12;
 
@@ -68,6 +69,9 @@ const MAX_WORK_PER_PAINT = 40;
 const ON_DATA_BATCH_COUNT = 1024 * 10;
 //const ON_DATA_BATCH_COUNT = 1;
 
+// NOTE: This used to be small, for atomic writes, but it causes lag, and... why do we need atomic writes? I believe we use a file per thread, so... we will be the only writer.
+const WRITE_CHUNK_SIZE = 1024 * 1024 * 16;
+
 
 
 const MAX_LOCAL_CACHED_FILES = 1000 * 10;
@@ -113,6 +117,15 @@ export function getFileTimeStamp(path: string): {
         dateStr += ":00:00.000Z";
     }
     let startTime = new Date(dateStr).getTime();
+    if (!startTime) {
+        let pathParts = path.split("/");
+        let year = parseInt(pathParts[0], 10);
+        let month = parseInt(pathParts[1], 10) - 1;
+        let day = parseInt(pathParts[2], 10);
+        let hour = parseInt(pathParts[3], 10);
+        let hourStart = Date.UTC(year, month, day, hour);
+        startTime = hourStart;
+    }
     return {
         startTime,
         endTime: startTime + timeInHour,
@@ -148,8 +161,13 @@ export class FastArchiveAppendable<Datum> {
         }
     }
 
-    public getArchives = lazy(() => {
-        let archives = getArchivesBackblazePrivateImmutable(getDomain());
+    public getArchives = cache((forceGetPublic: boolean) => {
+        let archives: Archives;
+        if (!isPublic() && !forceGetPublic) {
+            archives = getArchivesLocal(getDomain());
+        } else {
+            archives = getArchivesBackblazePrivateImmutable(getDomain());
+        }
         return nestArchives("fast-logs/" + this.rootPath, archives);
     });
 
@@ -201,8 +219,7 @@ export class FastArchiveAppendable<Datum> {
         }
     }
 
-    // NOTE: This is disk writing, which should be fast, but if it's slow we might be able to remove the measureWrap (as technically spending 50% of our time writing to the disk is fine, and won't lag anything).
-    @measureFnc
+    // NOTE: No timing on this as it's just waiting on the disk. It's not actually slowing anything down. 
     public async flushNow(now = Date.now()) {
 
         await appendableSerialLock(async () => {
@@ -226,35 +243,35 @@ export class FastArchiveAppendable<Datum> {
                 // File not existing is fine
             }
 
-            // NOTE: We can't use anything but JSON, as we need it to be scannable before decoding it (otherwise scanning takes 100X longer)
-            let writeData = this.pendingWriteQueue.map(v => {
-                let buffer = Buffer.from(JSON.stringify(v));
-                this.escapeDelimitter(buffer);
-                return buffer;
-            });
-            this.pendingWriteQueue = [];
+            let chunks: Buffer[][] = [];
+            measureBlock(() => {
+                // NOTE: We can't use anything but JSON, as we need it to be scannable before decoding it (otherwise scanning takes 100X longer)
+                let writeData = this.pendingWriteQueue.map(v => {
+                    let buffer = Buffer.from(JSON.stringify(v));
+                    this.escapeDelimitter(buffer);
+                    return buffer;
+                });
+                this.pendingWriteQueue = [];
 
-            // Apparently, anything more than this and our writes might not be atomic
-            const WRITE_ATOMIC_LIMIT = 4096;
+                // Group lines into WRITE_ATOMIC_LIMIT byte chunks
 
-            // Group lines into WRITE_ATOMIC_LIMIT byte chunks
-            let chunks: Buffer[][] = [];
-            let currentChunk: Buffer[] = [];
-            let currentSize = 0;
-            for (let line of writeData) {
-                if (currentSize + line.length + objectDelimitterBuffer.length > WRITE_ATOMIC_LIMIT && currentChunk.length > 0) {
+                let currentChunk: Buffer[] = [];
+                let currentSize = 0;
+                for (let line of writeData) {
+                    if (currentSize + line.length + objectDelimitterBuffer.length > WRITE_CHUNK_SIZE && currentChunk.length > 0) {
+                        chunks.push(currentChunk);
+                        currentChunk = [];
+                        currentSize = 0;
+                    }
+                    currentChunk.push(line);
+                    currentSize += line.length;
+                    currentChunk.push(objectDelimitterBuffer);
+                    currentSize += objectDelimitterBuffer.length;
+                }
+                if (currentChunk.length > 0) {
                     chunks.push(currentChunk);
-                    currentChunk = [];
-                    currentSize = 0;
                 }
-                currentChunk.push(line);
-                currentSize += line.length;
-                currentChunk.push(objectDelimitterBuffer);
-                currentSize += objectDelimitterBuffer.length;
-            }
-            if (currentChunk.length > 0) {
-                chunks.push(currentChunk);
-            }
+            }, `FastArchiveAppendable|serialize log data`);
 
             for (let chunk of chunks) {
                 await fs.promises.appendFile(localCachePath, Buffer.concat(chunk));
@@ -286,8 +303,13 @@ export class FastArchiveAppendable<Datum> {
             let rootCacheFolder = this.baseGetLocalPathRoot();
             if (!await fsExistsAsync(rootCacheFolder)) return;
             console.log(magenta(`Moving old logs to Backblaze from ${rootCacheFolder}`));
+            // Ugh... what is this even? Is it... hot reloading?
+            if (rootCacheFolder.includes("undefined/undefined/undefined/")) {
+                require("debugbreak")(2);
+                debugger;
+            }
 
-            let archives = this.getArchives();
+            let archives = this.getArchives(false);
             async function moveLogsForFolder(threadId: string) {
                 let threadDir = rootCacheFolder + threadId + "/";
                 if (!await fsExistsAsync(threadDir)) return;
@@ -361,11 +383,13 @@ export class FastArchiveAppendable<Datum> {
             endTime: number;
         };
         cacheBust: number;
+        forceGetPublic?: boolean;
         scanFnc?: ScanFnc;
         getWantData?: (file: FileMetadata) => Promise<((posStart: number, posEnd: number, data: Buffer) => boolean) | undefined>;
         onData: (datum: Datum[], file: FileMetadata) => void;
         // Called after onData
         onStats?: (stats: DatumStats, file: FileMetadata) => void;
+        onError?: (error: Error, file: FileMetadata) => void;
         onFinish?: () => void;
 
         onProgress?: (progress: {
@@ -379,10 +403,14 @@ export class FastArchiveAppendable<Datum> {
         };
         stopSynchronize: () => void;
     } | "cancelled"> {
-        let { onData, onStats } = config;
+        let { onData, onStats, onError } = config;
         // Create unique client sync ID upfront
         let syncId = nextId();
 
+        console.log(`Synchronizing ${this.rootPath} with syncId ${syncId}`, config);
+
+        let isPublicValue = isPublic() || config.forceGetPublic || false;
+
         // Register progress callback immediately so we can receive progress during setup
         // - It also helps with cancellation
         FastArchiveAppendableControllerBase.progressCallbacks.set(syncId, config.onProgress ?? (() => { }));
@@ -400,6 +428,7 @@ export class FastArchiveAppendable<Datum> {
 
         let parametersHash = sha256(JSON.stringify({
             range: config.range,
+            isPublicValue,
             cacheBust: config.cacheBust,
             version: 1,
         })) + ".parameters";
@@ -506,6 +535,7 @@ export class FastArchiveAppendable<Datum> {
                     syncId,
                     range: config.range,
                     rootPath: this.rootPath,
+                    forceGetPublic: config.forceGetPublic,
                 });
                 syncResult.createTime = Date.now();
                 await findingFiles(syncResult.files.length, syncResult.files.length, true);
@@ -618,18 +648,13 @@ export class FastArchiveAppendable<Datum> {
                 })();
 
                 // Fetch the file data in a streaming manner
-                let urlObj = new URL(file.url);
-                urlObj.searchParams.set("cacheBust", config.cacheBust.toString());
-                if (file.nodeId && downloadSyncId) {
-                    let args = JSON.parse(urlObj.searchParams.get("args") || "");
-                    args[0] = downloadSyncId;
-                    urlObj.searchParams.set("args", JSON.stringify(args));
-                }
-                let url = urlObj.toString();
 
                 // TODO: Stream from the local cache instead? It should be possible, we can get the total size, and read chunks.
                 let hash = getFileMetadataHash(file) + ".file";
                 let contents = await localCache.get(hash);
+                if (contents?.length !== file.size) {
+                    contents = undefined;
+                }
                 if (stoppedPromise.resolveCalled) return;
                 await runInner(async () => {
                     if (contents?.length) {
@@ -639,6 +664,15 @@ export class FastArchiveAppendable<Datum> {
                             await onFetchedData(data);
                         }
                     } else {
+                        let urlObj = new URL(file.url);
+                        urlObj.searchParams.set("cacheBust", config.cacheBust.toString());
+                        urlObj.searchParams.set("isPublic", isPublicValue.toString());
+                        if (file.nodeId && downloadSyncId) {
+                            let args = JSON.parse(urlObj.searchParams.get("args") || "");
+                            args[0] = downloadSyncId;
+                            urlObj.searchParams.set("args", JSON.stringify(args));
+                        }
+                        let url = urlObj.toString();
                         const response = await fetch(url);
                         if (!response.ok) {
                             throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
@@ -693,6 +727,9 @@ export class FastArchiveAppendable<Datum> {
                         } catch (e: any) {
                             console.warn(`Failed to download and parse file ${file.path}:\n${e.stack}`);
                             await failedFiles(1, 1, true);
+                            if (onError) {
+                                onError(e, file);
+                            }
                         }
                         if (stoppedPromise.resolveCalled) return;
                         await fileProgress(1, 0, true);