querysub 0.438.0 → 0.439.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.438.0",
+  "version": "0.439.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -62,7 +62,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "querysub": "^0.357.0",
-    "socket-function": "^1.1.20",
+    "socket-function": "^1.1.21",
     "terser": "^5.31.0",
     "typesafecss": "^0.28.0",
     "yaml": "^2.5.0",

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -1,12 +1,15 @@
 import { sha256 } from "js-sha256";
 import { getArchives } from "../../-a-archives/archives";
 import { getAllNodeIds, getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
-import { pathValueArchives } from "../pathValueArchives";
+import { archives, pathValueArchives } from "../pathValueArchives";
 import { ignoreErrors, logErrors, timeoutToUndefinedSilent } from "../../errors";
 import { green, magenta } from "socket-function/src/formatting/logColors";
 import { devDebugbreak, isPublic } from "../../config";
 import { sort } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
+import { decodeCborx, encodeCborx } from "../../misc/cloneHelpers";
+import { runInParallel } from "socket-function/src/batching";
+import { formatNumber } from "socket-function/src/formatting/format";
 
 const snapshots = lazy(() => getArchives("snapshots"));
 
@@ -140,6 +143,39 @@ async function getSnapshotBase(snapshotFile: string | "live") {
     };
 }
 
+export interface ArchiveSnapshotPayloadFile {
+    path: string;
+    data: Buffer;
+}
+
+export async function downloadSnapshot(snapshotFile: string | "live"): Promise<Buffer> {
+    let { files } = await getSnapshotBase(snapshotFile);
+    let buffers = await pathValueArchives.getRawValueFiles(files, { includeRecycleBin: true });
+    let payload: ArchiveSnapshotPayloadFile[] = [];
+    for (let i = 0; i < files.length; i++) {
+        let data = buffers[i];
+        if (!data) continue;
+        payload.push({ path: files[i], data });
+    }
+    return encodeCborx(payload);
+}
+
+export async function uploadSnapshot(buffer: Buffer): Promise<ArchiveSnapshotOverview> {
+    let payload = decodeCborx<ArchiveSnapshotPayloadFile[]>(buffer);
+    let liveArchives = archives();
+    async function uploadFile(path: string, data: Buffer) {
+        let existing = await liveArchives.getInfo(path);
+        if (existing) return;
+        console.log(`Creating missing file (${formatNumber(data.length)}B): ${path}`);
+        await liveArchives.set(path, Buffer.from(data));
+    }
+    let parallel = runInParallel({
+        parallelCount: 32,
+    }, uploadFile);
+    await Promise.all(payload.map(file => parallel(file.path, file.data)));
+    return await saveSnapshot({ files: payload.map(x => x.path) });
+}
+
 // EXTREME DANGEROUS! Should only be done when all servers are down (except one, which
 //  is run on a developers machine). Worse case scenario, where everything is broken,
 //  and you need to just go back to a previous state.

package/src/0-path-value-core/pathValueCore.ts

@@ -394,6 +394,18 @@ export function debugPathValue(pathValue: PathValue, write?: boolean): string {
 export function debugTime(time: Time): string {
     return `${time.time}[${time.version}]@${time.creatorId.toString().slice(4, 12)}`;
 }
+export function parseDebugTime(debugTime: string): {
+    time: number;
+    version: number;
+    creatorIdPart: string;
+} {
+    let parts = debugTime.split(/[\[\]\@]/g);
+    return {
+        time: +(parts[0]),
+        version: +(parts[1]),
+        creatorIdPart: parts[2],
+    };
+}
 
 
 let getCompressNetworkBase = () => false;

package/src/3-path-functions/PathFunctionRunner.ts

@@ -9,7 +9,7 @@ import { getPathFromStr, getPathIndex } from "../path";
 import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
 import { getProxyPath } from "../2-proxy/pathValueProxy";
 import { atomicObjectRead, atomicObjectWrite, doProxyOptions, isProxyBlockedByOrder, isSynced, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { authorityStorage, compareTime, debugTime, MAX_ACCEPTED_CHANGE_AGE, PathValue, Time } from "../0-path-value-core/pathValueCore";
+import { authorityStorage, compareTime, debugTime, MAX_ACCEPTED_CHANGE_AGE, parseDebugTime, PathValue, Time } from "../0-path-value-core/pathValueCore";
 import { getModuleFromSpec } from "./pathFunctionLoader";
 import debugbreak from "debugbreak";
 import { parseArgs } from "./PathFunctionHelpers";
@@ -164,6 +164,15 @@ function getDebugName(call: CallSpec, functionSpec: FunctionSpec | undefined, co
         return `${mainPart}`;
     }
 }
+export function parseDebugName(debugName: string) {
+    let parts = debugName.split("|");
+    if (parts.length !== 3) return undefined;
+    return {
+        file: parts[0],
+        functionId: parts[1],
+        runAtTime: parseDebugTime(parts[2]),
+    };
+}
 
 type PermissionsCheckType = {
     new(callerMachineId: { callerMachineId: string; callerIP: string; }): {
@@ -531,7 +540,9 @@ export class PathFunctionRunner {
             return;
         }
 
-        console.info(`New function call: ${getDebugName(callSpec, functionSpec, true)}`, {
+        let debugNameColored = getDebugName(callSpec, functionSpec, true);
+        let debugName = getDebugName(callSpec, functionSpec);
+        console.info(`New function call: ${debugNameColored}`, {
             callId: callSpec.CallId,
             timeId: callSpec.runAtTime.time,
             functionId: callSpec.FunctionId,
@@ -564,13 +575,13 @@ export class PathFunctionRunner {
             }
             if (PathFunctionRunner.DEBUG_CALLS) {
                 let fraction = getRoutingOverridePart(callSpec.CallId)?.route;
-                console.log(`${yellow("Function run fallback")} (primary servery failed to run function) after ${formatTime(secondaryDelay)}. Fraction ${fraction?.toFixed(3)}. Primary ${this.config.shardRange.startFraction.toFixed(3)} to ${this.config.shardRange.endFraction.toFixed(3)}. Secondary ${this.config.secondaryShardRange?.startFraction.toFixed(3)} to ${this.config.secondaryShardRange?.endFraction.toFixed(3)}. for ${getDebugName(callSpec, functionSpec, true)}`);
+                console.log(`${yellow("Function run fallback")} (primary servery failed to run function) after ${formatTime(secondaryDelay)}. Fraction ${fraction?.toFixed(3)}. Primary ${this.config.shardRange.startFraction.toFixed(3)} to ${this.config.shardRange.endFraction.toFixed(3)}. Secondary ${this.config.secondaryShardRange?.startFraction.toFixed(3)} to ${this.config.secondaryShardRange?.endFraction.toFixed(3)}. for ${debugNameColored}`);
             }
         }
 
         PathFunctionRunner.RUN_START_COUNT++;
         if (PathFunctionRunner.DEBUG_CALLS) {
-            console.log(`STARTING ${getDebugName(callSpec, functionSpec, true)}`);
+            console.log(`STARTING ${debugNameColored}`);
         }
         let startTime = Date.now();
         let runCount = 0;
@@ -618,29 +629,29 @@ export class PathFunctionRunner {
                     devFunctionMetadata = schemaObj?.functionMetadata?.[functionSpec.FunctionId];
                 }
 
-                console.info(`Running function: ${getDebugName(callSpec, functionSpec, true)}`, {
+                console.info(`Running function: ${debugNameColored}`, {
                     callId: callSpec.CallId,
                     outerLoop: retries,
                 });
 
                 await proxyWatcher.commitFunction({
                     canWrite: true,
-                    debugName: getDebugName(callSpec, functionSpec),
+                    debugName: debugName,
+                    source: debugName,
                     runAtTime: callSpec.runAtTime,
                     getPermissionsCheck: PermissionsChecker && (() => new PermissionsChecker(callSpec)),
                     nestedCalls: "inline",
                     temporary: true,
                     maxLocksOverride: devFunctionMetadata ? devFunctionMetadata.maxLocksOverride : functionSpec.maxLocksOverride,
-                    source: callSpec.CallId,
                     watchFunction: function runCallWatcher() {
                         runCount++;
                         stats.lastInternalLoopCount = runCount;
                         stats.totalInternalLoopCount++;
                         if (PathFunctionRunner.DEBUG_CALLS) {
-                            console.log(`Evaluating (try count ${runCount}) ${getDebugName(callSpec, functionSpec, true)}`);
+                            console.log(`Evaluating (try count ${runCount}) ${debugNameColored}`);
                         }
                         if (runCount > PathFunctionRunner.MAX_WATCH_LOOPS) {
-                            let errorMessage = `MAX_WATCH_LOOPS exceeded for ${getDebugName(callSpec, functionSpec, true)}. All accesses have to be consistent. So Querysub.time() instead of Date.now() and Querysub.nextId() instead of nextId() / Math.random(). If you need multiple random numbers, keep track of an index, and pass it to Querysub.nextId() for the nth random number.`;
+                            let errorMessage = `MAX_WATCH_LOOPS exceeded for ${debugNameColored}. All accesses have to be consistent. So Querysub.time() instead of Date.now() and Querysub.nextId() instead of nextId() / Math.random(). If you need multiple random numbers, keep track of an index, and pass it to Querysub.nextId() for the nth random number.`;
                             console.error(errorMessage, {
                                 callId: callSpec.CallId,
                                 runCount,
@@ -662,7 +673,7 @@ export class PathFunctionRunner {
                                 atomicObjectRead(syncedModule.Sources[callSpec.FunctionId])
                             );
                             if (!syncedSpec) {
-                                throw new Error(`Function spec not found for ${getDebugName(callSpec, functionSpec, true)}`);
+                                throw new Error(`Function spec not found for ${debugNameColored}`);
                             }
 
                             //  (We also need to depend on the RIGHT function spec).
@@ -775,7 +786,7 @@ export class PathFunctionRunner {
                 // }
                 //if (PathFunctionRunner.DEBUG_CALLS)
                 {
-                    console.log(`Function run error ${getDebugName(callSpec, functionSpec, true)}`, {
+                    console.log(`Function run error ${debugNameColored}`, {
                         callId: callSpec.CallId,
                         error: e.stack,
                     });
@@ -799,7 +810,7 @@ export class PathFunctionRunner {
         }
 
         if (PathFunctionRunner.DEBUG_CALLS) {
-            console.log(`FINISHED${nooped ? " (skipped)" : ""} ${getDebugName(callSpec, functionSpec, true)}, writes: ${finalWrites?.length}, took ${blue(formatTime(Date.now() - startTime))}`);
+            console.log(`FINISHED${nooped ? " (skipped)" : ""} ${debugNameColored}, writes: ${finalWrites?.length}, took ${blue(formatTime(Date.now() - startTime))}`);
         }
 
         let wallTime = Date.now() - startTime;
@@ -814,7 +825,7 @@ export class PathFunctionRunner {
             });
         }
 
-        console.info(`Finished function evaluation: ${getDebugName(callSpec, functionSpec, true)}`, {
+        console.info(`Finished function evaluation: ${debugNameColored}`, {
             callId: callSpec.CallId,
             outerLoop: retries,
             finalWrites: finalWrites?.length,

package/src/5-diagnostics/qreactDebug.tsx

@@ -17,6 +17,12 @@ import { InputLabel, InputLabelURL } from "../library-components/InputLabel";
 import { URLParam } from "../library-components/URLParam";
 import { hotReloadingGuard, isHotReloading, onHotReload } from "socket-function/hot/HotReloadController";
 import { isCallerDynamicModule, isDynamicModule } from "../3-path-functions/pathFunctionLoader";
+import { ATag } from "../library-components/ATag";
+import { managementPageURL, showingManagementURL } from "../diagnostics/managementPages";
+import { endTimeParam, startTimeParam } from "../diagnostics/logs/TimeRangeSelector";
+import { timeInHour } from "socket-function/src/misc";
+import { additionalSearchURL, lifecycleIdURL } from "../diagnostics/logs/lifeCycleAnalysis/LifeCyclePage";
+import { parseDebugName } from "../3-path-functions/PathFunctionRunner";
 
 // Map, so hot reloading doesn't break things
 let componentButtons = new Map<string, { title: string, callback: (component: ExternalRenderClass) => void }>();
@@ -290,9 +296,6 @@ class WatchModal extends qreact.Component<{
                                     return undefined;
                                 }
                                 let valueObj = authorityStorage.getValueAtTime(path);
-                                // if (!path.includes("LOCAL") && valueObj?.source) {
-                                //     debugger;
-                                // }
                                 let value = pathValueSerializer.getPathValue(valueObj);
                                 let valueStr = String(value);
                                 try {
@@ -311,6 +314,9 @@ class WatchModal extends qreact.Component<{
                                 let logOnRead = PathValueProxyWatcher.LOG_WRITES_INCLUDES.has(path);
                                 let fncBreak = PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES.has(path);
                                 let selectedButton = css.hsl(120, 75, 75);
+
+                                let sourceObj = parseDebugName(valueObj?.source || "");
+
                                 return (
                                     <div class={css.hbox(10, 0).fillWidth.wrap}>
                                         <button class={breakOnWrites && selectedButton || ""} onClick={() => {
@@ -353,6 +359,23 @@ class WatchModal extends qreact.Component<{
                                         }}>
                                             Caller*
                                         </button>
+                                        {
+                                            valueObj && sourceObj && <ATag values={[
+                                                showingManagementURL.getOverride(true),
+                                                managementPageURL.getOverride("LifeCyclePage"),
+                                                startTimeParam.getOverride(valueObj.time.time - timeInHour),
+                                                endTimeParam.getOverride(valueObj.time.time + timeInHour),
+                                                lifecycleIdURL.getOverride("1772917451506.018_0.441679673600166"),
+                                                additionalSearchURL.getOverride(`${sourceObj.functionId} & ${sourceObj.runAtTime.time}`),
+                                            ]}
+                                                title={valueObj.source || ""}
+                                            >
+                                                View Call
+                                            </ATag>
+                                            || <div title={valueObj?.source || ""}>
+                                                Source
+                                            </div>
+                                        }
                                         <div class={css.hbox(4).wrap.button} onClick={() => {
                                             console.log(path);
                                             return navigator.clipboard.writeText(path);

package/src/diagnostics/PathDistributionInfo.tsx

@@ -47,7 +47,7 @@ function getNodeForPath(path: string): string | "missing" | undefined {
 }
 
 export class PathDistributionInfo extends qreact.Component {
-    render() {
+    renderBase() {
         if (!isCurrentUserSuperUser()) return undefined;
 
         let nodeSpecs = querysubController(SocketFunction.browserNodeId()).debugGetNodeSpecs();
@@ -99,4 +99,12 @@ export class PathDistributionInfo extends qreact.Component {
             {parts.length > 0 && parts || "..."}
         </div>;
     }
+    render() {
+        try {
+            return this.renderBase();
+        } catch (error) {
+            console.error("Error in rendering PathDistributionInfo:", error);
+            return undefined;
+        }
+    }
 }

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -276,7 +276,7 @@ export class BufferUnitIndex {
                 // Only inaccurate if unique units approaches or exceeds 65k
                 return uniqueCount * 2;
             }
-        }, `collectUniqueUnits`);
+        }, `estimatedUniqueUnits`);
 
         // Step 3: Calculate hash table size and mask
         const { hashTableCapacity, mask } = measureBlock(() => {

package/src/diagnostics/logs/diskLogger.ts

@@ -21,6 +21,11 @@ if (isNode()) {
     }));
 }
 
+let loggingToDiskDisabled = false;
+export function disableLoggingToDisk() {
+    loggingToDiskDisabled = true;
+}
+
 // NOTE: When logging we spread objects. If we encounter strings, we set the field `param${index}`
 export type LogDatum = Record<string, unknown> & {
     time: number;
@@ -109,6 +114,7 @@ const logDiskDontShim = logDisk;
 /** @deprecated, Don't call this directly, call console info instead, which our shim will prevent from logging to the console, but it will still call logDisk. */
 export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknown[]) {
     if (!isNode()) return;
+    if (loggingToDiskDisabled) return;
     try {
         if (args.length === 0) return;
         let logType = args.find(x => typeof x === "string") as string | undefined;

package/src/diagnostics/misc-pages/SnapshotViewer.tsx

@@ -1,6 +1,6 @@
 module.allowclient = true;
 
-import { ArchiveSnapshotOverview, ArchiveSnapshotRead, getSnapshot, getSnapshotList, loadSnapshot, saveSnapshot } from "../../0-path-value-core/archiveLocks/archiveSnapshots";
+import { ArchiveSnapshotOverview, ArchiveSnapshotRead, downloadSnapshot, getSnapshot, getSnapshotList, loadSnapshot, saveSnapshot, uploadSnapshot } from "../../0-path-value-core/archiveLocks/archiveSnapshots";
 import { qreact } from "../../4-dom/qreact";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
@@ -30,6 +30,8 @@ export class SnapshotViewer extends qreact.Component {
         // file => true
         expanded: t.lookup(t.boolean),
         saving: t.boolean(false),
+        uploading: t.boolean(false),
+        downloading: t.lookup(t.boolean),
     });
     render() {
         let controller = SnapshotViewerSynced(getBrowserUrlNode());
@@ -40,6 +42,11 @@ export class SnapshotViewer extends qreact.Component {
             return (
                 <div class={css.pad2(10).vbox(10)}>
                     <h1>Snapshots are taken before every change. "zombie" means the file exists, but it has no confirm (so it will be ignored).</h1>
+                    <div class={css.hbox(10)}>
+                        <Button onClick={() => triggerSnapshotUpload(this)}>
+                            {this.state.uploading && "Uploading..." || "Upload Snapshot"}
+                        </Button>
+                    </div>
                     <Table
                         rows={snapshotList}
                         columns={{
@@ -68,6 +75,9 @@ export class SnapshotViewer extends qreact.Component {
                                                 {this.state.saving ? "Saving..." : "Save"}
                                             </Button>
                                         }
+                                        <Button onClick={() => downloadSnapshotFile(this, file)}>
+                                            {this.state.downloading[file] && "Downloading..." || "Download"}
+                                        </Button>
                                     </div>;
                                 }
                             }
@@ -142,6 +152,61 @@ export class SnapshotViewer extends qreact.Component {
     }
 }
 
+async function downloadSnapshotFile(component: SnapshotViewer, file: string) {
+    if (component.state.downloading[file]) return;
+    Querysub.localCommit(() => {
+        component.state.downloading[file] = true;
+    });
+    try {
+        let buffer = await SnapshotViewerController.nodes[getBrowserUrlNode()].downloadSnapshot(file);
+        let blob = new Blob([buffer]);
+        let url = URL.createObjectURL(blob);
+        let anchor = document.createElement("a");
+        anchor.href = url;
+        anchor.download = file + ".cborx";
+        if (file === "live") {
+            anchor.download = `${formatDateTime(Date.now())}.cborx`;
+        }
+        document.body.appendChild(anchor);
+        anchor.click();
+        document.body.removeChild(anchor);
+        URL.revokeObjectURL(url);
+    } catch (e: any) {
+        console.error(red(`Failed to download snapshot ${file}: ${e.stack}`));
+    } finally {
+        Querysub.localCommit(() => {
+            delete component.state.downloading[file];
+        });
+    }
+}
+
+async function triggerSnapshotUpload(component: SnapshotViewer) {
+    if (component.state.uploading) return;
+    let input = document.createElement("input");
+    input.type = "file";
+    input.accept = ".cborx";
+    input.onchange = async () => {
+        let selected = input.files?.[0];
+        if (!selected) return;
+        Querysub.localCommit(() => {
+            component.state.uploading = true;
+        });
+        try {
+            let arrayBuffer = await selected.arrayBuffer();
+            let buffer = Buffer.from(arrayBuffer);
+            await SnapshotViewerController.nodes[getBrowserUrlNode()].uploadSnapshot(buffer);
+            console.log(green("Uploaded snapshot"));
+        } catch (e: any) {
+            console.error(red(`Failed to upload snapshot: ${e.stack}`));
+        } finally {
+            Querysub.localCommit(() => {
+                component.state.uploading = false;
+            });
+        }
+    };
+    input.click();
+}
+
 async function seriousConfirm(config: {
     message: string;
 }): Promise<boolean> {
@@ -192,6 +257,14 @@ class SnapshotViewerControllerBase {
         let files = await getSnapshot("live");
         await saveSnapshot({ files: files.files.map(x => x.file) });
     }
+
+    public async downloadSnapshot(snapshotFile: string | "live"): Promise<Buffer> {
+        return await downloadSnapshot(snapshotFile);
+    }
+
+    public async uploadSnapshot(buffer: Buffer): Promise<ArchiveSnapshotOverview> {
+        return await uploadSnapshot(buffer);
+    }
 }
 
 export const SnapshotViewerController = SocketFunction.register(
@@ -202,6 +275,9 @@ export const SnapshotViewerController = SocketFunction.register(
         loadSnapshot: {},
         getSnapshot: {},
         saveLiveSnapshot: {},
+        downloadSnapshot: {},
+        uploadSnapshot: {},
+        example: {}
     }),
     () => ({
         hooks: [assertIsManagementUser],
@@ -217,5 +293,6 @@ const SnapshotViewerSynced = getSyncedController(SnapshotViewerController, {
     },
     writes: {
         saveLiveSnapshot: ["snapshots"],
+        uploadSnapshot: ["snapshots"],
     }
 });

package/src/library-components/uncaughtToast.tsx

@@ -13,6 +13,8 @@ function onUncaught(...args: unknown[]) {
     // Ignore ResizeObserver errors, they are spurious
     //  - https://github.com/vercel/next.js/discussions/51551
     if (error.message.startsWith("ResizeObserver loop")) return;
+    // We should really do this better. Basically, if we're disposing or canceling, it's not actually an error, just ignore it. 
+    if (error.message.startsWith("Dispose")) return;
 
     onMessage({
         type: "error",