querysub 0.186.0 → 0.187.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.186.0",
+  "version": "0.187.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/1-path-client/RemoteWatcher.ts

@@ -128,15 +128,19 @@ export class RemoteWatcher {
     public watchLatest(config: WatchConfig & { debugName?: string }) {
         logErrors(this.watchLatestPromise(config));
     }
+    // IMPORTANT! If this shows up as being slow in a profile, try again with devtools closed! It is often slow because async calls are many times slower when devtools is open...
     // NOTE: This is private, as who wants to wait for watch to finish?
     //      1) The value won't be ready when it finishes.
     //      2) If it errors out, they can't do anything to fix the error
     //      3) We retry internally anyways
     private watchLatestPromise(config: WatchConfig & { debugName?: string }) {
-        // NOTE: If none of the values are remote... early out. This has been profile to save a bit of time,
+        // NOTE: If none of the values are remote... early out. This has been profiled to save a bit of time,
         //  mostly due to avoiding the async call.
-        if (config.parentPaths.length === 0 && config.paths.every(x => pathValueAuthority2.isSelfAuthority(x))) {
-            return;
+        if (config.parentPaths.length === 0) {
+            let isSelf = measureBlock(() => config.paths.every(x => pathValueAuthority2.isSelfAuthority(x)), "RemoteWatcher()|watchLatestPromise|isSelfCheck");
+            if (isSelf) {
+                return;
+            }
         }
         return this.watchLatestBase(config);
     }

package/src/1-path-client/pathValueClientWatcher.ts

@@ -440,6 +440,7 @@ export class ClientWatcher {
         this.allWatchers.delete(callback);
     }
 
+    @measureFnc
     private updateUnwatches(watchSpec: WatchSpec) {
         const { callback, paths, parentPaths } = watchSpec;
         let prevSpec = this.allWatchers.get(callback);
@@ -488,14 +489,15 @@ export class ClientWatcher {
         watchSpec = prevSpec;
 
         let expiryTime = Date.now() + ClientWatcher.WATCH_STICK_TIME;
-        if (isDevDebugbreak()) {
-            for (let path of fullyUnwatchedPaths) {
-                auditLog("clientWatcher UNWATCH", { path });
-            }
-            for (let path of fullyUnwatchedParents) {
-                auditLog("clientWatcher UNWATCH PARENT", { path });
-            }
-        }
+        // Commented out, as this is showing up as a bit slow in profiling
+        // if (isDevDebugbreak()) {
+        //     for (let path of fullyUnwatchedPaths) {
+        //         auditLog("clientWatcher UNWATCH", { path });
+        //     }
+        //     for (let path of fullyUnwatchedParents) {
+        //         auditLog("clientWatcher UNWATCH PARENT", { path });
+        //     }
+        // }
         for (let path of fullyUnwatchedPaths) {
             this.pendingUnwatches.set(path, expiryTime);
         }
@@ -550,35 +552,39 @@ export class ClientWatcher {
         this.updateUnwatches(watchSpec);
 
         this.allWatchers.set(callback, watchSpec);
-        for (let path of paths) {
-            let watchers = this.valueFunctionWatchers.get(path);
-            if (!watchers) {
-                watchers = new Map();
-                this.valueFunctionWatchers.set(path, watchers);
-            }
-            watchers.set(callback, watchSpec);
-            this.pendingUnwatches.delete(path);
-        }
-        for (let path of parentPaths) {
-            let basePath = hack_stripPackedPath(path);
-            let watchersBase = this.parentValueFunctionWatchers.get(basePath);
-            if (!watchersBase) {
-                watchersBase = new Map();
-                this.parentValueFunctionWatchers.set(basePath, watchersBase);
+        measureBlock(() => {
+            for (let path of paths) {
+                let watchers = this.valueFunctionWatchers.get(path);
+                if (!watchers) {
+                    watchers = new Map();
+                    this.valueFunctionWatchers.set(path, watchers);
+                }
+                watchers.set(callback, watchSpec);
+                this.pendingUnwatches.delete(path);
             }
-            let watchers = watchersBase.get(path);
-            if (!watchers) {
-                let range = decodeParentFilter(path) || { start: 0, end: 1 };
-                watchers = {
-                    start: range.start,
-                    end: range.end,
-                    lookup: new Map()
-                };
-                watchersBase.set(path, watchers);
+        }, "ClientWatcher()|setWatches|paths");
+        measureBlock(() => {
+            for (let path of parentPaths) {
+                let basePath = hack_stripPackedPath(path);
+                let watchersBase = this.parentValueFunctionWatchers.get(basePath);
+                if (!watchersBase) {
+                    watchersBase = new Map();
+                    this.parentValueFunctionWatchers.set(basePath, watchersBase);
+                }
+                let watchers = watchersBase.get(path);
+                if (!watchers) {
+                    let range = decodeParentFilter(path) || { start: 0, end: 1 };
+                    watchers = {
+                        start: range.start,
+                        end: range.end,
+                        lookup: new Map()
+                    };
+                    watchersBase.set(path, watchers);
+                }
+                watchers.lookup.set(callback, watchSpec);
+                this.pendingParentUnwatches.delete(path);
             }
-            watchers.lookup.set(callback, watchSpec);
-            this.pendingParentUnwatches.delete(path);
-        }
+        }, "ClientWatcher()|setWatches|parentPaths");
 
         // Audit code to check if we failed to correctly maintain our paths
         //  OR (more likely), the watcher mutated the paths Set, which breaks our watching
@@ -604,27 +610,33 @@ export class ClientWatcher {
         }
         */
 
-        let pathsArray = Array.from(paths);
-        let parentPathsArray = Array.from(parentPaths);
+        let pathsArray!: string[];
+        let parentPathsArray!: string[];
+        measureBlock(() => {
+            pathsArray = Array.from(paths);
+            parentPathsArray = Array.from(parentPaths);
+        }, "ClientWatcher()|setWatches|Array.from");
         let debugName = watchSpec.callback.name;
 
         if (pathsArray.length === 0 && parentPathsArray.length === 0) {
             return;
         }
 
-        // Watch it locally as well, so when we get the value we know to trigger the client callback
-        pathWatcher.watchPath({
-            callback: getOwnNodeId(),
-            paths: pathsArray,
-            parentPaths: parentPathsArray,
-            debugName,
-            noInitialTrigger: watchSpec.noInitialTrigger,
-        });
-        remoteWatcher.watchLatest({
-            paths: pathsArray,
-            parentPaths: parentPathsArray,
-            debugName,
-        });
+        measureBlock(() => {
+            // Watch it locally as well, so when we get the value we know to trigger the client callback
+            pathWatcher.watchPath({
+                callback: getOwnNodeId(),
+                paths: pathsArray,
+                parentPaths: parentPathsArray,
+                debugName,
+                noInitialTrigger: watchSpec.noInitialTrigger,
+            });
+            remoteWatcher.watchLatest({
+                paths: pathsArray,
+                parentPaths: parentPathsArray,
+                debugName,
+            });
+        }, "ClientWatcher()|setWatches|watchCalls");
     }
 
     private lastVersions = registerResource("paths|lastVersion", new Map<number, number>());

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -32,7 +32,7 @@ import { DEPTH_TO_DATA, MODULE_INDEX, getCurrentCall, getCurrentCallObj } from "
 import { inlineNestedCalls } from "../3-path-functions/syncSchema";
 import { interceptCallsBase, runCall } from "../3-path-functions/PathFunctionHelpers";
 import { deepCloneCborx } from "../misc/cloneHelpers";
-import { formatPercent } from "socket-function/src/formatting/format";
+import { formatPercent, formatTime } from "socket-function/src/formatting/format";
 import { addStatPeriodic, interceptCalls, onAllPredictionsFinished, onTimeProfile } from "../-0-hooks/hooks";
 import { onNextPaint } from "../functional/onNextPaint";
 
@@ -202,6 +202,12 @@ export interface WatcherOptions<Result> {
      *          what you want, it depends on the usecase.
      */
     runOncePerPaint?: string;
+
+    /** Logs time to go from unsynced to synced, and the paths that were unloaded.
+     *      - In order to tell what paths to pre-load, and how much time we are waiting to load data.
+     *      - Also to tell us how long and how many paths are loading on startup.
+    */
+    logSyncTimings?: boolean;
 }
 
 let harvestableReadyLoopCount = 0;
@@ -330,6 +336,15 @@ export type SyncWatcher = {
     hackHistory: { message: string; time: number }[];
 
     createTime: number;
+
+    logSyncTimings?: {
+        startTime: number;
+        unsyncedPaths: Set<string>;
+        unsyncedStages: {
+            paths: string[];
+            time: number;
+        }[];
+    };
 }
 function addToHistory(watcher: SyncWatcher, message: string) {
     watcher.hackHistory.push({ message, time: Date.now() });
@@ -860,7 +875,6 @@ export class PathValueProxyWatcher {
             return getKeys(pathValue?.value) as string[];
         }
 
-        let keys: string[] = [];
         let childPaths = authorityStorage.getPathsFromParent(pathStr);
 
         // We need to also get keys from pendingWrites
@@ -877,23 +891,26 @@ export class PathValueProxyWatcher {
             }
         }
 
+        let keys = new Set<string>();
         if (childPaths) {
             let targetDepth = getPathDepth(pathStr);
             for (let childPath of childPaths) {
                 let key = getPathIndexAssert(childPath, targetDepth);
                 let childValue = this.getCallback(childPath, undefined, "readTransparent");
                 if (childValue?.value !== undefined) {
-                    keys.push(key);
+                    keys.add(key);
                 }
             }
         }
 
+        let keysArray = Array.from(keys);
+
         // NOTE: Because getPathsFromParent does not preserve order, we have to sort here to ensure
         //  we provide a consistent order (which might not be the order the user wants, but at least
         //  it will always fail if it does fail).
-        keys.sort();
+        keysArray.sort();
 
-        return keys;
+        return keysArray;
     };
 
     private getSymbol = (pathStr: string, symbol: symbol): { value: unknown } | undefined => {
@@ -1255,8 +1272,56 @@ export class PathValueProxyWatcher {
             const specialPromiseUnsynced = watcher.specialPromiseUnsynced;
             watcher.specialPromiseUnsynced = false;
 
+            let anyUnsynced = watcher.hasAnyUnsyncedAccesses();
+            if (watcher.options.logSyncTimings) {
+                if (anyUnsynced) {
+                    watcher.logSyncTimings = watcher.logSyncTimings || {
+                        startTime: Date.now(),
+                        unsyncedPaths: new Set(),
+                        unsyncedStages: [],
+                    };
+                    let newPaths: string[] = [];
+                    let set = watcher.logSyncTimings.unsyncedPaths;
+                    function addPath(path: string) {
+                        if (set.has(path)) return;
+                        set.add(path);
+                        newPaths.push(path);
+                    }
+                    for (let path of watcher.lastUnsyncedAccesses) {
+                        addPath(path);
+                    }
+                    for (let path of watcher.lastUnsyncedParentAccesses) {
+                        addPath(path);
+                    }
+                    if (newPaths.length > 0) {
+                        watcher.logSyncTimings.unsyncedStages.push({
+                            paths: newPaths,
+                            time: Date.now(),
+                        });
+                    }
+                } else {
+                    if (watcher.logSyncTimings) {
+                        let now = Date.now();
+                        console.groupCollapsed(`${watcher.debugName} synced in ${formatTime(now - watcher.logSyncTimings.startTime)}`);
+                        // Log the stages
+                        let stages = watcher.logSyncTimings.unsyncedStages;
+                        for (let i = 0; i < stages.length; i++) {
+                            let nextTime = stages[i + 1]?.time || now;
+                            let stage = stages[i];
+                            console.groupCollapsed(`${formatTime(nextTime - stage.time)}`);
+                            for (let path of stage.paths) {
+                                console.log(path);
+                            }
+                            console.groupEnd();
+                        }
+                        console.groupEnd();
+                        watcher.logSyncTimings = undefined;
+                    }
+                }
+            }
+
             if (!watcher.options.static) {
-                if (!watcher.hasAnyUnsyncedAccesses()) {
+                if (!anyUnsynced) {
                     watcher.countSinceLastFullSync = 0;
                     watcher.lastSyncTime = Date.now();
                     watcher.syncRunCount++;

package/src/4-dom/qreact.tsx

@@ -134,6 +134,9 @@ export interface QComponentStatic {
      *           UNLESS the child explicitly sets this to false.
      * */
     multiRendersPerPaint?: boolean;
+
+    /** If true, logs time it takes for component to load synced data (if at all), and the paths that were unloaded. */
+    logLoadTime?: boolean;
 }
 
 /** See QComponentStatic.jsonComparePropUpdates */
@@ -416,7 +419,14 @@ function getAncestor(
 }
 function onDispose(fnc: () => void) {
     let component = getRenderingComponent();
-    if (!component) throw new Error(`Can only call onDispose if inside of a render (or componentDidMount) function. If an an event callback, call qreact.getRenderingComponent in render, and then call qreact.watchDispose(component, callback).`);
+    if (!component) {
+        if (QRenderClass.renderingComponentId !== undefined) {
+            // If it's already disposed... call the function immediately?
+            fnc();
+            return;
+        }
+        throw new Error(`Can only call onDispose if inside of a render (or componentDidMount) function. If an an event callback, call qreact.getRenderingComponent in render, and then call qreact.watchDispose(component, callback).`);
+    }
     watchDispose(component, fnc);
 }
 function watchDispose(renderClass: ExternalRenderClass, fnc: () => void) {
@@ -616,6 +626,7 @@ class QRenderClass {
         const renderWatcher = this.renderWatcher = proxyWatcher.createWatcher({
             debugName: getDebugName("render"),
             canWrite: true,
+            logSyncTimings: statics.logLoadTime,
             runOncePerPaint: !multiRendersPerPaint && `${this.debugName}|${self.id}` || undefined,
             watchFunction() {
 
@@ -859,6 +870,10 @@ class QRenderClass {
                         if ("data-break" in nextProps) {
                             debugger;
                         }
+                        todonext;
+                        // First of all, we are accessing deepProps on the WRONG value. We should be accessing it on the child statics, not our statics!
+                        // Second of all... we need to check if our child is JSON comparing, and if so, do an atomic write of the props. Usually we don't want to do an atomic write, as this would force an update every render, but... if it is JSON comparing, it won't force an update.
+                        //  - Which is actually trivial. We just literally set .props = nextProps. Done.
                         if (statics.deepProps) {
                             // NOTE: By removing atomic we can leverage the proxy automatically decomposing
                             //  the props to each individual primitive value, and checking each individual
@@ -1953,14 +1968,17 @@ function updateDOMNodeFields(domNode: DOMNode, vNode: VirtualDOM, prevVNode: Vir
                         if (typeof v === "number" && !IS_NON_DIMENSIONAL.test(key)) {
                             v = v + "px";
                         }
-                        (domNode as any).style[key] = v ?? "";
+                        if (String(v).endsWith("!important")) {
+                            (domNode as any).style.setProperty(key, String(v).slice(0, -"!important".length), "important");
+                        } else {
+                            (domNode as any).style[key] = v ?? "";
+                        }
                     }
                     if (canHaveChildren(prevValue)) {
                         for (let key in prevValue) {
                             if (!(key in value)) {
                                 (domNode as any).style[key] = "";
                             }
-
                         }
                     }
                 }

package/src/4-querysub/Querysub.ts

@@ -361,6 +361,9 @@ export class Querysub {
             allowProxyResults: true,
         });
     }
+    public static fastReadAsync<T>(fnc: () => T, options?: Partial<WatcherOptions<T>>) {
+        return Querysub.serviceWrite(fnc, { ...options, allowProxyResults: true });
+    }
     public static localRead<T>(fnc: () => T, options?: Partial<WatcherOptions<T>>) {
         return proxyWatcher.runOnce({
             watchFunction: fnc,
@@ -447,7 +450,7 @@ export class Querysub {
      *      you can't call this at the start of your function (as nothing will have been called yet).
      *  NOTE: This can also be used to prevent
      */
-    public static afterPredictions(callback: () => Promise<void>) {
+    public static afterPredictions(callback: () => MaybePromise<void>) {
         let calls = proxyWatcher.getTriggeredWatcher().pendingCalls.map(x => x.call);
         Querysub.onCommitFinished(async () => {
             await Promise.all(calls.map(x => Querysub.onCallPredict(x)));

package/src/5-diagnostics/qreactDebug.tsx

@@ -186,10 +186,15 @@ class WatchModal extends qreact.Component<{
                     || {}
                 }
                 style={
-                    pos === "fill" && {}
-                    || pos === "top" && { maxHeight: "100%" }
-                    || pos === "bottom" && { maxHeight: "100%" }
-                    || {}
+                    {
+                        ...(
+                            pos === "fill" && {}
+                            || pos === "top" && { maxHeight: "100%" }
+                            || pos === "bottom" && { maxHeight: "100%" }
+                            || {}
+                        ),
+                        background: "rgb(255, 255, 255)!important",
+                    }
                 }
                 onCancel={() => {
                     if (this.state.pos !== "fill") {
@@ -197,7 +202,7 @@ class WatchModal extends qreact.Component<{
                     }
                 }}
             >
-                <div class={css.hsl(0, 0, 100).vbox(6)}>
+                <div class={css.hsl(0, 0, 100).hslcolor(0, 0, 10).vbox(6)}>
                     <div class={css.hbox(10).fillWidth}>
                         {component.debugName}
                         {parent && <Button onClick={() => this.state.parentNavigate++}>

package/src/diagnostics/misc-pages/ArchiveViewerTree.tsx

@@ -53,7 +53,10 @@ export class ArchiveViewerTree extends qreact.Component<{
         let { remainingWidth, nodePath, scaleType } = config;
         const selectedPath = archiveTreeSelected.value;
 
-        const sumFormatter = this.props.sumFormatter || formatNumber;
+        let sumFormatter: (value: number) => string = formatNumber;
+        if (archiveTreeScale.value === "size") {
+            sumFormatter = this.props.sumFormatter || formatNumber;
+        }
 
         let isSelected = selectedPath === nodePath;
         let isDescendantSelected = selectedPath.startsWith(nodePath);

package/src/errors.ts

@@ -112,6 +112,7 @@ export function errorify(error: any, messageOverride?: string) {
         errorObj.stack = error;
     } else {
         errorObj.message = error;
+        errorObj.stack = error + "\n" + errorObj.stack;
     }
     if (messageOverride) {
         errorObj.message = messageOverride;

package/src/library-components/ATag.tsx

@@ -29,6 +29,7 @@ export type ATagProps = (
         rawLink?: boolean;
         lightMode?: boolean;
         noStyles?: boolean;
+        onRef?: (element: HTMLAnchorElement | null) => void;
     }
 );
 
@@ -51,6 +52,7 @@ export class ATag extends qreact.Component<ATagProps> {
             <a
                 tabIndex={0}
                 {...props}
+                ref={props.onRef}
                 className={
                     (isCurrent ? css.color(`hsl(110, 75%, ${lightness}%)`) : css.color(`hsl(210, 100%, ${lightness}%)`))
                     + css.textDecoration("none")

package/src/user-implementation/UserPage.tsx

@@ -251,16 +251,18 @@ export class UserPage extends qreact.Component {
                         <table>
                             <tr>
                                 <th>User ID</th>
+                                <th>Email</th>
                                 <th>Time</th>
                             </tr>
-                            {sort(Object.entries(userObj.invitedUsers), x => -x[1].time).map(([userId, inviteData]) => {
+                            {sort(Object.entries(userObj.invitedUsers2), x => -x[1].time).map(([email, inviteData]) => {
                                 return (
                                     <tr>
                                         <td>
-                                            <Anchor values={[{ param: viewingUserURL, value: userId }]}>
-                                                {userId}
+                                            <Anchor values={[{ param: viewingUserURL, value: inviteData.userId }]}>
+                                                {inviteData.userId}
                                             </Anchor>
                                         </td>
+                                        <td>{inviteData.email}</td>
                                         <td>{inviteData.time}</td>
                                     </tr>
                                 );

package/src/user-implementation/userData.ts

@@ -133,8 +133,10 @@ export type User = {
     };
 
     invitesRemaining: number;
-    invitedUsers: {
-        [userId: string]: {
+    invitedUsers2: {
+        [email: string]: {
+            userId: string;
+            email: string;
             time: number;
         };
     };
@@ -427,7 +429,7 @@ function getLoadingUserObj() {
         email: "",
         settings: { displayName: "" }, userType: "anonymous" as const,
         createTime: 0,
-        machineIds: {}, allowedIPs: {}, loginTokens: {}, lastPageLoadsIPs: {}, invitedUsers: {},
+        machineIds: {}, allowedIPs: {}, loginTokens: {}, lastPageLoadsIPs: {}, invitedUsers2: {},
         invitesRemaining: 0
     };
 }
@@ -452,7 +454,7 @@ export function getCurrentUserObj(): User | undefined {
             allowedIPs: {},
             loginTokens: {},
             lastPageLoadsIPs: {},
-            invitedUsers: {},
+            invitedUsers2: {},
             invitesRemaining: 0,
         };
     }
@@ -537,9 +539,10 @@ function internalCreateUser(config: {
             allowedIPs: {},
             loginTokens: {},
             lastPageLoadsIPs: {},
-            invitedUsers: {},
+            invitedUsers2: {},
             invitesRemaining: 0,
         };
+        data().secure.emailToUserId[email] = userId;
     }
     return data().users[userId];
 }
@@ -776,7 +779,7 @@ function registerPageLoadTime() {
 function inviteUser(config: { email: string }) {
     Querysub.ignorePermissionsChecks(() => {
         let curUserObj = getUserObjAssert();
-        if (config.email in curUserObj.invitedUsers) {
+        if (config.email in curUserObj.invitedUsers2) {
             console.info(`User ${config.email} already invited`);
             return;
         }
@@ -793,7 +796,9 @@ function inviteUser(config: { email: string }) {
             throw new Error("No invites remaining");
         }
 
-        curUserObj.invitedUsers[config.email] = atomicObjectWrite({
+        curUserObj.invitedUsers2[email] = atomicObjectWrite({
+            userId,
+            email,
             time: Querysub.getCallTime(),
         });
         curUserObj.invitesRemaining--;
@@ -923,7 +928,7 @@ export function scriptCreateUser(config: {
         allowedIPs: {},
         loginTokens: {},
         lastPageLoadsIPs: {},
-        invitedUsers: {},
+        invitedUsers2: {},
         invitesRemaining: 0,
     };
     data().secure.emailToUserId[email] = userId;
@@ -980,7 +985,7 @@ export async function registerServicePermissions() {
                 allowedIPs: {},
                 loginTokens: {},
                 lastPageLoadsIPs: {},
-                invitedUsers: {},
+                invitedUsers2: {},
                 invitesRemaining: 0,
             };
         }