querysub 0.325.0 → 0.327.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.325.0",
+  "version": "0.327.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-a-archives/archiveCache.ts

@@ -12,10 +12,11 @@ import { Args } from "socket-function/src/types";
 import { getArchivesBackblaze } from "./archivesBackBlaze";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { SizeLimiter } from "../diagnostics/SizeLimiter";
+import { isPublic } from "../config";
 
 const SIZE_LIMIT = new SizeLimiter({
     diskRoot: getStorageDir(),
-    maxBytes: 1024 * 1024 * 1024 * 50,
+    maxBytes: isPublic() ? 1024 * 1024 * 1024 * 250 : 1024 * 1024 * 1024 * 50,
     // Anything less than this and we can't even load enough weights models for a single task
     minBytes: 1024 * 1024 * 1024 * 8,
     maxDiskFraction: 0.3,

package/src/-c-identity/IdentityController.ts

@@ -191,6 +191,7 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
     };
     let signature = sign(threadKeyCert, payload);
     await timeoutToError(
+        // NOTE: This timeout has to be small as if we try to connect to a node to send it something time sensitive such as a PathValue and it takes too long it might result in us having a PathValue which is expired. The threshold is around 60 seconds and so we want to build a timeout calling a few different nodes before the PathValue expires. 
         10 * 1000,
         IdentityController.nodes[nodeId].changeIdentity(signature, payload),
         () => new Error(`Timeout calling changeIdentity for ${nodeId}`)

package/src/-d-trust/NetworkTrust2.ts

@@ -1,7 +1,7 @@
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { getIdentityCA, getMachineId, getOwnMachineId } from "../-a-auth/certs";
 import { getArchives } from "../-a-archives/archives";
-import { isNode, throttleFunction, timeInSecond } from "socket-function/src/misc";
+import { isNode, throttleFunction, timeInHour, timeInSecond } from "socket-function/src/misc";
 import { SocketFunctionHook } from "socket-function/SocketFunctionTypes";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { IdentityController_getMachineId } from "../-c-identity/IdentityController";
@@ -19,6 +19,7 @@ import { magenta } from "socket-function/src/formatting/logColors";
 // Cache the untrust list, to prevent bugs from causing too many backend reads (while also allowing
 //  bad servers which make request before their trust is verified from staying broken).
 const UNTRUST_CACHE_TIME = 30 * timeInSecond;
+const TRUSTED_CACHE_RESET_INTERVAL = timeInHour;
 
 const archives = lazy(() => getArchives("trust2/"));
 
@@ -57,6 +58,8 @@ export const isTrusted = measureWrap(async function isTrusted(machineId: string)
     // See the comment in requiresNetworkTrustHook for why clients have to trust all callers.
     if (isClient()) return true;
 
+    await populateTrustedCache();
+
     if (trustedCache.has(machineId)) {
         return true;
     }
@@ -70,30 +73,11 @@ export const isTrusted = measureWrap(async function isTrusted(machineId: string)
         return false;
     }
 
-    return await isTrustedBase(machineId);
-});
-let trustedCachePopulated = false;
-const isTrustedBase = runInSerial(measureWrap(async function isTrustedBase(machineId: string) {
-    if (!trustedCachePopulated) {
-        trustedCachePopulated = true;
-        let trustedMachineIds = await archives().find("");
-        lastArchivesTrusted = trustedMachineIds.slice();
-        for (let trustedMachineId of trustedMachineIds) {
-            trustedCache.add(trustedMachineId);
-            // NOTE: We don't load trust certs here, as we need to load them on demand in case the trust changes after our initial startup.
-        }
-    } else {
-        // Checking a single entry is a lot faster (as find is slow)
-        let trusted = await archives().get(machineId);
-        if (trusted) {
-            trustedCache.add(machineId);
-        }
+    // Checking a single entry is fast and if we don't trust them they'll be added to untrusted cache so it shouldn't slow things down by too much. 
+    let trusted = await archives().get(machineId);
+    if (trusted) {
+        trustedCache.add(machineId);
     }
-    // Always trust ourself
-    trustedCache.add(getOwnMachineId());
-
-    // NOTE: This only happens to servers that we connect to. Also we only allow the machine ID to be this special ID in the case it's on our domain. And because we use HTTPS when connecting to domains, it means that it must be implicitly trusted if it has a certificate for our domain.
-    trustedCache.add("127-0-0-1");
 
     if (!trustedCache.has(machineId)) {
         untrustedCache.set(machineId, Date.now() + UNTRUST_CACHE_TIME);
@@ -101,7 +85,24 @@ const isTrustedBase = runInSerial(measureWrap(async function isTrustedBase(machi
     } else {
         return true;
     }
-}));
+});
+let populateTrustedCache = lazy(async () => {
+    let trustedMachineIds = await archives().find("");
+    lastArchivesTrusted = trustedMachineIds.slice();
+    for (let trustedMachineId of trustedMachineIds) {
+        trustedCache.add(trustedMachineId);
+    }
+    // Always trust ourself
+    trustedCache.add(getOwnMachineId());
+
+    // NOTE: This only happens to servers that we connect to. Also we only allow the machine ID to be this special ID in the case it's on our domain. And because we use HTTPS when connecting to domains, it means that it must be implicitly trusted if it has a certificate for our domain.
+    trustedCache.add("127-0-0-1");
+
+    setTimeout(() => {
+        trustedCache.clear();
+        populateTrustedCache.reset();
+    }, TRUSTED_CACHE_RESET_INTERVAL);
+});
 
 export async function isNodeTrusted(nodeId: string) {
     let domainName = getNodeIdDomainMaybeUndefined(nodeId);

package/src/diagnostics/logs/FastArchiveAppendable.ts

@@ -516,6 +516,7 @@ export class FastArchiveAppendable<Datum> {
                 });
 
                 const onDecompressedData = createLogScanner({
+                    debugName: file.path,
                     onParsedData,
                 });
                 let batchedData: Buffer[] = [];
@@ -697,14 +698,18 @@ export class FastArchiveAppendable<Datum> {
 
 
 export function createLogScanner(config: {
+    debugName: string;
     onParsedData: (posStart: number, posEnd: number, buffer: Buffer | "done") => MaybePromise<void>;
 }): (data: Buffer | "done") => Promise<void> {
     const { onParsedData } = config;
     let pendingData: Buffer[] = [];
 
+    let finished = false;
+
     let delimitterMatchIndex = 0;
-    return runInSerial(async (data: Buffer | "done") => {
+    return (async (data: Buffer | "done") => {
         if (data === "done") {
+            finished = true;
             // Flush any pending data, even though we have no delimitter. It will probably fail to parse, but... maybe it will work?
             if (pendingData.length > 0) {
                 let combinedBuffer = Buffer.concat(pendingData);
@@ -714,6 +719,9 @@ export function createLogScanner(config: {
             await onParsedData(0, 0, "done");
             return;
         }
+        if (finished) {
+            throw new Error(`Finished scan, but we received more data: ${data.length}, sample is: ${data.slice(0, 100).toString("hex")}, ${config.debugName}`);
+        }
 
         let lastStart = 0;
         await measureBlock(async () => {
@@ -734,6 +742,7 @@ export function createLogScanner(config: {
                             ...pendingData,
                             data.slice(lastStart, i + 1),
                         ]).slice(0, -objectDelimitterBuffer.length);
+                        pendingData = [];
                         posStart = 0;
                         posEnd = buffer.length;
                     } else {
@@ -741,19 +750,12 @@ export function createLogScanner(config: {
                         posStart = lastStart;
                         posEnd = i + 1 - objectDelimitterBuffer.length;
                     }
-                    // Delimitter was the start of the chunk, and it's the first chunk. Just skip it.
-                    if (posStart === posEnd && i === 0) {
-                        lastStart = i + 1;
-                        continue;
-                    }
-
                     // Only sometimes awaiting here makes scanning almost 2X faster, in the normal case, somehow?
                     let maybePromise = onParsedData(posStart, posEnd, buffer);
                     if (maybePromise) {
                         await maybePromise;
                     }
 
-                    pendingData = [];
                     lastStart = i + 1;
                 }
             }

package/src/diagnostics/logs/FastArchiveViewer.tsx

@@ -15,7 +15,7 @@ import { ButtonSelector } from "../../library-components/ButtonSelector";
 import { Button } from "../../library-components/Button";
 import { lazy } from "socket-function/src/caching";
 import { LOG_LIMIT_FLAG } from "./diskLogger";
-import { canHaveChildren } from "socket-function/src/types";
+import { MaybePromise, canHaveChildren } from "socket-function/src/types";
 import { niceParse } from "../../niceStringify";
 import { FileMetadata } from "./FastArchiveController";
 
@@ -30,7 +30,8 @@ const caseInsensitiveParam = new URLParam("caseInsensitive", false);
 
 export class FastArchiveViewer<T> extends qreact.Component<{
     fastArchives: FastArchiveAppendable<T>[];
-    onStart: () => void;
+    runOnLoad?: boolean;
+    onStart: () => MaybePromise<void>;
     getWantData?: (file: FileMetadata) => Promise<((posStart: number, posEnd: number, data: Buffer) => boolean) | undefined>;
     onDatums: (source: FastArchiveAppendable<T>, datums: T[], metadata: FileMetadata) => void;
     // Called after onData
@@ -38,12 +39,13 @@ export class FastArchiveViewer<T> extends qreact.Component<{
     onFinish?: () => void;
 }> {
     state = t.state({
+        runCount: t.atomic<number>(0),
         // rootPath =>
         fileMetadata: t.atomic<({
             files: FileMetadata[];
             createTime?: number;
         } | undefined)[]>([]),
-        finished: t.atomic(false),
+        finished: t.atomic(true),
         error: t.atomic<string | undefined>(undefined),
         pendingSyncInitializations: t.atomic<number>(0),
 
@@ -108,6 +110,11 @@ export class FastArchiveViewer<T> extends qreact.Component<{
         this.latestSequenceNumber = this.currentSequenceNumber;
         const mySequenceNumber = this.currentSequenceNumber;
 
+        // Increment run count for each new run
+        Querysub.commit(() => {
+            this.state.runCount++;
+        });
+
         // Helper function to check if this sequence number is still the latest
         const isLatestSync = () => mySequenceNumber === this.latestSequenceNumber;
 
@@ -150,7 +157,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
 
             this.histogramStartTime = timeRange.startTime - this.histogramBucketTime * 2;
             this.histogramEndTime = timeRange.endTime + this.histogramBucketTime * 2;
-            const bucketCount = Math.ceil((this.histogramEndTime - this.histogramStartTime) / this.histogramBucketTime);
+            const bucketCount = clamp(Math.ceil((this.histogramEndTime - this.histogramStartTime) / this.histogramBucketTime), 1, 10000);
 
             this.histogramAllDataCounts = new Float64Array(bucketCount);
             this.histogramSelectedDataCounts = new Float64Array(bucketCount);
@@ -181,9 +188,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
             }
         };
         try {
-            ifLatest(() => {
-                onStart();
-            });
+            await onStart();
 
             const caseInsensitive = Querysub.fastRead(() => caseInsensitiveParam.value);
             let caseInsensitiveMapping = new Uint8Array(256);
@@ -572,7 +577,9 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         fillWidth
                         onKeyUp={this.handleDownload}
                         ref2={() => {
-                            void this.handleDownload();
+                            if (this.props.runOnLoad) {
+                                void this.handleDownload();
+                            }
                         }}
                         noEnterKeyBlur
                         placeholder="Filter terms, ex x | y & z"
@@ -587,17 +594,21 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         flavor="small"
                     />
                     <div className={css.vbox(10)}>
-                        {this.state.fileMetadata
-                            && (
-                                <div
-                                    className={infoDisplay(120)}
-                                    title={this.state.fileMetadata.map(x => x?.files || []).flat().map(x =>
-                                        `${x.path} (${formatNumber(x.size)})`
-                                    ).join("\n")}
-                                >
-                                    File count: {formatNumber(totalFileCount)}, Backblaze size: {formatNumber(totalBackblazeByteCount)}B (compressed), Disk size: {formatNumber(totalLocalByteCount)}B (uncompressed)
-                                </div>
-                            )}
+                        {this.state.runCount > 0 && (
+                            <div
+                                className={infoDisplay(120)}
+                                title={this.state.fileMetadata.map(x => x?.files || []).flat().map(x =>
+                                    `${x.path} (${formatNumber(x.size)})`
+                                ).join("\n")}
+                            >
+                                File count: {formatNumber(totalFileCount)}, Backblaze size: {formatNumber(totalBackblazeByteCount)}B (compressed), Disk size: {formatNumber(totalLocalByteCount)}B (uncompressed)
+                            </div>
+                        )}
+                        {this.state.runCount === 0 && (
+                            <div className={infoDisplay(200)}>
+                                No data downloaded yet. Click Run to download data.
+                            </div>
+                        )}
                         {this.state.finished && <div
                             className={infoDisplay(60).button}
                             onClick={() => {

package/src/diagnostics/logs/LogViewer2.tsx

@@ -144,7 +144,8 @@ export class LogViewer2 extends qreact.Component {
                 <FastArchiveViewer
                     ref2={x => this.fastArchiveViewer = x}
                     fastArchives={logs}
-                    onStart={() => {
+                    runOnLoad={errorNotifyToggleURL.value}
+                    onStart={async () => {
                         this.datumCount = 0;
                         this.notMatchedCount = 0;
                         this.errors = 0;
@@ -167,6 +168,9 @@ export class LogViewer2 extends qreact.Component {
                                 });
                             }
                         })();
+                        // ALWAYS update it, as the synchronous one might be out of date, and if we use an outdated one extra errors show up.
+                        suppressionList = await suppressionController.getSuppressionList.promise();
+
                     }}
                     getWantData={async (file) => {
                         if (!hasErrorNotifyToggle) return undefined;

package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts

@@ -224,22 +224,26 @@ class SuppressionList {
         suppressionUpdatedChannel.watch(() => {
             void this.updateEntriesNow();
         });
+        await runInfinitePollCallAtStart(SUPPRESSION_POLL_INTERVAL, async () => {
+            await this.updateEntriesNow();
+        });
     });
+    private cacheEntries: SuppressionListBase | undefined = undefined;
     private updateEntriesNow = async () => {
         let entries = await suppressionListArchive.get(suppressionListKey);
         if (!entries) {
             entries = { entries: {} };
         }
-        this.getEntries.set(Promise.resolve(entries));
+        this.cacheEntries = entries;
     };
-    private getEntries = lazy(async (): Promise<SuppressionListBase> => {
+    private async getEntries(): Promise<SuppressionListBase> {
         await this.init();
-        await runInfinitePollCallAtStart(SUPPRESSION_POLL_INTERVAL, async () => {
-            await this.updateEntriesNow();
-        });
+        if (!this.cacheEntries) {
+            throw new Error("Cache entries not set? Should be impossible.");
+        }
         // Infinite poll will have set this, so we don't infinitely loop
-        return await this.getEntries();
-    });
+        return this.cacheEntries;
+    }
 
     public async filterObjsToNonSuppressed(objs: LogDatum[]): Promise<LogDatum[]> {
         // NOTE: Streamed data should be rare enough, that handling this inefficiently is okay.
@@ -259,6 +263,7 @@ class SuppressionList {
         }
         let buffer = Buffer.concat(parts);
         let scanner = await this.scanForRecentErrors({
+            debugName: "filterObjsToNonSuppressed",
             startTime,
             endTime,
         });
@@ -266,6 +271,7 @@ class SuppressionList {
         return await scanner.finish();
     }
     public async scanForRecentErrors(config: {
+        debugName: string;
         startTime: number;
         endTime: number;
     }): Promise<{
@@ -285,6 +291,7 @@ class SuppressionList {
         // for the suppression key. 
         let obj: { outdatedSuppressionKey?: string } = {};
         let callback = createLogScanner({
+            debugName: config.debugName,
             onParsedData: (posStart, posEnd, buffer) => {
                 if (buffer === "done") {
                     return;
@@ -297,7 +304,7 @@ class SuppressionList {
                 try {
                     datum = JSON.parse(buffer.slice(posStart, posEnd).toString()) as LogDatum;
                 } catch (e: any) {
-                    process.stderr.write(`Failed to parse log datum in around ${buffer.slice(posStart, posEnd).slice(0, 100).toString("hex")}, error is:\n${e.stack}`);
+                    process.stderr.write(`Failed to parse log datum in around ${buffer.slice(posStart, posEnd).slice(0, 100).toString("hex")}, in source ${config.debugName}, error is:\n${e.stack}`);
                     return;
                 }
                 if (obj.outdatedSuppressionKey) {
@@ -307,18 +314,19 @@ class SuppressionList {
             },
         });
         let lastWaitTime = Date.now();
+        const stream = runInSerial(async (buffer: Buffer | "done") => {
+            // TODO: Maybe we should add this pattern to batching.ts? Basically, if we get called fast, we allow the calls through. BUT, if we called slowly OR we are doing a lot of processing (and so we are working for all of SELF_THROTTLE_INTERVAL), then we wait. This prevents this from taking over the machine. The back off is steep though, and if the machine is lagging we might reduce to a trickle, just getting 1 call in per SELF_THROTTLE_DELAY + synchronous lag from work in other parts of the program.
+            let now = Date.now();
+            if (now - lastWaitTime > SELF_THROTTLE_INTERVAL) {
+                await delay(SELF_THROTTLE_DELAY);
+                lastWaitTime = now;
+            }
+            await callback(buffer);
+        });
         return {
-            onData: runInSerial(async (buffer) => {
-                // TODO: Maybe we should add this pattern to batching.ts? Basically, if we get called fast, we allow the calls through. BUT, if we called slowly OR we are doing a lot of processing (and so we are working for all of SELF_THROTTLE_INTERVAL), then we wait. This prevents this from taking over the machine. The back off is steep though, and if the machine is lagging we might reduce to a trickle, just getting 1 call in per SELF_THROTTLE_DELAY + synchronous lag from work in other parts of the program.
-                let now = Date.now();
-                if (now - lastWaitTime > SELF_THROTTLE_INTERVAL) {
-                    await delay(SELF_THROTTLE_DELAY);
-                    lastWaitTime = now;
-                }
-                await callback(buffer);
-            }),
+            onData: stream,
             finish: async () => {
-                await callback("done");
+                await stream("done");
                 // NOTE: We COULD limit as we run, however... how many errors are we really going to encounter that AREN'T suppressed? Suppression is supposed to prevent overload anyways. I guess worst case scenario, yes, we could get overloaded, but... if we are logging more NEW errors than we can store in memory, we have bigger problems...
                 return limitRecentErrors(datums);
             },
@@ -341,7 +349,8 @@ class SuppressionList {
     }
 
     public async getSuppressionList(): Promise<SuppressionEntry[]> {
-        return Object.values((await this.getEntries()).entries);
+        let entries = Object.values((await this.getEntries()).entries);
+        return entries;
     }
 }
 const suppressionList = new SuppressionList();
@@ -597,6 +606,7 @@ class RecentErrors {
                     path = await urlCache.getURLLocalPath(file.url, hash);
                     if (!path) continue;
                     let scanner = await suppressionList.scanForRecentErrors({
+                        debugName: file.url,
                         startTime: file.startTime,
                         endTime: file.endTime,
                     });

package/src/diagnostics/logs/errorNotifications/ErrorWarning.tsx

@@ -10,7 +10,7 @@ import { ATag } from "../../../library-components/ATag";
 import { managementPageURL, showingManagementURL } from "../../managementPages";
 import { errorNotifyToggleURL } from "../LogViewer2";
 import { Querysub } from "../../../4-querysub/QuerysubController";
-import { nextId, timeInDay, timeInHour } from "socket-function/src/misc";
+import { deepCloneJSON, nextId, timeInDay, timeInHour } from "socket-function/src/misc";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { Icon } from "../../../library-components/icons";
 import { filterParam } from "../FastArchiveViewer";
@@ -133,14 +133,15 @@ export class ErrorWarning extends qreact.Component {
                 {topExpired &&
                     <div className={css.hbox(8)}>
                         <Button onClick={() => {
+                            let newObj = deepCloneJSON({
+                                ...topExpired!,
+                                expiresAt: Date.now() + timeInDay,
+                            });
                             void Querysub.onCommitFinished(async () => {
-                                await suppressionController.setSuppressionEntry.promise({
-                                    ...topExpired!,
-                                    expiresAt: Date.now() + timeInDay,
-                                });
+                                await suppressionController.setSuppressionEntry.promise(newObj);
                             });
                         }}>
-                            Ignore Again
+                            Ignore Again ({formatDateJSX(topExpired.expiresAt)})
                         </Button>
                         <div>
                             Match Pattern =

package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx

@@ -4,23 +4,42 @@ import { RecentErrorsController, recentErrorsChannel, watchRecentErrors } from "
 import { timeInSecond } from "socket-function/src/misc";
 import { formatDateTime } from "socket-function/src/formatting/format";
 
-
 //todonext
 
+// 1) Fix whatever is causing our memory usage to highly fluctuate. We seem to just go up and up and up from 100 megabytes all the way up to 300 megabytes and then we GC going back to 100. I turned on the socket function logging and I don't think we're sending network messages. And I ran a profile and it doesn't seem to be doing anything slow, but what are we even doing? We need to fix this. 
+//      - It could be the FPS display. It's pretty easy to check. 
+// YEP! I think it's the FPS display. I disabled it. Now we could see if the air parsing lag goes away and if it does and then comes back when we turn back on the FPS display then you know we're gonna have to fix that. I could understand lagging but why would it be allocating so much? Is it the animation rendering? It definitely could be that because the animations are slow to render and can lag Chrome and can lag the GPU and can lag other windows. 
+
+// 2) The error parsing page is clearly lagging. Even when loading just the error data for the last few days, which is less than a megabytes of total logs, the error loading animation is locking up. Stuttering that is. We need to fix this. 
+
+
+// Deploy everything and then update the server runner itself. 
+
+// The constant error notifications might be fixed now. We'll see tomorrow after all the rolling updates finish. 
+// 4) fix whatever's causing constant error notifications. Something is broadcasting on the Recent Errors Change channel constantly. 
+//      - I guess usually there's no server that's going to be listening on it. So it's... Not that big of a deal, but it's still annoying. 
 
-// 5) Add a channel to watch suppression entries, as once we suppress it, we don't want notifications anymore anywhere, And we don't want to have to wait
-//      - test by logging when we get errors and then having our script which automatically keeps adding errors to keep running so we can verify that we keep logging new errors and then ignore the errors and this suppression should propagate and cause us to no longer omit the errors.
-//      - This will also cause the errors to be suppressed across different HTTP servers. 
 
-// 5) Get IMs actually sending. 
+
+
+// Back to getting a Node.js watcher that works smoothly without receiving a whole bunch of extra errors etc.
+//  4.1) update channel watching so you can specify that you want to watch only on a specific node ID and then update our code so we only watch it on the controller node ID that we're interfacing with. 
+
+// 5) Verify our suppression updates broadcast across the channel correctly, causing us to be able to suppress a notification and our watching script to then stop seeing the new updates. Realistically, it's the calling script that stops setting them, but same thing. 
+
+// 5) Set up the Instant Messaging Sending API. 
+//  - Discord. With beeper it won't really matter what we're messaging. We could also do WhatsApp. It's really all the same.  
 
 // 6) Set up all the code to properly rate limit IMs, batch them, link back to the log page, etc. 
 //      - Just link to the error page for the last week. We don't need to link to anything specific. 
 //      - properly getting the node ID that we're going to be watching, and if it goes down, getting a new one, and ignoring messages from the old node. 
 //      - And if no node exists, we need to warn and then wait. 
 
+
 // 7) Write the digest script, which is very different, but will run in the same entry.
-//      - Separate warnings and errors and also bucket by time bucket 
+//      - Separate warnings and errors and also bucket by time bucket
+//      - suppressed errors by time bucket (but no type, as we definitely don't want to parse all suppressed errors...)
+//      - Time the entire thing, and put that, and the profile, in the digest too! That will give us a good gauge on if the errors/suppressions are getting slow (due to a lot of errors, or a lot of suppression checks!)
 // 8) Write a page that shows the results of the digest in tabs, writing the digest probably just to backblaze 
 //      - For now, just have two tabs, one for errors and one for warnings. 
 //      - If we're going to do a full scan, we might as well show time series data as well. It's trivial. 

package/src/diagnostics/logs/lifeCycleAnalysis/spec.md

@@ -20,16 +20,6 @@ Very small amount of data
     - For now this will just be for:
         - non-suppressed errors
         - suppressed errors
-    - Eventually the goal of this is to add our tracking charts to this. There are some really useful metrics we can track. 
-        - unique visit IPs. 
-        - Percent bounces. 
-        - average visit length and also median visit length top 95" bottom 5" 
-        - average stories read average story percentages read 
-        - percent of first page story views by subscribers percent of fifth page story view by subscribers 
-        - Number of users visiting the fifth page.
-        - New subscriptions
-        - Subscription cancellations
-        - Total subscriptions
 
 
 
@@ -55,51 +45,9 @@ Very small amount of data
         2.1) Collections of life cycles so we can further reduce the complexity. 
     - Uses FastArchiveViewer, but instead of showing a table, shows lifecycles (a derived concept)
         - We save them in backblaze, with a bit of cache for loading them
-    - List of life cycles
-    - Life cycle
-        - Title
-        - Operation list (each supports | / &, but having multiple is even better)
-            - Match filter
-            - Group key extractions (optional, if not set it becomes a singleton)
-                - Just a field name
-                - CAN have multiple, which adds us as multiple life cycles
-                    - With each one being namespaced using the key, so we can tell them apart
-            - Global value setting (optional, if not set it has no global state impact)
-                - A list of set values
-                    - Each one is an expression which can use fields in the object, ex:
-                    - `alivePathValueServers.$threadId = true`
-            - Show AND SHOULDN'T include match filters!
-                - So when we should the count matched, we can show stats for these, which will often be should have "finished", and shouldn't have "error", so we can see completed, and errors
-                - Similar to error notifications, but... it's nice to also have this here, as we could miss the notification, or suppress it, but when we are looking at a life cycle it's relevant skyrockets.
-                - ALSO for start, so we can see cutoff starts!
-    OH! How do we handle cut off starts?
-        - Maybe... we have a "read preload" duration, and... we read that, BUT, only include life cycles which are also in our main selected time. So we don't cut anything off in our main time, but don't add new values which also get cut off!
-    - Same time/machine/thread selector as log viewer
-    - Allow filtering to specific life cycles
-        - After download, shows matches per life cycle
-        - Button to reset to all
-    - Download logs, and scan for selected life cycles
-        - Immediately on load, showing progress and throttling, so it's not too bad
-    - Result 
-        - List of life cycles, with count of each
-        - Table of individual life cycles?
-        - Can then filter within these life cycles by searching
-        - BUT, importantly, if any log is matched in a life cycle, the entire life cycle is matched
-        - AND, global settings for ALL life cycles are applied, not just filtered ones!
-    - Table of result life cycles
-        - Preview shows first matched line
-        - ALSO, shows duration of life cycle!
-        - And start date
-        - Expand to see pin that specific life cycle above
-            - Show list of logs in it, in another table, with searching on each of them
-            - Start date AND duration of each line!
-        - Can pin multiple life cycles (I guess might as well)
-    - Show list of global value expressions as well (limited, but with filtering to search them easily)
-        - Can expand a global value to see object (but again... limited?)
-    - Can select a specific global value path, to have it injected into
-        - a column for before and after each life cycle
-        - A column for after each line within a life cycle
-        - Can also select parent values, to show all paths under that (use ShowMore, to make them manageable...)
+    - show the life cycles and allow viewing just the specific life cycles and then drilling into those life cycles. Also allow viewing multiple at once so we can view a stream that has many life cycles (as in life cycle collections, which we can also save. )
+    - Show overlap when we're showing the list of life cycles by having it first sorted by start time and then have some kind of indicator for how many values after the value it overlaps with. So if it's serial there'll be no overlap. If everything is being queued up and then run in serial we'll see the overlap go from 1, 2, 3, 4, 5, 6, 7, 8, etc. If it's just in parallel, it'll count up to a number and then go up and down a little bit as values are added and removed. If everything's added at once and it's in parallel, then the values will go up, but then they'll very quickly go down.
+        - Maybe we should have some indication for how much parallel overlap there is? Or kind of like how much gap there is to the start time of the next thing and to the end time of the next thing, something like that. 
 
 6) Add life cycles for
     - Node discovery life cycle (new node, check for alive, check for paths, etc)
@@ -112,6 +60,7 @@ Very small amount of data
         - Use different keys for threadId, and, triggeredNodeId, so we can track how a node discovers other nodes, vs how a node is discovered
         - Mark as dead, dead count increases, remove node as dead
             - MAYBE there should be 2 lifecycles, one for all, and one for just creation type stuff (no discovery)
+    - There's a life cycle for something dying which starts when we first see it's dead and ends when the count reaches the maximum. We want to use the capability to change the ending state so we can have it as partially dead but not fully dead or fully dead so we can see if the dead life cycles keep starting but not finishing which would be bad. 
     - Trusted machine lifecycle
         - Check if we need to add trust
         - Add trust to archives
@@ -138,6 +87,10 @@ Very small amount of data
     - Re-enable all of our services as well.
     - Just logs MIGHT be able to do it, but... life cycles should make it a lot easier to correlate logs, which is something we need to do anyways to solve it...
 
+Make sure we check our life cycles for nodes being added and removed to make sure that life cycle path is pretty empty and there aren't nodes constantly being lost and re-added. 
+
+Check the startup lifecycle to make sure we can detect the nodes pretty fast and in parallel, instead of serial
+
 10) Verify old user/fast-log-cache machine folders are deleted
 
 

package/src/library-components/SyncedController.ts

@@ -309,11 +309,21 @@ export function getSyncedController<T extends SocketRegistered>(
                         obj.promise = undefined;
                         obj.invalidated = true;
                         call(...args);
+                        // Assign to itself, to preset the type assumptions typescript makes (otherwise we get an error below)
+                        obj = obj as any;
                         let promise = atomic(obj.promise);
                         if (!promise) {
                             debugger;
                             throw new Error(`Impossible, called function, but promise is not found for ${fncName}`);
                         }
+                        // Don't cache promise calls
+                        void promise.finally(() => {
+                            Querysub.fastRead(() => {
+                                if (obj.promise === promise) {
+                                    obj.promise = undefined;
+                                }
+                            });
+                        });
                         return promise;
                     });
                 };

package/src/library-components/icons.tsx

@@ -9,6 +9,9 @@ import { renderToString } from "./renderToString";
 import { debugTime } from "../../src/0-path-value-core/pathValueCore";
 import { measureBlock } from "socket-function/src/profiling/measure";
 
+// TODO: Create more icons with: https://www.recraft.ai/project/e2a2200f-bed2-4426-b8f1-e5120b9dc990
+//  - This should also make creating animated SVGs a lot easier. If the base is nice, and we add comments, we can probably get the AI to help animate it
+
 // Most icons are from https://www.figma.com/file/eVpKKmt8uOKmSYKW4LyusF/Free-Icon-Pack-1600%2B-icons-(Community)?node-id=1654-9894&t=0bDbK0bA9KGpswRE-0
 
 // TODO: Add a build step that does this (storing the .svgs in a file), so we don't

package/src/server.ts

@@ -33,7 +33,8 @@ async function main() {
 
     Error.stackTraceLimit = 20;
 
-    //SocketFunction.logMessages = true;
+    // SocketFunction.logMessages = true;
+    // SocketFunction.silent = false;
 
     // ClientWatcher.DEBUG_READS = true;
     // ClientWatcher.DEBUG_WRITES = true;