querysub 0.312.0 → 0.314.0

package/src/diagnostics/logs/errorNotifications/ErrorSuppressionUI.tsx

@@ -0,0 +1,225 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../../4-dom/qreact";
+import { NOT_AN_ERROR_EXPIRE_TIME, RecentErrorsController, SuppressionEntry, SuppressionListController, getSuppressEntryChecker } from "./ErrorNotificationController";
+import { t } from "../../../2-proxy/schema2";
+import { css } from "../../../4-dom/css";
+import { InputLabel } from "../../../library-components/InputLabel";
+import { Button } from "../../../library-components/Button";
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { nextId, sort, timeInDay } from "socket-function/src/misc";
+import { formatNumber, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
+import { formatDateJSX } from "../../../misc/formatJSX";
+import { LogDatum } from "../diskLogger";
+
+export class ErrorSuppressionUI extends qreact.Component<{
+    dataSeqNum: number;
+    suppressionCounts: Map<string, number>;
+    expiredSuppressionCounts: Map<string, number>;
+    datums: LogDatum[];
+    rerunFilters: () => void;
+}> {
+    state = t.state({
+        matchedInput: t.string(""),
+    });
+
+    private calculatePreviewMatchCount(pattern: string): number {
+        if (!pattern.trim()) return 0;
+
+        try {
+            let entry: SuppressionEntry = {
+                key: "preview",
+                match: pattern,
+                comment: "",
+                lastUpdateTime: Date.now(),
+                expiresAt: Date.now() + timeInDay,
+            };
+            let checker = getSuppressEntryChecker(entry);
+
+            let matchCount = 0;
+            for (let datum of this.props.datums.slice(0, 1000)) {
+                let datumStr = JSON.stringify(datum);
+                let buffer = Buffer.from(datumStr);
+                if (checker.fnc(buffer, 0, buffer.length)) {
+                    matchCount++;
+                }
+            }
+            return matchCount;
+        } catch (error) {
+            return 0;
+        }
+    }
+
+    public render() {
+        this.props.dataSeqNum;
+        const controller = SuppressionListController(SocketFunction.browserNodeId());
+        const entries = (controller.getSuppressionList() || []);
+        sort(entries, x => -x.lastUpdateTime);
+        sort(entries, x => -(this.props.suppressionCounts.get(x.key) ? 1 : 0));
+        sort(entries, x => -(this.props.expiredSuppressionCounts.get(x.key) ? 1 : 0));
+
+        const previewMatchCount = this.calculatePreviewMatchCount(this.state.matchedInput);
+
+        return <div className={css.vbox(16).pad2(16).fillWidth.bord2(0, 0, 50, 5).hsl(0, 0, 80)}>
+            <h2 className={css.fontSize(18).marginTop(0)}>Error Suppression List</h2>
+
+            <div className={css.hbox(8).fillWidth}>
+                <InputLabel
+                    label="Add Match"
+                    value={this.state.matchedInput}
+                    hot
+                    onChangeValue={(value: string) => this.state.matchedInput = value}
+                    onKeyDown={e => {
+                        if (e.key === "Enter") {
+                            let value = e.currentTarget.value;
+                            e.currentTarget.value = "";
+                            void Querysub.onCommitFinished(async () => {
+                                await controller.setSuppressionEntry.promise({
+                                    key: nextId(),
+                                    match: value,
+                                    comment: "",
+                                    lastUpdateTime: Date.now(),
+                                    expiresAt: Date.now() + timeInDay,
+                                });
+                                if (!e.shiftKey) {
+                                    Querysub.commit(() => {
+                                        this.props.rerunFilters();
+                                    });
+                                }
+                            });
+                        }
+                    }}
+                    fillWidth
+                    placeholder="* is supported as a wildcard (enter to submit and refresh, shift+enter to submit and do not refresh)"
+                />
+
+                {this.state.matchedInput.trim() && <span className={css.opacity(0.6).fontSize(12).flexShrink0}>
+                    matches {formatNumber(previewMatchCount)} (in top 1000)
+                </span>}
+                <Button onClick={() => {
+                    let value = this.state.matchedInput;
+                    this.state.matchedInput = "";
+                    void Querysub.onCommitFinished(async () => {
+                        await controller.setSuppressionEntry.promise({
+                            key: nextId(),
+                            match: value,
+                            comment: "",
+                            lastUpdateTime: Date.now(),
+                            expiresAt: NOT_AN_ERROR_EXPIRE_TIME,
+                        });
+                        Querysub.commit(() => {
+                            this.props.rerunFilters();
+                        });
+                    });
+                }}>
+                    Not an error
+                </Button>
+                <Button
+                    onClick={() => {
+                        let value = this.state.matchedInput;
+                        this.state.matchedInput = "";
+                        void Querysub.onCommitFinished(async () => {
+                            await controller.setSuppressionEntry.promise({
+                                key: nextId(),
+                                match: value,
+                                comment: "",
+                                lastUpdateTime: Date.now(),
+                                expiresAt: Date.now() + timeInDay,
+                            });
+                            Querysub.commit(() => {
+                                this.props.rerunFilters();
+                            });
+                        });
+                    }}
+                    title="Fixed, or will fix within a day"
+                >
+                    Fixed
+                </Button>
+            </div>
+
+            <div className={css.vbox(8).fillWidth.overflowAuto.maxHeight("30vh")}>
+                {entries.map((entry) => {
+                    const updateEntry = (changes: Partial<SuppressionEntry>) => {
+                        let newEntry = { ...entry, ...changes };
+                        void Querysub.onCommitFinished(async () => {
+                            await controller.setSuppressionEntry.promise(newEntry);
+                        });
+                    };
+                    let count = this.props.suppressionCounts.get(entry.key) || 0;
+                    let expiredCount = this.props.expiredSuppressionCounts.get(entry.key) || 0;
+                    return <div
+                        key={entry.match}
+                        className={
+                            css.hbox(8).pad2(12).bord2(0, 0, 10).fillWidth
+                            //+ (entry.expiresAt < Date.now() && expiredCount > 0 && css.opacity(0.5))
+                            + ((count === 0 && expiredCount === 0) && css.opacity(0.6))
+                            + (
+                                count > 0 && entry.expiresAt !== NOT_AN_ERROR_EXPIRE_TIME && css.hsla(0, 50, 50, 0.5)
+                                || css.hsla(0, 0, 0, 0.1)
+                            )
+                        }
+                    >
+                        <span className={css.width(100).hbox(2) + (expiredCount === 0 && count === 0 && css.opacity(0.8))}>
+                            ({formatNumber(count)})
+                            {
+                                expiredCount > 0 && <span className={css.hsl(0, 50, 50).pad2(4, 2).colorhsl(0, 50, 95)}>
+                                    + {formatNumber(expiredCount)} EXPIRED
+                                </span>
+                            }
+                        </span>
+                        <InputLabel
+                            label="Match Pattern"
+                            value={entry.match}
+                            onChangeValue={value => updateEntry({ match: value })}
+                            fillWidth={0.5}
+                        />
+                        <InputLabel
+                            label="Comment"
+                            value={entry.comment}
+                            onChangeValue={value => updateEntry({ comment: value })}
+                            fillWidth
+                        />
+
+                        <Button
+                            onClick={() => updateEntry({ expiresAt: Date.now() + timeInDay })}
+                            title="Fixed, or will fix within a day"
+                        >
+                            Fixed
+                        </Button>
+                        <Button onClick={() => updateEntry({ expiresAt: Date.now() + timeInDay * 7 })}
+                            title="Ignore for a week"
+                        >
+                            Ignore (for a week)
+                        </Button>
+                        <Button onClick={() => updateEntry({ expiresAt: NOT_AN_ERROR_EXPIRE_TIME })}>
+                            Not an error
+                        </Button>
+                        <Button
+                            onClick={() => updateEntry({ expiresAt: Date.now() - timeInDay * 7 })}
+                            title="Unignore"
+                        >
+                            Unignore
+                        </Button>
+                        {entry.expiresAt === NOT_AN_ERROR_EXPIRE_TIME && <span>
+                            Not an error
+                        </span>}
+                        {entry.expiresAt < Date.now() && <span>
+                            Expired
+                        </span>}
+                        {entry.expiresAt > Date.now() && entry.expiresAt !== NOT_AN_ERROR_EXPIRE_TIME && <span>
+                            Expires in {formatDateJSX(Date.now() * 2 - entry.expiresAt)}
+                        </span>}
+                        <span className={css.opacity(0.5)}>
+                            (Last Updated: {formatDateJSX(entry.lastUpdateTime)})
+                        </span>
+                        <Button
+                            onClick={() => controller.removeSuppressionEntry.promise(entry.key)}
+                            hue={120}
+                        >
+                            Remove
+                        </Button>
+                    </div>;
+                })}
+            </div>
+        </div>;
+    }
+}

package/src/diagnostics/logs/errorNotifications/ErrorWarning.tsx

@@ -0,0 +1,207 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../../4-dom/qreact";
+import { css } from "../../../4-dom/css";
+import { isCurrentUserSuperUser } from "../../../user-implementation/userData";
+import { RecentErrorsController, SuppressionListController, watchRecentErrors, MAX_RECENT_ERRORS, NOT_AN_ERROR_EXPIRE_TIME, SuppressionEntry } from "./ErrorNotificationController";
+import { t } from "../../../2-proxy/schema2";
+import { InputLabel } from "../../../library-components/InputLabel";
+import { Button } from "../../../library-components/Button";
+import { ATag } from "../../../library-components/ATag";
+import { managementPageURL, showingManagementURL } from "../../managementPages";
+import { errorNotifyToggleURL } from "../LogViewer2";
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { nextId, timeInDay } from "socket-function/src/misc";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { Icon } from "../../../library-components/icons";
+
+export class ErrorWarning extends qreact.Component {
+    state = t.state({
+        suppressionInput: t.string(""),
+        reloading: t.boolean(false),
+    });
+
+    public render() {
+        if (!isCurrentUserSuperUser()) return undefined;
+        watchRecentErrors();
+
+        let recentController = RecentErrorsController(SocketFunction.browserNodeId());
+        let suppressionController = SuppressionListController(SocketFunction.browserNodeId());
+
+        let recentErrors = recentController.getRecentErrors();
+
+        const refreshButton =
+            <Button
+                onClick={() => {
+                    this.state.reloading = true;
+                    void Querysub.onCommitFinished(async () => {
+                        try {
+                            await recentController.rescanAllErrorsNow.promise();
+                        } finally {
+                            Querysub.commit(() => {
+                                this.state.reloading = false;
+                            });
+                        }
+                    });
+                }}
+                flavor="noui"
+                className={
+                    css.button.hsla(0, 0, 50, 1).pad2(4, 1).borderRadius(2)
+                    + (this.state.reloading && css.animation("ErrorWarning-rotate 1s linear infinite"))
+                }
+            >
+                ⟳
+                <style>
+                    {
+                        `@keyframes ErrorWarning-rotate {
+                        from {
+                            transform: rotate(0deg);
+                        }
+                        to {
+                            transform: rotate(360deg);
+                        }
+                    }
+                    `}
+                </style>
+            </Button>;
+
+        if (!recentErrors || recentErrors.length === 0) {
+            return <span className={css.hbox(8)}>
+                <ATag target="_blank" values={[showingManagementURL.getOverride(true), managementPageURL.getOverride("LogViewer2"), errorNotifyToggleURL.getOverride(true)]}>
+                    No Errors
+                </ATag>
+                {refreshButton}
+            </span>;
+        }
+
+        // Count unique files
+        let fileSet = new Set<string>();
+        for (let error of recentErrors) {
+            let file = String(error.__FILE__) || "";
+            if (file) {
+                fileSet.add(file);
+            }
+        }
+        let fileCount = fileSet.size;
+        let fileCountText = `${formatNumber(fileCount)} files matched`;
+        if (recentErrors.length >= MAX_RECENT_ERRORS) {
+            fileCountText = "> " + fileCountText;
+        }
+
+        const addSuppressionEntry = async (match: string, expiresAt: number) => {
+            await suppressionController.setSuppressionEntry.promise({
+                key: nextId(),
+                match: match,
+                comment: "",
+                lastUpdateTime: Date.now(),
+                expiresAt: expiresAt,
+            });
+        };
+
+        let suppressionList = suppressionController.getSuppressionList() || [];
+        let topExpired: SuppressionEntry | undefined = undefined;
+        if (recentErrors[0].__matchedOutdatedSuppressionKey) {
+            let key = recentErrors[0].__matchedOutdatedSuppressionKey;
+            topExpired = suppressionList.find(x => x.key === key);
+
+        }
+
+        return (
+            <div className={css.vbox(8).fillWidth}>
+                <div className={css.hbox(8).wrap}>
+                    <div className={css}>
+                        ⚠️ {fileCountText} files with errors
+                    </div>
+
+                    <ATag target="_blank" values={[showingManagementURL.getOverride(true), managementPageURL.getOverride("LogViewer2"), errorNotifyToggleURL.getOverride(true)]}>
+                        View Logs
+                    </ATag>
+                    {refreshButton}
+                </div>
+
+                {topExpired &&
+                    <div className={css.hbox(8)}>
+                        <Button onClick={() => {
+                            void Querysub.onCommitFinished(async () => {
+                                await suppressionController.setSuppressionEntry.promise({
+                                    ...topExpired!,
+                                    expiresAt: Date.now() + timeInDay,
+                                });
+                            });
+                        }}>
+                            Ignore Again
+                        </Button>
+                        <div>
+                            Match Pattern =
+                        </div>
+                        <div className={css.hsl(200, 40, 40).pad2(4, 2).colorhsl(0, 0, 95)}>
+                            {topExpired.match}
+                        </div>
+                    </div>
+                }
+                <div className={css.hbox(8).hsl(0, 50, 50).pad2(4, 2).colorhsl(0, 50, 95)}>
+                    {recentErrors[0].param0} ({recentErrors[0].__NAME__})
+                </div>
+
+                <div className={css.hbox(8).fillWidth}>
+                    <InputLabel
+                        value={this.state.suppressionInput}
+                        noEnterKeyBlur
+                        onChangeValue={(value: string) => this.state.suppressionInput = value}
+                        onKeyDown={e => {
+                            if (e.key === "Enter") {
+                                let value = e.currentTarget.value;
+                                if (value.trim()) {
+                                    e.currentTarget.value = "";
+                                    void Querysub.onCommitFinished(async () => {
+                                        await addSuppressionEntry(value, Date.now() + timeInDay);
+                                        Querysub.commit(() => {
+                                            this.state.suppressionInput = "";
+                                        });
+                                    });
+                                }
+                            }
+                        }}
+                        placeholder="Quick suppress pattern (press enter to add)"
+                        fillWidth
+                    />
+
+
+                    <Button
+                        onClick={() => {
+                            let value = this.state.suppressionInput;
+                            if (value.trim()) {
+                                void Querysub.onCommitFinished(async () => {
+                                    await addSuppressionEntry(value, Date.now() + timeInDay);
+                                    Querysub.commit(() => {
+                                        this.state.suppressionInput = "";
+                                    });
+                                });
+                            }
+                        }}
+                        className={css.fontSize(12).pad2(4, 6)}
+                        disabled={!this.state.suppressionInput.trim()}
+                    >
+                        Fixed
+                    </Button>
+                    <Button
+                        onClick={() => {
+                            let value = this.state.suppressionInput;
+                            if (value.trim()) {
+                                void Querysub.onCommitFinished(async () => {
+                                    await addSuppressionEntry(value, NOT_AN_ERROR_EXPIRE_TIME);
+                                    Querysub.commit(() => {
+                                        this.state.suppressionInput = "";
+                                    });
+                                });
+                            }
+                        }}
+                        className={css.fontSize(12).pad2(4, 6)}
+                        disabled={!this.state.suppressionInput.trim()}
+                    >
+                        Not a bug
+                    </Button>
+                </div>
+            </div>
+        );
+    }
+}

package/src/diagnostics/logs/importLogsEntry.ts

@@ -0,0 +1,38 @@
+import { Querysub } from "../../4-querysub/Querysub";
+import fs from "fs";
+import { Zip } from "../../zip";
+import { getLoggers } from "./diskLogger";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { shutdown } from "../periodic";
+
+async function main() {
+    let { logLogs } = getLoggers()!;
+    //let root = "X:/root/machine-services/cyoa-1-dply/git/database-storage/disklogs";
+    /*
+    let root = "D:/repos/qs-cyoa/database-storage/disklogs/";
+    let files = await fs.promises.readdir(root);
+    for (let fileName of files) {
+        let path = root + fileName;
+        if (!path.endsWith(".log")) continue;
+        let contents = await fs.promises.readFile(path);
+        let lines = contents.toString("utf8").split("\n");
+        for (let line of lines) {
+            if (!line) continue;
+            try {
+                let log = JSON.parse(line);
+                logLogs.append(log);
+            } catch (e) {
+                require("debugbreak")(2);
+                debugger;
+                console.error(e, line);
+            }
+        }
+        console.log(`Flushing ${path}, line count: ${formatNumber(lines.length)}, ${formatNumber(contents.length)}B`);
+        let time = Number(fileName.split("-")[0]) || Date.now();
+        await logLogs.flushNow(time);
+    }
+    */
+    await logLogs.moveLogsToBackblaze();
+    //await shutdown();
+}
+main().catch(console.error).finally(() => process.exit());

package/src/diagnostics/logs/injectFileLocationToConsole.ts

@@ -1,9 +1,10 @@
 import { css, isNode } from "typesafecss";
 import { compileTransformBefore } from "typenode";
+
 // IMPORTANT! Don't add any imports BEFORE we add compileTransform
 if (isNode()) {
-    // NOTE: While mapping line numbers would be nice, just adding the file is easy,
-    //  and provides a huge benefit. And... you should be able to trace down the specific
+    // NOTE: While mapping line numbers would be nice, just adding the file is easier,
+    //  and still provides a huge benefit. And... you should be able to trace down the specific
     //  line from the message (and if not, you should make the message more specific).
     compileTransformBefore((contents, path, module) => {
         if (![".js", ".jsx", ".ts", ".tsx"].some(x => path.endsWith(x))) {
@@ -34,21 +35,10 @@ if (isNode()) {
             return newConsole;
         }
 
-        // Use line numbers BEFORE we inject the compile transform. The line numbers are for vs code,
-        //  so this makes them accurate for original source.
-        contents = contents.split("\n").map((line, index) => {
-            return line
-                .replaceAll(`(0, diskLogger_1.diskLog)(`, `(0, diskLogger_1.diskLog)(console.__context, { __LINE__: ${index} }, `)
-                .replaceAll(`  diskLog(`, ` diskLog(console.__context, { __LINE__: ${index} },`)
-                .replaceAll(`(0, diskLogger_1.logDisk)(`, `(0, diskLogger_1.logDisk)(console.__context, { __LINE__: ${index} }, `)
-                .replaceAll(`  logDisk(`, ` logDisk(console.__context, { __LINE__: ${index} },`)
-                ;
-        }).join("\n");
-        // IMPORTANT! No newlines, so the line numbers match up with the original source.
+        // IMPORTANT! No newlines, so we don't break sourcemaps
         return `var console = (${shimConsole.toString().replaceAll("\n", " ")})(); ${contents}`;
     });
 
-    // We want to call shimConsoleLogs early. Before we call it, our additional context isn't
-    //      screened out, which makes the terminal hard to read.
-    void import("./diskShimConsoleLogs").then(x => x.shimConsoleLogs());
-}
+    let diskShimConsoleLogs = require("./diskShimConsoleLogs") as typeof import("./diskShimConsoleLogs");
+    diskShimConsoleLogs.shimConsoleLogs();
+}

package/src/diagnostics/logs/lifeCycleAnalysis/spec.md

@@ -0,0 +1,153 @@
+Huge amount of data:
+    https://127-0-0-1.querysubtest.com:7007/?hot&enableLogs&page=login&caseInsensitive&showingmanagement&endTime=1757732880000&startTime=1746997620000&managementpage=LogViewer2
+Very small amount of data
+    https://127-0-0-1.querysubtest.com:7007/?hot&enableLogs&page=login&filter=%22431%22&showingmanagement&endTime=1755140880000&startTime=1754950020000&managementpage=LogViewer2
+
+
+
+4) Setup new hetnzer server
+
+yarn setup-machine 176.9.2.136
+
+5) Update all services, and move them to that machine
+5) Verify the hezner server can run the site well
+6) Take down our digital ocean server
+7) Destroy our digital ocean server
+
+
+6) Update URLParam to allow linking it to other parameters, resetting when they change.
+    - With a function, and have standard one beside URLParam (that uses page and tab)
+        - ALSO managementPageURL
+    - Reset filter in FastArchiveViewer
+        - First observe the overlap with it and the BookOverview
+        - If we are actually on the book overview page and we close the management page then that shouldn't reset it. We just want to reset it when you change pages. Because we want you to be able to hide and show the management page quickly if you want to double check something. Generally speaking though, you won't be on a page with filter and then going back and forth. And if you are, whatever. That's just the management page. We just want to avoid the overall confusion and annoyance of having lots of pre-filled values (And mostly the confusion of having filter prefilled all the time because it's always going to be set because everyone uses it and no one resets it at the moment. 
+
+
+
+
+5) IM error notifications - allow immediately knowing about production issues, for better testing
+    - Create a dedicated entry point which acts like a client of the HTTP server, using RecentErrorControllers.getRecentErrors
+        - Getting it working in a script will be interesting, but... in theory it should just work?
+    - Just for new errors
+    - Using backblaze to track when we send it, so we can heavily limit IMs and email
+    - IM api key tracked in secrets (like email api key)
+    - Once we get it working, deploy to production
+
+6) IM + email digests (daily / weekly?)
+    - a very short digest for the instant message which then links to a page on the site with a larger digest
+        - which has tabs, and each part in the instant message links to the correct tab
+    - Augments the error notifications entry point, having it also queue stuff up for digests. 
+    - Some notifications will never be immediate and will always be only in digests. 
+    - For now this will just be for:
+        - non-suppressed errors
+        - suppressed errors
+    - Eventually the goal of this is to add our tracking charts to this. There are some really useful metrics we can track. 
+        - unique visit IPs. 
+        - Percent bounces. 
+        - average visit length and also median visit length top 95" bottom 5" 
+        - average stories read average story percentages read 
+        - percent of first page story views by subscribers percent of fifth page story view by subscribers 
+        - Number of users visiting the fifth page.
+        - New subscriptions
+        - Subscription cancellations
+        - Total subscriptions
+
+
+
+5) Life cycle analyzer
+    - Uses FastArchiveViewer, but instead of showing a table, shows lifecycles (a derived concept)
+        - We save them in backblaze, with a bit of cache for loading them
+    - List of life cycles
+    - Life cycle
+        - Title
+        - Operation list (each supports | / &, but having multiple is even better)
+            - Match filter
+            - Group key extractions (optional, if not set it becomes a singleton)
+                - Just a field name
+                - CAN have multiple, which adds us as multiple life cycles
+                    - With each one being namespaced using the key, so we can tell them apart
+            - Global value setting (optional, if not set it has no global state impact)
+                - A list of set values
+                    - Each one is an expression which can use fields in the object, ex:
+                    - `alivePathValueServers.$threadId = true`
+            - Show AND SHOULDN'T include match filters!
+                - So when we should the count matched, we can show stats for these, which will often be should have "finished", and shouldn't have "error", so we can see completed, and errors
+                - Similar to error notifications, but... it's nice to also have this here, as we could miss the notification, or suppress it, but when we are looking at a life cycle it's relevant skyrockets.
+                - ALSO for start, so we can see cutoff starts!
+    OH! How do we handle cut off starts?
+        - Maybe... we have a "read preload" duration, and... we read that, BUT, only include life cycles which are also in our main selected time. So we don't cut anything off in our main time, but don't add new values which also get cut off!
+    - Same time/machine/thread selector as log viewer
+    - Allow filtering to specific life cycles
+        - After download, shows matches per life cycle
+        - Button to reset to all
+    - Download logs, and scan for selected life cycles
+        - Immediately on load, showing progress and throttling, so it's not too bad
+    - Result 
+        - List of life cycles, with count of each
+        - Table of individual life cycles?
+        - Can then filter within these life cycles by searching
+        - BUT, importantly, if any log is matched in a life cycle, the entire life cycle is matched
+        - AND, global settings for ALL life cycles are applied, not just filtered ones!
+    - Table of result life cycles
+        - Preview shows first matched line
+        - ALSO, shows duration of life cycle!
+        - And start date
+        - Expand to see pin that specific life cycle above
+            - Show list of logs in it, in another table, with searching on each of them
+            - Start date AND duration of each line!
+        - Can pin multiple life cycles (I guess might as well)
+    - Show list of global value expressions as well (limited, but with filtering to search them easily)
+        - Can expand a global value to see object (but again... limited?)
+    - Can select a specific global value path, to have it injected into
+        - a column for before and after each life cycle
+        - A column for after each line within a life cycle
+        - Can also select parent values, to show all paths under that (use ShowMore, to make them manageable...)
+
+6) Add life cycles for
+    - Node discovery life cycle (new node, check for alive, check for paths, etc)
+        - Discover nodes
+        - Check if trusted
+        - Sets authorities
+        - Sets nodes which exist
+        - Ready broadcast
+        - Resync is also part of life cycle
+        - Use different keys for threadId, and, triggeredNodeId, so we can track how a node discovers other nodes, vs how a node is discovered
+        - Mark as dead, dead count increases, remove node as dead
+            - MAYBE there should be 2 lifecycles, one for all, and one for just creation type stuff (no discovery)
+    - Trusted machine lifecycle
+        - Check if we need to add trust
+        - Add trust to archives
+        - Read trusted nodes from archives
+        => Set of trusted machines per threadId
+    - Functions evaluation in FunctionRunner
+        - Adding in http, evaluation, and maybe even forwarding the final PathValue writes to it?
+    - Known nodes
+        - Not really a life cycle, but does set global state
+    - Value syncing
+        - Crosses node boundaries, with the server having to include the client key, so we can track a proxy access => ask the server => validate permissions => send value => receive value => notify proxy
+    - Value creation
+        - Source, propagation, writing to archives
+    - Permissions (probably part of value syncing)
+    - SOCKET connections, open, stats, closed, error, etc.
+    - Archive management
+        - Queue (count)
+        - Archive (count + file)
+        - Joining, deleting, etc, etc
+
+6) Redeploy everything to the servers again, updating our machine apply script, all of our services, deploying our code, etc
+
+7) Use log viewer to find rejections / errors during startup, and then trace down the sync life cycle for the path write, and find where it is lagging out!
+    - Re-enable all of our services as well.
+    - Just logs MIGHT be able to do it, but... life cycles should make it a lot easier to correlate logs, which is something we need to do anyways to solve it...
+
+10) Verify old user/fast-log-cache machine folders are deleted
+
+
+SPECIAL UI links for certain errors in log view
+    - Probably dynamically created, based on contents of log
+    - LINKS to filters for all these special errors on a special page
+    - Special errors
+        - Rejection errors
+            - Link to life cycle filtering show the life cycles involving the path (we need to hardcode the life cycle ids for these, after we create the life cycles)
+        - Audit failures
+            - Same info as rejection errors, except for the path, and the failing lock path