querysub 0.394.0 → 0.395.0

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -1,94 +1,129 @@
-/*
-todonext
-
-Now nothing runs. I guess more that our disk is not working. Task manager won't open either. Fuck, we need to restart our computer. 
-
--1) Deploy everything again!
-
--2) Fix synchronization speed from remote to local.
-    - I think we just have a lot of data, which our code will detect and log now.
-        - And we'll see how much data we are even sending
-
-Write Monthly Summary
-
-*/
-
-
-
-//todonext
-// Reorganize our lifecycles ideas
-//  - Scans over past logs, and only finished, never pending
-//         - Probably... realtime creating cache, BUT, if we add new lifecycles, we then only add them on demand. So when we are in the debugging mindset, and adding and changing them, it's fast (only as needed), but when we come back later when we aren't in the middle of debugging, it'll be super fast so we can fix the bug quickly and move on.
-//      - OR... we can do pending dynamically, but we don't cache it? Hmm... then what's the point? Hmm...
-//      - MAYBE, no caching, but we restrict the time by a lot? HMM...
-//      - I mean, we have to implement it without caching anyways to start, so we can just do that, and then cache it later if we see the need...
-//          - WELL, we need SOME kind of caching... Maybe... we DO use a strict time range, and then... if we've scanned a pending file in the past, we can cache that, because we know we've got all of it's values. Hmm...
-//  - Use lifecycles to debug rejections. To a point, until it is likely we are out of sync, then we should write the sync verification code.
-//      - Maybe it isn't necessarily rejections, but definitely if we're running stuff locally and remotely it breaks. Could it be the case that we can't run the path value server both locally and remotely? And if so, we really should prevent this in some way. 
-//      - Annoyingly enough, we're actually going to have to add a case where we can scan the local logs as well as the remote logs to debug running it locally...
-//          - ALTHOUGH, It does fail just if we have the local running server, But we're on the remote site. So we might be able to just debug it from that life cycle, from the remote life cycles. 
-
-
-// todonext;
-// Hmm... so... should we index it, so we can search it? HMM... I think we might want to?
-//      - Although the searches might get a bit complicated...
-//      - I think we need to limit lifecycle lengths? Hmm... as otherwise we need a lot in memory at once?
-//          - We could always do it based on size, so if we have too many logs the max time length is less
-// 7) Decide how we're going to store it, and setup the controller
-// 8) Get the AI to set up some basic UI to manage it.
-//    - For now, we'll run the phases one after the other. Controlled by the caller. Caching is going to come much later.
-//    - The first phase I think we can run when we receive the logs directly. This will make progress nice. And then the second phase will run automatically in the finish function. We'll have to add additional progress for that, which shouldn't be too difficult.
-//      - Having the finish function fork off into an async is probably fine, although we will have to expose the cancellation promise to the finish function?
-//          - Or do we even need cancellation? Are we ever going to even keep this code? So if we have a server precaching the values, we still need the pending values to be parsed somewhere. Hmm... That's annoying. Are we just not going to use fast archives for this?
-//          - MAYBE We can have the Fast Archives support external caching, so it'll give us the name of the file, and we'll tell it if it's cached or not. Actually, it can give us the name of all the files, and we'll tell it if we have a cached result for it. And if we do, we'll load the cache result, and it won't do any of the parsing of that file. 
-// 9) Stress test it, by automating something to load a page repeatedly (which should cause a lot of path logs). We should try at least 10K page refreshes, and nothing should break...
-//      - If my estimates are correct, and a page load of a large book results in ~5K logs, that will use ~2.5MB of uncompressed space, (~100KB compressed). Which... is acceptable. 10K page loads is 1GB, so a million would cost 1 USD/month to store. AND that's if we keep storing all audits in the logs, which we don't really have to do...
-//      - Processing it, on the other hand, will be kind of slow. Looks like we decompress about ten MB a second. But... there's not much we can do about that... unless... we searched on the compressed data directly? Hmm... or at least optimized based on it.
-// 10) Add log file overview page, so we can keep an eye on the size of logs
-// 10.1) Start storing the log count in the log file names
-//      - WHICH, requires breaking all of our old logs, but... that's not too big of deal
-//      - ONLY for the non-pending logs
-// 11) Pull requests our lz4-wasm code. It's purely AI generated, but it might be of some interest.
-//  - As in, supporting streaming and decoding from a stream
-//  - REMEMBER to delete all of our test files and test code, and also really clean up the pull request before we make it. 
-/*
-Two phases, and second phase has limitted, as some of our life cycles might explode quadratically.
-*/
-
-
-// logs => life cycle related => life cycle group by key => life cycle list, with each one being expandable
-
-// Searching in previous state
-// Using variables from logs?
-// Using variables from lifecycle? 
-// The FORM we match will be pretty much always the same (not depending on variables). If we absolutely need to we can do some late filtering in the second phase, but I don't think we'll need to. However the life cycle it becomes a part of, that will depend on the state.
-// OH! The start lifecycle entry will create a lifecycle, THEN we match. The later matches might not be as restrictive as the start life cycle. SOO...
-//   start = [path, clientId]
-//   watch = [path]
-// AND SO, This is what we can do to kind of expand our paths. So really, it's more of a debug thing, where the start is something that provides a key for what we're debugging. And then after it, we'll have different lines which can match whichever data we want.
-//      - The only restriction is the other lines have to happen after the start and before the end.
-//      - The end will be the first end matched as well
-//          - AND Optionally, you can make it so that the start ends the previous one
-//      - And you can also just not provide an end. There's no reason the life cycle needs to end.
-//      - I think technically if you add a start and it keeps triggering and you don't tell it that it should end the previous one, then there'll be a lot of overlapping and some of the life cycles will get quite large. But we can handle that by throttling and lazy evaluation, so it shouldn't actually be an issue. 
-// todonext;
-// // Okay, so it's looking like the first thing we're going to do is match the start, and then once we find the start, if we want to expand that life cycle, then we do a second phase search.
-// todonext;
-// // HMM... So actually what kind of variables do we need? I guess if we want to map between IDs. But then can't we just Match some line That gives us the mapping. And then, presumably, that'll be early on in the life cycle....
-// todonext;
-// // OH! Okay, so we want to allow some logs in the life cycle to match before the start? But that's all. They just match before the start. And I think we have to define how long they can match before the start. Obviously, we can't search forever. And then they're just normal lines after that. They can set life cycle variables, and we can access life cycle variables. 
-// todonext;
-// // Yes, so... The variables are going to be a lot simple. It's just the lifecycle will have variables associated with it, which lines in the life cycle can set.
-// todonext;
-// // Let's not have any regex parsing either. Lifecycles entries can just say, I care about these variables and we expect them to be a part of the log object, and then we add them to the lifecycle.
-// //      - Which is really more for UI organization, as we can say these are important. And prevents the bug of some random log having to have happen to having a thread ID, but we have a thread ID and we mean the client thread ID, etc., etc. We just make it explicit per line, and then we don't run into that bug ever. 
-
-// TODO: Eventually, if we need it, we can have entries find other life cycles, either matching many, the latest, etc, and take variables from them PURELY FOR DISPLAY PURPOSES. But right at this moment, that's not needed.
-
-type LifeCycle = {
-
+import { lazy } from "socket-function/src/caching";
+import { nestArchives } from "../../../-a-archives/archives";
+import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
+import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
+import { getDomain } from "../../../config";
+import { runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { timeInMinute } from "socket-function/src/misc";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { assertIsManagementUser } from "../../managementPages";
+import { getSyncedController } from "../../../library-components/SyncedController";
+
+export type LifeCycle = {
+    id: string;
+    title: string;
+    entries: LifeCycleEntry[];
 };
 
-type LifeCycleEntry = {
+export type LifeCycleEntry = {
+    description?: string;
+
+    sourceType: "log" | "error" | "info" | "warning";
+
+    matchPattern: string;
+
+    // There could be multiple starts and multiple ends. Ideally the UI should move anything set to the start or the end to the start and end of the entries list.
+    isStart?: boolean;
+    isEnd?: boolean;
 
+    groupByKeys: {
+        ourKey: string;
+        // Defaults to the same key, which is usually the case
+        startKey?: string;
+    }[];
+
+    // All of the special variables, which are marked. The title defaults to the key, but can be overriden for the purposes of adding notes.
+    variables: {
+        [key: string]: {
+            title?: string;
+        };
+    };
 };
+export function getVariables(entry: LifeCycleEntry): { key: string, title?: string }[] {
+    let variables: { key: string, title?: string }[] = [];
+    for (let groupByKey of entry.groupByKeys) {
+        variables.push({ key: groupByKey.ourKey, title: "Group By Key" });
+    }
+    for (let [key, value] of Object.entries(entry.variables)) {
+        variables.push({ key, title: value.title });
+    }
+    return variables;
+}
+
+const lifeCycles = archiveJSONT<LifeCycle>(() => nestArchives("logs/life-cycles/", getArchivesBackblaze(getDomain())));
+let lifeCyclesCache: LifeCycle[] = [];
+
+let ensureWatching = lazy(async () => {
+    await runInfinitePollCallAtStart(timeInMinute * 5, updateNow);
+});
+
+async function updateNow() {
+    lifeCyclesCache = await lifeCycles.values();
+}
+
+async function getLifeCyclesEntries(): Promise<LifeCycle[]> {
+    await ensureWatching();
+    return lifeCyclesCache;
+}
+
+class LifeCyclesService {
+    public async getLifeCycles() {
+        return await getLifeCyclesEntries();
+    }
+
+    public async setLifeCycle(lifeCycle: LifeCycle) {
+        let promise = lifeCycles.set(lifeCycle.id, lifeCycle);
+        let prevIndex = lifeCyclesCache.findIndex(e => e.id === lifeCycle.id);
+        if (prevIndex !== -1) {
+            lifeCyclesCache[prevIndex] = lifeCycle;
+        } else {
+            lifeCyclesCache.push(lifeCycle);
+        }
+        void promise.finally(() => {
+            void updateNow();
+        });
+    }
+
+    public async deleteLifeCycle(id: string) {
+        let promise = lifeCycles.delete(id);
+        let prevIndex = lifeCyclesCache.findIndex(e => e.id === id);
+        if (prevIndex !== -1) {
+            lifeCyclesCache.splice(prevIndex, 1);
+        }
+        void promise.finally(() => {
+            void updateNow();
+        });
+    }
+}
+
+const LifeCyclesServiceBase = SocketFunction.register(
+    "LifeCyclesService-019cb585-0381-730b-9a25-a2c647e503ee",
+    new LifeCyclesService(),
+    () => ({
+        getLifeCycles: {},
+        setLifeCycle: {},
+        deleteLifeCycle: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+);
+
+export const LifeCyclesController = getSyncedController(LifeCyclesServiceBase, {
+    reads: {
+        getLifeCycles: ["lifeCycles"],
+    },
+    writes: {
+        setLifeCycle: ["lifeCycles"],
+        deleteLifeCycle: ["lifeCycles"],
+    },
+});
+
+
+
+
+
+
+
+
+

package/src/diagnostics/managementPages.tsx

@@ -27,6 +27,7 @@ import { addComponentUI } from "../5-diagnostics/qreactDebug";
 import { addComponentButton } from "../5-diagnostics/qreactDebug";
 import { closeAllModals } from "../5-diagnostics/Modal";
 import { delay } from "socket-function/src/batching";
+import { currentViewParam, selectedServiceIdParam } from "../deployManager/urlParams";
 
 export const managementPageURL = new URLParam("managementpage", "");
 export const showingManagementURL = new URLParam("showingmanagement", false);
@@ -73,6 +74,7 @@ export async function registerManagementPages2(config: {
     let inputPages: typeof config.pages = [];
 
 
+    // IMPORTANT! The controller name has to be the name of the controller in the file. No human would just randomly put some junk there, but the AI keeps putting random junk there and being surprised when they get an error because that controller doesn't actually exist in that file.
     inputPages.push({
         title: "Machines",
         componentName: "MachinesPage",
@@ -89,6 +91,12 @@ export async function registerManagementPages2(config: {
         componentName: "LogViewer3",
         getModule: () => import("./logs/IndexedLogs/LogViewer3"),
     });
+    inputPages.push({
+        title: "Life Cycles",
+        componentName: "LifeCyclePage",
+        controllerName: "",
+        getModule: () => import("./logs/lifeCycleAnalysis/LifeCyclePage"),
+    });
     inputPages.push({
         title: "Error Notifications",
         componentName: "ErrorNotificationPage",
@@ -330,8 +338,16 @@ class ManagementRoot extends qreact.Component {
                             }
                         `}
                     </style>
-                    <div class={css.fillWidth.hbox(30, 10).wrap.hsl(245, 25, 80).pad2(10).pointerEvents("all")}>
+                    <div class={css.fillWidth.hbox(20, 4).wrap.hsl(245, 25, 80).pad2(20, 4).pointerEvents("all")}>
                         <ErrorWarning />
+                        <ATag values={[
+                            managementPageURL.getOverride("MachinesPage"),
+                            currentViewParam.getOverride("services"),
+                        ]}>Deploy Machines</ATag>
+                        <ATag values={[
+                            managementPageURL.getOverride("MachinesPage"),
+                            currentViewParam.getOverride("deploy"),
+                        ]}>Deploy Application</ATag>
                         {pages.map(page =>
                             <ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>
                         )}

package/src/functional/{limitProcessing.ts → throttleProcessing.ts}

@@ -1,7 +1,7 @@
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { MaybePromise } from "socket-function/src/types";
 
-export class LimitGroup {
+export class ThrottleGroup {
     /** We have processing sections. In each section, if we exceed the current maximum wait, then any new processing will be told that it needs to wait. And then at the end of the section we wait wait time. This can be zero, which is fine, and we'll should wait enough time for networking, etc to run. */
     constructor(public config: {
         maxTimePerBeforeWait: number;

package/src/library-components/StartEllipsis.tsx

@@ -0,0 +1,13 @@
+import { qreact } from "../4-dom/qreact";
+import { css } from "typesafecss";
+
+export class StartEllipsis extends qreact.Component<{
+    maxWidth: number;
+}> {
+    render() {
+        return <div className={css.relative.maxWidth(this.props.maxWidth).overflowHidden.whiteSpace("nowrap").direction("rtl")}>
+            <span className={css.relative.direction("ltr")}>{this.props.children}</span>
+            <span className={css.absolute.left(`calc((100% - ${this.props.maxWidth}px) * -10000)`).top(0).height("100%").width(30).background("linear-gradient(to right, white, transparent)").pointerEvents("none")}></span>
+        </div>;
+    }
+}

package/src/misc.ts

@@ -190,4 +190,8 @@ export function streamToIteratable<T>(reader: {
 const AsyncFunction = (async () => { }).constructor;
 export function isAsyncFunction(func: unknown): boolean {
     return func instanceof AsyncFunction;
+}
+
+export function maybeUndefined<T>(value: T): T | undefined {
+    return value;
 }

package/src/diagnostics/logs/lifeCycleAnalysis/test.wat

@@ -1,106 +0,0 @@
-(module
-  ;; Import memory from JavaScript (with memory64 support for >4GB).
-  ;; Callers use allocateBuffer(exports, size) from supportWat.ts to
-  ;; request scratch regions; memory will be grown automatically as needed.
-  (import "env" "memory" (memory i64 1))
-
-  ;; Export the heap base so the allocator knows where static data ends.
-  ;; With no static data section here, the heap starts at byte 0.
-  (global (export "__heap_base") i32 (i32.const 0))
-
-  ;; Add function: adds two i32 integers
-  (func $add (param $a i32) (param $b i32) (result i32)
-    local.get $a
-    local.get $b
-    i32.add
-  )
-  (export "add" (func $add))
-
-  ;; Multiply function: multiplies two i32 integers
-  (func $multiply (param $a i32) (param $b i32) (result i32)
-    local.get $a
-    local.get $b
-    i32.mul
-  )
-  (export "multiply" (func $multiply))
-
-  ;; stddev(offset: i32, count: i32) -> f64
-  ;;
-  ;; Computes the population standard deviation of `count` f64 values
-  ;; stored contiguously in memory starting at byte `offset`.
-  ;; Each value occupies 8 bytes (little-endian IEEE 754 double).
-  ;; Returns 0.0 when count == 0.
-  (func $stddev (param $offset i32) (param $count i32) (result f64)
-    (local $i i32)
-    (local $sum f64)
-    (local $mean f64)
-    (local $diff f64)
-    (local $variance_sum f64)
-
-    ;; Guard: return 0 if count == 0
-    (if (i32.eqz (local.get $count))
-      (then (return (f64.const 0)))
-    )
-
-    ;; Pass 1: compute sum
-    (local.set $i (i32.const 0))
-    (local.set $sum (f64.const 0))
-    (block $break1
-      (loop $loop1
-        (br_if $break1 (i32.ge_u (local.get $i) (local.get $count)))
-        (local.set $sum
-          (f64.add
-            (local.get $sum)
-            (f64.load
-              (i32.add
-                (local.get $offset)
-                (i32.mul (local.get $i) (i32.const 8))
-              )
-            )
-          )
-        )
-        (local.set $i (i32.add (local.get $i) (i32.const 1)))
-        (br $loop1)
-      )
-    )
-
-    ;; mean = sum / count
-    (local.set $mean
-      (f64.div (local.get $sum) (f64.convert_i32_u (local.get $count)))
-    )
-
-    ;; Pass 2: compute sum of squared deviations
-    (local.set $i (i32.const 0))
-    (local.set $variance_sum (f64.const 0))
-    (block $break2
-      (loop $loop2
-        (br_if $break2 (i32.ge_u (local.get $i) (local.get $count)))
-        (local.set $diff
-          (f64.sub
-            (f64.load
-              (i32.add
-                (local.get $offset)
-                (i32.mul (local.get $i) (i32.const 8))
-              )
-            )
-            (local.get $mean)
-          )
-        )
-        (local.set $variance_sum
-          (f64.add
-            (local.get $variance_sum)
-            (f64.mul (local.get $diff) (local.get $diff))
-          )
-        )
-        (local.set $i (i32.add (local.get $i) (i32.const 1)))
-        (br $loop2)
-      )
-    )
-
-    ;; return sqrt(variance_sum / count)
-    (f64.sqrt
-      (f64.div (local.get $variance_sum) (f64.convert_i32_u (local.get $count)))
-    )
-  )
-  (export "stddev" (func $stddev))
-)

package/src/diagnostics/logs/lifeCycleAnalysis/test.wat.d.ts

@@ -1,2 +0,0 @@
-export const add: (a: number, b: number) => number;
-export const multiply: (a: number, b: number) => number;

package/src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts

@@ -1,5 +0,0 @@
-import "../../../forceProduction";
-import "../../../inject";
-import { chdir } from "process";
-chdir("D:/repos/qs-cyoa/");
-import "./test";