querysub 0.356.0 → 0.358.0

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -1,16 +1,151 @@
-//todonext
-// Welp... no logs are showing up. That's a problem..
-// Ah, okay, we don't store logs locally. Because... we only log to backblaze? Alright, fair enough...
 /*
-0) LIFECYCLES!
-    - To debug auditting (to some degree)
-    - To debug why servers are getting sent values from the wrong authority
-        - This would be VERY interesting
-    - Rewrite the spec for how to do this
-        - One big thing is value association. As in, we want logs to be able to associate a session with a user. Even dynamically, so we might log somewhere saying this is the current user, and then we might log after that saying this is the session. And both of the logs might contain something saying this is the thread. And so we could say, okay, for the thread, the current user is this. And then when it says what the session is, we say, okay, well, the current user, which was defined earlier, has this session. 
-            - The biggest issue with this is caching, as now everything becomes one big bucket
-        - We want to be able to cache the parsing of the logs with the different lifecycle specifications. That way, we can view even just specific life cycles, and it can be very fast to load it, even if we have billions of logs, which take hours to parse. 
-        - I think we do want the lifecycle parsing to be run in the background. We already have a lot of background scripts (although we haven't been running them for a long time).
-            - Our log parsing is already pretty fast, which is good. So, then this will be even faster. 
-            - We will be need to be able to dynamically parse it though. And I guess we'll want to be able to parse it in the browser. And then that becomes a whole complicated thing to manage, but I think it's worth it. 
-*/
+todonext
+
+OKAY! CORE CONCEPTS
+    - Index needs to be fast searched, so it's uncompressed form has to be small
+        - Nesting compression is fine, as long as it's small enough that each search decompresses very little
+    - The enemy is storing every single position, we get around this by:
+        - In-exact matches, only storing block indexes.
+            - Requires full search to verify, but... in practice has low false hit rate
+        - Input key (unit) is only inexact matched.
+            - Input query almost always has more than 1 byte of uniqueness. This means if even part is only 50% accurate, we can still get very high hit rates (8 bytes means 255/256 hit rate!)
+
+
+todonext
+
+
+
+IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs should ever break. The code should be waiting until everything's fully flushed before it allows the shutdown handler to finish running. If we see any more errors, we need to investigate them. 
+
+
+
+4) Make it easy to enable or disable an entire server, regardless of what services are on it.
+- This is annoying, but it would be very useful. I think the apply loop can probably figure it out. We should probably ask the AI to do it. I'm sure it'll fuck it up, but it'll give us a start at least. And we can also just tell it, okay, find the actual code that we're going to need to change, but not change it, and just keep and maybe even have it put a comment there. And then we just keep doing that until we're absolutely certain that we found every place that we need to change to make this work. And then the AI might be able to help with the refactor. 
+3) Start the servers again, and deploy all of our code
+
+2) Create lot of remote server logs
+    - Via our refresh loop
+
+2.0) SUPPORT reading pending from multiple servers
+    - The main controller has to find a node on each other machine, and call it. Only one node per machine though, so it shouldn't be too difficult.
+        - We'll cache the last node per machine that we picked. 
+        - If the cache value doesn't exist, or if it doesn't work, if it throws an error when we try to verify it works, then we'll call a function to get the entry point on all of the nodes for that machine
+            - After we receive the first result, we'll wait at least a second so we get some more results, and then we'll prioritize the one that's the function endpoint, which will end in function.js. 
+
+2) Add a UI toggle to read public logs (only shows up on a non-public server though, as otherwise it wouldn't make sense)
+    - Basically, just changes the code we're reading from multiple servers to select public servers instead, and then, of course, skip ourselves. 
+
+3) Verify true remote reads are reasonable fast
+
+3) Deploy service for movelogs
+0) Run move logs in function runner, in development, just so we don't get too far behind
+
+
+
+
+
+1) Fix missing __NAME__
+     "Received PathValue for path" misses name?
+     - Maybe the missing name only happens when we rate limit?
+
+
+
+
+Rewrite error notification code
+    THINK about how to do a somewhat generic logs => derived thing, as... we will need the exact same thing for life cycles!
+        - Maybe make it generic immediately? Having it abstracted it kind of nice for development anyways...
+    - New service that manages it, instead of doing it on demand
+        - It asks everyone to send it error logs
+    - Stores cached error logs => { unsuppressed logs, suppressionSummary }
+        - Unsuppressed logs only for suppression which is old enough. 
+        - Only when logs are old enough. 
+    - Stores in memory with all suppressionSummaries
+    AND, the only watcher will be the watcher service. You can't get recent errors, or any errors, without going through one of those
+    NO dev errors. They are usually red-herrings anyways... and we should just be using public servers for regular usage
+        - And we still have dev logs we can check if to see if an error happened locally
+
+
+
+Remove all old LogViewer/FastArchiveAppendable code
+
+
+0) Add LZ4 compression to socket-function by default
+    - Allow setting "compress" to "none" or "lz4" or "zip" or "zip0" or "zip3", etc, for levels.
+        - default is "lz4"
+    - REQUIRES feature checking the remote, to make sure it is new enough to accept this.
+        - A generic thing which gets the version is probably fine.
+    - LZ4 compression is fast enough that this should cause basically no overhead, and in many cases greatly reduce the bandwidth (which will increase the speed).
+    - We're gonna have to investigate how we're sending buffers anyway. I think this should be easy, but we 
+0.1) Verify the size distance with some local testing
+    - ALSO, verify the processing overhead is acceptable.
+1) Deploy, which SHOULD be backwards compatible with everything?
+
+*/
+
+
+
+
+
+// todonext;
+// Hmm... so... should we index it, so we can search it? HMM... I think we might want to?
+//      - Although the searches might get a bit complicated...
+//      - I think we need to limit lifecycle lengths? Hmm... as otherwise we need a lot in memory at once?
+//          - We could always do it based on size, so if we have too many logs the max time length is less
+// 7) Decide how we're going to store it, and setup the controller
+// 8) Get the AI to set up some basic UI to manage it.
+//    - For now, we'll run the phases one after the other. Controlled by the caller. Caching is going to come much later.
+//    - The first phase I think we can run when we receive the logs directly. This will make progress nice. And then the second phase will run automatically in the finish function. We'll have to add additional progress for that, which shouldn't be too difficult.
+//      - Having the finish function fork off into an async is probably fine, although we will have to expose the cancellation promise to the finish function?
+//          - Or do we even need cancellation? Are we ever going to even keep this code? So if we have a server precaching the values, we still need the pending values to be parsed somewhere. Hmm... That's annoying. Are we just not going to use fast archives for this?
+//          - MAYBE We can have the Fast Archives support external caching, so it'll give us the name of the file, and we'll tell it if it's cached or not. Actually, it can give us the name of all the files, and we'll tell it if we have a cached result for it. And if we do, we'll load the cache result, and it won't do any of the parsing of that file. 
+// 9) Stress test it, by automating something to load a page repeatedly (which should cause a lot of path logs). We should try at least 10K page refreshes, and nothing should break...
+//      - If my estimates are correct, and a page load of a large book results in ~5K logs, that will use ~2.5MB of uncompressed space, (~100KB compressed). Which... is acceptable. 10K page loads is 1GB, so a million would cost 1 USD/month to store. AND that's if we keep storing all audits in the logs, which we don't really have to do...
+//      - Processing it, on the other hand, will be kind of slow. Looks like we decompress about ten MB a second. But... there's not much we can do about that... unless... we searched on the compressed data directly? Hmm... or at least optimized based on it.
+// 10) Add log file overview page, so we can keep an eye on the size of logs
+// 10.1) Start storing the log count in the log file names
+//      - WHICH, requires breaking all of our old logs, but... that's not too big of deal
+//      - ONLY for the non-pending logs
+// 11) Pull requests our lz4-wasm code. It's purely AI generated, but it might be of some interest.
+//  - As in, supporting streaming and decoding from a stream
+//  - REMEMBER to delete all of our test files and test code, and also really clean up the pull request before we make it. 
+/*
+Two phases, and second phase has limitted, as some of our life cycles might explode quadratically.
+*/
+
+
+// logs => life cycle related => life cycle group by key => life cycle list, with each one being expandable
+
+// Searching in previous state
+// Using variables from logs?
+// Using variables from lifecycle? 
+// The FORM we match will be pretty much always the same (not depending on variables). If we absolutely need to we can do some late filtering in the second phase, but I don't think we'll need to. However the life cycle it becomes a part of, that will depend on the state.
+// OH! The start lifecycle entry will create a lifecycle, THEN we match. The later matches might not be as restrictive as the start life cycle. SOO...
+//   start = [path, clientId]
+//   watch = [path]
+// AND SO, This is what we can do to kind of expand our paths. So really, it's more of a debug thing, where the start is something that provides a key for what we're debugging. And then after it, we'll have different lines which can match whichever data we want.
+//      - The only restriction is the other lines have to happen after the start and before the end.
+//      - The end will be the first end matched as well
+//          - AND Optionally, you can make it so that the start ends the previous one
+//      - And you can also just not provide an end. There's no reason the life cycle needs to end.
+//      - I think technically if you add a start and it keeps triggering and you don't tell it that it should end the previous one, then there'll be a lot of overlapping and some of the life cycles will get quite large. But we can handle that by throttling and lazy evaluation, so it shouldn't actually be an issue. 
+// todonext;
+// // Okay, so it's looking like the first thing we're going to do is match the start, and then once we find the start, if we want to expand that life cycle, then we do a second phase search.
+// todonext;
+// // HMM... So actually what kind of variables do we need? I guess if we want to map between IDs. But then can't we just Match some line That gives us the mapping. And then, presumably, that'll be early on in the life cycle....
+// todonext;
+// // OH! Okay, so we want to allow some logs in the life cycle to match before the start? But that's all. They just match before the start. And I think we have to define how long they can match before the start. Obviously, we can't search forever. And then they're just normal lines after that. They can set life cycle variables, and we can access life cycle variables. 
+// todonext;
+// // Yes, so... The variables are going to be a lot simple. It's just the lifecycle will have variables associated with it, which lines in the life cycle can set.
+// todonext;
+// // Let's not have any regex parsing either. Lifecycles entries can just say, I care about these variables and we expect them to be a part of the log object, and then we add them to the lifecycle.
+// //      - Which is really more for UI organization, as we can say these are important. And prevents the bug of some random log having to have happen to having a thread ID, but we have a thread ID and we mean the client thread ID, etc., etc. We just make it explicit per line, and then we don't run into that bug ever. 
+
+// TODO: Eventually, if we need it, we can have entries find other life cycles, either matching many, the latest, etc, and take variables from them PURELY FOR DISPLAY PURPOSES. But right at this moment, that's not needed.
+
+type LifeCycle = {
+
+};
+
+type LifeCycleEntry = {
+
+};

package/src/diagnostics/logs/lifeCycleAnalysis/test.wat

@@ -0,0 +1,106 @@
+(module
+  ;; Import memory from JavaScript (with memory64 support for >4GB).
+  ;; Callers use allocateBuffer(exports, size) from supportWat.ts to
+  ;; request scratch regions; memory will be grown automatically as needed.
+  (import "env" "memory" (memory i64 1))
+
+  ;; Export the heap base so the allocator knows where static data ends.
+  ;; With no static data section here, the heap starts at byte 0.
+  (global (export "__heap_base") i32 (i32.const 0))
+
+  ;; Add function: adds two i32 integers
+  (func $add (param $a i32) (param $b i32) (result i32)
+    local.get $a
+    local.get $b
+    i32.add
+  )
+  (export "add" (func $add))
+
+  ;; Multiply function: multiplies two i32 integers
+  (func $multiply (param $a i32) (param $b i32) (result i32)
+    local.get $a
+    local.get $b
+    i32.mul
+  )
+  (export "multiply" (func $multiply))
+
+  ;; stddev(offset: i32, count: i32) -> f64
+  ;;
+  ;; Computes the population standard deviation of `count` f64 values
+  ;; stored contiguously in memory starting at byte `offset`.
+  ;; Each value occupies 8 bytes (little-endian IEEE 754 double).
+  ;; Returns 0.0 when count == 0.
+  (func $stddev (param $offset i32) (param $count i32) (result f64)
+    (local $i i32)
+    (local $sum f64)
+    (local $mean f64)
+    (local $diff f64)
+    (local $variance_sum f64)
+
+    ;; Guard: return 0 if count == 0
+    (if (i32.eqz (local.get $count))
+      (then (return (f64.const 0)))
+    )
+
+    ;; Pass 1: compute sum
+    (local.set $i (i32.const 0))
+    (local.set $sum (f64.const 0))
+    (block $break1
+      (loop $loop1
+        (br_if $break1 (i32.ge_u (local.get $i) (local.get $count)))
+        (local.set $sum
+          (f64.add
+            (local.get $sum)
+            (f64.load
+              (i32.add
+                (local.get $offset)
+                (i32.mul (local.get $i) (i32.const 8))
+              )
+            )
+          )
+        )
+        (local.set $i (i32.add (local.get $i) (i32.const 1)))
+        (br $loop1)
+      )
+    )
+
+    ;; mean = sum / count
+    (local.set $mean
+      (f64.div (local.get $sum) (f64.convert_i32_u (local.get $count)))
+    )
+
+    ;; Pass 2: compute sum of squared deviations
+    (local.set $i (i32.const 0))
+    (local.set $variance_sum (f64.const 0))
+    (block $break2
+      (loop $loop2
+        (br_if $break2 (i32.ge_u (local.get $i) (local.get $count)))
+        (local.set $diff
+          (f64.sub
+            (f64.load
+              (i32.add
+                (local.get $offset)
+                (i32.mul (local.get $i) (i32.const 8))
+              )
+            )
+            (local.get $mean)
+          )
+        )
+        (local.set $variance_sum
+          (f64.add
+            (local.get $variance_sum)
+            (f64.mul (local.get $diff) (local.get $diff))
+          )
+        )
+        (local.set $i (i32.add (local.get $i) (i32.const 1)))
+        (br $loop2)
+      )
+    )
+
+    ;; return sqrt(variance_sum / count)
+    (f64.sqrt
+      (f64.div (local.get $variance_sum) (f64.convert_i32_u (local.get $count)))
+    )
+  )
+  (export "stddev" (func $stddev))
+)

package/src/diagnostics/logs/lifeCycleAnalysis/test.wat.d.ts

@@ -0,0 +1,2 @@
+export const add: (a: number, b: number) => number;
+export const multiply: (a: number, b: number) => number;

package/src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts

@@ -0,0 +1,5 @@
+import "../../../forceProduction";
+import "../../../inject";
+import { chdir } from "process";
+chdir("D:/repos/qs-cyoa/");
+import "./test";

package/src/diagnostics/logs/logViewerExtractField.ts

@@ -2,7 +2,7 @@ import { ScanFnc } from "./FastArchiveViewer";
 
 export function createLogViewerExtractField(
     fieldName: string,
-    onFieldValue: (value: string) => void,
+    onFieldValue: (value: string, unpackedSize: number) => void,
 ): ScanFnc {
     let fieldNameBuffer = Buffer.from(`${JSON.stringify(fieldName)}:"`);
     let quoteChar = Buffer.from(`"`)[0];
@@ -19,7 +19,7 @@ export function createLogViewerExtractField(
                         let start = i + j + 1;
                         for (let k = start; k < posEnd; k++) {
                             if (data[k] === quoteChar) {
-                                onFieldValue(data.slice(start, k).toString());
+                                onFieldValue(data.slice(start, k).toString(), posEnd - start);
                                 break;
                             }
                             if (data[k] === escapeChar) {
@@ -31,7 +31,6 @@ export function createLogViewerExtractField(
                 }
             }
         }
-        onFieldValue("");
         return false;
     };
 }

package/src/diagnostics/managementPages.tsx

@@ -91,6 +91,16 @@ export async function registerManagementPages2(config: {
         componentName: "LogViewer2",
         getModule: () => import("./logs/LogViewer2"),
     });
+    inputPages.push({
+        title: "Logs3",
+        componentName: "LogViewer3",
+        getModule: () => import("./logs/IndexedLogs/LogViewer3"),
+    });
+    inputPages.push({
+        title: "Life Cycles",
+        componentName: "LifeCyclePages",
+        getModule: () => import("./logs/lifeCycleAnalysis/LifeCyclePages"),
+    });
     inputPages.push({
         title: "Error Digests",
         componentName: "ErrorDigestPage",

package/src/diagnostics/trackResources.ts

@@ -86,7 +86,7 @@ function logResourcesNow() {
 }
 
 registerMeasureInfo(() => {
-    return `${formatNumber(getUsedHeapSize())}B+${formatNumber(getBufferUsage())}B/${formatNumber(getHeapSize())}B`;
+    return `MEM ${formatNumber(getUsedHeapSize())}B+${formatNumber(getBufferUsage())}B/${formatNumber(getHeapSize())}B `;
 });
 
 registerPeriodic(logResourcesNow);

package/src/functional/limitProcessing.ts

@@ -0,0 +1,39 @@
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { MaybePromise } from "socket-function/src/types";
+
+export class LimitGroup {
+    /** We have processing sections. In each section, if we exceed the current maximum wait, then any new processing will be told that it needs to wait. And then at the end of the section we wait wait time. This can be zero, which is fine, and we'll should wait enough time for networking, etc to run. */
+    constructor(public config: {
+        maxTimePerBeforeWait: number;
+        waitTime: number;
+    }) { }
+
+    private sectionStartTime: number | undefined = undefined;
+    private afterSectionResolvers: (() => void)[] = [];
+
+    @measureFnc
+    public wait(): MaybePromise<void> {
+        if (this.sectionStartTime === undefined) {
+            this.sectionStartTime = Date.now();
+            setTimeout(async () => {
+                await new Promise(resolve => setTimeout(resolve, this.config.waitTime));
+
+                this.sectionStartTime = undefined;
+                const resolvers = this.afterSectionResolvers;
+                this.afterSectionResolvers = [];
+                for (const resolve of resolvers) {
+                    resolve();
+                }
+            }, 0);
+        }
+
+        const elapsed = Date.now() - this.sectionStartTime;
+        if (elapsed >= this.config.maxTimePerBeforeWait) {
+            return new Promise<void>((resolve) => {
+                this.afterSectionResolvers.push(resolve);
+            });
+        }
+
+        return undefined;
+    }
+}

package/src/misc/lz4_wasm_nodejs.d.ts

@@ -0,0 +1,34 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/**
+ * Streaming LZ4 compressor (frame format with linked blocks).
+ * Concatenate all output chunks to form a complete LZ4 frame.
+ */
+export class Lz4StreamCompressor {
+    free(): void;
+    [Symbol.dispose](): void;
+    compress(input: Uint8Array): Uint8Array;
+    constructor();
+}
+
+/**
+ * One-shot block compression with size prepended.
+ */
+export function compress(input: Uint8Array): Uint8Array;
+
+/**
+ * One-shot block decompression with size prepended.
+ */
+export function decompress(input: Uint8Array): Uint8Array;
+
+/**
+ * Decompress an LZ4 stream (frame format).
+ * Auto-injects end marker if missing. On error, returns partial data and sets a warning.
+ */
+export function decompress_stream(input: Uint8Array): Uint8Array;
+
+/**
+ * Get and clear the last warning from decompression.
+ */
+export function get_last_warning(): string | undefined;