querysub 0.356.0 → 0.358.0

package/src/zip.ts

@@ -8,94 +8,8 @@ import { unzipThreaded } from "./zipThreaded";
 import debugbreak from "debugbreak";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { runInSerial } from "socket-function/src/batching";
+import { MaybePromise } from "socket-function/src/types";
+import { Zip } from "socket-function/src/Zip";
 setFlag(require, "pako", "allowclient", true);
 
-export class Zip {
-    @measureFnc
-    public static async gzip(buffer: Buffer, level?: number): Promise<Buffer> {
-        if (isNode()) {
-            return new Promise((resolve, reject) => {
-                zlib.gzip(buffer, { level }, (err: any, result: Buffer) => {
-                    if (err) reject(err);
-                    else resolve(result);
-                });
-            });
-        } else {
-            // @ts-ignore
-            return await doStream(new CompressionStream("gzip"), buffer);
-        }
-    }
-    @measureFnc
-    public static async gunzip(buffer: Buffer): Promise<Buffer> {
-        return Zip.gunzipUntracked(buffer);
-    }
-    // A base function, so we can avoid instrumentation for batch calls
-    public static async gunzipUntracked(buffer: Buffer): Promise<Buffer> {
-        if (isNode()) {
-            let test = await unzipThreaded(buffer);
-            // let test2 = await new Promise<Buffer>((resolve, reject) => {
-            //     zlib.gunzip(buffer, (err: any, result: Buffer) => {
-            //         if (err) reject(err);
-            //         else resolve(result);
-            //     });
-            // });
-            // if (test.length !== test2.length) {
-            //     debugbreak(2);
-            //     debugger;
-            // }
-            // for (let i = 0; i < test.length; i++) {
-            //     if (test[i] !== test2[i]) {
-            //         debugbreak(2);
-            //         debugger;
-            //     }
-            // }
-
-            return test;
-        } else {
-            // NOTE: pako seems to be faster, at least clientside.
-            // TIMING: 700ms vs 1200ms
-            //  - This might just be faster for small files.
-            return Buffer.from(pako.inflate(buffer));
-            // @ts-ignore
-            // return await doStream(new DecompressionStream("gzip"), buffer);
-        }
-    }
-
-    @measureFnc
-    public static gunzipSync(buffer: Buffer): Buffer {
-        if (isNode()) {
-            return Buffer.from(zlib.gunzipSync(buffer));
-        } else {
-            return Buffer.from(pako.inflate(buffer));
-        }
-    }
-
-    @measureFnc
-    public static async gunzipBatch(buffers: Buffer[]): Promise<Buffer[]> {
-        let time = Date.now();
-        buffers = await Promise.all(buffers.map(Zip.gunzipUntracked));
-        time = Date.now() - time;
-        let totalSize = buffers.reduce((acc, buffer) => acc + buffer.length, 0);
-        //console.log(`Gunzip ${formatNumber(totalSize)}B at ${formatNumber(totalSize / time * 1000)}B/s`);
-        return buffers;
-    }
-}
-
-async function doStream(stream: GenericTransformStream, buffer: Buffer): Promise<Buffer> {
-    let reader = stream.readable.getReader();
-    let writer = stream.writable.getWriter();
-    let writePromise = writer.write(buffer);
-    let closePromise = writer.close();
-
-    let outputBuffers: Buffer[] = [];
-    while (true) {
-        let { value, done } = await reader.read();
-        if (done) {
-            await writePromise;
-            await closePromise;
-            return Buffer.concat(outputBuffers);
-        }
-        outputBuffers.push(Buffer.from(value));
-    }
-
-}
+export { Zip };

package/src/diagnostics/logs/lifeCycleAnalysis/spec.md

@@ -1,125 +0,0 @@
-Huge amount of data:
-    https://127-0-0-1.querysubtest.com:7007/?hot&enableLogs&page=login&caseInsensitive&showingmanagement&endTime=1757732880000&startTime=1746997620000&managementpage=LogViewer2
-Very small amount of data
-    https://127-0-0-1.querysubtest.com:7007/?hot&enableLogs&page=login&filter=%22431%22&showingmanagement&endTime=1755140880000&startTime=1754950020000&managementpage=LogViewer2
-
-
-BUG: Digest email isn't limitting per file? We got 12 from a single file?
-
-5) Life cycle analyzer
-    - Implement regular range lifecycles first (matching an === object field)
-        - THEN, Add lifecycle collections where we can define life cycles that have a certain type. 
-            - A collection that has everything by default and we remove life cycles from it would also be useful.
-        - AFTER that, MAYBE, support global values, setting them, and referencing them in the life cycle (which makes them shown as a whole, AND on each line in the life cycle, so we can see the progression).
-            - We have to allow selecting the value we want as well based on a lifecycle state that it can set with its logs. And then we're basically building expression. The whole thing becomes very complicated. So maybe we won't even do it. 
-            - We should definitely wait until we create life cycles and find use out of them before we start adding global values. 
-    - It seems like life cycles have two types. 
-        1) Set global values for user in other logs
-            - Allows for providing specific context, such as what are all the nodes that exist at this time, What are all the paths, authorities that exist, etc, So we can tell what the results should be by essentially knowing what the other values are on other servers inst... antly in a way that the current node couldn't possibly know at that time (And because it can't know it, it couldn't log it. So it's the only way to get this information).
-        2) Range based, a bunch of values connected via an === object field, making one range
-            - allows reducing the complexity of an analysis, by taking lots of different logs and reducing it to just one.. 
-                - IMPORTANT! Show the time and the count of logs in the life cycle.  
-                - also allows us to look at them as a whole, as in how long did it take. Which is minor because we could just measure it, However, sometimes it's annoying to pass the start time information around And if we don't pass the start time around, then we need to look at the start time from the first log and the end time for the last log, which again, would require looking at it as a lifecycle analysis. 
-            - allows adding information based on the existence or the non-existence of a log in that group, such as A setup process missing the last step. 
-                - IMPORTANT! we need to support this by allowing defining life cycles as requiring certain field value combinations which we will end up making basically just the field will always be type and the value will be the type in that life cycle and we'll hard code all the steps that are required and then we can warn when there's a missing step
-                    - I guess also show the count of every single step because that's useful to see if some things were repeated. 
-            - Allows drilling into a lifecycle to see just the specific information of the lifecycle cross server
-                - Otherwise, it's somewhat annoying to look at things cross-server, and while we can search for a specific key that we log, which is exactly what life cycles will do, it's easier to just click to drilldown, rather than having to make the query dynamically every time we want it. 
-        2.1) Collections of life cycles so we can further reduce the complexity. 
-    - Uses FastArchiveViewer, but instead of showing a table, shows lifecycles (a derived concept)
-        - We save them in backblaze, with a bit of cache for loading them
-    - show the life cycles and allow viewing just the specific life cycles and then drilling into those life cycles. Also allow viewing multiple at once so we can view a stream that has many life cycles (as in life cycle collections, which we can also save. )
-    - Show overlap when we're showing the list of life cycles by having it first sorted by start time and then have some kind of indicator for how many values after the value it overlaps with. So if it's serial there'll be no overlap. If everything is being queued up and then run in serial we'll see the overlap go from 1, 2, 3, 4, 5, 6, 7, 8, etc. If it's just in parallel, it'll count up to a number and then go up and down a little bit as values are added and removed. If everything's added at once and it's in parallel, then the values will go up, but then they'll very quickly go down.
-        - Maybe we should have some indication for how much parallel overlap there is? Or kind of like how much gap there is to the start time of the next thing and to the end time of the next thing, something like that. 
-
-6) Add life cycles for
-    - Node discovery life cycle (new node, check for alive, check for paths, etc)
-        - Discover nodes
-        - Check if trusted
-        - Sets authorities
-        - Sets nodes which exist
-        - Ready broadcast
-        - Resync is also part of life cycle
-        - Use different keys for threadId, and, triggeredNodeId, so we can track how a node discovers other nodes, vs how a node is discovered
-        - Mark as dead, dead count increases, remove node as dead
-            - MAYBE there should be 2 lifecycles, one for all, and one for just creation type stuff (no discovery)
-    - There's a life cycle for something dying which starts when we first see it's dead and ends when the count reaches the maximum. We want to use the capability to change the ending state so we can have it as partially dead but not fully dead or fully dead so we can see if the dead life cycles keep starting but not finishing which would be bad. 
-    - Trusted machine lifecycle
-        - Check if we need to add trust
-        - Add trust to archives
-        - Read trusted nodes from archives
-        => Set of trusted machines per threadId
-    - Functions evaluation in FunctionRunner
-        - Adding in http, evaluation, and maybe even forwarding the final PathValue writes to it?
-    - Known nodes
-        - Not really a life cycle, but does set global state
-    - Value syncing
-        - Crosses node boundaries, with the server having to include the client key, so we can track a proxy access => ask the server => validate permissions => send value => receive value => notify proxy
-    - Value creation
-        - Source, propagation, writing to archives
-    - Permissions (probably part of value syncing)
-    - SOCKET connections, open, stats, closed, error, etc.
-    - Archive management
-        - Queue (count)
-        - Archive (count + file)
-        - Joining, deleting, etc, etc
-
-6) Redeploy everything to the servers again, updating our machine apply script, all of our services, deploying our code, etc
-
-7) Use log viewer to find rejections / errors during startup, and then trace down the sync life cycle for the path write, and find where it is lagging out!
-    - Re-enable all of our services as well.
-    - Just logs MIGHT be able to do it, but... life cycles should make it a lot easier to correlate logs, which is something we need to do anyways to solve it...
-
-Make sure we check our life cycles for nodes being added and removed to make sure that life cycle path is pretty empty and there aren't nodes constantly being lost and re-added. 
-
-Check the startup lifecycle to make sure we can detect the nodes pretty fast and in parallel, instead of serial
-
-11) Take all of the errors that we are ignoring and use the life cycles to detect why they're happening. A lot of them really shouldn't be happening. 
-    - Receiving values from different authorities and the ones we're watching is weird. Why does that keep happening?
-    - And we keep running into audit mismatches? Why does that keep happening? Is it only because of our local development server? 
-
-DEBUG: Deploy hash updates. 
-    - Forced refresh now, and then immediately refreshing is STILL not giving us the latest code. Even though we waited for everything to reload the UI, which took forever.
-    - It's probably an issue with the routing information being out of date, I think it's cached in Cloudflare. We could at least use life cycles to verify the values we have, and then if they're different than the values in the client, then I guess it must be in Cloudflare. We can also verify our timing, as I'm pretty sure we're supposed to be waiting for the cloud flare values to update, and if we're not, then that's a problem.
-
-DEBUG: Suppression creation propagation 
-    - It didn't propagate to all the servers?
-
-DEBUG: Machine always up script not restoring dead node?
-    - Add lifecycle for service launching + checking
-    - Does it just not restart killed services... at all?
-    - Maybe machine always up just gets in a bad state (maybe the main loop exits?)
-        - But it's an infinite loop, so it can't?
-
-DEBUG: Why we would get into an infinite identify loop:
-    Identified (loading) f194b2e77661e660.a794fbcf7b104c68.querysubtest.com:34139 as a path authority for:
-        =-.,querysubtest._com.,/
-    Waiting for 1 authorities to be ready:
-        =-.,querysubtest._com.,/
-    Starting to identify f194b2e77661e660.a794fbcf7b104c68.querysubtest.com:34139 as a path authority
-    Identified (loading) f194b2e77661e660.a794fbcf7b104c68.querysubtest.com:34139 as a path authority for:
-        =-.,querysubtest._com.,/
-    Waiting for 1 authorities to be ready:
-        =-.,querysubtest._com.,/
-    Starting to identify f194b2e77661e660.a794fbcf7b104c68.querysubtest.com:34139 as a path authority
-    Identified (loading) f194b2e77661e660.a794fbcf7b104c68.querysubtest.com:34139 as a path authority for:
-        =-.,querysubtest._com.,/
-    Starting to identify f194b2e77661e660.a794fbcf7b104c68.querysubtest.com:34139 as a path authority
-    Identified (loading) f194b2e77661e660.a794fbcf7b104c68.querysubtest.com:34139 as a path authority for:
-        =-.,querysubtest._com.,/
-IMPORTANT! The infinite identify loop causes servers to never finish identification, and so new PathValueServers couldn't start. SO THIS IS A REALLY BIG ISSUE!
-    - Restarting the server fixed it. I'm not sure if it was one service, or all of them?
-
-DEBUG: Backblaze errors
-    - Add a life cycle which handles the fact that one request might result in errors and retries. It also might result in an initial connection being done. It might be waiting on the initial connection, or it might be the one that triggers the initial connection.
-    - Being able to split up the different life cycles by various values, such as if they had to connect, if they had to retry, etc., would be useful as well. And then also splitting up the timing. And graphing various timing such as how long the request actually takes (minus the retries, initial connection, throttling, etc), how long the overall call takes, etc.
-    - And being able to plot by time would be useful as well, so we can see if the errors are time-based. 
-
-SPECIAL UI links for certain errors in log view
-    - Probably dynamically created, based on contents of log
-    - LINKS to filters for all these special errors on a special page
-    - Special errors
-        - Rejection errors
-            - Link to life cycle filtering show the life cycles involving the path (we need to hardcode the life cycle ids for these, after we create the life cycles)
-        - Audit failures
-            - Same info as rejection errors, except for the path, and the failing lock path