querysub 0.374.0 → 0.376.0

package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts

@@ -147,7 +147,7 @@ export async function moveLogsToPublic(config: {
         }));
         if (buffers.length === 0) continue;
 
-        console.log(magenta(`Merging and moving ${group.length} log files backblaze`));
+        console.log(magenta(`Merging and moving ${group.length} log files backblaze (${blue(config.loggerName)})`));
 
         let allBuffers: Buffer[][] = [];
 
@@ -188,7 +188,7 @@ export async function moveLogsToPublic(config: {
             for (let buffer of group) {
                 uncompressedSize += buffer.length;
             }
-            let obj = BufferIndex.encodeAll({
+            let obj = await BufferIndex.encodeAll({
                 data: group,
             });
             encoded.push({
@@ -221,7 +221,7 @@ export async function moveLogsToPublic(config: {
             await localLogs.del(path.fullPath + indexExtension);
         }));
 
-        console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`));
+        console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
     }
 
     // Clean up orphaned index files (index files without corresponding data files)

package/src/diagnostics/logs/diskLogger.ts

@@ -5,7 +5,6 @@ import { lazy } from "socket-function/src/caching";
 import { timeInMinute } from "socket-function/src/misc";
 import { formatTime } from "socket-function/src/formatting/format";
 import { addEpsilons } from "../../bits";
-import { FileMetadata } from "./FastArchiveController";
 import { getPathStr2 } from "../../path";
 import { isPublic } from "../../config";
 // IMPORTANT! We can't have any real imports here, because we are depended on so early in startup!
@@ -32,8 +31,6 @@ export type LogDatum = Record<string, unknown> & {
     __DIR__?: string;
     __NAME__?: string;
     __LINE__?: string;
-    /** Dynamically set at runtime in the frontend. */
-    __metadata?: FileMetadata;
     /** Dynamically set when matching recent errors only. */
     __matchedOutdatedSuppressionKey?: string;
 };
@@ -68,36 +65,6 @@ export const LOG_LINE_LIMIT_FLAG = String.fromCharCode(44534) + "LOGS_LINE_LIMIT
 /** If this key exists in the logged object, as in a key in one of the objects logged, then we will use the value of it as the limit ID. This is useful as it allows us to either override a limit or limit something independently from other logs in the file. */
 export const LOG_LINE_LIMIT_ID = "LIMIT_LINE_ID";
 
-export const getLoggers = lazy(function () {
-    const { FastArchiveAppendable } = require("./FastArchiveAppendable") as typeof import("./FastArchiveAppendable");
-    if (!FastArchiveAppendable) {
-        setImmediate(() => {
-            getLoggers.reset();
-        });
-        return undefined;
-    }
-    return {
-        logLogs: new FastArchiveAppendable<LogDatum>("logs-log/"),
-        warnLogs: new FastArchiveAppendable<LogDatum>("logs-warn/"),
-        infoLogs: new FastArchiveAppendable<LogDatum>("logs-info/"),
-        errorLogs: new FastArchiveAppendable<LogDatum>("logs-error/"),
-    };
-});
-setImmediate(() => {
-    // If we don't import it at all, then it doesn't work client-side. 
-    require("./FastArchiveAppendable") as typeof import("./FastArchiveAppendable");
-});
-const getNotifyErrors = lazy(function () {
-    const { notifyWatchersOfError: notifyErrors } = require("./errorNotifications/ErrorNotificationController") as typeof import("./errorNotifications/ErrorNotificationController");
-    if (typeof notifyErrors !== "function") {
-        setImmediate(() => {
-            getNotifyErrors.reset();
-        });
-        return undefined;
-    }
-    return notifyErrors;
-});
-
 export const getLoggers2 = lazy(function () {
     const { IndexedLogs } = require("./IndexedLogs/IndexedLogs") as typeof import("./IndexedLogs/IndexedLogs");
 
@@ -189,11 +156,6 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
             }
         }
 
-        if (type === "warn" || type === "error") {
-            // Dropping notifies is fine, as long as they get added to the logs, we'll see them eventually...
-            void getNotifyErrors()?.(logObj);
-        }
-
     } catch (e: any) {
         process.stdout.write(`Error writing to disk logs: ${e.stack || e}\n\t${String(args[0])}\n`);
     }

package/src/diagnostics/logs/errorNotifications2/errorNotifications2.ts

@@ -0,0 +1,9 @@
+
+//todonext
+
+// single service, no history search, discord messaging, suppression and suppression page
+//  - Storing suppression history just in memory? (although suppression values, obviously, on disk)
+//  - receive all errors, and service will do suppression
+
+// Start with page itself showing errors and suppressing them, for debugging/testing
+// - Ai to automatically create suppression searches, and combine existing ones, etc

package/src/diagnostics/logs/injectFileLocationToConsole.ts

@@ -41,4 +41,7 @@ if (isNode()) {
 
     let diskShimConsoleLogs = require("./diskShimConsoleLogs") as typeof import("./diskShimConsoleLogs");
     diskShimConsoleLogs.shimConsoleLogs();
+} else {
+    // NOTE: Disable console.info, as serverside we use it for disk only logs
+    console.info = () => { };
 }

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -20,12 +20,6 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 
 
 
-3) Ensure our moveloops are working correctly, on both dev and public
-
-
-1) Fix missing __NAME__
-    "Received PathValue for path" misses name?
-    - Maybe the missing name only happens when we rate limit?
 
 2) Create lot of remote server logs
     - Via our refresh loop
@@ -39,23 +33,18 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 
 
 Rewrite error notification code
-    THINK about how to do a somewhat generic logs => derived thing, as... we will need the exact same thing for life cycles!
-        - Maybe make it generic immediately? Having it abstracted it kind of nice for development anyways...
-    - New service that manages it, instead of doing it on demand
-        - It asks everyone to send it error logs
-    - Stores cached error logs => { unsuppressed logs, suppressionSummary }
-        - Unsuppressed logs only for suppression which is old enough. 
-        - Only when logs are old enough. 
-    - Stores in memory with all suppressionSummaries
-    AND, the only watcher will be the watcher service. You can't get recent errors, or any errors, without going through one of those
-    NO dev errors. They are usually red-herrings anyways... and we should just be using public servers for regular usage
-        - And we still have dev logs we can check if to see if an error happened locally
-    ALSO owns the discord messaging code
-
+    - Single service, which talks to all machines
+        - We'll have a dev mode, mostly for testing, although we won't usually run it.
+        - BROADCASTS messages to all http servers so they show them
+            - Broadcast messages only has one type now "error notification", but add TODO for more types, which may display in different ways (via toast, etc)
+    - Maybe only streaming? We'll miss some, but... not that much...
+    - ALSO owns the discord messaging code
+    - Suppression... will work in the service itself. Because there shouldn't be THAT many errors happening!
+    - Suppression will use AI to group?
+    - BUT, we will still have page that shows suppression list, with counts and times
+        - Page links to search, which tries to find those errors (we should use the same format for suppression as search, so it should be fairly easy to do that)
 
 
-Remove all old LogViewer/FastArchiveAppendable code
-    - Make sure to find all links and update them as well
 
 
 0) Add LZ4 compression to socket-function by default
@@ -63,12 +52,25 @@ Remove all old LogViewer/FastArchiveAppendable code
         - default is "lz4"
     - REQUIRES feature checking the remote, to make sure it is new enough to accept this.
         - A generic thing which gets the version is probably fine.
+    - When decoding, I think we should just check the first byte. It should be a magic number which tells us which compression it's using. 
     - LZ4 compression is fast enough that this should cause basically no overhead, and in many cases greatly reduce the bandwidth (which will increase the speed).
     - We're gonna have to investigate how we're sending buffers anyway. I think this should be easy, but we 
-0.1) Verify the size distance with some local testing
+0.1) Verify the size difference with some local testing
     - ALSO, verify the processing overhead is acceptable.
 1) Deploy, which SHOULD be backwards compatible with everything?
 
+
+-1) Change PathValueSerialize to use LZ4
+    - I think we already have a compression flag here, so it should be easy enough to just support it to change it to support LZ4. 
+    - I think we want to disable the server, and then when we launch the server locally, it should have to read from disk. 
+-2) It would probably be valuable to test the synchronization speed between two different servers. It seems like when the remote is running and we start a local server, it really takes a long time for it to get ready. As in minutes. 
+    - This might have already been fixed by us improving the compression speed to not use gzip, maybe, but we should test it again, and if not, we should see what's slow
+        - We might need to jump into the server and look at its profiles there by forcing them to omit, which isn't too hard, but it is kind of annoying. 
+
+-3) During server startup (when there is a remote server), we spend a lot of time in identityHook.
+    - BUT... Our check to see if the async call that we're doing underneath is taking more than 200 milliseconds, is never firing. Even though apparently plenty of calls are taking over a second in total, so what why is it slow? Are some of the synchronous functions we're calling slow? 
+    - Basically, there's nowhere that could be slow, and we're timing it, and it times as being fast. However, in the profiles, it shows up as being slow. 
+        - It could also be an asynchronous issue, so some other timing is getting alias to it. That is feasible...
 */
 
 

package/src/diagnostics/managementPages.tsx

@@ -28,8 +28,6 @@ import { addComponentButton } from "../5-diagnostics/qreactDebug";
 import { closeAllModals } from "../5-diagnostics/Modal";
 import { delay } from "socket-function/src/batching";
 
-const ErrorWarning = createLazyComponent(() => import("./logs/errorNotifications/ErrorWarning"))("ErrorWarning");
-
 export const managementPageURL = new URLParam("managementpage", "");
 export const showingManagementURL = new URLParam("showingmanagement", false);
 
@@ -86,26 +84,11 @@ export async function registerManagementPages2(config: {
         controllerName: "NodeViewerController",
         getModule: () => import("./NodeViewer"),
     });
-    inputPages.push({
-        title: "Logs",
-        componentName: "LogViewer2",
-        getModule: () => import("./logs/LogViewer2"),
-    });
     inputPages.push({
         title: "Logs3",
         componentName: "LogViewer3",
         getModule: () => import("./logs/IndexedLogs/LogViewer3"),
     });
-    inputPages.push({
-        title: "Life Cycles",
-        componentName: "LifeCyclePages",
-        getModule: () => import("./logs/lifeCycleAnalysis/LifeCyclePages"),
-    });
-    inputPages.push({
-        title: "Error Digests",
-        componentName: "ErrorDigestPage",
-        getModule: () => import("./logs/errorNotifications/ErrorDigestPage"),
-    });
     inputPages.push({
         title: "Security",
         componentName: "SecurityPage",
@@ -339,7 +322,6 @@ class ManagementRoot extends qreact.Component {
                         `}
                     </style>
                     <div class={css.fillWidth.hbox(30, 10).wrap.hsl(245, 25, 60).pad2(10).pointerEvents("all")}>
-                        <ErrorWarning />
                         {pages.map(page =>
                             <ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>
                         )}

package/test.ts

@@ -1,11 +1,6 @@
 import { Querysub } from "./src/4-querysub/QuerysubController";
 // Import querysub, so inject is run
 Querysub;
-import { formatTime } from "socket-function/src/formatting/format";
-import { getMachineId, getOwnMachineId } from "./src/-a-auth/certs";
-import { testLogs } from "./src/diagnostics/logs/LogViewer2";
-import { shutdown } from "./src/diagnostics/periodic";
-import crypto from "crypto";
 import { testMain } from "./testEntry2";
 
 testMain().catch(console.error).finally(() => {

package/bin/error-email.js

@@ -1,8 +0,0 @@
-#!/usr/bin/env node
-
-// public, so we can connect to it from any node (for diagnostics, etc)
-process.argv.push("--public");
-process.argv.push("--nobreak");
-
-require("typenode");
-require("../src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx");

package/bin/error-im.js

@@ -1,8 +0,0 @@
-#!/usr/bin/env node
-
-// public, so we can connect to it from any node (for diagnostics, etc)
-process.argv.push("--public");
-process.argv.push("--nobreak");
-
-require("typenode");
-require("../src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx");