querysub 0.327.0 → 0.328.0

package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx

@@ -1,79 +1,221 @@
 import { batchFunction, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { getControllerNodeId } from "../../../-g-core-values/NodeCapabilities";
-import { RecentErrorsController, recentErrorsChannel, watchRecentErrors } from "./ErrorNotificationController";
-import { timeInSecond } from "socket-function/src/misc";
-import { formatDateTime } from "socket-function/src/formatting/format";
+import { RecentErrors, RecentErrorsController, errorWatcherBase, recentErrorsChannel, suppressionList, watchRecentErrors } from "./ErrorNotificationController";
+import { sort, timeInDay, timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { formatDate, formatDateTime } from "socket-function/src/formatting/format";
+import { getDomain } from "../../../config";
+import { Querysub, QuerysubController } from "../../../4-querysub/QuerysubController";
+import { formatPercent } from "socket-function/src/formatting/format";
+import { LogDatum, getLogFile } from "../diskLogger";
+import { sendDiscordMessage } from "../../../email_ims_notifications/discord";
+import { user_data } from "../../../user-implementation/userData";
+import { createLink } from "../../../library-components/ATag";
+import { getLogsLinkParts } from "./ErrorWarning";
 
-//todonext
+const MAX_IMS_PER_DAY = 3;
+const MAX_IMS_PER_HOURS = 1;
+const MAX_IMS_PER_FILE_PER_DAY = 1;
+const MAX_PER_FILE_PER_IM = 2;
+const MAX_PER_IM = 10;
 
-// 1) Fix whatever is causing our memory usage to highly fluctuate. We seem to just go up and up and up from 100 megabytes all the way up to 300 megabytes and then we GC going back to 100. I turned on the socket function logging and I don't think we're sending network messages. And I ran a profile and it doesn't seem to be doing anything slow, but what are we even doing? We need to fix this. 
-//      - It could be the FPS display. It's pretty easy to check. 
-// YEP! I think it's the FPS display. I disabled it. Now we could see if the air parsing lag goes away and if it does and then comes back when we turn back on the FPS display then you know we're gonna have to fix that. I could understand lagging but why would it be allocating so much? Is it the animation rendering? It definitely could be that because the animations are slow to render and can lag Chrome and can lag the GPU and can lag other windows. 
+// Wait a bit, because it's likely if there's one error, there are more errors. 
+const BATCH_TIME = timeInSecond * 30;
+const DIGEST_INTERVAL = timeInDay * 3;
 
-// 2) The error parsing page is clearly lagging. Even when loading just the error data for the last few days, which is less than a megabytes of total logs, the error loading animation is locking up. Stuttering that is. We need to fix this. 
 
+// 4) Deploy
+// 4) Create local errors and make sure the remote server doesn't get them 
+// 5) Create remote errors and make sure that the all servers get them (Local and remote servers)
 
-// Deploy everything and then update the server runner itself. 
 
-// The constant error notifications might be fixed now. We'll see tomorrow after all the rolling updates finish. 
-// 4) fix whatever's causing constant error notifications. Something is broadcasting on the Recent Errors Change channel constantly. 
-//      - I guess usually there's no server that's going to be listening on it. So it's... Not that big of a deal, but it's still annoying. 
+// Abstract out RecentErrors scanning, so we can reuse it
+//  (appendables, timeRange, callback)
+// - Oh, we also need a way to tell it to not limit to recent errors.
+// - And also to disable throttling. 
 
 
+// createDigest(appendables, timeRange) => Promise<DigestInfo>
 
+// 7) Write the digest script, which does a fresh scan on just a specific range of data using start synchronize internal and suppression list, etc. etc. 
+//      - The most recent ten unsuppressed errors. Counts of unsuppressed errors by file. Or by line id? If they have it. Counts of suppressed and unsuppressed errors by hour group. 
+//      - Oh, also everything we do for errors do for warnings as well, and split the values. We want warnings and errors. 
+//      - Time the entire thing, and put that, and the profile, in the digest too! That will give us a good gauge on if the errors/suppressions are getting slow (due to a lot of errors, or a lot of suppression checks!)
+// store the digest results in Back Blaze. 
+// 8) Write a page that will show the results from Backblaze, allowing you to see a list of all of the digests and to view a specific one. Just showing all the data in a chart, whatever. Showed two charts, one with just the unsuppressed errors, and the other with both suppressed and unsuppressed, because you really only care about suppressed when you're looking at the fraction of suppressed to unsuppressed, and the sheer quantity of unsuppressed. 
+// 9) send an email every time period, and also send an IM that has smaller information
+//      - Both will link to the actual web page that has the digest, deep linking to the specific tabs. 
+//      - Show the chart in the email as well, but just format it like ASCII Because image dependencies are annoying and I don't want to implement them right now as it might take a few days to get working. 
 
-// Back to getting a Node.js watcher that works smoothly without receiving a whole bunch of extra errors etc.
-//  4.1) update channel watching so you can specify that you want to watch only on a specific node ID and then update our code so we only watch it on the controller node ID that we're interfacing with. 
-
-// 5) Verify our suppression updates broadcast across the channel correctly, causing us to be able to suppress a notification and our watching script to then stop seeing the new updates. Realistically, it's the calling script that stops setting them, but same thing. 
 
-// 5) Set up the Instant Messaging Sending API. 
-//  - Discord. With beeper it won't really matter what we're messaging. We could also do WhatsApp. It's really all the same.  
 
-// 6) Set up all the code to properly rate limit IMs, batch them, link back to the log page, etc. 
-//      - Just link to the error page for the last week. We don't need to link to anything specific. 
-//      - properly getting the node ID that we're going to be watching, and if it goes down, getting a new one, and ignoring messages from the old node. 
-//      - And if no node exists, we need to warn and then wait. 
+// NOTE: Yes, this is stored in memory, so if the server reboots or if this script keeps crashing, we might send a lot of instant messages. However, one, Discord will probably late rate limit us, and two, this means something is really wrong, especially if it happens a lot, and we really should fix it right away. 
+// NOTE: If we decide not to send IMs, we don't queue them up, because that's just extremely annoying. The point of the limit isn't to send the maximum we can that stays just under the limit! The point of the limit is if a lot happens at once, ignore most of it. 
+type IMInfo = {
+    time: number;
+    perFile: {
+        [file: string]: LogDatum[];
+    };
+};
+
+let imHistory: IMInfo[] = [];
+function filterIMInfo(info: IMInfo): {
+    info: IMInfo;
+    countFiltered: number;
+} {
+    let countFiltered = 0;
+    // Don't prefer the most warnings, prefer The oldest warnings, which are generally the ones that are first in the object. 
+    // filter based on files per im, and max per im
+    for (let [key, value] of Object.entries(info.perFile)) {
+        if (value.length > MAX_PER_FILE_PER_IM) {
+            countFiltered += value.length - MAX_PER_FILE_PER_IM;
+            value = value.slice(0, MAX_PER_FILE_PER_IM);
+        }
+        info.perFile[key] = value;
+    }
+    let entries = Object.entries(info.perFile);
+    if (entries.length > MAX_PER_IM) {
+        let removed = entries.slice(MAX_PER_IM);
+        for (let [key, value] of removed) {
+            countFiltered += value.length;
+        }
+        entries = entries.slice(0, MAX_PER_IM);
+    }
+    info.perFile = Object.fromEntries(entries);
+    // Also ignore files if they've been mentioned too many times today. 
+    let dayThreshold = Date.now() - timeInDay;
+    let historyInDay = imHistory.filter(x => x.time > dayThreshold);
+    let countByFile = new Map<string, number>();
+    for (let obj of historyInDay) {
+        for (let [key, value] of Object.entries(obj.perFile)) {
+            countByFile.set(key, (countByFile.get(key) || 0) + value.length);
+        }
+    }
+    for (let key of Object.keys(info.perFile)) {
+        let count = countByFile.get(key) || 0;
+        if (count >= MAX_IMS_PER_FILE_PER_DAY) {
+            countFiltered += count;
+            delete info.perFile[key];
+        }
+    }
+    return {
+        info,
+        countFiltered,
+    };
+}
+function canSendNow(info: IMInfo) {
+    let dayThreshold = Date.now() - timeInDay;
+    let historyInDay = imHistory.filter(x => x.time > dayThreshold);
+    let hourThreshold = Date.now() - timeInHour;
+    let historyInHour = historyInDay.filter(x => x.time > hourThreshold);
+
+    if (historyInDay.length >= MAX_IMS_PER_DAY) {
+        return false;
+    }
+    if (historyInHour.length >= MAX_IMS_PER_HOURS) {
+        return false;
+    }
+    return true;
+}
 
+const sendIMs = batchFunction(({ delay: BATCH_TIME }), async (logsAll: LogDatum[][]) => {
+    let logs = logsAll.flat();
+    let infoBase: IMInfo = {
+        time: Date.now(),
+        perFile: {},
+    };
+    for (let log of logs) {
+        let file = getLogFile(log);
+        let array = infoBase.perFile[file];
+        if (!array) {
+            array = [];
+            infoBase.perFile[file] = array;
+        }
+        array.push(log);
+    }
+    let { info, countFiltered } = filterIMInfo(infoBase);
+    if (canSendNow(info)) {
+        imHistory.push(info);
+        let webhookURL = await Querysub.commitAsync(() => {
+            return user_data().secure.notifyDiscordWebhookURL;
+        });
+        if (!webhookURL) {
+            console.error(`No Discord webhook URL set, cannot send warning instant messages`);
+            return;
+        }
+        let url = createLink(getLogsLinkParts());
+        let message = Object.values(info.perFile).flat().map(
+            x => `[${formatDateTime(x.time)}](${url}) | ${x.param0} (${x.__NAME__})`
+        ).join("\n");
+        if (countFiltered > 0) {
+            message += `\n+${countFiltered} more errors`;
+        }
+        void sendDiscordMessage({
+            webhookURL,
+            message,
+        });
+    }
+});
 
-// 7) Write the digest script, which is very different, but will run in the same entry.
-//      - Separate warnings and errors and also bucket by time bucket
-//      - suppressed errors by time bucket (but no type, as we definitely don't want to parse all suppressed errors...)
-//      - Time the entire thing, and put that, and the profile, in the digest too! That will give us a good gauge on if the errors/suppressions are getting slow (due to a lot of errors, or a lot of suppression checks!)
-// 8) Write a page that shows the results of the digest in tabs, writing the digest probably just to backblaze 
-//      - For now, just have two tabs, one for errors and one for warnings. 
-//      - If we're going to do a full scan, we might as well show time series data as well. It's trivial. 
-//      - Also track the number of suppressed errors as well. We won't have details on these such as a breakdown, but we can at least show the count (and the count by time)
-// 9) send an email every time period, and also send an IM that has smaller information
-//      - Both will link to the actual web page that has the digest, deep linking to the specific tabs. 
-//      - Show the chart in the email as well, but just format it like ASCII Because image dependencies are annoying and I don't want to implement them right now as it might take a few days to get working. 
 
 async function runIMNotifies() {
-    let controllerNodeId = await getControllerNodeId(RecentErrorsController.base);
-    if (!controllerNodeId) throw new Error("No controller node id found");
-    //todonext
-    //  Temporary hardcode to use the local server
-    controllerNodeId = "127-0-0-1.querysubtest.com:7007";
-
-    let controller = RecentErrorsController.base.nodes[controllerNodeId];
-    recentErrorsChannel.watch(() => {
-        void updateRecentErrors(undefined);
-    });
-    const updateRecentErrors = batchFunction(
-        //todonext
-        //  Increase this after we finish testing
-        { delay: 1000 },
-        async function updateRecentErrors() {
-            let recentErrors = await controller.getRecentErrors();
-            console.log(`Received ${recentErrors.length} recent errors at ${formatDateTime(Date.now())}`);
-            for (let error of recentErrors) {
-                console.log(`    ${error.param0}`);
+    await Querysub.hostService("error-notifications");
+
+    // NOTE: This should be fine, as realistically how many errors are we going to see in the last two weeks. At 10 million, we're still probably only going to allocate 160 megs of memory, assuming we allocate 16 bytes per number. If we have more than 16 million for a single thread, it'll fail because the max set size... However, they will likely be somewhat distributed between threads. 
+    let errorByThreadIdByDay = new Map<number, Map<string, Set<number>>>();
+    function getDay(time: number) {
+        return Math.floor(time / timeInDay) * timeInDay;
+    }
+    function isDuplicate(obj: LogDatum): boolean {
+        // Checks if it's a duplicate, and if it's not, adds it. 
+        let day = getDay(obj.time);
+        let threadId = obj.__threadId || "";
+        let dayMap = errorByThreadIdByDay.get(day);
+        if (!dayMap) {
+            dayMap = new Map<string, Set<number>>();
+            errorByThreadIdByDay.set(day, dayMap);
+        }
+        let threadIdMap = dayMap.get(threadId);
+        if (!threadIdMap) {
+            threadIdMap = new Set<number>();
+            dayMap.set(threadId, threadIdMap);
+        }
+        if (threadIdMap.has(obj.time)) {
+            return true;
+        }
+        threadIdMap.add(obj.time);
+        return false;
+
+    }
+    function clearOldDays() {
+        // Clear all the days that are more than 14 days older than our current day.
+        let now = Date.now();
+        let currentDay = getDay(now);
+        let cutOffDay = currentDay - 14;
+        for (let day of errorByThreadIdByDay.keys()) {
+            if (day < cutOffDay) {
+                errorByThreadIdByDay.delete(day);
+                console.log(`Cleared old day ${formatDateTime(day)}`);
             }
-            console.log();
-            console.log();
         }
-    );
-    await updateRecentErrors(undefined);
+    }
+    errorWatcherBase.watch(async (objs) => {
+        clearOldDays();
+        objs = await suppressionList.filterObjsToNonSuppressed(objs);
+        objs = objs.filter(x => !isDuplicate(x));
+        if (objs.length === 0) return;
+        // The oldest first as they are most likely the cause. 
+        sort(objs, x => -x.time);
+        void sendIMs(objs);
+        console.log();
+        console.log();
+        console.log(`Received ${objs.length} recent errors at ${formatDateTime(Date.now())}`);
+        for (let obj of objs) {
+            console.log(`    ${obj.param0}`);
+        }
+        console.log();
+        console.log();
+    });
+
 }
 
 async function main() {

package/src/diagnostics/logs/lifeCycleAnalysis/spec.md

@@ -91,8 +91,9 @@ Make sure we check our life cycles for nodes being added and removed to make sur
 
 Check the startup lifecycle to make sure we can detect the nodes pretty fast and in parallel, instead of serial
 
-10) Verify old user/fast-log-cache machine folders are deleted
-
+11) Take all of the errors that we are ignoring and use the life cycles to detect why they're happening. A lot of them really shouldn't be happening. 
+    - Receiving values from different authorities and the ones we're watching is weird. Why does that keep happening?
+    - And we keep running into audit mismatches? Why does that keep happening? Is it only because of our local development server? 
 
 SPECIAL UI links for certain errors in log view
     - Probably dynamically created, based on contents of log

package/src/diagnostics/managementPages.tsx

@@ -90,6 +90,11 @@ export async function registerManagementPages2(config: {
         componentName: "LogViewer2",
         getModule: () => import("./logs/LogViewer2"),
     });
+    inputPages.push({
+        title: "Security",
+        componentName: "SecurityPage",
+        getModule: () => import("../user-implementation/SecurityPage"),
+    });
     inputPages.push({
         title: "Audit Paths",
         componentName: "AuditLogPage",

package/src/email_ims_notifications/discord.tsx

@@ -0,0 +1,203 @@
+import { httpsRequest } from "../https";
+
+export interface DiscordEmbedFooter {
+    text: string;
+    icon_url?: string;
+}
+
+export interface DiscordEmbedImage {
+    url: string;
+}
+
+export interface DiscordEmbedThumbnail {
+    url: string;
+}
+
+export interface DiscordEmbedVideo {
+    url: string;
+}
+
+export interface DiscordEmbedProvider {
+    name?: string;
+    url?: string;
+}
+
+export interface DiscordEmbedAuthor {
+    name: string;
+    url?: string;
+    icon_url?: string;
+}
+
+export interface DiscordEmbedField {
+    name: string;
+    value: string;
+    inline?: boolean;
+}
+
+export interface DiscordEmbed {
+    title?: string;
+    type?: "rich" | "image" | "video" | "gifv" | "article" | "link";
+    description?: string;
+    url?: string;
+    timestamp?: string; // ISO8601 timestamp
+    color?: number; // integer color value
+    footer?: DiscordEmbedFooter;
+    image?: DiscordEmbedImage;
+    thumbnail?: DiscordEmbedThumbnail;
+    video?: DiscordEmbedVideo;
+    provider?: DiscordEmbedProvider;
+    author?: DiscordEmbedAuthor;
+    fields?: DiscordEmbedField[];
+}
+
+export interface DiscordAllowedMentions {
+    parse?: ("roles" | "users" | "everyone")[];
+    roles?: string[];
+    users?: string[];
+    replied_user?: boolean;
+}
+
+export interface DiscordAttachment {
+    id: string;
+    filename: string;
+    description?: string;
+    content_type?: string;
+    size: number;
+    url: string;
+    proxy_url: string;
+}
+
+export interface DiscordEmoji {
+    id?: string;
+    name?: string;
+    animated?: boolean;
+}
+
+export interface DiscordSelectOption {
+    label: string;
+    value: string;
+    description?: string;
+    emoji?: DiscordEmoji;
+    default?: boolean;
+}
+
+export interface DiscordButtonComponent {
+    type: 2; // Button component type
+    style: 1 | 2 | 3 | 4 | 5; // Primary, Secondary, Success, Danger, Link
+    label?: string;
+    emoji?: DiscordEmoji;
+    custom_id?: string; // Required for non-link buttons
+    url?: string; // Required for link buttons
+    disabled?: boolean;
+}
+
+export interface DiscordSelectMenuComponent {
+    type: 3; // String select menu component type
+    custom_id: string;
+    options: DiscordSelectOption[];
+    placeholder?: string;
+    min_values?: number;
+    max_values?: number;
+    disabled?: boolean;
+}
+
+export interface DiscordUserSelectMenuComponent {
+    type: 5; // User select menu component type
+    custom_id: string;
+    placeholder?: string;
+    min_values?: number;
+    max_values?: number;
+    disabled?: boolean;
+}
+
+export interface DiscordRoleSelectMenuComponent {
+    type: 6; // Role select menu component type
+    custom_id: string;
+    placeholder?: string;
+    min_values?: number;
+    max_values?: number;
+    disabled?: boolean;
+}
+
+export interface DiscordMentionableSelectMenuComponent {
+    type: 7; // Mentionable (users + roles) select menu component type
+    custom_id: string;
+    placeholder?: string;
+    min_values?: number;
+    max_values?: number;
+    disabled?: boolean;
+}
+
+export interface DiscordChannelSelectMenuComponent {
+    type: 8; // Channel select menu component type
+    custom_id: string;
+    placeholder?: string;
+    min_values?: number;
+    max_values?: number;
+    disabled?: boolean;
+    channel_types?: number[]; // Array of channel type integers to filter by
+}
+
+export interface DiscordTextInputComponent {
+    type: 4; // Text input component type
+    custom_id: string;
+    style: 1 | 2; // Short (1) or Paragraph (2)
+    label: string;
+    min_length?: number;
+    max_length?: number;
+    required?: boolean;
+    value?: string;
+    placeholder?: string;
+}
+
+export type DiscordComponent =
+    | DiscordButtonComponent
+    | DiscordSelectMenuComponent
+    | DiscordUserSelectMenuComponent
+    | DiscordRoleSelectMenuComponent
+    | DiscordMentionableSelectMenuComponent
+    | DiscordChannelSelectMenuComponent
+    | DiscordTextInputComponent;
+
+export interface DiscordActionRow {
+    type: 1; // Action row component type
+    components: DiscordComponent[];
+}
+
+export interface DiscordWebhookMessage {
+    content?: string;
+    username?: string;
+    avatar_url?: string;
+    tts?: boolean;
+    embeds?: DiscordEmbed[];
+    allowed_mentions?: DiscordAllowedMentions;
+    components?: DiscordActionRow[];
+    files?: any[]; // File attachments - these are handled differently in multipart requests
+    payload_json?: string;
+    attachments?: DiscordAttachment[];
+    flags?: number;
+    thread_name?: string;
+}
+
+export async function sendDiscordMessage(config: {
+    webhookURL: string;
+    message: DiscordWebhookMessage | string;
+}) {
+    const { webhookURL, message } = config;
+
+    const payload: DiscordWebhookMessage = typeof message === "string"
+        ? { content: message }
+        : message;
+
+    await httpsRequest(
+        webhookURL,
+        Buffer.from(JSON.stringify(payload)),
+        "POST",
+        false,
+        {
+            headers: {
+                "Content-Type": "application/json",
+            },
+        }
+    );
+}

package/src/fs.ts

@@ -69,4 +69,13 @@ export async function* readDirRecursive(dir: string): AsyncGenerator<string> {
             } catch { }
         }
     } catch { }
+}
+
+export async function fsExistsAsync(path: string) {
+    try {
+        await fs.promises.stat(path);
+        return true;
+    } catch {
+        return false;
+    }
 }

package/src/functional/SocketChannel.ts

@@ -55,6 +55,7 @@ export class SocketChannel<T> {
     }
 
     private watchAllNodes = lazy(async () => {
+        // NOTE: By watching instead of having nodes broadcast, it naturally prevents non-public servers from sending messages to public servers. This is really nice, and helps keep non-public servers (development servers), actually non-public...
         watchNodeIds((nodeIds) => {
             for (let nodeId of nodeIds) {
                 void errorToUndefinedSilent(this.controller.nodes[nodeId]._internal_watchMessages());
@@ -68,4 +69,12 @@ export class SocketChannel<T> {
             this.localWatchers.delete(callback);
         };
     }
+    // NOTE: We also get notifications for watching on all nodes, which should be fine...
+    public async watchSingleNode(nodeId: string, callback: (message: T) => void) {
+        await this.controller.nodes[nodeId]._internal_watchMessages();
+        this.localWatchers.add(callback);
+        return () => {
+            this.localWatchers.delete(callback);
+        };
+    }
 }