querysub 0.374.0 → 0.376.0

package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx

@@ -1,209 +0,0 @@
-import { batchFunction, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { getControllerNodeId } from "../../../-g-core-values/NodeCapabilities";
-import { RecentErrors, RecentErrorsController, errorWatcherBase, recentErrorsChannel, suppressionList, watchRecentErrors } from "./ErrorNotificationController";
-import { sort, timeInDay, timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
-import { formatDate, formatDateTime } from "socket-function/src/formatting/format";
-import { getDomain } from "../../../config";
-import { Querysub, QuerysubController } from "../../../4-querysub/QuerysubController";
-import { formatPercent } from "socket-function/src/formatting/format";
-import { LogDatum, getLogFile } from "../diskLogger";
-import { sendDiscordMessage } from "../../../email_ims_notifications/discord";
-import { user_data } from "../../../user-implementation/userData";
-import { createLink } from "../../../library-components/ATag";
-import { getErrorLogsLink } from "./ErrorWarning";
-import { magenta } from "socket-function/src/formatting/logColors";
-import { ellipsisMiddle } from "../../../misc";
-
-const MAX_IMS_PER_DAY = 3;
-const MAX_IMS_PER_HOURS = 1;
-const MAX_IMS_PER_FILE_PER_DAY = 1;
-const MAX_PER_FILE_PER_IM = 2;
-const MAX_PER_IM = 10;
-
-// Wait a bit, because it's likely if there's one error, there are more errors. 
-const BATCH_TIME = timeInSecond * 30;
-
-
-// NOTE: Yes, this is stored in memory, so if the server reboots or if this script keeps crashing, we might send a lot of instant messages. However, one, Discord will probably late rate limit us, and two, this means something is really wrong, especially if it happens a lot, and we really should fix it right away. 
-// NOTE: If we decide not to send IMs, we don't queue them up, because that's just extremely annoying. The point of the limit isn't to send the maximum we can that stays just under the limit! The point of the limit is if a lot happens at once, ignore most of it. 
-type IMInfo = {
-    time: number;
-    perFile: {
-        [file: string]: LogDatum[];
-    };
-};
-
-let imHistory: IMInfo[] = [];
-function filterIMInfo(info: IMInfo): {
-    info: IMInfo;
-    countFiltered: number;
-} {
-    let countFiltered = 0;
-    // Don't prefer the most warnings, prefer The oldest warnings, which are generally the ones that are first in the object. 
-    // filter based on files per im, and max per im
-    for (let [key, value] of Object.entries(info.perFile)) {
-        if (value.length > MAX_PER_FILE_PER_IM) {
-            countFiltered += value.length - MAX_PER_FILE_PER_IM;
-            value = value.slice(0, MAX_PER_FILE_PER_IM);
-        }
-        info.perFile[key] = value;
-    }
-    let entries = Object.entries(info.perFile);
-    if (entries.length > MAX_PER_IM) {
-        let removed = entries.slice(MAX_PER_IM);
-        for (let [key, value] of removed) {
-            countFiltered += value.length;
-        }
-        entries = entries.slice(0, MAX_PER_IM);
-    }
-    info.perFile = Object.fromEntries(entries);
-    // Also ignore files if they've been mentioned too many times today. 
-    let dayThreshold = Date.now() - timeInDay;
-    let historyInDay = imHistory.filter(x => x.time > dayThreshold);
-    let countByFile = new Map<string, number>();
-    for (let obj of historyInDay) {
-        for (let [key, value] of Object.entries(obj.perFile)) {
-            countByFile.set(key, (countByFile.get(key) || 0) + value.length);
-        }
-    }
-    for (let key of Object.keys(info.perFile)) {
-        let count = countByFile.get(key) || 0;
-        if (count >= MAX_IMS_PER_FILE_PER_DAY) {
-            countFiltered += count;
-            delete info.perFile[key];
-        }
-    }
-    return {
-        info,
-        countFiltered,
-    };
-}
-function canSendNow(info: IMInfo) {
-    let dayThreshold = Date.now() - timeInDay;
-    let historyInDay = imHistory.filter(x => x.time > dayThreshold);
-    let hourThreshold = Date.now() - timeInHour;
-    let historyInHour = historyInDay.filter(x => x.time > hourThreshold);
-
-    if (historyInDay.length >= MAX_IMS_PER_DAY) {
-        console.log(magenta(`Not sending IM, because we've sent ${historyInDay.length} > ${MAX_IMS_PER_DAY} IMs in the last day`));
-        return false;
-    }
-    if (historyInHour.length >= MAX_IMS_PER_HOURS) {
-        console.log(magenta(`Not sending IM, because we've sent ${historyInHour.length} > ${MAX_IMS_PER_HOURS} IMs in the last hour`));
-        return false;
-    }
-    return true;
-}
-
-const sendIMs = batchFunction(({ delay: BATCH_TIME }), async (logsAll: LogDatum[][]) => {
-    let logs = logsAll.flat();
-    let infoBase: IMInfo = {
-        time: Date.now(),
-        perFile: {},
-    };
-    for (let log of logs) {
-        let file = getLogFile(log);
-        let array = infoBase.perFile[file];
-        if (!array) {
-            array = [];
-            infoBase.perFile[file] = array;
-        }
-        array.push(log);
-    }
-    let { info, countFiltered } = filterIMInfo(infoBase);
-    if (canSendNow(info)) {
-        imHistory.push(info);
-        let webhookURL = await Querysub.commitAsync(() => {
-            return user_data().secure.notifyDiscordWebhookURL;
-        });
-        if (!webhookURL) {
-            console.error(`No Discord webhook URL set, cannot send warning instant messages`);
-            return;
-        }
-        let url = createLink(getErrorLogsLink());
-        let flat = Object.values(info.perFile).flat();
-        if (flat.length === 0) {
-            console.log(`All messages have hit their IM limit, so not sending any message at all.`);
-            return;
-        }
-        let message = flat.map(
-            x => `[${formatDateTime(x.time)}](${url}) | ${ellipsisMiddle(String(x.param0), 100)} (${x.__NAME__})`
-        ).join("\n");
-        message = ellipsisMiddle(message, 900);
-        if (countFiltered > 0) {
-            message += `\n+${countFiltered} more errors`;
-        }
-        console.log(`Discord message: ${message}`);
-        void sendDiscordMessage({
-            webhookURL,
-            message,
-        });
-    }
-});
-
-
-async function runIMNotifies() {
-    await Querysub.hostService("error-notifications");
-
-    // NOTE: This should be fine, as realistically how many errors are we going to see in the last two weeks. At 10 million, we're still probably only going to allocate 160 megs of memory, assuming we allocate 16 bytes per number. If we have more than 16 million for a single thread, it'll fail because the max set size... However, they will likely be somewhat distributed between threads. 
-    let errorByThreadIdByDay = new Map<number, Map<string, Set<number>>>();
-    function getDay(time: number) {
-        return Math.floor(time / timeInDay) * timeInDay;
-    }
-    function isDuplicate(obj: LogDatum): boolean {
-        // Checks if it's a duplicate, and if it's not, adds it. 
-        let day = getDay(obj.time);
-        let threadId = obj.__threadId || "";
-        let dayMap = errorByThreadIdByDay.get(day);
-        if (!dayMap) {
-            dayMap = new Map<string, Set<number>>();
-            errorByThreadIdByDay.set(day, dayMap);
-        }
-        let threadIdMap = dayMap.get(threadId);
-        if (!threadIdMap) {
-            threadIdMap = new Set<number>();
-            dayMap.set(threadId, threadIdMap);
-        }
-        if (threadIdMap.has(obj.time)) {
-            return true;
-        }
-        threadIdMap.add(obj.time);
-        return false;
-    }
-    function clearOldDays() {
-        // Clear all the days that are more than 14 days older than our current day.
-        let now = Date.now();
-        let currentDay = getDay(now);
-        let cutOffDay = currentDay - 14;
-        for (let day of errorByThreadIdByDay.keys()) {
-            if (day < cutOffDay) {
-                errorByThreadIdByDay.delete(day);
-                console.log(`Cleared old day ${formatDateTime(day)}`);
-            }
-        }
-    }
-    errorWatcherBase.watch(async (objs) => {
-        clearOldDays();
-        console.log(magenta(`Pre-suppression: received ${objs.length} recent errors at ${formatDateTime(Date.now())}`));
-        objs = await suppressionList.filterObjsToNonSuppressed(objs);
-        objs = objs.filter(x => !isDuplicate(x));
-        if (objs.length === 0) return;
-        // The oldest first as they are most likely the cause. 
-        sort(objs, x => -x.time);
-        void sendIMs(objs);
-        console.log();
-        console.log();
-        console.log(`Received ${objs.length} recent errors at ${formatDateTime(Date.now())}`);
-        for (let obj of objs) {
-            console.log(`    ${obj.param0}`);
-        }
-        console.log();
-        console.log();
-    });
-
-}
-
-async function main() {
-    void runIMNotifies();
-}
-void main();

package/src/diagnostics/logs/importLogsEntry.ts

@@ -1,38 +0,0 @@
-import { Querysub } from "../../4-querysub/Querysub";
-import fs from "fs";
-import { Zip } from "../../zip";
-import { getLoggers } from "./diskLogger";
-import { formatNumber } from "socket-function/src/formatting/format";
-import { shutdown } from "../periodic";
-
-async function main() {
-    let { logLogs } = getLoggers()!;
-    //let root = "X:/root/machine-services/cyoa-1-dply/git/database-storage/disklogs";
-    /*
-    let root = "D:/repos/qs-cyoa/database-storage/disklogs/";
-    let files = await fs.promises.readdir(root);
-    for (let fileName of files) {
-        let path = root + fileName;
-        if (!path.endsWith(".log")) continue;
-        let contents = await fs.promises.readFile(path);
-        let lines = contents.toString("utf8").split("\n");
-        for (let line of lines) {
-            if (!line) continue;
-            try {
-                let log = JSON.parse(line);
-                logLogs.append(log);
-            } catch (e) {
-                require("debugbreak")(2);
-                debugger;
-                console.error(e, line);
-            }
-        }
-        console.log(`Flushing ${path}, line count: ${formatNumber(lines.length)}, ${formatNumber(contents.length)}B`);
-        let time = Number(fileName.split("-")[0]) || Date.now();
-        await logLogs.flushNow(time);
-    }
-    */
-    await logLogs.moveLogsToBackblaze();
-    //await shutdown();
-}
-main().catch(console.error).finally(() => process.exit());

package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePages.tsx

@@ -1,150 +0,0 @@
-module.hotreload = true;
-import { qreact } from "../../../4-dom/qreact";
-import { css } from "../../../4-dom/css";
-import { t } from "../../../2-proxy/schema2";
-import { Querysub } from "../../../4-querysub/QuerysubController";
-import { sort } from "socket-function/src/misc";
-import { FastArchiveViewer } from "../FastArchiveViewer";
-import { LogDatum, getLoggers } from "../diskLogger";
-import { Table } from "../../../5-diagnostics/Table";
-import { formatDateTime } from "socket-function/src/formatting/format";
-import { FastArchiveAppendable } from "../FastArchiveAppendable";
-import { getTimeRange } from "../TimeRangeSelector";
-import { URLParam } from "../../../library-components/URLParam";
-import { InputLabelURL } from "../../../library-components/InputLabel";
-import { isPublic } from "../../../config";
-
-const forceGetPublicURL = new URLParam("forceGetPublic", false);
-
-export class LifeCyclePages extends qreact.Component {
-    state = t.state({
-        dataSeqNum: t.atomic<number>(0),
-    });
-
-    private datums: LogDatum[] = [];
-    private latestByName = new Map<string, LogDatum>();
-    private fastArchiveViewer: FastArchiveViewer<LogDatum> | undefined = undefined;
-
-    rerun() {
-        Querysub.onCommitFinished(async () => {
-            void this.fastArchiveViewer?.synchronizeDataThrottled();
-        });
-    }
-
-    getLoggers = () => {
-        let logs: FastArchiveAppendable<LogDatum>[] = [];
-        let loggers = getLoggers();
-        if (!loggers) {
-            throw new Error("Loggers not available?");
-        }
-        let { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
-        logs.push(logLogs);
-        logs.push(infoLogs);
-        logs.push(warnLogs);
-        logs.push(errorLogs);
-        return logs;
-    };
-
-    renderContent() {
-        let latestEntries = Array.from(this.latestByName.entries());
-        sort(latestEntries, x => -(x[1].time || 0));
-
-        let rows = latestEntries.map(([name, datum]) => ({
-            name,
-            time: datum.time,
-            datum,
-        }));
-
-        return <>
-            <div className={css.hbox(10)}>
-                <div>
-                    Total unique __NAME__ values: {this.latestByName.size}
-                </div>
-                <div>
-                    Total logs processed: {this.datums.length}
-                </div>
-            </div>
-
-            {rows.length === 0 && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
-                No logs matched, either increase the time range or run the search.
-            </div>}
-
-            <Table
-                rows={rows}
-                columns={{
-                    name: {
-                        title: "Name",
-                        formatter: x => <div>{String(x)}</div>
-                    },
-                    time: {
-                        title: "Last Time",
-                        formatter: x => <div>{formatDateTime(Number(x))}</div>
-                    },
-                }}
-                lineLimit={4}
-                initialLimit={20}
-                characterLimit={400}
-            />
-        </>;
-    }
-
-    render() {
-        this.state.dataSeqNum;
-
-        const timeRange = getTimeRange();
-        return (
-            <div className={css.vbox(20).pad2(20).fillBoth}>
-                <div className={css.hbox(10)}>
-                    <div className={css.fontSize(14)}>
-                        Life Cycle Analysis
-                    </div>
-                    {!isPublic() && <InputLabelURL
-                        label="Force Public Logs" checkbox url={forceGetPublicURL}
-                        onChangeValue={(newValue) => {
-                            this.rerun();
-                        }}
-                    />}
-                </div>
-
-                <FastArchiveViewer
-                    ref2={x => this.fastArchiveViewer = x}
-                    fastArchives={this.getLoggers}
-                    runOnLoad={false}
-                    forceGetPublic={forceGetPublicURL.value}
-                    onStart={async () => {
-                        this.datums = [];
-                        this.latestByName = new Map();
-                    }}
-                    onDatums={(source, datums, file) => {
-                        for (let datum of datums) {
-                            let time = datum.time;
-                            if (time && (time < timeRange.startTime || time > timeRange.endTime)) {
-                                continue;
-                            }
-                            this.datums.push(datum);
-                        }
-                    }}
-                    onFinish={() => {
-                        // Sort all datums by time
-                        sort(this.datums, x => x.time || 0);
-
-                        // Get the latest value (last time) for every __NAME__
-                        this.latestByName.clear();
-                        for (let datum of this.datums) {
-                            const name = datum.__NAME__;
-                            if (name) {
-                                this.latestByName.set(name, datum);
-                            }
-                        }
-
-                        Querysub.commit(() => {
-                            this.state.dataSeqNum++;
-                        });
-                    }}
-                >
-                    {this.renderContent()}
-                </FastArchiveViewer>
-            </div>
-        );
-    }
-}

package/src/diagnostics/logs/logViewerExtractField.ts

@@ -1,36 +0,0 @@
-import { ScanFnc } from "./FastArchiveViewer";
-
-export function createLogViewerExtractField(
-    fieldName: string,
-    onFieldValue: (value: string, unpackedSize: number) => void,
-): ScanFnc {
-    let fieldNameBuffer = Buffer.from(`${JSON.stringify(fieldName)}:"`);
-    let quoteChar = Buffer.from(`"`)[0];
-    let escapeChar = Buffer.from(`\\`)[0];
-    return (posStart: number, posEnd: number, data: Buffer) => {
-        for (let i = posStart; i < posEnd; i++) {
-            if (data[i] === fieldNameBuffer[0]) {
-                for (let j = 1; j < fieldNameBuffer.length; j++) {
-                    if (data[i + j] !== fieldNameBuffer[j]) {
-                        break;
-                    }
-                    if (j === fieldNameBuffer.length - 1) {
-                        // Parse until the string ends, ignoring escaped characters
-                        let start = i + j + 1;
-                        for (let k = start; k < posEnd; k++) {
-                            if (data[k] === quoteChar) {
-                                onFieldValue(data.slice(start, k).toString(), posEnd - start);
-                                break;
-                            }
-                            if (data[k] === escapeChar) {
-                                k++;
-                            }
-                        }
-                        return true;
-                    }
-                }
-            }
-        }
-        return false;
-    };
-}