querysub 0.312.0 → 0.313.0

package/src/diagnostics/errorLogs/ErrorLogCore.ts

@@ -1,274 +0,0 @@
-import { cache, lazy } from "socket-function/src/caching";
-import { timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
-import { getArchives } from "../../-a-archives/archives";
-import { Zip } from "../../zip";
-import yaml from "yaml";
-import { measureWrap } from "socket-function/src/profiling/measure";
-
-export const MAX_EXAMPLE_LENGTH = 1024;
-export const MAX_EXAMPLES = 16;
-export const MAX_UNGROUPED_EXAMPLES = 1000;
-export const LOG_FLUSH_INTERVAL = timeInHour;
-export function getMaxExamples(classifier: LogClass): number {
-    if (isUngroupedClass(classifier)) return MAX_UNGROUPED_EXAMPLES;
-    return classifier.maxExamples ?? MAX_EXAMPLES;
-}
-
-export const LOG_ARCHIVES = lazy(() => getArchives("notify-error-logs"));
-export const LOG_CLASSES_PATH = "classes.yaml";
-export const LOG_BLOCK_EXTENSION = ".log";
-
-export const ungroupedPrefix = "ungrouped_";
-
-export type LogRaw = {
-    time: number;
-    message: string;
-};
-
-export type LogType = (
-    // NOTE: Most of these levels come fron the type of console[fnc] used
-    //  - The type controls the rendering style, as well as the default suppression time.
-    "info" | "warn" | "error"
-    // NOTE: Fatal is any error that includes "FATAL" in the first line
-    //  TODO: On (new) fatal errors, we should also send an email to the admin
-    | "fatal"
-);
-export type LogClass = {
-    id: string;
-    title: string;
-    description: string;
-    /** Defaults to 0. Higher means it will take logs before lower priority LogClasses. */
-    priority?: number;
-    createTime: number;
-
-    match: {
-        type: "includes";
-        // Filters for log.toLowerCase().includes(value),
-        //  matching sanitizeMessageForMatch(message)
-        value: string;
-    }[];
-
-    // Suppress from notifications (in the UI, or email, or however it is setup)
-    //      until time > suppressUntil.
-    //      - By default "info" type logs set this to Number.MAX_SAFE_INTEGER.
-    suppressUntil: number;
-
-    // If set overrides MAX_EXAMPLES (useful for getting live logs).
-    maxExamples?: number;
-
-    type: LogType;
-};
-let allTypes: { [type in LogType]: 1 } = {
-    fatal: 1,
-    error: 1,
-    warn: 1,
-    info: 1,
-};
-
-export type LogBlock = {
-    classes: { [classId: string]: LogClassSummary; };
-    startTime: number;
-    endTime: number;
-
-    threadId: string;
-    machineId: string;
-
-    // NOTE: These counts are just for navigation of files, AND for the default type for unclassed values.
-    //  Once a value has a class that determines the type (which might be different than the input class).
-    fatalCount: number;
-    errorCount: number;
-    warnCount: number;
-    infoCount: number;
-
-    seqNum: number;
-
-    count: number;
-};
-export type LogBlockInfo = {
-    startTime: number;
-    endTime: number;
-
-    seqNum: number;
-
-    count: number;
-
-    fatalCount: number;
-    errorCount: number;
-    warnCount: number;
-    infoCount: number;
-
-    threadId: string;
-    machineId: string;
-
-    fileName: string;
-};
-
-// NOTE: After enough classes we will start taking ungrouped values and
-//      putting them in an "Other" class.
-export type LogClassSummary = {
-    count: number;
-    // NOTE: The
-    logClassId: string;
-    logClassType: LogType;
-    examples: LogRaw[];
-    firstTime: number;
-    lastTime: number;
-};
-
-export function sanitizeMessageForMatch(message: string): string {
-    message = message.toLowerCase();
-    // Replace multiple whitespaces with a single space
-    message = message.replace(/\s+/g, " ");
-    return message;
-}
-
-export function isClassifierMatch(logClass: LogClass, log: LogRaw): boolean {
-    // TODO: Cache sanitizeMessageForMatch when looping thorugh logClasses
-    let logMessage = sanitizeMessageForMatch(log.message);
-    for (const match of logClass.match) {
-        if (logMessage.includes(match.value.toLowerCase())) {
-            return true;
-        }
-    }
-    return false;
-}
-
-export function getLogClassCategorizer(logClasses: LogClass[]): (log: LogRaw, typeHint: LogType) => LogClass {
-    return measureWrap(function classifyRawLog(log, typeHint) {
-        for (const logClass of logClasses) {
-            if (isClassifierMatch(logClass, log)) {
-                return logClass;
-            }
-        }
-        return ungroupedClass(typeHint);
-    });
-}
-
-export const ungroupedClass = cache((type: LogType): LogClass => {
-    return {
-        id: `${ungroupedPrefix}${type}`,
-        title: `Ungrouped ${type}`,
-        createTime: 0,
-        description: "Ungrouped logs",
-        match: [],
-        suppressUntil: 0,
-        type,
-    };
-});
-export function isUngroupedClass(classObj: LogClass): boolean {
-    return classObj.id.startsWith(ungroupedPrefix);
-}
-
-export function addToLogBlock(block: LogBlock, logClass: LogClass, log: LogRaw) {
-    let classSummary = block.classes[logClass.id];
-    let now = Date.now();
-    if (!classSummary) {
-        classSummary = createLogClassSummary(now, logClass);
-        block.classes[logClass.id] = classSummary;
-    }
-    let maxExamples = getMaxExamples(logClass);
-    if (classSummary.examples.length < maxExamples) {
-        if (log.message.length > MAX_EXAMPLE_LENGTH) {
-            log = { ...log };
-            log.message = log.message.slice(0, MAX_EXAMPLE_LENGTH - 3) + "...";
-        }
-        classSummary.examples.push(log);
-    }
-    classSummary.count++;
-    if (now > classSummary.lastTime) classSummary.lastTime = now;
-    if (now > block.endTime) block.endTime = now;
-    block.count++;
-    if (logClass.type === "fatal") block.fatalCount++;
-    if (logClass.type === "error") block.errorCount++;
-    if (logClass.type === "warn") block.warnCount++;
-    if (logClass.type === "info") block.infoCount++;
-}
-function createLogClassSummary(now: number, logClass: LogClass): LogClassSummary {
-    return {
-        count: 0,
-        logClassId: logClass.id,
-        examples: [],
-        firstTime: now,
-        lastTime: now,
-        logClassType: logClass.type,
-    };
-}
-
-export async function logBlockToBuffer(block: LogBlock): Promise<{
-    fileName: string;
-    buffer: Buffer;
-}> {
-    let buffer = await Zip.gzip(Buffer.from(JSON.stringify(block)));
-    let info = getLogBlockInfoFromBlock(block);
-    let fileName = getLogBlockFileName(info);
-    return { fileName, buffer };
-}
-export async function logBlockFromBuffer(buffer: Buffer): Promise<LogBlock> {
-    buffer = await Zip.gunzip(buffer);
-    return JSON.parse(buffer.toString());
-}
-function getLogBlockInfoFromBlock(block: LogBlock): LogBlockInfo {
-    return {
-        startTime: block.startTime,
-        endTime: block.endTime,
-        seqNum: block.seqNum,
-        count: block.count,
-        fatalCount: block.fatalCount,
-        errorCount: block.errorCount,
-        warnCount: block.warnCount,
-        infoCount: block.infoCount,
-        threadId: block.threadId,
-        machineId: block.machineId,
-        fileName: [
-            "count " + block.count,
-            "seq " + block.seqNum,
-            "fatal " + block.fatalCount,
-            "errors " + block.errorCount,
-            "warns " + block.warnCount,
-            "infos " + block.infoCount,
-            "thread " + block.threadId,
-            "machine " + block.machineId,
-            block.startTime + " to " + block.endTime,
-            LOG_BLOCK_EXTENSION
-        ].join("   "),
-    };
-}
-export function getLogBlockFileName(info: LogBlockInfo): string {
-    return info.fileName;
-}
-export function getLogBlockInfo(fileName: string): LogBlockInfo {
-    let parts = fileName.split("   ");
-    return {
-        count: Number(parts[0].split(" ")[1]),
-        seqNum: Number(parts[1].split(" ")[1]),
-        fatalCount: Number(parts[2].split(" ")[1]),
-        errorCount: Number(parts[3].split(" ")[1]),
-        warnCount: Number(parts[4].split(" ")[1]),
-        infoCount: Number(parts[5].split(" ")[1]),
-        threadId: parts[6].split(" ")[1],
-        machineId: parts[7].split(" ")[1],
-        startTime: Number(parts[8].split(" to ")[0]),
-        endTime: parseFloat(parts[8].split(" to ")[1]),
-        fileName,
-    };
-}
-
-export function logClassesToString(logClasses: LogClass[]): string {
-    return lineYamlToString(logClasses.filter(x => !isUngroupedClass(x)));
-}
-export function logClassesFromString(file: string): LogClass[] {
-    let result = lineYamlFromString(file) as LogClass[];
-    for (let key in allTypes) {
-        result.push(ungroupedClass(key as LogType));
-    }
-    return result;
-}
-
-const LINE_DELIMITTER = "\n=============================\n";
-export function lineYamlToString(values: unknown[]): string {
-    return values.map(value => yaml.stringify(value)).join(LINE_DELIMITTER);
-}
-export function lineYamlFromString<T>(file: string): T[] {
-    if (!file.trim()) return [];
-    return file.split(LINE_DELIMITTER).map(value => yaml.parse(value));
-}

package/src/diagnostics/errorLogs/LogClassifiers.tsx

@@ -1,308 +0,0 @@
-import preact from "preact"; import { qreact } from "../../../src/4-dom/qreact";
-import { LogViewerController, LogViewerSynced, formatTypeColor } from "./LogViewer";
-import { getBrowserUrlNode } from "../../../src/-f-node-discovery/NodeDiscovery";
-import { css } from "../../../src/4-dom/css";
-import { LogClass, LogType, getMaxExamples, isUngroupedClass } from "../../../src/diagnostics/errorLogs/ErrorLogCore";
-import { Table } from "../../../src/5-diagnostics/Table";
-import { deepCloneJSON, nextId, sort, timeInDay } from "socket-function/src/misc";
-import { FormatContext, JSXFormatter, formatValue, toSpaceCase } from "../../../src/5-diagnostics/GenericFormat";
-import { Input } from "../../../src/library-components/Input";
-import { InputLabel } from "../../../src/library-components/InputLabel";
-import { fatalColor, errorColor, warnColor, infoColor } from "./logFiltering";
-import { Button } from "../../../src/library-components/Button";
-import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { errorMessage } from "../../../src/library-components/colors";
-import { DropdownCustom } from "../../../src/library-components/DropdownCustom";
-import { createURLSync } from "../../../src/library-components/URLParam";
-import { Querysub } from "../../../src/4-querysub/Querysub";
-
-export const logClassIdURL = createURLSync("id", "");
-
-export class LogClassifiers extends qreact.Component {
-    state = {
-        add: ""
-    };
-    render() {
-        let controller = LogViewerSynced(getBrowserUrlNode());
-        let classes = controller.getClasses() || [];
-        classes = classes.filter(x => !isUngroupedClass(x));
-        sort(classes, x => -x.createTime || Number.MAX_SAFE_INTEGER);
-
-        let specificId = logClassIdURL.value;
-        if (specificId) {
-            classes = classes.filter(x => x.id === specificId);
-        }
-
-        let editFormatter: JSXFormatter<unknown, LogClass> = (value, context) => {
-            const columnName = context?.columnName;
-            if (!columnName) throw new Error(`Column name not provided to formatter.`);
-            const row = context?.row;
-            if (!row) throw new Error(`Row not provided to formatter.`);
-
-            let valueFormatted: preact.ComponentChild = formatValue(value);
-            if (columnName === "type") {
-                valueFormatted = formatTypeColor(value as LogType, String(value));
-            }
-
-            const updateValue = async function updateValue(newValue: unknown) {
-                let updateController = LogViewerController.nodes[getBrowserUrlNode()];
-                let newClass: LogClass = { ...row };
-                newClass[columnName] = newValue as never;
-                await updateController.updateClass(newClass);
-                controller.resetAll();
-            };
-
-            if (columnName === "type") {
-                return <DropdownCustom
-                    optionClass={
-                        css.backgroundColor("hsla(0, 0%, 100%, 0.5)", "important")
-                            .padding("2px 6px", "important")
-                    }
-                    value={value}
-                    options={[
-                        { label: () => formatTypeColor("fatal", "Fatal"), value: "fatal" },
-                        { label: () => formatTypeColor("error", "Error"), value: "error" },
-                        { label: () => formatTypeColor("warn", "Warn"), value: "warn" },
-                        { label: () => formatTypeColor("info", "Info"), value: "info" },
-                    ]}
-                    onChange={(newValue) => updateValue(newValue)}
-                />;
-            }
-            if (columnName === "match") {
-                let valueT = value as LogClass["match"];
-                valueT = deepCloneJSON(valueT);
-                return (
-                    valueT.map((match, index) => {
-                        let matchType = match.type;
-                        if (matchType === "includes") {
-                            return <Input
-                                value={match.value}
-                                onChangeValue={async newValue => {
-                                    let newMatch = { ...match, value: newValue };
-                                    let newMatches = [...valueT];
-                                    newMatches[index] = newMatch;
-                                    await updateValue(newMatches);
-                                }}
-                            />;
-                        }
-                        let unhandledType = matchType;
-                        return JSON.stringify(match);
-                    })
-                );
-            }
-
-            if (columnName === "maxExamples" && !valueFormatted) {
-                value = getMaxExamples(row);
-                valueFormatted = String(value);
-            }
-            if (columnName === "priority") {
-                value = value ?? 0;
-            }
-            if (columnName === "createTime") {
-                return formatValue(Number(value) || "");
-            }
-
-            if (columnName === "suppressUntil") {
-                return <SuppressUntil
-                    type={row.type}
-                    time={row.suppressUntil}
-                    updateClass={async (newFields) => {
-                        let updateController = LogViewerController.nodes[getBrowserUrlNode()];
-                        let newClass: LogClass = { ...row, ...newFields };
-                        await updateController.updateClass(newClass);
-                        controller.resetAll();
-                    }}
-                />;
-            }
-
-
-            let edit = true;
-            let isNumber = false;
-            let isTextArea = false;
-            let isDatetime = false;
-            if (typeof value === "number") {
-                isNumber = true;
-            }
-            if (columnName === "description") {
-                isTextArea = true;
-            }
-            return (
-                <InputLabel
-                    edit={edit}
-                    editValue={valueFormatted}
-                    textarea={isTextArea}
-                    isDatetime={isDatetime}
-                    value={String(value)}
-                    number={isNumber}
-                    onChangeValue={async newValue => {
-                        if (isNumber) {
-                            newValue = Number(newValue) as any;
-                        }
-                        await updateValue(newValue);
-                    }}
-                />
-            );
-        };
-
-        const createClassifier = async () => {
-            let previewFilter = this.state.add;
-            // Wait for the blur to finish before resetting previewFilter
-            Querysub.onCommitFinished(() => {
-                setImmediate(() => {
-                    Querysub.localCommit(() => {
-                        this.state.add = "";
-                    });
-                });
-            });
-            await LogViewerController.nodes[getBrowserUrlNode()].updateClass({
-                id: nextId(),
-                title: toSpaceCase(previewFilter),
-                createTime: Date.now(),
-                description: previewFilter,
-                match: [{ type: "includes", value: previewFilter }],
-                suppressUntil: 0,
-                type: "error",
-            });
-            controller.resetAll();
-        };
-        return (
-            <div class={css.pad2(10).vbox(10).fillWidth}>
-                {specificId &&
-                    <div class={css.hbox(4).fontSize(20)}>
-                        Viewing single classifer,
-                        <Button flavor="large" onClick={() => logClassIdURL.value = ""}>
-                            Reset to all classifiers
-                        </Button>
-                    </div>
-                }
-                <div class={css.hbox(4)}>
-                    <InputLabel
-                        label="Create Classifier"
-                        value={this.state.add}
-                        hot
-                        onChangeValue={(value) => (this.state.add = value)}
-                        onKeyDown={(e) => e.code === "Enter" && createClassifier()}
-                    />
-                    {this.state.add && (
-                        <Button class={css.hbox(6)} onClick={() => createClassifier()}>
-                            <span>Create Classifier</span>
-                        </Button>
-                    )}
-                </div>
-                <Table
-                    columns={{
-                        title: { formatter: editFormatter },
-                        match: { formatter: editFormatter },
-                        priority: { formatter: editFormatter },
-                        suppressUntil: { formatter: editFormatter },
-                        maxExamples: { formatter: editFormatter },
-                        type: { formatter: editFormatter },
-                        createTime: { formatter: editFormatter },
-                        description: { formatter: editFormatter },
-                        ["actions" as any]: {
-                            formatter: (value: unknown, context: FormatContext) => {
-                                return (
-                                    <Button class={errorMessage} onClick={async () => {
-                                        let updateController = LogViewerController.nodes[getBrowserUrlNode()];
-                                        await updateController.updateClass({ id: context?.row?.id as any, action: "delete" });
-                                        controller.resetAll();
-                                    }}>
-                                        Delete
-                                    </Button>
-                                );
-                            }
-                        } as any
-                    }}
-                    rows={classes || []}
-                />
-            </div>
-        );
-    }
-}
-
-export class SuppressUntil extends qreact.Component<{
-    type: LogType;
-    time: number;
-    updateClass: (newFields: Partial<LogClass>) => void;
-}> {
-    state = {
-        editting: false
-    };
-    render() {
-        let { time, type, updateClass } = this.props;
-        let label = "";
-        let now = Date.now();
-        time = time || 0;
-        if (!time) {
-            label = "Will fix in";
-        } else if (time < now) {
-            label = `Should have been fixed ${formatTime(now - time)} AGO`;
-        } else {
-            label = `Will fix within ${formatTime(time - now)}`;
-        }
-
-        let changeLabel = "";
-        let changeTime = 0;
-        if (type === "fatal" || type === "error") {
-            changeLabel = "a day";
-            changeTime = now + timeInDay;
-        } else if (type === "warn" || type === "info") {
-            changeLabel = "a week";
-            changeTime = now + timeInDay * 7;
-        }
-
-        return (
-            <div class={css.hbox(8)}>
-                <div
-                    class={css
-                        .hbox(4)
-                        .pad2(4, 2)
-                        .wrap
-                        .button
-                        .hsla(0, 0, 100, 0.3)
-                        .background("hsla(0, 0%, 100%, 0.6)", "hover")
-                    }
-                    onClick={(e) => {
-                        // If not clicking on input
-                        if ((e.target as HTMLElement).tagName === "INPUT") return;
-                        this.state.editting = !this.state.editting;
-                    }}
-                >
-                    <span>{label}</span>
-                    {this.state.editting && (
-                        <>
-                            <Button onClick={() => updateClass({ suppressUntil: changeTime })}>{changeLabel}</Button>
-                            <InputLabel
-                                class={css.hsl(0, 0, 100).hslcolor(0, 0, 7)}
-                                value={time}
-                                isDatetime
-                                onChangeValue={(newValue) => {
-                                    updateClass({ suppressUntil: Number(newValue) });
-                                    this.state.editting = false;
-                                }}
-                            />
-                        </>
-                    )}
-                </div>
-                {type !== "info" && <Button
-                    onClick={() => updateClass({ suppressUntil: changeTime })}
-                    class={css.hsl(infoColor.h, infoColor.s, infoColor.l)}
-                >
-                    Fixed
-                </Button>}
-                {type !== "info" && <Button
-                    onClick={() => updateClass({ type: "info" })}
-                    class={css.hsl(infoColor.h, infoColor.s, infoColor.l)}
-                >
-                    Not a Bug
-                </Button>}
-                {type === "info" && <Button
-                    onClick={() => updateClass({ type: "error" })}
-                    class={css.hsl(errorColor.h, errorColor.s, errorColor.l)}
-                >
-                    Is a Bug
-                </Button>}
-            </div>
-        );
-    }
-}

package/src/diagnostics/errorLogs/LogFilterUI.tsx

@@ -1,84 +0,0 @@
-import preact from "preact"; import { qreact } from "../../../src/4-dom/qreact";
-import { getBrowserUrlNode } from "../../../src/-f-node-discovery/NodeDiscovery";
-import { InputLabelURL } from "../../../src/library-components/InputLabel";
-import { URLParam } from "../../../src/library-components/URLParam";
-import { getPathFromStr, getPathStr } from "../../../src/path";
-import { css } from "typesafecss";
-import { InputPicker } from "../../../src/library-components/InputPicker";
-import { LogViewerSynced } from "./LogViewer";
-import { showOldestLogsFirstURL, showLiveLogsURL, filterToClassesURL, filterMachineIdsURL, filterThreadIdsURL, getPathArrayLax } from "./logFiltering";
-
-export class LogFilterUI extends qreact.Component<{
-    allMachineIds: Set<string> | null;
-    allThreadIds: Set<string> | null;
-}> {
-    render() {
-        let classList = LogViewerSynced(getBrowserUrlNode()).getClasses();
-        function strPicker(
-            label: preact.ComponentChild,
-            url: URLParam<string>,
-            options: {
-                value: string;
-                matchText?: string;
-                label?: preact.ComponentChild;
-            }[]
-        ) {
-            return <InputPicker
-                label={label}
-                allowNonOptions
-                picked={getPathArrayLax(url.value)}
-                options={options}
-                addPicked={(value) => {
-                    let picked = getPathArrayLax(url.value);
-                    picked.push(value);
-                    url.value = getPathStr(picked);
-                }}
-                removePicked={(value) => {
-                    let picked = getPathArrayLax(url.value);
-                    let index = picked.indexOf(value);
-                    if (index !== -1) {
-                        picked.splice(index, 1);
-                        if (picked.length === 0) {
-                            url.value = "";
-                        } else {
-                            url.value = getPathStr(picked);
-                        }
-                    }
-                }}
-            />;
-        }
-        return (
-            <div class={css.vbox(10).flexExpand.wrap.alignItems("start")}>
-                <div class={css.hbox(10).alignItems("start")}>
-                    <InputLabelURL
-                        label="Oldest Logs First"
-                        checkbox
-                        url={showOldestLogsFirstURL}
-                    />
-                    <InputLabelURL
-                        label="Show Live Logs"
-                        checkbox
-                        url={showLiveLogsURL}
-                    />
-                </div>
-                <div class={css.hbox(10).alignItems("start")}>
-                    {strPicker(
-                        "Filter Classes",
-                        filterToClassesURL,
-                        classList?.map(x => ({ value: x.id, label: `${x.title} (${x.type})` })) || []
-                    )}
-                    {strPicker(
-                        "Filter Machines",
-                        filterMachineIdsURL,
-                        Array.from(this.props.allMachineIds || []).map(x => ({ value: x })) || []
-                    )}
-                    {strPicker(
-                        "Filter Threads",
-                        filterThreadIdsURL,
-                        Array.from(this.props.allThreadIds || []).map(x => ({ value: x })) || []
-                    )}
-                </div>
-            </div>
-        );
-    }
-}