pi-roundtable 0.1.0

package/src/core/ops/error-reporter.test.ts

@@ -0,0 +1,283 @@
+import { describe, expect, test } from "bun:test";
+import pino from "pino";
+import type { ScheduledOutcome } from "../contract/channels.ts";
+import { createLogger, type LogEntry } from "../log.ts";
+import type { ChannelKey } from "../sessions.ts";
+import { ErrorReporter, fingerprint, reportText } from "./error-reporter.ts";
+
+const HOUR = 3_600_000;
+
+/** The ops agent as the delivery sees it: active with a channel, or not. */
+interface OpsAgent {
+	status: "active" | "archived";
+	channelId?: string;
+}
+
+const infra: OpsAgent = { status: "active", channelId: "42" };
+
+function stack(file: string, line: number): string {
+	return `Error: boom\n    at run (/opt/app/${file}:${line}:7)\n    at next (/opt/app/node_modules/x/index.js:1:1)`;
+}
+
+function entry(msg: string, fields: LogEntry = {}): LogEntry {
+	return {
+		level: 50,
+		time: Date.now(),
+		pid: 1,
+		hostname: "h",
+		app: "roundtable",
+		msg,
+		...fields,
+	};
+}
+
+function setup(options: { agent?: OpsAgent; turn?: () => Promise<void> } = {}) {
+	let now = 1_000_000;
+	const timers: { run: () => void; at: number }[] = [];
+	const reporter = new ErrorReporter({
+		opsAgent: "infra",
+		app: "Roundtable",
+		now: () => now,
+		setTimer: (run, ms) => timers.push({ run, at: now + ms }),
+		ignored: [/expected noise/],
+	});
+	const posts: { channel: ChannelKey; text: string }[] = [];
+	const turns: string[] = [];
+	const lines: LogEntry[] = [];
+	const logger = pino(
+		{ base: { app: "roundtable" } },
+		{
+			write(line: string) {
+				const logged = JSON.parse(line) as LogEntry;
+				lines.push(logged);
+				reporter.record(logged);
+			},
+		},
+	);
+	const agent = "agent" in options ? options.agent : infra;
+	const delivery = {
+		channelOf: (name: string): ChannelKey | undefined =>
+			name === "infra" && agent?.status === "active" && agent.channelId
+				? `discord:${agent.channelId}`
+				: undefined,
+		post: async (channel: ChannelKey, text: string) => {
+			posts.push({ channel, text });
+		},
+		turn: async (_channel: ChannelKey, text: string) => {
+			turns.push(text);
+			await options.turn?.();
+			return { status: "ran" } as ScheduledOutcome;
+		},
+		logger,
+	};
+	return {
+		reporter,
+		delivery,
+		posts,
+		turns,
+		lines,
+		logger,
+		connect: () => reporter.connect(delivery),
+		advance(ms: number) {
+			now += ms;
+			for (const timer of timers.splice(0))
+				if (timer.at <= now) timer.run();
+				else timers.push(timer);
+		},
+	};
+}
+
+const settle = () => new Promise((resolve) => setTimeout(resolve, 0));
+
+describe("ErrorReporter", () => {
+	test("an error is posted in Infra's channel and starts its report turn", async () => {
+		const { reporter, connect, posts, turns } = setup();
+		connect();
+		reporter.record(
+			entry("webhook post failed", {
+				module: "discord",
+				channel: "discord:7",
+				err: { type: "Error", message: "boom", stack: stack("src/a.ts", 12) },
+			}),
+		);
+		await settle();
+		expect(posts).toHaveLength(1);
+		expect(posts[0]?.channel).toBe("discord:42");
+		const text = posts[0]?.text ?? "";
+		expect(text).toStartWith(
+			"Roundtable logged an error: webhook post failed\n",
+		);
+		expect(text).toContain("app: roundtable · module: discord");
+		expect(text).toContain("channel: discord:7");
+		expect(text).toContain("at run (/opt/app/src/a.ts:12:7)");
+		expect(text).not.toContain("pid");
+		expect(turns).toEqual([text]);
+	});
+
+	test("a long report is trimmed", () => {
+		const text = reportText(
+			entry("big", {
+				err: { message: "x", stack: `Error: ${"y".repeat(5000)}` },
+			}),
+			"Roundtable",
+		);
+		expect(text.length).toBeLessThanOrEqual(1500);
+		expect(text).toEndWith("…");
+	});
+
+	test("the fingerprint ignores times and ids but not where the error was raised", () => {
+		const a = entry("failed", {
+			channel: "discord:1",
+			err: { stack: stack("src/a.ts", 3) },
+		});
+		const b = entry("failed", {
+			channel: "discord:2",
+			time: 5,
+			err: { stack: stack("src/a.ts", 3) },
+		});
+		const c = entry("failed", { err: { stack: stack("src/b.ts", 3) } });
+		expect(fingerprint(a)).toBe(fingerprint(b));
+		expect(fingerprint(a)).not.toBe(fingerprint(c));
+		expect(fingerprint(entry("no stack", { job: 1 }))).toBe(
+			fingerprint(entry("no stack", { job: 2 })),
+		);
+	});
+
+	test("the same error twice in an hour is reported once, and again after it", async () => {
+		const { reporter, connect, posts, advance } = setup();
+		connect();
+		const same = () =>
+			entry("failed", { err: { stack: stack("src/a.ts", 3) } });
+		reporter.record(same());
+		await settle();
+		advance(HOUR - 1);
+		reporter.record(same());
+		await settle();
+		expect(posts).toHaveLength(1);
+		advance(1);
+		reporter.record(same());
+		await settle();
+		expect(posts).toHaveLength(2);
+	});
+
+	test("six distinct errors in an hour make five reports and a later summary", async () => {
+		const { reporter, connect, posts, turns, advance } = setup();
+		connect();
+		for (let i = 0; i < 6; i++) {
+			reporter.record(entry(`failure ${i}`));
+			advance(60_000);
+		}
+		reporter.record(entry("failure 5"));
+		await settle();
+		expect(posts).toHaveLength(5);
+		advance(HOUR);
+		await settle();
+		expect(posts).toHaveLength(6);
+		expect(posts[5]?.text).toBe(
+			"Roundtable error reports: 2 more errors suppressed: failure 5 (×2)",
+		);
+		expect(turns).toHaveLength(6);
+	});
+
+	test("errors logged while Infra runs an error-report turn are summed up after it", async () => {
+		let finish = () => {};
+		const running = new Promise<void>((resolve) => {
+			finish = resolve;
+		});
+		const { reporter, connect, posts, logger } = setup({ turn: () => running });
+		connect();
+		reporter.record(entry("first"));
+		await settle();
+		reporter.record(entry("during the turn"));
+		logger.error("also during the turn");
+		await settle();
+		expect(posts).toHaveLength(1);
+		finish();
+		await settle();
+		await settle();
+		expect(posts).toHaveLength(2);
+		expect(posts[1]?.text).toStartWith(
+			"Roundtable error reports: 2 more errors suppressed:",
+		);
+		expect(posts[1]?.text).toContain("during the turn");
+		expect(posts[1]?.text).toContain("also during the turn");
+		await settle();
+		reporter.record(entry("after the turn"));
+		await settle();
+		expect(posts).toHaveLength(3);
+	});
+
+	test("the reporter's own failures are only logged", async () => {
+		const { reporter, delivery, lines, turns } = setup();
+		delivery.post = async () => {
+			throw new Error("discord down");
+		};
+		delivery.turn = async () => {
+			throw new Error("turn broke");
+		};
+		reporter.connect(delivery);
+		reporter.record(entry("first"));
+		await settle();
+		await settle();
+		expect(lines.map((l) => l.msg)).toEqual([
+			"error report not posted",
+			"error report turn failed",
+		]);
+		expect(turns).toHaveLength(0);
+	});
+
+	test("nothing is reported when Infra is missing or archived", async () => {
+		for (const agent of [
+			undefined,
+			{ ...infra, status: "archived" as const },
+		]) {
+			const { reporter, connect, posts, turns } = setup({ agent });
+			connect();
+			reporter.record(entry("failed"));
+			await settle();
+			expect(posts).toHaveLength(0);
+			expect(turns).toHaveLength(0);
+		}
+	});
+
+	test("warnings and ignored errors are not reported", async () => {
+		const { reporter, connect, posts } = setup();
+		connect();
+		reporter.record({ ...entry("just a warning"), level: 40 });
+		reporter.record(entry("some expected noise here"));
+		await settle();
+		expect(posts).toHaveLength(0);
+	});
+
+	test("errors before the agent server is ready wait, the last twenty kept", async () => {
+		const { reporter, connect, posts, lines } = setup();
+		for (let i = 0; i < 23; i++) reporter.record(entry(`early ${i}`));
+		expect(posts).toHaveLength(0);
+		connect();
+		await settle();
+		expect(posts.map((p) => p.text.split("\n")[0])).toEqual(
+			[3, 4, 5, 6, 7].map((i) => `Roundtable logged an error: early ${i}`),
+		);
+		expect(lines[0]).toMatchObject({
+			level: 40,
+			dropped: 3,
+			msg: "early errors dropped before reporting started",
+		});
+	});
+});
+
+describe("createLogger", () => {
+	test("hands error and fatal entries to the sink, not warnings", () => {
+		const taken: LogEntry[] = [];
+		const logger = createLogger("roundtable", (logged) => taken.push(logged));
+		logger.level = "error";
+		logger.warn("a warning");
+		logger.error({ job: 1 }, "an error");
+		logger.fatal("fatal");
+		expect(taken.map((t) => [t.level, t.msg, t.app])).toEqual([
+			[50, "an error", "roundtable"],
+			[60, "fatal", "roundtable"],
+		]);
+		expect(taken[0]?.job).toBe(1);
+	});
+});

package/src/core/ops/error-reporter.ts

@@ -0,0 +1,271 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import type { ScheduledOutcome } from "../contract/channels.ts";
+import type { LogEntry, Logger } from "../log.ts";
+import type { ChannelKey } from "../sessions.ts";
+
+/** Errors that are expected and never reported, matched against the message and the error's. */
+export const IGNORED_ERRORS: readonly RegExp[] = [];
+
+const HOUR_MS = 3_600_000;
+/** Reports, summaries included, per rolling hour. */
+const REPORTS_PER_HOUR = 5;
+/** Errors kept from before the agent server is ready; older ones are dropped and counted. */
+const EARLY_BUFFER = 20;
+const REPORT_LIMIT = 1500;
+const FIELD_LIMIT = 200;
+const STACK_FRAMES = 8;
+/** pino's own keys, and the ones the report shows apart from the context. */
+const OWN_KEYS = new Set(["level", "time", "pid", "hostname", "msg", "err"]);
+const HEAD_KEYS = ["app", "module"];
+const APP_FRAME = /((?:src|shared|worker)\/[^\s():]+\.ts):(\d+)/;
+
+/** Where the reports go, known once the Discord surface and the agent team are up. */
+export interface ErrorReportDelivery {
+	/** The agent's channel while it is active; undefined for an archived or unknown agent. */
+	channelOf(agent: string): ChannelKey | undefined;
+	/** The visible message in the agent's channel. */
+	post(channel: ChannelKey, text: string): Promise<void>;
+	/** The agent's report turn; resolves when it has run. */
+	turn(channel: ChannelKey, text: string): Promise<ScheduledOutcome>;
+	logger: Logger;
+}
+
+export interface ErrorReporterOptions {
+	/** The agent that investigates the errors, reported in its channel. */
+	opsAgent: string;
+	/** The name the reports give the process, such as "Roundtable". */
+	app: string;
+	ignored?: readonly RegExp[];
+	/** Replaceable in tests. */
+	now?: () => number;
+	/** Replaceable in tests. */
+	setTimer?: (run: () => void, ms: number) => void;
+}
+
+/**
+ * The process's own `error` and `fatal` log entries, each reported to the ops agent as a visible
+ * message and a report turn in its channel. The same error goes at most once an hour, and at
+ * most five reports an hour overall; the rest are counted and summed up in one report when the
+ * hour allows. Errors from the reporting itself are only logged; errors logged elsewhere while
+ * an error-report turn runs are counted and summed up once it ends.
+ */
+export class ErrorReporter {
+	readonly #opsAgent: string;
+	readonly #app: string;
+	readonly #ignored: readonly RegExp[];
+	readonly #now: () => number;
+	readonly #setTimer: (run: () => void, ms: number) => void;
+	/** Set inside the reporting path: posting, the turn, and the reporter's own logging. */
+	readonly #reporting = new AsyncLocalStorage<true>();
+	#delivery: ErrorReportDelivery | undefined;
+	#early: LogEntry[] = [];
+	#earlyDropped = 0;
+	/** Report turns started and not yet finished. */
+	#turns = 0;
+	#sent: number[] = [];
+	readonly #lastSent = new Map<string, number>();
+	readonly #suppressed = new Map<string, number>();
+	#summaryDue = false;
+
+	constructor(options: ErrorReporterOptions) {
+		this.#opsAgent = options.opsAgent;
+		this.#app = options.app;
+		this.#ignored = options.ignored ?? IGNORED_ERRORS;
+		this.#now = options.now ?? Date.now;
+		this.#setTimer =
+			options.setTimer ??
+			((run, ms) => {
+				setTimeout(run, ms).unref();
+			});
+	}
+
+	/** Takes one log entry; the logger calls it for every `error` and `fatal` line. */
+	record(entry: LogEntry): void {
+		if (typeof entry.level !== "number" || entry.level < 50) return;
+		if (this.#reporting.getStore()) return;
+		if (this.#isIgnored(entry)) return;
+		if (this.#turns > 0) {
+			// The ops agent is already looking at an error; these wait for the summary after its turn.
+			const key = fingerprint(entry);
+			this.#suppressed.set(key, (this.#suppressed.get(key) ?? 0) + 1);
+			return;
+		}
+		if (!this.#delivery) {
+			this.#early.push(entry);
+			if (this.#early.length > EARLY_BUFFER) {
+				this.#early.shift();
+				this.#earlyDropped += 1;
+			}
+			return;
+		}
+		this.#consider(entry);
+	}
+
+	/** Starts reporting, the errors kept from startup first; later calls do nothing. */
+	connect(delivery: ErrorReportDelivery): void {
+		if (this.#delivery) return;
+		this.#delivery = delivery;
+		const early = this.#early;
+		this.#early = [];
+		if (this.#earlyDropped > 0)
+			this.#reporting.run(true, () =>
+				delivery.logger.warn(
+					{ dropped: this.#earlyDropped },
+					"early errors dropped before reporting started",
+				),
+			);
+		this.#earlyDropped = 0;
+		for (const entry of early) this.#consider(entry);
+	}
+
+	#isIgnored(entry: LogEntry): boolean {
+		const text = `${String(entry.msg ?? "")}\n${errorOf(entry)?.message ?? ""}`;
+		return this.#ignored.some((pattern) => pattern.test(text));
+	}
+
+	#consider(entry: LogEntry): void {
+		const now = this.#now();
+		const key = fingerprint(entry);
+		const last = this.#lastSent.get(key);
+		if (last !== undefined && now - last < HOUR_MS) return;
+		if (!this.#hasRoom(now)) {
+			this.#suppressed.set(key, (this.#suppressed.get(key) ?? 0) + 1);
+			this.#scheduleSummary(now);
+			return;
+		}
+		this.#lastSent.set(key, now);
+		this.#sent.push(now);
+		void this.#deliver(reportText(entry, this.#app));
+	}
+
+	#hasRoom(now: number): boolean {
+		this.#sent = this.#sent.filter((at) => now - at < HOUR_MS);
+		return this.#sent.length < REPORTS_PER_HOUR;
+	}
+
+	#scheduleSummary(now: number): void {
+		if (this.#summaryDue) return;
+		this.#summaryDue = true;
+		const oldest = this.#sent[0] ?? now;
+		this.#setTimer(() => this.#summarize(), oldest + HOUR_MS - now);
+	}
+
+	#summarize(): void {
+		this.#summaryDue = false;
+		const now = this.#now();
+		if (this.#suppressed.size === 0) return;
+		if (!this.#hasRoom(now)) {
+			this.#scheduleSummary(now);
+			return;
+		}
+		const counts = [...this.#suppressed];
+		this.#suppressed.clear();
+		const total = counts.reduce((sum, [, n]) => sum + n, 0);
+		for (const [key] of counts) this.#lastSent.set(key, now);
+		this.#sent.push(now);
+		const list = counts
+			.map(([key, n]) => (n > 1 ? `${key} (×${n})` : key))
+			.join("; ");
+		void this.#deliver(
+			trim(
+				`${this.#app} error reports: ${total} more errors suppressed: ${list}`,
+			),
+		);
+	}
+
+	#deliver(text: string): Promise<void> {
+		const delivery = this.#delivery;
+		if (!delivery) return Promise.resolve();
+		return this.#reporting.run(true, async () => {
+			const channel = delivery.channelOf(this.#opsAgent);
+			if (!channel) return;
+			try {
+				await delivery.post(channel, text);
+				// pi-lens-ignore: missing-error-propagation — a report never throws back into the logger
+			} catch (error) {
+				delivery.logger.error(
+					{ channel, err: error },
+					"error report not posted",
+				);
+			}
+			this.#turns += 1;
+			try {
+				const outcome = await delivery.turn(channel, text);
+				if (outcome.status !== "ran")
+					delivery.logger.warn(
+						{ channel, outcome },
+						"error report not answered",
+					);
+				// pi-lens-ignore: missing-error-propagation — a report never throws back into the logger
+			} catch (error) {
+				delivery.logger.error(
+					{ channel, err: error },
+					"error report turn failed",
+				);
+			} finally {
+				this.#turns -= 1;
+				if (this.#turns === 0 && this.#suppressed.size > 0) this.#summarize();
+			}
+		});
+	}
+}
+
+function errorOf(
+	entry: LogEntry,
+): { type?: string; message?: string; stack?: string } | undefined {
+	const err = entry.err;
+	return err && typeof err === "object"
+		? (err as Record<string, string>)
+		: undefined;
+}
+
+/**
+ * What makes two errors the same: the message and where it was raised, the first frame of
+ * the process's own code, or else the names of the logged fields. Never times or ids.
+ */
+export function fingerprint(entry: LogEntry): string {
+	const frame = errorOf(entry)?.stack?.match(APP_FRAME);
+	const where = frame
+		? `${frame[1]}:${frame[2]}`
+		: // pi-lens-ignore: no-sort-without-comparator — field names, in string order
+			contextKeys(entry).sort().join(",");
+	const msg = String(entry.msg ?? "");
+	return where ? `${msg} @ ${where}` : msg;
+}
+
+function contextKeys(entry: LogEntry): string[] {
+	return Object.keys(entry).filter(
+		(key) => !OWN_KEYS.has(key) && !HEAD_KEYS.includes(key),
+	);
+}
+
+/** The report: the message, app and module, the logged fields, and a trimmed stack. */
+export function reportText(entry: LogEntry, app: string): string {
+	const lines = [`${app} logged an error: ${String(entry.msg ?? "")}`];
+	const head = HEAD_KEYS.flatMap((key) =>
+		entry[key] === undefined ? [] : [`${key}: ${String(entry[key])}`],
+	);
+	if (head.length > 0) lines.push(head.join(" · "));
+	for (const key of contextKeys(entry))
+		lines.push(`${key}: ${short(entry[key])}`);
+	const err = errorOf(entry);
+	if (err?.stack) {
+		const [first, ...frames] = err.stack.split("\n");
+		lines.push(
+			first ?? "",
+			...frames.slice(0, STACK_FRAMES).map((frame) => frame.trimEnd()),
+		);
+	} else if (err?.message) lines.push(`${err.type ?? "Error"}: ${err.message}`);
+	return trim(lines.join("\n"));
+}
+
+function short(value: unknown): string {
+	const text = typeof value === "string" ? value : JSON.stringify(value);
+	return text.length > FIELD_LIMIT ? `${text.slice(0, FIELD_LIMIT)}…` : text;
+}
+
+function trim(text: string): string {
+	return text.length > REPORT_LIMIT
+		? `${text.slice(0, REPORT_LIMIT - 1)}…`
+		: text;
+}