pi-roundtable-line 0.0.0-stage → 0.9.13

package/src/index.ts

@@ -0,0 +1,21 @@
+export { errorCode, LineApiError, LineError } from "./errors.ts";
+export type { ParsedEvent, SourceKind } from "./events.ts";
+export { parseSourceKey, sourceKeyOf } from "./events.ts";
+export type {
+	InboxEvent,
+	InboxOptions,
+	InboxProcessor,
+	InboxResult,
+} from "./inbox.ts";
+export type { LineOptions } from "./options.ts";
+export { line } from "./plugin.ts";
+export type { LineBridgePort } from "./port.ts";
+export { LINE } from "./port.ts";
+export type {
+	LineSource,
+	LineSourceSummary,
+	LineSources,
+	SourceStatus,
+} from "./sources.ts";
+export type { LineCredentials, TokenMode } from "./token.ts";
+export { WEBHOOK_MAX_BYTES } from "./webhook.ts";

package/src/intake.ts

@@ -0,0 +1,119 @@
+import type { SQL } from "bun";
+import { type ParsedEvent, parseSourceKey } from "./events.ts";
+import type { SourceStatus } from "./sources.ts";
+
+/** What persisting one request did, in counts. */
+export interface IntakeResult {
+	/** Events kept for the inbox worker. */
+	queued: number;
+	/** Events dropped unread because their chat is not allowed (or they name none). */
+	skipped: number;
+	/** Events an earlier delivery had already stored. */
+	duplicates: number;
+}
+
+/**
+ * Events that carry no customer content, only the fact that the bot came or went. They are kept for
+ * a chat that is not allowed too: a join is answered with a notice whatever the allowlist says, and
+ * a leave keeps the chat's status true.
+ */
+const MEMBERSHIP = new Set(["join", "leave"]);
+
+/**
+ * PostgreSQL refuses U+0000 in text and in JSONB, and a lone surrogate in JSONB; either would fail
+ * the whole request's transaction and with it every other event stored beside this one. They are
+ * replaced with U+FFFD, which loses nothing a chat mirror could use.
+ */
+function clean(text: string): string {
+	return text.replaceAll("\u0000", "\uFFFD").toWellFormed();
+}
+
+function scrub(value: unknown): unknown {
+	if (typeof value === "string") return clean(value);
+	if (Array.isArray(value)) return value.map(scrub);
+	if (typeof value === "object" && value !== null)
+		return Object.fromEntries(
+			Object.entries(value).map(([key, item]) => [clean(key), scrub(item)]),
+		);
+	return value;
+}
+
+/** Why an event is dropped, or undefined when it is kept. */
+function dropReason(
+	event: ParsedEvent,
+	status: SourceStatus | undefined,
+): string | undefined {
+	if (!event.active) return "standby";
+	if (status === undefined) return "no_source";
+	if (status === "allowed") return undefined;
+	if (status !== "blocked" && MEMBERSHIP.has(event.type)) return undefined;
+	return status === "blocked" ? "source_blocked" : "source_not_allowed";
+}
+
+/**
+ * Stores a request's events in one transaction, so the caller may answer `200` once this returns.
+ * An unknown chat is recorded as pending with counts only; the content of an event of a chat that
+ * is not allowed is never written. An event already stored (same `webhookEventId`) is left as it
+ * is, so a redelivery is processed once.
+ */
+export async function persistEvents(
+	sql: SQL,
+	events: readonly ParsedEvent[],
+	now: Date,
+): Promise<IntakeResult> {
+	const result: IntakeResult = { queued: 0, skipped: 0, duplicates: 0 };
+	// Sorted, so two requests that share chats take their locks in one order and cannot deadlock.
+	const keys = [
+		...new Set(events.flatMap((e) => (e.sourceKey ? [e.sourceKey] : []))),
+	].sort();
+	await sql.begin(async (tx) => {
+		const statuses = new Map<string, SourceStatus>();
+		for (const key of keys) {
+			const source = parseSourceKey(key);
+			if (!source) continue;
+			await tx`INSERT INTO line_sources
+				(source_key, kind, status, first_seen_at, last_seen_at, status_changed_at)
+				VALUES (${key}, ${source.kind}, 'pending', ${now}, ${now}, ${now})
+				ON CONFLICT (source_key) DO NOTHING`;
+		}
+		if (keys.length > 0) {
+			// The row locks last to the end of the request: allow() and block() wait, so each event meets one status.
+			const rows: { source_key: string; status: SourceStatus }[] =
+				await tx`SELECT source_key, status FROM line_sources
+					WHERE source_key IN ${tx(keys)} ORDER BY source_key FOR UPDATE`;
+			for (const row of rows) statuses.set(row.source_key, row.status);
+		}
+		for (const event of events) {
+			const status = event.sourceKey
+				? statuses.get(event.sourceKey)
+				: undefined;
+			const reason = dropReason(event, status);
+			const payload = reason ? null : JSON.stringify(scrub(event.raw));
+			const id = clean(event.id);
+			const type = clean(event.type);
+			const messageId =
+				event.messageId === null ? null : clean(event.messageId);
+			const inserted: { webhook_event_id: string }[] =
+				await tx`INSERT INTO line_inbox
+				(webhook_event_id, source_key, event_type, message_id, event_ts, payload, status,
+				 next_attempt_at, last_error_code, created_at, completed_at)
+				VALUES (${id}, ${event.sourceKey}, ${type}, ${messageId}, ${event.timestamp},
+					${payload}::text::jsonb, ${reason ? "skipped" : "pending"}, ${now}, ${reason ?? null}, ${now},
+					${reason ? now : null})
+				ON CONFLICT (webhook_event_id) DO NOTHING
+				RETURNING webhook_event_id`;
+			if (inserted.length === 0) {
+				result.duplicates++;
+				continue;
+			}
+			if (reason) result.skipped++;
+			else result.queued++;
+			if (event.sourceKey)
+				await tx`UPDATE line_sources
+					SET last_seen_at = ${now}, event_count = event_count + 1,
+						message_count = message_count + ${event.type === "message" ? 1 : 0}
+					WHERE source_key = ${event.sourceKey}`;
+		}
+	});
+	return result;
+}

package/src/migrations.ts

@@ -0,0 +1,45 @@
+import type { Migration } from "pi-roundtable";
+
+/**
+ * The sources and the inbox. Later migrations add the tables of messages, tombstones and the
+ * outbox; this one is never edited once released.
+ */
+export const INBOX_MIGRATION: Migration = {
+	name: "inbox",
+	up: async (sql) => {
+		await sql`CREATE TABLE IF NOT EXISTS line_sources (
+			source_key TEXT PRIMARY KEY,
+			kind TEXT NOT NULL CHECK (kind IN ('group', 'room', 'user')),
+			status TEXT NOT NULL CHECK (status IN ('pending', 'allowed', 'left', 'blocked')),
+			first_seen_at TIMESTAMPTZ NOT NULL,
+			last_seen_at TIMESTAMPTZ NOT NULL,
+			status_changed_at TIMESTAMPTZ NOT NULL,
+			event_count BIGINT NOT NULL DEFAULT 0,
+			message_count BIGINT NOT NULL DEFAULT 0,
+			membership_ts BIGINT NOT NULL DEFAULT 0
+		)`;
+		await sql`CREATE TABLE IF NOT EXISTS line_inbox (
+			seq BIGSERIAL NOT NULL,
+			webhook_event_id TEXT PRIMARY KEY,
+			source_key TEXT,
+			event_type TEXT NOT NULL,
+			message_id TEXT,
+			event_ts BIGINT NOT NULL,
+			payload JSONB,
+			status TEXT NOT NULL CHECK (status IN ('pending', 'processing', 'done', 'skipped', 'dead')),
+			attempts INTEGER NOT NULL DEFAULT 0,
+			requeues INTEGER NOT NULL DEFAULT 0,
+			next_attempt_at TIMESTAMPTZ NOT NULL,
+			claimed_at TIMESTAMPTZ,
+			last_error_code TEXT,
+			created_at TIMESTAMPTZ NOT NULL,
+			completed_at TIMESTAMPTZ
+		)`;
+		await sql`CREATE INDEX IF NOT EXISTS line_inbox_claim
+			ON line_inbox (event_ts, seq) WHERE status = 'pending'`;
+		await sql`CREATE INDEX IF NOT EXISTS line_inbox_processing
+			ON line_inbox (source_key) WHERE status = 'processing'`;
+		await sql`CREATE INDEX IF NOT EXISTS line_inbox_message
+			ON line_inbox (message_id) WHERE message_id IS NOT NULL`;
+	},
+};

package/src/options.ts

@@ -0,0 +1,63 @@
+import type { InboxOptions } from "./inbox.ts";
+import type { LineCredentials } from "./token.ts";
+
+export interface LineOptions {
+	/** The channel id and secret, and how the access token is obtained. */
+	credentials: LineCredentials;
+	/** Where LINE posts events; default `/line/webhook`. Set this URL in the channel's settings. */
+	webhookPath?: string;
+	/** The configured listener the route attaches to; default `public`. */
+	listener?: string;
+	/** Tunes the inbox worker. */
+	inbox?: Partial<InboxOptions>;
+	/** Replaces `fetch` for every call to LINE, for tests. */
+	fetch?: typeof fetch;
+}
+
+const INBOX_KEYS: readonly (keyof InboxOptions)[] = [
+	"leaseMs",
+	"maxHandleMs",
+	"maxAttempts",
+	"maxRequeues",
+	"backoffBaseMs",
+	"backoffCapMs",
+	"concurrency",
+	"pollMs",
+	"messageGraceMs",
+	"dependencyRetryMs",
+];
+
+/** The longest delay `setTimeout` and `setInterval` hold; past it they fire at once. */
+const MAX_TIMER_MS = 2_147_483_647;
+
+/** The options that become a timer's delay. */
+const TIMER_KEYS: readonly (keyof InboxOptions)[] = [
+	"leaseMs",
+	"maxHandleMs",
+	"pollMs",
+];
+
+/** Refuses a mistake in the options, naming the field; the host stops before it starts. */
+export function checkOptions(options: LineOptions): { path: string } {
+	const path = options.webhookPath ?? "/line/webhook";
+	if (!/^\/[A-Za-z0-9._~/-]*[A-Za-z0-9._~-]$/.test(path))
+		throw new Error(
+			`line: webhookPath ${JSON.stringify(path)} must start with / and not end with one, such as /line/webhook`,
+		);
+	for (const [key, value] of Object.entries(options.inbox ?? {})) {
+		if (!INBOX_KEYS.includes(key as keyof InboxOptions))
+			throw new Error(`line: inbox.${key} is not an inbox option`);
+		if (!Number.isInteger(value) || (value as number) < 1)
+			throw new Error(
+				`line: inbox.${key} must be a positive whole number; got ${String(value)}`,
+			);
+		if (
+			TIMER_KEYS.includes(key as keyof InboxOptions) &&
+			(value as number) > MAX_TIMER_MS
+		)
+			throw new Error(
+				`line: inbox.${key} must be at most ${MAX_TIMER_MS} milliseconds (about 24 days); got ${String(value)}`,
+			);
+	}
+	return { path };
+}

package/src/plugin.ts

@@ -0,0 +1,105 @@
+import {
+	definePlugin,
+	type PluginContext,
+	type RoundtablePlugin,
+} from "pi-roundtable";
+import { LineApi } from "./api.ts";
+import { BotIdentity } from "./bot.ts";
+import { type InboxProcessor, InboxWorker } from "./inbox.ts";
+import { INBOX_MIGRATION } from "./migrations.ts";
+import { checkOptions, type LineOptions } from "./options.ts";
+import { LINE, type LineBridgePort } from "./port.ts";
+import { PgSources } from "./sources.ts";
+import { TokenSource } from "./token.ts";
+import { webhookHandler } from "./webhook.ts";
+
+/**
+ * The LINE transport: a webhook on the host's HTTP listener that stores every signed event before
+ * it answers, an inbox worker that hands the stored events of allowed chats to one processor, and
+ * the `LINE` service other plugins read the allowlist and health from. A chat the bot has not been
+ * told to mirror is recorded as pending with counts only; nothing of what its members write is kept.
+ * Configuration mistakes throw here, before the host starts.
+ */
+export function line(options: LineOptions): RoundtablePlugin {
+	const { path } = checkOptions(options);
+	// Built now so bad credentials stop the configuration rather than the boot.
+	const tokens = new TokenSource({
+		credentials: options.credentials,
+		...(options.fetch ? { fetch: options.fetch } : {}),
+	});
+	return definePlugin({
+		name: "line",
+		migrations: [INBOX_MIGRATION],
+		provides: [LINE],
+		setup: (context: PluginContext) => {
+			const sql = context.database();
+			const now = () => context.env.now();
+			const api = new LineApi({
+				tokens,
+				...(options.fetch ? { fetch: options.fetch } : {}),
+			});
+			const bot = new BotIdentity({ api, logger: context.logger, now });
+			const sources = new PgSources(sql, () => api, now);
+			let processor: InboxProcessor | undefined;
+			const worker = new InboxWorker({
+				sql,
+				logger: context.logger,
+				now,
+				sources,
+				processor: () => processor,
+				...(options.inbox ? { options: options.inbox } : {}),
+			});
+			const port: LineBridgePort = {
+				sources,
+				botUserId: () => bot.current(),
+				health: () => {
+					const health = tokens.health();
+					return {
+						tokensIssued: health.issued,
+						tokenFailure: health.lastFailure,
+					};
+				},
+				attachProcessor: (attached) => {
+					if (processor)
+						throw new Error("line: an inbox processor is already attached");
+					processor = attached;
+					worker.kick();
+					return () => {
+						if (processor === attached) processor = undefined;
+					};
+				},
+			};
+			context.services.provide(LINE, port);
+			return {
+				services: [
+					{
+						name: "line-bot",
+						startInBackground: () => bot.refresh(),
+					},
+					{
+						name: "line-inbox",
+						start: () => worker.start(),
+						stop: () => worker.stop(),
+						busy: () => worker.busy(),
+					},
+				],
+				http: [
+					{
+						name: "line-webhook",
+						listener: options.listener ?? "public",
+						path: { exact: path },
+						methods: ["POST"],
+						handle: webhookHandler({
+							sql,
+							tokens,
+							logger: context.logger,
+							now,
+							botUserId: () => bot.current(),
+							stored: () => worker.kick(),
+						}),
+					},
+				],
+			};
+		},
+	});
+}

package/src/port.ts

@@ -0,0 +1,25 @@
+import { serviceKey } from "pi-roundtable";
+import type { InboxProcessor } from "./inbox.ts";
+import type { LineSources } from "./sources.ts";
+
+/** What other plugins read from this one, through the `LINE` service key. */
+export interface LineBridgePort {
+	/** The chats the bot has seen, and the allowlist over them. */
+	readonly sources: LineSources;
+	/** The bot's user id once LINE has told it, else undefined. */
+	botUserId(): string | undefined;
+	/** The token source's health: how many tokens were issued and why the last attempt failed, if it did. */
+	health(): { tokensIssued: number; tokenFailure: string | undefined };
+	/**
+	 * Takes the inbox: from now on each stored event of an allowed chat (and the join and leave of
+	 * any chat) is handed to `processor`, oldest first within a chat. Only one processor may be
+	 * attached; returns the function that detaches it. Until one is attached events wait in the
+	 * inbox, payload and all, until retention removes them.
+	 */
+	attachProcessor(processor: InboxProcessor): () => void;
+}
+
+export const LINE = serviceKey<LineBridgePort>("roundtable.line", {
+	absent:
+		"add the line() plugin from pi-roundtable-line to the host's plugins before the plugin that reads it",
+});

package/src/sources.ts

@@ -0,0 +1,186 @@
+import type { SQL } from "bun";
+import type { LineApi } from "./api.ts";
+import { LineError } from "./errors.ts";
+import { parseSourceKey, type SourceKind } from "./events.ts";
+
+export type SourceStatus = "pending" | "allowed" | "left" | "blocked";
+
+/** A chat the bot has seen. Only counts and times are kept for a chat that is not allowed. */
+export interface LineSource {
+	key: string;
+	kind: SourceKind;
+	status: SourceStatus;
+	firstSeenAt: string;
+	lastSeenAt: string;
+	statusChangedAt: string;
+	eventCount: number;
+	messageCount: number;
+}
+
+export interface LineSourceSummary extends LineSource {
+	/** The group's name as LINE shows it; null for a room or a person, which have none. */
+	name: string | null;
+}
+
+interface Row {
+	source_key: string;
+	kind: SourceKind;
+	status: SourceStatus;
+	first_seen_at: Date;
+	last_seen_at: Date;
+	status_changed_at: Date;
+	event_count: string | number;
+	message_count: string | number;
+}
+
+const sourceOf = (row: Row): LineSource => ({
+	key: row.source_key,
+	kind: row.kind,
+	status: row.status,
+	firstSeenAt: row.first_seen_at.toISOString(),
+	lastSeenAt: row.last_seen_at.toISOString(),
+	statusChangedAt: row.status_changed_at.toISOString(),
+	eventCount: Number(row.event_count),
+	messageCount: Number(row.message_count),
+});
+
+/** What the host does with the chats the bot has seen: list them, allow or block them. It never creates a channel. */
+export interface LineSources {
+	/** Chats waiting for a decision, oldest first. */
+	pending(): Promise<LineSource[]>;
+	/** Every chat, or only those with `status`, oldest first. */
+	list(status?: SourceStatus): Promise<LineSource[]>;
+	/** One chat, or undefined when the bot has not seen it. */
+	get(key: string): Promise<LineSource | undefined>;
+	/**
+	 * Mirrors a chat from now on. Events that arrived before are gone: they were dropped unread.
+	 * Throws `LineError` `unknown_source` for a chat never seen, and `source_left` for one the
+	 * bot has left.
+	 */
+	allow(key: string): Promise<LineSource>;
+	/** Stops mirroring a chat and drops the events of it still waiting in the inbox. Throws `unknown_source` for a chat never seen. */
+	block(key: string): Promise<LineSource>;
+	/** The chat with its group name, asked of LINE; throws when LINE refuses or does not answer. */
+	summary(key: string): Promise<LineSourceSummary>;
+}
+
+function checkedKey(key: string): { kind: SourceKind; id: string } {
+	const parsed = parseSourceKey(key);
+	if (!parsed)
+		throw new LineError(
+			"bad_source_key",
+			"line: a source key is line:<group|room|user>:<id>",
+		);
+	return parsed;
+}
+
+export class PgSources implements LineSources {
+	readonly #sql: SQL;
+	readonly #api: () => LineApi;
+	readonly #now: () => Date;
+
+	constructor(sql: SQL, api: () => LineApi, now: () => Date) {
+		this.#sql = sql;
+		this.#api = api;
+		this.#now = now;
+	}
+
+	async pending(): Promise<LineSource[]> {
+		return this.list("pending");
+	}
+
+	async list(status?: SourceStatus): Promise<LineSource[]> {
+		const rows: Row[] = status
+			? await this
+					.#sql`SELECT * FROM line_sources WHERE status = ${status} ORDER BY first_seen_at, source_key`
+			: await this
+					.#sql`SELECT * FROM line_sources ORDER BY first_seen_at, source_key`;
+		return rows.map(sourceOf);
+	}
+
+	async get(key: string): Promise<LineSource | undefined> {
+		checkedKey(key);
+		const rows: Row[] = await this
+			.#sql`SELECT * FROM line_sources WHERE source_key = ${key}`;
+		return rows[0] ? sourceOf(rows[0]) : undefined;
+	}
+
+	async allow(key: string): Promise<LineSource> {
+		checkedKey(key);
+		const now = this.#now();
+		const rows: Row[] = await this.#sql`UPDATE line_sources
+			SET status = 'allowed', status_changed_at = ${now}
+			WHERE source_key = ${key} AND status IN ('pending', 'blocked', 'allowed')
+			RETURNING *`;
+		if (rows[0]) return sourceOf(rows[0]);
+		throw (await this.get(key))
+			? new LineError("source_left", "line: the bot has left that chat")
+			: new LineError("unknown_source", "line: the bot has not seen that chat");
+	}
+
+	async block(key: string): Promise<LineSource> {
+		checkedKey(key);
+		const now = this.#now();
+		return this.#sql.begin(async (sql) => {
+			const rows: Row[] = await sql`UPDATE line_sources
+				SET status = 'blocked', status_changed_at = ${now}
+				WHERE source_key = ${key} RETURNING *`;
+			if (!rows[0])
+				throw new LineError(
+					"unknown_source",
+					"line: the bot has not seen that chat",
+				);
+			// What waits unread is dropped with its payload; an event being handled finishes.
+			await sql`UPDATE line_inbox
+				SET status = 'skipped', payload = NULL, last_error_code = 'source_blocked', completed_at = ${now}
+				WHERE source_key = ${key} AND status = 'pending'`;
+			return sourceOf(rows[0]);
+		});
+	}
+
+	async summary(key: string): Promise<LineSourceSummary> {
+		const { kind, id } = checkedKey(key);
+		const source = await this.get(key);
+		if (!source)
+			throw new LineError(
+				"unknown_source",
+				"line: the bot has not seen that chat",
+			);
+		const name = kind === "group" ? await this.#api().groupName(id) : null;
+		return { ...source, name };
+	}
+
+	/**
+	 * Records that the bot is in a chat (a `join` event): a chat it had left needs a new decision,
+	 * so it goes back to pending. Ignores an event older than the last membership change applied.
+	 */
+	async markJoined(key: string, timestamp: number): Promise<void> {
+		await this.#sql`UPDATE line_sources
+			SET membership_ts = ${timestamp},
+				status = CASE WHEN status = 'left' THEN 'pending' ELSE status END,
+				status_changed_at = CASE WHEN status = 'left' THEN ${this.#now()} ELSE status_changed_at END
+			WHERE source_key = ${key} AND membership_ts < ${timestamp}`;
+	}
+
+	/**
+	 * Records that the bot was removed from a chat (a `leave` event); a blocked chat stays blocked.
+	 * Whatever else the chat has waiting (an event backing off, say) is dropped with its payload,
+	 * apart from its join and leave and its unsends (which carry nothing and only take content down),
+	 * so it cannot be handed out long after, when the bot is back.
+	 */
+	async markLeft(key: string, timestamp: number): Promise<void> {
+		const now = this.#now();
+		await this.#sql.begin(async (sql) => {
+			const rows: { status: SourceStatus }[] = await sql`UPDATE line_sources
+				SET membership_ts = ${timestamp},
+					status = CASE WHEN status = 'blocked' THEN status ELSE 'left' END,
+					status_changed_at = CASE WHEN status = 'blocked' THEN status_changed_at ELSE ${now} END
+				WHERE source_key = ${key} AND membership_ts < ${timestamp}
+				RETURNING status`;
+			if (rows[0]?.status !== "left") return;
+			await sql`UPDATE line_inbox
+				SET status = 'skipped', payload = NULL, last_error_code = 'source_left', completed_at = ${now}
+				WHERE source_key = ${key} AND status = 'pending' AND event_type NOT IN ('join', 'leave', 'unsend')`;
+		});
+	}
+}