pi-roundtable 0.1.0

package/src/core/i18n/schedules.ts

@@ -0,0 +1,92 @@
+import type { CatalogContext } from "./types.ts";
+
+const WEEKDAYS_ZH: Record<string, string> = {
+	sun: "日",
+	mon: "一",
+	tue: "二",
+	wed: "三",
+	thu: "四",
+	fri: "五",
+	sat: "六",
+};
+
+const WEEKDAYS_EN: Record<string, string> = {
+	sun: "Sun",
+	mon: "Mon",
+	tue: "Tue",
+	wed: "Wed",
+	thu: "Thu",
+	fri: "Fri",
+	sat: "Sat",
+};
+
+/** The schedule panel, the schedule slash commands, and how a recurrence reads. */
+export function schedulesEn(ctx: CatalogContext) {
+	return {
+		scheduleWeekday: (day: string) => WEEKDAYS_EN[day] ?? day,
+		scheduleOnce: (date: string, time: string) => `Once on ${date} at ${time}`,
+		scheduleDaily: (time: string) => `Daily at ${time}`,
+		scheduleEveryDays: (days: number, time: string, startDate: string) =>
+			`Every ${days} days at ${time} (counting from ${startDate})`,
+		scheduleWeekly: (days: readonly string[], time: string) =>
+			`Weekly on ${days.join(", ")} at ${time}`,
+		scheduleHead: (id: number, title: string, channel: string, mode: string) =>
+			`**#${id} ${title}** · ${channel} · ${mode}`,
+		scheduleModeOwner: ctx.assistant,
+		scheduleModeParty: "party role",
+		scheduleFooter: `Schedules are created and edited by the bot in conversation; here you can review and cancel them. \`/${ctx.root} schedule cancel\` cancels one.`,
+		scheduleGroupDescription: "Schedules",
+		scheduleListDescription: "List the schedules of every channel",
+		scheduleCancelDescription: "Cancel a schedule",
+		scheduleIdDescription: "The schedule to cancel",
+		scheduleLastRun: (unix: number, status: string) =>
+			`; last run <t:${unix}:R> (${status})`,
+		scheduleNextRun: (recurrence: string, unix: number) =>
+			`${recurrence}; next run <t:${unix}:f>`,
+		scheduleSetBy: (name: string, lastRun: string) =>
+			`Set by ${name}${lastRun}`,
+		scheduleChoice: (id: number, title: string, recurrence: string) =>
+			`#${id} ${title} (${recurrence})`,
+		scheduleTitle: "Schedules",
+		scheduleNone: "No schedules right now.",
+		schedulePickOne: "Pick a schedule from the list.",
+		scheduleUnknown: (id: string) => `There is no schedule #${id}.`,
+		scheduleCancelled: "Schedule cancelled",
+	};
+}
+
+export function schedulesZhTW(
+	ctx: CatalogContext,
+): ReturnType<typeof schedulesEn> {
+	return {
+		scheduleWeekday: (day: string) => WEEKDAYS_ZH[day] ?? day,
+		scheduleOnce: (date: string, time: string) => `單次 ${date} ${time}`,
+		scheduleDaily: (time: string) => `每天 ${time}`,
+		scheduleEveryDays: (days: number, time: string, startDate: string) =>
+			`每 ${days} 天 ${time}（從 ${startDate} 起算）`,
+		scheduleWeekly: (days: readonly string[], time: string) =>
+			`每週${days.join("、")} ${time}`,
+		scheduleHead: (id: number, title: string, channel: string, mode: string) =>
+			`**#${id} ${title}**　${channel}　${mode}`,
+		scheduleModeOwner: ctx.assistant,
+		scheduleModeParty: "party 角色",
+		scheduleFooter: `排程由 bot 在對話中建立與修改；這裡可以總覽與取消。\`/${ctx.root} schedule cancel\` 取消。`,
+		scheduleGroupDescription: "排程",
+		scheduleListDescription: "列出所有頻道的排程",
+		scheduleCancelDescription: "取消一個排程",
+		scheduleIdDescription: "要取消的排程",
+		scheduleLastRun: (unix: number, status: string) =>
+			`；上次 <t:${unix}:R>（${status}）`,
+		scheduleNextRun: (recurrence: string, unix: number) =>
+			`${recurrence}；下次 <t:${unix}:f>`,
+		scheduleSetBy: (name: string, lastRun: string) =>
+			`由 ${name} 設定${lastRun}`,
+		scheduleChoice: (id: number, title: string, recurrence: string) =>
+			`#${id} ${title}（${recurrence}）`,
+		scheduleTitle: "排程",
+		scheduleNone: "目前沒有排程。",
+		schedulePickOne: "請從清單選一個排程。",
+		scheduleUnknown: (id: string) => `沒有 #${id} 這個排程。`,
+		scheduleCancelled: "排程已取消",
+	};
+}

package/src/core/i18n/time-zones.ts

@@ -0,0 +1,27 @@
+/** A zone's place name: the last part of its IANA id, underscores as spaces ("America/New_York" is "New York"). */
+function zoneCity(zone: string): string {
+	return (zone.split("/").at(-1) ?? zone).replaceAll("_", " ");
+}
+
+/** The zone wording of one catalog, from how it names a zone. */
+function zoneWording(place: (zone: string) => string) {
+	const zoneTime = (zone: string) => `${place(zone)} time`;
+	return {
+		zoneName: place,
+		zoneTime,
+		atTimeError: (zone: string) =>
+			`at must be ${zoneTime(zone)} as "YYYY-MM-DD HH:MM"`,
+		repeatingTimeError: (zone: string) =>
+			`give at ("YYYY-MM-DD HH:MM") for one run, or time ("HH:MM", ${place(zone)}) to repeat`,
+	};
+}
+
+/** Time-zone wording shared by prompts and schedule tools; the English catalog names a zone by its IANA id. */
+export function timeZonesEn() {
+	return zoneWording((zone) => zone);
+}
+
+/** The Traditional Chinese catalog names a zone by its city, so no zone has a wording of its own. */
+export function timeZonesZhTW(): ReturnType<typeof timeZonesEn> {
+	return zoneWording(zoneCity);
+}

package/src/core/i18n/types.ts

@@ -0,0 +1,7 @@
+/** What every catalog factory knows about the deployment it speaks for. */
+export interface CatalogContext {
+	/** The assistant's display name, such as "Roundtable". */
+	assistant: string;
+	/** The name of the root slash command, without the slash. */
+	root: string;
+}

package/src/core/i18n/zh-tw.ts

@@ -0,0 +1,26 @@
+import { agentPanelZhTW } from "./agent-panel.ts";
+import { cardsZhTW } from "./cards.ts";
+import { channelsZhTW } from "./channels.ts";
+import { conversationZhTW } from "./conversation.ts";
+import { dashboardZhTW } from "./dashboard.ts";
+import { discordZhTW } from "./discord.ts";
+import type { Messages } from "./en.ts";
+import { ownerZhTW } from "./owner.ts";
+import { schedulesZhTW } from "./schedules.ts";
+import { timeZonesZhTW } from "./time-zones.ts";
+import type { CatalogContext } from "./types.ts";
+
+/** The Traditional Chinese catalog. */
+export function createZhTW(ctx: CatalogContext): Messages {
+	return {
+		...cardsZhTW(ctx),
+		...ownerZhTW(ctx),
+		...schedulesZhTW(ctx),
+		...timeZonesZhTW(),
+		...agentPanelZhTW(ctx),
+		...conversationZhTW(ctx),
+		...dashboardZhTW(ctx),
+		...channelsZhTW(ctx),
+		...discordZhTW(ctx),
+	};
+}

package/src/core/identity.test.ts

@@ -0,0 +1,18 @@
+import { expect, test } from "bun:test";
+import { ownerWords } from "./identity.ts";
+
+test("owner words capitalize the subject and possessive for a sentence's start", () => {
+	expect(
+		ownerWords({
+			name: "Alice",
+			pronouns: { subject: "she", object: "her", possessive: "her" },
+		}),
+	).toEqual({
+		name: "Alice",
+		he: "she",
+		He: "She",
+		him: "her",
+		his: "her",
+		His: "Her",
+	});
+});

package/src/core/identity.ts

@@ -0,0 +1,31 @@
+/** Who the agents work for, as prompts and tool descriptions name them. */
+export interface OwnerIdentity {
+	name: string;
+	/** How prompts refer back to the owner, such as he, him, his. */
+	pronouns: { subject: string; object: string; possessive: string };
+}
+
+const capital = (word: string) => word.charAt(0).toUpperCase() + word.slice(1);
+
+/** The owner's name and pronouns as prompt text uses them, capitalized for a sentence's start. */
+export function ownerWords(owner: OwnerIdentity): OwnerWords {
+	const { subject, object, possessive } = owner.pronouns;
+	return {
+		name: owner.name,
+		he: subject,
+		He: capital(subject),
+		him: object,
+		his: possessive,
+		His: capital(possessive),
+	};
+}
+
+/** The owner's words as `ownerWords` gives them. */
+export interface OwnerWords {
+	name: string;
+	he: string;
+	He: string;
+	him: string;
+	his: string;
+	His: string;
+}

package/src/core/judging/confirmation-judge.ts

@@ -0,0 +1,68 @@
+import type { Judge } from "../contract/providers.ts";
+import type { PendingConfirmation } from "../domain/conversation.ts";
+import type { Logger } from "../log.ts";
+
+export interface ConfirmationJudgeOptions {
+	judge: Judge;
+	threshold: number;
+	/** How the judge's instructions name the assistant that held the actions. */
+	assistant?: string;
+	logger: Logger;
+}
+
+/** Decides with the judge whether the owner's reply confirms the actions the assistant held for them. */
+export class ConfirmationJudge {
+	readonly #options: ConfirmationJudgeOptions;
+
+	constructor(options: ConfirmationJudgeOptions) {
+		this.#options = options;
+	}
+
+	/** True only for a confident approval; a failed request counts as no approval. */
+	async approves(
+		pending: PendingConfirmation,
+		reply: string,
+	): Promise<boolean> {
+		const {
+			judge,
+			threshold,
+			logger,
+			assistant = "The assistant",
+		} = this.#options;
+		try {
+			const answer = await judge.askChoice(
+				{
+					held_actions: pending.calls.map((call) => call.action),
+					owner_reply: reply,
+				},
+				"confirmation",
+				{
+					type: "choice",
+					instructions: `${assistant} asked the owner to confirm the held_actions before running them. Does owner_reply give the go-ahead for them as described?`,
+					criteria: {
+						approve: `The owner clearly agrees that ${assistant} should go ahead with the held actions as described.`,
+						decline:
+							"The owner refuses, cancels, postpones, or asks to change the actions first.",
+						other:
+							"The reply is about something else or does not clearly answer.",
+					},
+				},
+			);
+			const approved =
+				answer.choice === "approve" && answer.confidence >= threshold;
+			logger.info(
+				{
+					choice: answer.choice,
+					confidence: answer.confidence,
+					approved,
+					actions: pending.calls.map((call) => call.action),
+				},
+				"confirmation decided",
+			);
+			return approved;
+		} catch (error) {
+			logger.warn({ err: error }, "confirmation request failed; not approving");
+			return false;
+		}
+	}
+}

package/src/core/judging/effort-judge.test.ts

@@ -0,0 +1,112 @@
+import { describe, expect, test } from "bun:test";
+import type { Judge, ScoreQuestion } from "../contract/providers.ts";
+import { JudgeError } from "../errors.ts";
+import { silentLogger } from "../log.ts";
+import {
+	AGENT_BRIEF,
+	EffortJudge,
+	levelIndex,
+	moveIndex,
+} from "./effort-judge.ts";
+
+class FakeJudge implements Judge {
+	constructor(private readonly scores: number[] | Error) {}
+	async askYesNo(): Promise<Record<string, number>> {
+		throw new Error("not used here");
+	}
+	async askChoice(): Promise<never> {
+		throw new Error("not used here");
+	}
+	scored: { state: Record<string, unknown>; criteria: readonly string[] }[] =
+		[];
+	async askScore(
+		state: Record<string, unknown>,
+		_name: string,
+		question: ScoreQuestion,
+	): Promise<number[]> {
+		this.scored.push({ state, criteria: question.criteria });
+		if (this.scores instanceof Error) throw this.scores;
+		return this.scores;
+	}
+}
+
+describe("EffortJudge", () => {
+	type Level = "low" | "medium" | "high" | "xhigh";
+	const judge = (
+		scores: number[] | Error,
+		previous: { reply?: string; level?: Level } = {},
+		fallback: Level = "low",
+	) => {
+		const fakeJudge = new FakeJudge(scores);
+		const level = new EffortJudge({
+			judge: fakeJudge,
+			brief: AGENT_BRIEF,
+			fallback,
+			threshold: 0.6,
+			logger: silentLogger(),
+		}).judge("sounds good, go ahead", previous);
+		return { level, fakeJudge };
+	};
+
+	test("a first turn takes the median of the judge's distribution up to high", async () => {
+		expect(await judge([0.9, 0.1, 0, 0]).level).toBe("low");
+		expect(await judge([0.2, 0.6, 0.2, 0]).level).toBe("medium");
+		expect(await judge([0, 0.1, 0.9, 0]).level).toBe("high");
+	});
+
+	test("xhigh takes a near-certain answer, otherwise it stays high", async () => {
+		expect(await judge([0, 0, 0.1, 0.9]).level).toBe("xhigh");
+		expect(await judge([0, 0, 0.3, 0.7]).level).toBe("high");
+		expect(await judge([0, 0, 0.3, 0.7], { level: "high" }).level).toBe("high");
+	});
+
+	test("a split first answer lands where the mass is, not in the middle", () => {
+		expect(levelIndex([0.6, 0, 0.4, 0])).toBe(0);
+		expect(levelIndex([0.4, 0, 0.6, 0])).toBe(2);
+	});
+
+	test("an unsure answer keeps the previous turn's level", async () => {
+		expect(await judge([0.4, 0.15, 0.45, 0], { level: "high" }).level).toBe(
+			"high",
+		);
+		expect(await judge([0.45, 0.1, 0.45, 0], { level: "low" }).level).toBe(
+			"low",
+		);
+	});
+
+	test("a confident answer moves off the previous level either way", async () => {
+		expect(await judge([0, 0.3, 0.7, 0], { level: "low" }).level).toBe("high");
+		expect(await judge([0.55, 0.45, 0, 0], { level: "high" }).level).toBe(
+			"medium",
+		);
+		expect(await judge([0.9, 0.1, 0, 0], { level: "high" }).level).toBe("low");
+	});
+
+	test("moveIndex checks upgrades before downgrades", () => {
+		expect(moveIndex(1, [0.5, 0, 0.5, 0], 0.5)).toBe(2);
+	});
+
+	test("a judge failure keeps the previous level, or runs at the fallback", async () => {
+		expect(await judge(new JudgeError("down")).level).toBe("low");
+		expect(await judge(new JudgeError("down"), {}, "medium").level).toBe(
+			"medium",
+		);
+		expect(
+			await judge(new JudgeError("down"), { level: "high" }, "medium").level,
+		).toBe("high");
+	});
+
+	test("the judge reads the previous reply with the message and the brief's rubric", async () => {
+		const { level, fakeJudge } = judge([1, 0, 0, 0], {
+			reply: "Plan: move the database first, then switch DNS.",
+		});
+		await level;
+		expect(fakeJudge.scored[0]).toEqual({
+			state: {
+				message: "sounds good, go ahead",
+				previous_reply: "Plan: move the database first, then switch DNS.",
+			},
+			criteria: AGENT_BRIEF.criteria,
+		});
+	});
+});

package/src/core/judging/effort-judge.ts

@@ -0,0 +1,158 @@
+import type { Judge } from "../contract/providers.ts";
+import type { Logger } from "../log.ts";
+import type { ThinkingLevel } from "../models.ts";
+
+/** The levels the judge chooses between, lowest first; criterion i of a brief asks for level i. */
+export const EFFORT_LEVELS = ["low", "medium", "high", "xhigh"] as const;
+export type EffortLevel = (typeof EFFORT_LEVELS)[number];
+
+/** What the judge is told about the conversation, and one criterion per effort level. */
+export interface EffortBrief {
+	instructions: string;
+	criteria: readonly [string, string, string, string];
+}
+
+export const JUDGE_WORK =
+	"How much step-by-step reasoning does handling the message well require? Judge the work " +
+	"the message asks for, not its length or tone.";
+
+/** The default brief: an assistant agent that works through tools for its owner. */
+export const AGENT_BRIEF: EffortBrief = {
+	instructions:
+		"The owner sent this message to a personal assistant agent that works through tools: " +
+		"notes, schedules, research, coding, and infrastructure. previous_reply, when present, is " +
+		`the assistant's last answer in this conversation. ${JUDGE_WORK} A short reply that ` +
+		"continues earlier work asks for as much as the work it continues.",
+	criteria: [
+		"Chat or a quick lookup: a greeting, thanks, a short factual question, reading one note " +
+			"or calendar entry, setting a simple reminder, confirming or declining held actions.",
+		"Routine work: a few tool calls in a row, a small edit or a message draft, summarizing a " +
+			"page or a few notes, a research question with a clear answer.",
+		"Substantial work: debugging, planning or changing code or infrastructure, research that " +
+			"compares sources, a decision with real tradeoffs, work spanning several tools or systems.",
+		"An exceptional problem that sustained reasoning would still get wrong: architecture or " +
+			"migration across systems, a subtle correctness or security problem, synthesizing a " +
+			"large body of material into a design, or the owner explicitly asks for the deepest " +
+			"thinking available.",
+	],
+};
+
+/**
+ * How much of the distribution must reach each level before a first turn runs at it. The
+ * top level is slow and costly, so it takes a near-certain answer; below it, half is enough.
+ */
+const REQUIRED_MASS = [0, 0.5, 0.5, 0.8] as const;
+
+/** The judge reads this much of the previous reply; its gist is at the start and the end. */
+const PREVIOUS_REPLY_CHARS = 1_500;
+
+export interface EffortJudgeOptions<Fallback extends ThinkingLevel> {
+	judge: Judge;
+	brief: EffortBrief;
+	/** The level of a first turn the judge could not judge. */
+	fallback: Fallback;
+	/** The share of the judge's answer a turn needs to move off the previous turn's level. */
+	threshold: number;
+	logger: Logger;
+}
+
+/** What the conversation's previous turn left: its answer and the level it ran at. */
+export interface PreviousTurn<Level extends ThinkingLevel> {
+	reply?: string;
+	level?: Level;
+}
+
+/** Picks the thinking level of one turn from the judge's reading of its message. */
+export class EffortJudge<Fallback extends ThinkingLevel = ThinkingLevel> {
+	readonly #options: EffortJudgeOptions<Fallback>;
+
+	constructor(options: EffortJudgeOptions<Fallback>) {
+		this.#options = options;
+	}
+
+	/** Keeps the previous turn's level when the judge is unsure or fails. */
+	async judge(
+		message: string,
+		previous: PreviousTurn<EffortLevel | Fallback> = {},
+	): Promise<EffortLevel | Fallback> {
+		const { judge, brief, fallback, threshold, logger } = this.#options;
+		try {
+			const probabilities = await judge.askScore(
+				{
+					message,
+					...(previous.reply
+						? { previous_reply: excerpt(previous.reply) }
+						: {}),
+				},
+				"effort",
+				{
+					type: "score",
+					instructions: brief.instructions,
+					criteria: brief.criteria,
+				},
+			);
+			const from = EFFORT_LEVELS.indexOf(previous.level as EffortLevel);
+			const index =
+				from === -1
+					? levelIndex(probabilities)
+					: moveIndex(from, probabilities, threshold);
+			const level =
+				index === from && previous.level
+					? previous.level
+					: (EFFORT_LEVELS[index] ?? "low");
+			logger.info(
+				{ probabilities, previous: previous.level, level },
+				"effort decided",
+			);
+			return level;
+		} catch (error) {
+			const level = previous.level ?? fallback;
+			logger.warn({ err: error, level }, "effort request failed");
+			return level;
+		}
+	}
+}
+
+/**
+ * A first turn's level: the highest level whose share of the distribution at or above it
+ * meets REQUIRED_MASS. Below the top level that is the median of an ordinal answer, which
+ * unlike the mean never lands on a middle level nothing voted for.
+ */
+export function levelIndex(probabilities: readonly number[]): number {
+	let reached = 0;
+	for (let index = probabilities.length - 1; index > 0; index--) {
+		reached += probabilities[index] ?? 0;
+		if (reached >= (REQUIRED_MASS[index] ?? 0.5)) return index;
+	}
+	return 0;
+}
+
+/**
+ * A later turn's level, starting from the previous one: up to the highest level that
+ * `threshold` of the answer reaches, and never less than a first turn needs, else down to
+ * the lowest level that `threshold` of it stays at or under, else where it was. Upgrades
+ * are checked first, because thinking too much costs less than answering wrong.
+ */
+export function moveIndex(
+	from: number,
+	probabilities: readonly number[],
+	threshold: number,
+): number {
+	const sum = (values: readonly number[]) =>
+		values.reduce((total, value) => total + value, 0);
+	for (let index = probabilities.length - 1; index > from; index--)
+		if (
+			sum(probabilities.slice(index)) >=
+			Math.max(threshold, REQUIRED_MASS[index] ?? 0)
+		)
+			return index;
+	for (let index = 0; index < from; index++)
+		if (sum(probabilities.slice(0, index + 1)) >= threshold) return index;
+	return from;
+}
+
+function excerpt(text: string): string {
+	if (text.length <= PREVIOUS_REPLY_CHARS) return text;
+	const half = PREVIOUS_REPLY_CHARS / 2;
+	return `${text.slice(0, half)}\n…\n${text.slice(-half)}`;
+}

package/src/core/judging/model-judge.test.ts

@@ -0,0 +1,126 @@
+import { describe, expect, test } from "bun:test";
+import { JudgeError } from "../errors.ts";
+import { type JudgeModel, ModelJudge } from "./model-judge.ts";
+
+/** A model that answers each prompt with the next scripted text and records the prompts. */
+function scripted(...answers: string[]): {
+	model: JudgeModel;
+	prompts: string[];
+} {
+	const prompts: string[] = [];
+	return {
+		prompts,
+		model: async (_system, prompt) => {
+			prompts.push(prompt);
+			const answer = answers.shift();
+			if (answer === undefined) throw new Error("no scripted answer left");
+			return answer;
+		},
+	};
+}
+
+/** What the promise rejected with; undefined when it resolved. */
+async function rejection(promise: Promise<unknown>): Promise<unknown> {
+	try {
+		await promise;
+	} catch (error) {
+		return error;
+	}
+	return undefined;
+}
+
+const approval = {
+	approves: {
+		type: "yesno" as const,
+		instructions: "Does the reply approve the held actions?",
+		criteria: { true: "the owner agrees", false: "the owner declines" },
+	},
+};
+
+describe("ModelJudge", () => {
+	test("a yes-or-no answer gives each question's probability, and the prompt carries the state", async () => {
+		const { model, prompts } = scripted('{"approves": 0.92}');
+		const judge = new ModelJudge(model);
+		expect(
+			await judge.askYesNo({ reply: "sounds good, send it" }, approval),
+		).toEqual({
+			approves: 0.92,
+		});
+		expect(prompts[0]).toContain('"reply": "sounds good, send it"');
+		expect(prompts[0]).toContain("Yes means: the owner agrees.");
+		// A bare "how sure are you" made a model rate its own certainty, so a clear no scored 0.99.
+		expect(prompts[0]).toContain("probability that the answer is yes");
+	});
+
+	test("a code fence or words around the object are tolerated", async () => {
+		const { model } = scripted('Sure:\n```json\n{"approves": 0.1}\n```');
+		expect(await new ModelJudge(model).askYesNo({}, approval)).toEqual({
+			approves: 0.1,
+		});
+	});
+
+	test("a missing or out-of-range probability throws", async () => {
+		const { model } = scripted('{"other": 1}', '{"approves": 1.5}', "no JSON");
+		const judge = new ModelJudge(model);
+		for (let i = 0; i < 3; i++)
+			expect(await rejection(judge.askYesNo({}, approval))).toBeInstanceOf(
+				JudgeError,
+			);
+	});
+
+	test("a choice must name a listed option with a confidence", async () => {
+		const question = {
+			type: "choice" as const,
+			instructions: "Which expression fits the reply?",
+			criteria: { happy: "cheerful", neutral: "plain" },
+		};
+		const { model } = scripted(
+			'{"choice": "happy", "confidence": 0.8}',
+			'{"choice": "angry", "confidence": 0.8}',
+			'{"choice": "neutral"}',
+		);
+		const judge = new ModelJudge(model);
+		expect(await judge.askChoice({}, "expression", question)).toEqual({
+			choice: "happy",
+			confidence: 0.8,
+		});
+		expect(
+			await rejection(judge.askChoice({}, "expression", question)),
+		).toBeInstanceOf(JudgeError);
+		expect(
+			await rejection(judge.askChoice({}, "expression", question)),
+		).toBeInstanceOf(JudgeError);
+	});
+
+	test("a score gives one probability per level, normalized to sum to one", async () => {
+		const question = {
+			type: "score" as const,
+			instructions: "How much thinking does the turn need?",
+			criteria: ["low", "medium", "high"],
+		};
+		const { model } = scripted(
+			'{"probabilities": [0.2, 0.2, 0.4]}',
+			'{"probabilities": [0.5, 0.5]}',
+			'{"probabilities": [0, 0, 0]}',
+		);
+		const judge = new ModelJudge(model);
+		expect(await judge.askScore({}, "effort", question)).toEqual([
+			0.25, 0.25, 0.5,
+		]);
+		expect(
+			await rejection(judge.askScore({}, "effort", question)),
+		).toBeInstanceOf(JudgeError);
+		expect(
+			await rejection(judge.askScore({}, "effort", question)),
+		).toBeInstanceOf(JudgeError);
+	});
+
+	test("a failing model's error reaches the caller", async () => {
+		const judge = new ModelJudge(async () => {
+			throw new Error("offline");
+		});
+		expect(String(await rejection(judge.askYesNo({}, approval)))).toContain(
+			"offline",
+		);
+	});
+});